aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/md/bitmap.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/md/bitmap.c')
-rw-r--r--drivers/md/bitmap.c1100
1 files changed, 632 insertions, 468 deletions
diff --git a/drivers/md/bitmap.c b/drivers/md/bitmap.c
index 17e2b472e16d..15dbe03117e4 100644
--- a/drivers/md/bitmap.c
+++ b/drivers/md/bitmap.c
@@ -45,7 +45,7 @@ static inline char *bmname(struct bitmap *bitmap)
45 * if we find our page, we increment the page's refcount so that it stays 45 * if we find our page, we increment the page's refcount so that it stays
46 * allocated while we're using it 46 * allocated while we're using it
47 */ 47 */
48static int bitmap_checkpage(struct bitmap *bitmap, 48static int bitmap_checkpage(struct bitmap_counts *bitmap,
49 unsigned long page, int create) 49 unsigned long page, int create)
50__releases(bitmap->lock) 50__releases(bitmap->lock)
51__acquires(bitmap->lock) 51__acquires(bitmap->lock)
@@ -76,8 +76,7 @@ __acquires(bitmap->lock)
76 spin_lock_irq(&bitmap->lock); 76 spin_lock_irq(&bitmap->lock);
77 77
78 if (mappage == NULL) { 78 if (mappage == NULL) {
79 pr_debug("%s: bitmap map page allocation failed, hijacking\n", 79 pr_debug("md/bitmap: map page allocation failed, hijacking\n");
80 bmname(bitmap));
81 /* failed - set the hijacked flag so that we can use the 80 /* failed - set the hijacked flag so that we can use the
82 * pointer as a counter */ 81 * pointer as a counter */
83 if (!bitmap->bp[page].map) 82 if (!bitmap->bp[page].map)
@@ -100,7 +99,7 @@ __acquires(bitmap->lock)
100/* if page is completely empty, put it back on the free list, or dealloc it */ 99/* if page is completely empty, put it back on the free list, or dealloc it */
101/* if page was hijacked, unmark the flag so it might get alloced next time */ 100/* if page was hijacked, unmark the flag so it might get alloced next time */
102/* Note: lock should be held when calling this */ 101/* Note: lock should be held when calling this */
103static void bitmap_checkfree(struct bitmap *bitmap, unsigned long page) 102static void bitmap_checkfree(struct bitmap_counts *bitmap, unsigned long page)
104{ 103{
105 char *ptr; 104 char *ptr;
106 105
@@ -130,22 +129,14 @@ static void bitmap_checkfree(struct bitmap *bitmap, unsigned long page)
130 */ 129 */
131 130
132/* IO operations when bitmap is stored near all superblocks */ 131/* IO operations when bitmap is stored near all superblocks */
133static struct page *read_sb_page(struct mddev *mddev, loff_t offset, 132static int read_sb_page(struct mddev *mddev, loff_t offset,
134 struct page *page, 133 struct page *page,
135 unsigned long index, int size) 134 unsigned long index, int size)
136{ 135{
137 /* choose a good rdev and read the page from there */ 136 /* choose a good rdev and read the page from there */
138 137
139 struct md_rdev *rdev; 138 struct md_rdev *rdev;
140 sector_t target; 139 sector_t target;
141 int did_alloc = 0;
142
143 if (!page) {
144 page = alloc_page(GFP_KERNEL);
145 if (!page)
146 return ERR_PTR(-ENOMEM);
147 did_alloc = 1;
148 }
149 140
150 rdev_for_each(rdev, mddev) { 141 rdev_for_each(rdev, mddev) {
151 if (! test_bit(In_sync, &rdev->flags) 142 if (! test_bit(In_sync, &rdev->flags)
@@ -158,15 +149,10 @@ static struct page *read_sb_page(struct mddev *mddev, loff_t offset,
158 roundup(size, bdev_logical_block_size(rdev->bdev)), 149 roundup(size, bdev_logical_block_size(rdev->bdev)),
159 page, READ, true)) { 150 page, READ, true)) {
160 page->index = index; 151 page->index = index;
161 attach_page_buffers(page, NULL); /* so that free_buffer will 152 return 0;
162 * quietly no-op */
163 return page;
164 } 153 }
165 } 154 }
166 if (did_alloc) 155 return -EIO;
167 put_page(page);
168 return ERR_PTR(-EIO);
169
170} 156}
171 157
172static struct md_rdev *next_active_rdev(struct md_rdev *rdev, struct mddev *mddev) 158static struct md_rdev *next_active_rdev(struct md_rdev *rdev, struct mddev *mddev)
@@ -208,6 +194,7 @@ static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait)
208 struct md_rdev *rdev = NULL; 194 struct md_rdev *rdev = NULL;
209 struct block_device *bdev; 195 struct block_device *bdev;
210 struct mddev *mddev = bitmap->mddev; 196 struct mddev *mddev = bitmap->mddev;
197 struct bitmap_storage *store = &bitmap->storage;
211 198
212 while ((rdev = next_active_rdev(rdev, mddev)) != NULL) { 199 while ((rdev = next_active_rdev(rdev, mddev)) != NULL) {
213 int size = PAGE_SIZE; 200 int size = PAGE_SIZE;
@@ -215,9 +202,13 @@ static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait)
215 202
216 bdev = (rdev->meta_bdev) ? rdev->meta_bdev : rdev->bdev; 203 bdev = (rdev->meta_bdev) ? rdev->meta_bdev : rdev->bdev;
217 204
218 if (page->index == bitmap->file_pages-1) 205 if (page->index == store->file_pages-1) {
219 size = roundup(bitmap->last_page_size, 206 int last_page_size = store->bytes & (PAGE_SIZE-1);
207 if (last_page_size == 0)
208 last_page_size = PAGE_SIZE;
209 size = roundup(last_page_size,
220 bdev_logical_block_size(bdev)); 210 bdev_logical_block_size(bdev));
211 }
221 /* Just make sure we aren't corrupting data or 212 /* Just make sure we aren't corrupting data or
222 * metadata 213 * metadata
223 */ 214 */
@@ -276,10 +267,10 @@ static void write_page(struct bitmap *bitmap, struct page *page, int wait)
276{ 267{
277 struct buffer_head *bh; 268 struct buffer_head *bh;
278 269
279 if (bitmap->file == NULL) { 270 if (bitmap->storage.file == NULL) {
280 switch (write_sb_page(bitmap, page, wait)) { 271 switch (write_sb_page(bitmap, page, wait)) {
281 case -EINVAL: 272 case -EINVAL:
282 bitmap->flags |= BITMAP_WRITE_ERROR; 273 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
283 } 274 }
284 } else { 275 } else {
285 276
@@ -297,20 +288,16 @@ static void write_page(struct bitmap *bitmap, struct page *page, int wait)
297 wait_event(bitmap->write_wait, 288 wait_event(bitmap->write_wait,
298 atomic_read(&bitmap->pending_writes)==0); 289 atomic_read(&bitmap->pending_writes)==0);
299 } 290 }
300 if (bitmap->flags & BITMAP_WRITE_ERROR) 291 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
301 bitmap_file_kick(bitmap); 292 bitmap_file_kick(bitmap);
302} 293}
303 294
304static void end_bitmap_write(struct buffer_head *bh, int uptodate) 295static void end_bitmap_write(struct buffer_head *bh, int uptodate)
305{ 296{
306 struct bitmap *bitmap = bh->b_private; 297 struct bitmap *bitmap = bh->b_private;
307 unsigned long flags;
308 298
309 if (!uptodate) { 299 if (!uptodate)
310 spin_lock_irqsave(&bitmap->lock, flags); 300 set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
311 bitmap->flags |= BITMAP_WRITE_ERROR;
312 spin_unlock_irqrestore(&bitmap->lock, flags);
313 }
314 if (atomic_dec_and_test(&bitmap->pending_writes)) 301 if (atomic_dec_and_test(&bitmap->pending_writes))
315 wake_up(&bitmap->write_wait); 302 wake_up(&bitmap->write_wait);
316} 303}
@@ -325,8 +312,12 @@ __clear_page_buffers(struct page *page)
325} 312}
326static void free_buffers(struct page *page) 313static void free_buffers(struct page *page)
327{ 314{
328 struct buffer_head *bh = page_buffers(page); 315 struct buffer_head *bh;
329 316
317 if (!PagePrivate(page))
318 return;
319
320 bh = page_buffers(page);
330 while (bh) { 321 while (bh) {
331 struct buffer_head *next = bh->b_this_page; 322 struct buffer_head *next = bh->b_this_page;
332 free_buffer_head(bh); 323 free_buffer_head(bh);
@@ -343,11 +334,12 @@ static void free_buffers(struct page *page)
343 * This usage is similar to how swap files are handled, and allows us 334 * This usage is similar to how swap files are handled, and allows us
344 * to write to a file with no concerns of memory allocation failing. 335 * to write to a file with no concerns of memory allocation failing.
345 */ 336 */
346static struct page *read_page(struct file *file, unsigned long index, 337static int read_page(struct file *file, unsigned long index,
347 struct bitmap *bitmap, 338 struct bitmap *bitmap,
348 unsigned long count) 339 unsigned long count,
340 struct page *page)
349{ 341{
350 struct page *page = NULL; 342 int ret = 0;
351 struct inode *inode = file->f_path.dentry->d_inode; 343 struct inode *inode = file->f_path.dentry->d_inode;
352 struct buffer_head *bh; 344 struct buffer_head *bh;
353 sector_t block; 345 sector_t block;
@@ -355,16 +347,9 @@ static struct page *read_page(struct file *file, unsigned long index,
355 pr_debug("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE, 347 pr_debug("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE,
356 (unsigned long long)index << PAGE_SHIFT); 348 (unsigned long long)index << PAGE_SHIFT);
357 349
358 page = alloc_page(GFP_KERNEL);
359 if (!page)
360 page = ERR_PTR(-ENOMEM);
361 if (IS_ERR(page))
362 goto out;
363
364 bh = alloc_page_buffers(page, 1<<inode->i_blkbits, 0); 350 bh = alloc_page_buffers(page, 1<<inode->i_blkbits, 0);
365 if (!bh) { 351 if (!bh) {
366 put_page(page); 352 ret = -ENOMEM;
367 page = ERR_PTR(-ENOMEM);
368 goto out; 353 goto out;
369 } 354 }
370 attach_page_buffers(page, bh); 355 attach_page_buffers(page, bh);
@@ -376,8 +361,7 @@ static struct page *read_page(struct file *file, unsigned long index,
376 bh->b_blocknr = bmap(inode, block); 361 bh->b_blocknr = bmap(inode, block);
377 if (bh->b_blocknr == 0) { 362 if (bh->b_blocknr == 0) {
378 /* Cannot use this file! */ 363 /* Cannot use this file! */
379 free_buffers(page); 364 ret = -EINVAL;
380 page = ERR_PTR(-EINVAL);
381 goto out; 365 goto out;
382 } 366 }
383 bh->b_bdev = inode->i_sb->s_bdev; 367 bh->b_bdev = inode->i_sb->s_bdev;
@@ -400,17 +384,15 @@ static struct page *read_page(struct file *file, unsigned long index,
400 384
401 wait_event(bitmap->write_wait, 385 wait_event(bitmap->write_wait,
402 atomic_read(&bitmap->pending_writes)==0); 386 atomic_read(&bitmap->pending_writes)==0);
403 if (bitmap->flags & BITMAP_WRITE_ERROR) { 387 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
404 free_buffers(page); 388 ret = -EIO;
405 page = ERR_PTR(-EIO);
406 }
407out: 389out:
408 if (IS_ERR(page)) 390 if (ret)
409 printk(KERN_ALERT "md: bitmap read error: (%dB @ %llu): %ld\n", 391 printk(KERN_ALERT "md: bitmap read error: (%dB @ %llu): %d\n",
410 (int)PAGE_SIZE, 392 (int)PAGE_SIZE,
411 (unsigned long long)index << PAGE_SHIFT, 393 (unsigned long long)index << PAGE_SHIFT,
412 PTR_ERR(page)); 394 ret);
413 return page; 395 return ret;
414} 396}
415 397
416/* 398/*
@@ -426,9 +408,9 @@ void bitmap_update_sb(struct bitmap *bitmap)
426 return; 408 return;
427 if (bitmap->mddev->bitmap_info.external) 409 if (bitmap->mddev->bitmap_info.external)
428 return; 410 return;
429 if (!bitmap->sb_page) /* no superblock */ 411 if (!bitmap->storage.sb_page) /* no superblock */
430 return; 412 return;
431 sb = kmap_atomic(bitmap->sb_page); 413 sb = kmap_atomic(bitmap->storage.sb_page);
432 sb->events = cpu_to_le64(bitmap->mddev->events); 414 sb->events = cpu_to_le64(bitmap->mddev->events);
433 if (bitmap->mddev->events < bitmap->events_cleared) 415 if (bitmap->mddev->events < bitmap->events_cleared)
434 /* rocking back to read-only */ 416 /* rocking back to read-only */
@@ -438,8 +420,13 @@ void bitmap_update_sb(struct bitmap *bitmap)
438 /* Just in case these have been changed via sysfs: */ 420 /* Just in case these have been changed via sysfs: */
439 sb->daemon_sleep = cpu_to_le32(bitmap->mddev->bitmap_info.daemon_sleep/HZ); 421 sb->daemon_sleep = cpu_to_le32(bitmap->mddev->bitmap_info.daemon_sleep/HZ);
440 sb->write_behind = cpu_to_le32(bitmap->mddev->bitmap_info.max_write_behind); 422 sb->write_behind = cpu_to_le32(bitmap->mddev->bitmap_info.max_write_behind);
423 /* This might have been changed by a reshape */
424 sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
425 sb->chunksize = cpu_to_le32(bitmap->mddev->bitmap_info.chunksize);
426 sb->sectors_reserved = cpu_to_le32(bitmap->mddev->
427 bitmap_info.space);
441 kunmap_atomic(sb); 428 kunmap_atomic(sb);
442 write_page(bitmap, bitmap->sb_page, 1); 429 write_page(bitmap, bitmap->storage.sb_page, 1);
443} 430}
444 431
445/* print out the bitmap file superblock */ 432/* print out the bitmap file superblock */
@@ -447,9 +434,9 @@ void bitmap_print_sb(struct bitmap *bitmap)
447{ 434{
448 bitmap_super_t *sb; 435 bitmap_super_t *sb;
449 436
450 if (!bitmap || !bitmap->sb_page) 437 if (!bitmap || !bitmap->storage.sb_page)
451 return; 438 return;
452 sb = kmap_atomic(bitmap->sb_page); 439 sb = kmap_atomic(bitmap->storage.sb_page);
453 printk(KERN_DEBUG "%s: bitmap file superblock:\n", bmname(bitmap)); 440 printk(KERN_DEBUG "%s: bitmap file superblock:\n", bmname(bitmap));
454 printk(KERN_DEBUG " magic: %08x\n", le32_to_cpu(sb->magic)); 441 printk(KERN_DEBUG " magic: %08x\n", le32_to_cpu(sb->magic));
455 printk(KERN_DEBUG " version: %d\n", le32_to_cpu(sb->version)); 442 printk(KERN_DEBUG " version: %d\n", le32_to_cpu(sb->version));
@@ -488,15 +475,15 @@ static int bitmap_new_disk_sb(struct bitmap *bitmap)
488 unsigned long chunksize, daemon_sleep, write_behind; 475 unsigned long chunksize, daemon_sleep, write_behind;
489 int err = -EINVAL; 476 int err = -EINVAL;
490 477
491 bitmap->sb_page = alloc_page(GFP_KERNEL); 478 bitmap->storage.sb_page = alloc_page(GFP_KERNEL);
492 if (IS_ERR(bitmap->sb_page)) { 479 if (IS_ERR(bitmap->storage.sb_page)) {
493 err = PTR_ERR(bitmap->sb_page); 480 err = PTR_ERR(bitmap->storage.sb_page);
494 bitmap->sb_page = NULL; 481 bitmap->storage.sb_page = NULL;
495 return err; 482 return err;
496 } 483 }
497 bitmap->sb_page->index = 0; 484 bitmap->storage.sb_page->index = 0;
498 485
499 sb = kmap_atomic(bitmap->sb_page); 486 sb = kmap_atomic(bitmap->storage.sb_page);
500 487
501 sb->magic = cpu_to_le32(BITMAP_MAGIC); 488 sb->magic = cpu_to_le32(BITMAP_MAGIC);
502 sb->version = cpu_to_le32(BITMAP_MAJOR_HI); 489 sb->version = cpu_to_le32(BITMAP_MAJOR_HI);
@@ -534,8 +521,8 @@ static int bitmap_new_disk_sb(struct bitmap *bitmap)
534 521
535 memcpy(sb->uuid, bitmap->mddev->uuid, 16); 522 memcpy(sb->uuid, bitmap->mddev->uuid, 16);
536 523
537 bitmap->flags |= BITMAP_STALE; 524 set_bit(BITMAP_STALE, &bitmap->flags);
538 sb->state |= cpu_to_le32(BITMAP_STALE); 525 sb->state = cpu_to_le32(bitmap->flags);
539 bitmap->events_cleared = bitmap->mddev->events; 526 bitmap->events_cleared = bitmap->mddev->events;
540 sb->events_cleared = cpu_to_le64(bitmap->mddev->events); 527 sb->events_cleared = cpu_to_le64(bitmap->mddev->events);
541 528
@@ -551,31 +538,45 @@ static int bitmap_read_sb(struct bitmap *bitmap)
551 bitmap_super_t *sb; 538 bitmap_super_t *sb;
552 unsigned long chunksize, daemon_sleep, write_behind; 539 unsigned long chunksize, daemon_sleep, write_behind;
553 unsigned long long events; 540 unsigned long long events;
541 unsigned long sectors_reserved = 0;
554 int err = -EINVAL; 542 int err = -EINVAL;
543 struct page *sb_page;
555 544
545 if (!bitmap->storage.file && !bitmap->mddev->bitmap_info.offset) {
546 chunksize = 128 * 1024 * 1024;
547 daemon_sleep = 5 * HZ;
548 write_behind = 0;
549 set_bit(BITMAP_STALE, &bitmap->flags);
550 err = 0;
551 goto out_no_sb;
552 }
556 /* page 0 is the superblock, read it... */ 553 /* page 0 is the superblock, read it... */
557 if (bitmap->file) { 554 sb_page = alloc_page(GFP_KERNEL);
558 loff_t isize = i_size_read(bitmap->file->f_mapping->host); 555 if (!sb_page)
556 return -ENOMEM;
557 bitmap->storage.sb_page = sb_page;
558
559 if (bitmap->storage.file) {
560 loff_t isize = i_size_read(bitmap->storage.file->f_mapping->host);
559 int bytes = isize > PAGE_SIZE ? PAGE_SIZE : isize; 561 int bytes = isize > PAGE_SIZE ? PAGE_SIZE : isize;
560 562
561 bitmap->sb_page = read_page(bitmap->file, 0, bitmap, bytes); 563 err = read_page(bitmap->storage.file, 0,
564 bitmap, bytes, sb_page);
562 } else { 565 } else {
563 bitmap->sb_page = read_sb_page(bitmap->mddev, 566 err = read_sb_page(bitmap->mddev,
564 bitmap->mddev->bitmap_info.offset, 567 bitmap->mddev->bitmap_info.offset,
565 NULL, 568 sb_page,
566 0, sizeof(bitmap_super_t)); 569 0, sizeof(bitmap_super_t));
567 } 570 }
568 if (IS_ERR(bitmap->sb_page)) { 571 if (err)
569 err = PTR_ERR(bitmap->sb_page);
570 bitmap->sb_page = NULL;
571 return err; 572 return err;
572 }
573 573
574 sb = kmap_atomic(bitmap->sb_page); 574 sb = kmap_atomic(sb_page);
575 575
576 chunksize = le32_to_cpu(sb->chunksize); 576 chunksize = le32_to_cpu(sb->chunksize);
577 daemon_sleep = le32_to_cpu(sb->daemon_sleep) * HZ; 577 daemon_sleep = le32_to_cpu(sb->daemon_sleep) * HZ;
578 write_behind = le32_to_cpu(sb->write_behind); 578 write_behind = le32_to_cpu(sb->write_behind);
579 sectors_reserved = le32_to_cpu(sb->sectors_reserved);
579 580
580 /* verify that the bitmap-specific fields are valid */ 581 /* verify that the bitmap-specific fields are valid */
581 if (sb->magic != cpu_to_le32(BITMAP_MAGIC)) 582 if (sb->magic != cpu_to_le32(BITMAP_MAGIC))
@@ -618,60 +619,32 @@ static int bitmap_read_sb(struct bitmap *bitmap)
618 "-- forcing full recovery\n", 619 "-- forcing full recovery\n",
619 bmname(bitmap), events, 620 bmname(bitmap), events,
620 (unsigned long long) bitmap->mddev->events); 621 (unsigned long long) bitmap->mddev->events);
621 sb->state |= cpu_to_le32(BITMAP_STALE); 622 set_bit(BITMAP_STALE, &bitmap->flags);
622 } 623 }
623 } 624 }
624 625
625 /* assign fields using values from superblock */ 626 /* assign fields using values from superblock */
626 bitmap->mddev->bitmap_info.chunksize = chunksize;
627 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
628 bitmap->mddev->bitmap_info.max_write_behind = write_behind;
629 bitmap->flags |= le32_to_cpu(sb->state); 627 bitmap->flags |= le32_to_cpu(sb->state);
630 if (le32_to_cpu(sb->version) == BITMAP_MAJOR_HOSTENDIAN) 628 if (le32_to_cpu(sb->version) == BITMAP_MAJOR_HOSTENDIAN)
631 bitmap->flags |= BITMAP_HOSTENDIAN; 629 set_bit(BITMAP_HOSTENDIAN, &bitmap->flags);
632 bitmap->events_cleared = le64_to_cpu(sb->events_cleared); 630 bitmap->events_cleared = le64_to_cpu(sb->events_cleared);
633 if (bitmap->flags & BITMAP_STALE)
634 bitmap->events_cleared = bitmap->mddev->events;
635 err = 0; 631 err = 0;
636out: 632out:
637 kunmap_atomic(sb); 633 kunmap_atomic(sb);
634out_no_sb:
635 if (test_bit(BITMAP_STALE, &bitmap->flags))
636 bitmap->events_cleared = bitmap->mddev->events;
637 bitmap->mddev->bitmap_info.chunksize = chunksize;
638 bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
639 bitmap->mddev->bitmap_info.max_write_behind = write_behind;
640 if (bitmap->mddev->bitmap_info.space == 0 ||
641 bitmap->mddev->bitmap_info.space > sectors_reserved)
642 bitmap->mddev->bitmap_info.space = sectors_reserved;
638 if (err) 643 if (err)
639 bitmap_print_sb(bitmap); 644 bitmap_print_sb(bitmap);
640 return err; 645 return err;
641} 646}
642 647
643enum bitmap_mask_op {
644 MASK_SET,
645 MASK_UNSET
646};
647
648/* record the state of the bitmap in the superblock. Return the old value */
649static int bitmap_mask_state(struct bitmap *bitmap, enum bitmap_state bits,
650 enum bitmap_mask_op op)
651{
652 bitmap_super_t *sb;
653 int old;
654
655 if (!bitmap->sb_page) /* can't set the state */
656 return 0;
657 sb = kmap_atomic(bitmap->sb_page);
658 old = le32_to_cpu(sb->state) & bits;
659 switch (op) {
660 case MASK_SET:
661 sb->state |= cpu_to_le32(bits);
662 bitmap->flags |= bits;
663 break;
664 case MASK_UNSET:
665 sb->state &= cpu_to_le32(~bits);
666 bitmap->flags &= ~bits;
667 break;
668 default:
669 BUG();
670 }
671 kunmap_atomic(sb);
672 return old;
673}
674
675/* 648/*
676 * general bitmap file operations 649 * general bitmap file operations
677 */ 650 */
@@ -683,17 +656,19 @@ static int bitmap_mask_state(struct bitmap *bitmap, enum bitmap_state bits,
683 * file a page at a time. There's a superblock at the start of the file. 656 * file a page at a time. There's a superblock at the start of the file.
684 */ 657 */
685/* calculate the index of the page that contains this bit */ 658/* calculate the index of the page that contains this bit */
686static inline unsigned long file_page_index(struct bitmap *bitmap, unsigned long chunk) 659static inline unsigned long file_page_index(struct bitmap_storage *store,
660 unsigned long chunk)
687{ 661{
688 if (!bitmap->mddev->bitmap_info.external) 662 if (store->sb_page)
689 chunk += sizeof(bitmap_super_t) << 3; 663 chunk += sizeof(bitmap_super_t) << 3;
690 return chunk >> PAGE_BIT_SHIFT; 664 return chunk >> PAGE_BIT_SHIFT;
691} 665}
692 666
693/* calculate the (bit) offset of this bit within a page */ 667/* calculate the (bit) offset of this bit within a page */
694static inline unsigned long file_page_offset(struct bitmap *bitmap, unsigned long chunk) 668static inline unsigned long file_page_offset(struct bitmap_storage *store,
669 unsigned long chunk)
695{ 670{
696 if (!bitmap->mddev->bitmap_info.external) 671 if (store->sb_page)
697 chunk += sizeof(bitmap_super_t) << 3; 672 chunk += sizeof(bitmap_super_t) << 3;
698 return chunk & (PAGE_BITS - 1); 673 return chunk & (PAGE_BITS - 1);
699} 674}
@@ -705,57 +680,86 @@ static inline unsigned long file_page_offset(struct bitmap *bitmap, unsigned lon
705 * 1 page (e.g., x86) or less than 1 page -- so the bitmap might start on page 680 * 1 page (e.g., x86) or less than 1 page -- so the bitmap might start on page
706 * 0 or page 1 681 * 0 or page 1
707 */ 682 */
708static inline struct page *filemap_get_page(struct bitmap *bitmap, 683static inline struct page *filemap_get_page(struct bitmap_storage *store,
709 unsigned long chunk) 684 unsigned long chunk)
710{ 685{
711 if (file_page_index(bitmap, chunk) >= bitmap->file_pages) 686 if (file_page_index(store, chunk) >= store->file_pages)
712 return NULL; 687 return NULL;
713 return bitmap->filemap[file_page_index(bitmap, chunk) 688 return store->filemap[file_page_index(store, chunk)
714 - file_page_index(bitmap, 0)]; 689 - file_page_index(store, 0)];
715} 690}
716 691
717static void bitmap_file_unmap(struct bitmap *bitmap) 692static int bitmap_storage_alloc(struct bitmap_storage *store,
693 unsigned long chunks, int with_super)
694{
695 int pnum;
696 unsigned long num_pages;
697 unsigned long bytes;
698
699 bytes = DIV_ROUND_UP(chunks, 8);
700 if (with_super)
701 bytes += sizeof(bitmap_super_t);
702
703 num_pages = DIV_ROUND_UP(bytes, PAGE_SIZE);
704
705 store->filemap = kmalloc(sizeof(struct page *)
706 * num_pages, GFP_KERNEL);
707 if (!store->filemap)
708 return -ENOMEM;
709
710 if (with_super && !store->sb_page) {
711 store->sb_page = alloc_page(GFP_KERNEL|__GFP_ZERO);
712 if (store->sb_page == NULL)
713 return -ENOMEM;
714 store->sb_page->index = 0;
715 }
716 pnum = 0;
717 if (store->sb_page) {
718 store->filemap[0] = store->sb_page;
719 pnum = 1;
720 }
721 for ( ; pnum < num_pages; pnum++) {
722 store->filemap[pnum] = alloc_page(GFP_KERNEL|__GFP_ZERO);
723 if (!store->filemap[pnum]) {
724 store->file_pages = pnum;
725 return -ENOMEM;
726 }
727 store->filemap[pnum]->index = pnum;
728 }
729 store->file_pages = pnum;
730
731 /* We need 4 bits per page, rounded up to a multiple
732 * of sizeof(unsigned long) */
733 store->filemap_attr = kzalloc(
734 roundup(DIV_ROUND_UP(num_pages*4, 8), sizeof(unsigned long)),
735 GFP_KERNEL);
736 if (!store->filemap_attr)
737 return -ENOMEM;
738
739 store->bytes = bytes;
740
741 return 0;
742}
743
744static void bitmap_file_unmap(struct bitmap_storage *store)
718{ 745{
719 struct page **map, *sb_page; 746 struct page **map, *sb_page;
720 unsigned long *attr;
721 int pages; 747 int pages;
722 unsigned long flags; 748 struct file *file;
723 749
724 spin_lock_irqsave(&bitmap->lock, flags); 750 file = store->file;
725 map = bitmap->filemap; 751 map = store->filemap;
726 bitmap->filemap = NULL; 752 pages = store->file_pages;
727 attr = bitmap->filemap_attr; 753 sb_page = store->sb_page;
728 bitmap->filemap_attr = NULL;
729 pages = bitmap->file_pages;
730 bitmap->file_pages = 0;
731 sb_page = bitmap->sb_page;
732 bitmap->sb_page = NULL;
733 spin_unlock_irqrestore(&bitmap->lock, flags);
734 754
735 while (pages--) 755 while (pages--)
736 if (map[pages] != sb_page) /* 0 is sb_page, release it below */ 756 if (map[pages] != sb_page) /* 0 is sb_page, release it below */
737 free_buffers(map[pages]); 757 free_buffers(map[pages]);
738 kfree(map); 758 kfree(map);
739 kfree(attr); 759 kfree(store->filemap_attr);
740 760
741 if (sb_page) 761 if (sb_page)
742 free_buffers(sb_page); 762 free_buffers(sb_page);
743}
744
745static void bitmap_file_put(struct bitmap *bitmap)
746{
747 struct file *file;
748 unsigned long flags;
749
750 spin_lock_irqsave(&bitmap->lock, flags);
751 file = bitmap->file;
752 bitmap->file = NULL;
753 spin_unlock_irqrestore(&bitmap->lock, flags);
754
755 if (file)
756 wait_event(bitmap->write_wait,
757 atomic_read(&bitmap->pending_writes)==0);
758 bitmap_file_unmap(bitmap);
759 763
760 if (file) { 764 if (file) {
761 struct inode *inode = file->f_path.dentry->d_inode; 765 struct inode *inode = file->f_path.dentry->d_inode;
@@ -773,14 +777,14 @@ static void bitmap_file_kick(struct bitmap *bitmap)
773{ 777{
774 char *path, *ptr = NULL; 778 char *path, *ptr = NULL;
775 779
776 if (bitmap_mask_state(bitmap, BITMAP_STALE, MASK_SET) == 0) { 780 if (!test_and_set_bit(BITMAP_STALE, &bitmap->flags)) {
777 bitmap_update_sb(bitmap); 781 bitmap_update_sb(bitmap);
778 782
779 if (bitmap->file) { 783 if (bitmap->storage.file) {
780 path = kmalloc(PAGE_SIZE, GFP_KERNEL); 784 path = kmalloc(PAGE_SIZE, GFP_KERNEL);
781 if (path) 785 if (path)
782 ptr = d_path(&bitmap->file->f_path, path, 786 ptr = d_path(&bitmap->storage.file->f_path,
783 PAGE_SIZE); 787 path, PAGE_SIZE);
784 788
785 printk(KERN_ALERT 789 printk(KERN_ALERT
786 "%s: kicking failed bitmap file %s from array!\n", 790 "%s: kicking failed bitmap file %s from array!\n",
@@ -792,10 +796,6 @@ static void bitmap_file_kick(struct bitmap *bitmap)
792 "%s: disabling internal bitmap due to errors\n", 796 "%s: disabling internal bitmap due to errors\n",
793 bmname(bitmap)); 797 bmname(bitmap));
794 } 798 }
795
796 bitmap_file_put(bitmap);
797
798 return;
799} 799}
800 800
801enum bitmap_page_attr { 801enum bitmap_page_attr {
@@ -805,24 +805,30 @@ enum bitmap_page_attr {
805 BITMAP_PAGE_NEEDWRITE = 2, /* there are cleared bits that need to be synced */ 805 BITMAP_PAGE_NEEDWRITE = 2, /* there are cleared bits that need to be synced */
806}; 806};
807 807
808static inline void set_page_attr(struct bitmap *bitmap, struct page *page, 808static inline void set_page_attr(struct bitmap *bitmap, int pnum,
809 enum bitmap_page_attr attr) 809 enum bitmap_page_attr attr)
810{ 810{
811 __set_bit((page->index<<2) + attr, bitmap->filemap_attr); 811 set_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
812} 812}
813 813
814static inline void clear_page_attr(struct bitmap *bitmap, struct page *page, 814static inline void clear_page_attr(struct bitmap *bitmap, int pnum,
815 enum bitmap_page_attr attr) 815 enum bitmap_page_attr attr)
816{ 816{
817 __clear_bit((page->index<<2) + attr, bitmap->filemap_attr); 817 clear_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
818} 818}
819 819
820static inline unsigned long test_page_attr(struct bitmap *bitmap, struct page *page, 820static inline int test_page_attr(struct bitmap *bitmap, int pnum,
821 enum bitmap_page_attr attr) 821 enum bitmap_page_attr attr)
822{ 822{
823 return test_bit((page->index<<2) + attr, bitmap->filemap_attr); 823 return test_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
824} 824}
825 825
826static inline int test_and_clear_page_attr(struct bitmap *bitmap, int pnum,
827 enum bitmap_page_attr attr)
828{
829 return test_and_clear_bit((pnum<<2) + attr,
830 bitmap->storage.filemap_attr);
831}
826/* 832/*
827 * bitmap_file_set_bit -- called before performing a write to the md device 833 * bitmap_file_set_bit -- called before performing a write to the md device
828 * to set (and eventually sync) a particular bit in the bitmap file 834 * to set (and eventually sync) a particular bit in the bitmap file
@@ -835,26 +841,46 @@ static void bitmap_file_set_bit(struct bitmap *bitmap, sector_t block)
835 unsigned long bit; 841 unsigned long bit;
836 struct page *page; 842 struct page *page;
837 void *kaddr; 843 void *kaddr;
838 unsigned long chunk = block >> bitmap->chunkshift; 844 unsigned long chunk = block >> bitmap->counts.chunkshift;
839 845
840 if (!bitmap->filemap) 846 page = filemap_get_page(&bitmap->storage, chunk);
841 return;
842
843 page = filemap_get_page(bitmap, chunk);
844 if (!page) 847 if (!page)
845 return; 848 return;
846 bit = file_page_offset(bitmap, chunk); 849 bit = file_page_offset(&bitmap->storage, chunk);
847 850
848 /* set the bit */ 851 /* set the bit */
849 kaddr = kmap_atomic(page); 852 kaddr = kmap_atomic(page);
850 if (bitmap->flags & BITMAP_HOSTENDIAN) 853 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
851 set_bit(bit, kaddr); 854 set_bit(bit, kaddr);
852 else 855 else
853 __set_bit_le(bit, kaddr); 856 test_and_set_bit_le(bit, kaddr);
854 kunmap_atomic(kaddr); 857 kunmap_atomic(kaddr);
855 pr_debug("set file bit %lu page %lu\n", bit, page->index); 858 pr_debug("set file bit %lu page %lu\n", bit, page->index);
856 /* record page number so it gets flushed to disk when unplug occurs */ 859 /* record page number so it gets flushed to disk when unplug occurs */
857 set_page_attr(bitmap, page, BITMAP_PAGE_DIRTY); 860 set_page_attr(bitmap, page->index, BITMAP_PAGE_DIRTY);
861}
862
863static void bitmap_file_clear_bit(struct bitmap *bitmap, sector_t block)
864{
865 unsigned long bit;
866 struct page *page;
867 void *paddr;
868 unsigned long chunk = block >> bitmap->counts.chunkshift;
869
870 page = filemap_get_page(&bitmap->storage, chunk);
871 if (!page)
872 return;
873 bit = file_page_offset(&bitmap->storage, chunk);
874 paddr = kmap_atomic(page);
875 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
876 clear_bit(bit, paddr);
877 else
878 test_and_clear_bit_le(bit, paddr);
879 kunmap_atomic(paddr);
880 if (!test_page_attr(bitmap, page->index, BITMAP_PAGE_NEEDWRITE)) {
881 set_page_attr(bitmap, page->index, BITMAP_PAGE_PENDING);
882 bitmap->allclean = 0;
883 }
858} 884}
859 885
860/* this gets called when the md device is ready to unplug its underlying 886/* this gets called when the md device is ready to unplug its underlying
@@ -862,42 +888,37 @@ static void bitmap_file_set_bit(struct bitmap *bitmap, sector_t block)
862 * sync the dirty pages of the bitmap file to disk */ 888 * sync the dirty pages of the bitmap file to disk */
863void bitmap_unplug(struct bitmap *bitmap) 889void bitmap_unplug(struct bitmap *bitmap)
864{ 890{
865 unsigned long i, flags; 891 unsigned long i;
866 int dirty, need_write; 892 int dirty, need_write;
867 struct page *page;
868 int wait = 0; 893 int wait = 0;
869 894
870 if (!bitmap) 895 if (!bitmap || !bitmap->storage.filemap ||
896 test_bit(BITMAP_STALE, &bitmap->flags))
871 return; 897 return;
872 898
873 /* look at each page to see if there are any set bits that need to be 899 /* look at each page to see if there are any set bits that need to be
874 * flushed out to disk */ 900 * flushed out to disk */
875 for (i = 0; i < bitmap->file_pages; i++) { 901 for (i = 0; i < bitmap->storage.file_pages; i++) {
876 spin_lock_irqsave(&bitmap->lock, flags); 902 if (!bitmap->storage.filemap)
877 if (!bitmap->filemap) {
878 spin_unlock_irqrestore(&bitmap->lock, flags);
879 return; 903 return;
904 dirty = test_and_clear_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
905 need_write = test_and_clear_page_attr(bitmap, i,
906 BITMAP_PAGE_NEEDWRITE);
907 if (dirty || need_write) {
908 clear_page_attr(bitmap, i, BITMAP_PAGE_PENDING);
909 write_page(bitmap, bitmap->storage.filemap[i], 0);
880 } 910 }
881 page = bitmap->filemap[i];
882 dirty = test_page_attr(bitmap, page, BITMAP_PAGE_DIRTY);
883 need_write = test_page_attr(bitmap, page, BITMAP_PAGE_NEEDWRITE);
884 clear_page_attr(bitmap, page, BITMAP_PAGE_DIRTY);
885 clear_page_attr(bitmap, page, BITMAP_PAGE_NEEDWRITE);
886 if (dirty) 911 if (dirty)
887 wait = 1; 912 wait = 1;
888 spin_unlock_irqrestore(&bitmap->lock, flags);
889
890 if (dirty || need_write)
891 write_page(bitmap, page, 0);
892 } 913 }
893 if (wait) { /* if any writes were performed, we need to wait on them */ 914 if (wait) { /* if any writes were performed, we need to wait on them */
894 if (bitmap->file) 915 if (bitmap->storage.file)
895 wait_event(bitmap->write_wait, 916 wait_event(bitmap->write_wait,
896 atomic_read(&bitmap->pending_writes)==0); 917 atomic_read(&bitmap->pending_writes)==0);
897 else 918 else
898 md_super_wait(bitmap->mddev); 919 md_super_wait(bitmap->mddev);
899 } 920 }
900 if (bitmap->flags & BITMAP_WRITE_ERROR) 921 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
901 bitmap_file_kick(bitmap); 922 bitmap_file_kick(bitmap);
902} 923}
903EXPORT_SYMBOL(bitmap_unplug); 924EXPORT_SYMBOL(bitmap_unplug);
@@ -917,98 +938,77 @@ static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int n
917static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start) 938static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
918{ 939{
919 unsigned long i, chunks, index, oldindex, bit; 940 unsigned long i, chunks, index, oldindex, bit;
920 struct page *page = NULL, *oldpage = NULL; 941 struct page *page = NULL;
921 unsigned long num_pages, bit_cnt = 0; 942 unsigned long bit_cnt = 0;
922 struct file *file; 943 struct file *file;
923 unsigned long bytes, offset; 944 unsigned long offset;
924 int outofdate; 945 int outofdate;
925 int ret = -ENOSPC; 946 int ret = -ENOSPC;
926 void *paddr; 947 void *paddr;
948 struct bitmap_storage *store = &bitmap->storage;
927 949
928 chunks = bitmap->chunks; 950 chunks = bitmap->counts.chunks;
929 file = bitmap->file; 951 file = store->file;
930 952
931 BUG_ON(!file && !bitmap->mddev->bitmap_info.offset); 953 if (!file && !bitmap->mddev->bitmap_info.offset) {
954 /* No permanent bitmap - fill with '1s'. */
955 store->filemap = NULL;
956 store->file_pages = 0;
957 for (i = 0; i < chunks ; i++) {
958 /* if the disk bit is set, set the memory bit */
959 int needed = ((sector_t)(i+1) << (bitmap->counts.chunkshift)
960 >= start);
961 bitmap_set_memory_bits(bitmap,
962 (sector_t)i << bitmap->counts.chunkshift,
963 needed);
964 }
965 return 0;
966 }
932 967
933 outofdate = bitmap->flags & BITMAP_STALE; 968 outofdate = test_bit(BITMAP_STALE, &bitmap->flags);
934 if (outofdate) 969 if (outofdate)
935 printk(KERN_INFO "%s: bitmap file is out of date, doing full " 970 printk(KERN_INFO "%s: bitmap file is out of date, doing full "
936 "recovery\n", bmname(bitmap)); 971 "recovery\n", bmname(bitmap));
937 972
938 bytes = DIV_ROUND_UP(bitmap->chunks, 8); 973 if (file && i_size_read(file->f_mapping->host) < store->bytes) {
939 if (!bitmap->mddev->bitmap_info.external)
940 bytes += sizeof(bitmap_super_t);
941
942 num_pages = DIV_ROUND_UP(bytes, PAGE_SIZE);
943
944 if (file && i_size_read(file->f_mapping->host) < bytes) {
945 printk(KERN_INFO "%s: bitmap file too short %lu < %lu\n", 974 printk(KERN_INFO "%s: bitmap file too short %lu < %lu\n",
946 bmname(bitmap), 975 bmname(bitmap),
947 (unsigned long) i_size_read(file->f_mapping->host), 976 (unsigned long) i_size_read(file->f_mapping->host),
948 bytes); 977 store->bytes);
949 goto err; 978 goto err;
950 } 979 }
951 980
952 ret = -ENOMEM;
953
954 bitmap->filemap = kmalloc(sizeof(struct page *) * num_pages, GFP_KERNEL);
955 if (!bitmap->filemap)
956 goto err;
957
958 /* We need 4 bits per page, rounded up to a multiple of sizeof(unsigned long) */
959 bitmap->filemap_attr = kzalloc(
960 roundup(DIV_ROUND_UP(num_pages*4, 8), sizeof(unsigned long)),
961 GFP_KERNEL);
962 if (!bitmap->filemap_attr)
963 goto err;
964
965 oldindex = ~0L; 981 oldindex = ~0L;
982 offset = 0;
983 if (!bitmap->mddev->bitmap_info.external)
984 offset = sizeof(bitmap_super_t);
966 985
967 for (i = 0; i < chunks; i++) { 986 for (i = 0; i < chunks; i++) {
968 int b; 987 int b;
969 index = file_page_index(bitmap, i); 988 index = file_page_index(&bitmap->storage, i);
970 bit = file_page_offset(bitmap, i); 989 bit = file_page_offset(&bitmap->storage, i);
971 if (index != oldindex) { /* this is a new page, read it in */ 990 if (index != oldindex) { /* this is a new page, read it in */
972 int count; 991 int count;
973 /* unmap the old page, we're done with it */ 992 /* unmap the old page, we're done with it */
974 if (index == num_pages-1) 993 if (index == store->file_pages-1)
975 count = bytes - index * PAGE_SIZE; 994 count = store->bytes - index * PAGE_SIZE;
976 else 995 else
977 count = PAGE_SIZE; 996 count = PAGE_SIZE;
978 if (index == 0 && bitmap->sb_page) { 997 page = store->filemap[index];
979 /* 998 if (file)
980 * if we're here then the superblock page 999 ret = read_page(file, index, bitmap,
981 * contains some bits (PAGE_SIZE != sizeof sb) 1000 count, page);
982 * we've already read it in, so just use it 1001 else
983 */ 1002 ret = read_sb_page(
984 page = bitmap->sb_page; 1003 bitmap->mddev,
985 offset = sizeof(bitmap_super_t); 1004 bitmap->mddev->bitmap_info.offset,
986 if (!file) 1005 page,
987 page = read_sb_page( 1006 index, count);
988 bitmap->mddev, 1007
989 bitmap->mddev->bitmap_info.offset, 1008 if (ret)
990 page,
991 index, count);
992 } else if (file) {
993 page = read_page(file, index, bitmap, count);
994 offset = 0;
995 } else {
996 page = read_sb_page(bitmap->mddev,
997 bitmap->mddev->bitmap_info.offset,
998 NULL,
999 index, count);
1000 offset = 0;
1001 }
1002 if (IS_ERR(page)) { /* read error */
1003 ret = PTR_ERR(page);
1004 goto err; 1009 goto err;
1005 }
1006 1010
1007 oldindex = index; 1011 oldindex = index;
1008 oldpage = page;
1009
1010 bitmap->filemap[bitmap->file_pages++] = page;
1011 bitmap->last_page_size = count;
1012 1012
1013 if (outofdate) { 1013 if (outofdate) {
1014 /* 1014 /*
@@ -1022,39 +1022,33 @@ static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
1022 write_page(bitmap, page, 1); 1022 write_page(bitmap, page, 1);
1023 1023
1024 ret = -EIO; 1024 ret = -EIO;
1025 if (bitmap->flags & BITMAP_WRITE_ERROR) 1025 if (test_bit(BITMAP_WRITE_ERROR,
1026 &bitmap->flags))
1026 goto err; 1027 goto err;
1027 } 1028 }
1028 } 1029 }
1029 paddr = kmap_atomic(page); 1030 paddr = kmap_atomic(page);
1030 if (bitmap->flags & BITMAP_HOSTENDIAN) 1031 if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
1031 b = test_bit(bit, paddr); 1032 b = test_bit(bit, paddr);
1032 else 1033 else
1033 b = test_bit_le(bit, paddr); 1034 b = test_bit_le(bit, paddr);
1034 kunmap_atomic(paddr); 1035 kunmap_atomic(paddr);
1035 if (b) { 1036 if (b) {
1036 /* if the disk bit is set, set the memory bit */ 1037 /* if the disk bit is set, set the memory bit */
1037 int needed = ((sector_t)(i+1) << bitmap->chunkshift 1038 int needed = ((sector_t)(i+1) << bitmap->counts.chunkshift
1038 >= start); 1039 >= start);
1039 bitmap_set_memory_bits(bitmap, 1040 bitmap_set_memory_bits(bitmap,
1040 (sector_t)i << bitmap->chunkshift, 1041 (sector_t)i << bitmap->counts.chunkshift,
1041 needed); 1042 needed);
1042 bit_cnt++; 1043 bit_cnt++;
1043 } 1044 }
1044 } 1045 offset = 0;
1045
1046 /* everything went OK */
1047 ret = 0;
1048 bitmap_mask_state(bitmap, BITMAP_STALE, MASK_UNSET);
1049
1050 if (bit_cnt) { /* Kick recovery if any bits were set */
1051 set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery);
1052 md_wakeup_thread(bitmap->mddev->thread);
1053 } 1046 }
1054 1047
1055 printk(KERN_INFO "%s: bitmap initialized from disk: " 1048 printk(KERN_INFO "%s: bitmap initialized from disk: "
1056 "read %lu/%lu pages, set %lu of %lu bits\n", 1049 "read %lu pages, set %lu of %lu bits\n",
1057 bmname(bitmap), bitmap->file_pages, num_pages, bit_cnt, chunks); 1050 bmname(bitmap), store->file_pages,
1051 bit_cnt, chunks);
1058 1052
1059 return 0; 1053 return 0;
1060 1054
@@ -1071,22 +1065,38 @@ void bitmap_write_all(struct bitmap *bitmap)
1071 */ 1065 */
1072 int i; 1066 int i;
1073 1067
1074 spin_lock_irq(&bitmap->lock); 1068 if (!bitmap || !bitmap->storage.filemap)
1075 for (i = 0; i < bitmap->file_pages; i++) 1069 return;
1076 set_page_attr(bitmap, bitmap->filemap[i], 1070 if (bitmap->storage.file)
1071 /* Only one copy, so nothing needed */
1072 return;
1073
1074 for (i = 0; i < bitmap->storage.file_pages; i++)
1075 set_page_attr(bitmap, i,
1077 BITMAP_PAGE_NEEDWRITE); 1076 BITMAP_PAGE_NEEDWRITE);
1078 bitmap->allclean = 0; 1077 bitmap->allclean = 0;
1079 spin_unlock_irq(&bitmap->lock);
1080} 1078}
1081 1079
1082static void bitmap_count_page(struct bitmap *bitmap, sector_t offset, int inc) 1080static void bitmap_count_page(struct bitmap_counts *bitmap,
1081 sector_t offset, int inc)
1083{ 1082{
1084 sector_t chunk = offset >> bitmap->chunkshift; 1083 sector_t chunk = offset >> bitmap->chunkshift;
1085 unsigned long page = chunk >> PAGE_COUNTER_SHIFT; 1084 unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1086 bitmap->bp[page].count += inc; 1085 bitmap->bp[page].count += inc;
1087 bitmap_checkfree(bitmap, page); 1086 bitmap_checkfree(bitmap, page);
1088} 1087}
1089static bitmap_counter_t *bitmap_get_counter(struct bitmap *bitmap, 1088
1089static void bitmap_set_pending(struct bitmap_counts *bitmap, sector_t offset)
1090{
1091 sector_t chunk = offset >> bitmap->chunkshift;
1092 unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1093 struct bitmap_page *bp = &bitmap->bp[page];
1094
1095 if (!bp->pending)
1096 bp->pending = 1;
1097}
1098
1099static bitmap_counter_t *bitmap_get_counter(struct bitmap_counts *bitmap,
1090 sector_t offset, sector_t *blocks, 1100 sector_t offset, sector_t *blocks,
1091 int create); 1101 int create);
1092 1102
@@ -1099,10 +1109,9 @@ void bitmap_daemon_work(struct mddev *mddev)
1099{ 1109{
1100 struct bitmap *bitmap; 1110 struct bitmap *bitmap;
1101 unsigned long j; 1111 unsigned long j;
1102 unsigned long flags; 1112 unsigned long nextpage;
1103 struct page *page = NULL, *lastpage = NULL;
1104 sector_t blocks; 1113 sector_t blocks;
1105 void *paddr; 1114 struct bitmap_counts *counts;
1106 1115
1107 /* Use a mutex to guard daemon_work against 1116 /* Use a mutex to guard daemon_work against
1108 * bitmap_destroy. 1117 * bitmap_destroy.
@@ -1124,112 +1133,90 @@ void bitmap_daemon_work(struct mddev *mddev)
1124 } 1133 }
1125 bitmap->allclean = 1; 1134 bitmap->allclean = 1;
1126 1135
1127 spin_lock_irqsave(&bitmap->lock, flags); 1136 /* Any file-page which is PENDING now needs to be written.
1128 for (j = 0; j < bitmap->chunks; j++) { 1137 * So set NEEDWRITE now, then after we make any last-minute changes
1138 * we will write it.
1139 */
1140 for (j = 0; j < bitmap->storage.file_pages; j++)
1141 if (test_and_clear_page_attr(bitmap, j,
1142 BITMAP_PAGE_PENDING))
1143 set_page_attr(bitmap, j,
1144 BITMAP_PAGE_NEEDWRITE);
1145
1146 if (bitmap->need_sync &&
1147 mddev->bitmap_info.external == 0) {
1148 /* Arrange for superblock update as well as
1149 * other changes */
1150 bitmap_super_t *sb;
1151 bitmap->need_sync = 0;
1152 if (bitmap->storage.filemap) {
1153 sb = kmap_atomic(bitmap->storage.sb_page);
1154 sb->events_cleared =
1155 cpu_to_le64(bitmap->events_cleared);
1156 kunmap_atomic(sb);
1157 set_page_attr(bitmap, 0,
1158 BITMAP_PAGE_NEEDWRITE);
1159 }
1160 }
1161 /* Now look at the bitmap counters and if any are '2' or '1',
1162 * decrement and handle accordingly.
1163 */
1164 counts = &bitmap->counts;
1165 spin_lock_irq(&counts->lock);
1166 nextpage = 0;
1167 for (j = 0; j < counts->chunks; j++) {
1129 bitmap_counter_t *bmc; 1168 bitmap_counter_t *bmc;
1130 if (!bitmap->filemap) 1169 sector_t block = (sector_t)j << counts->chunkshift;
1131 /* error or shutdown */
1132 break;
1133 1170
1134 page = filemap_get_page(bitmap, j); 1171 if (j == nextpage) {
1135 1172 nextpage += PAGE_COUNTER_RATIO;
1136 if (page != lastpage) { 1173 if (!counts->bp[j >> PAGE_COUNTER_SHIFT].pending) {
1137 /* skip this page unless it's marked as needing cleaning */ 1174 j |= PAGE_COUNTER_MASK;
1138 if (!test_page_attr(bitmap, page, BITMAP_PAGE_PENDING)) {
1139 int need_write = test_page_attr(bitmap, page,
1140 BITMAP_PAGE_NEEDWRITE);
1141 if (need_write)
1142 clear_page_attr(bitmap, page, BITMAP_PAGE_NEEDWRITE);
1143
1144 spin_unlock_irqrestore(&bitmap->lock, flags);
1145 if (need_write)
1146 write_page(bitmap, page, 0);
1147 spin_lock_irqsave(&bitmap->lock, flags);
1148 j |= (PAGE_BITS - 1);
1149 continue; 1175 continue;
1150 } 1176 }
1151 1177 counts->bp[j >> PAGE_COUNTER_SHIFT].pending = 0;
1152 /* grab the new page, sync and release the old */
1153 if (lastpage != NULL) {
1154 if (test_page_attr(bitmap, lastpage,
1155 BITMAP_PAGE_NEEDWRITE)) {
1156 clear_page_attr(bitmap, lastpage,
1157 BITMAP_PAGE_NEEDWRITE);
1158 spin_unlock_irqrestore(&bitmap->lock, flags);
1159 write_page(bitmap, lastpage, 0);
1160 } else {
1161 set_page_attr(bitmap, lastpage,
1162 BITMAP_PAGE_NEEDWRITE);
1163 bitmap->allclean = 0;
1164 spin_unlock_irqrestore(&bitmap->lock, flags);
1165 }
1166 } else
1167 spin_unlock_irqrestore(&bitmap->lock, flags);
1168 lastpage = page;
1169
1170 /* We are possibly going to clear some bits, so make
1171 * sure that events_cleared is up-to-date.
1172 */
1173 if (bitmap->need_sync &&
1174 mddev->bitmap_info.external == 0) {
1175 bitmap_super_t *sb;
1176 bitmap->need_sync = 0;
1177 sb = kmap_atomic(bitmap->sb_page);
1178 sb->events_cleared =
1179 cpu_to_le64(bitmap->events_cleared);
1180 kunmap_atomic(sb);
1181 write_page(bitmap, bitmap->sb_page, 1);
1182 }
1183 spin_lock_irqsave(&bitmap->lock, flags);
1184 if (!bitmap->need_sync)
1185 clear_page_attr(bitmap, page, BITMAP_PAGE_PENDING);
1186 else
1187 bitmap->allclean = 0;
1188 } 1178 }
1189 bmc = bitmap_get_counter(bitmap, 1179 bmc = bitmap_get_counter(counts,
1190 (sector_t)j << bitmap->chunkshift, 1180 block,
1191 &blocks, 0); 1181 &blocks, 0);
1192 if (!bmc) 1182
1183 if (!bmc) {
1193 j |= PAGE_COUNTER_MASK; 1184 j |= PAGE_COUNTER_MASK;
1194 else if (*bmc) { 1185 continue;
1195 if (*bmc == 1 && !bitmap->need_sync) {
1196 /* we can clear the bit */
1197 *bmc = 0;
1198 bitmap_count_page(bitmap,
1199 (sector_t)j << bitmap->chunkshift,
1200 -1);
1201
1202 /* clear the bit */
1203 paddr = kmap_atomic(page);
1204 if (bitmap->flags & BITMAP_HOSTENDIAN)
1205 clear_bit(file_page_offset(bitmap, j),
1206 paddr);
1207 else
1208 __clear_bit_le(
1209 file_page_offset(bitmap,
1210 j),
1211 paddr);
1212 kunmap_atomic(paddr);
1213 } else if (*bmc <= 2) {
1214 *bmc = 1; /* maybe clear the bit next time */
1215 set_page_attr(bitmap, page, BITMAP_PAGE_PENDING);
1216 bitmap->allclean = 0;
1217 }
1218 } 1186 }
1219 } 1187 if (*bmc == 1 && !bitmap->need_sync) {
1220 spin_unlock_irqrestore(&bitmap->lock, flags); 1188 /* We can clear the bit */
1221 1189 *bmc = 0;
1222 /* now sync the final page */ 1190 bitmap_count_page(counts, block, -1);
1223 if (lastpage != NULL) { 1191 bitmap_file_clear_bit(bitmap, block);
1224 spin_lock_irqsave(&bitmap->lock, flags); 1192 } else if (*bmc && *bmc <= 2) {
1225 if (test_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE)) { 1193 *bmc = 1;
1226 clear_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE); 1194 bitmap_set_pending(counts, block);
1227 spin_unlock_irqrestore(&bitmap->lock, flags);
1228 write_page(bitmap, lastpage, 0);
1229 } else {
1230 set_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE);
1231 bitmap->allclean = 0; 1195 bitmap->allclean = 0;
1232 spin_unlock_irqrestore(&bitmap->lock, flags); 1196 }
1197 }
1198 spin_unlock_irq(&counts->lock);
1199
1200 /* Now start writeout on any page in NEEDWRITE that isn't DIRTY.
1201 * DIRTY pages need to be written by bitmap_unplug so it can wait
1202 * for them.
1203 * If we find any DIRTY page we stop there and let bitmap_unplug
1204 * handle all the rest. This is important in the case where
1205 * the first blocking holds the superblock and it has been updated.
1206 * We mustn't write any other blocks before the superblock.
1207 */
1208 for (j = 0;
1209 j < bitmap->storage.file_pages
1210 && !test_bit(BITMAP_STALE, &bitmap->flags);
1211 j++) {
1212
1213 if (test_page_attr(bitmap, j,
1214 BITMAP_PAGE_DIRTY))
1215 /* bitmap_unplug will handle the rest */
1216 break;
1217 if (test_and_clear_page_attr(bitmap, j,
1218 BITMAP_PAGE_NEEDWRITE)) {
1219 write_page(bitmap, bitmap->storage.filemap[j], 0);
1233 } 1220 }
1234 } 1221 }
1235 1222
@@ -1240,7 +1227,7 @@ void bitmap_daemon_work(struct mddev *mddev)
1240 mutex_unlock(&mddev->bitmap_info.mutex); 1227 mutex_unlock(&mddev->bitmap_info.mutex);
1241} 1228}
1242 1229
1243static bitmap_counter_t *bitmap_get_counter(struct bitmap *bitmap, 1230static bitmap_counter_t *bitmap_get_counter(struct bitmap_counts *bitmap,
1244 sector_t offset, sector_t *blocks, 1231 sector_t offset, sector_t *blocks,
1245 int create) 1232 int create)
1246__releases(bitmap->lock) 1233__releases(bitmap->lock)
@@ -1302,10 +1289,10 @@ int bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sect
1302 sector_t blocks; 1289 sector_t blocks;
1303 bitmap_counter_t *bmc; 1290 bitmap_counter_t *bmc;
1304 1291
1305 spin_lock_irq(&bitmap->lock); 1292 spin_lock_irq(&bitmap->counts.lock);
1306 bmc = bitmap_get_counter(bitmap, offset, &blocks, 1); 1293 bmc = bitmap_get_counter(&bitmap->counts, offset, &blocks, 1);
1307 if (!bmc) { 1294 if (!bmc) {
1308 spin_unlock_irq(&bitmap->lock); 1295 spin_unlock_irq(&bitmap->counts.lock);
1309 return 0; 1296 return 0;
1310 } 1297 }
1311 1298
@@ -1317,7 +1304,7 @@ int bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sect
1317 */ 1304 */
1318 prepare_to_wait(&bitmap->overflow_wait, &__wait, 1305 prepare_to_wait(&bitmap->overflow_wait, &__wait,
1319 TASK_UNINTERRUPTIBLE); 1306 TASK_UNINTERRUPTIBLE);
1320 spin_unlock_irq(&bitmap->lock); 1307 spin_unlock_irq(&bitmap->counts.lock);
1321 io_schedule(); 1308 io_schedule();
1322 finish_wait(&bitmap->overflow_wait, &__wait); 1309 finish_wait(&bitmap->overflow_wait, &__wait);
1323 continue; 1310 continue;
@@ -1326,7 +1313,7 @@ int bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sect
1326 switch (*bmc) { 1313 switch (*bmc) {
1327 case 0: 1314 case 0:
1328 bitmap_file_set_bit(bitmap, offset); 1315 bitmap_file_set_bit(bitmap, offset);
1329 bitmap_count_page(bitmap, offset, 1); 1316 bitmap_count_page(&bitmap->counts, offset, 1);
1330 /* fall through */ 1317 /* fall through */
1331 case 1: 1318 case 1:
1332 *bmc = 2; 1319 *bmc = 2;
@@ -1334,7 +1321,7 @@ int bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sect
1334 1321
1335 (*bmc)++; 1322 (*bmc)++;
1336 1323
1337 spin_unlock_irq(&bitmap->lock); 1324 spin_unlock_irq(&bitmap->counts.lock);
1338 1325
1339 offset += blocks; 1326 offset += blocks;
1340 if (sectors > blocks) 1327 if (sectors > blocks)
@@ -1364,10 +1351,10 @@ void bitmap_endwrite(struct bitmap *bitmap, sector_t offset, unsigned long secto
1364 unsigned long flags; 1351 unsigned long flags;
1365 bitmap_counter_t *bmc; 1352 bitmap_counter_t *bmc;
1366 1353
1367 spin_lock_irqsave(&bitmap->lock, flags); 1354 spin_lock_irqsave(&bitmap->counts.lock, flags);
1368 bmc = bitmap_get_counter(bitmap, offset, &blocks, 0); 1355 bmc = bitmap_get_counter(&bitmap->counts, offset, &blocks, 0);
1369 if (!bmc) { 1356 if (!bmc) {
1370 spin_unlock_irqrestore(&bitmap->lock, flags); 1357 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1371 return; 1358 return;
1372 } 1359 }
1373 1360
@@ -1386,14 +1373,10 @@ void bitmap_endwrite(struct bitmap *bitmap, sector_t offset, unsigned long secto
1386 1373
1387 (*bmc)--; 1374 (*bmc)--;
1388 if (*bmc <= 2) { 1375 if (*bmc <= 2) {
1389 set_page_attr(bitmap, 1376 bitmap_set_pending(&bitmap->counts, offset);
1390 filemap_get_page(
1391 bitmap,
1392 offset >> bitmap->chunkshift),
1393 BITMAP_PAGE_PENDING);
1394 bitmap->allclean = 0; 1377 bitmap->allclean = 0;
1395 } 1378 }
1396 spin_unlock_irqrestore(&bitmap->lock, flags); 1379 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1397 offset += blocks; 1380 offset += blocks;
1398 if (sectors > blocks) 1381 if (sectors > blocks)
1399 sectors -= blocks; 1382 sectors -= blocks;
@@ -1412,8 +1395,8 @@ static int __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t
1412 *blocks = 1024; 1395 *blocks = 1024;
1413 return 1; /* always resync if no bitmap */ 1396 return 1; /* always resync if no bitmap */
1414 } 1397 }
1415 spin_lock_irq(&bitmap->lock); 1398 spin_lock_irq(&bitmap->counts.lock);
1416 bmc = bitmap_get_counter(bitmap, offset, blocks, 0); 1399 bmc = bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1417 rv = 0; 1400 rv = 0;
1418 if (bmc) { 1401 if (bmc) {
1419 /* locked */ 1402 /* locked */
@@ -1427,7 +1410,7 @@ static int __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t
1427 } 1410 }
1428 } 1411 }
1429 } 1412 }
1430 spin_unlock_irq(&bitmap->lock); 1413 spin_unlock_irq(&bitmap->counts.lock);
1431 return rv; 1414 return rv;
1432} 1415}
1433 1416
@@ -1464,8 +1447,8 @@ void bitmap_end_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, i
1464 *blocks = 1024; 1447 *blocks = 1024;
1465 return; 1448 return;
1466 } 1449 }
1467 spin_lock_irqsave(&bitmap->lock, flags); 1450 spin_lock_irqsave(&bitmap->counts.lock, flags);
1468 bmc = bitmap_get_counter(bitmap, offset, blocks, 0); 1451 bmc = bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
1469 if (bmc == NULL) 1452 if (bmc == NULL)
1470 goto unlock; 1453 goto unlock;
1471 /* locked */ 1454 /* locked */
@@ -1476,15 +1459,13 @@ void bitmap_end_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, i
1476 *bmc |= NEEDED_MASK; 1459 *bmc |= NEEDED_MASK;
1477 else { 1460 else {
1478 if (*bmc <= 2) { 1461 if (*bmc <= 2) {
1479 set_page_attr(bitmap, 1462 bitmap_set_pending(&bitmap->counts, offset);
1480 filemap_get_page(bitmap, offset >> bitmap->chunkshift),
1481 BITMAP_PAGE_PENDING);
1482 bitmap->allclean = 0; 1463 bitmap->allclean = 0;
1483 } 1464 }
1484 } 1465 }
1485 } 1466 }
1486 unlock: 1467 unlock:
1487 spin_unlock_irqrestore(&bitmap->lock, flags); 1468 spin_unlock_irqrestore(&bitmap->counts.lock, flags);
1488} 1469}
1489EXPORT_SYMBOL(bitmap_end_sync); 1470EXPORT_SYMBOL(bitmap_end_sync);
1490 1471
@@ -1524,7 +1505,7 @@ void bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector)
1524 1505
1525 bitmap->mddev->curr_resync_completed = sector; 1506 bitmap->mddev->curr_resync_completed = sector;
1526 set_bit(MD_CHANGE_CLEAN, &bitmap->mddev->flags); 1507 set_bit(MD_CHANGE_CLEAN, &bitmap->mddev->flags);
1527 sector &= ~((1ULL << bitmap->chunkshift) - 1); 1508 sector &= ~((1ULL << bitmap->counts.chunkshift) - 1);
1528 s = 0; 1509 s = 0;
1529 while (s < sector && s < bitmap->mddev->resync_max_sectors) { 1510 while (s < sector && s < bitmap->mddev->resync_max_sectors) {
1530 bitmap_end_sync(bitmap, s, &blocks, 0); 1511 bitmap_end_sync(bitmap, s, &blocks, 0);
@@ -1538,27 +1519,25 @@ EXPORT_SYMBOL(bitmap_cond_end_sync);
1538static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed) 1519static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed)
1539{ 1520{
1540 /* For each chunk covered by any of these sectors, set the 1521 /* For each chunk covered by any of these sectors, set the
1541 * counter to 1 and set resync_needed. They should all 1522 * counter to 2 and possibly set resync_needed. They should all
1542 * be 0 at this point 1523 * be 0 at this point
1543 */ 1524 */
1544 1525
1545 sector_t secs; 1526 sector_t secs;
1546 bitmap_counter_t *bmc; 1527 bitmap_counter_t *bmc;
1547 spin_lock_irq(&bitmap->lock); 1528 spin_lock_irq(&bitmap->counts.lock);
1548 bmc = bitmap_get_counter(bitmap, offset, &secs, 1); 1529 bmc = bitmap_get_counter(&bitmap->counts, offset, &secs, 1);
1549 if (!bmc) { 1530 if (!bmc) {
1550 spin_unlock_irq(&bitmap->lock); 1531 spin_unlock_irq(&bitmap->counts.lock);
1551 return; 1532 return;
1552 } 1533 }
1553 if (!*bmc) { 1534 if (!*bmc) {
1554 struct page *page;
1555 *bmc = 2 | (needed ? NEEDED_MASK : 0); 1535 *bmc = 2 | (needed ? NEEDED_MASK : 0);
1556 bitmap_count_page(bitmap, offset, 1); 1536 bitmap_count_page(&bitmap->counts, offset, 1);
1557 page = filemap_get_page(bitmap, offset >> bitmap->chunkshift); 1537 bitmap_set_pending(&bitmap->counts, offset);
1558 set_page_attr(bitmap, page, BITMAP_PAGE_PENDING);
1559 bitmap->allclean = 0; 1538 bitmap->allclean = 0;
1560 } 1539 }
1561 spin_unlock_irq(&bitmap->lock); 1540 spin_unlock_irq(&bitmap->counts.lock);
1562} 1541}
1563 1542
1564/* dirty the memory and file bits for bitmap chunks "s" to "e" */ 1543/* dirty the memory and file bits for bitmap chunks "s" to "e" */
@@ -1567,11 +1546,9 @@ void bitmap_dirty_bits(struct bitmap *bitmap, unsigned long s, unsigned long e)
1567 unsigned long chunk; 1546 unsigned long chunk;
1568 1547
1569 for (chunk = s; chunk <= e; chunk++) { 1548 for (chunk = s; chunk <= e; chunk++) {
1570 sector_t sec = (sector_t)chunk << bitmap->chunkshift; 1549 sector_t sec = (sector_t)chunk << bitmap->counts.chunkshift;
1571 bitmap_set_memory_bits(bitmap, sec, 1); 1550 bitmap_set_memory_bits(bitmap, sec, 1);
1572 spin_lock_irq(&bitmap->lock);
1573 bitmap_file_set_bit(bitmap, sec); 1551 bitmap_file_set_bit(bitmap, sec);
1574 spin_unlock_irq(&bitmap->lock);
1575 if (sec < bitmap->mddev->recovery_cp) 1552 if (sec < bitmap->mddev->recovery_cp)
1576 /* We are asserting that the array is dirty, 1553 /* We are asserting that the array is dirty,
1577 * so move the recovery_cp address back so 1554 * so move the recovery_cp address back so
@@ -1616,11 +1593,15 @@ static void bitmap_free(struct bitmap *bitmap)
1616 if (!bitmap) /* there was no bitmap */ 1593 if (!bitmap) /* there was no bitmap */
1617 return; 1594 return;
1618 1595
1619 /* release the bitmap file and kill the daemon */ 1596 /* Shouldn't be needed - but just in case.... */
1620 bitmap_file_put(bitmap); 1597 wait_event(bitmap->write_wait,
1598 atomic_read(&bitmap->pending_writes) == 0);
1599
1600 /* release the bitmap file */
1601 bitmap_file_unmap(&bitmap->storage);
1621 1602
1622 bp = bitmap->bp; 1603 bp = bitmap->counts.bp;
1623 pages = bitmap->pages; 1604 pages = bitmap->counts.pages;
1624 1605
1625 /* free all allocated memory */ 1606 /* free all allocated memory */
1626 1607
@@ -1659,25 +1640,19 @@ int bitmap_create(struct mddev *mddev)
1659{ 1640{
1660 struct bitmap *bitmap; 1641 struct bitmap *bitmap;
1661 sector_t blocks = mddev->resync_max_sectors; 1642 sector_t blocks = mddev->resync_max_sectors;
1662 unsigned long chunks;
1663 unsigned long pages;
1664 struct file *file = mddev->bitmap_info.file; 1643 struct file *file = mddev->bitmap_info.file;
1665 int err; 1644 int err;
1666 struct sysfs_dirent *bm = NULL; 1645 struct sysfs_dirent *bm = NULL;
1667 1646
1668 BUILD_BUG_ON(sizeof(bitmap_super_t) != 256); 1647 BUILD_BUG_ON(sizeof(bitmap_super_t) != 256);
1669 1648
1670 if (!file
1671 && !mddev->bitmap_info.offset) /* bitmap disabled, nothing to do */
1672 return 0;
1673
1674 BUG_ON(file && mddev->bitmap_info.offset); 1649 BUG_ON(file && mddev->bitmap_info.offset);
1675 1650
1676 bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL); 1651 bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL);
1677 if (!bitmap) 1652 if (!bitmap)
1678 return -ENOMEM; 1653 return -ENOMEM;
1679 1654
1680 spin_lock_init(&bitmap->lock); 1655 spin_lock_init(&bitmap->counts.lock);
1681 atomic_set(&bitmap->pending_writes, 0); 1656 atomic_set(&bitmap->pending_writes, 0);
1682 init_waitqueue_head(&bitmap->write_wait); 1657 init_waitqueue_head(&bitmap->write_wait);
1683 init_waitqueue_head(&bitmap->overflow_wait); 1658 init_waitqueue_head(&bitmap->overflow_wait);
@@ -1693,7 +1668,7 @@ int bitmap_create(struct mddev *mddev)
1693 } else 1668 } else
1694 bitmap->sysfs_can_clear = NULL; 1669 bitmap->sysfs_can_clear = NULL;
1695 1670
1696 bitmap->file = file; 1671 bitmap->storage.file = file;
1697 if (file) { 1672 if (file) {
1698 get_file(file); 1673 get_file(file);
1699 /* As future accesses to this file will use bmap, 1674 /* As future accesses to this file will use bmap,
@@ -1724,32 +1699,15 @@ int bitmap_create(struct mddev *mddev)
1724 goto error; 1699 goto error;
1725 1700
1726 bitmap->daemon_lastrun = jiffies; 1701 bitmap->daemon_lastrun = jiffies;
1727 bitmap->chunkshift = (ffz(~mddev->bitmap_info.chunksize) 1702 err = bitmap_resize(bitmap, blocks, mddev->bitmap_info.chunksize, 1);
1728 - BITMAP_BLOCK_SHIFT); 1703 if (err)
1729
1730 chunks = (blocks + (1 << bitmap->chunkshift) - 1) >>
1731 bitmap->chunkshift;
1732 pages = (chunks + PAGE_COUNTER_RATIO - 1) / PAGE_COUNTER_RATIO;
1733
1734 BUG_ON(!pages);
1735
1736 bitmap->chunks = chunks;
1737 bitmap->pages = pages;
1738 bitmap->missing_pages = pages;
1739
1740 bitmap->bp = kzalloc(pages * sizeof(*bitmap->bp), GFP_KERNEL);
1741
1742 err = -ENOMEM;
1743 if (!bitmap->bp)
1744 goto error; 1704 goto error;
1745 1705
1746 printk(KERN_INFO "created bitmap (%lu pages) for device %s\n", 1706 printk(KERN_INFO "created bitmap (%lu pages) for device %s\n",
1747 pages, bmname(bitmap)); 1707 bitmap->counts.pages, bmname(bitmap));
1748 1708
1749 mddev->bitmap = bitmap; 1709 mddev->bitmap = bitmap;
1750 1710 return test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0;
1751
1752 return (bitmap->flags & BITMAP_WRITE_ERROR) ? -EIO : 0;
1753 1711
1754 error: 1712 error:
1755 bitmap_free(bitmap); 1713 bitmap_free(bitmap);
@@ -1790,13 +1748,17 @@ int bitmap_load(struct mddev *mddev)
1790 1748
1791 if (err) 1749 if (err)
1792 goto out; 1750 goto out;
1751 clear_bit(BITMAP_STALE, &bitmap->flags);
1752
1753 /* Kick recovery in case any bits were set */
1754 set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery);
1793 1755
1794 mddev->thread->timeout = mddev->bitmap_info.daemon_sleep; 1756 mddev->thread->timeout = mddev->bitmap_info.daemon_sleep;
1795 md_wakeup_thread(mddev->thread); 1757 md_wakeup_thread(mddev->thread);
1796 1758
1797 bitmap_update_sb(bitmap); 1759 bitmap_update_sb(bitmap);
1798 1760
1799 if (bitmap->flags & BITMAP_WRITE_ERROR) 1761 if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
1800 err = -EIO; 1762 err = -EIO;
1801out: 1763out:
1802 return err; 1764 return err;
@@ -1806,30 +1768,194 @@ EXPORT_SYMBOL_GPL(bitmap_load);
1806void bitmap_status(struct seq_file *seq, struct bitmap *bitmap) 1768void bitmap_status(struct seq_file *seq, struct bitmap *bitmap)
1807{ 1769{
1808 unsigned long chunk_kb; 1770 unsigned long chunk_kb;
1809 unsigned long flags; 1771 struct bitmap_counts *counts;
1810 1772
1811 if (!bitmap) 1773 if (!bitmap)
1812 return; 1774 return;
1813 1775
1814 spin_lock_irqsave(&bitmap->lock, flags); 1776 counts = &bitmap->counts;
1777
1815 chunk_kb = bitmap->mddev->bitmap_info.chunksize >> 10; 1778 chunk_kb = bitmap->mddev->bitmap_info.chunksize >> 10;
1816 seq_printf(seq, "bitmap: %lu/%lu pages [%luKB], " 1779 seq_printf(seq, "bitmap: %lu/%lu pages [%luKB], "
1817 "%lu%s chunk", 1780 "%lu%s chunk",
1818 bitmap->pages - bitmap->missing_pages, 1781 counts->pages - counts->missing_pages,
1819 bitmap->pages, 1782 counts->pages,
1820 (bitmap->pages - bitmap->missing_pages) 1783 (counts->pages - counts->missing_pages)
1821 << (PAGE_SHIFT - 10), 1784 << (PAGE_SHIFT - 10),
1822 chunk_kb ? chunk_kb : bitmap->mddev->bitmap_info.chunksize, 1785 chunk_kb ? chunk_kb : bitmap->mddev->bitmap_info.chunksize,
1823 chunk_kb ? "KB" : "B"); 1786 chunk_kb ? "KB" : "B");
1824 if (bitmap->file) { 1787 if (bitmap->storage.file) {
1825 seq_printf(seq, ", file: "); 1788 seq_printf(seq, ", file: ");
1826 seq_path(seq, &bitmap->file->f_path, " \t\n"); 1789 seq_path(seq, &bitmap->storage.file->f_path, " \t\n");
1827 } 1790 }
1828 1791
1829 seq_printf(seq, "\n"); 1792 seq_printf(seq, "\n");
1830 spin_unlock_irqrestore(&bitmap->lock, flags);
1831} 1793}
1832 1794
1795int bitmap_resize(struct bitmap *bitmap, sector_t blocks,
1796 int chunksize, int init)
1797{
1798 /* If chunk_size is 0, choose an appropriate chunk size.
1799 * Then possibly allocate new storage space.
1800 * Then quiesce, copy bits, replace bitmap, and re-start
1801 *
1802 * This function is called both to set up the initial bitmap
1803 * and to resize the bitmap while the array is active.
1804 * If this happens as a result of the array being resized,
1805 * chunksize will be zero, and we need to choose a suitable
1806 * chunksize, otherwise we use what we are given.
1807 */
1808 struct bitmap_storage store;
1809 struct bitmap_counts old_counts;
1810 unsigned long chunks;
1811 sector_t block;
1812 sector_t old_blocks, new_blocks;
1813 int chunkshift;
1814 int ret = 0;
1815 long pages;
1816 struct bitmap_page *new_bp;
1817
1818 if (chunksize == 0) {
1819 /* If there is enough space, leave the chunk size unchanged,
1820 * else increase by factor of two until there is enough space.
1821 */
1822 long bytes;
1823 long space = bitmap->mddev->bitmap_info.space;
1824
1825 if (space == 0) {
1826 /* We don't know how much space there is, so limit
1827 * to current size - in sectors.
1828 */
1829 bytes = DIV_ROUND_UP(bitmap->counts.chunks, 8);
1830 if (!bitmap->mddev->bitmap_info.external)
1831 bytes += sizeof(bitmap_super_t);
1832 space = DIV_ROUND_UP(bytes, 512);
1833 bitmap->mddev->bitmap_info.space = space;
1834 }
1835 chunkshift = bitmap->counts.chunkshift;
1836 chunkshift--;
1837 do {
1838 /* 'chunkshift' is shift from block size to chunk size */
1839 chunkshift++;
1840 chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
1841 bytes = DIV_ROUND_UP(chunks, 8);
1842 if (!bitmap->mddev->bitmap_info.external)
1843 bytes += sizeof(bitmap_super_t);
1844 } while (bytes > (space << 9));
1845 } else
1846 chunkshift = ffz(~chunksize) - BITMAP_BLOCK_SHIFT;
1847
1848 chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
1849 memset(&store, 0, sizeof(store));
1850 if (bitmap->mddev->bitmap_info.offset || bitmap->mddev->bitmap_info.file)
1851 ret = bitmap_storage_alloc(&store, chunks,
1852 !bitmap->mddev->bitmap_info.external);
1853 if (ret)
1854 goto err;
1855
1856 pages = DIV_ROUND_UP(chunks, PAGE_COUNTER_RATIO);
1857
1858 new_bp = kzalloc(pages * sizeof(*new_bp), GFP_KERNEL);
1859 ret = -ENOMEM;
1860 if (!new_bp) {
1861 bitmap_file_unmap(&store);
1862 goto err;
1863 }
1864
1865 if (!init)
1866 bitmap->mddev->pers->quiesce(bitmap->mddev, 1);
1867
1868 store.file = bitmap->storage.file;
1869 bitmap->storage.file = NULL;
1870
1871 if (store.sb_page && bitmap->storage.sb_page)
1872 memcpy(page_address(store.sb_page),
1873 page_address(bitmap->storage.sb_page),
1874 sizeof(bitmap_super_t));
1875 bitmap_file_unmap(&bitmap->storage);
1876 bitmap->storage = store;
1877
1878 old_counts = bitmap->counts;
1879 bitmap->counts.bp = new_bp;
1880 bitmap->counts.pages = pages;
1881 bitmap->counts.missing_pages = pages;
1882 bitmap->counts.chunkshift = chunkshift;
1883 bitmap->counts.chunks = chunks;
1884 bitmap->mddev->bitmap_info.chunksize = 1 << (chunkshift +
1885 BITMAP_BLOCK_SHIFT);
1886
1887 blocks = min(old_counts.chunks << old_counts.chunkshift,
1888 chunks << chunkshift);
1889
1890 spin_lock_irq(&bitmap->counts.lock);
1891 for (block = 0; block < blocks; ) {
1892 bitmap_counter_t *bmc_old, *bmc_new;
1893 int set;
1894
1895 bmc_old = bitmap_get_counter(&old_counts, block,
1896 &old_blocks, 0);
1897 set = bmc_old && NEEDED(*bmc_old);
1898
1899 if (set) {
1900 bmc_new = bitmap_get_counter(&bitmap->counts, block,
1901 &new_blocks, 1);
1902 if (*bmc_new == 0) {
1903 /* need to set on-disk bits too. */
1904 sector_t end = block + new_blocks;
1905 sector_t start = block >> chunkshift;
1906 start <<= chunkshift;
1907 while (start < end) {
1908 bitmap_file_set_bit(bitmap, block);
1909 start += 1 << chunkshift;
1910 }
1911 *bmc_new = 2;
1912 bitmap_count_page(&bitmap->counts,
1913 block, 1);
1914 bitmap_set_pending(&bitmap->counts,
1915 block);
1916 }
1917 *bmc_new |= NEEDED_MASK;
1918 if (new_blocks < old_blocks)
1919 old_blocks = new_blocks;
1920 }
1921 block += old_blocks;
1922 }
1923
1924 if (!init) {
1925 int i;
1926 while (block < (chunks << chunkshift)) {
1927 bitmap_counter_t *bmc;
1928 bmc = bitmap_get_counter(&bitmap->counts, block,
1929 &new_blocks, 1);
1930 if (bmc) {
1931 /* new space. It needs to be resynced, so
1932 * we set NEEDED_MASK.
1933 */
1934 if (*bmc == 0) {
1935 *bmc = NEEDED_MASK | 2;
1936 bitmap_count_page(&bitmap->counts,
1937 block, 1);
1938 bitmap_set_pending(&bitmap->counts,
1939 block);
1940 }
1941 }
1942 block += new_blocks;
1943 }
1944 for (i = 0; i < bitmap->storage.file_pages; i++)
1945 set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
1946 }
1947 spin_unlock_irq(&bitmap->counts.lock);
1948
1949 if (!init) {
1950 bitmap_unplug(bitmap);
1951 bitmap->mddev->pers->quiesce(bitmap->mddev, 0);
1952 }
1953 ret = 0;
1954err:
1955 return ret;
1956}
1957EXPORT_SYMBOL_GPL(bitmap_resize);
1958
1833static ssize_t 1959static ssize_t
1834location_show(struct mddev *mddev, char *page) 1960location_show(struct mddev *mddev, char *page)
1835{ 1961{
@@ -1923,6 +2049,43 @@ location_store(struct mddev *mddev, const char *buf, size_t len)
1923static struct md_sysfs_entry bitmap_location = 2049static struct md_sysfs_entry bitmap_location =
1924__ATTR(location, S_IRUGO|S_IWUSR, location_show, location_store); 2050__ATTR(location, S_IRUGO|S_IWUSR, location_show, location_store);
1925 2051
2052/* 'bitmap/space' is the space available at 'location' for the
2053 * bitmap. This allows the kernel to know when it is safe to
2054 * resize the bitmap to match a resized array.
2055 */
2056static ssize_t
2057space_show(struct mddev *mddev, char *page)
2058{
2059 return sprintf(page, "%lu\n", mddev->bitmap_info.space);
2060}
2061
2062static ssize_t
2063space_store(struct mddev *mddev, const char *buf, size_t len)
2064{
2065 unsigned long sectors;
2066 int rv;
2067
2068 rv = kstrtoul(buf, 10, &sectors);
2069 if (rv)
2070 return rv;
2071
2072 if (sectors == 0)
2073 return -EINVAL;
2074
2075 if (mddev->bitmap &&
2076 sectors < (mddev->bitmap->storage.bytes + 511) >> 9)
2077 return -EFBIG; /* Bitmap is too big for this small space */
2078
2079 /* could make sure it isn't too big, but that isn't really
2080 * needed - user-space should be careful.
2081 */
2082 mddev->bitmap_info.space = sectors;
2083 return len;
2084}
2085
2086static struct md_sysfs_entry bitmap_space =
2087__ATTR(space, S_IRUGO|S_IWUSR, space_show, space_store);
2088
1926static ssize_t 2089static ssize_t
1927timeout_show(struct mddev *mddev, char *page) 2090timeout_show(struct mddev *mddev, char *page)
1928{ 2091{
@@ -2098,6 +2261,7 @@ __ATTR(max_backlog_used, S_IRUGO | S_IWUSR,
2098 2261
2099static struct attribute *md_bitmap_attrs[] = { 2262static struct attribute *md_bitmap_attrs[] = {
2100 &bitmap_location.attr, 2263 &bitmap_location.attr,
2264 &bitmap_space.attr,
2101 &bitmap_timeout.attr, 2265 &bitmap_timeout.attr,
2102 &bitmap_backlog.attr, 2266 &bitmap_backlog.attr,
2103 &bitmap_chunksize.attr, 2267 &bitmap_chunksize.attr,
generated by cgit v1.2.2 (git 2.25.0) at 2025-09-20 17:42:10 -0400