diff options
Diffstat (limited to 'drivers/md/bitmap.c')
-rw-r--r-- | drivers/md/bitmap.c | 483 |
1 files changed, 167 insertions, 316 deletions
diff --git a/drivers/md/bitmap.c b/drivers/md/bitmap.c index f8ffaee20ff8..ebbd2d856256 100644 --- a/drivers/md/bitmap.c +++ b/drivers/md/bitmap.c | |||
@@ -7,7 +7,6 @@ | |||
7 | * additions, Copyright (C) 2003-2004, Paul Clements, SteelEye Technology, Inc.: | 7 | * additions, Copyright (C) 2003-2004, Paul Clements, SteelEye Technology, Inc.: |
8 | * - added disk storage for bitmap | 8 | * - added disk storage for bitmap |
9 | * - changes to allow various bitmap chunk sizes | 9 | * - changes to allow various bitmap chunk sizes |
10 | * - added bitmap daemon (to asynchronously clear bitmap bits from disk) | ||
11 | */ | 10 | */ |
12 | 11 | ||
13 | /* | 12 | /* |
@@ -15,9 +14,6 @@ | |||
15 | * | 14 | * |
16 | * flush after percent set rather than just time based. (maybe both). | 15 | * flush after percent set rather than just time based. (maybe both). |
17 | * wait if count gets too high, wake when it drops to half. | 16 | * wait if count gets too high, wake when it drops to half. |
18 | * allow bitmap to be mirrored with superblock (before or after...) | ||
19 | * allow hot-add to re-instate a current device. | ||
20 | * allow hot-add of bitmap after quiessing device | ||
21 | */ | 17 | */ |
22 | 18 | ||
23 | #include <linux/module.h> | 19 | #include <linux/module.h> |
@@ -73,24 +69,6 @@ static inline char * bmname(struct bitmap *bitmap) | |||
73 | 69 | ||
74 | 70 | ||
75 | /* | 71 | /* |
76 | * test if the bitmap is active | ||
77 | */ | ||
78 | int bitmap_active(struct bitmap *bitmap) | ||
79 | { | ||
80 | unsigned long flags; | ||
81 | int res = 0; | ||
82 | |||
83 | if (!bitmap) | ||
84 | return res; | ||
85 | spin_lock_irqsave(&bitmap->lock, flags); | ||
86 | res = bitmap->flags & BITMAP_ACTIVE; | ||
87 | spin_unlock_irqrestore(&bitmap->lock, flags); | ||
88 | return res; | ||
89 | } | ||
90 | |||
91 | #define WRITE_POOL_SIZE 256 | ||
92 | |||
93 | /* | ||
94 | * just a placeholder - calls kmalloc for bitmap pages | 72 | * just a placeholder - calls kmalloc for bitmap pages |
95 | */ | 73 | */ |
96 | static unsigned char *bitmap_alloc_page(struct bitmap *bitmap) | 74 | static unsigned char *bitmap_alloc_page(struct bitmap *bitmap) |
@@ -269,6 +247,8 @@ static struct page *read_sb_page(mddev_t *mddev, long offset, unsigned long inde | |||
269 | 247 | ||
270 | if (sync_page_io(rdev->bdev, target, PAGE_SIZE, page, READ)) { | 248 | if (sync_page_io(rdev->bdev, target, PAGE_SIZE, page, READ)) { |
271 | page->index = index; | 249 | page->index = index; |
250 | attach_page_buffers(page, NULL); /* so that free_buffer will | ||
251 | * quietly no-op */ | ||
272 | return page; | 252 | return page; |
273 | } | 253 | } |
274 | } | 254 | } |
@@ -300,77 +280,132 @@ static int write_sb_page(mddev_t *mddev, long offset, struct page *page, int wai | |||
300 | */ | 280 | */ |
301 | static int write_page(struct bitmap *bitmap, struct page *page, int wait) | 281 | static int write_page(struct bitmap *bitmap, struct page *page, int wait) |
302 | { | 282 | { |
303 | int ret = -ENOMEM; | 283 | struct buffer_head *bh; |
304 | 284 | ||
305 | if (bitmap->file == NULL) | 285 | if (bitmap->file == NULL) |
306 | return write_sb_page(bitmap->mddev, bitmap->offset, page, wait); | 286 | return write_sb_page(bitmap->mddev, bitmap->offset, page, wait); |
307 | 287 | ||
308 | flush_dcache_page(page); /* make sure visible to anyone reading the file */ | 288 | bh = page_buffers(page); |
309 | 289 | ||
310 | if (wait) | 290 | while (bh && bh->b_blocknr) { |
311 | lock_page(page); | 291 | atomic_inc(&bitmap->pending_writes); |
312 | else { | 292 | set_buffer_locked(bh); |
313 | if (TestSetPageLocked(page)) | 293 | set_buffer_mapped(bh); |
314 | return -EAGAIN; /* already locked */ | 294 | submit_bh(WRITE, bh); |
315 | if (PageWriteback(page)) { | 295 | bh = bh->b_this_page; |
316 | unlock_page(page); | ||
317 | return -EAGAIN; | ||
318 | } | ||
319 | } | 296 | } |
320 | 297 | ||
321 | ret = page->mapping->a_ops->prepare_write(bitmap->file, page, 0, PAGE_SIZE); | 298 | if (wait) { |
322 | if (!ret) | 299 | wait_event(bitmap->write_wait, |
323 | ret = page->mapping->a_ops->commit_write(bitmap->file, page, 0, | 300 | atomic_read(&bitmap->pending_writes)==0); |
324 | PAGE_SIZE); | 301 | return (bitmap->flags & BITMAP_WRITE_ERROR) ? -EIO : 0; |
325 | if (ret) { | ||
326 | unlock_page(page); | ||
327 | return ret; | ||
328 | } | 302 | } |
303 | return 0; | ||
304 | } | ||
329 | 305 | ||
330 | set_page_dirty(page); /* force it to be written out */ | 306 | static void end_bitmap_write(struct buffer_head *bh, int uptodate) |
331 | 307 | { | |
332 | if (!wait) { | 308 | struct bitmap *bitmap = bh->b_private; |
333 | /* add to list to be waited for by daemon */ | 309 | unsigned long flags; |
334 | struct page_list *item = mempool_alloc(bitmap->write_pool, GFP_NOIO); | 310 | |
335 | item->page = page; | 311 | if (!uptodate) { |
336 | get_page(page); | 312 | spin_lock_irqsave(&bitmap->lock, flags); |
337 | spin_lock(&bitmap->write_lock); | 313 | bitmap->flags |= BITMAP_WRITE_ERROR; |
338 | list_add(&item->list, &bitmap->complete_pages); | 314 | spin_unlock_irqrestore(&bitmap->lock, flags); |
339 | spin_unlock(&bitmap->write_lock); | 315 | } |
340 | md_wakeup_thread(bitmap->writeback_daemon); | 316 | if (atomic_dec_and_test(&bitmap->pending_writes)) |
317 | wake_up(&bitmap->write_wait); | ||
318 | } | ||
319 | |||
320 | /* copied from buffer.c */ | ||
321 | static void | ||
322 | __clear_page_buffers(struct page *page) | ||
323 | { | ||
324 | ClearPagePrivate(page); | ||
325 | set_page_private(page, 0); | ||
326 | page_cache_release(page); | ||
327 | } | ||
328 | static void free_buffers(struct page *page) | ||
329 | { | ||
330 | struct buffer_head *bh = page_buffers(page); | ||
331 | |||
332 | while (bh) { | ||
333 | struct buffer_head *next = bh->b_this_page; | ||
334 | free_buffer_head(bh); | ||
335 | bh = next; | ||
341 | } | 336 | } |
342 | return write_one_page(page, wait); | 337 | __clear_page_buffers(page); |
338 | put_page(page); | ||
343 | } | 339 | } |
344 | 340 | ||
345 | /* read a page from a file, pinning it into cache, and return bytes_read */ | 341 | /* read a page from a file. |
342 | * We both read the page, and attach buffers to the page to record the | ||
343 | * address of each block (using bmap). These addresses will be used | ||
344 | * to write the block later, completely bypassing the filesystem. | ||
345 | * This usage is similar to how swap files are handled, and allows us | ||
346 | * to write to a file with no concerns of memory allocation failing. | ||
347 | */ | ||
346 | static struct page *read_page(struct file *file, unsigned long index, | 348 | static struct page *read_page(struct file *file, unsigned long index, |
347 | unsigned long *bytes_read) | 349 | struct bitmap *bitmap, |
350 | unsigned long count) | ||
348 | { | 351 | { |
349 | struct inode *inode = file->f_mapping->host; | ||
350 | struct page *page = NULL; | 352 | struct page *page = NULL; |
351 | loff_t isize = i_size_read(inode); | 353 | struct inode *inode = file->f_dentry->d_inode; |
352 | unsigned long end_index = isize >> PAGE_SHIFT; | 354 | struct buffer_head *bh; |
355 | sector_t block; | ||
353 | 356 | ||
354 | PRINTK("read bitmap file (%dB @ %Lu)\n", (int)PAGE_SIZE, | 357 | PRINTK("read bitmap file (%dB @ %Lu)\n", (int)PAGE_SIZE, |
355 | (unsigned long long)index << PAGE_SHIFT); | 358 | (unsigned long long)index << PAGE_SHIFT); |
356 | 359 | ||
357 | page = read_cache_page(inode->i_mapping, index, | 360 | page = alloc_page(GFP_KERNEL); |
358 | (filler_t *)inode->i_mapping->a_ops->readpage, file); | 361 | if (!page) |
362 | page = ERR_PTR(-ENOMEM); | ||
359 | if (IS_ERR(page)) | 363 | if (IS_ERR(page)) |
360 | goto out; | 364 | goto out; |
361 | wait_on_page_locked(page); | 365 | |
362 | if (!PageUptodate(page) || PageError(page)) { | 366 | bh = alloc_page_buffers(page, 1<<inode->i_blkbits, 0); |
367 | if (!bh) { | ||
363 | put_page(page); | 368 | put_page(page); |
364 | page = ERR_PTR(-EIO); | 369 | page = ERR_PTR(-ENOMEM); |
365 | goto out; | 370 | goto out; |
366 | } | 371 | } |
372 | attach_page_buffers(page, bh); | ||
373 | block = index << (PAGE_SHIFT - inode->i_blkbits); | ||
374 | while (bh) { | ||
375 | if (count == 0) | ||
376 | bh->b_blocknr = 0; | ||
377 | else { | ||
378 | bh->b_blocknr = bmap(inode, block); | ||
379 | if (bh->b_blocknr == 0) { | ||
380 | /* Cannot use this file! */ | ||
381 | free_buffers(page); | ||
382 | page = ERR_PTR(-EINVAL); | ||
383 | goto out; | ||
384 | } | ||
385 | bh->b_bdev = inode->i_sb->s_bdev; | ||
386 | if (count < (1<<inode->i_blkbits)) | ||
387 | count = 0; | ||
388 | else | ||
389 | count -= (1<<inode->i_blkbits); | ||
390 | |||
391 | bh->b_end_io = end_bitmap_write; | ||
392 | bh->b_private = bitmap; | ||
393 | atomic_inc(&bitmap->pending_writes); | ||
394 | set_buffer_locked(bh); | ||
395 | set_buffer_mapped(bh); | ||
396 | submit_bh(READ, bh); | ||
397 | } | ||
398 | block++; | ||
399 | bh = bh->b_this_page; | ||
400 | } | ||
401 | page->index = index; | ||
367 | 402 | ||
368 | if (index > end_index) /* we have read beyond EOF */ | 403 | wait_event(bitmap->write_wait, |
369 | *bytes_read = 0; | 404 | atomic_read(&bitmap->pending_writes)==0); |
370 | else if (index == end_index) /* possible short read */ | 405 | if (bitmap->flags & BITMAP_WRITE_ERROR) { |
371 | *bytes_read = isize & ~PAGE_MASK; | 406 | free_buffers(page); |
372 | else | 407 | page = ERR_PTR(-EIO); |
373 | *bytes_read = PAGE_SIZE; /* got a full page */ | 408 | } |
374 | out: | 409 | out: |
375 | if (IS_ERR(page)) | 410 | if (IS_ERR(page)) |
376 | printk(KERN_ALERT "md: bitmap read error: (%dB @ %Lu): %ld\n", | 411 | printk(KERN_ALERT "md: bitmap read error: (%dB @ %Lu): %ld\n", |
@@ -441,16 +476,14 @@ static int bitmap_read_sb(struct bitmap *bitmap) | |||
441 | char *reason = NULL; | 476 | char *reason = NULL; |
442 | bitmap_super_t *sb; | 477 | bitmap_super_t *sb; |
443 | unsigned long chunksize, daemon_sleep, write_behind; | 478 | unsigned long chunksize, daemon_sleep, write_behind; |
444 | unsigned long bytes_read; | ||
445 | unsigned long long events; | 479 | unsigned long long events; |
446 | int err = -EINVAL; | 480 | int err = -EINVAL; |
447 | 481 | ||
448 | /* page 0 is the superblock, read it... */ | 482 | /* page 0 is the superblock, read it... */ |
449 | if (bitmap->file) | 483 | if (bitmap->file) |
450 | bitmap->sb_page = read_page(bitmap->file, 0, &bytes_read); | 484 | bitmap->sb_page = read_page(bitmap->file, 0, bitmap, PAGE_SIZE); |
451 | else { | 485 | else { |
452 | bitmap->sb_page = read_sb_page(bitmap->mddev, bitmap->offset, 0); | 486 | bitmap->sb_page = read_sb_page(bitmap->mddev, bitmap->offset, 0); |
453 | bytes_read = PAGE_SIZE; | ||
454 | } | 487 | } |
455 | if (IS_ERR(bitmap->sb_page)) { | 488 | if (IS_ERR(bitmap->sb_page)) { |
456 | err = PTR_ERR(bitmap->sb_page); | 489 | err = PTR_ERR(bitmap->sb_page); |
@@ -460,13 +493,6 @@ static int bitmap_read_sb(struct bitmap *bitmap) | |||
460 | 493 | ||
461 | sb = (bitmap_super_t *)kmap_atomic(bitmap->sb_page, KM_USER0); | 494 | sb = (bitmap_super_t *)kmap_atomic(bitmap->sb_page, KM_USER0); |
462 | 495 | ||
463 | if (bytes_read < sizeof(*sb)) { /* short read */ | ||
464 | printk(KERN_INFO "%s: bitmap file superblock truncated\n", | ||
465 | bmname(bitmap)); | ||
466 | err = -ENOSPC; | ||
467 | goto out; | ||
468 | } | ||
469 | |||
470 | chunksize = le32_to_cpu(sb->chunksize); | 496 | chunksize = le32_to_cpu(sb->chunksize); |
471 | daemon_sleep = le32_to_cpu(sb->daemon_sleep); | 497 | daemon_sleep = le32_to_cpu(sb->daemon_sleep); |
472 | write_behind = le32_to_cpu(sb->write_behind); | 498 | write_behind = le32_to_cpu(sb->write_behind); |
@@ -550,7 +576,6 @@ static void bitmap_mask_state(struct bitmap *bitmap, enum bitmap_state bits, | |||
550 | spin_unlock_irqrestore(&bitmap->lock, flags); | 576 | spin_unlock_irqrestore(&bitmap->lock, flags); |
551 | return; | 577 | return; |
552 | } | 578 | } |
553 | get_page(bitmap->sb_page); | ||
554 | spin_unlock_irqrestore(&bitmap->lock, flags); | 579 | spin_unlock_irqrestore(&bitmap->lock, flags); |
555 | sb = (bitmap_super_t *)kmap_atomic(bitmap->sb_page, KM_USER0); | 580 | sb = (bitmap_super_t *)kmap_atomic(bitmap->sb_page, KM_USER0); |
556 | switch (op) { | 581 | switch (op) { |
@@ -561,7 +586,6 @@ static void bitmap_mask_state(struct bitmap *bitmap, enum bitmap_state bits, | |||
561 | default: BUG(); | 586 | default: BUG(); |
562 | } | 587 | } |
563 | kunmap_atomic(sb, KM_USER0); | 588 | kunmap_atomic(sb, KM_USER0); |
564 | put_page(bitmap->sb_page); | ||
565 | } | 589 | } |
566 | 590 | ||
567 | /* | 591 | /* |
@@ -614,48 +638,17 @@ static void bitmap_file_unmap(struct bitmap *bitmap) | |||
614 | 638 | ||
615 | while (pages--) | 639 | while (pages--) |
616 | if (map[pages]->index != 0) /* 0 is sb_page, release it below */ | 640 | if (map[pages]->index != 0) /* 0 is sb_page, release it below */ |
617 | put_page(map[pages]); | 641 | free_buffers(map[pages]); |
618 | kfree(map); | 642 | kfree(map); |
619 | kfree(attr); | 643 | kfree(attr); |
620 | 644 | ||
621 | safe_put_page(sb_page); | 645 | if (sb_page) |
622 | } | 646 | free_buffers(sb_page); |
623 | |||
624 | static void bitmap_stop_daemon(struct bitmap *bitmap); | ||
625 | |||
626 | /* dequeue the next item in a page list -- don't call from irq context */ | ||
627 | static struct page_list *dequeue_page(struct bitmap *bitmap) | ||
628 | { | ||
629 | struct page_list *item = NULL; | ||
630 | struct list_head *head = &bitmap->complete_pages; | ||
631 | |||
632 | spin_lock(&bitmap->write_lock); | ||
633 | if (list_empty(head)) | ||
634 | goto out; | ||
635 | item = list_entry(head->prev, struct page_list, list); | ||
636 | list_del(head->prev); | ||
637 | out: | ||
638 | spin_unlock(&bitmap->write_lock); | ||
639 | return item; | ||
640 | } | ||
641 | |||
642 | static void drain_write_queues(struct bitmap *bitmap) | ||
643 | { | ||
644 | struct page_list *item; | ||
645 | |||
646 | while ((item = dequeue_page(bitmap))) { | ||
647 | /* don't bother to wait */ | ||
648 | put_page(item->page); | ||
649 | mempool_free(item, bitmap->write_pool); | ||
650 | } | ||
651 | |||
652 | wake_up(&bitmap->write_wait); | ||
653 | } | 647 | } |
654 | 648 | ||
655 | static void bitmap_file_put(struct bitmap *bitmap) | 649 | static void bitmap_file_put(struct bitmap *bitmap) |
656 | { | 650 | { |
657 | struct file *file; | 651 | struct file *file; |
658 | struct inode *inode; | ||
659 | unsigned long flags; | 652 | unsigned long flags; |
660 | 653 | ||
661 | spin_lock_irqsave(&bitmap->lock, flags); | 654 | spin_lock_irqsave(&bitmap->lock, flags); |
@@ -663,17 +656,14 @@ static void bitmap_file_put(struct bitmap *bitmap) | |||
663 | bitmap->file = NULL; | 656 | bitmap->file = NULL; |
664 | spin_unlock_irqrestore(&bitmap->lock, flags); | 657 | spin_unlock_irqrestore(&bitmap->lock, flags); |
665 | 658 | ||
666 | bitmap_stop_daemon(bitmap); | 659 | if (file) |
667 | 660 | wait_event(bitmap->write_wait, | |
668 | drain_write_queues(bitmap); | 661 | atomic_read(&bitmap->pending_writes)==0); |
669 | |||
670 | bitmap_file_unmap(bitmap); | 662 | bitmap_file_unmap(bitmap); |
671 | 663 | ||
672 | if (file) { | 664 | if (file) { |
673 | inode = file->f_mapping->host; | 665 | struct inode *inode = file->f_dentry->d_inode; |
674 | spin_lock(&inode->i_lock); | 666 | invalidate_inode_pages(inode->i_mapping); |
675 | atomic_set(&inode->i_writecount, 1); /* allow writes again */ | ||
676 | spin_unlock(&inode->i_lock); | ||
677 | fput(file); | 667 | fput(file); |
678 | } | 668 | } |
679 | } | 669 | } |
@@ -708,26 +698,27 @@ static void bitmap_file_kick(struct bitmap *bitmap) | |||
708 | } | 698 | } |
709 | 699 | ||
710 | enum bitmap_page_attr { | 700 | enum bitmap_page_attr { |
711 | BITMAP_PAGE_DIRTY = 1, // there are set bits that need to be synced | 701 | BITMAP_PAGE_DIRTY = 0, // there are set bits that need to be synced |
712 | BITMAP_PAGE_CLEAN = 2, // there are bits that might need to be cleared | 702 | BITMAP_PAGE_CLEAN = 1, // there are bits that might need to be cleared |
713 | BITMAP_PAGE_NEEDWRITE=4, // there are cleared bits that need to be synced | 703 | BITMAP_PAGE_NEEDWRITE=2, // there are cleared bits that need to be synced |
714 | }; | 704 | }; |
715 | 705 | ||
716 | static inline void set_page_attr(struct bitmap *bitmap, struct page *page, | 706 | static inline void set_page_attr(struct bitmap *bitmap, struct page *page, |
717 | enum bitmap_page_attr attr) | 707 | enum bitmap_page_attr attr) |
718 | { | 708 | { |
719 | bitmap->filemap_attr[page->index] |= attr; | 709 | __set_bit((page->index<<2) + attr, bitmap->filemap_attr); |
720 | } | 710 | } |
721 | 711 | ||
722 | static inline void clear_page_attr(struct bitmap *bitmap, struct page *page, | 712 | static inline void clear_page_attr(struct bitmap *bitmap, struct page *page, |
723 | enum bitmap_page_attr attr) | 713 | enum bitmap_page_attr attr) |
724 | { | 714 | { |
725 | bitmap->filemap_attr[page->index] &= ~attr; | 715 | __clear_bit((page->index<<2) + attr, bitmap->filemap_attr); |
726 | } | 716 | } |
727 | 717 | ||
728 | static inline unsigned long get_page_attr(struct bitmap *bitmap, struct page *page) | 718 | static inline unsigned long test_page_attr(struct bitmap *bitmap, struct page *page, |
719 | enum bitmap_page_attr attr) | ||
729 | { | 720 | { |
730 | return bitmap->filemap_attr[page->index]; | 721 | return test_bit((page->index<<2) + attr, bitmap->filemap_attr); |
731 | } | 722 | } |
732 | 723 | ||
733 | /* | 724 | /* |
@@ -751,11 +742,6 @@ static void bitmap_file_set_bit(struct bitmap *bitmap, sector_t block) | |||
751 | page = filemap_get_page(bitmap, chunk); | 742 | page = filemap_get_page(bitmap, chunk); |
752 | bit = file_page_offset(chunk); | 743 | bit = file_page_offset(chunk); |
753 | 744 | ||
754 | |||
755 | /* make sure the page stays cached until it gets written out */ | ||
756 | if (! (get_page_attr(bitmap, page) & BITMAP_PAGE_DIRTY)) | ||
757 | get_page(page); | ||
758 | |||
759 | /* set the bit */ | 745 | /* set the bit */ |
760 | kaddr = kmap_atomic(page, KM_USER0); | 746 | kaddr = kmap_atomic(page, KM_USER0); |
761 | if (bitmap->flags & BITMAP_HOSTENDIAN) | 747 | if (bitmap->flags & BITMAP_HOSTENDIAN) |
@@ -775,7 +761,8 @@ static void bitmap_file_set_bit(struct bitmap *bitmap, sector_t block) | |||
775 | * sync the dirty pages of the bitmap file to disk */ | 761 | * sync the dirty pages of the bitmap file to disk */ |
776 | int bitmap_unplug(struct bitmap *bitmap) | 762 | int bitmap_unplug(struct bitmap *bitmap) |
777 | { | 763 | { |
778 | unsigned long i, attr, flags; | 764 | unsigned long i, flags; |
765 | int dirty, need_write; | ||
779 | struct page *page; | 766 | struct page *page; |
780 | int wait = 0; | 767 | int wait = 0; |
781 | int err; | 768 | int err; |
@@ -792,35 +779,26 @@ int bitmap_unplug(struct bitmap *bitmap) | |||
792 | return 0; | 779 | return 0; |
793 | } | 780 | } |
794 | page = bitmap->filemap[i]; | 781 | page = bitmap->filemap[i]; |
795 | attr = get_page_attr(bitmap, page); | 782 | dirty = test_page_attr(bitmap, page, BITMAP_PAGE_DIRTY); |
783 | need_write = test_page_attr(bitmap, page, BITMAP_PAGE_NEEDWRITE); | ||
796 | clear_page_attr(bitmap, page, BITMAP_PAGE_DIRTY); | 784 | clear_page_attr(bitmap, page, BITMAP_PAGE_DIRTY); |
797 | clear_page_attr(bitmap, page, BITMAP_PAGE_NEEDWRITE); | 785 | clear_page_attr(bitmap, page, BITMAP_PAGE_NEEDWRITE); |
798 | if ((attr & BITMAP_PAGE_DIRTY)) | 786 | if (dirty) |
799 | wait = 1; | 787 | wait = 1; |
800 | spin_unlock_irqrestore(&bitmap->lock, flags); | 788 | spin_unlock_irqrestore(&bitmap->lock, flags); |
801 | 789 | ||
802 | if (attr & (BITMAP_PAGE_DIRTY | BITMAP_PAGE_NEEDWRITE)) { | 790 | if (dirty | need_write) |
803 | err = write_page(bitmap, page, 0); | 791 | err = write_page(bitmap, page, 0); |
804 | if (err == -EAGAIN) { | ||
805 | if (attr & BITMAP_PAGE_DIRTY) | ||
806 | err = write_page(bitmap, page, 1); | ||
807 | else | ||
808 | err = 0; | ||
809 | } | ||
810 | if (err) | ||
811 | return 1; | ||
812 | } | ||
813 | } | 792 | } |
814 | if (wait) { /* if any writes were performed, we need to wait on them */ | 793 | if (wait) { /* if any writes were performed, we need to wait on them */ |
815 | if (bitmap->file) { | 794 | if (bitmap->file) |
816 | spin_lock_irq(&bitmap->write_lock); | 795 | wait_event(bitmap->write_wait, |
817 | wait_event_lock_irq(bitmap->write_wait, | 796 | atomic_read(&bitmap->pending_writes)==0); |
818 | list_empty(&bitmap->complete_pages), bitmap->write_lock, | 797 | else |
819 | wake_up_process(bitmap->writeback_daemon->tsk)); | ||
820 | spin_unlock_irq(&bitmap->write_lock); | ||
821 | } else | ||
822 | md_super_wait(bitmap->mddev); | 798 | md_super_wait(bitmap->mddev); |
823 | } | 799 | } |
800 | if (bitmap->flags & BITMAP_WRITE_ERROR) | ||
801 | bitmap_file_kick(bitmap); | ||
824 | return 0; | 802 | return 0; |
825 | } | 803 | } |
826 | 804 | ||
@@ -842,7 +820,7 @@ static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start) | |||
842 | struct page *page = NULL, *oldpage = NULL; | 820 | struct page *page = NULL, *oldpage = NULL; |
843 | unsigned long num_pages, bit_cnt = 0; | 821 | unsigned long num_pages, bit_cnt = 0; |
844 | struct file *file; | 822 | struct file *file; |
845 | unsigned long bytes, offset, dummy; | 823 | unsigned long bytes, offset; |
846 | int outofdate; | 824 | int outofdate; |
847 | int ret = -ENOSPC; | 825 | int ret = -ENOSPC; |
848 | void *paddr; | 826 | void *paddr; |
@@ -879,7 +857,12 @@ static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start) | |||
879 | if (!bitmap->filemap) | 857 | if (!bitmap->filemap) |
880 | goto out; | 858 | goto out; |
881 | 859 | ||
882 | bitmap->filemap_attr = kzalloc(sizeof(long) * num_pages, GFP_KERNEL); | 860 | /* We need 4 bits per page, rounded up to a multiple of sizeof(unsigned long) */ |
861 | bitmap->filemap_attr = kzalloc( | ||
862 | (((num_pages*4/8)+sizeof(unsigned long)-1) | ||
863 | /sizeof(unsigned long)) | ||
864 | *sizeof(unsigned long), | ||
865 | GFP_KERNEL); | ||
883 | if (!bitmap->filemap_attr) | 866 | if (!bitmap->filemap_attr) |
884 | goto out; | 867 | goto out; |
885 | 868 | ||
@@ -890,7 +873,12 @@ static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start) | |||
890 | index = file_page_index(i); | 873 | index = file_page_index(i); |
891 | bit = file_page_offset(i); | 874 | bit = file_page_offset(i); |
892 | if (index != oldindex) { /* this is a new page, read it in */ | 875 | if (index != oldindex) { /* this is a new page, read it in */ |
876 | int count; | ||
893 | /* unmap the old page, we're done with it */ | 877 | /* unmap the old page, we're done with it */ |
878 | if (index == num_pages-1) | ||
879 | count = bytes - index * PAGE_SIZE; | ||
880 | else | ||
881 | count = PAGE_SIZE; | ||
894 | if (index == 0) { | 882 | if (index == 0) { |
895 | /* | 883 | /* |
896 | * if we're here then the superblock page | 884 | * if we're here then the superblock page |
@@ -900,7 +888,7 @@ static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start) | |||
900 | page = bitmap->sb_page; | 888 | page = bitmap->sb_page; |
901 | offset = sizeof(bitmap_super_t); | 889 | offset = sizeof(bitmap_super_t); |
902 | } else if (file) { | 890 | } else if (file) { |
903 | page = read_page(file, index, &dummy); | 891 | page = read_page(file, index, bitmap, count); |
904 | offset = 0; | 892 | offset = 0; |
905 | } else { | 893 | } else { |
906 | page = read_sb_page(bitmap->mddev, bitmap->offset, index); | 894 | page = read_sb_page(bitmap->mddev, bitmap->offset, index); |
@@ -971,12 +959,11 @@ void bitmap_write_all(struct bitmap *bitmap) | |||
971 | /* We don't actually write all bitmap blocks here, | 959 | /* We don't actually write all bitmap blocks here, |
972 | * just flag them as needing to be written | 960 | * just flag them as needing to be written |
973 | */ | 961 | */ |
962 | int i; | ||
974 | 963 | ||
975 | unsigned long chunks = bitmap->chunks; | 964 | for (i=0; i < bitmap->file_pages; i++) |
976 | unsigned long bytes = (chunks+7)/8 + sizeof(bitmap_super_t); | 965 | set_page_attr(bitmap, bitmap->filemap[i], |
977 | unsigned long num_pages = (bytes + PAGE_SIZE-1) / PAGE_SIZE; | 966 | BITMAP_PAGE_NEEDWRITE); |
978 | while (num_pages--) | ||
979 | bitmap->filemap_attr[num_pages] |= BITMAP_PAGE_NEEDWRITE; | ||
980 | } | 967 | } |
981 | 968 | ||
982 | 969 | ||
@@ -1007,7 +994,6 @@ int bitmap_daemon_work(struct bitmap *bitmap) | |||
1007 | struct page *page = NULL, *lastpage = NULL; | 994 | struct page *page = NULL, *lastpage = NULL; |
1008 | int err = 0; | 995 | int err = 0; |
1009 | int blocks; | 996 | int blocks; |
1010 | int attr; | ||
1011 | void *paddr; | 997 | void *paddr; |
1012 | 998 | ||
1013 | if (bitmap == NULL) | 999 | if (bitmap == NULL) |
@@ -1029,43 +1015,34 @@ int bitmap_daemon_work(struct bitmap *bitmap) | |||
1029 | 1015 | ||
1030 | if (page != lastpage) { | 1016 | if (page != lastpage) { |
1031 | /* skip this page unless it's marked as needing cleaning */ | 1017 | /* skip this page unless it's marked as needing cleaning */ |
1032 | if (!((attr=get_page_attr(bitmap, page)) & BITMAP_PAGE_CLEAN)) { | 1018 | if (!test_page_attr(bitmap, page, BITMAP_PAGE_CLEAN)) { |
1033 | if (attr & BITMAP_PAGE_NEEDWRITE) { | 1019 | int need_write = test_page_attr(bitmap, page, |
1034 | get_page(page); | 1020 | BITMAP_PAGE_NEEDWRITE); |
1021 | if (need_write) | ||
1035 | clear_page_attr(bitmap, page, BITMAP_PAGE_NEEDWRITE); | 1022 | clear_page_attr(bitmap, page, BITMAP_PAGE_NEEDWRITE); |
1036 | } | 1023 | |
1037 | spin_unlock_irqrestore(&bitmap->lock, flags); | 1024 | spin_unlock_irqrestore(&bitmap->lock, flags); |
1038 | if (attr & BITMAP_PAGE_NEEDWRITE) { | 1025 | if (need_write) { |
1039 | switch (write_page(bitmap, page, 0)) { | 1026 | switch (write_page(bitmap, page, 0)) { |
1040 | case -EAGAIN: | ||
1041 | set_page_attr(bitmap, page, BITMAP_PAGE_NEEDWRITE); | ||
1042 | break; | ||
1043 | case 0: | 1027 | case 0: |
1044 | break; | 1028 | break; |
1045 | default: | 1029 | default: |
1046 | bitmap_file_kick(bitmap); | 1030 | bitmap_file_kick(bitmap); |
1047 | } | 1031 | } |
1048 | put_page(page); | ||
1049 | } | 1032 | } |
1050 | continue; | 1033 | continue; |
1051 | } | 1034 | } |
1052 | 1035 | ||
1053 | /* grab the new page, sync and release the old */ | 1036 | /* grab the new page, sync and release the old */ |
1054 | get_page(page); | ||
1055 | if (lastpage != NULL) { | 1037 | if (lastpage != NULL) { |
1056 | if (get_page_attr(bitmap, lastpage) & BITMAP_PAGE_NEEDWRITE) { | 1038 | if (test_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE)) { |
1057 | clear_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE); | 1039 | clear_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE); |
1058 | spin_unlock_irqrestore(&bitmap->lock, flags); | 1040 | spin_unlock_irqrestore(&bitmap->lock, flags); |
1059 | err = write_page(bitmap, lastpage, 0); | 1041 | err = write_page(bitmap, lastpage, 0); |
1060 | if (err == -EAGAIN) { | ||
1061 | err = 0; | ||
1062 | set_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE); | ||
1063 | } | ||
1064 | } else { | 1042 | } else { |
1065 | set_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE); | 1043 | set_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE); |
1066 | spin_unlock_irqrestore(&bitmap->lock, flags); | 1044 | spin_unlock_irqrestore(&bitmap->lock, flags); |
1067 | } | 1045 | } |
1068 | put_page(lastpage); | ||
1069 | if (err) | 1046 | if (err) |
1070 | bitmap_file_kick(bitmap); | 1047 | bitmap_file_kick(bitmap); |
1071 | } else | 1048 | } else |
@@ -1107,131 +1084,19 @@ int bitmap_daemon_work(struct bitmap *bitmap) | |||
1107 | /* now sync the final page */ | 1084 | /* now sync the final page */ |
1108 | if (lastpage != NULL) { | 1085 | if (lastpage != NULL) { |
1109 | spin_lock_irqsave(&bitmap->lock, flags); | 1086 | spin_lock_irqsave(&bitmap->lock, flags); |
1110 | if (get_page_attr(bitmap, lastpage) &BITMAP_PAGE_NEEDWRITE) { | 1087 | if (test_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE)) { |
1111 | clear_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE); | 1088 | clear_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE); |
1112 | spin_unlock_irqrestore(&bitmap->lock, flags); | 1089 | spin_unlock_irqrestore(&bitmap->lock, flags); |
1113 | err = write_page(bitmap, lastpage, 0); | 1090 | err = write_page(bitmap, lastpage, 0); |
1114 | if (err == -EAGAIN) { | ||
1115 | set_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE); | ||
1116 | err = 0; | ||
1117 | } | ||
1118 | } else { | 1091 | } else { |
1119 | set_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE); | 1092 | set_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE); |
1120 | spin_unlock_irqrestore(&bitmap->lock, flags); | 1093 | spin_unlock_irqrestore(&bitmap->lock, flags); |
1121 | } | 1094 | } |
1122 | |||
1123 | put_page(lastpage); | ||
1124 | } | 1095 | } |
1125 | 1096 | ||
1126 | return err; | 1097 | return err; |
1127 | } | 1098 | } |
1128 | 1099 | ||
1129 | static void daemon_exit(struct bitmap *bitmap, mdk_thread_t **daemon) | ||
1130 | { | ||
1131 | mdk_thread_t *dmn; | ||
1132 | unsigned long flags; | ||
1133 | |||
1134 | /* if no one is waiting on us, we'll free the md thread struct | ||
1135 | * and exit, otherwise we let the waiter clean things up */ | ||
1136 | spin_lock_irqsave(&bitmap->lock, flags); | ||
1137 | if ((dmn = *daemon)) { /* no one is waiting, cleanup and exit */ | ||
1138 | *daemon = NULL; | ||
1139 | spin_unlock_irqrestore(&bitmap->lock, flags); | ||
1140 | kfree(dmn); | ||
1141 | complete_and_exit(NULL, 0); /* do_exit not exported */ | ||
1142 | } | ||
1143 | spin_unlock_irqrestore(&bitmap->lock, flags); | ||
1144 | } | ||
1145 | |||
1146 | static void bitmap_writeback_daemon(mddev_t *mddev) | ||
1147 | { | ||
1148 | struct bitmap *bitmap = mddev->bitmap; | ||
1149 | struct page *page; | ||
1150 | struct page_list *item; | ||
1151 | int err = 0; | ||
1152 | |||
1153 | if (signal_pending(current)) { | ||
1154 | printk(KERN_INFO | ||
1155 | "%s: bitmap writeback daemon got signal, exiting...\n", | ||
1156 | bmname(bitmap)); | ||
1157 | err = -EINTR; | ||
1158 | goto out; | ||
1159 | } | ||
1160 | if (bitmap == NULL) | ||
1161 | /* about to be stopped. */ | ||
1162 | return; | ||
1163 | |||
1164 | PRINTK("%s: bitmap writeback daemon woke up...\n", bmname(bitmap)); | ||
1165 | /* wait on bitmap page writebacks */ | ||
1166 | while ((item = dequeue_page(bitmap))) { | ||
1167 | page = item->page; | ||
1168 | mempool_free(item, bitmap->write_pool); | ||
1169 | PRINTK("wait on page writeback: %p\n", page); | ||
1170 | wait_on_page_writeback(page); | ||
1171 | PRINTK("finished page writeback: %p\n", page); | ||
1172 | |||
1173 | err = PageError(page); | ||
1174 | put_page(page); | ||
1175 | if (err) { | ||
1176 | printk(KERN_WARNING "%s: bitmap file writeback " | ||
1177 | "failed (page %lu): %d\n", | ||
1178 | bmname(bitmap), page->index, err); | ||
1179 | bitmap_file_kick(bitmap); | ||
1180 | goto out; | ||
1181 | } | ||
1182 | } | ||
1183 | out: | ||
1184 | wake_up(&bitmap->write_wait); | ||
1185 | if (err) { | ||
1186 | printk(KERN_INFO "%s: bitmap writeback daemon exiting (%d)\n", | ||
1187 | bmname(bitmap), err); | ||
1188 | daemon_exit(bitmap, &bitmap->writeback_daemon); | ||
1189 | } | ||
1190 | } | ||
1191 | |||
1192 | static mdk_thread_t *bitmap_start_daemon(struct bitmap *bitmap, | ||
1193 | void (*func)(mddev_t *), char *name) | ||
1194 | { | ||
1195 | mdk_thread_t *daemon; | ||
1196 | char namebuf[32]; | ||
1197 | |||
1198 | #ifdef INJECT_FATAL_FAULT_2 | ||
1199 | daemon = NULL; | ||
1200 | #else | ||
1201 | sprintf(namebuf, "%%s_%s", name); | ||
1202 | daemon = md_register_thread(func, bitmap->mddev, namebuf); | ||
1203 | #endif | ||
1204 | if (!daemon) { | ||
1205 | printk(KERN_ERR "%s: failed to start bitmap daemon\n", | ||
1206 | bmname(bitmap)); | ||
1207 | return ERR_PTR(-ECHILD); | ||
1208 | } | ||
1209 | |||
1210 | md_wakeup_thread(daemon); /* start it running */ | ||
1211 | |||
1212 | PRINTK("%s: %s daemon (pid %d) started...\n", | ||
1213 | bmname(bitmap), name, daemon->tsk->pid); | ||
1214 | |||
1215 | return daemon; | ||
1216 | } | ||
1217 | |||
1218 | static void bitmap_stop_daemon(struct bitmap *bitmap) | ||
1219 | { | ||
1220 | /* the daemon can't stop itself... it'll just exit instead... */ | ||
1221 | if (bitmap->writeback_daemon && ! IS_ERR(bitmap->writeback_daemon) && | ||
1222 | current->pid != bitmap->writeback_daemon->tsk->pid) { | ||
1223 | mdk_thread_t *daemon; | ||
1224 | unsigned long flags; | ||
1225 | |||
1226 | spin_lock_irqsave(&bitmap->lock, flags); | ||
1227 | daemon = bitmap->writeback_daemon; | ||
1228 | bitmap->writeback_daemon = NULL; | ||
1229 | spin_unlock_irqrestore(&bitmap->lock, flags); | ||
1230 | if (daemon && ! IS_ERR(daemon)) | ||
1231 | md_unregister_thread(daemon); /* destroy the thread */ | ||
1232 | } | ||
1233 | } | ||
1234 | |||
1235 | static bitmap_counter_t *bitmap_get_counter(struct bitmap *bitmap, | 1100 | static bitmap_counter_t *bitmap_get_counter(struct bitmap *bitmap, |
1236 | sector_t offset, int *blocks, | 1101 | sector_t offset, int *blocks, |
1237 | int create) | 1102 | int create) |
@@ -1500,8 +1365,6 @@ static void bitmap_free(struct bitmap *bitmap) | |||
1500 | 1365 | ||
1501 | /* free all allocated memory */ | 1366 | /* free all allocated memory */ |
1502 | 1367 | ||
1503 | mempool_destroy(bitmap->write_pool); | ||
1504 | |||
1505 | if (bp) /* deallocate the page memory */ | 1368 | if (bp) /* deallocate the page memory */ |
1506 | for (k = 0; k < pages; k++) | 1369 | for (k = 0; k < pages; k++) |
1507 | if (bp[k].map && !bp[k].hijacked) | 1370 | if (bp[k].map && !bp[k].hijacked) |
@@ -1549,20 +1412,20 @@ int bitmap_create(mddev_t *mddev) | |||
1549 | return -ENOMEM; | 1412 | return -ENOMEM; |
1550 | 1413 | ||
1551 | spin_lock_init(&bitmap->lock); | 1414 | spin_lock_init(&bitmap->lock); |
1552 | bitmap->mddev = mddev; | 1415 | atomic_set(&bitmap->pending_writes, 0); |
1553 | |||
1554 | spin_lock_init(&bitmap->write_lock); | ||
1555 | INIT_LIST_HEAD(&bitmap->complete_pages); | ||
1556 | init_waitqueue_head(&bitmap->write_wait); | 1416 | init_waitqueue_head(&bitmap->write_wait); |
1557 | bitmap->write_pool = mempool_create_kmalloc_pool(WRITE_POOL_SIZE, | 1417 | |
1558 | sizeof(struct page_list)); | 1418 | bitmap->mddev = mddev; |
1559 | err = -ENOMEM; | ||
1560 | if (!bitmap->write_pool) | ||
1561 | goto error; | ||
1562 | 1419 | ||
1563 | bitmap->file = file; | 1420 | bitmap->file = file; |
1564 | bitmap->offset = mddev->bitmap_offset; | 1421 | bitmap->offset = mddev->bitmap_offset; |
1565 | if (file) get_file(file); | 1422 | if (file) { |
1423 | get_file(file); | ||
1424 | do_sync_file_range(file, 0, LLONG_MAX, | ||
1425 | SYNC_FILE_RANGE_WAIT_BEFORE | | ||
1426 | SYNC_FILE_RANGE_WRITE | | ||
1427 | SYNC_FILE_RANGE_WAIT_AFTER); | ||
1428 | } | ||
1566 | /* read superblock from bitmap file (this sets bitmap->chunksize) */ | 1429 | /* read superblock from bitmap file (this sets bitmap->chunksize) */ |
1567 | err = bitmap_read_sb(bitmap); | 1430 | err = bitmap_read_sb(bitmap); |
1568 | if (err) | 1431 | if (err) |
@@ -1594,8 +1457,6 @@ int bitmap_create(mddev_t *mddev) | |||
1594 | if (!bitmap->bp) | 1457 | if (!bitmap->bp) |
1595 | goto error; | 1458 | goto error; |
1596 | 1459 | ||
1597 | bitmap->flags |= BITMAP_ACTIVE; | ||
1598 | |||
1599 | /* now that we have some pages available, initialize the in-memory | 1460 | /* now that we have some pages available, initialize the in-memory |
1600 | * bitmap from the on-disk bitmap */ | 1461 | * bitmap from the on-disk bitmap */ |
1601 | start = 0; | 1462 | start = 0; |
@@ -1613,15 +1474,6 @@ int bitmap_create(mddev_t *mddev) | |||
1613 | 1474 | ||
1614 | mddev->bitmap = bitmap; | 1475 | mddev->bitmap = bitmap; |
1615 | 1476 | ||
1616 | if (file) | ||
1617 | /* kick off the bitmap writeback daemon */ | ||
1618 | bitmap->writeback_daemon = | ||
1619 | bitmap_start_daemon(bitmap, | ||
1620 | bitmap_writeback_daemon, | ||
1621 | "bitmap_wb"); | ||
1622 | |||
1623 | if (IS_ERR(bitmap->writeback_daemon)) | ||
1624 | return PTR_ERR(bitmap->writeback_daemon); | ||
1625 | mddev->thread->timeout = bitmap->daemon_sleep * HZ; | 1477 | mddev->thread->timeout = bitmap->daemon_sleep * HZ; |
1626 | 1478 | ||
1627 | return bitmap_update_sb(bitmap); | 1479 | return bitmap_update_sb(bitmap); |
@@ -1638,4 +1490,3 @@ EXPORT_SYMBOL(bitmap_start_sync); | |||
1638 | EXPORT_SYMBOL(bitmap_end_sync); | 1490 | EXPORT_SYMBOL(bitmap_end_sync); |
1639 | EXPORT_SYMBOL(bitmap_unplug); | 1491 | EXPORT_SYMBOL(bitmap_unplug); |
1640 | EXPORT_SYMBOL(bitmap_close_sync); | 1492 | EXPORT_SYMBOL(bitmap_close_sync); |
1641 | EXPORT_SYMBOL(bitmap_daemon_work); | ||