aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/md/bitmap.c
diff options
context:
space:
mode:
authorMichal Marek <mmarek@suse.cz>2011-03-09 10:15:44 -0500
committerMichal Marek <mmarek@suse.cz>2011-03-09 10:15:44 -0500
commit2d8ad8719591fa803b0d589ed057fa46f49b7155 (patch)
tree4ae051577dad1161c91dafbf4207bb10a9dc91bb /drivers/md/bitmap.c
parent9b4ce7bce5f30712fd926ab4599a803314a07719 (diff)
parentc56eb8fb6dccb83d9fe62fd4dc00c834de9bc470 (diff)
Merge commit 'v2.6.38-rc1' into kbuild/packaging
Diffstat (limited to 'drivers/md/bitmap.c')
-rw-r--r--drivers/md/bitmap.c597
1 files changed, 340 insertions, 257 deletions
diff --git a/drivers/md/bitmap.c b/drivers/md/bitmap.c
index 26ac8aad0b19..9a35320fb59f 100644
--- a/drivers/md/bitmap.c
+++ b/drivers/md/bitmap.c
@@ -13,7 +13,6 @@
13 * Still to do: 13 * Still to do:
14 * 14 *
15 * flush after percent set rather than just time based. (maybe both). 15 * flush after percent set rather than just time based. (maybe both).
16 * wait if count gets too high, wake when it drops to half.
17 */ 16 */
18 17
19#include <linux/blkdev.h> 18#include <linux/blkdev.h>
@@ -30,6 +29,7 @@
30#include "md.h" 29#include "md.h"
31#include "bitmap.h" 30#include "bitmap.h"
32 31
32#include <linux/dm-dirty-log.h>
33/* debug macros */ 33/* debug macros */
34 34
35#define DEBUG 0 35#define DEBUG 0
@@ -51,9 +51,6 @@
51#define INJECT_FATAL_FAULT_3 0 /* undef */ 51#define INJECT_FATAL_FAULT_3 0 /* undef */
52#endif 52#endif
53 53
54//#define DPRINTK PRINTK /* set this NULL to avoid verbose debug output */
55#define DPRINTK(x...) do { } while(0)
56
57#ifndef PRINTK 54#ifndef PRINTK
58# if DEBUG > 0 55# if DEBUG > 0
59# define PRINTK(x...) printk(KERN_DEBUG x) 56# define PRINTK(x...) printk(KERN_DEBUG x)
@@ -62,12 +59,11 @@
62# endif 59# endif
63#endif 60#endif
64 61
65static inline char * bmname(struct bitmap *bitmap) 62static inline char *bmname(struct bitmap *bitmap)
66{ 63{
67 return bitmap->mddev ? mdname(bitmap->mddev) : "mdX"; 64 return bitmap->mddev ? mdname(bitmap->mddev) : "mdX";
68} 65}
69 66
70
71/* 67/*
72 * just a placeholder - calls kmalloc for bitmap pages 68 * just a placeholder - calls kmalloc for bitmap pages
73 */ 69 */
@@ -78,7 +74,7 @@ static unsigned char *bitmap_alloc_page(struct bitmap *bitmap)
78#ifdef INJECT_FAULTS_1 74#ifdef INJECT_FAULTS_1
79 page = NULL; 75 page = NULL;
80#else 76#else
81 page = kmalloc(PAGE_SIZE, GFP_NOIO); 77 page = kzalloc(PAGE_SIZE, GFP_NOIO);
82#endif 78#endif
83 if (!page) 79 if (!page)
84 printk("%s: bitmap_alloc_page FAILED\n", bmname(bitmap)); 80 printk("%s: bitmap_alloc_page FAILED\n", bmname(bitmap));
@@ -107,7 +103,8 @@ static void bitmap_free_page(struct bitmap *bitmap, unsigned char *page)
107 * if we find our page, we increment the page's refcount so that it stays 103 * if we find our page, we increment the page's refcount so that it stays
108 * allocated while we're using it 104 * allocated while we're using it
109 */ 105 */
110static int bitmap_checkpage(struct bitmap *bitmap, unsigned long page, int create) 106static int bitmap_checkpage(struct bitmap *bitmap,
107 unsigned long page, int create)
111__releases(bitmap->lock) 108__releases(bitmap->lock)
112__acquires(bitmap->lock) 109__acquires(bitmap->lock)
113{ 110{
@@ -121,7 +118,6 @@ __acquires(bitmap->lock)
121 return -EINVAL; 118 return -EINVAL;
122 } 119 }
123 120
124
125 if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */ 121 if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */
126 return 0; 122 return 0;
127 123
@@ -131,43 +127,34 @@ __acquires(bitmap->lock)
131 if (!create) 127 if (!create)
132 return -ENOENT; 128 return -ENOENT;
133 129
134 spin_unlock_irq(&bitmap->lock);
135
136 /* this page has not been allocated yet */ 130 /* this page has not been allocated yet */
137 131
138 if ((mappage = bitmap_alloc_page(bitmap)) == NULL) { 132 spin_unlock_irq(&bitmap->lock);
133 mappage = bitmap_alloc_page(bitmap);
134 spin_lock_irq(&bitmap->lock);
135
136 if (mappage == NULL) {
139 PRINTK("%s: bitmap map page allocation failed, hijacking\n", 137 PRINTK("%s: bitmap map page allocation failed, hijacking\n",
140 bmname(bitmap)); 138 bmname(bitmap));
141 /* failed - set the hijacked flag so that we can use the 139 /* failed - set the hijacked flag so that we can use the
142 * pointer as a counter */ 140 * pointer as a counter */
143 spin_lock_irq(&bitmap->lock);
144 if (!bitmap->bp[page].map) 141 if (!bitmap->bp[page].map)
145 bitmap->bp[page].hijacked = 1; 142 bitmap->bp[page].hijacked = 1;
146 goto out; 143 } else if (bitmap->bp[page].map ||
147 } 144 bitmap->bp[page].hijacked) {
148
149 /* got a page */
150
151 spin_lock_irq(&bitmap->lock);
152
153 /* recheck the page */
154
155 if (bitmap->bp[page].map || bitmap->bp[page].hijacked) {
156 /* somebody beat us to getting the page */ 145 /* somebody beat us to getting the page */
157 bitmap_free_page(bitmap, mappage); 146 bitmap_free_page(bitmap, mappage);
158 return 0; 147 return 0;
159 } 148 } else {
160 149
161 /* no page was in place and we have one, so install it */ 150 /* no page was in place and we have one, so install it */
162 151
163 memset(mappage, 0, PAGE_SIZE); 152 bitmap->bp[page].map = mappage;
164 bitmap->bp[page].map = mappage; 153 bitmap->missing_pages--;
165 bitmap->missing_pages--; 154 }
166out:
167 return 0; 155 return 0;
168} 156}
169 157
170
171/* if page is completely empty, put it back on the free list, or dealloc it */ 158/* if page is completely empty, put it back on the free list, or dealloc it */
172/* if page was hijacked, unmark the flag so it might get alloced next time */ 159/* if page was hijacked, unmark the flag so it might get alloced next time */
173/* Note: lock should be held when calling this */ 160/* Note: lock should be held when calling this */
@@ -183,26 +170,15 @@ static void bitmap_checkfree(struct bitmap *bitmap, unsigned long page)
183 if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */ 170 if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */
184 bitmap->bp[page].hijacked = 0; 171 bitmap->bp[page].hijacked = 0;
185 bitmap->bp[page].map = NULL; 172 bitmap->bp[page].map = NULL;
186 return; 173 } else {
174 /* normal case, free the page */
175 ptr = bitmap->bp[page].map;
176 bitmap->bp[page].map = NULL;
177 bitmap->missing_pages++;
178 bitmap_free_page(bitmap, ptr);
187 } 179 }
188
189 /* normal case, free the page */
190
191#if 0
192/* actually ... let's not. We will probably need the page again exactly when
193 * memory is tight and we are flusing to disk
194 */
195 return;
196#else
197 ptr = bitmap->bp[page].map;
198 bitmap->bp[page].map = NULL;
199 bitmap->missing_pages++;
200 bitmap_free_page(bitmap, ptr);
201 return;
202#endif
203} 180}
204 181
205
206/* 182/*
207 * bitmap file handling - read and write the bitmap file and its superblock 183 * bitmap file handling - read and write the bitmap file and its superblock
208 */ 184 */
@@ -220,28 +196,33 @@ static struct page *read_sb_page(mddev_t *mddev, loff_t offset,
220 196
221 mdk_rdev_t *rdev; 197 mdk_rdev_t *rdev;
222 sector_t target; 198 sector_t target;
199 int did_alloc = 0;
223 200
224 if (!page) 201 if (!page) {
225 page = alloc_page(GFP_KERNEL); 202 page = alloc_page(GFP_KERNEL);
226 if (!page) 203 if (!page)
227 return ERR_PTR(-ENOMEM); 204 return ERR_PTR(-ENOMEM);
205 did_alloc = 1;
206 }
228 207
229 list_for_each_entry(rdev, &mddev->disks, same_set) { 208 list_for_each_entry(rdev, &mddev->disks, same_set) {
230 if (! test_bit(In_sync, &rdev->flags) 209 if (! test_bit(In_sync, &rdev->flags)
231 || test_bit(Faulty, &rdev->flags)) 210 || test_bit(Faulty, &rdev->flags))
232 continue; 211 continue;
233 212
234 target = rdev->sb_start + offset + index * (PAGE_SIZE/512); 213 target = offset + index * (PAGE_SIZE/512);
235 214
236 if (sync_page_io(rdev->bdev, target, 215 if (sync_page_io(rdev, target,
237 roundup(size, bdev_logical_block_size(rdev->bdev)), 216 roundup(size, bdev_logical_block_size(rdev->bdev)),
238 page, READ)) { 217 page, READ, true)) {
239 page->index = index; 218 page->index = index;
240 attach_page_buffers(page, NULL); /* so that free_buffer will 219 attach_page_buffers(page, NULL); /* so that free_buffer will
241 * quietly no-op */ 220 * quietly no-op */
242 return page; 221 return page;
243 } 222 }
244 } 223 }
224 if (did_alloc)
225 put_page(page);
245 return ERR_PTR(-EIO); 226 return ERR_PTR(-EIO);
246 227
247} 228}
@@ -283,52 +264,58 @@ static mdk_rdev_t *next_active_rdev(mdk_rdev_t *rdev, mddev_t *mddev)
283static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait) 264static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait)
284{ 265{
285 mdk_rdev_t *rdev = NULL; 266 mdk_rdev_t *rdev = NULL;
267 struct block_device *bdev;
286 mddev_t *mddev = bitmap->mddev; 268 mddev_t *mddev = bitmap->mddev;
287 269
288 while ((rdev = next_active_rdev(rdev, mddev)) != NULL) { 270 while ((rdev = next_active_rdev(rdev, mddev)) != NULL) {
289 int size = PAGE_SIZE; 271 int size = PAGE_SIZE;
290 loff_t offset = mddev->bitmap_info.offset; 272 loff_t offset = mddev->bitmap_info.offset;
291 if (page->index == bitmap->file_pages-1) 273
292 size = roundup(bitmap->last_page_size, 274 bdev = (rdev->meta_bdev) ? rdev->meta_bdev : rdev->bdev;
293 bdev_logical_block_size(rdev->bdev)); 275
294 /* Just make sure we aren't corrupting data or 276 if (page->index == bitmap->file_pages-1)
295 * metadata 277 size = roundup(bitmap->last_page_size,
296 */ 278 bdev_logical_block_size(bdev));
297 if (mddev->external) { 279 /* Just make sure we aren't corrupting data or
298 /* Bitmap could be anywhere. */ 280 * metadata
299 if (rdev->sb_start + offset + (page->index *(PAGE_SIZE/512)) > 281 */
300 rdev->data_offset && 282 if (mddev->external) {
301 rdev->sb_start + offset < 283 /* Bitmap could be anywhere. */
302 rdev->data_offset + mddev->dev_sectors + 284 if (rdev->sb_start + offset + (page->index
303 (PAGE_SIZE/512)) 285 * (PAGE_SIZE/512))
304 goto bad_alignment; 286 > rdev->data_offset
305 } else if (offset < 0) { 287 &&
306 /* DATA BITMAP METADATA */ 288 rdev->sb_start + offset
307 if (offset 289 < (rdev->data_offset + mddev->dev_sectors
308 + (long)(page->index * (PAGE_SIZE/512)) 290 + (PAGE_SIZE/512)))
309 + size/512 > 0) 291 goto bad_alignment;
310 /* bitmap runs in to metadata */ 292 } else if (offset < 0) {
311 goto bad_alignment; 293 /* DATA BITMAP METADATA */
312 if (rdev->data_offset + mddev->dev_sectors 294 if (offset
313 > rdev->sb_start + offset) 295 + (long)(page->index * (PAGE_SIZE/512))
314 /* data runs in to bitmap */ 296 + size/512 > 0)
315 goto bad_alignment; 297 /* bitmap runs in to metadata */
316 } else if (rdev->sb_start < rdev->data_offset) { 298 goto bad_alignment;
317 /* METADATA BITMAP DATA */ 299 if (rdev->data_offset + mddev->dev_sectors
318 if (rdev->sb_start 300 > rdev->sb_start + offset)
319 + offset 301 /* data runs in to bitmap */
320 + page->index*(PAGE_SIZE/512) + size/512 302 goto bad_alignment;
321 > rdev->data_offset) 303 } else if (rdev->sb_start < rdev->data_offset) {
322 /* bitmap runs in to data */ 304 /* METADATA BITMAP DATA */
323 goto bad_alignment; 305 if (rdev->sb_start
324 } else { 306 + offset
325 /* DATA METADATA BITMAP - no problems */ 307 + page->index*(PAGE_SIZE/512) + size/512
326 } 308 > rdev->data_offset)
327 md_super_write(mddev, rdev, 309 /* bitmap runs in to data */
328 rdev->sb_start + offset 310 goto bad_alignment;
329 + page->index * (PAGE_SIZE/512), 311 } else {
330 size, 312 /* DATA METADATA BITMAP - no problems */
331 page); 313 }
314 md_super_write(mddev, rdev,
315 rdev->sb_start + offset
316 + page->index * (PAGE_SIZE/512),
317 size,
318 page);
332 } 319 }
333 320
334 if (wait) 321 if (wait)
@@ -360,14 +347,13 @@ static void write_page(struct bitmap *bitmap, struct page *page, int wait)
360 atomic_inc(&bitmap->pending_writes); 347 atomic_inc(&bitmap->pending_writes);
361 set_buffer_locked(bh); 348 set_buffer_locked(bh);
362 set_buffer_mapped(bh); 349 set_buffer_mapped(bh);
363 submit_bh(WRITE, bh); 350 submit_bh(WRITE | REQ_UNPLUG | REQ_SYNC, bh);
364 bh = bh->b_this_page; 351 bh = bh->b_this_page;
365 } 352 }
366 353
367 if (wait) { 354 if (wait)
368 wait_event(bitmap->write_wait, 355 wait_event(bitmap->write_wait,
369 atomic_read(&bitmap->pending_writes)==0); 356 atomic_read(&bitmap->pending_writes)==0);
370 }
371 } 357 }
372 if (bitmap->flags & BITMAP_WRITE_ERROR) 358 if (bitmap->flags & BITMAP_WRITE_ERROR)
373 bitmap_file_kick(bitmap); 359 bitmap_file_kick(bitmap);
@@ -424,7 +410,7 @@ static struct page *read_page(struct file *file, unsigned long index,
424 struct buffer_head *bh; 410 struct buffer_head *bh;
425 sector_t block; 411 sector_t block;
426 412
427 PRINTK("read bitmap file (%dB @ %Lu)\n", (int)PAGE_SIZE, 413 PRINTK("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE,
428 (unsigned long long)index << PAGE_SHIFT); 414 (unsigned long long)index << PAGE_SHIFT);
429 415
430 page = alloc_page(GFP_KERNEL); 416 page = alloc_page(GFP_KERNEL);
@@ -478,7 +464,7 @@ static struct page *read_page(struct file *file, unsigned long index,
478 } 464 }
479out: 465out:
480 if (IS_ERR(page)) 466 if (IS_ERR(page))
481 printk(KERN_ALERT "md: bitmap read error: (%dB @ %Lu): %ld\n", 467 printk(KERN_ALERT "md: bitmap read error: (%dB @ %llu): %ld\n",
482 (int)PAGE_SIZE, 468 (int)PAGE_SIZE,
483 (unsigned long long)index << PAGE_SHIFT, 469 (unsigned long long)index << PAGE_SHIFT,
484 PTR_ERR(page)); 470 PTR_ERR(page));
@@ -505,7 +491,7 @@ void bitmap_update_sb(struct bitmap *bitmap)
505 return; 491 return;
506 } 492 }
507 spin_unlock_irqrestore(&bitmap->lock, flags); 493 spin_unlock_irqrestore(&bitmap->lock, flags);
508 sb = (bitmap_super_t *)kmap_atomic(bitmap->sb_page, KM_USER0); 494 sb = kmap_atomic(bitmap->sb_page, KM_USER0);
509 sb->events = cpu_to_le64(bitmap->mddev->events); 495 sb->events = cpu_to_le64(bitmap->mddev->events);
510 if (bitmap->mddev->events < bitmap->events_cleared) { 496 if (bitmap->mddev->events < bitmap->events_cleared) {
511 /* rocking back to read-only */ 497 /* rocking back to read-only */
@@ -526,7 +512,7 @@ void bitmap_print_sb(struct bitmap *bitmap)
526 512
527 if (!bitmap || !bitmap->sb_page) 513 if (!bitmap || !bitmap->sb_page)
528 return; 514 return;
529 sb = (bitmap_super_t *)kmap_atomic(bitmap->sb_page, KM_USER0); 515 sb = kmap_atomic(bitmap->sb_page, KM_USER0);
530 printk(KERN_DEBUG "%s: bitmap file superblock:\n", bmname(bitmap)); 516 printk(KERN_DEBUG "%s: bitmap file superblock:\n", bmname(bitmap));
531 printk(KERN_DEBUG " magic: %08x\n", le32_to_cpu(sb->magic)); 517 printk(KERN_DEBUG " magic: %08x\n", le32_to_cpu(sb->magic));
532 printk(KERN_DEBUG " version: %d\n", le32_to_cpu(sb->version)); 518 printk(KERN_DEBUG " version: %d\n", le32_to_cpu(sb->version));
@@ -575,7 +561,7 @@ static int bitmap_read_sb(struct bitmap *bitmap)
575 return err; 561 return err;
576 } 562 }
577 563
578 sb = (bitmap_super_t *)kmap_atomic(bitmap->sb_page, KM_USER0); 564 sb = kmap_atomic(bitmap->sb_page, KM_USER0);
579 565
580 chunksize = le32_to_cpu(sb->chunksize); 566 chunksize = le32_to_cpu(sb->chunksize);
581 daemon_sleep = le32_to_cpu(sb->daemon_sleep) * HZ; 567 daemon_sleep = le32_to_cpu(sb->daemon_sleep) * HZ;
@@ -661,14 +647,17 @@ static int bitmap_mask_state(struct bitmap *bitmap, enum bitmap_state bits,
661 return 0; 647 return 0;
662 } 648 }
663 spin_unlock_irqrestore(&bitmap->lock, flags); 649 spin_unlock_irqrestore(&bitmap->lock, flags);
664 sb = (bitmap_super_t *)kmap_atomic(bitmap->sb_page, KM_USER0); 650 sb = kmap_atomic(bitmap->sb_page, KM_USER0);
665 old = le32_to_cpu(sb->state) & bits; 651 old = le32_to_cpu(sb->state) & bits;
666 switch (op) { 652 switch (op) {
667 case MASK_SET: sb->state |= cpu_to_le32(bits); 653 case MASK_SET:
668 break; 654 sb->state |= cpu_to_le32(bits);
669 case MASK_UNSET: sb->state &= cpu_to_le32(~bits); 655 break;
670 break; 656 case MASK_UNSET:
671 default: BUG(); 657 sb->state &= cpu_to_le32(~bits);
658 break;
659 default:
660 BUG();
672 } 661 }
673 kunmap_atomic(sb, KM_USER0); 662 kunmap_atomic(sb, KM_USER0);
674 return old; 663 return old;
@@ -710,12 +699,14 @@ static inline unsigned long file_page_offset(struct bitmap *bitmap, unsigned lon
710static inline struct page *filemap_get_page(struct bitmap *bitmap, 699static inline struct page *filemap_get_page(struct bitmap *bitmap,
711 unsigned long chunk) 700 unsigned long chunk)
712{ 701{
713 if (file_page_index(bitmap, chunk) >= bitmap->file_pages) return NULL; 702 if (bitmap->filemap == NULL)
703 return NULL;
704 if (file_page_index(bitmap, chunk) >= bitmap->file_pages)
705 return NULL;
714 return bitmap->filemap[file_page_index(bitmap, chunk) 706 return bitmap->filemap[file_page_index(bitmap, chunk)
715 - file_page_index(bitmap, 0)]; 707 - file_page_index(bitmap, 0)];
716} 708}
717 709
718
719static void bitmap_file_unmap(struct bitmap *bitmap) 710static void bitmap_file_unmap(struct bitmap *bitmap)
720{ 711{
721 struct page **map, *sb_page; 712 struct page **map, *sb_page;
@@ -766,7 +757,6 @@ static void bitmap_file_put(struct bitmap *bitmap)
766 } 757 }
767} 758}
768 759
769
770/* 760/*
771 * bitmap_file_kick - if an error occurs while manipulating the bitmap file 761 * bitmap_file_kick - if an error occurs while manipulating the bitmap file
772 * then it is no longer reliable, so we stop using it and we mark the file 762 * then it is no longer reliable, so we stop using it and we mark the file
@@ -785,7 +775,6 @@ static void bitmap_file_kick(struct bitmap *bitmap)
785 ptr = d_path(&bitmap->file->f_path, path, 775 ptr = d_path(&bitmap->file->f_path, path,
786 PAGE_SIZE); 776 PAGE_SIZE);
787 777
788
789 printk(KERN_ALERT 778 printk(KERN_ALERT
790 "%s: kicking failed bitmap file %s from array!\n", 779 "%s: kicking failed bitmap file %s from array!\n",
791 bmname(bitmap), IS_ERR(ptr) ? "" : ptr); 780 bmname(bitmap), IS_ERR(ptr) ? "" : ptr);
@@ -803,27 +792,36 @@ static void bitmap_file_kick(struct bitmap *bitmap)
803} 792}
804 793
805enum bitmap_page_attr { 794enum bitmap_page_attr {
806 BITMAP_PAGE_DIRTY = 0, // there are set bits that need to be synced 795 BITMAP_PAGE_DIRTY = 0, /* there are set bits that need to be synced */
807 BITMAP_PAGE_CLEAN = 1, // there are bits that might need to be cleared 796 BITMAP_PAGE_CLEAN = 1, /* there are bits that might need to be cleared */
808 BITMAP_PAGE_NEEDWRITE=2, // there are cleared bits that need to be synced 797 BITMAP_PAGE_NEEDWRITE = 2, /* there are cleared bits that need to be synced */
809}; 798};
810 799
811static inline void set_page_attr(struct bitmap *bitmap, struct page *page, 800static inline void set_page_attr(struct bitmap *bitmap, struct page *page,
812 enum bitmap_page_attr attr) 801 enum bitmap_page_attr attr)
813{ 802{
814 __set_bit((page->index<<2) + attr, bitmap->filemap_attr); 803 if (page)
804 __set_bit((page->index<<2) + attr, bitmap->filemap_attr);
805 else
806 __set_bit(attr, &bitmap->logattrs);
815} 807}
816 808
817static inline void clear_page_attr(struct bitmap *bitmap, struct page *page, 809static inline void clear_page_attr(struct bitmap *bitmap, struct page *page,
818 enum bitmap_page_attr attr) 810 enum bitmap_page_attr attr)
819{ 811{
820 __clear_bit((page->index<<2) + attr, bitmap->filemap_attr); 812 if (page)
813 __clear_bit((page->index<<2) + attr, bitmap->filemap_attr);
814 else
815 __clear_bit(attr, &bitmap->logattrs);
821} 816}
822 817
823static inline unsigned long test_page_attr(struct bitmap *bitmap, struct page *page, 818static inline unsigned long test_page_attr(struct bitmap *bitmap, struct page *page,
824 enum bitmap_page_attr attr) 819 enum bitmap_page_attr attr)
825{ 820{
826 return test_bit((page->index<<2) + attr, bitmap->filemap_attr); 821 if (page)
822 return test_bit((page->index<<2) + attr, bitmap->filemap_attr);
823 else
824 return test_bit(attr, &bitmap->logattrs);
827} 825}
828 826
829/* 827/*
@@ -836,30 +834,32 @@ static inline unsigned long test_page_attr(struct bitmap *bitmap, struct page *p
836static void bitmap_file_set_bit(struct bitmap *bitmap, sector_t block) 834static void bitmap_file_set_bit(struct bitmap *bitmap, sector_t block)
837{ 835{
838 unsigned long bit; 836 unsigned long bit;
839 struct page *page; 837 struct page *page = NULL;
840 void *kaddr; 838 void *kaddr;
841 unsigned long chunk = block >> CHUNK_BLOCK_SHIFT(bitmap); 839 unsigned long chunk = block >> CHUNK_BLOCK_SHIFT(bitmap);
842 840
843 if (!bitmap->filemap) { 841 if (!bitmap->filemap) {
844 return; 842 struct dm_dirty_log *log = bitmap->mddev->bitmap_info.log;
845 } 843 if (log)
846 844 log->type->mark_region(log, chunk);
847 page = filemap_get_page(bitmap, chunk); 845 } else {
848 if (!page) return;
849 bit = file_page_offset(bitmap, chunk);
850 846
851 /* set the bit */ 847 page = filemap_get_page(bitmap, chunk);
852 kaddr = kmap_atomic(page, KM_USER0); 848 if (!page)
853 if (bitmap->flags & BITMAP_HOSTENDIAN) 849 return;
854 set_bit(bit, kaddr); 850 bit = file_page_offset(bitmap, chunk);
855 else
856 ext2_set_bit(bit, kaddr);
857 kunmap_atomic(kaddr, KM_USER0);
858 PRINTK("set file bit %lu page %lu\n", bit, page->index);
859 851
852 /* set the bit */
853 kaddr = kmap_atomic(page, KM_USER0);
854 if (bitmap->flags & BITMAP_HOSTENDIAN)
855 set_bit(bit, kaddr);
856 else
857 ext2_set_bit(bit, kaddr);
858 kunmap_atomic(kaddr, KM_USER0);
859 PRINTK("set file bit %lu page %lu\n", bit, page->index);
860 }
860 /* record page number so it gets flushed to disk when unplug occurs */ 861 /* record page number so it gets flushed to disk when unplug occurs */
861 set_page_attr(bitmap, page, BITMAP_PAGE_DIRTY); 862 set_page_attr(bitmap, page, BITMAP_PAGE_DIRTY);
862
863} 863}
864 864
865/* this gets called when the md device is ready to unplug its underlying 865/* this gets called when the md device is ready to unplug its underlying
@@ -874,6 +874,16 @@ void bitmap_unplug(struct bitmap *bitmap)
874 874
875 if (!bitmap) 875 if (!bitmap)
876 return; 876 return;
877 if (!bitmap->filemap) {
878 /* Must be using a dirty_log */
879 struct dm_dirty_log *log = bitmap->mddev->bitmap_info.log;
880 dirty = test_and_clear_bit(BITMAP_PAGE_DIRTY, &bitmap->logattrs);
881 need_write = test_and_clear_bit(BITMAP_PAGE_NEEDWRITE, &bitmap->logattrs);
882 if (dirty || need_write)
883 if (log->type->flush(log))
884 bitmap->flags |= BITMAP_WRITE_ERROR;
885 goto out;
886 }
877 887
878 /* look at each page to see if there are any set bits that need to be 888 /* look at each page to see if there are any set bits that need to be
879 * flushed out to disk */ 889 * flushed out to disk */
@@ -892,7 +902,7 @@ void bitmap_unplug(struct bitmap *bitmap)
892 wait = 1; 902 wait = 1;
893 spin_unlock_irqrestore(&bitmap->lock, flags); 903 spin_unlock_irqrestore(&bitmap->lock, flags);
894 904
895 if (dirty | need_write) 905 if (dirty || need_write)
896 write_page(bitmap, page, 0); 906 write_page(bitmap, page, 0);
897 } 907 }
898 if (wait) { /* if any writes were performed, we need to wait on them */ 908 if (wait) { /* if any writes were performed, we need to wait on them */
@@ -902,9 +912,11 @@ void bitmap_unplug(struct bitmap *bitmap)
902 else 912 else
903 md_super_wait(bitmap->mddev); 913 md_super_wait(bitmap->mddev);
904 } 914 }
915out:
905 if (bitmap->flags & BITMAP_WRITE_ERROR) 916 if (bitmap->flags & BITMAP_WRITE_ERROR)
906 bitmap_file_kick(bitmap); 917 bitmap_file_kick(bitmap);
907} 918}
919EXPORT_SYMBOL(bitmap_unplug);
908 920
909static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed); 921static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);
910/* * bitmap_init_from_disk -- called at bitmap_create time to initialize 922/* * bitmap_init_from_disk -- called at bitmap_create time to initialize
@@ -943,12 +955,11 @@ static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
943 printk(KERN_INFO "%s: bitmap file is out of date, doing full " 955 printk(KERN_INFO "%s: bitmap file is out of date, doing full "
944 "recovery\n", bmname(bitmap)); 956 "recovery\n", bmname(bitmap));
945 957
946 bytes = (chunks + 7) / 8; 958 bytes = DIV_ROUND_UP(bitmap->chunks, 8);
947 if (!bitmap->mddev->bitmap_info.external) 959 if (!bitmap->mddev->bitmap_info.external)
948 bytes += sizeof(bitmap_super_t); 960 bytes += sizeof(bitmap_super_t);
949 961
950 962 num_pages = DIV_ROUND_UP(bytes, PAGE_SIZE);
951 num_pages = (bytes + PAGE_SIZE - 1) / PAGE_SIZE;
952 963
953 if (file && i_size_read(file->f_mapping->host) < bytes) { 964 if (file && i_size_read(file->f_mapping->host) < bytes) {
954 printk(KERN_INFO "%s: bitmap file too short %lu < %lu\n", 965 printk(KERN_INFO "%s: bitmap file too short %lu < %lu\n",
@@ -966,7 +977,7 @@ static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
966 977
967 /* We need 4 bits per page, rounded up to a multiple of sizeof(unsigned long) */ 978 /* We need 4 bits per page, rounded up to a multiple of sizeof(unsigned long) */
968 bitmap->filemap_attr = kzalloc( 979 bitmap->filemap_attr = kzalloc(
969 roundup( DIV_ROUND_UP(num_pages*4, 8), sizeof(unsigned long)), 980 roundup(DIV_ROUND_UP(num_pages*4, 8), sizeof(unsigned long)),
970 GFP_KERNEL); 981 GFP_KERNEL);
971 if (!bitmap->filemap_attr) 982 if (!bitmap->filemap_attr)
972 goto err; 983 goto err;
@@ -993,10 +1004,11 @@ static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
993 page = bitmap->sb_page; 1004 page = bitmap->sb_page;
994 offset = sizeof(bitmap_super_t); 1005 offset = sizeof(bitmap_super_t);
995 if (!file) 1006 if (!file)
996 read_sb_page(bitmap->mddev, 1007 page = read_sb_page(
997 bitmap->mddev->bitmap_info.offset, 1008 bitmap->mddev,
998 page, 1009 bitmap->mddev->bitmap_info.offset,
999 index, count); 1010 page,
1011 index, count);
1000 } else if (file) { 1012 } else if (file) {
1001 page = read_page(file, index, bitmap, count); 1013 page = read_page(file, index, bitmap, count);
1002 offset = 0; 1014 offset = 0;
@@ -1021,7 +1033,7 @@ static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
1021 if (outofdate) { 1033 if (outofdate) {
1022 /* 1034 /*
1023 * if bitmap is out of date, dirty the 1035 * if bitmap is out of date, dirty the
1024 * whole page and write it out 1036 * whole page and write it out
1025 */ 1037 */
1026 paddr = kmap_atomic(page, KM_USER0); 1038 paddr = kmap_atomic(page, KM_USER0);
1027 memset(paddr + offset, 0xff, 1039 memset(paddr + offset, 0xff,
@@ -1052,7 +1064,7 @@ static int bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
1052 } 1064 }
1053 } 1065 }
1054 1066
1055 /* everything went OK */ 1067 /* everything went OK */
1056 ret = 0; 1068 ret = 0;
1057 bitmap_mask_state(bitmap, BITMAP_STALE, MASK_UNSET); 1069 bitmap_mask_state(bitmap, BITMAP_STALE, MASK_UNSET);
1058 1070
@@ -1080,25 +1092,20 @@ void bitmap_write_all(struct bitmap *bitmap)
1080 */ 1092 */
1081 int i; 1093 int i;
1082 1094
1083 for (i=0; i < bitmap->file_pages; i++) 1095 for (i = 0; i < bitmap->file_pages; i++)
1084 set_page_attr(bitmap, bitmap->filemap[i], 1096 set_page_attr(bitmap, bitmap->filemap[i],
1085 BITMAP_PAGE_NEEDWRITE); 1097 BITMAP_PAGE_NEEDWRITE);
1086} 1098}
1087 1099
1088
1089static void bitmap_count_page(struct bitmap *bitmap, sector_t offset, int inc) 1100static void bitmap_count_page(struct bitmap *bitmap, sector_t offset, int inc)
1090{ 1101{
1091 sector_t chunk = offset >> CHUNK_BLOCK_SHIFT(bitmap); 1102 sector_t chunk = offset >> CHUNK_BLOCK_SHIFT(bitmap);
1092 unsigned long page = chunk >> PAGE_COUNTER_SHIFT; 1103 unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1093 bitmap->bp[page].count += inc; 1104 bitmap->bp[page].count += inc;
1094/*
1095 if (page == 0) printk("count page 0, offset %llu: %d gives %d\n",
1096 (unsigned long long)offset, inc, bitmap->bp[page].count);
1097*/
1098 bitmap_checkfree(bitmap, page); 1105 bitmap_checkfree(bitmap, page);
1099} 1106}
1100static bitmap_counter_t *bitmap_get_counter(struct bitmap *bitmap, 1107static bitmap_counter_t *bitmap_get_counter(struct bitmap *bitmap,
1101 sector_t offset, int *blocks, 1108 sector_t offset, sector_t *blocks,
1102 int create); 1109 int create);
1103 1110
1104/* 1111/*
@@ -1112,8 +1119,9 @@ void bitmap_daemon_work(mddev_t *mddev)
1112 unsigned long j; 1119 unsigned long j;
1113 unsigned long flags; 1120 unsigned long flags;
1114 struct page *page = NULL, *lastpage = NULL; 1121 struct page *page = NULL, *lastpage = NULL;
1115 int blocks; 1122 sector_t blocks;
1116 void *paddr; 1123 void *paddr;
1124 struct dm_dirty_log *log = mddev->bitmap_info.log;
1117 1125
1118 /* Use a mutex to guard daemon_work against 1126 /* Use a mutex to guard daemon_work against
1119 * bitmap_destroy. 1127 * bitmap_destroy.
@@ -1138,11 +1146,12 @@ void bitmap_daemon_work(mddev_t *mddev)
1138 spin_lock_irqsave(&bitmap->lock, flags); 1146 spin_lock_irqsave(&bitmap->lock, flags);
1139 for (j = 0; j < bitmap->chunks; j++) { 1147 for (j = 0; j < bitmap->chunks; j++) {
1140 bitmap_counter_t *bmc; 1148 bitmap_counter_t *bmc;
1141 if (!bitmap->filemap) 1149 if (!bitmap->filemap) {
1142 /* error or shutdown */ 1150 if (!log)
1143 break; 1151 /* error or shutdown */
1144 1152 break;
1145 page = filemap_get_page(bitmap, j); 1153 } else
1154 page = filemap_get_page(bitmap, j);
1146 1155
1147 if (page != lastpage) { 1156 if (page != lastpage) {
1148 /* skip this page unless it's marked as needing cleaning */ 1157 /* skip this page unless it's marked as needing cleaning */
@@ -1197,14 +1206,11 @@ void bitmap_daemon_work(mddev_t *mddev)
1197 (sector_t)j << CHUNK_BLOCK_SHIFT(bitmap), 1206 (sector_t)j << CHUNK_BLOCK_SHIFT(bitmap),
1198 &blocks, 0); 1207 &blocks, 0);
1199 if (bmc) { 1208 if (bmc) {
1200/*
1201 if (j < 100) printk("bitmap: j=%lu, *bmc = 0x%x\n", j, *bmc);
1202*/
1203 if (*bmc) 1209 if (*bmc)
1204 bitmap->allclean = 0; 1210 bitmap->allclean = 0;
1205 1211
1206 if (*bmc == 2) { 1212 if (*bmc == 2) {
1207 *bmc=1; /* maybe clear the bit next time */ 1213 *bmc = 1; /* maybe clear the bit next time */
1208 set_page_attr(bitmap, page, BITMAP_PAGE_CLEAN); 1214 set_page_attr(bitmap, page, BITMAP_PAGE_CLEAN);
1209 } else if (*bmc == 1 && !bitmap->need_sync) { 1215 } else if (*bmc == 1 && !bitmap->need_sync) {
1210 /* we can clear the bit */ 1216 /* we can clear the bit */
@@ -1214,14 +1220,17 @@ void bitmap_daemon_work(mddev_t *mddev)
1214 -1); 1220 -1);
1215 1221
1216 /* clear the bit */ 1222 /* clear the bit */
1217 paddr = kmap_atomic(page, KM_USER0); 1223 if (page) {
1218 if (bitmap->flags & BITMAP_HOSTENDIAN) 1224 paddr = kmap_atomic(page, KM_USER0);
1219 clear_bit(file_page_offset(bitmap, j), 1225 if (bitmap->flags & BITMAP_HOSTENDIAN)
1220 paddr); 1226 clear_bit(file_page_offset(bitmap, j),
1221 else 1227 paddr);
1222 ext2_clear_bit(file_page_offset(bitmap, j), 1228 else
1223 paddr); 1229 ext2_clear_bit(file_page_offset(bitmap, j),
1224 kunmap_atomic(paddr, KM_USER0); 1230 paddr);
1231 kunmap_atomic(paddr, KM_USER0);
1232 } else
1233 log->type->clear_region(log, j);
1225 } 1234 }
1226 } else 1235 } else
1227 j |= PAGE_COUNTER_MASK; 1236 j |= PAGE_COUNTER_MASK;
@@ -1229,12 +1238,16 @@ void bitmap_daemon_work(mddev_t *mddev)
1229 spin_unlock_irqrestore(&bitmap->lock, flags); 1238 spin_unlock_irqrestore(&bitmap->lock, flags);
1230 1239
1231 /* now sync the final page */ 1240 /* now sync the final page */
1232 if (lastpage != NULL) { 1241 if (lastpage != NULL || log != NULL) {
1233 spin_lock_irqsave(&bitmap->lock, flags); 1242 spin_lock_irqsave(&bitmap->lock, flags);
1234 if (test_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE)) { 1243 if (test_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE)) {
1235 clear_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE); 1244 clear_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE);
1236 spin_unlock_irqrestore(&bitmap->lock, flags); 1245 spin_unlock_irqrestore(&bitmap->lock, flags);
1237 write_page(bitmap, lastpage, 0); 1246 if (lastpage)
1247 write_page(bitmap, lastpage, 0);
1248 else
1249 if (log->type->flush(log))
1250 bitmap->flags |= BITMAP_WRITE_ERROR;
1238 } else { 1251 } else {
1239 set_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE); 1252 set_page_attr(bitmap, lastpage, BITMAP_PAGE_NEEDWRITE);
1240 spin_unlock_irqrestore(&bitmap->lock, flags); 1253 spin_unlock_irqrestore(&bitmap->lock, flags);
@@ -1243,13 +1256,13 @@ void bitmap_daemon_work(mddev_t *mddev)
1243 1256
1244 done: 1257 done:
1245 if (bitmap->allclean == 0) 1258 if (bitmap->allclean == 0)
1246 bitmap->mddev->thread->timeout = 1259 bitmap->mddev->thread->timeout =
1247 bitmap->mddev->bitmap_info.daemon_sleep; 1260 bitmap->mddev->bitmap_info.daemon_sleep;
1248 mutex_unlock(&mddev->bitmap_info.mutex); 1261 mutex_unlock(&mddev->bitmap_info.mutex);
1249} 1262}
1250 1263
1251static bitmap_counter_t *bitmap_get_counter(struct bitmap *bitmap, 1264static bitmap_counter_t *bitmap_get_counter(struct bitmap *bitmap,
1252 sector_t offset, int *blocks, 1265 sector_t offset, sector_t *blocks,
1253 int create) 1266 int create)
1254__releases(bitmap->lock) 1267__releases(bitmap->lock)
1255__acquires(bitmap->lock) 1268__acquires(bitmap->lock)
@@ -1262,43 +1275,52 @@ __acquires(bitmap->lock)
1262 unsigned long page = chunk >> PAGE_COUNTER_SHIFT; 1275 unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
1263 unsigned long pageoff = (chunk & PAGE_COUNTER_MASK) << COUNTER_BYTE_SHIFT; 1276 unsigned long pageoff = (chunk & PAGE_COUNTER_MASK) << COUNTER_BYTE_SHIFT;
1264 sector_t csize; 1277 sector_t csize;
1278 int err;
1279
1280 err = bitmap_checkpage(bitmap, page, create);
1265 1281
1266 if (bitmap_checkpage(bitmap, page, create) < 0) { 1282 if (bitmap->bp[page].hijacked ||
1283 bitmap->bp[page].map == NULL)
1284 csize = ((sector_t)1) << (CHUNK_BLOCK_SHIFT(bitmap) +
1285 PAGE_COUNTER_SHIFT - 1);
1286 else
1267 csize = ((sector_t)1) << (CHUNK_BLOCK_SHIFT(bitmap)); 1287 csize = ((sector_t)1) << (CHUNK_BLOCK_SHIFT(bitmap));
1268 *blocks = csize - (offset & (csize- 1)); 1288 *blocks = csize - (offset & (csize - 1));
1289
1290 if (err < 0)
1269 return NULL; 1291 return NULL;
1270 } 1292
1271 /* now locked ... */ 1293 /* now locked ... */
1272 1294
1273 if (bitmap->bp[page].hijacked) { /* hijacked pointer */ 1295 if (bitmap->bp[page].hijacked) { /* hijacked pointer */
1274 /* should we use the first or second counter field 1296 /* should we use the first or second counter field
1275 * of the hijacked pointer? */ 1297 * of the hijacked pointer? */
1276 int hi = (pageoff > PAGE_COUNTER_MASK); 1298 int hi = (pageoff > PAGE_COUNTER_MASK);
1277 csize = ((sector_t)1) << (CHUNK_BLOCK_SHIFT(bitmap) +
1278 PAGE_COUNTER_SHIFT - 1);
1279 *blocks = csize - (offset & (csize- 1));
1280 return &((bitmap_counter_t *) 1299 return &((bitmap_counter_t *)
1281 &bitmap->bp[page].map)[hi]; 1300 &bitmap->bp[page].map)[hi];
1282 } else { /* page is allocated */ 1301 } else /* page is allocated */
1283 csize = ((sector_t)1) << (CHUNK_BLOCK_SHIFT(bitmap));
1284 *blocks = csize - (offset & (csize- 1));
1285 return (bitmap_counter_t *) 1302 return (bitmap_counter_t *)
1286 &(bitmap->bp[page].map[pageoff]); 1303 &(bitmap->bp[page].map[pageoff]);
1287 }
1288} 1304}
1289 1305
1290int bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors, int behind) 1306int bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors, int behind)
1291{ 1307{
1292 if (!bitmap) return 0; 1308 if (!bitmap)
1309 return 0;
1293 1310
1294 if (behind) { 1311 if (behind) {
1312 int bw;
1295 atomic_inc(&bitmap->behind_writes); 1313 atomic_inc(&bitmap->behind_writes);
1314 bw = atomic_read(&bitmap->behind_writes);
1315 if (bw > bitmap->behind_writes_used)
1316 bitmap->behind_writes_used = bw;
1317
1296 PRINTK(KERN_DEBUG "inc write-behind count %d/%d\n", 1318 PRINTK(KERN_DEBUG "inc write-behind count %d/%d\n",
1297 atomic_read(&bitmap->behind_writes), bitmap->max_write_behind); 1319 bw, bitmap->max_write_behind);
1298 } 1320 }
1299 1321
1300 while (sectors) { 1322 while (sectors) {
1301 int blocks; 1323 sector_t blocks;
1302 bitmap_counter_t *bmc; 1324 bitmap_counter_t *bmc;
1303 1325
1304 spin_lock_irq(&bitmap->lock); 1326 spin_lock_irq(&bitmap->lock);
@@ -1317,17 +1339,16 @@ int bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sect
1317 prepare_to_wait(&bitmap->overflow_wait, &__wait, 1339 prepare_to_wait(&bitmap->overflow_wait, &__wait,
1318 TASK_UNINTERRUPTIBLE); 1340 TASK_UNINTERRUPTIBLE);
1319 spin_unlock_irq(&bitmap->lock); 1341 spin_unlock_irq(&bitmap->lock);
1320 blk_unplug(bitmap->mddev->queue); 1342 md_unplug(bitmap->mddev);
1321 schedule(); 1343 schedule();
1322 finish_wait(&bitmap->overflow_wait, &__wait); 1344 finish_wait(&bitmap->overflow_wait, &__wait);
1323 continue; 1345 continue;
1324 } 1346 }
1325 1347
1326 switch(*bmc) { 1348 switch (*bmc) {
1327 case 0: 1349 case 0:
1328 bitmap_file_set_bit(bitmap, offset); 1350 bitmap_file_set_bit(bitmap, offset);
1329 bitmap_count_page(bitmap,offset, 1); 1351 bitmap_count_page(bitmap, offset, 1);
1330 blk_plug_device_unlocked(bitmap->mddev->queue);
1331 /* fall through */ 1352 /* fall through */
1332 case 1: 1353 case 1:
1333 *bmc = 2; 1354 *bmc = 2;
@@ -1340,18 +1361,22 @@ int bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sect
1340 offset += blocks; 1361 offset += blocks;
1341 if (sectors > blocks) 1362 if (sectors > blocks)
1342 sectors -= blocks; 1363 sectors -= blocks;
1343 else sectors = 0; 1364 else
1365 sectors = 0;
1344 } 1366 }
1345 bitmap->allclean = 0; 1367 bitmap->allclean = 0;
1346 return 0; 1368 return 0;
1347} 1369}
1370EXPORT_SYMBOL(bitmap_startwrite);
1348 1371
1349void bitmap_endwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors, 1372void bitmap_endwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors,
1350 int success, int behind) 1373 int success, int behind)
1351{ 1374{
1352 if (!bitmap) return; 1375 if (!bitmap)
1376 return;
1353 if (behind) { 1377 if (behind) {
1354 atomic_dec(&bitmap->behind_writes); 1378 if (atomic_dec_and_test(&bitmap->behind_writes))
1379 wake_up(&bitmap->behind_wait);
1355 PRINTK(KERN_DEBUG "dec write-behind count %d/%d\n", 1380 PRINTK(KERN_DEBUG "dec write-behind count %d/%d\n",
1356 atomic_read(&bitmap->behind_writes), bitmap->max_write_behind); 1381 atomic_read(&bitmap->behind_writes), bitmap->max_write_behind);
1357 } 1382 }
@@ -1360,7 +1385,7 @@ void bitmap_endwrite(struct bitmap *bitmap, sector_t offset, unsigned long secto
1360 success = 0; 1385 success = 0;
1361 1386
1362 while (sectors) { 1387 while (sectors) {
1363 int blocks; 1388 sector_t blocks;
1364 unsigned long flags; 1389 unsigned long flags;
1365 bitmap_counter_t *bmc; 1390 bitmap_counter_t *bmc;
1366 1391
@@ -1375,7 +1400,7 @@ void bitmap_endwrite(struct bitmap *bitmap, sector_t offset, unsigned long secto
1375 bitmap->events_cleared < bitmap->mddev->events) { 1400 bitmap->events_cleared < bitmap->mddev->events) {
1376 bitmap->events_cleared = bitmap->mddev->events; 1401 bitmap->events_cleared = bitmap->mddev->events;
1377 bitmap->need_sync = 1; 1402 bitmap->need_sync = 1;
1378 sysfs_notify_dirent(bitmap->sysfs_can_clear); 1403 sysfs_notify_dirent_safe(bitmap->sysfs_can_clear);
1379 } 1404 }
1380 1405
1381 if (!success && ! (*bmc & NEEDED_MASK)) 1406 if (!success && ! (*bmc & NEEDED_MASK))
@@ -1385,20 +1410,24 @@ void bitmap_endwrite(struct bitmap *bitmap, sector_t offset, unsigned long secto
1385 wake_up(&bitmap->overflow_wait); 1410 wake_up(&bitmap->overflow_wait);
1386 1411
1387 (*bmc)--; 1412 (*bmc)--;
1388 if (*bmc <= 2) { 1413 if (*bmc <= 2)
1389 set_page_attr(bitmap, 1414 set_page_attr(bitmap,
1390 filemap_get_page(bitmap, offset >> CHUNK_BLOCK_SHIFT(bitmap)), 1415 filemap_get_page(
1416 bitmap,
1417 offset >> CHUNK_BLOCK_SHIFT(bitmap)),
1391 BITMAP_PAGE_CLEAN); 1418 BITMAP_PAGE_CLEAN);
1392 } 1419
1393 spin_unlock_irqrestore(&bitmap->lock, flags); 1420 spin_unlock_irqrestore(&bitmap->lock, flags);
1394 offset += blocks; 1421 offset += blocks;
1395 if (sectors > blocks) 1422 if (sectors > blocks)
1396 sectors -= blocks; 1423 sectors -= blocks;
1397 else sectors = 0; 1424 else
1425 sectors = 0;
1398 } 1426 }
1399} 1427}
1428EXPORT_SYMBOL(bitmap_endwrite);
1400 1429
1401static int __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, int *blocks, 1430static int __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1402 int degraded) 1431 int degraded)
1403{ 1432{
1404 bitmap_counter_t *bmc; 1433 bitmap_counter_t *bmc;
@@ -1427,7 +1456,7 @@ static int __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, int *bloc
1427 return rv; 1456 return rv;
1428} 1457}
1429 1458
1430int bitmap_start_sync(struct bitmap *bitmap, sector_t offset, int *blocks, 1459int bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
1431 int degraded) 1460 int degraded)
1432{ 1461{
1433 /* bitmap_start_sync must always report on multiples of whole 1462 /* bitmap_start_sync must always report on multiples of whole
@@ -1438,7 +1467,7 @@ int bitmap_start_sync(struct bitmap *bitmap, sector_t offset, int *blocks,
1438 * Return the 'or' of the result. 1467 * Return the 'or' of the result.
1439 */ 1468 */
1440 int rv = 0; 1469 int rv = 0;
1441 int blocks1; 1470 sector_t blocks1;
1442 1471
1443 *blocks = 0; 1472 *blocks = 0;
1444 while (*blocks < (PAGE_SIZE>>9)) { 1473 while (*blocks < (PAGE_SIZE>>9)) {
@@ -1449,14 +1478,14 @@ int bitmap_start_sync(struct bitmap *bitmap, sector_t offset, int *blocks,
1449 } 1478 }
1450 return rv; 1479 return rv;
1451} 1480}
1481EXPORT_SYMBOL(bitmap_start_sync);
1452 1482
1453void bitmap_end_sync(struct bitmap *bitmap, sector_t offset, int *blocks, int aborted) 1483void bitmap_end_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, int aborted)
1454{ 1484{
1455 bitmap_counter_t *bmc; 1485 bitmap_counter_t *bmc;
1456 unsigned long flags; 1486 unsigned long flags;
1457/* 1487
1458 if (offset == 0) printk("bitmap_end_sync 0 (%d)\n", aborted); 1488 if (bitmap == NULL) {
1459*/ if (bitmap == NULL) {
1460 *blocks = 1024; 1489 *blocks = 1024;
1461 return; 1490 return;
1462 } 1491 }
@@ -1465,26 +1494,23 @@ void bitmap_end_sync(struct bitmap *bitmap, sector_t offset, int *blocks, int ab
1465 if (bmc == NULL) 1494 if (bmc == NULL)
1466 goto unlock; 1495 goto unlock;
1467 /* locked */ 1496 /* locked */
1468/*
1469 if (offset == 0) printk("bitmap_end sync found 0x%x, blocks %d\n", *bmc, *blocks);
1470*/
1471 if (RESYNC(*bmc)) { 1497 if (RESYNC(*bmc)) {
1472 *bmc &= ~RESYNC_MASK; 1498 *bmc &= ~RESYNC_MASK;
1473 1499
1474 if (!NEEDED(*bmc) && aborted) 1500 if (!NEEDED(*bmc) && aborted)
1475 *bmc |= NEEDED_MASK; 1501 *bmc |= NEEDED_MASK;
1476 else { 1502 else {
1477 if (*bmc <= 2) { 1503 if (*bmc <= 2)
1478 set_page_attr(bitmap, 1504 set_page_attr(bitmap,
1479 filemap_get_page(bitmap, offset >> CHUNK_BLOCK_SHIFT(bitmap)), 1505 filemap_get_page(bitmap, offset >> CHUNK_BLOCK_SHIFT(bitmap)),
1480 BITMAP_PAGE_CLEAN); 1506 BITMAP_PAGE_CLEAN);
1481 }
1482 } 1507 }
1483 } 1508 }
1484 unlock: 1509 unlock:
1485 spin_unlock_irqrestore(&bitmap->lock, flags); 1510 spin_unlock_irqrestore(&bitmap->lock, flags);
1486 bitmap->allclean = 0; 1511 bitmap->allclean = 0;
1487} 1512}
1513EXPORT_SYMBOL(bitmap_end_sync);
1488 1514
1489void bitmap_close_sync(struct bitmap *bitmap) 1515void bitmap_close_sync(struct bitmap *bitmap)
1490{ 1516{
@@ -1493,7 +1519,7 @@ void bitmap_close_sync(struct bitmap *bitmap)
1493 * RESYNC bit wherever it is still on 1519 * RESYNC bit wherever it is still on
1494 */ 1520 */
1495 sector_t sector = 0; 1521 sector_t sector = 0;
1496 int blocks; 1522 sector_t blocks;
1497 if (!bitmap) 1523 if (!bitmap)
1498 return; 1524 return;
1499 while (sector < bitmap->mddev->resync_max_sectors) { 1525 while (sector < bitmap->mddev->resync_max_sectors) {
@@ -1501,11 +1527,12 @@ void bitmap_close_sync(struct bitmap *bitmap)
1501 sector += blocks; 1527 sector += blocks;
1502 } 1528 }
1503} 1529}
1530EXPORT_SYMBOL(bitmap_close_sync);
1504 1531
1505void bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector) 1532void bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector)
1506{ 1533{
1507 sector_t s = 0; 1534 sector_t s = 0;
1508 int blocks; 1535 sector_t blocks;
1509 1536
1510 if (!bitmap) 1537 if (!bitmap)
1511 return; 1538 return;
@@ -1519,7 +1546,7 @@ void bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector)
1519 wait_event(bitmap->mddev->recovery_wait, 1546 wait_event(bitmap->mddev->recovery_wait,
1520 atomic_read(&bitmap->mddev->recovery_active) == 0); 1547 atomic_read(&bitmap->mddev->recovery_active) == 0);
1521 1548
1522 bitmap->mddev->curr_resync_completed = bitmap->mddev->curr_resync; 1549 bitmap->mddev->curr_resync_completed = sector;
1523 set_bit(MD_CHANGE_CLEAN, &bitmap->mddev->flags); 1550 set_bit(MD_CHANGE_CLEAN, &bitmap->mddev->flags);
1524 sector &= ~((1ULL << CHUNK_BLOCK_SHIFT(bitmap)) - 1); 1551 sector &= ~((1ULL << CHUNK_BLOCK_SHIFT(bitmap)) - 1);
1525 s = 0; 1552 s = 0;
@@ -1530,6 +1557,7 @@ void bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector)
1530 bitmap->last_end_sync = jiffies; 1557 bitmap->last_end_sync = jiffies;
1531 sysfs_notify(&bitmap->mddev->kobj, NULL, "sync_completed"); 1558 sysfs_notify(&bitmap->mddev->kobj, NULL, "sync_completed");
1532} 1559}
1560EXPORT_SYMBOL(bitmap_cond_end_sync);
1533 1561
1534static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed) 1562static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed)
1535{ 1563{
@@ -1538,7 +1566,7 @@ static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int n
1538 * be 0 at this point 1566 * be 0 at this point
1539 */ 1567 */
1540 1568
1541 int secs; 1569 sector_t secs;
1542 bitmap_counter_t *bmc; 1570 bitmap_counter_t *bmc;
1543 spin_lock_irq(&bitmap->lock); 1571 spin_lock_irq(&bitmap->lock);
1544 bmc = bitmap_get_counter(bitmap, offset, &secs, 1); 1572 bmc = bitmap_get_counter(bitmap, offset, &secs, 1);
@@ -1546,9 +1574,9 @@ static void bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int n
1546 spin_unlock_irq(&bitmap->lock); 1574 spin_unlock_irq(&bitmap->lock);
1547 return; 1575 return;
1548 } 1576 }
1549 if (! *bmc) { 1577 if (!*bmc) {
1550 struct page *page; 1578 struct page *page;
1551 *bmc = 1 | (needed?NEEDED_MASK:0); 1579 *bmc = 1 | (needed ? NEEDED_MASK : 0);
1552 bitmap_count_page(bitmap, offset, 1); 1580 bitmap_count_page(bitmap, offset, 1);
1553 page = filemap_get_page(bitmap, offset >> CHUNK_BLOCK_SHIFT(bitmap)); 1581 page = filemap_get_page(bitmap, offset >> CHUNK_BLOCK_SHIFT(bitmap));
1554 set_page_attr(bitmap, page, BITMAP_PAGE_CLEAN); 1582 set_page_attr(bitmap, page, BITMAP_PAGE_CLEAN);
@@ -1657,15 +1685,17 @@ int bitmap_create(mddev_t *mddev)
1657 unsigned long pages; 1685 unsigned long pages;
1658 struct file *file = mddev->bitmap_info.file; 1686 struct file *file = mddev->bitmap_info.file;
1659 int err; 1687 int err;
1660 sector_t start; 1688 struct sysfs_dirent *bm = NULL;
1661 struct sysfs_dirent *bm;
1662 1689
1663 BUILD_BUG_ON(sizeof(bitmap_super_t) != 256); 1690 BUILD_BUG_ON(sizeof(bitmap_super_t) != 256);
1664 1691
1665 if (!file && !mddev->bitmap_info.offset) /* bitmap disabled, nothing to do */ 1692 if (!file
1693 && !mddev->bitmap_info.offset
1694 && !mddev->bitmap_info.log) /* bitmap disabled, nothing to do */
1666 return 0; 1695 return 0;
1667 1696
1668 BUG_ON(file && mddev->bitmap_info.offset); 1697 BUG_ON(file && mddev->bitmap_info.offset);
1698 BUG_ON(mddev->bitmap_info.offset && mddev->bitmap_info.log);
1669 1699
1670 bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL); 1700 bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL);
1671 if (!bitmap) 1701 if (!bitmap)
@@ -1675,12 +1705,14 @@ int bitmap_create(mddev_t *mddev)
1675 atomic_set(&bitmap->pending_writes, 0); 1705 atomic_set(&bitmap->pending_writes, 0);
1676 init_waitqueue_head(&bitmap->write_wait); 1706 init_waitqueue_head(&bitmap->write_wait);
1677 init_waitqueue_head(&bitmap->overflow_wait); 1707 init_waitqueue_head(&bitmap->overflow_wait);
1708 init_waitqueue_head(&bitmap->behind_wait);
1678 1709
1679 bitmap->mddev = mddev; 1710 bitmap->mddev = mddev;
1680 1711
1681 bm = sysfs_get_dirent(mddev->kobj.sd, "bitmap"); 1712 if (mddev->kobj.sd)
1713 bm = sysfs_get_dirent(mddev->kobj.sd, NULL, "bitmap");
1682 if (bm) { 1714 if (bm) {
1683 bitmap->sysfs_can_clear = sysfs_get_dirent(bm, "can_clear"); 1715 bitmap->sysfs_can_clear = sysfs_get_dirent(bm, NULL, "can_clear");
1684 sysfs_put(bm); 1716 sysfs_put(bm);
1685 } else 1717 } else
1686 bitmap->sysfs_can_clear = NULL; 1718 bitmap->sysfs_can_clear = NULL;
@@ -1692,7 +1724,7 @@ int bitmap_create(mddev_t *mddev)
1692 * and bypass the page cache, we must sync the file 1724 * and bypass the page cache, we must sync the file
1693 * first. 1725 * first.
1694 */ 1726 */
1695 vfs_fsync(file, file->f_dentry, 1); 1727 vfs_fsync(file, 1);
1696 } 1728 }
1697 /* read superblock from bitmap file (this sets mddev->bitmap_info.chunksize) */ 1729 /* read superblock from bitmap file (this sets mddev->bitmap_info.chunksize) */
1698 if (!mddev->bitmap_info.external) 1730 if (!mddev->bitmap_info.external)
@@ -1712,9 +1744,9 @@ int bitmap_create(mddev_t *mddev)
1712 bitmap->chunkshift = ffz(~mddev->bitmap_info.chunksize); 1744 bitmap->chunkshift = ffz(~mddev->bitmap_info.chunksize);
1713 1745
1714 /* now that chunksize and chunkshift are set, we can use these macros */ 1746 /* now that chunksize and chunkshift are set, we can use these macros */
1715 chunks = (blocks + CHUNK_BLOCK_RATIO(bitmap) - 1) >> 1747 chunks = (blocks + CHUNK_BLOCK_RATIO(bitmap) - 1) >>
1716 CHUNK_BLOCK_SHIFT(bitmap); 1748 CHUNK_BLOCK_SHIFT(bitmap);
1717 pages = (chunks + PAGE_COUNTER_RATIO - 1) / PAGE_COUNTER_RATIO; 1749 pages = (chunks + PAGE_COUNTER_RATIO - 1) / PAGE_COUNTER_RATIO;
1718 1750
1719 BUG_ON(!pages); 1751 BUG_ON(!pages);
1720 1752
@@ -1734,27 +1766,11 @@ int bitmap_create(mddev_t *mddev)
1734 if (!bitmap->bp) 1766 if (!bitmap->bp)
1735 goto error; 1767 goto error;
1736 1768
1737 /* now that we have some pages available, initialize the in-memory
1738 * bitmap from the on-disk bitmap */
1739 start = 0;
1740 if (mddev->degraded == 0
1741 || bitmap->events_cleared == mddev->events)
1742 /* no need to keep dirty bits to optimise a re-add of a missing device */
1743 start = mddev->recovery_cp;
1744 err = bitmap_init_from_disk(bitmap, start);
1745
1746 if (err)
1747 goto error;
1748
1749 printk(KERN_INFO "created bitmap (%lu pages) for device %s\n", 1769 printk(KERN_INFO "created bitmap (%lu pages) for device %s\n",
1750 pages, bmname(bitmap)); 1770 pages, bmname(bitmap));
1751 1771
1752 mddev->bitmap = bitmap; 1772 mddev->bitmap = bitmap;
1753 1773
1754 mddev->thread->timeout = mddev->bitmap_info.daemon_sleep;
1755 md_wakeup_thread(mddev->thread);
1756
1757 bitmap_update_sb(bitmap);
1758 1774
1759 return (bitmap->flags & BITMAP_WRITE_ERROR) ? -EIO : 0; 1775 return (bitmap->flags & BITMAP_WRITE_ERROR) ? -EIO : 0;
1760 1776
@@ -1763,15 +1779,69 @@ int bitmap_create(mddev_t *mddev)
1763 return err; 1779 return err;
1764} 1780}
1765 1781
1782int bitmap_load(mddev_t *mddev)
1783{
1784 int err = 0;
1785 sector_t sector = 0;
1786 struct bitmap *bitmap = mddev->bitmap;
1787
1788 if (!bitmap)
1789 goto out;
1790
1791 /* Clear out old bitmap info first: Either there is none, or we
1792 * are resuming after someone else has possibly changed things,
1793 * so we should forget old cached info.
1794 * All chunks should be clean, but some might need_sync.
1795 */
1796 while (sector < mddev->resync_max_sectors) {
1797 sector_t blocks;
1798 bitmap_start_sync(bitmap, sector, &blocks, 0);
1799 sector += blocks;
1800 }
1801 bitmap_close_sync(bitmap);
1802
1803 if (mddev->bitmap_info.log) {
1804 unsigned long i;
1805 struct dm_dirty_log *log = mddev->bitmap_info.log;
1806 for (i = 0; i < bitmap->chunks; i++)
1807 if (!log->type->in_sync(log, i, 1))
1808 bitmap_set_memory_bits(bitmap,
1809 (sector_t)i << CHUNK_BLOCK_SHIFT(bitmap),
1810 1);
1811 } else {
1812 sector_t start = 0;
1813 if (mddev->degraded == 0
1814 || bitmap->events_cleared == mddev->events)
1815 /* no need to keep dirty bits to optimise a
1816 * re-add of a missing device */
1817 start = mddev->recovery_cp;
1818
1819 err = bitmap_init_from_disk(bitmap, start);
1820 }
1821 if (err)
1822 goto out;
1823
1824 mddev->thread->timeout = mddev->bitmap_info.daemon_sleep;
1825 md_wakeup_thread(mddev->thread);
1826
1827 bitmap_update_sb(bitmap);
1828
1829 if (bitmap->flags & BITMAP_WRITE_ERROR)
1830 err = -EIO;
1831out:
1832 return err;
1833}
1834EXPORT_SYMBOL_GPL(bitmap_load);
1835
1766static ssize_t 1836static ssize_t
1767location_show(mddev_t *mddev, char *page) 1837location_show(mddev_t *mddev, char *page)
1768{ 1838{
1769 ssize_t len; 1839 ssize_t len;
1770 if (mddev->bitmap_info.file) { 1840 if (mddev->bitmap_info.file)
1771 len = sprintf(page, "file"); 1841 len = sprintf(page, "file");
1772 } else if (mddev->bitmap_info.offset) { 1842 else if (mddev->bitmap_info.offset)
1773 len = sprintf(page, "%+lld", (long long)mddev->bitmap_info.offset); 1843 len = sprintf(page, "%+lld", (long long)mddev->bitmap_info.offset);
1774 } else 1844 else
1775 len = sprintf(page, "none"); 1845 len = sprintf(page, "none");
1776 len += sprintf(page+len, "\n"); 1846 len += sprintf(page+len, "\n");
1777 return len; 1847 return len;
@@ -1860,7 +1930,7 @@ timeout_show(mddev_t *mddev, char *page)
1860 ssize_t len; 1930 ssize_t len;
1861 unsigned long secs = mddev->bitmap_info.daemon_sleep / HZ; 1931 unsigned long secs = mddev->bitmap_info.daemon_sleep / HZ;
1862 unsigned long jifs = mddev->bitmap_info.daemon_sleep % HZ; 1932 unsigned long jifs = mddev->bitmap_info.daemon_sleep % HZ;
1863 1933
1864 len = sprintf(page, "%lu", secs); 1934 len = sprintf(page, "%lu", secs);
1865 if (jifs) 1935 if (jifs)
1866 len += sprintf(page+len, ".%03u", jiffies_to_msecs(jifs)); 1936 len += sprintf(page+len, ".%03u", jiffies_to_msecs(jifs));
@@ -2006,6 +2076,27 @@ static ssize_t can_clear_store(mddev_t *mddev, const char *buf, size_t len)
2006static struct md_sysfs_entry bitmap_can_clear = 2076static struct md_sysfs_entry bitmap_can_clear =
2007__ATTR(can_clear, S_IRUGO|S_IWUSR, can_clear_show, can_clear_store); 2077__ATTR(can_clear, S_IRUGO|S_IWUSR, can_clear_show, can_clear_store);
2008 2078
2079static ssize_t
2080behind_writes_used_show(mddev_t *mddev, char *page)
2081{
2082 if (mddev->bitmap == NULL)
2083 return sprintf(page, "0\n");
2084 return sprintf(page, "%lu\n",
2085 mddev->bitmap->behind_writes_used);
2086}
2087
2088static ssize_t
2089behind_writes_used_reset(mddev_t *mddev, const char *buf, size_t len)
2090{
2091 if (mddev->bitmap)
2092 mddev->bitmap->behind_writes_used = 0;
2093 return len;
2094}
2095
2096static struct md_sysfs_entry max_backlog_used =
2097__ATTR(max_backlog_used, S_IRUGO | S_IWUSR,
2098 behind_writes_used_show, behind_writes_used_reset);
2099
2009static struct attribute *md_bitmap_attrs[] = { 2100static struct attribute *md_bitmap_attrs[] = {
2010 &bitmap_location.attr, 2101 &bitmap_location.attr,
2011 &bitmap_timeout.attr, 2102 &bitmap_timeout.attr,
@@ -2013,6 +2104,7 @@ static struct attribute *md_bitmap_attrs[] = {
2013 &bitmap_chunksize.attr, 2104 &bitmap_chunksize.attr,
2014 &bitmap_metadata.attr, 2105 &bitmap_metadata.attr,
2015 &bitmap_can_clear.attr, 2106 &bitmap_can_clear.attr,
2107 &max_backlog_used.attr,
2016 NULL 2108 NULL
2017}; 2109};
2018struct attribute_group md_bitmap_group = { 2110struct attribute_group md_bitmap_group = {
@@ -2020,12 +2112,3 @@ struct attribute_group md_bitmap_group = {
2020 .attrs = md_bitmap_attrs, 2112 .attrs = md_bitmap_attrs,
2021}; 2113};
2022 2114
2023
2024/* the bitmap API -- for raid personalities */
2025EXPORT_SYMBOL(bitmap_startwrite);
2026EXPORT_SYMBOL(bitmap_endwrite);
2027EXPORT_SYMBOL(bitmap_start_sync);
2028EXPORT_SYMBOL(bitmap_end_sync);
2029EXPORT_SYMBOL(bitmap_unplug);
2030EXPORT_SYMBOL(bitmap_close_sync);
2031EXPORT_SYMBOL(bitmap_cond_end_sync);