diff options
| author | Minchan Kim <minchan@kernel.org> | 2014-01-30 18:46:02 -0500 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2014-01-30 19:56:55 -0500 |
| commit | deb0bdeb2f3d6b81d37fc778316dae46b6daab56 (patch) | |
| tree | f191670e9a1740af77adc01a46ec859f9c7a2338 /drivers/block | |
| parent | 874e3cddc33f0c0f9cc08ad2b73fa0cbe7dfaa63 (diff) | |
zram: use atomic operation for stat
Some of fields in zram->stats are protected by zram->lock which is
rather coarse-grained so let's use atomic operation without explict
locking.
This patch is ready for removing dependency of zram->lock in read path
which is very coarse-grained rw_semaphore. Of course, this patch adds
new atomic operation so it might make slow but my 12CPU test couldn't
spot any regression. All gain/lose is marginal within stddev.
iozone -t -T -l 12 -u 12 -r 16K -s 60M -I +Z -V 0
==Initial write ==Initial write
records: 50 records: 50
avg: 412875.17 avg: 415638.23
std: 38543.12 (9.34%) std: 36601.11 (8.81%)
max: 521262.03 max: 502976.72
min: 343263.13 min: 351389.12
==Rewrite ==Rewrite
records: 50 records: 50
avg: 416640.34 avg: 397914.33
std: 60798.92 (14.59%) std: 46150.42 (11.60%)
max: 543057.07 max: 522669.17
min: 304071.67 min: 316588.77
==Read ==Read
records: 50 records: 50
avg: 4147338.63 avg: 4070736.51
std: 179333.25 (4.32%) std: 223499.89 (5.49%)
max: 4459295.28 max: 4539514.44
min: 3753057.53 min: 3444686.31
==Re-read ==Re-read
records: 50 records: 50
avg: 4096706.71 avg: 4117218.57
std: 229735.04 (5.61%) std: 171676.25 (4.17%)
max: 4430012.09 max: 4459263.94
min: 2987217.80 min: 3666904.28
==Reverse Read ==Reverse Read
records: 50 records: 50
avg: 4062763.83 avg: 4078508.32
std: 186208.46 (4.58%) std: 172684.34 (4.23%)
max: 4401358.78 max: 4424757.22
min: 3381625.00 min: 3679359.94
==Stride read ==Stride read
records: 50 records: 50
avg: 4094933.49 avg: 4082170.22
std: 185710.52 (4.54%) std: 196346.68 (4.81%)
max: 4478241.25 max: 4460060.97
min: 3732593.23 min: 3584125.78
==Random read ==Random read
records: 50 records: 50
avg: 4031070.04 avg: 4074847.49
std: 192065.51 (4.76%) std: 206911.33 (5.08%)
max: 4356931.16 max: 4399442.56
min: 3481619.62 min: 3548372.44
==Mixed workload ==Mixed workload
records: 50 records: 50
avg: 149925.73 avg: 149675.54
std: 7701.26 (5.14%) std: 6902.09 (4.61%)
max: 191301.56 max: 175162.05
min: 133566.28 min: 137762.87
==Random write ==Random write
records: 50 records: 50
avg: 404050.11 avg: 393021.47
std: 58887.57 (14.57%) std: 42813.70 (10.89%)
max: 601798.09 max: 524533.43
min: 325176.99 min: 313255.34
==Pwrite ==Pwrite
records: 50 records: 50
avg: 411217.70 avg: 411237.96
std: 43114.99 (10.48%) std: 33136.29 (8.06%)
max: 530766.79 max: 471899.76
min: 320786.84 min: 317906.94
==Pread ==Pread
records: 50 records: 50
avg: 4154908.65 avg: 4087121.92
std: 151272.08 (3.64%) std: 219505.04 (5.37%)
max: 4459478.12 max: 4435857.38
min: 3730512.41 min: 3101101.67
Signed-off-by: Minchan Kim <minchan@kernel.org>
Cc: Nitin Gupta <ngupta@vflare.org>
Tested-by: Sergey Senozhatsky <sergey.senozhatsky@gmail.com>
Cc: Jerome Marchand <jmarchan@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
Diffstat (limited to 'drivers/block')
| -rw-r--r-- | drivers/block/zram/zram_drv.c | 20 | ||||
| -rw-r--r-- | drivers/block/zram/zram_drv.h | 16 |
2 files changed, 16 insertions, 20 deletions
diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index ebfddd852272..9ab884999420 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c | |||
| @@ -104,7 +104,7 @@ static ssize_t zero_pages_show(struct device *dev, | |||
| 104 | { | 104 | { |
| 105 | struct zram *zram = dev_to_zram(dev); | 105 | struct zram *zram = dev_to_zram(dev); |
| 106 | 106 | ||
| 107 | return sprintf(buf, "%u\n", zram->stats.pages_zero); | 107 | return sprintf(buf, "%u\n", atomic_read(&zram->stats.pages_zero)); |
| 108 | } | 108 | } |
| 109 | 109 | ||
| 110 | static ssize_t orig_data_size_show(struct device *dev, | 110 | static ssize_t orig_data_size_show(struct device *dev, |
| @@ -113,7 +113,7 @@ static ssize_t orig_data_size_show(struct device *dev, | |||
| 113 | struct zram *zram = dev_to_zram(dev); | 113 | struct zram *zram = dev_to_zram(dev); |
| 114 | 114 | ||
| 115 | return sprintf(buf, "%llu\n", | 115 | return sprintf(buf, "%llu\n", |
| 116 | (u64)(zram->stats.pages_stored) << PAGE_SHIFT); | 116 | (u64)(atomic_read(&zram->stats.pages_stored)) << PAGE_SHIFT); |
| 117 | } | 117 | } |
| 118 | 118 | ||
| 119 | static ssize_t compr_data_size_show(struct device *dev, | 119 | static ssize_t compr_data_size_show(struct device *dev, |
| @@ -293,21 +293,21 @@ static void zram_free_page(struct zram *zram, size_t index) | |||
| 293 | */ | 293 | */ |
| 294 | if (zram_test_flag(meta, index, ZRAM_ZERO)) { | 294 | if (zram_test_flag(meta, index, ZRAM_ZERO)) { |
| 295 | zram_clear_flag(meta, index, ZRAM_ZERO); | 295 | zram_clear_flag(meta, index, ZRAM_ZERO); |
| 296 | zram->stats.pages_zero--; | 296 | atomic_dec(&zram->stats.pages_zero); |
| 297 | } | 297 | } |
| 298 | return; | 298 | return; |
| 299 | } | 299 | } |
| 300 | 300 | ||
| 301 | if (unlikely(size > max_zpage_size)) | 301 | if (unlikely(size > max_zpage_size)) |
| 302 | zram->stats.bad_compress--; | 302 | atomic_dec(&zram->stats.bad_compress); |
| 303 | 303 | ||
| 304 | zs_free(meta->mem_pool, handle); | 304 | zs_free(meta->mem_pool, handle); |
| 305 | 305 | ||
| 306 | if (size <= PAGE_SIZE / 2) | 306 | if (size <= PAGE_SIZE / 2) |
| 307 | zram->stats.good_compress--; | 307 | atomic_dec(&zram->stats.good_compress); |
| 308 | 308 | ||
| 309 | atomic64_sub(meta->table[index].size, &zram->stats.compr_size); | 309 | atomic64_sub(meta->table[index].size, &zram->stats.compr_size); |
| 310 | zram->stats.pages_stored--; | 310 | atomic_dec(&zram->stats.pages_stored); |
| 311 | 311 | ||
| 312 | meta->table[index].handle = 0; | 312 | meta->table[index].handle = 0; |
| 313 | meta->table[index].size = 0; | 313 | meta->table[index].size = 0; |
| @@ -435,7 +435,7 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index, | |||
| 435 | /* Free memory associated with this sector now. */ | 435 | /* Free memory associated with this sector now. */ |
| 436 | zram_free_page(zram, index); | 436 | zram_free_page(zram, index); |
| 437 | 437 | ||
| 438 | zram->stats.pages_zero++; | 438 | atomic_inc(&zram->stats.pages_zero); |
| 439 | zram_set_flag(meta, index, ZRAM_ZERO); | 439 | zram_set_flag(meta, index, ZRAM_ZERO); |
| 440 | ret = 0; | 440 | ret = 0; |
| 441 | goto out; | 441 | goto out; |
| @@ -456,7 +456,7 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index, | |||
| 456 | } | 456 | } |
| 457 | 457 | ||
| 458 | if (unlikely(clen > max_zpage_size)) { | 458 | if (unlikely(clen > max_zpage_size)) { |
| 459 | zram->stats.bad_compress++; | 459 | atomic_inc(&zram->stats.bad_compress); |
| 460 | clen = PAGE_SIZE; | 460 | clen = PAGE_SIZE; |
| 461 | src = NULL; | 461 | src = NULL; |
| 462 | if (is_partial_io(bvec)) | 462 | if (is_partial_io(bvec)) |
| @@ -493,9 +493,9 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index, | |||
| 493 | 493 | ||
| 494 | /* Update stats */ | 494 | /* Update stats */ |
| 495 | atomic64_add(clen, &zram->stats.compr_size); | 495 | atomic64_add(clen, &zram->stats.compr_size); |
| 496 | zram->stats.pages_stored++; | 496 | atomic_inc(&zram->stats.pages_stored); |
| 497 | if (clen <= PAGE_SIZE / 2) | 497 | if (clen <= PAGE_SIZE / 2) |
| 498 | zram->stats.good_compress++; | 498 | atomic_inc(&zram->stats.good_compress); |
| 499 | 499 | ||
| 500 | out: | 500 | out: |
| 501 | if (is_partial_io(bvec)) | 501 | if (is_partial_io(bvec)) |
diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h index 0e46953c08e9..81b0170de369 100644 --- a/drivers/block/zram/zram_drv.h +++ b/drivers/block/zram/zram_drv.h | |||
| @@ -68,10 +68,6 @@ struct table { | |||
| 68 | u8 flags; | 68 | u8 flags; |
| 69 | } __aligned(4); | 69 | } __aligned(4); |
| 70 | 70 | ||
| 71 | /* | ||
| 72 | * All 64bit fields should only be manipulated by 64bit atomic accessors. | ||
| 73 | * All modifications to 32bit counter should be protected by zram->lock. | ||
| 74 | */ | ||
| 75 | struct zram_stats { | 71 | struct zram_stats { |
| 76 | atomic64_t compr_size; /* compressed size of pages stored */ | 72 | atomic64_t compr_size; /* compressed size of pages stored */ |
| 77 | atomic64_t num_reads; /* failed + successful */ | 73 | atomic64_t num_reads; /* failed + successful */ |
| @@ -80,10 +76,10 @@ struct zram_stats { | |||
| 80 | atomic64_t failed_writes; /* can happen when memory is too low */ | 76 | atomic64_t failed_writes; /* can happen when memory is too low */ |
| 81 | atomic64_t invalid_io; /* non-page-aligned I/O requests */ | 77 | atomic64_t invalid_io; /* non-page-aligned I/O requests */ |
| 82 | atomic64_t notify_free; /* no. of swap slot free notifications */ | 78 | atomic64_t notify_free; /* no. of swap slot free notifications */ |
| 83 | u32 pages_zero; /* no. of zero filled pages */ | 79 | atomic_t pages_zero; /* no. of zero filled pages */ |
| 84 | u32 pages_stored; /* no. of pages currently stored */ | 80 | atomic_t pages_stored; /* no. of pages currently stored */ |
| 85 | u32 good_compress; /* % of pages with compression ratio<=50% */ | 81 | atomic_t good_compress; /* % of pages with compression ratio<=50% */ |
| 86 | u32 bad_compress; /* % of pages with compression ratio>=75% */ | 82 | atomic_t bad_compress; /* % of pages with compression ratio>=75% */ |
| 87 | }; | 83 | }; |
| 88 | 84 | ||
| 89 | struct zram_meta { | 85 | struct zram_meta { |
| @@ -101,8 +97,8 @@ struct zram_slot_free { | |||
| 101 | struct zram { | 97 | struct zram { |
| 102 | struct zram_meta *meta; | 98 | struct zram_meta *meta; |
| 103 | struct rw_semaphore lock; /* protect compression buffers, table, | 99 | struct rw_semaphore lock; /* protect compression buffers, table, |
| 104 | * 32bit stat counters against concurrent | 100 | * reads and writes |
| 105 | * notifications, reads and writes */ | 101 | */ |
| 106 | 102 | ||
| 107 | struct work_struct free_work; /* handle pending free request */ | 103 | struct work_struct free_work; /* handle pending free request */ |
| 108 | struct zram_slot_free *slot_free_rq; /* list head of free request */ | 104 | struct zram_slot_free *slot_free_rq; /* list head of free request */ |