diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2014-04-07 19:38:06 -0400 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2014-04-07 19:38:06 -0400 |
| commit | 26c12d93348f0bda0756aff83f4867d9ae58a5a6 (patch) | |
| tree | 65221f6837c66a9260c5c973e5fb908b10e0d504 /drivers/block | |
| parent | dc5ed40686a4da95881c35d913b60f867755cbe2 (diff) | |
| parent | fdc5813fbbd484a54c88477f91a78934cda8bb32 (diff) | |
Merge branch 'akpm' (incoming from Andrew)
Merge second patch-bomb from Andrew Morton:
- the rest of MM
- zram updates
- zswap updates
- exit
- procfs
- exec
- wait
- crash dump
- lib/idr
- rapidio
- adfs, affs, bfs, ufs
- cris
- Kconfig things
- initramfs
- small amount of IPC material
- percpu enhancements
- early ioremap support
- various other misc things
* emailed patches from Andrew Morton <akpm@linux-foundation.org>: (156 commits)
MAINTAINERS: update Intel C600 SAS driver maintainers
fs/ufs: remove unused ufs_super_block_third pointer
fs/ufs: remove unused ufs_super_block_second pointer
fs/ufs: remove unused ufs_super_block_first pointer
fs/ufs/super.c: add __init to init_inodecache()
doc/kernel-parameters.txt: add early_ioremap_debug
arm64: add early_ioremap support
arm64: initialize pgprot info earlier in boot
x86: use generic early_ioremap
mm: create generic early_ioremap() support
x86/mm: sparse warning fix for early_memremap
lglock: map to spinlock when !CONFIG_SMP
percpu: add preemption checks to __this_cpu ops
vmstat: use raw_cpu_ops to avoid false positives on preemption checks
slub: use raw_cpu_inc for incrementing statistics
net: replace __this_cpu_inc in route.c with raw_cpu_inc
modules: use raw_cpu_write for initialization of per cpu refcount.
mm: use raw_cpu ops for determining current NUMA node
percpu: add raw_cpu_ops
slub: fix leak of 'name' in sysfs_slab_add
...
Diffstat (limited to 'drivers/block')
| -rw-r--r-- | drivers/block/zram/Kconfig | 10 | ||||
| -rw-r--r-- | drivers/block/zram/Makefile | 4 | ||||
| -rw-r--r-- | drivers/block/zram/zcomp.c | 353 | ||||
| -rw-r--r-- | drivers/block/zram/zcomp.h | 68 | ||||
| -rw-r--r-- | drivers/block/zram/zcomp_lz4.c | 47 | ||||
| -rw-r--r-- | drivers/block/zram/zcomp_lz4.h | 17 | ||||
| -rw-r--r-- | drivers/block/zram/zcomp_lzo.c | 47 | ||||
| -rw-r--r-- | drivers/block/zram/zcomp_lzo.h | 17 | ||||
| -rw-r--r-- | drivers/block/zram/zram_drv.c | 383 | ||||
| -rw-r--r-- | drivers/block/zram/zram_drv.h | 21 |
10 files changed, 797 insertions, 170 deletions
diff --git a/drivers/block/zram/Kconfig b/drivers/block/zram/Kconfig index 3450be850399..6489c0fd0ea6 100644 --- a/drivers/block/zram/Kconfig +++ b/drivers/block/zram/Kconfig | |||
| @@ -15,6 +15,16 @@ config ZRAM | |||
| 15 | 15 | ||
| 16 | See zram.txt for more information. | 16 | See zram.txt for more information. |
| 17 | 17 | ||
| 18 | config ZRAM_LZ4_COMPRESS | ||
| 19 | bool "Enable LZ4 algorithm support" | ||
| 20 | depends on ZRAM | ||
| 21 | select LZ4_COMPRESS | ||
| 22 | select LZ4_DECOMPRESS | ||
| 23 | default n | ||
| 24 | help | ||
| 25 | This option enables LZ4 compression algorithm support. Compression | ||
| 26 | algorithm can be changed using `comp_algorithm' device attribute. | ||
| 27 | |||
| 18 | config ZRAM_DEBUG | 28 | config ZRAM_DEBUG |
| 19 | bool "Compressed RAM block device debug support" | 29 | bool "Compressed RAM block device debug support" |
| 20 | depends on ZRAM | 30 | depends on ZRAM |
diff --git a/drivers/block/zram/Makefile b/drivers/block/zram/Makefile index cb0f9ced6a93..be0763ff57a2 100644 --- a/drivers/block/zram/Makefile +++ b/drivers/block/zram/Makefile | |||
| @@ -1,3 +1,5 @@ | |||
| 1 | zram-y := zram_drv.o | 1 | zram-y := zcomp_lzo.o zcomp.o zram_drv.o |
| 2 | |||
| 3 | zram-$(CONFIG_ZRAM_LZ4_COMPRESS) += zcomp_lz4.o | ||
| 2 | 4 | ||
| 3 | obj-$(CONFIG_ZRAM) += zram.o | 5 | obj-$(CONFIG_ZRAM) += zram.o |
diff --git a/drivers/block/zram/zcomp.c b/drivers/block/zram/zcomp.c new file mode 100644 index 000000000000..f1ff39a3d1c1 --- /dev/null +++ b/drivers/block/zram/zcomp.c | |||
| @@ -0,0 +1,353 @@ | |||
| 1 | /* | ||
| 2 | * Copyright (C) 2014 Sergey Senozhatsky. | ||
| 3 | * | ||
| 4 | * This program is free software; you can redistribute it and/or | ||
| 5 | * modify it under the terms of the GNU General Public License | ||
| 6 | * as published by the Free Software Foundation; either version | ||
| 7 | * 2 of the License, or (at your option) any later version. | ||
| 8 | */ | ||
| 9 | |||
| 10 | #include <linux/kernel.h> | ||
| 11 | #include <linux/string.h> | ||
| 12 | #include <linux/err.h> | ||
| 13 | #include <linux/slab.h> | ||
| 14 | #include <linux/wait.h> | ||
| 15 | #include <linux/sched.h> | ||
| 16 | |||
| 17 | #include "zcomp.h" | ||
| 18 | #include "zcomp_lzo.h" | ||
| 19 | #ifdef CONFIG_ZRAM_LZ4_COMPRESS | ||
| 20 | #include "zcomp_lz4.h" | ||
| 21 | #endif | ||
| 22 | |||
| 23 | /* | ||
| 24 | * single zcomp_strm backend | ||
| 25 | */ | ||
| 26 | struct zcomp_strm_single { | ||
| 27 | struct mutex strm_lock; | ||
| 28 | struct zcomp_strm *zstrm; | ||
| 29 | }; | ||
| 30 | |||
| 31 | /* | ||
| 32 | * multi zcomp_strm backend | ||
| 33 | */ | ||
| 34 | struct zcomp_strm_multi { | ||
| 35 | /* protect strm list */ | ||
| 36 | spinlock_t strm_lock; | ||
| 37 | /* max possible number of zstrm streams */ | ||
| 38 | int max_strm; | ||
| 39 | /* number of available zstrm streams */ | ||
| 40 | int avail_strm; | ||
| 41 | /* list of available strms */ | ||
| 42 | struct list_head idle_strm; | ||
| 43 | wait_queue_head_t strm_wait; | ||
| 44 | }; | ||
| 45 | |||
| 46 | static struct zcomp_backend *backends[] = { | ||
| 47 | &zcomp_lzo, | ||
| 48 | #ifdef CONFIG_ZRAM_LZ4_COMPRESS | ||
| 49 | &zcomp_lz4, | ||
| 50 | #endif | ||
| 51 | NULL | ||
| 52 | }; | ||
| 53 | |||
| 54 | static struct zcomp_backend *find_backend(const char *compress) | ||
| 55 | { | ||
| 56 | int i = 0; | ||
| 57 | while (backends[i]) { | ||
| 58 | if (sysfs_streq(compress, backends[i]->name)) | ||
| 59 | break; | ||
| 60 | i++; | ||
| 61 | } | ||
| 62 | return backends[i]; | ||
| 63 | } | ||
| 64 | |||
| 65 | static void zcomp_strm_free(struct zcomp *comp, struct zcomp_strm *zstrm) | ||
| 66 | { | ||
| 67 | if (zstrm->private) | ||
| 68 | comp->backend->destroy(zstrm->private); | ||
| 69 | free_pages((unsigned long)zstrm->buffer, 1); | ||
| 70 | kfree(zstrm); | ||
| 71 | } | ||
| 72 | |||
| 73 | /* | ||
| 74 | * allocate new zcomp_strm structure with ->private initialized by | ||
| 75 | * backend, return NULL on error | ||
| 76 | */ | ||
| 77 | static struct zcomp_strm *zcomp_strm_alloc(struct zcomp *comp) | ||
| 78 | { | ||
| 79 | struct zcomp_strm *zstrm = kmalloc(sizeof(*zstrm), GFP_KERNEL); | ||
| 80 | if (!zstrm) | ||
| 81 | return NULL; | ||
| 82 | |||
| 83 | zstrm->private = comp->backend->create(); | ||
| 84 | /* | ||
| 85 | * allocate 2 pages. 1 for compressed data, plus 1 extra for the | ||
| 86 | * case when compressed size is larger than the original one | ||
| 87 | */ | ||
| 88 | zstrm->buffer = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); | ||
| 89 | if (!zstrm->private || !zstrm->buffer) { | ||
| 90 | zcomp_strm_free(comp, zstrm); | ||
| 91 | zstrm = NULL; | ||
| 92 | } | ||
| 93 | return zstrm; | ||
| 94 | } | ||
| 95 | |||
| 96 | /* | ||
| 97 | * get idle zcomp_strm or wait until other process release | ||
| 98 | * (zcomp_strm_release()) one for us | ||
| 99 | */ | ||
| 100 | static struct zcomp_strm *zcomp_strm_multi_find(struct zcomp *comp) | ||
| 101 | { | ||
| 102 | struct zcomp_strm_multi *zs = comp->stream; | ||
| 103 | struct zcomp_strm *zstrm; | ||
| 104 | |||
| 105 | while (1) { | ||
| 106 | spin_lock(&zs->strm_lock); | ||
| 107 | if (!list_empty(&zs->idle_strm)) { | ||
| 108 | zstrm = list_entry(zs->idle_strm.next, | ||
| 109 | struct zcomp_strm, list); | ||
| 110 | list_del(&zstrm->list); | ||
| 111 | spin_unlock(&zs->strm_lock); | ||
| 112 | return zstrm; | ||
| 113 | } | ||
| 114 | /* zstrm streams limit reached, wait for idle stream */ | ||
| 115 | if (zs->avail_strm >= zs->max_strm) { | ||
| 116 | spin_unlock(&zs->strm_lock); | ||
| 117 | wait_event(zs->strm_wait, !list_empty(&zs->idle_strm)); | ||
| 118 | continue; | ||
| 119 | } | ||
| 120 | /* allocate new zstrm stream */ | ||
| 121 | zs->avail_strm++; | ||
| 122 | spin_unlock(&zs->strm_lock); | ||
| 123 | |||
| 124 | zstrm = zcomp_strm_alloc(comp); | ||
| 125 | if (!zstrm) { | ||
| 126 | spin_lock(&zs->strm_lock); | ||
| 127 | zs->avail_strm--; | ||
| 128 | spin_unlock(&zs->strm_lock); | ||
| 129 | wait_event(zs->strm_wait, !list_empty(&zs->idle_strm)); | ||
| 130 | continue; | ||
| 131 | } | ||
| 132 | break; | ||
| 133 | } | ||
| 134 | return zstrm; | ||
| 135 | } | ||
| 136 | |||
| 137 | /* add stream back to idle list and wake up waiter or free the stream */ | ||
| 138 | static void zcomp_strm_multi_release(struct zcomp *comp, struct zcomp_strm *zstrm) | ||
| 139 | { | ||
| 140 | struct zcomp_strm_multi *zs = comp->stream; | ||
| 141 | |||
| 142 | spin_lock(&zs->strm_lock); | ||
| 143 | if (zs->avail_strm <= zs->max_strm) { | ||
| 144 | list_add(&zstrm->list, &zs->idle_strm); | ||
| 145 | spin_unlock(&zs->strm_lock); | ||
| 146 | wake_up(&zs->strm_wait); | ||
| 147 | return; | ||
| 148 | } | ||
| 149 | |||
| 150 | zs->avail_strm--; | ||
| 151 | spin_unlock(&zs->strm_lock); | ||
| 152 | zcomp_strm_free(comp, zstrm); | ||
| 153 | } | ||
| 154 | |||
| 155 | /* change max_strm limit */ | ||
| 156 | static bool zcomp_strm_multi_set_max_streams(struct zcomp *comp, int num_strm) | ||
| 157 | { | ||
| 158 | struct zcomp_strm_multi *zs = comp->stream; | ||
| 159 | struct zcomp_strm *zstrm; | ||
| 160 | |||
| 161 | spin_lock(&zs->strm_lock); | ||
| 162 | zs->max_strm = num_strm; | ||
| 163 | /* | ||
| 164 | * if user has lowered the limit and there are idle streams, | ||
| 165 | * immediately free as much streams (and memory) as we can. | ||
| 166 | */ | ||
| 167 | while (zs->avail_strm > num_strm && !list_empty(&zs->idle_strm)) { | ||
| 168 | zstrm = list_entry(zs->idle_strm.next, | ||
| 169 | struct zcomp_strm, list); | ||
| 170 | list_del(&zstrm->list); | ||
| 171 | zcomp_strm_free(comp, zstrm); | ||
| 172 | zs->avail_strm--; | ||
| 173 | } | ||
| 174 | spin_unlock(&zs->strm_lock); | ||
| 175 | return true; | ||
| 176 | } | ||
| 177 | |||
| 178 | static void zcomp_strm_multi_destroy(struct zcomp *comp) | ||
| 179 | { | ||
| 180 | struct zcomp_strm_multi *zs = comp->stream; | ||
| 181 | struct zcomp_strm *zstrm; | ||
| 182 | |||
| 183 | while (!list_empty(&zs->idle_strm)) { | ||
| 184 | zstrm = list_entry(zs->idle_strm.next, | ||
| 185 | struct zcomp_strm, list); | ||
| 186 | list_del(&zstrm->list); | ||
| 187 | zcomp_strm_free(comp, zstrm); | ||
| 188 | } | ||
| 189 | kfree(zs); | ||
| 190 | } | ||
| 191 | |||
| 192 | static int zcomp_strm_multi_create(struct zcomp *comp, int max_strm) | ||
| 193 | { | ||
| 194 | struct zcomp_strm *zstrm; | ||
| 195 | struct zcomp_strm_multi *zs; | ||
| 196 | |||
| 197 | comp->destroy = zcomp_strm_multi_destroy; | ||
| 198 | comp->strm_find = zcomp_strm_multi_find; | ||
| 199 | comp->strm_release = zcomp_strm_multi_release; | ||
| 200 | comp->set_max_streams = zcomp_strm_multi_set_max_streams; | ||
| 201 | zs = kmalloc(sizeof(struct zcomp_strm_multi), GFP_KERNEL); | ||
| 202 | if (!zs) | ||
| 203 | return -ENOMEM; | ||
| 204 | |||
| 205 | comp->stream = zs; | ||
| 206 | spin_lock_init(&zs->strm_lock); | ||
| 207 | INIT_LIST_HEAD(&zs->idle_strm); | ||
| 208 | init_waitqueue_head(&zs->strm_wait); | ||
| 209 | zs->max_strm = max_strm; | ||
| 210 | zs->avail_strm = 1; | ||
| 211 | |||
| 212 | zstrm = zcomp_strm_alloc(comp); | ||
| 213 | if (!zstrm) { | ||
| 214 | kfree(zs); | ||
| 215 | return -ENOMEM; | ||
| 216 | } | ||
| 217 | list_add(&zstrm->list, &zs->idle_strm); | ||
| 218 | return 0; | ||
| 219 | } | ||
| 220 | |||
| 221 | static struct zcomp_strm *zcomp_strm_single_find(struct zcomp *comp) | ||
| 222 | { | ||
| 223 | struct zcomp_strm_single *zs = comp->stream; | ||
| 224 | mutex_lock(&zs->strm_lock); | ||
| 225 | return zs->zstrm; | ||
| 226 | } | ||
| 227 | |||
| 228 | static void zcomp_strm_single_release(struct zcomp *comp, | ||
| 229 | struct zcomp_strm *zstrm) | ||
| 230 | { | ||
| 231 | struct zcomp_strm_single *zs = comp->stream; | ||
| 232 | mutex_unlock(&zs->strm_lock); | ||
| 233 | } | ||
| 234 | |||
| 235 | static bool zcomp_strm_single_set_max_streams(struct zcomp *comp, int num_strm) | ||
| 236 | { | ||
| 237 | /* zcomp_strm_single support only max_comp_streams == 1 */ | ||
| 238 | return false; | ||
| 239 | } | ||
| 240 | |||
| 241 | static void zcomp_strm_single_destroy(struct zcomp *comp) | ||
| 242 | { | ||
| 243 | struct zcomp_strm_single *zs = comp->stream; | ||
| 244 | zcomp_strm_free(comp, zs->zstrm); | ||
| 245 | kfree(zs); | ||
| 246 | } | ||
| 247 | |||
| 248 | static int zcomp_strm_single_create(struct zcomp *comp) | ||
| 249 | { | ||
| 250 | struct zcomp_strm_single *zs; | ||
| 251 | |||
| 252 | comp->destroy = zcomp_strm_single_destroy; | ||
| 253 | comp->strm_find = zcomp_strm_single_find; | ||
| 254 | comp->strm_release = zcomp_strm_single_release; | ||
| 255 | comp->set_max_streams = zcomp_strm_single_set_max_streams; | ||
| 256 | zs = kmalloc(sizeof(struct zcomp_strm_single), GFP_KERNEL); | ||
| 257 | if (!zs) | ||
| 258 | return -ENOMEM; | ||
| 259 | |||
| 260 | comp->stream = zs; | ||
| 261 | mutex_init(&zs->strm_lock); | ||
| 262 | zs->zstrm = zcomp_strm_alloc(comp); | ||
| 263 | if (!zs->zstrm) { | ||
| 264 | kfree(zs); | ||
| 265 | return -ENOMEM; | ||
| 266 | } | ||
| 267 | return 0; | ||
| 268 | } | ||
| 269 | |||
| 270 | /* show available compressors */ | ||
| 271 | ssize_t zcomp_available_show(const char *comp, char *buf) | ||
| 272 | { | ||
| 273 | ssize_t sz = 0; | ||
| 274 | int i = 0; | ||
| 275 | |||
| 276 | while (backends[i]) { | ||
| 277 | if (sysfs_streq(comp, backends[i]->name)) | ||
| 278 | sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, | ||
| 279 | "[%s] ", backends[i]->name); | ||
| 280 | else | ||
| 281 | sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2, | ||
| 282 | "%s ", backends[i]->name); | ||
| 283 | i++; | ||
| 284 | } | ||
| 285 | sz += scnprintf(buf + sz, PAGE_SIZE - sz, "\n"); | ||
| 286 | return sz; | ||
| 287 | } | ||
| 288 | |||
| 289 | bool zcomp_set_max_streams(struct zcomp *comp, int num_strm) | ||
| 290 | { | ||
| 291 | return comp->set_max_streams(comp, num_strm); | ||
| 292 | } | ||
| 293 | |||
| 294 | struct zcomp_strm *zcomp_strm_find(struct zcomp *comp) | ||
| 295 | { | ||
| 296 | return comp->strm_find(comp); | ||
| 297 | } | ||
| 298 | |||
| 299 | void zcomp_strm_release(struct zcomp *comp, struct zcomp_strm *zstrm) | ||
| 300 | { | ||
| 301 | comp->strm_release(comp, zstrm); | ||
| 302 | } | ||
| 303 | |||
| 304 | int zcomp_compress(struct zcomp *comp, struct zcomp_strm *zstrm, | ||
| 305 | const unsigned char *src, size_t *dst_len) | ||
| 306 | { | ||
| 307 | return comp->backend->compress(src, zstrm->buffer, dst_len, | ||
| 308 | zstrm->private); | ||
| 309 | } | ||
| 310 | |||
| 311 | int zcomp_decompress(struct zcomp *comp, const unsigned char *src, | ||
| 312 | size_t src_len, unsigned char *dst) | ||
| 313 | { | ||
| 314 | return comp->backend->decompress(src, src_len, dst); | ||
| 315 | } | ||
| 316 | |||
| 317 | void zcomp_destroy(struct zcomp *comp) | ||
| 318 | { | ||
| 319 | comp->destroy(comp); | ||
| 320 | kfree(comp); | ||
| 321 | } | ||
| 322 | |||
| 323 | /* | ||
| 324 | * search available compressors for requested algorithm. | ||
| 325 | * allocate new zcomp and initialize it. return compressing | ||
| 326 | * backend pointer or ERR_PTR if things went bad. ERR_PTR(-EINVAL) | ||
| 327 | * if requested algorithm is not supported, ERR_PTR(-ENOMEM) in | ||
| 328 | * case of allocation error. | ||
| 329 | */ | ||
| 330 | struct zcomp *zcomp_create(const char *compress, int max_strm) | ||
| 331 | { | ||
| 332 | struct zcomp *comp; | ||
| 333 | struct zcomp_backend *backend; | ||
| 334 | |||
| 335 | backend = find_backend(compress); | ||
| 336 | if (!backend) | ||
| 337 | return ERR_PTR(-EINVAL); | ||
| 338 | |||
| 339 | comp = kzalloc(sizeof(struct zcomp), GFP_KERNEL); | ||
| 340 | if (!comp) | ||
| 341 | return ERR_PTR(-ENOMEM); | ||
| 342 | |||
| 343 | comp->backend = backend; | ||
| 344 | if (max_strm > 1) | ||
| 345 | zcomp_strm_multi_create(comp, max_strm); | ||
| 346 | else | ||
| 347 | zcomp_strm_single_create(comp); | ||
| 348 | if (!comp->stream) { | ||
| 349 | kfree(comp); | ||
| 350 | return ERR_PTR(-ENOMEM); | ||
| 351 | } | ||
| 352 | return comp; | ||
| 353 | } | ||
diff --git a/drivers/block/zram/zcomp.h b/drivers/block/zram/zcomp.h new file mode 100644 index 000000000000..c59d1fca72c0 --- /dev/null +++ b/drivers/block/zram/zcomp.h | |||
| @@ -0,0 +1,68 @@ | |||
| 1 | /* | ||
| 2 | * Copyright (C) 2014 Sergey Senozhatsky. | ||
| 3 | * | ||
| 4 | * This program is free software; you can redistribute it and/or | ||
| 5 | * modify it under the terms of the GNU General Public License | ||
| 6 | * as published by the Free Software Foundation; either version | ||
| 7 | * 2 of the License, or (at your option) any later version. | ||
| 8 | */ | ||
| 9 | |||
| 10 | #ifndef _ZCOMP_H_ | ||
| 11 | #define _ZCOMP_H_ | ||
| 12 | |||
| 13 | #include <linux/mutex.h> | ||
| 14 | |||
| 15 | struct zcomp_strm { | ||
| 16 | /* compression/decompression buffer */ | ||
| 17 | void *buffer; | ||
| 18 | /* | ||
| 19 | * The private data of the compression stream, only compression | ||
| 20 | * stream backend can touch this (e.g. compression algorithm | ||
| 21 | * working memory) | ||
| 22 | */ | ||
| 23 | void *private; | ||
| 24 | /* used in multi stream backend, protected by backend strm_lock */ | ||
| 25 | struct list_head list; | ||
| 26 | }; | ||
| 27 | |||
| 28 | /* static compression backend */ | ||
| 29 | struct zcomp_backend { | ||
| 30 | int (*compress)(const unsigned char *src, unsigned char *dst, | ||
| 31 | size_t *dst_len, void *private); | ||
| 32 | |||
| 33 | int (*decompress)(const unsigned char *src, size_t src_len, | ||
| 34 | unsigned char *dst); | ||
| 35 | |||
| 36 | void *(*create)(void); | ||
| 37 | void (*destroy)(void *private); | ||
| 38 | |||
| 39 | const char *name; | ||
| 40 | }; | ||
| 41 | |||
| 42 | /* dynamic per-device compression frontend */ | ||
| 43 | struct zcomp { | ||
| 44 | void *stream; | ||
| 45 | struct zcomp_backend *backend; | ||
| 46 | |||
| 47 | struct zcomp_strm *(*strm_find)(struct zcomp *comp); | ||
| 48 | void (*strm_release)(struct zcomp *comp, struct zcomp_strm *zstrm); | ||
| 49 | bool (*set_max_streams)(struct zcomp *comp, int num_strm); | ||
| 50 | void (*destroy)(struct zcomp *comp); | ||
| 51 | }; | ||
| 52 | |||
| 53 | ssize_t zcomp_available_show(const char *comp, char *buf); | ||
| 54 | |||
| 55 | struct zcomp *zcomp_create(const char *comp, int max_strm); | ||
| 56 | void zcomp_destroy(struct zcomp *comp); | ||
| 57 | |||
| 58 | struct zcomp_strm *zcomp_strm_find(struct zcomp *comp); | ||
| 59 | void zcomp_strm_release(struct zcomp *comp, struct zcomp_strm *zstrm); | ||
| 60 | |||
| 61 | int zcomp_compress(struct zcomp *comp, struct zcomp_strm *zstrm, | ||
| 62 | const unsigned char *src, size_t *dst_len); | ||
| 63 | |||
| 64 | int zcomp_decompress(struct zcomp *comp, const unsigned char *src, | ||
| 65 | size_t src_len, unsigned char *dst); | ||
| 66 | |||
| 67 | bool zcomp_set_max_streams(struct zcomp *comp, int num_strm); | ||
| 68 | #endif /* _ZCOMP_H_ */ | ||
diff --git a/drivers/block/zram/zcomp_lz4.c b/drivers/block/zram/zcomp_lz4.c new file mode 100644 index 000000000000..f2afb7e988c3 --- /dev/null +++ b/drivers/block/zram/zcomp_lz4.c | |||
| @@ -0,0 +1,47 @@ | |||
| 1 | /* | ||
| 2 | * Copyright (C) 2014 Sergey Senozhatsky. | ||
| 3 | * | ||
| 4 | * This program is free software; you can redistribute it and/or | ||
| 5 | * modify it under the terms of the GNU General Public License | ||
| 6 | * as published by the Free Software Foundation; either version | ||
| 7 | * 2 of the License, or (at your option) any later version. | ||
| 8 | */ | ||
| 9 | |||
| 10 | #include <linux/kernel.h> | ||
| 11 | #include <linux/slab.h> | ||
| 12 | #include <linux/lz4.h> | ||
| 13 | |||
| 14 | #include "zcomp_lz4.h" | ||
| 15 | |||
| 16 | static void *zcomp_lz4_create(void) | ||
| 17 | { | ||
| 18 | return kzalloc(LZ4_MEM_COMPRESS, GFP_KERNEL); | ||
| 19 | } | ||
| 20 | |||
| 21 | static void zcomp_lz4_destroy(void *private) | ||
| 22 | { | ||
| 23 | kfree(private); | ||
| 24 | } | ||
| 25 | |||
| 26 | static int zcomp_lz4_compress(const unsigned char *src, unsigned char *dst, | ||
| 27 | size_t *dst_len, void *private) | ||
| 28 | { | ||
| 29 | /* return : Success if return 0 */ | ||
| 30 | return lz4_compress(src, PAGE_SIZE, dst, dst_len, private); | ||
| 31 | } | ||
| 32 | |||
| 33 | static int zcomp_lz4_decompress(const unsigned char *src, size_t src_len, | ||
| 34 | unsigned char *dst) | ||
| 35 | { | ||
| 36 | size_t dst_len = PAGE_SIZE; | ||
| 37 | /* return : Success if return 0 */ | ||
| 38 | return lz4_decompress_unknownoutputsize(src, src_len, dst, &dst_len); | ||
| 39 | } | ||
| 40 | |||
| 41 | struct zcomp_backend zcomp_lz4 = { | ||
| 42 | .compress = zcomp_lz4_compress, | ||
| 43 | .decompress = zcomp_lz4_decompress, | ||
| 44 | .create = zcomp_lz4_create, | ||
| 45 | .destroy = zcomp_lz4_destroy, | ||
| 46 | .name = "lz4", | ||
| 47 | }; | ||
diff --git a/drivers/block/zram/zcomp_lz4.h b/drivers/block/zram/zcomp_lz4.h new file mode 100644 index 000000000000..60613fb29dd8 --- /dev/null +++ b/drivers/block/zram/zcomp_lz4.h | |||
| @@ -0,0 +1,17 @@ | |||
| 1 | /* | ||
| 2 | * Copyright (C) 2014 Sergey Senozhatsky. | ||
| 3 | * | ||
| 4 | * This program is free software; you can redistribute it and/or | ||
| 5 | * modify it under the terms of the GNU General Public License | ||
| 6 | * as published by the Free Software Foundation; either version | ||
| 7 | * 2 of the License, or (at your option) any later version. | ||
| 8 | */ | ||
| 9 | |||
| 10 | #ifndef _ZCOMP_LZ4_H_ | ||
| 11 | #define _ZCOMP_LZ4_H_ | ||
| 12 | |||
| 13 | #include "zcomp.h" | ||
| 14 | |||
| 15 | extern struct zcomp_backend zcomp_lz4; | ||
| 16 | |||
| 17 | #endif /* _ZCOMP_LZ4_H_ */ | ||
diff --git a/drivers/block/zram/zcomp_lzo.c b/drivers/block/zram/zcomp_lzo.c new file mode 100644 index 000000000000..da1bc47d588e --- /dev/null +++ b/drivers/block/zram/zcomp_lzo.c | |||
| @@ -0,0 +1,47 @@ | |||
| 1 | /* | ||
| 2 | * Copyright (C) 2014 Sergey Senozhatsky. | ||
| 3 | * | ||
| 4 | * This program is free software; you can redistribute it and/or | ||
| 5 | * modify it under the terms of the GNU General Public License | ||
| 6 | * as published by the Free Software Foundation; either version | ||
| 7 | * 2 of the License, or (at your option) any later version. | ||
| 8 | */ | ||
| 9 | |||
| 10 | #include <linux/kernel.h> | ||
| 11 | #include <linux/slab.h> | ||
| 12 | #include <linux/lzo.h> | ||
| 13 | |||
| 14 | #include "zcomp_lzo.h" | ||
| 15 | |||
| 16 | static void *lzo_create(void) | ||
| 17 | { | ||
| 18 | return kzalloc(LZO1X_MEM_COMPRESS, GFP_KERNEL); | ||
| 19 | } | ||
| 20 | |||
| 21 | static void lzo_destroy(void *private) | ||
| 22 | { | ||
| 23 | kfree(private); | ||
| 24 | } | ||
| 25 | |||
| 26 | static int lzo_compress(const unsigned char *src, unsigned char *dst, | ||
| 27 | size_t *dst_len, void *private) | ||
| 28 | { | ||
| 29 | int ret = lzo1x_1_compress(src, PAGE_SIZE, dst, dst_len, private); | ||
| 30 | return ret == LZO_E_OK ? 0 : ret; | ||
| 31 | } | ||
| 32 | |||
| 33 | static int lzo_decompress(const unsigned char *src, size_t src_len, | ||
| 34 | unsigned char *dst) | ||
| 35 | { | ||
| 36 | size_t dst_len = PAGE_SIZE; | ||
| 37 | int ret = lzo1x_decompress_safe(src, src_len, dst, &dst_len); | ||
| 38 | return ret == LZO_E_OK ? 0 : ret; | ||
| 39 | } | ||
| 40 | |||
| 41 | struct zcomp_backend zcomp_lzo = { | ||
| 42 | .compress = lzo_compress, | ||
| 43 | .decompress = lzo_decompress, | ||
| 44 | .create = lzo_create, | ||
| 45 | .destroy = lzo_destroy, | ||
| 46 | .name = "lzo", | ||
| 47 | }; | ||
diff --git a/drivers/block/zram/zcomp_lzo.h b/drivers/block/zram/zcomp_lzo.h new file mode 100644 index 000000000000..128c5807fa14 --- /dev/null +++ b/drivers/block/zram/zcomp_lzo.h | |||
| @@ -0,0 +1,17 @@ | |||
| 1 | /* | ||
| 2 | * Copyright (C) 2014 Sergey Senozhatsky. | ||
| 3 | * | ||
| 4 | * This program is free software; you can redistribute it and/or | ||
| 5 | * modify it under the terms of the GNU General Public License | ||
| 6 | * as published by the Free Software Foundation; either version | ||
| 7 | * 2 of the License, or (at your option) any later version. | ||
| 8 | */ | ||
| 9 | |||
| 10 | #ifndef _ZCOMP_LZO_H_ | ||
| 11 | #define _ZCOMP_LZO_H_ | ||
| 12 | |||
| 13 | #include "zcomp.h" | ||
| 14 | |||
| 15 | extern struct zcomp_backend zcomp_lzo; | ||
| 16 | |||
| 17 | #endif /* _ZCOMP_LZO_H_ */ | ||
diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c index 51c557cfd92b..9849b5233bf4 100644 --- a/drivers/block/zram/zram_drv.c +++ b/drivers/block/zram/zram_drv.c | |||
| @@ -29,19 +29,36 @@ | |||
| 29 | #include <linux/genhd.h> | 29 | #include <linux/genhd.h> |
| 30 | #include <linux/highmem.h> | 30 | #include <linux/highmem.h> |
| 31 | #include <linux/slab.h> | 31 | #include <linux/slab.h> |
| 32 | #include <linux/lzo.h> | ||
| 33 | #include <linux/string.h> | 32 | #include <linux/string.h> |
| 34 | #include <linux/vmalloc.h> | 33 | #include <linux/vmalloc.h> |
| 34 | #include <linux/err.h> | ||
| 35 | 35 | ||
| 36 | #include "zram_drv.h" | 36 | #include "zram_drv.h" |
| 37 | 37 | ||
| 38 | /* Globals */ | 38 | /* Globals */ |
| 39 | static int zram_major; | 39 | static int zram_major; |
| 40 | static struct zram *zram_devices; | 40 | static struct zram *zram_devices; |
| 41 | static const char *default_compressor = "lzo"; | ||
| 41 | 42 | ||
| 42 | /* Module params (documentation at end) */ | 43 | /* Module params (documentation at end) */ |
| 43 | static unsigned int num_devices = 1; | 44 | static unsigned int num_devices = 1; |
| 44 | 45 | ||
| 46 | #define ZRAM_ATTR_RO(name) \ | ||
| 47 | static ssize_t zram_attr_##name##_show(struct device *d, \ | ||
| 48 | struct device_attribute *attr, char *b) \ | ||
| 49 | { \ | ||
| 50 | struct zram *zram = dev_to_zram(d); \ | ||
| 51 | return scnprintf(b, PAGE_SIZE, "%llu\n", \ | ||
| 52 | (u64)atomic64_read(&zram->stats.name)); \ | ||
| 53 | } \ | ||
| 54 | static struct device_attribute dev_attr_##name = \ | ||
| 55 | __ATTR(name, S_IRUGO, zram_attr_##name##_show, NULL); | ||
| 56 | |||
| 57 | static inline int init_done(struct zram *zram) | ||
| 58 | { | ||
| 59 | return zram->meta != NULL; | ||
| 60 | } | ||
| 61 | |||
| 45 | static inline struct zram *dev_to_zram(struct device *dev) | 62 | static inline struct zram *dev_to_zram(struct device *dev) |
| 46 | { | 63 | { |
| 47 | return (struct zram *)dev_to_disk(dev)->private_data; | 64 | return (struct zram *)dev_to_disk(dev)->private_data; |
| @@ -52,92 +69,114 @@ static ssize_t disksize_show(struct device *dev, | |||
| 52 | { | 69 | { |
| 53 | struct zram *zram = dev_to_zram(dev); | 70 | struct zram *zram = dev_to_zram(dev); |
| 54 | 71 | ||
| 55 | return sprintf(buf, "%llu\n", zram->disksize); | 72 | return scnprintf(buf, PAGE_SIZE, "%llu\n", zram->disksize); |
| 56 | } | 73 | } |
| 57 | 74 | ||
| 58 | static ssize_t initstate_show(struct device *dev, | 75 | static ssize_t initstate_show(struct device *dev, |
| 59 | struct device_attribute *attr, char *buf) | 76 | struct device_attribute *attr, char *buf) |
| 60 | { | 77 | { |
| 78 | u32 val; | ||
| 61 | struct zram *zram = dev_to_zram(dev); | 79 | struct zram *zram = dev_to_zram(dev); |
| 62 | 80 | ||
| 63 | return sprintf(buf, "%u\n", zram->init_done); | 81 | down_read(&zram->init_lock); |
| 64 | } | 82 | val = init_done(zram); |
| 65 | 83 | up_read(&zram->init_lock); | |
| 66 | static ssize_t num_reads_show(struct device *dev, | ||
| 67 | struct device_attribute *attr, char *buf) | ||
| 68 | { | ||
| 69 | struct zram *zram = dev_to_zram(dev); | ||
| 70 | 84 | ||
| 71 | return sprintf(buf, "%llu\n", | 85 | return scnprintf(buf, PAGE_SIZE, "%u\n", val); |
| 72 | (u64)atomic64_read(&zram->stats.num_reads)); | ||
| 73 | } | 86 | } |
| 74 | 87 | ||
| 75 | static ssize_t num_writes_show(struct device *dev, | 88 | static ssize_t orig_data_size_show(struct device *dev, |
| 76 | struct device_attribute *attr, char *buf) | 89 | struct device_attribute *attr, char *buf) |
| 77 | { | 90 | { |
| 78 | struct zram *zram = dev_to_zram(dev); | 91 | struct zram *zram = dev_to_zram(dev); |
| 79 | 92 | ||
| 80 | return sprintf(buf, "%llu\n", | 93 | return scnprintf(buf, PAGE_SIZE, "%llu\n", |
| 81 | (u64)atomic64_read(&zram->stats.num_writes)); | 94 | (u64)(atomic64_read(&zram->stats.pages_stored)) << PAGE_SHIFT); |
| 82 | } | 95 | } |
| 83 | 96 | ||
| 84 | static ssize_t invalid_io_show(struct device *dev, | 97 | static ssize_t mem_used_total_show(struct device *dev, |
| 85 | struct device_attribute *attr, char *buf) | 98 | struct device_attribute *attr, char *buf) |
| 86 | { | 99 | { |
| 100 | u64 val = 0; | ||
| 87 | struct zram *zram = dev_to_zram(dev); | 101 | struct zram *zram = dev_to_zram(dev); |
| 102 | struct zram_meta *meta = zram->meta; | ||
| 88 | 103 | ||
| 89 | return sprintf(buf, "%llu\n", | 104 | down_read(&zram->init_lock); |
| 90 | (u64)atomic64_read(&zram->stats.invalid_io)); | 105 | if (init_done(zram)) |
| 91 | } | 106 | val = zs_get_total_size_bytes(meta->mem_pool); |
| 92 | 107 | up_read(&zram->init_lock); | |
| 93 | static ssize_t notify_free_show(struct device *dev, | ||
| 94 | struct device_attribute *attr, char *buf) | ||
| 95 | { | ||
| 96 | struct zram *zram = dev_to_zram(dev); | ||
| 97 | 108 | ||
| 98 | return sprintf(buf, "%llu\n", | 109 | return scnprintf(buf, PAGE_SIZE, "%llu\n", val); |
| 99 | (u64)atomic64_read(&zram->stats.notify_free)); | ||
| 100 | } | 110 | } |
| 101 | 111 | ||
| 102 | static ssize_t zero_pages_show(struct device *dev, | 112 | static ssize_t max_comp_streams_show(struct device *dev, |
| 103 | struct device_attribute *attr, char *buf) | 113 | struct device_attribute *attr, char *buf) |
| 104 | { | 114 | { |
| 115 | int val; | ||
| 105 | struct zram *zram = dev_to_zram(dev); | 116 | struct zram *zram = dev_to_zram(dev); |
| 106 | 117 | ||
| 107 | return sprintf(buf, "%u\n", atomic_read(&zram->stats.pages_zero)); | 118 | down_read(&zram->init_lock); |
| 119 | val = zram->max_comp_streams; | ||
| 120 | up_read(&zram->init_lock); | ||
| 121 | |||
| 122 | return scnprintf(buf, PAGE_SIZE, "%d\n", val); | ||
| 108 | } | 123 | } |
| 109 | 124 | ||
| 110 | static ssize_t orig_data_size_show(struct device *dev, | 125 | static ssize_t max_comp_streams_store(struct device *dev, |
| 111 | struct device_attribute *attr, char *buf) | 126 | struct device_attribute *attr, const char *buf, size_t len) |
| 112 | { | 127 | { |
| 128 | int num; | ||
| 113 | struct zram *zram = dev_to_zram(dev); | 129 | struct zram *zram = dev_to_zram(dev); |
| 130 | int ret; | ||
| 114 | 131 | ||
| 115 | return sprintf(buf, "%llu\n", | 132 | ret = kstrtoint(buf, 0, &num); |
| 116 | (u64)(atomic_read(&zram->stats.pages_stored)) << PAGE_SHIFT); | 133 | if (ret < 0) |
| 117 | } | 134 | return ret; |
| 135 | if (num < 1) | ||
| 136 | return -EINVAL; | ||
| 118 | 137 | ||
| 119 | static ssize_t compr_data_size_show(struct device *dev, | 138 | down_write(&zram->init_lock); |
| 120 | struct device_attribute *attr, char *buf) | 139 | if (init_done(zram)) { |
| 121 | { | 140 | if (!zcomp_set_max_streams(zram->comp, num)) { |
| 122 | struct zram *zram = dev_to_zram(dev); | 141 | pr_info("Cannot change max compression streams\n"); |
| 142 | ret = -EINVAL; | ||
| 143 | goto out; | ||
| 144 | } | ||
| 145 | } | ||
| 123 | 146 | ||
| 124 | return sprintf(buf, "%llu\n", | 147 | zram->max_comp_streams = num; |
| 125 | (u64)atomic64_read(&zram->stats.compr_size)); | 148 | ret = len; |
| 149 | out: | ||
| 150 | up_write(&zram->init_lock); | ||
| 151 | return ret; | ||
| 126 | } | 152 | } |
| 127 | 153 | ||
| 128 | static ssize_t mem_used_total_show(struct device *dev, | 154 | static ssize_t comp_algorithm_show(struct device *dev, |
| 129 | struct device_attribute *attr, char *buf) | 155 | struct device_attribute *attr, char *buf) |
| 130 | { | 156 | { |
| 131 | u64 val = 0; | 157 | size_t sz; |
| 132 | struct zram *zram = dev_to_zram(dev); | 158 | struct zram *zram = dev_to_zram(dev); |
| 133 | struct zram_meta *meta = zram->meta; | ||
| 134 | 159 | ||
| 135 | down_read(&zram->init_lock); | 160 | down_read(&zram->init_lock); |
| 136 | if (zram->init_done) | 161 | sz = zcomp_available_show(zram->compressor, buf); |
| 137 | val = zs_get_total_size_bytes(meta->mem_pool); | ||
| 138 | up_read(&zram->init_lock); | 162 | up_read(&zram->init_lock); |
| 139 | 163 | ||
| 140 | return sprintf(buf, "%llu\n", val); | 164 | return sz; |
| 165 | } | ||
| 166 | |||
| 167 | static ssize_t comp_algorithm_store(struct device *dev, | ||
| 168 | struct device_attribute *attr, const char *buf, size_t len) | ||
| 169 | { | ||
| 170 | struct zram *zram = dev_to_zram(dev); | ||
| 171 | down_write(&zram->init_lock); | ||
| 172 | if (init_done(zram)) { | ||
| 173 | up_write(&zram->init_lock); | ||
| 174 | pr_info("Can't change algorithm for initialized device\n"); | ||
| 175 | return -EBUSY; | ||
| 176 | } | ||
| 177 | strlcpy(zram->compressor, buf, sizeof(zram->compressor)); | ||
| 178 | up_write(&zram->init_lock); | ||
| 179 | return len; | ||
| 141 | } | 180 | } |
| 142 | 181 | ||
| 143 | /* flag operations needs meta->tb_lock */ | 182 | /* flag operations needs meta->tb_lock */ |
| @@ -192,8 +231,6 @@ static inline int valid_io_request(struct zram *zram, struct bio *bio) | |||
| 192 | static void zram_meta_free(struct zram_meta *meta) | 231 | static void zram_meta_free(struct zram_meta *meta) |
| 193 | { | 232 | { |
| 194 | zs_destroy_pool(meta->mem_pool); | 233 | zs_destroy_pool(meta->mem_pool); |
| 195 | kfree(meta->compress_workmem); | ||
| 196 | free_pages((unsigned long)meta->compress_buffer, 1); | ||
| 197 | vfree(meta->table); | 234 | vfree(meta->table); |
| 198 | kfree(meta); | 235 | kfree(meta); |
| 199 | } | 236 | } |
| @@ -205,22 +242,11 @@ static struct zram_meta *zram_meta_alloc(u64 disksize) | |||
| 205 | if (!meta) | 242 | if (!meta) |
| 206 | goto out; | 243 | goto out; |
| 207 | 244 | ||
| 208 | meta->compress_workmem = kzalloc(LZO1X_MEM_COMPRESS, GFP_KERNEL); | ||
| 209 | if (!meta->compress_workmem) | ||
| 210 | goto free_meta; | ||
| 211 | |||
| 212 | meta->compress_buffer = | ||
| 213 | (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); | ||
| 214 | if (!meta->compress_buffer) { | ||
| 215 | pr_err("Error allocating compressor buffer space\n"); | ||
| 216 | goto free_workmem; | ||
| 217 | } | ||
| 218 | |||
| 219 | num_pages = disksize >> PAGE_SHIFT; | 245 | num_pages = disksize >> PAGE_SHIFT; |
| 220 | meta->table = vzalloc(num_pages * sizeof(*meta->table)); | 246 | meta->table = vzalloc(num_pages * sizeof(*meta->table)); |
| 221 | if (!meta->table) { | 247 | if (!meta->table) { |
| 222 | pr_err("Error allocating zram address table\n"); | 248 | pr_err("Error allocating zram address table\n"); |
| 223 | goto free_buffer; | 249 | goto free_meta; |
| 224 | } | 250 | } |
| 225 | 251 | ||
| 226 | meta->mem_pool = zs_create_pool(GFP_NOIO | __GFP_HIGHMEM); | 252 | meta->mem_pool = zs_create_pool(GFP_NOIO | __GFP_HIGHMEM); |
| @@ -230,15 +256,10 @@ static struct zram_meta *zram_meta_alloc(u64 disksize) | |||
| 230 | } | 256 | } |
| 231 | 257 | ||
| 232 | rwlock_init(&meta->tb_lock); | 258 | rwlock_init(&meta->tb_lock); |
| 233 | mutex_init(&meta->buffer_lock); | ||
| 234 | return meta; | 259 | return meta; |
| 235 | 260 | ||
| 236 | free_table: | 261 | free_table: |
| 237 | vfree(meta->table); | 262 | vfree(meta->table); |
| 238 | free_buffer: | ||
| 239 | free_pages((unsigned long)meta->compress_buffer, 1); | ||
| 240 | free_workmem: | ||
| 241 | kfree(meta->compress_workmem); | ||
| 242 | free_meta: | 263 | free_meta: |
| 243 | kfree(meta); | 264 | kfree(meta); |
| 244 | meta = NULL; | 265 | meta = NULL; |
| @@ -288,7 +309,6 @@ static void zram_free_page(struct zram *zram, size_t index) | |||
| 288 | { | 309 | { |
| 289 | struct zram_meta *meta = zram->meta; | 310 | struct zram_meta *meta = zram->meta; |
| 290 | unsigned long handle = meta->table[index].handle; | 311 | unsigned long handle = meta->table[index].handle; |
| 291 | u16 size = meta->table[index].size; | ||
| 292 | 312 | ||
| 293 | if (unlikely(!handle)) { | 313 | if (unlikely(!handle)) { |
| 294 | /* | 314 | /* |
| @@ -297,21 +317,15 @@ static void zram_free_page(struct zram *zram, size_t index) | |||
| 297 | */ | 317 | */ |
| 298 | if (zram_test_flag(meta, index, ZRAM_ZERO)) { | 318 | if (zram_test_flag(meta, index, ZRAM_ZERO)) { |
| 299 | zram_clear_flag(meta, index, ZRAM_ZERO); | 319 | zram_clear_flag(meta, index, ZRAM_ZERO); |
| 300 | atomic_dec(&zram->stats.pages_zero); | 320 | atomic64_dec(&zram->stats.zero_pages); |
| 301 | } | 321 | } |
| 302 | return; | 322 | return; |
| 303 | } | 323 | } |
| 304 | 324 | ||
| 305 | if (unlikely(size > max_zpage_size)) | ||
| 306 | atomic_dec(&zram->stats.bad_compress); | ||
| 307 | |||
| 308 | zs_free(meta->mem_pool, handle); | 325 | zs_free(meta->mem_pool, handle); |
| 309 | 326 | ||
| 310 | if (size <= PAGE_SIZE / 2) | 327 | atomic64_sub(meta->table[index].size, &zram->stats.compr_data_size); |
| 311 | atomic_dec(&zram->stats.good_compress); | 328 | atomic64_dec(&zram->stats.pages_stored); |
| 312 | |||
| 313 | atomic64_sub(meta->table[index].size, &zram->stats.compr_size); | ||
| 314 | atomic_dec(&zram->stats.pages_stored); | ||
| 315 | 329 | ||
| 316 | meta->table[index].handle = 0; | 330 | meta->table[index].handle = 0; |
| 317 | meta->table[index].size = 0; | 331 | meta->table[index].size = 0; |
| @@ -319,8 +333,7 @@ static void zram_free_page(struct zram *zram, size_t index) | |||
| 319 | 333 | ||
| 320 | static int zram_decompress_page(struct zram *zram, char *mem, u32 index) | 334 | static int zram_decompress_page(struct zram *zram, char *mem, u32 index) |
| 321 | { | 335 | { |
| 322 | int ret = LZO_E_OK; | 336 | int ret = 0; |
| 323 | size_t clen = PAGE_SIZE; | ||
| 324 | unsigned char *cmem; | 337 | unsigned char *cmem; |
| 325 | struct zram_meta *meta = zram->meta; | 338 | struct zram_meta *meta = zram->meta; |
| 326 | unsigned long handle; | 339 | unsigned long handle; |
| @@ -340,12 +353,12 @@ static int zram_decompress_page(struct zram *zram, char *mem, u32 index) | |||
| 340 | if (size == PAGE_SIZE) | 353 | if (size == PAGE_SIZE) |
| 341 | copy_page(mem, cmem); | 354 | copy_page(mem, cmem); |
| 342 | else | 355 | else |
| 343 | ret = lzo1x_decompress_safe(cmem, size, mem, &clen); | 356 | ret = zcomp_decompress(zram->comp, cmem, size, mem); |
| 344 | zs_unmap_object(meta->mem_pool, handle); | 357 | zs_unmap_object(meta->mem_pool, handle); |
| 345 | read_unlock(&meta->tb_lock); | 358 | read_unlock(&meta->tb_lock); |
| 346 | 359 | ||
| 347 | /* Should NEVER happen. Return bio error if it does. */ | 360 | /* Should NEVER happen. Return bio error if it does. */ |
| 348 | if (unlikely(ret != LZO_E_OK)) { | 361 | if (unlikely(ret)) { |
| 349 | pr_err("Decompression failed! err=%d, page=%u\n", ret, index); | 362 | pr_err("Decompression failed! err=%d, page=%u\n", ret, index); |
| 350 | atomic64_inc(&zram->stats.failed_reads); | 363 | atomic64_inc(&zram->stats.failed_reads); |
| 351 | return ret; | 364 | return ret; |
| @@ -388,7 +401,7 @@ static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, | |||
| 388 | 401 | ||
| 389 | ret = zram_decompress_page(zram, uncmem, index); | 402 | ret = zram_decompress_page(zram, uncmem, index); |
| 390 | /* Should NEVER happen. Return bio error if it does. */ | 403 | /* Should NEVER happen. Return bio error if it does. */ |
| 391 | if (unlikely(ret != LZO_E_OK)) | 404 | if (unlikely(ret)) |
| 392 | goto out_cleanup; | 405 | goto out_cleanup; |
| 393 | 406 | ||
| 394 | if (is_partial_io(bvec)) | 407 | if (is_partial_io(bvec)) |
| @@ -413,11 +426,10 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index, | |||
| 413 | struct page *page; | 426 | struct page *page; |
| 414 | unsigned char *user_mem, *cmem, *src, *uncmem = NULL; | 427 | unsigned char *user_mem, *cmem, *src, *uncmem = NULL; |
| 415 | struct zram_meta *meta = zram->meta; | 428 | struct zram_meta *meta = zram->meta; |
| 429 | struct zcomp_strm *zstrm; | ||
| 416 | bool locked = false; | 430 | bool locked = false; |
| 417 | 431 | ||
| 418 | page = bvec->bv_page; | 432 | page = bvec->bv_page; |
| 419 | src = meta->compress_buffer; | ||
| 420 | |||
| 421 | if (is_partial_io(bvec)) { | 433 | if (is_partial_io(bvec)) { |
| 422 | /* | 434 | /* |
| 423 | * This is a partial IO. We need to read the full page | 435 | * This is a partial IO. We need to read the full page |
| @@ -433,7 +445,7 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index, | |||
| 433 | goto out; | 445 | goto out; |
| 434 | } | 446 | } |
| 435 | 447 | ||
| 436 | mutex_lock(&meta->buffer_lock); | 448 | zstrm = zcomp_strm_find(zram->comp); |
| 437 | locked = true; | 449 | locked = true; |
| 438 | user_mem = kmap_atomic(page); | 450 | user_mem = kmap_atomic(page); |
| 439 | 451 | ||
| @@ -454,28 +466,25 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index, | |||
| 454 | zram_set_flag(meta, index, ZRAM_ZERO); | 466 | zram_set_flag(meta, index, ZRAM_ZERO); |
| 455 | write_unlock(&zram->meta->tb_lock); | 467 | write_unlock(&zram->meta->tb_lock); |
| 456 | 468 | ||
| 457 | atomic_inc(&zram->stats.pages_zero); | 469 | atomic64_inc(&zram->stats.zero_pages); |
| 458 | ret = 0; | 470 | ret = 0; |
| 459 | goto out; | 471 | goto out; |
| 460 | } | 472 | } |
| 461 | 473 | ||
| 462 | ret = lzo1x_1_compress(uncmem, PAGE_SIZE, src, &clen, | 474 | ret = zcomp_compress(zram->comp, zstrm, uncmem, &clen); |
| 463 | meta->compress_workmem); | ||
| 464 | if (!is_partial_io(bvec)) { | 475 | if (!is_partial_io(bvec)) { |
| 465 | kunmap_atomic(user_mem); | 476 | kunmap_atomic(user_mem); |
| 466 | user_mem = NULL; | 477 | user_mem = NULL; |
| 467 | uncmem = NULL; | 478 | uncmem = NULL; |
| 468 | } | 479 | } |
| 469 | 480 | ||
| 470 | if (unlikely(ret != LZO_E_OK)) { | 481 | if (unlikely(ret)) { |
| 471 | pr_err("Compression failed! err=%d\n", ret); | 482 | pr_err("Compression failed! err=%d\n", ret); |
| 472 | goto out; | 483 | goto out; |
| 473 | } | 484 | } |
| 474 | 485 | src = zstrm->buffer; | |
| 475 | if (unlikely(clen > max_zpage_size)) { | 486 | if (unlikely(clen > max_zpage_size)) { |
| 476 | atomic_inc(&zram->stats.bad_compress); | ||
| 477 | clen = PAGE_SIZE; | 487 | clen = PAGE_SIZE; |
| 478 | src = NULL; | ||
| 479 | if (is_partial_io(bvec)) | 488 | if (is_partial_io(bvec)) |
| 480 | src = uncmem; | 489 | src = uncmem; |
| 481 | } | 490 | } |
| @@ -497,6 +506,8 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index, | |||
| 497 | memcpy(cmem, src, clen); | 506 | memcpy(cmem, src, clen); |
| 498 | } | 507 | } |
| 499 | 508 | ||
| 509 | zcomp_strm_release(zram->comp, zstrm); | ||
| 510 | locked = false; | ||
| 500 | zs_unmap_object(meta->mem_pool, handle); | 511 | zs_unmap_object(meta->mem_pool, handle); |
| 501 | 512 | ||
| 502 | /* | 513 | /* |
| @@ -511,49 +522,88 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index, | |||
| 511 | write_unlock(&zram->meta->tb_lock); | 522 | write_unlock(&zram->meta->tb_lock); |
| 512 | 523 | ||
| 513 | /* Update stats */ | 524 | /* Update stats */ |
| 514 | atomic64_add(clen, &zram->stats.compr_size); | 525 | atomic64_add(clen, &zram->stats.compr_data_size); |
| 515 | atomic_inc(&zram->stats.pages_stored); | 526 | atomic64_inc(&zram->stats.pages_stored); |
| 516 | if (clen <= PAGE_SIZE / 2) | ||
| 517 | atomic_inc(&zram->stats.good_compress); | ||
| 518 | |||
| 519 | out: | 527 | out: |
| 520 | if (locked) | 528 | if (locked) |
| 521 | mutex_unlock(&meta->buffer_lock); | 529 | zcomp_strm_release(zram->comp, zstrm); |
| 522 | if (is_partial_io(bvec)) | 530 | if (is_partial_io(bvec)) |
| 523 | kfree(uncmem); | 531 | kfree(uncmem); |
| 524 | |||
| 525 | if (ret) | 532 | if (ret) |
| 526 | atomic64_inc(&zram->stats.failed_writes); | 533 | atomic64_inc(&zram->stats.failed_writes); |
| 527 | return ret; | 534 | return ret; |
| 528 | } | 535 | } |
| 529 | 536 | ||
| 530 | static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index, | 537 | static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index, |
| 531 | int offset, struct bio *bio, int rw) | 538 | int offset, struct bio *bio) |
| 532 | { | 539 | { |
| 533 | int ret; | 540 | int ret; |
| 541 | int rw = bio_data_dir(bio); | ||
| 534 | 542 | ||
| 535 | if (rw == READ) | 543 | if (rw == READ) { |
| 544 | atomic64_inc(&zram->stats.num_reads); | ||
| 536 | ret = zram_bvec_read(zram, bvec, index, offset, bio); | 545 | ret = zram_bvec_read(zram, bvec, index, offset, bio); |
| 537 | else | 546 | } else { |
| 547 | atomic64_inc(&zram->stats.num_writes); | ||
| 538 | ret = zram_bvec_write(zram, bvec, index, offset); | 548 | ret = zram_bvec_write(zram, bvec, index, offset); |
| 549 | } | ||
| 539 | 550 | ||
| 540 | return ret; | 551 | return ret; |
| 541 | } | 552 | } |
| 542 | 553 | ||
| 554 | /* | ||
| 555 | * zram_bio_discard - handler on discard request | ||
| 556 | * @index: physical block index in PAGE_SIZE units | ||
| 557 | * @offset: byte offset within physical block | ||
| 558 | */ | ||
| 559 | static void zram_bio_discard(struct zram *zram, u32 index, | ||
| 560 | int offset, struct bio *bio) | ||
| 561 | { | ||
| 562 | size_t n = bio->bi_iter.bi_size; | ||
| 563 | |||
| 564 | /* | ||
| 565 | * zram manages data in physical block size units. Because logical block | ||
| 566 | * size isn't identical with physical block size on some arch, we | ||
| 567 | * could get a discard request pointing to a specific offset within a | ||
| 568 | * certain physical block. Although we can handle this request by | ||
| 569 | * reading that physiclal block and decompressing and partially zeroing | ||
| 570 | * and re-compressing and then re-storing it, this isn't reasonable | ||
| 571 | * because our intent with a discard request is to save memory. So | ||
| 572 | * skipping this logical block is appropriate here. | ||
| 573 | */ | ||
| 574 | if (offset) { | ||
| 575 | if (n < offset) | ||
| 576 | return; | ||
| 577 | |||
| 578 | n -= offset; | ||
| 579 | index++; | ||
| 580 | } | ||
| 581 | |||
| 582 | while (n >= PAGE_SIZE) { | ||
| 583 | /* | ||
| 584 | * Discard request can be large so the lock hold times could be | ||
| 585 | * lengthy. So take the lock once per page. | ||
| 586 | */ | ||
| 587 | write_lock(&zram->meta->tb_lock); | ||
| 588 | zram_free_page(zram, index); | ||
| 589 | write_unlock(&zram->meta->tb_lock); | ||
| 590 | index++; | ||
| 591 | n -= PAGE_SIZE; | ||
| 592 | } | ||
| 593 | } | ||
| 594 | |||
| 543 | static void zram_reset_device(struct zram *zram, bool reset_capacity) | 595 | static void zram_reset_device(struct zram *zram, bool reset_capacity) |
| 544 | { | 596 | { |
| 545 | size_t index; | 597 | size_t index; |
| 546 | struct zram_meta *meta; | 598 | struct zram_meta *meta; |
| 547 | 599 | ||
| 548 | down_write(&zram->init_lock); | 600 | down_write(&zram->init_lock); |
| 549 | if (!zram->init_done) { | 601 | if (!init_done(zram)) { |
| 550 | up_write(&zram->init_lock); | 602 | up_write(&zram->init_lock); |
| 551 | return; | 603 | return; |
| 552 | } | 604 | } |
| 553 | 605 | ||
| 554 | meta = zram->meta; | 606 | meta = zram->meta; |
| 555 | zram->init_done = 0; | ||
| 556 | |||
| 557 | /* Free all pages that are still in this zram device */ | 607 | /* Free all pages that are still in this zram device */ |
| 558 | for (index = 0; index < zram->disksize >> PAGE_SHIFT; index++) { | 608 | for (index = 0; index < zram->disksize >> PAGE_SHIFT; index++) { |
| 559 | unsigned long handle = meta->table[index].handle; | 609 | unsigned long handle = meta->table[index].handle; |
| @@ -563,6 +613,9 @@ static void zram_reset_device(struct zram *zram, bool reset_capacity) | |||
| 563 | zs_free(meta->mem_pool, handle); | 613 | zs_free(meta->mem_pool, handle); |
| 564 | } | 614 | } |
| 565 | 615 | ||
| 616 | zcomp_destroy(zram->comp); | ||
| 617 | zram->max_comp_streams = 1; | ||
| 618 | |||
| 566 | zram_meta_free(zram->meta); | 619 | zram_meta_free(zram->meta); |
| 567 | zram->meta = NULL; | 620 | zram->meta = NULL; |
| 568 | /* Reset stats */ | 621 | /* Reset stats */ |
| @@ -574,37 +627,14 @@ static void zram_reset_device(struct zram *zram, bool reset_capacity) | |||
| 574 | up_write(&zram->init_lock); | 627 | up_write(&zram->init_lock); |
| 575 | } | 628 | } |
| 576 | 629 | ||
| 577 | static void zram_init_device(struct zram *zram, struct zram_meta *meta) | ||
| 578 | { | ||
| 579 | if (zram->disksize > 2 * (totalram_pages << PAGE_SHIFT)) { | ||
| 580 | pr_info( | ||
| 581 | "There is little point creating a zram of greater than " | ||
| 582 | "twice the size of memory since we expect a 2:1 compression " | ||
| 583 | "ratio. Note that zram uses about 0.1%% of the size of " | ||
| 584 | "the disk when not in use so a huge zram is " | ||
| 585 | "wasteful.\n" | ||
| 586 | "\tMemory Size: %lu kB\n" | ||
| 587 | "\tSize you selected: %llu kB\n" | ||
| 588 | "Continuing anyway ...\n", | ||
| 589 | (totalram_pages << PAGE_SHIFT) >> 10, zram->disksize >> 10 | ||
| 590 | ); | ||
| 591 | } | ||
| 592 | |||
| 593 | /* zram devices sort of resembles non-rotational disks */ | ||
| 594 | queue_flag_set_unlocked(QUEUE_FLAG_NONROT, zram->disk->queue); | ||
| 595 | |||
| 596 | zram->meta = meta; | ||
| 597 | zram->init_done = 1; | ||
| 598 | |||
| 599 | pr_debug("Initialization done!\n"); | ||
| 600 | } | ||
| 601 | |||
| 602 | static ssize_t disksize_store(struct device *dev, | 630 | static ssize_t disksize_store(struct device *dev, |
| 603 | struct device_attribute *attr, const char *buf, size_t len) | 631 | struct device_attribute *attr, const char *buf, size_t len) |
| 604 | { | 632 | { |
| 605 | u64 disksize; | 633 | u64 disksize; |
| 634 | struct zcomp *comp; | ||
| 606 | struct zram_meta *meta; | 635 | struct zram_meta *meta; |
| 607 | struct zram *zram = dev_to_zram(dev); | 636 | struct zram *zram = dev_to_zram(dev); |
| 637 | int err; | ||
| 608 | 638 | ||
| 609 | disksize = memparse(buf, NULL); | 639 | disksize = memparse(buf, NULL); |
| 610 | if (!disksize) | 640 | if (!disksize) |
| @@ -614,20 +644,35 @@ static ssize_t disksize_store(struct device *dev, | |||
| 614 | meta = zram_meta_alloc(disksize); | 644 | meta = zram_meta_alloc(disksize); |
| 615 | if (!meta) | 645 | if (!meta) |
| 616 | return -ENOMEM; | 646 | return -ENOMEM; |
| 647 | |||
| 648 | comp = zcomp_create(zram->compressor, zram->max_comp_streams); | ||
| 649 | if (IS_ERR(comp)) { | ||
| 650 | pr_info("Cannot initialise %s compressing backend\n", | ||
| 651 | zram->compressor); | ||
| 652 | err = PTR_ERR(comp); | ||
| 653 | goto out_free_meta; | ||
| 654 | } | ||
| 655 | |||
| 617 | down_write(&zram->init_lock); | 656 | down_write(&zram->init_lock); |
| 618 | if (zram->init_done) { | 657 | if (init_done(zram)) { |
| 619 | up_write(&zram->init_lock); | ||
| 620 | zram_meta_free(meta); | ||
| 621 | pr_info("Cannot change disksize for initialized device\n"); | 658 | pr_info("Cannot change disksize for initialized device\n"); |
| 622 | return -EBUSY; | 659 | err = -EBUSY; |
| 660 | goto out_destroy_comp; | ||
| 623 | } | 661 | } |
| 624 | 662 | ||
| 663 | zram->meta = meta; | ||
| 664 | zram->comp = comp; | ||
| 625 | zram->disksize = disksize; | 665 | zram->disksize = disksize; |
| 626 | set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT); | 666 | set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT); |
| 627 | zram_init_device(zram, meta); | ||
| 628 | up_write(&zram->init_lock); | 667 | up_write(&zram->init_lock); |
| 629 | |||
| 630 | return len; | 668 | return len; |
| 669 | |||
| 670 | out_destroy_comp: | ||
| 671 | up_write(&zram->init_lock); | ||
| 672 | zcomp_destroy(comp); | ||
| 673 | out_free_meta: | ||
| 674 | zram_meta_free(meta); | ||
| 675 | return err; | ||
| 631 | } | 676 | } |
| 632 | 677 | ||
| 633 | static ssize_t reset_store(struct device *dev, | 678 | static ssize_t reset_store(struct device *dev, |
| @@ -671,26 +716,23 @@ out: | |||
| 671 | return ret; | 716 | return ret; |
| 672 | } | 717 | } |
| 673 | 718 | ||
| 674 | static void __zram_make_request(struct zram *zram, struct bio *bio, int rw) | 719 | static void __zram_make_request(struct zram *zram, struct bio *bio) |
| 675 | { | 720 | { |
| 676 | int offset; | 721 | int offset; |
| 677 | u32 index; | 722 | u32 index; |
| 678 | struct bio_vec bvec; | 723 | struct bio_vec bvec; |
| 679 | struct bvec_iter iter; | 724 | struct bvec_iter iter; |
| 680 | 725 | ||
| 681 | switch (rw) { | ||
| 682 | case READ: | ||
| 683 | atomic64_inc(&zram->stats.num_reads); | ||
| 684 | break; | ||
| 685 | case WRITE: | ||
| 686 | atomic64_inc(&zram->stats.num_writes); | ||
| 687 | break; | ||
| 688 | } | ||
| 689 | |||
| 690 | index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; | 726 | index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; |
| 691 | offset = (bio->bi_iter.bi_sector & | 727 | offset = (bio->bi_iter.bi_sector & |
| 692 | (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT; | 728 | (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT; |
| 693 | 729 | ||
| 730 | if (unlikely(bio->bi_rw & REQ_DISCARD)) { | ||
| 731 | zram_bio_discard(zram, index, offset, bio); | ||
| 732 | bio_endio(bio, 0); | ||
| 733 | return; | ||
| 734 | } | ||
| 735 | |||
| 694 | bio_for_each_segment(bvec, bio, iter) { | 736 | bio_for_each_segment(bvec, bio, iter) { |
| 695 | int max_transfer_size = PAGE_SIZE - offset; | 737 | int max_transfer_size = PAGE_SIZE - offset; |
| 696 | 738 | ||
| @@ -705,16 +747,15 @@ static void __zram_make_request(struct zram *zram, struct bio *bio, int rw) | |||
| 705 | bv.bv_len = max_transfer_size; | 747 | bv.bv_len = max_transfer_size; |
| 706 | bv.bv_offset = bvec.bv_offset; | 748 | bv.bv_offset = bvec.bv_offset; |
| 707 | 749 | ||
| 708 | if (zram_bvec_rw(zram, &bv, index, offset, bio, rw) < 0) | 750 | if (zram_bvec_rw(zram, &bv, index, offset, bio) < 0) |
| 709 | goto out; | 751 | goto out; |
| 710 | 752 | ||
| 711 | bv.bv_len = bvec.bv_len - max_transfer_size; | 753 | bv.bv_len = bvec.bv_len - max_transfer_size; |
| 712 | bv.bv_offset += max_transfer_size; | 754 | bv.bv_offset += max_transfer_size; |
| 713 | if (zram_bvec_rw(zram, &bv, index+1, 0, bio, rw) < 0) | 755 | if (zram_bvec_rw(zram, &bv, index + 1, 0, bio) < 0) |
| 714 | goto out; | 756 | goto out; |
| 715 | } else | 757 | } else |
| 716 | if (zram_bvec_rw(zram, &bvec, index, offset, bio, rw) | 758 | if (zram_bvec_rw(zram, &bvec, index, offset, bio) < 0) |
| 717 | < 0) | ||
| 718 | goto out; | 759 | goto out; |
| 719 | 760 | ||
| 720 | update_position(&index, &offset, &bvec); | 761 | update_position(&index, &offset, &bvec); |
| @@ -736,7 +777,7 @@ static void zram_make_request(struct request_queue *queue, struct bio *bio) | |||
| 736 | struct zram *zram = queue->queuedata; | 777 | struct zram *zram = queue->queuedata; |
| 737 | 778 | ||
| 738 | down_read(&zram->init_lock); | 779 | down_read(&zram->init_lock); |
| 739 | if (unlikely(!zram->init_done)) | 780 | if (unlikely(!init_done(zram))) |
| 740 | goto error; | 781 | goto error; |
| 741 | 782 | ||
| 742 | if (!valid_io_request(zram, bio)) { | 783 | if (!valid_io_request(zram, bio)) { |
| @@ -744,7 +785,7 @@ static void zram_make_request(struct request_queue *queue, struct bio *bio) | |||
| 744 | goto error; | 785 | goto error; |
| 745 | } | 786 | } |
| 746 | 787 | ||
| 747 | __zram_make_request(zram, bio, bio_data_dir(bio)); | 788 | __zram_make_request(zram, bio); |
| 748 | up_read(&zram->init_lock); | 789 | up_read(&zram->init_lock); |
| 749 | 790 | ||
| 750 | return; | 791 | return; |
| @@ -778,14 +819,21 @@ static DEVICE_ATTR(disksize, S_IRUGO | S_IWUSR, | |||
| 778 | disksize_show, disksize_store); | 819 | disksize_show, disksize_store); |
| 779 | static DEVICE_ATTR(initstate, S_IRUGO, initstate_show, NULL); | 820 | static DEVICE_ATTR(initstate, S_IRUGO, initstate_show, NULL); |
| 780 | static DEVICE_ATTR(reset, S_IWUSR, NULL, reset_store); | 821 | static DEVICE_ATTR(reset, S_IWUSR, NULL, reset_store); |
| 781 | static DEVICE_ATTR(num_reads, S_IRUGO, num_reads_show, NULL); | ||
| 782 | static DEVICE_ATTR(num_writes, S_IRUGO, num_writes_show, NULL); | ||
| 783 | static DEVICE_ATTR(invalid_io, S_IRUGO, invalid_io_show, NULL); | ||
| 784 | static DEVICE_ATTR(notify_free, S_IRUGO, notify_free_show, NULL); | ||
| 785 | static DEVICE_ATTR(zero_pages, S_IRUGO, zero_pages_show, NULL); | ||
| 786 | static DEVICE_ATTR(orig_data_size, S_IRUGO, orig_data_size_show, NULL); | 822 | static DEVICE_ATTR(orig_data_size, S_IRUGO, orig_data_size_show, NULL); |
| 787 | static DEVICE_ATTR(compr_data_size, S_IRUGO, compr_data_size_show, NULL); | ||
| 788 | static DEVICE_ATTR(mem_used_total, S_IRUGO, mem_used_total_show, NULL); | 823 | static DEVICE_ATTR(mem_used_total, S_IRUGO, mem_used_total_show, NULL); |
| 824 | static DEVICE_ATTR(max_comp_streams, S_IRUGO | S_IWUSR, | ||
| 825 | max_comp_streams_show, max_comp_streams_store); | ||
| 826 | static DEVICE_ATTR(comp_algorithm, S_IRUGO | S_IWUSR, | ||
| 827 | comp_algorithm_show, comp_algorithm_store); | ||
| 828 | |||
| 829 | ZRAM_ATTR_RO(num_reads); | ||
| 830 | ZRAM_ATTR_RO(num_writes); | ||
| 831 | ZRAM_ATTR_RO(failed_reads); | ||
| 832 | ZRAM_ATTR_RO(failed_writes); | ||
| 833 | ZRAM_ATTR_RO(invalid_io); | ||
| 834 | ZRAM_ATTR_RO(notify_free); | ||
| 835 | ZRAM_ATTR_RO(zero_pages); | ||
| 836 | ZRAM_ATTR_RO(compr_data_size); | ||
| 789 | 837 | ||
| 790 | static struct attribute *zram_disk_attrs[] = { | 838 | static struct attribute *zram_disk_attrs[] = { |
| 791 | &dev_attr_disksize.attr, | 839 | &dev_attr_disksize.attr, |
| @@ -793,12 +841,16 @@ static struct attribute *zram_disk_attrs[] = { | |||
| 793 | &dev_attr_reset.attr, | 841 | &dev_attr_reset.attr, |
| 794 | &dev_attr_num_reads.attr, | 842 | &dev_attr_num_reads.attr, |
| 795 | &dev_attr_num_writes.attr, | 843 | &dev_attr_num_writes.attr, |
| 844 | &dev_attr_failed_reads.attr, | ||
| 845 | &dev_attr_failed_writes.attr, | ||
| 796 | &dev_attr_invalid_io.attr, | 846 | &dev_attr_invalid_io.attr, |
| 797 | &dev_attr_notify_free.attr, | 847 | &dev_attr_notify_free.attr, |
| 798 | &dev_attr_zero_pages.attr, | 848 | &dev_attr_zero_pages.attr, |
| 799 | &dev_attr_orig_data_size.attr, | 849 | &dev_attr_orig_data_size.attr, |
| 800 | &dev_attr_compr_data_size.attr, | 850 | &dev_attr_compr_data_size.attr, |
| 801 | &dev_attr_mem_used_total.attr, | 851 | &dev_attr_mem_used_total.attr, |
| 852 | &dev_attr_max_comp_streams.attr, | ||
| 853 | &dev_attr_comp_algorithm.attr, | ||
| 802 | NULL, | 854 | NULL, |
| 803 | }; | 855 | }; |
| 804 | 856 | ||
| @@ -839,7 +891,8 @@ static int create_device(struct zram *zram, int device_id) | |||
| 839 | 891 | ||
| 840 | /* Actual capacity set using syfs (/sys/block/zram<id>/disksize */ | 892 | /* Actual capacity set using syfs (/sys/block/zram<id>/disksize */ |
| 841 | set_capacity(zram->disk, 0); | 893 | set_capacity(zram->disk, 0); |
| 842 | 894 | /* zram devices sort of resembles non-rotational disks */ | |
| 895 | queue_flag_set_unlocked(QUEUE_FLAG_NONROT, zram->disk->queue); | ||
| 843 | /* | 896 | /* |
| 844 | * To ensure that we always get PAGE_SIZE aligned | 897 | * To ensure that we always get PAGE_SIZE aligned |
| 845 | * and n*PAGE_SIZED sized I/O requests. | 898 | * and n*PAGE_SIZED sized I/O requests. |
| @@ -849,6 +902,21 @@ static int create_device(struct zram *zram, int device_id) | |||
| 849 | ZRAM_LOGICAL_BLOCK_SIZE); | 902 | ZRAM_LOGICAL_BLOCK_SIZE); |
| 850 | blk_queue_io_min(zram->disk->queue, PAGE_SIZE); | 903 | blk_queue_io_min(zram->disk->queue, PAGE_SIZE); |
| 851 | blk_queue_io_opt(zram->disk->queue, PAGE_SIZE); | 904 | blk_queue_io_opt(zram->disk->queue, PAGE_SIZE); |
| 905 | zram->disk->queue->limits.discard_granularity = PAGE_SIZE; | ||
| 906 | zram->disk->queue->limits.max_discard_sectors = UINT_MAX; | ||
| 907 | /* | ||
| 908 | * zram_bio_discard() will clear all logical blocks if logical block | ||
| 909 | * size is identical with physical block size(PAGE_SIZE). But if it is | ||
| 910 | * different, we will skip discarding some parts of logical blocks in | ||
| 911 | * the part of the request range which isn't aligned to physical block | ||
| 912 | * size. So we can't ensure that all discarded logical blocks are | ||
| 913 | * zeroed. | ||
| 914 | */ | ||
| 915 | if (ZRAM_LOGICAL_BLOCK_SIZE == PAGE_SIZE) | ||
| 916 | zram->disk->queue->limits.discard_zeroes_data = 1; | ||
| 917 | else | ||
| 918 | zram->disk->queue->limits.discard_zeroes_data = 0; | ||
| 919 | queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, zram->disk->queue); | ||
| 852 | 920 | ||
| 853 | add_disk(zram->disk); | 921 | add_disk(zram->disk); |
| 854 | 922 | ||
| @@ -858,8 +926,9 @@ static int create_device(struct zram *zram, int device_id) | |||
| 858 | pr_warn("Error creating sysfs group"); | 926 | pr_warn("Error creating sysfs group"); |
| 859 | goto out_free_disk; | 927 | goto out_free_disk; |
| 860 | } | 928 | } |
| 861 | 929 | strlcpy(zram->compressor, default_compressor, sizeof(zram->compressor)); | |
| 862 | zram->init_done = 0; | 930 | zram->meta = NULL; |
| 931 | zram->max_comp_streams = 1; | ||
| 863 | return 0; | 932 | return 0; |
| 864 | 933 | ||
| 865 | out_free_disk: | 934 | out_free_disk: |
diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h index ad8aa35bae00..7f21c145e317 100644 --- a/drivers/block/zram/zram_drv.h +++ b/drivers/block/zram/zram_drv.h | |||
| @@ -16,9 +16,10 @@ | |||
| 16 | #define _ZRAM_DRV_H_ | 16 | #define _ZRAM_DRV_H_ |
| 17 | 17 | ||
| 18 | #include <linux/spinlock.h> | 18 | #include <linux/spinlock.h> |
| 19 | #include <linux/mutex.h> | ||
| 20 | #include <linux/zsmalloc.h> | 19 | #include <linux/zsmalloc.h> |
| 21 | 20 | ||
| 21 | #include "zcomp.h" | ||
| 22 | |||
| 22 | /* | 23 | /* |
| 23 | * Some arbitrary value. This is just to catch | 24 | * Some arbitrary value. This is just to catch |
| 24 | * invalid value for num_devices module parameter. | 25 | * invalid value for num_devices module parameter. |
| @@ -64,38 +65,33 @@ enum zram_pageflags { | |||
| 64 | struct table { | 65 | struct table { |
| 65 | unsigned long handle; | 66 | unsigned long handle; |
| 66 | u16 size; /* object size (excluding header) */ | 67 | u16 size; /* object size (excluding header) */ |
| 67 | u8 count; /* object ref count (not yet used) */ | ||
| 68 | u8 flags; | 68 | u8 flags; |
| 69 | } __aligned(4); | 69 | } __aligned(4); |
| 70 | 70 | ||
| 71 | struct zram_stats { | 71 | struct zram_stats { |
| 72 | atomic64_t compr_size; /* compressed size of pages stored */ | 72 | atomic64_t compr_data_size; /* compressed size of pages stored */ |
| 73 | atomic64_t num_reads; /* failed + successful */ | 73 | atomic64_t num_reads; /* failed + successful */ |
| 74 | atomic64_t num_writes; /* --do-- */ | 74 | atomic64_t num_writes; /* --do-- */ |
| 75 | atomic64_t failed_reads; /* should NEVER! happen */ | 75 | atomic64_t failed_reads; /* should NEVER! happen */ |
| 76 | atomic64_t failed_writes; /* can happen when memory is too low */ | 76 | atomic64_t failed_writes; /* can happen when memory is too low */ |
| 77 | atomic64_t invalid_io; /* non-page-aligned I/O requests */ | 77 | atomic64_t invalid_io; /* non-page-aligned I/O requests */ |
| 78 | atomic64_t notify_free; /* no. of swap slot free notifications */ | 78 | atomic64_t notify_free; /* no. of swap slot free notifications */ |
| 79 | atomic_t pages_zero; /* no. of zero filled pages */ | 79 | atomic64_t zero_pages; /* no. of zero filled pages */ |
| 80 | atomic_t pages_stored; /* no. of pages currently stored */ | 80 | atomic64_t pages_stored; /* no. of pages currently stored */ |
| 81 | atomic_t good_compress; /* % of pages with compression ratio<=50% */ | ||
| 82 | atomic_t bad_compress; /* % of pages with compression ratio>=75% */ | ||
| 83 | }; | 81 | }; |
| 84 | 82 | ||
| 85 | struct zram_meta { | 83 | struct zram_meta { |
| 86 | rwlock_t tb_lock; /* protect table */ | 84 | rwlock_t tb_lock; /* protect table */ |
| 87 | void *compress_workmem; | ||
| 88 | void *compress_buffer; | ||
| 89 | struct table *table; | 85 | struct table *table; |
| 90 | struct zs_pool *mem_pool; | 86 | struct zs_pool *mem_pool; |
| 91 | struct mutex buffer_lock; /* protect compress buffers */ | ||
| 92 | }; | 87 | }; |
| 93 | 88 | ||
| 94 | struct zram { | 89 | struct zram { |
| 95 | struct zram_meta *meta; | 90 | struct zram_meta *meta; |
| 96 | struct request_queue *queue; | 91 | struct request_queue *queue; |
| 97 | struct gendisk *disk; | 92 | struct gendisk *disk; |
| 98 | int init_done; | 93 | struct zcomp *comp; |
| 94 | |||
| 99 | /* Prevent concurrent execution of device init, reset and R/W request */ | 95 | /* Prevent concurrent execution of device init, reset and R/W request */ |
| 100 | struct rw_semaphore init_lock; | 96 | struct rw_semaphore init_lock; |
| 101 | /* | 97 | /* |
| @@ -103,7 +99,8 @@ struct zram { | |||
| 103 | * we can store in a disk. | 99 | * we can store in a disk. |
| 104 | */ | 100 | */ |
| 105 | u64 disksize; /* bytes */ | 101 | u64 disksize; /* bytes */ |
| 106 | 102 | int max_comp_streams; | |
| 107 | struct zram_stats stats; | 103 | struct zram_stats stats; |
| 104 | char compressor[10]; | ||
| 108 | }; | 105 | }; |
| 109 | #endif | 106 | #endif |
