aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/block
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2014-04-07 19:38:06 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2014-04-07 19:38:06 -0400
commit26c12d93348f0bda0756aff83f4867d9ae58a5a6 (patch)
tree65221f6837c66a9260c5c973e5fb908b10e0d504 /drivers/block
parentdc5ed40686a4da95881c35d913b60f867755cbe2 (diff)
parentfdc5813fbbd484a54c88477f91a78934cda8bb32 (diff)
Merge branch 'akpm' (incoming from Andrew)
Merge second patch-bomb from Andrew Morton: - the rest of MM - zram updates - zswap updates - exit - procfs - exec - wait - crash dump - lib/idr - rapidio - adfs, affs, bfs, ufs - cris - Kconfig things - initramfs - small amount of IPC material - percpu enhancements - early ioremap support - various other misc things * emailed patches from Andrew Morton <akpm@linux-foundation.org>: (156 commits) MAINTAINERS: update Intel C600 SAS driver maintainers fs/ufs: remove unused ufs_super_block_third pointer fs/ufs: remove unused ufs_super_block_second pointer fs/ufs: remove unused ufs_super_block_first pointer fs/ufs/super.c: add __init to init_inodecache() doc/kernel-parameters.txt: add early_ioremap_debug arm64: add early_ioremap support arm64: initialize pgprot info earlier in boot x86: use generic early_ioremap mm: create generic early_ioremap() support x86/mm: sparse warning fix for early_memremap lglock: map to spinlock when !CONFIG_SMP percpu: add preemption checks to __this_cpu ops vmstat: use raw_cpu_ops to avoid false positives on preemption checks slub: use raw_cpu_inc for incrementing statistics net: replace __this_cpu_inc in route.c with raw_cpu_inc modules: use raw_cpu_write for initialization of per cpu refcount. mm: use raw_cpu ops for determining current NUMA node percpu: add raw_cpu_ops slub: fix leak of 'name' in sysfs_slab_add ...
Diffstat (limited to 'drivers/block')
-rw-r--r--drivers/block/zram/Kconfig10
-rw-r--r--drivers/block/zram/Makefile4
-rw-r--r--drivers/block/zram/zcomp.c353
-rw-r--r--drivers/block/zram/zcomp.h68
-rw-r--r--drivers/block/zram/zcomp_lz4.c47
-rw-r--r--drivers/block/zram/zcomp_lz4.h17
-rw-r--r--drivers/block/zram/zcomp_lzo.c47
-rw-r--r--drivers/block/zram/zcomp_lzo.h17
-rw-r--r--drivers/block/zram/zram_drv.c383
-rw-r--r--drivers/block/zram/zram_drv.h21
10 files changed, 797 insertions, 170 deletions
diff --git a/drivers/block/zram/Kconfig b/drivers/block/zram/Kconfig
index 3450be850399..6489c0fd0ea6 100644
--- a/drivers/block/zram/Kconfig
+++ b/drivers/block/zram/Kconfig
@@ -15,6 +15,16 @@ config ZRAM
15 15
16 See zram.txt for more information. 16 See zram.txt for more information.
17 17
18config ZRAM_LZ4_COMPRESS
19 bool "Enable LZ4 algorithm support"
20 depends on ZRAM
21 select LZ4_COMPRESS
22 select LZ4_DECOMPRESS
23 default n
24 help
25 This option enables LZ4 compression algorithm support. Compression
26 algorithm can be changed using `comp_algorithm' device attribute.
27
18config ZRAM_DEBUG 28config ZRAM_DEBUG
19 bool "Compressed RAM block device debug support" 29 bool "Compressed RAM block device debug support"
20 depends on ZRAM 30 depends on ZRAM
diff --git a/drivers/block/zram/Makefile b/drivers/block/zram/Makefile
index cb0f9ced6a93..be0763ff57a2 100644
--- a/drivers/block/zram/Makefile
+++ b/drivers/block/zram/Makefile
@@ -1,3 +1,5 @@
1zram-y := zram_drv.o 1zram-y := zcomp_lzo.o zcomp.o zram_drv.o
2
3zram-$(CONFIG_ZRAM_LZ4_COMPRESS) += zcomp_lz4.o
2 4
3obj-$(CONFIG_ZRAM) += zram.o 5obj-$(CONFIG_ZRAM) += zram.o
diff --git a/drivers/block/zram/zcomp.c b/drivers/block/zram/zcomp.c
new file mode 100644
index 000000000000..f1ff39a3d1c1
--- /dev/null
+++ b/drivers/block/zram/zcomp.c
@@ -0,0 +1,353 @@
1/*
2 * Copyright (C) 2014 Sergey Senozhatsky.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */
9
10#include <linux/kernel.h>
11#include <linux/string.h>
12#include <linux/err.h>
13#include <linux/slab.h>
14#include <linux/wait.h>
15#include <linux/sched.h>
16
17#include "zcomp.h"
18#include "zcomp_lzo.h"
19#ifdef CONFIG_ZRAM_LZ4_COMPRESS
20#include "zcomp_lz4.h"
21#endif
22
23/*
24 * single zcomp_strm backend
25 */
26struct zcomp_strm_single {
27 struct mutex strm_lock;
28 struct zcomp_strm *zstrm;
29};
30
31/*
32 * multi zcomp_strm backend
33 */
34struct zcomp_strm_multi {
35 /* protect strm list */
36 spinlock_t strm_lock;
37 /* max possible number of zstrm streams */
38 int max_strm;
39 /* number of available zstrm streams */
40 int avail_strm;
41 /* list of available strms */
42 struct list_head idle_strm;
43 wait_queue_head_t strm_wait;
44};
45
46static struct zcomp_backend *backends[] = {
47 &zcomp_lzo,
48#ifdef CONFIG_ZRAM_LZ4_COMPRESS
49 &zcomp_lz4,
50#endif
51 NULL
52};
53
54static struct zcomp_backend *find_backend(const char *compress)
55{
56 int i = 0;
57 while (backends[i]) {
58 if (sysfs_streq(compress, backends[i]->name))
59 break;
60 i++;
61 }
62 return backends[i];
63}
64
65static void zcomp_strm_free(struct zcomp *comp, struct zcomp_strm *zstrm)
66{
67 if (zstrm->private)
68 comp->backend->destroy(zstrm->private);
69 free_pages((unsigned long)zstrm->buffer, 1);
70 kfree(zstrm);
71}
72
73/*
74 * allocate new zcomp_strm structure with ->private initialized by
75 * backend, return NULL on error
76 */
77static struct zcomp_strm *zcomp_strm_alloc(struct zcomp *comp)
78{
79 struct zcomp_strm *zstrm = kmalloc(sizeof(*zstrm), GFP_KERNEL);
80 if (!zstrm)
81 return NULL;
82
83 zstrm->private = comp->backend->create();
84 /*
85 * allocate 2 pages. 1 for compressed data, plus 1 extra for the
86 * case when compressed size is larger than the original one
87 */
88 zstrm->buffer = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1);
89 if (!zstrm->private || !zstrm->buffer) {
90 zcomp_strm_free(comp, zstrm);
91 zstrm = NULL;
92 }
93 return zstrm;
94}
95
96/*
97 * get idle zcomp_strm or wait until other process release
98 * (zcomp_strm_release()) one for us
99 */
100static struct zcomp_strm *zcomp_strm_multi_find(struct zcomp *comp)
101{
102 struct zcomp_strm_multi *zs = comp->stream;
103 struct zcomp_strm *zstrm;
104
105 while (1) {
106 spin_lock(&zs->strm_lock);
107 if (!list_empty(&zs->idle_strm)) {
108 zstrm = list_entry(zs->idle_strm.next,
109 struct zcomp_strm, list);
110 list_del(&zstrm->list);
111 spin_unlock(&zs->strm_lock);
112 return zstrm;
113 }
114 /* zstrm streams limit reached, wait for idle stream */
115 if (zs->avail_strm >= zs->max_strm) {
116 spin_unlock(&zs->strm_lock);
117 wait_event(zs->strm_wait, !list_empty(&zs->idle_strm));
118 continue;
119 }
120 /* allocate new zstrm stream */
121 zs->avail_strm++;
122 spin_unlock(&zs->strm_lock);
123
124 zstrm = zcomp_strm_alloc(comp);
125 if (!zstrm) {
126 spin_lock(&zs->strm_lock);
127 zs->avail_strm--;
128 spin_unlock(&zs->strm_lock);
129 wait_event(zs->strm_wait, !list_empty(&zs->idle_strm));
130 continue;
131 }
132 break;
133 }
134 return zstrm;
135}
136
137/* add stream back to idle list and wake up waiter or free the stream */
138static void zcomp_strm_multi_release(struct zcomp *comp, struct zcomp_strm *zstrm)
139{
140 struct zcomp_strm_multi *zs = comp->stream;
141
142 spin_lock(&zs->strm_lock);
143 if (zs->avail_strm <= zs->max_strm) {
144 list_add(&zstrm->list, &zs->idle_strm);
145 spin_unlock(&zs->strm_lock);
146 wake_up(&zs->strm_wait);
147 return;
148 }
149
150 zs->avail_strm--;
151 spin_unlock(&zs->strm_lock);
152 zcomp_strm_free(comp, zstrm);
153}
154
155/* change max_strm limit */
156static bool zcomp_strm_multi_set_max_streams(struct zcomp *comp, int num_strm)
157{
158 struct zcomp_strm_multi *zs = comp->stream;
159 struct zcomp_strm *zstrm;
160
161 spin_lock(&zs->strm_lock);
162 zs->max_strm = num_strm;
163 /*
164 * if user has lowered the limit and there are idle streams,
165 * immediately free as much streams (and memory) as we can.
166 */
167 while (zs->avail_strm > num_strm && !list_empty(&zs->idle_strm)) {
168 zstrm = list_entry(zs->idle_strm.next,
169 struct zcomp_strm, list);
170 list_del(&zstrm->list);
171 zcomp_strm_free(comp, zstrm);
172 zs->avail_strm--;
173 }
174 spin_unlock(&zs->strm_lock);
175 return true;
176}
177
178static void zcomp_strm_multi_destroy(struct zcomp *comp)
179{
180 struct zcomp_strm_multi *zs = comp->stream;
181 struct zcomp_strm *zstrm;
182
183 while (!list_empty(&zs->idle_strm)) {
184 zstrm = list_entry(zs->idle_strm.next,
185 struct zcomp_strm, list);
186 list_del(&zstrm->list);
187 zcomp_strm_free(comp, zstrm);
188 }
189 kfree(zs);
190}
191
192static int zcomp_strm_multi_create(struct zcomp *comp, int max_strm)
193{
194 struct zcomp_strm *zstrm;
195 struct zcomp_strm_multi *zs;
196
197 comp->destroy = zcomp_strm_multi_destroy;
198 comp->strm_find = zcomp_strm_multi_find;
199 comp->strm_release = zcomp_strm_multi_release;
200 comp->set_max_streams = zcomp_strm_multi_set_max_streams;
201 zs = kmalloc(sizeof(struct zcomp_strm_multi), GFP_KERNEL);
202 if (!zs)
203 return -ENOMEM;
204
205 comp->stream = zs;
206 spin_lock_init(&zs->strm_lock);
207 INIT_LIST_HEAD(&zs->idle_strm);
208 init_waitqueue_head(&zs->strm_wait);
209 zs->max_strm = max_strm;
210 zs->avail_strm = 1;
211
212 zstrm = zcomp_strm_alloc(comp);
213 if (!zstrm) {
214 kfree(zs);
215 return -ENOMEM;
216 }
217 list_add(&zstrm->list, &zs->idle_strm);
218 return 0;
219}
220
221static struct zcomp_strm *zcomp_strm_single_find(struct zcomp *comp)
222{
223 struct zcomp_strm_single *zs = comp->stream;
224 mutex_lock(&zs->strm_lock);
225 return zs->zstrm;
226}
227
228static void zcomp_strm_single_release(struct zcomp *comp,
229 struct zcomp_strm *zstrm)
230{
231 struct zcomp_strm_single *zs = comp->stream;
232 mutex_unlock(&zs->strm_lock);
233}
234
235static bool zcomp_strm_single_set_max_streams(struct zcomp *comp, int num_strm)
236{
237 /* zcomp_strm_single support only max_comp_streams == 1 */
238 return false;
239}
240
241static void zcomp_strm_single_destroy(struct zcomp *comp)
242{
243 struct zcomp_strm_single *zs = comp->stream;
244 zcomp_strm_free(comp, zs->zstrm);
245 kfree(zs);
246}
247
248static int zcomp_strm_single_create(struct zcomp *comp)
249{
250 struct zcomp_strm_single *zs;
251
252 comp->destroy = zcomp_strm_single_destroy;
253 comp->strm_find = zcomp_strm_single_find;
254 comp->strm_release = zcomp_strm_single_release;
255 comp->set_max_streams = zcomp_strm_single_set_max_streams;
256 zs = kmalloc(sizeof(struct zcomp_strm_single), GFP_KERNEL);
257 if (!zs)
258 return -ENOMEM;
259
260 comp->stream = zs;
261 mutex_init(&zs->strm_lock);
262 zs->zstrm = zcomp_strm_alloc(comp);
263 if (!zs->zstrm) {
264 kfree(zs);
265 return -ENOMEM;
266 }
267 return 0;
268}
269
270/* show available compressors */
271ssize_t zcomp_available_show(const char *comp, char *buf)
272{
273 ssize_t sz = 0;
274 int i = 0;
275
276 while (backends[i]) {
277 if (sysfs_streq(comp, backends[i]->name))
278 sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2,
279 "[%s] ", backends[i]->name);
280 else
281 sz += scnprintf(buf + sz, PAGE_SIZE - sz - 2,
282 "%s ", backends[i]->name);
283 i++;
284 }
285 sz += scnprintf(buf + sz, PAGE_SIZE - sz, "\n");
286 return sz;
287}
288
289bool zcomp_set_max_streams(struct zcomp *comp, int num_strm)
290{
291 return comp->set_max_streams(comp, num_strm);
292}
293
294struct zcomp_strm *zcomp_strm_find(struct zcomp *comp)
295{
296 return comp->strm_find(comp);
297}
298
299void zcomp_strm_release(struct zcomp *comp, struct zcomp_strm *zstrm)
300{
301 comp->strm_release(comp, zstrm);
302}
303
304int zcomp_compress(struct zcomp *comp, struct zcomp_strm *zstrm,
305 const unsigned char *src, size_t *dst_len)
306{
307 return comp->backend->compress(src, zstrm->buffer, dst_len,
308 zstrm->private);
309}
310
311int zcomp_decompress(struct zcomp *comp, const unsigned char *src,
312 size_t src_len, unsigned char *dst)
313{
314 return comp->backend->decompress(src, src_len, dst);
315}
316
317void zcomp_destroy(struct zcomp *comp)
318{
319 comp->destroy(comp);
320 kfree(comp);
321}
322
323/*
324 * search available compressors for requested algorithm.
325 * allocate new zcomp and initialize it. return compressing
326 * backend pointer or ERR_PTR if things went bad. ERR_PTR(-EINVAL)
327 * if requested algorithm is not supported, ERR_PTR(-ENOMEM) in
328 * case of allocation error.
329 */
330struct zcomp *zcomp_create(const char *compress, int max_strm)
331{
332 struct zcomp *comp;
333 struct zcomp_backend *backend;
334
335 backend = find_backend(compress);
336 if (!backend)
337 return ERR_PTR(-EINVAL);
338
339 comp = kzalloc(sizeof(struct zcomp), GFP_KERNEL);
340 if (!comp)
341 return ERR_PTR(-ENOMEM);
342
343 comp->backend = backend;
344 if (max_strm > 1)
345 zcomp_strm_multi_create(comp, max_strm);
346 else
347 zcomp_strm_single_create(comp);
348 if (!comp->stream) {
349 kfree(comp);
350 return ERR_PTR(-ENOMEM);
351 }
352 return comp;
353}
diff --git a/drivers/block/zram/zcomp.h b/drivers/block/zram/zcomp.h
new file mode 100644
index 000000000000..c59d1fca72c0
--- /dev/null
+++ b/drivers/block/zram/zcomp.h
@@ -0,0 +1,68 @@
1/*
2 * Copyright (C) 2014 Sergey Senozhatsky.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */
9
10#ifndef _ZCOMP_H_
11#define _ZCOMP_H_
12
13#include <linux/mutex.h>
14
15struct zcomp_strm {
16 /* compression/decompression buffer */
17 void *buffer;
18 /*
19 * The private data of the compression stream, only compression
20 * stream backend can touch this (e.g. compression algorithm
21 * working memory)
22 */
23 void *private;
24 /* used in multi stream backend, protected by backend strm_lock */
25 struct list_head list;
26};
27
28/* static compression backend */
29struct zcomp_backend {
30 int (*compress)(const unsigned char *src, unsigned char *dst,
31 size_t *dst_len, void *private);
32
33 int (*decompress)(const unsigned char *src, size_t src_len,
34 unsigned char *dst);
35
36 void *(*create)(void);
37 void (*destroy)(void *private);
38
39 const char *name;
40};
41
42/* dynamic per-device compression frontend */
43struct zcomp {
44 void *stream;
45 struct zcomp_backend *backend;
46
47 struct zcomp_strm *(*strm_find)(struct zcomp *comp);
48 void (*strm_release)(struct zcomp *comp, struct zcomp_strm *zstrm);
49 bool (*set_max_streams)(struct zcomp *comp, int num_strm);
50 void (*destroy)(struct zcomp *comp);
51};
52
53ssize_t zcomp_available_show(const char *comp, char *buf);
54
55struct zcomp *zcomp_create(const char *comp, int max_strm);
56void zcomp_destroy(struct zcomp *comp);
57
58struct zcomp_strm *zcomp_strm_find(struct zcomp *comp);
59void zcomp_strm_release(struct zcomp *comp, struct zcomp_strm *zstrm);
60
61int zcomp_compress(struct zcomp *comp, struct zcomp_strm *zstrm,
62 const unsigned char *src, size_t *dst_len);
63
64int zcomp_decompress(struct zcomp *comp, const unsigned char *src,
65 size_t src_len, unsigned char *dst);
66
67bool zcomp_set_max_streams(struct zcomp *comp, int num_strm);
68#endif /* _ZCOMP_H_ */
diff --git a/drivers/block/zram/zcomp_lz4.c b/drivers/block/zram/zcomp_lz4.c
new file mode 100644
index 000000000000..f2afb7e988c3
--- /dev/null
+++ b/drivers/block/zram/zcomp_lz4.c
@@ -0,0 +1,47 @@
1/*
2 * Copyright (C) 2014 Sergey Senozhatsky.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */
9
10#include <linux/kernel.h>
11#include <linux/slab.h>
12#include <linux/lz4.h>
13
14#include "zcomp_lz4.h"
15
16static void *zcomp_lz4_create(void)
17{
18 return kzalloc(LZ4_MEM_COMPRESS, GFP_KERNEL);
19}
20
21static void zcomp_lz4_destroy(void *private)
22{
23 kfree(private);
24}
25
26static int zcomp_lz4_compress(const unsigned char *src, unsigned char *dst,
27 size_t *dst_len, void *private)
28{
29 /* return : Success if return 0 */
30 return lz4_compress(src, PAGE_SIZE, dst, dst_len, private);
31}
32
33static int zcomp_lz4_decompress(const unsigned char *src, size_t src_len,
34 unsigned char *dst)
35{
36 size_t dst_len = PAGE_SIZE;
37 /* return : Success if return 0 */
38 return lz4_decompress_unknownoutputsize(src, src_len, dst, &dst_len);
39}
40
41struct zcomp_backend zcomp_lz4 = {
42 .compress = zcomp_lz4_compress,
43 .decompress = zcomp_lz4_decompress,
44 .create = zcomp_lz4_create,
45 .destroy = zcomp_lz4_destroy,
46 .name = "lz4",
47};
diff --git a/drivers/block/zram/zcomp_lz4.h b/drivers/block/zram/zcomp_lz4.h
new file mode 100644
index 000000000000..60613fb29dd8
--- /dev/null
+++ b/drivers/block/zram/zcomp_lz4.h
@@ -0,0 +1,17 @@
1/*
2 * Copyright (C) 2014 Sergey Senozhatsky.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */
9
10#ifndef _ZCOMP_LZ4_H_
11#define _ZCOMP_LZ4_H_
12
13#include "zcomp.h"
14
15extern struct zcomp_backend zcomp_lz4;
16
17#endif /* _ZCOMP_LZ4_H_ */
diff --git a/drivers/block/zram/zcomp_lzo.c b/drivers/block/zram/zcomp_lzo.c
new file mode 100644
index 000000000000..da1bc47d588e
--- /dev/null
+++ b/drivers/block/zram/zcomp_lzo.c
@@ -0,0 +1,47 @@
1/*
2 * Copyright (C) 2014 Sergey Senozhatsky.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */
9
10#include <linux/kernel.h>
11#include <linux/slab.h>
12#include <linux/lzo.h>
13
14#include "zcomp_lzo.h"
15
16static void *lzo_create(void)
17{
18 return kzalloc(LZO1X_MEM_COMPRESS, GFP_KERNEL);
19}
20
21static void lzo_destroy(void *private)
22{
23 kfree(private);
24}
25
26static int lzo_compress(const unsigned char *src, unsigned char *dst,
27 size_t *dst_len, void *private)
28{
29 int ret = lzo1x_1_compress(src, PAGE_SIZE, dst, dst_len, private);
30 return ret == LZO_E_OK ? 0 : ret;
31}
32
33static int lzo_decompress(const unsigned char *src, size_t src_len,
34 unsigned char *dst)
35{
36 size_t dst_len = PAGE_SIZE;
37 int ret = lzo1x_decompress_safe(src, src_len, dst, &dst_len);
38 return ret == LZO_E_OK ? 0 : ret;
39}
40
41struct zcomp_backend zcomp_lzo = {
42 .compress = lzo_compress,
43 .decompress = lzo_decompress,
44 .create = lzo_create,
45 .destroy = lzo_destroy,
46 .name = "lzo",
47};
diff --git a/drivers/block/zram/zcomp_lzo.h b/drivers/block/zram/zcomp_lzo.h
new file mode 100644
index 000000000000..128c5807fa14
--- /dev/null
+++ b/drivers/block/zram/zcomp_lzo.h
@@ -0,0 +1,17 @@
1/*
2 * Copyright (C) 2014 Sergey Senozhatsky.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */
9
10#ifndef _ZCOMP_LZO_H_
11#define _ZCOMP_LZO_H_
12
13#include "zcomp.h"
14
15extern struct zcomp_backend zcomp_lzo;
16
17#endif /* _ZCOMP_LZO_H_ */
diff --git a/drivers/block/zram/zram_drv.c b/drivers/block/zram/zram_drv.c
index 51c557cfd92b..9849b5233bf4 100644
--- a/drivers/block/zram/zram_drv.c
+++ b/drivers/block/zram/zram_drv.c
@@ -29,19 +29,36 @@
29#include <linux/genhd.h> 29#include <linux/genhd.h>
30#include <linux/highmem.h> 30#include <linux/highmem.h>
31#include <linux/slab.h> 31#include <linux/slab.h>
32#include <linux/lzo.h>
33#include <linux/string.h> 32#include <linux/string.h>
34#include <linux/vmalloc.h> 33#include <linux/vmalloc.h>
34#include <linux/err.h>
35 35
36#include "zram_drv.h" 36#include "zram_drv.h"
37 37
38/* Globals */ 38/* Globals */
39static int zram_major; 39static int zram_major;
40static struct zram *zram_devices; 40static struct zram *zram_devices;
41static const char *default_compressor = "lzo";
41 42
42/* Module params (documentation at end) */ 43/* Module params (documentation at end) */
43static unsigned int num_devices = 1; 44static unsigned int num_devices = 1;
44 45
46#define ZRAM_ATTR_RO(name) \
47static ssize_t zram_attr_##name##_show(struct device *d, \
48 struct device_attribute *attr, char *b) \
49{ \
50 struct zram *zram = dev_to_zram(d); \
51 return scnprintf(b, PAGE_SIZE, "%llu\n", \
52 (u64)atomic64_read(&zram->stats.name)); \
53} \
54static struct device_attribute dev_attr_##name = \
55 __ATTR(name, S_IRUGO, zram_attr_##name##_show, NULL);
56
57static inline int init_done(struct zram *zram)
58{
59 return zram->meta != NULL;
60}
61
45static inline struct zram *dev_to_zram(struct device *dev) 62static inline struct zram *dev_to_zram(struct device *dev)
46{ 63{
47 return (struct zram *)dev_to_disk(dev)->private_data; 64 return (struct zram *)dev_to_disk(dev)->private_data;
@@ -52,92 +69,114 @@ static ssize_t disksize_show(struct device *dev,
52{ 69{
53 struct zram *zram = dev_to_zram(dev); 70 struct zram *zram = dev_to_zram(dev);
54 71
55 return sprintf(buf, "%llu\n", zram->disksize); 72 return scnprintf(buf, PAGE_SIZE, "%llu\n", zram->disksize);
56} 73}
57 74
58static ssize_t initstate_show(struct device *dev, 75static ssize_t initstate_show(struct device *dev,
59 struct device_attribute *attr, char *buf) 76 struct device_attribute *attr, char *buf)
60{ 77{
78 u32 val;
61 struct zram *zram = dev_to_zram(dev); 79 struct zram *zram = dev_to_zram(dev);
62 80
63 return sprintf(buf, "%u\n", zram->init_done); 81 down_read(&zram->init_lock);
64} 82 val = init_done(zram);
65 83 up_read(&zram->init_lock);
66static ssize_t num_reads_show(struct device *dev,
67 struct device_attribute *attr, char *buf)
68{
69 struct zram *zram = dev_to_zram(dev);
70 84
71 return sprintf(buf, "%llu\n", 85 return scnprintf(buf, PAGE_SIZE, "%u\n", val);
72 (u64)atomic64_read(&zram->stats.num_reads));
73} 86}
74 87
75static ssize_t num_writes_show(struct device *dev, 88static ssize_t orig_data_size_show(struct device *dev,
76 struct device_attribute *attr, char *buf) 89 struct device_attribute *attr, char *buf)
77{ 90{
78 struct zram *zram = dev_to_zram(dev); 91 struct zram *zram = dev_to_zram(dev);
79 92
80 return sprintf(buf, "%llu\n", 93 return scnprintf(buf, PAGE_SIZE, "%llu\n",
81 (u64)atomic64_read(&zram->stats.num_writes)); 94 (u64)(atomic64_read(&zram->stats.pages_stored)) << PAGE_SHIFT);
82} 95}
83 96
84static ssize_t invalid_io_show(struct device *dev, 97static ssize_t mem_used_total_show(struct device *dev,
85 struct device_attribute *attr, char *buf) 98 struct device_attribute *attr, char *buf)
86{ 99{
100 u64 val = 0;
87 struct zram *zram = dev_to_zram(dev); 101 struct zram *zram = dev_to_zram(dev);
102 struct zram_meta *meta = zram->meta;
88 103
89 return sprintf(buf, "%llu\n", 104 down_read(&zram->init_lock);
90 (u64)atomic64_read(&zram->stats.invalid_io)); 105 if (init_done(zram))
91} 106 val = zs_get_total_size_bytes(meta->mem_pool);
92 107 up_read(&zram->init_lock);
93static ssize_t notify_free_show(struct device *dev,
94 struct device_attribute *attr, char *buf)
95{
96 struct zram *zram = dev_to_zram(dev);
97 108
98 return sprintf(buf, "%llu\n", 109 return scnprintf(buf, PAGE_SIZE, "%llu\n", val);
99 (u64)atomic64_read(&zram->stats.notify_free));
100} 110}
101 111
102static ssize_t zero_pages_show(struct device *dev, 112static ssize_t max_comp_streams_show(struct device *dev,
103 struct device_attribute *attr, char *buf) 113 struct device_attribute *attr, char *buf)
104{ 114{
115 int val;
105 struct zram *zram = dev_to_zram(dev); 116 struct zram *zram = dev_to_zram(dev);
106 117
107 return sprintf(buf, "%u\n", atomic_read(&zram->stats.pages_zero)); 118 down_read(&zram->init_lock);
119 val = zram->max_comp_streams;
120 up_read(&zram->init_lock);
121
122 return scnprintf(buf, PAGE_SIZE, "%d\n", val);
108} 123}
109 124
110static ssize_t orig_data_size_show(struct device *dev, 125static ssize_t max_comp_streams_store(struct device *dev,
111 struct device_attribute *attr, char *buf) 126 struct device_attribute *attr, const char *buf, size_t len)
112{ 127{
128 int num;
113 struct zram *zram = dev_to_zram(dev); 129 struct zram *zram = dev_to_zram(dev);
130 int ret;
114 131
115 return sprintf(buf, "%llu\n", 132 ret = kstrtoint(buf, 0, &num);
116 (u64)(atomic_read(&zram->stats.pages_stored)) << PAGE_SHIFT); 133 if (ret < 0)
117} 134 return ret;
135 if (num < 1)
136 return -EINVAL;
118 137
119static ssize_t compr_data_size_show(struct device *dev, 138 down_write(&zram->init_lock);
120 struct device_attribute *attr, char *buf) 139 if (init_done(zram)) {
121{ 140 if (!zcomp_set_max_streams(zram->comp, num)) {
122 struct zram *zram = dev_to_zram(dev); 141 pr_info("Cannot change max compression streams\n");
142 ret = -EINVAL;
143 goto out;
144 }
145 }
123 146
124 return sprintf(buf, "%llu\n", 147 zram->max_comp_streams = num;
125 (u64)atomic64_read(&zram->stats.compr_size)); 148 ret = len;
149out:
150 up_write(&zram->init_lock);
151 return ret;
126} 152}
127 153
128static ssize_t mem_used_total_show(struct device *dev, 154static ssize_t comp_algorithm_show(struct device *dev,
129 struct device_attribute *attr, char *buf) 155 struct device_attribute *attr, char *buf)
130{ 156{
131 u64 val = 0; 157 size_t sz;
132 struct zram *zram = dev_to_zram(dev); 158 struct zram *zram = dev_to_zram(dev);
133 struct zram_meta *meta = zram->meta;
134 159
135 down_read(&zram->init_lock); 160 down_read(&zram->init_lock);
136 if (zram->init_done) 161 sz = zcomp_available_show(zram->compressor, buf);
137 val = zs_get_total_size_bytes(meta->mem_pool);
138 up_read(&zram->init_lock); 162 up_read(&zram->init_lock);
139 163
140 return sprintf(buf, "%llu\n", val); 164 return sz;
165}
166
167static ssize_t comp_algorithm_store(struct device *dev,
168 struct device_attribute *attr, const char *buf, size_t len)
169{
170 struct zram *zram = dev_to_zram(dev);
171 down_write(&zram->init_lock);
172 if (init_done(zram)) {
173 up_write(&zram->init_lock);
174 pr_info("Can't change algorithm for initialized device\n");
175 return -EBUSY;
176 }
177 strlcpy(zram->compressor, buf, sizeof(zram->compressor));
178 up_write(&zram->init_lock);
179 return len;
141} 180}
142 181
143/* flag operations needs meta->tb_lock */ 182/* flag operations needs meta->tb_lock */
@@ -192,8 +231,6 @@ static inline int valid_io_request(struct zram *zram, struct bio *bio)
192static void zram_meta_free(struct zram_meta *meta) 231static void zram_meta_free(struct zram_meta *meta)
193{ 232{
194 zs_destroy_pool(meta->mem_pool); 233 zs_destroy_pool(meta->mem_pool);
195 kfree(meta->compress_workmem);
196 free_pages((unsigned long)meta->compress_buffer, 1);
197 vfree(meta->table); 234 vfree(meta->table);
198 kfree(meta); 235 kfree(meta);
199} 236}
@@ -205,22 +242,11 @@ static struct zram_meta *zram_meta_alloc(u64 disksize)
205 if (!meta) 242 if (!meta)
206 goto out; 243 goto out;
207 244
208 meta->compress_workmem = kzalloc(LZO1X_MEM_COMPRESS, GFP_KERNEL);
209 if (!meta->compress_workmem)
210 goto free_meta;
211
212 meta->compress_buffer =
213 (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1);
214 if (!meta->compress_buffer) {
215 pr_err("Error allocating compressor buffer space\n");
216 goto free_workmem;
217 }
218
219 num_pages = disksize >> PAGE_SHIFT; 245 num_pages = disksize >> PAGE_SHIFT;
220 meta->table = vzalloc(num_pages * sizeof(*meta->table)); 246 meta->table = vzalloc(num_pages * sizeof(*meta->table));
221 if (!meta->table) { 247 if (!meta->table) {
222 pr_err("Error allocating zram address table\n"); 248 pr_err("Error allocating zram address table\n");
223 goto free_buffer; 249 goto free_meta;
224 } 250 }
225 251
226 meta->mem_pool = zs_create_pool(GFP_NOIO | __GFP_HIGHMEM); 252 meta->mem_pool = zs_create_pool(GFP_NOIO | __GFP_HIGHMEM);
@@ -230,15 +256,10 @@ static struct zram_meta *zram_meta_alloc(u64 disksize)
230 } 256 }
231 257
232 rwlock_init(&meta->tb_lock); 258 rwlock_init(&meta->tb_lock);
233 mutex_init(&meta->buffer_lock);
234 return meta; 259 return meta;
235 260
236free_table: 261free_table:
237 vfree(meta->table); 262 vfree(meta->table);
238free_buffer:
239 free_pages((unsigned long)meta->compress_buffer, 1);
240free_workmem:
241 kfree(meta->compress_workmem);
242free_meta: 263free_meta:
243 kfree(meta); 264 kfree(meta);
244 meta = NULL; 265 meta = NULL;
@@ -288,7 +309,6 @@ static void zram_free_page(struct zram *zram, size_t index)
288{ 309{
289 struct zram_meta *meta = zram->meta; 310 struct zram_meta *meta = zram->meta;
290 unsigned long handle = meta->table[index].handle; 311 unsigned long handle = meta->table[index].handle;
291 u16 size = meta->table[index].size;
292 312
293 if (unlikely(!handle)) { 313 if (unlikely(!handle)) {
294 /* 314 /*
@@ -297,21 +317,15 @@ static void zram_free_page(struct zram *zram, size_t index)
297 */ 317 */
298 if (zram_test_flag(meta, index, ZRAM_ZERO)) { 318 if (zram_test_flag(meta, index, ZRAM_ZERO)) {
299 zram_clear_flag(meta, index, ZRAM_ZERO); 319 zram_clear_flag(meta, index, ZRAM_ZERO);
300 atomic_dec(&zram->stats.pages_zero); 320 atomic64_dec(&zram->stats.zero_pages);
301 } 321 }
302 return; 322 return;
303 } 323 }
304 324
305 if (unlikely(size > max_zpage_size))
306 atomic_dec(&zram->stats.bad_compress);
307
308 zs_free(meta->mem_pool, handle); 325 zs_free(meta->mem_pool, handle);
309 326
310 if (size <= PAGE_SIZE / 2) 327 atomic64_sub(meta->table[index].size, &zram->stats.compr_data_size);
311 atomic_dec(&zram->stats.good_compress); 328 atomic64_dec(&zram->stats.pages_stored);
312
313 atomic64_sub(meta->table[index].size, &zram->stats.compr_size);
314 atomic_dec(&zram->stats.pages_stored);
315 329
316 meta->table[index].handle = 0; 330 meta->table[index].handle = 0;
317 meta->table[index].size = 0; 331 meta->table[index].size = 0;
@@ -319,8 +333,7 @@ static void zram_free_page(struct zram *zram, size_t index)
319 333
320static int zram_decompress_page(struct zram *zram, char *mem, u32 index) 334static int zram_decompress_page(struct zram *zram, char *mem, u32 index)
321{ 335{
322 int ret = LZO_E_OK; 336 int ret = 0;
323 size_t clen = PAGE_SIZE;
324 unsigned char *cmem; 337 unsigned char *cmem;
325 struct zram_meta *meta = zram->meta; 338 struct zram_meta *meta = zram->meta;
326 unsigned long handle; 339 unsigned long handle;
@@ -340,12 +353,12 @@ static int zram_decompress_page(struct zram *zram, char *mem, u32 index)
340 if (size == PAGE_SIZE) 353 if (size == PAGE_SIZE)
341 copy_page(mem, cmem); 354 copy_page(mem, cmem);
342 else 355 else
343 ret = lzo1x_decompress_safe(cmem, size, mem, &clen); 356 ret = zcomp_decompress(zram->comp, cmem, size, mem);
344 zs_unmap_object(meta->mem_pool, handle); 357 zs_unmap_object(meta->mem_pool, handle);
345 read_unlock(&meta->tb_lock); 358 read_unlock(&meta->tb_lock);
346 359
347 /* Should NEVER happen. Return bio error if it does. */ 360 /* Should NEVER happen. Return bio error if it does. */
348 if (unlikely(ret != LZO_E_OK)) { 361 if (unlikely(ret)) {
349 pr_err("Decompression failed! err=%d, page=%u\n", ret, index); 362 pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
350 atomic64_inc(&zram->stats.failed_reads); 363 atomic64_inc(&zram->stats.failed_reads);
351 return ret; 364 return ret;
@@ -388,7 +401,7 @@ static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
388 401
389 ret = zram_decompress_page(zram, uncmem, index); 402 ret = zram_decompress_page(zram, uncmem, index);
390 /* Should NEVER happen. Return bio error if it does. */ 403 /* Should NEVER happen. Return bio error if it does. */
391 if (unlikely(ret != LZO_E_OK)) 404 if (unlikely(ret))
392 goto out_cleanup; 405 goto out_cleanup;
393 406
394 if (is_partial_io(bvec)) 407 if (is_partial_io(bvec))
@@ -413,11 +426,10 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
413 struct page *page; 426 struct page *page;
414 unsigned char *user_mem, *cmem, *src, *uncmem = NULL; 427 unsigned char *user_mem, *cmem, *src, *uncmem = NULL;
415 struct zram_meta *meta = zram->meta; 428 struct zram_meta *meta = zram->meta;
429 struct zcomp_strm *zstrm;
416 bool locked = false; 430 bool locked = false;
417 431
418 page = bvec->bv_page; 432 page = bvec->bv_page;
419 src = meta->compress_buffer;
420
421 if (is_partial_io(bvec)) { 433 if (is_partial_io(bvec)) {
422 /* 434 /*
423 * This is a partial IO. We need to read the full page 435 * This is a partial IO. We need to read the full page
@@ -433,7 +445,7 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
433 goto out; 445 goto out;
434 } 446 }
435 447
436 mutex_lock(&meta->buffer_lock); 448 zstrm = zcomp_strm_find(zram->comp);
437 locked = true; 449 locked = true;
438 user_mem = kmap_atomic(page); 450 user_mem = kmap_atomic(page);
439 451
@@ -454,28 +466,25 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
454 zram_set_flag(meta, index, ZRAM_ZERO); 466 zram_set_flag(meta, index, ZRAM_ZERO);
455 write_unlock(&zram->meta->tb_lock); 467 write_unlock(&zram->meta->tb_lock);
456 468
457 atomic_inc(&zram->stats.pages_zero); 469 atomic64_inc(&zram->stats.zero_pages);
458 ret = 0; 470 ret = 0;
459 goto out; 471 goto out;
460 } 472 }
461 473
462 ret = lzo1x_1_compress(uncmem, PAGE_SIZE, src, &clen, 474 ret = zcomp_compress(zram->comp, zstrm, uncmem, &clen);
463 meta->compress_workmem);
464 if (!is_partial_io(bvec)) { 475 if (!is_partial_io(bvec)) {
465 kunmap_atomic(user_mem); 476 kunmap_atomic(user_mem);
466 user_mem = NULL; 477 user_mem = NULL;
467 uncmem = NULL; 478 uncmem = NULL;
468 } 479 }
469 480
470 if (unlikely(ret != LZO_E_OK)) { 481 if (unlikely(ret)) {
471 pr_err("Compression failed! err=%d\n", ret); 482 pr_err("Compression failed! err=%d\n", ret);
472 goto out; 483 goto out;
473 } 484 }
474 485 src = zstrm->buffer;
475 if (unlikely(clen > max_zpage_size)) { 486 if (unlikely(clen > max_zpage_size)) {
476 atomic_inc(&zram->stats.bad_compress);
477 clen = PAGE_SIZE; 487 clen = PAGE_SIZE;
478 src = NULL;
479 if (is_partial_io(bvec)) 488 if (is_partial_io(bvec))
480 src = uncmem; 489 src = uncmem;
481 } 490 }
@@ -497,6 +506,8 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
497 memcpy(cmem, src, clen); 506 memcpy(cmem, src, clen);
498 } 507 }
499 508
509 zcomp_strm_release(zram->comp, zstrm);
510 locked = false;
500 zs_unmap_object(meta->mem_pool, handle); 511 zs_unmap_object(meta->mem_pool, handle);
501 512
502 /* 513 /*
@@ -511,49 +522,88 @@ static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
511 write_unlock(&zram->meta->tb_lock); 522 write_unlock(&zram->meta->tb_lock);
512 523
513 /* Update stats */ 524 /* Update stats */
514 atomic64_add(clen, &zram->stats.compr_size); 525 atomic64_add(clen, &zram->stats.compr_data_size);
515 atomic_inc(&zram->stats.pages_stored); 526 atomic64_inc(&zram->stats.pages_stored);
516 if (clen <= PAGE_SIZE / 2)
517 atomic_inc(&zram->stats.good_compress);
518
519out: 527out:
520 if (locked) 528 if (locked)
521 mutex_unlock(&meta->buffer_lock); 529 zcomp_strm_release(zram->comp, zstrm);
522 if (is_partial_io(bvec)) 530 if (is_partial_io(bvec))
523 kfree(uncmem); 531 kfree(uncmem);
524
525 if (ret) 532 if (ret)
526 atomic64_inc(&zram->stats.failed_writes); 533 atomic64_inc(&zram->stats.failed_writes);
527 return ret; 534 return ret;
528} 535}
529 536
530static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index, 537static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,
531 int offset, struct bio *bio, int rw) 538 int offset, struct bio *bio)
532{ 539{
533 int ret; 540 int ret;
541 int rw = bio_data_dir(bio);
534 542
535 if (rw == READ) 543 if (rw == READ) {
544 atomic64_inc(&zram->stats.num_reads);
536 ret = zram_bvec_read(zram, bvec, index, offset, bio); 545 ret = zram_bvec_read(zram, bvec, index, offset, bio);
537 else 546 } else {
547 atomic64_inc(&zram->stats.num_writes);
538 ret = zram_bvec_write(zram, bvec, index, offset); 548 ret = zram_bvec_write(zram, bvec, index, offset);
549 }
539 550
540 return ret; 551 return ret;
541} 552}
542 553
554/*
555 * zram_bio_discard - handler on discard request
556 * @index: physical block index in PAGE_SIZE units
557 * @offset: byte offset within physical block
558 */
559static void zram_bio_discard(struct zram *zram, u32 index,
560 int offset, struct bio *bio)
561{
562 size_t n = bio->bi_iter.bi_size;
563
564 /*
565 * zram manages data in physical block size units. Because logical block
566 * size isn't identical with physical block size on some arch, we
567 * could get a discard request pointing to a specific offset within a
568 * certain physical block. Although we can handle this request by
569 * reading that physiclal block and decompressing and partially zeroing
570 * and re-compressing and then re-storing it, this isn't reasonable
571 * because our intent with a discard request is to save memory. So
572 * skipping this logical block is appropriate here.
573 */
574 if (offset) {
575 if (n < offset)
576 return;
577
578 n -= offset;
579 index++;
580 }
581
582 while (n >= PAGE_SIZE) {
583 /*
584 * Discard request can be large so the lock hold times could be
585 * lengthy. So take the lock once per page.
586 */
587 write_lock(&zram->meta->tb_lock);
588 zram_free_page(zram, index);
589 write_unlock(&zram->meta->tb_lock);
590 index++;
591 n -= PAGE_SIZE;
592 }
593}
594
543static void zram_reset_device(struct zram *zram, bool reset_capacity) 595static void zram_reset_device(struct zram *zram, bool reset_capacity)
544{ 596{
545 size_t index; 597 size_t index;
546 struct zram_meta *meta; 598 struct zram_meta *meta;
547 599
548 down_write(&zram->init_lock); 600 down_write(&zram->init_lock);
549 if (!zram->init_done) { 601 if (!init_done(zram)) {
550 up_write(&zram->init_lock); 602 up_write(&zram->init_lock);
551 return; 603 return;
552 } 604 }
553 605
554 meta = zram->meta; 606 meta = zram->meta;
555 zram->init_done = 0;
556
557 /* Free all pages that are still in this zram device */ 607 /* Free all pages that are still in this zram device */
558 for (index = 0; index < zram->disksize >> PAGE_SHIFT; index++) { 608 for (index = 0; index < zram->disksize >> PAGE_SHIFT; index++) {
559 unsigned long handle = meta->table[index].handle; 609 unsigned long handle = meta->table[index].handle;
@@ -563,6 +613,9 @@ static void zram_reset_device(struct zram *zram, bool reset_capacity)
563 zs_free(meta->mem_pool, handle); 613 zs_free(meta->mem_pool, handle);
564 } 614 }
565 615
616 zcomp_destroy(zram->comp);
617 zram->max_comp_streams = 1;
618
566 zram_meta_free(zram->meta); 619 zram_meta_free(zram->meta);
567 zram->meta = NULL; 620 zram->meta = NULL;
568 /* Reset stats */ 621 /* Reset stats */
@@ -574,37 +627,14 @@ static void zram_reset_device(struct zram *zram, bool reset_capacity)
574 up_write(&zram->init_lock); 627 up_write(&zram->init_lock);
575} 628}
576 629
577static void zram_init_device(struct zram *zram, struct zram_meta *meta)
578{
579 if (zram->disksize > 2 * (totalram_pages << PAGE_SHIFT)) {
580 pr_info(
581 "There is little point creating a zram of greater than "
582 "twice the size of memory since we expect a 2:1 compression "
583 "ratio. Note that zram uses about 0.1%% of the size of "
584 "the disk when not in use so a huge zram is "
585 "wasteful.\n"
586 "\tMemory Size: %lu kB\n"
587 "\tSize you selected: %llu kB\n"
588 "Continuing anyway ...\n",
589 (totalram_pages << PAGE_SHIFT) >> 10, zram->disksize >> 10
590 );
591 }
592
593 /* zram devices sort of resembles non-rotational disks */
594 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, zram->disk->queue);
595
596 zram->meta = meta;
597 zram->init_done = 1;
598
599 pr_debug("Initialization done!\n");
600}
601
602static ssize_t disksize_store(struct device *dev, 630static ssize_t disksize_store(struct device *dev,
603 struct device_attribute *attr, const char *buf, size_t len) 631 struct device_attribute *attr, const char *buf, size_t len)
604{ 632{
605 u64 disksize; 633 u64 disksize;
634 struct zcomp *comp;
606 struct zram_meta *meta; 635 struct zram_meta *meta;
607 struct zram *zram = dev_to_zram(dev); 636 struct zram *zram = dev_to_zram(dev);
637 int err;
608 638
609 disksize = memparse(buf, NULL); 639 disksize = memparse(buf, NULL);
610 if (!disksize) 640 if (!disksize)
@@ -614,20 +644,35 @@ static ssize_t disksize_store(struct device *dev,
614 meta = zram_meta_alloc(disksize); 644 meta = zram_meta_alloc(disksize);
615 if (!meta) 645 if (!meta)
616 return -ENOMEM; 646 return -ENOMEM;
647
648 comp = zcomp_create(zram->compressor, zram->max_comp_streams);
649 if (IS_ERR(comp)) {
650 pr_info("Cannot initialise %s compressing backend\n",
651 zram->compressor);
652 err = PTR_ERR(comp);
653 goto out_free_meta;
654 }
655
617 down_write(&zram->init_lock); 656 down_write(&zram->init_lock);
618 if (zram->init_done) { 657 if (init_done(zram)) {
619 up_write(&zram->init_lock);
620 zram_meta_free(meta);
621 pr_info("Cannot change disksize for initialized device\n"); 658 pr_info("Cannot change disksize for initialized device\n");
622 return -EBUSY; 659 err = -EBUSY;
660 goto out_destroy_comp;
623 } 661 }
624 662
663 zram->meta = meta;
664 zram->comp = comp;
625 zram->disksize = disksize; 665 zram->disksize = disksize;
626 set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT); 666 set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT);
627 zram_init_device(zram, meta);
628 up_write(&zram->init_lock); 667 up_write(&zram->init_lock);
629
630 return len; 668 return len;
669
670out_destroy_comp:
671 up_write(&zram->init_lock);
672 zcomp_destroy(comp);
673out_free_meta:
674 zram_meta_free(meta);
675 return err;
631} 676}
632 677
633static ssize_t reset_store(struct device *dev, 678static ssize_t reset_store(struct device *dev,
@@ -671,26 +716,23 @@ out:
671 return ret; 716 return ret;
672} 717}
673 718
674static void __zram_make_request(struct zram *zram, struct bio *bio, int rw) 719static void __zram_make_request(struct zram *zram, struct bio *bio)
675{ 720{
676 int offset; 721 int offset;
677 u32 index; 722 u32 index;
678 struct bio_vec bvec; 723 struct bio_vec bvec;
679 struct bvec_iter iter; 724 struct bvec_iter iter;
680 725
681 switch (rw) {
682 case READ:
683 atomic64_inc(&zram->stats.num_reads);
684 break;
685 case WRITE:
686 atomic64_inc(&zram->stats.num_writes);
687 break;
688 }
689
690 index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT; 726 index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
691 offset = (bio->bi_iter.bi_sector & 727 offset = (bio->bi_iter.bi_sector &
692 (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT; 728 (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT;
693 729
730 if (unlikely(bio->bi_rw & REQ_DISCARD)) {
731 zram_bio_discard(zram, index, offset, bio);
732 bio_endio(bio, 0);
733 return;
734 }
735
694 bio_for_each_segment(bvec, bio, iter) { 736 bio_for_each_segment(bvec, bio, iter) {
695 int max_transfer_size = PAGE_SIZE - offset; 737 int max_transfer_size = PAGE_SIZE - offset;
696 738
@@ -705,16 +747,15 @@ static void __zram_make_request(struct zram *zram, struct bio *bio, int rw)
705 bv.bv_len = max_transfer_size; 747 bv.bv_len = max_transfer_size;
706 bv.bv_offset = bvec.bv_offset; 748 bv.bv_offset = bvec.bv_offset;
707 749
708 if (zram_bvec_rw(zram, &bv, index, offset, bio, rw) < 0) 750 if (zram_bvec_rw(zram, &bv, index, offset, bio) < 0)
709 goto out; 751 goto out;
710 752
711 bv.bv_len = bvec.bv_len - max_transfer_size; 753 bv.bv_len = bvec.bv_len - max_transfer_size;
712 bv.bv_offset += max_transfer_size; 754 bv.bv_offset += max_transfer_size;
713 if (zram_bvec_rw(zram, &bv, index+1, 0, bio, rw) < 0) 755 if (zram_bvec_rw(zram, &bv, index + 1, 0, bio) < 0)
714 goto out; 756 goto out;
715 } else 757 } else
716 if (zram_bvec_rw(zram, &bvec, index, offset, bio, rw) 758 if (zram_bvec_rw(zram, &bvec, index, offset, bio) < 0)
717 < 0)
718 goto out; 759 goto out;
719 760
720 update_position(&index, &offset, &bvec); 761 update_position(&index, &offset, &bvec);
@@ -736,7 +777,7 @@ static void zram_make_request(struct request_queue *queue, struct bio *bio)
736 struct zram *zram = queue->queuedata; 777 struct zram *zram = queue->queuedata;
737 778
738 down_read(&zram->init_lock); 779 down_read(&zram->init_lock);
739 if (unlikely(!zram->init_done)) 780 if (unlikely(!init_done(zram)))
740 goto error; 781 goto error;
741 782
742 if (!valid_io_request(zram, bio)) { 783 if (!valid_io_request(zram, bio)) {
@@ -744,7 +785,7 @@ static void zram_make_request(struct request_queue *queue, struct bio *bio)
744 goto error; 785 goto error;
745 } 786 }
746 787
747 __zram_make_request(zram, bio, bio_data_dir(bio)); 788 __zram_make_request(zram, bio);
748 up_read(&zram->init_lock); 789 up_read(&zram->init_lock);
749 790
750 return; 791 return;
@@ -778,14 +819,21 @@ static DEVICE_ATTR(disksize, S_IRUGO | S_IWUSR,
778 disksize_show, disksize_store); 819 disksize_show, disksize_store);
779static DEVICE_ATTR(initstate, S_IRUGO, initstate_show, NULL); 820static DEVICE_ATTR(initstate, S_IRUGO, initstate_show, NULL);
780static DEVICE_ATTR(reset, S_IWUSR, NULL, reset_store); 821static DEVICE_ATTR(reset, S_IWUSR, NULL, reset_store);
781static DEVICE_ATTR(num_reads, S_IRUGO, num_reads_show, NULL);
782static DEVICE_ATTR(num_writes, S_IRUGO, num_writes_show, NULL);
783static DEVICE_ATTR(invalid_io, S_IRUGO, invalid_io_show, NULL);
784static DEVICE_ATTR(notify_free, S_IRUGO, notify_free_show, NULL);
785static DEVICE_ATTR(zero_pages, S_IRUGO, zero_pages_show, NULL);
786static DEVICE_ATTR(orig_data_size, S_IRUGO, orig_data_size_show, NULL); 822static DEVICE_ATTR(orig_data_size, S_IRUGO, orig_data_size_show, NULL);
787static DEVICE_ATTR(compr_data_size, S_IRUGO, compr_data_size_show, NULL);
788static DEVICE_ATTR(mem_used_total, S_IRUGO, mem_used_total_show, NULL); 823static DEVICE_ATTR(mem_used_total, S_IRUGO, mem_used_total_show, NULL);
824static DEVICE_ATTR(max_comp_streams, S_IRUGO | S_IWUSR,
825 max_comp_streams_show, max_comp_streams_store);
826static DEVICE_ATTR(comp_algorithm, S_IRUGO | S_IWUSR,
827 comp_algorithm_show, comp_algorithm_store);
828
829ZRAM_ATTR_RO(num_reads);
830ZRAM_ATTR_RO(num_writes);
831ZRAM_ATTR_RO(failed_reads);
832ZRAM_ATTR_RO(failed_writes);
833ZRAM_ATTR_RO(invalid_io);
834ZRAM_ATTR_RO(notify_free);
835ZRAM_ATTR_RO(zero_pages);
836ZRAM_ATTR_RO(compr_data_size);
789 837
790static struct attribute *zram_disk_attrs[] = { 838static struct attribute *zram_disk_attrs[] = {
791 &dev_attr_disksize.attr, 839 &dev_attr_disksize.attr,
@@ -793,12 +841,16 @@ static struct attribute *zram_disk_attrs[] = {
793 &dev_attr_reset.attr, 841 &dev_attr_reset.attr,
794 &dev_attr_num_reads.attr, 842 &dev_attr_num_reads.attr,
795 &dev_attr_num_writes.attr, 843 &dev_attr_num_writes.attr,
844 &dev_attr_failed_reads.attr,
845 &dev_attr_failed_writes.attr,
796 &dev_attr_invalid_io.attr, 846 &dev_attr_invalid_io.attr,
797 &dev_attr_notify_free.attr, 847 &dev_attr_notify_free.attr,
798 &dev_attr_zero_pages.attr, 848 &dev_attr_zero_pages.attr,
799 &dev_attr_orig_data_size.attr, 849 &dev_attr_orig_data_size.attr,
800 &dev_attr_compr_data_size.attr, 850 &dev_attr_compr_data_size.attr,
801 &dev_attr_mem_used_total.attr, 851 &dev_attr_mem_used_total.attr,
852 &dev_attr_max_comp_streams.attr,
853 &dev_attr_comp_algorithm.attr,
802 NULL, 854 NULL,
803}; 855};
804 856
@@ -839,7 +891,8 @@ static int create_device(struct zram *zram, int device_id)
839 891
840 /* Actual capacity set using syfs (/sys/block/zram<id>/disksize */ 892 /* Actual capacity set using syfs (/sys/block/zram<id>/disksize */
841 set_capacity(zram->disk, 0); 893 set_capacity(zram->disk, 0);
842 894 /* zram devices sort of resembles non-rotational disks */
895 queue_flag_set_unlocked(QUEUE_FLAG_NONROT, zram->disk->queue);
843 /* 896 /*
844 * To ensure that we always get PAGE_SIZE aligned 897 * To ensure that we always get PAGE_SIZE aligned
845 * and n*PAGE_SIZED sized I/O requests. 898 * and n*PAGE_SIZED sized I/O requests.
@@ -849,6 +902,21 @@ static int create_device(struct zram *zram, int device_id)
849 ZRAM_LOGICAL_BLOCK_SIZE); 902 ZRAM_LOGICAL_BLOCK_SIZE);
850 blk_queue_io_min(zram->disk->queue, PAGE_SIZE); 903 blk_queue_io_min(zram->disk->queue, PAGE_SIZE);
851 blk_queue_io_opt(zram->disk->queue, PAGE_SIZE); 904 blk_queue_io_opt(zram->disk->queue, PAGE_SIZE);
905 zram->disk->queue->limits.discard_granularity = PAGE_SIZE;
906 zram->disk->queue->limits.max_discard_sectors = UINT_MAX;
907 /*
908 * zram_bio_discard() will clear all logical blocks if logical block
909 * size is identical with physical block size(PAGE_SIZE). But if it is
910 * different, we will skip discarding some parts of logical blocks in
911 * the part of the request range which isn't aligned to physical block
912 * size. So we can't ensure that all discarded logical blocks are
913 * zeroed.
914 */
915 if (ZRAM_LOGICAL_BLOCK_SIZE == PAGE_SIZE)
916 zram->disk->queue->limits.discard_zeroes_data = 1;
917 else
918 zram->disk->queue->limits.discard_zeroes_data = 0;
919 queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, zram->disk->queue);
852 920
853 add_disk(zram->disk); 921 add_disk(zram->disk);
854 922
@@ -858,8 +926,9 @@ static int create_device(struct zram *zram, int device_id)
858 pr_warn("Error creating sysfs group"); 926 pr_warn("Error creating sysfs group");
859 goto out_free_disk; 927 goto out_free_disk;
860 } 928 }
861 929 strlcpy(zram->compressor, default_compressor, sizeof(zram->compressor));
862 zram->init_done = 0; 930 zram->meta = NULL;
931 zram->max_comp_streams = 1;
863 return 0; 932 return 0;
864 933
865out_free_disk: 934out_free_disk:
diff --git a/drivers/block/zram/zram_drv.h b/drivers/block/zram/zram_drv.h
index ad8aa35bae00..7f21c145e317 100644
--- a/drivers/block/zram/zram_drv.h
+++ b/drivers/block/zram/zram_drv.h
@@ -16,9 +16,10 @@
16#define _ZRAM_DRV_H_ 16#define _ZRAM_DRV_H_
17 17
18#include <linux/spinlock.h> 18#include <linux/spinlock.h>
19#include <linux/mutex.h>
20#include <linux/zsmalloc.h> 19#include <linux/zsmalloc.h>
21 20
21#include "zcomp.h"
22
22/* 23/*
23 * Some arbitrary value. This is just to catch 24 * Some arbitrary value. This is just to catch
24 * invalid value for num_devices module parameter. 25 * invalid value for num_devices module parameter.
@@ -64,38 +65,33 @@ enum zram_pageflags {
64struct table { 65struct table {
65 unsigned long handle; 66 unsigned long handle;
66 u16 size; /* object size (excluding header) */ 67 u16 size; /* object size (excluding header) */
67 u8 count; /* object ref count (not yet used) */
68 u8 flags; 68 u8 flags;
69} __aligned(4); 69} __aligned(4);
70 70
71struct zram_stats { 71struct zram_stats {
72 atomic64_t compr_size; /* compressed size of pages stored */ 72 atomic64_t compr_data_size; /* compressed size of pages stored */
73 atomic64_t num_reads; /* failed + successful */ 73 atomic64_t num_reads; /* failed + successful */
74 atomic64_t num_writes; /* --do-- */ 74 atomic64_t num_writes; /* --do-- */
75 atomic64_t failed_reads; /* should NEVER! happen */ 75 atomic64_t failed_reads; /* should NEVER! happen */
76 atomic64_t failed_writes; /* can happen when memory is too low */ 76 atomic64_t failed_writes; /* can happen when memory is too low */
77 atomic64_t invalid_io; /* non-page-aligned I/O requests */ 77 atomic64_t invalid_io; /* non-page-aligned I/O requests */
78 atomic64_t notify_free; /* no. of swap slot free notifications */ 78 atomic64_t notify_free; /* no. of swap slot free notifications */
79 atomic_t pages_zero; /* no. of zero filled pages */ 79 atomic64_t zero_pages; /* no. of zero filled pages */
80 atomic_t pages_stored; /* no. of pages currently stored */ 80 atomic64_t pages_stored; /* no. of pages currently stored */
81 atomic_t good_compress; /* % of pages with compression ratio<=50% */
82 atomic_t bad_compress; /* % of pages with compression ratio>=75% */
83}; 81};
84 82
85struct zram_meta { 83struct zram_meta {
86 rwlock_t tb_lock; /* protect table */ 84 rwlock_t tb_lock; /* protect table */
87 void *compress_workmem;
88 void *compress_buffer;
89 struct table *table; 85 struct table *table;
90 struct zs_pool *mem_pool; 86 struct zs_pool *mem_pool;
91 struct mutex buffer_lock; /* protect compress buffers */
92}; 87};
93 88
94struct zram { 89struct zram {
95 struct zram_meta *meta; 90 struct zram_meta *meta;
96 struct request_queue *queue; 91 struct request_queue *queue;
97 struct gendisk *disk; 92 struct gendisk *disk;
98 int init_done; 93 struct zcomp *comp;
94
99 /* Prevent concurrent execution of device init, reset and R/W request */ 95 /* Prevent concurrent execution of device init, reset and R/W request */
100 struct rw_semaphore init_lock; 96 struct rw_semaphore init_lock;
101 /* 97 /*
@@ -103,7 +99,8 @@ struct zram {
103 * we can store in a disk. 99 * we can store in a disk.
104 */ 100 */
105 u64 disksize; /* bytes */ 101 u64 disksize; /* bytes */
106 102 int max_comp_streams;
107 struct zram_stats stats; 103 struct zram_stats stats;
104 char compressor[10];
108}; 105};
109#endif 106#endif