aboutsummaryrefslogtreecommitdiffstats
path: root/fs
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2013-11-20 17:51:37 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2013-11-20 17:51:37 -0500
commitaf2e2f328052082f58f041d574ed50c7f21c598f (patch)
treeaad7fdd9fe6f3016596b1be8098eaaaa7e0a85f7 /fs
parent8b2e9b712f6139df9c754af0d67fecc4bbc88545 (diff)
parented4f381ec15e5f11724cdbc68cffd2c22d1eaebd (diff)
Merge tag 'squashfs-updates' of git://git.kernel.org/pub/scm/linux/kernel/git/pkl/squashfs-next
Pull squashfs updates from Phillip Lougher: "These patches optionally improve the multi-threading peformance of Squashfs by adding parallel decompression, and direct decompression into the page cache, eliminating an intermediate buffer (removing memcpy overhead and lock contention)" * tag 'squashfs-updates' of git://git.kernel.org/pub/scm/linux/kernel/git/pkl/squashfs-next: Squashfs: Check stream is not NULL in decompressor_multi.c Squashfs: Directly decompress into the page cache for file data Squashfs: Restructure squashfs_readpage() Squashfs: Generalise paging handling in the decompressors Squashfs: add multi-threaded decompression using percpu variable squashfs: Enhance parallel I/O Squashfs: Refactor decompressor interface and code
Diffstat (limited to 'fs')
-rw-r--r--fs/squashfs/Kconfig72
-rw-r--r--fs/squashfs/Makefile5
-rw-r--r--fs/squashfs/block.c36
-rw-r--r--fs/squashfs/cache.c28
-rw-r--r--fs/squashfs/decompressor.c59
-rw-r--r--fs/squashfs/decompressor.h24
-rw-r--r--fs/squashfs/decompressor_multi.c198
-rw-r--r--fs/squashfs/decompressor_multi_percpu.c97
-rw-r--r--fs/squashfs/decompressor_single.c85
-rw-r--r--fs/squashfs/file.c142
-rw-r--r--fs/squashfs/file_cache.c38
-rw-r--r--fs/squashfs/file_direct.c173
-rw-r--r--fs/squashfs/lzo_wrapper.c47
-rw-r--r--fs/squashfs/page_actor.c100
-rw-r--r--fs/squashfs/page_actor.h81
-rw-r--r--fs/squashfs/squashfs.h20
-rw-r--r--fs/squashfs/squashfs_fs_sb.h4
-rw-r--r--fs/squashfs/super.c10
-rw-r--r--fs/squashfs/xz_wrapper.c105
-rw-r--r--fs/squashfs/zlib_wrapper.c64
20 files changed, 1145 insertions, 243 deletions
diff --git a/fs/squashfs/Kconfig b/fs/squashfs/Kconfig
index c70111ebefd4..b6fa8657dcbc 100644
--- a/fs/squashfs/Kconfig
+++ b/fs/squashfs/Kconfig
@@ -25,6 +25,78 @@ config SQUASHFS
25 25
26 If unsure, say N. 26 If unsure, say N.
27 27
28choice
29 prompt "File decompression options"
30 depends on SQUASHFS
31 help
32 Squashfs now supports two options for decompressing file
33 data. Traditionally Squashfs has decompressed into an
34 intermediate buffer and then memcopied it into the page cache.
35 Squashfs now supports the ability to decompress directly into
36 the page cache.
37
38 If unsure, select "Decompress file data into an intermediate buffer"
39
40config SQUASHFS_FILE_CACHE
41 bool "Decompress file data into an intermediate buffer"
42 help
43 Decompress file data into an intermediate buffer and then
44 memcopy it into the page cache.
45
46config SQUASHFS_FILE_DIRECT
47 bool "Decompress files directly into the page cache"
48 help
49 Directly decompress file data into the page cache.
50 Doing so can significantly improve performance because
51 it eliminates a memcpy and it also removes the lock contention
52 on the single buffer.
53
54endchoice
55
56choice
57 prompt "Decompressor parallelisation options"
58 depends on SQUASHFS
59 help
60 Squashfs now supports three parallelisation options for
61 decompression. Each one exhibits various trade-offs between
62 decompression performance and CPU and memory usage.
63
64 If in doubt, select "Single threaded compression"
65
66config SQUASHFS_DECOMP_SINGLE
67 bool "Single threaded compression"
68 help
69 Traditionally Squashfs has used single-threaded decompression.
70 Only one block (data or metadata) can be decompressed at any
71 one time. This limits CPU and memory usage to a minimum.
72
73config SQUASHFS_DECOMP_MULTI
74 bool "Use multiple decompressors for parallel I/O"
75 help
76 By default Squashfs uses a single decompressor but it gives
77 poor performance on parallel I/O workloads when using multiple CPU
78 machines due to waiting on decompressor availability.
79
80 If you have a parallel I/O workload and your system has enough memory,
81 using this option may improve overall I/O performance.
82
83 This decompressor implementation uses up to two parallel
84 decompressors per core. It dynamically allocates decompressors
85 on a demand basis.
86
87config SQUASHFS_DECOMP_MULTI_PERCPU
88 bool "Use percpu multiple decompressors for parallel I/O"
89 help
90 By default Squashfs uses a single decompressor but it gives
91 poor performance on parallel I/O workloads when using multiple CPU
92 machines due to waiting on decompressor availability.
93
94 This decompressor implementation uses a maximum of one
95 decompressor per core. It uses percpu variables to ensure
96 decompression is load-balanced across the cores.
97
98endchoice
99
28config SQUASHFS_XATTR 100config SQUASHFS_XATTR
29 bool "Squashfs XATTR support" 101 bool "Squashfs XATTR support"
30 depends on SQUASHFS 102 depends on SQUASHFS
diff --git a/fs/squashfs/Makefile b/fs/squashfs/Makefile
index 110b0476f3b4..4132520b4ff2 100644
--- a/fs/squashfs/Makefile
+++ b/fs/squashfs/Makefile
@@ -5,6 +5,11 @@
5obj-$(CONFIG_SQUASHFS) += squashfs.o 5obj-$(CONFIG_SQUASHFS) += squashfs.o
6squashfs-y += block.o cache.o dir.o export.o file.o fragment.o id.o inode.o 6squashfs-y += block.o cache.o dir.o export.o file.o fragment.o id.o inode.o
7squashfs-y += namei.o super.o symlink.o decompressor.o 7squashfs-y += namei.o super.o symlink.o decompressor.o
8squashfs-$(CONFIG_SQUASHFS_FILE_CACHE) += file_cache.o
9squashfs-$(CONFIG_SQUASHFS_FILE_DIRECT) += file_direct.o page_actor.o
10squashfs-$(CONFIG_SQUASHFS_DECOMP_SINGLE) += decompressor_single.o
11squashfs-$(CONFIG_SQUASHFS_DECOMP_MULTI) += decompressor_multi.o
12squashfs-$(CONFIG_SQUASHFS_DECOMP_MULTI_PERCPU) += decompressor_multi_percpu.o
8squashfs-$(CONFIG_SQUASHFS_XATTR) += xattr.o xattr_id.o 13squashfs-$(CONFIG_SQUASHFS_XATTR) += xattr.o xattr_id.o
9squashfs-$(CONFIG_SQUASHFS_LZO) += lzo_wrapper.o 14squashfs-$(CONFIG_SQUASHFS_LZO) += lzo_wrapper.o
10squashfs-$(CONFIG_SQUASHFS_XZ) += xz_wrapper.o 15squashfs-$(CONFIG_SQUASHFS_XZ) += xz_wrapper.o
diff --git a/fs/squashfs/block.c b/fs/squashfs/block.c
index 41d108ecc9be..0cea9b9236d0 100644
--- a/fs/squashfs/block.c
+++ b/fs/squashfs/block.c
@@ -36,6 +36,7 @@
36#include "squashfs_fs_sb.h" 36#include "squashfs_fs_sb.h"
37#include "squashfs.h" 37#include "squashfs.h"
38#include "decompressor.h" 38#include "decompressor.h"
39#include "page_actor.h"
39 40
40/* 41/*
41 * Read the metadata block length, this is stored in the first two 42 * Read the metadata block length, this is stored in the first two
@@ -86,16 +87,16 @@ static struct buffer_head *get_block_length(struct super_block *sb,
86 * generated a larger block - this does occasionally happen with compression 87 * generated a larger block - this does occasionally happen with compression
87 * algorithms). 88 * algorithms).
88 */ 89 */
89int squashfs_read_data(struct super_block *sb, void **buffer, u64 index, 90int squashfs_read_data(struct super_block *sb, u64 index, int length,
90 int length, u64 *next_index, int srclength, int pages) 91 u64 *next_index, struct squashfs_page_actor *output)
91{ 92{
92 struct squashfs_sb_info *msblk = sb->s_fs_info; 93 struct squashfs_sb_info *msblk = sb->s_fs_info;
93 struct buffer_head **bh; 94 struct buffer_head **bh;
94 int offset = index & ((1 << msblk->devblksize_log2) - 1); 95 int offset = index & ((1 << msblk->devblksize_log2) - 1);
95 u64 cur_index = index >> msblk->devblksize_log2; 96 u64 cur_index = index >> msblk->devblksize_log2;
96 int bytes, compressed, b = 0, k = 0, page = 0, avail; 97 int bytes, compressed, b = 0, k = 0, avail, i;
97 98
98 bh = kcalloc(((srclength + msblk->devblksize - 1) 99 bh = kcalloc(((output->length + msblk->devblksize - 1)
99 >> msblk->devblksize_log2) + 1, sizeof(*bh), GFP_KERNEL); 100 >> msblk->devblksize_log2) + 1, sizeof(*bh), GFP_KERNEL);
100 if (bh == NULL) 101 if (bh == NULL)
101 return -ENOMEM; 102 return -ENOMEM;
@@ -111,9 +112,9 @@ int squashfs_read_data(struct super_block *sb, void **buffer, u64 index,
111 *next_index = index + length; 112 *next_index = index + length;
112 113
113 TRACE("Block @ 0x%llx, %scompressed size %d, src size %d\n", 114 TRACE("Block @ 0x%llx, %scompressed size %d, src size %d\n",
114 index, compressed ? "" : "un", length, srclength); 115 index, compressed ? "" : "un", length, output->length);
115 116
116 if (length < 0 || length > srclength || 117 if (length < 0 || length > output->length ||
117 (index + length) > msblk->bytes_used) 118 (index + length) > msblk->bytes_used)
118 goto read_failure; 119 goto read_failure;
119 120
@@ -145,7 +146,7 @@ int squashfs_read_data(struct super_block *sb, void **buffer, u64 index,
145 TRACE("Block @ 0x%llx, %scompressed size %d\n", index, 146 TRACE("Block @ 0x%llx, %scompressed size %d\n", index,
146 compressed ? "" : "un", length); 147 compressed ? "" : "un", length);
147 148
148 if (length < 0 || length > srclength || 149 if (length < 0 || length > output->length ||
149 (index + length) > msblk->bytes_used) 150 (index + length) > msblk->bytes_used)
150 goto block_release; 151 goto block_release;
151 152
@@ -158,9 +159,15 @@ int squashfs_read_data(struct super_block *sb, void **buffer, u64 index,
158 ll_rw_block(READ, b - 1, bh + 1); 159 ll_rw_block(READ, b - 1, bh + 1);
159 } 160 }
160 161
162 for (i = 0; i < b; i++) {
163 wait_on_buffer(bh[i]);
164 if (!buffer_uptodate(bh[i]))
165 goto block_release;
166 }
167
161 if (compressed) { 168 if (compressed) {
162 length = squashfs_decompress(msblk, buffer, bh, b, offset, 169 length = squashfs_decompress(msblk, bh, b, offset, length,
163 length, srclength, pages); 170 output);
164 if (length < 0) 171 if (length < 0)
165 goto read_failure; 172 goto read_failure;
166 } else { 173 } else {
@@ -168,22 +175,20 @@ int squashfs_read_data(struct super_block *sb, void **buffer, u64 index,
168 * Block is uncompressed. 175 * Block is uncompressed.
169 */ 176 */
170 int in, pg_offset = 0; 177 int in, pg_offset = 0;
178 void *data = squashfs_first_page(output);
171 179
172 for (bytes = length; k < b; k++) { 180 for (bytes = length; k < b; k++) {
173 in = min(bytes, msblk->devblksize - offset); 181 in = min(bytes, msblk->devblksize - offset);
174 bytes -= in; 182 bytes -= in;
175 wait_on_buffer(bh[k]);
176 if (!buffer_uptodate(bh[k]))
177 goto block_release;
178 while (in) { 183 while (in) {
179 if (pg_offset == PAGE_CACHE_SIZE) { 184 if (pg_offset == PAGE_CACHE_SIZE) {
180 page++; 185 data = squashfs_next_page(output);
181 pg_offset = 0; 186 pg_offset = 0;
182 } 187 }
183 avail = min_t(int, in, PAGE_CACHE_SIZE - 188 avail = min_t(int, in, PAGE_CACHE_SIZE -
184 pg_offset); 189 pg_offset);
185 memcpy(buffer[page] + pg_offset, 190 memcpy(data + pg_offset, bh[k]->b_data + offset,
186 bh[k]->b_data + offset, avail); 191 avail);
187 in -= avail; 192 in -= avail;
188 pg_offset += avail; 193 pg_offset += avail;
189 offset += avail; 194 offset += avail;
@@ -191,6 +196,7 @@ int squashfs_read_data(struct super_block *sb, void **buffer, u64 index,
191 offset = 0; 196 offset = 0;
192 put_bh(bh[k]); 197 put_bh(bh[k]);
193 } 198 }
199 squashfs_finish_page(output);
194 } 200 }
195 201
196 kfree(bh); 202 kfree(bh);
diff --git a/fs/squashfs/cache.c b/fs/squashfs/cache.c
index af0b73802592..1cb70a0b2168 100644
--- a/fs/squashfs/cache.c
+++ b/fs/squashfs/cache.c
@@ -56,6 +56,7 @@
56#include "squashfs_fs.h" 56#include "squashfs_fs.h"
57#include "squashfs_fs_sb.h" 57#include "squashfs_fs_sb.h"
58#include "squashfs.h" 58#include "squashfs.h"
59#include "page_actor.h"
59 60
60/* 61/*
61 * Look-up block in cache, and increment usage count. If not in cache, read 62 * Look-up block in cache, and increment usage count. If not in cache, read
@@ -119,9 +120,8 @@ struct squashfs_cache_entry *squashfs_cache_get(struct super_block *sb,
119 entry->error = 0; 120 entry->error = 0;
120 spin_unlock(&cache->lock); 121 spin_unlock(&cache->lock);
121 122
122 entry->length = squashfs_read_data(sb, entry->data, 123 entry->length = squashfs_read_data(sb, block, length,
123 block, length, &entry->next_index, 124 &entry->next_index, entry->actor);
124 cache->block_size, cache->pages);
125 125
126 spin_lock(&cache->lock); 126 spin_lock(&cache->lock);
127 127
@@ -220,6 +220,7 @@ void squashfs_cache_delete(struct squashfs_cache *cache)
220 kfree(cache->entry[i].data[j]); 220 kfree(cache->entry[i].data[j]);
221 kfree(cache->entry[i].data); 221 kfree(cache->entry[i].data);
222 } 222 }
223 kfree(cache->entry[i].actor);
223 } 224 }
224 225
225 kfree(cache->entry); 226 kfree(cache->entry);
@@ -280,6 +281,13 @@ struct squashfs_cache *squashfs_cache_init(char *name, int entries,
280 goto cleanup; 281 goto cleanup;
281 } 282 }
282 } 283 }
284
285 entry->actor = squashfs_page_actor_init(entry->data,
286 cache->pages, 0);
287 if (entry->actor == NULL) {
288 ERROR("Failed to allocate %s cache entry\n", name);
289 goto cleanup;
290 }
283 } 291 }
284 292
285 return cache; 293 return cache;
@@ -410,6 +418,7 @@ void *squashfs_read_table(struct super_block *sb, u64 block, int length)
410 int pages = (length + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT; 418 int pages = (length + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
411 int i, res; 419 int i, res;
412 void *table, *buffer, **data; 420 void *table, *buffer, **data;
421 struct squashfs_page_actor *actor;
413 422
414 table = buffer = kmalloc(length, GFP_KERNEL); 423 table = buffer = kmalloc(length, GFP_KERNEL);
415 if (table == NULL) 424 if (table == NULL)
@@ -421,19 +430,28 @@ void *squashfs_read_table(struct super_block *sb, u64 block, int length)
421 goto failed; 430 goto failed;
422 } 431 }
423 432
433 actor = squashfs_page_actor_init(data, pages, length);
434 if (actor == NULL) {
435 res = -ENOMEM;
436 goto failed2;
437 }
438
424 for (i = 0; i < pages; i++, buffer += PAGE_CACHE_SIZE) 439 for (i = 0; i < pages; i++, buffer += PAGE_CACHE_SIZE)
425 data[i] = buffer; 440 data[i] = buffer;
426 441
427 res = squashfs_read_data(sb, data, block, length | 442 res = squashfs_read_data(sb, block, length |
428 SQUASHFS_COMPRESSED_BIT_BLOCK, NULL, length, pages); 443 SQUASHFS_COMPRESSED_BIT_BLOCK, NULL, actor);
429 444
430 kfree(data); 445 kfree(data);
446 kfree(actor);
431 447
432 if (res < 0) 448 if (res < 0)
433 goto failed; 449 goto failed;
434 450
435 return table; 451 return table;
436 452
453failed2:
454 kfree(data);
437failed: 455failed:
438 kfree(table); 456 kfree(table);
439 return ERR_PTR(res); 457 return ERR_PTR(res);
diff --git a/fs/squashfs/decompressor.c b/fs/squashfs/decompressor.c
index 3f6271d86abc..ac22fe73b0ad 100644
--- a/fs/squashfs/decompressor.c
+++ b/fs/squashfs/decompressor.c
@@ -30,6 +30,7 @@
30#include "squashfs_fs_sb.h" 30#include "squashfs_fs_sb.h"
31#include "decompressor.h" 31#include "decompressor.h"
32#include "squashfs.h" 32#include "squashfs.h"
33#include "page_actor.h"
33 34
34/* 35/*
35 * This file (and decompressor.h) implements a decompressor framework for 36 * This file (and decompressor.h) implements a decompressor framework for
@@ -37,29 +38,29 @@
37 */ 38 */
38 39
39static const struct squashfs_decompressor squashfs_lzma_unsupported_comp_ops = { 40static const struct squashfs_decompressor squashfs_lzma_unsupported_comp_ops = {
40 NULL, NULL, NULL, LZMA_COMPRESSION, "lzma", 0 41 NULL, NULL, NULL, NULL, LZMA_COMPRESSION, "lzma", 0
41}; 42};
42 43
43#ifndef CONFIG_SQUASHFS_LZO 44#ifndef CONFIG_SQUASHFS_LZO
44static const struct squashfs_decompressor squashfs_lzo_comp_ops = { 45static const struct squashfs_decompressor squashfs_lzo_comp_ops = {
45 NULL, NULL, NULL, LZO_COMPRESSION, "lzo", 0 46 NULL, NULL, NULL, NULL, LZO_COMPRESSION, "lzo", 0
46}; 47};
47#endif 48#endif
48 49
49#ifndef CONFIG_SQUASHFS_XZ 50#ifndef CONFIG_SQUASHFS_XZ
50static const struct squashfs_decompressor squashfs_xz_comp_ops = { 51static const struct squashfs_decompressor squashfs_xz_comp_ops = {
51 NULL, NULL, NULL, XZ_COMPRESSION, "xz", 0 52 NULL, NULL, NULL, NULL, XZ_COMPRESSION, "xz", 0
52}; 53};
53#endif 54#endif
54 55
55#ifndef CONFIG_SQUASHFS_ZLIB 56#ifndef CONFIG_SQUASHFS_ZLIB
56static const struct squashfs_decompressor squashfs_zlib_comp_ops = { 57static const struct squashfs_decompressor squashfs_zlib_comp_ops = {
57 NULL, NULL, NULL, ZLIB_COMPRESSION, "zlib", 0 58 NULL, NULL, NULL, NULL, ZLIB_COMPRESSION, "zlib", 0
58}; 59};
59#endif 60#endif
60 61
61static const struct squashfs_decompressor squashfs_unknown_comp_ops = { 62static const struct squashfs_decompressor squashfs_unknown_comp_ops = {
62 NULL, NULL, NULL, 0, "unknown", 0 63 NULL, NULL, NULL, NULL, 0, "unknown", 0
63}; 64};
64 65
65static const struct squashfs_decompressor *decompressor[] = { 66static const struct squashfs_decompressor *decompressor[] = {
@@ -83,10 +84,11 @@ const struct squashfs_decompressor *squashfs_lookup_decompressor(int id)
83} 84}
84 85
85 86
86void *squashfs_decompressor_init(struct super_block *sb, unsigned short flags) 87static void *get_comp_opts(struct super_block *sb, unsigned short flags)
87{ 88{
88 struct squashfs_sb_info *msblk = sb->s_fs_info; 89 struct squashfs_sb_info *msblk = sb->s_fs_info;
89 void *strm, *buffer = NULL; 90 void *buffer = NULL, *comp_opts;
91 struct squashfs_page_actor *actor = NULL;
90 int length = 0; 92 int length = 0;
91 93
92 /* 94 /*
@@ -94,23 +96,46 @@ void *squashfs_decompressor_init(struct super_block *sb, unsigned short flags)
94 */ 96 */
95 if (SQUASHFS_COMP_OPTS(flags)) { 97 if (SQUASHFS_COMP_OPTS(flags)) {
96 buffer = kmalloc(PAGE_CACHE_SIZE, GFP_KERNEL); 98 buffer = kmalloc(PAGE_CACHE_SIZE, GFP_KERNEL);
97 if (buffer == NULL) 99 if (buffer == NULL) {
98 return ERR_PTR(-ENOMEM); 100 comp_opts = ERR_PTR(-ENOMEM);
101 goto out;
102 }
103
104 actor = squashfs_page_actor_init(&buffer, 1, 0);
105 if (actor == NULL) {
106 comp_opts = ERR_PTR(-ENOMEM);
107 goto out;
108 }
99 109
100 length = squashfs_read_data(sb, &buffer, 110 length = squashfs_read_data(sb,
101 sizeof(struct squashfs_super_block), 0, NULL, 111 sizeof(struct squashfs_super_block), 0, NULL, actor);
102 PAGE_CACHE_SIZE, 1);
103 112
104 if (length < 0) { 113 if (length < 0) {
105 strm = ERR_PTR(length); 114 comp_opts = ERR_PTR(length);
106 goto finished; 115 goto out;
107 } 116 }
108 } 117 }
109 118
110 strm = msblk->decompressor->init(msblk, buffer, length); 119 comp_opts = squashfs_comp_opts(msblk, buffer, length);
111 120
112finished: 121out:
122 kfree(actor);
113 kfree(buffer); 123 kfree(buffer);
124 return comp_opts;
125}
126
127
128void *squashfs_decompressor_setup(struct super_block *sb, unsigned short flags)
129{
130 struct squashfs_sb_info *msblk = sb->s_fs_info;
131 void *stream, *comp_opts = get_comp_opts(sb, flags);
132
133 if (IS_ERR(comp_opts))
134 return comp_opts;
135
136 stream = squashfs_decompressor_create(msblk, comp_opts);
137 if (IS_ERR(stream))
138 kfree(comp_opts);
114 139
115 return strm; 140 return stream;
116} 141}
diff --git a/fs/squashfs/decompressor.h b/fs/squashfs/decompressor.h
index 330073e29029..af0985321808 100644
--- a/fs/squashfs/decompressor.h
+++ b/fs/squashfs/decompressor.h
@@ -24,28 +24,22 @@
24 */ 24 */
25 25
26struct squashfs_decompressor { 26struct squashfs_decompressor {
27 void *(*init)(struct squashfs_sb_info *, void *, int); 27 void *(*init)(struct squashfs_sb_info *, void *);
28 void *(*comp_opts)(struct squashfs_sb_info *, void *, int);
28 void (*free)(void *); 29 void (*free)(void *);
29 int (*decompress)(struct squashfs_sb_info *, void **, 30 int (*decompress)(struct squashfs_sb_info *, void *,
30 struct buffer_head **, int, int, int, int, int); 31 struct buffer_head **, int, int, int,
32 struct squashfs_page_actor *);
31 int id; 33 int id;
32 char *name; 34 char *name;
33 int supported; 35 int supported;
34}; 36};
35 37
36static inline void squashfs_decompressor_free(struct squashfs_sb_info *msblk, 38static inline void *squashfs_comp_opts(struct squashfs_sb_info *msblk,
37 void *s) 39 void *buff, int length)
38{ 40{
39 if (msblk->decompressor) 41 return msblk->decompressor->comp_opts ?
40 msblk->decompressor->free(s); 42 msblk->decompressor->comp_opts(msblk, buff, length) : NULL;
41}
42
43static inline int squashfs_decompress(struct squashfs_sb_info *msblk,
44 void **buffer, struct buffer_head **bh, int b, int offset, int length,
45 int srclength, int pages)
46{
47 return msblk->decompressor->decompress(msblk, buffer, bh, b, offset,
48 length, srclength, pages);
49} 43}
50 44
51#ifdef CONFIG_SQUASHFS_XZ 45#ifdef CONFIG_SQUASHFS_XZ
diff --git a/fs/squashfs/decompressor_multi.c b/fs/squashfs/decompressor_multi.c
new file mode 100644
index 000000000000..d6008a636479
--- /dev/null
+++ b/fs/squashfs/decompressor_multi.c
@@ -0,0 +1,198 @@
1/*
2 * Copyright (c) 2013
3 * Minchan Kim <minchan@kernel.org>
4 *
5 * This work is licensed under the terms of the GNU GPL, version 2. See
6 * the COPYING file in the top-level directory.
7 */
8#include <linux/types.h>
9#include <linux/mutex.h>
10#include <linux/slab.h>
11#include <linux/buffer_head.h>
12#include <linux/sched.h>
13#include <linux/wait.h>
14#include <linux/cpumask.h>
15
16#include "squashfs_fs.h"
17#include "squashfs_fs_sb.h"
18#include "decompressor.h"
19#include "squashfs.h"
20
21/*
22 * This file implements multi-threaded decompression in the
23 * decompressor framework
24 */
25
26
27/*
28 * The reason that multiply two is that a CPU can request new I/O
29 * while it is waiting previous request.
30 */
31#define MAX_DECOMPRESSOR (num_online_cpus() * 2)
32
33
34int squashfs_max_decompressors(void)
35{
36 return MAX_DECOMPRESSOR;
37}
38
39
40struct squashfs_stream {
41 void *comp_opts;
42 struct list_head strm_list;
43 struct mutex mutex;
44 int avail_decomp;
45 wait_queue_head_t wait;
46};
47
48
49struct decomp_stream {
50 void *stream;
51 struct list_head list;
52};
53
54
55static void put_decomp_stream(struct decomp_stream *decomp_strm,
56 struct squashfs_stream *stream)
57{
58 mutex_lock(&stream->mutex);
59 list_add(&decomp_strm->list, &stream->strm_list);
60 mutex_unlock(&stream->mutex);
61 wake_up(&stream->wait);
62}
63
64void *squashfs_decompressor_create(struct squashfs_sb_info *msblk,
65 void *comp_opts)
66{
67 struct squashfs_stream *stream;
68 struct decomp_stream *decomp_strm = NULL;
69 int err = -ENOMEM;
70
71 stream = kzalloc(sizeof(*stream), GFP_KERNEL);
72 if (!stream)
73 goto out;
74
75 stream->comp_opts = comp_opts;
76 mutex_init(&stream->mutex);
77 INIT_LIST_HEAD(&stream->strm_list);
78 init_waitqueue_head(&stream->wait);
79
80 /*
81 * We should have a decompressor at least as default
82 * so if we fail to allocate new decompressor dynamically,
83 * we could always fall back to default decompressor and
84 * file system works.
85 */
86 decomp_strm = kmalloc(sizeof(*decomp_strm), GFP_KERNEL);
87 if (!decomp_strm)
88 goto out;
89
90 decomp_strm->stream = msblk->decompressor->init(msblk,
91 stream->comp_opts);
92 if (IS_ERR(decomp_strm->stream)) {
93 err = PTR_ERR(decomp_strm->stream);
94 goto out;
95 }
96
97 list_add(&decomp_strm->list, &stream->strm_list);
98 stream->avail_decomp = 1;
99 return stream;
100
101out:
102 kfree(decomp_strm);
103 kfree(stream);
104 return ERR_PTR(err);
105}
106
107
108void squashfs_decompressor_destroy(struct squashfs_sb_info *msblk)
109{
110 struct squashfs_stream *stream = msblk->stream;
111 if (stream) {
112 struct decomp_stream *decomp_strm;
113
114 while (!list_empty(&stream->strm_list)) {
115 decomp_strm = list_entry(stream->strm_list.prev,
116 struct decomp_stream, list);
117 list_del(&decomp_strm->list);
118 msblk->decompressor->free(decomp_strm->stream);
119 kfree(decomp_strm);
120 stream->avail_decomp--;
121 }
122 WARN_ON(stream->avail_decomp);
123 kfree(stream->comp_opts);
124 kfree(stream);
125 }
126}
127
128
129static struct decomp_stream *get_decomp_stream(struct squashfs_sb_info *msblk,
130 struct squashfs_stream *stream)
131{
132 struct decomp_stream *decomp_strm;
133
134 while (1) {
135 mutex_lock(&stream->mutex);
136
137 /* There is available decomp_stream */
138 if (!list_empty(&stream->strm_list)) {
139 decomp_strm = list_entry(stream->strm_list.prev,
140 struct decomp_stream, list);
141 list_del(&decomp_strm->list);
142 mutex_unlock(&stream->mutex);
143 break;
144 }
145
146 /*
147 * If there is no available decomp and already full,
148 * let's wait for releasing decomp from other users.
149 */
150 if (stream->avail_decomp >= MAX_DECOMPRESSOR)
151 goto wait;
152
153 /* Let's allocate new decomp */
154 decomp_strm = kmalloc(sizeof(*decomp_strm), GFP_KERNEL);
155 if (!decomp_strm)
156 goto wait;
157
158 decomp_strm->stream = msblk->decompressor->init(msblk,
159 stream->comp_opts);
160 if (IS_ERR(decomp_strm->stream)) {
161 kfree(decomp_strm);
162 goto wait;
163 }
164
165 stream->avail_decomp++;
166 WARN_ON(stream->avail_decomp > MAX_DECOMPRESSOR);
167
168 mutex_unlock(&stream->mutex);
169 break;
170wait:
171 /*
172 * If system memory is tough, let's for other's
173 * releasing instead of hurting VM because it could
174 * make page cache thrashing.
175 */
176 mutex_unlock(&stream->mutex);
177 wait_event(stream->wait,
178 !list_empty(&stream->strm_list));
179 }
180
181 return decomp_strm;
182}
183
184
185int squashfs_decompress(struct squashfs_sb_info *msblk, struct buffer_head **bh,
186 int b, int offset, int length, struct squashfs_page_actor *output)
187{
188 int res;
189 struct squashfs_stream *stream = msblk->stream;
190 struct decomp_stream *decomp_stream = get_decomp_stream(msblk, stream);
191 res = msblk->decompressor->decompress(msblk, decomp_stream->stream,
192 bh, b, offset, length, output);
193 put_decomp_stream(decomp_stream, stream);
194 if (res < 0)
195 ERROR("%s decompression failed, data probably corrupt\n",
196 msblk->decompressor->name);
197 return res;
198}
diff --git a/fs/squashfs/decompressor_multi_percpu.c b/fs/squashfs/decompressor_multi_percpu.c
new file mode 100644
index 000000000000..23a9c28ad8ea
--- /dev/null
+++ b/fs/squashfs/decompressor_multi_percpu.c
@@ -0,0 +1,97 @@
1/*
2 * Copyright (c) 2013
3 * Phillip Lougher <phillip@squashfs.org.uk>
4 *
5 * This work is licensed under the terms of the GNU GPL, version 2. See
6 * the COPYING file in the top-level directory.
7 */
8
9#include <linux/types.h>
10#include <linux/slab.h>
11#include <linux/percpu.h>
12#include <linux/buffer_head.h>
13
14#include "squashfs_fs.h"
15#include "squashfs_fs_sb.h"
16#include "decompressor.h"
17#include "squashfs.h"
18
19/*
20 * This file implements multi-threaded decompression using percpu
21 * variables, one thread per cpu core.
22 */
23
24struct squashfs_stream {
25 void *stream;
26};
27
28void *squashfs_decompressor_create(struct squashfs_sb_info *msblk,
29 void *comp_opts)
30{
31 struct squashfs_stream *stream;
32 struct squashfs_stream __percpu *percpu;
33 int err, cpu;
34
35 percpu = alloc_percpu(struct squashfs_stream);
36 if (percpu == NULL)
37 return ERR_PTR(-ENOMEM);
38
39 for_each_possible_cpu(cpu) {
40 stream = per_cpu_ptr(percpu, cpu);
41 stream->stream = msblk->decompressor->init(msblk, comp_opts);
42 if (IS_ERR(stream->stream)) {
43 err = PTR_ERR(stream->stream);
44 goto out;
45 }
46 }
47
48 kfree(comp_opts);
49 return (__force void *) percpu;
50
51out:
52 for_each_possible_cpu(cpu) {
53 stream = per_cpu_ptr(percpu, cpu);
54 if (!IS_ERR_OR_NULL(stream->stream))
55 msblk->decompressor->free(stream->stream);
56 }
57 free_percpu(percpu);
58 return ERR_PTR(err);
59}
60
61void squashfs_decompressor_destroy(struct squashfs_sb_info *msblk)
62{
63 struct squashfs_stream __percpu *percpu =
64 (struct squashfs_stream __percpu *) msblk->stream;
65 struct squashfs_stream *stream;
66 int cpu;
67
68 if (msblk->stream) {
69 for_each_possible_cpu(cpu) {
70 stream = per_cpu_ptr(percpu, cpu);
71 msblk->decompressor->free(stream->stream);
72 }
73 free_percpu(percpu);
74 }
75}
76
77int squashfs_decompress(struct squashfs_sb_info *msblk, struct buffer_head **bh,
78 int b, int offset, int length, struct squashfs_page_actor *output)
79{
80 struct squashfs_stream __percpu *percpu =
81 (struct squashfs_stream __percpu *) msblk->stream;
82 struct squashfs_stream *stream = get_cpu_ptr(percpu);
83 int res = msblk->decompressor->decompress(msblk, stream->stream, bh, b,
84 offset, length, output);
85 put_cpu_ptr(stream);
86
87 if (res < 0)
88 ERROR("%s decompression failed, data probably corrupt\n",
89 msblk->decompressor->name);
90
91 return res;
92}
93
94int squashfs_max_decompressors(void)
95{
96 return num_possible_cpus();
97}
diff --git a/fs/squashfs/decompressor_single.c b/fs/squashfs/decompressor_single.c
new file mode 100644
index 000000000000..a6c75929a00e
--- /dev/null
+++ b/fs/squashfs/decompressor_single.c
@@ -0,0 +1,85 @@
1/*
2 * Copyright (c) 2013
3 * Phillip Lougher <phillip@squashfs.org.uk>
4 *
5 * This work is licensed under the terms of the GNU GPL, version 2. See
6 * the COPYING file in the top-level directory.
7 */
8
9#include <linux/types.h>
10#include <linux/mutex.h>
11#include <linux/slab.h>
12#include <linux/buffer_head.h>
13
14#include "squashfs_fs.h"
15#include "squashfs_fs_sb.h"
16#include "decompressor.h"
17#include "squashfs.h"
18
19/*
20 * This file implements single-threaded decompression in the
21 * decompressor framework
22 */
23
24struct squashfs_stream {
25 void *stream;
26 struct mutex mutex;
27};
28
29void *squashfs_decompressor_create(struct squashfs_sb_info *msblk,
30 void *comp_opts)
31{
32 struct squashfs_stream *stream;
33 int err = -ENOMEM;
34
35 stream = kmalloc(sizeof(*stream), GFP_KERNEL);
36 if (stream == NULL)
37 goto out;
38
39 stream->stream = msblk->decompressor->init(msblk, comp_opts);
40 if (IS_ERR(stream->stream)) {
41 err = PTR_ERR(stream->stream);
42 goto out;
43 }
44
45 kfree(comp_opts);
46 mutex_init(&stream->mutex);
47 return stream;
48
49out:
50 kfree(stream);
51 return ERR_PTR(err);
52}
53
54void squashfs_decompressor_destroy(struct squashfs_sb_info *msblk)
55{
56 struct squashfs_stream *stream = msblk->stream;
57
58 if (stream) {
59 msblk->decompressor->free(stream->stream);
60 kfree(stream);
61 }
62}
63
64int squashfs_decompress(struct squashfs_sb_info *msblk, struct buffer_head **bh,
65 int b, int offset, int length, struct squashfs_page_actor *output)
66{
67 int res;
68 struct squashfs_stream *stream = msblk->stream;
69
70 mutex_lock(&stream->mutex);
71 res = msblk->decompressor->decompress(msblk, stream->stream, bh, b,
72 offset, length, output);
73 mutex_unlock(&stream->mutex);
74
75 if (res < 0)
76 ERROR("%s decompression failed, data probably corrupt\n",
77 msblk->decompressor->name);
78
79 return res;
80}
81
82int squashfs_max_decompressors(void)
83{
84 return 1;
85}
diff --git a/fs/squashfs/file.c b/fs/squashfs/file.c
index 8ca62c28fe12..e5c9689062ba 100644
--- a/fs/squashfs/file.c
+++ b/fs/squashfs/file.c
@@ -370,77 +370,15 @@ static int read_blocklist(struct inode *inode, int index, u64 *block)
370 return le32_to_cpu(size); 370 return le32_to_cpu(size);
371} 371}
372 372
373 373/* Copy data into page cache */
374static int squashfs_readpage(struct file *file, struct page *page) 374void squashfs_copy_cache(struct page *page, struct squashfs_cache_entry *buffer,
375 int bytes, int offset)
375{ 376{
376 struct inode *inode = page->mapping->host; 377 struct inode *inode = page->mapping->host;
377 struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info; 378 struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info;
378 int bytes, i, offset = 0, sparse = 0;
379 struct squashfs_cache_entry *buffer = NULL;
380 void *pageaddr; 379 void *pageaddr;
381 380 int i, mask = (1 << (msblk->block_log - PAGE_CACHE_SHIFT)) - 1;
382 int mask = (1 << (msblk->block_log - PAGE_CACHE_SHIFT)) - 1; 381 int start_index = page->index & ~mask, end_index = start_index | mask;
383 int index = page->index >> (msblk->block_log - PAGE_CACHE_SHIFT);
384 int start_index = page->index & ~mask;
385 int end_index = start_index | mask;
386 int file_end = i_size_read(inode) >> msblk->block_log;
387
388 TRACE("Entered squashfs_readpage, page index %lx, start block %llx\n",
389 page->index, squashfs_i(inode)->start);
390
391 if (page->index >= ((i_size_read(inode) + PAGE_CACHE_SIZE - 1) >>
392 PAGE_CACHE_SHIFT))
393 goto out;
394
395 if (index < file_end || squashfs_i(inode)->fragment_block ==
396 SQUASHFS_INVALID_BLK) {
397 /*
398 * Reading a datablock from disk. Need to read block list
399 * to get location and block size.
400 */
401 u64 block = 0;
402 int bsize = read_blocklist(inode, index, &block);
403 if (bsize < 0)
404 goto error_out;
405
406 if (bsize == 0) { /* hole */
407 bytes = index == file_end ?
408 (i_size_read(inode) & (msblk->block_size - 1)) :
409 msblk->block_size;
410 sparse = 1;
411 } else {
412 /*
413 * Read and decompress datablock.
414 */
415 buffer = squashfs_get_datablock(inode->i_sb,
416 block, bsize);
417 if (buffer->error) {
418 ERROR("Unable to read page, block %llx, size %x"
419 "\n", block, bsize);
420 squashfs_cache_put(buffer);
421 goto error_out;
422 }
423 bytes = buffer->length;
424 }
425 } else {
426 /*
427 * Datablock is stored inside a fragment (tail-end packed
428 * block).
429 */
430 buffer = squashfs_get_fragment(inode->i_sb,
431 squashfs_i(inode)->fragment_block,
432 squashfs_i(inode)->fragment_size);
433
434 if (buffer->error) {
435 ERROR("Unable to read page, block %llx, size %x\n",
436 squashfs_i(inode)->fragment_block,
437 squashfs_i(inode)->fragment_size);
438 squashfs_cache_put(buffer);
439 goto error_out;
440 }
441 bytes = i_size_read(inode) & (msblk->block_size - 1);
442 offset = squashfs_i(inode)->fragment_offset;
443 }
444 382
445 /* 383 /*
446 * Loop copying datablock into pages. As the datablock likely covers 384 * Loop copying datablock into pages. As the datablock likely covers
@@ -451,7 +389,7 @@ static int squashfs_readpage(struct file *file, struct page *page)
451 for (i = start_index; i <= end_index && bytes > 0; i++, 389 for (i = start_index; i <= end_index && bytes > 0; i++,
452 bytes -= PAGE_CACHE_SIZE, offset += PAGE_CACHE_SIZE) { 390 bytes -= PAGE_CACHE_SIZE, offset += PAGE_CACHE_SIZE) {
453 struct page *push_page; 391 struct page *push_page;
454 int avail = sparse ? 0 : min_t(int, bytes, PAGE_CACHE_SIZE); 392 int avail = buffer ? min_t(int, bytes, PAGE_CACHE_SIZE) : 0;
455 393
456 TRACE("bytes %d, i %d, available_bytes %d\n", bytes, i, avail); 394 TRACE("bytes %d, i %d, available_bytes %d\n", bytes, i, avail);
457 395
@@ -475,11 +413,75 @@ skip_page:
475 if (i != page->index) 413 if (i != page->index)
476 page_cache_release(push_page); 414 page_cache_release(push_page);
477 } 415 }
416}
417
418/* Read datablock stored packed inside a fragment (tail-end packed block) */
419static int squashfs_readpage_fragment(struct page *page)
420{
421 struct inode *inode = page->mapping->host;
422 struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info;
423 struct squashfs_cache_entry *buffer = squashfs_get_fragment(inode->i_sb,
424 squashfs_i(inode)->fragment_block,
425 squashfs_i(inode)->fragment_size);
426 int res = buffer->error;
427
428 if (res)
429 ERROR("Unable to read page, block %llx, size %x\n",
430 squashfs_i(inode)->fragment_block,
431 squashfs_i(inode)->fragment_size);
432 else
433 squashfs_copy_cache(page, buffer, i_size_read(inode) &
434 (msblk->block_size - 1),
435 squashfs_i(inode)->fragment_offset);
436
437 squashfs_cache_put(buffer);
438 return res;
439}
478 440
479 if (!sparse) 441static int squashfs_readpage_sparse(struct page *page, int index, int file_end)
480 squashfs_cache_put(buffer); 442{
443 struct inode *inode = page->mapping->host;
444 struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info;
445 int bytes = index == file_end ?
446 (i_size_read(inode) & (msblk->block_size - 1)) :
447 msblk->block_size;
481 448
449 squashfs_copy_cache(page, NULL, bytes, 0);
482 return 0; 450 return 0;
451}
452
453static int squashfs_readpage(struct file *file, struct page *page)
454{
455 struct inode *inode = page->mapping->host;
456 struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info;
457 int index = page->index >> (msblk->block_log - PAGE_CACHE_SHIFT);
458 int file_end = i_size_read(inode) >> msblk->block_log;
459 int res;
460 void *pageaddr;
461
462 TRACE("Entered squashfs_readpage, page index %lx, start block %llx\n",
463 page->index, squashfs_i(inode)->start);
464
465 if (page->index >= ((i_size_read(inode) + PAGE_CACHE_SIZE - 1) >>
466 PAGE_CACHE_SHIFT))
467 goto out;
468
469 if (index < file_end || squashfs_i(inode)->fragment_block ==
470 SQUASHFS_INVALID_BLK) {
471 u64 block = 0;
472 int bsize = read_blocklist(inode, index, &block);
473 if (bsize < 0)
474 goto error_out;
475
476 if (bsize == 0)
477 res = squashfs_readpage_sparse(page, index, file_end);
478 else
479 res = squashfs_readpage_block(page, block, bsize);
480 } else
481 res = squashfs_readpage_fragment(page);
482
483 if (!res)
484 return 0;
483 485
484error_out: 486error_out:
485 SetPageError(page); 487 SetPageError(page);
diff --git a/fs/squashfs/file_cache.c b/fs/squashfs/file_cache.c
new file mode 100644
index 000000000000..f2310d2a2019
--- /dev/null
+++ b/fs/squashfs/file_cache.c
@@ -0,0 +1,38 @@
1/*
2 * Copyright (c) 2013
3 * Phillip Lougher <phillip@squashfs.org.uk>
4 *
5 * This work is licensed under the terms of the GNU GPL, version 2. See
6 * the COPYING file in the top-level directory.
7 */
8
9#include <linux/fs.h>
10#include <linux/vfs.h>
11#include <linux/kernel.h>
12#include <linux/slab.h>
13#include <linux/string.h>
14#include <linux/pagemap.h>
15#include <linux/mutex.h>
16
17#include "squashfs_fs.h"
18#include "squashfs_fs_sb.h"
19#include "squashfs_fs_i.h"
20#include "squashfs.h"
21
22/* Read separately compressed datablock and memcopy into page cache */
23int squashfs_readpage_block(struct page *page, u64 block, int bsize)
24{
25 struct inode *i = page->mapping->host;
26 struct squashfs_cache_entry *buffer = squashfs_get_datablock(i->i_sb,
27 block, bsize);
28 int res = buffer->error;
29
30 if (res)
31 ERROR("Unable to read page, block %llx, size %x\n", block,
32 bsize);
33 else
34 squashfs_copy_cache(page, buffer, buffer->length, 0);
35
36 squashfs_cache_put(buffer);
37 return res;
38}
diff --git a/fs/squashfs/file_direct.c b/fs/squashfs/file_direct.c
new file mode 100644
index 000000000000..2943b2bfae48
--- /dev/null
+++ b/fs/squashfs/file_direct.c
@@ -0,0 +1,173 @@
1/*
2 * Copyright (c) 2013
3 * Phillip Lougher <phillip@squashfs.org.uk>
4 *
5 * This work is licensed under the terms of the GNU GPL, version 2. See
6 * the COPYING file in the top-level directory.
7 */
8
9#include <linux/fs.h>
10#include <linux/vfs.h>
11#include <linux/kernel.h>
12#include <linux/slab.h>
13#include <linux/string.h>
14#include <linux/pagemap.h>
15#include <linux/mutex.h>
16
17#include "squashfs_fs.h"
18#include "squashfs_fs_sb.h"
19#include "squashfs_fs_i.h"
20#include "squashfs.h"
21#include "page_actor.h"
22
23static int squashfs_read_cache(struct page *target_page, u64 block, int bsize,
24 int pages, struct page **page);
25
26/* Read separately compressed datablock directly into page cache */
27int squashfs_readpage_block(struct page *target_page, u64 block, int bsize)
28
29{
30 struct inode *inode = target_page->mapping->host;
31 struct squashfs_sb_info *msblk = inode->i_sb->s_fs_info;
32
33 int file_end = (i_size_read(inode) - 1) >> PAGE_CACHE_SHIFT;
34 int mask = (1 << (msblk->block_log - PAGE_CACHE_SHIFT)) - 1;
35 int start_index = target_page->index & ~mask;
36 int end_index = start_index | mask;
37 int i, n, pages, missing_pages, bytes, res = -ENOMEM;
38 struct page **page;
39 struct squashfs_page_actor *actor;
40 void *pageaddr;
41
42 if (end_index > file_end)
43 end_index = file_end;
44
45 pages = end_index - start_index + 1;
46
47 page = kmalloc(sizeof(void *) * pages, GFP_KERNEL);
48 if (page == NULL)
49 return res;
50
51 /*
52 * Create a "page actor" which will kmap and kunmap the
53 * page cache pages appropriately within the decompressor
54 */
55 actor = squashfs_page_actor_init_special(page, pages, 0);
56 if (actor == NULL)
57 goto out;
58
59 /* Try to grab all the pages covered by the Squashfs block */
60 for (missing_pages = 0, i = 0, n = start_index; i < pages; i++, n++) {
61 page[i] = (n == target_page->index) ? target_page :
62 grab_cache_page_nowait(target_page->mapping, n);
63
64 if (page[i] == NULL) {
65 missing_pages++;
66 continue;
67 }
68
69 if (PageUptodate(page[i])) {
70 unlock_page(page[i]);
71 page_cache_release(page[i]);
72 page[i] = NULL;
73 missing_pages++;
74 }
75 }
76
77 if (missing_pages) {
78 /*
79 * Couldn't get one or more pages, this page has either
80 * been VM reclaimed, but others are still in the page cache
81 * and uptodate, or we're racing with another thread in
82 * squashfs_readpage also trying to grab them. Fall back to
83 * using an intermediate buffer.
84 */
85 res = squashfs_read_cache(target_page, block, bsize, pages,
86 page);
87 goto out;
88 }
89
90 /* Decompress directly into the page cache buffers */
91 res = squashfs_read_data(inode->i_sb, block, bsize, NULL, actor);
92 if (res < 0)
93 goto mark_errored;
94
95 /* Last page may have trailing bytes not filled */
96 bytes = res % PAGE_CACHE_SIZE;
97 if (bytes) {
98 pageaddr = kmap_atomic(page[pages - 1]);
99 memset(pageaddr + bytes, 0, PAGE_CACHE_SIZE - bytes);
100 kunmap_atomic(pageaddr);
101 }
102
103 /* Mark pages as uptodate, unlock and release */
104 for (i = 0; i < pages; i++) {
105 flush_dcache_page(page[i]);
106 SetPageUptodate(page[i]);
107 unlock_page(page[i]);
108 if (page[i] != target_page)
109 page_cache_release(page[i]);
110 }
111
112 kfree(actor);
113 kfree(page);
114
115 return 0;
116
117mark_errored:
118 /* Decompression failed, mark pages as errored. Target_page is
119 * dealt with by the caller
120 */
121 for (i = 0; i < pages; i++) {
122 if (page[i] == target_page)
123 continue;
124 flush_dcache_page(page[i]);
125 SetPageError(page[i]);
126 unlock_page(page[i]);
127 page_cache_release(page[i]);
128 }
129
130out:
131 kfree(actor);
132 kfree(page);
133 return res;
134}
135
136
137static int squashfs_read_cache(struct page *target_page, u64 block, int bsize,
138 int pages, struct page **page)
139{
140 struct inode *i = target_page->mapping->host;
141 struct squashfs_cache_entry *buffer = squashfs_get_datablock(i->i_sb,
142 block, bsize);
143 int bytes = buffer->length, res = buffer->error, n, offset = 0;
144 void *pageaddr;
145
146 if (res) {
147 ERROR("Unable to read page, block %llx, size %x\n", block,
148 bsize);
149 goto out;
150 }
151
152 for (n = 0; n < pages && bytes > 0; n++,
153 bytes -= PAGE_CACHE_SIZE, offset += PAGE_CACHE_SIZE) {
154 int avail = min_t(int, bytes, PAGE_CACHE_SIZE);
155
156 if (page[n] == NULL)
157 continue;
158
159 pageaddr = kmap_atomic(page[n]);
160 squashfs_copy_data(pageaddr, buffer, offset, avail);
161 memset(pageaddr + avail, 0, PAGE_CACHE_SIZE - avail);
162 kunmap_atomic(pageaddr);
163 flush_dcache_page(page[n]);
164 SetPageUptodate(page[n]);
165 unlock_page(page[n]);
166 if (page[n] != target_page)
167 page_cache_release(page[n]);
168 }
169
170out:
171 squashfs_cache_put(buffer);
172 return res;
173}
diff --git a/fs/squashfs/lzo_wrapper.c b/fs/squashfs/lzo_wrapper.c
index 00f4dfc5f088..244b9fbfff7b 100644
--- a/fs/squashfs/lzo_wrapper.c
+++ b/fs/squashfs/lzo_wrapper.c
@@ -31,13 +31,14 @@
31#include "squashfs_fs_sb.h" 31#include "squashfs_fs_sb.h"
32#include "squashfs.h" 32#include "squashfs.h"
33#include "decompressor.h" 33#include "decompressor.h"
34#include "page_actor.h"
34 35
35struct squashfs_lzo { 36struct squashfs_lzo {
36 void *input; 37 void *input;
37 void *output; 38 void *output;
38}; 39};
39 40
40static void *lzo_init(struct squashfs_sb_info *msblk, void *buff, int len) 41static void *lzo_init(struct squashfs_sb_info *msblk, void *buff)
41{ 42{
42 int block_size = max_t(int, msblk->block_size, SQUASHFS_METADATA_SIZE); 43 int block_size = max_t(int, msblk->block_size, SQUASHFS_METADATA_SIZE);
43 44
@@ -74,22 +75,16 @@ static void lzo_free(void *strm)
74} 75}
75 76
76 77
77static int lzo_uncompress(struct squashfs_sb_info *msblk, void **buffer, 78static int lzo_uncompress(struct squashfs_sb_info *msblk, void *strm,
78 struct buffer_head **bh, int b, int offset, int length, int srclength, 79 struct buffer_head **bh, int b, int offset, int length,
79 int pages) 80 struct squashfs_page_actor *output)
80{ 81{
81 struct squashfs_lzo *stream = msblk->stream; 82 struct squashfs_lzo *stream = strm;
82 void *buff = stream->input; 83 void *buff = stream->input, *data;
83 int avail, i, bytes = length, res; 84 int avail, i, bytes = length, res;
84 size_t out_len = srclength; 85 size_t out_len = output->length;
85
86 mutex_lock(&msblk->read_data_mutex);
87 86
88 for (i = 0; i < b; i++) { 87 for (i = 0; i < b; i++) {
89 wait_on_buffer(bh[i]);
90 if (!buffer_uptodate(bh[i]))
91 goto block_release;
92
93 avail = min(bytes, msblk->devblksize - offset); 88 avail = min(bytes, msblk->devblksize - offset);
94 memcpy(buff, bh[i]->b_data + offset, avail); 89 memcpy(buff, bh[i]->b_data + offset, avail);
95 buff += avail; 90 buff += avail;
@@ -104,24 +99,24 @@ static int lzo_uncompress(struct squashfs_sb_info *msblk, void **buffer,
104 goto failed; 99 goto failed;
105 100
106 res = bytes = (int)out_len; 101 res = bytes = (int)out_len;
107 for (i = 0, buff = stream->output; bytes && i < pages; i++) { 102 data = squashfs_first_page(output);
108 avail = min_t(int, bytes, PAGE_CACHE_SIZE); 103 buff = stream->output;
109 memcpy(buffer[i], buff, avail); 104 while (data) {
110 buff += avail; 105 if (bytes <= PAGE_CACHE_SIZE) {
111 bytes -= avail; 106 memcpy(data, buff, bytes);
107 break;
108 } else {
109 memcpy(data, buff, PAGE_CACHE_SIZE);
110 buff += PAGE_CACHE_SIZE;
111 bytes -= PAGE_CACHE_SIZE;
112 data = squashfs_next_page(output);
113 }
112 } 114 }
115 squashfs_finish_page(output);
113 116
114 mutex_unlock(&msblk->read_data_mutex);
115 return res; 117 return res;
116 118
117block_release:
118 for (; i < b; i++)
119 put_bh(bh[i]);
120
121failed: 119failed:
122 mutex_unlock(&msblk->read_data_mutex);
123
124 ERROR("lzo decompression failed, data probably corrupt\n");
125 return -EIO; 120 return -EIO;
126} 121}
127 122
diff --git a/fs/squashfs/page_actor.c b/fs/squashfs/page_actor.c
new file mode 100644
index 000000000000..5a1c11f56441
--- /dev/null
+++ b/fs/squashfs/page_actor.c
@@ -0,0 +1,100 @@
1/*
2 * Copyright (c) 2013
3 * Phillip Lougher <phillip@squashfs.org.uk>
4 *
5 * This work is licensed under the terms of the GNU GPL, version 2. See
6 * the COPYING file in the top-level directory.
7 */
8
9#include <linux/kernel.h>
10#include <linux/slab.h>
11#include <linux/pagemap.h>
12#include "page_actor.h"
13
14/*
15 * This file contains implementations of page_actor for decompressing into
16 * an intermediate buffer, and for decompressing directly into the
17 * page cache.
18 *
19 * Calling code should avoid sleeping between calls to squashfs_first_page()
20 * and squashfs_finish_page().
21 */
22
23/* Implementation of page_actor for decompressing into intermediate buffer */
24static void *cache_first_page(struct squashfs_page_actor *actor)
25{
26 actor->next_page = 1;
27 return actor->buffer[0];
28}
29
30static void *cache_next_page(struct squashfs_page_actor *actor)
31{
32 if (actor->next_page == actor->pages)
33 return NULL;
34
35 return actor->buffer[actor->next_page++];
36}
37
38static void cache_finish_page(struct squashfs_page_actor *actor)
39{
40 /* empty */
41}
42
43struct squashfs_page_actor *squashfs_page_actor_init(void **buffer,
44 int pages, int length)
45{
46 struct squashfs_page_actor *actor = kmalloc(sizeof(*actor), GFP_KERNEL);
47
48 if (actor == NULL)
49 return NULL;
50
51 actor->length = length ? : pages * PAGE_CACHE_SIZE;
52 actor->buffer = buffer;
53 actor->pages = pages;
54 actor->next_page = 0;
55 actor->squashfs_first_page = cache_first_page;
56 actor->squashfs_next_page = cache_next_page;
57 actor->squashfs_finish_page = cache_finish_page;
58 return actor;
59}
60
61/* Implementation of page_actor for decompressing directly into page cache. */
62static void *direct_first_page(struct squashfs_page_actor *actor)
63{
64 actor->next_page = 1;
65 return actor->pageaddr = kmap_atomic(actor->page[0]);
66}
67
68static void *direct_next_page(struct squashfs_page_actor *actor)
69{
70 if (actor->pageaddr)
71 kunmap_atomic(actor->pageaddr);
72
73 return actor->pageaddr = actor->next_page == actor->pages ? NULL :
74 kmap_atomic(actor->page[actor->next_page++]);
75}
76
77static void direct_finish_page(struct squashfs_page_actor *actor)
78{
79 if (actor->pageaddr)
80 kunmap_atomic(actor->pageaddr);
81}
82
83struct squashfs_page_actor *squashfs_page_actor_init_special(struct page **page,
84 int pages, int length)
85{
86 struct squashfs_page_actor *actor = kmalloc(sizeof(*actor), GFP_KERNEL);
87
88 if (actor == NULL)
89 return NULL;
90
91 actor->length = length ? : pages * PAGE_CACHE_SIZE;
92 actor->page = page;
93 actor->pages = pages;
94 actor->next_page = 0;
95 actor->pageaddr = NULL;
96 actor->squashfs_first_page = direct_first_page;
97 actor->squashfs_next_page = direct_next_page;
98 actor->squashfs_finish_page = direct_finish_page;
99 return actor;
100}
diff --git a/fs/squashfs/page_actor.h b/fs/squashfs/page_actor.h
new file mode 100644
index 000000000000..26dd82008b82
--- /dev/null
+++ b/fs/squashfs/page_actor.h
@@ -0,0 +1,81 @@
1#ifndef PAGE_ACTOR_H
2#define PAGE_ACTOR_H
3/*
4 * Copyright (c) 2013
5 * Phillip Lougher <phillip@squashfs.org.uk>
6 *
7 * This work is licensed under the terms of the GNU GPL, version 2. See
8 * the COPYING file in the top-level directory.
9 */
10
11#ifndef CONFIG_SQUASHFS_FILE_DIRECT
12struct squashfs_page_actor {
13 void **page;
14 int pages;
15 int length;
16 int next_page;
17};
18
19static inline struct squashfs_page_actor *squashfs_page_actor_init(void **page,
20 int pages, int length)
21{
22 struct squashfs_page_actor *actor = kmalloc(sizeof(*actor), GFP_KERNEL);
23
24 if (actor == NULL)
25 return NULL;
26
27 actor->length = length ? : pages * PAGE_CACHE_SIZE;
28 actor->page = page;
29 actor->pages = pages;
30 actor->next_page = 0;
31 return actor;
32}
33
34static inline void *squashfs_first_page(struct squashfs_page_actor *actor)
35{
36 actor->next_page = 1;
37 return actor->page[0];
38}
39
40static inline void *squashfs_next_page(struct squashfs_page_actor *actor)
41{
42 return actor->next_page == actor->pages ? NULL :
43 actor->page[actor->next_page++];
44}
45
46static inline void squashfs_finish_page(struct squashfs_page_actor *actor)
47{
48 /* empty */
49}
50#else
51struct squashfs_page_actor {
52 union {
53 void **buffer;
54 struct page **page;
55 };
56 void *pageaddr;
57 void *(*squashfs_first_page)(struct squashfs_page_actor *);
58 void *(*squashfs_next_page)(struct squashfs_page_actor *);
59 void (*squashfs_finish_page)(struct squashfs_page_actor *);
60 int pages;
61 int length;
62 int next_page;
63};
64
65extern struct squashfs_page_actor *squashfs_page_actor_init(void **, int, int);
66extern struct squashfs_page_actor *squashfs_page_actor_init_special(struct page
67 **, int, int);
68static inline void *squashfs_first_page(struct squashfs_page_actor *actor)
69{
70 return actor->squashfs_first_page(actor);
71}
72static inline void *squashfs_next_page(struct squashfs_page_actor *actor)
73{
74 return actor->squashfs_next_page(actor);
75}
76static inline void squashfs_finish_page(struct squashfs_page_actor *actor)
77{
78 actor->squashfs_finish_page(actor);
79}
80#endif
81#endif
diff --git a/fs/squashfs/squashfs.h b/fs/squashfs/squashfs.h
index d1266516ed08..9e1bb79f7e6f 100644
--- a/fs/squashfs/squashfs.h
+++ b/fs/squashfs/squashfs.h
@@ -28,8 +28,8 @@
28#define WARNING(s, args...) pr_warning("SQUASHFS: "s, ## args) 28#define WARNING(s, args...) pr_warning("SQUASHFS: "s, ## args)
29 29
30/* block.c */ 30/* block.c */
31extern int squashfs_read_data(struct super_block *, void **, u64, int, u64 *, 31extern int squashfs_read_data(struct super_block *, u64, int, u64 *,
32 int, int); 32 struct squashfs_page_actor *);
33 33
34/* cache.c */ 34/* cache.c */
35extern struct squashfs_cache *squashfs_cache_init(char *, int, int); 35extern struct squashfs_cache *squashfs_cache_init(char *, int, int);
@@ -48,7 +48,14 @@ extern void *squashfs_read_table(struct super_block *, u64, int);
48 48
49/* decompressor.c */ 49/* decompressor.c */
50extern const struct squashfs_decompressor *squashfs_lookup_decompressor(int); 50extern const struct squashfs_decompressor *squashfs_lookup_decompressor(int);
51extern void *squashfs_decompressor_init(struct super_block *, unsigned short); 51extern void *squashfs_decompressor_setup(struct super_block *, unsigned short);
52
53/* decompressor_xxx.c */
54extern void *squashfs_decompressor_create(struct squashfs_sb_info *, void *);
55extern void squashfs_decompressor_destroy(struct squashfs_sb_info *);
56extern int squashfs_decompress(struct squashfs_sb_info *, struct buffer_head **,
57 int, int, int, struct squashfs_page_actor *);
58extern int squashfs_max_decompressors(void);
52 59
53/* export.c */ 60/* export.c */
54extern __le64 *squashfs_read_inode_lookup_table(struct super_block *, u64, u64, 61extern __le64 *squashfs_read_inode_lookup_table(struct super_block *, u64, u64,
@@ -59,6 +66,13 @@ extern int squashfs_frag_lookup(struct super_block *, unsigned int, u64 *);
59extern __le64 *squashfs_read_fragment_index_table(struct super_block *, 66extern __le64 *squashfs_read_fragment_index_table(struct super_block *,
60 u64, u64, unsigned int); 67 u64, u64, unsigned int);
61 68
69/* file.c */
70void squashfs_copy_cache(struct page *, struct squashfs_cache_entry *, int,
71 int);
72
73/* file_xxx.c */
74extern int squashfs_readpage_block(struct page *, u64, int);
75
62/* id.c */ 76/* id.c */
63extern int squashfs_get_id(struct super_block *, unsigned int, unsigned int *); 77extern int squashfs_get_id(struct super_block *, unsigned int, unsigned int *);
64extern __le64 *squashfs_read_id_index_table(struct super_block *, u64, u64, 78extern __le64 *squashfs_read_id_index_table(struct super_block *, u64, u64,
diff --git a/fs/squashfs/squashfs_fs_sb.h b/fs/squashfs/squashfs_fs_sb.h
index 52934a22f296..1da565cb50c3 100644
--- a/fs/squashfs/squashfs_fs_sb.h
+++ b/fs/squashfs/squashfs_fs_sb.h
@@ -50,6 +50,7 @@ struct squashfs_cache_entry {
50 wait_queue_head_t wait_queue; 50 wait_queue_head_t wait_queue;
51 struct squashfs_cache *cache; 51 struct squashfs_cache *cache;
52 void **data; 52 void **data;
53 struct squashfs_page_actor *actor;
53}; 54};
54 55
55struct squashfs_sb_info { 56struct squashfs_sb_info {
@@ -63,10 +64,9 @@ struct squashfs_sb_info {
63 __le64 *id_table; 64 __le64 *id_table;
64 __le64 *fragment_index; 65 __le64 *fragment_index;
65 __le64 *xattr_id_table; 66 __le64 *xattr_id_table;
66 struct mutex read_data_mutex;
67 struct mutex meta_index_mutex; 67 struct mutex meta_index_mutex;
68 struct meta_index *meta_index; 68 struct meta_index *meta_index;
69 void *stream; 69 struct squashfs_stream *stream;
70 __le64 *inode_lookup_table; 70 __le64 *inode_lookup_table;
71 u64 inode_table; 71 u64 inode_table;
72 u64 directory_table; 72 u64 directory_table;
diff --git a/fs/squashfs/super.c b/fs/squashfs/super.c
index 60553a9053ca..202df6312d4e 100644
--- a/fs/squashfs/super.c
+++ b/fs/squashfs/super.c
@@ -98,7 +98,6 @@ static int squashfs_fill_super(struct super_block *sb, void *data, int silent)
98 msblk->devblksize = sb_min_blocksize(sb, SQUASHFS_DEVBLK_SIZE); 98 msblk->devblksize = sb_min_blocksize(sb, SQUASHFS_DEVBLK_SIZE);
99 msblk->devblksize_log2 = ffz(~msblk->devblksize); 99 msblk->devblksize_log2 = ffz(~msblk->devblksize);
100 100
101 mutex_init(&msblk->read_data_mutex);
102 mutex_init(&msblk->meta_index_mutex); 101 mutex_init(&msblk->meta_index_mutex);
103 102
104 /* 103 /*
@@ -206,13 +205,14 @@ static int squashfs_fill_super(struct super_block *sb, void *data, int silent)
206 goto failed_mount; 205 goto failed_mount;
207 206
208 /* Allocate read_page block */ 207 /* Allocate read_page block */
209 msblk->read_page = squashfs_cache_init("data", 1, msblk->block_size); 208 msblk->read_page = squashfs_cache_init("data",
209 squashfs_max_decompressors(), msblk->block_size);
210 if (msblk->read_page == NULL) { 210 if (msblk->read_page == NULL) {
211 ERROR("Failed to allocate read_page block\n"); 211 ERROR("Failed to allocate read_page block\n");
212 goto failed_mount; 212 goto failed_mount;
213 } 213 }
214 214
215 msblk->stream = squashfs_decompressor_init(sb, flags); 215 msblk->stream = squashfs_decompressor_setup(sb, flags);
216 if (IS_ERR(msblk->stream)) { 216 if (IS_ERR(msblk->stream)) {
217 err = PTR_ERR(msblk->stream); 217 err = PTR_ERR(msblk->stream);
218 msblk->stream = NULL; 218 msblk->stream = NULL;
@@ -336,7 +336,7 @@ failed_mount:
336 squashfs_cache_delete(msblk->block_cache); 336 squashfs_cache_delete(msblk->block_cache);
337 squashfs_cache_delete(msblk->fragment_cache); 337 squashfs_cache_delete(msblk->fragment_cache);
338 squashfs_cache_delete(msblk->read_page); 338 squashfs_cache_delete(msblk->read_page);
339 squashfs_decompressor_free(msblk, msblk->stream); 339 squashfs_decompressor_destroy(msblk);
340 kfree(msblk->inode_lookup_table); 340 kfree(msblk->inode_lookup_table);
341 kfree(msblk->fragment_index); 341 kfree(msblk->fragment_index);
342 kfree(msblk->id_table); 342 kfree(msblk->id_table);
@@ -383,7 +383,7 @@ static void squashfs_put_super(struct super_block *sb)
383 squashfs_cache_delete(sbi->block_cache); 383 squashfs_cache_delete(sbi->block_cache);
384 squashfs_cache_delete(sbi->fragment_cache); 384 squashfs_cache_delete(sbi->fragment_cache);
385 squashfs_cache_delete(sbi->read_page); 385 squashfs_cache_delete(sbi->read_page);
386 squashfs_decompressor_free(sbi, sbi->stream); 386 squashfs_decompressor_destroy(sbi);
387 kfree(sbi->id_table); 387 kfree(sbi->id_table);
388 kfree(sbi->fragment_index); 388 kfree(sbi->fragment_index);
389 kfree(sbi->meta_index); 389 kfree(sbi->meta_index);
diff --git a/fs/squashfs/xz_wrapper.c b/fs/squashfs/xz_wrapper.c
index 1760b7d108f6..c609624e4b8a 100644
--- a/fs/squashfs/xz_wrapper.c
+++ b/fs/squashfs/xz_wrapper.c
@@ -32,44 +32,70 @@
32#include "squashfs_fs_sb.h" 32#include "squashfs_fs_sb.h"
33#include "squashfs.h" 33#include "squashfs.h"
34#include "decompressor.h" 34#include "decompressor.h"
35#include "page_actor.h"
35 36
36struct squashfs_xz { 37struct squashfs_xz {
37 struct xz_dec *state; 38 struct xz_dec *state;
38 struct xz_buf buf; 39 struct xz_buf buf;
39}; 40};
40 41
41struct comp_opts { 42struct disk_comp_opts {
42 __le32 dictionary_size; 43 __le32 dictionary_size;
43 __le32 flags; 44 __le32 flags;
44}; 45};
45 46
46static void *squashfs_xz_init(struct squashfs_sb_info *msblk, void *buff, 47struct comp_opts {
47 int len) 48 int dict_size;
49};
50
51static void *squashfs_xz_comp_opts(struct squashfs_sb_info *msblk,
52 void *buff, int len)
48{ 53{
49 struct comp_opts *comp_opts = buff; 54 struct disk_comp_opts *comp_opts = buff;
50 struct squashfs_xz *stream; 55 struct comp_opts *opts;
51 int dict_size = msblk->block_size; 56 int err = 0, n;
52 int err, n; 57
58 opts = kmalloc(sizeof(*opts), GFP_KERNEL);
59 if (opts == NULL) {
60 err = -ENOMEM;
61 goto out2;
62 }
53 63
54 if (comp_opts) { 64 if (comp_opts) {
55 /* check compressor options are the expected length */ 65 /* check compressor options are the expected length */
56 if (len < sizeof(*comp_opts)) { 66 if (len < sizeof(*comp_opts)) {
57 err = -EIO; 67 err = -EIO;
58 goto failed; 68 goto out;
59 } 69 }
60 70
61 dict_size = le32_to_cpu(comp_opts->dictionary_size); 71 opts->dict_size = le32_to_cpu(comp_opts->dictionary_size);
62 72
63 /* the dictionary size should be 2^n or 2^n+2^(n+1) */ 73 /* the dictionary size should be 2^n or 2^n+2^(n+1) */
64 n = ffs(dict_size) - 1; 74 n = ffs(opts->dict_size) - 1;
65 if (dict_size != (1 << n) && dict_size != (1 << n) + 75 if (opts->dict_size != (1 << n) && opts->dict_size != (1 << n) +
66 (1 << (n + 1))) { 76 (1 << (n + 1))) {
67 err = -EIO; 77 err = -EIO;
68 goto failed; 78 goto out;
69 } 79 }
70 } 80 } else
81 /* use defaults */
82 opts->dict_size = max_t(int, msblk->block_size,
83 SQUASHFS_METADATA_SIZE);
84
85 return opts;
86
87out:
88 kfree(opts);
89out2:
90 return ERR_PTR(err);
91}
92
71 93
72 dict_size = max_t(int, dict_size, SQUASHFS_METADATA_SIZE); 94static void *squashfs_xz_init(struct squashfs_sb_info *msblk, void *buff)
95{
96 struct comp_opts *comp_opts = buff;
97 struct squashfs_xz *stream;
98 int err;
73 99
74 stream = kmalloc(sizeof(*stream), GFP_KERNEL); 100 stream = kmalloc(sizeof(*stream), GFP_KERNEL);
75 if (stream == NULL) { 101 if (stream == NULL) {
@@ -77,7 +103,7 @@ static void *squashfs_xz_init(struct squashfs_sb_info *msblk, void *buff,
77 goto failed; 103 goto failed;
78 } 104 }
79 105
80 stream->state = xz_dec_init(XZ_PREALLOC, dict_size); 106 stream->state = xz_dec_init(XZ_PREALLOC, comp_opts->dict_size);
81 if (stream->state == NULL) { 107 if (stream->state == NULL) {
82 kfree(stream); 108 kfree(stream);
83 err = -ENOMEM; 109 err = -ENOMEM;
@@ -103,42 +129,37 @@ static void squashfs_xz_free(void *strm)
103} 129}
104 130
105 131
106static int squashfs_xz_uncompress(struct squashfs_sb_info *msblk, void **buffer, 132static int squashfs_xz_uncompress(struct squashfs_sb_info *msblk, void *strm,
107 struct buffer_head **bh, int b, int offset, int length, int srclength, 133 struct buffer_head **bh, int b, int offset, int length,
108 int pages) 134 struct squashfs_page_actor *output)
109{ 135{
110 enum xz_ret xz_err; 136 enum xz_ret xz_err;
111 int avail, total = 0, k = 0, page = 0; 137 int avail, total = 0, k = 0;
112 struct squashfs_xz *stream = msblk->stream; 138 struct squashfs_xz *stream = strm;
113
114 mutex_lock(&msblk->read_data_mutex);
115 139
116 xz_dec_reset(stream->state); 140 xz_dec_reset(stream->state);
117 stream->buf.in_pos = 0; 141 stream->buf.in_pos = 0;
118 stream->buf.in_size = 0; 142 stream->buf.in_size = 0;
119 stream->buf.out_pos = 0; 143 stream->buf.out_pos = 0;
120 stream->buf.out_size = PAGE_CACHE_SIZE; 144 stream->buf.out_size = PAGE_CACHE_SIZE;
121 stream->buf.out = buffer[page++]; 145 stream->buf.out = squashfs_first_page(output);
122 146
123 do { 147 do {
124 if (stream->buf.in_pos == stream->buf.in_size && k < b) { 148 if (stream->buf.in_pos == stream->buf.in_size && k < b) {
125 avail = min(length, msblk->devblksize - offset); 149 avail = min(length, msblk->devblksize - offset);
126 length -= avail; 150 length -= avail;
127 wait_on_buffer(bh[k]);
128 if (!buffer_uptodate(bh[k]))
129 goto release_mutex;
130
131 stream->buf.in = bh[k]->b_data + offset; 151 stream->buf.in = bh[k]->b_data + offset;
132 stream->buf.in_size = avail; 152 stream->buf.in_size = avail;
133 stream->buf.in_pos = 0; 153 stream->buf.in_pos = 0;
134 offset = 0; 154 offset = 0;
135 } 155 }
136 156
137 if (stream->buf.out_pos == stream->buf.out_size 157 if (stream->buf.out_pos == stream->buf.out_size) {
138 && page < pages) { 158 stream->buf.out = squashfs_next_page(output);
139 stream->buf.out = buffer[page++]; 159 if (stream->buf.out != NULL) {
140 stream->buf.out_pos = 0; 160 stream->buf.out_pos = 0;
141 total += PAGE_CACHE_SIZE; 161 total += PAGE_CACHE_SIZE;
162 }
142 } 163 }
143 164
144 xz_err = xz_dec_run(stream->state, &stream->buf); 165 xz_err = xz_dec_run(stream->state, &stream->buf);
@@ -147,23 +168,14 @@ static int squashfs_xz_uncompress(struct squashfs_sb_info *msblk, void **buffer,
147 put_bh(bh[k++]); 168 put_bh(bh[k++]);
148 } while (xz_err == XZ_OK); 169 } while (xz_err == XZ_OK);
149 170
150 if (xz_err != XZ_STREAM_END) { 171 squashfs_finish_page(output);
151 ERROR("xz_dec_run error, data probably corrupt\n");
152 goto release_mutex;
153 }
154
155 if (k < b) {
156 ERROR("xz_uncompress error, input remaining\n");
157 goto release_mutex;
158 }
159 172
160 total += stream->buf.out_pos; 173 if (xz_err != XZ_STREAM_END || k < b)
161 mutex_unlock(&msblk->read_data_mutex); 174 goto out;
162 return total;
163 175
164release_mutex: 176 return total + stream->buf.out_pos;
165 mutex_unlock(&msblk->read_data_mutex);
166 177
178out:
167 for (; k < b; k++) 179 for (; k < b; k++)
168 put_bh(bh[k]); 180 put_bh(bh[k]);
169 181
@@ -172,6 +184,7 @@ release_mutex:
172 184
173const struct squashfs_decompressor squashfs_xz_comp_ops = { 185const struct squashfs_decompressor squashfs_xz_comp_ops = {
174 .init = squashfs_xz_init, 186 .init = squashfs_xz_init,
187 .comp_opts = squashfs_xz_comp_opts,
175 .free = squashfs_xz_free, 188 .free = squashfs_xz_free,
176 .decompress = squashfs_xz_uncompress, 189 .decompress = squashfs_xz_uncompress,
177 .id = XZ_COMPRESSION, 190 .id = XZ_COMPRESSION,
diff --git a/fs/squashfs/zlib_wrapper.c b/fs/squashfs/zlib_wrapper.c
index 55d918fd2d86..8727caba6882 100644
--- a/fs/squashfs/zlib_wrapper.c
+++ b/fs/squashfs/zlib_wrapper.c
@@ -32,8 +32,9 @@
32#include "squashfs_fs_sb.h" 32#include "squashfs_fs_sb.h"
33#include "squashfs.h" 33#include "squashfs.h"
34#include "decompressor.h" 34#include "decompressor.h"
35#include "page_actor.h"
35 36
36static void *zlib_init(struct squashfs_sb_info *dummy, void *buff, int len) 37static void *zlib_init(struct squashfs_sb_info *dummy, void *buff)
37{ 38{
38 z_stream *stream = kmalloc(sizeof(z_stream), GFP_KERNEL); 39 z_stream *stream = kmalloc(sizeof(z_stream), GFP_KERNEL);
39 if (stream == NULL) 40 if (stream == NULL)
@@ -61,44 +62,37 @@ static void zlib_free(void *strm)
61} 62}
62 63
63 64
64static int zlib_uncompress(struct squashfs_sb_info *msblk, void **buffer, 65static int zlib_uncompress(struct squashfs_sb_info *msblk, void *strm,
65 struct buffer_head **bh, int b, int offset, int length, int srclength, 66 struct buffer_head **bh, int b, int offset, int length,
66 int pages) 67 struct squashfs_page_actor *output)
67{ 68{
68 int zlib_err, zlib_init = 0; 69 int zlib_err, zlib_init = 0, k = 0;
69 int k = 0, page = 0; 70 z_stream *stream = strm;
70 z_stream *stream = msblk->stream;
71
72 mutex_lock(&msblk->read_data_mutex);
73 71
74 stream->avail_out = 0; 72 stream->avail_out = PAGE_CACHE_SIZE;
73 stream->next_out = squashfs_first_page(output);
75 stream->avail_in = 0; 74 stream->avail_in = 0;
76 75
77 do { 76 do {
78 if (stream->avail_in == 0 && k < b) { 77 if (stream->avail_in == 0 && k < b) {
79 int avail = min(length, msblk->devblksize - offset); 78 int avail = min(length, msblk->devblksize - offset);
80 length -= avail; 79 length -= avail;
81 wait_on_buffer(bh[k]);
82 if (!buffer_uptodate(bh[k]))
83 goto release_mutex;
84
85 stream->next_in = bh[k]->b_data + offset; 80 stream->next_in = bh[k]->b_data + offset;
86 stream->avail_in = avail; 81 stream->avail_in = avail;
87 offset = 0; 82 offset = 0;
88 } 83 }
89 84
90 if (stream->avail_out == 0 && page < pages) { 85 if (stream->avail_out == 0) {
91 stream->next_out = buffer[page++]; 86 stream->next_out = squashfs_next_page(output);
92 stream->avail_out = PAGE_CACHE_SIZE; 87 if (stream->next_out != NULL)
88 stream->avail_out = PAGE_CACHE_SIZE;
93 } 89 }
94 90
95 if (!zlib_init) { 91 if (!zlib_init) {
96 zlib_err = zlib_inflateInit(stream); 92 zlib_err = zlib_inflateInit(stream);
97 if (zlib_err != Z_OK) { 93 if (zlib_err != Z_OK) {
98 ERROR("zlib_inflateInit returned unexpected " 94 squashfs_finish_page(output);
99 "result 0x%x, srclength %d\n", 95 goto out;
100 zlib_err, srclength);
101 goto release_mutex;
102 } 96 }
103 zlib_init = 1; 97 zlib_init = 1;
104 } 98 }
@@ -109,29 +103,21 @@ static int zlib_uncompress(struct squashfs_sb_info *msblk, void **buffer,
109 put_bh(bh[k++]); 103 put_bh(bh[k++]);
110 } while (zlib_err == Z_OK); 104 } while (zlib_err == Z_OK);
111 105
112 if (zlib_err != Z_STREAM_END) { 106 squashfs_finish_page(output);
113 ERROR("zlib_inflate error, data probably corrupt\n");
114 goto release_mutex;
115 }
116 107
117 zlib_err = zlib_inflateEnd(stream); 108 if (zlib_err != Z_STREAM_END)
118 if (zlib_err != Z_OK) { 109 goto out;
119 ERROR("zlib_inflate error, data probably corrupt\n");
120 goto release_mutex;
121 }
122 110
123 if (k < b) { 111 zlib_err = zlib_inflateEnd(stream);
124 ERROR("zlib_uncompress error, data remaining\n"); 112 if (zlib_err != Z_OK)
125 goto release_mutex; 113 goto out;
126 }
127 114
128 length = stream->total_out; 115 if (k < b)
129 mutex_unlock(&msblk->read_data_mutex); 116 goto out;
130 return length;
131 117
132release_mutex: 118 return stream->total_out;
133 mutex_unlock(&msblk->read_data_mutex);
134 119
120out:
135 for (; k < b; k++) 121 for (; k < b; k++)
136 put_bh(bh[k]); 122 put_bh(bh[k]);
137 123
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-10 19:15:12 -0500