diff options
author | Theodore Ts'o <tytso@mit.edu> | 2015-04-08 00:00:32 -0400 |
---|---|---|
committer | Theodore Ts'o <tytso@mit.edu> | 2015-04-08 00:00:32 -0400 |
commit | f64e02fe9bc6a359cab95632b33900094d225ae1 (patch) | |
tree | bbaa7c514a5b6d7371760ac3ad367586f4e983f0 | |
parent | e12fb97222fc41e8442896934f76d39ef99b590a (diff) |
ext4 crypto: add ext4_mpage_readpages()
This takes code from fs/mpage.c and optimizes it for ext4. Its
primary reason is to allow us to more easily add encryption to ext4's
read path in an efficient manner.
Signed-off-by: Theodore Ts'o <tytso@mit.edu>
-rw-r--r-- | fs/ext4/Makefile | 2 | ||||
-rw-r--r-- | fs/ext4/ext4.h | 4 | ||||
-rw-r--r-- | fs/ext4/inode.c | 4 | ||||
-rw-r--r-- | fs/ext4/readpage.c | 264 |
4 files changed, 271 insertions, 3 deletions
diff --git a/fs/ext4/Makefile b/fs/ext4/Makefile index 0310fec2ee3d..cd6f50fce278 100644 --- a/fs/ext4/Makefile +++ b/fs/ext4/Makefile | |||
@@ -8,7 +8,7 @@ ext4-y := balloc.o bitmap.o dir.o file.o fsync.o ialloc.o inode.o page-io.o \ | |||
8 | ioctl.o namei.o super.o symlink.o hash.o resize.o extents.o \ | 8 | ioctl.o namei.o super.o symlink.o hash.o resize.o extents.o \ |
9 | ext4_jbd2.o migrate.o mballoc.o block_validity.o move_extent.o \ | 9 | ext4_jbd2.o migrate.o mballoc.o block_validity.o move_extent.o \ |
10 | mmp.o indirect.o extents_status.o xattr.o xattr_user.o \ | 10 | mmp.o indirect.o extents_status.o xattr.o xattr_user.o \ |
11 | xattr_trusted.o inline.o | 11 | xattr_trusted.o inline.o readpage.o |
12 | 12 | ||
13 | ext4-$(CONFIG_EXT4_FS_POSIX_ACL) += acl.o | 13 | ext4-$(CONFIG_EXT4_FS_POSIX_ACL) += acl.o |
14 | ext4-$(CONFIG_EXT4_FS_SECURITY) += xattr_security.o | 14 | ext4-$(CONFIG_EXT4_FS_SECURITY) += xattr_security.o |
diff --git a/fs/ext4/ext4.h b/fs/ext4/ext4.h index f63c3d5805c4..f7f3f5871796 100644 --- a/fs/ext4/ext4.h +++ b/fs/ext4/ext4.h | |||
@@ -2699,6 +2699,10 @@ static inline void ext4_set_de_type(struct super_block *sb, | |||
2699 | de->file_type = ext4_type_by_mode[(mode & S_IFMT)>>S_SHIFT]; | 2699 | de->file_type = ext4_type_by_mode[(mode & S_IFMT)>>S_SHIFT]; |
2700 | } | 2700 | } |
2701 | 2701 | ||
2702 | /* readpages.c */ | ||
2703 | extern int ext4_mpage_readpages(struct address_space *mapping, | ||
2704 | struct list_head *pages, struct page *page, | ||
2705 | unsigned nr_pages); | ||
2702 | 2706 | ||
2703 | /* symlink.c */ | 2707 | /* symlink.c */ |
2704 | extern const struct inode_operations ext4_symlink_inode_operations; | 2708 | extern const struct inode_operations ext4_symlink_inode_operations; |
diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c index ff0c111e52eb..cd3009152ae2 100644 --- a/fs/ext4/inode.c +++ b/fs/ext4/inode.c | |||
@@ -2820,7 +2820,7 @@ static int ext4_readpage(struct file *file, struct page *page) | |||
2820 | ret = ext4_readpage_inline(inode, page); | 2820 | ret = ext4_readpage_inline(inode, page); |
2821 | 2821 | ||
2822 | if (ret == -EAGAIN) | 2822 | if (ret == -EAGAIN) |
2823 | return mpage_readpage(page, ext4_get_block); | 2823 | return ext4_mpage_readpages(page->mapping, NULL, page, 1); |
2824 | 2824 | ||
2825 | return ret; | 2825 | return ret; |
2826 | } | 2826 | } |
@@ -2835,7 +2835,7 @@ ext4_readpages(struct file *file, struct address_space *mapping, | |||
2835 | if (ext4_has_inline_data(inode)) | 2835 | if (ext4_has_inline_data(inode)) |
2836 | return 0; | 2836 | return 0; |
2837 | 2837 | ||
2838 | return mpage_readpages(mapping, pages, nr_pages, ext4_get_block); | 2838 | return ext4_mpage_readpages(mapping, pages, NULL, nr_pages); |
2839 | } | 2839 | } |
2840 | 2840 | ||
2841 | static void ext4_invalidatepage(struct page *page, unsigned int offset, | 2841 | static void ext4_invalidatepage(struct page *page, unsigned int offset, |
diff --git a/fs/ext4/readpage.c b/fs/ext4/readpage.c new file mode 100644 index 000000000000..fff9fe6aacf8 --- /dev/null +++ b/fs/ext4/readpage.c | |||
@@ -0,0 +1,264 @@ | |||
1 | /* | ||
2 | * linux/fs/ext4/readpage.c | ||
3 | * | ||
4 | * Copyright (C) 2002, Linus Torvalds. | ||
5 | * Copyright (C) 2015, Google, Inc. | ||
6 | * | ||
7 | * This was originally taken from fs/mpage.c | ||
8 | * | ||
9 | * The intent is the ext4_mpage_readpages() function here is intended | ||
10 | * to replace mpage_readpages() in the general case, not just for | ||
11 | * encrypted files. It has some limitations (see below), where it | ||
12 | * will fall back to read_block_full_page(), but these limitations | ||
13 | * should only be hit when page_size != block_size. | ||
14 | * | ||
15 | * This will allow us to attach a callback function to support ext4 | ||
16 | * encryption. | ||
17 | * | ||
18 | * If anything unusual happens, such as: | ||
19 | * | ||
20 | * - encountering a page which has buffers | ||
21 | * - encountering a page which has a non-hole after a hole | ||
22 | * - encountering a page with non-contiguous blocks | ||
23 | * | ||
24 | * then this code just gives up and calls the buffer_head-based read function. | ||
25 | * It does handle a page which has holes at the end - that is a common case: | ||
26 | * the end-of-file on blocksize < PAGE_CACHE_SIZE setups. | ||
27 | * | ||
28 | */ | ||
29 | |||
30 | #include <linux/kernel.h> | ||
31 | #include <linux/export.h> | ||
32 | #include <linux/mm.h> | ||
33 | #include <linux/kdev_t.h> | ||
34 | #include <linux/gfp.h> | ||
35 | #include <linux/bio.h> | ||
36 | #include <linux/fs.h> | ||
37 | #include <linux/buffer_head.h> | ||
38 | #include <linux/blkdev.h> | ||
39 | #include <linux/highmem.h> | ||
40 | #include <linux/prefetch.h> | ||
41 | #include <linux/mpage.h> | ||
42 | #include <linux/writeback.h> | ||
43 | #include <linux/backing-dev.h> | ||
44 | #include <linux/pagevec.h> | ||
45 | #include <linux/cleancache.h> | ||
46 | |||
47 | #include "ext4.h" | ||
48 | |||
49 | /* | ||
50 | * I/O completion handler for multipage BIOs. | ||
51 | * | ||
52 | * The mpage code never puts partial pages into a BIO (except for end-of-file). | ||
53 | * If a page does not map to a contiguous run of blocks then it simply falls | ||
54 | * back to block_read_full_page(). | ||
55 | * | ||
56 | * Why is this? If a page's completion depends on a number of different BIOs | ||
57 | * which can complete in any order (or at the same time) then determining the | ||
58 | * status of that page is hard. See end_buffer_async_read() for the details. | ||
59 | * There is no point in duplicating all that complexity. | ||
60 | */ | ||
61 | static void mpage_end_io(struct bio *bio, int err) | ||
62 | { | ||
63 | struct bio_vec *bv; | ||
64 | int i; | ||
65 | |||
66 | bio_for_each_segment_all(bv, bio, i) { | ||
67 | struct page *page = bv->bv_page; | ||
68 | |||
69 | if (!err) { | ||
70 | SetPageUptodate(page); | ||
71 | } else { | ||
72 | ClearPageUptodate(page); | ||
73 | SetPageError(page); | ||
74 | } | ||
75 | unlock_page(page); | ||
76 | } | ||
77 | |||
78 | bio_put(bio); | ||
79 | } | ||
80 | |||
81 | int ext4_mpage_readpages(struct address_space *mapping, | ||
82 | struct list_head *pages, struct page *page, | ||
83 | unsigned nr_pages) | ||
84 | { | ||
85 | struct bio *bio = NULL; | ||
86 | unsigned page_idx; | ||
87 | sector_t last_block_in_bio = 0; | ||
88 | |||
89 | struct inode *inode = mapping->host; | ||
90 | const unsigned blkbits = inode->i_blkbits; | ||
91 | const unsigned blocks_per_page = PAGE_CACHE_SIZE >> blkbits; | ||
92 | const unsigned blocksize = 1 << blkbits; | ||
93 | sector_t block_in_file; | ||
94 | sector_t last_block; | ||
95 | sector_t last_block_in_file; | ||
96 | sector_t blocks[MAX_BUF_PER_PAGE]; | ||
97 | unsigned page_block; | ||
98 | struct block_device *bdev = inode->i_sb->s_bdev; | ||
99 | int length; | ||
100 | unsigned relative_block = 0; | ||
101 | struct ext4_map_blocks map; | ||
102 | |||
103 | map.m_pblk = 0; | ||
104 | map.m_lblk = 0; | ||
105 | map.m_len = 0; | ||
106 | map.m_flags = 0; | ||
107 | |||
108 | for (page_idx = 0; nr_pages; page_idx++, nr_pages--) { | ||
109 | int fully_mapped = 1; | ||
110 | unsigned first_hole = blocks_per_page; | ||
111 | |||
112 | prefetchw(&page->flags); | ||
113 | if (pages) { | ||
114 | page = list_entry(pages->prev, struct page, lru); | ||
115 | list_del(&page->lru); | ||
116 | if (add_to_page_cache_lru(page, mapping, | ||
117 | page->index, GFP_KERNEL)) | ||
118 | goto next_page; | ||
119 | } | ||
120 | |||
121 | if (page_has_buffers(page)) | ||
122 | goto confused; | ||
123 | |||
124 | block_in_file = (sector_t)page->index << (PAGE_CACHE_SHIFT - blkbits); | ||
125 | last_block = block_in_file + nr_pages * blocks_per_page; | ||
126 | last_block_in_file = (i_size_read(inode) + blocksize - 1) >> blkbits; | ||
127 | if (last_block > last_block_in_file) | ||
128 | last_block = last_block_in_file; | ||
129 | page_block = 0; | ||
130 | |||
131 | /* | ||
132 | * Map blocks using the previous result first. | ||
133 | */ | ||
134 | if ((map.m_flags & EXT4_MAP_MAPPED) && | ||
135 | block_in_file > map.m_lblk && | ||
136 | block_in_file < (map.m_lblk + map.m_len)) { | ||
137 | unsigned map_offset = block_in_file - map.m_lblk; | ||
138 | unsigned last = map.m_len - map_offset; | ||
139 | |||
140 | for (relative_block = 0; ; relative_block++) { | ||
141 | if (relative_block == last) { | ||
142 | /* needed? */ | ||
143 | map.m_flags &= ~EXT4_MAP_MAPPED; | ||
144 | break; | ||
145 | } | ||
146 | if (page_block == blocks_per_page) | ||
147 | break; | ||
148 | blocks[page_block] = map.m_pblk + map_offset + | ||
149 | relative_block; | ||
150 | page_block++; | ||
151 | block_in_file++; | ||
152 | } | ||
153 | } | ||
154 | |||
155 | /* | ||
156 | * Then do more ext4_map_blocks() calls until we are | ||
157 | * done with this page. | ||
158 | */ | ||
159 | while (page_block < blocks_per_page) { | ||
160 | if (block_in_file < last_block) { | ||
161 | map.m_lblk = block_in_file; | ||
162 | map.m_len = last_block - block_in_file; | ||
163 | |||
164 | if (ext4_map_blocks(NULL, inode, &map, 0) < 0) { | ||
165 | set_error_page: | ||
166 | SetPageError(page); | ||
167 | zero_user_segment(page, 0, | ||
168 | PAGE_CACHE_SIZE); | ||
169 | unlock_page(page); | ||
170 | goto next_page; | ||
171 | } | ||
172 | } | ||
173 | if ((map.m_flags & EXT4_MAP_MAPPED) == 0) { | ||
174 | fully_mapped = 0; | ||
175 | if (first_hole == blocks_per_page) | ||
176 | first_hole = page_block; | ||
177 | page_block++; | ||
178 | block_in_file++; | ||
179 | continue; | ||
180 | } | ||
181 | if (first_hole != blocks_per_page) | ||
182 | goto confused; /* hole -> non-hole */ | ||
183 | |||
184 | /* Contiguous blocks? */ | ||
185 | if (page_block && blocks[page_block-1] != map.m_pblk-1) | ||
186 | goto confused; | ||
187 | for (relative_block = 0; ; relative_block++) { | ||
188 | if (relative_block == map.m_len) { | ||
189 | /* needed? */ | ||
190 | map.m_flags &= ~EXT4_MAP_MAPPED; | ||
191 | break; | ||
192 | } else if (page_block == blocks_per_page) | ||
193 | break; | ||
194 | blocks[page_block] = map.m_pblk+relative_block; | ||
195 | page_block++; | ||
196 | block_in_file++; | ||
197 | } | ||
198 | } | ||
199 | if (first_hole != blocks_per_page) { | ||
200 | zero_user_segment(page, first_hole << blkbits, | ||
201 | PAGE_CACHE_SIZE); | ||
202 | if (first_hole == 0) { | ||
203 | SetPageUptodate(page); | ||
204 | unlock_page(page); | ||
205 | goto next_page; | ||
206 | } | ||
207 | } else if (fully_mapped) { | ||
208 | SetPageMappedToDisk(page); | ||
209 | } | ||
210 | if (fully_mapped && blocks_per_page == 1 && | ||
211 | !PageUptodate(page) && cleancache_get_page(page) == 0) { | ||
212 | SetPageUptodate(page); | ||
213 | goto confused; | ||
214 | } | ||
215 | |||
216 | /* | ||
217 | * This page will go to BIO. Do we need to send this | ||
218 | * BIO off first? | ||
219 | */ | ||
220 | if (bio && (last_block_in_bio != blocks[0] - 1)) { | ||
221 | submit_and_realloc: | ||
222 | submit_bio(READ, bio); | ||
223 | bio = NULL; | ||
224 | } | ||
225 | if (bio == NULL) { | ||
226 | bio = bio_alloc(GFP_KERNEL, | ||
227 | min_t(int, nr_pages, bio_get_nr_vecs(bdev))); | ||
228 | if (!bio) | ||
229 | goto set_error_page; | ||
230 | bio->bi_bdev = bdev; | ||
231 | bio->bi_iter.bi_sector = blocks[0] << (blkbits - 9); | ||
232 | bio->bi_end_io = mpage_end_io; | ||
233 | } | ||
234 | |||
235 | length = first_hole << blkbits; | ||
236 | if (bio_add_page(bio, page, length, 0) < length) | ||
237 | goto submit_and_realloc; | ||
238 | |||
239 | if (((map.m_flags & EXT4_MAP_BOUNDARY) && | ||
240 | (relative_block == map.m_len)) || | ||
241 | (first_hole != blocks_per_page)) { | ||
242 | submit_bio(READ, bio); | ||
243 | bio = NULL; | ||
244 | } else | ||
245 | last_block_in_bio = blocks[blocks_per_page - 1]; | ||
246 | goto next_page; | ||
247 | confused: | ||
248 | if (bio) { | ||
249 | submit_bio(READ, bio); | ||
250 | bio = NULL; | ||
251 | } | ||
252 | if (!PageUptodate(page)) | ||
253 | block_read_full_page(page, ext4_get_block); | ||
254 | else | ||
255 | unlock_page(page); | ||
256 | next_page: | ||
257 | if (pages) | ||
258 | page_cache_release(page); | ||
259 | } | ||
260 | BUG_ON(pages && !list_empty(pages)); | ||
261 | if (bio) | ||
262 | submit_bio(READ, bio); | ||
263 | return 0; | ||
264 | } | ||