aboutsummaryrefslogtreecommitdiffstats
path: root/fs
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2014-12-10 18:41:28 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2014-12-10 18:41:28 -0500
commit4b0a268eeccae14d42ff5fb9f19b612913c0007c (patch)
treef8a9e9345b50b437b205421d8c7898c7ccaafdf4 /fs
parenta6b849578ef3e0b131b1ea4063473a4f935a65e9 (diff)
parent635aee1fefef921ae4124b127fced62ea6008839 (diff)
Merge tag 'for-f2fs-3.19' of git://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs
Pull f2fs updates from Jaegeuk Kim: "This patch-set includes lots of bug fixes based on clean-ups and refactored codes. And inline_dir was introduced and two minor mount options were added. Details from signed tag: This series includes the following enhancement with refactored flows. - fix inmemory page operations - fix wrong inline_data & inline_dir logics - enhance memory and IO control under memory pressure - consider preemption on radix_tree operation - fix memory leaks and deadlocks But also, there are a couple of new features: - support inline_dir to store dentries inside inode page - add -o fastboot to reduce booting time - implement -o dirsync And a lot of clean-ups and minor bug fixes as well" * tag 'for-f2fs-3.19' of git://git.kernel.org/pub/scm/linux/kernel/git/jaegeuk/f2fs: (88 commits) f2fs: avoid to ra unneeded blocks in recover flow f2fs: introduce is_valid_blkaddr to cleanup codes in ra_meta_pages f2fs: fix to enable readahead for SSA/CP blocks f2fs: use atomic for counting inode with inline_{dir,inode} flag f2fs: cleanup path to need cp at fsync f2fs: check if inode state is dirty at fsync f2fs: count the number of inmemory pages f2fs: release inmemory pages when the file was closed f2fs: set page private for inmemory pages for truncation f2fs: count inline_xx in do_read_inode f2fs: do retry operations with cond_resched f2fs: call radix_tree_preload before radix_tree_insert f2fs: use rw_semaphore for nat entry lock f2fs: fix missing kmem_cache_free f2fs: more fast lookup for gc_inode list f2fs: cleanup redundant macro f2fs: fix to return correct error number in f2fs_write_begin f2fs: cleanup if-statement of phase in gc_data_segment f2fs: fix to recover converted inline_data f2fs: make clean the page before writing ...
Diffstat (limited to 'fs')
-rw-r--r--fs/f2fs/acl.c148
-rw-r--r--fs/f2fs/acl.h5
-rw-r--r--fs/f2fs/checkpoint.c186
-rw-r--r--fs/f2fs/data.c166
-rw-r--r--fs/f2fs/debug.c15
-rw-r--r--fs/f2fs/dir.c308
-rw-r--r--fs/f2fs/f2fs.h176
-rw-r--r--fs/f2fs/file.c212
-rw-r--r--fs/f2fs/gc.c89
-rw-r--r--fs/f2fs/gc.h5
-rw-r--r--fs/f2fs/inline.c482
-rw-r--r--fs/f2fs/inode.c44
-rw-r--r--fs/f2fs/namei.c58
-rw-r--r--fs/f2fs/node.c163
-rw-r--r--fs/f2fs/node.h8
-rw-r--r--fs/f2fs/recovery.c14
-rw-r--r--fs/f2fs/segment.c122
-rw-r--r--fs/f2fs/segment.h8
-rw-r--r--fs/f2fs/super.c29
-rw-r--r--fs/f2fs/xattr.c6
-rw-r--r--fs/f2fs/xattr.h6
21 files changed, 1549 insertions, 701 deletions
diff --git a/fs/f2fs/acl.c b/fs/f2fs/acl.c
index 83b9b5a8d112..1ccb26bc2a0b 100644
--- a/fs/f2fs/acl.c
+++ b/fs/f2fs/acl.c
@@ -162,7 +162,8 @@ fail:
162 return ERR_PTR(-EINVAL); 162 return ERR_PTR(-EINVAL);
163} 163}
164 164
165struct posix_acl *f2fs_get_acl(struct inode *inode, int type) 165static struct posix_acl *__f2fs_get_acl(struct inode *inode, int type,
166 struct page *dpage)
166{ 167{
167 int name_index = F2FS_XATTR_INDEX_POSIX_ACL_DEFAULT; 168 int name_index = F2FS_XATTR_INDEX_POSIX_ACL_DEFAULT;
168 void *value = NULL; 169 void *value = NULL;
@@ -172,12 +173,13 @@ struct posix_acl *f2fs_get_acl(struct inode *inode, int type)
172 if (type == ACL_TYPE_ACCESS) 173 if (type == ACL_TYPE_ACCESS)
173 name_index = F2FS_XATTR_INDEX_POSIX_ACL_ACCESS; 174 name_index = F2FS_XATTR_INDEX_POSIX_ACL_ACCESS;
174 175
175 retval = f2fs_getxattr(inode, name_index, "", NULL, 0); 176 retval = f2fs_getxattr(inode, name_index, "", NULL, 0, dpage);
176 if (retval > 0) { 177 if (retval > 0) {
177 value = kmalloc(retval, GFP_F2FS_ZERO); 178 value = kmalloc(retval, GFP_F2FS_ZERO);
178 if (!value) 179 if (!value)
179 return ERR_PTR(-ENOMEM); 180 return ERR_PTR(-ENOMEM);
180 retval = f2fs_getxattr(inode, name_index, "", value, retval); 181 retval = f2fs_getxattr(inode, name_index, "", value,
182 retval, dpage);
181 } 183 }
182 184
183 if (retval > 0) 185 if (retval > 0)
@@ -194,6 +196,11 @@ struct posix_acl *f2fs_get_acl(struct inode *inode, int type)
194 return acl; 196 return acl;
195} 197}
196 198
199struct posix_acl *f2fs_get_acl(struct inode *inode, int type)
200{
201 return __f2fs_get_acl(inode, type, NULL);
202}
203
197static int __f2fs_set_acl(struct inode *inode, int type, 204static int __f2fs_set_acl(struct inode *inode, int type,
198 struct posix_acl *acl, struct page *ipage) 205 struct posix_acl *acl, struct page *ipage)
199{ 206{
@@ -229,7 +236,7 @@ static int __f2fs_set_acl(struct inode *inode, int type,
229 if (acl) { 236 if (acl) {
230 value = f2fs_acl_to_disk(acl, &size); 237 value = f2fs_acl_to_disk(acl, &size);
231 if (IS_ERR(value)) { 238 if (IS_ERR(value)) {
232 cond_clear_inode_flag(fi, FI_ACL_MODE); 239 clear_inode_flag(fi, FI_ACL_MODE);
233 return (int)PTR_ERR(value); 240 return (int)PTR_ERR(value);
234 } 241 }
235 } 242 }
@@ -240,7 +247,7 @@ static int __f2fs_set_acl(struct inode *inode, int type,
240 if (!error) 247 if (!error)
241 set_cached_acl(inode, type, acl); 248 set_cached_acl(inode, type, acl);
242 249
243 cond_clear_inode_flag(fi, FI_ACL_MODE); 250 clear_inode_flag(fi, FI_ACL_MODE);
244 return error; 251 return error;
245} 252}
246 253
@@ -249,12 +256,137 @@ int f2fs_set_acl(struct inode *inode, struct posix_acl *acl, int type)
249 return __f2fs_set_acl(inode, type, acl, NULL); 256 return __f2fs_set_acl(inode, type, acl, NULL);
250} 257}
251 258
252int f2fs_init_acl(struct inode *inode, struct inode *dir, struct page *ipage) 259/*
260 * Most part of f2fs_acl_clone, f2fs_acl_create_masq, f2fs_acl_create
261 * are copied from posix_acl.c
262 */
263static struct posix_acl *f2fs_acl_clone(const struct posix_acl *acl,
264 gfp_t flags)
265{
266 struct posix_acl *clone = NULL;
267
268 if (acl) {
269 int size = sizeof(struct posix_acl) + acl->a_count *
270 sizeof(struct posix_acl_entry);
271 clone = kmemdup(acl, size, flags);
272 if (clone)
273 atomic_set(&clone->a_refcount, 1);
274 }
275 return clone;
276}
277
278static int f2fs_acl_create_masq(struct posix_acl *acl, umode_t *mode_p)
279{
280 struct posix_acl_entry *pa, *pe;
281 struct posix_acl_entry *group_obj = NULL, *mask_obj = NULL;
282 umode_t mode = *mode_p;
283 int not_equiv = 0;
284
285 /* assert(atomic_read(acl->a_refcount) == 1); */
286
287 FOREACH_ACL_ENTRY(pa, acl, pe) {
288 switch(pa->e_tag) {
289 case ACL_USER_OBJ:
290 pa->e_perm &= (mode >> 6) | ~S_IRWXO;
291 mode &= (pa->e_perm << 6) | ~S_IRWXU;
292 break;
293
294 case ACL_USER:
295 case ACL_GROUP:
296 not_equiv = 1;
297 break;
298
299 case ACL_GROUP_OBJ:
300 group_obj = pa;
301 break;
302
303 case ACL_OTHER:
304 pa->e_perm &= mode | ~S_IRWXO;
305 mode &= pa->e_perm | ~S_IRWXO;
306 break;
307
308 case ACL_MASK:
309 mask_obj = pa;
310 not_equiv = 1;
311 break;
312
313 default:
314 return -EIO;
315 }
316 }
317
318 if (mask_obj) {
319 mask_obj->e_perm &= (mode >> 3) | ~S_IRWXO;
320 mode &= (mask_obj->e_perm << 3) | ~S_IRWXG;
321 } else {
322 if (!group_obj)
323 return -EIO;
324 group_obj->e_perm &= (mode >> 3) | ~S_IRWXO;
325 mode &= (group_obj->e_perm << 3) | ~S_IRWXG;
326 }
327
328 *mode_p = (*mode_p & ~S_IRWXUGO) | mode;
329 return not_equiv;
330}
331
332static int f2fs_acl_create(struct inode *dir, umode_t *mode,
333 struct posix_acl **default_acl, struct posix_acl **acl,
334 struct page *dpage)
335{
336 struct posix_acl *p;
337 int ret;
338
339 if (S_ISLNK(*mode) || !IS_POSIXACL(dir))
340 goto no_acl;
341
342 p = __f2fs_get_acl(dir, ACL_TYPE_DEFAULT, dpage);
343 if (IS_ERR(p)) {
344 if (p == ERR_PTR(-EOPNOTSUPP))
345 goto apply_umask;
346 return PTR_ERR(p);
347 }
348
349 if (!p)
350 goto apply_umask;
351
352 *acl = f2fs_acl_clone(p, GFP_NOFS);
353 if (!*acl)
354 return -ENOMEM;
355
356 ret = f2fs_acl_create_masq(*acl, mode);
357 if (ret < 0) {
358 posix_acl_release(*acl);
359 return -ENOMEM;
360 }
361
362 if (ret == 0) {
363 posix_acl_release(*acl);
364 *acl = NULL;
365 }
366
367 if (!S_ISDIR(*mode)) {
368 posix_acl_release(p);
369 *default_acl = NULL;
370 } else {
371 *default_acl = p;
372 }
373 return 0;
374
375apply_umask:
376 *mode &= ~current_umask();
377no_acl:
378 *default_acl = NULL;
379 *acl = NULL;
380 return 0;
381}
382
383int f2fs_init_acl(struct inode *inode, struct inode *dir, struct page *ipage,
384 struct page *dpage)
253{ 385{
254 struct posix_acl *default_acl, *acl; 386 struct posix_acl *default_acl = NULL, *acl = NULL;
255 int error = 0; 387 int error = 0;
256 388
257 error = posix_acl_create(dir, &inode->i_mode, &default_acl, &acl); 389 error = f2fs_acl_create(dir, &inode->i_mode, &default_acl, &acl, dpage);
258 if (error) 390 if (error)
259 return error; 391 return error;
260 392
diff --git a/fs/f2fs/acl.h b/fs/f2fs/acl.h
index e0864651cdc1..997ca8edb6cb 100644
--- a/fs/f2fs/acl.h
+++ b/fs/f2fs/acl.h
@@ -38,14 +38,15 @@ struct f2fs_acl_header {
38 38
39extern struct posix_acl *f2fs_get_acl(struct inode *, int); 39extern struct posix_acl *f2fs_get_acl(struct inode *, int);
40extern int f2fs_set_acl(struct inode *inode, struct posix_acl *acl, int type); 40extern int f2fs_set_acl(struct inode *inode, struct posix_acl *acl, int type);
41extern int f2fs_init_acl(struct inode *, struct inode *, struct page *); 41extern int f2fs_init_acl(struct inode *, struct inode *, struct page *,
42 struct page *);
42#else 43#else
43#define f2fs_check_acl NULL 44#define f2fs_check_acl NULL
44#define f2fs_get_acl NULL 45#define f2fs_get_acl NULL
45#define f2fs_set_acl NULL 46#define f2fs_set_acl NULL
46 47
47static inline int f2fs_init_acl(struct inode *inode, struct inode *dir, 48static inline int f2fs_init_acl(struct inode *inode, struct inode *dir,
48 struct page *page) 49 struct page *ipage, struct page *dpage)
49{ 50{
50 return 0; 51 return 0;
51} 52}
diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c
index dd10a031c052..e6c271fefaca 100644
--- a/fs/f2fs/checkpoint.c
+++ b/fs/f2fs/checkpoint.c
@@ -72,36 +72,36 @@ out:
72 return page; 72 return page;
73} 73}
74 74
75struct page *get_meta_page_ra(struct f2fs_sb_info *sbi, pgoff_t index) 75static inline bool is_valid_blkaddr(struct f2fs_sb_info *sbi,
76{ 76 block_t blkaddr, int type)
77 bool readahead = false;
78 struct page *page;
79
80 page = find_get_page(META_MAPPING(sbi), index);
81 if (!page || (page && !PageUptodate(page)))
82 readahead = true;
83 f2fs_put_page(page, 0);
84
85 if (readahead)
86 ra_meta_pages(sbi, index, MAX_BIO_BLOCKS(sbi), META_POR);
87 return get_meta_page(sbi, index);
88}
89
90static inline block_t get_max_meta_blks(struct f2fs_sb_info *sbi, int type)
91{ 77{
92 switch (type) { 78 switch (type) {
93 case META_NAT: 79 case META_NAT:
94 return NM_I(sbi)->max_nid / NAT_ENTRY_PER_BLOCK; 80 break;
95 case META_SIT: 81 case META_SIT:
96 return SIT_BLK_CNT(sbi); 82 if (unlikely(blkaddr >= SIT_BLK_CNT(sbi)))
83 return false;
84 break;
97 case META_SSA: 85 case META_SSA:
86 if (unlikely(blkaddr >= MAIN_BLKADDR(sbi) ||
87 blkaddr < SM_I(sbi)->ssa_blkaddr))
88 return false;
89 break;
98 case META_CP: 90 case META_CP:
99 return 0; 91 if (unlikely(blkaddr >= SIT_I(sbi)->sit_base_addr ||
92 blkaddr < __start_cp_addr(sbi)))
93 return false;
94 break;
100 case META_POR: 95 case META_POR:
101 return MAX_BLKADDR(sbi); 96 if (unlikely(blkaddr >= MAX_BLKADDR(sbi) ||
97 blkaddr < MAIN_BLKADDR(sbi)))
98 return false;
99 break;
102 default: 100 default:
103 BUG(); 101 BUG();
104 } 102 }
103
104 return true;
105} 105}
106 106
107/* 107/*
@@ -112,7 +112,6 @@ int ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages, int type
112 block_t prev_blk_addr = 0; 112 block_t prev_blk_addr = 0;
113 struct page *page; 113 struct page *page;
114 block_t blkno = start; 114 block_t blkno = start;
115 block_t max_blks = get_max_meta_blks(sbi, type);
116 115
117 struct f2fs_io_info fio = { 116 struct f2fs_io_info fio = {
118 .type = META, 117 .type = META,
@@ -122,18 +121,20 @@ int ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages, int type
122 for (; nrpages-- > 0; blkno++) { 121 for (; nrpages-- > 0; blkno++) {
123 block_t blk_addr; 122 block_t blk_addr;
124 123
124 if (!is_valid_blkaddr(sbi, blkno, type))
125 goto out;
126
125 switch (type) { 127 switch (type) {
126 case META_NAT: 128 case META_NAT:
127 /* get nat block addr */ 129 if (unlikely(blkno >=
128 if (unlikely(blkno >= max_blks)) 130 NAT_BLOCK_OFFSET(NM_I(sbi)->max_nid)))
129 blkno = 0; 131 blkno = 0;
132 /* get nat block addr */
130 blk_addr = current_nat_addr(sbi, 133 blk_addr = current_nat_addr(sbi,
131 blkno * NAT_ENTRY_PER_BLOCK); 134 blkno * NAT_ENTRY_PER_BLOCK);
132 break; 135 break;
133 case META_SIT: 136 case META_SIT:
134 /* get sit block addr */ 137 /* get sit block addr */
135 if (unlikely(blkno >= max_blks))
136 goto out;
137 blk_addr = current_sit_addr(sbi, 138 blk_addr = current_sit_addr(sbi,
138 blkno * SIT_ENTRY_PER_BLOCK); 139 blkno * SIT_ENTRY_PER_BLOCK);
139 if (blkno != start && prev_blk_addr + 1 != blk_addr) 140 if (blkno != start && prev_blk_addr + 1 != blk_addr)
@@ -143,10 +144,6 @@ int ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages, int type
143 case META_SSA: 144 case META_SSA:
144 case META_CP: 145 case META_CP:
145 case META_POR: 146 case META_POR:
146 if (unlikely(blkno >= max_blks))
147 goto out;
148 if (unlikely(blkno < SEG0_BLKADDR(sbi)))
149 goto out;
150 blk_addr = blkno; 147 blk_addr = blkno;
151 break; 148 break;
152 default: 149 default:
@@ -169,6 +166,20 @@ out:
169 return blkno - start; 166 return blkno - start;
170} 167}
171 168
169void ra_meta_pages_cond(struct f2fs_sb_info *sbi, pgoff_t index)
170{
171 struct page *page;
172 bool readahead = false;
173
174 page = find_get_page(META_MAPPING(sbi), index);
175 if (!page || (page && !PageUptodate(page)))
176 readahead = true;
177 f2fs_put_page(page, 0);
178
179 if (readahead)
180 ra_meta_pages(sbi, index, MAX_BIO_BLOCKS(sbi), META_POR);
181}
182
172static int f2fs_write_meta_page(struct page *page, 183static int f2fs_write_meta_page(struct page *page,
173 struct writeback_control *wbc) 184 struct writeback_control *wbc)
174{ 185{
@@ -178,7 +189,7 @@ static int f2fs_write_meta_page(struct page *page,
178 189
179 if (unlikely(sbi->por_doing)) 190 if (unlikely(sbi->por_doing))
180 goto redirty_out; 191 goto redirty_out;
181 if (wbc->for_reclaim) 192 if (wbc->for_reclaim && page->index < GET_SUM_BLOCK(sbi, 0))
182 goto redirty_out; 193 goto redirty_out;
183 if (unlikely(f2fs_cp_error(sbi))) 194 if (unlikely(f2fs_cp_error(sbi)))
184 goto redirty_out; 195 goto redirty_out;
@@ -187,6 +198,9 @@ static int f2fs_write_meta_page(struct page *page,
187 write_meta_page(sbi, page); 198 write_meta_page(sbi, page);
188 dec_page_count(sbi, F2FS_DIRTY_META); 199 dec_page_count(sbi, F2FS_DIRTY_META);
189 unlock_page(page); 200 unlock_page(page);
201
202 if (wbc->for_reclaim)
203 f2fs_submit_merged_bio(sbi, META, WRITE);
190 return 0; 204 return 0;
191 205
192redirty_out: 206redirty_out:
@@ -298,46 +312,57 @@ const struct address_space_operations f2fs_meta_aops = {
298 312
299static void __add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type) 313static void __add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type)
300{ 314{
315 struct inode_management *im = &sbi->im[type];
301 struct ino_entry *e; 316 struct ino_entry *e;
302retry: 317retry:
303 spin_lock(&sbi->ino_lock[type]); 318 if (radix_tree_preload(GFP_NOFS)) {
319 cond_resched();
320 goto retry;
321 }
322
323 spin_lock(&im->ino_lock);
304 324
305 e = radix_tree_lookup(&sbi->ino_root[type], ino); 325 e = radix_tree_lookup(&im->ino_root, ino);
306 if (!e) { 326 if (!e) {
307 e = kmem_cache_alloc(ino_entry_slab, GFP_ATOMIC); 327 e = kmem_cache_alloc(ino_entry_slab, GFP_ATOMIC);
308 if (!e) { 328 if (!e) {
309 spin_unlock(&sbi->ino_lock[type]); 329 spin_unlock(&im->ino_lock);
330 radix_tree_preload_end();
310 goto retry; 331 goto retry;
311 } 332 }
312 if (radix_tree_insert(&sbi->ino_root[type], ino, e)) { 333 if (radix_tree_insert(&im->ino_root, ino, e)) {
313 spin_unlock(&sbi->ino_lock[type]); 334 spin_unlock(&im->ino_lock);
314 kmem_cache_free(ino_entry_slab, e); 335 kmem_cache_free(ino_entry_slab, e);
336 radix_tree_preload_end();
315 goto retry; 337 goto retry;
316 } 338 }
317 memset(e, 0, sizeof(struct ino_entry)); 339 memset(e, 0, sizeof(struct ino_entry));
318 e->ino = ino; 340 e->ino = ino;
319 341
320 list_add_tail(&e->list, &sbi->ino_list[type]); 342 list_add_tail(&e->list, &im->ino_list);
343 if (type != ORPHAN_INO)
344 im->ino_num++;
321 } 345 }
322 spin_unlock(&sbi->ino_lock[type]); 346 spin_unlock(&im->ino_lock);
347 radix_tree_preload_end();
323} 348}
324 349
325static void __remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type) 350static void __remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type)
326{ 351{
352 struct inode_management *im = &sbi->im[type];
327 struct ino_entry *e; 353 struct ino_entry *e;
328 354
329 spin_lock(&sbi->ino_lock[type]); 355 spin_lock(&im->ino_lock);
330 e = radix_tree_lookup(&sbi->ino_root[type], ino); 356 e = radix_tree_lookup(&im->ino_root, ino);
331 if (e) { 357 if (e) {
332 list_del(&e->list); 358 list_del(&e->list);
333 radix_tree_delete(&sbi->ino_root[type], ino); 359 radix_tree_delete(&im->ino_root, ino);
334 if (type == ORPHAN_INO) 360 im->ino_num--;
335 sbi->n_orphans--; 361 spin_unlock(&im->ino_lock);
336 spin_unlock(&sbi->ino_lock[type]);
337 kmem_cache_free(ino_entry_slab, e); 362 kmem_cache_free(ino_entry_slab, e);
338 return; 363 return;
339 } 364 }
340 spin_unlock(&sbi->ino_lock[type]); 365 spin_unlock(&im->ino_lock);
341} 366}
342 367
343void add_dirty_inode(struct f2fs_sb_info *sbi, nid_t ino, int type) 368void add_dirty_inode(struct f2fs_sb_info *sbi, nid_t ino, int type)
@@ -355,10 +380,12 @@ void remove_dirty_inode(struct f2fs_sb_info *sbi, nid_t ino, int type)
355/* mode should be APPEND_INO or UPDATE_INO */ 380/* mode should be APPEND_INO or UPDATE_INO */
356bool exist_written_data(struct f2fs_sb_info *sbi, nid_t ino, int mode) 381bool exist_written_data(struct f2fs_sb_info *sbi, nid_t ino, int mode)
357{ 382{
383 struct inode_management *im = &sbi->im[mode];
358 struct ino_entry *e; 384 struct ino_entry *e;
359 spin_lock(&sbi->ino_lock[mode]); 385
360 e = radix_tree_lookup(&sbi->ino_root[mode], ino); 386 spin_lock(&im->ino_lock);
361 spin_unlock(&sbi->ino_lock[mode]); 387 e = radix_tree_lookup(&im->ino_root, ino);
388 spin_unlock(&im->ino_lock);
362 return e ? true : false; 389 return e ? true : false;
363} 390}
364 391
@@ -368,36 +395,42 @@ void release_dirty_inode(struct f2fs_sb_info *sbi)
368 int i; 395 int i;
369 396
370 for (i = APPEND_INO; i <= UPDATE_INO; i++) { 397 for (i = APPEND_INO; i <= UPDATE_INO; i++) {
371 spin_lock(&sbi->ino_lock[i]); 398 struct inode_management *im = &sbi->im[i];
372 list_for_each_entry_safe(e, tmp, &sbi->ino_list[i], list) { 399
400 spin_lock(&im->ino_lock);
401 list_for_each_entry_safe(e, tmp, &im->ino_list, list) {
373 list_del(&e->list); 402 list_del(&e->list);
374 radix_tree_delete(&sbi->ino_root[i], e->ino); 403 radix_tree_delete(&im->ino_root, e->ino);
375 kmem_cache_free(ino_entry_slab, e); 404 kmem_cache_free(ino_entry_slab, e);
405 im->ino_num--;
376 } 406 }
377 spin_unlock(&sbi->ino_lock[i]); 407 spin_unlock(&im->ino_lock);
378 } 408 }
379} 409}
380 410
381int acquire_orphan_inode(struct f2fs_sb_info *sbi) 411int acquire_orphan_inode(struct f2fs_sb_info *sbi)
382{ 412{
413 struct inode_management *im = &sbi->im[ORPHAN_INO];
383 int err = 0; 414 int err = 0;
384 415
385 spin_lock(&sbi->ino_lock[ORPHAN_INO]); 416 spin_lock(&im->ino_lock);
386 if (unlikely(sbi->n_orphans >= sbi->max_orphans)) 417 if (unlikely(im->ino_num >= sbi->max_orphans))
387 err = -ENOSPC; 418 err = -ENOSPC;
388 else 419 else
389 sbi->n_orphans++; 420 im->ino_num++;
390 spin_unlock(&sbi->ino_lock[ORPHAN_INO]); 421 spin_unlock(&im->ino_lock);
391 422
392 return err; 423 return err;
393} 424}
394 425
395void release_orphan_inode(struct f2fs_sb_info *sbi) 426void release_orphan_inode(struct f2fs_sb_info *sbi)
396{ 427{
397 spin_lock(&sbi->ino_lock[ORPHAN_INO]); 428 struct inode_management *im = &sbi->im[ORPHAN_INO];
398 f2fs_bug_on(sbi, sbi->n_orphans == 0); 429
399 sbi->n_orphans--; 430 spin_lock(&im->ino_lock);
400 spin_unlock(&sbi->ino_lock[ORPHAN_INO]); 431 f2fs_bug_on(sbi, im->ino_num == 0);
432 im->ino_num--;
433 spin_unlock(&im->ino_lock);
401} 434}
402 435
403void add_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino) 436void add_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino)
@@ -460,17 +493,19 @@ static void write_orphan_inodes(struct f2fs_sb_info *sbi, block_t start_blk)
460 struct f2fs_orphan_block *orphan_blk = NULL; 493 struct f2fs_orphan_block *orphan_blk = NULL;
461 unsigned int nentries = 0; 494 unsigned int nentries = 0;
462 unsigned short index; 495 unsigned short index;
463 unsigned short orphan_blocks = 496 unsigned short orphan_blocks;
464 (unsigned short)GET_ORPHAN_BLOCKS(sbi->n_orphans);
465 struct page *page = NULL; 497 struct page *page = NULL;
466 struct ino_entry *orphan = NULL; 498 struct ino_entry *orphan = NULL;
499 struct inode_management *im = &sbi->im[ORPHAN_INO];
500
501 orphan_blocks = GET_ORPHAN_BLOCKS(im->ino_num);
467 502
468 for (index = 0; index < orphan_blocks; index++) 503 for (index = 0; index < orphan_blocks; index++)
469 grab_meta_page(sbi, start_blk + index); 504 grab_meta_page(sbi, start_blk + index);
470 505
471 index = 1; 506 index = 1;
472 spin_lock(&sbi->ino_lock[ORPHAN_INO]); 507 spin_lock(&im->ino_lock);
473 head = &sbi->ino_list[ORPHAN_INO]; 508 head = &im->ino_list;
474 509
475 /* loop for each orphan inode entry and write them in Jornal block */ 510 /* loop for each orphan inode entry and write them in Jornal block */
476 list_for_each_entry(orphan, head, list) { 511 list_for_each_entry(orphan, head, list) {
@@ -510,7 +545,7 @@ static void write_orphan_inodes(struct f2fs_sb_info *sbi, block_t start_blk)
510 f2fs_put_page(page, 1); 545 f2fs_put_page(page, 1);
511 } 546 }
512 547
513 spin_unlock(&sbi->ino_lock[ORPHAN_INO]); 548 spin_unlock(&im->ino_lock);
514} 549}
515 550
516static struct page *validate_checkpoint(struct f2fs_sb_info *sbi, 551static struct page *validate_checkpoint(struct f2fs_sb_info *sbi,
@@ -731,6 +766,9 @@ void sync_dirty_dir_inodes(struct f2fs_sb_info *sbi)
731 struct dir_inode_entry *entry; 766 struct dir_inode_entry *entry;
732 struct inode *inode; 767 struct inode *inode;
733retry: 768retry:
769 if (unlikely(f2fs_cp_error(sbi)))
770 return;
771
734 spin_lock(&sbi->dir_inode_lock); 772 spin_lock(&sbi->dir_inode_lock);
735 773
736 head = &sbi->dir_inode_list; 774 head = &sbi->dir_inode_list;
@@ -830,6 +868,7 @@ static void do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
830 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi); 868 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
831 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_WARM_NODE); 869 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_WARM_NODE);
832 struct f2fs_nm_info *nm_i = NM_I(sbi); 870 struct f2fs_nm_info *nm_i = NM_I(sbi);
871 unsigned long orphan_num = sbi->im[ORPHAN_INO].ino_num;
833 nid_t last_nid = nm_i->next_scan_nid; 872 nid_t last_nid = nm_i->next_scan_nid;
834 block_t start_blk; 873 block_t start_blk;
835 struct page *cp_page; 874 struct page *cp_page;
@@ -889,7 +928,7 @@ static void do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
889 else 928 else
890 clear_ckpt_flags(ckpt, CP_COMPACT_SUM_FLAG); 929 clear_ckpt_flags(ckpt, CP_COMPACT_SUM_FLAG);
891 930
892 orphan_blocks = GET_ORPHAN_BLOCKS(sbi->n_orphans); 931 orphan_blocks = GET_ORPHAN_BLOCKS(orphan_num);
893 ckpt->cp_pack_start_sum = cpu_to_le32(1 + cp_payload_blks + 932 ckpt->cp_pack_start_sum = cpu_to_le32(1 + cp_payload_blks +
894 orphan_blocks); 933 orphan_blocks);
895 934
@@ -905,7 +944,7 @@ static void do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
905 orphan_blocks); 944 orphan_blocks);
906 } 945 }
907 946
908 if (sbi->n_orphans) 947 if (orphan_num)
909 set_ckpt_flags(ckpt, CP_ORPHAN_PRESENT_FLAG); 948 set_ckpt_flags(ckpt, CP_ORPHAN_PRESENT_FLAG);
910 else 949 else
911 clear_ckpt_flags(ckpt, CP_ORPHAN_PRESENT_FLAG); 950 clear_ckpt_flags(ckpt, CP_ORPHAN_PRESENT_FLAG);
@@ -940,7 +979,7 @@ static void do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
940 f2fs_put_page(cp_page, 1); 979 f2fs_put_page(cp_page, 1);
941 } 980 }
942 981
943 if (sbi->n_orphans) { 982 if (orphan_num) {
944 write_orphan_inodes(sbi, start_blk); 983 write_orphan_inodes(sbi, start_blk);
945 start_blk += orphan_blocks; 984 start_blk += orphan_blocks;
946 } 985 }
@@ -975,6 +1014,9 @@ static void do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
975 /* Here, we only have one bio having CP pack */ 1014 /* Here, we only have one bio having CP pack */
976 sync_meta_pages(sbi, META_FLUSH, LONG_MAX); 1015 sync_meta_pages(sbi, META_FLUSH, LONG_MAX);
977 1016
1017 /* wait for previous submitted meta pages writeback */
1018 wait_on_all_pages_writeback(sbi);
1019
978 release_dirty_inode(sbi); 1020 release_dirty_inode(sbi);
979 1021
980 if (unlikely(f2fs_cp_error(sbi))) 1022 if (unlikely(f2fs_cp_error(sbi)))
@@ -1036,9 +1078,12 @@ void init_ino_entry_info(struct f2fs_sb_info *sbi)
1036 int i; 1078 int i;
1037 1079
1038 for (i = 0; i < MAX_INO_ENTRY; i++) { 1080 for (i = 0; i < MAX_INO_ENTRY; i++) {
1039 INIT_RADIX_TREE(&sbi->ino_root[i], GFP_ATOMIC); 1081 struct inode_management *im = &sbi->im[i];
1040 spin_lock_init(&sbi->ino_lock[i]); 1082
1041 INIT_LIST_HEAD(&sbi->ino_list[i]); 1083 INIT_RADIX_TREE(&im->ino_root, GFP_ATOMIC);
1084 spin_lock_init(&im->ino_lock);
1085 INIT_LIST_HEAD(&im->ino_list);
1086 im->ino_num = 0;
1042 } 1087 }
1043 1088
1044 /* 1089 /*
@@ -1047,7 +1092,6 @@ void init_ino_entry_info(struct f2fs_sb_info *sbi)
1047 * orphan entries with the limitation one reserved segment 1092 * orphan entries with the limitation one reserved segment
1048 * for cp pack we can have max 1020*504 orphan entries 1093 * for cp pack we can have max 1020*504 orphan entries
1049 */ 1094 */
1050 sbi->n_orphans = 0;
1051 sbi->max_orphans = (sbi->blocks_per_seg - F2FS_CP_PACKS - 1095 sbi->max_orphans = (sbi->blocks_per_seg - F2FS_CP_PACKS -
1052 NR_CURSEG_TYPE) * F2FS_ORPHANS_PER_BLOCK; 1096 NR_CURSEG_TYPE) * F2FS_ORPHANS_PER_BLOCK;
1053} 1097}
diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
index 8e58c4cc2cb9..7ec697b37f19 100644
--- a/fs/f2fs/data.c
+++ b/fs/f2fs/data.c
@@ -61,11 +61,6 @@ static void f2fs_write_end_io(struct bio *bio, int err)
61 dec_page_count(sbi, F2FS_WRITEBACK); 61 dec_page_count(sbi, F2FS_WRITEBACK);
62 } 62 }
63 63
64 if (sbi->wait_io) {
65 complete(sbi->wait_io);
66 sbi->wait_io = NULL;
67 }
68
69 if (!get_pages(sbi, F2FS_WRITEBACK) && 64 if (!get_pages(sbi, F2FS_WRITEBACK) &&
70 !list_empty(&sbi->cp_wait.task_list)) 65 !list_empty(&sbi->cp_wait.task_list))
71 wake_up(&sbi->cp_wait); 66 wake_up(&sbi->cp_wait);
@@ -95,34 +90,18 @@ static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr,
95static void __submit_merged_bio(struct f2fs_bio_info *io) 90static void __submit_merged_bio(struct f2fs_bio_info *io)
96{ 91{
97 struct f2fs_io_info *fio = &io->fio; 92 struct f2fs_io_info *fio = &io->fio;
98 int rw;
99 93
100 if (!io->bio) 94 if (!io->bio)
101 return; 95 return;
102 96
103 rw = fio->rw; 97 if (is_read_io(fio->rw))
104 98 trace_f2fs_submit_read_bio(io->sbi->sb, fio->rw,
105 if (is_read_io(rw)) { 99 fio->type, io->bio);
106 trace_f2fs_submit_read_bio(io->sbi->sb, rw, 100 else
107 fio->type, io->bio); 101 trace_f2fs_submit_write_bio(io->sbi->sb, fio->rw,
108 submit_bio(rw, io->bio); 102 fio->type, io->bio);
109 } else {
110 trace_f2fs_submit_write_bio(io->sbi->sb, rw,
111 fio->type, io->bio);
112 /*
113 * META_FLUSH is only from the checkpoint procedure, and we
114 * should wait this metadata bio for FS consistency.
115 */
116 if (fio->type == META_FLUSH) {
117 DECLARE_COMPLETION_ONSTACK(wait);
118 io->sbi->wait_io = &wait;
119 submit_bio(rw, io->bio);
120 wait_for_completion(&wait);
121 } else {
122 submit_bio(rw, io->bio);
123 }
124 }
125 103
104 submit_bio(fio->rw, io->bio);
126 io->bio = NULL; 105 io->bio = NULL;
127} 106}
128 107
@@ -257,9 +236,6 @@ int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
257 bool need_put = dn->inode_page ? false : true; 236 bool need_put = dn->inode_page ? false : true;
258 int err; 237 int err;
259 238
260 /* if inode_page exists, index should be zero */
261 f2fs_bug_on(F2FS_I_SB(dn->inode), !need_put && index);
262
263 err = get_dnode_of_data(dn, index, ALLOC_NODE); 239 err = get_dnode_of_data(dn, index, ALLOC_NODE);
264 if (err) 240 if (err)
265 return err; 241 return err;
@@ -740,14 +716,14 @@ int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
740static int f2fs_read_data_page(struct file *file, struct page *page) 716static int f2fs_read_data_page(struct file *file, struct page *page)
741{ 717{
742 struct inode *inode = page->mapping->host; 718 struct inode *inode = page->mapping->host;
743 int ret; 719 int ret = -EAGAIN;
744 720
745 trace_f2fs_readpage(page, DATA); 721 trace_f2fs_readpage(page, DATA);
746 722
747 /* If the file has inline data, try to read it directly */ 723 /* If the file has inline data, try to read it directly */
748 if (f2fs_has_inline_data(inode)) 724 if (f2fs_has_inline_data(inode))
749 ret = f2fs_read_inline_data(inode, page); 725 ret = f2fs_read_inline_data(inode, page);
750 else 726 if (ret == -EAGAIN)
751 ret = mpage_readpage(page, get_data_block); 727 ret = mpage_readpage(page, get_data_block);
752 728
753 return ret; 729 return ret;
@@ -859,10 +835,11 @@ write:
859 else if (has_not_enough_free_secs(sbi, 0)) 835 else if (has_not_enough_free_secs(sbi, 0))
860 goto redirty_out; 836 goto redirty_out;
861 837
838 err = -EAGAIN;
862 f2fs_lock_op(sbi); 839 f2fs_lock_op(sbi);
863 if (f2fs_has_inline_data(inode) || f2fs_may_inline(inode)) 840 if (f2fs_has_inline_data(inode))
864 err = f2fs_write_inline_data(inode, page, offset); 841 err = f2fs_write_inline_data(inode, page);
865 else 842 if (err == -EAGAIN)
866 err = do_write_data_page(page, &fio); 843 err = do_write_data_page(page, &fio);
867 f2fs_unlock_op(sbi); 844 f2fs_unlock_op(sbi);
868done: 845done:
@@ -951,7 +928,7 @@ static int f2fs_write_begin(struct file *file, struct address_space *mapping,
951{ 928{
952 struct inode *inode = mapping->host; 929 struct inode *inode = mapping->host;
953 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 930 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
954 struct page *page; 931 struct page *page, *ipage;
955 pgoff_t index = ((unsigned long long) pos) >> PAGE_CACHE_SHIFT; 932 pgoff_t index = ((unsigned long long) pos) >> PAGE_CACHE_SHIFT;
956 struct dnode_of_data dn; 933 struct dnode_of_data dn;
957 int err = 0; 934 int err = 0;
@@ -959,45 +936,60 @@ static int f2fs_write_begin(struct file *file, struct address_space *mapping,
959 trace_f2fs_write_begin(inode, pos, len, flags); 936 trace_f2fs_write_begin(inode, pos, len, flags);
960 937
961 f2fs_balance_fs(sbi); 938 f2fs_balance_fs(sbi);
962repeat:
963 err = f2fs_convert_inline_data(inode, pos + len, NULL);
964 if (err)
965 goto fail;
966 939
940 /*
941 * We should check this at this moment to avoid deadlock on inode page
942 * and #0 page. The locking rule for inline_data conversion should be:
943 * lock_page(page #0) -> lock_page(inode_page)
944 */
945 if (index != 0) {
946 err = f2fs_convert_inline_inode(inode);
947 if (err)
948 goto fail;
949 }
950repeat:
967 page = grab_cache_page_write_begin(mapping, index, flags); 951 page = grab_cache_page_write_begin(mapping, index, flags);
968 if (!page) { 952 if (!page) {
969 err = -ENOMEM; 953 err = -ENOMEM;
970 goto fail; 954 goto fail;
971 } 955 }
972 956
973 /* to avoid latency during memory pressure */
974 unlock_page(page);
975
976 *pagep = page; 957 *pagep = page;
977 958
978 if (f2fs_has_inline_data(inode) && (pos + len) <= MAX_INLINE_DATA)
979 goto inline_data;
980
981 f2fs_lock_op(sbi); 959 f2fs_lock_op(sbi);
982 set_new_dnode(&dn, inode, NULL, NULL, 0); 960
983 err = f2fs_reserve_block(&dn, index); 961 /* check inline_data */
984 f2fs_unlock_op(sbi); 962 ipage = get_node_page(sbi, inode->i_ino);
985 if (err) { 963 if (IS_ERR(ipage)) {
986 f2fs_put_page(page, 0); 964 err = PTR_ERR(ipage);
987 goto fail; 965 goto unlock_fail;
988 }
989inline_data:
990 lock_page(page);
991 if (unlikely(page->mapping != mapping)) {
992 f2fs_put_page(page, 1);
993 goto repeat;
994 } 966 }
995 967
996 f2fs_wait_on_page_writeback(page, DATA); 968 set_new_dnode(&dn, inode, ipage, ipage, 0);
969
970 if (f2fs_has_inline_data(inode)) {
971 if (pos + len <= MAX_INLINE_DATA) {
972 read_inline_data(page, ipage);
973 set_inode_flag(F2FS_I(inode), FI_DATA_EXIST);
974 sync_inode_page(&dn);
975 goto put_next;
976 }
977 err = f2fs_convert_inline_page(&dn, page);
978 if (err)
979 goto put_fail;
980 }
981 err = f2fs_reserve_block(&dn, index);
982 if (err)
983 goto put_fail;
984put_next:
985 f2fs_put_dnode(&dn);
986 f2fs_unlock_op(sbi);
997 987
998 if ((len == PAGE_CACHE_SIZE) || PageUptodate(page)) 988 if ((len == PAGE_CACHE_SIZE) || PageUptodate(page))
999 return 0; 989 return 0;
1000 990
991 f2fs_wait_on_page_writeback(page, DATA);
992
1001 if ((pos & PAGE_CACHE_MASK) >= i_size_read(inode)) { 993 if ((pos & PAGE_CACHE_MASK) >= i_size_read(inode)) {
1002 unsigned start = pos & (PAGE_CACHE_SIZE - 1); 994 unsigned start = pos & (PAGE_CACHE_SIZE - 1);
1003 unsigned end = start + len; 995 unsigned end = start + len;
@@ -1010,18 +1002,10 @@ inline_data:
1010 if (dn.data_blkaddr == NEW_ADDR) { 1002 if (dn.data_blkaddr == NEW_ADDR) {
1011 zero_user_segment(page, 0, PAGE_CACHE_SIZE); 1003 zero_user_segment(page, 0, PAGE_CACHE_SIZE);
1012 } else { 1004 } else {
1013 if (f2fs_has_inline_data(inode)) { 1005 err = f2fs_submit_page_bio(sbi, page, dn.data_blkaddr,
1014 err = f2fs_read_inline_data(inode, page); 1006 READ_SYNC);
1015 if (err) { 1007 if (err)
1016 page_cache_release(page); 1008 goto fail;
1017 goto fail;
1018 }
1019 } else {
1020 err = f2fs_submit_page_bio(sbi, page, dn.data_blkaddr,
1021 READ_SYNC);
1022 if (err)
1023 goto fail;
1024 }
1025 1009
1026 lock_page(page); 1010 lock_page(page);
1027 if (unlikely(!PageUptodate(page))) { 1011 if (unlikely(!PageUptodate(page))) {
@@ -1038,6 +1022,12 @@ out:
1038 SetPageUptodate(page); 1022 SetPageUptodate(page);
1039 clear_cold_data(page); 1023 clear_cold_data(page);
1040 return 0; 1024 return 0;
1025
1026put_fail:
1027 f2fs_put_dnode(&dn);
1028unlock_fail:
1029 f2fs_unlock_op(sbi);
1030 f2fs_put_page(page, 1);
1041fail: 1031fail:
1042 f2fs_write_failed(mapping, pos + len); 1032 f2fs_write_failed(mapping, pos + len);
1043 return err; 1033 return err;
@@ -1052,10 +1042,7 @@ static int f2fs_write_end(struct file *file,
1052 1042
1053 trace_f2fs_write_end(inode, pos, len, copied); 1043 trace_f2fs_write_end(inode, pos, len, copied);
1054 1044
1055 if (f2fs_is_atomic_file(inode) || f2fs_is_volatile_file(inode)) 1045 set_page_dirty(page);
1056 register_inmem_page(inode, page);
1057 else
1058 set_page_dirty(page);
1059 1046
1060 if (pos + copied > i_size_read(inode)) { 1047 if (pos + copied > i_size_read(inode)) {
1061 i_size_write(inode, pos + copied); 1048 i_size_write(inode, pos + copied);
@@ -1093,9 +1080,12 @@ static ssize_t f2fs_direct_IO(int rw, struct kiocb *iocb,
1093 size_t count = iov_iter_count(iter); 1080 size_t count = iov_iter_count(iter);
1094 int err; 1081 int err;
1095 1082
1096 /* Let buffer I/O handle the inline data case. */ 1083 /* we don't need to use inline_data strictly */
1097 if (f2fs_has_inline_data(inode)) 1084 if (f2fs_has_inline_data(inode)) {
1098 return 0; 1085 err = f2fs_convert_inline_inode(inode);
1086 if (err)
1087 return err;
1088 }
1099 1089
1100 if (check_direct_IO(inode, rw, iter, offset)) 1090 if (check_direct_IO(inode, rw, iter, offset))
1101 return 0; 1091 return 0;
@@ -1119,6 +1109,9 @@ static void f2fs_invalidate_data_page(struct page *page, unsigned int offset,
1119 if (offset % PAGE_CACHE_SIZE || length != PAGE_CACHE_SIZE) 1109 if (offset % PAGE_CACHE_SIZE || length != PAGE_CACHE_SIZE)
1120 return; 1110 return;
1121 1111
1112 if (f2fs_is_atomic_file(inode) || f2fs_is_volatile_file(inode))
1113 invalidate_inmem_page(inode, page);
1114
1122 if (PageDirty(page)) 1115 if (PageDirty(page))
1123 inode_dec_dirty_pages(inode); 1116 inode_dec_dirty_pages(inode);
1124 ClearPagePrivate(page); 1117 ClearPagePrivate(page);
@@ -1138,6 +1131,12 @@ static int f2fs_set_data_page_dirty(struct page *page)
1138 trace_f2fs_set_page_dirty(page, DATA); 1131 trace_f2fs_set_page_dirty(page, DATA);
1139 1132
1140 SetPageUptodate(page); 1133 SetPageUptodate(page);
1134
1135 if (f2fs_is_atomic_file(inode) || f2fs_is_volatile_file(inode)) {
1136 register_inmem_page(inode, page);
1137 return 1;
1138 }
1139
1141 mark_inode_dirty(inode); 1140 mark_inode_dirty(inode);
1142 1141
1143 if (!PageDirty(page)) { 1142 if (!PageDirty(page)) {
@@ -1152,9 +1151,12 @@ static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
1152{ 1151{
1153 struct inode *inode = mapping->host; 1152 struct inode *inode = mapping->host;
1154 1153
1155 if (f2fs_has_inline_data(inode)) 1154 /* we don't need to use inline_data strictly */
1156 return 0; 1155 if (f2fs_has_inline_data(inode)) {
1157 1156 int err = f2fs_convert_inline_inode(inode);
1157 if (err)
1158 return err;
1159 }
1158 return generic_block_bmap(mapping, block, get_data_block); 1160 return generic_block_bmap(mapping, block, get_data_block);
1159} 1161}
1160 1162
diff --git a/fs/f2fs/debug.c b/fs/f2fs/debug.c
index 0a91ab813a9e..91e8f699ab30 100644
--- a/fs/f2fs/debug.c
+++ b/fs/f2fs/debug.c
@@ -39,13 +39,15 @@ static void update_general_status(struct f2fs_sb_info *sbi)
39 si->ndirty_dent = get_pages(sbi, F2FS_DIRTY_DENTS); 39 si->ndirty_dent = get_pages(sbi, F2FS_DIRTY_DENTS);
40 si->ndirty_dirs = sbi->n_dirty_dirs; 40 si->ndirty_dirs = sbi->n_dirty_dirs;
41 si->ndirty_meta = get_pages(sbi, F2FS_DIRTY_META); 41 si->ndirty_meta = get_pages(sbi, F2FS_DIRTY_META);
42 si->inmem_pages = get_pages(sbi, F2FS_INMEM_PAGES);
42 si->total_count = (int)sbi->user_block_count / sbi->blocks_per_seg; 43 si->total_count = (int)sbi->user_block_count / sbi->blocks_per_seg;
43 si->rsvd_segs = reserved_segments(sbi); 44 si->rsvd_segs = reserved_segments(sbi);
44 si->overp_segs = overprovision_segments(sbi); 45 si->overp_segs = overprovision_segments(sbi);
45 si->valid_count = valid_user_blocks(sbi); 46 si->valid_count = valid_user_blocks(sbi);
46 si->valid_node_count = valid_node_count(sbi); 47 si->valid_node_count = valid_node_count(sbi);
47 si->valid_inode_count = valid_inode_count(sbi); 48 si->valid_inode_count = valid_inode_count(sbi);
48 si->inline_inode = sbi->inline_inode; 49 si->inline_inode = atomic_read(&sbi->inline_inode);
50 si->inline_dir = atomic_read(&sbi->inline_dir);
49 si->utilization = utilization(sbi); 51 si->utilization = utilization(sbi);
50 52
51 si->free_segs = free_segments(sbi); 53 si->free_segs = free_segments(sbi);
@@ -118,6 +120,7 @@ static void update_mem_info(struct f2fs_sb_info *sbi)
118{ 120{
119 struct f2fs_stat_info *si = F2FS_STAT(sbi); 121 struct f2fs_stat_info *si = F2FS_STAT(sbi);
120 unsigned npages; 122 unsigned npages;
123 int i;
121 124
122 if (si->base_mem) 125 if (si->base_mem)
123 goto get_cache; 126 goto get_cache;
@@ -167,8 +170,9 @@ get_cache:
167 si->cache_mem += npages << PAGE_CACHE_SHIFT; 170 si->cache_mem += npages << PAGE_CACHE_SHIFT;
168 npages = META_MAPPING(sbi)->nrpages; 171 npages = META_MAPPING(sbi)->nrpages;
169 si->cache_mem += npages << PAGE_CACHE_SHIFT; 172 si->cache_mem += npages << PAGE_CACHE_SHIFT;
170 si->cache_mem += sbi->n_orphans * sizeof(struct ino_entry);
171 si->cache_mem += sbi->n_dirty_dirs * sizeof(struct dir_inode_entry); 173 si->cache_mem += sbi->n_dirty_dirs * sizeof(struct dir_inode_entry);
174 for (i = 0; i <= UPDATE_INO; i++)
175 si->cache_mem += sbi->im[i].ino_num * sizeof(struct ino_entry);
172} 176}
173 177
174static int stat_show(struct seq_file *s, void *v) 178static int stat_show(struct seq_file *s, void *v)
@@ -200,6 +204,8 @@ static int stat_show(struct seq_file *s, void *v)
200 si->valid_count - si->valid_node_count); 204 si->valid_count - si->valid_node_count);
201 seq_printf(s, " - Inline_data Inode: %u\n", 205 seq_printf(s, " - Inline_data Inode: %u\n",
202 si->inline_inode); 206 si->inline_inode);
207 seq_printf(s, " - Inline_dentry Inode: %u\n",
208 si->inline_dir);
203 seq_printf(s, "\nMain area: %d segs, %d secs %d zones\n", 209 seq_printf(s, "\nMain area: %d segs, %d secs %d zones\n",
204 si->main_area_segs, si->main_area_sections, 210 si->main_area_segs, si->main_area_sections,
205 si->main_area_zones); 211 si->main_area_zones);
@@ -244,6 +250,8 @@ static int stat_show(struct seq_file *s, void *v)
244 seq_printf(s, "\nExtent Hit Ratio: %d / %d\n", 250 seq_printf(s, "\nExtent Hit Ratio: %d / %d\n",
245 si->hit_ext, si->total_ext); 251 si->hit_ext, si->total_ext);
246 seq_puts(s, "\nBalancing F2FS Async:\n"); 252 seq_puts(s, "\nBalancing F2FS Async:\n");
253 seq_printf(s, " - inmem: %4d\n",
254 si->inmem_pages);
247 seq_printf(s, " - nodes: %4d in %4d\n", 255 seq_printf(s, " - nodes: %4d in %4d\n",
248 si->ndirty_node, si->node_pages); 256 si->ndirty_node, si->node_pages);
249 seq_printf(s, " - dents: %4d in dirs:%4d\n", 257 seq_printf(s, " - dents: %4d in dirs:%4d\n",
@@ -321,6 +329,9 @@ int f2fs_build_stats(struct f2fs_sb_info *sbi)
321 si->sbi = sbi; 329 si->sbi = sbi;
322 sbi->stat_info = si; 330 sbi->stat_info = si;
323 331
332 atomic_set(&sbi->inline_inode, 0);
333 atomic_set(&sbi->inline_dir, 0);
334
324 mutex_lock(&f2fs_stat_mutex); 335 mutex_lock(&f2fs_stat_mutex);
325 list_add_tail(&si->stat_list, &f2fs_stat_list); 336 list_add_tail(&si->stat_list, &f2fs_stat_list);
326 mutex_unlock(&f2fs_stat_mutex); 337 mutex_unlock(&f2fs_stat_mutex);
diff --git a/fs/f2fs/dir.c b/fs/f2fs/dir.c
index b54f87149c09..b1a7d5737cd0 100644
--- a/fs/f2fs/dir.c
+++ b/fs/f2fs/dir.c
@@ -37,7 +37,7 @@ static unsigned int bucket_blocks(unsigned int level)
37 return 4; 37 return 4;
38} 38}
39 39
40static unsigned char f2fs_filetype_table[F2FS_FT_MAX] = { 40unsigned char f2fs_filetype_table[F2FS_FT_MAX] = {
41 [F2FS_FT_UNKNOWN] = DT_UNKNOWN, 41 [F2FS_FT_UNKNOWN] = DT_UNKNOWN,
42 [F2FS_FT_REG_FILE] = DT_REG, 42 [F2FS_FT_REG_FILE] = DT_REG,
43 [F2FS_FT_DIR] = DT_DIR, 43 [F2FS_FT_DIR] = DT_DIR,
@@ -59,7 +59,7 @@ static unsigned char f2fs_type_by_mode[S_IFMT >> S_SHIFT] = {
59 [S_IFLNK >> S_SHIFT] = F2FS_FT_SYMLINK, 59 [S_IFLNK >> S_SHIFT] = F2FS_FT_SYMLINK,
60}; 60};
61 61
62static void set_de_type(struct f2fs_dir_entry *de, struct inode *inode) 62void set_de_type(struct f2fs_dir_entry *de, struct inode *inode)
63{ 63{
64 umode_t mode = inode->i_mode; 64 umode_t mode = inode->i_mode;
65 de->file_type = f2fs_type_by_mode[(mode & S_IFMT) >> S_SHIFT]; 65 de->file_type = f2fs_type_by_mode[(mode & S_IFMT) >> S_SHIFT];
@@ -90,51 +90,70 @@ static bool early_match_name(size_t namelen, f2fs_hash_t namehash,
90} 90}
91 91
92static struct f2fs_dir_entry *find_in_block(struct page *dentry_page, 92static struct f2fs_dir_entry *find_in_block(struct page *dentry_page,
93 struct qstr *name, int *max_slots, 93 struct qstr *name, int *max_slots,
94 f2fs_hash_t namehash, struct page **res_page) 94 struct page **res_page)
95{
96 struct f2fs_dentry_block *dentry_blk;
97 struct f2fs_dir_entry *de;
98 struct f2fs_dentry_ptr d;
99
100 dentry_blk = (struct f2fs_dentry_block *)kmap(dentry_page);
101
102 make_dentry_ptr(&d, (void *)dentry_blk, 1);
103 de = find_target_dentry(name, max_slots, &d);
104
105 if (de)
106 *res_page = dentry_page;
107 else
108 kunmap(dentry_page);
109
110 /*
111 * For the most part, it should be a bug when name_len is zero.
112 * We stop here for figuring out where the bugs has occurred.
113 */
114 f2fs_bug_on(F2FS_P_SB(dentry_page), d.max < 0);
115 return de;
116}
117
118struct f2fs_dir_entry *find_target_dentry(struct qstr *name, int *max_slots,
119 struct f2fs_dentry_ptr *d)
95{ 120{
96 struct f2fs_dir_entry *de; 121 struct f2fs_dir_entry *de;
97 unsigned long bit_pos = 0; 122 unsigned long bit_pos = 0;
98 struct f2fs_dentry_block *dentry_blk = kmap(dentry_page); 123 f2fs_hash_t namehash = f2fs_dentry_hash(name);
99 const void *dentry_bits = &dentry_blk->dentry_bitmap;
100 int max_len = 0; 124 int max_len = 0;
101 125
102 while (bit_pos < NR_DENTRY_IN_BLOCK) { 126 if (max_slots)
103 if (!test_bit_le(bit_pos, dentry_bits)) { 127 *max_slots = 0;
128 while (bit_pos < d->max) {
129 if (!test_bit_le(bit_pos, d->bitmap)) {
104 if (bit_pos == 0) 130 if (bit_pos == 0)
105 max_len = 1; 131 max_len = 1;
106 else if (!test_bit_le(bit_pos - 1, dentry_bits)) 132 else if (!test_bit_le(bit_pos - 1, d->bitmap))
107 max_len++; 133 max_len++;
108 bit_pos++; 134 bit_pos++;
109 continue; 135 continue;
110 } 136 }
111 de = &dentry_blk->dentry[bit_pos]; 137 de = &d->dentry[bit_pos];
112 if (early_match_name(name->len, namehash, de)) { 138 if (early_match_name(name->len, namehash, de) &&
113 if (!memcmp(dentry_blk->filename[bit_pos], 139 !memcmp(d->filename[bit_pos], name->name, name->len))
114 name->name, 140 goto found;
115 name->len)) { 141
116 *res_page = dentry_page; 142 if (max_slots && *max_slots >= 0 && max_len > *max_slots) {
117 goto found;
118 }
119 }
120 if (max_len > *max_slots) {
121 *max_slots = max_len; 143 *max_slots = max_len;
122 max_len = 0; 144 max_len = 0;
123 } 145 }
124 146
125 /* 147 /* remain bug on condition */
126 * For the most part, it should be a bug when name_len is zero. 148 if (unlikely(!de->name_len))
127 * We stop here for figuring out where the bugs has occurred. 149 d->max = -1;
128 */
129 f2fs_bug_on(F2FS_P_SB(dentry_page), !de->name_len);
130 150
131 bit_pos += GET_DENTRY_SLOTS(le16_to_cpu(de->name_len)); 151 bit_pos += GET_DENTRY_SLOTS(le16_to_cpu(de->name_len));
132 } 152 }
133 153
134 de = NULL; 154 de = NULL;
135 kunmap(dentry_page);
136found: 155found:
137 if (max_len > *max_slots) 156 if (max_slots && max_len > *max_slots)
138 *max_slots = max_len; 157 *max_slots = max_len;
139 return de; 158 return de;
140} 159}
@@ -149,7 +168,7 @@ static struct f2fs_dir_entry *find_in_level(struct inode *dir,
149 struct page *dentry_page; 168 struct page *dentry_page;
150 struct f2fs_dir_entry *de = NULL; 169 struct f2fs_dir_entry *de = NULL;
151 bool room = false; 170 bool room = false;
152 int max_slots = 0; 171 int max_slots;
153 172
154 f2fs_bug_on(F2FS_I_SB(dir), level > MAX_DIR_HASH_DEPTH); 173 f2fs_bug_on(F2FS_I_SB(dir), level > MAX_DIR_HASH_DEPTH);
155 174
@@ -168,8 +187,7 @@ static struct f2fs_dir_entry *find_in_level(struct inode *dir,
168 continue; 187 continue;
169 } 188 }
170 189
171 de = find_in_block(dentry_page, name, &max_slots, 190 de = find_in_block(dentry_page, name, &max_slots, res_page);
172 namehash, res_page);
173 if (de) 191 if (de)
174 break; 192 break;
175 193
@@ -201,6 +219,9 @@ struct f2fs_dir_entry *f2fs_find_entry(struct inode *dir,
201 unsigned int max_depth; 219 unsigned int max_depth;
202 unsigned int level; 220 unsigned int level;
203 221
222 if (f2fs_has_inline_dentry(dir))
223 return find_in_inline_dir(dir, child, res_page);
224
204 if (npages == 0) 225 if (npages == 0)
205 return NULL; 226 return NULL;
206 227
@@ -227,6 +248,9 @@ struct f2fs_dir_entry *f2fs_parent_dir(struct inode *dir, struct page **p)
227 struct f2fs_dir_entry *de; 248 struct f2fs_dir_entry *de;
228 struct f2fs_dentry_block *dentry_blk; 249 struct f2fs_dentry_block *dentry_blk;
229 250
251 if (f2fs_has_inline_dentry(dir))
252 return f2fs_parent_inline_dir(dir, p);
253
230 page = get_lock_data_page(dir, 0); 254 page = get_lock_data_page(dir, 0);
231 if (IS_ERR(page)) 255 if (IS_ERR(page))
232 return NULL; 256 return NULL;
@@ -247,7 +271,7 @@ ino_t f2fs_inode_by_name(struct inode *dir, struct qstr *qstr)
247 de = f2fs_find_entry(dir, qstr, &page); 271 de = f2fs_find_entry(dir, qstr, &page);
248 if (de) { 272 if (de) {
249 res = le32_to_cpu(de->ino); 273 res = le32_to_cpu(de->ino);
250 kunmap(page); 274 f2fs_dentry_kunmap(dir, page);
251 f2fs_put_page(page, 0); 275 f2fs_put_page(page, 0);
252 } 276 }
253 277
@@ -257,11 +281,13 @@ ino_t f2fs_inode_by_name(struct inode *dir, struct qstr *qstr)
257void f2fs_set_link(struct inode *dir, struct f2fs_dir_entry *de, 281void f2fs_set_link(struct inode *dir, struct f2fs_dir_entry *de,
258 struct page *page, struct inode *inode) 282 struct page *page, struct inode *inode)
259{ 283{
284 enum page_type type = f2fs_has_inline_dentry(dir) ? NODE : DATA;
260 lock_page(page); 285 lock_page(page);
261 f2fs_wait_on_page_writeback(page, DATA); 286 f2fs_wait_on_page_writeback(page, type);
262 de->ino = cpu_to_le32(inode->i_ino); 287 de->ino = cpu_to_le32(inode->i_ino);
263 set_de_type(de, inode); 288 set_de_type(de, inode);
264 kunmap(page); 289 if (!f2fs_has_inline_dentry(dir))
290 kunmap(page);
265 set_page_dirty(page); 291 set_page_dirty(page);
266 dir->i_mtime = dir->i_ctime = CURRENT_TIME; 292 dir->i_mtime = dir->i_ctime = CURRENT_TIME;
267 mark_inode_dirty(dir); 293 mark_inode_dirty(dir);
@@ -296,36 +322,48 @@ int update_dent_inode(struct inode *inode, const struct qstr *name)
296 return 0; 322 return 0;
297} 323}
298 324
299static int make_empty_dir(struct inode *inode, 325void do_make_empty_dir(struct inode *inode, struct inode *parent,
300 struct inode *parent, struct page *page) 326 struct f2fs_dentry_ptr *d)
301{ 327{
302 struct page *dentry_page;
303 struct f2fs_dentry_block *dentry_blk;
304 struct f2fs_dir_entry *de; 328 struct f2fs_dir_entry *de;
305 329
306 dentry_page = get_new_data_page(inode, page, 0, true); 330 de = &d->dentry[0];
307 if (IS_ERR(dentry_page))
308 return PTR_ERR(dentry_page);
309
310
311 dentry_blk = kmap_atomic(dentry_page);
312
313 de = &dentry_blk->dentry[0];
314 de->name_len = cpu_to_le16(1); 331 de->name_len = cpu_to_le16(1);
315 de->hash_code = 0; 332 de->hash_code = 0;
316 de->ino = cpu_to_le32(inode->i_ino); 333 de->ino = cpu_to_le32(inode->i_ino);
317 memcpy(dentry_blk->filename[0], ".", 1); 334 memcpy(d->filename[0], ".", 1);
318 set_de_type(de, inode); 335 set_de_type(de, inode);
319 336
320 de = &dentry_blk->dentry[1]; 337 de = &d->dentry[1];
321 de->hash_code = 0; 338 de->hash_code = 0;
322 de->name_len = cpu_to_le16(2); 339 de->name_len = cpu_to_le16(2);
323 de->ino = cpu_to_le32(parent->i_ino); 340 de->ino = cpu_to_le32(parent->i_ino);
324 memcpy(dentry_blk->filename[1], "..", 2); 341 memcpy(d->filename[1], "..", 2);
325 set_de_type(de, inode); 342 set_de_type(de, inode);
326 343
327 test_and_set_bit_le(0, &dentry_blk->dentry_bitmap); 344 test_and_set_bit_le(0, (void *)d->bitmap);
328 test_and_set_bit_le(1, &dentry_blk->dentry_bitmap); 345 test_and_set_bit_le(1, (void *)d->bitmap);
346}
347
348static int make_empty_dir(struct inode *inode,
349 struct inode *parent, struct page *page)
350{
351 struct page *dentry_page;
352 struct f2fs_dentry_block *dentry_blk;
353 struct f2fs_dentry_ptr d;
354
355 if (f2fs_has_inline_dentry(inode))
356 return make_empty_inline_dir(inode, parent, page);
357
358 dentry_page = get_new_data_page(inode, page, 0, true);
359 if (IS_ERR(dentry_page))
360 return PTR_ERR(dentry_page);
361
362 dentry_blk = kmap_atomic(dentry_page);
363
364 make_dentry_ptr(&d, (void *)dentry_blk, 1);
365 do_make_empty_dir(inode, parent, &d);
366
329 kunmap_atomic(dentry_blk); 367 kunmap_atomic(dentry_blk);
330 368
331 set_page_dirty(dentry_page); 369 set_page_dirty(dentry_page);
@@ -333,8 +371,8 @@ static int make_empty_dir(struct inode *inode,
333 return 0; 371 return 0;
334} 372}
335 373
336static struct page *init_inode_metadata(struct inode *inode, 374struct page *init_inode_metadata(struct inode *inode, struct inode *dir,
337 struct inode *dir, const struct qstr *name) 375 const struct qstr *name, struct page *dpage)
338{ 376{
339 struct page *page; 377 struct page *page;
340 int err; 378 int err;
@@ -350,7 +388,7 @@ static struct page *init_inode_metadata(struct inode *inode,
350 goto error; 388 goto error;
351 } 389 }
352 390
353 err = f2fs_init_acl(inode, dir, page); 391 err = f2fs_init_acl(inode, dir, page, dpage);
354 if (err) 392 if (err)
355 goto put_error; 393 goto put_error;
356 394
@@ -395,7 +433,7 @@ error:
395 return ERR_PTR(err); 433 return ERR_PTR(err);
396} 434}
397 435
398static void update_parent_metadata(struct inode *dir, struct inode *inode, 436void update_parent_metadata(struct inode *dir, struct inode *inode,
399 unsigned int current_depth) 437 unsigned int current_depth)
400{ 438{
401 if (is_inode_flag_set(F2FS_I(inode), FI_NEW_INODE)) { 439 if (is_inode_flag_set(F2FS_I(inode), FI_NEW_INODE)) {
@@ -417,27 +455,23 @@ static void update_parent_metadata(struct inode *dir, struct inode *inode,
417 clear_inode_flag(F2FS_I(inode), FI_INC_LINK); 455 clear_inode_flag(F2FS_I(inode), FI_INC_LINK);
418} 456}
419 457
420static int room_for_filename(struct f2fs_dentry_block *dentry_blk, int slots) 458int room_for_filename(const void *bitmap, int slots, int max_slots)
421{ 459{
422 int bit_start = 0; 460 int bit_start = 0;
423 int zero_start, zero_end; 461 int zero_start, zero_end;
424next: 462next:
425 zero_start = find_next_zero_bit_le(&dentry_blk->dentry_bitmap, 463 zero_start = find_next_zero_bit_le(bitmap, max_slots, bit_start);
426 NR_DENTRY_IN_BLOCK, 464 if (zero_start >= max_slots)
427 bit_start); 465 return max_slots;
428 if (zero_start >= NR_DENTRY_IN_BLOCK)
429 return NR_DENTRY_IN_BLOCK;
430 466
431 zero_end = find_next_bit_le(&dentry_blk->dentry_bitmap, 467 zero_end = find_next_bit_le(bitmap, max_slots, zero_start);
432 NR_DENTRY_IN_BLOCK,
433 zero_start);
434 if (zero_end - zero_start >= slots) 468 if (zero_end - zero_start >= slots)
435 return zero_start; 469 return zero_start;
436 470
437 bit_start = zero_end + 1; 471 bit_start = zero_end + 1;
438 472
439 if (zero_end + 1 >= NR_DENTRY_IN_BLOCK) 473 if (zero_end + 1 >= max_slots)
440 return NR_DENTRY_IN_BLOCK; 474 return max_slots;
441 goto next; 475 goto next;
442} 476}
443 477
@@ -463,6 +497,14 @@ int __f2fs_add_link(struct inode *dir, const struct qstr *name,
463 int err = 0; 497 int err = 0;
464 int i; 498 int i;
465 499
500 if (f2fs_has_inline_dentry(dir)) {
501 err = f2fs_add_inline_entry(dir, name, inode);
502 if (!err || err != -EAGAIN)
503 return err;
504 else
505 err = 0;
506 }
507
466 dentry_hash = f2fs_dentry_hash(name); 508 dentry_hash = f2fs_dentry_hash(name);
467 level = 0; 509 level = 0;
468 current_depth = F2FS_I(dir)->i_current_depth; 510 current_depth = F2FS_I(dir)->i_current_depth;
@@ -491,7 +533,8 @@ start:
491 return PTR_ERR(dentry_page); 533 return PTR_ERR(dentry_page);
492 534
493 dentry_blk = kmap(dentry_page); 535 dentry_blk = kmap(dentry_page);
494 bit_pos = room_for_filename(dentry_blk, slots); 536 bit_pos = room_for_filename(&dentry_blk->dentry_bitmap,
537 slots, NR_DENTRY_IN_BLOCK);
495 if (bit_pos < NR_DENTRY_IN_BLOCK) 538 if (bit_pos < NR_DENTRY_IN_BLOCK)
496 goto add_dentry; 539 goto add_dentry;
497 540
@@ -506,7 +549,7 @@ add_dentry:
506 f2fs_wait_on_page_writeback(dentry_page, DATA); 549 f2fs_wait_on_page_writeback(dentry_page, DATA);
507 550
508 down_write(&F2FS_I(inode)->i_sem); 551 down_write(&F2FS_I(inode)->i_sem);
509 page = init_inode_metadata(inode, dir, name); 552 page = init_inode_metadata(inode, dir, name, NULL);
510 if (IS_ERR(page)) { 553 if (IS_ERR(page)) {
511 err = PTR_ERR(page); 554 err = PTR_ERR(page);
512 goto fail; 555 goto fail;
@@ -545,7 +588,7 @@ int f2fs_do_tmpfile(struct inode *inode, struct inode *dir)
545 int err = 0; 588 int err = 0;
546 589
547 down_write(&F2FS_I(inode)->i_sem); 590 down_write(&F2FS_I(inode)->i_sem);
548 page = init_inode_metadata(inode, dir, NULL); 591 page = init_inode_metadata(inode, dir, NULL, NULL);
549 if (IS_ERR(page)) { 592 if (IS_ERR(page)) {
550 err = PTR_ERR(page); 593 err = PTR_ERR(page);
551 goto fail; 594 goto fail;
@@ -560,26 +603,57 @@ fail:
560 return err; 603 return err;
561} 604}
562 605
606void f2fs_drop_nlink(struct inode *dir, struct inode *inode, struct page *page)
607{
608 struct f2fs_sb_info *sbi = F2FS_I_SB(dir);
609
610 down_write(&F2FS_I(inode)->i_sem);
611
612 if (S_ISDIR(inode->i_mode)) {
613 drop_nlink(dir);
614 if (page)
615 update_inode(dir, page);
616 else
617 update_inode_page(dir);
618 }
619 inode->i_ctime = CURRENT_TIME;
620
621 drop_nlink(inode);
622 if (S_ISDIR(inode->i_mode)) {
623 drop_nlink(inode);
624 i_size_write(inode, 0);
625 }
626 up_write(&F2FS_I(inode)->i_sem);
627 update_inode_page(inode);
628
629 if (inode->i_nlink == 0)
630 add_orphan_inode(sbi, inode->i_ino);
631 else
632 release_orphan_inode(sbi);
633}
634
563/* 635/*
564 * It only removes the dentry from the dentry page, corresponding name 636 * It only removes the dentry from the dentry page, corresponding name
565 * entry in name page does not need to be touched during deletion. 637 * entry in name page does not need to be touched during deletion.
566 */ 638 */
567void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct page *page, 639void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct page *page,
568 struct inode *inode) 640 struct inode *dir, struct inode *inode)
569{ 641{
570 struct f2fs_dentry_block *dentry_blk; 642 struct f2fs_dentry_block *dentry_blk;
571 unsigned int bit_pos; 643 unsigned int bit_pos;
572 struct inode *dir = page->mapping->host;
573 int slots = GET_DENTRY_SLOTS(le16_to_cpu(dentry->name_len)); 644 int slots = GET_DENTRY_SLOTS(le16_to_cpu(dentry->name_len));
574 int i; 645 int i;
575 646
647 if (f2fs_has_inline_dentry(dir))
648 return f2fs_delete_inline_entry(dentry, page, dir, inode);
649
576 lock_page(page); 650 lock_page(page);
577 f2fs_wait_on_page_writeback(page, DATA); 651 f2fs_wait_on_page_writeback(page, DATA);
578 652
579 dentry_blk = page_address(page); 653 dentry_blk = page_address(page);
580 bit_pos = dentry - dentry_blk->dentry; 654 bit_pos = dentry - dentry_blk->dentry;
581 for (i = 0; i < slots; i++) 655 for (i = 0; i < slots; i++)
582 test_and_clear_bit_le(bit_pos + i, &dentry_blk->dentry_bitmap); 656 clear_bit_le(bit_pos + i, &dentry_blk->dentry_bitmap);
583 657
584 /* Let's check and deallocate this dentry page */ 658 /* Let's check and deallocate this dentry page */
585 bit_pos = find_next_bit_le(&dentry_blk->dentry_bitmap, 659 bit_pos = find_next_bit_le(&dentry_blk->dentry_bitmap,
@@ -590,29 +664,8 @@ void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct page *page,
590 664
591 dir->i_ctime = dir->i_mtime = CURRENT_TIME; 665 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
592 666
593 if (inode) { 667 if (inode)
594 struct f2fs_sb_info *sbi = F2FS_I_SB(dir); 668 f2fs_drop_nlink(dir, inode, NULL);
595
596 down_write(&F2FS_I(inode)->i_sem);
597
598 if (S_ISDIR(inode->i_mode)) {
599 drop_nlink(dir);
600 update_inode_page(dir);
601 }
602 inode->i_ctime = CURRENT_TIME;
603 drop_nlink(inode);
604 if (S_ISDIR(inode->i_mode)) {
605 drop_nlink(inode);
606 i_size_write(inode, 0);
607 }
608 up_write(&F2FS_I(inode)->i_sem);
609 update_inode_page(inode);
610
611 if (inode->i_nlink == 0)
612 add_orphan_inode(sbi, inode->i_ino);
613 else
614 release_orphan_inode(sbi);
615 }
616 669
617 if (bit_pos == NR_DENTRY_IN_BLOCK) { 670 if (bit_pos == NR_DENTRY_IN_BLOCK) {
618 truncate_hole(dir, page->index, page->index + 1); 671 truncate_hole(dir, page->index, page->index + 1);
@@ -628,9 +681,12 @@ bool f2fs_empty_dir(struct inode *dir)
628 unsigned long bidx; 681 unsigned long bidx;
629 struct page *dentry_page; 682 struct page *dentry_page;
630 unsigned int bit_pos; 683 unsigned int bit_pos;
631 struct f2fs_dentry_block *dentry_blk; 684 struct f2fs_dentry_block *dentry_blk;
632 unsigned long nblock = dir_blocks(dir); 685 unsigned long nblock = dir_blocks(dir);
633 686
687 if (f2fs_has_inline_dentry(dir))
688 return f2fs_empty_inline_dir(dir);
689
634 for (bidx = 0; bidx < nblock; bidx++) { 690 for (bidx = 0; bidx < nblock; bidx++) {
635 dentry_page = get_lock_data_page(dir, bidx); 691 dentry_page = get_lock_data_page(dir, bidx);
636 if (IS_ERR(dentry_page)) { 692 if (IS_ERR(dentry_page)) {
@@ -640,7 +696,6 @@ bool f2fs_empty_dir(struct inode *dir)
640 return false; 696 return false;
641 } 697 }
642 698
643
644 dentry_blk = kmap_atomic(dentry_page); 699 dentry_blk = kmap_atomic(dentry_page);
645 if (bidx == 0) 700 if (bidx == 0)
646 bit_pos = 2; 701 bit_pos = 2;
@@ -659,19 +714,48 @@ bool f2fs_empty_dir(struct inode *dir)
659 return true; 714 return true;
660} 715}
661 716
717bool f2fs_fill_dentries(struct dir_context *ctx, struct f2fs_dentry_ptr *d,
718 unsigned int start_pos)
719{
720 unsigned char d_type = DT_UNKNOWN;
721 unsigned int bit_pos;
722 struct f2fs_dir_entry *de = NULL;
723
724 bit_pos = ((unsigned long)ctx->pos % d->max);
725
726 while (bit_pos < d->max) {
727 bit_pos = find_next_bit_le(d->bitmap, d->max, bit_pos);
728 if (bit_pos >= d->max)
729 break;
730
731 de = &d->dentry[bit_pos];
732 if (de->file_type < F2FS_FT_MAX)
733 d_type = f2fs_filetype_table[de->file_type];
734 else
735 d_type = DT_UNKNOWN;
736 if (!dir_emit(ctx, d->filename[bit_pos],
737 le16_to_cpu(de->name_len),
738 le32_to_cpu(de->ino), d_type))
739 return true;
740
741 bit_pos += GET_DENTRY_SLOTS(le16_to_cpu(de->name_len));
742 ctx->pos = start_pos + bit_pos;
743 }
744 return false;
745}
746
662static int f2fs_readdir(struct file *file, struct dir_context *ctx) 747static int f2fs_readdir(struct file *file, struct dir_context *ctx)
663{ 748{
664 struct inode *inode = file_inode(file); 749 struct inode *inode = file_inode(file);
665 unsigned long npages = dir_blocks(inode); 750 unsigned long npages = dir_blocks(inode);
666 unsigned int bit_pos = 0;
667 struct f2fs_dentry_block *dentry_blk = NULL; 751 struct f2fs_dentry_block *dentry_blk = NULL;
668 struct f2fs_dir_entry *de = NULL;
669 struct page *dentry_page = NULL; 752 struct page *dentry_page = NULL;
670 struct file_ra_state *ra = &file->f_ra; 753 struct file_ra_state *ra = &file->f_ra;
671 unsigned int n = ((unsigned long)ctx->pos / NR_DENTRY_IN_BLOCK); 754 unsigned int n = ((unsigned long)ctx->pos / NR_DENTRY_IN_BLOCK);
672 unsigned char d_type = DT_UNKNOWN; 755 struct f2fs_dentry_ptr d;
673 756
674 bit_pos = ((unsigned long)ctx->pos % NR_DENTRY_IN_BLOCK); 757 if (f2fs_has_inline_dentry(inode))
758 return f2fs_read_inline_dir(file, ctx);
675 759
676 /* readahead for multi pages of dir */ 760 /* readahead for multi pages of dir */
677 if (npages - n > 1 && !ra_has_index(ra, n)) 761 if (npages - n > 1 && !ra_has_index(ra, n))
@@ -684,28 +768,12 @@ static int f2fs_readdir(struct file *file, struct dir_context *ctx)
684 continue; 768 continue;
685 769
686 dentry_blk = kmap(dentry_page); 770 dentry_blk = kmap(dentry_page);
687 while (bit_pos < NR_DENTRY_IN_BLOCK) {
688 bit_pos = find_next_bit_le(&dentry_blk->dentry_bitmap,
689 NR_DENTRY_IN_BLOCK,
690 bit_pos);
691 if (bit_pos >= NR_DENTRY_IN_BLOCK)
692 break;
693
694 de = &dentry_blk->dentry[bit_pos];
695 if (de->file_type < F2FS_FT_MAX)
696 d_type = f2fs_filetype_table[de->file_type];
697 else
698 d_type = DT_UNKNOWN;
699 if (!dir_emit(ctx,
700 dentry_blk->filename[bit_pos],
701 le16_to_cpu(de->name_len),
702 le32_to_cpu(de->ino), d_type))
703 goto stop;
704 771
705 bit_pos += GET_DENTRY_SLOTS(le16_to_cpu(de->name_len)); 772 make_dentry_ptr(&d, (void *)dentry_blk, 1);
706 ctx->pos = n * NR_DENTRY_IN_BLOCK + bit_pos; 773
707 } 774 if (f2fs_fill_dentries(ctx, &d, n * NR_DENTRY_IN_BLOCK))
708 bit_pos = 0; 775 goto stop;
776
709 ctx->pos = (n + 1) * NR_DENTRY_IN_BLOCK; 777 ctx->pos = (n + 1) * NR_DENTRY_IN_BLOCK;
710 kunmap(dentry_page); 778 kunmap(dentry_page);
711 f2fs_put_page(dentry_page, 1); 779 f2fs_put_page(dentry_page, 1);
diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
index 8171e80b2ee9..ec58bb2373fc 100644
--- a/fs/f2fs/f2fs.h
+++ b/fs/f2fs/f2fs.h
@@ -46,8 +46,10 @@
46#define F2FS_MOUNT_DISABLE_EXT_IDENTIFY 0x00000040 46#define F2FS_MOUNT_DISABLE_EXT_IDENTIFY 0x00000040
47#define F2FS_MOUNT_INLINE_XATTR 0x00000080 47#define F2FS_MOUNT_INLINE_XATTR 0x00000080
48#define F2FS_MOUNT_INLINE_DATA 0x00000100 48#define F2FS_MOUNT_INLINE_DATA 0x00000100
49#define F2FS_MOUNT_FLUSH_MERGE 0x00000200 49#define F2FS_MOUNT_INLINE_DENTRY 0x00000200
50#define F2FS_MOUNT_NOBARRIER 0x00000400 50#define F2FS_MOUNT_FLUSH_MERGE 0x00000400
51#define F2FS_MOUNT_NOBARRIER 0x00000800
52#define F2FS_MOUNT_FASTBOOT 0x00001000
51 53
52#define clear_opt(sbi, option) (sbi->mount_opt.opt &= ~F2FS_MOUNT_##option) 54#define clear_opt(sbi, option) (sbi->mount_opt.opt &= ~F2FS_MOUNT_##option)
53#define set_opt(sbi, option) (sbi->mount_opt.opt |= F2FS_MOUNT_##option) 55#define set_opt(sbi, option) (sbi->mount_opt.opt |= F2FS_MOUNT_##option)
@@ -211,6 +213,32 @@ static inline bool __has_cursum_space(struct f2fs_summary_block *sum, int size,
211/* 213/*
212 * For INODE and NODE manager 214 * For INODE and NODE manager
213 */ 215 */
216/* for directory operations */
217struct f2fs_dentry_ptr {
218 const void *bitmap;
219 struct f2fs_dir_entry *dentry;
220 __u8 (*filename)[F2FS_SLOT_LEN];
221 int max;
222};
223
224static inline void make_dentry_ptr(struct f2fs_dentry_ptr *d,
225 void *src, int type)
226{
227 if (type == 1) {
228 struct f2fs_dentry_block *t = (struct f2fs_dentry_block *)src;
229 d->max = NR_DENTRY_IN_BLOCK;
230 d->bitmap = &t->dentry_bitmap;
231 d->dentry = t->dentry;
232 d->filename = t->filename;
233 } else {
234 struct f2fs_inline_dentry *t = (struct f2fs_inline_dentry *)src;
235 d->max = NR_INLINE_DENTRY;
236 d->bitmap = &t->dentry_bitmap;
237 d->dentry = t->dentry;
238 d->filename = t->filename;
239 }
240}
241
214/* 242/*
215 * XATTR_NODE_OFFSET stores xattrs to one node block per file keeping -1 243 * XATTR_NODE_OFFSET stores xattrs to one node block per file keeping -1
216 * as its node offset to distinguish from index node blocks. 244 * as its node offset to distinguish from index node blocks.
@@ -269,6 +297,7 @@ struct f2fs_inode_info {
269 struct extent_info ext; /* in-memory extent cache entry */ 297 struct extent_info ext; /* in-memory extent cache entry */
270 struct dir_inode_entry *dirty_dir; /* the pointer of dirty dir */ 298 struct dir_inode_entry *dirty_dir; /* the pointer of dirty dir */
271 299
300 struct radix_tree_root inmem_root; /* radix tree for inmem pages */
272 struct list_head inmem_pages; /* inmemory pages managed by f2fs */ 301 struct list_head inmem_pages; /* inmemory pages managed by f2fs */
273 struct mutex inmem_lock; /* lock for inmemory pages */ 302 struct mutex inmem_lock; /* lock for inmemory pages */
274}; 303};
@@ -303,7 +332,7 @@ struct f2fs_nm_info {
303 /* NAT cache management */ 332 /* NAT cache management */
304 struct radix_tree_root nat_root;/* root of the nat entry cache */ 333 struct radix_tree_root nat_root;/* root of the nat entry cache */
305 struct radix_tree_root nat_set_root;/* root of the nat set cache */ 334 struct radix_tree_root nat_set_root;/* root of the nat set cache */
306 rwlock_t nat_tree_lock; /* protect nat_tree_lock */ 335 struct rw_semaphore nat_tree_lock; /* protect nat_tree_lock */
307 struct list_head nat_entries; /* cached nat entry list (clean) */ 336 struct list_head nat_entries; /* cached nat entry list (clean) */
308 unsigned int nat_cnt; /* the # of cached nat entries */ 337 unsigned int nat_cnt; /* the # of cached nat entries */
309 unsigned int dirty_nat_cnt; /* total num of nat entries in set */ 338 unsigned int dirty_nat_cnt; /* total num of nat entries in set */
@@ -433,6 +462,7 @@ enum count_type {
433 F2FS_DIRTY_DENTS, 462 F2FS_DIRTY_DENTS,
434 F2FS_DIRTY_NODES, 463 F2FS_DIRTY_NODES,
435 F2FS_DIRTY_META, 464 F2FS_DIRTY_META,
465 F2FS_INMEM_PAGES,
436 NR_COUNT_TYPE, 466 NR_COUNT_TYPE,
437}; 467};
438 468
@@ -470,6 +500,14 @@ struct f2fs_bio_info {
470 struct rw_semaphore io_rwsem; /* blocking op for bio */ 500 struct rw_semaphore io_rwsem; /* blocking op for bio */
471}; 501};
472 502
503/* for inner inode cache management */
504struct inode_management {
505 struct radix_tree_root ino_root; /* ino entry array */
506 spinlock_t ino_lock; /* for ino entry lock */
507 struct list_head ino_list; /* inode list head */
508 unsigned long ino_num; /* number of entries */
509};
510
473struct f2fs_sb_info { 511struct f2fs_sb_info {
474 struct super_block *sb; /* pointer to VFS super block */ 512 struct super_block *sb; /* pointer to VFS super block */
475 struct proc_dir_entry *s_proc; /* proc entry */ 513 struct proc_dir_entry *s_proc; /* proc entry */
@@ -488,7 +526,6 @@ struct f2fs_sb_info {
488 /* for bio operations */ 526 /* for bio operations */
489 struct f2fs_bio_info read_io; /* for read bios */ 527 struct f2fs_bio_info read_io; /* for read bios */
490 struct f2fs_bio_info write_io[NR_PAGE_TYPE]; /* for write bios */ 528 struct f2fs_bio_info write_io[NR_PAGE_TYPE]; /* for write bios */
491 struct completion *wait_io; /* for completion bios */
492 529
493 /* for checkpoint */ 530 /* for checkpoint */
494 struct f2fs_checkpoint *ckpt; /* raw checkpoint pointer */ 531 struct f2fs_checkpoint *ckpt; /* raw checkpoint pointer */
@@ -500,13 +537,9 @@ struct f2fs_sb_info {
500 bool por_doing; /* recovery is doing or not */ 537 bool por_doing; /* recovery is doing or not */
501 wait_queue_head_t cp_wait; 538 wait_queue_head_t cp_wait;
502 539
503 /* for inode management */ 540 struct inode_management im[MAX_INO_ENTRY]; /* manage inode cache */
504 struct radix_tree_root ino_root[MAX_INO_ENTRY]; /* ino entry array */
505 spinlock_t ino_lock[MAX_INO_ENTRY]; /* for ino entry lock */
506 struct list_head ino_list[MAX_INO_ENTRY]; /* inode list head */
507 541
508 /* for orphan inode, use 0'th array */ 542 /* for orphan inode, use 0'th array */
509 unsigned int n_orphans; /* # of orphan inodes */
510 unsigned int max_orphans; /* max orphan inodes */ 543 unsigned int max_orphans; /* max orphan inodes */
511 544
512 /* for directory inode management */ 545 /* for directory inode management */
@@ -557,7 +590,8 @@ struct f2fs_sb_info {
557 unsigned int segment_count[2]; /* # of allocated segments */ 590 unsigned int segment_count[2]; /* # of allocated segments */
558 unsigned int block_count[2]; /* # of allocated blocks */ 591 unsigned int block_count[2]; /* # of allocated blocks */
559 int total_hit_ext, read_hit_ext; /* extent cache hit ratio */ 592 int total_hit_ext, read_hit_ext; /* extent cache hit ratio */
560 int inline_inode; /* # of inline_data inodes */ 593 atomic_t inline_inode; /* # of inline_data inodes */
594 atomic_t inline_dir; /* # of inline_dentry inodes */
561 int bg_gc; /* background gc calls */ 595 int bg_gc; /* background gc calls */
562 unsigned int n_dirty_dirs; /* # of dir inodes */ 596 unsigned int n_dirty_dirs; /* # of dir inodes */
563#endif 597#endif
@@ -988,6 +1022,13 @@ retry:
988 return entry; 1022 return entry;
989} 1023}
990 1024
1025static inline void f2fs_radix_tree_insert(struct radix_tree_root *root,
1026 unsigned long index, void *item)
1027{
1028 while (radix_tree_insert(root, index, item))
1029 cond_resched();
1030}
1031
991#define RAW_IS_INODE(p) ((p)->footer.nid == (p)->footer.ino) 1032#define RAW_IS_INODE(p) ((p)->footer.nid == (p)->footer.ino)
992 1033
993static inline bool IS_INODE(struct page *page) 1034static inline bool IS_INODE(struct page *page)
@@ -1020,7 +1061,7 @@ static inline int f2fs_test_bit(unsigned int nr, char *addr)
1020 return mask & *addr; 1061 return mask & *addr;
1021} 1062}
1022 1063
1023static inline int f2fs_set_bit(unsigned int nr, char *addr) 1064static inline int f2fs_test_and_set_bit(unsigned int nr, char *addr)
1024{ 1065{
1025 int mask; 1066 int mask;
1026 int ret; 1067 int ret;
@@ -1032,7 +1073,7 @@ static inline int f2fs_set_bit(unsigned int nr, char *addr)
1032 return ret; 1073 return ret;
1033} 1074}
1034 1075
1035static inline int f2fs_clear_bit(unsigned int nr, char *addr) 1076static inline int f2fs_test_and_clear_bit(unsigned int nr, char *addr)
1036{ 1077{
1037 int mask; 1078 int mask;
1038 int ret; 1079 int ret;
@@ -1044,6 +1085,15 @@ static inline int f2fs_clear_bit(unsigned int nr, char *addr)
1044 return ret; 1085 return ret;
1045} 1086}
1046 1087
1088static inline void f2fs_change_bit(unsigned int nr, char *addr)
1089{
1090 int mask;
1091
1092 addr += (nr >> 3);
1093 mask = 1 << (7 - (nr & 0x07));
1094 *addr ^= mask;
1095}
1096
1047/* used for f2fs_inode_info->flags */ 1097/* used for f2fs_inode_info->flags */
1048enum { 1098enum {
1049 FI_NEW_INODE, /* indicate newly allocated inode */ 1099 FI_NEW_INODE, /* indicate newly allocated inode */
@@ -1057,11 +1107,13 @@ enum {
1057 FI_NO_EXTENT, /* not to use the extent cache */ 1107 FI_NO_EXTENT, /* not to use the extent cache */
1058 FI_INLINE_XATTR, /* used for inline xattr */ 1108 FI_INLINE_XATTR, /* used for inline xattr */
1059 FI_INLINE_DATA, /* used for inline data*/ 1109 FI_INLINE_DATA, /* used for inline data*/
1110 FI_INLINE_DENTRY, /* used for inline dentry */
1060 FI_APPEND_WRITE, /* inode has appended data */ 1111 FI_APPEND_WRITE, /* inode has appended data */
1061 FI_UPDATE_WRITE, /* inode has in-place-update data */ 1112 FI_UPDATE_WRITE, /* inode has in-place-update data */
1062 FI_NEED_IPU, /* used for ipu per file */ 1113 FI_NEED_IPU, /* used for ipu per file */
1063 FI_ATOMIC_FILE, /* indicate atomic file */ 1114 FI_ATOMIC_FILE, /* indicate atomic file */
1064 FI_VOLATILE_FILE, /* indicate volatile file */ 1115 FI_VOLATILE_FILE, /* indicate volatile file */
1116 FI_DATA_EXIST, /* indicate data exists */
1065}; 1117};
1066 1118
1067static inline void set_inode_flag(struct f2fs_inode_info *fi, int flag) 1119static inline void set_inode_flag(struct f2fs_inode_info *fi, int flag)
@@ -1087,15 +1139,6 @@ static inline void set_acl_inode(struct f2fs_inode_info *fi, umode_t mode)
1087 set_inode_flag(fi, FI_ACL_MODE); 1139 set_inode_flag(fi, FI_ACL_MODE);
1088} 1140}
1089 1141
1090static inline int cond_clear_inode_flag(struct f2fs_inode_info *fi, int flag)
1091{
1092 if (is_inode_flag_set(fi, FI_ACL_MODE)) {
1093 clear_inode_flag(fi, FI_ACL_MODE);
1094 return 1;
1095 }
1096 return 0;
1097}
1098
1099static inline void get_inline_info(struct f2fs_inode_info *fi, 1142static inline void get_inline_info(struct f2fs_inode_info *fi,
1100 struct f2fs_inode *ri) 1143 struct f2fs_inode *ri)
1101{ 1144{
@@ -1103,6 +1146,10 @@ static inline void get_inline_info(struct f2fs_inode_info *fi,
1103 set_inode_flag(fi, FI_INLINE_XATTR); 1146 set_inode_flag(fi, FI_INLINE_XATTR);
1104 if (ri->i_inline & F2FS_INLINE_DATA) 1147 if (ri->i_inline & F2FS_INLINE_DATA)
1105 set_inode_flag(fi, FI_INLINE_DATA); 1148 set_inode_flag(fi, FI_INLINE_DATA);
1149 if (ri->i_inline & F2FS_INLINE_DENTRY)
1150 set_inode_flag(fi, FI_INLINE_DENTRY);
1151 if (ri->i_inline & F2FS_DATA_EXIST)
1152 set_inode_flag(fi, FI_DATA_EXIST);
1106} 1153}
1107 1154
1108static inline void set_raw_inline(struct f2fs_inode_info *fi, 1155static inline void set_raw_inline(struct f2fs_inode_info *fi,
@@ -1114,6 +1161,10 @@ static inline void set_raw_inline(struct f2fs_inode_info *fi,
1114 ri->i_inline |= F2FS_INLINE_XATTR; 1161 ri->i_inline |= F2FS_INLINE_XATTR;
1115 if (is_inode_flag_set(fi, FI_INLINE_DATA)) 1162 if (is_inode_flag_set(fi, FI_INLINE_DATA))
1116 ri->i_inline |= F2FS_INLINE_DATA; 1163 ri->i_inline |= F2FS_INLINE_DATA;
1164 if (is_inode_flag_set(fi, FI_INLINE_DENTRY))
1165 ri->i_inline |= F2FS_INLINE_DENTRY;
1166 if (is_inode_flag_set(fi, FI_DATA_EXIST))
1167 ri->i_inline |= F2FS_DATA_EXIST;
1117} 1168}
1118 1169
1119static inline int f2fs_has_inline_xattr(struct inode *inode) 1170static inline int f2fs_has_inline_xattr(struct inode *inode)
@@ -1148,6 +1199,17 @@ static inline int f2fs_has_inline_data(struct inode *inode)
1148 return is_inode_flag_set(F2FS_I(inode), FI_INLINE_DATA); 1199 return is_inode_flag_set(F2FS_I(inode), FI_INLINE_DATA);
1149} 1200}
1150 1201
1202static inline void f2fs_clear_inline_inode(struct inode *inode)
1203{
1204 clear_inode_flag(F2FS_I(inode), FI_INLINE_DATA);
1205 clear_inode_flag(F2FS_I(inode), FI_DATA_EXIST);
1206}
1207
1208static inline int f2fs_exist_data(struct inode *inode)
1209{
1210 return is_inode_flag_set(F2FS_I(inode), FI_DATA_EXIST);
1211}
1212
1151static inline bool f2fs_is_atomic_file(struct inode *inode) 1213static inline bool f2fs_is_atomic_file(struct inode *inode)
1152{ 1214{
1153 return is_inode_flag_set(F2FS_I(inode), FI_ATOMIC_FILE); 1215 return is_inode_flag_set(F2FS_I(inode), FI_ATOMIC_FILE);
@@ -1164,6 +1226,23 @@ static inline void *inline_data_addr(struct page *page)
1164 return (void *)&(ri->i_addr[1]); 1226 return (void *)&(ri->i_addr[1]);
1165} 1227}
1166 1228
1229static inline int f2fs_has_inline_dentry(struct inode *inode)
1230{
1231 return is_inode_flag_set(F2FS_I(inode), FI_INLINE_DENTRY);
1232}
1233
1234static inline void *inline_dentry_addr(struct page *page)
1235{
1236 struct f2fs_inode *ri = F2FS_INODE(page);
1237 return (void *)&(ri->i_addr[1]);
1238}
1239
1240static inline void f2fs_dentry_kunmap(struct inode *dir, struct page *page)
1241{
1242 if (!f2fs_has_inline_dentry(dir))
1243 kunmap(page);
1244}
1245
1167static inline int f2fs_readonly(struct super_block *sb) 1246static inline int f2fs_readonly(struct super_block *sb)
1168{ 1247{
1169 return sb->s_flags & MS_RDONLY; 1248 return sb->s_flags & MS_RDONLY;
@@ -1224,6 +1303,19 @@ struct dentry *f2fs_get_parent(struct dentry *child);
1224/* 1303/*
1225 * dir.c 1304 * dir.c
1226 */ 1305 */
1306extern unsigned char f2fs_filetype_table[F2FS_FT_MAX];
1307void set_de_type(struct f2fs_dir_entry *, struct inode *);
1308struct f2fs_dir_entry *find_target_dentry(struct qstr *, int *,
1309 struct f2fs_dentry_ptr *);
1310bool f2fs_fill_dentries(struct dir_context *, struct f2fs_dentry_ptr *,
1311 unsigned int);
1312void do_make_empty_dir(struct inode *, struct inode *,
1313 struct f2fs_dentry_ptr *);
1314struct page *init_inode_metadata(struct inode *, struct inode *,
1315 const struct qstr *, struct page *);
1316void update_parent_metadata(struct inode *, struct inode *, unsigned int);
1317int room_for_filename(const void *, int, int);
1318void f2fs_drop_nlink(struct inode *, struct inode *, struct page *);
1227struct f2fs_dir_entry *f2fs_find_entry(struct inode *, struct qstr *, 1319struct f2fs_dir_entry *f2fs_find_entry(struct inode *, struct qstr *,
1228 struct page **); 1320 struct page **);
1229struct f2fs_dir_entry *f2fs_parent_dir(struct inode *, struct page **); 1321struct f2fs_dir_entry *f2fs_parent_dir(struct inode *, struct page **);
@@ -1232,7 +1324,8 @@ void f2fs_set_link(struct inode *, struct f2fs_dir_entry *,
1232 struct page *, struct inode *); 1324 struct page *, struct inode *);
1233int update_dent_inode(struct inode *, const struct qstr *); 1325int update_dent_inode(struct inode *, const struct qstr *);
1234int __f2fs_add_link(struct inode *, const struct qstr *, struct inode *); 1326int __f2fs_add_link(struct inode *, const struct qstr *, struct inode *);
1235void f2fs_delete_entry(struct f2fs_dir_entry *, struct page *, struct inode *); 1327void f2fs_delete_entry(struct f2fs_dir_entry *, struct page *, struct inode *,
1328 struct inode *);
1236int f2fs_do_tmpfile(struct inode *, struct inode *); 1329int f2fs_do_tmpfile(struct inode *, struct inode *);
1237int f2fs_make_empty(struct inode *, struct inode *); 1330int f2fs_make_empty(struct inode *, struct inode *);
1238bool f2fs_empty_dir(struct inode *); 1331bool f2fs_empty_dir(struct inode *);
@@ -1296,6 +1389,7 @@ void destroy_node_manager_caches(void);
1296 * segment.c 1389 * segment.c
1297 */ 1390 */
1298void register_inmem_page(struct inode *, struct page *); 1391void register_inmem_page(struct inode *, struct page *);
1392void invalidate_inmem_page(struct inode *, struct page *);
1299void commit_inmem_pages(struct inode *, bool); 1393void commit_inmem_pages(struct inode *, bool);
1300void f2fs_balance_fs(struct f2fs_sb_info *); 1394void f2fs_balance_fs(struct f2fs_sb_info *);
1301void f2fs_balance_fs_bg(struct f2fs_sb_info *); 1395void f2fs_balance_fs_bg(struct f2fs_sb_info *);
@@ -1337,8 +1431,8 @@ void destroy_segment_manager_caches(void);
1337 */ 1431 */
1338struct page *grab_meta_page(struct f2fs_sb_info *, pgoff_t); 1432struct page *grab_meta_page(struct f2fs_sb_info *, pgoff_t);
1339struct page *get_meta_page(struct f2fs_sb_info *, pgoff_t); 1433struct page *get_meta_page(struct f2fs_sb_info *, pgoff_t);
1340struct page *get_meta_page_ra(struct f2fs_sb_info *, pgoff_t);
1341int ra_meta_pages(struct f2fs_sb_info *, block_t, int, int); 1434int ra_meta_pages(struct f2fs_sb_info *, block_t, int, int);
1435void ra_meta_pages_cond(struct f2fs_sb_info *, pgoff_t);
1342long sync_meta_pages(struct f2fs_sb_info *, enum page_type, long); 1436long sync_meta_pages(struct f2fs_sb_info *, enum page_type, long);
1343void add_dirty_inode(struct f2fs_sb_info *, nid_t, int type); 1437void add_dirty_inode(struct f2fs_sb_info *, nid_t, int type);
1344void remove_dirty_inode(struct f2fs_sb_info *, nid_t, int type); 1438void remove_dirty_inode(struct f2fs_sb_info *, nid_t, int type);
@@ -1405,7 +1499,7 @@ struct f2fs_stat_info {
1405 int ndirty_node, ndirty_dent, ndirty_dirs, ndirty_meta; 1499 int ndirty_node, ndirty_dent, ndirty_dirs, ndirty_meta;
1406 int nats, sits, fnids; 1500 int nats, sits, fnids;
1407 int total_count, utilization; 1501 int total_count, utilization;
1408 int bg_gc, inline_inode; 1502 int bg_gc, inline_inode, inline_dir, inmem_pages;
1409 unsigned int valid_count, valid_node_count, valid_inode_count; 1503 unsigned int valid_count, valid_node_count, valid_inode_count;
1410 unsigned int bimodal, avg_vblocks; 1504 unsigned int bimodal, avg_vblocks;
1411 int util_free, util_valid, util_invalid; 1505 int util_free, util_valid, util_invalid;
@@ -1438,14 +1532,23 @@ static inline struct f2fs_stat_info *F2FS_STAT(struct f2fs_sb_info *sbi)
1438#define stat_inc_inline_inode(inode) \ 1532#define stat_inc_inline_inode(inode) \
1439 do { \ 1533 do { \
1440 if (f2fs_has_inline_data(inode)) \ 1534 if (f2fs_has_inline_data(inode)) \
1441 ((F2FS_I_SB(inode))->inline_inode++); \ 1535 (atomic_inc(&F2FS_I_SB(inode)->inline_inode)); \
1442 } while (0) 1536 } while (0)
1443#define stat_dec_inline_inode(inode) \ 1537#define stat_dec_inline_inode(inode) \
1444 do { \ 1538 do { \
1445 if (f2fs_has_inline_data(inode)) \ 1539 if (f2fs_has_inline_data(inode)) \
1446 ((F2FS_I_SB(inode))->inline_inode--); \ 1540 (atomic_dec(&F2FS_I_SB(inode)->inline_inode)); \
1541 } while (0)
1542#define stat_inc_inline_dir(inode) \
1543 do { \
1544 if (f2fs_has_inline_dentry(inode)) \
1545 (atomic_inc(&F2FS_I_SB(inode)->inline_dir)); \
1546 } while (0)
1547#define stat_dec_inline_dir(inode) \
1548 do { \
1549 if (f2fs_has_inline_dentry(inode)) \
1550 (atomic_dec(&F2FS_I_SB(inode)->inline_dir)); \
1447 } while (0) 1551 } while (0)
1448
1449#define stat_inc_seg_type(sbi, curseg) \ 1552#define stat_inc_seg_type(sbi, curseg) \
1450 ((sbi)->segment_count[(curseg)->alloc_type]++) 1553 ((sbi)->segment_count[(curseg)->alloc_type]++)
1451#define stat_inc_block_count(sbi, curseg) \ 1554#define stat_inc_block_count(sbi, curseg) \
@@ -1492,6 +1595,8 @@ void f2fs_destroy_root_stats(void);
1492#define stat_inc_read_hit(sb) 1595#define stat_inc_read_hit(sb)
1493#define stat_inc_inline_inode(inode) 1596#define stat_inc_inline_inode(inode)
1494#define stat_dec_inline_inode(inode) 1597#define stat_dec_inline_inode(inode)
1598#define stat_inc_inline_dir(inode)
1599#define stat_dec_inline_dir(inode)
1495#define stat_inc_seg_type(sbi, curseg) 1600#define stat_inc_seg_type(sbi, curseg)
1496#define stat_inc_block_count(sbi, curseg) 1601#define stat_inc_block_count(sbi, curseg)
1497#define stat_inc_seg_count(si, type) 1602#define stat_inc_seg_count(si, type)
@@ -1519,9 +1624,20 @@ extern const struct inode_operations f2fs_special_inode_operations;
1519 * inline.c 1624 * inline.c
1520 */ 1625 */
1521bool f2fs_may_inline(struct inode *); 1626bool f2fs_may_inline(struct inode *);
1627void read_inline_data(struct page *, struct page *);
1522int f2fs_read_inline_data(struct inode *, struct page *); 1628int f2fs_read_inline_data(struct inode *, struct page *);
1523int f2fs_convert_inline_data(struct inode *, pgoff_t, struct page *); 1629int f2fs_convert_inline_page(struct dnode_of_data *, struct page *);
1524int f2fs_write_inline_data(struct inode *, struct page *, unsigned int); 1630int f2fs_convert_inline_inode(struct inode *);
1525void truncate_inline_data(struct inode *, u64); 1631int f2fs_write_inline_data(struct inode *, struct page *);
1632void truncate_inline_data(struct page *, u64);
1526bool recover_inline_data(struct inode *, struct page *); 1633bool recover_inline_data(struct inode *, struct page *);
1634struct f2fs_dir_entry *find_in_inline_dir(struct inode *, struct qstr *,
1635 struct page **);
1636struct f2fs_dir_entry *f2fs_parent_inline_dir(struct inode *, struct page **);
1637int make_empty_inline_dir(struct inode *inode, struct inode *, struct page *);
1638int f2fs_add_inline_entry(struct inode *, const struct qstr *, struct inode *);
1639void f2fs_delete_inline_entry(struct f2fs_dir_entry *, struct page *,
1640 struct inode *, struct inode *);
1641bool f2fs_empty_inline_dir(struct inode *);
1642int f2fs_read_inline_dir(struct file *, struct dir_context *);
1527#endif 1643#endif
diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c
index 8e68bb64f835..3c27e0ecb3bc 100644
--- a/fs/f2fs/file.c
+++ b/fs/f2fs/file.c
@@ -41,18 +41,18 @@ static int f2fs_vm_page_mkwrite(struct vm_area_struct *vma,
41 41
42 sb_start_pagefault(inode->i_sb); 42 sb_start_pagefault(inode->i_sb);
43 43
44 /* force to convert with normal data indices */ 44 f2fs_bug_on(sbi, f2fs_has_inline_data(inode));
45 err = f2fs_convert_inline_data(inode, MAX_INLINE_DATA + 1, page);
46 if (err)
47 goto out;
48 45
49 /* block allocation */ 46 /* block allocation */
50 f2fs_lock_op(sbi); 47 f2fs_lock_op(sbi);
51 set_new_dnode(&dn, inode, NULL, NULL, 0); 48 set_new_dnode(&dn, inode, NULL, NULL, 0);
52 err = f2fs_reserve_block(&dn, page->index); 49 err = f2fs_reserve_block(&dn, page->index);
53 f2fs_unlock_op(sbi); 50 if (err) {
54 if (err) 51 f2fs_unlock_op(sbi);
55 goto out; 52 goto out;
53 }
54 f2fs_put_dnode(&dn);
55 f2fs_unlock_op(sbi);
56 56
57 file_update_time(vma->vm_file); 57 file_update_time(vma->vm_file);
58 lock_page(page); 58 lock_page(page);
@@ -130,10 +130,45 @@ static inline bool need_do_checkpoint(struct inode *inode)
130 need_cp = true; 130 need_cp = true;
131 else if (F2FS_I(inode)->xattr_ver == cur_cp_version(F2FS_CKPT(sbi))) 131 else if (F2FS_I(inode)->xattr_ver == cur_cp_version(F2FS_CKPT(sbi)))
132 need_cp = true; 132 need_cp = true;
133 else if (test_opt(sbi, FASTBOOT))
134 need_cp = true;
135 else if (sbi->active_logs == 2)
136 need_cp = true;
133 137
134 return need_cp; 138 return need_cp;
135} 139}
136 140
141static bool need_inode_page_update(struct f2fs_sb_info *sbi, nid_t ino)
142{
143 struct page *i = find_get_page(NODE_MAPPING(sbi), ino);
144 bool ret = false;
145 /* But we need to avoid that there are some inode updates */
146 if ((i && PageDirty(i)) || need_inode_block_update(sbi, ino))
147 ret = true;
148 f2fs_put_page(i, 0);
149 return ret;
150}
151
152static void try_to_fix_pino(struct inode *inode)
153{
154 struct f2fs_inode_info *fi = F2FS_I(inode);
155 nid_t pino;
156
157 down_write(&fi->i_sem);
158 fi->xattr_ver = 0;
159 if (file_wrong_pino(inode) && inode->i_nlink == 1 &&
160 get_parent_ino(inode, &pino)) {
161 fi->i_pino = pino;
162 file_got_pino(inode);
163 up_write(&fi->i_sem);
164
165 mark_inode_dirty_sync(inode);
166 f2fs_write_inode(inode, NULL);
167 } else {
168 up_write(&fi->i_sem);
169 }
170}
171
137int f2fs_sync_file(struct file *file, loff_t start, loff_t end, int datasync) 172int f2fs_sync_file(struct file *file, loff_t start, loff_t end, int datasync)
138{ 173{
139 struct inode *inode = file->f_mapping->host; 174 struct inode *inode = file->f_mapping->host;
@@ -164,19 +199,21 @@ int f2fs_sync_file(struct file *file, loff_t start, loff_t end, int datasync)
164 return ret; 199 return ret;
165 } 200 }
166 201
202 /* if the inode is dirty, let's recover all the time */
203 if (!datasync && is_inode_flag_set(fi, FI_DIRTY_INODE)) {
204 update_inode_page(inode);
205 goto go_write;
206 }
207
167 /* 208 /*
168 * if there is no written data, don't waste time to write recovery info. 209 * if there is no written data, don't waste time to write recovery info.
169 */ 210 */
170 if (!is_inode_flag_set(fi, FI_APPEND_WRITE) && 211 if (!is_inode_flag_set(fi, FI_APPEND_WRITE) &&
171 !exist_written_data(sbi, ino, APPEND_INO)) { 212 !exist_written_data(sbi, ino, APPEND_INO)) {
172 struct page *i = find_get_page(NODE_MAPPING(sbi), ino);
173 213
174 /* But we need to avoid that there are some inode updates */ 214 /* it may call write_inode just prior to fsync */
175 if ((i && PageDirty(i)) || need_inode_block_update(sbi, ino)) { 215 if (need_inode_page_update(sbi, ino))
176 f2fs_put_page(i, 0);
177 goto go_write; 216 goto go_write;
178 }
179 f2fs_put_page(i, 0);
180 217
181 if (is_inode_flag_set(fi, FI_UPDATE_WRITE) || 218 if (is_inode_flag_set(fi, FI_UPDATE_WRITE) ||
182 exist_written_data(sbi, ino, UPDATE_INO)) 219 exist_written_data(sbi, ino, UPDATE_INO))
@@ -196,49 +233,36 @@ go_write:
196 up_read(&fi->i_sem); 233 up_read(&fi->i_sem);
197 234
198 if (need_cp) { 235 if (need_cp) {
199 nid_t pino;
200
201 /* all the dirty node pages should be flushed for POR */ 236 /* all the dirty node pages should be flushed for POR */
202 ret = f2fs_sync_fs(inode->i_sb, 1); 237 ret = f2fs_sync_fs(inode->i_sb, 1);
203 238
204 down_write(&fi->i_sem); 239 /*
205 F2FS_I(inode)->xattr_ver = 0; 240 * We've secured consistency through sync_fs. Following pino
206 if (file_wrong_pino(inode) && inode->i_nlink == 1 && 241 * will be used only for fsynced inodes after checkpoint.
207 get_parent_ino(inode, &pino)) { 242 */
208 F2FS_I(inode)->i_pino = pino; 243 try_to_fix_pino(inode);
209 file_got_pino(inode); 244 goto out;
210 up_write(&fi->i_sem); 245 }
211 mark_inode_dirty_sync(inode);
212 ret = f2fs_write_inode(inode, NULL);
213 if (ret)
214 goto out;
215 } else {
216 up_write(&fi->i_sem);
217 }
218 } else {
219sync_nodes: 246sync_nodes:
220 sync_node_pages(sbi, ino, &wbc); 247 sync_node_pages(sbi, ino, &wbc);
221
222 if (need_inode_block_update(sbi, ino)) {
223 mark_inode_dirty_sync(inode);
224 ret = f2fs_write_inode(inode, NULL);
225 if (ret)
226 goto out;
227 goto sync_nodes;
228 }
229 248
230 ret = wait_on_node_pages_writeback(sbi, ino); 249 if (need_inode_block_update(sbi, ino)) {
231 if (ret) 250 mark_inode_dirty_sync(inode);
232 goto out; 251 f2fs_write_inode(inode, NULL);
252 goto sync_nodes;
253 }
254
255 ret = wait_on_node_pages_writeback(sbi, ino);
256 if (ret)
257 goto out;
233 258
234 /* once recovery info is written, don't need to tack this */ 259 /* once recovery info is written, don't need to tack this */
235 remove_dirty_inode(sbi, ino, APPEND_INO); 260 remove_dirty_inode(sbi, ino, APPEND_INO);
236 clear_inode_flag(fi, FI_APPEND_WRITE); 261 clear_inode_flag(fi, FI_APPEND_WRITE);
237flush_out: 262flush_out:
238 remove_dirty_inode(sbi, ino, UPDATE_INO); 263 remove_dirty_inode(sbi, ino, UPDATE_INO);
239 clear_inode_flag(fi, FI_UPDATE_WRITE); 264 clear_inode_flag(fi, FI_UPDATE_WRITE);
240 ret = f2fs_issue_flush(F2FS_I_SB(inode)); 265 ret = f2fs_issue_flush(sbi);
241 }
242out: 266out:
243 trace_f2fs_sync_file_exit(inode, need_cp, datasync, ret); 267 trace_f2fs_sync_file_exit(inode, need_cp, datasync, ret);
244 return ret; 268 return ret;
@@ -296,7 +320,7 @@ static loff_t f2fs_seek_block(struct file *file, loff_t offset, int whence)
296 goto fail; 320 goto fail;
297 321
298 /* handle inline data case */ 322 /* handle inline data case */
299 if (f2fs_has_inline_data(inode)) { 323 if (f2fs_has_inline_data(inode) || f2fs_has_inline_dentry(inode)) {
300 if (whence == SEEK_HOLE) 324 if (whence == SEEK_HOLE)
301 data_ofs = isize; 325 data_ofs = isize;
302 goto found; 326 goto found;
@@ -374,6 +398,15 @@ static loff_t f2fs_llseek(struct file *file, loff_t offset, int whence)
374 398
375static int f2fs_file_mmap(struct file *file, struct vm_area_struct *vma) 399static int f2fs_file_mmap(struct file *file, struct vm_area_struct *vma)
376{ 400{
401 struct inode *inode = file_inode(file);
402
403 /* we don't need to use inline_data strictly */
404 if (f2fs_has_inline_data(inode)) {
405 int err = f2fs_convert_inline_inode(inode);
406 if (err)
407 return err;
408 }
409
377 file_accessed(file); 410 file_accessed(file);
378 vma->vm_ops = &f2fs_file_vm_ops; 411 vma->vm_ops = &f2fs_file_vm_ops;
379 return 0; 412 return 0;
@@ -415,20 +448,17 @@ void truncate_data_blocks(struct dnode_of_data *dn)
415 truncate_data_blocks_range(dn, ADDRS_PER_BLOCK); 448 truncate_data_blocks_range(dn, ADDRS_PER_BLOCK);
416} 449}
417 450
418static void truncate_partial_data_page(struct inode *inode, u64 from) 451static int truncate_partial_data_page(struct inode *inode, u64 from)
419{ 452{
420 unsigned offset = from & (PAGE_CACHE_SIZE - 1); 453 unsigned offset = from & (PAGE_CACHE_SIZE - 1);
421 struct page *page; 454 struct page *page;
422 455
423 if (f2fs_has_inline_data(inode))
424 return truncate_inline_data(inode, from);
425
426 if (!offset) 456 if (!offset)
427 return; 457 return 0;
428 458
429 page = find_data_page(inode, from >> PAGE_CACHE_SHIFT, false); 459 page = find_data_page(inode, from >> PAGE_CACHE_SHIFT, false);
430 if (IS_ERR(page)) 460 if (IS_ERR(page))
431 return; 461 return 0;
432 462
433 lock_page(page); 463 lock_page(page);
434 if (unlikely(!PageUptodate(page) || 464 if (unlikely(!PageUptodate(page) ||
@@ -438,9 +468,9 @@ static void truncate_partial_data_page(struct inode *inode, u64 from)
438 f2fs_wait_on_page_writeback(page, DATA); 468 f2fs_wait_on_page_writeback(page, DATA);
439 zero_user(page, offset, PAGE_CACHE_SIZE - offset); 469 zero_user(page, offset, PAGE_CACHE_SIZE - offset);
440 set_page_dirty(page); 470 set_page_dirty(page);
441
442out: 471out:
443 f2fs_put_page(page, 1); 472 f2fs_put_page(page, 1);
473 return 0;
444} 474}
445 475
446int truncate_blocks(struct inode *inode, u64 from, bool lock) 476int truncate_blocks(struct inode *inode, u64 from, bool lock)
@@ -450,27 +480,33 @@ int truncate_blocks(struct inode *inode, u64 from, bool lock)
450 struct dnode_of_data dn; 480 struct dnode_of_data dn;
451 pgoff_t free_from; 481 pgoff_t free_from;
452 int count = 0, err = 0; 482 int count = 0, err = 0;
483 struct page *ipage;
453 484
454 trace_f2fs_truncate_blocks_enter(inode, from); 485 trace_f2fs_truncate_blocks_enter(inode, from);
455 486
456 if (f2fs_has_inline_data(inode))
457 goto done;
458
459 free_from = (pgoff_t) 487 free_from = (pgoff_t)
460 ((from + blocksize - 1) >> (sbi->log_blocksize)); 488 ((from + blocksize - 1) >> (sbi->log_blocksize));
461 489
462 if (lock) 490 if (lock)
463 f2fs_lock_op(sbi); 491 f2fs_lock_op(sbi);
464 492
465 set_new_dnode(&dn, inode, NULL, NULL, 0); 493 ipage = get_node_page(sbi, inode->i_ino);
494 if (IS_ERR(ipage)) {
495 err = PTR_ERR(ipage);
496 goto out;
497 }
498
499 if (f2fs_has_inline_data(inode)) {
500 f2fs_put_page(ipage, 1);
501 goto out;
502 }
503
504 set_new_dnode(&dn, inode, ipage, NULL, 0);
466 err = get_dnode_of_data(&dn, free_from, LOOKUP_NODE); 505 err = get_dnode_of_data(&dn, free_from, LOOKUP_NODE);
467 if (err) { 506 if (err) {
468 if (err == -ENOENT) 507 if (err == -ENOENT)
469 goto free_next; 508 goto free_next;
470 if (lock) 509 goto out;
471 f2fs_unlock_op(sbi);
472 trace_f2fs_truncate_blocks_exit(inode, err);
473 return err;
474 } 510 }
475 511
476 count = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode)); 512 count = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode));
@@ -486,11 +522,13 @@ int truncate_blocks(struct inode *inode, u64 from, bool lock)
486 f2fs_put_dnode(&dn); 522 f2fs_put_dnode(&dn);
487free_next: 523free_next:
488 err = truncate_inode_blocks(inode, free_from); 524 err = truncate_inode_blocks(inode, free_from);
525out:
489 if (lock) 526 if (lock)
490 f2fs_unlock_op(sbi); 527 f2fs_unlock_op(sbi);
491done: 528
492 /* lastly zero out the first data page */ 529 /* lastly zero out the first data page */
493 truncate_partial_data_page(inode, from); 530 if (!err)
531 err = truncate_partial_data_page(inode, from);
494 532
495 trace_f2fs_truncate_blocks_exit(inode, err); 533 trace_f2fs_truncate_blocks_exit(inode, err);
496 return err; 534 return err;
@@ -504,6 +542,12 @@ void f2fs_truncate(struct inode *inode)
504 542
505 trace_f2fs_truncate(inode); 543 trace_f2fs_truncate(inode);
506 544
545 /* we should check inline_data size */
546 if (f2fs_has_inline_data(inode) && !f2fs_may_inline(inode)) {
547 if (f2fs_convert_inline_inode(inode))
548 return;
549 }
550
507 if (!truncate_blocks(inode, i_size_read(inode), true)) { 551 if (!truncate_blocks(inode, i_size_read(inode), true)) {
508 inode->i_mtime = inode->i_ctime = CURRENT_TIME; 552 inode->i_mtime = inode->i_ctime = CURRENT_TIME;
509 mark_inode_dirty(inode); 553 mark_inode_dirty(inode);
@@ -561,10 +605,6 @@ int f2fs_setattr(struct dentry *dentry, struct iattr *attr)
561 return err; 605 return err;
562 606
563 if (attr->ia_valid & ATTR_SIZE) { 607 if (attr->ia_valid & ATTR_SIZE) {
564 err = f2fs_convert_inline_data(inode, attr->ia_size, NULL);
565 if (err)
566 return err;
567
568 if (attr->ia_size != i_size_read(inode)) { 608 if (attr->ia_size != i_size_read(inode)) {
569 truncate_setsize(inode, attr->ia_size); 609 truncate_setsize(inode, attr->ia_size);
570 f2fs_truncate(inode); 610 f2fs_truncate(inode);
@@ -665,9 +705,11 @@ static int punch_hole(struct inode *inode, loff_t offset, loff_t len)
665 if (offset >= inode->i_size) 705 if (offset >= inode->i_size)
666 return ret; 706 return ret;
667 707
668 ret = f2fs_convert_inline_data(inode, MAX_INLINE_DATA + 1, NULL); 708 if (f2fs_has_inline_data(inode)) {
669 if (ret) 709 ret = f2fs_convert_inline_inode(inode);
670 return ret; 710 if (ret)
711 return ret;
712 }
671 713
672 pg_start = ((unsigned long long) offset) >> PAGE_CACHE_SHIFT; 714 pg_start = ((unsigned long long) offset) >> PAGE_CACHE_SHIFT;
673 pg_end = ((unsigned long long) offset + len) >> PAGE_CACHE_SHIFT; 715 pg_end = ((unsigned long long) offset + len) >> PAGE_CACHE_SHIFT;
@@ -721,9 +763,11 @@ static int expand_inode_data(struct inode *inode, loff_t offset,
721 if (ret) 763 if (ret)
722 return ret; 764 return ret;
723 765
724 ret = f2fs_convert_inline_data(inode, offset + len, NULL); 766 if (f2fs_has_inline_data(inode)) {
725 if (ret) 767 ret = f2fs_convert_inline_inode(inode);
726 return ret; 768 if (ret)
769 return ret;
770 }
727 771
728 pg_start = ((unsigned long long) offset) >> PAGE_CACHE_SHIFT; 772 pg_start = ((unsigned long long) offset) >> PAGE_CACHE_SHIFT;
729 pg_end = ((unsigned long long) offset + len) >> PAGE_CACHE_SHIFT; 773 pg_end = ((unsigned long long) offset + len) >> PAGE_CACHE_SHIFT;
@@ -874,7 +918,15 @@ static int f2fs_ioc_start_atomic_write(struct file *filp)
874 918
875 set_inode_flag(F2FS_I(inode), FI_ATOMIC_FILE); 919 set_inode_flag(F2FS_I(inode), FI_ATOMIC_FILE);
876 920
877 return f2fs_convert_inline_data(inode, MAX_INLINE_DATA + 1, NULL); 921 return f2fs_convert_inline_inode(inode);
922}
923
924static int f2fs_release_file(struct inode *inode, struct file *filp)
925{
926 /* some remained atomic pages should discarded */
927 if (f2fs_is_atomic_file(inode) || f2fs_is_volatile_file(inode))
928 commit_inmem_pages(inode, true);
929 return 0;
878} 930}
879 931
880static int f2fs_ioc_commit_atomic_write(struct file *filp) 932static int f2fs_ioc_commit_atomic_write(struct file *filp)
@@ -908,7 +960,8 @@ static int f2fs_ioc_start_volatile_write(struct file *filp)
908 return -EACCES; 960 return -EACCES;
909 961
910 set_inode_flag(F2FS_I(inode), FI_VOLATILE_FILE); 962 set_inode_flag(F2FS_I(inode), FI_VOLATILE_FILE);
911 return 0; 963
964 return f2fs_convert_inline_inode(inode);
912} 965}
913 966
914static int f2fs_ioc_fitrim(struct file *filp, unsigned long arg) 967static int f2fs_ioc_fitrim(struct file *filp, unsigned long arg)
@@ -985,6 +1038,7 @@ const struct file_operations f2fs_file_operations = {
985 .read_iter = generic_file_read_iter, 1038 .read_iter = generic_file_read_iter,
986 .write_iter = generic_file_write_iter, 1039 .write_iter = generic_file_write_iter,
987 .open = generic_file_open, 1040 .open = generic_file_open,
1041 .release = f2fs_release_file,
988 .mmap = f2fs_file_mmap, 1042 .mmap = f2fs_file_mmap,
989 .fsync = f2fs_sync_file, 1043 .fsync = f2fs_sync_file,
990 .fallocate = f2fs_fallocate, 1044 .fallocate = f2fs_fallocate,
diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c
index 2a8f4acdb86b..eec0933a4819 100644
--- a/fs/f2fs/gc.c
+++ b/fs/f2fs/gc.c
@@ -96,8 +96,6 @@ int start_gc_thread(struct f2fs_sb_info *sbi)
96 dev_t dev = sbi->sb->s_bdev->bd_dev; 96 dev_t dev = sbi->sb->s_bdev->bd_dev;
97 int err = 0; 97 int err = 0;
98 98
99 if (!test_opt(sbi, BG_GC))
100 goto out;
101 gc_th = kmalloc(sizeof(struct f2fs_gc_kthread), GFP_KERNEL); 99 gc_th = kmalloc(sizeof(struct f2fs_gc_kthread), GFP_KERNEL);
102 if (!gc_th) { 100 if (!gc_th) {
103 err = -ENOMEM; 101 err = -ENOMEM;
@@ -340,34 +338,39 @@ static const struct victim_selection default_v_ops = {
340 .get_victim = get_victim_by_default, 338 .get_victim = get_victim_by_default,
341}; 339};
342 340
343static struct inode *find_gc_inode(nid_t ino, struct list_head *ilist) 341static struct inode *find_gc_inode(struct gc_inode_list *gc_list, nid_t ino)
344{ 342{
345 struct inode_entry *ie; 343 struct inode_entry *ie;
346 344
347 list_for_each_entry(ie, ilist, list) 345 ie = radix_tree_lookup(&gc_list->iroot, ino);
348 if (ie->inode->i_ino == ino) 346 if (ie)
349 return ie->inode; 347 return ie->inode;
350 return NULL; 348 return NULL;
351} 349}
352 350
353static void add_gc_inode(struct inode *inode, struct list_head *ilist) 351static void add_gc_inode(struct gc_inode_list *gc_list, struct inode *inode)
354{ 352{
355 struct inode_entry *new_ie; 353 struct inode_entry *new_ie;
356 354
357 if (inode == find_gc_inode(inode->i_ino, ilist)) { 355 if (inode == find_gc_inode(gc_list, inode->i_ino)) {
358 iput(inode); 356 iput(inode);
359 return; 357 return;
360 } 358 }
361
362 new_ie = f2fs_kmem_cache_alloc(winode_slab, GFP_NOFS); 359 new_ie = f2fs_kmem_cache_alloc(winode_slab, GFP_NOFS);
363 new_ie->inode = inode; 360 new_ie->inode = inode;
364 list_add_tail(&new_ie->list, ilist); 361retry:
362 if (radix_tree_insert(&gc_list->iroot, inode->i_ino, new_ie)) {
363 cond_resched();
364 goto retry;
365 }
366 list_add_tail(&new_ie->list, &gc_list->ilist);
365} 367}
366 368
367static void put_gc_inode(struct list_head *ilist) 369static void put_gc_inode(struct gc_inode_list *gc_list)
368{ 370{
369 struct inode_entry *ie, *next_ie; 371 struct inode_entry *ie, *next_ie;
370 list_for_each_entry_safe(ie, next_ie, ilist, list) { 372 list_for_each_entry_safe(ie, next_ie, &gc_list->ilist, list) {
373 radix_tree_delete(&gc_list->iroot, ie->inode->i_ino);
371 iput(ie->inode); 374 iput(ie->inode);
372 list_del(&ie->list); 375 list_del(&ie->list);
373 kmem_cache_free(winode_slab, ie); 376 kmem_cache_free(winode_slab, ie);
@@ -553,7 +556,7 @@ out:
553 * the victim data block is ignored. 556 * the victim data block is ignored.
554 */ 557 */
555static void gc_data_segment(struct f2fs_sb_info *sbi, struct f2fs_summary *sum, 558static void gc_data_segment(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
556 struct list_head *ilist, unsigned int segno, int gc_type) 559 struct gc_inode_list *gc_list, unsigned int segno, int gc_type)
557{ 560{
558 struct super_block *sb = sbi->sb; 561 struct super_block *sb = sbi->sb;
559 struct f2fs_summary *entry; 562 struct f2fs_summary *entry;
@@ -605,27 +608,27 @@ next_step:
605 608
606 data_page = find_data_page(inode, 609 data_page = find_data_page(inode,
607 start_bidx + ofs_in_node, false); 610 start_bidx + ofs_in_node, false);
608 if (IS_ERR(data_page)) 611 if (IS_ERR(data_page)) {
609 goto next_iput; 612 iput(inode);
613 continue;
614 }
610 615
611 f2fs_put_page(data_page, 0); 616 f2fs_put_page(data_page, 0);
612 add_gc_inode(inode, ilist); 617 add_gc_inode(gc_list, inode);
613 } else { 618 continue;
614 inode = find_gc_inode(dni.ino, ilist); 619 }
615 if (inode) { 620
616 start_bidx = start_bidx_of_node(nofs, 621 /* phase 3 */
617 F2FS_I(inode)); 622 inode = find_gc_inode(gc_list, dni.ino);
618 data_page = get_lock_data_page(inode, 623 if (inode) {
624 start_bidx = start_bidx_of_node(nofs, F2FS_I(inode));
625 data_page = get_lock_data_page(inode,
619 start_bidx + ofs_in_node); 626 start_bidx + ofs_in_node);
620 if (IS_ERR(data_page)) 627 if (IS_ERR(data_page))
621 continue; 628 continue;
622 move_data_page(inode, data_page, gc_type); 629 move_data_page(inode, data_page, gc_type);
623 stat_inc_data_blk_count(sbi, 1); 630 stat_inc_data_blk_count(sbi, 1);
624 }
625 } 631 }
626 continue;
627next_iput:
628 iput(inode);
629 } 632 }
630 633
631 if (++phase < 4) 634 if (++phase < 4)
@@ -646,18 +649,20 @@ next_iput:
646} 649}
647 650
648static int __get_victim(struct f2fs_sb_info *sbi, unsigned int *victim, 651static int __get_victim(struct f2fs_sb_info *sbi, unsigned int *victim,
649 int gc_type, int type) 652 int gc_type)
650{ 653{
651 struct sit_info *sit_i = SIT_I(sbi); 654 struct sit_info *sit_i = SIT_I(sbi);
652 int ret; 655 int ret;
656
653 mutex_lock(&sit_i->sentry_lock); 657 mutex_lock(&sit_i->sentry_lock);
654 ret = DIRTY_I(sbi)->v_ops->get_victim(sbi, victim, gc_type, type, LFS); 658 ret = DIRTY_I(sbi)->v_ops->get_victim(sbi, victim, gc_type,
659 NO_CHECK_TYPE, LFS);
655 mutex_unlock(&sit_i->sentry_lock); 660 mutex_unlock(&sit_i->sentry_lock);
656 return ret; 661 return ret;
657} 662}
658 663
659static void do_garbage_collect(struct f2fs_sb_info *sbi, unsigned int segno, 664static void do_garbage_collect(struct f2fs_sb_info *sbi, unsigned int segno,
660 struct list_head *ilist, int gc_type) 665 struct gc_inode_list *gc_list, int gc_type)
661{ 666{
662 struct page *sum_page; 667 struct page *sum_page;
663 struct f2fs_summary_block *sum; 668 struct f2fs_summary_block *sum;
@@ -675,7 +680,7 @@ static void do_garbage_collect(struct f2fs_sb_info *sbi, unsigned int segno,
675 gc_node_segment(sbi, sum->entries, segno, gc_type); 680 gc_node_segment(sbi, sum->entries, segno, gc_type);
676 break; 681 break;
677 case SUM_TYPE_DATA: 682 case SUM_TYPE_DATA:
678 gc_data_segment(sbi, sum->entries, ilist, segno, gc_type); 683 gc_data_segment(sbi, sum->entries, gc_list, segno, gc_type);
679 break; 684 break;
680 } 685 }
681 blk_finish_plug(&plug); 686 blk_finish_plug(&plug);
@@ -688,16 +693,18 @@ static void do_garbage_collect(struct f2fs_sb_info *sbi, unsigned int segno,
688 693
689int f2fs_gc(struct f2fs_sb_info *sbi) 694int f2fs_gc(struct f2fs_sb_info *sbi)
690{ 695{
691 struct list_head ilist;
692 unsigned int segno, i; 696 unsigned int segno, i;
693 int gc_type = BG_GC; 697 int gc_type = BG_GC;
694 int nfree = 0; 698 int nfree = 0;
695 int ret = -1; 699 int ret = -1;
696 struct cp_control cpc = { 700 struct cp_control cpc;
697 .reason = CP_SYNC, 701 struct gc_inode_list gc_list = {
702 .ilist = LIST_HEAD_INIT(gc_list.ilist),
703 .iroot = RADIX_TREE_INIT(GFP_NOFS),
698 }; 704 };
699 705
700 INIT_LIST_HEAD(&ilist); 706 cpc.reason = test_opt(sbi, FASTBOOT) ? CP_UMOUNT : CP_SYNC;
707
701gc_more: 708gc_more:
702 if (unlikely(!(sbi->sb->s_flags & MS_ACTIVE))) 709 if (unlikely(!(sbi->sb->s_flags & MS_ACTIVE)))
703 goto stop; 710 goto stop;
@@ -709,7 +716,7 @@ gc_more:
709 write_checkpoint(sbi, &cpc); 716 write_checkpoint(sbi, &cpc);
710 } 717 }
711 718
712 if (!__get_victim(sbi, &segno, gc_type, NO_CHECK_TYPE)) 719 if (!__get_victim(sbi, &segno, gc_type))
713 goto stop; 720 goto stop;
714 ret = 0; 721 ret = 0;
715 722
@@ -719,7 +726,7 @@ gc_more:
719 META_SSA); 726 META_SSA);
720 727
721 for (i = 0; i < sbi->segs_per_sec; i++) 728 for (i = 0; i < sbi->segs_per_sec; i++)
722 do_garbage_collect(sbi, segno + i, &ilist, gc_type); 729 do_garbage_collect(sbi, segno + i, &gc_list, gc_type);
723 730
724 if (gc_type == FG_GC) { 731 if (gc_type == FG_GC) {
725 sbi->cur_victim_sec = NULL_SEGNO; 732 sbi->cur_victim_sec = NULL_SEGNO;
@@ -735,7 +742,7 @@ gc_more:
735stop: 742stop:
736 mutex_unlock(&sbi->gc_mutex); 743 mutex_unlock(&sbi->gc_mutex);
737 744
738 put_gc_inode(&ilist); 745 put_gc_inode(&gc_list);
739 return ret; 746 return ret;
740} 747}
741 748
diff --git a/fs/f2fs/gc.h b/fs/f2fs/gc.h
index 16f0b2b22999..6ff7ad38463e 100644
--- a/fs/f2fs/gc.h
+++ b/fs/f2fs/gc.h
@@ -40,6 +40,11 @@ struct inode_entry {
40 struct inode *inode; 40 struct inode *inode;
41}; 41};
42 42
43struct gc_inode_list {
44 struct list_head ilist;
45 struct radix_tree_root iroot;
46};
47
43/* 48/*
44 * inline functions 49 * inline functions
45 */ 50 */
diff --git a/fs/f2fs/inline.c b/fs/f2fs/inline.c
index 88036fd75797..f2d3c581e776 100644
--- a/fs/f2fs/inline.c
+++ b/fs/f2fs/inline.c
@@ -15,35 +15,44 @@
15 15
16bool f2fs_may_inline(struct inode *inode) 16bool f2fs_may_inline(struct inode *inode)
17{ 17{
18 block_t nr_blocks;
19 loff_t i_size;
20
21 if (!test_opt(F2FS_I_SB(inode), INLINE_DATA)) 18 if (!test_opt(F2FS_I_SB(inode), INLINE_DATA))
22 return false; 19 return false;
23 20
24 if (f2fs_is_atomic_file(inode)) 21 if (f2fs_is_atomic_file(inode))
25 return false; 22 return false;
26 23
27 nr_blocks = F2FS_I(inode)->i_xattr_nid ? 3 : 2; 24 if (!S_ISREG(inode->i_mode))
28 if (inode->i_blocks > nr_blocks)
29 return false; 25 return false;
30 26
31 i_size = i_size_read(inode); 27 if (i_size_read(inode) > MAX_INLINE_DATA)
32 if (i_size > MAX_INLINE_DATA)
33 return false; 28 return false;
34 29
35 return true; 30 return true;
36} 31}
37 32
38int f2fs_read_inline_data(struct inode *inode, struct page *page) 33void read_inline_data(struct page *page, struct page *ipage)
39{ 34{
40 struct page *ipage;
41 void *src_addr, *dst_addr; 35 void *src_addr, *dst_addr;
42 36
43 if (page->index) { 37 if (PageUptodate(page))
44 zero_user_segment(page, 0, PAGE_CACHE_SIZE); 38 return;
45 goto out; 39
46 } 40 f2fs_bug_on(F2FS_P_SB(page), page->index);
41
42 zero_user_segment(page, MAX_INLINE_DATA, PAGE_CACHE_SIZE);
43
44 /* Copy the whole inline data block */
45 src_addr = inline_data_addr(ipage);
46 dst_addr = kmap_atomic(page);
47 memcpy(dst_addr, src_addr, MAX_INLINE_DATA);
48 flush_dcache_page(page);
49 kunmap_atomic(dst_addr);
50 SetPageUptodate(page);
51}
52
53int f2fs_read_inline_data(struct inode *inode, struct page *page)
54{
55 struct page *ipage;
47 56
48 ipage = get_node_page(F2FS_I_SB(inode), inode->i_ino); 57 ipage = get_node_page(F2FS_I_SB(inode), inode->i_ino);
49 if (IS_ERR(ipage)) { 58 if (IS_ERR(ipage)) {
@@ -51,112 +60,116 @@ int f2fs_read_inline_data(struct inode *inode, struct page *page)
51 return PTR_ERR(ipage); 60 return PTR_ERR(ipage);
52 } 61 }
53 62
54 zero_user_segment(page, MAX_INLINE_DATA, PAGE_CACHE_SIZE); 63 if (!f2fs_has_inline_data(inode)) {
64 f2fs_put_page(ipage, 1);
65 return -EAGAIN;
66 }
55 67
56 /* Copy the whole inline data block */ 68 if (page->index)
57 src_addr = inline_data_addr(ipage); 69 zero_user_segment(page, 0, PAGE_CACHE_SIZE);
58 dst_addr = kmap(page); 70 else
59 memcpy(dst_addr, src_addr, MAX_INLINE_DATA); 71 read_inline_data(page, ipage);
60 kunmap(page);
61 f2fs_put_page(ipage, 1);
62 72
63out:
64 SetPageUptodate(page); 73 SetPageUptodate(page);
74 f2fs_put_page(ipage, 1);
65 unlock_page(page); 75 unlock_page(page);
66
67 return 0; 76 return 0;
68} 77}
69 78
70static int __f2fs_convert_inline_data(struct inode *inode, struct page *page) 79int f2fs_convert_inline_page(struct dnode_of_data *dn, struct page *page)
71{ 80{
72 int err = 0;
73 struct page *ipage;
74 struct dnode_of_data dn;
75 void *src_addr, *dst_addr; 81 void *src_addr, *dst_addr;
76 block_t new_blk_addr; 82 block_t new_blk_addr;
77 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
78 struct f2fs_io_info fio = { 83 struct f2fs_io_info fio = {
79 .type = DATA, 84 .type = DATA,
80 .rw = WRITE_SYNC | REQ_PRIO, 85 .rw = WRITE_SYNC | REQ_PRIO,
81 }; 86 };
87 int dirty, err;
82 88
83 f2fs_lock_op(sbi); 89 f2fs_bug_on(F2FS_I_SB(dn->inode), page->index);
84 ipage = get_node_page(sbi, inode->i_ino);
85 if (IS_ERR(ipage)) {
86 err = PTR_ERR(ipage);
87 goto out;
88 }
89 90
90 /* someone else converted inline_data already */ 91 if (!f2fs_exist_data(dn->inode))
91 if (!f2fs_has_inline_data(inode)) 92 goto clear_out;
92 goto out;
93 93
94 /* 94 err = f2fs_reserve_block(dn, 0);
95 * i_addr[0] is not used for inline data,
96 * so reserving new block will not destroy inline data
97 */
98 set_new_dnode(&dn, inode, ipage, NULL, 0);
99 err = f2fs_reserve_block(&dn, 0);
100 if (err) 95 if (err)
101 goto out; 96 return err;
102 97
103 f2fs_wait_on_page_writeback(page, DATA); 98 f2fs_wait_on_page_writeback(page, DATA);
99
100 if (PageUptodate(page))
101 goto no_update;
102
104 zero_user_segment(page, MAX_INLINE_DATA, PAGE_CACHE_SIZE); 103 zero_user_segment(page, MAX_INLINE_DATA, PAGE_CACHE_SIZE);
105 104
106 /* Copy the whole inline data block */ 105 /* Copy the whole inline data block */
107 src_addr = inline_data_addr(ipage); 106 src_addr = inline_data_addr(dn->inode_page);
108 dst_addr = kmap(page); 107 dst_addr = kmap_atomic(page);
109 memcpy(dst_addr, src_addr, MAX_INLINE_DATA); 108 memcpy(dst_addr, src_addr, MAX_INLINE_DATA);
110 kunmap(page); 109 flush_dcache_page(page);
110 kunmap_atomic(dst_addr);
111 SetPageUptodate(page); 111 SetPageUptodate(page);
112no_update:
113 /* clear dirty state */
114 dirty = clear_page_dirty_for_io(page);
112 115
113 /* write data page to try to make data consistent */ 116 /* write data page to try to make data consistent */
114 set_page_writeback(page); 117 set_page_writeback(page);
115 write_data_page(page, &dn, &new_blk_addr, &fio); 118
116 update_extent_cache(new_blk_addr, &dn); 119 write_data_page(page, dn, &new_blk_addr, &fio);
120 update_extent_cache(new_blk_addr, dn);
117 f2fs_wait_on_page_writeback(page, DATA); 121 f2fs_wait_on_page_writeback(page, DATA);
122 if (dirty)
123 inode_dec_dirty_pages(dn->inode);
118 124
119 /* clear inline data and flag after data writeback */ 125 /* this converted inline_data should be recovered. */
120 zero_user_segment(ipage, INLINE_DATA_OFFSET, 126 set_inode_flag(F2FS_I(dn->inode), FI_APPEND_WRITE);
121 INLINE_DATA_OFFSET + MAX_INLINE_DATA);
122 clear_inode_flag(F2FS_I(inode), FI_INLINE_DATA);
123 stat_dec_inline_inode(inode);
124 127
125 sync_inode_page(&dn); 128 /* clear inline data and flag after data writeback */
126 f2fs_put_dnode(&dn); 129 truncate_inline_data(dn->inode_page, 0);
127out: 130clear_out:
128 f2fs_unlock_op(sbi); 131 stat_dec_inline_inode(dn->inode);
129 return err; 132 f2fs_clear_inline_inode(dn->inode);
133 sync_inode_page(dn);
134 f2fs_put_dnode(dn);
135 return 0;
130} 136}
131 137
132int f2fs_convert_inline_data(struct inode *inode, pgoff_t to_size, 138int f2fs_convert_inline_inode(struct inode *inode)
133 struct page *page)
134{ 139{
135 struct page *new_page = page; 140 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
136 int err; 141 struct dnode_of_data dn;
142 struct page *ipage, *page;
143 int err = 0;
137 144
138 if (!f2fs_has_inline_data(inode)) 145 page = grab_cache_page(inode->i_mapping, 0);
139 return 0; 146 if (!page)
140 else if (to_size <= MAX_INLINE_DATA) 147 return -ENOMEM;
141 return 0; 148
149 f2fs_lock_op(sbi);
142 150
143 if (!page || page->index != 0) { 151 ipage = get_node_page(sbi, inode->i_ino);
144 new_page = grab_cache_page(inode->i_mapping, 0); 152 if (IS_ERR(ipage)) {
145 if (!new_page) 153 err = PTR_ERR(ipage);
146 return -ENOMEM; 154 goto out;
147 } 155 }
148 156
149 err = __f2fs_convert_inline_data(inode, new_page); 157 set_new_dnode(&dn, inode, ipage, ipage, 0);
150 if (!page || page->index != 0) 158
151 f2fs_put_page(new_page, 1); 159 if (f2fs_has_inline_data(inode))
160 err = f2fs_convert_inline_page(&dn, page);
161
162 f2fs_put_dnode(&dn);
163out:
164 f2fs_unlock_op(sbi);
165
166 f2fs_put_page(page, 1);
152 return err; 167 return err;
153} 168}
154 169
155int f2fs_write_inline_data(struct inode *inode, 170int f2fs_write_inline_data(struct inode *inode, struct page *page)
156 struct page *page, unsigned size)
157{ 171{
158 void *src_addr, *dst_addr; 172 void *src_addr, *dst_addr;
159 struct page *ipage;
160 struct dnode_of_data dn; 173 struct dnode_of_data dn;
161 int err; 174 int err;
162 175
@@ -164,47 +177,39 @@ int f2fs_write_inline_data(struct inode *inode,
164 err = get_dnode_of_data(&dn, 0, LOOKUP_NODE); 177 err = get_dnode_of_data(&dn, 0, LOOKUP_NODE);
165 if (err) 178 if (err)
166 return err; 179 return err;
167 ipage = dn.inode_page;
168 180
169 f2fs_wait_on_page_writeback(ipage, NODE);
170 zero_user_segment(ipage, INLINE_DATA_OFFSET,
171 INLINE_DATA_OFFSET + MAX_INLINE_DATA);
172 src_addr = kmap(page);
173 dst_addr = inline_data_addr(ipage);
174 memcpy(dst_addr, src_addr, size);
175 kunmap(page);
176
177 /* Release the first data block if it is allocated */
178 if (!f2fs_has_inline_data(inode)) { 181 if (!f2fs_has_inline_data(inode)) {
179 truncate_data_blocks_range(&dn, 1); 182 f2fs_put_dnode(&dn);
180 set_inode_flag(F2FS_I(inode), FI_INLINE_DATA); 183 return -EAGAIN;
181 stat_inc_inline_inode(inode);
182 } 184 }
183 185
186 f2fs_bug_on(F2FS_I_SB(inode), page->index);
187
188 f2fs_wait_on_page_writeback(dn.inode_page, NODE);
189 src_addr = kmap_atomic(page);
190 dst_addr = inline_data_addr(dn.inode_page);
191 memcpy(dst_addr, src_addr, MAX_INLINE_DATA);
192 kunmap_atomic(src_addr);
193
184 set_inode_flag(F2FS_I(inode), FI_APPEND_WRITE); 194 set_inode_flag(F2FS_I(inode), FI_APPEND_WRITE);
195 set_inode_flag(F2FS_I(inode), FI_DATA_EXIST);
196
185 sync_inode_page(&dn); 197 sync_inode_page(&dn);
186 f2fs_put_dnode(&dn); 198 f2fs_put_dnode(&dn);
187
188 return 0; 199 return 0;
189} 200}
190 201
191void truncate_inline_data(struct inode *inode, u64 from) 202void truncate_inline_data(struct page *ipage, u64 from)
192{ 203{
193 struct page *ipage; 204 void *addr;
194 205
195 if (from >= MAX_INLINE_DATA) 206 if (from >= MAX_INLINE_DATA)
196 return; 207 return;
197 208
198 ipage = get_node_page(F2FS_I_SB(inode), inode->i_ino);
199 if (IS_ERR(ipage))
200 return;
201
202 f2fs_wait_on_page_writeback(ipage, NODE); 209 f2fs_wait_on_page_writeback(ipage, NODE);
203 210
204 zero_user_segment(ipage, INLINE_DATA_OFFSET + from, 211 addr = inline_data_addr(ipage);
205 INLINE_DATA_OFFSET + MAX_INLINE_DATA); 212 memset(addr + from, 0, MAX_INLINE_DATA - from);
206 set_page_dirty(ipage);
207 f2fs_put_page(ipage, 1);
208} 213}
209 214
210bool recover_inline_data(struct inode *inode, struct page *npage) 215bool recover_inline_data(struct inode *inode, struct page *npage)
@@ -236,6 +241,10 @@ process_inline:
236 src_addr = inline_data_addr(npage); 241 src_addr = inline_data_addr(npage);
237 dst_addr = inline_data_addr(ipage); 242 dst_addr = inline_data_addr(ipage);
238 memcpy(dst_addr, src_addr, MAX_INLINE_DATA); 243 memcpy(dst_addr, src_addr, MAX_INLINE_DATA);
244
245 set_inode_flag(F2FS_I(inode), FI_INLINE_DATA);
246 set_inode_flag(F2FS_I(inode), FI_DATA_EXIST);
247
239 update_inode(inode, ipage); 248 update_inode(inode, ipage);
240 f2fs_put_page(ipage, 1); 249 f2fs_put_page(ipage, 1);
241 return true; 250 return true;
@@ -244,16 +253,279 @@ process_inline:
244 if (f2fs_has_inline_data(inode)) { 253 if (f2fs_has_inline_data(inode)) {
245 ipage = get_node_page(sbi, inode->i_ino); 254 ipage = get_node_page(sbi, inode->i_ino);
246 f2fs_bug_on(sbi, IS_ERR(ipage)); 255 f2fs_bug_on(sbi, IS_ERR(ipage));
247 f2fs_wait_on_page_writeback(ipage, NODE); 256 truncate_inline_data(ipage, 0);
248 zero_user_segment(ipage, INLINE_DATA_OFFSET, 257 f2fs_clear_inline_inode(inode);
249 INLINE_DATA_OFFSET + MAX_INLINE_DATA);
250 clear_inode_flag(F2FS_I(inode), FI_INLINE_DATA);
251 update_inode(inode, ipage); 258 update_inode(inode, ipage);
252 f2fs_put_page(ipage, 1); 259 f2fs_put_page(ipage, 1);
253 } else if (ri && (ri->i_inline & F2FS_INLINE_DATA)) { 260 } else if (ri && (ri->i_inline & F2FS_INLINE_DATA)) {
254 truncate_blocks(inode, 0, false); 261 truncate_blocks(inode, 0, false);
255 set_inode_flag(F2FS_I(inode), FI_INLINE_DATA);
256 goto process_inline; 262 goto process_inline;
257 } 263 }
258 return false; 264 return false;
259} 265}
266
267struct f2fs_dir_entry *find_in_inline_dir(struct inode *dir,
268 struct qstr *name, struct page **res_page)
269{
270 struct f2fs_sb_info *sbi = F2FS_SB(dir->i_sb);
271 struct f2fs_inline_dentry *inline_dentry;
272 struct f2fs_dir_entry *de;
273 struct f2fs_dentry_ptr d;
274 struct page *ipage;
275
276 ipage = get_node_page(sbi, dir->i_ino);
277 if (IS_ERR(ipage))
278 return NULL;
279
280 inline_dentry = inline_data_addr(ipage);
281
282 make_dentry_ptr(&d, (void *)inline_dentry, 2);
283 de = find_target_dentry(name, NULL, &d);
284
285 unlock_page(ipage);
286 if (de)
287 *res_page = ipage;
288 else
289 f2fs_put_page(ipage, 0);
290
291 /*
292 * For the most part, it should be a bug when name_len is zero.
293 * We stop here for figuring out where the bugs has occurred.
294 */
295 f2fs_bug_on(sbi, d.max < 0);
296 return de;
297}
298
299struct f2fs_dir_entry *f2fs_parent_inline_dir(struct inode *dir,
300 struct page **p)
301{
302 struct f2fs_sb_info *sbi = F2FS_I_SB(dir);
303 struct page *ipage;
304 struct f2fs_dir_entry *de;
305 struct f2fs_inline_dentry *dentry_blk;
306
307 ipage = get_node_page(sbi, dir->i_ino);
308 if (IS_ERR(ipage))
309 return NULL;
310
311 dentry_blk = inline_data_addr(ipage);
312 de = &dentry_blk->dentry[1];
313 *p = ipage;
314 unlock_page(ipage);
315 return de;
316}
317
318int make_empty_inline_dir(struct inode *inode, struct inode *parent,
319 struct page *ipage)
320{
321 struct f2fs_inline_dentry *dentry_blk;
322 struct f2fs_dentry_ptr d;
323
324 dentry_blk = inline_data_addr(ipage);
325
326 make_dentry_ptr(&d, (void *)dentry_blk, 2);
327 do_make_empty_dir(inode, parent, &d);
328
329 set_page_dirty(ipage);
330
331 /* update i_size to MAX_INLINE_DATA */
332 if (i_size_read(inode) < MAX_INLINE_DATA) {
333 i_size_write(inode, MAX_INLINE_DATA);
334 set_inode_flag(F2FS_I(inode), FI_UPDATE_DIR);
335 }
336 return 0;
337}
338
339static int f2fs_convert_inline_dir(struct inode *dir, struct page *ipage,
340 struct f2fs_inline_dentry *inline_dentry)
341{
342 struct page *page;
343 struct dnode_of_data dn;
344 struct f2fs_dentry_block *dentry_blk;
345 int err;
346
347 page = grab_cache_page(dir->i_mapping, 0);
348 if (!page)
349 return -ENOMEM;
350
351 set_new_dnode(&dn, dir, ipage, NULL, 0);
352 err = f2fs_reserve_block(&dn, 0);
353 if (err)
354 goto out;
355
356 f2fs_wait_on_page_writeback(page, DATA);
357 zero_user_segment(page, 0, PAGE_CACHE_SIZE);
358
359 dentry_blk = kmap_atomic(page);
360
361 /* copy data from inline dentry block to new dentry block */
362 memcpy(dentry_blk->dentry_bitmap, inline_dentry->dentry_bitmap,
363 INLINE_DENTRY_BITMAP_SIZE);
364 memcpy(dentry_blk->dentry, inline_dentry->dentry,
365 sizeof(struct f2fs_dir_entry) * NR_INLINE_DENTRY);
366 memcpy(dentry_blk->filename, inline_dentry->filename,
367 NR_INLINE_DENTRY * F2FS_SLOT_LEN);
368
369 kunmap_atomic(dentry_blk);
370 SetPageUptodate(page);
371 set_page_dirty(page);
372
373 /* clear inline dir and flag after data writeback */
374 truncate_inline_data(ipage, 0);
375
376 stat_dec_inline_dir(dir);
377 clear_inode_flag(F2FS_I(dir), FI_INLINE_DENTRY);
378
379 if (i_size_read(dir) < PAGE_CACHE_SIZE) {
380 i_size_write(dir, PAGE_CACHE_SIZE);
381 set_inode_flag(F2FS_I(dir), FI_UPDATE_DIR);
382 }
383
384 sync_inode_page(&dn);
385out:
386 f2fs_put_page(page, 1);
387 return err;
388}
389
390int f2fs_add_inline_entry(struct inode *dir, const struct qstr *name,
391 struct inode *inode)
392{
393 struct f2fs_sb_info *sbi = F2FS_I_SB(dir);
394 struct page *ipage;
395 unsigned int bit_pos;
396 f2fs_hash_t name_hash;
397 struct f2fs_dir_entry *de;
398 size_t namelen = name->len;
399 struct f2fs_inline_dentry *dentry_blk = NULL;
400 int slots = GET_DENTRY_SLOTS(namelen);
401 struct page *page;
402 int err = 0;
403 int i;
404
405 name_hash = f2fs_dentry_hash(name);
406
407 ipage = get_node_page(sbi, dir->i_ino);
408 if (IS_ERR(ipage))
409 return PTR_ERR(ipage);
410
411 dentry_blk = inline_data_addr(ipage);
412 bit_pos = room_for_filename(&dentry_blk->dentry_bitmap,
413 slots, NR_INLINE_DENTRY);
414 if (bit_pos >= NR_INLINE_DENTRY) {
415 err = f2fs_convert_inline_dir(dir, ipage, dentry_blk);
416 if (!err)
417 err = -EAGAIN;
418 goto out;
419 }
420
421 down_write(&F2FS_I(inode)->i_sem);
422 page = init_inode_metadata(inode, dir, name, ipage);
423 if (IS_ERR(page)) {
424 err = PTR_ERR(page);
425 goto fail;
426 }
427
428 f2fs_wait_on_page_writeback(ipage, NODE);
429 de = &dentry_blk->dentry[bit_pos];
430 de->hash_code = name_hash;
431 de->name_len = cpu_to_le16(namelen);
432 memcpy(dentry_blk->filename[bit_pos], name->name, name->len);
433 de->ino = cpu_to_le32(inode->i_ino);
434 set_de_type(de, inode);
435 for (i = 0; i < slots; i++)
436 test_and_set_bit_le(bit_pos + i, &dentry_blk->dentry_bitmap);
437 set_page_dirty(ipage);
438
439 /* we don't need to mark_inode_dirty now */
440 F2FS_I(inode)->i_pino = dir->i_ino;
441 update_inode(inode, page);
442 f2fs_put_page(page, 1);
443
444 update_parent_metadata(dir, inode, 0);
445fail:
446 up_write(&F2FS_I(inode)->i_sem);
447
448 if (is_inode_flag_set(F2FS_I(dir), FI_UPDATE_DIR)) {
449 update_inode(dir, ipage);
450 clear_inode_flag(F2FS_I(dir), FI_UPDATE_DIR);
451 }
452out:
453 f2fs_put_page(ipage, 1);
454 return err;
455}
456
457void f2fs_delete_inline_entry(struct f2fs_dir_entry *dentry, struct page *page,
458 struct inode *dir, struct inode *inode)
459{
460 struct f2fs_inline_dentry *inline_dentry;
461 int slots = GET_DENTRY_SLOTS(le16_to_cpu(dentry->name_len));
462 unsigned int bit_pos;
463 int i;
464
465 lock_page(page);
466 f2fs_wait_on_page_writeback(page, NODE);
467
468 inline_dentry = inline_data_addr(page);
469 bit_pos = dentry - inline_dentry->dentry;
470 for (i = 0; i < slots; i++)
471 test_and_clear_bit_le(bit_pos + i,
472 &inline_dentry->dentry_bitmap);
473
474 set_page_dirty(page);
475
476 dir->i_ctime = dir->i_mtime = CURRENT_TIME;
477
478 if (inode)
479 f2fs_drop_nlink(dir, inode, page);
480
481 f2fs_put_page(page, 1);
482}
483
484bool f2fs_empty_inline_dir(struct inode *dir)
485{
486 struct f2fs_sb_info *sbi = F2FS_I_SB(dir);
487 struct page *ipage;
488 unsigned int bit_pos = 2;
489 struct f2fs_inline_dentry *dentry_blk;
490
491 ipage = get_node_page(sbi, dir->i_ino);
492 if (IS_ERR(ipage))
493 return false;
494
495 dentry_blk = inline_data_addr(ipage);
496 bit_pos = find_next_bit_le(&dentry_blk->dentry_bitmap,
497 NR_INLINE_DENTRY,
498 bit_pos);
499
500 f2fs_put_page(ipage, 1);
501
502 if (bit_pos < NR_INLINE_DENTRY)
503 return false;
504
505 return true;
506}
507
508int f2fs_read_inline_dir(struct file *file, struct dir_context *ctx)
509{
510 struct inode *inode = file_inode(file);
511 struct f2fs_inline_dentry *inline_dentry = NULL;
512 struct page *ipage = NULL;
513 struct f2fs_dentry_ptr d;
514
515 if (ctx->pos == NR_INLINE_DENTRY)
516 return 0;
517
518 ipage = get_node_page(F2FS_I_SB(inode), inode->i_ino);
519 if (IS_ERR(ipage))
520 return PTR_ERR(ipage);
521
522 inline_dentry = inline_data_addr(ipage);
523
524 make_dentry_ptr(&d, (void *)inline_dentry, 2);
525
526 if (!f2fs_fill_dentries(ctx, &d, 0))
527 ctx->pos = NR_INLINE_DENTRY;
528
529 f2fs_put_page(ipage, 1);
530 return 0;
531}
diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c
index 0deead4505e7..196cc7843aaf 100644
--- a/fs/f2fs/inode.c
+++ b/fs/f2fs/inode.c
@@ -67,12 +67,38 @@ static void __set_inode_rdev(struct inode *inode, struct f2fs_inode *ri)
67 } 67 }
68} 68}
69 69
70static int __recover_inline_status(struct inode *inode, struct page *ipage)
71{
72 void *inline_data = inline_data_addr(ipage);
73 struct f2fs_inode *ri;
74 void *zbuf;
75
76 zbuf = kzalloc(MAX_INLINE_DATA, GFP_NOFS);
77 if (!zbuf)
78 return -ENOMEM;
79
80 if (!memcmp(zbuf, inline_data, MAX_INLINE_DATA)) {
81 kfree(zbuf);
82 return 0;
83 }
84 kfree(zbuf);
85
86 f2fs_wait_on_page_writeback(ipage, NODE);
87 set_inode_flag(F2FS_I(inode), FI_DATA_EXIST);
88
89 ri = F2FS_INODE(ipage);
90 set_raw_inline(F2FS_I(inode), ri);
91 set_page_dirty(ipage);
92 return 0;
93}
94
70static int do_read_inode(struct inode *inode) 95static int do_read_inode(struct inode *inode)
71{ 96{
72 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 97 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
73 struct f2fs_inode_info *fi = F2FS_I(inode); 98 struct f2fs_inode_info *fi = F2FS_I(inode);
74 struct page *node_page; 99 struct page *node_page;
75 struct f2fs_inode *ri; 100 struct f2fs_inode *ri;
101 int err = 0;
76 102
77 /* Check if ino is within scope */ 103 /* Check if ino is within scope */
78 if (check_nid_range(sbi, inode->i_ino)) { 104 if (check_nid_range(sbi, inode->i_ino)) {
@@ -114,11 +140,19 @@ static int do_read_inode(struct inode *inode)
114 get_extent_info(&fi->ext, ri->i_ext); 140 get_extent_info(&fi->ext, ri->i_ext);
115 get_inline_info(fi, ri); 141 get_inline_info(fi, ri);
116 142
143 /* check data exist */
144 if (f2fs_has_inline_data(inode) && !f2fs_exist_data(inode))
145 err = __recover_inline_status(inode, node_page);
146
117 /* get rdev by using inline_info */ 147 /* get rdev by using inline_info */
118 __get_inode_rdev(inode, ri); 148 __get_inode_rdev(inode, ri);
119 149
120 f2fs_put_page(node_page, 1); 150 f2fs_put_page(node_page, 1);
121 return 0; 151
152 stat_inc_inline_inode(inode);
153 stat_inc_inline_dir(inode);
154
155 return err;
122} 156}
123 157
124struct inode *f2fs_iget(struct super_block *sb, unsigned long ino) 158struct inode *f2fs_iget(struct super_block *sb, unsigned long ino)
@@ -156,7 +190,7 @@ make_now:
156 inode->i_op = &f2fs_dir_inode_operations; 190 inode->i_op = &f2fs_dir_inode_operations;
157 inode->i_fop = &f2fs_dir_operations; 191 inode->i_fop = &f2fs_dir_operations;
158 inode->i_mapping->a_ops = &f2fs_dblock_aops; 192 inode->i_mapping->a_ops = &f2fs_dblock_aops;
159 mapping_set_gfp_mask(inode->i_mapping, GFP_F2FS_ZERO); 193 mapping_set_gfp_mask(inode->i_mapping, GFP_F2FS_HIGH_ZERO);
160 } else if (S_ISLNK(inode->i_mode)) { 194 } else if (S_ISLNK(inode->i_mode)) {
161 inode->i_op = &f2fs_symlink_inode_operations; 195 inode->i_op = &f2fs_symlink_inode_operations;
162 inode->i_mapping->a_ops = &f2fs_dblock_aops; 196 inode->i_mapping->a_ops = &f2fs_dblock_aops;
@@ -295,11 +329,12 @@ void f2fs_evict_inode(struct inode *inode)
295 329
296 f2fs_lock_op(sbi); 330 f2fs_lock_op(sbi);
297 remove_inode_page(inode); 331 remove_inode_page(inode);
298 stat_dec_inline_inode(inode);
299 f2fs_unlock_op(sbi); 332 f2fs_unlock_op(sbi);
300 333
301 sb_end_intwrite(inode->i_sb); 334 sb_end_intwrite(inode->i_sb);
302no_delete: 335no_delete:
336 stat_dec_inline_dir(inode);
337 stat_dec_inline_inode(inode);
303 invalidate_mapping_pages(NODE_MAPPING(sbi), inode->i_ino, inode->i_ino); 338 invalidate_mapping_pages(NODE_MAPPING(sbi), inode->i_ino, inode->i_ino);
304 if (xnid) 339 if (xnid)
305 invalidate_mapping_pages(NODE_MAPPING(sbi), xnid, xnid); 340 invalidate_mapping_pages(NODE_MAPPING(sbi), xnid, xnid);
@@ -325,8 +360,9 @@ void handle_failed_inode(struct inode *inode)
325 f2fs_truncate(inode); 360 f2fs_truncate(inode);
326 361
327 remove_inode_page(inode); 362 remove_inode_page(inode);
328 stat_dec_inline_inode(inode);
329 363
364 clear_inode_flag(F2FS_I(inode), FI_INLINE_DATA);
365 clear_inode_flag(F2FS_I(inode), FI_INLINE_DENTRY);
330 alloc_nid_failed(sbi, inode->i_ino); 366 alloc_nid_failed(sbi, inode->i_ino);
331 f2fs_unlock_op(sbi); 367 f2fs_unlock_op(sbi);
332 368
diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c
index 0d2526e5aa11..547a2deeb1ac 100644
--- a/fs/f2fs/namei.c
+++ b/fs/f2fs/namei.c
@@ -54,6 +54,12 @@ static struct inode *f2fs_new_inode(struct inode *dir, umode_t mode)
54 nid_free = true; 54 nid_free = true;
55 goto out; 55 goto out;
56 } 56 }
57
58 if (f2fs_may_inline(inode))
59 set_inode_flag(F2FS_I(inode), FI_INLINE_DATA);
60 if (test_opt(sbi, INLINE_DENTRY) && S_ISDIR(inode->i_mode))
61 set_inode_flag(F2FS_I(inode), FI_INLINE_DENTRY);
62
57 trace_f2fs_new_inode(inode, 0); 63 trace_f2fs_new_inode(inode, 0);
58 mark_inode_dirty(inode); 64 mark_inode_dirty(inode);
59 return inode; 65 return inode;
@@ -129,8 +135,12 @@ static int f2fs_create(struct inode *dir, struct dentry *dentry, umode_t mode,
129 135
130 alloc_nid_done(sbi, ino); 136 alloc_nid_done(sbi, ino);
131 137
138 stat_inc_inline_inode(inode);
132 d_instantiate(dentry, inode); 139 d_instantiate(dentry, inode);
133 unlock_new_inode(inode); 140 unlock_new_inode(inode);
141
142 if (IS_DIRSYNC(dir))
143 f2fs_sync_fs(sbi->sb, 1);
134 return 0; 144 return 0;
135out: 145out:
136 handle_failed_inode(inode); 146 handle_failed_inode(inode);
@@ -157,6 +167,9 @@ static int f2fs_link(struct dentry *old_dentry, struct inode *dir,
157 f2fs_unlock_op(sbi); 167 f2fs_unlock_op(sbi);
158 168
159 d_instantiate(dentry, inode); 169 d_instantiate(dentry, inode);
170
171 if (IS_DIRSYNC(dir))
172 f2fs_sync_fs(sbi->sb, 1);
160 return 0; 173 return 0;
161out: 174out:
162 clear_inode_flag(F2FS_I(inode), FI_INC_LINK); 175 clear_inode_flag(F2FS_I(inode), FI_INC_LINK);
@@ -187,14 +200,12 @@ static struct dentry *f2fs_lookup(struct inode *dir, struct dentry *dentry,
187 de = f2fs_find_entry(dir, &dentry->d_name, &page); 200 de = f2fs_find_entry(dir, &dentry->d_name, &page);
188 if (de) { 201 if (de) {
189 nid_t ino = le32_to_cpu(de->ino); 202 nid_t ino = le32_to_cpu(de->ino);
190 kunmap(page); 203 f2fs_dentry_kunmap(dir, page);
191 f2fs_put_page(page, 0); 204 f2fs_put_page(page, 0);
192 205
193 inode = f2fs_iget(dir->i_sb, ino); 206 inode = f2fs_iget(dir->i_sb, ino);
194 if (IS_ERR(inode)) 207 if (IS_ERR(inode))
195 return ERR_CAST(inode); 208 return ERR_CAST(inode);
196
197 stat_inc_inline_inode(inode);
198 } 209 }
199 210
200 return d_splice_alias(inode, dentry); 211 return d_splice_alias(inode, dentry);
@@ -219,15 +230,18 @@ static int f2fs_unlink(struct inode *dir, struct dentry *dentry)
219 err = acquire_orphan_inode(sbi); 230 err = acquire_orphan_inode(sbi);
220 if (err) { 231 if (err) {
221 f2fs_unlock_op(sbi); 232 f2fs_unlock_op(sbi);
222 kunmap(page); 233 f2fs_dentry_kunmap(dir, page);
223 f2fs_put_page(page, 0); 234 f2fs_put_page(page, 0);
224 goto fail; 235 goto fail;
225 } 236 }
226 f2fs_delete_entry(de, page, inode); 237 f2fs_delete_entry(de, page, dir, inode);
227 f2fs_unlock_op(sbi); 238 f2fs_unlock_op(sbi);
228 239
229 /* In order to evict this inode, we set it dirty */ 240 /* In order to evict this inode, we set it dirty */
230 mark_inode_dirty(inode); 241 mark_inode_dirty(inode);
242
243 if (IS_DIRSYNC(dir))
244 f2fs_sync_fs(sbi->sb, 1);
231fail: 245fail:
232 trace_f2fs_unlink_exit(inode, err); 246 trace_f2fs_unlink_exit(inode, err);
233 return err; 247 return err;
@@ -261,6 +275,9 @@ static int f2fs_symlink(struct inode *dir, struct dentry *dentry,
261 275
262 d_instantiate(dentry, inode); 276 d_instantiate(dentry, inode);
263 unlock_new_inode(inode); 277 unlock_new_inode(inode);
278
279 if (IS_DIRSYNC(dir))
280 f2fs_sync_fs(sbi->sb, 1);
264 return err; 281 return err;
265out: 282out:
266 handle_failed_inode(inode); 283 handle_failed_inode(inode);
@@ -291,11 +308,14 @@ static int f2fs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
291 goto out_fail; 308 goto out_fail;
292 f2fs_unlock_op(sbi); 309 f2fs_unlock_op(sbi);
293 310
311 stat_inc_inline_dir(inode);
294 alloc_nid_done(sbi, inode->i_ino); 312 alloc_nid_done(sbi, inode->i_ino);
295 313
296 d_instantiate(dentry, inode); 314 d_instantiate(dentry, inode);
297 unlock_new_inode(inode); 315 unlock_new_inode(inode);
298 316
317 if (IS_DIRSYNC(dir))
318 f2fs_sync_fs(sbi->sb, 1);
299 return 0; 319 return 0;
300 320
301out_fail: 321out_fail:
@@ -338,8 +358,12 @@ static int f2fs_mknod(struct inode *dir, struct dentry *dentry,
338 f2fs_unlock_op(sbi); 358 f2fs_unlock_op(sbi);
339 359
340 alloc_nid_done(sbi, inode->i_ino); 360 alloc_nid_done(sbi, inode->i_ino);
361
341 d_instantiate(dentry, inode); 362 d_instantiate(dentry, inode);
342 unlock_new_inode(inode); 363 unlock_new_inode(inode);
364
365 if (IS_DIRSYNC(dir))
366 f2fs_sync_fs(sbi->sb, 1);
343 return 0; 367 return 0;
344out: 368out:
345 handle_failed_inode(inode); 369 handle_failed_inode(inode);
@@ -435,7 +459,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
435 old_inode->i_ctime = CURRENT_TIME; 459 old_inode->i_ctime = CURRENT_TIME;
436 mark_inode_dirty(old_inode); 460 mark_inode_dirty(old_inode);
437 461
438 f2fs_delete_entry(old_entry, old_page, NULL); 462 f2fs_delete_entry(old_entry, old_page, old_dir, NULL);
439 463
440 if (old_dir_entry) { 464 if (old_dir_entry) {
441 if (old_dir != new_dir) { 465 if (old_dir != new_dir) {
@@ -443,7 +467,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
443 old_dir_page, new_dir); 467 old_dir_page, new_dir);
444 update_inode_page(old_inode); 468 update_inode_page(old_inode);
445 } else { 469 } else {
446 kunmap(old_dir_page); 470 f2fs_dentry_kunmap(old_inode, old_dir_page);
447 f2fs_put_page(old_dir_page, 0); 471 f2fs_put_page(old_dir_page, 0);
448 } 472 }
449 drop_nlink(old_dir); 473 drop_nlink(old_dir);
@@ -452,19 +476,22 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
452 } 476 }
453 477
454 f2fs_unlock_op(sbi); 478 f2fs_unlock_op(sbi);
479
480 if (IS_DIRSYNC(old_dir) || IS_DIRSYNC(new_dir))
481 f2fs_sync_fs(sbi->sb, 1);
455 return 0; 482 return 0;
456 483
457put_out_dir: 484put_out_dir:
458 f2fs_unlock_op(sbi); 485 f2fs_unlock_op(sbi);
459 kunmap(new_page); 486 f2fs_dentry_kunmap(new_dir, new_page);
460 f2fs_put_page(new_page, 0); 487 f2fs_put_page(new_page, 0);
461out_dir: 488out_dir:
462 if (old_dir_entry) { 489 if (old_dir_entry) {
463 kunmap(old_dir_page); 490 f2fs_dentry_kunmap(old_inode, old_dir_page);
464 f2fs_put_page(old_dir_page, 0); 491 f2fs_put_page(old_dir_page, 0);
465 } 492 }
466out_old: 493out_old:
467 kunmap(old_page); 494 f2fs_dentry_kunmap(old_dir, old_page);
468 f2fs_put_page(old_page, 0); 495 f2fs_put_page(old_page, 0);
469out: 496out:
470 return err; 497 return err;
@@ -588,6 +615,9 @@ static int f2fs_cross_rename(struct inode *old_dir, struct dentry *old_dentry,
588 update_inode_page(new_dir); 615 update_inode_page(new_dir);
589 616
590 f2fs_unlock_op(sbi); 617 f2fs_unlock_op(sbi);
618
619 if (IS_DIRSYNC(old_dir) || IS_DIRSYNC(new_dir))
620 f2fs_sync_fs(sbi->sb, 1);
591 return 0; 621 return 0;
592out_undo: 622out_undo:
593 /* Still we may fail to recover name info of f2fs_inode here */ 623 /* Still we may fail to recover name info of f2fs_inode here */
@@ -596,19 +626,19 @@ out_unlock:
596 f2fs_unlock_op(sbi); 626 f2fs_unlock_op(sbi);
597out_new_dir: 627out_new_dir:
598 if (new_dir_entry) { 628 if (new_dir_entry) {
599 kunmap(new_dir_page); 629 f2fs_dentry_kunmap(new_inode, new_dir_page);
600 f2fs_put_page(new_dir_page, 0); 630 f2fs_put_page(new_dir_page, 0);
601 } 631 }
602out_old_dir: 632out_old_dir:
603 if (old_dir_entry) { 633 if (old_dir_entry) {
604 kunmap(old_dir_page); 634 f2fs_dentry_kunmap(old_inode, old_dir_page);
605 f2fs_put_page(old_dir_page, 0); 635 f2fs_put_page(old_dir_page, 0);
606 } 636 }
607out_new: 637out_new:
608 kunmap(new_page); 638 f2fs_dentry_kunmap(new_dir, new_page);
609 f2fs_put_page(new_page, 0); 639 f2fs_put_page(new_page, 0);
610out_old: 640out_old:
611 kunmap(old_page); 641 f2fs_dentry_kunmap(old_dir, old_page);
612 f2fs_put_page(old_page, 0); 642 f2fs_put_page(old_page, 0);
613out: 643out:
614 return err; 644 return err;
diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c
index 44b8afef43d9..f83326ca32ef 100644
--- a/fs/f2fs/node.c
+++ b/fs/f2fs/node.c
@@ -31,22 +31,38 @@ bool available_free_memory(struct f2fs_sb_info *sbi, int type)
31{ 31{
32 struct f2fs_nm_info *nm_i = NM_I(sbi); 32 struct f2fs_nm_info *nm_i = NM_I(sbi);
33 struct sysinfo val; 33 struct sysinfo val;
34 unsigned long avail_ram;
34 unsigned long mem_size = 0; 35 unsigned long mem_size = 0;
35 bool res = false; 36 bool res = false;
36 37
37 si_meminfo(&val); 38 si_meminfo(&val);
38 /* give 25%, 25%, 50% memory for each components respectively */ 39
40 /* only uses low memory */
41 avail_ram = val.totalram - val.totalhigh;
42
43 /* give 25%, 25%, 50%, 50% memory for each components respectively */
39 if (type == FREE_NIDS) { 44 if (type == FREE_NIDS) {
40 mem_size = (nm_i->fcnt * sizeof(struct free_nid)) >> 12; 45 mem_size = (nm_i->fcnt * sizeof(struct free_nid)) >>
41 res = mem_size < ((val.totalram * nm_i->ram_thresh / 100) >> 2); 46 PAGE_CACHE_SHIFT;
47 res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2);
42 } else if (type == NAT_ENTRIES) { 48 } else if (type == NAT_ENTRIES) {
43 mem_size = (nm_i->nat_cnt * sizeof(struct nat_entry)) >> 12; 49 mem_size = (nm_i->nat_cnt * sizeof(struct nat_entry)) >>
44 res = mem_size < ((val.totalram * nm_i->ram_thresh / 100) >> 2); 50 PAGE_CACHE_SHIFT;
51 res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 2);
45 } else if (type == DIRTY_DENTS) { 52 } else if (type == DIRTY_DENTS) {
46 if (sbi->sb->s_bdi->dirty_exceeded) 53 if (sbi->sb->s_bdi->dirty_exceeded)
47 return false; 54 return false;
48 mem_size = get_pages(sbi, F2FS_DIRTY_DENTS); 55 mem_size = get_pages(sbi, F2FS_DIRTY_DENTS);
49 res = mem_size < ((val.totalram * nm_i->ram_thresh / 100) >> 1); 56 res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1);
57 } else if (type == INO_ENTRIES) {
58 int i;
59
60 if (sbi->sb->s_bdi->dirty_exceeded)
61 return false;
62 for (i = 0; i <= UPDATE_INO; i++)
63 mem_size += (sbi->im[i].ino_num *
64 sizeof(struct ino_entry)) >> PAGE_CACHE_SHIFT;
65 res = mem_size < ((avail_ram * nm_i->ram_thresh / 100) >> 1);
50 } 66 }
51 return res; 67 return res;
52} 68}
@@ -131,7 +147,7 @@ static void __set_nat_cache_dirty(struct f2fs_nm_info *nm_i,
131 147
132 if (get_nat_flag(ne, IS_DIRTY)) 148 if (get_nat_flag(ne, IS_DIRTY))
133 return; 149 return;
134retry: 150
135 head = radix_tree_lookup(&nm_i->nat_set_root, set); 151 head = radix_tree_lookup(&nm_i->nat_set_root, set);
136 if (!head) { 152 if (!head) {
137 head = f2fs_kmem_cache_alloc(nat_entry_set_slab, GFP_ATOMIC); 153 head = f2fs_kmem_cache_alloc(nat_entry_set_slab, GFP_ATOMIC);
@@ -140,11 +156,7 @@ retry:
140 INIT_LIST_HEAD(&head->set_list); 156 INIT_LIST_HEAD(&head->set_list);
141 head->set = set; 157 head->set = set;
142 head->entry_cnt = 0; 158 head->entry_cnt = 0;
143 159 f2fs_radix_tree_insert(&nm_i->nat_set_root, set, head);
144 if (radix_tree_insert(&nm_i->nat_set_root, set, head)) {
145 cond_resched();
146 goto retry;
147 }
148 } 160 }
149 list_move_tail(&ne->list, &head->entry_list); 161 list_move_tail(&ne->list, &head->entry_list);
150 nm_i->dirty_nat_cnt++; 162 nm_i->dirty_nat_cnt++;
@@ -155,7 +167,7 @@ retry:
155static void __clear_nat_cache_dirty(struct f2fs_nm_info *nm_i, 167static void __clear_nat_cache_dirty(struct f2fs_nm_info *nm_i,
156 struct nat_entry *ne) 168 struct nat_entry *ne)
157{ 169{
158 nid_t set = ne->ni.nid / NAT_ENTRY_PER_BLOCK; 170 nid_t set = NAT_BLOCK_OFFSET(ne->ni.nid);
159 struct nat_entry_set *head; 171 struct nat_entry_set *head;
160 172
161 head = radix_tree_lookup(&nm_i->nat_set_root, set); 173 head = radix_tree_lookup(&nm_i->nat_set_root, set);
@@ -180,11 +192,11 @@ bool is_checkpointed_node(struct f2fs_sb_info *sbi, nid_t nid)
180 struct nat_entry *e; 192 struct nat_entry *e;
181 bool is_cp = true; 193 bool is_cp = true;
182 194
183 read_lock(&nm_i->nat_tree_lock); 195 down_read(&nm_i->nat_tree_lock);
184 e = __lookup_nat_cache(nm_i, nid); 196 e = __lookup_nat_cache(nm_i, nid);
185 if (e && !get_nat_flag(e, IS_CHECKPOINTED)) 197 if (e && !get_nat_flag(e, IS_CHECKPOINTED))
186 is_cp = false; 198 is_cp = false;
187 read_unlock(&nm_i->nat_tree_lock); 199 up_read(&nm_i->nat_tree_lock);
188 return is_cp; 200 return is_cp;
189} 201}
190 202
@@ -194,11 +206,11 @@ bool has_fsynced_inode(struct f2fs_sb_info *sbi, nid_t ino)
194 struct nat_entry *e; 206 struct nat_entry *e;
195 bool fsynced = false; 207 bool fsynced = false;
196 208
197 read_lock(&nm_i->nat_tree_lock); 209 down_read(&nm_i->nat_tree_lock);
198 e = __lookup_nat_cache(nm_i, ino); 210 e = __lookup_nat_cache(nm_i, ino);
199 if (e && get_nat_flag(e, HAS_FSYNCED_INODE)) 211 if (e && get_nat_flag(e, HAS_FSYNCED_INODE))
200 fsynced = true; 212 fsynced = true;
201 read_unlock(&nm_i->nat_tree_lock); 213 up_read(&nm_i->nat_tree_lock);
202 return fsynced; 214 return fsynced;
203} 215}
204 216
@@ -208,13 +220,13 @@ bool need_inode_block_update(struct f2fs_sb_info *sbi, nid_t ino)
208 struct nat_entry *e; 220 struct nat_entry *e;
209 bool need_update = true; 221 bool need_update = true;
210 222
211 read_lock(&nm_i->nat_tree_lock); 223 down_read(&nm_i->nat_tree_lock);
212 e = __lookup_nat_cache(nm_i, ino); 224 e = __lookup_nat_cache(nm_i, ino);
213 if (e && get_nat_flag(e, HAS_LAST_FSYNC) && 225 if (e && get_nat_flag(e, HAS_LAST_FSYNC) &&
214 (get_nat_flag(e, IS_CHECKPOINTED) || 226 (get_nat_flag(e, IS_CHECKPOINTED) ||
215 get_nat_flag(e, HAS_FSYNCED_INODE))) 227 get_nat_flag(e, HAS_FSYNCED_INODE)))
216 need_update = false; 228 need_update = false;
217 read_unlock(&nm_i->nat_tree_lock); 229 up_read(&nm_i->nat_tree_lock);
218 return need_update; 230 return need_update;
219} 231}
220 232
@@ -222,13 +234,8 @@ static struct nat_entry *grab_nat_entry(struct f2fs_nm_info *nm_i, nid_t nid)
222{ 234{
223 struct nat_entry *new; 235 struct nat_entry *new;
224 236
225 new = kmem_cache_alloc(nat_entry_slab, GFP_ATOMIC); 237 new = f2fs_kmem_cache_alloc(nat_entry_slab, GFP_ATOMIC);
226 if (!new) 238 f2fs_radix_tree_insert(&nm_i->nat_root, nid, new);
227 return NULL;
228 if (radix_tree_insert(&nm_i->nat_root, nid, new)) {
229 kmem_cache_free(nat_entry_slab, new);
230 return NULL;
231 }
232 memset(new, 0, sizeof(struct nat_entry)); 239 memset(new, 0, sizeof(struct nat_entry));
233 nat_set_nid(new, nid); 240 nat_set_nid(new, nid);
234 nat_reset_flag(new); 241 nat_reset_flag(new);
@@ -241,18 +248,14 @@ static void cache_nat_entry(struct f2fs_nm_info *nm_i, nid_t nid,
241 struct f2fs_nat_entry *ne) 248 struct f2fs_nat_entry *ne)
242{ 249{
243 struct nat_entry *e; 250 struct nat_entry *e;
244retry: 251
245 write_lock(&nm_i->nat_tree_lock); 252 down_write(&nm_i->nat_tree_lock);
246 e = __lookup_nat_cache(nm_i, nid); 253 e = __lookup_nat_cache(nm_i, nid);
247 if (!e) { 254 if (!e) {
248 e = grab_nat_entry(nm_i, nid); 255 e = grab_nat_entry(nm_i, nid);
249 if (!e) {
250 write_unlock(&nm_i->nat_tree_lock);
251 goto retry;
252 }
253 node_info_from_raw_nat(&e->ni, ne); 256 node_info_from_raw_nat(&e->ni, ne);
254 } 257 }
255 write_unlock(&nm_i->nat_tree_lock); 258 up_write(&nm_i->nat_tree_lock);
256} 259}
257 260
258static void set_node_addr(struct f2fs_sb_info *sbi, struct node_info *ni, 261static void set_node_addr(struct f2fs_sb_info *sbi, struct node_info *ni,
@@ -260,15 +263,11 @@ static void set_node_addr(struct f2fs_sb_info *sbi, struct node_info *ni,
260{ 263{
261 struct f2fs_nm_info *nm_i = NM_I(sbi); 264 struct f2fs_nm_info *nm_i = NM_I(sbi);
262 struct nat_entry *e; 265 struct nat_entry *e;
263retry: 266
264 write_lock(&nm_i->nat_tree_lock); 267 down_write(&nm_i->nat_tree_lock);
265 e = __lookup_nat_cache(nm_i, ni->nid); 268 e = __lookup_nat_cache(nm_i, ni->nid);
266 if (!e) { 269 if (!e) {
267 e = grab_nat_entry(nm_i, ni->nid); 270 e = grab_nat_entry(nm_i, ni->nid);
268 if (!e) {
269 write_unlock(&nm_i->nat_tree_lock);
270 goto retry;
271 }
272 e->ni = *ni; 271 e->ni = *ni;
273 f2fs_bug_on(sbi, ni->blk_addr == NEW_ADDR); 272 f2fs_bug_on(sbi, ni->blk_addr == NEW_ADDR);
274 } else if (new_blkaddr == NEW_ADDR) { 273 } else if (new_blkaddr == NEW_ADDR) {
@@ -310,7 +309,7 @@ retry:
310 set_nat_flag(e, HAS_FSYNCED_INODE, true); 309 set_nat_flag(e, HAS_FSYNCED_INODE, true);
311 set_nat_flag(e, HAS_LAST_FSYNC, fsync_done); 310 set_nat_flag(e, HAS_LAST_FSYNC, fsync_done);
312 } 311 }
313 write_unlock(&nm_i->nat_tree_lock); 312 up_write(&nm_i->nat_tree_lock);
314} 313}
315 314
316int try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink) 315int try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink)
@@ -320,7 +319,7 @@ int try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink)
320 if (available_free_memory(sbi, NAT_ENTRIES)) 319 if (available_free_memory(sbi, NAT_ENTRIES))
321 return 0; 320 return 0;
322 321
323 write_lock(&nm_i->nat_tree_lock); 322 down_write(&nm_i->nat_tree_lock);
324 while (nr_shrink && !list_empty(&nm_i->nat_entries)) { 323 while (nr_shrink && !list_empty(&nm_i->nat_entries)) {
325 struct nat_entry *ne; 324 struct nat_entry *ne;
326 ne = list_first_entry(&nm_i->nat_entries, 325 ne = list_first_entry(&nm_i->nat_entries,
@@ -328,7 +327,7 @@ int try_to_free_nats(struct f2fs_sb_info *sbi, int nr_shrink)
328 __del_from_nat_cache(nm_i, ne); 327 __del_from_nat_cache(nm_i, ne);
329 nr_shrink--; 328 nr_shrink--;
330 } 329 }
331 write_unlock(&nm_i->nat_tree_lock); 330 up_write(&nm_i->nat_tree_lock);
332 return nr_shrink; 331 return nr_shrink;
333} 332}
334 333
@@ -351,14 +350,14 @@ void get_node_info(struct f2fs_sb_info *sbi, nid_t nid, struct node_info *ni)
351 ni->nid = nid; 350 ni->nid = nid;
352 351
353 /* Check nat cache */ 352 /* Check nat cache */
354 read_lock(&nm_i->nat_tree_lock); 353 down_read(&nm_i->nat_tree_lock);
355 e = __lookup_nat_cache(nm_i, nid); 354 e = __lookup_nat_cache(nm_i, nid);
356 if (e) { 355 if (e) {
357 ni->ino = nat_get_ino(e); 356 ni->ino = nat_get_ino(e);
358 ni->blk_addr = nat_get_blkaddr(e); 357 ni->blk_addr = nat_get_blkaddr(e);
359 ni->version = nat_get_version(e); 358 ni->version = nat_get_version(e);
360 } 359 }
361 read_unlock(&nm_i->nat_tree_lock); 360 up_read(&nm_i->nat_tree_lock);
362 if (e) 361 if (e)
363 return; 362 return;
364 363
@@ -1298,16 +1297,22 @@ static int f2fs_write_node_page(struct page *page,
1298 return 0; 1297 return 0;
1299 } 1298 }
1300 1299
1301 if (wbc->for_reclaim) 1300 if (wbc->for_reclaim) {
1302 goto redirty_out; 1301 if (!down_read_trylock(&sbi->node_write))
1303 1302 goto redirty_out;
1304 down_read(&sbi->node_write); 1303 } else {
1304 down_read(&sbi->node_write);
1305 }
1305 set_page_writeback(page); 1306 set_page_writeback(page);
1306 write_node_page(sbi, page, &fio, nid, ni.blk_addr, &new_addr); 1307 write_node_page(sbi, page, &fio, nid, ni.blk_addr, &new_addr);
1307 set_node_addr(sbi, &ni, new_addr, is_fsync_dnode(page)); 1308 set_node_addr(sbi, &ni, new_addr, is_fsync_dnode(page));
1308 dec_page_count(sbi, F2FS_DIRTY_NODES); 1309 dec_page_count(sbi, F2FS_DIRTY_NODES);
1309 up_read(&sbi->node_write); 1310 up_read(&sbi->node_write);
1310 unlock_page(page); 1311 unlock_page(page);
1312
1313 if (wbc->for_reclaim)
1314 f2fs_submit_merged_bio(sbi, NODE, WRITE);
1315
1311 return 0; 1316 return 0;
1312 1317
1313redirty_out: 1318redirty_out:
@@ -1410,13 +1415,13 @@ static int add_free_nid(struct f2fs_sb_info *sbi, nid_t nid, bool build)
1410 1415
1411 if (build) { 1416 if (build) {
1412 /* do not add allocated nids */ 1417 /* do not add allocated nids */
1413 read_lock(&nm_i->nat_tree_lock); 1418 down_read(&nm_i->nat_tree_lock);
1414 ne = __lookup_nat_cache(nm_i, nid); 1419 ne = __lookup_nat_cache(nm_i, nid);
1415 if (ne && 1420 if (ne &&
1416 (!get_nat_flag(ne, IS_CHECKPOINTED) || 1421 (!get_nat_flag(ne, IS_CHECKPOINTED) ||
1417 nat_get_blkaddr(ne) != NULL_ADDR)) 1422 nat_get_blkaddr(ne) != NULL_ADDR))
1418 allocated = true; 1423 allocated = true;
1419 read_unlock(&nm_i->nat_tree_lock); 1424 up_read(&nm_i->nat_tree_lock);
1420 if (allocated) 1425 if (allocated)
1421 return 0; 1426 return 0;
1422 } 1427 }
@@ -1425,15 +1430,22 @@ static int add_free_nid(struct f2fs_sb_info *sbi, nid_t nid, bool build)
1425 i->nid = nid; 1430 i->nid = nid;
1426 i->state = NID_NEW; 1431 i->state = NID_NEW;
1427 1432
1433 if (radix_tree_preload(GFP_NOFS)) {
1434 kmem_cache_free(free_nid_slab, i);
1435 return 0;
1436 }
1437
1428 spin_lock(&nm_i->free_nid_list_lock); 1438 spin_lock(&nm_i->free_nid_list_lock);
1429 if (radix_tree_insert(&nm_i->free_nid_root, i->nid, i)) { 1439 if (radix_tree_insert(&nm_i->free_nid_root, i->nid, i)) {
1430 spin_unlock(&nm_i->free_nid_list_lock); 1440 spin_unlock(&nm_i->free_nid_list_lock);
1441 radix_tree_preload_end();
1431 kmem_cache_free(free_nid_slab, i); 1442 kmem_cache_free(free_nid_slab, i);
1432 return 0; 1443 return 0;
1433 } 1444 }
1434 list_add_tail(&i->list, &nm_i->free_nid_list); 1445 list_add_tail(&i->list, &nm_i->free_nid_list);
1435 nm_i->fcnt++; 1446 nm_i->fcnt++;
1436 spin_unlock(&nm_i->free_nid_list_lock); 1447 spin_unlock(&nm_i->free_nid_list_lock);
1448 radix_tree_preload_end();
1437 return 1; 1449 return 1;
1438} 1450}
1439 1451
@@ -1804,21 +1816,15 @@ static void remove_nats_in_journal(struct f2fs_sb_info *sbi)
1804 nid_t nid = le32_to_cpu(nid_in_journal(sum, i)); 1816 nid_t nid = le32_to_cpu(nid_in_journal(sum, i));
1805 1817
1806 raw_ne = nat_in_journal(sum, i); 1818 raw_ne = nat_in_journal(sum, i);
1807retry:
1808 write_lock(&nm_i->nat_tree_lock);
1809 ne = __lookup_nat_cache(nm_i, nid);
1810 if (ne)
1811 goto found;
1812 1819
1813 ne = grab_nat_entry(nm_i, nid); 1820 down_write(&nm_i->nat_tree_lock);
1821 ne = __lookup_nat_cache(nm_i, nid);
1814 if (!ne) { 1822 if (!ne) {
1815 write_unlock(&nm_i->nat_tree_lock); 1823 ne = grab_nat_entry(nm_i, nid);
1816 goto retry; 1824 node_info_from_raw_nat(&ne->ni, &raw_ne);
1817 } 1825 }
1818 node_info_from_raw_nat(&ne->ni, &raw_ne);
1819found:
1820 __set_nat_cache_dirty(nm_i, ne); 1826 __set_nat_cache_dirty(nm_i, ne);
1821 write_unlock(&nm_i->nat_tree_lock); 1827 up_write(&nm_i->nat_tree_lock);
1822 } 1828 }
1823 update_nats_in_cursum(sum, -i); 1829 update_nats_in_cursum(sum, -i);
1824 mutex_unlock(&curseg->curseg_mutex); 1830 mutex_unlock(&curseg->curseg_mutex);
@@ -1889,10 +1895,10 @@ static void __flush_nat_entry_set(struct f2fs_sb_info *sbi,
1889 } 1895 }
1890 raw_nat_from_node_info(raw_ne, &ne->ni); 1896 raw_nat_from_node_info(raw_ne, &ne->ni);
1891 1897
1892 write_lock(&NM_I(sbi)->nat_tree_lock); 1898 down_write(&NM_I(sbi)->nat_tree_lock);
1893 nat_reset_flag(ne); 1899 nat_reset_flag(ne);
1894 __clear_nat_cache_dirty(NM_I(sbi), ne); 1900 __clear_nat_cache_dirty(NM_I(sbi), ne);
1895 write_unlock(&NM_I(sbi)->nat_tree_lock); 1901 up_write(&NM_I(sbi)->nat_tree_lock);
1896 1902
1897 if (nat_get_blkaddr(ne) == NULL_ADDR) 1903 if (nat_get_blkaddr(ne) == NULL_ADDR)
1898 add_free_nid(sbi, nid, false); 1904 add_free_nid(sbi, nid, false);
@@ -1903,10 +1909,10 @@ static void __flush_nat_entry_set(struct f2fs_sb_info *sbi,
1903 else 1909 else
1904 f2fs_put_page(page, 1); 1910 f2fs_put_page(page, 1);
1905 1911
1906 if (!set->entry_cnt) { 1912 f2fs_bug_on(sbi, set->entry_cnt);
1907 radix_tree_delete(&NM_I(sbi)->nat_set_root, set->set); 1913
1908 kmem_cache_free(nat_entry_set_slab, set); 1914 radix_tree_delete(&NM_I(sbi)->nat_set_root, set->set);
1909 } 1915 kmem_cache_free(nat_entry_set_slab, set);
1910} 1916}
1911 1917
1912/* 1918/*
@@ -1923,6 +1929,8 @@ void flush_nat_entries(struct f2fs_sb_info *sbi)
1923 nid_t set_idx = 0; 1929 nid_t set_idx = 0;
1924 LIST_HEAD(sets); 1930 LIST_HEAD(sets);
1925 1931
1932 if (!nm_i->dirty_nat_cnt)
1933 return;
1926 /* 1934 /*
1927 * if there are no enough space in journal to store dirty nat 1935 * if there are no enough space in journal to store dirty nat
1928 * entries, remove all entries from journal and merge them 1936 * entries, remove all entries from journal and merge them
@@ -1931,9 +1939,6 @@ void flush_nat_entries(struct f2fs_sb_info *sbi)
1931 if (!__has_cursum_space(sum, nm_i->dirty_nat_cnt, NAT_JOURNAL)) 1939 if (!__has_cursum_space(sum, nm_i->dirty_nat_cnt, NAT_JOURNAL))
1932 remove_nats_in_journal(sbi); 1940 remove_nats_in_journal(sbi);
1933 1941
1934 if (!nm_i->dirty_nat_cnt)
1935 return;
1936
1937 while ((found = __gang_lookup_nat_set(nm_i, 1942 while ((found = __gang_lookup_nat_set(nm_i,
1938 set_idx, NATVEC_SIZE, setvec))) { 1943 set_idx, NATVEC_SIZE, setvec))) {
1939 unsigned idx; 1944 unsigned idx;
@@ -1973,13 +1978,13 @@ static int init_node_manager(struct f2fs_sb_info *sbi)
1973 1978
1974 INIT_RADIX_TREE(&nm_i->free_nid_root, GFP_ATOMIC); 1979 INIT_RADIX_TREE(&nm_i->free_nid_root, GFP_ATOMIC);
1975 INIT_LIST_HEAD(&nm_i->free_nid_list); 1980 INIT_LIST_HEAD(&nm_i->free_nid_list);
1976 INIT_RADIX_TREE(&nm_i->nat_root, GFP_ATOMIC); 1981 INIT_RADIX_TREE(&nm_i->nat_root, GFP_NOIO);
1977 INIT_RADIX_TREE(&nm_i->nat_set_root, GFP_ATOMIC); 1982 INIT_RADIX_TREE(&nm_i->nat_set_root, GFP_NOIO);
1978 INIT_LIST_HEAD(&nm_i->nat_entries); 1983 INIT_LIST_HEAD(&nm_i->nat_entries);
1979 1984
1980 mutex_init(&nm_i->build_lock); 1985 mutex_init(&nm_i->build_lock);
1981 spin_lock_init(&nm_i->free_nid_list_lock); 1986 spin_lock_init(&nm_i->free_nid_list_lock);
1982 rwlock_init(&nm_i->nat_tree_lock); 1987 init_rwsem(&nm_i->nat_tree_lock);
1983 1988
1984 nm_i->next_scan_nid = le32_to_cpu(sbi->ckpt->next_free_nid); 1989 nm_i->next_scan_nid = le32_to_cpu(sbi->ckpt->next_free_nid);
1985 nm_i->bitmap_size = __bitmap_size(sbi, NAT_BITMAP); 1990 nm_i->bitmap_size = __bitmap_size(sbi, NAT_BITMAP);
@@ -2035,7 +2040,7 @@ void destroy_node_manager(struct f2fs_sb_info *sbi)
2035 spin_unlock(&nm_i->free_nid_list_lock); 2040 spin_unlock(&nm_i->free_nid_list_lock);
2036 2041
2037 /* destroy nat cache */ 2042 /* destroy nat cache */
2038 write_lock(&nm_i->nat_tree_lock); 2043 down_write(&nm_i->nat_tree_lock);
2039 while ((found = __gang_lookup_nat_cache(nm_i, 2044 while ((found = __gang_lookup_nat_cache(nm_i,
2040 nid, NATVEC_SIZE, natvec))) { 2045 nid, NATVEC_SIZE, natvec))) {
2041 unsigned idx; 2046 unsigned idx;
@@ -2044,7 +2049,7 @@ void destroy_node_manager(struct f2fs_sb_info *sbi)
2044 __del_from_nat_cache(nm_i, natvec[idx]); 2049 __del_from_nat_cache(nm_i, natvec[idx]);
2045 } 2050 }
2046 f2fs_bug_on(sbi, nm_i->nat_cnt); 2051 f2fs_bug_on(sbi, nm_i->nat_cnt);
2047 write_unlock(&nm_i->nat_tree_lock); 2052 up_write(&nm_i->nat_tree_lock);
2048 2053
2049 kfree(nm_i->nat_bitmap); 2054 kfree(nm_i->nat_bitmap);
2050 sbi->nm_info = NULL; 2055 sbi->nm_info = NULL;
@@ -2061,17 +2066,17 @@ int __init create_node_manager_caches(void)
2061 free_nid_slab = f2fs_kmem_cache_create("free_nid", 2066 free_nid_slab = f2fs_kmem_cache_create("free_nid",
2062 sizeof(struct free_nid)); 2067 sizeof(struct free_nid));
2063 if (!free_nid_slab) 2068 if (!free_nid_slab)
2064 goto destory_nat_entry; 2069 goto destroy_nat_entry;
2065 2070
2066 nat_entry_set_slab = f2fs_kmem_cache_create("nat_entry_set", 2071 nat_entry_set_slab = f2fs_kmem_cache_create("nat_entry_set",
2067 sizeof(struct nat_entry_set)); 2072 sizeof(struct nat_entry_set));
2068 if (!nat_entry_set_slab) 2073 if (!nat_entry_set_slab)
2069 goto destory_free_nid; 2074 goto destroy_free_nid;
2070 return 0; 2075 return 0;
2071 2076
2072destory_free_nid: 2077destroy_free_nid:
2073 kmem_cache_destroy(free_nid_slab); 2078 kmem_cache_destroy(free_nid_slab);
2074destory_nat_entry: 2079destroy_nat_entry:
2075 kmem_cache_destroy(nat_entry_slab); 2080 kmem_cache_destroy(nat_entry_slab);
2076fail: 2081fail:
2077 return -ENOMEM; 2082 return -ENOMEM;
diff --git a/fs/f2fs/node.h b/fs/f2fs/node.h
index 8d5e6e0dd840..d10b6448a671 100644
--- a/fs/f2fs/node.h
+++ b/fs/f2fs/node.h
@@ -106,7 +106,8 @@ static inline void raw_nat_from_node_info(struct f2fs_nat_entry *raw_ne,
106enum mem_type { 106enum mem_type {
107 FREE_NIDS, /* indicates the free nid list */ 107 FREE_NIDS, /* indicates the free nid list */
108 NAT_ENTRIES, /* indicates the cached nat entry */ 108 NAT_ENTRIES, /* indicates the cached nat entry */
109 DIRTY_DENTS /* indicates dirty dentry pages */ 109 DIRTY_DENTS, /* indicates dirty dentry pages */
110 INO_ENTRIES, /* indicates inode entries */
110}; 111};
111 112
112struct nat_entry_set { 113struct nat_entry_set {
@@ -192,10 +193,7 @@ static inline void set_to_next_nat(struct f2fs_nm_info *nm_i, nid_t start_nid)
192{ 193{
193 unsigned int block_off = NAT_BLOCK_OFFSET(start_nid); 194 unsigned int block_off = NAT_BLOCK_OFFSET(start_nid);
194 195
195 if (f2fs_test_bit(block_off, nm_i->nat_bitmap)) 196 f2fs_change_bit(block_off, nm_i->nat_bitmap);
196 f2fs_clear_bit(block_off, nm_i->nat_bitmap);
197 else
198 f2fs_set_bit(block_off, nm_i->nat_bitmap);
199} 197}
200 198
201static inline void fill_node_footer(struct page *page, nid_t nid, 199static inline void fill_node_footer(struct page *page, nid_t nid,
diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c
index ebd013225788..9160a37e1c7a 100644
--- a/fs/f2fs/recovery.c
+++ b/fs/f2fs/recovery.c
@@ -111,7 +111,7 @@ retry:
111 iput(einode); 111 iput(einode);
112 goto out_unmap_put; 112 goto out_unmap_put;
113 } 113 }
114 f2fs_delete_entry(de, page, einode); 114 f2fs_delete_entry(de, page, dir, einode);
115 iput(einode); 115 iput(einode);
116 goto retry; 116 goto retry;
117 } 117 }
@@ -129,7 +129,7 @@ retry:
129 goto out; 129 goto out;
130 130
131out_unmap_put: 131out_unmap_put:
132 kunmap(page); 132 f2fs_dentry_kunmap(dir, page);
133 f2fs_put_page(page, 0); 133 f2fs_put_page(page, 0);
134out_err: 134out_err:
135 iput(dir); 135 iput(dir);
@@ -170,13 +170,15 @@ static int find_fsync_dnodes(struct f2fs_sb_info *sbi, struct list_head *head)
170 curseg = CURSEG_I(sbi, CURSEG_WARM_NODE); 170 curseg = CURSEG_I(sbi, CURSEG_WARM_NODE);
171 blkaddr = NEXT_FREE_BLKADDR(sbi, curseg); 171 blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
172 172
173 ra_meta_pages(sbi, blkaddr, 1, META_POR);
174
173 while (1) { 175 while (1) {
174 struct fsync_inode_entry *entry; 176 struct fsync_inode_entry *entry;
175 177
176 if (blkaddr < MAIN_BLKADDR(sbi) || blkaddr >= MAX_BLKADDR(sbi)) 178 if (blkaddr < MAIN_BLKADDR(sbi) || blkaddr >= MAX_BLKADDR(sbi))
177 return 0; 179 return 0;
178 180
179 page = get_meta_page_ra(sbi, blkaddr); 181 page = get_meta_page(sbi, blkaddr);
180 182
181 if (cp_ver != cpver_of_node(page)) 183 if (cp_ver != cpver_of_node(page))
182 break; 184 break;
@@ -227,6 +229,8 @@ next:
227 /* check next segment */ 229 /* check next segment */
228 blkaddr = next_blkaddr_of_node(page); 230 blkaddr = next_blkaddr_of_node(page);
229 f2fs_put_page(page, 1); 231 f2fs_put_page(page, 1);
232
233 ra_meta_pages_cond(sbi, blkaddr);
230 } 234 }
231 f2fs_put_page(page, 1); 235 f2fs_put_page(page, 1);
232 return err; 236 return err;
@@ -436,7 +440,9 @@ static int recover_data(struct f2fs_sb_info *sbi,
436 if (blkaddr < MAIN_BLKADDR(sbi) || blkaddr >= MAX_BLKADDR(sbi)) 440 if (blkaddr < MAIN_BLKADDR(sbi) || blkaddr >= MAX_BLKADDR(sbi))
437 break; 441 break;
438 442
439 page = get_meta_page_ra(sbi, blkaddr); 443 ra_meta_pages_cond(sbi, blkaddr);
444
445 page = get_meta_page(sbi, blkaddr);
440 446
441 if (cp_ver != cpver_of_node(page)) { 447 if (cp_ver != cpver_of_node(page)) {
442 f2fs_put_page(page, 1); 448 f2fs_put_page(page, 1);
diff --git a/fs/f2fs/segment.c b/fs/f2fs/segment.c
index 923cb76fdc46..42607a679923 100644
--- a/fs/f2fs/segment.c
+++ b/fs/f2fs/segment.c
@@ -178,17 +178,47 @@ void register_inmem_page(struct inode *inode, struct page *page)
178{ 178{
179 struct f2fs_inode_info *fi = F2FS_I(inode); 179 struct f2fs_inode_info *fi = F2FS_I(inode);
180 struct inmem_pages *new; 180 struct inmem_pages *new;
181 int err;
182
183 SetPagePrivate(page);
181 184
182 new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS); 185 new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS);
183 186
184 /* add atomic page indices to the list */ 187 /* add atomic page indices to the list */
185 new->page = page; 188 new->page = page;
186 INIT_LIST_HEAD(&new->list); 189 INIT_LIST_HEAD(&new->list);
187 190retry:
188 /* increase reference count with clean state */ 191 /* increase reference count with clean state */
189 mutex_lock(&fi->inmem_lock); 192 mutex_lock(&fi->inmem_lock);
193 err = radix_tree_insert(&fi->inmem_root, page->index, new);
194 if (err == -EEXIST) {
195 mutex_unlock(&fi->inmem_lock);
196 kmem_cache_free(inmem_entry_slab, new);
197 return;
198 } else if (err) {
199 mutex_unlock(&fi->inmem_lock);
200 goto retry;
201 }
190 get_page(page); 202 get_page(page);
191 list_add_tail(&new->list, &fi->inmem_pages); 203 list_add_tail(&new->list, &fi->inmem_pages);
204 inc_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
205 mutex_unlock(&fi->inmem_lock);
206}
207
208void invalidate_inmem_page(struct inode *inode, struct page *page)
209{
210 struct f2fs_inode_info *fi = F2FS_I(inode);
211 struct inmem_pages *cur;
212
213 mutex_lock(&fi->inmem_lock);
214 cur = radix_tree_lookup(&fi->inmem_root, page->index);
215 if (cur) {
216 radix_tree_delete(&fi->inmem_root, cur->page->index);
217 f2fs_put_page(cur->page, 0);
218 list_del(&cur->list);
219 kmem_cache_free(inmem_entry_slab, cur);
220 dec_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
221 }
192 mutex_unlock(&fi->inmem_lock); 222 mutex_unlock(&fi->inmem_lock);
193} 223}
194 224
@@ -203,7 +233,16 @@ void commit_inmem_pages(struct inode *inode, bool abort)
203 .rw = WRITE_SYNC, 233 .rw = WRITE_SYNC,
204 }; 234 };
205 235
206 f2fs_balance_fs(sbi); 236 /*
237 * The abort is true only when f2fs_evict_inode is called.
238 * Basically, the f2fs_evict_inode doesn't produce any data writes, so
239 * that we don't need to call f2fs_balance_fs.
240 * Otherwise, f2fs_gc in f2fs_balance_fs can wait forever until this
241 * inode becomes free by iget_locked in f2fs_iget.
242 */
243 if (!abort)
244 f2fs_balance_fs(sbi);
245
207 f2fs_lock_op(sbi); 246 f2fs_lock_op(sbi);
208 247
209 mutex_lock(&fi->inmem_lock); 248 mutex_lock(&fi->inmem_lock);
@@ -216,9 +255,11 @@ void commit_inmem_pages(struct inode *inode, bool abort)
216 do_write_data_page(cur->page, &fio); 255 do_write_data_page(cur->page, &fio);
217 submit_bio = true; 256 submit_bio = true;
218 } 257 }
258 radix_tree_delete(&fi->inmem_root, cur->page->index);
219 f2fs_put_page(cur->page, 1); 259 f2fs_put_page(cur->page, 1);
220 list_del(&cur->list); 260 list_del(&cur->list);
221 kmem_cache_free(inmem_entry_slab, cur); 261 kmem_cache_free(inmem_entry_slab, cur);
262 dec_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
222 } 263 }
223 if (submit_bio) 264 if (submit_bio)
224 f2fs_submit_merged_bio(sbi, DATA, WRITE); 265 f2fs_submit_merged_bio(sbi, DATA, WRITE);
@@ -248,7 +289,8 @@ void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi)
248{ 289{
249 /* check the # of cached NAT entries and prefree segments */ 290 /* check the # of cached NAT entries and prefree segments */
250 if (try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK) || 291 if (try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK) ||
251 excess_prefree_segs(sbi)) 292 excess_prefree_segs(sbi) ||
293 available_free_memory(sbi, INO_ENTRIES))
252 f2fs_sync_fs(sbi->sb, true); 294 f2fs_sync_fs(sbi->sb, true);
253} 295}
254 296
@@ -441,10 +483,33 @@ void discard_next_dnode(struct f2fs_sb_info *sbi, block_t blkaddr)
441 } 483 }
442} 484}
443 485
444static void add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc) 486static void __add_discard_entry(struct f2fs_sb_info *sbi,
487 struct cp_control *cpc, unsigned int start, unsigned int end)
445{ 488{
446 struct list_head *head = &SM_I(sbi)->discard_list; 489 struct list_head *head = &SM_I(sbi)->discard_list;
447 struct discard_entry *new; 490 struct discard_entry *new, *last;
491
492 if (!list_empty(head)) {
493 last = list_last_entry(head, struct discard_entry, list);
494 if (START_BLOCK(sbi, cpc->trim_start) + start ==
495 last->blkaddr + last->len) {
496 last->len += end - start;
497 goto done;
498 }
499 }
500
501 new = f2fs_kmem_cache_alloc(discard_entry_slab, GFP_NOFS);
502 INIT_LIST_HEAD(&new->list);
503 new->blkaddr = START_BLOCK(sbi, cpc->trim_start) + start;
504 new->len = end - start;
505 list_add_tail(&new->list, head);
506done:
507 SM_I(sbi)->nr_discards += end - start;
508 cpc->trimmed += end - start;
509}
510
511static void add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc)
512{
448 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long); 513 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
449 int max_blocks = sbi->blocks_per_seg; 514 int max_blocks = sbi->blocks_per_seg;
450 struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start); 515 struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);
@@ -473,13 +538,7 @@ static void add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc)
473 } 538 }
474 mutex_unlock(&dirty_i->seglist_lock); 539 mutex_unlock(&dirty_i->seglist_lock);
475 540
476 new = f2fs_kmem_cache_alloc(discard_entry_slab, GFP_NOFS); 541 __add_discard_entry(sbi, cpc, 0, sbi->blocks_per_seg);
477 INIT_LIST_HEAD(&new->list);
478 new->blkaddr = START_BLOCK(sbi, cpc->trim_start);
479 new->len = sbi->blocks_per_seg;
480 list_add_tail(&new->list, head);
481 SM_I(sbi)->nr_discards += sbi->blocks_per_seg;
482 cpc->trimmed += sbi->blocks_per_seg;
483 return; 542 return;
484 } 543 }
485 544
@@ -489,7 +548,7 @@ static void add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc)
489 548
490 /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */ 549 /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */
491 for (i = 0; i < entries; i++) 550 for (i = 0; i < entries; i++)
492 dmap[i] = (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i]; 551 dmap[i] = ~(cur_map[i] | ckpt_map[i]);
493 552
494 while (force || SM_I(sbi)->nr_discards <= SM_I(sbi)->max_discards) { 553 while (force || SM_I(sbi)->nr_discards <= SM_I(sbi)->max_discards) {
495 start = __find_rev_next_bit(dmap, max_blocks, end + 1); 554 start = __find_rev_next_bit(dmap, max_blocks, end + 1);
@@ -501,14 +560,7 @@ static void add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc)
501 if (end - start < cpc->trim_minlen) 560 if (end - start < cpc->trim_minlen)
502 continue; 561 continue;
503 562
504 new = f2fs_kmem_cache_alloc(discard_entry_slab, GFP_NOFS); 563 __add_discard_entry(sbi, cpc, start, end);
505 INIT_LIST_HEAD(&new->list);
506 new->blkaddr = START_BLOCK(sbi, cpc->trim_start) + start;
507 new->len = end - start;
508 cpc->trimmed += end - start;
509
510 list_add_tail(&new->list, head);
511 SM_I(sbi)->nr_discards += end - start;
512 } 564 }
513} 565}
514 566
@@ -620,10 +672,10 @@ static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
620 672
621 /* Update valid block bitmap */ 673 /* Update valid block bitmap */
622 if (del > 0) { 674 if (del > 0) {
623 if (f2fs_set_bit(offset, se->cur_valid_map)) 675 if (f2fs_test_and_set_bit(offset, se->cur_valid_map))
624 f2fs_bug_on(sbi, 1); 676 f2fs_bug_on(sbi, 1);
625 } else { 677 } else {
626 if (!f2fs_clear_bit(offset, se->cur_valid_map)) 678 if (!f2fs_test_and_clear_bit(offset, se->cur_valid_map))
627 f2fs_bug_on(sbi, 1); 679 f2fs_bug_on(sbi, 1);
628 } 680 }
629 if (!f2fs_test_bit(offset, se->ckpt_valid_map)) 681 if (!f2fs_test_bit(offset, se->ckpt_valid_map))
@@ -1004,6 +1056,7 @@ int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
1004 range->len < sbi->blocksize) 1056 range->len < sbi->blocksize)
1005 return -EINVAL; 1057 return -EINVAL;
1006 1058
1059 cpc.trimmed = 0;
1007 if (end <= MAIN_BLKADDR(sbi)) 1060 if (end <= MAIN_BLKADDR(sbi))
1008 goto out; 1061 goto out;
1009 1062
@@ -1015,10 +1068,11 @@ int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
1015 cpc.trim_start = start_segno; 1068 cpc.trim_start = start_segno;
1016 cpc.trim_end = end_segno; 1069 cpc.trim_end = end_segno;
1017 cpc.trim_minlen = range->minlen >> sbi->log_blocksize; 1070 cpc.trim_minlen = range->minlen >> sbi->log_blocksize;
1018 cpc.trimmed = 0;
1019 1071
1020 /* do checkpoint to issue discard commands safely */ 1072 /* do checkpoint to issue discard commands safely */
1073 mutex_lock(&sbi->gc_mutex);
1021 write_checkpoint(sbi, &cpc); 1074 write_checkpoint(sbi, &cpc);
1075 mutex_unlock(&sbi->gc_mutex);
1022out: 1076out:
1023 range->len = cpc.trimmed << sbi->log_blocksize; 1077 range->len = cpc.trimmed << sbi->log_blocksize;
1024 return 0; 1078 return 0;
@@ -1050,8 +1104,8 @@ static int __get_segment_type_4(struct page *page, enum page_type p_type)
1050 else 1104 else
1051 return CURSEG_COLD_DATA; 1105 return CURSEG_COLD_DATA;
1052 } else { 1106 } else {
1053 if (IS_DNODE(page) && !is_cold_node(page)) 1107 if (IS_DNODE(page) && is_cold_node(page))
1054 return CURSEG_HOT_NODE; 1108 return CURSEG_WARM_NODE;
1055 else 1109 else
1056 return CURSEG_COLD_NODE; 1110 return CURSEG_COLD_NODE;
1057 } 1111 }
@@ -1524,17 +1578,7 @@ int lookup_journal_in_cursum(struct f2fs_summary_block *sum, int type,
1524static struct page *get_current_sit_page(struct f2fs_sb_info *sbi, 1578static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
1525 unsigned int segno) 1579 unsigned int segno)
1526{ 1580{
1527 struct sit_info *sit_i = SIT_I(sbi); 1581 return get_meta_page(sbi, current_sit_addr(sbi, segno));
1528 unsigned int offset = SIT_BLOCK_OFFSET(segno);
1529 block_t blk_addr = sit_i->sit_base_addr + offset;
1530
1531 check_seg_range(sbi, segno);
1532
1533 /* calculate sit block address */
1534 if (f2fs_test_bit(offset, sit_i->sit_bitmap))
1535 blk_addr += sit_i->sit_blocks;
1536
1537 return get_meta_page(sbi, blk_addr);
1538} 1582}
1539 1583
1540static struct page *get_next_sit_page(struct f2fs_sb_info *sbi, 1584static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
@@ -1687,7 +1731,7 @@ void flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
1687 * #2, flush sit entries to sit page. 1731 * #2, flush sit entries to sit page.
1688 */ 1732 */
1689 list_for_each_entry_safe(ses, tmp, head, set_list) { 1733 list_for_each_entry_safe(ses, tmp, head, set_list) {
1690 struct page *page; 1734 struct page *page = NULL;
1691 struct f2fs_sit_block *raw_sit = NULL; 1735 struct f2fs_sit_block *raw_sit = NULL;
1692 unsigned int start_segno = ses->start_segno; 1736 unsigned int start_segno = ses->start_segno;
1693 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK, 1737 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK,
@@ -2200,7 +2244,7 @@ int __init create_segment_manager_caches(void)
2200 goto fail; 2244 goto fail;
2201 2245
2202 sit_entry_set_slab = f2fs_kmem_cache_create("sit_entry_set", 2246 sit_entry_set_slab = f2fs_kmem_cache_create("sit_entry_set",
2203 sizeof(struct nat_entry_set)); 2247 sizeof(struct sit_entry_set));
2204 if (!sit_entry_set_slab) 2248 if (!sit_entry_set_slab)
2205 goto destory_discard_entry; 2249 goto destory_discard_entry;
2206 2250
diff --git a/fs/f2fs/segment.h b/fs/f2fs/segment.h
index 2495bec1c621..7f327c0ba4e3 100644
--- a/fs/f2fs/segment.h
+++ b/fs/f2fs/segment.h
@@ -657,10 +657,7 @@ static inline void set_to_next_sit(struct sit_info *sit_i, unsigned int start)
657{ 657{
658 unsigned int block_off = SIT_BLOCK_OFFSET(start); 658 unsigned int block_off = SIT_BLOCK_OFFSET(start);
659 659
660 if (f2fs_test_bit(block_off, sit_i->sit_bitmap)) 660 f2fs_change_bit(block_off, sit_i->sit_bitmap);
661 f2fs_clear_bit(block_off, sit_i->sit_bitmap);
662 else
663 f2fs_set_bit(block_off, sit_i->sit_bitmap);
664} 661}
665 662
666static inline unsigned long long get_mtime(struct f2fs_sb_info *sbi) 663static inline unsigned long long get_mtime(struct f2fs_sb_info *sbi)
@@ -714,6 +711,9 @@ static inline unsigned int max_hw_blocks(struct f2fs_sb_info *sbi)
714 */ 711 */
715static inline int nr_pages_to_skip(struct f2fs_sb_info *sbi, int type) 712static inline int nr_pages_to_skip(struct f2fs_sb_info *sbi, int type)
716{ 713{
714 if (sbi->sb->s_bdi->dirty_exceeded)
715 return 0;
716
717 if (type == DATA) 717 if (type == DATA)
718 return sbi->blocks_per_seg; 718 return sbi->blocks_per_seg;
719 else if (type == NODE) 719 else if (type == NODE)
diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c
index 41d6f700f4ee..f71421d70475 100644
--- a/fs/f2fs/super.c
+++ b/fs/f2fs/super.c
@@ -51,8 +51,10 @@ enum {
51 Opt_disable_ext_identify, 51 Opt_disable_ext_identify,
52 Opt_inline_xattr, 52 Opt_inline_xattr,
53 Opt_inline_data, 53 Opt_inline_data,
54 Opt_inline_dentry,
54 Opt_flush_merge, 55 Opt_flush_merge,
55 Opt_nobarrier, 56 Opt_nobarrier,
57 Opt_fastboot,
56 Opt_err, 58 Opt_err,
57}; 59};
58 60
@@ -69,8 +71,10 @@ static match_table_t f2fs_tokens = {
69 {Opt_disable_ext_identify, "disable_ext_identify"}, 71 {Opt_disable_ext_identify, "disable_ext_identify"},
70 {Opt_inline_xattr, "inline_xattr"}, 72 {Opt_inline_xattr, "inline_xattr"},
71 {Opt_inline_data, "inline_data"}, 73 {Opt_inline_data, "inline_data"},
74 {Opt_inline_dentry, "inline_dentry"},
72 {Opt_flush_merge, "flush_merge"}, 75 {Opt_flush_merge, "flush_merge"},
73 {Opt_nobarrier, "nobarrier"}, 76 {Opt_nobarrier, "nobarrier"},
77 {Opt_fastboot, "fastboot"},
74 {Opt_err, NULL}, 78 {Opt_err, NULL},
75}; 79};
76 80
@@ -340,12 +344,18 @@ static int parse_options(struct super_block *sb, char *options)
340 case Opt_inline_data: 344 case Opt_inline_data:
341 set_opt(sbi, INLINE_DATA); 345 set_opt(sbi, INLINE_DATA);
342 break; 346 break;
347 case Opt_inline_dentry:
348 set_opt(sbi, INLINE_DENTRY);
349 break;
343 case Opt_flush_merge: 350 case Opt_flush_merge:
344 set_opt(sbi, FLUSH_MERGE); 351 set_opt(sbi, FLUSH_MERGE);
345 break; 352 break;
346 case Opt_nobarrier: 353 case Opt_nobarrier:
347 set_opt(sbi, NOBARRIER); 354 set_opt(sbi, NOBARRIER);
348 break; 355 break;
356 case Opt_fastboot:
357 set_opt(sbi, FASTBOOT);
358 break;
349 default: 359 default:
350 f2fs_msg(sb, KERN_ERR, 360 f2fs_msg(sb, KERN_ERR,
351 "Unrecognized mount option \"%s\" or missing value", 361 "Unrecognized mount option \"%s\" or missing value",
@@ -373,6 +383,7 @@ static struct inode *f2fs_alloc_inode(struct super_block *sb)
373 fi->i_advise = 0; 383 fi->i_advise = 0;
374 rwlock_init(&fi->ext.ext_lock); 384 rwlock_init(&fi->ext.ext_lock);
375 init_rwsem(&fi->i_sem); 385 init_rwsem(&fi->i_sem);
386 INIT_RADIX_TREE(&fi->inmem_root, GFP_NOFS);
376 INIT_LIST_HEAD(&fi->inmem_pages); 387 INIT_LIST_HEAD(&fi->inmem_pages);
377 mutex_init(&fi->inmem_lock); 388 mutex_init(&fi->inmem_lock);
378 389
@@ -473,9 +484,9 @@ int f2fs_sync_fs(struct super_block *sb, int sync)
473 trace_f2fs_sync_fs(sb, sync); 484 trace_f2fs_sync_fs(sb, sync);
474 485
475 if (sync) { 486 if (sync) {
476 struct cp_control cpc = { 487 struct cp_control cpc;
477 .reason = CP_SYNC, 488
478 }; 489 cpc.reason = test_opt(sbi, FASTBOOT) ? CP_UMOUNT : CP_SYNC;
479 mutex_lock(&sbi->gc_mutex); 490 mutex_lock(&sbi->gc_mutex);
480 write_checkpoint(sbi, &cpc); 491 write_checkpoint(sbi, &cpc);
481 mutex_unlock(&sbi->gc_mutex); 492 mutex_unlock(&sbi->gc_mutex);
@@ -562,10 +573,14 @@ static int f2fs_show_options(struct seq_file *seq, struct dentry *root)
562 seq_puts(seq, ",disable_ext_identify"); 573 seq_puts(seq, ",disable_ext_identify");
563 if (test_opt(sbi, INLINE_DATA)) 574 if (test_opt(sbi, INLINE_DATA))
564 seq_puts(seq, ",inline_data"); 575 seq_puts(seq, ",inline_data");
576 if (test_opt(sbi, INLINE_DENTRY))
577 seq_puts(seq, ",inline_dentry");
565 if (!f2fs_readonly(sbi->sb) && test_opt(sbi, FLUSH_MERGE)) 578 if (!f2fs_readonly(sbi->sb) && test_opt(sbi, FLUSH_MERGE))
566 seq_puts(seq, ",flush_merge"); 579 seq_puts(seq, ",flush_merge");
567 if (test_opt(sbi, NOBARRIER)) 580 if (test_opt(sbi, NOBARRIER))
568 seq_puts(seq, ",nobarrier"); 581 seq_puts(seq, ",nobarrier");
582 if (test_opt(sbi, FASTBOOT))
583 seq_puts(seq, ",fastboot");
569 seq_printf(seq, ",active_logs=%u", sbi->active_logs); 584 seq_printf(seq, ",active_logs=%u", sbi->active_logs);
570 585
571 return 0; 586 return 0;
@@ -654,7 +669,7 @@ static int f2fs_remount(struct super_block *sb, int *flags, char *data)
654 f2fs_sync_fs(sb, 1); 669 f2fs_sync_fs(sb, 1);
655 need_restart_gc = true; 670 need_restart_gc = true;
656 } 671 }
657 } else if (test_opt(sbi, BG_GC) && !sbi->gc_thread) { 672 } else if (!sbi->gc_thread) {
658 err = start_gc_thread(sbi); 673 err = start_gc_thread(sbi);
659 if (err) 674 if (err)
660 goto restore_opts; 675 goto restore_opts;
@@ -667,7 +682,7 @@ static int f2fs_remount(struct super_block *sb, int *flags, char *data)
667 */ 682 */
668 if ((*flags & MS_RDONLY) || !test_opt(sbi, FLUSH_MERGE)) { 683 if ((*flags & MS_RDONLY) || !test_opt(sbi, FLUSH_MERGE)) {
669 destroy_flush_cmd_control(sbi); 684 destroy_flush_cmd_control(sbi);
670 } else if (test_opt(sbi, FLUSH_MERGE) && !SM_I(sbi)->cmd_control_info) { 685 } else if (!SM_I(sbi)->cmd_control_info) {
671 err = create_flush_cmd_control(sbi); 686 err = create_flush_cmd_control(sbi);
672 if (err) 687 if (err)
673 goto restore_gc; 688 goto restore_gc;
@@ -922,7 +937,7 @@ retry:
922static int f2fs_fill_super(struct super_block *sb, void *data, int silent) 937static int f2fs_fill_super(struct super_block *sb, void *data, int silent)
923{ 938{
924 struct f2fs_sb_info *sbi; 939 struct f2fs_sb_info *sbi;
925 struct f2fs_super_block *raw_super; 940 struct f2fs_super_block *raw_super = NULL;
926 struct buffer_head *raw_super_buf; 941 struct buffer_head *raw_super_buf;
927 struct inode *root; 942 struct inode *root;
928 long err = -EINVAL; 943 long err = -EINVAL;
@@ -1123,7 +1138,7 @@ try_onemore:
1123 * If filesystem is not mounted as read-only then 1138 * If filesystem is not mounted as read-only then
1124 * do start the gc_thread. 1139 * do start the gc_thread.
1125 */ 1140 */
1126 if (!f2fs_readonly(sb)) { 1141 if (test_opt(sbi, BG_GC) && !f2fs_readonly(sb)) {
1127 /* After POR, we can run background GC thread.*/ 1142 /* After POR, we can run background GC thread.*/
1128 err = start_gc_thread(sbi); 1143 err = start_gc_thread(sbi);
1129 if (err) 1144 if (err)
diff --git a/fs/f2fs/xattr.c b/fs/f2fs/xattr.c
index deca8728117b..5072bf9ae0ef 100644
--- a/fs/f2fs/xattr.c
+++ b/fs/f2fs/xattr.c
@@ -83,7 +83,7 @@ static int f2fs_xattr_generic_get(struct dentry *dentry, const char *name,
83 } 83 }
84 if (strcmp(name, "") == 0) 84 if (strcmp(name, "") == 0)
85 return -EINVAL; 85 return -EINVAL;
86 return f2fs_getxattr(dentry->d_inode, type, name, buffer, size); 86 return f2fs_getxattr(dentry->d_inode, type, name, buffer, size, NULL);
87} 87}
88 88
89static int f2fs_xattr_generic_set(struct dentry *dentry, const char *name, 89static int f2fs_xattr_generic_set(struct dentry *dentry, const char *name,
@@ -398,7 +398,7 @@ static inline int write_all_xattrs(struct inode *inode, __u32 hsize,
398} 398}
399 399
400int f2fs_getxattr(struct inode *inode, int index, const char *name, 400int f2fs_getxattr(struct inode *inode, int index, const char *name,
401 void *buffer, size_t buffer_size) 401 void *buffer, size_t buffer_size, struct page *ipage)
402{ 402{
403 struct f2fs_xattr_entry *entry; 403 struct f2fs_xattr_entry *entry;
404 void *base_addr; 404 void *base_addr;
@@ -412,7 +412,7 @@ int f2fs_getxattr(struct inode *inode, int index, const char *name,
412 if (len > F2FS_NAME_LEN) 412 if (len > F2FS_NAME_LEN)
413 return -ERANGE; 413 return -ERANGE;
414 414
415 base_addr = read_all_xattrs(inode, NULL); 415 base_addr = read_all_xattrs(inode, ipage);
416 if (!base_addr) 416 if (!base_addr)
417 return -ENOMEM; 417 return -ENOMEM;
418 418
diff --git a/fs/f2fs/xattr.h b/fs/f2fs/xattr.h
index 34ab7dbcf5e3..969d792ca362 100644
--- a/fs/f2fs/xattr.h
+++ b/fs/f2fs/xattr.h
@@ -115,7 +115,8 @@ extern const struct xattr_handler *f2fs_xattr_handlers[];
115 115
116extern int f2fs_setxattr(struct inode *, int, const char *, 116extern int f2fs_setxattr(struct inode *, int, const char *,
117 const void *, size_t, struct page *, int); 117 const void *, size_t, struct page *, int);
118extern int f2fs_getxattr(struct inode *, int, const char *, void *, size_t); 118extern int f2fs_getxattr(struct inode *, int, const char *, void *,
119 size_t, struct page *);
119extern ssize_t f2fs_listxattr(struct dentry *, char *, size_t); 120extern ssize_t f2fs_listxattr(struct dentry *, char *, size_t);
120#else 121#else
121 122
@@ -126,7 +127,8 @@ static inline int f2fs_setxattr(struct inode *inode, int index,
126 return -EOPNOTSUPP; 127 return -EOPNOTSUPP;
127} 128}
128static inline int f2fs_getxattr(struct inode *inode, int index, 129static inline int f2fs_getxattr(struct inode *inode, int index,
129 const char *name, void *buffer, size_t buffer_size) 130 const char *name, void *buffer,
131 size_t buffer_size, struct page *dpage)
130{ 132{
131 return -EOPNOTSUPP; 133 return -EOPNOTSUPP;
132} 134}
generated by cgit v1.2.2 (git 2.25.0) at 2025-05-31 22:59:41 -0400