aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJaegeuk Kim <jaegeuk.kim@samsung.com>2012-11-22 02:21:29 -0500
committerJaegeuk Kim <jaegeuk.kim@samsung.com>2013-04-09 05:21:18 -0400
commit399368372ed9f3c396eadb5c2bbc98be8c774a39 (patch)
treeb496c910ebf04d95e5ad6cb1b65d8cbb45aeef89
parent1127a3d448bcf4de338e60a7cc695d54c5767433 (diff)
f2fs: introduce a new global lock scheme
In the previous version, f2fs uses global locks according to the usage types, such as directory operations, block allocation, block write, and so on. Reference the following lock types in f2fs.h. enum lock_type { RENAME, /* for renaming operations */ DENTRY_OPS, /* for directory operations */ DATA_WRITE, /* for data write */ DATA_NEW, /* for data allocation */ DATA_TRUNC, /* for data truncate */ NODE_NEW, /* for node allocation */ NODE_TRUNC, /* for node truncate */ NODE_WRITE, /* for node write */ NR_LOCK_TYPE, }; In that case, we lose the performance under the multi-threading environment, since every types of operations must be conducted one at a time. In order to address the problem, let's share the locks globally with a mutex array regardless of any types. So, let users grab a mutex and perform their jobs in parallel as much as possbile. For this, I propose a new global lock scheme as follows. 0. Data structure - f2fs_sb_info -> mutex_lock[NR_GLOBAL_LOCKS] - f2fs_sb_info -> node_write 1. mutex_lock_op(sbi) - try to get an avaiable lock from the array. - returns the index of the gottern lock variable. 2. mutex_unlock_op(sbi, index of the lock) - unlock the given index of the lock. 3. mutex_lock_all(sbi) - grab all the locks in the array before the checkpoint. 4. mutex_unlock_all(sbi) - release all the locks in the array after checkpoint. 5. block_operations() - call mutex_lock_all() - sync_dirty_dir_inodes() - grab node_write - sync_node_pages() Note that, the pairs of mutex_lock_op()/mutex_unlock_op() and mutex_lock_all()/mutex_unlock_all() should be used together. Signed-off-by: Jaegeuk Kim <jaegeuk.kim@samsung.com>
-rw-r--r--fs/f2fs/checkpoint.c39
-rw-r--r--fs/f2fs/data.c77
-rw-r--r--fs/f2fs/dir.c105
-rw-r--r--fs/f2fs/f2fs.h63
-rw-r--r--fs/f2fs/file.c46
-rw-r--r--fs/f2fs/gc.c2
-rw-r--r--fs/f2fs/inode.c48
-rw-r--r--fs/f2fs/namei.c44
-rw-r--r--fs/f2fs/node.c44
-rw-r--r--fs/f2fs/recovery.c8
-rw-r--r--fs/f2fs/super.c4
-rw-r--r--fs/f2fs/xattr.c10
12 files changed, 248 insertions, 242 deletions
diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c
index 93fd57d491ac..be6aa2eef894 100644
--- a/fs/f2fs/checkpoint.c
+++ b/fs/f2fs/checkpoint.c
@@ -543,54 +543,39 @@ retry:
543 */ 543 */
544static void block_operations(struct f2fs_sb_info *sbi) 544static void block_operations(struct f2fs_sb_info *sbi)
545{ 545{
546 int t;
547 struct writeback_control wbc = { 546 struct writeback_control wbc = {
548 .sync_mode = WB_SYNC_ALL, 547 .sync_mode = WB_SYNC_ALL,
549 .nr_to_write = LONG_MAX, 548 .nr_to_write = LONG_MAX,
550 .for_reclaim = 0, 549 .for_reclaim = 0,
551 }; 550 };
551retry_flush_dents:
552 mutex_lock_all(sbi);
552 553
553 /* Stop renaming operation */
554 mutex_lock_op(sbi, RENAME);
555 mutex_lock_op(sbi, DENTRY_OPS);
556
557retry_dents:
558 /* write all the dirty dentry pages */ 554 /* write all the dirty dentry pages */
559 sync_dirty_dir_inodes(sbi);
560
561 mutex_lock_op(sbi, DATA_WRITE);
562 if (get_pages(sbi, F2FS_DIRTY_DENTS)) { 555 if (get_pages(sbi, F2FS_DIRTY_DENTS)) {
563 mutex_unlock_op(sbi, DATA_WRITE); 556 mutex_unlock_all(sbi);
564 goto retry_dents; 557 sync_dirty_dir_inodes(sbi);
558 goto retry_flush_dents;
565 } 559 }
566 560
567 /* block all the operations */
568 for (t = DATA_NEW; t <= NODE_TRUNC; t++)
569 mutex_lock_op(sbi, t);
570
571 mutex_lock(&sbi->write_inode);
572
573 /* 561 /*
574 * POR: we should ensure that there is no dirty node pages 562 * POR: we should ensure that there is no dirty node pages
575 * until finishing nat/sit flush. 563 * until finishing nat/sit flush.
576 */ 564 */
577retry: 565retry_flush_nodes:
578 sync_node_pages(sbi, 0, &wbc); 566 mutex_lock(&sbi->node_write);
579
580 mutex_lock_op(sbi, NODE_WRITE);
581 567
582 if (get_pages(sbi, F2FS_DIRTY_NODES)) { 568 if (get_pages(sbi, F2FS_DIRTY_NODES)) {
583 mutex_unlock_op(sbi, NODE_WRITE); 569 mutex_unlock(&sbi->node_write);
584 goto retry; 570 sync_node_pages(sbi, 0, &wbc);
571 goto retry_flush_nodes;
585 } 572 }
586 mutex_unlock(&sbi->write_inode);
587} 573}
588 574
589static void unblock_operations(struct f2fs_sb_info *sbi) 575static void unblock_operations(struct f2fs_sb_info *sbi)
590{ 576{
591 int t; 577 mutex_unlock(&sbi->node_write);
592 for (t = NODE_WRITE; t >= RENAME; t--) 578 mutex_unlock_all(sbi);
593 mutex_unlock_op(sbi, t);
594} 579}
595 580
596static void do_checkpoint(struct f2fs_sb_info *sbi, bool is_umount) 581static void do_checkpoint(struct f2fs_sb_info *sbi, bool is_umount)
diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
index cf9ff5f76134..72a1b30f720a 100644
--- a/fs/f2fs/data.c
+++ b/fs/f2fs/data.c
@@ -260,6 +260,9 @@ struct page *get_lock_data_page(struct inode *inode, pgoff_t index)
260/* 260/*
261 * Caller ensures that this data page is never allocated. 261 * Caller ensures that this data page is never allocated.
262 * A new zero-filled data page is allocated in the page cache. 262 * A new zero-filled data page is allocated in the page cache.
263 *
264 * Also, caller should grab and release a mutex by calling mutex_lock_op() and
265 * mutex_unlock_op().
263 */ 266 */
264struct page *get_new_data_page(struct inode *inode, pgoff_t index, 267struct page *get_new_data_page(struct inode *inode, pgoff_t index,
265 bool new_i_size) 268 bool new_i_size)
@@ -479,10 +482,11 @@ static int f2fs_write_data_page(struct page *page,
479 const pgoff_t end_index = ((unsigned long long) i_size) 482 const pgoff_t end_index = ((unsigned long long) i_size)
480 >> PAGE_CACHE_SHIFT; 483 >> PAGE_CACHE_SHIFT;
481 unsigned offset; 484 unsigned offset;
485 bool need_balance_fs = false;
482 int err = 0; 486 int err = 0;
483 487
484 if (page->index < end_index) 488 if (page->index < end_index)
485 goto out; 489 goto write;
486 490
487 /* 491 /*
488 * If the offset is out-of-range of file size, 492 * If the offset is out-of-range of file size,
@@ -494,50 +498,46 @@ static int f2fs_write_data_page(struct page *page,
494 dec_page_count(sbi, F2FS_DIRTY_DENTS); 498 dec_page_count(sbi, F2FS_DIRTY_DENTS);
495 inode_dec_dirty_dents(inode); 499 inode_dec_dirty_dents(inode);
496 } 500 }
497 goto unlock_out; 501 goto out;
498 } 502 }
499 503
500 zero_user_segment(page, offset, PAGE_CACHE_SIZE); 504 zero_user_segment(page, offset, PAGE_CACHE_SIZE);
501out: 505write:
502 if (sbi->por_doing) 506 if (sbi->por_doing) {
503 goto redirty_out; 507 err = AOP_WRITEPAGE_ACTIVATE;
504
505 if (wbc->for_reclaim && !S_ISDIR(inode->i_mode) && !is_cold_data(page))
506 goto redirty_out; 508 goto redirty_out;
509 }
507 510
508 mutex_lock_op(sbi, DATA_WRITE); 511 /* Dentry blocks are controlled by checkpoint */
509 if (S_ISDIR(inode->i_mode)) { 512 if (S_ISDIR(inode->i_mode)) {
510 dec_page_count(sbi, F2FS_DIRTY_DENTS); 513 dec_page_count(sbi, F2FS_DIRTY_DENTS);
511 inode_dec_dirty_dents(inode); 514 inode_dec_dirty_dents(inode);
515 err = do_write_data_page(page);
516 } else {
517 int ilock = mutex_lock_op(sbi);
518 err = do_write_data_page(page);
519 mutex_unlock_op(sbi, ilock);
520 need_balance_fs = true;
512 } 521 }
513 err = do_write_data_page(page); 522 if (err == -ENOENT)
514 if (err && err != -ENOENT) { 523 goto out;
515 wbc->pages_skipped++; 524 else if (err)
516 set_page_dirty(page); 525 goto redirty_out;
517 }
518 mutex_unlock_op(sbi, DATA_WRITE);
519 526
520 if (wbc->for_reclaim) 527 if (wbc->for_reclaim)
521 f2fs_submit_bio(sbi, DATA, true); 528 f2fs_submit_bio(sbi, DATA, true);
522 529
523 if (err == -ENOENT)
524 goto unlock_out;
525
526 clear_cold_data(page); 530 clear_cold_data(page);
531out:
527 unlock_page(page); 532 unlock_page(page);
528 533 if (need_balance_fs)
529 if (!wbc->for_reclaim && !S_ISDIR(inode->i_mode))
530 f2fs_balance_fs(sbi); 534 f2fs_balance_fs(sbi);
531 return 0; 535 return 0;
532 536
533unlock_out:
534 unlock_page(page);
535 return (err == -ENOENT) ? 0 : err;
536
537redirty_out: 537redirty_out:
538 wbc->pages_skipped++; 538 wbc->pages_skipped++;
539 set_page_dirty(page); 539 set_page_dirty(page);
540 return AOP_WRITEPAGE_ACTIVATE; 540 return err;
541} 541}
542 542
543#define MAX_DESIRED_PAGES_WP 4096 543#define MAX_DESIRED_PAGES_WP 4096
@@ -592,6 +592,7 @@ static int f2fs_write_begin(struct file *file, struct address_space *mapping,
592 pgoff_t index = ((unsigned long long) pos) >> PAGE_CACHE_SHIFT; 592 pgoff_t index = ((unsigned long long) pos) >> PAGE_CACHE_SHIFT;
593 struct dnode_of_data dn; 593 struct dnode_of_data dn;
594 int err = 0; 594 int err = 0;
595 int ilock;
595 596
596 /* for nobh_write_end */ 597 /* for nobh_write_end */
597 *fsdata = NULL; 598 *fsdata = NULL;
@@ -603,28 +604,21 @@ static int f2fs_write_begin(struct file *file, struct address_space *mapping,
603 return -ENOMEM; 604 return -ENOMEM;
604 *pagep = page; 605 *pagep = page;
605 606
606 mutex_lock_op(sbi, DATA_NEW); 607 ilock = mutex_lock_op(sbi);
607 608
608 set_new_dnode(&dn, inode, NULL, NULL, 0); 609 set_new_dnode(&dn, inode, NULL, NULL, 0);
609 err = get_dnode_of_data(&dn, index, ALLOC_NODE); 610 err = get_dnode_of_data(&dn, index, ALLOC_NODE);
610 if (err) { 611 if (err)
611 mutex_unlock_op(sbi, DATA_NEW); 612 goto err;
612 f2fs_put_page(page, 1);
613 return err;
614 }
615 613
616 if (dn.data_blkaddr == NULL_ADDR) { 614 if (dn.data_blkaddr == NULL_ADDR)
617 err = reserve_new_block(&dn); 615 err = reserve_new_block(&dn);
618 if (err) { 616
619 f2fs_put_dnode(&dn);
620 mutex_unlock_op(sbi, DATA_NEW);
621 f2fs_put_page(page, 1);
622 return err;
623 }
624 }
625 f2fs_put_dnode(&dn); 617 f2fs_put_dnode(&dn);
618 if (err)
619 goto err;
626 620
627 mutex_unlock_op(sbi, DATA_NEW); 621 mutex_unlock_op(sbi, ilock);
628 622
629 if ((len == PAGE_CACHE_SIZE) || PageUptodate(page)) 623 if ((len == PAGE_CACHE_SIZE) || PageUptodate(page))
630 return 0; 624 return 0;
@@ -654,6 +648,11 @@ out:
654 SetPageUptodate(page); 648 SetPageUptodate(page);
655 clear_cold_data(page); 649 clear_cold_data(page);
656 return 0; 650 return 0;
651
652err:
653 mutex_unlock_op(sbi, ilock);
654 f2fs_put_page(page, 1);
655 return err;
657} 656}
658 657
659static ssize_t f2fs_direct_IO(int rw, struct kiocb *iocb, 658static ssize_t f2fs_direct_IO(int rw, struct kiocb *iocb,
diff --git a/fs/f2fs/dir.c b/fs/f2fs/dir.c
index 2851ae6948a1..cd3342d4a3a7 100644
--- a/fs/f2fs/dir.c
+++ b/fs/f2fs/dir.c
@@ -249,9 +249,6 @@ ino_t f2fs_inode_by_name(struct inode *dir, struct qstr *qstr)
249void f2fs_set_link(struct inode *dir, struct f2fs_dir_entry *de, 249void f2fs_set_link(struct inode *dir, struct f2fs_dir_entry *de,
250 struct page *page, struct inode *inode) 250 struct page *page, struct inode *inode)
251{ 251{
252 struct f2fs_sb_info *sbi = F2FS_SB(dir->i_sb);
253
254 mutex_lock_op(sbi, DENTRY_OPS);
255 lock_page(page); 252 lock_page(page);
256 wait_on_page_writeback(page); 253 wait_on_page_writeback(page);
257 de->ino = cpu_to_le32(inode->i_ino); 254 de->ino = cpu_to_le32(inode->i_ino);
@@ -265,7 +262,6 @@ void f2fs_set_link(struct inode *dir, struct f2fs_dir_entry *de,
265 F2FS_I(inode)->i_pino = dir->i_ino; 262 F2FS_I(inode)->i_pino = dir->i_ino;
266 263
267 f2fs_put_page(page, 1); 264 f2fs_put_page(page, 1);
268 mutex_unlock_op(sbi, DENTRY_OPS);
269} 265}
270 266
271void init_dent_inode(const struct qstr *name, struct page *ipage) 267void init_dent_inode(const struct qstr *name, struct page *ipage)
@@ -284,6 +280,43 @@ void init_dent_inode(const struct qstr *name, struct page *ipage)
284 set_page_dirty(ipage); 280 set_page_dirty(ipage);
285} 281}
286 282
283static int make_empty_dir(struct inode *inode, struct inode *parent)
284{
285 struct page *dentry_page;
286 struct f2fs_dentry_block *dentry_blk;
287 struct f2fs_dir_entry *de;
288 void *kaddr;
289
290 dentry_page = get_new_data_page(inode, 0, true);
291 if (IS_ERR(dentry_page))
292 return PTR_ERR(dentry_page);
293
294 kaddr = kmap_atomic(dentry_page);
295 dentry_blk = (struct f2fs_dentry_block *)kaddr;
296
297 de = &dentry_blk->dentry[0];
298 de->name_len = cpu_to_le16(1);
299 de->hash_code = 0;
300 de->ino = cpu_to_le32(inode->i_ino);
301 memcpy(dentry_blk->filename[0], ".", 1);
302 set_de_type(de, inode);
303
304 de = &dentry_blk->dentry[1];
305 de->hash_code = 0;
306 de->name_len = cpu_to_le16(2);
307 de->ino = cpu_to_le32(parent->i_ino);
308 memcpy(dentry_blk->filename[1], "..", 2);
309 set_de_type(de, inode);
310
311 test_and_set_bit_le(0, &dentry_blk->dentry_bitmap);
312 test_and_set_bit_le(1, &dentry_blk->dentry_bitmap);
313 kunmap_atomic(kaddr);
314
315 set_page_dirty(dentry_page);
316 f2fs_put_page(dentry_page, 1);
317 return 0;
318}
319
287static int init_inode_metadata(struct inode *inode, 320static int init_inode_metadata(struct inode *inode,
288 struct inode *dir, const struct qstr *name) 321 struct inode *dir, const struct qstr *name)
289{ 322{
@@ -294,7 +327,7 @@ static int init_inode_metadata(struct inode *inode,
294 return err; 327 return err;
295 328
296 if (S_ISDIR(inode->i_mode)) { 329 if (S_ISDIR(inode->i_mode)) {
297 err = f2fs_make_empty(inode, dir); 330 err = make_empty_dir(inode, dir);
298 if (err) { 331 if (err) {
299 remove_inode_page(inode); 332 remove_inode_page(inode);
300 return err; 333 return err;
@@ -317,7 +350,7 @@ static int init_inode_metadata(struct inode *inode,
317 } 350 }
318 if (is_inode_flag_set(F2FS_I(inode), FI_INC_LINK)) { 351 if (is_inode_flag_set(F2FS_I(inode), FI_INC_LINK)) {
319 inc_nlink(inode); 352 inc_nlink(inode);
320 f2fs_write_inode(inode, NULL); 353 update_inode_page(inode);
321 } 354 }
322 return 0; 355 return 0;
323} 356}
@@ -341,7 +374,7 @@ static void update_parent_metadata(struct inode *dir, struct inode *inode,
341 } 374 }
342 375
343 if (need_dir_update) 376 if (need_dir_update)
344 f2fs_write_inode(dir, NULL); 377 update_inode_page(dir);
345 else 378 else
346 mark_inode_dirty(dir); 379 mark_inode_dirty(dir);
347 380
@@ -373,6 +406,10 @@ next:
373 goto next; 406 goto next;
374} 407}
375 408
409/*
410 * Caller should grab and release a mutex by calling mutex_lock_op() and
411 * mutex_unlock_op().
412 */
376int __f2fs_add_link(struct inode *dir, const struct qstr *name, struct inode *inode) 413int __f2fs_add_link(struct inode *dir, const struct qstr *name, struct inode *inode)
377{ 414{
378 unsigned int bit_pos; 415 unsigned int bit_pos;
@@ -382,7 +419,6 @@ int __f2fs_add_link(struct inode *dir, const struct qstr *name, struct inode *in
382 f2fs_hash_t dentry_hash; 419 f2fs_hash_t dentry_hash;
383 struct f2fs_dir_entry *de; 420 struct f2fs_dir_entry *de;
384 unsigned int nbucket, nblock; 421 unsigned int nbucket, nblock;
385 struct f2fs_sb_info *sbi = F2FS_SB(dir->i_sb);
386 size_t namelen = name->len; 422 size_t namelen = name->len;
387 struct page *dentry_page = NULL; 423 struct page *dentry_page = NULL;
388 struct f2fs_dentry_block *dentry_blk = NULL; 424 struct f2fs_dentry_block *dentry_blk = NULL;
@@ -412,12 +448,9 @@ start:
412 bidx = dir_block_index(level, (le32_to_cpu(dentry_hash) % nbucket)); 448 bidx = dir_block_index(level, (le32_to_cpu(dentry_hash) % nbucket));
413 449
414 for (block = bidx; block <= (bidx + nblock - 1); block++) { 450 for (block = bidx; block <= (bidx + nblock - 1); block++) {
415 mutex_lock_op(sbi, DENTRY_OPS);
416 dentry_page = get_new_data_page(dir, block, true); 451 dentry_page = get_new_data_page(dir, block, true);
417 if (IS_ERR(dentry_page)) { 452 if (IS_ERR(dentry_page))
418 mutex_unlock_op(sbi, DENTRY_OPS);
419 return PTR_ERR(dentry_page); 453 return PTR_ERR(dentry_page);
420 }
421 454
422 dentry_blk = kmap(dentry_page); 455 dentry_blk = kmap(dentry_page);
423 bit_pos = room_for_filename(dentry_blk, slots); 456 bit_pos = room_for_filename(dentry_blk, slots);
@@ -426,7 +459,6 @@ start:
426 459
427 kunmap(dentry_page); 460 kunmap(dentry_page);
428 f2fs_put_page(dentry_page, 1); 461 f2fs_put_page(dentry_page, 1);
429 mutex_unlock_op(sbi, DENTRY_OPS);
430 } 462 }
431 463
432 /* Move to next level to find the empty slot for new dentry */ 464 /* Move to next level to find the empty slot for new dentry */
@@ -456,7 +488,6 @@ add_dentry:
456fail: 488fail:
457 kunmap(dentry_page); 489 kunmap(dentry_page);
458 f2fs_put_page(dentry_page, 1); 490 f2fs_put_page(dentry_page, 1);
459 mutex_unlock_op(sbi, DENTRY_OPS);
460 return err; 491 return err;
461} 492}
462 493
@@ -476,8 +507,6 @@ void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct page *page,
476 void *kaddr = page_address(page); 507 void *kaddr = page_address(page);
477 int i; 508 int i;
478 509
479 mutex_lock_op(sbi, DENTRY_OPS);
480
481 lock_page(page); 510 lock_page(page);
482 wait_on_page_writeback(page); 511 wait_on_page_writeback(page);
483 512
@@ -497,7 +526,7 @@ void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct page *page,
497 526
498 if (inode && S_ISDIR(inode->i_mode)) { 527 if (inode && S_ISDIR(inode->i_mode)) {
499 drop_nlink(dir); 528 drop_nlink(dir);
500 f2fs_write_inode(dir, NULL); 529 update_inode_page(dir);
501 } else { 530 } else {
502 mark_inode_dirty(dir); 531 mark_inode_dirty(dir);
503 } 532 }
@@ -509,7 +538,8 @@ void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct page *page,
509 drop_nlink(inode); 538 drop_nlink(inode);
510 i_size_write(inode, 0); 539 i_size_write(inode, 0);
511 } 540 }
512 f2fs_write_inode(inode, NULL); 541 update_inode_page(inode);
542
513 if (inode->i_nlink == 0) 543 if (inode->i_nlink == 0)
514 add_orphan_inode(sbi, inode->i_ino); 544 add_orphan_inode(sbi, inode->i_ino);
515 } 545 }
@@ -522,45 +552,6 @@ void f2fs_delete_entry(struct f2fs_dir_entry *dentry, struct page *page,
522 inode_dec_dirty_dents(dir); 552 inode_dec_dirty_dents(dir);
523 } 553 }
524 f2fs_put_page(page, 1); 554 f2fs_put_page(page, 1);
525
526 mutex_unlock_op(sbi, DENTRY_OPS);
527}
528
529int f2fs_make_empty(struct inode *inode, struct inode *parent)
530{
531 struct page *dentry_page;
532 struct f2fs_dentry_block *dentry_blk;
533 struct f2fs_dir_entry *de;
534 void *kaddr;
535
536 dentry_page = get_new_data_page(inode, 0, true);
537 if (IS_ERR(dentry_page))
538 return PTR_ERR(dentry_page);
539
540 kaddr = kmap_atomic(dentry_page);
541 dentry_blk = (struct f2fs_dentry_block *)kaddr;
542
543 de = &dentry_blk->dentry[0];
544 de->name_len = cpu_to_le16(1);
545 de->hash_code = f2fs_dentry_hash(".", 1);
546 de->ino = cpu_to_le32(inode->i_ino);
547 memcpy(dentry_blk->filename[0], ".", 1);
548 set_de_type(de, inode);
549
550 de = &dentry_blk->dentry[1];
551 de->hash_code = f2fs_dentry_hash("..", 2);
552 de->name_len = cpu_to_le16(2);
553 de->ino = cpu_to_le32(parent->i_ino);
554 memcpy(dentry_blk->filename[1], "..", 2);
555 set_de_type(de, inode);
556
557 test_and_set_bit_le(0, &dentry_blk->dentry_bitmap);
558 test_and_set_bit_le(1, &dentry_blk->dentry_bitmap);
559 kunmap_atomic(kaddr);
560
561 set_page_dirty(dentry_page);
562 f2fs_put_page(dentry_page, 1);
563 return 0;
564} 555}
565 556
566bool f2fs_empty_dir(struct inode *dir) 557bool f2fs_empty_dir(struct inode *dir)
diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
index 71eacd373916..06cc75c66c88 100644
--- a/fs/f2fs/f2fs.h
+++ b/fs/f2fs/f2fs.h
@@ -309,23 +309,12 @@ enum count_type {
309}; 309};
310 310
311/* 311/*
312 * FS_LOCK nesting subclasses for the lock validator: 312 * Uses as sbi->fs_lock[NR_GLOBAL_LOCKS].
313 * 313 * The checkpoint procedure blocks all the locks in this fs_lock array.
314 * The locking order between these classes is 314 * Some FS operations grab free locks, and if there is no free lock,
315 * RENAME -> DENTRY_OPS -> DATA_WRITE -> DATA_NEW 315 * then wait to grab a lock in a round-robin manner.
316 * -> DATA_TRUNC -> NODE_WRITE -> NODE_NEW -> NODE_TRUNC
317 */ 316 */
318enum lock_type { 317#define NR_GLOBAL_LOCKS 8
319 RENAME, /* for renaming operations */
320 DENTRY_OPS, /* for directory operations */
321 DATA_WRITE, /* for data write */
322 DATA_NEW, /* for data allocation */
323 DATA_TRUNC, /* for data truncate */
324 NODE_NEW, /* for node allocation */
325 NODE_TRUNC, /* for node truncate */
326 NODE_WRITE, /* for node write */
327 NR_LOCK_TYPE,
328};
329 318
330/* 319/*
331 * The below are the page types of bios used in submti_bio(). 320 * The below are the page types of bios used in submti_bio().
@@ -365,10 +354,11 @@ struct f2fs_sb_info {
365 /* for checkpoint */ 354 /* for checkpoint */
366 struct f2fs_checkpoint *ckpt; /* raw checkpoint pointer */ 355 struct f2fs_checkpoint *ckpt; /* raw checkpoint pointer */
367 struct inode *meta_inode; /* cache meta blocks */ 356 struct inode *meta_inode; /* cache meta blocks */
368 struct mutex cp_mutex; /* for checkpoint procedure */ 357 struct mutex cp_mutex; /* checkpoint procedure lock */
369 struct mutex fs_lock[NR_LOCK_TYPE]; /* for blocking FS operations */ 358 struct mutex fs_lock[NR_GLOBAL_LOCKS]; /* blocking FS operations */
370 struct mutex write_inode; /* mutex for write inode */ 359 struct mutex node_write; /* locking node writes */
371 struct mutex writepages; /* mutex for writepages() */ 360 struct mutex writepages; /* mutex for writepages() */
361 unsigned char next_lock_num; /* round-robin global locks */
372 int por_doing; /* recovery is doing or not */ 362 int por_doing; /* recovery is doing or not */
373 363
374 /* for orphan inode management */ 364 /* for orphan inode management */
@@ -503,14 +493,40 @@ static inline void clear_ckpt_flags(struct f2fs_checkpoint *cp, unsigned int f)
503 cp->ckpt_flags = cpu_to_le32(ckpt_flags); 493 cp->ckpt_flags = cpu_to_le32(ckpt_flags);
504} 494}
505 495
506static inline void mutex_lock_op(struct f2fs_sb_info *sbi, enum lock_type t) 496static inline void mutex_lock_all(struct f2fs_sb_info *sbi)
497{
498 int i = 0;
499 for (; i < NR_GLOBAL_LOCKS; i++)
500 mutex_lock(&sbi->fs_lock[i]);
501}
502
503static inline void mutex_unlock_all(struct f2fs_sb_info *sbi)
507{ 504{
508 mutex_lock_nested(&sbi->fs_lock[t], t); 505 int i = 0;
506 for (; i < NR_GLOBAL_LOCKS; i++)
507 mutex_unlock(&sbi->fs_lock[i]);
509} 508}
510 509
511static inline void mutex_unlock_op(struct f2fs_sb_info *sbi, enum lock_type t) 510static inline int mutex_lock_op(struct f2fs_sb_info *sbi)
512{ 511{
513 mutex_unlock(&sbi->fs_lock[t]); 512 unsigned char next_lock = sbi->next_lock_num % NR_GLOBAL_LOCKS;
513 int i = 0;
514
515 for (; i < NR_GLOBAL_LOCKS; i++)
516 if (mutex_trylock(&sbi->fs_lock[i]))
517 return i;
518
519 mutex_lock(&sbi->fs_lock[next_lock]);
520 sbi->next_lock_num++;
521 return next_lock;
522}
523
524static inline void mutex_unlock_op(struct f2fs_sb_info *sbi, int ilock)
525{
526 if (ilock < 0)
527 return;
528 BUG_ON(ilock >= NR_GLOBAL_LOCKS);
529 mutex_unlock(&sbi->fs_lock[ilock]);
514} 530}
515 531
516/* 532/*
@@ -879,6 +895,7 @@ long f2fs_compat_ioctl(struct file *, unsigned int, unsigned long);
879void f2fs_set_inode_flags(struct inode *); 895void f2fs_set_inode_flags(struct inode *);
880struct inode *f2fs_iget(struct super_block *, unsigned long); 896struct inode *f2fs_iget(struct super_block *, unsigned long);
881void update_inode(struct inode *, struct page *); 897void update_inode(struct inode *, struct page *);
898int update_inode_page(struct inode *);
882int f2fs_write_inode(struct inode *, struct writeback_control *); 899int f2fs_write_inode(struct inode *, struct writeback_control *);
883void f2fs_evict_inode(struct inode *); 900void f2fs_evict_inode(struct inode *);
884 901
diff --git a/fs/f2fs/file.c b/fs/f2fs/file.c
index 155b362dad63..07be88ddb9f8 100644
--- a/fs/f2fs/file.c
+++ b/fs/f2fs/file.c
@@ -34,19 +34,18 @@ static int f2fs_vm_page_mkwrite(struct vm_area_struct *vma,
34 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb); 34 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
35 block_t old_blk_addr; 35 block_t old_blk_addr;
36 struct dnode_of_data dn; 36 struct dnode_of_data dn;
37 int err; 37 int err, ilock;
38 38
39 f2fs_balance_fs(sbi); 39 f2fs_balance_fs(sbi);
40 40
41 sb_start_pagefault(inode->i_sb); 41 sb_start_pagefault(inode->i_sb);
42 42
43 mutex_lock_op(sbi, DATA_NEW);
44
45 /* block allocation */ 43 /* block allocation */
44 ilock = mutex_lock_op(sbi);
46 set_new_dnode(&dn, inode, NULL, NULL, 0); 45 set_new_dnode(&dn, inode, NULL, NULL, 0);
47 err = get_dnode_of_data(&dn, page->index, ALLOC_NODE); 46 err = get_dnode_of_data(&dn, page->index, ALLOC_NODE);
48 if (err) { 47 if (err) {
49 mutex_unlock_op(sbi, DATA_NEW); 48 mutex_unlock_op(sbi, ilock);
50 goto out; 49 goto out;
51 } 50 }
52 51
@@ -56,13 +55,12 @@ static int f2fs_vm_page_mkwrite(struct vm_area_struct *vma,
56 err = reserve_new_block(&dn); 55 err = reserve_new_block(&dn);
57 if (err) { 56 if (err) {
58 f2fs_put_dnode(&dn); 57 f2fs_put_dnode(&dn);
59 mutex_unlock_op(sbi, DATA_NEW); 58 mutex_unlock_op(sbi, ilock);
60 goto out; 59 goto out;
61 } 60 }
62 } 61 }
63 f2fs_put_dnode(&dn); 62 f2fs_put_dnode(&dn);
64 63 mutex_unlock_op(sbi, ilock);
65 mutex_unlock_op(sbi, DATA_NEW);
66 64
67 lock_page(page); 65 lock_page(page);
68 if (page->mapping != inode->i_mapping || 66 if (page->mapping != inode->i_mapping ||
@@ -223,20 +221,19 @@ static int truncate_blocks(struct inode *inode, u64 from)
223 unsigned int blocksize = inode->i_sb->s_blocksize; 221 unsigned int blocksize = inode->i_sb->s_blocksize;
224 struct dnode_of_data dn; 222 struct dnode_of_data dn;
225 pgoff_t free_from; 223 pgoff_t free_from;
226 int count = 0; 224 int count = 0, ilock = -1;
227 int err; 225 int err;
228 226
229 free_from = (pgoff_t) 227 free_from = (pgoff_t)
230 ((from + blocksize - 1) >> (sbi->log_blocksize)); 228 ((from + blocksize - 1) >> (sbi->log_blocksize));
231 229
232 mutex_lock_op(sbi, DATA_TRUNC); 230 ilock = mutex_lock_op(sbi);
233
234 set_new_dnode(&dn, inode, NULL, NULL, 0); 231 set_new_dnode(&dn, inode, NULL, NULL, 0);
235 err = get_dnode_of_data(&dn, free_from, LOOKUP_NODE); 232 err = get_dnode_of_data(&dn, free_from, LOOKUP_NODE);
236 if (err) { 233 if (err) {
237 if (err == -ENOENT) 234 if (err == -ENOENT)
238 goto free_next; 235 goto free_next;
239 mutex_unlock_op(sbi, DATA_TRUNC); 236 mutex_unlock_op(sbi, ilock);
240 return err; 237 return err;
241 } 238 }
242 239
@@ -247,6 +244,7 @@ static int truncate_blocks(struct inode *inode, u64 from)
247 244
248 count -= dn.ofs_in_node; 245 count -= dn.ofs_in_node;
249 BUG_ON(count < 0); 246 BUG_ON(count < 0);
247
250 if (dn.ofs_in_node || IS_INODE(dn.node_page)) { 248 if (dn.ofs_in_node || IS_INODE(dn.node_page)) {
251 truncate_data_blocks_range(&dn, count); 249 truncate_data_blocks_range(&dn, count);
252 free_from += count; 250 free_from += count;
@@ -255,7 +253,7 @@ static int truncate_blocks(struct inode *inode, u64 from)
255 f2fs_put_dnode(&dn); 253 f2fs_put_dnode(&dn);
256free_next: 254free_next:
257 err = truncate_inode_blocks(inode, free_from); 255 err = truncate_inode_blocks(inode, free_from);
258 mutex_unlock_op(sbi, DATA_TRUNC); 256 mutex_unlock_op(sbi, ilock);
259 257
260 /* lastly zero out the first data page */ 258 /* lastly zero out the first data page */
261 truncate_partial_data_page(inode, from); 259 truncate_partial_data_page(inode, from);
@@ -363,15 +361,16 @@ static void fill_zero(struct inode *inode, pgoff_t index,
363{ 361{
364 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb); 362 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
365 struct page *page; 363 struct page *page;
364 int ilock;
366 365
367 if (!len) 366 if (!len)
368 return; 367 return;
369 368
370 f2fs_balance_fs(sbi); 369 f2fs_balance_fs(sbi);
371 370
372 mutex_lock_op(sbi, DATA_NEW); 371 ilock = mutex_lock_op(sbi);
373 page = get_new_data_page(inode, index, false); 372 page = get_new_data_page(inode, index, false);
374 mutex_unlock_op(sbi, DATA_NEW); 373 mutex_unlock_op(sbi, ilock);
375 374
376 if (!IS_ERR(page)) { 375 if (!IS_ERR(page)) {
377 wait_on_page_writeback(page); 376 wait_on_page_writeback(page);
@@ -388,13 +387,10 @@ int truncate_hole(struct inode *inode, pgoff_t pg_start, pgoff_t pg_end)
388 387
389 for (index = pg_start; index < pg_end; index++) { 388 for (index = pg_start; index < pg_end; index++) {
390 struct dnode_of_data dn; 389 struct dnode_of_data dn;
391 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
392 390
393 mutex_lock_op(sbi, DATA_TRUNC);
394 set_new_dnode(&dn, inode, NULL, NULL, 0); 391 set_new_dnode(&dn, inode, NULL, NULL, 0);
395 err = get_dnode_of_data(&dn, index, LOOKUP_NODE); 392 err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
396 if (err) { 393 if (err) {
397 mutex_unlock_op(sbi, DATA_TRUNC);
398 if (err == -ENOENT) 394 if (err == -ENOENT)
399 continue; 395 continue;
400 return err; 396 return err;
@@ -403,7 +399,6 @@ int truncate_hole(struct inode *inode, pgoff_t pg_start, pgoff_t pg_end)
403 if (dn.data_blkaddr != NULL_ADDR) 399 if (dn.data_blkaddr != NULL_ADDR)
404 truncate_data_blocks_range(&dn, 1); 400 truncate_data_blocks_range(&dn, 1);
405 f2fs_put_dnode(&dn); 401 f2fs_put_dnode(&dn);
406 mutex_unlock_op(sbi, DATA_TRUNC);
407 } 402 }
408 return 0; 403 return 0;
409} 404}
@@ -434,6 +429,7 @@ static int punch_hole(struct inode *inode, loff_t offset, loff_t len, int mode)
434 struct address_space *mapping = inode->i_mapping; 429 struct address_space *mapping = inode->i_mapping;
435 loff_t blk_start, blk_end; 430 loff_t blk_start, blk_end;
436 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb); 431 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
432 int ilock;
437 433
438 f2fs_balance_fs(sbi); 434 f2fs_balance_fs(sbi);
439 435
@@ -441,7 +437,10 @@ static int punch_hole(struct inode *inode, loff_t offset, loff_t len, int mode)
441 blk_end = pg_end << PAGE_CACHE_SHIFT; 437 blk_end = pg_end << PAGE_CACHE_SHIFT;
442 truncate_inode_pages_range(mapping, blk_start, 438 truncate_inode_pages_range(mapping, blk_start,
443 blk_end - 1); 439 blk_end - 1);
440
441 ilock = mutex_lock_op(sbi);
444 ret = truncate_hole(inode, pg_start, pg_end); 442 ret = truncate_hole(inode, pg_start, pg_end);
443 mutex_unlock_op(sbi, ilock);
445 } 444 }
446 } 445 }
447 446
@@ -475,13 +474,13 @@ static int expand_inode_data(struct inode *inode, loff_t offset,
475 474
476 for (index = pg_start; index <= pg_end; index++) { 475 for (index = pg_start; index <= pg_end; index++) {
477 struct dnode_of_data dn; 476 struct dnode_of_data dn;
477 int ilock;
478 478
479 mutex_lock_op(sbi, DATA_NEW); 479 ilock = mutex_lock_op(sbi);
480
481 set_new_dnode(&dn, inode, NULL, NULL, 0); 480 set_new_dnode(&dn, inode, NULL, NULL, 0);
482 ret = get_dnode_of_data(&dn, index, ALLOC_NODE); 481 ret = get_dnode_of_data(&dn, index, ALLOC_NODE);
483 if (ret) { 482 if (ret) {
484 mutex_unlock_op(sbi, DATA_NEW); 483 mutex_unlock_op(sbi, ilock);
485 break; 484 break;
486 } 485 }
487 486
@@ -489,13 +488,12 @@ static int expand_inode_data(struct inode *inode, loff_t offset,
489 ret = reserve_new_block(&dn); 488 ret = reserve_new_block(&dn);
490 if (ret) { 489 if (ret) {
491 f2fs_put_dnode(&dn); 490 f2fs_put_dnode(&dn);
492 mutex_unlock_op(sbi, DATA_NEW); 491 mutex_unlock_op(sbi, ilock);
493 break; 492 break;
494 } 493 }
495 } 494 }
496 f2fs_put_dnode(&dn); 495 f2fs_put_dnode(&dn);
497 496 mutex_unlock_op(sbi, ilock);
498 mutex_unlock_op(sbi, DATA_NEW);
499 497
500 if (pg_start == pg_end) 498 if (pg_start == pg_end)
501 new_size = offset + len; 499 new_size = offset + len;
diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c
index e97f30157aa6..83cec8f868c6 100644
--- a/fs/f2fs/gc.c
+++ b/fs/f2fs/gc.c
@@ -510,7 +510,6 @@ static void move_data_page(struct inode *inode, struct page *page, int gc_type)
510 wait_on_page_writeback(page); 510 wait_on_page_writeback(page);
511 } 511 }
512 512
513 mutex_lock_op(sbi, DATA_WRITE);
514 if (clear_page_dirty_for_io(page) && 513 if (clear_page_dirty_for_io(page) &&
515 S_ISDIR(inode->i_mode)) { 514 S_ISDIR(inode->i_mode)) {
516 dec_page_count(sbi, F2FS_DIRTY_DENTS); 515 dec_page_count(sbi, F2FS_DIRTY_DENTS);
@@ -518,7 +517,6 @@ static void move_data_page(struct inode *inode, struct page *page, int gc_type)
518 } 517 }
519 set_cold_data(page); 518 set_cold_data(page);
520 do_write_data_page(page); 519 do_write_data_page(page);
521 mutex_unlock_op(sbi, DATA_WRITE);
522 clear_cold_data(page); 520 clear_cold_data(page);
523 } 521 }
524out: 522out:
diff --git a/fs/f2fs/inode.c b/fs/f2fs/inode.c
index f798ddf2c8a8..60105b710958 100644
--- a/fs/f2fs/inode.c
+++ b/fs/f2fs/inode.c
@@ -195,46 +195,49 @@ void update_inode(struct inode *inode, struct page *node_page)
195 set_page_dirty(node_page); 195 set_page_dirty(node_page);
196} 196}
197 197
198int f2fs_write_inode(struct inode *inode, struct writeback_control *wbc) 198int update_inode_page(struct inode *inode)
199{ 199{
200 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb); 200 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
201 struct page *node_page; 201 struct page *node_page;
202 bool need_lock = false;
203
204 if (inode->i_ino == F2FS_NODE_INO(sbi) ||
205 inode->i_ino == F2FS_META_INO(sbi))
206 return 0;
207
208 if (wbc)
209 f2fs_balance_fs(sbi);
210 202
211 node_page = get_node_page(sbi, inode->i_ino); 203 node_page = get_node_page(sbi, inode->i_ino);
212 if (IS_ERR(node_page)) 204 if (IS_ERR(node_page))
213 return PTR_ERR(node_page); 205 return PTR_ERR(node_page);
214 206
215 if (!PageDirty(node_page)) {
216 need_lock = true;
217 f2fs_put_page(node_page, 1);
218 mutex_lock(&sbi->write_inode);
219 node_page = get_node_page(sbi, inode->i_ino);
220 if (IS_ERR(node_page)) {
221 mutex_unlock(&sbi->write_inode);
222 return PTR_ERR(node_page);
223 }
224 }
225 update_inode(inode, node_page); 207 update_inode(inode, node_page);
226 f2fs_put_page(node_page, 1); 208 f2fs_put_page(node_page, 1);
227 if (need_lock)
228 mutex_unlock(&sbi->write_inode);
229 return 0; 209 return 0;
230} 210}
231 211
212int f2fs_write_inode(struct inode *inode, struct writeback_control *wbc)
213{
214 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
215 int ret, ilock;
216
217 if (inode->i_ino == F2FS_NODE_INO(sbi) ||
218 inode->i_ino == F2FS_META_INO(sbi))
219 return 0;
220
221 if (wbc)
222 f2fs_balance_fs(sbi);
223
224 /*
225 * We need to lock here to prevent from producing dirty node pages
226 * during the urgent cleaning time when runing out of free sections.
227 */
228 ilock = mutex_lock_op(sbi);
229 ret = update_inode_page(inode);
230 mutex_unlock_op(sbi, ilock);
231 return ret;
232}
233
232/* 234/*
233 * Called at the last iput() if i_nlink is zero 235 * Called at the last iput() if i_nlink is zero
234 */ 236 */
235void f2fs_evict_inode(struct inode *inode) 237void f2fs_evict_inode(struct inode *inode)
236{ 238{
237 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb); 239 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
240 int ilock;
238 241
239 truncate_inode_pages(&inode->i_data, 0); 242 truncate_inode_pages(&inode->i_data, 0);
240 243
@@ -255,7 +258,10 @@ void f2fs_evict_inode(struct inode *inode)
255 if (F2FS_HAS_BLOCKS(inode)) 258 if (F2FS_HAS_BLOCKS(inode))
256 f2fs_truncate(inode); 259 f2fs_truncate(inode);
257 260
261 ilock = mutex_lock_op(sbi);
258 remove_inode_page(inode); 262 remove_inode_page(inode);
263 mutex_unlock_op(sbi, ilock);
264
259 sb_end_intwrite(inode->i_sb); 265 sb_end_intwrite(inode->i_sb);
260no_delete: 266no_delete:
261 clear_inode(inode); 267 clear_inode(inode);
diff --git a/fs/f2fs/namei.c b/fs/f2fs/namei.c
index 7c6e219a479c..841f6b486bd6 100644
--- a/fs/f2fs/namei.c
+++ b/fs/f2fs/namei.c
@@ -26,19 +26,19 @@ static struct inode *f2fs_new_inode(struct inode *dir, umode_t mode)
26 nid_t ino; 26 nid_t ino;
27 struct inode *inode; 27 struct inode *inode;
28 bool nid_free = false; 28 bool nid_free = false;
29 int err; 29 int err, ilock;
30 30
31 inode = new_inode(sb); 31 inode = new_inode(sb);
32 if (!inode) 32 if (!inode)
33 return ERR_PTR(-ENOMEM); 33 return ERR_PTR(-ENOMEM);
34 34
35 mutex_lock_op(sbi, NODE_NEW); 35 ilock = mutex_lock_op(sbi);
36 if (!alloc_nid(sbi, &ino)) { 36 if (!alloc_nid(sbi, &ino)) {
37 mutex_unlock_op(sbi, NODE_NEW); 37 mutex_unlock_op(sbi, ilock);
38 err = -ENOSPC; 38 err = -ENOSPC;
39 goto fail; 39 goto fail;
40 } 40 }
41 mutex_unlock_op(sbi, NODE_NEW); 41 mutex_unlock_op(sbi, ilock);
42 42
43 inode->i_uid = current_fsuid(); 43 inode->i_uid = current_fsuid();
44 44
@@ -122,7 +122,7 @@ static int f2fs_create(struct inode *dir, struct dentry *dentry, umode_t mode,
122 struct f2fs_sb_info *sbi = F2FS_SB(sb); 122 struct f2fs_sb_info *sbi = F2FS_SB(sb);
123 struct inode *inode; 123 struct inode *inode;
124 nid_t ino = 0; 124 nid_t ino = 0;
125 int err; 125 int err, ilock;
126 126
127 f2fs_balance_fs(sbi); 127 f2fs_balance_fs(sbi);
128 128
@@ -138,7 +138,9 @@ static int f2fs_create(struct inode *dir, struct dentry *dentry, umode_t mode,
138 inode->i_mapping->a_ops = &f2fs_dblock_aops; 138 inode->i_mapping->a_ops = &f2fs_dblock_aops;
139 ino = inode->i_ino; 139 ino = inode->i_ino;
140 140
141 ilock = mutex_lock_op(sbi);
141 err = f2fs_add_link(dentry, inode); 142 err = f2fs_add_link(dentry, inode);
143 mutex_unlock_op(sbi, ilock);
142 if (err) 144 if (err)
143 goto out; 145 goto out;
144 146
@@ -162,7 +164,7 @@ static int f2fs_link(struct dentry *old_dentry, struct inode *dir,
162 struct inode *inode = old_dentry->d_inode; 164 struct inode *inode = old_dentry->d_inode;
163 struct super_block *sb = dir->i_sb; 165 struct super_block *sb = dir->i_sb;
164 struct f2fs_sb_info *sbi = F2FS_SB(sb); 166 struct f2fs_sb_info *sbi = F2FS_SB(sb);
165 int err; 167 int err, ilock;
166 168
167 f2fs_balance_fs(sbi); 169 f2fs_balance_fs(sbi);
168 170
@@ -170,7 +172,9 @@ static int f2fs_link(struct dentry *old_dentry, struct inode *dir,
170 atomic_inc(&inode->i_count); 172 atomic_inc(&inode->i_count);
171 173
172 set_inode_flag(F2FS_I(inode), FI_INC_LINK); 174 set_inode_flag(F2FS_I(inode), FI_INC_LINK);
175 ilock = mutex_lock_op(sbi);
173 err = f2fs_add_link(dentry, inode); 176 err = f2fs_add_link(dentry, inode);
177 mutex_unlock_op(sbi, ilock);
174 if (err) 178 if (err)
175 goto out; 179 goto out;
176 180
@@ -229,6 +233,7 @@ static int f2fs_unlink(struct inode *dir, struct dentry *dentry)
229 struct f2fs_dir_entry *de; 233 struct f2fs_dir_entry *de;
230 struct page *page; 234 struct page *page;
231 int err = -ENOENT; 235 int err = -ENOENT;
236 int ilock;
232 237
233 f2fs_balance_fs(sbi); 238 f2fs_balance_fs(sbi);
234 239
@@ -243,7 +248,9 @@ static int f2fs_unlink(struct inode *dir, struct dentry *dentry)
243 goto fail; 248 goto fail;
244 } 249 }
245 250
251 ilock = mutex_lock_op(sbi);
246 f2fs_delete_entry(de, page, inode); 252 f2fs_delete_entry(de, page, inode);
253 mutex_unlock_op(sbi, ilock);
247 254
248 /* In order to evict this inode, we set it dirty */ 255 /* In order to evict this inode, we set it dirty */
249 mark_inode_dirty(inode); 256 mark_inode_dirty(inode);
@@ -258,7 +265,7 @@ static int f2fs_symlink(struct inode *dir, struct dentry *dentry,
258 struct f2fs_sb_info *sbi = F2FS_SB(sb); 265 struct f2fs_sb_info *sbi = F2FS_SB(sb);
259 struct inode *inode; 266 struct inode *inode;
260 size_t symlen = strlen(symname) + 1; 267 size_t symlen = strlen(symname) + 1;
261 int err; 268 int err, ilock;
262 269
263 f2fs_balance_fs(sbi); 270 f2fs_balance_fs(sbi);
264 271
@@ -269,7 +276,9 @@ static int f2fs_symlink(struct inode *dir, struct dentry *dentry,
269 inode->i_op = &f2fs_symlink_inode_operations; 276 inode->i_op = &f2fs_symlink_inode_operations;
270 inode->i_mapping->a_ops = &f2fs_dblock_aops; 277 inode->i_mapping->a_ops = &f2fs_dblock_aops;
271 278
279 ilock = mutex_lock_op(sbi);
272 err = f2fs_add_link(dentry, inode); 280 err = f2fs_add_link(dentry, inode);
281 mutex_unlock_op(sbi, ilock);
273 if (err) 282 if (err)
274 goto out; 283 goto out;
275 284
@@ -291,7 +300,7 @@ static int f2fs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
291{ 300{
292 struct f2fs_sb_info *sbi = F2FS_SB(dir->i_sb); 301 struct f2fs_sb_info *sbi = F2FS_SB(dir->i_sb);
293 struct inode *inode; 302 struct inode *inode;
294 int err; 303 int err, ilock;
295 304
296 f2fs_balance_fs(sbi); 305 f2fs_balance_fs(sbi);
297 306
@@ -305,7 +314,9 @@ static int f2fs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
305 mapping_set_gfp_mask(inode->i_mapping, GFP_F2FS_ZERO); 314 mapping_set_gfp_mask(inode->i_mapping, GFP_F2FS_ZERO);
306 315
307 set_inode_flag(F2FS_I(inode), FI_INC_LINK); 316 set_inode_flag(F2FS_I(inode), FI_INC_LINK);
317 ilock = mutex_lock_op(sbi);
308 err = f2fs_add_link(dentry, inode); 318 err = f2fs_add_link(dentry, inode);
319 mutex_unlock_op(sbi, ilock);
309 if (err) 320 if (err)
310 goto out_fail; 321 goto out_fail;
311 322
@@ -340,6 +351,7 @@ static int f2fs_mknod(struct inode *dir, struct dentry *dentry,
340 struct f2fs_sb_info *sbi = F2FS_SB(sb); 351 struct f2fs_sb_info *sbi = F2FS_SB(sb);
341 struct inode *inode; 352 struct inode *inode;
342 int err = 0; 353 int err = 0;
354 int ilock;
343 355
344 if (!new_valid_dev(rdev)) 356 if (!new_valid_dev(rdev))
345 return -EINVAL; 357 return -EINVAL;
@@ -353,7 +365,9 @@ static int f2fs_mknod(struct inode *dir, struct dentry *dentry,
353 init_special_inode(inode, inode->i_mode, rdev); 365 init_special_inode(inode, inode->i_mode, rdev);
354 inode->i_op = &f2fs_special_inode_operations; 366 inode->i_op = &f2fs_special_inode_operations;
355 367
368 ilock = mutex_lock_op(sbi);
356 err = f2fs_add_link(dentry, inode); 369 err = f2fs_add_link(dentry, inode);
370 mutex_unlock_op(sbi, ilock);
357 if (err) 371 if (err)
358 goto out; 372 goto out;
359 373
@@ -381,7 +395,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
381 struct f2fs_dir_entry *old_dir_entry = NULL; 395 struct f2fs_dir_entry *old_dir_entry = NULL;
382 struct f2fs_dir_entry *old_entry; 396 struct f2fs_dir_entry *old_entry;
383 struct f2fs_dir_entry *new_entry; 397 struct f2fs_dir_entry *new_entry;
384 int err = -ENOENT; 398 int err = -ENOENT, ilock = -1;
385 399
386 f2fs_balance_fs(sbi); 400 f2fs_balance_fs(sbi);
387 401
@@ -396,7 +410,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
396 goto out_old; 410 goto out_old;
397 } 411 }
398 412
399 mutex_lock_op(sbi, RENAME); 413 ilock = mutex_lock_op(sbi);
400 414
401 if (new_inode) { 415 if (new_inode) {
402 struct page *new_page; 416 struct page *new_page;
@@ -419,7 +433,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
419 drop_nlink(new_inode); 433 drop_nlink(new_inode);
420 if (!new_inode->i_nlink) 434 if (!new_inode->i_nlink)
421 add_orphan_inode(sbi, new_inode->i_ino); 435 add_orphan_inode(sbi, new_inode->i_ino);
422 f2fs_write_inode(new_inode, NULL); 436 update_inode_page(new_inode);
423 } else { 437 } else {
424 err = f2fs_add_link(new_dentry, old_inode); 438 err = f2fs_add_link(new_dentry, old_inode);
425 if (err) 439 if (err)
@@ -427,7 +441,7 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
427 441
428 if (old_dir_entry) { 442 if (old_dir_entry) {
429 inc_nlink(new_dir); 443 inc_nlink(new_dir);
430 f2fs_write_inode(new_dir, NULL); 444 update_inode_page(new_dir);
431 } 445 }
432 } 446 }
433 447
@@ -445,10 +459,10 @@ static int f2fs_rename(struct inode *old_dir, struct dentry *old_dentry,
445 f2fs_put_page(old_dir_page, 0); 459 f2fs_put_page(old_dir_page, 0);
446 } 460 }
447 drop_nlink(old_dir); 461 drop_nlink(old_dir);
448 f2fs_write_inode(old_dir, NULL); 462 update_inode_page(old_dir);
449 } 463 }
450 464
451 mutex_unlock_op(sbi, RENAME); 465 mutex_unlock_op(sbi, ilock);
452 return 0; 466 return 0;
453 467
454out_dir: 468out_dir:
@@ -456,7 +470,7 @@ out_dir:
456 kunmap(old_dir_page); 470 kunmap(old_dir_page);
457 f2fs_put_page(old_dir_page, 0); 471 f2fs_put_page(old_dir_page, 0);
458 } 472 }
459 mutex_unlock_op(sbi, RENAME); 473 mutex_unlock_op(sbi, ilock);
460out_old: 474out_old:
461 kunmap(old_page); 475 kunmap(old_page);
462 f2fs_put_page(old_page, 0); 476 f2fs_put_page(old_page, 0);
diff --git a/fs/f2fs/node.c b/fs/f2fs/node.c
index ad3adbee842a..5a7edf90ca45 100644
--- a/fs/f2fs/node.c
+++ b/fs/f2fs/node.c
@@ -385,6 +385,9 @@ got:
385 385
386/* 386/*
387 * Caller should call f2fs_put_dnode(dn). 387 * Caller should call f2fs_put_dnode(dn).
388 * Also, it should grab and release a mutex by calling mutex_lock_op() and
389 * mutex_unlock_op() only if ro is not set RDONLY_NODE.
390 * In the case of RDONLY_NODE, we don't need to care about mutex.
388 */ 391 */
389int get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode) 392int get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode)
390{ 393{
@@ -415,11 +418,8 @@ int get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode)
415 bool done = false; 418 bool done = false;
416 419
417 if (!nids[i] && mode == ALLOC_NODE) { 420 if (!nids[i] && mode == ALLOC_NODE) {
418 mutex_lock_op(sbi, NODE_NEW);
419
420 /* alloc new node */ 421 /* alloc new node */
421 if (!alloc_nid(sbi, &(nids[i]))) { 422 if (!alloc_nid(sbi, &(nids[i]))) {
422 mutex_unlock_op(sbi, NODE_NEW);
423 err = -ENOSPC; 423 err = -ENOSPC;
424 goto release_pages; 424 goto release_pages;
425 } 425 }
@@ -428,14 +428,12 @@ int get_dnode_of_data(struct dnode_of_data *dn, pgoff_t index, int mode)
428 npage[i] = new_node_page(dn, noffset[i]); 428 npage[i] = new_node_page(dn, noffset[i]);
429 if (IS_ERR(npage[i])) { 429 if (IS_ERR(npage[i])) {
430 alloc_nid_failed(sbi, nids[i]); 430 alloc_nid_failed(sbi, nids[i]);
431 mutex_unlock_op(sbi, NODE_NEW);
432 err = PTR_ERR(npage[i]); 431 err = PTR_ERR(npage[i]);
433 goto release_pages; 432 goto release_pages;
434 } 433 }
435 434
436 set_nid(parent, offset[i - 1], nids[i], i == 1); 435 set_nid(parent, offset[i - 1], nids[i], i == 1);
437 alloc_nid_done(sbi, nids[i]); 436 alloc_nid_done(sbi, nids[i]);
438 mutex_unlock_op(sbi, NODE_NEW);
439 done = true; 437 done = true;
440 } else if (mode == LOOKUP_NODE_RA && i == level && level > 1) { 438 } else if (mode == LOOKUP_NODE_RA && i == level && level > 1) {
441 npage[i] = get_node_page_ra(parent, offset[i - 1]); 439 npage[i] = get_node_page_ra(parent, offset[i - 1]);
@@ -745,6 +743,10 @@ fail:
745 return err > 0 ? 0 : err; 743 return err > 0 ? 0 : err;
746} 744}
747 745
746/*
747 * Caller should grab and release a mutex by calling mutex_lock_op() and
748 * mutex_unlock_op().
749 */
748int remove_inode_page(struct inode *inode) 750int remove_inode_page(struct inode *inode)
749{ 751{
750 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb); 752 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
@@ -752,21 +754,16 @@ int remove_inode_page(struct inode *inode)
752 nid_t ino = inode->i_ino; 754 nid_t ino = inode->i_ino;
753 struct dnode_of_data dn; 755 struct dnode_of_data dn;
754 756
755 mutex_lock_op(sbi, NODE_TRUNC);
756 page = get_node_page(sbi, ino); 757 page = get_node_page(sbi, ino);
757 if (IS_ERR(page)) { 758 if (IS_ERR(page))
758 mutex_unlock_op(sbi, NODE_TRUNC);
759 return PTR_ERR(page); 759 return PTR_ERR(page);
760 }
761 760
762 if (F2FS_I(inode)->i_xattr_nid) { 761 if (F2FS_I(inode)->i_xattr_nid) {
763 nid_t nid = F2FS_I(inode)->i_xattr_nid; 762 nid_t nid = F2FS_I(inode)->i_xattr_nid;
764 struct page *npage = get_node_page(sbi, nid); 763 struct page *npage = get_node_page(sbi, nid);
765 764
766 if (IS_ERR(npage)) { 765 if (IS_ERR(npage))
767 mutex_unlock_op(sbi, NODE_TRUNC);
768 return PTR_ERR(npage); 766 return PTR_ERR(npage);
769 }
770 767
771 F2FS_I(inode)->i_xattr_nid = 0; 768 F2FS_I(inode)->i_xattr_nid = 0;
772 set_new_dnode(&dn, inode, page, npage, nid); 769 set_new_dnode(&dn, inode, page, npage, nid);
@@ -778,23 +775,18 @@ int remove_inode_page(struct inode *inode)
778 BUG_ON(inode->i_blocks != 0 && inode->i_blocks != 1); 775 BUG_ON(inode->i_blocks != 0 && inode->i_blocks != 1);
779 set_new_dnode(&dn, inode, page, page, ino); 776 set_new_dnode(&dn, inode, page, page, ino);
780 truncate_node(&dn); 777 truncate_node(&dn);
781
782 mutex_unlock_op(sbi, NODE_TRUNC);
783 return 0; 778 return 0;
784} 779}
785 780
786int new_inode_page(struct inode *inode, const struct qstr *name) 781int new_inode_page(struct inode *inode, const struct qstr *name)
787{ 782{
788 struct f2fs_sb_info *sbi = F2FS_SB(inode->i_sb);
789 struct page *page; 783 struct page *page;
790 struct dnode_of_data dn; 784 struct dnode_of_data dn;
791 785
792 /* allocate inode page for new inode */ 786 /* allocate inode page for new inode */
793 set_new_dnode(&dn, inode, NULL, NULL, inode->i_ino); 787 set_new_dnode(&dn, inode, NULL, NULL, inode->i_ino);
794 mutex_lock_op(sbi, NODE_NEW);
795 page = new_node_page(&dn, 0); 788 page = new_node_page(&dn, 0);
796 init_dent_inode(name, page); 789 init_dent_inode(name, page);
797 mutex_unlock_op(sbi, NODE_NEW);
798 if (IS_ERR(page)) 790 if (IS_ERR(page))
799 return PTR_ERR(page); 791 return PTR_ERR(page);
800 f2fs_put_page(page, 1); 792 f2fs_put_page(page, 1);
@@ -985,7 +977,7 @@ void sync_inode_page(struct dnode_of_data *dn)
985 if (!dn->inode_page_locked) 977 if (!dn->inode_page_locked)
986 unlock_page(dn->inode_page); 978 unlock_page(dn->inode_page);
987 } else { 979 } else {
988 f2fs_write_inode(dn->inode, NULL); 980 update_inode_page(dn->inode);
989 } 981 }
990} 982}
991 983
@@ -1102,8 +1094,6 @@ static int f2fs_write_node_page(struct page *page,
1102 1094
1103 wait_on_page_writeback(page); 1095 wait_on_page_writeback(page);
1104 1096
1105 mutex_lock_op(sbi, NODE_WRITE);
1106
1107 /* get old block addr of this node page */ 1097 /* get old block addr of this node page */
1108 nid = nid_of_node(page); 1098 nid = nid_of_node(page);
1109 BUG_ON(page->index != nid); 1099 BUG_ON(page->index != nid);
@@ -1111,25 +1101,25 @@ static int f2fs_write_node_page(struct page *page,
1111 get_node_info(sbi, nid, &ni); 1101 get_node_info(sbi, nid, &ni);
1112 1102
1113 /* This page is already truncated */ 1103 /* This page is already truncated */
1114 if (ni.blk_addr == NULL_ADDR) 1104 if (ni.blk_addr == NULL_ADDR) {
1115 goto out; 1105 dec_page_count(sbi, F2FS_DIRTY_NODES);
1106 unlock_page(page);
1107 return 0;
1108 }
1116 1109
1117 if (wbc->for_reclaim) { 1110 if (wbc->for_reclaim) {
1118 dec_page_count(sbi, F2FS_DIRTY_NODES); 1111 dec_page_count(sbi, F2FS_DIRTY_NODES);
1119 wbc->pages_skipped++; 1112 wbc->pages_skipped++;
1120 set_page_dirty(page); 1113 set_page_dirty(page);
1121 mutex_unlock_op(sbi, NODE_WRITE);
1122 return AOP_WRITEPAGE_ACTIVATE; 1114 return AOP_WRITEPAGE_ACTIVATE;
1123 } 1115 }
1124 1116
1117 mutex_lock(&sbi->node_write);
1125 set_page_writeback(page); 1118 set_page_writeback(page);
1126
1127 /* insert node offset */
1128 write_node_page(sbi, page, nid, ni.blk_addr, &new_addr); 1119 write_node_page(sbi, page, nid, ni.blk_addr, &new_addr);
1129 set_node_addr(sbi, &ni, new_addr); 1120 set_node_addr(sbi, &ni, new_addr);
1130out:
1131 dec_page_count(sbi, F2FS_DIRTY_NODES); 1121 dec_page_count(sbi, F2FS_DIRTY_NODES);
1132 mutex_unlock_op(sbi, NODE_WRITE); 1122 mutex_unlock(&sbi->node_write);
1133 unlock_page(page); 1123 unlock_page(page);
1134 return 0; 1124 return 0;
1135} 1125}
diff --git a/fs/f2fs/recovery.c b/fs/f2fs/recovery.c
index 61bdaa755906..f16d12df8e99 100644
--- a/fs/f2fs/recovery.c
+++ b/fs/f2fs/recovery.c
@@ -242,6 +242,7 @@ static int do_recover_data(struct f2fs_sb_info *sbi, struct inode *inode,
242 struct f2fs_summary sum; 242 struct f2fs_summary sum;
243 struct node_info ni; 243 struct node_info ni;
244 int err = 0; 244 int err = 0;
245 int ilock;
245 246
246 start = start_bidx_of_node(ofs_of_node(page)); 247 start = start_bidx_of_node(ofs_of_node(page));
247 if (IS_INODE(page)) 248 if (IS_INODE(page))
@@ -249,10 +250,14 @@ static int do_recover_data(struct f2fs_sb_info *sbi, struct inode *inode,
249 else 250 else
250 end = start + ADDRS_PER_BLOCK; 251 end = start + ADDRS_PER_BLOCK;
251 252
253 ilock = mutex_lock_op(sbi);
252 set_new_dnode(&dn, inode, NULL, NULL, 0); 254 set_new_dnode(&dn, inode, NULL, NULL, 0);
255
253 err = get_dnode_of_data(&dn, start, ALLOC_NODE); 256 err = get_dnode_of_data(&dn, start, ALLOC_NODE);
254 if (err) 257 if (err) {
258 mutex_unlock_op(sbi, ilock);
255 return err; 259 return err;
260 }
256 261
257 wait_on_page_writeback(dn.node_page); 262 wait_on_page_writeback(dn.node_page);
258 263
@@ -297,6 +302,7 @@ static int do_recover_data(struct f2fs_sb_info *sbi, struct inode *inode,
297 302
298 recover_node_page(sbi, dn.node_page, &sum, &ni, blkaddr); 303 recover_node_page(sbi, dn.node_page, &sum, &ni, blkaddr);
299 f2fs_put_dnode(&dn); 304 f2fs_put_dnode(&dn);
305 mutex_unlock_op(sbi, ilock);
300 return 0; 306 return 0;
301} 307}
302 308
diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c
index ca5413346653..2643c49c38a5 100644
--- a/fs/f2fs/super.c
+++ b/fs/f2fs/super.c
@@ -556,11 +556,11 @@ static int f2fs_fill_super(struct super_block *sb, void *data, int silent)
556 sbi->raw_super = raw_super; 556 sbi->raw_super = raw_super;
557 sbi->raw_super_buf = raw_super_buf; 557 sbi->raw_super_buf = raw_super_buf;
558 mutex_init(&sbi->gc_mutex); 558 mutex_init(&sbi->gc_mutex);
559 mutex_init(&sbi->write_inode);
560 mutex_init(&sbi->writepages); 559 mutex_init(&sbi->writepages);
561 mutex_init(&sbi->cp_mutex); 560 mutex_init(&sbi->cp_mutex);
562 for (i = 0; i < NR_LOCK_TYPE; i++) 561 for (i = 0; i < NR_GLOBAL_LOCKS; i++)
563 mutex_init(&sbi->fs_lock[i]); 562 mutex_init(&sbi->fs_lock[i]);
563 mutex_init(&sbi->node_write);
564 sbi->por_doing = 0; 564 sbi->por_doing = 0;
565 spin_lock_init(&sbi->stat_lock); 565 spin_lock_init(&sbi->stat_lock);
566 init_rwsem(&sbi->bio_sem); 566 init_rwsem(&sbi->bio_sem);
diff --git a/fs/f2fs/xattr.c b/fs/f2fs/xattr.c
index 3bfea80610ff..0b02dce31356 100644
--- a/fs/f2fs/xattr.c
+++ b/fs/f2fs/xattr.c
@@ -307,6 +307,7 @@ int f2fs_setxattr(struct inode *inode, int name_index, const char *name,
307 int error, found, free, newsize; 307 int error, found, free, newsize;
308 size_t name_len; 308 size_t name_len;
309 char *pval; 309 char *pval;
310 int ilock;
310 311
311 if (name == NULL) 312 if (name == NULL)
312 return -EINVAL; 313 return -EINVAL;
@@ -321,7 +322,8 @@ int f2fs_setxattr(struct inode *inode, int name_index, const char *name,
321 322
322 f2fs_balance_fs(sbi); 323 f2fs_balance_fs(sbi);
323 324
324 mutex_lock_op(sbi, NODE_NEW); 325 ilock = mutex_lock_op(sbi);
326
325 if (!fi->i_xattr_nid) { 327 if (!fi->i_xattr_nid) {
326 /* Allocate new attribute block */ 328 /* Allocate new attribute block */
327 struct dnode_of_data dn; 329 struct dnode_of_data dn;
@@ -433,13 +435,13 @@ int f2fs_setxattr(struct inode *inode, int name_index, const char *name,
433 inode->i_ctime = CURRENT_TIME; 435 inode->i_ctime = CURRENT_TIME;
434 clear_inode_flag(fi, FI_ACL_MODE); 436 clear_inode_flag(fi, FI_ACL_MODE);
435 } 437 }
436 f2fs_write_inode(inode, NULL); 438 update_inode_page(inode);
437 mutex_unlock_op(sbi, NODE_NEW); 439 mutex_unlock_op(sbi, ilock);
438 440
439 return 0; 441 return 0;
440cleanup: 442cleanup:
441 f2fs_put_page(page, 1); 443 f2fs_put_page(page, 1);
442exit: 444exit:
443 mutex_unlock_op(sbi, NODE_NEW); 445 mutex_unlock_op(sbi, ilock);
444 return error; 446 return error;
445} 447}