1 files changed, 147 insertions, 71 deletions

diff --git a/fs/f2fs/data.c b/fs/f2fs/data.c
index 7ec697b37f19..985ed023a750 100644
--- a/fs/f2fs/data.c
+++ b/fs/f2fs/data.c
@@ -22,6 +22,7 @@
 #include "f2fs.h"
 #include "node.h"
 #include "segment.h"
+#include "trace.h"
 #include <trace/events/f2fs.h>
 
 static void f2fs_read_end_io(struct bio *bio, int err)
@@ -95,11 +96,9 @@ static void __submit_merged_bio(struct f2fs_bio_info *io)
                 return;
 
         if (is_read_io(fio->rw))
-                trace_f2fs_submit_read_bio(io->sbi->sb, fio->rw,
-                                                        fio->type, io->bio);
+                trace_f2fs_submit_read_bio(io->sbi->sb, fio, io->bio);
         else
-                trace_f2fs_submit_write_bio(io->sbi->sb, fio->rw,
-                                                        fio->type, io->bio);
+                trace_f2fs_submit_write_bio(io->sbi->sb, fio, io->bio);
 
         submit_bio(fio->rw, io->bio);
         io->bio = NULL;
@@ -132,14 +131,15 @@ void f2fs_submit_merged_bio(struct f2fs_sb_info *sbi,
  * Return unlocked page.
  */
 int f2fs_submit_page_bio(struct f2fs_sb_info *sbi, struct page *page,
-                                        block_t blk_addr, int rw)
+                                        struct f2fs_io_info *fio)
 {
         struct bio *bio;
 
-        trace_f2fs_submit_page_bio(page, blk_addr, rw);
+        trace_f2fs_submit_page_bio(page, fio);
+        f2fs_trace_ios(page, fio, 0);
 
         /* Allocate a new bio */
-        bio = __bio_alloc(sbi, blk_addr, 1, is_read_io(rw));
+        bio = __bio_alloc(sbi, fio->blk_addr, 1, is_read_io(fio->rw));
 
         if (bio_add_page(bio, page, PAGE_CACHE_SIZE, 0) < PAGE_CACHE_SIZE) {
                 bio_put(bio);
@@ -147,12 +147,12 @@ int f2fs_submit_page_bio(struct f2fs_sb_info *sbi, struct page *page,
                 return -EFAULT;
         }
 
-        submit_bio(rw, bio);
+        submit_bio(fio->rw, bio);
         return 0;
 }
 
 void f2fs_submit_page_mbio(struct f2fs_sb_info *sbi, struct page *page,
-                        block_t blk_addr, struct f2fs_io_info *fio)
+                                        struct f2fs_io_info *fio)
 {
         enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
         struct f2fs_bio_info *io;
@@ -160,21 +160,21 @@ void f2fs_submit_page_mbio(struct f2fs_sb_info *sbi, struct page *page,
 
         io = is_read ? &sbi->read_io : &sbi->write_io[btype];
 
-        verify_block_addr(sbi, blk_addr);
+        verify_block_addr(sbi, fio->blk_addr);
 
         down_write(&io->io_rwsem);
 
         if (!is_read)
                 inc_page_count(sbi, F2FS_WRITEBACK);
 
-        if (io->bio && (io->last_block_in_bio != blk_addr - 1 ||
+        if (io->bio && (io->last_block_in_bio != fio->blk_addr - 1 ||
                                                 io->fio.rw != fio->rw))
                 __submit_merged_bio(io);
 alloc_new:
         if (io->bio == NULL) {
                 int bio_blocks = MAX_BIO_BLOCKS(sbi);
 
-                io->bio = __bio_alloc(sbi, blk_addr, bio_blocks, is_read);
+                io->bio = __bio_alloc(sbi, fio->blk_addr, bio_blocks, is_read);
                 io->fio = *fio;
         }
 
@@ -184,10 +184,11 @@ alloc_new:
                 goto alloc_new;
         }
 
-        io->last_block_in_bio = blk_addr;
+        io->last_block_in_bio = fio->blk_addr;
+        f2fs_trace_ios(page, fio, 0);
 
         up_write(&io->io_rwsem);
-        trace_f2fs_submit_page_mbio(page, fio->rw, fio->type, blk_addr);
+        trace_f2fs_submit_page_mbio(page, fio);
 }
 
 /*
@@ -196,7 +197,7 @@ alloc_new:
  *  ->node_page
  *    update block addresses in the node page
  */
-static void __set_data_blkaddr(struct dnode_of_data *dn, block_t new_addr)
+static void __set_data_blkaddr(struct dnode_of_data *dn)
 {
         struct f2fs_node *rn;
         __le32 *addr_array;
@@ -209,7 +210,7 @@ static void __set_data_blkaddr(struct dnode_of_data *dn, block_t new_addr)
 
         /* Get physical address of data block */
         addr_array = blkaddr_in_node(rn);
-        addr_array[ofs_in_node] = cpu_to_le32(new_addr);
+        addr_array[ofs_in_node] = cpu_to_le32(dn->data_blkaddr);
         set_page_dirty(node_page);
 }
 
@@ -224,8 +225,8 @@ int reserve_new_block(struct dnode_of_data *dn)
 
         trace_f2fs_reserve_new_block(dn->inode, dn->nid, dn->ofs_in_node);
 
-        __set_data_blkaddr(dn, NEW_ADDR);
         dn->data_blkaddr = NEW_ADDR;
+        __set_data_blkaddr(dn);
         mark_inode_dirty(dn->inode);
         sync_inode_page(dn);
         return 0;
@@ -273,7 +274,7 @@ static int check_extent_cache(struct inode *inode, pgoff_t pgofs,
                 unsigned int blkbits = inode->i_sb->s_blocksize_bits;
                 size_t count;
 
-                clear_buffer_new(bh_result);
+                set_buffer_new(bh_result);
                 map_bh(bh_result, inode->i_sb,
                                 start_blkaddr + pgofs - start_fofs);
                 count = end_fofs - pgofs + 1;
@@ -290,23 +291,24 @@ static int check_extent_cache(struct inode *inode, pgoff_t pgofs,
         return 0;
 }
 
-void update_extent_cache(block_t blk_addr, struct dnode_of_data *dn)
+void update_extent_cache(struct dnode_of_data *dn)
 {
         struct f2fs_inode_info *fi = F2FS_I(dn->inode);
         pgoff_t fofs, start_fofs, end_fofs;
         block_t start_blkaddr, end_blkaddr;
         int need_update = true;
 
-        f2fs_bug_on(F2FS_I_SB(dn->inode), blk_addr == NEW_ADDR);
-        fofs = start_bidx_of_node(ofs_of_node(dn->node_page), fi) +
-                                                        dn->ofs_in_node;
+        f2fs_bug_on(F2FS_I_SB(dn->inode), dn->data_blkaddr == NEW_ADDR);
 
         /* Update the page address in the parent node */
-        __set_data_blkaddr(dn, blk_addr);
+        __set_data_blkaddr(dn);
 
         if (is_inode_flag_set(fi, FI_NO_EXTENT))
                 return;
 
+        fofs = start_bidx_of_node(ofs_of_node(dn->node_page), fi) +
+                                                        dn->ofs_in_node;
+
         write_lock(&fi->ext.ext_lock);
 
         start_fofs = fi->ext.fofs;
@@ -320,16 +322,16 @@ void update_extent_cache(block_t blk_addr, struct dnode_of_data *dn)
 
         /* Initial extent */
         if (fi->ext.len == 0) {
-                if (blk_addr != NULL_ADDR) {
+                if (dn->data_blkaddr != NULL_ADDR) {
                         fi->ext.fofs = fofs;
-                        fi->ext.blk_addr = blk_addr;
+                        fi->ext.blk_addr = dn->data_blkaddr;
                         fi->ext.len = 1;
                 }
                 goto end_update;
         }
 
         /* Front merge */
-        if (fofs == start_fofs - 1 && blk_addr == start_blkaddr - 1) {
+        if (fofs == start_fofs - 1 && dn->data_blkaddr == start_blkaddr - 1) {
                 fi->ext.fofs--;
                 fi->ext.blk_addr--;
                 fi->ext.len++;
@@ -337,7 +339,7 @@ void update_extent_cache(block_t blk_addr, struct dnode_of_data *dn)
         }
 
         /* Back merge */
-        if (fofs == end_fofs + 1 && blk_addr == end_blkaddr + 1) {
+        if (fofs == end_fofs + 1 && dn->data_blkaddr == end_blkaddr + 1) {
                 fi->ext.len++;
                 goto end_update;
         }
@@ -376,6 +378,10 @@ struct page *find_data_page(struct inode *inode, pgoff_t index, bool sync)
         struct dnode_of_data dn;
         struct page *page;
         int err;
+        struct f2fs_io_info fio = {
+                .type = DATA,
+                .rw = sync ? READ_SYNC : READA,
+        };
 
         page = find_get_page(mapping, index);
         if (page && PageUptodate(page))
@@ -404,8 +410,8 @@ struct page *find_data_page(struct inode *inode, pgoff_t index, bool sync)
                 return page;
         }
 
-        err = f2fs_submit_page_bio(F2FS_I_SB(inode), page, dn.data_blkaddr,
-                                        sync ? READ_SYNC : READA);
+        fio.blk_addr = dn.data_blkaddr;
+        err = f2fs_submit_page_bio(F2FS_I_SB(inode), page, &fio);
         if (err)
                 return ERR_PTR(err);
 
@@ -430,7 +436,10 @@ struct page *get_lock_data_page(struct inode *inode, pgoff_t index)
         struct dnode_of_data dn;
         struct page *page;
         int err;
-
+        struct f2fs_io_info fio = {
+                .type = DATA,
+                .rw = READ_SYNC,
+        };
 repeat:
         page = grab_cache_page(mapping, index);
         if (!page)
@@ -464,8 +473,8 @@ repeat:
                 return page;
         }
 
-        err = f2fs_submit_page_bio(F2FS_I_SB(inode), page,
-                                        dn.data_blkaddr, READ_SYNC);
+        fio.blk_addr = dn.data_blkaddr;
+        err = f2fs_submit_page_bio(F2FS_I_SB(inode), page, &fio);
         if (err)
                 return ERR_PTR(err);
 
@@ -515,8 +524,12 @@ repeat:
                 zero_user_segment(page, 0, PAGE_CACHE_SIZE);
                 SetPageUptodate(page);
         } else {
-                err = f2fs_submit_page_bio(F2FS_I_SB(inode), page,
-                                                dn.data_blkaddr, READ_SYNC);
+                struct f2fs_io_info fio = {
+                        .type = DATA,
+                        .rw = READ_SYNC,
+                        .blk_addr = dn.data_blkaddr,
+                };
+                err = f2fs_submit_page_bio(F2FS_I_SB(inode), page, &fio);
                 if (err)
                         goto put_err;
 
@@ -550,30 +563,25 @@ static int __allocate_data_block(struct dnode_of_data *dn)
         struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
         struct f2fs_inode_info *fi = F2FS_I(dn->inode);
         struct f2fs_summary sum;
-        block_t new_blkaddr;
         struct node_info ni;
+        int seg = CURSEG_WARM_DATA;
         pgoff_t fofs;
-        int type;
 
         if (unlikely(is_inode_flag_set(F2FS_I(dn->inode), FI_NO_ALLOC)))
                 return -EPERM;
         if (unlikely(!inc_valid_block_count(sbi, dn->inode, 1)))
                 return -ENOSPC;
 
-        __set_data_blkaddr(dn, NEW_ADDR);
-        dn->data_blkaddr = NEW_ADDR;
-
         get_node_info(sbi, dn->nid, &ni);
         set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
 
-        type = CURSEG_WARM_DATA;
+        if (dn->ofs_in_node == 0 && dn->inode_page == dn->node_page)
+                seg = CURSEG_DIRECT_IO;
 
-        allocate_data_block(sbi, NULL, NULL_ADDR, &new_blkaddr, &sum, type);
+        allocate_data_block(sbi, NULL, NULL_ADDR, &dn->data_blkaddr, &sum, seg);
 
         /* direct IO doesn't use extent cache to maximize the performance */
-        set_inode_flag(F2FS_I(dn->inode), FI_NO_EXTENT);
-        update_extent_cache(new_blkaddr, dn);
-        clear_inode_flag(F2FS_I(dn->inode), FI_NO_EXTENT);
+        __set_data_blkaddr(dn);
 
         /* update i_size */
         fofs = start_bidx_of_node(ofs_of_node(dn->node_page), fi) +
@@ -581,10 +589,59 @@ static int __allocate_data_block(struct dnode_of_data *dn)
         if (i_size_read(dn->inode) < ((fofs + 1) << PAGE_CACHE_SHIFT))
                 i_size_write(dn->inode, ((fofs + 1) << PAGE_CACHE_SHIFT));
 
-        dn->data_blkaddr = new_blkaddr;
         return 0;
 }
 
+static void __allocate_data_blocks(struct inode *inode, loff_t offset,
+                                                        size_t count)
+{
+        struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
+        struct dnode_of_data dn;
+        u64 start = F2FS_BYTES_TO_BLK(offset);
+        u64 len = F2FS_BYTES_TO_BLK(count);
+        bool allocated;
+        u64 end_offset;
+
+        while (len) {
+                f2fs_balance_fs(sbi);
+                f2fs_lock_op(sbi);
+
+                /* When reading holes, we need its node page */
+                set_new_dnode(&dn, inode, NULL, NULL, 0);
+                if (get_dnode_of_data(&dn, start, ALLOC_NODE))
+                        goto out;
+
+                allocated = false;
+                end_offset = ADDRS_PER_PAGE(dn.node_page, F2FS_I(inode));
+
+                while (dn.ofs_in_node < end_offset && len) {
+                        if (dn.data_blkaddr == NULL_ADDR) {
+                                if (__allocate_data_block(&dn))
+                                        goto sync_out;
+                                allocated = true;
+                        }
+                        len--;
+                        start++;
+                        dn.ofs_in_node++;
+                }
+
+                if (allocated)
+                        sync_inode_page(&dn);
+
+                f2fs_put_dnode(&dn);
+                f2fs_unlock_op(sbi);
+        }
+        return;
+
+sync_out:
+        if (allocated)
+                sync_inode_page(&dn);
+        f2fs_put_dnode(&dn);
+out:
+        f2fs_unlock_op(sbi);
+        return;
+}
+
 /*
  * get_data_block() now supported readahead/bmap/rw direct_IO with mapped bh.
  * If original data blocks are allocated, then give them to blockdev.
@@ -610,10 +667,8 @@ static int __get_data_block(struct inode *inode, sector_t iblock,
         if (check_extent_cache(inode, pgofs, bh_result))
                 goto out;
 
-        if (create) {
-                f2fs_balance_fs(F2FS_I_SB(inode));
+        if (create)
                 f2fs_lock_op(F2FS_I_SB(inode));
-        }
 
         /* When reading holes, we need its node page */
         set_new_dnode(&dn, inode, NULL, NULL, 0);
@@ -627,12 +682,14 @@ static int __get_data_block(struct inode *inode, sector_t iblock,
                 goto put_out;
 
         if (dn.data_blkaddr != NULL_ADDR) {
+                set_buffer_new(bh_result);
                 map_bh(bh_result, inode->i_sb, dn.data_blkaddr);
         } else if (create) {
                 err = __allocate_data_block(&dn);
                 if (err)
                         goto put_out;
                 allocated = true;
+                set_buffer_new(bh_result);
                 map_bh(bh_result, inode->i_sb, dn.data_blkaddr);
         } else {
                 goto put_out;
@@ -745,7 +802,6 @@ static int f2fs_read_data_pages(struct file *file,
 int do_write_data_page(struct page *page, struct f2fs_io_info *fio)
 {
         struct inode *inode = page->mapping->host;
-        block_t old_blkaddr, new_blkaddr;
         struct dnode_of_data dn;
         int err = 0;
 
@@ -754,10 +810,10 @@ int do_write_data_page(struct page *page, struct f2fs_io_info *fio)
         if (err)
                 return err;
 
-        old_blkaddr = dn.data_blkaddr;
+        fio->blk_addr = dn.data_blkaddr;
 
         /* This page is already truncated */
-        if (old_blkaddr == NULL_ADDR)
+        if (fio->blk_addr == NULL_ADDR)
                 goto out_writepage;
 
         set_page_writeback(page);
@@ -766,14 +822,14 @@ int do_write_data_page(struct page *page, struct f2fs_io_info *fio)
          * If current allocation needs SSR,
          * it had better in-place writes for updated data.
          */
-        if (unlikely(old_blkaddr != NEW_ADDR &&
+        if (unlikely(fio->blk_addr != NEW_ADDR &&
                         !is_cold_data(page) &&
                         need_inplace_update(inode))) {
-                rewrite_data_page(page, old_blkaddr, fio);
+                rewrite_data_page(page, fio);
                 set_inode_flag(F2FS_I(inode), FI_UPDATE_WRITE);
         } else {
-                write_data_page(page, &dn, &new_blkaddr, fio);
-                update_extent_cache(new_blkaddr, &dn);
+                write_data_page(page, &dn, fio);
+                update_extent_cache(&dn);
                 set_inode_flag(F2FS_I(inode), FI_APPEND_WRITE);
         }
 out_writepage:
@@ -812,7 +868,12 @@ static int f2fs_write_data_page(struct page *page,
 
         zero_user_segment(page, offset, PAGE_CACHE_SIZE);
 write:
-        if (unlikely(sbi->por_doing))
+        if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
+                goto redirty_out;
+        if (f2fs_is_drop_cache(inode))
+                goto out;
+        if (f2fs_is_volatile_file(inode) && !wbc->for_reclaim &&
+                        available_free_memory(sbi, BASE_CHECK))
                 goto redirty_out;
 
         /* Dentry blocks are controlled by checkpoint */
@@ -826,7 +887,6 @@ write:
         /* we should bypass data pages to proceed the kworkder jobs */
         if (unlikely(f2fs_cp_error(sbi))) {
                 SetPageError(page);
-                unlock_page(page);
                 goto out;
         }
 
@@ -1002,8 +1062,12 @@ put_next:
         if (dn.data_blkaddr == NEW_ADDR) {
                 zero_user_segment(page, 0, PAGE_CACHE_SIZE);
         } else {
-                err = f2fs_submit_page_bio(sbi, page, dn.data_blkaddr,
-                                           READ_SYNC);
+                struct f2fs_io_info fio = {
+                        .type = DATA,
+                        .rw = READ_SYNC,
+                        .blk_addr = dn.data_blkaddr,
+                };
+                err = f2fs_submit_page_bio(sbi, page, &fio);
                 if (err)
                         goto fail;
 
@@ -1092,6 +1156,9 @@ static ssize_t f2fs_direct_IO(int rw, struct kiocb *iocb,
 
         trace_f2fs_direct_IO_enter(inode, offset, count, rw);
 
+        if (rw & WRITE)
+                __allocate_data_blocks(inode, offset, count);
+
         err = blockdev_direct_IO(rw, iocb, inode, iter, offset, get_data_block);
         if (err < 0 && (rw & WRITE))
                 f2fs_write_failed(mapping, offset + count);
@@ -1101,24 +1168,33 @@ static ssize_t f2fs_direct_IO(int rw, struct kiocb *iocb,
         return err;
 }
 
-static void f2fs_invalidate_data_page(struct page *page, unsigned int offset,
-                                      unsigned int length)
+void f2fs_invalidate_page(struct page *page, unsigned int offset,
+                                                        unsigned int length)
 {
         struct inode *inode = page->mapping->host;
+        struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 
-        if (offset % PAGE_CACHE_SIZE || length != PAGE_CACHE_SIZE)
+        if (inode->i_ino >= F2FS_ROOT_INO(sbi) &&
+                (offset % PAGE_CACHE_SIZE || length != PAGE_CACHE_SIZE))
                 return;
 
-        if (f2fs_is_atomic_file(inode) || f2fs_is_volatile_file(inode))
-                invalidate_inmem_page(inode, page);
-
-        if (PageDirty(page))
-                inode_dec_dirty_pages(inode);
+        if (PageDirty(page)) {
+                if (inode->i_ino == F2FS_META_INO(sbi))
+                        dec_page_count(sbi, F2FS_DIRTY_META);
+                else if (inode->i_ino == F2FS_NODE_INO(sbi))
+                        dec_page_count(sbi, F2FS_DIRTY_NODES);
+                else
+                        inode_dec_dirty_pages(inode);
+        }
         ClearPagePrivate(page);
 }
 
-static int f2fs_release_data_page(struct page *page, gfp_t wait)
+int f2fs_release_page(struct page *page, gfp_t wait)
 {
+        /* If this is dirty page, keep PagePrivate */
+        if (PageDirty(page))
+                return 0;
+
         ClearPagePrivate(page);
         return 1;
 }
@@ -1132,7 +1208,7 @@ static int f2fs_set_data_page_dirty(struct page *page)
 
         SetPageUptodate(page);
 
-        if (f2fs_is_atomic_file(inode) || f2fs_is_volatile_file(inode)) {
+        if (f2fs_is_atomic_file(inode)) {
                 register_inmem_page(inode, page);
                 return 1;
         }
@@ -1168,8 +1244,8 @@ const struct address_space_operations f2fs_dblock_aops = {
         .write_begin    = f2fs_write_begin,
         .write_end      = f2fs_write_end,
         .set_page_dirty = f2fs_set_data_page_dirty,
-        .invalidatepage = f2fs_invalidate_data_page,
-        .releasepage    = f2fs_release_data_page,
+        .invalidatepage = f2fs_invalidate_page,
+        .releasepage    = f2fs_release_page,
         .direct_IO      = f2fs_direct_IO,
         .bmap           = f2fs_bmap,
 };