ext4: add DAX functionality

This is a port of the DAX functionality found in the current version of
ext2.

[matthew.r.wilcox@intel.com: heavily tweaked]
[akpm@linux-foundation.org: remap_pages went away]
Signed-off-by: Ross Zwisler <ross.zwisler@linux.intel.com>
Reviewed-by: Andreas Dilger <andreas.dilger@intel.com>
Signed-off-by: Matthew Wilcox <matthew.r.wilcox@intel.com>
Cc: Boaz Harrosh <boaz@plexistor.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Jens Axboe <axboe@kernel.dk>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Cc: Randy Dunlap <rdunlap@infradead.org>
Cc: Theodore Ts'o <tytso@mit.edu>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 62 insertions, 27 deletions

diff --git a/fs/ext4/inode.c b/fs/ext4/inode.c
index 5653fa42930b..28555f191b62 100644
--- a/fs/ext4/inode.c
+++ b/fs/ext4/inode.c
@@ -657,6 +657,18 @@ has_zeroout:
         return retval;
 }
 
+static void ext4_end_io_unwritten(struct buffer_head *bh, int uptodate)
+{
+        struct inode *inode = bh->b_assoc_map->host;
+        /* XXX: breaks on 32-bit > 16GB. Is that even supported? */
+        loff_t offset = (loff_t)(uintptr_t)bh->b_private << inode->i_blkbits;
+        int err;
+        if (!uptodate)
+                return;
+        WARN_ON(!buffer_unwritten(bh));
+        err = ext4_convert_unwritten_extents(NULL, inode, offset, bh->b_size);
+}
+
 /* Maximum number of blocks we map for direct IO at once. */
 #define DIO_MAX_BLOCKS 4096
 
@@ -694,6 +706,11 @@ static int _ext4_get_block(struct inode *inode, sector_t iblock,
 
                 map_bh(bh, inode->i_sb, map.m_pblk);
                 bh->b_state = (bh->b_state & ~EXT4_MAP_FLAGS) | map.m_flags;
+                if (IS_DAX(inode) && buffer_unwritten(bh) && !io_end) {
+                        bh->b_assoc_map = inode->i_mapping;
+                        bh->b_private = (void *)(unsigned long)iblock;
+                        bh->b_end_io = ext4_end_io_unwritten;
+                }
                 if (io_end && io_end->flag & EXT4_IO_END_UNWRITTEN)
                         set_buffer_defer_completion(bh);
                 bh->b_size = inode->i_sb->s_blocksize * map.m_len;
@@ -3010,13 +3027,14 @@ static ssize_t ext4_ext_direct_IO(int rw, struct kiocb *iocb,
                 get_block_func = ext4_get_block_write;
                 dio_flags = DIO_LOCKING;
         }
-        ret = __blockdev_direct_IO(rw, iocb, inode,
-                                   inode->i_sb->s_bdev, iter,
-                                   offset,
-                                   get_block_func,
-                                   ext4_end_io_dio,
-                                   NULL,
-                                   dio_flags);
+        if (IS_DAX(inode))
+                ret = dax_do_io(rw, iocb, inode, iter, offset, get_block_func,
+                                ext4_end_io_dio, dio_flags);
+        else
+                ret = __blockdev_direct_IO(rw, iocb, inode,
+                                           inode->i_sb->s_bdev, iter, offset,
+                                           get_block_func,
+                                           ext4_end_io_dio, NULL, dio_flags);
 
         /*
          * Put our reference to io_end. This can free the io_end structure e.g.
@@ -3180,19 +3198,12 @@ void ext4_set_aops(struct inode *inode)
                 inode->i_mapping->a_ops = &ext4_aops;
 }
 
-/*
- * ext4_block_zero_page_range() zeros out a mapping of length 'length'
- * starting from file offset 'from'.  The range to be zero'd must
- * be contained with in one block.  If the specified range exceeds
- * the end of the block it will be shortened to end of the block
- * that cooresponds to 'from'
- */
-static int ext4_block_zero_page_range(handle_t *handle,
+static int __ext4_block_zero_page_range(handle_t *handle,
                 struct address_space *mapping, loff_t from, loff_t length)
 {
         ext4_fsblk_t index = from >> PAGE_CACHE_SHIFT;
         unsigned offset = from & (PAGE_CACHE_SIZE-1);
-        unsigned blocksize, max, pos;
+        unsigned blocksize, pos;
         ext4_lblk_t iblock;
         struct inode *inode = mapping->host;
         struct buffer_head *bh;
@@ -3205,14 +3216,6 @@ static int ext4_block_zero_page_range(handle_t *handle,
                 return -ENOMEM;
 
         blocksize = inode->i_sb->s_blocksize;
-        max = blocksize - (offset & (blocksize - 1));
-
-        /*
-         * correct length if it does not fall between
-         * 'from' and the end of the block
-         */
-        if (length > max || length < 0)
-                length = max;
 
         iblock = index << (PAGE_CACHE_SHIFT - inode->i_sb->s_blocksize_bits);
 
@@ -3278,6 +3281,33 @@ unlock:
 }
 
 /*
+ * ext4_block_zero_page_range() zeros out a mapping of length 'length'
+ * starting from file offset 'from'.  The range to be zero'd must
+ * be contained with in one block.  If the specified range exceeds
+ * the end of the block it will be shortened to end of the block
+ * that cooresponds to 'from'
+ */
+static int ext4_block_zero_page_range(handle_t *handle,
+                struct address_space *mapping, loff_t from, loff_t length)
+{
+        struct inode *inode = mapping->host;
+        unsigned offset = from & (PAGE_CACHE_SIZE-1);
+        unsigned blocksize = inode->i_sb->s_blocksize;
+        unsigned max = blocksize - (offset & (blocksize - 1));
+
+        /*
+         * correct length if it does not fall between
+         * 'from' and the end of the block
+         */
+        if (length > max || length < 0)
+                length = max;
+
+        if (IS_DAX(inode))
+                return dax_zero_page_range(inode, from, length, ext4_get_block);
+        return __ext4_block_zero_page_range(handle, mapping, from, length);
+}
+
+/*
  * ext4_block_truncate_page() zeroes out a mapping from file offset `from'
  * up to the end of the block which corresponds to `from'.
  * This required during truncate. We need to physically zero the tail end
@@ -3798,8 +3828,10 @@ void ext4_set_inode_flags(struct inode *inode)
                 new_fl |= S_NOATIME;
         if (flags & EXT4_DIRSYNC_FL)
                 new_fl |= S_DIRSYNC;
+        if (test_opt(inode->i_sb, DAX))
+                new_fl |= S_DAX;
         inode_set_flags(inode, new_fl,
-                        S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC);
+                        S_SYNC|S_APPEND|S_IMMUTABLE|S_NOATIME|S_DIRSYNC|S_DAX);
 }
 
 /* Propagate flags from i_flags to EXT4_I(inode)->i_flags */
@@ -4052,7 +4084,10 @@ struct inode *ext4_iget(struct super_block *sb, unsigned long ino)
 
         if (S_ISREG(inode->i_mode)) {
                 inode->i_op = &ext4_file_inode_operations;
-                inode->i_fop = &ext4_file_operations;
+                if (test_opt(inode->i_sb, DAX))
+                        inode->i_fop = &ext4_dax_file_operations;
+                else
+                        inode->i_fop = &ext4_file_operations;
                 ext4_set_aops(inode);
         } else if (S_ISDIR(inode->i_mode)) {
                 inode->i_op = &ext4_dir_inode_operations;
@@ -4534,7 +4569,7 @@ int ext4_setattr(struct dentry *dentry, struct iattr *attr)
                  * Truncate pagecache after we've waited for commit
                  * in data=journal mode to make pages freeable.
                  */
-                        truncate_pagecache(inode, inode->i_size);
+                truncate_pagecache(inode, inode->i_size);
         }
         /*
          * We want to call ext4_truncate() even if attr->ia_size ==