1 files changed, 83 insertions, 56 deletions

diff --git a/mm/shmem.c b/mm/shmem.c
index f65f84062db5..080b09a57a8f 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -28,6 +28,7 @@
 #include <linux/file.h>
 #include <linux/mm.h>
 #include <linux/module.h>
+#include <linux/percpu_counter.h>
 #include <linux/swap.h>
 
 static struct vfsmount *shm_mnt;
@@ -233,10 +234,10 @@ static void shmem_free_blocks(struct inode *inode, long pages)
 {
         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
         if (sbinfo->max_blocks) {
-                spin_lock(&sbinfo->stat_lock);
-                sbinfo->free_blocks += pages;
+                percpu_counter_add(&sbinfo->used_blocks, -pages);
+                spin_lock(&inode->i_lock);
                 inode->i_blocks -= pages*BLOCKS_PER_PAGE;
-                spin_unlock(&sbinfo->stat_lock);
+                spin_unlock(&inode->i_lock);
         }
 }
 
@@ -416,19 +417,17 @@ static swp_entry_t *shmem_swp_alloc(struct shmem_inode_info *info, unsigned long
                 if (sgp == SGP_READ)
                         return shmem_swp_map(ZERO_PAGE(0));
                 /*
-                 * Test free_blocks against 1 not 0, since we have 1 data
+                 * Test used_blocks against 1 less max_blocks, since we have 1 data
                  * page (and perhaps indirect index pages) yet to allocate:
                  * a waste to allocate index if we cannot allocate data.
                  */
                 if (sbinfo->max_blocks) {
-                        spin_lock(&sbinfo->stat_lock);
-                        if (sbinfo->free_blocks <= 1) {
-                                spin_unlock(&sbinfo->stat_lock);
+                        if (percpu_counter_compare(&sbinfo->used_blocks, (sbinfo->max_blocks - 1)) > 0)
                                 return ERR_PTR(-ENOSPC);
-                        }
-                        sbinfo->free_blocks--;
+                        percpu_counter_inc(&sbinfo->used_blocks);
+                        spin_lock(&inode->i_lock);
                         inode->i_blocks += BLOCKS_PER_PAGE;
-                        spin_unlock(&sbinfo->stat_lock);
+                        spin_unlock(&inode->i_lock);
                 }
 
                 spin_unlock(&info->lock);
@@ -767,6 +766,10 @@ static int shmem_notify_change(struct dentry *dentry, struct iattr *attr)
         loff_t newsize = attr->ia_size;
         int error;
 
+        error = inode_change_ok(inode, attr);
+        if (error)
+                return error;
+
         if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)
                                         && newsize != inode->i_size) {
                 struct page *page = NULL;
@@ -801,25 +804,22 @@ static int shmem_notify_change(struct dentry *dentry, struct iattr *attr)
                         }
                 }
 
-                error = simple_setsize(inode, newsize);
+                /* XXX(truncate): truncate_setsize should be called last */
+                truncate_setsize(inode, newsize);
                 if (page)
                         page_cache_release(page);
-                if (error)
-                        return error;
                 shmem_truncate_range(inode, newsize, (loff_t)-1);
         }
 
-        error = inode_change_ok(inode, attr);
-        if (!error)
-                generic_setattr(inode, attr);
+        setattr_copy(inode, attr);
 #ifdef CONFIG_TMPFS_POSIX_ACL
-        if (!error && (attr->ia_valid & ATTR_MODE))
+        if (attr->ia_valid & ATTR_MODE)
                 error = generic_acl_chmod(inode);
 #endif
         return error;
 }
 
-static void shmem_delete_inode(struct inode *inode)
+static void shmem_evict_inode(struct inode *inode)
 {
         struct shmem_inode_info *info = SHMEM_I(inode);
 
@@ -836,7 +836,7 @@ static void shmem_delete_inode(struct inode *inode)
         }
         BUG_ON(inode->i_blocks);
         shmem_free_inode(inode->i_sb);
-        clear_inode(inode);
+        end_writeback(inode);
 }
 
 static inline int shmem_find_swp(swp_entry_t entry, swp_entry_t *dir, swp_entry_t *edir)
@@ -933,7 +933,7 @@ found:
 
         /*
          * Move _head_ to start search for next from here.
-         * But be careful: shmem_delete_inode checks list_empty without taking
+         * But be careful: shmem_evict_inode checks list_empty without taking
          * mutex, and there's an instant in list_move_tail when info->swaplist
          * would appear empty, if it were the only one on shmem_swaplist.  We
          * could avoid doing it if inode NULL; or use this minor optimization.
@@ -1223,6 +1223,7 @@ static int shmem_getpage(struct inode *inode, unsigned long idx,
         struct shmem_sb_info *sbinfo;
         struct page *filepage = *pagep;
         struct page *swappage;
+        struct page *prealloc_page = NULL;
         swp_entry_t *entry;
         swp_entry_t swap;
         gfp_t gfp;
@@ -1247,7 +1248,6 @@ repeat:
                 filepage = find_lock_page(mapping, idx);
         if (filepage && PageUptodate(filepage))
                 goto done;
-        error = 0;
         gfp = mapping_gfp_mask(mapping);
         if (!filepage) {
                 /*
@@ -1258,7 +1258,19 @@ repeat:
                 if (error)
                         goto failed;
                 radix_tree_preload_end();
+                if (sgp != SGP_READ && !prealloc_page) {
+                        /* We don't care if this fails */
+                        prealloc_page = shmem_alloc_page(gfp, info, idx);
+                        if (prealloc_page) {
+                                if (mem_cgroup_cache_charge(prealloc_page,
+                                                current->mm, GFP_KERNEL)) {
+                                        page_cache_release(prealloc_page);
+                                        prealloc_page = NULL;
+                                }
+                        }
+                }
         }
+        error = 0;
 
         spin_lock(&info->lock);
         shmem_recalc_inode(inode);
@@ -1387,17 +1399,16 @@ repeat:
                 shmem_swp_unmap(entry);
                 sbinfo = SHMEM_SB(inode->i_sb);
                 if (sbinfo->max_blocks) {
-                        spin_lock(&sbinfo->stat_lock);
-                        if (sbinfo->free_blocks == 0 ||
+                        if ((percpu_counter_compare(&sbinfo->used_blocks, sbinfo->max_blocks) > 0) ||
                             shmem_acct_block(info->flags)) {
-                                spin_unlock(&sbinfo->stat_lock);
                                 spin_unlock(&info->lock);
                                 error = -ENOSPC;
                                 goto failed;
                         }
-                        sbinfo->free_blocks--;
+                        percpu_counter_inc(&sbinfo->used_blocks);
+                        spin_lock(&inode->i_lock);
                         inode->i_blocks += BLOCKS_PER_PAGE;
-                        spin_unlock(&sbinfo->stat_lock);
+                        spin_unlock(&inode->i_lock);
                 } else if (shmem_acct_block(info->flags)) {
                         spin_unlock(&info->lock);
                         error = -ENOSPC;
@@ -1407,28 +1418,38 @@ repeat:
                 if (!filepage) {
                         int ret;
 
-                        spin_unlock(&info->lock);
-                        filepage = shmem_alloc_page(gfp, info, idx);
-                        if (!filepage) {
-                                shmem_unacct_blocks(info->flags, 1);
-                                shmem_free_blocks(inode, 1);
-                                error = -ENOMEM;
-                                goto failed;
-                        }
-                        SetPageSwapBacked(filepage);
+                        if (!prealloc_page) {
+                                spin_unlock(&info->lock);
+                                filepage = shmem_alloc_page(gfp, info, idx);
+                                if (!filepage) {
+                                        shmem_unacct_blocks(info->flags, 1);
+                                        shmem_free_blocks(inode, 1);
+                                        error = -ENOMEM;
+                                        goto failed;
+                                }
+                                SetPageSwapBacked(filepage);
 
-                        /* Precharge page while we can wait, compensate after */
-                        error = mem_cgroup_cache_charge(filepage, current->mm,
-                                        GFP_KERNEL);
-                        if (error) {
-                                page_cache_release(filepage);
-                                shmem_unacct_blocks(info->flags, 1);
-                                shmem_free_blocks(inode, 1);
-                                filepage = NULL;
-                                goto failed;
+                                /*
+                                 * Precharge page while we can wait, compensate
+                                 * after
+                                 */
+                                error = mem_cgroup_cache_charge(filepage,
+                                        current->mm, GFP_KERNEL);
+                                if (error) {
+                                        page_cache_release(filepage);
+                                        shmem_unacct_blocks(info->flags, 1);
+                                        shmem_free_blocks(inode, 1);
+                                        filepage = NULL;
+                                        goto failed;
+                                }
+
+                                spin_lock(&info->lock);
+                        } else {
+                                filepage = prealloc_page;
+                                prealloc_page = NULL;
+                                SetPageSwapBacked(filepage);
                         }
 
-                        spin_lock(&info->lock);
                         entry = shmem_swp_alloc(info, idx, sgp);
                         if (IS_ERR(entry))
                                 error = PTR_ERR(entry);
@@ -1469,13 +1490,19 @@ repeat:
         }
 done:
         *pagep = filepage;
-        return 0;
+        error = 0;
+        goto out;
 
 failed:
         if (*pagep != filepage) {
                 unlock_page(filepage);
                 page_cache_release(filepage);
         }
+out:
+        if (prealloc_page) {
+                mem_cgroup_uncharge_cache_page(prealloc_page);
+                page_cache_release(prealloc_page);
+        }
         return error;
 }
 
@@ -1791,17 +1818,16 @@ static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
         buf->f_type = TMPFS_MAGIC;
         buf->f_bsize = PAGE_CACHE_SIZE;
         buf->f_namelen = NAME_MAX;
-        spin_lock(&sbinfo->stat_lock);
         if (sbinfo->max_blocks) {
                 buf->f_blocks = sbinfo->max_blocks;
-                buf->f_bavail = buf->f_bfree = sbinfo->free_blocks;
+                buf->f_bavail = buf->f_bfree =
+                                sbinfo->max_blocks - percpu_counter_sum(&sbinfo->used_blocks);
         }
         if (sbinfo->max_inodes) {
                 buf->f_files = sbinfo->max_inodes;
                 buf->f_ffree = sbinfo->free_inodes;
         }
         /* else leave those fields 0 like simple_statfs */
-        spin_unlock(&sbinfo->stat_lock);
         return 0;
 }
 
@@ -2242,7 +2268,6 @@ static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
 {
         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
         struct shmem_sb_info config = *sbinfo;
-        unsigned long blocks;
         unsigned long inodes;
         int error = -EINVAL;
 
@@ -2250,9 +2275,8 @@ static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
                 return error;
 
         spin_lock(&sbinfo->stat_lock);
-        blocks = sbinfo->max_blocks - sbinfo->free_blocks;
         inodes = sbinfo->max_inodes - sbinfo->free_inodes;
-        if (config.max_blocks < blocks)
+        if (percpu_counter_compare(&sbinfo->used_blocks, config.max_blocks) > 0)
                 goto out;
         if (config.max_inodes < inodes)
                 goto out;
@@ -2269,7 +2293,6 @@ static int shmem_remount_fs(struct super_block *sb, int *flags, char *data)
 
         error = 0;
         sbinfo->max_blocks  = config.max_blocks;
-        sbinfo->free_blocks = config.max_blocks - blocks;
         sbinfo->max_inodes  = config.max_inodes;
         sbinfo->free_inodes = config.max_inodes - inodes;
 
@@ -2302,7 +2325,10 @@ static int shmem_show_options(struct seq_file *seq, struct vfsmount *vfs)
 
 static void shmem_put_super(struct super_block *sb)
 {
-        kfree(sb->s_fs_info);
+        struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
+
+        percpu_counter_destroy(&sbinfo->used_blocks);
+        kfree(sbinfo);
         sb->s_fs_info = NULL;
 }
 
@@ -2344,7 +2370,8 @@ int shmem_fill_super(struct super_block *sb, void *data, int silent)
 #endif
 
         spin_lock_init(&sbinfo->stat_lock);
-        sbinfo->free_blocks = sbinfo->max_blocks;
+        if (percpu_counter_init(&sbinfo->used_blocks, 0))
+                goto failed;
         sbinfo->free_inodes = sbinfo->max_inodes;
 
         sb->s_maxbytes = SHMEM_MAX_BYTES;
@@ -2496,7 +2523,7 @@ static const struct super_operations shmem_ops = {
         .remount_fs     = shmem_remount_fs,
         .show_options   = shmem_show_options,
 #endif
-        .delete_inode   = shmem_delete_inode,
+        .evict_inode    = shmem_evict_inode,
         .drop_inode     = generic_delete_inode,
         .put_super      = shmem_put_super,
 };