1 files changed, 227 insertions, 268 deletions

diff --git a/mm/shmem.c b/mm/shmem.c
index 51b3d6ccddab..0f246c44a574 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -78,11 +78,10 @@
 
 /* Flag allocation requirements to shmem_getpage and shmem_swp_alloc */
 enum sgp_type {
-        SGP_QUICK,      /* don't try more than file page cache lookup */
         SGP_READ,       /* don't exceed i_size, don't allocate page */
         SGP_CACHE,      /* don't exceed i_size, may allocate page */
+        SGP_DIRTY,      /* like SGP_CACHE, but set new page dirty */
         SGP_WRITE,      /* may exceed i_size, may allocate page */
-        SGP_FAULT,      /* same as SGP_CACHE, return with page locked */
 };
 
 static int shmem_getpage(struct inode *inode, unsigned long idx,
@@ -194,7 +193,7 @@ static struct backing_dev_info shmem_backing_dev_info  __read_mostly = {
 };
 
 static LIST_HEAD(shmem_swaplist);
-static DEFINE_SPINLOCK(shmem_swaplist_lock);
+static DEFINE_MUTEX(shmem_swaplist_mutex);
 
 static void shmem_free_blocks(struct inode *inode, long pages)
 {
@@ -207,6 +206,31 @@ static void shmem_free_blocks(struct inode *inode, long pages)
         }
 }
 
+static int shmem_reserve_inode(struct super_block *sb)
+{
+        struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
+        if (sbinfo->max_inodes) {
+                spin_lock(&sbinfo->stat_lock);
+                if (!sbinfo->free_inodes) {
+                        spin_unlock(&sbinfo->stat_lock);
+                        return -ENOSPC;
+                }
+                sbinfo->free_inodes--;
+                spin_unlock(&sbinfo->stat_lock);
+        }
+        return 0;
+}
+
+static void shmem_free_inode(struct super_block *sb)
+{
+        struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
+        if (sbinfo->max_inodes) {
+                spin_lock(&sbinfo->stat_lock);
+                sbinfo->free_inodes++;
+                spin_unlock(&sbinfo->stat_lock);
+        }
+}
+
 /*
  * shmem_recalc_inode - recalculate the size of an inode
  *
@@ -731,6 +755,8 @@ static int shmem_notify_change(struct dentry *dentry, struct iattr *attr)
                                 (void) shmem_getpage(inode,
                                         attr->ia_size>>PAGE_CACHE_SHIFT,
                                                 &page, SGP_READ, NULL);
+                                if (page)
+                                        unlock_page(page);
                         }
                         /*
                          * Reset SHMEM_PAGEIN flag so that shmem_truncate can
@@ -762,7 +788,6 @@ static int shmem_notify_change(struct dentry *dentry, struct iattr *attr)
 
 static void shmem_delete_inode(struct inode *inode)
 {
-        struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
         struct shmem_inode_info *info = SHMEM_I(inode);
 
         if (inode->i_op->truncate == shmem_truncate) {
@@ -771,17 +796,13 @@ static void shmem_delete_inode(struct inode *inode)
                 inode->i_size = 0;
                 shmem_truncate(inode);
                 if (!list_empty(&info->swaplist)) {
-                        spin_lock(&shmem_swaplist_lock);
+                        mutex_lock(&shmem_swaplist_mutex);
                         list_del_init(&info->swaplist);
-                        spin_unlock(&shmem_swaplist_lock);
+                        mutex_unlock(&shmem_swaplist_mutex);
                 }
         }
         BUG_ON(inode->i_blocks);
-        if (sbinfo->max_inodes) {
-                spin_lock(&sbinfo->stat_lock);
-                sbinfo->free_inodes++;
-                spin_unlock(&sbinfo->stat_lock);
-        }
+        shmem_free_inode(inode->i_sb);
         clear_inode(inode);
 }
 
@@ -807,19 +828,22 @@ static int shmem_unuse_inode(struct shmem_inode_info *info, swp_entry_t entry, s
         struct page *subdir;
         swp_entry_t *ptr;
         int offset;
+        int error;
 
         idx = 0;
         ptr = info->i_direct;
         spin_lock(&info->lock);
+        if (!info->swapped) {
+                list_del_init(&info->swaplist);
+                goto lost2;
+        }
         limit = info->next_index;
         size = limit;
         if (size > SHMEM_NR_DIRECT)
                 size = SHMEM_NR_DIRECT;
         offset = shmem_find_swp(entry, ptr, ptr+size);
-        if (offset >= 0) {
-                shmem_swp_balance_unmap();
+        if (offset >= 0)
                 goto found;
-        }
         if (!info->i_indirect)
                 goto lost2;
 
@@ -829,6 +853,14 @@ static int shmem_unuse_inode(struct shmem_inode_info *info, swp_entry_t entry, s
         for (idx = SHMEM_NR_DIRECT; idx < limit; idx += ENTRIES_PER_PAGE, dir++) {
                 if (unlikely(idx == stage)) {
                         shmem_dir_unmap(dir-1);
+                        if (cond_resched_lock(&info->lock)) {
+                                /* check it has not been truncated */
+                                if (limit > info->next_index) {
+                                        limit = info->next_index;
+                                        if (idx >= limit)
+                                                goto lost2;
+                                }
+                        }
                         dir = shmem_dir_map(info->i_indirect) +
                             ENTRIES_PER_PAGE/2 + idx/ENTRIES_PER_PAGEPAGE;
                         while (!*dir) {
@@ -849,11 +881,11 @@ static int shmem_unuse_inode(struct shmem_inode_info *info, swp_entry_t entry, s
                         if (size > ENTRIES_PER_PAGE)
                                 size = ENTRIES_PER_PAGE;
                         offset = shmem_find_swp(entry, ptr, ptr+size);
+                        shmem_swp_unmap(ptr);
                         if (offset >= 0) {
                                 shmem_dir_unmap(dir);
                                 goto found;
                         }
-                        shmem_swp_unmap(ptr);
                 }
         }
 lost1:
@@ -863,19 +895,63 @@ lost2:
         return 0;
 found:
         idx += offset;
-        inode = &info->vfs_inode;
-        if (move_from_swap_cache(page, idx, inode->i_mapping) == 0) {
-                info->flags |= SHMEM_PAGEIN;
-                shmem_swp_set(info, ptr + offset, 0);
-        }
-        shmem_swp_unmap(ptr);
+        inode = igrab(&info->vfs_inode);
         spin_unlock(&info->lock);
+
         /*
-         * Decrement swap count even when the entry is left behind:
-         * try_to_unuse will skip over mms, then reincrement count.
+         * Move _head_ to start search for next from here.
+         * But be careful: shmem_delete_inode checks list_empty without taking
+         * mutex, and there's an instant in list_move_tail when info->swaplist
+         * would appear empty, if it were the only one on shmem_swaplist.  We
+         * could avoid doing it if inode NULL; or use this minor optimization.
          */
-        swap_free(entry);
-        return 1;
+        if (shmem_swaplist.next != &info->swaplist)
+                list_move_tail(&shmem_swaplist, &info->swaplist);
+        mutex_unlock(&shmem_swaplist_mutex);
+
+        error = 1;
+        if (!inode)
+                goto out;
+        error = radix_tree_preload(GFP_KERNEL);
+        if (error)
+                goto out;
+        error = 1;
+
+        spin_lock(&info->lock);
+        ptr = shmem_swp_entry(info, idx, NULL);
+        if (ptr && ptr->val == entry.val)
+                error = add_to_page_cache(page, inode->i_mapping,
+                                                idx, GFP_NOWAIT);
+        if (error == -EEXIST) {
+                struct page *filepage = find_get_page(inode->i_mapping, idx);
+                error = 1;
+                if (filepage) {
+                        /*
+                         * There might be a more uptodate page coming down
+                         * from a stacked writepage: forget our swappage if so.
+                         */
+                        if (PageUptodate(filepage))
+                                error = 0;
+                        page_cache_release(filepage);
+                }
+        }
+        if (!error) {
+                delete_from_swap_cache(page);
+                set_page_dirty(page);
+                info->flags |= SHMEM_PAGEIN;
+                shmem_swp_set(info, ptr, 0);
+                swap_free(entry);
+                error = 1;      /* not an error, but entry was found */
+        }
+        if (ptr)
+                shmem_swp_unmap(ptr);
+        spin_unlock(&info->lock);
+        radix_tree_preload_end();
+out:
+        unlock_page(page);
+        page_cache_release(page);
+        iput(inode);            /* allows for NULL */
+        return error;
 }
 
 /*
@@ -887,20 +963,16 @@ int shmem_unuse(swp_entry_t entry, struct page *page)
         struct shmem_inode_info *info;
         int found = 0;
 
-        spin_lock(&shmem_swaplist_lock);
+        mutex_lock(&shmem_swaplist_mutex);
         list_for_each_safe(p, next, &shmem_swaplist) {
                 info = list_entry(p, struct shmem_inode_info, swaplist);
-                if (!info->swapped)
-                        list_del_init(&info->swaplist);
-                else if (shmem_unuse_inode(info, entry, page)) {
-                        /* move head to start search for next from here */
-                        list_move_tail(&shmem_swaplist, &info->swaplist);
-                        found = 1;
-                        break;
-                }
+                found = shmem_unuse_inode(info, entry, page);
+                cond_resched();
+                if (found)
+                        goto out;
         }
-        spin_unlock(&shmem_swaplist_lock);
-        return found;
+        mutex_unlock(&shmem_swaplist_mutex);
+out:    return found;   /* 0 or 1 or -ENOMEM */
 }
 
 /*
@@ -915,54 +987,65 @@ static int shmem_writepage(struct page *page, struct writeback_control *wbc)
         struct inode *inode;
 
         BUG_ON(!PageLocked(page));
-        /*
-         * shmem_backing_dev_info's capabilities prevent regular writeback or
-         * sync from ever calling shmem_writepage; but a stacking filesystem
-         * may use the ->writepage of its underlying filesystem, in which case
-         * we want to do nothing when that underlying filesystem is tmpfs
-         * (writing out to swap is useful as a response to memory pressure, but
-         * of no use to stabilize the data) - just redirty the page, unlock it
-         * and claim success in this case.  AOP_WRITEPAGE_ACTIVATE, and the
-         * page_mapped check below, must be avoided unless we're in reclaim.
-         */
-        if (!wbc->for_reclaim) {
-                set_page_dirty(page);
-                unlock_page(page);
-                return 0;
-        }
-        BUG_ON(page_mapped(page));
-
         mapping = page->mapping;
         index = page->index;
         inode = mapping->host;
         info = SHMEM_I(inode);
         if (info->flags & VM_LOCKED)
                 goto redirty;
-        swap = get_swap_page();
-        if (!swap.val)
+        if (!total_swap_pages)
                 goto redirty;
 
+        /*
+         * shmem_backing_dev_info's capabilities prevent regular writeback or
+         * sync from ever calling shmem_writepage; but a stacking filesystem
+         * may use the ->writepage of its underlying filesystem, in which case
+         * tmpfs should write out to swap only in response to memory pressure,
+         * and not for pdflush or sync.  However, in those cases, we do still
+         * want to check if there's a redundant swappage to be discarded.
+         */
+        if (wbc->for_reclaim)
+                swap = get_swap_page();
+        else
+                swap.val = 0;
+
         spin_lock(&info->lock);
-        shmem_recalc_inode(inode);
         if (index >= info->next_index) {
                 BUG_ON(!(info->flags & SHMEM_TRUNCATE));
                 goto unlock;
         }
         entry = shmem_swp_entry(info, index, NULL);
-        BUG_ON(!entry);
-        BUG_ON(entry->val);
+        if (entry->val) {
+                /*
+                 * The more uptodate page coming down from a stacked
+                 * writepage should replace our old swappage.
+                 */
+                free_swap_and_cache(*entry);
+                shmem_swp_set(info, entry, 0);
+        }
+        shmem_recalc_inode(inode);
 
-        if (move_to_swap_cache(page, swap) == 0) {
+        if (swap.val && add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
+                remove_from_page_cache(page);
                 shmem_swp_set(info, entry, swap.val);
                 shmem_swp_unmap(entry);
+                if (list_empty(&info->swaplist))
+                        inode = igrab(inode);
+                else
+                        inode = NULL;
                 spin_unlock(&info->lock);
-                if (list_empty(&info->swaplist)) {
-                        spin_lock(&shmem_swaplist_lock);
+                swap_duplicate(swap);
+                BUG_ON(page_mapped(page));
+                page_cache_release(page);       /* pagecache ref */
+                set_page_dirty(page);
+                unlock_page(page);
+                if (inode) {
+                        mutex_lock(&shmem_swaplist_mutex);
                         /* move instead of add in case we're racing */
                         list_move_tail(&info->swaplist, &shmem_swaplist);
-                        spin_unlock(&shmem_swaplist_lock);
+                        mutex_unlock(&shmem_swaplist_mutex);
+                        iput(inode);
                 }
-                unlock_page(page);
                 return 0;
         }
 
@@ -972,7 +1055,10 @@ unlock:
         swap_free(swap);
 redirty:
         set_page_dirty(page);
-        return AOP_WRITEPAGE_ACTIVATE;  /* Return with the page locked */
+        if (wbc->for_reclaim)
+                return AOP_WRITEPAGE_ACTIVATE;  /* Return with page locked */
+        unlock_page(page);
+        return 0;
 }
 
 #ifdef CONFIG_NUMA
@@ -1025,53 +1111,33 @@ out:
         return err;
 }
 
-static struct page *shmem_swapin_async(struct shared_policy *p,
-                                       swp_entry_t entry, unsigned long idx)
+static struct page *shmem_swapin(swp_entry_t entry, gfp_t gfp,
+                        struct shmem_inode_info *info, unsigned long idx)
 {
-        struct page *page;
         struct vm_area_struct pvma;
+        struct page *page;
 
         /* Create a pseudo vma that just contains the policy */
-        memset(&pvma, 0, sizeof(struct vm_area_struct));
-        pvma.vm_end = PAGE_SIZE;
+        pvma.vm_start = 0;
         pvma.vm_pgoff = idx;
-        pvma.vm_policy = mpol_shared_policy_lookup(p, idx);
-        page = read_swap_cache_async(entry, &pvma, 0);
+        pvma.vm_ops = NULL;
+        pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, idx);
+        page = swapin_readahead(entry, gfp, &pvma, 0);
         mpol_free(pvma.vm_policy);
         return page;
 }
 
-static struct page *shmem_swapin(struct shmem_inode_info *info,
-                                 swp_entry_t entry, unsigned long idx)
-{
-        struct shared_policy *p = &info->policy;
-        int i, num;
-        struct page *page;
-        unsigned long offset;
-
-        num = valid_swaphandles(entry, &offset);
-        for (i = 0; i < num; offset++, i++) {
-                page = shmem_swapin_async(p,
-                                swp_entry(swp_type(entry), offset), idx);
-                if (!page)
-                        break;
-                page_cache_release(page);
-        }
-        lru_add_drain();        /* Push any new pages onto the LRU now */
-        return shmem_swapin_async(p, entry, idx);
-}
-
-static struct page *
-shmem_alloc_page(gfp_t gfp, struct shmem_inode_info *info,
-                 unsigned long idx)
+static struct page *shmem_alloc_page(gfp_t gfp,
+                        struct shmem_inode_info *info, unsigned long idx)
 {
         struct vm_area_struct pvma;
         struct page *page;
 
-        memset(&pvma, 0, sizeof(struct vm_area_struct));
-        pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, idx);
+        /* Create a pseudo vma that just contains the policy */
+        pvma.vm_start = 0;
         pvma.vm_pgoff = idx;
-        pvma.vm_end = PAGE_SIZE;
+        pvma.vm_ops = NULL;
+        pvma.vm_policy = mpol_shared_policy_lookup(&info->policy, idx);
         page = alloc_page_vma(gfp, &pvma, 0);
         mpol_free(pvma.vm_policy);
         return page;
@@ -1083,15 +1149,14 @@ static inline int shmem_parse_mpol(char *value, int *policy,
         return 1;
 }
 
-static inline struct page *
-shmem_swapin(struct shmem_inode_info *info,swp_entry_t entry,unsigned long idx)
+static inline struct page *shmem_swapin(swp_entry_t entry, gfp_t gfp,
+                        struct shmem_inode_info *info, unsigned long idx)
 {
-        swapin_readahead(entry, 0, NULL);
-        return read_swap_cache_async(entry, NULL, 0);
+        return swapin_readahead(entry, gfp, NULL, 0);
 }
 
-static inline struct page *
-shmem_alloc_page(gfp_t gfp,struct shmem_inode_info *info, unsigned long idx)
+static inline struct page *shmem_alloc_page(gfp_t gfp,
+                        struct shmem_inode_info *info, unsigned long idx)
 {
         return alloc_page(gfp);
 }
@@ -1114,6 +1179,7 @@ static int shmem_getpage(struct inode *inode, unsigned long idx,
         struct page *swappage;
         swp_entry_t *entry;
         swp_entry_t swap;
+        gfp_t gfp;
         int error;
 
         if (idx >= SHMEM_MAX_INDEX)
@@ -1126,7 +1192,7 @@ static int shmem_getpage(struct inode *inode, unsigned long idx,
          * Normally, filepage is NULL on entry, and either found
          * uptodate immediately, or allocated and zeroed, or read
          * in under swappage, which is then assigned to filepage.
-         * But shmem_readpage and shmem_write_begin pass in a locked
+         * But shmem_readpage (required for splice) passes in a locked
          * filepage, which may be found not uptodate by other callers
          * too, and may need to be copied from the swappage read in.
          */
@@ -1136,8 +1202,17 @@ repeat:
         if (filepage && PageUptodate(filepage))
                 goto done;
         error = 0;
-        if (sgp == SGP_QUICK)
-                goto failed;
+        gfp = mapping_gfp_mask(mapping);
+        if (!filepage) {
+                /*
+                 * Try to preload while we can wait, to not make a habit of
+                 * draining atomic reserves; but don't latch on to this cpu.
+                 */
+                error = radix_tree_preload(gfp & ~__GFP_HIGHMEM);
+                if (error)
+                        goto failed;
+                radix_tree_preload_end();
+        }
 
         spin_lock(&info->lock);
         shmem_recalc_inode(inode);
@@ -1160,7 +1235,7 @@ repeat:
                                 *type |= VM_FAULT_MAJOR;
                         }
                         spin_unlock(&info->lock);
-                        swappage = shmem_swapin(info, swap, idx);
+                        swappage = shmem_swapin(swap, gfp, info, idx);
                         if (!swappage) {
                                 spin_lock(&info->lock);
                                 entry = shmem_swp_alloc(info, idx, sgp);
@@ -1218,23 +1293,21 @@ repeat:
                         SetPageUptodate(filepage);
                         set_page_dirty(filepage);
                         swap_free(swap);
-                } else if (!(error = move_from_swap_cache(
-                                swappage, idx, mapping))) {
+                } else if (!(error = add_to_page_cache(
+                                swappage, mapping, idx, GFP_NOWAIT))) {
                         info->flags |= SHMEM_PAGEIN;
                         shmem_swp_set(info, entry, 0);
                         shmem_swp_unmap(entry);
+                        delete_from_swap_cache(swappage);
                         spin_unlock(&info->lock);
                         filepage = swappage;
+                        set_page_dirty(filepage);
                         swap_free(swap);
                 } else {
                         shmem_swp_unmap(entry);
                         spin_unlock(&info->lock);
                         unlock_page(swappage);
                         page_cache_release(swappage);
-                        if (error == -ENOMEM) {
-                                /* let kswapd refresh zone for GFP_ATOMICs */
-                                congestion_wait(WRITE, HZ/50);
-                        }
                         goto repeat;
                 }
         } else if (sgp == SGP_READ && !filepage) {
@@ -1272,9 +1345,7 @@ repeat:
 
                 if (!filepage) {
                         spin_unlock(&info->lock);
-                        filepage = shmem_alloc_page(mapping_gfp_mask(mapping),
-                                                    info,
-                                                    idx);
+                        filepage = shmem_alloc_page(gfp, info, idx);
                         if (!filepage) {
                                 shmem_unacct_blocks(info->flags, 1);
                                 shmem_free_blocks(inode, 1);
@@ -1291,7 +1362,7 @@ repeat:
                                 shmem_swp_unmap(entry);
                         }
                         if (error || swap.val || 0 != add_to_page_cache_lru(
-                                        filepage, mapping, idx, GFP_ATOMIC)) {
+                                        filepage, mapping, idx, GFP_NOWAIT)) {
                                 spin_unlock(&info->lock);
                                 page_cache_release(filepage);
                                 shmem_unacct_blocks(info->flags, 1);
@@ -1309,14 +1380,11 @@ repeat:
                 clear_highpage(filepage);
                 flush_dcache_page(filepage);
                 SetPageUptodate(filepage);
+                if (sgp == SGP_DIRTY)
+                        set_page_dirty(filepage);
         }
 done:
-        if (*pagep != filepage) {
-                *pagep = filepage;
-                if (sgp != SGP_FAULT)
-                        unlock_page(filepage);
-
-        }
+        *pagep = filepage;
         return 0;
 
 failed:
@@ -1336,7 +1404,7 @@ static int shmem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
         if (((loff_t)vmf->pgoff << PAGE_CACHE_SHIFT) >= i_size_read(inode))
                 return VM_FAULT_SIGBUS;
 
-        error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_FAULT, &ret);
+        error = shmem_getpage(inode, vmf->pgoff, &vmf->page, SGP_CACHE, &ret);
         if (error)
                 return ((error == -ENOMEM) ? VM_FAULT_OOM : VM_FAULT_SIGBUS);
 
@@ -1399,15 +1467,8 @@ shmem_get_inode(struct super_block *sb, int mode, dev_t dev)
         struct shmem_inode_info *info;
         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
 
-        if (sbinfo->max_inodes) {
-                spin_lock(&sbinfo->stat_lock);
-                if (!sbinfo->free_inodes) {
-                        spin_unlock(&sbinfo->stat_lock);
-                        return NULL;
-                }
-                sbinfo->free_inodes--;
-                spin_unlock(&sbinfo->stat_lock);
-        }
+        if (shmem_reserve_inode(sb))
+                return NULL;
 
         inode = new_inode(sb);
         if (inode) {
@@ -1451,11 +1512,8 @@ shmem_get_inode(struct super_block *sb, int mode, dev_t dev)
                                                 NULL);
                         break;
                 }
-        } else if (sbinfo->max_inodes) {
-                spin_lock(&sbinfo->stat_lock);
-                sbinfo->free_inodes++;
-                spin_unlock(&sbinfo->stat_lock);
-        }
+        } else
+                shmem_free_inode(sb);
         return inode;
 }
 
@@ -1494,123 +1552,30 @@ shmem_write_end(struct file *file, struct address_space *mapping,
 {
         struct inode *inode = mapping->host;
 
+        if (pos + copied > inode->i_size)
+                i_size_write(inode, pos + copied);
+
+        unlock_page(page);
         set_page_dirty(page);
         page_cache_release(page);
 
-        if (pos+copied > inode->i_size)
-                i_size_write(inode, pos+copied);
-
         return copied;
 }
 
-static ssize_t
-shmem_file_write(struct file *file, const char __user *buf, size_t count, loff_t *ppos)
-{
-        struct inode    *inode = file->f_path.dentry->d_inode;
-        loff_t          pos;
-        unsigned long   written;
-        ssize_t         err;
-
-        if ((ssize_t) count < 0)
-                return -EINVAL;
-
-        if (!access_ok(VERIFY_READ, buf, count))
-                return -EFAULT;
-
-        mutex_lock(&inode->i_mutex);
-
-        pos = *ppos;
-        written = 0;
-
-        err = generic_write_checks(file, &pos, &count, 0);
-        if (err || !count)
-                goto out;
-
-        err = remove_suid(file->f_path.dentry);
-        if (err)
-                goto out;
-
-        inode->i_ctime = inode->i_mtime = CURRENT_TIME;
-
-        do {
-                struct page *page = NULL;
-                unsigned long bytes, index, offset;
-                char *kaddr;
-                int left;
-
-                offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */
-                index = pos >> PAGE_CACHE_SHIFT;
-                bytes = PAGE_CACHE_SIZE - offset;
-                if (bytes > count)
-                        bytes = count;
-
-                /*
-                 * We don't hold page lock across copy from user -
-                 * what would it guard against? - so no deadlock here.
-                 * But it still may be a good idea to prefault below.
-                 */
-
-                err = shmem_getpage(inode, index, &page, SGP_WRITE, NULL);
-                if (err)
-                        break;
-
-                left = bytes;
-                if (PageHighMem(page)) {
-                        volatile unsigned char dummy;
-                        __get_user(dummy, buf);
-                        __get_user(dummy, buf + bytes - 1);
-
-                        kaddr = kmap_atomic(page, KM_USER0);
-                        left = __copy_from_user_inatomic(kaddr + offset,
-                                                        buf, bytes);
-                        kunmap_atomic(kaddr, KM_USER0);
-                }
-                if (left) {
-                        kaddr = kmap(page);
-                        left = __copy_from_user(kaddr + offset, buf, bytes);
-                        kunmap(page);
-                }
-
-                written += bytes;
-                count -= bytes;
-                pos += bytes;
-                buf += bytes;
-                if (pos > inode->i_size)
-                        i_size_write(inode, pos);
-
-                flush_dcache_page(page);
-                set_page_dirty(page);
-                mark_page_accessed(page);
-                page_cache_release(page);
-
-                if (left) {
-                        pos -= left;
-                        written -= left;
-                        err = -EFAULT;
-                        break;
-                }
-
-                /*
-                 * Our dirty pages are not counted in nr_dirty,
-                 * and we do not attempt to balance dirty pages.
-                 */
-
-                cond_resched();
-        } while (count);
-
-        *ppos = pos;
-        if (written)
-                err = written;
-out:
-        mutex_unlock(&inode->i_mutex);
-        return err;
-}
-
 static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_t *desc, read_actor_t actor)
 {
         struct inode *inode = filp->f_path.dentry->d_inode;
         struct address_space *mapping = inode->i_mapping;
         unsigned long index, offset;
+        enum sgp_type sgp = SGP_READ;
+
+        /*
+         * Might this read be for a stacking filesystem?  Then when reading
+         * holes of a sparse file, we actually need to allocate those pages,
+         * and even mark them dirty, so it cannot exceed the max_blocks limit.
+         */
+        if (segment_eq(get_fs(), KERNEL_DS))
+                sgp = SGP_DIRTY;
 
         index = *ppos >> PAGE_CACHE_SHIFT;
         offset = *ppos & ~PAGE_CACHE_MASK;
@@ -1629,12 +1594,14 @@ static void do_shmem_file_read(struct file *filp, loff_t *ppos, read_descriptor_
                                 break;
                 }
 
-                desc->error = shmem_getpage(inode, index, &page, SGP_READ, NULL);
+                desc->error = shmem_getpage(inode, index, &page, sgp, NULL);
                 if (desc->error) {
                         if (desc->error == -EINVAL)
                                 desc->error = 0;
                         break;
                 }
+                if (page)
+                        unlock_page(page);
 
                 /*
                  * We must evaluate after, since reads (unlike writes)
@@ -1798,22 +1765,16 @@ static int shmem_create(struct inode *dir, struct dentry *dentry, int mode,
 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
 {
         struct inode *inode = old_dentry->d_inode;
-        struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
+        int ret;
 
         /*
          * No ordinary (disk based) filesystem counts links as inodes;
          * but each new link needs a new dentry, pinning lowmem, and
          * tmpfs dentries cannot be pruned until they are unlinked.
          */
-        if (sbinfo->max_inodes) {
-                spin_lock(&sbinfo->stat_lock);
-                if (!sbinfo->free_inodes) {
-                        spin_unlock(&sbinfo->stat_lock);
-                        return -ENOSPC;
-                }
-                sbinfo->free_inodes--;
-                spin_unlock(&sbinfo->stat_lock);
-        }
+        ret = shmem_reserve_inode(inode->i_sb);
+        if (ret)
+                goto out;
 
         dir->i_size += BOGO_DIRENT_SIZE;
         inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
@@ -1821,21 +1782,16 @@ static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentr
         atomic_inc(&inode->i_count);    /* New dentry reference */
         dget(dentry);           /* Extra pinning count for the created dentry */
         d_instantiate(dentry, inode);
-        return 0;
+out:
+        return ret;
 }
 
 static int shmem_unlink(struct inode *dir, struct dentry *dentry)
 {
         struct inode *inode = dentry->d_inode;
 
-        if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode)) {
-                struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
-                if (sbinfo->max_inodes) {
-                        spin_lock(&sbinfo->stat_lock);
-                        sbinfo->free_inodes++;
-                        spin_unlock(&sbinfo->stat_lock);
-                }
-        }
+        if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
+                shmem_free_inode(inode->i_sb);
 
         dir->i_size -= BOGO_DIRENT_SIZE;
         inode->i_ctime = dir->i_ctime = dir->i_mtime = CURRENT_TIME;
@@ -1924,6 +1880,7 @@ static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *s
                         iput(inode);
                         return error;
                 }
+                unlock_page(page);
                 inode->i_op = &shmem_symlink_inode_operations;
                 kaddr = kmap_atomic(page, KM_USER0);
                 memcpy(kaddr, symname, len);
@@ -1951,6 +1908,8 @@ static void *shmem_follow_link(struct dentry *dentry, struct nameidata *nd)
         struct page *page = NULL;
         int res = shmem_getpage(dentry->d_inode, 0, &page, SGP_READ, NULL);
         nd_set_link(nd, res ? ERR_PTR(res) : kmap(page));
+        if (page)
+                unlock_page(page);
         return page;
 }
 
@@ -1996,8 +1955,7 @@ static int shmem_xattr_security_get(struct inode *inode, const char *name,
 {
         if (strcmp(name, "") == 0)
                 return -EINVAL;
-        return security_inode_getsecurity(inode, name, buffer, size,
-                                          -EOPNOTSUPP);
+        return xattr_getsecurity(inode, name, buffer, size);
 }
 
 static int shmem_xattr_security_set(struct inode *inode, const char *name,
@@ -2138,7 +2096,7 @@ static int shmem_parse_options(char *options, int *mode, uid_t *uid,
                         }
                         if (*rest)
                                 goto bad_val;
-                        *blocks = size >> PAGE_CACHE_SHIFT;
+                        *blocks = DIV_ROUND_UP(size, PAGE_CACHE_SIZE);
                 } else if (!strcmp(this_char,"nr_blocks")) {
                         *blocks = memparse(value,&rest);
                         if (*rest)
@@ -2375,7 +2333,8 @@ static const struct file_operations shmem_file_operations = {
 #ifdef CONFIG_TMPFS
         .llseek         = generic_file_llseek,
         .read           = shmem_file_read,
-        .write          = shmem_file_write,
+        .write          = do_sync_write,
+        .aio_write      = generic_file_aio_write,
         .fsync          = simple_sync_file,
         .splice_read    = generic_file_splice_read,
         .splice_write   = generic_file_splice_write,