Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs

Pull vfs pile 4 from Al Viro:
 "list_lru pile, mostly"

This came out of Andrew's pile, Al ended up doing the merge work so that
Andrew didn't have to.

Additionally, a few fixes.

* 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/viro/vfs: (42 commits)
  super: fix for destroy lrus
  list_lru: dynamically adjust node arrays
  shrinker: Kill old ->shrink API.
  shrinker: convert remaining shrinkers to count/scan API
  staging/lustre/libcfs: cleanup linux-mem.h
  staging/lustre/ptlrpc: convert to new shrinker API
  staging/lustre/obdclass: convert lu_object shrinker to count/scan API
  staging/lustre/ldlm: convert to shrinkers to count/scan API
  hugepage: convert huge zero page shrinker to new shrinker API
  i915: bail out earlier when shrinker cannot acquire mutex
  drivers: convert shrinkers to new count/scan API
  fs: convert fs shrinkers to new scan/count API
  xfs: fix dquot isolation hang
  xfs-convert-dquot-cache-lru-to-list_lru-fix
  xfs: convert dquot cache lru to list_lru
  xfs: rework buffer dispose list tracking
  xfs-convert-buftarg-lru-to-generic-code-fix
  xfs: convert buftarg LRU to generic code
  fs: convert inode and dentry shrinking to be node aware
  vmscan: per-node deferred work
  ...

5 files changed, 293 insertions, 108 deletions

diff --git a/mm/Makefile b/mm/Makefile
index f00803386a67..305d10acd081 100644
--- a/mm/Makefile
+++ b/mm/Makefile
@@ -17,7 +17,7 @@ obj-y			:= filemap.o mempool.o oom_kill.o fadvise.o \
                            util.o mmzone.o vmstat.o backing-dev.o \
                            mm_init.o mmu_context.o percpu.o slab_common.o \
                            compaction.o balloon_compaction.o \
-                           interval_tree.o $(mmu-y)
+                           interval_tree.o list_lru.o $(mmu-y)
 
 obj-y += init-mm.o
 
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index 963e14c0486f..d66010e0049d 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -211,24 +211,29 @@ static void put_huge_zero_page(void)
         BUG_ON(atomic_dec_and_test(&huge_zero_refcount));
 }
 
-static int shrink_huge_zero_page(struct shrinker *shrink,
-                struct shrink_control *sc)
+static unsigned long shrink_huge_zero_page_count(struct shrinker *shrink,
+                                        struct shrink_control *sc)
 {
-        if (!sc->nr_to_scan)
-                /* we can free zero page only if last reference remains */
-                return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0;
+        /* we can free zero page only if last reference remains */
+        return atomic_read(&huge_zero_refcount) == 1 ? HPAGE_PMD_NR : 0;
+}
 
+static unsigned long shrink_huge_zero_page_scan(struct shrinker *shrink,
+                                       struct shrink_control *sc)
+{
         if (atomic_cmpxchg(&huge_zero_refcount, 1, 0) == 1) {
                 struct page *zero_page = xchg(&huge_zero_page, NULL);
                 BUG_ON(zero_page == NULL);
                 __free_page(zero_page);
+                return HPAGE_PMD_NR;
         }
 
         return 0;
 }
 
 static struct shrinker huge_zero_page_shrinker = {
-        .shrink = shrink_huge_zero_page,
+        .count_objects = shrink_huge_zero_page_count,
+        .scan_objects = shrink_huge_zero_page_scan,
         .seeks = DEFAULT_SEEKS,
 };
 
diff --git a/mm/list_lru.c b/mm/list_lru.c
new file mode 100644
index 000000000000..72467914b856
--- /dev/null
+++ b/mm/list_lru.c
@@ -0,0 +1,139 @@
+/*
+ * Copyright (c) 2013 Red Hat, Inc. and Parallels Inc. All rights reserved.
+ * Authors: David Chinner and Glauber Costa
+ *
+ * Generic LRU infrastructure
+ */
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/mm.h>
+#include <linux/list_lru.h>
+#include <linux/slab.h>
+
+bool list_lru_add(struct list_lru *lru, struct list_head *item)
+{
+        int nid = page_to_nid(virt_to_page(item));
+        struct list_lru_node *nlru = &lru->node[nid];
+
+        spin_lock(&nlru->lock);
+        WARN_ON_ONCE(nlru->nr_items < 0);
+        if (list_empty(item)) {
+                list_add_tail(item, &nlru->list);
+                if (nlru->nr_items++ == 0)
+                        node_set(nid, lru->active_nodes);
+                spin_unlock(&nlru->lock);
+                return true;
+        }
+        spin_unlock(&nlru->lock);
+        return false;
+}
+EXPORT_SYMBOL_GPL(list_lru_add);
+
+bool list_lru_del(struct list_lru *lru, struct list_head *item)
+{
+        int nid = page_to_nid(virt_to_page(item));
+        struct list_lru_node *nlru = &lru->node[nid];
+
+        spin_lock(&nlru->lock);
+        if (!list_empty(item)) {
+                list_del_init(item);
+                if (--nlru->nr_items == 0)
+                        node_clear(nid, lru->active_nodes);
+                WARN_ON_ONCE(nlru->nr_items < 0);
+                spin_unlock(&nlru->lock);
+                return true;
+        }
+        spin_unlock(&nlru->lock);
+        return false;
+}
+EXPORT_SYMBOL_GPL(list_lru_del);
+
+unsigned long
+list_lru_count_node(struct list_lru *lru, int nid)
+{
+        unsigned long count = 0;
+        struct list_lru_node *nlru = &lru->node[nid];
+
+        spin_lock(&nlru->lock);
+        WARN_ON_ONCE(nlru->nr_items < 0);
+        count += nlru->nr_items;
+        spin_unlock(&nlru->lock);
+
+        return count;
+}
+EXPORT_SYMBOL_GPL(list_lru_count_node);
+
+unsigned long
+list_lru_walk_node(struct list_lru *lru, int nid, list_lru_walk_cb isolate,
+                   void *cb_arg, unsigned long *nr_to_walk)
+{
+
+        struct list_lru_node    *nlru = &lru->node[nid];
+        struct list_head *item, *n;
+        unsigned long isolated = 0;
+
+        spin_lock(&nlru->lock);
+restart:
+        list_for_each_safe(item, n, &nlru->list) {
+                enum lru_status ret;
+
+                /*
+                 * decrement nr_to_walk first so that we don't livelock if we
+                 * get stuck on large numbesr of LRU_RETRY items
+                 */
+                if (--(*nr_to_walk) == 0)
+                        break;
+
+                ret = isolate(item, &nlru->lock, cb_arg);
+                switch (ret) {
+                case LRU_REMOVED:
+                        if (--nlru->nr_items == 0)
+                                node_clear(nid, lru->active_nodes);
+                        WARN_ON_ONCE(nlru->nr_items < 0);
+                        isolated++;
+                        break;
+                case LRU_ROTATE:
+                        list_move_tail(item, &nlru->list);
+                        break;
+                case LRU_SKIP:
+                        break;
+                case LRU_RETRY:
+                        /*
+                         * The lru lock has been dropped, our list traversal is
+                         * now invalid and so we have to restart from scratch.
+                         */
+                        goto restart;
+                default:
+                        BUG();
+                }
+        }
+
+        spin_unlock(&nlru->lock);
+        return isolated;
+}
+EXPORT_SYMBOL_GPL(list_lru_walk_node);
+
+int list_lru_init(struct list_lru *lru)
+{
+        int i;
+        size_t size = sizeof(*lru->node) * nr_node_ids;
+
+        lru->node = kzalloc(size, GFP_KERNEL);
+        if (!lru->node)
+                return -ENOMEM;
+
+        nodes_clear(lru->active_nodes);
+        for (i = 0; i < nr_node_ids; i++) {
+                spin_lock_init(&lru->node[i].lock);
+                INIT_LIST_HEAD(&lru->node[i].list);
+                lru->node[i].nr_items = 0;
+        }
+        return 0;
+}
+EXPORT_SYMBOL_GPL(list_lru_init);
+
+void list_lru_destroy(struct list_lru *lru)
+{
+        kfree(lru->node);
+}
+EXPORT_SYMBOL_GPL(list_lru_destroy);
diff --git a/mm/memory-failure.c b/mm/memory-failure.c
index d472e14c6808..947ed5413279 100644
--- a/mm/memory-failure.c
+++ b/mm/memory-failure.c
@@ -248,10 +248,12 @@ void shake_page(struct page *p, int access)
          */
         if (access) {
                 int nr;
+                int nid = page_to_nid(p);
                 do {
                         struct shrink_control shrink = {
                                 .gfp_mask = GFP_KERNEL,
                         };
+                        node_set(nid, shrink.nodes_to_scan);
 
                         nr = shrink_slab(&shrink, 1000, 1000);
                         if (page_count(p) == 1)
diff --git a/mm/vmscan.c b/mm/vmscan.c
index fe715daeb8bc..beb35778c69f 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -174,14 +174,31 @@ static unsigned long get_lru_size(struct lruvec *lruvec, enum lru_list lru)
 }
 
 /*
- * Add a shrinker callback to be called from the vm
+ * Add a shrinker callback to be called from the vm.
  */
-void register_shrinker(struct shrinker *shrinker)
+int register_shrinker(struct shrinker *shrinker)
 {
-        atomic_long_set(&shrinker->nr_in_batch, 0);
+        size_t size = sizeof(*shrinker->nr_deferred);
+
+        /*
+         * If we only have one possible node in the system anyway, save
+         * ourselves the trouble and disable NUMA aware behavior. This way we
+         * will save memory and some small loop time later.
+         */
+        if (nr_node_ids == 1)
+                shrinker->flags &= ~SHRINKER_NUMA_AWARE;
+
+        if (shrinker->flags & SHRINKER_NUMA_AWARE)
+                size *= nr_node_ids;
+
+        shrinker->nr_deferred = kzalloc(size, GFP_KERNEL);
+        if (!shrinker->nr_deferred)
+                return -ENOMEM;
+
         down_write(&shrinker_rwsem);
         list_add_tail(&shrinker->list, &shrinker_list);
         up_write(&shrinker_rwsem);
+        return 0;
 }
 EXPORT_SYMBOL(register_shrinker);
 
@@ -196,15 +213,102 @@ void unregister_shrinker(struct shrinker *shrinker)
 }
 EXPORT_SYMBOL(unregister_shrinker);
 
-static inline int do_shrinker_shrink(struct shrinker *shrinker,
-                                     struct shrink_control *sc,
-                                     unsigned long nr_to_scan)
-{
-        sc->nr_to_scan = nr_to_scan;
-        return (*shrinker->shrink)(shrinker, sc);
+#define SHRINK_BATCH 128
+
+static unsigned long
+shrink_slab_node(struct shrink_control *shrinkctl, struct shrinker *shrinker,
+                 unsigned long nr_pages_scanned, unsigned long lru_pages)
+{
+        unsigned long freed = 0;
+        unsigned long long delta;
+        long total_scan;
+        long max_pass;
+        long nr;
+        long new_nr;
+        int nid = shrinkctl->nid;
+        long batch_size = shrinker->batch ? shrinker->batch
+                                          : SHRINK_BATCH;
+
+        max_pass = shrinker->count_objects(shrinker, shrinkctl);
+        if (max_pass == 0)
+                return 0;
+
+        /*
+         * copy the current shrinker scan count into a local variable
+         * and zero it so that other concurrent shrinker invocations
+         * don't also do this scanning work.
+         */
+        nr = atomic_long_xchg(&shrinker->nr_deferred[nid], 0);
+
+        total_scan = nr;
+        delta = (4 * nr_pages_scanned) / shrinker->seeks;
+        delta *= max_pass;
+        do_div(delta, lru_pages + 1);
+        total_scan += delta;
+        if (total_scan < 0) {
+                printk(KERN_ERR
+                "shrink_slab: %pF negative objects to delete nr=%ld\n",
+                       shrinker->scan_objects, total_scan);
+                total_scan = max_pass;
+        }
+
+        /*
+         * We need to avoid excessive windup on filesystem shrinkers
+         * due to large numbers of GFP_NOFS allocations causing the
+         * shrinkers to return -1 all the time. This results in a large
+         * nr being built up so when a shrink that can do some work
+         * comes along it empties the entire cache due to nr >>>
+         * max_pass.  This is bad for sustaining a working set in
+         * memory.
+         *
+         * Hence only allow the shrinker to scan the entire cache when
+         * a large delta change is calculated directly.
+         */
+        if (delta < max_pass / 4)
+                total_scan = min(total_scan, max_pass / 2);
+
+        /*
+         * Avoid risking looping forever due to too large nr value:
+         * never try to free more than twice the estimate number of
+         * freeable entries.
+         */
+        if (total_scan > max_pass * 2)
+                total_scan = max_pass * 2;
+
+        trace_mm_shrink_slab_start(shrinker, shrinkctl, nr,
+                                nr_pages_scanned, lru_pages,
+                                max_pass, delta, total_scan);
+
+        while (total_scan >= batch_size) {
+                unsigned long ret;
+
+                shrinkctl->nr_to_scan = batch_size;
+                ret = shrinker->scan_objects(shrinker, shrinkctl);
+                if (ret == SHRINK_STOP)
+                        break;
+                freed += ret;
+
+                count_vm_events(SLABS_SCANNED, batch_size);
+                total_scan -= batch_size;
+
+                cond_resched();
+        }
+
+        /*
+         * move the unused scan count back into the shrinker in a
+         * manner that handles concurrent updates. If we exhausted the
+         * scan, there is no need to do an update.
+         */
+        if (total_scan > 0)
+                new_nr = atomic_long_add_return(total_scan,
+                                                &shrinker->nr_deferred[nid]);
+        else
+                new_nr = atomic_long_read(&shrinker->nr_deferred[nid]);
+
+        trace_mm_shrink_slab_end(shrinker, freed, nr, new_nr);
+        return freed;
 }
 
-#define SHRINK_BATCH 128
 /*
  * Call the shrink functions to age shrinkable caches
  *
@@ -224,115 +328,45 @@ static inline int do_shrinker_shrink(struct shrinker *shrinker,
  *
  * Returns the number of slab objects which we shrunk.
  */
-unsigned long shrink_slab(struct shrink_control *shrink,
+unsigned long shrink_slab(struct shrink_control *shrinkctl,
                           unsigned long nr_pages_scanned,
                           unsigned long lru_pages)
 {
         struct shrinker *shrinker;
-        unsigned long ret = 0;
+        unsigned long freed = 0;
 
         if (nr_pages_scanned == 0)
                 nr_pages_scanned = SWAP_CLUSTER_MAX;
 
         if (!down_read_trylock(&shrinker_rwsem)) {
-                /* Assume we'll be able to shrink next time */
-                ret = 1;
+                /*
+                 * If we would return 0, our callers would understand that we
+                 * have nothing else to shrink and give up trying. By returning
+                 * 1 we keep it going and assume we'll be able to shrink next
+                 * time.
+                 */
+                freed = 1;
                 goto out;
         }
 
         list_for_each_entry(shrinker, &shrinker_list, list) {
-                unsigned long long delta;
-                long total_scan;
-                long max_pass;
-                int shrink_ret = 0;
-                long nr;
-                long new_nr;
-                long batch_size = shrinker->batch ? shrinker->batch
-                                                  : SHRINK_BATCH;
-
-                max_pass = do_shrinker_shrink(shrinker, shrink, 0);
-                if (max_pass <= 0)
-                        continue;
-
-                /*
-                 * copy the current shrinker scan count into a local variable
-                 * and zero it so that other concurrent shrinker invocations
-                 * don't also do this scanning work.
-                 */
-                nr = atomic_long_xchg(&shrinker->nr_in_batch, 0);
-
-                total_scan = nr;
-                delta = (4 * nr_pages_scanned) / shrinker->seeks;
-                delta *= max_pass;
-                do_div(delta, lru_pages + 1);
-                total_scan += delta;
-                if (total_scan < 0) {
-                        printk(KERN_ERR "shrink_slab: %pF negative objects to "
-                               "delete nr=%ld\n",
-                               shrinker->shrink, total_scan);
-                        total_scan = max_pass;
-                }
-
-                /*
-                 * We need to avoid excessive windup on filesystem shrinkers
-                 * due to large numbers of GFP_NOFS allocations causing the
-                 * shrinkers to return -1 all the time. This results in a large
-                 * nr being built up so when a shrink that can do some work
-                 * comes along it empties the entire cache due to nr >>>
-                 * max_pass.  This is bad for sustaining a working set in
-                 * memory.
-                 *
-                 * Hence only allow the shrinker to scan the entire cache when
-                 * a large delta change is calculated directly.
-                 */
-                if (delta < max_pass / 4)
-                        total_scan = min(total_scan, max_pass / 2);
-
-                /*
-                 * Avoid risking looping forever due to too large nr value:
-                 * never try to free more than twice the estimate number of
-                 * freeable entries.
-                 */
-                if (total_scan > max_pass * 2)
-                        total_scan = max_pass * 2;
-
-                trace_mm_shrink_slab_start(shrinker, shrink, nr,
-                                        nr_pages_scanned, lru_pages,
-                                        max_pass, delta, total_scan);
-
-                while (total_scan >= batch_size) {
-                        int nr_before;
+                for_each_node_mask(shrinkctl->nid, shrinkctl->nodes_to_scan) {
+                        if (!node_online(shrinkctl->nid))
+                                continue;
 
-                        nr_before = do_shrinker_shrink(shrinker, shrink, 0);
-                        shrink_ret = do_shrinker_shrink(shrinker, shrink,
-                                                        batch_size);
-                        if (shrink_ret == -1)
+                        if (!(shrinker->flags & SHRINKER_NUMA_AWARE) &&
+                            (shrinkctl->nid != 0))
                                 break;
-                        if (shrink_ret < nr_before)
-                                ret += nr_before - shrink_ret;
-                        count_vm_events(SLABS_SCANNED, batch_size);
-                        total_scan -= batch_size;
 
-                        cond_resched();
-                }
+                        freed += shrink_slab_node(shrinkctl, shrinker,
+                                 nr_pages_scanned, lru_pages);
 
-                /*
-                 * move the unused scan count back into the shrinker in a
-                 * manner that handles concurrent updates. If we exhausted the
-                 * scan, there is no need to do an update.
-                 */
-                if (total_scan > 0)
-                        new_nr = atomic_long_add_return(total_scan,
-                                        &shrinker->nr_in_batch);
-                else
-                        new_nr = atomic_long_read(&shrinker->nr_in_batch);
-
-                trace_mm_shrink_slab_end(shrinker, shrink_ret, nr, new_nr);
+                }
         }
         up_read(&shrinker_rwsem);
 out:
         cond_resched();
-        return ret;
+        return freed;
 }
 
 static inline int is_page_cache_freeable(struct page *page)
@@ -2368,12 +2402,16 @@ static unsigned long do_try_to_free_pages(struct zonelist *zonelist,
                  */
                 if (global_reclaim(sc)) {
                         unsigned long lru_pages = 0;
+
+                        nodes_clear(shrink->nodes_to_scan);
                         for_each_zone_zonelist(zone, z, zonelist,
                                         gfp_zone(sc->gfp_mask)) {
                                 if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
                                         continue;
 
                                 lru_pages += zone_reclaimable_pages(zone);
+                                node_set(zone_to_nid(zone),
+                                         shrink->nodes_to_scan);
                         }
 
                         shrink_slab(shrink, sc->nr_scanned, lru_pages);
@@ -2829,6 +2867,8 @@ static bool kswapd_shrink_zone(struct zone *zone,
                 return true;
 
         shrink_zone(zone, sc);
+        nodes_clear(shrink.nodes_to_scan);
+        node_set(zone_to_nid(zone), shrink.nodes_to_scan);
 
         reclaim_state->reclaimed_slab = 0;
         shrink_slab(&shrink, sc->nr_scanned, lru_pages);
@@ -3520,10 +3560,9 @@ static int __zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
                  * number of slab pages and shake the slab until it is reduced
                  * by the same nr_pages that we used for reclaiming unmapped
                  * pages.
-                 *
-                 * Note that shrink_slab will free memory on all zones and may
-                 * take a long time.
                  */
+                nodes_clear(shrink.nodes_to_scan);
+                node_set(zone_to_nid(zone), shrink.nodes_to_scan);
                 for (;;) {
                         unsigned long lru_pages = zone_reclaimable_pages(zone);