1 files changed, 29 insertions, 1 deletions

diff --git a/mm/workingset.c b/mm/workingset.c
index f874b2c663e3..9a26a60368d2 100644
--- a/mm/workingset.c
+++ b/mm/workingset.c
@@ -156,8 +156,19 @@
                          ZONES_SHIFT + NODES_SHIFT)
 #define EVICTION_MASK   (~0UL >> EVICTION_SHIFT)
 
+/*
+ * Eviction timestamps need to be able to cover the full range of
+ * actionable refaults. However, bits are tight in the radix tree
+ * entry, and after storing the identifier for the lruvec there might
+ * not be enough left to represent every single actionable refault. In
+ * that case, we have to sacrifice granularity for distance, and group
+ * evictions into coarser buckets by shaving off lower timestamp bits.
+ */
+static unsigned int bucket_order __read_mostly;
+
 static void *pack_shadow(unsigned long eviction, struct zone *zone)
 {
+        eviction >>= bucket_order;
         eviction = (eviction << NODES_SHIFT) | zone_to_nid(zone);
         eviction = (eviction << ZONES_SHIFT) | zone_idx(zone);
         eviction = (eviction << RADIX_TREE_EXCEPTIONAL_SHIFT);
@@ -178,7 +189,7 @@ static void unpack_shadow(void *shadow, struct zone **zonep,
         entry >>= NODES_SHIFT;
 
         *zonep = NODE_DATA(nid)->node_zones + zid;
-        *evictionp = entry;
+        *evictionp = entry << bucket_order;
 }
 
 /**
@@ -400,8 +411,25 @@ static struct lock_class_key shadow_nodes_key;
 
 static int __init workingset_init(void)
 {
+        unsigned int timestamp_bits;
+        unsigned int max_order;
         int ret;
 
+        BUILD_BUG_ON(BITS_PER_LONG < EVICTION_SHIFT);
+        /*
+         * Calculate the eviction bucket size to cover the longest
+         * actionable refault distance, which is currently half of
+         * memory (totalram_pages/2). However, memory hotplug may add
+         * some more pages at runtime, so keep working with up to
+         * double the initial memory by using totalram_pages as-is.
+         */
+        timestamp_bits = BITS_PER_LONG - EVICTION_SHIFT;
+        max_order = fls_long(totalram_pages - 1);
+        if (max_order > timestamp_bits)
+                bucket_order = max_order - timestamp_bits;
+        printk("workingset: timestamp_bits=%d max_order=%d bucket_order=%u\n",
+               timestamp_bits, max_order, bucket_order);
+
         ret = list_lru_init_key(&workingset_shadow_nodes, &shadow_nodes_key);
         if (ret)
                 goto err;