1 files changed, 88 insertions, 24 deletions

diff --git a/mm/memory_hotplug.c b/mm/memory_hotplug.c
index ca1dd3aa5eee..0eb1a1df649d 100644
--- a/mm/memory_hotplug.c
+++ b/mm/memory_hotplug.c
@@ -30,6 +30,7 @@
 #include <linux/mm_inline.h>
 #include <linux/firmware-map.h>
 #include <linux/stop_machine.h>
+#include <linux/hugetlb.h>
 
 #include <asm/tlbflush.h>
 
@@ -194,7 +195,7 @@ void register_page_bootmem_info_node(struct pglist_data *pgdat)
 
         zone = &pgdat->node_zones[0];
         for (; zone < pgdat->node_zones + MAX_NR_ZONES - 1; zone++) {
-                if (zone->wait_table) {
+                if (zone_is_initialized(zone)) {
                         nr_pages = zone->wait_table_hash_nr_entries
                                 * sizeof(wait_queue_head_t);
                         nr_pages = PAGE_ALIGN(nr_pages) >> PAGE_SHIFT;
@@ -229,8 +230,8 @@ static void grow_zone_span(struct zone *zone, unsigned long start_pfn,
 
         zone_span_writelock(zone);
 
-        old_zone_end_pfn = zone->zone_start_pfn + zone->spanned_pages;
-        if (!zone->spanned_pages || start_pfn < zone->zone_start_pfn)
+        old_zone_end_pfn = zone_end_pfn(zone);
+        if (zone_is_empty(zone) || start_pfn < zone->zone_start_pfn)
                 zone->zone_start_pfn = start_pfn;
 
         zone->spanned_pages = max(old_zone_end_pfn, end_pfn) -
@@ -305,7 +306,7 @@ static int __meminit move_pfn_range_left(struct zone *z1, struct zone *z2,
                 goto out_fail;
 
         /* use start_pfn for z1's start_pfn if z1 is empty */
-        if (z1->spanned_pages)
+        if (!zone_is_empty(z1))
                 z1_start_pfn = z1->zone_start_pfn;
         else
                 z1_start_pfn = start_pfn;
@@ -347,7 +348,7 @@ static int __meminit move_pfn_range_right(struct zone *z1, struct zone *z2,
                 goto out_fail;
 
         /* use end_pfn for z2's end_pfn if z2 is empty */
-        if (z2->spanned_pages)
+        if (!zone_is_empty(z2))
                 z2_end_pfn = zone_end_pfn(z2);
         else
                 z2_end_pfn = end_pfn;
@@ -514,8 +515,9 @@ static int find_biggest_section_pfn(int nid, struct zone *zone,
 static void shrink_zone_span(struct zone *zone, unsigned long start_pfn,
                              unsigned long end_pfn)
 {
-        unsigned long zone_start_pfn =  zone->zone_start_pfn;
-        unsigned long zone_end_pfn = zone->zone_start_pfn + zone->spanned_pages;
+        unsigned long zone_start_pfn = zone->zone_start_pfn;
+        unsigned long z = zone_end_pfn(zone); /* zone_end_pfn namespace clash */
+        unsigned long zone_end_pfn = z;
         unsigned long pfn;
         struct mem_section *ms;
         int nid = zone_to_nid(zone);
@@ -1069,6 +1071,23 @@ out:
         return ret;
 }
 
+static int check_hotplug_memory_range(u64 start, u64 size)
+{
+        u64 start_pfn = start >> PAGE_SHIFT;
+        u64 nr_pages = size >> PAGE_SHIFT;
+
+        /* Memory range must be aligned with section */
+        if ((start_pfn & ~PAGE_SECTION_MASK) ||
+            (nr_pages % PAGES_PER_SECTION) || (!nr_pages)) {
+                pr_err("Section-unaligned hotplug range: start 0x%llx, size 0x%llx\n",
+                                (unsigned long long)start,
+                                (unsigned long long)size);
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
 /* we are OK calling __meminit stuff here - we have CONFIG_MEMORY_HOTPLUG */
 int __ref add_memory(int nid, u64 start, u64 size)
 {
@@ -1078,6 +1097,10 @@ int __ref add_memory(int nid, u64 start, u64 size)
         struct resource *res;
         int ret;
 
+        ret = check_hotplug_memory_range(start, size);
+        if (ret)
+                return ret;
+
         lock_memory_hotplug();
 
         res = register_memory_resource(start, size);
@@ -1208,10 +1231,12 @@ static int test_pages_in_a_zone(unsigned long start_pfn, unsigned long end_pfn)
 }
 
 /*
- * Scanning pfn is much easier than scanning lru list.
- * Scan pfn from start to end and Find LRU page.
+ * Scan pfn range [start,end) to find movable/migratable pages (LRU pages
+ * and hugepages). We scan pfn because it's much easier than scanning over
+ * linked list. This function returns the pfn of the first found movable
+ * page if it's found, otherwise 0.
  */
-static unsigned long scan_lru_pages(unsigned long start, unsigned long end)
+static unsigned long scan_movable_pages(unsigned long start, unsigned long end)
 {
         unsigned long pfn;
         struct page *page;
@@ -1220,6 +1245,13 @@ static unsigned long scan_lru_pages(unsigned long start, unsigned long end)
                         page = pfn_to_page(pfn);
                         if (PageLRU(page))
                                 return pfn;
+                        if (PageHuge(page)) {
+                                if (is_hugepage_active(page))
+                                        return pfn;
+                                else
+                                        pfn = round_up(pfn + 1,
+                                                1 << compound_order(page)) - 1;
+                        }
                 }
         }
         return 0;
@@ -1240,6 +1272,19 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
                 if (!pfn_valid(pfn))
                         continue;
                 page = pfn_to_page(pfn);
+
+                if (PageHuge(page)) {
+                        struct page *head = compound_head(page);
+                        pfn = page_to_pfn(head) + (1<<compound_order(head)) - 1;
+                        if (compound_order(head) > PFN_SECTION_SHIFT) {
+                                ret = -EBUSY;
+                                break;
+                        }
+                        if (isolate_huge_page(page, &source))
+                                move_pages -= 1 << compound_order(head);
+                        continue;
+                }
+
                 if (!get_page_unless_zero(page))
                         continue;
                 /*
@@ -1272,7 +1317,7 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
         }
         if (!list_empty(&source)) {
                 if (not_managed) {
-                        putback_lru_pages(&source);
+                        putback_movable_pages(&source);
                         goto out;
                 }
 
@@ -1283,7 +1328,7 @@ do_migrate_range(unsigned long start_pfn, unsigned long end_pfn)
                 ret = migrate_pages(&source, alloc_migrate_target, 0,
                                         MIGRATE_SYNC, MR_MEMORY_HOTPLUG);
                 if (ret)
-                        putback_lru_pages(&source);
+                        putback_movable_pages(&source);
         }
 out:
         return ret;
@@ -1472,7 +1517,6 @@ static int __ref __offline_pages(unsigned long start_pfn,
         struct zone *zone;
         struct memory_notify arg;
 
-        BUG_ON(start_pfn >= end_pfn);
         /* at least, alignment against pageblock is necessary */
         if (!IS_ALIGNED(start_pfn, pageblock_nr_pages))
                 return -EINVAL;
@@ -1527,8 +1571,8 @@ repeat:
                 drain_all_pages();
         }
 
-        pfn = scan_lru_pages(start_pfn, end_pfn);
-        if (pfn) { /* We have page on LRU */
+        pfn = scan_movable_pages(start_pfn, end_pfn);
+        if (pfn) { /* We have movable pages */
                 ret = do_migrate_range(pfn, end_pfn);
                 if (!ret) {
                         drain = 1;
@@ -1547,6 +1591,11 @@ repeat:
         yield();
         /* drain pcp pages, this is synchronous. */
         drain_all_pages();
+        /*
+         * dissolve free hugepages in the memory block before doing offlining
+         * actually in order to make hugetlbfs's object counting consistent.
+         */
+        dissolve_free_huge_pages(start_pfn, end_pfn);
         /* check again */
         offlined_pages = check_pages_isolated(start_pfn, end_pfn);
         if (offlined_pages < 0) {
@@ -1674,9 +1723,8 @@ static int is_memblock_offlined_cb(struct memory_block *mem, void *arg)
         return ret;
 }
 
-static int check_cpu_on_node(void *data)
+static int check_cpu_on_node(pg_data_t *pgdat)
 {
-        struct pglist_data *pgdat = data;
         int cpu;
 
         for_each_present_cpu(cpu) {
@@ -1691,10 +1739,9 @@ static int check_cpu_on_node(void *data)
         return 0;
 }
 
-static void unmap_cpu_on_node(void *data)
+static void unmap_cpu_on_node(pg_data_t *pgdat)
 {
 #ifdef CONFIG_ACPI_NUMA
-        struct pglist_data *pgdat = data;
         int cpu;
 
         for_each_possible_cpu(cpu)
@@ -1703,10 +1750,11 @@ static void unmap_cpu_on_node(void *data)
 #endif
 }
 
-static int check_and_unmap_cpu_on_node(void *data)
+static int check_and_unmap_cpu_on_node(pg_data_t *pgdat)
 {
-        int ret = check_cpu_on_node(data);
+        int ret;
 
+        ret = check_cpu_on_node(pgdat);
         if (ret)
                 return ret;
 
@@ -1715,11 +1763,18 @@ static int check_and_unmap_cpu_on_node(void *data)
          * the cpu_to_node() now.
          */
 
-        unmap_cpu_on_node(data);
+        unmap_cpu_on_node(pgdat);
         return 0;
 }
 
-/* offline the node if all memory sections of this node are removed */
+/**
+ * try_offline_node
+ *
+ * Offline a node if all memory sections and cpus of the node are removed.
+ *
+ * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
+ * and online/offline operations before this call.
+ */
 void try_offline_node(int nid)
 {
         pg_data_t *pgdat = NODE_DATA(nid);
@@ -1745,7 +1800,7 @@ void try_offline_node(int nid)
                 return;
         }
 
-        if (stop_machine(check_and_unmap_cpu_on_node, pgdat, NULL))
+        if (check_and_unmap_cpu_on_node(pgdat))
                 return;
 
         /*
@@ -1782,10 +1837,19 @@ void try_offline_node(int nid)
 }
 EXPORT_SYMBOL(try_offline_node);
 
+/**
+ * remove_memory
+ *
+ * NOTE: The caller must call lock_device_hotplug() to serialize hotplug
+ * and online/offline operations before this call, as required by
+ * try_offline_node().
+ */
 void __ref remove_memory(int nid, u64 start, u64 size)
 {
         int ret;
 
+        BUG_ON(check_hotplug_memory_range(start, size));
+
         lock_memory_hotplug();
 
         /*