mm: move buddy list manipulations into helpers

In preparation for runtime randomization of the zone lists, take all
(well, most of) the list_*() functions in the buddy allocator and put
them in helper functions.  Provide a common control point for injecting
additional behavior when freeing pages.

[dan.j.williams@intel.com: fix buddy list helpers]
  Link: http://lkml.kernel.org/r/155033679702.1773410.13041474192173212653.stgit@dwillia2-desk3.amr.corp.intel.com
[vbabka@suse.cz: remove del_page_from_free_area() migratetype parameter]
  Link: http://lkml.kernel.org/r/4672701b-6775-6efd-0797-b6242591419e@suse.cz
Link: http://lkml.kernel.org/r/154899812264.3165233.5219320056406926223.stgit@dwillia2-desk3.amr.corp.intel.com
Signed-off-by: Dan Williams <dan.j.williams@intel.com>
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Tested-by: Tetsuo Handa <penguin-kernel@i-love.sakura.ne.jp>
Acked-by: Michal Hocko <mhocko@suse.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Keith Busch <keith.busch@intel.com>
Cc: Robert Elliott <elliott@hpe.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 22 insertions, 43 deletions

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 548f8f5d3295..b674625762c4 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -756,12 +756,6 @@ static inline void set_page_order(struct page *page, unsigned int order)
         __SetPageBuddy(page);
 }
 
-static inline void rmv_page_order(struct page *page)
-{
-        __ClearPageBuddy(page);
-        set_page_private(page, 0);
-}
-
 /*
  * This function checks whether a page is free && is the buddy
  * we can coalesce a page and its buddy if
@@ -919,13 +913,10 @@ continue_merging:
                  * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
                  * merge with it and move up one order.
                  */
-                if (page_is_guard(buddy)) {
+                if (page_is_guard(buddy))
                         clear_page_guard(zone, buddy, order, migratetype);
-                } else {
-                        list_del(&buddy->lru);
-                        zone->free_area[order].nr_free--;
-                        rmv_page_order(buddy);
-                }
+                else
+                        del_page_from_free_area(buddy, &zone->free_area[order]);
                 combined_pfn = buddy_pfn & pfn;
                 page = page + (combined_pfn - pfn);
                 pfn = combined_pfn;
@@ -975,15 +966,13 @@ done_merging:
                 higher_buddy = higher_page + (buddy_pfn - combined_pfn);
                 if (pfn_valid_within(buddy_pfn) &&
                     page_is_buddy(higher_page, higher_buddy, order + 1)) {
-                        list_add_tail(&page->lru,
-                                &zone->free_area[order].free_list[migratetype]);
-                        goto out;
+                        add_to_free_area_tail(page, &zone->free_area[order],
+                                              migratetype);
+                        return;
                 }
         }
 
-        list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
-out:
-        zone->free_area[order].nr_free++;
+        add_to_free_area(page, &zone->free_area[order], migratetype);
 }
 
 /*
@@ -1974,8 +1963,7 @@ static inline void expand(struct zone *zone, struct page *page,
                 if (set_page_guard(zone, &page[size], high, migratetype))
                         continue;
 
-                list_add(&page[size].lru, &area->free_list[migratetype]);
-                area->nr_free++;
+                add_to_free_area(&page[size], area, migratetype);
                 set_page_order(&page[size], high);
         }
 }
@@ -2117,13 +2105,10 @@ struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
         /* Find a page of the appropriate size in the preferred list */
         for (current_order = order; current_order < MAX_ORDER; ++current_order) {
                 area = &(zone->free_area[current_order]);
-                page = list_first_entry_or_null(&area->free_list[migratetype],
-                                                        struct page, lru);
+                page = get_page_from_free_area(area, migratetype);
                 if (!page)
                         continue;
-                list_del(&page->lru);
-                rmv_page_order(page);
-                area->nr_free--;
+                del_page_from_free_area(page, area);
                 expand(zone, page, order, current_order, area, migratetype);
                 set_pcppage_migratetype(page, migratetype);
                 return page;
@@ -2209,8 +2194,7 @@ static int move_freepages(struct zone *zone,
                 }
 
                 order = page_order(page);
-                list_move(&page->lru,
-                          &zone->free_area[order].free_list[migratetype]);
+                move_to_free_area(page, &zone->free_area[order], migratetype);
                 page += 1 << order;
                 pages_moved += 1 << order;
         }
@@ -2398,7 +2382,7 @@ static void steal_suitable_fallback(struct zone *zone, struct page *page,
 
 single_page:
         area = &zone->free_area[current_order];
-        list_move(&page->lru, &area->free_list[start_type]);
+        move_to_free_area(page, area, start_type);
 }
 
 /*
@@ -2422,7 +2406,7 @@ int find_suitable_fallback(struct free_area *area, unsigned int order,
                 if (fallback_mt == MIGRATE_TYPES)
                         break;
 
-                if (list_empty(&area->free_list[fallback_mt]))
+                if (free_area_empty(area, fallback_mt))
                         continue;
 
                 if (can_steal_fallback(order, migratetype))
@@ -2509,9 +2493,7 @@ static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
                 for (order = 0; order < MAX_ORDER; order++) {
                         struct free_area *area = &(zone->free_area[order]);
 
-                        page = list_first_entry_or_null(
-                                        &area->free_list[MIGRATE_HIGHATOMIC],
-                                        struct page, lru);
+                        page = get_page_from_free_area(area, MIGRATE_HIGHATOMIC);
                         if (!page)
                                 continue;
 
@@ -2634,8 +2616,7 @@ find_smallest:
         VM_BUG_ON(current_order == MAX_ORDER);
 
 do_steal:
-        page = list_first_entry(&area->free_list[fallback_mt],
-                                                        struct page, lru);
+        page = get_page_from_free_area(area, fallback_mt);
 
         steal_suitable_fallback(zone, page, alloc_flags, start_migratetype,
                                                                 can_steal);
@@ -3072,6 +3053,7 @@ EXPORT_SYMBOL_GPL(split_page);
 
 int __isolate_free_page(struct page *page, unsigned int order)
 {
+        struct free_area *area = &page_zone(page)->free_area[order];
         unsigned long watermark;
         struct zone *zone;
         int mt;
@@ -3096,9 +3078,8 @@ int __isolate_free_page(struct page *page, unsigned int order)
         }
 
         /* Remove page from free list */
-        list_del(&page->lru);
-        zone->free_area[order].nr_free--;
-        rmv_page_order(page);
+
+        del_page_from_free_area(page, area);
 
         /*
          * Set the pageblock if the isolated page is at least half of a
@@ -3395,13 +3376,13 @@ bool __zone_watermark_ok(struct zone *z, unsigned int order, unsigned long mark,
                         continue;
 
                 for (mt = 0; mt < MIGRATE_PCPTYPES; mt++) {
-                        if (!list_empty(&area->free_list[mt]))
+                        if (!free_area_empty(area, mt))
                                 return true;
                 }
 
 #ifdef CONFIG_CMA
                 if ((alloc_flags & ALLOC_CMA) &&
-                    !list_empty(&area->free_list[MIGRATE_CMA])) {
+                    !free_area_empty(area, MIGRATE_CMA)) {
                         return true;
                 }
 #endif
@@ -5328,7 +5309,7 @@ void show_free_areas(unsigned int filter, nodemask_t *nodemask)
 
                         types[order] = 0;
                         for (type = 0; type < MIGRATE_TYPES; type++) {
-                                if (!list_empty(&area->free_list[type]))
+                                if (!free_area_empty(area, type))
                                         types[order] |= 1 << type;
                         }
                 }
@@ -8501,9 +8482,7 @@ __offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
                 pr_info("remove from free list %lx %d %lx\n",
                         pfn, 1 << order, end_pfn);
 #endif
-                list_del(&page->lru);
-                rmv_page_order(page);
-                zone->free_area[order].nr_free--;
+                del_page_from_free_area(page, &zone->free_area[order]);
                 for (i = 0; i < (1 << order); i++)
                         SetPageReserved((page+i));
                 pfn += (1 << order);