1 files changed, 37 insertions, 25 deletions

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 2c25de46c58f..2f1118b4dda4 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -1948,23 +1948,44 @@ static bool can_steal_fallback(unsigned int order, int start_mt)
  * use it's pages as requested migratetype in the future.
  */
 static void steal_suitable_fallback(struct zone *zone, struct page *page,
-                                                          int start_type)
+                                        int start_type, bool whole_block)
 {
         unsigned int current_order = page_order(page);
+        struct free_area *area;
         int pages;
 
+        /*
+         * This can happen due to races and we want to prevent broken
+         * highatomic accounting.
+         */
+        if (is_migrate_highatomic_page(page))
+                goto single_page;
+
         /* Take ownership for orders >= pageblock_order */
         if (current_order >= pageblock_order) {
                 change_pageblock_range(page, current_order, start_type);
-                return;
+                goto single_page;
         }
 
+        /* We are not allowed to try stealing from the whole block */
+        if (!whole_block)
+                goto single_page;
+
         pages = move_freepages_block(zone, page, start_type);
+        /* moving whole block can fail due to zone boundary conditions */
+        if (!pages)
+                goto single_page;
 
         /* Claim the whole block if over half of it is free */
         if (pages >= (1 << (pageblock_order-1)) ||
                         page_group_by_mobility_disabled)
                 set_pageblock_migratetype(page, start_type);
+
+        return;
+
+single_page:
+        area = &zone->free_area[current_order];
+        list_move(&page->lru, &area->free_list[start_type]);
 }
 
 /*
@@ -2123,8 +2144,13 @@ static bool unreserve_highatomic_pageblock(const struct alloc_context *ac,
         return false;
 }
 
-/* Remove an element from the buddy allocator from the fallback list */
-static inline struct page *
+/*
+ * Try finding a free buddy page on the fallback list and put it on the free
+ * list of requested migratetype, possibly along with other pages from the same
+ * block, depending on fragmentation avoidance heuristics. Returns true if
+ * fallback was found so that __rmqueue_smallest() can grab it.
+ */
+static inline bool
 __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype)
 {
         struct free_area *area;
@@ -2145,32 +2171,17 @@ __rmqueue_fallback(struct zone *zone, unsigned int order, int start_migratetype)
 
                 page = list_first_entry(&area->free_list[fallback_mt],
                                                 struct page, lru);
-                if (can_steal && !is_migrate_highatomic_page(page))
-                        steal_suitable_fallback(zone, page, start_migratetype);
 
-                /* Remove the page from the freelists */
-                area->nr_free--;
-                list_del(&page->lru);
-                rmv_page_order(page);
-
-                expand(zone, page, order, current_order, area,
-                                        start_migratetype);
-                /*
-                 * The pcppage_migratetype may differ from pageblock's
-                 * migratetype depending on the decisions in
-                 * find_suitable_fallback(). This is OK as long as it does not
-                 * differ for MIGRATE_CMA pageblocks. Those can be used as
-                 * fallback only via special __rmqueue_cma_fallback() function
-                 */
-                set_pcppage_migratetype(page, start_migratetype);
+                steal_suitable_fallback(zone, page, start_migratetype,
+                                                                can_steal);
 
                 trace_mm_page_alloc_extfrag(page, order, current_order,
                         start_migratetype, fallback_mt);
 
-                return page;
+                return true;
         }
 
-        return NULL;
+        return false;
 }
 
 /*
@@ -2182,13 +2193,14 @@ static struct page *__rmqueue(struct zone *zone, unsigned int order,
 {
         struct page *page;
 
+retry:
         page = __rmqueue_smallest(zone, order, migratetype);
         if (unlikely(!page)) {
                 if (migratetype == MIGRATE_MOVABLE)
                         page = __rmqueue_cma_fallback(zone, order);
 
-                if (!page)
-                        page = __rmqueue_fallback(zone, order, migratetype);
+                if (!page && __rmqueue_fallback(zone, order, migratetype))
+                        goto retry;
         }
 
         trace_mm_page_alloc_zone_locked(page, order, migratetype);