1 files changed, 32 insertions, 20 deletions

diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 485932c577e7..6e937809c87a 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -6028,53 +6028,65 @@ static inline int pfn_to_bitidx(struct zone *zone, unsigned long pfn)
  * @end_bitidx: The last bit of interest
  * returns pageblock_bits flags
  */
-unsigned long get_pageblock_flags_group(struct page *page,
-                                        int start_bitidx, int end_bitidx)
+unsigned long get_pageblock_flags_mask(struct page *page,
+                                        unsigned long end_bitidx,
+                                        unsigned long mask)
 {
         struct zone *zone;
         unsigned long *bitmap;
-        unsigned long pfn, bitidx;
-        unsigned long flags = 0;
-        unsigned long value = 1;
+        unsigned long pfn, bitidx, word_bitidx;
+        unsigned long word;
 
         zone = page_zone(page);
         pfn = page_to_pfn(page);
         bitmap = get_pageblock_bitmap(zone, pfn);
         bitidx = pfn_to_bitidx(zone, pfn);
+        word_bitidx = bitidx / BITS_PER_LONG;
+        bitidx &= (BITS_PER_LONG-1);
 
-        for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1)
-                if (test_bit(bitidx + start_bitidx, bitmap))
-                        flags |= value;
-
-        return flags;
+        word = bitmap[word_bitidx];
+        bitidx += end_bitidx;
+        return (word >> (BITS_PER_LONG - bitidx - 1)) & mask;
 }
 
 /**
- * set_pageblock_flags_group - Set the requested group of flags for a pageblock_nr_pages block of pages
+ * set_pageblock_flags_mask - Set the requested group of flags for a pageblock_nr_pages block of pages
  * @page: The page within the block of interest
  * @start_bitidx: The first bit of interest
  * @end_bitidx: The last bit of interest
  * @flags: The flags to set
  */
-void set_pageblock_flags_group(struct page *page, unsigned long flags,
-                                        int start_bitidx, int end_bitidx)
+void set_pageblock_flags_mask(struct page *page, unsigned long flags,
+                                        unsigned long end_bitidx,
+                                        unsigned long mask)
 {
         struct zone *zone;
         unsigned long *bitmap;
-        unsigned long pfn, bitidx;
-        unsigned long value = 1;
+        unsigned long pfn, bitidx, word_bitidx;
+        unsigned long old_word, word;
+
+        BUILD_BUG_ON(NR_PAGEBLOCK_BITS != 4);
 
         zone = page_zone(page);
         pfn = page_to_pfn(page);
         bitmap = get_pageblock_bitmap(zone, pfn);
         bitidx = pfn_to_bitidx(zone, pfn);
+        word_bitidx = bitidx / BITS_PER_LONG;
+        bitidx &= (BITS_PER_LONG-1);
+
         VM_BUG_ON_PAGE(!zone_spans_pfn(zone, pfn), page);
 
-        for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1)
-                if (flags & value)
-                        __set_bit(bitidx + start_bitidx, bitmap);
-                else
-                        __clear_bit(bitidx + start_bitidx, bitmap);
+        bitidx += end_bitidx;
+        mask <<= (BITS_PER_LONG - bitidx - 1);
+        flags <<= (BITS_PER_LONG - bitidx - 1);
+
+        word = ACCESS_ONCE(bitmap[word_bitidx]);
+        for (;;) {
+                old_word = cmpxchg(&bitmap[word_bitidx], word, (word & ~mask) | flags);
+                if (word == old_word)
+                        break;
+                word = old_word;
+        }
 }
 
 /*