s390: use _refcount for pgtables

Patch series "Rearrange struct page", v6.

As presented at LSFMM, this patch-set rearranges struct page to give
more contiguous usable space to users who have allocated a struct page
for their own purposes.  For a graphical view of before-and-after, see
the first two tabs of

  https://docs.google.com/spreadsheets/d/1tvCszs_7FXrjei9_mtFiKV6nW1FLnYyvPvW-qNZhdog/edit?usp=sharing

Highlights:
 - deferred_list now really exists in struct page instead of just a comment.
 - hmm_data also exists in struct page instead of being a nasty hack.
 - x86's PGD pages have a real pointer to the mm_struct.
 - VMalloc pages now have all sorts of extra information stored in them
   to help with debugging and tuning.
 - rcu_head is no longer tied to slab in case anyone else wants to
   free pages by RCU.
 - slub's counters no longer share space with _refcount.
 - slub's freelist+counters are now naturally dword aligned.
 - slub loses a parameter to a lot of functions and a sysfs file.

This patch (of 17):

s390 borrows the storage used for _mapcount in struct page in order to
account whether the bottom or top half is being used for 2kB page tables.
I want to use that for something else, so use the top byte of _refcount
instead of the bottom byte of _mapcount.  _refcount may temporarily be
incremented by other CPUs that see a stale pointer to this page in the
page cache, but each CPU can only increment it by one, and there are no
systems with 2^24 CPUs today, so they will not change the upper byte of
_refcount.  We do have to be a little careful not to lose any of their
writes (as they will subsequently decrement the counter).

Link: http://lkml.kernel.org/r/20180518194519.3820-2-willy@infradead.org
Signed-off-by: Matthew Wilcox <mawilcox@microsoft.com>
Acked-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
Cc: "Kirill A . Shutemov" <kirill.shutemov@linux.intel.com>
Cc: Christoph Lameter <cl@linux.com>
Cc: Lai Jiangshan <jiangshanlai@gmail.com>
Cc: Pekka Enberg <penberg@kernel.org>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Jérôme Glisse <jglisse@redhat.com>
Cc: Randy Dunlap <rdunlap@infradead.org>
Cc: Andrey Ryabinin <aryabinin@virtuozzo.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 12 insertions, 9 deletions

diff --git a/arch/s390/mm/pgalloc.c b/arch/s390/mm/pgalloc.c
index 562f72955956..84bd6329a88d 100644
--- a/arch/s390/mm/pgalloc.c
+++ b/arch/s390/mm/pgalloc.c
@@ -190,14 +190,15 @@ unsigned long *page_table_alloc(struct mm_struct *mm)
                 if (!list_empty(&mm->context.pgtable_list)) {
                         page = list_first_entry(&mm->context.pgtable_list,
                                                 struct page, lru);
-                        mask = atomic_read(&page->_mapcount);
+                        mask = atomic_read(&page->_refcount) >> 24;
                         mask = (mask | (mask >> 4)) & 3;
                         if (mask != 3) {
                                 table = (unsigned long *) page_to_phys(page);
                                 bit = mask & 1;         /* =1 -> second 2K */
                                 if (bit)
                                         table += PTRS_PER_PTE;
-                                atomic_xor_bits(&page->_mapcount, 1U << bit);
+                                atomic_xor_bits(&page->_refcount,
+                                                        1U << (bit + 24));
                                 list_del(&page->lru);
                         }
                 }
@@ -218,12 +219,12 @@ unsigned long *page_table_alloc(struct mm_struct *mm)
         table = (unsigned long *) page_to_phys(page);
         if (mm_alloc_pgste(mm)) {
                 /* Return 4K page table with PGSTEs */
-                atomic_set(&page->_mapcount, 3);
+                atomic_xor_bits(&page->_refcount, 3 << 24);
                 memset64((u64 *)table, _PAGE_INVALID, PTRS_PER_PTE);
                 memset64((u64 *)table + PTRS_PER_PTE, 0, PTRS_PER_PTE);
         } else {
                 /* Return the first 2K fragment of the page */
-                atomic_set(&page->_mapcount, 1);
+                atomic_xor_bits(&page->_refcount, 1 << 24);
                 memset64((u64 *)table, _PAGE_INVALID, 2 * PTRS_PER_PTE);
                 spin_lock_bh(&mm->context.lock);
                 list_add(&page->lru, &mm->context.pgtable_list);
@@ -242,7 +243,8 @@ void page_table_free(struct mm_struct *mm, unsigned long *table)
                 /* Free 2K page table fragment of a 4K page */
                 bit = (__pa(table) & ~PAGE_MASK)/(PTRS_PER_PTE*sizeof(pte_t));
                 spin_lock_bh(&mm->context.lock);
-                mask = atomic_xor_bits(&page->_mapcount, 1U << bit);
+                mask = atomic_xor_bits(&page->_refcount, 1U << (bit + 24));
+                mask >>= 24;
                 if (mask & 3)
                         list_add(&page->lru, &mm->context.pgtable_list);
                 else
@@ -253,7 +255,6 @@ void page_table_free(struct mm_struct *mm, unsigned long *table)
         }
 
         pgtable_page_dtor(page);
-        atomic_set(&page->_mapcount, -1);
         __free_page(page);
 }
 
@@ -274,7 +275,8 @@ void page_table_free_rcu(struct mmu_gather *tlb, unsigned long *table,
         }
         bit = (__pa(table) & ~PAGE_MASK) / (PTRS_PER_PTE*sizeof(pte_t));
         spin_lock_bh(&mm->context.lock);
-        mask = atomic_xor_bits(&page->_mapcount, 0x11U << bit);
+        mask = atomic_xor_bits(&page->_refcount, 0x11U << (bit + 24));
+        mask >>= 24;
         if (mask & 3)
                 list_add_tail(&page->lru, &mm->context.pgtable_list);
         else
@@ -296,12 +298,13 @@ static void __tlb_remove_table(void *_table)
                 break;
         case 1:         /* lower 2K of a 4K page table */
         case 2:         /* higher 2K of a 4K page table */
-                if (atomic_xor_bits(&page->_mapcount, mask << 4) != 0)
+                mask = atomic_xor_bits(&page->_refcount, mask << (4 + 24));
+                mask >>= 24;
+                if (mask != 0)
                         break;
                 /* fallthrough */
         case 3:         /* 4K page table with pgstes */
                 pgtable_page_dtor(page);
-                atomic_set(&page->_mapcount, -1);
                 __free_page(page);
                 break;
         }