1 files changed, 123 insertions, 18 deletions

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 1813b162b0a8..82d7024f0765 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -395,19 +395,81 @@ static inline void put_page(struct page *page)
 /*
  * The zone field is never updated after free_area_init_core()
  * sets it, so none of the operations on it need to be atomic.
- * We'll have up to (MAX_NUMNODES * MAX_NR_ZONES) zones total,
- * so we use (MAX_NODES_SHIFT + MAX_ZONES_SHIFT) here to get enough bits.
  */
-#define NODEZONE_SHIFT (sizeof(page_flags_t)*8 - MAX_NODES_SHIFT - MAX_ZONES_SHIFT)
-#define NODEZONE(node, zone)    ((node << ZONES_SHIFT) | zone)
+
+
+/*
+ * page->flags layout:
+ *
+ * There are three possibilities for how page->flags get
+ * laid out.  The first is for the normal case, without
+ * sparsemem.  The second is for sparsemem when there is
+ * plenty of space for node and section.  The last is when
+ * we have run out of space and have to fall back to an
+ * alternate (slower) way of determining the node.
+ *
+ *        No sparsemem: |       NODE     | ZONE | ... | FLAGS |
+ * with space for node: | SECTION | NODE | ZONE | ... | FLAGS |
+ *   no space for node: | SECTION |     ZONE    | ... | FLAGS |
+ */
+#ifdef CONFIG_SPARSEMEM
+#define SECTIONS_WIDTH          SECTIONS_SHIFT
+#else
+#define SECTIONS_WIDTH          0
+#endif
+
+#define ZONES_WIDTH             ZONES_SHIFT
+
+#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT <= FLAGS_RESERVED
+#define NODES_WIDTH             NODES_SHIFT
+#else
+#define NODES_WIDTH             0
+#endif
+
+/* Page flags: | [SECTION] | [NODE] | ZONE | ... | FLAGS | */
+#define SECTIONS_PGOFF          ((sizeof(page_flags_t)*8) - SECTIONS_WIDTH)
+#define NODES_PGOFF             (SECTIONS_PGOFF - NODES_WIDTH)
+#define ZONES_PGOFF             (NODES_PGOFF - ZONES_WIDTH)
+
+/*
+ * We are going to use the flags for the page to node mapping if its in
+ * there.  This includes the case where there is no node, so it is implicit.
+ */
+#define FLAGS_HAS_NODE          (NODES_WIDTH > 0 || NODES_SHIFT == 0)
+
+#ifndef PFN_SECTION_SHIFT
+#define PFN_SECTION_SHIFT 0
+#endif
+
+/*
+ * Define the bit shifts to access each section.  For non-existant
+ * sections we define the shift as 0; that plus a 0 mask ensures
+ * the compiler will optimise away reference to them.
+ */
+#define SECTIONS_PGSHIFT        (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))
+#define NODES_PGSHIFT           (NODES_PGOFF * (NODES_WIDTH != 0))
+#define ZONES_PGSHIFT           (ZONES_PGOFF * (ZONES_WIDTH != 0))
+
+/* NODE:ZONE or SECTION:ZONE is used to lookup the zone from a page. */
+#if FLAGS_HAS_NODE
+#define ZONETABLE_SHIFT         (NODES_SHIFT + ZONES_SHIFT)
+#else
+#define ZONETABLE_SHIFT         (SECTIONS_SHIFT + ZONES_SHIFT)
+#endif
+#define ZONETABLE_PGSHIFT       ZONES_PGSHIFT
+
+#if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > FLAGS_RESERVED
+#error SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > FLAGS_RESERVED
+#endif
+
+#define ZONES_MASK              ((1UL << ZONES_WIDTH) - 1)
+#define NODES_MASK              ((1UL << NODES_WIDTH) - 1)
+#define SECTIONS_MASK           ((1UL << SECTIONS_WIDTH) - 1)
+#define ZONETABLE_MASK          ((1UL << ZONETABLE_SHIFT) - 1)
 
 static inline unsigned long page_zonenum(struct page *page)
 {
-        return (page->flags >> NODEZONE_SHIFT) & (~(~0UL << ZONES_SHIFT));
-}
-static inline unsigned long page_to_nid(struct page *page)
-{
-        return (page->flags >> (NODEZONE_SHIFT + ZONES_SHIFT));
+        return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;
 }
 
 struct zone;
@@ -415,13 +477,44 @@ extern struct zone *zone_table[];
 
 static inline struct zone *page_zone(struct page *page)
 {
-        return zone_table[page->flags >> NODEZONE_SHIFT];
+        return zone_table[(page->flags >> ZONETABLE_PGSHIFT) &
+                        ZONETABLE_MASK];
+}
+
+static inline unsigned long page_to_nid(struct page *page)
+{
+        if (FLAGS_HAS_NODE)
+                return (page->flags >> NODES_PGSHIFT) & NODES_MASK;
+        else
+                return page_zone(page)->zone_pgdat->node_id;
+}
+static inline unsigned long page_to_section(struct page *page)
+{
+        return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
 }
 
-static inline void set_page_zone(struct page *page, unsigned long nodezone_num)
+static inline void set_page_zone(struct page *page, unsigned long zone)
 {
-        page->flags &= ~(~0UL << NODEZONE_SHIFT);
-        page->flags |= nodezone_num << NODEZONE_SHIFT;
+        page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
+        page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;
+}
+static inline void set_page_node(struct page *page, unsigned long node)
+{
+        page->flags &= ~(NODES_MASK << NODES_PGSHIFT);
+        page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;
+}
+static inline void set_page_section(struct page *page, unsigned long section)
+{
+        page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
+        page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
+}
+
+static inline void set_page_links(struct page *page, unsigned long zone,
+        unsigned long node, unsigned long pfn)
+{
+        set_page_zone(page, zone);
+        set_page_node(page, node);
+        set_page_section(page, pfn_to_section_nr(pfn));
 }
 
 #ifndef CONFIG_DISCONTIGMEM
@@ -532,10 +625,16 @@ static inline int page_mapped(struct page *page)
  * Used to decide whether a process gets delivered SIGBUS or
  * just gets major/minor fault counters bumped up.
  */
-#define VM_FAULT_OOM    (-1)
-#define VM_FAULT_SIGBUS 0
-#define VM_FAULT_MINOR  1
-#define VM_FAULT_MAJOR  2
+#define VM_FAULT_OOM    0x00
+#define VM_FAULT_SIGBUS 0x01
+#define VM_FAULT_MINOR  0x02
+#define VM_FAULT_MAJOR  0x03
+
+/* 
+ * Special case for get_user_pages.
+ * Must be in a distinct bit from the above VM_FAULT_ flags.
+ */
+#define VM_FAULT_WRITE  0x10
 
 #define offset_in_page(p)       ((unsigned long)(p) & ~PAGE_MASK)
 
@@ -611,7 +710,13 @@ extern pte_t *FASTCALL(pte_alloc_kernel(struct mm_struct *mm, pmd_t *pmd, unsign
 extern pte_t *FASTCALL(pte_alloc_map(struct mm_struct *mm, pmd_t *pmd, unsigned long address));
 extern int install_page(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, struct page *page, pgprot_t prot);
 extern int install_file_pte(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long addr, unsigned long pgoff, pgprot_t prot);
-extern int handle_mm_fault(struct mm_struct *mm,struct vm_area_struct *vma, unsigned long address, int write_access);
+extern int __handle_mm_fault(struct mm_struct *mm,struct vm_area_struct *vma, unsigned long address, int write_access);
+
+static inline int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, int write_access)
+{
+        return __handle_mm_fault(mm, vma, address, write_access) & (~VM_FAULT_WRITE);
+}
+
 extern int make_pages_present(unsigned long addr, unsigned long end);
 extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write);
 void install_arg_page(struct vm_area_struct *, struct page *, unsigned long);