aboutsummaryrefslogtreecommitdiffstats
path: root/include/linux
diff options
context:
space:
mode:
Diffstat (limited to 'include/linux')
-rw-r--r--include/linux/acpi.h8
-rw-r--r--include/linux/bootmem.h1
-rw-r--r--include/linux/compaction.h5
-rw-r--r--include/linux/firmware-map.h6
-rw-r--r--include/linux/highmem.h6
-rw-r--r--include/linux/huge_mm.h2
-rw-r--r--include/linux/hugetlb.h6
-rw-r--r--include/linux/ksm.h18
-rw-r--r--include/linux/memblock.h2
-rw-r--r--include/linux/memcontrol.h7
-rw-r--r--include/linux/memory_hotplug.h20
-rw-r--r--include/linux/migrate.h14
-rw-r--r--include/linux/mm.h178
-rw-r--r--include/linux/mm_types.h9
-rw-r--r--include/linux/mman.h4
-rw-r--r--include/linux/mmzone.h58
-rw-r--r--include/linux/page-flags-layout.h88
-rw-r--r--include/linux/page-isolation.h19
-rw-r--r--include/linux/pm.h1
-rw-r--r--include/linux/pm_runtime.h3
-rw-r--r--include/linux/rmap.h2
-rw-r--r--include/linux/sched.h22
-rw-r--r--include/linux/swap.h49
-rw-r--r--include/linux/vm_event_item.h1
-rw-r--r--include/linux/vmstat.h2
25 files changed, 356 insertions, 175 deletions
diff --git a/include/linux/acpi.h b/include/linux/acpi.h
index bcbdd7484e58..f46cfd73a553 100644
--- a/include/linux/acpi.h
+++ b/include/linux/acpi.h
@@ -485,6 +485,14 @@ static inline bool acpi_driver_match_device(struct device *dev,
485 485
486#endif /* !CONFIG_ACPI */ 486#endif /* !CONFIG_ACPI */
487 487
488#ifdef CONFIG_ACPI_NUMA
489void __init early_parse_srat(void);
490#else
491static inline void early_parse_srat(void)
492{
493}
494#endif
495
488#ifdef CONFIG_ACPI 496#ifdef CONFIG_ACPI
489void acpi_os_set_prepare_sleep(int (*func)(u8 sleep_state, 497void acpi_os_set_prepare_sleep(int (*func)(u8 sleep_state,
490 u32 pm1a_ctrl, u32 pm1b_ctrl)); 498 u32 pm1a_ctrl, u32 pm1b_ctrl));
diff --git a/include/linux/bootmem.h b/include/linux/bootmem.h
index 3cd16ba82f15..cdc3bab01832 100644
--- a/include/linux/bootmem.h
+++ b/include/linux/bootmem.h
@@ -53,6 +53,7 @@ extern void free_bootmem_node(pg_data_t *pgdat,
53 unsigned long size); 53 unsigned long size);
54extern void free_bootmem(unsigned long physaddr, unsigned long size); 54extern void free_bootmem(unsigned long physaddr, unsigned long size);
55extern void free_bootmem_late(unsigned long physaddr, unsigned long size); 55extern void free_bootmem_late(unsigned long physaddr, unsigned long size);
56extern void __free_pages_bootmem(struct page *page, unsigned int order);
56 57
57/* 58/*
58 * Flags for reserve_bootmem (also if CONFIG_HAVE_ARCH_BOOTMEM_NODE, 59 * Flags for reserve_bootmem (also if CONFIG_HAVE_ARCH_BOOTMEM_NODE,
diff --git a/include/linux/compaction.h b/include/linux/compaction.h
index cc7bddeaf553..091d72e70d8a 100644
--- a/include/linux/compaction.h
+++ b/include/linux/compaction.h
@@ -23,7 +23,7 @@ extern int fragmentation_index(struct zone *zone, unsigned int order);
23extern unsigned long try_to_compact_pages(struct zonelist *zonelist, 23extern unsigned long try_to_compact_pages(struct zonelist *zonelist,
24 int order, gfp_t gfp_mask, nodemask_t *mask, 24 int order, gfp_t gfp_mask, nodemask_t *mask,
25 bool sync, bool *contended); 25 bool sync, bool *contended);
26extern int compact_pgdat(pg_data_t *pgdat, int order); 26extern void compact_pgdat(pg_data_t *pgdat, int order);
27extern void reset_isolation_suitable(pg_data_t *pgdat); 27extern void reset_isolation_suitable(pg_data_t *pgdat);
28extern unsigned long compaction_suitable(struct zone *zone, int order); 28extern unsigned long compaction_suitable(struct zone *zone, int order);
29 29
@@ -80,9 +80,8 @@ static inline unsigned long try_to_compact_pages(struct zonelist *zonelist,
80 return COMPACT_CONTINUE; 80 return COMPACT_CONTINUE;
81} 81}
82 82
83static inline int compact_pgdat(pg_data_t *pgdat, int order) 83static inline void compact_pgdat(pg_data_t *pgdat, int order)
84{ 84{
85 return COMPACT_CONTINUE;
86} 85}
87 86
88static inline void reset_isolation_suitable(pg_data_t *pgdat) 87static inline void reset_isolation_suitable(pg_data_t *pgdat)
diff --git a/include/linux/firmware-map.h b/include/linux/firmware-map.h
index 43fe52fcef0f..71d4fa721db9 100644
--- a/include/linux/firmware-map.h
+++ b/include/linux/firmware-map.h
@@ -25,6 +25,7 @@
25 25
26int firmware_map_add_early(u64 start, u64 end, const char *type); 26int firmware_map_add_early(u64 start, u64 end, const char *type);
27int firmware_map_add_hotplug(u64 start, u64 end, const char *type); 27int firmware_map_add_hotplug(u64 start, u64 end, const char *type);
28int firmware_map_remove(u64 start, u64 end, const char *type);
28 29
29#else /* CONFIG_FIRMWARE_MEMMAP */ 30#else /* CONFIG_FIRMWARE_MEMMAP */
30 31
@@ -38,6 +39,11 @@ static inline int firmware_map_add_hotplug(u64 start, u64 end, const char *type)
38 return 0; 39 return 0;
39} 40}
40 41
42static inline int firmware_map_remove(u64 start, u64 end, const char *type)
43{
44 return 0;
45}
46
41#endif /* CONFIG_FIRMWARE_MEMMAP */ 47#endif /* CONFIG_FIRMWARE_MEMMAP */
42 48
43#endif /* _LINUX_FIRMWARE_MAP_H */ 49#endif /* _LINUX_FIRMWARE_MAP_H */
diff --git a/include/linux/highmem.h b/include/linux/highmem.h
index ef788b5b4a35..7fb31da45d03 100644
--- a/include/linux/highmem.h
+++ b/include/linux/highmem.h
@@ -219,12 +219,6 @@ static inline void zero_user(struct page *page,
219 zero_user_segments(page, start, start + size, 0, 0); 219 zero_user_segments(page, start, start + size, 0, 0);
220} 220}
221 221
222static inline void __deprecated memclear_highpage_flush(struct page *page,
223 unsigned int offset, unsigned int size)
224{
225 zero_user(page, offset, size);
226}
227
228#ifndef __HAVE_ARCH_COPY_USER_HIGHPAGE 222#ifndef __HAVE_ARCH_COPY_USER_HIGHPAGE
229 223
230static inline void copy_user_highpage(struct page *to, struct page *from, 224static inline void copy_user_highpage(struct page *to, struct page *from,
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h
index 1d76f8ca90f0..ee1c244a62a1 100644
--- a/include/linux/huge_mm.h
+++ b/include/linux/huge_mm.h
@@ -113,7 +113,7 @@ extern void __split_huge_page_pmd(struct vm_area_struct *vma,
113 do { \ 113 do { \
114 pmd_t *____pmd = (__pmd); \ 114 pmd_t *____pmd = (__pmd); \
115 anon_vma_lock_write(__anon_vma); \ 115 anon_vma_lock_write(__anon_vma); \
116 anon_vma_unlock(__anon_vma); \ 116 anon_vma_unlock_write(__anon_vma); \
117 BUG_ON(pmd_trans_splitting(*____pmd) || \ 117 BUG_ON(pmd_trans_splitting(*____pmd) || \
118 pmd_trans_huge(*____pmd)); \ 118 pmd_trans_huge(*____pmd)); \
119 } while (0) 119 } while (0)
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index 0c80d3f57a5b..eedc334fb6f5 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -43,9 +43,9 @@ int hugetlb_mempolicy_sysctl_handler(struct ctl_table *, int,
43#endif 43#endif
44 44
45int copy_hugetlb_page_range(struct mm_struct *, struct mm_struct *, struct vm_area_struct *); 45int copy_hugetlb_page_range(struct mm_struct *, struct mm_struct *, struct vm_area_struct *);
46int follow_hugetlb_page(struct mm_struct *, struct vm_area_struct *, 46long follow_hugetlb_page(struct mm_struct *, struct vm_area_struct *,
47 struct page **, struct vm_area_struct **, 47 struct page **, struct vm_area_struct **,
48 unsigned long *, int *, int, unsigned int flags); 48 unsigned long *, unsigned long *, long, unsigned int);
49void unmap_hugepage_range(struct vm_area_struct *, 49void unmap_hugepage_range(struct vm_area_struct *,
50 unsigned long, unsigned long, struct page *); 50 unsigned long, unsigned long, struct page *);
51void __unmap_hugepage_range_final(struct mmu_gather *tlb, 51void __unmap_hugepage_range_final(struct mmu_gather *tlb,
diff --git a/include/linux/ksm.h b/include/linux/ksm.h
index 3319a6967626..45c9b6a17bcb 100644
--- a/include/linux/ksm.h
+++ b/include/linux/ksm.h
@@ -16,9 +16,6 @@
16struct stable_node; 16struct stable_node;
17struct mem_cgroup; 17struct mem_cgroup;
18 18
19struct page *ksm_does_need_to_copy(struct page *page,
20 struct vm_area_struct *vma, unsigned long address);
21
22#ifdef CONFIG_KSM 19#ifdef CONFIG_KSM
23int ksm_madvise(struct vm_area_struct *vma, unsigned long start, 20int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
24 unsigned long end, int advice, unsigned long *vm_flags); 21 unsigned long end, int advice, unsigned long *vm_flags);
@@ -73,15 +70,8 @@ static inline void set_page_stable_node(struct page *page,
73 * We'd like to make this conditional on vma->vm_flags & VM_MERGEABLE, 70 * We'd like to make this conditional on vma->vm_flags & VM_MERGEABLE,
74 * but what if the vma was unmerged while the page was swapped out? 71 * but what if the vma was unmerged while the page was swapped out?
75 */ 72 */
76static inline int ksm_might_need_to_copy(struct page *page, 73struct page *ksm_might_need_to_copy(struct page *page,
77 struct vm_area_struct *vma, unsigned long address) 74 struct vm_area_struct *vma, unsigned long address);
78{
79 struct anon_vma *anon_vma = page_anon_vma(page);
80
81 return anon_vma &&
82 (anon_vma->root != vma->anon_vma->root ||
83 page->index != linear_page_index(vma, address));
84}
85 75
86int page_referenced_ksm(struct page *page, 76int page_referenced_ksm(struct page *page,
87 struct mem_cgroup *memcg, unsigned long *vm_flags); 77 struct mem_cgroup *memcg, unsigned long *vm_flags);
@@ -113,10 +103,10 @@ static inline int ksm_madvise(struct vm_area_struct *vma, unsigned long start,
113 return 0; 103 return 0;
114} 104}
115 105
116static inline int ksm_might_need_to_copy(struct page *page, 106static inline struct page *ksm_might_need_to_copy(struct page *page,
117 struct vm_area_struct *vma, unsigned long address) 107 struct vm_area_struct *vma, unsigned long address)
118{ 108{
119 return 0; 109 return page;
120} 110}
121 111
122static inline int page_referenced_ksm(struct page *page, 112static inline int page_referenced_ksm(struct page *page,
diff --git a/include/linux/memblock.h b/include/linux/memblock.h
index f388203db7e8..3e5ecb2d790e 100644
--- a/include/linux/memblock.h
+++ b/include/linux/memblock.h
@@ -42,6 +42,7 @@ struct memblock {
42 42
43extern struct memblock memblock; 43extern struct memblock memblock;
44extern int memblock_debug; 44extern int memblock_debug;
45extern struct movablemem_map movablemem_map;
45 46
46#define memblock_dbg(fmt, ...) \ 47#define memblock_dbg(fmt, ...) \
47 if (memblock_debug) printk(KERN_INFO pr_fmt(fmt), ##__VA_ARGS__) 48 if (memblock_debug) printk(KERN_INFO pr_fmt(fmt), ##__VA_ARGS__)
@@ -60,6 +61,7 @@ int memblock_reserve(phys_addr_t base, phys_addr_t size);
60void memblock_trim_memory(phys_addr_t align); 61void memblock_trim_memory(phys_addr_t align);
61 62
62#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP 63#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
64
63void __next_mem_pfn_range(int *idx, int nid, unsigned long *out_start_pfn, 65void __next_mem_pfn_range(int *idx, int nid, unsigned long *out_start_pfn,
64 unsigned long *out_end_pfn, int *out_nid); 66 unsigned long *out_end_pfn, int *out_nid);
65 67
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 28bd5fa2ff2e..d6183f06d8c1 100644
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -116,7 +116,6 @@ void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *);
116 * For memory reclaim. 116 * For memory reclaim.
117 */ 117 */
118int mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec); 118int mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec);
119int mem_cgroup_inactive_file_is_low(struct lruvec *lruvec);
120int mem_cgroup_select_victim_node(struct mem_cgroup *memcg); 119int mem_cgroup_select_victim_node(struct mem_cgroup *memcg);
121unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list); 120unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list);
122void mem_cgroup_update_lru_size(struct lruvec *, enum lru_list, int); 121void mem_cgroup_update_lru_size(struct lruvec *, enum lru_list, int);
@@ -321,12 +320,6 @@ mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec)
321 return 1; 320 return 1;
322} 321}
323 322
324static inline int
325mem_cgroup_inactive_file_is_low(struct lruvec *lruvec)
326{
327 return 1;
328}
329
330static inline unsigned long 323static inline unsigned long
331mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru) 324mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru)
332{ 325{
diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h
index 4a45c4e50025..b6a3be7d47bf 100644
--- a/include/linux/memory_hotplug.h
+++ b/include/linux/memory_hotplug.h
@@ -96,6 +96,7 @@ extern void __online_page_free(struct page *page);
96 96
97#ifdef CONFIG_MEMORY_HOTREMOVE 97#ifdef CONFIG_MEMORY_HOTREMOVE
98extern bool is_pageblock_removable_nolock(struct page *page); 98extern bool is_pageblock_removable_nolock(struct page *page);
99extern int arch_remove_memory(u64 start, u64 size);
99#endif /* CONFIG_MEMORY_HOTREMOVE */ 100#endif /* CONFIG_MEMORY_HOTREMOVE */
100 101
101/* reasonably generic interface to expand the physical pages in a zone */ 102/* reasonably generic interface to expand the physical pages in a zone */
@@ -173,17 +174,16 @@ static inline void arch_refresh_nodedata(int nid, pg_data_t *pgdat)
173#endif /* CONFIG_NUMA */ 174#endif /* CONFIG_NUMA */
174#endif /* CONFIG_HAVE_ARCH_NODEDATA_EXTENSION */ 175#endif /* CONFIG_HAVE_ARCH_NODEDATA_EXTENSION */
175 176
176#ifdef CONFIG_SPARSEMEM_VMEMMAP 177#ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE
178extern void register_page_bootmem_info_node(struct pglist_data *pgdat);
179#else
177static inline void register_page_bootmem_info_node(struct pglist_data *pgdat) 180static inline void register_page_bootmem_info_node(struct pglist_data *pgdat)
178{ 181{
179} 182}
180static inline void put_page_bootmem(struct page *page)
181{
182}
183#else
184extern void register_page_bootmem_info_node(struct pglist_data *pgdat);
185extern void put_page_bootmem(struct page *page);
186#endif 183#endif
184extern void put_page_bootmem(struct page *page);
185extern void get_page_bootmem(unsigned long ingo, struct page *page,
186 unsigned long type);
187 187
188/* 188/*
189 * Lock for memory hotplug guarantees 1) all callbacks for memory hotplug 189 * Lock for memory hotplug guarantees 1) all callbacks for memory hotplug
@@ -233,6 +233,7 @@ static inline void unlock_memory_hotplug(void) {}
233#ifdef CONFIG_MEMORY_HOTREMOVE 233#ifdef CONFIG_MEMORY_HOTREMOVE
234 234
235extern int is_mem_section_removable(unsigned long pfn, unsigned long nr_pages); 235extern int is_mem_section_removable(unsigned long pfn, unsigned long nr_pages);
236extern void try_offline_node(int nid);
236 237
237#else 238#else
238static inline int is_mem_section_removable(unsigned long pfn, 239static inline int is_mem_section_removable(unsigned long pfn,
@@ -240,6 +241,8 @@ static inline int is_mem_section_removable(unsigned long pfn,
240{ 241{
241 return 0; 242 return 0;
242} 243}
244
245static inline void try_offline_node(int nid) {}
243#endif /* CONFIG_MEMORY_HOTREMOVE */ 246#endif /* CONFIG_MEMORY_HOTREMOVE */
244 247
245extern int mem_online_node(int nid); 248extern int mem_online_node(int nid);
@@ -247,7 +250,8 @@ extern int add_memory(int nid, u64 start, u64 size);
247extern int arch_add_memory(int nid, u64 start, u64 size); 250extern int arch_add_memory(int nid, u64 start, u64 size);
248extern int offline_pages(unsigned long start_pfn, unsigned long nr_pages); 251extern int offline_pages(unsigned long start_pfn, unsigned long nr_pages);
249extern int offline_memory_block(struct memory_block *mem); 252extern int offline_memory_block(struct memory_block *mem);
250extern int remove_memory(u64 start, u64 size); 253extern bool is_memblock_offlined(struct memory_block *mem);
254extern int remove_memory(int nid, u64 start, u64 size);
251extern int sparse_add_one_section(struct zone *zone, unsigned long start_pfn, 255extern int sparse_add_one_section(struct zone *zone, unsigned long start_pfn,
252 int nr_pages); 256 int nr_pages);
253extern void sparse_remove_one_section(struct zone *zone, struct mem_section *ms); 257extern void sparse_remove_one_section(struct zone *zone, struct mem_section *ms);
diff --git a/include/linux/migrate.h b/include/linux/migrate.h
index 1e9f627967a3..a405d3dc0f61 100644
--- a/include/linux/migrate.h
+++ b/include/linux/migrate.h
@@ -40,11 +40,9 @@ extern void putback_movable_pages(struct list_head *l);
40extern int migrate_page(struct address_space *, 40extern int migrate_page(struct address_space *,
41 struct page *, struct page *, enum migrate_mode); 41 struct page *, struct page *, enum migrate_mode);
42extern int migrate_pages(struct list_head *l, new_page_t x, 42extern int migrate_pages(struct list_head *l, new_page_t x,
43 unsigned long private, bool offlining, 43 unsigned long private, enum migrate_mode mode, int reason);
44 enum migrate_mode mode, int reason);
45extern int migrate_huge_page(struct page *, new_page_t x, 44extern int migrate_huge_page(struct page *, new_page_t x,
46 unsigned long private, bool offlining, 45 unsigned long private, enum migrate_mode mode);
47 enum migrate_mode mode);
48 46
49extern int fail_migrate_page(struct address_space *, 47extern int fail_migrate_page(struct address_space *,
50 struct page *, struct page *); 48 struct page *, struct page *);
@@ -62,11 +60,11 @@ extern int migrate_huge_page_move_mapping(struct address_space *mapping,
62static inline void putback_lru_pages(struct list_head *l) {} 60static inline void putback_lru_pages(struct list_head *l) {}
63static inline void putback_movable_pages(struct list_head *l) {} 61static inline void putback_movable_pages(struct list_head *l) {}
64static inline int migrate_pages(struct list_head *l, new_page_t x, 62static inline int migrate_pages(struct list_head *l, new_page_t x,
65 unsigned long private, bool offlining, 63 unsigned long private, enum migrate_mode mode, int reason)
66 enum migrate_mode mode, int reason) { return -ENOSYS; } 64 { return -ENOSYS; }
67static inline int migrate_huge_page(struct page *page, new_page_t x, 65static inline int migrate_huge_page(struct page *page, new_page_t x,
68 unsigned long private, bool offlining, 66 unsigned long private, enum migrate_mode mode)
69 enum migrate_mode mode) { return -ENOSYS; } 67 { return -ENOSYS; }
70 68
71static inline int migrate_prep(void) { return -ENOSYS; } 69static inline int migrate_prep(void) { return -ENOSYS; }
72static inline int migrate_prep_local(void) { return -ENOSYS; } 70static inline int migrate_prep_local(void) { return -ENOSYS; }
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 9d9dcc35d6a1..e7c3f9a0111a 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -87,6 +87,7 @@ extern unsigned int kobjsize(const void *objp);
87#define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */ 87#define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */
88#define VM_DENYWRITE 0x00000800 /* ETXTBSY on write attempts.. */ 88#define VM_DENYWRITE 0x00000800 /* ETXTBSY on write attempts.. */
89 89
90#define VM_POPULATE 0x00001000
90#define VM_LOCKED 0x00002000 91#define VM_LOCKED 0x00002000
91#define VM_IO 0x00004000 /* Memory mapped I/O or similar */ 92#define VM_IO 0x00004000 /* Memory mapped I/O or similar */
92 93
@@ -366,7 +367,7 @@ static inline struct page *compound_head(struct page *page)
366 * both from it and to it can be tracked, using atomic_inc_and_test 367 * both from it and to it can be tracked, using atomic_inc_and_test
367 * and atomic_add_negative(-1). 368 * and atomic_add_negative(-1).
368 */ 369 */
369static inline void reset_page_mapcount(struct page *page) 370static inline void page_mapcount_reset(struct page *page)
370{ 371{
371 atomic_set(&(page)->_mapcount, -1); 372 atomic_set(&(page)->_mapcount, -1);
372} 373}
@@ -580,50 +581,11 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
580 * sets it, so none of the operations on it need to be atomic. 581 * sets it, so none of the operations on it need to be atomic.
581 */ 582 */
582 583
583 584/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_NID] | ... | FLAGS | */
584/*
585 * page->flags layout:
586 *
587 * There are three possibilities for how page->flags get
588 * laid out. The first is for the normal case, without
589 * sparsemem. The second is for sparsemem when there is
590 * plenty of space for node and section. The last is when
591 * we have run out of space and have to fall back to an
592 * alternate (slower) way of determining the node.
593 *
594 * No sparsemem or sparsemem vmemmap: | NODE | ZONE | ... | FLAGS |
595 * classic sparse with space for node:| SECTION | NODE | ZONE | ... | FLAGS |
596 * classic sparse no space for node: | SECTION | ZONE | ... | FLAGS |
597 */
598#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
599#define SECTIONS_WIDTH SECTIONS_SHIFT
600#else
601#define SECTIONS_WIDTH 0
602#endif
603
604#define ZONES_WIDTH ZONES_SHIFT
605
606#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS
607#define NODES_WIDTH NODES_SHIFT
608#else
609#ifdef CONFIG_SPARSEMEM_VMEMMAP
610#error "Vmemmap: No space for nodes field in page flags"
611#endif
612#define NODES_WIDTH 0
613#endif
614
615/* Page flags: | [SECTION] | [NODE] | ZONE | ... | FLAGS | */
616#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH) 585#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)
617#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH) 586#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH)
618#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH) 587#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)
619 588#define LAST_NID_PGOFF (ZONES_PGOFF - LAST_NID_WIDTH)
620/*
621 * We are going to use the flags for the page to node mapping if its in
622 * there. This includes the case where there is no node, so it is implicit.
623 */
624#if !(NODES_WIDTH > 0 || NODES_SHIFT == 0)
625#define NODE_NOT_IN_PAGE_FLAGS
626#endif
627 589
628/* 590/*
629 * Define the bit shifts to access each section. For non-existent 591 * Define the bit shifts to access each section. For non-existent
@@ -633,6 +595,7 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
633#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0)) 595#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))
634#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0)) 596#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0))
635#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0)) 597#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0))
598#define LAST_NID_PGSHIFT (LAST_NID_PGOFF * (LAST_NID_WIDTH != 0))
636 599
637/* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */ 600/* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */
638#ifdef NODE_NOT_IN_PAGE_FLAGS 601#ifdef NODE_NOT_IN_PAGE_FLAGS
@@ -654,6 +617,7 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
654#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1) 617#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1)
655#define NODES_MASK ((1UL << NODES_WIDTH) - 1) 618#define NODES_MASK ((1UL << NODES_WIDTH) - 1)
656#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1) 619#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)
620#define LAST_NID_MASK ((1UL << LAST_NID_WIDTH) - 1)
657#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1) 621#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1)
658 622
659static inline enum zone_type page_zonenum(const struct page *page) 623static inline enum zone_type page_zonenum(const struct page *page)
@@ -661,6 +625,10 @@ static inline enum zone_type page_zonenum(const struct page *page)
661 return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK; 625 return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;
662} 626}
663 627
628#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
629#define SECTION_IN_PAGE_FLAGS
630#endif
631
664/* 632/*
665 * The identification function is only used by the buddy allocator for 633 * The identification function is only used by the buddy allocator for
666 * determining if two pages could be buddies. We are not really 634 * determining if two pages could be buddies. We are not really
@@ -693,31 +661,48 @@ static inline int page_to_nid(const struct page *page)
693#endif 661#endif
694 662
695#ifdef CONFIG_NUMA_BALANCING 663#ifdef CONFIG_NUMA_BALANCING
696static inline int page_xchg_last_nid(struct page *page, int nid) 664#ifdef LAST_NID_NOT_IN_PAGE_FLAGS
665static inline int page_nid_xchg_last(struct page *page, int nid)
697{ 666{
698 return xchg(&page->_last_nid, nid); 667 return xchg(&page->_last_nid, nid);
699} 668}
700 669
701static inline int page_last_nid(struct page *page) 670static inline int page_nid_last(struct page *page)
702{ 671{
703 return page->_last_nid; 672 return page->_last_nid;
704} 673}
705static inline void reset_page_last_nid(struct page *page) 674static inline void page_nid_reset_last(struct page *page)
706{ 675{
707 page->_last_nid = -1; 676 page->_last_nid = -1;
708} 677}
709#else 678#else
710static inline int page_xchg_last_nid(struct page *page, int nid) 679static inline int page_nid_last(struct page *page)
680{
681 return (page->flags >> LAST_NID_PGSHIFT) & LAST_NID_MASK;
682}
683
684extern int page_nid_xchg_last(struct page *page, int nid);
685
686static inline void page_nid_reset_last(struct page *page)
687{
688 int nid = (1 << LAST_NID_SHIFT) - 1;
689
690 page->flags &= ~(LAST_NID_MASK << LAST_NID_PGSHIFT);
691 page->flags |= (nid & LAST_NID_MASK) << LAST_NID_PGSHIFT;
692}
693#endif /* LAST_NID_NOT_IN_PAGE_FLAGS */
694#else
695static inline int page_nid_xchg_last(struct page *page, int nid)
711{ 696{
712 return page_to_nid(page); 697 return page_to_nid(page);
713} 698}
714 699
715static inline int page_last_nid(struct page *page) 700static inline int page_nid_last(struct page *page)
716{ 701{
717 return page_to_nid(page); 702 return page_to_nid(page);
718} 703}
719 704
720static inline void reset_page_last_nid(struct page *page) 705static inline void page_nid_reset_last(struct page *page)
721{ 706{
722} 707}
723#endif 708#endif
@@ -727,7 +712,7 @@ static inline struct zone *page_zone(const struct page *page)
727 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)]; 712 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];
728} 713}
729 714
730#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) 715#ifdef SECTION_IN_PAGE_FLAGS
731static inline void set_page_section(struct page *page, unsigned long section) 716static inline void set_page_section(struct page *page, unsigned long section)
732{ 717{
733 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT); 718 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
@@ -757,7 +742,7 @@ static inline void set_page_links(struct page *page, enum zone_type zone,
757{ 742{
758 set_page_zone(page, zone); 743 set_page_zone(page, zone);
759 set_page_node(page, node); 744 set_page_node(page, node);
760#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) 745#ifdef SECTION_IN_PAGE_FLAGS
761 set_page_section(page, pfn_to_section_nr(pfn)); 746 set_page_section(page, pfn_to_section_nr(pfn));
762#endif 747#endif
763} 748}
@@ -817,18 +802,7 @@ void page_address_init(void);
817#define PAGE_MAPPING_KSM 2 802#define PAGE_MAPPING_KSM 2
818#define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM) 803#define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM)
819 804
820extern struct address_space swapper_space; 805extern struct address_space *page_mapping(struct page *page);
821static inline struct address_space *page_mapping(struct page *page)
822{
823 struct address_space *mapping = page->mapping;
824
825 VM_BUG_ON(PageSlab(page));
826 if (unlikely(PageSwapCache(page)))
827 mapping = &swapper_space;
828 else if ((unsigned long)mapping & PAGE_MAPPING_ANON)
829 mapping = NULL;
830 return mapping;
831}
832 806
833/* Neutral page->mapping pointer to address_space or anon_vma or other */ 807/* Neutral page->mapping pointer to address_space or anon_vma or other */
834static inline void *page_rmapping(struct page *page) 808static inline void *page_rmapping(struct page *page)
@@ -1035,18 +1009,18 @@ static inline int fixup_user_fault(struct task_struct *tsk,
1035} 1009}
1036#endif 1010#endif
1037 1011
1038extern int make_pages_present(unsigned long addr, unsigned long end);
1039extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write); 1012extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write);
1040extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, 1013extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
1041 void *buf, int len, int write); 1014 void *buf, int len, int write);
1042 1015
1043int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, 1016long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
1044 unsigned long start, int len, unsigned int foll_flags, 1017 unsigned long start, unsigned long nr_pages,
1045 struct page **pages, struct vm_area_struct **vmas, 1018 unsigned int foll_flags, struct page **pages,
1046 int *nonblocking); 1019 struct vm_area_struct **vmas, int *nonblocking);
1047int get_user_pages(struct task_struct *tsk, struct mm_struct *mm, 1020long get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
1048 unsigned long start, int nr_pages, int write, int force, 1021 unsigned long start, unsigned long nr_pages,
1049 struct page **pages, struct vm_area_struct **vmas); 1022 int write, int force, struct page **pages,
1023 struct vm_area_struct **vmas);
1050int get_user_pages_fast(unsigned long start, int nr_pages, int write, 1024int get_user_pages_fast(unsigned long start, int nr_pages, int write,
1051 struct page **pages); 1025 struct page **pages);
1052struct kvec; 1026struct kvec;
@@ -1359,6 +1333,24 @@ extern void free_bootmem_with_active_regions(int nid,
1359 unsigned long max_low_pfn); 1333 unsigned long max_low_pfn);
1360extern void sparse_memory_present_with_active_regions(int nid); 1334extern void sparse_memory_present_with_active_regions(int nid);
1361 1335
1336#define MOVABLEMEM_MAP_MAX MAX_NUMNODES
1337struct movablemem_entry {
1338 unsigned long start_pfn; /* start pfn of memory segment */
1339 unsigned long end_pfn; /* end pfn of memory segment (exclusive) */
1340};
1341
1342struct movablemem_map {
1343 bool acpi; /* true if using SRAT info */
1344 int nr_map;
1345 struct movablemem_entry map[MOVABLEMEM_MAP_MAX];
1346 nodemask_t numa_nodes_hotplug; /* on which nodes we specify memory */
1347 nodemask_t numa_nodes_kernel; /* on which nodes kernel resides in */
1348};
1349
1350extern void __init insert_movablemem_map(unsigned long start_pfn,
1351 unsigned long end_pfn);
1352extern int __init movablemem_map_overlap(unsigned long start_pfn,
1353 unsigned long end_pfn);
1362#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ 1354#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
1363 1355
1364#if !defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) && \ 1356#if !defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) && \
@@ -1395,6 +1387,9 @@ extern void setup_per_cpu_pageset(void);
1395extern void zone_pcp_update(struct zone *zone); 1387extern void zone_pcp_update(struct zone *zone);
1396extern void zone_pcp_reset(struct zone *zone); 1388extern void zone_pcp_reset(struct zone *zone);
1397 1389
1390/* page_alloc.c */
1391extern int min_free_kbytes;
1392
1398/* nommu.c */ 1393/* nommu.c */
1399extern atomic_long_t mmap_pages_allocated; 1394extern atomic_long_t mmap_pages_allocated;
1400extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t); 1395extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
@@ -1472,13 +1467,24 @@ extern int install_special_mapping(struct mm_struct *mm,
1472extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long); 1467extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
1473 1468
1474extern unsigned long mmap_region(struct file *file, unsigned long addr, 1469extern unsigned long mmap_region(struct file *file, unsigned long addr,
1475 unsigned long len, unsigned long flags, 1470 unsigned long len, vm_flags_t vm_flags, unsigned long pgoff);
1476 vm_flags_t vm_flags, unsigned long pgoff); 1471extern unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
1477extern unsigned long do_mmap_pgoff(struct file *, unsigned long, 1472 unsigned long len, unsigned long prot, unsigned long flags,
1478 unsigned long, unsigned long, 1473 unsigned long pgoff, unsigned long *populate);
1479 unsigned long, unsigned long);
1480extern int do_munmap(struct mm_struct *, unsigned long, size_t); 1474extern int do_munmap(struct mm_struct *, unsigned long, size_t);
1481 1475
1476#ifdef CONFIG_MMU
1477extern int __mm_populate(unsigned long addr, unsigned long len,
1478 int ignore_errors);
1479static inline void mm_populate(unsigned long addr, unsigned long len)
1480{
1481 /* Ignore errors */
1482 (void) __mm_populate(addr, len, 1);
1483}
1484#else
1485static inline void mm_populate(unsigned long addr, unsigned long len) {}
1486#endif
1487
1482/* These take the mm semaphore themselves */ 1488/* These take the mm semaphore themselves */
1483extern unsigned long vm_brk(unsigned long, unsigned long); 1489extern unsigned long vm_brk(unsigned long, unsigned long);
1484extern int vm_munmap(unsigned long, size_t); 1490extern int vm_munmap(unsigned long, size_t);
@@ -1623,8 +1629,17 @@ int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
1623int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr, 1629int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
1624 unsigned long pfn); 1630 unsigned long pfn);
1625 1631
1626struct page *follow_page(struct vm_area_struct *, unsigned long address, 1632struct page *follow_page_mask(struct vm_area_struct *vma,
1627 unsigned int foll_flags); 1633 unsigned long address, unsigned int foll_flags,
1634 unsigned int *page_mask);
1635
1636static inline struct page *follow_page(struct vm_area_struct *vma,
1637 unsigned long address, unsigned int foll_flags)
1638{
1639 unsigned int unused_page_mask;
1640 return follow_page_mask(vma, address, foll_flags, &unused_page_mask);
1641}
1642
1628#define FOLL_WRITE 0x01 /* check pte is writable */ 1643#define FOLL_WRITE 0x01 /* check pte is writable */
1629#define FOLL_TOUCH 0x02 /* mark page accessed */ 1644#define FOLL_TOUCH 0x02 /* mark page accessed */
1630#define FOLL_GET 0x04 /* do get_page on page */ 1645#define FOLL_GET 0x04 /* do get_page on page */
@@ -1636,6 +1651,7 @@ struct page *follow_page(struct vm_area_struct *, unsigned long address,
1636#define FOLL_SPLIT 0x80 /* don't return transhuge pages, split them */ 1651#define FOLL_SPLIT 0x80 /* don't return transhuge pages, split them */
1637#define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */ 1652#define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */
1638#define FOLL_NUMA 0x200 /* force NUMA hinting page fault */ 1653#define FOLL_NUMA 0x200 /* force NUMA hinting page fault */
1654#define FOLL_MIGRATION 0x400 /* wait for page to replace migration entry */
1639 1655
1640typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr, 1656typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
1641 void *data); 1657 void *data);
@@ -1707,7 +1723,11 @@ int vmemmap_populate_basepages(struct page *start_page,
1707 unsigned long pages, int node); 1723 unsigned long pages, int node);
1708int vmemmap_populate(struct page *start_page, unsigned long pages, int node); 1724int vmemmap_populate(struct page *start_page, unsigned long pages, int node);
1709void vmemmap_populate_print_last(void); 1725void vmemmap_populate_print_last(void);
1710 1726#ifdef CONFIG_MEMORY_HOTPLUG
1727void vmemmap_free(struct page *memmap, unsigned long nr_pages);
1728#endif
1729void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
1730 unsigned long size);
1711 1731
1712enum mf_flags { 1732enum mf_flags {
1713 MF_COUNT_INCREASED = 1 << 0, 1733 MF_COUNT_INCREASED = 1 << 0,
@@ -1720,7 +1740,7 @@ extern int unpoison_memory(unsigned long pfn);
1720extern int sysctl_memory_failure_early_kill; 1740extern int sysctl_memory_failure_early_kill;
1721extern int sysctl_memory_failure_recovery; 1741extern int sysctl_memory_failure_recovery;
1722extern void shake_page(struct page *p, int access); 1742extern void shake_page(struct page *p, int access);
1723extern atomic_long_t mce_bad_pages; 1743extern atomic_long_t num_poisoned_pages;
1724extern int soft_offline_page(struct page *page, int flags); 1744extern int soft_offline_page(struct page *page, int flags);
1725 1745
1726extern void dump_page(struct page *page); 1746extern void dump_page(struct page *page);
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index f8f5162a3571..ace9a5f01c64 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -12,6 +12,7 @@
12#include <linux/cpumask.h> 12#include <linux/cpumask.h>
13#include <linux/page-debug-flags.h> 13#include <linux/page-debug-flags.h>
14#include <linux/uprobes.h> 14#include <linux/uprobes.h>
15#include <linux/page-flags-layout.h>
15#include <asm/page.h> 16#include <asm/page.h>
16#include <asm/mmu.h> 17#include <asm/mmu.h>
17 18
@@ -173,7 +174,7 @@ struct page {
173 void *shadow; 174 void *shadow;
174#endif 175#endif
175 176
176#ifdef CONFIG_NUMA_BALANCING 177#ifdef LAST_NID_NOT_IN_PAGE_FLAGS
177 int _last_nid; 178 int _last_nid;
178#endif 179#endif
179} 180}
@@ -414,9 +415,9 @@ struct mm_struct {
414#endif 415#endif
415#ifdef CONFIG_NUMA_BALANCING 416#ifdef CONFIG_NUMA_BALANCING
416 /* 417 /*
417 * numa_next_scan is the next time when the PTEs will me marked 418 * numa_next_scan is the next time that the PTEs will be marked
418 * pte_numa to gather statistics and migrate pages to new nodes 419 * pte_numa. NUMA hinting faults will gather statistics and migrate
419 * if necessary 420 * pages to new nodes if necessary.
420 */ 421 */
421 unsigned long numa_next_scan; 422 unsigned long numa_next_scan;
422 423
diff --git a/include/linux/mman.h b/include/linux/mman.h
index 9aa863da287f..61c7a87e5d2b 100644
--- a/include/linux/mman.h
+++ b/include/linux/mman.h
@@ -79,6 +79,8 @@ calc_vm_flag_bits(unsigned long flags)
79{ 79{
80 return _calc_vm_trans(flags, MAP_GROWSDOWN, VM_GROWSDOWN ) | 80 return _calc_vm_trans(flags, MAP_GROWSDOWN, VM_GROWSDOWN ) |
81 _calc_vm_trans(flags, MAP_DENYWRITE, VM_DENYWRITE ) | 81 _calc_vm_trans(flags, MAP_DENYWRITE, VM_DENYWRITE ) |
82 _calc_vm_trans(flags, MAP_LOCKED, VM_LOCKED ); 82 ((flags & MAP_LOCKED) ? (VM_LOCKED | VM_POPULATE) : 0) |
83 (((flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE) ?
84 VM_POPULATE : 0);
83} 85}
84#endif /* _LINUX_MMAN_H */ 86#endif /* _LINUX_MMAN_H */
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 73b64a38b984..ede274957e05 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -15,7 +15,7 @@
15#include <linux/seqlock.h> 15#include <linux/seqlock.h>
16#include <linux/nodemask.h> 16#include <linux/nodemask.h>
17#include <linux/pageblock-flags.h> 17#include <linux/pageblock-flags.h>
18#include <generated/bounds.h> 18#include <linux/page-flags-layout.h>
19#include <linux/atomic.h> 19#include <linux/atomic.h>
20#include <asm/page.h> 20#include <asm/page.h>
21 21
@@ -57,7 +57,9 @@ enum {
57 */ 57 */
58 MIGRATE_CMA, 58 MIGRATE_CMA,
59#endif 59#endif
60#ifdef CONFIG_MEMORY_ISOLATION
60 MIGRATE_ISOLATE, /* can't allocate from here */ 61 MIGRATE_ISOLATE, /* can't allocate from here */
62#endif
61 MIGRATE_TYPES 63 MIGRATE_TYPES
62}; 64};
63 65
@@ -308,24 +310,6 @@ enum zone_type {
308 310
309#ifndef __GENERATING_BOUNDS_H 311#ifndef __GENERATING_BOUNDS_H
310 312
311/*
312 * When a memory allocation must conform to specific limitations (such
313 * as being suitable for DMA) the caller will pass in hints to the
314 * allocator in the gfp_mask, in the zone modifier bits. These bits
315 * are used to select a priority ordered list of memory zones which
316 * match the requested limits. See gfp_zone() in include/linux/gfp.h
317 */
318
319#if MAX_NR_ZONES < 2
320#define ZONES_SHIFT 0
321#elif MAX_NR_ZONES <= 2
322#define ZONES_SHIFT 1
323#elif MAX_NR_ZONES <= 4
324#define ZONES_SHIFT 2
325#else
326#error ZONES_SHIFT -- too many zones configured adjust calculation
327#endif
328
329struct zone { 313struct zone {
330 /* Fields commonly accessed by the page allocator */ 314 /* Fields commonly accessed by the page allocator */
331 315
@@ -543,6 +527,26 @@ static inline int zone_is_oom_locked(const struct zone *zone)
543 return test_bit(ZONE_OOM_LOCKED, &zone->flags); 527 return test_bit(ZONE_OOM_LOCKED, &zone->flags);
544} 528}
545 529
530static inline unsigned zone_end_pfn(const struct zone *zone)
531{
532 return zone->zone_start_pfn + zone->spanned_pages;
533}
534
535static inline bool zone_spans_pfn(const struct zone *zone, unsigned long pfn)
536{
537 return zone->zone_start_pfn <= pfn && pfn < zone_end_pfn(zone);
538}
539
540static inline bool zone_is_initialized(struct zone *zone)
541{
542 return !!zone->wait_table;
543}
544
545static inline bool zone_is_empty(struct zone *zone)
546{
547 return zone->spanned_pages == 0;
548}
549
546/* 550/*
547 * The "priority" of VM scanning is how much of the queues we will scan in one 551 * The "priority" of VM scanning is how much of the queues we will scan in one
548 * go. A value of 12 for DEF_PRIORITY implies that we will scan 1/4096th of the 552 * go. A value of 12 for DEF_PRIORITY implies that we will scan 1/4096th of the
@@ -752,11 +756,17 @@ typedef struct pglist_data {
752#define nid_page_nr(nid, pagenr) pgdat_page_nr(NODE_DATA(nid),(pagenr)) 756#define nid_page_nr(nid, pagenr) pgdat_page_nr(NODE_DATA(nid),(pagenr))
753 757
754#define node_start_pfn(nid) (NODE_DATA(nid)->node_start_pfn) 758#define node_start_pfn(nid) (NODE_DATA(nid)->node_start_pfn)
759#define node_end_pfn(nid) pgdat_end_pfn(NODE_DATA(nid))
755 760
756#define node_end_pfn(nid) ({\ 761static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat)
757 pg_data_t *__pgdat = NODE_DATA(nid);\ 762{
758 __pgdat->node_start_pfn + __pgdat->node_spanned_pages;\ 763 return pgdat->node_start_pfn + pgdat->node_spanned_pages;
759}) 764}
765
766static inline bool pgdat_is_empty(pg_data_t *pgdat)
767{
768 return !pgdat->node_start_pfn && !pgdat->node_spanned_pages;
769}
760 770
761#include <linux/memory_hotplug.h> 771#include <linux/memory_hotplug.h>
762 772
@@ -1053,8 +1063,6 @@ static inline unsigned long early_pfn_to_nid(unsigned long pfn)
1053 * PA_SECTION_SHIFT physical address to/from section number 1063 * PA_SECTION_SHIFT physical address to/from section number
1054 * PFN_SECTION_SHIFT pfn to/from section number 1064 * PFN_SECTION_SHIFT pfn to/from section number
1055 */ 1065 */
1056#define SECTIONS_SHIFT (MAX_PHYSMEM_BITS - SECTION_SIZE_BITS)
1057
1058#define PA_SECTION_SHIFT (SECTION_SIZE_BITS) 1066#define PA_SECTION_SHIFT (SECTION_SIZE_BITS)
1059#define PFN_SECTION_SHIFT (SECTION_SIZE_BITS - PAGE_SHIFT) 1067#define PFN_SECTION_SHIFT (SECTION_SIZE_BITS - PAGE_SHIFT)
1060 1068
diff --git a/include/linux/page-flags-layout.h b/include/linux/page-flags-layout.h
new file mode 100644
index 000000000000..93506a114034
--- /dev/null
+++ b/include/linux/page-flags-layout.h
@@ -0,0 +1,88 @@
1#ifndef PAGE_FLAGS_LAYOUT_H
2#define PAGE_FLAGS_LAYOUT_H
3
4#include <linux/numa.h>
5#include <generated/bounds.h>
6
7/*
8 * When a memory allocation must conform to specific limitations (such
9 * as being suitable for DMA) the caller will pass in hints to the
10 * allocator in the gfp_mask, in the zone modifier bits. These bits
11 * are used to select a priority ordered list of memory zones which
12 * match the requested limits. See gfp_zone() in include/linux/gfp.h
13 */
14#if MAX_NR_ZONES < 2
15#define ZONES_SHIFT 0
16#elif MAX_NR_ZONES <= 2
17#define ZONES_SHIFT 1
18#elif MAX_NR_ZONES <= 4
19#define ZONES_SHIFT 2
20#else
21#error ZONES_SHIFT -- too many zones configured adjust calculation
22#endif
23
24#ifdef CONFIG_SPARSEMEM
25#include <asm/sparsemem.h>
26
27/* SECTION_SHIFT #bits space required to store a section # */
28#define SECTIONS_SHIFT (MAX_PHYSMEM_BITS - SECTION_SIZE_BITS)
29
30#endif /* CONFIG_SPARSEMEM */
31
32/*
33 * page->flags layout:
34 *
35 * There are five possibilities for how page->flags get laid out. The first
36 * pair is for the normal case without sparsemem. The second pair is for
37 * sparsemem when there is plenty of space for node and section information.
38 * The last is when there is insufficient space in page->flags and a separate
39 * lookup is necessary.
40 *
41 * No sparsemem or sparsemem vmemmap: | NODE | ZONE | ... | FLAGS |
42 * " plus space for last_nid: | NODE | ZONE | LAST_NID ... | FLAGS |
43 * classic sparse with space for node:| SECTION | NODE | ZONE | ... | FLAGS |
44 * " plus space for last_nid: | SECTION | NODE | ZONE | LAST_NID ... | FLAGS |
45 * classic sparse no space for node: | SECTION | ZONE | ... | FLAGS |
46 */
47#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
48#define SECTIONS_WIDTH SECTIONS_SHIFT
49#else
50#define SECTIONS_WIDTH 0
51#endif
52
53#define ZONES_WIDTH ZONES_SHIFT
54
55#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS
56#define NODES_WIDTH NODES_SHIFT
57#else
58#ifdef CONFIG_SPARSEMEM_VMEMMAP
59#error "Vmemmap: No space for nodes field in page flags"
60#endif
61#define NODES_WIDTH 0
62#endif
63
64#ifdef CONFIG_NUMA_BALANCING
65#define LAST_NID_SHIFT NODES_SHIFT
66#else
67#define LAST_NID_SHIFT 0
68#endif
69
70#if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT+LAST_NID_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS
71#define LAST_NID_WIDTH LAST_NID_SHIFT
72#else
73#define LAST_NID_WIDTH 0
74#endif
75
76/*
77 * We are going to use the flags for the page to node mapping if its in
78 * there. This includes the case where there is no node, so it is implicit.
79 */
80#if !(NODES_WIDTH > 0 || NODES_SHIFT == 0)
81#define NODE_NOT_IN_PAGE_FLAGS
82#endif
83
84#if defined(CONFIG_NUMA_BALANCING) && LAST_NID_WIDTH == 0
85#define LAST_NID_NOT_IN_PAGE_FLAGS
86#endif
87
88#endif /* _LINUX_PAGE_FLAGS_LAYOUT */
diff --git a/include/linux/page-isolation.h b/include/linux/page-isolation.h
index a92061e08d48..3fff8e774067 100644
--- a/include/linux/page-isolation.h
+++ b/include/linux/page-isolation.h
@@ -1,6 +1,25 @@
1#ifndef __LINUX_PAGEISOLATION_H 1#ifndef __LINUX_PAGEISOLATION_H
2#define __LINUX_PAGEISOLATION_H 2#define __LINUX_PAGEISOLATION_H
3 3
4#ifdef CONFIG_MEMORY_ISOLATION
5static inline bool is_migrate_isolate_page(struct page *page)
6{
7 return get_pageblock_migratetype(page) == MIGRATE_ISOLATE;
8}
9static inline bool is_migrate_isolate(int migratetype)
10{
11 return migratetype == MIGRATE_ISOLATE;
12}
13#else
14static inline bool is_migrate_isolate_page(struct page *page)
15{
16 return false;
17}
18static inline bool is_migrate_isolate(int migratetype)
19{
20 return false;
21}
22#endif
4 23
5bool has_unmovable_pages(struct zone *zone, struct page *page, int count, 24bool has_unmovable_pages(struct zone *zone, struct page *page, int count,
6 bool skip_hwpoisoned_pages); 25 bool skip_hwpoisoned_pages);
diff --git a/include/linux/pm.h b/include/linux/pm.h
index 97bcf23e045a..e5d7230332a4 100644
--- a/include/linux/pm.h
+++ b/include/linux/pm.h
@@ -537,6 +537,7 @@ struct dev_pm_info {
537 unsigned int irq_safe:1; 537 unsigned int irq_safe:1;
538 unsigned int use_autosuspend:1; 538 unsigned int use_autosuspend:1;
539 unsigned int timer_autosuspends:1; 539 unsigned int timer_autosuspends:1;
540 unsigned int memalloc_noio:1;
540 enum rpm_request request; 541 enum rpm_request request;
541 enum rpm_status runtime_status; 542 enum rpm_status runtime_status;
542 int runtime_error; 543 int runtime_error;
diff --git a/include/linux/pm_runtime.h b/include/linux/pm_runtime.h
index c785c215abfc..7d7e09efff9b 100644
--- a/include/linux/pm_runtime.h
+++ b/include/linux/pm_runtime.h
@@ -47,6 +47,7 @@ extern void pm_runtime_set_autosuspend_delay(struct device *dev, int delay);
47extern unsigned long pm_runtime_autosuspend_expiration(struct device *dev); 47extern unsigned long pm_runtime_autosuspend_expiration(struct device *dev);
48extern void pm_runtime_update_max_time_suspended(struct device *dev, 48extern void pm_runtime_update_max_time_suspended(struct device *dev,
49 s64 delta_ns); 49 s64 delta_ns);
50extern void pm_runtime_set_memalloc_noio(struct device *dev, bool enable);
50 51
51static inline bool pm_children_suspended(struct device *dev) 52static inline bool pm_children_suspended(struct device *dev)
52{ 53{
@@ -156,6 +157,8 @@ static inline void pm_runtime_set_autosuspend_delay(struct device *dev,
156 int delay) {} 157 int delay) {}
157static inline unsigned long pm_runtime_autosuspend_expiration( 158static inline unsigned long pm_runtime_autosuspend_expiration(
158 struct device *dev) { return 0; } 159 struct device *dev) { return 0; }
160static inline void pm_runtime_set_memalloc_noio(struct device *dev,
161 bool enable){}
159 162
160#endif /* !CONFIG_PM_RUNTIME */ 163#endif /* !CONFIG_PM_RUNTIME */
161 164
diff --git a/include/linux/rmap.h b/include/linux/rmap.h
index c20635c527a9..6dacb93a6d94 100644
--- a/include/linux/rmap.h
+++ b/include/linux/rmap.h
@@ -123,7 +123,7 @@ static inline void anon_vma_lock_write(struct anon_vma *anon_vma)
123 down_write(&anon_vma->root->rwsem); 123 down_write(&anon_vma->root->rwsem);
124} 124}
125 125
126static inline void anon_vma_unlock(struct anon_vma *anon_vma) 126static inline void anon_vma_unlock_write(struct anon_vma *anon_vma)
127{ 127{
128 up_write(&anon_vma->root->rwsem); 128 up_write(&anon_vma->root->rwsem);
129} 129}
diff --git a/include/linux/sched.h b/include/linux/sched.h
index e4112aad2964..c2182b53dace 100644
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -51,6 +51,7 @@ struct sched_param {
51#include <linux/cred.h> 51#include <linux/cred.h>
52#include <linux/llist.h> 52#include <linux/llist.h>
53#include <linux/uidgid.h> 53#include <linux/uidgid.h>
54#include <linux/gfp.h>
54 55
55#include <asm/processor.h> 56#include <asm/processor.h>
56 57
@@ -1791,6 +1792,7 @@ extern void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut,
1791#define PF_FROZEN 0x00010000 /* frozen for system suspend */ 1792#define PF_FROZEN 0x00010000 /* frozen for system suspend */
1792#define PF_FSTRANS 0x00020000 /* inside a filesystem transaction */ 1793#define PF_FSTRANS 0x00020000 /* inside a filesystem transaction */
1793#define PF_KSWAPD 0x00040000 /* I am kswapd */ 1794#define PF_KSWAPD 0x00040000 /* I am kswapd */
1795#define PF_MEMALLOC_NOIO 0x00080000 /* Allocating memory without IO involved */
1794#define PF_LESS_THROTTLE 0x00100000 /* Throttle me less: I clean memory */ 1796#define PF_LESS_THROTTLE 0x00100000 /* Throttle me less: I clean memory */
1795#define PF_KTHREAD 0x00200000 /* I am a kernel thread */ 1797#define PF_KTHREAD 0x00200000 /* I am a kernel thread */
1796#define PF_RANDOMIZE 0x00400000 /* randomize virtual address space */ 1798#define PF_RANDOMIZE 0x00400000 /* randomize virtual address space */
@@ -1828,6 +1830,26 @@ extern void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut,
1828#define tsk_used_math(p) ((p)->flags & PF_USED_MATH) 1830#define tsk_used_math(p) ((p)->flags & PF_USED_MATH)
1829#define used_math() tsk_used_math(current) 1831#define used_math() tsk_used_math(current)
1830 1832
1833/* __GFP_IO isn't allowed if PF_MEMALLOC_NOIO is set in current->flags */
1834static inline gfp_t memalloc_noio_flags(gfp_t flags)
1835{
1836 if (unlikely(current->flags & PF_MEMALLOC_NOIO))
1837 flags &= ~__GFP_IO;
1838 return flags;
1839}
1840
1841static inline unsigned int memalloc_noio_save(void)
1842{
1843 unsigned int flags = current->flags & PF_MEMALLOC_NOIO;
1844 current->flags |= PF_MEMALLOC_NOIO;
1845 return flags;
1846}
1847
1848static inline void memalloc_noio_restore(unsigned int flags)
1849{
1850 current->flags = (current->flags & ~PF_MEMALLOC_NOIO) | flags;
1851}
1852
1831/* 1853/*
1832 * task->jobctl flags 1854 * task->jobctl flags
1833 */ 1855 */
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 68df9c17fbbb..2818a123f3ea 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -8,7 +8,7 @@
8#include <linux/memcontrol.h> 8#include <linux/memcontrol.h>
9#include <linux/sched.h> 9#include <linux/sched.h>
10#include <linux/node.h> 10#include <linux/node.h>
11 11#include <linux/fs.h>
12#include <linux/atomic.h> 12#include <linux/atomic.h>
13#include <asm/page.h> 13#include <asm/page.h>
14 14
@@ -156,7 +156,7 @@ enum {
156 SWP_SCANNING = (1 << 8), /* refcount in scan_swap_map */ 156 SWP_SCANNING = (1 << 8), /* refcount in scan_swap_map */
157}; 157};
158 158
159#define SWAP_CLUSTER_MAX 32 159#define SWAP_CLUSTER_MAX 32UL
160#define COMPACT_CLUSTER_MAX SWAP_CLUSTER_MAX 160#define COMPACT_CLUSTER_MAX SWAP_CLUSTER_MAX
161 161
162/* 162/*
@@ -202,6 +202,18 @@ struct swap_info_struct {
202 unsigned long *frontswap_map; /* frontswap in-use, one bit per page */ 202 unsigned long *frontswap_map; /* frontswap in-use, one bit per page */
203 atomic_t frontswap_pages; /* frontswap pages in-use counter */ 203 atomic_t frontswap_pages; /* frontswap pages in-use counter */
204#endif 204#endif
205 spinlock_t lock; /*
206 * protect map scan related fields like
207 * swap_map, lowest_bit, highest_bit,
208 * inuse_pages, cluster_next,
209 * cluster_nr, lowest_alloc and
210 * highest_alloc. other fields are only
211 * changed at swapon/swapoff, so are
212 * protected by swap_lock. changing
213 * flags need hold this lock and
214 * swap_lock. If both locks need hold,
215 * hold swap_lock first.
216 */
205}; 217};
206 218
207struct swap_list_t { 219struct swap_list_t {
@@ -209,15 +221,12 @@ struct swap_list_t {
209 int next; /* swapfile to be used next */ 221 int next; /* swapfile to be used next */
210}; 222};
211 223
212/* Swap 50% full? Release swapcache more aggressively.. */
213#define vm_swap_full() (nr_swap_pages*2 < total_swap_pages)
214
215/* linux/mm/page_alloc.c */ 224/* linux/mm/page_alloc.c */
216extern unsigned long totalram_pages; 225extern unsigned long totalram_pages;
217extern unsigned long totalreserve_pages; 226extern unsigned long totalreserve_pages;
218extern unsigned long dirty_balance_reserve; 227extern unsigned long dirty_balance_reserve;
219extern unsigned int nr_free_buffer_pages(void); 228extern unsigned long nr_free_buffer_pages(void);
220extern unsigned int nr_free_pagecache_pages(void); 229extern unsigned long nr_free_pagecache_pages(void);
221 230
222/* Definition of global_page_state not available yet */ 231/* Definition of global_page_state not available yet */
223#define nr_free_pages() global_page_state(NR_FREE_PAGES) 232#define nr_free_pages() global_page_state(NR_FREE_PAGES)
@@ -266,7 +275,7 @@ extern unsigned long mem_cgroup_shrink_node_zone(struct mem_cgroup *mem,
266extern unsigned long shrink_all_memory(unsigned long nr_pages); 275extern unsigned long shrink_all_memory(unsigned long nr_pages);
267extern int vm_swappiness; 276extern int vm_swappiness;
268extern int remove_mapping(struct address_space *mapping, struct page *page); 277extern int remove_mapping(struct address_space *mapping, struct page *page);
269extern long vm_total_pages; 278extern unsigned long vm_total_pages;
270 279
271#ifdef CONFIG_NUMA 280#ifdef CONFIG_NUMA
272extern int zone_reclaim_mode; 281extern int zone_reclaim_mode;
@@ -330,8 +339,9 @@ int generic_swapfile_activate(struct swap_info_struct *, struct file *,
330 sector_t *); 339 sector_t *);
331 340
332/* linux/mm/swap_state.c */ 341/* linux/mm/swap_state.c */
333extern struct address_space swapper_space; 342extern struct address_space swapper_spaces[];
334#define total_swapcache_pages swapper_space.nrpages 343#define swap_address_space(entry) (&swapper_spaces[swp_type(entry)])
344extern unsigned long total_swapcache_pages(void);
335extern void show_swap_cache_info(void); 345extern void show_swap_cache_info(void);
336extern int add_to_swap(struct page *); 346extern int add_to_swap(struct page *);
337extern int add_to_swap_cache(struct page *, swp_entry_t, gfp_t); 347extern int add_to_swap_cache(struct page *, swp_entry_t, gfp_t);
@@ -346,8 +356,20 @@ extern struct page *swapin_readahead(swp_entry_t, gfp_t,
346 struct vm_area_struct *vma, unsigned long addr); 356 struct vm_area_struct *vma, unsigned long addr);
347 357
348/* linux/mm/swapfile.c */ 358/* linux/mm/swapfile.c */
349extern long nr_swap_pages; 359extern atomic_long_t nr_swap_pages;
350extern long total_swap_pages; 360extern long total_swap_pages;
361
362/* Swap 50% full? Release swapcache more aggressively.. */
363static inline bool vm_swap_full(void)
364{
365 return atomic_long_read(&nr_swap_pages) * 2 < total_swap_pages;
366}
367
368static inline long get_nr_swap_pages(void)
369{
370 return atomic_long_read(&nr_swap_pages);
371}
372
351extern void si_swapinfo(struct sysinfo *); 373extern void si_swapinfo(struct sysinfo *);
352extern swp_entry_t get_swap_page(void); 374extern swp_entry_t get_swap_page(void);
353extern swp_entry_t get_swap_page_of_type(int); 375extern swp_entry_t get_swap_page_of_type(int);
@@ -380,9 +402,10 @@ mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout)
380 402
381#else /* CONFIG_SWAP */ 403#else /* CONFIG_SWAP */
382 404
383#define nr_swap_pages 0L 405#define get_nr_swap_pages() 0L
384#define total_swap_pages 0L 406#define total_swap_pages 0L
385#define total_swapcache_pages 0UL 407#define total_swapcache_pages() 0UL
408#define vm_swap_full() 0
386 409
387#define si_swapinfo(val) \ 410#define si_swapinfo(val) \
388 do { (val)->freeswap = (val)->totalswap = 0; } while (0) 411 do { (val)->freeswap = (val)->totalswap = 0; } while (0)
diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
index fce0a2799d43..bd6cf61142be 100644
--- a/include/linux/vm_event_item.h
+++ b/include/linux/vm_event_item.h
@@ -36,7 +36,6 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
36#endif 36#endif
37 PGINODESTEAL, SLABS_SCANNED, KSWAPD_INODESTEAL, 37 PGINODESTEAL, SLABS_SCANNED, KSWAPD_INODESTEAL,
38 KSWAPD_LOW_WMARK_HIT_QUICKLY, KSWAPD_HIGH_WMARK_HIT_QUICKLY, 38 KSWAPD_LOW_WMARK_HIT_QUICKLY, KSWAPD_HIGH_WMARK_HIT_QUICKLY,
39 KSWAPD_SKIP_CONGESTION_WAIT,
40 PAGEOUTRUN, ALLOCSTALL, PGROTATED, 39 PAGEOUTRUN, ALLOCSTALL, PGROTATED,
41#ifdef CONFIG_NUMA_BALANCING 40#ifdef CONFIG_NUMA_BALANCING
42 NUMA_PTE_UPDATES, 41 NUMA_PTE_UPDATES,
diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h
index a13291f7da88..5fd71a7d0dfd 100644
--- a/include/linux/vmstat.h
+++ b/include/linux/vmstat.h
@@ -85,7 +85,7 @@ static inline void vm_events_fold_cpu(int cpu)
85#define count_vm_numa_events(x, y) count_vm_events(x, y) 85#define count_vm_numa_events(x, y) count_vm_events(x, y)
86#else 86#else
87#define count_vm_numa_event(x) do {} while (0) 87#define count_vm_numa_event(x) do {} while (0)
88#define count_vm_numa_events(x, y) do {} while (0) 88#define count_vm_numa_events(x, y) do { (void)(y); } while (0)
89#endif /* CONFIG_NUMA_BALANCING */ 89#endif /* CONFIG_NUMA_BALANCING */
90 90
91#define __count_zone_vm_events(item, zone, delta) \ 91#define __count_zone_vm_events(item, zone, delta) \