diff options
Diffstat (limited to 'include/linux')
-rw-r--r-- | include/linux/acpi.h | 8 | ||||
-rw-r--r-- | include/linux/bootmem.h | 1 | ||||
-rw-r--r-- | include/linux/compaction.h | 5 | ||||
-rw-r--r-- | include/linux/firmware-map.h | 6 | ||||
-rw-r--r-- | include/linux/highmem.h | 6 | ||||
-rw-r--r-- | include/linux/huge_mm.h | 2 | ||||
-rw-r--r-- | include/linux/hugetlb.h | 6 | ||||
-rw-r--r-- | include/linux/ksm.h | 18 | ||||
-rw-r--r-- | include/linux/memblock.h | 2 | ||||
-rw-r--r-- | include/linux/memcontrol.h | 7 | ||||
-rw-r--r-- | include/linux/memory_hotplug.h | 20 | ||||
-rw-r--r-- | include/linux/migrate.h | 14 | ||||
-rw-r--r-- | include/linux/mm.h | 178 | ||||
-rw-r--r-- | include/linux/mm_types.h | 9 | ||||
-rw-r--r-- | include/linux/mman.h | 4 | ||||
-rw-r--r-- | include/linux/mmzone.h | 58 | ||||
-rw-r--r-- | include/linux/page-flags-layout.h | 88 | ||||
-rw-r--r-- | include/linux/page-isolation.h | 19 | ||||
-rw-r--r-- | include/linux/pm.h | 1 | ||||
-rw-r--r-- | include/linux/pm_runtime.h | 3 | ||||
-rw-r--r-- | include/linux/rmap.h | 2 | ||||
-rw-r--r-- | include/linux/sched.h | 22 | ||||
-rw-r--r-- | include/linux/swap.h | 49 | ||||
-rw-r--r-- | include/linux/vm_event_item.h | 1 | ||||
-rw-r--r-- | include/linux/vmstat.h | 2 |
25 files changed, 356 insertions, 175 deletions
diff --git a/include/linux/acpi.h b/include/linux/acpi.h index bcbdd7484e58..f46cfd73a553 100644 --- a/include/linux/acpi.h +++ b/include/linux/acpi.h | |||
@@ -485,6 +485,14 @@ static inline bool acpi_driver_match_device(struct device *dev, | |||
485 | 485 | ||
486 | #endif /* !CONFIG_ACPI */ | 486 | #endif /* !CONFIG_ACPI */ |
487 | 487 | ||
488 | #ifdef CONFIG_ACPI_NUMA | ||
489 | void __init early_parse_srat(void); | ||
490 | #else | ||
491 | static inline void early_parse_srat(void) | ||
492 | { | ||
493 | } | ||
494 | #endif | ||
495 | |||
488 | #ifdef CONFIG_ACPI | 496 | #ifdef CONFIG_ACPI |
489 | void acpi_os_set_prepare_sleep(int (*func)(u8 sleep_state, | 497 | void acpi_os_set_prepare_sleep(int (*func)(u8 sleep_state, |
490 | u32 pm1a_ctrl, u32 pm1b_ctrl)); | 498 | u32 pm1a_ctrl, u32 pm1b_ctrl)); |
diff --git a/include/linux/bootmem.h b/include/linux/bootmem.h index 3cd16ba82f15..cdc3bab01832 100644 --- a/include/linux/bootmem.h +++ b/include/linux/bootmem.h | |||
@@ -53,6 +53,7 @@ extern void free_bootmem_node(pg_data_t *pgdat, | |||
53 | unsigned long size); | 53 | unsigned long size); |
54 | extern void free_bootmem(unsigned long physaddr, unsigned long size); | 54 | extern void free_bootmem(unsigned long physaddr, unsigned long size); |
55 | extern void free_bootmem_late(unsigned long physaddr, unsigned long size); | 55 | extern void free_bootmem_late(unsigned long physaddr, unsigned long size); |
56 | extern void __free_pages_bootmem(struct page *page, unsigned int order); | ||
56 | 57 | ||
57 | /* | 58 | /* |
58 | * Flags for reserve_bootmem (also if CONFIG_HAVE_ARCH_BOOTMEM_NODE, | 59 | * Flags for reserve_bootmem (also if CONFIG_HAVE_ARCH_BOOTMEM_NODE, |
diff --git a/include/linux/compaction.h b/include/linux/compaction.h index cc7bddeaf553..091d72e70d8a 100644 --- a/include/linux/compaction.h +++ b/include/linux/compaction.h | |||
@@ -23,7 +23,7 @@ extern int fragmentation_index(struct zone *zone, unsigned int order); | |||
23 | extern unsigned long try_to_compact_pages(struct zonelist *zonelist, | 23 | extern unsigned long try_to_compact_pages(struct zonelist *zonelist, |
24 | int order, gfp_t gfp_mask, nodemask_t *mask, | 24 | int order, gfp_t gfp_mask, nodemask_t *mask, |
25 | bool sync, bool *contended); | 25 | bool sync, bool *contended); |
26 | extern int compact_pgdat(pg_data_t *pgdat, int order); | 26 | extern void compact_pgdat(pg_data_t *pgdat, int order); |
27 | extern void reset_isolation_suitable(pg_data_t *pgdat); | 27 | extern void reset_isolation_suitable(pg_data_t *pgdat); |
28 | extern unsigned long compaction_suitable(struct zone *zone, int order); | 28 | extern unsigned long compaction_suitable(struct zone *zone, int order); |
29 | 29 | ||
@@ -80,9 +80,8 @@ static inline unsigned long try_to_compact_pages(struct zonelist *zonelist, | |||
80 | return COMPACT_CONTINUE; | 80 | return COMPACT_CONTINUE; |
81 | } | 81 | } |
82 | 82 | ||
83 | static inline int compact_pgdat(pg_data_t *pgdat, int order) | 83 | static inline void compact_pgdat(pg_data_t *pgdat, int order) |
84 | { | 84 | { |
85 | return COMPACT_CONTINUE; | ||
86 | } | 85 | } |
87 | 86 | ||
88 | static inline void reset_isolation_suitable(pg_data_t *pgdat) | 87 | static inline void reset_isolation_suitable(pg_data_t *pgdat) |
diff --git a/include/linux/firmware-map.h b/include/linux/firmware-map.h index 43fe52fcef0f..71d4fa721db9 100644 --- a/include/linux/firmware-map.h +++ b/include/linux/firmware-map.h | |||
@@ -25,6 +25,7 @@ | |||
25 | 25 | ||
26 | int firmware_map_add_early(u64 start, u64 end, const char *type); | 26 | int firmware_map_add_early(u64 start, u64 end, const char *type); |
27 | int firmware_map_add_hotplug(u64 start, u64 end, const char *type); | 27 | int firmware_map_add_hotplug(u64 start, u64 end, const char *type); |
28 | int firmware_map_remove(u64 start, u64 end, const char *type); | ||
28 | 29 | ||
29 | #else /* CONFIG_FIRMWARE_MEMMAP */ | 30 | #else /* CONFIG_FIRMWARE_MEMMAP */ |
30 | 31 | ||
@@ -38,6 +39,11 @@ static inline int firmware_map_add_hotplug(u64 start, u64 end, const char *type) | |||
38 | return 0; | 39 | return 0; |
39 | } | 40 | } |
40 | 41 | ||
42 | static inline int firmware_map_remove(u64 start, u64 end, const char *type) | ||
43 | { | ||
44 | return 0; | ||
45 | } | ||
46 | |||
41 | #endif /* CONFIG_FIRMWARE_MEMMAP */ | 47 | #endif /* CONFIG_FIRMWARE_MEMMAP */ |
42 | 48 | ||
43 | #endif /* _LINUX_FIRMWARE_MAP_H */ | 49 | #endif /* _LINUX_FIRMWARE_MAP_H */ |
diff --git a/include/linux/highmem.h b/include/linux/highmem.h index ef788b5b4a35..7fb31da45d03 100644 --- a/include/linux/highmem.h +++ b/include/linux/highmem.h | |||
@@ -219,12 +219,6 @@ static inline void zero_user(struct page *page, | |||
219 | zero_user_segments(page, start, start + size, 0, 0); | 219 | zero_user_segments(page, start, start + size, 0, 0); |
220 | } | 220 | } |
221 | 221 | ||
222 | static inline void __deprecated memclear_highpage_flush(struct page *page, | ||
223 | unsigned int offset, unsigned int size) | ||
224 | { | ||
225 | zero_user(page, offset, size); | ||
226 | } | ||
227 | |||
228 | #ifndef __HAVE_ARCH_COPY_USER_HIGHPAGE | 222 | #ifndef __HAVE_ARCH_COPY_USER_HIGHPAGE |
229 | 223 | ||
230 | static inline void copy_user_highpage(struct page *to, struct page *from, | 224 | static inline void copy_user_highpage(struct page *to, struct page *from, |
diff --git a/include/linux/huge_mm.h b/include/linux/huge_mm.h index 1d76f8ca90f0..ee1c244a62a1 100644 --- a/include/linux/huge_mm.h +++ b/include/linux/huge_mm.h | |||
@@ -113,7 +113,7 @@ extern void __split_huge_page_pmd(struct vm_area_struct *vma, | |||
113 | do { \ | 113 | do { \ |
114 | pmd_t *____pmd = (__pmd); \ | 114 | pmd_t *____pmd = (__pmd); \ |
115 | anon_vma_lock_write(__anon_vma); \ | 115 | anon_vma_lock_write(__anon_vma); \ |
116 | anon_vma_unlock(__anon_vma); \ | 116 | anon_vma_unlock_write(__anon_vma); \ |
117 | BUG_ON(pmd_trans_splitting(*____pmd) || \ | 117 | BUG_ON(pmd_trans_splitting(*____pmd) || \ |
118 | pmd_trans_huge(*____pmd)); \ | 118 | pmd_trans_huge(*____pmd)); \ |
119 | } while (0) | 119 | } while (0) |
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h index 0c80d3f57a5b..eedc334fb6f5 100644 --- a/include/linux/hugetlb.h +++ b/include/linux/hugetlb.h | |||
@@ -43,9 +43,9 @@ int hugetlb_mempolicy_sysctl_handler(struct ctl_table *, int, | |||
43 | #endif | 43 | #endif |
44 | 44 | ||
45 | int copy_hugetlb_page_range(struct mm_struct *, struct mm_struct *, struct vm_area_struct *); | 45 | int copy_hugetlb_page_range(struct mm_struct *, struct mm_struct *, struct vm_area_struct *); |
46 | int follow_hugetlb_page(struct mm_struct *, struct vm_area_struct *, | 46 | long follow_hugetlb_page(struct mm_struct *, struct vm_area_struct *, |
47 | struct page **, struct vm_area_struct **, | 47 | struct page **, struct vm_area_struct **, |
48 | unsigned long *, int *, int, unsigned int flags); | 48 | unsigned long *, unsigned long *, long, unsigned int); |
49 | void unmap_hugepage_range(struct vm_area_struct *, | 49 | void unmap_hugepage_range(struct vm_area_struct *, |
50 | unsigned long, unsigned long, struct page *); | 50 | unsigned long, unsigned long, struct page *); |
51 | void __unmap_hugepage_range_final(struct mmu_gather *tlb, | 51 | void __unmap_hugepage_range_final(struct mmu_gather *tlb, |
diff --git a/include/linux/ksm.h b/include/linux/ksm.h index 3319a6967626..45c9b6a17bcb 100644 --- a/include/linux/ksm.h +++ b/include/linux/ksm.h | |||
@@ -16,9 +16,6 @@ | |||
16 | struct stable_node; | 16 | struct stable_node; |
17 | struct mem_cgroup; | 17 | struct mem_cgroup; |
18 | 18 | ||
19 | struct page *ksm_does_need_to_copy(struct page *page, | ||
20 | struct vm_area_struct *vma, unsigned long address); | ||
21 | |||
22 | #ifdef CONFIG_KSM | 19 | #ifdef CONFIG_KSM |
23 | int ksm_madvise(struct vm_area_struct *vma, unsigned long start, | 20 | int ksm_madvise(struct vm_area_struct *vma, unsigned long start, |
24 | unsigned long end, int advice, unsigned long *vm_flags); | 21 | unsigned long end, int advice, unsigned long *vm_flags); |
@@ -73,15 +70,8 @@ static inline void set_page_stable_node(struct page *page, | |||
73 | * We'd like to make this conditional on vma->vm_flags & VM_MERGEABLE, | 70 | * We'd like to make this conditional on vma->vm_flags & VM_MERGEABLE, |
74 | * but what if the vma was unmerged while the page was swapped out? | 71 | * but what if the vma was unmerged while the page was swapped out? |
75 | */ | 72 | */ |
76 | static inline int ksm_might_need_to_copy(struct page *page, | 73 | struct page *ksm_might_need_to_copy(struct page *page, |
77 | struct vm_area_struct *vma, unsigned long address) | 74 | struct vm_area_struct *vma, unsigned long address); |
78 | { | ||
79 | struct anon_vma *anon_vma = page_anon_vma(page); | ||
80 | |||
81 | return anon_vma && | ||
82 | (anon_vma->root != vma->anon_vma->root || | ||
83 | page->index != linear_page_index(vma, address)); | ||
84 | } | ||
85 | 75 | ||
86 | int page_referenced_ksm(struct page *page, | 76 | int page_referenced_ksm(struct page *page, |
87 | struct mem_cgroup *memcg, unsigned long *vm_flags); | 77 | struct mem_cgroup *memcg, unsigned long *vm_flags); |
@@ -113,10 +103,10 @@ static inline int ksm_madvise(struct vm_area_struct *vma, unsigned long start, | |||
113 | return 0; | 103 | return 0; |
114 | } | 104 | } |
115 | 105 | ||
116 | static inline int ksm_might_need_to_copy(struct page *page, | 106 | static inline struct page *ksm_might_need_to_copy(struct page *page, |
117 | struct vm_area_struct *vma, unsigned long address) | 107 | struct vm_area_struct *vma, unsigned long address) |
118 | { | 108 | { |
119 | return 0; | 109 | return page; |
120 | } | 110 | } |
121 | 111 | ||
122 | static inline int page_referenced_ksm(struct page *page, | 112 | static inline int page_referenced_ksm(struct page *page, |
diff --git a/include/linux/memblock.h b/include/linux/memblock.h index f388203db7e8..3e5ecb2d790e 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h | |||
@@ -42,6 +42,7 @@ struct memblock { | |||
42 | 42 | ||
43 | extern struct memblock memblock; | 43 | extern struct memblock memblock; |
44 | extern int memblock_debug; | 44 | extern int memblock_debug; |
45 | extern struct movablemem_map movablemem_map; | ||
45 | 46 | ||
46 | #define memblock_dbg(fmt, ...) \ | 47 | #define memblock_dbg(fmt, ...) \ |
47 | if (memblock_debug) printk(KERN_INFO pr_fmt(fmt), ##__VA_ARGS__) | 48 | if (memblock_debug) printk(KERN_INFO pr_fmt(fmt), ##__VA_ARGS__) |
@@ -60,6 +61,7 @@ int memblock_reserve(phys_addr_t base, phys_addr_t size); | |||
60 | void memblock_trim_memory(phys_addr_t align); | 61 | void memblock_trim_memory(phys_addr_t align); |
61 | 62 | ||
62 | #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP | 63 | #ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP |
64 | |||
63 | void __next_mem_pfn_range(int *idx, int nid, unsigned long *out_start_pfn, | 65 | void __next_mem_pfn_range(int *idx, int nid, unsigned long *out_start_pfn, |
64 | unsigned long *out_end_pfn, int *out_nid); | 66 | unsigned long *out_end_pfn, int *out_nid); |
65 | 67 | ||
diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 28bd5fa2ff2e..d6183f06d8c1 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h | |||
@@ -116,7 +116,6 @@ void mem_cgroup_iter_break(struct mem_cgroup *, struct mem_cgroup *); | |||
116 | * For memory reclaim. | 116 | * For memory reclaim. |
117 | */ | 117 | */ |
118 | int mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec); | 118 | int mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec); |
119 | int mem_cgroup_inactive_file_is_low(struct lruvec *lruvec); | ||
120 | int mem_cgroup_select_victim_node(struct mem_cgroup *memcg); | 119 | int mem_cgroup_select_victim_node(struct mem_cgroup *memcg); |
121 | unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list); | 120 | unsigned long mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list); |
122 | void mem_cgroup_update_lru_size(struct lruvec *, enum lru_list, int); | 121 | void mem_cgroup_update_lru_size(struct lruvec *, enum lru_list, int); |
@@ -321,12 +320,6 @@ mem_cgroup_inactive_anon_is_low(struct lruvec *lruvec) | |||
321 | return 1; | 320 | return 1; |
322 | } | 321 | } |
323 | 322 | ||
324 | static inline int | ||
325 | mem_cgroup_inactive_file_is_low(struct lruvec *lruvec) | ||
326 | { | ||
327 | return 1; | ||
328 | } | ||
329 | |||
330 | static inline unsigned long | 323 | static inline unsigned long |
331 | mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru) | 324 | mem_cgroup_get_lru_size(struct lruvec *lruvec, enum lru_list lru) |
332 | { | 325 | { |
diff --git a/include/linux/memory_hotplug.h b/include/linux/memory_hotplug.h index 4a45c4e50025..b6a3be7d47bf 100644 --- a/include/linux/memory_hotplug.h +++ b/include/linux/memory_hotplug.h | |||
@@ -96,6 +96,7 @@ extern void __online_page_free(struct page *page); | |||
96 | 96 | ||
97 | #ifdef CONFIG_MEMORY_HOTREMOVE | 97 | #ifdef CONFIG_MEMORY_HOTREMOVE |
98 | extern bool is_pageblock_removable_nolock(struct page *page); | 98 | extern bool is_pageblock_removable_nolock(struct page *page); |
99 | extern int arch_remove_memory(u64 start, u64 size); | ||
99 | #endif /* CONFIG_MEMORY_HOTREMOVE */ | 100 | #endif /* CONFIG_MEMORY_HOTREMOVE */ |
100 | 101 | ||
101 | /* reasonably generic interface to expand the physical pages in a zone */ | 102 | /* reasonably generic interface to expand the physical pages in a zone */ |
@@ -173,17 +174,16 @@ static inline void arch_refresh_nodedata(int nid, pg_data_t *pgdat) | |||
173 | #endif /* CONFIG_NUMA */ | 174 | #endif /* CONFIG_NUMA */ |
174 | #endif /* CONFIG_HAVE_ARCH_NODEDATA_EXTENSION */ | 175 | #endif /* CONFIG_HAVE_ARCH_NODEDATA_EXTENSION */ |
175 | 176 | ||
176 | #ifdef CONFIG_SPARSEMEM_VMEMMAP | 177 | #ifdef CONFIG_HAVE_BOOTMEM_INFO_NODE |
178 | extern void register_page_bootmem_info_node(struct pglist_data *pgdat); | ||
179 | #else | ||
177 | static inline void register_page_bootmem_info_node(struct pglist_data *pgdat) | 180 | static inline void register_page_bootmem_info_node(struct pglist_data *pgdat) |
178 | { | 181 | { |
179 | } | 182 | } |
180 | static inline void put_page_bootmem(struct page *page) | ||
181 | { | ||
182 | } | ||
183 | #else | ||
184 | extern void register_page_bootmem_info_node(struct pglist_data *pgdat); | ||
185 | extern void put_page_bootmem(struct page *page); | ||
186 | #endif | 183 | #endif |
184 | extern void put_page_bootmem(struct page *page); | ||
185 | extern void get_page_bootmem(unsigned long ingo, struct page *page, | ||
186 | unsigned long type); | ||
187 | 187 | ||
188 | /* | 188 | /* |
189 | * Lock for memory hotplug guarantees 1) all callbacks for memory hotplug | 189 | * Lock for memory hotplug guarantees 1) all callbacks for memory hotplug |
@@ -233,6 +233,7 @@ static inline void unlock_memory_hotplug(void) {} | |||
233 | #ifdef CONFIG_MEMORY_HOTREMOVE | 233 | #ifdef CONFIG_MEMORY_HOTREMOVE |
234 | 234 | ||
235 | extern int is_mem_section_removable(unsigned long pfn, unsigned long nr_pages); | 235 | extern int is_mem_section_removable(unsigned long pfn, unsigned long nr_pages); |
236 | extern void try_offline_node(int nid); | ||
236 | 237 | ||
237 | #else | 238 | #else |
238 | static inline int is_mem_section_removable(unsigned long pfn, | 239 | static inline int is_mem_section_removable(unsigned long pfn, |
@@ -240,6 +241,8 @@ static inline int is_mem_section_removable(unsigned long pfn, | |||
240 | { | 241 | { |
241 | return 0; | 242 | return 0; |
242 | } | 243 | } |
244 | |||
245 | static inline void try_offline_node(int nid) {} | ||
243 | #endif /* CONFIG_MEMORY_HOTREMOVE */ | 246 | #endif /* CONFIG_MEMORY_HOTREMOVE */ |
244 | 247 | ||
245 | extern int mem_online_node(int nid); | 248 | extern int mem_online_node(int nid); |
@@ -247,7 +250,8 @@ extern int add_memory(int nid, u64 start, u64 size); | |||
247 | extern int arch_add_memory(int nid, u64 start, u64 size); | 250 | extern int arch_add_memory(int nid, u64 start, u64 size); |
248 | extern int offline_pages(unsigned long start_pfn, unsigned long nr_pages); | 251 | extern int offline_pages(unsigned long start_pfn, unsigned long nr_pages); |
249 | extern int offline_memory_block(struct memory_block *mem); | 252 | extern int offline_memory_block(struct memory_block *mem); |
250 | extern int remove_memory(u64 start, u64 size); | 253 | extern bool is_memblock_offlined(struct memory_block *mem); |
254 | extern int remove_memory(int nid, u64 start, u64 size); | ||
251 | extern int sparse_add_one_section(struct zone *zone, unsigned long start_pfn, | 255 | extern int sparse_add_one_section(struct zone *zone, unsigned long start_pfn, |
252 | int nr_pages); | 256 | int nr_pages); |
253 | extern void sparse_remove_one_section(struct zone *zone, struct mem_section *ms); | 257 | extern void sparse_remove_one_section(struct zone *zone, struct mem_section *ms); |
diff --git a/include/linux/migrate.h b/include/linux/migrate.h index 1e9f627967a3..a405d3dc0f61 100644 --- a/include/linux/migrate.h +++ b/include/linux/migrate.h | |||
@@ -40,11 +40,9 @@ extern void putback_movable_pages(struct list_head *l); | |||
40 | extern int migrate_page(struct address_space *, | 40 | extern int migrate_page(struct address_space *, |
41 | struct page *, struct page *, enum migrate_mode); | 41 | struct page *, struct page *, enum migrate_mode); |
42 | extern int migrate_pages(struct list_head *l, new_page_t x, | 42 | extern int migrate_pages(struct list_head *l, new_page_t x, |
43 | unsigned long private, bool offlining, | 43 | unsigned long private, enum migrate_mode mode, int reason); |
44 | enum migrate_mode mode, int reason); | ||
45 | extern int migrate_huge_page(struct page *, new_page_t x, | 44 | extern int migrate_huge_page(struct page *, new_page_t x, |
46 | unsigned long private, bool offlining, | 45 | unsigned long private, enum migrate_mode mode); |
47 | enum migrate_mode mode); | ||
48 | 46 | ||
49 | extern int fail_migrate_page(struct address_space *, | 47 | extern int fail_migrate_page(struct address_space *, |
50 | struct page *, struct page *); | 48 | struct page *, struct page *); |
@@ -62,11 +60,11 @@ extern int migrate_huge_page_move_mapping(struct address_space *mapping, | |||
62 | static inline void putback_lru_pages(struct list_head *l) {} | 60 | static inline void putback_lru_pages(struct list_head *l) {} |
63 | static inline void putback_movable_pages(struct list_head *l) {} | 61 | static inline void putback_movable_pages(struct list_head *l) {} |
64 | static inline int migrate_pages(struct list_head *l, new_page_t x, | 62 | static inline int migrate_pages(struct list_head *l, new_page_t x, |
65 | unsigned long private, bool offlining, | 63 | unsigned long private, enum migrate_mode mode, int reason) |
66 | enum migrate_mode mode, int reason) { return -ENOSYS; } | 64 | { return -ENOSYS; } |
67 | static inline int migrate_huge_page(struct page *page, new_page_t x, | 65 | static inline int migrate_huge_page(struct page *page, new_page_t x, |
68 | unsigned long private, bool offlining, | 66 | unsigned long private, enum migrate_mode mode) |
69 | enum migrate_mode mode) { return -ENOSYS; } | 67 | { return -ENOSYS; } |
70 | 68 | ||
71 | static inline int migrate_prep(void) { return -ENOSYS; } | 69 | static inline int migrate_prep(void) { return -ENOSYS; } |
72 | static inline int migrate_prep_local(void) { return -ENOSYS; } | 70 | static inline int migrate_prep_local(void) { return -ENOSYS; } |
diff --git a/include/linux/mm.h b/include/linux/mm.h index 9d9dcc35d6a1..e7c3f9a0111a 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h | |||
@@ -87,6 +87,7 @@ extern unsigned int kobjsize(const void *objp); | |||
87 | #define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */ | 87 | #define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */ |
88 | #define VM_DENYWRITE 0x00000800 /* ETXTBSY on write attempts.. */ | 88 | #define VM_DENYWRITE 0x00000800 /* ETXTBSY on write attempts.. */ |
89 | 89 | ||
90 | #define VM_POPULATE 0x00001000 | ||
90 | #define VM_LOCKED 0x00002000 | 91 | #define VM_LOCKED 0x00002000 |
91 | #define VM_IO 0x00004000 /* Memory mapped I/O or similar */ | 92 | #define VM_IO 0x00004000 /* Memory mapped I/O or similar */ |
92 | 93 | ||
@@ -366,7 +367,7 @@ static inline struct page *compound_head(struct page *page) | |||
366 | * both from it and to it can be tracked, using atomic_inc_and_test | 367 | * both from it and to it can be tracked, using atomic_inc_and_test |
367 | * and atomic_add_negative(-1). | 368 | * and atomic_add_negative(-1). |
368 | */ | 369 | */ |
369 | static inline void reset_page_mapcount(struct page *page) | 370 | static inline void page_mapcount_reset(struct page *page) |
370 | { | 371 | { |
371 | atomic_set(&(page)->_mapcount, -1); | 372 | atomic_set(&(page)->_mapcount, -1); |
372 | } | 373 | } |
@@ -580,50 +581,11 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) | |||
580 | * sets it, so none of the operations on it need to be atomic. | 581 | * sets it, so none of the operations on it need to be atomic. |
581 | */ | 582 | */ |
582 | 583 | ||
583 | 584 | /* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_NID] | ... | FLAGS | */ | |
584 | /* | ||
585 | * page->flags layout: | ||
586 | * | ||
587 | * There are three possibilities for how page->flags get | ||
588 | * laid out. The first is for the normal case, without | ||
589 | * sparsemem. The second is for sparsemem when there is | ||
590 | * plenty of space for node and section. The last is when | ||
591 | * we have run out of space and have to fall back to an | ||
592 | * alternate (slower) way of determining the node. | ||
593 | * | ||
594 | * No sparsemem or sparsemem vmemmap: | NODE | ZONE | ... | FLAGS | | ||
595 | * classic sparse with space for node:| SECTION | NODE | ZONE | ... | FLAGS | | ||
596 | * classic sparse no space for node: | SECTION | ZONE | ... | FLAGS | | ||
597 | */ | ||
598 | #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) | ||
599 | #define SECTIONS_WIDTH SECTIONS_SHIFT | ||
600 | #else | ||
601 | #define SECTIONS_WIDTH 0 | ||
602 | #endif | ||
603 | |||
604 | #define ZONES_WIDTH ZONES_SHIFT | ||
605 | |||
606 | #if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS | ||
607 | #define NODES_WIDTH NODES_SHIFT | ||
608 | #else | ||
609 | #ifdef CONFIG_SPARSEMEM_VMEMMAP | ||
610 | #error "Vmemmap: No space for nodes field in page flags" | ||
611 | #endif | ||
612 | #define NODES_WIDTH 0 | ||
613 | #endif | ||
614 | |||
615 | /* Page flags: | [SECTION] | [NODE] | ZONE | ... | FLAGS | */ | ||
616 | #define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH) | 585 | #define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH) |
617 | #define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH) | 586 | #define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH) |
618 | #define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH) | 587 | #define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH) |
619 | 588 | #define LAST_NID_PGOFF (ZONES_PGOFF - LAST_NID_WIDTH) | |
620 | /* | ||
621 | * We are going to use the flags for the page to node mapping if its in | ||
622 | * there. This includes the case where there is no node, so it is implicit. | ||
623 | */ | ||
624 | #if !(NODES_WIDTH > 0 || NODES_SHIFT == 0) | ||
625 | #define NODE_NOT_IN_PAGE_FLAGS | ||
626 | #endif | ||
627 | 589 | ||
628 | /* | 590 | /* |
629 | * Define the bit shifts to access each section. For non-existent | 591 | * Define the bit shifts to access each section. For non-existent |
@@ -633,6 +595,7 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) | |||
633 | #define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0)) | 595 | #define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0)) |
634 | #define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0)) | 596 | #define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0)) |
635 | #define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0)) | 597 | #define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0)) |
598 | #define LAST_NID_PGSHIFT (LAST_NID_PGOFF * (LAST_NID_WIDTH != 0)) | ||
636 | 599 | ||
637 | /* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */ | 600 | /* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */ |
638 | #ifdef NODE_NOT_IN_PAGE_FLAGS | 601 | #ifdef NODE_NOT_IN_PAGE_FLAGS |
@@ -654,6 +617,7 @@ static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) | |||
654 | #define ZONES_MASK ((1UL << ZONES_WIDTH) - 1) | 617 | #define ZONES_MASK ((1UL << ZONES_WIDTH) - 1) |
655 | #define NODES_MASK ((1UL << NODES_WIDTH) - 1) | 618 | #define NODES_MASK ((1UL << NODES_WIDTH) - 1) |
656 | #define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1) | 619 | #define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1) |
620 | #define LAST_NID_MASK ((1UL << LAST_NID_WIDTH) - 1) | ||
657 | #define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1) | 621 | #define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1) |
658 | 622 | ||
659 | static inline enum zone_type page_zonenum(const struct page *page) | 623 | static inline enum zone_type page_zonenum(const struct page *page) |
@@ -661,6 +625,10 @@ static inline enum zone_type page_zonenum(const struct page *page) | |||
661 | return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK; | 625 | return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK; |
662 | } | 626 | } |
663 | 627 | ||
628 | #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) | ||
629 | #define SECTION_IN_PAGE_FLAGS | ||
630 | #endif | ||
631 | |||
664 | /* | 632 | /* |
665 | * The identification function is only used by the buddy allocator for | 633 | * The identification function is only used by the buddy allocator for |
666 | * determining if two pages could be buddies. We are not really | 634 | * determining if two pages could be buddies. We are not really |
@@ -693,31 +661,48 @@ static inline int page_to_nid(const struct page *page) | |||
693 | #endif | 661 | #endif |
694 | 662 | ||
695 | #ifdef CONFIG_NUMA_BALANCING | 663 | #ifdef CONFIG_NUMA_BALANCING |
696 | static inline int page_xchg_last_nid(struct page *page, int nid) | 664 | #ifdef LAST_NID_NOT_IN_PAGE_FLAGS |
665 | static inline int page_nid_xchg_last(struct page *page, int nid) | ||
697 | { | 666 | { |
698 | return xchg(&page->_last_nid, nid); | 667 | return xchg(&page->_last_nid, nid); |
699 | } | 668 | } |
700 | 669 | ||
701 | static inline int page_last_nid(struct page *page) | 670 | static inline int page_nid_last(struct page *page) |
702 | { | 671 | { |
703 | return page->_last_nid; | 672 | return page->_last_nid; |
704 | } | 673 | } |
705 | static inline void reset_page_last_nid(struct page *page) | 674 | static inline void page_nid_reset_last(struct page *page) |
706 | { | 675 | { |
707 | page->_last_nid = -1; | 676 | page->_last_nid = -1; |
708 | } | 677 | } |
709 | #else | 678 | #else |
710 | static inline int page_xchg_last_nid(struct page *page, int nid) | 679 | static inline int page_nid_last(struct page *page) |
680 | { | ||
681 | return (page->flags >> LAST_NID_PGSHIFT) & LAST_NID_MASK; | ||
682 | } | ||
683 | |||
684 | extern int page_nid_xchg_last(struct page *page, int nid); | ||
685 | |||
686 | static inline void page_nid_reset_last(struct page *page) | ||
687 | { | ||
688 | int nid = (1 << LAST_NID_SHIFT) - 1; | ||
689 | |||
690 | page->flags &= ~(LAST_NID_MASK << LAST_NID_PGSHIFT); | ||
691 | page->flags |= (nid & LAST_NID_MASK) << LAST_NID_PGSHIFT; | ||
692 | } | ||
693 | #endif /* LAST_NID_NOT_IN_PAGE_FLAGS */ | ||
694 | #else | ||
695 | static inline int page_nid_xchg_last(struct page *page, int nid) | ||
711 | { | 696 | { |
712 | return page_to_nid(page); | 697 | return page_to_nid(page); |
713 | } | 698 | } |
714 | 699 | ||
715 | static inline int page_last_nid(struct page *page) | 700 | static inline int page_nid_last(struct page *page) |
716 | { | 701 | { |
717 | return page_to_nid(page); | 702 | return page_to_nid(page); |
718 | } | 703 | } |
719 | 704 | ||
720 | static inline void reset_page_last_nid(struct page *page) | 705 | static inline void page_nid_reset_last(struct page *page) |
721 | { | 706 | { |
722 | } | 707 | } |
723 | #endif | 708 | #endif |
@@ -727,7 +712,7 @@ static inline struct zone *page_zone(const struct page *page) | |||
727 | return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)]; | 712 | return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)]; |
728 | } | 713 | } |
729 | 714 | ||
730 | #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) | 715 | #ifdef SECTION_IN_PAGE_FLAGS |
731 | static inline void set_page_section(struct page *page, unsigned long section) | 716 | static inline void set_page_section(struct page *page, unsigned long section) |
732 | { | 717 | { |
733 | page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT); | 718 | page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT); |
@@ -757,7 +742,7 @@ static inline void set_page_links(struct page *page, enum zone_type zone, | |||
757 | { | 742 | { |
758 | set_page_zone(page, zone); | 743 | set_page_zone(page, zone); |
759 | set_page_node(page, node); | 744 | set_page_node(page, node); |
760 | #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) | 745 | #ifdef SECTION_IN_PAGE_FLAGS |
761 | set_page_section(page, pfn_to_section_nr(pfn)); | 746 | set_page_section(page, pfn_to_section_nr(pfn)); |
762 | #endif | 747 | #endif |
763 | } | 748 | } |
@@ -817,18 +802,7 @@ void page_address_init(void); | |||
817 | #define PAGE_MAPPING_KSM 2 | 802 | #define PAGE_MAPPING_KSM 2 |
818 | #define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM) | 803 | #define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM) |
819 | 804 | ||
820 | extern struct address_space swapper_space; | 805 | extern struct address_space *page_mapping(struct page *page); |
821 | static inline struct address_space *page_mapping(struct page *page) | ||
822 | { | ||
823 | struct address_space *mapping = page->mapping; | ||
824 | |||
825 | VM_BUG_ON(PageSlab(page)); | ||
826 | if (unlikely(PageSwapCache(page))) | ||
827 | mapping = &swapper_space; | ||
828 | else if ((unsigned long)mapping & PAGE_MAPPING_ANON) | ||
829 | mapping = NULL; | ||
830 | return mapping; | ||
831 | } | ||
832 | 806 | ||
833 | /* Neutral page->mapping pointer to address_space or anon_vma or other */ | 807 | /* Neutral page->mapping pointer to address_space or anon_vma or other */ |
834 | static inline void *page_rmapping(struct page *page) | 808 | static inline void *page_rmapping(struct page *page) |
@@ -1035,18 +1009,18 @@ static inline int fixup_user_fault(struct task_struct *tsk, | |||
1035 | } | 1009 | } |
1036 | #endif | 1010 | #endif |
1037 | 1011 | ||
1038 | extern int make_pages_present(unsigned long addr, unsigned long end); | ||
1039 | extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write); | 1012 | extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write); |
1040 | extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, | 1013 | extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, |
1041 | void *buf, int len, int write); | 1014 | void *buf, int len, int write); |
1042 | 1015 | ||
1043 | int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, | 1016 | long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, |
1044 | unsigned long start, int len, unsigned int foll_flags, | 1017 | unsigned long start, unsigned long nr_pages, |
1045 | struct page **pages, struct vm_area_struct **vmas, | 1018 | unsigned int foll_flags, struct page **pages, |
1046 | int *nonblocking); | 1019 | struct vm_area_struct **vmas, int *nonblocking); |
1047 | int get_user_pages(struct task_struct *tsk, struct mm_struct *mm, | 1020 | long get_user_pages(struct task_struct *tsk, struct mm_struct *mm, |
1048 | unsigned long start, int nr_pages, int write, int force, | 1021 | unsigned long start, unsigned long nr_pages, |
1049 | struct page **pages, struct vm_area_struct **vmas); | 1022 | int write, int force, struct page **pages, |
1023 | struct vm_area_struct **vmas); | ||
1050 | int get_user_pages_fast(unsigned long start, int nr_pages, int write, | 1024 | int get_user_pages_fast(unsigned long start, int nr_pages, int write, |
1051 | struct page **pages); | 1025 | struct page **pages); |
1052 | struct kvec; | 1026 | struct kvec; |
@@ -1359,6 +1333,24 @@ extern void free_bootmem_with_active_regions(int nid, | |||
1359 | unsigned long max_low_pfn); | 1333 | unsigned long max_low_pfn); |
1360 | extern void sparse_memory_present_with_active_regions(int nid); | 1334 | extern void sparse_memory_present_with_active_regions(int nid); |
1361 | 1335 | ||
1336 | #define MOVABLEMEM_MAP_MAX MAX_NUMNODES | ||
1337 | struct movablemem_entry { | ||
1338 | unsigned long start_pfn; /* start pfn of memory segment */ | ||
1339 | unsigned long end_pfn; /* end pfn of memory segment (exclusive) */ | ||
1340 | }; | ||
1341 | |||
1342 | struct movablemem_map { | ||
1343 | bool acpi; /* true if using SRAT info */ | ||
1344 | int nr_map; | ||
1345 | struct movablemem_entry map[MOVABLEMEM_MAP_MAX]; | ||
1346 | nodemask_t numa_nodes_hotplug; /* on which nodes we specify memory */ | ||
1347 | nodemask_t numa_nodes_kernel; /* on which nodes kernel resides in */ | ||
1348 | }; | ||
1349 | |||
1350 | extern void __init insert_movablemem_map(unsigned long start_pfn, | ||
1351 | unsigned long end_pfn); | ||
1352 | extern int __init movablemem_map_overlap(unsigned long start_pfn, | ||
1353 | unsigned long end_pfn); | ||
1362 | #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ | 1354 | #endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */ |
1363 | 1355 | ||
1364 | #if !defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) && \ | 1356 | #if !defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) && \ |
@@ -1395,6 +1387,9 @@ extern void setup_per_cpu_pageset(void); | |||
1395 | extern void zone_pcp_update(struct zone *zone); | 1387 | extern void zone_pcp_update(struct zone *zone); |
1396 | extern void zone_pcp_reset(struct zone *zone); | 1388 | extern void zone_pcp_reset(struct zone *zone); |
1397 | 1389 | ||
1390 | /* page_alloc.c */ | ||
1391 | extern int min_free_kbytes; | ||
1392 | |||
1398 | /* nommu.c */ | 1393 | /* nommu.c */ |
1399 | extern atomic_long_t mmap_pages_allocated; | 1394 | extern atomic_long_t mmap_pages_allocated; |
1400 | extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t); | 1395 | extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t); |
@@ -1472,13 +1467,24 @@ extern int install_special_mapping(struct mm_struct *mm, | |||
1472 | extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long); | 1467 | extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long); |
1473 | 1468 | ||
1474 | extern unsigned long mmap_region(struct file *file, unsigned long addr, | 1469 | extern unsigned long mmap_region(struct file *file, unsigned long addr, |
1475 | unsigned long len, unsigned long flags, | 1470 | unsigned long len, vm_flags_t vm_flags, unsigned long pgoff); |
1476 | vm_flags_t vm_flags, unsigned long pgoff); | 1471 | extern unsigned long do_mmap_pgoff(struct file *file, unsigned long addr, |
1477 | extern unsigned long do_mmap_pgoff(struct file *, unsigned long, | 1472 | unsigned long len, unsigned long prot, unsigned long flags, |
1478 | unsigned long, unsigned long, | 1473 | unsigned long pgoff, unsigned long *populate); |
1479 | unsigned long, unsigned long); | ||
1480 | extern int do_munmap(struct mm_struct *, unsigned long, size_t); | 1474 | extern int do_munmap(struct mm_struct *, unsigned long, size_t); |
1481 | 1475 | ||
1476 | #ifdef CONFIG_MMU | ||
1477 | extern int __mm_populate(unsigned long addr, unsigned long len, | ||
1478 | int ignore_errors); | ||
1479 | static inline void mm_populate(unsigned long addr, unsigned long len) | ||
1480 | { | ||
1481 | /* Ignore errors */ | ||
1482 | (void) __mm_populate(addr, len, 1); | ||
1483 | } | ||
1484 | #else | ||
1485 | static inline void mm_populate(unsigned long addr, unsigned long len) {} | ||
1486 | #endif | ||
1487 | |||
1482 | /* These take the mm semaphore themselves */ | 1488 | /* These take the mm semaphore themselves */ |
1483 | extern unsigned long vm_brk(unsigned long, unsigned long); | 1489 | extern unsigned long vm_brk(unsigned long, unsigned long); |
1484 | extern int vm_munmap(unsigned long, size_t); | 1490 | extern int vm_munmap(unsigned long, size_t); |
@@ -1623,8 +1629,17 @@ int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr, | |||
1623 | int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr, | 1629 | int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr, |
1624 | unsigned long pfn); | 1630 | unsigned long pfn); |
1625 | 1631 | ||
1626 | struct page *follow_page(struct vm_area_struct *, unsigned long address, | 1632 | struct page *follow_page_mask(struct vm_area_struct *vma, |
1627 | unsigned int foll_flags); | 1633 | unsigned long address, unsigned int foll_flags, |
1634 | unsigned int *page_mask); | ||
1635 | |||
1636 | static inline struct page *follow_page(struct vm_area_struct *vma, | ||
1637 | unsigned long address, unsigned int foll_flags) | ||
1638 | { | ||
1639 | unsigned int unused_page_mask; | ||
1640 | return follow_page_mask(vma, address, foll_flags, &unused_page_mask); | ||
1641 | } | ||
1642 | |||
1628 | #define FOLL_WRITE 0x01 /* check pte is writable */ | 1643 | #define FOLL_WRITE 0x01 /* check pte is writable */ |
1629 | #define FOLL_TOUCH 0x02 /* mark page accessed */ | 1644 | #define FOLL_TOUCH 0x02 /* mark page accessed */ |
1630 | #define FOLL_GET 0x04 /* do get_page on page */ | 1645 | #define FOLL_GET 0x04 /* do get_page on page */ |
@@ -1636,6 +1651,7 @@ struct page *follow_page(struct vm_area_struct *, unsigned long address, | |||
1636 | #define FOLL_SPLIT 0x80 /* don't return transhuge pages, split them */ | 1651 | #define FOLL_SPLIT 0x80 /* don't return transhuge pages, split them */ |
1637 | #define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */ | 1652 | #define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */ |
1638 | #define FOLL_NUMA 0x200 /* force NUMA hinting page fault */ | 1653 | #define FOLL_NUMA 0x200 /* force NUMA hinting page fault */ |
1654 | #define FOLL_MIGRATION 0x400 /* wait for page to replace migration entry */ | ||
1639 | 1655 | ||
1640 | typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr, | 1656 | typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr, |
1641 | void *data); | 1657 | void *data); |
@@ -1707,7 +1723,11 @@ int vmemmap_populate_basepages(struct page *start_page, | |||
1707 | unsigned long pages, int node); | 1723 | unsigned long pages, int node); |
1708 | int vmemmap_populate(struct page *start_page, unsigned long pages, int node); | 1724 | int vmemmap_populate(struct page *start_page, unsigned long pages, int node); |
1709 | void vmemmap_populate_print_last(void); | 1725 | void vmemmap_populate_print_last(void); |
1710 | 1726 | #ifdef CONFIG_MEMORY_HOTPLUG | |
1727 | void vmemmap_free(struct page *memmap, unsigned long nr_pages); | ||
1728 | #endif | ||
1729 | void register_page_bootmem_memmap(unsigned long section_nr, struct page *map, | ||
1730 | unsigned long size); | ||
1711 | 1731 | ||
1712 | enum mf_flags { | 1732 | enum mf_flags { |
1713 | MF_COUNT_INCREASED = 1 << 0, | 1733 | MF_COUNT_INCREASED = 1 << 0, |
@@ -1720,7 +1740,7 @@ extern int unpoison_memory(unsigned long pfn); | |||
1720 | extern int sysctl_memory_failure_early_kill; | 1740 | extern int sysctl_memory_failure_early_kill; |
1721 | extern int sysctl_memory_failure_recovery; | 1741 | extern int sysctl_memory_failure_recovery; |
1722 | extern void shake_page(struct page *p, int access); | 1742 | extern void shake_page(struct page *p, int access); |
1723 | extern atomic_long_t mce_bad_pages; | 1743 | extern atomic_long_t num_poisoned_pages; |
1724 | extern int soft_offline_page(struct page *page, int flags); | 1744 | extern int soft_offline_page(struct page *page, int flags); |
1725 | 1745 | ||
1726 | extern void dump_page(struct page *page); | 1746 | extern void dump_page(struct page *page); |
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index f8f5162a3571..ace9a5f01c64 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h | |||
@@ -12,6 +12,7 @@ | |||
12 | #include <linux/cpumask.h> | 12 | #include <linux/cpumask.h> |
13 | #include <linux/page-debug-flags.h> | 13 | #include <linux/page-debug-flags.h> |
14 | #include <linux/uprobes.h> | 14 | #include <linux/uprobes.h> |
15 | #include <linux/page-flags-layout.h> | ||
15 | #include <asm/page.h> | 16 | #include <asm/page.h> |
16 | #include <asm/mmu.h> | 17 | #include <asm/mmu.h> |
17 | 18 | ||
@@ -173,7 +174,7 @@ struct page { | |||
173 | void *shadow; | 174 | void *shadow; |
174 | #endif | 175 | #endif |
175 | 176 | ||
176 | #ifdef CONFIG_NUMA_BALANCING | 177 | #ifdef LAST_NID_NOT_IN_PAGE_FLAGS |
177 | int _last_nid; | 178 | int _last_nid; |
178 | #endif | 179 | #endif |
179 | } | 180 | } |
@@ -414,9 +415,9 @@ struct mm_struct { | |||
414 | #endif | 415 | #endif |
415 | #ifdef CONFIG_NUMA_BALANCING | 416 | #ifdef CONFIG_NUMA_BALANCING |
416 | /* | 417 | /* |
417 | * numa_next_scan is the next time when the PTEs will me marked | 418 | * numa_next_scan is the next time that the PTEs will be marked |
418 | * pte_numa to gather statistics and migrate pages to new nodes | 419 | * pte_numa. NUMA hinting faults will gather statistics and migrate |
419 | * if necessary | 420 | * pages to new nodes if necessary. |
420 | */ | 421 | */ |
421 | unsigned long numa_next_scan; | 422 | unsigned long numa_next_scan; |
422 | 423 | ||
diff --git a/include/linux/mman.h b/include/linux/mman.h index 9aa863da287f..61c7a87e5d2b 100644 --- a/include/linux/mman.h +++ b/include/linux/mman.h | |||
@@ -79,6 +79,8 @@ calc_vm_flag_bits(unsigned long flags) | |||
79 | { | 79 | { |
80 | return _calc_vm_trans(flags, MAP_GROWSDOWN, VM_GROWSDOWN ) | | 80 | return _calc_vm_trans(flags, MAP_GROWSDOWN, VM_GROWSDOWN ) | |
81 | _calc_vm_trans(flags, MAP_DENYWRITE, VM_DENYWRITE ) | | 81 | _calc_vm_trans(flags, MAP_DENYWRITE, VM_DENYWRITE ) | |
82 | _calc_vm_trans(flags, MAP_LOCKED, VM_LOCKED ); | 82 | ((flags & MAP_LOCKED) ? (VM_LOCKED | VM_POPULATE) : 0) | |
83 | (((flags & (MAP_POPULATE | MAP_NONBLOCK)) == MAP_POPULATE) ? | ||
84 | VM_POPULATE : 0); | ||
83 | } | 85 | } |
84 | #endif /* _LINUX_MMAN_H */ | 86 | #endif /* _LINUX_MMAN_H */ |
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 73b64a38b984..ede274957e05 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h | |||
@@ -15,7 +15,7 @@ | |||
15 | #include <linux/seqlock.h> | 15 | #include <linux/seqlock.h> |
16 | #include <linux/nodemask.h> | 16 | #include <linux/nodemask.h> |
17 | #include <linux/pageblock-flags.h> | 17 | #include <linux/pageblock-flags.h> |
18 | #include <generated/bounds.h> | 18 | #include <linux/page-flags-layout.h> |
19 | #include <linux/atomic.h> | 19 | #include <linux/atomic.h> |
20 | #include <asm/page.h> | 20 | #include <asm/page.h> |
21 | 21 | ||
@@ -57,7 +57,9 @@ enum { | |||
57 | */ | 57 | */ |
58 | MIGRATE_CMA, | 58 | MIGRATE_CMA, |
59 | #endif | 59 | #endif |
60 | #ifdef CONFIG_MEMORY_ISOLATION | ||
60 | MIGRATE_ISOLATE, /* can't allocate from here */ | 61 | MIGRATE_ISOLATE, /* can't allocate from here */ |
62 | #endif | ||
61 | MIGRATE_TYPES | 63 | MIGRATE_TYPES |
62 | }; | 64 | }; |
63 | 65 | ||
@@ -308,24 +310,6 @@ enum zone_type { | |||
308 | 310 | ||
309 | #ifndef __GENERATING_BOUNDS_H | 311 | #ifndef __GENERATING_BOUNDS_H |
310 | 312 | ||
311 | /* | ||
312 | * When a memory allocation must conform to specific limitations (such | ||
313 | * as being suitable for DMA) the caller will pass in hints to the | ||
314 | * allocator in the gfp_mask, in the zone modifier bits. These bits | ||
315 | * are used to select a priority ordered list of memory zones which | ||
316 | * match the requested limits. See gfp_zone() in include/linux/gfp.h | ||
317 | */ | ||
318 | |||
319 | #if MAX_NR_ZONES < 2 | ||
320 | #define ZONES_SHIFT 0 | ||
321 | #elif MAX_NR_ZONES <= 2 | ||
322 | #define ZONES_SHIFT 1 | ||
323 | #elif MAX_NR_ZONES <= 4 | ||
324 | #define ZONES_SHIFT 2 | ||
325 | #else | ||
326 | #error ZONES_SHIFT -- too many zones configured adjust calculation | ||
327 | #endif | ||
328 | |||
329 | struct zone { | 313 | struct zone { |
330 | /* Fields commonly accessed by the page allocator */ | 314 | /* Fields commonly accessed by the page allocator */ |
331 | 315 | ||
@@ -543,6 +527,26 @@ static inline int zone_is_oom_locked(const struct zone *zone) | |||
543 | return test_bit(ZONE_OOM_LOCKED, &zone->flags); | 527 | return test_bit(ZONE_OOM_LOCKED, &zone->flags); |
544 | } | 528 | } |
545 | 529 | ||
530 | static inline unsigned zone_end_pfn(const struct zone *zone) | ||
531 | { | ||
532 | return zone->zone_start_pfn + zone->spanned_pages; | ||
533 | } | ||
534 | |||
535 | static inline bool zone_spans_pfn(const struct zone *zone, unsigned long pfn) | ||
536 | { | ||
537 | return zone->zone_start_pfn <= pfn && pfn < zone_end_pfn(zone); | ||
538 | } | ||
539 | |||
540 | static inline bool zone_is_initialized(struct zone *zone) | ||
541 | { | ||
542 | return !!zone->wait_table; | ||
543 | } | ||
544 | |||
545 | static inline bool zone_is_empty(struct zone *zone) | ||
546 | { | ||
547 | return zone->spanned_pages == 0; | ||
548 | } | ||
549 | |||
546 | /* | 550 | /* |
547 | * The "priority" of VM scanning is how much of the queues we will scan in one | 551 | * The "priority" of VM scanning is how much of the queues we will scan in one |
548 | * go. A value of 12 for DEF_PRIORITY implies that we will scan 1/4096th of the | 552 | * go. A value of 12 for DEF_PRIORITY implies that we will scan 1/4096th of the |
@@ -752,11 +756,17 @@ typedef struct pglist_data { | |||
752 | #define nid_page_nr(nid, pagenr) pgdat_page_nr(NODE_DATA(nid),(pagenr)) | 756 | #define nid_page_nr(nid, pagenr) pgdat_page_nr(NODE_DATA(nid),(pagenr)) |
753 | 757 | ||
754 | #define node_start_pfn(nid) (NODE_DATA(nid)->node_start_pfn) | 758 | #define node_start_pfn(nid) (NODE_DATA(nid)->node_start_pfn) |
759 | #define node_end_pfn(nid) pgdat_end_pfn(NODE_DATA(nid)) | ||
755 | 760 | ||
756 | #define node_end_pfn(nid) ({\ | 761 | static inline unsigned long pgdat_end_pfn(pg_data_t *pgdat) |
757 | pg_data_t *__pgdat = NODE_DATA(nid);\ | 762 | { |
758 | __pgdat->node_start_pfn + __pgdat->node_spanned_pages;\ | 763 | return pgdat->node_start_pfn + pgdat->node_spanned_pages; |
759 | }) | 764 | } |
765 | |||
766 | static inline bool pgdat_is_empty(pg_data_t *pgdat) | ||
767 | { | ||
768 | return !pgdat->node_start_pfn && !pgdat->node_spanned_pages; | ||
769 | } | ||
760 | 770 | ||
761 | #include <linux/memory_hotplug.h> | 771 | #include <linux/memory_hotplug.h> |
762 | 772 | ||
@@ -1053,8 +1063,6 @@ static inline unsigned long early_pfn_to_nid(unsigned long pfn) | |||
1053 | * PA_SECTION_SHIFT physical address to/from section number | 1063 | * PA_SECTION_SHIFT physical address to/from section number |
1054 | * PFN_SECTION_SHIFT pfn to/from section number | 1064 | * PFN_SECTION_SHIFT pfn to/from section number |
1055 | */ | 1065 | */ |
1056 | #define SECTIONS_SHIFT (MAX_PHYSMEM_BITS - SECTION_SIZE_BITS) | ||
1057 | |||
1058 | #define PA_SECTION_SHIFT (SECTION_SIZE_BITS) | 1066 | #define PA_SECTION_SHIFT (SECTION_SIZE_BITS) |
1059 | #define PFN_SECTION_SHIFT (SECTION_SIZE_BITS - PAGE_SHIFT) | 1067 | #define PFN_SECTION_SHIFT (SECTION_SIZE_BITS - PAGE_SHIFT) |
1060 | 1068 | ||
diff --git a/include/linux/page-flags-layout.h b/include/linux/page-flags-layout.h new file mode 100644 index 000000000000..93506a114034 --- /dev/null +++ b/include/linux/page-flags-layout.h | |||
@@ -0,0 +1,88 @@ | |||
1 | #ifndef PAGE_FLAGS_LAYOUT_H | ||
2 | #define PAGE_FLAGS_LAYOUT_H | ||
3 | |||
4 | #include <linux/numa.h> | ||
5 | #include <generated/bounds.h> | ||
6 | |||
7 | /* | ||
8 | * When a memory allocation must conform to specific limitations (such | ||
9 | * as being suitable for DMA) the caller will pass in hints to the | ||
10 | * allocator in the gfp_mask, in the zone modifier bits. These bits | ||
11 | * are used to select a priority ordered list of memory zones which | ||
12 | * match the requested limits. See gfp_zone() in include/linux/gfp.h | ||
13 | */ | ||
14 | #if MAX_NR_ZONES < 2 | ||
15 | #define ZONES_SHIFT 0 | ||
16 | #elif MAX_NR_ZONES <= 2 | ||
17 | #define ZONES_SHIFT 1 | ||
18 | #elif MAX_NR_ZONES <= 4 | ||
19 | #define ZONES_SHIFT 2 | ||
20 | #else | ||
21 | #error ZONES_SHIFT -- too many zones configured adjust calculation | ||
22 | #endif | ||
23 | |||
24 | #ifdef CONFIG_SPARSEMEM | ||
25 | #include <asm/sparsemem.h> | ||
26 | |||
27 | /* SECTION_SHIFT #bits space required to store a section # */ | ||
28 | #define SECTIONS_SHIFT (MAX_PHYSMEM_BITS - SECTION_SIZE_BITS) | ||
29 | |||
30 | #endif /* CONFIG_SPARSEMEM */ | ||
31 | |||
32 | /* | ||
33 | * page->flags layout: | ||
34 | * | ||
35 | * There are five possibilities for how page->flags get laid out. The first | ||
36 | * pair is for the normal case without sparsemem. The second pair is for | ||
37 | * sparsemem when there is plenty of space for node and section information. | ||
38 | * The last is when there is insufficient space in page->flags and a separate | ||
39 | * lookup is necessary. | ||
40 | * | ||
41 | * No sparsemem or sparsemem vmemmap: | NODE | ZONE | ... | FLAGS | | ||
42 | * " plus space for last_nid: | NODE | ZONE | LAST_NID ... | FLAGS | | ||
43 | * classic sparse with space for node:| SECTION | NODE | ZONE | ... | FLAGS | | ||
44 | * " plus space for last_nid: | SECTION | NODE | ZONE | LAST_NID ... | FLAGS | | ||
45 | * classic sparse no space for node: | SECTION | ZONE | ... | FLAGS | | ||
46 | */ | ||
47 | #if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP) | ||
48 | #define SECTIONS_WIDTH SECTIONS_SHIFT | ||
49 | #else | ||
50 | #define SECTIONS_WIDTH 0 | ||
51 | #endif | ||
52 | |||
53 | #define ZONES_WIDTH ZONES_SHIFT | ||
54 | |||
55 | #if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS | ||
56 | #define NODES_WIDTH NODES_SHIFT | ||
57 | #else | ||
58 | #ifdef CONFIG_SPARSEMEM_VMEMMAP | ||
59 | #error "Vmemmap: No space for nodes field in page flags" | ||
60 | #endif | ||
61 | #define NODES_WIDTH 0 | ||
62 | #endif | ||
63 | |||
64 | #ifdef CONFIG_NUMA_BALANCING | ||
65 | #define LAST_NID_SHIFT NODES_SHIFT | ||
66 | #else | ||
67 | #define LAST_NID_SHIFT 0 | ||
68 | #endif | ||
69 | |||
70 | #if SECTIONS_WIDTH+ZONES_WIDTH+NODES_SHIFT+LAST_NID_SHIFT <= BITS_PER_LONG - NR_PAGEFLAGS | ||
71 | #define LAST_NID_WIDTH LAST_NID_SHIFT | ||
72 | #else | ||
73 | #define LAST_NID_WIDTH 0 | ||
74 | #endif | ||
75 | |||
76 | /* | ||
77 | * We are going to use the flags for the page to node mapping if its in | ||
78 | * there. This includes the case where there is no node, so it is implicit. | ||
79 | */ | ||
80 | #if !(NODES_WIDTH > 0 || NODES_SHIFT == 0) | ||
81 | #define NODE_NOT_IN_PAGE_FLAGS | ||
82 | #endif | ||
83 | |||
84 | #if defined(CONFIG_NUMA_BALANCING) && LAST_NID_WIDTH == 0 | ||
85 | #define LAST_NID_NOT_IN_PAGE_FLAGS | ||
86 | #endif | ||
87 | |||
88 | #endif /* _LINUX_PAGE_FLAGS_LAYOUT */ | ||
diff --git a/include/linux/page-isolation.h b/include/linux/page-isolation.h index a92061e08d48..3fff8e774067 100644 --- a/include/linux/page-isolation.h +++ b/include/linux/page-isolation.h | |||
@@ -1,6 +1,25 @@ | |||
1 | #ifndef __LINUX_PAGEISOLATION_H | 1 | #ifndef __LINUX_PAGEISOLATION_H |
2 | #define __LINUX_PAGEISOLATION_H | 2 | #define __LINUX_PAGEISOLATION_H |
3 | 3 | ||
4 | #ifdef CONFIG_MEMORY_ISOLATION | ||
5 | static inline bool is_migrate_isolate_page(struct page *page) | ||
6 | { | ||
7 | return get_pageblock_migratetype(page) == MIGRATE_ISOLATE; | ||
8 | } | ||
9 | static inline bool is_migrate_isolate(int migratetype) | ||
10 | { | ||
11 | return migratetype == MIGRATE_ISOLATE; | ||
12 | } | ||
13 | #else | ||
14 | static inline bool is_migrate_isolate_page(struct page *page) | ||
15 | { | ||
16 | return false; | ||
17 | } | ||
18 | static inline bool is_migrate_isolate(int migratetype) | ||
19 | { | ||
20 | return false; | ||
21 | } | ||
22 | #endif | ||
4 | 23 | ||
5 | bool has_unmovable_pages(struct zone *zone, struct page *page, int count, | 24 | bool has_unmovable_pages(struct zone *zone, struct page *page, int count, |
6 | bool skip_hwpoisoned_pages); | 25 | bool skip_hwpoisoned_pages); |
diff --git a/include/linux/pm.h b/include/linux/pm.h index 97bcf23e045a..e5d7230332a4 100644 --- a/include/linux/pm.h +++ b/include/linux/pm.h | |||
@@ -537,6 +537,7 @@ struct dev_pm_info { | |||
537 | unsigned int irq_safe:1; | 537 | unsigned int irq_safe:1; |
538 | unsigned int use_autosuspend:1; | 538 | unsigned int use_autosuspend:1; |
539 | unsigned int timer_autosuspends:1; | 539 | unsigned int timer_autosuspends:1; |
540 | unsigned int memalloc_noio:1; | ||
540 | enum rpm_request request; | 541 | enum rpm_request request; |
541 | enum rpm_status runtime_status; | 542 | enum rpm_status runtime_status; |
542 | int runtime_error; | 543 | int runtime_error; |
diff --git a/include/linux/pm_runtime.h b/include/linux/pm_runtime.h index c785c215abfc..7d7e09efff9b 100644 --- a/include/linux/pm_runtime.h +++ b/include/linux/pm_runtime.h | |||
@@ -47,6 +47,7 @@ extern void pm_runtime_set_autosuspend_delay(struct device *dev, int delay); | |||
47 | extern unsigned long pm_runtime_autosuspend_expiration(struct device *dev); | 47 | extern unsigned long pm_runtime_autosuspend_expiration(struct device *dev); |
48 | extern void pm_runtime_update_max_time_suspended(struct device *dev, | 48 | extern void pm_runtime_update_max_time_suspended(struct device *dev, |
49 | s64 delta_ns); | 49 | s64 delta_ns); |
50 | extern void pm_runtime_set_memalloc_noio(struct device *dev, bool enable); | ||
50 | 51 | ||
51 | static inline bool pm_children_suspended(struct device *dev) | 52 | static inline bool pm_children_suspended(struct device *dev) |
52 | { | 53 | { |
@@ -156,6 +157,8 @@ static inline void pm_runtime_set_autosuspend_delay(struct device *dev, | |||
156 | int delay) {} | 157 | int delay) {} |
157 | static inline unsigned long pm_runtime_autosuspend_expiration( | 158 | static inline unsigned long pm_runtime_autosuspend_expiration( |
158 | struct device *dev) { return 0; } | 159 | struct device *dev) { return 0; } |
160 | static inline void pm_runtime_set_memalloc_noio(struct device *dev, | ||
161 | bool enable){} | ||
159 | 162 | ||
160 | #endif /* !CONFIG_PM_RUNTIME */ | 163 | #endif /* !CONFIG_PM_RUNTIME */ |
161 | 164 | ||
diff --git a/include/linux/rmap.h b/include/linux/rmap.h index c20635c527a9..6dacb93a6d94 100644 --- a/include/linux/rmap.h +++ b/include/linux/rmap.h | |||
@@ -123,7 +123,7 @@ static inline void anon_vma_lock_write(struct anon_vma *anon_vma) | |||
123 | down_write(&anon_vma->root->rwsem); | 123 | down_write(&anon_vma->root->rwsem); |
124 | } | 124 | } |
125 | 125 | ||
126 | static inline void anon_vma_unlock(struct anon_vma *anon_vma) | 126 | static inline void anon_vma_unlock_write(struct anon_vma *anon_vma) |
127 | { | 127 | { |
128 | up_write(&anon_vma->root->rwsem); | 128 | up_write(&anon_vma->root->rwsem); |
129 | } | 129 | } |
diff --git a/include/linux/sched.h b/include/linux/sched.h index e4112aad2964..c2182b53dace 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h | |||
@@ -51,6 +51,7 @@ struct sched_param { | |||
51 | #include <linux/cred.h> | 51 | #include <linux/cred.h> |
52 | #include <linux/llist.h> | 52 | #include <linux/llist.h> |
53 | #include <linux/uidgid.h> | 53 | #include <linux/uidgid.h> |
54 | #include <linux/gfp.h> | ||
54 | 55 | ||
55 | #include <asm/processor.h> | 56 | #include <asm/processor.h> |
56 | 57 | ||
@@ -1791,6 +1792,7 @@ extern void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, | |||
1791 | #define PF_FROZEN 0x00010000 /* frozen for system suspend */ | 1792 | #define PF_FROZEN 0x00010000 /* frozen for system suspend */ |
1792 | #define PF_FSTRANS 0x00020000 /* inside a filesystem transaction */ | 1793 | #define PF_FSTRANS 0x00020000 /* inside a filesystem transaction */ |
1793 | #define PF_KSWAPD 0x00040000 /* I am kswapd */ | 1794 | #define PF_KSWAPD 0x00040000 /* I am kswapd */ |
1795 | #define PF_MEMALLOC_NOIO 0x00080000 /* Allocating memory without IO involved */ | ||
1794 | #define PF_LESS_THROTTLE 0x00100000 /* Throttle me less: I clean memory */ | 1796 | #define PF_LESS_THROTTLE 0x00100000 /* Throttle me less: I clean memory */ |
1795 | #define PF_KTHREAD 0x00200000 /* I am a kernel thread */ | 1797 | #define PF_KTHREAD 0x00200000 /* I am a kernel thread */ |
1796 | #define PF_RANDOMIZE 0x00400000 /* randomize virtual address space */ | 1798 | #define PF_RANDOMIZE 0x00400000 /* randomize virtual address space */ |
@@ -1828,6 +1830,26 @@ extern void thread_group_cputime_adjusted(struct task_struct *p, cputime_t *ut, | |||
1828 | #define tsk_used_math(p) ((p)->flags & PF_USED_MATH) | 1830 | #define tsk_used_math(p) ((p)->flags & PF_USED_MATH) |
1829 | #define used_math() tsk_used_math(current) | 1831 | #define used_math() tsk_used_math(current) |
1830 | 1832 | ||
1833 | /* __GFP_IO isn't allowed if PF_MEMALLOC_NOIO is set in current->flags */ | ||
1834 | static inline gfp_t memalloc_noio_flags(gfp_t flags) | ||
1835 | { | ||
1836 | if (unlikely(current->flags & PF_MEMALLOC_NOIO)) | ||
1837 | flags &= ~__GFP_IO; | ||
1838 | return flags; | ||
1839 | } | ||
1840 | |||
1841 | static inline unsigned int memalloc_noio_save(void) | ||
1842 | { | ||
1843 | unsigned int flags = current->flags & PF_MEMALLOC_NOIO; | ||
1844 | current->flags |= PF_MEMALLOC_NOIO; | ||
1845 | return flags; | ||
1846 | } | ||
1847 | |||
1848 | static inline void memalloc_noio_restore(unsigned int flags) | ||
1849 | { | ||
1850 | current->flags = (current->flags & ~PF_MEMALLOC_NOIO) | flags; | ||
1851 | } | ||
1852 | |||
1831 | /* | 1853 | /* |
1832 | * task->jobctl flags | 1854 | * task->jobctl flags |
1833 | */ | 1855 | */ |
diff --git a/include/linux/swap.h b/include/linux/swap.h index 68df9c17fbbb..2818a123f3ea 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h | |||
@@ -8,7 +8,7 @@ | |||
8 | #include <linux/memcontrol.h> | 8 | #include <linux/memcontrol.h> |
9 | #include <linux/sched.h> | 9 | #include <linux/sched.h> |
10 | #include <linux/node.h> | 10 | #include <linux/node.h> |
11 | 11 | #include <linux/fs.h> | |
12 | #include <linux/atomic.h> | 12 | #include <linux/atomic.h> |
13 | #include <asm/page.h> | 13 | #include <asm/page.h> |
14 | 14 | ||
@@ -156,7 +156,7 @@ enum { | |||
156 | SWP_SCANNING = (1 << 8), /* refcount in scan_swap_map */ | 156 | SWP_SCANNING = (1 << 8), /* refcount in scan_swap_map */ |
157 | }; | 157 | }; |
158 | 158 | ||
159 | #define SWAP_CLUSTER_MAX 32 | 159 | #define SWAP_CLUSTER_MAX 32UL |
160 | #define COMPACT_CLUSTER_MAX SWAP_CLUSTER_MAX | 160 | #define COMPACT_CLUSTER_MAX SWAP_CLUSTER_MAX |
161 | 161 | ||
162 | /* | 162 | /* |
@@ -202,6 +202,18 @@ struct swap_info_struct { | |||
202 | unsigned long *frontswap_map; /* frontswap in-use, one bit per page */ | 202 | unsigned long *frontswap_map; /* frontswap in-use, one bit per page */ |
203 | atomic_t frontswap_pages; /* frontswap pages in-use counter */ | 203 | atomic_t frontswap_pages; /* frontswap pages in-use counter */ |
204 | #endif | 204 | #endif |
205 | spinlock_t lock; /* | ||
206 | * protect map scan related fields like | ||
207 | * swap_map, lowest_bit, highest_bit, | ||
208 | * inuse_pages, cluster_next, | ||
209 | * cluster_nr, lowest_alloc and | ||
210 | * highest_alloc. other fields are only | ||
211 | * changed at swapon/swapoff, so are | ||
212 | * protected by swap_lock. changing | ||
213 | * flags need hold this lock and | ||
214 | * swap_lock. If both locks need hold, | ||
215 | * hold swap_lock first. | ||
216 | */ | ||
205 | }; | 217 | }; |
206 | 218 | ||
207 | struct swap_list_t { | 219 | struct swap_list_t { |
@@ -209,15 +221,12 @@ struct swap_list_t { | |||
209 | int next; /* swapfile to be used next */ | 221 | int next; /* swapfile to be used next */ |
210 | }; | 222 | }; |
211 | 223 | ||
212 | /* Swap 50% full? Release swapcache more aggressively.. */ | ||
213 | #define vm_swap_full() (nr_swap_pages*2 < total_swap_pages) | ||
214 | |||
215 | /* linux/mm/page_alloc.c */ | 224 | /* linux/mm/page_alloc.c */ |
216 | extern unsigned long totalram_pages; | 225 | extern unsigned long totalram_pages; |
217 | extern unsigned long totalreserve_pages; | 226 | extern unsigned long totalreserve_pages; |
218 | extern unsigned long dirty_balance_reserve; | 227 | extern unsigned long dirty_balance_reserve; |
219 | extern unsigned int nr_free_buffer_pages(void); | 228 | extern unsigned long nr_free_buffer_pages(void); |
220 | extern unsigned int nr_free_pagecache_pages(void); | 229 | extern unsigned long nr_free_pagecache_pages(void); |
221 | 230 | ||
222 | /* Definition of global_page_state not available yet */ | 231 | /* Definition of global_page_state not available yet */ |
223 | #define nr_free_pages() global_page_state(NR_FREE_PAGES) | 232 | #define nr_free_pages() global_page_state(NR_FREE_PAGES) |
@@ -266,7 +275,7 @@ extern unsigned long mem_cgroup_shrink_node_zone(struct mem_cgroup *mem, | |||
266 | extern unsigned long shrink_all_memory(unsigned long nr_pages); | 275 | extern unsigned long shrink_all_memory(unsigned long nr_pages); |
267 | extern int vm_swappiness; | 276 | extern int vm_swappiness; |
268 | extern int remove_mapping(struct address_space *mapping, struct page *page); | 277 | extern int remove_mapping(struct address_space *mapping, struct page *page); |
269 | extern long vm_total_pages; | 278 | extern unsigned long vm_total_pages; |
270 | 279 | ||
271 | #ifdef CONFIG_NUMA | 280 | #ifdef CONFIG_NUMA |
272 | extern int zone_reclaim_mode; | 281 | extern int zone_reclaim_mode; |
@@ -330,8 +339,9 @@ int generic_swapfile_activate(struct swap_info_struct *, struct file *, | |||
330 | sector_t *); | 339 | sector_t *); |
331 | 340 | ||
332 | /* linux/mm/swap_state.c */ | 341 | /* linux/mm/swap_state.c */ |
333 | extern struct address_space swapper_space; | 342 | extern struct address_space swapper_spaces[]; |
334 | #define total_swapcache_pages swapper_space.nrpages | 343 | #define swap_address_space(entry) (&swapper_spaces[swp_type(entry)]) |
344 | extern unsigned long total_swapcache_pages(void); | ||
335 | extern void show_swap_cache_info(void); | 345 | extern void show_swap_cache_info(void); |
336 | extern int add_to_swap(struct page *); | 346 | extern int add_to_swap(struct page *); |
337 | extern int add_to_swap_cache(struct page *, swp_entry_t, gfp_t); | 347 | extern int add_to_swap_cache(struct page *, swp_entry_t, gfp_t); |
@@ -346,8 +356,20 @@ extern struct page *swapin_readahead(swp_entry_t, gfp_t, | |||
346 | struct vm_area_struct *vma, unsigned long addr); | 356 | struct vm_area_struct *vma, unsigned long addr); |
347 | 357 | ||
348 | /* linux/mm/swapfile.c */ | 358 | /* linux/mm/swapfile.c */ |
349 | extern long nr_swap_pages; | 359 | extern atomic_long_t nr_swap_pages; |
350 | extern long total_swap_pages; | 360 | extern long total_swap_pages; |
361 | |||
362 | /* Swap 50% full? Release swapcache more aggressively.. */ | ||
363 | static inline bool vm_swap_full(void) | ||
364 | { | ||
365 | return atomic_long_read(&nr_swap_pages) * 2 < total_swap_pages; | ||
366 | } | ||
367 | |||
368 | static inline long get_nr_swap_pages(void) | ||
369 | { | ||
370 | return atomic_long_read(&nr_swap_pages); | ||
371 | } | ||
372 | |||
351 | extern void si_swapinfo(struct sysinfo *); | 373 | extern void si_swapinfo(struct sysinfo *); |
352 | extern swp_entry_t get_swap_page(void); | 374 | extern swp_entry_t get_swap_page(void); |
353 | extern swp_entry_t get_swap_page_of_type(int); | 375 | extern swp_entry_t get_swap_page_of_type(int); |
@@ -380,9 +402,10 @@ mem_cgroup_uncharge_swapcache(struct page *page, swp_entry_t ent, bool swapout) | |||
380 | 402 | ||
381 | #else /* CONFIG_SWAP */ | 403 | #else /* CONFIG_SWAP */ |
382 | 404 | ||
383 | #define nr_swap_pages 0L | 405 | #define get_nr_swap_pages() 0L |
384 | #define total_swap_pages 0L | 406 | #define total_swap_pages 0L |
385 | #define total_swapcache_pages 0UL | 407 | #define total_swapcache_pages() 0UL |
408 | #define vm_swap_full() 0 | ||
386 | 409 | ||
387 | #define si_swapinfo(val) \ | 410 | #define si_swapinfo(val) \ |
388 | do { (val)->freeswap = (val)->totalswap = 0; } while (0) | 411 | do { (val)->freeswap = (val)->totalswap = 0; } while (0) |
diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h index fce0a2799d43..bd6cf61142be 100644 --- a/include/linux/vm_event_item.h +++ b/include/linux/vm_event_item.h | |||
@@ -36,7 +36,6 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, | |||
36 | #endif | 36 | #endif |
37 | PGINODESTEAL, SLABS_SCANNED, KSWAPD_INODESTEAL, | 37 | PGINODESTEAL, SLABS_SCANNED, KSWAPD_INODESTEAL, |
38 | KSWAPD_LOW_WMARK_HIT_QUICKLY, KSWAPD_HIGH_WMARK_HIT_QUICKLY, | 38 | KSWAPD_LOW_WMARK_HIT_QUICKLY, KSWAPD_HIGH_WMARK_HIT_QUICKLY, |
39 | KSWAPD_SKIP_CONGESTION_WAIT, | ||
40 | PAGEOUTRUN, ALLOCSTALL, PGROTATED, | 39 | PAGEOUTRUN, ALLOCSTALL, PGROTATED, |
41 | #ifdef CONFIG_NUMA_BALANCING | 40 | #ifdef CONFIG_NUMA_BALANCING |
42 | NUMA_PTE_UPDATES, | 41 | NUMA_PTE_UPDATES, |
diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h index a13291f7da88..5fd71a7d0dfd 100644 --- a/include/linux/vmstat.h +++ b/include/linux/vmstat.h | |||
@@ -85,7 +85,7 @@ static inline void vm_events_fold_cpu(int cpu) | |||
85 | #define count_vm_numa_events(x, y) count_vm_events(x, y) | 85 | #define count_vm_numa_events(x, y) count_vm_events(x, y) |
86 | #else | 86 | #else |
87 | #define count_vm_numa_event(x) do {} while (0) | 87 | #define count_vm_numa_event(x) do {} while (0) |
88 | #define count_vm_numa_events(x, y) do {} while (0) | 88 | #define count_vm_numa_events(x, y) do { (void)(y); } while (0) |
89 | #endif /* CONFIG_NUMA_BALANCING */ | 89 | #endif /* CONFIG_NUMA_BALANCING */ |
90 | 90 | ||
91 | #define __count_zone_vm_events(item, zone, delta) \ | 91 | #define __count_zone_vm_events(item, zone, delta) \ |