summaryrefslogtreecommitdiffstats
path: root/mm/vmscan.c
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2019-07-17 11:58:04 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2019-07-17 11:58:04 -0400
commit57a8ec387e1441ea5e1232bc0749fb99a8cba7e7 (patch)
treeb5fb03fc6bc5754de8b5b1f8b0e4f36d67c8315c /mm/vmscan.c
parent0a8ad0ffa4d80a544f6cbff703bf6394339afcdf (diff)
parent43e11fa2d1d3b6e35629fa556eb7d571edba2010 (diff)
Merge branch 'akpm' (patches from Andrew)
Merge more updates from Andrew Morton: "VM: - z3fold fixes and enhancements by Henry Burns and Vitaly Wool - more accurate reclaimed slab caches calculations by Yafang Shao - fix MAP_UNINITIALIZED UAPI symbol to not depend on config, by Christoph Hellwig - !CONFIG_MMU fixes by Christoph Hellwig - new novmcoredd parameter to omit device dumps from vmcore, by Kairui Song - new test_meminit module for testing heap and pagealloc initialization, by Alexander Potapenko - ioremap improvements for huge mappings, by Anshuman Khandual - generalize kprobe page fault handling, by Anshuman Khandual - device-dax hotplug fixes and improvements, by Pavel Tatashin - enable synchronous DAX fault on powerpc, by Aneesh Kumar K.V - add pte_devmap() support for arm64, by Robin Murphy - unify locked_vm accounting with a helper, by Daniel Jordan - several misc fixes core/lib: - new typeof_member() macro including some users, by Alexey Dobriyan - make BIT() and GENMASK() available in asm, by Masahiro Yamada - changed LIST_POISON2 on x86_64 to 0xdead000000000122 for better code generation, by Alexey Dobriyan - rbtree code size optimizations, by Michel Lespinasse - convert struct pid count to refcount_t, by Joel Fernandes get_maintainer.pl: - add --no-moderated switch to skip moderated ML's, by Joe Perches misc: - ptrace PTRACE_GET_SYSCALL_INFO interface - coda updates - gdb scripts, various" [ Using merge message suggestion from Vlastimil Babka, with some editing - Linus ] * emailed patches from Andrew Morton <akpm@linux-foundation.org>: (100 commits) fs/select.c: use struct_size() in kmalloc() mm: add account_locked_vm utility function arm64: mm: implement pte_devmap support mm: introduce ARCH_HAS_PTE_DEVMAP mm: clean up is_device_*_page() definitions mm/mmap: move common defines to mman-common.h mm: move MAP_SYNC to asm-generic/mman-common.h device-dax: "Hotremove" persistent memory that is used like normal RAM mm/hotplug: make remove_memory() interface usable device-dax: fix memory and resource leak if hotplug fails include/linux/lz4.h: fix spelling and copy-paste errors in documentation ipc/mqueue.c: only perform resource calculation if user valid include/asm-generic/bug.h: fix "cut here" for WARN_ON for __WARN_TAINT architectures scripts/gdb: add helpers to find and list devices scripts/gdb: add lx-genpd-summary command drivers/pps/pps.c: clear offset flags in PPS_SETPARAMS ioctl kernel/pid.c: convert struct pid count to refcount_t drivers/rapidio/devices/rio_mport_cdev.c: NUL terminate some strings select: shift restore_saved_sigmask_unless() into poll_select_copy_remaining() select: change do_poll() to return -ERESTARTNOHAND rather than -EINTR ...
Diffstat (limited to 'mm/vmscan.c')
-rw-r--r--mm/vmscan.c44
1 files changed, 29 insertions, 15 deletions
diff --git a/mm/vmscan.c b/mm/vmscan.c
index f8e3dcd527b8..44df66a98f2a 100644
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -131,6 +131,9 @@ struct scan_control {
131 unsigned int file_taken; 131 unsigned int file_taken;
132 unsigned int taken; 132 unsigned int taken;
133 } nr; 133 } nr;
134
135 /* for recording the reclaimed slab by now */
136 struct reclaim_state reclaim_state;
134}; 137};
135 138
136#ifdef ARCH_HAS_PREFETCH 139#ifdef ARCH_HAS_PREFETCH
@@ -238,6 +241,18 @@ static void unregister_memcg_shrinker(struct shrinker *shrinker)
238} 241}
239#endif /* CONFIG_MEMCG_KMEM */ 242#endif /* CONFIG_MEMCG_KMEM */
240 243
244static void set_task_reclaim_state(struct task_struct *task,
245 struct reclaim_state *rs)
246{
247 /* Check for an overwrite */
248 WARN_ON_ONCE(rs && task->reclaim_state);
249
250 /* Check for the nulling of an already-nulled member */
251 WARN_ON_ONCE(!rs && !task->reclaim_state);
252
253 task->reclaim_state = rs;
254}
255
241#ifdef CONFIG_MEMCG 256#ifdef CONFIG_MEMCG
242static bool global_reclaim(struct scan_control *sc) 257static bool global_reclaim(struct scan_control *sc)
243{ 258{
@@ -3191,11 +3206,13 @@ unsigned long try_to_free_pages(struct zonelist *zonelist, int order,
3191 if (throttle_direct_reclaim(sc.gfp_mask, zonelist, nodemask)) 3206 if (throttle_direct_reclaim(sc.gfp_mask, zonelist, nodemask))
3192 return 1; 3207 return 1;
3193 3208
3209 set_task_reclaim_state(current, &sc.reclaim_state);
3194 trace_mm_vmscan_direct_reclaim_begin(order, sc.gfp_mask); 3210 trace_mm_vmscan_direct_reclaim_begin(order, sc.gfp_mask);
3195 3211
3196 nr_reclaimed = do_try_to_free_pages(zonelist, &sc); 3212 nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3197 3213
3198 trace_mm_vmscan_direct_reclaim_end(nr_reclaimed); 3214 trace_mm_vmscan_direct_reclaim_end(nr_reclaimed);
3215 set_task_reclaim_state(current, NULL);
3199 3216
3200 return nr_reclaimed; 3217 return nr_reclaimed;
3201} 3218}
@@ -3218,6 +3235,7 @@ unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg,
3218 }; 3235 };
3219 unsigned long lru_pages; 3236 unsigned long lru_pages;
3220 3237
3238 set_task_reclaim_state(current, &sc.reclaim_state);
3221 sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) | 3239 sc.gfp_mask = (gfp_mask & GFP_RECLAIM_MASK) |
3222 (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK); 3240 (GFP_HIGHUSER_MOVABLE & ~GFP_RECLAIM_MASK);
3223 3241
@@ -3235,7 +3253,9 @@ unsigned long mem_cgroup_shrink_node(struct mem_cgroup *memcg,
3235 3253
3236 trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed); 3254 trace_mm_vmscan_memcg_softlimit_reclaim_end(sc.nr_reclaimed);
3237 3255
3256 set_task_reclaim_state(current, NULL);
3238 *nr_scanned = sc.nr_scanned; 3257 *nr_scanned = sc.nr_scanned;
3258
3239 return sc.nr_reclaimed; 3259 return sc.nr_reclaimed;
3240} 3260}
3241 3261
@@ -3262,6 +3282,7 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
3262 .may_shrinkslab = 1, 3282 .may_shrinkslab = 1,
3263 }; 3283 };
3264 3284
3285 set_task_reclaim_state(current, &sc.reclaim_state);
3265 /* 3286 /*
3266 * Unlike direct reclaim via alloc_pages(), memcg's reclaim doesn't 3287 * Unlike direct reclaim via alloc_pages(), memcg's reclaim doesn't
3267 * take care of from where we get pages. So the node where we start the 3288 * take care of from where we get pages. So the node where we start the
@@ -3282,6 +3303,7 @@ unsigned long try_to_free_mem_cgroup_pages(struct mem_cgroup *memcg,
3282 psi_memstall_leave(&pflags); 3303 psi_memstall_leave(&pflags);
3283 3304
3284 trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed); 3305 trace_mm_vmscan_memcg_reclaim_end(nr_reclaimed);
3306 set_task_reclaim_state(current, NULL);
3285 3307
3286 return nr_reclaimed; 3308 return nr_reclaimed;
3287} 3309}
@@ -3483,6 +3505,7 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int classzone_idx)
3483 .may_unmap = 1, 3505 .may_unmap = 1,
3484 }; 3506 };
3485 3507
3508 set_task_reclaim_state(current, &sc.reclaim_state);
3486 psi_memstall_enter(&pflags); 3509 psi_memstall_enter(&pflags);
3487 __fs_reclaim_acquire(); 3510 __fs_reclaim_acquire();
3488 3511
@@ -3664,6 +3687,8 @@ out:
3664 snapshot_refaults(NULL, pgdat); 3687 snapshot_refaults(NULL, pgdat);
3665 __fs_reclaim_release(); 3688 __fs_reclaim_release();
3666 psi_memstall_leave(&pflags); 3689 psi_memstall_leave(&pflags);
3690 set_task_reclaim_state(current, NULL);
3691
3667 /* 3692 /*
3668 * Return the order kswapd stopped reclaiming at as 3693 * Return the order kswapd stopped reclaiming at as
3669 * prepare_kswapd_sleep() takes it into account. If another caller 3694 * prepare_kswapd_sleep() takes it into account. If another caller
@@ -3787,15 +3812,10 @@ static int kswapd(void *p)
3787 unsigned int classzone_idx = MAX_NR_ZONES - 1; 3812 unsigned int classzone_idx = MAX_NR_ZONES - 1;
3788 pg_data_t *pgdat = (pg_data_t*)p; 3813 pg_data_t *pgdat = (pg_data_t*)p;
3789 struct task_struct *tsk = current; 3814 struct task_struct *tsk = current;
3790
3791 struct reclaim_state reclaim_state = {
3792 .reclaimed_slab = 0,
3793 };
3794 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id); 3815 const struct cpumask *cpumask = cpumask_of_node(pgdat->node_id);
3795 3816
3796 if (!cpumask_empty(cpumask)) 3817 if (!cpumask_empty(cpumask))
3797 set_cpus_allowed_ptr(tsk, cpumask); 3818 set_cpus_allowed_ptr(tsk, cpumask);
3798 current->reclaim_state = &reclaim_state;
3799 3819
3800 /* 3820 /*
3801 * Tell the memory management that we're a "memory allocator", 3821 * Tell the memory management that we're a "memory allocator",
@@ -3857,7 +3877,6 @@ kswapd_try_sleep:
3857 } 3877 }
3858 3878
3859 tsk->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD); 3879 tsk->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD);
3860 current->reclaim_state = NULL;
3861 3880
3862 return 0; 3881 return 0;
3863} 3882}
@@ -3922,7 +3941,6 @@ void wakeup_kswapd(struct zone *zone, gfp_t gfp_flags, int order,
3922 */ 3941 */
3923unsigned long shrink_all_memory(unsigned long nr_to_reclaim) 3942unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
3924{ 3943{
3925 struct reclaim_state reclaim_state;
3926 struct scan_control sc = { 3944 struct scan_control sc = {
3927 .nr_to_reclaim = nr_to_reclaim, 3945 .nr_to_reclaim = nr_to_reclaim,
3928 .gfp_mask = GFP_HIGHUSER_MOVABLE, 3946 .gfp_mask = GFP_HIGHUSER_MOVABLE,
@@ -3934,18 +3952,16 @@ unsigned long shrink_all_memory(unsigned long nr_to_reclaim)
3934 .hibernation_mode = 1, 3952 .hibernation_mode = 1,
3935 }; 3953 };
3936 struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask); 3954 struct zonelist *zonelist = node_zonelist(numa_node_id(), sc.gfp_mask);
3937 struct task_struct *p = current;
3938 unsigned long nr_reclaimed; 3955 unsigned long nr_reclaimed;
3939 unsigned int noreclaim_flag; 3956 unsigned int noreclaim_flag;
3940 3957
3941 fs_reclaim_acquire(sc.gfp_mask); 3958 fs_reclaim_acquire(sc.gfp_mask);
3942 noreclaim_flag = memalloc_noreclaim_save(); 3959 noreclaim_flag = memalloc_noreclaim_save();
3943 reclaim_state.reclaimed_slab = 0; 3960 set_task_reclaim_state(current, &sc.reclaim_state);
3944 p->reclaim_state = &reclaim_state;
3945 3961
3946 nr_reclaimed = do_try_to_free_pages(zonelist, &sc); 3962 nr_reclaimed = do_try_to_free_pages(zonelist, &sc);
3947 3963
3948 p->reclaim_state = NULL; 3964 set_task_reclaim_state(current, NULL);
3949 memalloc_noreclaim_restore(noreclaim_flag); 3965 memalloc_noreclaim_restore(noreclaim_flag);
3950 fs_reclaim_release(sc.gfp_mask); 3966 fs_reclaim_release(sc.gfp_mask);
3951 3967
@@ -4110,7 +4126,6 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in
4110 /* Minimum pages needed in order to stay on node */ 4126 /* Minimum pages needed in order to stay on node */
4111 const unsigned long nr_pages = 1 << order; 4127 const unsigned long nr_pages = 1 << order;
4112 struct task_struct *p = current; 4128 struct task_struct *p = current;
4113 struct reclaim_state reclaim_state;
4114 unsigned int noreclaim_flag; 4129 unsigned int noreclaim_flag;
4115 struct scan_control sc = { 4130 struct scan_control sc = {
4116 .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX), 4131 .nr_to_reclaim = max(nr_pages, SWAP_CLUSTER_MAX),
@@ -4135,8 +4150,7 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in
4135 */ 4150 */
4136 noreclaim_flag = memalloc_noreclaim_save(); 4151 noreclaim_flag = memalloc_noreclaim_save();
4137 p->flags |= PF_SWAPWRITE; 4152 p->flags |= PF_SWAPWRITE;
4138 reclaim_state.reclaimed_slab = 0; 4153 set_task_reclaim_state(p, &sc.reclaim_state);
4139 p->reclaim_state = &reclaim_state;
4140 4154
4141 if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages) { 4155 if (node_pagecache_reclaimable(pgdat) > pgdat->min_unmapped_pages) {
4142 /* 4156 /*
@@ -4148,7 +4162,7 @@ static int __node_reclaim(struct pglist_data *pgdat, gfp_t gfp_mask, unsigned in
4148 } while (sc.nr_reclaimed < nr_pages && --sc.priority >= 0); 4162 } while (sc.nr_reclaimed < nr_pages && --sc.priority >= 0);
4149 } 4163 }
4150 4164
4151 p->reclaim_state = NULL; 4165 set_task_reclaim_state(p, NULL);
4152 current->flags &= ~PF_SWAPWRITE; 4166 current->flags &= ~PF_SWAPWRITE;
4153 memalloc_noreclaim_restore(noreclaim_flag); 4167 memalloc_noreclaim_restore(noreclaim_flag);
4154 fs_reclaim_release(sc.gfp_mask); 4168 fs_reclaim_release(sc.gfp_mask);