aboutsummaryrefslogtreecommitdiffstats
path: root/mm/memcontrol.c
diff options
context:
space:
mode:
Diffstat (limited to 'mm/memcontrol.c')
-rw-r--r--mm/memcontrol.c18
1 files changed, 10 insertions, 8 deletions
diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 6f8b5b3b38b2..10846b9656aa 100644
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -66,11 +66,10 @@ struct mem_cgroup_stat {
66/* 66/*
67 * For accounting under irq disable, no need for increment preempt count. 67 * For accounting under irq disable, no need for increment preempt count.
68 */ 68 */
69static void __mem_cgroup_stat_add_safe(struct mem_cgroup_stat *stat, 69static inline void __mem_cgroup_stat_add_safe(struct mem_cgroup_stat_cpu *stat,
70 enum mem_cgroup_stat_index idx, int val) 70 enum mem_cgroup_stat_index idx, int val)
71{ 71{
72 int cpu = smp_processor_id(); 72 stat->count[idx] += val;
73 stat->cpustat[cpu].count[idx] += val;
74} 73}
75 74
76static s64 mem_cgroup_read_stat(struct mem_cgroup_stat *stat, 75static s64 mem_cgroup_read_stat(struct mem_cgroup_stat *stat,
@@ -190,18 +189,21 @@ static void mem_cgroup_charge_statistics(struct mem_cgroup *mem, int flags,
190{ 189{
191 int val = (charge)? 1 : -1; 190 int val = (charge)? 1 : -1;
192 struct mem_cgroup_stat *stat = &mem->stat; 191 struct mem_cgroup_stat *stat = &mem->stat;
192 struct mem_cgroup_stat_cpu *cpustat;
193 193
194 VM_BUG_ON(!irqs_disabled()); 194 VM_BUG_ON(!irqs_disabled());
195
196 cpustat = &stat->cpustat[smp_processor_id()];
195 if (flags & PAGE_CGROUP_FLAG_CACHE) 197 if (flags & PAGE_CGROUP_FLAG_CACHE)
196 __mem_cgroup_stat_add_safe(stat, MEM_CGROUP_STAT_CACHE, val); 198 __mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_CACHE, val);
197 else 199 else
198 __mem_cgroup_stat_add_safe(stat, MEM_CGROUP_STAT_RSS, val); 200 __mem_cgroup_stat_add_safe(cpustat, MEM_CGROUP_STAT_RSS, val);
199 201
200 if (charge) 202 if (charge)
201 __mem_cgroup_stat_add_safe(stat, 203 __mem_cgroup_stat_add_safe(cpustat,
202 MEM_CGROUP_STAT_PGPGIN_COUNT, 1); 204 MEM_CGROUP_STAT_PGPGIN_COUNT, 1);
203 else 205 else
204 __mem_cgroup_stat_add_safe(stat, 206 __mem_cgroup_stat_add_safe(cpustat,
205 MEM_CGROUP_STAT_PGPGOUT_COUNT, 1); 207 MEM_CGROUP_STAT_PGPGOUT_COUNT, 1);
206} 208}
207 209
@@ -558,7 +560,7 @@ static int mem_cgroup_charge_common(struct page *page, struct mm_struct *mm,
558 css_get(&memcg->css); 560 css_get(&memcg->css);
559 } 561 }
560 562
561 while (res_counter_charge(&mem->res, PAGE_SIZE)) { 563 while (unlikely(res_counter_charge(&mem->res, PAGE_SIZE))) {
562 if (!(gfp_mask & __GFP_WAIT)) 564 if (!(gfp_mask & __GFP_WAIT))
563 goto out; 565 goto out;
564 566