diff options
-rw-r--r-- | lib/percpu_counter.c | 15 |
1 files changed, 9 insertions, 6 deletions
diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c index 93c5d5ecff4e..7473ee3b4ee7 100644 --- a/lib/percpu_counter.c +++ b/lib/percpu_counter.c | |||
@@ -60,14 +60,15 @@ static inline void debug_percpu_counter_deactivate(struct percpu_counter *fbc) | |||
60 | void percpu_counter_set(struct percpu_counter *fbc, s64 amount) | 60 | void percpu_counter_set(struct percpu_counter *fbc, s64 amount) |
61 | { | 61 | { |
62 | int cpu; | 62 | int cpu; |
63 | unsigned long flags; | ||
63 | 64 | ||
64 | raw_spin_lock(&fbc->lock); | 65 | raw_spin_lock_irqsave(&fbc->lock, flags); |
65 | for_each_possible_cpu(cpu) { | 66 | for_each_possible_cpu(cpu) { |
66 | s32 *pcount = per_cpu_ptr(fbc->counters, cpu); | 67 | s32 *pcount = per_cpu_ptr(fbc->counters, cpu); |
67 | *pcount = 0; | 68 | *pcount = 0; |
68 | } | 69 | } |
69 | fbc->count = amount; | 70 | fbc->count = amount; |
70 | raw_spin_unlock(&fbc->lock); | 71 | raw_spin_unlock_irqrestore(&fbc->lock, flags); |
71 | } | 72 | } |
72 | EXPORT_SYMBOL(percpu_counter_set); | 73 | EXPORT_SYMBOL(percpu_counter_set); |
73 | 74 | ||
@@ -78,9 +79,10 @@ void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch) | |||
78 | preempt_disable(); | 79 | preempt_disable(); |
79 | count = __this_cpu_read(*fbc->counters) + amount; | 80 | count = __this_cpu_read(*fbc->counters) + amount; |
80 | if (count >= batch || count <= -batch) { | 81 | if (count >= batch || count <= -batch) { |
81 | raw_spin_lock(&fbc->lock); | 82 | unsigned long flags; |
83 | raw_spin_lock_irqsave(&fbc->lock, flags); | ||
82 | fbc->count += count; | 84 | fbc->count += count; |
83 | raw_spin_unlock(&fbc->lock); | 85 | raw_spin_unlock_irqrestore(&fbc->lock, flags); |
84 | __this_cpu_write(*fbc->counters, 0); | 86 | __this_cpu_write(*fbc->counters, 0); |
85 | } else { | 87 | } else { |
86 | __this_cpu_write(*fbc->counters, count); | 88 | __this_cpu_write(*fbc->counters, count); |
@@ -97,14 +99,15 @@ s64 __percpu_counter_sum(struct percpu_counter *fbc) | |||
97 | { | 99 | { |
98 | s64 ret; | 100 | s64 ret; |
99 | int cpu; | 101 | int cpu; |
102 | unsigned long flags; | ||
100 | 103 | ||
101 | raw_spin_lock(&fbc->lock); | 104 | raw_spin_lock_irqsave(&fbc->lock, flags); |
102 | ret = fbc->count; | 105 | ret = fbc->count; |
103 | for_each_online_cpu(cpu) { | 106 | for_each_online_cpu(cpu) { |
104 | s32 *pcount = per_cpu_ptr(fbc->counters, cpu); | 107 | s32 *pcount = per_cpu_ptr(fbc->counters, cpu); |
105 | ret += *pcount; | 108 | ret += *pcount; |
106 | } | 109 | } |
107 | raw_spin_unlock(&fbc->lock); | 110 | raw_spin_unlock_irqrestore(&fbc->lock, flags); |
108 | return ret; | 111 | return ret; |
109 | } | 112 | } |
110 | EXPORT_SYMBOL(__percpu_counter_sum); | 113 | EXPORT_SYMBOL(__percpu_counter_sum); |