diff options
| -rw-r--r-- | lib/Makefile | 1 | ||||
| -rw-r--r-- | lib/percpu_counter.c | 46 | ||||
| -rw-r--r-- | mm/swap.c | 42 |
3 files changed, 47 insertions, 42 deletions
diff --git a/lib/Makefile b/lib/Makefile index b830c9a15541..79358ad1f113 100644 --- a/lib/Makefile +++ b/lib/Makefile | |||
| @@ -46,6 +46,7 @@ obj-$(CONFIG_TEXTSEARCH) += textsearch.o | |||
| 46 | obj-$(CONFIG_TEXTSEARCH_KMP) += ts_kmp.o | 46 | obj-$(CONFIG_TEXTSEARCH_KMP) += ts_kmp.o |
| 47 | obj-$(CONFIG_TEXTSEARCH_BM) += ts_bm.o | 47 | obj-$(CONFIG_TEXTSEARCH_BM) += ts_bm.o |
| 48 | obj-$(CONFIG_TEXTSEARCH_FSM) += ts_fsm.o | 48 | obj-$(CONFIG_TEXTSEARCH_FSM) += ts_fsm.o |
| 49 | obj-$(CONFIG_SMP) += percpu_counter.o | ||
| 49 | 50 | ||
| 50 | obj-$(CONFIG_SWIOTLB) += swiotlb.o | 51 | obj-$(CONFIG_SWIOTLB) += swiotlb.o |
| 51 | 52 | ||
diff --git a/lib/percpu_counter.c b/lib/percpu_counter.c new file mode 100644 index 000000000000..7a87003f8e8f --- /dev/null +++ b/lib/percpu_counter.c | |||
| @@ -0,0 +1,46 @@ | |||
| 1 | /* | ||
| 2 | * Fast batching percpu counters. | ||
| 3 | */ | ||
| 4 | |||
| 5 | #include <linux/percpu_counter.h> | ||
| 6 | #include <linux/module.h> | ||
| 7 | |||
| 8 | void percpu_counter_mod(struct percpu_counter *fbc, long amount) | ||
| 9 | { | ||
| 10 | long count; | ||
| 11 | long *pcount; | ||
| 12 | int cpu = get_cpu(); | ||
| 13 | |||
| 14 | pcount = per_cpu_ptr(fbc->counters, cpu); | ||
| 15 | count = *pcount + amount; | ||
| 16 | if (count >= FBC_BATCH || count <= -FBC_BATCH) { | ||
| 17 | spin_lock(&fbc->lock); | ||
| 18 | fbc->count += count; | ||
| 19 | *pcount = 0; | ||
| 20 | spin_unlock(&fbc->lock); | ||
| 21 | } else { | ||
| 22 | *pcount = count; | ||
| 23 | } | ||
| 24 | put_cpu(); | ||
| 25 | } | ||
| 26 | EXPORT_SYMBOL(percpu_counter_mod); | ||
| 27 | |||
| 28 | /* | ||
| 29 | * Add up all the per-cpu counts, return the result. This is a more accurate | ||
| 30 | * but much slower version of percpu_counter_read_positive() | ||
| 31 | */ | ||
| 32 | long percpu_counter_sum(struct percpu_counter *fbc) | ||
| 33 | { | ||
| 34 | long ret; | ||
| 35 | int cpu; | ||
| 36 | |||
| 37 | spin_lock(&fbc->lock); | ||
| 38 | ret = fbc->count; | ||
| 39 | for_each_possible_cpu(cpu) { | ||
| 40 | long *pcount = per_cpu_ptr(fbc->counters, cpu); | ||
| 41 | ret += *pcount; | ||
| 42 | } | ||
| 43 | spin_unlock(&fbc->lock); | ||
| 44 | return ret < 0 ? 0 : ret; | ||
| 45 | } | ||
| 46 | EXPORT_SYMBOL(percpu_counter_sum); | ||
| @@ -480,48 +480,6 @@ static int cpu_swap_callback(struct notifier_block *nfb, | |||
| 480 | #endif /* CONFIG_HOTPLUG_CPU */ | 480 | #endif /* CONFIG_HOTPLUG_CPU */ |
| 481 | #endif /* CONFIG_SMP */ | 481 | #endif /* CONFIG_SMP */ |
| 482 | 482 | ||
| 483 | #ifdef CONFIG_SMP | ||
| 484 | void percpu_counter_mod(struct percpu_counter *fbc, long amount) | ||
| 485 | { | ||
| 486 | long count; | ||
| 487 | long *pcount; | ||
| 488 | int cpu = get_cpu(); | ||
| 489 | |||
| 490 | pcount = per_cpu_ptr(fbc->counters, cpu); | ||
| 491 | count = *pcount + amount; | ||
| 492 | if (count >= FBC_BATCH || count <= -FBC_BATCH) { | ||
| 493 | spin_lock(&fbc->lock); | ||
| 494 | fbc->count += count; | ||
| 495 | *pcount = 0; | ||
| 496 | spin_unlock(&fbc->lock); | ||
| 497 | } else { | ||
| 498 | *pcount = count; | ||
| 499 | } | ||
| 500 | put_cpu(); | ||
| 501 | } | ||
| 502 | EXPORT_SYMBOL(percpu_counter_mod); | ||
| 503 | |||
| 504 | /* | ||
| 505 | * Add up all the per-cpu counts, return the result. This is a more accurate | ||
| 506 | * but much slower version of percpu_counter_read_positive() | ||
| 507 | */ | ||
| 508 | long percpu_counter_sum(struct percpu_counter *fbc) | ||
| 509 | { | ||
| 510 | long ret; | ||
| 511 | int cpu; | ||
| 512 | |||
| 513 | spin_lock(&fbc->lock); | ||
| 514 | ret = fbc->count; | ||
| 515 | for_each_possible_cpu(cpu) { | ||
| 516 | long *pcount = per_cpu_ptr(fbc->counters, cpu); | ||
| 517 | ret += *pcount; | ||
| 518 | } | ||
| 519 | spin_unlock(&fbc->lock); | ||
| 520 | return ret < 0 ? 0 : ret; | ||
| 521 | } | ||
| 522 | EXPORT_SYMBOL(percpu_counter_sum); | ||
| 523 | #endif | ||
| 524 | |||
| 525 | /* | 483 | /* |
| 526 | * Perform any setup for the swap system | 484 | * Perform any setup for the swap system |
| 527 | */ | 485 | */ |
