aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--include/linux/pcounter.h80
-rw-r--r--lib/pcounter.c42
2 files changed, 63 insertions, 59 deletions
diff --git a/include/linux/pcounter.h b/include/linux/pcounter.h
index 9c4760a328f3..a82d9f2628ca 100644
--- a/include/linux/pcounter.h
+++ b/include/linux/pcounter.h
@@ -1,41 +1,39 @@
1#ifndef __LINUX_PCOUNTER_H 1#ifndef __LINUX_PCOUNTER_H
2#define __LINUX_PCOUNTER_H 2#define __LINUX_PCOUNTER_H
3 3/*
4 * Using a dynamic percpu 'int' variable has a cost :
5 * 1) Extra dereference
6 * Current per_cpu_ptr() implementation uses an array per 'percpu variable'.
7 * 2) memory cost of NR_CPUS*(32+sizeof(void *)) instead of num_possible_cpus()*4
8 *
9 * This pcounter implementation is an abstraction to be able to use
10 * either a static or a dynamic per cpu variable.
11 * One dynamic per cpu variable gets a fast & cheap implementation, we can
12 * change pcounter implementation too.
13 */
4struct pcounter { 14struct pcounter {
5#ifdef CONFIG_SMP 15#ifdef CONFIG_SMP
6 void (*add)(struct pcounter *self, int inc); 16 void (*add)(struct pcounter *self, int inc);
7 int (*getval)(const struct pcounter *self); 17 int (*getval)(const struct pcounter *self, int cpu);
8 int *per_cpu_values; 18 int *per_cpu_values;
9#else 19#else
10 int val; 20 int val;
11#endif 21#endif
12}; 22};
13 23
14/*
15 * Special macros to let pcounters use a fast version of {getvalue|add}
16 * using a static percpu variable per pcounter instead of an allocated one,
17 * saving one dereference.
18 * This might be changed if/when dynamic percpu vars become fast.
19 */
20#ifdef CONFIG_SMP 24#ifdef CONFIG_SMP
21#include <linux/cpumask.h>
22#include <linux/percpu.h> 25#include <linux/percpu.h>
23 26
24#define DEFINE_PCOUNTER(NAME) \ 27#define DEFINE_PCOUNTER(NAME) \
25static DEFINE_PER_CPU(int, NAME##_pcounter_values); \ 28static DEFINE_PER_CPU(int, NAME##_pcounter_values); \
26static void NAME##_pcounter_add(struct pcounter *self, int inc) \ 29static void NAME##_pcounter_add(struct pcounter *self, int val) \
27{ \ 30{ \
28 __get_cpu_var(NAME##_pcounter_values) += inc; \ 31 __get_cpu_var(NAME##_pcounter_values) += val; \
29} \ 32} \
30 \ 33static int NAME##_pcounter_getval(const struct pcounter *self, int cpu) \
31static int NAME##_pcounter_getval(const struct pcounter *self) \ 34{ \
32{ \ 35 return per_cpu(NAME##_pcounter_values, cpu); \
33 int res = 0, cpu; \ 36} \
34 \
35 for_each_possible_cpu(cpu) \
36 res += per_cpu(NAME##_pcounter_values, cpu); \
37 return res; \
38}
39 37
40#define PCOUNTER_MEMBER_INITIALIZER(NAME, MEMBER) \ 38#define PCOUNTER_MEMBER_INITIALIZER(NAME, MEMBER) \
41 MEMBER = { \ 39 MEMBER = { \
@@ -43,42 +41,16 @@ static int NAME##_pcounter_getval(const struct pcounter *self) \
43 .getval = NAME##_pcounter_getval, \ 41 .getval = NAME##_pcounter_getval, \
44 } 42 }
45 43
46extern void pcounter_def_add(struct pcounter *self, int inc);
47extern int pcounter_def_getval(const struct pcounter *self);
48
49static inline int pcounter_alloc(struct pcounter *self)
50{
51 int rc = 0;
52 if (self->add == NULL) {
53 self->per_cpu_values = alloc_percpu(int);
54 if (self->per_cpu_values != NULL) {
55 self->add = pcounter_def_add;
56 self->getval = pcounter_def_getval;
57 } else
58 rc = 1;
59 }
60 return rc;
61}
62
63static inline void pcounter_free(struct pcounter *self)
64{
65 if (self->per_cpu_values != NULL) {
66 free_percpu(self->per_cpu_values);
67 self->per_cpu_values = NULL;
68 self->getval = NULL;
69 self->add = NULL;
70 }
71}
72 44
73static inline void pcounter_add(struct pcounter *self, int inc) 45static inline void pcounter_add(struct pcounter *self, int inc)
74{ 46{
75 self->add(self, inc); 47 self->add(self, inc);
76} 48}
77 49
78static inline int pcounter_getval(const struct pcounter *self) 50extern int pcounter_getval(const struct pcounter *self);
79{ 51extern int pcounter_alloc(struct pcounter *self);
80 return self->getval(self); 52extern void pcounter_free(struct pcounter *self);
81} 53
82 54
83#else /* CONFIG_SMP */ 55#else /* CONFIG_SMP */
84 56
diff --git a/lib/pcounter.c b/lib/pcounter.c
index 93feea598251..9b56807da93b 100644
--- a/lib/pcounter.c
+++ b/lib/pcounter.c
@@ -7,20 +7,52 @@
7#include <linux/module.h> 7#include <linux/module.h>
8#include <linux/pcounter.h> 8#include <linux/pcounter.h>
9#include <linux/smp.h> 9#include <linux/smp.h>
10#include <linux/cpumask.h>
10 11
11void pcounter_def_add(struct pcounter *self, int inc) 12static void pcounter_dyn_add(struct pcounter *self, int inc)
12{ 13{
13 per_cpu_ptr(self->per_cpu_values, smp_processor_id())[0] += inc; 14 per_cpu_ptr(self->per_cpu_values, smp_processor_id())[0] += inc;
14} 15}
15 16
16EXPORT_SYMBOL_GPL(pcounter_def_add); 17static int pcounter_dyn_getval(const struct pcounter *self, int cpu)
18{
19 return per_cpu_ptr(self->per_cpu_values, cpu)[0];
20}
17 21
18int pcounter_def_getval(const struct pcounter *self) 22int pcounter_getval(const struct pcounter *self)
19{ 23{
20 int res = 0, cpu; 24 int res = 0, cpu;
25
21 for_each_possible_cpu(cpu) 26 for_each_possible_cpu(cpu)
22 res += per_cpu_ptr(self->per_cpu_values, cpu)[0]; 27 res += self->getval(self, cpu);
28
23 return res; 29 return res;
24} 30}
31EXPORT_SYMBOL_GPL(pcounter_getval);
32
33int pcounter_alloc(struct pcounter *self)
34{
35 int rc = 0;
36 if (self->add == NULL) {
37 self->per_cpu_values = alloc_percpu(int);
38 if (self->per_cpu_values != NULL) {
39 self->add = pcounter_dyn_add;
40 self->getval = pcounter_dyn_getval;
41 } else
42 rc = 1;
43 }
44 return rc;
45}
46EXPORT_SYMBOL_GPL(pcounter_alloc);
47
48void pcounter_free(struct pcounter *self)
49{
50 if (self->per_cpu_values != NULL) {
51 free_percpu(self->per_cpu_values);
52 self->per_cpu_values = NULL;
53 self->getval = NULL;
54 self->add = NULL;
55 }
56}
57EXPORT_SYMBOL_GPL(pcounter_free);
25 58
26EXPORT_SYMBOL_GPL(pcounter_def_getval);
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-26 00:51:42 -0500