aboutsummaryrefslogtreecommitdiffstats
path: root/net/core
diff options
context:
space:
mode:
authorTimo Teräs <timo.teras@iki.fi>2010-04-06 20:30:04 -0400
committerDavid S. Miller <davem@davemloft.net>2010-04-07 06:43:18 -0400
commitfe1a5f031e76bd8761a7803d75b95ee96e84a574 (patch)
treec74392cef02c1529b00df6c5d0b8f4239fe091c3 /net/core
parent8020eb82d4c37d21dade0abeb8feed265a01819e (diff)
flow: virtualize flow cache entry methods
This allows to validate the cached object before returning it. It also allows to destruct object properly, if the last reference was held in flow cache. This is also a prepartion for caching bundles in the flow cache. In return for virtualizing the methods, we save on: - not having to regenerate the whole flow cache on policy removal: each flow matching a killed policy gets refreshed as the getter function notices it smartly. - we do not have to call flow_cache_flush from policy gc, since the flow cache now properly deletes the object if it had any references Signed-off-by: Timo Teras <timo.teras@iki.fi> Acked-by: Herbert Xu <herbert@gondor.apana.org.au> Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'net/core')
-rw-r--r--net/core/flow.c128
1 files changed, 68 insertions, 60 deletions
diff --git a/net/core/flow.c b/net/core/flow.c
index 1d27ca6b421d..521df52a77d2 100644
--- a/net/core/flow.c
+++ b/net/core/flow.c
@@ -26,17 +26,16 @@
26#include <linux/security.h> 26#include <linux/security.h>
27 27
28struct flow_cache_entry { 28struct flow_cache_entry {
29 struct flow_cache_entry *next; 29 struct flow_cache_entry *next;
30 u16 family; 30 u16 family;
31 u8 dir; 31 u8 dir;
32 u32 genid; 32 u32 genid;
33 struct flowi key; 33 struct flowi key;
34 void *object; 34 struct flow_cache_object *object;
35 atomic_t *object_ref;
36}; 35};
37 36
38struct flow_cache_percpu { 37struct flow_cache_percpu {
39 struct flow_cache_entry ** hash_table; 38 struct flow_cache_entry **hash_table;
40 int hash_count; 39 int hash_count;
41 u32 hash_rnd; 40 u32 hash_rnd;
42 int hash_rnd_recalc; 41 int hash_rnd_recalc;
@@ -44,7 +43,7 @@ struct flow_cache_percpu {
44}; 43};
45 44
46struct flow_flush_info { 45struct flow_flush_info {
47 struct flow_cache * cache; 46 struct flow_cache *cache;
48 atomic_t cpuleft; 47 atomic_t cpuleft;
49 struct completion completion; 48 struct completion completion;
50}; 49};
@@ -52,7 +51,7 @@ struct flow_flush_info {
52struct flow_cache { 51struct flow_cache {
53 u32 hash_shift; 52 u32 hash_shift;
54 unsigned long order; 53 unsigned long order;
55 struct flow_cache_percpu * percpu; 54 struct flow_cache_percpu *percpu;
56 struct notifier_block hotcpu_notifier; 55 struct notifier_block hotcpu_notifier;
57 int low_watermark; 56 int low_watermark;
58 int high_watermark; 57 int high_watermark;
@@ -78,12 +77,21 @@ static void flow_cache_new_hashrnd(unsigned long arg)
78 add_timer(&fc->rnd_timer); 77 add_timer(&fc->rnd_timer);
79} 78}
80 79
80static int flow_entry_valid(struct flow_cache_entry *fle)
81{
82 if (atomic_read(&flow_cache_genid) != fle->genid)
83 return 0;
84 if (fle->object && !fle->object->ops->check(fle->object))
85 return 0;
86 return 1;
87}
88
81static void flow_entry_kill(struct flow_cache *fc, 89static void flow_entry_kill(struct flow_cache *fc,
82 struct flow_cache_percpu *fcp, 90 struct flow_cache_percpu *fcp,
83 struct flow_cache_entry *fle) 91 struct flow_cache_entry *fle)
84{ 92{
85 if (fle->object) 93 if (fle->object)
86 atomic_dec(fle->object_ref); 94 fle->object->ops->delete(fle->object);
87 kmem_cache_free(flow_cachep, fle); 95 kmem_cache_free(flow_cachep, fle);
88 fcp->hash_count--; 96 fcp->hash_count--;
89} 97}
@@ -96,16 +104,18 @@ static void __flow_cache_shrink(struct flow_cache *fc,
96 int i; 104 int i;
97 105
98 for (i = 0; i < flow_cache_hash_size(fc); i++) { 106 for (i = 0; i < flow_cache_hash_size(fc); i++) {
99 int k = 0; 107 int saved = 0;
100 108
101 flp = &fcp->hash_table[i]; 109 flp = &fcp->hash_table[i];
102 while ((fle = *flp) != NULL && k < shrink_to) {
103 k++;
104 flp = &fle->next;
105 }
106 while ((fle = *flp) != NULL) { 110 while ((fle = *flp) != NULL) {
107 *flp = fle->next; 111 if (saved < shrink_to &&
108 flow_entry_kill(fc, fcp, fle); 112 flow_entry_valid(fle)) {
113 saved++;
114 flp = &fle->next;
115 } else {
116 *flp = fle->next;
117 flow_entry_kill(fc, fcp, fle);
118 }
109 } 119 }
110 } 120 }
111} 121}
@@ -166,18 +176,21 @@ static int flow_key_compare(struct flowi *key1, struct flowi *key2)
166 return 0; 176 return 0;
167} 177}
168 178
169void *flow_cache_lookup(struct net *net, struct flowi *key, u16 family, u8 dir, 179struct flow_cache_object *
170 flow_resolve_t resolver) 180flow_cache_lookup(struct net *net, struct flowi *key, u16 family, u8 dir,
181 flow_resolve_t resolver, void *ctx)
171{ 182{
172 struct flow_cache *fc = &flow_cache_global; 183 struct flow_cache *fc = &flow_cache_global;
173 struct flow_cache_percpu *fcp; 184 struct flow_cache_percpu *fcp;
174 struct flow_cache_entry *fle, **head; 185 struct flow_cache_entry *fle, **head;
186 struct flow_cache_object *flo;
175 unsigned int hash; 187 unsigned int hash;
176 188
177 local_bh_disable(); 189 local_bh_disable();
178 fcp = per_cpu_ptr(fc->percpu, smp_processor_id()); 190 fcp = per_cpu_ptr(fc->percpu, smp_processor_id());
179 191
180 fle = NULL; 192 fle = NULL;
193 flo = NULL;
181 /* Packet really early in init? Making flow_cache_init a 194 /* Packet really early in init? Making flow_cache_init a
182 * pre-smp initcall would solve this. --RR */ 195 * pre-smp initcall would solve this. --RR */
183 if (!fcp->hash_table) 196 if (!fcp->hash_table)
@@ -185,27 +198,17 @@ void *flow_cache_lookup(struct net *net, struct flowi *key, u16 family, u8 dir,
185 198
186 if (fcp->hash_rnd_recalc) 199 if (fcp->hash_rnd_recalc)
187 flow_new_hash_rnd(fc, fcp); 200 flow_new_hash_rnd(fc, fcp);
188 hash = flow_hash_code(fc, fcp, key);
189 201
202 hash = flow_hash_code(fc, fcp, key);
190 head = &fcp->hash_table[hash]; 203 head = &fcp->hash_table[hash];
191 for (fle = *head; fle; fle = fle->next) { 204 for (fle = *head; fle; fle = fle->next) {
192 if (fle->family == family && 205 if (fle->family == family &&
193 fle->dir == dir && 206 fle->dir == dir &&
194 flow_key_compare(key, &fle->key) == 0) { 207 flow_key_compare(key, &fle->key) == 0)
195 if (fle->genid == atomic_read(&flow_cache_genid)) {
196 void *ret = fle->object;
197
198 if (ret)
199 atomic_inc(fle->object_ref);
200 local_bh_enable();
201
202 return ret;
203 }
204 break; 208 break;
205 }
206 } 209 }
207 210
208 if (!fle) { 211 if (unlikely(!fle)) {
209 if (fcp->hash_count > fc->high_watermark) 212 if (fcp->hash_count > fc->high_watermark)
210 flow_cache_shrink(fc, fcp); 213 flow_cache_shrink(fc, fcp);
211 214
@@ -219,33 +222,39 @@ void *flow_cache_lookup(struct net *net, struct flowi *key, u16 family, u8 dir,
219 fle->object = NULL; 222 fle->object = NULL;
220 fcp->hash_count++; 223 fcp->hash_count++;
221 } 224 }
225 } else if (likely(fle->genid == atomic_read(&flow_cache_genid))) {
226 flo = fle->object;
227 if (!flo)
228 goto ret_object;
229 flo = flo->ops->get(flo);
230 if (flo)
231 goto ret_object;
232 } else if (fle->object) {
233 flo = fle->object;
234 flo->ops->delete(flo);
235 fle->object = NULL;
222 } 236 }
223 237
224nocache: 238nocache:
225 { 239 flo = NULL;
226 int err; 240 if (fle) {
227 void *obj; 241 flo = fle->object;
228 atomic_t *obj_ref; 242 fle->object = NULL;
229 243 }
230 err = resolver(net, key, family, dir, &obj, &obj_ref); 244 flo = resolver(net, key, family, dir, flo, ctx);
231 245 if (fle) {
232 if (fle && !err) { 246 fle->genid = atomic_read(&flow_cache_genid);
233 fle->genid = atomic_read(&flow_cache_genid); 247 if (!IS_ERR(flo))
234 248 fle->object = flo;
235 if (fle->object) 249 else
236 atomic_dec(fle->object_ref); 250 fle->genid--;
237 251 } else {
238 fle->object = obj; 252 if (flo && !IS_ERR(flo))
239 fle->object_ref = obj_ref; 253 flo->ops->delete(flo);
240 if (obj)
241 atomic_inc(fle->object_ref);
242 }
243 local_bh_enable();
244
245 if (err)
246 obj = ERR_PTR(err);
247 return obj;
248 } 254 }
255ret_object:
256 local_bh_enable();
257 return flo;
249} 258}
250 259
251static void flow_cache_flush_tasklet(unsigned long data) 260static void flow_cache_flush_tasklet(unsigned long data)
@@ -261,13 +270,12 @@ static void flow_cache_flush_tasklet(unsigned long data)
261 270
262 fle = fcp->hash_table[i]; 271 fle = fcp->hash_table[i];
263 for (; fle; fle = fle->next) { 272 for (; fle; fle = fle->next) {
264 unsigned genid = atomic_read(&flow_cache_genid); 273 if (flow_entry_valid(fle))
265
266 if (!fle->object || fle->genid == genid)
267 continue; 274 continue;
268 275
276 if (fle->object)
277 fle->object->ops->delete(fle->object);
269 fle->object = NULL; 278 fle->object = NULL;
270 atomic_dec(fle->object_ref);
271 } 279 }
272 } 280 }
273 281