aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorTimo Teras <timo.teras@iki.fi>2008-02-29 00:31:08 -0500
committerDavid S. Miller <davem@davemloft.net>2008-02-29 00:31:08 -0500
commit4c563f7669c10a12354b72b518c2287ffc6ebfb3 (patch)
tree056ec93f192f31640f32983c9e11bc7ce1c0692f
parent1e04d530705280770e003ac8db516722cca54758 (diff)
[XFRM]: Speed up xfrm_policy and xfrm_state walking
Change xfrm_policy and xfrm_state walking algorithm from O(n^2) to O(n). This is achieved adding the entries to one more list which is used solely for walking the entries. This also fixes some races where the dump can have duplicate or missing entries when the SPD/SADB is modified during an ongoing dump. Dumping SADB with 20000 entries using "time ip xfrm state" the sys time dropped from 1.012s to 0.080s. Signed-off-by: Timo Teras <timo.teras@iki.fi> Signed-off-by: David S. Miller <davem@davemloft.net>
-rw-r--r--include/linux/xfrm.h3
-rw-r--r--include/net/xfrm.h52
-rw-r--r--net/key/af_key.c24
-rw-r--r--net/xfrm/xfrm_policy.c79
-rw-r--r--net/xfrm/xfrm_state.c53
-rw-r--r--net/xfrm/xfrm_user.c71
6 files changed, 197 insertions, 85 deletions
diff --git a/include/linux/xfrm.h b/include/linux/xfrm.h
index e31b8c84f2c9..0c82c80b277f 100644
--- a/include/linux/xfrm.h
+++ b/include/linux/xfrm.h
@@ -113,7 +113,8 @@ enum
113{ 113{
114 XFRM_POLICY_TYPE_MAIN = 0, 114 XFRM_POLICY_TYPE_MAIN = 0,
115 XFRM_POLICY_TYPE_SUB = 1, 115 XFRM_POLICY_TYPE_SUB = 1,
116 XFRM_POLICY_TYPE_MAX = 2 116 XFRM_POLICY_TYPE_MAX = 2,
117 XFRM_POLICY_TYPE_ANY = 255
117}; 118};
118 119
119enum 120enum
diff --git a/include/net/xfrm.h b/include/net/xfrm.h
index eea7785cc757..9b6205665190 100644
--- a/include/net/xfrm.h
+++ b/include/net/xfrm.h
@@ -121,6 +121,7 @@ extern struct mutex xfrm_cfg_mutex;
121struct xfrm_state 121struct xfrm_state
122{ 122{
123 /* Note: bydst is re-used during gc */ 123 /* Note: bydst is re-used during gc */
124 struct list_head all;
124 struct hlist_node bydst; 125 struct hlist_node bydst;
125 struct hlist_node bysrc; 126 struct hlist_node bysrc;
126 struct hlist_node byspi; 127 struct hlist_node byspi;
@@ -424,6 +425,7 @@ struct xfrm_tmpl
424struct xfrm_policy 425struct xfrm_policy
425{ 426{
426 struct xfrm_policy *next; 427 struct xfrm_policy *next;
428 struct list_head bytype;
427 struct hlist_node bydst; 429 struct hlist_node bydst;
428 struct hlist_node byidx; 430 struct hlist_node byidx;
429 431
@@ -1160,6 +1162,18 @@ struct xfrm6_tunnel {
1160 int priority; 1162 int priority;
1161}; 1163};
1162 1164
1165struct xfrm_state_walk {
1166 struct xfrm_state *state;
1167 int count;
1168 u8 proto;
1169};
1170
1171struct xfrm_policy_walk {
1172 struct xfrm_policy *policy;
1173 int count;
1174 u8 type, cur_type;
1175};
1176
1163extern void xfrm_init(void); 1177extern void xfrm_init(void);
1164extern void xfrm4_init(void); 1178extern void xfrm4_init(void);
1165extern void xfrm_state_init(void); 1179extern void xfrm_state_init(void);
@@ -1184,7 +1198,23 @@ static inline void xfrm6_fini(void)
1184extern int xfrm_proc_init(void); 1198extern int xfrm_proc_init(void);
1185#endif 1199#endif
1186 1200
1187extern int xfrm_state_walk(u8 proto, int (*func)(struct xfrm_state *, int, void*), void *); 1201static inline void xfrm_state_walk_init(struct xfrm_state_walk *walk, u8 proto)
1202{
1203 walk->proto = proto;
1204 walk->state = NULL;
1205 walk->count = 0;
1206}
1207
1208static inline void xfrm_state_walk_done(struct xfrm_state_walk *walk)
1209{
1210 if (walk->state != NULL) {
1211 xfrm_state_put(walk->state);
1212 walk->state = NULL;
1213 }
1214}
1215
1216extern int xfrm_state_walk(struct xfrm_state_walk *walk,
1217 int (*func)(struct xfrm_state *, int, void*), void *);
1188extern struct xfrm_state *xfrm_state_alloc(void); 1218extern struct xfrm_state *xfrm_state_alloc(void);
1189extern struct xfrm_state *xfrm_state_find(xfrm_address_t *daddr, xfrm_address_t *saddr, 1219extern struct xfrm_state *xfrm_state_find(xfrm_address_t *daddr, xfrm_address_t *saddr,
1190 struct flowi *fl, struct xfrm_tmpl *tmpl, 1220 struct flowi *fl, struct xfrm_tmpl *tmpl,
@@ -1306,7 +1336,25 @@ static inline int xfrm4_udp_encap_rcv(struct sock *sk, struct sk_buff *skb)
1306#endif 1336#endif
1307 1337
1308struct xfrm_policy *xfrm_policy_alloc(gfp_t gfp); 1338struct xfrm_policy *xfrm_policy_alloc(gfp_t gfp);
1309extern int xfrm_policy_walk(u8 type, int (*func)(struct xfrm_policy *, int, int, void*), void *); 1339
1340static inline void xfrm_policy_walk_init(struct xfrm_policy_walk *walk, u8 type)
1341{
1342 walk->cur_type = XFRM_POLICY_TYPE_MAIN;
1343 walk->type = type;
1344 walk->policy = NULL;
1345 walk->count = 0;
1346}
1347
1348static inline void xfrm_policy_walk_done(struct xfrm_policy_walk *walk)
1349{
1350 if (walk->policy != NULL) {
1351 xfrm_pol_put(walk->policy);
1352 walk->policy = NULL;
1353 }
1354}
1355
1356extern int xfrm_policy_walk(struct xfrm_policy_walk *walk,
1357 int (*func)(struct xfrm_policy *, int, int, void*), void *);
1310int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl); 1358int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl);
1311struct xfrm_policy *xfrm_policy_bysel_ctx(u8 type, int dir, 1359struct xfrm_policy *xfrm_policy_bysel_ctx(u8 type, int dir,
1312 struct xfrm_selector *sel, 1360 struct xfrm_selector *sel,
diff --git a/net/key/af_key.c b/net/key/af_key.c
index 8b5f486ac80f..7cb6f1213360 100644
--- a/net/key/af_key.c
+++ b/net/key/af_key.c
@@ -1742,12 +1742,18 @@ static int pfkey_dump(struct sock *sk, struct sk_buff *skb, struct sadb_msg *hdr
1742{ 1742{
1743 u8 proto; 1743 u8 proto;
1744 struct pfkey_dump_data data = { .skb = skb, .hdr = hdr, .sk = sk }; 1744 struct pfkey_dump_data data = { .skb = skb, .hdr = hdr, .sk = sk };
1745 struct xfrm_state_walk walk;
1746 int rc;
1745 1747
1746 proto = pfkey_satype2proto(hdr->sadb_msg_satype); 1748 proto = pfkey_satype2proto(hdr->sadb_msg_satype);
1747 if (proto == 0) 1749 if (proto == 0)
1748 return -EINVAL; 1750 return -EINVAL;
1749 1751
1750 return xfrm_state_walk(proto, dump_sa, &data); 1752 xfrm_state_walk_init(&walk, proto);
1753 rc = xfrm_state_walk(&walk, dump_sa, &data);
1754 xfrm_state_walk_done(&walk);
1755
1756 return rc;
1751} 1757}
1752 1758
1753static int pfkey_promisc(struct sock *sk, struct sk_buff *skb, struct sadb_msg *hdr, void **ext_hdrs) 1759static int pfkey_promisc(struct sock *sk, struct sk_buff *skb, struct sadb_msg *hdr, void **ext_hdrs)
@@ -1780,7 +1786,9 @@ static int check_reqid(struct xfrm_policy *xp, int dir, int count, void *ptr)
1780 1786
1781static u32 gen_reqid(void) 1787static u32 gen_reqid(void)
1782{ 1788{
1789 struct xfrm_policy_walk walk;
1783 u32 start; 1790 u32 start;
1791 int rc;
1784 static u32 reqid = IPSEC_MANUAL_REQID_MAX; 1792 static u32 reqid = IPSEC_MANUAL_REQID_MAX;
1785 1793
1786 start = reqid; 1794 start = reqid;
@@ -1788,8 +1796,10 @@ static u32 gen_reqid(void)
1788 ++reqid; 1796 ++reqid;
1789 if (reqid == 0) 1797 if (reqid == 0)
1790 reqid = IPSEC_MANUAL_REQID_MAX+1; 1798 reqid = IPSEC_MANUAL_REQID_MAX+1;
1791 if (xfrm_policy_walk(XFRM_POLICY_TYPE_MAIN, check_reqid, 1799 xfrm_policy_walk_init(&walk, XFRM_POLICY_TYPE_MAIN);
1792 (void*)&reqid) != -EEXIST) 1800 rc = xfrm_policy_walk(&walk, check_reqid, (void*)&reqid);
1801 xfrm_policy_walk_done(&walk);
1802 if (rc != -EEXIST)
1793 return reqid; 1803 return reqid;
1794 } while (reqid != start); 1804 } while (reqid != start);
1795 return 0; 1805 return 0;
@@ -2665,8 +2675,14 @@ static int dump_sp(struct xfrm_policy *xp, int dir, int count, void *ptr)
2665static int pfkey_spddump(struct sock *sk, struct sk_buff *skb, struct sadb_msg *hdr, void **ext_hdrs) 2675static int pfkey_spddump(struct sock *sk, struct sk_buff *skb, struct sadb_msg *hdr, void **ext_hdrs)
2666{ 2676{
2667 struct pfkey_dump_data data = { .skb = skb, .hdr = hdr, .sk = sk }; 2677 struct pfkey_dump_data data = { .skb = skb, .hdr = hdr, .sk = sk };
2678 struct xfrm_policy_walk walk;
2679 int rc;
2680
2681 xfrm_policy_walk_init(&walk, XFRM_POLICY_TYPE_MAIN);
2682 rc = xfrm_policy_walk(&walk, dump_sp, &data);
2683 xfrm_policy_walk_done(&walk);
2668 2684
2669 return xfrm_policy_walk(XFRM_POLICY_TYPE_MAIN, dump_sp, &data); 2685 return rc;
2670} 2686}
2671 2687
2672static int key_notify_policy_flush(struct km_event *c) 2688static int key_notify_policy_flush(struct km_event *c)
diff --git a/net/xfrm/xfrm_policy.c b/net/xfrm/xfrm_policy.c
index 9fc4c315f6cd..bae94a8031a2 100644
--- a/net/xfrm/xfrm_policy.c
+++ b/net/xfrm/xfrm_policy.c
@@ -46,6 +46,7 @@ EXPORT_SYMBOL(xfrm_cfg_mutex);
46 46
47static DEFINE_RWLOCK(xfrm_policy_lock); 47static DEFINE_RWLOCK(xfrm_policy_lock);
48 48
49static struct list_head xfrm_policy_bytype[XFRM_POLICY_TYPE_MAX];
49unsigned int xfrm_policy_count[XFRM_POLICY_MAX*2]; 50unsigned int xfrm_policy_count[XFRM_POLICY_MAX*2];
50EXPORT_SYMBOL(xfrm_policy_count); 51EXPORT_SYMBOL(xfrm_policy_count);
51 52
@@ -208,6 +209,7 @@ struct xfrm_policy *xfrm_policy_alloc(gfp_t gfp)
208 policy = kzalloc(sizeof(struct xfrm_policy), gfp); 209 policy = kzalloc(sizeof(struct xfrm_policy), gfp);
209 210
210 if (policy) { 211 if (policy) {
212 INIT_LIST_HEAD(&policy->bytype);
211 INIT_HLIST_NODE(&policy->bydst); 213 INIT_HLIST_NODE(&policy->bydst);
212 INIT_HLIST_NODE(&policy->byidx); 214 INIT_HLIST_NODE(&policy->byidx);
213 rwlock_init(&policy->lock); 215 rwlock_init(&policy->lock);
@@ -230,6 +232,10 @@ void xfrm_policy_destroy(struct xfrm_policy *policy)
230 if (del_timer(&policy->timer)) 232 if (del_timer(&policy->timer))
231 BUG(); 233 BUG();
232 234
235 write_lock_bh(&xfrm_policy_lock);
236 list_del(&policy->bytype);
237 write_unlock_bh(&xfrm_policy_lock);
238
233 security_xfrm_policy_free(policy); 239 security_xfrm_policy_free(policy);
234 kfree(policy); 240 kfree(policy);
235} 241}
@@ -584,6 +590,7 @@ int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl)
584 policy->curlft.use_time = 0; 590 policy->curlft.use_time = 0;
585 if (!mod_timer(&policy->timer, jiffies + HZ)) 591 if (!mod_timer(&policy->timer, jiffies + HZ))
586 xfrm_pol_hold(policy); 592 xfrm_pol_hold(policy);
593 list_add_tail(&policy->bytype, &xfrm_policy_bytype[policy->type]);
587 write_unlock_bh(&xfrm_policy_lock); 594 write_unlock_bh(&xfrm_policy_lock);
588 595
589 if (delpol) 596 if (delpol)
@@ -822,57 +829,60 @@ out:
822} 829}
823EXPORT_SYMBOL(xfrm_policy_flush); 830EXPORT_SYMBOL(xfrm_policy_flush);
824 831
825int xfrm_policy_walk(u8 type, int (*func)(struct xfrm_policy *, int, int, void*), 832int xfrm_policy_walk(struct xfrm_policy_walk *walk,
833 int (*func)(struct xfrm_policy *, int, int, void*),
826 void *data) 834 void *data)
827{ 835{
828 struct xfrm_policy *pol, *last = NULL; 836 struct xfrm_policy *old, *pol, *last = NULL;
829 struct hlist_node *entry; 837 int error = 0;
830 int dir, last_dir = 0, count, error; 838
839 if (walk->type >= XFRM_POLICY_TYPE_MAX &&
840 walk->type != XFRM_POLICY_TYPE_ANY)
841 return -EINVAL;
831 842
843 if (walk->policy == NULL && walk->count != 0)
844 return 0;
845
846 old = pol = walk->policy;
847 walk->policy = NULL;
832 read_lock_bh(&xfrm_policy_lock); 848 read_lock_bh(&xfrm_policy_lock);
833 count = 0;
834 849
835 for (dir = 0; dir < 2*XFRM_POLICY_MAX; dir++) { 850 for (; walk->cur_type < XFRM_POLICY_TYPE_MAX; walk->cur_type++) {
836 struct hlist_head *table = xfrm_policy_bydst[dir].table; 851 if (walk->type != walk->cur_type &&
837 int i; 852 walk->type != XFRM_POLICY_TYPE_ANY)
853 continue;
838 854
839 hlist_for_each_entry(pol, entry, 855 if (pol == NULL) {
840 &xfrm_policy_inexact[dir], bydst) { 856 pol = list_first_entry(&xfrm_policy_bytype[walk->cur_type],
841 if (pol->type != type) 857 struct xfrm_policy, bytype);
858 }
859 list_for_each_entry_from(pol, &xfrm_policy_bytype[walk->cur_type], bytype) {
860 if (pol->dead)
842 continue; 861 continue;
843 if (last) { 862 if (last) {
844 error = func(last, last_dir % XFRM_POLICY_MAX, 863 error = func(last, xfrm_policy_id2dir(last->index),
845 count, data); 864 walk->count, data);
846 if (error) 865 if (error) {
866 xfrm_pol_hold(last);
867 walk->policy = last;
847 goto out; 868 goto out;
848 }
849 last = pol;
850 last_dir = dir;
851 count++;
852 }
853 for (i = xfrm_policy_bydst[dir].hmask; i >= 0; i--) {
854 hlist_for_each_entry(pol, entry, table + i, bydst) {
855 if (pol->type != type)
856 continue;
857 if (last) {
858 error = func(last, last_dir % XFRM_POLICY_MAX,
859 count, data);
860 if (error)
861 goto out;
862 } 869 }
863 last = pol;
864 last_dir = dir;
865 count++;
866 } 870 }
871 last = pol;
872 walk->count++;
867 } 873 }
874 pol = NULL;
868 } 875 }
869 if (count == 0) { 876 if (walk->count == 0) {
870 error = -ENOENT; 877 error = -ENOENT;
871 goto out; 878 goto out;
872 } 879 }
873 error = func(last, last_dir % XFRM_POLICY_MAX, 0, data); 880 if (last)
881 error = func(last, xfrm_policy_id2dir(last->index), 0, data);
874out: 882out:
875 read_unlock_bh(&xfrm_policy_lock); 883 read_unlock_bh(&xfrm_policy_lock);
884 if (old != NULL)
885 xfrm_pol_put(old);
876 return error; 886 return error;
877} 887}
878EXPORT_SYMBOL(xfrm_policy_walk); 888EXPORT_SYMBOL(xfrm_policy_walk);
@@ -2365,6 +2375,9 @@ static void __init xfrm_policy_init(void)
2365 panic("XFRM: failed to allocate bydst hash\n"); 2375 panic("XFRM: failed to allocate bydst hash\n");
2366 } 2376 }
2367 2377
2378 for (dir = 0; dir < XFRM_POLICY_TYPE_MAX; dir++)
2379 INIT_LIST_HEAD(&xfrm_policy_bytype[dir]);
2380
2368 INIT_WORK(&xfrm_policy_gc_work, xfrm_policy_gc_task); 2381 INIT_WORK(&xfrm_policy_gc_work, xfrm_policy_gc_task);
2369 register_netdevice_notifier(&xfrm_dev_notifier); 2382 register_netdevice_notifier(&xfrm_dev_notifier);
2370} 2383}
diff --git a/net/xfrm/xfrm_state.c b/net/xfrm/xfrm_state.c
index 7ba65e82941c..9880b792e6a5 100644
--- a/net/xfrm/xfrm_state.c
+++ b/net/xfrm/xfrm_state.c
@@ -50,6 +50,7 @@ static DEFINE_SPINLOCK(xfrm_state_lock);
50 * Main use is finding SA after policy selected tunnel or transport mode. 50 * Main use is finding SA after policy selected tunnel or transport mode.
51 * Also, it can be used by ah/esp icmp error handler to find offending SA. 51 * Also, it can be used by ah/esp icmp error handler to find offending SA.
52 */ 52 */
53static LIST_HEAD(xfrm_state_all);
53static struct hlist_head *xfrm_state_bydst __read_mostly; 54static struct hlist_head *xfrm_state_bydst __read_mostly;
54static struct hlist_head *xfrm_state_bysrc __read_mostly; 55static struct hlist_head *xfrm_state_bysrc __read_mostly;
55static struct hlist_head *xfrm_state_byspi __read_mostly; 56static struct hlist_head *xfrm_state_byspi __read_mostly;
@@ -510,6 +511,7 @@ struct xfrm_state *xfrm_state_alloc(void)
510 if (x) { 511 if (x) {
511 atomic_set(&x->refcnt, 1); 512 atomic_set(&x->refcnt, 1);
512 atomic_set(&x->tunnel_users, 0); 513 atomic_set(&x->tunnel_users, 0);
514 INIT_LIST_HEAD(&x->all);
513 INIT_HLIST_NODE(&x->bydst); 515 INIT_HLIST_NODE(&x->bydst);
514 INIT_HLIST_NODE(&x->bysrc); 516 INIT_HLIST_NODE(&x->bysrc);
515 INIT_HLIST_NODE(&x->byspi); 517 INIT_HLIST_NODE(&x->byspi);
@@ -533,6 +535,10 @@ void __xfrm_state_destroy(struct xfrm_state *x)
533{ 535{
534 BUG_TRAP(x->km.state == XFRM_STATE_DEAD); 536 BUG_TRAP(x->km.state == XFRM_STATE_DEAD);
535 537
538 spin_lock_bh(&xfrm_state_lock);
539 list_del(&x->all);
540 spin_unlock_bh(&xfrm_state_lock);
541
536 spin_lock_bh(&xfrm_state_gc_lock); 542 spin_lock_bh(&xfrm_state_gc_lock);
537 hlist_add_head(&x->bydst, &xfrm_state_gc_list); 543 hlist_add_head(&x->bydst, &xfrm_state_gc_list);
538 spin_unlock_bh(&xfrm_state_gc_lock); 544 spin_unlock_bh(&xfrm_state_gc_lock);
@@ -909,6 +915,8 @@ static void __xfrm_state_insert(struct xfrm_state *x)
909 915
910 x->genid = ++xfrm_state_genid; 916 x->genid = ++xfrm_state_genid;
911 917
918 list_add_tail(&x->all, &xfrm_state_all);
919
912 h = xfrm_dst_hash(&x->id.daddr, &x->props.saddr, 920 h = xfrm_dst_hash(&x->id.daddr, &x->props.saddr,
913 x->props.reqid, x->props.family); 921 x->props.reqid, x->props.family);
914 hlist_add_head(&x->bydst, xfrm_state_bydst+h); 922 hlist_add_head(&x->bydst, xfrm_state_bydst+h);
@@ -1518,36 +1526,47 @@ unlock:
1518} 1526}
1519EXPORT_SYMBOL(xfrm_alloc_spi); 1527EXPORT_SYMBOL(xfrm_alloc_spi);
1520 1528
1521int xfrm_state_walk(u8 proto, int (*func)(struct xfrm_state *, int, void*), 1529int xfrm_state_walk(struct xfrm_state_walk *walk,
1530 int (*func)(struct xfrm_state *, int, void*),
1522 void *data) 1531 void *data)
1523{ 1532{
1524 int i; 1533 struct xfrm_state *old, *x, *last = NULL;
1525 struct xfrm_state *x, *last = NULL;
1526 struct hlist_node *entry;
1527 int count = 0;
1528 int err = 0; 1534 int err = 0;
1529 1535
1536 if (walk->state == NULL && walk->count != 0)
1537 return 0;
1538
1539 old = x = walk->state;
1540 walk->state = NULL;
1530 spin_lock_bh(&xfrm_state_lock); 1541 spin_lock_bh(&xfrm_state_lock);
1531 for (i = 0; i <= xfrm_state_hmask; i++) { 1542 if (x == NULL)
1532 hlist_for_each_entry(x, entry, xfrm_state_bydst+i, bydst) { 1543 x = list_first_entry(&xfrm_state_all, struct xfrm_state, all);
1533 if (!xfrm_id_proto_match(x->id.proto, proto)) 1544 list_for_each_entry_from(x, &xfrm_state_all, all) {
1534 continue; 1545 if (x->km.state == XFRM_STATE_DEAD)
1535 if (last) { 1546 continue;
1536 err = func(last, count, data); 1547 if (!xfrm_id_proto_match(x->id.proto, walk->proto))
1537 if (err) 1548 continue;
1538 goto out; 1549 if (last) {
1550 err = func(last, walk->count, data);
1551 if (err) {
1552 xfrm_state_hold(last);
1553 walk->state = last;
1554 goto out;
1539 } 1555 }
1540 last = x;
1541 count++;
1542 } 1556 }
1557 last = x;
1558 walk->count++;
1543 } 1559 }
1544 if (count == 0) { 1560 if (walk->count == 0) {
1545 err = -ENOENT; 1561 err = -ENOENT;
1546 goto out; 1562 goto out;
1547 } 1563 }
1548 err = func(last, 0, data); 1564 if (last)
1565 err = func(last, 0, data);
1549out: 1566out:
1550 spin_unlock_bh(&xfrm_state_lock); 1567 spin_unlock_bh(&xfrm_state_lock);
1568 if (old != NULL)
1569 xfrm_state_put(old);
1551 return err; 1570 return err;
1552} 1571}
1553EXPORT_SYMBOL(xfrm_state_walk); 1572EXPORT_SYMBOL(xfrm_state_walk);
diff --git a/net/xfrm/xfrm_user.c b/net/xfrm/xfrm_user.c
index f971ca5645f8..f5fd5b3147cc 100644
--- a/net/xfrm/xfrm_user.c
+++ b/net/xfrm/xfrm_user.c
@@ -532,8 +532,6 @@ struct xfrm_dump_info {
532 struct sk_buff *out_skb; 532 struct sk_buff *out_skb;
533 u32 nlmsg_seq; 533 u32 nlmsg_seq;
534 u16 nlmsg_flags; 534 u16 nlmsg_flags;
535 int start_idx;
536 int this_idx;
537}; 535};
538 536
539static int copy_sec_ctx(struct xfrm_sec_ctx *s, struct sk_buff *skb) 537static int copy_sec_ctx(struct xfrm_sec_ctx *s, struct sk_buff *skb)
@@ -600,9 +598,6 @@ static int dump_one_state(struct xfrm_state *x, int count, void *ptr)
600 struct nlmsghdr *nlh; 598 struct nlmsghdr *nlh;
601 int err; 599 int err;
602 600
603 if (sp->this_idx < sp->start_idx)
604 goto out;
605
606 nlh = nlmsg_put(skb, NETLINK_CB(in_skb).pid, sp->nlmsg_seq, 601 nlh = nlmsg_put(skb, NETLINK_CB(in_skb).pid, sp->nlmsg_seq,
607 XFRM_MSG_NEWSA, sizeof(*p), sp->nlmsg_flags); 602 XFRM_MSG_NEWSA, sizeof(*p), sp->nlmsg_flags);
608 if (nlh == NULL) 603 if (nlh == NULL)
@@ -615,8 +610,6 @@ static int dump_one_state(struct xfrm_state *x, int count, void *ptr)
615 goto nla_put_failure; 610 goto nla_put_failure;
616 611
617 nlmsg_end(skb, nlh); 612 nlmsg_end(skb, nlh);
618out:
619 sp->this_idx++;
620 return 0; 613 return 0;
621 614
622nla_put_failure: 615nla_put_failure:
@@ -624,18 +617,32 @@ nla_put_failure:
624 return err; 617 return err;
625} 618}
626 619
620static int xfrm_dump_sa_done(struct netlink_callback *cb)
621{
622 struct xfrm_state_walk *walk = (struct xfrm_state_walk *) &cb->args[1];
623 xfrm_state_walk_done(walk);
624 return 0;
625}
626
627static int xfrm_dump_sa(struct sk_buff *skb, struct netlink_callback *cb) 627static int xfrm_dump_sa(struct sk_buff *skb, struct netlink_callback *cb)
628{ 628{
629 struct xfrm_state_walk *walk = (struct xfrm_state_walk *) &cb->args[1];
629 struct xfrm_dump_info info; 630 struct xfrm_dump_info info;
630 631
632 BUILD_BUG_ON(sizeof(struct xfrm_state_walk) >
633 sizeof(cb->args) - sizeof(cb->args[0]));
634
631 info.in_skb = cb->skb; 635 info.in_skb = cb->skb;
632 info.out_skb = skb; 636 info.out_skb = skb;
633 info.nlmsg_seq = cb->nlh->nlmsg_seq; 637 info.nlmsg_seq = cb->nlh->nlmsg_seq;
634 info.nlmsg_flags = NLM_F_MULTI; 638 info.nlmsg_flags = NLM_F_MULTI;
635 info.this_idx = 0; 639
636 info.start_idx = cb->args[0]; 640 if (!cb->args[0]) {
637 (void) xfrm_state_walk(0, dump_one_state, &info); 641 cb->args[0] = 1;
638 cb->args[0] = info.this_idx; 642 xfrm_state_walk_init(walk, 0);
643 }
644
645 (void) xfrm_state_walk(walk, dump_one_state, &info);
639 646
640 return skb->len; 647 return skb->len;
641} 648}
@@ -654,7 +661,6 @@ static struct sk_buff *xfrm_state_netlink(struct sk_buff *in_skb,
654 info.out_skb = skb; 661 info.out_skb = skb;
655 info.nlmsg_seq = seq; 662 info.nlmsg_seq = seq;
656 info.nlmsg_flags = 0; 663 info.nlmsg_flags = 0;
657 info.this_idx = info.start_idx = 0;
658 664
659 if (dump_one_state(x, 0, &info)) { 665 if (dump_one_state(x, 0, &info)) {
660 kfree_skb(skb); 666 kfree_skb(skb);
@@ -1232,9 +1238,6 @@ static int dump_one_policy(struct xfrm_policy *xp, int dir, int count, void *ptr
1232 struct sk_buff *skb = sp->out_skb; 1238 struct sk_buff *skb = sp->out_skb;
1233 struct nlmsghdr *nlh; 1239 struct nlmsghdr *nlh;
1234 1240
1235 if (sp->this_idx < sp->start_idx)
1236 goto out;
1237
1238 nlh = nlmsg_put(skb, NETLINK_CB(in_skb).pid, sp->nlmsg_seq, 1241 nlh = nlmsg_put(skb, NETLINK_CB(in_skb).pid, sp->nlmsg_seq,
1239 XFRM_MSG_NEWPOLICY, sizeof(*p), sp->nlmsg_flags); 1242 XFRM_MSG_NEWPOLICY, sizeof(*p), sp->nlmsg_flags);
1240 if (nlh == NULL) 1243 if (nlh == NULL)
@@ -1250,8 +1253,6 @@ static int dump_one_policy(struct xfrm_policy *xp, int dir, int count, void *ptr
1250 goto nlmsg_failure; 1253 goto nlmsg_failure;
1251 1254
1252 nlmsg_end(skb, nlh); 1255 nlmsg_end(skb, nlh);
1253out:
1254 sp->this_idx++;
1255 return 0; 1256 return 0;
1256 1257
1257nlmsg_failure: 1258nlmsg_failure:
@@ -1259,21 +1260,33 @@ nlmsg_failure:
1259 return -EMSGSIZE; 1260 return -EMSGSIZE;
1260} 1261}
1261 1262
1263static int xfrm_dump_policy_done(struct netlink_callback *cb)
1264{
1265 struct xfrm_policy_walk *walk = (struct xfrm_policy_walk *) &cb->args[1];
1266
1267 xfrm_policy_walk_done(walk);
1268 return 0;
1269}
1270
1262static int xfrm_dump_policy(struct sk_buff *skb, struct netlink_callback *cb) 1271static int xfrm_dump_policy(struct sk_buff *skb, struct netlink_callback *cb)
1263{ 1272{
1273 struct xfrm_policy_walk *walk = (struct xfrm_policy_walk *) &cb->args[1];
1264 struct xfrm_dump_info info; 1274 struct xfrm_dump_info info;
1265 1275
1276 BUILD_BUG_ON(sizeof(struct xfrm_policy_walk) >
1277 sizeof(cb->args) - sizeof(cb->args[0]));
1278
1266 info.in_skb = cb->skb; 1279 info.in_skb = cb->skb;
1267 info.out_skb = skb; 1280 info.out_skb = skb;
1268 info.nlmsg_seq = cb->nlh->nlmsg_seq; 1281 info.nlmsg_seq = cb->nlh->nlmsg_seq;
1269 info.nlmsg_flags = NLM_F_MULTI; 1282 info.nlmsg_flags = NLM_F_MULTI;
1270 info.this_idx = 0; 1283
1271 info.start_idx = cb->args[0]; 1284 if (!cb->args[0]) {
1272 (void) xfrm_policy_walk(XFRM_POLICY_TYPE_MAIN, dump_one_policy, &info); 1285 cb->args[0] = 1;
1273#ifdef CONFIG_XFRM_SUB_POLICY 1286 xfrm_policy_walk_init(walk, XFRM_POLICY_TYPE_ANY);
1274 (void) xfrm_policy_walk(XFRM_POLICY_TYPE_SUB, dump_one_policy, &info); 1287 }
1275#endif 1288
1276 cb->args[0] = info.this_idx; 1289 (void) xfrm_policy_walk(walk, dump_one_policy, &info);
1277 1290
1278 return skb->len; 1291 return skb->len;
1279} 1292}
@@ -1293,7 +1306,6 @@ static struct sk_buff *xfrm_policy_netlink(struct sk_buff *in_skb,
1293 info.out_skb = skb; 1306 info.out_skb = skb;
1294 info.nlmsg_seq = seq; 1307 info.nlmsg_seq = seq;
1295 info.nlmsg_flags = 0; 1308 info.nlmsg_flags = 0;
1296 info.this_idx = info.start_idx = 0;
1297 1309
1298 if (dump_one_policy(xp, dir, 0, &info) < 0) { 1310 if (dump_one_policy(xp, dir, 0, &info) < 0) {
1299 kfree_skb(skb); 1311 kfree_skb(skb);
@@ -1891,15 +1903,18 @@ static const struct nla_policy xfrma_policy[XFRMA_MAX+1] = {
1891static struct xfrm_link { 1903static struct xfrm_link {
1892 int (*doit)(struct sk_buff *, struct nlmsghdr *, struct nlattr **); 1904 int (*doit)(struct sk_buff *, struct nlmsghdr *, struct nlattr **);
1893 int (*dump)(struct sk_buff *, struct netlink_callback *); 1905 int (*dump)(struct sk_buff *, struct netlink_callback *);
1906 int (*done)(struct netlink_callback *);
1894} xfrm_dispatch[XFRM_NR_MSGTYPES] = { 1907} xfrm_dispatch[XFRM_NR_MSGTYPES] = {
1895 [XFRM_MSG_NEWSA - XFRM_MSG_BASE] = { .doit = xfrm_add_sa }, 1908 [XFRM_MSG_NEWSA - XFRM_MSG_BASE] = { .doit = xfrm_add_sa },
1896 [XFRM_MSG_DELSA - XFRM_MSG_BASE] = { .doit = xfrm_del_sa }, 1909 [XFRM_MSG_DELSA - XFRM_MSG_BASE] = { .doit = xfrm_del_sa },
1897 [XFRM_MSG_GETSA - XFRM_MSG_BASE] = { .doit = xfrm_get_sa, 1910 [XFRM_MSG_GETSA - XFRM_MSG_BASE] = { .doit = xfrm_get_sa,
1898 .dump = xfrm_dump_sa }, 1911 .dump = xfrm_dump_sa,
1912 .done = xfrm_dump_sa_done },
1899 [XFRM_MSG_NEWPOLICY - XFRM_MSG_BASE] = { .doit = xfrm_add_policy }, 1913 [XFRM_MSG_NEWPOLICY - XFRM_MSG_BASE] = { .doit = xfrm_add_policy },
1900 [XFRM_MSG_DELPOLICY - XFRM_MSG_BASE] = { .doit = xfrm_get_policy }, 1914 [XFRM_MSG_DELPOLICY - XFRM_MSG_BASE] = { .doit = xfrm_get_policy },
1901 [XFRM_MSG_GETPOLICY - XFRM_MSG_BASE] = { .doit = xfrm_get_policy, 1915 [XFRM_MSG_GETPOLICY - XFRM_MSG_BASE] = { .doit = xfrm_get_policy,
1902 .dump = xfrm_dump_policy }, 1916 .dump = xfrm_dump_policy,
1917 .done = xfrm_dump_policy_done },
1903 [XFRM_MSG_ALLOCSPI - XFRM_MSG_BASE] = { .doit = xfrm_alloc_userspi }, 1918 [XFRM_MSG_ALLOCSPI - XFRM_MSG_BASE] = { .doit = xfrm_alloc_userspi },
1904 [XFRM_MSG_ACQUIRE - XFRM_MSG_BASE] = { .doit = xfrm_add_acquire }, 1919 [XFRM_MSG_ACQUIRE - XFRM_MSG_BASE] = { .doit = xfrm_add_acquire },
1905 [XFRM_MSG_EXPIRE - XFRM_MSG_BASE] = { .doit = xfrm_add_sa_expire }, 1920 [XFRM_MSG_EXPIRE - XFRM_MSG_BASE] = { .doit = xfrm_add_sa_expire },
@@ -1938,7 +1953,7 @@ static int xfrm_user_rcv_msg(struct sk_buff *skb, struct nlmsghdr *nlh)
1938 if (link->dump == NULL) 1953 if (link->dump == NULL)
1939 return -EINVAL; 1954 return -EINVAL;
1940 1955
1941 return netlink_dump_start(xfrm_nl, skb, nlh, link->dump, NULL); 1956 return netlink_dump_start(xfrm_nl, skb, nlh, link->dump, link->done);
1942 } 1957 }
1943 1958
1944 err = nlmsg_parse(nlh, xfrm_msg_min[type], attrs, XFRMA_MAX, 1959 err = nlmsg_parse(nlh, xfrm_msg_min[type], attrs, XFRMA_MAX,