aboutsummaryrefslogtreecommitdiffstats
path: root/net/rds
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2011-10-25 07:25:22 -0400
committerLinus Torvalds <torvalds@linux-foundation.org>2011-10-25 07:25:22 -0400
commit8a9ea3237e7eb5c25f09e429ad242ae5a3d5ea22 (patch)
treea0a63398a9983667d52cbbbf4e2405b4f22b1d83 /net/rds
parent1be025d3cb40cd295123af2c394f7229ef9b30ca (diff)
parent8b3408f8ee994973869d8ba32c5bf482bc4ddca4 (diff)
Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/net-next
* git://git.kernel.org/pub/scm/linux/kernel/git/davem/net-next: (1745 commits) dp83640: free packet queues on remove dp83640: use proper function to free transmit time stamping packets ipv6: Do not use routes from locally generated RAs |PATCH net-next] tg3: add tx_dropped counter be2net: don't create multiple RX/TX rings in multi channel mode be2net: don't create multiple TXQs in BE2 be2net: refactor VF setup/teardown code into be_vf_setup/clear() be2net: add vlan/rx-mode/flow-control config to be_setup() net_sched: cls_flow: use skb_header_pointer() ipv4: avoid useless call of the function check_peer_pmtu TCP: remove TCP_DEBUG net: Fix driver name for mdio-gpio.c ipv4: tcp: fix TOS value in ACK messages sent from TIME_WAIT rtnetlink: Add missing manual netlink notification in dev_change_net_namespaces ipv4: fix ipsec forward performance regression jme: fix irq storm after suspend/resume route: fix ICMP redirect validation net: hold sock reference while processing tx timestamps tcp: md5: add more const attributes Add ethtool -g support to virtio_net ... Fix up conflicts in: - drivers/net/Kconfig: The split-up generated a trivial conflict with removal of a stale reference to Documentation/networking/net-modules.txt. Remove it from the new location instead. - fs/sysfs/dir.c: Fairly nasty conflicts with the sysfs rb-tree usage, conflicting with Eric Biederman's changes for tagged directories.
Diffstat (limited to 'net/rds')
-rw-r--r--net/rds/Kconfig1
-rw-r--r--net/rds/ib_rdma.c112
-rw-r--r--net/rds/xlist.h80
3 files changed, 52 insertions, 141 deletions
diff --git a/net/rds/Kconfig b/net/rds/Kconfig
index ec753b3ae72a..4cf6dc7910e4 100644
--- a/net/rds/Kconfig
+++ b/net/rds/Kconfig
@@ -9,6 +9,7 @@ config RDS
9 9
10config RDS_RDMA 10config RDS_RDMA
11 tristate "RDS over Infiniband and iWARP" 11 tristate "RDS over Infiniband and iWARP"
12 select LLIST
12 depends on RDS && INFINIBAND && INFINIBAND_ADDR_TRANS 13 depends on RDS && INFINIBAND && INFINIBAND_ADDR_TRANS
13 ---help--- 14 ---help---
14 Allow RDS to use Infiniband and iWARP as a transport. 15 Allow RDS to use Infiniband and iWARP as a transport.
diff --git a/net/rds/ib_rdma.c b/net/rds/ib_rdma.c
index 819c35a0d9cb..e8fdb172adbb 100644
--- a/net/rds/ib_rdma.c
+++ b/net/rds/ib_rdma.c
@@ -33,10 +33,10 @@
33#include <linux/kernel.h> 33#include <linux/kernel.h>
34#include <linux/slab.h> 34#include <linux/slab.h>
35#include <linux/rculist.h> 35#include <linux/rculist.h>
36#include <linux/llist.h>
36 37
37#include "rds.h" 38#include "rds.h"
38#include "ib.h" 39#include "ib.h"
39#include "xlist.h"
40 40
41static DEFINE_PER_CPU(unsigned long, clean_list_grace); 41static DEFINE_PER_CPU(unsigned long, clean_list_grace);
42#define CLEAN_LIST_BUSY_BIT 0 42#define CLEAN_LIST_BUSY_BIT 0
@@ -49,7 +49,7 @@ struct rds_ib_mr {
49 struct rds_ib_mr_pool *pool; 49 struct rds_ib_mr_pool *pool;
50 struct ib_fmr *fmr; 50 struct ib_fmr *fmr;
51 51
52 struct xlist_head xlist; 52 struct llist_node llnode;
53 53
54 /* unmap_list is for freeing */ 54 /* unmap_list is for freeing */
55 struct list_head unmap_list; 55 struct list_head unmap_list;
@@ -71,9 +71,9 @@ struct rds_ib_mr_pool {
71 atomic_t item_count; /* total # of MRs */ 71 atomic_t item_count; /* total # of MRs */
72 atomic_t dirty_count; /* # dirty of MRs */ 72 atomic_t dirty_count; /* # dirty of MRs */
73 73
74 struct xlist_head drop_list; /* MRs that have reached their max_maps limit */ 74 struct llist_head drop_list; /* MRs that have reached their max_maps limit */
75 struct xlist_head free_list; /* unused MRs */ 75 struct llist_head free_list; /* unused MRs */
76 struct xlist_head clean_list; /* global unused & unamapped MRs */ 76 struct llist_head clean_list; /* global unused & unamapped MRs */
77 wait_queue_head_t flush_wait; 77 wait_queue_head_t flush_wait;
78 78
79 atomic_t free_pinned; /* memory pinned by free MRs */ 79 atomic_t free_pinned; /* memory pinned by free MRs */
@@ -220,9 +220,9 @@ struct rds_ib_mr_pool *rds_ib_create_mr_pool(struct rds_ib_device *rds_ibdev)
220 if (!pool) 220 if (!pool)
221 return ERR_PTR(-ENOMEM); 221 return ERR_PTR(-ENOMEM);
222 222
223 INIT_XLIST_HEAD(&pool->free_list); 223 init_llist_head(&pool->free_list);
224 INIT_XLIST_HEAD(&pool->drop_list); 224 init_llist_head(&pool->drop_list);
225 INIT_XLIST_HEAD(&pool->clean_list); 225 init_llist_head(&pool->clean_list);
226 mutex_init(&pool->flush_lock); 226 mutex_init(&pool->flush_lock);
227 init_waitqueue_head(&pool->flush_wait); 227 init_waitqueue_head(&pool->flush_wait);
228 INIT_DELAYED_WORK(&pool->flush_worker, rds_ib_mr_pool_flush_worker); 228 INIT_DELAYED_WORK(&pool->flush_worker, rds_ib_mr_pool_flush_worker);
@@ -260,26 +260,18 @@ void rds_ib_destroy_mr_pool(struct rds_ib_mr_pool *pool)
260 kfree(pool); 260 kfree(pool);
261} 261}
262 262
263static void refill_local(struct rds_ib_mr_pool *pool, struct xlist_head *xl,
264 struct rds_ib_mr **ibmr_ret)
265{
266 struct xlist_head *ibmr_xl;
267 ibmr_xl = xlist_del_head_fast(xl);
268 *ibmr_ret = list_entry(ibmr_xl, struct rds_ib_mr, xlist);
269}
270
271static inline struct rds_ib_mr *rds_ib_reuse_fmr(struct rds_ib_mr_pool *pool) 263static inline struct rds_ib_mr *rds_ib_reuse_fmr(struct rds_ib_mr_pool *pool)
272{ 264{
273 struct rds_ib_mr *ibmr = NULL; 265 struct rds_ib_mr *ibmr = NULL;
274 struct xlist_head *ret; 266 struct llist_node *ret;
275 unsigned long *flag; 267 unsigned long *flag;
276 268
277 preempt_disable(); 269 preempt_disable();
278 flag = &__get_cpu_var(clean_list_grace); 270 flag = &__get_cpu_var(clean_list_grace);
279 set_bit(CLEAN_LIST_BUSY_BIT, flag); 271 set_bit(CLEAN_LIST_BUSY_BIT, flag);
280 ret = xlist_del_head(&pool->clean_list); 272 ret = llist_del_first(&pool->clean_list);
281 if (ret) 273 if (ret)
282 ibmr = list_entry(ret, struct rds_ib_mr, xlist); 274 ibmr = llist_entry(ret, struct rds_ib_mr, llnode);
283 275
284 clear_bit(CLEAN_LIST_BUSY_BIT, flag); 276 clear_bit(CLEAN_LIST_BUSY_BIT, flag);
285 preempt_enable(); 277 preempt_enable();
@@ -529,46 +521,44 @@ static inline unsigned int rds_ib_flush_goal(struct rds_ib_mr_pool *pool, int fr
529} 521}
530 522
531/* 523/*
532 * given an xlist of mrs, put them all into the list_head for more processing 524 * given an llist of mrs, put them all into the list_head for more processing
533 */ 525 */
534static void xlist_append_to_list(struct xlist_head *xlist, struct list_head *list) 526static void llist_append_to_list(struct llist_head *llist, struct list_head *list)
535{ 527{
536 struct rds_ib_mr *ibmr; 528 struct rds_ib_mr *ibmr;
537 struct xlist_head splice; 529 struct llist_node *node;
538 struct xlist_head *cur; 530 struct llist_node *next;
539 struct xlist_head *next; 531
540 532 node = llist_del_all(llist);
541 splice.next = NULL; 533 while (node) {
542 xlist_splice(xlist, &splice); 534 next = node->next;
543 cur = splice.next; 535 ibmr = llist_entry(node, struct rds_ib_mr, llnode);
544 while (cur) {
545 next = cur->next;
546 ibmr = list_entry(cur, struct rds_ib_mr, xlist);
547 list_add_tail(&ibmr->unmap_list, list); 536 list_add_tail(&ibmr->unmap_list, list);
548 cur = next; 537 node = next;
549 } 538 }
550} 539}
551 540
552/* 541/*
553 * this takes a list head of mrs and turns it into an xlist of clusters. 542 * this takes a list head of mrs and turns it into linked llist nodes
554 * each cluster has an xlist of MR_CLUSTER_SIZE mrs that are ready for 543 * of clusters. Each cluster has linked llist nodes of
555 * reuse. 544 * MR_CLUSTER_SIZE mrs that are ready for reuse.
556 */ 545 */
557static void list_append_to_xlist(struct rds_ib_mr_pool *pool, 546static void list_to_llist_nodes(struct rds_ib_mr_pool *pool,
558 struct list_head *list, struct xlist_head *xlist, 547 struct list_head *list,
559 struct xlist_head **tail_ret) 548 struct llist_node **nodes_head,
549 struct llist_node **nodes_tail)
560{ 550{
561 struct rds_ib_mr *ibmr; 551 struct rds_ib_mr *ibmr;
562 struct xlist_head *cur_mr = xlist; 552 struct llist_node *cur = NULL;
563 struct xlist_head *tail_mr = NULL; 553 struct llist_node **next = nodes_head;
564 554
565 list_for_each_entry(ibmr, list, unmap_list) { 555 list_for_each_entry(ibmr, list, unmap_list) {
566 tail_mr = &ibmr->xlist; 556 cur = &ibmr->llnode;
567 tail_mr->next = NULL; 557 *next = cur;
568 cur_mr->next = tail_mr; 558 next = &cur->next;
569 cur_mr = tail_mr;
570 } 559 }
571 *tail_ret = tail_mr; 560 *next = NULL;
561 *nodes_tail = cur;
572} 562}
573 563
574/* 564/*
@@ -581,8 +571,8 @@ static int rds_ib_flush_mr_pool(struct rds_ib_mr_pool *pool,
581 int free_all, struct rds_ib_mr **ibmr_ret) 571 int free_all, struct rds_ib_mr **ibmr_ret)
582{ 572{
583 struct rds_ib_mr *ibmr, *next; 573 struct rds_ib_mr *ibmr, *next;
584 struct xlist_head clean_xlist; 574 struct llist_node *clean_nodes;
585 struct xlist_head *clean_tail; 575 struct llist_node *clean_tail;
586 LIST_HEAD(unmap_list); 576 LIST_HEAD(unmap_list);
587 LIST_HEAD(fmr_list); 577 LIST_HEAD(fmr_list);
588 unsigned long unpinned = 0; 578 unsigned long unpinned = 0;
@@ -603,7 +593,7 @@ static int rds_ib_flush_mr_pool(struct rds_ib_mr_pool *pool,
603 593
604 prepare_to_wait(&pool->flush_wait, &wait, 594 prepare_to_wait(&pool->flush_wait, &wait,
605 TASK_UNINTERRUPTIBLE); 595 TASK_UNINTERRUPTIBLE);
606 if (xlist_empty(&pool->clean_list)) 596 if (llist_empty(&pool->clean_list))
607 schedule(); 597 schedule();
608 598
609 ibmr = rds_ib_reuse_fmr(pool); 599 ibmr = rds_ib_reuse_fmr(pool);
@@ -628,10 +618,10 @@ static int rds_ib_flush_mr_pool(struct rds_ib_mr_pool *pool,
628 /* Get the list of all MRs to be dropped. Ordering matters - 618 /* Get the list of all MRs to be dropped. Ordering matters -
629 * we want to put drop_list ahead of free_list. 619 * we want to put drop_list ahead of free_list.
630 */ 620 */
631 xlist_append_to_list(&pool->drop_list, &unmap_list); 621 llist_append_to_list(&pool->drop_list, &unmap_list);
632 xlist_append_to_list(&pool->free_list, &unmap_list); 622 llist_append_to_list(&pool->free_list, &unmap_list);
633 if (free_all) 623 if (free_all)
634 xlist_append_to_list(&pool->clean_list, &unmap_list); 624 llist_append_to_list(&pool->clean_list, &unmap_list);
635 625
636 free_goal = rds_ib_flush_goal(pool, free_all); 626 free_goal = rds_ib_flush_goal(pool, free_all);
637 627
@@ -663,22 +653,22 @@ static int rds_ib_flush_mr_pool(struct rds_ib_mr_pool *pool,
663 if (!list_empty(&unmap_list)) { 653 if (!list_empty(&unmap_list)) {
664 /* we have to make sure that none of the things we're about 654 /* we have to make sure that none of the things we're about
665 * to put on the clean list would race with other cpus trying 655 * to put on the clean list would race with other cpus trying
666 * to pull items off. The xlist would explode if we managed to 656 * to pull items off. The llist would explode if we managed to
667 * remove something from the clean list and then add it back again 657 * remove something from the clean list and then add it back again
668 * while another CPU was spinning on that same item in xlist_del_head. 658 * while another CPU was spinning on that same item in llist_del_first.
669 * 659 *
670 * This is pretty unlikely, but just in case wait for an xlist grace period 660 * This is pretty unlikely, but just in case wait for an llist grace period
671 * here before adding anything back into the clean list. 661 * here before adding anything back into the clean list.
672 */ 662 */
673 wait_clean_list_grace(); 663 wait_clean_list_grace();
674 664
675 list_append_to_xlist(pool, &unmap_list, &clean_xlist, &clean_tail); 665 list_to_llist_nodes(pool, &unmap_list, &clean_nodes, &clean_tail);
676 if (ibmr_ret) 666 if (ibmr_ret)
677 refill_local(pool, &clean_xlist, ibmr_ret); 667 *ibmr_ret = llist_entry(clean_nodes, struct rds_ib_mr, llnode);
678 668
679 /* refill_local may have emptied our list */ 669 /* more than one entry in llist nodes */
680 if (!xlist_empty(&clean_xlist)) 670 if (clean_nodes->next)
681 xlist_add(clean_xlist.next, clean_tail, &pool->clean_list); 671 llist_add_batch(clean_nodes->next, clean_tail, &pool->clean_list);
682 672
683 } 673 }
684 674
@@ -711,9 +701,9 @@ void rds_ib_free_mr(void *trans_private, int invalidate)
711 701
712 /* Return it to the pool's free list */ 702 /* Return it to the pool's free list */
713 if (ibmr->remap_count >= pool->fmr_attr.max_maps) 703 if (ibmr->remap_count >= pool->fmr_attr.max_maps)
714 xlist_add(&ibmr->xlist, &ibmr->xlist, &pool->drop_list); 704 llist_add(&ibmr->llnode, &pool->drop_list);
715 else 705 else
716 xlist_add(&ibmr->xlist, &ibmr->xlist, &pool->free_list); 706 llist_add(&ibmr->llnode, &pool->free_list);
717 707
718 atomic_add(ibmr->sg_len, &pool->free_pinned); 708 atomic_add(ibmr->sg_len, &pool->free_pinned);
719 atomic_inc(&pool->dirty_count); 709 atomic_inc(&pool->dirty_count);
diff --git a/net/rds/xlist.h b/net/rds/xlist.h
deleted file mode 100644
index e6b5190daddd..000000000000
--- a/net/rds/xlist.h
+++ /dev/null
@@ -1,80 +0,0 @@
1#ifndef _LINUX_XLIST_H
2#define _LINUX_XLIST_H
3
4#include <linux/stddef.h>
5#include <linux/poison.h>
6#include <linux/prefetch.h>
7#include <asm/system.h>
8
9struct xlist_head {
10 struct xlist_head *next;
11};
12
13static inline void INIT_XLIST_HEAD(struct xlist_head *list)
14{
15 list->next = NULL;
16}
17
18static inline int xlist_empty(struct xlist_head *head)
19{
20 return head->next == NULL;
21}
22
23static inline void xlist_add(struct xlist_head *new, struct xlist_head *tail,
24 struct xlist_head *head)
25{
26 struct xlist_head *cur;
27 struct xlist_head *check;
28
29 while (1) {
30 cur = head->next;
31 tail->next = cur;
32 check = cmpxchg(&head->next, cur, new);
33 if (check == cur)
34 break;
35 }
36}
37
38static inline struct xlist_head *xlist_del_head(struct xlist_head *head)
39{
40 struct xlist_head *cur;
41 struct xlist_head *check;
42 struct xlist_head *next;
43
44 while (1) {
45 cur = head->next;
46 if (!cur)
47 goto out;
48
49 next = cur->next;
50 check = cmpxchg(&head->next, cur, next);
51 if (check == cur)
52 goto out;
53 }
54out:
55 return cur;
56}
57
58static inline struct xlist_head *xlist_del_head_fast(struct xlist_head *head)
59{
60 struct xlist_head *cur;
61
62 cur = head->next;
63 if (!cur)
64 return NULL;
65
66 head->next = cur->next;
67 return cur;
68}
69
70static inline void xlist_splice(struct xlist_head *list,
71 struct xlist_head *head)
72{
73 struct xlist_head *cur;
74
75 WARN_ON(head->next);
76 cur = xchg(&list->next, NULL);
77 head->next = cur;
78}
79
80#endif