aboutsummaryrefslogtreecommitdiffstats
path: root/net/core/pktgen.c
diff options
context:
space:
mode:
authorEric Dumazet <eric.dumazet@gmail.com>2011-01-25 16:26:05 -0500
committerDavid S. Miller <davem@davemloft.net>2011-01-25 16:26:05 -0500
commit26ad787962ef84677a48c56039d3c9769b84f847 (patch)
tree46ca79b2bb1a5723ac60a0138d624efdf1e4d3e3 /net/core/pktgen.c
parent672bda337060fa2ff99866a6ebfa3ae036f8b23b (diff)
pktgen: speedup fragmented skbs
We spend lot of time clearing pages in pktgen. (Or not clearing them on ipv6 and leaking kernel memory) Since we dont modify them, we can use one zeroed page, and get references on it. This page can use NUMA affinity as well. Define pktgen_finalize_skb() helper, used both in ipv4 and ipv6 Results using skbs with one frag : Before patch : Result: OK: 608980458(c608978520+d1938) nsec, 1000000000 (100byte,1frags) 1642088pps 1313Mb/sec (1313670400bps) errors: 0 After patch : Result: OK: 345285014(c345283891+d1123) nsec, 1000000000 (100byte,1frags) 2896158pps 2316Mb/sec (2316926400bps) errors: 0 Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com> Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'net/core/pktgen.c')
-rw-r--r--net/core/pktgen.c234
1 files changed, 93 insertions, 141 deletions
diff --git a/net/core/pktgen.c b/net/core/pktgen.c
index a9e7fc4c461f..d73b77adb676 100644
--- a/net/core/pktgen.c
+++ b/net/core/pktgen.c
@@ -251,6 +251,7 @@ struct pktgen_dev {
251 int max_pkt_size; /* = ETH_ZLEN; */ 251 int max_pkt_size; /* = ETH_ZLEN; */
252 int pkt_overhead; /* overhead for MPLS, VLANs, IPSEC etc */ 252 int pkt_overhead; /* overhead for MPLS, VLANs, IPSEC etc */
253 int nfrags; 253 int nfrags;
254 struct page *page;
254 u64 delay; /* nano-seconds */ 255 u64 delay; /* nano-seconds */
255 256
256 __u64 count; /* Default No packets to send */ 257 __u64 count; /* Default No packets to send */
@@ -1134,6 +1135,10 @@ static ssize_t pktgen_if_write(struct file *file,
1134 if (node_possible(value)) { 1135 if (node_possible(value)) {
1135 pkt_dev->node = value; 1136 pkt_dev->node = value;
1136 sprintf(pg_result, "OK: node=%d", pkt_dev->node); 1137 sprintf(pg_result, "OK: node=%d", pkt_dev->node);
1138 if (pkt_dev->page) {
1139 put_page(pkt_dev->page);
1140 pkt_dev->page = NULL;
1141 }
1137 } 1142 }
1138 else 1143 else
1139 sprintf(pg_result, "ERROR: node not possible"); 1144 sprintf(pg_result, "ERROR: node not possible");
@@ -2605,6 +2610,90 @@ static inline __be16 build_tci(unsigned int id, unsigned int cfi,
2605 return htons(id | (cfi << 12) | (prio << 13)); 2610 return htons(id | (cfi << 12) | (prio << 13));
2606} 2611}
2607 2612
2613static void pktgen_finalize_skb(struct pktgen_dev *pkt_dev, struct sk_buff *skb,
2614 int datalen)
2615{
2616 struct timeval timestamp;
2617 struct pktgen_hdr *pgh;
2618
2619 pgh = (struct pktgen_hdr *)skb_put(skb, sizeof(*pgh));
2620 datalen -= sizeof(*pgh);
2621
2622 if (pkt_dev->nfrags <= 0) {
2623 pgh = (struct pktgen_hdr *)skb_put(skb, datalen);
2624 memset(pgh + 1, 0, datalen);
2625 } else {
2626 int frags = pkt_dev->nfrags;
2627 int i, len;
2628
2629
2630 if (frags > MAX_SKB_FRAGS)
2631 frags = MAX_SKB_FRAGS;
2632 len = datalen - frags * PAGE_SIZE;
2633 if (len > 0) {
2634 memset(skb_put(skb, len), 0, len);
2635 datalen = frags * PAGE_SIZE;
2636 }
2637
2638 i = 0;
2639 while (datalen > 0) {
2640 if (unlikely(!pkt_dev->page)) {
2641 int node = numa_node_id();
2642
2643 if (pkt_dev->node >= 0 && (pkt_dev->flags & F_NODE))
2644 node = pkt_dev->node;
2645 pkt_dev->page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
2646 if (!pkt_dev->page)
2647 break;
2648 }
2649 skb_shinfo(skb)->frags[i].page = pkt_dev->page;
2650 get_page(pkt_dev->page);
2651 skb_shinfo(skb)->frags[i].page_offset = 0;
2652 skb_shinfo(skb)->frags[i].size =
2653 (datalen < PAGE_SIZE ? datalen : PAGE_SIZE);
2654 datalen -= skb_shinfo(skb)->frags[i].size;
2655 skb->len += skb_shinfo(skb)->frags[i].size;
2656 skb->data_len += skb_shinfo(skb)->frags[i].size;
2657 i++;
2658 skb_shinfo(skb)->nr_frags = i;
2659 }
2660
2661 while (i < frags) {
2662 int rem;
2663
2664 if (i == 0)
2665 break;
2666
2667 rem = skb_shinfo(skb)->frags[i - 1].size / 2;
2668 if (rem == 0)
2669 break;
2670
2671 skb_shinfo(skb)->frags[i - 1].size -= rem;
2672
2673 skb_shinfo(skb)->frags[i] =
2674 skb_shinfo(skb)->frags[i - 1];
2675 get_page(skb_shinfo(skb)->frags[i].page);
2676 skb_shinfo(skb)->frags[i].page =
2677 skb_shinfo(skb)->frags[i - 1].page;
2678 skb_shinfo(skb)->frags[i].page_offset +=
2679 skb_shinfo(skb)->frags[i - 1].size;
2680 skb_shinfo(skb)->frags[i].size = rem;
2681 i++;
2682 skb_shinfo(skb)->nr_frags = i;
2683 }
2684 }
2685
2686 /* Stamp the time, and sequence number,
2687 * convert them to network byte order
2688 */
2689 pgh->pgh_magic = htonl(PKTGEN_MAGIC);
2690 pgh->seq_num = htonl(pkt_dev->seq_num);
2691
2692 do_gettimeofday(&timestamp);
2693 pgh->tv_sec = htonl(timestamp.tv_sec);
2694 pgh->tv_usec = htonl(timestamp.tv_usec);
2695}
2696
2608static struct sk_buff *fill_packet_ipv4(struct net_device *odev, 2697static struct sk_buff *fill_packet_ipv4(struct net_device *odev,
2609 struct pktgen_dev *pkt_dev) 2698 struct pktgen_dev *pkt_dev)
2610{ 2699{
@@ -2613,7 +2702,6 @@ static struct sk_buff *fill_packet_ipv4(struct net_device *odev,
2613 struct udphdr *udph; 2702 struct udphdr *udph;
2614 int datalen, iplen; 2703 int datalen, iplen;
2615 struct iphdr *iph; 2704 struct iphdr *iph;
2616 struct pktgen_hdr *pgh = NULL;
2617 __be16 protocol = htons(ETH_P_IP); 2705 __be16 protocol = htons(ETH_P_IP);
2618 __be32 *mpls; 2706 __be32 *mpls;
2619 __be16 *vlan_tci = NULL; /* Encapsulates priority and VLAN ID */ 2707 __be16 *vlan_tci = NULL; /* Encapsulates priority and VLAN ID */
@@ -2729,76 +2817,7 @@ static struct sk_buff *fill_packet_ipv4(struct net_device *odev,
2729 pkt_dev->pkt_overhead); 2817 pkt_dev->pkt_overhead);
2730 skb->dev = odev; 2818 skb->dev = odev;
2731 skb->pkt_type = PACKET_HOST; 2819 skb->pkt_type = PACKET_HOST;
2732 2820 pktgen_finalize_skb(pkt_dev, skb, datalen);
2733 if (pkt_dev->nfrags <= 0) {
2734 pgh = (struct pktgen_hdr *)skb_put(skb, datalen);
2735 memset(pgh + 1, 0, datalen - sizeof(struct pktgen_hdr));
2736 } else {
2737 int frags = pkt_dev->nfrags;
2738 int i, len;
2739
2740 pgh = (struct pktgen_hdr *)(((char *)(udph)) + 8);
2741
2742 if (frags > MAX_SKB_FRAGS)
2743 frags = MAX_SKB_FRAGS;
2744 if (datalen > frags * PAGE_SIZE) {
2745 len = datalen - frags * PAGE_SIZE;
2746 memset(skb_put(skb, len), 0, len);
2747 datalen = frags * PAGE_SIZE;
2748 }
2749
2750 i = 0;
2751 while (datalen > 0) {
2752 struct page *page = alloc_pages(GFP_KERNEL | __GFP_ZERO, 0);
2753 skb_shinfo(skb)->frags[i].page = page;
2754 skb_shinfo(skb)->frags[i].page_offset = 0;
2755 skb_shinfo(skb)->frags[i].size =
2756 (datalen < PAGE_SIZE ? datalen : PAGE_SIZE);
2757 datalen -= skb_shinfo(skb)->frags[i].size;
2758 skb->len += skb_shinfo(skb)->frags[i].size;
2759 skb->data_len += skb_shinfo(skb)->frags[i].size;
2760 i++;
2761 skb_shinfo(skb)->nr_frags = i;
2762 }
2763
2764 while (i < frags) {
2765 int rem;
2766
2767 if (i == 0)
2768 break;
2769
2770 rem = skb_shinfo(skb)->frags[i - 1].size / 2;
2771 if (rem == 0)
2772 break;
2773
2774 skb_shinfo(skb)->frags[i - 1].size -= rem;
2775
2776 skb_shinfo(skb)->frags[i] =
2777 skb_shinfo(skb)->frags[i - 1];
2778 get_page(skb_shinfo(skb)->frags[i].page);
2779 skb_shinfo(skb)->frags[i].page =
2780 skb_shinfo(skb)->frags[i - 1].page;
2781 skb_shinfo(skb)->frags[i].page_offset +=
2782 skb_shinfo(skb)->frags[i - 1].size;
2783 skb_shinfo(skb)->frags[i].size = rem;
2784 i++;
2785 skb_shinfo(skb)->nr_frags = i;
2786 }
2787 }
2788
2789 /* Stamp the time, and sequence number,
2790 * convert them to network byte order
2791 */
2792 if (pgh) {
2793 struct timeval timestamp;
2794
2795 pgh->pgh_magic = htonl(PKTGEN_MAGIC);
2796 pgh->seq_num = htonl(pkt_dev->seq_num);
2797
2798 do_gettimeofday(&timestamp);
2799 pgh->tv_sec = htonl(timestamp.tv_sec);
2800 pgh->tv_usec = htonl(timestamp.tv_usec);
2801 }
2802 2821
2803#ifdef CONFIG_XFRM 2822#ifdef CONFIG_XFRM
2804 if (!process_ipsec(pkt_dev, skb, protocol)) 2823 if (!process_ipsec(pkt_dev, skb, protocol))
@@ -2980,7 +2999,6 @@ static struct sk_buff *fill_packet_ipv6(struct net_device *odev,
2980 struct udphdr *udph; 2999 struct udphdr *udph;
2981 int datalen; 3000 int datalen;
2982 struct ipv6hdr *iph; 3001 struct ipv6hdr *iph;
2983 struct pktgen_hdr *pgh = NULL;
2984 __be16 protocol = htons(ETH_P_IPV6); 3002 __be16 protocol = htons(ETH_P_IPV6);
2985 __be32 *mpls; 3003 __be32 *mpls;
2986 __be16 *vlan_tci = NULL; /* Encapsulates priority and VLAN ID */ 3004 __be16 *vlan_tci = NULL; /* Encapsulates priority and VLAN ID */
@@ -3083,75 +3101,7 @@ static struct sk_buff *fill_packet_ipv6(struct net_device *odev,
3083 skb->dev = odev; 3101 skb->dev = odev;
3084 skb->pkt_type = PACKET_HOST; 3102 skb->pkt_type = PACKET_HOST;
3085 3103
3086 if (pkt_dev->nfrags <= 0) 3104 pktgen_finalize_skb(pkt_dev, skb, datalen);
3087 pgh = (struct pktgen_hdr *)skb_put(skb, datalen);
3088 else {
3089 int frags = pkt_dev->nfrags;
3090 int i;
3091
3092 pgh = (struct pktgen_hdr *)(((char *)(udph)) + 8);
3093
3094 if (frags > MAX_SKB_FRAGS)
3095 frags = MAX_SKB_FRAGS;
3096 if (datalen > frags * PAGE_SIZE) {
3097 skb_put(skb, datalen - frags * PAGE_SIZE);
3098 datalen = frags * PAGE_SIZE;
3099 }
3100
3101 i = 0;
3102 while (datalen > 0) {
3103 struct page *page = alloc_pages(GFP_KERNEL, 0);
3104 skb_shinfo(skb)->frags[i].page = page;
3105 skb_shinfo(skb)->frags[i].page_offset = 0;
3106 skb_shinfo(skb)->frags[i].size =
3107 (datalen < PAGE_SIZE ? datalen : PAGE_SIZE);
3108 datalen -= skb_shinfo(skb)->frags[i].size;
3109 skb->len += skb_shinfo(skb)->frags[i].size;
3110 skb->data_len += skb_shinfo(skb)->frags[i].size;
3111 i++;
3112 skb_shinfo(skb)->nr_frags = i;
3113 }
3114
3115 while (i < frags) {
3116 int rem;
3117
3118 if (i == 0)
3119 break;
3120
3121 rem = skb_shinfo(skb)->frags[i - 1].size / 2;
3122 if (rem == 0)
3123 break;
3124
3125 skb_shinfo(skb)->frags[i - 1].size -= rem;
3126
3127 skb_shinfo(skb)->frags[i] =
3128 skb_shinfo(skb)->frags[i - 1];
3129 get_page(skb_shinfo(skb)->frags[i].page);
3130 skb_shinfo(skb)->frags[i].page =
3131 skb_shinfo(skb)->frags[i - 1].page;
3132 skb_shinfo(skb)->frags[i].page_offset +=
3133 skb_shinfo(skb)->frags[i - 1].size;
3134 skb_shinfo(skb)->frags[i].size = rem;
3135 i++;
3136 skb_shinfo(skb)->nr_frags = i;
3137 }
3138 }
3139
3140 /* Stamp the time, and sequence number,
3141 * convert them to network byte order
3142 * should we update cloned packets too ?
3143 */
3144 if (pgh) {
3145 struct timeval timestamp;
3146
3147 pgh->pgh_magic = htonl(PKTGEN_MAGIC);
3148 pgh->seq_num = htonl(pkt_dev->seq_num);
3149
3150 do_gettimeofday(&timestamp);
3151 pgh->tv_sec = htonl(timestamp.tv_sec);
3152 pgh->tv_usec = htonl(timestamp.tv_usec);
3153 }
3154 /* pkt_dev->seq_num++; FF: you really mean this? */
3155 3105
3156 return skb; 3106 return skb;
3157} 3107}
@@ -3884,6 +3834,8 @@ static int pktgen_remove_device(struct pktgen_thread *t,
3884 free_SAs(pkt_dev); 3834 free_SAs(pkt_dev);
3885#endif 3835#endif
3886 vfree(pkt_dev->flows); 3836 vfree(pkt_dev->flows);
3837 if (pkt_dev->page)
3838 put_page(pkt_dev->page);
3887 kfree(pkt_dev); 3839 kfree(pkt_dev);
3888 return 0; 3840 return 0;
3889} 3841}