litmus-rt-edfsc.git - LITMUS^RT with the EDF-SC plugin for Real-Time Systems journal paper

diff options

author	Andrew Victor <linux@maxim.org.za>	2008-04-16 15:43:49 -0400
committer	Russell King <rmk+kernel@arm.linux.org.uk>	2008-04-17 10:55:54 -0400
commit	ad48ce74f70a201c4c1cf3b4e8f6b6203a2e4a8d (patch)
tree	982ac3f881a9c09115ae9b9723c343ed00dcfc8d /arch/arm/mach-at91/at91sam926x_time.c
parent	11aadac4f6adc032cfd1e41c348a7a568819ed94 (diff)

[ARM] 4989/1: [AT91] SAM9 ClockSource / ClockEvents

Update AT91SAM9/CAP9 PIT driver to use generic time and clockevent infrastructure: - Clocksource gives sub-microsecond timestamp precision, assuming memory is clocked at over 16 MHz. It's less than a 32 bit counter, unless it's is also generating IRQs. - Clockevent device supports periodic mode only; no oneshot support from this hardware. No IRQs generated unless it's the active clocksource. Later, another timer (probably from a TC module) can provide a oneshot clockevent device to get NO_HZ and High-Res-Timer behavior. This also updates the timekeeping to use the actual master clock rate on the system, instead of compile-time <asm/arch/timex.h> constants matching what Atmel's EK boards use. (Product boards may well differ!) Plus cleanup: rename "*_timer*" symbols to "*_pit*" (there are other timers, but only one PIT); shorter lines; remove needless CPP stuff; make several symbols static; etc. Signed-off-by: David Brownell <dbrownell@users.sourceforge.net> Signed-off-by: Andrew Victor <linux@maxim.org.za> Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>

Diffstat (limited to 'arch/arm/mach-at91/at91sam926x_time.c')

-rw-r--r--

arch/arm/mach-at91/at91sam926x_time.c

171

1 files changed, 125 insertions, 46 deletions


diff --git a/arch/arm/mach-at91/at91sam926x_time.c b/arch/arm/mach-at91/at91sam926x_time.c index e38d23770992..5cecbd7de6a6 100644 --- a/arch/arm/mach-at91/at91sam926x_time.c +++ b/arch/arm/mach-at91/at91sam926x_time.c
@@ -1,23 +1,20 @@
1	/*	1	/*
2	* linux/arch/arm/mach-at91/at91sam926x_time.c	2	* at91sam926x_time.c - Periodic Interval Timer (PIT) for at91sam926x
3	*	3	*
4	* Copyright (C) 2005-2006 M. Amine SAYA, ATMEL Rousset, France	4	* Copyright (C) 2005-2006 M. Amine SAYA, ATMEL Rousset, France
5	* Revision 2005 M. Nicolas Diremdjian, ATMEL Rousset, France	5	* Revision 2005 M. Nicolas Diremdjian, ATMEL Rousset, France
		6	* Converted to ClockSource/ClockEvents by David Brownell.
6	*	7	*
7	* This program is free software; you can redistribute it and/or modify	8	* This program is free software; you can redistribute it and/or modify
8	* it under the terms of the GNU General Public License version 2 as	9	* it under the terms of the GNU General Public License version 2 as
9	* published by the Free Software Foundation.	10	* published by the Free Software Foundation.
10	*/	11	*/
11
12	#include <linux/init.h>
13	#include <linux/interrupt.h>	12	#include <linux/interrupt.h>
14	#include <linux/irq.h>	13	#include <linux/irq.h>
15	#include <linux/kernel.h>	14	#include <linux/kernel.h>
16	#include <linux/sched.h>	15	#include <linux/clk.h>
17	#include <linux/time.h>	16	#include <linux/clockchips.h>
18		17
19	#include <asm/hardware.h>
20	#include <asm/io.h>
21	#include <asm/mach/time.h>	18	#include <asm/mach/time.h>
22		19
23	#include <asm/arch/at91_pit.h>	20	#include <asm/arch/at91_pit.h>
@@ -26,85 +23,167 @@
26	#define PIT_CPIV(x) ((x) & AT91_PIT_CPIV)	23	#define PIT_CPIV(x) ((x) & AT91_PIT_CPIV)
27	#define PIT_PICNT(x) (((x) & AT91_PIT_PICNT) >> 20)	24	#define PIT_PICNT(x) (((x) & AT91_PIT_PICNT) >> 20)
28		25
		26	static u32 pit_cycle; /* write-once */
		27	static u32 pit_cnt; /* access only w/system irq blocked */
		28
		29
29	/*	30	/*
30	* Returns number of microseconds since last timer interrupt. Note that interrupts	31	* Clocksource: just a monotonic counter of MCK/16 cycles.
31	* will have been disabled by do_gettimeofday()	32	* We don't care whether or not PIT irqs are enabled.
32	* 'LATCH' is hwclock ticks (see CLOCK_TICK_RATE in timex.h) per jiffy.
33	*/	33	*/
34	static unsigned long at91sam926x_gettimeoffset(void)	34	static cycle_t read_pit_clk(void)
35	{	35	{
36	unsigned long elapsed;	36	unsigned long flags;
37	unsigned long t = at91_sys_read(AT91_PIT_PIIR);	37	u32 elapsed;
		38	u32 t;
		39
		40	raw_local_irq_save(flags);
		41	elapsed = pit_cnt;
		42	t = at91_sys_read(AT91_PIT_PIIR);
		43	raw_local_irq_restore(flags);
		44
		45	elapsed += PIT_PICNT(t) * pit_cycle;
		46	elapsed += PIT_CPIV(t);
		47	return elapsed;
		48	}
		49
		50	static struct clocksource pit_clk = {
		51	.name = "pit",
		52	.rating = 175,
		53	.read = read_pit_clk,
		54	.shift = 20,
		55	.flags = CLOCK_SOURCE_IS_CONTINUOUS,
		56	};
38		57
39	elapsed = (PIT_PICNT(t) * LATCH) + PIT_CPIV(t); /* hardware clock cycles */
40		58
41	return (unsigned long)(elapsed * jiffies_to_usecs(1)) / LATCH;	59	/*
		60	* Clockevent device: interrupts every 1/HZ (== pit_cycles * MCK/16)
		61	*/
		62	static void
		63	pit_clkevt_mode(enum clock_event_mode mode, struct clock_event_device *dev)
		64	{
		65	unsigned long flags;
		66
		67	switch (mode) {
		68	case CLOCK_EVT_MODE_PERIODIC:
		69	/* update clocksource counter, then enable the IRQ */
		70	raw_local_irq_save(flags);
		71	pit_cnt += pit_cycle * PIT_PICNT(at91_sys_read(AT91_PIT_PIVR));
		72	at91_sys_write(AT91_PIT_MR, (pit_cycle - 1) \| AT91_PIT_PITEN
		73	\| AT91_PIT_PITIEN);
		74	raw_local_irq_restore(flags);
		75	break;
		76	case CLOCK_EVT_MODE_ONESHOT:
		77	BUG();
		78	/* FALLTHROUGH */
		79	case CLOCK_EVT_MODE_SHUTDOWN:
		80	case CLOCK_EVT_MODE_UNUSED:
		81	/* disable irq, leaving the clocksource active */
		82	at91_sys_write(AT91_PIT_MR, (pit_cycle - 1) \| AT91_PIT_PITEN);
		83	break;
		84	case CLOCK_EVT_MODE_RESUME:
		85	break;
		86	}
42	}	87	}
43		88
		89	static struct clock_event_device pit_clkevt = {
		90	.name = "pit",
		91	.features = CLOCK_EVT_FEAT_PERIODIC,
		92	.shift = 32,
		93	.rating = 100,
		94	.cpumask = CPU_MASK_CPU0,
		95	.set_mode = pit_clkevt_mode,
		96	};
		97
		98
44	/*	99	/*
45	* IRQ handler for the timer.	100	* IRQ handler for the timer.
46	*/	101	*/
47	static irqreturn_t at91sam926x_timer_interrupt(int irq, void *dev_id)	102	static irqreturn_t at91sam926x_pit_interrupt(int irq, void *dev_id)
48	{	103	{
49	volatile long nr_ticks;
50		104
51	if (at91_sys_read(AT91_PIT_SR) & AT91_PIT_PITS) { /* This is a shared interrupt */	105	/* The PIT interrupt may be disabled, and is shared */
52	/* Get number to ticks performed before interrupt and clear PIT interrupt */	106	if ((pit_clkevt.mode == CLOCK_EVT_MODE_PERIODIC)
		107	&& (at91_sys_read(AT91_PIT_SR) & AT91_PIT_PITS)) {
		108	unsigned nr_ticks;
		109
		110	/* Get number of ticks performed before irq, and ack it */
53	nr_ticks = PIT_PICNT(at91_sys_read(AT91_PIT_PIVR));	111	nr_ticks = PIT_PICNT(at91_sys_read(AT91_PIT_PIVR));
54	do {	112	do {
55	timer_tick();	113	pit_cnt += pit_cycle;
		114	pit_clkevt.event_handler(&pit_clkevt);
56	nr_ticks--;	115	nr_ticks--;
57	} while (nr_ticks);	116	} while (nr_ticks);
58		117
59	return IRQ_HANDLED;	118	return IRQ_HANDLED;
60	} else	119	}
61	return IRQ_NONE; /* not handled */	120
		121	return IRQ_NONE;
62	}	122	}
63		123
64	static struct irqaction at91sam926x_timer_irq = {	124	static struct irqaction at91sam926x_pit_irq = {
65	.name = "at91_tick",	125	.name = "at91_tick",
66	.flags = IRQF_SHARED \| IRQF_DISABLED \| IRQF_TIMER \| IRQF_IRQPOLL,	126	.flags = IRQF_SHARED \| IRQF_DISABLED \| IRQF_TIMER \| IRQF_IRQPOLL,
67	.handler = at91sam926x_timer_interrupt	127	.handler = at91sam926x_pit_interrupt
68	};	128	};
69		129
70	void at91sam926x_timer_reset(void)	130	static void at91sam926x_pit_reset(void)
71	{	131	{
72	/* Disable timer */	132	/* Disable timer and irqs */
73	at91_sys_write(AT91_PIT_MR, 0);	133	at91_sys_write(AT91_PIT_MR, 0);
74		134
75	/* Clear any pending interrupts */	135	/* Clear any pending interrupts, wait for PIT to stop counting */
76	(void) at91_sys_read(AT91_PIT_PIVR);	136	while (PIT_CPIV(at91_sys_read(AT91_PIT_PIVR)) != 0)
		137	cpu_relax();
77		138
78	/* Set Period Interval timer and enable its interrupt */	139	/* Start PIT but don't enable IRQ */
79	at91_sys_write(AT91_PIT_MR, (LATCH & AT91_PIT_PIV) \| AT91_PIT_PITIEN \| AT91_PIT_PITEN);	140	at91_sys_write(AT91_PIT_MR, (pit_cycle - 1) \| AT91_PIT_PITEN);
80	}	141	}
81		142
82	/*	143	/*
83	* Set up timer interrupt.	144	* Set up both clocksource and clockevent support.
84	*/	145	*/
85	void __init at91sam926x_timer_init(void)	146	static void __init at91sam926x_pit_init(void)
86	{	147	{
87	/* Initialize and enable the timer */	148	unsigned long pit_rate;
88	at91sam926x_timer_reset();	149	unsigned bits;
		150
		151	/*
		152	* Use our actual MCK to figure out how many MCK/16 ticks per
		153	* 1/HZ period (instead of a compile-time constant LATCH).
		154	*/
		155	pit_rate = clk_get_rate(clk_get(NULL, "mck")) / 16;
		156	pit_cycle = (pit_rate + HZ/2) / HZ;
		157	WARN_ON(((pit_cycle - 1) & ~AT91_PIT_PIV) != 0);
89		158
90	/* Make IRQs happen for the system timer. */	159	/* Initialize and enable the timer */
91	setup_irq(AT91_ID_SYS, &at91sam926x_timer_irq);	160	at91sam926x_pit_reset();
		161
		162	/*
		163	* Register clocksource. The high order bits of PIV are unused,
		164	* so this isn't a 32-bit counter unless we get clockevent irqs.
		165	*/
		166	pit_clk.mult = clocksource_hz2mult(pit_rate, pit_clk.shift);
		167	bits = 12 /* PICNT / + ilog2(pit_cycle) / PIV */;
		168	pit_clk.mask = CLOCKSOURCE_MASK(bits);
		169	clocksource_register(&pit_clk);
		170
		171	/* Set up irq handler */
		172	setup_irq(AT91_ID_SYS, &at91sam926x_pit_irq);
		173
		174	/* Set up and register clockevents */
		175	pit_clkevt.mult = div_sc(pit_rate, NSEC_PER_SEC, pit_clkevt.shift);
		176	clockevents_register_device(&pit_clkevt);
92	}	177	}
93		178
94	#ifdef CONFIG_PM	179	static void at91sam926x_pit_suspend(void)
95	static void at91sam926x_timer_suspend(void)
96	{	180	{
97	/* Disable timer */	181	/* Disable timer */
98	at91_sys_write(AT91_PIT_MR, 0);	182	at91_sys_write(AT91_PIT_MR, 0);
99	}	183	}
100	#else
101	#define at91sam926x_timer_suspend NULL
102	#endif
103		184
104	struct sys_timer at91sam926x_timer = {	185	struct sys_timer at91sam926x_timer = {
105	.init = at91sam926x_timer_init,	186	.init = at91sam926x_pit_init,
106	.offset = at91sam926x_gettimeoffset,	187	.suspend = at91sam926x_pit_suspend,
107	.suspend = at91sam926x_timer_suspend,	188	.resume = at91sam926x_pit_reset,
108	.resume = at91sam926x_timer_reset,
109	};	189	};
110

/******************************************************************************* Intel 10 Gigabit PCI Express Linux driver Copyright(c) 1999 - 2008 Intel Corporation. This program is free software; you can redistribute it and/or modify it under the terms and conditions of the GNU General Public License, version 2, as published by the Free Software Foundation. This program is distributed in the hope it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. The full GNU General Public License is included in this distribution in the file called "COPYING". Contact Information: e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 *******************************************************************************/ #include <linux/types.h> #include <linux/module.h> #include <linux/pci.h> #include <linux/netdevice.h> #include <linux/vmalloc.h> #include <linux/string.h> #include <linux/in.h> #include <linux/ip.h> #include <linux/tcp.h> #include <linux/ipv6.h> #include <net/checksum.h> #include <net/ip6_checksum.h> #include <linux/ethtool.h> #include <linux/if_vlan.h> #include "ixgbe.h" #include "ixgbe_common.h" char ixgbe_driver_name[] = "ixgbe"; static const char ixgbe_driver_string[] = "Intel(R) 10 Gigabit PCI Express Network Driver"; #define DRV_VERSION "1.3.30-k2" const char ixgbe_driver_version[] = DRV_VERSION; static char ixgbe_copyright[] = "Copyright (c) 1999-2007 Intel Corporation."; static const struct ixgbe_info *ixgbe_info_tbl[] = { [board_82598] = &ixgbe_82598_info, }; /* ixgbe_pci_tbl - PCI Device ID Table * * Wildcard entries (PCI_ANY_ID) should come last * Last entry must be all 0s * * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, * Class, Class Mask, private data (not used) } */ static struct pci_device_id ixgbe_pci_tbl[] = { {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_DUAL_PORT), board_82598 }, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AF_SINGLE_PORT), board_82598 }, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598AT), board_82598 }, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_CX4), board_82598 }, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_CX4_DUAL_PORT), board_82598 }, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_DA_DUAL_PORT), board_82598 }, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598_SR_DUAL_PORT_EM), board_82598 }, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_XF_LR), board_82598 }, {PCI_VDEVICE(INTEL, IXGBE_DEV_ID_82598EB_SFP_LOM), board_82598 }, /* required last entry */ {0, } }; MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl); #ifdef CONFIG_IXGBE_DCA static int ixgbe_notify_dca(struct notifier_block *, unsigned long event, void *p); static struct notifier_block dca_notifier = { .notifier_call = ixgbe_notify_dca, .next = NULL, .priority = 0 }; #endif MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver"); MODULE_LICENSE("GPL"); MODULE_VERSION(DRV_VERSION); #define DEFAULT_DEBUG_LEVEL_SHIFT 3 static void ixgbe_release_hw_control(struct ixgbe_adapter *adapter) { u32 ctrl_ext; /* Let firmware take over control of h/w */ ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext & ~IXGBE_CTRL_EXT_DRV_LOAD); } static void ixgbe_get_hw_control(struct ixgbe_adapter *adapter) { u32 ctrl_ext; /* Let firmware know the driver has taken over */ ctrl_ext = IXGBE_READ_REG(&adapter->hw, IXGBE_CTRL_EXT); IXGBE_WRITE_REG(&adapter->hw, IXGBE_CTRL_EXT, ctrl_ext | IXGBE_CTRL_EXT_DRV_LOAD); } static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, u16 int_alloc_entry, u8 msix_vector) { u32 ivar, index; msix_vector |= IXGBE_IVAR_ALLOC_VAL; index = (int_alloc_entry >> 2) & 0x1F; ivar = IXGBE_READ_REG(&adapter->hw, IXGBE_IVAR(index)); ivar &= ~(0xFF << (8 * (int_alloc_entry & 0x3))); ivar |= (msix_vector << (8 * (int_alloc_entry & 0x3))); IXGBE_WRITE_REG(&adapter->hw, IXGBE_IVAR(index), ivar); } static void ixgbe_unmap_and_free_tx_resource(struct ixgbe_adapter *adapter, struct ixgbe_tx_buffer *tx_buffer_info) { if (tx_buffer_info->dma) { pci_unmap_page(adapter->pdev, tx_buffer_info->dma, tx_buffer_info->length, PCI_DMA_TODEVICE); tx_buffer_info->dma = 0; } if (tx_buffer_info->skb) { dev_kfree_skb_any(tx_buffer_info->skb); tx_buffer_info->skb = NULL; } /* tx_buffer_info must be completely set up in the transmit path */ } static inline bool ixgbe_check_tx_hang(struct ixgbe_adapter *adapter, struct ixgbe_ring *tx_ring, unsigned int eop) { struct ixgbe_hw *hw = &adapter->hw; u32 head, tail; /* Detect a transmit hang in hardware, this serializes the * check with the clearing of time_stamp and movement of eop */ head = IXGBE_READ_REG(hw, tx_ring->head); tail = IXGBE_READ_REG(hw, tx_ring->tail); adapter->detect_tx_hung = false; if ((head != tail) && tx_ring->tx_buffer_info[eop].time_stamp && time_after(jiffies, tx_ring->tx_buffer_info[eop].time_stamp + HZ) && !(IXGBE_READ_REG(&adapter->hw, IXGBE_TFCS) & IXGBE_TFCS_TXOFF)) { /* detected Tx unit hang */ union ixgbe_adv_tx_desc *tx_desc; tx_desc = IXGBE_TX_DESC_ADV(*tx_ring, eop); DPRINTK(DRV, ERR, "Detected Tx Unit Hang\n" " Tx Queue <%d>\n" " TDH, TDT <%x>, <%x>\n" " next_to_use <%x>\n" " next_to_clean <%x>\n" "tx_buffer_info[next_to_clean]\n" " time_stamp <%lx>\n" " jiffies <%lx>\n", tx_ring->queue_index, head, tail, tx_ring->next_to_use, eop, tx_ring->tx_buffer_info[eop].time_stamp, jiffies); return true; } return false; } #define IXGBE_MAX_TXD_PWR 14 #define IXGBE_MAX_DATA_PER_TXD (1 << IXGBE_MAX_TXD_PWR) /* Tx Descriptors needed, worst case */ #define TXD_USE_COUNT(S) (((S) >> IXGBE_MAX_TXD_PWR) + \ (((S) & (IXGBE_MAX_DATA_PER_TXD - 1)) ? 1 : 0)) #define DESC_NEEDED (TXD_USE_COUNT(IXGBE_MAX_DATA_PER_TXD) /* skb->data */ + \ MAX_SKB_FRAGS * TXD_USE_COUNT(PAGE_SIZE) + 1) /* for context */ #define GET_TX_HEAD_FROM_RING(ring) (\ *(volatile u32 *) \ ((union ixgbe_adv_tx_desc *)(ring)->desc + (ring)->count)) static void ixgbe_tx_timeout(struct net_device *netdev); /** * ixgbe_clean_tx_irq - Reclaim resources after transmit completes * @adapter: board private structure * @tx_ring: tx ring to clean **/ static bool ixgbe_clean_tx_irq(struct ixgbe_adapter *adapter, struct ixgbe_ring *tx_ring) { union ixgbe_adv_tx_desc *tx_desc; struct ixgbe_tx_buffer *tx_buffer_info; struct net_device *netdev = adapter->netdev; struct sk_buff *skb; unsigned int i; u32 head, oldhead; unsigned int count = 0; unsigned int total_bytes = 0, total_packets = 0; rmb(); head = GET_TX_HEAD_FROM_RING(tx_ring); head = le32_to_cpu(head); i = tx_ring->next_to_clean; while (1) { while (i != head) { tx_desc = IXGBE_TX_DESC_ADV(*tx_ring, i); tx_buffer_info = &tx_ring->tx_buffer_info[i]; skb = tx_buffer_info->skb; if (skb) { unsigned int segs, bytecount; /* gso_segs is currently only valid for tcp */ segs = skb_shinfo(skb)->gso_segs ?: 1; /* multiply data chunks by size of headers */ bytecount = ((segs - 1) * skb_headlen(skb)) + skb->len; total_packets += segs; total_bytes += bytecount; } ixgbe_unmap_and_free_tx_resource(adapter, tx_buffer_info); i++; if (i == tx_ring->count) i = 0; count++; if (count == tx_ring->count) goto done_cleaning; } oldhead = head; rmb(); head = GET_TX_HEAD_FROM_RING(tx_ring); head = le32_to_cpu(head); if (head == oldhead) goto done_cleaning; } /* while (1) */ done_cleaning: tx_ring->next_to_clean = i; #define TX_WAKE_THRESHOLD (DESC_NEEDED * 2) if (unlikely(count && netif_carrier_ok(netdev) && (IXGBE_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) { /* Make sure that anybody stopping the queue after this * sees the new next_to_clean. */ smp_mb(); if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) && !test_bit(__IXGBE_DOWN, &adapter->state)) { netif_wake_subqueue(netdev, tx_ring->queue_index); ++adapter->restart_queue; } } if (adapter->detect_tx_hung) { if (ixgbe_check_tx_hang(adapter, tx_ring, i)) { /* schedule immediate reset if we believe we hung */ DPRINTK(PROBE, INFO, "tx hang %d detected, resetting adapter\n", adapter->tx_timeout_count + 1); ixgbe_tx_timeout(adapter->netdev); } } /* re-arm the interrupt */ if ((total_packets >= tx_ring->work_limit) || (count == tx_ring->count)) IXGBE_WRITE_REG(&adapter->hw, IXGBE_EICS, tx_ring->v_idx); tx_ring->total_bytes += total_bytes; tx_ring->total_packets += total_packets; tx_ring->stats.bytes += total_bytes; tx_ring->stats.packets += total_packets; adapter->net_stats.tx_bytes += total_bytes; adapter->net_stats.tx_packets += total_packets; return (total_packets ? true : false); } #ifdef CONFIG_IXGBE_DCA static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter, struct ixgbe_ring *rx_ring) { u32 rxctrl; int cpu = get_cpu(); int q = rx_ring - adapter->rx_ring; if (rx_ring->cpu != cpu) { rxctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_DCA_RXCTRL(q)); rxctrl &= ~IXGBE_DCA_RXCTRL_CPUID_MASK; rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu); rxctrl |= IXGBE_DCA_RXCTRL_DESC_DCA_EN; rxctrl |= IXGBE_DCA_RXCTRL_HEAD_DCA_EN; IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_RXCTRL(q), rxctrl); rx_ring->cpu = cpu; } put_cpu(); } static void ixgbe_update_tx_dca(struct ixgbe_adapter *adapter, struct ixgbe_ring *tx_ring) { u32 txctrl; int cpu = get_cpu(); int q = tx_ring - adapter->tx_ring; if (tx_ring->cpu != cpu) { txctrl = IXGBE_READ_REG(&adapter->hw, IXGBE_DCA_TXCTRL(q)); txctrl &= ~IXGBE_DCA_TXCTRL_CPUID_MASK; txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu); txctrl |= IXGBE_DCA_TXCTRL_DESC_DCA_EN; IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_TXCTRL(q), txctrl); tx_ring->cpu = cpu; } put_cpu(); } static void ixgbe_setup_dca(struct ixgbe_adapter *adapter) { int i; if (!(adapter->flags & IXGBE_FLAG_DCA_ENABLED)) return; for (i = 0; i < adapter->num_tx_queues; i++) { adapter->tx_ring[i].cpu = -1; ixgbe_update_tx_dca(adapter, &adapter->tx_ring[i]); } for (i = 0; i < adapter->num_rx_queues; i++) { adapter->rx_ring[i].cpu = -1; ixgbe_update_rx_dca(adapter, &adapter->rx_ring[i]); } } static int __ixgbe_notify_dca(struct device *dev, void *data) { struct net_device *netdev = dev_get_drvdata(dev); struct ixgbe_adapter *adapter = netdev_priv(netdev); unsigned long event = *(unsigned long *)data; switch (event) { case DCA_PROVIDER_ADD: /* if we're already enabled, don't do it again */ if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) break; /* Always use CB2 mode, difference is masked * in the CB driver. */ IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 2); if (dca_add_requester(dev) == 0) { adapter->flags |= IXGBE_FLAG_DCA_ENABLED; ixgbe_setup_dca(adapter); break; } /* Fall Through since DCA is disabled. */ case DCA_PROVIDER_REMOVE: if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) { dca_remove_requester(dev); adapter->flags &= ~IXGBE_FLAG_DCA_ENABLED; IXGBE_WRITE_REG(&adapter->hw, IXGBE_DCA_CTRL, 1); } break; } return 0; } #endif /* CONFIG_IXGBE_DCA */ /** * ixgbe_receive_skb - Send a completed packet up the stack * @adapter: board private structure * @skb: packet to send up * @status: hardware indication of status of receive * @rx_ring: rx descriptor ring (for a specific queue) to setup * @rx_desc: rx descriptor **/ static void ixgbe_receive_skb(struct ixgbe_adapter *adapter, struct sk_buff *skb, u8 status, struct ixgbe_ring *ring, union ixgbe_adv_rx_desc *rx_desc) { bool is_vlan = (status & IXGBE_RXD_STAT_VP); u16 tag = le16_to_cpu(rx_desc->wb.upper.vlan); if (adapter->netdev->features & NETIF_F_LRO && skb->ip_summed == CHECKSUM_UNNECESSARY) { if (adapter->vlgrp && is_vlan && (tag != 0)) lro_vlan_hwaccel_receive_skb(&ring->lro_mgr, skb, adapter->vlgrp, tag, rx_desc); else lro_receive_skb(&ring->lro_mgr, skb, rx_desc); ring->lro_used = true; } else { if (!(adapter->flags & IXGBE_FLAG_IN_NETPOLL)) { if (adapter->vlgrp && is_vlan && (tag != 0)) vlan_hwaccel_receive_skb(skb, adapter->vlgrp, tag); else netif_receive_skb(skb); } else { if (adapter->vlgrp && is_vlan && (tag != 0)) vlan_hwaccel_rx(skb, adapter->vlgrp, tag); else netif_rx(skb); } } } /** * ixgbe_rx_checksum - indicate in skb if hw indicated a good cksum * @adapter: address of board private structure * @status_err: hardware indication of status of receive * @skb: skb currently being received and modified **/ static inline void ixgbe_rx_checksum(struct ixgbe_adapter *adapter, u32 status_err, struct sk_buff *skb) { skb->ip_summed = CHECKSUM_NONE; /* Rx csum disabled */ if (!(adapter->flags & IXGBE_FLAG_RX_CSUM_ENABLED)) return; /* if IP and error */ if ((status_err & IXGBE_RXD_STAT_IPCS) && (status_err & IXGBE_RXDADV_ERR_IPE)) { adapter->hw_csum_rx_error++; return; } if (!(status_err & IXGBE_RXD_STAT_L4CS)) return; if (status_err & IXGBE_RXDADV_ERR_TCPE) { adapter->hw_csum_rx_error++; return; } /* It must be a TCP or UDP packet with a valid checksum */ skb->ip_summed = CHECKSUM_UNNECESSARY; adapter->hw_csum_rx_good++; } /** * ixgbe_alloc_rx_buffers - Replace used receive buffers; packet split * @adapter: address of board private structure **/ static void ixgbe_alloc_rx_buffers(struct ixgbe_adapter *adapter, struct ixgbe_ring *rx_ring, int cleaned_count) { struct pci_dev *pdev = adapter->pdev; union ixgbe_adv_rx_desc *rx_desc; struct ixgbe_rx_buffer *bi; unsigned int i; unsigned int bufsz = rx_ring->rx_buf_len + NET_IP_ALIGN; i = rx_ring->next_to_use; bi = &rx_ring->rx_buffer_info[i]; while (cleaned_count--) { rx_desc = IXGBE_RX_DESC_ADV(*rx_ring, i); if (!bi->page_dma && (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED)) { if (!bi->page) { bi->page = alloc_page(GFP_ATOMIC); if (!bi->page) { adapter->alloc_rx_page_failed++; goto no_buffers; } bi->page_offset = 0; } else { /* use a half page if we're re-using */ bi->page_offset ^= (PAGE_SIZE / 2); } bi->page_dma = pci_map_page(pdev, bi->page, bi->page_offset, (PAGE_SIZE / 2), PCI_DMA_FROMDEVICE); } if (!bi->skb) { struct sk_buff *skb = netdev_alloc_skb(adapter->netdev, bufsz); if (!skb) { adapter->alloc_rx_buff_failed++; goto no_buffers; } /* * Make buffer alignment 2 beyond a 16 byte boundary * this will result in a 16 byte aligned IP header after * the 14 byte MAC header is removed */ skb_reserve(skb, NET_IP_ALIGN); bi->skb = skb; bi->dma = pci_map_single(pdev, skb->data, bufsz, PCI_DMA_FROMDEVICE); } /* Refresh the desc even if buffer_addrs didn't change because * each write-back erases this info. */ if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) { rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma); rx_desc->read.hdr_addr = cpu_to_le64(bi->dma); } else { rx_desc->read.pkt_addr = cpu_to_le64(bi->dma); } i++; if (i == rx_ring->count) i = 0; bi = &rx_ring->rx_buffer_info[i]; } no_buffers: if (rx_ring->next_to_use != i) { rx_ring->next_to_use = i; if (i-- == 0) i = (rx_ring->count - 1); /* * Force memory writes to complete before letting h/w * know there are new descriptors to fetch. (Only * applicable for weak-ordered memory model archs, * such as IA-64). */ wmb(); writel(i, adapter->hw.hw_addr + rx_ring->tail); } } static inline u16 ixgbe_get_hdr_info(union ixgbe_adv_rx_desc *rx_desc) { return rx_desc->wb.lower.lo_dword.hs_rss.hdr_info; } static inline u16 ixgbe_get_pkt_info(union ixgbe_adv_rx_desc *rx_desc) { return rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; } static bool ixgbe_clean_rx_irq(struct ixgbe_adapter *adapter, struct ixgbe_ring *rx_ring, int *work_done, int work_to_do) { struct pci_dev *pdev = adapter->pdev; union ixgbe_adv_rx_desc *rx_desc, *next_rxd; struct ixgbe_rx_buffer *rx_buffer_info, *next_buffer; struct sk_buff *skb; unsigned int i; u32 len, staterr; u16 hdr_info; bool cleaned = false; int cleaned_count = 0; unsigned int total_rx_bytes = 0, total_rx_packets = 0; i = rx_ring->next_to_clean; rx_desc = IXGBE_RX_DESC_ADV(*rx_ring, i); staterr = le32_to_cpu(rx_desc->wb.upper.status_error); rx_buffer_info = &rx_ring->rx_buffer_info[i]; while (staterr & IXGBE_RXD_STAT_DD) { u32 upper_len = 0; if (*work_done >= work_to_do) break; (*work_done)++; if (adapter->flags & IXGBE_FLAG_RX_PS_ENABLED) { hdr_info = le16_to_cpu(ixgbe_get_hdr_info(rx_desc)); len = (hdr_info & IXGBE_RXDADV_HDRBUFLEN_MASK) >> IXGBE_RXDADV_HDRBUFLEN_SHIFT; if (hdr_info & IXGBE_RXDADV_SPH) adapter->rx_hdr_split++; if (len > IXGBE_RX_HDR_SIZE) len = IXGBE_RX_HDR_SIZE; upper_len = le16_to_cpu(rx_desc->wb.upper.length); } else { len = le16_to_cpu(rx_desc->wb.upper.length); } cleaned = true; skb = rx_buffer_info->skb; prefetch(skb->data - NET_IP_ALIGN); rx_buffer_info->skb = NULL; if (len && !skb_shinfo(skb)->nr_frags) { pci_unmap_single(pdev, rx_buffer_info->dma, rx_ring->rx_buf_len + NET_IP_ALIGN, PCI_DMA_FROMDEVICE); skb_put(skb, len); } if (upper_len) { pci_unmap_page(pdev, rx_buffer_info->page_dma, PAGE_SIZE / 2, PCI_DMA_FROMDEVICE); rx_buffer_info->page_dma = 0; skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, rx_buffer_info->page, rx_buffer_info->page_offset, upper_len); if ((rx_ring->rx_buf_len > (PAGE_SIZE / 2)) || (page_count(rx_buffer_info->page) != 1)) rx_buffer_info->page = NULL; else get_page(rx_buffer_info->page); skb->len += upper_len; skb->data_len += upper_len; skb->truesize += upper_len; } i++; if (i == rx_ring->count) i = 0; next_buffer = &rx_ring->rx_buffer_info[i]; next_rxd = IXGBE_RX_DESC_ADV(*rx_ring, i); prefetch(next_rxd); cleaned_count++; if (staterr & IXGBE_RXD_STAT_EOP) { rx_ring->stats.packets++; rx_ring->stats.bytes += skb->len; } else { rx_buffer_info->skb = next_buffer->skb; rx_buffer_info->dma = next_buffer->dma; next_buffer->skb = skb; next_buffer->dma = 0; adapter->non_eop_descs++; goto next_desc; } if (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) { dev_kfree_skb_irq(skb); goto next_desc; } ixgbe_rx_checksum(adapter, staterr, skb); /* probably a little skewed due to removing CRC */ total_rx_bytes += skb->len; total_rx_packets++; skb->protocol = eth_type_trans(skb, adapter->netdev); ixgbe_receive_skb(adapter, skb, staterr, rx_ring, rx_desc); next_desc: rx_desc->wb.upper.status_error = 0; /* return some buffers to hardware, one at a time is too slow */ if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) { ixgbe_alloc_rx_buffers(adapter, rx_ring, cleaned_count); cleaned_count = 0; } /* use prefetched values */ rx_desc = next_rxd; rx_buffer_info = next_buffer; staterr = le32_to_cpu(rx_desc->wb.upper.status_error); } if (rx_ring->lro_used) { lro_flush_all(&rx_ring->lro_mgr); rx_ring->lro_used = false; } rx_ring->next_to_clean = i; cleaned_count = IXGBE_DESC_UNUSED(rx_ring); if (cleaned_count) ixgbe_alloc_rx_buffers(adapter, rx_ring, cleaned_count); rx_ring->total_packets += total_rx_packets; rx_ring->total_bytes += total_rx_bytes; adapter->net_stats.rx_bytes += total_rx_bytes; adapter->net_stats.rx_packets += total_rx_packets; return cleaned; } static int ixgbe_clean_rxonly(struct napi_struct *, int); /** * ixgbe_configure_msix - Configure MSI-X hardware * @adapter: board private structure * * ixgbe_configure_msix sets up the hardware to properly generate MSI-X * interrupts. **/ static void ixgbe_configure_msix(struct ixgbe_adapter *adapter) { struct ixgbe_q_vector *q_vector; int i, j, q_vectors, v_idx, r_idx; u32 mask; q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; /* Populate the IVAR table and set the ITR values to the * corresponding register. */ for (v_idx = 0; v_idx < q_vectors; v_idx++) { q_vector = &adapter->q_vector[v_idx]; /* XXX for_each_bit(...) */ r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); for (i = 0; i < q_vector->rxr_count; i++) { j = adapter->rx_ring[r_idx].reg_idx; ixgbe_set_ivar(adapter, IXGBE_IVAR_RX_QUEUE(j), v_idx); r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues, r_idx + 1); } r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues); for (i = 0; i < q_vector->txr_count; i++) { j = adapter->tx_ring[r_idx].reg_idx; ixgbe_set_ivar(adapter, IXGBE_IVAR_TX_QUEUE(j), v_idx); r_idx = find_next_bit(q_vector->txr_idx, adapter->num_tx_queues, r_idx + 1); } /* if this is a tx only vector halve the interrupt rate */ if (q_vector->txr_count && !q_vector->rxr_count) q_vector->eitr = (adapter->eitr_param >> 1); else /* rx only */ q_vector->eitr = adapter->eitr_param; IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), EITR_INTS_PER_SEC_TO_REG(q_vector->eitr)); } ixgbe_set_ivar(adapter, IXGBE_IVAR_OTHER_CAUSES_INDEX, v_idx); IXGBE_WRITE_REG(&adapter->hw, IXGBE_EITR(v_idx), 1950); /* set up to autoclear timer, and the vectors */ mask = IXGBE_EIMS_ENABLE_MASK; mask &= ~(IXGBE_EIMS_OTHER | IXGBE_EIMS_LSC); IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIAC, mask); } enum latency_range { lowest_latency = 0, low_latency = 1, bulk_latency = 2, latency_invalid = 255 }; /** * ixgbe_update_itr - update the dynamic ITR value based on statistics * @adapter: pointer to adapter * @eitr: eitr setting (ints per sec) to give last timeslice * @itr_setting: current throttle rate in ints/second * @packets: the number of packets during this measurement interval * @bytes: the number of bytes during this measurement interval * * Stores a new ITR value based on packets and byte * counts during the last interrupt. The advantage of per interrupt * computation is faster updates and more accurate ITR for the current * traffic pattern. Constants in this function were computed * based on theoretical maximum wire speed and thresholds were set based * on testing data as well as attempting to minimize response time * while increasing bulk throughput. * this functionality is controlled by the InterruptThrottleRate module * parameter (see ixgbe_param.c) **/ static u8 ixgbe_update_itr(struct ixgbe_adapter *adapter, u32 eitr, u8 itr_setting, int packets, int bytes) { unsigned int retval = itr_setting; u32 timepassed_us; u64 bytes_perint; if (packets == 0) goto update_itr_done; /* simple throttlerate management * 0-20MB/s lowest (100000 ints/s) * 20-100MB/s low (20000 ints/s) * 100-1249MB/s bulk (8000 ints/s) */ /* what was last interrupt timeslice? */ timepassed_us = 1000000/eitr; bytes_perint = bytes / timepassed_us; /* bytes/usec */ switch (itr_setting) { case lowest_latency: if (bytes_perint > adapter->eitr_low) retval = low_latency; break; case low_latency: if (bytes_perint > adapter->eitr_high) retval = bulk_latency; else if (bytes_perint <= adapter->eitr_low) retval = lowest_latency; break; case bulk_latency: if (bytes_perint <= adapter->eitr_high) retval = low_latency; break; } update_itr_done: return retval; } static void ixgbe_set_itr_msix(struct ixgbe_q_vector *q_vector) { struct ixgbe_adapter *adapter = q_vector->adapter; struct ixgbe_hw *hw = &adapter->hw; u32 new_itr; u8 current_itr, ret_itr; int i, r_idx, v_idx = ((void *)q_vector - (void *)(adapter->q_vector)) / sizeof(struct ixgbe_q_vector); struct ixgbe_ring *rx_ring, *tx_ring; r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues); for (i = 0; i < q_vector->txr_count; i++) { tx_ring = &(adapter->tx_ring[r_idx]); ret_itr = ixgbe_update_itr(adapter, q_vector->eitr, q_vector->tx_itr, tx_ring->total_packets, tx_ring->total_bytes); /* if the result for this queue would decrease interrupt * rate for this vector then use that result */ q_vector->tx_itr = ((q_vector->tx_itr > ret_itr) ? q_vector->tx_itr - 1 : ret_itr); r_idx = find_next_bit(q_vector->txr_idx, adapter->num_tx_queues, r_idx + 1); } r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); for (i = 0; i < q_vector->rxr_count; i++) { rx_ring = &(adapter->rx_ring[r_idx]); ret_itr = ixgbe_update_itr(adapter, q_vector->eitr, q_vector->rx_itr, rx_ring->total_packets, rx_ring->total_bytes); /* if the result for this queue would decrease interrupt * rate for this vector then use that result */ q_vector->rx_itr = ((q_vector->rx_itr > ret_itr) ? q_vector->rx_itr - 1 : ret_itr); r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues, r_idx + 1); } current_itr = max(q_vector->rx_itr, q_vector->tx_itr); switch (current_itr) { /* counts and packets in update_itr are dependent on these numbers */ case lowest_latency: new_itr = 100000; break; case low_latency: new_itr = 20000; /* aka hwitr = ~200 */ break; case bulk_latency: default: new_itr = 8000; break; } if (new_itr != q_vector->eitr) { u32 itr_reg; /* do an exponential smoothing */ new_itr = ((q_vector->eitr * 90)/100) + ((new_itr * 10)/100); q_vector->eitr = new_itr; itr_reg = EITR_INTS_PER_SEC_TO_REG(new_itr); /* must write high and low 16 bits to reset counter */ DPRINTK(TX_ERR, DEBUG, "writing eitr(%d): %08X\n", v_idx, itr_reg); IXGBE_WRITE_REG(hw, IXGBE_EITR(v_idx), itr_reg | (itr_reg)<<16); } return; } static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr) { struct ixgbe_hw *hw = &adapter->hw; if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) && (eicr & IXGBE_EICR_GPI_SDP1)) { DPRINTK(PROBE, CRIT, "Fan has stopped, replace the adapter\n"); /* write to clear the interrupt */ IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); } } static void ixgbe_check_lsc(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; adapter->lsc_int++; adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; adapter->link_check_timeout = jiffies; if (!test_bit(__IXGBE_DOWN, &adapter->state)) { IXGBE_WRITE_REG(hw, IXGBE_EIMC, IXGBE_EIMC_LSC); schedule_work(&adapter->watchdog_task); } } static irqreturn_t ixgbe_msix_lsc(int irq, void *data) { struct net_device *netdev = data; struct ixgbe_adapter *adapter = netdev_priv(netdev); struct ixgbe_hw *hw = &adapter->hw; u32 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); if (eicr & IXGBE_EICR_LSC) ixgbe_check_lsc(adapter); ixgbe_check_fan_failure(adapter, eicr); if (!test_bit(__IXGBE_DOWN, &adapter->state)) IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_OTHER); return IRQ_HANDLED; } static irqreturn_t ixgbe_msix_clean_tx(int irq, void *data) { struct ixgbe_q_vector *q_vector = data; struct ixgbe_adapter *adapter = q_vector->adapter; struct ixgbe_ring *tx_ring; int i, r_idx; if (!q_vector->txr_count) return IRQ_HANDLED; r_idx = find_first_bit(q_vector->txr_idx, adapter->num_tx_queues); for (i = 0; i < q_vector->txr_count; i++) { tx_ring = &(adapter->tx_ring[r_idx]); #ifdef CONFIG_IXGBE_DCA if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) ixgbe_update_tx_dca(adapter, tx_ring); #endif tx_ring->total_bytes = 0; tx_ring->total_packets = 0; ixgbe_clean_tx_irq(adapter, tx_ring); r_idx = find_next_bit(q_vector->txr_idx, adapter->num_tx_queues, r_idx + 1); } return IRQ_HANDLED; } /** * ixgbe_msix_clean_rx - single unshared vector rx clean (all queues) * @irq: unused * @data: pointer to our q_vector struct for this interrupt vector **/ static irqreturn_t ixgbe_msix_clean_rx(int irq, void *data) { struct ixgbe_q_vector *q_vector = data; struct ixgbe_adapter *adapter = q_vector->adapter; struct ixgbe_ring *rx_ring; int r_idx; int i; r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); for (i = 0; i < q_vector->rxr_count; i++) { rx_ring = &(adapter->rx_ring[r_idx]); rx_ring->total_bytes = 0; rx_ring->total_packets = 0; r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues, r_idx + 1); } if (!q_vector->rxr_count) return IRQ_HANDLED; r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); rx_ring = &(adapter->rx_ring[r_idx]); /* disable interrupts on this vector only */ IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMC, rx_ring->v_idx); netif_rx_schedule(adapter->netdev, &q_vector->napi); return IRQ_HANDLED; } static irqreturn_t ixgbe_msix_clean_many(int irq, void *data) { ixgbe_msix_clean_rx(irq, data); ixgbe_msix_clean_tx(irq, data); return IRQ_HANDLED; } /** * ixgbe_clean_rxonly - msix (aka one shot) rx clean routine * @napi: napi struct with our devices info in it * @budget: amount of work driver is allowed to do this pass, in packets * * This function is optimized for cleaning one queue only on a single * q_vector!!! **/ static int ixgbe_clean_rxonly(struct napi_struct *napi, int budget) { struct ixgbe_q_vector *q_vector = container_of(napi, struct ixgbe_q_vector, napi); struct ixgbe_adapter *adapter = q_vector->adapter; struct ixgbe_ring *rx_ring = NULL; int work_done = 0; long r_idx; r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); rx_ring = &(adapter->rx_ring[r_idx]); #ifdef CONFIG_IXGBE_DCA if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) ixgbe_update_rx_dca(adapter, rx_ring); #endif ixgbe_clean_rx_irq(adapter, rx_ring, &work_done, budget); /* If all Rx work done, exit the polling mode */ if (work_done < budget) { netif_rx_complete(adapter->netdev, napi); if (adapter->itr_setting & 3) ixgbe_set_itr_msix(q_vector); if (!test_bit(__IXGBE_DOWN, &adapter->state)) IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, rx_ring->v_idx); } return work_done; } /** * ixgbe_clean_rxonly_many - msix (aka one shot) rx clean routine * @napi: napi struct with our devices info in it * @budget: amount of work driver is allowed to do this pass, in packets * * This function will clean more than one rx queue associated with a * q_vector. **/ static int ixgbe_clean_rxonly_many(struct napi_struct *napi, int budget) { struct ixgbe_q_vector *q_vector = container_of(napi, struct ixgbe_q_vector, napi); struct ixgbe_adapter *adapter = q_vector->adapter; struct ixgbe_ring *rx_ring = NULL; int work_done = 0, i; long r_idx; u16 enable_mask = 0; /* attempt to distribute budget to each queue fairly, but don't allow * the budget to go below 1 because we'll exit polling */ budget /= (q_vector->rxr_count ?: 1); budget = max(budget, 1); r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); for (i = 0; i < q_vector->rxr_count; i++) { rx_ring = &(adapter->rx_ring[r_idx]); #ifdef CONFIG_IXGBE_DCA if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) ixgbe_update_rx_dca(adapter, rx_ring); #endif ixgbe_clean_rx_irq(adapter, rx_ring, &work_done, budget); enable_mask |= rx_ring->v_idx; r_idx = find_next_bit(q_vector->rxr_idx, adapter->num_rx_queues, r_idx + 1); } r_idx = find_first_bit(q_vector->rxr_idx, adapter->num_rx_queues); rx_ring = &(adapter->rx_ring[r_idx]); /* If all Rx work done, exit the polling mode */ if (work_done < budget) { netif_rx_complete(adapter->netdev, napi); if (adapter->itr_setting & 3) ixgbe_set_itr_msix(q_vector); if (!test_bit(__IXGBE_DOWN, &adapter->state)) IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, enable_mask); return 0; } return work_done; } static inline void map_vector_to_rxq(struct ixgbe_adapter *a, int v_idx, int r_idx) { a->q_vector[v_idx].adapter = a; set_bit(r_idx, a->q_vector[v_idx].rxr_idx); a->q_vector[v_idx].rxr_count++; a->rx_ring[r_idx].v_idx = 1 << v_idx; } static inline void map_vector_to_txq(struct ixgbe_adapter *a, int v_idx, int r_idx) { a->q_vector[v_idx].adapter = a; set_bit(r_idx, a->q_vector[v_idx].txr_idx); a->q_vector[v_idx].txr_count++; a->tx_ring[r_idx].v_idx = 1 << v_idx; } /** * ixgbe_map_rings_to_vectors - Maps descriptor rings to vectors * @adapter: board private structure to initialize * @vectors: allotted vector count for descriptor rings * * This function maps descriptor rings to the queue-specific vectors * we were allotted through the MSI-X enabling code. Ideally, we'd have * one vector per ring/queue, but on a constrained vector budget, we * group the rings as "efficiently" as possible. You would add new * mapping configurations in here. **/ static int ixgbe_map_rings_to_vectors(struct ixgbe_adapter *adapter, int vectors) { int v_start = 0; int rxr_idx = 0, txr_idx = 0; int rxr_remaining = adapter->num_rx_queues; int txr_remaining = adapter->num_tx_queues; int i, j; int rqpv, tqpv; int err = 0; /* No mapping required if MSI-X is disabled. */ if (!(adapter->flags & IXGBE_FLAG_MSIX_ENABLED)) goto out; /* * The ideal configuration... * We have enough vectors to map one per queue. */ if (vectors == adapter->num_rx_queues + adapter->num_tx_queues) { for (; rxr_idx < rxr_remaining; v_start++, rxr_idx++) map_vector_to_rxq(adapter, v_start, rxr_idx); for (; txr_idx < txr_remaining; v_start++, txr_idx++) map_vector_to_txq(adapter, v_start, txr_idx); goto out; } /* * If we don't have enough vectors for a 1-to-1 * mapping, we'll have to group them so there are * multiple queues per vector. */ /* Re-adjusting *qpv takes care of the remainder. */ for (i = v_start; i < vectors; i++) { rqpv = DIV_ROUND_UP(rxr_remaining, vectors - i); for (j = 0; j < rqpv; j++) { map_vector_to_rxq(adapter, i, rxr_idx); rxr_idx++; rxr_remaining--; } } for (i = v_start; i < vectors; i++) { tqpv = DIV_ROUND_UP(txr_remaining, vectors - i); for (j = 0; j < tqpv; j++) { map_vector_to_txq(adapter, i, txr_idx); txr_idx++; txr_remaining--; } } out: return err; } /** * ixgbe_request_msix_irqs - Initialize MSI-X interrupts * @adapter: board private structure * * ixgbe_request_msix_irqs allocates MSI-X vectors and requests * interrupts from the kernel. **/ static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter) { struct net_device *netdev = adapter->netdev; irqreturn_t (*handler)(int, void *); int i, vector, q_vectors, err; int ri=0, ti=0; /* Decrement for Other and TCP Timer vectors */ q_vectors = adapter->num_msix_vectors - NON_Q_VECTORS; /* Map the Tx/Rx rings to the vectors we were allotted. */ err = ixgbe_map_rings_to_vectors(adapter, q_vectors); if (err) goto out; #define SET_HANDLER(_v) ((!(_v)->rxr_count) ? &ixgbe_msix_clean_tx : \ (!(_v)->txr_count) ? &ixgbe_msix_clean_rx : \ &ixgbe_msix_clean_many) for (vector = 0; vector < q_vectors; vector++) { handler = SET_HANDLER(&adapter->q_vector[vector]); if(handler == &ixgbe_msix_clean_rx) { sprintf(adapter->name[vector], "%s-%s-%d", netdev->name, "rx", ri++); } else if(handler == &ixgbe_msix_clean_tx) { sprintf(adapter->name[vector], "%s-%s-%d", netdev->name, "tx", ti++); } else sprintf(adapter->name[vector], "%s-%s-%d", netdev->name, "TxRx", vector); err = request_irq(adapter->msix_entries[vector].vector, handler, 0, adapter->name[vector], &(adapter->q_vector[vector])); if (err) { DPRINTK(PROBE, ERR, "request_irq failed for MSIX interrupt " "Error: %d\n", err); goto free_queue_irqs; } } sprintf(adapter->name[vector], "%s:lsc", netdev->name); err = request_irq(adapter->msix_entries[vector].vector, &ixgbe_msix_lsc, 0, adapter->name[vector], netdev); if (err) { DPRINTK(PROBE, ERR, "request_irq for msix_lsc failed: %d\n", err); goto free_queue_irqs; } return 0; free_queue_irqs: for (i = vector - 1; i >= 0; i--) free_irq(adapter->msix_entries[--vector].vector, &(adapter->q_vector[i])); adapter->flags &= ~IXGBE_FLAG_MSIX_ENABLED; pci_disable_msix(adapter->pdev); kfree(adapter->msix_entries); adapter->msix_entries = NULL; out: return err; } static void ixgbe_set_itr(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; struct ixgbe_q_vector *q_vector = adapter->q_vector; u8 current_itr; u32 new_itr = q_vector->eitr;


context:
space:
mode: