aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorShannon Nelson <shannon.nelson@intel.com>2007-10-16 04:27:42 -0400
committerLinus Torvalds <torvalds@woody.linux-foundation.org>2007-10-16 12:43:09 -0400
commit2ed6dc34f9ed39bb8e4c81ea1056f0ba56315841 (patch)
treee3f6ca7961f9c4e34453d06e584c0bc98ec630d7
parent7589670f37736bcc119ebfbd69aafea6d585d1d4 (diff)
I/OAT: Add DCA services
Add code to connect to the DCA driver and provide cpu tags for use by drivers that would like to use Direct Cache Access hints. [Adrian Bunk] Several Kconfig cleanup items [Andrew Morten, Chris Leech] Fix for using cpu_physical_id() even when built for uni-processor Signed-off-by: Shannon Nelson <shannon.nelson@intel.com> Acked-by: David S. Miller <davem@davemloft.net> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
-rw-r--r--drivers/dca/Kconfig6
-rw-r--r--drivers/dma/Kconfig60
-rw-r--r--drivers/dma/Makefile2
-rw-r--r--drivers/dma/ioat.c17
-rw-r--r--drivers/dma/ioat_dca.c263
-rw-r--r--drivers/dma/ioatdma.h3
-rw-r--r--include/asm-x86/cpufeature_32.h1
7 files changed, 320 insertions, 32 deletions
diff --git a/drivers/dca/Kconfig b/drivers/dca/Kconfig
index e2f7436da05..94f0364a0ef 100644
--- a/drivers/dca/Kconfig
+++ b/drivers/dca/Kconfig
@@ -3,9 +3,5 @@
3# 3#
4 4
5config DCA 5config DCA
6 tristate "DCA support for clients and providers" 6 tristate
7 default m
8 help
9 This is a server to help modules that want to use Direct Cache
10 Access to find DCA providers that will supply correct CPU tags.
11 7
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index 8f670dae53b..9c91b0fd134 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -2,42 +2,52 @@
2# DMA engine configuration 2# DMA engine configuration
3# 3#
4 4
5menu "DMA Engine support" 5menuconfig DMADEVICES
6 depends on HAS_DMA 6 bool "DMA Offload Engine support"
7 depends on (PCI && X86) || ARCH_IOP32X || ARCH_IOP33X || ARCH_IOP13XX
8 help
9 Intel(R) offload engines enable offloading memory copies in the
10 network stack and RAID operations in the MD driver.
11
12if DMADEVICES
13
14comment "DMA Devices"
15
16config INTEL_IOATDMA
17 tristate "Intel I/OAT DMA support"
18 depends on PCI && X86
19 select DMA_ENGINE
20 select DCA
21 help
22 Enable support for the Intel(R) I/OAT DMA engine present
23 in recent Intel Xeon chipsets.
24
25 Say Y here if you have such a chipset.
26
27 If unsure, say N.
28
29config INTEL_IOP_ADMA
30 tristate "Intel IOP ADMA support"
31 depends on ARCH_IOP32X || ARCH_IOP33X || ARCH_IOP13XX
32 select ASYNC_CORE
33 select DMA_ENGINE
34 help
35 Enable support for the Intel(R) IOP Series RAID engines.
7 36
8config DMA_ENGINE 37config DMA_ENGINE
9 bool "Support for DMA engines" 38 bool
10 ---help---
11 DMA engines offload bulk memory operations from the CPU to dedicated
12 hardware, allowing the operations to happen asynchronously.
13 39
14comment "DMA Clients" 40comment "DMA Clients"
41 depends on DMA_ENGINE
15 42
16config NET_DMA 43config NET_DMA
17 bool "Network: TCP receive copy offload" 44 bool "Network: TCP receive copy offload"
18 depends on DMA_ENGINE && NET 45 depends on DMA_ENGINE && NET
19 default y 46 default y
20 ---help--- 47 help
21 This enables the use of DMA engines in the network stack to 48 This enables the use of DMA engines in the network stack to
22 offload receive copy-to-user operations, freeing CPU cycles. 49 offload receive copy-to-user operations, freeing CPU cycles.
23 Since this is the main user of the DMA engine, it should be enabled; 50 Since this is the main user of the DMA engine, it should be enabled;
24 say Y here. 51 say Y here.
25 52
26comment "DMA Devices" 53endif
27
28config INTEL_IOATDMA
29 tristate "Intel I/OAT DMA support"
30 depends on DMA_ENGINE && PCI
31 default m
32 ---help---
33 Enable support for the Intel(R) I/OAT DMA engine.
34
35config INTEL_IOP_ADMA
36 tristate "Intel IOP ADMA support"
37 depends on DMA_ENGINE && (ARCH_IOP32X || ARCH_IOP33X || ARCH_IOP13XX)
38 select ASYNC_CORE
39 default m
40 ---help---
41 Enable support for the Intel(R) IOP Series RAID engines.
42
43endmenu
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
index cec0c9c5df6..b152cd84e12 100644
--- a/drivers/dma/Makefile
+++ b/drivers/dma/Makefile
@@ -1,5 +1,5 @@
1obj-$(CONFIG_DMA_ENGINE) += dmaengine.o 1obj-$(CONFIG_DMA_ENGINE) += dmaengine.o
2obj-$(CONFIG_NET_DMA) += iovlock.o 2obj-$(CONFIG_NET_DMA) += iovlock.o
3obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o 3obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o
4ioatdma-objs := ioat.o ioat_dma.o 4ioatdma-objs := ioat.o ioat_dma.o ioat_dca.o
5obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o 5obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o
diff --git a/drivers/dma/ioat.c b/drivers/dma/ioat.c
index ae5817bfc01..f7276bf2fe7 100644
--- a/drivers/dma/ioat.c
+++ b/drivers/dma/ioat.c
@@ -1,6 +1,6 @@
1/* 1/*
2 * Intel I/OAT DMA Linux driver 2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2004 - 2007 Intel Corporation. 3 * Copyright(c) 2007 Intel Corporation.
4 * 4 *
5 * This program is free software; you can redistribute it and/or modify it 5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License, 6 * under the terms and conditions of the GNU General Public License,
@@ -29,6 +29,7 @@
29#include <linux/module.h> 29#include <linux/module.h>
30#include <linux/pci.h> 30#include <linux/pci.h>
31#include <linux/interrupt.h> 31#include <linux/interrupt.h>
32#include <linux/dca.h>
32#include "ioatdma.h" 33#include "ioatdma.h"
33#include "ioatdma_registers.h" 34#include "ioatdma_registers.h"
34#include "ioatdma_hw.h" 35#include "ioatdma_hw.h"
@@ -49,6 +50,7 @@ struct ioat_device {
49 struct pci_dev *pdev; 50 struct pci_dev *pdev;
50 void __iomem *iobase; 51 void __iomem *iobase;
51 struct ioatdma_device *dma; 52 struct ioatdma_device *dma;
53 struct dca_provider *dca;
52}; 54};
53 55
54static int __devinit ioat_probe(struct pci_dev *pdev, 56static int __devinit ioat_probe(struct pci_dev *pdev,
@@ -57,6 +59,10 @@ static int __devinit ioat_probe(struct pci_dev *pdev,
57static void __devexit ioat_remove(struct pci_dev *pdev); 59static void __devexit ioat_remove(struct pci_dev *pdev);
58#endif 60#endif
59 61
62static int ioat_dca_enabled = 1;
63module_param(ioat_dca_enabled, int, 0644);
64MODULE_PARM_DESC(ioat_dca_enabled, "control support of dca service (default: 1)");
65
60static int ioat_setup_functionality(struct pci_dev *pdev, void __iomem *iobase) 66static int ioat_setup_functionality(struct pci_dev *pdev, void __iomem *iobase)
61{ 67{
62 struct ioat_device *device = pci_get_drvdata(pdev); 68 struct ioat_device *device = pci_get_drvdata(pdev);
@@ -67,6 +73,8 @@ static int ioat_setup_functionality(struct pci_dev *pdev, void __iomem *iobase)
67 switch (version) { 73 switch (version) {
68 case IOAT_VER_1_2: 74 case IOAT_VER_1_2:
69 device->dma = ioat_dma_probe(pdev, iobase); 75 device->dma = ioat_dma_probe(pdev, iobase);
76 if (ioat_dca_enabled)
77 device->dca = ioat_dca_init(pdev, iobase);
70 break; 78 break;
71 default: 79 default:
72 err = -ENODEV; 80 err = -ENODEV;
@@ -83,6 +91,13 @@ static void ioat_shutdown_functionality(struct pci_dev *pdev)
83 ioat_dma_remove(device->dma); 91 ioat_dma_remove(device->dma);
84 device->dma = NULL; 92 device->dma = NULL;
85 } 93 }
94
95 if (device->dca) {
96 unregister_dca_provider(device->dca);
97 free_dca_provider(device->dca);
98 device->dca = NULL;
99 }
100
86} 101}
87 102
88static struct pci_driver ioat_pci_drv = { 103static struct pci_driver ioat_pci_drv = {
diff --git a/drivers/dma/ioat_dca.c b/drivers/dma/ioat_dca.c
new file mode 100644
index 00000000000..2ae04c30ede
--- /dev/null
+++ b/drivers/dma/ioat_dca.c
@@ -0,0 +1,263 @@
1/*
2 * Intel I/OAT DMA Linux driver
3 * Copyright(c) 2007 Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc.,
16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 * The full GNU General Public License is included in this distribution in
19 * the file called "COPYING".
20 *
21 */
22
23#include <linux/kernel.h>
24#include <linux/pci.h>
25#include <linux/smp.h>
26#include <linux/interrupt.h>
27#include <linux/dca.h>
28
29/* either a kernel change is needed, or we need something like this in kernel */
30#ifndef CONFIG_SMP
31#include <asm/smp.h>
32#undef cpu_physical_id
33#define cpu_physical_id(cpu) (cpuid_ebx(1) >> 24)
34#endif
35
36#include "ioatdma.h"
37#include "ioatdma_registers.h"
38
39/*
40 * Bit 16 of a tag map entry is the "valid" bit, if it is set then bits 0:15
41 * contain the bit number of the APIC ID to map into the DCA tag. If the valid
42 * bit is not set, then the value must be 0 or 1 and defines the bit in the tag.
43 */
44#define DCA_TAG_MAP_VALID 0x80
45
46/*
47 * "Legacy" DCA systems do not implement the DCA register set in the
48 * I/OAT device. Software needs direct support for their tag mappings.
49 */
50
51#define APICID_BIT(x) (DCA_TAG_MAP_VALID | (x))
52#define IOAT_TAG_MAP_LEN 8
53
54static u8 ioat_tag_map_BNB[IOAT_TAG_MAP_LEN] = {
55 1, APICID_BIT(1), APICID_BIT(2), APICID_BIT(2), };
56static u8 ioat_tag_map_SCNB[IOAT_TAG_MAP_LEN] = {
57 1, APICID_BIT(1), APICID_BIT(2), APICID_BIT(2), };
58static u8 ioat_tag_map_CNB[IOAT_TAG_MAP_LEN] = {
59 1, APICID_BIT(1), APICID_BIT(3), APICID_BIT(4), APICID_BIT(2), };
60static u8 ioat_tag_map_UNISYS[IOAT_TAG_MAP_LEN] = { 0 };
61
62/* pack PCI B/D/F into a u16 */
63static inline u16 dcaid_from_pcidev(struct pci_dev *pci)
64{
65 return (pci->bus->number << 8) | pci->devfn;
66}
67
68static int dca_enabled_in_bios(void)
69{
70 /* CPUID level 9 returns DCA configuration */
71 /* Bit 0 indicates DCA enabled by the BIOS */
72 unsigned long cpuid_level_9;
73 int res;
74
75 cpuid_level_9 = cpuid_eax(9);
76 res = test_bit(0, &cpuid_level_9);
77 if (!res)
78 printk(KERN_ERR "ioat dma: DCA is disabled in BIOS\n");
79
80 return res;
81}
82
83static int system_has_dca_enabled(void)
84{
85 if (boot_cpu_has(X86_FEATURE_DCA))
86 return dca_enabled_in_bios();
87
88 printk(KERN_ERR "ioat dma: boot cpu doesn't have X86_FEATURE_DCA\n");
89 return 0;
90}
91
92struct ioat_dca_slot {
93 struct pci_dev *pdev; /* requester device */
94 u16 rid; /* requester id, as used by IOAT */
95};
96
97#define IOAT_DCA_MAX_REQ 6
98
99struct ioat_dca_priv {
100 void __iomem *iobase;
101 void *dca_base;
102 int max_requesters;
103 int requester_count;
104 u8 tag_map[IOAT_TAG_MAP_LEN];
105 struct ioat_dca_slot req_slots[0];
106};
107
108/* 5000 series chipset DCA Port Requester ID Table Entry Format
109 * [15:8] PCI-Express Bus Number
110 * [7:3] PCI-Express Device Number
111 * [2:0] PCI-Express Function Number
112 *
113 * 5000 series chipset DCA control register format
114 * [7:1] Reserved (0)
115 * [0] Ignore Function Number
116 */
117
118static int ioat_dca_add_requester(struct dca_provider *dca, struct device *dev)
119{
120 struct ioat_dca_priv *ioatdca = dca_priv(dca);
121 struct pci_dev *pdev;
122 int i;
123 u16 id;
124
125 /* This implementation only supports PCI-Express */
126 if (dev->bus != &pci_bus_type)
127 return -ENODEV;
128 pdev = to_pci_dev(dev);
129 id = dcaid_from_pcidev(pdev);
130
131 if (ioatdca->requester_count == ioatdca->max_requesters)
132 return -ENODEV;
133
134 for (i = 0; i < ioatdca->max_requesters; i++) {
135 if (ioatdca->req_slots[i].pdev == NULL) {
136 /* found an empty slot */
137 ioatdca->requester_count++;
138 ioatdca->req_slots[i].pdev = pdev;
139 ioatdca->req_slots[i].rid = id;
140 writew(id, ioatdca->dca_base + (i * 4));
141 /* make sure the ignore function bit is off */
142 writeb(0, ioatdca->dca_base + (i * 4) + 2);
143 return i;
144 }
145 }
146 /* Error, ioatdma->requester_count is out of whack */
147 return -EFAULT;
148}
149
150static int ioat_dca_remove_requester(struct dca_provider *dca,
151 struct device *dev)
152{
153 struct ioat_dca_priv *ioatdca = dca_priv(dca);
154 struct pci_dev *pdev;
155 int i;
156
157 /* This implementation only supports PCI-Express */
158 if (dev->bus != &pci_bus_type)
159 return -ENODEV;
160 pdev = to_pci_dev(dev);
161
162 for (i = 0; i < ioatdca->max_requesters; i++) {
163 if (ioatdca->req_slots[i].pdev == pdev) {
164 writew(0, ioatdca->dca_base + (i * 4));
165 ioatdca->req_slots[i].pdev = NULL;
166 ioatdca->req_slots[i].rid = 0;
167 ioatdca->requester_count--;
168 return i;
169 }
170 }
171 return -ENODEV;
172}
173
174static u8 ioat_dca_get_tag(struct dca_provider *dca, int cpu)
175{
176 struct ioat_dca_priv *ioatdca = dca_priv(dca);
177 int i, apic_id, bit, value;
178 u8 entry, tag;
179
180 tag = 0;
181 apic_id = cpu_physical_id(cpu);
182
183 for (i = 0; i < IOAT_TAG_MAP_LEN; i++) {
184 entry = ioatdca->tag_map[i];
185 if (entry & DCA_TAG_MAP_VALID) {
186 bit = entry & ~DCA_TAG_MAP_VALID;
187 value = (apic_id & (1 << bit)) ? 1 : 0;
188 } else {
189 value = entry ? 1 : 0;
190 }
191 tag |= (value << i);
192 }
193 return tag;
194}
195
196static struct dca_ops ioat_dca_ops = {
197 .add_requester = ioat_dca_add_requester,
198 .remove_requester = ioat_dca_remove_requester,
199 .get_tag = ioat_dca_get_tag,
200};
201
202
203struct dca_provider *ioat_dca_init(struct pci_dev *pdev, void __iomem *iobase)
204{
205 struct dca_provider *dca;
206 struct ioat_dca_priv *ioatdca;
207 u8 *tag_map = NULL;
208 int i;
209 int err;
210
211 if (!system_has_dca_enabled())
212 return NULL;
213
214 /* I/OAT v1 systems must have a known tag_map to support DCA */
215 switch (pdev->vendor) {
216 case PCI_VENDOR_ID_INTEL:
217 switch (pdev->device) {
218 case PCI_DEVICE_ID_INTEL_IOAT:
219 tag_map = ioat_tag_map_BNB;
220 break;
221 case PCI_DEVICE_ID_INTEL_IOAT_CNB:
222 tag_map = ioat_tag_map_CNB;
223 break;
224 case PCI_DEVICE_ID_INTEL_IOAT_SCNB:
225 tag_map = ioat_tag_map_SCNB;
226 break;
227 }
228 break;
229 case PCI_VENDOR_ID_UNISYS:
230 switch (pdev->device) {
231 case PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR:
232 tag_map = ioat_tag_map_UNISYS;
233 break;
234 }
235 break;
236 }
237 if (tag_map == NULL)
238 return NULL;
239
240 dca = alloc_dca_provider(&ioat_dca_ops,
241 sizeof(*ioatdca) +
242 (sizeof(struct ioat_dca_slot) * IOAT_DCA_MAX_REQ));
243 if (!dca)
244 return NULL;
245
246 ioatdca = dca_priv(dca);
247 ioatdca->max_requesters = IOAT_DCA_MAX_REQ;
248
249 ioatdca->dca_base = iobase + 0x54;
250
251 /* copy over the APIC ID to DCA tag mapping */
252 for (i = 0; i < IOAT_TAG_MAP_LEN; i++)
253 ioatdca->tag_map[i] = tag_map[i];
254
255 err = register_dca_provider(dca, &pdev->dev);
256 if (err) {
257 free_dca_provider(dca);
258 return NULL;
259 }
260
261 return dca;
262}
263
diff --git a/drivers/dma/ioatdma.h b/drivers/dma/ioatdma.h
index 2abf0b88a97..2a319e124ec 100644
--- a/drivers/dma/ioatdma.h
+++ b/drivers/dma/ioatdma.h
@@ -132,9 +132,12 @@ struct ioat_desc_sw {
132struct ioatdma_device *ioat_dma_probe(struct pci_dev *pdev, 132struct ioatdma_device *ioat_dma_probe(struct pci_dev *pdev,
133 void __iomem *iobase); 133 void __iomem *iobase);
134void ioat_dma_remove(struct ioatdma_device *device); 134void ioat_dma_remove(struct ioatdma_device *device);
135struct dca_provider *ioat_dca_init(struct pci_dev *pdev,
136 void __iomem *iobase);
135#else 137#else
136#define ioat_dma_probe(pdev, iobase) NULL 138#define ioat_dma_probe(pdev, iobase) NULL
137#define ioat_dma_remove(device) do { } while (0) 139#define ioat_dma_remove(device) do { } while (0)
140#define ioat_dca_init(pdev, iobase) NULL
138#endif 141#endif
139 142
140#endif /* IOATDMA_H */ 143#endif /* IOATDMA_H */
diff --git a/include/asm-x86/cpufeature_32.h b/include/asm-x86/cpufeature_32.h
index 7b3aa28ebc6..f17e688dfb0 100644
--- a/include/asm-x86/cpufeature_32.h
+++ b/include/asm-x86/cpufeature_32.h
@@ -92,6 +92,7 @@
92#define X86_FEATURE_CID (4*32+10) /* Context ID */ 92#define X86_FEATURE_CID (4*32+10) /* Context ID */
93#define X86_FEATURE_CX16 (4*32+13) /* CMPXCHG16B */ 93#define X86_FEATURE_CX16 (4*32+13) /* CMPXCHG16B */
94#define X86_FEATURE_XTPR (4*32+14) /* Send Task Priority Messages */ 94#define X86_FEATURE_XTPR (4*32+14) /* Send Task Priority Messages */
95#define X86_FEATURE_DCA (4*32+18) /* Direct Cache Access */
95 96
96/* VIA/Cyrix/Centaur-defined CPU features, CPUID level 0xC0000001, word 5 */ 97/* VIA/Cyrix/Centaur-defined CPU features, CPUID level 0xC0000001, word 5 */
97#define X86_FEATURE_XSTORE (5*32+ 2) /* on-CPU RNG present (xstore insn) */ 98#define X86_FEATURE_XSTORE (5*32+ 2) /* on-CPU RNG present (xstore insn) */