aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/dma
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/dma')
-rw-r--r--drivers/dma/Kconfig34
-rw-r--r--drivers/dma/Makefile3
-rw-r--r--drivers/dma/dmaengine.c408
-rw-r--r--drivers/dma/ioatdma.c840
-rw-r--r--drivers/dma/ioatdma.h125
-rw-r--r--drivers/dma/ioatdma_hw.h52
-rw-r--r--drivers/dma/ioatdma_io.h118
-rw-r--r--drivers/dma/ioatdma_registers.h126
-rw-r--r--drivers/dma/iovlock.c301
9 files changed, 2007 insertions, 0 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
new file mode 100644
index 00000000000..30d021d1a07
--- /dev/null
+++ b/drivers/dma/Kconfig
@@ -0,0 +1,34 @@
1#
2# DMA engine configuration
3#
4
5menu "DMA Engine support"
6
7config DMA_ENGINE
8 bool "Support for DMA engines"
9 ---help---
10 DMA engines offload copy operations from the CPU to dedicated
11 hardware, allowing the copies to happen asynchronously.
12
13comment "DMA Clients"
14
15config NET_DMA
16 bool "Network: TCP receive copy offload"
17 depends on DMA_ENGINE && NET
18 default y
19 ---help---
20 This enables the use of DMA engines in the network stack to
21 offload receive copy-to-user operations, freeing CPU cycles.
22 Since this is the main user of the DMA engine, it should be enabled;
23 say Y here.
24
25comment "DMA Devices"
26
27config INTEL_IOATDMA
28 tristate "Intel I/OAT DMA support"
29 depends on DMA_ENGINE && PCI
30 default m
31 ---help---
32 Enable support for the Intel(R) I/OAT DMA engine.
33
34endmenu
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
new file mode 100644
index 00000000000..bdcfdbdb1ae
--- /dev/null
+++ b/drivers/dma/Makefile
@@ -0,0 +1,3 @@
1obj-$(CONFIG_DMA_ENGINE) += dmaengine.o
2obj-$(CONFIG_NET_DMA) += iovlock.o
3obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
new file mode 100644
index 00000000000..5829143558e
--- /dev/null
+++ b/drivers/dma/dmaengine.c
@@ -0,0 +1,408 @@
1/*
2 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the Free
6 * Software Foundation; either version 2 of the License, or (at your option)
7 * any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc., 59
16 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * The full GNU General Public License is included in this distribution in the
19 * file called COPYING.
20 */
21
22/*
23 * This code implements the DMA subsystem. It provides a HW-neutral interface
24 * for other kernel code to use asynchronous memory copy capabilities,
25 * if present, and allows different HW DMA drivers to register as providing
26 * this capability.
27 *
28 * Due to the fact we are accelerating what is already a relatively fast
29 * operation, the code goes to great lengths to avoid additional overhead,
30 * such as locking.
31 *
32 * LOCKING:
33 *
34 * The subsystem keeps two global lists, dma_device_list and dma_client_list.
35 * Both of these are protected by a mutex, dma_list_mutex.
36 *
37 * Each device has a channels list, which runs unlocked but is never modified
38 * once the device is registered, it's just setup by the driver.
39 *
40 * Each client has a channels list, it's only modified under the client->lock
41 * and in an RCU callback, so it's safe to read under rcu_read_lock().
42 *
43 * Each device has a kref, which is initialized to 1 when the device is
44 * registered. A kref_put is done for each class_device registered. When the
45 * class_device is released, the coresponding kref_put is done in the release
46 * method. Every time one of the device's channels is allocated to a client,
47 * a kref_get occurs. When the channel is freed, the coresponding kref_put
48 * happens. The device's release function does a completion, so
49 * unregister_device does a remove event, class_device_unregister, a kref_put
50 * for the first reference, then waits on the completion for all other
51 * references to finish.
52 *
53 * Each channel has an open-coded implementation of Rusty Russell's "bigref,"
54 * with a kref and a per_cpu local_t. A single reference is set when on an
55 * ADDED event, and removed with a REMOVE event. Net DMA client takes an
56 * extra reference per outstanding transaction. The relase function does a
57 * kref_put on the device. -ChrisL
58 */
59
60#include <linux/init.h>
61#include <linux/module.h>
62#include <linux/device.h>
63#include <linux/dmaengine.h>
64#include <linux/hardirq.h>
65#include <linux/spinlock.h>
66#include <linux/percpu.h>
67#include <linux/rcupdate.h>
68#include <linux/mutex.h>
69
70static DEFINE_MUTEX(dma_list_mutex);
71static LIST_HEAD(dma_device_list);
72static LIST_HEAD(dma_client_list);
73
74/* --- sysfs implementation --- */
75
76static ssize_t show_memcpy_count(struct class_device *cd, char *buf)
77{
78 struct dma_chan *chan = container_of(cd, struct dma_chan, class_dev);
79 unsigned long count = 0;
80 int i;
81
82 for_each_possible_cpu(i)
83 count += per_cpu_ptr(chan->local, i)->memcpy_count;
84
85 return sprintf(buf, "%lu\n", count);
86}
87
88static ssize_t show_bytes_transferred(struct class_device *cd, char *buf)
89{
90 struct dma_chan *chan = container_of(cd, struct dma_chan, class_dev);
91 unsigned long count = 0;
92 int i;
93
94 for_each_possible_cpu(i)
95 count += per_cpu_ptr(chan->local, i)->bytes_transferred;
96
97 return sprintf(buf, "%lu\n", count);
98}
99
100static ssize_t show_in_use(struct class_device *cd, char *buf)
101{
102 struct dma_chan *chan = container_of(cd, struct dma_chan, class_dev);
103
104 return sprintf(buf, "%d\n", (chan->client ? 1 : 0));
105}
106
107static struct class_device_attribute dma_class_attrs[] = {
108 __ATTR(memcpy_count, S_IRUGO, show_memcpy_count, NULL),
109 __ATTR(bytes_transferred, S_IRUGO, show_bytes_transferred, NULL),
110 __ATTR(in_use, S_IRUGO, show_in_use, NULL),
111 __ATTR_NULL
112};
113
114static void dma_async_device_cleanup(struct kref *kref);
115
116static void dma_class_dev_release(struct class_device *cd)
117{
118 struct dma_chan *chan = container_of(cd, struct dma_chan, class_dev);
119 kref_put(&chan->device->refcount, dma_async_device_cleanup);
120}
121
122static struct class dma_devclass = {
123 .name = "dma",
124 .class_dev_attrs = dma_class_attrs,
125 .release = dma_class_dev_release,
126};
127
128/* --- client and device registration --- */
129
130/**
131 * dma_client_chan_alloc - try to allocate a channel to a client
132 * @client: &dma_client
133 *
134 * Called with dma_list_mutex held.
135 */
136static struct dma_chan *dma_client_chan_alloc(struct dma_client *client)
137{
138 struct dma_device *device;
139 struct dma_chan *chan;
140 unsigned long flags;
141 int desc; /* allocated descriptor count */
142
143 /* Find a channel, any DMA engine will do */
144 list_for_each_entry(device, &dma_device_list, global_node) {
145 list_for_each_entry(chan, &device->channels, device_node) {
146 if (chan->client)
147 continue;
148
149 desc = chan->device->device_alloc_chan_resources(chan);
150 if (desc >= 0) {
151 kref_get(&device->refcount);
152 kref_init(&chan->refcount);
153 chan->slow_ref = 0;
154 INIT_RCU_HEAD(&chan->rcu);
155 chan->client = client;
156 spin_lock_irqsave(&client->lock, flags);
157 list_add_tail_rcu(&chan->client_node,
158 &client->channels);
159 spin_unlock_irqrestore(&client->lock, flags);
160 return chan;
161 }
162 }
163 }
164
165 return NULL;
166}
167
168/**
169 * dma_client_chan_free - release a DMA channel
170 * @chan: &dma_chan
171 */
172void dma_chan_cleanup(struct kref *kref)
173{
174 struct dma_chan *chan = container_of(kref, struct dma_chan, refcount);
175 chan->device->device_free_chan_resources(chan);
176 chan->client = NULL;
177 kref_put(&chan->device->refcount, dma_async_device_cleanup);
178}
179
180static void dma_chan_free_rcu(struct rcu_head *rcu)
181{
182 struct dma_chan *chan = container_of(rcu, struct dma_chan, rcu);
183 int bias = 0x7FFFFFFF;
184 int i;
185 for_each_possible_cpu(i)
186 bias -= local_read(&per_cpu_ptr(chan->local, i)->refcount);
187 atomic_sub(bias, &chan->refcount.refcount);
188 kref_put(&chan->refcount, dma_chan_cleanup);
189}
190
191static void dma_client_chan_free(struct dma_chan *chan)
192{
193 atomic_add(0x7FFFFFFF, &chan->refcount.refcount);
194 chan->slow_ref = 1;
195 call_rcu(&chan->rcu, dma_chan_free_rcu);
196}
197
198/**
199 * dma_chans_rebalance - reallocate channels to clients
200 *
201 * When the number of DMA channel in the system changes,
202 * channels need to be rebalanced among clients
203 */
204static void dma_chans_rebalance(void)
205{
206 struct dma_client *client;
207 struct dma_chan *chan;
208 unsigned long flags;
209
210 mutex_lock(&dma_list_mutex);
211
212 list_for_each_entry(client, &dma_client_list, global_node) {
213 while (client->chans_desired > client->chan_count) {
214 chan = dma_client_chan_alloc(client);
215 if (!chan)
216 break;
217 client->chan_count++;
218 client->event_callback(client,
219 chan,
220 DMA_RESOURCE_ADDED);
221 }
222 while (client->chans_desired < client->chan_count) {
223 spin_lock_irqsave(&client->lock, flags);
224 chan = list_entry(client->channels.next,
225 struct dma_chan,
226 client_node);
227 list_del_rcu(&chan->client_node);
228 spin_unlock_irqrestore(&client->lock, flags);
229 client->chan_count--;
230 client->event_callback(client,
231 chan,
232 DMA_RESOURCE_REMOVED);
233 dma_client_chan_free(chan);
234 }
235 }
236
237 mutex_unlock(&dma_list_mutex);
238}
239
240/**
241 * dma_async_client_register - allocate and register a &dma_client
242 * @event_callback: callback for notification of channel addition/removal
243 */
244struct dma_client *dma_async_client_register(dma_event_callback event_callback)
245{
246 struct dma_client *client;
247
248 client = kzalloc(sizeof(*client), GFP_KERNEL);
249 if (!client)
250 return NULL;
251
252 INIT_LIST_HEAD(&client->channels);
253 spin_lock_init(&client->lock);
254 client->chans_desired = 0;
255 client->chan_count = 0;
256 client->event_callback = event_callback;
257
258 mutex_lock(&dma_list_mutex);
259 list_add_tail(&client->global_node, &dma_client_list);
260 mutex_unlock(&dma_list_mutex);
261
262 return client;
263}
264
265/**
266 * dma_async_client_unregister - unregister a client and free the &dma_client
267 * @client:
268 *
269 * Force frees any allocated DMA channels, frees the &dma_client memory
270 */
271void dma_async_client_unregister(struct dma_client *client)
272{
273 struct dma_chan *chan;
274
275 if (!client)
276 return;
277
278 rcu_read_lock();
279 list_for_each_entry_rcu(chan, &client->channels, client_node)
280 dma_client_chan_free(chan);
281 rcu_read_unlock();
282
283 mutex_lock(&dma_list_mutex);
284 list_del(&client->global_node);
285 mutex_unlock(&dma_list_mutex);
286
287 kfree(client);
288 dma_chans_rebalance();
289}
290
291/**
292 * dma_async_client_chan_request - request DMA channels
293 * @client: &dma_client
294 * @number: count of DMA channels requested
295 *
296 * Clients call dma_async_client_chan_request() to specify how many
297 * DMA channels they need, 0 to free all currently allocated.
298 * The resulting allocations/frees are indicated to the client via the
299 * event callback.
300 */
301void dma_async_client_chan_request(struct dma_client *client,
302 unsigned int number)
303{
304 client->chans_desired = number;
305 dma_chans_rebalance();
306}
307
308/**
309 * dma_async_device_register -
310 * @device: &dma_device
311 */
312int dma_async_device_register(struct dma_device *device)
313{
314 static int id;
315 int chancnt = 0;
316 struct dma_chan* chan;
317
318 if (!device)
319 return -ENODEV;
320
321 init_completion(&device->done);
322 kref_init(&device->refcount);
323 device->dev_id = id++;
324
325 /* represent channels in sysfs. Probably want devs too */
326 list_for_each_entry(chan, &device->channels, device_node) {
327 chan->local = alloc_percpu(typeof(*chan->local));
328 if (chan->local == NULL)
329 continue;
330
331 chan->chan_id = chancnt++;
332 chan->class_dev.class = &dma_devclass;
333 chan->class_dev.dev = NULL;
334 snprintf(chan->class_dev.class_id, BUS_ID_SIZE, "dma%dchan%d",
335 device->dev_id, chan->chan_id);
336
337 kref_get(&device->refcount);
338 class_device_register(&chan->class_dev);
339 }
340
341 mutex_lock(&dma_list_mutex);
342 list_add_tail(&device->global_node, &dma_device_list);
343 mutex_unlock(&dma_list_mutex);
344
345 dma_chans_rebalance();
346
347 return 0;
348}
349
350/**
351 * dma_async_device_unregister -
352 * @device: &dma_device
353 */
354static void dma_async_device_cleanup(struct kref *kref)
355{
356 struct dma_device *device;
357
358 device = container_of(kref, struct dma_device, refcount);
359 complete(&device->done);
360}
361
362void dma_async_device_unregister(struct dma_device* device)
363{
364 struct dma_chan *chan;
365 unsigned long flags;
366
367 mutex_lock(&dma_list_mutex);
368 list_del(&device->global_node);
369 mutex_unlock(&dma_list_mutex);
370
371 list_for_each_entry(chan, &device->channels, device_node) {
372 if (chan->client) {
373 spin_lock_irqsave(&chan->client->lock, flags);
374 list_del(&chan->client_node);
375 chan->client->chan_count--;
376 spin_unlock_irqrestore(&chan->client->lock, flags);
377 chan->client->event_callback(chan->client,
378 chan,
379 DMA_RESOURCE_REMOVED);
380 dma_client_chan_free(chan);
381 }
382 class_device_unregister(&chan->class_dev);
383 }
384 dma_chans_rebalance();
385
386 kref_put(&device->refcount, dma_async_device_cleanup);
387 wait_for_completion(&device->done);
388}
389
390static int __init dma_bus_init(void)
391{
392 mutex_init(&dma_list_mutex);
393 return class_register(&dma_devclass);
394}
395
396subsys_initcall(dma_bus_init);
397
398EXPORT_SYMBOL(dma_async_client_register);
399EXPORT_SYMBOL(dma_async_client_unregister);
400EXPORT_SYMBOL(dma_async_client_chan_request);
401EXPORT_SYMBOL(dma_async_memcpy_buf_to_buf);
402EXPORT_SYMBOL(dma_async_memcpy_buf_to_pg);
403EXPORT_SYMBOL(dma_async_memcpy_pg_to_pg);
404EXPORT_SYMBOL(dma_async_memcpy_complete);
405EXPORT_SYMBOL(dma_async_memcpy_issue_pending);
406EXPORT_SYMBOL(dma_async_device_register);
407EXPORT_SYMBOL(dma_async_device_unregister);
408EXPORT_SYMBOL(dma_chan_cleanup);
diff --git a/drivers/dma/ioatdma.c b/drivers/dma/ioatdma.c
new file mode 100644
index 00000000000..0fdf7fbd649
--- /dev/null
+++ b/drivers/dma/ioatdma.c
@@ -0,0 +1,840 @@
1/*
2 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the Free
6 * Software Foundation; either version 2 of the License, or (at your option)
7 * any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc., 59
16 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * The full GNU General Public License is included in this distribution in the
19 * file called COPYING.
20 */
21
22/*
23 * This driver supports an Intel I/OAT DMA engine, which does asynchronous
24 * copy operations.
25 */
26
27#include <linux/init.h>
28#include <linux/module.h>
29#include <linux/pci.h>
30#include <linux/interrupt.h>
31#include <linux/dmaengine.h>
32#include <linux/delay.h>
33#include <linux/dma-mapping.h>
34#include "ioatdma.h"
35#include "ioatdma_io.h"
36#include "ioatdma_registers.h"
37#include "ioatdma_hw.h"
38
39#define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common)
40#define to_ioat_device(dev) container_of(dev, struct ioat_device, common)
41#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
42
43/* internal functions */
44static int __devinit ioat_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
45static void __devexit ioat_remove(struct pci_dev *pdev);
46
47static int enumerate_dma_channels(struct ioat_device *device)
48{
49 u8 xfercap_scale;
50 u32 xfercap;
51 int i;
52 struct ioat_dma_chan *ioat_chan;
53
54 device->common.chancnt = ioatdma_read8(device, IOAT_CHANCNT_OFFSET);
55 xfercap_scale = ioatdma_read8(device, IOAT_XFERCAP_OFFSET);
56 xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
57
58 for (i = 0; i < device->common.chancnt; i++) {
59 ioat_chan = kzalloc(sizeof(*ioat_chan), GFP_KERNEL);
60 if (!ioat_chan) {
61 device->common.chancnt = i;
62 break;
63 }
64
65 ioat_chan->device = device;
66 ioat_chan->reg_base = device->reg_base + (0x80 * (i + 1));
67 ioat_chan->xfercap = xfercap;
68 spin_lock_init(&ioat_chan->cleanup_lock);
69 spin_lock_init(&ioat_chan->desc_lock);
70 INIT_LIST_HEAD(&ioat_chan->free_desc);
71 INIT_LIST_HEAD(&ioat_chan->used_desc);
72 /* This should be made common somewhere in dmaengine.c */
73 ioat_chan->common.device = &device->common;
74 ioat_chan->common.client = NULL;
75 list_add_tail(&ioat_chan->common.device_node,
76 &device->common.channels);
77 }
78 return device->common.chancnt;
79}
80
81static struct ioat_desc_sw *ioat_dma_alloc_descriptor(
82 struct ioat_dma_chan *ioat_chan,
83 int flags)
84{
85 struct ioat_dma_descriptor *desc;
86 struct ioat_desc_sw *desc_sw;
87 struct ioat_device *ioat_device;
88 dma_addr_t phys;
89
90 ioat_device = to_ioat_device(ioat_chan->common.device);
91 desc = pci_pool_alloc(ioat_device->dma_pool, flags, &phys);
92 if (unlikely(!desc))
93 return NULL;
94
95 desc_sw = kzalloc(sizeof(*desc_sw), flags);
96 if (unlikely(!desc_sw)) {
97 pci_pool_free(ioat_device->dma_pool, desc, phys);
98 return NULL;
99 }
100
101 memset(desc, 0, sizeof(*desc));
102 desc_sw->hw = desc;
103 desc_sw->phys = phys;
104
105 return desc_sw;
106}
107
108#define INITIAL_IOAT_DESC_COUNT 128
109
110static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan);
111
112/* returns the actual number of allocated descriptors */
113static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)
114{
115 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
116 struct ioat_desc_sw *desc = NULL;
117 u16 chanctrl;
118 u32 chanerr;
119 int i;
120 LIST_HEAD(tmp_list);
121
122 /*
123 * In-use bit automatically set by reading chanctrl
124 * If 0, we got it, if 1, someone else did
125 */
126 chanctrl = ioatdma_chan_read16(ioat_chan, IOAT_CHANCTRL_OFFSET);
127 if (chanctrl & IOAT_CHANCTRL_CHANNEL_IN_USE)
128 return -EBUSY;
129
130 /* Setup register to interrupt and write completion status on error */
131 chanctrl = IOAT_CHANCTRL_CHANNEL_IN_USE |
132 IOAT_CHANCTRL_ERR_INT_EN |
133 IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
134 IOAT_CHANCTRL_ERR_COMPLETION_EN;
135 ioatdma_chan_write16(ioat_chan, IOAT_CHANCTRL_OFFSET, chanctrl);
136
137 chanerr = ioatdma_chan_read32(ioat_chan, IOAT_CHANERR_OFFSET);
138 if (chanerr) {
139 printk("IOAT: CHANERR = %x, clearing\n", chanerr);
140 ioatdma_chan_write32(ioat_chan, IOAT_CHANERR_OFFSET, chanerr);
141 }
142
143 /* Allocate descriptors */
144 for (i = 0; i < INITIAL_IOAT_DESC_COUNT; i++) {
145 desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL);
146 if (!desc) {
147 printk(KERN_ERR "IOAT: Only %d initial descriptors\n", i);
148 break;
149 }
150 list_add_tail(&desc->node, &tmp_list);
151 }
152 spin_lock_bh(&ioat_chan->desc_lock);
153 list_splice(&tmp_list, &ioat_chan->free_desc);
154 spin_unlock_bh(&ioat_chan->desc_lock);
155
156 /* allocate a completion writeback area */
157 /* doing 2 32bit writes to mmio since 1 64b write doesn't work */
158 ioat_chan->completion_virt =
159 pci_pool_alloc(ioat_chan->device->completion_pool,
160 GFP_KERNEL,
161 &ioat_chan->completion_addr);
162 memset(ioat_chan->completion_virt, 0,
163 sizeof(*ioat_chan->completion_virt));
164 ioatdma_chan_write32(ioat_chan, IOAT_CHANCMP_OFFSET_LOW,
165 ((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF);
166 ioatdma_chan_write32(ioat_chan, IOAT_CHANCMP_OFFSET_HIGH,
167 ((u64) ioat_chan->completion_addr) >> 32);
168
169 ioat_start_null_desc(ioat_chan);
170 return i;
171}
172
173static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *ioat_chan);
174
175static void ioat_dma_free_chan_resources(struct dma_chan *chan)
176{
177 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
178 struct ioat_device *ioat_device = to_ioat_device(chan->device);
179 struct ioat_desc_sw *desc, *_desc;
180 u16 chanctrl;
181 int in_use_descs = 0;
182
183 ioat_dma_memcpy_cleanup(ioat_chan);
184
185 ioatdma_chan_write8(ioat_chan, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_RESET);
186
187 spin_lock_bh(&ioat_chan->desc_lock);
188 list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) {
189 in_use_descs++;
190 list_del(&desc->node);
191 pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys);
192 kfree(desc);
193 }
194 list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) {
195 list_del(&desc->node);
196 pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys);
197 kfree(desc);
198 }
199 spin_unlock_bh(&ioat_chan->desc_lock);
200
201 pci_pool_free(ioat_device->completion_pool,
202 ioat_chan->completion_virt,
203 ioat_chan->completion_addr);
204
205 /* one is ok since we left it on there on purpose */
206 if (in_use_descs > 1)
207 printk(KERN_ERR "IOAT: Freeing %d in use descriptors!\n",
208 in_use_descs - 1);
209
210 ioat_chan->last_completion = ioat_chan->completion_addr = 0;
211
212 /* Tell hw the chan is free */
213 chanctrl = ioatdma_chan_read16(ioat_chan, IOAT_CHANCTRL_OFFSET);
214 chanctrl &= ~IOAT_CHANCTRL_CHANNEL_IN_USE;
215 ioatdma_chan_write16(ioat_chan, IOAT_CHANCTRL_OFFSET, chanctrl);
216}
217
218/**
219 * do_ioat_dma_memcpy - actual function that initiates a IOAT DMA transaction
220 * @chan: IOAT DMA channel handle
221 * @dest: DMA destination address
222 * @src: DMA source address
223 * @len: transaction length in bytes
224 */
225
226static dma_cookie_t do_ioat_dma_memcpy(struct ioat_dma_chan *ioat_chan,
227 dma_addr_t dest,
228 dma_addr_t src,
229 size_t len)
230{
231 struct ioat_desc_sw *first;
232 struct ioat_desc_sw *prev;
233 struct ioat_desc_sw *new;
234 dma_cookie_t cookie;
235 LIST_HEAD(new_chain);
236 u32 copy;
237 size_t orig_len;
238 dma_addr_t orig_src, orig_dst;
239 unsigned int desc_count = 0;
240 unsigned int append = 0;
241
242 if (!ioat_chan || !dest || !src)
243 return -EFAULT;
244
245 if (!len)
246 return ioat_chan->common.cookie;
247
248 orig_len = len;
249 orig_src = src;
250 orig_dst = dest;
251
252 first = NULL;
253 prev = NULL;
254
255 spin_lock_bh(&ioat_chan->desc_lock);
256
257 while (len) {
258 if (!list_empty(&ioat_chan->free_desc)) {
259 new = to_ioat_desc(ioat_chan->free_desc.next);
260 list_del(&new->node);
261 } else {
262 /* try to get another desc */
263 new = ioat_dma_alloc_descriptor(ioat_chan, GFP_ATOMIC);
264 /* will this ever happen? */
265 /* TODO add upper limit on these */
266 BUG_ON(!new);
267 }
268
269 copy = min((u32) len, ioat_chan->xfercap);
270
271 new->hw->size = copy;
272 new->hw->ctl = 0;
273 new->hw->src_addr = src;
274 new->hw->dst_addr = dest;
275 new->cookie = 0;
276
277 /* chain together the physical address list for the HW */
278 if (!first)
279 first = new;
280 else
281 prev->hw->next = (u64) new->phys;
282
283 prev = new;
284
285 len -= copy;
286 dest += copy;
287 src += copy;
288
289 list_add_tail(&new->node, &new_chain);
290 desc_count++;
291 }
292 new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
293 new->hw->next = 0;
294
295 /* cookie incr and addition to used_list must be atomic */
296
297 cookie = ioat_chan->common.cookie;
298 cookie++;
299 if (cookie < 0)
300 cookie = 1;
301 ioat_chan->common.cookie = new->cookie = cookie;
302
303 pci_unmap_addr_set(new, src, orig_src);
304 pci_unmap_addr_set(new, dst, orig_dst);
305 pci_unmap_len_set(new, src_len, orig_len);
306 pci_unmap_len_set(new, dst_len, orig_len);
307
308 /* write address into NextDescriptor field of last desc in chain */
309 to_ioat_desc(ioat_chan->used_desc.prev)->hw->next = first->phys;
310 list_splice_init(&new_chain, ioat_chan->used_desc.prev);
311
312 ioat_chan->pending += desc_count;
313 if (ioat_chan->pending >= 20) {
314 append = 1;
315 ioat_chan->pending = 0;
316 }
317
318 spin_unlock_bh(&ioat_chan->desc_lock);
319
320 if (append)
321 ioatdma_chan_write8(ioat_chan,
322 IOAT_CHANCMD_OFFSET,
323 IOAT_CHANCMD_APPEND);
324 return cookie;
325}
326
327/**
328 * ioat_dma_memcpy_buf_to_buf - wrapper that takes src & dest bufs
329 * @chan: IOAT DMA channel handle
330 * @dest: DMA destination address
331 * @src: DMA source address
332 * @len: transaction length in bytes
333 */
334
335static dma_cookie_t ioat_dma_memcpy_buf_to_buf(struct dma_chan *chan,
336 void *dest,
337 void *src,
338 size_t len)
339{
340 dma_addr_t dest_addr;
341 dma_addr_t src_addr;
342 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
343
344 dest_addr = pci_map_single(ioat_chan->device->pdev,
345 dest, len, PCI_DMA_FROMDEVICE);
346 src_addr = pci_map_single(ioat_chan->device->pdev,
347 src, len, PCI_DMA_TODEVICE);
348
349 return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
350}
351
352/**
353 * ioat_dma_memcpy_buf_to_pg - wrapper, copying from a buf to a page
354 * @chan: IOAT DMA channel handle
355 * @page: pointer to the page to copy to
356 * @offset: offset into that page
357 * @src: DMA source address
358 * @len: transaction length in bytes
359 */
360
361static dma_cookie_t ioat_dma_memcpy_buf_to_pg(struct dma_chan *chan,
362 struct page *page,
363 unsigned int offset,
364 void *src,
365 size_t len)
366{
367 dma_addr_t dest_addr;
368 dma_addr_t src_addr;
369 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
370
371 dest_addr = pci_map_page(ioat_chan->device->pdev,
372 page, offset, len, PCI_DMA_FROMDEVICE);
373 src_addr = pci_map_single(ioat_chan->device->pdev,
374 src, len, PCI_DMA_TODEVICE);
375
376 return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
377}
378
379/**
380 * ioat_dma_memcpy_pg_to_pg - wrapper, copying between two pages
381 * @chan: IOAT DMA channel handle
382 * @dest_pg: pointer to the page to copy to
383 * @dest_off: offset into that page
384 * @src_pg: pointer to the page to copy from
385 * @src_off: offset into that page
386 * @len: transaction length in bytes. This is guaranteed to not make a copy
387 * across a page boundary.
388 */
389
390static dma_cookie_t ioat_dma_memcpy_pg_to_pg(struct dma_chan *chan,
391 struct page *dest_pg,
392 unsigned int dest_off,
393 struct page *src_pg,
394 unsigned int src_off,
395 size_t len)
396{
397 dma_addr_t dest_addr;
398 dma_addr_t src_addr;
399 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
400
401 dest_addr = pci_map_page(ioat_chan->device->pdev,
402 dest_pg, dest_off, len, PCI_DMA_FROMDEVICE);
403 src_addr = pci_map_page(ioat_chan->device->pdev,
404 src_pg, src_off, len, PCI_DMA_TODEVICE);
405
406 return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
407}
408
409/**
410 * ioat_dma_memcpy_issue_pending - push potentially unrecognoized appended descriptors to hw
411 * @chan: DMA channel handle
412 */
413
414static void ioat_dma_memcpy_issue_pending(struct dma_chan *chan)
415{
416 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
417
418 if (ioat_chan->pending != 0) {
419 ioat_chan->pending = 0;
420 ioatdma_chan_write8(ioat_chan,
421 IOAT_CHANCMD_OFFSET,
422 IOAT_CHANCMD_APPEND);
423 }
424}
425
426static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan)
427{
428 unsigned long phys_complete;
429 struct ioat_desc_sw *desc, *_desc;
430 dma_cookie_t cookie = 0;
431
432 prefetch(chan->completion_virt);
433
434 if (!spin_trylock(&chan->cleanup_lock))
435 return;
436
437 /* The completion writeback can happen at any time,
438 so reads by the driver need to be atomic operations
439 The descriptor physical addresses are limited to 32-bits
440 when the CPU can only do a 32-bit mov */
441
442#if (BITS_PER_LONG == 64)
443 phys_complete =
444 chan->completion_virt->full & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR;
445#else
446 phys_complete = chan->completion_virt->low & IOAT_LOW_COMPLETION_MASK;
447#endif
448
449 if ((chan->completion_virt->full & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
450 IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
451 printk("IOAT: Channel halted, chanerr = %x\n",
452 ioatdma_chan_read32(chan, IOAT_CHANERR_OFFSET));
453
454 /* TODO do something to salvage the situation */
455 }
456
457 if (phys_complete == chan->last_completion) {
458 spin_unlock(&chan->cleanup_lock);
459 return;
460 }
461
462 spin_lock_bh(&chan->desc_lock);
463 list_for_each_entry_safe(desc, _desc, &chan->used_desc, node) {
464
465 /*
466 * Incoming DMA requests may use multiple descriptors, due to
467 * exceeding xfercap, perhaps. If so, only the last one will
468 * have a cookie, and require unmapping.
469 */
470 if (desc->cookie) {
471 cookie = desc->cookie;
472
473 /* yes we are unmapping both _page and _single alloc'd
474 regions with unmap_page. Is this *really* that bad?
475 */
476 pci_unmap_page(chan->device->pdev,
477 pci_unmap_addr(desc, dst),
478 pci_unmap_len(desc, dst_len),
479 PCI_DMA_FROMDEVICE);
480 pci_unmap_page(chan->device->pdev,
481 pci_unmap_addr(desc, src),
482 pci_unmap_len(desc, src_len),
483 PCI_DMA_TODEVICE);
484 }
485
486 if (desc->phys != phys_complete) {
487 /* a completed entry, but not the last, so cleanup */
488 list_del(&desc->node);
489 list_add_tail(&desc->node, &chan->free_desc);
490 } else {
491 /* last used desc. Do not remove, so we can append from
492 it, but don't look at it next time, either */
493 desc->cookie = 0;
494
495 /* TODO check status bits? */
496 break;
497 }
498 }
499
500 spin_unlock_bh(&chan->desc_lock);
501
502 chan->last_completion = phys_complete;
503 if (cookie != 0)
504 chan->completed_cookie = cookie;
505
506 spin_unlock(&chan->cleanup_lock);
507}
508
509/**
510 * ioat_dma_is_complete - poll the status of a IOAT DMA transaction
511 * @chan: IOAT DMA channel handle
512 * @cookie: DMA transaction identifier
513 */
514
515static enum dma_status ioat_dma_is_complete(struct dma_chan *chan,
516 dma_cookie_t cookie,
517 dma_cookie_t *done,
518 dma_cookie_t *used)
519{
520 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
521 dma_cookie_t last_used;
522 dma_cookie_t last_complete;
523 enum dma_status ret;
524
525 last_used = chan->cookie;
526 last_complete = ioat_chan->completed_cookie;
527
528 if (done)
529 *done= last_complete;
530 if (used)
531 *used = last_used;
532
533 ret = dma_async_is_complete(cookie, last_complete, last_used);
534 if (ret == DMA_SUCCESS)
535 return ret;
536
537 ioat_dma_memcpy_cleanup(ioat_chan);
538
539 last_used = chan->cookie;
540 last_complete = ioat_chan->completed_cookie;
541
542 if (done)
543 *done= last_complete;
544 if (used)
545 *used = last_used;
546
547 return dma_async_is_complete(cookie, last_complete, last_used);
548}
549
550/* PCI API */
551
552static struct pci_device_id ioat_pci_tbl[] = {
553 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT) },
554 { 0, }
555};
556
557static struct pci_driver ioat_pci_drv = {
558 .name = "ioatdma",
559 .id_table = ioat_pci_tbl,
560 .probe = ioat_probe,
561 .remove = __devexit_p(ioat_remove),
562};
563
564static irqreturn_t ioat_do_interrupt(int irq, void *data, struct pt_regs *regs)
565{
566 struct ioat_device *instance = data;
567 unsigned long attnstatus;
568 u8 intrctrl;
569
570 intrctrl = ioatdma_read8(instance, IOAT_INTRCTRL_OFFSET);
571
572 if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
573 return IRQ_NONE;
574
575 if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
576 ioatdma_write8(instance, IOAT_INTRCTRL_OFFSET, intrctrl);
577 return IRQ_NONE;
578 }
579
580 attnstatus = ioatdma_read32(instance, IOAT_ATTNSTATUS_OFFSET);
581
582 printk(KERN_ERR "ioatdma error: interrupt! status %lx\n", attnstatus);
583
584 ioatdma_write8(instance, IOAT_INTRCTRL_OFFSET, intrctrl);
585 return IRQ_HANDLED;
586}
587
588static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan)
589{
590 struct ioat_desc_sw *desc;
591
592 spin_lock_bh(&ioat_chan->desc_lock);
593
594 if (!list_empty(&ioat_chan->free_desc)) {
595 desc = to_ioat_desc(ioat_chan->free_desc.next);
596 list_del(&desc->node);
597 } else {
598 /* try to get another desc */
599 spin_unlock_bh(&ioat_chan->desc_lock);
600 desc = ioat_dma_alloc_descriptor(ioat_chan, GFP_KERNEL);
601 spin_lock_bh(&ioat_chan->desc_lock);
602 /* will this ever happen? */
603 BUG_ON(!desc);
604 }
605
606 desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL;
607 desc->hw->next = 0;
608
609 list_add_tail(&desc->node, &ioat_chan->used_desc);
610 spin_unlock_bh(&ioat_chan->desc_lock);
611
612#if (BITS_PER_LONG == 64)
613 ioatdma_chan_write64(ioat_chan, IOAT_CHAINADDR_OFFSET, desc->phys);
614#else
615 ioatdma_chan_write32(ioat_chan,
616 IOAT_CHAINADDR_OFFSET_LOW,
617 (u32) desc->phys);
618 ioatdma_chan_write32(ioat_chan, IOAT_CHAINADDR_OFFSET_HIGH, 0);
619#endif
620 ioatdma_chan_write8(ioat_chan, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_START);
621}
622
623/*
624 * Perform a IOAT transaction to verify the HW works.
625 */
626#define IOAT_TEST_SIZE 2000
627
628static int ioat_self_test(struct ioat_device *device)
629{
630 int i;
631 u8 *src;
632 u8 *dest;
633 struct dma_chan *dma_chan;
634 dma_cookie_t cookie;
635 int err = 0;
636
637 src = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, SLAB_KERNEL);
638 if (!src)
639 return -ENOMEM;
640 dest = kzalloc(sizeof(u8) * IOAT_TEST_SIZE, SLAB_KERNEL);
641 if (!dest) {
642 kfree(src);
643 return -ENOMEM;
644 }
645
646 /* Fill in src buffer */
647 for (i = 0; i < IOAT_TEST_SIZE; i++)
648 src[i] = (u8)i;
649
650 /* Start copy, using first DMA channel */
651 dma_chan = container_of(device->common.channels.next,
652 struct dma_chan,
653 device_node);
654 if (ioat_dma_alloc_chan_resources(dma_chan) < 1) {
655 err = -ENODEV;
656 goto out;
657 }
658
659 cookie = ioat_dma_memcpy_buf_to_buf(dma_chan, dest, src, IOAT_TEST_SIZE);
660 ioat_dma_memcpy_issue_pending(dma_chan);
661 msleep(1);
662
663 if (ioat_dma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
664 printk(KERN_ERR "ioatdma: Self-test copy timed out, disabling\n");
665 err = -ENODEV;
666 goto free_resources;
667 }
668 if (memcmp(src, dest, IOAT_TEST_SIZE)) {
669 printk(KERN_ERR "ioatdma: Self-test copy failed compare, disabling\n");
670 err = -ENODEV;
671 goto free_resources;
672 }
673
674free_resources:
675 ioat_dma_free_chan_resources(dma_chan);
676out:
677 kfree(src);
678 kfree(dest);
679 return err;
680}
681
682static int __devinit ioat_probe(struct pci_dev *pdev,
683 const struct pci_device_id *ent)
684{
685 int err;
686 unsigned long mmio_start, mmio_len;
687 void *reg_base;
688 struct ioat_device *device;
689
690 err = pci_enable_device(pdev);
691 if (err)
692 goto err_enable_device;
693
694 err = pci_set_dma_mask(pdev, DMA_64BIT_MASK);
695 if (err)
696 err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
697 if (err)
698 goto err_set_dma_mask;
699
700 err = pci_request_regions(pdev, ioat_pci_drv.name);
701 if (err)
702 goto err_request_regions;
703
704 mmio_start = pci_resource_start(pdev, 0);
705 mmio_len = pci_resource_len(pdev, 0);
706
707 reg_base = ioremap(mmio_start, mmio_len);
708 if (!reg_base) {
709 err = -ENOMEM;
710 goto err_ioremap;
711 }
712
713 device = kzalloc(sizeof(*device), GFP_KERNEL);
714 if (!device) {
715 err = -ENOMEM;
716 goto err_kzalloc;
717 }
718
719 /* DMA coherent memory pool for DMA descriptor allocations */
720 device->dma_pool = pci_pool_create("dma_desc_pool", pdev,
721 sizeof(struct ioat_dma_descriptor), 64, 0);
722 if (!device->dma_pool) {
723 err = -ENOMEM;
724 goto err_dma_pool;
725 }
726
727 device->completion_pool = pci_pool_create("completion_pool", pdev, sizeof(u64), SMP_CACHE_BYTES, SMP_CACHE_BYTES);
728 if (!device->completion_pool) {
729 err = -ENOMEM;
730 goto err_completion_pool;
731 }
732
733 device->pdev = pdev;
734 pci_set_drvdata(pdev, device);
735#ifdef CONFIG_PCI_MSI
736 if (pci_enable_msi(pdev) == 0) {
737 device->msi = 1;
738 } else {
739 device->msi = 0;
740 }
741#endif
742 err = request_irq(pdev->irq, &ioat_do_interrupt, SA_SHIRQ, "ioat",
743 device);
744 if (err)
745 goto err_irq;
746
747 device->reg_base = reg_base;
748
749 ioatdma_write8(device, IOAT_INTRCTRL_OFFSET, IOAT_INTRCTRL_MASTER_INT_EN);
750 pci_set_master(pdev);
751
752 INIT_LIST_HEAD(&device->common.channels);
753 enumerate_dma_channels(device);
754
755 device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources;
756 device->common.device_free_chan_resources = ioat_dma_free_chan_resources;
757 device->common.device_memcpy_buf_to_buf = ioat_dma_memcpy_buf_to_buf;
758 device->common.device_memcpy_buf_to_pg = ioat_dma_memcpy_buf_to_pg;
759 device->common.device_memcpy_pg_to_pg = ioat_dma_memcpy_pg_to_pg;
760 device->common.device_memcpy_complete = ioat_dma_is_complete;
761 device->common.device_memcpy_issue_pending = ioat_dma_memcpy_issue_pending;
762 printk(KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n",
763 device->common.chancnt);
764
765 err = ioat_self_test(device);
766 if (err)
767 goto err_self_test;
768
769 dma_async_device_register(&device->common);
770
771 return 0;
772
773err_self_test:
774err_irq:
775 pci_pool_destroy(device->completion_pool);
776err_completion_pool:
777 pci_pool_destroy(device->dma_pool);
778err_dma_pool:
779 kfree(device);
780err_kzalloc:
781 iounmap(reg_base);
782err_ioremap:
783 pci_release_regions(pdev);
784err_request_regions:
785err_set_dma_mask:
786 pci_disable_device(pdev);
787err_enable_device:
788 return err;
789}
790
791static void __devexit ioat_remove(struct pci_dev *pdev)
792{
793 struct ioat_device *device;
794 struct dma_chan *chan, *_chan;
795 struct ioat_dma_chan *ioat_chan;
796
797 device = pci_get_drvdata(pdev);
798 dma_async_device_unregister(&device->common);
799
800 free_irq(device->pdev->irq, device);
801#ifdef CONFIG_PCI_MSI
802 if (device->msi)
803 pci_disable_msi(device->pdev);
804#endif
805 pci_pool_destroy(device->dma_pool);
806 pci_pool_destroy(device->completion_pool);
807 iounmap(device->reg_base);
808 pci_release_regions(pdev);
809 pci_disable_device(pdev);
810 list_for_each_entry_safe(chan, _chan, &device->common.channels, device_node) {
811 ioat_chan = to_ioat_chan(chan);
812 list_del(&chan->device_node);
813 kfree(ioat_chan);
814 }
815 kfree(device);
816}
817
818/* MODULE API */
819MODULE_VERSION("1.7");
820MODULE_LICENSE("GPL");
821MODULE_AUTHOR("Intel Corporation");
822
823static int __init ioat_init_module(void)
824{
825 /* it's currently unsafe to unload this module */
826 /* if forced, worst case is that rmmod hangs */
827 if (THIS_MODULE != NULL)
828 THIS_MODULE->unsafe = 1;
829
830 return pci_module_init(&ioat_pci_drv);
831}
832
833module_init(ioat_init_module);
834
835static void __exit ioat_exit_module(void)
836{
837 pci_unregister_driver(&ioat_pci_drv);
838}
839
840module_exit(ioat_exit_module);
diff --git a/drivers/dma/ioatdma.h b/drivers/dma/ioatdma.h
new file mode 100644
index 00000000000..a5d3b364416
--- /dev/null
+++ b/drivers/dma/ioatdma.h
@@ -0,0 +1,125 @@
1/*
2 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the Free
6 * Software Foundation; either version 2 of the License, or (at your option)
7 * any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc., 59
16 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * The full GNU General Public License is included in this distribution in the
19 * file called COPYING.
20 */
21#ifndef IOATDMA_H
22#define IOATDMA_H
23
24#include <linux/dmaengine.h>
25#include "ioatdma_hw.h"
26#include <linux/init.h>
27#include <linux/dmapool.h>
28#include <linux/cache.h>
29#include <linux/pci_ids.h>
30
31#define IOAT_LOW_COMPLETION_MASK 0xffffffc0
32
33extern struct list_head dma_device_list;
34extern struct list_head dma_client_list;
35
36/**
37 * struct ioat_device - internal representation of a IOAT device
38 * @pdev: PCI-Express device
39 * @reg_base: MMIO register space base address
40 * @dma_pool: for allocating DMA descriptors
41 * @common: embedded struct dma_device
42 * @msi: Message Signaled Interrupt number
43 */
44
45struct ioat_device {
46 struct pci_dev *pdev;
47 void *reg_base;
48 struct pci_pool *dma_pool;
49 struct pci_pool *completion_pool;
50
51 struct dma_device common;
52 u8 msi;
53};
54
55/**
56 * struct ioat_dma_chan - internal representation of a DMA channel
57 * @device:
58 * @reg_base:
59 * @sw_in_use:
60 * @completion:
61 * @completion_low:
62 * @completion_high:
63 * @completed_cookie: last cookie seen completed on cleanup
64 * @cookie: value of last cookie given to client
65 * @last_completion:
66 * @xfercap:
67 * @desc_lock:
68 * @free_desc:
69 * @used_desc:
70 * @resource:
71 * @device_node:
72 */
73
74struct ioat_dma_chan {
75
76 void *reg_base;
77
78 dma_cookie_t completed_cookie;
79 unsigned long last_completion;
80
81 u32 xfercap; /* XFERCAP register value expanded out */
82
83 spinlock_t cleanup_lock;
84 spinlock_t desc_lock;
85 struct list_head free_desc;
86 struct list_head used_desc;
87
88 int pending;
89
90 struct ioat_device *device;
91 struct dma_chan common;
92
93 dma_addr_t completion_addr;
94 union {
95 u64 full; /* HW completion writeback */
96 struct {
97 u32 low;
98 u32 high;
99 };
100 } *completion_virt;
101};
102
103/* wrapper around hardware descriptor format + additional software fields */
104
105/**
106 * struct ioat_desc_sw - wrapper around hardware descriptor
107 * @hw: hardware DMA descriptor
108 * @node:
109 * @cookie:
110 * @phys:
111 */
112
113struct ioat_desc_sw {
114 struct ioat_dma_descriptor *hw;
115 struct list_head node;
116 dma_cookie_t cookie;
117 dma_addr_t phys;
118 DECLARE_PCI_UNMAP_ADDR(src)
119 DECLARE_PCI_UNMAP_LEN(src_len)
120 DECLARE_PCI_UNMAP_ADDR(dst)
121 DECLARE_PCI_UNMAP_LEN(dst_len)
122};
123
124#endif /* IOATDMA_H */
125
diff --git a/drivers/dma/ioatdma_hw.h b/drivers/dma/ioatdma_hw.h
new file mode 100644
index 00000000000..4d7a12880be
--- /dev/null
+++ b/drivers/dma/ioatdma_hw.h
@@ -0,0 +1,52 @@
1/*
2 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the Free
6 * Software Foundation; either version 2 of the License, or (at your option)
7 * any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc., 59
16 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * The full GNU General Public License is included in this distribution in the
19 * file called COPYING.
20 */
21#ifndef _IOAT_HW_H_
22#define _IOAT_HW_H_
23
24/* PCI Configuration Space Values */
25#define IOAT_PCI_VID 0x8086
26#define IOAT_PCI_DID 0x1A38
27#define IOAT_PCI_RID 0x00
28#define IOAT_PCI_SVID 0x8086
29#define IOAT_PCI_SID 0x8086
30#define IOAT_VER 0x12 /* Version 1.2 */
31
32struct ioat_dma_descriptor {
33 uint32_t size;
34 uint32_t ctl;
35 uint64_t src_addr;
36 uint64_t dst_addr;
37 uint64_t next;
38 uint64_t rsv1;
39 uint64_t rsv2;
40 uint64_t user1;
41 uint64_t user2;
42};
43
44#define IOAT_DMA_DESCRIPTOR_CTL_INT_GN 0x00000001
45#define IOAT_DMA_DESCRIPTOR_CTL_SRC_SN 0x00000002
46#define IOAT_DMA_DESCRIPTOR_CTL_DST_SN 0x00000004
47#define IOAT_DMA_DESCRIPTOR_CTL_CP_STS 0x00000008
48#define IOAT_DMA_DESCRIPTOR_CTL_FRAME 0x00000010
49#define IOAT_DMA_DESCRIPTOR_NUL 0x00000020
50#define IOAT_DMA_DESCRIPTOR_OPCODE 0xFF000000
51
52#endif
diff --git a/drivers/dma/ioatdma_io.h b/drivers/dma/ioatdma_io.h
new file mode 100644
index 00000000000..c0b4bf66c92
--- /dev/null
+++ b/drivers/dma/ioatdma_io.h
@@ -0,0 +1,118 @@
1/*
2 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the Free
6 * Software Foundation; either version 2 of the License, or (at your option)
7 * any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc., 59
16 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * The full GNU General Public License is included in this distribution in the
19 * file called COPYING.
20 */
21#ifndef IOATDMA_IO_H
22#define IOATDMA_IO_H
23
24#include <asm/io.h>
25
26/*
27 * device and per-channel MMIO register read and write functions
28 * this is a lot of anoying inline functions, but it's typesafe
29 */
30
31static inline u8 ioatdma_read8(struct ioat_device *device,
32 unsigned int offset)
33{
34 return readb(device->reg_base + offset);
35}
36
37static inline u16 ioatdma_read16(struct ioat_device *device,
38 unsigned int offset)
39{
40 return readw(device->reg_base + offset);
41}
42
43static inline u32 ioatdma_read32(struct ioat_device *device,
44 unsigned int offset)
45{
46 return readl(device->reg_base + offset);
47}
48
49static inline void ioatdma_write8(struct ioat_device *device,
50 unsigned int offset, u8 value)
51{
52 writeb(value, device->reg_base + offset);
53}
54
55static inline void ioatdma_write16(struct ioat_device *device,
56 unsigned int offset, u16 value)
57{
58 writew(value, device->reg_base + offset);
59}
60
61static inline void ioatdma_write32(struct ioat_device *device,
62 unsigned int offset, u32 value)
63{
64 writel(value, device->reg_base + offset);
65}
66
67static inline u8 ioatdma_chan_read8(struct ioat_dma_chan *chan,
68 unsigned int offset)
69{
70 return readb(chan->reg_base + offset);
71}
72
73static inline u16 ioatdma_chan_read16(struct ioat_dma_chan *chan,
74 unsigned int offset)
75{
76 return readw(chan->reg_base + offset);
77}
78
79static inline u32 ioatdma_chan_read32(struct ioat_dma_chan *chan,
80 unsigned int offset)
81{
82 return readl(chan->reg_base + offset);
83}
84
85static inline void ioatdma_chan_write8(struct ioat_dma_chan *chan,
86 unsigned int offset, u8 value)
87{
88 writeb(value, chan->reg_base + offset);
89}
90
91static inline void ioatdma_chan_write16(struct ioat_dma_chan *chan,
92 unsigned int offset, u16 value)
93{
94 writew(value, chan->reg_base + offset);
95}
96
97static inline void ioatdma_chan_write32(struct ioat_dma_chan *chan,
98 unsigned int offset, u32 value)
99{
100 writel(value, chan->reg_base + offset);
101}
102
103#if (BITS_PER_LONG == 64)
104static inline u64 ioatdma_chan_read64(struct ioat_dma_chan *chan,
105 unsigned int offset)
106{
107 return readq(chan->reg_base + offset);
108}
109
110static inline void ioatdma_chan_write64(struct ioat_dma_chan *chan,
111 unsigned int offset, u64 value)
112{
113 writeq(value, chan->reg_base + offset);
114}
115#endif
116
117#endif /* IOATDMA_IO_H */
118
diff --git a/drivers/dma/ioatdma_registers.h b/drivers/dma/ioatdma_registers.h
new file mode 100644
index 00000000000..41a21ab2b00
--- /dev/null
+++ b/drivers/dma/ioatdma_registers.h
@@ -0,0 +1,126 @@
1/*
2 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the Free
6 * Software Foundation; either version 2 of the License, or (at your option)
7 * any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc., 59
16 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * The full GNU General Public License is included in this distribution in the
19 * file called COPYING.
20 */
21#ifndef _IOAT_REGISTERS_H_
22#define _IOAT_REGISTERS_H_
23
24
25/* MMIO Device Registers */
26#define IOAT_CHANCNT_OFFSET 0x00 /* 8-bit */
27
28#define IOAT_XFERCAP_OFFSET 0x01 /* 8-bit */
29#define IOAT_XFERCAP_4KB 12
30#define IOAT_XFERCAP_8KB 13
31#define IOAT_XFERCAP_16KB 14
32#define IOAT_XFERCAP_32KB 15
33#define IOAT_XFERCAP_32GB 0
34
35#define IOAT_GENCTRL_OFFSET 0x02 /* 8-bit */
36#define IOAT_GENCTRL_DEBUG_EN 0x01
37
38#define IOAT_INTRCTRL_OFFSET 0x03 /* 8-bit */
39#define IOAT_INTRCTRL_MASTER_INT_EN 0x01 /* Master Interrupt Enable */
40#define IOAT_INTRCTRL_INT_STATUS 0x02 /* ATTNSTATUS -or- Channel Int */
41#define IOAT_INTRCTRL_INT 0x04 /* INT_STATUS -and- MASTER_INT_EN */
42
43#define IOAT_ATTNSTATUS_OFFSET 0x04 /* Each bit is a channel */
44
45#define IOAT_VER_OFFSET 0x08 /* 8-bit */
46#define IOAT_VER_MAJOR_MASK 0xF0
47#define IOAT_VER_MINOR_MASK 0x0F
48#define GET_IOAT_VER_MAJOR(x) ((x) & IOAT_VER_MAJOR_MASK)
49#define GET_IOAT_VER_MINOR(x) ((x) & IOAT_VER_MINOR_MASK)
50
51#define IOAT_PERPORTOFFSET_OFFSET 0x0A /* 16-bit */
52
53#define IOAT_INTRDELAY_OFFSET 0x0C /* 16-bit */
54#define IOAT_INTRDELAY_INT_DELAY_MASK 0x3FFF /* Interrupt Delay Time */
55#define IOAT_INTRDELAY_COALESE_SUPPORT 0x8000 /* Interrupt Coalesing Supported */
56
57#define IOAT_DEVICE_STATUS_OFFSET 0x0E /* 16-bit */
58#define IOAT_DEVICE_STATUS_DEGRADED_MODE 0x0001
59
60
61#define IOAT_CHANNEL_MMIO_SIZE 0x80 /* Each Channel MMIO space is this size */
62
63/* DMA Channel Registers */
64#define IOAT_CHANCTRL_OFFSET 0x00 /* 16-bit Channel Control Register */
65#define IOAT_CHANCTRL_CHANNEL_PRIORITY_MASK 0xF000
66#define IOAT_CHANCTRL_CHANNEL_IN_USE 0x0100
67#define IOAT_CHANCTRL_DESCRIPTOR_ADDR_SNOOP_CONTROL 0x0020
68#define IOAT_CHANCTRL_ERR_INT_EN 0x0010
69#define IOAT_CHANCTRL_ANY_ERR_ABORT_EN 0x0008
70#define IOAT_CHANCTRL_ERR_COMPLETION_EN 0x0004
71#define IOAT_CHANCTRL_INT_DISABLE 0x0001
72
73#define IOAT_DMA_COMP_OFFSET 0x02 /* 16-bit DMA channel compatability */
74#define IOAT_DMA_COMP_V1 0x0001 /* Compatability with DMA version 1 */
75
76#define IOAT_CHANSTS_OFFSET 0x04 /* 64-bit Channel Status Register */
77#define IOAT_CHANSTS_OFFSET_LOW 0x04
78#define IOAT_CHANSTS_OFFSET_HIGH 0x08
79#define IOAT_CHANSTS_COMPLETED_DESCRIPTOR_ADDR 0xFFFFFFFFFFFFFFC0
80#define IOAT_CHANSTS_SOFT_ERR 0x0000000000000010
81#define IOAT_CHANSTS_DMA_TRANSFER_STATUS 0x0000000000000007
82#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_ACTIVE 0x0
83#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_DONE 0x1
84#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_SUSPENDED 0x2
85#define IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED 0x3
86
87#define IOAT_CHAINADDR_OFFSET 0x0C /* 64-bit Descriptor Chain Address Register */
88#define IOAT_CHAINADDR_OFFSET_LOW 0x0C
89#define IOAT_CHAINADDR_OFFSET_HIGH 0x10
90
91#define IOAT_CHANCMD_OFFSET 0x14 /* 8-bit DMA Channel Command Register */
92#define IOAT_CHANCMD_RESET 0x20
93#define IOAT_CHANCMD_RESUME 0x10
94#define IOAT_CHANCMD_ABORT 0x08
95#define IOAT_CHANCMD_SUSPEND 0x04
96#define IOAT_CHANCMD_APPEND 0x02
97#define IOAT_CHANCMD_START 0x01
98
99#define IOAT_CHANCMP_OFFSET 0x18 /* 64-bit Channel Completion Address Register */
100#define IOAT_CHANCMP_OFFSET_LOW 0x18
101#define IOAT_CHANCMP_OFFSET_HIGH 0x1C
102
103#define IOAT_CDAR_OFFSET 0x20 /* 64-bit Current Descriptor Address Register */
104#define IOAT_CDAR_OFFSET_LOW 0x20
105#define IOAT_CDAR_OFFSET_HIGH 0x24
106
107#define IOAT_CHANERR_OFFSET 0x28 /* 32-bit Channel Error Register */
108#define IOAT_CHANERR_DMA_TRANSFER_SRC_ADDR_ERR 0x0001
109#define IOAT_CHANERR_DMA_TRANSFER_DEST_ADDR_ERR 0x0002
110#define IOAT_CHANERR_NEXT_DESCRIPTOR_ADDR_ERR 0x0004
111#define IOAT_CHANERR_NEXT_DESCRIPTOR_ALIGNMENT_ERR 0x0008
112#define IOAT_CHANERR_CHAIN_ADDR_VALUE_ERR 0x0010
113#define IOAT_CHANERR_CHANCMD_ERR 0x0020
114#define IOAT_CHANERR_CHIPSET_UNCORRECTABLE_DATA_INTEGRITY_ERR 0x0040
115#define IOAT_CHANERR_DMA_UNCORRECTABLE_DATA_INTEGRITY_ERR 0x0080
116#define IOAT_CHANERR_READ_DATA_ERR 0x0100
117#define IOAT_CHANERR_WRITE_DATA_ERR 0x0200
118#define IOAT_CHANERR_DESCRIPTOR_CONTROL_ERR 0x0400
119#define IOAT_CHANERR_DESCRIPTOR_LENGTH_ERR 0x0800
120#define IOAT_CHANERR_COMPLETION_ADDR_ERR 0x1000
121#define IOAT_CHANERR_INT_CONFIGURATION_ERR 0x2000
122#define IOAT_CHANERR_SOFT_ERR 0x4000
123
124#define IOAT_CHANERR_MASK_OFFSET 0x2C /* 32-bit Channel Error Register */
125
126#endif /* _IOAT_REGISTERS_H_ */
diff --git a/drivers/dma/iovlock.c b/drivers/dma/iovlock.c
new file mode 100644
index 00000000000..5ed327e453a
--- /dev/null
+++ b/drivers/dma/iovlock.c
@@ -0,0 +1,301 @@
1/*
2 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
3 * Portions based on net/core/datagram.c and copyrighted by their authors.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License as published by the Free
7 * Software Foundation; either version 2 of the License, or (at your option)
8 * any later version.
9 *
10 * This program is distributed in the hope that it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * more details.
14 *
15 * You should have received a copy of the GNU General Public License along with
16 * this program; if not, write to the Free Software Foundation, Inc., 59
17 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
18 *
19 * The full GNU General Public License is included in this distribution in the
20 * file called COPYING.
21 */
22
23/*
24 * This code allows the net stack to make use of a DMA engine for
25 * skb to iovec copies.
26 */
27
28#include <linux/dmaengine.h>
29#include <linux/pagemap.h>
30#include <net/tcp.h> /* for memcpy_toiovec */
31#include <asm/io.h>
32#include <asm/uaccess.h>
33
34int num_pages_spanned(struct iovec *iov)
35{
36 return
37 ((PAGE_ALIGN((unsigned long)iov->iov_base + iov->iov_len) -
38 ((unsigned long)iov->iov_base & PAGE_MASK)) >> PAGE_SHIFT);
39}
40
41/*
42 * Pin down all the iovec pages needed for len bytes.
43 * Return a struct dma_pinned_list to keep track of pages pinned down.
44 *
45 * We are allocating a single chunk of memory, and then carving it up into
46 * 3 sections, the latter 2 whose size depends on the number of iovecs and the
47 * total number of pages, respectively.
48 */
49struct dma_pinned_list *dma_pin_iovec_pages(struct iovec *iov, size_t len)
50{
51 struct dma_pinned_list *local_list;
52 struct page **pages;
53 int i;
54 int ret;
55 int nr_iovecs = 0;
56 int iovec_len_used = 0;
57 int iovec_pages_used = 0;
58 long err;
59
60 /* don't pin down non-user-based iovecs */
61 if (segment_eq(get_fs(), KERNEL_DS))
62 return NULL;
63
64 /* determine how many iovecs/pages there are, up front */
65 do {
66 iovec_len_used += iov[nr_iovecs].iov_len;
67 iovec_pages_used += num_pages_spanned(&iov[nr_iovecs]);
68 nr_iovecs++;
69 } while (iovec_len_used < len);
70
71 /* single kmalloc for pinned list, page_list[], and the page arrays */
72 local_list = kmalloc(sizeof(*local_list)
73 + (nr_iovecs * sizeof (struct dma_page_list))
74 + (iovec_pages_used * sizeof (struct page*)), GFP_KERNEL);
75 if (!local_list) {
76 err = -ENOMEM;
77 goto out;
78 }
79
80 /* list of pages starts right after the page list array */
81 pages = (struct page **) &local_list->page_list[nr_iovecs];
82
83 for (i = 0; i < nr_iovecs; i++) {
84 struct dma_page_list *page_list = &local_list->page_list[i];
85
86 len -= iov[i].iov_len;
87
88 if (!access_ok(VERIFY_WRITE, iov[i].iov_base, iov[i].iov_len)) {
89 err = -EFAULT;
90 goto unpin;
91 }
92
93 page_list->nr_pages = num_pages_spanned(&iov[i]);
94 page_list->base_address = iov[i].iov_base;
95
96 page_list->pages = pages;
97 pages += page_list->nr_pages;
98
99 /* pin pages down */
100 down_read(&current->mm->mmap_sem);
101 ret = get_user_pages(
102 current,
103 current->mm,
104 (unsigned long) iov[i].iov_base,
105 page_list->nr_pages,
106 1, /* write */
107 0, /* force */
108 page_list->pages,
109 NULL);
110 up_read(&current->mm->mmap_sem);
111
112 if (ret != page_list->nr_pages) {
113 err = -ENOMEM;
114 goto unpin;
115 }
116
117 local_list->nr_iovecs = i + 1;
118 }
119
120 return local_list;
121
122unpin:
123 dma_unpin_iovec_pages(local_list);
124out:
125 return ERR_PTR(err);
126}
127
128void dma_unpin_iovec_pages(struct dma_pinned_list *pinned_list)
129{
130 int i, j;
131
132 if (!pinned_list)
133 return;
134
135 for (i = 0; i < pinned_list->nr_iovecs; i++) {
136 struct dma_page_list *page_list = &pinned_list->page_list[i];
137 for (j = 0; j < page_list->nr_pages; j++) {
138 set_page_dirty_lock(page_list->pages[j]);
139 page_cache_release(page_list->pages[j]);
140 }
141 }
142
143 kfree(pinned_list);
144}
145
146static dma_cookie_t dma_memcpy_to_kernel_iovec(struct dma_chan *chan, struct
147 iovec *iov, unsigned char *kdata, size_t len)
148{
149 dma_cookie_t dma_cookie = 0;
150
151 while (len > 0) {
152 if (iov->iov_len) {
153 int copy = min_t(unsigned int, iov->iov_len, len);
154 dma_cookie = dma_async_memcpy_buf_to_buf(
155 chan,
156 iov->iov_base,
157 kdata,
158 copy);
159 kdata += copy;
160 len -= copy;
161 iov->iov_len -= copy;
162 iov->iov_base += copy;
163 }
164 iov++;
165 }
166
167 return dma_cookie;
168}
169
170/*
171 * We have already pinned down the pages we will be using in the iovecs.
172 * Each entry in iov array has corresponding entry in pinned_list->page_list.
173 * Using array indexing to keep iov[] and page_list[] in sync.
174 * Initial elements in iov array's iov->iov_len will be 0 if already copied into
175 * by another call.
176 * iov array length remaining guaranteed to be bigger than len.
177 */
178dma_cookie_t dma_memcpy_to_iovec(struct dma_chan *chan, struct iovec *iov,
179 struct dma_pinned_list *pinned_list, unsigned char *kdata, size_t len)
180{
181 int iov_byte_offset;
182 int copy;
183 dma_cookie_t dma_cookie = 0;
184 int iovec_idx;
185 int page_idx;
186
187 if (!chan)
188 return memcpy_toiovec(iov, kdata, len);
189
190 /* -> kernel copies (e.g. smbfs) */
191 if (!pinned_list)
192 return dma_memcpy_to_kernel_iovec(chan, iov, kdata, len);
193
194 iovec_idx = 0;
195 while (iovec_idx < pinned_list->nr_iovecs) {
196 struct dma_page_list *page_list;
197
198 /* skip already used-up iovecs */
199 while (!iov[iovec_idx].iov_len)
200 iovec_idx++;
201
202 page_list = &pinned_list->page_list[iovec_idx];
203
204 iov_byte_offset = ((unsigned long)iov[iovec_idx].iov_base & ~PAGE_MASK);
205 page_idx = (((unsigned long)iov[iovec_idx].iov_base & PAGE_MASK)
206 - ((unsigned long)page_list->base_address & PAGE_MASK)) >> PAGE_SHIFT;
207
208 /* break up copies to not cross page boundary */
209 while (iov[iovec_idx].iov_len) {
210 copy = min_t(int, PAGE_SIZE - iov_byte_offset, len);
211 copy = min_t(int, copy, iov[iovec_idx].iov_len);
212
213 dma_cookie = dma_async_memcpy_buf_to_pg(chan,
214 page_list->pages[page_idx],
215 iov_byte_offset,
216 kdata,
217 copy);
218
219 len -= copy;
220 iov[iovec_idx].iov_len -= copy;
221 iov[iovec_idx].iov_base += copy;
222
223 if (!len)
224 return dma_cookie;
225
226 kdata += copy;
227 iov_byte_offset = 0;
228 page_idx++;
229 }
230 iovec_idx++;
231 }
232
233 /* really bad if we ever run out of iovecs */
234 BUG();
235 return -EFAULT;
236}
237
238dma_cookie_t dma_memcpy_pg_to_iovec(struct dma_chan *chan, struct iovec *iov,
239 struct dma_pinned_list *pinned_list, struct page *page,
240 unsigned int offset, size_t len)
241{
242 int iov_byte_offset;
243 int copy;
244 dma_cookie_t dma_cookie = 0;
245 int iovec_idx;
246 int page_idx;
247 int err;
248
249 /* this needs as-yet-unimplemented buf-to-buff, so punt. */
250 /* TODO: use dma for this */
251 if (!chan || !pinned_list) {
252 u8 *vaddr = kmap(page);
253 err = memcpy_toiovec(iov, vaddr + offset, len);
254 kunmap(page);
255 return err;
256 }
257
258 iovec_idx = 0;
259 while (iovec_idx < pinned_list->nr_iovecs) {
260 struct dma_page_list *page_list;
261
262 /* skip already used-up iovecs */
263 while (!iov[iovec_idx].iov_len)
264 iovec_idx++;
265
266 page_list = &pinned_list->page_list[iovec_idx];
267
268 iov_byte_offset = ((unsigned long)iov[iovec_idx].iov_base & ~PAGE_MASK);
269 page_idx = (((unsigned long)iov[iovec_idx].iov_base & PAGE_MASK)
270 - ((unsigned long)page_list->base_address & PAGE_MASK)) >> PAGE_SHIFT;
271
272 /* break up copies to not cross page boundary */
273 while (iov[iovec_idx].iov_len) {
274 copy = min_t(int, PAGE_SIZE - iov_byte_offset, len);
275 copy = min_t(int, copy, iov[iovec_idx].iov_len);
276
277 dma_cookie = dma_async_memcpy_pg_to_pg(chan,
278 page_list->pages[page_idx],
279 iov_byte_offset,
280 page,
281 offset,
282 copy);
283
284 len -= copy;
285 iov[iovec_idx].iov_len -= copy;
286 iov[iovec_idx].iov_base += copy;
287
288 if (!len)
289 return dma_cookie;
290
291 offset += copy;
292 iov_byte_offset = 0;
293 page_idx++;
294 }
295 iovec_idx++;
296 }
297
298 /* really bad if we ever run out of iovecs */
299 BUG();
300 return -EFAULT;
301}
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-25 23:38:52 -0500