aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/dma
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@woody.linux-foundation.org>2007-07-13 13:52:27 -0400
committerLinus Torvalds <torvalds@woody.linux-foundation.org>2007-07-13 13:52:27 -0400
commite030dbf91a87da7e8be3be3ca781558695bea683 (patch)
tree4ff2e01621a888be4098ca48c404775e56a55a0d /drivers/dma
parent12a22960549979c10a95cc97f8ec63b461c55692 (diff)
parent3039f0735a280b54c7364fbfe6a9287f7f0b510a (diff)
Merge branch 'ioat-md-accel-for-linus' of git://lost.foo-projects.org/~dwillia2/git/iop
* 'ioat-md-accel-for-linus' of git://lost.foo-projects.org/~dwillia2/git/iop: (28 commits) ioatdma: add the unisys "i/oat" pci vendor/device id ARM: Add drivers/dma to arch/arm/Kconfig iop3xx: surface the iop3xx DMA and AAU units to the iop-adma driver iop13xx: surface the iop13xx adma units to the iop-adma driver dmaengine: driver for the iop32x, iop33x, and iop13xx raid engines md: remove raid5 compute_block and compute_parity5 md: handle_stripe5 - request io processing in raid5_run_ops md: handle_stripe5 - add request/completion logic for async expand ops md: handle_stripe5 - add request/completion logic for async read ops md: handle_stripe5 - add request/completion logic for async check ops md: handle_stripe5 - add request/completion logic for async compute ops md: handle_stripe5 - add request/completion logic for async write ops md: common infrastructure for running operations with raid5_run_ops md: raid5_run_ops - run stripe operations outside sh->lock raid5: replace custom debug PRINTKs with standard pr_debug raid5: refactor handle_stripe5 and handle_stripe6 (v3) async_tx: add the async_tx api xor: make 'xor_blocks' a library routine for use with async_tx dmaengine: make clients responsible for managing channels dmaengine: refactor dmaengine around dma_async_tx_descriptor ...
Diffstat (limited to 'drivers/dma')
-rw-r--r--drivers/dma/Kconfig12
-rw-r--r--drivers/dma/Makefile1
-rw-r--r--drivers/dma/dmaengine.c419
-rw-r--r--drivers/dma/ioatdma.c369
-rw-r--r--drivers/dma/ioatdma.h16
-rw-r--r--drivers/dma/ioatdma_io.h118
-rw-r--r--drivers/dma/iop-adma.c1467
7 files changed, 1974 insertions, 428 deletions
diff --git a/drivers/dma/Kconfig b/drivers/dma/Kconfig
index 72be6c63edfc..b31756d59978 100644
--- a/drivers/dma/Kconfig
+++ b/drivers/dma/Kconfig
@@ -8,8 +8,8 @@ menu "DMA Engine support"
8config DMA_ENGINE 8config DMA_ENGINE
9 bool "Support for DMA engines" 9 bool "Support for DMA engines"
10 ---help--- 10 ---help---
11 DMA engines offload copy operations from the CPU to dedicated 11 DMA engines offload bulk memory operations from the CPU to dedicated
12 hardware, allowing the copies to happen asynchronously. 12 hardware, allowing the operations to happen asynchronously.
13 13
14comment "DMA Clients" 14comment "DMA Clients"
15 15
@@ -32,4 +32,12 @@ config INTEL_IOATDMA
32 ---help--- 32 ---help---
33 Enable support for the Intel(R) I/OAT DMA engine. 33 Enable support for the Intel(R) I/OAT DMA engine.
34 34
35config INTEL_IOP_ADMA
36 tristate "Intel IOP ADMA support"
37 depends on DMA_ENGINE && (ARCH_IOP32X || ARCH_IOP33X || ARCH_IOP13XX)
38 select ASYNC_CORE
39 default m
40 ---help---
41 Enable support for the Intel(R) IOP Series RAID engines.
42
35endmenu 43endmenu
diff --git a/drivers/dma/Makefile b/drivers/dma/Makefile
index bdcfdbdb1aec..b3839b687ae0 100644
--- a/drivers/dma/Makefile
+++ b/drivers/dma/Makefile
@@ -1,3 +1,4 @@
1obj-$(CONFIG_DMA_ENGINE) += dmaengine.o 1obj-$(CONFIG_DMA_ENGINE) += dmaengine.o
2obj-$(CONFIG_NET_DMA) += iovlock.o 2obj-$(CONFIG_NET_DMA) += iovlock.o
3obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o 3obj-$(CONFIG_INTEL_IOATDMA) += ioatdma.o
4obj-$(CONFIG_INTEL_IOP_ADMA) += iop-adma.o
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index 322ee2984e3d..82489923af09 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -37,11 +37,11 @@
37 * Each device has a channels list, which runs unlocked but is never modified 37 * Each device has a channels list, which runs unlocked but is never modified
38 * once the device is registered, it's just setup by the driver. 38 * once the device is registered, it's just setup by the driver.
39 * 39 *
40 * Each client has a channels list, it's only modified under the client->lock 40 * Each client is responsible for keeping track of the channels it uses. See
41 * and in an RCU callback, so it's safe to read under rcu_read_lock(). 41 * the definition of dma_event_callback in dmaengine.h.
42 * 42 *
43 * Each device has a kref, which is initialized to 1 when the device is 43 * Each device has a kref, which is initialized to 1 when the device is
44 * registered. A kref_put is done for each class_device registered. When the 44 * registered. A kref_get is done for each class_device registered. When the
45 * class_device is released, the coresponding kref_put is done in the release 45 * class_device is released, the coresponding kref_put is done in the release
46 * method. Every time one of the device's channels is allocated to a client, 46 * method. Every time one of the device's channels is allocated to a client,
47 * a kref_get occurs. When the channel is freed, the coresponding kref_put 47 * a kref_get occurs. When the channel is freed, the coresponding kref_put
@@ -51,14 +51,17 @@
51 * references to finish. 51 * references to finish.
52 * 52 *
53 * Each channel has an open-coded implementation of Rusty Russell's "bigref," 53 * Each channel has an open-coded implementation of Rusty Russell's "bigref,"
54 * with a kref and a per_cpu local_t. A single reference is set when on an 54 * with a kref and a per_cpu local_t. A dma_chan_get is called when a client
55 * ADDED event, and removed with a REMOVE event. Net DMA client takes an 55 * signals that it wants to use a channel, and dma_chan_put is called when
56 * extra reference per outstanding transaction. The relase function does a 56 * a channel is removed or a client using it is unregesitered. A client can
57 * kref_put on the device. -ChrisL 57 * take extra references per outstanding transaction, as is the case with
58 * the NET DMA client. The release function does a kref_put on the device.
59 * -ChrisL, DanW
58 */ 60 */
59 61
60#include <linux/init.h> 62#include <linux/init.h>
61#include <linux/module.h> 63#include <linux/module.h>
64#include <linux/mm.h>
62#include <linux/device.h> 65#include <linux/device.h>
63#include <linux/dmaengine.h> 66#include <linux/dmaengine.h>
64#include <linux/hardirq.h> 67#include <linux/hardirq.h>
@@ -66,6 +69,7 @@
66#include <linux/percpu.h> 69#include <linux/percpu.h>
67#include <linux/rcupdate.h> 70#include <linux/rcupdate.h>
68#include <linux/mutex.h> 71#include <linux/mutex.h>
72#include <linux/jiffies.h>
69 73
70static DEFINE_MUTEX(dma_list_mutex); 74static DEFINE_MUTEX(dma_list_mutex);
71static LIST_HEAD(dma_device_list); 75static LIST_HEAD(dma_device_list);
@@ -100,8 +104,19 @@ static ssize_t show_bytes_transferred(struct class_device *cd, char *buf)
100static ssize_t show_in_use(struct class_device *cd, char *buf) 104static ssize_t show_in_use(struct class_device *cd, char *buf)
101{ 105{
102 struct dma_chan *chan = container_of(cd, struct dma_chan, class_dev); 106 struct dma_chan *chan = container_of(cd, struct dma_chan, class_dev);
107 int in_use = 0;
108
109 if (unlikely(chan->slow_ref) &&
110 atomic_read(&chan->refcount.refcount) > 1)
111 in_use = 1;
112 else {
113 if (local_read(&(per_cpu_ptr(chan->local,
114 get_cpu())->refcount)) > 0)
115 in_use = 1;
116 put_cpu();
117 }
103 118
104 return sprintf(buf, "%d\n", (chan->client ? 1 : 0)); 119 return sprintf(buf, "%d\n", in_use);
105} 120}
106 121
107static struct class_device_attribute dma_class_attrs[] = { 122static struct class_device_attribute dma_class_attrs[] = {
@@ -127,43 +142,72 @@ static struct class dma_devclass = {
127 142
128/* --- client and device registration --- */ 143/* --- client and device registration --- */
129 144
145#define dma_chan_satisfies_mask(chan, mask) \
146 __dma_chan_satisfies_mask((chan), &(mask))
147static int
148__dma_chan_satisfies_mask(struct dma_chan *chan, dma_cap_mask_t *want)
149{
150 dma_cap_mask_t has;
151
152 bitmap_and(has.bits, want->bits, chan->device->cap_mask.bits,
153 DMA_TX_TYPE_END);
154 return bitmap_equal(want->bits, has.bits, DMA_TX_TYPE_END);
155}
156
130/** 157/**
131 * dma_client_chan_alloc - try to allocate a channel to a client 158 * dma_client_chan_alloc - try to allocate channels to a client
132 * @client: &dma_client 159 * @client: &dma_client
133 * 160 *
134 * Called with dma_list_mutex held. 161 * Called with dma_list_mutex held.
135 */ 162 */
136static struct dma_chan *dma_client_chan_alloc(struct dma_client *client) 163static void dma_client_chan_alloc(struct dma_client *client)
137{ 164{
138 struct dma_device *device; 165 struct dma_device *device;
139 struct dma_chan *chan; 166 struct dma_chan *chan;
140 unsigned long flags;
141 int desc; /* allocated descriptor count */ 167 int desc; /* allocated descriptor count */
168 enum dma_state_client ack;
142 169
143 /* Find a channel, any DMA engine will do */ 170 /* Find a channel */
144 list_for_each_entry(device, &dma_device_list, global_node) { 171 list_for_each_entry(device, &dma_device_list, global_node)
145 list_for_each_entry(chan, &device->channels, device_node) { 172 list_for_each_entry(chan, &device->channels, device_node) {
146 if (chan->client) 173 if (!dma_chan_satisfies_mask(chan, client->cap_mask))
147 continue; 174 continue;
148 175
149 desc = chan->device->device_alloc_chan_resources(chan); 176 desc = chan->device->device_alloc_chan_resources(chan);
150 if (desc >= 0) { 177 if (desc >= 0) {
151 kref_get(&device->refcount); 178 ack = client->event_callback(client,
152 kref_init(&chan->refcount); 179 chan,
153 chan->slow_ref = 0; 180 DMA_RESOURCE_AVAILABLE);
154 INIT_RCU_HEAD(&chan->rcu); 181
155 chan->client = client; 182 /* we are done once this client rejects
156 spin_lock_irqsave(&client->lock, flags); 183 * an available resource
157 list_add_tail_rcu(&chan->client_node, 184 */
158 &client->channels); 185 if (ack == DMA_ACK) {
159 spin_unlock_irqrestore(&client->lock, flags); 186 dma_chan_get(chan);
160 return chan; 187 kref_get(&device->refcount);
188 } else if (ack == DMA_NAK)
189 return;
161 } 190 }
162 } 191 }
163 } 192}
193
194enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie)
195{
196 enum dma_status status;
197 unsigned long dma_sync_wait_timeout = jiffies + msecs_to_jiffies(5000);
198
199 dma_async_issue_pending(chan);
200 do {
201 status = dma_async_is_tx_complete(chan, cookie, NULL, NULL);
202 if (time_after_eq(jiffies, dma_sync_wait_timeout)) {
203 printk(KERN_ERR "dma_sync_wait_timeout!\n");
204 return DMA_ERROR;
205 }
206 } while (status == DMA_IN_PROGRESS);
164 207
165 return NULL; 208 return status;
166} 209}
210EXPORT_SYMBOL(dma_sync_wait);
167 211
168/** 212/**
169 * dma_chan_cleanup - release a DMA channel's resources 213 * dma_chan_cleanup - release a DMA channel's resources
@@ -173,7 +217,6 @@ void dma_chan_cleanup(struct kref *kref)
173{ 217{
174 struct dma_chan *chan = container_of(kref, struct dma_chan, refcount); 218 struct dma_chan *chan = container_of(kref, struct dma_chan, refcount);
175 chan->device->device_free_chan_resources(chan); 219 chan->device->device_free_chan_resources(chan);
176 chan->client = NULL;
177 kref_put(&chan->device->refcount, dma_async_device_cleanup); 220 kref_put(&chan->device->refcount, dma_async_device_cleanup);
178} 221}
179EXPORT_SYMBOL(dma_chan_cleanup); 222EXPORT_SYMBOL(dma_chan_cleanup);
@@ -189,7 +232,7 @@ static void dma_chan_free_rcu(struct rcu_head *rcu)
189 kref_put(&chan->refcount, dma_chan_cleanup); 232 kref_put(&chan->refcount, dma_chan_cleanup);
190} 233}
191 234
192static void dma_client_chan_free(struct dma_chan *chan) 235static void dma_chan_release(struct dma_chan *chan)
193{ 236{
194 atomic_add(0x7FFFFFFF, &chan->refcount.refcount); 237 atomic_add(0x7FFFFFFF, &chan->refcount.refcount);
195 chan->slow_ref = 1; 238 chan->slow_ref = 1;
@@ -197,70 +240,57 @@ static void dma_client_chan_free(struct dma_chan *chan)
197} 240}
198 241
199/** 242/**
200 * dma_chans_rebalance - reallocate channels to clients 243 * dma_chans_notify_available - broadcast available channels to the clients
201 *
202 * When the number of DMA channel in the system changes,
203 * channels need to be rebalanced among clients.
204 */ 244 */
205static void dma_chans_rebalance(void) 245static void dma_clients_notify_available(void)
206{ 246{
207 struct dma_client *client; 247 struct dma_client *client;
208 struct dma_chan *chan;
209 unsigned long flags;
210 248
211 mutex_lock(&dma_list_mutex); 249 mutex_lock(&dma_list_mutex);
212 250
213 list_for_each_entry(client, &dma_client_list, global_node) { 251 list_for_each_entry(client, &dma_client_list, global_node)
214 while (client->chans_desired > client->chan_count) { 252 dma_client_chan_alloc(client);
215 chan = dma_client_chan_alloc(client);
216 if (!chan)
217 break;
218 client->chan_count++;
219 client->event_callback(client,
220 chan,
221 DMA_RESOURCE_ADDED);
222 }
223 while (client->chans_desired < client->chan_count) {
224 spin_lock_irqsave(&client->lock, flags);
225 chan = list_entry(client->channels.next,
226 struct dma_chan,
227 client_node);
228 list_del_rcu(&chan->client_node);
229 spin_unlock_irqrestore(&client->lock, flags);
230 client->chan_count--;
231 client->event_callback(client,
232 chan,
233 DMA_RESOURCE_REMOVED);
234 dma_client_chan_free(chan);
235 }
236 }
237 253
238 mutex_unlock(&dma_list_mutex); 254 mutex_unlock(&dma_list_mutex);
239} 255}
240 256
241/** 257/**
242 * dma_async_client_register - allocate and register a &dma_client 258 * dma_chans_notify_available - tell the clients that a channel is going away
243 * @event_callback: callback for notification of channel addition/removal 259 * @chan: channel on its way out
244 */ 260 */
245struct dma_client *dma_async_client_register(dma_event_callback event_callback) 261static void dma_clients_notify_removed(struct dma_chan *chan)
246{ 262{
247 struct dma_client *client; 263 struct dma_client *client;
264 enum dma_state_client ack;
248 265
249 client = kzalloc(sizeof(*client), GFP_KERNEL); 266 mutex_lock(&dma_list_mutex);
250 if (!client)
251 return NULL;
252 267
253 INIT_LIST_HEAD(&client->channels); 268 list_for_each_entry(client, &dma_client_list, global_node) {
254 spin_lock_init(&client->lock); 269 ack = client->event_callback(client, chan,
255 client->chans_desired = 0; 270 DMA_RESOURCE_REMOVED);
256 client->chan_count = 0; 271
257 client->event_callback = event_callback; 272 /* client was holding resources for this channel so
273 * free it
274 */
275 if (ack == DMA_ACK) {
276 dma_chan_put(chan);
277 kref_put(&chan->device->refcount,
278 dma_async_device_cleanup);
279 }
280 }
258 281
282 mutex_unlock(&dma_list_mutex);
283}
284
285/**
286 * dma_async_client_register - register a &dma_client
287 * @client: ptr to a client structure with valid 'event_callback' and 'cap_mask'
288 */
289void dma_async_client_register(struct dma_client *client)
290{
259 mutex_lock(&dma_list_mutex); 291 mutex_lock(&dma_list_mutex);
260 list_add_tail(&client->global_node, &dma_client_list); 292 list_add_tail(&client->global_node, &dma_client_list);
261 mutex_unlock(&dma_list_mutex); 293 mutex_unlock(&dma_list_mutex);
262
263 return client;
264} 294}
265EXPORT_SYMBOL(dma_async_client_register); 295EXPORT_SYMBOL(dma_async_client_register);
266 296
@@ -272,40 +302,42 @@ EXPORT_SYMBOL(dma_async_client_register);
272 */ 302 */
273void dma_async_client_unregister(struct dma_client *client) 303void dma_async_client_unregister(struct dma_client *client)
274{ 304{
305 struct dma_device *device;
275 struct dma_chan *chan; 306 struct dma_chan *chan;
307 enum dma_state_client ack;
276 308
277 if (!client) 309 if (!client)
278 return; 310 return;
279 311
280 rcu_read_lock();
281 list_for_each_entry_rcu(chan, &client->channels, client_node)
282 dma_client_chan_free(chan);
283 rcu_read_unlock();
284
285 mutex_lock(&dma_list_mutex); 312 mutex_lock(&dma_list_mutex);
313 /* free all channels the client is holding */
314 list_for_each_entry(device, &dma_device_list, global_node)
315 list_for_each_entry(chan, &device->channels, device_node) {
316 ack = client->event_callback(client, chan,
317 DMA_RESOURCE_REMOVED);
318
319 if (ack == DMA_ACK) {
320 dma_chan_put(chan);
321 kref_put(&chan->device->refcount,
322 dma_async_device_cleanup);
323 }
324 }
325
286 list_del(&client->global_node); 326 list_del(&client->global_node);
287 mutex_unlock(&dma_list_mutex); 327 mutex_unlock(&dma_list_mutex);
288
289 kfree(client);
290 dma_chans_rebalance();
291} 328}
292EXPORT_SYMBOL(dma_async_client_unregister); 329EXPORT_SYMBOL(dma_async_client_unregister);
293 330
294/** 331/**
295 * dma_async_client_chan_request - request DMA channels 332 * dma_async_client_chan_request - send all available channels to the
296 * @client: &dma_client 333 * client that satisfy the capability mask
297 * @number: count of DMA channels requested 334 * @client - requester
298 *
299 * Clients call dma_async_client_chan_request() to specify how many
300 * DMA channels they need, 0 to free all currently allocated.
301 * The resulting allocations/frees are indicated to the client via the
302 * event callback.
303 */ 335 */
304void dma_async_client_chan_request(struct dma_client *client, 336void dma_async_client_chan_request(struct dma_client *client)
305 unsigned int number)
306{ 337{
307 client->chans_desired = number; 338 mutex_lock(&dma_list_mutex);
308 dma_chans_rebalance(); 339 dma_client_chan_alloc(client);
340 mutex_unlock(&dma_list_mutex);
309} 341}
310EXPORT_SYMBOL(dma_async_client_chan_request); 342EXPORT_SYMBOL(dma_async_client_chan_request);
311 343
@@ -316,12 +348,31 @@ EXPORT_SYMBOL(dma_async_client_chan_request);
316int dma_async_device_register(struct dma_device *device) 348int dma_async_device_register(struct dma_device *device)
317{ 349{
318 static int id; 350 static int id;
319 int chancnt = 0; 351 int chancnt = 0, rc;
320 struct dma_chan* chan; 352 struct dma_chan* chan;
321 353
322 if (!device) 354 if (!device)
323 return -ENODEV; 355 return -ENODEV;
324 356
357 /* validate device routines */
358 BUG_ON(dma_has_cap(DMA_MEMCPY, device->cap_mask) &&
359 !device->device_prep_dma_memcpy);
360 BUG_ON(dma_has_cap(DMA_XOR, device->cap_mask) &&
361 !device->device_prep_dma_xor);
362 BUG_ON(dma_has_cap(DMA_ZERO_SUM, device->cap_mask) &&
363 !device->device_prep_dma_zero_sum);
364 BUG_ON(dma_has_cap(DMA_MEMSET, device->cap_mask) &&
365 !device->device_prep_dma_memset);
366 BUG_ON(dma_has_cap(DMA_ZERO_SUM, device->cap_mask) &&
367 !device->device_prep_dma_interrupt);
368
369 BUG_ON(!device->device_alloc_chan_resources);
370 BUG_ON(!device->device_free_chan_resources);
371 BUG_ON(!device->device_dependency_added);
372 BUG_ON(!device->device_is_tx_complete);
373 BUG_ON(!device->device_issue_pending);
374 BUG_ON(!device->dev);
375
325 init_completion(&device->done); 376 init_completion(&device->done);
326 kref_init(&device->refcount); 377 kref_init(&device->refcount);
327 device->dev_id = id++; 378 device->dev_id = id++;
@@ -338,17 +389,38 @@ int dma_async_device_register(struct dma_device *device)
338 snprintf(chan->class_dev.class_id, BUS_ID_SIZE, "dma%dchan%d", 389 snprintf(chan->class_dev.class_id, BUS_ID_SIZE, "dma%dchan%d",
339 device->dev_id, chan->chan_id); 390 device->dev_id, chan->chan_id);
340 391
392 rc = class_device_register(&chan->class_dev);
393 if (rc) {
394 chancnt--;
395 free_percpu(chan->local);
396 chan->local = NULL;
397 goto err_out;
398 }
399
341 kref_get(&device->refcount); 400 kref_get(&device->refcount);
342 class_device_register(&chan->class_dev); 401 kref_init(&chan->refcount);
402 chan->slow_ref = 0;
403 INIT_RCU_HEAD(&chan->rcu);
343 } 404 }
344 405
345 mutex_lock(&dma_list_mutex); 406 mutex_lock(&dma_list_mutex);
346 list_add_tail(&device->global_node, &dma_device_list); 407 list_add_tail(&device->global_node, &dma_device_list);
347 mutex_unlock(&dma_list_mutex); 408 mutex_unlock(&dma_list_mutex);
348 409
349 dma_chans_rebalance(); 410 dma_clients_notify_available();
350 411
351 return 0; 412 return 0;
413
414err_out:
415 list_for_each_entry(chan, &device->channels, device_node) {
416 if (chan->local == NULL)
417 continue;
418 kref_put(&device->refcount, dma_async_device_cleanup);
419 class_device_unregister(&chan->class_dev);
420 chancnt--;
421 free_percpu(chan->local);
422 }
423 return rc;
352} 424}
353EXPORT_SYMBOL(dma_async_device_register); 425EXPORT_SYMBOL(dma_async_device_register);
354 426
@@ -371,32 +443,165 @@ static void dma_async_device_cleanup(struct kref *kref)
371void dma_async_device_unregister(struct dma_device *device) 443void dma_async_device_unregister(struct dma_device *device)
372{ 444{
373 struct dma_chan *chan; 445 struct dma_chan *chan;
374 unsigned long flags;
375 446
376 mutex_lock(&dma_list_mutex); 447 mutex_lock(&dma_list_mutex);
377 list_del(&device->global_node); 448 list_del(&device->global_node);
378 mutex_unlock(&dma_list_mutex); 449 mutex_unlock(&dma_list_mutex);
379 450
380 list_for_each_entry(chan, &device->channels, device_node) { 451 list_for_each_entry(chan, &device->channels, device_node) {
381 if (chan->client) { 452 dma_clients_notify_removed(chan);
382 spin_lock_irqsave(&chan->client->lock, flags);
383 list_del(&chan->client_node);
384 chan->client->chan_count--;
385 spin_unlock_irqrestore(&chan->client->lock, flags);
386 chan->client->event_callback(chan->client,
387 chan,
388 DMA_RESOURCE_REMOVED);
389 dma_client_chan_free(chan);
390 }
391 class_device_unregister(&chan->class_dev); 453 class_device_unregister(&chan->class_dev);
454 dma_chan_release(chan);
392 } 455 }
393 dma_chans_rebalance();
394 456
395 kref_put(&device->refcount, dma_async_device_cleanup); 457 kref_put(&device->refcount, dma_async_device_cleanup);
396 wait_for_completion(&device->done); 458 wait_for_completion(&device->done);
397} 459}
398EXPORT_SYMBOL(dma_async_device_unregister); 460EXPORT_SYMBOL(dma_async_device_unregister);
399 461
462/**
463 * dma_async_memcpy_buf_to_buf - offloaded copy between virtual addresses
464 * @chan: DMA channel to offload copy to
465 * @dest: destination address (virtual)
466 * @src: source address (virtual)
467 * @len: length
468 *
469 * Both @dest and @src must be mappable to a bus address according to the
470 * DMA mapping API rules for streaming mappings.
471 * Both @dest and @src must stay memory resident (kernel memory or locked
472 * user space pages).
473 */
474dma_cookie_t
475dma_async_memcpy_buf_to_buf(struct dma_chan *chan, void *dest,
476 void *src, size_t len)
477{
478 struct dma_device *dev = chan->device;
479 struct dma_async_tx_descriptor *tx;
480 dma_addr_t addr;
481 dma_cookie_t cookie;
482 int cpu;
483
484 tx = dev->device_prep_dma_memcpy(chan, len, 0);
485 if (!tx)
486 return -ENOMEM;
487
488 tx->ack = 1;
489 tx->callback = NULL;
490 addr = dma_map_single(dev->dev, src, len, DMA_TO_DEVICE);
491 tx->tx_set_src(addr, tx, 0);
492 addr = dma_map_single(dev->dev, dest, len, DMA_FROM_DEVICE);
493 tx->tx_set_dest(addr, tx, 0);
494 cookie = tx->tx_submit(tx);
495
496 cpu = get_cpu();
497 per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
498 per_cpu_ptr(chan->local, cpu)->memcpy_count++;
499 put_cpu();
500
501 return cookie;
502}
503EXPORT_SYMBOL(dma_async_memcpy_buf_to_buf);
504
505/**
506 * dma_async_memcpy_buf_to_pg - offloaded copy from address to page
507 * @chan: DMA channel to offload copy to
508 * @page: destination page
509 * @offset: offset in page to copy to
510 * @kdata: source address (virtual)
511 * @len: length
512 *
513 * Both @page/@offset and @kdata must be mappable to a bus address according
514 * to the DMA mapping API rules for streaming mappings.
515 * Both @page/@offset and @kdata must stay memory resident (kernel memory or
516 * locked user space pages)
517 */
518dma_cookie_t
519dma_async_memcpy_buf_to_pg(struct dma_chan *chan, struct page *page,
520 unsigned int offset, void *kdata, size_t len)
521{
522 struct dma_device *dev = chan->device;
523 struct dma_async_tx_descriptor *tx;
524 dma_addr_t addr;
525 dma_cookie_t cookie;
526 int cpu;
527
528 tx = dev->device_prep_dma_memcpy(chan, len, 0);
529 if (!tx)
530 return -ENOMEM;
531
532 tx->ack = 1;
533 tx->callback = NULL;
534 addr = dma_map_single(dev->dev, kdata, len, DMA_TO_DEVICE);
535 tx->tx_set_src(addr, tx, 0);
536 addr = dma_map_page(dev->dev, page, offset, len, DMA_FROM_DEVICE);
537 tx->tx_set_dest(addr, tx, 0);
538 cookie = tx->tx_submit(tx);
539
540 cpu = get_cpu();
541 per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
542 per_cpu_ptr(chan->local, cpu)->memcpy_count++;
543 put_cpu();
544
545 return cookie;
546}
547EXPORT_SYMBOL(dma_async_memcpy_buf_to_pg);
548
549/**
550 * dma_async_memcpy_pg_to_pg - offloaded copy from page to page
551 * @chan: DMA channel to offload copy to
552 * @dest_pg: destination page
553 * @dest_off: offset in page to copy to
554 * @src_pg: source page
555 * @src_off: offset in page to copy from
556 * @len: length
557 *
558 * Both @dest_page/@dest_off and @src_page/@src_off must be mappable to a bus
559 * address according to the DMA mapping API rules for streaming mappings.
560 * Both @dest_page/@dest_off and @src_page/@src_off must stay memory resident
561 * (kernel memory or locked user space pages).
562 */
563dma_cookie_t
564dma_async_memcpy_pg_to_pg(struct dma_chan *chan, struct page *dest_pg,
565 unsigned int dest_off, struct page *src_pg, unsigned int src_off,
566 size_t len)
567{
568 struct dma_device *dev = chan->device;
569 struct dma_async_tx_descriptor *tx;
570 dma_addr_t addr;
571 dma_cookie_t cookie;
572 int cpu;
573
574 tx = dev->device_prep_dma_memcpy(chan, len, 0);
575 if (!tx)
576 return -ENOMEM;
577
578 tx->ack = 1;
579 tx->callback = NULL;
580 addr = dma_map_page(dev->dev, src_pg, src_off, len, DMA_TO_DEVICE);
581 tx->tx_set_src(addr, tx, 0);
582 addr = dma_map_page(dev->dev, dest_pg, dest_off, len, DMA_FROM_DEVICE);
583 tx->tx_set_dest(addr, tx, 0);
584 cookie = tx->tx_submit(tx);
585
586 cpu = get_cpu();
587 per_cpu_ptr(chan->local, cpu)->bytes_transferred += len;
588 per_cpu_ptr(chan->local, cpu)->memcpy_count++;
589 put_cpu();
590
591 return cookie;
592}
593EXPORT_SYMBOL(dma_async_memcpy_pg_to_pg);
594
595void dma_async_tx_descriptor_init(struct dma_async_tx_descriptor *tx,
596 struct dma_chan *chan)
597{
598 tx->chan = chan;
599 spin_lock_init(&tx->lock);
600 INIT_LIST_HEAD(&tx->depend_node);
601 INIT_LIST_HEAD(&tx->depend_list);
602}
603EXPORT_SYMBOL(dma_async_tx_descriptor_init);
604
400static int __init dma_bus_init(void) 605static int __init dma_bus_init(void)
401{ 606{
402 mutex_init(&dma_list_mutex); 607 mutex_init(&dma_list_mutex);
diff --git a/drivers/dma/ioatdma.c b/drivers/dma/ioatdma.c
index 850014139556..5fbe56b5cea0 100644
--- a/drivers/dma/ioatdma.c
+++ b/drivers/dma/ioatdma.c
@@ -32,16 +32,17 @@
32#include <linux/delay.h> 32#include <linux/delay.h>
33#include <linux/dma-mapping.h> 33#include <linux/dma-mapping.h>
34#include "ioatdma.h" 34#include "ioatdma.h"
35#include "ioatdma_io.h"
36#include "ioatdma_registers.h" 35#include "ioatdma_registers.h"
37#include "ioatdma_hw.h" 36#include "ioatdma_hw.h"
38 37
39#define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common) 38#define to_ioat_chan(chan) container_of(chan, struct ioat_dma_chan, common)
40#define to_ioat_device(dev) container_of(dev, struct ioat_device, common) 39#define to_ioat_device(dev) container_of(dev, struct ioat_device, common)
41#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node) 40#define to_ioat_desc(lh) container_of(lh, struct ioat_desc_sw, node)
41#define tx_to_ioat_desc(tx) container_of(tx, struct ioat_desc_sw, async_tx)
42 42
43/* internal functions */ 43/* internal functions */
44static int __devinit ioat_probe(struct pci_dev *pdev, const struct pci_device_id *ent); 44static int __devinit ioat_probe(struct pci_dev *pdev, const struct pci_device_id *ent);
45static void ioat_shutdown(struct pci_dev *pdev);
45static void __devexit ioat_remove(struct pci_dev *pdev); 46static void __devexit ioat_remove(struct pci_dev *pdev);
46 47
47static int enumerate_dma_channels(struct ioat_device *device) 48static int enumerate_dma_channels(struct ioat_device *device)
@@ -51,8 +52,8 @@ static int enumerate_dma_channels(struct ioat_device *device)
51 int i; 52 int i;
52 struct ioat_dma_chan *ioat_chan; 53 struct ioat_dma_chan *ioat_chan;
53 54
54 device->common.chancnt = ioatdma_read8(device, IOAT_CHANCNT_OFFSET); 55 device->common.chancnt = readb(device->reg_base + IOAT_CHANCNT_OFFSET);
55 xfercap_scale = ioatdma_read8(device, IOAT_XFERCAP_OFFSET); 56 xfercap_scale = readb(device->reg_base + IOAT_XFERCAP_OFFSET);
56 xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale)); 57 xfercap = (xfercap_scale == 0 ? -1 : (1UL << xfercap_scale));
57 58
58 for (i = 0; i < device->common.chancnt; i++) { 59 for (i = 0; i < device->common.chancnt; i++) {
@@ -71,13 +72,79 @@ static int enumerate_dma_channels(struct ioat_device *device)
71 INIT_LIST_HEAD(&ioat_chan->used_desc); 72 INIT_LIST_HEAD(&ioat_chan->used_desc);
72 /* This should be made common somewhere in dmaengine.c */ 73 /* This should be made common somewhere in dmaengine.c */
73 ioat_chan->common.device = &device->common; 74 ioat_chan->common.device = &device->common;
74 ioat_chan->common.client = NULL;
75 list_add_tail(&ioat_chan->common.device_node, 75 list_add_tail(&ioat_chan->common.device_node,
76 &device->common.channels); 76 &device->common.channels);
77 } 77 }
78 return device->common.chancnt; 78 return device->common.chancnt;
79} 79}
80 80
81static void
82ioat_set_src(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index)
83{
84 struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx);
85 struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
86
87 pci_unmap_addr_set(desc, src, addr);
88
89 list_for_each_entry(iter, &desc->async_tx.tx_list, node) {
90 iter->hw->src_addr = addr;
91 addr += ioat_chan->xfercap;
92 }
93
94}
95
96static void
97ioat_set_dest(dma_addr_t addr, struct dma_async_tx_descriptor *tx, int index)
98{
99 struct ioat_desc_sw *iter, *desc = tx_to_ioat_desc(tx);
100 struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
101
102 pci_unmap_addr_set(desc, dst, addr);
103
104 list_for_each_entry(iter, &desc->async_tx.tx_list, node) {
105 iter->hw->dst_addr = addr;
106 addr += ioat_chan->xfercap;
107 }
108}
109
110static dma_cookie_t
111ioat_tx_submit(struct dma_async_tx_descriptor *tx)
112{
113 struct ioat_dma_chan *ioat_chan = to_ioat_chan(tx->chan);
114 struct ioat_desc_sw *desc = tx_to_ioat_desc(tx);
115 int append = 0;
116 dma_cookie_t cookie;
117 struct ioat_desc_sw *group_start;
118
119 group_start = list_entry(desc->async_tx.tx_list.next,
120 struct ioat_desc_sw, node);
121 spin_lock_bh(&ioat_chan->desc_lock);
122 /* cookie incr and addition to used_list must be atomic */
123 cookie = ioat_chan->common.cookie;
124 cookie++;
125 if (cookie < 0)
126 cookie = 1;
127 ioat_chan->common.cookie = desc->async_tx.cookie = cookie;
128
129 /* write address into NextDescriptor field of last desc in chain */
130 to_ioat_desc(ioat_chan->used_desc.prev)->hw->next =
131 group_start->async_tx.phys;
132 list_splice_init(&desc->async_tx.tx_list, ioat_chan->used_desc.prev);
133
134 ioat_chan->pending += desc->tx_cnt;
135 if (ioat_chan->pending >= 4) {
136 append = 1;
137 ioat_chan->pending = 0;
138 }
139 spin_unlock_bh(&ioat_chan->desc_lock);
140
141 if (append)
142 writeb(IOAT_CHANCMD_APPEND,
143 ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
144
145 return cookie;
146}
147
81static struct ioat_desc_sw *ioat_dma_alloc_descriptor( 148static struct ioat_desc_sw *ioat_dma_alloc_descriptor(
82 struct ioat_dma_chan *ioat_chan, 149 struct ioat_dma_chan *ioat_chan,
83 gfp_t flags) 150 gfp_t flags)
@@ -99,8 +166,13 @@ static struct ioat_desc_sw *ioat_dma_alloc_descriptor(
99 } 166 }
100 167
101 memset(desc, 0, sizeof(*desc)); 168 memset(desc, 0, sizeof(*desc));
169 dma_async_tx_descriptor_init(&desc_sw->async_tx, &ioat_chan->common);
170 desc_sw->async_tx.tx_set_src = ioat_set_src;
171 desc_sw->async_tx.tx_set_dest = ioat_set_dest;
172 desc_sw->async_tx.tx_submit = ioat_tx_submit;
173 INIT_LIST_HEAD(&desc_sw->async_tx.tx_list);
102 desc_sw->hw = desc; 174 desc_sw->hw = desc;
103 desc_sw->phys = phys; 175 desc_sw->async_tx.phys = phys;
104 176
105 return desc_sw; 177 return desc_sw;
106} 178}
@@ -123,7 +195,7 @@ static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)
123 * In-use bit automatically set by reading chanctrl 195 * In-use bit automatically set by reading chanctrl
124 * If 0, we got it, if 1, someone else did 196 * If 0, we got it, if 1, someone else did
125 */ 197 */
126 chanctrl = ioatdma_chan_read16(ioat_chan, IOAT_CHANCTRL_OFFSET); 198 chanctrl = readw(ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
127 if (chanctrl & IOAT_CHANCTRL_CHANNEL_IN_USE) 199 if (chanctrl & IOAT_CHANCTRL_CHANNEL_IN_USE)
128 return -EBUSY; 200 return -EBUSY;
129 201
@@ -132,12 +204,12 @@ static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)
132 IOAT_CHANCTRL_ERR_INT_EN | 204 IOAT_CHANCTRL_ERR_INT_EN |
133 IOAT_CHANCTRL_ANY_ERR_ABORT_EN | 205 IOAT_CHANCTRL_ANY_ERR_ABORT_EN |
134 IOAT_CHANCTRL_ERR_COMPLETION_EN; 206 IOAT_CHANCTRL_ERR_COMPLETION_EN;
135 ioatdma_chan_write16(ioat_chan, IOAT_CHANCTRL_OFFSET, chanctrl); 207 writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
136 208
137 chanerr = ioatdma_chan_read32(ioat_chan, IOAT_CHANERR_OFFSET); 209 chanerr = readl(ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
138 if (chanerr) { 210 if (chanerr) {
139 printk("IOAT: CHANERR = %x, clearing\n", chanerr); 211 printk("IOAT: CHANERR = %x, clearing\n", chanerr);
140 ioatdma_chan_write32(ioat_chan, IOAT_CHANERR_OFFSET, chanerr); 212 writel(chanerr, ioat_chan->reg_base + IOAT_CHANERR_OFFSET);
141 } 213 }
142 214
143 /* Allocate descriptors */ 215 /* Allocate descriptors */
@@ -161,10 +233,10 @@ static int ioat_dma_alloc_chan_resources(struct dma_chan *chan)
161 &ioat_chan->completion_addr); 233 &ioat_chan->completion_addr);
162 memset(ioat_chan->completion_virt, 0, 234 memset(ioat_chan->completion_virt, 0,
163 sizeof(*ioat_chan->completion_virt)); 235 sizeof(*ioat_chan->completion_virt));
164 ioatdma_chan_write32(ioat_chan, IOAT_CHANCMP_OFFSET_LOW, 236 writel(((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF,
165 ((u64) ioat_chan->completion_addr) & 0x00000000FFFFFFFF); 237 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_LOW);
166 ioatdma_chan_write32(ioat_chan, IOAT_CHANCMP_OFFSET_HIGH, 238 writel(((u64) ioat_chan->completion_addr) >> 32,
167 ((u64) ioat_chan->completion_addr) >> 32); 239 ioat_chan->reg_base + IOAT_CHANCMP_OFFSET_HIGH);
168 240
169 ioat_start_null_desc(ioat_chan); 241 ioat_start_null_desc(ioat_chan);
170 return i; 242 return i;
@@ -182,18 +254,20 @@ static void ioat_dma_free_chan_resources(struct dma_chan *chan)
182 254
183 ioat_dma_memcpy_cleanup(ioat_chan); 255 ioat_dma_memcpy_cleanup(ioat_chan);
184 256
185 ioatdma_chan_write8(ioat_chan, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_RESET); 257 writeb(IOAT_CHANCMD_RESET, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
186 258
187 spin_lock_bh(&ioat_chan->desc_lock); 259 spin_lock_bh(&ioat_chan->desc_lock);
188 list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) { 260 list_for_each_entry_safe(desc, _desc, &ioat_chan->used_desc, node) {
189 in_use_descs++; 261 in_use_descs++;
190 list_del(&desc->node); 262 list_del(&desc->node);
191 pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys); 263 pci_pool_free(ioat_device->dma_pool, desc->hw,
264 desc->async_tx.phys);
192 kfree(desc); 265 kfree(desc);
193 } 266 }
194 list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) { 267 list_for_each_entry_safe(desc, _desc, &ioat_chan->free_desc, node) {
195 list_del(&desc->node); 268 list_del(&desc->node);
196 pci_pool_free(ioat_device->dma_pool, desc->hw, desc->phys); 269 pci_pool_free(ioat_device->dma_pool, desc->hw,
270 desc->async_tx.phys);
197 kfree(desc); 271 kfree(desc);
198 } 272 }
199 spin_unlock_bh(&ioat_chan->desc_lock); 273 spin_unlock_bh(&ioat_chan->desc_lock);
@@ -210,50 +284,30 @@ static void ioat_dma_free_chan_resources(struct dma_chan *chan)
210 ioat_chan->last_completion = ioat_chan->completion_addr = 0; 284 ioat_chan->last_completion = ioat_chan->completion_addr = 0;
211 285
212 /* Tell hw the chan is free */ 286 /* Tell hw the chan is free */
213 chanctrl = ioatdma_chan_read16(ioat_chan, IOAT_CHANCTRL_OFFSET); 287 chanctrl = readw(ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
214 chanctrl &= ~IOAT_CHANCTRL_CHANNEL_IN_USE; 288 chanctrl &= ~IOAT_CHANCTRL_CHANNEL_IN_USE;
215 ioatdma_chan_write16(ioat_chan, IOAT_CHANCTRL_OFFSET, chanctrl); 289 writew(chanctrl, ioat_chan->reg_base + IOAT_CHANCTRL_OFFSET);
216} 290}
217 291
218/** 292static struct dma_async_tx_descriptor *
219 * do_ioat_dma_memcpy - actual function that initiates a IOAT DMA transaction 293ioat_dma_prep_memcpy(struct dma_chan *chan, size_t len, int int_en)
220 * @ioat_chan: IOAT DMA channel handle
221 * @dest: DMA destination address
222 * @src: DMA source address
223 * @len: transaction length in bytes
224 */
225
226static dma_cookie_t do_ioat_dma_memcpy(struct ioat_dma_chan *ioat_chan,
227 dma_addr_t dest,
228 dma_addr_t src,
229 size_t len)
230{ 294{
231 struct ioat_desc_sw *first; 295 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
232 struct ioat_desc_sw *prev; 296 struct ioat_desc_sw *first, *prev, *new;
233 struct ioat_desc_sw *new;
234 dma_cookie_t cookie;
235 LIST_HEAD(new_chain); 297 LIST_HEAD(new_chain);
236 u32 copy; 298 u32 copy;
237 size_t orig_len; 299 size_t orig_len;
238 dma_addr_t orig_src, orig_dst; 300 int desc_count = 0;
239 unsigned int desc_count = 0;
240 unsigned int append = 0;
241
242 if (!ioat_chan || !dest || !src)
243 return -EFAULT;
244 301
245 if (!len) 302 if (!len)
246 return ioat_chan->common.cookie; 303 return NULL;
247 304
248 orig_len = len; 305 orig_len = len;
249 orig_src = src;
250 orig_dst = dest;
251 306
252 first = NULL; 307 first = NULL;
253 prev = NULL; 308 prev = NULL;
254 309
255 spin_lock_bh(&ioat_chan->desc_lock); 310 spin_lock_bh(&ioat_chan->desc_lock);
256
257 while (len) { 311 while (len) {
258 if (!list_empty(&ioat_chan->free_desc)) { 312 if (!list_empty(&ioat_chan->free_desc)) {
259 new = to_ioat_desc(ioat_chan->free_desc.next); 313 new = to_ioat_desc(ioat_chan->free_desc.next);
@@ -270,141 +324,36 @@ static dma_cookie_t do_ioat_dma_memcpy(struct ioat_dma_chan *ioat_chan,
270 324
271 new->hw->size = copy; 325 new->hw->size = copy;
272 new->hw->ctl = 0; 326 new->hw->ctl = 0;
273 new->hw->src_addr = src; 327 new->async_tx.cookie = 0;
274 new->hw->dst_addr = dest; 328 new->async_tx.ack = 1;
275 new->cookie = 0;
276 329
277 /* chain together the physical address list for the HW */ 330 /* chain together the physical address list for the HW */
278 if (!first) 331 if (!first)
279 first = new; 332 first = new;
280 else 333 else
281 prev->hw->next = (u64) new->phys; 334 prev->hw->next = (u64) new->async_tx.phys;
282 335
283 prev = new; 336 prev = new;
284
285 len -= copy; 337 len -= copy;
286 dest += copy;
287 src += copy;
288
289 list_add_tail(&new->node, &new_chain); 338 list_add_tail(&new->node, &new_chain);
290 desc_count++; 339 desc_count++;
291 } 340 }
292 new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
293 new->hw->next = 0;
294 341
295 /* cookie incr and addition to used_list must be atomic */ 342 list_splice(&new_chain, &new->async_tx.tx_list);
296 343
297 cookie = ioat_chan->common.cookie; 344 new->hw->ctl = IOAT_DMA_DESCRIPTOR_CTL_CP_STS;
298 cookie++; 345 new->hw->next = 0;
299 if (cookie < 0) 346 new->tx_cnt = desc_count;
300 cookie = 1; 347 new->async_tx.ack = 0; /* client is in control of this ack */
301 ioat_chan->common.cookie = new->cookie = cookie; 348 new->async_tx.cookie = -EBUSY;
302 349
303 pci_unmap_addr_set(new, src, orig_src);
304 pci_unmap_addr_set(new, dst, orig_dst);
305 pci_unmap_len_set(new, src_len, orig_len); 350 pci_unmap_len_set(new, src_len, orig_len);
306 pci_unmap_len_set(new, dst_len, orig_len); 351 pci_unmap_len_set(new, dst_len, orig_len);
307
308 /* write address into NextDescriptor field of last desc in chain */
309 to_ioat_desc(ioat_chan->used_desc.prev)->hw->next = first->phys;
310 list_splice_init(&new_chain, ioat_chan->used_desc.prev);
311
312 ioat_chan->pending += desc_count;
313 if (ioat_chan->pending >= 20) {
314 append = 1;
315 ioat_chan->pending = 0;
316 }
317
318 spin_unlock_bh(&ioat_chan->desc_lock); 352 spin_unlock_bh(&ioat_chan->desc_lock);
319 353
320 if (append) 354 return new ? &new->async_tx : NULL;
321 ioatdma_chan_write8(ioat_chan,
322 IOAT_CHANCMD_OFFSET,
323 IOAT_CHANCMD_APPEND);
324 return cookie;
325}
326
327/**
328 * ioat_dma_memcpy_buf_to_buf - wrapper that takes src & dest bufs
329 * @chan: IOAT DMA channel handle
330 * @dest: DMA destination address
331 * @src: DMA source address
332 * @len: transaction length in bytes
333 */
334
335static dma_cookie_t ioat_dma_memcpy_buf_to_buf(struct dma_chan *chan,
336 void *dest,
337 void *src,
338 size_t len)
339{
340 dma_addr_t dest_addr;
341 dma_addr_t src_addr;
342 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
343
344 dest_addr = pci_map_single(ioat_chan->device->pdev,
345 dest, len, PCI_DMA_FROMDEVICE);
346 src_addr = pci_map_single(ioat_chan->device->pdev,
347 src, len, PCI_DMA_TODEVICE);
348
349 return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
350} 355}
351 356
352/**
353 * ioat_dma_memcpy_buf_to_pg - wrapper, copying from a buf to a page
354 * @chan: IOAT DMA channel handle
355 * @page: pointer to the page to copy to
356 * @offset: offset into that page
357 * @src: DMA source address
358 * @len: transaction length in bytes
359 */
360
361static dma_cookie_t ioat_dma_memcpy_buf_to_pg(struct dma_chan *chan,
362 struct page *page,
363 unsigned int offset,
364 void *src,
365 size_t len)
366{
367 dma_addr_t dest_addr;
368 dma_addr_t src_addr;
369 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
370
371 dest_addr = pci_map_page(ioat_chan->device->pdev,
372 page, offset, len, PCI_DMA_FROMDEVICE);
373 src_addr = pci_map_single(ioat_chan->device->pdev,
374 src, len, PCI_DMA_TODEVICE);
375
376 return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
377}
378
379/**
380 * ioat_dma_memcpy_pg_to_pg - wrapper, copying between two pages
381 * @chan: IOAT DMA channel handle
382 * @dest_pg: pointer to the page to copy to
383 * @dest_off: offset into that page
384 * @src_pg: pointer to the page to copy from
385 * @src_off: offset into that page
386 * @len: transaction length in bytes. This is guaranteed not to make a copy
387 * across a page boundary.
388 */
389
390static dma_cookie_t ioat_dma_memcpy_pg_to_pg(struct dma_chan *chan,
391 struct page *dest_pg,
392 unsigned int dest_off,
393 struct page *src_pg,
394 unsigned int src_off,
395 size_t len)
396{
397 dma_addr_t dest_addr;
398 dma_addr_t src_addr;
399 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
400
401 dest_addr = pci_map_page(ioat_chan->device->pdev,
402 dest_pg, dest_off, len, PCI_DMA_FROMDEVICE);
403 src_addr = pci_map_page(ioat_chan->device->pdev,
404 src_pg, src_off, len, PCI_DMA_TODEVICE);
405
406 return do_ioat_dma_memcpy(ioat_chan, dest_addr, src_addr, len);
407}
408 357
409/** 358/**
410 * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended descriptors to hw 359 * ioat_dma_memcpy_issue_pending - push potentially unrecognized appended descriptors to hw
@@ -417,9 +366,8 @@ static void ioat_dma_memcpy_issue_pending(struct dma_chan *chan)
417 366
418 if (ioat_chan->pending != 0) { 367 if (ioat_chan->pending != 0) {
419 ioat_chan->pending = 0; 368 ioat_chan->pending = 0;
420 ioatdma_chan_write8(ioat_chan, 369 writeb(IOAT_CHANCMD_APPEND,
421 IOAT_CHANCMD_OFFSET, 370 ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
422 IOAT_CHANCMD_APPEND);
423 } 371 }
424} 372}
425 373
@@ -449,7 +397,7 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan)
449 if ((chan->completion_virt->full & IOAT_CHANSTS_DMA_TRANSFER_STATUS) == 397 if ((chan->completion_virt->full & IOAT_CHANSTS_DMA_TRANSFER_STATUS) ==
450 IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) { 398 IOAT_CHANSTS_DMA_TRANSFER_STATUS_HALTED) {
451 printk("IOAT: Channel halted, chanerr = %x\n", 399 printk("IOAT: Channel halted, chanerr = %x\n",
452 ioatdma_chan_read32(chan, IOAT_CHANERR_OFFSET)); 400 readl(chan->reg_base + IOAT_CHANERR_OFFSET));
453 401
454 /* TODO do something to salvage the situation */ 402 /* TODO do something to salvage the situation */
455 } 403 }
@@ -467,8 +415,8 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan)
467 * exceeding xfercap, perhaps. If so, only the last one will 415 * exceeding xfercap, perhaps. If so, only the last one will
468 * have a cookie, and require unmapping. 416 * have a cookie, and require unmapping.
469 */ 417 */
470 if (desc->cookie) { 418 if (desc->async_tx.cookie) {
471 cookie = desc->cookie; 419 cookie = desc->async_tx.cookie;
472 420
473 /* yes we are unmapping both _page and _single alloc'd 421 /* yes we are unmapping both _page and _single alloc'd
474 regions with unmap_page. Is this *really* that bad? 422 regions with unmap_page. Is this *really* that bad?
@@ -483,14 +431,19 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan)
483 PCI_DMA_TODEVICE); 431 PCI_DMA_TODEVICE);
484 } 432 }
485 433
486 if (desc->phys != phys_complete) { 434 if (desc->async_tx.phys != phys_complete) {
487 /* a completed entry, but not the last, so cleanup */ 435 /* a completed entry, but not the last, so cleanup
488 list_del(&desc->node); 436 * if the client is done with the descriptor
489 list_add_tail(&desc->node, &chan->free_desc); 437 */
438 if (desc->async_tx.ack) {
439 list_del(&desc->node);
440 list_add_tail(&desc->node, &chan->free_desc);
441 } else
442 desc->async_tx.cookie = 0;
490 } else { 443 } else {
491 /* last used desc. Do not remove, so we can append from 444 /* last used desc. Do not remove, so we can append from
492 it, but don't look at it next time, either */ 445 it, but don't look at it next time, either */
493 desc->cookie = 0; 446 desc->async_tx.cookie = 0;
494 447
495 /* TODO check status bits? */ 448 /* TODO check status bits? */
496 break; 449 break;
@@ -506,6 +459,17 @@ static void ioat_dma_memcpy_cleanup(struct ioat_dma_chan *chan)
506 spin_unlock(&chan->cleanup_lock); 459 spin_unlock(&chan->cleanup_lock);
507} 460}
508 461
462static void ioat_dma_dependency_added(struct dma_chan *chan)
463{
464 struct ioat_dma_chan *ioat_chan = to_ioat_chan(chan);
465 spin_lock_bh(&ioat_chan->desc_lock);
466 if (ioat_chan->pending == 0) {
467 spin_unlock_bh(&ioat_chan->desc_lock);
468 ioat_dma_memcpy_cleanup(ioat_chan);
469 } else
470 spin_unlock_bh(&ioat_chan->desc_lock);
471}
472
509/** 473/**
510 * ioat_dma_is_complete - poll the status of a IOAT DMA transaction 474 * ioat_dma_is_complete - poll the status of a IOAT DMA transaction
511 * @chan: IOAT DMA channel handle 475 * @chan: IOAT DMA channel handle
@@ -553,6 +517,8 @@ static enum dma_status ioat_dma_is_complete(struct dma_chan *chan,
553 517
554static struct pci_device_id ioat_pci_tbl[] = { 518static struct pci_device_id ioat_pci_tbl[] = {
555 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT) }, 519 { PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT) },
520 { PCI_DEVICE(PCI_VENDOR_ID_UNISYS,
521 PCI_DEVICE_ID_UNISYS_DMA_DIRECTOR) },
556 { 0, } 522 { 0, }
557}; 523};
558 524
@@ -560,6 +526,7 @@ static struct pci_driver ioat_pci_driver = {
560 .name = "ioatdma", 526 .name = "ioatdma",
561 .id_table = ioat_pci_tbl, 527 .id_table = ioat_pci_tbl,
562 .probe = ioat_probe, 528 .probe = ioat_probe,
529 .shutdown = ioat_shutdown,
563 .remove = __devexit_p(ioat_remove), 530 .remove = __devexit_p(ioat_remove),
564}; 531};
565 532
@@ -569,21 +536,21 @@ static irqreturn_t ioat_do_interrupt(int irq, void *data)
569 unsigned long attnstatus; 536 unsigned long attnstatus;
570 u8 intrctrl; 537 u8 intrctrl;
571 538
572 intrctrl = ioatdma_read8(instance, IOAT_INTRCTRL_OFFSET); 539 intrctrl = readb(instance->reg_base + IOAT_INTRCTRL_OFFSET);
573 540
574 if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN)) 541 if (!(intrctrl & IOAT_INTRCTRL_MASTER_INT_EN))
575 return IRQ_NONE; 542 return IRQ_NONE;
576 543
577 if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) { 544 if (!(intrctrl & IOAT_INTRCTRL_INT_STATUS)) {
578 ioatdma_write8(instance, IOAT_INTRCTRL_OFFSET, intrctrl); 545 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
579 return IRQ_NONE; 546 return IRQ_NONE;
580 } 547 }
581 548
582 attnstatus = ioatdma_read32(instance, IOAT_ATTNSTATUS_OFFSET); 549 attnstatus = readl(instance->reg_base + IOAT_ATTNSTATUS_OFFSET);
583 550
584 printk(KERN_ERR "ioatdma error: interrupt! status %lx\n", attnstatus); 551 printk(KERN_ERR "ioatdma error: interrupt! status %lx\n", attnstatus);
585 552
586 ioatdma_write8(instance, IOAT_INTRCTRL_OFFSET, intrctrl); 553 writeb(intrctrl, instance->reg_base + IOAT_INTRCTRL_OFFSET);
587 return IRQ_HANDLED; 554 return IRQ_HANDLED;
588} 555}
589 556
@@ -607,19 +574,17 @@ static void ioat_start_null_desc(struct ioat_dma_chan *ioat_chan)
607 574
608 desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL; 575 desc->hw->ctl = IOAT_DMA_DESCRIPTOR_NUL;
609 desc->hw->next = 0; 576 desc->hw->next = 0;
577 desc->async_tx.ack = 1;
610 578
611 list_add_tail(&desc->node, &ioat_chan->used_desc); 579 list_add_tail(&desc->node, &ioat_chan->used_desc);
612 spin_unlock_bh(&ioat_chan->desc_lock); 580 spin_unlock_bh(&ioat_chan->desc_lock);
613 581
614#if (BITS_PER_LONG == 64) 582 writel(((u64) desc->async_tx.phys) & 0x00000000FFFFFFFF,
615 ioatdma_chan_write64(ioat_chan, IOAT_CHAINADDR_OFFSET, desc->phys); 583 ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_LOW);
616#else 584 writel(((u64) desc->async_tx.phys) >> 32,
617 ioatdma_chan_write32(ioat_chan, 585 ioat_chan->reg_base + IOAT_CHAINADDR_OFFSET_HIGH);
618 IOAT_CHAINADDR_OFFSET_LOW, 586
619 (u32) desc->phys); 587 writeb(IOAT_CHANCMD_START, ioat_chan->reg_base + IOAT_CHANCMD_OFFSET);
620 ioatdma_chan_write32(ioat_chan, IOAT_CHAINADDR_OFFSET_HIGH, 0);
621#endif
622 ioatdma_chan_write8(ioat_chan, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_START);
623} 588}
624 589
625/* 590/*
@@ -633,6 +598,8 @@ static int ioat_self_test(struct ioat_device *device)
633 u8 *src; 598 u8 *src;
634 u8 *dest; 599 u8 *dest;
635 struct dma_chan *dma_chan; 600 struct dma_chan *dma_chan;
601 struct dma_async_tx_descriptor *tx;
602 dma_addr_t addr;
636 dma_cookie_t cookie; 603 dma_cookie_t cookie;
637 int err = 0; 604 int err = 0;
638 605
@@ -658,7 +625,15 @@ static int ioat_self_test(struct ioat_device *device)
658 goto out; 625 goto out;
659 } 626 }
660 627
661 cookie = ioat_dma_memcpy_buf_to_buf(dma_chan, dest, src, IOAT_TEST_SIZE); 628 tx = ioat_dma_prep_memcpy(dma_chan, IOAT_TEST_SIZE, 0);
629 async_tx_ack(tx);
630 addr = dma_map_single(dma_chan->device->dev, src, IOAT_TEST_SIZE,
631 DMA_TO_DEVICE);
632 ioat_set_src(addr, tx, 0);
633 addr = dma_map_single(dma_chan->device->dev, dest, IOAT_TEST_SIZE,
634 DMA_FROM_DEVICE);
635 ioat_set_dest(addr, tx, 0);
636 cookie = ioat_tx_submit(tx);
662 ioat_dma_memcpy_issue_pending(dma_chan); 637 ioat_dma_memcpy_issue_pending(dma_chan);
663 msleep(1); 638 msleep(1);
664 639
@@ -748,19 +723,20 @@ static int __devinit ioat_probe(struct pci_dev *pdev,
748 723
749 device->reg_base = reg_base; 724 device->reg_base = reg_base;
750 725
751 ioatdma_write8(device, IOAT_INTRCTRL_OFFSET, IOAT_INTRCTRL_MASTER_INT_EN); 726 writeb(IOAT_INTRCTRL_MASTER_INT_EN, device->reg_base + IOAT_INTRCTRL_OFFSET);
752 pci_set_master(pdev); 727 pci_set_master(pdev);
753 728
754 INIT_LIST_HEAD(&device->common.channels); 729 INIT_LIST_HEAD(&device->common.channels);
755 enumerate_dma_channels(device); 730 enumerate_dma_channels(device);
756 731
732 dma_cap_set(DMA_MEMCPY, device->common.cap_mask);
757 device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources; 733 device->common.device_alloc_chan_resources = ioat_dma_alloc_chan_resources;
758 device->common.device_free_chan_resources = ioat_dma_free_chan_resources; 734 device->common.device_free_chan_resources = ioat_dma_free_chan_resources;
759 device->common.device_memcpy_buf_to_buf = ioat_dma_memcpy_buf_to_buf; 735 device->common.device_prep_dma_memcpy = ioat_dma_prep_memcpy;
760 device->common.device_memcpy_buf_to_pg = ioat_dma_memcpy_buf_to_pg; 736 device->common.device_is_tx_complete = ioat_dma_is_complete;
761 device->common.device_memcpy_pg_to_pg = ioat_dma_memcpy_pg_to_pg; 737 device->common.device_issue_pending = ioat_dma_memcpy_issue_pending;
762 device->common.device_memcpy_complete = ioat_dma_is_complete; 738 device->common.device_dependency_added = ioat_dma_dependency_added;
763 device->common.device_memcpy_issue_pending = ioat_dma_memcpy_issue_pending; 739 device->common.dev = &pdev->dev;
764 printk(KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n", 740 printk(KERN_INFO "Intel(R) I/OAT DMA Engine found, %d channels\n",
765 device->common.chancnt); 741 device->common.chancnt);
766 742
@@ -787,9 +763,20 @@ err_request_regions:
787err_set_dma_mask: 763err_set_dma_mask:
788 pci_disable_device(pdev); 764 pci_disable_device(pdev);
789err_enable_device: 765err_enable_device:
766
767 printk(KERN_ERR "Intel(R) I/OAT DMA Engine initialization failed\n");
768
790 return err; 769 return err;
791} 770}
792 771
772static void ioat_shutdown(struct pci_dev *pdev)
773{
774 struct ioat_device *device;
775 device = pci_get_drvdata(pdev);
776
777 dma_async_device_unregister(&device->common);
778}
779
793static void __devexit ioat_remove(struct pci_dev *pdev) 780static void __devexit ioat_remove(struct pci_dev *pdev)
794{ 781{
795 struct ioat_device *device; 782 struct ioat_device *device;
@@ -818,7 +805,7 @@ static void __devexit ioat_remove(struct pci_dev *pdev)
818} 805}
819 806
820/* MODULE API */ 807/* MODULE API */
821MODULE_VERSION("1.7"); 808MODULE_VERSION("1.9");
822MODULE_LICENSE("GPL"); 809MODULE_LICENSE("GPL");
823MODULE_AUTHOR("Intel Corporation"); 810MODULE_AUTHOR("Intel Corporation");
824 811
diff --git a/drivers/dma/ioatdma.h b/drivers/dma/ioatdma.h
index 62b26a9be4c9..d3726478031a 100644
--- a/drivers/dma/ioatdma.h
+++ b/drivers/dma/ioatdma.h
@@ -30,9 +30,6 @@
30 30
31#define IOAT_LOW_COMPLETION_MASK 0xffffffc0 31#define IOAT_LOW_COMPLETION_MASK 0xffffffc0
32 32
33extern struct list_head dma_device_list;
34extern struct list_head dma_client_list;
35
36/** 33/**
37 * struct ioat_device - internal representation of a IOAT device 34 * struct ioat_device - internal representation of a IOAT device
38 * @pdev: PCI-Express device 35 * @pdev: PCI-Express device
@@ -105,21 +102,20 @@ struct ioat_dma_chan {
105/** 102/**
106 * struct ioat_desc_sw - wrapper around hardware descriptor 103 * struct ioat_desc_sw - wrapper around hardware descriptor
107 * @hw: hardware DMA descriptor 104 * @hw: hardware DMA descriptor
108 * @node: 105 * @node: this descriptor will either be on the free list,
109 * @cookie: 106 * or attached to a transaction list (async_tx.tx_list)
110 * @phys: 107 * @tx_cnt: number of descriptors required to complete the transaction
108 * @async_tx: the generic software descriptor for all engines
111 */ 109 */
112
113struct ioat_desc_sw { 110struct ioat_desc_sw {
114 struct ioat_dma_descriptor *hw; 111 struct ioat_dma_descriptor *hw;
115 struct list_head node; 112 struct list_head node;
116 dma_cookie_t cookie; 113 int tx_cnt;
117 dma_addr_t phys;
118 DECLARE_PCI_UNMAP_ADDR(src) 114 DECLARE_PCI_UNMAP_ADDR(src)
119 DECLARE_PCI_UNMAP_LEN(src_len) 115 DECLARE_PCI_UNMAP_LEN(src_len)
120 DECLARE_PCI_UNMAP_ADDR(dst) 116 DECLARE_PCI_UNMAP_ADDR(dst)
121 DECLARE_PCI_UNMAP_LEN(dst_len) 117 DECLARE_PCI_UNMAP_LEN(dst_len)
118 struct dma_async_tx_descriptor async_tx;
122}; 119};
123 120
124#endif /* IOATDMA_H */ 121#endif /* IOATDMA_H */
125
diff --git a/drivers/dma/ioatdma_io.h b/drivers/dma/ioatdma_io.h
deleted file mode 100644
index c0b4bf66c920..000000000000
--- a/drivers/dma/ioatdma_io.h
+++ /dev/null
@@ -1,118 +0,0 @@
1/*
2 * Copyright(c) 2004 - 2006 Intel Corporation. All rights reserved.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the Free
6 * Software Foundation; either version 2 of the License, or (at your option)
7 * any later version.
8 *
9 * This program is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc., 59
16 * Temple Place - Suite 330, Boston, MA 02111-1307, USA.
17 *
18 * The full GNU General Public License is included in this distribution in the
19 * file called COPYING.
20 */
21#ifndef IOATDMA_IO_H
22#define IOATDMA_IO_H
23
24#include <asm/io.h>
25
26/*
27 * device and per-channel MMIO register read and write functions
28 * this is a lot of anoying inline functions, but it's typesafe
29 */
30
31static inline u8 ioatdma_read8(struct ioat_device *device,
32 unsigned int offset)
33{
34 return readb(device->reg_base + offset);
35}
36
37static inline u16 ioatdma_read16(struct ioat_device *device,
38 unsigned int offset)
39{
40 return readw(device->reg_base + offset);
41}
42
43static inline u32 ioatdma_read32(struct ioat_device *device,
44 unsigned int offset)
45{
46 return readl(device->reg_base + offset);
47}
48
49static inline void ioatdma_write8(struct ioat_device *device,
50 unsigned int offset, u8 value)
51{
52 writeb(value, device->reg_base + offset);
53}
54
55static inline void ioatdma_write16(struct ioat_device *device,
56 unsigned int offset, u16 value)
57{
58 writew(value, device->reg_base + offset);
59}
60
61static inline void ioatdma_write32(struct ioat_device *device,
62 unsigned int offset, u32 value)
63{
64 writel(value, device->reg_base + offset);
65}
66
67static inline u8 ioatdma_chan_read8(struct ioat_dma_chan *chan,
68 unsigned int offset)
69{
70 return readb(chan->reg_base + offset);
71}
72
73static inline u16 ioatdma_chan_read16(struct ioat_dma_chan *chan,
74 unsigned int offset)
75{
76 return readw(chan->reg_base + offset);
77}
78
79static inline u32 ioatdma_chan_read32(struct ioat_dma_chan *chan,
80 unsigned int offset)
81{
82 return readl(chan->reg_base + offset);
83}
84
85static inline void ioatdma_chan_write8(struct ioat_dma_chan *chan,
86 unsigned int offset, u8 value)
87{
88 writeb(value, chan->reg_base + offset);
89}
90
91static inline void ioatdma_chan_write16(struct ioat_dma_chan *chan,
92 unsigned int offset, u16 value)
93{
94 writew(value, chan->reg_base + offset);
95}
96
97static inline void ioatdma_chan_write32(struct ioat_dma_chan *chan,
98 unsigned int offset, u32 value)
99{
100 writel(value, chan->reg_base + offset);
101}
102
103#if (BITS_PER_LONG == 64)
104static inline u64 ioatdma_chan_read64(struct ioat_dma_chan *chan,
105 unsigned int offset)
106{
107 return readq(chan->reg_base + offset);
108}
109
110static inline void ioatdma_chan_write64(struct ioat_dma_chan *chan,
111 unsigned int offset, u64 value)
112{
113 writeq(value, chan->reg_base + offset);
114}
115#endif
116
117#endif /* IOATDMA_IO_H */
118
diff --git a/drivers/dma/iop-adma.c b/drivers/dma/iop-adma.c
new file mode 100644
index 000000000000..5a1d426744d6
--- /dev/null
+++ b/drivers/dma/iop-adma.c
@@ -0,0 +1,1467 @@
1/*
2 * offload engine driver for the Intel Xscale series of i/o processors
3 * Copyright © 2006, Intel Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc.,
16 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
17 *
18 */
19
20/*
21 * This driver supports the asynchrounous DMA copy and RAID engines available
22 * on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x)
23 */
24
25#include <linux/init.h>
26#include <linux/module.h>
27#include <linux/async_tx.h>
28#include <linux/delay.h>
29#include <linux/dma-mapping.h>
30#include <linux/spinlock.h>
31#include <linux/interrupt.h>
32#include <linux/platform_device.h>
33#include <linux/memory.h>
34#include <linux/ioport.h>
35
36#include <asm/arch/adma.h>
37
38#define to_iop_adma_chan(chan) container_of(chan, struct iop_adma_chan, common)
39#define to_iop_adma_device(dev) \
40 container_of(dev, struct iop_adma_device, common)
41#define tx_to_iop_adma_slot(tx) \
42 container_of(tx, struct iop_adma_desc_slot, async_tx)
43
44/**
45 * iop_adma_free_slots - flags descriptor slots for reuse
46 * @slot: Slot to free
47 * Caller must hold &iop_chan->lock while calling this function
48 */
49static void iop_adma_free_slots(struct iop_adma_desc_slot *slot)
50{
51 int stride = slot->slots_per_op;
52
53 while (stride--) {
54 slot->slots_per_op = 0;
55 slot = list_entry(slot->slot_node.next,
56 struct iop_adma_desc_slot,
57 slot_node);
58 }
59}
60
61static dma_cookie_t
62iop_adma_run_tx_complete_actions(struct iop_adma_desc_slot *desc,
63 struct iop_adma_chan *iop_chan, dma_cookie_t cookie)
64{
65 BUG_ON(desc->async_tx.cookie < 0);
66 spin_lock_bh(&desc->async_tx.lock);
67 if (desc->async_tx.cookie > 0) {
68 cookie = desc->async_tx.cookie;
69 desc->async_tx.cookie = 0;
70
71 /* call the callback (must not sleep or submit new
72 * operations to this channel)
73 */
74 if (desc->async_tx.callback)
75 desc->async_tx.callback(
76 desc->async_tx.callback_param);
77
78 /* unmap dma addresses
79 * (unmap_single vs unmap_page?)
80 */
81 if (desc->group_head && desc->unmap_len) {
82 struct iop_adma_desc_slot *unmap = desc->group_head;
83 struct device *dev =
84 &iop_chan->device->pdev->dev;
85 u32 len = unmap->unmap_len;
86 u32 src_cnt = unmap->unmap_src_cnt;
87 dma_addr_t addr = iop_desc_get_dest_addr(unmap,
88 iop_chan);
89
90 dma_unmap_page(dev, addr, len, DMA_FROM_DEVICE);
91 while (src_cnt--) {
92 addr = iop_desc_get_src_addr(unmap,
93 iop_chan,
94 src_cnt);
95 dma_unmap_page(dev, addr, len,
96 DMA_TO_DEVICE);
97 }
98 desc->group_head = NULL;
99 }
100 }
101
102 /* run dependent operations */
103 async_tx_run_dependencies(&desc->async_tx);
104 spin_unlock_bh(&desc->async_tx.lock);
105
106 return cookie;
107}
108
109static int
110iop_adma_clean_slot(struct iop_adma_desc_slot *desc,
111 struct iop_adma_chan *iop_chan)
112{
113 /* the client is allowed to attach dependent operations
114 * until 'ack' is set
115 */
116 if (!desc->async_tx.ack)
117 return 0;
118
119 /* leave the last descriptor in the chain
120 * so we can append to it
121 */
122 if (desc->chain_node.next == &iop_chan->chain)
123 return 1;
124
125 dev_dbg(iop_chan->device->common.dev,
126 "\tfree slot: %d slots_per_op: %d\n",
127 desc->idx, desc->slots_per_op);
128
129 list_del(&desc->chain_node);
130 iop_adma_free_slots(desc);
131
132 return 0;
133}
134
135static void __iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)
136{
137 struct iop_adma_desc_slot *iter, *_iter, *grp_start = NULL;
138 dma_cookie_t cookie = 0;
139 u32 current_desc = iop_chan_get_current_descriptor(iop_chan);
140 int busy = iop_chan_is_busy(iop_chan);
141 int seen_current = 0, slot_cnt = 0, slots_per_op = 0;
142
143 dev_dbg(iop_chan->device->common.dev, "%s\n", __FUNCTION__);
144 /* free completed slots from the chain starting with
145 * the oldest descriptor
146 */
147 list_for_each_entry_safe(iter, _iter, &iop_chan->chain,
148 chain_node) {
149 pr_debug("\tcookie: %d slot: %d busy: %d "
150 "this_desc: %#x next_desc: %#x ack: %d\n",
151 iter->async_tx.cookie, iter->idx, busy,
152 iter->async_tx.phys, iop_desc_get_next_desc(iter),
153 iter->async_tx.ack);
154 prefetch(_iter);
155 prefetch(&_iter->async_tx);
156
157 /* do not advance past the current descriptor loaded into the
158 * hardware channel, subsequent descriptors are either in
159 * process or have not been submitted
160 */
161 if (seen_current)
162 break;
163
164 /* stop the search if we reach the current descriptor and the
165 * channel is busy, or if it appears that the current descriptor
166 * needs to be re-read (i.e. has been appended to)
167 */
168 if (iter->async_tx.phys == current_desc) {
169 BUG_ON(seen_current++);
170 if (busy || iop_desc_get_next_desc(iter))
171 break;
172 }
173
174 /* detect the start of a group transaction */
175 if (!slot_cnt && !slots_per_op) {
176 slot_cnt = iter->slot_cnt;
177 slots_per_op = iter->slots_per_op;
178 if (slot_cnt <= slots_per_op) {
179 slot_cnt = 0;
180 slots_per_op = 0;
181 }
182 }
183
184 if (slot_cnt) {
185 pr_debug("\tgroup++\n");
186 if (!grp_start)
187 grp_start = iter;
188 slot_cnt -= slots_per_op;
189 }
190
191 /* all the members of a group are complete */
192 if (slots_per_op != 0 && slot_cnt == 0) {
193 struct iop_adma_desc_slot *grp_iter, *_grp_iter;
194 int end_of_chain = 0;
195 pr_debug("\tgroup end\n");
196
197 /* collect the total results */
198 if (grp_start->xor_check_result) {
199 u32 zero_sum_result = 0;
200 slot_cnt = grp_start->slot_cnt;
201 grp_iter = grp_start;
202
203 list_for_each_entry_from(grp_iter,
204 &iop_chan->chain, chain_node) {
205 zero_sum_result |=
206 iop_desc_get_zero_result(grp_iter);
207 pr_debug("\titer%d result: %d\n",
208 grp_iter->idx, zero_sum_result);
209 slot_cnt -= slots_per_op;
210 if (slot_cnt == 0)
211 break;
212 }
213 pr_debug("\tgrp_start->xor_check_result: %p\n",
214 grp_start->xor_check_result);
215 *grp_start->xor_check_result = zero_sum_result;
216 }
217
218 /* clean up the group */
219 slot_cnt = grp_start->slot_cnt;
220 grp_iter = grp_start;
221 list_for_each_entry_safe_from(grp_iter, _grp_iter,
222 &iop_chan->chain, chain_node) {
223 cookie = iop_adma_run_tx_complete_actions(
224 grp_iter, iop_chan, cookie);
225
226 slot_cnt -= slots_per_op;
227 end_of_chain = iop_adma_clean_slot(grp_iter,
228 iop_chan);
229
230 if (slot_cnt == 0 || end_of_chain)
231 break;
232 }
233
234 /* the group should be complete at this point */
235 BUG_ON(slot_cnt);
236
237 slots_per_op = 0;
238 grp_start = NULL;
239 if (end_of_chain)
240 break;
241 else
242 continue;
243 } else if (slots_per_op) /* wait for group completion */
244 continue;
245
246 /* write back zero sum results (single descriptor case) */
247 if (iter->xor_check_result && iter->async_tx.cookie)
248 *iter->xor_check_result =
249 iop_desc_get_zero_result(iter);
250
251 cookie = iop_adma_run_tx_complete_actions(
252 iter, iop_chan, cookie);
253
254 if (iop_adma_clean_slot(iter, iop_chan))
255 break;
256 }
257
258 BUG_ON(!seen_current);
259
260 iop_chan_idle(busy, iop_chan);
261
262 if (cookie > 0) {
263 iop_chan->completed_cookie = cookie;
264 pr_debug("\tcompleted cookie %d\n", cookie);
265 }
266}
267
268static void
269iop_adma_slot_cleanup(struct iop_adma_chan *iop_chan)
270{
271 spin_lock_bh(&iop_chan->lock);
272 __iop_adma_slot_cleanup(iop_chan);
273 spin_unlock_bh(&iop_chan->lock);
274}
275
276static void iop_adma_tasklet(unsigned long data)
277{
278 struct iop_adma_chan *chan = (struct iop_adma_chan *) data;
279 __iop_adma_slot_cleanup(chan);
280}
281
282static struct iop_adma_desc_slot *
283iop_adma_alloc_slots(struct iop_adma_chan *iop_chan, int num_slots,
284 int slots_per_op)
285{
286 struct iop_adma_desc_slot *iter, *_iter, *alloc_start = NULL;
287 struct list_head chain = LIST_HEAD_INIT(chain);
288 int slots_found, retry = 0;
289
290 /* start search from the last allocated descrtiptor
291 * if a contiguous allocation can not be found start searching
292 * from the beginning of the list
293 */
294retry:
295 slots_found = 0;
296 if (retry == 0)
297 iter = iop_chan->last_used;
298 else
299 iter = list_entry(&iop_chan->all_slots,
300 struct iop_adma_desc_slot,
301 slot_node);
302
303 list_for_each_entry_safe_continue(
304 iter, _iter, &iop_chan->all_slots, slot_node) {
305 prefetch(_iter);
306 prefetch(&_iter->async_tx);
307 if (iter->slots_per_op) {
308 /* give up after finding the first busy slot
309 * on the second pass through the list
310 */
311 if (retry)
312 break;
313
314 slots_found = 0;
315 continue;
316 }
317
318 /* start the allocation if the slot is correctly aligned */
319 if (!slots_found++) {
320 if (iop_desc_is_aligned(iter, slots_per_op))
321 alloc_start = iter;
322 else {
323 slots_found = 0;
324 continue;
325 }
326 }
327
328 if (slots_found == num_slots) {
329 struct iop_adma_desc_slot *alloc_tail = NULL;
330 struct iop_adma_desc_slot *last_used = NULL;
331 iter = alloc_start;
332 while (num_slots) {
333 int i;
334 dev_dbg(iop_chan->device->common.dev,
335 "allocated slot: %d "
336 "(desc %p phys: %#x) slots_per_op %d\n",
337 iter->idx, iter->hw_desc,
338 iter->async_tx.phys, slots_per_op);
339
340 /* pre-ack all but the last descriptor */
341 if (num_slots != slots_per_op)
342 iter->async_tx.ack = 1;
343 else
344 iter->async_tx.ack = 0;
345
346 list_add_tail(&iter->chain_node, &chain);
347 alloc_tail = iter;
348 iter->async_tx.cookie = 0;
349 iter->slot_cnt = num_slots;
350 iter->xor_check_result = NULL;
351 for (i = 0; i < slots_per_op; i++) {
352 iter->slots_per_op = slots_per_op - i;
353 last_used = iter;
354 iter = list_entry(iter->slot_node.next,
355 struct iop_adma_desc_slot,
356 slot_node);
357 }
358 num_slots -= slots_per_op;
359 }
360 alloc_tail->group_head = alloc_start;
361 alloc_tail->async_tx.cookie = -EBUSY;
362 list_splice(&chain, &alloc_tail->async_tx.tx_list);
363 iop_chan->last_used = last_used;
364 iop_desc_clear_next_desc(alloc_start);
365 iop_desc_clear_next_desc(alloc_tail);
366 return alloc_tail;
367 }
368 }
369 if (!retry++)
370 goto retry;
371
372 /* try to free some slots if the allocation fails */
373 tasklet_schedule(&iop_chan->irq_tasklet);
374
375 return NULL;
376}
377
378static dma_cookie_t
379iop_desc_assign_cookie(struct iop_adma_chan *iop_chan,
380 struct iop_adma_desc_slot *desc)
381{
382 dma_cookie_t cookie = iop_chan->common.cookie;
383 cookie++;
384 if (cookie < 0)
385 cookie = 1;
386 iop_chan->common.cookie = desc->async_tx.cookie = cookie;
387 return cookie;
388}
389
390static void iop_adma_check_threshold(struct iop_adma_chan *iop_chan)
391{
392 dev_dbg(iop_chan->device->common.dev, "pending: %d\n",
393 iop_chan->pending);
394
395 if (iop_chan->pending >= IOP_ADMA_THRESHOLD) {
396 iop_chan->pending = 0;
397 iop_chan_append(iop_chan);
398 }
399}
400
401static dma_cookie_t
402iop_adma_tx_submit(struct dma_async_tx_descriptor *tx)
403{
404 struct iop_adma_desc_slot *sw_desc = tx_to_iop_adma_slot(tx);
405 struct iop_adma_chan *iop_chan = to_iop_adma_chan(tx->chan);
406 struct iop_adma_desc_slot *grp_start, *old_chain_tail;
407 int slot_cnt;
408 int slots_per_op;
409 dma_cookie_t cookie;
410
411 grp_start = sw_desc->group_head;
412 slot_cnt = grp_start->slot_cnt;
413 slots_per_op = grp_start->slots_per_op;
414
415 spin_lock_bh(&iop_chan->lock);
416 cookie = iop_desc_assign_cookie(iop_chan, sw_desc);
417
418 old_chain_tail = list_entry(iop_chan->chain.prev,
419 struct iop_adma_desc_slot, chain_node);
420 list_splice_init(&sw_desc->async_tx.tx_list,
421 &old_chain_tail->chain_node);
422
423 /* fix up the hardware chain */
424 iop_desc_set_next_desc(old_chain_tail, grp_start->async_tx.phys);
425
426 /* 1/ don't add pre-chained descriptors
427 * 2/ dummy read to flush next_desc write
428 */
429 BUG_ON(iop_desc_get_next_desc(sw_desc));
430
431 /* increment the pending count by the number of slots
432 * memcpy operations have a 1:1 (slot:operation) relation
433 * other operations are heavier and will pop the threshold
434 * more often.
435 */
436 iop_chan->pending += slot_cnt;
437 iop_adma_check_threshold(iop_chan);
438 spin_unlock_bh(&iop_chan->lock);
439
440 dev_dbg(iop_chan->device->common.dev, "%s cookie: %d slot: %d\n",
441 __FUNCTION__, sw_desc->async_tx.cookie, sw_desc->idx);
442
443 return cookie;
444}
445
446static void
447iop_adma_set_dest(dma_addr_t addr, struct dma_async_tx_descriptor *tx,
448 int index)
449{
450 struct iop_adma_desc_slot *sw_desc = tx_to_iop_adma_slot(tx);
451 struct iop_adma_chan *iop_chan = to_iop_adma_chan(tx->chan);
452
453 /* to do: support transfers lengths > IOP_ADMA_MAX_BYTE_COUNT */
454 iop_desc_set_dest_addr(sw_desc->group_head, iop_chan, addr);
455}
456
457static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan);
458static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan);
459
460/* returns the number of allocated descriptors */
461static int iop_adma_alloc_chan_resources(struct dma_chan *chan)
462{
463 char *hw_desc;
464 int idx;
465 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
466 struct iop_adma_desc_slot *slot = NULL;
467 int init = iop_chan->slots_allocated ? 0 : 1;
468 struct iop_adma_platform_data *plat_data =
469 iop_chan->device->pdev->dev.platform_data;
470 int num_descs_in_pool = plat_data->pool_size/IOP_ADMA_SLOT_SIZE;
471
472 /* Allocate descriptor slots */
473 do {
474 idx = iop_chan->slots_allocated;
475 if (idx == num_descs_in_pool)
476 break;
477
478 slot = kzalloc(sizeof(*slot), GFP_KERNEL);
479 if (!slot) {
480 printk(KERN_INFO "IOP ADMA Channel only initialized"
481 " %d descriptor slots", idx);
482 break;
483 }
484 hw_desc = (char *) iop_chan->device->dma_desc_pool_virt;
485 slot->hw_desc = (void *) &hw_desc[idx * IOP_ADMA_SLOT_SIZE];
486
487 dma_async_tx_descriptor_init(&slot->async_tx, chan);
488 slot->async_tx.tx_submit = iop_adma_tx_submit;
489 slot->async_tx.tx_set_dest = iop_adma_set_dest;
490 INIT_LIST_HEAD(&slot->chain_node);
491 INIT_LIST_HEAD(&slot->slot_node);
492 INIT_LIST_HEAD(&slot->async_tx.tx_list);
493 hw_desc = (char *) iop_chan->device->dma_desc_pool;
494 slot->async_tx.phys =
495 (dma_addr_t) &hw_desc[idx * IOP_ADMA_SLOT_SIZE];
496 slot->idx = idx;
497
498 spin_lock_bh(&iop_chan->lock);
499 iop_chan->slots_allocated++;
500 list_add_tail(&slot->slot_node, &iop_chan->all_slots);
501 spin_unlock_bh(&iop_chan->lock);
502 } while (iop_chan->slots_allocated < num_descs_in_pool);
503
504 if (idx && !iop_chan->last_used)
505 iop_chan->last_used = list_entry(iop_chan->all_slots.next,
506 struct iop_adma_desc_slot,
507 slot_node);
508
509 dev_dbg(iop_chan->device->common.dev,
510 "allocated %d descriptor slots last_used: %p\n",
511 iop_chan->slots_allocated, iop_chan->last_used);
512
513 /* initialize the channel and the chain with a null operation */
514 if (init) {
515 if (dma_has_cap(DMA_MEMCPY,
516 iop_chan->device->common.cap_mask))
517 iop_chan_start_null_memcpy(iop_chan);
518 else if (dma_has_cap(DMA_XOR,
519 iop_chan->device->common.cap_mask))
520 iop_chan_start_null_xor(iop_chan);
521 else
522 BUG();
523 }
524
525 return (idx > 0) ? idx : -ENOMEM;
526}
527
528static struct dma_async_tx_descriptor *
529iop_adma_prep_dma_interrupt(struct dma_chan *chan)
530{
531 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
532 struct iop_adma_desc_slot *sw_desc, *grp_start;
533 int slot_cnt, slots_per_op;
534
535 dev_dbg(iop_chan->device->common.dev, "%s\n", __FUNCTION__);
536
537 spin_lock_bh(&iop_chan->lock);
538 slot_cnt = iop_chan_interrupt_slot_count(&slots_per_op, iop_chan);
539 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
540 if (sw_desc) {
541 grp_start = sw_desc->group_head;
542 iop_desc_init_interrupt(grp_start, iop_chan);
543 grp_start->unmap_len = 0;
544 }
545 spin_unlock_bh(&iop_chan->lock);
546
547 return sw_desc ? &sw_desc->async_tx : NULL;
548}
549
550static void
551iop_adma_memcpy_set_src(dma_addr_t addr, struct dma_async_tx_descriptor *tx,
552 int index)
553{
554 struct iop_adma_desc_slot *sw_desc = tx_to_iop_adma_slot(tx);
555 struct iop_adma_desc_slot *grp_start = sw_desc->group_head;
556
557 iop_desc_set_memcpy_src_addr(grp_start, addr);
558}
559
560static struct dma_async_tx_descriptor *
561iop_adma_prep_dma_memcpy(struct dma_chan *chan, size_t len, int int_en)
562{
563 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
564 struct iop_adma_desc_slot *sw_desc, *grp_start;
565 int slot_cnt, slots_per_op;
566
567 if (unlikely(!len))
568 return NULL;
569 BUG_ON(unlikely(len > IOP_ADMA_MAX_BYTE_COUNT));
570
571 dev_dbg(iop_chan->device->common.dev, "%s len: %u\n",
572 __FUNCTION__, len);
573
574 spin_lock_bh(&iop_chan->lock);
575 slot_cnt = iop_chan_memcpy_slot_count(len, &slots_per_op);
576 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
577 if (sw_desc) {
578 grp_start = sw_desc->group_head;
579 iop_desc_init_memcpy(grp_start, int_en);
580 iop_desc_set_byte_count(grp_start, iop_chan, len);
581 sw_desc->unmap_src_cnt = 1;
582 sw_desc->unmap_len = len;
583 sw_desc->async_tx.tx_set_src = iop_adma_memcpy_set_src;
584 }
585 spin_unlock_bh(&iop_chan->lock);
586
587 return sw_desc ? &sw_desc->async_tx : NULL;
588}
589
590static struct dma_async_tx_descriptor *
591iop_adma_prep_dma_memset(struct dma_chan *chan, int value, size_t len,
592 int int_en)
593{
594 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
595 struct iop_adma_desc_slot *sw_desc, *grp_start;
596 int slot_cnt, slots_per_op;
597
598 if (unlikely(!len))
599 return NULL;
600 BUG_ON(unlikely(len > IOP_ADMA_MAX_BYTE_COUNT));
601
602 dev_dbg(iop_chan->device->common.dev, "%s len: %u\n",
603 __FUNCTION__, len);
604
605 spin_lock_bh(&iop_chan->lock);
606 slot_cnt = iop_chan_memset_slot_count(len, &slots_per_op);
607 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
608 if (sw_desc) {
609 grp_start = sw_desc->group_head;
610 iop_desc_init_memset(grp_start, int_en);
611 iop_desc_set_byte_count(grp_start, iop_chan, len);
612 iop_desc_set_block_fill_val(grp_start, value);
613 sw_desc->unmap_src_cnt = 1;
614 sw_desc->unmap_len = len;
615 }
616 spin_unlock_bh(&iop_chan->lock);
617
618 return sw_desc ? &sw_desc->async_tx : NULL;
619}
620
621static void
622iop_adma_xor_set_src(dma_addr_t addr, struct dma_async_tx_descriptor *tx,
623 int index)
624{
625 struct iop_adma_desc_slot *sw_desc = tx_to_iop_adma_slot(tx);
626 struct iop_adma_desc_slot *grp_start = sw_desc->group_head;
627
628 iop_desc_set_xor_src_addr(grp_start, index, addr);
629}
630
631static struct dma_async_tx_descriptor *
632iop_adma_prep_dma_xor(struct dma_chan *chan, unsigned int src_cnt, size_t len,
633 int int_en)
634{
635 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
636 struct iop_adma_desc_slot *sw_desc, *grp_start;
637 int slot_cnt, slots_per_op;
638
639 if (unlikely(!len))
640 return NULL;
641 BUG_ON(unlikely(len > IOP_ADMA_XOR_MAX_BYTE_COUNT));
642
643 dev_dbg(iop_chan->device->common.dev,
644 "%s src_cnt: %d len: %u int_en: %d\n",
645 __FUNCTION__, src_cnt, len, int_en);
646
647 spin_lock_bh(&iop_chan->lock);
648 slot_cnt = iop_chan_xor_slot_count(len, src_cnt, &slots_per_op);
649 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
650 if (sw_desc) {
651 grp_start = sw_desc->group_head;
652 iop_desc_init_xor(grp_start, src_cnt, int_en);
653 iop_desc_set_byte_count(grp_start, iop_chan, len);
654 sw_desc->unmap_src_cnt = src_cnt;
655 sw_desc->unmap_len = len;
656 sw_desc->async_tx.tx_set_src = iop_adma_xor_set_src;
657 }
658 spin_unlock_bh(&iop_chan->lock);
659
660 return sw_desc ? &sw_desc->async_tx : NULL;
661}
662
663static void
664iop_adma_xor_zero_sum_set_src(dma_addr_t addr,
665 struct dma_async_tx_descriptor *tx,
666 int index)
667{
668 struct iop_adma_desc_slot *sw_desc = tx_to_iop_adma_slot(tx);
669 struct iop_adma_desc_slot *grp_start = sw_desc->group_head;
670
671 iop_desc_set_zero_sum_src_addr(grp_start, index, addr);
672}
673
674static struct dma_async_tx_descriptor *
675iop_adma_prep_dma_zero_sum(struct dma_chan *chan, unsigned int src_cnt,
676 size_t len, u32 *result, int int_en)
677{
678 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
679 struct iop_adma_desc_slot *sw_desc, *grp_start;
680 int slot_cnt, slots_per_op;
681
682 if (unlikely(!len))
683 return NULL;
684
685 dev_dbg(iop_chan->device->common.dev, "%s src_cnt: %d len: %u\n",
686 __FUNCTION__, src_cnt, len);
687
688 spin_lock_bh(&iop_chan->lock);
689 slot_cnt = iop_chan_zero_sum_slot_count(len, src_cnt, &slots_per_op);
690 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
691 if (sw_desc) {
692 grp_start = sw_desc->group_head;
693 iop_desc_init_zero_sum(grp_start, src_cnt, int_en);
694 iop_desc_set_zero_sum_byte_count(grp_start, len);
695 grp_start->xor_check_result = result;
696 pr_debug("\t%s: grp_start->xor_check_result: %p\n",
697 __FUNCTION__, grp_start->xor_check_result);
698 sw_desc->unmap_src_cnt = src_cnt;
699 sw_desc->unmap_len = len;
700 sw_desc->async_tx.tx_set_src = iop_adma_xor_zero_sum_set_src;
701 }
702 spin_unlock_bh(&iop_chan->lock);
703
704 return sw_desc ? &sw_desc->async_tx : NULL;
705}
706
707static void iop_adma_dependency_added(struct dma_chan *chan)
708{
709 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
710 tasklet_schedule(&iop_chan->irq_tasklet);
711}
712
713static void iop_adma_free_chan_resources(struct dma_chan *chan)
714{
715 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
716 struct iop_adma_desc_slot *iter, *_iter;
717 int in_use_descs = 0;
718
719 iop_adma_slot_cleanup(iop_chan);
720
721 spin_lock_bh(&iop_chan->lock);
722 list_for_each_entry_safe(iter, _iter, &iop_chan->chain,
723 chain_node) {
724 in_use_descs++;
725 list_del(&iter->chain_node);
726 }
727 list_for_each_entry_safe_reverse(
728 iter, _iter, &iop_chan->all_slots, slot_node) {
729 list_del(&iter->slot_node);
730 kfree(iter);
731 iop_chan->slots_allocated--;
732 }
733 iop_chan->last_used = NULL;
734
735 dev_dbg(iop_chan->device->common.dev, "%s slots_allocated %d\n",
736 __FUNCTION__, iop_chan->slots_allocated);
737 spin_unlock_bh(&iop_chan->lock);
738
739 /* one is ok since we left it on there on purpose */
740 if (in_use_descs > 1)
741 printk(KERN_ERR "IOP: Freeing %d in use descriptors!\n",
742 in_use_descs - 1);
743}
744
745/**
746 * iop_adma_is_complete - poll the status of an ADMA transaction
747 * @chan: ADMA channel handle
748 * @cookie: ADMA transaction identifier
749 */
750static enum dma_status iop_adma_is_complete(struct dma_chan *chan,
751 dma_cookie_t cookie,
752 dma_cookie_t *done,
753 dma_cookie_t *used)
754{
755 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
756 dma_cookie_t last_used;
757 dma_cookie_t last_complete;
758 enum dma_status ret;
759
760 last_used = chan->cookie;
761 last_complete = iop_chan->completed_cookie;
762
763 if (done)
764 *done = last_complete;
765 if (used)
766 *used = last_used;
767
768 ret = dma_async_is_complete(cookie, last_complete, last_used);
769 if (ret == DMA_SUCCESS)
770 return ret;
771
772 iop_adma_slot_cleanup(iop_chan);
773
774 last_used = chan->cookie;
775 last_complete = iop_chan->completed_cookie;
776
777 if (done)
778 *done = last_complete;
779 if (used)
780 *used = last_used;
781
782 return dma_async_is_complete(cookie, last_complete, last_used);
783}
784
785static irqreturn_t iop_adma_eot_handler(int irq, void *data)
786{
787 struct iop_adma_chan *chan = data;
788
789 dev_dbg(chan->device->common.dev, "%s\n", __FUNCTION__);
790
791 tasklet_schedule(&chan->irq_tasklet);
792
793 iop_adma_device_clear_eot_status(chan);
794
795 return IRQ_HANDLED;
796}
797
798static irqreturn_t iop_adma_eoc_handler(int irq, void *data)
799{
800 struct iop_adma_chan *chan = data;
801
802 dev_dbg(chan->device->common.dev, "%s\n", __FUNCTION__);
803
804 tasklet_schedule(&chan->irq_tasklet);
805
806 iop_adma_device_clear_eoc_status(chan);
807
808 return IRQ_HANDLED;
809}
810
811static irqreturn_t iop_adma_err_handler(int irq, void *data)
812{
813 struct iop_adma_chan *chan = data;
814 unsigned long status = iop_chan_get_status(chan);
815
816 dev_printk(KERN_ERR, chan->device->common.dev,
817 "error ( %s%s%s%s%s%s%s)\n",
818 iop_is_err_int_parity(status, chan) ? "int_parity " : "",
819 iop_is_err_mcu_abort(status, chan) ? "mcu_abort " : "",
820 iop_is_err_int_tabort(status, chan) ? "int_tabort " : "",
821 iop_is_err_int_mabort(status, chan) ? "int_mabort " : "",
822 iop_is_err_pci_tabort(status, chan) ? "pci_tabort " : "",
823 iop_is_err_pci_mabort(status, chan) ? "pci_mabort " : "",
824 iop_is_err_split_tx(status, chan) ? "split_tx " : "");
825
826 iop_adma_device_clear_err_status(chan);
827
828 BUG();
829
830 return IRQ_HANDLED;
831}
832
833static void iop_adma_issue_pending(struct dma_chan *chan)
834{
835 struct iop_adma_chan *iop_chan = to_iop_adma_chan(chan);
836
837 if (iop_chan->pending) {
838 iop_chan->pending = 0;
839 iop_chan_append(iop_chan);
840 }
841}
842
843/*
844 * Perform a transaction to verify the HW works.
845 */
846#define IOP_ADMA_TEST_SIZE 2000
847
848static int __devinit iop_adma_memcpy_self_test(struct iop_adma_device *device)
849{
850 int i;
851 void *src, *dest;
852 dma_addr_t src_dma, dest_dma;
853 struct dma_chan *dma_chan;
854 dma_cookie_t cookie;
855 struct dma_async_tx_descriptor *tx;
856 int err = 0;
857 struct iop_adma_chan *iop_chan;
858
859 dev_dbg(device->common.dev, "%s\n", __FUNCTION__);
860
861 src = kzalloc(sizeof(u8) * IOP_ADMA_TEST_SIZE, GFP_KERNEL);
862 if (!src)
863 return -ENOMEM;
864 dest = kzalloc(sizeof(u8) * IOP_ADMA_TEST_SIZE, GFP_KERNEL);
865 if (!dest) {
866 kfree(src);
867 return -ENOMEM;
868 }
869
870 /* Fill in src buffer */
871 for (i = 0; i < IOP_ADMA_TEST_SIZE; i++)
872 ((u8 *) src)[i] = (u8)i;
873
874 memset(dest, 0, IOP_ADMA_TEST_SIZE);
875
876 /* Start copy, using first DMA channel */
877 dma_chan = container_of(device->common.channels.next,
878 struct dma_chan,
879 device_node);
880 if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
881 err = -ENODEV;
882 goto out;
883 }
884
885 tx = iop_adma_prep_dma_memcpy(dma_chan, IOP_ADMA_TEST_SIZE, 1);
886 dest_dma = dma_map_single(dma_chan->device->dev, dest,
887 IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE);
888 iop_adma_set_dest(dest_dma, tx, 0);
889 src_dma = dma_map_single(dma_chan->device->dev, src,
890 IOP_ADMA_TEST_SIZE, DMA_TO_DEVICE);
891 iop_adma_memcpy_set_src(src_dma, tx, 0);
892
893 cookie = iop_adma_tx_submit(tx);
894 iop_adma_issue_pending(dma_chan);
895 async_tx_ack(tx);
896 msleep(1);
897
898 if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) !=
899 DMA_SUCCESS) {
900 dev_printk(KERN_ERR, dma_chan->device->dev,
901 "Self-test copy timed out, disabling\n");
902 err = -ENODEV;
903 goto free_resources;
904 }
905
906 iop_chan = to_iop_adma_chan(dma_chan);
907 dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma,
908 IOP_ADMA_TEST_SIZE, DMA_FROM_DEVICE);
909 if (memcmp(src, dest, IOP_ADMA_TEST_SIZE)) {
910 dev_printk(KERN_ERR, dma_chan->device->dev,
911 "Self-test copy failed compare, disabling\n");
912 err = -ENODEV;
913 goto free_resources;
914 }
915
916free_resources:
917 iop_adma_free_chan_resources(dma_chan);
918out:
919 kfree(src);
920 kfree(dest);
921 return err;
922}
923
924#define IOP_ADMA_NUM_SRC_TEST 4 /* must be <= 15 */
925static int __devinit
926iop_adma_xor_zero_sum_self_test(struct iop_adma_device *device)
927{
928 int i, src_idx;
929 struct page *dest;
930 struct page *xor_srcs[IOP_ADMA_NUM_SRC_TEST];
931 struct page *zero_sum_srcs[IOP_ADMA_NUM_SRC_TEST + 1];
932 dma_addr_t dma_addr, dest_dma;
933 struct dma_async_tx_descriptor *tx;
934 struct dma_chan *dma_chan;
935 dma_cookie_t cookie;
936 u8 cmp_byte = 0;
937 u32 cmp_word;
938 u32 zero_sum_result;
939 int err = 0;
940 struct iop_adma_chan *iop_chan;
941
942 dev_dbg(device->common.dev, "%s\n", __FUNCTION__);
943
944 for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) {
945 xor_srcs[src_idx] = alloc_page(GFP_KERNEL);
946 if (!xor_srcs[src_idx])
947 while (src_idx--) {
948 __free_page(xor_srcs[src_idx]);
949 return -ENOMEM;
950 }
951 }
952
953 dest = alloc_page(GFP_KERNEL);
954 if (!dest)
955 while (src_idx--) {
956 __free_page(xor_srcs[src_idx]);
957 return -ENOMEM;
958 }
959
960 /* Fill in src buffers */
961 for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++) {
962 u8 *ptr = page_address(xor_srcs[src_idx]);
963 for (i = 0; i < PAGE_SIZE; i++)
964 ptr[i] = (1 << src_idx);
965 }
966
967 for (src_idx = 0; src_idx < IOP_ADMA_NUM_SRC_TEST; src_idx++)
968 cmp_byte ^= (u8) (1 << src_idx);
969
970 cmp_word = (cmp_byte << 24) | (cmp_byte << 16) |
971 (cmp_byte << 8) | cmp_byte;
972
973 memset(page_address(dest), 0, PAGE_SIZE);
974
975 dma_chan = container_of(device->common.channels.next,
976 struct dma_chan,
977 device_node);
978 if (iop_adma_alloc_chan_resources(dma_chan) < 1) {
979 err = -ENODEV;
980 goto out;
981 }
982
983 /* test xor */
984 tx = iop_adma_prep_dma_xor(dma_chan, IOP_ADMA_NUM_SRC_TEST,
985 PAGE_SIZE, 1);
986 dest_dma = dma_map_page(dma_chan->device->dev, dest, 0,
987 PAGE_SIZE, DMA_FROM_DEVICE);
988 iop_adma_set_dest(dest_dma, tx, 0);
989
990 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++) {
991 dma_addr = dma_map_page(dma_chan->device->dev, xor_srcs[i], 0,
992 PAGE_SIZE, DMA_TO_DEVICE);
993 iop_adma_xor_set_src(dma_addr, tx, i);
994 }
995
996 cookie = iop_adma_tx_submit(tx);
997 iop_adma_issue_pending(dma_chan);
998 async_tx_ack(tx);
999 msleep(8);
1000
1001 if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) !=
1002 DMA_SUCCESS) {
1003 dev_printk(KERN_ERR, dma_chan->device->dev,
1004 "Self-test xor timed out, disabling\n");
1005 err = -ENODEV;
1006 goto free_resources;
1007 }
1008
1009 iop_chan = to_iop_adma_chan(dma_chan);
1010 dma_sync_single_for_cpu(&iop_chan->device->pdev->dev, dest_dma,
1011 PAGE_SIZE, DMA_FROM_DEVICE);
1012 for (i = 0; i < (PAGE_SIZE / sizeof(u32)); i++) {
1013 u32 *ptr = page_address(dest);
1014 if (ptr[i] != cmp_word) {
1015 dev_printk(KERN_ERR, dma_chan->device->dev,
1016 "Self-test xor failed compare, disabling\n");
1017 err = -ENODEV;
1018 goto free_resources;
1019 }
1020 }
1021 dma_sync_single_for_device(&iop_chan->device->pdev->dev, dest_dma,
1022 PAGE_SIZE, DMA_TO_DEVICE);
1023
1024 /* skip zero sum if the capability is not present */
1025 if (!dma_has_cap(DMA_ZERO_SUM, dma_chan->device->cap_mask))
1026 goto free_resources;
1027
1028 /* zero sum the sources with the destintation page */
1029 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST; i++)
1030 zero_sum_srcs[i] = xor_srcs[i];
1031 zero_sum_srcs[i] = dest;
1032
1033 zero_sum_result = 1;
1034
1035 tx = iop_adma_prep_dma_zero_sum(dma_chan, IOP_ADMA_NUM_SRC_TEST + 1,
1036 PAGE_SIZE, &zero_sum_result, 1);
1037 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++) {
1038 dma_addr = dma_map_page(dma_chan->device->dev, zero_sum_srcs[i],
1039 0, PAGE_SIZE, DMA_TO_DEVICE);
1040 iop_adma_xor_zero_sum_set_src(dma_addr, tx, i);
1041 }
1042
1043 cookie = iop_adma_tx_submit(tx);
1044 iop_adma_issue_pending(dma_chan);
1045 async_tx_ack(tx);
1046 msleep(8);
1047
1048 if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
1049 dev_printk(KERN_ERR, dma_chan->device->dev,
1050 "Self-test zero sum timed out, disabling\n");
1051 err = -ENODEV;
1052 goto free_resources;
1053 }
1054
1055 if (zero_sum_result != 0) {
1056 dev_printk(KERN_ERR, dma_chan->device->dev,
1057 "Self-test zero sum failed compare, disabling\n");
1058 err = -ENODEV;
1059 goto free_resources;
1060 }
1061
1062 /* test memset */
1063 tx = iop_adma_prep_dma_memset(dma_chan, 0, PAGE_SIZE, 1);
1064 dma_addr = dma_map_page(dma_chan->device->dev, dest, 0,
1065 PAGE_SIZE, DMA_FROM_DEVICE);
1066 iop_adma_set_dest(dma_addr, tx, 0);
1067
1068 cookie = iop_adma_tx_submit(tx);
1069 iop_adma_issue_pending(dma_chan);
1070 async_tx_ack(tx);
1071 msleep(8);
1072
1073 if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
1074 dev_printk(KERN_ERR, dma_chan->device->dev,
1075 "Self-test memset timed out, disabling\n");
1076 err = -ENODEV;
1077 goto free_resources;
1078 }
1079
1080 for (i = 0; i < PAGE_SIZE/sizeof(u32); i++) {
1081 u32 *ptr = page_address(dest);
1082 if (ptr[i]) {
1083 dev_printk(KERN_ERR, dma_chan->device->dev,
1084 "Self-test memset failed compare, disabling\n");
1085 err = -ENODEV;
1086 goto free_resources;
1087 }
1088 }
1089
1090 /* test for non-zero parity sum */
1091 zero_sum_result = 0;
1092 tx = iop_adma_prep_dma_zero_sum(dma_chan, IOP_ADMA_NUM_SRC_TEST + 1,
1093 PAGE_SIZE, &zero_sum_result, 1);
1094 for (i = 0; i < IOP_ADMA_NUM_SRC_TEST + 1; i++) {
1095 dma_addr = dma_map_page(dma_chan->device->dev, zero_sum_srcs[i],
1096 0, PAGE_SIZE, DMA_TO_DEVICE);
1097 iop_adma_xor_zero_sum_set_src(dma_addr, tx, i);
1098 }
1099
1100 cookie = iop_adma_tx_submit(tx);
1101 iop_adma_issue_pending(dma_chan);
1102 async_tx_ack(tx);
1103 msleep(8);
1104
1105 if (iop_adma_is_complete(dma_chan, cookie, NULL, NULL) != DMA_SUCCESS) {
1106 dev_printk(KERN_ERR, dma_chan->device->dev,
1107 "Self-test non-zero sum timed out, disabling\n");
1108 err = -ENODEV;
1109 goto free_resources;
1110 }
1111
1112 if (zero_sum_result != 1) {
1113 dev_printk(KERN_ERR, dma_chan->device->dev,
1114 "Self-test non-zero sum failed compare, disabling\n");
1115 err = -ENODEV;
1116 goto free_resources;
1117 }
1118
1119free_resources:
1120 iop_adma_free_chan_resources(dma_chan);
1121out:
1122 src_idx = IOP_ADMA_NUM_SRC_TEST;
1123 while (src_idx--)
1124 __free_page(xor_srcs[src_idx]);
1125 __free_page(dest);
1126 return err;
1127}
1128
1129static int __devexit iop_adma_remove(struct platform_device *dev)
1130{
1131 struct iop_adma_device *device = platform_get_drvdata(dev);
1132 struct dma_chan *chan, *_chan;
1133 struct iop_adma_chan *iop_chan;
1134 int i;
1135 struct iop_adma_platform_data *plat_data = dev->dev.platform_data;
1136
1137 dma_async_device_unregister(&device->common);
1138
1139 for (i = 0; i < 3; i++) {
1140 unsigned int irq;
1141 irq = platform_get_irq(dev, i);
1142 free_irq(irq, device);
1143 }
1144
1145 dma_free_coherent(&dev->dev, plat_data->pool_size,
1146 device->dma_desc_pool_virt, device->dma_desc_pool);
1147
1148 do {
1149 struct resource *res;
1150 res = platform_get_resource(dev, IORESOURCE_MEM, 0);
1151 release_mem_region(res->start, res->end - res->start);
1152 } while (0);
1153
1154 list_for_each_entry_safe(chan, _chan, &device->common.channels,
1155 device_node) {
1156 iop_chan = to_iop_adma_chan(chan);
1157 list_del(&chan->device_node);
1158 kfree(iop_chan);
1159 }
1160 kfree(device);
1161
1162 return 0;
1163}
1164
1165static int __devinit iop_adma_probe(struct platform_device *pdev)
1166{
1167 struct resource *res;
1168 int ret = 0, i;
1169 struct iop_adma_device *adev;
1170 struct iop_adma_chan *iop_chan;
1171 struct dma_device *dma_dev;
1172 struct iop_adma_platform_data *plat_data = pdev->dev.platform_data;
1173
1174 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1175 if (!res)
1176 return -ENODEV;
1177
1178 if (!devm_request_mem_region(&pdev->dev, res->start,
1179 res->end - res->start, pdev->name))
1180 return -EBUSY;
1181
1182 adev = kzalloc(sizeof(*adev), GFP_KERNEL);
1183 if (!adev)
1184 return -ENOMEM;
1185 dma_dev = &adev->common;
1186
1187 /* allocate coherent memory for hardware descriptors
1188 * note: writecombine gives slightly better performance, but
1189 * requires that we explicitly flush the writes
1190 */
1191 if ((adev->dma_desc_pool_virt = dma_alloc_writecombine(&pdev->dev,
1192 plat_data->pool_size,
1193 &adev->dma_desc_pool,
1194 GFP_KERNEL)) == NULL) {
1195 ret = -ENOMEM;
1196 goto err_free_adev;
1197 }
1198
1199 dev_dbg(&pdev->dev, "%s: allocted descriptor pool virt %p phys %p\n",
1200 __FUNCTION__, adev->dma_desc_pool_virt,
1201 (void *) adev->dma_desc_pool);
1202
1203 adev->id = plat_data->hw_id;
1204
1205 /* discover transaction capabilites from the platform data */
1206 dma_dev->cap_mask = plat_data->cap_mask;
1207
1208 adev->pdev = pdev;
1209 platform_set_drvdata(pdev, adev);
1210
1211 INIT_LIST_HEAD(&dma_dev->channels);
1212
1213 /* set base routines */
1214 dma_dev->device_alloc_chan_resources = iop_adma_alloc_chan_resources;
1215 dma_dev->device_free_chan_resources = iop_adma_free_chan_resources;
1216 dma_dev->device_is_tx_complete = iop_adma_is_complete;
1217 dma_dev->device_issue_pending = iop_adma_issue_pending;
1218 dma_dev->device_dependency_added = iop_adma_dependency_added;
1219 dma_dev->dev = &pdev->dev;
1220
1221 /* set prep routines based on capability */
1222 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask))
1223 dma_dev->device_prep_dma_memcpy = iop_adma_prep_dma_memcpy;
1224 if (dma_has_cap(DMA_MEMSET, dma_dev->cap_mask))
1225 dma_dev->device_prep_dma_memset = iop_adma_prep_dma_memset;
1226 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) {
1227 dma_dev->max_xor = iop_adma_get_max_xor();
1228 dma_dev->device_prep_dma_xor = iop_adma_prep_dma_xor;
1229 }
1230 if (dma_has_cap(DMA_ZERO_SUM, dma_dev->cap_mask))
1231 dma_dev->device_prep_dma_zero_sum =
1232 iop_adma_prep_dma_zero_sum;
1233 if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask))
1234 dma_dev->device_prep_dma_interrupt =
1235 iop_adma_prep_dma_interrupt;
1236
1237 iop_chan = kzalloc(sizeof(*iop_chan), GFP_KERNEL);
1238 if (!iop_chan) {
1239 ret = -ENOMEM;
1240 goto err_free_dma;
1241 }
1242 iop_chan->device = adev;
1243
1244 iop_chan->mmr_base = devm_ioremap(&pdev->dev, res->start,
1245 res->end - res->start);
1246 if (!iop_chan->mmr_base) {
1247 ret = -ENOMEM;
1248 goto err_free_iop_chan;
1249 }
1250 tasklet_init(&iop_chan->irq_tasklet, iop_adma_tasklet, (unsigned long)
1251 iop_chan);
1252
1253 /* clear errors before enabling interrupts */
1254 iop_adma_device_clear_err_status(iop_chan);
1255
1256 for (i = 0; i < 3; i++) {
1257 irq_handler_t handler[] = { iop_adma_eot_handler,
1258 iop_adma_eoc_handler,
1259 iop_adma_err_handler };
1260 int irq = platform_get_irq(pdev, i);
1261 if (irq < 0) {
1262 ret = -ENXIO;
1263 goto err_free_iop_chan;
1264 } else {
1265 ret = devm_request_irq(&pdev->dev, irq,
1266 handler[i], 0, pdev->name, iop_chan);
1267 if (ret)
1268 goto err_free_iop_chan;
1269 }
1270 }
1271
1272 spin_lock_init(&iop_chan->lock);
1273 init_timer(&iop_chan->cleanup_watchdog);
1274 iop_chan->cleanup_watchdog.data = (unsigned long) iop_chan;
1275 iop_chan->cleanup_watchdog.function = iop_adma_tasklet;
1276 INIT_LIST_HEAD(&iop_chan->chain);
1277 INIT_LIST_HEAD(&iop_chan->all_slots);
1278 INIT_RCU_HEAD(&iop_chan->common.rcu);
1279 iop_chan->common.device = dma_dev;
1280 list_add_tail(&iop_chan->common.device_node, &dma_dev->channels);
1281
1282 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) {
1283 ret = iop_adma_memcpy_self_test(adev);
1284 dev_dbg(&pdev->dev, "memcpy self test returned %d\n", ret);
1285 if (ret)
1286 goto err_free_iop_chan;
1287 }
1288
1289 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask) ||
1290 dma_has_cap(DMA_MEMSET, dma_dev->cap_mask)) {
1291 ret = iop_adma_xor_zero_sum_self_test(adev);
1292 dev_dbg(&pdev->dev, "xor self test returned %d\n", ret);
1293 if (ret)
1294 goto err_free_iop_chan;
1295 }
1296
1297 dev_printk(KERN_INFO, &pdev->dev, "Intel(R) IOP: "
1298 "( %s%s%s%s%s%s%s%s%s%s)\n",
1299 dma_has_cap(DMA_PQ_XOR, dma_dev->cap_mask) ? "pq_xor " : "",
1300 dma_has_cap(DMA_PQ_UPDATE, dma_dev->cap_mask) ? "pq_update " : "",
1301 dma_has_cap(DMA_PQ_ZERO_SUM, dma_dev->cap_mask) ? "pq_zero_sum " : "",
1302 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "",
1303 dma_has_cap(DMA_DUAL_XOR, dma_dev->cap_mask) ? "dual_xor " : "",
1304 dma_has_cap(DMA_ZERO_SUM, dma_dev->cap_mask) ? "xor_zero_sum " : "",
1305 dma_has_cap(DMA_MEMSET, dma_dev->cap_mask) ? "fill " : "",
1306 dma_has_cap(DMA_MEMCPY_CRC32C, dma_dev->cap_mask) ? "cpy+crc " : "",
1307 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "cpy " : "",
1308 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "intr " : "");
1309
1310 dma_async_device_register(dma_dev);
1311 goto out;
1312
1313 err_free_iop_chan:
1314 kfree(iop_chan);
1315 err_free_dma:
1316 dma_free_coherent(&adev->pdev->dev, plat_data->pool_size,
1317 adev->dma_desc_pool_virt, adev->dma_desc_pool);
1318 err_free_adev:
1319 kfree(adev);
1320 out:
1321 return ret;
1322}
1323
1324static void iop_chan_start_null_memcpy(struct iop_adma_chan *iop_chan)
1325{
1326 struct iop_adma_desc_slot *sw_desc, *grp_start;
1327 dma_cookie_t cookie;
1328 int slot_cnt, slots_per_op;
1329
1330 dev_dbg(iop_chan->device->common.dev, "%s\n", __FUNCTION__);
1331
1332 spin_lock_bh(&iop_chan->lock);
1333 slot_cnt = iop_chan_memcpy_slot_count(0, &slots_per_op);
1334 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
1335 if (sw_desc) {
1336 grp_start = sw_desc->group_head;
1337
1338 list_splice_init(&sw_desc->async_tx.tx_list, &iop_chan->chain);
1339 sw_desc->async_tx.ack = 1;
1340 iop_desc_init_memcpy(grp_start, 0);
1341 iop_desc_set_byte_count(grp_start, iop_chan, 0);
1342 iop_desc_set_dest_addr(grp_start, iop_chan, 0);
1343 iop_desc_set_memcpy_src_addr(grp_start, 0);
1344
1345 cookie = iop_chan->common.cookie;
1346 cookie++;
1347 if (cookie <= 1)
1348 cookie = 2;
1349
1350 /* initialize the completed cookie to be less than
1351 * the most recently used cookie
1352 */
1353 iop_chan->completed_cookie = cookie - 1;
1354 iop_chan->common.cookie = sw_desc->async_tx.cookie = cookie;
1355
1356 /* channel should not be busy */
1357 BUG_ON(iop_chan_is_busy(iop_chan));
1358
1359 /* clear any prior error-status bits */
1360 iop_adma_device_clear_err_status(iop_chan);
1361
1362 /* disable operation */
1363 iop_chan_disable(iop_chan);
1364
1365 /* set the descriptor address */
1366 iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys);
1367
1368 /* 1/ don't add pre-chained descriptors
1369 * 2/ dummy read to flush next_desc write
1370 */
1371 BUG_ON(iop_desc_get_next_desc(sw_desc));
1372
1373 /* run the descriptor */
1374 iop_chan_enable(iop_chan);
1375 } else
1376 dev_printk(KERN_ERR, iop_chan->device->common.dev,
1377 "failed to allocate null descriptor\n");
1378 spin_unlock_bh(&iop_chan->lock);
1379}
1380
1381static void iop_chan_start_null_xor(struct iop_adma_chan *iop_chan)
1382{
1383 struct iop_adma_desc_slot *sw_desc, *grp_start;
1384 dma_cookie_t cookie;
1385 int slot_cnt, slots_per_op;
1386
1387 dev_dbg(iop_chan->device->common.dev, "%s\n", __FUNCTION__);
1388
1389 spin_lock_bh(&iop_chan->lock);
1390 slot_cnt = iop_chan_xor_slot_count(0, 2, &slots_per_op);
1391 sw_desc = iop_adma_alloc_slots(iop_chan, slot_cnt, slots_per_op);
1392 if (sw_desc) {
1393 grp_start = sw_desc->group_head;
1394 list_splice_init(&sw_desc->async_tx.tx_list, &iop_chan->chain);
1395 sw_desc->async_tx.ack = 1;
1396 iop_desc_init_null_xor(grp_start, 2, 0);
1397 iop_desc_set_byte_count(grp_start, iop_chan, 0);
1398 iop_desc_set_dest_addr(grp_start, iop_chan, 0);
1399 iop_desc_set_xor_src_addr(grp_start, 0, 0);
1400 iop_desc_set_xor_src_addr(grp_start, 1, 0);
1401
1402 cookie = iop_chan->common.cookie;
1403 cookie++;
1404 if (cookie <= 1)
1405 cookie = 2;
1406
1407 /* initialize the completed cookie to be less than
1408 * the most recently used cookie
1409 */
1410 iop_chan->completed_cookie = cookie - 1;
1411 iop_chan->common.cookie = sw_desc->async_tx.cookie = cookie;
1412
1413 /* channel should not be busy */
1414 BUG_ON(iop_chan_is_busy(iop_chan));
1415
1416 /* clear any prior error-status bits */
1417 iop_adma_device_clear_err_status(iop_chan);
1418
1419 /* disable operation */
1420 iop_chan_disable(iop_chan);
1421
1422 /* set the descriptor address */
1423 iop_chan_set_next_descriptor(iop_chan, sw_desc->async_tx.phys);
1424
1425 /* 1/ don't add pre-chained descriptors
1426 * 2/ dummy read to flush next_desc write
1427 */
1428 BUG_ON(iop_desc_get_next_desc(sw_desc));
1429
1430 /* run the descriptor */
1431 iop_chan_enable(iop_chan);
1432 } else
1433 dev_printk(KERN_ERR, iop_chan->device->common.dev,
1434 "failed to allocate null descriptor\n");
1435 spin_unlock_bh(&iop_chan->lock);
1436}
1437
1438static struct platform_driver iop_adma_driver = {
1439 .probe = iop_adma_probe,
1440 .remove = iop_adma_remove,
1441 .driver = {
1442 .owner = THIS_MODULE,
1443 .name = "iop-adma",
1444 },
1445};
1446
1447static int __init iop_adma_init (void)
1448{
1449 /* it's currently unsafe to unload this module */
1450 /* if forced, worst case is that rmmod hangs */
1451 __unsafe(THIS_MODULE);
1452
1453 return platform_driver_register(&iop_adma_driver);
1454}
1455
1456static void __exit iop_adma_exit (void)
1457{
1458 platform_driver_unregister(&iop_adma_driver);
1459 return;
1460}
1461
1462module_init(iop_adma_init);
1463module_exit(iop_adma_exit);
1464
1465MODULE_AUTHOR("Intel Corporation");
1466MODULE_DESCRIPTION("IOP ADMA Engine Driver");
1467MODULE_LICENSE("GPL");
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-28 13:24:51 -0400