aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/dma/dmaengine.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/dma/dmaengine.c')
-rw-r--r--drivers/dma/dmaengine.c168
1 files changed, 167 insertions, 1 deletions
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c
index d4d925912c47..87a8cd4791ed 100644
--- a/drivers/dma/dmaengine.c
+++ b/drivers/dma/dmaengine.c
@@ -295,6 +295,164 @@ static void dma_chan_release(struct dma_chan *chan)
295} 295}
296 296
297/** 297/**
298 * dma_cap_mask_all - enable iteration over all operation types
299 */
300static dma_cap_mask_t dma_cap_mask_all;
301
302/**
303 * dma_chan_tbl_ent - tracks channel allocations per core/operation
304 * @chan - associated channel for this entry
305 */
306struct dma_chan_tbl_ent {
307 struct dma_chan *chan;
308};
309
310/**
311 * channel_table - percpu lookup table for memory-to-memory offload providers
312 */
313static struct dma_chan_tbl_ent *channel_table[DMA_TX_TYPE_END];
314
315static int __init dma_channel_table_init(void)
316{
317 enum dma_transaction_type cap;
318 int err = 0;
319
320 bitmap_fill(dma_cap_mask_all.bits, DMA_TX_TYPE_END);
321
322 /* 'interrupt' and 'slave' are channel capabilities, but are not
323 * associated with an operation so they do not need an entry in the
324 * channel_table
325 */
326 clear_bit(DMA_INTERRUPT, dma_cap_mask_all.bits);
327 clear_bit(DMA_SLAVE, dma_cap_mask_all.bits);
328
329 for_each_dma_cap_mask(cap, dma_cap_mask_all) {
330 channel_table[cap] = alloc_percpu(struct dma_chan_tbl_ent);
331 if (!channel_table[cap]) {
332 err = -ENOMEM;
333 break;
334 }
335 }
336
337 if (err) {
338 pr_err("dmaengine: initialization failure\n");
339 for_each_dma_cap_mask(cap, dma_cap_mask_all)
340 if (channel_table[cap])
341 free_percpu(channel_table[cap]);
342 }
343
344 return err;
345}
346subsys_initcall(dma_channel_table_init);
347
348/**
349 * dma_find_channel - find a channel to carry out the operation
350 * @tx_type: transaction type
351 */
352struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type)
353{
354 struct dma_chan *chan;
355 int cpu;
356
357 WARN_ONCE(dmaengine_ref_count == 0,
358 "client called %s without a reference", __func__);
359
360 cpu = get_cpu();
361 chan = per_cpu_ptr(channel_table[tx_type], cpu)->chan;
362 put_cpu();
363
364 return chan;
365}
366EXPORT_SYMBOL(dma_find_channel);
367
368/**
369 * nth_chan - returns the nth channel of the given capability
370 * @cap: capability to match
371 * @n: nth channel desired
372 *
373 * Defaults to returning the channel with the desired capability and the
374 * lowest reference count when 'n' cannot be satisfied. Must be called
375 * under dma_list_mutex.
376 */
377static struct dma_chan *nth_chan(enum dma_transaction_type cap, int n)
378{
379 struct dma_device *device;
380 struct dma_chan *chan;
381 struct dma_chan *ret = NULL;
382 struct dma_chan *min = NULL;
383
384 list_for_each_entry(device, &dma_device_list, global_node) {
385 if (!dma_has_cap(cap, device->cap_mask))
386 continue;
387 list_for_each_entry(chan, &device->channels, device_node) {
388 if (!chan->client_count)
389 continue;
390 if (!min)
391 min = chan;
392 else if (chan->table_count < min->table_count)
393 min = chan;
394
395 if (n-- == 0) {
396 ret = chan;
397 break; /* done */
398 }
399 }
400 if (ret)
401 break; /* done */
402 }
403
404 if (!ret)
405 ret = min;
406
407 if (ret)
408 ret->table_count++;
409
410 return ret;
411}
412
413/**
414 * dma_channel_rebalance - redistribute the available channels
415 *
416 * Optimize for cpu isolation (each cpu gets a dedicated channel for an
417 * operation type) in the SMP case, and operation isolation (avoid
418 * multi-tasking channels) in the non-SMP case. Must be called under
419 * dma_list_mutex.
420 */
421static void dma_channel_rebalance(void)
422{
423 struct dma_chan *chan;
424 struct dma_device *device;
425 int cpu;
426 int cap;
427 int n;
428
429 /* undo the last distribution */
430 for_each_dma_cap_mask(cap, dma_cap_mask_all)
431 for_each_possible_cpu(cpu)
432 per_cpu_ptr(channel_table[cap], cpu)->chan = NULL;
433
434 list_for_each_entry(device, &dma_device_list, global_node)
435 list_for_each_entry(chan, &device->channels, device_node)
436 chan->table_count = 0;
437
438 /* don't populate the channel_table if no clients are available */
439 if (!dmaengine_ref_count)
440 return;
441
442 /* redistribute available channels */
443 n = 0;
444 for_each_dma_cap_mask(cap, dma_cap_mask_all)
445 for_each_online_cpu(cpu) {
446 if (num_possible_cpus() > 1)
447 chan = nth_chan(cap, n++);
448 else
449 chan = nth_chan(cap, -1);
450
451 per_cpu_ptr(channel_table[cap], cpu)->chan = chan;
452 }
453}
454
455/**
298 * dma_chans_notify_available - broadcast available channels to the clients 456 * dma_chans_notify_available - broadcast available channels to the clients
299 */ 457 */
300static void dma_clients_notify_available(void) 458static void dma_clients_notify_available(void)
@@ -339,7 +497,12 @@ void dma_async_client_register(struct dma_client *client)
339 dev_name(&chan->dev), err); 497 dev_name(&chan->dev), err);
340 } 498 }
341 499
342 500 /* if this is the first reference and there were channels
501 * waiting we need to rebalance to get those channels
502 * incorporated into the channel table
503 */
504 if (dmaengine_ref_count == 1)
505 dma_channel_rebalance();
343 list_add_tail(&client->global_node, &dma_client_list); 506 list_add_tail(&client->global_node, &dma_client_list);
344 mutex_unlock(&dma_list_mutex); 507 mutex_unlock(&dma_list_mutex);
345} 508}
@@ -473,6 +636,7 @@ int dma_async_device_register(struct dma_device *device)
473 } 636 }
474 } 637 }
475 list_add_tail(&device->global_node, &dma_device_list); 638 list_add_tail(&device->global_node, &dma_device_list);
639 dma_channel_rebalance();
476 mutex_unlock(&dma_list_mutex); 640 mutex_unlock(&dma_list_mutex);
477 641
478 dma_clients_notify_available(); 642 dma_clients_notify_available();
@@ -514,6 +678,7 @@ void dma_async_device_unregister(struct dma_device *device)
514 678
515 mutex_lock(&dma_list_mutex); 679 mutex_lock(&dma_list_mutex);
516 list_del(&device->global_node); 680 list_del(&device->global_node);
681 dma_channel_rebalance();
517 mutex_unlock(&dma_list_mutex); 682 mutex_unlock(&dma_list_mutex);
518 683
519 list_for_each_entry(chan, &device->channels, device_node) { 684 list_for_each_entry(chan, &device->channels, device_node) {
@@ -768,3 +933,4 @@ static int __init dma_bus_init(void)
768} 933}
769subsys_initcall(dma_bus_init); 934subsys_initcall(dma_bus_init);
770 935
936