diff options
Diffstat (limited to 'drivers/dma')
-rw-r--r-- | drivers/dma/dmaengine.c | 168 |
1 files changed, 167 insertions, 1 deletions
diff --git a/drivers/dma/dmaengine.c b/drivers/dma/dmaengine.c index d4d925912c47..87a8cd4791ed 100644 --- a/drivers/dma/dmaengine.c +++ b/drivers/dma/dmaengine.c | |||
@@ -295,6 +295,164 @@ static void dma_chan_release(struct dma_chan *chan) | |||
295 | } | 295 | } |
296 | 296 | ||
297 | /** | 297 | /** |
298 | * dma_cap_mask_all - enable iteration over all operation types | ||
299 | */ | ||
300 | static dma_cap_mask_t dma_cap_mask_all; | ||
301 | |||
302 | /** | ||
303 | * dma_chan_tbl_ent - tracks channel allocations per core/operation | ||
304 | * @chan - associated channel for this entry | ||
305 | */ | ||
306 | struct dma_chan_tbl_ent { | ||
307 | struct dma_chan *chan; | ||
308 | }; | ||
309 | |||
310 | /** | ||
311 | * channel_table - percpu lookup table for memory-to-memory offload providers | ||
312 | */ | ||
313 | static struct dma_chan_tbl_ent *channel_table[DMA_TX_TYPE_END]; | ||
314 | |||
315 | static int __init dma_channel_table_init(void) | ||
316 | { | ||
317 | enum dma_transaction_type cap; | ||
318 | int err = 0; | ||
319 | |||
320 | bitmap_fill(dma_cap_mask_all.bits, DMA_TX_TYPE_END); | ||
321 | |||
322 | /* 'interrupt' and 'slave' are channel capabilities, but are not | ||
323 | * associated with an operation so they do not need an entry in the | ||
324 | * channel_table | ||
325 | */ | ||
326 | clear_bit(DMA_INTERRUPT, dma_cap_mask_all.bits); | ||
327 | clear_bit(DMA_SLAVE, dma_cap_mask_all.bits); | ||
328 | |||
329 | for_each_dma_cap_mask(cap, dma_cap_mask_all) { | ||
330 | channel_table[cap] = alloc_percpu(struct dma_chan_tbl_ent); | ||
331 | if (!channel_table[cap]) { | ||
332 | err = -ENOMEM; | ||
333 | break; | ||
334 | } | ||
335 | } | ||
336 | |||
337 | if (err) { | ||
338 | pr_err("dmaengine: initialization failure\n"); | ||
339 | for_each_dma_cap_mask(cap, dma_cap_mask_all) | ||
340 | if (channel_table[cap]) | ||
341 | free_percpu(channel_table[cap]); | ||
342 | } | ||
343 | |||
344 | return err; | ||
345 | } | ||
346 | subsys_initcall(dma_channel_table_init); | ||
347 | |||
348 | /** | ||
349 | * dma_find_channel - find a channel to carry out the operation | ||
350 | * @tx_type: transaction type | ||
351 | */ | ||
352 | struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type) | ||
353 | { | ||
354 | struct dma_chan *chan; | ||
355 | int cpu; | ||
356 | |||
357 | WARN_ONCE(dmaengine_ref_count == 0, | ||
358 | "client called %s without a reference", __func__); | ||
359 | |||
360 | cpu = get_cpu(); | ||
361 | chan = per_cpu_ptr(channel_table[tx_type], cpu)->chan; | ||
362 | put_cpu(); | ||
363 | |||
364 | return chan; | ||
365 | } | ||
366 | EXPORT_SYMBOL(dma_find_channel); | ||
367 | |||
368 | /** | ||
369 | * nth_chan - returns the nth channel of the given capability | ||
370 | * @cap: capability to match | ||
371 | * @n: nth channel desired | ||
372 | * | ||
373 | * Defaults to returning the channel with the desired capability and the | ||
374 | * lowest reference count when 'n' cannot be satisfied. Must be called | ||
375 | * under dma_list_mutex. | ||
376 | */ | ||
377 | static struct dma_chan *nth_chan(enum dma_transaction_type cap, int n) | ||
378 | { | ||
379 | struct dma_device *device; | ||
380 | struct dma_chan *chan; | ||
381 | struct dma_chan *ret = NULL; | ||
382 | struct dma_chan *min = NULL; | ||
383 | |||
384 | list_for_each_entry(device, &dma_device_list, global_node) { | ||
385 | if (!dma_has_cap(cap, device->cap_mask)) | ||
386 | continue; | ||
387 | list_for_each_entry(chan, &device->channels, device_node) { | ||
388 | if (!chan->client_count) | ||
389 | continue; | ||
390 | if (!min) | ||
391 | min = chan; | ||
392 | else if (chan->table_count < min->table_count) | ||
393 | min = chan; | ||
394 | |||
395 | if (n-- == 0) { | ||
396 | ret = chan; | ||
397 | break; /* done */ | ||
398 | } | ||
399 | } | ||
400 | if (ret) | ||
401 | break; /* done */ | ||
402 | } | ||
403 | |||
404 | if (!ret) | ||
405 | ret = min; | ||
406 | |||
407 | if (ret) | ||
408 | ret->table_count++; | ||
409 | |||
410 | return ret; | ||
411 | } | ||
412 | |||
413 | /** | ||
414 | * dma_channel_rebalance - redistribute the available channels | ||
415 | * | ||
416 | * Optimize for cpu isolation (each cpu gets a dedicated channel for an | ||
417 | * operation type) in the SMP case, and operation isolation (avoid | ||
418 | * multi-tasking channels) in the non-SMP case. Must be called under | ||
419 | * dma_list_mutex. | ||
420 | */ | ||
421 | static void dma_channel_rebalance(void) | ||
422 | { | ||
423 | struct dma_chan *chan; | ||
424 | struct dma_device *device; | ||
425 | int cpu; | ||
426 | int cap; | ||
427 | int n; | ||
428 | |||
429 | /* undo the last distribution */ | ||
430 | for_each_dma_cap_mask(cap, dma_cap_mask_all) | ||
431 | for_each_possible_cpu(cpu) | ||
432 | per_cpu_ptr(channel_table[cap], cpu)->chan = NULL; | ||
433 | |||
434 | list_for_each_entry(device, &dma_device_list, global_node) | ||
435 | list_for_each_entry(chan, &device->channels, device_node) | ||
436 | chan->table_count = 0; | ||
437 | |||
438 | /* don't populate the channel_table if no clients are available */ | ||
439 | if (!dmaengine_ref_count) | ||
440 | return; | ||
441 | |||
442 | /* redistribute available channels */ | ||
443 | n = 0; | ||
444 | for_each_dma_cap_mask(cap, dma_cap_mask_all) | ||
445 | for_each_online_cpu(cpu) { | ||
446 | if (num_possible_cpus() > 1) | ||
447 | chan = nth_chan(cap, n++); | ||
448 | else | ||
449 | chan = nth_chan(cap, -1); | ||
450 | |||
451 | per_cpu_ptr(channel_table[cap], cpu)->chan = chan; | ||
452 | } | ||
453 | } | ||
454 | |||
455 | /** | ||
298 | * dma_chans_notify_available - broadcast available channels to the clients | 456 | * dma_chans_notify_available - broadcast available channels to the clients |
299 | */ | 457 | */ |
300 | static void dma_clients_notify_available(void) | 458 | static void dma_clients_notify_available(void) |
@@ -339,7 +497,12 @@ void dma_async_client_register(struct dma_client *client) | |||
339 | dev_name(&chan->dev), err); | 497 | dev_name(&chan->dev), err); |
340 | } | 498 | } |
341 | 499 | ||
342 | 500 | /* if this is the first reference and there were channels | |
501 | * waiting we need to rebalance to get those channels | ||
502 | * incorporated into the channel table | ||
503 | */ | ||
504 | if (dmaengine_ref_count == 1) | ||
505 | dma_channel_rebalance(); | ||
343 | list_add_tail(&client->global_node, &dma_client_list); | 506 | list_add_tail(&client->global_node, &dma_client_list); |
344 | mutex_unlock(&dma_list_mutex); | 507 | mutex_unlock(&dma_list_mutex); |
345 | } | 508 | } |
@@ -473,6 +636,7 @@ int dma_async_device_register(struct dma_device *device) | |||
473 | } | 636 | } |
474 | } | 637 | } |
475 | list_add_tail(&device->global_node, &dma_device_list); | 638 | list_add_tail(&device->global_node, &dma_device_list); |
639 | dma_channel_rebalance(); | ||
476 | mutex_unlock(&dma_list_mutex); | 640 | mutex_unlock(&dma_list_mutex); |
477 | 641 | ||
478 | dma_clients_notify_available(); | 642 | dma_clients_notify_available(); |
@@ -514,6 +678,7 @@ void dma_async_device_unregister(struct dma_device *device) | |||
514 | 678 | ||
515 | mutex_lock(&dma_list_mutex); | 679 | mutex_lock(&dma_list_mutex); |
516 | list_del(&device->global_node); | 680 | list_del(&device->global_node); |
681 | dma_channel_rebalance(); | ||
517 | mutex_unlock(&dma_list_mutex); | 682 | mutex_unlock(&dma_list_mutex); |
518 | 683 | ||
519 | list_for_each_entry(chan, &device->channels, device_node) { | 684 | list_for_each_entry(chan, &device->channels, device_node) { |
@@ -768,3 +933,4 @@ static int __init dma_bus_init(void) | |||
768 | } | 933 | } |
769 | subsys_initcall(dma_bus_init); | 934 | subsys_initcall(dma_bus_init); |
770 | 935 | ||
936 | |||