aboutsummaryrefslogtreecommitdiffstats
path: root/lib/dma-debug.c
diff options
context:
space:
mode:
generated by cgit v1.2.2 (git 2.25.0) at 2025-07-05 13:29:29 -0400
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software 17 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 18 */ 19 20#include <linux/scatterlist.h> 21#include <linux/dma-mapping.h> 22#include <linux/stacktrace.h> 23#include <linux/dma-debug.h> 24#include <linux/spinlock.h> 25#include <linux/debugfs.h> 26#include <linux/uaccess.h> 27#include <linux/device.h> 28#include <linux/types.h> 29#include <linux/sched.h> 30#include <linux/ctype.h> 31#include <linux/list.h> 32#include <linux/slab.h> 33 34#include <asm/sections.h> 35 36#define HASH_SIZE 1024ULL 37#define HASH_FN_SHIFT 13 38#define HASH_FN_MASK (HASH_SIZE - 1) 39 40enum { 41 dma_debug_single, 42 dma_debug_page, 43 dma_debug_sg, 44 dma_debug_coherent, 45}; 46 47#define DMA_DEBUG_STACKTRACE_ENTRIES 5 48 49struct dma_debug_entry { 50 struct list_head list; 51 struct device *dev; 52 int type; 53 phys_addr_t paddr; 54 u64 dev_addr; 55 u64 size; 56 int direction; 57 int sg_call_ents; 58 int sg_mapped_ents; 59#ifdef CONFIG_STACKTRACE 60 struct stack_trace stacktrace; 61 unsigned long st_entries[DMA_DEBUG_STACKTRACE_ENTRIES]; 62#endif 63}; 64 65struct hash_bucket { 66 struct list_head list; 67 spinlock_t lock; 68} ____cacheline_aligned_in_smp; 69 70/* Hash list to save the allocated dma addresses */ 71static struct hash_bucket dma_entry_hash[HASH_SIZE]; 72/* List of pre-allocated dma_debug_entry's */ 73static LIST_HEAD(free_entries); 74/* Lock for the list above */ 75static DEFINE_SPINLOCK(free_entries_lock); 76 77/* Global disable flag - will be set in case of an error */ 78static bool global_disable __read_mostly; 79 80/* Global error count */ 81static u32 error_count; 82 83/* Global error show enable*/ 84static u32 show_all_errors __read_mostly; 85/* Number of errors to show */ 86static u32 show_num_errors = 1; 87 88static u32 num_free_entries; 89static u32 min_free_entries; 90static u32 nr_total_entries; 91 92/* number of preallocated entries requested by kernel cmdline */ 93static u32 req_entries; 94 95/* debugfs dentry's for the stuff above */ 96static struct dentry *dma_debug_dent __read_mostly; 97static struct dentry *global_disable_dent __read_mostly; 98static struct dentry *error_count_dent __read_mostly; 99static struct dentry *show_all_errors_dent __read_mostly; 100static struct dentry *show_num_errors_dent __read_mostly; 101static struct dentry *num_free_entries_dent __read_mostly; 102static struct dentry *min_free_entries_dent __read_mostly; 103static struct dentry *filter_dent __read_mostly; 104 105/* per-driver filter related state */ 106 107#define NAME_MAX_LEN 64 108 109static char current_driver_name[NAME_MAX_LEN] __read_mostly; 110static struct device_driver *current_driver __read_mostly; 111 112static DEFINE_RWLOCK(driver_name_lock); 113 114static const char *type2name[4] = { "single", "page", 115 "scather-gather", "coherent" }; 116 117static const char *dir2name[4] = { "DMA_BIDIRECTIONAL", "DMA_TO_DEVICE", 118 "DMA_FROM_DEVICE", "DMA_NONE" }; 119 120/* little merge helper - remove it after the merge window */ 121#ifndef BUS_NOTIFY_UNBOUND_DRIVER 122#define BUS_NOTIFY_UNBOUND_DRIVER 0x0005 123#endif 124 125/* 126 * The access to some variables in this macro is racy. We can't use atomic_t 127 * here because all these variables are exported to debugfs. Some of them even 128 * writeable. This is also the reason why a lock won't help much. But anyway, 129 * the races are no big deal. Here is why: 130 * 131 * error_count: the addition is racy, but the worst thing that can happen is 132 * that we don't count some errors 133 * show_num_errors: the subtraction is racy. Also no big deal because in 134 * worst case this will result in one warning more in the 135 * system log than the user configured. This variable is 136 * writeable via debugfs. 137 */ 138static inline void dump_entry_trace(struct dma_debug_entry *entry) 139{ 140#ifdef CONFIG_STACKTRACE 141 if (entry) { 142 pr_warning("Mapped at:\n"); 143 print_stack_trace(&entry->stacktrace, 0); 144 } 145#endif 146} 147 148static bool driver_filter(struct device *dev) 149{ 150 struct device_driver *drv; 151 unsigned long flags; 152 bool ret; 153 154 /* driver filter off */ 155 if (likely(!current_driver_name[0])) 156 return true; 157 158 /* driver filter on and initialized */ 159 if (current_driver && dev->driver == current_driver) 160 return true; 161 162 if (current_driver || !current_driver_name[0]) 163 return false; 164 165 /* driver filter on but not yet initialized */ 166 drv = get_driver(dev->driver); 167 if (!drv) 168 return false; 169 170 /* lock to protect against change of current_driver_name */ 171 read_lock_irqsave(&driver_name_lock, flags); 172 173 ret = false; 174 if (drv->name && 175 strncmp(current_driver_name, drv->name, NAME_MAX_LEN - 1) == 0) { 176 current_driver = drv; 177 ret = true; 178 } 179 180 read_unlock_irqrestore(&driver_name_lock, flags); 181 put_driver(drv); 182 183 return ret; 184} 185 186#define err_printk(dev, entry, format, arg...) do { \ 187 error_count += 1; \ 188 if (driver_filter(dev) && \ 189 (show_all_errors || show_num_errors > 0)) { \ 190 WARN(1, "%s %s: " format, \ 191 dev_driver_string(dev), \ 192 dev_name(dev) , ## arg); \ 193 dump_entry_trace(entry); \ 194 } \ 195 if (!show_all_errors && show_num_errors > 0) \ 196 show_num_errors -= 1; \ 197 } while (0); 198 199/* 200 * Hash related functions 201 * 202 * Every DMA-API request is saved into a struct dma_debug_entry. To 203 * have quick access to these structs they are stored into a hash. 204 */ 205static int hash_fn(struct dma_debug_entry *entry) 206{ 207 /* 208 * Hash function is based on the dma address. 209 * We use bits 20-27 here as the index into the hash 210 */ 211 return (entry->dev_addr >> HASH_FN_SHIFT) & HASH_FN_MASK; 212} 213 214/* 215 * Request exclusive access to a hash bucket for a given dma_debug_entry. 216 */ 217static struct hash_bucket *get_hash_bucket(struct dma_debug_entry *entry, 218 unsigned long *flags) 219{ 220 int idx = hash_fn(entry); 221 unsigned long __flags; 222 223 spin_lock_irqsave(&dma_entry_hash[idx].lock, __flags); 224 *flags = __flags; 225 return &dma_entry_hash[idx]; 226} 227 228/* 229 * Give up exclusive access to the hash bucket 230 */ 231static void put_hash_bucket(struct hash_bucket *bucket, 232 unsigned long *flags) 233{ 234 unsigned long __flags = *flags; 235 236 spin_unlock_irqrestore(&bucket->lock, __flags); 237} 238 239/* 240 * Search a given entry in the hash bucket list 241 */ 242static struct dma_debug_entry *hash_bucket_find(struct hash_bucket *bucket, 243 struct dma_debug_entry *ref) 244{ 245 struct dma_debug_entry *entry, *ret = NULL; 246 int matches = 0, match_lvl, last_lvl = 0; 247 248 list_for_each_entry(entry, &bucket->list, list) { 249 if ((entry->dev_addr != ref->dev_addr) || 250 (entry->dev != ref->dev)) 251 continue; 252 253 /* 254 * Some drivers map the same physical address multiple 255 * times. Without a hardware IOMMU this results in the 256 * same device addresses being put into the dma-debug 257 * hash multiple times too. This can result in false 258 * positives being reported. Therfore we implement a 259 * best-fit algorithm here which returns the entry from 260 * the hash which fits best to the reference value 261 * instead of the first-fit. 262 */ 263 matches += 1; 264 match_lvl = 0; 265 entry->size == ref->size ? ++match_lvl : match_lvl; 266 entry->type == ref->type ? ++match_lvl : match_lvl; 267 entry->direction == ref->direction ? ++match_lvl : match_lvl; 268 269 if (match_lvl == 3) { 270 /* perfect-fit - return the result */ 271 return entry; 272 } else if (match_lvl > last_lvl) { 273 /* 274 * We found an entry that fits better then the 275 * previous one 276 */ 277 last_lvl = match_lvl; 278 ret = entry; 279 } 280 } 281 282 /* 283 * If we have multiple matches but no perfect-fit, just return 284 * NULL. 285 */ 286 ret = (matches == 1) ? ret : NULL; 287 288 return ret; 289} 290 291/* 292 * Add an entry to a hash bucket 293 */ 294static void hash_bucket_add(struct hash_bucket *bucket, 295 struct dma_debug_entry *entry) 296{ 297 list_add_tail(&entry->list, &bucket->list); 298} 299 300/* 301 * Remove entry from a hash bucket list 302 */ 303static void hash_bucket_del(struct dma_debug_entry *entry) 304{ 305 list_del(&entry->list); 306} 307 308/* 309 * Dump mapping entries for debugging purposes 310 */ 311void debug_dma_dump_mappings(struct device *dev) 312{ 313 int idx; 314 315 for (idx = 0; idx < HASH_SIZE; idx++) { 316 struct hash_bucket *bucket = &dma_entry_hash[idx]; 317 struct dma_debug_entry *entry; 318 unsigned long flags; 319 320 spin_lock_irqsave(&bucket->lock, flags); 321 322 list_for_each_entry(entry, &bucket->list, list) { 323 if (!dev || dev == entry->dev) { 324 dev_info(entry->dev, 325 "%s idx %d P=%Lx D=%Lx L=%Lx %s\n", 326 type2name[entry->type], idx, 327 (unsigned long long)entry->paddr, 328 entry->dev_addr, entry->size, 329 dir2name[entry->direction]); 330 } 331 } 332 333 spin_unlock_irqrestore(&bucket->lock, flags); 334 } 335} 336EXPORT_SYMBOL(debug_dma_dump_mappings); 337 338/* 339 * Wrapper function for adding an entry to the hash. 340 * This function takes care of locking itself. 341 */ 342static void add_dma_entry(struct dma_debug_entry *entry) 343{ 344 struct hash_bucket *bucket; 345 unsigned long flags; 346 347 bucket = get_hash_bucket(entry, &flags); 348 hash_bucket_add(bucket, entry); 349 put_hash_bucket(bucket, &flags); 350} 351 352static struct dma_debug_entry *__dma_entry_alloc(void) 353{ 354 struct dma_debug_entry *entry; 355 356 entry = list_entry(free_entries.next, struct dma_debug_entry, list); 357 list_del(&entry->list); 358 memset(entry, 0, sizeof(*entry)); 359 360 num_free_entries -= 1; 361 if (num_free_entries < min_free_entries) 362 min_free_entries = num_free_entries; 363 364 return entry; 365} 366 367/* struct dma_entry allocator 368 * 369 * The next two functions implement the allocator for 370 * struct dma_debug_entries. 371 */ 372static struct dma_debug_entry *dma_entry_alloc(void) 373{ 374 struct dma_debug_entry *entry = NULL; 375 unsigned long flags; 376 377 spin_lock_irqsave(&free_entries_lock, flags); 378 379 if (list_empty(&free_entries)) { 380 pr_err("DMA-API: debugging out of memory - disabling\n"); 381 global_disable = true; 382 goto out; 383 } 384 385 entry = __dma_entry_alloc(); 386 387#ifdef CONFIG_STACKTRACE 388 entry->stacktrace.max_entries = DMA_DEBUG_STACKTRACE_ENTRIES; 389 entry->stacktrace.entries = entry->st_entries; 390 entry->stacktrace.skip = 2; 391 save_stack_trace(&entry->stacktrace); 392#endif 393 394out: 395 spin_unlock_irqrestore(&free_entries_lock, flags); 396 397 return entry; 398} 399 400static void dma_entry_free(struct dma_debug_entry *entry) 401{ 402 unsigned long flags; 403 404 /* 405 * add to beginning of the list - this way the entries are 406 * more likely cache hot when they are reallocated. 407 */ 408 spin_lock_irqsave(&free_entries_lock, flags); 409 list_add(&entry->list, &free_entries); 410 num_free_entries += 1; 411 spin_unlock_irqrestore(&free_entries_lock, flags); 412} 413 414int dma_debug_resize_entries(u32 num_entries) 415{ 416 int i, delta, ret = 0; 417 unsigned long flags; 418 struct dma_debug_entry *entry; 419 LIST_HEAD(tmp); 420 421 spin_lock_irqsave(&free_entries_lock, flags); 422 423 if (nr_total_entries < num_entries) { 424 delta = num_entries - nr_total_entries; 425 426 spin_unlock_irqrestore(&free_entries_lock, flags); 427 428 for (i = 0; i < delta; i++) { 429 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 430 if (!entry) 431 break; 432 433 list_add_tail(&entry->list, &tmp); 434 } 435 436 spin_lock_irqsave(&free_entries_lock, flags); 437 438 list_splice(&tmp, &free_entries); 439 nr_total_entries += i; 440 num_free_entries += i; 441 } else { 442 delta = nr_total_entries - num_entries; 443 444 for (i = 0; i < delta && !list_empty(&free_entries); i++) { 445 entry = __dma_entry_alloc(); 446 kfree(entry); 447 } 448 449 nr_total_entries -= i; 450 } 451 452 if (nr_total_entries != num_entries) 453 ret = 1; 454 455 spin_unlock_irqrestore(&free_entries_lock, flags); 456 457 return ret; 458} 459EXPORT_SYMBOL(dma_debug_resize_entries); 460 461/* 462 * DMA-API debugging init code 463 * 464 * The init code does two things: 465 * 1. Initialize core data structures 466 * 2. Preallocate a given number of dma_debug_entry structs 467 */ 468 469static int prealloc_memory(u32 num_entries) 470{ 471 struct dma_debug_entry *entry, *next_entry; 472 int i; 473 474 for (i = 0; i < num_entries; ++i) { 475 entry = kzalloc(sizeof(*entry), GFP_KERNEL); 476 if (!entry) 477 goto out_err; 478 479 list_add_tail(&entry->list, &free_entries); 480 } 481 482 num_free_entries = num_entries; 483 min_free_entries = num_entries; 484 485 pr_info("DMA-API: preallocated %d debug entries\n", num_entries); 486 487 return 0; 488 489out_err: 490 491 list_for_each_entry_safe(entry, next_entry, &free_entries, list) { 492 list_del(&entry->list); 493 kfree(entry); 494 } 495 496 return -ENOMEM; 497} 498 499static ssize_t filter_read(struct file *file, char __user *user_buf, 500 size_t count, loff_t *ppos) 501{ 502 char buf[NAME_MAX_LEN + 1]; 503 unsigned long flags; 504 int len; 505 506 if (!current_driver_name[0]) 507 return 0; 508 509 /* 510 * We can't copy to userspace directly because current_driver_name can 511 * only be read under the driver_name_lock with irqs disabled. So 512 * create a temporary copy first. 513 */ 514 read_lock_irqsave(&driver_name_lock, flags); 515 len = scnprintf(buf, NAME_MAX_LEN + 1, "%s\n", current_driver_name); 516 read_unlock_irqrestore(&driver_name_lock, flags); 517 518 return simple_read_from_buffer(user_buf, count, ppos, buf, len); 519} 520 521static ssize_t filter_write(struct file *file, const char __user *userbuf, 522 size_t count, loff_t *ppos) 523{ 524 char buf[NAME_MAX_LEN]; 525 unsigned long flags; 526 size_t len; 527 int i; 528 529 /* 530 * We can't copy from userspace directly. Access to 531 * current_driver_name is protected with a write_lock with irqs 532 * disabled. Since copy_from_user can fault and may sleep we 533 * need to copy to temporary buffer first 534 */ 535 len = min(count, (size_t)(NAME_MAX_LEN - 1)); 536 if (copy_from_user(buf, userbuf, len)) 537 return -EFAULT; 538 539 buf[len] = 0; 540 541 write_lock_irqsave(&driver_name_lock, flags); 542 543 /* 544 * Now handle the string we got from userspace very carefully. 545 * The rules are: 546 * - only use the first token we got 547 * - token delimiter is everything looking like a space 548 * character (' ', '\n', '\t' ...) 549 * 550 */ 551 if (!isalnum(buf[0])) { 552 /* 553 * If the first character userspace gave us is not 554 * alphanumerical then assume the filter should be 555 * switched off. 556 */ 557 if (current_driver_name[0]) 558 pr_info("DMA-API: switching off dma-debug driver filter\n"); 559 current_driver_name[0] = 0; 560 current_driver = NULL; 561 goto out_unlock; 562 } 563 564 /* 565 * Now parse out the first token and use it as the name for the 566 * driver to filter for. 567 */ 568 for (i = 0; i < NAME_MAX_LEN; ++i) { 569 current_driver_name[i] = buf[i]; 570 if (isspace(buf[i]) || buf[i] == ' ' || buf[i] == 0) 571 break; 572 } 573 current_driver_name[i] = 0; 574 current_driver = NULL; 575 576 pr_info("DMA-API: enable driver filter for driver [%s]\n", 577 current_driver_name); 578 579out_unlock: 580 write_unlock_irqrestore(&driver_name_lock, flags); 581 582 return count; 583} 584 585const struct file_operations filter_fops = { 586 .read = filter_read, 587 .write = filter_write, 588}; 589 590static int dma_debug_fs_init(void) 591{ 592 dma_debug_dent = debugfs_create_dir("dma-api", NULL); 593 if (!dma_debug_dent) { 594 pr_err("DMA-API: can not create debugfs directory\n"); 595 return -ENOMEM; 596 } 597 598 global_disable_dent = debugfs_create_bool("disabled", 0444, 599 dma_debug_dent, 600 (u32 *)&global_disable); 601 if (!global_disable_dent) 602 goto out_err; 603 604 error_count_dent = debugfs_create_u32("error_count", 0444, 605 dma_debug_dent, &error_count); 606 if (!error_count_dent) 607 goto out_err; 608 609 show_all_errors_dent = debugfs_create_u32("all_errors", 0644, 610 dma_debug_dent, 611 &show_all_errors); 612 if (!show_all_errors_dent) 613 goto out_err; 614 615 show_num_errors_dent = debugfs_create_u32("num_errors", 0644, 616 dma_debug_dent, 617 &show_num_errors); 618 if (!show_num_errors_dent) 619 goto out_err; 620 621 num_free_entries_dent = debugfs_create_u32("num_free_entries", 0444, 622 dma_debug_dent, 623 &num_free_entries); 624 if (!num_free_entries_dent) 625 goto out_err; 626 627 min_free_entries_dent = debugfs_create_u32("min_free_entries", 0444, 628 dma_debug_dent, 629 &min_free_entries); 630 if (!min_free_entries_dent) 631 goto out_err; 632 633 filter_dent = debugfs_create_file("driver_filter", 0644, 634 dma_debug_dent, NULL, &filter_fops); 635 if (!filter_dent) 636 goto out_err; 637 638 return 0; 639 640out_err: 641 debugfs_remove_recursive(dma_debug_dent); 642 643 return -ENOMEM; 644} 645 646static int device_dma_allocations(struct device *dev) 647{ 648 struct dma_debug_entry *entry; 649 unsigned long flags; 650 int count = 0, i; 651 652 local_irq_save(flags); 653 654 for (i = 0; i < HASH_SIZE; ++i) { 655 spin_lock(&dma_entry_hash[i].lock); 656 list_for_each_entry(entry, &dma_entry_hash[i].list, list) { 657 if (entry->dev == dev) 658 count += 1; 659 } 660 spin_unlock(&dma_entry_hash[i].lock); 661 } 662 663 local_irq_restore(flags); 664 665 return count; 666} 667 668static int dma_debug_device_change(struct notifier_block *nb, 669 unsigned long action, void *data) 670{ 671 struct device *dev = data; 672 int count; 673 674 675 switch (action) { 676 case BUS_NOTIFY_UNBOUND_DRIVER: 677 count = device_dma_allocations(dev); 678 if (count == 0) 679 break; 680 err_printk(dev, NULL, "DMA-API: device driver has pending " 681 "DMA allocations while released from device " 682 "[count=%d]\n", count); 683 break; 684 default: 685 break; 686 } 687 688 return 0; 689} 690 691void dma_debug_add_bus(struct bus_type *bus) 692{ 693 struct notifier_block *nb; 694 695 nb = kzalloc(sizeof(struct notifier_block), GFP_KERNEL); 696 if (nb == NULL) { 697 pr_err("dma_debug_add_bus: out of memory\n"); 698 return; 699 } 700 701 nb->notifier_call = dma_debug_device_change; 702 703 bus_register_notifier(bus, nb); 704} 705 706/* 707 * Let the architectures decide how many entries should be preallocated. 708 */ 709void dma_debug_init(u32 num_entries) 710{ 711 int i; 712 713 if (global_disable) 714 return; 715 716 for (i = 0; i < HASH_SIZE; ++i) { 717 INIT_LIST_HEAD(&dma_entry_hash[i].list); 718 dma_entry_hash[i].lock = SPIN_LOCK_UNLOCKED; 719 } 720 721 if (dma_debug_fs_init() != 0) { 722 pr_err("DMA-API: error creating debugfs entries - disabling\n"); 723 global_disable = true; 724 725 return; 726 } 727 728 if (req_entries) 729 num_entries = req_entries; 730 731 if (prealloc_memory(num_entries) != 0) { 732 pr_err("DMA-API: debugging out of memory error - disabled\n"); 733 global_disable = true; 734 735 return; 736 } 737 738 nr_total_entries = num_free_entries; 739 740 pr_info("DMA-API: debugging enabled by kernel config\n"); 741} 742 743static __init int dma_debug_cmdline(char *str) 744{ 745 if (!str) 746 return -EINVAL; 747 748 if (strncmp(str, "off", 3) == 0) { 749 pr_info("DMA-API: debugging disabled on kernel command line\n"); 750 global_disable = true; 751 } 752 753 return 0; 754} 755 756static __init int dma_debug_entries_cmdline(char *str) 757{ 758 int res; 759 760 if (!str) 761 return -EINVAL; 762 763 res = get_option(&str, &req_entries); 764 765 if (!res) 766 req_entries = 0; 767 768 return 0; 769} 770 771__setup("dma_debug=", dma_debug_cmdline); 772__setup("dma_debug_entries=", dma_debug_entries_cmdline); 773 774static void check_unmap(struct dma_debug_entry *ref) 775{ 776 struct dma_debug_entry *entry; 777 struct hash_bucket *bucket; 778 unsigned long flags; 779 780 if (dma_mapping_error(ref->dev, ref->dev_addr)) { 781 err_printk(ref->dev, NULL, "DMA-API: device driver tries " 782 "to free an invalid DMA memory address\n"); 783 return; 784 } 785 786 bucket = get_hash_bucket(ref, &flags); 787 entry = hash_bucket_find(bucket, ref); 788 789 if (!entry) { 790 err_printk(ref->dev, NULL, "DMA-API: device driver tries " 791 "to free DMA memory it has not allocated " 792 "[device address=0x%016llx] [size=%llu bytes]\n", 793 ref->dev_addr, ref->size); 794 goto out; 795 } 796 797 if (ref->size != entry->size) { 798 err_printk(ref->dev, entry, "DMA-API: device driver frees " 799 "DMA memory with different size " 800 "[device address=0x%016llx] [map size=%llu bytes] " 801 "[unmap size=%llu bytes]\n", 802 ref->dev_addr, entry->size, ref->size); 803 } 804 805 if (ref->type != entry->type) { 806 err_printk(ref->dev, entry, "DMA-API: device driver frees " 807 "DMA memory with wrong function " 808 "[device address=0x%016llx] [size=%llu bytes] " 809 "[mapped as %s] [unmapped as %s]\n", 810 ref->dev_addr, ref->size, 811 type2name[entry->type], type2name[ref->type]); 812 } else if ((entry->type == dma_debug_coherent) && 813 (ref->paddr != entry->paddr)) { 814 err_printk(ref->dev, entry, "DMA-API: device driver frees " 815 "DMA memory with different CPU address " 816 "[device address=0x%016llx] [size=%llu bytes] " 817 "[cpu alloc address=%p] [cpu free address=%p]", 818 ref->dev_addr, ref->size, 819 (void *)entry->paddr, (void *)ref->paddr); 820 } 821 822 if (ref->sg_call_ents && ref->type == dma_debug_sg && 823 ref->sg_call_ents != entry->sg_call_ents) { 824 err_printk(ref->dev, entry, "DMA-API: device driver frees " 825 "DMA sg list with different entry count " 826 "[map count=%d] [unmap count=%d]\n", 827 entry->sg_call_ents, ref->sg_call_ents); 828 } 829 830 /* 831 * This may be no bug in reality - but most implementations of the 832 * DMA API don't handle this properly, so check for it here 833 */ 834 if (ref->direction != entry->direction) { 835 err_printk(ref->dev, entry, "DMA-API: device driver frees " 836 "DMA memory with different direction " 837 "[device address=0x%016llx] [size=%llu bytes] " 838 "[mapped with %s] [unmapped with %s]\n", 839 ref->dev_addr, ref->size, 840 dir2name[entry->direction], 841 dir2name[ref->direction]); 842 } 843 844 hash_bucket_del(entry); 845 dma_entry_free(entry); 846 847out: 848 put_hash_bucket(bucket, &flags); 849} 850 851static void check_for_stack(struct device *dev, void *addr) 852{ 853 if (object_is_on_stack(addr)) 854 err_printk(dev, NULL, "DMA-API: device driver maps memory from" 855 "stack [addr=%p]\n", addr); 856} 857 858static inline bool overlap(void *addr, u64 size, void *start, void *end) 859{ 860 void *addr2 = (char *)addr + size; 861 862 return ((addr >= start && addr < end) || 863 (addr2 >= start && addr2 < end) || 864 ((addr < start) && (addr2 >= end))); 865} 866 867static void check_for_illegal_area(struct device *dev, void *addr, u64 size) 868{ 869 if (overlap(addr, size, _text, _etext) || 870 overlap(addr, size, __start_rodata, __end_rodata)) 871 err_printk(dev, NULL, "DMA-API: device driver maps " 872 "memory from kernel text or rodata " 873 "[addr=%p] [size=%llu]\n", addr, size); 874} 875 876static void check_sync(struct device *dev, dma_addr_t addr, 877 u64 size, u64 offset, int direction, bool to_cpu) 878{ 879 struct dma_debug_entry ref = { 880 .dev = dev, 881 .dev_addr = addr, 882 .size = size, 883 .direction = direction, 884 }; 885 struct dma_debug_entry *entry; 886 struct hash_bucket *bucket; 887 unsigned long flags; 888 889 bucket = get_hash_bucket(&ref, &flags); 890 891 entry = hash_bucket_find(bucket, &ref); 892 893 if (!entry) { 894 err_printk(dev, NULL, "DMA-API: device driver tries " 895 "to sync DMA memory it has not allocated " 896 "[device address=0x%016llx] [size=%llu bytes]\n", 897 (unsigned long long)addr, size); 898 goto out; 899 } 900 901 if ((offset + size) > entry->size) { 902 err_printk(dev, entry, "DMA-API: device driver syncs" 903 " DMA memory outside allocated range " 904 "[device address=0x%016llx] " 905 "[allocation size=%llu bytes] [sync offset=%llu] " 906 "[sync size=%llu]\n", entry->dev_addr, entry->size, 907 offset, size); 908 } 909 910 if (direction != entry->direction) { 911 err_printk(dev, entry, "DMA-API: device driver syncs " 912 "DMA memory with different direction " 913 "[device address=0x%016llx] [size=%llu bytes] " 914 "[mapped with %s] [synced with %s]\n", 915 (unsigned long long)addr, entry->size, 916 dir2name[entry->direction], 917 dir2name[direction]); 918 } 919 920 if (entry->direction == DMA_BIDIRECTIONAL) 921 goto out; 922 923 if (to_cpu && !(entry->direction == DMA_FROM_DEVICE) && 924 !(direction == DMA_TO_DEVICE)) 925 err_printk(dev, entry, "DMA-API: device driver syncs " 926 "device read-only DMA memory for cpu " 927 "[device address=0x%016llx] [size=%llu bytes] " 928 "[mapped with %s] [synced with %s]\n", 929 (unsigned long long)addr, entry->size, 930 dir2name[entry->direction], 931 dir2name[direction]); 932 933 if (!to_cpu && !(entry->direction == DMA_TO_DEVICE) && 934 !(direction == DMA_FROM_DEVICE)) 935 err_printk(dev, entry, "DMA-API: device driver syncs " 936 "device write-only DMA memory to device " 937 "[device address=0x%016llx] [size=%llu bytes] " 938 "[mapped with %s] [synced with %s]\n", 939 (unsigned long long)addr, entry->size, 940 dir2name[entry->direction], 941 dir2name[direction]); 942 943out: 944 put_hash_bucket(bucket, &flags); 945 946} 947 948void debug_dma_map_page(struct device *dev, struct page *page, size_t offset, 949 size_t size, int direction, dma_addr_t dma_addr, 950 bool map_single) 951{ 952 struct dma_debug_entry *entry; 953 954 if (unlikely(global_disable)) 955 return; 956 957 if (unlikely(dma_mapping_error(dev, dma_addr))) 958 return; 959 960 entry = dma_entry_alloc(); 961 if (!entry) 962 return; 963 964 entry->dev = dev; 965 entry->type = dma_debug_page; 966 entry->paddr = page_to_phys(page) + offset; 967 entry->dev_addr = dma_addr; 968 entry->size = size; 969 entry->direction = direction; 970 971 if (map_single) 972 entry->type = dma_debug_single; 973 974 if (!PageHighMem(page)) { 975 void *addr = ((char *)page_address(page)) + offset; 976 check_for_stack(dev, addr); 977 check_for_illegal_area(dev, addr, size); 978 } 979 980 add_dma_entry(entry); 981} 982EXPORT_SYMBOL(debug_dma_map_page); 983 984void debug_dma_unmap_page(struct device *dev, dma_addr_t addr, 985 size_t size, int direction, bool map_single) 986{ 987 struct dma_debug_entry ref = { 988 .type = dma_debug_page, 989 .dev = dev, 990 .dev_addr = addr, 991 .size = size, 992 .direction = direction, 993 }; 994 995 if (unlikely(global_disable)) 996 return; 997 998 if (map_single) 999 ref.type = dma_debug_single; 1000 1001 check_unmap(&ref); 1002} 1003EXPORT_SYMBOL(debug_dma_unmap_page); 1004 1005void debug_dma_map_sg(struct device *dev, struct scatterlist *sg, 1006 int nents, int mapped_ents, int direction) 1007{ 1008 struct dma_debug_entry *entry; 1009 struct scatterlist *s; 1010 int i; 1011 1012 if (unlikely(global_disable)) 1013 return; 1014 1015 for_each_sg(sg, s, mapped_ents, i) { 1016 entry = dma_entry_alloc(); 1017 if (!entry) 1018 return; 1019 1020 entry->type = dma_debug_sg; 1021 entry->dev = dev; 1022 entry->paddr = sg_phys(s); 1023 entry->size = sg_dma_len(s); 1024 entry->dev_addr = sg_dma_address(s); 1025 entry->direction = direction; 1026 entry->sg_call_ents = nents; 1027 entry->sg_mapped_ents = mapped_ents; 1028 1029 if (!PageHighMem(sg_page(s))) { 1030 check_for_stack(dev, sg_virt(s)); 1031 check_for_illegal_area(dev, sg_virt(s), sg_dma_len(s)); 1032 } 1033 1034 add_dma_entry(entry); 1035 } 1036} 1037EXPORT_SYMBOL(debug_dma_map_sg); 1038 1039static int get_nr_mapped_entries(struct device *dev, struct scatterlist *s) 1040{ 1041 struct dma_debug_entry *entry, ref; 1042 struct hash_bucket *bucket; 1043 unsigned long flags; 1044 int mapped_ents; 1045 1046 ref.dev = dev; 1047 ref.dev_addr = sg_dma_address(s); 1048 ref.size = sg_dma_len(s), 1049 1050 bucket = get_hash_bucket(&ref, &flags); 1051 entry = hash_bucket_find(bucket, &ref); 1052 mapped_ents = 0; 1053 1054 if (entry) 1055 mapped_ents = entry->sg_mapped_ents; 1056 put_hash_bucket(bucket, &flags); 1057 1058 return mapped_ents; 1059} 1060 1061void debug_dma_unmap_sg(struct device *dev, struct scatterlist *sglist, 1062 int nelems, int dir) 1063{ 1064 struct scatterlist *s; 1065 int mapped_ents = 0, i; 1066 1067 if (unlikely(global_disable)) 1068 return; 1069 1070 for_each_sg(sglist, s, nelems, i) { 1071 1072 struct dma_debug_entry ref = { 1073 .type = dma_debug_sg, 1074 .dev = dev, 1075 .paddr = sg_phys(s), 1076 .dev_addr = sg_dma_address(s), 1077 .size = sg_dma_len(s), 1078 .direction = dir, 1079 .sg_call_ents = 0, 1080 }; 1081 1082 if (mapped_ents && i >= mapped_ents) 1083 break; 1084 1085 if (!i) { 1086 ref.sg_call_ents = nelems; 1087 mapped_ents = get_nr_mapped_entries(dev, s); 1088 } 1089 1090 check_unmap(&ref); 1091 } 1092} 1093EXPORT_SYMBOL(debug_dma_unmap_sg); 1094 1095void debug_dma_alloc_coherent(struct device *dev, size_t size, 1096 dma_addr_t dma_addr, void *virt) 1097{ 1098 struct dma_debug_entry *entry; 1099 1100 if (unlikely(global_disable)) 1101 return; 1102 1103 if (unlikely(virt == NULL)) 1104 return; 1105 1106 entry = dma_entry_alloc(); 1107 if (!entry) 1108 return; 1109 1110 entry->type = dma_debug_coherent; 1111 entry->dev = dev; 1112 entry->paddr = virt_to_phys(virt); 1113 entry->size = size; 1114 entry->dev_addr = dma_addr; 1115 entry->direction = DMA_BIDIRECTIONAL; 1116 1117 add_dma_entry(entry); 1118} 1119EXPORT_SYMBOL(debug_dma_alloc_coherent); 1120 1121void debug_dma_free_coherent(struct device *dev, size_t size, 1122 void *virt, dma_addr_t addr) 1123{ 1124 struct dma_debug_entry ref = { 1125 .type = dma_debug_coherent, 1126 .dev = dev, 1127 .paddr = virt_to_phys(virt), 1128 .dev_addr = addr, 1129 .size = size, 1130 .direction = DMA_BIDIRECTIONAL, 1131 }; 1132 1133 if (unlikely(global_disable)) 1134 return; 1135 1136 check_unmap(&ref); 1137} 1138EXPORT_SYMBOL(debug_dma_free_coherent); 1139 1140void debug_dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle, 1141 size_t size, int direction) 1142{ 1143 if (unlikely(global_disable)) 1144 return; 1145 1146 check_sync(dev, dma_handle, size, 0, direction, true); 1147} 1148EXPORT_SYMBOL(debug_dma_sync_single_for_cpu); 1149 1150void debug_dma_sync_single_for_device(struct device *dev, 1151 dma_addr_t dma_handle, size_t size, 1152 int direction) 1153{ 1154 if (unlikely(global_disable)) 1155 return; 1156 1157 check_sync(dev, dma_handle, size, 0, direction, false); 1158} 1159EXPORT_SYMBOL(debug_dma_sync_single_for_device); 1160 1161void debug_dma_sync_single_range_for_cpu(struct device *dev, 1162 dma_addr_t dma_handle, 1163 unsigned long offset, size_t size, 1164 int direction) 1165{ 1166 if (unlikely(global_disable)) 1167 return; 1168 1169 check_sync(dev, dma_handle, size, offset, direction, true); 1170} 1171EXPORT_SYMBOL(debug_dma_sync_single_range_for_cpu); 1172 1173void debug_dma_sync_single_range_for_device(struct device *dev, 1174 dma_addr_t dma_handle, 1175 unsigned long offset, 1176 size_t size, int direction) 1177{ 1178 if (unlikely(global_disable)) 1179 return; 1180 1181 check_sync(dev, dma_handle, size, offset, direction, false); 1182} 1183EXPORT_SYMBOL(debug_dma_sync_single_range_for_device); 1184 1185void debug_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 1186 int nelems, int direction) 1187{ 1188 struct scatterlist *s; 1189 int mapped_ents = 0, i; 1190 1191 if (unlikely(global_disable)) 1192 return; 1193 1194 for_each_sg(sg, s, nelems, i) { 1195 if (!i) 1196 mapped_ents = get_nr_mapped_entries(dev, s); 1197 1198 if (i >= mapped_ents) 1199 break; 1200 1201 check_sync(dev, sg_dma_address(s), sg_dma_len(s), 0, 1202 direction, true); 1203 } 1204} 1205EXPORT_SYMBOL(debug_dma_sync_sg_for_cpu); 1206 1207void debug_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, 1208 int nelems, int direction) 1209{ 1210 struct scatterlist *s; 1211 int mapped_ents = 0, i; 1212 1213 if (unlikely(global_disable)) 1214 return; 1215 1216 for_each_sg(sg, s, nelems, i) { 1217 if (!i) 1218 mapped_ents = get_nr_mapped_entries(dev, s); 1219 1220 if (i >= mapped_ents) 1221 break; 1222 1223 check_sync(dev, sg_dma_address(s), sg_dma_len(s), 0, 1224 direction, false); 1225 } 1226} 1227EXPORT_SYMBOL(debug_dma_sync_sg_for_device); 1228 1229static int __init dma_debug_driver_setup(char *str) 1230{ 1231 int i; 1232 1233 for (i = 0; i < NAME_MAX_LEN - 1; ++i, ++str) { 1234 current_driver_name[i] = *str; 1235 if (*str == 0) 1236 break; 1237 } 1238 1239 if (current_driver_name[0]) 1240 pr_info("DMA-API: enable driver filter for driver [%s]\n", 1241 current_driver_name); 1242 1243 1244 return 1; 1245} 1246__setup("dma_debug_driver=", dma_debug_driver_setup);
generated by cgit v1.2.2 (git 2.25.0) at 2025-07-05 13:29:29 -0400