diff options
author | Ingo Molnar <mingo@elte.hu> | 2011-05-10 16:15:57 -0400 |
---|---|---|
committer | Ingo Molnar <mingo@elte.hu> | 2011-05-10 16:15:57 -0400 |
commit | 2c14ddc135d93cb4b33ee4bd4a2b05e21a4a762c (patch) | |
tree | 47cc9da388b0bae4bbb2255e1fbe9eef724d088c /arch/x86/kernel/amd_iommu.c | |
parent | 54b333529df25b21da462c7dcc16c7dc779d9f26 (diff) | |
parent | 72fe00f01f9a3240a1073be27aeaf4fc476cc662 (diff) |
Merge branch 'iommu/2.6.40' of git://git.kernel.org/pub/scm/linux/kernel/git/joro/linux-2.6-iommu into core/iommu
Diffstat (limited to 'arch/x86/kernel/amd_iommu.c')
-rw-r--r-- | arch/x86/kernel/amd_iommu.c | 526 |
1 files changed, 308 insertions, 218 deletions
diff --git a/arch/x86/kernel/amd_iommu.c b/arch/x86/kernel/amd_iommu.c index 57ca7778722..873e7e1ead7 100644 --- a/arch/x86/kernel/amd_iommu.c +++ b/arch/x86/kernel/amd_iommu.c | |||
@@ -18,6 +18,7 @@ | |||
18 | */ | 18 | */ |
19 | 19 | ||
20 | #include <linux/pci.h> | 20 | #include <linux/pci.h> |
21 | #include <linux/pci-ats.h> | ||
21 | #include <linux/bitmap.h> | 22 | #include <linux/bitmap.h> |
22 | #include <linux/slab.h> | 23 | #include <linux/slab.h> |
23 | #include <linux/debugfs.h> | 24 | #include <linux/debugfs.h> |
@@ -25,6 +26,7 @@ | |||
25 | #include <linux/dma-mapping.h> | 26 | #include <linux/dma-mapping.h> |
26 | #include <linux/iommu-helper.h> | 27 | #include <linux/iommu-helper.h> |
27 | #include <linux/iommu.h> | 28 | #include <linux/iommu.h> |
29 | #include <linux/delay.h> | ||
28 | #include <asm/proto.h> | 30 | #include <asm/proto.h> |
29 | #include <asm/iommu.h> | 31 | #include <asm/iommu.h> |
30 | #include <asm/gart.h> | 32 | #include <asm/gart.h> |
@@ -34,7 +36,7 @@ | |||
34 | 36 | ||
35 | #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28)) | 37 | #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28)) |
36 | 38 | ||
37 | #define EXIT_LOOP_COUNT 10000000 | 39 | #define LOOP_TIMEOUT 100000 |
38 | 40 | ||
39 | static DEFINE_RWLOCK(amd_iommu_devtable_lock); | 41 | static DEFINE_RWLOCK(amd_iommu_devtable_lock); |
40 | 42 | ||
@@ -57,7 +59,6 @@ struct iommu_cmd { | |||
57 | u32 data[4]; | 59 | u32 data[4]; |
58 | }; | 60 | }; |
59 | 61 | ||
60 | static void reset_iommu_command_buffer(struct amd_iommu *iommu); | ||
61 | static void update_domain(struct protection_domain *domain); | 62 | static void update_domain(struct protection_domain *domain); |
62 | 63 | ||
63 | /**************************************************************************** | 64 | /**************************************************************************** |
@@ -322,8 +323,6 @@ static void iommu_print_event(struct amd_iommu *iommu, void *__evt) | |||
322 | break; | 323 | break; |
323 | case EVENT_TYPE_ILL_CMD: | 324 | case EVENT_TYPE_ILL_CMD: |
324 | printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address); | 325 | printk("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n", address); |
325 | iommu->reset_in_progress = true; | ||
326 | reset_iommu_command_buffer(iommu); | ||
327 | dump_command(address); | 326 | dump_command(address); |
328 | break; | 327 | break; |
329 | case EVENT_TYPE_CMD_HARD_ERR: | 328 | case EVENT_TYPE_CMD_HARD_ERR: |
@@ -367,7 +366,7 @@ static void iommu_poll_events(struct amd_iommu *iommu) | |||
367 | spin_unlock_irqrestore(&iommu->lock, flags); | 366 | spin_unlock_irqrestore(&iommu->lock, flags); |
368 | } | 367 | } |
369 | 368 | ||
370 | irqreturn_t amd_iommu_int_handler(int irq, void *data) | 369 | irqreturn_t amd_iommu_int_thread(int irq, void *data) |
371 | { | 370 | { |
372 | struct amd_iommu *iommu; | 371 | struct amd_iommu *iommu; |
373 | 372 | ||
@@ -377,192 +376,300 @@ irqreturn_t amd_iommu_int_handler(int irq, void *data) | |||
377 | return IRQ_HANDLED; | 376 | return IRQ_HANDLED; |
378 | } | 377 | } |
379 | 378 | ||
379 | irqreturn_t amd_iommu_int_handler(int irq, void *data) | ||
380 | { | ||
381 | return IRQ_WAKE_THREAD; | ||
382 | } | ||
383 | |||
380 | /**************************************************************************** | 384 | /**************************************************************************** |
381 | * | 385 | * |
382 | * IOMMU command queuing functions | 386 | * IOMMU command queuing functions |
383 | * | 387 | * |
384 | ****************************************************************************/ | 388 | ****************************************************************************/ |
385 | 389 | ||
386 | /* | 390 | static int wait_on_sem(volatile u64 *sem) |
387 | * Writes the command to the IOMMUs command buffer and informs the | 391 | { |
388 | * hardware about the new command. Must be called with iommu->lock held. | 392 | int i = 0; |
389 | */ | 393 | |
390 | static int __iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd) | 394 | while (*sem == 0 && i < LOOP_TIMEOUT) { |
395 | udelay(1); | ||
396 | i += 1; | ||
397 | } | ||
398 | |||
399 | if (i == LOOP_TIMEOUT) { | ||
400 | pr_alert("AMD-Vi: Completion-Wait loop timed out\n"); | ||
401 | return -EIO; | ||
402 | } | ||
403 | |||
404 | return 0; | ||
405 | } | ||
406 | |||
407 | static void copy_cmd_to_buffer(struct amd_iommu *iommu, | ||
408 | struct iommu_cmd *cmd, | ||
409 | u32 tail) | ||
391 | { | 410 | { |
392 | u32 tail, head; | ||
393 | u8 *target; | 411 | u8 *target; |
394 | 412 | ||
395 | WARN_ON(iommu->cmd_buf_size & CMD_BUFFER_UNINITIALIZED); | ||
396 | tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET); | ||
397 | target = iommu->cmd_buf + tail; | 413 | target = iommu->cmd_buf + tail; |
398 | memcpy_toio(target, cmd, sizeof(*cmd)); | 414 | tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size; |
399 | tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size; | 415 | |
400 | head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET); | 416 | /* Copy command to buffer */ |
401 | if (tail == head) | 417 | memcpy(target, cmd, sizeof(*cmd)); |
402 | return -ENOMEM; | 418 | |
419 | /* Tell the IOMMU about it */ | ||
403 | writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET); | 420 | writel(tail, iommu->mmio_base + MMIO_CMD_TAIL_OFFSET); |
421 | } | ||
404 | 422 | ||
405 | return 0; | 423 | static void build_completion_wait(struct iommu_cmd *cmd, u64 address) |
424 | { | ||
425 | WARN_ON(address & 0x7ULL); | ||
426 | |||
427 | memset(cmd, 0, sizeof(*cmd)); | ||
428 | cmd->data[0] = lower_32_bits(__pa(address)) | CMD_COMPL_WAIT_STORE_MASK; | ||
429 | cmd->data[1] = upper_32_bits(__pa(address)); | ||
430 | cmd->data[2] = 1; | ||
431 | CMD_SET_TYPE(cmd, CMD_COMPL_WAIT); | ||
432 | } | ||
433 | |||
434 | static void build_inv_dte(struct iommu_cmd *cmd, u16 devid) | ||
435 | { | ||
436 | memset(cmd, 0, sizeof(*cmd)); | ||
437 | cmd->data[0] = devid; | ||
438 | CMD_SET_TYPE(cmd, CMD_INV_DEV_ENTRY); | ||
439 | } | ||
440 | |||
441 | static void build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address, | ||
442 | size_t size, u16 domid, int pde) | ||
443 | { | ||
444 | u64 pages; | ||
445 | int s; | ||
446 | |||
447 | pages = iommu_num_pages(address, size, PAGE_SIZE); | ||
448 | s = 0; | ||
449 | |||
450 | if (pages > 1) { | ||
451 | /* | ||
452 | * If we have to flush more than one page, flush all | ||
453 | * TLB entries for this domain | ||
454 | */ | ||
455 | address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS; | ||
456 | s = 1; | ||
457 | } | ||
458 | |||
459 | address &= PAGE_MASK; | ||
460 | |||
461 | memset(cmd, 0, sizeof(*cmd)); | ||
462 | cmd->data[1] |= domid; | ||
463 | cmd->data[2] = lower_32_bits(address); | ||
464 | cmd->data[3] = upper_32_bits(address); | ||
465 | CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES); | ||
466 | if (s) /* size bit - we flush more than one 4kb page */ | ||
467 | cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK; | ||
468 | if (pde) /* PDE bit - we wan't flush everything not only the PTEs */ | ||
469 | cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK; | ||
470 | } | ||
471 | |||
472 | static void build_inv_iotlb_pages(struct iommu_cmd *cmd, u16 devid, int qdep, | ||
473 | u64 address, size_t size) | ||
474 | { | ||
475 | u64 pages; | ||
476 | int s; | ||
477 | |||
478 | pages = iommu_num_pages(address, size, PAGE_SIZE); | ||
479 | s = 0; | ||
480 | |||
481 | if (pages > 1) { | ||
482 | /* | ||
483 | * If we have to flush more than one page, flush all | ||
484 | * TLB entries for this domain | ||
485 | */ | ||
486 | address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS; | ||
487 | s = 1; | ||
488 | } | ||
489 | |||
490 | address &= PAGE_MASK; | ||
491 | |||
492 | memset(cmd, 0, sizeof(*cmd)); | ||
493 | cmd->data[0] = devid; | ||
494 | cmd->data[0] |= (qdep & 0xff) << 24; | ||
495 | cmd->data[1] = devid; | ||
496 | cmd->data[2] = lower_32_bits(address); | ||
497 | cmd->data[3] = upper_32_bits(address); | ||
498 | CMD_SET_TYPE(cmd, CMD_INV_IOTLB_PAGES); | ||
499 | if (s) | ||
500 | cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK; | ||
501 | } | ||
502 | |||
503 | static void build_inv_all(struct iommu_cmd *cmd) | ||
504 | { | ||
505 | memset(cmd, 0, sizeof(*cmd)); | ||
506 | CMD_SET_TYPE(cmd, CMD_INV_ALL); | ||
406 | } | 507 | } |
407 | 508 | ||
408 | /* | 509 | /* |
409 | * General queuing function for commands. Takes iommu->lock and calls | 510 | * Writes the command to the IOMMUs command buffer and informs the |
410 | * __iommu_queue_command(). | 511 | * hardware about the new command. |
411 | */ | 512 | */ |
412 | static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd) | 513 | static int iommu_queue_command(struct amd_iommu *iommu, struct iommu_cmd *cmd) |
413 | { | 514 | { |
515 | u32 left, tail, head, next_tail; | ||
414 | unsigned long flags; | 516 | unsigned long flags; |
415 | int ret; | ||
416 | 517 | ||
518 | WARN_ON(iommu->cmd_buf_size & CMD_BUFFER_UNINITIALIZED); | ||
519 | |||
520 | again: | ||
417 | spin_lock_irqsave(&iommu->lock, flags); | 521 | spin_lock_irqsave(&iommu->lock, flags); |
418 | ret = __iommu_queue_command(iommu, cmd); | ||
419 | if (!ret) | ||
420 | iommu->need_sync = true; | ||
421 | spin_unlock_irqrestore(&iommu->lock, flags); | ||
422 | 522 | ||
423 | return ret; | 523 | head = readl(iommu->mmio_base + MMIO_CMD_HEAD_OFFSET); |
424 | } | 524 | tail = readl(iommu->mmio_base + MMIO_CMD_TAIL_OFFSET); |
525 | next_tail = (tail + sizeof(*cmd)) % iommu->cmd_buf_size; | ||
526 | left = (head - next_tail) % iommu->cmd_buf_size; | ||
425 | 527 | ||
426 | /* | 528 | if (left <= 2) { |
427 | * This function waits until an IOMMU has completed a completion | 529 | struct iommu_cmd sync_cmd; |
428 | * wait command | 530 | volatile u64 sem = 0; |
429 | */ | 531 | int ret; |
430 | static void __iommu_wait_for_completion(struct amd_iommu *iommu) | ||
431 | { | ||
432 | int ready = 0; | ||
433 | unsigned status = 0; | ||
434 | unsigned long i = 0; | ||
435 | 532 | ||
436 | INC_STATS_COUNTER(compl_wait); | 533 | build_completion_wait(&sync_cmd, (u64)&sem); |
534 | copy_cmd_to_buffer(iommu, &sync_cmd, tail); | ||
437 | 535 | ||
438 | while (!ready && (i < EXIT_LOOP_COUNT)) { | 536 | spin_unlock_irqrestore(&iommu->lock, flags); |
439 | ++i; | 537 | |
440 | /* wait for the bit to become one */ | 538 | if ((ret = wait_on_sem(&sem)) != 0) |
441 | status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); | 539 | return ret; |
442 | ready = status & MMIO_STATUS_COM_WAIT_INT_MASK; | 540 | |
541 | goto again; | ||
443 | } | 542 | } |
444 | 543 | ||
445 | /* set bit back to zero */ | 544 | copy_cmd_to_buffer(iommu, cmd, tail); |
446 | status &= ~MMIO_STATUS_COM_WAIT_INT_MASK; | 545 | |
447 | writel(status, iommu->mmio_base + MMIO_STATUS_OFFSET); | 546 | /* We need to sync now to make sure all commands are processed */ |
547 | iommu->need_sync = true; | ||
548 | |||
549 | spin_unlock_irqrestore(&iommu->lock, flags); | ||
448 | 550 | ||
449 | if (unlikely(i == EXIT_LOOP_COUNT)) | 551 | return 0; |
450 | iommu->reset_in_progress = true; | ||
451 | } | 552 | } |
452 | 553 | ||
453 | /* | 554 | /* |
454 | * This function queues a completion wait command into the command | 555 | * This function queues a completion wait command into the command |
455 | * buffer of an IOMMU | 556 | * buffer of an IOMMU |
456 | */ | 557 | */ |
457 | static int __iommu_completion_wait(struct amd_iommu *iommu) | 558 | static int iommu_completion_wait(struct amd_iommu *iommu) |
458 | { | 559 | { |
459 | struct iommu_cmd cmd; | 560 | struct iommu_cmd cmd; |
561 | volatile u64 sem = 0; | ||
562 | int ret; | ||
460 | 563 | ||
461 | memset(&cmd, 0, sizeof(cmd)); | 564 | if (!iommu->need_sync) |
462 | cmd.data[0] = CMD_COMPL_WAIT_INT_MASK; | 565 | return 0; |
463 | CMD_SET_TYPE(&cmd, CMD_COMPL_WAIT); | ||
464 | 566 | ||
465 | return __iommu_queue_command(iommu, &cmd); | 567 | build_completion_wait(&cmd, (u64)&sem); |
568 | |||
569 | ret = iommu_queue_command(iommu, &cmd); | ||
570 | if (ret) | ||
571 | return ret; | ||
572 | |||
573 | return wait_on_sem(&sem); | ||
466 | } | 574 | } |
467 | 575 | ||
468 | /* | 576 | static int iommu_flush_dte(struct amd_iommu *iommu, u16 devid) |
469 | * This function is called whenever we need to ensure that the IOMMU has | ||
470 | * completed execution of all commands we sent. It sends a | ||
471 | * COMPLETION_WAIT command and waits for it to finish. The IOMMU informs | ||
472 | * us about that by writing a value to a physical address we pass with | ||
473 | * the command. | ||
474 | */ | ||
475 | static int iommu_completion_wait(struct amd_iommu *iommu) | ||
476 | { | 577 | { |
477 | int ret = 0; | 578 | struct iommu_cmd cmd; |
478 | unsigned long flags; | ||
479 | 579 | ||
480 | spin_lock_irqsave(&iommu->lock, flags); | 580 | build_inv_dte(&cmd, devid); |
481 | 581 | ||
482 | if (!iommu->need_sync) | 582 | return iommu_queue_command(iommu, &cmd); |
483 | goto out; | 583 | } |
484 | 584 | ||
485 | ret = __iommu_completion_wait(iommu); | 585 | static void iommu_flush_dte_all(struct amd_iommu *iommu) |
586 | { | ||
587 | u32 devid; | ||
486 | 588 | ||
487 | iommu->need_sync = false; | 589 | for (devid = 0; devid <= 0xffff; ++devid) |
590 | iommu_flush_dte(iommu, devid); | ||
488 | 591 | ||
489 | if (ret) | 592 | iommu_completion_wait(iommu); |
490 | goto out; | 593 | } |
491 | |||
492 | __iommu_wait_for_completion(iommu); | ||
493 | 594 | ||
494 | out: | 595 | /* |
495 | spin_unlock_irqrestore(&iommu->lock, flags); | 596 | * This function uses heavy locking and may disable irqs for some time. But |
597 | * this is no issue because it is only called during resume. | ||
598 | */ | ||
599 | static void iommu_flush_tlb_all(struct amd_iommu *iommu) | ||
600 | { | ||
601 | u32 dom_id; | ||
496 | 602 | ||
497 | if (iommu->reset_in_progress) | 603 | for (dom_id = 0; dom_id <= 0xffff; ++dom_id) { |
498 | reset_iommu_command_buffer(iommu); | 604 | struct iommu_cmd cmd; |
605 | build_inv_iommu_pages(&cmd, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, | ||
606 | dom_id, 1); | ||
607 | iommu_queue_command(iommu, &cmd); | ||
608 | } | ||
499 | 609 | ||
500 | return 0; | 610 | iommu_completion_wait(iommu); |
501 | } | 611 | } |
502 | 612 | ||
503 | static void iommu_flush_complete(struct protection_domain *domain) | 613 | static void iommu_flush_all(struct amd_iommu *iommu) |
504 | { | 614 | { |
505 | int i; | 615 | struct iommu_cmd cmd; |
506 | 616 | ||
507 | for (i = 0; i < amd_iommus_present; ++i) { | 617 | build_inv_all(&cmd); |
508 | if (!domain->dev_iommu[i]) | ||
509 | continue; | ||
510 | 618 | ||
511 | /* | 619 | iommu_queue_command(iommu, &cmd); |
512 | * Devices of this domain are behind this IOMMU | 620 | iommu_completion_wait(iommu); |
513 | * We need to wait for completion of all commands. | 621 | } |
514 | */ | 622 | |
515 | iommu_completion_wait(amd_iommus[i]); | 623 | void iommu_flush_all_caches(struct amd_iommu *iommu) |
624 | { | ||
625 | if (iommu_feature(iommu, FEATURE_IA)) { | ||
626 | iommu_flush_all(iommu); | ||
627 | } else { | ||
628 | iommu_flush_dte_all(iommu); | ||
629 | iommu_flush_tlb_all(iommu); | ||
516 | } | 630 | } |
517 | } | 631 | } |
518 | 632 | ||
519 | /* | 633 | /* |
520 | * Command send function for invalidating a device table entry | 634 | * Command send function for flushing on-device TLB |
521 | */ | 635 | */ |
522 | static int iommu_flush_device(struct device *dev) | 636 | static int device_flush_iotlb(struct device *dev, u64 address, size_t size) |
523 | { | 637 | { |
638 | struct pci_dev *pdev = to_pci_dev(dev); | ||
524 | struct amd_iommu *iommu; | 639 | struct amd_iommu *iommu; |
525 | struct iommu_cmd cmd; | 640 | struct iommu_cmd cmd; |
526 | u16 devid; | 641 | u16 devid; |
642 | int qdep; | ||
527 | 643 | ||
644 | qdep = pci_ats_queue_depth(pdev); | ||
528 | devid = get_device_id(dev); | 645 | devid = get_device_id(dev); |
529 | iommu = amd_iommu_rlookup_table[devid]; | 646 | iommu = amd_iommu_rlookup_table[devid]; |
530 | 647 | ||
531 | /* Build command */ | 648 | build_inv_iotlb_pages(&cmd, devid, qdep, address, size); |
532 | memset(&cmd, 0, sizeof(cmd)); | ||
533 | CMD_SET_TYPE(&cmd, CMD_INV_DEV_ENTRY); | ||
534 | cmd.data[0] = devid; | ||
535 | 649 | ||
536 | return iommu_queue_command(iommu, &cmd); | 650 | return iommu_queue_command(iommu, &cmd); |
537 | } | 651 | } |
538 | 652 | ||
539 | static void __iommu_build_inv_iommu_pages(struct iommu_cmd *cmd, u64 address, | ||
540 | u16 domid, int pde, int s) | ||
541 | { | ||
542 | memset(cmd, 0, sizeof(*cmd)); | ||
543 | address &= PAGE_MASK; | ||
544 | CMD_SET_TYPE(cmd, CMD_INV_IOMMU_PAGES); | ||
545 | cmd->data[1] |= domid; | ||
546 | cmd->data[2] = lower_32_bits(address); | ||
547 | cmd->data[3] = upper_32_bits(address); | ||
548 | if (s) /* size bit - we flush more than one 4kb page */ | ||
549 | cmd->data[2] |= CMD_INV_IOMMU_PAGES_SIZE_MASK; | ||
550 | if (pde) /* PDE bit - we wan't flush everything not only the PTEs */ | ||
551 | cmd->data[2] |= CMD_INV_IOMMU_PAGES_PDE_MASK; | ||
552 | } | ||
553 | |||
554 | /* | 653 | /* |
555 | * Generic command send function for invalidaing TLB entries | 654 | * Command send function for invalidating a device table entry |
556 | */ | 655 | */ |
557 | static int iommu_queue_inv_iommu_pages(struct amd_iommu *iommu, | 656 | static int device_flush_dte(struct device *dev) |
558 | u64 address, u16 domid, int pde, int s) | ||
559 | { | 657 | { |
560 | struct iommu_cmd cmd; | 658 | struct amd_iommu *iommu; |
659 | struct pci_dev *pdev; | ||
660 | u16 devid; | ||
561 | int ret; | 661 | int ret; |
562 | 662 | ||
563 | __iommu_build_inv_iommu_pages(&cmd, address, domid, pde, s); | 663 | pdev = to_pci_dev(dev); |
664 | devid = get_device_id(dev); | ||
665 | iommu = amd_iommu_rlookup_table[devid]; | ||
564 | 666 | ||
565 | ret = iommu_queue_command(iommu, &cmd); | 667 | ret = iommu_flush_dte(iommu, devid); |
668 | if (ret) | ||
669 | return ret; | ||
670 | |||
671 | if (pci_ats_enabled(pdev)) | ||
672 | ret = device_flush_iotlb(dev, 0, ~0UL); | ||
566 | 673 | ||
567 | return ret; | 674 | return ret; |
568 | } | 675 | } |
@@ -572,23 +679,14 @@ static int iommu_queue_inv_iommu_pages(struct amd_iommu *iommu, | |||
572 | * It invalidates a single PTE if the range to flush is within a single | 679 | * It invalidates a single PTE if the range to flush is within a single |
573 | * page. Otherwise it flushes the whole TLB of the IOMMU. | 680 | * page. Otherwise it flushes the whole TLB of the IOMMU. |
574 | */ | 681 | */ |
575 | static void __iommu_flush_pages(struct protection_domain *domain, | 682 | static void __domain_flush_pages(struct protection_domain *domain, |
576 | u64 address, size_t size, int pde) | 683 | u64 address, size_t size, int pde) |
577 | { | 684 | { |
578 | int s = 0, i; | 685 | struct iommu_dev_data *dev_data; |
579 | unsigned long pages = iommu_num_pages(address, size, PAGE_SIZE); | 686 | struct iommu_cmd cmd; |
580 | 687 | int ret = 0, i; | |
581 | address &= PAGE_MASK; | ||
582 | |||
583 | if (pages > 1) { | ||
584 | /* | ||
585 | * If we have to flush more than one page, flush all | ||
586 | * TLB entries for this domain | ||
587 | */ | ||
588 | address = CMD_INV_IOMMU_ALL_PAGES_ADDRESS; | ||
589 | s = 1; | ||
590 | } | ||
591 | 688 | ||
689 | build_inv_iommu_pages(&cmd, address, size, domain->id, pde); | ||
592 | 690 | ||
593 | for (i = 0; i < amd_iommus_present; ++i) { | 691 | for (i = 0; i < amd_iommus_present; ++i) { |
594 | if (!domain->dev_iommu[i]) | 692 | if (!domain->dev_iommu[i]) |
@@ -598,101 +696,70 @@ static void __iommu_flush_pages(struct protection_domain *domain, | |||
598 | * Devices of this domain are behind this IOMMU | 696 | * Devices of this domain are behind this IOMMU |
599 | * We need a TLB flush | 697 | * We need a TLB flush |
600 | */ | 698 | */ |
601 | iommu_queue_inv_iommu_pages(amd_iommus[i], address, | 699 | ret |= iommu_queue_command(amd_iommus[i], &cmd); |
602 | domain->id, pde, s); | 700 | } |
701 | |||
702 | list_for_each_entry(dev_data, &domain->dev_list, list) { | ||
703 | struct pci_dev *pdev = to_pci_dev(dev_data->dev); | ||
704 | |||
705 | if (!pci_ats_enabled(pdev)) | ||
706 | continue; | ||
707 | |||
708 | ret |= device_flush_iotlb(dev_data->dev, address, size); | ||
603 | } | 709 | } |
604 | 710 | ||
605 | return; | 711 | WARN_ON(ret); |
606 | } | 712 | } |
607 | 713 | ||
608 | static void iommu_flush_pages(struct protection_domain *domain, | 714 | static void domain_flush_pages(struct protection_domain *domain, |
609 | u64 address, size_t size) | 715 | u64 address, size_t size) |
610 | { | 716 | { |
611 | __iommu_flush_pages(domain, address, size, 0); | 717 | __domain_flush_pages(domain, address, size, 0); |
612 | } | 718 | } |
613 | 719 | ||
614 | /* Flush the whole IO/TLB for a given protection domain */ | 720 | /* Flush the whole IO/TLB for a given protection domain */ |
615 | static void iommu_flush_tlb(struct protection_domain *domain) | 721 | static void domain_flush_tlb(struct protection_domain *domain) |
616 | { | 722 | { |
617 | __iommu_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 0); | 723 | __domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 0); |
618 | } | 724 | } |
619 | 725 | ||
620 | /* Flush the whole IO/TLB for a given protection domain - including PDE */ | 726 | /* Flush the whole IO/TLB for a given protection domain - including PDE */ |
621 | static void iommu_flush_tlb_pde(struct protection_domain *domain) | 727 | static void domain_flush_tlb_pde(struct protection_domain *domain) |
622 | { | 728 | { |
623 | __iommu_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1); | 729 | __domain_flush_pages(domain, 0, CMD_INV_IOMMU_ALL_PAGES_ADDRESS, 1); |
624 | } | ||
625 | |||
626 | |||
627 | /* | ||
628 | * This function flushes the DTEs for all devices in domain | ||
629 | */ | ||
630 | static void iommu_flush_domain_devices(struct protection_domain *domain) | ||
631 | { | ||
632 | struct iommu_dev_data *dev_data; | ||
633 | unsigned long flags; | ||
634 | |||
635 | spin_lock_irqsave(&domain->lock, flags); | ||
636 | |||
637 | list_for_each_entry(dev_data, &domain->dev_list, list) | ||
638 | iommu_flush_device(dev_data->dev); | ||
639 | |||
640 | spin_unlock_irqrestore(&domain->lock, flags); | ||
641 | } | 730 | } |
642 | 731 | ||
643 | static void iommu_flush_all_domain_devices(void) | 732 | static void domain_flush_complete(struct protection_domain *domain) |
644 | { | 733 | { |
645 | struct protection_domain *domain; | 734 | int i; |
646 | unsigned long flags; | ||
647 | 735 | ||
648 | spin_lock_irqsave(&amd_iommu_pd_lock, flags); | 736 | for (i = 0; i < amd_iommus_present; ++i) { |
737 | if (!domain->dev_iommu[i]) | ||
738 | continue; | ||
649 | 739 | ||
650 | list_for_each_entry(domain, &amd_iommu_pd_list, list) { | 740 | /* |
651 | iommu_flush_domain_devices(domain); | 741 | * Devices of this domain are behind this IOMMU |
652 | iommu_flush_complete(domain); | 742 | * We need to wait for completion of all commands. |
743 | */ | ||
744 | iommu_completion_wait(amd_iommus[i]); | ||
653 | } | 745 | } |
654 | |||
655 | spin_unlock_irqrestore(&amd_iommu_pd_lock, flags); | ||
656 | } | 746 | } |
657 | 747 | ||
658 | void amd_iommu_flush_all_devices(void) | ||
659 | { | ||
660 | iommu_flush_all_domain_devices(); | ||
661 | } | ||
662 | 748 | ||
663 | /* | 749 | /* |
664 | * This function uses heavy locking and may disable irqs for some time. But | 750 | * This function flushes the DTEs for all devices in domain |
665 | * this is no issue because it is only called during resume. | ||
666 | */ | 751 | */ |
667 | void amd_iommu_flush_all_domains(void) | 752 | static void domain_flush_devices(struct protection_domain *domain) |
668 | { | 753 | { |
669 | struct protection_domain *domain; | 754 | struct iommu_dev_data *dev_data; |
670 | unsigned long flags; | 755 | unsigned long flags; |
671 | 756 | ||
672 | spin_lock_irqsave(&amd_iommu_pd_lock, flags); | 757 | spin_lock_irqsave(&domain->lock, flags); |
673 | |||
674 | list_for_each_entry(domain, &amd_iommu_pd_list, list) { | ||
675 | spin_lock(&domain->lock); | ||
676 | iommu_flush_tlb_pde(domain); | ||
677 | iommu_flush_complete(domain); | ||
678 | spin_unlock(&domain->lock); | ||
679 | } | ||
680 | |||
681 | spin_unlock_irqrestore(&amd_iommu_pd_lock, flags); | ||
682 | } | ||
683 | |||
684 | static void reset_iommu_command_buffer(struct amd_iommu *iommu) | ||
685 | { | ||
686 | pr_err("AMD-Vi: Resetting IOMMU command buffer\n"); | ||
687 | |||
688 | if (iommu->reset_in_progress) | ||
689 | panic("AMD-Vi: ILLEGAL_COMMAND_ERROR while resetting command buffer\n"); | ||
690 | 758 | ||
691 | amd_iommu_reset_cmd_buffer(iommu); | 759 | list_for_each_entry(dev_data, &domain->dev_list, list) |
692 | amd_iommu_flush_all_devices(); | 760 | device_flush_dte(dev_data->dev); |
693 | amd_iommu_flush_all_domains(); | ||
694 | 761 | ||
695 | iommu->reset_in_progress = false; | 762 | spin_unlock_irqrestore(&domain->lock, flags); |
696 | } | 763 | } |
697 | 764 | ||
698 | /**************************************************************************** | 765 | /**************************************************************************** |
@@ -1410,17 +1477,22 @@ static bool dma_ops_domain(struct protection_domain *domain) | |||
1410 | return domain->flags & PD_DMA_OPS_MASK; | 1477 | return domain->flags & PD_DMA_OPS_MASK; |
1411 | } | 1478 | } |
1412 | 1479 | ||
1413 | static void set_dte_entry(u16 devid, struct protection_domain *domain) | 1480 | static void set_dte_entry(u16 devid, struct protection_domain *domain, bool ats) |
1414 | { | 1481 | { |
1415 | u64 pte_root = virt_to_phys(domain->pt_root); | 1482 | u64 pte_root = virt_to_phys(domain->pt_root); |
1483 | u32 flags = 0; | ||
1416 | 1484 | ||
1417 | pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK) | 1485 | pte_root |= (domain->mode & DEV_ENTRY_MODE_MASK) |
1418 | << DEV_ENTRY_MODE_SHIFT; | 1486 | << DEV_ENTRY_MODE_SHIFT; |
1419 | pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV; | 1487 | pte_root |= IOMMU_PTE_IR | IOMMU_PTE_IW | IOMMU_PTE_P | IOMMU_PTE_TV; |
1420 | 1488 | ||
1421 | amd_iommu_dev_table[devid].data[2] = domain->id; | 1489 | if (ats) |
1422 | amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root); | 1490 | flags |= DTE_FLAG_IOTLB; |
1423 | amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root); | 1491 | |
1492 | amd_iommu_dev_table[devid].data[3] |= flags; | ||
1493 | amd_iommu_dev_table[devid].data[2] = domain->id; | ||
1494 | amd_iommu_dev_table[devid].data[1] = upper_32_bits(pte_root); | ||
1495 | amd_iommu_dev_table[devid].data[0] = lower_32_bits(pte_root); | ||
1424 | } | 1496 | } |
1425 | 1497 | ||
1426 | static void clear_dte_entry(u16 devid) | 1498 | static void clear_dte_entry(u16 devid) |
@@ -1437,34 +1509,42 @@ static void do_attach(struct device *dev, struct protection_domain *domain) | |||
1437 | { | 1509 | { |
1438 | struct iommu_dev_data *dev_data; | 1510 | struct iommu_dev_data *dev_data; |
1439 | struct amd_iommu *iommu; | 1511 | struct amd_iommu *iommu; |
1512 | struct pci_dev *pdev; | ||
1513 | bool ats = false; | ||
1440 | u16 devid; | 1514 | u16 devid; |
1441 | 1515 | ||
1442 | devid = get_device_id(dev); | 1516 | devid = get_device_id(dev); |
1443 | iommu = amd_iommu_rlookup_table[devid]; | 1517 | iommu = amd_iommu_rlookup_table[devid]; |
1444 | dev_data = get_dev_data(dev); | 1518 | dev_data = get_dev_data(dev); |
1519 | pdev = to_pci_dev(dev); | ||
1520 | |||
1521 | if (amd_iommu_iotlb_sup) | ||
1522 | ats = pci_ats_enabled(pdev); | ||
1445 | 1523 | ||
1446 | /* Update data structures */ | 1524 | /* Update data structures */ |
1447 | dev_data->domain = domain; | 1525 | dev_data->domain = domain; |
1448 | list_add(&dev_data->list, &domain->dev_list); | 1526 | list_add(&dev_data->list, &domain->dev_list); |
1449 | set_dte_entry(devid, domain); | 1527 | set_dte_entry(devid, domain, ats); |
1450 | 1528 | ||
1451 | /* Do reference counting */ | 1529 | /* Do reference counting */ |
1452 | domain->dev_iommu[iommu->index] += 1; | 1530 | domain->dev_iommu[iommu->index] += 1; |
1453 | domain->dev_cnt += 1; | 1531 | domain->dev_cnt += 1; |
1454 | 1532 | ||
1455 | /* Flush the DTE entry */ | 1533 | /* Flush the DTE entry */ |
1456 | iommu_flush_device(dev); | 1534 | device_flush_dte(dev); |
1457 | } | 1535 | } |
1458 | 1536 | ||
1459 | static void do_detach(struct device *dev) | 1537 | static void do_detach(struct device *dev) |
1460 | { | 1538 | { |
1461 | struct iommu_dev_data *dev_data; | 1539 | struct iommu_dev_data *dev_data; |
1462 | struct amd_iommu *iommu; | 1540 | struct amd_iommu *iommu; |
1541 | struct pci_dev *pdev; | ||
1463 | u16 devid; | 1542 | u16 devid; |
1464 | 1543 | ||
1465 | devid = get_device_id(dev); | 1544 | devid = get_device_id(dev); |
1466 | iommu = amd_iommu_rlookup_table[devid]; | 1545 | iommu = amd_iommu_rlookup_table[devid]; |
1467 | dev_data = get_dev_data(dev); | 1546 | dev_data = get_dev_data(dev); |
1547 | pdev = to_pci_dev(dev); | ||
1468 | 1548 | ||
1469 | /* decrease reference counters */ | 1549 | /* decrease reference counters */ |
1470 | dev_data->domain->dev_iommu[iommu->index] -= 1; | 1550 | dev_data->domain->dev_iommu[iommu->index] -= 1; |
@@ -1476,7 +1556,7 @@ static void do_detach(struct device *dev) | |||
1476 | clear_dte_entry(devid); | 1556 | clear_dte_entry(devid); |
1477 | 1557 | ||
1478 | /* Flush the DTE entry */ | 1558 | /* Flush the DTE entry */ |
1479 | iommu_flush_device(dev); | 1559 | device_flush_dte(dev); |
1480 | } | 1560 | } |
1481 | 1561 | ||
1482 | /* | 1562 | /* |
@@ -1539,9 +1619,13 @@ out_unlock: | |||
1539 | static int attach_device(struct device *dev, | 1619 | static int attach_device(struct device *dev, |
1540 | struct protection_domain *domain) | 1620 | struct protection_domain *domain) |
1541 | { | 1621 | { |
1622 | struct pci_dev *pdev = to_pci_dev(dev); | ||
1542 | unsigned long flags; | 1623 | unsigned long flags; |
1543 | int ret; | 1624 | int ret; |
1544 | 1625 | ||
1626 | if (amd_iommu_iotlb_sup) | ||
1627 | pci_enable_ats(pdev, PAGE_SHIFT); | ||
1628 | |||
1545 | write_lock_irqsave(&amd_iommu_devtable_lock, flags); | 1629 | write_lock_irqsave(&amd_iommu_devtable_lock, flags); |
1546 | ret = __attach_device(dev, domain); | 1630 | ret = __attach_device(dev, domain); |
1547 | write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); | 1631 | write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); |
@@ -1551,7 +1635,7 @@ static int attach_device(struct device *dev, | |||
1551 | * left the caches in the IOMMU dirty. So we have to flush | 1635 | * left the caches in the IOMMU dirty. So we have to flush |
1552 | * here to evict all dirty stuff. | 1636 | * here to evict all dirty stuff. |
1553 | */ | 1637 | */ |
1554 | iommu_flush_tlb_pde(domain); | 1638 | domain_flush_tlb_pde(domain); |
1555 | 1639 | ||
1556 | return ret; | 1640 | return ret; |
1557 | } | 1641 | } |
@@ -1598,12 +1682,16 @@ static void __detach_device(struct device *dev) | |||
1598 | */ | 1682 | */ |
1599 | static void detach_device(struct device *dev) | 1683 | static void detach_device(struct device *dev) |
1600 | { | 1684 | { |
1685 | struct pci_dev *pdev = to_pci_dev(dev); | ||
1601 | unsigned long flags; | 1686 | unsigned long flags; |
1602 | 1687 | ||
1603 | /* lock device table */ | 1688 | /* lock device table */ |
1604 | write_lock_irqsave(&amd_iommu_devtable_lock, flags); | 1689 | write_lock_irqsave(&amd_iommu_devtable_lock, flags); |
1605 | __detach_device(dev); | 1690 | __detach_device(dev); |
1606 | write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); | 1691 | write_unlock_irqrestore(&amd_iommu_devtable_lock, flags); |
1692 | |||
1693 | if (amd_iommu_iotlb_sup && pci_ats_enabled(pdev)) | ||
1694 | pci_disable_ats(pdev); | ||
1607 | } | 1695 | } |
1608 | 1696 | ||
1609 | /* | 1697 | /* |
@@ -1692,7 +1780,7 @@ static int device_change_notifier(struct notifier_block *nb, | |||
1692 | goto out; | 1780 | goto out; |
1693 | } | 1781 | } |
1694 | 1782 | ||
1695 | iommu_flush_device(dev); | 1783 | device_flush_dte(dev); |
1696 | iommu_completion_wait(iommu); | 1784 | iommu_completion_wait(iommu); |
1697 | 1785 | ||
1698 | out: | 1786 | out: |
@@ -1753,8 +1841,9 @@ static void update_device_table(struct protection_domain *domain) | |||
1753 | struct iommu_dev_data *dev_data; | 1841 | struct iommu_dev_data *dev_data; |
1754 | 1842 | ||
1755 | list_for_each_entry(dev_data, &domain->dev_list, list) { | 1843 | list_for_each_entry(dev_data, &domain->dev_list, list) { |
1844 | struct pci_dev *pdev = to_pci_dev(dev_data->dev); | ||
1756 | u16 devid = get_device_id(dev_data->dev); | 1845 | u16 devid = get_device_id(dev_data->dev); |
1757 | set_dte_entry(devid, domain); | 1846 | set_dte_entry(devid, domain, pci_ats_enabled(pdev)); |
1758 | } | 1847 | } |
1759 | } | 1848 | } |
1760 | 1849 | ||
@@ -1764,8 +1853,9 @@ static void update_domain(struct protection_domain *domain) | |||
1764 | return; | 1853 | return; |
1765 | 1854 | ||
1766 | update_device_table(domain); | 1855 | update_device_table(domain); |
1767 | iommu_flush_domain_devices(domain); | 1856 | |
1768 | iommu_flush_tlb_pde(domain); | 1857 | domain_flush_devices(domain); |
1858 | domain_flush_tlb_pde(domain); | ||
1769 | 1859 | ||
1770 | domain->updated = false; | 1860 | domain->updated = false; |
1771 | } | 1861 | } |
@@ -1924,10 +2014,10 @@ retry: | |||
1924 | ADD_STATS_COUNTER(alloced_io_mem, size); | 2014 | ADD_STATS_COUNTER(alloced_io_mem, size); |
1925 | 2015 | ||
1926 | if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) { | 2016 | if (unlikely(dma_dom->need_flush && !amd_iommu_unmap_flush)) { |
1927 | iommu_flush_tlb(&dma_dom->domain); | 2017 | domain_flush_tlb(&dma_dom->domain); |
1928 | dma_dom->need_flush = false; | 2018 | dma_dom->need_flush = false; |
1929 | } else if (unlikely(amd_iommu_np_cache)) | 2019 | } else if (unlikely(amd_iommu_np_cache)) |
1930 | iommu_flush_pages(&dma_dom->domain, address, size); | 2020 | domain_flush_pages(&dma_dom->domain, address, size); |
1931 | 2021 | ||
1932 | out: | 2022 | out: |
1933 | return address; | 2023 | return address; |
@@ -1976,7 +2066,7 @@ static void __unmap_single(struct dma_ops_domain *dma_dom, | |||
1976 | dma_ops_free_addresses(dma_dom, dma_addr, pages); | 2066 | dma_ops_free_addresses(dma_dom, dma_addr, pages); |
1977 | 2067 | ||
1978 | if (amd_iommu_unmap_flush || dma_dom->need_flush) { | 2068 | if (amd_iommu_unmap_flush || dma_dom->need_flush) { |
1979 | iommu_flush_pages(&dma_dom->domain, flush_addr, size); | 2069 | domain_flush_pages(&dma_dom->domain, flush_addr, size); |
1980 | dma_dom->need_flush = false; | 2070 | dma_dom->need_flush = false; |
1981 | } | 2071 | } |
1982 | } | 2072 | } |
@@ -2012,7 +2102,7 @@ static dma_addr_t map_page(struct device *dev, struct page *page, | |||
2012 | if (addr == DMA_ERROR_CODE) | 2102 | if (addr == DMA_ERROR_CODE) |
2013 | goto out; | 2103 | goto out; |
2014 | 2104 | ||
2015 | iommu_flush_complete(domain); | 2105 | domain_flush_complete(domain); |
2016 | 2106 | ||
2017 | out: | 2107 | out: |
2018 | spin_unlock_irqrestore(&domain->lock, flags); | 2108 | spin_unlock_irqrestore(&domain->lock, flags); |
@@ -2039,7 +2129,7 @@ static void unmap_page(struct device *dev, dma_addr_t dma_addr, size_t size, | |||
2039 | 2129 | ||
2040 | __unmap_single(domain->priv, dma_addr, size, dir); | 2130 | __unmap_single(domain->priv, dma_addr, size, dir); |
2041 | 2131 | ||
2042 | iommu_flush_complete(domain); | 2132 | domain_flush_complete(domain); |
2043 | 2133 | ||
2044 | spin_unlock_irqrestore(&domain->lock, flags); | 2134 | spin_unlock_irqrestore(&domain->lock, flags); |
2045 | } | 2135 | } |
@@ -2104,7 +2194,7 @@ static int map_sg(struct device *dev, struct scatterlist *sglist, | |||
2104 | goto unmap; | 2194 | goto unmap; |
2105 | } | 2195 | } |
2106 | 2196 | ||
2107 | iommu_flush_complete(domain); | 2197 | domain_flush_complete(domain); |
2108 | 2198 | ||
2109 | out: | 2199 | out: |
2110 | spin_unlock_irqrestore(&domain->lock, flags); | 2200 | spin_unlock_irqrestore(&domain->lock, flags); |
@@ -2150,7 +2240,7 @@ static void unmap_sg(struct device *dev, struct scatterlist *sglist, | |||
2150 | s->dma_address = s->dma_length = 0; | 2240 | s->dma_address = s->dma_length = 0; |
2151 | } | 2241 | } |
2152 | 2242 | ||
2153 | iommu_flush_complete(domain); | 2243 | domain_flush_complete(domain); |
2154 | 2244 | ||
2155 | spin_unlock_irqrestore(&domain->lock, flags); | 2245 | spin_unlock_irqrestore(&domain->lock, flags); |
2156 | } | 2246 | } |
@@ -2200,7 +2290,7 @@ static void *alloc_coherent(struct device *dev, size_t size, | |||
2200 | goto out_free; | 2290 | goto out_free; |
2201 | } | 2291 | } |
2202 | 2292 | ||
2203 | iommu_flush_complete(domain); | 2293 | domain_flush_complete(domain); |
2204 | 2294 | ||
2205 | spin_unlock_irqrestore(&domain->lock, flags); | 2295 | spin_unlock_irqrestore(&domain->lock, flags); |
2206 | 2296 | ||
@@ -2232,7 +2322,7 @@ static void free_coherent(struct device *dev, size_t size, | |||
2232 | 2322 | ||
2233 | __unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL); | 2323 | __unmap_single(domain->priv, dma_addr, size, DMA_BIDIRECTIONAL); |
2234 | 2324 | ||
2235 | iommu_flush_complete(domain); | 2325 | domain_flush_complete(domain); |
2236 | 2326 | ||
2237 | spin_unlock_irqrestore(&domain->lock, flags); | 2327 | spin_unlock_irqrestore(&domain->lock, flags); |
2238 | 2328 | ||
@@ -2476,7 +2566,7 @@ static void amd_iommu_detach_device(struct iommu_domain *dom, | |||
2476 | if (!iommu) | 2566 | if (!iommu) |
2477 | return; | 2567 | return; |
2478 | 2568 | ||
2479 | iommu_flush_device(dev); | 2569 | device_flush_dte(dev); |
2480 | iommu_completion_wait(iommu); | 2570 | iommu_completion_wait(iommu); |
2481 | } | 2571 | } |
2482 | 2572 | ||
@@ -2542,7 +2632,7 @@ static int amd_iommu_unmap(struct iommu_domain *dom, unsigned long iova, | |||
2542 | unmap_size = iommu_unmap_page(domain, iova, page_size); | 2632 | unmap_size = iommu_unmap_page(domain, iova, page_size); |
2543 | mutex_unlock(&domain->api_lock); | 2633 | mutex_unlock(&domain->api_lock); |
2544 | 2634 | ||
2545 | iommu_flush_tlb_pde(domain); | 2635 | domain_flush_tlb_pde(domain); |
2546 | 2636 | ||
2547 | return get_order(unmap_size); | 2637 | return get_order(unmap_size); |
2548 | } | 2638 | } |