diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2015-02-10 16:56:56 -0500 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2015-02-10 16:56:56 -0500 |
commit | bdccc4edeb03ad68c55053b0260bdaaac547bbd9 (patch) | |
tree | c9fdce7c6bb13632f5a64925a1fb3fa306b7cfe1 | |
parent | 98368ab436538103a557fc1f15f54afd8aab6712 (diff) | |
parent | 72978b2fe2f2cdf9f319c6c6dcdbe92b38de2be2 (diff) |
Merge tag 'stable/for-linus-3.20-rc0-tag' of git://git.kernel.org/pub/scm/linux/kernel/git/xen/tip
Pull xen features and fixes from David Vrabel:
- Reworked handling for foreign (grant mapped) pages to simplify the
code, enable a number of additional use cases and fix a number of
long-standing bugs.
- Prefer the TSC over the Xen PV clock when dom0 (and the TSC is
stable).
- Assorted other cleanup and minor bug fixes.
* tag 'stable/for-linus-3.20-rc0-tag' of git://git.kernel.org/pub/scm/linux/kernel/git/xen/tip: (25 commits)
xen/manage: Fix USB interaction issues when resuming
xenbus: Add proper handling of XS_ERROR from Xenbus for transactions.
xen/gntdev: provide find_special_page VMA operation
xen/gntdev: mark userspace PTEs as special on x86 PV guests
xen-blkback: safely unmap grants in case they are still in use
xen/gntdev: safely unmap grants in case they are still in use
xen/gntdev: convert priv->lock to a mutex
xen/grant-table: add a mechanism to safely unmap pages that are in use
xen-netback: use foreign page information from the pages themselves
xen: mark grant mapped pages as foreign
xen/grant-table: add helpers for allocating pages
x86/xen: require ballooned pages for grant maps
xen: remove scratch frames for ballooned pages and m2p override
xen/grant-table: pre-populate kernel unmap ops for xen_gnttab_unmap_refs()
mm: add 'foreign' alias for the 'pinned' page flag
mm: provide a find_special_page vma operation
x86/xen: cleanup arch/x86/xen/mmu.c
x86/xen: add some __init annotations in arch/x86/xen/mmu.c
x86/xen: add some __init and static annotations in arch/x86/xen/setup.c
x86/xen: use correct types for addresses in arch/x86/xen/setup.c
...
28 files changed, 509 insertions, 594 deletions
diff --git a/arch/arm/include/asm/xen/page.h b/arch/arm/include/asm/xen/page.h index 68c739b3fdf4..2f7e6ff67d51 100644 --- a/arch/arm/include/asm/xen/page.h +++ b/arch/arm/include/asm/xen/page.h | |||
@@ -92,7 +92,7 @@ extern int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops, | |||
92 | struct page **pages, unsigned int count); | 92 | struct page **pages, unsigned int count); |
93 | 93 | ||
94 | extern int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops, | 94 | extern int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops, |
95 | struct gnttab_map_grant_ref *kmap_ops, | 95 | struct gnttab_unmap_grant_ref *kunmap_ops, |
96 | struct page **pages, unsigned int count); | 96 | struct page **pages, unsigned int count); |
97 | 97 | ||
98 | bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn); | 98 | bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn); |
diff --git a/arch/arm/xen/enlighten.c b/arch/arm/xen/enlighten.c index c7ca936ebd99..263a2044c65b 100644 --- a/arch/arm/xen/enlighten.c +++ b/arch/arm/xen/enlighten.c | |||
@@ -29,10 +29,10 @@ | |||
29 | 29 | ||
30 | struct start_info _xen_start_info; | 30 | struct start_info _xen_start_info; |
31 | struct start_info *xen_start_info = &_xen_start_info; | 31 | struct start_info *xen_start_info = &_xen_start_info; |
32 | EXPORT_SYMBOL_GPL(xen_start_info); | 32 | EXPORT_SYMBOL(xen_start_info); |
33 | 33 | ||
34 | enum xen_domain_type xen_domain_type = XEN_NATIVE; | 34 | enum xen_domain_type xen_domain_type = XEN_NATIVE; |
35 | EXPORT_SYMBOL_GPL(xen_domain_type); | 35 | EXPORT_SYMBOL(xen_domain_type); |
36 | 36 | ||
37 | struct shared_info xen_dummy_shared_info; | 37 | struct shared_info xen_dummy_shared_info; |
38 | struct shared_info *HYPERVISOR_shared_info = (void *)&xen_dummy_shared_info; | 38 | struct shared_info *HYPERVISOR_shared_info = (void *)&xen_dummy_shared_info; |
diff --git a/arch/arm/xen/mm.c b/arch/arm/xen/mm.c index 351b24a979d4..793551d15f1d 100644 --- a/arch/arm/xen/mm.c +++ b/arch/arm/xen/mm.c | |||
@@ -149,7 +149,7 @@ void xen_destroy_contiguous_region(phys_addr_t pstart, unsigned int order) | |||
149 | EXPORT_SYMBOL_GPL(xen_destroy_contiguous_region); | 149 | EXPORT_SYMBOL_GPL(xen_destroy_contiguous_region); |
150 | 150 | ||
151 | struct dma_map_ops *xen_dma_ops; | 151 | struct dma_map_ops *xen_dma_ops; |
152 | EXPORT_SYMBOL_GPL(xen_dma_ops); | 152 | EXPORT_SYMBOL(xen_dma_ops); |
153 | 153 | ||
154 | static struct dma_map_ops xen_swiotlb_dma_ops = { | 154 | static struct dma_map_ops xen_swiotlb_dma_ops = { |
155 | .mapping_error = xen_swiotlb_dma_mapping_error, | 155 | .mapping_error = xen_swiotlb_dma_mapping_error, |
diff --git a/arch/arm/xen/p2m.c b/arch/arm/xen/p2m.c index 054857776254..cb7a14c5cd69 100644 --- a/arch/arm/xen/p2m.c +++ b/arch/arm/xen/p2m.c | |||
@@ -102,7 +102,7 @@ int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops, | |||
102 | EXPORT_SYMBOL_GPL(set_foreign_p2m_mapping); | 102 | EXPORT_SYMBOL_GPL(set_foreign_p2m_mapping); |
103 | 103 | ||
104 | int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops, | 104 | int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops, |
105 | struct gnttab_map_grant_ref *kmap_ops, | 105 | struct gnttab_unmap_grant_ref *kunmap_ops, |
106 | struct page **pages, unsigned int count) | 106 | struct page **pages, unsigned int count) |
107 | { | 107 | { |
108 | int i; | 108 | int i; |
diff --git a/arch/x86/include/asm/xen/page.h b/arch/x86/include/asm/xen/page.h index 5eea09915a15..358dcd338915 100644 --- a/arch/x86/include/asm/xen/page.h +++ b/arch/x86/include/asm/xen/page.h | |||
@@ -55,9 +55,8 @@ extern int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops, | |||
55 | struct gnttab_map_grant_ref *kmap_ops, | 55 | struct gnttab_map_grant_ref *kmap_ops, |
56 | struct page **pages, unsigned int count); | 56 | struct page **pages, unsigned int count); |
57 | extern int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops, | 57 | extern int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops, |
58 | struct gnttab_map_grant_ref *kmap_ops, | 58 | struct gnttab_unmap_grant_ref *kunmap_ops, |
59 | struct page **pages, unsigned int count); | 59 | struct page **pages, unsigned int count); |
60 | extern unsigned long m2p_find_override_pfn(unsigned long mfn, unsigned long pfn); | ||
61 | 60 | ||
62 | /* | 61 | /* |
63 | * Helper functions to write or read unsigned long values to/from | 62 | * Helper functions to write or read unsigned long values to/from |
@@ -154,21 +153,12 @@ static inline unsigned long mfn_to_pfn(unsigned long mfn) | |||
154 | return mfn; | 153 | return mfn; |
155 | 154 | ||
156 | pfn = mfn_to_pfn_no_overrides(mfn); | 155 | pfn = mfn_to_pfn_no_overrides(mfn); |
157 | if (__pfn_to_mfn(pfn) != mfn) { | 156 | if (__pfn_to_mfn(pfn) != mfn) |
158 | /* | 157 | pfn = ~0; |
159 | * If this appears to be a foreign mfn (because the pfn | ||
160 | * doesn't map back to the mfn), then check the local override | ||
161 | * table to see if there's a better pfn to use. | ||
162 | * | ||
163 | * m2p_find_override_pfn returns ~0 if it doesn't find anything. | ||
164 | */ | ||
165 | pfn = m2p_find_override_pfn(mfn, ~0); | ||
166 | } | ||
167 | 158 | ||
168 | /* | 159 | /* |
169 | * pfn is ~0 if there are no entries in the m2p for mfn or if the | 160 | * pfn is ~0 if there are no entries in the m2p for mfn or the |
170 | * entry doesn't map back to the mfn and m2p_override doesn't have a | 161 | * entry doesn't map back to the mfn. |
171 | * valid entry for it. | ||
172 | */ | 162 | */ |
173 | if (pfn == ~0 && __pfn_to_mfn(mfn) == IDENTITY_FRAME(mfn)) | 163 | if (pfn == ~0 && __pfn_to_mfn(mfn) == IDENTITY_FRAME(mfn)) |
174 | pfn = mfn; | 164 | pfn = mfn; |
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c index 5c1f9ace7ae7..adca9e2b6553 100644 --- a/arch/x86/xen/mmu.c +++ b/arch/x86/xen/mmu.c | |||
@@ -1489,7 +1489,7 @@ static void __init xen_set_pte_init(pte_t *ptep, pte_t pte) | |||
1489 | native_set_pte(ptep, pte); | 1489 | native_set_pte(ptep, pte); |
1490 | } | 1490 | } |
1491 | 1491 | ||
1492 | static void pin_pagetable_pfn(unsigned cmd, unsigned long pfn) | 1492 | static void __init pin_pagetable_pfn(unsigned cmd, unsigned long pfn) |
1493 | { | 1493 | { |
1494 | struct mmuext_op op; | 1494 | struct mmuext_op op; |
1495 | op.cmd = cmd; | 1495 | op.cmd = cmd; |
@@ -1657,7 +1657,7 @@ void __init xen_reserve_top(void) | |||
1657 | * Like __va(), but returns address in the kernel mapping (which is | 1657 | * Like __va(), but returns address in the kernel mapping (which is |
1658 | * all we have until the physical memory mapping has been set up. | 1658 | * all we have until the physical memory mapping has been set up. |
1659 | */ | 1659 | */ |
1660 | static void *__ka(phys_addr_t paddr) | 1660 | static void * __init __ka(phys_addr_t paddr) |
1661 | { | 1661 | { |
1662 | #ifdef CONFIG_X86_64 | 1662 | #ifdef CONFIG_X86_64 |
1663 | return (void *)(paddr + __START_KERNEL_map); | 1663 | return (void *)(paddr + __START_KERNEL_map); |
@@ -1667,7 +1667,7 @@ static void *__ka(phys_addr_t paddr) | |||
1667 | } | 1667 | } |
1668 | 1668 | ||
1669 | /* Convert a machine address to physical address */ | 1669 | /* Convert a machine address to physical address */ |
1670 | static unsigned long m2p(phys_addr_t maddr) | 1670 | static unsigned long __init m2p(phys_addr_t maddr) |
1671 | { | 1671 | { |
1672 | phys_addr_t paddr; | 1672 | phys_addr_t paddr; |
1673 | 1673 | ||
@@ -1678,13 +1678,14 @@ static unsigned long m2p(phys_addr_t maddr) | |||
1678 | } | 1678 | } |
1679 | 1679 | ||
1680 | /* Convert a machine address to kernel virtual */ | 1680 | /* Convert a machine address to kernel virtual */ |
1681 | static void *m2v(phys_addr_t maddr) | 1681 | static void * __init m2v(phys_addr_t maddr) |
1682 | { | 1682 | { |
1683 | return __ka(m2p(maddr)); | 1683 | return __ka(m2p(maddr)); |
1684 | } | 1684 | } |
1685 | 1685 | ||
1686 | /* Set the page permissions on an identity-mapped pages */ | 1686 | /* Set the page permissions on an identity-mapped pages */ |
1687 | static void set_page_prot_flags(void *addr, pgprot_t prot, unsigned long flags) | 1687 | static void __init set_page_prot_flags(void *addr, pgprot_t prot, |
1688 | unsigned long flags) | ||
1688 | { | 1689 | { |
1689 | unsigned long pfn = __pa(addr) >> PAGE_SHIFT; | 1690 | unsigned long pfn = __pa(addr) >> PAGE_SHIFT; |
1690 | pte_t pte = pfn_pte(pfn, prot); | 1691 | pte_t pte = pfn_pte(pfn, prot); |
@@ -1696,7 +1697,7 @@ static void set_page_prot_flags(void *addr, pgprot_t prot, unsigned long flags) | |||
1696 | if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, flags)) | 1697 | if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, flags)) |
1697 | BUG(); | 1698 | BUG(); |
1698 | } | 1699 | } |
1699 | static void set_page_prot(void *addr, pgprot_t prot) | 1700 | static void __init set_page_prot(void *addr, pgprot_t prot) |
1700 | { | 1701 | { |
1701 | return set_page_prot_flags(addr, prot, UVMF_NONE); | 1702 | return set_page_prot_flags(addr, prot, UVMF_NONE); |
1702 | } | 1703 | } |
@@ -1733,10 +1734,8 @@ static void __init xen_map_identity_early(pmd_t *pmd, unsigned long max_pfn) | |||
1733 | for (pteidx = 0; pteidx < PTRS_PER_PTE; pteidx++, pfn++) { | 1734 | for (pteidx = 0; pteidx < PTRS_PER_PTE; pteidx++, pfn++) { |
1734 | pte_t pte; | 1735 | pte_t pte; |
1735 | 1736 | ||
1736 | #ifdef CONFIG_X86_32 | ||
1737 | if (pfn > max_pfn_mapped) | 1737 | if (pfn > max_pfn_mapped) |
1738 | max_pfn_mapped = pfn; | 1738 | max_pfn_mapped = pfn; |
1739 | #endif | ||
1740 | 1739 | ||
1741 | if (!pte_none(pte_page[pteidx])) | 1740 | if (!pte_none(pte_page[pteidx])) |
1742 | continue; | 1741 | continue; |
@@ -1769,7 +1768,7 @@ void __init xen_setup_machphys_mapping(void) | |||
1769 | } | 1768 | } |
1770 | 1769 | ||
1771 | #ifdef CONFIG_X86_64 | 1770 | #ifdef CONFIG_X86_64 |
1772 | static void convert_pfn_mfn(void *v) | 1771 | static void __init convert_pfn_mfn(void *v) |
1773 | { | 1772 | { |
1774 | pte_t *pte = v; | 1773 | pte_t *pte = v; |
1775 | int i; | 1774 | int i; |
diff --git a/arch/x86/xen/p2m.c b/arch/x86/xen/p2m.c index 70fb5075c901..f18fd1d411f6 100644 --- a/arch/x86/xen/p2m.c +++ b/arch/x86/xen/p2m.c | |||
@@ -84,8 +84,6 @@ | |||
84 | 84 | ||
85 | #define PMDS_PER_MID_PAGE (P2M_MID_PER_PAGE / PTRS_PER_PTE) | 85 | #define PMDS_PER_MID_PAGE (P2M_MID_PER_PAGE / PTRS_PER_PTE) |
86 | 86 | ||
87 | static void __init m2p_override_init(void); | ||
88 | |||
89 | unsigned long *xen_p2m_addr __read_mostly; | 87 | unsigned long *xen_p2m_addr __read_mostly; |
90 | EXPORT_SYMBOL_GPL(xen_p2m_addr); | 88 | EXPORT_SYMBOL_GPL(xen_p2m_addr); |
91 | unsigned long xen_p2m_size __read_mostly; | 89 | unsigned long xen_p2m_size __read_mostly; |
@@ -402,8 +400,6 @@ void __init xen_vmalloc_p2m_tree(void) | |||
402 | xen_p2m_size = xen_max_p2m_pfn; | 400 | xen_p2m_size = xen_max_p2m_pfn; |
403 | 401 | ||
404 | xen_inv_extra_mem(); | 402 | xen_inv_extra_mem(); |
405 | |||
406 | m2p_override_init(); | ||
407 | } | 403 | } |
408 | 404 | ||
409 | unsigned long get_phys_to_machine(unsigned long pfn) | 405 | unsigned long get_phys_to_machine(unsigned long pfn) |
@@ -652,100 +648,21 @@ bool set_phys_to_machine(unsigned long pfn, unsigned long mfn) | |||
652 | return true; | 648 | return true; |
653 | } | 649 | } |
654 | 650 | ||
655 | #define M2P_OVERRIDE_HASH_SHIFT 10 | ||
656 | #define M2P_OVERRIDE_HASH (1 << M2P_OVERRIDE_HASH_SHIFT) | ||
657 | |||
658 | static struct list_head *m2p_overrides; | ||
659 | static DEFINE_SPINLOCK(m2p_override_lock); | ||
660 | |||
661 | static void __init m2p_override_init(void) | ||
662 | { | ||
663 | unsigned i; | ||
664 | |||
665 | m2p_overrides = alloc_bootmem_align( | ||
666 | sizeof(*m2p_overrides) * M2P_OVERRIDE_HASH, | ||
667 | sizeof(unsigned long)); | ||
668 | |||
669 | for (i = 0; i < M2P_OVERRIDE_HASH; i++) | ||
670 | INIT_LIST_HEAD(&m2p_overrides[i]); | ||
671 | } | ||
672 | |||
673 | static unsigned long mfn_hash(unsigned long mfn) | ||
674 | { | ||
675 | return hash_long(mfn, M2P_OVERRIDE_HASH_SHIFT); | ||
676 | } | ||
677 | |||
678 | /* Add an MFN override for a particular page */ | ||
679 | static int m2p_add_override(unsigned long mfn, struct page *page, | ||
680 | struct gnttab_map_grant_ref *kmap_op) | ||
681 | { | ||
682 | unsigned long flags; | ||
683 | unsigned long pfn; | ||
684 | unsigned long uninitialized_var(address); | ||
685 | unsigned level; | ||
686 | pte_t *ptep = NULL; | ||
687 | |||
688 | pfn = page_to_pfn(page); | ||
689 | if (!PageHighMem(page)) { | ||
690 | address = (unsigned long)__va(pfn << PAGE_SHIFT); | ||
691 | ptep = lookup_address(address, &level); | ||
692 | if (WARN(ptep == NULL || level != PG_LEVEL_4K, | ||
693 | "m2p_add_override: pfn %lx not mapped", pfn)) | ||
694 | return -EINVAL; | ||
695 | } | ||
696 | |||
697 | if (kmap_op != NULL) { | ||
698 | if (!PageHighMem(page)) { | ||
699 | struct multicall_space mcs = | ||
700 | xen_mc_entry(sizeof(*kmap_op)); | ||
701 | |||
702 | MULTI_grant_table_op(mcs.mc, | ||
703 | GNTTABOP_map_grant_ref, kmap_op, 1); | ||
704 | |||
705 | xen_mc_issue(PARAVIRT_LAZY_MMU); | ||
706 | } | ||
707 | } | ||
708 | spin_lock_irqsave(&m2p_override_lock, flags); | ||
709 | list_add(&page->lru, &m2p_overrides[mfn_hash(mfn)]); | ||
710 | spin_unlock_irqrestore(&m2p_override_lock, flags); | ||
711 | |||
712 | /* p2m(m2p(mfn)) == mfn: the mfn is already present somewhere in | ||
713 | * this domain. Set the FOREIGN_FRAME_BIT in the p2m for the other | ||
714 | * pfn so that the following mfn_to_pfn(mfn) calls will return the | ||
715 | * pfn from the m2p_override (the backend pfn) instead. | ||
716 | * We need to do this because the pages shared by the frontend | ||
717 | * (xen-blkfront) can be already locked (lock_page, called by | ||
718 | * do_read_cache_page); when the userspace backend tries to use them | ||
719 | * with direct_IO, mfn_to_pfn returns the pfn of the frontend, so | ||
720 | * do_blockdev_direct_IO is going to try to lock the same pages | ||
721 | * again resulting in a deadlock. | ||
722 | * As a side effect get_user_pages_fast might not be safe on the | ||
723 | * frontend pages while they are being shared with the backend, | ||
724 | * because mfn_to_pfn (that ends up being called by GUPF) will | ||
725 | * return the backend pfn rather than the frontend pfn. */ | ||
726 | pfn = mfn_to_pfn_no_overrides(mfn); | ||
727 | if (__pfn_to_mfn(pfn) == mfn) | ||
728 | set_phys_to_machine(pfn, FOREIGN_FRAME(mfn)); | ||
729 | |||
730 | return 0; | ||
731 | } | ||
732 | |||
733 | int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops, | 651 | int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops, |
734 | struct gnttab_map_grant_ref *kmap_ops, | 652 | struct gnttab_map_grant_ref *kmap_ops, |
735 | struct page **pages, unsigned int count) | 653 | struct page **pages, unsigned int count) |
736 | { | 654 | { |
737 | int i, ret = 0; | 655 | int i, ret = 0; |
738 | bool lazy = false; | ||
739 | pte_t *pte; | 656 | pte_t *pte; |
740 | 657 | ||
741 | if (xen_feature(XENFEAT_auto_translated_physmap)) | 658 | if (xen_feature(XENFEAT_auto_translated_physmap)) |
742 | return 0; | 659 | return 0; |
743 | 660 | ||
744 | if (kmap_ops && | 661 | if (kmap_ops) { |
745 | !in_interrupt() && | 662 | ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, |
746 | paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) { | 663 | kmap_ops, count); |
747 | arch_enter_lazy_mmu_mode(); | 664 | if (ret) |
748 | lazy = true; | 665 | goto out; |
749 | } | 666 | } |
750 | 667 | ||
751 | for (i = 0; i < count; i++) { | 668 | for (i = 0; i < count; i++) { |
@@ -764,170 +681,28 @@ int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops, | |||
764 | } | 681 | } |
765 | pfn = page_to_pfn(pages[i]); | 682 | pfn = page_to_pfn(pages[i]); |
766 | 683 | ||
767 | WARN_ON(PagePrivate(pages[i])); | 684 | WARN(pfn_to_mfn(pfn) != INVALID_P2M_ENTRY, "page must be ballooned"); |
768 | SetPagePrivate(pages[i]); | ||
769 | set_page_private(pages[i], mfn); | ||
770 | pages[i]->index = pfn_to_mfn(pfn); | ||
771 | 685 | ||
772 | if (unlikely(!set_phys_to_machine(pfn, FOREIGN_FRAME(mfn)))) { | 686 | if (unlikely(!set_phys_to_machine(pfn, FOREIGN_FRAME(mfn)))) { |
773 | ret = -ENOMEM; | 687 | ret = -ENOMEM; |
774 | goto out; | 688 | goto out; |
775 | } | 689 | } |
776 | |||
777 | if (kmap_ops) { | ||
778 | ret = m2p_add_override(mfn, pages[i], &kmap_ops[i]); | ||
779 | if (ret) | ||
780 | goto out; | ||
781 | } | ||
782 | } | 690 | } |
783 | 691 | ||
784 | out: | 692 | out: |
785 | if (lazy) | ||
786 | arch_leave_lazy_mmu_mode(); | ||
787 | |||
788 | return ret; | 693 | return ret; |
789 | } | 694 | } |
790 | EXPORT_SYMBOL_GPL(set_foreign_p2m_mapping); | 695 | EXPORT_SYMBOL_GPL(set_foreign_p2m_mapping); |
791 | 696 | ||
792 | static struct page *m2p_find_override(unsigned long mfn) | ||
793 | { | ||
794 | unsigned long flags; | ||
795 | struct list_head *bucket; | ||
796 | struct page *p, *ret; | ||
797 | |||
798 | if (unlikely(!m2p_overrides)) | ||
799 | return NULL; | ||
800 | |||
801 | ret = NULL; | ||
802 | bucket = &m2p_overrides[mfn_hash(mfn)]; | ||
803 | |||
804 | spin_lock_irqsave(&m2p_override_lock, flags); | ||
805 | |||
806 | list_for_each_entry(p, bucket, lru) { | ||
807 | if (page_private(p) == mfn) { | ||
808 | ret = p; | ||
809 | break; | ||
810 | } | ||
811 | } | ||
812 | |||
813 | spin_unlock_irqrestore(&m2p_override_lock, flags); | ||
814 | |||
815 | return ret; | ||
816 | } | ||
817 | |||
818 | static int m2p_remove_override(struct page *page, | ||
819 | struct gnttab_map_grant_ref *kmap_op, | ||
820 | unsigned long mfn) | ||
821 | { | ||
822 | unsigned long flags; | ||
823 | unsigned long pfn; | ||
824 | unsigned long uninitialized_var(address); | ||
825 | unsigned level; | ||
826 | pte_t *ptep = NULL; | ||
827 | |||
828 | pfn = page_to_pfn(page); | ||
829 | |||
830 | if (!PageHighMem(page)) { | ||
831 | address = (unsigned long)__va(pfn << PAGE_SHIFT); | ||
832 | ptep = lookup_address(address, &level); | ||
833 | |||
834 | if (WARN(ptep == NULL || level != PG_LEVEL_4K, | ||
835 | "m2p_remove_override: pfn %lx not mapped", pfn)) | ||
836 | return -EINVAL; | ||
837 | } | ||
838 | |||
839 | spin_lock_irqsave(&m2p_override_lock, flags); | ||
840 | list_del(&page->lru); | ||
841 | spin_unlock_irqrestore(&m2p_override_lock, flags); | ||
842 | |||
843 | if (kmap_op != NULL) { | ||
844 | if (!PageHighMem(page)) { | ||
845 | struct multicall_space mcs; | ||
846 | struct gnttab_unmap_and_replace *unmap_op; | ||
847 | struct page *scratch_page = get_balloon_scratch_page(); | ||
848 | unsigned long scratch_page_address = (unsigned long) | ||
849 | __va(page_to_pfn(scratch_page) << PAGE_SHIFT); | ||
850 | |||
851 | /* | ||
852 | * It might be that we queued all the m2p grant table | ||
853 | * hypercalls in a multicall, then m2p_remove_override | ||
854 | * get called before the multicall has actually been | ||
855 | * issued. In this case handle is going to -1 because | ||
856 | * it hasn't been modified yet. | ||
857 | */ | ||
858 | if (kmap_op->handle == -1) | ||
859 | xen_mc_flush(); | ||
860 | /* | ||
861 | * Now if kmap_op->handle is negative it means that the | ||
862 | * hypercall actually returned an error. | ||
863 | */ | ||
864 | if (kmap_op->handle == GNTST_general_error) { | ||
865 | pr_warn("m2p_remove_override: pfn %lx mfn %lx, failed to modify kernel mappings", | ||
866 | pfn, mfn); | ||
867 | put_balloon_scratch_page(); | ||
868 | return -1; | ||
869 | } | ||
870 | |||
871 | xen_mc_batch(); | ||
872 | |||
873 | mcs = __xen_mc_entry( | ||
874 | sizeof(struct gnttab_unmap_and_replace)); | ||
875 | unmap_op = mcs.args; | ||
876 | unmap_op->host_addr = kmap_op->host_addr; | ||
877 | unmap_op->new_addr = scratch_page_address; | ||
878 | unmap_op->handle = kmap_op->handle; | ||
879 | |||
880 | MULTI_grant_table_op(mcs.mc, | ||
881 | GNTTABOP_unmap_and_replace, unmap_op, 1); | ||
882 | |||
883 | mcs = __xen_mc_entry(0); | ||
884 | MULTI_update_va_mapping(mcs.mc, scratch_page_address, | ||
885 | pfn_pte(page_to_pfn(scratch_page), | ||
886 | PAGE_KERNEL_RO), 0); | ||
887 | |||
888 | xen_mc_issue(PARAVIRT_LAZY_MMU); | ||
889 | |||
890 | kmap_op->host_addr = 0; | ||
891 | put_balloon_scratch_page(); | ||
892 | } | ||
893 | } | ||
894 | |||
895 | /* p2m(m2p(mfn)) == FOREIGN_FRAME(mfn): the mfn is already present | ||
896 | * somewhere in this domain, even before being added to the | ||
897 | * m2p_override (see comment above in m2p_add_override). | ||
898 | * If there are no other entries in the m2p_override corresponding | ||
899 | * to this mfn, then remove the FOREIGN_FRAME_BIT from the p2m for | ||
900 | * the original pfn (the one shared by the frontend): the backend | ||
901 | * cannot do any IO on this page anymore because it has been | ||
902 | * unshared. Removing the FOREIGN_FRAME_BIT from the p2m entry of | ||
903 | * the original pfn causes mfn_to_pfn(mfn) to return the frontend | ||
904 | * pfn again. */ | ||
905 | mfn &= ~FOREIGN_FRAME_BIT; | ||
906 | pfn = mfn_to_pfn_no_overrides(mfn); | ||
907 | if (__pfn_to_mfn(pfn) == FOREIGN_FRAME(mfn) && | ||
908 | m2p_find_override(mfn) == NULL) | ||
909 | set_phys_to_machine(pfn, mfn); | ||
910 | |||
911 | return 0; | ||
912 | } | ||
913 | |||
914 | int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops, | 697 | int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops, |
915 | struct gnttab_map_grant_ref *kmap_ops, | 698 | struct gnttab_unmap_grant_ref *kunmap_ops, |
916 | struct page **pages, unsigned int count) | 699 | struct page **pages, unsigned int count) |
917 | { | 700 | { |
918 | int i, ret = 0; | 701 | int i, ret = 0; |
919 | bool lazy = false; | ||
920 | 702 | ||
921 | if (xen_feature(XENFEAT_auto_translated_physmap)) | 703 | if (xen_feature(XENFEAT_auto_translated_physmap)) |
922 | return 0; | 704 | return 0; |
923 | 705 | ||
924 | if (kmap_ops && | ||
925 | !in_interrupt() && | ||
926 | paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) { | ||
927 | arch_enter_lazy_mmu_mode(); | ||
928 | lazy = true; | ||
929 | } | ||
930 | |||
931 | for (i = 0; i < count; i++) { | 706 | for (i = 0; i < count; i++) { |
932 | unsigned long mfn = __pfn_to_mfn(page_to_pfn(pages[i])); | 707 | unsigned long mfn = __pfn_to_mfn(page_to_pfn(pages[i])); |
933 | unsigned long pfn = page_to_pfn(pages[i]); | 708 | unsigned long pfn = page_to_pfn(pages[i]); |
@@ -937,36 +712,16 @@ int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops, | |||
937 | goto out; | 712 | goto out; |
938 | } | 713 | } |
939 | 714 | ||
940 | set_page_private(pages[i], INVALID_P2M_ENTRY); | 715 | set_phys_to_machine(pfn, INVALID_P2M_ENTRY); |
941 | WARN_ON(!PagePrivate(pages[i])); | ||
942 | ClearPagePrivate(pages[i]); | ||
943 | set_phys_to_machine(pfn, pages[i]->index); | ||
944 | |||
945 | if (kmap_ops) | ||
946 | ret = m2p_remove_override(pages[i], &kmap_ops[i], mfn); | ||
947 | if (ret) | ||
948 | goto out; | ||
949 | } | 716 | } |
950 | 717 | if (kunmap_ops) | |
718 | ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, | ||
719 | kunmap_ops, count); | ||
951 | out: | 720 | out: |
952 | if (lazy) | ||
953 | arch_leave_lazy_mmu_mode(); | ||
954 | return ret; | 721 | return ret; |
955 | } | 722 | } |
956 | EXPORT_SYMBOL_GPL(clear_foreign_p2m_mapping); | 723 | EXPORT_SYMBOL_GPL(clear_foreign_p2m_mapping); |
957 | 724 | ||
958 | unsigned long m2p_find_override_pfn(unsigned long mfn, unsigned long pfn) | ||
959 | { | ||
960 | struct page *p = m2p_find_override(mfn); | ||
961 | unsigned long ret = pfn; | ||
962 | |||
963 | if (p) | ||
964 | ret = page_to_pfn(p); | ||
965 | |||
966 | return ret; | ||
967 | } | ||
968 | EXPORT_SYMBOL_GPL(m2p_find_override_pfn); | ||
969 | |||
970 | #ifdef CONFIG_XEN_DEBUG_FS | 725 | #ifdef CONFIG_XEN_DEBUG_FS |
971 | #include <linux/debugfs.h> | 726 | #include <linux/debugfs.h> |
972 | #include "debugfs.h" | 727 | #include "debugfs.h" |
diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c index 865e56cea7a0..55f388ef481a 100644 --- a/arch/x86/xen/setup.c +++ b/arch/x86/xen/setup.c | |||
@@ -32,16 +32,6 @@ | |||
32 | #include "p2m.h" | 32 | #include "p2m.h" |
33 | #include "mmu.h" | 33 | #include "mmu.h" |
34 | 34 | ||
35 | /* These are code, but not functions. Defined in entry.S */ | ||
36 | extern const char xen_hypervisor_callback[]; | ||
37 | extern const char xen_failsafe_callback[]; | ||
38 | #ifdef CONFIG_X86_64 | ||
39 | extern asmlinkage void nmi(void); | ||
40 | #endif | ||
41 | extern void xen_sysenter_target(void); | ||
42 | extern void xen_syscall_target(void); | ||
43 | extern void xen_syscall32_target(void); | ||
44 | |||
45 | /* Amount of extra memory space we add to the e820 ranges */ | 35 | /* Amount of extra memory space we add to the e820 ranges */ |
46 | struct xen_memory_region xen_extra_mem[XEN_EXTRA_MEM_MAX_REGIONS] __initdata; | 36 | struct xen_memory_region xen_extra_mem[XEN_EXTRA_MEM_MAX_REGIONS] __initdata; |
47 | 37 | ||
@@ -74,7 +64,7 @@ static unsigned long xen_remap_mfn __initdata = INVALID_P2M_ENTRY; | |||
74 | */ | 64 | */ |
75 | #define EXTRA_MEM_RATIO (10) | 65 | #define EXTRA_MEM_RATIO (10) |
76 | 66 | ||
77 | static void __init xen_add_extra_mem(u64 start, u64 size) | 67 | static void __init xen_add_extra_mem(phys_addr_t start, phys_addr_t size) |
78 | { | 68 | { |
79 | int i; | 69 | int i; |
80 | 70 | ||
@@ -97,10 +87,10 @@ static void __init xen_add_extra_mem(u64 start, u64 size) | |||
97 | memblock_reserve(start, size); | 87 | memblock_reserve(start, size); |
98 | } | 88 | } |
99 | 89 | ||
100 | static void __init xen_del_extra_mem(u64 start, u64 size) | 90 | static void __init xen_del_extra_mem(phys_addr_t start, phys_addr_t size) |
101 | { | 91 | { |
102 | int i; | 92 | int i; |
103 | u64 start_r, size_r; | 93 | phys_addr_t start_r, size_r; |
104 | 94 | ||
105 | for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) { | 95 | for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) { |
106 | start_r = xen_extra_mem[i].start; | 96 | start_r = xen_extra_mem[i].start; |
@@ -267,7 +257,7 @@ static void __init xen_set_identity_and_release_chunk(unsigned long start_pfn, | |||
267 | static void __init xen_update_mem_tables(unsigned long pfn, unsigned long mfn) | 257 | static void __init xen_update_mem_tables(unsigned long pfn, unsigned long mfn) |
268 | { | 258 | { |
269 | struct mmu_update update = { | 259 | struct mmu_update update = { |
270 | .ptr = ((unsigned long long)mfn << PAGE_SHIFT) | MMU_MACHPHYS_UPDATE, | 260 | .ptr = ((uint64_t)mfn << PAGE_SHIFT) | MMU_MACHPHYS_UPDATE, |
271 | .val = pfn | 261 | .val = pfn |
272 | }; | 262 | }; |
273 | 263 | ||
@@ -545,20 +535,21 @@ static unsigned long __init xen_get_max_pages(void) | |||
545 | return min(max_pages, MAX_DOMAIN_PAGES); | 535 | return min(max_pages, MAX_DOMAIN_PAGES); |
546 | } | 536 | } |
547 | 537 | ||
548 | static void xen_align_and_add_e820_region(u64 start, u64 size, int type) | 538 | static void __init xen_align_and_add_e820_region(phys_addr_t start, |
539 | phys_addr_t size, int type) | ||
549 | { | 540 | { |
550 | u64 end = start + size; | 541 | phys_addr_t end = start + size; |
551 | 542 | ||
552 | /* Align RAM regions to page boundaries. */ | 543 | /* Align RAM regions to page boundaries. */ |
553 | if (type == E820_RAM) { | 544 | if (type == E820_RAM) { |
554 | start = PAGE_ALIGN(start); | 545 | start = PAGE_ALIGN(start); |
555 | end &= ~((u64)PAGE_SIZE - 1); | 546 | end &= ~((phys_addr_t)PAGE_SIZE - 1); |
556 | } | 547 | } |
557 | 548 | ||
558 | e820_add_region(start, end - start, type); | 549 | e820_add_region(start, end - start, type); |
559 | } | 550 | } |
560 | 551 | ||
561 | void xen_ignore_unusable(struct e820entry *list, size_t map_size) | 552 | static void __init xen_ignore_unusable(struct e820entry *list, size_t map_size) |
562 | { | 553 | { |
563 | struct e820entry *entry; | 554 | struct e820entry *entry; |
564 | unsigned int i; | 555 | unsigned int i; |
@@ -577,7 +568,7 @@ char * __init xen_memory_setup(void) | |||
577 | static struct e820entry map[E820MAX] __initdata; | 568 | static struct e820entry map[E820MAX] __initdata; |
578 | 569 | ||
579 | unsigned long max_pfn = xen_start_info->nr_pages; | 570 | unsigned long max_pfn = xen_start_info->nr_pages; |
580 | unsigned long long mem_end; | 571 | phys_addr_t mem_end; |
581 | int rc; | 572 | int rc; |
582 | struct xen_memory_map memmap; | 573 | struct xen_memory_map memmap; |
583 | unsigned long max_pages; | 574 | unsigned long max_pages; |
@@ -652,16 +643,16 @@ char * __init xen_memory_setup(void) | |||
652 | extra_pages); | 643 | extra_pages); |
653 | i = 0; | 644 | i = 0; |
654 | while (i < memmap.nr_entries) { | 645 | while (i < memmap.nr_entries) { |
655 | u64 addr = map[i].addr; | 646 | phys_addr_t addr = map[i].addr; |
656 | u64 size = map[i].size; | 647 | phys_addr_t size = map[i].size; |
657 | u32 type = map[i].type; | 648 | u32 type = map[i].type; |
658 | 649 | ||
659 | if (type == E820_RAM) { | 650 | if (type == E820_RAM) { |
660 | if (addr < mem_end) { | 651 | if (addr < mem_end) { |
661 | size = min(size, mem_end - addr); | 652 | size = min(size, mem_end - addr); |
662 | } else if (extra_pages) { | 653 | } else if (extra_pages) { |
663 | size = min(size, (u64)extra_pages * PAGE_SIZE); | 654 | size = min(size, PFN_PHYS(extra_pages)); |
664 | extra_pages -= size / PAGE_SIZE; | 655 | extra_pages -= PFN_DOWN(size); |
665 | xen_add_extra_mem(addr, size); | 656 | xen_add_extra_mem(addr, size); |
666 | xen_max_p2m_pfn = PFN_DOWN(addr + size); | 657 | xen_max_p2m_pfn = PFN_DOWN(addr + size); |
667 | } else | 658 | } else |
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c index 4c071aeb8417..08e8489c47f1 100644 --- a/arch/x86/xen/smp.c +++ b/arch/x86/xen/smp.c | |||
@@ -507,7 +507,7 @@ static int xen_cpu_disable(void) | |||
507 | static void xen_cpu_die(unsigned int cpu) | 507 | static void xen_cpu_die(unsigned int cpu) |
508 | { | 508 | { |
509 | while (xen_pv_domain() && HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL)) { | 509 | while (xen_pv_domain() && HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL)) { |
510 | current->state = TASK_UNINTERRUPTIBLE; | 510 | __set_current_state(TASK_UNINTERRUPTIBLE); |
511 | schedule_timeout(HZ/10); | 511 | schedule_timeout(HZ/10); |
512 | } | 512 | } |
513 | 513 | ||
diff --git a/arch/x86/xen/time.c b/arch/x86/xen/time.c index 69087341d9ae..55da33b1d51c 100644 --- a/arch/x86/xen/time.c +++ b/arch/x86/xen/time.c | |||
@@ -479,6 +479,10 @@ static void __init xen_time_init(void) | |||
479 | int cpu = smp_processor_id(); | 479 | int cpu = smp_processor_id(); |
480 | struct timespec tp; | 480 | struct timespec tp; |
481 | 481 | ||
482 | /* As Dom0 is never moved, no penalty on using TSC there */ | ||
483 | if (xen_initial_domain()) | ||
484 | xen_clocksource.rating = 275; | ||
485 | |||
482 | clocksource_register_hz(&xen_clocksource, NSEC_PER_SEC); | 486 | clocksource_register_hz(&xen_clocksource, NSEC_PER_SEC); |
483 | 487 | ||
484 | if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL) == 0) { | 488 | if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL) == 0) { |
diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h index 5686bd9d58cc..9e195c683549 100644 --- a/arch/x86/xen/xen-ops.h +++ b/arch/x86/xen/xen-ops.h | |||
@@ -10,6 +10,12 @@ | |||
10 | extern const char xen_hypervisor_callback[]; | 10 | extern const char xen_hypervisor_callback[]; |
11 | extern const char xen_failsafe_callback[]; | 11 | extern const char xen_failsafe_callback[]; |
12 | 12 | ||
13 | void xen_sysenter_target(void); | ||
14 | #ifdef CONFIG_X86_64 | ||
15 | void xen_syscall_target(void); | ||
16 | void xen_syscall32_target(void); | ||
17 | #endif | ||
18 | |||
13 | extern void *xen_initial_gdt; | 19 | extern void *xen_initial_gdt; |
14 | 20 | ||
15 | struct trap_info; | 21 | struct trap_info; |
diff --git a/drivers/block/xen-blkback/blkback.c b/drivers/block/xen-blkback/blkback.c index 63fc7f06a014..2a04d341e598 100644 --- a/drivers/block/xen-blkback/blkback.c +++ b/drivers/block/xen-blkback/blkback.c | |||
@@ -47,6 +47,7 @@ | |||
47 | #include <asm/xen/hypervisor.h> | 47 | #include <asm/xen/hypervisor.h> |
48 | #include <asm/xen/hypercall.h> | 48 | #include <asm/xen/hypercall.h> |
49 | #include <xen/balloon.h> | 49 | #include <xen/balloon.h> |
50 | #include <xen/grant_table.h> | ||
50 | #include "common.h" | 51 | #include "common.h" |
51 | 52 | ||
52 | /* | 53 | /* |
@@ -100,7 +101,7 @@ module_param(log_stats, int, 0644); | |||
100 | 101 | ||
101 | #define BLKBACK_INVALID_HANDLE (~0) | 102 | #define BLKBACK_INVALID_HANDLE (~0) |
102 | 103 | ||
103 | /* Number of free pages to remove on each call to free_xenballooned_pages */ | 104 | /* Number of free pages to remove on each call to gnttab_free_pages */ |
104 | #define NUM_BATCH_FREE_PAGES 10 | 105 | #define NUM_BATCH_FREE_PAGES 10 |
105 | 106 | ||
106 | static inline int get_free_page(struct xen_blkif *blkif, struct page **page) | 107 | static inline int get_free_page(struct xen_blkif *blkif, struct page **page) |
@@ -111,7 +112,7 @@ static inline int get_free_page(struct xen_blkif *blkif, struct page **page) | |||
111 | if (list_empty(&blkif->free_pages)) { | 112 | if (list_empty(&blkif->free_pages)) { |
112 | BUG_ON(blkif->free_pages_num != 0); | 113 | BUG_ON(blkif->free_pages_num != 0); |
113 | spin_unlock_irqrestore(&blkif->free_pages_lock, flags); | 114 | spin_unlock_irqrestore(&blkif->free_pages_lock, flags); |
114 | return alloc_xenballooned_pages(1, page, false); | 115 | return gnttab_alloc_pages(1, page); |
115 | } | 116 | } |
116 | BUG_ON(blkif->free_pages_num == 0); | 117 | BUG_ON(blkif->free_pages_num == 0); |
117 | page[0] = list_first_entry(&blkif->free_pages, struct page, lru); | 118 | page[0] = list_first_entry(&blkif->free_pages, struct page, lru); |
@@ -151,14 +152,14 @@ static inline void shrink_free_pagepool(struct xen_blkif *blkif, int num) | |||
151 | blkif->free_pages_num--; | 152 | blkif->free_pages_num--; |
152 | if (++num_pages == NUM_BATCH_FREE_PAGES) { | 153 | if (++num_pages == NUM_BATCH_FREE_PAGES) { |
153 | spin_unlock_irqrestore(&blkif->free_pages_lock, flags); | 154 | spin_unlock_irqrestore(&blkif->free_pages_lock, flags); |
154 | free_xenballooned_pages(num_pages, page); | 155 | gnttab_free_pages(num_pages, page); |
155 | spin_lock_irqsave(&blkif->free_pages_lock, flags); | 156 | spin_lock_irqsave(&blkif->free_pages_lock, flags); |
156 | num_pages = 0; | 157 | num_pages = 0; |
157 | } | 158 | } |
158 | } | 159 | } |
159 | spin_unlock_irqrestore(&blkif->free_pages_lock, flags); | 160 | spin_unlock_irqrestore(&blkif->free_pages_lock, flags); |
160 | if (num_pages != 0) | 161 | if (num_pages != 0) |
161 | free_xenballooned_pages(num_pages, page); | 162 | gnttab_free_pages(num_pages, page); |
162 | } | 163 | } |
163 | 164 | ||
164 | #define vaddr(page) ((unsigned long)pfn_to_kaddr(page_to_pfn(page))) | 165 | #define vaddr(page) ((unsigned long)pfn_to_kaddr(page_to_pfn(page))) |
@@ -262,6 +263,17 @@ static void put_persistent_gnt(struct xen_blkif *blkif, | |||
262 | atomic_dec(&blkif->persistent_gnt_in_use); | 263 | atomic_dec(&blkif->persistent_gnt_in_use); |
263 | } | 264 | } |
264 | 265 | ||
266 | static void free_persistent_gnts_unmap_callback(int result, | ||
267 | struct gntab_unmap_queue_data *data) | ||
268 | { | ||
269 | struct completion *c = data->data; | ||
270 | |||
271 | /* BUG_ON used to reproduce existing behaviour, | ||
272 | but is this the best way to deal with this? */ | ||
273 | BUG_ON(result); | ||
274 | complete(c); | ||
275 | } | ||
276 | |||
265 | static void free_persistent_gnts(struct xen_blkif *blkif, struct rb_root *root, | 277 | static void free_persistent_gnts(struct xen_blkif *blkif, struct rb_root *root, |
266 | unsigned int num) | 278 | unsigned int num) |
267 | { | 279 | { |
@@ -269,8 +281,17 @@ static void free_persistent_gnts(struct xen_blkif *blkif, struct rb_root *root, | |||
269 | struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST]; | 281 | struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST]; |
270 | struct persistent_gnt *persistent_gnt; | 282 | struct persistent_gnt *persistent_gnt; |
271 | struct rb_node *n; | 283 | struct rb_node *n; |
272 | int ret = 0; | ||
273 | int segs_to_unmap = 0; | 284 | int segs_to_unmap = 0; |
285 | struct gntab_unmap_queue_data unmap_data; | ||
286 | struct completion unmap_completion; | ||
287 | |||
288 | init_completion(&unmap_completion); | ||
289 | |||
290 | unmap_data.data = &unmap_completion; | ||
291 | unmap_data.done = &free_persistent_gnts_unmap_callback; | ||
292 | unmap_data.pages = pages; | ||
293 | unmap_data.unmap_ops = unmap; | ||
294 | unmap_data.kunmap_ops = NULL; | ||
274 | 295 | ||
275 | foreach_grant_safe(persistent_gnt, n, root, node) { | 296 | foreach_grant_safe(persistent_gnt, n, root, node) { |
276 | BUG_ON(persistent_gnt->handle == | 297 | BUG_ON(persistent_gnt->handle == |
@@ -285,9 +306,11 @@ static void free_persistent_gnts(struct xen_blkif *blkif, struct rb_root *root, | |||
285 | 306 | ||
286 | if (++segs_to_unmap == BLKIF_MAX_SEGMENTS_PER_REQUEST || | 307 | if (++segs_to_unmap == BLKIF_MAX_SEGMENTS_PER_REQUEST || |
287 | !rb_next(&persistent_gnt->node)) { | 308 | !rb_next(&persistent_gnt->node)) { |
288 | ret = gnttab_unmap_refs(unmap, NULL, pages, | 309 | |
289 | segs_to_unmap); | 310 | unmap_data.count = segs_to_unmap; |
290 | BUG_ON(ret); | 311 | gnttab_unmap_refs_async(&unmap_data); |
312 | wait_for_completion(&unmap_completion); | ||
313 | |||
291 | put_free_pages(blkif, pages, segs_to_unmap); | 314 | put_free_pages(blkif, pages, segs_to_unmap); |
292 | segs_to_unmap = 0; | 315 | segs_to_unmap = 0; |
293 | } | 316 | } |
@@ -653,18 +676,14 @@ void xen_blkbk_free_caches(struct xen_blkif *blkif) | |||
653 | shrink_free_pagepool(blkif, 0 /* All */); | 676 | shrink_free_pagepool(blkif, 0 /* All */); |
654 | } | 677 | } |
655 | 678 | ||
656 | /* | 679 | static unsigned int xen_blkbk_unmap_prepare( |
657 | * Unmap the grant references, and also remove the M2P over-rides | 680 | struct xen_blkif *blkif, |
658 | * used in the 'pending_req'. | 681 | struct grant_page **pages, |
659 | */ | 682 | unsigned int num, |
660 | static void xen_blkbk_unmap(struct xen_blkif *blkif, | 683 | struct gnttab_unmap_grant_ref *unmap_ops, |
661 | struct grant_page *pages[], | 684 | struct page **unmap_pages) |
662 | int num) | ||
663 | { | 685 | { |
664 | struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST]; | ||
665 | struct page *unmap_pages[BLKIF_MAX_SEGMENTS_PER_REQUEST]; | ||
666 | unsigned int i, invcount = 0; | 686 | unsigned int i, invcount = 0; |
667 | int ret; | ||
668 | 687 | ||
669 | for (i = 0; i < num; i++) { | 688 | for (i = 0; i < num; i++) { |
670 | if (pages[i]->persistent_gnt != NULL) { | 689 | if (pages[i]->persistent_gnt != NULL) { |
@@ -674,21 +693,95 @@ static void xen_blkbk_unmap(struct xen_blkif *blkif, | |||
674 | if (pages[i]->handle == BLKBACK_INVALID_HANDLE) | 693 | if (pages[i]->handle == BLKBACK_INVALID_HANDLE) |
675 | continue; | 694 | continue; |
676 | unmap_pages[invcount] = pages[i]->page; | 695 | unmap_pages[invcount] = pages[i]->page; |
677 | gnttab_set_unmap_op(&unmap[invcount], vaddr(pages[i]->page), | 696 | gnttab_set_unmap_op(&unmap_ops[invcount], vaddr(pages[i]->page), |
678 | GNTMAP_host_map, pages[i]->handle); | 697 | GNTMAP_host_map, pages[i]->handle); |
679 | pages[i]->handle = BLKBACK_INVALID_HANDLE; | 698 | pages[i]->handle = BLKBACK_INVALID_HANDLE; |
680 | if (++invcount == BLKIF_MAX_SEGMENTS_PER_REQUEST) { | 699 | invcount++; |
681 | ret = gnttab_unmap_refs(unmap, NULL, unmap_pages, | 700 | } |
682 | invcount); | 701 | |
702 | return invcount; | ||
703 | } | ||
704 | |||
705 | static void xen_blkbk_unmap_and_respond_callback(int result, struct gntab_unmap_queue_data *data) | ||
706 | { | ||
707 | struct pending_req* pending_req = (struct pending_req*) (data->data); | ||
708 | struct xen_blkif *blkif = pending_req->blkif; | ||
709 | |||
710 | /* BUG_ON used to reproduce existing behaviour, | ||
711 | but is this the best way to deal with this? */ | ||
712 | BUG_ON(result); | ||
713 | |||
714 | put_free_pages(blkif, data->pages, data->count); | ||
715 | make_response(blkif, pending_req->id, | ||
716 | pending_req->operation, pending_req->status); | ||
717 | free_req(blkif, pending_req); | ||
718 | /* | ||
719 | * Make sure the request is freed before releasing blkif, | ||
720 | * or there could be a race between free_req and the | ||
721 | * cleanup done in xen_blkif_free during shutdown. | ||
722 | * | ||
723 | * NB: The fact that we might try to wake up pending_free_wq | ||
724 | * before drain_complete (in case there's a drain going on) | ||
725 | * it's not a problem with our current implementation | ||
726 | * because we can assure there's no thread waiting on | ||
727 | * pending_free_wq if there's a drain going on, but it has | ||
728 | * to be taken into account if the current model is changed. | ||
729 | */ | ||
730 | if (atomic_dec_and_test(&blkif->inflight) && atomic_read(&blkif->drain)) { | ||
731 | complete(&blkif->drain_complete); | ||
732 | } | ||
733 | xen_blkif_put(blkif); | ||
734 | } | ||
735 | |||
736 | static void xen_blkbk_unmap_and_respond(struct pending_req *req) | ||
737 | { | ||
738 | struct gntab_unmap_queue_data* work = &req->gnttab_unmap_data; | ||
739 | struct xen_blkif *blkif = req->blkif; | ||
740 | struct grant_page **pages = req->segments; | ||
741 | unsigned int invcount; | ||
742 | |||
743 | invcount = xen_blkbk_unmap_prepare(blkif, pages, req->nr_pages, | ||
744 | req->unmap, req->unmap_pages); | ||
745 | |||
746 | work->data = req; | ||
747 | work->done = xen_blkbk_unmap_and_respond_callback; | ||
748 | work->unmap_ops = req->unmap; | ||
749 | work->kunmap_ops = NULL; | ||
750 | work->pages = req->unmap_pages; | ||
751 | work->count = invcount; | ||
752 | |||
753 | gnttab_unmap_refs_async(&req->gnttab_unmap_data); | ||
754 | } | ||
755 | |||
756 | |||
757 | /* | ||
758 | * Unmap the grant references. | ||
759 | * | ||
760 | * This could accumulate ops up to the batch size to reduce the number | ||
761 | * of hypercalls, but since this is only used in error paths there's | ||
762 | * no real need. | ||
763 | */ | ||
764 | static void xen_blkbk_unmap(struct xen_blkif *blkif, | ||
765 | struct grant_page *pages[], | ||
766 | int num) | ||
767 | { | ||
768 | struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST]; | ||
769 | struct page *unmap_pages[BLKIF_MAX_SEGMENTS_PER_REQUEST]; | ||
770 | unsigned int invcount = 0; | ||
771 | int ret; | ||
772 | |||
773 | while (num) { | ||
774 | unsigned int batch = min(num, BLKIF_MAX_SEGMENTS_PER_REQUEST); | ||
775 | |||
776 | invcount = xen_blkbk_unmap_prepare(blkif, pages, batch, | ||
777 | unmap, unmap_pages); | ||
778 | if (invcount) { | ||
779 | ret = gnttab_unmap_refs(unmap, NULL, unmap_pages, invcount); | ||
683 | BUG_ON(ret); | 780 | BUG_ON(ret); |
684 | put_free_pages(blkif, unmap_pages, invcount); | 781 | put_free_pages(blkif, unmap_pages, invcount); |
685 | invcount = 0; | ||
686 | } | 782 | } |
687 | } | 783 | pages += batch; |
688 | if (invcount) { | 784 | num -= batch; |
689 | ret = gnttab_unmap_refs(unmap, NULL, unmap_pages, invcount); | ||
690 | BUG_ON(ret); | ||
691 | put_free_pages(blkif, unmap_pages, invcount); | ||
692 | } | 785 | } |
693 | } | 786 | } |
694 | 787 | ||
@@ -982,32 +1075,8 @@ static void __end_block_io_op(struct pending_req *pending_req, int error) | |||
982 | * the grant references associated with 'request' and provide | 1075 | * the grant references associated with 'request' and provide |
983 | * the proper response on the ring. | 1076 | * the proper response on the ring. |
984 | */ | 1077 | */ |
985 | if (atomic_dec_and_test(&pending_req->pendcnt)) { | 1078 | if (atomic_dec_and_test(&pending_req->pendcnt)) |
986 | struct xen_blkif *blkif = pending_req->blkif; | 1079 | xen_blkbk_unmap_and_respond(pending_req); |
987 | |||
988 | xen_blkbk_unmap(blkif, | ||
989 | pending_req->segments, | ||
990 | pending_req->nr_pages); | ||
991 | make_response(blkif, pending_req->id, | ||
992 | pending_req->operation, pending_req->status); | ||
993 | free_req(blkif, pending_req); | ||
994 | /* | ||
995 | * Make sure the request is freed before releasing blkif, | ||
996 | * or there could be a race between free_req and the | ||
997 | * cleanup done in xen_blkif_free during shutdown. | ||
998 | * | ||
999 | * NB: The fact that we might try to wake up pending_free_wq | ||
1000 | * before drain_complete (in case there's a drain going on) | ||
1001 | * it's not a problem with our current implementation | ||
1002 | * because we can assure there's no thread waiting on | ||
1003 | * pending_free_wq if there's a drain going on, but it has | ||
1004 | * to be taken into account if the current model is changed. | ||
1005 | */ | ||
1006 | if (atomic_dec_and_test(&blkif->inflight) && atomic_read(&blkif->drain)) { | ||
1007 | complete(&blkif->drain_complete); | ||
1008 | } | ||
1009 | xen_blkif_put(blkif); | ||
1010 | } | ||
1011 | } | 1080 | } |
1012 | 1081 | ||
1013 | /* | 1082 | /* |
diff --git a/drivers/block/xen-blkback/common.h b/drivers/block/xen-blkback/common.h index f65b807e3236..cc90a840e616 100644 --- a/drivers/block/xen-blkback/common.h +++ b/drivers/block/xen-blkback/common.h | |||
@@ -350,6 +350,9 @@ struct pending_req { | |||
350 | struct grant_page *indirect_pages[MAX_INDIRECT_PAGES]; | 350 | struct grant_page *indirect_pages[MAX_INDIRECT_PAGES]; |
351 | struct seg_buf seg[MAX_INDIRECT_SEGMENTS]; | 351 | struct seg_buf seg[MAX_INDIRECT_SEGMENTS]; |
352 | struct bio *biolist[MAX_INDIRECT_SEGMENTS]; | 352 | struct bio *biolist[MAX_INDIRECT_SEGMENTS]; |
353 | struct gnttab_unmap_grant_ref unmap[MAX_INDIRECT_SEGMENTS]; | ||
354 | struct page *unmap_pages[MAX_INDIRECT_SEGMENTS]; | ||
355 | struct gntab_unmap_queue_data gnttab_unmap_data; | ||
353 | }; | 356 | }; |
354 | 357 | ||
355 | 358 | ||
diff --git a/drivers/net/xen-netback/interface.c b/drivers/net/xen-netback/interface.c index 037f74f0fcf6..12f9e2708afb 100644 --- a/drivers/net/xen-netback/interface.c +++ b/drivers/net/xen-netback/interface.c | |||
@@ -483,9 +483,8 @@ int xenvif_init_queue(struct xenvif_queue *queue) | |||
483 | * better enable it. The long term solution would be to use just a | 483 | * better enable it. The long term solution would be to use just a |
484 | * bunch of valid page descriptors, without dependency on ballooning | 484 | * bunch of valid page descriptors, without dependency on ballooning |
485 | */ | 485 | */ |
486 | err = alloc_xenballooned_pages(MAX_PENDING_REQS, | 486 | err = gnttab_alloc_pages(MAX_PENDING_REQS, |
487 | queue->mmap_pages, | 487 | queue->mmap_pages); |
488 | false); | ||
489 | if (err) { | 488 | if (err) { |
490 | netdev_err(queue->vif->dev, "Could not reserve mmap_pages\n"); | 489 | netdev_err(queue->vif->dev, "Could not reserve mmap_pages\n"); |
491 | return -ENOMEM; | 490 | return -ENOMEM; |
@@ -664,7 +663,7 @@ void xenvif_disconnect(struct xenvif *vif) | |||
664 | */ | 663 | */ |
665 | void xenvif_deinit_queue(struct xenvif_queue *queue) | 664 | void xenvif_deinit_queue(struct xenvif_queue *queue) |
666 | { | 665 | { |
667 | free_xenballooned_pages(MAX_PENDING_REQS, queue->mmap_pages); | 666 | gnttab_free_pages(MAX_PENDING_REQS, queue->mmap_pages); |
668 | } | 667 | } |
669 | 668 | ||
670 | void xenvif_free(struct xenvif *vif) | 669 | void xenvif_free(struct xenvif *vif) |
diff --git a/drivers/net/xen-netback/netback.c b/drivers/net/xen-netback/netback.c index c8ce701a7efb..7dc2d64db3cb 100644 --- a/drivers/net/xen-netback/netback.c +++ b/drivers/net/xen-netback/netback.c | |||
@@ -314,9 +314,7 @@ static struct xenvif_rx_meta *get_next_rx_buffer(struct xenvif_queue *queue, | |||
314 | static void xenvif_gop_frag_copy(struct xenvif_queue *queue, struct sk_buff *skb, | 314 | static void xenvif_gop_frag_copy(struct xenvif_queue *queue, struct sk_buff *skb, |
315 | struct netrx_pending_operations *npo, | 315 | struct netrx_pending_operations *npo, |
316 | struct page *page, unsigned long size, | 316 | struct page *page, unsigned long size, |
317 | unsigned long offset, int *head, | 317 | unsigned long offset, int *head) |
318 | struct xenvif_queue *foreign_queue, | ||
319 | grant_ref_t foreign_gref) | ||
320 | { | 318 | { |
321 | struct gnttab_copy *copy_gop; | 319 | struct gnttab_copy *copy_gop; |
322 | struct xenvif_rx_meta *meta; | 320 | struct xenvif_rx_meta *meta; |
@@ -333,6 +331,8 @@ static void xenvif_gop_frag_copy(struct xenvif_queue *queue, struct sk_buff *skb | |||
333 | offset &= ~PAGE_MASK; | 331 | offset &= ~PAGE_MASK; |
334 | 332 | ||
335 | while (size > 0) { | 333 | while (size > 0) { |
334 | struct xen_page_foreign *foreign; | ||
335 | |||
336 | BUG_ON(offset >= PAGE_SIZE); | 336 | BUG_ON(offset >= PAGE_SIZE); |
337 | BUG_ON(npo->copy_off > MAX_BUFFER_OFFSET); | 337 | BUG_ON(npo->copy_off > MAX_BUFFER_OFFSET); |
338 | 338 | ||
@@ -361,9 +361,10 @@ static void xenvif_gop_frag_copy(struct xenvif_queue *queue, struct sk_buff *skb | |||
361 | copy_gop->flags = GNTCOPY_dest_gref; | 361 | copy_gop->flags = GNTCOPY_dest_gref; |
362 | copy_gop->len = bytes; | 362 | copy_gop->len = bytes; |
363 | 363 | ||
364 | if (foreign_queue) { | 364 | foreign = xen_page_foreign(page); |
365 | copy_gop->source.domid = foreign_queue->vif->domid; | 365 | if (foreign) { |
366 | copy_gop->source.u.ref = foreign_gref; | 366 | copy_gop->source.domid = foreign->domid; |
367 | copy_gop->source.u.ref = foreign->gref; | ||
367 | copy_gop->flags |= GNTCOPY_source_gref; | 368 | copy_gop->flags |= GNTCOPY_source_gref; |
368 | } else { | 369 | } else { |
369 | copy_gop->source.domid = DOMID_SELF; | 370 | copy_gop->source.domid = DOMID_SELF; |
@@ -406,35 +407,6 @@ static void xenvif_gop_frag_copy(struct xenvif_queue *queue, struct sk_buff *skb | |||
406 | } | 407 | } |
407 | 408 | ||
408 | /* | 409 | /* |
409 | * Find the grant ref for a given frag in a chain of struct ubuf_info's | ||
410 | * skb: the skb itself | ||
411 | * i: the frag's number | ||
412 | * ubuf: a pointer to an element in the chain. It should not be NULL | ||
413 | * | ||
414 | * Returns a pointer to the element in the chain where the page were found. If | ||
415 | * not found, returns NULL. | ||
416 | * See the definition of callback_struct in common.h for more details about | ||
417 | * the chain. | ||
418 | */ | ||
419 | static const struct ubuf_info *xenvif_find_gref(const struct sk_buff *const skb, | ||
420 | const int i, | ||
421 | const struct ubuf_info *ubuf) | ||
422 | { | ||
423 | struct xenvif_queue *foreign_queue = ubuf_to_queue(ubuf); | ||
424 | |||
425 | do { | ||
426 | u16 pending_idx = ubuf->desc; | ||
427 | |||
428 | if (skb_shinfo(skb)->frags[i].page.p == | ||
429 | foreign_queue->mmap_pages[pending_idx]) | ||
430 | break; | ||
431 | ubuf = (struct ubuf_info *) ubuf->ctx; | ||
432 | } while (ubuf); | ||
433 | |||
434 | return ubuf; | ||
435 | } | ||
436 | |||
437 | /* | ||
438 | * Prepare an SKB to be transmitted to the frontend. | 410 | * Prepare an SKB to be transmitted to the frontend. |
439 | * | 411 | * |
440 | * This function is responsible for allocating grant operations, meta | 412 | * This function is responsible for allocating grant operations, meta |
@@ -459,8 +431,6 @@ static int xenvif_gop_skb(struct sk_buff *skb, | |||
459 | int head = 1; | 431 | int head = 1; |
460 | int old_meta_prod; | 432 | int old_meta_prod; |
461 | int gso_type; | 433 | int gso_type; |
462 | const struct ubuf_info *ubuf = skb_shinfo(skb)->destructor_arg; | ||
463 | const struct ubuf_info *const head_ubuf = ubuf; | ||
464 | 434 | ||
465 | old_meta_prod = npo->meta_prod; | 435 | old_meta_prod = npo->meta_prod; |
466 | 436 | ||
@@ -507,68 +477,16 @@ static int xenvif_gop_skb(struct sk_buff *skb, | |||
507 | len = skb_tail_pointer(skb) - data; | 477 | len = skb_tail_pointer(skb) - data; |
508 | 478 | ||
509 | xenvif_gop_frag_copy(queue, skb, npo, | 479 | xenvif_gop_frag_copy(queue, skb, npo, |
510 | virt_to_page(data), len, offset, &head, | 480 | virt_to_page(data), len, offset, &head); |
511 | NULL, | ||
512 | 0); | ||
513 | data += len; | 481 | data += len; |
514 | } | 482 | } |
515 | 483 | ||
516 | for (i = 0; i < nr_frags; i++) { | 484 | for (i = 0; i < nr_frags; i++) { |
517 | /* This variable also signals whether foreign_gref has a real | ||
518 | * value or not. | ||
519 | */ | ||
520 | struct xenvif_queue *foreign_queue = NULL; | ||
521 | grant_ref_t foreign_gref; | ||
522 | |||
523 | if ((skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) && | ||
524 | (ubuf->callback == &xenvif_zerocopy_callback)) { | ||
525 | const struct ubuf_info *const startpoint = ubuf; | ||
526 | |||
527 | /* Ideally ubuf points to the chain element which | ||
528 | * belongs to this frag. Or if frags were removed from | ||
529 | * the beginning, then shortly before it. | ||
530 | */ | ||
531 | ubuf = xenvif_find_gref(skb, i, ubuf); | ||
532 | |||
533 | /* Try again from the beginning of the list, if we | ||
534 | * haven't tried from there. This only makes sense in | ||
535 | * the unlikely event of reordering the original frags. | ||
536 | * For injected local pages it's an unnecessary second | ||
537 | * run. | ||
538 | */ | ||
539 | if (unlikely(!ubuf) && startpoint != head_ubuf) | ||
540 | ubuf = xenvif_find_gref(skb, i, head_ubuf); | ||
541 | |||
542 | if (likely(ubuf)) { | ||
543 | u16 pending_idx = ubuf->desc; | ||
544 | |||
545 | foreign_queue = ubuf_to_queue(ubuf); | ||
546 | foreign_gref = | ||
547 | foreign_queue->pending_tx_info[pending_idx].req.gref; | ||
548 | /* Just a safety measure. If this was the last | ||
549 | * element on the list, the for loop will | ||
550 | * iterate again if a local page were added to | ||
551 | * the end. Using head_ubuf here prevents the | ||
552 | * second search on the chain. Or the original | ||
553 | * frags changed order, but that's less likely. | ||
554 | * In any way, ubuf shouldn't be NULL. | ||
555 | */ | ||
556 | ubuf = ubuf->ctx ? | ||
557 | (struct ubuf_info *) ubuf->ctx : | ||
558 | head_ubuf; | ||
559 | } else | ||
560 | /* This frag was a local page, added to the | ||
561 | * array after the skb left netback. | ||
562 | */ | ||
563 | ubuf = head_ubuf; | ||
564 | } | ||
565 | xenvif_gop_frag_copy(queue, skb, npo, | 485 | xenvif_gop_frag_copy(queue, skb, npo, |
566 | skb_frag_page(&skb_shinfo(skb)->frags[i]), | 486 | skb_frag_page(&skb_shinfo(skb)->frags[i]), |
567 | skb_frag_size(&skb_shinfo(skb)->frags[i]), | 487 | skb_frag_size(&skb_shinfo(skb)->frags[i]), |
568 | skb_shinfo(skb)->frags[i].page_offset, | 488 | skb_shinfo(skb)->frags[i].page_offset, |
569 | &head, | 489 | &head); |
570 | foreign_queue, | ||
571 | foreign_queue ? foreign_gref : UINT_MAX); | ||
572 | } | 490 | } |
573 | 491 | ||
574 | return npo->meta_prod - old_meta_prod; | 492 | return npo->meta_prod - old_meta_prod; |
@@ -1241,12 +1159,6 @@ static void xenvif_fill_frags(struct xenvif_queue *queue, struct sk_buff *skb) | |||
1241 | /* Take an extra reference to offset network stack's put_page */ | 1159 | /* Take an extra reference to offset network stack's put_page */ |
1242 | get_page(queue->mmap_pages[pending_idx]); | 1160 | get_page(queue->mmap_pages[pending_idx]); |
1243 | } | 1161 | } |
1244 | /* FIXME: __skb_fill_page_desc set this to true because page->pfmemalloc | ||
1245 | * overlaps with "index", and "mapping" is not set. I think mapping | ||
1246 | * should be set. If delivered to local stack, it would drop this | ||
1247 | * skb in sk_filter unless the socket has the right to use it. | ||
1248 | */ | ||
1249 | skb->pfmemalloc = false; | ||
1250 | } | 1162 | } |
1251 | 1163 | ||
1252 | static int xenvif_get_extras(struct xenvif_queue *queue, | 1164 | static int xenvif_get_extras(struct xenvif_queue *queue, |
diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c index 3860d02729dc..0b52d92cb2e5 100644 --- a/drivers/xen/balloon.c +++ b/drivers/xen/balloon.c | |||
@@ -92,7 +92,6 @@ EXPORT_SYMBOL_GPL(balloon_stats); | |||
92 | 92 | ||
93 | /* We increase/decrease in batches which fit in a page */ | 93 | /* We increase/decrease in batches which fit in a page */ |
94 | static xen_pfn_t frame_list[PAGE_SIZE / sizeof(unsigned long)]; | 94 | static xen_pfn_t frame_list[PAGE_SIZE / sizeof(unsigned long)]; |
95 | static DEFINE_PER_CPU(struct page *, balloon_scratch_page); | ||
96 | 95 | ||
97 | 96 | ||
98 | /* List of ballooned pages, threaded through the mem_map array. */ | 97 | /* List of ballooned pages, threaded through the mem_map array. */ |
@@ -423,22 +422,12 @@ static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp) | |||
423 | page = pfn_to_page(pfn); | 422 | page = pfn_to_page(pfn); |
424 | 423 | ||
425 | #ifdef CONFIG_XEN_HAVE_PVMMU | 424 | #ifdef CONFIG_XEN_HAVE_PVMMU |
426 | /* | ||
427 | * Ballooned out frames are effectively replaced with | ||
428 | * a scratch frame. Ensure direct mappings and the | ||
429 | * p2m are consistent. | ||
430 | */ | ||
431 | if (!xen_feature(XENFEAT_auto_translated_physmap)) { | 425 | if (!xen_feature(XENFEAT_auto_translated_physmap)) { |
432 | if (!PageHighMem(page)) { | 426 | if (!PageHighMem(page)) { |
433 | struct page *scratch_page = get_balloon_scratch_page(); | ||
434 | |||
435 | ret = HYPERVISOR_update_va_mapping( | 427 | ret = HYPERVISOR_update_va_mapping( |
436 | (unsigned long)__va(pfn << PAGE_SHIFT), | 428 | (unsigned long)__va(pfn << PAGE_SHIFT), |
437 | pfn_pte(page_to_pfn(scratch_page), | 429 | __pte_ma(0), 0); |
438 | PAGE_KERNEL_RO), 0); | ||
439 | BUG_ON(ret); | 430 | BUG_ON(ret); |
440 | |||
441 | put_balloon_scratch_page(); | ||
442 | } | 431 | } |
443 | __set_phys_to_machine(pfn, INVALID_P2M_ENTRY); | 432 | __set_phys_to_machine(pfn, INVALID_P2M_ENTRY); |
444 | } | 433 | } |
@@ -500,18 +489,6 @@ static void balloon_process(struct work_struct *work) | |||
500 | mutex_unlock(&balloon_mutex); | 489 | mutex_unlock(&balloon_mutex); |
501 | } | 490 | } |
502 | 491 | ||
503 | struct page *get_balloon_scratch_page(void) | ||
504 | { | ||
505 | struct page *ret = get_cpu_var(balloon_scratch_page); | ||
506 | BUG_ON(ret == NULL); | ||
507 | return ret; | ||
508 | } | ||
509 | |||
510 | void put_balloon_scratch_page(void) | ||
511 | { | ||
512 | put_cpu_var(balloon_scratch_page); | ||
513 | } | ||
514 | |||
515 | /* Resets the Xen limit, sets new target, and kicks off processing. */ | 492 | /* Resets the Xen limit, sets new target, and kicks off processing. */ |
516 | void balloon_set_new_target(unsigned long target) | 493 | void balloon_set_new_target(unsigned long target) |
517 | { | 494 | { |
@@ -605,61 +582,13 @@ static void __init balloon_add_region(unsigned long start_pfn, | |||
605 | } | 582 | } |
606 | } | 583 | } |
607 | 584 | ||
608 | static int alloc_balloon_scratch_page(int cpu) | ||
609 | { | ||
610 | if (per_cpu(balloon_scratch_page, cpu) != NULL) | ||
611 | return 0; | ||
612 | |||
613 | per_cpu(balloon_scratch_page, cpu) = alloc_page(GFP_KERNEL); | ||
614 | if (per_cpu(balloon_scratch_page, cpu) == NULL) { | ||
615 | pr_warn("Failed to allocate balloon_scratch_page for cpu %d\n", cpu); | ||
616 | return -ENOMEM; | ||
617 | } | ||
618 | |||
619 | return 0; | ||
620 | } | ||
621 | |||
622 | |||
623 | static int balloon_cpu_notify(struct notifier_block *self, | ||
624 | unsigned long action, void *hcpu) | ||
625 | { | ||
626 | int cpu = (long)hcpu; | ||
627 | switch (action) { | ||
628 | case CPU_UP_PREPARE: | ||
629 | if (alloc_balloon_scratch_page(cpu)) | ||
630 | return NOTIFY_BAD; | ||
631 | break; | ||
632 | default: | ||
633 | break; | ||
634 | } | ||
635 | return NOTIFY_OK; | ||
636 | } | ||
637 | |||
638 | static struct notifier_block balloon_cpu_notifier = { | ||
639 | .notifier_call = balloon_cpu_notify, | ||
640 | }; | ||
641 | |||
642 | static int __init balloon_init(void) | 585 | static int __init balloon_init(void) |
643 | { | 586 | { |
644 | int i, cpu; | 587 | int i; |
645 | 588 | ||
646 | if (!xen_domain()) | 589 | if (!xen_domain()) |
647 | return -ENODEV; | 590 | return -ENODEV; |
648 | 591 | ||
649 | if (!xen_feature(XENFEAT_auto_translated_physmap)) { | ||
650 | register_cpu_notifier(&balloon_cpu_notifier); | ||
651 | |||
652 | get_online_cpus(); | ||
653 | for_each_online_cpu(cpu) { | ||
654 | if (alloc_balloon_scratch_page(cpu)) { | ||
655 | put_online_cpus(); | ||
656 | unregister_cpu_notifier(&balloon_cpu_notifier); | ||
657 | return -ENOMEM; | ||
658 | } | ||
659 | } | ||
660 | put_online_cpus(); | ||
661 | } | ||
662 | |||
663 | pr_info("Initialising balloon driver\n"); | 592 | pr_info("Initialising balloon driver\n"); |
664 | 593 | ||
665 | balloon_stats.current_pages = xen_pv_domain() | 594 | balloon_stats.current_pages = xen_pv_domain() |
@@ -696,15 +625,4 @@ static int __init balloon_init(void) | |||
696 | 625 | ||
697 | subsys_initcall(balloon_init); | 626 | subsys_initcall(balloon_init); |
698 | 627 | ||
699 | static int __init balloon_clear(void) | ||
700 | { | ||
701 | int cpu; | ||
702 | |||
703 | for_each_possible_cpu(cpu) | ||
704 | per_cpu(balloon_scratch_page, cpu) = NULL; | ||
705 | |||
706 | return 0; | ||
707 | } | ||
708 | early_initcall(balloon_clear); | ||
709 | |||
710 | MODULE_LICENSE("GPL"); | 628 | MODULE_LICENSE("GPL"); |
diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c index 073b4a19a8b0..d5bb1a33d0a3 100644 --- a/drivers/xen/gntdev.c +++ b/drivers/xen/gntdev.c | |||
@@ -67,7 +67,7 @@ struct gntdev_priv { | |||
67 | * Only populated if populate_freeable_maps == 1 */ | 67 | * Only populated if populate_freeable_maps == 1 */ |
68 | struct list_head freeable_maps; | 68 | struct list_head freeable_maps; |
69 | /* lock protects maps and freeable_maps */ | 69 | /* lock protects maps and freeable_maps */ |
70 | spinlock_t lock; | 70 | struct mutex lock; |
71 | struct mm_struct *mm; | 71 | struct mm_struct *mm; |
72 | struct mmu_notifier mn; | 72 | struct mmu_notifier mn; |
73 | }; | 73 | }; |
@@ -91,7 +91,9 @@ struct grant_map { | |||
91 | struct gnttab_map_grant_ref *map_ops; | 91 | struct gnttab_map_grant_ref *map_ops; |
92 | struct gnttab_unmap_grant_ref *unmap_ops; | 92 | struct gnttab_unmap_grant_ref *unmap_ops; |
93 | struct gnttab_map_grant_ref *kmap_ops; | 93 | struct gnttab_map_grant_ref *kmap_ops; |
94 | struct gnttab_unmap_grant_ref *kunmap_ops; | ||
94 | struct page **pages; | 95 | struct page **pages; |
96 | unsigned long pages_vm_start; | ||
95 | }; | 97 | }; |
96 | 98 | ||
97 | static int unmap_grant_pages(struct grant_map *map, int offset, int pages); | 99 | static int unmap_grant_pages(struct grant_map *map, int offset, int pages); |
@@ -118,12 +120,13 @@ static void gntdev_free_map(struct grant_map *map) | |||
118 | return; | 120 | return; |
119 | 121 | ||
120 | if (map->pages) | 122 | if (map->pages) |
121 | free_xenballooned_pages(map->count, map->pages); | 123 | gnttab_free_pages(map->count, map->pages); |
122 | kfree(map->pages); | 124 | kfree(map->pages); |
123 | kfree(map->grants); | 125 | kfree(map->grants); |
124 | kfree(map->map_ops); | 126 | kfree(map->map_ops); |
125 | kfree(map->unmap_ops); | 127 | kfree(map->unmap_ops); |
126 | kfree(map->kmap_ops); | 128 | kfree(map->kmap_ops); |
129 | kfree(map->kunmap_ops); | ||
127 | kfree(map); | 130 | kfree(map); |
128 | } | 131 | } |
129 | 132 | ||
@@ -140,21 +143,24 @@ static struct grant_map *gntdev_alloc_map(struct gntdev_priv *priv, int count) | |||
140 | add->map_ops = kcalloc(count, sizeof(add->map_ops[0]), GFP_KERNEL); | 143 | add->map_ops = kcalloc(count, sizeof(add->map_ops[0]), GFP_KERNEL); |
141 | add->unmap_ops = kcalloc(count, sizeof(add->unmap_ops[0]), GFP_KERNEL); | 144 | add->unmap_ops = kcalloc(count, sizeof(add->unmap_ops[0]), GFP_KERNEL); |
142 | add->kmap_ops = kcalloc(count, sizeof(add->kmap_ops[0]), GFP_KERNEL); | 145 | add->kmap_ops = kcalloc(count, sizeof(add->kmap_ops[0]), GFP_KERNEL); |
146 | add->kunmap_ops = kcalloc(count, sizeof(add->kunmap_ops[0]), GFP_KERNEL); | ||
143 | add->pages = kcalloc(count, sizeof(add->pages[0]), GFP_KERNEL); | 147 | add->pages = kcalloc(count, sizeof(add->pages[0]), GFP_KERNEL); |
144 | if (NULL == add->grants || | 148 | if (NULL == add->grants || |
145 | NULL == add->map_ops || | 149 | NULL == add->map_ops || |
146 | NULL == add->unmap_ops || | 150 | NULL == add->unmap_ops || |
147 | NULL == add->kmap_ops || | 151 | NULL == add->kmap_ops || |
152 | NULL == add->kunmap_ops || | ||
148 | NULL == add->pages) | 153 | NULL == add->pages) |
149 | goto err; | 154 | goto err; |
150 | 155 | ||
151 | if (alloc_xenballooned_pages(count, add->pages, false /* lowmem */)) | 156 | if (gnttab_alloc_pages(count, add->pages)) |
152 | goto err; | 157 | goto err; |
153 | 158 | ||
154 | for (i = 0; i < count; i++) { | 159 | for (i = 0; i < count; i++) { |
155 | add->map_ops[i].handle = -1; | 160 | add->map_ops[i].handle = -1; |
156 | add->unmap_ops[i].handle = -1; | 161 | add->unmap_ops[i].handle = -1; |
157 | add->kmap_ops[i].handle = -1; | 162 | add->kmap_ops[i].handle = -1; |
163 | add->kunmap_ops[i].handle = -1; | ||
158 | } | 164 | } |
159 | 165 | ||
160 | add->index = 0; | 166 | add->index = 0; |
@@ -216,9 +222,9 @@ static void gntdev_put_map(struct gntdev_priv *priv, struct grant_map *map) | |||
216 | } | 222 | } |
217 | 223 | ||
218 | if (populate_freeable_maps && priv) { | 224 | if (populate_freeable_maps && priv) { |
219 | spin_lock(&priv->lock); | 225 | mutex_lock(&priv->lock); |
220 | list_del(&map->next); | 226 | list_del(&map->next); |
221 | spin_unlock(&priv->lock); | 227 | mutex_unlock(&priv->lock); |
222 | } | 228 | } |
223 | 229 | ||
224 | if (map->pages && !use_ptemod) | 230 | if (map->pages && !use_ptemod) |
@@ -239,6 +245,14 @@ static int find_grant_ptes(pte_t *pte, pgtable_t token, | |||
239 | BUG_ON(pgnr >= map->count); | 245 | BUG_ON(pgnr >= map->count); |
240 | pte_maddr = arbitrary_virt_to_machine(pte).maddr; | 246 | pte_maddr = arbitrary_virt_to_machine(pte).maddr; |
241 | 247 | ||
248 | /* | ||
249 | * Set the PTE as special to force get_user_pages_fast() fall | ||
250 | * back to the slow path. If this is not supported as part of | ||
251 | * the grant map, it will be done afterwards. | ||
252 | */ | ||
253 | if (xen_feature(XENFEAT_gnttab_map_avail_bits)) | ||
254 | flags |= (1 << _GNTMAP_guest_avail0); | ||
255 | |||
242 | gnttab_set_map_op(&map->map_ops[pgnr], pte_maddr, flags, | 256 | gnttab_set_map_op(&map->map_ops[pgnr], pte_maddr, flags, |
243 | map->grants[pgnr].ref, | 257 | map->grants[pgnr].ref, |
244 | map->grants[pgnr].domid); | 258 | map->grants[pgnr].domid); |
@@ -247,6 +261,15 @@ static int find_grant_ptes(pte_t *pte, pgtable_t token, | |||
247 | return 0; | 261 | return 0; |
248 | } | 262 | } |
249 | 263 | ||
264 | #ifdef CONFIG_X86 | ||
265 | static int set_grant_ptes_as_special(pte_t *pte, pgtable_t token, | ||
266 | unsigned long addr, void *data) | ||
267 | { | ||
268 | set_pte_at(current->mm, addr, pte, pte_mkspecial(*pte)); | ||
269 | return 0; | ||
270 | } | ||
271 | #endif | ||
272 | |||
250 | static int map_grant_pages(struct grant_map *map) | 273 | static int map_grant_pages(struct grant_map *map) |
251 | { | 274 | { |
252 | int i, err = 0; | 275 | int i, err = 0; |
@@ -280,6 +303,8 @@ static int map_grant_pages(struct grant_map *map) | |||
280 | map->flags | GNTMAP_host_map, | 303 | map->flags | GNTMAP_host_map, |
281 | map->grants[i].ref, | 304 | map->grants[i].ref, |
282 | map->grants[i].domid); | 305 | map->grants[i].domid); |
306 | gnttab_set_unmap_op(&map->kunmap_ops[i], address, | ||
307 | map->flags | GNTMAP_host_map, -1); | ||
283 | } | 308 | } |
284 | } | 309 | } |
285 | 310 | ||
@@ -290,20 +315,42 @@ static int map_grant_pages(struct grant_map *map) | |||
290 | return err; | 315 | return err; |
291 | 316 | ||
292 | for (i = 0; i < map->count; i++) { | 317 | for (i = 0; i < map->count; i++) { |
293 | if (map->map_ops[i].status) | 318 | if (map->map_ops[i].status) { |
294 | err = -EINVAL; | 319 | err = -EINVAL; |
295 | else { | 320 | continue; |
296 | BUG_ON(map->map_ops[i].handle == -1); | ||
297 | map->unmap_ops[i].handle = map->map_ops[i].handle; | ||
298 | pr_debug("map handle=%d\n", map->map_ops[i].handle); | ||
299 | } | 321 | } |
322 | |||
323 | map->unmap_ops[i].handle = map->map_ops[i].handle; | ||
324 | if (use_ptemod) | ||
325 | map->kunmap_ops[i].handle = map->kmap_ops[i].handle; | ||
300 | } | 326 | } |
301 | return err; | 327 | return err; |
302 | } | 328 | } |
303 | 329 | ||
330 | struct unmap_grant_pages_callback_data | ||
331 | { | ||
332 | struct completion completion; | ||
333 | int result; | ||
334 | }; | ||
335 | |||
336 | static void unmap_grant_callback(int result, | ||
337 | struct gntab_unmap_queue_data *data) | ||
338 | { | ||
339 | struct unmap_grant_pages_callback_data* d = data->data; | ||
340 | |||
341 | d->result = result; | ||
342 | complete(&d->completion); | ||
343 | } | ||
344 | |||
304 | static int __unmap_grant_pages(struct grant_map *map, int offset, int pages) | 345 | static int __unmap_grant_pages(struct grant_map *map, int offset, int pages) |
305 | { | 346 | { |
306 | int i, err = 0; | 347 | int i, err = 0; |
348 | struct gntab_unmap_queue_data unmap_data; | ||
349 | struct unmap_grant_pages_callback_data data; | ||
350 | |||
351 | init_completion(&data.completion); | ||
352 | unmap_data.data = &data; | ||
353 | unmap_data.done= &unmap_grant_callback; | ||
307 | 354 | ||
308 | if (map->notify.flags & UNMAP_NOTIFY_CLEAR_BYTE) { | 355 | if (map->notify.flags & UNMAP_NOTIFY_CLEAR_BYTE) { |
309 | int pgno = (map->notify.addr >> PAGE_SHIFT); | 356 | int pgno = (map->notify.addr >> PAGE_SHIFT); |
@@ -315,11 +362,16 @@ static int __unmap_grant_pages(struct grant_map *map, int offset, int pages) | |||
315 | } | 362 | } |
316 | } | 363 | } |
317 | 364 | ||
318 | err = gnttab_unmap_refs(map->unmap_ops + offset, | 365 | unmap_data.unmap_ops = map->unmap_ops + offset; |
319 | use_ptemod ? map->kmap_ops + offset : NULL, map->pages + offset, | 366 | unmap_data.kunmap_ops = use_ptemod ? map->kunmap_ops + offset : NULL; |
320 | pages); | 367 | unmap_data.pages = map->pages + offset; |
321 | if (err) | 368 | unmap_data.count = pages; |
322 | return err; | 369 | |
370 | gnttab_unmap_refs_async(&unmap_data); | ||
371 | |||
372 | wait_for_completion(&data.completion); | ||
373 | if (data.result) | ||
374 | return data.result; | ||
323 | 375 | ||
324 | for (i = 0; i < pages; i++) { | 376 | for (i = 0; i < pages; i++) { |
325 | if (map->unmap_ops[offset+i].status) | 377 | if (map->unmap_ops[offset+i].status) |
@@ -387,17 +439,26 @@ static void gntdev_vma_close(struct vm_area_struct *vma) | |||
387 | * not do any unmapping, since that has been done prior to | 439 | * not do any unmapping, since that has been done prior to |
388 | * closing the vma, but it may still iterate the unmap_ops list. | 440 | * closing the vma, but it may still iterate the unmap_ops list. |
389 | */ | 441 | */ |
390 | spin_lock(&priv->lock); | 442 | mutex_lock(&priv->lock); |
391 | map->vma = NULL; | 443 | map->vma = NULL; |
392 | spin_unlock(&priv->lock); | 444 | mutex_unlock(&priv->lock); |
393 | } | 445 | } |
394 | vma->vm_private_data = NULL; | 446 | vma->vm_private_data = NULL; |
395 | gntdev_put_map(priv, map); | 447 | gntdev_put_map(priv, map); |
396 | } | 448 | } |
397 | 449 | ||
450 | static struct page *gntdev_vma_find_special_page(struct vm_area_struct *vma, | ||
451 | unsigned long addr) | ||
452 | { | ||
453 | struct grant_map *map = vma->vm_private_data; | ||
454 | |||
455 | return map->pages[(addr - map->pages_vm_start) >> PAGE_SHIFT]; | ||
456 | } | ||
457 | |||
398 | static struct vm_operations_struct gntdev_vmops = { | 458 | static struct vm_operations_struct gntdev_vmops = { |
399 | .open = gntdev_vma_open, | 459 | .open = gntdev_vma_open, |
400 | .close = gntdev_vma_close, | 460 | .close = gntdev_vma_close, |
461 | .find_special_page = gntdev_vma_find_special_page, | ||
401 | }; | 462 | }; |
402 | 463 | ||
403 | /* ------------------------------------------------------------------ */ | 464 | /* ------------------------------------------------------------------ */ |
@@ -433,14 +494,14 @@ static void mn_invl_range_start(struct mmu_notifier *mn, | |||
433 | struct gntdev_priv *priv = container_of(mn, struct gntdev_priv, mn); | 494 | struct gntdev_priv *priv = container_of(mn, struct gntdev_priv, mn); |
434 | struct grant_map *map; | 495 | struct grant_map *map; |
435 | 496 | ||
436 | spin_lock(&priv->lock); | 497 | mutex_lock(&priv->lock); |
437 | list_for_each_entry(map, &priv->maps, next) { | 498 | list_for_each_entry(map, &priv->maps, next) { |
438 | unmap_if_in_range(map, start, end); | 499 | unmap_if_in_range(map, start, end); |
439 | } | 500 | } |
440 | list_for_each_entry(map, &priv->freeable_maps, next) { | 501 | list_for_each_entry(map, &priv->freeable_maps, next) { |
441 | unmap_if_in_range(map, start, end); | 502 | unmap_if_in_range(map, start, end); |
442 | } | 503 | } |
443 | spin_unlock(&priv->lock); | 504 | mutex_unlock(&priv->lock); |
444 | } | 505 | } |
445 | 506 | ||
446 | static void mn_invl_page(struct mmu_notifier *mn, | 507 | static void mn_invl_page(struct mmu_notifier *mn, |
@@ -457,7 +518,7 @@ static void mn_release(struct mmu_notifier *mn, | |||
457 | struct grant_map *map; | 518 | struct grant_map *map; |
458 | int err; | 519 | int err; |
459 | 520 | ||
460 | spin_lock(&priv->lock); | 521 | mutex_lock(&priv->lock); |
461 | list_for_each_entry(map, &priv->maps, next) { | 522 | list_for_each_entry(map, &priv->maps, next) { |
462 | if (!map->vma) | 523 | if (!map->vma) |
463 | continue; | 524 | continue; |
@@ -476,7 +537,7 @@ static void mn_release(struct mmu_notifier *mn, | |||
476 | err = unmap_grant_pages(map, /* offset */ 0, map->count); | 537 | err = unmap_grant_pages(map, /* offset */ 0, map->count); |
477 | WARN_ON(err); | 538 | WARN_ON(err); |
478 | } | 539 | } |
479 | spin_unlock(&priv->lock); | 540 | mutex_unlock(&priv->lock); |
480 | } | 541 | } |
481 | 542 | ||
482 | static struct mmu_notifier_ops gntdev_mmu_ops = { | 543 | static struct mmu_notifier_ops gntdev_mmu_ops = { |
@@ -498,7 +559,7 @@ static int gntdev_open(struct inode *inode, struct file *flip) | |||
498 | 559 | ||
499 | INIT_LIST_HEAD(&priv->maps); | 560 | INIT_LIST_HEAD(&priv->maps); |
500 | INIT_LIST_HEAD(&priv->freeable_maps); | 561 | INIT_LIST_HEAD(&priv->freeable_maps); |
501 | spin_lock_init(&priv->lock); | 562 | mutex_init(&priv->lock); |
502 | 563 | ||
503 | if (use_ptemod) { | 564 | if (use_ptemod) { |
504 | priv->mm = get_task_mm(current); | 565 | priv->mm = get_task_mm(current); |
@@ -572,10 +633,10 @@ static long gntdev_ioctl_map_grant_ref(struct gntdev_priv *priv, | |||
572 | return -EFAULT; | 633 | return -EFAULT; |
573 | } | 634 | } |
574 | 635 | ||
575 | spin_lock(&priv->lock); | 636 | mutex_lock(&priv->lock); |
576 | gntdev_add_map(priv, map); | 637 | gntdev_add_map(priv, map); |
577 | op.index = map->index << PAGE_SHIFT; | 638 | op.index = map->index << PAGE_SHIFT; |
578 | spin_unlock(&priv->lock); | 639 | mutex_unlock(&priv->lock); |
579 | 640 | ||
580 | if (copy_to_user(u, &op, sizeof(op)) != 0) | 641 | if (copy_to_user(u, &op, sizeof(op)) != 0) |
581 | return -EFAULT; | 642 | return -EFAULT; |
@@ -594,7 +655,7 @@ static long gntdev_ioctl_unmap_grant_ref(struct gntdev_priv *priv, | |||
594 | return -EFAULT; | 655 | return -EFAULT; |
595 | pr_debug("priv %p, del %d+%d\n", priv, (int)op.index, (int)op.count); | 656 | pr_debug("priv %p, del %d+%d\n", priv, (int)op.index, (int)op.count); |
596 | 657 | ||
597 | spin_lock(&priv->lock); | 658 | mutex_lock(&priv->lock); |
598 | map = gntdev_find_map_index(priv, op.index >> PAGE_SHIFT, op.count); | 659 | map = gntdev_find_map_index(priv, op.index >> PAGE_SHIFT, op.count); |
599 | if (map) { | 660 | if (map) { |
600 | list_del(&map->next); | 661 | list_del(&map->next); |
@@ -602,7 +663,7 @@ static long gntdev_ioctl_unmap_grant_ref(struct gntdev_priv *priv, | |||
602 | list_add_tail(&map->next, &priv->freeable_maps); | 663 | list_add_tail(&map->next, &priv->freeable_maps); |
603 | err = 0; | 664 | err = 0; |
604 | } | 665 | } |
605 | spin_unlock(&priv->lock); | 666 | mutex_unlock(&priv->lock); |
606 | if (map) | 667 | if (map) |
607 | gntdev_put_map(priv, map); | 668 | gntdev_put_map(priv, map); |
608 | return err; | 669 | return err; |
@@ -670,7 +731,7 @@ static long gntdev_ioctl_notify(struct gntdev_priv *priv, void __user *u) | |||
670 | out_flags = op.action; | 731 | out_flags = op.action; |
671 | out_event = op.event_channel_port; | 732 | out_event = op.event_channel_port; |
672 | 733 | ||
673 | spin_lock(&priv->lock); | 734 | mutex_lock(&priv->lock); |
674 | 735 | ||
675 | list_for_each_entry(map, &priv->maps, next) { | 736 | list_for_each_entry(map, &priv->maps, next) { |
676 | uint64_t begin = map->index << PAGE_SHIFT; | 737 | uint64_t begin = map->index << PAGE_SHIFT; |
@@ -698,7 +759,7 @@ static long gntdev_ioctl_notify(struct gntdev_priv *priv, void __user *u) | |||
698 | rc = 0; | 759 | rc = 0; |
699 | 760 | ||
700 | unlock_out: | 761 | unlock_out: |
701 | spin_unlock(&priv->lock); | 762 | mutex_unlock(&priv->lock); |
702 | 763 | ||
703 | /* Drop the reference to the event channel we did not save in the map */ | 764 | /* Drop the reference to the event channel we did not save in the map */ |
704 | if (out_flags & UNMAP_NOTIFY_SEND_EVENT) | 765 | if (out_flags & UNMAP_NOTIFY_SEND_EVENT) |
@@ -748,7 +809,7 @@ static int gntdev_mmap(struct file *flip, struct vm_area_struct *vma) | |||
748 | pr_debug("map %d+%d at %lx (pgoff %lx)\n", | 809 | pr_debug("map %d+%d at %lx (pgoff %lx)\n", |
749 | index, count, vma->vm_start, vma->vm_pgoff); | 810 | index, count, vma->vm_start, vma->vm_pgoff); |
750 | 811 | ||
751 | spin_lock(&priv->lock); | 812 | mutex_lock(&priv->lock); |
752 | map = gntdev_find_map_index(priv, index, count); | 813 | map = gntdev_find_map_index(priv, index, count); |
753 | if (!map) | 814 | if (!map) |
754 | goto unlock_out; | 815 | goto unlock_out; |
@@ -783,7 +844,7 @@ static int gntdev_mmap(struct file *flip, struct vm_area_struct *vma) | |||
783 | map->flags |= GNTMAP_readonly; | 844 | map->flags |= GNTMAP_readonly; |
784 | } | 845 | } |
785 | 846 | ||
786 | spin_unlock(&priv->lock); | 847 | mutex_unlock(&priv->lock); |
787 | 848 | ||
788 | if (use_ptemod) { | 849 | if (use_ptemod) { |
789 | err = apply_to_page_range(vma->vm_mm, vma->vm_start, | 850 | err = apply_to_page_range(vma->vm_mm, vma->vm_start, |
@@ -806,16 +867,34 @@ static int gntdev_mmap(struct file *flip, struct vm_area_struct *vma) | |||
806 | if (err) | 867 | if (err) |
807 | goto out_put_map; | 868 | goto out_put_map; |
808 | } | 869 | } |
870 | } else { | ||
871 | #ifdef CONFIG_X86 | ||
872 | /* | ||
873 | * If the PTEs were not made special by the grant map | ||
874 | * hypercall, do so here. | ||
875 | * | ||
876 | * This is racy since the mapping is already visible | ||
877 | * to userspace but userspace should be well-behaved | ||
878 | * enough to not touch it until the mmap() call | ||
879 | * returns. | ||
880 | */ | ||
881 | if (!xen_feature(XENFEAT_gnttab_map_avail_bits)) { | ||
882 | apply_to_page_range(vma->vm_mm, vma->vm_start, | ||
883 | vma->vm_end - vma->vm_start, | ||
884 | set_grant_ptes_as_special, NULL); | ||
885 | } | ||
886 | #endif | ||
887 | map->pages_vm_start = vma->vm_start; | ||
809 | } | 888 | } |
810 | 889 | ||
811 | return 0; | 890 | return 0; |
812 | 891 | ||
813 | unlock_out: | 892 | unlock_out: |
814 | spin_unlock(&priv->lock); | 893 | mutex_unlock(&priv->lock); |
815 | return err; | 894 | return err; |
816 | 895 | ||
817 | out_unlock_put: | 896 | out_unlock_put: |
818 | spin_unlock(&priv->lock); | 897 | mutex_unlock(&priv->lock); |
819 | out_put_map: | 898 | out_put_map: |
820 | if (use_ptemod) | 899 | if (use_ptemod) |
821 | map->vma = NULL; | 900 | map->vma = NULL; |
diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c index 7786291ba229..17972fbacddc 100644 --- a/drivers/xen/grant-table.c +++ b/drivers/xen/grant-table.c | |||
@@ -42,6 +42,7 @@ | |||
42 | #include <linux/io.h> | 42 | #include <linux/io.h> |
43 | #include <linux/delay.h> | 43 | #include <linux/delay.h> |
44 | #include <linux/hardirq.h> | 44 | #include <linux/hardirq.h> |
45 | #include <linux/workqueue.h> | ||
45 | 46 | ||
46 | #include <xen/xen.h> | 47 | #include <xen/xen.h> |
47 | #include <xen/interface/xen.h> | 48 | #include <xen/interface/xen.h> |
@@ -50,6 +51,7 @@ | |||
50 | #include <xen/interface/memory.h> | 51 | #include <xen/interface/memory.h> |
51 | #include <xen/hvc-console.h> | 52 | #include <xen/hvc-console.h> |
52 | #include <xen/swiotlb-xen.h> | 53 | #include <xen/swiotlb-xen.h> |
54 | #include <xen/balloon.h> | ||
53 | #include <asm/xen/hypercall.h> | 55 | #include <asm/xen/hypercall.h> |
54 | #include <asm/xen/interface.h> | 56 | #include <asm/xen/interface.h> |
55 | 57 | ||
@@ -671,6 +673,59 @@ void gnttab_free_auto_xlat_frames(void) | |||
671 | } | 673 | } |
672 | EXPORT_SYMBOL_GPL(gnttab_free_auto_xlat_frames); | 674 | EXPORT_SYMBOL_GPL(gnttab_free_auto_xlat_frames); |
673 | 675 | ||
676 | /** | ||
677 | * gnttab_alloc_pages - alloc pages suitable for grant mapping into | ||
678 | * @nr_pages: number of pages to alloc | ||
679 | * @pages: returns the pages | ||
680 | */ | ||
681 | int gnttab_alloc_pages(int nr_pages, struct page **pages) | ||
682 | { | ||
683 | int i; | ||
684 | int ret; | ||
685 | |||
686 | ret = alloc_xenballooned_pages(nr_pages, pages, false); | ||
687 | if (ret < 0) | ||
688 | return ret; | ||
689 | |||
690 | for (i = 0; i < nr_pages; i++) { | ||
691 | #if BITS_PER_LONG < 64 | ||
692 | struct xen_page_foreign *foreign; | ||
693 | |||
694 | foreign = kzalloc(sizeof(*foreign), GFP_KERNEL); | ||
695 | if (!foreign) { | ||
696 | gnttab_free_pages(nr_pages, pages); | ||
697 | return -ENOMEM; | ||
698 | } | ||
699 | set_page_private(pages[i], (unsigned long)foreign); | ||
700 | #endif | ||
701 | SetPagePrivate(pages[i]); | ||
702 | } | ||
703 | |||
704 | return 0; | ||
705 | } | ||
706 | EXPORT_SYMBOL(gnttab_alloc_pages); | ||
707 | |||
708 | /** | ||
709 | * gnttab_free_pages - free pages allocated by gnttab_alloc_pages() | ||
710 | * @nr_pages; number of pages to free | ||
711 | * @pages: the pages | ||
712 | */ | ||
713 | void gnttab_free_pages(int nr_pages, struct page **pages) | ||
714 | { | ||
715 | int i; | ||
716 | |||
717 | for (i = 0; i < nr_pages; i++) { | ||
718 | if (PagePrivate(pages[i])) { | ||
719 | #if BITS_PER_LONG < 64 | ||
720 | kfree((void *)page_private(pages[i])); | ||
721 | #endif | ||
722 | ClearPagePrivate(pages[i]); | ||
723 | } | ||
724 | } | ||
725 | free_xenballooned_pages(nr_pages, pages); | ||
726 | } | ||
727 | EXPORT_SYMBOL(gnttab_free_pages); | ||
728 | |||
674 | /* Handling of paged out grant targets (GNTST_eagain) */ | 729 | /* Handling of paged out grant targets (GNTST_eagain) */ |
675 | #define MAX_DELAY 256 | 730 | #define MAX_DELAY 256 |
676 | static inline void | 731 | static inline void |
@@ -727,30 +782,87 @@ int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops, | |||
727 | if (ret) | 782 | if (ret) |
728 | return ret; | 783 | return ret; |
729 | 784 | ||
730 | /* Retry eagain maps */ | 785 | for (i = 0; i < count; i++) { |
731 | for (i = 0; i < count; i++) | 786 | /* Retry eagain maps */ |
732 | if (map_ops[i].status == GNTST_eagain) | 787 | if (map_ops[i].status == GNTST_eagain) |
733 | gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, map_ops + i, | 788 | gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, map_ops + i, |
734 | &map_ops[i].status, __func__); | 789 | &map_ops[i].status, __func__); |
735 | 790 | ||
791 | if (map_ops[i].status == GNTST_okay) { | ||
792 | struct xen_page_foreign *foreign; | ||
793 | |||
794 | SetPageForeign(pages[i]); | ||
795 | foreign = xen_page_foreign(pages[i]); | ||
796 | foreign->domid = map_ops[i].dom; | ||
797 | foreign->gref = map_ops[i].ref; | ||
798 | } | ||
799 | } | ||
800 | |||
736 | return set_foreign_p2m_mapping(map_ops, kmap_ops, pages, count); | 801 | return set_foreign_p2m_mapping(map_ops, kmap_ops, pages, count); |
737 | } | 802 | } |
738 | EXPORT_SYMBOL_GPL(gnttab_map_refs); | 803 | EXPORT_SYMBOL_GPL(gnttab_map_refs); |
739 | 804 | ||
740 | int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops, | 805 | int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops, |
741 | struct gnttab_map_grant_ref *kmap_ops, | 806 | struct gnttab_unmap_grant_ref *kunmap_ops, |
742 | struct page **pages, unsigned int count) | 807 | struct page **pages, unsigned int count) |
743 | { | 808 | { |
809 | unsigned int i; | ||
744 | int ret; | 810 | int ret; |
745 | 811 | ||
746 | ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count); | 812 | ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count); |
747 | if (ret) | 813 | if (ret) |
748 | return ret; | 814 | return ret; |
749 | 815 | ||
750 | return clear_foreign_p2m_mapping(unmap_ops, kmap_ops, pages, count); | 816 | for (i = 0; i < count; i++) |
817 | ClearPageForeign(pages[i]); | ||
818 | |||
819 | return clear_foreign_p2m_mapping(unmap_ops, kunmap_ops, pages, count); | ||
751 | } | 820 | } |
752 | EXPORT_SYMBOL_GPL(gnttab_unmap_refs); | 821 | EXPORT_SYMBOL_GPL(gnttab_unmap_refs); |
753 | 822 | ||
823 | #define GNTTAB_UNMAP_REFS_DELAY 5 | ||
824 | |||
825 | static void __gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item); | ||
826 | |||
827 | static void gnttab_unmap_work(struct work_struct *work) | ||
828 | { | ||
829 | struct gntab_unmap_queue_data | ||
830 | *unmap_data = container_of(work, | ||
831 | struct gntab_unmap_queue_data, | ||
832 | gnttab_work.work); | ||
833 | if (unmap_data->age != UINT_MAX) | ||
834 | unmap_data->age++; | ||
835 | __gnttab_unmap_refs_async(unmap_data); | ||
836 | } | ||
837 | |||
838 | static void __gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item) | ||
839 | { | ||
840 | int ret; | ||
841 | int pc; | ||
842 | |||
843 | for (pc = 0; pc < item->count; pc++) { | ||
844 | if (page_count(item->pages[pc]) > 1) { | ||
845 | unsigned long delay = GNTTAB_UNMAP_REFS_DELAY * (item->age + 1); | ||
846 | schedule_delayed_work(&item->gnttab_work, | ||
847 | msecs_to_jiffies(delay)); | ||
848 | return; | ||
849 | } | ||
850 | } | ||
851 | |||
852 | ret = gnttab_unmap_refs(item->unmap_ops, item->kunmap_ops, | ||
853 | item->pages, item->count); | ||
854 | item->done(ret, item); | ||
855 | } | ||
856 | |||
857 | void gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item) | ||
858 | { | ||
859 | INIT_DELAYED_WORK(&item->gnttab_work, gnttab_unmap_work); | ||
860 | item->age = 0; | ||
861 | |||
862 | __gnttab_unmap_refs_async(item); | ||
863 | } | ||
864 | EXPORT_SYMBOL_GPL(gnttab_unmap_refs_async); | ||
865 | |||
754 | static int gnttab_map_frames_v1(xen_pfn_t *frames, unsigned int nr_gframes) | 866 | static int gnttab_map_frames_v1(xen_pfn_t *frames, unsigned int nr_gframes) |
755 | { | 867 | { |
756 | int rc; | 868 | int rc; |
diff --git a/drivers/xen/manage.c b/drivers/xen/manage.c index f8bb36f9d9ce..bf1940706422 100644 --- a/drivers/xen/manage.c +++ b/drivers/xen/manage.c | |||
@@ -105,10 +105,16 @@ static void do_suspend(void) | |||
105 | 105 | ||
106 | err = freeze_processes(); | 106 | err = freeze_processes(); |
107 | if (err) { | 107 | if (err) { |
108 | pr_err("%s: freeze failed %d\n", __func__, err); | 108 | pr_err("%s: freeze processes failed %d\n", __func__, err); |
109 | goto out; | 109 | goto out; |
110 | } | 110 | } |
111 | 111 | ||
112 | err = freeze_kernel_threads(); | ||
113 | if (err) { | ||
114 | pr_err("%s: freeze kernel threads failed %d\n", __func__, err); | ||
115 | goto out_thaw; | ||
116 | } | ||
117 | |||
112 | err = dpm_suspend_start(PMSG_FREEZE); | 118 | err = dpm_suspend_start(PMSG_FREEZE); |
113 | if (err) { | 119 | if (err) { |
114 | pr_err("%s: dpm_suspend_start %d\n", __func__, err); | 120 | pr_err("%s: dpm_suspend_start %d\n", __func__, err); |
diff --git a/drivers/xen/tmem.c b/drivers/xen/tmem.c index 83b5c53bec6b..8a65423bc696 100644 --- a/drivers/xen/tmem.c +++ b/drivers/xen/tmem.c | |||
@@ -374,7 +374,7 @@ static struct frontswap_ops tmem_frontswap_ops = { | |||
374 | }; | 374 | }; |
375 | #endif | 375 | #endif |
376 | 376 | ||
377 | static int xen_tmem_init(void) | 377 | static int __init xen_tmem_init(void) |
378 | { | 378 | { |
379 | if (!xen_domain()) | 379 | if (!xen_domain()) |
380 | return 0; | 380 | return 0; |
diff --git a/drivers/xen/xen-scsiback.c b/drivers/xen/xen-scsiback.c index e999496eda3e..ecd540a7a562 100644 --- a/drivers/xen/xen-scsiback.c +++ b/drivers/xen/xen-scsiback.c | |||
@@ -227,7 +227,7 @@ static void put_free_pages(struct page **page, int num) | |||
227 | return; | 227 | return; |
228 | if (i > scsiback_max_buffer_pages) { | 228 | if (i > scsiback_max_buffer_pages) { |
229 | n = min(num, i - scsiback_max_buffer_pages); | 229 | n = min(num, i - scsiback_max_buffer_pages); |
230 | free_xenballooned_pages(n, page + num - n); | 230 | gnttab_free_pages(n, page + num - n); |
231 | n = num - n; | 231 | n = num - n; |
232 | } | 232 | } |
233 | spin_lock_irqsave(&free_pages_lock, flags); | 233 | spin_lock_irqsave(&free_pages_lock, flags); |
@@ -244,7 +244,7 @@ static int get_free_page(struct page **page) | |||
244 | spin_lock_irqsave(&free_pages_lock, flags); | 244 | spin_lock_irqsave(&free_pages_lock, flags); |
245 | if (list_empty(&scsiback_free_pages)) { | 245 | if (list_empty(&scsiback_free_pages)) { |
246 | spin_unlock_irqrestore(&free_pages_lock, flags); | 246 | spin_unlock_irqrestore(&free_pages_lock, flags); |
247 | return alloc_xenballooned_pages(1, page, false); | 247 | return gnttab_alloc_pages(1, page); |
248 | } | 248 | } |
249 | page[0] = list_first_entry(&scsiback_free_pages, struct page, lru); | 249 | page[0] = list_first_entry(&scsiback_free_pages, struct page, lru); |
250 | list_del(&page[0]->lru); | 250 | list_del(&page[0]->lru); |
@@ -2106,7 +2106,7 @@ static void __exit scsiback_exit(void) | |||
2106 | while (free_pages_num) { | 2106 | while (free_pages_num) { |
2107 | if (get_free_page(&page)) | 2107 | if (get_free_page(&page)) |
2108 | BUG(); | 2108 | BUG(); |
2109 | free_xenballooned_pages(1, &page); | 2109 | gnttab_free_pages(1, &page); |
2110 | } | 2110 | } |
2111 | scsiback_deregister_configfs(); | 2111 | scsiback_deregister_configfs(); |
2112 | xenbus_unregister_driver(&scsiback_driver); | 2112 | xenbus_unregister_driver(&scsiback_driver); |
diff --git a/drivers/xen/xenbus/xenbus_dev_frontend.c b/drivers/xen/xenbus/xenbus_dev_frontend.c index 85534ea63555..9433e46518c8 100644 --- a/drivers/xen/xenbus/xenbus_dev_frontend.c +++ b/drivers/xen/xenbus/xenbus_dev_frontend.c | |||
@@ -326,10 +326,13 @@ static int xenbus_write_transaction(unsigned msg_type, | |||
326 | } | 326 | } |
327 | 327 | ||
328 | if (msg_type == XS_TRANSACTION_START) { | 328 | if (msg_type == XS_TRANSACTION_START) { |
329 | trans->handle.id = simple_strtoul(reply, NULL, 0); | 329 | if (u->u.msg.type == XS_ERROR) |
330 | 330 | kfree(trans); | |
331 | list_add(&trans->list, &u->transactions); | 331 | else { |
332 | } else if (msg_type == XS_TRANSACTION_END) { | 332 | trans->handle.id = simple_strtoul(reply, NULL, 0); |
333 | list_add(&trans->list, &u->transactions); | ||
334 | } | ||
335 | } else if (u->u.msg.type == XS_TRANSACTION_END) { | ||
333 | list_for_each_entry(trans, &u->transactions, list) | 336 | list_for_each_entry(trans, &u->transactions, list) |
334 | if (trans->handle.id == u->u.msg.tx_id) | 337 | if (trans->handle.id == u->u.msg.tx_id) |
335 | break; | 338 | break; |
diff --git a/include/linux/mm.h b/include/linux/mm.h index dd5ea3016fc4..237b3ba29225 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h | |||
@@ -290,6 +290,14 @@ struct vm_operations_struct { | |||
290 | /* called by sys_remap_file_pages() to populate non-linear mapping */ | 290 | /* called by sys_remap_file_pages() to populate non-linear mapping */ |
291 | int (*remap_pages)(struct vm_area_struct *vma, unsigned long addr, | 291 | int (*remap_pages)(struct vm_area_struct *vma, unsigned long addr, |
292 | unsigned long size, pgoff_t pgoff); | 292 | unsigned long size, pgoff_t pgoff); |
293 | |||
294 | /* | ||
295 | * Called by vm_normal_page() for special PTEs to find the | ||
296 | * page for @addr. This is useful if the default behavior | ||
297 | * (using pte_page()) would not find the correct page. | ||
298 | */ | ||
299 | struct page *(*find_special_page)(struct vm_area_struct *vma, | ||
300 | unsigned long addr); | ||
293 | }; | 301 | }; |
294 | 302 | ||
295 | struct mmu_gather; | 303 | struct mmu_gather; |
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h index e1f5fcd79792..5ed7bdaf22d5 100644 --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h | |||
@@ -121,8 +121,12 @@ enum pageflags { | |||
121 | PG_fscache = PG_private_2, /* page backed by cache */ | 121 | PG_fscache = PG_private_2, /* page backed by cache */ |
122 | 122 | ||
123 | /* XEN */ | 123 | /* XEN */ |
124 | /* Pinned in Xen as a read-only pagetable page. */ | ||
124 | PG_pinned = PG_owner_priv_1, | 125 | PG_pinned = PG_owner_priv_1, |
126 | /* Pinned as part of domain save (see xen_mm_pin_all()). */ | ||
125 | PG_savepinned = PG_dirty, | 127 | PG_savepinned = PG_dirty, |
128 | /* Has a grant mapping of another (foreign) domain's page. */ | ||
129 | PG_foreign = PG_owner_priv_1, | ||
126 | 130 | ||
127 | /* SLOB */ | 131 | /* SLOB */ |
128 | PG_slob_free = PG_private, | 132 | PG_slob_free = PG_private, |
@@ -215,6 +219,7 @@ __PAGEFLAG(Slab, slab) | |||
215 | PAGEFLAG(Checked, checked) /* Used by some filesystems */ | 219 | PAGEFLAG(Checked, checked) /* Used by some filesystems */ |
216 | PAGEFLAG(Pinned, pinned) TESTSCFLAG(Pinned, pinned) /* Xen */ | 220 | PAGEFLAG(Pinned, pinned) TESTSCFLAG(Pinned, pinned) /* Xen */ |
217 | PAGEFLAG(SavePinned, savepinned); /* Xen */ | 221 | PAGEFLAG(SavePinned, savepinned); /* Xen */ |
222 | PAGEFLAG(Foreign, foreign); /* Xen */ | ||
218 | PAGEFLAG(Reserved, reserved) __CLEARPAGEFLAG(Reserved, reserved) | 223 | PAGEFLAG(Reserved, reserved) __CLEARPAGEFLAG(Reserved, reserved) |
219 | PAGEFLAG(SwapBacked, swapbacked) __CLEARPAGEFLAG(SwapBacked, swapbacked) | 224 | PAGEFLAG(SwapBacked, swapbacked) __CLEARPAGEFLAG(SwapBacked, swapbacked) |
220 | __SETPAGEFLAG(SwapBacked, swapbacked) | 225 | __SETPAGEFLAG(SwapBacked, swapbacked) |
diff --git a/include/xen/grant_table.h b/include/xen/grant_table.h index 3387465b9caa..143ca5ffab7a 100644 --- a/include/xen/grant_table.h +++ b/include/xen/grant_table.h | |||
@@ -45,6 +45,8 @@ | |||
45 | #include <asm/xen/hypervisor.h> | 45 | #include <asm/xen/hypervisor.h> |
46 | 46 | ||
47 | #include <xen/features.h> | 47 | #include <xen/features.h> |
48 | #include <linux/mm_types.h> | ||
49 | #include <linux/page-flags.h> | ||
48 | 50 | ||
49 | #define GNTTAB_RESERVED_XENSTORE 1 | 51 | #define GNTTAB_RESERVED_XENSTORE 1 |
50 | 52 | ||
@@ -58,6 +60,22 @@ struct gnttab_free_callback { | |||
58 | u16 count; | 60 | u16 count; |
59 | }; | 61 | }; |
60 | 62 | ||
63 | struct gntab_unmap_queue_data; | ||
64 | |||
65 | typedef void (*gnttab_unmap_refs_done)(int result, struct gntab_unmap_queue_data *data); | ||
66 | |||
67 | struct gntab_unmap_queue_data | ||
68 | { | ||
69 | struct delayed_work gnttab_work; | ||
70 | void *data; | ||
71 | gnttab_unmap_refs_done done; | ||
72 | struct gnttab_unmap_grant_ref *unmap_ops; | ||
73 | struct gnttab_unmap_grant_ref *kunmap_ops; | ||
74 | struct page **pages; | ||
75 | unsigned int count; | ||
76 | unsigned int age; | ||
77 | }; | ||
78 | |||
61 | int gnttab_init(void); | 79 | int gnttab_init(void); |
62 | int gnttab_suspend(void); | 80 | int gnttab_suspend(void); |
63 | int gnttab_resume(void); | 81 | int gnttab_resume(void); |
@@ -163,12 +181,17 @@ void gnttab_free_auto_xlat_frames(void); | |||
163 | 181 | ||
164 | #define gnttab_map_vaddr(map) ((void *)(map.host_virt_addr)) | 182 | #define gnttab_map_vaddr(map) ((void *)(map.host_virt_addr)) |
165 | 183 | ||
184 | int gnttab_alloc_pages(int nr_pages, struct page **pages); | ||
185 | void gnttab_free_pages(int nr_pages, struct page **pages); | ||
186 | |||
166 | int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops, | 187 | int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops, |
167 | struct gnttab_map_grant_ref *kmap_ops, | 188 | struct gnttab_map_grant_ref *kmap_ops, |
168 | struct page **pages, unsigned int count); | 189 | struct page **pages, unsigned int count); |
169 | int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops, | 190 | int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops, |
170 | struct gnttab_map_grant_ref *kunmap_ops, | 191 | struct gnttab_unmap_grant_ref *kunmap_ops, |
171 | struct page **pages, unsigned int count); | 192 | struct page **pages, unsigned int count); |
193 | void gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item); | ||
194 | |||
172 | 195 | ||
173 | /* Perform a batch of grant map/copy operations. Retry every batch slot | 196 | /* Perform a batch of grant map/copy operations. Retry every batch slot |
174 | * for which the hypervisor returns GNTST_eagain. This is typically due | 197 | * for which the hypervisor returns GNTST_eagain. This is typically due |
@@ -182,4 +205,22 @@ int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops, | |||
182 | void gnttab_batch_map(struct gnttab_map_grant_ref *batch, unsigned count); | 205 | void gnttab_batch_map(struct gnttab_map_grant_ref *batch, unsigned count); |
183 | void gnttab_batch_copy(struct gnttab_copy *batch, unsigned count); | 206 | void gnttab_batch_copy(struct gnttab_copy *batch, unsigned count); |
184 | 207 | ||
208 | |||
209 | struct xen_page_foreign { | ||
210 | domid_t domid; | ||
211 | grant_ref_t gref; | ||
212 | }; | ||
213 | |||
214 | static inline struct xen_page_foreign *xen_page_foreign(struct page *page) | ||
215 | { | ||
216 | if (!PageForeign(page)) | ||
217 | return NULL; | ||
218 | #if BITS_PER_LONG < 64 | ||
219 | return (struct xen_page_foreign *)page->private; | ||
220 | #else | ||
221 | BUILD_BUG_ON(sizeof(struct xen_page_foreign) > BITS_PER_LONG); | ||
222 | return (struct xen_page_foreign *)&page->private; | ||
223 | #endif | ||
224 | } | ||
225 | |||
185 | #endif /* __ASM_GNTTAB_H__ */ | 226 | #endif /* __ASM_GNTTAB_H__ */ |
diff --git a/include/xen/interface/features.h b/include/xen/interface/features.h index 131a6ccdba25..6ad3d110bb81 100644 --- a/include/xen/interface/features.h +++ b/include/xen/interface/features.h | |||
@@ -41,6 +41,12 @@ | |||
41 | /* x86: Does this Xen host support the MMU_PT_UPDATE_PRESERVE_AD hypercall? */ | 41 | /* x86: Does this Xen host support the MMU_PT_UPDATE_PRESERVE_AD hypercall? */ |
42 | #define XENFEAT_mmu_pt_update_preserve_ad 5 | 42 | #define XENFEAT_mmu_pt_update_preserve_ad 5 |
43 | 43 | ||
44 | /* | ||
45 | * If set, GNTTABOP_map_grant_ref honors flags to be placed into guest kernel | ||
46 | * available pte bits. | ||
47 | */ | ||
48 | #define XENFEAT_gnttab_map_avail_bits 7 | ||
49 | |||
44 | /* x86: Does this Xen host support the HVM callback vector type? */ | 50 | /* x86: Does this Xen host support the HVM callback vector type? */ |
45 | #define XENFEAT_hvm_callback_vector 8 | 51 | #define XENFEAT_hvm_callback_vector 8 |
46 | 52 | ||
diff --git a/include/xen/interface/grant_table.h b/include/xen/interface/grant_table.h index bcce56439d64..56806bc90c2f 100644 --- a/include/xen/interface/grant_table.h +++ b/include/xen/interface/grant_table.h | |||
@@ -526,6 +526,13 @@ DEFINE_GUEST_HANDLE_STRUCT(gnttab_cache_flush); | |||
526 | #define GNTMAP_contains_pte (1<<_GNTMAP_contains_pte) | 526 | #define GNTMAP_contains_pte (1<<_GNTMAP_contains_pte) |
527 | 527 | ||
528 | /* | 528 | /* |
529 | * Bits to be placed in guest kernel available PTE bits (architecture | ||
530 | * dependent; only supported when XENFEAT_gnttab_map_avail_bits is set). | ||
531 | */ | ||
532 | #define _GNTMAP_guest_avail0 (16) | ||
533 | #define GNTMAP_guest_avail_mask ((uint32_t)~0 << _GNTMAP_guest_avail0) | ||
534 | |||
535 | /* | ||
529 | * Values for error status returns. All errors are -ve. | 536 | * Values for error status returns. All errors are -ve. |
530 | */ | 537 | */ |
531 | #define GNTST_okay (0) /* Normal return. */ | 538 | #define GNTST_okay (0) /* Normal return. */ |
diff --git a/mm/memory.c b/mm/memory.c index 2c3536cc6c63..d707c4dfbbb4 100644 --- a/mm/memory.c +++ b/mm/memory.c | |||
@@ -754,6 +754,8 @@ struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, | |||
754 | if (HAVE_PTE_SPECIAL) { | 754 | if (HAVE_PTE_SPECIAL) { |
755 | if (likely(!pte_special(pte))) | 755 | if (likely(!pte_special(pte))) |
756 | goto check_pfn; | 756 | goto check_pfn; |
757 | if (vma->vm_ops && vma->vm_ops->find_special_page) | ||
758 | return vma->vm_ops->find_special_page(vma, addr); | ||
757 | if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)) | 759 | if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)) |
758 | return NULL; | 760 | return NULL; |
759 | if (!is_zero_pfn(pfn)) | 761 | if (!is_zero_pfn(pfn)) |