aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2015-02-10 16:56:56 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2015-02-10 16:56:56 -0500
commitbdccc4edeb03ad68c55053b0260bdaaac547bbd9 (patch)
treec9fdce7c6bb13632f5a64925a1fb3fa306b7cfe1
parent98368ab436538103a557fc1f15f54afd8aab6712 (diff)
parent72978b2fe2f2cdf9f319c6c6dcdbe92b38de2be2 (diff)
Merge tag 'stable/for-linus-3.20-rc0-tag' of git://git.kernel.org/pub/scm/linux/kernel/git/xen/tip
Pull xen features and fixes from David Vrabel: - Reworked handling for foreign (grant mapped) pages to simplify the code, enable a number of additional use cases and fix a number of long-standing bugs. - Prefer the TSC over the Xen PV clock when dom0 (and the TSC is stable). - Assorted other cleanup and minor bug fixes. * tag 'stable/for-linus-3.20-rc0-tag' of git://git.kernel.org/pub/scm/linux/kernel/git/xen/tip: (25 commits) xen/manage: Fix USB interaction issues when resuming xenbus: Add proper handling of XS_ERROR from Xenbus for transactions. xen/gntdev: provide find_special_page VMA operation xen/gntdev: mark userspace PTEs as special on x86 PV guests xen-blkback: safely unmap grants in case they are still in use xen/gntdev: safely unmap grants in case they are still in use xen/gntdev: convert priv->lock to a mutex xen/grant-table: add a mechanism to safely unmap pages that are in use xen-netback: use foreign page information from the pages themselves xen: mark grant mapped pages as foreign xen/grant-table: add helpers for allocating pages x86/xen: require ballooned pages for grant maps xen: remove scratch frames for ballooned pages and m2p override xen/grant-table: pre-populate kernel unmap ops for xen_gnttab_unmap_refs() mm: add 'foreign' alias for the 'pinned' page flag mm: provide a find_special_page vma operation x86/xen: cleanup arch/x86/xen/mmu.c x86/xen: add some __init annotations in arch/x86/xen/mmu.c x86/xen: add some __init and static annotations in arch/x86/xen/setup.c x86/xen: use correct types for addresses in arch/x86/xen/setup.c ...
Diffstat
-rw-r--r--arch/arm/include/asm/xen/page.h2
-rw-r--r--arch/arm/xen/enlighten.c4
-rw-r--r--arch/arm/xen/mm.c2
-rw-r--r--arch/arm/xen/p2m.c2
-rw-r--r--arch/x86/include/asm/xen/page.h20
-rw-r--r--arch/x86/xen/mmu.c17
-rw-r--r--arch/x86/xen/p2m.c267
-rw-r--r--arch/x86/xen/setup.c37
-rw-r--r--arch/x86/xen/smp.c2
-rw-r--r--arch/x86/xen/time.c4
-rw-r--r--arch/x86/xen/xen-ops.h6
-rw-r--r--drivers/block/xen-blkback/blkback.c177
-rw-r--r--drivers/block/xen-blkback/common.h3
-rw-r--r--drivers/net/xen-netback/interface.c7
-rw-r--r--drivers/net/xen-netback/netback.c106
-rw-r--r--drivers/xen/balloon.c86
-rw-r--r--drivers/xen/gntdev.c143
-rw-r--r--drivers/xen/grant-table.c120
-rw-r--r--drivers/xen/manage.c8
-rw-r--r--drivers/xen/tmem.c2
-rw-r--r--drivers/xen/xen-scsiback.c6
-rw-r--r--drivers/xen/xenbus/xenbus_dev_frontend.c11
-rw-r--r--include/linux/mm.h8
-rw-r--r--include/linux/page-flags.h5
-rw-r--r--include/xen/grant_table.h43
-rw-r--r--include/xen/interface/features.h6
-rw-r--r--include/xen/interface/grant_table.h7
-rw-r--r--mm/memory.c2
28 files changed, 509 insertions, 594 deletions
diff --git a/arch/arm/include/asm/xen/page.h b/arch/arm/include/asm/xen/page.h
index 68c739b3fdf4..2f7e6ff67d51 100644
--- a/arch/arm/include/asm/xen/page.h
+++ b/arch/arm/include/asm/xen/page.h
@@ -92,7 +92,7 @@ extern int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops,
92 struct page **pages, unsigned int count); 92 struct page **pages, unsigned int count);
93 93
94extern int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops, 94extern int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops,
95 struct gnttab_map_grant_ref *kmap_ops, 95 struct gnttab_unmap_grant_ref *kunmap_ops,
96 struct page **pages, unsigned int count); 96 struct page **pages, unsigned int count);
97 97
98bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn); 98bool __set_phys_to_machine(unsigned long pfn, unsigned long mfn);
diff --git a/arch/arm/xen/enlighten.c b/arch/arm/xen/enlighten.c
index c7ca936ebd99..263a2044c65b 100644
--- a/arch/arm/xen/enlighten.c
+++ b/arch/arm/xen/enlighten.c
@@ -29,10 +29,10 @@
29 29
30struct start_info _xen_start_info; 30struct start_info _xen_start_info;
31struct start_info *xen_start_info = &_xen_start_info; 31struct start_info *xen_start_info = &_xen_start_info;
32EXPORT_SYMBOL_GPL(xen_start_info); 32EXPORT_SYMBOL(xen_start_info);
33 33
34enum xen_domain_type xen_domain_type = XEN_NATIVE; 34enum xen_domain_type xen_domain_type = XEN_NATIVE;
35EXPORT_SYMBOL_GPL(xen_domain_type); 35EXPORT_SYMBOL(xen_domain_type);
36 36
37struct shared_info xen_dummy_shared_info; 37struct shared_info xen_dummy_shared_info;
38struct shared_info *HYPERVISOR_shared_info = (void *)&xen_dummy_shared_info; 38struct shared_info *HYPERVISOR_shared_info = (void *)&xen_dummy_shared_info;
diff --git a/arch/arm/xen/mm.c b/arch/arm/xen/mm.c
index 351b24a979d4..793551d15f1d 100644
--- a/arch/arm/xen/mm.c
+++ b/arch/arm/xen/mm.c
@@ -149,7 +149,7 @@ void xen_destroy_contiguous_region(phys_addr_t pstart, unsigned int order)
149EXPORT_SYMBOL_GPL(xen_destroy_contiguous_region); 149EXPORT_SYMBOL_GPL(xen_destroy_contiguous_region);
150 150
151struct dma_map_ops *xen_dma_ops; 151struct dma_map_ops *xen_dma_ops;
152EXPORT_SYMBOL_GPL(xen_dma_ops); 152EXPORT_SYMBOL(xen_dma_ops);
153 153
154static struct dma_map_ops xen_swiotlb_dma_ops = { 154static struct dma_map_ops xen_swiotlb_dma_ops = {
155 .mapping_error = xen_swiotlb_dma_mapping_error, 155 .mapping_error = xen_swiotlb_dma_mapping_error,
diff --git a/arch/arm/xen/p2m.c b/arch/arm/xen/p2m.c
index 054857776254..cb7a14c5cd69 100644
--- a/arch/arm/xen/p2m.c
+++ b/arch/arm/xen/p2m.c
@@ -102,7 +102,7 @@ int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops,
102EXPORT_SYMBOL_GPL(set_foreign_p2m_mapping); 102EXPORT_SYMBOL_GPL(set_foreign_p2m_mapping);
103 103
104int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops, 104int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops,
105 struct gnttab_map_grant_ref *kmap_ops, 105 struct gnttab_unmap_grant_ref *kunmap_ops,
106 struct page **pages, unsigned int count) 106 struct page **pages, unsigned int count)
107{ 107{
108 int i; 108 int i;
diff --git a/arch/x86/include/asm/xen/page.h b/arch/x86/include/asm/xen/page.h
index 5eea09915a15..358dcd338915 100644
--- a/arch/x86/include/asm/xen/page.h
+++ b/arch/x86/include/asm/xen/page.h
@@ -55,9 +55,8 @@ extern int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops,
55 struct gnttab_map_grant_ref *kmap_ops, 55 struct gnttab_map_grant_ref *kmap_ops,
56 struct page **pages, unsigned int count); 56 struct page **pages, unsigned int count);
57extern int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops, 57extern int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops,
58 struct gnttab_map_grant_ref *kmap_ops, 58 struct gnttab_unmap_grant_ref *kunmap_ops,
59 struct page **pages, unsigned int count); 59 struct page **pages, unsigned int count);
60extern unsigned long m2p_find_override_pfn(unsigned long mfn, unsigned long pfn);
61 60
62/* 61/*
63 * Helper functions to write or read unsigned long values to/from 62 * Helper functions to write or read unsigned long values to/from
@@ -154,21 +153,12 @@ static inline unsigned long mfn_to_pfn(unsigned long mfn)
154 return mfn; 153 return mfn;
155 154
156 pfn = mfn_to_pfn_no_overrides(mfn); 155 pfn = mfn_to_pfn_no_overrides(mfn);
157 if (__pfn_to_mfn(pfn) != mfn) { 156 if (__pfn_to_mfn(pfn) != mfn)
158 /* 157 pfn = ~0;
159 * If this appears to be a foreign mfn (because the pfn
160 * doesn't map back to the mfn), then check the local override
161 * table to see if there's a better pfn to use.
162 *
163 * m2p_find_override_pfn returns ~0 if it doesn't find anything.
164 */
165 pfn = m2p_find_override_pfn(mfn, ~0);
166 }
167 158
168 /* 159 /*
169 * pfn is ~0 if there are no entries in the m2p for mfn or if the 160 * pfn is ~0 if there are no entries in the m2p for mfn or the
170 * entry doesn't map back to the mfn and m2p_override doesn't have a 161 * entry doesn't map back to the mfn.
171 * valid entry for it.
172 */ 162 */
173 if (pfn == ~0 && __pfn_to_mfn(mfn) == IDENTITY_FRAME(mfn)) 163 if (pfn == ~0 && __pfn_to_mfn(mfn) == IDENTITY_FRAME(mfn))
174 pfn = mfn; 164 pfn = mfn;
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 5c1f9ace7ae7..adca9e2b6553 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1489,7 +1489,7 @@ static void __init xen_set_pte_init(pte_t *ptep, pte_t pte)
1489 native_set_pte(ptep, pte); 1489 native_set_pte(ptep, pte);
1490} 1490}
1491 1491
1492static void pin_pagetable_pfn(unsigned cmd, unsigned long pfn) 1492static void __init pin_pagetable_pfn(unsigned cmd, unsigned long pfn)
1493{ 1493{
1494 struct mmuext_op op; 1494 struct mmuext_op op;
1495 op.cmd = cmd; 1495 op.cmd = cmd;
@@ -1657,7 +1657,7 @@ void __init xen_reserve_top(void)
1657 * Like __va(), but returns address in the kernel mapping (which is 1657 * Like __va(), but returns address in the kernel mapping (which is
1658 * all we have until the physical memory mapping has been set up. 1658 * all we have until the physical memory mapping has been set up.
1659 */ 1659 */
1660static void *__ka(phys_addr_t paddr) 1660static void * __init __ka(phys_addr_t paddr)
1661{ 1661{
1662#ifdef CONFIG_X86_64 1662#ifdef CONFIG_X86_64
1663 return (void *)(paddr + __START_KERNEL_map); 1663 return (void *)(paddr + __START_KERNEL_map);
@@ -1667,7 +1667,7 @@ static void *__ka(phys_addr_t paddr)
1667} 1667}
1668 1668
1669/* Convert a machine address to physical address */ 1669/* Convert a machine address to physical address */
1670static unsigned long m2p(phys_addr_t maddr) 1670static unsigned long __init m2p(phys_addr_t maddr)
1671{ 1671{
1672 phys_addr_t paddr; 1672 phys_addr_t paddr;
1673 1673
@@ -1678,13 +1678,14 @@ static unsigned long m2p(phys_addr_t maddr)
1678} 1678}
1679 1679
1680/* Convert a machine address to kernel virtual */ 1680/* Convert a machine address to kernel virtual */
1681static void *m2v(phys_addr_t maddr) 1681static void * __init m2v(phys_addr_t maddr)
1682{ 1682{
1683 return __ka(m2p(maddr)); 1683 return __ka(m2p(maddr));
1684} 1684}
1685 1685
1686/* Set the page permissions on an identity-mapped pages */ 1686/* Set the page permissions on an identity-mapped pages */
1687static void set_page_prot_flags(void *addr, pgprot_t prot, unsigned long flags) 1687static void __init set_page_prot_flags(void *addr, pgprot_t prot,
1688 unsigned long flags)
1688{ 1689{
1689 unsigned long pfn = __pa(addr) >> PAGE_SHIFT; 1690 unsigned long pfn = __pa(addr) >> PAGE_SHIFT;
1690 pte_t pte = pfn_pte(pfn, prot); 1691 pte_t pte = pfn_pte(pfn, prot);
@@ -1696,7 +1697,7 @@ static void set_page_prot_flags(void *addr, pgprot_t prot, unsigned long flags)
1696 if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, flags)) 1697 if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, flags))
1697 BUG(); 1698 BUG();
1698} 1699}
1699static void set_page_prot(void *addr, pgprot_t prot) 1700static void __init set_page_prot(void *addr, pgprot_t prot)
1700{ 1701{
1701 return set_page_prot_flags(addr, prot, UVMF_NONE); 1702 return set_page_prot_flags(addr, prot, UVMF_NONE);
1702} 1703}
@@ -1733,10 +1734,8 @@ static void __init xen_map_identity_early(pmd_t *pmd, unsigned long max_pfn)
1733 for (pteidx = 0; pteidx < PTRS_PER_PTE; pteidx++, pfn++) { 1734 for (pteidx = 0; pteidx < PTRS_PER_PTE; pteidx++, pfn++) {
1734 pte_t pte; 1735 pte_t pte;
1735 1736
1736#ifdef CONFIG_X86_32
1737 if (pfn > max_pfn_mapped) 1737 if (pfn > max_pfn_mapped)
1738 max_pfn_mapped = pfn; 1738 max_pfn_mapped = pfn;
1739#endif
1740 1739
1741 if (!pte_none(pte_page[pteidx])) 1740 if (!pte_none(pte_page[pteidx]))
1742 continue; 1741 continue;
@@ -1769,7 +1768,7 @@ void __init xen_setup_machphys_mapping(void)
1769} 1768}
1770 1769
1771#ifdef CONFIG_X86_64 1770#ifdef CONFIG_X86_64
1772static void convert_pfn_mfn(void *v) 1771static void __init convert_pfn_mfn(void *v)
1773{ 1772{
1774 pte_t *pte = v; 1773 pte_t *pte = v;
1775 int i; 1774 int i;
diff --git a/arch/x86/xen/p2m.c b/arch/x86/xen/p2m.c
index 70fb5075c901..f18fd1d411f6 100644
--- a/arch/x86/xen/p2m.c
+++ b/arch/x86/xen/p2m.c
@@ -84,8 +84,6 @@
84 84
85#define PMDS_PER_MID_PAGE (P2M_MID_PER_PAGE / PTRS_PER_PTE) 85#define PMDS_PER_MID_PAGE (P2M_MID_PER_PAGE / PTRS_PER_PTE)
86 86
87static void __init m2p_override_init(void);
88
89unsigned long *xen_p2m_addr __read_mostly; 87unsigned long *xen_p2m_addr __read_mostly;
90EXPORT_SYMBOL_GPL(xen_p2m_addr); 88EXPORT_SYMBOL_GPL(xen_p2m_addr);
91unsigned long xen_p2m_size __read_mostly; 89unsigned long xen_p2m_size __read_mostly;
@@ -402,8 +400,6 @@ void __init xen_vmalloc_p2m_tree(void)
402 xen_p2m_size = xen_max_p2m_pfn; 400 xen_p2m_size = xen_max_p2m_pfn;
403 401
404 xen_inv_extra_mem(); 402 xen_inv_extra_mem();
405
406 m2p_override_init();
407} 403}
408 404
409unsigned long get_phys_to_machine(unsigned long pfn) 405unsigned long get_phys_to_machine(unsigned long pfn)
@@ -652,100 +648,21 @@ bool set_phys_to_machine(unsigned long pfn, unsigned long mfn)
652 return true; 648 return true;
653} 649}
654 650
655#define M2P_OVERRIDE_HASH_SHIFT 10
656#define M2P_OVERRIDE_HASH (1 << M2P_OVERRIDE_HASH_SHIFT)
657
658static struct list_head *m2p_overrides;
659static DEFINE_SPINLOCK(m2p_override_lock);
660
661static void __init m2p_override_init(void)
662{
663 unsigned i;
664
665 m2p_overrides = alloc_bootmem_align(
666 sizeof(*m2p_overrides) * M2P_OVERRIDE_HASH,
667 sizeof(unsigned long));
668
669 for (i = 0; i < M2P_OVERRIDE_HASH; i++)
670 INIT_LIST_HEAD(&m2p_overrides[i]);
671}
672
673static unsigned long mfn_hash(unsigned long mfn)
674{
675 return hash_long(mfn, M2P_OVERRIDE_HASH_SHIFT);
676}
677
678/* Add an MFN override for a particular page */
679static int m2p_add_override(unsigned long mfn, struct page *page,
680 struct gnttab_map_grant_ref *kmap_op)
681{
682 unsigned long flags;
683 unsigned long pfn;
684 unsigned long uninitialized_var(address);
685 unsigned level;
686 pte_t *ptep = NULL;
687
688 pfn = page_to_pfn(page);
689 if (!PageHighMem(page)) {
690 address = (unsigned long)__va(pfn << PAGE_SHIFT);
691 ptep = lookup_address(address, &level);
692 if (WARN(ptep == NULL || level != PG_LEVEL_4K,
693 "m2p_add_override: pfn %lx not mapped", pfn))
694 return -EINVAL;
695 }
696
697 if (kmap_op != NULL) {
698 if (!PageHighMem(page)) {
699 struct multicall_space mcs =
700 xen_mc_entry(sizeof(*kmap_op));
701
702 MULTI_grant_table_op(mcs.mc,
703 GNTTABOP_map_grant_ref, kmap_op, 1);
704
705 xen_mc_issue(PARAVIRT_LAZY_MMU);
706 }
707 }
708 spin_lock_irqsave(&m2p_override_lock, flags);
709 list_add(&page->lru, &m2p_overrides[mfn_hash(mfn)]);
710 spin_unlock_irqrestore(&m2p_override_lock, flags);
711
712 /* p2m(m2p(mfn)) == mfn: the mfn is already present somewhere in
713 * this domain. Set the FOREIGN_FRAME_BIT in the p2m for the other
714 * pfn so that the following mfn_to_pfn(mfn) calls will return the
715 * pfn from the m2p_override (the backend pfn) instead.
716 * We need to do this because the pages shared by the frontend
717 * (xen-blkfront) can be already locked (lock_page, called by
718 * do_read_cache_page); when the userspace backend tries to use them
719 * with direct_IO, mfn_to_pfn returns the pfn of the frontend, so
720 * do_blockdev_direct_IO is going to try to lock the same pages
721 * again resulting in a deadlock.
722 * As a side effect get_user_pages_fast might not be safe on the
723 * frontend pages while they are being shared with the backend,
724 * because mfn_to_pfn (that ends up being called by GUPF) will
725 * return the backend pfn rather than the frontend pfn. */
726 pfn = mfn_to_pfn_no_overrides(mfn);
727 if (__pfn_to_mfn(pfn) == mfn)
728 set_phys_to_machine(pfn, FOREIGN_FRAME(mfn));
729
730 return 0;
731}
732
733int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops, 651int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops,
734 struct gnttab_map_grant_ref *kmap_ops, 652 struct gnttab_map_grant_ref *kmap_ops,
735 struct page **pages, unsigned int count) 653 struct page **pages, unsigned int count)
736{ 654{
737 int i, ret = 0; 655 int i, ret = 0;
738 bool lazy = false;
739 pte_t *pte; 656 pte_t *pte;
740 657
741 if (xen_feature(XENFEAT_auto_translated_physmap)) 658 if (xen_feature(XENFEAT_auto_translated_physmap))
742 return 0; 659 return 0;
743 660
744 if (kmap_ops && 661 if (kmap_ops) {
745 !in_interrupt() && 662 ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref,
746 paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) { 663 kmap_ops, count);
747 arch_enter_lazy_mmu_mode(); 664 if (ret)
748 lazy = true; 665 goto out;
749 } 666 }
750 667
751 for (i = 0; i < count; i++) { 668 for (i = 0; i < count; i++) {
@@ -764,170 +681,28 @@ int set_foreign_p2m_mapping(struct gnttab_map_grant_ref *map_ops,
764 } 681 }
765 pfn = page_to_pfn(pages[i]); 682 pfn = page_to_pfn(pages[i]);
766 683
767 WARN_ON(PagePrivate(pages[i])); 684 WARN(pfn_to_mfn(pfn) != INVALID_P2M_ENTRY, "page must be ballooned");
768 SetPagePrivate(pages[i]);
769 set_page_private(pages[i], mfn);
770 pages[i]->index = pfn_to_mfn(pfn);
771 685
772 if (unlikely(!set_phys_to_machine(pfn, FOREIGN_FRAME(mfn)))) { 686 if (unlikely(!set_phys_to_machine(pfn, FOREIGN_FRAME(mfn)))) {
773 ret = -ENOMEM; 687 ret = -ENOMEM;
774 goto out; 688 goto out;
775 } 689 }
776
777 if (kmap_ops) {
778 ret = m2p_add_override(mfn, pages[i], &kmap_ops[i]);
779 if (ret)
780 goto out;
781 }
782 } 690 }
783 691
784out: 692out:
785 if (lazy)
786 arch_leave_lazy_mmu_mode();
787
788 return ret; 693 return ret;
789} 694}
790EXPORT_SYMBOL_GPL(set_foreign_p2m_mapping); 695EXPORT_SYMBOL_GPL(set_foreign_p2m_mapping);
791 696
792static struct page *m2p_find_override(unsigned long mfn)
793{
794 unsigned long flags;
795 struct list_head *bucket;
796 struct page *p, *ret;
797
798 if (unlikely(!m2p_overrides))
799 return NULL;
800
801 ret = NULL;
802 bucket = &m2p_overrides[mfn_hash(mfn)];
803
804 spin_lock_irqsave(&m2p_override_lock, flags);
805
806 list_for_each_entry(p, bucket, lru) {
807 if (page_private(p) == mfn) {
808 ret = p;
809 break;
810 }
811 }
812
813 spin_unlock_irqrestore(&m2p_override_lock, flags);
814
815 return ret;
816}
817
818static int m2p_remove_override(struct page *page,
819 struct gnttab_map_grant_ref *kmap_op,
820 unsigned long mfn)
821{
822 unsigned long flags;
823 unsigned long pfn;
824 unsigned long uninitialized_var(address);
825 unsigned level;
826 pte_t *ptep = NULL;
827
828 pfn = page_to_pfn(page);
829
830 if (!PageHighMem(page)) {
831 address = (unsigned long)__va(pfn << PAGE_SHIFT);
832 ptep = lookup_address(address, &level);
833
834 if (WARN(ptep == NULL || level != PG_LEVEL_4K,
835 "m2p_remove_override: pfn %lx not mapped", pfn))
836 return -EINVAL;
837 }
838
839 spin_lock_irqsave(&m2p_override_lock, flags);
840 list_del(&page->lru);
841 spin_unlock_irqrestore(&m2p_override_lock, flags);
842
843 if (kmap_op != NULL) {
844 if (!PageHighMem(page)) {
845 struct multicall_space mcs;
846 struct gnttab_unmap_and_replace *unmap_op;
847 struct page *scratch_page = get_balloon_scratch_page();
848 unsigned long scratch_page_address = (unsigned long)
849 __va(page_to_pfn(scratch_page) << PAGE_SHIFT);
850
851 /*
852 * It might be that we queued all the m2p grant table
853 * hypercalls in a multicall, then m2p_remove_override
854 * get called before the multicall has actually been
855 * issued. In this case handle is going to -1 because
856 * it hasn't been modified yet.
857 */
858 if (kmap_op->handle == -1)
859 xen_mc_flush();
860 /*
861 * Now if kmap_op->handle is negative it means that the
862 * hypercall actually returned an error.
863 */
864 if (kmap_op->handle == GNTST_general_error) {
865 pr_warn("m2p_remove_override: pfn %lx mfn %lx, failed to modify kernel mappings",
866 pfn, mfn);
867 put_balloon_scratch_page();
868 return -1;
869 }
870
871 xen_mc_batch();
872
873 mcs = __xen_mc_entry(
874 sizeof(struct gnttab_unmap_and_replace));
875 unmap_op = mcs.args;
876 unmap_op->host_addr = kmap_op->host_addr;
877 unmap_op->new_addr = scratch_page_address;
878 unmap_op->handle = kmap_op->handle;
879
880 MULTI_grant_table_op(mcs.mc,
881 GNTTABOP_unmap_and_replace, unmap_op, 1);
882
883 mcs = __xen_mc_entry(0);
884 MULTI_update_va_mapping(mcs.mc, scratch_page_address,
885 pfn_pte(page_to_pfn(scratch_page),
886 PAGE_KERNEL_RO), 0);
887
888 xen_mc_issue(PARAVIRT_LAZY_MMU);
889
890 kmap_op->host_addr = 0;
891 put_balloon_scratch_page();
892 }
893 }
894
895 /* p2m(m2p(mfn)) == FOREIGN_FRAME(mfn): the mfn is already present
896 * somewhere in this domain, even before being added to the
897 * m2p_override (see comment above in m2p_add_override).
898 * If there are no other entries in the m2p_override corresponding
899 * to this mfn, then remove the FOREIGN_FRAME_BIT from the p2m for
900 * the original pfn (the one shared by the frontend): the backend
901 * cannot do any IO on this page anymore because it has been
902 * unshared. Removing the FOREIGN_FRAME_BIT from the p2m entry of
903 * the original pfn causes mfn_to_pfn(mfn) to return the frontend
904 * pfn again. */
905 mfn &= ~FOREIGN_FRAME_BIT;
906 pfn = mfn_to_pfn_no_overrides(mfn);
907 if (__pfn_to_mfn(pfn) == FOREIGN_FRAME(mfn) &&
908 m2p_find_override(mfn) == NULL)
909 set_phys_to_machine(pfn, mfn);
910
911 return 0;
912}
913
914int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops, 697int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops,
915 struct gnttab_map_grant_ref *kmap_ops, 698 struct gnttab_unmap_grant_ref *kunmap_ops,
916 struct page **pages, unsigned int count) 699 struct page **pages, unsigned int count)
917{ 700{
918 int i, ret = 0; 701 int i, ret = 0;
919 bool lazy = false;
920 702
921 if (xen_feature(XENFEAT_auto_translated_physmap)) 703 if (xen_feature(XENFEAT_auto_translated_physmap))
922 return 0; 704 return 0;
923 705
924 if (kmap_ops &&
925 !in_interrupt() &&
926 paravirt_get_lazy_mode() == PARAVIRT_LAZY_NONE) {
927 arch_enter_lazy_mmu_mode();
928 lazy = true;
929 }
930
931 for (i = 0; i < count; i++) { 706 for (i = 0; i < count; i++) {
932 unsigned long mfn = __pfn_to_mfn(page_to_pfn(pages[i])); 707 unsigned long mfn = __pfn_to_mfn(page_to_pfn(pages[i]));
933 unsigned long pfn = page_to_pfn(pages[i]); 708 unsigned long pfn = page_to_pfn(pages[i]);
@@ -937,36 +712,16 @@ int clear_foreign_p2m_mapping(struct gnttab_unmap_grant_ref *unmap_ops,
937 goto out; 712 goto out;
938 } 713 }
939 714
940 set_page_private(pages[i], INVALID_P2M_ENTRY); 715 set_phys_to_machine(pfn, INVALID_P2M_ENTRY);
941 WARN_ON(!PagePrivate(pages[i]));
942 ClearPagePrivate(pages[i]);
943 set_phys_to_machine(pfn, pages[i]->index);
944
945 if (kmap_ops)
946 ret = m2p_remove_override(pages[i], &kmap_ops[i], mfn);
947 if (ret)
948 goto out;
949 } 716 }
950 717 if (kunmap_ops)
718 ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref,
719 kunmap_ops, count);
951out: 720out:
952 if (lazy)
953 arch_leave_lazy_mmu_mode();
954 return ret; 721 return ret;
955} 722}
956EXPORT_SYMBOL_GPL(clear_foreign_p2m_mapping); 723EXPORT_SYMBOL_GPL(clear_foreign_p2m_mapping);
957 724
958unsigned long m2p_find_override_pfn(unsigned long mfn, unsigned long pfn)
959{
960 struct page *p = m2p_find_override(mfn);
961 unsigned long ret = pfn;
962
963 if (p)
964 ret = page_to_pfn(p);
965
966 return ret;
967}
968EXPORT_SYMBOL_GPL(m2p_find_override_pfn);
969
970#ifdef CONFIG_XEN_DEBUG_FS 725#ifdef CONFIG_XEN_DEBUG_FS
971#include <linux/debugfs.h> 726#include <linux/debugfs.h>
972#include "debugfs.h" 727#include "debugfs.h"
diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c
index 865e56cea7a0..55f388ef481a 100644
--- a/arch/x86/xen/setup.c
+++ b/arch/x86/xen/setup.c
@@ -32,16 +32,6 @@
32#include "p2m.h" 32#include "p2m.h"
33#include "mmu.h" 33#include "mmu.h"
34 34
35/* These are code, but not functions. Defined in entry.S */
36extern const char xen_hypervisor_callback[];
37extern const char xen_failsafe_callback[];
38#ifdef CONFIG_X86_64
39extern asmlinkage void nmi(void);
40#endif
41extern void xen_sysenter_target(void);
42extern void xen_syscall_target(void);
43extern void xen_syscall32_target(void);
44
45/* Amount of extra memory space we add to the e820 ranges */ 35/* Amount of extra memory space we add to the e820 ranges */
46struct xen_memory_region xen_extra_mem[XEN_EXTRA_MEM_MAX_REGIONS] __initdata; 36struct xen_memory_region xen_extra_mem[XEN_EXTRA_MEM_MAX_REGIONS] __initdata;
47 37
@@ -74,7 +64,7 @@ static unsigned long xen_remap_mfn __initdata = INVALID_P2M_ENTRY;
74 */ 64 */
75#define EXTRA_MEM_RATIO (10) 65#define EXTRA_MEM_RATIO (10)
76 66
77static void __init xen_add_extra_mem(u64 start, u64 size) 67static void __init xen_add_extra_mem(phys_addr_t start, phys_addr_t size)
78{ 68{
79 int i; 69 int i;
80 70
@@ -97,10 +87,10 @@ static void __init xen_add_extra_mem(u64 start, u64 size)
97 memblock_reserve(start, size); 87 memblock_reserve(start, size);
98} 88}
99 89
100static void __init xen_del_extra_mem(u64 start, u64 size) 90static void __init xen_del_extra_mem(phys_addr_t start, phys_addr_t size)
101{ 91{
102 int i; 92 int i;
103 u64 start_r, size_r; 93 phys_addr_t start_r, size_r;
104 94
105 for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) { 95 for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++) {
106 start_r = xen_extra_mem[i].start; 96 start_r = xen_extra_mem[i].start;
@@ -267,7 +257,7 @@ static void __init xen_set_identity_and_release_chunk(unsigned long start_pfn,
267static void __init xen_update_mem_tables(unsigned long pfn, unsigned long mfn) 257static void __init xen_update_mem_tables(unsigned long pfn, unsigned long mfn)
268{ 258{
269 struct mmu_update update = { 259 struct mmu_update update = {
270 .ptr = ((unsigned long long)mfn << PAGE_SHIFT) | MMU_MACHPHYS_UPDATE, 260 .ptr = ((uint64_t)mfn << PAGE_SHIFT) | MMU_MACHPHYS_UPDATE,
271 .val = pfn 261 .val = pfn
272 }; 262 };
273 263
@@ -545,20 +535,21 @@ static unsigned long __init xen_get_max_pages(void)
545 return min(max_pages, MAX_DOMAIN_PAGES); 535 return min(max_pages, MAX_DOMAIN_PAGES);
546} 536}
547 537
548static void xen_align_and_add_e820_region(u64 start, u64 size, int type) 538static void __init xen_align_and_add_e820_region(phys_addr_t start,
539 phys_addr_t size, int type)
549{ 540{
550 u64 end = start + size; 541 phys_addr_t end = start + size;
551 542
552 /* Align RAM regions to page boundaries. */ 543 /* Align RAM regions to page boundaries. */
553 if (type == E820_RAM) { 544 if (type == E820_RAM) {
554 start = PAGE_ALIGN(start); 545 start = PAGE_ALIGN(start);
555 end &= ~((u64)PAGE_SIZE - 1); 546 end &= ~((phys_addr_t)PAGE_SIZE - 1);
556 } 547 }
557 548
558 e820_add_region(start, end - start, type); 549 e820_add_region(start, end - start, type);
559} 550}
560 551
561void xen_ignore_unusable(struct e820entry *list, size_t map_size) 552static void __init xen_ignore_unusable(struct e820entry *list, size_t map_size)
562{ 553{
563 struct e820entry *entry; 554 struct e820entry *entry;
564 unsigned int i; 555 unsigned int i;
@@ -577,7 +568,7 @@ char * __init xen_memory_setup(void)
577 static struct e820entry map[E820MAX] __initdata; 568 static struct e820entry map[E820MAX] __initdata;
578 569
579 unsigned long max_pfn = xen_start_info->nr_pages; 570 unsigned long max_pfn = xen_start_info->nr_pages;
580 unsigned long long mem_end; 571 phys_addr_t mem_end;
581 int rc; 572 int rc;
582 struct xen_memory_map memmap; 573 struct xen_memory_map memmap;
583 unsigned long max_pages; 574 unsigned long max_pages;
@@ -652,16 +643,16 @@ char * __init xen_memory_setup(void)
652 extra_pages); 643 extra_pages);
653 i = 0; 644 i = 0;
654 while (i < memmap.nr_entries) { 645 while (i < memmap.nr_entries) {
655 u64 addr = map[i].addr; 646 phys_addr_t addr = map[i].addr;
656 u64 size = map[i].size; 647 phys_addr_t size = map[i].size;
657 u32 type = map[i].type; 648 u32 type = map[i].type;
658 649
659 if (type == E820_RAM) { 650 if (type == E820_RAM) {
660 if (addr < mem_end) { 651 if (addr < mem_end) {
661 size = min(size, mem_end - addr); 652 size = min(size, mem_end - addr);
662 } else if (extra_pages) { 653 } else if (extra_pages) {
663 size = min(size, (u64)extra_pages * PAGE_SIZE); 654 size = min(size, PFN_PHYS(extra_pages));
664 extra_pages -= size / PAGE_SIZE; 655 extra_pages -= PFN_DOWN(size);
665 xen_add_extra_mem(addr, size); 656 xen_add_extra_mem(addr, size);
666 xen_max_p2m_pfn = PFN_DOWN(addr + size); 657 xen_max_p2m_pfn = PFN_DOWN(addr + size);
667 } else 658 } else
diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c
index 4c071aeb8417..08e8489c47f1 100644
--- a/arch/x86/xen/smp.c
+++ b/arch/x86/xen/smp.c
@@ -507,7 +507,7 @@ static int xen_cpu_disable(void)
507static void xen_cpu_die(unsigned int cpu) 507static void xen_cpu_die(unsigned int cpu)
508{ 508{
509 while (xen_pv_domain() && HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL)) { 509 while (xen_pv_domain() && HYPERVISOR_vcpu_op(VCPUOP_is_up, cpu, NULL)) {
510 current->state = TASK_UNINTERRUPTIBLE; 510 __set_current_state(TASK_UNINTERRUPTIBLE);
511 schedule_timeout(HZ/10); 511 schedule_timeout(HZ/10);
512 } 512 }
513 513
diff --git a/arch/x86/xen/time.c b/arch/x86/xen/time.c
index 69087341d9ae..55da33b1d51c 100644
--- a/arch/x86/xen/time.c
+++ b/arch/x86/xen/time.c
@@ -479,6 +479,10 @@ static void __init xen_time_init(void)
479 int cpu = smp_processor_id(); 479 int cpu = smp_processor_id();
480 struct timespec tp; 480 struct timespec tp;
481 481
482 /* As Dom0 is never moved, no penalty on using TSC there */
483 if (xen_initial_domain())
484 xen_clocksource.rating = 275;
485
482 clocksource_register_hz(&xen_clocksource, NSEC_PER_SEC); 486 clocksource_register_hz(&xen_clocksource, NSEC_PER_SEC);
483 487
484 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL) == 0) { 488 if (HYPERVISOR_vcpu_op(VCPUOP_stop_periodic_timer, cpu, NULL) == 0) {
diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h
index 5686bd9d58cc..9e195c683549 100644
--- a/arch/x86/xen/xen-ops.h
+++ b/arch/x86/xen/xen-ops.h
@@ -10,6 +10,12 @@
10extern const char xen_hypervisor_callback[]; 10extern const char xen_hypervisor_callback[];
11extern const char xen_failsafe_callback[]; 11extern const char xen_failsafe_callback[];
12 12
13void xen_sysenter_target(void);
14#ifdef CONFIG_X86_64
15void xen_syscall_target(void);
16void xen_syscall32_target(void);
17#endif
18
13extern void *xen_initial_gdt; 19extern void *xen_initial_gdt;
14 20
15struct trap_info; 21struct trap_info;
diff --git a/drivers/block/xen-blkback/blkback.c b/drivers/block/xen-blkback/blkback.c
index 63fc7f06a014..2a04d341e598 100644
--- a/drivers/block/xen-blkback/blkback.c
+++ b/drivers/block/xen-blkback/blkback.c
@@ -47,6 +47,7 @@
47#include <asm/xen/hypervisor.h> 47#include <asm/xen/hypervisor.h>
48#include <asm/xen/hypercall.h> 48#include <asm/xen/hypercall.h>
49#include <xen/balloon.h> 49#include <xen/balloon.h>
50#include <xen/grant_table.h>
50#include "common.h" 51#include "common.h"
51 52
52/* 53/*
@@ -100,7 +101,7 @@ module_param(log_stats, int, 0644);
100 101
101#define BLKBACK_INVALID_HANDLE (~0) 102#define BLKBACK_INVALID_HANDLE (~0)
102 103
103/* Number of free pages to remove on each call to free_xenballooned_pages */ 104/* Number of free pages to remove on each call to gnttab_free_pages */
104#define NUM_BATCH_FREE_PAGES 10 105#define NUM_BATCH_FREE_PAGES 10
105 106
106static inline int get_free_page(struct xen_blkif *blkif, struct page **page) 107static inline int get_free_page(struct xen_blkif *blkif, struct page **page)
@@ -111,7 +112,7 @@ static inline int get_free_page(struct xen_blkif *blkif, struct page **page)
111 if (list_empty(&blkif->free_pages)) { 112 if (list_empty(&blkif->free_pages)) {
112 BUG_ON(blkif->free_pages_num != 0); 113 BUG_ON(blkif->free_pages_num != 0);
113 spin_unlock_irqrestore(&blkif->free_pages_lock, flags); 114 spin_unlock_irqrestore(&blkif->free_pages_lock, flags);
114 return alloc_xenballooned_pages(1, page, false); 115 return gnttab_alloc_pages(1, page);
115 } 116 }
116 BUG_ON(blkif->free_pages_num == 0); 117 BUG_ON(blkif->free_pages_num == 0);
117 page[0] = list_first_entry(&blkif->free_pages, struct page, lru); 118 page[0] = list_first_entry(&blkif->free_pages, struct page, lru);
@@ -151,14 +152,14 @@ static inline void shrink_free_pagepool(struct xen_blkif *blkif, int num)
151 blkif->free_pages_num--; 152 blkif->free_pages_num--;
152 if (++num_pages == NUM_BATCH_FREE_PAGES) { 153 if (++num_pages == NUM_BATCH_FREE_PAGES) {
153 spin_unlock_irqrestore(&blkif->free_pages_lock, flags); 154 spin_unlock_irqrestore(&blkif->free_pages_lock, flags);
154 free_xenballooned_pages(num_pages, page); 155 gnttab_free_pages(num_pages, page);
155 spin_lock_irqsave(&blkif->free_pages_lock, flags); 156 spin_lock_irqsave(&blkif->free_pages_lock, flags);
156 num_pages = 0; 157 num_pages = 0;
157 } 158 }
158 } 159 }
159 spin_unlock_irqrestore(&blkif->free_pages_lock, flags); 160 spin_unlock_irqrestore(&blkif->free_pages_lock, flags);
160 if (num_pages != 0) 161 if (num_pages != 0)
161 free_xenballooned_pages(num_pages, page); 162 gnttab_free_pages(num_pages, page);
162} 163}
163 164
164#define vaddr(page) ((unsigned long)pfn_to_kaddr(page_to_pfn(page))) 165#define vaddr(page) ((unsigned long)pfn_to_kaddr(page_to_pfn(page)))
@@ -262,6 +263,17 @@ static void put_persistent_gnt(struct xen_blkif *blkif,
262 atomic_dec(&blkif->persistent_gnt_in_use); 263 atomic_dec(&blkif->persistent_gnt_in_use);
263} 264}
264 265
266static void free_persistent_gnts_unmap_callback(int result,
267 struct gntab_unmap_queue_data *data)
268{
269 struct completion *c = data->data;
270
271 /* BUG_ON used to reproduce existing behaviour,
272 but is this the best way to deal with this? */
273 BUG_ON(result);
274 complete(c);
275}
276
265static void free_persistent_gnts(struct xen_blkif *blkif, struct rb_root *root, 277static void free_persistent_gnts(struct xen_blkif *blkif, struct rb_root *root,
266 unsigned int num) 278 unsigned int num)
267{ 279{
@@ -269,8 +281,17 @@ static void free_persistent_gnts(struct xen_blkif *blkif, struct rb_root *root,
269 struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST]; 281 struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST];
270 struct persistent_gnt *persistent_gnt; 282 struct persistent_gnt *persistent_gnt;
271 struct rb_node *n; 283 struct rb_node *n;
272 int ret = 0;
273 int segs_to_unmap = 0; 284 int segs_to_unmap = 0;
285 struct gntab_unmap_queue_data unmap_data;
286 struct completion unmap_completion;
287
288 init_completion(&unmap_completion);
289
290 unmap_data.data = &unmap_completion;
291 unmap_data.done = &free_persistent_gnts_unmap_callback;
292 unmap_data.pages = pages;
293 unmap_data.unmap_ops = unmap;
294 unmap_data.kunmap_ops = NULL;
274 295
275 foreach_grant_safe(persistent_gnt, n, root, node) { 296 foreach_grant_safe(persistent_gnt, n, root, node) {
276 BUG_ON(persistent_gnt->handle == 297 BUG_ON(persistent_gnt->handle ==
@@ -285,9 +306,11 @@ static void free_persistent_gnts(struct xen_blkif *blkif, struct rb_root *root,
285 306
286 if (++segs_to_unmap == BLKIF_MAX_SEGMENTS_PER_REQUEST || 307 if (++segs_to_unmap == BLKIF_MAX_SEGMENTS_PER_REQUEST ||
287 !rb_next(&persistent_gnt->node)) { 308 !rb_next(&persistent_gnt->node)) {
288 ret = gnttab_unmap_refs(unmap, NULL, pages, 309
289 segs_to_unmap); 310 unmap_data.count = segs_to_unmap;
290 BUG_ON(ret); 311 gnttab_unmap_refs_async(&unmap_data);
312 wait_for_completion(&unmap_completion);
313
291 put_free_pages(blkif, pages, segs_to_unmap); 314 put_free_pages(blkif, pages, segs_to_unmap);
292 segs_to_unmap = 0; 315 segs_to_unmap = 0;
293 } 316 }
@@ -653,18 +676,14 @@ void xen_blkbk_free_caches(struct xen_blkif *blkif)
653 shrink_free_pagepool(blkif, 0 /* All */); 676 shrink_free_pagepool(blkif, 0 /* All */);
654} 677}
655 678
656/* 679static unsigned int xen_blkbk_unmap_prepare(
657 * Unmap the grant references, and also remove the M2P over-rides 680 struct xen_blkif *blkif,
658 * used in the 'pending_req'. 681 struct grant_page **pages,
659 */ 682 unsigned int num,
660static void xen_blkbk_unmap(struct xen_blkif *blkif, 683 struct gnttab_unmap_grant_ref *unmap_ops,
661 struct grant_page *pages[], 684 struct page **unmap_pages)
662 int num)
663{ 685{
664 struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST];
665 struct page *unmap_pages[BLKIF_MAX_SEGMENTS_PER_REQUEST];
666 unsigned int i, invcount = 0; 686 unsigned int i, invcount = 0;
667 int ret;
668 687
669 for (i = 0; i < num; i++) { 688 for (i = 0; i < num; i++) {
670 if (pages[i]->persistent_gnt != NULL) { 689 if (pages[i]->persistent_gnt != NULL) {
@@ -674,21 +693,95 @@ static void xen_blkbk_unmap(struct xen_blkif *blkif,
674 if (pages[i]->handle == BLKBACK_INVALID_HANDLE) 693 if (pages[i]->handle == BLKBACK_INVALID_HANDLE)
675 continue; 694 continue;
676 unmap_pages[invcount] = pages[i]->page; 695 unmap_pages[invcount] = pages[i]->page;
677 gnttab_set_unmap_op(&unmap[invcount], vaddr(pages[i]->page), 696 gnttab_set_unmap_op(&unmap_ops[invcount], vaddr(pages[i]->page),
678 GNTMAP_host_map, pages[i]->handle); 697 GNTMAP_host_map, pages[i]->handle);
679 pages[i]->handle = BLKBACK_INVALID_HANDLE; 698 pages[i]->handle = BLKBACK_INVALID_HANDLE;
680 if (++invcount == BLKIF_MAX_SEGMENTS_PER_REQUEST) { 699 invcount++;
681 ret = gnttab_unmap_refs(unmap, NULL, unmap_pages, 700 }
682 invcount); 701
702 return invcount;
703}
704
705static void xen_blkbk_unmap_and_respond_callback(int result, struct gntab_unmap_queue_data *data)
706{
707 struct pending_req* pending_req = (struct pending_req*) (data->data);
708 struct xen_blkif *blkif = pending_req->blkif;
709
710 /* BUG_ON used to reproduce existing behaviour,
711 but is this the best way to deal with this? */
712 BUG_ON(result);
713
714 put_free_pages(blkif, data->pages, data->count);
715 make_response(blkif, pending_req->id,
716 pending_req->operation, pending_req->status);
717 free_req(blkif, pending_req);
718 /*
719 * Make sure the request is freed before releasing blkif,
720 * or there could be a race between free_req and the
721 * cleanup done in xen_blkif_free during shutdown.
722 *
723 * NB: The fact that we might try to wake up pending_free_wq
724 * before drain_complete (in case there's a drain going on)
725 * it's not a problem with our current implementation
726 * because we can assure there's no thread waiting on
727 * pending_free_wq if there's a drain going on, but it has
728 * to be taken into account if the current model is changed.
729 */
730 if (atomic_dec_and_test(&blkif->inflight) && atomic_read(&blkif->drain)) {
731 complete(&blkif->drain_complete);
732 }
733 xen_blkif_put(blkif);
734}
735
736static void xen_blkbk_unmap_and_respond(struct pending_req *req)
737{
738 struct gntab_unmap_queue_data* work = &req->gnttab_unmap_data;
739 struct xen_blkif *blkif = req->blkif;
740 struct grant_page **pages = req->segments;
741 unsigned int invcount;
742
743 invcount = xen_blkbk_unmap_prepare(blkif, pages, req->nr_pages,
744 req->unmap, req->unmap_pages);
745
746 work->data = req;
747 work->done = xen_blkbk_unmap_and_respond_callback;
748 work->unmap_ops = req->unmap;
749 work->kunmap_ops = NULL;
750 work->pages = req->unmap_pages;
751 work->count = invcount;
752
753 gnttab_unmap_refs_async(&req->gnttab_unmap_data);
754}
755
756
757/*
758 * Unmap the grant references.
759 *
760 * This could accumulate ops up to the batch size to reduce the number
761 * of hypercalls, but since this is only used in error paths there's
762 * no real need.
763 */
764static void xen_blkbk_unmap(struct xen_blkif *blkif,
765 struct grant_page *pages[],
766 int num)
767{
768 struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST];
769 struct page *unmap_pages[BLKIF_MAX_SEGMENTS_PER_REQUEST];
770 unsigned int invcount = 0;
771 int ret;
772
773 while (num) {
774 unsigned int batch = min(num, BLKIF_MAX_SEGMENTS_PER_REQUEST);
775
776 invcount = xen_blkbk_unmap_prepare(blkif, pages, batch,
777 unmap, unmap_pages);
778 if (invcount) {
779 ret = gnttab_unmap_refs(unmap, NULL, unmap_pages, invcount);
683 BUG_ON(ret); 780 BUG_ON(ret);
684 put_free_pages(blkif, unmap_pages, invcount); 781 put_free_pages(blkif, unmap_pages, invcount);
685 invcount = 0;
686 } 782 }
687 } 783 pages += batch;
688 if (invcount) { 784 num -= batch;
689 ret = gnttab_unmap_refs(unmap, NULL, unmap_pages, invcount);
690 BUG_ON(ret);
691 put_free_pages(blkif, unmap_pages, invcount);
692 } 785 }
693} 786}
694 787
@@ -982,32 +1075,8 @@ static void __end_block_io_op(struct pending_req *pending_req, int error)
982 * the grant references associated with 'request' and provide 1075 * the grant references associated with 'request' and provide
983 * the proper response on the ring. 1076 * the proper response on the ring.
984 */ 1077 */
985 if (atomic_dec_and_test(&pending_req->pendcnt)) { 1078 if (atomic_dec_and_test(&pending_req->pendcnt))
986 struct xen_blkif *blkif = pending_req->blkif; 1079 xen_blkbk_unmap_and_respond(pending_req);
987
988 xen_blkbk_unmap(blkif,
989 pending_req->segments,
990 pending_req->nr_pages);
991 make_response(blkif, pending_req->id,
992 pending_req->operation, pending_req->status);
993 free_req(blkif, pending_req);
994 /*
995 * Make sure the request is freed before releasing blkif,
996 * or there could be a race between free_req and the
997 * cleanup done in xen_blkif_free during shutdown.
998 *
999 * NB: The fact that we might try to wake up pending_free_wq
1000 * before drain_complete (in case there's a drain going on)
1001 * it's not a problem with our current implementation
1002 * because we can assure there's no thread waiting on
1003 * pending_free_wq if there's a drain going on, but it has
1004 * to be taken into account if the current model is changed.
1005 */
1006 if (atomic_dec_and_test(&blkif->inflight) && atomic_read(&blkif->drain)) {
1007 complete(&blkif->drain_complete);
1008 }
1009 xen_blkif_put(blkif);
1010 }
1011} 1080}
1012 1081
1013/* 1082/*
diff --git a/drivers/block/xen-blkback/common.h b/drivers/block/xen-blkback/common.h
index f65b807e3236..cc90a840e616 100644
--- a/drivers/block/xen-blkback/common.h
+++ b/drivers/block/xen-blkback/common.h
@@ -350,6 +350,9 @@ struct pending_req {
350 struct grant_page *indirect_pages[MAX_INDIRECT_PAGES]; 350 struct grant_page *indirect_pages[MAX_INDIRECT_PAGES];
351 struct seg_buf seg[MAX_INDIRECT_SEGMENTS]; 351 struct seg_buf seg[MAX_INDIRECT_SEGMENTS];
352 struct bio *biolist[MAX_INDIRECT_SEGMENTS]; 352 struct bio *biolist[MAX_INDIRECT_SEGMENTS];
353 struct gnttab_unmap_grant_ref unmap[MAX_INDIRECT_SEGMENTS];
354 struct page *unmap_pages[MAX_INDIRECT_SEGMENTS];
355 struct gntab_unmap_queue_data gnttab_unmap_data;
353}; 356};
354 357
355 358
diff --git a/drivers/net/xen-netback/interface.c b/drivers/net/xen-netback/interface.c
index 037f74f0fcf6..12f9e2708afb 100644
--- a/drivers/net/xen-netback/interface.c
+++ b/drivers/net/xen-netback/interface.c
@@ -483,9 +483,8 @@ int xenvif_init_queue(struct xenvif_queue *queue)
483 * better enable it. The long term solution would be to use just a 483 * better enable it. The long term solution would be to use just a
484 * bunch of valid page descriptors, without dependency on ballooning 484 * bunch of valid page descriptors, without dependency on ballooning
485 */ 485 */
486 err = alloc_xenballooned_pages(MAX_PENDING_REQS, 486 err = gnttab_alloc_pages(MAX_PENDING_REQS,
487 queue->mmap_pages, 487 queue->mmap_pages);
488 false);
489 if (err) { 488 if (err) {
490 netdev_err(queue->vif->dev, "Could not reserve mmap_pages\n"); 489 netdev_err(queue->vif->dev, "Could not reserve mmap_pages\n");
491 return -ENOMEM; 490 return -ENOMEM;
@@ -664,7 +663,7 @@ void xenvif_disconnect(struct xenvif *vif)
664 */ 663 */
665void xenvif_deinit_queue(struct xenvif_queue *queue) 664void xenvif_deinit_queue(struct xenvif_queue *queue)
666{ 665{
667 free_xenballooned_pages(MAX_PENDING_REQS, queue->mmap_pages); 666 gnttab_free_pages(MAX_PENDING_REQS, queue->mmap_pages);
668} 667}
669 668
670void xenvif_free(struct xenvif *vif) 669void xenvif_free(struct xenvif *vif)
diff --git a/drivers/net/xen-netback/netback.c b/drivers/net/xen-netback/netback.c
index c8ce701a7efb..7dc2d64db3cb 100644
--- a/drivers/net/xen-netback/netback.c
+++ b/drivers/net/xen-netback/netback.c
@@ -314,9 +314,7 @@ static struct xenvif_rx_meta *get_next_rx_buffer(struct xenvif_queue *queue,
314static void xenvif_gop_frag_copy(struct xenvif_queue *queue, struct sk_buff *skb, 314static void xenvif_gop_frag_copy(struct xenvif_queue *queue, struct sk_buff *skb,
315 struct netrx_pending_operations *npo, 315 struct netrx_pending_operations *npo,
316 struct page *page, unsigned long size, 316 struct page *page, unsigned long size,
317 unsigned long offset, int *head, 317 unsigned long offset, int *head)
318 struct xenvif_queue *foreign_queue,
319 grant_ref_t foreign_gref)
320{ 318{
321 struct gnttab_copy *copy_gop; 319 struct gnttab_copy *copy_gop;
322 struct xenvif_rx_meta *meta; 320 struct xenvif_rx_meta *meta;
@@ -333,6 +331,8 @@ static void xenvif_gop_frag_copy(struct xenvif_queue *queue, struct sk_buff *skb
333 offset &= ~PAGE_MASK; 331 offset &= ~PAGE_MASK;
334 332
335 while (size > 0) { 333 while (size > 0) {
334 struct xen_page_foreign *foreign;
335
336 BUG_ON(offset >= PAGE_SIZE); 336 BUG_ON(offset >= PAGE_SIZE);
337 BUG_ON(npo->copy_off > MAX_BUFFER_OFFSET); 337 BUG_ON(npo->copy_off > MAX_BUFFER_OFFSET);
338 338
@@ -361,9 +361,10 @@ static void xenvif_gop_frag_copy(struct xenvif_queue *queue, struct sk_buff *skb
361 copy_gop->flags = GNTCOPY_dest_gref; 361 copy_gop->flags = GNTCOPY_dest_gref;
362 copy_gop->len = bytes; 362 copy_gop->len = bytes;
363 363
364 if (foreign_queue) { 364 foreign = xen_page_foreign(page);
365 copy_gop->source.domid = foreign_queue->vif->domid; 365 if (foreign) {
366 copy_gop->source.u.ref = foreign_gref; 366 copy_gop->source.domid = foreign->domid;
367 copy_gop->source.u.ref = foreign->gref;
367 copy_gop->flags |= GNTCOPY_source_gref; 368 copy_gop->flags |= GNTCOPY_source_gref;
368 } else { 369 } else {
369 copy_gop->source.domid = DOMID_SELF; 370 copy_gop->source.domid = DOMID_SELF;
@@ -406,35 +407,6 @@ static void xenvif_gop_frag_copy(struct xenvif_queue *queue, struct sk_buff *skb
406} 407}
407 408
408/* 409/*
409 * Find the grant ref for a given frag in a chain of struct ubuf_info's
410 * skb: the skb itself
411 * i: the frag's number
412 * ubuf: a pointer to an element in the chain. It should not be NULL
413 *
414 * Returns a pointer to the element in the chain where the page were found. If
415 * not found, returns NULL.
416 * See the definition of callback_struct in common.h for more details about
417 * the chain.
418 */
419static const struct ubuf_info *xenvif_find_gref(const struct sk_buff *const skb,
420 const int i,
421 const struct ubuf_info *ubuf)
422{
423 struct xenvif_queue *foreign_queue = ubuf_to_queue(ubuf);
424
425 do {
426 u16 pending_idx = ubuf->desc;
427
428 if (skb_shinfo(skb)->frags[i].page.p ==
429 foreign_queue->mmap_pages[pending_idx])
430 break;
431 ubuf = (struct ubuf_info *) ubuf->ctx;
432 } while (ubuf);
433
434 return ubuf;
435}
436
437/*
438 * Prepare an SKB to be transmitted to the frontend. 410 * Prepare an SKB to be transmitted to the frontend.
439 * 411 *
440 * This function is responsible for allocating grant operations, meta 412 * This function is responsible for allocating grant operations, meta
@@ -459,8 +431,6 @@ static int xenvif_gop_skb(struct sk_buff *skb,
459 int head = 1; 431 int head = 1;
460 int old_meta_prod; 432 int old_meta_prod;
461 int gso_type; 433 int gso_type;
462 const struct ubuf_info *ubuf = skb_shinfo(skb)->destructor_arg;
463 const struct ubuf_info *const head_ubuf = ubuf;
464 434
465 old_meta_prod = npo->meta_prod; 435 old_meta_prod = npo->meta_prod;
466 436
@@ -507,68 +477,16 @@ static int xenvif_gop_skb(struct sk_buff *skb,
507 len = skb_tail_pointer(skb) - data; 477 len = skb_tail_pointer(skb) - data;
508 478
509 xenvif_gop_frag_copy(queue, skb, npo, 479 xenvif_gop_frag_copy(queue, skb, npo,
510 virt_to_page(data), len, offset, &head, 480 virt_to_page(data), len, offset, &head);
511 NULL,
512 0);
513 data += len; 481 data += len;
514 } 482 }
515 483
516 for (i = 0; i < nr_frags; i++) { 484 for (i = 0; i < nr_frags; i++) {
517 /* This variable also signals whether foreign_gref has a real
518 * value or not.
519 */
520 struct xenvif_queue *foreign_queue = NULL;
521 grant_ref_t foreign_gref;
522
523 if ((skb_shinfo(skb)->tx_flags & SKBTX_DEV_ZEROCOPY) &&
524 (ubuf->callback == &xenvif_zerocopy_callback)) {
525 const struct ubuf_info *const startpoint = ubuf;
526
527 /* Ideally ubuf points to the chain element which
528 * belongs to this frag. Or if frags were removed from
529 * the beginning, then shortly before it.
530 */
531 ubuf = xenvif_find_gref(skb, i, ubuf);
532
533 /* Try again from the beginning of the list, if we
534 * haven't tried from there. This only makes sense in
535 * the unlikely event of reordering the original frags.
536 * For injected local pages it's an unnecessary second
537 * run.
538 */
539 if (unlikely(!ubuf) && startpoint != head_ubuf)
540 ubuf = xenvif_find_gref(skb, i, head_ubuf);
541
542 if (likely(ubuf)) {
543 u16 pending_idx = ubuf->desc;
544
545 foreign_queue = ubuf_to_queue(ubuf);
546 foreign_gref =
547 foreign_queue->pending_tx_info[pending_idx].req.gref;
548 /* Just a safety measure. If this was the last
549 * element on the list, the for loop will
550 * iterate again if a local page were added to
551 * the end. Using head_ubuf here prevents the
552 * second search on the chain. Or the original
553 * frags changed order, but that's less likely.
554 * In any way, ubuf shouldn't be NULL.
555 */
556 ubuf = ubuf->ctx ?
557 (struct ubuf_info *) ubuf->ctx :
558 head_ubuf;
559 } else
560 /* This frag was a local page, added to the
561 * array after the skb left netback.
562 */
563 ubuf = head_ubuf;
564 }
565 xenvif_gop_frag_copy(queue, skb, npo, 485 xenvif_gop_frag_copy(queue, skb, npo,
566 skb_frag_page(&skb_shinfo(skb)->frags[i]), 486 skb_frag_page(&skb_shinfo(skb)->frags[i]),
567 skb_frag_size(&skb_shinfo(skb)->frags[i]), 487 skb_frag_size(&skb_shinfo(skb)->frags[i]),
568 skb_shinfo(skb)->frags[i].page_offset, 488 skb_shinfo(skb)->frags[i].page_offset,
569 &head, 489 &head);
570 foreign_queue,
571 foreign_queue ? foreign_gref : UINT_MAX);
572 } 490 }
573 491
574 return npo->meta_prod - old_meta_prod; 492 return npo->meta_prod - old_meta_prod;
@@ -1241,12 +1159,6 @@ static void xenvif_fill_frags(struct xenvif_queue *queue, struct sk_buff *skb)
1241 /* Take an extra reference to offset network stack's put_page */ 1159 /* Take an extra reference to offset network stack's put_page */
1242 get_page(queue->mmap_pages[pending_idx]); 1160 get_page(queue->mmap_pages[pending_idx]);
1243 } 1161 }
1244 /* FIXME: __skb_fill_page_desc set this to true because page->pfmemalloc
1245 * overlaps with "index", and "mapping" is not set. I think mapping
1246 * should be set. If delivered to local stack, it would drop this
1247 * skb in sk_filter unless the socket has the right to use it.
1248 */
1249 skb->pfmemalloc = false;
1250} 1162}
1251 1163
1252static int xenvif_get_extras(struct xenvif_queue *queue, 1164static int xenvif_get_extras(struct xenvif_queue *queue,
diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c
index 3860d02729dc..0b52d92cb2e5 100644
--- a/drivers/xen/balloon.c
+++ b/drivers/xen/balloon.c
@@ -92,7 +92,6 @@ EXPORT_SYMBOL_GPL(balloon_stats);
92 92
93/* We increase/decrease in batches which fit in a page */ 93/* We increase/decrease in batches which fit in a page */
94static xen_pfn_t frame_list[PAGE_SIZE / sizeof(unsigned long)]; 94static xen_pfn_t frame_list[PAGE_SIZE / sizeof(unsigned long)];
95static DEFINE_PER_CPU(struct page *, balloon_scratch_page);
96 95
97 96
98/* List of ballooned pages, threaded through the mem_map array. */ 97/* List of ballooned pages, threaded through the mem_map array. */
@@ -423,22 +422,12 @@ static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp)
423 page = pfn_to_page(pfn); 422 page = pfn_to_page(pfn);
424 423
425#ifdef CONFIG_XEN_HAVE_PVMMU 424#ifdef CONFIG_XEN_HAVE_PVMMU
426 /*
427 * Ballooned out frames are effectively replaced with
428 * a scratch frame. Ensure direct mappings and the
429 * p2m are consistent.
430 */
431 if (!xen_feature(XENFEAT_auto_translated_physmap)) { 425 if (!xen_feature(XENFEAT_auto_translated_physmap)) {
432 if (!PageHighMem(page)) { 426 if (!PageHighMem(page)) {
433 struct page *scratch_page = get_balloon_scratch_page();
434
435 ret = HYPERVISOR_update_va_mapping( 427 ret = HYPERVISOR_update_va_mapping(
436 (unsigned long)__va(pfn << PAGE_SHIFT), 428 (unsigned long)__va(pfn << PAGE_SHIFT),
437 pfn_pte(page_to_pfn(scratch_page), 429 __pte_ma(0), 0);
438 PAGE_KERNEL_RO), 0);
439 BUG_ON(ret); 430 BUG_ON(ret);
440
441 put_balloon_scratch_page();
442 } 431 }
443 __set_phys_to_machine(pfn, INVALID_P2M_ENTRY); 432 __set_phys_to_machine(pfn, INVALID_P2M_ENTRY);
444 } 433 }
@@ -500,18 +489,6 @@ static void balloon_process(struct work_struct *work)
500 mutex_unlock(&balloon_mutex); 489 mutex_unlock(&balloon_mutex);
501} 490}
502 491
503struct page *get_balloon_scratch_page(void)
504{
505 struct page *ret = get_cpu_var(balloon_scratch_page);
506 BUG_ON(ret == NULL);
507 return ret;
508}
509
510void put_balloon_scratch_page(void)
511{
512 put_cpu_var(balloon_scratch_page);
513}
514
515/* Resets the Xen limit, sets new target, and kicks off processing. */ 492/* Resets the Xen limit, sets new target, and kicks off processing. */
516void balloon_set_new_target(unsigned long target) 493void balloon_set_new_target(unsigned long target)
517{ 494{
@@ -605,61 +582,13 @@ static void __init balloon_add_region(unsigned long start_pfn,
605 } 582 }
606} 583}
607 584
608static int alloc_balloon_scratch_page(int cpu)
609{
610 if (per_cpu(balloon_scratch_page, cpu) != NULL)
611 return 0;
612
613 per_cpu(balloon_scratch_page, cpu) = alloc_page(GFP_KERNEL);
614 if (per_cpu(balloon_scratch_page, cpu) == NULL) {
615 pr_warn("Failed to allocate balloon_scratch_page for cpu %d\n", cpu);
616 return -ENOMEM;
617 }
618
619 return 0;
620}
621
622
623static int balloon_cpu_notify(struct notifier_block *self,
624 unsigned long action, void *hcpu)
625{
626 int cpu = (long)hcpu;
627 switch (action) {
628 case CPU_UP_PREPARE:
629 if (alloc_balloon_scratch_page(cpu))
630 return NOTIFY_BAD;
631 break;
632 default:
633 break;
634 }
635 return NOTIFY_OK;
636}
637
638static struct notifier_block balloon_cpu_notifier = {
639 .notifier_call = balloon_cpu_notify,
640};
641
642static int __init balloon_init(void) 585static int __init balloon_init(void)
643{ 586{
644 int i, cpu; 587 int i;
645 588
646 if (!xen_domain()) 589 if (!xen_domain())
647 return -ENODEV; 590 return -ENODEV;
648 591
649 if (!xen_feature(XENFEAT_auto_translated_physmap)) {
650 register_cpu_notifier(&balloon_cpu_notifier);
651
652 get_online_cpus();
653 for_each_online_cpu(cpu) {
654 if (alloc_balloon_scratch_page(cpu)) {
655 put_online_cpus();
656 unregister_cpu_notifier(&balloon_cpu_notifier);
657 return -ENOMEM;
658 }
659 }
660 put_online_cpus();
661 }
662
663 pr_info("Initialising balloon driver\n"); 592 pr_info("Initialising balloon driver\n");
664 593
665 balloon_stats.current_pages = xen_pv_domain() 594 balloon_stats.current_pages = xen_pv_domain()
@@ -696,15 +625,4 @@ static int __init balloon_init(void)
696 625
697subsys_initcall(balloon_init); 626subsys_initcall(balloon_init);
698 627
699static int __init balloon_clear(void)
700{
701 int cpu;
702
703 for_each_possible_cpu(cpu)
704 per_cpu(balloon_scratch_page, cpu) = NULL;
705
706 return 0;
707}
708early_initcall(balloon_clear);
709
710MODULE_LICENSE("GPL"); 628MODULE_LICENSE("GPL");
diff --git a/drivers/xen/gntdev.c b/drivers/xen/gntdev.c
index 073b4a19a8b0..d5bb1a33d0a3 100644
--- a/drivers/xen/gntdev.c
+++ b/drivers/xen/gntdev.c
@@ -67,7 +67,7 @@ struct gntdev_priv {
67 * Only populated if populate_freeable_maps == 1 */ 67 * Only populated if populate_freeable_maps == 1 */
68 struct list_head freeable_maps; 68 struct list_head freeable_maps;
69 /* lock protects maps and freeable_maps */ 69 /* lock protects maps and freeable_maps */
70 spinlock_t lock; 70 struct mutex lock;
71 struct mm_struct *mm; 71 struct mm_struct *mm;
72 struct mmu_notifier mn; 72 struct mmu_notifier mn;
73}; 73};
@@ -91,7 +91,9 @@ struct grant_map {
91 struct gnttab_map_grant_ref *map_ops; 91 struct gnttab_map_grant_ref *map_ops;
92 struct gnttab_unmap_grant_ref *unmap_ops; 92 struct gnttab_unmap_grant_ref *unmap_ops;
93 struct gnttab_map_grant_ref *kmap_ops; 93 struct gnttab_map_grant_ref *kmap_ops;
94 struct gnttab_unmap_grant_ref *kunmap_ops;
94 struct page **pages; 95 struct page **pages;
96 unsigned long pages_vm_start;
95}; 97};
96 98
97static int unmap_grant_pages(struct grant_map *map, int offset, int pages); 99static int unmap_grant_pages(struct grant_map *map, int offset, int pages);
@@ -118,12 +120,13 @@ static void gntdev_free_map(struct grant_map *map)
118 return; 120 return;
119 121
120 if (map->pages) 122 if (map->pages)
121 free_xenballooned_pages(map->count, map->pages); 123 gnttab_free_pages(map->count, map->pages);
122 kfree(map->pages); 124 kfree(map->pages);
123 kfree(map->grants); 125 kfree(map->grants);
124 kfree(map->map_ops); 126 kfree(map->map_ops);
125 kfree(map->unmap_ops); 127 kfree(map->unmap_ops);
126 kfree(map->kmap_ops); 128 kfree(map->kmap_ops);
129 kfree(map->kunmap_ops);
127 kfree(map); 130 kfree(map);
128} 131}
129 132
@@ -140,21 +143,24 @@ static struct grant_map *gntdev_alloc_map(struct gntdev_priv *priv, int count)
140 add->map_ops = kcalloc(count, sizeof(add->map_ops[0]), GFP_KERNEL); 143 add->map_ops = kcalloc(count, sizeof(add->map_ops[0]), GFP_KERNEL);
141 add->unmap_ops = kcalloc(count, sizeof(add->unmap_ops[0]), GFP_KERNEL); 144 add->unmap_ops = kcalloc(count, sizeof(add->unmap_ops[0]), GFP_KERNEL);
142 add->kmap_ops = kcalloc(count, sizeof(add->kmap_ops[0]), GFP_KERNEL); 145 add->kmap_ops = kcalloc(count, sizeof(add->kmap_ops[0]), GFP_KERNEL);
146 add->kunmap_ops = kcalloc(count, sizeof(add->kunmap_ops[0]), GFP_KERNEL);
143 add->pages = kcalloc(count, sizeof(add->pages[0]), GFP_KERNEL); 147 add->pages = kcalloc(count, sizeof(add->pages[0]), GFP_KERNEL);
144 if (NULL == add->grants || 148 if (NULL == add->grants ||
145 NULL == add->map_ops || 149 NULL == add->map_ops ||
146 NULL == add->unmap_ops || 150 NULL == add->unmap_ops ||
147 NULL == add->kmap_ops || 151 NULL == add->kmap_ops ||
152 NULL == add->kunmap_ops ||
148 NULL == add->pages) 153 NULL == add->pages)
149 goto err; 154 goto err;
150 155
151 if (alloc_xenballooned_pages(count, add->pages, false /* lowmem */)) 156 if (gnttab_alloc_pages(count, add->pages))
152 goto err; 157 goto err;
153 158
154 for (i = 0; i < count; i++) { 159 for (i = 0; i < count; i++) {
155 add->map_ops[i].handle = -1; 160 add->map_ops[i].handle = -1;
156 add->unmap_ops[i].handle = -1; 161 add->unmap_ops[i].handle = -1;
157 add->kmap_ops[i].handle = -1; 162 add->kmap_ops[i].handle = -1;
163 add->kunmap_ops[i].handle = -1;
158 } 164 }
159 165
160 add->index = 0; 166 add->index = 0;
@@ -216,9 +222,9 @@ static void gntdev_put_map(struct gntdev_priv *priv, struct grant_map *map)
216 } 222 }
217 223
218 if (populate_freeable_maps && priv) { 224 if (populate_freeable_maps && priv) {
219 spin_lock(&priv->lock); 225 mutex_lock(&priv->lock);
220 list_del(&map->next); 226 list_del(&map->next);
221 spin_unlock(&priv->lock); 227 mutex_unlock(&priv->lock);
222 } 228 }
223 229
224 if (map->pages && !use_ptemod) 230 if (map->pages && !use_ptemod)
@@ -239,6 +245,14 @@ static int find_grant_ptes(pte_t *pte, pgtable_t token,
239 BUG_ON(pgnr >= map->count); 245 BUG_ON(pgnr >= map->count);
240 pte_maddr = arbitrary_virt_to_machine(pte).maddr; 246 pte_maddr = arbitrary_virt_to_machine(pte).maddr;
241 247
248 /*
249 * Set the PTE as special to force get_user_pages_fast() fall
250 * back to the slow path. If this is not supported as part of
251 * the grant map, it will be done afterwards.
252 */
253 if (xen_feature(XENFEAT_gnttab_map_avail_bits))
254 flags |= (1 << _GNTMAP_guest_avail0);
255
242 gnttab_set_map_op(&map->map_ops[pgnr], pte_maddr, flags, 256 gnttab_set_map_op(&map->map_ops[pgnr], pte_maddr, flags,
243 map->grants[pgnr].ref, 257 map->grants[pgnr].ref,
244 map->grants[pgnr].domid); 258 map->grants[pgnr].domid);
@@ -247,6 +261,15 @@ static int find_grant_ptes(pte_t *pte, pgtable_t token,
247 return 0; 261 return 0;
248} 262}
249 263
264#ifdef CONFIG_X86
265static int set_grant_ptes_as_special(pte_t *pte, pgtable_t token,
266 unsigned long addr, void *data)
267{
268 set_pte_at(current->mm, addr, pte, pte_mkspecial(*pte));
269 return 0;
270}
271#endif
272
250static int map_grant_pages(struct grant_map *map) 273static int map_grant_pages(struct grant_map *map)
251{ 274{
252 int i, err = 0; 275 int i, err = 0;
@@ -280,6 +303,8 @@ static int map_grant_pages(struct grant_map *map)
280 map->flags | GNTMAP_host_map, 303 map->flags | GNTMAP_host_map,
281 map->grants[i].ref, 304 map->grants[i].ref,
282 map->grants[i].domid); 305 map->grants[i].domid);
306 gnttab_set_unmap_op(&map->kunmap_ops[i], address,
307 map->flags | GNTMAP_host_map, -1);
283 } 308 }
284 } 309 }
285 310
@@ -290,20 +315,42 @@ static int map_grant_pages(struct grant_map *map)
290 return err; 315 return err;
291 316
292 for (i = 0; i < map->count; i++) { 317 for (i = 0; i < map->count; i++) {
293 if (map->map_ops[i].status) 318 if (map->map_ops[i].status) {
294 err = -EINVAL; 319 err = -EINVAL;
295 else { 320 continue;
296 BUG_ON(map->map_ops[i].handle == -1);
297 map->unmap_ops[i].handle = map->map_ops[i].handle;
298 pr_debug("map handle=%d\n", map->map_ops[i].handle);
299 } 321 }
322
323 map->unmap_ops[i].handle = map->map_ops[i].handle;
324 if (use_ptemod)
325 map->kunmap_ops[i].handle = map->kmap_ops[i].handle;
300 } 326 }
301 return err; 327 return err;
302} 328}
303 329
330struct unmap_grant_pages_callback_data
331{
332 struct completion completion;
333 int result;
334};
335
336static void unmap_grant_callback(int result,
337 struct gntab_unmap_queue_data *data)
338{
339 struct unmap_grant_pages_callback_data* d = data->data;
340
341 d->result = result;
342 complete(&d->completion);
343}
344
304static int __unmap_grant_pages(struct grant_map *map, int offset, int pages) 345static int __unmap_grant_pages(struct grant_map *map, int offset, int pages)
305{ 346{
306 int i, err = 0; 347 int i, err = 0;
348 struct gntab_unmap_queue_data unmap_data;
349 struct unmap_grant_pages_callback_data data;
350
351 init_completion(&data.completion);
352 unmap_data.data = &data;
353 unmap_data.done= &unmap_grant_callback;
307 354
308 if (map->notify.flags & UNMAP_NOTIFY_CLEAR_BYTE) { 355 if (map->notify.flags & UNMAP_NOTIFY_CLEAR_BYTE) {
309 int pgno = (map->notify.addr >> PAGE_SHIFT); 356 int pgno = (map->notify.addr >> PAGE_SHIFT);
@@ -315,11 +362,16 @@ static int __unmap_grant_pages(struct grant_map *map, int offset, int pages)
315 } 362 }
316 } 363 }
317 364
318 err = gnttab_unmap_refs(map->unmap_ops + offset, 365 unmap_data.unmap_ops = map->unmap_ops + offset;
319 use_ptemod ? map->kmap_ops + offset : NULL, map->pages + offset, 366 unmap_data.kunmap_ops = use_ptemod ? map->kunmap_ops + offset : NULL;
320 pages); 367 unmap_data.pages = map->pages + offset;
321 if (err) 368 unmap_data.count = pages;
322 return err; 369
370 gnttab_unmap_refs_async(&unmap_data);
371
372 wait_for_completion(&data.completion);
373 if (data.result)
374 return data.result;
323 375
324 for (i = 0; i < pages; i++) { 376 for (i = 0; i < pages; i++) {
325 if (map->unmap_ops[offset+i].status) 377 if (map->unmap_ops[offset+i].status)
@@ -387,17 +439,26 @@ static void gntdev_vma_close(struct vm_area_struct *vma)
387 * not do any unmapping, since that has been done prior to 439 * not do any unmapping, since that has been done prior to
388 * closing the vma, but it may still iterate the unmap_ops list. 440 * closing the vma, but it may still iterate the unmap_ops list.
389 */ 441 */
390 spin_lock(&priv->lock); 442 mutex_lock(&priv->lock);
391 map->vma = NULL; 443 map->vma = NULL;
392 spin_unlock(&priv->lock); 444 mutex_unlock(&priv->lock);
393 } 445 }
394 vma->vm_private_data = NULL; 446 vma->vm_private_data = NULL;
395 gntdev_put_map(priv, map); 447 gntdev_put_map(priv, map);
396} 448}
397 449
450static struct page *gntdev_vma_find_special_page(struct vm_area_struct *vma,
451 unsigned long addr)
452{
453 struct grant_map *map = vma->vm_private_data;
454
455 return map->pages[(addr - map->pages_vm_start) >> PAGE_SHIFT];
456}
457
398static struct vm_operations_struct gntdev_vmops = { 458static struct vm_operations_struct gntdev_vmops = {
399 .open = gntdev_vma_open, 459 .open = gntdev_vma_open,
400 .close = gntdev_vma_close, 460 .close = gntdev_vma_close,
461 .find_special_page = gntdev_vma_find_special_page,
401}; 462};
402 463
403/* ------------------------------------------------------------------ */ 464/* ------------------------------------------------------------------ */
@@ -433,14 +494,14 @@ static void mn_invl_range_start(struct mmu_notifier *mn,
433 struct gntdev_priv *priv = container_of(mn, struct gntdev_priv, mn); 494 struct gntdev_priv *priv = container_of(mn, struct gntdev_priv, mn);
434 struct grant_map *map; 495 struct grant_map *map;
435 496
436 spin_lock(&priv->lock); 497 mutex_lock(&priv->lock);
437 list_for_each_entry(map, &priv->maps, next) { 498 list_for_each_entry(map, &priv->maps, next) {
438 unmap_if_in_range(map, start, end); 499 unmap_if_in_range(map, start, end);
439 } 500 }
440 list_for_each_entry(map, &priv->freeable_maps, next) { 501 list_for_each_entry(map, &priv->freeable_maps, next) {
441 unmap_if_in_range(map, start, end); 502 unmap_if_in_range(map, start, end);
442 } 503 }
443 spin_unlock(&priv->lock); 504 mutex_unlock(&priv->lock);
444} 505}
445 506
446static void mn_invl_page(struct mmu_notifier *mn, 507static void mn_invl_page(struct mmu_notifier *mn,
@@ -457,7 +518,7 @@ static void mn_release(struct mmu_notifier *mn,
457 struct grant_map *map; 518 struct grant_map *map;
458 int err; 519 int err;
459 520
460 spin_lock(&priv->lock); 521 mutex_lock(&priv->lock);
461 list_for_each_entry(map, &priv->maps, next) { 522 list_for_each_entry(map, &priv->maps, next) {
462 if (!map->vma) 523 if (!map->vma)
463 continue; 524 continue;
@@ -476,7 +537,7 @@ static void mn_release(struct mmu_notifier *mn,
476 err = unmap_grant_pages(map, /* offset */ 0, map->count); 537 err = unmap_grant_pages(map, /* offset */ 0, map->count);
477 WARN_ON(err); 538 WARN_ON(err);
478 } 539 }
479 spin_unlock(&priv->lock); 540 mutex_unlock(&priv->lock);
480} 541}
481 542
482static struct mmu_notifier_ops gntdev_mmu_ops = { 543static struct mmu_notifier_ops gntdev_mmu_ops = {
@@ -498,7 +559,7 @@ static int gntdev_open(struct inode *inode, struct file *flip)
498 559
499 INIT_LIST_HEAD(&priv->maps); 560 INIT_LIST_HEAD(&priv->maps);
500 INIT_LIST_HEAD(&priv->freeable_maps); 561 INIT_LIST_HEAD(&priv->freeable_maps);
501 spin_lock_init(&priv->lock); 562 mutex_init(&priv->lock);
502 563
503 if (use_ptemod) { 564 if (use_ptemod) {
504 priv->mm = get_task_mm(current); 565 priv->mm = get_task_mm(current);
@@ -572,10 +633,10 @@ static long gntdev_ioctl_map_grant_ref(struct gntdev_priv *priv,
572 return -EFAULT; 633 return -EFAULT;
573 } 634 }
574 635
575 spin_lock(&priv->lock); 636 mutex_lock(&priv->lock);
576 gntdev_add_map(priv, map); 637 gntdev_add_map(priv, map);
577 op.index = map->index << PAGE_SHIFT; 638 op.index = map->index << PAGE_SHIFT;
578 spin_unlock(&priv->lock); 639 mutex_unlock(&priv->lock);
579 640
580 if (copy_to_user(u, &op, sizeof(op)) != 0) 641 if (copy_to_user(u, &op, sizeof(op)) != 0)
581 return -EFAULT; 642 return -EFAULT;
@@ -594,7 +655,7 @@ static long gntdev_ioctl_unmap_grant_ref(struct gntdev_priv *priv,
594 return -EFAULT; 655 return -EFAULT;
595 pr_debug("priv %p, del %d+%d\n", priv, (int)op.index, (int)op.count); 656 pr_debug("priv %p, del %d+%d\n", priv, (int)op.index, (int)op.count);
596 657
597 spin_lock(&priv->lock); 658 mutex_lock(&priv->lock);
598 map = gntdev_find_map_index(priv, op.index >> PAGE_SHIFT, op.count); 659 map = gntdev_find_map_index(priv, op.index >> PAGE_SHIFT, op.count);
599 if (map) { 660 if (map) {
600 list_del(&map->next); 661 list_del(&map->next);
@@ -602,7 +663,7 @@ static long gntdev_ioctl_unmap_grant_ref(struct gntdev_priv *priv,
602 list_add_tail(&map->next, &priv->freeable_maps); 663 list_add_tail(&map->next, &priv->freeable_maps);
603 err = 0; 664 err = 0;
604 } 665 }
605 spin_unlock(&priv->lock); 666 mutex_unlock(&priv->lock);
606 if (map) 667 if (map)
607 gntdev_put_map(priv, map); 668 gntdev_put_map(priv, map);
608 return err; 669 return err;
@@ -670,7 +731,7 @@ static long gntdev_ioctl_notify(struct gntdev_priv *priv, void __user *u)
670 out_flags = op.action; 731 out_flags = op.action;
671 out_event = op.event_channel_port; 732 out_event = op.event_channel_port;
672 733
673 spin_lock(&priv->lock); 734 mutex_lock(&priv->lock);
674 735
675 list_for_each_entry(map, &priv->maps, next) { 736 list_for_each_entry(map, &priv->maps, next) {
676 uint64_t begin = map->index << PAGE_SHIFT; 737 uint64_t begin = map->index << PAGE_SHIFT;
@@ -698,7 +759,7 @@ static long gntdev_ioctl_notify(struct gntdev_priv *priv, void __user *u)
698 rc = 0; 759 rc = 0;
699 760
700 unlock_out: 761 unlock_out:
701 spin_unlock(&priv->lock); 762 mutex_unlock(&priv->lock);
702 763
703 /* Drop the reference to the event channel we did not save in the map */ 764 /* Drop the reference to the event channel we did not save in the map */
704 if (out_flags & UNMAP_NOTIFY_SEND_EVENT) 765 if (out_flags & UNMAP_NOTIFY_SEND_EVENT)
@@ -748,7 +809,7 @@ static int gntdev_mmap(struct file *flip, struct vm_area_struct *vma)
748 pr_debug("map %d+%d at %lx (pgoff %lx)\n", 809 pr_debug("map %d+%d at %lx (pgoff %lx)\n",
749 index, count, vma->vm_start, vma->vm_pgoff); 810 index, count, vma->vm_start, vma->vm_pgoff);
750 811
751 spin_lock(&priv->lock); 812 mutex_lock(&priv->lock);
752 map = gntdev_find_map_index(priv, index, count); 813 map = gntdev_find_map_index(priv, index, count);
753 if (!map) 814 if (!map)
754 goto unlock_out; 815 goto unlock_out;
@@ -783,7 +844,7 @@ static int gntdev_mmap(struct file *flip, struct vm_area_struct *vma)
783 map->flags |= GNTMAP_readonly; 844 map->flags |= GNTMAP_readonly;
784 } 845 }
785 846
786 spin_unlock(&priv->lock); 847 mutex_unlock(&priv->lock);
787 848
788 if (use_ptemod) { 849 if (use_ptemod) {
789 err = apply_to_page_range(vma->vm_mm, vma->vm_start, 850 err = apply_to_page_range(vma->vm_mm, vma->vm_start,
@@ -806,16 +867,34 @@ static int gntdev_mmap(struct file *flip, struct vm_area_struct *vma)
806 if (err) 867 if (err)
807 goto out_put_map; 868 goto out_put_map;
808 } 869 }
870 } else {
871#ifdef CONFIG_X86
872 /*
873 * If the PTEs were not made special by the grant map
874 * hypercall, do so here.
875 *
876 * This is racy since the mapping is already visible
877 * to userspace but userspace should be well-behaved
878 * enough to not touch it until the mmap() call
879 * returns.
880 */
881 if (!xen_feature(XENFEAT_gnttab_map_avail_bits)) {
882 apply_to_page_range(vma->vm_mm, vma->vm_start,
883 vma->vm_end - vma->vm_start,
884 set_grant_ptes_as_special, NULL);
885 }
886#endif
887 map->pages_vm_start = vma->vm_start;
809 } 888 }
810 889
811 return 0; 890 return 0;
812 891
813unlock_out: 892unlock_out:
814 spin_unlock(&priv->lock); 893 mutex_unlock(&priv->lock);
815 return err; 894 return err;
816 895
817out_unlock_put: 896out_unlock_put:
818 spin_unlock(&priv->lock); 897 mutex_unlock(&priv->lock);
819out_put_map: 898out_put_map:
820 if (use_ptemod) 899 if (use_ptemod)
821 map->vma = NULL; 900 map->vma = NULL;
diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c
index 7786291ba229..17972fbacddc 100644
--- a/drivers/xen/grant-table.c
+++ b/drivers/xen/grant-table.c
@@ -42,6 +42,7 @@
42#include <linux/io.h> 42#include <linux/io.h>
43#include <linux/delay.h> 43#include <linux/delay.h>
44#include <linux/hardirq.h> 44#include <linux/hardirq.h>
45#include <linux/workqueue.h>
45 46
46#include <xen/xen.h> 47#include <xen/xen.h>
47#include <xen/interface/xen.h> 48#include <xen/interface/xen.h>
@@ -50,6 +51,7 @@
50#include <xen/interface/memory.h> 51#include <xen/interface/memory.h>
51#include <xen/hvc-console.h> 52#include <xen/hvc-console.h>
52#include <xen/swiotlb-xen.h> 53#include <xen/swiotlb-xen.h>
54#include <xen/balloon.h>
53#include <asm/xen/hypercall.h> 55#include <asm/xen/hypercall.h>
54#include <asm/xen/interface.h> 56#include <asm/xen/interface.h>
55 57
@@ -671,6 +673,59 @@ void gnttab_free_auto_xlat_frames(void)
671} 673}
672EXPORT_SYMBOL_GPL(gnttab_free_auto_xlat_frames); 674EXPORT_SYMBOL_GPL(gnttab_free_auto_xlat_frames);
673 675
676/**
677 * gnttab_alloc_pages - alloc pages suitable for grant mapping into
678 * @nr_pages: number of pages to alloc
679 * @pages: returns the pages
680 */
681int gnttab_alloc_pages(int nr_pages, struct page **pages)
682{
683 int i;
684 int ret;
685
686 ret = alloc_xenballooned_pages(nr_pages, pages, false);
687 if (ret < 0)
688 return ret;
689
690 for (i = 0; i < nr_pages; i++) {
691#if BITS_PER_LONG < 64
692 struct xen_page_foreign *foreign;
693
694 foreign = kzalloc(sizeof(*foreign), GFP_KERNEL);
695 if (!foreign) {
696 gnttab_free_pages(nr_pages, pages);
697 return -ENOMEM;
698 }
699 set_page_private(pages[i], (unsigned long)foreign);
700#endif
701 SetPagePrivate(pages[i]);
702 }
703
704 return 0;
705}
706EXPORT_SYMBOL(gnttab_alloc_pages);
707
708/**
709 * gnttab_free_pages - free pages allocated by gnttab_alloc_pages()
710 * @nr_pages; number of pages to free
711 * @pages: the pages
712 */
713void gnttab_free_pages(int nr_pages, struct page **pages)
714{
715 int i;
716
717 for (i = 0; i < nr_pages; i++) {
718 if (PagePrivate(pages[i])) {
719#if BITS_PER_LONG < 64
720 kfree((void *)page_private(pages[i]));
721#endif
722 ClearPagePrivate(pages[i]);
723 }
724 }
725 free_xenballooned_pages(nr_pages, pages);
726}
727EXPORT_SYMBOL(gnttab_free_pages);
728
674/* Handling of paged out grant targets (GNTST_eagain) */ 729/* Handling of paged out grant targets (GNTST_eagain) */
675#define MAX_DELAY 256 730#define MAX_DELAY 256
676static inline void 731static inline void
@@ -727,30 +782,87 @@ int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,
727 if (ret) 782 if (ret)
728 return ret; 783 return ret;
729 784
730 /* Retry eagain maps */ 785 for (i = 0; i < count; i++) {
731 for (i = 0; i < count; i++) 786 /* Retry eagain maps */
732 if (map_ops[i].status == GNTST_eagain) 787 if (map_ops[i].status == GNTST_eagain)
733 gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, map_ops + i, 788 gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, map_ops + i,
734 &map_ops[i].status, __func__); 789 &map_ops[i].status, __func__);
735 790
791 if (map_ops[i].status == GNTST_okay) {
792 struct xen_page_foreign *foreign;
793
794 SetPageForeign(pages[i]);
795 foreign = xen_page_foreign(pages[i]);
796 foreign->domid = map_ops[i].dom;
797 foreign->gref = map_ops[i].ref;
798 }
799 }
800
736 return set_foreign_p2m_mapping(map_ops, kmap_ops, pages, count); 801 return set_foreign_p2m_mapping(map_ops, kmap_ops, pages, count);
737} 802}
738EXPORT_SYMBOL_GPL(gnttab_map_refs); 803EXPORT_SYMBOL_GPL(gnttab_map_refs);
739 804
740int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops, 805int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,
741 struct gnttab_map_grant_ref *kmap_ops, 806 struct gnttab_unmap_grant_ref *kunmap_ops,
742 struct page **pages, unsigned int count) 807 struct page **pages, unsigned int count)
743{ 808{
809 unsigned int i;
744 int ret; 810 int ret;
745 811
746 ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count); 812 ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count);
747 if (ret) 813 if (ret)
748 return ret; 814 return ret;
749 815
750 return clear_foreign_p2m_mapping(unmap_ops, kmap_ops, pages, count); 816 for (i = 0; i < count; i++)
817 ClearPageForeign(pages[i]);
818
819 return clear_foreign_p2m_mapping(unmap_ops, kunmap_ops, pages, count);
751} 820}
752EXPORT_SYMBOL_GPL(gnttab_unmap_refs); 821EXPORT_SYMBOL_GPL(gnttab_unmap_refs);
753 822
823#define GNTTAB_UNMAP_REFS_DELAY 5
824
825static void __gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item);
826
827static void gnttab_unmap_work(struct work_struct *work)
828{
829 struct gntab_unmap_queue_data
830 *unmap_data = container_of(work,
831 struct gntab_unmap_queue_data,
832 gnttab_work.work);
833 if (unmap_data->age != UINT_MAX)
834 unmap_data->age++;
835 __gnttab_unmap_refs_async(unmap_data);
836}
837
838static void __gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item)
839{
840 int ret;
841 int pc;
842
843 for (pc = 0; pc < item->count; pc++) {
844 if (page_count(item->pages[pc]) > 1) {
845 unsigned long delay = GNTTAB_UNMAP_REFS_DELAY * (item->age + 1);
846 schedule_delayed_work(&item->gnttab_work,
847 msecs_to_jiffies(delay));
848 return;
849 }
850 }
851
852 ret = gnttab_unmap_refs(item->unmap_ops, item->kunmap_ops,
853 item->pages, item->count);
854 item->done(ret, item);
855}
856
857void gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item)
858{
859 INIT_DELAYED_WORK(&item->gnttab_work, gnttab_unmap_work);
860 item->age = 0;
861
862 __gnttab_unmap_refs_async(item);
863}
864EXPORT_SYMBOL_GPL(gnttab_unmap_refs_async);
865
754static int gnttab_map_frames_v1(xen_pfn_t *frames, unsigned int nr_gframes) 866static int gnttab_map_frames_v1(xen_pfn_t *frames, unsigned int nr_gframes)
755{ 867{
756 int rc; 868 int rc;
diff --git a/drivers/xen/manage.c b/drivers/xen/manage.c
index f8bb36f9d9ce..bf1940706422 100644
--- a/drivers/xen/manage.c
+++ b/drivers/xen/manage.c
@@ -105,10 +105,16 @@ static void do_suspend(void)
105 105
106 err = freeze_processes(); 106 err = freeze_processes();
107 if (err) { 107 if (err) {
108 pr_err("%s: freeze failed %d\n", __func__, err); 108 pr_err("%s: freeze processes failed %d\n", __func__, err);
109 goto out; 109 goto out;
110 } 110 }
111 111
112 err = freeze_kernel_threads();
113 if (err) {
114 pr_err("%s: freeze kernel threads failed %d\n", __func__, err);
115 goto out_thaw;
116 }
117
112 err = dpm_suspend_start(PMSG_FREEZE); 118 err = dpm_suspend_start(PMSG_FREEZE);
113 if (err) { 119 if (err) {
114 pr_err("%s: dpm_suspend_start %d\n", __func__, err); 120 pr_err("%s: dpm_suspend_start %d\n", __func__, err);
diff --git a/drivers/xen/tmem.c b/drivers/xen/tmem.c
index 83b5c53bec6b..8a65423bc696 100644
--- a/drivers/xen/tmem.c
+++ b/drivers/xen/tmem.c
@@ -374,7 +374,7 @@ static struct frontswap_ops tmem_frontswap_ops = {
374}; 374};
375#endif 375#endif
376 376
377static int xen_tmem_init(void) 377static int __init xen_tmem_init(void)
378{ 378{
379 if (!xen_domain()) 379 if (!xen_domain())
380 return 0; 380 return 0;
diff --git a/drivers/xen/xen-scsiback.c b/drivers/xen/xen-scsiback.c
index e999496eda3e..ecd540a7a562 100644
--- a/drivers/xen/xen-scsiback.c
+++ b/drivers/xen/xen-scsiback.c
@@ -227,7 +227,7 @@ static void put_free_pages(struct page **page, int num)
227 return; 227 return;
228 if (i > scsiback_max_buffer_pages) { 228 if (i > scsiback_max_buffer_pages) {
229 n = min(num, i - scsiback_max_buffer_pages); 229 n = min(num, i - scsiback_max_buffer_pages);
230 free_xenballooned_pages(n, page + num - n); 230 gnttab_free_pages(n, page + num - n);
231 n = num - n; 231 n = num - n;
232 } 232 }
233 spin_lock_irqsave(&free_pages_lock, flags); 233 spin_lock_irqsave(&free_pages_lock, flags);
@@ -244,7 +244,7 @@ static int get_free_page(struct page **page)
244 spin_lock_irqsave(&free_pages_lock, flags); 244 spin_lock_irqsave(&free_pages_lock, flags);
245 if (list_empty(&scsiback_free_pages)) { 245 if (list_empty(&scsiback_free_pages)) {
246 spin_unlock_irqrestore(&free_pages_lock, flags); 246 spin_unlock_irqrestore(&free_pages_lock, flags);
247 return alloc_xenballooned_pages(1, page, false); 247 return gnttab_alloc_pages(1, page);
248 } 248 }
249 page[0] = list_first_entry(&scsiback_free_pages, struct page, lru); 249 page[0] = list_first_entry(&scsiback_free_pages, struct page, lru);
250 list_del(&page[0]->lru); 250 list_del(&page[0]->lru);
@@ -2106,7 +2106,7 @@ static void __exit scsiback_exit(void)
2106 while (free_pages_num) { 2106 while (free_pages_num) {
2107 if (get_free_page(&page)) 2107 if (get_free_page(&page))
2108 BUG(); 2108 BUG();
2109 free_xenballooned_pages(1, &page); 2109 gnttab_free_pages(1, &page);
2110 } 2110 }
2111 scsiback_deregister_configfs(); 2111 scsiback_deregister_configfs();
2112 xenbus_unregister_driver(&scsiback_driver); 2112 xenbus_unregister_driver(&scsiback_driver);
diff --git a/drivers/xen/xenbus/xenbus_dev_frontend.c b/drivers/xen/xenbus/xenbus_dev_frontend.c
index 85534ea63555..9433e46518c8 100644
--- a/drivers/xen/xenbus/xenbus_dev_frontend.c
+++ b/drivers/xen/xenbus/xenbus_dev_frontend.c
@@ -326,10 +326,13 @@ static int xenbus_write_transaction(unsigned msg_type,
326 } 326 }
327 327
328 if (msg_type == XS_TRANSACTION_START) { 328 if (msg_type == XS_TRANSACTION_START) {
329 trans->handle.id = simple_strtoul(reply, NULL, 0); 329 if (u->u.msg.type == XS_ERROR)
330 330 kfree(trans);
331 list_add(&trans->list, &u->transactions); 331 else {
332 } else if (msg_type == XS_TRANSACTION_END) { 332 trans->handle.id = simple_strtoul(reply, NULL, 0);
333 list_add(&trans->list, &u->transactions);
334 }
335 } else if (u->u.msg.type == XS_TRANSACTION_END) {
333 list_for_each_entry(trans, &u->transactions, list) 336 list_for_each_entry(trans, &u->transactions, list)
334 if (trans->handle.id == u->u.msg.tx_id) 337 if (trans->handle.id == u->u.msg.tx_id)
335 break; 338 break;
diff --git a/include/linux/mm.h b/include/linux/mm.h
index dd5ea3016fc4..237b3ba29225 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -290,6 +290,14 @@ struct vm_operations_struct {
290 /* called by sys_remap_file_pages() to populate non-linear mapping */ 290 /* called by sys_remap_file_pages() to populate non-linear mapping */
291 int (*remap_pages)(struct vm_area_struct *vma, unsigned long addr, 291 int (*remap_pages)(struct vm_area_struct *vma, unsigned long addr,
292 unsigned long size, pgoff_t pgoff); 292 unsigned long size, pgoff_t pgoff);
293
294 /*
295 * Called by vm_normal_page() for special PTEs to find the
296 * page for @addr. This is useful if the default behavior
297 * (using pte_page()) would not find the correct page.
298 */
299 struct page *(*find_special_page)(struct vm_area_struct *vma,
300 unsigned long addr);
293}; 301};
294 302
295struct mmu_gather; 303struct mmu_gather;
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index e1f5fcd79792..5ed7bdaf22d5 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -121,8 +121,12 @@ enum pageflags {
121 PG_fscache = PG_private_2, /* page backed by cache */ 121 PG_fscache = PG_private_2, /* page backed by cache */
122 122
123 /* XEN */ 123 /* XEN */
124 /* Pinned in Xen as a read-only pagetable page. */
124 PG_pinned = PG_owner_priv_1, 125 PG_pinned = PG_owner_priv_1,
126 /* Pinned as part of domain save (see xen_mm_pin_all()). */
125 PG_savepinned = PG_dirty, 127 PG_savepinned = PG_dirty,
128 /* Has a grant mapping of another (foreign) domain's page. */
129 PG_foreign = PG_owner_priv_1,
126 130
127 /* SLOB */ 131 /* SLOB */
128 PG_slob_free = PG_private, 132 PG_slob_free = PG_private,
@@ -215,6 +219,7 @@ __PAGEFLAG(Slab, slab)
215PAGEFLAG(Checked, checked) /* Used by some filesystems */ 219PAGEFLAG(Checked, checked) /* Used by some filesystems */
216PAGEFLAG(Pinned, pinned) TESTSCFLAG(Pinned, pinned) /* Xen */ 220PAGEFLAG(Pinned, pinned) TESTSCFLAG(Pinned, pinned) /* Xen */
217PAGEFLAG(SavePinned, savepinned); /* Xen */ 221PAGEFLAG(SavePinned, savepinned); /* Xen */
222PAGEFLAG(Foreign, foreign); /* Xen */
218PAGEFLAG(Reserved, reserved) __CLEARPAGEFLAG(Reserved, reserved) 223PAGEFLAG(Reserved, reserved) __CLEARPAGEFLAG(Reserved, reserved)
219PAGEFLAG(SwapBacked, swapbacked) __CLEARPAGEFLAG(SwapBacked, swapbacked) 224PAGEFLAG(SwapBacked, swapbacked) __CLEARPAGEFLAG(SwapBacked, swapbacked)
220 __SETPAGEFLAG(SwapBacked, swapbacked) 225 __SETPAGEFLAG(SwapBacked, swapbacked)
diff --git a/include/xen/grant_table.h b/include/xen/grant_table.h
index 3387465b9caa..143ca5ffab7a 100644
--- a/include/xen/grant_table.h
+++ b/include/xen/grant_table.h
@@ -45,6 +45,8 @@
45#include <asm/xen/hypervisor.h> 45#include <asm/xen/hypervisor.h>
46 46
47#include <xen/features.h> 47#include <xen/features.h>
48#include <linux/mm_types.h>
49#include <linux/page-flags.h>
48 50
49#define GNTTAB_RESERVED_XENSTORE 1 51#define GNTTAB_RESERVED_XENSTORE 1
50 52
@@ -58,6 +60,22 @@ struct gnttab_free_callback {
58 u16 count; 60 u16 count;
59}; 61};
60 62
63struct gntab_unmap_queue_data;
64
65typedef void (*gnttab_unmap_refs_done)(int result, struct gntab_unmap_queue_data *data);
66
67struct gntab_unmap_queue_data
68{
69 struct delayed_work gnttab_work;
70 void *data;
71 gnttab_unmap_refs_done done;
72 struct gnttab_unmap_grant_ref *unmap_ops;
73 struct gnttab_unmap_grant_ref *kunmap_ops;
74 struct page **pages;
75 unsigned int count;
76 unsigned int age;
77};
78
61int gnttab_init(void); 79int gnttab_init(void);
62int gnttab_suspend(void); 80int gnttab_suspend(void);
63int gnttab_resume(void); 81int gnttab_resume(void);
@@ -163,12 +181,17 @@ void gnttab_free_auto_xlat_frames(void);
163 181
164#define gnttab_map_vaddr(map) ((void *)(map.host_virt_addr)) 182#define gnttab_map_vaddr(map) ((void *)(map.host_virt_addr))
165 183
184int gnttab_alloc_pages(int nr_pages, struct page **pages);
185void gnttab_free_pages(int nr_pages, struct page **pages);
186
166int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops, 187int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,
167 struct gnttab_map_grant_ref *kmap_ops, 188 struct gnttab_map_grant_ref *kmap_ops,
168 struct page **pages, unsigned int count); 189 struct page **pages, unsigned int count);
169int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops, 190int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,
170 struct gnttab_map_grant_ref *kunmap_ops, 191 struct gnttab_unmap_grant_ref *kunmap_ops,
171 struct page **pages, unsigned int count); 192 struct page **pages, unsigned int count);
193void gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item);
194
172 195
173/* Perform a batch of grant map/copy operations. Retry every batch slot 196/* Perform a batch of grant map/copy operations. Retry every batch slot
174 * for which the hypervisor returns GNTST_eagain. This is typically due 197 * for which the hypervisor returns GNTST_eagain. This is typically due
@@ -182,4 +205,22 @@ int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,
182void gnttab_batch_map(struct gnttab_map_grant_ref *batch, unsigned count); 205void gnttab_batch_map(struct gnttab_map_grant_ref *batch, unsigned count);
183void gnttab_batch_copy(struct gnttab_copy *batch, unsigned count); 206void gnttab_batch_copy(struct gnttab_copy *batch, unsigned count);
184 207
208
209struct xen_page_foreign {
210 domid_t domid;
211 grant_ref_t gref;
212};
213
214static inline struct xen_page_foreign *xen_page_foreign(struct page *page)
215{
216 if (!PageForeign(page))
217 return NULL;
218#if BITS_PER_LONG < 64
219 return (struct xen_page_foreign *)page->private;
220#else
221 BUILD_BUG_ON(sizeof(struct xen_page_foreign) > BITS_PER_LONG);
222 return (struct xen_page_foreign *)&page->private;
223#endif
224}
225
185#endif /* __ASM_GNTTAB_H__ */ 226#endif /* __ASM_GNTTAB_H__ */
diff --git a/include/xen/interface/features.h b/include/xen/interface/features.h
index 131a6ccdba25..6ad3d110bb81 100644
--- a/include/xen/interface/features.h
+++ b/include/xen/interface/features.h
@@ -41,6 +41,12 @@
41/* x86: Does this Xen host support the MMU_PT_UPDATE_PRESERVE_AD hypercall? */ 41/* x86: Does this Xen host support the MMU_PT_UPDATE_PRESERVE_AD hypercall? */
42#define XENFEAT_mmu_pt_update_preserve_ad 5 42#define XENFEAT_mmu_pt_update_preserve_ad 5
43 43
44/*
45 * If set, GNTTABOP_map_grant_ref honors flags to be placed into guest kernel
46 * available pte bits.
47 */
48#define XENFEAT_gnttab_map_avail_bits 7
49
44/* x86: Does this Xen host support the HVM callback vector type? */ 50/* x86: Does this Xen host support the HVM callback vector type? */
45#define XENFEAT_hvm_callback_vector 8 51#define XENFEAT_hvm_callback_vector 8
46 52
diff --git a/include/xen/interface/grant_table.h b/include/xen/interface/grant_table.h
index bcce56439d64..56806bc90c2f 100644
--- a/include/xen/interface/grant_table.h
+++ b/include/xen/interface/grant_table.h
@@ -526,6 +526,13 @@ DEFINE_GUEST_HANDLE_STRUCT(gnttab_cache_flush);
526#define GNTMAP_contains_pte (1<<_GNTMAP_contains_pte) 526#define GNTMAP_contains_pte (1<<_GNTMAP_contains_pte)
527 527
528/* 528/*
529 * Bits to be placed in guest kernel available PTE bits (architecture
530 * dependent; only supported when XENFEAT_gnttab_map_avail_bits is set).
531 */
532#define _GNTMAP_guest_avail0 (16)
533#define GNTMAP_guest_avail_mask ((uint32_t)~0 << _GNTMAP_guest_avail0)
534
535/*
529 * Values for error status returns. All errors are -ve. 536 * Values for error status returns. All errors are -ve.
530 */ 537 */
531#define GNTST_okay (0) /* Normal return. */ 538#define GNTST_okay (0) /* Normal return. */
diff --git a/mm/memory.c b/mm/memory.c
index 2c3536cc6c63..d707c4dfbbb4 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -754,6 +754,8 @@ struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
754 if (HAVE_PTE_SPECIAL) { 754 if (HAVE_PTE_SPECIAL) {
755 if (likely(!pte_special(pte))) 755 if (likely(!pte_special(pte)))
756 goto check_pfn; 756 goto check_pfn;
757 if (vma->vm_ops && vma->vm_ops->find_special_page)
758 return vma->vm_ops->find_special_page(vma, addr);
757 if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP)) 759 if (vma->vm_flags & (VM_PFNMAP | VM_MIXEDMAP))
758 return NULL; 760 return NULL;
759 if (!is_zero_pfn(pfn)) 761 if (!is_zero_pfn(pfn))
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-29 16:09:48 -0400