aboutsummaryrefslogtreecommitdiffstats
path: root/arch/tile/include
diff options
context:
space:
mode:
authorChris Metcalf <cmetcalf@tilera.com>2012-04-01 14:04:21 -0400
committerChris Metcalf <cmetcalf@tilera.com>2012-05-25 12:48:27 -0400
commit621b19551507c8fd9d721f4038509c5bb155a983 (patch)
tree62d8d5e7a783364940153b4523fcfba821cee241 /arch/tile/include
parentd9ed9faac283a3be73f0e11a2ef49ee55aece4db (diff)
arch/tile: support multiple huge page sizes dynamically
This change adds support for a new "super" bit in the PTE, using the new arch_make_huge_pte() method. The Tilera hypervisor sees the bit set at a given level of the page table and gangs together 4, 16, or 64 consecutive pages from that level of the hierarchy to create a larger TLB entry. One extra "super" page size can be specified at each of the three levels of the page table hierarchy on tilegx, using the "hugepagesz" argument on the boot command line. A new hypervisor API is added to allow Linux to tell the hypervisor how many PTEs to gang together at each level of the page table. To allow pre-allocating huge pages larger than the buddy allocator can handle, this change modifies the Tilera bootmem support to put all of memory on tilegx platforms into bootmem. As part of this change I eliminate the vestigial CONFIG_HIGHPTE support, which never worked anyway, and eliminate the hv_page_size() API in favor of the standard vma_kernel_pagesize() API. Signed-off-by: Chris Metcalf <cmetcalf@tilera.com>
Diffstat (limited to 'arch/tile/include')
-rw-r--r--arch/tile/include/asm/hugetlb.h21
-rw-r--r--arch/tile/include/asm/page.h5
-rw-r--r--arch/tile/include/asm/pgtable.h12
-rw-r--r--arch/tile/include/asm/tlbflush.h17
-rw-r--r--arch/tile/include/hv/hypervisor.h70
5 files changed, 102 insertions, 23 deletions
diff --git a/arch/tile/include/asm/hugetlb.h b/arch/tile/include/asm/hugetlb.h
index d396d1805163..b2042380a5aa 100644
--- a/arch/tile/include/asm/hugetlb.h
+++ b/arch/tile/include/asm/hugetlb.h
@@ -106,4 +106,25 @@ static inline void arch_release_hugepage(struct page *page)
106{ 106{
107} 107}
108 108
109#ifdef CONFIG_HUGETLB_SUPER_PAGES
110static inline pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma,
111 struct page *page, int writable)
112{
113 size_t pagesize = huge_page_size(hstate_vma(vma));
114 if (pagesize != PUD_SIZE && pagesize != PMD_SIZE)
115 entry = pte_mksuper(entry);
116 return entry;
117}
118#define arch_make_huge_pte arch_make_huge_pte
119
120/* Sizes to scale up page size for PTEs with HV_PTE_SUPER bit. */
121enum {
122 HUGE_SHIFT_PGDIR = 0,
123 HUGE_SHIFT_PMD = 1,
124 HUGE_SHIFT_PAGE = 2,
125 HUGE_SHIFT_ENTRIES
126};
127extern int huge_shift[HUGE_SHIFT_ENTRIES];
128#endif
129
109#endif /* _ASM_TILE_HUGETLB_H */ 130#endif /* _ASM_TILE_HUGETLB_H */
diff --git a/arch/tile/include/asm/page.h b/arch/tile/include/asm/page.h
index c750943f961e..9d9131e5c552 100644
--- a/arch/tile/include/asm/page.h
+++ b/arch/tile/include/asm/page.h
@@ -87,8 +87,7 @@ typedef HV_PTE pgprot_t;
87/* 87/*
88 * User L2 page tables are managed as one L2 page table per page, 88 * User L2 page tables are managed as one L2 page table per page,
89 * because we use the page allocator for them. This keeps the allocation 89 * because we use the page allocator for them. This keeps the allocation
90 * simple and makes it potentially useful to implement HIGHPTE at some point. 90 * simple, but it's also inefficient, since L2 page tables are much smaller
91 * However, it's also inefficient, since L2 page tables are much smaller
92 * than pages (currently 2KB vs 64KB). So we should revisit this. 91 * than pages (currently 2KB vs 64KB). So we should revisit this.
93 */ 92 */
94typedef struct page *pgtable_t; 93typedef struct page *pgtable_t;
@@ -137,7 +136,7 @@ static inline __attribute_const__ int get_order(unsigned long size)
137 136
138#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) 137#define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT)
139 138
140#define HUGE_MAX_HSTATE 2 139#define HUGE_MAX_HSTATE 6
141 140
142#ifdef CONFIG_HUGETLB_PAGE 141#ifdef CONFIG_HUGETLB_PAGE
143#define HAVE_ARCH_HUGETLB_UNMAPPED_AREA 142#define HAVE_ARCH_HUGETLB_UNMAPPED_AREA
diff --git a/arch/tile/include/asm/pgtable.h b/arch/tile/include/asm/pgtable.h
index 319f4826d972..73b1a4c9ad03 100644
--- a/arch/tile/include/asm/pgtable.h
+++ b/arch/tile/include/asm/pgtable.h
@@ -71,6 +71,7 @@ extern void set_page_homes(void);
71 71
72#define _PAGE_PRESENT HV_PTE_PRESENT 72#define _PAGE_PRESENT HV_PTE_PRESENT
73#define _PAGE_HUGE_PAGE HV_PTE_PAGE 73#define _PAGE_HUGE_PAGE HV_PTE_PAGE
74#define _PAGE_SUPER_PAGE HV_PTE_SUPER
74#define _PAGE_READABLE HV_PTE_READABLE 75#define _PAGE_READABLE HV_PTE_READABLE
75#define _PAGE_WRITABLE HV_PTE_WRITABLE 76#define _PAGE_WRITABLE HV_PTE_WRITABLE
76#define _PAGE_EXECUTABLE HV_PTE_EXECUTABLE 77#define _PAGE_EXECUTABLE HV_PTE_EXECUTABLE
@@ -87,6 +88,7 @@ extern void set_page_homes(void);
87#define _PAGE_ALL (\ 88#define _PAGE_ALL (\
88 _PAGE_PRESENT | \ 89 _PAGE_PRESENT | \
89 _PAGE_HUGE_PAGE | \ 90 _PAGE_HUGE_PAGE | \
91 _PAGE_SUPER_PAGE | \
90 _PAGE_READABLE | \ 92 _PAGE_READABLE | \
91 _PAGE_WRITABLE | \ 93 _PAGE_WRITABLE | \
92 _PAGE_EXECUTABLE | \ 94 _PAGE_EXECUTABLE | \
@@ -197,6 +199,7 @@ static inline void __pte_clear(pte_t *ptep)
197#define pte_write hv_pte_get_writable 199#define pte_write hv_pte_get_writable
198#define pte_exec hv_pte_get_executable 200#define pte_exec hv_pte_get_executable
199#define pte_huge hv_pte_get_page 201#define pte_huge hv_pte_get_page
202#define pte_super hv_pte_get_super
200#define pte_rdprotect hv_pte_clear_readable 203#define pte_rdprotect hv_pte_clear_readable
201#define pte_exprotect hv_pte_clear_executable 204#define pte_exprotect hv_pte_clear_executable
202#define pte_mkclean hv_pte_clear_dirty 205#define pte_mkclean hv_pte_clear_dirty
@@ -209,6 +212,7 @@ static inline void __pte_clear(pte_t *ptep)
209#define pte_mkyoung hv_pte_set_accessed 212#define pte_mkyoung hv_pte_set_accessed
210#define pte_mkwrite hv_pte_set_writable 213#define pte_mkwrite hv_pte_set_writable
211#define pte_mkhuge hv_pte_set_page 214#define pte_mkhuge hv_pte_set_page
215#define pte_mksuper hv_pte_set_super
212 216
213#define pte_special(pte) 0 217#define pte_special(pte) 0
214#define pte_mkspecial(pte) (pte) 218#define pte_mkspecial(pte) (pte)
@@ -338,13 +342,8 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
338 */ 342 */
339#define pgd_offset_k(address) pgd_offset(&init_mm, address) 343#define pgd_offset_k(address) pgd_offset(&init_mm, address)
340 344
341#if defined(CONFIG_HIGHPTE)
342extern pte_t *pte_offset_map(pmd_t *, unsigned long address);
343#define pte_unmap(pte) kunmap_atomic(pte)
344#else
345#define pte_offset_map(dir, address) pte_offset_kernel(dir, address) 345#define pte_offset_map(dir, address) pte_offset_kernel(dir, address)
346#define pte_unmap(pte) do { } while (0) 346#define pte_unmap(pte) do { } while (0)
347#endif
348 347
349/* Clear a non-executable kernel PTE and flush it from the TLB. */ 348/* Clear a non-executable kernel PTE and flush it from the TLB. */
350#define kpte_clear_flush(ptep, vaddr) \ 349#define kpte_clear_flush(ptep, vaddr) \
@@ -537,7 +536,8 @@ static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address)
537/* Support /proc/NN/pgtable API. */ 536/* Support /proc/NN/pgtable API. */
538struct seq_file; 537struct seq_file;
539int arch_proc_pgtable_show(struct seq_file *m, struct mm_struct *mm, 538int arch_proc_pgtable_show(struct seq_file *m, struct mm_struct *mm,
540 unsigned long vaddr, pte_t *ptep, void **datap); 539 unsigned long vaddr, unsigned long pagesize,
540 pte_t *ptep, void **datap);
541 541
542#endif /* !__ASSEMBLY__ */ 542#endif /* !__ASSEMBLY__ */
543 543
diff --git a/arch/tile/include/asm/tlbflush.h b/arch/tile/include/asm/tlbflush.h
index 96199d214fb8..dcf91b25a1e5 100644
--- a/arch/tile/include/asm/tlbflush.h
+++ b/arch/tile/include/asm/tlbflush.h
@@ -38,16 +38,11 @@ DECLARE_PER_CPU(int, current_asid);
38/* The hypervisor tells us what ASIDs are available to us. */ 38/* The hypervisor tells us what ASIDs are available to us. */
39extern int min_asid, max_asid; 39extern int min_asid, max_asid;
40 40
41static inline unsigned long hv_page_size(const struct vm_area_struct *vma)
42{
43 return (vma->vm_flags & VM_HUGETLB) ? HPAGE_SIZE : PAGE_SIZE;
44}
45
46/* Pass as vma pointer for non-executable mapping, if no vma available. */ 41/* Pass as vma pointer for non-executable mapping, if no vma available. */
47#define FLUSH_NONEXEC ((const struct vm_area_struct *)-1UL) 42#define FLUSH_NONEXEC ((struct vm_area_struct *)-1UL)
48 43
49/* Flush a single user page on this cpu. */ 44/* Flush a single user page on this cpu. */
50static inline void local_flush_tlb_page(const struct vm_area_struct *vma, 45static inline void local_flush_tlb_page(struct vm_area_struct *vma,
51 unsigned long addr, 46 unsigned long addr,
52 unsigned long page_size) 47 unsigned long page_size)
53{ 48{
@@ -60,7 +55,7 @@ static inline void local_flush_tlb_page(const struct vm_area_struct *vma,
60} 55}
61 56
62/* Flush range of user pages on this cpu. */ 57/* Flush range of user pages on this cpu. */
63static inline void local_flush_tlb_pages(const struct vm_area_struct *vma, 58static inline void local_flush_tlb_pages(struct vm_area_struct *vma,
64 unsigned long addr, 59 unsigned long addr,
65 unsigned long page_size, 60 unsigned long page_size,
66 unsigned long len) 61 unsigned long len)
@@ -117,10 +112,10 @@ extern void flush_tlb_all(void);
117extern void flush_tlb_kernel_range(unsigned long start, unsigned long end); 112extern void flush_tlb_kernel_range(unsigned long start, unsigned long end);
118extern void flush_tlb_current_task(void); 113extern void flush_tlb_current_task(void);
119extern void flush_tlb_mm(struct mm_struct *); 114extern void flush_tlb_mm(struct mm_struct *);
120extern void flush_tlb_page(const struct vm_area_struct *, unsigned long); 115extern void flush_tlb_page(struct vm_area_struct *, unsigned long);
121extern void flush_tlb_page_mm(const struct vm_area_struct *, 116extern void flush_tlb_page_mm(struct vm_area_struct *,
122 struct mm_struct *, unsigned long); 117 struct mm_struct *, unsigned long);
123extern void flush_tlb_range(const struct vm_area_struct *, 118extern void flush_tlb_range(struct vm_area_struct *,
124 unsigned long start, unsigned long end); 119 unsigned long start, unsigned long end);
125 120
126#define flush_tlb() flush_tlb_current_task() 121#define flush_tlb() flush_tlb_current_task()
diff --git a/arch/tile/include/hv/hypervisor.h b/arch/tile/include/hv/hypervisor.h
index f27871775b7a..85e5cab4c2f0 100644
--- a/arch/tile/include/hv/hypervisor.h
+++ b/arch/tile/include/hv/hypervisor.h
@@ -66,6 +66,22 @@
66#define HV_DEFAULT_PAGE_SIZE_LARGE \ 66#define HV_DEFAULT_PAGE_SIZE_LARGE \
67 (__HV_SIZE_ONE << HV_LOG2_DEFAULT_PAGE_SIZE_LARGE) 67 (__HV_SIZE_ONE << HV_LOG2_DEFAULT_PAGE_SIZE_LARGE)
68 68
69#if CHIP_VA_WIDTH() > 32
70
71/** The log2 of the initial size of jumbo pages, in bytes.
72 * See HV_DEFAULT_PAGE_SIZE_JUMBO.
73 */
74#define HV_LOG2_DEFAULT_PAGE_SIZE_JUMBO 32
75
76/** The initial size of jumbo pages, in bytes. This value should
77 * be verified at runtime by calling hv_sysconf(HV_SYSCONF_PAGE_SIZE_JUMBO).
78 * It may also be modified when installing a new context.
79 */
80#define HV_DEFAULT_PAGE_SIZE_JUMBO \
81 (__HV_SIZE_ONE << HV_LOG2_DEFAULT_PAGE_SIZE_JUMBO)
82
83#endif
84
69/** The log2 of the granularity at which page tables must be aligned; 85/** The log2 of the granularity at which page tables must be aligned;
70 * in other words, the CPA for a page table must have this many zero 86 * in other words, the CPA for a page table must have this many zero
71 * bits at the bottom of the address. 87 * bits at the bottom of the address.
@@ -284,8 +300,11 @@
284#define HV_DISPATCH_GET_IPI_PTE 56 300#define HV_DISPATCH_GET_IPI_PTE 56
285#endif 301#endif
286 302
303/** hv_set_pte_super_shift */
304#define HV_DISPATCH_SET_PTE_SUPER_SHIFT 57
305
287/** One more than the largest dispatch value */ 306/** One more than the largest dispatch value */
288#define _HV_DISPATCH_END 57 307#define _HV_DISPATCH_END 58
289 308
290 309
291#ifndef __ASSEMBLER__ 310#ifndef __ASSEMBLER__
@@ -413,6 +432,11 @@ typedef enum {
413 */ 432 */
414 HV_SYSCONF_VALID_PAGE_SIZES = 7, 433 HV_SYSCONF_VALID_PAGE_SIZES = 7,
415 434
435 /** The size of jumbo pages, in bytes.
436 * If no jumbo pages are available, zero will be returned.
437 */
438 HV_SYSCONF_PAGE_SIZE_JUMBO = 8,
439
416} HV_SysconfQuery; 440} HV_SysconfQuery;
417 441
418/** Offset to subtract from returned Kelvin temperature to get degrees 442/** Offset to subtract from returned Kelvin temperature to get degrees
@@ -695,6 +719,29 @@ int hv_install_context(HV_PhysAddr page_table, HV_PTE access, HV_ASID asid,
695 719
696#ifndef __ASSEMBLER__ 720#ifndef __ASSEMBLER__
697 721
722
723/** Set the number of pages ganged together by HV_PTE_SUPER at a
724 * particular level of the page table.
725 *
726 * The current TILE-Gx hardware only supports powers of four
727 * (i.e. log2_count must be a multiple of two), and the requested
728 * "super" page size must be less than the span of the next level in
729 * the page table. The largest size that can be requested is 64GB.
730 *
731 * The shift value is initially "0" for all page table levels,
732 * indicating that the HV_PTE_SUPER bit is effectively ignored.
733 *
734 * If you change the count from one non-zero value to another, the
735 * hypervisor will flush the entire TLB and TSB to avoid confusion.
736 *
737 * @param level Page table level (0, 1, or 2)
738 * @param log2_count Base-2 log of the number of pages to gang together,
739 * i.e. how much to shift left the base page size for the super page size.
740 * @return Zero on success, or a hypervisor error code on failure.
741 */
742int hv_set_pte_super_shift(int level, int log2_count);
743
744
698/** Value returned from hv_inquire_context(). */ 745/** Value returned from hv_inquire_context(). */
699typedef struct 746typedef struct
700{ 747{
@@ -1891,8 +1938,9 @@ int hv_flush_remote(HV_PhysAddr cache_pa, unsigned long cache_control,
1891#define HV_PTE_INDEX_USER 10 /**< Page is user-accessible */ 1938#define HV_PTE_INDEX_USER 10 /**< Page is user-accessible */
1892#define HV_PTE_INDEX_ACCESSED 11 /**< Page has been accessed */ 1939#define HV_PTE_INDEX_ACCESSED 11 /**< Page has been accessed */
1893#define HV_PTE_INDEX_DIRTY 12 /**< Page has been written */ 1940#define HV_PTE_INDEX_DIRTY 12 /**< Page has been written */
1894 /* Bits 13-15 are reserved for 1941 /* Bits 13-14 are reserved for
1895 future use. */ 1942 future use. */
1943#define HV_PTE_INDEX_SUPER 15 /**< Pages ganged together for TLB */
1896#define HV_PTE_INDEX_MODE 16 /**< Page mode; see HV_PTE_MODE_xxx */ 1944#define HV_PTE_INDEX_MODE 16 /**< Page mode; see HV_PTE_MODE_xxx */
1897#define HV_PTE_MODE_BITS 3 /**< Number of bits in mode */ 1945#define HV_PTE_MODE_BITS 3 /**< Number of bits in mode */
1898#define HV_PTE_INDEX_CLIENT2 19 /**< Page client state 2 */ 1946#define HV_PTE_INDEX_CLIENT2 19 /**< Page client state 2 */
@@ -1987,7 +2035,10 @@ int hv_flush_remote(HV_PhysAddr cache_pa, unsigned long cache_control,
1987 2035
1988/** Does this PTE map a page? 2036/** Does this PTE map a page?
1989 * 2037 *
1990 * If this bit is set in the level-1 page table, the entry should be 2038 * If this bit is set in a level-0 page table, the entry should be
2039 * interpreted as a level-2 page table entry mapping a jumbo page.
2040 *
2041 * If this bit is set in a level-1 page table, the entry should be
1991 * interpreted as a level-2 page table entry mapping a large page. 2042 * interpreted as a level-2 page table entry mapping a large page.
1992 * 2043 *
1993 * This bit should not be modified by the client while PRESENT is set, as 2044 * This bit should not be modified by the client while PRESENT is set, as
@@ -1997,6 +2048,18 @@ int hv_flush_remote(HV_PhysAddr cache_pa, unsigned long cache_control,
1997 */ 2048 */
1998#define HV_PTE_PAGE (__HV_PTE_ONE << HV_PTE_INDEX_PAGE) 2049#define HV_PTE_PAGE (__HV_PTE_ONE << HV_PTE_INDEX_PAGE)
1999 2050
2051/** Does this PTE implicitly reference multiple pages?
2052 *
2053 * If this bit is set in the page table (either in the level-2 page table,
2054 * or in a higher level page table in conjunction with the PAGE bit)
2055 * then the PTE specifies a range of contiguous pages, not a single page.
2056 * The hv_set_pte_super_shift() allows you to specify the count for
2057 * each level of the page table.
2058 *
2059 * Note: this bit is not supported on TILEPro systems.
2060 */
2061#define HV_PTE_SUPER (__HV_PTE_ONE << HV_PTE_INDEX_SUPER)
2062
2000/** Is this a global (non-ASID) mapping? 2063/** Is this a global (non-ASID) mapping?
2001 * 2064 *
2002 * If this bit is set, the translations established by this PTE will 2065 * If this bit is set, the translations established by this PTE will
@@ -2215,6 +2278,7 @@ hv_pte_clear_##name(HV_PTE pte) \
2215 */ 2278 */
2216_HV_BIT(present, PRESENT) 2279_HV_BIT(present, PRESENT)
2217_HV_BIT(page, PAGE) 2280_HV_BIT(page, PAGE)
2281_HV_BIT(super, SUPER)
2218_HV_BIT(client0, CLIENT0) 2282_HV_BIT(client0, CLIENT0)
2219_HV_BIT(client1, CLIENT1) 2283_HV_BIT(client1, CLIENT1)
2220_HV_BIT(client2, CLIENT2) 2284_HV_BIT(client2, CLIENT2)