aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
authorRussell King <rmk+kernel@arm.linux.org.uk>2011-12-08 13:02:04 -0500
committerRussell King <rmk+kernel@arm.linux.org.uk>2011-12-08 13:02:04 -0500
commit6ae25a5b9d7ba86d6ac19c403dfa57dae6caa73d (patch)
tree41d04269f268d6162e5f1866496dd42fbc79d2a4 /arch
parent3ee0fc5ca129cbae81c073756febcb1c552af446 (diff)
parent497b7e943d0dc5743454de56dcdb67352bbf96b2 (diff)
Merge branch 'for-rmk' of git://git.kernel.org/pub/scm/linux/kernel/git/cmarinas/linux into devel-stable
Conflicts: arch/arm/mm/ioremap.c
Diffstat (limited to 'arch')
-rw-r--r--arch/arm/Kconfig2
-rw-r--r--arch/arm/boot/compressed/head.S1
-rw-r--r--arch/arm/include/asm/assembler.h11
-rw-r--r--arch/arm/include/asm/page.h4
-rw-r--r--arch/arm/include/asm/pgalloc.h26
-rw-r--r--arch/arm/include/asm/pgtable-2level.h41
-rw-r--r--arch/arm/include/asm/pgtable-3level-hwdef.h77
-rw-r--r--arch/arm/include/asm/pgtable-3level-types.h70
-rw-r--r--arch/arm/include/asm/pgtable-3level.h155
-rw-r--r--arch/arm/include/asm/pgtable-hwdef.h4
-rw-r--r--arch/arm/include/asm/pgtable.h43
-rw-r--r--arch/arm/include/asm/proc-fns.h21
-rw-r--r--arch/arm/include/asm/system.h8
-rw-r--r--arch/arm/include/asm/tlb.h12
-rw-r--r--arch/arm/kernel/head.S47
-rw-r--r--arch/arm/kernel/hw_breakpoint.c8
-rw-r--r--arch/arm/kernel/sleep.S2
-rw-r--r--arch/arm/mm/Kconfig17
-rw-r--r--arch/arm/mm/alignment.c2
-rw-r--r--arch/arm/mm/context.c19
-rw-r--r--arch/arm/mm/fault.c111
-rw-r--r--arch/arm/mm/fault.h27
-rw-r--r--arch/arm/mm/fsr-2level.c78
-rw-r--r--arch/arm/mm/fsr-3level.c68
-rw-r--r--arch/arm/mm/idmap.c28
-rw-r--r--arch/arm/mm/ioremap.c39
-rw-r--r--arch/arm/mm/mmu.c46
-rw-r--r--arch/arm/mm/pgd.c51
-rw-r--r--arch/arm/mm/proc-macros.S5
-rw-r--r--arch/arm/mm/proc-v7-2level.S171
-rw-r--r--arch/arm/mm/proc-v7-3level.S150
-rw-r--r--arch/arm/mm/proc-v7.S177
32 files changed, 1199 insertions, 322 deletions
diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig
index e084b7e981e..4b01d71bdbd 100644
--- a/arch/arm/Kconfig
+++ b/arch/arm/Kconfig
@@ -1971,7 +1971,7 @@ endchoice
1971 1971
1972config XIP_KERNEL 1972config XIP_KERNEL
1973 bool "Kernel Execute-In-Place from ROM" 1973 bool "Kernel Execute-In-Place from ROM"
1974 depends on !ZBOOT_ROM 1974 depends on !ZBOOT_ROM && !ARM_LPAE
1975 help 1975 help
1976 Execute-In-Place allows the kernel to run from non-volatile storage 1976 Execute-In-Place allows the kernel to run from non-volatile storage
1977 directly addressable by the CPU, such as NOR flash. This saves RAM 1977 directly addressable by the CPU, such as NOR flash. This saves RAM
diff --git a/arch/arm/boot/compressed/head.S b/arch/arm/boot/compressed/head.S
index c2effc91725..c5d60250d43 100644
--- a/arch/arm/boot/compressed/head.S
+++ b/arch/arm/boot/compressed/head.S
@@ -659,6 +659,7 @@ __armv7_mmu_cache_on:
659 mcrne p15, 0, r3, c2, c0, 0 @ load page table pointer 659 mcrne p15, 0, r3, c2, c0, 0 @ load page table pointer
660 mcrne p15, 0, r1, c3, c0, 0 @ load domain access control 660 mcrne p15, 0, r1, c3, c0, 0 @ load domain access control
661#endif 661#endif
662 mcr p15, 0, r0, c7, c5, 4 @ ISB
662 mcr p15, 0, r0, c1, c0, 0 @ load control register 663 mcr p15, 0, r0, c1, c0, 0 @ load control register
663 mrc p15, 0, r0, c1, c0, 0 @ and read it back 664 mrc p15, 0, r0, c1, c0, 0 @ and read it back
664 mov r0, #0 665 mov r0, #0
diff --git a/arch/arm/include/asm/assembler.h b/arch/arm/include/asm/assembler.h
index 29035e86a59..b6e65dedfd7 100644
--- a/arch/arm/include/asm/assembler.h
+++ b/arch/arm/include/asm/assembler.h
@@ -187,6 +187,17 @@
187#endif 187#endif
188 188
189/* 189/*
190 * Instruction barrier
191 */
192 .macro instr_sync
193#if __LINUX_ARM_ARCH__ >= 7
194 isb
195#elif __LINUX_ARM_ARCH__ == 6
196 mcr p15, 0, r0, c7, c5, 4
197#endif
198 .endm
199
200/*
190 * SMP data memory barrier 201 * SMP data memory barrier
191 */ 202 */
192 .macro smp_dmb mode 203 .macro smp_dmb mode
diff --git a/arch/arm/include/asm/page.h b/arch/arm/include/asm/page.h
index ca94653f1ec..97b440c25c5 100644
--- a/arch/arm/include/asm/page.h
+++ b/arch/arm/include/asm/page.h
@@ -151,7 +151,11 @@ extern void __cpu_copy_user_highpage(struct page *to, struct page *from,
151#define clear_page(page) memset((void *)(page), 0, PAGE_SIZE) 151#define clear_page(page) memset((void *)(page), 0, PAGE_SIZE)
152extern void copy_page(void *to, const void *from); 152extern void copy_page(void *to, const void *from);
153 153
154#ifdef CONFIG_ARM_LPAE
155#include <asm/pgtable-3level-types.h>
156#else
154#include <asm/pgtable-2level-types.h> 157#include <asm/pgtable-2level-types.h>
158#endif
155 159
156#endif /* CONFIG_MMU */ 160#endif /* CONFIG_MMU */
157 161
diff --git a/arch/arm/include/asm/pgalloc.h b/arch/arm/include/asm/pgalloc.h
index 3e08fd3fbb6..943504f53f5 100644
--- a/arch/arm/include/asm/pgalloc.h
+++ b/arch/arm/include/asm/pgalloc.h
@@ -25,12 +25,34 @@
25#define _PAGE_USER_TABLE (PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_USER)) 25#define _PAGE_USER_TABLE (PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_USER))
26#define _PAGE_KERNEL_TABLE (PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_KERNEL)) 26#define _PAGE_KERNEL_TABLE (PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_KERNEL))
27 27
28#ifdef CONFIG_ARM_LPAE
29
30static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
31{
32 return (pmd_t *)get_zeroed_page(GFP_KERNEL | __GFP_REPEAT);
33}
34
35static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
36{
37 BUG_ON((unsigned long)pmd & (PAGE_SIZE-1));
38 free_page((unsigned long)pmd);
39}
40
41static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
42{
43 set_pud(pud, __pud(__pa(pmd) | PMD_TYPE_TABLE));
44}
45
46#else /* !CONFIG_ARM_LPAE */
47
28/* 48/*
29 * Since we have only two-level page tables, these are trivial 49 * Since we have only two-level page tables, these are trivial
30 */ 50 */
31#define pmd_alloc_one(mm,addr) ({ BUG(); ((pmd_t *)2); }) 51#define pmd_alloc_one(mm,addr) ({ BUG(); ((pmd_t *)2); })
32#define pmd_free(mm, pmd) do { } while (0) 52#define pmd_free(mm, pmd) do { } while (0)
33#define pgd_populate(mm,pmd,pte) BUG() 53#define pud_populate(mm,pmd,pte) BUG()
54
55#endif /* CONFIG_ARM_LPAE */
34 56
35extern pgd_t *pgd_alloc(struct mm_struct *mm); 57extern pgd_t *pgd_alloc(struct mm_struct *mm);
36extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); 58extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);
@@ -109,7 +131,9 @@ static inline void __pmd_populate(pmd_t *pmdp, phys_addr_t pte,
109{ 131{
110 pmdval_t pmdval = (pte + PTE_HWTABLE_OFF) | prot; 132 pmdval_t pmdval = (pte + PTE_HWTABLE_OFF) | prot;
111 pmdp[0] = __pmd(pmdval); 133 pmdp[0] = __pmd(pmdval);
134#ifndef CONFIG_ARM_LPAE
112 pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t)); 135 pmdp[1] = __pmd(pmdval + 256 * sizeof(pte_t));
136#endif
113 flush_pmd_entry(pmdp); 137 flush_pmd_entry(pmdp);
114} 138}
115 139
diff --git a/arch/arm/include/asm/pgtable-2level.h b/arch/arm/include/asm/pgtable-2level.h
index 470457e1cfc..2317a71c8f8 100644
--- a/arch/arm/include/asm/pgtable-2level.h
+++ b/arch/arm/include/asm/pgtable-2level.h
@@ -140,4 +140,45 @@
140#define L_PTE_MT_DEV_CACHED (_AT(pteval_t, 0x0b) << 2) /* 1011 */ 140#define L_PTE_MT_DEV_CACHED (_AT(pteval_t, 0x0b) << 2) /* 1011 */
141#define L_PTE_MT_MASK (_AT(pteval_t, 0x0f) << 2) 141#define L_PTE_MT_MASK (_AT(pteval_t, 0x0f) << 2)
142 142
143#ifndef __ASSEMBLY__
144
145/*
146 * The "pud_xxx()" functions here are trivial when the pmd is folded into
147 * the pud: the pud entry is never bad, always exists, and can't be set or
148 * cleared.
149 */
150#define pud_none(pud) (0)
151#define pud_bad(pud) (0)
152#define pud_present(pud) (1)
153#define pud_clear(pudp) do { } while (0)
154#define set_pud(pud,pudp) do { } while (0)
155
156static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr)
157{
158 return (pmd_t *)pud;
159}
160
161#define pmd_bad(pmd) (pmd_val(pmd) & 2)
162
163#define copy_pmd(pmdpd,pmdps) \
164 do { \
165 pmdpd[0] = pmdps[0]; \
166 pmdpd[1] = pmdps[1]; \
167 flush_pmd_entry(pmdpd); \
168 } while (0)
169
170#define pmd_clear(pmdp) \
171 do { \
172 pmdp[0] = __pmd(0); \
173 pmdp[1] = __pmd(0); \
174 clean_pmd_entry(pmdp); \
175 } while (0)
176
177/* we don't need complex calculations here as the pmd is folded into the pgd */
178#define pmd_addr_end(addr,end) (end)
179
180#define set_pte_ext(ptep,pte,ext) cpu_set_pte_ext(ptep,pte,ext)
181
182#endif /* __ASSEMBLY__ */
183
143#endif /* _ASM_PGTABLE_2LEVEL_H */ 184#endif /* _ASM_PGTABLE_2LEVEL_H */
diff --git a/arch/arm/include/asm/pgtable-3level-hwdef.h b/arch/arm/include/asm/pgtable-3level-hwdef.h
new file mode 100644
index 00000000000..d7952824c5c
--- /dev/null
+++ b/arch/arm/include/asm/pgtable-3level-hwdef.h
@@ -0,0 +1,77 @@
1/*
2 * arch/arm/include/asm/pgtable-3level-hwdef.h
3 *
4 * Copyright (C) 2011 ARM Ltd.
5 * Author: Catalin Marinas <catalin.marinas@arm.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 */
20#ifndef _ASM_PGTABLE_3LEVEL_HWDEF_H
21#define _ASM_PGTABLE_3LEVEL_HWDEF_H
22
23/*
24 * Hardware page table definitions.
25 *
26 * + Level 1/2 descriptor
27 * - common
28 */
29#define PMD_TYPE_MASK (_AT(pmdval_t, 3) << 0)
30#define PMD_TYPE_FAULT (_AT(pmdval_t, 0) << 0)
31#define PMD_TYPE_TABLE (_AT(pmdval_t, 3) << 0)
32#define PMD_TYPE_SECT (_AT(pmdval_t, 1) << 0)
33#define PMD_BIT4 (_AT(pmdval_t, 0))
34#define PMD_DOMAIN(x) (_AT(pmdval_t, 0))
35
36/*
37 * - section
38 */
39#define PMD_SECT_BUFFERABLE (_AT(pmdval_t, 1) << 2)
40#define PMD_SECT_CACHEABLE (_AT(pmdval_t, 1) << 3)
41#define PMD_SECT_S (_AT(pmdval_t, 3) << 8)
42#define PMD_SECT_AF (_AT(pmdval_t, 1) << 10)
43#define PMD_SECT_nG (_AT(pmdval_t, 1) << 11)
44#define PMD_SECT_XN (_AT(pmdval_t, 1) << 54)
45#define PMD_SECT_AP_WRITE (_AT(pmdval_t, 0))
46#define PMD_SECT_AP_READ (_AT(pmdval_t, 0))
47#define PMD_SECT_TEX(x) (_AT(pmdval_t, 0))
48
49/*
50 * AttrIndx[2:0] encoding (mapping attributes defined in the MAIR* registers).
51 */
52#define PMD_SECT_UNCACHED (_AT(pmdval_t, 0) << 2) /* strongly ordered */
53#define PMD_SECT_BUFFERED (_AT(pmdval_t, 1) << 2) /* normal non-cacheable */
54#define PMD_SECT_WT (_AT(pmdval_t, 2) << 2) /* normal inner write-through */
55#define PMD_SECT_WB (_AT(pmdval_t, 3) << 2) /* normal inner write-back */
56#define PMD_SECT_WBWA (_AT(pmdval_t, 7) << 2) /* normal inner write-alloc */
57
58/*
59 * + Level 3 descriptor (PTE)
60 */
61#define PTE_TYPE_MASK (_AT(pteval_t, 3) << 0)
62#define PTE_TYPE_FAULT (_AT(pteval_t, 0) << 0)
63#define PTE_TYPE_PAGE (_AT(pteval_t, 3) << 0)
64#define PTE_BUFFERABLE (_AT(pteval_t, 1) << 2) /* AttrIndx[0] */
65#define PTE_CACHEABLE (_AT(pteval_t, 1) << 3) /* AttrIndx[1] */
66#define PTE_EXT_SHARED (_AT(pteval_t, 3) << 8) /* SH[1:0], inner shareable */
67#define PTE_EXT_AF (_AT(pteval_t, 1) << 10) /* Access Flag */
68#define PTE_EXT_NG (_AT(pteval_t, 1) << 11) /* nG */
69#define PTE_EXT_XN (_AT(pteval_t, 1) << 54) /* XN */
70
71/*
72 * 40-bit physical address supported.
73 */
74#define PHYS_MASK_SHIFT (40)
75#define PHYS_MASK ((1ULL << PHYS_MASK_SHIFT) - 1)
76
77#endif
diff --git a/arch/arm/include/asm/pgtable-3level-types.h b/arch/arm/include/asm/pgtable-3level-types.h
new file mode 100644
index 00000000000..921aa30259c
--- /dev/null
+++ b/arch/arm/include/asm/pgtable-3level-types.h
@@ -0,0 +1,70 @@
1/*
2 * arch/arm/include/asm/pgtable-3level-types.h
3 *
4 * Copyright (C) 2011 ARM Ltd.
5 * Author: Catalin Marinas <catalin.marinas@arm.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 */
20#ifndef _ASM_PGTABLE_3LEVEL_TYPES_H
21#define _ASM_PGTABLE_3LEVEL_TYPES_H
22
23#include <asm/types.h>
24
25typedef u64 pteval_t;
26typedef u64 pmdval_t;
27typedef u64 pgdval_t;
28
29#undef STRICT_MM_TYPECHECKS
30
31#ifdef STRICT_MM_TYPECHECKS
32
33/*
34 * These are used to make use of C type-checking..
35 */
36typedef struct { pteval_t pte; } pte_t;
37typedef struct { pmdval_t pmd; } pmd_t;
38typedef struct { pgdval_t pgd; } pgd_t;
39typedef struct { pteval_t pgprot; } pgprot_t;
40
41#define pte_val(x) ((x).pte)
42#define pmd_val(x) ((x).pmd)
43#define pgd_val(x) ((x).pgd)
44#define pgprot_val(x) ((x).pgprot)
45
46#define __pte(x) ((pte_t) { (x) } )
47#define __pmd(x) ((pmd_t) { (x) } )
48#define __pgd(x) ((pgd_t) { (x) } )
49#define __pgprot(x) ((pgprot_t) { (x) } )
50
51#else /* !STRICT_MM_TYPECHECKS */
52
53typedef pteval_t pte_t;
54typedef pmdval_t pmd_t;
55typedef pgdval_t pgd_t;
56typedef pteval_t pgprot_t;
57
58#define pte_val(x) (x)
59#define pmd_val(x) (x)
60#define pgd_val(x) (x)
61#define pgprot_val(x) (x)
62
63#define __pte(x) (x)
64#define __pmd(x) (x)
65#define __pgd(x) (x)
66#define __pgprot(x) (x)
67
68#endif /* STRICT_MM_TYPECHECKS */
69
70#endif /* _ASM_PGTABLE_3LEVEL_TYPES_H */
diff --git a/arch/arm/include/asm/pgtable-3level.h b/arch/arm/include/asm/pgtable-3level.h
new file mode 100644
index 00000000000..759af70f9a0
--- /dev/null
+++ b/arch/arm/include/asm/pgtable-3level.h
@@ -0,0 +1,155 @@
1/*
2 * arch/arm/include/asm/pgtable-3level.h
3 *
4 * Copyright (C) 2011 ARM Ltd.
5 * Author: Catalin Marinas <catalin.marinas@arm.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 */
20#ifndef _ASM_PGTABLE_3LEVEL_H
21#define _ASM_PGTABLE_3LEVEL_H
22
23/*
24 * With LPAE, there are 3 levels of page tables. Each level has 512 entries of
25 * 8 bytes each, occupying a 4K page. The first level table covers a range of
26 * 512GB, each entry representing 1GB. Since we are limited to 4GB input
27 * address range, only 4 entries in the PGD are used.
28 *
29 * There are enough spare bits in a page table entry for the kernel specific
30 * state.
31 */
32#define PTRS_PER_PTE 512
33#define PTRS_PER_PMD 512
34#define PTRS_PER_PGD 4
35
36#define PTE_HWTABLE_PTRS (PTRS_PER_PTE)
37#define PTE_HWTABLE_OFF (0)
38#define PTE_HWTABLE_SIZE (PTRS_PER_PTE * sizeof(u64))
39
40/*
41 * PGDIR_SHIFT determines the size a top-level page table entry can map.
42 */
43#define PGDIR_SHIFT 30
44
45/*
46 * PMD_SHIFT determines the size a middle-level page table entry can map.
47 */
48#define PMD_SHIFT 21
49
50#define PMD_SIZE (1UL << PMD_SHIFT)
51#define PMD_MASK (~(PMD_SIZE-1))
52#define PGDIR_SIZE (1UL << PGDIR_SHIFT)
53#define PGDIR_MASK (~(PGDIR_SIZE-1))
54
55/*
56 * section address mask and size definitions.
57 */
58#define SECTION_SHIFT 21
59#define SECTION_SIZE (1UL << SECTION_SHIFT)
60#define SECTION_MASK (~(SECTION_SIZE-1))
61
62#define USER_PTRS_PER_PGD (PAGE_OFFSET / PGDIR_SIZE)
63
64/*
65 * "Linux" PTE definitions for LPAE.
66 *
67 * These bits overlap with the hardware bits but the naming is preserved for
68 * consistency with the classic page table format.
69 */
70#define L_PTE_PRESENT (_AT(pteval_t, 3) << 0) /* Valid */
71#define L_PTE_FILE (_AT(pteval_t, 1) << 2) /* only when !PRESENT */
72#define L_PTE_BUFFERABLE (_AT(pteval_t, 1) << 2) /* AttrIndx[0] */
73#define L_PTE_CACHEABLE (_AT(pteval_t, 1) << 3) /* AttrIndx[1] */
74#define L_PTE_USER (_AT(pteval_t, 1) << 6) /* AP[1] */
75#define L_PTE_RDONLY (_AT(pteval_t, 1) << 7) /* AP[2] */
76#define L_PTE_SHARED (_AT(pteval_t, 3) << 8) /* SH[1:0], inner shareable */
77#define L_PTE_YOUNG (_AT(pteval_t, 1) << 10) /* AF */
78#define L_PTE_XN (_AT(pteval_t, 1) << 54) /* XN */
79#define L_PTE_DIRTY (_AT(pteval_t, 1) << 55) /* unused */
80#define L_PTE_SPECIAL (_AT(pteval_t, 1) << 56) /* unused */
81
82/*
83 * To be used in assembly code with the upper page attributes.
84 */
85#define L_PTE_XN_HIGH (1 << (54 - 32))
86#define L_PTE_DIRTY_HIGH (1 << (55 - 32))
87
88/*
89 * AttrIndx[2:0] encoding (mapping attributes defined in the MAIR* registers).
90 */
91#define L_PTE_MT_UNCACHED (_AT(pteval_t, 0) << 2) /* strongly ordered */
92#define L_PTE_MT_BUFFERABLE (_AT(pteval_t, 1) << 2) /* normal non-cacheable */
93#define L_PTE_MT_WRITETHROUGH (_AT(pteval_t, 2) << 2) /* normal inner write-through */
94#define L_PTE_MT_WRITEBACK (_AT(pteval_t, 3) << 2) /* normal inner write-back */
95#define L_PTE_MT_WRITEALLOC (_AT(pteval_t, 7) << 2) /* normal inner write-alloc */
96#define L_PTE_MT_DEV_SHARED (_AT(pteval_t, 4) << 2) /* device */
97#define L_PTE_MT_DEV_NONSHARED (_AT(pteval_t, 4) << 2) /* device */
98#define L_PTE_MT_DEV_WC (_AT(pteval_t, 1) << 2) /* normal non-cacheable */
99#define L_PTE_MT_DEV_CACHED (_AT(pteval_t, 3) << 2) /* normal inner write-back */
100#define L_PTE_MT_MASK (_AT(pteval_t, 7) << 2)
101
102/*
103 * Software PGD flags.
104 */
105#define L_PGD_SWAPPER (_AT(pgdval_t, 1) << 55) /* swapper_pg_dir entry */
106
107#ifndef __ASSEMBLY__
108
109#define pud_none(pud) (!pud_val(pud))
110#define pud_bad(pud) (!(pud_val(pud) & 2))
111#define pud_present(pud) (pud_val(pud))
112
113#define pud_clear(pudp) \
114 do { \
115 *pudp = __pud(0); \
116 clean_pmd_entry(pudp); \
117 } while (0)
118
119#define set_pud(pudp, pud) \
120 do { \
121 *pudp = pud; \
122 flush_pmd_entry(pudp); \
123 } while (0)
124
125static inline pmd_t *pud_page_vaddr(pud_t pud)
126{
127 return __va(pud_val(pud) & PHYS_MASK & (s32)PAGE_MASK);
128}
129
130/* Find an entry in the second-level page table.. */
131#define pmd_index(addr) (((addr) >> PMD_SHIFT) & (PTRS_PER_PMD - 1))
132static inline pmd_t *pmd_offset(pud_t *pud, unsigned long addr)
133{
134 return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(addr);
135}
136
137#define pmd_bad(pmd) (!(pmd_val(pmd) & 2))
138
139#define copy_pmd(pmdpd,pmdps) \
140 do { \
141 *pmdpd = *pmdps; \
142 flush_pmd_entry(pmdpd); \
143 } while (0)
144
145#define pmd_clear(pmdp) \
146 do { \
147 *pmdp = __pmd(0); \
148 clean_pmd_entry(pmdp); \
149 } while (0)
150
151#define set_pte_ext(ptep,pte,ext) cpu_set_pte_ext(ptep,__pte(pte_val(pte)|(ext)))
152
153#endif /* __ASSEMBLY__ */
154
155#endif /* _ASM_PGTABLE_3LEVEL_H */
diff --git a/arch/arm/include/asm/pgtable-hwdef.h b/arch/arm/include/asm/pgtable-hwdef.h
index 183111164ce..8426229ba29 100644
--- a/arch/arm/include/asm/pgtable-hwdef.h
+++ b/arch/arm/include/asm/pgtable-hwdef.h
@@ -10,6 +10,10 @@
10#ifndef _ASMARM_PGTABLE_HWDEF_H 10#ifndef _ASMARM_PGTABLE_HWDEF_H
11#define _ASMARM_PGTABLE_HWDEF_H 11#define _ASMARM_PGTABLE_HWDEF_H
12 12
13#ifdef CONFIG_ARM_LPAE
14#include <asm/pgtable-3level-hwdef.h>
15#else
13#include <asm/pgtable-2level-hwdef.h> 16#include <asm/pgtable-2level-hwdef.h>
17#endif
14 18
15#endif 19#endif
diff --git a/arch/arm/include/asm/pgtable.h b/arch/arm/include/asm/pgtable.h
index a784859cc7a..3f2f0eb7621 100644
--- a/arch/arm/include/asm/pgtable.h
+++ b/arch/arm/include/asm/pgtable.h
@@ -11,19 +11,24 @@
11#define _ASMARM_PGTABLE_H 11#define _ASMARM_PGTABLE_H
12 12
13#include <linux/const.h> 13#include <linux/const.h>
14#include <asm-generic/4level-fixup.h>
15#include <asm/proc-fns.h> 14#include <asm/proc-fns.h>
16 15
17#ifndef CONFIG_MMU 16#ifndef CONFIG_MMU
18 17
18#include <asm-generic/4level-fixup.h>
19#include "pgtable-nommu.h" 19#include "pgtable-nommu.h"
20 20
21#else 21#else
22 22
23#include <asm-generic/pgtable-nopud.h>
23#include <asm/memory.h> 24#include <asm/memory.h>
24#include <asm/pgtable-hwdef.h> 25#include <asm/pgtable-hwdef.h>
25 26
27#ifdef CONFIG_ARM_LPAE
28#include <asm/pgtable-3level.h>
29#else
26#include <asm/pgtable-2level.h> 30#include <asm/pgtable-2level.h>
31#endif
27 32
28/* 33/*
29 * Just any arbitrary offset to the start of the vmalloc VM area: the 34 * Just any arbitrary offset to the start of the vmalloc VM area: the
@@ -164,39 +169,8 @@ extern pgd_t swapper_pg_dir[PTRS_PER_PGD];
164/* to find an entry in a kernel page-table-directory */ 169/* to find an entry in a kernel page-table-directory */
165#define pgd_offset_k(addr) pgd_offset(&init_mm, addr) 170#define pgd_offset_k(addr) pgd_offset(&init_mm, addr)
166 171
167/*
168 * The "pgd_xxx()" functions here are trivial for a folded two-level
169 * setup: the pgd is never bad, and a pmd always exists (as it's folded
170 * into the pgd entry)
171 */
172#define pgd_none(pgd) (0)
173#define pgd_bad(pgd) (0)
174#define pgd_present(pgd) (1)
175#define pgd_clear(pgdp) do { } while (0)
176#define set_pgd(pgd,pgdp) do { } while (0)
177#define set_pud(pud,pudp) do { } while (0)
178
179
180/* Find an entry in the second-level page table.. */
181#define pmd_offset(dir, addr) ((pmd_t *)(dir))
182
183#define pmd_none(pmd) (!pmd_val(pmd)) 172#define pmd_none(pmd) (!pmd_val(pmd))
184#define pmd_present(pmd) (pmd_val(pmd)) 173#define pmd_present(pmd) (pmd_val(pmd))
185#define pmd_bad(pmd) (pmd_val(pmd) & 2)
186
187#define copy_pmd(pmdpd,pmdps) \
188 do { \
189 pmdpd[0] = pmdps[0]; \
190 pmdpd[1] = pmdps[1]; \
191 flush_pmd_entry(pmdpd); \
192 } while (0)
193
194#define pmd_clear(pmdp) \
195 do { \
196 pmdp[0] = __pmd(0); \
197 pmdp[1] = __pmd(0); \
198 clean_pmd_entry(pmdp); \
199 } while (0)
200 174
201static inline pte_t *pmd_page_vaddr(pmd_t pmd) 175static inline pte_t *pmd_page_vaddr(pmd_t pmd)
202{ 176{
@@ -205,10 +179,6 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd)
205 179
206#define pmd_page(pmd) pfn_to_page(__phys_to_pfn(pmd_val(pmd) & PHYS_MASK)) 180#define pmd_page(pmd) pfn_to_page(__phys_to_pfn(pmd_val(pmd) & PHYS_MASK))
207 181
208/* we don't need complex calculations here as the pmd is folded into the pgd */
209#define pmd_addr_end(addr,end) (end)
210
211
212#ifndef CONFIG_HIGHPTE 182#ifndef CONFIG_HIGHPTE
213#define __pte_map(pmd) pmd_page_vaddr(*(pmd)) 183#define __pte_map(pmd) pmd_page_vaddr(*(pmd))
214#define __pte_unmap(pte) do { } while (0) 184#define __pte_unmap(pte) do { } while (0)
@@ -230,7 +200,6 @@ static inline pte_t *pmd_page_vaddr(pmd_t pmd)
230#define pte_page(pte) pfn_to_page(pte_pfn(pte)) 200#define pte_page(pte) pfn_to_page(pte_pfn(pte))
231#define mk_pte(page,prot) pfn_pte(page_to_pfn(page), prot) 201#define mk_pte(page,prot) pfn_pte(page_to_pfn(page), prot)
232 202
233#define set_pte_ext(ptep,pte,ext) cpu_set_pte_ext(ptep,pte,ext)
234#define pte_clear(mm,addr,ptep) set_pte_ext(ptep, __pte(0), 0) 203#define pte_clear(mm,addr,ptep) set_pte_ext(ptep, __pte(0), 0)
235 204
236#if __LINUX_ARM_ARCH__ < 6 205#if __LINUX_ARM_ARCH__ < 6
diff --git a/arch/arm/include/asm/proc-fns.h b/arch/arm/include/asm/proc-fns.h
index 9e92cb205e6..f3628fb3d2b 100644
--- a/arch/arm/include/asm/proc-fns.h
+++ b/arch/arm/include/asm/proc-fns.h
@@ -65,7 +65,11 @@ extern struct processor {
65 * Set a possibly extended PTE. Non-extended PTEs should 65 * Set a possibly extended PTE. Non-extended PTEs should
66 * ignore 'ext'. 66 * ignore 'ext'.
67 */ 67 */
68#ifdef CONFIG_ARM_LPAE
69 void (*set_pte_ext)(pte_t *ptep, pte_t pte);
70#else
68 void (*set_pte_ext)(pte_t *ptep, pte_t pte, unsigned int ext); 71 void (*set_pte_ext)(pte_t *ptep, pte_t pte, unsigned int ext);
72#endif
69 73
70 /* Suspend/resume */ 74 /* Suspend/resume */
71 unsigned int suspend_size; 75 unsigned int suspend_size;
@@ -79,7 +83,11 @@ extern void cpu_proc_fin(void);
79extern int cpu_do_idle(void); 83extern int cpu_do_idle(void);
80extern void cpu_dcache_clean_area(void *, int); 84extern void cpu_dcache_clean_area(void *, int);
81extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm); 85extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm);
86#ifdef CONFIG_ARM_LPAE
87extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte);
88#else
82extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext); 89extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte, unsigned int ext);
90#endif
83extern void cpu_reset(unsigned long addr) __attribute__((noreturn)); 91extern void cpu_reset(unsigned long addr) __attribute__((noreturn));
84 92
85/* These three are private to arch/arm/kernel/suspend.c */ 93/* These three are private to arch/arm/kernel/suspend.c */
@@ -107,6 +115,18 @@ extern void cpu_resume(void);
107 115
108#define cpu_switch_mm(pgd,mm) cpu_do_switch_mm(virt_to_phys(pgd),mm) 116#define cpu_switch_mm(pgd,mm) cpu_do_switch_mm(virt_to_phys(pgd),mm)
109 117
118#ifdef CONFIG_ARM_LPAE
119#define cpu_get_pgd() \
120 ({ \
121 unsigned long pg, pg2; \
122 __asm__("mrrc p15, 0, %0, %1, c2" \
123 : "=r" (pg), "=r" (pg2) \
124 : \
125 : "cc"); \
126 pg &= ~(PTRS_PER_PGD*sizeof(pgd_t)-1); \
127 (pgd_t *)phys_to_virt(pg); \
128 })
129#else
110#define cpu_get_pgd() \ 130#define cpu_get_pgd() \
111 ({ \ 131 ({ \
112 unsigned long pg; \ 132 unsigned long pg; \
@@ -115,6 +135,7 @@ extern void cpu_resume(void);
115 pg &= ~0x3fff; \ 135 pg &= ~0x3fff; \
116 (pgd_t *)phys_to_virt(pg); \ 136 (pgd_t *)phys_to_virt(pg); \
117 }) 137 })
138#endif
118 139
119#endif 140#endif
120 141
diff --git a/arch/arm/include/asm/system.h b/arch/arm/include/asm/system.h
index fe7de7571ba..53785828744 100644
--- a/arch/arm/include/asm/system.h
+++ b/arch/arm/include/asm/system.h
@@ -80,6 +80,14 @@ struct siginfo;
80void arm_notify_die(const char *str, struct pt_regs *regs, struct siginfo *info, 80void arm_notify_die(const char *str, struct pt_regs *regs, struct siginfo *info,
81 unsigned long err, unsigned long trap); 81 unsigned long err, unsigned long trap);
82 82
83#ifdef CONFIG_ARM_LPAE
84#define FAULT_CODE_ALIGNMENT 33
85#define FAULT_CODE_DEBUG 34
86#else
87#define FAULT_CODE_ALIGNMENT 1
88#define FAULT_CODE_DEBUG 2
89#endif
90
83void hook_fault_code(int nr, int (*fn)(unsigned long, unsigned int, 91void hook_fault_code(int nr, int (*fn)(unsigned long, unsigned int,
84 struct pt_regs *), 92 struct pt_regs *),
85 int sig, int code, const char *name); 93 int sig, int code, const char *name);
diff --git a/arch/arm/include/asm/tlb.h b/arch/arm/include/asm/tlb.h
index 265f908c4a6..5d3ed7e3856 100644
--- a/arch/arm/include/asm/tlb.h
+++ b/arch/arm/include/asm/tlb.h
@@ -202,8 +202,18 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
202 tlb_remove_page(tlb, pte); 202 tlb_remove_page(tlb, pte);
203} 203}
204 204
205static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
206 unsigned long addr)
207{
208#ifdef CONFIG_ARM_LPAE
209 tlb_add_flush(tlb, addr);
210 tlb_remove_page(tlb, virt_to_page(pmdp));
211#endif
212}
213
205#define pte_free_tlb(tlb, ptep, addr) __pte_free_tlb(tlb, ptep, addr) 214#define pte_free_tlb(tlb, ptep, addr) __pte_free_tlb(tlb, ptep, addr)
206#define pmd_free_tlb(tlb, pmdp, addr) pmd_free((tlb)->mm, pmdp) 215#define pmd_free_tlb(tlb, pmdp, addr) __pmd_free_tlb(tlb, pmdp, addr)
216#define pud_free_tlb(tlb, pudp, addr) pud_free((tlb)->mm, pudp)
207 217
208#define tlb_migrate_finish(mm) do { } while (0) 218#define tlb_migrate_finish(mm) do { } while (0)
209 219
diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S
index fb2945be5a5..14e277d2ff9 100644
--- a/arch/arm/kernel/head.S
+++ b/arch/arm/kernel/head.S
@@ -39,8 +39,14 @@
39#error KERNEL_RAM_VADDR must start at 0xXXXX8000 39#error KERNEL_RAM_VADDR must start at 0xXXXX8000
40#endif 40#endif
41 41
42#ifdef CONFIG_ARM_LPAE
43 /* LPAE requires an additional page for the PGD */
44#define PG_DIR_SIZE 0x5000
45#define PMD_ORDER 3
46#else
42#define PG_DIR_SIZE 0x4000 47#define PG_DIR_SIZE 0x4000
43#define PMD_ORDER 2 48#define PMD_ORDER 2
49#endif
44 50
45 .globl swapper_pg_dir 51 .globl swapper_pg_dir
46 .equ swapper_pg_dir, KERNEL_RAM_VADDR - PG_DIR_SIZE 52 .equ swapper_pg_dir, KERNEL_RAM_VADDR - PG_DIR_SIZE
@@ -164,6 +170,25 @@ __create_page_tables:
164 teq r0, r6 170 teq r0, r6
165 bne 1b 171 bne 1b
166 172
173#ifdef CONFIG_ARM_LPAE
174 /*
175 * Build the PGD table (first level) to point to the PMD table. A PGD
176 * entry is 64-bit wide.
177 */
178 mov r0, r4
179 add r3, r4, #0x1000 @ first PMD table address
180 orr r3, r3, #3 @ PGD block type
181 mov r6, #4 @ PTRS_PER_PGD
182 mov r7, #1 << (55 - 32) @ L_PGD_SWAPPER
1831: str r3, [r0], #4 @ set bottom PGD entry bits
184 str r7, [r0], #4 @ set top PGD entry bits
185 add r3, r3, #0x1000 @ next PMD table
186 subs r6, r6, #1
187 bne 1b
188
189 add r4, r4, #0x1000 @ point to the PMD tables
190#endif
191
167 ldr r7, [r10, #PROCINFO_MM_MMUFLAGS] @ mm_mmuflags 192 ldr r7, [r10, #PROCINFO_MM_MMUFLAGS] @ mm_mmuflags
168 193
169 /* 194 /*
@@ -219,8 +244,8 @@ __create_page_tables:
219#endif 244#endif
220 245
221 /* 246 /*
222 * Then map boot params address in r2 or 247 * Then map boot params address in r2 or the first 1MB (2MB with LPAE)
223 * the first 1MB of ram if boot params address is not specified. 248 * of ram if boot params address is not specified.
224 */ 249 */
225 mov r0, r2, lsr #SECTION_SHIFT 250 mov r0, r2, lsr #SECTION_SHIFT
226 movs r0, r0, lsl #SECTION_SHIFT 251 movs r0, r0, lsl #SECTION_SHIFT
@@ -251,7 +276,15 @@ __create_page_tables:
251 mov r3, r7, lsr #SECTION_SHIFT 276 mov r3, r7, lsr #SECTION_SHIFT
252 ldr r7, [r10, #PROCINFO_IO_MMUFLAGS] @ io_mmuflags 277 ldr r7, [r10, #PROCINFO_IO_MMUFLAGS] @ io_mmuflags
253 orr r3, r7, r3, lsl #SECTION_SHIFT 278 orr r3, r7, r3, lsl #SECTION_SHIFT
279#ifdef CONFIG_ARM_LPAE
280 mov r7, #1 << (54 - 32) @ XN
281#else
282 orr r3, r3, #PMD_SECT_XN
283#endif
2541: str r3, [r0], #4 2841: str r3, [r0], #4
285#ifdef CONFIG_ARM_LPAE
286 str r7, [r0], #4
287#endif
255 add r3, r3, #1 << SECTION_SHIFT 288 add r3, r3, #1 << SECTION_SHIFT
256 cmp r0, r6 289 cmp r0, r6
257 blo 1b 290 blo 1b
@@ -283,6 +316,9 @@ __create_page_tables:
283 str r3, [r0] 316 str r3, [r0]
284#endif 317#endif
285#endif 318#endif
319#ifdef CONFIG_ARM_LPAE
320 sub r4, r4, #0x1000 @ point to the PGD table
321#endif
286 mov pc, lr 322 mov pc, lr
287ENDPROC(__create_page_tables) 323ENDPROC(__create_page_tables)
288 .ltorg 324 .ltorg
@@ -374,12 +410,17 @@ __enable_mmu:
374#ifdef CONFIG_CPU_ICACHE_DISABLE 410#ifdef CONFIG_CPU_ICACHE_DISABLE
375 bic r0, r0, #CR_I 411 bic r0, r0, #CR_I
376#endif 412#endif
413#ifdef CONFIG_ARM_LPAE
414 mov r5, #0
415 mcrr p15, 0, r4, r5, c2 @ load TTBR0
416#else
377 mov r5, #(domain_val(DOMAIN_USER, DOMAIN_MANAGER) | \ 417 mov r5, #(domain_val(DOMAIN_USER, DOMAIN_MANAGER) | \
378 domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \ 418 domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \
379 domain_val(DOMAIN_TABLE, DOMAIN_MANAGER) | \ 419 domain_val(DOMAIN_TABLE, DOMAIN_MANAGER) | \
380 domain_val(DOMAIN_IO, DOMAIN_CLIENT)) 420 domain_val(DOMAIN_IO, DOMAIN_CLIENT))
381 mcr p15, 0, r5, c3, c0, 0 @ load domain access register 421 mcr p15, 0, r5, c3, c0, 0 @ load domain access register
382 mcr p15, 0, r4, c2, c0, 0 @ load page table pointer 422 mcr p15, 0, r4, c2, c0, 0 @ load page table pointer
423#endif
383 b __turn_mmu_on 424 b __turn_mmu_on
384ENDPROC(__enable_mmu) 425ENDPROC(__enable_mmu)
385 426
@@ -401,8 +442,10 @@ ENDPROC(__enable_mmu)
401 .pushsection .idmap.text, "ax" 442 .pushsection .idmap.text, "ax"
402ENTRY(__turn_mmu_on) 443ENTRY(__turn_mmu_on)
403 mov r0, r0 444 mov r0, r0
445 instr_sync
404 mcr p15, 0, r0, c1, c0, 0 @ write control reg 446 mcr p15, 0, r0, c1, c0, 0 @ write control reg
405 mrc p15, 0, r3, c0, c0, 0 @ read id reg 447 mrc p15, 0, r3, c0, c0, 0 @ read id reg
448 instr_sync
406 mov r3, r3 449 mov r3, r3
407 mov r3, r13 450 mov r3, r13
408 mov pc, r3 451 mov pc, r3
diff --git a/arch/arm/kernel/hw_breakpoint.c b/arch/arm/kernel/hw_breakpoint.c
index 814a52a9dc3..d6a95ef9131 100644
--- a/arch/arm/kernel/hw_breakpoint.c
+++ b/arch/arm/kernel/hw_breakpoint.c
@@ -1016,10 +1016,10 @@ static int __init arch_hw_breakpoint_init(void)
1016 } 1016 }
1017 1017
1018 /* Register debug fault handler. */ 1018 /* Register debug fault handler. */
1019 hook_fault_code(2, hw_breakpoint_pending, SIGTRAP, TRAP_HWBKPT, 1019 hook_fault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP,
1020 "watchpoint debug exception"); 1020 TRAP_HWBKPT, "watchpoint debug exception");
1021 hook_ifault_code(2, hw_breakpoint_pending, SIGTRAP, TRAP_HWBKPT, 1021 hook_ifault_code(FAULT_CODE_DEBUG, hw_breakpoint_pending, SIGTRAP,
1022 "breakpoint debug exception"); 1022 TRAP_HWBKPT, "breakpoint debug exception");
1023 1023
1024 /* Register hotplug notifier. */ 1024 /* Register hotplug notifier. */
1025 register_cpu_notifier(&dbg_reset_nb); 1025 register_cpu_notifier(&dbg_reset_nb);
diff --git a/arch/arm/kernel/sleep.S b/arch/arm/kernel/sleep.S
index 9e64231c8cf..1f268bda455 100644
--- a/arch/arm/kernel/sleep.S
+++ b/arch/arm/kernel/sleep.S
@@ -57,8 +57,10 @@ ENDPROC(cpu_suspend_abort)
57 .pushsection .idmap.text,"ax" 57 .pushsection .idmap.text,"ax"
58ENTRY(cpu_resume_mmu) 58ENTRY(cpu_resume_mmu)
59 ldr r3, =cpu_resume_after_mmu 59 ldr r3, =cpu_resume_after_mmu
60 instr_sync
60 mcr p15, 0, r0, c1, c0, 0 @ turn on MMU, I-cache, etc 61 mcr p15, 0, r0, c1, c0, 0 @ turn on MMU, I-cache, etc
61 mrc p15, 0, r0, c0, c0, 0 @ read id reg 62 mrc p15, 0, r0, c0, c0, 0 @ read id reg
63 instr_sync
62 mov r0, r0 64 mov r0, r0
63 mov r0, r0 65 mov r0, r0
64 mov pc, r3 @ jump to virtual address 66 mov pc, r3 @ jump to virtual address
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
index 67f75a0b66d..5cf7922ff5e 100644
--- a/arch/arm/mm/Kconfig
+++ b/arch/arm/mm/Kconfig
@@ -629,6 +629,23 @@ config IO_36
629 629
630comment "Processor Features" 630comment "Processor Features"
631 631
632config ARM_LPAE
633 bool "Support for the Large Physical Address Extension"
634 depends on MMU && CPU_V7
635 help
636 Say Y if you have an ARMv7 processor supporting the LPAE page
637 table format and you would like to access memory beyond the
638 4GB limit. The resulting kernel image will not run on
639 processors without the LPA extension.
640
641 If unsure, say N.
642
643config ARCH_PHYS_ADDR_T_64BIT
644 def_bool ARM_LPAE
645
646config ARCH_DMA_ADDR_T_64BIT
647 bool
648
632config ARM_THUMB 649config ARM_THUMB
633 bool "Support Thumb user binaries" 650 bool "Support Thumb user binaries"
634 depends on CPU_ARM720T || CPU_ARM740T || CPU_ARM920T || CPU_ARM922T || CPU_ARM925T || CPU_ARM926T || CPU_ARM940T || CPU_ARM946E || CPU_ARM1020 || CPU_ARM1020E || CPU_ARM1022 || CPU_ARM1026 || CPU_XSCALE || CPU_XSC3 || CPU_MOHAWK || CPU_V6 || CPU_V6K || CPU_V7 || CPU_FEROCEON 651 depends on CPU_ARM720T || CPU_ARM740T || CPU_ARM920T || CPU_ARM922T || CPU_ARM925T || CPU_ARM926T || CPU_ARM940T || CPU_ARM946E || CPU_ARM1020 || CPU_ARM1020E || CPU_ARM1022 || CPU_ARM1026 || CPU_XSCALE || CPU_XSC3 || CPU_MOHAWK || CPU_V6 || CPU_V6K || CPU_V7 || CPU_FEROCEON
diff --git a/arch/arm/mm/alignment.c b/arch/arm/mm/alignment.c
index c335c76e0d8..caf14dc059e 100644
--- a/arch/arm/mm/alignment.c
+++ b/arch/arm/mm/alignment.c
@@ -968,7 +968,7 @@ static int __init alignment_init(void)
968 ai_usermode = safe_usermode(ai_usermode, false); 968 ai_usermode = safe_usermode(ai_usermode, false);
969 } 969 }
970 970
971 hook_fault_code(1, do_alignment, SIGBUS, BUS_ADRALN, 971 hook_fault_code(FAULT_CODE_ALIGNMENT, do_alignment, SIGBUS, BUS_ADRALN,
972 "alignment exception"); 972 "alignment exception");
973 973
974 /* 974 /*
diff --git a/arch/arm/mm/context.c b/arch/arm/mm/context.c
index 93aac068da9..ee9bb363d60 100644
--- a/arch/arm/mm/context.c
+++ b/arch/arm/mm/context.c
@@ -22,6 +22,21 @@ unsigned int cpu_last_asid = ASID_FIRST_VERSION;
22DEFINE_PER_CPU(struct mm_struct *, current_mm); 22DEFINE_PER_CPU(struct mm_struct *, current_mm);
23#endif 23#endif
24 24
25#ifdef CONFIG_ARM_LPAE
26#define cpu_set_asid(asid) { \
27 unsigned long ttbl, ttbh; \
28 asm volatile( \
29 " mrrc p15, 0, %0, %1, c2 @ read TTBR0\n" \
30 " mov %1, %2, lsl #(48 - 32) @ set ASID\n" \
31 " mcrr p15, 0, %0, %1, c2 @ set TTBR0\n" \
32 : "=&r" (ttbl), "=&r" (ttbh) \
33 : "r" (asid & ~ASID_MASK)); \
34}
35#else
36#define cpu_set_asid(asid) \
37 asm(" mcr p15, 0, %0, c13, c0, 1\n" : : "r" (asid))
38#endif
39
25/* 40/*
26 * We fork()ed a process, and we need a new context for the child 41 * We fork()ed a process, and we need a new context for the child
27 * to run in. We reserve version 0 for initial tasks so we will 42 * to run in. We reserve version 0 for initial tasks so we will
@@ -37,7 +52,7 @@ void __init_new_context(struct task_struct *tsk, struct mm_struct *mm)
37static void flush_context(void) 52static void flush_context(void)
38{ 53{
39 /* set the reserved ASID before flushing the TLB */ 54 /* set the reserved ASID before flushing the TLB */
40 asm("mcr p15, 0, %0, c13, c0, 1\n" : : "r" (0)); 55 cpu_set_asid(0);
41 isb(); 56 isb();
42 local_flush_tlb_all(); 57 local_flush_tlb_all();
43 if (icache_is_vivt_asid_tagged()) { 58 if (icache_is_vivt_asid_tagged()) {
@@ -99,7 +114,7 @@ static void reset_context(void *info)
99 set_mm_context(mm, asid); 114 set_mm_context(mm, asid);
100 115
101 /* set the new ASID */ 116 /* set the new ASID */
102 asm("mcr p15, 0, %0, c13, c0, 1\n" : : "r" (mm->context.id)); 117 cpu_set_asid(mm->context.id);
103 isb(); 118 isb();
104} 119}
105 120
diff --git a/arch/arm/mm/fault.c b/arch/arm/mm/fault.c
index aa33949fef6..eb5520fc755 100644
--- a/arch/arm/mm/fault.c
+++ b/arch/arm/mm/fault.c
@@ -27,19 +27,6 @@
27 27
28#include "fault.h" 28#include "fault.h"
29 29
30/*
31 * Fault status register encodings. We steal bit 31 for our own purposes.
32 */
33#define FSR_LNX_PF (1 << 31)
34#define FSR_WRITE (1 << 11)
35#define FSR_FS4 (1 << 10)
36#define FSR_FS3_0 (15)
37
38static inline int fsr_fs(unsigned int fsr)
39{
40 return (fsr & FSR_FS3_0) | (fsr & FSR_FS4) >> 6;
41}
42
43#ifdef CONFIG_MMU 30#ifdef CONFIG_MMU
44 31
45#ifdef CONFIG_KPROBES 32#ifdef CONFIG_KPROBES
@@ -123,8 +110,10 @@ void show_pte(struct mm_struct *mm, unsigned long addr)
123 110
124 pte = pte_offset_map(pmd, addr); 111 pte = pte_offset_map(pmd, addr);
125 printk(", *pte=%08llx", (long long)pte_val(*pte)); 112 printk(", *pte=%08llx", (long long)pte_val(*pte));
113#ifndef CONFIG_ARM_LPAE
126 printk(", *ppte=%08llx", 114 printk(", *ppte=%08llx",
127 (long long)pte_val(pte[PTE_HWTABLE_PTRS])); 115 (long long)pte_val(pte[PTE_HWTABLE_PTRS]));
116#endif
128 pte_unmap(pte); 117 pte_unmap(pte);
129 } while(0); 118 } while(0);
130 119
@@ -441,6 +430,12 @@ do_translation_fault(unsigned long addr, unsigned int fsr,
441 pmd = pmd_offset(pud, addr); 430 pmd = pmd_offset(pud, addr);
442 pmd_k = pmd_offset(pud_k, addr); 431 pmd_k = pmd_offset(pud_k, addr);
443 432
433#ifdef CONFIG_ARM_LPAE
434 /*
435 * Only one hardware entry per PMD with LPAE.
436 */
437 index = 0;
438#else
444 /* 439 /*
445 * On ARM one Linux PGD entry contains two hardware entries (see page 440 * On ARM one Linux PGD entry contains two hardware entries (see page
446 * tables layout in pgtable.h). We normally guarantee that we always 441 * tables layout in pgtable.h). We normally guarantee that we always
@@ -450,6 +445,7 @@ do_translation_fault(unsigned long addr, unsigned int fsr,
450 * for the first of pair. 445 * for the first of pair.
451 */ 446 */
452 index = (addr >> SECTION_SHIFT) & 1; 447 index = (addr >> SECTION_SHIFT) & 1;
448#endif
453 if (pmd_none(pmd_k[index])) 449 if (pmd_none(pmd_k[index]))
454 goto bad_area; 450 goto bad_area;
455 451
@@ -489,55 +485,20 @@ do_bad(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
489 return 1; 485 return 1;
490} 486}
491 487
492static struct fsr_info { 488struct fsr_info {
493 int (*fn)(unsigned long addr, unsigned int fsr, struct pt_regs *regs); 489 int (*fn)(unsigned long addr, unsigned int fsr, struct pt_regs *regs);
494 int sig; 490 int sig;
495 int code; 491 int code;
496 const char *name; 492 const char *name;
497} fsr_info[] = {
498 /*
499 * The following are the standard ARMv3 and ARMv4 aborts. ARMv5
500 * defines these to be "precise" aborts.
501 */
502 { do_bad, SIGSEGV, 0, "vector exception" },
503 { do_bad, SIGBUS, BUS_ADRALN, "alignment exception" },
504 { do_bad, SIGKILL, 0, "terminal exception" },
505 { do_bad, SIGBUS, BUS_ADRALN, "alignment exception" },
506 { do_bad, SIGBUS, 0, "external abort on linefetch" },
507 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "section translation fault" },
508 { do_bad, SIGBUS, 0, "external abort on linefetch" },
509 { do_page_fault, SIGSEGV, SEGV_MAPERR, "page translation fault" },
510 { do_bad, SIGBUS, 0, "external abort on non-linefetch" },
511 { do_bad, SIGSEGV, SEGV_ACCERR, "section domain fault" },
512 { do_bad, SIGBUS, 0, "external abort on non-linefetch" },
513 { do_bad, SIGSEGV, SEGV_ACCERR, "page domain fault" },
514 { do_bad, SIGBUS, 0, "external abort on translation" },
515 { do_sect_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" },
516 { do_bad, SIGBUS, 0, "external abort on translation" },
517 { do_page_fault, SIGSEGV, SEGV_ACCERR, "page permission fault" },
518 /*
519 * The following are "imprecise" aborts, which are signalled by bit
520 * 10 of the FSR, and may not be recoverable. These are only
521 * supported if the CPU abort handler supports bit 10.
522 */
523 { do_bad, SIGBUS, 0, "unknown 16" },
524 { do_bad, SIGBUS, 0, "unknown 17" },
525 { do_bad, SIGBUS, 0, "unknown 18" },
526 { do_bad, SIGBUS, 0, "unknown 19" },
527 { do_bad, SIGBUS, 0, "lock abort" }, /* xscale */
528 { do_bad, SIGBUS, 0, "unknown 21" },
529 { do_bad, SIGBUS, BUS_OBJERR, "imprecise external abort" }, /* xscale */
530 { do_bad, SIGBUS, 0, "unknown 23" },
531 { do_bad, SIGBUS, 0, "dcache parity error" }, /* xscale */
532 { do_bad, SIGBUS, 0, "unknown 25" },
533 { do_bad, SIGBUS, 0, "unknown 26" },
534 { do_bad, SIGBUS, 0, "unknown 27" },
535 { do_bad, SIGBUS, 0, "unknown 28" },
536 { do_bad, SIGBUS, 0, "unknown 29" },
537 { do_bad, SIGBUS, 0, "unknown 30" },
538 { do_bad, SIGBUS, 0, "unknown 31" }
539}; 493};
540 494
495/* FSR definition */
496#ifdef CONFIG_ARM_LPAE
497#include "fsr-3level.c"
498#else
499#include "fsr-2level.c"
500#endif
501
541void __init 502void __init
542hook_fault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *), 503hook_fault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *),
543 int sig, int code, const char *name) 504 int sig, int code, const char *name)
@@ -573,42 +534,6 @@ do_DataAbort(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
573 arm_notify_die("", regs, &info, fsr, 0); 534 arm_notify_die("", regs, &info, fsr, 0);
574} 535}
575 536
576
577static struct fsr_info ifsr_info[] = {
578 { do_bad, SIGBUS, 0, "unknown 0" },
579 { do_bad, SIGBUS, 0, "unknown 1" },
580 { do_bad, SIGBUS, 0, "debug event" },
581 { do_bad, SIGSEGV, SEGV_ACCERR, "section access flag fault" },
582 { do_bad, SIGBUS, 0, "unknown 4" },
583 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "section translation fault" },
584 { do_bad, SIGSEGV, SEGV_ACCERR, "page access flag fault" },
585 { do_page_fault, SIGSEGV, SEGV_MAPERR, "page translation fault" },
586 { do_bad, SIGBUS, 0, "external abort on non-linefetch" },
587 { do_bad, SIGSEGV, SEGV_ACCERR, "section domain fault" },
588 { do_bad, SIGBUS, 0, "unknown 10" },
589 { do_bad, SIGSEGV, SEGV_ACCERR, "page domain fault" },
590 { do_bad, SIGBUS, 0, "external abort on translation" },
591 { do_sect_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" },
592 { do_bad, SIGBUS, 0, "external abort on translation" },
593 { do_page_fault, SIGSEGV, SEGV_ACCERR, "page permission fault" },
594 { do_bad, SIGBUS, 0, "unknown 16" },
595 { do_bad, SIGBUS, 0, "unknown 17" },
596 { do_bad, SIGBUS, 0, "unknown 18" },
597 { do_bad, SIGBUS, 0, "unknown 19" },
598 { do_bad, SIGBUS, 0, "unknown 20" },
599 { do_bad, SIGBUS, 0, "unknown 21" },
600 { do_bad, SIGBUS, 0, "unknown 22" },
601 { do_bad, SIGBUS, 0, "unknown 23" },
602 { do_bad, SIGBUS, 0, "unknown 24" },
603 { do_bad, SIGBUS, 0, "unknown 25" },
604 { do_bad, SIGBUS, 0, "unknown 26" },
605 { do_bad, SIGBUS, 0, "unknown 27" },
606 { do_bad, SIGBUS, 0, "unknown 28" },
607 { do_bad, SIGBUS, 0, "unknown 29" },
608 { do_bad, SIGBUS, 0, "unknown 30" },
609 { do_bad, SIGBUS, 0, "unknown 31" },
610};
611
612void __init 537void __init
613hook_ifault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *), 538hook_ifault_code(int nr, int (*fn)(unsigned long, unsigned int, struct pt_regs *),
614 int sig, int code, const char *name) 539 int sig, int code, const char *name)
@@ -641,6 +566,7 @@ do_PrefetchAbort(unsigned long addr, unsigned int ifsr, struct pt_regs *regs)
641 arm_notify_die("", regs, &info, ifsr, 0); 566 arm_notify_die("", regs, &info, ifsr, 0);
642} 567}
643 568
569#ifndef CONFIG_ARM_LPAE
644static int __init exceptions_init(void) 570static int __init exceptions_init(void)
645{ 571{
646 if (cpu_architecture() >= CPU_ARCH_ARMv6) { 572 if (cpu_architecture() >= CPU_ARCH_ARMv6) {
@@ -663,3 +589,4 @@ static int __init exceptions_init(void)
663} 589}
664 590
665arch_initcall(exceptions_init); 591arch_initcall(exceptions_init);
592#endif
diff --git a/arch/arm/mm/fault.h b/arch/arm/mm/fault.h
index 49e9e3804de..cf08bdfbe0d 100644
--- a/arch/arm/mm/fault.h
+++ b/arch/arm/mm/fault.h
@@ -1,3 +1,28 @@
1void do_bad_area(unsigned long addr, unsigned int fsr, struct pt_regs *regs); 1#ifndef __ARCH_ARM_FAULT_H
2#define __ARCH_ARM_FAULT_H
3
4/*
5 * Fault status register encodings. We steal bit 31 for our own purposes.
6 */
7#define FSR_LNX_PF (1 << 31)
8#define FSR_WRITE (1 << 11)
9#define FSR_FS4 (1 << 10)
10#define FSR_FS3_0 (15)
11#define FSR_FS5_0 (0x3f)
12
13#ifdef CONFIG_ARM_LPAE
14static inline int fsr_fs(unsigned int fsr)
15{
16 return fsr & FSR_FS5_0;
17}
18#else
19static inline int fsr_fs(unsigned int fsr)
20{
21 return (fsr & FSR_FS3_0) | (fsr & FSR_FS4) >> 6;
22}
23#endif
2 24
25void do_bad_area(unsigned long addr, unsigned int fsr, struct pt_regs *regs);
3unsigned long search_exception_table(unsigned long addr); 26unsigned long search_exception_table(unsigned long addr);
27
28#endif /* __ARCH_ARM_FAULT_H */
diff --git a/arch/arm/mm/fsr-2level.c b/arch/arm/mm/fsr-2level.c
new file mode 100644
index 00000000000..18ca74c0f34
--- /dev/null
+++ b/arch/arm/mm/fsr-2level.c
@@ -0,0 +1,78 @@
1static struct fsr_info fsr_info[] = {
2 /*
3 * The following are the standard ARMv3 and ARMv4 aborts. ARMv5
4 * defines these to be "precise" aborts.
5 */
6 { do_bad, SIGSEGV, 0, "vector exception" },
7 { do_bad, SIGBUS, BUS_ADRALN, "alignment exception" },
8 { do_bad, SIGKILL, 0, "terminal exception" },
9 { do_bad, SIGBUS, BUS_ADRALN, "alignment exception" },
10 { do_bad, SIGBUS, 0, "external abort on linefetch" },
11 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "section translation fault" },
12 { do_bad, SIGBUS, 0, "external abort on linefetch" },
13 { do_page_fault, SIGSEGV, SEGV_MAPERR, "page translation fault" },
14 { do_bad, SIGBUS, 0, "external abort on non-linefetch" },
15 { do_bad, SIGSEGV, SEGV_ACCERR, "section domain fault" },
16 { do_bad, SIGBUS, 0, "external abort on non-linefetch" },
17 { do_bad, SIGSEGV, SEGV_ACCERR, "page domain fault" },
18 { do_bad, SIGBUS, 0, "external abort on translation" },
19 { do_sect_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" },
20 { do_bad, SIGBUS, 0, "external abort on translation" },
21 { do_page_fault, SIGSEGV, SEGV_ACCERR, "page permission fault" },
22 /*
23 * The following are "imprecise" aborts, which are signalled by bit
24 * 10 of the FSR, and may not be recoverable. These are only
25 * supported if the CPU abort handler supports bit 10.
26 */
27 { do_bad, SIGBUS, 0, "unknown 16" },
28 { do_bad, SIGBUS, 0, "unknown 17" },
29 { do_bad, SIGBUS, 0, "unknown 18" },
30 { do_bad, SIGBUS, 0, "unknown 19" },
31 { do_bad, SIGBUS, 0, "lock abort" }, /* xscale */
32 { do_bad, SIGBUS, 0, "unknown 21" },
33 { do_bad, SIGBUS, BUS_OBJERR, "imprecise external abort" }, /* xscale */
34 { do_bad, SIGBUS, 0, "unknown 23" },
35 { do_bad, SIGBUS, 0, "dcache parity error" }, /* xscale */
36 { do_bad, SIGBUS, 0, "unknown 25" },
37 { do_bad, SIGBUS, 0, "unknown 26" },
38 { do_bad, SIGBUS, 0, "unknown 27" },
39 { do_bad, SIGBUS, 0, "unknown 28" },
40 { do_bad, SIGBUS, 0, "unknown 29" },
41 { do_bad, SIGBUS, 0, "unknown 30" },
42 { do_bad, SIGBUS, 0, "unknown 31" },
43};
44
45static struct fsr_info ifsr_info[] = {
46 { do_bad, SIGBUS, 0, "unknown 0" },
47 { do_bad, SIGBUS, 0, "unknown 1" },
48 { do_bad, SIGBUS, 0, "debug event" },
49 { do_bad, SIGSEGV, SEGV_ACCERR, "section access flag fault" },
50 { do_bad, SIGBUS, 0, "unknown 4" },
51 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "section translation fault" },
52 { do_bad, SIGSEGV, SEGV_ACCERR, "page access flag fault" },
53 { do_page_fault, SIGSEGV, SEGV_MAPERR, "page translation fault" },
54 { do_bad, SIGBUS, 0, "external abort on non-linefetch" },
55 { do_bad, SIGSEGV, SEGV_ACCERR, "section domain fault" },
56 { do_bad, SIGBUS, 0, "unknown 10" },
57 { do_bad, SIGSEGV, SEGV_ACCERR, "page domain fault" },
58 { do_bad, SIGBUS, 0, "external abort on translation" },
59 { do_sect_fault, SIGSEGV, SEGV_ACCERR, "section permission fault" },
60 { do_bad, SIGBUS, 0, "external abort on translation" },
61 { do_page_fault, SIGSEGV, SEGV_ACCERR, "page permission fault" },
62 { do_bad, SIGBUS, 0, "unknown 16" },
63 { do_bad, SIGBUS, 0, "unknown 17" },
64 { do_bad, SIGBUS, 0, "unknown 18" },
65 { do_bad, SIGBUS, 0, "unknown 19" },
66 { do_bad, SIGBUS, 0, "unknown 20" },
67 { do_bad, SIGBUS, 0, "unknown 21" },
68 { do_bad, SIGBUS, 0, "unknown 22" },
69 { do_bad, SIGBUS, 0, "unknown 23" },
70 { do_bad, SIGBUS, 0, "unknown 24" },
71 { do_bad, SIGBUS, 0, "unknown 25" },
72 { do_bad, SIGBUS, 0, "unknown 26" },
73 { do_bad, SIGBUS, 0, "unknown 27" },
74 { do_bad, SIGBUS, 0, "unknown 28" },
75 { do_bad, SIGBUS, 0, "unknown 29" },
76 { do_bad, SIGBUS, 0, "unknown 30" },
77 { do_bad, SIGBUS, 0, "unknown 31" },
78};
diff --git a/arch/arm/mm/fsr-3level.c b/arch/arm/mm/fsr-3level.c
new file mode 100644
index 00000000000..05a4e943183
--- /dev/null
+++ b/arch/arm/mm/fsr-3level.c
@@ -0,0 +1,68 @@
1static struct fsr_info fsr_info[] = {
2 { do_bad, SIGBUS, 0, "unknown 0" },
3 { do_bad, SIGBUS, 0, "unknown 1" },
4 { do_bad, SIGBUS, 0, "unknown 2" },
5 { do_bad, SIGBUS, 0, "unknown 3" },
6 { do_bad, SIGBUS, 0, "reserved translation fault" },
7 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 1 translation fault" },
8 { do_translation_fault, SIGSEGV, SEGV_MAPERR, "level 2 translation fault" },
9 { do_page_fault, SIGSEGV, SEGV_MAPERR, "level 3 translation fault" },
10 { do_bad, SIGBUS, 0, "reserved access flag fault" },
11 { do_bad, SIGSEGV, SEGV_ACCERR, "level 1 access flag fault" },
12 { do_bad, SIGSEGV, SEGV_ACCERR, "level 2 access flag fault" },
13 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 access flag fault" },
14 { do_bad, SIGBUS, 0, "reserved permission fault" },
15 { do_bad, SIGSEGV, SEGV_ACCERR, "level 1 permission fault" },
16 { do_sect_fault, SIGSEGV, SEGV_ACCERR, "level 2 permission fault" },
17 { do_page_fault, SIGSEGV, SEGV_ACCERR, "level 3 permission fault" },
18 { do_bad, SIGBUS, 0, "synchronous external abort" },
19 { do_bad, SIGBUS, 0, "asynchronous external abort" },
20 { do_bad, SIGBUS, 0, "unknown 18" },
21 { do_bad, SIGBUS, 0, "unknown 19" },
22 { do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" },
23 { do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" },
24 { do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" },
25 { do_bad, SIGBUS, 0, "synchronous abort (translation table walk)" },
26 { do_bad, SIGBUS, 0, "synchronous parity error" },
27 { do_bad, SIGBUS, 0, "asynchronous parity error" },
28 { do_bad, SIGBUS, 0, "unknown 26" },
29 { do_bad, SIGBUS, 0, "unknown 27" },
30 { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk" },
31 { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk" },
32 { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk" },
33 { do_bad, SIGBUS, 0, "synchronous parity error (translation table walk" },
34 { do_bad, SIGBUS, 0, "unknown 32" },
35 { do_bad, SIGBUS, BUS_ADRALN, "alignment fault" },
36 { do_bad, SIGBUS, 0, "debug event" },
37 { do_bad, SIGBUS, 0, "unknown 35" },
38 { do_bad, SIGBUS, 0, "unknown 36" },
39 { do_bad, SIGBUS, 0, "unknown 37" },
40 { do_bad, SIGBUS, 0, "unknown 38" },
41 { do_bad, SIGBUS, 0, "unknown 39" },
42 { do_bad, SIGBUS, 0, "unknown 40" },
43 { do_bad, SIGBUS, 0, "unknown 41" },
44 { do_bad, SIGBUS, 0, "unknown 42" },
45 { do_bad, SIGBUS, 0, "unknown 43" },
46 { do_bad, SIGBUS, 0, "unknown 44" },
47 { do_bad, SIGBUS, 0, "unknown 45" },
48 { do_bad, SIGBUS, 0, "unknown 46" },
49 { do_bad, SIGBUS, 0, "unknown 47" },
50 { do_bad, SIGBUS, 0, "unknown 48" },
51 { do_bad, SIGBUS, 0, "unknown 49" },
52 { do_bad, SIGBUS, 0, "unknown 50" },
53 { do_bad, SIGBUS, 0, "unknown 51" },
54 { do_bad, SIGBUS, 0, "implementation fault (lockdown abort)" },
55 { do_bad, SIGBUS, 0, "unknown 53" },
56 { do_bad, SIGBUS, 0, "unknown 54" },
57 { do_bad, SIGBUS, 0, "unknown 55" },
58 { do_bad, SIGBUS, 0, "unknown 56" },
59 { do_bad, SIGBUS, 0, "unknown 57" },
60 { do_bad, SIGBUS, 0, "implementation fault (coprocessor abort)" },
61 { do_bad, SIGBUS, 0, "unknown 59" },
62 { do_bad, SIGBUS, 0, "unknown 60" },
63 { do_bad, SIGBUS, 0, "unknown 61" },
64 { do_bad, SIGBUS, 0, "unknown 62" },
65 { do_bad, SIGBUS, 0, "unknown 63" },
66};
67
68#define ifsr_info fsr_info
diff --git a/arch/arm/mm/idmap.c b/arch/arm/mm/idmap.c
index 660f1bc68f9..feacf4c7671 100644
--- a/arch/arm/mm/idmap.c
+++ b/arch/arm/mm/idmap.c
@@ -8,6 +8,31 @@
8 8
9pgd_t *idmap_pgd; 9pgd_t *idmap_pgd;
10 10
11#ifdef CONFIG_ARM_LPAE
12static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end,
13 unsigned long prot)
14{
15 pmd_t *pmd;
16 unsigned long next;
17
18 if (pud_none_or_clear_bad(pud) || (pud_val(*pud) & L_PGD_SWAPPER)) {
19 pmd = pmd_alloc_one(&init_mm, addr);
20 if (!pmd) {
21 pr_warning("Failed to allocate identity pmd.\n");
22 return;
23 }
24 pud_populate(&init_mm, pud, pmd);
25 pmd += pmd_index(addr);
26 } else
27 pmd = pmd_offset(pud, addr);
28
29 do {
30 next = pmd_addr_end(addr, end);
31 *pmd = __pmd((addr & PMD_MASK) | prot);
32 flush_pmd_entry(pmd);
33 } while (pmd++, addr = next, addr != end);
34}
35#else /* !CONFIG_ARM_LPAE */
11static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end, 36static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end,
12 unsigned long prot) 37 unsigned long prot)
13{ 38{
@@ -19,6 +44,7 @@ static void idmap_add_pmd(pud_t *pud, unsigned long addr, unsigned long end,
19 pmd[1] = __pmd(addr); 44 pmd[1] = __pmd(addr);
20 flush_pmd_entry(pmd); 45 flush_pmd_entry(pmd);
21} 46}
47#endif /* CONFIG_ARM_LPAE */
22 48
23static void idmap_add_pud(pgd_t *pgd, unsigned long addr, unsigned long end, 49static void idmap_add_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
24 unsigned long prot) 50 unsigned long prot)
@@ -36,7 +62,7 @@ static void identity_mapping_add(pgd_t *pgd, unsigned long addr, unsigned long e
36{ 62{
37 unsigned long prot, next; 63 unsigned long prot, next;
38 64
39 prot = PMD_TYPE_SECT | PMD_SECT_AP_WRITE; 65 prot = PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AF;
40 if (cpu_architecture() <= CPU_ARCH_ARMv5TEJ && !cpu_is_xscale()) 66 if (cpu_architecture() <= CPU_ARCH_ARMv5TEJ && !cpu_is_xscale())
41 prot |= PMD_BIT4; 67 prot |= PMD_BIT4;
42 68
diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c
index 12c7ad215ce..80632e8d753 100644
--- a/arch/arm/mm/ioremap.c
+++ b/arch/arm/mm/ioremap.c
@@ -58,7 +58,7 @@ void __check_kvm_seq(struct mm_struct *mm)
58 } while (seq != init_mm.context.kvm_seq); 58 } while (seq != init_mm.context.kvm_seq);
59} 59}
60 60
61#ifndef CONFIG_SMP 61#if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE)
62/* 62/*
63 * Section support is unsafe on SMP - If you iounmap and ioremap a region, 63 * Section support is unsafe on SMP - If you iounmap and ioremap a region,
64 * the other CPUs will not see this change until their next context switch. 64 * the other CPUs will not see this change until their next context switch.
@@ -73,13 +73,16 @@ static void unmap_area_sections(unsigned long virt, unsigned long size)
73{ 73{
74 unsigned long addr = virt, end = virt + (size & ~(SZ_1M - 1)); 74 unsigned long addr = virt, end = virt + (size & ~(SZ_1M - 1));
75 pgd_t *pgd; 75 pgd_t *pgd;
76 pud_t *pud;
77 pmd_t *pmdp;
76 78
77 flush_cache_vunmap(addr, end); 79 flush_cache_vunmap(addr, end);
78 pgd = pgd_offset_k(addr); 80 pgd = pgd_offset_k(addr);
81 pud = pud_offset(pgd, addr);
82 pmdp = pmd_offset(pud, addr);
79 do { 83 do {
80 pmd_t pmd, *pmdp = pmd_offset(pgd, addr); 84 pmd_t pmd = *pmdp;
81 85
82 pmd = *pmdp;
83 if (!pmd_none(pmd)) { 86 if (!pmd_none(pmd)) {
84 /* 87 /*
85 * Clear the PMD from the page table, and 88 * Clear the PMD from the page table, and
@@ -98,8 +101,8 @@ static void unmap_area_sections(unsigned long virt, unsigned long size)
98 pte_free_kernel(&init_mm, pmd_page_vaddr(pmd)); 101 pte_free_kernel(&init_mm, pmd_page_vaddr(pmd));
99 } 102 }
100 103
101 addr += PGDIR_SIZE; 104 addr += PMD_SIZE;
102 pgd++; 105 pmdp += 2;
103 } while (addr < end); 106 } while (addr < end);
104 107
105 /* 108 /*
@@ -118,6 +121,8 @@ remap_area_sections(unsigned long virt, unsigned long pfn,
118{ 121{
119 unsigned long addr = virt, end = virt + size; 122 unsigned long addr = virt, end = virt + size;
120 pgd_t *pgd; 123 pgd_t *pgd;
124 pud_t *pud;
125 pmd_t *pmd;
121 126
122 /* 127 /*
123 * Remove and free any PTE-based mapping, and 128 * Remove and free any PTE-based mapping, and
@@ -126,17 +131,17 @@ remap_area_sections(unsigned long virt, unsigned long pfn,
126 unmap_area_sections(virt, size); 131 unmap_area_sections(virt, size);
127 132
128 pgd = pgd_offset_k(addr); 133 pgd = pgd_offset_k(addr);
134 pud = pud_offset(pgd, addr);
135 pmd = pmd_offset(pud, addr);
129 do { 136 do {
130 pmd_t *pmd = pmd_offset(pgd, addr);
131
132 pmd[0] = __pmd(__pfn_to_phys(pfn) | type->prot_sect); 137 pmd[0] = __pmd(__pfn_to_phys(pfn) | type->prot_sect);
133 pfn += SZ_1M >> PAGE_SHIFT; 138 pfn += SZ_1M >> PAGE_SHIFT;
134 pmd[1] = __pmd(__pfn_to_phys(pfn) | type->prot_sect); 139 pmd[1] = __pmd(__pfn_to_phys(pfn) | type->prot_sect);
135 pfn += SZ_1M >> PAGE_SHIFT; 140 pfn += SZ_1M >> PAGE_SHIFT;
136 flush_pmd_entry(pmd); 141 flush_pmd_entry(pmd);
137 142
138 addr += PGDIR_SIZE; 143 addr += PMD_SIZE;
139 pgd++; 144 pmd += 2;
140 } while (addr < end); 145 } while (addr < end);
141 146
142 return 0; 147 return 0;
@@ -148,6 +153,8 @@ remap_area_supersections(unsigned long virt, unsigned long pfn,
148{ 153{
149 unsigned long addr = virt, end = virt + size; 154 unsigned long addr = virt, end = virt + size;
150 pgd_t *pgd; 155 pgd_t *pgd;
156 pud_t *pud;
157 pmd_t *pmd;
151 158
152 /* 159 /*
153 * Remove and free any PTE-based mapping, and 160 * Remove and free any PTE-based mapping, and
@@ -156,6 +163,8 @@ remap_area_supersections(unsigned long virt, unsigned long pfn,
156 unmap_area_sections(virt, size); 163 unmap_area_sections(virt, size);
157 164
158 pgd = pgd_offset_k(virt); 165 pgd = pgd_offset_k(virt);
166 pud = pud_offset(pgd, addr);
167 pmd = pmd_offset(pud, addr);
159 do { 168 do {
160 unsigned long super_pmd_val, i; 169 unsigned long super_pmd_val, i;
161 170
@@ -164,14 +173,12 @@ remap_area_supersections(unsigned long virt, unsigned long pfn,
164 super_pmd_val |= ((pfn >> (32 - PAGE_SHIFT)) & 0xf) << 20; 173 super_pmd_val |= ((pfn >> (32 - PAGE_SHIFT)) & 0xf) << 20;
165 174
166 for (i = 0; i < 8; i++) { 175 for (i = 0; i < 8; i++) {
167 pmd_t *pmd = pmd_offset(pgd, addr);
168
169 pmd[0] = __pmd(super_pmd_val); 176 pmd[0] = __pmd(super_pmd_val);
170 pmd[1] = __pmd(super_pmd_val); 177 pmd[1] = __pmd(super_pmd_val);
171 flush_pmd_entry(pmd); 178 flush_pmd_entry(pmd);
172 179
173 addr += PGDIR_SIZE; 180 addr += PMD_SIZE;
174 pgd++; 181 pmd += 2;
175 } 182 }
176 183
177 pfn += SUPERSECTION_SIZE >> PAGE_SHIFT; 184 pfn += SUPERSECTION_SIZE >> PAGE_SHIFT;
@@ -189,11 +196,13 @@ void __iomem * __arm_ioremap_pfn_caller(unsigned long pfn,
189 unsigned long addr; 196 unsigned long addr;
190 struct vm_struct * area; 197 struct vm_struct * area;
191 198
199#ifndef CONFIG_ARM_LPAE
192 /* 200 /*
193 * High mappings must be supersection aligned 201 * High mappings must be supersection aligned
194 */ 202 */
195 if (pfn >= 0x100000 && (__pfn_to_phys(pfn) & ~SUPERSECTION_MASK)) 203 if (pfn >= 0x100000 && (__pfn_to_phys(pfn) & ~SUPERSECTION_MASK))
196 return NULL; 204 return NULL;
205#endif
197 206
198 type = get_mem_type(mtype); 207 type = get_mem_type(mtype);
199 if (!type) 208 if (!type)
@@ -237,7 +246,7 @@ void __iomem * __arm_ioremap_pfn_caller(unsigned long pfn,
237 return NULL; 246 return NULL;
238 addr = (unsigned long)area->addr; 247 addr = (unsigned long)area->addr;
239 248
240#ifndef CONFIG_SMP 249#if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE)
241 if (DOMAIN_IO == 0 && 250 if (DOMAIN_IO == 0 &&
242 (((cpu_architecture() >= CPU_ARCH_ARMv6) && (get_cr() & CR_XP)) || 251 (((cpu_architecture() >= CPU_ARCH_ARMv6) && (get_cr() & CR_XP)) ||
243 cpu_is_xsc3()) && pfn >= 0x100000 && 252 cpu_is_xsc3()) && pfn >= 0x100000 &&
@@ -343,7 +352,7 @@ void __iounmap(volatile void __iomem *io_addr)
343 read_unlock(&vmlist_lock); 352 read_unlock(&vmlist_lock);
344 return; 353 return;
345 } 354 }
346#ifndef CONFIG_SMP 355#if !defined(CONFIG_SMP) && !defined(CONFIG_ARM_LPAE)
347 /* 356 /*
348 * If this is a section based mapping we need to handle it 357 * If this is a section based mapping we need to handle it
349 * specially as the VM subsystem does not know how to handle 358 * specially as the VM subsystem does not know how to handle
diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c
index 27e366af67f..94c5a0c94f5 100644
--- a/arch/arm/mm/mmu.c
+++ b/arch/arm/mm/mmu.c
@@ -151,6 +151,7 @@ static int __init early_nowrite(char *__unused)
151} 151}
152early_param("nowb", early_nowrite); 152early_param("nowb", early_nowrite);
153 153
154#ifndef CONFIG_ARM_LPAE
154static int __init early_ecc(char *p) 155static int __init early_ecc(char *p)
155{ 156{
156 if (memcmp(p, "on", 2) == 0) 157 if (memcmp(p, "on", 2) == 0)
@@ -160,6 +161,7 @@ static int __init early_ecc(char *p)
160 return 0; 161 return 0;
161} 162}
162early_param("ecc", early_ecc); 163early_param("ecc", early_ecc);
164#endif
163 165
164static int __init noalign_setup(char *__unused) 166static int __init noalign_setup(char *__unused)
165{ 167{
@@ -229,10 +231,12 @@ static struct mem_type mem_types[] = {
229 .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN, 231 .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN,
230 .domain = DOMAIN_KERNEL, 232 .domain = DOMAIN_KERNEL,
231 }, 233 },
234#ifndef CONFIG_ARM_LPAE
232 [MT_MINICLEAN] = { 235 [MT_MINICLEAN] = {
233 .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN | PMD_SECT_MINICACHE, 236 .prot_sect = PMD_TYPE_SECT | PMD_SECT_XN | PMD_SECT_MINICACHE,
234 .domain = DOMAIN_KERNEL, 237 .domain = DOMAIN_KERNEL,
235 }, 238 },
239#endif
236 [MT_LOW_VECTORS] = { 240 [MT_LOW_VECTORS] = {
237 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | 241 .prot_pte = L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY |
238 L_PTE_RDONLY, 242 L_PTE_RDONLY,
@@ -430,6 +434,7 @@ static void __init build_mem_type_table(void)
430 * ARMv6 and above have extended page tables. 434 * ARMv6 and above have extended page tables.
431 */ 435 */
432 if (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP)) { 436 if (cpu_arch >= CPU_ARCH_ARMv6 && (cr & CR_XP)) {
437#ifndef CONFIG_ARM_LPAE
433 /* 438 /*
434 * Mark cache clean areas and XIP ROM read only 439 * Mark cache clean areas and XIP ROM read only
435 * from SVC mode and no access from userspace. 440 * from SVC mode and no access from userspace.
@@ -437,6 +442,7 @@ static void __init build_mem_type_table(void)
437 mem_types[MT_ROM].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; 442 mem_types[MT_ROM].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
438 mem_types[MT_MINICLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; 443 mem_types[MT_MINICLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
439 mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE; 444 mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_APX|PMD_SECT_AP_WRITE;
445#endif
440 446
441 if (is_smp()) { 447 if (is_smp()) {
442 /* 448 /*
@@ -475,6 +481,18 @@ static void __init build_mem_type_table(void)
475 mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_BUFFERABLE; 481 mem_types[MT_MEMORY_NONCACHED].prot_sect |= PMD_SECT_BUFFERABLE;
476 } 482 }
477 483
484#ifdef CONFIG_ARM_LPAE
485 /*
486 * Do not generate access flag faults for the kernel mappings.
487 */
488 for (i = 0; i < ARRAY_SIZE(mem_types); i++) {
489 mem_types[i].prot_pte |= PTE_EXT_AF;
490 mem_types[i].prot_sect |= PMD_SECT_AF;
491 }
492 kern_pgprot |= PTE_EXT_AF;
493 vecs_pgprot |= PTE_EXT_AF;
494#endif
495
478 for (i = 0; i < 16; i++) { 496 for (i = 0; i < 16; i++) {
479 unsigned long v = pgprot_val(protection_map[i]); 497 unsigned long v = pgprot_val(protection_map[i]);
480 protection_map[i] = __pgprot(v | user_pgprot); 498 protection_map[i] = __pgprot(v | user_pgprot);
@@ -578,8 +596,10 @@ static void __init alloc_init_section(pud_t *pud, unsigned long addr,
578 if (((addr | end | phys) & ~SECTION_MASK) == 0) { 596 if (((addr | end | phys) & ~SECTION_MASK) == 0) {
579 pmd_t *p = pmd; 597 pmd_t *p = pmd;
580 598
599#ifndef CONFIG_ARM_LPAE
581 if (addr & SECTION_SIZE) 600 if (addr & SECTION_SIZE)
582 pmd++; 601 pmd++;
602#endif
583 603
584 do { 604 do {
585 *pmd = __pmd(phys | type->prot_sect); 605 *pmd = __pmd(phys | type->prot_sect);
@@ -609,6 +629,7 @@ static void alloc_init_pud(pgd_t *pgd, unsigned long addr, unsigned long end,
609 } while (pud++, addr = next, addr != end); 629 } while (pud++, addr = next, addr != end);
610} 630}
611 631
632#ifndef CONFIG_ARM_LPAE
612static void __init create_36bit_mapping(struct map_desc *md, 633static void __init create_36bit_mapping(struct map_desc *md,
613 const struct mem_type *type) 634 const struct mem_type *type)
614{ 635{
@@ -668,6 +689,7 @@ static void __init create_36bit_mapping(struct map_desc *md,
668 pgd += SUPERSECTION_SIZE >> PGDIR_SHIFT; 689 pgd += SUPERSECTION_SIZE >> PGDIR_SHIFT;
669 } while (addr != end); 690 } while (addr != end);
670} 691}
692#endif /* !CONFIG_ARM_LPAE */
671 693
672/* 694/*
673 * Create the page directory entries and any necessary 695 * Create the page directory entries and any necessary
@@ -700,6 +722,7 @@ static void __init create_mapping(struct map_desc *md)
700 722
701 type = &mem_types[md->type]; 723 type = &mem_types[md->type];
702 724
725#ifndef CONFIG_ARM_LPAE
703 /* 726 /*
704 * Catch 36-bit addresses 727 * Catch 36-bit addresses
705 */ 728 */
@@ -707,6 +730,7 @@ static void __init create_mapping(struct map_desc *md)
707 create_36bit_mapping(md, type); 730 create_36bit_mapping(md, type);
708 return; 731 return;
709 } 732 }
733#endif
710 734
711 addr = md->virtual & PAGE_MASK; 735 addr = md->virtual & PAGE_MASK;
712 phys = __pfn_to_phys(md->pfn); 736 phys = __pfn_to_phys(md->pfn);
@@ -797,6 +821,9 @@ void __init sanity_check_meminfo(void)
797 struct membank *bank = &meminfo.bank[j]; 821 struct membank *bank = &meminfo.bank[j];
798 *bank = meminfo.bank[i]; 822 *bank = meminfo.bank[i];
799 823
824 if (bank->start > ULONG_MAX)
825 highmem = 1;
826
800#ifdef CONFIG_HIGHMEM 827#ifdef CONFIG_HIGHMEM
801 if (__va(bank->start) >= vmalloc_min || 828 if (__va(bank->start) >= vmalloc_min ||
802 __va(bank->start) < (void *)PAGE_OFFSET) 829 __va(bank->start) < (void *)PAGE_OFFSET)
@@ -808,7 +835,7 @@ void __init sanity_check_meminfo(void)
808 * Split those memory banks which are partially overlapping 835 * Split those memory banks which are partially overlapping
809 * the vmalloc area greatly simplifying things later. 836 * the vmalloc area greatly simplifying things later.
810 */ 837 */
811 if (__va(bank->start) < vmalloc_min && 838 if (!highmem && __va(bank->start) < vmalloc_min &&
812 bank->size > vmalloc_min - __va(bank->start)) { 839 bank->size > vmalloc_min - __va(bank->start)) {
813 if (meminfo.nr_banks >= NR_BANKS) { 840 if (meminfo.nr_banks >= NR_BANKS) {
814 printk(KERN_CRIT "NR_BANKS too low, " 841 printk(KERN_CRIT "NR_BANKS too low, "
@@ -829,6 +856,17 @@ void __init sanity_check_meminfo(void)
829 bank->highmem = highmem; 856 bank->highmem = highmem;
830 857
831 /* 858 /*
859 * Highmem banks not allowed with !CONFIG_HIGHMEM.
860 */
861 if (highmem) {
862 printk(KERN_NOTICE "Ignoring RAM at %.8llx-%.8llx "
863 "(!CONFIG_HIGHMEM).\n",
864 (unsigned long long)bank->start,
865 (unsigned long long)bank->start + bank->size - 1);
866 continue;
867 }
868
869 /*
832 * Check whether this memory bank would entirely overlap 870 * Check whether this memory bank would entirely overlap
833 * the vmalloc area. 871 * the vmalloc area.
834 */ 872 */
@@ -920,7 +958,13 @@ static inline void prepare_page_table(void)
920 pmd_clear(pmd_off_k(addr)); 958 pmd_clear(pmd_off_k(addr));
921} 959}
922 960
961#ifdef CONFIG_ARM_LPAE
962/* the first page is reserved for pgd */
963#define SWAPPER_PG_DIR_SIZE (PAGE_SIZE + \
964 PTRS_PER_PGD * PTRS_PER_PMD * sizeof(pmd_t))
965#else
923#define SWAPPER_PG_DIR_SIZE (PTRS_PER_PGD * sizeof(pgd_t)) 966#define SWAPPER_PG_DIR_SIZE (PTRS_PER_PGD * sizeof(pgd_t))
967#endif
924 968
925/* 969/*
926 * Reserve the special regions of memory 970 * Reserve the special regions of memory
diff --git a/arch/arm/mm/pgd.c b/arch/arm/mm/pgd.c
index b2027c154b2..a3e78ccabd6 100644
--- a/arch/arm/mm/pgd.c
+++ b/arch/arm/mm/pgd.c
@@ -10,6 +10,7 @@
10#include <linux/mm.h> 10#include <linux/mm.h>
11#include <linux/gfp.h> 11#include <linux/gfp.h>
12#include <linux/highmem.h> 12#include <linux/highmem.h>
13#include <linux/slab.h>
13 14
14#include <asm/pgalloc.h> 15#include <asm/pgalloc.h>
15#include <asm/page.h> 16#include <asm/page.h>
@@ -17,6 +18,14 @@
17 18
18#include "mm.h" 19#include "mm.h"
19 20
21#ifdef CONFIG_ARM_LPAE
22#define __pgd_alloc() kmalloc(PTRS_PER_PGD * sizeof(pgd_t), GFP_KERNEL)
23#define __pgd_free(pgd) kfree(pgd)
24#else
25#define __pgd_alloc() (pgd_t *)__get_free_pages(GFP_KERNEL, 2)
26#define __pgd_free(pgd) free_pages((unsigned long)pgd, 2)
27#endif
28
20/* 29/*
21 * need to get a 16k page for level 1 30 * need to get a 16k page for level 1
22 */ 31 */
@@ -27,7 +36,7 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
27 pmd_t *new_pmd, *init_pmd; 36 pmd_t *new_pmd, *init_pmd;
28 pte_t *new_pte, *init_pte; 37 pte_t *new_pte, *init_pte;
29 38
30 new_pgd = (pgd_t *)__get_free_pages(GFP_KERNEL, 2); 39 new_pgd = __pgd_alloc();
31 if (!new_pgd) 40 if (!new_pgd)
32 goto no_pgd; 41 goto no_pgd;
33 42
@@ -42,10 +51,25 @@ pgd_t *pgd_alloc(struct mm_struct *mm)
42 51
43 clean_dcache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t)); 52 clean_dcache_area(new_pgd, PTRS_PER_PGD * sizeof(pgd_t));
44 53
54#ifdef CONFIG_ARM_LPAE
55 /*
56 * Allocate PMD table for modules and pkmap mappings.
57 */
58 new_pud = pud_alloc(mm, new_pgd + pgd_index(MODULES_VADDR),
59 MODULES_VADDR);
60 if (!new_pud)
61 goto no_pud;
62
63 new_pmd = pmd_alloc(mm, new_pud, 0);
64 if (!new_pmd)
65 goto no_pmd;
66#endif
67
45 if (!vectors_high()) { 68 if (!vectors_high()) {
46 /* 69 /*
47 * On ARM, first page must always be allocated since it 70 * On ARM, first page must always be allocated since it
48 * contains the machine vectors. 71 * contains the machine vectors. The vectors are always high
72 * with LPAE.
49 */ 73 */
50 new_pud = pud_alloc(mm, new_pgd, 0); 74 new_pud = pud_alloc(mm, new_pgd, 0);
51 if (!new_pud) 75 if (!new_pud)
@@ -74,7 +98,7 @@ no_pte:
74no_pmd: 98no_pmd:
75 pud_free(mm, new_pud); 99 pud_free(mm, new_pud);
76no_pud: 100no_pud:
77 free_pages((unsigned long)new_pgd, 2); 101 __pgd_free(new_pgd);
78no_pgd: 102no_pgd:
79 return NULL; 103 return NULL;
80} 104}
@@ -111,5 +135,24 @@ no_pud:
111 pgd_clear(pgd); 135 pgd_clear(pgd);
112 pud_free(mm, pud); 136 pud_free(mm, pud);
113no_pgd: 137no_pgd:
114 free_pages((unsigned long) pgd_base, 2); 138#ifdef CONFIG_ARM_LPAE
139 /*
140 * Free modules/pkmap or identity pmd tables.
141 */
142 for (pgd = pgd_base; pgd < pgd_base + PTRS_PER_PGD; pgd++) {
143 if (pgd_none_or_clear_bad(pgd))
144 continue;
145 if (pgd_val(*pgd) & L_PGD_SWAPPER)
146 continue;
147 pud = pud_offset(pgd, 0);
148 if (pud_none_or_clear_bad(pud))
149 continue;
150 pmd = pmd_offset(pud, 0);
151 pud_clear(pud);
152 pmd_free(mm, pmd);
153 pgd_clear(pgd);
154 pud_free(mm, pud);
155 }
156#endif
157 __pgd_free(pgd_base);
115} 158}
diff --git a/arch/arm/mm/proc-macros.S b/arch/arm/mm/proc-macros.S
index 307a4def8d3..2d8ff3ad86d 100644
--- a/arch/arm/mm/proc-macros.S
+++ b/arch/arm/mm/proc-macros.S
@@ -91,8 +91,9 @@
91#if L_PTE_SHARED != PTE_EXT_SHARED 91#if L_PTE_SHARED != PTE_EXT_SHARED
92#error PTE shared bit mismatch 92#error PTE shared bit mismatch
93#endif 93#endif
94#if (L_PTE_XN+L_PTE_USER+L_PTE_RDONLY+L_PTE_DIRTY+L_PTE_YOUNG+\ 94#if !defined (CONFIG_ARM_LPAE) && \
95 L_PTE_FILE+L_PTE_PRESENT) > L_PTE_SHARED 95 (L_PTE_XN+L_PTE_USER+L_PTE_RDONLY+L_PTE_DIRTY+L_PTE_YOUNG+\
96 L_PTE_FILE+L_PTE_PRESENT) > L_PTE_SHARED
96#error Invalid Linux PTE bit settings 97#error Invalid Linux PTE bit settings
97#endif 98#endif
98#endif /* CONFIG_MMU */ 99#endif /* CONFIG_MMU */
diff --git a/arch/arm/mm/proc-v7-2level.S b/arch/arm/mm/proc-v7-2level.S
new file mode 100644
index 00000000000..3a4b3e7b888
--- /dev/null
+++ b/arch/arm/mm/proc-v7-2level.S
@@ -0,0 +1,171 @@
1/*
2 * arch/arm/mm/proc-v7-2level.S
3 *
4 * Copyright (C) 2001 Deep Blue Solutions Ltd.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#define TTB_S (1 << 1)
12#define TTB_RGN_NC (0 << 3)
13#define TTB_RGN_OC_WBWA (1 << 3)
14#define TTB_RGN_OC_WT (2 << 3)
15#define TTB_RGN_OC_WB (3 << 3)
16#define TTB_NOS (1 << 5)
17#define TTB_IRGN_NC ((0 << 0) | (0 << 6))
18#define TTB_IRGN_WBWA ((0 << 0) | (1 << 6))
19#define TTB_IRGN_WT ((1 << 0) | (0 << 6))
20#define TTB_IRGN_WB ((1 << 0) | (1 << 6))
21
22/* PTWs cacheable, inner WB not shareable, outer WB not shareable */
23#define TTB_FLAGS_UP TTB_IRGN_WB|TTB_RGN_OC_WB
24#define PMD_FLAGS_UP PMD_SECT_WB
25
26/* PTWs cacheable, inner WBWA shareable, outer WBWA not shareable */
27#define TTB_FLAGS_SMP TTB_IRGN_WBWA|TTB_S|TTB_NOS|TTB_RGN_OC_WBWA
28#define PMD_FLAGS_SMP PMD_SECT_WBWA|PMD_SECT_S
29
30/*
31 * cpu_v7_switch_mm(pgd_phys, tsk)
32 *
33 * Set the translation table base pointer to be pgd_phys
34 *
35 * - pgd_phys - physical address of new TTB
36 *
37 * It is assumed that:
38 * - we are not using split page tables
39 */
40ENTRY(cpu_v7_switch_mm)
41#ifdef CONFIG_MMU
42 mov r2, #0
43 ldr r1, [r1, #MM_CONTEXT_ID] @ get mm->context.id
44 ALT_SMP(orr r0, r0, #TTB_FLAGS_SMP)
45 ALT_UP(orr r0, r0, #TTB_FLAGS_UP)
46#ifdef CONFIG_ARM_ERRATA_430973
47 mcr p15, 0, r2, c7, c5, 6 @ flush BTAC/BTB
48#endif
49#ifdef CONFIG_ARM_ERRATA_754322
50 dsb
51#endif
52 mcr p15, 0, r2, c13, c0, 1 @ set reserved context ID
53 isb
541: mcr p15, 0, r0, c2, c0, 0 @ set TTB 0
55 isb
56#ifdef CONFIG_ARM_ERRATA_754322
57 dsb
58#endif
59 mcr p15, 0, r1, c13, c0, 1 @ set context ID
60 isb
61#endif
62 mov pc, lr
63ENDPROC(cpu_v7_switch_mm)
64
65/*
66 * cpu_v7_set_pte_ext(ptep, pte)
67 *
68 * Set a level 2 translation table entry.
69 *
70 * - ptep - pointer to level 2 translation table entry
71 * (hardware version is stored at +2048 bytes)
72 * - pte - PTE value to store
73 * - ext - value for extended PTE bits
74 */
75ENTRY(cpu_v7_set_pte_ext)
76#ifdef CONFIG_MMU
77 str r1, [r0] @ linux version
78
79 bic r3, r1, #0x000003f0
80 bic r3, r3, #PTE_TYPE_MASK
81 orr r3, r3, r2
82 orr r3, r3, #PTE_EXT_AP0 | 2
83
84 tst r1, #1 << 4
85 orrne r3, r3, #PTE_EXT_TEX(1)
86
87 eor r1, r1, #L_PTE_DIRTY
88 tst r1, #L_PTE_RDONLY | L_PTE_DIRTY
89 orrne r3, r3, #PTE_EXT_APX
90
91 tst r1, #L_PTE_USER
92 orrne r3, r3, #PTE_EXT_AP1
93#ifdef CONFIG_CPU_USE_DOMAINS
94 @ allow kernel read/write access to read-only user pages
95 tstne r3, #PTE_EXT_APX
96 bicne r3, r3, #PTE_EXT_APX | PTE_EXT_AP0
97#endif
98
99 tst r1, #L_PTE_XN
100 orrne r3, r3, #PTE_EXT_XN
101
102 tst r1, #L_PTE_YOUNG
103 tstne r1, #L_PTE_PRESENT
104 moveq r3, #0
105
106 ARM( str r3, [r0, #2048]! )
107 THUMB( add r0, r0, #2048 )
108 THUMB( str r3, [r0] )
109 mcr p15, 0, r0, c7, c10, 1 @ flush_pte
110#endif
111 mov pc, lr
112ENDPROC(cpu_v7_set_pte_ext)
113
114 /*
115 * Memory region attributes with SCTLR.TRE=1
116 *
117 * n = TEX[0],C,B
118 * TR = PRRR[2n+1:2n] - memory type
119 * IR = NMRR[2n+1:2n] - inner cacheable property
120 * OR = NMRR[2n+17:2n+16] - outer cacheable property
121 *
122 * n TR IR OR
123 * UNCACHED 000 00
124 * BUFFERABLE 001 10 00 00
125 * WRITETHROUGH 010 10 10 10
126 * WRITEBACK 011 10 11 11
127 * reserved 110
128 * WRITEALLOC 111 10 01 01
129 * DEV_SHARED 100 01
130 * DEV_NONSHARED 100 01
131 * DEV_WC 001 10
132 * DEV_CACHED 011 10
133 *
134 * Other attributes:
135 *
136 * DS0 = PRRR[16] = 0 - device shareable property
137 * DS1 = PRRR[17] = 1 - device shareable property
138 * NS0 = PRRR[18] = 0 - normal shareable property
139 * NS1 = PRRR[19] = 1 - normal shareable property
140 * NOS = PRRR[24+n] = 1 - not outer shareable
141 */
142.equ PRRR, 0xff0a81a8
143.equ NMRR, 0x40e040e0
144
145 /*
146 * Macro for setting up the TTBRx and TTBCR registers.
147 * - \ttb0 and \ttb1 updated with the corresponding flags.
148 */
149 .macro v7_ttb_setup, zero, ttbr0, ttbr1, tmp
150 mcr p15, 0, \zero, c2, c0, 2 @ TTB control register
151 ALT_SMP(orr \ttbr0, \ttbr0, #TTB_FLAGS_SMP)
152 ALT_UP(orr \ttbr0, \ttbr0, #TTB_FLAGS_UP)
153 ALT_SMP(orr \ttbr1, \ttbr1, #TTB_FLAGS_SMP)
154 ALT_UP(orr \ttbr1, \ttbr1, #TTB_FLAGS_UP)
155 mcr p15, 0, \ttbr1, c2, c0, 1 @ load TTB1
156 .endm
157
158 __CPUINIT
159
160 /* AT
161 * TFR EV X F I D LR S
162 * .EEE ..EE PUI. .T.T 4RVI ZWRS BLDP WCAM
163 * rxxx rrxx xxx0 0101 xxxx xxxx x111 xxxx < forced
164 * 1 0 110 0011 1100 .111 1101 < we want
165 */
166 .align 2
167 .type v7_crval, #object
168v7_crval:
169 crval clear=0x0120c302, mmuset=0x10c03c7d, ucset=0x00c01c7c
170
171 .previous
diff --git a/arch/arm/mm/proc-v7-3level.S b/arch/arm/mm/proc-v7-3level.S
new file mode 100644
index 00000000000..8de0f1dd154
--- /dev/null
+++ b/arch/arm/mm/proc-v7-3level.S
@@ -0,0 +1,150 @@
1/*
2 * arch/arm/mm/proc-v7-3level.S
3 *
4 * Copyright (C) 2001 Deep Blue Solutions Ltd.
5 * Copyright (C) 2011 ARM Ltd.
6 * Author: Catalin Marinas <catalin.marinas@arm.com>
7 * based on arch/arm/mm/proc-v7-2level.S
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 *
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
17 *
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, write to the Free Software
20 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 */
22
23#define TTB_IRGN_NC (0 << 8)
24#define TTB_IRGN_WBWA (1 << 8)
25#define TTB_IRGN_WT (2 << 8)
26#define TTB_IRGN_WB (3 << 8)
27#define TTB_RGN_NC (0 << 10)
28#define TTB_RGN_OC_WBWA (1 << 10)
29#define TTB_RGN_OC_WT (2 << 10)
30#define TTB_RGN_OC_WB (3 << 10)
31#define TTB_S (3 << 12)
32#define TTB_EAE (1 << 31)
33
34/* PTWs cacheable, inner WB not shareable, outer WB not shareable */
35#define TTB_FLAGS_UP (TTB_IRGN_WB|TTB_RGN_OC_WB)
36#define PMD_FLAGS_UP (PMD_SECT_WB)
37
38/* PTWs cacheable, inner WBWA shareable, outer WBWA not shareable */
39#define TTB_FLAGS_SMP (TTB_IRGN_WBWA|TTB_S|TTB_RGN_OC_WBWA)
40#define PMD_FLAGS_SMP (PMD_SECT_WBWA|PMD_SECT_S)
41
42/*
43 * cpu_v7_switch_mm(pgd_phys, tsk)
44 *
45 * Set the translation table base pointer to be pgd_phys (physical address of
46 * the new TTB).
47 */
48ENTRY(cpu_v7_switch_mm)
49#ifdef CONFIG_MMU
50 ldr r1, [r1, #MM_CONTEXT_ID] @ get mm->context.id
51 and r3, r1, #0xff
52 mov r3, r3, lsl #(48 - 32) @ ASID
53 mcrr p15, 0, r0, r3, c2 @ set TTB 0
54 isb
55#endif
56 mov pc, lr
57ENDPROC(cpu_v7_switch_mm)
58
59/*
60 * cpu_v7_set_pte_ext(ptep, pte)
61 *
62 * Set a level 2 translation table entry.
63 * - ptep - pointer to level 3 translation table entry
64 * - pte - PTE value to store (64-bit in r2 and r3)
65 */
66ENTRY(cpu_v7_set_pte_ext)
67#ifdef CONFIG_MMU
68 tst r2, #L_PTE_PRESENT
69 beq 1f
70 tst r3, #1 << (55 - 32) @ L_PTE_DIRTY
71 orreq r2, #L_PTE_RDONLY
721: strd r2, r3, [r0]
73 mcr p15, 0, r0, c7, c10, 1 @ flush_pte
74#endif
75 mov pc, lr
76ENDPROC(cpu_v7_set_pte_ext)
77
78 /*
79 * Memory region attributes for LPAE (defined in pgtable-3level.h):
80 *
81 * n = AttrIndx[2:0]
82 *
83 * n MAIR
84 * UNCACHED 000 00000000
85 * BUFFERABLE 001 01000100
86 * DEV_WC 001 01000100
87 * WRITETHROUGH 010 10101010
88 * WRITEBACK 011 11101110
89 * DEV_CACHED 011 11101110
90 * DEV_SHARED 100 00000100
91 * DEV_NONSHARED 100 00000100
92 * unused 101
93 * unused 110
94 * WRITEALLOC 111 11111111
95 */
96.equ PRRR, 0xeeaa4400 @ MAIR0
97.equ NMRR, 0xff000004 @ MAIR1
98
99 /*
100 * Macro for setting up the TTBRx and TTBCR registers.
101 * - \ttbr1 updated.
102 */
103 .macro v7_ttb_setup, zero, ttbr0, ttbr1, tmp
104 ldr \tmp, =swapper_pg_dir @ swapper_pg_dir virtual address
105 cmp \ttbr1, \tmp @ PHYS_OFFSET > PAGE_OFFSET? (branch below)
106 mrc p15, 0, \tmp, c2, c0, 2 @ TTB control register
107 orr \tmp, \tmp, #TTB_EAE
108 ALT_SMP(orr \tmp, \tmp, #TTB_FLAGS_SMP)
109 ALT_UP(orr \tmp, \tmp, #TTB_FLAGS_UP)
110 ALT_SMP(orr \tmp, \tmp, #TTB_FLAGS_SMP << 16)
111 ALT_UP(orr \tmp, \tmp, #TTB_FLAGS_UP << 16)
112 /*
113 * TTBR0/TTBR1 split (PAGE_OFFSET):
114 * 0x40000000: T0SZ = 2, T1SZ = 0 (not used)
115 * 0x80000000: T0SZ = 0, T1SZ = 1
116 * 0xc0000000: T0SZ = 0, T1SZ = 2
117 *
118 * Only use this feature if PHYS_OFFSET <= PAGE_OFFSET, otherwise
119 * booting secondary CPUs would end up using TTBR1 for the identity
120 * mapping set up in TTBR0.
121 */
122 bhi 9001f @ PHYS_OFFSET > PAGE_OFFSET?
123 orr \tmp, \tmp, #(((PAGE_OFFSET >> 30) - 1) << 16) @ TTBCR.T1SZ
124#if defined CONFIG_VMSPLIT_2G
125 /* PAGE_OFFSET == 0x80000000, T1SZ == 1 */
126 add \ttbr1, \ttbr1, #1 << 4 @ skip two L1 entries
127#elif defined CONFIG_VMSPLIT_3G
128 /* PAGE_OFFSET == 0xc0000000, T1SZ == 2 */
129 add \ttbr1, \ttbr1, #4096 * (1 + 3) @ only L2 used, skip pgd+3*pmd
130#endif
131 /* CONFIG_VMSPLIT_1G does not need TTBR1 adjustment */
1329001: mcr p15, 0, \tmp, c2, c0, 2 @ TTB control register
133 mcrr p15, 1, \ttbr1, \zero, c2 @ load TTBR1
134 .endm
135
136 __CPUINIT
137
138 /*
139 * AT
140 * TFR EV X F IHD LR S
141 * .EEE ..EE PUI. .TAT 4RVI ZWRS BLDP WCAM
142 * rxxx rrxx xxx0 0101 xxxx xxxx x111 xxxx < forced
143 * 11 0 110 1 0011 1100 .111 1101 < we want
144 */
145 .align 2
146 .type v7_crval, #object
147v7_crval:
148 crval clear=0x0120c302, mmuset=0x30c23c7d, ucset=0x00c01c7c
149
150 .previous
diff --git a/arch/arm/mm/proc-v7.S b/arch/arm/mm/proc-v7.S
index 66a185f018a..7efa2a721d5 100644
--- a/arch/arm/mm/proc-v7.S
+++ b/arch/arm/mm/proc-v7.S
@@ -19,24 +19,11 @@
19 19
20#include "proc-macros.S" 20#include "proc-macros.S"
21 21
22#define TTB_S (1 << 1) 22#ifdef CONFIG_ARM_LPAE
23#define TTB_RGN_NC (0 << 3) 23#include "proc-v7-3level.S"
24#define TTB_RGN_OC_WBWA (1 << 3) 24#else
25#define TTB_RGN_OC_WT (2 << 3) 25#include "proc-v7-2level.S"
26#define TTB_RGN_OC_WB (3 << 3) 26#endif
27#define TTB_NOS (1 << 5)
28#define TTB_IRGN_NC ((0 << 0) | (0 << 6))
29#define TTB_IRGN_WBWA ((0 << 0) | (1 << 6))
30#define TTB_IRGN_WT ((1 << 0) | (0 << 6))
31#define TTB_IRGN_WB ((1 << 0) | (1 << 6))
32
33/* PTWs cacheable, inner WB not shareable, outer WB not shareable */
34#define TTB_FLAGS_UP TTB_IRGN_WB|TTB_RGN_OC_WB
35#define PMD_FLAGS_UP PMD_SECT_WB
36
37/* PTWs cacheable, inner WBWA shareable, outer WBWA not shareable */
38#define TTB_FLAGS_SMP TTB_IRGN_WBWA|TTB_S|TTB_NOS|TTB_RGN_OC_WBWA
39#define PMD_FLAGS_SMP PMD_SECT_WBWA|PMD_SECT_S
40 27
41ENTRY(cpu_v7_proc_init) 28ENTRY(cpu_v7_proc_init)
42 mov pc, lr 29 mov pc, lr
@@ -99,127 +86,12 @@ ENTRY(cpu_v7_dcache_clean_area)
99 mov pc, lr 86 mov pc, lr
100ENDPROC(cpu_v7_dcache_clean_area) 87ENDPROC(cpu_v7_dcache_clean_area)
101 88
102/*
103 * cpu_v7_switch_mm(pgd_phys, tsk)
104 *
105 * Set the translation table base pointer to be pgd_phys
106 *
107 * - pgd_phys - physical address of new TTB
108 *
109 * It is assumed that:
110 * - we are not using split page tables
111 */
112ENTRY(cpu_v7_switch_mm)
113#ifdef CONFIG_MMU
114 mov r2, #0
115 ldr r1, [r1, #MM_CONTEXT_ID] @ get mm->context.id
116 ALT_SMP(orr r0, r0, #TTB_FLAGS_SMP)
117 ALT_UP(orr r0, r0, #TTB_FLAGS_UP)
118#ifdef CONFIG_ARM_ERRATA_430973
119 mcr p15, 0, r2, c7, c5, 6 @ flush BTAC/BTB
120#endif
121#ifdef CONFIG_ARM_ERRATA_754322
122 dsb
123#endif
124 mcr p15, 0, r2, c13, c0, 1 @ set reserved context ID
125 isb
1261: mcr p15, 0, r0, c2, c0, 0 @ set TTB 0
127 isb
128#ifdef CONFIG_ARM_ERRATA_754322
129 dsb
130#endif
131 mcr p15, 0, r1, c13, c0, 1 @ set context ID
132 isb
133#endif
134 mov pc, lr
135ENDPROC(cpu_v7_switch_mm)
136
137/*
138 * cpu_v7_set_pte_ext(ptep, pte)
139 *
140 * Set a level 2 translation table entry.
141 *
142 * - ptep - pointer to level 2 translation table entry
143 * (hardware version is stored at +2048 bytes)
144 * - pte - PTE value to store
145 * - ext - value for extended PTE bits
146 */
147ENTRY(cpu_v7_set_pte_ext)
148#ifdef CONFIG_MMU
149 str r1, [r0] @ linux version
150
151 bic r3, r1, #0x000003f0
152 bic r3, r3, #PTE_TYPE_MASK
153 orr r3, r3, r2
154 orr r3, r3, #PTE_EXT_AP0 | 2
155
156 tst r1, #1 << 4
157 orrne r3, r3, #PTE_EXT_TEX(1)
158
159 eor r1, r1, #L_PTE_DIRTY
160 tst r1, #L_PTE_RDONLY | L_PTE_DIRTY
161 orrne r3, r3, #PTE_EXT_APX
162
163 tst r1, #L_PTE_USER
164 orrne r3, r3, #PTE_EXT_AP1
165#ifdef CONFIG_CPU_USE_DOMAINS
166 @ allow kernel read/write access to read-only user pages
167 tstne r3, #PTE_EXT_APX
168 bicne r3, r3, #PTE_EXT_APX | PTE_EXT_AP0
169#endif
170
171 tst r1, #L_PTE_XN
172 orrne r3, r3, #PTE_EXT_XN
173
174 tst r1, #L_PTE_YOUNG
175 tstne r1, #L_PTE_PRESENT
176 moveq r3, #0
177
178 ARM( str r3, [r0, #2048]! )
179 THUMB( add r0, r0, #2048 )
180 THUMB( str r3, [r0] )
181 mcr p15, 0, r0, c7, c10, 1 @ flush_pte
182#endif
183 mov pc, lr
184ENDPROC(cpu_v7_set_pte_ext)
185
186 string cpu_v7_name, "ARMv7 Processor" 89 string cpu_v7_name, "ARMv7 Processor"
187 .align 90 .align
188 91
189 /*
190 * Memory region attributes with SCTLR.TRE=1
191 *
192 * n = TEX[0],C,B
193 * TR = PRRR[2n+1:2n] - memory type
194 * IR = NMRR[2n+1:2n] - inner cacheable property
195 * OR = NMRR[2n+17:2n+16] - outer cacheable property
196 *
197 * n TR IR OR
198 * UNCACHED 000 00
199 * BUFFERABLE 001 10 00 00
200 * WRITETHROUGH 010 10 10 10
201 * WRITEBACK 011 10 11 11
202 * reserved 110
203 * WRITEALLOC 111 10 01 01
204 * DEV_SHARED 100 01
205 * DEV_NONSHARED 100 01
206 * DEV_WC 001 10
207 * DEV_CACHED 011 10
208 *
209 * Other attributes:
210 *
211 * DS0 = PRRR[16] = 0 - device shareable property
212 * DS1 = PRRR[17] = 1 - device shareable property
213 * NS0 = PRRR[18] = 0 - normal shareable property
214 * NS1 = PRRR[19] = 1 - normal shareable property
215 * NOS = PRRR[24+n] = 1 - not outer shareable
216 */
217.equ PRRR, 0xff0a81a8
218.equ NMRR, 0x40e040e0
219
220/* Suspend/resume support: derived from arch/arm/mach-s5pv210/sleep.S */ 92/* Suspend/resume support: derived from arch/arm/mach-s5pv210/sleep.S */
221.globl cpu_v7_suspend_size 93.globl cpu_v7_suspend_size
222.equ cpu_v7_suspend_size, 4 * 7 94.equ cpu_v7_suspend_size, 4 * 8
223#ifdef CONFIG_ARM_CPU_SUSPEND 95#ifdef CONFIG_ARM_CPU_SUSPEND
224ENTRY(cpu_v7_do_suspend) 96ENTRY(cpu_v7_do_suspend)
225 stmfd sp!, {r4 - r10, lr} 97 stmfd sp!, {r4 - r10, lr}
@@ -228,10 +100,11 @@ ENTRY(cpu_v7_do_suspend)
228 stmia r0!, {r4 - r5} 100 stmia r0!, {r4 - r5}
229 mrc p15, 0, r6, c3, c0, 0 @ Domain ID 101 mrc p15, 0, r6, c3, c0, 0 @ Domain ID
230 mrc p15, 0, r7, c2, c0, 1 @ TTB 1 102 mrc p15, 0, r7, c2, c0, 1 @ TTB 1
103 mrc p15, 0, r11, c2, c0, 2 @ TTB control register
231 mrc p15, 0, r8, c1, c0, 0 @ Control register 104 mrc p15, 0, r8, c1, c0, 0 @ Control register
232 mrc p15, 0, r9, c1, c0, 1 @ Auxiliary control register 105 mrc p15, 0, r9, c1, c0, 1 @ Auxiliary control register
233 mrc p15, 0, r10, c1, c0, 2 @ Co-processor access control 106 mrc p15, 0, r10, c1, c0, 2 @ Co-processor access control
234 stmia r0, {r6 - r10} 107 stmia r0, {r6 - r11}
235 ldmfd sp!, {r4 - r10, pc} 108 ldmfd sp!, {r4 - r10, pc}
236ENDPROC(cpu_v7_do_suspend) 109ENDPROC(cpu_v7_do_suspend)
237 110
@@ -243,13 +116,15 @@ ENTRY(cpu_v7_do_resume)
243 ldmia r0!, {r4 - r5} 116 ldmia r0!, {r4 - r5}
244 mcr p15, 0, r4, c13, c0, 0 @ FCSE/PID 117 mcr p15, 0, r4, c13, c0, 0 @ FCSE/PID
245 mcr p15, 0, r5, c13, c0, 3 @ User r/o thread ID 118 mcr p15, 0, r5, c13, c0, 3 @ User r/o thread ID
246 ldmia r0, {r6 - r10} 119 ldmia r0, {r6 - r11}
247 mcr p15, 0, r6, c3, c0, 0 @ Domain ID 120 mcr p15, 0, r6, c3, c0, 0 @ Domain ID
121#ifndef CONFIG_ARM_LPAE
248 ALT_SMP(orr r1, r1, #TTB_FLAGS_SMP) 122 ALT_SMP(orr r1, r1, #TTB_FLAGS_SMP)
249 ALT_UP(orr r1, r1, #TTB_FLAGS_UP) 123 ALT_UP(orr r1, r1, #TTB_FLAGS_UP)
124#endif
250 mcr p15, 0, r1, c2, c0, 0 @ TTB 0 125 mcr p15, 0, r1, c2, c0, 0 @ TTB 0
251 mcr p15, 0, r7, c2, c0, 1 @ TTB 1 126 mcr p15, 0, r7, c2, c0, 1 @ TTB 1
252 mcr p15, 0, ip, c2, c0, 2 @ TTB control register 127 mcr p15, 0, r11, c2, c0, 2 @ TTB control register
253 mrc p15, 0, r4, c1, c0, 1 @ Read Auxiliary control register 128 mrc p15, 0, r4, c1, c0, 1 @ Read Auxiliary control register
254 teq r4, r9 @ Is it already set? 129 teq r4, r9 @ Is it already set?
255 mcrne p15, 0, r9, c1, c0, 1 @ No, so write it 130 mcrne p15, 0, r9, c1, c0, 1 @ No, so write it
@@ -379,12 +254,7 @@ __v7_setup:
379 dsb 254 dsb
380#ifdef CONFIG_MMU 255#ifdef CONFIG_MMU
381 mcr p15, 0, r10, c8, c7, 0 @ invalidate I + D TLBs 256 mcr p15, 0, r10, c8, c7, 0 @ invalidate I + D TLBs
382 mcr p15, 0, r10, c2, c0, 2 @ TTB control register 257 v7_ttb_setup r10, r4, r8, r5 @ TTBCR, TTBRx setup
383 ALT_SMP(orr r4, r4, #TTB_FLAGS_SMP)
384 ALT_UP(orr r4, r4, #TTB_FLAGS_UP)
385 ALT_SMP(orr r8, r8, #TTB_FLAGS_SMP)
386 ALT_UP(orr r8, r8, #TTB_FLAGS_UP)
387 mcr p15, 0, r8, c2, c0, 1 @ load TTB1
388 ldr r5, =PRRR @ PRRR 258 ldr r5, =PRRR @ PRRR
389 ldr r6, =NMRR @ NMRR 259 ldr r6, =NMRR @ NMRR
390 mcr p15, 0, r5, c10, c2, 0 @ write PRRR 260 mcr p15, 0, r5, c10, c2, 0 @ write PRRR
@@ -406,16 +276,7 @@ __v7_setup:
406 mov pc, lr @ return to head.S:__ret 276 mov pc, lr @ return to head.S:__ret
407ENDPROC(__v7_setup) 277ENDPROC(__v7_setup)
408 278
409 /* AT 279 .align 2
410 * TFR EV X F I D LR S
411 * .EEE ..EE PUI. .T.T 4RVI ZWRS BLDP WCAM
412 * rxxx rrxx xxx0 0101 xxxx xxxx x111 xxxx < forced
413 * 1 0 110 0011 1100 .111 1101 < we want
414 */
415 .type v7_crval, #object
416v7_crval:
417 crval clear=0x0120c302, mmuset=0x10c03c7d, ucset=0x00c01c7c
418
419__v7_setup_stack: 280__v7_setup_stack:
420 .space 4 * 11 @ 11 registers 281 .space 4 * 11 @ 11 registers
421 282
@@ -437,11 +298,11 @@ __v7_setup_stack:
437 */ 298 */
438.macro __v7_proc initfunc, mm_mmuflags = 0, io_mmuflags = 0, hwcaps = 0 299.macro __v7_proc initfunc, mm_mmuflags = 0, io_mmuflags = 0, hwcaps = 0
439 ALT_SMP(.long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | \ 300 ALT_SMP(.long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | \
440 PMD_FLAGS_SMP | \mm_mmuflags) 301 PMD_SECT_AF | PMD_FLAGS_SMP | \mm_mmuflags)
441 ALT_UP(.long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | \ 302 ALT_UP(.long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | PMD_SECT_AP_READ | \
442 PMD_FLAGS_UP | \mm_mmuflags) 303 PMD_SECT_AF | PMD_FLAGS_UP | \mm_mmuflags)
443 .long PMD_TYPE_SECT | PMD_SECT_XN | PMD_SECT_AP_WRITE | \ 304 .long PMD_TYPE_SECT | PMD_SECT_AP_WRITE | \
444 PMD_SECT_AP_READ | \io_mmuflags 305 PMD_SECT_AP_READ | PMD_SECT_AF | \io_mmuflags
445 W(b) \initfunc 306 W(b) \initfunc
446 .long cpu_arch_name 307 .long cpu_arch_name
447 .long cpu_elf_name 308 .long cpu_elf_name
@@ -454,6 +315,7 @@ __v7_setup_stack:
454 .long v7_cache_fns 315 .long v7_cache_fns
455.endm 316.endm
456 317
318#ifndef CONFIG_ARM_LPAE
457 /* 319 /*
458 * ARM Ltd. Cortex A5 processor. 320 * ARM Ltd. Cortex A5 processor.
459 */ 321 */
@@ -473,6 +335,7 @@ __v7_ca9mp_proc_info:
473 .long 0xff0ffff0 335 .long 0xff0ffff0
474 __v7_proc __v7_ca9mp_setup 336 __v7_proc __v7_ca9mp_setup
475 .size __v7_ca9mp_proc_info, . - __v7_ca9mp_proc_info 337 .size __v7_ca9mp_proc_info, . - __v7_ca9mp_proc_info
338#endif /* CONFIG_ARM_LPAE */
476 339
477 /* 340 /*
478 * ARM Ltd. Cortex A15 processor. 341 * ARM Ltd. Cortex A15 processor.
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-27 02:10:55 -0500