aboutsummaryrefslogtreecommitdiffstats
path: root/arch/arm64/mm
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2015-11-04 17:47:13 -0500
committerLinus Torvalds <torvalds@linux-foundation.org>2015-11-04 17:47:13 -0500
commit2dc10ad81fc017837037e60439662e1b16bdffb9 (patch)
treefc2f77874339b2f79499e3b34dc5ecb496b68dfc /arch/arm64/mm
parente627078a0cbdc0c391efeb5a2c4eb287328fd633 (diff)
parentf8f8bdc48851da979c6e0e4808b6031122e4af47 (diff)
Merge tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux
Pull arm64 updates from Catalin Marinas: - "genirq: Introduce generic irq migration for cpu hotunplugged" patch merged from tip/irq/for-arm to allow the arm64-specific part to be upstreamed via the arm64 tree - CPU feature detection reworked to cope with heterogeneous systems where CPUs may not have exactly the same features. The features reported by the kernel via internal data structures or ELF_HWCAP are delayed until all the CPUs are up (and before user space starts) - Support for 16KB pages, with the additional bonus of a 36-bit VA space, though the latter only depending on EXPERT - Implement native {relaxed, acquire, release} atomics for arm64 - New ASID allocation algorithm which avoids IPI on roll-over, together with TLB invalidation optimisations (using local vs global where feasible) - KASan support for arm64 - EFI_STUB clean-up and isolation for the kernel proper (required by KASan) - copy_{to,from,in}_user optimisations (sharing the memcpy template) - perf: moving arm64 to the arm32/64 shared PMU framework - L1_CACHE_BYTES increased to 128 to accommodate Cavium hardware - Support for the contiguous PTE hint on kernel mapping (16 consecutive entries may be able to use a single TLB entry) - Generic CONFIG_HZ now used on arm64 - defconfig updates * tag 'arm64-upstream' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux: (91 commits) arm64/efi: fix libstub build under CONFIG_MODVERSIONS ARM64: Enable multi-core scheduler support by default arm64/efi: move arm64 specific stub C code to libstub arm64: page-align sections for DEBUG_RODATA arm64: Fix build with CONFIG_ZONE_DMA=n arm64: Fix compat register mappings arm64: Increase the max granular size arm64: remove bogus TASK_SIZE_64 check arm64: make Timer Interrupt Frequency selectable arm64/mm: use PAGE_ALIGNED instead of IS_ALIGNED arm64: cachetype: fix definitions of ICACHEF_* flags arm64: cpufeature: declare enable_cpu_capabilities as static genirq: Make the cpuhotplug migration code less noisy arm64: Constify hwcap name string arrays arm64/kvm: Make use of the system wide safe values arm64/debug: Make use of the system wide safe value arm64: Move FP/ASIMD hwcap handling to common code arm64/HWCAP: Use system wide safe values arm64/capabilities: Make use of system wide safe value arm64: Delay cpu feature capability checks ...
Diffstat (limited to 'arch/arm64/mm')
-rw-r--r--arch/arm64/mm/Makefile3
-rw-r--r--arch/arm64/mm/cache.S10
-rw-r--r--arch/arm64/mm/context.c236
-rw-r--r--arch/arm64/mm/dump.c18
-rw-r--r--arch/arm64/mm/fault.c2
-rw-r--r--arch/arm64/mm/init.c19
-rw-r--r--arch/arm64/mm/kasan_init.c165
-rw-r--r--arch/arm64/mm/mmu.c145
-rw-r--r--arch/arm64/mm/pageattr.c2
-rw-r--r--arch/arm64/mm/pgd.c2
-rw-r--r--arch/arm64/mm/proc.S10
11 files changed, 449 insertions, 163 deletions
diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile
index 773d37a14039..57f57fde5722 100644
--- a/arch/arm64/mm/Makefile
+++ b/arch/arm64/mm/Makefile
@@ -4,3 +4,6 @@ obj-y := dma-mapping.o extable.o fault.o init.o \
4 context.o proc.o pageattr.o 4 context.o proc.o pageattr.o
5obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o 5obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
6obj-$(CONFIG_ARM64_PTDUMP) += dump.o 6obj-$(CONFIG_ARM64_PTDUMP) += dump.o
7
8obj-$(CONFIG_KASAN) += kasan_init.o
9KASAN_SANITIZE_kasan_init.o := n
diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S
index eb48d5df4a0f..cfa44a6adc0a 100644
--- a/arch/arm64/mm/cache.S
+++ b/arch/arm64/mm/cache.S
@@ -98,7 +98,7 @@ ENTRY(__flush_dcache_area)
98 b.lo 1b 98 b.lo 1b
99 dsb sy 99 dsb sy
100 ret 100 ret
101ENDPROC(__flush_dcache_area) 101ENDPIPROC(__flush_dcache_area)
102 102
103/* 103/*
104 * __inval_cache_range(start, end) 104 * __inval_cache_range(start, end)
@@ -131,7 +131,7 @@ __dma_inv_range:
131 b.lo 2b 131 b.lo 2b
132 dsb sy 132 dsb sy
133 ret 133 ret
134ENDPROC(__inval_cache_range) 134ENDPIPROC(__inval_cache_range)
135ENDPROC(__dma_inv_range) 135ENDPROC(__dma_inv_range)
136 136
137/* 137/*
@@ -171,7 +171,7 @@ ENTRY(__dma_flush_range)
171 b.lo 1b 171 b.lo 1b
172 dsb sy 172 dsb sy
173 ret 173 ret
174ENDPROC(__dma_flush_range) 174ENDPIPROC(__dma_flush_range)
175 175
176/* 176/*
177 * __dma_map_area(start, size, dir) 177 * __dma_map_area(start, size, dir)
@@ -184,7 +184,7 @@ ENTRY(__dma_map_area)
184 cmp w2, #DMA_FROM_DEVICE 184 cmp w2, #DMA_FROM_DEVICE
185 b.eq __dma_inv_range 185 b.eq __dma_inv_range
186 b __dma_clean_range 186 b __dma_clean_range
187ENDPROC(__dma_map_area) 187ENDPIPROC(__dma_map_area)
188 188
189/* 189/*
190 * __dma_unmap_area(start, size, dir) 190 * __dma_unmap_area(start, size, dir)
@@ -197,4 +197,4 @@ ENTRY(__dma_unmap_area)
197 cmp w2, #DMA_TO_DEVICE 197 cmp w2, #DMA_TO_DEVICE
198 b.ne __dma_inv_range 198 b.ne __dma_inv_range
199 ret 199 ret
200ENDPROC(__dma_unmap_area) 200ENDPIPROC(__dma_unmap_area)
diff --git a/arch/arm64/mm/context.c b/arch/arm64/mm/context.c
index d70ff14dbdbd..f636a2639f03 100644
--- a/arch/arm64/mm/context.c
+++ b/arch/arm64/mm/context.c
@@ -17,135 +17,185 @@
17 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */ 18 */
19 19
20#include <linux/init.h> 20#include <linux/bitops.h>
21#include <linux/sched.h> 21#include <linux/sched.h>
22#include <linux/slab.h>
22#include <linux/mm.h> 23#include <linux/mm.h>
23#include <linux/smp.h>
24#include <linux/percpu.h>
25 24
25#include <asm/cpufeature.h>
26#include <asm/mmu_context.h> 26#include <asm/mmu_context.h>
27#include <asm/tlbflush.h> 27#include <asm/tlbflush.h>
28#include <asm/cachetype.h>
29 28
30#define asid_bits(reg) \ 29static u32 asid_bits;
31 (((read_cpuid(ID_AA64MMFR0_EL1) & 0xf0) >> 2) + 8) 30static DEFINE_RAW_SPINLOCK(cpu_asid_lock);
32 31
33#define ASID_FIRST_VERSION (1 << MAX_ASID_BITS) 32static atomic64_t asid_generation;
33static unsigned long *asid_map;
34 34
35static DEFINE_RAW_SPINLOCK(cpu_asid_lock); 35static DEFINE_PER_CPU(atomic64_t, active_asids);
36unsigned int cpu_last_asid = ASID_FIRST_VERSION; 36static DEFINE_PER_CPU(u64, reserved_asids);
37static cpumask_t tlb_flush_pending;
37 38
38/* 39#define ASID_MASK (~GENMASK(asid_bits - 1, 0))
39 * We fork()ed a process, and we need a new context for the child to run in. 40#define ASID_FIRST_VERSION (1UL << asid_bits)
40 */ 41#define NUM_USER_ASIDS ASID_FIRST_VERSION
41void __init_new_context(struct task_struct *tsk, struct mm_struct *mm)
42{
43 mm->context.id = 0;
44 raw_spin_lock_init(&mm->context.id_lock);
45}
46 42
47static void flush_context(void) 43static void flush_context(unsigned int cpu)
48{ 44{
49 /* set the reserved TTBR0 before flushing the TLB */ 45 int i;
50 cpu_set_reserved_ttbr0(); 46 u64 asid;
51 flush_tlb_all();
52 if (icache_is_aivivt())
53 __flush_icache_all();
54}
55 47
56static void set_mm_context(struct mm_struct *mm, unsigned int asid) 48 /* Update the list of reserved ASIDs and the ASID bitmap. */
57{ 49 bitmap_clear(asid_map, 0, NUM_USER_ASIDS);
58 unsigned long flags;
59 50
60 /* 51 /*
61 * Locking needed for multi-threaded applications where the same 52 * Ensure the generation bump is observed before we xchg the
62 * mm->context.id could be set from different CPUs during the 53 * active_asids.
63 * broadcast. This function is also called via IPI so the
64 * mm->context.id_lock has to be IRQ-safe.
65 */ 54 */
66 raw_spin_lock_irqsave(&mm->context.id_lock, flags); 55 smp_wmb();
67 if (likely((mm->context.id ^ cpu_last_asid) >> MAX_ASID_BITS)) { 56
57 for_each_possible_cpu(i) {
58 asid = atomic64_xchg_relaxed(&per_cpu(active_asids, i), 0);
68 /* 59 /*
69 * Old version of ASID found. Set the new one and reset 60 * If this CPU has already been through a
70 * mm_cpumask(mm). 61 * rollover, but hasn't run another task in
62 * the meantime, we must preserve its reserved
63 * ASID, as this is the only trace we have of
64 * the process it is still running.
71 */ 65 */
72 mm->context.id = asid; 66 if (asid == 0)
73 cpumask_clear(mm_cpumask(mm)); 67 asid = per_cpu(reserved_asids, i);
68 __set_bit(asid & ~ASID_MASK, asid_map);
69 per_cpu(reserved_asids, i) = asid;
74 } 70 }
75 raw_spin_unlock_irqrestore(&mm->context.id_lock, flags);
76 71
77 /* 72 /* Queue a TLB invalidate and flush the I-cache if necessary. */
78 * Set the mm_cpumask(mm) bit for the current CPU. 73 cpumask_setall(&tlb_flush_pending);
79 */ 74
80 cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm)); 75 if (icache_is_aivivt())
76 __flush_icache_all();
81} 77}
82 78
83/* 79static int is_reserved_asid(u64 asid)
84 * Reset the ASID on the current CPU. This function call is broadcast from the 80{
85 * CPU handling the ASID rollover and holding cpu_asid_lock. 81 int cpu;
86 */ 82 for_each_possible_cpu(cpu)
87static void reset_context(void *info) 83 if (per_cpu(reserved_asids, cpu) == asid)
84 return 1;
85 return 0;
86}
87
88static u64 new_context(struct mm_struct *mm, unsigned int cpu)
88{ 89{
89 unsigned int asid; 90 static u32 cur_idx = 1;
90 unsigned int cpu = smp_processor_id(); 91 u64 asid = atomic64_read(&mm->context.id);
91 struct mm_struct *mm = current->active_mm; 92 u64 generation = atomic64_read(&asid_generation);
93
94 if (asid != 0) {
95 /*
96 * If our current ASID was active during a rollover, we
97 * can continue to use it and this was just a false alarm.
98 */
99 if (is_reserved_asid(asid))
100 return generation | (asid & ~ASID_MASK);
101
102 /*
103 * We had a valid ASID in a previous life, so try to re-use
104 * it if possible.
105 */
106 asid &= ~ASID_MASK;
107 if (!__test_and_set_bit(asid, asid_map))
108 goto bump_gen;
109 }
92 110
93 /* 111 /*
94 * current->active_mm could be init_mm for the idle thread immediately 112 * Allocate a free ASID. If we can't find one, take a note of the
95 * after secondary CPU boot or hotplug. TTBR0_EL1 is already set to 113 * currently active ASIDs and mark the TLBs as requiring flushes.
96 * the reserved value, so no need to reset any context. 114 * We always count from ASID #1, as we use ASID #0 when setting a
115 * reserved TTBR0 for the init_mm.
97 */ 116 */
98 if (mm == &init_mm) 117 asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, cur_idx);
99 return; 118 if (asid != NUM_USER_ASIDS)
119 goto set_asid;
100 120
101 smp_rmb(); 121 /* We're out of ASIDs, so increment the global generation count */
102 asid = cpu_last_asid + cpu; 122 generation = atomic64_add_return_relaxed(ASID_FIRST_VERSION,
123 &asid_generation);
124 flush_context(cpu);
103 125
104 flush_context(); 126 /* We have at least 1 ASID per CPU, so this will always succeed */
105 set_mm_context(mm, asid); 127 asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, 1);
106 128
107 /* set the new ASID */ 129set_asid:
108 cpu_switch_mm(mm->pgd, mm); 130 __set_bit(asid, asid_map);
131 cur_idx = asid;
132
133bump_gen:
134 asid |= generation;
135 return asid;
109} 136}
110 137
111void __new_context(struct mm_struct *mm) 138void check_and_switch_context(struct mm_struct *mm, unsigned int cpu)
112{ 139{
113 unsigned int asid; 140 unsigned long flags;
114 unsigned int bits = asid_bits(); 141 u64 asid;
142
143 asid = atomic64_read(&mm->context.id);
115 144
116 raw_spin_lock(&cpu_asid_lock);
117 /* 145 /*
118 * Check the ASID again, in case the change was broadcast from another 146 * The memory ordering here is subtle. We rely on the control
119 * CPU before we acquired the lock. 147 * dependency between the generation read and the update of
148 * active_asids to ensure that we are synchronised with a
149 * parallel rollover (i.e. this pairs with the smp_wmb() in
150 * flush_context).
120 */ 151 */
121 if (!unlikely((mm->context.id ^ cpu_last_asid) >> MAX_ASID_BITS)) { 152 if (!((asid ^ atomic64_read(&asid_generation)) >> asid_bits)
122 cpumask_set_cpu(smp_processor_id(), mm_cpumask(mm)); 153 && atomic64_xchg_relaxed(&per_cpu(active_asids, cpu), asid))
123 raw_spin_unlock(&cpu_asid_lock); 154 goto switch_mm_fastpath;
124 return; 155
156 raw_spin_lock_irqsave(&cpu_asid_lock, flags);
157 /* Check that our ASID belongs to the current generation. */
158 asid = atomic64_read(&mm->context.id);
159 if ((asid ^ atomic64_read(&asid_generation)) >> asid_bits) {
160 asid = new_context(mm, cpu);
161 atomic64_set(&mm->context.id, asid);
125 } 162 }
126 /*
127 * At this point, it is guaranteed that the current mm (with an old
128 * ASID) isn't active on any other CPU since the ASIDs are changed
129 * simultaneously via IPI.
130 */
131 asid = ++cpu_last_asid;
132 163
133 /* 164 if (cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending))
134 * If we've used up all our ASIDs, we need to start a new version and 165 local_flush_tlb_all();
135 * flush the TLB. 166
136 */ 167 atomic64_set(&per_cpu(active_asids, cpu), asid);
137 if (unlikely((asid & ((1 << bits) - 1)) == 0)) { 168 raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
138 /* increment the ASID version */ 169
139 cpu_last_asid += (1 << MAX_ASID_BITS) - (1 << bits); 170switch_mm_fastpath:
140 if (cpu_last_asid == 0) 171 cpu_switch_mm(mm->pgd, mm);
141 cpu_last_asid = ASID_FIRST_VERSION; 172}
142 asid = cpu_last_asid + smp_processor_id(); 173
143 flush_context(); 174static int asids_init(void)
144 smp_wmb(); 175{
145 smp_call_function(reset_context, NULL, 1); 176 int fld = cpuid_feature_extract_field(read_cpuid(ID_AA64MMFR0_EL1), 4);
146 cpu_last_asid += NR_CPUS - 1; 177
178 switch (fld) {
179 default:
180 pr_warn("Unknown ASID size (%d); assuming 8-bit\n", fld);
181 /* Fallthrough */
182 case 0:
183 asid_bits = 8;
184 break;
185 case 2:
186 asid_bits = 16;
147 } 187 }
148 188
149 set_mm_context(mm, asid); 189 /* If we end up with more CPUs than ASIDs, expect things to crash */
150 raw_spin_unlock(&cpu_asid_lock); 190 WARN_ON(NUM_USER_ASIDS < num_possible_cpus());
191 atomic64_set(&asid_generation, ASID_FIRST_VERSION);
192 asid_map = kzalloc(BITS_TO_LONGS(NUM_USER_ASIDS) * sizeof(*asid_map),
193 GFP_KERNEL);
194 if (!asid_map)
195 panic("Failed to allocate bitmap for %lu ASIDs\n",
196 NUM_USER_ASIDS);
197
198 pr_info("ASID allocator initialised with %lu entries\n", NUM_USER_ASIDS);
199 return 0;
151} 200}
201early_initcall(asids_init);
diff --git a/arch/arm64/mm/dump.c b/arch/arm64/mm/dump.c
index f3d6221cd5bd..5a22a119a74c 100644
--- a/arch/arm64/mm/dump.c
+++ b/arch/arm64/mm/dump.c
@@ -67,6 +67,12 @@ static struct addr_marker address_markers[] = {
67 { -1, NULL }, 67 { -1, NULL },
68}; 68};
69 69
70/*
71 * The page dumper groups page table entries of the same type into a single
72 * description. It uses pg_state to track the range information while
73 * iterating over the pte entries. When the continuity is broken it then
74 * dumps out a description of the range.
75 */
70struct pg_state { 76struct pg_state {
71 struct seq_file *seq; 77 struct seq_file *seq;
72 const struct addr_marker *marker; 78 const struct addr_marker *marker;
@@ -114,6 +120,16 @@ static const struct prot_bits pte_bits[] = {
114 .set = "NG", 120 .set = "NG",
115 .clear = " ", 121 .clear = " ",
116 }, { 122 }, {
123 .mask = PTE_CONT,
124 .val = PTE_CONT,
125 .set = "CON",
126 .clear = " ",
127 }, {
128 .mask = PTE_TABLE_BIT,
129 .val = PTE_TABLE_BIT,
130 .set = " ",
131 .clear = "BLK",
132 }, {
117 .mask = PTE_UXN, 133 .mask = PTE_UXN,
118 .val = PTE_UXN, 134 .val = PTE_UXN,
119 .set = "UXN", 135 .set = "UXN",
@@ -198,7 +214,7 @@ static void note_page(struct pg_state *st, unsigned long addr, unsigned level,
198 unsigned long delta; 214 unsigned long delta;
199 215
200 if (st->current_prot) { 216 if (st->current_prot) {
201 seq_printf(st->seq, "0x%16lx-0x%16lx ", 217 seq_printf(st->seq, "0x%016lx-0x%016lx ",
202 st->start_address, addr); 218 st->start_address, addr);
203 219
204 delta = (addr - st->start_address) >> 10; 220 delta = (addr - st->start_address) >> 10;
diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
index 9fadf6d7039b..19211c4a8911 100644
--- a/arch/arm64/mm/fault.c
+++ b/arch/arm64/mm/fault.c
@@ -556,7 +556,7 @@ asmlinkage int __exception do_debug_exception(unsigned long addr,
556} 556}
557 557
558#ifdef CONFIG_ARM64_PAN 558#ifdef CONFIG_ARM64_PAN
559void cpu_enable_pan(void) 559void cpu_enable_pan(void *__unused)
560{ 560{
561 config_sctlr_el1(SCTLR_EL1_SPAN, 0); 561 config_sctlr_el1(SCTLR_EL1_SPAN, 0);
562} 562}
diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c
index f5c0680d17d9..17bf39ac83ba 100644
--- a/arch/arm64/mm/init.c
+++ b/arch/arm64/mm/init.c
@@ -86,10 +86,10 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max)
86 memset(zone_size, 0, sizeof(zone_size)); 86 memset(zone_size, 0, sizeof(zone_size));
87 87
88 /* 4GB maximum for 32-bit only capable devices */ 88 /* 4GB maximum for 32-bit only capable devices */
89 if (IS_ENABLED(CONFIG_ZONE_DMA)) { 89#ifdef CONFIG_ZONE_DMA
90 max_dma = PFN_DOWN(arm64_dma_phys_limit); 90 max_dma = PFN_DOWN(arm64_dma_phys_limit);
91 zone_size[ZONE_DMA] = max_dma - min; 91 zone_size[ZONE_DMA] = max_dma - min;
92 } 92#endif
93 zone_size[ZONE_NORMAL] = max - max_dma; 93 zone_size[ZONE_NORMAL] = max - max_dma;
94 94
95 memcpy(zhole_size, zone_size, sizeof(zhole_size)); 95 memcpy(zhole_size, zone_size, sizeof(zhole_size));
@@ -101,11 +101,12 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max)
101 if (start >= max) 101 if (start >= max)
102 continue; 102 continue;
103 103
104 if (IS_ENABLED(CONFIG_ZONE_DMA) && start < max_dma) { 104#ifdef CONFIG_ZONE_DMA
105 if (start < max_dma) {
105 unsigned long dma_end = min(end, max_dma); 106 unsigned long dma_end = min(end, max_dma);
106 zhole_size[ZONE_DMA] -= dma_end - start; 107 zhole_size[ZONE_DMA] -= dma_end - start;
107 } 108 }
108 109#endif
109 if (end > max_dma) { 110 if (end > max_dma) {
110 unsigned long normal_end = min(end, max); 111 unsigned long normal_end = min(end, max);
111 unsigned long normal_start = max(start, max_dma); 112 unsigned long normal_start = max(start, max_dma);
@@ -298,6 +299,9 @@ void __init mem_init(void)
298#define MLK_ROUNDUP(b, t) b, t, DIV_ROUND_UP(((t) - (b)), SZ_1K) 299#define MLK_ROUNDUP(b, t) b, t, DIV_ROUND_UP(((t) - (b)), SZ_1K)
299 300
300 pr_notice("Virtual kernel memory layout:\n" 301 pr_notice("Virtual kernel memory layout:\n"
302#ifdef CONFIG_KASAN
303 " kasan : 0x%16lx - 0x%16lx (%6ld GB)\n"
304#endif
301 " vmalloc : 0x%16lx - 0x%16lx (%6ld GB)\n" 305 " vmalloc : 0x%16lx - 0x%16lx (%6ld GB)\n"
302#ifdef CONFIG_SPARSEMEM_VMEMMAP 306#ifdef CONFIG_SPARSEMEM_VMEMMAP
303 " vmemmap : 0x%16lx - 0x%16lx (%6ld GB maximum)\n" 307 " vmemmap : 0x%16lx - 0x%16lx (%6ld GB maximum)\n"
@@ -310,6 +314,9 @@ void __init mem_init(void)
310 " .init : 0x%p" " - 0x%p" " (%6ld KB)\n" 314 " .init : 0x%p" " - 0x%p" " (%6ld KB)\n"
311 " .text : 0x%p" " - 0x%p" " (%6ld KB)\n" 315 " .text : 0x%p" " - 0x%p" " (%6ld KB)\n"
312 " .data : 0x%p" " - 0x%p" " (%6ld KB)\n", 316 " .data : 0x%p" " - 0x%p" " (%6ld KB)\n",
317#ifdef CONFIG_KASAN
318 MLG(KASAN_SHADOW_START, KASAN_SHADOW_END),
319#endif
313 MLG(VMALLOC_START, VMALLOC_END), 320 MLG(VMALLOC_START, VMALLOC_END),
314#ifdef CONFIG_SPARSEMEM_VMEMMAP 321#ifdef CONFIG_SPARSEMEM_VMEMMAP
315 MLG((unsigned long)vmemmap, 322 MLG((unsigned long)vmemmap,
diff --git a/arch/arm64/mm/kasan_init.c b/arch/arm64/mm/kasan_init.c
new file mode 100644
index 000000000000..cf038c7d9fa9
--- /dev/null
+++ b/arch/arm64/mm/kasan_init.c
@@ -0,0 +1,165 @@
1/*
2 * This file contains kasan initialization code for ARM64.
3 *
4 * Copyright (c) 2015 Samsung Electronics Co., Ltd.
5 * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 */
12
13#define pr_fmt(fmt) "kasan: " fmt
14#include <linux/kasan.h>
15#include <linux/kernel.h>
16#include <linux/memblock.h>
17#include <linux/start_kernel.h>
18
19#include <asm/page.h>
20#include <asm/pgalloc.h>
21#include <asm/pgtable.h>
22#include <asm/tlbflush.h>
23
24static pgd_t tmp_pg_dir[PTRS_PER_PGD] __initdata __aligned(PGD_SIZE);
25
26static void __init kasan_early_pte_populate(pmd_t *pmd, unsigned long addr,
27 unsigned long end)
28{
29 pte_t *pte;
30 unsigned long next;
31
32 if (pmd_none(*pmd))
33 pmd_populate_kernel(&init_mm, pmd, kasan_zero_pte);
34
35 pte = pte_offset_kernel(pmd, addr);
36 do {
37 next = addr + PAGE_SIZE;
38 set_pte(pte, pfn_pte(virt_to_pfn(kasan_zero_page),
39 PAGE_KERNEL));
40 } while (pte++, addr = next, addr != end && pte_none(*pte));
41}
42
43static void __init kasan_early_pmd_populate(pud_t *pud,
44 unsigned long addr,
45 unsigned long end)
46{
47 pmd_t *pmd;
48 unsigned long next;
49
50 if (pud_none(*pud))
51 pud_populate(&init_mm, pud, kasan_zero_pmd);
52
53 pmd = pmd_offset(pud, addr);
54 do {
55 next = pmd_addr_end(addr, end);
56 kasan_early_pte_populate(pmd, addr, next);
57 } while (pmd++, addr = next, addr != end && pmd_none(*pmd));
58}
59
60static void __init kasan_early_pud_populate(pgd_t *pgd,
61 unsigned long addr,
62 unsigned long end)
63{
64 pud_t *pud;
65 unsigned long next;
66
67 if (pgd_none(*pgd))
68 pgd_populate(&init_mm, pgd, kasan_zero_pud);
69
70 pud = pud_offset(pgd, addr);
71 do {
72 next = pud_addr_end(addr, end);
73 kasan_early_pmd_populate(pud, addr, next);
74 } while (pud++, addr = next, addr != end && pud_none(*pud));
75}
76
77static void __init kasan_map_early_shadow(void)
78{
79 unsigned long addr = KASAN_SHADOW_START;
80 unsigned long end = KASAN_SHADOW_END;
81 unsigned long next;
82 pgd_t *pgd;
83
84 pgd = pgd_offset_k(addr);
85 do {
86 next = pgd_addr_end(addr, end);
87 kasan_early_pud_populate(pgd, addr, next);
88 } while (pgd++, addr = next, addr != end);
89}
90
91asmlinkage void __init kasan_early_init(void)
92{
93 BUILD_BUG_ON(KASAN_SHADOW_OFFSET != KASAN_SHADOW_END - (1UL << 61));
94 BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_START, PGDIR_SIZE));
95 BUILD_BUG_ON(!IS_ALIGNED(KASAN_SHADOW_END, PGDIR_SIZE));
96 kasan_map_early_shadow();
97}
98
99static void __init clear_pgds(unsigned long start,
100 unsigned long end)
101{
102 /*
103 * Remove references to kasan page tables from
104 * swapper_pg_dir. pgd_clear() can't be used
105 * here because it's nop on 2,3-level pagetable setups
106 */
107 for (; start < end; start += PGDIR_SIZE)
108 set_pgd(pgd_offset_k(start), __pgd(0));
109}
110
111static void __init cpu_set_ttbr1(unsigned long ttbr1)
112{
113 asm(
114 " msr ttbr1_el1, %0\n"
115 " isb"
116 :
117 : "r" (ttbr1));
118}
119
120void __init kasan_init(void)
121{
122 struct memblock_region *reg;
123
124 /*
125 * We are going to perform proper setup of shadow memory.
126 * At first we should unmap early shadow (clear_pgds() call bellow).
127 * However, instrumented code couldn't execute without shadow memory.
128 * tmp_pg_dir used to keep early shadow mapped until full shadow
129 * setup will be finished.
130 */
131 memcpy(tmp_pg_dir, swapper_pg_dir, sizeof(tmp_pg_dir));
132 cpu_set_ttbr1(__pa(tmp_pg_dir));
133 flush_tlb_all();
134
135 clear_pgds(KASAN_SHADOW_START, KASAN_SHADOW_END);
136
137 kasan_populate_zero_shadow((void *)KASAN_SHADOW_START,
138 kasan_mem_to_shadow((void *)MODULES_VADDR));
139
140 for_each_memblock(memory, reg) {
141 void *start = (void *)__phys_to_virt(reg->base);
142 void *end = (void *)__phys_to_virt(reg->base + reg->size);
143
144 if (start >= end)
145 break;
146
147 /*
148 * end + 1 here is intentional. We check several shadow bytes in
149 * advance to slightly speed up fastpath. In some rare cases
150 * we could cross boundary of mapped shadow, so we just map
151 * some more here.
152 */
153 vmemmap_populate((unsigned long)kasan_mem_to_shadow(start),
154 (unsigned long)kasan_mem_to_shadow(end) + 1,
155 pfn_to_nid(virt_to_pfn(start)));
156 }
157
158 memset(kasan_zero_page, 0, PAGE_SIZE);
159 cpu_set_ttbr1(__pa(swapper_pg_dir));
160 flush_tlb_all();
161
162 /* At this point kasan is fully initialized. Enable error messages */
163 init_task.kasan_depth = 0;
164 pr_info("KernelAddressSanitizer initialized\n");
165}
diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 9211b8527f25..c2fa6b56613c 100644
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -32,6 +32,7 @@
32 32
33#include <asm/cputype.h> 33#include <asm/cputype.h>
34#include <asm/fixmap.h> 34#include <asm/fixmap.h>
35#include <asm/kernel-pgtable.h>
35#include <asm/sections.h> 36#include <asm/sections.h>
36#include <asm/setup.h> 37#include <asm/setup.h>
37#include <asm/sizes.h> 38#include <asm/sizes.h>
@@ -80,19 +81,55 @@ static void split_pmd(pmd_t *pmd, pte_t *pte)
80 do { 81 do {
81 /* 82 /*
82 * Need to have the least restrictive permissions available 83 * Need to have the least restrictive permissions available
83 * permissions will be fixed up later 84 * permissions will be fixed up later. Default the new page
85 * range as contiguous ptes.
84 */ 86 */
85 set_pte(pte, pfn_pte(pfn, PAGE_KERNEL_EXEC)); 87 set_pte(pte, pfn_pte(pfn, PAGE_KERNEL_EXEC_CONT));
86 pfn++; 88 pfn++;
87 } while (pte++, i++, i < PTRS_PER_PTE); 89 } while (pte++, i++, i < PTRS_PER_PTE);
88} 90}
89 91
92/*
93 * Given a PTE with the CONT bit set, determine where the CONT range
94 * starts, and clear the entire range of PTE CONT bits.
95 */
96static void clear_cont_pte_range(pte_t *pte, unsigned long addr)
97{
98 int i;
99
100 pte -= CONT_RANGE_OFFSET(addr);
101 for (i = 0; i < CONT_PTES; i++) {
102 set_pte(pte, pte_mknoncont(*pte));
103 pte++;
104 }
105 flush_tlb_all();
106}
107
108/*
109 * Given a range of PTEs set the pfn and provided page protection flags
110 */
111static void __populate_init_pte(pte_t *pte, unsigned long addr,
112 unsigned long end, phys_addr_t phys,
113 pgprot_t prot)
114{
115 unsigned long pfn = __phys_to_pfn(phys);
116
117 do {
118 /* clear all the bits except the pfn, then apply the prot */
119 set_pte(pte, pfn_pte(pfn, prot));
120 pte++;
121 pfn++;
122 addr += PAGE_SIZE;
123 } while (addr != end);
124}
125
90static void alloc_init_pte(pmd_t *pmd, unsigned long addr, 126static void alloc_init_pte(pmd_t *pmd, unsigned long addr,
91 unsigned long end, unsigned long pfn, 127 unsigned long end, phys_addr_t phys,
92 pgprot_t prot, 128 pgprot_t prot,
93 void *(*alloc)(unsigned long size)) 129 void *(*alloc)(unsigned long size))
94{ 130{
95 pte_t *pte; 131 pte_t *pte;
132 unsigned long next;
96 133
97 if (pmd_none(*pmd) || pmd_sect(*pmd)) { 134 if (pmd_none(*pmd) || pmd_sect(*pmd)) {
98 pte = alloc(PTRS_PER_PTE * sizeof(pte_t)); 135 pte = alloc(PTRS_PER_PTE * sizeof(pte_t));
@@ -105,9 +142,27 @@ static void alloc_init_pte(pmd_t *pmd, unsigned long addr,
105 142
106 pte = pte_offset_kernel(pmd, addr); 143 pte = pte_offset_kernel(pmd, addr);
107 do { 144 do {
108 set_pte(pte, pfn_pte(pfn, prot)); 145 next = min(end, (addr + CONT_SIZE) & CONT_MASK);
109 pfn++; 146 if (((addr | next | phys) & ~CONT_MASK) == 0) {
110 } while (pte++, addr += PAGE_SIZE, addr != end); 147 /* a block of CONT_PTES */
148 __populate_init_pte(pte, addr, next, phys,
149 prot | __pgprot(PTE_CONT));
150 } else {
151 /*
152 * If the range being split is already inside of a
153 * contiguous range but this PTE isn't going to be
154 * contiguous, then we want to unmark the adjacent
155 * ranges, then update the portion of the range we
156 * are interrested in.
157 */
158 clear_cont_pte_range(pte, addr);
159 __populate_init_pte(pte, addr, next, phys, prot);
160 }
161
162 pte += (next - addr) >> PAGE_SHIFT;
163 phys += next - addr;
164 addr = next;
165 } while (addr != end);
111} 166}
112 167
113void split_pud(pud_t *old_pud, pmd_t *pmd) 168void split_pud(pud_t *old_pud, pmd_t *pmd)
@@ -168,8 +223,7 @@ static void alloc_init_pmd(struct mm_struct *mm, pud_t *pud,
168 } 223 }
169 } 224 }
170 } else { 225 } else {
171 alloc_init_pte(pmd, addr, next, __phys_to_pfn(phys), 226 alloc_init_pte(pmd, addr, next, phys, prot, alloc);
172 prot, alloc);
173 } 227 }
174 phys += next - addr; 228 phys += next - addr;
175 } while (pmd++, addr = next, addr != end); 229 } while (pmd++, addr = next, addr != end);
@@ -353,14 +407,11 @@ static void __init map_mem(void)
353 * memory addressable from the initial direct kernel mapping. 407 * memory addressable from the initial direct kernel mapping.
354 * 408 *
355 * The initial direct kernel mapping, located at swapper_pg_dir, gives 409 * The initial direct kernel mapping, located at swapper_pg_dir, gives
356 * us PUD_SIZE (4K pages) or PMD_SIZE (64K pages) memory starting from 410 * us PUD_SIZE (with SECTION maps) or PMD_SIZE (without SECTION maps,
357 * PHYS_OFFSET (which must be aligned to 2MB as per 411 * memory starting from PHYS_OFFSET (which must be aligned to 2MB as
358 * Documentation/arm64/booting.txt). 412 * per Documentation/arm64/booting.txt).
359 */ 413 */
360 if (IS_ENABLED(CONFIG_ARM64_64K_PAGES)) 414 limit = PHYS_OFFSET + SWAPPER_INIT_MAP_SIZE;
361 limit = PHYS_OFFSET + PMD_SIZE;
362 else
363 limit = PHYS_OFFSET + PUD_SIZE;
364 memblock_set_current_limit(limit); 415 memblock_set_current_limit(limit);
365 416
366 /* map all the memory banks */ 417 /* map all the memory banks */
@@ -371,21 +422,24 @@ static void __init map_mem(void)
371 if (start >= end) 422 if (start >= end)
372 break; 423 break;
373 424
374#ifndef CONFIG_ARM64_64K_PAGES 425 if (ARM64_SWAPPER_USES_SECTION_MAPS) {
375 /* 426 /*
376 * For the first memory bank align the start address and 427 * For the first memory bank align the start address and
377 * current memblock limit to prevent create_mapping() from 428 * current memblock limit to prevent create_mapping() from
378 * allocating pte page tables from unmapped memory. 429 * allocating pte page tables from unmapped memory. With
379 * When 64K pages are enabled, the pte page table for the 430 * the section maps, if the first block doesn't end on section
380 * first PGDIR_SIZE is already present in swapper_pg_dir. 431 * size boundary, create_mapping() will try to allocate a pte
381 */ 432 * page, which may be returned from an unmapped area.
382 if (start < limit) 433 * When section maps are not used, the pte page table for the
383 start = ALIGN(start, PMD_SIZE); 434 * current limit is already present in swapper_pg_dir.
384 if (end < limit) { 435 */
385 limit = end & PMD_MASK; 436 if (start < limit)
386 memblock_set_current_limit(limit); 437 start = ALIGN(start, SECTION_SIZE);
438 if (end < limit) {
439 limit = end & SECTION_MASK;
440 memblock_set_current_limit(limit);
441 }
387 } 442 }
388#endif
389 __map_memblock(start, end); 443 __map_memblock(start, end);
390 } 444 }
391 445
@@ -456,7 +510,7 @@ void __init paging_init(void)
456 * point to zero page to avoid speculatively fetching new entries. 510 * point to zero page to avoid speculatively fetching new entries.
457 */ 511 */
458 cpu_set_reserved_ttbr0(); 512 cpu_set_reserved_ttbr0();
459 flush_tlb_all(); 513 local_flush_tlb_all();
460 cpu_set_default_tcr_t0sz(); 514 cpu_set_default_tcr_t0sz();
461} 515}
462 516
@@ -498,12 +552,12 @@ int kern_addr_valid(unsigned long addr)
498 return pfn_valid(pte_pfn(*pte)); 552 return pfn_valid(pte_pfn(*pte));
499} 553}
500#ifdef CONFIG_SPARSEMEM_VMEMMAP 554#ifdef CONFIG_SPARSEMEM_VMEMMAP
501#ifdef CONFIG_ARM64_64K_PAGES 555#if !ARM64_SWAPPER_USES_SECTION_MAPS
502int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) 556int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
503{ 557{
504 return vmemmap_populate_basepages(start, end, node); 558 return vmemmap_populate_basepages(start, end, node);
505} 559}
506#else /* !CONFIG_ARM64_64K_PAGES */ 560#else /* !ARM64_SWAPPER_USES_SECTION_MAPS */
507int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node) 561int __meminit vmemmap_populate(unsigned long start, unsigned long end, int node)
508{ 562{
509 unsigned long addr = start; 563 unsigned long addr = start;
@@ -638,7 +692,7 @@ void *__init fixmap_remap_fdt(phys_addr_t dt_phys)
638{ 692{
639 const u64 dt_virt_base = __fix_to_virt(FIX_FDT); 693 const u64 dt_virt_base = __fix_to_virt(FIX_FDT);
640 pgprot_t prot = PAGE_KERNEL | PTE_RDONLY; 694 pgprot_t prot = PAGE_KERNEL | PTE_RDONLY;
641 int granularity, size, offset; 695 int size, offset;
642 void *dt_virt; 696 void *dt_virt;
643 697
644 /* 698 /*
@@ -664,24 +718,15 @@ void *__init fixmap_remap_fdt(phys_addr_t dt_phys)
664 */ 718 */
665 BUILD_BUG_ON(dt_virt_base % SZ_2M); 719 BUILD_BUG_ON(dt_virt_base % SZ_2M);
666 720
667 if (IS_ENABLED(CONFIG_ARM64_64K_PAGES)) { 721 BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> SWAPPER_TABLE_SHIFT !=
668 BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> PMD_SHIFT != 722 __fix_to_virt(FIX_BTMAP_BEGIN) >> SWAPPER_TABLE_SHIFT);
669 __fix_to_virt(FIX_BTMAP_BEGIN) >> PMD_SHIFT);
670
671 granularity = PAGE_SIZE;
672 } else {
673 BUILD_BUG_ON(__fix_to_virt(FIX_FDT_END) >> PUD_SHIFT !=
674 __fix_to_virt(FIX_BTMAP_BEGIN) >> PUD_SHIFT);
675
676 granularity = PMD_SIZE;
677 }
678 723
679 offset = dt_phys % granularity; 724 offset = dt_phys % SWAPPER_BLOCK_SIZE;
680 dt_virt = (void *)dt_virt_base + offset; 725 dt_virt = (void *)dt_virt_base + offset;
681 726
682 /* map the first chunk so we can read the size from the header */ 727 /* map the first chunk so we can read the size from the header */
683 create_mapping(round_down(dt_phys, granularity), dt_virt_base, 728 create_mapping(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base,
684 granularity, prot); 729 SWAPPER_BLOCK_SIZE, prot);
685 730
686 if (fdt_check_header(dt_virt) != 0) 731 if (fdt_check_header(dt_virt) != 0)
687 return NULL; 732 return NULL;
@@ -690,9 +735,9 @@ void *__init fixmap_remap_fdt(phys_addr_t dt_phys)
690 if (size > MAX_FDT_SIZE) 735 if (size > MAX_FDT_SIZE)
691 return NULL; 736 return NULL;
692 737
693 if (offset + size > granularity) 738 if (offset + size > SWAPPER_BLOCK_SIZE)
694 create_mapping(round_down(dt_phys, granularity), dt_virt_base, 739 create_mapping(round_down(dt_phys, SWAPPER_BLOCK_SIZE), dt_virt_base,
695 round_up(offset + size, granularity), prot); 740 round_up(offset + size, SWAPPER_BLOCK_SIZE), prot);
696 741
697 memblock_reserve(dt_phys, size); 742 memblock_reserve(dt_phys, size);
698 743
diff --git a/arch/arm64/mm/pageattr.c b/arch/arm64/mm/pageattr.c
index e47ed1c5dce1..3571c7309c5e 100644
--- a/arch/arm64/mm/pageattr.c
+++ b/arch/arm64/mm/pageattr.c
@@ -45,7 +45,7 @@ static int change_memory_common(unsigned long addr, int numpages,
45 int ret; 45 int ret;
46 struct page_change_data data; 46 struct page_change_data data;
47 47
48 if (!IS_ALIGNED(addr, PAGE_SIZE)) { 48 if (!PAGE_ALIGNED(addr)) {
49 start &= PAGE_MASK; 49 start &= PAGE_MASK;
50 end = start + size; 50 end = start + size;
51 WARN_ON_ONCE(1); 51 WARN_ON_ONCE(1);
diff --git a/arch/arm64/mm/pgd.c b/arch/arm64/mm/pgd.c
index 71ca104f97bd..cb3ba1b812e7 100644
--- a/arch/arm64/mm/pgd.c
+++ b/arch/arm64/mm/pgd.c
@@ -28,8 +28,6 @@
28 28
29#include "mm.h" 29#include "mm.h"
30 30
31#define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t))
32
33static struct kmem_cache *pgd_cache; 31static struct kmem_cache *pgd_cache;
34 32
35pgd_t *pgd_alloc(struct mm_struct *mm) 33pgd_t *pgd_alloc(struct mm_struct *mm)
diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index 7783ff05f74c..cacecc4ad3e5 100644
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -30,7 +30,9 @@
30 30
31#ifdef CONFIG_ARM64_64K_PAGES 31#ifdef CONFIG_ARM64_64K_PAGES
32#define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K 32#define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K
33#else 33#elif defined(CONFIG_ARM64_16K_PAGES)
34#define TCR_TG_FLAGS TCR_TG0_16K | TCR_TG1_16K
35#else /* CONFIG_ARM64_4K_PAGES */
34#define TCR_TG_FLAGS TCR_TG0_4K | TCR_TG1_4K 36#define TCR_TG_FLAGS TCR_TG0_4K | TCR_TG1_4K
35#endif 37#endif
36 38
@@ -130,7 +132,7 @@ ENDPROC(cpu_do_resume)
130 * - pgd_phys - physical address of new TTB 132 * - pgd_phys - physical address of new TTB
131 */ 133 */
132ENTRY(cpu_do_switch_mm) 134ENTRY(cpu_do_switch_mm)
133 mmid w1, x1 // get mm->context.id 135 mmid x1, x1 // get mm->context.id
134 bfi x0, x1, #48, #16 // set the ASID 136 bfi x0, x1, #48, #16 // set the ASID
135 msr ttbr0_el1, x0 // set TTBR0 137 msr ttbr0_el1, x0 // set TTBR0
136 isb 138 isb
@@ -146,8 +148,8 @@ ENDPROC(cpu_do_switch_mm)
146 * value of the SCTLR_EL1 register. 148 * value of the SCTLR_EL1 register.
147 */ 149 */
148ENTRY(__cpu_setup) 150ENTRY(__cpu_setup)
149 tlbi vmalle1is // invalidate I + D TLBs 151 tlbi vmalle1 // Invalidate local TLB
150 dsb ish 152 dsb nsh
151 153
152 mov x0, #3 << 20 154 mov x0, #3 << 20
153 msr cpacr_el1, x0 // Enable FP/ASIMD 155 msr cpacr_el1, x0 // Enable FP/ASIMD
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-31 17:36:56 -0400