aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
Diffstat (limited to 'arch')
-rw-r--r--arch/arm/boot/dts/imx51.dtsi8
-rw-r--r--arch/arm/boot/dts/imx53.dtsi14
-rw-r--r--arch/arm/boot/dts/imx6q.dtsi14
-rw-r--r--arch/arm/mach-at91/at91sam9g45.c13
-rw-r--r--arch/arm/mach-at91/at91sam9g45_devices.c128
-rw-r--r--arch/arm/mach-at91/include/mach/at91sam9g45.h2
-rw-r--r--arch/arm/mach-imx/mm-imx25.c10
-rw-r--r--arch/arm/mach-imx/mm-imx3.c7
-rw-r--r--arch/arm/mach-imx/mm-imx5.c40
-rw-r--r--arch/arm/mach-omap2/clock3xxx_data.c14
-rw-r--r--arch/arm/plat-spear/include/plat/keyboard.h2
-rw-r--r--arch/m68k/Kconfig.bus7
-rw-r--r--arch/m68k/Kconfig.cpu18
-rw-r--r--arch/m68k/Makefile2
-rw-r--r--arch/m68k/include/asm/cacheflush_mm.h41
-rw-r--r--arch/m68k/include/asm/dma.h8
-rw-r--r--arch/m68k/include/asm/gpio.h179
-rw-r--r--arch/m68k/include/asm/io_mm.h50
-rw-r--r--arch/m68k/include/asm/m520xsim.h14
-rw-r--r--arch/m68k/include/asm/m523xsim.h1
-rw-r--r--arch/m68k/include/asm/m525xsim.h194
-rw-r--r--arch/m68k/include/asm/m527xsim.h1
-rw-r--r--arch/m68k/include/asm/m528xsim.h2
-rw-r--r--arch/m68k/include/asm/m532xsim.h17
-rw-r--r--arch/m68k/include/asm/m5441xsim.h276
-rw-r--r--arch/m68k/include/asm/m54xxacr.h4
-rw-r--r--arch/m68k/include/asm/m54xxpci.h138
-rw-r--r--arch/m68k/include/asm/m54xxsim.h3
-rw-r--r--arch/m68k/include/asm/mcfclk.h43
-rw-r--r--arch/m68k/include/asm/mcfgpio.h343
-rw-r--r--arch/m68k/include/asm/mcfsim.h5
-rw-r--r--arch/m68k/include/asm/mcftimer.h2
-rw-r--r--arch/m68k/include/asm/mcfuart.h4
-rw-r--r--arch/m68k/include/asm/pci.h6
-rw-r--r--arch/m68k/include/asm/pinmux.h30
-rw-r--r--arch/m68k/kernel/Makefile1
-rw-r--r--arch/m68k/kernel/dma.c5
-rw-r--r--arch/m68k/kernel/entry.S452
-rw-r--r--arch/m68k/kernel/entry_mm.S419
-rw-r--r--arch/m68k/kernel/entry_no.S130
-rw-r--r--arch/m68k/kernel/pcibios.c109
-rw-r--r--arch/m68k/mm/memory.c2
-rw-r--r--arch/m68k/platform/coldfire/Makefile7
-rw-r--r--arch/m68k/platform/coldfire/clk.c108
-rw-r--r--arch/m68k/platform/coldfire/device.c57
-rw-r--r--arch/m68k/platform/coldfire/gpio.c172
-rw-r--r--arch/m68k/platform/coldfire/head.S6
-rw-r--r--arch/m68k/platform/coldfire/intc-525x.c91
-rw-r--r--arch/m68k/platform/coldfire/intc-simr.c26
-rw-r--r--arch/m68k/platform/coldfire/m5206.c9
-rw-r--r--arch/m68k/platform/coldfire/m520x.c103
-rw-r--r--arch/m68k/platform/coldfire/m523x.c22
-rw-r--r--arch/m68k/platform/coldfire/m5249.c10
-rw-r--r--arch/m68k/platform/coldfire/m525x.c66
-rw-r--r--arch/m68k/platform/coldfire/m5272.c11
-rw-r--r--arch/m68k/platform/coldfire/m527x.c43
-rw-r--r--arch/m68k/platform/coldfire/m528x.c33
-rw-r--r--arch/m68k/platform/coldfire/m5307.c9
-rw-r--r--arch/m68k/platform/coldfire/m532x.c154
-rw-r--r--arch/m68k/platform/coldfire/m5407.c9
-rw-r--r--arch/m68k/platform/coldfire/m5441x.c261
-rw-r--r--arch/m68k/platform/coldfire/m54xx.c7
-rw-r--r--arch/m68k/platform/coldfire/mcf8390.c38
-rw-r--r--arch/m68k/platform/coldfire/pci.c327
-rw-r--r--arch/m68k/platform/coldfire/pinmux.c28
-rw-r--r--arch/m68k/platform/coldfire/pit.c4
-rw-r--r--arch/m68k/platform/coldfire/timers.c2
-rw-r--r--arch/mips/kernel/kspd.c2
-rw-r--r--arch/powerpc/Makefile1
-rw-r--r--arch/s390/crypto/crypto_des.h18
-rw-r--r--arch/sh/include/asm/siu.h1
-rw-r--r--arch/sh/kernel/cpu/sh4a/setup-sh7722.c1
-rw-r--r--arch/sparc/include/asm/fixmap.h110
-rw-r--r--arch/sparc/include/asm/highmem.h3
-rw-r--r--arch/sparc/include/asm/leon.h1
-rw-r--r--arch/sparc/include/asm/mmu_context_32.h8
-rw-r--r--arch/sparc/include/asm/page_32.h3
-rw-r--r--arch/sparc/include/asm/pgalloc_32.h29
-rw-r--r--arch/sparc/include/asm/pgtable_32.h44
-rw-r--r--arch/sparc/include/asm/vaddrs.h22
-rw-r--r--arch/sparc/kernel/head_32.S2
-rw-r--r--arch/sparc/kernel/leon_kernel.c16
-rw-r--r--arch/sparc/kernel/process_32.c4
-rw-r--r--arch/sparc/kernel/setup_32.c1
-rw-r--r--arch/sparc/kernel/sys_sparc_64.c17
-rw-r--r--arch/sparc/lib/NG2memcpy.S72
-rw-r--r--arch/sparc/lib/U1memcpy.S4
-rw-r--r--arch/sparc/lib/copy_page.S56
-rw-r--r--arch/sparc/mm/fault_32.c18
-rw-r--r--arch/sparc/mm/highmem.c42
-rw-r--r--arch/sparc/mm/init_32.c58
-rw-r--r--arch/sparc/mm/srmmu.c332
-rw-r--r--arch/sparc/prom/init_32.c7
-rw-r--r--arch/sparc/prom/init_64.c4
-rw-r--r--arch/tile/kernel/pci.c2
-rw-r--r--arch/tile/kernel/pci_gx.c2
-rw-r--r--arch/x86/Kconfig.debug19
-rw-r--r--arch/x86/boot/compressed/cmdline.c4
-rw-r--r--arch/x86/boot/compressed/early_serial_console.c4
-rw-r--r--arch/x86/boot/compressed/eboot.c198
-rw-r--r--arch/x86/boot/compressed/head_32.S10
-rw-r--r--arch/x86/boot/compressed/head_64.S10
-rw-r--r--arch/x86/boot/compressed/misc.c31
-rw-r--r--arch/x86/boot/compressed/misc.h27
-rw-r--r--arch/x86/boot/header.S11
-rw-r--r--arch/x86/crypto/Makefile14
-rw-r--r--arch/x86/crypto/ablk_helper.c149
-rw-r--r--arch/x86/crypto/aes_glue.c2
-rw-r--r--arch/x86/crypto/aesni-intel_glue.c110
-rw-r--r--arch/x86/crypto/camellia_glue.c355
-rw-r--r--arch/x86/crypto/glue_helper.c307
-rw-r--r--arch/x86/crypto/serpent-avx-x86_64-asm_64.S704
-rw-r--r--arch/x86/crypto/serpent_avx_glue.c636
-rw-r--r--arch/x86/crypto/serpent_sse2_glue.c513
-rw-r--r--arch/x86/crypto/sha1_ssse3_asm.S2
-rw-r--r--arch/x86/crypto/sha1_ssse3_glue.c6
-rw-r--r--arch/x86/crypto/twofish-avx-x86_64-asm_64.S300
-rw-r--r--arch/x86/crypto/twofish_avx_glue.c624
-rw-r--r--arch/x86/crypto/twofish_glue_3way.c409
-rw-r--r--arch/x86/include/asm/apic.h2
-rw-r--r--arch/x86/include/asm/bootparam.h1
-rw-r--r--arch/x86/include/asm/cpufeature.h2
-rw-r--r--arch/x86/include/asm/crypto/ablk_helper.h31
-rw-r--r--arch/x86/include/asm/crypto/aes.h (renamed from arch/x86/include/asm/aes.h)0
-rw-r--r--arch/x86/include/asm/crypto/glue_helper.h115
-rw-r--r--arch/x86/include/asm/crypto/serpent-avx.h32
-rw-r--r--arch/x86/include/asm/crypto/serpent-sse2.h (renamed from arch/x86/include/asm/serpent.h)4
-rw-r--r--arch/x86/include/asm/crypto/twofish.h46
-rw-r--r--arch/x86/include/asm/entry_arch.h9
-rw-r--r--arch/x86/include/asm/irq_vectors.h11
-rw-r--r--arch/x86/include/asm/paravirt.h5
-rw-r--r--arch/x86/include/asm/paravirt_types.h3
-rw-r--r--arch/x86/include/asm/percpu.h17
-rw-r--r--arch/x86/include/asm/processor.h13
-rw-r--r--arch/x86/include/asm/smp.h16
-rw-r--r--arch/x86/include/asm/tlb.h9
-rw-r--r--arch/x86/include/asm/tlbflush.h49
-rw-r--r--arch/x86/include/asm/uv/uv.h5
-rw-r--r--arch/x86/kernel/apic/apic.c6
-rw-r--r--arch/x86/kernel/cpu/Makefile2
-rw-r--r--arch/x86/kernel/cpu/common.c31
-rw-r--r--arch/x86/kernel/cpu/cpu.h9
-rw-r--r--arch/x86/kernel/cpu/intel.c176
-rw-r--r--arch/x86/kernel/cpu/sched.c55
-rw-r--r--arch/x86/kernel/entry_64.S18
-rw-r--r--arch/x86/kernel/irqinit.c73
-rw-r--r--arch/x86/kernel/setup_percpu.c2
-rw-r--r--arch/x86/kernel/smpboot.c8
-rw-r--r--arch/x86/mm/pageattr.c10
-rw-r--r--arch/x86/mm/tlb.c401
-rw-r--r--arch/x86/platform/efi/efi.c30
-rw-r--r--arch/x86/platform/uv/tlb_uv.c6
-rw-r--r--arch/x86/xen/mmu.c12
153 files changed, 7519 insertions, 3376 deletions
diff --git a/arch/arm/boot/dts/imx51.dtsi b/arch/arm/boot/dts/imx51.dtsi
index 922adefdd291..53cbaa3d4f90 100644
--- a/arch/arm/boot/dts/imx51.dtsi
+++ b/arch/arm/boot/dts/imx51.dtsi
@@ -127,7 +127,7 @@
127 }; 127 };
128 128
129 gpio1: gpio@73f84000 { 129 gpio1: gpio@73f84000 {
130 compatible = "fsl,imx51-gpio", "fsl,imx31-gpio"; 130 compatible = "fsl,imx51-gpio", "fsl,imx35-gpio";
131 reg = <0x73f84000 0x4000>; 131 reg = <0x73f84000 0x4000>;
132 interrupts = <50 51>; 132 interrupts = <50 51>;
133 gpio-controller; 133 gpio-controller;
@@ -137,7 +137,7 @@
137 }; 137 };
138 138
139 gpio2: gpio@73f88000 { 139 gpio2: gpio@73f88000 {
140 compatible = "fsl,imx51-gpio", "fsl,imx31-gpio"; 140 compatible = "fsl,imx51-gpio", "fsl,imx35-gpio";
141 reg = <0x73f88000 0x4000>; 141 reg = <0x73f88000 0x4000>;
142 interrupts = <52 53>; 142 interrupts = <52 53>;
143 gpio-controller; 143 gpio-controller;
@@ -147,7 +147,7 @@
147 }; 147 };
148 148
149 gpio3: gpio@73f8c000 { 149 gpio3: gpio@73f8c000 {
150 compatible = "fsl,imx51-gpio", "fsl,imx31-gpio"; 150 compatible = "fsl,imx51-gpio", "fsl,imx35-gpio";
151 reg = <0x73f8c000 0x4000>; 151 reg = <0x73f8c000 0x4000>;
152 interrupts = <54 55>; 152 interrupts = <54 55>;
153 gpio-controller; 153 gpio-controller;
@@ -157,7 +157,7 @@
157 }; 157 };
158 158
159 gpio4: gpio@73f90000 { 159 gpio4: gpio@73f90000 {
160 compatible = "fsl,imx51-gpio", "fsl,imx31-gpio"; 160 compatible = "fsl,imx51-gpio", "fsl,imx35-gpio";
161 reg = <0x73f90000 0x4000>; 161 reg = <0x73f90000 0x4000>;
162 interrupts = <56 57>; 162 interrupts = <56 57>;
163 gpio-controller; 163 gpio-controller;
diff --git a/arch/arm/boot/dts/imx53.dtsi b/arch/arm/boot/dts/imx53.dtsi
index 4e735edc78ed..fc79cdc4b4e6 100644
--- a/arch/arm/boot/dts/imx53.dtsi
+++ b/arch/arm/boot/dts/imx53.dtsi
@@ -129,7 +129,7 @@
129 }; 129 };
130 130
131 gpio1: gpio@53f84000 { 131 gpio1: gpio@53f84000 {
132 compatible = "fsl,imx53-gpio", "fsl,imx31-gpio"; 132 compatible = "fsl,imx53-gpio", "fsl,imx35-gpio";
133 reg = <0x53f84000 0x4000>; 133 reg = <0x53f84000 0x4000>;
134 interrupts = <50 51>; 134 interrupts = <50 51>;
135 gpio-controller; 135 gpio-controller;
@@ -139,7 +139,7 @@
139 }; 139 };
140 140
141 gpio2: gpio@53f88000 { 141 gpio2: gpio@53f88000 {
142 compatible = "fsl,imx53-gpio", "fsl,imx31-gpio"; 142 compatible = "fsl,imx53-gpio", "fsl,imx35-gpio";
143 reg = <0x53f88000 0x4000>; 143 reg = <0x53f88000 0x4000>;
144 interrupts = <52 53>; 144 interrupts = <52 53>;
145 gpio-controller; 145 gpio-controller;
@@ -149,7 +149,7 @@
149 }; 149 };
150 150
151 gpio3: gpio@53f8c000 { 151 gpio3: gpio@53f8c000 {
152 compatible = "fsl,imx53-gpio", "fsl,imx31-gpio"; 152 compatible = "fsl,imx53-gpio", "fsl,imx35-gpio";
153 reg = <0x53f8c000 0x4000>; 153 reg = <0x53f8c000 0x4000>;
154 interrupts = <54 55>; 154 interrupts = <54 55>;
155 gpio-controller; 155 gpio-controller;
@@ -159,7 +159,7 @@
159 }; 159 };
160 160
161 gpio4: gpio@53f90000 { 161 gpio4: gpio@53f90000 {
162 compatible = "fsl,imx53-gpio", "fsl,imx31-gpio"; 162 compatible = "fsl,imx53-gpio", "fsl,imx35-gpio";
163 reg = <0x53f90000 0x4000>; 163 reg = <0x53f90000 0x4000>;
164 interrupts = <56 57>; 164 interrupts = <56 57>;
165 gpio-controller; 165 gpio-controller;
@@ -197,7 +197,7 @@
197 }; 197 };
198 198
199 gpio5: gpio@53fdc000 { 199 gpio5: gpio@53fdc000 {
200 compatible = "fsl,imx53-gpio", "fsl,imx31-gpio"; 200 compatible = "fsl,imx53-gpio", "fsl,imx35-gpio";
201 reg = <0x53fdc000 0x4000>; 201 reg = <0x53fdc000 0x4000>;
202 interrupts = <103 104>; 202 interrupts = <103 104>;
203 gpio-controller; 203 gpio-controller;
@@ -207,7 +207,7 @@
207 }; 207 };
208 208
209 gpio6: gpio@53fe0000 { 209 gpio6: gpio@53fe0000 {
210 compatible = "fsl,imx53-gpio", "fsl,imx31-gpio"; 210 compatible = "fsl,imx53-gpio", "fsl,imx35-gpio";
211 reg = <0x53fe0000 0x4000>; 211 reg = <0x53fe0000 0x4000>;
212 interrupts = <105 106>; 212 interrupts = <105 106>;
213 gpio-controller; 213 gpio-controller;
@@ -217,7 +217,7 @@
217 }; 217 };
218 218
219 gpio7: gpio@53fe4000 { 219 gpio7: gpio@53fe4000 {
220 compatible = "fsl,imx53-gpio", "fsl,imx31-gpio"; 220 compatible = "fsl,imx53-gpio", "fsl,imx35-gpio";
221 reg = <0x53fe4000 0x4000>; 221 reg = <0x53fe4000 0x4000>;
222 interrupts = <107 108>; 222 interrupts = <107 108>;
223 gpio-controller; 223 gpio-controller;
diff --git a/arch/arm/boot/dts/imx6q.dtsi b/arch/arm/boot/dts/imx6q.dtsi
index c25d49584814..3d3c64b014e6 100644
--- a/arch/arm/boot/dts/imx6q.dtsi
+++ b/arch/arm/boot/dts/imx6q.dtsi
@@ -277,7 +277,7 @@
277 }; 277 };
278 278
279 gpio1: gpio@0209c000 { 279 gpio1: gpio@0209c000 {
280 compatible = "fsl,imx6q-gpio", "fsl,imx31-gpio"; 280 compatible = "fsl,imx6q-gpio", "fsl,imx35-gpio";
281 reg = <0x0209c000 0x4000>; 281 reg = <0x0209c000 0x4000>;
282 interrupts = <0 66 0x04 0 67 0x04>; 282 interrupts = <0 66 0x04 0 67 0x04>;
283 gpio-controller; 283 gpio-controller;
@@ -287,7 +287,7 @@
287 }; 287 };
288 288
289 gpio2: gpio@020a0000 { 289 gpio2: gpio@020a0000 {
290 compatible = "fsl,imx6q-gpio", "fsl,imx31-gpio"; 290 compatible = "fsl,imx6q-gpio", "fsl,imx35-gpio";
291 reg = <0x020a0000 0x4000>; 291 reg = <0x020a0000 0x4000>;
292 interrupts = <0 68 0x04 0 69 0x04>; 292 interrupts = <0 68 0x04 0 69 0x04>;
293 gpio-controller; 293 gpio-controller;
@@ -297,7 +297,7 @@
297 }; 297 };
298 298
299 gpio3: gpio@020a4000 { 299 gpio3: gpio@020a4000 {
300 compatible = "fsl,imx6q-gpio", "fsl,imx31-gpio"; 300 compatible = "fsl,imx6q-gpio", "fsl,imx35-gpio";
301 reg = <0x020a4000 0x4000>; 301 reg = <0x020a4000 0x4000>;
302 interrupts = <0 70 0x04 0 71 0x04>; 302 interrupts = <0 70 0x04 0 71 0x04>;
303 gpio-controller; 303 gpio-controller;
@@ -307,7 +307,7 @@
307 }; 307 };
308 308
309 gpio4: gpio@020a8000 { 309 gpio4: gpio@020a8000 {
310 compatible = "fsl,imx6q-gpio", "fsl,imx31-gpio"; 310 compatible = "fsl,imx6q-gpio", "fsl,imx35-gpio";
311 reg = <0x020a8000 0x4000>; 311 reg = <0x020a8000 0x4000>;
312 interrupts = <0 72 0x04 0 73 0x04>; 312 interrupts = <0 72 0x04 0 73 0x04>;
313 gpio-controller; 313 gpio-controller;
@@ -317,7 +317,7 @@
317 }; 317 };
318 318
319 gpio5: gpio@020ac000 { 319 gpio5: gpio@020ac000 {
320 compatible = "fsl,imx6q-gpio", "fsl,imx31-gpio"; 320 compatible = "fsl,imx6q-gpio", "fsl,imx35-gpio";
321 reg = <0x020ac000 0x4000>; 321 reg = <0x020ac000 0x4000>;
322 interrupts = <0 74 0x04 0 75 0x04>; 322 interrupts = <0 74 0x04 0 75 0x04>;
323 gpio-controller; 323 gpio-controller;
@@ -327,7 +327,7 @@
327 }; 327 };
328 328
329 gpio6: gpio@020b0000 { 329 gpio6: gpio@020b0000 {
330 compatible = "fsl,imx6q-gpio", "fsl,imx31-gpio"; 330 compatible = "fsl,imx6q-gpio", "fsl,imx35-gpio";
331 reg = <0x020b0000 0x4000>; 331 reg = <0x020b0000 0x4000>;
332 interrupts = <0 76 0x04 0 77 0x04>; 332 interrupts = <0 76 0x04 0 77 0x04>;
333 gpio-controller; 333 gpio-controller;
@@ -337,7 +337,7 @@
337 }; 337 };
338 338
339 gpio7: gpio@020b4000 { 339 gpio7: gpio@020b4000 {
340 compatible = "fsl,imx6q-gpio", "fsl,imx31-gpio"; 340 compatible = "fsl,imx6q-gpio", "fsl,imx35-gpio";
341 reg = <0x020b4000 0x4000>; 341 reg = <0x020b4000 0x4000>;
342 interrupts = <0 78 0x04 0 79 0x04>; 342 interrupts = <0 78 0x04 0 79 0x04>;
343 gpio-controller; 343 gpio-controller;
diff --git a/arch/arm/mach-at91/at91sam9g45.c b/arch/arm/mach-at91/at91sam9g45.c
index 977127368a7d..ef6cedd52e3c 100644
--- a/arch/arm/mach-at91/at91sam9g45.c
+++ b/arch/arm/mach-at91/at91sam9g45.c
@@ -183,6 +183,13 @@ static struct clk adc_op_clk = {
183 .rate_hz = 13200000, 183 .rate_hz = 13200000,
184}; 184};
185 185
186/* AES/TDES/SHA clock - Only for sam9m11/sam9g56 */
187static struct clk aestdessha_clk = {
188 .name = "aestdessha_clk",
189 .pmc_mask = 1 << AT91SAM9G45_ID_AESTDESSHA,
190 .type = CLK_TYPE_PERIPHERAL,
191};
192
186static struct clk *periph_clocks[] __initdata = { 193static struct clk *periph_clocks[] __initdata = {
187 &pioA_clk, 194 &pioA_clk,
188 &pioB_clk, 195 &pioB_clk,
@@ -212,6 +219,7 @@ static struct clk *periph_clocks[] __initdata = {
212 &udphs_clk, 219 &udphs_clk,
213 &mmc1_clk, 220 &mmc1_clk,
214 &adc_op_clk, 221 &adc_op_clk,
222 &aestdessha_clk,
215 // irq0 223 // irq0
216}; 224};
217 225
@@ -232,6 +240,9 @@ static struct clk_lookup periph_clocks_lookups[] = {
232 CLKDEV_CON_DEV_ID("pclk", "ssc.0", &ssc0_clk), 240 CLKDEV_CON_DEV_ID("pclk", "ssc.0", &ssc0_clk),
233 CLKDEV_CON_DEV_ID("pclk", "ssc.1", &ssc1_clk), 241 CLKDEV_CON_DEV_ID("pclk", "ssc.1", &ssc1_clk),
234 CLKDEV_CON_DEV_ID(NULL, "atmel-trng", &trng_clk), 242 CLKDEV_CON_DEV_ID(NULL, "atmel-trng", &trng_clk),
243 CLKDEV_CON_DEV_ID(NULL, "atmel_sha", &aestdessha_clk),
244 CLKDEV_CON_DEV_ID(NULL, "atmel_tdes", &aestdessha_clk),
245 CLKDEV_CON_DEV_ID(NULL, "atmel_aes", &aestdessha_clk),
235 /* more usart lookup table for DT entries */ 246 /* more usart lookup table for DT entries */
236 CLKDEV_CON_DEV_ID("usart", "ffffee00.serial", &mck), 247 CLKDEV_CON_DEV_ID("usart", "ffffee00.serial", &mck),
237 CLKDEV_CON_DEV_ID("usart", "fff8c000.serial", &usart0_clk), 248 CLKDEV_CON_DEV_ID("usart", "fff8c000.serial", &usart0_clk),
@@ -388,7 +399,7 @@ static unsigned int at91sam9g45_default_irq_priority[NR_AIC_IRQS] __initdata = {
388 3, /* Ethernet */ 399 3, /* Ethernet */
389 0, /* Image Sensor Interface */ 400 0, /* Image Sensor Interface */
390 2, /* USB Device High speed port */ 401 2, /* USB Device High speed port */
391 0, 402 0, /* AESTDESSHA Crypto HW Accelerators */
392 0, /* Multimedia Card Interface 1 */ 403 0, /* Multimedia Card Interface 1 */
393 0, 404 0,
394 0, /* Advanced Interrupt Controller (IRQ0) */ 405 0, /* Advanced Interrupt Controller (IRQ0) */
diff --git a/arch/arm/mach-at91/at91sam9g45_devices.c b/arch/arm/mach-at91/at91sam9g45_devices.c
index 40fb79df2de0..06073996a382 100644
--- a/arch/arm/mach-at91/at91sam9g45_devices.c
+++ b/arch/arm/mach-at91/at91sam9g45_devices.c
@@ -18,6 +18,7 @@
18#include <linux/platform_device.h> 18#include <linux/platform_device.h>
19#include <linux/i2c-gpio.h> 19#include <linux/i2c-gpio.h>
20#include <linux/atmel-mci.h> 20#include <linux/atmel-mci.h>
21#include <linux/platform_data/atmel-aes.h>
21 22
22#include <linux/platform_data/at91_adc.h> 23#include <linux/platform_data/at91_adc.h>
23 24
@@ -1830,6 +1831,130 @@ void __init at91_register_uart(unsigned id, unsigned portnr, unsigned pins) {}
1830void __init at91_add_device_serial(void) {} 1831void __init at91_add_device_serial(void) {}
1831#endif 1832#endif
1832 1833
1834/* --------------------------------------------------------------------
1835 * SHA1/SHA256
1836 * -------------------------------------------------------------------- */
1837
1838#if defined(CONFIG_CRYPTO_DEV_ATMEL_SHA) || defined(CONFIG_CRYPTO_DEV_ATMEL_SHA_MODULE)
1839static struct resource sha_resources[] = {
1840 {
1841 .start = AT91SAM9G45_BASE_SHA,
1842 .end = AT91SAM9G45_BASE_SHA + SZ_16K - 1,
1843 .flags = IORESOURCE_MEM,
1844 },
1845 [1] = {
1846 .start = AT91SAM9G45_ID_AESTDESSHA,
1847 .end = AT91SAM9G45_ID_AESTDESSHA,
1848 .flags = IORESOURCE_IRQ,
1849 },
1850};
1851
1852static struct platform_device at91sam9g45_sha_device = {
1853 .name = "atmel_sha",
1854 .id = -1,
1855 .resource = sha_resources,
1856 .num_resources = ARRAY_SIZE(sha_resources),
1857};
1858
1859static void __init at91_add_device_sha(void)
1860{
1861 platform_device_register(&at91sam9g45_sha_device);
1862}
1863#else
1864static void __init at91_add_device_sha(void) {}
1865#endif
1866
1867/* --------------------------------------------------------------------
1868 * DES/TDES
1869 * -------------------------------------------------------------------- */
1870
1871#if defined(CONFIG_CRYPTO_DEV_ATMEL_TDES) || defined(CONFIG_CRYPTO_DEV_ATMEL_TDES_MODULE)
1872static struct resource tdes_resources[] = {
1873 [0] = {
1874 .start = AT91SAM9G45_BASE_TDES,
1875 .end = AT91SAM9G45_BASE_TDES + SZ_16K - 1,
1876 .flags = IORESOURCE_MEM,
1877 },
1878 [1] = {
1879 .start = AT91SAM9G45_ID_AESTDESSHA,
1880 .end = AT91SAM9G45_ID_AESTDESSHA,
1881 .flags = IORESOURCE_IRQ,
1882 },
1883};
1884
1885static struct platform_device at91sam9g45_tdes_device = {
1886 .name = "atmel_tdes",
1887 .id = -1,
1888 .resource = tdes_resources,
1889 .num_resources = ARRAY_SIZE(tdes_resources),
1890};
1891
1892static void __init at91_add_device_tdes(void)
1893{
1894 platform_device_register(&at91sam9g45_tdes_device);
1895}
1896#else
1897static void __init at91_add_device_tdes(void) {}
1898#endif
1899
1900/* --------------------------------------------------------------------
1901 * AES
1902 * -------------------------------------------------------------------- */
1903
1904#if defined(CONFIG_CRYPTO_DEV_ATMEL_AES) || defined(CONFIG_CRYPTO_DEV_ATMEL_AES_MODULE)
1905static struct aes_platform_data aes_data;
1906static u64 aes_dmamask = DMA_BIT_MASK(32);
1907
1908static struct resource aes_resources[] = {
1909 [0] = {
1910 .start = AT91SAM9G45_BASE_AES,
1911 .end = AT91SAM9G45_BASE_AES + SZ_16K - 1,
1912 .flags = IORESOURCE_MEM,
1913 },
1914 [1] = {
1915 .start = AT91SAM9G45_ID_AESTDESSHA,
1916 .end = AT91SAM9G45_ID_AESTDESSHA,
1917 .flags = IORESOURCE_IRQ,
1918 },
1919};
1920
1921static struct platform_device at91sam9g45_aes_device = {
1922 .name = "atmel_aes",
1923 .id = -1,
1924 .dev = {
1925 .dma_mask = &aes_dmamask,
1926 .coherent_dma_mask = DMA_BIT_MASK(32),
1927 .platform_data = &aes_data,
1928 },
1929 .resource = aes_resources,
1930 .num_resources = ARRAY_SIZE(aes_resources),
1931};
1932
1933static void __init at91_add_device_aes(void)
1934{
1935 struct at_dma_slave *atslave;
1936 struct aes_dma_data *alt_atslave;
1937
1938 alt_atslave = kzalloc(sizeof(struct aes_dma_data), GFP_KERNEL);
1939
1940 /* DMA TX slave channel configuration */
1941 atslave = &alt_atslave->txdata;
1942 atslave->dma_dev = &at_hdmac_device.dev;
1943 atslave->cfg = ATC_FIFOCFG_ENOUGHSPACE | ATC_SRC_H2SEL_HW |
1944 ATC_SRC_PER(AT_DMA_ID_AES_RX);
1945
1946 /* DMA RX slave channel configuration */
1947 atslave = &alt_atslave->rxdata;
1948 atslave->dma_dev = &at_hdmac_device.dev;
1949 atslave->cfg = ATC_FIFOCFG_ENOUGHSPACE | ATC_DST_H2SEL_HW |
1950 ATC_DST_PER(AT_DMA_ID_AES_TX);
1951
1952 aes_data.dma_slave = alt_atslave;
1953 platform_device_register(&at91sam9g45_aes_device);
1954}
1955#else
1956static void __init at91_add_device_aes(void) {}
1957#endif
1833 1958
1834/* -------------------------------------------------------------------- */ 1959/* -------------------------------------------------------------------- */
1835/* 1960/*
@@ -1847,6 +1972,9 @@ static int __init at91_add_standard_devices(void)
1847 at91_add_device_trng(); 1972 at91_add_device_trng();
1848 at91_add_device_watchdog(); 1973 at91_add_device_watchdog();
1849 at91_add_device_tc(); 1974 at91_add_device_tc();
1975 at91_add_device_sha();
1976 at91_add_device_tdes();
1977 at91_add_device_aes();
1850 return 0; 1978 return 0;
1851} 1979}
1852 1980
diff --git a/arch/arm/mach-at91/include/mach/at91sam9g45.h b/arch/arm/mach-at91/include/mach/at91sam9g45.h
index 3a4da24d5911..8eba1021f533 100644
--- a/arch/arm/mach-at91/include/mach/at91sam9g45.h
+++ b/arch/arm/mach-at91/include/mach/at91sam9g45.h
@@ -136,6 +136,8 @@
136#define AT_DMA_ID_SSC1_RX 8 136#define AT_DMA_ID_SSC1_RX 8
137#define AT_DMA_ID_AC97_TX 9 137#define AT_DMA_ID_AC97_TX 9
138#define AT_DMA_ID_AC97_RX 10 138#define AT_DMA_ID_AC97_RX 10
139#define AT_DMA_ID_AES_TX 11
140#define AT_DMA_ID_AES_RX 12
139#define AT_DMA_ID_MCI1 13 141#define AT_DMA_ID_MCI1 13
140 142
141#endif 143#endif
diff --git a/arch/arm/mach-imx/mm-imx25.c b/arch/arm/mach-imx/mm-imx25.c
index 388928fdb11a..f3f5c6542ab4 100644
--- a/arch/arm/mach-imx/mm-imx25.c
+++ b/arch/arm/mach-imx/mm-imx25.c
@@ -89,11 +89,11 @@ static const struct resource imx25_audmux_res[] __initconst = {
89 89
90void __init imx25_soc_init(void) 90void __init imx25_soc_init(void)
91{ 91{
92 /* i.mx25 has the i.mx31 type gpio */ 92 /* i.mx25 has the i.mx35 type gpio */
93 mxc_register_gpio("imx31-gpio", 0, MX25_GPIO1_BASE_ADDR, SZ_16K, MX25_INT_GPIO1, 0); 93 mxc_register_gpio("imx35-gpio", 0, MX25_GPIO1_BASE_ADDR, SZ_16K, MX25_INT_GPIO1, 0);
94 mxc_register_gpio("imx31-gpio", 1, MX25_GPIO2_BASE_ADDR, SZ_16K, MX25_INT_GPIO2, 0); 94 mxc_register_gpio("imx35-gpio", 1, MX25_GPIO2_BASE_ADDR, SZ_16K, MX25_INT_GPIO2, 0);
95 mxc_register_gpio("imx31-gpio", 2, MX25_GPIO3_BASE_ADDR, SZ_16K, MX25_INT_GPIO3, 0); 95 mxc_register_gpio("imx35-gpio", 2, MX25_GPIO3_BASE_ADDR, SZ_16K, MX25_INT_GPIO3, 0);
96 mxc_register_gpio("imx31-gpio", 3, MX25_GPIO4_BASE_ADDR, SZ_16K, MX25_INT_GPIO4, 0); 96 mxc_register_gpio("imx35-gpio", 3, MX25_GPIO4_BASE_ADDR, SZ_16K, MX25_INT_GPIO4, 0);
97 97
98 pinctrl_provide_dummies(); 98 pinctrl_provide_dummies();
99 /* i.mx25 has the i.mx35 type sdma */ 99 /* i.mx25 has the i.mx35 type sdma */
diff --git a/arch/arm/mach-imx/mm-imx3.c b/arch/arm/mach-imx/mm-imx3.c
index fe96105109b3..9d2c843bde02 100644
--- a/arch/arm/mach-imx/mm-imx3.c
+++ b/arch/arm/mach-imx/mm-imx3.c
@@ -272,10 +272,9 @@ void __init imx35_soc_init(void)
272 272
273 imx3_init_l2x0(); 273 imx3_init_l2x0();
274 274
275 /* i.mx35 has the i.mx31 type gpio */ 275 mxc_register_gpio("imx35-gpio", 0, MX35_GPIO1_BASE_ADDR, SZ_16K, MX35_INT_GPIO1, 0);
276 mxc_register_gpio("imx31-gpio", 0, MX35_GPIO1_BASE_ADDR, SZ_16K, MX35_INT_GPIO1, 0); 276 mxc_register_gpio("imx35-gpio", 1, MX35_GPIO2_BASE_ADDR, SZ_16K, MX35_INT_GPIO2, 0);
277 mxc_register_gpio("imx31-gpio", 1, MX35_GPIO2_BASE_ADDR, SZ_16K, MX35_INT_GPIO2, 0); 277 mxc_register_gpio("imx35-gpio", 2, MX35_GPIO3_BASE_ADDR, SZ_16K, MX35_INT_GPIO3, 0);
278 mxc_register_gpio("imx31-gpio", 2, MX35_GPIO3_BASE_ADDR, SZ_16K, MX35_INT_GPIO3, 0);
279 278
280 pinctrl_provide_dummies(); 279 pinctrl_provide_dummies();
281 if (to_version == 1) { 280 if (to_version == 1) {
diff --git a/arch/arm/mach-imx/mm-imx5.c b/arch/arm/mach-imx/mm-imx5.c
index f19d604e1b2a..52d8f534be10 100644
--- a/arch/arm/mach-imx/mm-imx5.c
+++ b/arch/arm/mach-imx/mm-imx5.c
@@ -161,13 +161,13 @@ static const struct resource imx53_audmux_res[] __initconst = {
161 161
162void __init imx50_soc_init(void) 162void __init imx50_soc_init(void)
163{ 163{
164 /* i.mx50 has the i.mx31 type gpio */ 164 /* i.mx50 has the i.mx35 type gpio */
165 mxc_register_gpio("imx31-gpio", 0, MX50_GPIO1_BASE_ADDR, SZ_16K, MX50_INT_GPIO1_LOW, MX50_INT_GPIO1_HIGH); 165 mxc_register_gpio("imx35-gpio", 0, MX50_GPIO1_BASE_ADDR, SZ_16K, MX50_INT_GPIO1_LOW, MX50_INT_GPIO1_HIGH);
166 mxc_register_gpio("imx31-gpio", 1, MX50_GPIO2_BASE_ADDR, SZ_16K, MX50_INT_GPIO2_LOW, MX50_INT_GPIO2_HIGH); 166 mxc_register_gpio("imx35-gpio", 1, MX50_GPIO2_BASE_ADDR, SZ_16K, MX50_INT_GPIO2_LOW, MX50_INT_GPIO2_HIGH);
167 mxc_register_gpio("imx31-gpio", 2, MX50_GPIO3_BASE_ADDR, SZ_16K, MX50_INT_GPIO3_LOW, MX50_INT_GPIO3_HIGH); 167 mxc_register_gpio("imx35-gpio", 2, MX50_GPIO3_BASE_ADDR, SZ_16K, MX50_INT_GPIO3_LOW, MX50_INT_GPIO3_HIGH);
168 mxc_register_gpio("imx31-gpio", 3, MX50_GPIO4_BASE_ADDR, SZ_16K, MX50_INT_GPIO4_LOW, MX50_INT_GPIO4_HIGH); 168 mxc_register_gpio("imx35-gpio", 3, MX50_GPIO4_BASE_ADDR, SZ_16K, MX50_INT_GPIO4_LOW, MX50_INT_GPIO4_HIGH);
169 mxc_register_gpio("imx31-gpio", 4, MX50_GPIO5_BASE_ADDR, SZ_16K, MX50_INT_GPIO5_LOW, MX50_INT_GPIO5_HIGH); 169 mxc_register_gpio("imx35-gpio", 4, MX50_GPIO5_BASE_ADDR, SZ_16K, MX50_INT_GPIO5_LOW, MX50_INT_GPIO5_HIGH);
170 mxc_register_gpio("imx31-gpio", 5, MX50_GPIO6_BASE_ADDR, SZ_16K, MX50_INT_GPIO6_LOW, MX50_INT_GPIO6_HIGH); 170 mxc_register_gpio("imx35-gpio", 5, MX50_GPIO6_BASE_ADDR, SZ_16K, MX50_INT_GPIO6_LOW, MX50_INT_GPIO6_HIGH);
171 171
172 /* i.mx50 has the i.mx31 type audmux */ 172 /* i.mx50 has the i.mx31 type audmux */
173 platform_device_register_simple("imx31-audmux", 0, imx50_audmux_res, 173 platform_device_register_simple("imx31-audmux", 0, imx50_audmux_res,
@@ -176,11 +176,11 @@ void __init imx50_soc_init(void)
176 176
177void __init imx51_soc_init(void) 177void __init imx51_soc_init(void)
178{ 178{
179 /* i.mx51 has the i.mx31 type gpio */ 179 /* i.mx51 has the i.mx35 type gpio */
180 mxc_register_gpio("imx31-gpio", 0, MX51_GPIO1_BASE_ADDR, SZ_16K, MX51_INT_GPIO1_LOW, MX51_INT_GPIO1_HIGH); 180 mxc_register_gpio("imx35-gpio", 0, MX51_GPIO1_BASE_ADDR, SZ_16K, MX51_INT_GPIO1_LOW, MX51_INT_GPIO1_HIGH);
181 mxc_register_gpio("imx31-gpio", 1, MX51_GPIO2_BASE_ADDR, SZ_16K, MX51_INT_GPIO2_LOW, MX51_INT_GPIO2_HIGH); 181 mxc_register_gpio("imx35-gpio", 1, MX51_GPIO2_BASE_ADDR, SZ_16K, MX51_INT_GPIO2_LOW, MX51_INT_GPIO2_HIGH);
182 mxc_register_gpio("imx31-gpio", 2, MX51_GPIO3_BASE_ADDR, SZ_16K, MX51_INT_GPIO3_LOW, MX51_INT_GPIO3_HIGH); 182 mxc_register_gpio("imx35-gpio", 2, MX51_GPIO3_BASE_ADDR, SZ_16K, MX51_INT_GPIO3_LOW, MX51_INT_GPIO3_HIGH);
183 mxc_register_gpio("imx31-gpio", 3, MX51_GPIO4_BASE_ADDR, SZ_16K, MX51_INT_GPIO4_LOW, MX51_INT_GPIO4_HIGH); 183 mxc_register_gpio("imx35-gpio", 3, MX51_GPIO4_BASE_ADDR, SZ_16K, MX51_INT_GPIO4_LOW, MX51_INT_GPIO4_HIGH);
184 184
185 pinctrl_provide_dummies(); 185 pinctrl_provide_dummies();
186 186
@@ -198,14 +198,14 @@ void __init imx51_soc_init(void)
198 198
199void __init imx53_soc_init(void) 199void __init imx53_soc_init(void)
200{ 200{
201 /* i.mx53 has the i.mx31 type gpio */ 201 /* i.mx53 has the i.mx35 type gpio */
202 mxc_register_gpio("imx31-gpio", 0, MX53_GPIO1_BASE_ADDR, SZ_16K, MX53_INT_GPIO1_LOW, MX53_INT_GPIO1_HIGH); 202 mxc_register_gpio("imx35-gpio", 0, MX53_GPIO1_BASE_ADDR, SZ_16K, MX53_INT_GPIO1_LOW, MX53_INT_GPIO1_HIGH);
203 mxc_register_gpio("imx31-gpio", 1, MX53_GPIO2_BASE_ADDR, SZ_16K, MX53_INT_GPIO2_LOW, MX53_INT_GPIO2_HIGH); 203 mxc_register_gpio("imx35-gpio", 1, MX53_GPIO2_BASE_ADDR, SZ_16K, MX53_INT_GPIO2_LOW, MX53_INT_GPIO2_HIGH);
204 mxc_register_gpio("imx31-gpio", 2, MX53_GPIO3_BASE_ADDR, SZ_16K, MX53_INT_GPIO3_LOW, MX53_INT_GPIO3_HIGH); 204 mxc_register_gpio("imx35-gpio", 2, MX53_GPIO3_BASE_ADDR, SZ_16K, MX53_INT_GPIO3_LOW, MX53_INT_GPIO3_HIGH);
205 mxc_register_gpio("imx31-gpio", 3, MX53_GPIO4_BASE_ADDR, SZ_16K, MX53_INT_GPIO4_LOW, MX53_INT_GPIO4_HIGH); 205 mxc_register_gpio("imx35-gpio", 3, MX53_GPIO4_BASE_ADDR, SZ_16K, MX53_INT_GPIO4_LOW, MX53_INT_GPIO4_HIGH);
206 mxc_register_gpio("imx31-gpio", 4, MX53_GPIO5_BASE_ADDR, SZ_16K, MX53_INT_GPIO5_LOW, MX53_INT_GPIO5_HIGH); 206 mxc_register_gpio("imx35-gpio", 4, MX53_GPIO5_BASE_ADDR, SZ_16K, MX53_INT_GPIO5_LOW, MX53_INT_GPIO5_HIGH);
207 mxc_register_gpio("imx31-gpio", 5, MX53_GPIO6_BASE_ADDR, SZ_16K, MX53_INT_GPIO6_LOW, MX53_INT_GPIO6_HIGH); 207 mxc_register_gpio("imx35-gpio", 5, MX53_GPIO6_BASE_ADDR, SZ_16K, MX53_INT_GPIO6_LOW, MX53_INT_GPIO6_HIGH);
208 mxc_register_gpio("imx31-gpio", 6, MX53_GPIO7_BASE_ADDR, SZ_16K, MX53_INT_GPIO7_LOW, MX53_INT_GPIO7_HIGH); 208 mxc_register_gpio("imx35-gpio", 6, MX53_GPIO7_BASE_ADDR, SZ_16K, MX53_INT_GPIO7_LOW, MX53_INT_GPIO7_HIGH);
209 209
210 pinctrl_provide_dummies(); 210 pinctrl_provide_dummies();
211 /* i.mx53 has the i.mx35 type sdma */ 211 /* i.mx53 has the i.mx35 type sdma */
diff --git a/arch/arm/mach-omap2/clock3xxx_data.c b/arch/arm/mach-omap2/clock3xxx_data.c
index 91b3d5c60bfe..83bed9ad3017 100644
--- a/arch/arm/mach-omap2/clock3xxx_data.c
+++ b/arch/arm/mach-omap2/clock3xxx_data.c
@@ -3376,15 +3376,15 @@ static struct omap_clk omap3xxx_clks[] = {
3376 CLK(NULL, "usbhost_48m_fck", &usbhost_48m_fck, CK_3430ES2PLUS | CK_AM35XX | CK_36XX), 3376 CLK(NULL, "usbhost_48m_fck", &usbhost_48m_fck, CK_3430ES2PLUS | CK_AM35XX | CK_36XX),
3377 CLK(NULL, "usbhost_ick", &usbhost_ick, CK_3430ES2PLUS | CK_AM35XX | CK_36XX), 3377 CLK(NULL, "usbhost_ick", &usbhost_ick, CK_3430ES2PLUS | CK_AM35XX | CK_36XX),
3378 CLK("usbhs_omap", "usbhost_ick", &usbhost_ick, CK_3430ES2PLUS | CK_AM35XX | CK_36XX), 3378 CLK("usbhs_omap", "usbhost_ick", &usbhost_ick, CK_3430ES2PLUS | CK_AM35XX | CK_36XX),
3379 CLK("usbhs_omap", "utmi_p1_gfclk", &dummy_ck, CK_3XXX), 3379 CLK(NULL, "utmi_p1_gfclk", &dummy_ck, CK_3XXX),
3380 CLK("usbhs_omap", "utmi_p2_gfclk", &dummy_ck, CK_3XXX), 3380 CLK(NULL, "utmi_p2_gfclk", &dummy_ck, CK_3XXX),
3381 CLK("usbhs_omap", "xclk60mhsp1_ck", &dummy_ck, CK_3XXX), 3381 CLK(NULL, "xclk60mhsp1_ck", &dummy_ck, CK_3XXX),
3382 CLK("usbhs_omap", "xclk60mhsp2_ck", &dummy_ck, CK_3XXX), 3382 CLK(NULL, "xclk60mhsp2_ck", &dummy_ck, CK_3XXX),
3383 CLK("usbhs_omap", "usb_host_hs_utmi_p1_clk", &dummy_ck, CK_3XXX), 3383 CLK(NULL, "usb_host_hs_utmi_p1_clk", &dummy_ck, CK_3XXX),
3384 CLK("usbhs_omap", "usb_host_hs_utmi_p2_clk", &dummy_ck, CK_3XXX), 3384 CLK(NULL, "usb_host_hs_utmi_p2_clk", &dummy_ck, CK_3XXX),
3385 CLK("usbhs_omap", "usb_tll_hs_usb_ch0_clk", &dummy_ck, CK_3XXX), 3385 CLK("usbhs_omap", "usb_tll_hs_usb_ch0_clk", &dummy_ck, CK_3XXX),
3386 CLK("usbhs_omap", "usb_tll_hs_usb_ch1_clk", &dummy_ck, CK_3XXX), 3386 CLK("usbhs_omap", "usb_tll_hs_usb_ch1_clk", &dummy_ck, CK_3XXX),
3387 CLK("usbhs_omap", "init_60m_fclk", &dummy_ck, CK_3XXX), 3387 CLK(NULL, "init_60m_fclk", &dummy_ck, CK_3XXX),
3388 CLK(NULL, "usim_fck", &usim_fck, CK_3430ES2PLUS | CK_36XX), 3388 CLK(NULL, "usim_fck", &usim_fck, CK_3430ES2PLUS | CK_36XX),
3389 CLK(NULL, "gpt1_fck", &gpt1_fck, CK_3XXX), 3389 CLK(NULL, "gpt1_fck", &gpt1_fck, CK_3XXX),
3390 CLK(NULL, "wkup_32k_fck", &wkup_32k_fck, CK_3XXX), 3390 CLK(NULL, "wkup_32k_fck", &wkup_32k_fck, CK_3XXX),
diff --git a/arch/arm/plat-spear/include/plat/keyboard.h b/arch/arm/plat-spear/include/plat/keyboard.h
index 0562f134621d..9248e3a7e333 100644
--- a/arch/arm/plat-spear/include/plat/keyboard.h
+++ b/arch/arm/plat-spear/include/plat/keyboard.h
@@ -149,6 +149,7 @@ int _name[] = { \
149 * keymap: pointer to keymap data (table and size) 149 * keymap: pointer to keymap data (table and size)
150 * rep: enables key autorepeat 150 * rep: enables key autorepeat
151 * mode: choose keyboard support(9x9, 6x6, 2x2) 151 * mode: choose keyboard support(9x9, 6x6, 2x2)
152 * suspended_rate: rate at which keyboard would operate in suspended mode
152 * 153 *
153 * This structure is supposed to be used by platform code to supply 154 * This structure is supposed to be used by platform code to supply
154 * keymaps to drivers that implement keyboards. 155 * keymaps to drivers that implement keyboards.
@@ -157,6 +158,7 @@ struct kbd_platform_data {
157 const struct matrix_keymap_data *keymap; 158 const struct matrix_keymap_data *keymap;
158 bool rep; 159 bool rep;
159 unsigned int mode; 160 unsigned int mode;
161 unsigned int suspended_rate;
160}; 162};
161 163
162#endif /* __PLAT_KEYBOARD_H */ 164#endif /* __PLAT_KEYBOARD_H */
diff --git a/arch/m68k/Kconfig.bus b/arch/m68k/Kconfig.bus
index 3adb499584fb..ffc0601a2a19 100644
--- a/arch/m68k/Kconfig.bus
+++ b/arch/m68k/Kconfig.bus
@@ -48,6 +48,13 @@ config ISA
48config GENERIC_ISA_DMA 48config GENERIC_ISA_DMA
49 def_bool ISA 49 def_bool ISA
50 50
51config PCI
52 bool "PCI support"
53 depends on M54xx
54 help
55 Enable the PCI bus. Support for the PCI bus hardware built into the
56 ColdFire 547x and 548x processors.
57
51source "drivers/pci/Kconfig" 58source "drivers/pci/Kconfig"
52 59
53source "drivers/zorro/Kconfig" 60source "drivers/zorro/Kconfig"
diff --git a/arch/m68k/Kconfig.cpu b/arch/m68k/Kconfig.cpu
index 2b53254ad994..43a9f8f1b8eb 100644
--- a/arch/m68k/Kconfig.cpu
+++ b/arch/m68k/Kconfig.cpu
@@ -23,7 +23,7 @@ config M68KCLASSIC
23config COLDFIRE 23config COLDFIRE
24 bool "Coldfire CPU family support" 24 bool "Coldfire CPU family support"
25 select GENERIC_GPIO 25 select GENERIC_GPIO
26 select ARCH_REQUIRE_GPIOLIB 26 select ARCH_WANT_OPTIONAL_GPIOLIB
27 select ARCH_HAVE_CUSTOM_GPIO_H 27 select ARCH_HAVE_CUSTOM_GPIO_H
28 select CPU_HAS_NO_BITFIELDS 28 select CPU_HAS_NO_BITFIELDS
29 select CPU_HAS_NO_MULDIV64 29 select CPU_HAS_NO_MULDIV64
@@ -167,6 +167,14 @@ config M5249
167 help 167 help
168 Motorola ColdFire 5249 processor support. 168 Motorola ColdFire 5249 processor support.
169 169
170config M525x
171 bool "MCF525x"
172 depends on !MMU
173 select COLDFIRE_SW_A7
174 select HAVE_MBAR
175 help
176 Freescale (Motorola) Coldfire 5251/5253 processor support.
177
170config M527x 178config M527x
171 bool 179 bool
172 180
@@ -253,6 +261,14 @@ config M548x
253 help 261 help
254 Freescale ColdFire 5480/5481/5482/5483/5484/5485 processor support. 262 Freescale ColdFire 5480/5481/5482/5483/5484/5485 processor support.
255 263
264config M5441x
265 bool "MCF5441x"
266 depends on !MMU
267 select GENERIC_CLOCKEVENTS
268 select HAVE_CACHE_CB
269 help
270 Freescale Coldfire 54410/54415/54416/54417/54418 processor support.
271
256endif # COLDFIRE 272endif # COLDFIRE
257 273
258 274
diff --git a/arch/m68k/Makefile b/arch/m68k/Makefile
index b7f2e2d5cd2e..7636751f2f87 100644
--- a/arch/m68k/Makefile
+++ b/arch/m68k/Makefile
@@ -41,6 +41,7 @@ cpuflags-$(CONFIG_M68030) :=
41cpuflags-$(CONFIG_M68020) := 41cpuflags-$(CONFIG_M68020) :=
42cpuflags-$(CONFIG_M68360) := -m68332 42cpuflags-$(CONFIG_M68360) := -m68332
43cpuflags-$(CONFIG_M68000) := -m68000 43cpuflags-$(CONFIG_M68000) := -m68000
44cpuflags-$(CONFIG_M5441x) := $(call cc-option,-mcpu=54455,-mcfv4e)
44cpuflags-$(CONFIG_M54xx) := $(call cc-option,-mcpu=5475,-m5200) 45cpuflags-$(CONFIG_M54xx) := $(call cc-option,-mcpu=5475,-m5200)
45cpuflags-$(CONFIG_M5407) := $(call cc-option,-mcpu=5407,-m5200) 46cpuflags-$(CONFIG_M5407) := $(call cc-option,-mcpu=5407,-m5200)
46cpuflags-$(CONFIG_M532x) := $(call cc-option,-mcpu=532x,-m5307) 47cpuflags-$(CONFIG_M532x) := $(call cc-option,-mcpu=532x,-m5307)
@@ -50,6 +51,7 @@ cpuflags-$(CONFIG_M5275) := $(call cc-option,-mcpu=5275,-m5307)
50cpuflags-$(CONFIG_M5272) := $(call cc-option,-mcpu=5272,-m5307) 51cpuflags-$(CONFIG_M5272) := $(call cc-option,-mcpu=5272,-m5307)
51cpuflags-$(CONFIG_M5271) := $(call cc-option,-mcpu=5271,-m5307) 52cpuflags-$(CONFIG_M5271) := $(call cc-option,-mcpu=5271,-m5307)
52cpuflags-$(CONFIG_M523x) := $(call cc-option,-mcpu=523x,-m5307) 53cpuflags-$(CONFIG_M523x) := $(call cc-option,-mcpu=523x,-m5307)
54cpuflags-$(CONFIG_M525x) := $(call cc-option,-mcpu=5253,-m5200)
53cpuflags-$(CONFIG_M5249) := $(call cc-option,-mcpu=5249,-m5200) 55cpuflags-$(CONFIG_M5249) := $(call cc-option,-mcpu=5249,-m5200)
54cpuflags-$(CONFIG_M520x) := $(call cc-option,-mcpu=5208,-m5200) 56cpuflags-$(CONFIG_M520x) := $(call cc-option,-mcpu=5208,-m5200)
55cpuflags-$(CONFIG_M5206e) := $(call cc-option,-mcpu=5206e,-m5200) 57cpuflags-$(CONFIG_M5206e) := $(call cc-option,-mcpu=5206e,-m5200)
diff --git a/arch/m68k/include/asm/cacheflush_mm.h b/arch/m68k/include/asm/cacheflush_mm.h
index 8104bd874649..fa2c3d681d84 100644
--- a/arch/m68k/include/asm/cacheflush_mm.h
+++ b/arch/m68k/include/asm/cacheflush_mm.h
@@ -16,7 +16,48 @@
16#define DCACHE_MAX_ADDR 0 16#define DCACHE_MAX_ADDR 0
17#define DCACHE_SETMASK 0 17#define DCACHE_SETMASK 0
18#endif 18#endif
19#ifndef CACHE_MODE
20#define CACHE_MODE 0
21#define CACR_ICINVA 0
22#define CACR_DCINVA 0
23#define CACR_BCINVA 0
24#endif
25
26/*
27 * ColdFire architecture has no way to clear individual cache lines, so we
28 * are stuck invalidating all the cache entries when we want a clear operation.
29 */
30static inline void clear_cf_icache(unsigned long start, unsigned long end)
31{
32 __asm__ __volatile__ (
33 "movec %0,%%cacr\n\t"
34 "nop"
35 :
36 : "r" (CACHE_MODE | CACR_ICINVA | CACR_BCINVA));
37}
38
39static inline void clear_cf_dcache(unsigned long start, unsigned long end)
40{
41 __asm__ __volatile__ (
42 "movec %0,%%cacr\n\t"
43 "nop"
44 :
45 : "r" (CACHE_MODE | CACR_DCINVA));
46}
19 47
48static inline void clear_cf_bcache(unsigned long start, unsigned long end)
49{
50 __asm__ __volatile__ (
51 "movec %0,%%cacr\n\t"
52 "nop"
53 :
54 : "r" (CACHE_MODE | CACR_ICINVA | CACR_BCINVA | CACR_DCINVA));
55}
56
57/*
58 * Use the ColdFire cpushl instruction to push (and invalidate) cache lines.
59 * The start and end addresses are cache line numbers not memory addresses.
60 */
20static inline void flush_cf_icache(unsigned long start, unsigned long end) 61static inline void flush_cf_icache(unsigned long start, unsigned long end)
21{ 62{
22 unsigned long set; 63 unsigned long set;
diff --git a/arch/m68k/include/asm/dma.h b/arch/m68k/include/asm/dma.h
index 6fbdfe895104..0ff3fc6a6d9a 100644
--- a/arch/m68k/include/asm/dma.h
+++ b/arch/m68k/include/asm/dma.h
@@ -33,7 +33,9 @@
33 * Set number of channels of DMA on ColdFire for different implementations. 33 * Set number of channels of DMA on ColdFire for different implementations.
34 */ 34 */
35#if defined(CONFIG_M5249) || defined(CONFIG_M5307) || defined(CONFIG_M5407) || \ 35#if defined(CONFIG_M5249) || defined(CONFIG_M5307) || defined(CONFIG_M5407) || \
36 defined(CONFIG_M523x) || defined(CONFIG_M527x) || defined(CONFIG_M528x) 36 defined(CONFIG_M523x) || defined(CONFIG_M527x) || \
37 defined(CONFIG_M528x) || defined(CONFIG_M525x)
38
37#define MAX_M68K_DMA_CHANNELS 4 39#define MAX_M68K_DMA_CHANNELS 4
38#elif defined(CONFIG_M5272) 40#elif defined(CONFIG_M5272)
39#define MAX_M68K_DMA_CHANNELS 1 41#define MAX_M68K_DMA_CHANNELS 1
@@ -486,6 +488,10 @@ static __inline__ int get_dma_residue(unsigned int dmanr)
486extern int request_dma(unsigned int dmanr, const char * device_id); /* reserve a DMA channel */ 488extern int request_dma(unsigned int dmanr, const char * device_id); /* reserve a DMA channel */
487extern void free_dma(unsigned int dmanr); /* release it again */ 489extern void free_dma(unsigned int dmanr); /* release it again */
488 490
491#ifdef CONFIG_PCI
492extern int isa_dma_bridge_buggy;
493#else
489#define isa_dma_bridge_buggy (0) 494#define isa_dma_bridge_buggy (0)
495#endif
490 496
491#endif /* _M68K_DMA_H */ 497#endif /* _M68K_DMA_H */
diff --git a/arch/m68k/include/asm/gpio.h b/arch/m68k/include/asm/gpio.h
index 00d0071de4c3..4395ffc51fdb 100644
--- a/arch/m68k/include/asm/gpio.h
+++ b/arch/m68k/include/asm/gpio.h
@@ -17,170 +17,9 @@
17#define coldfire_gpio_h 17#define coldfire_gpio_h
18 18
19#include <linux/io.h> 19#include <linux/io.h>
20#include <asm-generic/gpio.h>
21#include <asm/coldfire.h> 20#include <asm/coldfire.h>
22#include <asm/mcfsim.h> 21#include <asm/mcfsim.h>
23 22#include <asm/mcfgpio.h>
24/*
25 * The Freescale Coldfire family is quite varied in how they implement GPIO.
26 * Some parts have 8 bit ports, some have 16bit and some have 32bit; some have
27 * only one port, others have multiple ports; some have a single data latch
28 * for both input and output, others have a separate pin data register to read
29 * input; some require a read-modify-write access to change an output, others
30 * have set and clear registers for some of the outputs; Some have all the
31 * GPIOs in a single control area, others have some GPIOs implemented in
32 * different modules.
33 *
34 * This implementation attempts accommodate the differences while presenting
35 * a generic interface that will optimize to as few instructions as possible.
36 */
37#if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \
38 defined(CONFIG_M520x) || defined(CONFIG_M523x) || \
39 defined(CONFIG_M527x) || defined(CONFIG_M528x) || \
40 defined(CONFIG_M532x) || defined(CONFIG_M54xx)
41
42/* These parts have GPIO organized by 8 bit ports */
43
44#define MCFGPIO_PORTTYPE u8
45#define MCFGPIO_PORTSIZE 8
46#define mcfgpio_read(port) __raw_readb(port)
47#define mcfgpio_write(data, port) __raw_writeb(data, port)
48
49#elif defined(CONFIG_M5307) || defined(CONFIG_M5407) || defined(CONFIG_M5272)
50
51/* These parts have GPIO organized by 16 bit ports */
52
53#define MCFGPIO_PORTTYPE u16
54#define MCFGPIO_PORTSIZE 16
55#define mcfgpio_read(port) __raw_readw(port)
56#define mcfgpio_write(data, port) __raw_writew(data, port)
57
58#elif defined(CONFIG_M5249)
59
60/* These parts have GPIO organized by 32 bit ports */
61
62#define MCFGPIO_PORTTYPE u32
63#define MCFGPIO_PORTSIZE 32
64#define mcfgpio_read(port) __raw_readl(port)
65#define mcfgpio_write(data, port) __raw_writel(data, port)
66
67#endif
68
69#define mcfgpio_bit(gpio) (1 << ((gpio) % MCFGPIO_PORTSIZE))
70#define mcfgpio_port(gpio) ((gpio) / MCFGPIO_PORTSIZE)
71
72#if defined(CONFIG_M520x) || defined(CONFIG_M523x) || \
73 defined(CONFIG_M527x) || defined(CONFIG_M528x) || defined(CONFIG_M532x)
74/*
75 * These parts have an 'Edge' Port module (external interrupt/GPIO) which uses
76 * read-modify-write to change an output and a GPIO module which has separate
77 * set/clr registers to directly change outputs with a single write access.
78 */
79#if defined(CONFIG_M528x)
80/*
81 * The 528x also has GPIOs in other modules (GPT, QADC) which use
82 * read-modify-write as well as those controlled by the EPORT and GPIO modules.
83 */
84#define MCFGPIO_SCR_START 40
85#else
86#define MCFGPIO_SCR_START 8
87#endif
88
89#define MCFGPIO_SETR_PORT(gpio) (MCFGPIO_SETR + \
90 mcfgpio_port(gpio - MCFGPIO_SCR_START))
91
92#define MCFGPIO_CLRR_PORT(gpio) (MCFGPIO_CLRR + \
93 mcfgpio_port(gpio - MCFGPIO_SCR_START))
94#else
95
96#define MCFGPIO_SCR_START MCFGPIO_PIN_MAX
97/* with MCFGPIO_SCR == MCFGPIO_PIN_MAX, these will be optimized away */
98#define MCFGPIO_SETR_PORT(gpio) 0
99#define MCFGPIO_CLRR_PORT(gpio) 0
100
101#endif
102/*
103 * Coldfire specific helper functions
104 */
105
106/* return the port pin data register for a gpio */
107static inline u32 __mcf_gpio_ppdr(unsigned gpio)
108{
109#if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \
110 defined(CONFIG_M5307) || defined(CONFIG_M5407)
111 return MCFSIM_PADAT;
112#elif defined(CONFIG_M5272)
113 if (gpio < 16)
114 return MCFSIM_PADAT;
115 else if (gpio < 32)
116 return MCFSIM_PBDAT;
117 else
118 return MCFSIM_PCDAT;
119#elif defined(CONFIG_M5249)
120 if (gpio < 32)
121 return MCFSIM2_GPIOREAD;
122 else
123 return MCFSIM2_GPIO1READ;
124#elif defined(CONFIG_M520x) || defined(CONFIG_M523x) || \
125 defined(CONFIG_M527x) || defined(CONFIG_M528x) || defined(CONFIG_M532x)
126 if (gpio < 8)
127 return MCFEPORT_EPPDR;
128#if defined(CONFIG_M528x)
129 else if (gpio < 16)
130 return MCFGPTA_GPTPORT;
131 else if (gpio < 24)
132 return MCFGPTB_GPTPORT;
133 else if (gpio < 32)
134 return MCFQADC_PORTQA;
135 else if (gpio < 40)
136 return MCFQADC_PORTQB;
137#endif
138 else
139 return MCFGPIO_PPDR + mcfgpio_port(gpio - MCFGPIO_SCR_START);
140#else
141 return 0;
142#endif
143}
144
145/* return the port output data register for a gpio */
146static inline u32 __mcf_gpio_podr(unsigned gpio)
147{
148#if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \
149 defined(CONFIG_M5307) || defined(CONFIG_M5407)
150 return MCFSIM_PADAT;
151#elif defined(CONFIG_M5272)
152 if (gpio < 16)
153 return MCFSIM_PADAT;
154 else if (gpio < 32)
155 return MCFSIM_PBDAT;
156 else
157 return MCFSIM_PCDAT;
158#elif defined(CONFIG_M5249)
159 if (gpio < 32)
160 return MCFSIM2_GPIOWRITE;
161 else
162 return MCFSIM2_GPIO1WRITE;
163#elif defined(CONFIG_M520x) || defined(CONFIG_M523x) || \
164 defined(CONFIG_M527x) || defined(CONFIG_M528x) || defined(CONFIG_M532x)
165 if (gpio < 8)
166 return MCFEPORT_EPDR;
167#if defined(CONFIG_M528x)
168 else if (gpio < 16)
169 return MCFGPTA_GPTPORT;
170 else if (gpio < 24)
171 return MCFGPTB_GPTPORT;
172 else if (gpio < 32)
173 return MCFQADC_PORTQA;
174 else if (gpio < 40)
175 return MCFQADC_PORTQB;
176#endif
177 else
178 return MCFGPIO_PODR + mcfgpio_port(gpio - MCFGPIO_SCR_START);
179#else
180 return 0;
181#endif
182}
183
184/* 23/*
185 * The Generic GPIO functions 24 * The Generic GPIO functions
186 * 25 *
@@ -191,7 +30,7 @@ static inline u32 __mcf_gpio_podr(unsigned gpio)
191static inline int gpio_get_value(unsigned gpio) 30static inline int gpio_get_value(unsigned gpio)
192{ 31{
193 if (__builtin_constant_p(gpio) && gpio < MCFGPIO_PIN_MAX) 32 if (__builtin_constant_p(gpio) && gpio < MCFGPIO_PIN_MAX)
194 return mcfgpio_read(__mcf_gpio_ppdr(gpio)) & mcfgpio_bit(gpio); 33 return mcfgpio_read(__mcfgpio_ppdr(gpio)) & mcfgpio_bit(gpio);
195 else 34 else
196 return __gpio_get_value(gpio); 35 return __gpio_get_value(gpio);
197} 36}
@@ -204,12 +43,12 @@ static inline void gpio_set_value(unsigned gpio, int value)
204 MCFGPIO_PORTTYPE data; 43 MCFGPIO_PORTTYPE data;
205 44
206 local_irq_save(flags); 45 local_irq_save(flags);
207 data = mcfgpio_read(__mcf_gpio_podr(gpio)); 46 data = mcfgpio_read(__mcfgpio_podr(gpio));
208 if (value) 47 if (value)
209 data |= mcfgpio_bit(gpio); 48 data |= mcfgpio_bit(gpio);
210 else 49 else
211 data &= ~mcfgpio_bit(gpio); 50 data &= ~mcfgpio_bit(gpio);
212 mcfgpio_write(data, __mcf_gpio_podr(gpio)); 51 mcfgpio_write(data, __mcfgpio_podr(gpio));
213 local_irq_restore(flags); 52 local_irq_restore(flags);
214 } else { 53 } else {
215 if (value) 54 if (value)
@@ -225,8 +64,14 @@ static inline void gpio_set_value(unsigned gpio, int value)
225 64
226static inline int gpio_to_irq(unsigned gpio) 65static inline int gpio_to_irq(unsigned gpio)
227{ 66{
228 return (gpio < MCFGPIO_IRQ_MAX) ? gpio + MCFGPIO_IRQ_VECBASE 67#if defined(MCFGPIO_IRQ_MIN)
229 : __gpio_to_irq(gpio); 68 if ((gpio >= MCFGPIO_IRQ_MIN) && (gpio < MCFGPIO_IRQ_MAX))
69#else
70 if (gpio < MCFGPIO_IRQ_MAX)
71#endif
72 return gpio + MCFGPIO_IRQ_VECBASE;
73 else
74 return __gpio_to_irq(gpio);
230} 75}
231 76
232static inline int irq_to_gpio(unsigned irq) 77static inline int irq_to_gpio(unsigned irq)
diff --git a/arch/m68k/include/asm/io_mm.h b/arch/m68k/include/asm/io_mm.h
index fa4324bcf566..a6686d26fe17 100644
--- a/arch/m68k/include/asm/io_mm.h
+++ b/arch/m68k/include/asm/io_mm.h
@@ -65,7 +65,53 @@
65 65
66 66
67 67
68#ifdef CONFIG_ISA 68#if defined(CONFIG_PCI) && defined(CONFIG_COLDFIRE)
69
70#define HAVE_ARCH_PIO_SIZE
71#define PIO_OFFSET 0
72#define PIO_MASK 0xffff
73#define PIO_RESERVED 0x10000
74
75u8 mcf_pci_inb(u32 addr);
76u16 mcf_pci_inw(u32 addr);
77u32 mcf_pci_inl(u32 addr);
78void mcf_pci_insb(u32 addr, u8 *buf, u32 len);
79void mcf_pci_insw(u32 addr, u16 *buf, u32 len);
80void mcf_pci_insl(u32 addr, u32 *buf, u32 len);
81
82void mcf_pci_outb(u8 v, u32 addr);
83void mcf_pci_outw(u16 v, u32 addr);
84void mcf_pci_outl(u32 v, u32 addr);
85void mcf_pci_outsb(u32 addr, const u8 *buf, u32 len);
86void mcf_pci_outsw(u32 addr, const u16 *buf, u32 len);
87void mcf_pci_outsl(u32 addr, const u32 *buf, u32 len);
88
89#define inb mcf_pci_inb
90#define inb_p mcf_pci_inb
91#define inw mcf_pci_inw
92#define inw_p mcf_pci_inw
93#define inl mcf_pci_inl
94#define inl_p mcf_pci_inl
95#define insb mcf_pci_insb
96#define insw mcf_pci_insw
97#define insl mcf_pci_insl
98
99#define outb mcf_pci_outb
100#define outb_p mcf_pci_outb
101#define outw mcf_pci_outw
102#define outw_p mcf_pci_outw
103#define outl mcf_pci_outl
104#define outl_p mcf_pci_outl
105#define outsb mcf_pci_outsb
106#define outsw mcf_pci_outsw
107#define outsl mcf_pci_outsl
108
109#define readb(addr) in_8(addr)
110#define writeb(v, addr) out_8((addr), (v))
111#define readw(addr) in_le16(addr)
112#define writew(v, addr) out_le16((addr), (v))
113
114#elif defined(CONFIG_ISA)
69 115
70#if MULTI_ISA == 0 116#if MULTI_ISA == 0
71#undef MULTI_ISA 117#undef MULTI_ISA
@@ -340,4 +386,6 @@ static inline void memcpy_toio(volatile void __iomem *dst, const void *src, int
340 */ 386 */
341#define xlate_dev_kmem_ptr(p) p 387#define xlate_dev_kmem_ptr(p) p
342 388
389#define ioport_map(port, nr) ((void __iomem *)(port))
390
343#endif /* _IO_H */ 391#endif /* _IO_H */
diff --git a/arch/m68k/include/asm/m520xsim.h b/arch/m68k/include/asm/m520xsim.h
index 17f2aab9cf97..db3f8ee4a6c6 100644
--- a/arch/m68k/include/asm/m520xsim.h
+++ b/arch/m68k/include/asm/m520xsim.h
@@ -42,6 +42,9 @@
42#define MCFINTC1_SIMR (0) 42#define MCFINTC1_SIMR (0)
43#define MCFINTC1_CIMR (0) 43#define MCFINTC1_CIMR (0)
44#define MCFINTC1_ICR0 (0) 44#define MCFINTC1_ICR0 (0)
45#define MCFINTC2_SIMR (0)
46#define MCFINTC2_CIMR (0)
47#define MCFINTC2_ICR0 (0)
45 48
46#define MCFINT_VECBASE 64 49#define MCFINT_VECBASE 64
47#define MCFINT_UART0 26 /* Interrupt number for UART0 */ 50#define MCFINT_UART0 26 /* Interrupt number for UART0 */
@@ -62,6 +65,7 @@
62#define MCF_IRQ_FECENTC0 (MCFINT_VECBASE + MCFINT_FECENTC0) 65#define MCF_IRQ_FECENTC0 (MCFINT_VECBASE + MCFINT_FECENTC0)
63 66
64#define MCF_IRQ_QSPI (MCFINT_VECBASE + MCFINT_QSPI) 67#define MCF_IRQ_QSPI (MCFINT_VECBASE + MCFINT_QSPI)
68#define MCF_IRQ_PIT1 (MCFINT_VECBASE + MCFINT_PIT1)
65 69
66/* 70/*
67 * SDRAM configuration registers. 71 * SDRAM configuration registers.
@@ -186,5 +190,15 @@
186#define MCF_RCR_SWRESET 0x80 /* Software reset bit */ 190#define MCF_RCR_SWRESET 0x80 /* Software reset bit */
187#define MCF_RCR_FRCSTOUT 0x40 /* Force external reset */ 191#define MCF_RCR_FRCSTOUT 0x40 /* Force external reset */
188 192
193/*
194 * Power Management.
195 */
196#define MCFPM_WCR 0xfc040013
197#define MCFPM_PPMSR0 0xfc04002c
198#define MCFPM_PPMCR0 0xfc04002d
199#define MCFPM_PPMHR0 0xfc040030
200#define MCFPM_PPMLR0 0xfc040034
201#define MCFPM_LPCR 0xfc0a0007
202
189/****************************************************************************/ 203/****************************************************************************/
190#endif /* m520xsim_h */ 204#endif /* m520xsim_h */
diff --git a/arch/m68k/include/asm/m523xsim.h b/arch/m68k/include/asm/m523xsim.h
index 075062d4eecd..91d3abc3f2a5 100644
--- a/arch/m68k/include/asm/m523xsim.h
+++ b/arch/m68k/include/asm/m523xsim.h
@@ -52,6 +52,7 @@
52#define MCF_IRQ_FECENTC0 (MCFINT_VECBASE + MCFINT_FECENTC0) 52#define MCF_IRQ_FECENTC0 (MCFINT_VECBASE + MCFINT_FECENTC0)
53 53
54#define MCF_IRQ_QSPI (MCFINT_VECBASE + MCFINT_QSPI) 54#define MCF_IRQ_QSPI (MCFINT_VECBASE + MCFINT_QSPI)
55#define MCF_IRQ_PIT1 (MCFINT_VECBASE + MCFINT_PIT1)
55 56
56/* 57/*
57 * SDRAM configuration registers. 58 * SDRAM configuration registers.
diff --git a/arch/m68k/include/asm/m525xsim.h b/arch/m68k/include/asm/m525xsim.h
new file mode 100644
index 000000000000..6da24f653902
--- /dev/null
+++ b/arch/m68k/include/asm/m525xsim.h
@@ -0,0 +1,194 @@
1/****************************************************************************/
2
3/*
4 * m525xsim.h -- ColdFire 525x System Integration Module support.
5 *
6 * (C) Copyright 2012, Steven king <sfking@fdwdc.com>
7 * (C) Copyright 2002, Greg Ungerer (gerg@snapgear.com)
8 */
9
10/****************************************************************************/
11#ifndef m525xsim_h
12#define m525xsim_h
13/****************************************************************************/
14
15#define CPU_NAME "COLDFIRE(m525x)"
16#define CPU_INSTR_PER_JIFFY 3
17#define MCF_BUSCLK (MCF_CLK / 2)
18
19#include <asm/m52xxacr.h>
20
21/*
22 * The 525x has a second MBAR region, define its address.
23 */
24#define MCF_MBAR2 0x80000000
25
26/*
27 * Define the 525x SIM register set addresses.
28 */
29#define MCFSIM_RSR 0x00 /* Reset Status reg (r/w) */
30#define MCFSIM_SYPCR 0x01 /* System Protection reg (r/w)*/
31#define MCFSIM_SWIVR 0x02 /* SW Watchdog intr reg (r/w) */
32#define MCFSIM_SWSR 0x03 /* SW Watchdog service (r/w) */
33#define MCFSIM_MPARK 0x0C /* BUS Master Control Reg*/
34#define MCFSIM_IPR 0x40 /* Interrupt Pend reg (r/w) */
35#define MCFSIM_IMR 0x44 /* Interrupt Mask reg (r/w) */
36#define MCFSIM_ICR0 0x4c /* Intr Ctrl reg 0 (r/w) */
37#define MCFSIM_ICR1 0x4d /* Intr Ctrl reg 1 (r/w) */
38#define MCFSIM_ICR2 0x4e /* Intr Ctrl reg 2 (r/w) */
39#define MCFSIM_ICR3 0x4f /* Intr Ctrl reg 3 (r/w) */
40#define MCFSIM_ICR4 0x50 /* Intr Ctrl reg 4 (r/w) */
41#define MCFSIM_ICR5 0x51 /* Intr Ctrl reg 5 (r/w) */
42#define MCFSIM_ICR6 0x52 /* Intr Ctrl reg 6 (r/w) */
43#define MCFSIM_ICR7 0x53 /* Intr Ctrl reg 7 (r/w) */
44#define MCFSIM_ICR8 0x54 /* Intr Ctrl reg 8 (r/w) */
45#define MCFSIM_ICR9 0x55 /* Intr Ctrl reg 9 (r/w) */
46#define MCFSIM_ICR10 0x56 /* Intr Ctrl reg 10 (r/w) */
47#define MCFSIM_ICR11 0x57 /* Intr Ctrl reg 11 (r/w) */
48
49#define MCFSIM_CSAR0 0x80 /* CS 0 Address 0 reg (r/w) */
50#define MCFSIM_CSMR0 0x84 /* CS 0 Mask 0 reg (r/w) */
51#define MCFSIM_CSCR0 0x8a /* CS 0 Control reg (r/w) */
52#define MCFSIM_CSAR1 0x8c /* CS 1 Address reg (r/w) */
53#define MCFSIM_CSMR1 0x90 /* CS 1 Mask reg (r/w) */
54#define MCFSIM_CSCR1 0x96 /* CS 1 Control reg (r/w) */
55#define MCFSIM_CSAR2 0x98 /* CS 2 Address reg (r/w) */
56#define MCFSIM_CSMR2 0x9c /* CS 2 Mask reg (r/w) */
57#define MCFSIM_CSCR2 0xa2 /* CS 2 Control reg (r/w) */
58#define MCFSIM_CSAR3 0xa4 /* CS 3 Address reg (r/w) */
59#define MCFSIM_CSMR3 0xa8 /* CS 3 Mask reg (r/w) */
60#define MCFSIM_CSCR3 0xae /* CS 3 Control reg (r/w) */
61#define MCFSIM_CSAR4 0xb0 /* CS 4 Address reg (r/w) */
62#define MCFSIM_CSMR4 0xb4 /* CS 4 Mask reg (r/w) */
63#define MCFSIM_CSCR4 0xba /* CS 4 Control reg (r/w) */
64
65#define MCFSIM_DCR (MCF_MBAR + 0x100) /* DRAM Control */
66#define MCFSIM_DACR0 (MCF_MBAR + 0x108) /* DRAM 0 Addr/Ctrl */
67#define MCFSIM_DMR0 (MCF_MBAR + 0x10c) /* DRAM 0 Mask */
68
69/*
70 * Secondary Interrupt Controller (in MBAR2)
71*/
72#define MCFINTC2_INTBASE (MCF_MBAR2 + 0x168) /* Base Vector Reg */
73#define MCFINTC2_INTPRI1 (MCF_MBAR2 + 0x140) /* 0-7 priority */
74#define MCFINTC2_INTPRI2 (MCF_MBAR2 + 0x144) /* 8-15 priority */
75#define MCFINTC2_INTPRI3 (MCF_MBAR2 + 0x148) /* 16-23 priority */
76#define MCFINTC2_INTPRI4 (MCF_MBAR2 + 0x14c) /* 24-31 priority */
77#define MCFINTC2_INTPRI5 (MCF_MBAR2 + 0x150) /* 32-39 priority */
78#define MCFINTC2_INTPRI6 (MCF_MBAR2 + 0x154) /* 40-47 priority */
79#define MCFINTC2_INTPRI7 (MCF_MBAR2 + 0x158) /* 48-55 priority */
80#define MCFINTC2_INTPRI8 (MCF_MBAR2 + 0x15c) /* 56-63 priority */
81
82#define MCFINTC2_INTPRI_REG(i) (MCFINTC2_INTPRI1 + \
83 ((((i) - MCFINTC2_VECBASE) / 8) * 4))
84#define MCFINTC2_INTPRI_BITS(b, i) ((b) << (((i) % 8) * 4))
85
86/*
87 * Timer module.
88 */
89#define MCFTIMER_BASE1 (MCF_MBAR + 0x140) /* Base of TIMER1 */
90#define MCFTIMER_BASE2 (MCF_MBAR + 0x180) /* Base of TIMER2 */
91
92/*
93 * UART module.
94 */
95#define MCFUART_BASE0 (MCF_MBAR + 0x1c0) /* Base address UART0 */
96#define MCFUART_BASE1 (MCF_MBAR + 0x200) /* Base address UART1 */
97
98/*
99 * QSPI module.
100 */
101#define MCFQSPI_BASE (MCF_MBAR + 0x300) /* Base address QSPI */
102#define MCFQSPI_SIZE 0x40 /* Register set size */
103
104
105#define MCFQSPI_CS0 15
106#define MCFQSPI_CS1 16
107#define MCFQSPI_CS2 24
108#define MCFQSPI_CS3 28
109
110/*
111 * I2C module.
112 */
113#define MCFI2C_BASE0 (MCF_MBAR + 0x280) /* Base addreess I2C0 */
114#define MCFI2C_SIZE0 0x20 /* Register set size */
115
116#define MCFI2C_BASE1 (MCF_MBAR2 + 0x440) /* Base addreess I2C1 */
117#define MCFI2C_SIZE1 0x20 /* Register set size */
118/*
119 * DMA unit base addresses.
120 */
121#define MCFDMA_BASE0 (MCF_MBAR + 0x300) /* Base address DMA 0 */
122#define MCFDMA_BASE1 (MCF_MBAR + 0x340) /* Base address DMA 1 */
123#define MCFDMA_BASE2 (MCF_MBAR + 0x380) /* Base address DMA 2 */
124#define MCFDMA_BASE3 (MCF_MBAR + 0x3C0) /* Base address DMA 3 */
125
126/*
127 * Some symbol defines for the above...
128 */
129#define MCFSIM_SWDICR MCFSIM_ICR0 /* Watchdog timer ICR */
130#define MCFSIM_TIMER1ICR MCFSIM_ICR1 /* Timer 1 ICR */
131#define MCFSIM_TIMER2ICR MCFSIM_ICR2 /* Timer 2 ICR */
132#define MCFSIM_I2CICR MCFSIM_ICR3 /* I2C ICR */
133#define MCFSIM_UART1ICR MCFSIM_ICR4 /* UART 1 ICR */
134#define MCFSIM_UART2ICR MCFSIM_ICR5 /* UART 2 ICR */
135#define MCFSIM_DMA0ICR MCFSIM_ICR6 /* DMA 0 ICR */
136#define MCFSIM_DMA1ICR MCFSIM_ICR7 /* DMA 1 ICR */
137#define MCFSIM_DMA2ICR MCFSIM_ICR8 /* DMA 2 ICR */
138#define MCFSIM_DMA3ICR MCFSIM_ICR9 /* DMA 3 ICR */
139#define MCFSIM_QSPIICR MCFSIM_ICR10 /* QSPI ICR */
140
141/*
142 * Define system peripheral IRQ usage.
143 */
144#define MCF_IRQ_QSPI 28 /* QSPI, Level 4 */
145#define MCF_IRQ_I2C0 29
146#define MCF_IRQ_TIMER 30 /* Timer0, Level 6 */
147#define MCF_IRQ_PROFILER 31 /* Timer1, Level 7 */
148
149#define MCF_IRQ_UART0 73 /* UART0 */
150#define MCF_IRQ_UART1 74 /* UART1 */
151
152/*
153 * Define the base interrupt for the second interrupt controller.
154 * We set it to 128, out of the way of the base interrupts, and plenty
155 * of room for its 64 interrupts.
156 */
157#define MCFINTC2_VECBASE 128
158
159#define MCF_IRQ_GPIO0 (MCFINTC2_VECBASE + 32)
160#define MCF_IRQ_GPIO1 (MCFINTC2_VECBASE + 33)
161#define MCF_IRQ_GPIO2 (MCFINTC2_VECBASE + 34)
162#define MCF_IRQ_GPIO3 (MCFINTC2_VECBASE + 35)
163#define MCF_IRQ_GPIO4 (MCFINTC2_VECBASE + 36)
164#define MCF_IRQ_GPIO5 (MCFINTC2_VECBASE + 37)
165#define MCF_IRQ_GPIO6 (MCFINTC2_VECBASE + 38)
166
167#define MCF_IRQ_USBWUP (MCFINTC2_VECBASE + 40)
168#define MCF_IRQ_I2C1 (MCFINTC2_VECBASE + 62)
169
170/*
171 * General purpose IO registers (in MBAR2).
172 */
173#define MCFSIM2_GPIOREAD (MCF_MBAR2 + 0x000) /* GPIO read values */
174#define MCFSIM2_GPIOWRITE (MCF_MBAR2 + 0x004) /* GPIO write values */
175#define MCFSIM2_GPIOENABLE (MCF_MBAR2 + 0x008) /* GPIO enabled */
176#define MCFSIM2_GPIOFUNC (MCF_MBAR2 + 0x00C) /* GPIO function */
177#define MCFSIM2_GPIO1READ (MCF_MBAR2 + 0x0B0) /* GPIO1 read values */
178#define MCFSIM2_GPIO1WRITE (MCF_MBAR2 + 0x0B4) /* GPIO1 write values */
179#define MCFSIM2_GPIO1ENABLE (MCF_MBAR2 + 0x0B8) /* GPIO1 enabled */
180#define MCFSIM2_GPIO1FUNC (MCF_MBAR2 + 0x0BC) /* GPIO1 function */
181
182#define MCFSIM2_GPIOINTSTAT (MCF_MBAR2 + 0xc0) /* GPIO intr status */
183#define MCFSIM2_GPIOINTCLEAR (MCF_MBAR2 + 0xc0) /* GPIO intr clear */
184#define MCFSIM2_GPIOINTENABLE (MCF_MBAR2 + 0xc4) /* GPIO intr enable */
185
186/*
187 * Generic GPIO support
188 */
189#define MCFGPIO_PIN_MAX 64
190#define MCFGPIO_IRQ_MAX 7
191#define MCFGPIO_IRQ_VECBASE MCF_IRQ_GPIO0
192
193/****************************************************************************/
194#endif /* m525xsim_h */
diff --git a/arch/m68k/include/asm/m527xsim.h b/arch/m68k/include/asm/m527xsim.h
index 83db8106f50a..71aa5104d3d6 100644
--- a/arch/m68k/include/asm/m527xsim.h
+++ b/arch/m68k/include/asm/m527xsim.h
@@ -60,6 +60,7 @@
60#define MCF_IRQ_FECENTC1 (MCFINT2_VECBASE + MCFINT2_FECENTC1) 60#define MCF_IRQ_FECENTC1 (MCFINT2_VECBASE + MCFINT2_FECENTC1)
61 61
62#define MCF_IRQ_QSPI (MCFINT_VECBASE + MCFINT_QSPI) 62#define MCF_IRQ_QSPI (MCFINT_VECBASE + MCFINT_QSPI)
63#define MCF_IRQ_PIT1 (MCFINT_VECBASE + MCFINT_PIT1)
63 64
64/* 65/*
65 * SDRAM configuration registers. 66 * SDRAM configuration registers.
diff --git a/arch/m68k/include/asm/m528xsim.h b/arch/m68k/include/asm/m528xsim.h
index 497c31c803ff..4acb3c0a642e 100644
--- a/arch/m68k/include/asm/m528xsim.h
+++ b/arch/m68k/include/asm/m528xsim.h
@@ -52,7 +52,7 @@
52#define MCF_IRQ_FECENTC0 (MCFINT_VECBASE + MCFINT_FECENTC0) 52#define MCF_IRQ_FECENTC0 (MCFINT_VECBASE + MCFINT_FECENTC0)
53 53
54#define MCF_IRQ_QSPI (MCFINT_VECBASE + MCFINT_QSPI) 54#define MCF_IRQ_QSPI (MCFINT_VECBASE + MCFINT_QSPI)
55 55#define MCF_IRQ_PIT1 (MCFINT_VECBASE + MCFINT_PIT1)
56/* 56/*
57 * SDRAM configuration registers. 57 * SDRAM configuration registers.
58 */ 58 */
diff --git a/arch/m68k/include/asm/m532xsim.h b/arch/m68k/include/asm/m532xsim.h
index 29b66e21413a..5ca7b298c6eb 100644
--- a/arch/m68k/include/asm/m532xsim.h
+++ b/arch/m68k/include/asm/m532xsim.h
@@ -82,6 +82,9 @@
82#define MCFINTC1_SIMR 0xFC04C01C 82#define MCFINTC1_SIMR 0xFC04C01C
83#define MCFINTC1_CIMR 0xFC04C01D 83#define MCFINTC1_CIMR 0xFC04C01D
84#define MCFINTC1_ICR0 0xFC04C040 84#define MCFINTC1_ICR0 0xFC04C040
85#define MCFINTC2_SIMR (0)
86#define MCFINTC2_CIMR (0)
87#define MCFINTC2_ICR0 (0)
85 88
86#define MCFSIM_ICR_TIMER1 (0xFC048040+32) 89#define MCFSIM_ICR_TIMER1 (0xFC048040+32)
87#define MCFSIM_ICR_TIMER2 (0xFC048040+33) 90#define MCFSIM_ICR_TIMER2 (0xFC048040+33)
@@ -135,6 +138,20 @@
135#define MCF_RCR_SWRESET 0x80 /* Software reset bit */ 138#define MCF_RCR_SWRESET 0x80 /* Software reset bit */
136#define MCF_RCR_FRCSTOUT 0x40 /* Force external reset */ 139#define MCF_RCR_FRCSTOUT 0x40 /* Force external reset */
137 140
141
142/*
143 * Power Management
144 */
145#define MCFPM_WCR 0xfc040013
146#define MCFPM_PPMSR0 0xfc04002c
147#define MCFPM_PPMCR0 0xfc04002d
148#define MCFPM_PPMSR1 0xfc04002e
149#define MCFPM_PPMCR1 0xfc04002f
150#define MCFPM_PPMHR0 0xfc040030
151#define MCFPM_PPMLR0 0xfc040034
152#define MCFPM_PPMHR1 0xfc040038
153#define MCFPM_LPCR 0xec090007
154
138/********************************************************************* 155/*********************************************************************
139 * 156 *
140 * Inter-IC (I2C) Module 157 * Inter-IC (I2C) Module
diff --git a/arch/m68k/include/asm/m5441xsim.h b/arch/m68k/include/asm/m5441xsim.h
new file mode 100644
index 000000000000..cc798ab9524b
--- /dev/null
+++ b/arch/m68k/include/asm/m5441xsim.h
@@ -0,0 +1,276 @@
1/*
2 * m5441xsim.h -- Coldfire 5441x register definitions
3 *
4 * (C) Copyright 2012, Steven King <sfking@fdwdc.com>
5*/
6
7#ifndef m5441xsim_h
8#define m5441xsim_h
9
10#define CPU_NAME "COLDFIRE(m5441x)"
11#define CPU_INSTR_PER_JIFFY 2
12#define MCF_BUSCLK (MCF_CLK / 2)
13
14#include <asm/m54xxacr.h>
15
16/*
17 * Reset Controller Module.
18 */
19
20#define MCF_RCR 0xec090000
21#define MCF_RSR 0xec090001
22
23#define MCF_RCR_SWRESET 0x80 /* Software reset bit */
24#define MCF_RCR_FRCSTOUT 0x40 /* Force external reset */
25
26/*
27 * Interrupt Controller Modules.
28 */
29/* the 5441x have 3 interrupt controllers, each control 64 interrupts */
30#define MCFINT_VECBASE 64
31#define MCFINT0_VECBASE MCFINT_VECBASE
32#define MCFINT1_VECBASE (MCFINT0_VECBASE + 64)
33#define MCFINT2_VECBASE (MCFINT1_VECBASE + 64)
34
35/* interrupt controller 0 */
36#define MCFINTC0_SIMR 0xfc04801c
37#define MCFINTC0_CIMR 0xfc04801d
38#define MCFINTC0_ICR0 0xfc048040
39/* interrupt controller 1 */
40#define MCFINTC1_SIMR 0xfc04c01c
41#define MCFINTC1_CIMR 0xfc04c01d
42#define MCFINTC1_ICR0 0xfc04c040
43/* interrupt controller 2 */
44#define MCFINTC2_SIMR 0xfc05001c
45#define MCFINTC2_CIMR 0xfc05001d
46#define MCFINTC2_ICR0 0xfc050040
47
48/* on interrupt controller 0 */
49#define MCFINT0_EPORT0 1
50#define MCFINT0_UART0 26
51#define MCFINT0_UART1 27
52#define MCFINT0_UART2 28
53#define MCFINT0_UART3 29
54#define MCFINT0_I2C0 30
55#define MCFINT0_DSPI0 31
56
57#define MCFINT0_TIMER0 32
58#define MCFINT0_TIMER1 33
59#define MCFINT0_TIMER2 34
60#define MCFINT0_TIMER3 35
61
62#define MCFINT0_FECRX0 36
63#define MCFINT0_FECTX0 40
64#define MCFINT0_FECENTC0 42
65
66#define MCFINT0_FECRX1 49
67#define MCFINT0_FECTX1 53
68#define MCFINT0_FECENTC1 55
69
70/* on interrupt controller 1 */
71#define MCFINT1_UART4 48
72#define MCFINT1_UART5 49
73#define MCFINT1_UART6 50
74#define MCFINT1_UART7 51
75#define MCFINT1_UART8 52
76#define MCFINT1_UART9 53
77#define MCFINT1_DSPI1 54
78#define MCFINT1_DSPI2 55
79#define MCFINT1_DSPI3 56
80#define MCFINT1_I2C1 57
81#define MCFINT1_I2C2 58
82#define MCFINT1_I2C3 59
83#define MCFINT1_I2C4 60
84#define MCFINT1_I2C5 61
85
86/* on interrupt controller 2 */
87#define MCFINT2_PIT0 13
88#define MCFINT2_PIT1 14
89#define MCFINT2_PIT2 15
90#define MCFINT2_PIT3 16
91#define MCFINT2_RTC 26
92
93/*
94 * PIT timer module.
95 */
96#define MCFPIT_BASE0 0xFC080000 /* Base address of TIMER0 */
97#define MCFPIT_BASE1 0xFC084000 /* Base address of TIMER1 */
98#define MCFPIT_BASE2 0xFC088000 /* Base address of TIMER2 */
99#define MCFPIT_BASE3 0xFC08C000 /* Base address of TIMER3 */
100
101
102#define MCF_IRQ_PIT1 (MCFINT2_VECBASE + MCFINT2_PIT1)
103
104/*
105 * Power Management
106 */
107#define MCFPM_WCR 0xfc040013
108#define MCFPM_PPMSR0 0xfc04002c
109#define MCFPM_PPMCR0 0xfc04002d
110#define MCFPM_PPMSR1 0xfc04002e
111#define MCFPM_PPMCR1 0xfc04002f
112#define MCFPM_PPMHR0 0xfc040030
113#define MCFPM_PPMLR0 0xfc040034
114#define MCFPM_PPMHR1 0xfc040038
115#define MCFPM_PPMLR1 0xfc04003c
116#define MCFPM_LPCR 0xec090007
117/*
118 * UART module.
119 */
120#define MCFUART_BASE0 0xfc060000 /* Base address of UART0 */
121#define MCFUART_BASE1 0xfc064000 /* Base address of UART1 */
122#define MCFUART_BASE2 0xfc068000 /* Base address of UART2 */
123#define MCFUART_BASE3 0xfc06c000 /* Base address of UART3 */
124#define MCFUART_BASE4 0xec060000 /* Base address of UART4 */
125#define MCFUART_BASE5 0xec064000 /* Base address of UART5 */
126#define MCFUART_BASE6 0xec068000 /* Base address of UART6 */
127#define MCFUART_BASE7 0xec06c000 /* Base address of UART7 */
128#define MCFUART_BASE8 0xec070000 /* Base address of UART8 */
129#define MCFUART_BASE9 0xec074000 /* Base address of UART9 */
130
131#define MCF_IRQ_UART0 (MCFINT0_VECBASE + MCFINT0_UART0)
132#define MCF_IRQ_UART1 (MCFINT0_VECBASE + MCFINT0_UART1)
133#define MCF_IRQ_UART2 (MCFINT0_VECBASE + MCFINT0_UART2)
134#define MCF_IRQ_UART3 (MCFINT0_VECBASE + MCFINT0_UART3)
135#define MCF_IRQ_UART4 (MCFINT1_VECBASE + MCFINT1_UART4)
136#define MCF_IRQ_UART5 (MCFINT1_VECBASE + MCFINT1_UART5)
137#define MCF_IRQ_UART6 (MCFINT1_VECBASE + MCFINT1_UART6)
138#define MCF_IRQ_UART7 (MCFINT1_VECBASE + MCFINT1_UART7)
139#define MCF_IRQ_UART8 (MCFINT1_VECBASE + MCFINT1_UART8)
140#define MCF_IRQ_UART9 (MCFINT1_VECBASE + MCFINT1_UART9)
141/*
142 * FEC modules.
143 */
144#define MCFFEC_BASE0 0xfc0d4000
145#define MCFFEC_SIZE0 0x800
146#define MCF_IRQ_FECRX0 (MCFINT0_VECBASE + MCFINT0_FECRX0)
147#define MCF_IRQ_FECTX0 (MCFINT0_VECBASE + MCFINT0_FECTX0)
148#define MCF_IRQ_FECENTC0 (MCFINT0_VECBASE + MCFINT0_FECENTC0)
149
150#define MCFFEC_BASE1 0xfc0d8000
151#define MCFFEC_SIZE1 0x800
152#define MCF_IRQ_FECRX1 (MCFINT0_VECBASE + MCFINT0_FECRX1)
153#define MCF_IRQ_FECTX1 (MCFINT0_VECBASE + MCFINT0_FECTX1)
154#define MCF_IRQ_FECENTC1 (MCFINT0_VECBASE + MCFINT0_FECENTC1)
155/*
156 * I2C modules.
157 */
158#define MCFI2C_BASE0 0xfc058000
159#define MCFI2C_SIZE0 0x20
160#define MCFI2C_BASE1 0xfc038000
161#define MCFI2C_SIZE1 0x20
162#define MCFI2C_BASE2 0xec010000
163#define MCFI2C_SIZE2 0x20
164#define MCFI2C_BASE3 0xec014000
165#define MCFI2C_SIZE3 0x20
166#define MCFI2C_BASE4 0xec018000
167#define MCFI2C_SIZE4 0x20
168#define MCFI2C_BASE5 0xec01c000
169#define MCFI2C_SIZE5 0x20
170
171#define MCF_IRQ_I2C0 (MCFINT0_VECBASE + MCFINT0_I2C0)
172#define MCF_IRQ_I2C1 (MCFINT1_VECBASE + MCFINT1_I2C1)
173#define MCF_IRQ_I2C2 (MCFINT1_VECBASE + MCFINT1_I2C2)
174#define MCF_IRQ_I2C3 (MCFINT1_VECBASE + MCFINT1_I2C3)
175#define MCF_IRQ_I2C4 (MCFINT1_VECBASE + MCFINT1_I2C4)
176#define MCF_IRQ_I2C5 (MCFINT1_VECBASE + MCFINT1_I2C5)
177/*
178 * EPORT Module.
179 */
180#define MCFEPORT_EPPAR 0xfc090000
181#define MCFEPORT_EPIER 0xfc090003
182#define MCFEPORT_EPFR 0xfc090006
183/*
184 * RTC Module.
185 */
186#define MCFRTC_BASE 0xfc0a8000
187#define MCFRTC_SIZE (0xfc0a8840 - 0xfc0a8000)
188#define MCF_IRQ_RTC (MCFINT2_VECBASE + MCFINT2_RTC)
189
190/*
191 * GPIO Module.
192 */
193#define MCFGPIO_PODR_A 0xec094000
194#define MCFGPIO_PODR_B 0xec094001
195#define MCFGPIO_PODR_C 0xec094002
196#define MCFGPIO_PODR_D 0xec094003
197#define MCFGPIO_PODR_E 0xec094004
198#define MCFGPIO_PODR_F 0xec094005
199#define MCFGPIO_PODR_G 0xec094006
200#define MCFGPIO_PODR_H 0xec094007
201#define MCFGPIO_PODR_I 0xec094008
202#define MCFGPIO_PODR_J 0xec094009
203#define MCFGPIO_PODR_K 0xec09400a
204
205#define MCFGPIO_PDDR_A 0xec09400c
206#define MCFGPIO_PDDR_B 0xec09400d
207#define MCFGPIO_PDDR_C 0xec09400e
208#define MCFGPIO_PDDR_D 0xec09400f
209#define MCFGPIO_PDDR_E 0xec094010
210#define MCFGPIO_PDDR_F 0xec094011
211#define MCFGPIO_PDDR_G 0xec094012
212#define MCFGPIO_PDDR_H 0xec094013
213#define MCFGPIO_PDDR_I 0xec094014
214#define MCFGPIO_PDDR_J 0xec094015
215#define MCFGPIO_PDDR_K 0xec094016
216
217#define MCFGPIO_PPDSDR_A 0xec094018
218#define MCFGPIO_PPDSDR_B 0xec094019
219#define MCFGPIO_PPDSDR_C 0xec09401a
220#define MCFGPIO_PPDSDR_D 0xec09401b
221#define MCFGPIO_PPDSDR_E 0xec09401c
222#define MCFGPIO_PPDSDR_F 0xec09401d
223#define MCFGPIO_PPDSDR_G 0xec09401e
224#define MCFGPIO_PPDSDR_H 0xec09401f
225#define MCFGPIO_PPDSDR_I 0xec094020
226#define MCFGPIO_PPDSDR_J 0xec094021
227#define MCFGPIO_PPDSDR_K 0xec094022
228
229#define MCFGPIO_PCLRR_A 0xec094024
230#define MCFGPIO_PCLRR_B 0xec094025
231#define MCFGPIO_PCLRR_C 0xec094026
232#define MCFGPIO_PCLRR_D 0xec094027
233#define MCFGPIO_PCLRR_E 0xec094028
234#define MCFGPIO_PCLRR_F 0xec094029
235#define MCFGPIO_PCLRR_G 0xec09402a
236#define MCFGPIO_PCLRR_H 0xec09402b
237#define MCFGPIO_PCLRR_I 0xec09402c
238#define MCFGPIO_PCLRR_J 0xec09402d
239#define MCFGPIO_PCLRR_K 0xec09402e
240
241#define MCFGPIO_PAR_FBCTL 0xec094048
242#define MCFGPIO_PAR_BE 0xec094049
243#define MCFGPIO_PAR_CS 0xec09404a
244#define MCFGPIO_PAR_CANI2C 0xec09404b
245#define MCFGPIO_PAR_IRQ0H 0xec09404c
246#define MCFGPIO_PAR_IRQ0L 0xec09404d
247#define MCFGPIO_PAR_DSPIOWH 0xec09404e
248#define MCFGPIO_PAR_DSPIOWL 0xec09404f
249#define MCFGPIO_PAR_TIMER 0xec094050
250#define MCFGPIO_PAR_UART2 0xec094051
251#define MCFGPIO_PAR_UART1 0xec094052
252#define MCFGPIO_PAR_UART0 0xec094053
253#define MCFGPIO_PAR_SDHCH 0xec094054
254#define MCFGPIO_PAR_SDHCL 0xec094055
255#define MCFGPIO_PAR_SIMP0H 0xec094056
256#define MCFGPIO_PAR_SIMP0L 0xec094057
257#define MCFGPIO_PAR_SSI0H 0xec094058
258#define MCFGPIO_PAR_SSI0L 0xec094059
259#define MCFGPIO_PAR_DEBUGH1 0xec09405a
260#define MCFGPIO_PAR_DEBUGH0 0xec09405b
261#define MCFGPIO_PAR_DEBUGl 0xec09405c
262#define MCFGPIO_PAR_FEC 0xec09405e
263
264/* generalization for generic gpio support */
265#define MCFGPIO_PODR MCFGPIO_PODR_A
266#define MCFGPIO_PDDR MCFGPIO_PDDR_A
267#define MCFGPIO_PPDR MCFGPIO_PPDSDR_A
268#define MCFGPIO_SETR MCFGPIO_PPDSDR_A
269#define MCFGPIO_CLRR MCFGPIO_PCLRR_A
270
271#define MCFGPIO_IRQ_MIN 17
272#define MCFGPIO_IRQ_MAX 24
273#define MCFGPIO_IRQ_VECBASE (MCFINT_VECBASE - MCFGPIO_IRQ_MIN)
274#define MCFGPIO_PIN_MAX 87
275
276#endif /* m5441xsim_h */
diff --git a/arch/m68k/include/asm/m54xxacr.h b/arch/m68k/include/asm/m54xxacr.h
index 47906aafbf67..192bbfeabf70 100644
--- a/arch/m68k/include/asm/m54xxacr.h
+++ b/arch/m68k/include/asm/m54xxacr.h
@@ -55,6 +55,10 @@
55#define ICACHE_SIZE 0x8000 /* instruction - 32k */ 55#define ICACHE_SIZE 0x8000 /* instruction - 32k */
56#define DCACHE_SIZE 0x8000 /* data - 32k */ 56#define DCACHE_SIZE 0x8000 /* data - 32k */
57 57
58#elif defined(CONFIG_M5441x)
59
60#define ICACHE_SIZE 0x2000 /* instruction - 8k */
61#define DCACHE_SIZE 0x2000 /* data - 8k */
58#endif 62#endif
59 63
60#define CACHE_LINE_SIZE 0x0010 /* 16 bytes */ 64#define CACHE_LINE_SIZE 0x0010 /* 16 bytes */
diff --git a/arch/m68k/include/asm/m54xxpci.h b/arch/m68k/include/asm/m54xxpci.h
new file mode 100644
index 000000000000..6fbf54f72f2e
--- /dev/null
+++ b/arch/m68k/include/asm/m54xxpci.h
@@ -0,0 +1,138 @@
1/****************************************************************************/
2
3/*
4 * m54xxpci.h -- ColdFire 547x and 548x PCI bus support
5 *
6 * (C) Copyright 2011, Greg Ungerer <gerg@uclinux.org>
7 *
8 * This file is subject to the terms and conditions of the GNU General Public
9 * License. See the file COPYING in the main directory of this archive
10 * for more details.
11 */
12
13/****************************************************************************/
14#ifndef M54XXPCI_H
15#define M54XXPCI_H
16/****************************************************************************/
17
18/*
19 * The core set of PCI support registers are mapped into the MBAR region.
20 */
21#define PCIIDR (CONFIG_MBAR + 0xb00) /* PCI device/vendor ID */
22#define PCISCR (CONFIG_MBAR + 0xb04) /* PCI status/command */
23#define PCICCRIR (CONFIG_MBAR + 0xb08) /* PCI class/revision */
24#define PCICR1 (CONFIG_MBAR + 0xb0c) /* PCI configuration 1 */
25#define PCIBAR0 (CONFIG_MBAR + 0xb10) /* PCI base address 0 */
26#define PCIBAR1 (CONFIG_MBAR + 0xb14) /* PCI base address 1 */
27#define PCICCPR (CONFIG_MBAR + 0xb28) /* PCI cardbus CIS pointer */
28#define PCISID (CONFIG_MBAR + 0xb2c) /* PCI subsystem IDs */
29#define PCIERBAR (CONFIG_MBAR + 0xb30) /* PCI expansion ROM */
30#define PCICPR (CONFIG_MBAR + 0xb34) /* PCI capabilities pointer */
31#define PCICR2 (CONFIG_MBAR + 0xb3c) /* PCI configuration 2 */
32
33#define PCIGSCR (CONFIG_MBAR + 0xb60) /* Global status/control */
34#define PCITBATR0 (CONFIG_MBAR + 0xb64) /* Target base translation 0 */
35#define PCITBATR1 (CONFIG_MBAR + 0xb68) /* Target base translation 1 */
36#define PCITCR (CONFIG_MBAR + 0xb6c) /* Target control */
37#define PCIIW0BTAR (CONFIG_MBAR + 0xb70) /* Initiator window 0 */
38#define PCIIW1BTAR (CONFIG_MBAR + 0xb74) /* Initiator window 1 */
39#define PCIIW2BTAR (CONFIG_MBAR + 0xb78) /* Initiator window 2 */
40#define PCIIWCR (CONFIG_MBAR + 0xb80) /* Initiator window config */
41#define PCIICR (CONFIG_MBAR + 0xb84) /* Initiator control */
42#define PCIISR (CONFIG_MBAR + 0xb88) /* Initiator status */
43#define PCICAR (CONFIG_MBAR + 0xbf8) /* Configuration address */
44
45#define PCITPSR (CONFIG_MBAR + 0x8400) /* TX packet size */
46#define PCITSAR (CONFIG_MBAR + 0x8404) /* TX start address */
47#define PCITTCR (CONFIG_MBAR + 0x8408) /* TX transaction control */
48#define PCITER (CONFIG_MBAR + 0x840c) /* TX enables */
49#define PCITNAR (CONFIG_MBAR + 0x8410) /* TX next address */
50#define PCITLWR (CONFIG_MBAR + 0x8414) /* TX last word */
51#define PCITDCR (CONFIG_MBAR + 0x8418) /* TX done counts */
52#define PCITSR (CONFIG_MBAR + 0x841c) /* TX status */
53#define PCITFDR (CONFIG_MBAR + 0x8440) /* TX FIFO data */
54#define PCITFSR (CONFIG_MBAR + 0x8444) /* TX FIFO status */
55#define PCITFCR (CONFIG_MBAR + 0x8448) /* TX FIFO control */
56#define PCITFAR (CONFIG_MBAR + 0x844c) /* TX FIFO alarm */
57#define PCITFRPR (CONFIG_MBAR + 0x8450) /* TX FIFO read pointer */
58#define PCITFWPR (CONFIG_MBAR + 0x8454) /* TX FIFO write pointer */
59
60#define PCIRPSR (CONFIG_MBAR + 0x8480) /* RX packet size */
61#define PCIRSAR (CONFIG_MBAR + 0x8484) /* RX start address */
62#define PCIRTCR (CONFIG_MBAR + 0x8488) /* RX transaction control */
63#define PCIRER (CONFIG_MBAR + 0x848c) /* RX enables */
64#define PCIRNAR (CONFIG_MBAR + 0x8490) /* RX next address */
65#define PCIRDCR (CONFIG_MBAR + 0x8498) /* RX done counts */
66#define PCIRSR (CONFIG_MBAR + 0x849c) /* RX status */
67#define PCIRFDR (CONFIG_MBAR + 0x84c0) /* RX FIFO data */
68#define PCIRFSR (CONFIG_MBAR + 0x84c4) /* RX FIFO status */
69#define PCIRFCR (CONFIG_MBAR + 0x84c8) /* RX FIFO control */
70#define PCIRFAR (CONFIG_MBAR + 0x84cc) /* RX FIFO alarm */
71#define PCIRFRPR (CONFIG_MBAR + 0x84d0) /* RX FIFO read pointer */
72#define PCIRFWPR (CONFIG_MBAR + 0x84d4) /* RX FIFO write pointer */
73
74#define PACR (CONFIG_MBAR + 0xc00) /* PCI arbiter control */
75#define PASR (COFNIG_MBAR + 0xc04) /* PCI arbiter status */
76
77/*
78 * Definitions for the Global status and control register.
79 */
80#define PCIGSCR_PE 0x20000000 /* Parity error detected */
81#define PCIGSCR_SE 0x10000000 /* System error detected */
82#define PCIGSCR_XCLKBIN 0x07000000 /* XLB2CLKIN mask */
83#define PCIGSCR_PEE 0x00002000 /* Parity error intr enable */
84#define PCIGSCR_SEE 0x00001000 /* System error intr enable */
85#define PCIGSCR_RESET 0x00000001 /* Reset bit */
86
87/*
88 * Bit definitions for the PCICAR configuration address register.
89 */
90#define PCICAR_E 0x80000000 /* Enable config space */
91#define PCICAR_BUSN 16 /* Move bus bits */
92#define PCICAR_DEVFNN 8 /* Move devfn bits */
93#define PCICAR_DWORDN 0 /* Move dword bits */
94
95/*
96 * The initiator windows hold the memory and IO mapping information.
97 * This macro creates the register values from the desired addresses.
98 */
99#define WXBTAR(hostaddr, pciaddr, size) \
100 (((hostaddr) & 0xff000000) | \
101 ((((size) - 1) & 0xff000000) >> 8) | \
102 (((pciaddr) & 0xff000000) >> 16))
103
104#define PCIIWCR_W0_MEM 0x00000000 /* Window 0 is memory */
105#define PCIIWCR_W0_IO 0x08000000 /* Window 0 is IO */
106#define PCIIWCR_W0_MRD 0x00000000 /* Window 0 memory read */
107#define PCIIWCR_W0_MRDL 0x02000000 /* Window 0 memory read line */
108#define PCIIWCR_W0_MRDM 0x04000000 /* Window 0 memory read mult */
109#define PCIIWCR_W0_E 0x01000000 /* Window 0 enable */
110
111#define PCIIWCR_W1_MEM 0x00000000 /* Window 0 is memory */
112#define PCIIWCR_W1_IO 0x00080000 /* Window 0 is IO */
113#define PCIIWCR_W1_MRD 0x00000000 /* Window 0 memory read */
114#define PCIIWCR_W1_MRDL 0x00020000 /* Window 0 memory read line */
115#define PCIIWCR_W1_MRDM 0x00040000 /* Window 0 memory read mult */
116#define PCIIWCR_W1_E 0x00010000 /* Window 0 enable */
117
118/*
119 * Bit definitions for the PCIBATR registers.
120 */
121#define PCITBATR0_E 0x00000001 /* Enable window 0 */
122#define PCITBATR1_E 0x00000001 /* Enable window 1 */
123
124/*
125 * PCI arbiter support definitions and macros.
126 */
127#define PACR_INTMPRI 0x00000001
128#define PACR_EXTMPRI(x) (((x) & 0x1f) << 1)
129#define PACR_INTMINTE 0x00010000
130#define PACR_EXTMINTE(x) (((x) & 0x1f) << 17)
131#define PACR_PKMD 0x40000000
132#define PACR_DS 0x80000000
133
134#define PCICR1_CL(x) ((x) & 0xf) /* Cacheline size field */
135#define PCICR1_LT(x) (((x) & 0xff) << 8) /* Latency timer field */
136
137/****************************************************************************/
138#endif /* M54XXPCI_H */
diff --git a/arch/m68k/include/asm/m54xxsim.h b/arch/m68k/include/asm/m54xxsim.h
index ae56b8848a9d..d3c5e0dbdadf 100644
--- a/arch/m68k/include/asm/m54xxsim.h
+++ b/arch/m68k/include/asm/m54xxsim.h
@@ -81,4 +81,7 @@
81#define MCF_PAR_PSC_RTS_RTS (0x30) 81#define MCF_PAR_PSC_RTS_RTS (0x30)
82#define MCF_PAR_PSC_CANRX (0x40) 82#define MCF_PAR_PSC_CANRX (0x40)
83 83
84#define MCF_PAR_PCIBG (CONFIG_MBAR + 0xa48) /* PCI bus grant */
85#define MCF_PAR_PCIBR (CONFIG_MBAR + 0xa4a) /* PCI */
86
84#endif /* m54xxsim_h */ 87#endif /* m54xxsim_h */
diff --git a/arch/m68k/include/asm/mcfclk.h b/arch/m68k/include/asm/mcfclk.h
new file mode 100644
index 000000000000..b676a02bb392
--- /dev/null
+++ b/arch/m68k/include/asm/mcfclk.h
@@ -0,0 +1,43 @@
1/*
2 * mcfclk.h -- coldfire specific clock structure
3 */
4
5
6#ifndef mcfclk_h
7#define mcfclk_h
8
9struct clk;
10
11#ifdef MCFPM_PPMCR0
12struct clk_ops {
13 void (*enable)(struct clk *);
14 void (*disable)(struct clk *);
15};
16
17struct clk {
18 const char *name;
19 struct clk_ops *clk_ops;
20 unsigned long rate;
21 unsigned long enabled;
22 u8 slot;
23};
24
25extern struct clk *mcf_clks[];
26extern struct clk_ops clk_ops0;
27#ifdef MCFPM_PPMCR1
28extern struct clk_ops clk_ops1;
29#endif /* MCFPM_PPMCR1 */
30
31#define DEFINE_CLK(clk_bank, clk_name, clk_slot, clk_rate) \
32static struct clk __clk_##clk_bank##_##clk_slot = { \
33 .name = clk_name, \
34 .clk_ops = &clk_ops##clk_bank, \
35 .rate = clk_rate, \
36 .slot = clk_slot, \
37}
38
39void __clk_init_enabled(struct clk *);
40void __clk_init_disabled(struct clk *);
41#endif /* MCFPM_PPMCR0 */
42
43#endif /* mcfclk_h */
diff --git a/arch/m68k/include/asm/mcfgpio.h b/arch/m68k/include/asm/mcfgpio.h
index fe468eaa51e0..fa1059f50dfc 100644
--- a/arch/m68k/include/asm/mcfgpio.h
+++ b/arch/m68k/include/asm/mcfgpio.h
@@ -16,82 +16,289 @@
16#ifndef mcfgpio_h 16#ifndef mcfgpio_h
17#define mcfgpio_h 17#define mcfgpio_h
18 18
19#include <linux/io.h> 19#ifdef CONFIG_GPIOLIB
20#include <asm-generic/gpio.h> 20#include <asm-generic/gpio.h>
21#else
22
23int __mcfgpio_get_value(unsigned gpio);
24void __mcfgpio_set_value(unsigned gpio, int value);
25int __mcfgpio_direction_input(unsigned gpio);
26int __mcfgpio_direction_output(unsigned gpio, int value);
27int __mcfgpio_request(unsigned gpio);
28void __mcfgpio_free(unsigned gpio);
29
30/* our alternate 'gpiolib' functions */
31static inline int __gpio_get_value(unsigned gpio)
32{
33 if (gpio < MCFGPIO_PIN_MAX)
34 return __mcfgpio_get_value(gpio);
35 else
36 return -EINVAL;
37}
38
39static inline void __gpio_set_value(unsigned gpio, int value)
40{
41 if (gpio < MCFGPIO_PIN_MAX)
42 __mcfgpio_set_value(gpio, value);
43}
44
45static inline int __gpio_cansleep(unsigned gpio)
46{
47 if (gpio < MCFGPIO_PIN_MAX)
48 return 0;
49 else
50 return -EINVAL;
51}
52
53static inline int __gpio_to_irq(unsigned gpio)
54{
55 return -EINVAL;
56}
57
58static inline int gpio_direction_input(unsigned gpio)
59{
60 if (gpio < MCFGPIO_PIN_MAX)
61 return __mcfgpio_direction_input(gpio);
62 else
63 return -EINVAL;
64}
65
66static inline int gpio_direction_output(unsigned gpio, int value)
67{
68 if (gpio < MCFGPIO_PIN_MAX)
69 return __mcfgpio_direction_output(gpio, value);
70 else
71 return -EINVAL;
72}
73
74static inline int gpio_request(unsigned gpio, const char *label)
75{
76 if (gpio < MCFGPIO_PIN_MAX)
77 return __mcfgpio_request(gpio);
78 else
79 return -EINVAL;
80}
81
82static inline void gpio_free(unsigned gpio)
83{
84 if (gpio < MCFGPIO_PIN_MAX)
85 __mcfgpio_free(gpio);
86}
87
88#endif /* CONFIG_GPIOLIB */
21 89
22struct mcf_gpio_chip {
23 struct gpio_chip gpio_chip;
24 void __iomem *pddr;
25 void __iomem *podr;
26 void __iomem *ppdr;
27 void __iomem *setr;
28 void __iomem *clrr;
29 const u8 *gpio_to_pinmux;
30};
31
32extern struct mcf_gpio_chip mcf_gpio_chips[];
33extern unsigned int mcf_gpio_chips_size;
34
35int mcf_gpio_direction_input(struct gpio_chip *, unsigned);
36int mcf_gpio_get_value(struct gpio_chip *, unsigned);
37int mcf_gpio_direction_output(struct gpio_chip *, unsigned, int);
38void mcf_gpio_set_value(struct gpio_chip *, unsigned, int);
39void mcf_gpio_set_value_fast(struct gpio_chip *, unsigned, int);
40int mcf_gpio_request(struct gpio_chip *, unsigned);
41void mcf_gpio_free(struct gpio_chip *, unsigned);
42 90
43/* 91/*
44 * Define macros to ease the pain of setting up the GPIO tables. There 92 * The Freescale Coldfire family is quite varied in how they implement GPIO.
45 * are two cases we need to deal with here, they cover all currently 93 * Some parts have 8 bit ports, some have 16bit and some have 32bit; some have
46 * available ColdFire GPIO hardware. There are of course minor differences 94 * only one port, others have multiple ports; some have a single data latch
47 * in the layout and number of bits in each ColdFire part, but the macros 95 * for both input and output, others have a separate pin data register to read
48 * take all that in. 96 * input; some require a read-modify-write access to change an output, others
97 * have set and clear registers for some of the outputs; Some have all the
98 * GPIOs in a single control area, others have some GPIOs implemented in
99 * different modules.
49 * 100 *
50 * Firstly is the conventional GPIO registers where we toggle individual 101 * This implementation attempts accommodate the differences while presenting
51 * bits in a register, preserving the other bits in the register. For 102 * a generic interface that will optimize to as few instructions as possible.
52 * lack of a better term I have called this the slow method. 103 */
104#if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \
105 defined(CONFIG_M520x) || defined(CONFIG_M523x) || \
106 defined(CONFIG_M527x) || defined(CONFIG_M528x) || \
107 defined(CONFIG_M532x) || defined(CONFIG_M54xx) || \
108 defined(CONFIG_M5441x)
109
110/* These parts have GPIO organized by 8 bit ports */
111
112#define MCFGPIO_PORTTYPE u8
113#define MCFGPIO_PORTSIZE 8
114#define mcfgpio_read(port) __raw_readb(port)
115#define mcfgpio_write(data, port) __raw_writeb(data, port)
116
117#elif defined(CONFIG_M5307) || defined(CONFIG_M5407) || defined(CONFIG_M5272)
118
119/* These parts have GPIO organized by 16 bit ports */
120
121#define MCFGPIO_PORTTYPE u16
122#define MCFGPIO_PORTSIZE 16
123#define mcfgpio_read(port) __raw_readw(port)
124#define mcfgpio_write(data, port) __raw_writew(data, port)
125
126#elif defined(CONFIG_M5249) || defined(CONFIG_M525x)
127
128/* These parts have GPIO organized by 32 bit ports */
129
130#define MCFGPIO_PORTTYPE u32
131#define MCFGPIO_PORTSIZE 32
132#define mcfgpio_read(port) __raw_readl(port)
133#define mcfgpio_write(data, port) __raw_writel(data, port)
134
135#endif
136
137#define mcfgpio_bit(gpio) (1 << ((gpio) % MCFGPIO_PORTSIZE))
138#define mcfgpio_port(gpio) ((gpio) / MCFGPIO_PORTSIZE)
139
140#if defined(CONFIG_M520x) || defined(CONFIG_M523x) || \
141 defined(CONFIG_M527x) || defined(CONFIG_M528x) || \
142 defined(CONFIG_M532x) || defined(CONFIG_M5441x)
143/*
144 * These parts have an 'Edge' Port module (external interrupt/GPIO) which uses
145 * read-modify-write to change an output and a GPIO module which has separate
146 * set/clr registers to directly change outputs with a single write access.
147 */
148#if defined(CONFIG_M528x)
149/*
150 * The 528x also has GPIOs in other modules (GPT, QADC) which use
151 * read-modify-write as well as those controlled by the EPORT and GPIO modules.
53 */ 152 */
54#define MCFGPS(mlabel, mbase, mngpio, mpddr, mpodr, mppdr) \ 153#define MCFGPIO_SCR_START 40
55 { \ 154#elif defined(CONFIGM5441x)
56 .gpio_chip = { \ 155/* The m5441x EPORT doesn't have its own GPIO port, uses PORT C */
57 .label = #mlabel, \ 156#define MCFGPIO_SCR_START 0
58 .request = mcf_gpio_request, \ 157#else
59 .free = mcf_gpio_free, \ 158#define MCFGPIO_SCR_START 8
60 .direction_input = mcf_gpio_direction_input, \ 159#endif
61 .direction_output = mcf_gpio_direction_output,\
62 .get = mcf_gpio_get_value, \
63 .set = mcf_gpio_set_value, \
64 .base = mbase, \
65 .ngpio = mngpio, \
66 }, \
67 .pddr = (void __iomem *) mpddr, \
68 .podr = (void __iomem *) mpodr, \
69 .ppdr = (void __iomem *) mppdr, \
70 }
71 160
161#define MCFGPIO_SETR_PORT(gpio) (MCFGPIO_SETR + \
162 mcfgpio_port(gpio - MCFGPIO_SCR_START))
163
164#define MCFGPIO_CLRR_PORT(gpio) (MCFGPIO_CLRR + \
165 mcfgpio_port(gpio - MCFGPIO_SCR_START))
166#else
167
168#define MCFGPIO_SCR_START MCFGPIO_PIN_MAX
169/* with MCFGPIO_SCR == MCFGPIO_PIN_MAX, these will be optimized away */
170#define MCFGPIO_SETR_PORT(gpio) 0
171#define MCFGPIO_CLRR_PORT(gpio) 0
172
173#endif
72/* 174/*
73 * Secondly is the faster case, where we have set and clear registers 175 * Coldfire specific helper functions
74 * that allow us to set or clear a bit with a single write, not having
75 * to worry about preserving other bits.
76 */ 176 */
77#define MCFGPF(mlabel, mbase, mngpio) \
78 { \
79 .gpio_chip = { \
80 .label = #mlabel, \
81 .request = mcf_gpio_request, \
82 .free = mcf_gpio_free, \
83 .direction_input = mcf_gpio_direction_input, \
84 .direction_output = mcf_gpio_direction_output,\
85 .get = mcf_gpio_get_value, \
86 .set = mcf_gpio_set_value_fast, \
87 .base = mbase, \
88 .ngpio = mngpio, \
89 }, \
90 .pddr = (void __iomem *) MCFGPIO_PDDR_##mlabel, \
91 .podr = (void __iomem *) MCFGPIO_PODR_##mlabel, \
92 .ppdr = (void __iomem *) MCFGPIO_PPDSDR_##mlabel, \
93 .setr = (void __iomem *) MCFGPIO_PPDSDR_##mlabel, \
94 .clrr = (void __iomem *) MCFGPIO_PCLRR_##mlabel, \
95 }
96 177
178/* return the port pin data register for a gpio */
179static inline u32 __mcfgpio_ppdr(unsigned gpio)
180{
181#if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \
182 defined(CONFIG_M5307) || defined(CONFIG_M5407)
183 return MCFSIM_PADAT;
184#elif defined(CONFIG_M5272)
185 if (gpio < 16)
186 return MCFSIM_PADAT;
187 else if (gpio < 32)
188 return MCFSIM_PBDAT;
189 else
190 return MCFSIM_PCDAT;
191#elif defined(CONFIG_M5249) || defined(CONFIG_M525x)
192 if (gpio < 32)
193 return MCFSIM2_GPIOREAD;
194 else
195 return MCFSIM2_GPIO1READ;
196#elif defined(CONFIG_M520x) || defined(CONFIG_M523x) || \
197 defined(CONFIG_M527x) || defined(CONFIG_M528x) || \
198 defined(CONFIG_M532x) || defined(CONFIG_M5441x)
199#if !defined(CONFIG_M5441x)
200 if (gpio < 8)
201 return MCFEPORT_EPPDR;
202#if defined(CONFIG_M528x)
203 else if (gpio < 16)
204 return MCFGPTA_GPTPORT;
205 else if (gpio < 24)
206 return MCFGPTB_GPTPORT;
207 else if (gpio < 32)
208 return MCFQADC_PORTQA;
209 else if (gpio < 40)
210 return MCFQADC_PORTQB;
211#endif /* defined(CONFIG_M528x) */
212 else
213#endif /* !defined(CONFIG_M5441x) */
214 return MCFGPIO_PPDR + mcfgpio_port(gpio - MCFGPIO_SCR_START);
215#else
216 return 0;
97#endif 217#endif
218}
219
220/* return the port output data register for a gpio */
221static inline u32 __mcfgpio_podr(unsigned gpio)
222{
223#if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \
224 defined(CONFIG_M5307) || defined(CONFIG_M5407)
225 return MCFSIM_PADAT;
226#elif defined(CONFIG_M5272)
227 if (gpio < 16)
228 return MCFSIM_PADAT;
229 else if (gpio < 32)
230 return MCFSIM_PBDAT;
231 else
232 return MCFSIM_PCDAT;
233#elif defined(CONFIG_M5249) || defined(CONFIG_M525x)
234 if (gpio < 32)
235 return MCFSIM2_GPIOWRITE;
236 else
237 return MCFSIM2_GPIO1WRITE;
238#elif defined(CONFIG_M520x) || defined(CONFIG_M523x) || \
239 defined(CONFIG_M527x) || defined(CONFIG_M528x) || \
240 defined(CONFIG_M532x) || defined(CONFIG_M5441x)
241#if !defined(CONFIG_M5441x)
242 if (gpio < 8)
243 return MCFEPORT_EPDR;
244#if defined(CONFIG_M528x)
245 else if (gpio < 16)
246 return MCFGPTA_GPTPORT;
247 else if (gpio < 24)
248 return MCFGPTB_GPTPORT;
249 else if (gpio < 32)
250 return MCFQADC_PORTQA;
251 else if (gpio < 40)
252 return MCFQADC_PORTQB;
253#endif /* defined(CONFIG_M528x) */
254 else
255#endif /* !defined(CONFIG_M5441x) */
256 return MCFGPIO_PODR + mcfgpio_port(gpio - MCFGPIO_SCR_START);
257#else
258 return 0;
259#endif
260}
261
262/* return the port direction data register for a gpio */
263static inline u32 __mcfgpio_pddr(unsigned gpio)
264{
265#if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \
266 defined(CONFIG_M5307) || defined(CONFIG_M5407)
267 return MCFSIM_PADDR;
268#elif defined(CONFIG_M5272)
269 if (gpio < 16)
270 return MCFSIM_PADDR;
271 else if (gpio < 32)
272 return MCFSIM_PBDDR;
273 else
274 return MCFSIM_PCDDR;
275#elif defined(CONFIG_M5249) || defined(CONFIG_M525x)
276 if (gpio < 32)
277 return MCFSIM2_GPIOENABLE;
278 else
279 return MCFSIM2_GPIO1ENABLE;
280#elif defined(CONFIG_M520x) || defined(CONFIG_M523x) || \
281 defined(CONFIG_M527x) || defined(CONFIG_M528x) || \
282 defined(CONFIG_M532x) || defined(CONFIG_M5441x)
283#if !defined(CONFIG_M5441x)
284 if (gpio < 8)
285 return MCFEPORT_EPDDR;
286#if defined(CONFIG_M528x)
287 else if (gpio < 16)
288 return MCFGPTA_GPTDDR;
289 else if (gpio < 24)
290 return MCFGPTB_GPTDDR;
291 else if (gpio < 32)
292 return MCFQADC_DDRQA;
293 else if (gpio < 40)
294 return MCFQADC_DDRQB;
295#endif /* defined(CONFIG_M528x) */
296 else
297#endif /* !defined(CONFIG_M5441x) */
298 return MCFGPIO_PDDR + mcfgpio_port(gpio - MCFGPIO_SCR_START);
299#else
300 return 0;
301#endif
302}
303
304#endif /* mcfgpio_h */
diff --git a/arch/m68k/include/asm/mcfsim.h b/arch/m68k/include/asm/mcfsim.h
index ebd0304054ad..7a83e619e73b 100644
--- a/arch/m68k/include/asm/mcfsim.h
+++ b/arch/m68k/include/asm/mcfsim.h
@@ -27,6 +27,9 @@
27#elif defined(CONFIG_M5249) 27#elif defined(CONFIG_M5249)
28#include <asm/m5249sim.h> 28#include <asm/m5249sim.h>
29#include <asm/mcfintc.h> 29#include <asm/mcfintc.h>
30#elif defined(CONFIG_M525x)
31#include <asm/m525xsim.h>
32#include <asm/mcfintc.h>
30#elif defined(CONFIG_M527x) 33#elif defined(CONFIG_M527x)
31#include <asm/m527xsim.h> 34#include <asm/m527xsim.h>
32#elif defined(CONFIG_M5272) 35#elif defined(CONFIG_M5272)
@@ -43,6 +46,8 @@
43#include <asm/mcfintc.h> 46#include <asm/mcfintc.h>
44#elif defined(CONFIG_M54xx) 47#elif defined(CONFIG_M54xx)
45#include <asm/m54xxsim.h> 48#include <asm/m54xxsim.h>
49#elif defined(CONFIG_M5441x)
50#include <asm/m5441xsim.h>
46#endif 51#endif
47 52
48/****************************************************************************/ 53/****************************************************************************/
diff --git a/arch/m68k/include/asm/mcftimer.h b/arch/m68k/include/asm/mcftimer.h
index 351c27237874..da2fa43c2e45 100644
--- a/arch/m68k/include/asm/mcftimer.h
+++ b/arch/m68k/include/asm/mcftimer.h
@@ -19,7 +19,7 @@
19#define MCFTIMER_TRR 0x04 /* Timer Reference (r/w) */ 19#define MCFTIMER_TRR 0x04 /* Timer Reference (r/w) */
20#define MCFTIMER_TCR 0x08 /* Timer Capture reg (r/w) */ 20#define MCFTIMER_TCR 0x08 /* Timer Capture reg (r/w) */
21#define MCFTIMER_TCN 0x0C /* Timer Counter reg (r/w) */ 21#define MCFTIMER_TCN 0x0C /* Timer Counter reg (r/w) */
22#if defined(CONFIG_M532x) 22#if defined(CONFIG_M532x) || defined(CONFIG_M5441x)
23#define MCFTIMER_TER 0x03 /* Timer Event reg (r/w) */ 23#define MCFTIMER_TER 0x03 /* Timer Event reg (r/w) */
24#else 24#else
25#define MCFTIMER_TER 0x11 /* Timer Event reg (r/w) */ 25#define MCFTIMER_TER 0x11 /* Timer Event reg (r/w) */
diff --git a/arch/m68k/include/asm/mcfuart.h b/arch/m68k/include/asm/mcfuart.h
index 2d3bc774b3c5..b40c20f66647 100644
--- a/arch/m68k/include/asm/mcfuart.h
+++ b/arch/m68k/include/asm/mcfuart.h
@@ -43,8 +43,8 @@ struct mcf_platform_uart {
43#define MCFUART_UFPD 0x30 /* Frac Prec. Divider (r/w) */ 43#define MCFUART_UFPD 0x30 /* Frac Prec. Divider (r/w) */
44#endif 44#endif
45#if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \ 45#if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \
46 defined(CONFIG_M5249) || defined(CONFIG_M5307) || \ 46 defined(CONFIG_M5249) || defined(CONFIG_M525x) || \
47 defined(CONFIG_M5407) 47 defined(CONFIG_M5307) || defined(CONFIG_M5407)
48#define MCFUART_UIVR 0x30 /* Interrupt Vector (r/w) */ 48#define MCFUART_UIVR 0x30 /* Interrupt Vector (r/w) */
49#endif 49#endif
50#define MCFUART_UIPR 0x34 /* Input Port (r) */ 50#define MCFUART_UIPR 0x34 /* Input Port (r) */
diff --git a/arch/m68k/include/asm/pci.h b/arch/m68k/include/asm/pci.h
index 4ad0aea48ab4..848c3dfaad50 100644
--- a/arch/m68k/include/asm/pci.h
+++ b/arch/m68k/include/asm/pci.h
@@ -2,6 +2,7 @@
2#define _ASM_M68K_PCI_H 2#define _ASM_M68K_PCI_H
3 3
4#include <asm-generic/pci-dma-compat.h> 4#include <asm-generic/pci-dma-compat.h>
5#include <asm-generic/pci.h>
5 6
6/* The PCI address space does equal the physical memory 7/* The PCI address space does equal the physical memory
7 * address space. The networking and block device layers use 8 * address space. The networking and block device layers use
@@ -9,4 +10,9 @@
9 */ 10 */
10#define PCI_DMA_BUS_IS_PHYS (1) 11#define PCI_DMA_BUS_IS_PHYS (1)
11 12
13#define pcibios_assign_all_busses() 1
14
15#define PCIBIOS_MIN_IO 0x00000100
16#define PCIBIOS_MIN_MEM 0x02000000
17
12#endif /* _ASM_M68K_PCI_H */ 18#endif /* _ASM_M68K_PCI_H */
diff --git a/arch/m68k/include/asm/pinmux.h b/arch/m68k/include/asm/pinmux.h
deleted file mode 100644
index 119ee686dbd1..000000000000
--- a/arch/m68k/include/asm/pinmux.h
+++ /dev/null
@@ -1,30 +0,0 @@
1/*
2 * Coldfire generic GPIO pinmux support.
3 *
4 * (C) Copyright 2009, Steven King <sfking@fdwdc.com>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; version 2 of the License.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 */
15
16#ifndef pinmux_h
17#define pinmux_h
18
19#define MCFPINMUX_NONE -1
20
21extern int mcf_pinmux_request(unsigned, unsigned);
22extern void mcf_pinmux_release(unsigned, unsigned);
23
24static inline int mcf_pinmux_is_valid(unsigned pinmux)
25{
26 return pinmux != MCFPINMUX_NONE;
27}
28
29#endif
30
diff --git a/arch/m68k/kernel/Makefile b/arch/m68k/kernel/Makefile
index 5c7070e21eb7..068ad49210d6 100644
--- a/arch/m68k/kernel/Makefile
+++ b/arch/m68k/kernel/Makefile
@@ -18,6 +18,7 @@ obj-y += setup.o signal.o sys_m68k.o syscalltable.o time.o traps.o
18 18
19obj-$(CONFIG_MMU_MOTOROLA) += ints.o vectors.o 19obj-$(CONFIG_MMU_MOTOROLA) += ints.o vectors.o
20obj-$(CONFIG_MMU_SUN3) += ints.o vectors.o 20obj-$(CONFIG_MMU_SUN3) += ints.o vectors.o
21obj-$(CONFIG_PCI) += pcibios.o
21 22
22ifndef CONFIG_MMU_SUN3 23ifndef CONFIG_MMU_SUN3
23obj-y += dma.o 24obj-y += dma.o
diff --git a/arch/m68k/kernel/dma.c b/arch/m68k/kernel/dma.c
index f6daf6e15d2e..e546a5534dd4 100644
--- a/arch/m68k/kernel/dma.c
+++ b/arch/m68k/kernel/dma.c
@@ -16,7 +16,7 @@
16 16
17#include <asm/pgalloc.h> 17#include <asm/pgalloc.h>
18 18
19#ifdef CONFIG_MMU 19#if defined(CONFIG_MMU) && !defined(CONFIG_COLDFIRE)
20 20
21void *dma_alloc_coherent(struct device *dev, size_t size, 21void *dma_alloc_coherent(struct device *dev, size_t size,
22 dma_addr_t *handle, gfp_t flag) 22 dma_addr_t *handle, gfp_t flag)
@@ -96,7 +96,7 @@ void dma_free_coherent(struct device *dev, size_t size,
96 free_pages((unsigned long)vaddr, get_order(size)); 96 free_pages((unsigned long)vaddr, get_order(size));
97} 97}
98 98
99#endif /* CONFIG_MMU */ 99#endif /* CONFIG_MMU && !CONFIG_COLDFIRE */
100 100
101EXPORT_SYMBOL(dma_alloc_coherent); 101EXPORT_SYMBOL(dma_alloc_coherent);
102EXPORT_SYMBOL(dma_free_coherent); 102EXPORT_SYMBOL(dma_free_coherent);
@@ -105,6 +105,7 @@ void dma_sync_single_for_device(struct device *dev, dma_addr_t handle,
105 size_t size, enum dma_data_direction dir) 105 size_t size, enum dma_data_direction dir)
106{ 106{
107 switch (dir) { 107 switch (dir) {
108 case DMA_BIDIRECTIONAL:
108 case DMA_TO_DEVICE: 109 case DMA_TO_DEVICE:
109 cache_push(handle, size); 110 cache_push(handle, size);
110 break; 111 break;
diff --git a/arch/m68k/kernel/entry.S b/arch/m68k/kernel/entry.S
index b8daf64e347d..165ee9f9d5c9 100644
--- a/arch/m68k/kernel/entry.S
+++ b/arch/m68k/kernel/entry.S
@@ -1,5 +1,451 @@
1#if defined(CONFIG_MMU) && !defined(CONFIG_COLDFIRE) 1/* -*- mode: asm -*-
2#include "entry_mm.S" 2 *
3 * linux/arch/m68k/kernel/entry.S
4 *
5 * Copyright (C) 1991, 1992 Linus Torvalds
6 *
7 * This file is subject to the terms and conditions of the GNU General Public
8 * License. See the file README.legal in the main directory of this archive
9 * for more details.
10 *
11 * Linux/m68k support by Hamish Macdonald
12 *
13 * 68060 fixes by Jesper Skov
14 *
15 */
16
17/*
18 * entry.S contains the system-call and fault low-level handling routines.
19 * This also contains the timer-interrupt handler, as well as all interrupts
20 * and faults that can result in a task-switch.
21 *
22 * NOTE: This code handles signal-recognition, which happens every time
23 * after a timer-interrupt and after each system call.
24 *
25 */
26
27/*
28 * 12/03/96 Jes: Currently we only support m68k single-cpu systems, so
29 * all pointers that used to be 'current' are now entry
30 * number 0 in the 'current_set' list.
31 *
32 * 6/05/00 RZ: addedd writeback completion after return from sighandler
33 * for 68040
34 */
35
36#include <linux/linkage.h>
37#include <asm/errno.h>
38#include <asm/setup.h>
39#include <asm/segment.h>
40#include <asm/traps.h>
41#include <asm/unistd.h>
42#include <asm/asm-offsets.h>
43#include <asm/entry.h>
44
45.globl system_call, buserr, trap, resume
46.globl sys_call_table
47.globl sys_fork, sys_clone, sys_vfork
48.globl ret_from_interrupt, bad_interrupt
49.globl auto_irqhandler_fixup
50.globl user_irqvec_fixup
51
52.text
53ENTRY(sys_fork)
54 SAVE_SWITCH_STACK
55 pea %sp@(SWITCH_STACK_SIZE)
56 jbsr m68k_fork
57 addql #4,%sp
58 RESTORE_SWITCH_STACK
59 rts
60
61ENTRY(sys_clone)
62 SAVE_SWITCH_STACK
63 pea %sp@(SWITCH_STACK_SIZE)
64 jbsr m68k_clone
65 addql #4,%sp
66 RESTORE_SWITCH_STACK
67 rts
68
69ENTRY(sys_vfork)
70 SAVE_SWITCH_STACK
71 pea %sp@(SWITCH_STACK_SIZE)
72 jbsr m68k_vfork
73 addql #4,%sp
74 RESTORE_SWITCH_STACK
75 rts
76
77ENTRY(sys_sigreturn)
78 SAVE_SWITCH_STACK
79 jbsr do_sigreturn
80 RESTORE_SWITCH_STACK
81 rts
82
83ENTRY(sys_rt_sigreturn)
84 SAVE_SWITCH_STACK
85 jbsr do_rt_sigreturn
86 RESTORE_SWITCH_STACK
87 rts
88
89ENTRY(buserr)
90 SAVE_ALL_INT
91 GET_CURRENT(%d0)
92 movel %sp,%sp@- | stack frame pointer argument
93 jbsr buserr_c
94 addql #4,%sp
95 jra ret_from_exception
96
97ENTRY(trap)
98 SAVE_ALL_INT
99 GET_CURRENT(%d0)
100 movel %sp,%sp@- | stack frame pointer argument
101 jbsr trap_c
102 addql #4,%sp
103 jra ret_from_exception
104
105 | After a fork we jump here directly from resume,
106 | so that %d1 contains the previous task
107 | schedule_tail now used regardless of CONFIG_SMP
108ENTRY(ret_from_fork)
109 movel %d1,%sp@-
110 jsr schedule_tail
111 addql #4,%sp
112 jra ret_from_exception
113
114#if defined(CONFIG_COLDFIRE) || !defined(CONFIG_MMU)
115
116#ifdef TRAP_DBG_INTERRUPT
117
118.globl dbginterrupt
119ENTRY(dbginterrupt)
120 SAVE_ALL_INT
121 GET_CURRENT(%d0)
122 movel %sp,%sp@- /* stack frame pointer argument */
123 jsr dbginterrupt_c
124 addql #4,%sp
125 jra ret_from_exception
126#endif
127
128ENTRY(reschedule)
129 /* save top of frame */
130 pea %sp@
131 jbsr set_esp0
132 addql #4,%sp
133 pea ret_from_exception
134 jmp schedule
135
136ENTRY(ret_from_user_signal)
137 moveq #__NR_sigreturn,%d0
138 trap #0
139
140ENTRY(ret_from_user_rt_signal)
141 movel #__NR_rt_sigreturn,%d0
142 trap #0
143
3#else 144#else
4#include "entry_no.S" 145
146do_trace_entry:
147 movel #-ENOSYS,%sp@(PT_OFF_D0)| needed for strace
148 subql #4,%sp
149 SAVE_SWITCH_STACK
150 jbsr syscall_trace
151 RESTORE_SWITCH_STACK
152 addql #4,%sp
153 movel %sp@(PT_OFF_ORIG_D0),%d0
154 cmpl #NR_syscalls,%d0
155 jcs syscall
156badsys:
157 movel #-ENOSYS,%sp@(PT_OFF_D0)
158 jra ret_from_syscall
159
160do_trace_exit:
161 subql #4,%sp
162 SAVE_SWITCH_STACK
163 jbsr syscall_trace
164 RESTORE_SWITCH_STACK
165 addql #4,%sp
166 jra .Lret_from_exception
167
168ENTRY(ret_from_signal)
169 movel %curptr@(TASK_STACK),%a1
170 tstb %a1@(TINFO_FLAGS+2)
171 jge 1f
172 jbsr syscall_trace
1731: RESTORE_SWITCH_STACK
174 addql #4,%sp
175/* on 68040 complete pending writebacks if any */
176#ifdef CONFIG_M68040
177 bfextu %sp@(PT_OFF_FORMATVEC){#0,#4},%d0
178 subql #7,%d0 | bus error frame ?
179 jbne 1f
180 movel %sp,%sp@-
181 jbsr berr_040cleanup
182 addql #4,%sp
1831:
184#endif
185 jra .Lret_from_exception
186
187ENTRY(system_call)
188 SAVE_ALL_SYS
189
190 GET_CURRENT(%d1)
191 movel %d1,%a1
192
193 | save top of frame
194 movel %sp,%curptr@(TASK_THREAD+THREAD_ESP0)
195
196 | syscall trace?
197 tstb %a1@(TINFO_FLAGS+2)
198 jmi do_trace_entry
199 cmpl #NR_syscalls,%d0
200 jcc badsys
201syscall:
202 jbsr @(sys_call_table,%d0:l:4)@(0)
203 movel %d0,%sp@(PT_OFF_D0) | save the return value
204ret_from_syscall:
205 |oriw #0x0700,%sr
206 movel %curptr@(TASK_STACK),%a1
207 movew %a1@(TINFO_FLAGS+2),%d0
208 jne syscall_exit_work
2091: RESTORE_ALL
210
211syscall_exit_work:
212 btst #5,%sp@(PT_OFF_SR) | check if returning to kernel
213 bnes 1b | if so, skip resched, signals
214 lslw #1,%d0
215 jcs do_trace_exit
216 jmi do_delayed_trace
217 lslw #8,%d0
218 jne do_signal_return
219 pea resume_userspace
220 jra schedule
221
222
223ENTRY(ret_from_exception)
224.Lret_from_exception:
225 btst #5,%sp@(PT_OFF_SR) | check if returning to kernel
226 bnes 1f | if so, skip resched, signals
227 | only allow interrupts when we are really the last one on the
228 | kernel stack, otherwise stack overflow can occur during
229 | heavy interrupt load
230 andw #ALLOWINT,%sr
231
232resume_userspace:
233 movel %curptr@(TASK_STACK),%a1
234 moveb %a1@(TINFO_FLAGS+3),%d0
235 jne exit_work
2361: RESTORE_ALL
237
238exit_work:
239 | save top of frame
240 movel %sp,%curptr@(TASK_THREAD+THREAD_ESP0)
241 lslb #1,%d0
242 jne do_signal_return
243 pea resume_userspace
244 jra schedule
245
246
247do_signal_return:
248 |andw #ALLOWINT,%sr
249 subql #4,%sp | dummy return address
250 SAVE_SWITCH_STACK
251 pea %sp@(SWITCH_STACK_SIZE)
252 bsrl do_notify_resume
253 addql #4,%sp
254 RESTORE_SWITCH_STACK
255 addql #4,%sp
256 jbra resume_userspace
257
258do_delayed_trace:
259 bclr #7,%sp@(PT_OFF_SR) | clear trace bit in SR
260 pea 1 | send SIGTRAP
261 movel %curptr,%sp@-
262 pea LSIGTRAP
263 jbsr send_sig
264 addql #8,%sp
265 addql #4,%sp
266 jbra resume_userspace
267
268
269/* This is the main interrupt handler for autovector interrupts */
270
271ENTRY(auto_inthandler)
272 SAVE_ALL_INT
273 GET_CURRENT(%d0)
274 movel %d0,%a1
275 addqb #1,%a1@(TINFO_PREEMPT+1)
276 | put exception # in d0
277 bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0
278 subw #VEC_SPUR,%d0
279
280 movel %sp,%sp@-
281 movel %d0,%sp@- | put vector # on stack
282auto_irqhandler_fixup = . + 2
283 jsr do_IRQ | process the IRQ
284 addql #8,%sp | pop parameters off stack
285
286ret_from_interrupt:
287 movel %curptr@(TASK_STACK),%a1
288 subqb #1,%a1@(TINFO_PREEMPT+1)
289 jeq ret_from_last_interrupt
2902: RESTORE_ALL
291
292 ALIGN
293ret_from_last_interrupt:
294 moveq #(~ALLOWINT>>8)&0xff,%d0
295 andb %sp@(PT_OFF_SR),%d0
296 jne 2b
297
298 /* check if we need to do software interrupts */
299 tstl irq_stat+CPUSTAT_SOFTIRQ_PENDING
300 jeq .Lret_from_exception
301 pea ret_from_exception
302 jra do_softirq
303
304/* Handler for user defined interrupt vectors */
305
306ENTRY(user_inthandler)
307 SAVE_ALL_INT
308 GET_CURRENT(%d0)
309 movel %d0,%a1
310 addqb #1,%a1@(TINFO_PREEMPT+1)
311 | put exception # in d0
312 bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0
313user_irqvec_fixup = . + 2
314 subw #VEC_USER,%d0
315
316 movel %sp,%sp@-
317 movel %d0,%sp@- | put vector # on stack
318 jsr do_IRQ | process the IRQ
319 addql #8,%sp | pop parameters off stack
320
321 movel %curptr@(TASK_STACK),%a1
322 subqb #1,%a1@(TINFO_PREEMPT+1)
323 jeq ret_from_last_interrupt
324 RESTORE_ALL
325
326/* Handler for uninitialized and spurious interrupts */
327
328ENTRY(bad_inthandler)
329 SAVE_ALL_INT
330 GET_CURRENT(%d0)
331 movel %d0,%a1
332 addqb #1,%a1@(TINFO_PREEMPT+1)
333
334 movel %sp,%sp@-
335 jsr handle_badint
336 addql #4,%sp
337
338 movel %curptr@(TASK_STACK),%a1
339 subqb #1,%a1@(TINFO_PREEMPT+1)
340 jeq ret_from_last_interrupt
341 RESTORE_ALL
342
343
344resume:
345 /*
346 * Beware - when entering resume, prev (the current task) is
347 * in a0, next (the new task) is in a1,so don't change these
348 * registers until their contents are no longer needed.
349 */
350
351 /* save sr */
352 movew %sr,%a0@(TASK_THREAD+THREAD_SR)
353
354 /* save fs (sfc,%dfc) (may be pointing to kernel memory) */
355 movec %sfc,%d0
356 movew %d0,%a0@(TASK_THREAD+THREAD_FS)
357
358 /* save usp */
359 /* it is better to use a movel here instead of a movew 8*) */
360 movec %usp,%d0
361 movel %d0,%a0@(TASK_THREAD+THREAD_USP)
362
363 /* save non-scratch registers on stack */
364 SAVE_SWITCH_STACK
365
366 /* save current kernel stack pointer */
367 movel %sp,%a0@(TASK_THREAD+THREAD_KSP)
368
369 /* save floating point context */
370#ifndef CONFIG_M68KFPU_EMU_ONLY
371#ifdef CONFIG_M68KFPU_EMU
372 tstl m68k_fputype
373 jeq 3f
374#endif
375 fsave %a0@(TASK_THREAD+THREAD_FPSTATE)
376
377#if defined(CONFIG_M68060)
378#if !defined(CPU_M68060_ONLY)
379 btst #3,m68k_cputype+3
380 beqs 1f
381#endif
382 /* The 060 FPU keeps status in bits 15-8 of the first longword */
383 tstb %a0@(TASK_THREAD+THREAD_FPSTATE+2)
384 jeq 3f
385#if !defined(CPU_M68060_ONLY)
386 jra 2f
387#endif
388#endif /* CONFIG_M68060 */
389#if !defined(CPU_M68060_ONLY)
3901: tstb %a0@(TASK_THREAD+THREAD_FPSTATE)
391 jeq 3f
392#endif
3932: fmovemx %fp0-%fp7,%a0@(TASK_THREAD+THREAD_FPREG)
394 fmoveml %fpcr/%fpsr/%fpiar,%a0@(TASK_THREAD+THREAD_FPCNTL)
3953:
396#endif /* CONFIG_M68KFPU_EMU_ONLY */
397 /* Return previous task in %d1 */
398 movel %curptr,%d1
399
400 /* switch to new task (a1 contains new task) */
401 movel %a1,%curptr
402
403 /* restore floating point context */
404#ifndef CONFIG_M68KFPU_EMU_ONLY
405#ifdef CONFIG_M68KFPU_EMU
406 tstl m68k_fputype
407 jeq 4f
408#endif
409#if defined(CONFIG_M68060)
410#if !defined(CPU_M68060_ONLY)
411 btst #3,m68k_cputype+3
412 beqs 1f
413#endif
414 /* The 060 FPU keeps status in bits 15-8 of the first longword */
415 tstb %a1@(TASK_THREAD+THREAD_FPSTATE+2)
416 jeq 3f
417#if !defined(CPU_M68060_ONLY)
418 jra 2f
419#endif
420#endif /* CONFIG_M68060 */
421#if !defined(CPU_M68060_ONLY)
4221: tstb %a1@(TASK_THREAD+THREAD_FPSTATE)
423 jeq 3f
5#endif 424#endif
4252: fmovemx %a1@(TASK_THREAD+THREAD_FPREG),%fp0-%fp7
426 fmoveml %a1@(TASK_THREAD+THREAD_FPCNTL),%fpcr/%fpsr/%fpiar
4273: frestore %a1@(TASK_THREAD+THREAD_FPSTATE)
4284:
429#endif /* CONFIG_M68KFPU_EMU_ONLY */
430
431 /* restore the kernel stack pointer */
432 movel %a1@(TASK_THREAD+THREAD_KSP),%sp
433
434 /* restore non-scratch registers */
435 RESTORE_SWITCH_STACK
436
437 /* restore user stack pointer */
438 movel %a1@(TASK_THREAD+THREAD_USP),%a0
439 movel %a0,%usp
440
441 /* restore fs (sfc,%dfc) */
442 movew %a1@(TASK_THREAD+THREAD_FS),%a0
443 movec %a0,%sfc
444 movec %a0,%dfc
445
446 /* restore status register */
447 movew %a1@(TASK_THREAD+THREAD_SR),%sr
448
449 rts
450
451#endif /* CONFIG_MMU && !CONFIG_COLDFIRE */
diff --git a/arch/m68k/kernel/entry_mm.S b/arch/m68k/kernel/entry_mm.S
deleted file mode 100644
index f29e73ca9dbb..000000000000
--- a/arch/m68k/kernel/entry_mm.S
+++ /dev/null
@@ -1,419 +0,0 @@
1/* -*- mode: asm -*-
2 *
3 * linux/arch/m68k/kernel/entry.S
4 *
5 * Copyright (C) 1991, 1992 Linus Torvalds
6 *
7 * This file is subject to the terms and conditions of the GNU General Public
8 * License. See the file README.legal in the main directory of this archive
9 * for more details.
10 *
11 * Linux/m68k support by Hamish Macdonald
12 *
13 * 68060 fixes by Jesper Skov
14 *
15 */
16
17/*
18 * entry.S contains the system-call and fault low-level handling routines.
19 * This also contains the timer-interrupt handler, as well as all interrupts
20 * and faults that can result in a task-switch.
21 *
22 * NOTE: This code handles signal-recognition, which happens every time
23 * after a timer-interrupt and after each system call.
24 *
25 */
26
27/*
28 * 12/03/96 Jes: Currently we only support m68k single-cpu systems, so
29 * all pointers that used to be 'current' are now entry
30 * number 0 in the 'current_set' list.
31 *
32 * 6/05/00 RZ: addedd writeback completion after return from sighandler
33 * for 68040
34 */
35
36#include <linux/linkage.h>
37#include <asm/entry.h>
38#include <asm/errno.h>
39#include <asm/setup.h>
40#include <asm/segment.h>
41#include <asm/traps.h>
42#include <asm/unistd.h>
43
44#include <asm/asm-offsets.h>
45
46.globl system_call, buserr, trap, resume
47.globl sys_call_table
48.globl sys_fork, sys_clone, sys_vfork
49.globl ret_from_interrupt, bad_interrupt
50.globl auto_irqhandler_fixup
51.globl user_irqvec_fixup
52
53.text
54ENTRY(buserr)
55 SAVE_ALL_INT
56 GET_CURRENT(%d0)
57 movel %sp,%sp@- | stack frame pointer argument
58 bsrl buserr_c
59 addql #4,%sp
60 jra .Lret_from_exception
61
62ENTRY(trap)
63 SAVE_ALL_INT
64 GET_CURRENT(%d0)
65 movel %sp,%sp@- | stack frame pointer argument
66 bsrl trap_c
67 addql #4,%sp
68 jra .Lret_from_exception
69
70 | After a fork we jump here directly from resume,
71 | so that %d1 contains the previous task
72 | schedule_tail now used regardless of CONFIG_SMP
73ENTRY(ret_from_fork)
74 movel %d1,%sp@-
75 jsr schedule_tail
76 addql #4,%sp
77 jra .Lret_from_exception
78
79do_trace_entry:
80 movel #-ENOSYS,%sp@(PT_OFF_D0)| needed for strace
81 subql #4,%sp
82 SAVE_SWITCH_STACK
83 jbsr syscall_trace
84 RESTORE_SWITCH_STACK
85 addql #4,%sp
86 movel %sp@(PT_OFF_ORIG_D0),%d0
87 cmpl #NR_syscalls,%d0
88 jcs syscall
89badsys:
90 movel #-ENOSYS,%sp@(PT_OFF_D0)
91 jra ret_from_syscall
92
93do_trace_exit:
94 subql #4,%sp
95 SAVE_SWITCH_STACK
96 jbsr syscall_trace
97 RESTORE_SWITCH_STACK
98 addql #4,%sp
99 jra .Lret_from_exception
100
101ENTRY(ret_from_signal)
102 movel %curptr@(TASK_STACK),%a1
103 tstb %a1@(TINFO_FLAGS+2)
104 jge 1f
105 jbsr syscall_trace
1061: RESTORE_SWITCH_STACK
107 addql #4,%sp
108/* on 68040 complete pending writebacks if any */
109#ifdef CONFIG_M68040
110 bfextu %sp@(PT_OFF_FORMATVEC){#0,#4},%d0
111 subql #7,%d0 | bus error frame ?
112 jbne 1f
113 movel %sp,%sp@-
114 jbsr berr_040cleanup
115 addql #4,%sp
1161:
117#endif
118 jra .Lret_from_exception
119
120ENTRY(system_call)
121 SAVE_ALL_SYS
122
123 GET_CURRENT(%d1)
124 movel %d1,%a1
125
126 | save top of frame
127 movel %sp,%curptr@(TASK_THREAD+THREAD_ESP0)
128
129 | syscall trace?
130 tstb %a1@(TINFO_FLAGS+2)
131 jmi do_trace_entry
132 cmpl #NR_syscalls,%d0
133 jcc badsys
134syscall:
135 jbsr @(sys_call_table,%d0:l:4)@(0)
136 movel %d0,%sp@(PT_OFF_D0) | save the return value
137ret_from_syscall:
138 |oriw #0x0700,%sr
139 movel %curptr@(TASK_STACK),%a1
140 movew %a1@(TINFO_FLAGS+2),%d0
141 jne syscall_exit_work
1421: RESTORE_ALL
143
144syscall_exit_work:
145 btst #5,%sp@(PT_OFF_SR) | check if returning to kernel
146 bnes 1b | if so, skip resched, signals
147 lslw #1,%d0
148 jcs do_trace_exit
149 jmi do_delayed_trace
150 lslw #8,%d0
151 jne do_signal_return
152 pea resume_userspace
153 jra schedule
154
155
156ENTRY(ret_from_exception)
157.Lret_from_exception:
158 btst #5,%sp@(PT_OFF_SR) | check if returning to kernel
159 bnes 1f | if so, skip resched, signals
160 | only allow interrupts when we are really the last one on the
161 | kernel stack, otherwise stack overflow can occur during
162 | heavy interrupt load
163 andw #ALLOWINT,%sr
164
165resume_userspace:
166 movel %curptr@(TASK_STACK),%a1
167 moveb %a1@(TINFO_FLAGS+3),%d0
168 jne exit_work
1691: RESTORE_ALL
170
171exit_work:
172 | save top of frame
173 movel %sp,%curptr@(TASK_THREAD+THREAD_ESP0)
174 lslb #1,%d0
175 jne do_signal_return
176 pea resume_userspace
177 jra schedule
178
179
180do_signal_return:
181 |andw #ALLOWINT,%sr
182 subql #4,%sp | dummy return address
183 SAVE_SWITCH_STACK
184 pea %sp@(SWITCH_STACK_SIZE)
185 bsrl do_notify_resume
186 addql #4,%sp
187 RESTORE_SWITCH_STACK
188 addql #4,%sp
189 jbra resume_userspace
190
191do_delayed_trace:
192 bclr #7,%sp@(PT_OFF_SR) | clear trace bit in SR
193 pea 1 | send SIGTRAP
194 movel %curptr,%sp@-
195 pea LSIGTRAP
196 jbsr send_sig
197 addql #8,%sp
198 addql #4,%sp
199 jbra resume_userspace
200
201
202/* This is the main interrupt handler for autovector interrupts */
203
204ENTRY(auto_inthandler)
205 SAVE_ALL_INT
206 GET_CURRENT(%d0)
207 movel %d0,%a1
208 addqb #1,%a1@(TINFO_PREEMPT+1)
209 | put exception # in d0
210 bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0
211 subw #VEC_SPUR,%d0
212
213 movel %sp,%sp@-
214 movel %d0,%sp@- | put vector # on stack
215auto_irqhandler_fixup = . + 2
216 jsr do_IRQ | process the IRQ
217 addql #8,%sp | pop parameters off stack
218
219ret_from_interrupt:
220 movel %curptr@(TASK_STACK),%a1
221 subqb #1,%a1@(TINFO_PREEMPT+1)
222 jeq ret_from_last_interrupt
2232: RESTORE_ALL
224
225 ALIGN
226ret_from_last_interrupt:
227 moveq #(~ALLOWINT>>8)&0xff,%d0
228 andb %sp@(PT_OFF_SR),%d0
229 jne 2b
230
231 /* check if we need to do software interrupts */
232 tstl irq_stat+CPUSTAT_SOFTIRQ_PENDING
233 jeq .Lret_from_exception
234 pea ret_from_exception
235 jra do_softirq
236
237/* Handler for user defined interrupt vectors */
238
239ENTRY(user_inthandler)
240 SAVE_ALL_INT
241 GET_CURRENT(%d0)
242 movel %d0,%a1
243 addqb #1,%a1@(TINFO_PREEMPT+1)
244 | put exception # in d0
245 bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0
246user_irqvec_fixup = . + 2
247 subw #VEC_USER,%d0
248
249 movel %sp,%sp@-
250 movel %d0,%sp@- | put vector # on stack
251 jsr do_IRQ | process the IRQ
252 addql #8,%sp | pop parameters off stack
253
254 movel %curptr@(TASK_STACK),%a1
255 subqb #1,%a1@(TINFO_PREEMPT+1)
256 jeq ret_from_last_interrupt
257 RESTORE_ALL
258
259/* Handler for uninitialized and spurious interrupts */
260
261ENTRY(bad_inthandler)
262 SAVE_ALL_INT
263 GET_CURRENT(%d0)
264 movel %d0,%a1
265 addqb #1,%a1@(TINFO_PREEMPT+1)
266
267 movel %sp,%sp@-
268 jsr handle_badint
269 addql #4,%sp
270
271 movel %curptr@(TASK_STACK),%a1
272 subqb #1,%a1@(TINFO_PREEMPT+1)
273 jeq ret_from_last_interrupt
274 RESTORE_ALL
275
276
277ENTRY(sys_fork)
278 SAVE_SWITCH_STACK
279 pea %sp@(SWITCH_STACK_SIZE)
280 jbsr m68k_fork
281 addql #4,%sp
282 RESTORE_SWITCH_STACK
283 rts
284
285ENTRY(sys_clone)
286 SAVE_SWITCH_STACK
287 pea %sp@(SWITCH_STACK_SIZE)
288 jbsr m68k_clone
289 addql #4,%sp
290 RESTORE_SWITCH_STACK
291 rts
292
293ENTRY(sys_vfork)
294 SAVE_SWITCH_STACK
295 pea %sp@(SWITCH_STACK_SIZE)
296 jbsr m68k_vfork
297 addql #4,%sp
298 RESTORE_SWITCH_STACK
299 rts
300
301ENTRY(sys_sigreturn)
302 SAVE_SWITCH_STACK
303 jbsr do_sigreturn
304 RESTORE_SWITCH_STACK
305 rts
306
307ENTRY(sys_rt_sigreturn)
308 SAVE_SWITCH_STACK
309 jbsr do_rt_sigreturn
310 RESTORE_SWITCH_STACK
311 rts
312
313resume:
314 /*
315 * Beware - when entering resume, prev (the current task) is
316 * in a0, next (the new task) is in a1,so don't change these
317 * registers until their contents are no longer needed.
318 */
319
320 /* save sr */
321 movew %sr,%a0@(TASK_THREAD+THREAD_SR)
322
323 /* save fs (sfc,%dfc) (may be pointing to kernel memory) */
324 movec %sfc,%d0
325 movew %d0,%a0@(TASK_THREAD+THREAD_FS)
326
327 /* save usp */
328 /* it is better to use a movel here instead of a movew 8*) */
329 movec %usp,%d0
330 movel %d0,%a0@(TASK_THREAD+THREAD_USP)
331
332 /* save non-scratch registers on stack */
333 SAVE_SWITCH_STACK
334
335 /* save current kernel stack pointer */
336 movel %sp,%a0@(TASK_THREAD+THREAD_KSP)
337
338 /* save floating point context */
339#ifndef CONFIG_M68KFPU_EMU_ONLY
340#ifdef CONFIG_M68KFPU_EMU
341 tstl m68k_fputype
342 jeq 3f
343#endif
344 fsave %a0@(TASK_THREAD+THREAD_FPSTATE)
345
346#if defined(CONFIG_M68060)
347#if !defined(CPU_M68060_ONLY)
348 btst #3,m68k_cputype+3
349 beqs 1f
350#endif
351 /* The 060 FPU keeps status in bits 15-8 of the first longword */
352 tstb %a0@(TASK_THREAD+THREAD_FPSTATE+2)
353 jeq 3f
354#if !defined(CPU_M68060_ONLY)
355 jra 2f
356#endif
357#endif /* CONFIG_M68060 */
358#if !defined(CPU_M68060_ONLY)
3591: tstb %a0@(TASK_THREAD+THREAD_FPSTATE)
360 jeq 3f
361#endif
3622: fmovemx %fp0-%fp7,%a0@(TASK_THREAD+THREAD_FPREG)
363 fmoveml %fpcr/%fpsr/%fpiar,%a0@(TASK_THREAD+THREAD_FPCNTL)
3643:
365#endif /* CONFIG_M68KFPU_EMU_ONLY */
366 /* Return previous task in %d1 */
367 movel %curptr,%d1
368
369 /* switch to new task (a1 contains new task) */
370 movel %a1,%curptr
371
372 /* restore floating point context */
373#ifndef CONFIG_M68KFPU_EMU_ONLY
374#ifdef CONFIG_M68KFPU_EMU
375 tstl m68k_fputype
376 jeq 4f
377#endif
378#if defined(CONFIG_M68060)
379#if !defined(CPU_M68060_ONLY)
380 btst #3,m68k_cputype+3
381 beqs 1f
382#endif
383 /* The 060 FPU keeps status in bits 15-8 of the first longword */
384 tstb %a1@(TASK_THREAD+THREAD_FPSTATE+2)
385 jeq 3f
386#if !defined(CPU_M68060_ONLY)
387 jra 2f
388#endif
389#endif /* CONFIG_M68060 */
390#if !defined(CPU_M68060_ONLY)
3911: tstb %a1@(TASK_THREAD+THREAD_FPSTATE)
392 jeq 3f
393#endif
3942: fmovemx %a1@(TASK_THREAD+THREAD_FPREG),%fp0-%fp7
395 fmoveml %a1@(TASK_THREAD+THREAD_FPCNTL),%fpcr/%fpsr/%fpiar
3963: frestore %a1@(TASK_THREAD+THREAD_FPSTATE)
3974:
398#endif /* CONFIG_M68KFPU_EMU_ONLY */
399
400 /* restore the kernel stack pointer */
401 movel %a1@(TASK_THREAD+THREAD_KSP),%sp
402
403 /* restore non-scratch registers */
404 RESTORE_SWITCH_STACK
405
406 /* restore user stack pointer */
407 movel %a1@(TASK_THREAD+THREAD_USP),%a0
408 movel %a0,%usp
409
410 /* restore fs (sfc,%dfc) */
411 movew %a1@(TASK_THREAD+THREAD_FS),%a0
412 movec %a0,%sfc
413 movec %a0,%dfc
414
415 /* restore status register */
416 movew %a1@(TASK_THREAD+THREAD_SR),%sr
417
418 rts
419
diff --git a/arch/m68k/kernel/entry_no.S b/arch/m68k/kernel/entry_no.S
deleted file mode 100644
index d80cba45589f..000000000000
--- a/arch/m68k/kernel/entry_no.S
+++ /dev/null
@@ -1,130 +0,0 @@
1/*
2 * linux/arch/m68knommu/kernel/entry.S
3 *
4 * Copyright (C) 1999-2002, Greg Ungerer (gerg@snapgear.com)
5 * Copyright (C) 1998 D. Jeff Dionne <jeff@lineo.ca>,
6 * Kenneth Albanowski <kjahds@kjahds.com>,
7 * Copyright (C) 2000 Lineo Inc. (www.lineo.com)
8 *
9 * Based on:
10 *
11 * linux/arch/m68k/kernel/entry.S
12 *
13 * Copyright (C) 1991, 1992 Linus Torvalds
14 *
15 * This file is subject to the terms and conditions of the GNU General Public
16 * License. See the file README.legal in the main directory of this archive
17 * for more details.
18 *
19 * Linux/m68k support by Hamish Macdonald
20 *
21 * 68060 fixes by Jesper Skov
22 * ColdFire support by Greg Ungerer (gerg@snapgear.com)
23 * 5307 fixes by David W. Miller
24 * linux 2.4 support David McCullough <davidm@snapgear.com>
25 */
26
27#include <linux/linkage.h>
28#include <asm/errno.h>
29#include <asm/setup.h>
30#include <asm/segment.h>
31#include <asm/asm-offsets.h>
32#include <asm/entry.h>
33#include <asm/unistd.h>
34
35.text
36
37.globl buserr
38.globl trap
39.globl ret_from_exception
40.globl ret_from_signal
41.globl sys_fork
42.globl sys_clone
43.globl sys_vfork
44
45ENTRY(buserr)
46 SAVE_ALL_INT
47 GET_CURRENT(%d0)
48 movel %sp,%sp@- /* stack frame pointer argument */
49 jsr buserr_c
50 addql #4,%sp
51 jra ret_from_exception
52
53ENTRY(trap)
54 SAVE_ALL_INT
55 GET_CURRENT(%d0)
56 movel %sp,%sp@- /* stack frame pointer argument */
57 jsr trap_c
58 addql #4,%sp
59 jra ret_from_exception
60
61#ifdef TRAP_DBG_INTERRUPT
62
63.globl dbginterrupt
64ENTRY(dbginterrupt)
65 SAVE_ALL_INT
66 GET_CURRENT(%d0)
67 movel %sp,%sp@- /* stack frame pointer argument */
68 jsr dbginterrupt_c
69 addql #4,%sp
70 jra ret_from_exception
71#endif
72
73ENTRY(reschedule)
74 /* save top of frame */
75 pea %sp@
76 jbsr set_esp0
77 addql #4,%sp
78 pea ret_from_exception
79 jmp schedule
80
81ENTRY(ret_from_fork)
82 movel %d1,%sp@-
83 jsr schedule_tail
84 addql #4,%sp
85 jra ret_from_exception
86
87ENTRY(sys_fork)
88 SAVE_SWITCH_STACK
89 pea %sp@(SWITCH_STACK_SIZE)
90 jbsr m68k_fork
91 addql #4,%sp
92 RESTORE_SWITCH_STACK
93 rts
94
95ENTRY(sys_vfork)
96 SAVE_SWITCH_STACK
97 pea %sp@(SWITCH_STACK_SIZE)
98 jbsr m68k_vfork
99 addql #4,%sp
100 RESTORE_SWITCH_STACK
101 rts
102
103ENTRY(sys_clone)
104 SAVE_SWITCH_STACK
105 pea %sp@(SWITCH_STACK_SIZE)
106 jbsr m68k_clone
107 addql #4,%sp
108 RESTORE_SWITCH_STACK
109 rts
110
111ENTRY(sys_sigreturn)
112 SAVE_SWITCH_STACK
113 jbsr do_sigreturn
114 RESTORE_SWITCH_STACK
115 rts
116
117ENTRY(sys_rt_sigreturn)
118 SAVE_SWITCH_STACK
119 jbsr do_rt_sigreturn
120 RESTORE_SWITCH_STACK
121 rts
122
123ENTRY(ret_from_user_signal)
124 moveq #__NR_sigreturn,%d0
125 trap #0
126
127ENTRY(ret_from_user_rt_signal)
128 movel #__NR_rt_sigreturn,%d0
129 trap #0
130
diff --git a/arch/m68k/kernel/pcibios.c b/arch/m68k/kernel/pcibios.c
new file mode 100644
index 000000000000..b2988aa1840b
--- /dev/null
+++ b/arch/m68k/kernel/pcibios.c
@@ -0,0 +1,109 @@
1/*
2 * pci.c -- basic PCI support code
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms of the GNU General Public License as published by the
6 * Free Software Foundation; either version 2 of the License, or (at your
7 * option) any later version.
8 *
9 * (C) Copyright 2011, Greg Ungerer <gerg@uclinux.org>
10 */
11
12#include <linux/kernel.h>
13#include <linux/types.h>
14#include <linux/mm.h>
15#include <linux/init.h>
16#include <linux/pci.h>
17
18/*
19 * From arch/i386/kernel/pci-i386.c:
20 *
21 * We need to avoid collisions with `mirrored' VGA ports
22 * and other strange ISA hardware, so we always want the
23 * addresses to be allocated in the 0x000-0x0ff region
24 * modulo 0x400.
25 *
26 * Why? Because some silly external IO cards only decode
27 * the low 10 bits of the IO address. The 0x00-0xff region
28 * is reserved for motherboard devices that decode all 16
29 * bits, so it's ok to allocate at, say, 0x2800-0x28ff,
30 * but we want to try to avoid allocating at 0x2900-0x2bff
31 * which might be mirrored at 0x0100-0x03ff..
32 */
33resource_size_t pcibios_align_resource(void *data, const struct resource *res,
34 resource_size_t size, resource_size_t align)
35{
36 resource_size_t start = res->start;
37
38 if ((res->flags & IORESOURCE_IO) && (start & 0x300))
39 start = (start + 0x3ff) & ~0x3ff;
40
41 start = (start + align - 1) & ~(align - 1);
42
43 return start;
44}
45
46/*
47 * This is taken from the ARM code for this.
48 */
49int pcibios_enable_device(struct pci_dev *dev, int mask)
50{
51 struct resource *r;
52 u16 cmd, newcmd;
53 int idx;
54
55 pci_read_config_word(dev, PCI_COMMAND, &cmd);
56 newcmd = cmd;
57
58 for (idx = 0; idx < 6; idx++) {
59 /* Only set up the requested stuff */
60 if (!(mask & (1 << idx)))
61 continue;
62
63 r = dev->resource + idx;
64 if (!r->start && r->end) {
65 pr_err(KERN_ERR "PCI: Device %s not available because of resource collisions\n",
66 pci_name(dev));
67 return -EINVAL;
68 }
69 if (r->flags & IORESOURCE_IO)
70 newcmd |= PCI_COMMAND_IO;
71 if (r->flags & IORESOURCE_MEM)
72 newcmd |= PCI_COMMAND_MEMORY;
73 }
74
75 /*
76 * Bridges (eg, cardbus bridges) need to be fully enabled
77 */
78 if ((dev->class >> 16) == PCI_BASE_CLASS_BRIDGE)
79 newcmd |= PCI_COMMAND_IO | PCI_COMMAND_MEMORY;
80
81
82 if (newcmd != cmd) {
83 pr_info("PCI: enabling device %s (0x%04x -> 0x%04x)\n",
84 pci_name(dev), cmd, newcmd);
85 pci_write_config_word(dev, PCI_COMMAND, newcmd);
86 }
87 return 0;
88}
89
90void pcibios_update_irq(struct pci_dev *dev, int irq)
91{
92 pci_write_config_byte(dev, PCI_INTERRUPT_LINE, irq);
93}
94
95void __devinit pcibios_fixup_bus(struct pci_bus *bus)
96{
97 struct pci_dev *dev;
98
99 list_for_each_entry(dev, &bus->devices, bus_list) {
100 pci_write_config_byte(dev, PCI_CACHE_LINE_SIZE, 8);
101 pci_write_config_byte(dev, PCI_LATENCY_TIMER, 32);
102 }
103}
104
105char __devinit *pcibios_setup(char *str)
106{
107 return str;
108}
109
diff --git a/arch/m68k/mm/memory.c b/arch/m68k/mm/memory.c
index 250b8b786f4f..51bc9d258ede 100644
--- a/arch/m68k/mm/memory.c
+++ b/arch/m68k/mm/memory.c
@@ -203,7 +203,7 @@ static inline void pushcl040(unsigned long paddr)
203void cache_clear (unsigned long paddr, int len) 203void cache_clear (unsigned long paddr, int len)
204{ 204{
205 if (CPU_IS_COLDFIRE) { 205 if (CPU_IS_COLDFIRE) {
206 flush_cf_bcache(0, DCACHE_MAX_ADDR); 206 clear_cf_bcache(0, DCACHE_MAX_ADDR);
207 } else if (CPU_IS_040_OR_060) { 207 } else if (CPU_IS_040_OR_060) {
208 int tmp; 208 int tmp;
209 209
diff --git a/arch/m68k/platform/coldfire/Makefile b/arch/m68k/platform/coldfire/Makefile
index 76d389d9a84e..02591a109f8c 100644
--- a/arch/m68k/platform/coldfire/Makefile
+++ b/arch/m68k/platform/coldfire/Makefile
@@ -20,6 +20,7 @@ obj-$(CONFIG_M5206e) += m5206.o timers.o intc.o reset.o
20obj-$(CONFIG_M520x) += m520x.o pit.o intc-simr.o reset.o 20obj-$(CONFIG_M520x) += m520x.o pit.o intc-simr.o reset.o
21obj-$(CONFIG_M523x) += m523x.o pit.o dma_timer.o intc-2.o reset.o 21obj-$(CONFIG_M523x) += m523x.o pit.o dma_timer.o intc-2.o reset.o
22obj-$(CONFIG_M5249) += m5249.o timers.o intc.o intc-5249.o reset.o 22obj-$(CONFIG_M5249) += m5249.o timers.o intc.o intc-5249.o reset.o
23obj-$(CONFIG_M525x) += m525x.o timers.o intc.o intc-525x.o reset.o
23obj-$(CONFIG_M527x) += m527x.o pit.o intc-2.o reset.o 24obj-$(CONFIG_M527x) += m527x.o pit.o intc-2.o reset.o
24obj-$(CONFIG_M5272) += m5272.o intc-5272.o timers.o 25obj-$(CONFIG_M5272) += m5272.o intc-5272.o timers.o
25obj-$(CONFIG_M528x) += m528x.o pit.o intc-2.o reset.o 26obj-$(CONFIG_M528x) += m528x.o pit.o intc-2.o reset.o
@@ -27,10 +28,14 @@ obj-$(CONFIG_M5307) += m5307.o timers.o intc.o reset.o
27obj-$(CONFIG_M532x) += m532x.o timers.o intc-simr.o reset.o 28obj-$(CONFIG_M532x) += m532x.o timers.o intc-simr.o reset.o
28obj-$(CONFIG_M5407) += m5407.o timers.o intc.o reset.o 29obj-$(CONFIG_M5407) += m5407.o timers.o intc.o reset.o
29obj-$(CONFIG_M54xx) += m54xx.o sltimers.o intc-2.o 30obj-$(CONFIG_M54xx) += m54xx.o sltimers.o intc-2.o
31obj-$(CONFIG_M5441x) += m5441x.o pit.o intc-simr.o reset.o
30 32
31obj-$(CONFIG_NETtel) += nettel.o 33obj-$(CONFIG_NETtel) += nettel.o
32obj-$(CONFIG_CLEOPATRA) += nettel.o 34obj-$(CONFIG_CLEOPATRA) += nettel.o
33obj-$(CONFIG_FIREBEE) += firebee.o 35obj-$(CONFIG_FIREBEE) += firebee.o
36obj-$(CONFIG_MCF8390) += mcf8390.o
34 37
35obj-y += pinmux.o gpio.o 38obj-$(CONFIG_PCI) += pci.o
39
40obj-y += gpio.o
36extra-y := head.o 41extra-y := head.o
diff --git a/arch/m68k/platform/coldfire/clk.c b/arch/m68k/platform/coldfire/clk.c
index 44da406897e5..75f9ee967ea7 100644
--- a/arch/m68k/platform/coldfire/clk.c
+++ b/arch/m68k/platform/coldfire/clk.c
@@ -10,11 +10,17 @@
10 10
11#include <linux/kernel.h> 11#include <linux/kernel.h>
12#include <linux/module.h> 12#include <linux/module.h>
13#include <linux/platform_device.h>
14#include <linux/mutex.h>
13#include <linux/clk.h> 15#include <linux/clk.h>
16#include <linux/io.h>
17#include <linux/err.h>
14#include <asm/coldfire.h> 18#include <asm/coldfire.h>
19#include <asm/mcfsim.h>
20#include <asm/mcfclk.h>
15 21
16/***************************************************************************/ 22/***************************************************************************/
17 23#ifndef MCFPM_PPMCR0
18struct clk *clk_get(struct device *dev, const char *id) 24struct clk *clk_get(struct device *dev, const char *id)
19{ 25{
20 return NULL; 26 return NULL;
@@ -42,11 +48,107 @@ unsigned long clk_get_rate(struct clk *clk)
42 return MCF_CLK; 48 return MCF_CLK;
43} 49}
44EXPORT_SYMBOL(clk_get_rate); 50EXPORT_SYMBOL(clk_get_rate);
51#else
52static DEFINE_SPINLOCK(clk_lock);
53
54struct clk *clk_get(struct device *dev, const char *id)
55{
56 const char *clk_name = dev ? dev_name(dev) : id ? id : NULL;
57 struct clk *clk;
58 unsigned i;
59
60 for (i = 0; (clk = mcf_clks[i]) != NULL; ++i)
61 if (!strcmp(clk->name, clk_name))
62 return clk;
63 pr_warn("clk_get: didn't find clock %s\n", clk_name);
64 return ERR_PTR(-ENOENT);
65}
66EXPORT_SYMBOL(clk_get);
67
68int clk_enable(struct clk *clk)
69{
70 unsigned long flags;
71 spin_lock_irqsave(&clk_lock, flags);
72 if ((clk->enabled++ == 0) && clk->clk_ops)
73 clk->clk_ops->enable(clk);
74 spin_unlock_irqrestore(&clk_lock, flags);
75
76 return 0;
77}
78EXPORT_SYMBOL(clk_enable);
79
80void clk_disable(struct clk *clk)
81{
82 unsigned long flags;
83 spin_lock_irqsave(&clk_lock, flags);
84 if ((--clk->enabled == 0) && clk->clk_ops)
85 clk->clk_ops->disable(clk);
86 spin_unlock_irqrestore(&clk_lock, flags);
87}
88EXPORT_SYMBOL(clk_disable);
89
90void clk_put(struct clk *clk)
91{
92 if (clk->enabled != 0)
93 pr_warn("clk_put %s still enabled\n", clk->name);
94}
95EXPORT_SYMBOL(clk_put);
96
97unsigned long clk_get_rate(struct clk *clk)
98{
99 return clk->rate;
100}
101EXPORT_SYMBOL(clk_get_rate);
102
103/***************************************************************************/
104
105void __clk_init_enabled(struct clk *clk)
106{
107 clk->enabled = 1;
108 clk->clk_ops->enable(clk);
109}
110
111void __clk_init_disabled(struct clk *clk)
112{
113 clk->enabled = 0;
114 clk->clk_ops->disable(clk);
115}
116
117static void __clk_enable0(struct clk *clk)
118{
119 __raw_writeb(clk->slot, MCFPM_PPMCR0);
120}
121
122static void __clk_disable0(struct clk *clk)
123{
124 __raw_writeb(clk->slot, MCFPM_PPMSR0);
125}
126
127struct clk_ops clk_ops0 = {
128 .enable = __clk_enable0,
129 .disable = __clk_disable0,
130};
131
132#ifdef MCFPM_PPMCR1
133static void __clk_enable1(struct clk *clk)
134{
135 __raw_writeb(clk->slot, MCFPM_PPMCR1);
136}
137
138static void __clk_disable1(struct clk *clk)
139{
140 __raw_writeb(clk->slot, MCFPM_PPMSR1);
141}
142
143struct clk_ops clk_ops1 = {
144 .enable = __clk_enable1,
145 .disable = __clk_disable1,
146};
147#endif /* MCFPM_PPMCR1 */
148#endif /* MCFPM_PPMCR0 */
45 149
46struct clk *devm_clk_get(struct device *dev, const char *id) 150struct clk *devm_clk_get(struct device *dev, const char *id)
47{ 151{
48 return NULL; 152 return NULL;
49} 153}
50EXPORT_SYMBOL(devm_clk_get); 154EXPORT_SYMBOL(devm_clk_get);
51
52/***************************************************************************/
diff --git a/arch/m68k/platform/coldfire/device.c b/arch/m68k/platform/coldfire/device.c
index 3aa77ddea89d..81f0fb5e51cf 100644
--- a/arch/m68k/platform/coldfire/device.c
+++ b/arch/m68k/platform/coldfire/device.c
@@ -13,6 +13,7 @@
13#include <linux/io.h> 13#include <linux/io.h>
14#include <linux/spi/spi.h> 14#include <linux/spi/spi.h>
15#include <linux/gpio.h> 15#include <linux/gpio.h>
16#include <linux/fec.h>
16#include <asm/traps.h> 17#include <asm/traps.h>
17#include <asm/coldfire.h> 18#include <asm/coldfire.h>
18#include <asm/mcfsim.h> 19#include <asm/mcfsim.h>
@@ -20,7 +21,7 @@
20#include <asm/mcfqspi.h> 21#include <asm/mcfqspi.h>
21 22
22/* 23/*
23 * All current ColdFire parts contain from 2, 3 or 4 UARTS. 24 * All current ColdFire parts contain from 2, 3, 4 or 10 UARTS.
24 */ 25 */
25static struct mcf_platform_uart mcf_uart_platform_data[] = { 26static struct mcf_platform_uart mcf_uart_platform_data[] = {
26 { 27 {
@@ -43,6 +44,42 @@ static struct mcf_platform_uart mcf_uart_platform_data[] = {
43 .irq = MCF_IRQ_UART3, 44 .irq = MCF_IRQ_UART3,
44 }, 45 },
45#endif 46#endif
47#ifdef MCFUART_BASE4
48 {
49 .mapbase = MCFUART_BASE4,
50 .irq = MCF_IRQ_UART4,
51 },
52#endif
53#ifdef MCFUART_BASE5
54 {
55 .mapbase = MCFUART_BASE5,
56 .irq = MCF_IRQ_UART5,
57 },
58#endif
59#ifdef MCFUART_BASE6
60 {
61 .mapbase = MCFUART_BASE6,
62 .irq = MCF_IRQ_UART6,
63 },
64#endif
65#ifdef MCFUART_BASE7
66 {
67 .mapbase = MCFUART_BASE7,
68 .irq = MCF_IRQ_UART7,
69 },
70#endif
71#ifdef MCFUART_BASE8
72 {
73 .mapbase = MCFUART_BASE8,
74 .irq = MCF_IRQ_UART8,
75 },
76#endif
77#ifdef MCFUART_BASE9
78 {
79 .mapbase = MCFUART_BASE9,
80 .irq = MCF_IRQ_UART9,
81 },
82#endif
46 { }, 83 { },
47}; 84};
48 85
@@ -53,6 +90,18 @@ static struct platform_device mcf_uart = {
53}; 90};
54 91
55#ifdef CONFIG_FEC 92#ifdef CONFIG_FEC
93
94#ifdef CONFIG_M5441x
95#define FEC_NAME "enet-fec"
96static struct fec_platform_data fec_pdata = {
97 .phy = PHY_INTERFACE_MODE_RMII,
98};
99#define FEC_PDATA (&fec_pdata)
100#else
101#define FEC_NAME "fec"
102#define FEC_PDATA NULL
103#endif
104
56/* 105/*
57 * Some ColdFire cores contain the Fast Ethernet Controller (FEC) 106 * Some ColdFire cores contain the Fast Ethernet Controller (FEC)
58 * block. It is Freescale's own hardware block. Some ColdFires 107 * block. It is Freescale's own hardware block. Some ColdFires
@@ -82,10 +131,11 @@ static struct resource mcf_fec0_resources[] = {
82}; 131};
83 132
84static struct platform_device mcf_fec0 = { 133static struct platform_device mcf_fec0 = {
85 .name = "fec", 134 .name = FEC_NAME,
86 .id = 0, 135 .id = 0,
87 .num_resources = ARRAY_SIZE(mcf_fec0_resources), 136 .num_resources = ARRAY_SIZE(mcf_fec0_resources),
88 .resource = mcf_fec0_resources, 137 .resource = mcf_fec0_resources,
138 .dev.platform_data = FEC_PDATA,
89}; 139};
90 140
91#ifdef MCFFEC_BASE1 141#ifdef MCFFEC_BASE1
@@ -113,10 +163,11 @@ static struct resource mcf_fec1_resources[] = {
113}; 163};
114 164
115static struct platform_device mcf_fec1 = { 165static struct platform_device mcf_fec1 = {
116 .name = "fec", 166 .name = FEC_NAME,
117 .id = 1, 167 .id = 1,
118 .num_resources = ARRAY_SIZE(mcf_fec1_resources), 168 .num_resources = ARRAY_SIZE(mcf_fec1_resources),
119 .resource = mcf_fec1_resources, 169 .resource = mcf_fec1_resources,
170 .dev.platform_data = FEC_PDATA,
120}; 171};
121#endif /* MCFFEC_BASE1 */ 172#endif /* MCFFEC_BASE1 */
122#endif /* CONFIG_FEC */ 173#endif /* CONFIG_FEC */
diff --git a/arch/m68k/platform/coldfire/gpio.c b/arch/m68k/platform/coldfire/gpio.c
index 4c8c42450a4e..9cd2b5c70519 100644
--- a/arch/m68k/platform/coldfire/gpio.c
+++ b/arch/m68k/platform/coldfire/gpio.c
@@ -14,119 +14,161 @@
14 */ 14 */
15 15
16#include <linux/kernel.h> 16#include <linux/kernel.h>
17#include <linux/module.h>
17#include <linux/init.h> 18#include <linux/init.h>
18#include <linux/device.h> 19#include <linux/device.h>
19 20
20#include <asm/gpio.h> 21#include <linux/io.h>
21#include <asm/pinmux.h> 22#include <asm/coldfire.h>
23#include <asm/mcfsim.h>
22#include <asm/mcfgpio.h> 24#include <asm/mcfgpio.h>
23 25
24#define MCF_CHIP(chip) container_of(chip, struct mcf_gpio_chip, gpio_chip) 26int __mcfgpio_get_value(unsigned gpio)
27{
28 return mcfgpio_read(__mcfgpio_ppdr(gpio)) & mcfgpio_bit(gpio);
29}
30EXPORT_SYMBOL(__mcfgpio_get_value);
31
32void __mcfgpio_set_value(unsigned gpio, int value)
33{
34 if (gpio < MCFGPIO_SCR_START) {
35 unsigned long flags;
36 MCFGPIO_PORTTYPE data;
37
38 local_irq_save(flags);
39 data = mcfgpio_read(__mcfgpio_podr(gpio));
40 if (value)
41 data |= mcfgpio_bit(gpio);
42 else
43 data &= ~mcfgpio_bit(gpio);
44 mcfgpio_write(data, __mcfgpio_podr(gpio));
45 local_irq_restore(flags);
46 } else {
47 if (value)
48 mcfgpio_write(mcfgpio_bit(gpio),
49 MCFGPIO_SETR_PORT(gpio));
50 else
51 mcfgpio_write(~mcfgpio_bit(gpio),
52 MCFGPIO_CLRR_PORT(gpio));
53 }
54}
55EXPORT_SYMBOL(__mcfgpio_set_value);
25 56
26int mcf_gpio_direction_input(struct gpio_chip *chip, unsigned offset) 57int __mcfgpio_direction_input(unsigned gpio)
27{ 58{
28 unsigned long flags; 59 unsigned long flags;
29 MCFGPIO_PORTTYPE dir; 60 MCFGPIO_PORTTYPE dir;
30 struct mcf_gpio_chip *mcf_chip = MCF_CHIP(chip);
31 61
32 local_irq_save(flags); 62 local_irq_save(flags);
33 dir = mcfgpio_read(mcf_chip->pddr); 63 dir = mcfgpio_read(__mcfgpio_pddr(gpio));
34 dir &= ~mcfgpio_bit(chip->base + offset); 64 dir &= ~mcfgpio_bit(gpio);
35 mcfgpio_write(dir, mcf_chip->pddr); 65 mcfgpio_write(dir, __mcfgpio_pddr(gpio));
36 local_irq_restore(flags); 66 local_irq_restore(flags);
37 67
38 return 0; 68 return 0;
39} 69}
70EXPORT_SYMBOL(__mcfgpio_direction_input);
40 71
41int mcf_gpio_get_value(struct gpio_chip *chip, unsigned offset) 72int __mcfgpio_direction_output(unsigned gpio, int value)
42{
43 struct mcf_gpio_chip *mcf_chip = MCF_CHIP(chip);
44
45 return mcfgpio_read(mcf_chip->ppdr) & mcfgpio_bit(chip->base + offset);
46}
47
48int mcf_gpio_direction_output(struct gpio_chip *chip, unsigned offset,
49 int value)
50{ 73{
51 unsigned long flags; 74 unsigned long flags;
52 MCFGPIO_PORTTYPE data; 75 MCFGPIO_PORTTYPE data;
53 struct mcf_gpio_chip *mcf_chip = MCF_CHIP(chip);
54 76
55 local_irq_save(flags); 77 local_irq_save(flags);
56 /* write the value to the output latch */ 78 data = mcfgpio_read(__mcfgpio_pddr(gpio));
57 data = mcfgpio_read(mcf_chip->podr);
58 if (value) 79 if (value)
59 data |= mcfgpio_bit(chip->base + offset); 80 data |= mcfgpio_bit(gpio);
60 else 81 else
61 data &= ~mcfgpio_bit(chip->base + offset); 82 data &= mcfgpio_bit(gpio);
62 mcfgpio_write(data, mcf_chip->podr); 83 mcfgpio_write(data, __mcfgpio_pddr(gpio));
63 84
64 /* now set the direction to output */ 85 /* now set the data to output */
65 data = mcfgpio_read(mcf_chip->pddr); 86 if (gpio < MCFGPIO_SCR_START) {
66 data |= mcfgpio_bit(chip->base + offset); 87 data = mcfgpio_read(__mcfgpio_podr(gpio));
67 mcfgpio_write(data, mcf_chip->pddr); 88 if (value)
89 data |= mcfgpio_bit(gpio);
90 else
91 data &= ~mcfgpio_bit(gpio);
92 mcfgpio_write(data, __mcfgpio_podr(gpio));
93 } else {
94 if (value)
95 mcfgpio_write(mcfgpio_bit(gpio),
96 MCFGPIO_SETR_PORT(gpio));
97 else
98 mcfgpio_write(~mcfgpio_bit(gpio),
99 MCFGPIO_CLRR_PORT(gpio));
100 }
68 local_irq_restore(flags); 101 local_irq_restore(flags);
102 return 0;
103}
104EXPORT_SYMBOL(__mcfgpio_direction_output);
69 105
106int __mcfgpio_request(unsigned gpio)
107{
70 return 0; 108 return 0;
71} 109}
110EXPORT_SYMBOL(__mcfgpio_request);
72 111
73void mcf_gpio_set_value(struct gpio_chip *chip, unsigned offset, int value) 112void __mcfgpio_free(unsigned gpio)
74{ 113{
75 struct mcf_gpio_chip *mcf_chip = MCF_CHIP(chip); 114 __mcfgpio_direction_input(gpio);
115}
116EXPORT_SYMBOL(__mcfgpio_free);
76 117
77 unsigned long flags; 118#ifdef CONFIG_GPIOLIB
78 MCFGPIO_PORTTYPE data;
79 119
80 local_irq_save(flags); 120int mcfgpio_direction_input(struct gpio_chip *chip, unsigned offset)
81 data = mcfgpio_read(mcf_chip->podr); 121{
82 if (value) 122 return __mcfgpio_direction_input(offset);
83 data |= mcfgpio_bit(chip->base + offset);
84 else
85 data &= ~mcfgpio_bit(chip->base + offset);
86 mcfgpio_write(data, mcf_chip->podr);
87 local_irq_restore(flags);
88} 123}
89 124
90void mcf_gpio_set_value_fast(struct gpio_chip *chip, unsigned offset, int value) 125int mcfgpio_get_value(struct gpio_chip *chip, unsigned offset)
91{ 126{
92 struct mcf_gpio_chip *mcf_chip = MCF_CHIP(chip); 127 return __mcfgpio_get_value(offset);
93
94 if (value)
95 mcfgpio_write(mcfgpio_bit(chip->base + offset), mcf_chip->setr);
96 else
97 mcfgpio_write(~mcfgpio_bit(chip->base + offset), mcf_chip->clrr);
98} 128}
99 129
100int mcf_gpio_request(struct gpio_chip *chip, unsigned offset) 130int mcfgpio_direction_output(struct gpio_chip *chip, unsigned offset, int value)
101{ 131{
102 struct mcf_gpio_chip *mcf_chip = MCF_CHIP(chip); 132 return __mcfgpio_direction_output(offset, value);
103
104 return mcf_chip->gpio_to_pinmux ?
105 mcf_pinmux_request(mcf_chip->gpio_to_pinmux[offset], 0) : 0;
106} 133}
107 134
108void mcf_gpio_free(struct gpio_chip *chip, unsigned offset) 135void mcfgpio_set_value(struct gpio_chip *chip, unsigned offset, int value)
109{ 136{
110 struct mcf_gpio_chip *mcf_chip = MCF_CHIP(chip); 137 __mcfgpio_set_value(offset, value);
138}
111 139
112 mcf_gpio_direction_input(chip, offset); 140int mcfgpio_request(struct gpio_chip *chip, unsigned offset)
141{
142 return __mcfgpio_request(offset);
143}
113 144
114 if (mcf_chip->gpio_to_pinmux) 145void mcfgpio_free(struct gpio_chip *chip, unsigned offset)
115 mcf_pinmux_release(mcf_chip->gpio_to_pinmux[offset], 0); 146{
147 __mcfgpio_free(offset);
116} 148}
117 149
118struct bus_type mcf_gpio_subsys = { 150struct bus_type mcfgpio_subsys = {
119 .name = "gpio", 151 .name = "gpio",
120 .dev_name = "gpio", 152 .dev_name = "gpio",
121}; 153};
122 154
123static int __init mcf_gpio_sysinit(void) 155static struct gpio_chip mcfgpio_chip = {
124{ 156 .label = "mcfgpio",
125 unsigned int i = 0; 157 .request = mcfgpio_request,
158 .free = mcfgpio_free,
159 .direction_input = mcfgpio_direction_input,
160 .direction_output = mcfgpio_direction_output,
161 .get = mcfgpio_get_value,
162 .set = mcfgpio_set_value,
163 .base = 0,
164 .ngpio = MCFGPIO_PIN_MAX,
165};
126 166
127 while (i < mcf_gpio_chips_size) 167static int __init mcfgpio_sysinit(void)
128 gpiochip_add((struct gpio_chip *)&mcf_gpio_chips[i++]); 168{
129 return subsys_system_register(&mcf_gpio_subsys, NULL); 169 gpiochip_add(&mcfgpio_chip);
170 return subsys_system_register(&mcfgpio_subsys, NULL);
130} 171}
131 172
132core_initcall(mcf_gpio_sysinit); 173core_initcall(mcfgpio_sysinit);
174#endif
diff --git a/arch/m68k/platform/coldfire/head.S b/arch/m68k/platform/coldfire/head.S
index c3db70ed33b3..4e0c9eb3bd1f 100644
--- a/arch/m68k/platform/coldfire/head.S
+++ b/arch/m68k/platform/coldfire/head.S
@@ -31,9 +31,9 @@
31.endm 31.endm
32 32
33#elif defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \ 33#elif defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \
34 defined(CONFIG_M5249) || defined(CONFIG_M527x) || \ 34 defined(CONFIG_M5249) || defined(CONFIG_M525x) || \
35 defined(CONFIG_M528x) || defined(CONFIG_M5307) || \ 35 defined(CONFIG_M527x) || defined(CONFIG_M528x) || \
36 defined(CONFIG_M5407) 36 defined(CONFIG_M5307) || defined(CONFIG_M5407)
37/* 37/*
38 * Not all these devices have exactly the same DRAM controller, 38 * Not all these devices have exactly the same DRAM controller,
39 * but the DCMR register is virtually identical - give or take 39 * but the DCMR register is virtually identical - give or take
diff --git a/arch/m68k/platform/coldfire/intc-525x.c b/arch/m68k/platform/coldfire/intc-525x.c
new file mode 100644
index 000000000000..b23204d059ac
--- /dev/null
+++ b/arch/m68k/platform/coldfire/intc-525x.c
@@ -0,0 +1,91 @@
1/*
2 * intc2.c -- support for the 2nd INTC controller of the 525x
3 *
4 * (C) Copyright 2012, Steven King <sfking@fdwdc.com>
5 * (C) Copyright 2009, Greg Ungerer <gerg@snapgear.com>
6 *
7 * This file is subject to the terms and conditions of the GNU General Public
8 * License. See the file COPYING in the main directory of this archive
9 * for more details.
10 */
11
12#include <linux/types.h>
13#include <linux/init.h>
14#include <linux/kernel.h>
15#include <linux/interrupt.h>
16#include <linux/irq.h>
17#include <linux/io.h>
18#include <asm/coldfire.h>
19#include <asm/mcfsim.h>
20
21static void intc2_irq_gpio_mask(struct irq_data *d)
22{
23 u32 imr = readl(MCFSIM2_GPIOINTENABLE);
24 u32 type = irqd_get_trigger_type(d);
25 int irq = d->irq - MCF_IRQ_GPIO0;
26
27 if (type & IRQ_TYPE_EDGE_RISING)
28 imr &= ~(0x001 << irq);
29 if (type & IRQ_TYPE_EDGE_FALLING)
30 imr &= ~(0x100 << irq);
31 writel(imr, MCFSIM2_GPIOINTENABLE);
32}
33
34static void intc2_irq_gpio_unmask(struct irq_data *d)
35{
36 u32 imr = readl(MCFSIM2_GPIOINTENABLE);
37 u32 type = irqd_get_trigger_type(d);
38 int irq = d->irq - MCF_IRQ_GPIO0;
39
40 if (type & IRQ_TYPE_EDGE_RISING)
41 imr |= (0x001 << irq);
42 if (type & IRQ_TYPE_EDGE_FALLING)
43 imr |= (0x100 << irq);
44 writel(imr, MCFSIM2_GPIOINTENABLE);
45}
46
47static void intc2_irq_gpio_ack(struct irq_data *d)
48{
49 u32 imr = 0;
50 u32 type = irqd_get_trigger_type(d);
51 int irq = d->irq - MCF_IRQ_GPIO0;
52
53 if (type & IRQ_TYPE_EDGE_RISING)
54 imr |= (0x001 << irq);
55 if (type & IRQ_TYPE_EDGE_FALLING)
56 imr |= (0x100 << irq);
57 writel(imr, MCFSIM2_GPIOINTCLEAR);
58}
59
60static int intc2_irq_gpio_set_type(struct irq_data *d, unsigned int f)
61{
62 if (f & ~IRQ_TYPE_EDGE_BOTH)
63 return -EINVAL;
64 return 0;
65}
66
67static struct irq_chip intc2_irq_gpio_chip = {
68 .name = "CF-INTC2",
69 .irq_mask = intc2_irq_gpio_mask,
70 .irq_unmask = intc2_irq_gpio_unmask,
71 .irq_ack = intc2_irq_gpio_ack,
72 .irq_set_type = intc2_irq_gpio_set_type,
73};
74
75static int __init mcf_intc2_init(void)
76{
77 int irq;
78
79 /* set the interrupt base for the second interrupt controller */
80 writel(MCFINTC2_VECBASE, MCFINTC2_INTBASE);
81
82 /* GPIO interrupt sources */
83 for (irq = MCF_IRQ_GPIO0; (irq <= MCF_IRQ_GPIO6); irq++) {
84 irq_set_chip(irq, &intc2_irq_gpio_chip);
85 irq_set_handler(irq, handle_edge_irq);
86 }
87
88 return 0;
89}
90
91arch_initcall(mcf_intc2_init);
diff --git a/arch/m68k/platform/coldfire/intc-simr.c b/arch/m68k/platform/coldfire/intc-simr.c
index 650d52e2927e..7cf2c156f72d 100644
--- a/arch/m68k/platform/coldfire/intc-simr.c
+++ b/arch/m68k/platform/coldfire/intc-simr.c
@@ -59,16 +59,18 @@ static unsigned int inline irq2ebit(unsigned int irq)
59#endif 59#endif
60 60
61/* 61/*
62 * There maybe one or two interrupt control units, each has 64 62 * There maybe one, two or three interrupt control units, each has 64
63 * interrupts. If there is no second unit then MCFINTC1_* defines 63 * interrupts. If there is no second or third unit then MCFINTC1_* or
64 * will be 0 (and code for them optimized away). 64 * MCFINTC2_* defines will be 0 (and code for them optimized away).
65 */ 65 */
66 66
67static void intc_irq_mask(struct irq_data *d) 67static void intc_irq_mask(struct irq_data *d)
68{ 68{
69 unsigned int irq = d->irq - MCFINT_VECBASE; 69 unsigned int irq = d->irq - MCFINT_VECBASE;
70 70
71 if (MCFINTC1_SIMR && (irq > 64)) 71 if (MCFINTC2_SIMR && (irq > 128))
72 __raw_writeb(irq - 128, MCFINTC2_SIMR);
73 else if (MCFINTC1_SIMR && (irq > 64))
72 __raw_writeb(irq - 64, MCFINTC1_SIMR); 74 __raw_writeb(irq - 64, MCFINTC1_SIMR);
73 else 75 else
74 __raw_writeb(irq, MCFINTC0_SIMR); 76 __raw_writeb(irq, MCFINTC0_SIMR);
@@ -78,7 +80,9 @@ static void intc_irq_unmask(struct irq_data *d)
78{ 80{
79 unsigned int irq = d->irq - MCFINT_VECBASE; 81 unsigned int irq = d->irq - MCFINT_VECBASE;
80 82
81 if (MCFINTC1_CIMR && (irq > 64)) 83 if (MCFINTC2_CIMR && (irq > 128))
84 __raw_writeb(irq - 128, MCFINTC2_CIMR);
85 else if (MCFINTC1_CIMR && (irq > 64))
82 __raw_writeb(irq - 64, MCFINTC1_CIMR); 86 __raw_writeb(irq - 64, MCFINTC1_CIMR);
83 else 87 else
84 __raw_writeb(irq, MCFINTC0_CIMR); 88 __raw_writeb(irq, MCFINTC0_CIMR);
@@ -99,9 +103,11 @@ static unsigned int intc_irq_startup(struct irq_data *d)
99 unsigned int ebit = irq2ebit(irq); 103 unsigned int ebit = irq2ebit(irq);
100 u8 v; 104 u8 v;
101 105
106#if defined(MCFEPORT_EPDDR)
102 /* Set EPORT line as input */ 107 /* Set EPORT line as input */
103 v = __raw_readb(MCFEPORT_EPDDR); 108 v = __raw_readb(MCFEPORT_EPDDR);
104 __raw_writeb(v & ~(0x1 << ebit), MCFEPORT_EPDDR); 109 __raw_writeb(v & ~(0x1 << ebit), MCFEPORT_EPDDR);
110#endif
105 111
106 /* Set EPORT line as interrupt source */ 112 /* Set EPORT line as interrupt source */
107 v = __raw_readb(MCFEPORT_EPIER); 113 v = __raw_readb(MCFEPORT_EPIER);
@@ -109,12 +115,13 @@ static unsigned int intc_irq_startup(struct irq_data *d)
109 } 115 }
110 116
111 irq -= MCFINT_VECBASE; 117 irq -= MCFINT_VECBASE;
112 if (MCFINTC1_ICR0 && (irq > 64)) 118 if (MCFINTC2_ICR0 && (irq > 128))
119 __raw_writeb(5, MCFINTC2_ICR0 + irq - 128);
120 else if (MCFINTC1_ICR0 && (irq > 64))
113 __raw_writeb(5, MCFINTC1_ICR0 + irq - 64); 121 __raw_writeb(5, MCFINTC1_ICR0 + irq - 64);
114 else 122 else
115 __raw_writeb(5, MCFINTC0_ICR0 + irq); 123 __raw_writeb(5, MCFINTC0_ICR0 + irq);
116 124
117
118 intc_irq_unmask(d); 125 intc_irq_unmask(d);
119 return 0; 126 return 0;
120} 127}
@@ -175,8 +182,11 @@ void __init init_IRQ(void)
175 __raw_writeb(0xff, MCFINTC0_SIMR); 182 __raw_writeb(0xff, MCFINTC0_SIMR);
176 if (MCFINTC1_SIMR) 183 if (MCFINTC1_SIMR)
177 __raw_writeb(0xff, MCFINTC1_SIMR); 184 __raw_writeb(0xff, MCFINTC1_SIMR);
185 if (MCFINTC2_SIMR)
186 __raw_writeb(0xff, MCFINTC2_SIMR);
178 187
179 eirq = MCFINT_VECBASE + 64 + (MCFINTC1_ICR0 ? 64 : 0); 188 eirq = MCFINT_VECBASE + 64 + (MCFINTC1_ICR0 ? 64 : 0) +
189 (MCFINTC2_ICR0 ? 64 : 0);
180 for (irq = MCFINT_VECBASE; (irq < eirq); irq++) { 190 for (irq = MCFINT_VECBASE; (irq < eirq); irq++) {
181 if ((irq >= EINT1) && (irq <= EINT7)) 191 if ((irq >= EINT1) && (irq <= EINT7))
182 irq_set_chip(irq, &intc_irq_chip_edge_port); 192 irq_set_chip(irq, &intc_irq_chip_edge_port);
diff --git a/arch/m68k/platform/coldfire/m5206.c b/arch/m68k/platform/coldfire/m5206.c
index a8b81df653f0..6bfbeebd231b 100644
--- a/arch/m68k/platform/coldfire/m5206.c
+++ b/arch/m68k/platform/coldfire/m5206.c
@@ -16,15 +16,6 @@
16#include <asm/machdep.h> 16#include <asm/machdep.h>
17#include <asm/coldfire.h> 17#include <asm/coldfire.h>
18#include <asm/mcfsim.h> 18#include <asm/mcfsim.h>
19#include <asm/mcfgpio.h>
20
21/***************************************************************************/
22
23struct mcf_gpio_chip mcf_gpio_chips[] = {
24 MCFGPS(PP, 0, 8, MCFSIM_PADDR, MCFSIM_PADAT, MCFSIM_PADAT),
25};
26
27unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips);
28 19
29/***************************************************************************/ 20/***************************************************************************/
30 21
diff --git a/arch/m68k/platform/coldfire/m520x.c b/arch/m68k/platform/coldfire/m520x.c
index 3264b8883d5f..ea1be0e98ad6 100644
--- a/arch/m68k/platform/coldfire/m520x.c
+++ b/arch/m68k/platform/coldfire/m520x.c
@@ -19,22 +19,102 @@
19#include <asm/coldfire.h> 19#include <asm/coldfire.h>
20#include <asm/mcfsim.h> 20#include <asm/mcfsim.h>
21#include <asm/mcfuart.h> 21#include <asm/mcfuart.h>
22#include <asm/mcfgpio.h> 22#include <asm/mcfclk.h>
23 23
24/***************************************************************************/ 24/***************************************************************************/
25 25
26struct mcf_gpio_chip mcf_gpio_chips[] = { 26DEFINE_CLK(0, "flexbus", 2, MCF_CLK);
27 MCFGPS(PIRQ, 0, 8, MCFEPORT_EPDDR, MCFEPORT_EPDR, MCFEPORT_EPPDR), 27DEFINE_CLK(0, "fec.0", 12, MCF_CLK);
28 MCFGPF(CS, 9, 3), 28DEFINE_CLK(0, "edma", 17, MCF_CLK);
29 MCFGPF(FECI2C, 16, 4), 29DEFINE_CLK(0, "intc.0", 18, MCF_CLK);
30 MCFGPF(QSPI, 24, 4), 30DEFINE_CLK(0, "iack.0", 21, MCF_CLK);
31 MCFGPF(TIMER, 32, 4), 31DEFINE_CLK(0, "mcfi2c.0", 22, MCF_CLK);
32 MCFGPF(UART, 40, 8), 32DEFINE_CLK(0, "mcfqspi.0", 23, MCF_CLK);
33 MCFGPF(FECH, 48, 8), 33DEFINE_CLK(0, "mcfuart.0", 24, MCF_BUSCLK);
34 MCFGPF(FECL, 56, 8), 34DEFINE_CLK(0, "mcfuart.1", 25, MCF_BUSCLK);
35DEFINE_CLK(0, "mcfuart.2", 26, MCF_BUSCLK);
36DEFINE_CLK(0, "mcftmr.0", 28, MCF_CLK);
37DEFINE_CLK(0, "mcftmr.1", 29, MCF_CLK);
38DEFINE_CLK(0, "mcftmr.2", 30, MCF_CLK);
39DEFINE_CLK(0, "mcftmr.3", 31, MCF_CLK);
40
41DEFINE_CLK(0, "mcfpit.0", 32, MCF_CLK);
42DEFINE_CLK(0, "mcfpit.1", 33, MCF_CLK);
43DEFINE_CLK(0, "mcfeport.0", 34, MCF_CLK);
44DEFINE_CLK(0, "mcfwdt.0", 35, MCF_CLK);
45DEFINE_CLK(0, "pll.0", 36, MCF_CLK);
46DEFINE_CLK(0, "sys.0", 40, MCF_BUSCLK);
47DEFINE_CLK(0, "gpio.0", 41, MCF_BUSCLK);
48DEFINE_CLK(0, "sdram.0", 42, MCF_CLK);
49
50struct clk *mcf_clks[] = {
51 &__clk_0_2, /* flexbus */
52 &__clk_0_12, /* fec.0 */
53 &__clk_0_17, /* edma */
54 &__clk_0_18, /* intc.0 */
55 &__clk_0_21, /* iack.0 */
56 &__clk_0_22, /* mcfi2c.0 */
57 &__clk_0_23, /* mcfqspi.0 */
58 &__clk_0_24, /* mcfuart.0 */
59 &__clk_0_25, /* mcfuart.1 */
60 &__clk_0_26, /* mcfuart.2 */
61 &__clk_0_28, /* mcftmr.0 */
62 &__clk_0_29, /* mcftmr.1 */
63 &__clk_0_30, /* mcftmr.2 */
64 &__clk_0_31, /* mcftmr.3 */
65
66 &__clk_0_32, /* mcfpit.0 */
67 &__clk_0_33, /* mcfpit.1 */
68 &__clk_0_34, /* mcfeport.0 */
69 &__clk_0_35, /* mcfwdt.0 */
70 &__clk_0_36, /* pll.0 */
71 &__clk_0_40, /* sys.0 */
72 &__clk_0_41, /* gpio.0 */
73 &__clk_0_42, /* sdram.0 */
74NULL,
35}; 75};
36 76
37unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips); 77static struct clk * const enable_clks[] __initconst = {
78 &__clk_0_2, /* flexbus */
79 &__clk_0_18, /* intc.0 */
80 &__clk_0_21, /* iack.0 */
81 &__clk_0_24, /* mcfuart.0 */
82 &__clk_0_25, /* mcfuart.1 */
83 &__clk_0_26, /* mcfuart.2 */
84
85 &__clk_0_32, /* mcfpit.0 */
86 &__clk_0_33, /* mcfpit.1 */
87 &__clk_0_34, /* mcfeport.0 */
88 &__clk_0_36, /* pll.0 */
89 &__clk_0_40, /* sys.0 */
90 &__clk_0_41, /* gpio.0 */
91 &__clk_0_42, /* sdram.0 */
92};
93
94static struct clk * const disable_clks[] __initconst = {
95 &__clk_0_12, /* fec.0 */
96 &__clk_0_17, /* edma */
97 &__clk_0_22, /* mcfi2c.0 */
98 &__clk_0_23, /* mcfqspi.0 */
99 &__clk_0_28, /* mcftmr.0 */
100 &__clk_0_29, /* mcftmr.1 */
101 &__clk_0_30, /* mcftmr.2 */
102 &__clk_0_31, /* mcftmr.3 */
103 &__clk_0_35, /* mcfwdt.0 */
104};
105
106
107static void __init m520x_clk_init(void)
108{
109 unsigned i;
110
111 /* make sure these clocks are enabled */
112 for (i = 0; i < ARRAY_SIZE(enable_clks); ++i)
113 __clk_init_enabled(enable_clks[i]);
114 /* make sure these clocks are disabled */
115 for (i = 0; i < ARRAY_SIZE(disable_clks); ++i)
116 __clk_init_disabled(disable_clks[i]);
117}
38 118
39/***************************************************************************/ 119/***************************************************************************/
40 120
@@ -93,6 +173,7 @@ static void __init m520x_fec_init(void)
93void __init config_BSP(char *commandp, int size) 173void __init config_BSP(char *commandp, int size)
94{ 174{
95 mach_sched_init = hw_timer_init; 175 mach_sched_init = hw_timer_init;
176 m520x_clk_init();
96 m520x_uarts_init(); 177 m520x_uarts_init();
97 m520x_fec_init(); 178 m520x_fec_init();
98#if IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI) 179#if IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI)
diff --git a/arch/m68k/platform/coldfire/m523x.c b/arch/m68k/platform/coldfire/m523x.c
index 5d57a4249412..d47dfd8f50a2 100644
--- a/arch/m68k/platform/coldfire/m523x.c
+++ b/arch/m68k/platform/coldfire/m523x.c
@@ -19,28 +19,6 @@
19#include <asm/machdep.h> 19#include <asm/machdep.h>
20#include <asm/coldfire.h> 20#include <asm/coldfire.h>
21#include <asm/mcfsim.h> 21#include <asm/mcfsim.h>
22#include <asm/mcfgpio.h>
23
24/***************************************************************************/
25
26struct mcf_gpio_chip mcf_gpio_chips[] = {
27 MCFGPS(PIRQ, 1, 7, MCFEPORT_EPDDR, MCFEPORT_EPDR, MCFEPORT_EPPDR),
28 MCFGPF(ADDR, 13, 3),
29 MCFGPF(DATAH, 16, 8),
30 MCFGPF(DATAL, 24, 8),
31 MCFGPF(BUSCTL, 32, 8),
32 MCFGPF(BS, 40, 4),
33 MCFGPF(CS, 49, 7),
34 MCFGPF(SDRAM, 56, 6),
35 MCFGPF(FECI2C, 64, 4),
36 MCFGPF(UARTH, 72, 2),
37 MCFGPF(UARTL, 80, 8),
38 MCFGPF(QSPI, 88, 5),
39 MCFGPF(TIMER, 96, 8),
40 MCFGPF(ETPU, 104, 3),
41};
42
43unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips);
44 22
45/***************************************************************************/ 23/***************************************************************************/
46 24
diff --git a/arch/m68k/platform/coldfire/m5249.c b/arch/m68k/platform/coldfire/m5249.c
index fdfa1edfd1ac..300e729a58d0 100644
--- a/arch/m68k/platform/coldfire/m5249.c
+++ b/arch/m68k/platform/coldfire/m5249.c
@@ -16,16 +16,6 @@
16#include <asm/machdep.h> 16#include <asm/machdep.h>
17#include <asm/coldfire.h> 17#include <asm/coldfire.h>
18#include <asm/mcfsim.h> 18#include <asm/mcfsim.h>
19#include <asm/mcfgpio.h>
20
21/***************************************************************************/
22
23struct mcf_gpio_chip mcf_gpio_chips[] = {
24 MCFGPS(GPIO0, 0, 32, MCFSIM2_GPIOENABLE, MCFSIM2_GPIOWRITE, MCFSIM2_GPIOREAD),
25 MCFGPS(GPIO1, 32, 32, MCFSIM2_GPIO1ENABLE, MCFSIM2_GPIO1WRITE, MCFSIM2_GPIO1READ),
26};
27
28unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips);
29 19
30/***************************************************************************/ 20/***************************************************************************/
31 21
diff --git a/arch/m68k/platform/coldfire/m525x.c b/arch/m68k/platform/coldfire/m525x.c
new file mode 100644
index 000000000000..8ce905f9b84f
--- /dev/null
+++ b/arch/m68k/platform/coldfire/m525x.c
@@ -0,0 +1,66 @@
1/***************************************************************************/
2
3/*
4 * 525x.c
5 *
6 * Copyright (C) 2012, Steven King <sfking@fdwdc.com>
7 */
8
9/***************************************************************************/
10
11#include <linux/kernel.h>
12#include <linux/param.h>
13#include <linux/init.h>
14#include <linux/io.h>
15#include <linux/platform_device.h>
16#include <asm/machdep.h>
17#include <asm/coldfire.h>
18#include <asm/mcfsim.h>
19
20/***************************************************************************/
21
22static void __init m525x_qspi_init(void)
23{
24#if IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI)
25 /* set the GPIO function for the qspi cs gpios */
26 /* FIXME: replace with pinmux/pinctl support */
27 u32 f = readl(MCFSIM2_GPIOFUNC);
28 f |= (1 << MCFQSPI_CS2) | (1 << MCFQSPI_CS1) | (1 << MCFQSPI_CS0);
29 writel(f, MCFSIM2_GPIOFUNC);
30
31 /* QSPI irq setup */
32 writeb(MCFSIM_ICR_AUTOVEC | MCFSIM_ICR_LEVEL4 | MCFSIM_ICR_PRI0,
33 MCF_MBAR + MCFSIM_QSPIICR);
34 mcf_mapirq2imr(MCF_IRQ_QSPI, MCFINTC_QSPI);
35#endif /* IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI) */
36}
37
38static void __init m525x_i2c_init(void)
39{
40#if IS_ENABLED(CONFIG_I2C_COLDFIRE)
41 u32 r;
42
43 /* first I2C controller uses regular irq setup */
44 writeb(MCFSIM_ICR_AUTOVEC | MCFSIM_ICR_LEVEL5 | MCFSIM_ICR_PRI0,
45 MCF_MBAR + MCFSIM_I2CICR);
46 mcf_mapirq2imr(MCF_IRQ_I2C0, MCFINTC_I2C);
47
48 /* second I2C controller is completely different */
49 r = readl(MCFINTC2_INTPRI_REG(MCF_IRQ_I2C1));
50 r &= ~MCFINTC2_INTPRI_BITS(0xf, MCF_IRQ_I2C1);
51 r |= MCFINTC2_INTPRI_BITS(0x5, MCF_IRQ_I2C1);
52 writel(r, MCFINTC2_INTPRI_REG(MCF_IRQ_I2C1));
53#endif /* IS_ENABLED(CONFIG_I2C_COLDFIRE) */
54}
55
56/***************************************************************************/
57
58void __init config_BSP(char *commandp, int size)
59{
60 mach_sched_init = hw_timer_init;
61
62 m525x_qspi_init();
63 m525x_i2c_init();
64}
65
66/***************************************************************************/
diff --git a/arch/m68k/platform/coldfire/m5272.c b/arch/m68k/platform/coldfire/m5272.c
index 43e36060da18..e68bc7a148eb 100644
--- a/arch/m68k/platform/coldfire/m5272.c
+++ b/arch/m68k/platform/coldfire/m5272.c
@@ -19,7 +19,6 @@
19#include <asm/coldfire.h> 19#include <asm/coldfire.h>
20#include <asm/mcfsim.h> 20#include <asm/mcfsim.h>
21#include <asm/mcfuart.h> 21#include <asm/mcfuart.h>
22#include <asm/mcfgpio.h>
23 22
24/***************************************************************************/ 23/***************************************************************************/
25 24
@@ -31,16 +30,6 @@ unsigned char ledbank = 0xff;
31 30
32/***************************************************************************/ 31/***************************************************************************/
33 32
34struct mcf_gpio_chip mcf_gpio_chips[] = {
35 MCFGPS(PA, 0, 16, MCFSIM_PADDR, MCFSIM_PADAT, MCFSIM_PADAT),
36 MCFGPS(PB, 16, 16, MCFSIM_PBDDR, MCFSIM_PBDAT, MCFSIM_PBDAT),
37 MCFGPS(Pc, 32, 16, MCFSIM_PCDDR, MCFSIM_PCDAT, MCFSIM_PCDAT),
38};
39
40unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips);
41
42/***************************************************************************/
43
44static void __init m5272_uarts_init(void) 33static void __init m5272_uarts_init(void)
45{ 34{
46 u32 v; 35 u32 v;
diff --git a/arch/m68k/platform/coldfire/m527x.c b/arch/m68k/platform/coldfire/m527x.c
index 9b0b66aabd1b..b3cb378c5e94 100644
--- a/arch/m68k/platform/coldfire/m527x.c
+++ b/arch/m68k/platform/coldfire/m527x.c
@@ -20,49 +20,6 @@
20#include <asm/coldfire.h> 20#include <asm/coldfire.h>
21#include <asm/mcfsim.h> 21#include <asm/mcfsim.h>
22#include <asm/mcfuart.h> 22#include <asm/mcfuart.h>
23#include <asm/mcfgpio.h>
24
25/***************************************************************************/
26
27struct mcf_gpio_chip mcf_gpio_chips[] = {
28#if defined(CONFIG_M5271)
29 MCFGPS(PIRQ, 1, 7, MCFEPORT_EPDDR, MCFEPORT_EPDR, MCFEPORT_EPPDR),
30 MCFGPF(ADDR, 13, 3),
31 MCFGPF(DATAH, 16, 8),
32 MCFGPF(DATAL, 24, 8),
33 MCFGPF(BUSCTL, 32, 8),
34 MCFGPF(BS, 40, 4),
35 MCFGPF(CS, 49, 7),
36 MCFGPF(SDRAM, 56, 6),
37 MCFGPF(FECI2C, 64, 4),
38 MCFGPF(UARTH, 72, 2),
39 MCFGPF(UARTL, 80, 8),
40 MCFGPF(QSPI, 88, 5),
41 MCFGPF(TIMER, 96, 8),
42#elif defined(CONFIG_M5275)
43 MCFGPS(PIRQ, 1, 7, MCFEPORT_EPDDR, MCFEPORT_EPDR, MCFEPORT_EPPDR),
44 MCFGPF(BUSCTL, 8, 8),
45 MCFGPF(ADDR, 21, 3),
46 MCFGPF(CS, 25, 7),
47 MCFGPF(FEC0H, 32, 8),
48 MCFGPF(FEC0L, 40, 8),
49 MCFGPF(FECI2C, 48, 6),
50 MCFGPF(QSPI, 56, 7),
51 MCFGPF(SDRAM, 64, 8),
52 MCFGPF(TIMERH, 72, 4),
53 MCFGPF(TIMERL, 80, 4),
54 MCFGPF(UARTL, 88, 8),
55 MCFGPF(FEC1H, 96, 8),
56 MCFGPF(FEC1L, 104, 8),
57 MCFGPF(BS, 114, 2),
58 MCFGPF(IRQ, 121, 7),
59 MCFGPF(USBH, 128, 1),
60 MCFGPF(USBL, 136, 8),
61 MCFGPF(UARTH, 144, 4),
62#endif
63};
64
65unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips);
66 23
67/***************************************************************************/ 24/***************************************************************************/
68 25
diff --git a/arch/m68k/platform/coldfire/m528x.c b/arch/m68k/platform/coldfire/m528x.c
index 7ed1276b29dc..f1319e5d2546 100644
--- a/arch/m68k/platform/coldfire/m528x.c
+++ b/arch/m68k/platform/coldfire/m528x.c
@@ -21,37 +21,6 @@
21#include <asm/coldfire.h> 21#include <asm/coldfire.h>
22#include <asm/mcfsim.h> 22#include <asm/mcfsim.h>
23#include <asm/mcfuart.h> 23#include <asm/mcfuart.h>
24#include <asm/mcfgpio.h>
25
26/***************************************************************************/
27
28struct mcf_gpio_chip mcf_gpio_chips[] = {
29 MCFGPS(NQ, 1, 7, MCFEPORT_EPDDR, MCFEPORT_EPDR, MCFEPORT_EPPDR),
30 MCFGPS(TA, 8, 4, MCFGPTA_GPTDDR, MCFGPTA_GPTPORT, MCFGPTB_GPTPORT),
31 MCFGPS(TB, 16, 4, MCFGPTB_GPTDDR, MCFGPTB_GPTPORT, MCFGPTB_GPTPORT),
32 MCFGPS(QA, 24, 4, MCFQADC_DDRQA, MCFQADC_PORTQA, MCFQADC_PORTQA),
33 MCFGPS(QB, 32, 4, MCFQADC_DDRQB, MCFQADC_PORTQB, MCFQADC_PORTQB),
34 MCFGPF(A, 40, 8),
35 MCFGPF(B, 48, 8),
36 MCFGPF(C, 56, 8),
37 MCFGPF(D, 64, 8),
38 MCFGPF(E, 72, 8),
39 MCFGPF(F, 80, 8),
40 MCFGPF(G, 88, 8),
41 MCFGPF(H, 96, 8),
42 MCFGPF(J, 104, 8),
43 MCFGPF(DD, 112, 8),
44 MCFGPF(EH, 120, 8),
45 MCFGPF(EL, 128, 8),
46 MCFGPF(AS, 136, 6),
47 MCFGPF(QS, 144, 7),
48 MCFGPF(SD, 152, 6),
49 MCFGPF(TC, 160, 4),
50 MCFGPF(TD, 168, 4),
51 MCFGPF(UA, 176, 4),
52};
53
54unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips);
55 24
56/***************************************************************************/ 25/***************************************************************************/
57 26
@@ -74,7 +43,7 @@ static void __init m528x_uarts_init(void)
74 /* make sure PUAPAR is set for UART0 and UART1 */ 43 /* make sure PUAPAR is set for UART0 and UART1 */
75 port = readb(MCF5282_GPIO_PUAPAR); 44 port = readb(MCF5282_GPIO_PUAPAR);
76 port |= 0x03 | (0x03 << 2); 45 port |= 0x03 | (0x03 << 2);
77 writeb(port, MCF5282_GPIO_PUAPAR); 46 writeb(port, MCFGPIO_PUAPAR);
78} 47}
79 48
80/***************************************************************************/ 49/***************************************************************************/
diff --git a/arch/m68k/platform/coldfire/m5307.c b/arch/m68k/platform/coldfire/m5307.c
index 93b484976ab3..a568d2870d15 100644
--- a/arch/m68k/platform/coldfire/m5307.c
+++ b/arch/m68k/platform/coldfire/m5307.c
@@ -16,7 +16,6 @@
16#include <asm/machdep.h> 16#include <asm/machdep.h>
17#include <asm/coldfire.h> 17#include <asm/coldfire.h>
18#include <asm/mcfsim.h> 18#include <asm/mcfsim.h>
19#include <asm/mcfgpio.h>
20#include <asm/mcfwdebug.h> 19#include <asm/mcfwdebug.h>
21 20
22/***************************************************************************/ 21/***************************************************************************/
@@ -29,14 +28,6 @@ unsigned char ledbank = 0xff;
29 28
30/***************************************************************************/ 29/***************************************************************************/
31 30
32struct mcf_gpio_chip mcf_gpio_chips[] = {
33 MCFGPS(PP, 0, 16, MCFSIM_PADDR, MCFSIM_PADAT, MCFSIM_PADAT),
34};
35
36unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips);
37
38/***************************************************************************/
39
40void __init config_BSP(char *commandp, int size) 31void __init config_BSP(char *commandp, int size)
41{ 32{
42#if defined(CONFIG_NETtel) || \ 33#if defined(CONFIG_NETtel) || \
diff --git a/arch/m68k/platform/coldfire/m532x.c b/arch/m68k/platform/coldfire/m532x.c
index 5394223639f8..4819a44991ed 100644
--- a/arch/m68k/platform/coldfire/m532x.c
+++ b/arch/m68k/platform/coldfire/m532x.c
@@ -26,32 +26,144 @@
26#include <asm/mcfsim.h> 26#include <asm/mcfsim.h>
27#include <asm/mcfuart.h> 27#include <asm/mcfuart.h>
28#include <asm/mcfdma.h> 28#include <asm/mcfdma.h>
29#include <asm/mcfgpio.h>
30#include <asm/mcfwdebug.h> 29#include <asm/mcfwdebug.h>
30#include <asm/mcfclk.h>
31 31
32/***************************************************************************/ 32/***************************************************************************/
33 33
34struct mcf_gpio_chip mcf_gpio_chips[] = { 34DEFINE_CLK(0, "flexbus", 2, MCF_CLK);
35 MCFGPS(PIRQ, 0, 8, MCFEPORT_EPDDR, MCFEPORT_EPDR, MCFEPORT_EPPDR), 35DEFINE_CLK(0, "mcfcan.0", 8, MCF_CLK);
36 MCFGPF(FECH, 8, 8), 36DEFINE_CLK(0, "fec.0", 12, MCF_CLK);
37 MCFGPF(FECL, 16, 8), 37DEFINE_CLK(0, "edma", 17, MCF_CLK);
38 MCFGPF(SSI, 24, 5), 38DEFINE_CLK(0, "intc.0", 18, MCF_CLK);
39 MCFGPF(BUSCTL, 32, 4), 39DEFINE_CLK(0, "intc.1", 19, MCF_CLK);
40 MCFGPF(BE, 40, 4), 40DEFINE_CLK(0, "iack.0", 21, MCF_CLK);
41 MCFGPF(CS, 49, 5), 41DEFINE_CLK(0, "mcfi2c.0", 22, MCF_CLK);
42 MCFGPF(PWM, 58, 4), 42DEFINE_CLK(0, "mcfqspi.0", 23, MCF_CLK);
43 MCFGPF(FECI2C, 64, 4), 43DEFINE_CLK(0, "mcfuart.0", 24, MCF_BUSCLK);
44 MCFGPF(UART, 72, 8), 44DEFINE_CLK(0, "mcfuart.1", 25, MCF_BUSCLK);
45 MCFGPF(QSPI, 80, 6), 45DEFINE_CLK(0, "mcfuart.2", 26, MCF_BUSCLK);
46 MCFGPF(TIMER, 88, 4), 46DEFINE_CLK(0, "mcftmr.0", 28, MCF_CLK);
47 MCFGPF(LCDDATAH, 96, 2), 47DEFINE_CLK(0, "mcftmr.1", 29, MCF_CLK);
48 MCFGPF(LCDDATAM, 104, 8), 48DEFINE_CLK(0, "mcftmr.2", 30, MCF_CLK);
49 MCFGPF(LCDDATAL, 112, 8), 49DEFINE_CLK(0, "mcftmr.3", 31, MCF_CLK);
50 MCFGPF(LCDCTLH, 120, 1), 50
51 MCFGPF(LCDCTLL, 128, 8), 51DEFINE_CLK(0, "mcfpit.0", 32, MCF_CLK);
52DEFINE_CLK(0, "mcfpit.1", 33, MCF_CLK);
53DEFINE_CLK(0, "mcfpit.2", 34, MCF_CLK);
54DEFINE_CLK(0, "mcfpit.3", 35, MCF_CLK);
55DEFINE_CLK(0, "mcfpwm.0", 36, MCF_CLK);
56DEFINE_CLK(0, "mcfeport.0", 37, MCF_CLK);
57DEFINE_CLK(0, "mcfwdt.0", 38, MCF_CLK);
58DEFINE_CLK(0, "sys.0", 40, MCF_BUSCLK);
59DEFINE_CLK(0, "gpio.0", 41, MCF_BUSCLK);
60DEFINE_CLK(0, "mcfrtc.0", 42, MCF_CLK);
61DEFINE_CLK(0, "mcflcd.0", 43, MCF_CLK);
62DEFINE_CLK(0, "mcfusb-otg.0", 44, MCF_CLK);
63DEFINE_CLK(0, "mcfusb-host.0", 45, MCF_CLK);
64DEFINE_CLK(0, "sdram.0", 46, MCF_CLK);
65DEFINE_CLK(0, "ssi.0", 47, MCF_CLK);
66DEFINE_CLK(0, "pll.0", 48, MCF_CLK);
67
68DEFINE_CLK(1, "mdha.0", 32, MCF_CLK);
69DEFINE_CLK(1, "skha.0", 33, MCF_CLK);
70DEFINE_CLK(1, "rng.0", 34, MCF_CLK);
71
72struct clk *mcf_clks[] = {
73 &__clk_0_2, /* flexbus */
74 &__clk_0_8, /* mcfcan.0 */
75 &__clk_0_12, /* fec.0 */
76 &__clk_0_17, /* edma */
77 &__clk_0_18, /* intc.0 */
78 &__clk_0_19, /* intc.1 */
79 &__clk_0_21, /* iack.0 */
80 &__clk_0_22, /* mcfi2c.0 */
81 &__clk_0_23, /* mcfqspi.0 */
82 &__clk_0_24, /* mcfuart.0 */
83 &__clk_0_25, /* mcfuart.1 */
84 &__clk_0_26, /* mcfuart.2 */
85 &__clk_0_28, /* mcftmr.0 */
86 &__clk_0_29, /* mcftmr.1 */
87 &__clk_0_30, /* mcftmr.2 */
88 &__clk_0_31, /* mcftmr.3 */
89
90 &__clk_0_32, /* mcfpit.0 */
91 &__clk_0_33, /* mcfpit.1 */
92 &__clk_0_34, /* mcfpit.2 */
93 &__clk_0_35, /* mcfpit.3 */
94 &__clk_0_36, /* mcfpwm.0 */
95 &__clk_0_37, /* mcfeport.0 */
96 &__clk_0_38, /* mcfwdt.0 */
97 &__clk_0_40, /* sys.0 */
98 &__clk_0_41, /* gpio.0 */
99 &__clk_0_42, /* mcfrtc.0 */
100 &__clk_0_43, /* mcflcd.0 */
101 &__clk_0_44, /* mcfusb-otg.0 */
102 &__clk_0_45, /* mcfusb-host.0 */
103 &__clk_0_46, /* sdram.0 */
104 &__clk_0_47, /* ssi.0 */
105 &__clk_0_48, /* pll.0 */
106
107 &__clk_1_32, /* mdha.0 */
108 &__clk_1_33, /* skha.0 */
109 &__clk_1_34, /* rng.0 */
110 NULL,
111};
112
113static struct clk * const enable_clks[] __initconst = {
114 &__clk_0_2, /* flexbus */
115 &__clk_0_18, /* intc.0 */
116 &__clk_0_19, /* intc.1 */
117 &__clk_0_21, /* iack.0 */
118 &__clk_0_24, /* mcfuart.0 */
119 &__clk_0_25, /* mcfuart.1 */
120 &__clk_0_26, /* mcfuart.2 */
121
122 &__clk_0_32, /* mcfpit.0 */
123 &__clk_0_33, /* mcfpit.1 */
124 &__clk_0_37, /* mcfeport.0 */
125 &__clk_0_40, /* sys.0 */
126 &__clk_0_41, /* gpio.0 */
127 &__clk_0_46, /* sdram.0 */
128 &__clk_0_48, /* pll.0 */
129};
130
131static struct clk * const disable_clks[] __initconst = {
132 &__clk_0_8, /* mcfcan.0 */
133 &__clk_0_12, /* fec.0 */
134 &__clk_0_17, /* edma */
135 &__clk_0_22, /* mcfi2c.0 */
136 &__clk_0_23, /* mcfqspi.0 */
137 &__clk_0_28, /* mcftmr.0 */
138 &__clk_0_29, /* mcftmr.1 */
139 &__clk_0_30, /* mcftmr.2 */
140 &__clk_0_31, /* mcftmr.3 */
141 &__clk_0_34, /* mcfpit.2 */
142 &__clk_0_35, /* mcfpit.3 */
143 &__clk_0_36, /* mcfpwm.0 */
144 &__clk_0_38, /* mcfwdt.0 */
145 &__clk_0_42, /* mcfrtc.0 */
146 &__clk_0_43, /* mcflcd.0 */
147 &__clk_0_44, /* mcfusb-otg.0 */
148 &__clk_0_45, /* mcfusb-host.0 */
149 &__clk_0_47, /* ssi.0 */
150 &__clk_1_32, /* mdha.0 */
151 &__clk_1_33, /* skha.0 */
152 &__clk_1_34, /* rng.0 */
52}; 153};
53 154
54unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips); 155
156static void __init m532x_clk_init(void)
157{
158 unsigned i;
159
160 /* make sure these clocks are enabled */
161 for (i = 0; i < ARRAY_SIZE(enable_clks); ++i)
162 __clk_init_enabled(enable_clks[i]);
163 /* make sure these clocks are disabled */
164 for (i = 0; i < ARRAY_SIZE(disable_clks); ++i)
165 __clk_init_disabled(disable_clks[i]);
166}
55 167
56/***************************************************************************/ 168/***************************************************************************/
57 169
@@ -98,8 +210,8 @@ void __init config_BSP(char *commandp, int size)
98 memset(commandp, 0, size); 210 memset(commandp, 0, size);
99 } 211 }
100#endif 212#endif
101
102 mach_sched_init = hw_timer_init; 213 mach_sched_init = hw_timer_init;
214 m532x_clk_init();
103 m532x_uarts_init(); 215 m532x_uarts_init();
104 m532x_fec_init(); 216 m532x_fec_init();
105#if IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI) 217#if IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI)
diff --git a/arch/m68k/platform/coldfire/m5407.c b/arch/m68k/platform/coldfire/m5407.c
index faa6680b3404..bb6c746ae819 100644
--- a/arch/m68k/platform/coldfire/m5407.c
+++ b/arch/m68k/platform/coldfire/m5407.c
@@ -16,15 +16,6 @@
16#include <asm/machdep.h> 16#include <asm/machdep.h>
17#include <asm/coldfire.h> 17#include <asm/coldfire.h>
18#include <asm/mcfsim.h> 18#include <asm/mcfsim.h>
19#include <asm/mcfgpio.h>
20
21/***************************************************************************/
22
23struct mcf_gpio_chip mcf_gpio_chips[] = {
24 MCFGPS(PP, 0, 16, MCFSIM_PADDR, MCFSIM_PADAT, MCFSIM_PADAT),
25};
26
27unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips);
28 19
29/***************************************************************************/ 20/***************************************************************************/
30 21
diff --git a/arch/m68k/platform/coldfire/m5441x.c b/arch/m68k/platform/coldfire/m5441x.c
new file mode 100644
index 000000000000..98a13cce93d8
--- /dev/null
+++ b/arch/m68k/platform/coldfire/m5441x.c
@@ -0,0 +1,261 @@
1/*
2 * m5441x.c -- support for Coldfire m5441x processors
3 *
4 * (C) Copyright Steven King <sfking@fdwdc.com>
5 */
6
7#include <linux/kernel.h>
8#include <linux/param.h>
9#include <linux/init.h>
10#include <linux/io.h>
11#include <linux/clk.h>
12#include <asm/machdep.h>
13#include <asm/coldfire.h>
14#include <asm/mcfsim.h>
15#include <asm/mcfuart.h>
16#include <asm/mcfdma.h>
17#include <asm/mcfclk.h>
18
19DEFINE_CLK(0, "flexbus", 2, MCF_CLK);
20DEFINE_CLK(0, "mcfcan.0", 8, MCF_CLK);
21DEFINE_CLK(0, "mcfcan.1", 9, MCF_CLK);
22DEFINE_CLK(0, "mcfi2c.1", 14, MCF_CLK);
23DEFINE_CLK(0, "mcfdspi.1", 15, MCF_CLK);
24DEFINE_CLK(0, "edma", 17, MCF_CLK);
25DEFINE_CLK(0, "intc.0", 18, MCF_CLK);
26DEFINE_CLK(0, "intc.1", 19, MCF_CLK);
27DEFINE_CLK(0, "intc.2", 20, MCF_CLK);
28DEFINE_CLK(0, "mcfi2c.0", 22, MCF_CLK);
29DEFINE_CLK(0, "mcfdspi.0", 23, MCF_CLK);
30DEFINE_CLK(0, "mcfuart.0", 24, MCF_BUSCLK);
31DEFINE_CLK(0, "mcfuart.1", 25, MCF_BUSCLK);
32DEFINE_CLK(0, "mcfuart.2", 26, MCF_BUSCLK);
33DEFINE_CLK(0, "mcfuart.3", 27, MCF_BUSCLK);
34DEFINE_CLK(0, "mcftmr.0", 28, MCF_CLK);
35DEFINE_CLK(0, "mcftmr.1", 29, MCF_CLK);
36DEFINE_CLK(0, "mcftmr.2", 30, MCF_CLK);
37DEFINE_CLK(0, "mcftmr.3", 31, MCF_CLK);
38DEFINE_CLK(0, "mcfpit.0", 32, MCF_CLK);
39DEFINE_CLK(0, "mcfpit.1", 33, MCF_CLK);
40DEFINE_CLK(0, "mcfpit.2", 34, MCF_CLK);
41DEFINE_CLK(0, "mcfpit.3", 35, MCF_CLK);
42DEFINE_CLK(0, "mcfeport.0", 37, MCF_CLK);
43DEFINE_CLK(0, "mcfadc.0", 38, MCF_CLK);
44DEFINE_CLK(0, "mcfdac.0", 39, MCF_CLK);
45DEFINE_CLK(0, "mcfrtc.0", 42, MCF_CLK);
46DEFINE_CLK(0, "mcfsim.0", 43, MCF_CLK);
47DEFINE_CLK(0, "mcfusb-otg.0", 44, MCF_CLK);
48DEFINE_CLK(0, "mcfusb-host.0", 45, MCF_CLK);
49DEFINE_CLK(0, "mcfddr-sram.0", 46, MCF_CLK);
50DEFINE_CLK(0, "mcfssi.0", 47, MCF_CLK);
51DEFINE_CLK(0, "pll.0", 48, MCF_CLK);
52DEFINE_CLK(0, "mcfrng.0", 49, MCF_CLK);
53DEFINE_CLK(0, "mcfssi.1", 50, MCF_CLK);
54DEFINE_CLK(0, "mcfsdhc.0", 51, MCF_CLK);
55DEFINE_CLK(0, "enet-fec.0", 53, MCF_CLK);
56DEFINE_CLK(0, "enet-fec.1", 54, MCF_CLK);
57DEFINE_CLK(0, "switch.0", 55, MCF_CLK);
58DEFINE_CLK(0, "switch.1", 56, MCF_CLK);
59DEFINE_CLK(0, "nand.0", 63, MCF_CLK);
60
61DEFINE_CLK(1, "mcfow.0", 2, MCF_CLK);
62DEFINE_CLK(1, "mcfi2c.2", 4, MCF_CLK);
63DEFINE_CLK(1, "mcfi2c.3", 5, MCF_CLK);
64DEFINE_CLK(1, "mcfi2c.4", 6, MCF_CLK);
65DEFINE_CLK(1, "mcfi2c.5", 7, MCF_CLK);
66DEFINE_CLK(1, "mcfuart.4", 24, MCF_BUSCLK);
67DEFINE_CLK(1, "mcfuart.5", 25, MCF_BUSCLK);
68DEFINE_CLK(1, "mcfuart.6", 26, MCF_BUSCLK);
69DEFINE_CLK(1, "mcfuart.7", 27, MCF_BUSCLK);
70DEFINE_CLK(1, "mcfuart.8", 28, MCF_BUSCLK);
71DEFINE_CLK(1, "mcfuart.9", 29, MCF_BUSCLK);
72DEFINE_CLK(1, "mcfpwm.0", 34, MCF_BUSCLK);
73DEFINE_CLK(1, "sys.0", 36, MCF_BUSCLK);
74DEFINE_CLK(1, "gpio.0", 37, MCF_BUSCLK);
75
76struct clk *mcf_clks[] = {
77 &__clk_0_2,
78 &__clk_0_8,
79 &__clk_0_9,
80 &__clk_0_14,
81 &__clk_0_15,
82 &__clk_0_17,
83 &__clk_0_18,
84 &__clk_0_19,
85 &__clk_0_20,
86 &__clk_0_22,
87 &__clk_0_23,
88 &__clk_0_24,
89 &__clk_0_25,
90 &__clk_0_26,
91 &__clk_0_27,
92 &__clk_0_28,
93 &__clk_0_29,
94 &__clk_0_30,
95 &__clk_0_31,
96 &__clk_0_32,
97 &__clk_0_33,
98 &__clk_0_34,
99 &__clk_0_35,
100 &__clk_0_37,
101 &__clk_0_38,
102 &__clk_0_39,
103 &__clk_0_42,
104 &__clk_0_43,
105 &__clk_0_44,
106 &__clk_0_45,
107 &__clk_0_46,
108 &__clk_0_47,
109 &__clk_0_48,
110 &__clk_0_49,
111 &__clk_0_50,
112 &__clk_0_51,
113 &__clk_0_53,
114 &__clk_0_54,
115 &__clk_0_55,
116 &__clk_0_56,
117 &__clk_0_63,
118
119 &__clk_1_2,
120 &__clk_1_4,
121 &__clk_1_5,
122 &__clk_1_6,
123 &__clk_1_7,
124 &__clk_1_24,
125 &__clk_1_25,
126 &__clk_1_26,
127 &__clk_1_27,
128 &__clk_1_28,
129 &__clk_1_29,
130 &__clk_1_34,
131 &__clk_1_36,
132 &__clk_1_37,
133 NULL,
134};
135
136
137static struct clk * const enable_clks[] __initconst = {
138 /* make sure these clocks are enabled */
139 &__clk_0_18, /* intc0 */
140 &__clk_0_19, /* intc0 */
141 &__clk_0_20, /* intc0 */
142 &__clk_0_24, /* uart0 */
143 &__clk_0_25, /* uart1 */
144 &__clk_0_26, /* uart2 */
145 &__clk_0_27, /* uart3 */
146
147 &__clk_0_33, /* pit.1 */
148 &__clk_0_37, /* eport */
149 &__clk_0_48, /* pll */
150
151 &__clk_1_36, /* CCM/reset module/Power management */
152 &__clk_1_37, /* gpio */
153};
154static struct clk * const disable_clks[] __initconst = {
155 &__clk_0_8, /* can.0 */
156 &__clk_0_9, /* can.1 */
157 &__clk_0_14, /* i2c.1 */
158 &__clk_0_15, /* dspi.1 */
159 &__clk_0_17, /* eDMA */
160 &__clk_0_22, /* i2c.0 */
161 &__clk_0_23, /* dspi.0 */
162 &__clk_0_28, /* tmr.1 */
163 &__clk_0_29, /* tmr.2 */
164 &__clk_0_30, /* tmr.2 */
165 &__clk_0_31, /* tmr.3 */
166 &__clk_0_32, /* pit.0 */
167 &__clk_0_34, /* pit.2 */
168 &__clk_0_35, /* pit.3 */
169 &__clk_0_38, /* adc */
170 &__clk_0_39, /* dac */
171 &__clk_0_44, /* usb otg */
172 &__clk_0_45, /* usb host */
173 &__clk_0_47, /* ssi.0 */
174 &__clk_0_49, /* rng */
175 &__clk_0_50, /* ssi.1 */
176 &__clk_0_51, /* eSDHC */
177 &__clk_0_53, /* enet-fec */
178 &__clk_0_54, /* enet-fec */
179 &__clk_0_55, /* switch.0 */
180 &__clk_0_56, /* switch.1 */
181
182 &__clk_1_2, /* 1-wire */
183 &__clk_1_4, /* i2c.2 */
184 &__clk_1_5, /* i2c.3 */
185 &__clk_1_6, /* i2c.4 */
186 &__clk_1_7, /* i2c.5 */
187 &__clk_1_24, /* uart 4 */
188 &__clk_1_25, /* uart 5 */
189 &__clk_1_26, /* uart 6 */
190 &__clk_1_27, /* uart 7 */
191 &__clk_1_28, /* uart 8 */
192 &__clk_1_29, /* uart 9 */
193};
194
195static void __init m5441x_clk_init(void)
196{
197 unsigned i;
198
199 for (i = 0; i < ARRAY_SIZE(enable_clks); ++i)
200 __clk_init_enabled(enable_clks[i]);
201 /* make sure these clocks are disabled */
202 for (i = 0; i < ARRAY_SIZE(disable_clks); ++i)
203 __clk_init_disabled(disable_clks[i]);
204}
205
206static void __init m5441x_uarts_init(void)
207{
208 __raw_writeb(0x0f, MCFGPIO_PAR_UART0);
209 __raw_writeb(0x00, MCFGPIO_PAR_UART1);
210 __raw_writeb(0x00, MCFGPIO_PAR_UART2);
211}
212
213static void __init m5441x_fec_init(void)
214{
215 __raw_writeb(0x03, MCFGPIO_PAR_FEC);
216}
217
218void __init config_BSP(char *commandp, int size)
219{
220 m5441x_clk_init();
221 mach_sched_init = hw_timer_init;
222 m5441x_uarts_init();
223 m5441x_fec_init();
224}
225
226
227#if IS_ENABLED(CONFIG_RTC_DRV_M5441x)
228static struct resource m5441x_rtc_resources[] = {
229 {
230 .start = MCFRTC_BASE,
231 .end = MCFRTC_BASE + MCFRTC_SIZE - 1,
232 .flags = IORESOURCE_MEM,
233 },
234 {
235 .start = MCF_IRQ_RTC,
236 .end = MCF_IRQ_RTC,
237 .flags = IORESOURCE_IRQ,
238 },
239};
240
241static struct platform_device m5441x_rtc = {
242 .name = "mcfrtc",
243 .id = 0,
244 .resource = m5441x_rtc_resources,
245 .num_resources = ARRAY_SIZE(m5441x_rtc_resources),
246};
247#endif
248
249static struct platform_device *m5441x_devices[] __initdata = {
250#if IS_ENABLED(CONFIG_RTC_DRV_M5441x)
251 &m5441x_rtc,
252#endif
253};
254
255static int __init init_BSP(void)
256{
257 platform_add_devices(m5441x_devices, ARRAY_SIZE(m5441x_devices));
258 return 0;
259}
260
261arch_initcall(init_BSP);
diff --git a/arch/m68k/platform/coldfire/m54xx.c b/arch/m68k/platform/coldfire/m54xx.c
index 20672dadb252..2081c6cbb3de 100644
--- a/arch/m68k/platform/coldfire/m54xx.c
+++ b/arch/m68k/platform/coldfire/m54xx.c
@@ -21,19 +21,12 @@
21#include <asm/m54xxsim.h> 21#include <asm/m54xxsim.h>
22#include <asm/mcfuart.h> 22#include <asm/mcfuart.h>
23#include <asm/m54xxgpt.h> 23#include <asm/m54xxgpt.h>
24#include <asm/mcfgpio.h>
25#ifdef CONFIG_MMU 24#ifdef CONFIG_MMU
26#include <asm/mmu_context.h> 25#include <asm/mmu_context.h>
27#endif 26#endif
28 27
29/***************************************************************************/ 28/***************************************************************************/
30 29
31struct mcf_gpio_chip mcf_gpio_chips[] = { };
32
33unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips);
34
35/***************************************************************************/
36
37static void __init m54xx_uarts_init(void) 30static void __init m54xx_uarts_init(void)
38{ 31{
39 /* enable io pins */ 32 /* enable io pins */
diff --git a/arch/m68k/platform/coldfire/mcf8390.c b/arch/m68k/platform/coldfire/mcf8390.c
new file mode 100644
index 000000000000..23a6874a3248
--- /dev/null
+++ b/arch/m68k/platform/coldfire/mcf8390.c
@@ -0,0 +1,38 @@
1/*
2 * mcf8390.c -- platform support for 8390 ethernet on many boards
3 *
4 * (C) Copyright 2012, Greg Ungerer <gerg@uclinux.org>
5 *
6 * This file is subject to the terms and conditions of the GNU General Public
7 * License. See the file COPYING in the main directory of this archive
8 * for more details.
9 */
10
11#include <linux/kernel.h>
12#include <linux/module.h>
13#include <linux/init.h>
14#include <linux/resource.h>
15#include <linux/platform_device.h>
16#include <asm/mcf8390.h>
17
18static struct resource mcf8390_resources[] = {
19 {
20 .start = NE2000_ADDR,
21 .end = NE2000_ADDR + NE2000_ADDRSIZE - 1,
22 .flags = IORESOURCE_MEM,
23 },
24 {
25 .start = NE2000_IRQ_VECTOR,
26 .end = NE2000_IRQ_VECTOR,
27 .flags = IORESOURCE_IRQ,
28 },
29};
30
31static int __init mcf8390_platform_init(void)
32{
33 platform_device_register_simple("mcf8390", -1, mcf8390_resources,
34 ARRAY_SIZE(mcf8390_resources));
35 return 0;
36}
37
38arch_initcall(mcf8390_platform_init);
diff --git a/arch/m68k/platform/coldfire/pci.c b/arch/m68k/platform/coldfire/pci.c
new file mode 100644
index 000000000000..553210d3d4c1
--- /dev/null
+++ b/arch/m68k/platform/coldfire/pci.c
@@ -0,0 +1,327 @@
1/*
2 * pci.c -- PCI bus support for ColdFire processors
3 *
4 * (C) Copyright 2012, Greg Ungerer <gerg@uclinux.com>
5 *
6 * This file is subject to the terms and conditions of the GNU General Public
7 * License. See the file COPYING in the main directory of this archive
8 * for more details.
9 */
10
11#include <linux/types.h>
12#include <linux/module.h>
13#include <linux/init.h>
14#include <linux/kernel.h>
15#include <linux/interrupt.h>
16#include <linux/irq.h>
17#include <linux/io.h>
18#include <linux/pci.h>
19#include <linux/delay.h>
20#include <asm/coldfire.h>
21#include <asm/mcfsim.h>
22#include <asm/m54xxpci.h>
23
24/*
25 * Memory and IO mappings. We use a 1:1 mapping for local host memory to
26 * PCI bus memory (no reason not to really). IO space doesn't matter, we
27 * always use access functions for that. The device configuration space is
28 * mapped over the IO map space when we enable it in the PCICAR register.
29 */
30#define PCI_MEM_PA 0xf0000000 /* Host physical address */
31#define PCI_MEM_BA 0xf0000000 /* Bus physical address */
32#define PCI_MEM_SIZE 0x08000000 /* 128 MB */
33#define PCI_MEM_MASK (PCI_MEM_SIZE - 1)
34
35#define PCI_IO_PA 0xf8000000 /* Host physical address */
36#define PCI_IO_BA 0x00000000 /* Bus physical address */
37#define PCI_IO_SIZE 0x00010000 /* 64k */
38#define PCI_IO_MASK (PCI_IO_SIZE - 1)
39
40static struct pci_bus *rootbus;
41static unsigned long iospace;
42
43/*
44 * We need to be carefull probing on bus 0 (directly connected to host
45 * bridge). We should only acccess the well defined possible devices in
46 * use, ignore aliases and the like.
47 */
48static unsigned char mcf_host_slot2sid[32] = {
49 0, 0, 0, 0, 0, 0, 0, 0,
50 0, 0, 0, 0, 0, 0, 0, 0,
51 0, 1, 2, 0, 3, 4, 0, 0,
52 0, 0, 0, 0, 0, 0, 0, 0,
53};
54
55static unsigned char mcf_host_irq[] = {
56 0, 69, 69, 71, 71,
57};
58
59
60static inline void syncio(void)
61{
62 /* The ColdFire "nop" instruction waits for all bus IO to complete */
63 __asm__ __volatile__ ("nop");
64}
65
66/*
67 * Configuration space access functions. Configuration space access is
68 * through the IO mapping window, enabling it via the PCICAR register.
69 */
70static unsigned long mcf_mk_pcicar(int bus, unsigned int devfn, int where)
71{
72 return (bus << PCICAR_BUSN) | (devfn << PCICAR_DEVFNN) | (where & 0xfc);
73}
74
75static int mcf_pci_readconfig(struct pci_bus *bus, unsigned int devfn,
76 int where, int size, u32 *value)
77{
78 unsigned long addr;
79
80 *value = 0xffffffff;
81
82 if (bus->number == 0) {
83 if (mcf_host_slot2sid[PCI_SLOT(devfn)] == 0)
84 return PCIBIOS_SUCCESSFUL;
85 }
86
87 syncio();
88 addr = mcf_mk_pcicar(bus->number, devfn, where);
89 __raw_writel(PCICAR_E | addr, PCICAR);
90 addr = iospace + (where & 0x3);
91
92 switch (size) {
93 case 1:
94 *value = __raw_readb(addr);
95 break;
96 case 2:
97 *value = le16_to_cpu(__raw_readw(addr));
98 break;
99 default:
100 *value = le32_to_cpu(__raw_readl(addr));
101 break;
102 }
103
104 syncio();
105 __raw_writel(0, PCICAR);
106 return PCIBIOS_SUCCESSFUL;
107}
108
109static int mcf_pci_writeconfig(struct pci_bus *bus, unsigned int devfn,
110 int where, int size, u32 value)
111{
112 unsigned long addr;
113
114 if (bus->number == 0) {
115 if (mcf_host_slot2sid[PCI_SLOT(devfn)] == 0)
116 return PCIBIOS_SUCCESSFUL;
117 }
118
119 syncio();
120 addr = mcf_mk_pcicar(bus->number, devfn, where);
121 __raw_writel(PCICAR_E | addr, PCICAR);
122 addr = iospace + (where & 0x3);
123
124 switch (size) {
125 case 1:
126 __raw_writeb(value, addr);
127 break;
128 case 2:
129 __raw_writew(cpu_to_le16(value), addr);
130 break;
131 default:
132 __raw_writel(cpu_to_le32(value), addr);
133 break;
134 }
135
136 syncio();
137 __raw_writel(0, PCICAR);
138 return PCIBIOS_SUCCESSFUL;
139}
140
141static struct pci_ops mcf_pci_ops = {
142 .read = mcf_pci_readconfig,
143 .write = mcf_pci_writeconfig,
144};
145
146/*
147 * IO address space access functions. Pretty strait forward, these are
148 * directly mapped in to the IO mapping window. And that is mapped into
149 * virtual address space.
150 */
151u8 mcf_pci_inb(u32 addr)
152{
153 return __raw_readb(iospace + (addr & PCI_IO_MASK));
154}
155EXPORT_SYMBOL(mcf_pci_inb);
156
157u16 mcf_pci_inw(u32 addr)
158{
159 return le16_to_cpu(__raw_readw(iospace + (addr & PCI_IO_MASK)));
160}
161EXPORT_SYMBOL(mcf_pci_inw);
162
163u32 mcf_pci_inl(u32 addr)
164{
165 return le32_to_cpu(__raw_readl(iospace + (addr & PCI_IO_MASK)));
166}
167EXPORT_SYMBOL(mcf_pci_inl);
168
169void mcf_pci_insb(u32 addr, u8 *buf, u32 len)
170{
171 for (; len; len--)
172 *buf++ = mcf_pci_inb(addr);
173}
174EXPORT_SYMBOL(mcf_pci_insb);
175
176void mcf_pci_insw(u32 addr, u16 *buf, u32 len)
177{
178 for (; len; len--)
179 *buf++ = mcf_pci_inw(addr);
180}
181EXPORT_SYMBOL(mcf_pci_insw);
182
183void mcf_pci_insl(u32 addr, u32 *buf, u32 len)
184{
185 for (; len; len--)
186 *buf++ = mcf_pci_inl(addr);
187}
188EXPORT_SYMBOL(mcf_pci_insl);
189
190void mcf_pci_outb(u8 v, u32 addr)
191{
192 __raw_writeb(v, iospace + (addr & PCI_IO_MASK));
193}
194EXPORT_SYMBOL(mcf_pci_outb);
195
196void mcf_pci_outw(u16 v, u32 addr)
197{
198 __raw_writew(cpu_to_le16(v), iospace + (addr & PCI_IO_MASK));
199}
200EXPORT_SYMBOL(mcf_pci_outw);
201
202void mcf_pci_outl(u32 v, u32 addr)
203{
204 __raw_writel(cpu_to_le32(v), iospace + (addr & PCI_IO_MASK));
205}
206EXPORT_SYMBOL(mcf_pci_outl);
207
208void mcf_pci_outsb(u32 addr, const u8 *buf, u32 len)
209{
210 for (; len; len--)
211 mcf_pci_outb(*buf++, addr);
212}
213EXPORT_SYMBOL(mcf_pci_outsb);
214
215void mcf_pci_outsw(u32 addr, const u16 *buf, u32 len)
216{
217 for (; len; len--)
218 mcf_pci_outw(*buf++, addr);
219}
220EXPORT_SYMBOL(mcf_pci_outsw);
221
222void mcf_pci_outsl(u32 addr, const u32 *buf, u32 len)
223{
224 for (; len; len--)
225 mcf_pci_outl(*buf++, addr);
226}
227EXPORT_SYMBOL(mcf_pci_outsl);
228
229/*
230 * Initialize the PCI bus registers, and scan the bus.
231 */
232static struct resource mcf_pci_mem = {
233 .name = "PCI Memory space",
234 .start = PCI_MEM_PA,
235 .end = PCI_MEM_PA + PCI_MEM_SIZE - 1,
236 .flags = IORESOURCE_MEM,
237};
238
239static struct resource mcf_pci_io = {
240 .name = "PCI IO space",
241 .start = 0x400,
242 .end = 0x10000 - 1,
243 .flags = IORESOURCE_IO,
244};
245
246/*
247 * Interrupt mapping and setting.
248 */
249static int mcf_pci_map_irq(const struct pci_dev *dev, u8 slot, u8 pin)
250{
251 int sid;
252
253 sid = mcf_host_slot2sid[slot];
254 if (sid)
255 return mcf_host_irq[sid];
256 return 0;
257}
258
259static int __init mcf_pci_init(void)
260{
261 pr_info("ColdFire: PCI bus initialization...\n");
262
263 /* Reset the external PCI bus */
264 __raw_writel(PCIGSCR_RESET, PCIGSCR);
265 __raw_writel(0, PCITCR);
266
267 request_resource(&iomem_resource, &mcf_pci_mem);
268 request_resource(&iomem_resource, &mcf_pci_io);
269
270 /* Configure PCI arbiter */
271 __raw_writel(PACR_INTMPRI | PACR_INTMINTE | PACR_EXTMPRI(0x1f) |
272 PACR_EXTMINTE(0x1f), PACR);
273
274 /* Set required multi-function pins for PCI bus use */
275 __raw_writew(0x3ff, MCF_PAR_PCIBG);
276 __raw_writew(0x3ff, MCF_PAR_PCIBR);
277
278 /* Set up config space for local host bus controller */
279 __raw_writel(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
280 PCI_COMMAND_INVALIDATE, PCISCR);
281 __raw_writel(PCICR1_LT(32) | PCICR1_CL(8), PCICR1);
282 __raw_writel(0, PCICR2);
283
284 /*
285 * Set up the initiator windows for memory and IO mapping.
286 * These give the CPU bus access onto the PCI bus. One for each of
287 * PCI memory and IO address spaces.
288 */
289 __raw_writel(WXBTAR(PCI_MEM_PA, PCI_MEM_BA, PCI_MEM_SIZE),
290 PCIIW0BTAR);
291 __raw_writel(WXBTAR(PCI_IO_PA, PCI_IO_BA, PCI_IO_SIZE),
292 PCIIW1BTAR);
293 __raw_writel(PCIIWCR_W0_MEM /*| PCIIWCR_W0_MRDL*/ | PCIIWCR_W0_E |
294 PCIIWCR_W1_IO | PCIIWCR_W1_E, PCIIWCR);
295
296 /*
297 * Set up the target windows for access from the PCI bus back to the
298 * CPU bus. All we need is access to system RAM (for mastering).
299 */
300 __raw_writel(CONFIG_RAMBASE, PCIBAR1);
301 __raw_writel(CONFIG_RAMBASE | PCITBATR1_E, PCITBATR1);
302
303 /* Keep a virtual mapping to IO/config space active */
304 iospace = (unsigned long) ioremap(PCI_IO_PA, PCI_IO_SIZE);
305 if (iospace == 0)
306 return -ENODEV;
307 pr_info("Coldfire: PCI IO/config window mapped to 0x%x\n",
308 (u32) iospace);
309
310 /* Turn of PCI reset, and wait for devices to settle */
311 __raw_writel(0, PCIGSCR);
312 set_current_state(TASK_UNINTERRUPTIBLE);
313 schedule_timeout(msecs_to_jiffies(200));
314
315 rootbus = pci_scan_bus(0, &mcf_pci_ops, NULL);
316 rootbus->resource[0] = &mcf_pci_io;
317 rootbus->resource[1] = &mcf_pci_mem;
318
319 pci_fixup_irqs(pci_common_swizzle, mcf_pci_map_irq);
320 pci_bus_size_bridges(rootbus);
321 pci_bus_assign_resources(rootbus);
322 pci_enable_bridges(rootbus);
323 pci_bus_add_devices(rootbus);
324 return 0;
325}
326
327subsys_initcall(mcf_pci_init);
diff --git a/arch/m68k/platform/coldfire/pinmux.c b/arch/m68k/platform/coldfire/pinmux.c
deleted file mode 100644
index 8c62b825939f..000000000000
--- a/arch/m68k/platform/coldfire/pinmux.c
+++ /dev/null
@@ -1,28 +0,0 @@
1/*
2 * Coldfire generic GPIO pinmux support.
3 *
4 * (C) Copyright 2009, Steven King <sfking@fdwdc.com>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; version 2 of the License.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 */
16
17#include <linux/kernel.h>
18
19#include <asm/pinmux.h>
20
21int mcf_pinmux_request(unsigned pinmux, unsigned func)
22{
23 return 0;
24}
25
26void mcf_pinmux_release(unsigned pinmux, unsigned func)
27{
28}
diff --git a/arch/m68k/platform/coldfire/pit.c b/arch/m68k/platform/coldfire/pit.c
index e62dbbcb10f6..e8f3b97b0f77 100644
--- a/arch/m68k/platform/coldfire/pit.c
+++ b/arch/m68k/platform/coldfire/pit.c
@@ -93,7 +93,7 @@ struct clock_event_device cf_pit_clockevent = {
93 .set_mode = init_cf_pit_timer, 93 .set_mode = init_cf_pit_timer,
94 .set_next_event = cf_pit_next_event, 94 .set_next_event = cf_pit_next_event,
95 .shift = 32, 95 .shift = 32,
96 .irq = MCFINT_VECBASE + MCFINT_PIT1, 96 .irq = MCF_IRQ_PIT1,
97}; 97};
98 98
99 99
@@ -159,7 +159,7 @@ void hw_timer_init(irq_handler_t handler)
159 clockevent_delta2ns(0x3f, &cf_pit_clockevent); 159 clockevent_delta2ns(0x3f, &cf_pit_clockevent);
160 clockevents_register_device(&cf_pit_clockevent); 160 clockevents_register_device(&cf_pit_clockevent);
161 161
162 setup_irq(MCFINT_VECBASE + MCFINT_PIT1, &pit_irq); 162 setup_irq(MCF_IRQ_PIT1, &pit_irq);
163 163
164 clocksource_register_hz(&pit_clk, FREQ); 164 clocksource_register_hz(&pit_clk, FREQ);
165} 165}
diff --git a/arch/m68k/platform/coldfire/timers.c b/arch/m68k/platform/coldfire/timers.c
index ed96ce50d79f..0a273e75408c 100644
--- a/arch/m68k/platform/coldfire/timers.c
+++ b/arch/m68k/platform/coldfire/timers.c
@@ -36,7 +36,7 @@
36 */ 36 */
37void coldfire_profile_init(void); 37void coldfire_profile_init(void);
38 38
39#if defined(CONFIG_M532x) 39#if defined(CONFIG_M532x) || defined(CONFIG_M5441x)
40#define __raw_readtrr __raw_readl 40#define __raw_readtrr __raw_readl
41#define __raw_writetrr __raw_writel 41#define __raw_writetrr __raw_writel
42#else 42#else
diff --git a/arch/mips/kernel/kspd.c b/arch/mips/kernel/kspd.c
index 84d0639e4580..b77f56bbb477 100644
--- a/arch/mips/kernel/kspd.c
+++ b/arch/mips/kernel/kspd.c
@@ -323,7 +323,7 @@ static void sp_cleanup(void)
323 fdt = files_fdtable(files); 323 fdt = files_fdtable(files);
324 for (;;) { 324 for (;;) {
325 unsigned long set; 325 unsigned long set;
326 i = j * __NFDBITS; 326 i = j * BITS_PER_LONG;
327 if (i >= fdt->max_fds) 327 if (i >= fdt->max_fds)
328 break; 328 break;
329 set = fdt->open_fds[j++]; 329 set = fdt->open_fds[j++];
diff --git a/arch/powerpc/Makefile b/arch/powerpc/Makefile
index 950d1f7a5a39..159e94f4b22a 100644
--- a/arch/powerpc/Makefile
+++ b/arch/powerpc/Makefile
@@ -149,7 +149,6 @@ core-$(CONFIG_KVM) += arch/powerpc/kvm/
149core-$(CONFIG_PERF_EVENTS) += arch/powerpc/perf/ 149core-$(CONFIG_PERF_EVENTS) += arch/powerpc/perf/
150 150
151drivers-$(CONFIG_OPROFILE) += arch/powerpc/oprofile/ 151drivers-$(CONFIG_OPROFILE) += arch/powerpc/oprofile/
152drivers-$(CONFIG_CRYPTO_DEV_NX) += drivers/crypto/nx/
153 152
154# Default to zImage, override when needed 153# Default to zImage, override when needed
155all: zImage 154all: zImage
diff --git a/arch/s390/crypto/crypto_des.h b/arch/s390/crypto/crypto_des.h
deleted file mode 100644
index 6210457ceebb..000000000000
--- a/arch/s390/crypto/crypto_des.h
+++ /dev/null
@@ -1,18 +0,0 @@
1/*
2 * Cryptographic API.
3 *
4 * Function for checking keys for the DES and Tripple DES Encryption
5 * algorithms.
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11 *
12 */
13#ifndef __CRYPTO_DES_H__
14#define __CRYPTO_DES_H__
15
16extern int crypto_des_check_key(const u8*, unsigned int, u32*);
17
18#endif /*__CRYPTO_DES_H__*/
diff --git a/arch/sh/include/asm/siu.h b/arch/sh/include/asm/siu.h
index 1d95c78808d1..580b7ac228b7 100644
--- a/arch/sh/include/asm/siu.h
+++ b/arch/sh/include/asm/siu.h
@@ -14,7 +14,6 @@
14struct device; 14struct device;
15 15
16struct siu_platform { 16struct siu_platform {
17 struct device *dma_dev;
18 unsigned int dma_slave_tx_a; 17 unsigned int dma_slave_tx_a;
19 unsigned int dma_slave_rx_a; 18 unsigned int dma_slave_rx_a;
20 unsigned int dma_slave_tx_b; 19 unsigned int dma_slave_tx_b;
diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7722.c b/arch/sh/kernel/cpu/sh4a/setup-sh7722.c
index 0f5a21907da6..65786c7f5ded 100644
--- a/arch/sh/kernel/cpu/sh4a/setup-sh7722.c
+++ b/arch/sh/kernel/cpu/sh4a/setup-sh7722.c
@@ -512,7 +512,6 @@ static struct platform_device tmu2_device = {
512}; 512};
513 513
514static struct siu_platform siu_platform_data = { 514static struct siu_platform siu_platform_data = {
515 .dma_dev = &dma_device.dev,
516 .dma_slave_tx_a = SHDMA_SLAVE_SIUA_TX, 515 .dma_slave_tx_a = SHDMA_SLAVE_SIUA_TX,
517 .dma_slave_rx_a = SHDMA_SLAVE_SIUA_RX, 516 .dma_slave_rx_a = SHDMA_SLAVE_SIUA_RX,
518 .dma_slave_tx_b = SHDMA_SLAVE_SIUB_TX, 517 .dma_slave_tx_b = SHDMA_SLAVE_SIUB_TX,
diff --git a/arch/sparc/include/asm/fixmap.h b/arch/sparc/include/asm/fixmap.h
deleted file mode 100644
index f18fc0755adf..000000000000
--- a/arch/sparc/include/asm/fixmap.h
+++ /dev/null
@@ -1,110 +0,0 @@
1/*
2 * fixmap.h: compile-time virtual memory allocation
3 *
4 * This file is subject to the terms and conditions of the GNU General Public
5 * License. See the file "COPYING" in the main directory of this archive
6 * for more details.
7 *
8 * Copyright (C) 1998 Ingo Molnar
9 *
10 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
11 */
12
13#ifndef _ASM_FIXMAP_H
14#define _ASM_FIXMAP_H
15
16#include <linux/kernel.h>
17#include <asm/page.h>
18#ifdef CONFIG_HIGHMEM
19#include <linux/threads.h>
20#include <asm/kmap_types.h>
21#endif
22
23/*
24 * Here we define all the compile-time 'special' virtual
25 * addresses. The point is to have a constant address at
26 * compile time, but to set the physical address only
27 * in the boot process. We allocate these special addresses
28 * from the top of unused virtual memory (0xfd000000 - 1 page) backwards.
29 * Also this lets us do fail-safe vmalloc(), we
30 * can guarantee that these special addresses and
31 * vmalloc()-ed addresses never overlap.
32 *
33 * these 'compile-time allocated' memory buffers are
34 * fixed-size 4k pages. (or larger if used with an increment
35 * highger than 1) use fixmap_set(idx,phys) to associate
36 * physical memory with fixmap indices.
37 *
38 * TLB entries of such buffers will not be flushed across
39 * task switches.
40 */
41
42/*
43 * on UP currently we will have no trace of the fixmap mechanism,
44 * no page table allocations, etc. This might change in the
45 * future, say framebuffers for the console driver(s) could be
46 * fix-mapped?
47 */
48enum fixed_addresses {
49 FIX_HOLE,
50#ifdef CONFIG_HIGHMEM
51 FIX_KMAP_BEGIN,
52 FIX_KMAP_END = FIX_KMAP_BEGIN+(KM_TYPE_NR*NR_CPUS)-1,
53#endif
54 __end_of_fixed_addresses
55};
56
57extern void __set_fixmap (enum fixed_addresses idx,
58 unsigned long phys, pgprot_t flags);
59
60#define set_fixmap(idx, phys) \
61 __set_fixmap(idx, phys, PAGE_KERNEL)
62/*
63 * Some hardware wants to get fixmapped without caching.
64 */
65#define set_fixmap_nocache(idx, phys) \
66 __set_fixmap(idx, phys, PAGE_KERNEL_NOCACHE)
67/*
68 * used by vmalloc.c.
69 *
70 * Leave one empty page between IO pages at 0xfd000000 and
71 * the start of the fixmap.
72 */
73#define FIXADDR_TOP (0xfcfff000UL)
74#define FIXADDR_SIZE ((__end_of_fixed_addresses) << PAGE_SHIFT)
75#define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE)
76
77#define __fix_to_virt(x) (FIXADDR_TOP - ((x) << PAGE_SHIFT))
78#define __virt_to_fix(x) ((FIXADDR_TOP - ((x)&PAGE_MASK)) >> PAGE_SHIFT)
79
80extern void __this_fixmap_does_not_exist(void);
81
82/*
83 * 'index to address' translation. If anyone tries to use the idx
84 * directly without tranlation, we catch the bug with a NULL-deference
85 * kernel oops. Illegal ranges of incoming indices are caught too.
86 */
87static inline unsigned long fix_to_virt(const unsigned int idx)
88{
89 /*
90 * this branch gets completely eliminated after inlining,
91 * except when someone tries to use fixaddr indices in an
92 * illegal way. (such as mixing up address types or using
93 * out-of-range indices).
94 *
95 * If it doesn't get removed, the linker will complain
96 * loudly with a reasonably clear error message..
97 */
98 if (idx >= __end_of_fixed_addresses)
99 __this_fixmap_does_not_exist();
100
101 return __fix_to_virt(idx);
102}
103
104static inline unsigned long virt_to_fix(const unsigned long vaddr)
105{
106 BUG_ON(vaddr >= FIXADDR_TOP || vaddr < FIXADDR_START);
107 return __virt_to_fix(vaddr);
108}
109
110#endif
diff --git a/arch/sparc/include/asm/highmem.h b/arch/sparc/include/asm/highmem.h
index 3b6e00dd96e5..4f9e15c757e2 100644
--- a/arch/sparc/include/asm/highmem.h
+++ b/arch/sparc/include/asm/highmem.h
@@ -21,7 +21,6 @@
21#ifdef __KERNEL__ 21#ifdef __KERNEL__
22 22
23#include <linux/interrupt.h> 23#include <linux/interrupt.h>
24#include <asm/fixmap.h>
25#include <asm/vaddrs.h> 24#include <asm/vaddrs.h>
26#include <asm/kmap_types.h> 25#include <asm/kmap_types.h>
27#include <asm/pgtable.h> 26#include <asm/pgtable.h>
@@ -29,7 +28,6 @@
29/* declarations for highmem.c */ 28/* declarations for highmem.c */
30extern unsigned long highstart_pfn, highend_pfn; 29extern unsigned long highstart_pfn, highend_pfn;
31 30
32extern pte_t *kmap_pte;
33extern pgprot_t kmap_prot; 31extern pgprot_t kmap_prot;
34extern pte_t *pkmap_page_table; 32extern pte_t *pkmap_page_table;
35 33
@@ -72,7 +70,6 @@ static inline void kunmap(struct page *page)
72 70
73extern void *kmap_atomic(struct page *page); 71extern void *kmap_atomic(struct page *page);
74extern void __kunmap_atomic(void *kvaddr); 72extern void __kunmap_atomic(void *kvaddr);
75extern struct page *kmap_atomic_to_page(void *vaddr);
76 73
77#define flush_cache_kmaps() flush_cache_all() 74#define flush_cache_kmaps() flush_cache_all()
78 75
diff --git a/arch/sparc/include/asm/leon.h b/arch/sparc/include/asm/leon.h
index 3375c6293893..15a716934e4d 100644
--- a/arch/sparc/include/asm/leon.h
+++ b/arch/sparc/include/asm/leon.h
@@ -82,7 +82,6 @@ static inline unsigned long leon_load_reg(unsigned long paddr)
82#define LEON_BYPASS_LOAD_PA(x) leon_load_reg((unsigned long)(x)) 82#define LEON_BYPASS_LOAD_PA(x) leon_load_reg((unsigned long)(x))
83#define LEON_BYPASS_STORE_PA(x, v) leon_store_reg((unsigned long)(x), (unsigned long)(v)) 83#define LEON_BYPASS_STORE_PA(x, v) leon_store_reg((unsigned long)(x), (unsigned long)(v))
84 84
85extern void leon_init(void);
86extern void leon_switch_mm(void); 85extern void leon_switch_mm(void);
87extern void leon_init_IRQ(void); 86extern void leon_init_IRQ(void);
88 87
diff --git a/arch/sparc/include/asm/mmu_context_32.h b/arch/sparc/include/asm/mmu_context_32.h
index 01456c900720..2df2a9be8f6d 100644
--- a/arch/sparc/include/asm/mmu_context_32.h
+++ b/arch/sparc/include/asm/mmu_context_32.h
@@ -9,14 +9,12 @@ static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
9{ 9{
10} 10}
11 11
12/* 12/* Initialize a new mmu context. This is invoked when a new
13 * Initialize a new mmu context. This is invoked when a new
14 * address space instance (unique or shared) is instantiated. 13 * address space instance (unique or shared) is instantiated.
15 */ 14 */
16#define init_new_context(tsk, mm) (((mm)->context = NO_CONTEXT), 0) 15int init_new_context(struct task_struct *tsk, struct mm_struct *mm);
17 16
18/* 17/* Destroy a dead context. This occurs when mmput drops the
19 * Destroy a dead context. This occurs when mmput drops the
20 * mm_users count to zero, the mmaps have been released, and 18 * mm_users count to zero, the mmaps have been released, and
21 * all the page tables have been flushed. Our job is to destroy 19 * all the page tables have been flushed. Our job is to destroy
22 * any remaining processor-specific state. 20 * any remaining processor-specific state.
diff --git a/arch/sparc/include/asm/page_32.h b/arch/sparc/include/asm/page_32.h
index fab78a308ebf..f82a1f36b655 100644
--- a/arch/sparc/include/asm/page_32.h
+++ b/arch/sparc/include/asm/page_32.h
@@ -107,8 +107,7 @@ typedef unsigned long iopgprot_t;
107 107
108typedef struct page *pgtable_t; 108typedef struct page *pgtable_t;
109 109
110extern unsigned long sparc_unmapped_base; 110#define TASK_UNMAPPED_BASE 0x50000000
111#define TASK_UNMAPPED_BASE sparc_unmapped_base
112 111
113#else /* !(__ASSEMBLY__) */ 112#else /* !(__ASSEMBLY__) */
114 113
diff --git a/arch/sparc/include/asm/pgalloc_32.h b/arch/sparc/include/asm/pgalloc_32.h
index e5b169b46d21..9b1c36de0f18 100644
--- a/arch/sparc/include/asm/pgalloc_32.h
+++ b/arch/sparc/include/asm/pgalloc_32.h
@@ -11,28 +11,15 @@
11 11
12struct page; 12struct page;
13 13
14extern struct pgtable_cache_struct { 14void *srmmu_get_nocache(int size, int align);
15 unsigned long *pgd_cache; 15void srmmu_free_nocache(void *addr, int size);
16 unsigned long *pte_cache;
17 unsigned long pgtable_cache_sz;
18 unsigned long pgd_cache_sz;
19} pgt_quicklists;
20
21unsigned long srmmu_get_nocache(int size, int align);
22void srmmu_free_nocache(unsigned long vaddr, int size);
23
24#define pgd_quicklist (pgt_quicklists.pgd_cache)
25#define pmd_quicklist ((unsigned long *)0)
26#define pte_quicklist (pgt_quicklists.pte_cache)
27#define pgtable_cache_size (pgt_quicklists.pgtable_cache_sz)
28#define pgd_cache_size (pgt_quicklists.pgd_cache_sz)
29 16
30#define check_pgt_cache() do { } while (0) 17#define check_pgt_cache() do { } while (0)
31 18
32pgd_t *get_pgd_fast(void); 19pgd_t *get_pgd_fast(void);
33static inline void free_pgd_fast(pgd_t *pgd) 20static inline void free_pgd_fast(pgd_t *pgd)
34{ 21{
35 srmmu_free_nocache((unsigned long)pgd, SRMMU_PGD_TABLE_SIZE); 22 srmmu_free_nocache(pgd, SRMMU_PGD_TABLE_SIZE);
36} 23}
37 24
38#define pgd_free(mm, pgd) free_pgd_fast(pgd) 25#define pgd_free(mm, pgd) free_pgd_fast(pgd)
@@ -50,13 +37,13 @@ static inline void pgd_set(pgd_t * pgdp, pmd_t * pmdp)
50static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, 37static inline pmd_t *pmd_alloc_one(struct mm_struct *mm,
51 unsigned long address) 38 unsigned long address)
52{ 39{
53 return (pmd_t *)srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, 40 return srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE,
54 SRMMU_PMD_TABLE_SIZE); 41 SRMMU_PMD_TABLE_SIZE);
55} 42}
56 43
57static inline void free_pmd_fast(pmd_t * pmd) 44static inline void free_pmd_fast(pmd_t * pmd)
58{ 45{
59 srmmu_free_nocache((unsigned long)pmd, SRMMU_PMD_TABLE_SIZE); 46 srmmu_free_nocache(pmd, SRMMU_PMD_TABLE_SIZE);
60} 47}
61 48
62#define pmd_free(mm, pmd) free_pmd_fast(pmd) 49#define pmd_free(mm, pmd) free_pmd_fast(pmd)
@@ -73,13 +60,13 @@ pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address);
73static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, 60static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
74 unsigned long address) 61 unsigned long address)
75{ 62{
76 return (pte_t *)srmmu_get_nocache(PTE_SIZE, PTE_SIZE); 63 return srmmu_get_nocache(PTE_SIZE, PTE_SIZE);
77} 64}
78 65
79 66
80static inline void free_pte_fast(pte_t *pte) 67static inline void free_pte_fast(pte_t *pte)
81{ 68{
82 srmmu_free_nocache((unsigned long)pte, PTE_SIZE); 69 srmmu_free_nocache(pte, PTE_SIZE);
83} 70}
84 71
85#define pte_free_kernel(mm, pte) free_pte_fast(pte) 72#define pte_free_kernel(mm, pte) free_pte_fast(pte)
diff --git a/arch/sparc/include/asm/pgtable_32.h b/arch/sparc/include/asm/pgtable_32.h
index cbbbed5cb3aa..6fc13483f702 100644
--- a/arch/sparc/include/asm/pgtable_32.h
+++ b/arch/sparc/include/asm/pgtable_32.h
@@ -52,8 +52,9 @@ extern unsigned long calc_highpages(void);
52#define PAGE_READONLY SRMMU_PAGE_RDONLY 52#define PAGE_READONLY SRMMU_PAGE_RDONLY
53#define PAGE_KERNEL SRMMU_PAGE_KERNEL 53#define PAGE_KERNEL SRMMU_PAGE_KERNEL
54 54
55/* Top-level page directory */ 55/* Top-level page directory - dummy used by init-mm.
56extern pgd_t swapper_pg_dir[1024]; 56 * srmmu.c will assign the real one (which is dynamically sized) */
57#define swapper_pg_dir NULL
57 58
58extern void paging_init(void); 59extern void paging_init(void);
59 60
@@ -78,8 +79,6 @@ extern unsigned long ptr_in_current_pgd;
78#define __S110 PAGE_SHARED 79#define __S110 PAGE_SHARED
79#define __S111 PAGE_SHARED 80#define __S111 PAGE_SHARED
80 81
81extern int num_contexts;
82
83/* First physical page can be anywhere, the following is needed so that 82/* First physical page can be anywhere, the following is needed so that
84 * va-->pa and vice versa conversions work properly without performance 83 * va-->pa and vice versa conversions work properly without performance
85 * hit for all __pa()/__va() operations. 84 * hit for all __pa()/__va() operations.
@@ -88,18 +87,11 @@ extern unsigned long phys_base;
88extern unsigned long pfn_base; 87extern unsigned long pfn_base;
89 88
90/* 89/*
91 * BAD_PAGETABLE is used when we need a bogus page-table, while
92 * BAD_PAGE is used for a bogus page.
93 *
94 * ZERO_PAGE is a global shared page that is always zero: used 90 * ZERO_PAGE is a global shared page that is always zero: used
95 * for zero-mapped memory areas etc.. 91 * for zero-mapped memory areas etc..
96 */ 92 */
97extern pte_t * __bad_pagetable(void);
98extern pte_t __bad_page(void);
99extern unsigned long empty_zero_page; 93extern unsigned long empty_zero_page;
100 94
101#define BAD_PAGETABLE __bad_pagetable()
102#define BAD_PAGE __bad_page()
103#define ZERO_PAGE(vaddr) (virt_to_page(&empty_zero_page)) 95#define ZERO_PAGE(vaddr) (virt_to_page(&empty_zero_page))
104 96
105/* 97/*
@@ -398,36 +390,6 @@ static inline pte_t pgoff_to_pte(unsigned long pgoff)
398 */ 390 */
399#define PTE_FILE_MAX_BITS 24 391#define PTE_FILE_MAX_BITS 24
400 392
401/*
402 */
403struct ctx_list {
404 struct ctx_list *next;
405 struct ctx_list *prev;
406 unsigned int ctx_number;
407 struct mm_struct *ctx_mm;
408};
409
410extern struct ctx_list *ctx_list_pool; /* Dynamically allocated */
411extern struct ctx_list ctx_free; /* Head of free list */
412extern struct ctx_list ctx_used; /* Head of used contexts list */
413
414#define NO_CONTEXT -1
415
416static inline void remove_from_ctx_list(struct ctx_list *entry)
417{
418 entry->next->prev = entry->prev;
419 entry->prev->next = entry->next;
420}
421
422static inline void add_to_ctx_list(struct ctx_list *head, struct ctx_list *entry)
423{
424 entry->next = head;
425 (entry->prev = head->prev)->next = entry;
426 head->prev = entry;
427}
428#define add_to_free_ctxlist(entry) add_to_ctx_list(&ctx_free, entry)
429#define add_to_used_ctxlist(entry) add_to_ctx_list(&ctx_used, entry)
430
431static inline unsigned long 393static inline unsigned long
432__get_phys (unsigned long addr) 394__get_phys (unsigned long addr)
433{ 395{
diff --git a/arch/sparc/include/asm/vaddrs.h b/arch/sparc/include/asm/vaddrs.h
index da6535d88a72..c3dbcf902034 100644
--- a/arch/sparc/include/asm/vaddrs.h
+++ b/arch/sparc/include/asm/vaddrs.h
@@ -30,6 +30,28 @@
30 */ 30 */
31#define SRMMU_NOCACHE_ALCRATIO 64 /* 256 pages per 64MB of system RAM */ 31#define SRMMU_NOCACHE_ALCRATIO 64 /* 256 pages per 64MB of system RAM */
32 32
33#ifndef __ASSEMBLY__
34#include <asm/kmap_types.h>
35
36enum fixed_addresses {
37 FIX_HOLE,
38#ifdef CONFIG_HIGHMEM
39 FIX_KMAP_BEGIN,
40 FIX_KMAP_END = (KM_TYPE_NR * NR_CPUS),
41#endif
42 __end_of_fixed_addresses
43};
44#endif
45
46/* Leave one empty page between IO pages at 0xfd000000 and
47 * the top of the fixmap.
48 */
49#define FIXADDR_TOP (0xfcfff000UL)
50#define FIXADDR_SIZE ((FIX_KMAP_END + 1) << PAGE_SHIFT)
51#define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE)
52
53#define __fix_to_virt(x) (FIXADDR_TOP - ((x) << PAGE_SHIFT))
54
33#define SUN4M_IOBASE_VADDR 0xfd000000 /* Base for mapping pages */ 55#define SUN4M_IOBASE_VADDR 0xfd000000 /* Base for mapping pages */
34#define IOBASE_VADDR 0xfe000000 56#define IOBASE_VADDR 0xfe000000
35#define IOBASE_END 0xfe600000 57#define IOBASE_END 0xfe600000
diff --git a/arch/sparc/kernel/head_32.S b/arch/sparc/kernel/head_32.S
index afeb1d770303..3d92c0a8f6c4 100644
--- a/arch/sparc/kernel/head_32.S
+++ b/arch/sparc/kernel/head_32.S
@@ -58,8 +58,6 @@ sun4e_notsup:
58/* This was the only reasonable way I could think of to properly align 58/* This was the only reasonable way I could think of to properly align
59 * these page-table data structures. 59 * these page-table data structures.
60 */ 60 */
61 .globl swapper_pg_dir
62swapper_pg_dir: .skip PAGE_SIZE
63 .globl empty_zero_page 61 .globl empty_zero_page
64empty_zero_page: .skip PAGE_SIZE 62empty_zero_page: .skip PAGE_SIZE
65 63
diff --git a/arch/sparc/kernel/leon_kernel.c b/arch/sparc/kernel/leon_kernel.c
index e34e2c40c060..f8b6eee40bde 100644
--- a/arch/sparc/kernel/leon_kernel.c
+++ b/arch/sparc/kernel/leon_kernel.c
@@ -486,17 +486,6 @@ void __init leon_trans_init(struct device_node *dp)
486 } 486 }
487} 487}
488 488
489void __initdata (*prom_amba_init)(struct device_node *dp, struct device_node ***nextp) = 0;
490
491void __init leon_node_init(struct device_node *dp, struct device_node ***nextp)
492{
493 if (prom_amba_init &&
494 strcmp(dp->type, "ambapp") == 0 &&
495 strcmp(dp->name, "ambapp0") == 0) {
496 prom_amba_init(dp, nextp);
497 }
498}
499
500#ifdef CONFIG_SMP 489#ifdef CONFIG_SMP
501void leon_clear_profile_irq(int cpu) 490void leon_clear_profile_irq(int cpu)
502{ 491{
@@ -522,8 +511,3 @@ void __init leon_init_IRQ(void)
522 sparc_config.clear_clock_irq = leon_clear_clock_irq; 511 sparc_config.clear_clock_irq = leon_clear_clock_irq;
523 sparc_config.load_profile_irq = leon_load_profile_irq; 512 sparc_config.load_profile_irq = leon_load_profile_irq;
524} 513}
525
526void __init leon_init(void)
527{
528 of_pdt_build_more = &leon_node_init;
529}
diff --git a/arch/sparc/kernel/process_32.c b/arch/sparc/kernel/process_32.c
index cb36e82dcd5d..14006d8aca28 100644
--- a/arch/sparc/kernel/process_32.c
+++ b/arch/sparc/kernel/process_32.c
@@ -333,9 +333,6 @@ int copy_thread(unsigned long clone_flags, unsigned long sp,
333 put_psr(get_psr() | PSR_EF); 333 put_psr(get_psr() | PSR_EF);
334 fpsave(&p->thread.float_regs[0], &p->thread.fsr, 334 fpsave(&p->thread.float_regs[0], &p->thread.fsr,
335 &p->thread.fpqueue[0], &p->thread.fpqdepth); 335 &p->thread.fpqueue[0], &p->thread.fpqdepth);
336#ifdef CONFIG_SMP
337 clear_thread_flag(TIF_USEDFPU);
338#endif
339 } 336 }
340 337
341 /* 338 /*
@@ -413,6 +410,7 @@ int copy_thread(unsigned long clone_flags, unsigned long sp,
413#ifdef CONFIG_SMP 410#ifdef CONFIG_SMP
414 /* FPU must be disabled on SMP. */ 411 /* FPU must be disabled on SMP. */
415 childregs->psr &= ~PSR_EF; 412 childregs->psr &= ~PSR_EF;
413 clear_tsk_thread_flag(p, TIF_USEDFPU);
416#endif 414#endif
417 415
418 /* Set the return value for the child. */ 416 /* Set the return value for the child. */
diff --git a/arch/sparc/kernel/setup_32.c b/arch/sparc/kernel/setup_32.c
index efe3e64bba38..38bf80a22f02 100644
--- a/arch/sparc/kernel/setup_32.c
+++ b/arch/sparc/kernel/setup_32.c
@@ -371,7 +371,6 @@ void __init setup_arch(char **cmdline_p)
371 (*(linux_dbvec->teach_debugger))(); 371 (*(linux_dbvec->teach_debugger))();
372 } 372 }
373 373
374 init_mm.context = (unsigned long) NO_CONTEXT;
375 init_task.thread.kregs = &fake_swapper_regs; 374 init_task.thread.kregs = &fake_swapper_regs;
376 375
377 /* Run-time patch instructions to match the cpu model */ 376 /* Run-time patch instructions to match the cpu model */
diff --git a/arch/sparc/kernel/sys_sparc_64.c b/arch/sparc/kernel/sys_sparc_64.c
index 275f74fd6f6a..c38e5aaae56f 100644
--- a/arch/sparc/kernel/sys_sparc_64.c
+++ b/arch/sparc/kernel/sys_sparc_64.c
@@ -66,23 +66,6 @@ static inline int invalid_64bit_range(unsigned long addr, unsigned long len)
66 return 0; 66 return 0;
67} 67}
68 68
69/* Does start,end straddle the VA-space hole? */
70static inline int straddles_64bit_va_hole(unsigned long start, unsigned long end)
71{
72 unsigned long va_exclude_start, va_exclude_end;
73
74 va_exclude_start = VA_EXCLUDE_START;
75 va_exclude_end = VA_EXCLUDE_END;
76
77 if (likely(start < va_exclude_start && end < va_exclude_start))
78 return 0;
79
80 if (likely(start >= va_exclude_end && end >= va_exclude_end))
81 return 0;
82
83 return 1;
84}
85
86/* These functions differ from the default implementations in 69/* These functions differ from the default implementations in
87 * mm/mmap.c in two ways: 70 * mm/mmap.c in two ways:
88 * 71 *
diff --git a/arch/sparc/lib/NG2memcpy.S b/arch/sparc/lib/NG2memcpy.S
index 0aed75653b50..03eadf66b0d3 100644
--- a/arch/sparc/lib/NG2memcpy.S
+++ b/arch/sparc/lib/NG2memcpy.S
@@ -90,49 +90,49 @@
90 faligndata %x7, %x8, %f14; 90 faligndata %x7, %x8, %f14;
91 91
92#define FREG_MOVE_1(x0) \ 92#define FREG_MOVE_1(x0) \
93 fmovd %x0, %f0; 93 fsrc2 %x0, %f0;
94#define FREG_MOVE_2(x0, x1) \ 94#define FREG_MOVE_2(x0, x1) \
95 fmovd %x0, %f0; \ 95 fsrc2 %x0, %f0; \
96 fmovd %x1, %f2; 96 fsrc2 %x1, %f2;
97#define FREG_MOVE_3(x0, x1, x2) \ 97#define FREG_MOVE_3(x0, x1, x2) \
98 fmovd %x0, %f0; \ 98 fsrc2 %x0, %f0; \
99 fmovd %x1, %f2; \ 99 fsrc2 %x1, %f2; \
100 fmovd %x2, %f4; 100 fsrc2 %x2, %f4;
101#define FREG_MOVE_4(x0, x1, x2, x3) \ 101#define FREG_MOVE_4(x0, x1, x2, x3) \
102 fmovd %x0, %f0; \ 102 fsrc2 %x0, %f0; \
103 fmovd %x1, %f2; \ 103 fsrc2 %x1, %f2; \
104 fmovd %x2, %f4; \ 104 fsrc2 %x2, %f4; \
105 fmovd %x3, %f6; 105 fsrc2 %x3, %f6;
106#define FREG_MOVE_5(x0, x1, x2, x3, x4) \ 106#define FREG_MOVE_5(x0, x1, x2, x3, x4) \
107 fmovd %x0, %f0; \ 107 fsrc2 %x0, %f0; \
108 fmovd %x1, %f2; \ 108 fsrc2 %x1, %f2; \
109 fmovd %x2, %f4; \ 109 fsrc2 %x2, %f4; \
110 fmovd %x3, %f6; \ 110 fsrc2 %x3, %f6; \
111 fmovd %x4, %f8; 111 fsrc2 %x4, %f8;
112#define FREG_MOVE_6(x0, x1, x2, x3, x4, x5) \ 112#define FREG_MOVE_6(x0, x1, x2, x3, x4, x5) \
113 fmovd %x0, %f0; \ 113 fsrc2 %x0, %f0; \
114 fmovd %x1, %f2; \ 114 fsrc2 %x1, %f2; \
115 fmovd %x2, %f4; \ 115 fsrc2 %x2, %f4; \
116 fmovd %x3, %f6; \ 116 fsrc2 %x3, %f6; \
117 fmovd %x4, %f8; \ 117 fsrc2 %x4, %f8; \
118 fmovd %x5, %f10; 118 fsrc2 %x5, %f10;
119#define FREG_MOVE_7(x0, x1, x2, x3, x4, x5, x6) \ 119#define FREG_MOVE_7(x0, x1, x2, x3, x4, x5, x6) \
120 fmovd %x0, %f0; \ 120 fsrc2 %x0, %f0; \
121 fmovd %x1, %f2; \ 121 fsrc2 %x1, %f2; \
122 fmovd %x2, %f4; \ 122 fsrc2 %x2, %f4; \
123 fmovd %x3, %f6; \ 123 fsrc2 %x3, %f6; \
124 fmovd %x4, %f8; \ 124 fsrc2 %x4, %f8; \
125 fmovd %x5, %f10; \ 125 fsrc2 %x5, %f10; \
126 fmovd %x6, %f12; 126 fsrc2 %x6, %f12;
127#define FREG_MOVE_8(x0, x1, x2, x3, x4, x5, x6, x7) \ 127#define FREG_MOVE_8(x0, x1, x2, x3, x4, x5, x6, x7) \
128 fmovd %x0, %f0; \ 128 fsrc2 %x0, %f0; \
129 fmovd %x1, %f2; \ 129 fsrc2 %x1, %f2; \
130 fmovd %x2, %f4; \ 130 fsrc2 %x2, %f4; \
131 fmovd %x3, %f6; \ 131 fsrc2 %x3, %f6; \
132 fmovd %x4, %f8; \ 132 fsrc2 %x4, %f8; \
133 fmovd %x5, %f10; \ 133 fsrc2 %x5, %f10; \
134 fmovd %x6, %f12; \ 134 fsrc2 %x6, %f12; \
135 fmovd %x7, %f14; 135 fsrc2 %x7, %f14;
136#define FREG_LOAD_1(base, x0) \ 136#define FREG_LOAD_1(base, x0) \
137 EX_LD(LOAD(ldd, base + 0x00, %x0)) 137 EX_LD(LOAD(ldd, base + 0x00, %x0))
138#define FREG_LOAD_2(base, x0, x1) \ 138#define FREG_LOAD_2(base, x0, x1) \
diff --git a/arch/sparc/lib/U1memcpy.S b/arch/sparc/lib/U1memcpy.S
index bafd2fc07acb..b67142b7768e 100644
--- a/arch/sparc/lib/U1memcpy.S
+++ b/arch/sparc/lib/U1memcpy.S
@@ -109,7 +109,7 @@
109#define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \ 109#define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \
110 subcc %left, 8, %left; \ 110 subcc %left, 8, %left; \
111 bl,pn %xcc, 95f; \ 111 bl,pn %xcc, 95f; \
112 fsrc1 %f0, %f1; 112 fsrc2 %f0, %f1;
113 113
114#define UNEVEN_VISCHUNK(dest, f0, f1, left) \ 114#define UNEVEN_VISCHUNK(dest, f0, f1, left) \
115 UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \ 115 UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \
@@ -201,7 +201,7 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
201 andn %o1, (0x40 - 1), %o1 201 andn %o1, (0x40 - 1), %o1
202 and %g2, 7, %g2 202 and %g2, 7, %g2
203 andncc %g3, 0x7, %g3 203 andncc %g3, 0x7, %g3
204 fmovd %f0, %f2 204 fsrc2 %f0, %f2
205 sub %g3, 0x8, %g3 205 sub %g3, 0x8, %g3
206 sub %o2, %GLOBAL_SPARE, %o2 206 sub %o2, %GLOBAL_SPARE, %o2
207 207
diff --git a/arch/sparc/lib/copy_page.S b/arch/sparc/lib/copy_page.S
index b243d3b606ba..4d2df328e514 100644
--- a/arch/sparc/lib/copy_page.S
+++ b/arch/sparc/lib/copy_page.S
@@ -34,10 +34,10 @@
34#endif 34#endif
35 35
36#define TOUCH(reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7) \ 36#define TOUCH(reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7) \
37 fmovd %reg0, %f48; fmovd %reg1, %f50; \ 37 fsrc2 %reg0, %f48; fsrc2 %reg1, %f50; \
38 fmovd %reg2, %f52; fmovd %reg3, %f54; \ 38 fsrc2 %reg2, %f52; fsrc2 %reg3, %f54; \
39 fmovd %reg4, %f56; fmovd %reg5, %f58; \ 39 fsrc2 %reg4, %f56; fsrc2 %reg5, %f58; \
40 fmovd %reg6, %f60; fmovd %reg7, %f62; 40 fsrc2 %reg6, %f60; fsrc2 %reg7, %f62;
41 41
42 .text 42 .text
43 43
@@ -104,60 +104,60 @@ cheetah_copy_page_insn:
104 prefetch [%o1 + 0x140], #one_read 104 prefetch [%o1 + 0x140], #one_read
105 ldd [%o1 + 0x010], %f4 105 ldd [%o1 + 0x010], %f4
106 prefetch [%o1 + 0x180], #one_read 106 prefetch [%o1 + 0x180], #one_read
107 fmovd %f0, %f16 107 fsrc2 %f0, %f16
108 ldd [%o1 + 0x018], %f6 108 ldd [%o1 + 0x018], %f6
109 fmovd %f2, %f18 109 fsrc2 %f2, %f18
110 ldd [%o1 + 0x020], %f8 110 ldd [%o1 + 0x020], %f8
111 fmovd %f4, %f20 111 fsrc2 %f4, %f20
112 ldd [%o1 + 0x028], %f10 112 ldd [%o1 + 0x028], %f10
113 fmovd %f6, %f22 113 fsrc2 %f6, %f22
114 ldd [%o1 + 0x030], %f12 114 ldd [%o1 + 0x030], %f12
115 fmovd %f8, %f24 115 fsrc2 %f8, %f24
116 ldd [%o1 + 0x038], %f14 116 ldd [%o1 + 0x038], %f14
117 fmovd %f10, %f26 117 fsrc2 %f10, %f26
118 ldd [%o1 + 0x040], %f0 118 ldd [%o1 + 0x040], %f0
1191: ldd [%o1 + 0x048], %f2 1191: ldd [%o1 + 0x048], %f2
120 fmovd %f12, %f28 120 fsrc2 %f12, %f28
121 ldd [%o1 + 0x050], %f4 121 ldd [%o1 + 0x050], %f4
122 fmovd %f14, %f30 122 fsrc2 %f14, %f30
123 stda %f16, [%o0] ASI_BLK_P 123 stda %f16, [%o0] ASI_BLK_P
124 ldd [%o1 + 0x058], %f6 124 ldd [%o1 + 0x058], %f6
125 fmovd %f0, %f16 125 fsrc2 %f0, %f16
126 ldd [%o1 + 0x060], %f8 126 ldd [%o1 + 0x060], %f8
127 fmovd %f2, %f18 127 fsrc2 %f2, %f18
128 ldd [%o1 + 0x068], %f10 128 ldd [%o1 + 0x068], %f10
129 fmovd %f4, %f20 129 fsrc2 %f4, %f20
130 ldd [%o1 + 0x070], %f12 130 ldd [%o1 + 0x070], %f12
131 fmovd %f6, %f22 131 fsrc2 %f6, %f22
132 ldd [%o1 + 0x078], %f14 132 ldd [%o1 + 0x078], %f14
133 fmovd %f8, %f24 133 fsrc2 %f8, %f24
134 ldd [%o1 + 0x080], %f0 134 ldd [%o1 + 0x080], %f0
135 prefetch [%o1 + 0x180], #one_read 135 prefetch [%o1 + 0x180], #one_read
136 fmovd %f10, %f26 136 fsrc2 %f10, %f26
137 subcc %o2, 1, %o2 137 subcc %o2, 1, %o2
138 add %o0, 0x40, %o0 138 add %o0, 0x40, %o0
139 bne,pt %xcc, 1b 139 bne,pt %xcc, 1b
140 add %o1, 0x40, %o1 140 add %o1, 0x40, %o1
141 141
142 ldd [%o1 + 0x048], %f2 142 ldd [%o1 + 0x048], %f2
143 fmovd %f12, %f28 143 fsrc2 %f12, %f28
144 ldd [%o1 + 0x050], %f4 144 ldd [%o1 + 0x050], %f4
145 fmovd %f14, %f30 145 fsrc2 %f14, %f30
146 stda %f16, [%o0] ASI_BLK_P 146 stda %f16, [%o0] ASI_BLK_P
147 ldd [%o1 + 0x058], %f6 147 ldd [%o1 + 0x058], %f6
148 fmovd %f0, %f16 148 fsrc2 %f0, %f16
149 ldd [%o1 + 0x060], %f8 149 ldd [%o1 + 0x060], %f8
150 fmovd %f2, %f18 150 fsrc2 %f2, %f18
151 ldd [%o1 + 0x068], %f10 151 ldd [%o1 + 0x068], %f10
152 fmovd %f4, %f20 152 fsrc2 %f4, %f20
153 ldd [%o1 + 0x070], %f12 153 ldd [%o1 + 0x070], %f12
154 fmovd %f6, %f22 154 fsrc2 %f6, %f22
155 add %o0, 0x40, %o0 155 add %o0, 0x40, %o0
156 ldd [%o1 + 0x078], %f14 156 ldd [%o1 + 0x078], %f14
157 fmovd %f8, %f24 157 fsrc2 %f8, %f24
158 fmovd %f10, %f26 158 fsrc2 %f10, %f26
159 fmovd %f12, %f28 159 fsrc2 %f12, %f28
160 fmovd %f14, %f30 160 fsrc2 %f14, %f30
161 stda %f16, [%o0] ASI_BLK_P 161 stda %f16, [%o0] ASI_BLK_P
162 membar #Sync 162 membar #Sync
163 VISExitHalf 163 VISExitHalf
diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c
index f46cf6be3370..77ac917be152 100644
--- a/arch/sparc/mm/fault_32.c
+++ b/arch/sparc/mm/fault_32.c
@@ -32,24 +32,6 @@
32 32
33int show_unhandled_signals = 1; 33int show_unhandled_signals = 1;
34 34
35/* At boot time we determine these two values necessary for setting
36 * up the segment maps and page table entries (pte's).
37 */
38
39int num_contexts;
40
41/* Return how much physical memory we have. */
42unsigned long probe_memory(void)
43{
44 unsigned long total = 0;
45 int i;
46
47 for (i = 0; sp_banks[i].num_bytes; i++)
48 total += sp_banks[i].num_bytes;
49
50 return total;
51}
52
53static void unhandled_fault(unsigned long, struct task_struct *, 35static void unhandled_fault(unsigned long, struct task_struct *,
54 struct pt_regs *) __attribute__ ((noreturn)); 36 struct pt_regs *) __attribute__ ((noreturn));
55 37
diff --git a/arch/sparc/mm/highmem.c b/arch/sparc/mm/highmem.c
index 055c66cf1bf4..449f864f0cef 100644
--- a/arch/sparc/mm/highmem.c
+++ b/arch/sparc/mm/highmem.c
@@ -22,13 +22,31 @@
22 * shared by CPUs, and so precious, and establishing them requires IPI. 22 * shared by CPUs, and so precious, and establishing them requires IPI.
23 * Atomic kmaps are lightweight and we may have NCPUS more of them. 23 * Atomic kmaps are lightweight and we may have NCPUS more of them.
24 */ 24 */
25#include <linux/mm.h>
26#include <linux/highmem.h> 25#include <linux/highmem.h>
27#include <linux/export.h> 26#include <linux/export.h>
28#include <asm/pgalloc.h> 27#include <linux/mm.h>
28
29#include <asm/cacheflush.h> 29#include <asm/cacheflush.h>
30#include <asm/tlbflush.h> 30#include <asm/tlbflush.h>
31#include <asm/fixmap.h> 31#include <asm/pgalloc.h>
32#include <asm/vaddrs.h>
33
34pgprot_t kmap_prot;
35
36static pte_t *kmap_pte;
37
38void __init kmap_init(void)
39{
40 unsigned long address;
41 pmd_t *dir;
42
43 address = __fix_to_virt(FIX_KMAP_BEGIN);
44 dir = pmd_offset(pgd_offset_k(address), address);
45
46 /* cache the first kmap pte */
47 kmap_pte = pte_offset_kernel(dir, address);
48 kmap_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV | SRMMU_CACHE);
49}
32 50
33void *kmap_atomic(struct page *page) 51void *kmap_atomic(struct page *page)
34{ 52{
@@ -110,21 +128,3 @@ void __kunmap_atomic(void *kvaddr)
110 pagefault_enable(); 128 pagefault_enable();
111} 129}
112EXPORT_SYMBOL(__kunmap_atomic); 130EXPORT_SYMBOL(__kunmap_atomic);
113
114/* We may be fed a pagetable here by ptep_to_xxx and others. */
115struct page *kmap_atomic_to_page(void *ptr)
116{
117 unsigned long idx, vaddr = (unsigned long)ptr;
118 pte_t *pte;
119
120 if (vaddr < SRMMU_NOCACHE_VADDR)
121 return virt_to_page(ptr);
122 if (vaddr < PKMAP_BASE)
123 return pfn_to_page(__nocache_pa(vaddr) >> PAGE_SHIFT);
124 BUG_ON(vaddr < FIXADDR_START);
125 BUG_ON(vaddr > FIXADDR_TOP);
126
127 idx = virt_to_fix(vaddr);
128 pte = kmap_pte - (idx - FIX_KMAP_BEGIN);
129 return pte_page(*pte);
130}
diff --git a/arch/sparc/mm/init_32.c b/arch/sparc/mm/init_32.c
index ef5c779ec855..dde85ef1c56d 100644
--- a/arch/sparc/mm/init_32.c
+++ b/arch/sparc/mm/init_32.c
@@ -45,9 +45,6 @@ unsigned long pfn_base;
45EXPORT_SYMBOL(pfn_base); 45EXPORT_SYMBOL(pfn_base);
46 46
47struct sparc_phys_banks sp_banks[SPARC_PHYS_BANKS+1]; 47struct sparc_phys_banks sp_banks[SPARC_PHYS_BANKS+1];
48unsigned long sparc_unmapped_base;
49
50struct pgtable_cache_struct pgt_quicklists;
51 48
52/* Initial ramdisk setup */ 49/* Initial ramdisk setup */
53extern unsigned int sparc_ramdisk_image; 50extern unsigned int sparc_ramdisk_image;
@@ -55,19 +52,6 @@ extern unsigned int sparc_ramdisk_size;
55 52
56unsigned long highstart_pfn, highend_pfn; 53unsigned long highstart_pfn, highend_pfn;
57 54
58pte_t *kmap_pte;
59pgprot_t kmap_prot;
60
61#define kmap_get_fixmap_pte(vaddr) \
62 pte_offset_kernel(pmd_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr))
63
64void __init kmap_init(void)
65{
66 /* cache the first kmap pte */
67 kmap_pte = kmap_get_fixmap_pte(__fix_to_virt(FIX_KMAP_BEGIN));
68 kmap_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV | SRMMU_CACHE);
69}
70
71void show_mem(unsigned int filter) 55void show_mem(unsigned int filter)
72{ 56{
73 printk("Mem-info:\n"); 57 printk("Mem-info:\n");
@@ -76,33 +60,8 @@ void show_mem(unsigned int filter)
76 nr_swap_pages << (PAGE_SHIFT-10)); 60 nr_swap_pages << (PAGE_SHIFT-10));
77 printk("%ld pages of RAM\n", totalram_pages); 61 printk("%ld pages of RAM\n", totalram_pages);
78 printk("%ld free pages\n", nr_free_pages()); 62 printk("%ld free pages\n", nr_free_pages());
79#if 0 /* undefined pgtable_cache_size, pgd_cache_size */
80 printk("%ld pages in page table cache\n",pgtable_cache_size);
81#ifndef CONFIG_SMP
82 if (sparc_cpu_model == sun4m || sparc_cpu_model == sun4d)
83 printk("%ld entries in page dir cache\n",pgd_cache_size);
84#endif
85#endif
86} 63}
87 64
88void __init sparc_context_init(int numctx)
89{
90 int ctx;
91
92 ctx_list_pool = __alloc_bootmem(numctx * sizeof(struct ctx_list), SMP_CACHE_BYTES, 0UL);
93
94 for(ctx = 0; ctx < numctx; ctx++) {
95 struct ctx_list *clist;
96
97 clist = (ctx_list_pool + ctx);
98 clist->ctx_number = ctx;
99 clist->ctx_mm = NULL;
100 }
101 ctx_free.next = ctx_free.prev = &ctx_free;
102 ctx_used.next = ctx_used.prev = &ctx_used;
103 for(ctx = 0; ctx < numctx; ctx++)
104 add_to_free_ctxlist(ctx_list_pool + ctx);
105}
106 65
107extern unsigned long cmdline_memory_size; 66extern unsigned long cmdline_memory_size;
108unsigned long last_valid_pfn; 67unsigned long last_valid_pfn;
@@ -292,22 +251,7 @@ extern void device_scan(void);
292 251
293void __init paging_init(void) 252void __init paging_init(void)
294{ 253{
295 switch(sparc_cpu_model) { 254 srmmu_paging_init();
296 case sparc_leon:
297 leon_init();
298 /* fall through */
299 case sun4m:
300 case sun4d:
301 srmmu_paging_init();
302 sparc_unmapped_base = 0x50000000;
303 break;
304 default:
305 prom_printf("paging_init: Cannot init paging on this Sparc\n");
306 prom_printf("paging_init: sparc_cpu_model = %d\n", sparc_cpu_model);
307 prom_printf("paging_init: Halting...\n");
308 prom_halt();
309 }
310
311 prom_build_devicetree(); 255 prom_build_devicetree();
312 of_fill_in_cpu_data(); 256 of_fill_in_cpu_data();
313 device_scan(); 257 device_scan();
diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c
index 62e3f5773303..c38bb72e3e80 100644
--- a/arch/sparc/mm/srmmu.c
+++ b/arch/sparc/mm/srmmu.c
@@ -8,45 +8,45 @@
8 * Copyright (C) 1999,2000 Anton Blanchard (anton@samba.org) 8 * Copyright (C) 1999,2000 Anton Blanchard (anton@samba.org)
9 */ 9 */
10 10
11#include <linux/kernel.h> 11#include <linux/seq_file.h>
12#include <linux/mm.h>
13#include <linux/vmalloc.h>
14#include <linux/pagemap.h>
15#include <linux/init.h>
16#include <linux/spinlock.h> 12#include <linux/spinlock.h>
17#include <linux/bootmem.h> 13#include <linux/bootmem.h>
18#include <linux/fs.h> 14#include <linux/pagemap.h>
19#include <linux/seq_file.h> 15#include <linux/vmalloc.h>
20#include <linux/kdebug.h> 16#include <linux/kdebug.h>
17#include <linux/kernel.h>
18#include <linux/init.h>
21#include <linux/log2.h> 19#include <linux/log2.h>
22#include <linux/gfp.h> 20#include <linux/gfp.h>
21#include <linux/fs.h>
22#include <linux/mm.h>
23 23
24#include <asm/bitext.h> 24#include <asm/mmu_context.h>
25#include <asm/page.h> 25#include <asm/cacheflush.h>
26#include <asm/tlbflush.h>
27#include <asm/io-unit.h>
26#include <asm/pgalloc.h> 28#include <asm/pgalloc.h>
27#include <asm/pgtable.h> 29#include <asm/pgtable.h>
28#include <asm/io.h> 30#include <asm/bitext.h>
29#include <asm/vaddrs.h> 31#include <asm/vaddrs.h>
30#include <asm/traps.h>
31#include <asm/smp.h>
32#include <asm/mbus.h>
33#include <asm/cache.h> 32#include <asm/cache.h>
33#include <asm/traps.h>
34#include <asm/oplib.h> 34#include <asm/oplib.h>
35#include <asm/mbus.h>
36#include <asm/page.h>
35#include <asm/asi.h> 37#include <asm/asi.h>
36#include <asm/msi.h> 38#include <asm/msi.h>
37#include <asm/mmu_context.h> 39#include <asm/smp.h>
38#include <asm/io-unit.h> 40#include <asm/io.h>
39#include <asm/cacheflush.h>
40#include <asm/tlbflush.h>
41 41
42/* Now the cpu specific definitions. */ 42/* Now the cpu specific definitions. */
43#include <asm/viking.h> 43#include <asm/turbosparc.h>
44#include <asm/mxcc.h>
45#include <asm/ross.h>
46#include <asm/tsunami.h> 44#include <asm/tsunami.h>
45#include <asm/viking.h>
47#include <asm/swift.h> 46#include <asm/swift.h>
48#include <asm/turbosparc.h>
49#include <asm/leon.h> 47#include <asm/leon.h>
48#include <asm/mxcc.h>
49#include <asm/ross.h>
50 50
51#include "srmmu.h" 51#include "srmmu.h"
52 52
@@ -55,10 +55,6 @@ static unsigned int hwbug_bitmask;
55int vac_cache_size; 55int vac_cache_size;
56int vac_line_size; 56int vac_line_size;
57 57
58struct ctx_list *ctx_list_pool;
59struct ctx_list ctx_free;
60struct ctx_list ctx_used;
61
62extern struct resource sparc_iomap; 58extern struct resource sparc_iomap;
63 59
64extern unsigned long last_valid_pfn; 60extern unsigned long last_valid_pfn;
@@ -136,8 +132,8 @@ void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, struct page *ptep)
136 } 132 }
137} 133}
138 134
139/* Find an entry in the third-level page table.. */ 135/* Find an entry in the third-level page table.. */
140pte_t *pte_offset_kernel(pmd_t * dir, unsigned long address) 136pte_t *pte_offset_kernel(pmd_t *dir, unsigned long address)
141{ 137{
142 void *pte; 138 void *pte;
143 139
@@ -151,55 +147,61 @@ pte_t *pte_offset_kernel(pmd_t * dir, unsigned long address)
151 * align: bytes, number to align at. 147 * align: bytes, number to align at.
152 * Returns the virtual address of the allocated area. 148 * Returns the virtual address of the allocated area.
153 */ 149 */
154static unsigned long __srmmu_get_nocache(int size, int align) 150static void *__srmmu_get_nocache(int size, int align)
155{ 151{
156 int offset; 152 int offset;
153 unsigned long addr;
157 154
158 if (size < SRMMU_NOCACHE_BITMAP_SHIFT) { 155 if (size < SRMMU_NOCACHE_BITMAP_SHIFT) {
159 printk("Size 0x%x too small for nocache request\n", size); 156 printk(KERN_ERR "Size 0x%x too small for nocache request\n",
157 size);
160 size = SRMMU_NOCACHE_BITMAP_SHIFT; 158 size = SRMMU_NOCACHE_BITMAP_SHIFT;
161 } 159 }
162 if (size & (SRMMU_NOCACHE_BITMAP_SHIFT-1)) { 160 if (size & (SRMMU_NOCACHE_BITMAP_SHIFT - 1)) {
163 printk("Size 0x%x unaligned int nocache request\n", size); 161 printk(KERN_ERR "Size 0x%x unaligned int nocache request\n",
164 size += SRMMU_NOCACHE_BITMAP_SHIFT-1; 162 size);
163 size += SRMMU_NOCACHE_BITMAP_SHIFT - 1;
165 } 164 }
166 BUG_ON(align > SRMMU_NOCACHE_ALIGN_MAX); 165 BUG_ON(align > SRMMU_NOCACHE_ALIGN_MAX);
167 166
168 offset = bit_map_string_get(&srmmu_nocache_map, 167 offset = bit_map_string_get(&srmmu_nocache_map,
169 size >> SRMMU_NOCACHE_BITMAP_SHIFT, 168 size >> SRMMU_NOCACHE_BITMAP_SHIFT,
170 align >> SRMMU_NOCACHE_BITMAP_SHIFT); 169 align >> SRMMU_NOCACHE_BITMAP_SHIFT);
171 if (offset == -1) { 170 if (offset == -1) {
172 printk("srmmu: out of nocache %d: %d/%d\n", 171 printk(KERN_ERR "srmmu: out of nocache %d: %d/%d\n",
173 size, (int) srmmu_nocache_size, 172 size, (int) srmmu_nocache_size,
174 srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT); 173 srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT);
175 return 0; 174 return 0;
176 } 175 }
177 176
178 return (SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT)); 177 addr = SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT);
178 return (void *)addr;
179} 179}
180 180
181unsigned long srmmu_get_nocache(int size, int align) 181void *srmmu_get_nocache(int size, int align)
182{ 182{
183 unsigned long tmp; 183 void *tmp;
184 184
185 tmp = __srmmu_get_nocache(size, align); 185 tmp = __srmmu_get_nocache(size, align);
186 186
187 if (tmp) 187 if (tmp)
188 memset((void *)tmp, 0, size); 188 memset(tmp, 0, size);
189 189
190 return tmp; 190 return tmp;
191} 191}
192 192
193void srmmu_free_nocache(unsigned long vaddr, int size) 193void srmmu_free_nocache(void *addr, int size)
194{ 194{
195 unsigned long vaddr;
195 int offset; 196 int offset;
196 197
198 vaddr = (unsigned long)addr;
197 if (vaddr < SRMMU_NOCACHE_VADDR) { 199 if (vaddr < SRMMU_NOCACHE_VADDR) {
198 printk("Vaddr %lx is smaller than nocache base 0x%lx\n", 200 printk("Vaddr %lx is smaller than nocache base 0x%lx\n",
199 vaddr, (unsigned long)SRMMU_NOCACHE_VADDR); 201 vaddr, (unsigned long)SRMMU_NOCACHE_VADDR);
200 BUG(); 202 BUG();
201 } 203 }
202 if (vaddr+size > srmmu_nocache_end) { 204 if (vaddr + size > srmmu_nocache_end) {
203 printk("Vaddr %lx is bigger than nocache end 0x%lx\n", 205 printk("Vaddr %lx is bigger than nocache end 0x%lx\n",
204 vaddr, srmmu_nocache_end); 206 vaddr, srmmu_nocache_end);
205 BUG(); 207 BUG();
@@ -212,7 +214,7 @@ void srmmu_free_nocache(unsigned long vaddr, int size)
212 printk("Size 0x%x is too small\n", size); 214 printk("Size 0x%x is too small\n", size);
213 BUG(); 215 BUG();
214 } 216 }
215 if (vaddr & (size-1)) { 217 if (vaddr & (size - 1)) {
216 printk("Vaddr %lx is not aligned to size 0x%x\n", vaddr, size); 218 printk("Vaddr %lx is not aligned to size 0x%x\n", vaddr, size);
217 BUG(); 219 BUG();
218 } 220 }
@@ -226,13 +228,23 @@ void srmmu_free_nocache(unsigned long vaddr, int size)
226static void srmmu_early_allocate_ptable_skeleton(unsigned long start, 228static void srmmu_early_allocate_ptable_skeleton(unsigned long start,
227 unsigned long end); 229 unsigned long end);
228 230
229extern unsigned long probe_memory(void); /* in fault.c */ 231/* Return how much physical memory we have. */
232static unsigned long __init probe_memory(void)
233{
234 unsigned long total = 0;
235 int i;
236
237 for (i = 0; sp_banks[i].num_bytes; i++)
238 total += sp_banks[i].num_bytes;
239
240 return total;
241}
230 242
231/* 243/*
232 * Reserve nocache dynamically proportionally to the amount of 244 * Reserve nocache dynamically proportionally to the amount of
233 * system RAM. -- Tomas Szepe <szepe@pinerecords.com>, June 2002 245 * system RAM. -- Tomas Szepe <szepe@pinerecords.com>, June 2002
234 */ 246 */
235static void srmmu_nocache_calcsize(void) 247static void __init srmmu_nocache_calcsize(void)
236{ 248{
237 unsigned long sysmemavail = probe_memory() / 1024; 249 unsigned long sysmemavail = probe_memory() / 1024;
238 int srmmu_nocache_npages; 250 int srmmu_nocache_npages;
@@ -271,7 +283,7 @@ static void __init srmmu_nocache_init(void)
271 srmmu_nocache_bitmap = __alloc_bootmem(bitmap_bits >> 3, SMP_CACHE_BYTES, 0UL); 283 srmmu_nocache_bitmap = __alloc_bootmem(bitmap_bits >> 3, SMP_CACHE_BYTES, 0UL);
272 bit_map_init(&srmmu_nocache_map, srmmu_nocache_bitmap, bitmap_bits); 284 bit_map_init(&srmmu_nocache_map, srmmu_nocache_bitmap, bitmap_bits);
273 285
274 srmmu_swapper_pg_dir = (pgd_t *)__srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE); 286 srmmu_swapper_pg_dir = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
275 memset(__nocache_fix(srmmu_swapper_pg_dir), 0, SRMMU_PGD_TABLE_SIZE); 287 memset(__nocache_fix(srmmu_swapper_pg_dir), 0, SRMMU_PGD_TABLE_SIZE);
276 init_mm.pgd = srmmu_swapper_pg_dir; 288 init_mm.pgd = srmmu_swapper_pg_dir;
277 289
@@ -304,7 +316,7 @@ pgd_t *get_pgd_fast(void)
304{ 316{
305 pgd_t *pgd = NULL; 317 pgd_t *pgd = NULL;
306 318
307 pgd = (pgd_t *)__srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE); 319 pgd = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
308 if (pgd) { 320 if (pgd) {
309 pgd_t *init = pgd_offset_k(0); 321 pgd_t *init = pgd_offset_k(0);
310 memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t)); 322 memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
@@ -330,7 +342,7 @@ pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
330 342
331 if ((pte = (unsigned long)pte_alloc_one_kernel(mm, address)) == 0) 343 if ((pte = (unsigned long)pte_alloc_one_kernel(mm, address)) == 0)
332 return NULL; 344 return NULL;
333 page = pfn_to_page( __nocache_pa(pte) >> PAGE_SHIFT ); 345 page = pfn_to_page(__nocache_pa(pte) >> PAGE_SHIFT);
334 pgtable_page_ctor(page); 346 pgtable_page_ctor(page);
335 return page; 347 return page;
336} 348}
@@ -344,18 +356,50 @@ void pte_free(struct mm_struct *mm, pgtable_t pte)
344 if (p == 0) 356 if (p == 0)
345 BUG(); 357 BUG();
346 p = page_to_pfn(pte) << PAGE_SHIFT; /* Physical address */ 358 p = page_to_pfn(pte) << PAGE_SHIFT; /* Physical address */
347 p = (unsigned long) __nocache_va(p); /* Nocached virtual */ 359
348 srmmu_free_nocache(p, PTE_SIZE); 360 /* free non cached virtual address*/
361 srmmu_free_nocache(__nocache_va(p), PTE_SIZE);
349} 362}
350 363
351/* 364/* context handling - a dynamically sized pool is used */
352 */ 365#define NO_CONTEXT -1
366
367struct ctx_list {
368 struct ctx_list *next;
369 struct ctx_list *prev;
370 unsigned int ctx_number;
371 struct mm_struct *ctx_mm;
372};
373
374static struct ctx_list *ctx_list_pool;
375static struct ctx_list ctx_free;
376static struct ctx_list ctx_used;
377
378/* At boot time we determine the number of contexts */
379static int num_contexts;
380
381static inline void remove_from_ctx_list(struct ctx_list *entry)
382{
383 entry->next->prev = entry->prev;
384 entry->prev->next = entry->next;
385}
386
387static inline void add_to_ctx_list(struct ctx_list *head, struct ctx_list *entry)
388{
389 entry->next = head;
390 (entry->prev = head->prev)->next = entry;
391 head->prev = entry;
392}
393#define add_to_free_ctxlist(entry) add_to_ctx_list(&ctx_free, entry)
394#define add_to_used_ctxlist(entry) add_to_ctx_list(&ctx_used, entry)
395
396
353static inline void alloc_context(struct mm_struct *old_mm, struct mm_struct *mm) 397static inline void alloc_context(struct mm_struct *old_mm, struct mm_struct *mm)
354{ 398{
355 struct ctx_list *ctxp; 399 struct ctx_list *ctxp;
356 400
357 ctxp = ctx_free.next; 401 ctxp = ctx_free.next;
358 if(ctxp != &ctx_free) { 402 if (ctxp != &ctx_free) {
359 remove_from_ctx_list(ctxp); 403 remove_from_ctx_list(ctxp);
360 add_to_used_ctxlist(ctxp); 404 add_to_used_ctxlist(ctxp);
361 mm->context = ctxp->ctx_number; 405 mm->context = ctxp->ctx_number;
@@ -363,9 +407,9 @@ static inline void alloc_context(struct mm_struct *old_mm, struct mm_struct *mm)
363 return; 407 return;
364 } 408 }
365 ctxp = ctx_used.next; 409 ctxp = ctx_used.next;
366 if(ctxp->ctx_mm == old_mm) 410 if (ctxp->ctx_mm == old_mm)
367 ctxp = ctxp->next; 411 ctxp = ctxp->next;
368 if(ctxp == &ctx_used) 412 if (ctxp == &ctx_used)
369 panic("out of mmu contexts"); 413 panic("out of mmu contexts");
370 flush_cache_mm(ctxp->ctx_mm); 414 flush_cache_mm(ctxp->ctx_mm);
371 flush_tlb_mm(ctxp->ctx_mm); 415 flush_tlb_mm(ctxp->ctx_mm);
@@ -385,11 +429,31 @@ static inline void free_context(int context)
385 add_to_free_ctxlist(ctx_old); 429 add_to_free_ctxlist(ctx_old);
386} 430}
387 431
432static void __init sparc_context_init(int numctx)
433{
434 int ctx;
435 unsigned long size;
436
437 size = numctx * sizeof(struct ctx_list);
438 ctx_list_pool = __alloc_bootmem(size, SMP_CACHE_BYTES, 0UL);
439
440 for (ctx = 0; ctx < numctx; ctx++) {
441 struct ctx_list *clist;
442
443 clist = (ctx_list_pool + ctx);
444 clist->ctx_number = ctx;
445 clist->ctx_mm = NULL;
446 }
447 ctx_free.next = ctx_free.prev = &ctx_free;
448 ctx_used.next = ctx_used.prev = &ctx_used;
449 for (ctx = 0; ctx < numctx; ctx++)
450 add_to_free_ctxlist(ctx_list_pool + ctx);
451}
388 452
389void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, 453void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm,
390 struct task_struct *tsk) 454 struct task_struct *tsk)
391{ 455{
392 if(mm->context == NO_CONTEXT) { 456 if (mm->context == NO_CONTEXT) {
393 spin_lock(&srmmu_context_spinlock); 457 spin_lock(&srmmu_context_spinlock);
394 alloc_context(old_mm, mm); 458 alloc_context(old_mm, mm);
395 spin_unlock(&srmmu_context_spinlock); 459 spin_unlock(&srmmu_context_spinlock);
@@ -407,7 +471,7 @@ void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm,
407 471
408/* Low level IO area allocation on the SRMMU. */ 472/* Low level IO area allocation on the SRMMU. */
409static inline void srmmu_mapioaddr(unsigned long physaddr, 473static inline void srmmu_mapioaddr(unsigned long physaddr,
410 unsigned long virt_addr, int bus_type) 474 unsigned long virt_addr, int bus_type)
411{ 475{
412 pgd_t *pgdp; 476 pgd_t *pgdp;
413 pmd_t *pmdp; 477 pmd_t *pmdp;
@@ -420,8 +484,7 @@ static inline void srmmu_mapioaddr(unsigned long physaddr,
420 ptep = pte_offset_kernel(pmdp, virt_addr); 484 ptep = pte_offset_kernel(pmdp, virt_addr);
421 tmp = (physaddr >> 4) | SRMMU_ET_PTE; 485 tmp = (physaddr >> 4) | SRMMU_ET_PTE;
422 486
423 /* 487 /* I need to test whether this is consistent over all
424 * I need to test whether this is consistent over all
425 * sun4m's. The bus_type represents the upper 4 bits of 488 * sun4m's. The bus_type represents the upper 4 bits of
426 * 36-bit physical address on the I/O space lines... 489 * 36-bit physical address on the I/O space lines...
427 */ 490 */
@@ -591,10 +654,10 @@ static void __init srmmu_early_allocate_ptable_skeleton(unsigned long start,
591 pmd_t *pmdp; 654 pmd_t *pmdp;
592 pte_t *ptep; 655 pte_t *ptep;
593 656
594 while(start < end) { 657 while (start < end) {
595 pgdp = pgd_offset_k(start); 658 pgdp = pgd_offset_k(start);
596 if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) { 659 if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) {
597 pmdp = (pmd_t *) __srmmu_get_nocache( 660 pmdp = __srmmu_get_nocache(
598 SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); 661 SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
599 if (pmdp == NULL) 662 if (pmdp == NULL)
600 early_pgtable_allocfail("pmd"); 663 early_pgtable_allocfail("pmd");
@@ -602,8 +665,8 @@ static void __init srmmu_early_allocate_ptable_skeleton(unsigned long start,
602 pgd_set(__nocache_fix(pgdp), pmdp); 665 pgd_set(__nocache_fix(pgdp), pmdp);
603 } 666 }
604 pmdp = pmd_offset(__nocache_fix(pgdp), start); 667 pmdp = pmd_offset(__nocache_fix(pgdp), start);
605 if(srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) { 668 if (srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) {
606 ptep = (pte_t *)__srmmu_get_nocache(PTE_SIZE, PTE_SIZE); 669 ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE);
607 if (ptep == NULL) 670 if (ptep == NULL)
608 early_pgtable_allocfail("pte"); 671 early_pgtable_allocfail("pte");
609 memset(__nocache_fix(ptep), 0, PTE_SIZE); 672 memset(__nocache_fix(ptep), 0, PTE_SIZE);
@@ -622,18 +685,18 @@ static void __init srmmu_allocate_ptable_skeleton(unsigned long start,
622 pmd_t *pmdp; 685 pmd_t *pmdp;
623 pte_t *ptep; 686 pte_t *ptep;
624 687
625 while(start < end) { 688 while (start < end) {
626 pgdp = pgd_offset_k(start); 689 pgdp = pgd_offset_k(start);
627 if (pgd_none(*pgdp)) { 690 if (pgd_none(*pgdp)) {
628 pmdp = (pmd_t *)__srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); 691 pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
629 if (pmdp == NULL) 692 if (pmdp == NULL)
630 early_pgtable_allocfail("pmd"); 693 early_pgtable_allocfail("pmd");
631 memset(pmdp, 0, SRMMU_PMD_TABLE_SIZE); 694 memset(pmdp, 0, SRMMU_PMD_TABLE_SIZE);
632 pgd_set(pgdp, pmdp); 695 pgd_set(pgdp, pmdp);
633 } 696 }
634 pmdp = pmd_offset(pgdp, start); 697 pmdp = pmd_offset(pgdp, start);
635 if(srmmu_pmd_none(*pmdp)) { 698 if (srmmu_pmd_none(*pmdp)) {
636 ptep = (pte_t *) __srmmu_get_nocache(PTE_SIZE, 699 ptep = __srmmu_get_nocache(PTE_SIZE,
637 PTE_SIZE); 700 PTE_SIZE);
638 if (ptep == NULL) 701 if (ptep == NULL)
639 early_pgtable_allocfail("pte"); 702 early_pgtable_allocfail("pte");
@@ -671,72 +734,76 @@ static inline unsigned long srmmu_probe(unsigned long vaddr)
671static void __init srmmu_inherit_prom_mappings(unsigned long start, 734static void __init srmmu_inherit_prom_mappings(unsigned long start,
672 unsigned long end) 735 unsigned long end)
673{ 736{
737 unsigned long probed;
738 unsigned long addr;
674 pgd_t *pgdp; 739 pgd_t *pgdp;
675 pmd_t *pmdp; 740 pmd_t *pmdp;
676 pte_t *ptep; 741 pte_t *ptep;
677 int what = 0; /* 0 = normal-pte, 1 = pmd-level pte, 2 = pgd-level pte */ 742 int what; /* 0 = normal-pte, 1 = pmd-level pte, 2 = pgd-level pte */
678 unsigned long prompte;
679 743
680 while(start <= end) { 744 while (start <= end) {
681 if (start == 0) 745 if (start == 0)
682 break; /* probably wrap around */ 746 break; /* probably wrap around */
683 if(start == 0xfef00000) 747 if (start == 0xfef00000)
684 start = KADB_DEBUGGER_BEGVM; 748 start = KADB_DEBUGGER_BEGVM;
685 if(!(prompte = srmmu_probe(start))) { 749 probed = srmmu_probe(start);
750 if (!probed) {
751 /* continue probing until we find an entry */
686 start += PAGE_SIZE; 752 start += PAGE_SIZE;
687 continue; 753 continue;
688 } 754 }
689 755
690 /* A red snapper, see what it really is. */ 756 /* A red snapper, see what it really is. */
691 what = 0; 757 what = 0;
692 758 addr = start - PAGE_SIZE;
693 if(!(start & ~(SRMMU_REAL_PMD_MASK))) { 759
694 if(srmmu_probe((start-PAGE_SIZE) + SRMMU_REAL_PMD_SIZE) == prompte) 760 if (!(start & ~(SRMMU_REAL_PMD_MASK))) {
761 if (srmmu_probe(addr + SRMMU_REAL_PMD_SIZE) == probed)
695 what = 1; 762 what = 1;
696 } 763 }
697 764
698 if(!(start & ~(SRMMU_PGDIR_MASK))) { 765 if (!(start & ~(SRMMU_PGDIR_MASK))) {
699 if(srmmu_probe((start-PAGE_SIZE) + SRMMU_PGDIR_SIZE) == 766 if (srmmu_probe(addr + SRMMU_PGDIR_SIZE) == probed)
700 prompte)
701 what = 2; 767 what = 2;
702 } 768 }
703 769
704 pgdp = pgd_offset_k(start); 770 pgdp = pgd_offset_k(start);
705 if(what == 2) { 771 if (what == 2) {
706 *(pgd_t *)__nocache_fix(pgdp) = __pgd(prompte); 772 *(pgd_t *)__nocache_fix(pgdp) = __pgd(probed);
707 start += SRMMU_PGDIR_SIZE; 773 start += SRMMU_PGDIR_SIZE;
708 continue; 774 continue;
709 } 775 }
710 if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) { 776 if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) {
711 pmdp = (pmd_t *)__srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); 777 pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE,
778 SRMMU_PMD_TABLE_SIZE);
712 if (pmdp == NULL) 779 if (pmdp == NULL)
713 early_pgtable_allocfail("pmd"); 780 early_pgtable_allocfail("pmd");
714 memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE); 781 memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE);
715 pgd_set(__nocache_fix(pgdp), pmdp); 782 pgd_set(__nocache_fix(pgdp), pmdp);
716 } 783 }
717 pmdp = pmd_offset(__nocache_fix(pgdp), start); 784 pmdp = pmd_offset(__nocache_fix(pgdp), start);
718 if(srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) { 785 if (srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) {
719 ptep = (pte_t *) __srmmu_get_nocache(PTE_SIZE, 786 ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE);
720 PTE_SIZE);
721 if (ptep == NULL) 787 if (ptep == NULL)
722 early_pgtable_allocfail("pte"); 788 early_pgtable_allocfail("pte");
723 memset(__nocache_fix(ptep), 0, PTE_SIZE); 789 memset(__nocache_fix(ptep), 0, PTE_SIZE);
724 pmd_set(__nocache_fix(pmdp), ptep); 790 pmd_set(__nocache_fix(pmdp), ptep);
725 } 791 }
726 if(what == 1) { 792 if (what == 1) {
727 /* 793 /* We bend the rule where all 16 PTPs in a pmd_t point
728 * We bend the rule where all 16 PTPs in a pmd_t point
729 * inside the same PTE page, and we leak a perfectly 794 * inside the same PTE page, and we leak a perfectly
730 * good hardware PTE piece. Alternatives seem worse. 795 * good hardware PTE piece. Alternatives seem worse.
731 */ 796 */
732 unsigned int x; /* Index of HW PMD in soft cluster */ 797 unsigned int x; /* Index of HW PMD in soft cluster */
798 unsigned long *val;
733 x = (start >> PMD_SHIFT) & 15; 799 x = (start >> PMD_SHIFT) & 15;
734 *(unsigned long *)__nocache_fix(&pmdp->pmdv[x]) = prompte; 800 val = &pmdp->pmdv[x];
801 *(unsigned long *)__nocache_fix(val) = probed;
735 start += SRMMU_REAL_PMD_SIZE; 802 start += SRMMU_REAL_PMD_SIZE;
736 continue; 803 continue;
737 } 804 }
738 ptep = pte_offset_kernel(__nocache_fix(pmdp), start); 805 ptep = pte_offset_kernel(__nocache_fix(pmdp), start);
739 *(pte_t *)__nocache_fix(ptep) = __pte(prompte); 806 *(pte_t *)__nocache_fix(ptep) = __pte(probed);
740 start += PAGE_SIZE; 807 start += PAGE_SIZE;
741 } 808 }
742} 809}
@@ -765,18 +832,18 @@ static unsigned long __init map_spbank(unsigned long vbase, int sp_entry)
765 832
766 if (vstart < min_vaddr || vstart >= max_vaddr) 833 if (vstart < min_vaddr || vstart >= max_vaddr)
767 return vstart; 834 return vstart;
768 835
769 if (vend > max_vaddr || vend < min_vaddr) 836 if (vend > max_vaddr || vend < min_vaddr)
770 vend = max_vaddr; 837 vend = max_vaddr;
771 838
772 while(vstart < vend) { 839 while (vstart < vend) {
773 do_large_mapping(vstart, pstart); 840 do_large_mapping(vstart, pstart);
774 vstart += SRMMU_PGDIR_SIZE; pstart += SRMMU_PGDIR_SIZE; 841 vstart += SRMMU_PGDIR_SIZE; pstart += SRMMU_PGDIR_SIZE;
775 } 842 }
776 return vstart; 843 return vstart;
777} 844}
778 845
779static inline void map_kernel(void) 846static void __init map_kernel(void)
780{ 847{
781 int i; 848 int i;
782 849
@@ -789,9 +856,6 @@ static inline void map_kernel(void)
789 } 856 }
790} 857}
791 858
792/* Paging initialization on the Sparc Reference MMU. */
793extern void sparc_context_init(int);
794
795void (*poke_srmmu)(void) __cpuinitdata = NULL; 859void (*poke_srmmu)(void) __cpuinitdata = NULL;
796 860
797extern unsigned long bootmem_init(unsigned long *pages_avail); 861extern unsigned long bootmem_init(unsigned long *pages_avail);
@@ -806,6 +870,7 @@ void __init srmmu_paging_init(void)
806 pte_t *pte; 870 pte_t *pte;
807 unsigned long pages_avail; 871 unsigned long pages_avail;
808 872
873 init_mm.context = (unsigned long) NO_CONTEXT;
809 sparc_iomap.start = SUN4M_IOBASE_VADDR; /* 16MB of IOSPACE on all sun4m's. */ 874 sparc_iomap.start = SUN4M_IOBASE_VADDR; /* 16MB of IOSPACE on all sun4m's. */
810 875
811 if (sparc_cpu_model == sun4d) 876 if (sparc_cpu_model == sun4d)
@@ -814,9 +879,9 @@ void __init srmmu_paging_init(void)
814 /* Find the number of contexts on the srmmu. */ 879 /* Find the number of contexts on the srmmu. */
815 cpunode = prom_getchild(prom_root_node); 880 cpunode = prom_getchild(prom_root_node);
816 num_contexts = 0; 881 num_contexts = 0;
817 while(cpunode != 0) { 882 while (cpunode != 0) {
818 prom_getstring(cpunode, "device_type", node_str, sizeof(node_str)); 883 prom_getstring(cpunode, "device_type", node_str, sizeof(node_str));
819 if(!strcmp(node_str, "cpu")) { 884 if (!strcmp(node_str, "cpu")) {
820 num_contexts = prom_getintdefault(cpunode, "mmu-nctx", 0x8); 885 num_contexts = prom_getintdefault(cpunode, "mmu-nctx", 0x8);
821 break; 886 break;
822 } 887 }
@@ -824,7 +889,7 @@ void __init srmmu_paging_init(void)
824 } 889 }
825 } 890 }
826 891
827 if(!num_contexts) { 892 if (!num_contexts) {
828 prom_printf("Something wrong, can't find cpu node in paging_init.\n"); 893 prom_printf("Something wrong, can't find cpu node in paging_init.\n");
829 prom_halt(); 894 prom_halt();
830 } 895 }
@@ -834,14 +899,14 @@ void __init srmmu_paging_init(void)
834 899
835 srmmu_nocache_calcsize(); 900 srmmu_nocache_calcsize();
836 srmmu_nocache_init(); 901 srmmu_nocache_init();
837 srmmu_inherit_prom_mappings(0xfe400000,(LINUX_OPPROM_ENDVM-PAGE_SIZE)); 902 srmmu_inherit_prom_mappings(0xfe400000, (LINUX_OPPROM_ENDVM - PAGE_SIZE));
838 map_kernel(); 903 map_kernel();
839 904
840 /* ctx table has to be physically aligned to its size */ 905 /* ctx table has to be physically aligned to its size */
841 srmmu_context_table = (ctxd_t *)__srmmu_get_nocache(num_contexts*sizeof(ctxd_t), num_contexts*sizeof(ctxd_t)); 906 srmmu_context_table = __srmmu_get_nocache(num_contexts * sizeof(ctxd_t), num_contexts * sizeof(ctxd_t));
842 srmmu_ctx_table_phys = (ctxd_t *)__nocache_pa((unsigned long)srmmu_context_table); 907 srmmu_ctx_table_phys = (ctxd_t *)__nocache_pa((unsigned long)srmmu_context_table);
843 908
844 for(i = 0; i < num_contexts; i++) 909 for (i = 0; i < num_contexts; i++)
845 srmmu_ctxd_set((ctxd_t *)__nocache_fix(&srmmu_context_table[i]), srmmu_swapper_pg_dir); 910 srmmu_ctxd_set((ctxd_t *)__nocache_fix(&srmmu_context_table[i]), srmmu_swapper_pg_dir);
846 911
847 flush_cache_all(); 912 flush_cache_all();
@@ -897,7 +962,7 @@ void __init srmmu_paging_init(void)
897 962
898void mmu_info(struct seq_file *m) 963void mmu_info(struct seq_file *m)
899{ 964{
900 seq_printf(m, 965 seq_printf(m,
901 "MMU type\t: %s\n" 966 "MMU type\t: %s\n"
902 "contexts\t: %d\n" 967 "contexts\t: %d\n"
903 "nocache total\t: %ld\n" 968 "nocache total\t: %ld\n"
@@ -908,10 +973,16 @@ void mmu_info(struct seq_file *m)
908 srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT); 973 srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT);
909} 974}
910 975
976int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
977{
978 mm->context = NO_CONTEXT;
979 return 0;
980}
981
911void destroy_context(struct mm_struct *mm) 982void destroy_context(struct mm_struct *mm)
912{ 983{
913 984
914 if(mm->context != NO_CONTEXT) { 985 if (mm->context != NO_CONTEXT) {
915 flush_cache_mm(mm); 986 flush_cache_mm(mm);
916 srmmu_ctxd_set(&srmmu_context_table[mm->context], srmmu_swapper_pg_dir); 987 srmmu_ctxd_set(&srmmu_context_table[mm->context], srmmu_swapper_pg_dir);
917 flush_tlb_mm(mm); 988 flush_tlb_mm(mm);
@@ -941,13 +1012,12 @@ static void __init init_vac_layout(void)
941#endif 1012#endif
942 1013
943 nd = prom_getchild(prom_root_node); 1014 nd = prom_getchild(prom_root_node);
944 while((nd = prom_getsibling(nd)) != 0) { 1015 while ((nd = prom_getsibling(nd)) != 0) {
945 prom_getstring(nd, "device_type", node_str, sizeof(node_str)); 1016 prom_getstring(nd, "device_type", node_str, sizeof(node_str));
946 if(!strcmp(node_str, "cpu")) { 1017 if (!strcmp(node_str, "cpu")) {
947 vac_line_size = prom_getint(nd, "cache-line-size"); 1018 vac_line_size = prom_getint(nd, "cache-line-size");
948 if (vac_line_size == -1) { 1019 if (vac_line_size == -1) {
949 prom_printf("can't determine cache-line-size, " 1020 prom_printf("can't determine cache-line-size, halting.\n");
950 "halting.\n");
951 prom_halt(); 1021 prom_halt();
952 } 1022 }
953 cache_lines = prom_getint(nd, "cache-nlines"); 1023 cache_lines = prom_getint(nd, "cache-nlines");
@@ -958,9 +1028,9 @@ static void __init init_vac_layout(void)
958 1028
959 vac_cache_size = cache_lines * vac_line_size; 1029 vac_cache_size = cache_lines * vac_line_size;
960#ifdef CONFIG_SMP 1030#ifdef CONFIG_SMP
961 if(vac_cache_size > max_size) 1031 if (vac_cache_size > max_size)
962 max_size = vac_cache_size; 1032 max_size = vac_cache_size;
963 if(vac_line_size < min_line_size) 1033 if (vac_line_size < min_line_size)
964 min_line_size = vac_line_size; 1034 min_line_size = vac_line_size;
965 //FIXME: cpus not contiguous!! 1035 //FIXME: cpus not contiguous!!
966 cpu++; 1036 cpu++;
@@ -971,7 +1041,7 @@ static void __init init_vac_layout(void)
971#endif 1041#endif
972 } 1042 }
973 } 1043 }
974 if(nd == 0) { 1044 if (nd == 0) {
975 prom_printf("No CPU nodes found, halting.\n"); 1045 prom_printf("No CPU nodes found, halting.\n");
976 prom_halt(); 1046 prom_halt();
977 } 1047 }
@@ -1082,7 +1152,7 @@ static void __init init_swift(void)
1082 "=r" (swift_rev) : 1152 "=r" (swift_rev) :
1083 "r" (SWIFT_MASKID_ADDR), "i" (ASI_M_BYPASS)); 1153 "r" (SWIFT_MASKID_ADDR), "i" (ASI_M_BYPASS));
1084 srmmu_name = "Fujitsu Swift"; 1154 srmmu_name = "Fujitsu Swift";
1085 switch(swift_rev) { 1155 switch (swift_rev) {
1086 case 0x11: 1156 case 0x11:
1087 case 0x20: 1157 case 0x20:
1088 case 0x23: 1158 case 0x23:
@@ -1222,10 +1292,11 @@ static void __cpuinit poke_turbosparc(void)
1222 1292
1223 /* Clear any crap from the cache or else... */ 1293 /* Clear any crap from the cache or else... */
1224 turbosparc_flush_cache_all(); 1294 turbosparc_flush_cache_all();
1225 mreg &= ~(TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* Temporarily disable I & D caches */ 1295 /* Temporarily disable I & D caches */
1296 mreg &= ~(TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE);
1226 mreg &= ~(TURBOSPARC_PCENABLE); /* Don't check parity */ 1297 mreg &= ~(TURBOSPARC_PCENABLE); /* Don't check parity */
1227 srmmu_set_mmureg(mreg); 1298 srmmu_set_mmureg(mreg);
1228 1299
1229 ccreg = turbosparc_get_ccreg(); 1300 ccreg = turbosparc_get_ccreg();
1230 1301
1231#ifdef TURBOSPARC_WRITEBACK 1302#ifdef TURBOSPARC_WRITEBACK
@@ -1248,7 +1319,7 @@ static void __cpuinit poke_turbosparc(void)
1248 default: 1319 default:
1249 ccreg |= (TURBOSPARC_SCENABLE); 1320 ccreg |= (TURBOSPARC_SCENABLE);
1250 } 1321 }
1251 turbosparc_set_ccreg (ccreg); 1322 turbosparc_set_ccreg(ccreg);
1252 1323
1253 mreg |= (TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* I & D caches on */ 1324 mreg |= (TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* I & D caches on */
1254 mreg |= (TURBOSPARC_ICSNOOP); /* Icache snooping on */ 1325 mreg |= (TURBOSPARC_ICSNOOP); /* Icache snooping on */
@@ -1342,7 +1413,7 @@ static void __cpuinit poke_viking(void)
1342 unsigned long bpreg; 1413 unsigned long bpreg;
1343 1414
1344 mreg &= ~(VIKING_TCENABLE); 1415 mreg &= ~(VIKING_TCENABLE);
1345 if(smp_catch++) { 1416 if (smp_catch++) {
1346 /* Must disable mixed-cmd mode here for other cpu's. */ 1417 /* Must disable mixed-cmd mode here for other cpu's. */
1347 bpreg = viking_get_bpreg(); 1418 bpreg = viking_get_bpreg();
1348 bpreg &= ~(VIKING_ACTION_MIX); 1419 bpreg &= ~(VIKING_ACTION_MIX);
@@ -1411,7 +1482,7 @@ static void __init init_viking(void)
1411 unsigned long mreg = srmmu_get_mmureg(); 1482 unsigned long mreg = srmmu_get_mmureg();
1412 1483
1413 /* Ahhh, the viking. SRMMU VLSI abortion number two... */ 1484 /* Ahhh, the viking. SRMMU VLSI abortion number two... */
1414 if(mreg & VIKING_MMODE) { 1485 if (mreg & VIKING_MMODE) {
1415 srmmu_name = "TI Viking"; 1486 srmmu_name = "TI Viking";
1416 viking_mxcc_present = 0; 1487 viking_mxcc_present = 0;
1417 msi_set_sync(); 1488 msi_set_sync();
@@ -1467,8 +1538,8 @@ static void __init get_srmmu_type(void)
1467 } 1538 }
1468 1539
1469 /* Second, check for HyperSparc or Cypress. */ 1540 /* Second, check for HyperSparc or Cypress. */
1470 if(mod_typ == 1) { 1541 if (mod_typ == 1) {
1471 switch(mod_rev) { 1542 switch (mod_rev) {
1472 case 7: 1543 case 7:
1473 /* UP or MP Hypersparc */ 1544 /* UP or MP Hypersparc */
1474 init_hypersparc(); 1545 init_hypersparc();
@@ -1488,9 +1559,8 @@ static void __init get_srmmu_type(void)
1488 } 1559 }
1489 return; 1560 return;
1490 } 1561 }
1491 1562
1492 /* 1563 /* Now Fujitsu TurboSparc. It might happen that it is
1493 * Now Fujitsu TurboSparc. It might happen that it is
1494 * in Swift emulation mode, so we will check later... 1564 * in Swift emulation mode, so we will check later...
1495 */ 1565 */
1496 if (psr_typ == 0 && psr_vers == 5) { 1566 if (psr_typ == 0 && psr_vers == 5) {
@@ -1499,15 +1569,15 @@ static void __init get_srmmu_type(void)
1499 } 1569 }
1500 1570
1501 /* Next check for Fujitsu Swift. */ 1571 /* Next check for Fujitsu Swift. */
1502 if(psr_typ == 0 && psr_vers == 4) { 1572 if (psr_typ == 0 && psr_vers == 4) {
1503 phandle cpunode; 1573 phandle cpunode;
1504 char node_str[128]; 1574 char node_str[128];
1505 1575
1506 /* Look if it is not a TurboSparc emulating Swift... */ 1576 /* Look if it is not a TurboSparc emulating Swift... */
1507 cpunode = prom_getchild(prom_root_node); 1577 cpunode = prom_getchild(prom_root_node);
1508 while((cpunode = prom_getsibling(cpunode)) != 0) { 1578 while ((cpunode = prom_getsibling(cpunode)) != 0) {
1509 prom_getstring(cpunode, "device_type", node_str, sizeof(node_str)); 1579 prom_getstring(cpunode, "device_type", node_str, sizeof(node_str));
1510 if(!strcmp(node_str, "cpu")) { 1580 if (!strcmp(node_str, "cpu")) {
1511 if (!prom_getintdefault(cpunode, "psr-implementation", 1) && 1581 if (!prom_getintdefault(cpunode, "psr-implementation", 1) &&
1512 prom_getintdefault(cpunode, "psr-version", 1) == 5) { 1582 prom_getintdefault(cpunode, "psr-version", 1) == 5) {
1513 init_turbosparc(); 1583 init_turbosparc();
@@ -1516,13 +1586,13 @@ static void __init get_srmmu_type(void)
1516 break; 1586 break;
1517 } 1587 }
1518 } 1588 }
1519 1589
1520 init_swift(); 1590 init_swift();
1521 return; 1591 return;
1522 } 1592 }
1523 1593
1524 /* Now the Viking family of srmmu. */ 1594 /* Now the Viking family of srmmu. */
1525 if(psr_typ == 4 && 1595 if (psr_typ == 4 &&
1526 ((psr_vers == 0) || 1596 ((psr_vers == 0) ||
1527 ((psr_vers == 1) && (mod_typ == 0) && (mod_rev == 0)))) { 1597 ((psr_vers == 1) && (mod_typ == 0) && (mod_rev == 0)))) {
1528 init_viking(); 1598 init_viking();
@@ -1530,7 +1600,7 @@ static void __init get_srmmu_type(void)
1530 } 1600 }
1531 1601
1532 /* Finally the Tsunami. */ 1602 /* Finally the Tsunami. */
1533 if(psr_typ == 4 && psr_vers == 1 && (mod_typ || mod_rev)) { 1603 if (psr_typ == 4 && psr_vers == 1 && (mod_typ || mod_rev)) {
1534 init_tsunami(); 1604 init_tsunami();
1535 return; 1605 return;
1536 } 1606 }
diff --git a/arch/sparc/prom/init_32.c b/arch/sparc/prom/init_32.c
index 26c64cea3c9c..9ac30c2b7dba 100644
--- a/arch/sparc/prom/init_32.c
+++ b/arch/sparc/prom/init_32.c
@@ -27,13 +27,10 @@ EXPORT_SYMBOL(prom_root_node);
27struct linux_nodeops *prom_nodeops; 27struct linux_nodeops *prom_nodeops;
28 28
29/* You must call prom_init() before you attempt to use any of the 29/* You must call prom_init() before you attempt to use any of the
30 * routines in the prom library. It returns 0 on success, 1 on 30 * routines in the prom library.
31 * failure. It gets passed the pointer to the PROM vector. 31 * It gets passed the pointer to the PROM vector.
32 */ 32 */
33 33
34extern void prom_meminit(void);
35extern void prom_ranges_init(void);
36
37void __init prom_init(struct linux_romvec *rp) 34void __init prom_init(struct linux_romvec *rp)
38{ 35{
39 romvec = rp; 36 romvec = rp;
diff --git a/arch/sparc/prom/init_64.c b/arch/sparc/prom/init_64.c
index 5016c5e20575..d95db755828f 100644
--- a/arch/sparc/prom/init_64.c
+++ b/arch/sparc/prom/init_64.c
@@ -22,8 +22,8 @@ int prom_stdout;
22phandle prom_chosen_node; 22phandle prom_chosen_node;
23 23
24/* You must call prom_init() before you attempt to use any of the 24/* You must call prom_init() before you attempt to use any of the
25 * routines in the prom library. It returns 0 on success, 1 on 25 * routines in the prom library.
26 * failure. It gets passed the pointer to the PROM vector. 26 * It gets passed the pointer to the PROM vector.
27 */ 27 */
28 28
29extern void prom_cif_init(void *, void *); 29extern void prom_cif_init(void *, void *);
diff --git a/arch/tile/kernel/pci.c b/arch/tile/kernel/pci.c
index 0fdd99d0d8b7..33c10864d2f7 100644
--- a/arch/tile/kernel/pci.c
+++ b/arch/tile/kernel/pci.c
@@ -369,7 +369,7 @@ int __init pcibios_init(void)
369 */ 369 */
370 if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && 370 if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI &&
371 (PCI_SLOT(dev->devfn) == 0)) { 371 (PCI_SLOT(dev->devfn) == 0)) {
372 next_bus = dev->busn_res.end; 372 next_bus = dev->subordinate;
373 controllers[i].mem_resources[0] = 373 controllers[i].mem_resources[0] =
374 *next_bus->resource[0]; 374 *next_bus->resource[0];
375 controllers[i].mem_resources[1] = 375 controllers[i].mem_resources[1] =
diff --git a/arch/tile/kernel/pci_gx.c b/arch/tile/kernel/pci_gx.c
index fa75264a82ae..0e213e35ffc3 100644
--- a/arch/tile/kernel/pci_gx.c
+++ b/arch/tile/kernel/pci_gx.c
@@ -853,7 +853,7 @@ int __init pcibios_init(void)
853 bus = pci_scan_root_bus(NULL, next_busno, controller->ops, 853 bus = pci_scan_root_bus(NULL, next_busno, controller->ops,
854 controller, &resources); 854 controller, &resources);
855 controller->root_bus = bus; 855 controller->root_bus = bus;
856 next_busno = bus->subordinate + 1; 856 next_busno = bus->busn_res.end + 1;
857 857
858 } 858 }
859 859
diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug
index e46c2147397f..b322f124ee3c 100644
--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -129,6 +129,25 @@ config DOUBLEFAULT
129 option saves about 4k and might cause you much additional grey 129 option saves about 4k and might cause you much additional grey
130 hair. 130 hair.
131 131
132config DEBUG_TLBFLUSH
133 bool "Set upper limit of TLB entries to flush one-by-one"
134 depends on DEBUG_KERNEL && (X86_64 || X86_INVLPG)
135 ---help---
136
137 X86-only for now.
138
139 This option allows the user to tune the amount of TLB entries the
140 kernel flushes one-by-one instead of doing a full TLB flush. In
141 certain situations, the former is cheaper. This is controlled by the
142 tlb_flushall_shift knob under /sys/kernel/debug/x86. If you set it
143 to -1, the code flushes the whole TLB unconditionally. Otherwise,
144 for positive values of it, the kernel will use single TLB entry
145 invalidating instructions according to the following formula:
146
147 flush_entries <= active_tlb_entries / 2^tlb_flushall_shift
148
149 If in doubt, say "N".
150
132config IOMMU_DEBUG 151config IOMMU_DEBUG
133 bool "Enable IOMMU debugging" 152 bool "Enable IOMMU debugging"
134 depends on GART_IOMMU && DEBUG_KERNEL 153 depends on GART_IOMMU && DEBUG_KERNEL
diff --git a/arch/x86/boot/compressed/cmdline.c b/arch/x86/boot/compressed/cmdline.c
index cb62f786990d..10f6b1178c68 100644
--- a/arch/x86/boot/compressed/cmdline.c
+++ b/arch/x86/boot/compressed/cmdline.c
@@ -1,5 +1,7 @@
1#include "misc.h" 1#include "misc.h"
2 2
3#ifdef CONFIG_EARLY_PRINTK
4
3static unsigned long fs; 5static unsigned long fs;
4static inline void set_fs(unsigned long seg) 6static inline void set_fs(unsigned long seg)
5{ 7{
@@ -19,3 +21,5 @@ int cmdline_find_option_bool(const char *option)
19{ 21{
20 return __cmdline_find_option_bool(real_mode->hdr.cmd_line_ptr, option); 22 return __cmdline_find_option_bool(real_mode->hdr.cmd_line_ptr, option);
21} 23}
24
25#endif
diff --git a/arch/x86/boot/compressed/early_serial_console.c b/arch/x86/boot/compressed/early_serial_console.c
index 261e81fb9582..d3d003cb5481 100644
--- a/arch/x86/boot/compressed/early_serial_console.c
+++ b/arch/x86/boot/compressed/early_serial_console.c
@@ -1,5 +1,9 @@
1#include "misc.h" 1#include "misc.h"
2 2
3#ifdef CONFIG_EARLY_PRINTK
4
3int early_serial_base; 5int early_serial_base;
4 6
5#include "../early_serial_console.c" 7#include "../early_serial_console.c"
8
9#endif
diff --git a/arch/x86/boot/compressed/eboot.c b/arch/x86/boot/compressed/eboot.c
index 4e85f5f85837..b3e0227df2c9 100644
--- a/arch/x86/boot/compressed/eboot.c
+++ b/arch/x86/boot/compressed/eboot.c
@@ -729,32 +729,68 @@ fail:
729 * need to create one ourselves (usually the bootloader would create 729 * need to create one ourselves (usually the bootloader would create
730 * one for us). 730 * one for us).
731 */ 731 */
732static efi_status_t make_boot_params(struct boot_params *boot_params, 732struct boot_params *make_boot_params(void *handle, efi_system_table_t *_table)
733 efi_loaded_image_t *image,
734 void *handle)
735{ 733{
736 struct efi_info *efi = &boot_params->efi_info; 734 struct boot_params *boot_params;
737 struct apm_bios_info *bi = &boot_params->apm_bios_info; 735 struct sys_desc_table *sdt;
738 struct sys_desc_table *sdt = &boot_params->sys_desc_table; 736 struct apm_bios_info *bi;
739 struct e820entry *e820_map = &boot_params->e820_map[0]; 737 struct setup_header *hdr;
740 struct e820entry *prev = NULL; 738 struct efi_info *efi;
741 struct setup_header *hdr = &boot_params->hdr; 739 efi_loaded_image_t *image;
742 unsigned long size, key, desc_size, _size; 740 void *options;
743 efi_memory_desc_t *mem_map; 741 u32 load_options_size;
744 void *options = image->load_options; 742 efi_guid_t proto = LOADED_IMAGE_PROTOCOL_GUID;
745 u32 load_options_size = image->load_options_size / 2; /* ASCII */
746 int options_size = 0; 743 int options_size = 0;
747 efi_status_t status; 744 efi_status_t status;
748 __u32 desc_version;
749 unsigned long cmdline; 745 unsigned long cmdline;
750 u8 nr_entries;
751 u16 *s2; 746 u16 *s2;
752 u8 *s1; 747 u8 *s1;
753 int i; 748 int i;
754 749
750 sys_table = _table;
751
752 /* Check if we were booted by the EFI firmware */
753 if (sys_table->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE)
754 return NULL;
755
756 status = efi_call_phys3(sys_table->boottime->handle_protocol,
757 handle, &proto, (void *)&image);
758 if (status != EFI_SUCCESS) {
759 efi_printk("Failed to get handle for LOADED_IMAGE_PROTOCOL\n");
760 return NULL;
761 }
762
763 status = low_alloc(0x4000, 1, (unsigned long *)&boot_params);
764 if (status != EFI_SUCCESS) {
765 efi_printk("Failed to alloc lowmem for boot params\n");
766 return NULL;
767 }
768
769 memset(boot_params, 0x0, 0x4000);
770
771 hdr = &boot_params->hdr;
772 efi = &boot_params->efi_info;
773 bi = &boot_params->apm_bios_info;
774 sdt = &boot_params->sys_desc_table;
775
776 /* Copy the second sector to boot_params */
777 memcpy(&hdr->jump, image->image_base + 512, 512);
778
779 /*
780 * Fill out some of the header fields ourselves because the
781 * EFI firmware loader doesn't load the first sector.
782 */
783 hdr->root_flags = 1;
784 hdr->vid_mode = 0xffff;
785 hdr->boot_flag = 0xAA55;
786
787 hdr->code32_start = (__u64)(unsigned long)image->image_base;
788
755 hdr->type_of_loader = 0x21; 789 hdr->type_of_loader = 0x21;
756 790
757 /* Convert unicode cmdline to ascii */ 791 /* Convert unicode cmdline to ascii */
792 options = image->load_options;
793 load_options_size = image->load_options_size / 2; /* ASCII */
758 cmdline = 0; 794 cmdline = 0;
759 s2 = (u16 *)options; 795 s2 = (u16 *)options;
760 796
@@ -791,18 +827,36 @@ static efi_status_t make_boot_params(struct boot_params *boot_params,
791 hdr->ramdisk_image = 0; 827 hdr->ramdisk_image = 0;
792 hdr->ramdisk_size = 0; 828 hdr->ramdisk_size = 0;
793 829
794 status = handle_ramdisks(image, hdr);
795 if (status != EFI_SUCCESS)
796 goto free_cmdline;
797
798 setup_graphics(boot_params);
799
800 /* Clear APM BIOS info */ 830 /* Clear APM BIOS info */
801 memset(bi, 0, sizeof(*bi)); 831 memset(bi, 0, sizeof(*bi));
802 832
803 memset(sdt, 0, sizeof(*sdt)); 833 memset(sdt, 0, sizeof(*sdt));
804 834
805 memcpy(&efi->efi_loader_signature, EFI_LOADER_SIGNATURE, sizeof(__u32)); 835 status = handle_ramdisks(image, hdr);
836 if (status != EFI_SUCCESS)
837 goto fail2;
838
839 return boot_params;
840fail2:
841 if (options_size)
842 low_free(options_size, hdr->cmd_line_ptr);
843fail:
844 low_free(0x4000, (unsigned long)boot_params);
845 return NULL;
846}
847
848static efi_status_t exit_boot(struct boot_params *boot_params,
849 void *handle)
850{
851 struct efi_info *efi = &boot_params->efi_info;
852 struct e820entry *e820_map = &boot_params->e820_map[0];
853 struct e820entry *prev = NULL;
854 unsigned long size, key, desc_size, _size;
855 efi_memory_desc_t *mem_map;
856 efi_status_t status;
857 __u32 desc_version;
858 u8 nr_entries;
859 int i;
806 860
807 size = sizeof(*mem_map) * 32; 861 size = sizeof(*mem_map) * 32;
808 862
@@ -811,7 +865,7 @@ again:
811 _size = size; 865 _size = size;
812 status = low_alloc(size, 1, (unsigned long *)&mem_map); 866 status = low_alloc(size, 1, (unsigned long *)&mem_map);
813 if (status != EFI_SUCCESS) 867 if (status != EFI_SUCCESS)
814 goto free_cmdline; 868 return status;
815 869
816 status = efi_call_phys5(sys_table->boottime->get_memory_map, &size, 870 status = efi_call_phys5(sys_table->boottime->get_memory_map, &size,
817 mem_map, &key, &desc_size, &desc_version); 871 mem_map, &key, &desc_size, &desc_version);
@@ -823,6 +877,7 @@ again:
823 if (status != EFI_SUCCESS) 877 if (status != EFI_SUCCESS)
824 goto free_mem_map; 878 goto free_mem_map;
825 879
880 memcpy(&efi->efi_loader_signature, EFI_LOADER_SIGNATURE, sizeof(__u32));
826 efi->efi_systab = (unsigned long)sys_table; 881 efi->efi_systab = (unsigned long)sys_table;
827 efi->efi_memdesc_size = desc_size; 882 efi->efi_memdesc_size = desc_size;
828 efi->efi_memdesc_version = desc_version; 883 efi->efi_memdesc_version = desc_version;
@@ -906,61 +961,13 @@ again:
906 961
907free_mem_map: 962free_mem_map:
908 low_free(_size, (unsigned long)mem_map); 963 low_free(_size, (unsigned long)mem_map);
909free_cmdline:
910 if (options_size)
911 low_free(options_size, hdr->cmd_line_ptr);
912fail:
913 return status; 964 return status;
914} 965}
915 966
916/* 967static efi_status_t relocate_kernel(struct setup_header *hdr)
917 * On success we return a pointer to a boot_params structure, and NULL
918 * on failure.
919 */
920struct boot_params *efi_main(void *handle, efi_system_table_t *_table)
921{ 968{
922 struct boot_params *boot_params;
923 unsigned long start, nr_pages; 969 unsigned long start, nr_pages;
924 struct desc_ptr *gdt, *idt;
925 efi_loaded_image_t *image;
926 struct setup_header *hdr;
927 efi_status_t status; 970 efi_status_t status;
928 efi_guid_t proto = LOADED_IMAGE_PROTOCOL_GUID;
929 struct desc_struct *desc;
930
931 sys_table = _table;
932
933 /* Check if we were booted by the EFI firmware */
934 if (sys_table->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE)
935 goto fail;
936
937 status = efi_call_phys3(sys_table->boottime->handle_protocol,
938 handle, &proto, (void *)&image);
939 if (status != EFI_SUCCESS) {
940 efi_printk("Failed to get handle for LOADED_IMAGE_PROTOCOL\n");
941 goto fail;
942 }
943
944 status = low_alloc(0x4000, 1, (unsigned long *)&boot_params);
945 if (status != EFI_SUCCESS) {
946 efi_printk("Failed to alloc lowmem for boot params\n");
947 goto fail;
948 }
949
950 memset(boot_params, 0x0, 0x4000);
951
952 hdr = &boot_params->hdr;
953
954 /* Copy the second sector to boot_params */
955 memcpy(&hdr->jump, image->image_base + 512, 512);
956
957 /*
958 * Fill out some of the header fields ourselves because the
959 * EFI firmware loader doesn't load the first sector.
960 */
961 hdr->root_flags = 1;
962 hdr->vid_mode = 0xffff;
963 hdr->boot_flag = 0xAA55;
964 971
965 /* 972 /*
966 * The EFI firmware loader could have placed the kernel image 973 * The EFI firmware loader could have placed the kernel image
@@ -978,16 +985,40 @@ struct boot_params *efi_main(void *handle, efi_system_table_t *_table)
978 if (status != EFI_SUCCESS) { 985 if (status != EFI_SUCCESS) {
979 status = low_alloc(hdr->init_size, hdr->kernel_alignment, 986 status = low_alloc(hdr->init_size, hdr->kernel_alignment,
980 &start); 987 &start);
981 if (status != EFI_SUCCESS) { 988 if (status != EFI_SUCCESS)
982 efi_printk("Failed to alloc mem for kernel\n"); 989 efi_printk("Failed to alloc mem for kernel\n");
983 goto fail;
984 }
985 } 990 }
986 991
992 if (status == EFI_SUCCESS)
993 memcpy((void *)start, (void *)(unsigned long)hdr->code32_start,
994 hdr->init_size);
995
996 hdr->pref_address = hdr->code32_start;
987 hdr->code32_start = (__u32)start; 997 hdr->code32_start = (__u32)start;
988 hdr->pref_address = (__u64)(unsigned long)image->image_base;
989 998
990 memcpy((void *)start, image->image_base, image->image_size); 999 return status;
1000}
1001
1002/*
1003 * On success we return a pointer to a boot_params structure, and NULL
1004 * on failure.
1005 */
1006struct boot_params *efi_main(void *handle, efi_system_table_t *_table,
1007 struct boot_params *boot_params)
1008{
1009 struct desc_ptr *gdt, *idt;
1010 efi_loaded_image_t *image;
1011 struct setup_header *hdr = &boot_params->hdr;
1012 efi_status_t status;
1013 struct desc_struct *desc;
1014
1015 sys_table = _table;
1016
1017 /* Check if we were booted by the EFI firmware */
1018 if (sys_table->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE)
1019 goto fail;
1020
1021 setup_graphics(boot_params);
991 1022
992 status = efi_call_phys3(sys_table->boottime->allocate_pool, 1023 status = efi_call_phys3(sys_table->boottime->allocate_pool,
993 EFI_LOADER_DATA, sizeof(*gdt), 1024 EFI_LOADER_DATA, sizeof(*gdt),
@@ -1015,7 +1046,18 @@ struct boot_params *efi_main(void *handle, efi_system_table_t *_table)
1015 idt->size = 0; 1046 idt->size = 0;
1016 idt->address = 0; 1047 idt->address = 0;
1017 1048
1018 status = make_boot_params(boot_params, image, handle); 1049 /*
1050 * If the kernel isn't already loaded at the preferred load
1051 * address, relocate it.
1052 */
1053 if (hdr->pref_address != hdr->code32_start) {
1054 status = relocate_kernel(hdr);
1055
1056 if (status != EFI_SUCCESS)
1057 goto fail;
1058 }
1059
1060 status = exit_boot(boot_params, handle);
1019 if (status != EFI_SUCCESS) 1061 if (status != EFI_SUCCESS)
1020 goto fail; 1062 goto fail;
1021 1063
diff --git a/arch/x86/boot/compressed/head_32.S b/arch/x86/boot/compressed/head_32.S
index c85e3ac99bba..aa4aaf1b2380 100644
--- a/arch/x86/boot/compressed/head_32.S
+++ b/arch/x86/boot/compressed/head_32.S
@@ -42,6 +42,16 @@ ENTRY(startup_32)
42 */ 42 */
43 add $0x4, %esp 43 add $0x4, %esp
44 44
45 call make_boot_params
46 cmpl $0, %eax
47 je 1f
48 movl 0x4(%esp), %esi
49 movl (%esp), %ecx
50 pushl %eax
51 pushl %esi
52 pushl %ecx
53
54 .org 0x30,0x90
45 call efi_main 55 call efi_main
46 cmpl $0, %eax 56 cmpl $0, %eax
47 movl %eax, %esi 57 movl %eax, %esi
diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S
index 87e03a13d8e3..2c4b171eec33 100644
--- a/arch/x86/boot/compressed/head_64.S
+++ b/arch/x86/boot/compressed/head_64.S
@@ -209,6 +209,16 @@ ENTRY(startup_64)
209 .org 0x210 209 .org 0x210
210 mov %rcx, %rdi 210 mov %rcx, %rdi
211 mov %rdx, %rsi 211 mov %rdx, %rsi
212 pushq %rdi
213 pushq %rsi
214 call make_boot_params
215 cmpq $0,%rax
216 je 1f
217 mov %rax, %rdx
218 popq %rsi
219 popq %rdi
220
221 .org 0x230,0x90
212 call efi_main 222 call efi_main
213 movq %rax,%rsi 223 movq %rax,%rsi
214 cmpq $0,%rax 224 cmpq $0,%rax
diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c
index 7116dcba0c9e..88f7ff6da404 100644
--- a/arch/x86/boot/compressed/misc.c
+++ b/arch/x86/boot/compressed/misc.c
@@ -108,8 +108,6 @@ static void error(char *m);
108 * This is set up by the setup-routine at boot-time 108 * This is set up by the setup-routine at boot-time
109 */ 109 */
110struct boot_params *real_mode; /* Pointer to real-mode data */ 110struct boot_params *real_mode; /* Pointer to real-mode data */
111static int quiet;
112static int debug;
113 111
114void *memset(void *s, int c, size_t n); 112void *memset(void *s, int c, size_t n);
115void *memcpy(void *dest, const void *src, size_t n); 113void *memcpy(void *dest, const void *src, size_t n);
@@ -170,15 +168,11 @@ static void serial_putchar(int ch)
170 outb(ch, early_serial_base + TXR); 168 outb(ch, early_serial_base + TXR);
171} 169}
172 170
173void __putstr(int error, const char *s) 171void __putstr(const char *s)
174{ 172{
175 int x, y, pos; 173 int x, y, pos;
176 char c; 174 char c;
177 175
178#ifndef CONFIG_X86_VERBOSE_BOOTUP
179 if (!error)
180 return;
181#endif
182 if (early_serial_base) { 176 if (early_serial_base) {
183 const char *str = s; 177 const char *str = s;
184 while (*str) { 178 while (*str) {
@@ -265,9 +259,9 @@ void *memcpy(void *dest, const void *src, size_t n)
265 259
266static void error(char *x) 260static void error(char *x)
267{ 261{
268 __putstr(1, "\n\n"); 262 error_putstr("\n\n");
269 __putstr(1, x); 263 error_putstr(x);
270 __putstr(1, "\n\n -- System halted"); 264 error_putstr("\n\n -- System halted");
271 265
272 while (1) 266 while (1)
273 asm("hlt"); 267 asm("hlt");
@@ -294,8 +288,7 @@ static void parse_elf(void *output)
294 return; 288 return;
295 } 289 }
296 290
297 if (!quiet) 291 debug_putstr("Parsing ELF... ");
298 putstr("Parsing ELF... ");
299 292
300 phdrs = malloc(sizeof(*phdrs) * ehdr.e_phnum); 293 phdrs = malloc(sizeof(*phdrs) * ehdr.e_phnum);
301 if (!phdrs) 294 if (!phdrs)
@@ -332,11 +325,6 @@ asmlinkage void decompress_kernel(void *rmode, memptr heap,
332{ 325{
333 real_mode = rmode; 326 real_mode = rmode;
334 327
335 if (cmdline_find_option_bool("quiet"))
336 quiet = 1;
337 if (cmdline_find_option_bool("debug"))
338 debug = 1;
339
340 if (real_mode->screen_info.orig_video_mode == 7) { 328 if (real_mode->screen_info.orig_video_mode == 7) {
341 vidmem = (char *) 0xb0000; 329 vidmem = (char *) 0xb0000;
342 vidport = 0x3b4; 330 vidport = 0x3b4;
@@ -349,8 +337,7 @@ asmlinkage void decompress_kernel(void *rmode, memptr heap,
349 cols = real_mode->screen_info.orig_video_cols; 337 cols = real_mode->screen_info.orig_video_cols;
350 338
351 console_init(); 339 console_init();
352 if (debug) 340 debug_putstr("early console in decompress_kernel\n");
353 putstr("early console in decompress_kernel\n");
354 341
355 free_mem_ptr = heap; /* Heap */ 342 free_mem_ptr = heap; /* Heap */
356 free_mem_end_ptr = heap + BOOT_HEAP_SIZE; 343 free_mem_end_ptr = heap + BOOT_HEAP_SIZE;
@@ -369,11 +356,9 @@ asmlinkage void decompress_kernel(void *rmode, memptr heap,
369 error("Wrong destination address"); 356 error("Wrong destination address");
370#endif 357#endif
371 358
372 if (!quiet) 359 debug_putstr("\nDecompressing Linux... ");
373 putstr("\nDecompressing Linux... ");
374 decompress(input_data, input_len, NULL, NULL, output, NULL, error); 360 decompress(input_data, input_len, NULL, NULL, output, NULL, error);
375 parse_elf(output); 361 parse_elf(output);
376 if (!quiet) 362 debug_putstr("done.\nBooting the kernel.\n");
377 putstr("done.\nBooting the kernel.\n");
378 return; 363 return;
379} 364}
diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h
index 3f19c81a6203..0e6dc0ee0eea 100644
--- a/arch/x86/boot/compressed/misc.h
+++ b/arch/x86/boot/compressed/misc.h
@@ -24,9 +24,21 @@
24 24
25/* misc.c */ 25/* misc.c */
26extern struct boot_params *real_mode; /* Pointer to real-mode data */ 26extern struct boot_params *real_mode; /* Pointer to real-mode data */
27void __putstr(int error, const char *s); 27void __putstr(const char *s);
28#define putstr(__x) __putstr(0, __x) 28#define error_putstr(__x) __putstr(__x)
29#define puts(__x) __putstr(0, __x) 29
30#ifdef CONFIG_X86_VERBOSE_BOOTUP
31
32#define debug_putstr(__x) __putstr(__x)
33
34#else
35
36static inline void debug_putstr(const char *s)
37{ }
38
39#endif
40
41#ifdef CONFIG_EARLY_PRINTK
30 42
31/* cmdline.c */ 43/* cmdline.c */
32int cmdline_find_option(const char *option, char *buffer, int bufsize); 44int cmdline_find_option(const char *option, char *buffer, int bufsize);
@@ -36,4 +48,13 @@ int cmdline_find_option_bool(const char *option);
36extern int early_serial_base; 48extern int early_serial_base;
37void console_init(void); 49void console_init(void);
38 50
51#else
52
53/* early_serial_console.c */
54static const int early_serial_base;
55static inline void console_init(void)
56{ }
57
58#endif
59
39#endif 60#endif
diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S
index efe5acfc79c3..b4e15dd6786a 100644
--- a/arch/x86/boot/header.S
+++ b/arch/x86/boot/header.S
@@ -283,7 +283,7 @@ _start:
283 # Part 2 of the header, from the old setup.S 283 # Part 2 of the header, from the old setup.S
284 284
285 .ascii "HdrS" # header signature 285 .ascii "HdrS" # header signature
286 .word 0x020a # header version number (>= 0x0105) 286 .word 0x020b # header version number (>= 0x0105)
287 # or else old loadlin-1.5 will fail) 287 # or else old loadlin-1.5 will fail)
288 .globl realmode_swtch 288 .globl realmode_swtch
289realmode_swtch: .word 0, 0 # default_switch, SETUPSEG 289realmode_swtch: .word 0, 0 # default_switch, SETUPSEG
@@ -401,18 +401,13 @@ pref_address: .quad LOAD_PHYSICAL_ADDR # preferred load addr
401#define INIT_SIZE VO_INIT_SIZE 401#define INIT_SIZE VO_INIT_SIZE
402#endif 402#endif
403init_size: .long INIT_SIZE # kernel initialization size 403init_size: .long INIT_SIZE # kernel initialization size
404handover_offset: .long 0x30 # offset to the handover
405 # protocol entry point
404 406
405# End of setup header ##################################################### 407# End of setup header #####################################################
406 408
407 .section ".entrytext", "ax" 409 .section ".entrytext", "ax"
408start_of_setup: 410start_of_setup:
409#ifdef SAFE_RESET_DISK_CONTROLLER
410# Reset the disk controller.
411 movw $0x0000, %ax # Reset disk controller
412 movb $0x80, %dl # All disks
413 int $0x13
414#endif
415
416# Force %es = %ds 411# Force %es = %ds
417 movw %ds, %ax 412 movw %ds, %ax
418 movw %ax, %es 413 movw %ax, %es
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index e191ac048b59..e908e5de82d3 100644
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -2,6 +2,9 @@
2# Arch-specific CryptoAPI modules. 2# Arch-specific CryptoAPI modules.
3# 3#
4 4
5obj-$(CONFIG_CRYPTO_ABLK_HELPER_X86) += ablk_helper.o
6obj-$(CONFIG_CRYPTO_GLUE_HELPER_X86) += glue_helper.o
7
5obj-$(CONFIG_CRYPTO_AES_586) += aes-i586.o 8obj-$(CONFIG_CRYPTO_AES_586) += aes-i586.o
6obj-$(CONFIG_CRYPTO_TWOFISH_586) += twofish-i586.o 9obj-$(CONFIG_CRYPTO_TWOFISH_586) += twofish-i586.o
7obj-$(CONFIG_CRYPTO_SALSA20_586) += salsa20-i586.o 10obj-$(CONFIG_CRYPTO_SALSA20_586) += salsa20-i586.o
@@ -12,8 +15,10 @@ obj-$(CONFIG_CRYPTO_CAMELLIA_X86_64) += camellia-x86_64.o
12obj-$(CONFIG_CRYPTO_BLOWFISH_X86_64) += blowfish-x86_64.o 15obj-$(CONFIG_CRYPTO_BLOWFISH_X86_64) += blowfish-x86_64.o
13obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o 16obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o
14obj-$(CONFIG_CRYPTO_TWOFISH_X86_64_3WAY) += twofish-x86_64-3way.o 17obj-$(CONFIG_CRYPTO_TWOFISH_X86_64_3WAY) += twofish-x86_64-3way.o
18obj-$(CONFIG_CRYPTO_TWOFISH_AVX_X86_64) += twofish-avx-x86_64.o
15obj-$(CONFIG_CRYPTO_SALSA20_X86_64) += salsa20-x86_64.o 19obj-$(CONFIG_CRYPTO_SALSA20_X86_64) += salsa20-x86_64.o
16obj-$(CONFIG_CRYPTO_SERPENT_SSE2_X86_64) += serpent-sse2-x86_64.o 20obj-$(CONFIG_CRYPTO_SERPENT_SSE2_X86_64) += serpent-sse2-x86_64.o
21obj-$(CONFIG_CRYPTO_SERPENT_AVX_X86_64) += serpent-avx-x86_64.o
17obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o 22obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o
18obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o 23obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o
19 24
@@ -30,16 +35,11 @@ camellia-x86_64-y := camellia-x86_64-asm_64.o camellia_glue.o
30blowfish-x86_64-y := blowfish-x86_64-asm_64.o blowfish_glue.o 35blowfish-x86_64-y := blowfish-x86_64-asm_64.o blowfish_glue.o
31twofish-x86_64-y := twofish-x86_64-asm_64.o twofish_glue.o 36twofish-x86_64-y := twofish-x86_64-asm_64.o twofish_glue.o
32twofish-x86_64-3way-y := twofish-x86_64-asm_64-3way.o twofish_glue_3way.o 37twofish-x86_64-3way-y := twofish-x86_64-asm_64-3way.o twofish_glue_3way.o
38twofish-avx-x86_64-y := twofish-avx-x86_64-asm_64.o twofish_avx_glue.o
33salsa20-x86_64-y := salsa20-x86_64-asm_64.o salsa20_glue.o 39salsa20-x86_64-y := salsa20-x86_64-asm_64.o salsa20_glue.o
34serpent-sse2-x86_64-y := serpent-sse2-x86_64-asm_64.o serpent_sse2_glue.o 40serpent-sse2-x86_64-y := serpent-sse2-x86_64-asm_64.o serpent_sse2_glue.o
41serpent-avx-x86_64-y := serpent-avx-x86_64-asm_64.o serpent_avx_glue.o
35 42
36aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o fpu.o 43aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o fpu.o
37
38ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o 44ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o
39
40# enable AVX support only when $(AS) can actually assemble the instructions
41ifeq ($(call as-instr,vpxor %xmm0$(comma)%xmm1$(comma)%xmm2,yes,no),yes)
42AFLAGS_sha1_ssse3_asm.o += -DSHA1_ENABLE_AVX_SUPPORT
43CFLAGS_sha1_ssse3_glue.o += -DSHA1_ENABLE_AVX_SUPPORT
44endif
45sha1-ssse3-y := sha1_ssse3_asm.o sha1_ssse3_glue.o 45sha1-ssse3-y := sha1_ssse3_asm.o sha1_ssse3_glue.o
diff --git a/arch/x86/crypto/ablk_helper.c b/arch/x86/crypto/ablk_helper.c
new file mode 100644
index 000000000000..43282fe04a8b
--- /dev/null
+++ b/arch/x86/crypto/ablk_helper.c
@@ -0,0 +1,149 @@
1/*
2 * Shared async block cipher helpers
3 *
4 * Copyright (c) 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
5 *
6 * Based on aesni-intel_glue.c by:
7 * Copyright (C) 2008, Intel Corp.
8 * Author: Huang Ying <ying.huang@intel.com>
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
14 *
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
19 *
20 * You should have received a copy of the GNU General Public License
21 * along with this program; if not, write to the Free Software
22 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
23 * USA
24 *
25 */
26
27#include <linux/kernel.h>
28#include <linux/crypto.h>
29#include <linux/init.h>
30#include <linux/module.h>
31#include <crypto/algapi.h>
32#include <crypto/cryptd.h>
33#include <asm/i387.h>
34#include <asm/crypto/ablk_helper.h>
35
36int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
37 unsigned int key_len)
38{
39 struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm);
40 struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base;
41 int err;
42
43 crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
44 crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm)
45 & CRYPTO_TFM_REQ_MASK);
46 err = crypto_ablkcipher_setkey(child, key, key_len);
47 crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child)
48 & CRYPTO_TFM_RES_MASK);
49 return err;
50}
51EXPORT_SYMBOL_GPL(ablk_set_key);
52
53int __ablk_encrypt(struct ablkcipher_request *req)
54{
55 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
56 struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm);
57 struct blkcipher_desc desc;
58
59 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
60 desc.info = req->info;
61 desc.flags = 0;
62
63 return crypto_blkcipher_crt(desc.tfm)->encrypt(
64 &desc, req->dst, req->src, req->nbytes);
65}
66EXPORT_SYMBOL_GPL(__ablk_encrypt);
67
68int ablk_encrypt(struct ablkcipher_request *req)
69{
70 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
71 struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm);
72
73 if (!irq_fpu_usable()) {
74 struct ablkcipher_request *cryptd_req =
75 ablkcipher_request_ctx(req);
76
77 memcpy(cryptd_req, req, sizeof(*req));
78 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
79
80 return crypto_ablkcipher_encrypt(cryptd_req);
81 } else {
82 return __ablk_encrypt(req);
83 }
84}
85EXPORT_SYMBOL_GPL(ablk_encrypt);
86
87int ablk_decrypt(struct ablkcipher_request *req)
88{
89 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
90 struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm);
91
92 if (!irq_fpu_usable()) {
93 struct ablkcipher_request *cryptd_req =
94 ablkcipher_request_ctx(req);
95
96 memcpy(cryptd_req, req, sizeof(*req));
97 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
98
99 return crypto_ablkcipher_decrypt(cryptd_req);
100 } else {
101 struct blkcipher_desc desc;
102
103 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
104 desc.info = req->info;
105 desc.flags = 0;
106
107 return crypto_blkcipher_crt(desc.tfm)->decrypt(
108 &desc, req->dst, req->src, req->nbytes);
109 }
110}
111EXPORT_SYMBOL_GPL(ablk_decrypt);
112
113void ablk_exit(struct crypto_tfm *tfm)
114{
115 struct async_helper_ctx *ctx = crypto_tfm_ctx(tfm);
116
117 cryptd_free_ablkcipher(ctx->cryptd_tfm);
118}
119EXPORT_SYMBOL_GPL(ablk_exit);
120
121int ablk_init_common(struct crypto_tfm *tfm, const char *drv_name)
122{
123 struct async_helper_ctx *ctx = crypto_tfm_ctx(tfm);
124 struct cryptd_ablkcipher *cryptd_tfm;
125
126 cryptd_tfm = cryptd_alloc_ablkcipher(drv_name, 0, 0);
127 if (IS_ERR(cryptd_tfm))
128 return PTR_ERR(cryptd_tfm);
129
130 ctx->cryptd_tfm = cryptd_tfm;
131 tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
132 crypto_ablkcipher_reqsize(&cryptd_tfm->base);
133
134 return 0;
135}
136EXPORT_SYMBOL_GPL(ablk_init_common);
137
138int ablk_init(struct crypto_tfm *tfm)
139{
140 char drv_name[CRYPTO_MAX_ALG_NAME];
141
142 snprintf(drv_name, sizeof(drv_name), "__driver-%s",
143 crypto_tfm_alg_driver_name(tfm));
144
145 return ablk_init_common(tfm, drv_name);
146}
147EXPORT_SYMBOL_GPL(ablk_init);
148
149MODULE_LICENSE("GPL");
diff --git a/arch/x86/crypto/aes_glue.c b/arch/x86/crypto/aes_glue.c
index 8efcf42a9d7e..59b37deb8c8d 100644
--- a/arch/x86/crypto/aes_glue.c
+++ b/arch/x86/crypto/aes_glue.c
@@ -5,7 +5,7 @@
5 5
6#include <linux/module.h> 6#include <linux/module.h>
7#include <crypto/aes.h> 7#include <crypto/aes.h>
8#include <asm/aes.h> 8#include <asm/crypto/aes.h>
9 9
10asmlinkage void aes_enc_blk(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in); 10asmlinkage void aes_enc_blk(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in);
11asmlinkage void aes_dec_blk(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in); 11asmlinkage void aes_dec_blk(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in);
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index ac7f5cd019e8..34fdcff4d2c8 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -30,7 +30,8 @@
30#include <crypto/ctr.h> 30#include <crypto/ctr.h>
31#include <asm/cpu_device_id.h> 31#include <asm/cpu_device_id.h>
32#include <asm/i387.h> 32#include <asm/i387.h>
33#include <asm/aes.h> 33#include <asm/crypto/aes.h>
34#include <asm/crypto/ablk_helper.h>
34#include <crypto/scatterwalk.h> 35#include <crypto/scatterwalk.h>
35#include <crypto/internal/aead.h> 36#include <crypto/internal/aead.h>
36#include <linux/workqueue.h> 37#include <linux/workqueue.h>
@@ -52,10 +53,6 @@
52#define HAS_XTS 53#define HAS_XTS
53#endif 54#endif
54 55
55struct async_aes_ctx {
56 struct cryptd_ablkcipher *cryptd_tfm;
57};
58
59/* This data is stored at the end of the crypto_tfm struct. 56/* This data is stored at the end of the crypto_tfm struct.
60 * It's a type of per "session" data storage location. 57 * It's a type of per "session" data storage location.
61 * This needs to be 16 byte aligned. 58 * This needs to be 16 byte aligned.
@@ -377,87 +374,6 @@ static int ctr_crypt(struct blkcipher_desc *desc,
377} 374}
378#endif 375#endif
379 376
380static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
381 unsigned int key_len)
382{
383 struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
384 struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base;
385 int err;
386
387 crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
388 crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm)
389 & CRYPTO_TFM_REQ_MASK);
390 err = crypto_ablkcipher_setkey(child, key, key_len);
391 crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child)
392 & CRYPTO_TFM_RES_MASK);
393 return err;
394}
395
396static int ablk_encrypt(struct ablkcipher_request *req)
397{
398 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
399 struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
400
401 if (!irq_fpu_usable()) {
402 struct ablkcipher_request *cryptd_req =
403 ablkcipher_request_ctx(req);
404 memcpy(cryptd_req, req, sizeof(*req));
405 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
406 return crypto_ablkcipher_encrypt(cryptd_req);
407 } else {
408 struct blkcipher_desc desc;
409 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
410 desc.info = req->info;
411 desc.flags = 0;
412 return crypto_blkcipher_crt(desc.tfm)->encrypt(
413 &desc, req->dst, req->src, req->nbytes);
414 }
415}
416
417static int ablk_decrypt(struct ablkcipher_request *req)
418{
419 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
420 struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
421
422 if (!irq_fpu_usable()) {
423 struct ablkcipher_request *cryptd_req =
424 ablkcipher_request_ctx(req);
425 memcpy(cryptd_req, req, sizeof(*req));
426 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
427 return crypto_ablkcipher_decrypt(cryptd_req);
428 } else {
429 struct blkcipher_desc desc;
430 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
431 desc.info = req->info;
432 desc.flags = 0;
433 return crypto_blkcipher_crt(desc.tfm)->decrypt(
434 &desc, req->dst, req->src, req->nbytes);
435 }
436}
437
438static void ablk_exit(struct crypto_tfm *tfm)
439{
440 struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
441
442 cryptd_free_ablkcipher(ctx->cryptd_tfm);
443}
444
445static int ablk_init_common(struct crypto_tfm *tfm, const char *drv_name)
446{
447 struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
448 struct cryptd_ablkcipher *cryptd_tfm;
449
450 cryptd_tfm = cryptd_alloc_ablkcipher(drv_name, 0, 0);
451 if (IS_ERR(cryptd_tfm))
452 return PTR_ERR(cryptd_tfm);
453
454 ctx->cryptd_tfm = cryptd_tfm;
455 tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
456 crypto_ablkcipher_reqsize(&cryptd_tfm->base);
457
458 return 0;
459}
460
461static int ablk_ecb_init(struct crypto_tfm *tfm) 377static int ablk_ecb_init(struct crypto_tfm *tfm)
462{ 378{
463 return ablk_init_common(tfm, "__driver-ecb-aes-aesni"); 379 return ablk_init_common(tfm, "__driver-ecb-aes-aesni");
@@ -613,7 +529,7 @@ static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
613 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); 529 struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
614 struct aesni_rfc4106_gcm_ctx *child_ctx = 530 struct aesni_rfc4106_gcm_ctx *child_ctx =
615 aesni_rfc4106_gcm_ctx_get(cryptd_child); 531 aesni_rfc4106_gcm_ctx_get(cryptd_child);
616 u8 *new_key_mem = NULL; 532 u8 *new_key_align, *new_key_mem = NULL;
617 533
618 if (key_len < 4) { 534 if (key_len < 4) {
619 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); 535 crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
@@ -637,9 +553,9 @@ static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
637 if (!new_key_mem) 553 if (!new_key_mem)
638 return -ENOMEM; 554 return -ENOMEM;
639 555
640 new_key_mem = PTR_ALIGN(new_key_mem, AESNI_ALIGN); 556 new_key_align = PTR_ALIGN(new_key_mem, AESNI_ALIGN);
641 memcpy(new_key_mem, key, key_len); 557 memcpy(new_key_align, key, key_len);
642 key = new_key_mem; 558 key = new_key_align;
643 } 559 }
644 560
645 if (!irq_fpu_usable()) 561 if (!irq_fpu_usable())
@@ -968,7 +884,7 @@ static struct crypto_alg aesni_algs[] = { {
968 .cra_priority = 400, 884 .cra_priority = 400,
969 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 885 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
970 .cra_blocksize = AES_BLOCK_SIZE, 886 .cra_blocksize = AES_BLOCK_SIZE,
971 .cra_ctxsize = sizeof(struct async_aes_ctx), 887 .cra_ctxsize = sizeof(struct async_helper_ctx),
972 .cra_alignmask = 0, 888 .cra_alignmask = 0,
973 .cra_type = &crypto_ablkcipher_type, 889 .cra_type = &crypto_ablkcipher_type,
974 .cra_module = THIS_MODULE, 890 .cra_module = THIS_MODULE,
@@ -989,7 +905,7 @@ static struct crypto_alg aesni_algs[] = { {
989 .cra_priority = 400, 905 .cra_priority = 400,
990 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 906 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
991 .cra_blocksize = AES_BLOCK_SIZE, 907 .cra_blocksize = AES_BLOCK_SIZE,
992 .cra_ctxsize = sizeof(struct async_aes_ctx), 908 .cra_ctxsize = sizeof(struct async_helper_ctx),
993 .cra_alignmask = 0, 909 .cra_alignmask = 0,
994 .cra_type = &crypto_ablkcipher_type, 910 .cra_type = &crypto_ablkcipher_type,
995 .cra_module = THIS_MODULE, 911 .cra_module = THIS_MODULE,
@@ -1033,7 +949,7 @@ static struct crypto_alg aesni_algs[] = { {
1033 .cra_priority = 400, 949 .cra_priority = 400,
1034 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 950 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1035 .cra_blocksize = 1, 951 .cra_blocksize = 1,
1036 .cra_ctxsize = sizeof(struct async_aes_ctx), 952 .cra_ctxsize = sizeof(struct async_helper_ctx),
1037 .cra_alignmask = 0, 953 .cra_alignmask = 0,
1038 .cra_type = &crypto_ablkcipher_type, 954 .cra_type = &crypto_ablkcipher_type,
1039 .cra_module = THIS_MODULE, 955 .cra_module = THIS_MODULE,
@@ -1098,7 +1014,7 @@ static struct crypto_alg aesni_algs[] = { {
1098 .cra_priority = 400, 1014 .cra_priority = 400,
1099 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1015 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1100 .cra_blocksize = 1, 1016 .cra_blocksize = 1,
1101 .cra_ctxsize = sizeof(struct async_aes_ctx), 1017 .cra_ctxsize = sizeof(struct async_helper_ctx),
1102 .cra_alignmask = 0, 1018 .cra_alignmask = 0,
1103 .cra_type = &crypto_ablkcipher_type, 1019 .cra_type = &crypto_ablkcipher_type,
1104 .cra_module = THIS_MODULE, 1020 .cra_module = THIS_MODULE,
@@ -1126,7 +1042,7 @@ static struct crypto_alg aesni_algs[] = { {
1126 .cra_priority = 400, 1042 .cra_priority = 400,
1127 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1043 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1128 .cra_blocksize = AES_BLOCK_SIZE, 1044 .cra_blocksize = AES_BLOCK_SIZE,
1129 .cra_ctxsize = sizeof(struct async_aes_ctx), 1045 .cra_ctxsize = sizeof(struct async_helper_ctx),
1130 .cra_alignmask = 0, 1046 .cra_alignmask = 0,
1131 .cra_type = &crypto_ablkcipher_type, 1047 .cra_type = &crypto_ablkcipher_type,
1132 .cra_module = THIS_MODULE, 1048 .cra_module = THIS_MODULE,
@@ -1150,7 +1066,7 @@ static struct crypto_alg aesni_algs[] = { {
1150 .cra_priority = 400, 1066 .cra_priority = 400,
1151 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1067 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1152 .cra_blocksize = AES_BLOCK_SIZE, 1068 .cra_blocksize = AES_BLOCK_SIZE,
1153 .cra_ctxsize = sizeof(struct async_aes_ctx), 1069 .cra_ctxsize = sizeof(struct async_helper_ctx),
1154 .cra_alignmask = 0, 1070 .cra_alignmask = 0,
1155 .cra_type = &crypto_ablkcipher_type, 1071 .cra_type = &crypto_ablkcipher_type,
1156 .cra_module = THIS_MODULE, 1072 .cra_module = THIS_MODULE,
@@ -1174,7 +1090,7 @@ static struct crypto_alg aesni_algs[] = { {
1174 .cra_priority = 400, 1090 .cra_priority = 400,
1175 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 1091 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
1176 .cra_blocksize = AES_BLOCK_SIZE, 1092 .cra_blocksize = AES_BLOCK_SIZE,
1177 .cra_ctxsize = sizeof(struct async_aes_ctx), 1093 .cra_ctxsize = sizeof(struct async_helper_ctx),
1178 .cra_alignmask = 0, 1094 .cra_alignmask = 0,
1179 .cra_type = &crypto_ablkcipher_type, 1095 .cra_type = &crypto_ablkcipher_type,
1180 .cra_module = THIS_MODULE, 1096 .cra_module = THIS_MODULE,
diff --git a/arch/x86/crypto/camellia_glue.c b/arch/x86/crypto/camellia_glue.c
index 3306dc0b139e..eeb2b3b743e9 100644
--- a/arch/x86/crypto/camellia_glue.c
+++ b/arch/x86/crypto/camellia_glue.c
@@ -5,10 +5,6 @@
5 * 5 *
6 * Camellia parts based on code by: 6 * Camellia parts based on code by:
7 * Copyright (C) 2006 NTT (Nippon Telegraph and Telephone Corporation) 7 * Copyright (C) 2006 NTT (Nippon Telegraph and Telephone Corporation)
8 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
9 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
10 * CTR part based on code (crypto/ctr.c) by:
11 * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
12 * 8 *
13 * This program is free software; you can redistribute it and/or modify 9 * This program is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by 10 * it under the terms of the GNU General Public License as published by
@@ -34,9 +30,9 @@
34#include <linux/module.h> 30#include <linux/module.h>
35#include <linux/types.h> 31#include <linux/types.h>
36#include <crypto/algapi.h> 32#include <crypto/algapi.h>
37#include <crypto/b128ops.h>
38#include <crypto/lrw.h> 33#include <crypto/lrw.h>
39#include <crypto/xts.h> 34#include <crypto/xts.h>
35#include <asm/crypto/glue_helper.h>
40 36
41#define CAMELLIA_MIN_KEY_SIZE 16 37#define CAMELLIA_MIN_KEY_SIZE 16
42#define CAMELLIA_MAX_KEY_SIZE 32 38#define CAMELLIA_MAX_KEY_SIZE 32
@@ -1312,307 +1308,128 @@ static int camellia_setkey(struct crypto_tfm *tfm, const u8 *in_key,
1312 &tfm->crt_flags); 1308 &tfm->crt_flags);
1313} 1309}
1314 1310
1315static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk, 1311static void camellia_decrypt_cbc_2way(void *ctx, u128 *dst, const u128 *src)
1316 void (*fn)(struct camellia_ctx *, u8 *, const u8 *),
1317 void (*fn_2way)(struct camellia_ctx *, u8 *, const u8 *))
1318{ 1312{
1319 struct camellia_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 1313 u128 iv = *src;
1320 unsigned int bsize = CAMELLIA_BLOCK_SIZE;
1321 unsigned int nbytes;
1322 int err;
1323
1324 err = blkcipher_walk_virt(desc, walk);
1325
1326 while ((nbytes = walk->nbytes)) {
1327 u8 *wsrc = walk->src.virt.addr;
1328 u8 *wdst = walk->dst.virt.addr;
1329
1330 /* Process two block batch */
1331 if (nbytes >= bsize * 2) {
1332 do {
1333 fn_2way(ctx, wdst, wsrc);
1334
1335 wsrc += bsize * 2;
1336 wdst += bsize * 2;
1337 nbytes -= bsize * 2;
1338 } while (nbytes >= bsize * 2);
1339
1340 if (nbytes < bsize)
1341 goto done;
1342 }
1343
1344 /* Handle leftovers */
1345 do {
1346 fn(ctx, wdst, wsrc);
1347
1348 wsrc += bsize;
1349 wdst += bsize;
1350 nbytes -= bsize;
1351 } while (nbytes >= bsize);
1352
1353done:
1354 err = blkcipher_walk_done(desc, walk, nbytes);
1355 }
1356
1357 return err;
1358}
1359
1360static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
1361 struct scatterlist *src, unsigned int nbytes)
1362{
1363 struct blkcipher_walk walk;
1364
1365 blkcipher_walk_init(&walk, dst, src, nbytes);
1366 return ecb_crypt(desc, &walk, camellia_enc_blk, camellia_enc_blk_2way);
1367}
1368 1314
1369static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 1315 camellia_dec_blk_2way(ctx, (u8 *)dst, (u8 *)src);
1370 struct scatterlist *src, unsigned int nbytes)
1371{
1372 struct blkcipher_walk walk;
1373
1374 blkcipher_walk_init(&walk, dst, src, nbytes);
1375 return ecb_crypt(desc, &walk, camellia_dec_blk, camellia_dec_blk_2way);
1376}
1377 1316
1378static unsigned int __cbc_encrypt(struct blkcipher_desc *desc, 1317 u128_xor(&dst[1], &dst[1], &iv);
1379 struct blkcipher_walk *walk)
1380{
1381 struct camellia_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
1382 unsigned int bsize = CAMELLIA_BLOCK_SIZE;
1383 unsigned int nbytes = walk->nbytes;
1384 u128 *src = (u128 *)walk->src.virt.addr;
1385 u128 *dst = (u128 *)walk->dst.virt.addr;
1386 u128 *iv = (u128 *)walk->iv;
1387
1388 do {
1389 u128_xor(dst, src, iv);
1390 camellia_enc_blk(ctx, (u8 *)dst, (u8 *)dst);
1391 iv = dst;
1392
1393 src += 1;
1394 dst += 1;
1395 nbytes -= bsize;
1396 } while (nbytes >= bsize);
1397
1398 u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv);
1399 return nbytes;
1400} 1318}
1401 1319
1402static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 1320static void camellia_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv)
1403 struct scatterlist *src, unsigned int nbytes)
1404{ 1321{
1405 struct blkcipher_walk walk; 1322 be128 ctrblk;
1406 int err;
1407 1323
1408 blkcipher_walk_init(&walk, dst, src, nbytes); 1324 if (dst != src)
1409 err = blkcipher_walk_virt(desc, &walk); 1325 *dst = *src;
1410 1326
1411 while ((nbytes = walk.nbytes)) { 1327 u128_to_be128(&ctrblk, iv);
1412 nbytes = __cbc_encrypt(desc, &walk); 1328 u128_inc(iv);
1413 err = blkcipher_walk_done(desc, &walk, nbytes);
1414 }
1415 1329
1416 return err; 1330 camellia_enc_blk_xor(ctx, (u8 *)dst, (u8 *)&ctrblk);
1417} 1331}
1418 1332
1419static unsigned int __cbc_decrypt(struct blkcipher_desc *desc, 1333static void camellia_crypt_ctr_2way(void *ctx, u128 *dst, const u128 *src,
1420 struct blkcipher_walk *walk) 1334 u128 *iv)
1421{ 1335{
1422 struct camellia_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 1336 be128 ctrblks[2];
1423 unsigned int bsize = CAMELLIA_BLOCK_SIZE;
1424 unsigned int nbytes = walk->nbytes;
1425 u128 *src = (u128 *)walk->src.virt.addr;
1426 u128 *dst = (u128 *)walk->dst.virt.addr;
1427 u128 ivs[2 - 1];
1428 u128 last_iv;
1429 1337
1430 /* Start of the last block. */ 1338 if (dst != src) {
1431 src += nbytes / bsize - 1; 1339 dst[0] = src[0];
1432 dst += nbytes / bsize - 1; 1340 dst[1] = src[1];
1433
1434 last_iv = *src;
1435
1436 /* Process two block batch */
1437 if (nbytes >= bsize * 2) {
1438 do {
1439 nbytes -= bsize * (2 - 1);
1440 src -= 2 - 1;
1441 dst -= 2 - 1;
1442
1443 ivs[0] = src[0];
1444
1445 camellia_dec_blk_2way(ctx, (u8 *)dst, (u8 *)src);
1446
1447 u128_xor(dst + 1, dst + 1, ivs + 0);
1448
1449 nbytes -= bsize;
1450 if (nbytes < bsize)
1451 goto done;
1452
1453 u128_xor(dst, dst, src - 1);
1454 src -= 1;
1455 dst -= 1;
1456 } while (nbytes >= bsize * 2);
1457
1458 if (nbytes < bsize)
1459 goto done;
1460 } 1341 }
1461 1342
1462 /* Handle leftovers */ 1343 u128_to_be128(&ctrblks[0], iv);
1463 for (;;) { 1344 u128_inc(iv);
1464 camellia_dec_blk(ctx, (u8 *)dst, (u8 *)src); 1345 u128_to_be128(&ctrblks[1], iv);
1465 1346 u128_inc(iv);
1466 nbytes -= bsize;
1467 if (nbytes < bsize)
1468 break;
1469 1347
1470 u128_xor(dst, dst, src - 1); 1348 camellia_enc_blk_xor_2way(ctx, (u8 *)dst, (u8 *)ctrblks);
1471 src -= 1;
1472 dst -= 1;
1473 }
1474
1475done:
1476 u128_xor(dst, dst, (u128 *)walk->iv);
1477 *(u128 *)walk->iv = last_iv;
1478
1479 return nbytes;
1480} 1349}
1481 1350
1482static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 1351static const struct common_glue_ctx camellia_enc = {
1483 struct scatterlist *src, unsigned int nbytes) 1352 .num_funcs = 2,
1484{ 1353 .fpu_blocks_limit = -1,
1485 struct blkcipher_walk walk; 1354
1486 int err; 1355 .funcs = { {
1487 1356 .num_blocks = 2,
1488 blkcipher_walk_init(&walk, dst, src, nbytes); 1357 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk_2way) }
1489 err = blkcipher_walk_virt(desc, &walk); 1358 }, {
1359 .num_blocks = 1,
1360 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk) }
1361 } }
1362};
1490 1363
1491 while ((nbytes = walk.nbytes)) { 1364static const struct common_glue_ctx camellia_ctr = {
1492 nbytes = __cbc_decrypt(desc, &walk); 1365 .num_funcs = 2,
1493 err = blkcipher_walk_done(desc, &walk, nbytes); 1366 .fpu_blocks_limit = -1,
1494 } 1367
1368 .funcs = { {
1369 .num_blocks = 2,
1370 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr_2way) }
1371 }, {
1372 .num_blocks = 1,
1373 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr) }
1374 } }
1375};
1495 1376
1496 return err; 1377static const struct common_glue_ctx camellia_dec = {
1497} 1378 .num_funcs = 2,
1379 .fpu_blocks_limit = -1,
1380
1381 .funcs = { {
1382 .num_blocks = 2,
1383 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk_2way) }
1384 }, {
1385 .num_blocks = 1,
1386 .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk) }
1387 } }
1388};
1498 1389
1499static inline void u128_to_be128(be128 *dst, const u128 *src) 1390static const struct common_glue_ctx camellia_dec_cbc = {
1500{ 1391 .num_funcs = 2,
1501 dst->a = cpu_to_be64(src->a); 1392 .fpu_blocks_limit = -1,
1502 dst->b = cpu_to_be64(src->b); 1393
1503} 1394 .funcs = { {
1395 .num_blocks = 2,
1396 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_decrypt_cbc_2way) }
1397 }, {
1398 .num_blocks = 1,
1399 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_dec_blk) }
1400 } }
1401};
1504 1402
1505static inline void be128_to_u128(u128 *dst, const be128 *src) 1403static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
1404 struct scatterlist *src, unsigned int nbytes)
1506{ 1405{
1507 dst->a = be64_to_cpu(src->a); 1406 return glue_ecb_crypt_128bit(&camellia_enc, desc, dst, src, nbytes);
1508 dst->b = be64_to_cpu(src->b);
1509} 1407}
1510 1408
1511static inline void u128_inc(u128 *i) 1409static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
1410 struct scatterlist *src, unsigned int nbytes)
1512{ 1411{
1513 i->b++; 1412 return glue_ecb_crypt_128bit(&camellia_dec, desc, dst, src, nbytes);
1514 if (!i->b)
1515 i->a++;
1516} 1413}
1517 1414
1518static void ctr_crypt_final(struct blkcipher_desc *desc, 1415static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
1519 struct blkcipher_walk *walk) 1416 struct scatterlist *src, unsigned int nbytes)
1520{ 1417{
1521 struct camellia_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 1418 return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(camellia_enc_blk), desc,
1522 u8 keystream[CAMELLIA_BLOCK_SIZE]; 1419 dst, src, nbytes);
1523 u8 *src = walk->src.virt.addr;
1524 u8 *dst = walk->dst.virt.addr;
1525 unsigned int nbytes = walk->nbytes;
1526 u128 ctrblk;
1527
1528 memcpy(keystream, src, nbytes);
1529 camellia_enc_blk_xor(ctx, keystream, walk->iv);
1530 memcpy(dst, keystream, nbytes);
1531
1532 be128_to_u128(&ctrblk, (be128 *)walk->iv);
1533 u128_inc(&ctrblk);
1534 u128_to_be128((be128 *)walk->iv, &ctrblk);
1535} 1420}
1536 1421
1537static unsigned int __ctr_crypt(struct blkcipher_desc *desc, 1422static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
1538 struct blkcipher_walk *walk) 1423 struct scatterlist *src, unsigned int nbytes)
1539{ 1424{
1540 struct camellia_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 1425 return glue_cbc_decrypt_128bit(&camellia_dec_cbc, desc, dst, src,
1541 unsigned int bsize = CAMELLIA_BLOCK_SIZE; 1426 nbytes);
1542 unsigned int nbytes = walk->nbytes;
1543 u128 *src = (u128 *)walk->src.virt.addr;
1544 u128 *dst = (u128 *)walk->dst.virt.addr;
1545 u128 ctrblk;
1546 be128 ctrblocks[2];
1547
1548 be128_to_u128(&ctrblk, (be128 *)walk->iv);
1549
1550 /* Process two block batch */
1551 if (nbytes >= bsize * 2) {
1552 do {
1553 if (dst != src) {
1554 dst[0] = src[0];
1555 dst[1] = src[1];
1556 }
1557
1558 /* create ctrblks for parallel encrypt */
1559 u128_to_be128(&ctrblocks[0], &ctrblk);
1560 u128_inc(&ctrblk);
1561 u128_to_be128(&ctrblocks[1], &ctrblk);
1562 u128_inc(&ctrblk);
1563
1564 camellia_enc_blk_xor_2way(ctx, (u8 *)dst,
1565 (u8 *)ctrblocks);
1566
1567 src += 2;
1568 dst += 2;
1569 nbytes -= bsize * 2;
1570 } while (nbytes >= bsize * 2);
1571
1572 if (nbytes < bsize)
1573 goto done;
1574 }
1575
1576 /* Handle leftovers */
1577 do {
1578 if (dst != src)
1579 *dst = *src;
1580
1581 u128_to_be128(&ctrblocks[0], &ctrblk);
1582 u128_inc(&ctrblk);
1583
1584 camellia_enc_blk_xor(ctx, (u8 *)dst, (u8 *)ctrblocks);
1585
1586 src += 1;
1587 dst += 1;
1588 nbytes -= bsize;
1589 } while (nbytes >= bsize);
1590
1591done:
1592 u128_to_be128((be128 *)walk->iv, &ctrblk);
1593 return nbytes;
1594} 1427}
1595 1428
1596static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, 1429static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
1597 struct scatterlist *src, unsigned int nbytes) 1430 struct scatterlist *src, unsigned int nbytes)
1598{ 1431{
1599 struct blkcipher_walk walk; 1432 return glue_ctr_crypt_128bit(&camellia_ctr, desc, dst, src, nbytes);
1600 int err;
1601
1602 blkcipher_walk_init(&walk, dst, src, nbytes);
1603 err = blkcipher_walk_virt_block(desc, &walk, CAMELLIA_BLOCK_SIZE);
1604
1605 while ((nbytes = walk.nbytes) >= CAMELLIA_BLOCK_SIZE) {
1606 nbytes = __ctr_crypt(desc, &walk);
1607 err = blkcipher_walk_done(desc, &walk, nbytes);
1608 }
1609
1610 if (walk.nbytes) {
1611 ctr_crypt_final(desc, &walk);
1612 err = blkcipher_walk_done(desc, &walk, 0);
1613 }
1614
1615 return err;
1616} 1433}
1617 1434
1618static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) 1435static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
diff --git a/arch/x86/crypto/glue_helper.c b/arch/x86/crypto/glue_helper.c
new file mode 100644
index 000000000000..4854f0f31e4f
--- /dev/null
+++ b/arch/x86/crypto/glue_helper.c
@@ -0,0 +1,307 @@
1/*
2 * Shared glue code for 128bit block ciphers
3 *
4 * Copyright (c) 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
5 *
6 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
7 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
8 * CTR part based on code (crypto/ctr.c) by:
9 * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or
14 * (at your option) any later version.
15 *
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU General Public License for more details.
20 *
21 * You should have received a copy of the GNU General Public License
22 * along with this program; if not, write to the Free Software
23 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
24 * USA
25 *
26 */
27
28#include <linux/module.h>
29#include <crypto/b128ops.h>
30#include <crypto/lrw.h>
31#include <crypto/xts.h>
32#include <asm/crypto/glue_helper.h>
33#include <crypto/scatterwalk.h>
34
35static int __glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx,
36 struct blkcipher_desc *desc,
37 struct blkcipher_walk *walk)
38{
39 void *ctx = crypto_blkcipher_ctx(desc->tfm);
40 const unsigned int bsize = 128 / 8;
41 unsigned int nbytes, i, func_bytes;
42 bool fpu_enabled = false;
43 int err;
44
45 err = blkcipher_walk_virt(desc, walk);
46
47 while ((nbytes = walk->nbytes)) {
48 u8 *wsrc = walk->src.virt.addr;
49 u8 *wdst = walk->dst.virt.addr;
50
51 fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
52 desc, fpu_enabled, nbytes);
53
54 for (i = 0; i < gctx->num_funcs; i++) {
55 func_bytes = bsize * gctx->funcs[i].num_blocks;
56
57 /* Process multi-block batch */
58 if (nbytes >= func_bytes) {
59 do {
60 gctx->funcs[i].fn_u.ecb(ctx, wdst,
61 wsrc);
62
63 wsrc += func_bytes;
64 wdst += func_bytes;
65 nbytes -= func_bytes;
66 } while (nbytes >= func_bytes);
67
68 if (nbytes < bsize)
69 goto done;
70 }
71 }
72
73done:
74 err = blkcipher_walk_done(desc, walk, nbytes);
75 }
76
77 glue_fpu_end(fpu_enabled);
78 return err;
79}
80
81int glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx,
82 struct blkcipher_desc *desc, struct scatterlist *dst,
83 struct scatterlist *src, unsigned int nbytes)
84{
85 struct blkcipher_walk walk;
86
87 blkcipher_walk_init(&walk, dst, src, nbytes);
88 return __glue_ecb_crypt_128bit(gctx, desc, &walk);
89}
90EXPORT_SYMBOL_GPL(glue_ecb_crypt_128bit);
91
92static unsigned int __glue_cbc_encrypt_128bit(const common_glue_func_t fn,
93 struct blkcipher_desc *desc,
94 struct blkcipher_walk *walk)
95{
96 void *ctx = crypto_blkcipher_ctx(desc->tfm);
97 const unsigned int bsize = 128 / 8;
98 unsigned int nbytes = walk->nbytes;
99 u128 *src = (u128 *)walk->src.virt.addr;
100 u128 *dst = (u128 *)walk->dst.virt.addr;
101 u128 *iv = (u128 *)walk->iv;
102
103 do {
104 u128_xor(dst, src, iv);
105 fn(ctx, (u8 *)dst, (u8 *)dst);
106 iv = dst;
107
108 src += 1;
109 dst += 1;
110 nbytes -= bsize;
111 } while (nbytes >= bsize);
112
113 u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv);
114 return nbytes;
115}
116
117int glue_cbc_encrypt_128bit(const common_glue_func_t fn,
118 struct blkcipher_desc *desc,
119 struct scatterlist *dst,
120 struct scatterlist *src, unsigned int nbytes)
121{
122 struct blkcipher_walk walk;
123 int err;
124
125 blkcipher_walk_init(&walk, dst, src, nbytes);
126 err = blkcipher_walk_virt(desc, &walk);
127
128 while ((nbytes = walk.nbytes)) {
129 nbytes = __glue_cbc_encrypt_128bit(fn, desc, &walk);
130 err = blkcipher_walk_done(desc, &walk, nbytes);
131 }
132
133 return err;
134}
135EXPORT_SYMBOL_GPL(glue_cbc_encrypt_128bit);
136
137static unsigned int
138__glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx,
139 struct blkcipher_desc *desc,
140 struct blkcipher_walk *walk)
141{
142 void *ctx = crypto_blkcipher_ctx(desc->tfm);
143 const unsigned int bsize = 128 / 8;
144 unsigned int nbytes = walk->nbytes;
145 u128 *src = (u128 *)walk->src.virt.addr;
146 u128 *dst = (u128 *)walk->dst.virt.addr;
147 u128 last_iv;
148 unsigned int num_blocks, func_bytes;
149 unsigned int i;
150
151 /* Start of the last block. */
152 src += nbytes / bsize - 1;
153 dst += nbytes / bsize - 1;
154
155 last_iv = *src;
156
157 for (i = 0; i < gctx->num_funcs; i++) {
158 num_blocks = gctx->funcs[i].num_blocks;
159 func_bytes = bsize * num_blocks;
160
161 /* Process multi-block batch */
162 if (nbytes >= func_bytes) {
163 do {
164 nbytes -= func_bytes - bsize;
165 src -= num_blocks - 1;
166 dst -= num_blocks - 1;
167
168 gctx->funcs[i].fn_u.cbc(ctx, dst, src);
169
170 nbytes -= bsize;
171 if (nbytes < bsize)
172 goto done;
173
174 u128_xor(dst, dst, src - 1);
175 src -= 1;
176 dst -= 1;
177 } while (nbytes >= func_bytes);
178
179 if (nbytes < bsize)
180 goto done;
181 }
182 }
183
184done:
185 u128_xor(dst, dst, (u128 *)walk->iv);
186 *(u128 *)walk->iv = last_iv;
187
188 return nbytes;
189}
190
191int glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx,
192 struct blkcipher_desc *desc,
193 struct scatterlist *dst,
194 struct scatterlist *src, unsigned int nbytes)
195{
196 const unsigned int bsize = 128 / 8;
197 bool fpu_enabled = false;
198 struct blkcipher_walk walk;
199 int err;
200
201 blkcipher_walk_init(&walk, dst, src, nbytes);
202 err = blkcipher_walk_virt(desc, &walk);
203
204 while ((nbytes = walk.nbytes)) {
205 fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
206 desc, fpu_enabled, nbytes);
207 nbytes = __glue_cbc_decrypt_128bit(gctx, desc, &walk);
208 err = blkcipher_walk_done(desc, &walk, nbytes);
209 }
210
211 glue_fpu_end(fpu_enabled);
212 return err;
213}
214EXPORT_SYMBOL_GPL(glue_cbc_decrypt_128bit);
215
216static void glue_ctr_crypt_final_128bit(const common_glue_ctr_func_t fn_ctr,
217 struct blkcipher_desc *desc,
218 struct blkcipher_walk *walk)
219{
220 void *ctx = crypto_blkcipher_ctx(desc->tfm);
221 u8 *src = (u8 *)walk->src.virt.addr;
222 u8 *dst = (u8 *)walk->dst.virt.addr;
223 unsigned int nbytes = walk->nbytes;
224 u128 ctrblk;
225 u128 tmp;
226
227 be128_to_u128(&ctrblk, (be128 *)walk->iv);
228
229 memcpy(&tmp, src, nbytes);
230 fn_ctr(ctx, &tmp, &tmp, &ctrblk);
231 memcpy(dst, &tmp, nbytes);
232
233 u128_to_be128((be128 *)walk->iv, &ctrblk);
234}
235EXPORT_SYMBOL_GPL(glue_ctr_crypt_final_128bit);
236
237static unsigned int __glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx,
238 struct blkcipher_desc *desc,
239 struct blkcipher_walk *walk)
240{
241 const unsigned int bsize = 128 / 8;
242 void *ctx = crypto_blkcipher_ctx(desc->tfm);
243 unsigned int nbytes = walk->nbytes;
244 u128 *src = (u128 *)walk->src.virt.addr;
245 u128 *dst = (u128 *)walk->dst.virt.addr;
246 u128 ctrblk;
247 unsigned int num_blocks, func_bytes;
248 unsigned int i;
249
250 be128_to_u128(&ctrblk, (be128 *)walk->iv);
251
252 /* Process multi-block batch */
253 for (i = 0; i < gctx->num_funcs; i++) {
254 num_blocks = gctx->funcs[i].num_blocks;
255 func_bytes = bsize * num_blocks;
256
257 if (nbytes >= func_bytes) {
258 do {
259 gctx->funcs[i].fn_u.ctr(ctx, dst, src, &ctrblk);
260
261 src += num_blocks;
262 dst += num_blocks;
263 nbytes -= func_bytes;
264 } while (nbytes >= func_bytes);
265
266 if (nbytes < bsize)
267 goto done;
268 }
269 }
270
271done:
272 u128_to_be128((be128 *)walk->iv, &ctrblk);
273 return nbytes;
274}
275
276int glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx,
277 struct blkcipher_desc *desc, struct scatterlist *dst,
278 struct scatterlist *src, unsigned int nbytes)
279{
280 const unsigned int bsize = 128 / 8;
281 bool fpu_enabled = false;
282 struct blkcipher_walk walk;
283 int err;
284
285 blkcipher_walk_init(&walk, dst, src, nbytes);
286 err = blkcipher_walk_virt_block(desc, &walk, bsize);
287
288 while ((nbytes = walk.nbytes) >= bsize) {
289 fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
290 desc, fpu_enabled, nbytes);
291 nbytes = __glue_ctr_crypt_128bit(gctx, desc, &walk);
292 err = blkcipher_walk_done(desc, &walk, nbytes);
293 }
294
295 glue_fpu_end(fpu_enabled);
296
297 if (walk.nbytes) {
298 glue_ctr_crypt_final_128bit(
299 gctx->funcs[gctx->num_funcs - 1].fn_u.ctr, desc, &walk);
300 err = blkcipher_walk_done(desc, &walk, 0);
301 }
302
303 return err;
304}
305EXPORT_SYMBOL_GPL(glue_ctr_crypt_128bit);
306
307MODULE_LICENSE("GPL");
diff --git a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
new file mode 100644
index 000000000000..504106bf04a2
--- /dev/null
+++ b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
@@ -0,0 +1,704 @@
1/*
2 * Serpent Cipher 8-way parallel algorithm (x86_64/AVX)
3 *
4 * Copyright (C) 2012 Johannes Goetzfried
5 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
6 *
7 * Based on arch/x86/crypto/serpent-sse2-x86_64-asm_64.S by
8 * Copyright (C) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
14 *
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
19 *
20 * You should have received a copy of the GNU General Public License
21 * along with this program; if not, write to the Free Software
22 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
23 * USA
24 *
25 */
26
27.file "serpent-avx-x86_64-asm_64.S"
28.text
29
30#define CTX %rdi
31
32/**********************************************************************
33 8-way AVX serpent
34 **********************************************************************/
35#define RA1 %xmm0
36#define RB1 %xmm1
37#define RC1 %xmm2
38#define RD1 %xmm3
39#define RE1 %xmm4
40
41#define tp %xmm5
42
43#define RA2 %xmm6
44#define RB2 %xmm7
45#define RC2 %xmm8
46#define RD2 %xmm9
47#define RE2 %xmm10
48
49#define RNOT %xmm11
50
51#define RK0 %xmm12
52#define RK1 %xmm13
53#define RK2 %xmm14
54#define RK3 %xmm15
55
56
57#define S0_1(x0, x1, x2, x3, x4) \
58 vpor x0, x3, tp; \
59 vpxor x3, x0, x0; \
60 vpxor x2, x3, x4; \
61 vpxor RNOT, x4, x4; \
62 vpxor x1, tp, x3; \
63 vpand x0, x1, x1; \
64 vpxor x4, x1, x1; \
65 vpxor x0, x2, x2;
66#define S0_2(x0, x1, x2, x3, x4) \
67 vpxor x3, x0, x0; \
68 vpor x0, x4, x4; \
69 vpxor x2, x0, x0; \
70 vpand x1, x2, x2; \
71 vpxor x2, x3, x3; \
72 vpxor RNOT, x1, x1; \
73 vpxor x4, x2, x2; \
74 vpxor x2, x1, x1;
75
76#define S1_1(x0, x1, x2, x3, x4) \
77 vpxor x0, x1, tp; \
78 vpxor x3, x0, x0; \
79 vpxor RNOT, x3, x3; \
80 vpand tp, x1, x4; \
81 vpor tp, x0, x0; \
82 vpxor x2, x3, x3; \
83 vpxor x3, x0, x0; \
84 vpxor x3, tp, x1;
85#define S1_2(x0, x1, x2, x3, x4) \
86 vpxor x4, x3, x3; \
87 vpor x4, x1, x1; \
88 vpxor x2, x4, x4; \
89 vpand x0, x2, x2; \
90 vpxor x1, x2, x2; \
91 vpor x0, x1, x1; \
92 vpxor RNOT, x0, x0; \
93 vpxor x2, x0, x0; \
94 vpxor x1, x4, x4;
95
96#define S2_1(x0, x1, x2, x3, x4) \
97 vpxor RNOT, x3, x3; \
98 vpxor x0, x1, x1; \
99 vpand x2, x0, tp; \
100 vpxor x3, tp, tp; \
101 vpor x0, x3, x3; \
102 vpxor x1, x2, x2; \
103 vpxor x1, x3, x3; \
104 vpand tp, x1, x1;
105#define S2_2(x0, x1, x2, x3, x4) \
106 vpxor x2, tp, tp; \
107 vpand x3, x2, x2; \
108 vpor x1, x3, x3; \
109 vpxor RNOT, tp, tp; \
110 vpxor tp, x3, x3; \
111 vpxor tp, x0, x4; \
112 vpxor x2, tp, x0; \
113 vpor x2, x1, x1;
114
115#define S3_1(x0, x1, x2, x3, x4) \
116 vpxor x3, x1, tp; \
117 vpor x0, x3, x3; \
118 vpand x0, x1, x4; \
119 vpxor x2, x0, x0; \
120 vpxor tp, x2, x2; \
121 vpand x3, tp, x1; \
122 vpxor x3, x2, x2; \
123 vpor x4, x0, x0; \
124 vpxor x3, x4, x4;
125#define S3_2(x0, x1, x2, x3, x4) \
126 vpxor x0, x1, x1; \
127 vpand x3, x0, x0; \
128 vpand x4, x3, x3; \
129 vpxor x2, x3, x3; \
130 vpor x1, x4, x4; \
131 vpand x1, x2, x2; \
132 vpxor x3, x4, x4; \
133 vpxor x3, x0, x0; \
134 vpxor x2, x3, x3;
135
136#define S4_1(x0, x1, x2, x3, x4) \
137 vpand x0, x3, tp; \
138 vpxor x3, x0, x0; \
139 vpxor x2, tp, tp; \
140 vpor x3, x2, x2; \
141 vpxor x1, x0, x0; \
142 vpxor tp, x3, x4; \
143 vpor x0, x2, x2; \
144 vpxor x1, x2, x2;
145#define S4_2(x0, x1, x2, x3, x4) \
146 vpand x0, x1, x1; \
147 vpxor x4, x1, x1; \
148 vpand x2, x4, x4; \
149 vpxor tp, x2, x2; \
150 vpxor x0, x4, x4; \
151 vpor x1, tp, x3; \
152 vpxor RNOT, x1, x1; \
153 vpxor x0, x3, x3;
154
155#define S5_1(x0, x1, x2, x3, x4) \
156 vpor x0, x1, tp; \
157 vpxor tp, x2, x2; \
158 vpxor RNOT, x3, x3; \
159 vpxor x0, x1, x4; \
160 vpxor x2, x0, x0; \
161 vpand x4, tp, x1; \
162 vpor x3, x4, x4; \
163 vpxor x0, x4, x4;
164#define S5_2(x0, x1, x2, x3, x4) \
165 vpand x3, x0, x0; \
166 vpxor x3, x1, x1; \
167 vpxor x2, x3, x3; \
168 vpxor x1, x0, x0; \
169 vpand x4, x2, x2; \
170 vpxor x2, x1, x1; \
171 vpand x0, x2, x2; \
172 vpxor x2, x3, x3;
173
174#define S6_1(x0, x1, x2, x3, x4) \
175 vpxor x0, x3, x3; \
176 vpxor x2, x1, tp; \
177 vpxor x0, x2, x2; \
178 vpand x3, x0, x0; \
179 vpor x3, tp, tp; \
180 vpxor RNOT, x1, x4; \
181 vpxor tp, x0, x0; \
182 vpxor x2, tp, x1;
183#define S6_2(x0, x1, x2, x3, x4) \
184 vpxor x4, x3, x3; \
185 vpxor x0, x4, x4; \
186 vpand x0, x2, x2; \
187 vpxor x1, x4, x4; \
188 vpxor x3, x2, x2; \
189 vpand x1, x3, x3; \
190 vpxor x0, x3, x3; \
191 vpxor x2, x1, x1;
192
193#define S7_1(x0, x1, x2, x3, x4) \
194 vpxor RNOT, x1, tp; \
195 vpxor RNOT, x0, x0; \
196 vpand x2, tp, x1; \
197 vpxor x3, x1, x1; \
198 vpor tp, x3, x3; \
199 vpxor x2, tp, x4; \
200 vpxor x3, x2, x2; \
201 vpxor x0, x3, x3; \
202 vpor x1, x0, x0;
203#define S7_2(x0, x1, x2, x3, x4) \
204 vpand x0, x2, x2; \
205 vpxor x4, x0, x0; \
206 vpxor x3, x4, x4; \
207 vpand x0, x3, x3; \
208 vpxor x1, x4, x4; \
209 vpxor x4, x2, x2; \
210 vpxor x1, x3, x3; \
211 vpor x0, x4, x4; \
212 vpxor x1, x4, x4;
213
214#define SI0_1(x0, x1, x2, x3, x4) \
215 vpxor x0, x1, x1; \
216 vpor x1, x3, tp; \
217 vpxor x1, x3, x4; \
218 vpxor RNOT, x0, x0; \
219 vpxor tp, x2, x2; \
220 vpxor x0, tp, x3; \
221 vpand x1, x0, x0; \
222 vpxor x2, x0, x0;
223#define SI0_2(x0, x1, x2, x3, x4) \
224 vpand x3, x2, x2; \
225 vpxor x4, x3, x3; \
226 vpxor x3, x2, x2; \
227 vpxor x3, x1, x1; \
228 vpand x0, x3, x3; \
229 vpxor x0, x1, x1; \
230 vpxor x2, x0, x0; \
231 vpxor x3, x4, x4;
232
233#define SI1_1(x0, x1, x2, x3, x4) \
234 vpxor x3, x1, x1; \
235 vpxor x2, x0, tp; \
236 vpxor RNOT, x2, x2; \
237 vpor x1, x0, x4; \
238 vpxor x3, x4, x4; \
239 vpand x1, x3, x3; \
240 vpxor x2, x1, x1; \
241 vpand x4, x2, x2;
242#define SI1_2(x0, x1, x2, x3, x4) \
243 vpxor x1, x4, x4; \
244 vpor x3, x1, x1; \
245 vpxor tp, x3, x3; \
246 vpxor tp, x2, x2; \
247 vpor x4, tp, x0; \
248 vpxor x4, x2, x2; \
249 vpxor x0, x1, x1; \
250 vpxor x1, x4, x4;
251
252#define SI2_1(x0, x1, x2, x3, x4) \
253 vpxor x1, x2, x2; \
254 vpxor RNOT, x3, tp; \
255 vpor x2, tp, tp; \
256 vpxor x3, x2, x2; \
257 vpxor x0, x3, x4; \
258 vpxor x1, tp, x3; \
259 vpor x2, x1, x1; \
260 vpxor x0, x2, x2;
261#define SI2_2(x0, x1, x2, x3, x4) \
262 vpxor x4, x1, x1; \
263 vpor x3, x4, x4; \
264 vpxor x3, x2, x2; \
265 vpxor x2, x4, x4; \
266 vpand x1, x2, x2; \
267 vpxor x3, x2, x2; \
268 vpxor x4, x3, x3; \
269 vpxor x0, x4, x4;
270
271#define SI3_1(x0, x1, x2, x3, x4) \
272 vpxor x1, x2, x2; \
273 vpand x2, x1, tp; \
274 vpxor x0, tp, tp; \
275 vpor x1, x0, x0; \
276 vpxor x3, x1, x4; \
277 vpxor x3, x0, x0; \
278 vpor tp, x3, x3; \
279 vpxor x2, tp, x1;
280#define SI3_2(x0, x1, x2, x3, x4) \
281 vpxor x3, x1, x1; \
282 vpxor x2, x0, x0; \
283 vpxor x3, x2, x2; \
284 vpand x1, x3, x3; \
285 vpxor x0, x1, x1; \
286 vpand x2, x0, x0; \
287 vpxor x3, x4, x4; \
288 vpxor x0, x3, x3; \
289 vpxor x1, x0, x0;
290
291#define SI4_1(x0, x1, x2, x3, x4) \
292 vpxor x3, x2, x2; \
293 vpand x1, x0, tp; \
294 vpxor x2, tp, tp; \
295 vpor x3, x2, x2; \
296 vpxor RNOT, x0, x4; \
297 vpxor tp, x1, x1; \
298 vpxor x2, tp, x0; \
299 vpand x4, x2, x2;
300#define SI4_2(x0, x1, x2, x3, x4) \
301 vpxor x0, x2, x2; \
302 vpor x4, x0, x0; \
303 vpxor x3, x0, x0; \
304 vpand x2, x3, x3; \
305 vpxor x3, x4, x4; \
306 vpxor x1, x3, x3; \
307 vpand x0, x1, x1; \
308 vpxor x1, x4, x4; \
309 vpxor x3, x0, x0;
310
311#define SI5_1(x0, x1, x2, x3, x4) \
312 vpor x2, x1, tp; \
313 vpxor x1, x2, x2; \
314 vpxor x3, tp, tp; \
315 vpand x1, x3, x3; \
316 vpxor x3, x2, x2; \
317 vpor x0, x3, x3; \
318 vpxor RNOT, x0, x0; \
319 vpxor x2, x3, x3; \
320 vpor x0, x2, x2;
321#define SI5_2(x0, x1, x2, x3, x4) \
322 vpxor tp, x1, x4; \
323 vpxor x4, x2, x2; \
324 vpand x0, x4, x4; \
325 vpxor tp, x0, x0; \
326 vpxor x3, tp, x1; \
327 vpand x2, x0, x0; \
328 vpxor x3, x2, x2; \
329 vpxor x2, x0, x0; \
330 vpxor x4, x2, x2; \
331 vpxor x3, x4, x4;
332
333#define SI6_1(x0, x1, x2, x3, x4) \
334 vpxor x2, x0, x0; \
335 vpand x3, x0, tp; \
336 vpxor x3, x2, x2; \
337 vpxor x2, tp, tp; \
338 vpxor x1, x3, x3; \
339 vpor x0, x2, x2; \
340 vpxor x3, x2, x2; \
341 vpand tp, x3, x3;
342#define SI6_2(x0, x1, x2, x3, x4) \
343 vpxor RNOT, tp, tp; \
344 vpxor x1, x3, x3; \
345 vpand x2, x1, x1; \
346 vpxor tp, x0, x4; \
347 vpxor x4, x3, x3; \
348 vpxor x2, x4, x4; \
349 vpxor x1, tp, x0; \
350 vpxor x0, x2, x2;
351
352#define SI7_1(x0, x1, x2, x3, x4) \
353 vpand x0, x3, tp; \
354 vpxor x2, x0, x0; \
355 vpor x3, x2, x2; \
356 vpxor x1, x3, x4; \
357 vpxor RNOT, x0, x0; \
358 vpor tp, x1, x1; \
359 vpxor x0, x4, x4; \
360 vpand x2, x0, x0; \
361 vpxor x1, x0, x0;
362#define SI7_2(x0, x1, x2, x3, x4) \
363 vpand x2, x1, x1; \
364 vpxor x2, tp, x3; \
365 vpxor x3, x4, x4; \
366 vpand x3, x2, x2; \
367 vpor x0, x3, x3; \
368 vpxor x4, x1, x1; \
369 vpxor x4, x3, x3; \
370 vpand x0, x4, x4; \
371 vpxor x2, x4, x4;
372
373#define get_key(i, j, t) \
374 vbroadcastss (4*(i)+(j))*4(CTX), t;
375
376#define K2(x0, x1, x2, x3, x4, i) \
377 get_key(i, 0, RK0); \
378 get_key(i, 1, RK1); \
379 get_key(i, 2, RK2); \
380 get_key(i, 3, RK3); \
381 vpxor RK0, x0 ## 1, x0 ## 1; \
382 vpxor RK1, x1 ## 1, x1 ## 1; \
383 vpxor RK2, x2 ## 1, x2 ## 1; \
384 vpxor RK3, x3 ## 1, x3 ## 1; \
385 vpxor RK0, x0 ## 2, x0 ## 2; \
386 vpxor RK1, x1 ## 2, x1 ## 2; \
387 vpxor RK2, x2 ## 2, x2 ## 2; \
388 vpxor RK3, x3 ## 2, x3 ## 2;
389
390#define LK2(x0, x1, x2, x3, x4, i) \
391 vpslld $13, x0 ## 1, x4 ## 1; \
392 vpsrld $(32 - 13), x0 ## 1, x0 ## 1; \
393 vpor x4 ## 1, x0 ## 1, x0 ## 1; \
394 vpxor x0 ## 1, x1 ## 1, x1 ## 1; \
395 vpslld $3, x2 ## 1, x4 ## 1; \
396 vpsrld $(32 - 3), x2 ## 1, x2 ## 1; \
397 vpor x4 ## 1, x2 ## 1, x2 ## 1; \
398 vpxor x2 ## 1, x1 ## 1, x1 ## 1; \
399 vpslld $13, x0 ## 2, x4 ## 2; \
400 vpsrld $(32 - 13), x0 ## 2, x0 ## 2; \
401 vpor x4 ## 2, x0 ## 2, x0 ## 2; \
402 vpxor x0 ## 2, x1 ## 2, x1 ## 2; \
403 vpslld $3, x2 ## 2, x4 ## 2; \
404 vpsrld $(32 - 3), x2 ## 2, x2 ## 2; \
405 vpor x4 ## 2, x2 ## 2, x2 ## 2; \
406 vpxor x2 ## 2, x1 ## 2, x1 ## 2; \
407 vpslld $1, x1 ## 1, x4 ## 1; \
408 vpsrld $(32 - 1), x1 ## 1, x1 ## 1; \
409 vpor x4 ## 1, x1 ## 1, x1 ## 1; \
410 vpslld $3, x0 ## 1, x4 ## 1; \
411 vpxor x2 ## 1, x3 ## 1, x3 ## 1; \
412 vpxor x4 ## 1, x3 ## 1, x3 ## 1; \
413 get_key(i, 1, RK1); \
414 vpslld $1, x1 ## 2, x4 ## 2; \
415 vpsrld $(32 - 1), x1 ## 2, x1 ## 2; \
416 vpor x4 ## 2, x1 ## 2, x1 ## 2; \
417 vpslld $3, x0 ## 2, x4 ## 2; \
418 vpxor x2 ## 2, x3 ## 2, x3 ## 2; \
419 vpxor x4 ## 2, x3 ## 2, x3 ## 2; \
420 get_key(i, 3, RK3); \
421 vpslld $7, x3 ## 1, x4 ## 1; \
422 vpsrld $(32 - 7), x3 ## 1, x3 ## 1; \
423 vpor x4 ## 1, x3 ## 1, x3 ## 1; \
424 vpslld $7, x1 ## 1, x4 ## 1; \
425 vpxor x1 ## 1, x0 ## 1, x0 ## 1; \
426 vpxor x3 ## 1, x0 ## 1, x0 ## 1; \
427 vpxor x3 ## 1, x2 ## 1, x2 ## 1; \
428 vpxor x4 ## 1, x2 ## 1, x2 ## 1; \
429 get_key(i, 0, RK0); \
430 vpslld $7, x3 ## 2, x4 ## 2; \
431 vpsrld $(32 - 7), x3 ## 2, x3 ## 2; \
432 vpor x4 ## 2, x3 ## 2, x3 ## 2; \
433 vpslld $7, x1 ## 2, x4 ## 2; \
434 vpxor x1 ## 2, x0 ## 2, x0 ## 2; \
435 vpxor x3 ## 2, x0 ## 2, x0 ## 2; \
436 vpxor x3 ## 2, x2 ## 2, x2 ## 2; \
437 vpxor x4 ## 2, x2 ## 2, x2 ## 2; \
438 get_key(i, 2, RK2); \
439 vpxor RK1, x1 ## 1, x1 ## 1; \
440 vpxor RK3, x3 ## 1, x3 ## 1; \
441 vpslld $5, x0 ## 1, x4 ## 1; \
442 vpsrld $(32 - 5), x0 ## 1, x0 ## 1; \
443 vpor x4 ## 1, x0 ## 1, x0 ## 1; \
444 vpslld $22, x2 ## 1, x4 ## 1; \
445 vpsrld $(32 - 22), x2 ## 1, x2 ## 1; \
446 vpor x4 ## 1, x2 ## 1, x2 ## 1; \
447 vpxor RK0, x0 ## 1, x0 ## 1; \
448 vpxor RK2, x2 ## 1, x2 ## 1; \
449 vpxor RK1, x1 ## 2, x1 ## 2; \
450 vpxor RK3, x3 ## 2, x3 ## 2; \
451 vpslld $5, x0 ## 2, x4 ## 2; \
452 vpsrld $(32 - 5), x0 ## 2, x0 ## 2; \
453 vpor x4 ## 2, x0 ## 2, x0 ## 2; \
454 vpslld $22, x2 ## 2, x4 ## 2; \
455 vpsrld $(32 - 22), x2 ## 2, x2 ## 2; \
456 vpor x4 ## 2, x2 ## 2, x2 ## 2; \
457 vpxor RK0, x0 ## 2, x0 ## 2; \
458 vpxor RK2, x2 ## 2, x2 ## 2;
459
460#define KL2(x0, x1, x2, x3, x4, i) \
461 vpxor RK0, x0 ## 1, x0 ## 1; \
462 vpxor RK2, x2 ## 1, x2 ## 1; \
463 vpsrld $5, x0 ## 1, x4 ## 1; \
464 vpslld $(32 - 5), x0 ## 1, x0 ## 1; \
465 vpor x4 ## 1, x0 ## 1, x0 ## 1; \
466 vpxor RK3, x3 ## 1, x3 ## 1; \
467 vpxor RK1, x1 ## 1, x1 ## 1; \
468 vpsrld $22, x2 ## 1, x4 ## 1; \
469 vpslld $(32 - 22), x2 ## 1, x2 ## 1; \
470 vpor x4 ## 1, x2 ## 1, x2 ## 1; \
471 vpxor x3 ## 1, x2 ## 1, x2 ## 1; \
472 vpxor RK0, x0 ## 2, x0 ## 2; \
473 vpxor RK2, x2 ## 2, x2 ## 2; \
474 vpsrld $5, x0 ## 2, x4 ## 2; \
475 vpslld $(32 - 5), x0 ## 2, x0 ## 2; \
476 vpor x4 ## 2, x0 ## 2, x0 ## 2; \
477 vpxor RK3, x3 ## 2, x3 ## 2; \
478 vpxor RK1, x1 ## 2, x1 ## 2; \
479 vpsrld $22, x2 ## 2, x4 ## 2; \
480 vpslld $(32 - 22), x2 ## 2, x2 ## 2; \
481 vpor x4 ## 2, x2 ## 2, x2 ## 2; \
482 vpxor x3 ## 2, x2 ## 2, x2 ## 2; \
483 vpxor x3 ## 1, x0 ## 1, x0 ## 1; \
484 vpslld $7, x1 ## 1, x4 ## 1; \
485 vpxor x1 ## 1, x0 ## 1, x0 ## 1; \
486 vpxor x4 ## 1, x2 ## 1, x2 ## 1; \
487 vpsrld $1, x1 ## 1, x4 ## 1; \
488 vpslld $(32 - 1), x1 ## 1, x1 ## 1; \
489 vpor x4 ## 1, x1 ## 1, x1 ## 1; \
490 vpxor x3 ## 2, x0 ## 2, x0 ## 2; \
491 vpslld $7, x1 ## 2, x4 ## 2; \
492 vpxor x1 ## 2, x0 ## 2, x0 ## 2; \
493 vpxor x4 ## 2, x2 ## 2, x2 ## 2; \
494 vpsrld $1, x1 ## 2, x4 ## 2; \
495 vpslld $(32 - 1), x1 ## 2, x1 ## 2; \
496 vpor x4 ## 2, x1 ## 2, x1 ## 2; \
497 vpsrld $7, x3 ## 1, x4 ## 1; \
498 vpslld $(32 - 7), x3 ## 1, x3 ## 1; \
499 vpor x4 ## 1, x3 ## 1, x3 ## 1; \
500 vpxor x0 ## 1, x1 ## 1, x1 ## 1; \
501 vpslld $3, x0 ## 1, x4 ## 1; \
502 vpxor x4 ## 1, x3 ## 1, x3 ## 1; \
503 vpsrld $7, x3 ## 2, x4 ## 2; \
504 vpslld $(32 - 7), x3 ## 2, x3 ## 2; \
505 vpor x4 ## 2, x3 ## 2, x3 ## 2; \
506 vpxor x0 ## 2, x1 ## 2, x1 ## 2; \
507 vpslld $3, x0 ## 2, x4 ## 2; \
508 vpxor x4 ## 2, x3 ## 2, x3 ## 2; \
509 vpsrld $13, x0 ## 1, x4 ## 1; \
510 vpslld $(32 - 13), x0 ## 1, x0 ## 1; \
511 vpor x4 ## 1, x0 ## 1, x0 ## 1; \
512 vpxor x2 ## 1, x1 ## 1, x1 ## 1; \
513 vpxor x2 ## 1, x3 ## 1, x3 ## 1; \
514 vpsrld $3, x2 ## 1, x4 ## 1; \
515 vpslld $(32 - 3), x2 ## 1, x2 ## 1; \
516 vpor x4 ## 1, x2 ## 1, x2 ## 1; \
517 vpsrld $13, x0 ## 2, x4 ## 2; \
518 vpslld $(32 - 13), x0 ## 2, x0 ## 2; \
519 vpor x4 ## 2, x0 ## 2, x0 ## 2; \
520 vpxor x2 ## 2, x1 ## 2, x1 ## 2; \
521 vpxor x2 ## 2, x3 ## 2, x3 ## 2; \
522 vpsrld $3, x2 ## 2, x4 ## 2; \
523 vpslld $(32 - 3), x2 ## 2, x2 ## 2; \
524 vpor x4 ## 2, x2 ## 2, x2 ## 2;
525
526#define S(SBOX, x0, x1, x2, x3, x4) \
527 SBOX ## _1(x0 ## 1, x1 ## 1, x2 ## 1, x3 ## 1, x4 ## 1); \
528 SBOX ## _2(x0 ## 1, x1 ## 1, x2 ## 1, x3 ## 1, x4 ## 1); \
529 SBOX ## _1(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2); \
530 SBOX ## _2(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2);
531
532#define SP(SBOX, x0, x1, x2, x3, x4, i) \
533 get_key(i, 0, RK0); \
534 SBOX ## _1(x0 ## 1, x1 ## 1, x2 ## 1, x3 ## 1, x4 ## 1); \
535 get_key(i, 2, RK2); \
536 SBOX ## _2(x0 ## 1, x1 ## 1, x2 ## 1, x3 ## 1, x4 ## 1); \
537 get_key(i, 3, RK3); \
538 SBOX ## _1(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2); \
539 get_key(i, 1, RK1); \
540 SBOX ## _2(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2); \
541
542#define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
543 vpunpckldq x1, x0, t0; \
544 vpunpckhdq x1, x0, t2; \
545 vpunpckldq x3, x2, t1; \
546 vpunpckhdq x3, x2, x3; \
547 \
548 vpunpcklqdq t1, t0, x0; \
549 vpunpckhqdq t1, t0, x1; \
550 vpunpcklqdq x3, t2, x2; \
551 vpunpckhqdq x3, t2, x3;
552
553#define read_blocks(in, x0, x1, x2, x3, t0, t1, t2) \
554 vmovdqu (0*4*4)(in), x0; \
555 vmovdqu (1*4*4)(in), x1; \
556 vmovdqu (2*4*4)(in), x2; \
557 vmovdqu (3*4*4)(in), x3; \
558 \
559 transpose_4x4(x0, x1, x2, x3, t0, t1, t2)
560
561#define write_blocks(out, x0, x1, x2, x3, t0, t1, t2) \
562 transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
563 \
564 vmovdqu x0, (0*4*4)(out); \
565 vmovdqu x1, (1*4*4)(out); \
566 vmovdqu x2, (2*4*4)(out); \
567 vmovdqu x3, (3*4*4)(out);
568
569#define xor_blocks(out, x0, x1, x2, x3, t0, t1, t2) \
570 transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
571 \
572 vpxor (0*4*4)(out), x0, x0; \
573 vmovdqu x0, (0*4*4)(out); \
574 vpxor (1*4*4)(out), x1, x1; \
575 vmovdqu x1, (1*4*4)(out); \
576 vpxor (2*4*4)(out), x2, x2; \
577 vmovdqu x2, (2*4*4)(out); \
578 vpxor (3*4*4)(out), x3, x3; \
579 vmovdqu x3, (3*4*4)(out);
580
581.align 8
582.global __serpent_enc_blk_8way_avx
583.type __serpent_enc_blk_8way_avx,@function;
584
585__serpent_enc_blk_8way_avx:
586 /* input:
587 * %rdi: ctx, CTX
588 * %rsi: dst
589 * %rdx: src
590 * %rcx: bool, if true: xor output
591 */
592
593 vpcmpeqd RNOT, RNOT, RNOT;
594
595 leaq (4*4*4)(%rdx), %rax;
596 read_blocks(%rdx, RA1, RB1, RC1, RD1, RK0, RK1, RK2);
597 read_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2);
598
599 K2(RA, RB, RC, RD, RE, 0);
600 S(S0, RA, RB, RC, RD, RE); LK2(RC, RB, RD, RA, RE, 1);
601 S(S1, RC, RB, RD, RA, RE); LK2(RE, RD, RA, RC, RB, 2);
602 S(S2, RE, RD, RA, RC, RB); LK2(RB, RD, RE, RC, RA, 3);
603 S(S3, RB, RD, RE, RC, RA); LK2(RC, RA, RD, RB, RE, 4);
604 S(S4, RC, RA, RD, RB, RE); LK2(RA, RD, RB, RE, RC, 5);
605 S(S5, RA, RD, RB, RE, RC); LK2(RC, RA, RD, RE, RB, 6);
606 S(S6, RC, RA, RD, RE, RB); LK2(RD, RB, RA, RE, RC, 7);
607 S(S7, RD, RB, RA, RE, RC); LK2(RC, RA, RE, RD, RB, 8);
608 S(S0, RC, RA, RE, RD, RB); LK2(RE, RA, RD, RC, RB, 9);
609 S(S1, RE, RA, RD, RC, RB); LK2(RB, RD, RC, RE, RA, 10);
610 S(S2, RB, RD, RC, RE, RA); LK2(RA, RD, RB, RE, RC, 11);
611 S(S3, RA, RD, RB, RE, RC); LK2(RE, RC, RD, RA, RB, 12);
612 S(S4, RE, RC, RD, RA, RB); LK2(RC, RD, RA, RB, RE, 13);
613 S(S5, RC, RD, RA, RB, RE); LK2(RE, RC, RD, RB, RA, 14);
614 S(S6, RE, RC, RD, RB, RA); LK2(RD, RA, RC, RB, RE, 15);
615 S(S7, RD, RA, RC, RB, RE); LK2(RE, RC, RB, RD, RA, 16);
616 S(S0, RE, RC, RB, RD, RA); LK2(RB, RC, RD, RE, RA, 17);
617 S(S1, RB, RC, RD, RE, RA); LK2(RA, RD, RE, RB, RC, 18);
618 S(S2, RA, RD, RE, RB, RC); LK2(RC, RD, RA, RB, RE, 19);
619 S(S3, RC, RD, RA, RB, RE); LK2(RB, RE, RD, RC, RA, 20);
620 S(S4, RB, RE, RD, RC, RA); LK2(RE, RD, RC, RA, RB, 21);
621 S(S5, RE, RD, RC, RA, RB); LK2(RB, RE, RD, RA, RC, 22);
622 S(S6, RB, RE, RD, RA, RC); LK2(RD, RC, RE, RA, RB, 23);
623 S(S7, RD, RC, RE, RA, RB); LK2(RB, RE, RA, RD, RC, 24);
624 S(S0, RB, RE, RA, RD, RC); LK2(RA, RE, RD, RB, RC, 25);
625 S(S1, RA, RE, RD, RB, RC); LK2(RC, RD, RB, RA, RE, 26);
626 S(S2, RC, RD, RB, RA, RE); LK2(RE, RD, RC, RA, RB, 27);
627 S(S3, RE, RD, RC, RA, RB); LK2(RA, RB, RD, RE, RC, 28);
628 S(S4, RA, RB, RD, RE, RC); LK2(RB, RD, RE, RC, RA, 29);
629 S(S5, RB, RD, RE, RC, RA); LK2(RA, RB, RD, RC, RE, 30);
630 S(S6, RA, RB, RD, RC, RE); LK2(RD, RE, RB, RC, RA, 31);
631 S(S7, RD, RE, RB, RC, RA); K2(RA, RB, RC, RD, RE, 32);
632
633 leaq (4*4*4)(%rsi), %rax;
634
635 testb %cl, %cl;
636 jnz __enc_xor8;
637
638 write_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2);
639 write_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2);
640
641 ret;
642
643__enc_xor8:
644 xor_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2);
645 xor_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2);
646
647 ret;
648
649.align 8
650.global serpent_dec_blk_8way_avx
651.type serpent_dec_blk_8way_avx,@function;
652
653serpent_dec_blk_8way_avx:
654 /* input:
655 * %rdi: ctx, CTX
656 * %rsi: dst
657 * %rdx: src
658 */
659
660 vpcmpeqd RNOT, RNOT, RNOT;
661
662 leaq (4*4*4)(%rdx), %rax;
663 read_blocks(%rdx, RA1, RB1, RC1, RD1, RK0, RK1, RK2);
664 read_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2);
665
666 K2(RA, RB, RC, RD, RE, 32);
667 SP(SI7, RA, RB, RC, RD, RE, 31); KL2(RB, RD, RA, RE, RC, 31);
668 SP(SI6, RB, RD, RA, RE, RC, 30); KL2(RA, RC, RE, RB, RD, 30);
669 SP(SI5, RA, RC, RE, RB, RD, 29); KL2(RC, RD, RA, RE, RB, 29);
670 SP(SI4, RC, RD, RA, RE, RB, 28); KL2(RC, RA, RB, RE, RD, 28);
671 SP(SI3, RC, RA, RB, RE, RD, 27); KL2(RB, RC, RD, RE, RA, 27);
672 SP(SI2, RB, RC, RD, RE, RA, 26); KL2(RC, RA, RE, RD, RB, 26);
673 SP(SI1, RC, RA, RE, RD, RB, 25); KL2(RB, RA, RE, RD, RC, 25);
674 SP(SI0, RB, RA, RE, RD, RC, 24); KL2(RE, RC, RA, RB, RD, 24);
675 SP(SI7, RE, RC, RA, RB, RD, 23); KL2(RC, RB, RE, RD, RA, 23);
676 SP(SI6, RC, RB, RE, RD, RA, 22); KL2(RE, RA, RD, RC, RB, 22);
677 SP(SI5, RE, RA, RD, RC, RB, 21); KL2(RA, RB, RE, RD, RC, 21);
678 SP(SI4, RA, RB, RE, RD, RC, 20); KL2(RA, RE, RC, RD, RB, 20);
679 SP(SI3, RA, RE, RC, RD, RB, 19); KL2(RC, RA, RB, RD, RE, 19);
680 SP(SI2, RC, RA, RB, RD, RE, 18); KL2(RA, RE, RD, RB, RC, 18);
681 SP(SI1, RA, RE, RD, RB, RC, 17); KL2(RC, RE, RD, RB, RA, 17);
682 SP(SI0, RC, RE, RD, RB, RA, 16); KL2(RD, RA, RE, RC, RB, 16);
683 SP(SI7, RD, RA, RE, RC, RB, 15); KL2(RA, RC, RD, RB, RE, 15);
684 SP(SI6, RA, RC, RD, RB, RE, 14); KL2(RD, RE, RB, RA, RC, 14);
685 SP(SI5, RD, RE, RB, RA, RC, 13); KL2(RE, RC, RD, RB, RA, 13);
686 SP(SI4, RE, RC, RD, RB, RA, 12); KL2(RE, RD, RA, RB, RC, 12);
687 SP(SI3, RE, RD, RA, RB, RC, 11); KL2(RA, RE, RC, RB, RD, 11);
688 SP(SI2, RA, RE, RC, RB, RD, 10); KL2(RE, RD, RB, RC, RA, 10);
689 SP(SI1, RE, RD, RB, RC, RA, 9); KL2(RA, RD, RB, RC, RE, 9);
690 SP(SI0, RA, RD, RB, RC, RE, 8); KL2(RB, RE, RD, RA, RC, 8);
691 SP(SI7, RB, RE, RD, RA, RC, 7); KL2(RE, RA, RB, RC, RD, 7);
692 SP(SI6, RE, RA, RB, RC, RD, 6); KL2(RB, RD, RC, RE, RA, 6);
693 SP(SI5, RB, RD, RC, RE, RA, 5); KL2(RD, RA, RB, RC, RE, 5);
694 SP(SI4, RD, RA, RB, RC, RE, 4); KL2(RD, RB, RE, RC, RA, 4);
695 SP(SI3, RD, RB, RE, RC, RA, 3); KL2(RE, RD, RA, RC, RB, 3);
696 SP(SI2, RE, RD, RA, RC, RB, 2); KL2(RD, RB, RC, RA, RE, 2);
697 SP(SI1, RD, RB, RC, RA, RE, 1); KL2(RE, RB, RC, RA, RD, 1);
698 S(SI0, RE, RB, RC, RA, RD); K2(RC, RD, RB, RE, RA, 0);
699
700 leaq (4*4*4)(%rsi), %rax;
701 write_blocks(%rsi, RC1, RD1, RB1, RE1, RK0, RK1, RK2);
702 write_blocks(%rax, RC2, RD2, RB2, RE2, RK0, RK1, RK2);
703
704 ret;
diff --git a/arch/x86/crypto/serpent_avx_glue.c b/arch/x86/crypto/serpent_avx_glue.c
new file mode 100644
index 000000000000..b36bdac237eb
--- /dev/null
+++ b/arch/x86/crypto/serpent_avx_glue.c
@@ -0,0 +1,636 @@
1/*
2 * Glue Code for AVX assembler versions of Serpent Cipher
3 *
4 * Copyright (C) 2012 Johannes Goetzfried
5 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
6 *
7 * Glue code based on serpent_sse2_glue.c by:
8 * Copyright (C) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; either version 2 of the License, or
13 * (at your option) any later version.
14 *
15 * This program is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 * GNU General Public License for more details.
19 *
20 * You should have received a copy of the GNU General Public License
21 * along with this program; if not, write to the Free Software
22 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
23 * USA
24 *
25 */
26
27#include <linux/module.h>
28#include <linux/hardirq.h>
29#include <linux/types.h>
30#include <linux/crypto.h>
31#include <linux/err.h>
32#include <crypto/algapi.h>
33#include <crypto/serpent.h>
34#include <crypto/cryptd.h>
35#include <crypto/b128ops.h>
36#include <crypto/ctr.h>
37#include <crypto/lrw.h>
38#include <crypto/xts.h>
39#include <asm/xcr.h>
40#include <asm/xsave.h>
41#include <asm/crypto/serpent-avx.h>
42#include <asm/crypto/ablk_helper.h>
43#include <asm/crypto/glue_helper.h>
44
45static void serpent_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src)
46{
47 u128 ivs[SERPENT_PARALLEL_BLOCKS - 1];
48 unsigned int j;
49
50 for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
51 ivs[j] = src[j];
52
53 serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
54
55 for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
56 u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);
57}
58
59static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv)
60{
61 be128 ctrblk;
62
63 u128_to_be128(&ctrblk, iv);
64 u128_inc(iv);
65
66 __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
67 u128_xor(dst, src, (u128 *)&ctrblk);
68}
69
70static void serpent_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src,
71 u128 *iv)
72{
73 be128 ctrblks[SERPENT_PARALLEL_BLOCKS];
74 unsigned int i;
75
76 for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) {
77 if (dst != src)
78 dst[i] = src[i];
79
80 u128_to_be128(&ctrblks[i], iv);
81 u128_inc(iv);
82 }
83
84 serpent_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks);
85}
86
87static const struct common_glue_ctx serpent_enc = {
88 .num_funcs = 2,
89 .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
90
91 .funcs = { {
92 .num_blocks = SERPENT_PARALLEL_BLOCKS,
93 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_enc_blk_xway) }
94 }, {
95 .num_blocks = 1,
96 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
97 } }
98};
99
100static const struct common_glue_ctx serpent_ctr = {
101 .num_funcs = 2,
102 .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
103
104 .funcs = { {
105 .num_blocks = SERPENT_PARALLEL_BLOCKS,
106 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr_xway) }
107 }, {
108 .num_blocks = 1,
109 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr) }
110 } }
111};
112
113static const struct common_glue_ctx serpent_dec = {
114 .num_funcs = 2,
115 .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
116
117 .funcs = { {
118 .num_blocks = SERPENT_PARALLEL_BLOCKS,
119 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_dec_blk_xway) }
120 }, {
121 .num_blocks = 1,
122 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
123 } }
124};
125
126static const struct common_glue_ctx serpent_dec_cbc = {
127 .num_funcs = 2,
128 .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
129
130 .funcs = { {
131 .num_blocks = SERPENT_PARALLEL_BLOCKS,
132 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_decrypt_cbc_xway) }
133 }, {
134 .num_blocks = 1,
135 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
136 } }
137};
138
139static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
140 struct scatterlist *src, unsigned int nbytes)
141{
142 return glue_ecb_crypt_128bit(&serpent_enc, desc, dst, src, nbytes);
143}
144
145static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
146 struct scatterlist *src, unsigned int nbytes)
147{
148 return glue_ecb_crypt_128bit(&serpent_dec, desc, dst, src, nbytes);
149}
150
151static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
152 struct scatterlist *src, unsigned int nbytes)
153{
154 return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__serpent_encrypt), desc,
155 dst, src, nbytes);
156}
157
158static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
159 struct scatterlist *src, unsigned int nbytes)
160{
161 return glue_cbc_decrypt_128bit(&serpent_dec_cbc, desc, dst, src,
162 nbytes);
163}
164
165static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
166 struct scatterlist *src, unsigned int nbytes)
167{
168 return glue_ctr_crypt_128bit(&serpent_ctr, desc, dst, src, nbytes);
169}
170
171static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes)
172{
173 return glue_fpu_begin(SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS,
174 NULL, fpu_enabled, nbytes);
175}
176
177static inline void serpent_fpu_end(bool fpu_enabled)
178{
179 glue_fpu_end(fpu_enabled);
180}
181
182struct crypt_priv {
183 struct serpent_ctx *ctx;
184 bool fpu_enabled;
185};
186
187static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
188{
189 const unsigned int bsize = SERPENT_BLOCK_SIZE;
190 struct crypt_priv *ctx = priv;
191 int i;
192
193 ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
194
195 if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
196 serpent_enc_blk_xway(ctx->ctx, srcdst, srcdst);
197 return;
198 }
199
200 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
201 __serpent_encrypt(ctx->ctx, srcdst, srcdst);
202}
203
204static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
205{
206 const unsigned int bsize = SERPENT_BLOCK_SIZE;
207 struct crypt_priv *ctx = priv;
208 int i;
209
210 ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
211
212 if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
213 serpent_dec_blk_xway(ctx->ctx, srcdst, srcdst);
214 return;
215 }
216
217 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
218 __serpent_decrypt(ctx->ctx, srcdst, srcdst);
219}
220
221struct serpent_lrw_ctx {
222 struct lrw_table_ctx lrw_table;
223 struct serpent_ctx serpent_ctx;
224};
225
226static int lrw_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
227 unsigned int keylen)
228{
229 struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
230 int err;
231
232 err = __serpent_setkey(&ctx->serpent_ctx, key, keylen -
233 SERPENT_BLOCK_SIZE);
234 if (err)
235 return err;
236
237 return lrw_init_table(&ctx->lrw_table, key + keylen -
238 SERPENT_BLOCK_SIZE);
239}
240
241static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
242 struct scatterlist *src, unsigned int nbytes)
243{
244 struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
245 be128 buf[SERPENT_PARALLEL_BLOCKS];
246 struct crypt_priv crypt_ctx = {
247 .ctx = &ctx->serpent_ctx,
248 .fpu_enabled = false,
249 };
250 struct lrw_crypt_req req = {
251 .tbuf = buf,
252 .tbuflen = sizeof(buf),
253
254 .table_ctx = &ctx->lrw_table,
255 .crypt_ctx = &crypt_ctx,
256 .crypt_fn = encrypt_callback,
257 };
258 int ret;
259
260 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
261 ret = lrw_crypt(desc, dst, src, nbytes, &req);
262 serpent_fpu_end(crypt_ctx.fpu_enabled);
263
264 return ret;
265}
266
267static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
268 struct scatterlist *src, unsigned int nbytes)
269{
270 struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
271 be128 buf[SERPENT_PARALLEL_BLOCKS];
272 struct crypt_priv crypt_ctx = {
273 .ctx = &ctx->serpent_ctx,
274 .fpu_enabled = false,
275 };
276 struct lrw_crypt_req req = {
277 .tbuf = buf,
278 .tbuflen = sizeof(buf),
279
280 .table_ctx = &ctx->lrw_table,
281 .crypt_ctx = &crypt_ctx,
282 .crypt_fn = decrypt_callback,
283 };
284 int ret;
285
286 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
287 ret = lrw_crypt(desc, dst, src, nbytes, &req);
288 serpent_fpu_end(crypt_ctx.fpu_enabled);
289
290 return ret;
291}
292
293static void lrw_exit_tfm(struct crypto_tfm *tfm)
294{
295 struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
296
297 lrw_free_table(&ctx->lrw_table);
298}
299
300struct serpent_xts_ctx {
301 struct serpent_ctx tweak_ctx;
302 struct serpent_ctx crypt_ctx;
303};
304
305static int xts_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
306 unsigned int keylen)
307{
308 struct serpent_xts_ctx *ctx = crypto_tfm_ctx(tfm);
309 u32 *flags = &tfm->crt_flags;
310 int err;
311
312 /* key consists of keys of equal size concatenated, therefore
313 * the length must be even
314 */
315 if (keylen % 2) {
316 *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
317 return -EINVAL;
318 }
319
320 /* first half of xts-key is for crypt */
321 err = __serpent_setkey(&ctx->crypt_ctx, key, keylen / 2);
322 if (err)
323 return err;
324
325 /* second half of xts-key is for tweak */
326 return __serpent_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2);
327}
328
329static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
330 struct scatterlist *src, unsigned int nbytes)
331{
332 struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
333 be128 buf[SERPENT_PARALLEL_BLOCKS];
334 struct crypt_priv crypt_ctx = {
335 .ctx = &ctx->crypt_ctx,
336 .fpu_enabled = false,
337 };
338 struct xts_crypt_req req = {
339 .tbuf = buf,
340 .tbuflen = sizeof(buf),
341
342 .tweak_ctx = &ctx->tweak_ctx,
343 .tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt),
344 .crypt_ctx = &crypt_ctx,
345 .crypt_fn = encrypt_callback,
346 };
347 int ret;
348
349 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
350 ret = xts_crypt(desc, dst, src, nbytes, &req);
351 serpent_fpu_end(crypt_ctx.fpu_enabled);
352
353 return ret;
354}
355
356static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
357 struct scatterlist *src, unsigned int nbytes)
358{
359 struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
360 be128 buf[SERPENT_PARALLEL_BLOCKS];
361 struct crypt_priv crypt_ctx = {
362 .ctx = &ctx->crypt_ctx,
363 .fpu_enabled = false,
364 };
365 struct xts_crypt_req req = {
366 .tbuf = buf,
367 .tbuflen = sizeof(buf),
368
369 .tweak_ctx = &ctx->tweak_ctx,
370 .tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt),
371 .crypt_ctx = &crypt_ctx,
372 .crypt_fn = decrypt_callback,
373 };
374 int ret;
375
376 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
377 ret = xts_crypt(desc, dst, src, nbytes, &req);
378 serpent_fpu_end(crypt_ctx.fpu_enabled);
379
380 return ret;
381}
382
383static struct crypto_alg serpent_algs[10] = { {
384 .cra_name = "__ecb-serpent-avx",
385 .cra_driver_name = "__driver-ecb-serpent-avx",
386 .cra_priority = 0,
387 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
388 .cra_blocksize = SERPENT_BLOCK_SIZE,
389 .cra_ctxsize = sizeof(struct serpent_ctx),
390 .cra_alignmask = 0,
391 .cra_type = &crypto_blkcipher_type,
392 .cra_module = THIS_MODULE,
393 .cra_list = LIST_HEAD_INIT(serpent_algs[0].cra_list),
394 .cra_u = {
395 .blkcipher = {
396 .min_keysize = SERPENT_MIN_KEY_SIZE,
397 .max_keysize = SERPENT_MAX_KEY_SIZE,
398 .setkey = serpent_setkey,
399 .encrypt = ecb_encrypt,
400 .decrypt = ecb_decrypt,
401 },
402 },
403}, {
404 .cra_name = "__cbc-serpent-avx",
405 .cra_driver_name = "__driver-cbc-serpent-avx",
406 .cra_priority = 0,
407 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
408 .cra_blocksize = SERPENT_BLOCK_SIZE,
409 .cra_ctxsize = sizeof(struct serpent_ctx),
410 .cra_alignmask = 0,
411 .cra_type = &crypto_blkcipher_type,
412 .cra_module = THIS_MODULE,
413 .cra_list = LIST_HEAD_INIT(serpent_algs[1].cra_list),
414 .cra_u = {
415 .blkcipher = {
416 .min_keysize = SERPENT_MIN_KEY_SIZE,
417 .max_keysize = SERPENT_MAX_KEY_SIZE,
418 .setkey = serpent_setkey,
419 .encrypt = cbc_encrypt,
420 .decrypt = cbc_decrypt,
421 },
422 },
423}, {
424 .cra_name = "__ctr-serpent-avx",
425 .cra_driver_name = "__driver-ctr-serpent-avx",
426 .cra_priority = 0,
427 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
428 .cra_blocksize = 1,
429 .cra_ctxsize = sizeof(struct serpent_ctx),
430 .cra_alignmask = 0,
431 .cra_type = &crypto_blkcipher_type,
432 .cra_module = THIS_MODULE,
433 .cra_list = LIST_HEAD_INIT(serpent_algs[2].cra_list),
434 .cra_u = {
435 .blkcipher = {
436 .min_keysize = SERPENT_MIN_KEY_SIZE,
437 .max_keysize = SERPENT_MAX_KEY_SIZE,
438 .ivsize = SERPENT_BLOCK_SIZE,
439 .setkey = serpent_setkey,
440 .encrypt = ctr_crypt,
441 .decrypt = ctr_crypt,
442 },
443 },
444}, {
445 .cra_name = "__lrw-serpent-avx",
446 .cra_driver_name = "__driver-lrw-serpent-avx",
447 .cra_priority = 0,
448 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
449 .cra_blocksize = SERPENT_BLOCK_SIZE,
450 .cra_ctxsize = sizeof(struct serpent_lrw_ctx),
451 .cra_alignmask = 0,
452 .cra_type = &crypto_blkcipher_type,
453 .cra_module = THIS_MODULE,
454 .cra_list = LIST_HEAD_INIT(serpent_algs[3].cra_list),
455 .cra_exit = lrw_exit_tfm,
456 .cra_u = {
457 .blkcipher = {
458 .min_keysize = SERPENT_MIN_KEY_SIZE +
459 SERPENT_BLOCK_SIZE,
460 .max_keysize = SERPENT_MAX_KEY_SIZE +
461 SERPENT_BLOCK_SIZE,
462 .ivsize = SERPENT_BLOCK_SIZE,
463 .setkey = lrw_serpent_setkey,
464 .encrypt = lrw_encrypt,
465 .decrypt = lrw_decrypt,
466 },
467 },
468}, {
469 .cra_name = "__xts-serpent-avx",
470 .cra_driver_name = "__driver-xts-serpent-avx",
471 .cra_priority = 0,
472 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
473 .cra_blocksize = SERPENT_BLOCK_SIZE,
474 .cra_ctxsize = sizeof(struct serpent_xts_ctx),
475 .cra_alignmask = 0,
476 .cra_type = &crypto_blkcipher_type,
477 .cra_module = THIS_MODULE,
478 .cra_list = LIST_HEAD_INIT(serpent_algs[4].cra_list),
479 .cra_u = {
480 .blkcipher = {
481 .min_keysize = SERPENT_MIN_KEY_SIZE * 2,
482 .max_keysize = SERPENT_MAX_KEY_SIZE * 2,
483 .ivsize = SERPENT_BLOCK_SIZE,
484 .setkey = xts_serpent_setkey,
485 .encrypt = xts_encrypt,
486 .decrypt = xts_decrypt,
487 },
488 },
489}, {
490 .cra_name = "ecb(serpent)",
491 .cra_driver_name = "ecb-serpent-avx",
492 .cra_priority = 500,
493 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
494 .cra_blocksize = SERPENT_BLOCK_SIZE,
495 .cra_ctxsize = sizeof(struct async_helper_ctx),
496 .cra_alignmask = 0,
497 .cra_type = &crypto_ablkcipher_type,
498 .cra_module = THIS_MODULE,
499 .cra_list = LIST_HEAD_INIT(serpent_algs[5].cra_list),
500 .cra_init = ablk_init,
501 .cra_exit = ablk_exit,
502 .cra_u = {
503 .ablkcipher = {
504 .min_keysize = SERPENT_MIN_KEY_SIZE,
505 .max_keysize = SERPENT_MAX_KEY_SIZE,
506 .setkey = ablk_set_key,
507 .encrypt = ablk_encrypt,
508 .decrypt = ablk_decrypt,
509 },
510 },
511}, {
512 .cra_name = "cbc(serpent)",
513 .cra_driver_name = "cbc-serpent-avx",
514 .cra_priority = 500,
515 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
516 .cra_blocksize = SERPENT_BLOCK_SIZE,
517 .cra_ctxsize = sizeof(struct async_helper_ctx),
518 .cra_alignmask = 0,
519 .cra_type = &crypto_ablkcipher_type,
520 .cra_module = THIS_MODULE,
521 .cra_list = LIST_HEAD_INIT(serpent_algs[6].cra_list),
522 .cra_init = ablk_init,
523 .cra_exit = ablk_exit,
524 .cra_u = {
525 .ablkcipher = {
526 .min_keysize = SERPENT_MIN_KEY_SIZE,
527 .max_keysize = SERPENT_MAX_KEY_SIZE,
528 .ivsize = SERPENT_BLOCK_SIZE,
529 .setkey = ablk_set_key,
530 .encrypt = __ablk_encrypt,
531 .decrypt = ablk_decrypt,
532 },
533 },
534}, {
535 .cra_name = "ctr(serpent)",
536 .cra_driver_name = "ctr-serpent-avx",
537 .cra_priority = 500,
538 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
539 .cra_blocksize = 1,
540 .cra_ctxsize = sizeof(struct async_helper_ctx),
541 .cra_alignmask = 0,
542 .cra_type = &crypto_ablkcipher_type,
543 .cra_module = THIS_MODULE,
544 .cra_list = LIST_HEAD_INIT(serpent_algs[7].cra_list),
545 .cra_init = ablk_init,
546 .cra_exit = ablk_exit,
547 .cra_u = {
548 .ablkcipher = {
549 .min_keysize = SERPENT_MIN_KEY_SIZE,
550 .max_keysize = SERPENT_MAX_KEY_SIZE,
551 .ivsize = SERPENT_BLOCK_SIZE,
552 .setkey = ablk_set_key,
553 .encrypt = ablk_encrypt,
554 .decrypt = ablk_encrypt,
555 .geniv = "chainiv",
556 },
557 },
558}, {
559 .cra_name = "lrw(serpent)",
560 .cra_driver_name = "lrw-serpent-avx",
561 .cra_priority = 500,
562 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
563 .cra_blocksize = SERPENT_BLOCK_SIZE,
564 .cra_ctxsize = sizeof(struct async_helper_ctx),
565 .cra_alignmask = 0,
566 .cra_type = &crypto_ablkcipher_type,
567 .cra_module = THIS_MODULE,
568 .cra_list = LIST_HEAD_INIT(serpent_algs[8].cra_list),
569 .cra_init = ablk_init,
570 .cra_exit = ablk_exit,
571 .cra_u = {
572 .ablkcipher = {
573 .min_keysize = SERPENT_MIN_KEY_SIZE +
574 SERPENT_BLOCK_SIZE,
575 .max_keysize = SERPENT_MAX_KEY_SIZE +
576 SERPENT_BLOCK_SIZE,
577 .ivsize = SERPENT_BLOCK_SIZE,
578 .setkey = ablk_set_key,
579 .encrypt = ablk_encrypt,
580 .decrypt = ablk_decrypt,
581 },
582 },
583}, {
584 .cra_name = "xts(serpent)",
585 .cra_driver_name = "xts-serpent-avx",
586 .cra_priority = 500,
587 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
588 .cra_blocksize = SERPENT_BLOCK_SIZE,
589 .cra_ctxsize = sizeof(struct async_helper_ctx),
590 .cra_alignmask = 0,
591 .cra_type = &crypto_ablkcipher_type,
592 .cra_module = THIS_MODULE,
593 .cra_list = LIST_HEAD_INIT(serpent_algs[9].cra_list),
594 .cra_init = ablk_init,
595 .cra_exit = ablk_exit,
596 .cra_u = {
597 .ablkcipher = {
598 .min_keysize = SERPENT_MIN_KEY_SIZE * 2,
599 .max_keysize = SERPENT_MAX_KEY_SIZE * 2,
600 .ivsize = SERPENT_BLOCK_SIZE,
601 .setkey = ablk_set_key,
602 .encrypt = ablk_encrypt,
603 .decrypt = ablk_decrypt,
604 },
605 },
606} };
607
608static int __init serpent_init(void)
609{
610 u64 xcr0;
611
612 if (!cpu_has_avx || !cpu_has_osxsave) {
613 printk(KERN_INFO "AVX instructions are not detected.\n");
614 return -ENODEV;
615 }
616
617 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
618 if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
619 printk(KERN_INFO "AVX detected but unusable.\n");
620 return -ENODEV;
621 }
622
623 return crypto_register_algs(serpent_algs, ARRAY_SIZE(serpent_algs));
624}
625
626static void __exit serpent_exit(void)
627{
628 crypto_unregister_algs(serpent_algs, ARRAY_SIZE(serpent_algs));
629}
630
631module_init(serpent_init);
632module_exit(serpent_exit);
633
634MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX optimized");
635MODULE_LICENSE("GPL");
636MODULE_ALIAS("serpent");
diff --git a/arch/x86/crypto/serpent_sse2_glue.c b/arch/x86/crypto/serpent_sse2_glue.c
index 4b21be85e0a1..d679c8675f4a 100644
--- a/arch/x86/crypto/serpent_sse2_glue.c
+++ b/arch/x86/crypto/serpent_sse2_glue.c
@@ -41,358 +41,145 @@
41#include <crypto/ctr.h> 41#include <crypto/ctr.h>
42#include <crypto/lrw.h> 42#include <crypto/lrw.h>
43#include <crypto/xts.h> 43#include <crypto/xts.h>
44#include <asm/i387.h> 44#include <asm/crypto/serpent-sse2.h>
45#include <asm/serpent.h> 45#include <asm/crypto/ablk_helper.h>
46#include <crypto/scatterwalk.h> 46#include <asm/crypto/glue_helper.h>
47#include <linux/workqueue.h>
48#include <linux/spinlock.h>
49
50struct async_serpent_ctx {
51 struct cryptd_ablkcipher *cryptd_tfm;
52};
53 47
54static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes) 48static void serpent_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src)
55{
56 if (fpu_enabled)
57 return true;
58
59 /* SSE2 is only used when chunk to be processed is large enough, so
60 * do not enable FPU until it is necessary.
61 */
62 if (nbytes < SERPENT_BLOCK_SIZE * SERPENT_PARALLEL_BLOCKS)
63 return false;
64
65 kernel_fpu_begin();
66 return true;
67}
68
69static inline void serpent_fpu_end(bool fpu_enabled)
70{ 49{
71 if (fpu_enabled) 50 u128 ivs[SERPENT_PARALLEL_BLOCKS - 1];
72 kernel_fpu_end(); 51 unsigned int j;
73}
74
75static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
76 bool enc)
77{
78 bool fpu_enabled = false;
79 struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
80 const unsigned int bsize = SERPENT_BLOCK_SIZE;
81 unsigned int nbytes;
82 int err;
83
84 err = blkcipher_walk_virt(desc, walk);
85 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
86
87 while ((nbytes = walk->nbytes)) {
88 u8 *wsrc = walk->src.virt.addr;
89 u8 *wdst = walk->dst.virt.addr;
90
91 fpu_enabled = serpent_fpu_begin(fpu_enabled, nbytes);
92
93 /* Process multi-block batch */
94 if (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS) {
95 do {
96 if (enc)
97 serpent_enc_blk_xway(ctx, wdst, wsrc);
98 else
99 serpent_dec_blk_xway(ctx, wdst, wsrc);
100
101 wsrc += bsize * SERPENT_PARALLEL_BLOCKS;
102 wdst += bsize * SERPENT_PARALLEL_BLOCKS;
103 nbytes -= bsize * SERPENT_PARALLEL_BLOCKS;
104 } while (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS);
105
106 if (nbytes < bsize)
107 goto done;
108 }
109
110 /* Handle leftovers */
111 do {
112 if (enc)
113 __serpent_encrypt(ctx, wdst, wsrc);
114 else
115 __serpent_decrypt(ctx, wdst, wsrc);
116
117 wsrc += bsize;
118 wdst += bsize;
119 nbytes -= bsize;
120 } while (nbytes >= bsize);
121
122done:
123 err = blkcipher_walk_done(desc, walk, nbytes);
124 }
125 52
126 serpent_fpu_end(fpu_enabled); 53 for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
127 return err; 54 ivs[j] = src[j];
128}
129 55
130static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 56 serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
131 struct scatterlist *src, unsigned int nbytes)
132{
133 struct blkcipher_walk walk;
134 57
135 blkcipher_walk_init(&walk, dst, src, nbytes); 58 for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
136 return ecb_crypt(desc, &walk, true); 59 u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);
137} 60}
138 61
139static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 62static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv)
140 struct scatterlist *src, unsigned int nbytes)
141{ 63{
142 struct blkcipher_walk walk; 64 be128 ctrblk;
143 65
144 blkcipher_walk_init(&walk, dst, src, nbytes); 66 u128_to_be128(&ctrblk, iv);
145 return ecb_crypt(desc, &walk, false); 67 u128_inc(iv);
146}
147 68
148static unsigned int __cbc_encrypt(struct blkcipher_desc *desc, 69 __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
149 struct blkcipher_walk *walk) 70 u128_xor(dst, src, (u128 *)&ctrblk);
150{
151 struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
152 const unsigned int bsize = SERPENT_BLOCK_SIZE;
153 unsigned int nbytes = walk->nbytes;
154 u128 *src = (u128 *)walk->src.virt.addr;
155 u128 *dst = (u128 *)walk->dst.virt.addr;
156 u128 *iv = (u128 *)walk->iv;
157
158 do {
159 u128_xor(dst, src, iv);
160 __serpent_encrypt(ctx, (u8 *)dst, (u8 *)dst);
161 iv = dst;
162
163 src += 1;
164 dst += 1;
165 nbytes -= bsize;
166 } while (nbytes >= bsize);
167
168 u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv);
169 return nbytes;
170} 71}
171 72
172static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 73static void serpent_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src,
173 struct scatterlist *src, unsigned int nbytes) 74 u128 *iv)
174{ 75{
175 struct blkcipher_walk walk; 76 be128 ctrblks[SERPENT_PARALLEL_BLOCKS];
176 int err; 77 unsigned int i;
177 78
178 blkcipher_walk_init(&walk, dst, src, nbytes); 79 for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) {
179 err = blkcipher_walk_virt(desc, &walk); 80 if (dst != src)
81 dst[i] = src[i];
180 82
181 while ((nbytes = walk.nbytes)) { 83 u128_to_be128(&ctrblks[i], iv);
182 nbytes = __cbc_encrypt(desc, &walk); 84 u128_inc(iv);
183 err = blkcipher_walk_done(desc, &walk, nbytes);
184 } 85 }
185 86
186 return err; 87 serpent_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks);
187} 88}
188 89
189static unsigned int __cbc_decrypt(struct blkcipher_desc *desc, 90static const struct common_glue_ctx serpent_enc = {
190 struct blkcipher_walk *walk) 91 .num_funcs = 2,
191{ 92 .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
192 struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
193 const unsigned int bsize = SERPENT_BLOCK_SIZE;
194 unsigned int nbytes = walk->nbytes;
195 u128 *src = (u128 *)walk->src.virt.addr;
196 u128 *dst = (u128 *)walk->dst.virt.addr;
197 u128 ivs[SERPENT_PARALLEL_BLOCKS - 1];
198 u128 last_iv;
199 int i;
200
201 /* Start of the last block. */
202 src += nbytes / bsize - 1;
203 dst += nbytes / bsize - 1;
204
205 last_iv = *src;
206
207 /* Process multi-block batch */
208 if (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS) {
209 do {
210 nbytes -= bsize * (SERPENT_PARALLEL_BLOCKS - 1);
211 src -= SERPENT_PARALLEL_BLOCKS - 1;
212 dst -= SERPENT_PARALLEL_BLOCKS - 1;
213
214 for (i = 0; i < SERPENT_PARALLEL_BLOCKS - 1; i++)
215 ivs[i] = src[i];
216
217 serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
218
219 for (i = 0; i < SERPENT_PARALLEL_BLOCKS - 1; i++)
220 u128_xor(dst + (i + 1), dst + (i + 1), ivs + i);
221
222 nbytes -= bsize;
223 if (nbytes < bsize)
224 goto done;
225 93
226 u128_xor(dst, dst, src - 1); 94 .funcs = { {
227 src -= 1; 95 .num_blocks = SERPENT_PARALLEL_BLOCKS,
228 dst -= 1; 96 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_enc_blk_xway) }
229 } while (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS); 97 }, {
230 98 .num_blocks = 1,
231 if (nbytes < bsize) 99 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
232 goto done; 100 } }
233 } 101};
234
235 /* Handle leftovers */
236 for (;;) {
237 __serpent_decrypt(ctx, (u8 *)dst, (u8 *)src);
238
239 nbytes -= bsize;
240 if (nbytes < bsize)
241 break;
242 102
243 u128_xor(dst, dst, src - 1); 103static const struct common_glue_ctx serpent_ctr = {
244 src -= 1; 104 .num_funcs = 2,
245 dst -= 1; 105 .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
246 } 106
107 .funcs = { {
108 .num_blocks = SERPENT_PARALLEL_BLOCKS,
109 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr_xway) }
110 }, {
111 .num_blocks = 1,
112 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr) }
113 } }
114};
247 115
248done: 116static const struct common_glue_ctx serpent_dec = {
249 u128_xor(dst, dst, (u128 *)walk->iv); 117 .num_funcs = 2,
250 *(u128 *)walk->iv = last_iv; 118 .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
119
120 .funcs = { {
121 .num_blocks = SERPENT_PARALLEL_BLOCKS,
122 .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_dec_blk_xway) }
123 }, {
124 .num_blocks = 1,
125 .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
126 } }
127};
251 128
252 return nbytes; 129static const struct common_glue_ctx serpent_dec_cbc = {
253} 130 .num_funcs = 2,
131 .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
132
133 .funcs = { {
134 .num_blocks = SERPENT_PARALLEL_BLOCKS,
135 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_decrypt_cbc_xway) }
136 }, {
137 .num_blocks = 1,
138 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
139 } }
140};
254 141
255static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 142static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
256 struct scatterlist *src, unsigned int nbytes) 143 struct scatterlist *src, unsigned int nbytes)
257{ 144{
258 bool fpu_enabled = false; 145 return glue_ecb_crypt_128bit(&serpent_enc, desc, dst, src, nbytes);
259 struct blkcipher_walk walk;
260 int err;
261
262 blkcipher_walk_init(&walk, dst, src, nbytes);
263 err = blkcipher_walk_virt(desc, &walk);
264 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
265
266 while ((nbytes = walk.nbytes)) {
267 fpu_enabled = serpent_fpu_begin(fpu_enabled, nbytes);
268 nbytes = __cbc_decrypt(desc, &walk);
269 err = blkcipher_walk_done(desc, &walk, nbytes);
270 }
271
272 serpent_fpu_end(fpu_enabled);
273 return err;
274} 146}
275 147
276static inline void u128_to_be128(be128 *dst, const u128 *src) 148static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
149 struct scatterlist *src, unsigned int nbytes)
277{ 150{
278 dst->a = cpu_to_be64(src->a); 151 return glue_ecb_crypt_128bit(&serpent_dec, desc, dst, src, nbytes);
279 dst->b = cpu_to_be64(src->b);
280} 152}
281 153
282static inline void be128_to_u128(u128 *dst, const be128 *src) 154static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
155 struct scatterlist *src, unsigned int nbytes)
283{ 156{
284 dst->a = be64_to_cpu(src->a); 157 return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__serpent_encrypt), desc,
285 dst->b = be64_to_cpu(src->b); 158 dst, src, nbytes);
286} 159}
287 160
288static inline void u128_inc(u128 *i) 161static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
162 struct scatterlist *src, unsigned int nbytes)
289{ 163{
290 i->b++; 164 return glue_cbc_decrypt_128bit(&serpent_dec_cbc, desc, dst, src,
291 if (!i->b) 165 nbytes);
292 i->a++;
293} 166}
294 167
295static void ctr_crypt_final(struct blkcipher_desc *desc, 168static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
296 struct blkcipher_walk *walk) 169 struct scatterlist *src, unsigned int nbytes)
297{ 170{
298 struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 171 return glue_ctr_crypt_128bit(&serpent_ctr, desc, dst, src, nbytes);
299 u8 *ctrblk = walk->iv;
300 u8 keystream[SERPENT_BLOCK_SIZE];
301 u8 *src = walk->src.virt.addr;
302 u8 *dst = walk->dst.virt.addr;
303 unsigned int nbytes = walk->nbytes;
304
305 __serpent_encrypt(ctx, keystream, ctrblk);
306 crypto_xor(keystream, src, nbytes);
307 memcpy(dst, keystream, nbytes);
308
309 crypto_inc(ctrblk, SERPENT_BLOCK_SIZE);
310} 172}
311 173
312static unsigned int __ctr_crypt(struct blkcipher_desc *desc, 174static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes)
313 struct blkcipher_walk *walk)
314{ 175{
315 struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 176 return glue_fpu_begin(SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS,
316 const unsigned int bsize = SERPENT_BLOCK_SIZE; 177 NULL, fpu_enabled, nbytes);
317 unsigned int nbytes = walk->nbytes;
318 u128 *src = (u128 *)walk->src.virt.addr;
319 u128 *dst = (u128 *)walk->dst.virt.addr;
320 u128 ctrblk;
321 be128 ctrblocks[SERPENT_PARALLEL_BLOCKS];
322 int i;
323
324 be128_to_u128(&ctrblk, (be128 *)walk->iv);
325
326 /* Process multi-block batch */
327 if (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS) {
328 do {
329 /* create ctrblks for parallel encrypt */
330 for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) {
331 if (dst != src)
332 dst[i] = src[i];
333
334 u128_to_be128(&ctrblocks[i], &ctrblk);
335 u128_inc(&ctrblk);
336 }
337
338 serpent_enc_blk_xway_xor(ctx, (u8 *)dst,
339 (u8 *)ctrblocks);
340
341 src += SERPENT_PARALLEL_BLOCKS;
342 dst += SERPENT_PARALLEL_BLOCKS;
343 nbytes -= bsize * SERPENT_PARALLEL_BLOCKS;
344 } while (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS);
345
346 if (nbytes < bsize)
347 goto done;
348 }
349
350 /* Handle leftovers */
351 do {
352 if (dst != src)
353 *dst = *src;
354
355 u128_to_be128(&ctrblocks[0], &ctrblk);
356 u128_inc(&ctrblk);
357
358 __serpent_encrypt(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks);
359 u128_xor(dst, dst, (u128 *)ctrblocks);
360
361 src += 1;
362 dst += 1;
363 nbytes -= bsize;
364 } while (nbytes >= bsize);
365
366done:
367 u128_to_be128((be128 *)walk->iv, &ctrblk);
368 return nbytes;
369} 178}
370 179
371static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, 180static inline void serpent_fpu_end(bool fpu_enabled)
372 struct scatterlist *src, unsigned int nbytes)
373{ 181{
374 bool fpu_enabled = false; 182 glue_fpu_end(fpu_enabled);
375 struct blkcipher_walk walk;
376 int err;
377
378 blkcipher_walk_init(&walk, dst, src, nbytes);
379 err = blkcipher_walk_virt_block(desc, &walk, SERPENT_BLOCK_SIZE);
380 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
381
382 while ((nbytes = walk.nbytes) >= SERPENT_BLOCK_SIZE) {
383 fpu_enabled = serpent_fpu_begin(fpu_enabled, nbytes);
384 nbytes = __ctr_crypt(desc, &walk);
385 err = blkcipher_walk_done(desc, &walk, nbytes);
386 }
387
388 serpent_fpu_end(fpu_enabled);
389
390 if (walk.nbytes) {
391 ctr_crypt_final(desc, &walk);
392 err = blkcipher_walk_done(desc, &walk, 0);
393 }
394
395 return err;
396} 183}
397 184
398struct crypt_priv { 185struct crypt_priv {
@@ -596,106 +383,6 @@ static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
596 return ret; 383 return ret;
597} 384}
598 385
599static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
600 unsigned int key_len)
601{
602 struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm);
603 struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base;
604 int err;
605
606 crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
607 crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm)
608 & CRYPTO_TFM_REQ_MASK);
609 err = crypto_ablkcipher_setkey(child, key, key_len);
610 crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child)
611 & CRYPTO_TFM_RES_MASK);
612 return err;
613}
614
615static int __ablk_encrypt(struct ablkcipher_request *req)
616{
617 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
618 struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm);
619 struct blkcipher_desc desc;
620
621 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
622 desc.info = req->info;
623 desc.flags = 0;
624
625 return crypto_blkcipher_crt(desc.tfm)->encrypt(
626 &desc, req->dst, req->src, req->nbytes);
627}
628
629static int ablk_encrypt(struct ablkcipher_request *req)
630{
631 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
632 struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm);
633
634 if (!irq_fpu_usable()) {
635 struct ablkcipher_request *cryptd_req =
636 ablkcipher_request_ctx(req);
637
638 memcpy(cryptd_req, req, sizeof(*req));
639 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
640
641 return crypto_ablkcipher_encrypt(cryptd_req);
642 } else {
643 return __ablk_encrypt(req);
644 }
645}
646
647static int ablk_decrypt(struct ablkcipher_request *req)
648{
649 struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
650 struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm);
651
652 if (!irq_fpu_usable()) {
653 struct ablkcipher_request *cryptd_req =
654 ablkcipher_request_ctx(req);
655
656 memcpy(cryptd_req, req, sizeof(*req));
657 ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
658
659 return crypto_ablkcipher_decrypt(cryptd_req);
660 } else {
661 struct blkcipher_desc desc;
662
663 desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
664 desc.info = req->info;
665 desc.flags = 0;
666
667 return crypto_blkcipher_crt(desc.tfm)->decrypt(
668 &desc, req->dst, req->src, req->nbytes);
669 }
670}
671
672static void ablk_exit(struct crypto_tfm *tfm)
673{
674 struct async_serpent_ctx *ctx = crypto_tfm_ctx(tfm);
675
676 cryptd_free_ablkcipher(ctx->cryptd_tfm);
677}
678
679static int ablk_init(struct crypto_tfm *tfm)
680{
681 struct async_serpent_ctx *ctx = crypto_tfm_ctx(tfm);
682 struct cryptd_ablkcipher *cryptd_tfm;
683 char drv_name[CRYPTO_MAX_ALG_NAME];
684
685 snprintf(drv_name, sizeof(drv_name), "__driver-%s",
686 crypto_tfm_alg_driver_name(tfm));
687
688 cryptd_tfm = cryptd_alloc_ablkcipher(drv_name, 0, 0);
689 if (IS_ERR(cryptd_tfm))
690 return PTR_ERR(cryptd_tfm);
691
692 ctx->cryptd_tfm = cryptd_tfm;
693 tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
694 crypto_ablkcipher_reqsize(&cryptd_tfm->base);
695
696 return 0;
697}
698
699static struct crypto_alg serpent_algs[10] = { { 386static struct crypto_alg serpent_algs[10] = { {
700 .cra_name = "__ecb-serpent-sse2", 387 .cra_name = "__ecb-serpent-sse2",
701 .cra_driver_name = "__driver-ecb-serpent-sse2", 388 .cra_driver_name = "__driver-ecb-serpent-sse2",
@@ -808,7 +495,7 @@ static struct crypto_alg serpent_algs[10] = { {
808 .cra_priority = 400, 495 .cra_priority = 400,
809 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 496 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
810 .cra_blocksize = SERPENT_BLOCK_SIZE, 497 .cra_blocksize = SERPENT_BLOCK_SIZE,
811 .cra_ctxsize = sizeof(struct async_serpent_ctx), 498 .cra_ctxsize = sizeof(struct async_helper_ctx),
812 .cra_alignmask = 0, 499 .cra_alignmask = 0,
813 .cra_type = &crypto_ablkcipher_type, 500 .cra_type = &crypto_ablkcipher_type,
814 .cra_module = THIS_MODULE, 501 .cra_module = THIS_MODULE,
@@ -830,7 +517,7 @@ static struct crypto_alg serpent_algs[10] = { {
830 .cra_priority = 400, 517 .cra_priority = 400,
831 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 518 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
832 .cra_blocksize = SERPENT_BLOCK_SIZE, 519 .cra_blocksize = SERPENT_BLOCK_SIZE,
833 .cra_ctxsize = sizeof(struct async_serpent_ctx), 520 .cra_ctxsize = sizeof(struct async_helper_ctx),
834 .cra_alignmask = 0, 521 .cra_alignmask = 0,
835 .cra_type = &crypto_ablkcipher_type, 522 .cra_type = &crypto_ablkcipher_type,
836 .cra_module = THIS_MODULE, 523 .cra_module = THIS_MODULE,
@@ -853,7 +540,7 @@ static struct crypto_alg serpent_algs[10] = { {
853 .cra_priority = 400, 540 .cra_priority = 400,
854 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 541 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
855 .cra_blocksize = 1, 542 .cra_blocksize = 1,
856 .cra_ctxsize = sizeof(struct async_serpent_ctx), 543 .cra_ctxsize = sizeof(struct async_helper_ctx),
857 .cra_alignmask = 0, 544 .cra_alignmask = 0,
858 .cra_type = &crypto_ablkcipher_type, 545 .cra_type = &crypto_ablkcipher_type,
859 .cra_module = THIS_MODULE, 546 .cra_module = THIS_MODULE,
@@ -877,7 +564,7 @@ static struct crypto_alg serpent_algs[10] = { {
877 .cra_priority = 400, 564 .cra_priority = 400,
878 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 565 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
879 .cra_blocksize = SERPENT_BLOCK_SIZE, 566 .cra_blocksize = SERPENT_BLOCK_SIZE,
880 .cra_ctxsize = sizeof(struct async_serpent_ctx), 567 .cra_ctxsize = sizeof(struct async_helper_ctx),
881 .cra_alignmask = 0, 568 .cra_alignmask = 0,
882 .cra_type = &crypto_ablkcipher_type, 569 .cra_type = &crypto_ablkcipher_type,
883 .cra_module = THIS_MODULE, 570 .cra_module = THIS_MODULE,
@@ -902,7 +589,7 @@ static struct crypto_alg serpent_algs[10] = { {
902 .cra_priority = 400, 589 .cra_priority = 400,
903 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, 590 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
904 .cra_blocksize = SERPENT_BLOCK_SIZE, 591 .cra_blocksize = SERPENT_BLOCK_SIZE,
905 .cra_ctxsize = sizeof(struct async_serpent_ctx), 592 .cra_ctxsize = sizeof(struct async_helper_ctx),
906 .cra_alignmask = 0, 593 .cra_alignmask = 0,
907 .cra_type = &crypto_ablkcipher_type, 594 .cra_type = &crypto_ablkcipher_type,
908 .cra_module = THIS_MODULE, 595 .cra_module = THIS_MODULE,
diff --git a/arch/x86/crypto/sha1_ssse3_asm.S b/arch/x86/crypto/sha1_ssse3_asm.S
index b2c2f57d70e8..49d6987a73d9 100644
--- a/arch/x86/crypto/sha1_ssse3_asm.S
+++ b/arch/x86/crypto/sha1_ssse3_asm.S
@@ -468,7 +468,7 @@ W_PRECALC_SSSE3
468 */ 468 */
469SHA1_VECTOR_ASM sha1_transform_ssse3 469SHA1_VECTOR_ASM sha1_transform_ssse3
470 470
471#ifdef SHA1_ENABLE_AVX_SUPPORT 471#ifdef CONFIG_AS_AVX
472 472
473.macro W_PRECALC_AVX 473.macro W_PRECALC_AVX
474 474
diff --git a/arch/x86/crypto/sha1_ssse3_glue.c b/arch/x86/crypto/sha1_ssse3_glue.c
index f916499d0abe..4a11a9d72451 100644
--- a/arch/x86/crypto/sha1_ssse3_glue.c
+++ b/arch/x86/crypto/sha1_ssse3_glue.c
@@ -35,7 +35,7 @@
35 35
36asmlinkage void sha1_transform_ssse3(u32 *digest, const char *data, 36asmlinkage void sha1_transform_ssse3(u32 *digest, const char *data,
37 unsigned int rounds); 37 unsigned int rounds);
38#ifdef SHA1_ENABLE_AVX_SUPPORT 38#ifdef CONFIG_AS_AVX
39asmlinkage void sha1_transform_avx(u32 *digest, const char *data, 39asmlinkage void sha1_transform_avx(u32 *digest, const char *data,
40 unsigned int rounds); 40 unsigned int rounds);
41#endif 41#endif
@@ -184,7 +184,7 @@ static struct shash_alg alg = {
184 } 184 }
185}; 185};
186 186
187#ifdef SHA1_ENABLE_AVX_SUPPORT 187#ifdef CONFIG_AS_AVX
188static bool __init avx_usable(void) 188static bool __init avx_usable(void)
189{ 189{
190 u64 xcr0; 190 u64 xcr0;
@@ -209,7 +209,7 @@ static int __init sha1_ssse3_mod_init(void)
209 if (cpu_has_ssse3) 209 if (cpu_has_ssse3)
210 sha1_transform_asm = sha1_transform_ssse3; 210 sha1_transform_asm = sha1_transform_ssse3;
211 211
212#ifdef SHA1_ENABLE_AVX_SUPPORT 212#ifdef CONFIG_AS_AVX
213 /* allow AVX to override SSSE3, it's a little faster */ 213 /* allow AVX to override SSSE3, it's a little faster */
214 if (avx_usable()) 214 if (avx_usable())
215 sha1_transform_asm = sha1_transform_avx; 215 sha1_transform_asm = sha1_transform_avx;
diff --git a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
new file mode 100644
index 000000000000..35f45574390d
--- /dev/null
+++ b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
@@ -0,0 +1,300 @@
1/*
2 * Twofish Cipher 8-way parallel algorithm (AVX/x86_64)
3 *
4 * Copyright (C) 2012 Johannes Goetzfried
5 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
20 * USA
21 *
22 */
23
24.file "twofish-avx-x86_64-asm_64.S"
25.text
26
27/* structure of crypto context */
28#define s0 0
29#define s1 1024
30#define s2 2048
31#define s3 3072
32#define w 4096
33#define k 4128
34
35/**********************************************************************
36 8-way AVX twofish
37 **********************************************************************/
38#define CTX %rdi
39
40#define RA1 %xmm0
41#define RB1 %xmm1
42#define RC1 %xmm2
43#define RD1 %xmm3
44
45#define RA2 %xmm4
46#define RB2 %xmm5
47#define RC2 %xmm6
48#define RD2 %xmm7
49
50#define RX %xmm8
51#define RY %xmm9
52
53#define RK1 %xmm10
54#define RK2 %xmm11
55
56#define RID1 %rax
57#define RID1b %al
58#define RID2 %rbx
59#define RID2b %bl
60
61#define RGI1 %rdx
62#define RGI1bl %dl
63#define RGI1bh %dh
64#define RGI2 %rcx
65#define RGI2bl %cl
66#define RGI2bh %ch
67
68#define RGS1 %r8
69#define RGS1d %r8d
70#define RGS2 %r9
71#define RGS2d %r9d
72#define RGS3 %r10
73#define RGS3d %r10d
74
75
76#define lookup_32bit(t0, t1, t2, t3, src, dst) \
77 movb src ## bl, RID1b; \
78 movb src ## bh, RID2b; \
79 movl t0(CTX, RID1, 4), dst ## d; \
80 xorl t1(CTX, RID2, 4), dst ## d; \
81 shrq $16, src; \
82 movb src ## bl, RID1b; \
83 movb src ## bh, RID2b; \
84 xorl t2(CTX, RID1, 4), dst ## d; \
85 xorl t3(CTX, RID2, 4), dst ## d;
86
87#define G(a, x, t0, t1, t2, t3) \
88 vmovq a, RGI1; \
89 vpsrldq $8, a, x; \
90 vmovq x, RGI2; \
91 \
92 lookup_32bit(t0, t1, t2, t3, RGI1, RGS1); \
93 shrq $16, RGI1; \
94 lookup_32bit(t0, t1, t2, t3, RGI1, RGS2); \
95 shlq $32, RGS2; \
96 orq RGS1, RGS2; \
97 \
98 lookup_32bit(t0, t1, t2, t3, RGI2, RGS1); \
99 shrq $16, RGI2; \
100 lookup_32bit(t0, t1, t2, t3, RGI2, RGS3); \
101 shlq $32, RGS3; \
102 orq RGS1, RGS3; \
103 \
104 vmovq RGS2, x; \
105 vpinsrq $1, RGS3, x, x;
106
107#define encround(a, b, c, d, x, y) \
108 G(a, x, s0, s1, s2, s3); \
109 G(b, y, s1, s2, s3, s0); \
110 vpaddd x, y, x; \
111 vpaddd y, x, y; \
112 vpaddd x, RK1, x; \
113 vpaddd y, RK2, y; \
114 vpxor x, c, c; \
115 vpsrld $1, c, x; \
116 vpslld $(32 - 1), c, c; \
117 vpor c, x, c; \
118 vpslld $1, d, x; \
119 vpsrld $(32 - 1), d, d; \
120 vpor d, x, d; \
121 vpxor d, y, d;
122
123#define decround(a, b, c, d, x, y) \
124 G(a, x, s0, s1, s2, s3); \
125 G(b, y, s1, s2, s3, s0); \
126 vpaddd x, y, x; \
127 vpaddd y, x, y; \
128 vpaddd y, RK2, y; \
129 vpxor d, y, d; \
130 vpsrld $1, d, y; \
131 vpslld $(32 - 1), d, d; \
132 vpor d, y, d; \
133 vpslld $1, c, y; \
134 vpsrld $(32 - 1), c, c; \
135 vpor c, y, c; \
136 vpaddd x, RK1, x; \
137 vpxor x, c, c;
138
139#define encrypt_round(n, a, b, c, d) \
140 vbroadcastss (k+4*(2*(n)))(CTX), RK1; \
141 vbroadcastss (k+4*(2*(n)+1))(CTX), RK2; \
142 encround(a ## 1, b ## 1, c ## 1, d ## 1, RX, RY); \
143 encround(a ## 2, b ## 2, c ## 2, d ## 2, RX, RY);
144
145#define decrypt_round(n, a, b, c, d) \
146 vbroadcastss (k+4*(2*(n)))(CTX), RK1; \
147 vbroadcastss (k+4*(2*(n)+1))(CTX), RK2; \
148 decround(a ## 1, b ## 1, c ## 1, d ## 1, RX, RY); \
149 decround(a ## 2, b ## 2, c ## 2, d ## 2, RX, RY);
150
151#define encrypt_cycle(n) \
152 encrypt_round((2*n), RA, RB, RC, RD); \
153 encrypt_round(((2*n) + 1), RC, RD, RA, RB);
154
155#define decrypt_cycle(n) \
156 decrypt_round(((2*n) + 1), RC, RD, RA, RB); \
157 decrypt_round((2*n), RA, RB, RC, RD);
158
159
160#define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
161 vpunpckldq x1, x0, t0; \
162 vpunpckhdq x1, x0, t2; \
163 vpunpckldq x3, x2, t1; \
164 vpunpckhdq x3, x2, x3; \
165 \
166 vpunpcklqdq t1, t0, x0; \
167 vpunpckhqdq t1, t0, x1; \
168 vpunpcklqdq x3, t2, x2; \
169 vpunpckhqdq x3, t2, x3;
170
171#define inpack_blocks(in, x0, x1, x2, x3, wkey, t0, t1, t2) \
172 vpxor (0*4*4)(in), wkey, x0; \
173 vpxor (1*4*4)(in), wkey, x1; \
174 vpxor (2*4*4)(in), wkey, x2; \
175 vpxor (3*4*4)(in), wkey, x3; \
176 \
177 transpose_4x4(x0, x1, x2, x3, t0, t1, t2)
178
179#define outunpack_blocks(out, x0, x1, x2, x3, wkey, t0, t1, t2) \
180 transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
181 \
182 vpxor x0, wkey, x0; \
183 vmovdqu x0, (0*4*4)(out); \
184 vpxor x1, wkey, x1; \
185 vmovdqu x1, (1*4*4)(out); \
186 vpxor x2, wkey, x2; \
187 vmovdqu x2, (2*4*4)(out); \
188 vpxor x3, wkey, x3; \
189 vmovdqu x3, (3*4*4)(out);
190
191#define outunpack_xor_blocks(out, x0, x1, x2, x3, wkey, t0, t1, t2) \
192 transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
193 \
194 vpxor x0, wkey, x0; \
195 vpxor (0*4*4)(out), x0, x0; \
196 vmovdqu x0, (0*4*4)(out); \
197 vpxor x1, wkey, x1; \
198 vpxor (1*4*4)(out), x1, x1; \
199 vmovdqu x1, (1*4*4)(out); \
200 vpxor x2, wkey, x2; \
201 vpxor (2*4*4)(out), x2, x2; \
202 vmovdqu x2, (2*4*4)(out); \
203 vpxor x3, wkey, x3; \
204 vpxor (3*4*4)(out), x3, x3; \
205 vmovdqu x3, (3*4*4)(out);
206
207.align 8
208.global __twofish_enc_blk_8way
209.type __twofish_enc_blk_8way,@function;
210
211__twofish_enc_blk_8way:
212 /* input:
213 * %rdi: ctx, CTX
214 * %rsi: dst
215 * %rdx: src
216 * %rcx: bool, if true: xor output
217 */
218
219 pushq %rbx;
220 pushq %rcx;
221
222 vmovdqu w(CTX), RK1;
223
224 leaq (4*4*4)(%rdx), %rax;
225 inpack_blocks(%rdx, RA1, RB1, RC1, RD1, RK1, RX, RY, RK2);
226 inpack_blocks(%rax, RA2, RB2, RC2, RD2, RK1, RX, RY, RK2);
227
228 xorq RID1, RID1;
229 xorq RID2, RID2;
230
231 encrypt_cycle(0);
232 encrypt_cycle(1);
233 encrypt_cycle(2);
234 encrypt_cycle(3);
235 encrypt_cycle(4);
236 encrypt_cycle(5);
237 encrypt_cycle(6);
238 encrypt_cycle(7);
239
240 vmovdqu (w+4*4)(CTX), RK1;
241
242 popq %rcx;
243 popq %rbx;
244
245 leaq (4*4*4)(%rsi), %rax;
246
247 testb %cl, %cl;
248 jnz __enc_xor8;
249
250 outunpack_blocks(%rsi, RC1, RD1, RA1, RB1, RK1, RX, RY, RK2);
251 outunpack_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX, RY, RK2);
252
253 ret;
254
255__enc_xor8:
256 outunpack_xor_blocks(%rsi, RC1, RD1, RA1, RB1, RK1, RX, RY, RK2);
257 outunpack_xor_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX, RY, RK2);
258
259 ret;
260
261.align 8
262.global twofish_dec_blk_8way
263.type twofish_dec_blk_8way,@function;
264
265twofish_dec_blk_8way:
266 /* input:
267 * %rdi: ctx, CTX
268 * %rsi: dst
269 * %rdx: src
270 */
271
272 pushq %rbx;
273
274 vmovdqu (w+4*4)(CTX), RK1;
275
276 leaq (4*4*4)(%rdx), %rax;
277 inpack_blocks(%rdx, RC1, RD1, RA1, RB1, RK1, RX, RY, RK2);
278 inpack_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX, RY, RK2);
279
280 xorq RID1, RID1;
281 xorq RID2, RID2;
282
283 decrypt_cycle(7);
284 decrypt_cycle(6);
285 decrypt_cycle(5);
286 decrypt_cycle(4);
287 decrypt_cycle(3);
288 decrypt_cycle(2);
289 decrypt_cycle(1);
290 decrypt_cycle(0);
291
292 vmovdqu (w)(CTX), RK1;
293
294 popq %rbx;
295
296 leaq (4*4*4)(%rsi), %rax;
297 outunpack_blocks(%rsi, RA1, RB1, RC1, RD1, RK1, RX, RY, RK2);
298 outunpack_blocks(%rax, RA2, RB2, RC2, RD2, RK1, RX, RY, RK2);
299
300 ret;
diff --git a/arch/x86/crypto/twofish_avx_glue.c b/arch/x86/crypto/twofish_avx_glue.c
new file mode 100644
index 000000000000..782b67ddaf6a
--- /dev/null
+++ b/arch/x86/crypto/twofish_avx_glue.c
@@ -0,0 +1,624 @@
1/*
2 * Glue Code for AVX assembler version of Twofish Cipher
3 *
4 * Copyright (C) 2012 Johannes Goetzfried
5 * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
20 * USA
21 *
22 */
23
24#include <linux/module.h>
25#include <linux/hardirq.h>
26#include <linux/types.h>
27#include <linux/crypto.h>
28#include <linux/err.h>
29#include <crypto/algapi.h>
30#include <crypto/twofish.h>
31#include <crypto/cryptd.h>
32#include <crypto/b128ops.h>
33#include <crypto/ctr.h>
34#include <crypto/lrw.h>
35#include <crypto/xts.h>
36#include <asm/i387.h>
37#include <asm/xcr.h>
38#include <asm/xsave.h>
39#include <asm/crypto/twofish.h>
40#include <asm/crypto/ablk_helper.h>
41#include <asm/crypto/glue_helper.h>
42#include <crypto/scatterwalk.h>
43#include <linux/workqueue.h>
44#include <linux/spinlock.h>
45
46#define TWOFISH_PARALLEL_BLOCKS 8
47
48static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
49 const u8 *src)
50{
51 __twofish_enc_blk_3way(ctx, dst, src, false);
52}
53
54/* 8-way parallel cipher functions */
55asmlinkage void __twofish_enc_blk_8way(struct twofish_ctx *ctx, u8 *dst,
56 const u8 *src, bool xor);
57asmlinkage void twofish_dec_blk_8way(struct twofish_ctx *ctx, u8 *dst,
58 const u8 *src);
59
60static inline void twofish_enc_blk_xway(struct twofish_ctx *ctx, u8 *dst,
61 const u8 *src)
62{
63 __twofish_enc_blk_8way(ctx, dst, src, false);
64}
65
66static inline void twofish_enc_blk_xway_xor(struct twofish_ctx *ctx, u8 *dst,
67 const u8 *src)
68{
69 __twofish_enc_blk_8way(ctx, dst, src, true);
70}
71
72static inline void twofish_dec_blk_xway(struct twofish_ctx *ctx, u8 *dst,
73 const u8 *src)
74{
75 twofish_dec_blk_8way(ctx, dst, src);
76}
77
78static void twofish_dec_blk_cbc_xway(void *ctx, u128 *dst, const u128 *src)
79{
80 u128 ivs[TWOFISH_PARALLEL_BLOCKS - 1];
81 unsigned int j;
82
83 for (j = 0; j < TWOFISH_PARALLEL_BLOCKS - 1; j++)
84 ivs[j] = src[j];
85
86 twofish_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
87
88 for (j = 0; j < TWOFISH_PARALLEL_BLOCKS - 1; j++)
89 u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);
90}
91
92static void twofish_enc_blk_ctr_xway(void *ctx, u128 *dst, const u128 *src,
93 u128 *iv)
94{
95 be128 ctrblks[TWOFISH_PARALLEL_BLOCKS];
96 unsigned int i;
97
98 for (i = 0; i < TWOFISH_PARALLEL_BLOCKS; i++) {
99 if (dst != src)
100 dst[i] = src[i];
101
102 u128_to_be128(&ctrblks[i], iv);
103 u128_inc(iv);
104 }
105
106 twofish_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks);
107}
108
109static const struct common_glue_ctx twofish_enc = {
110 .num_funcs = 3,
111 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
112
113 .funcs = { {
114 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
115 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_xway) }
116 }, {
117 .num_blocks = 3,
118 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
119 }, {
120 .num_blocks = 1,
121 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
122 } }
123};
124
125static const struct common_glue_ctx twofish_ctr = {
126 .num_funcs = 3,
127 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
128
129 .funcs = { {
130 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
131 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_xway) }
132 }, {
133 .num_blocks = 3,
134 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) }
135 }, {
136 .num_blocks = 1,
137 .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) }
138 } }
139};
140
141static const struct common_glue_ctx twofish_dec = {
142 .num_funcs = 3,
143 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
144
145 .funcs = { {
146 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
147 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_xway) }
148 }, {
149 .num_blocks = 3,
150 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
151 }, {
152 .num_blocks = 1,
153 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
154 } }
155};
156
157static const struct common_glue_ctx twofish_dec_cbc = {
158 .num_funcs = 3,
159 .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
160
161 .funcs = { {
162 .num_blocks = TWOFISH_PARALLEL_BLOCKS,
163 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_xway) }
164 }, {
165 .num_blocks = 3,
166 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
167 }, {
168 .num_blocks = 1,
169 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
170 } }
171};
172
173static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
174 struct scatterlist *src, unsigned int nbytes)
175{
176 return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes);
177}
178
179static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
180 struct scatterlist *src, unsigned int nbytes)
181{
182 return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes);
183}
184
185static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
186 struct scatterlist *src, unsigned int nbytes)
187{
188 return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc,
189 dst, src, nbytes);
190}
191
192static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
193 struct scatterlist *src, unsigned int nbytes)
194{
195 return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src,
196 nbytes);
197}
198
199static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
200 struct scatterlist *src, unsigned int nbytes)
201{
202 return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes);
203}
204
205static inline bool twofish_fpu_begin(bool fpu_enabled, unsigned int nbytes)
206{
207 return glue_fpu_begin(TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS, NULL,
208 fpu_enabled, nbytes);
209}
210
211static inline void twofish_fpu_end(bool fpu_enabled)
212{
213 glue_fpu_end(fpu_enabled);
214}
215
216struct crypt_priv {
217 struct twofish_ctx *ctx;
218 bool fpu_enabled;
219};
220
221static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
222{
223 const unsigned int bsize = TF_BLOCK_SIZE;
224 struct crypt_priv *ctx = priv;
225 int i;
226
227 ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
228
229 if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
230 twofish_enc_blk_xway(ctx->ctx, srcdst, srcdst);
231 return;
232 }
233
234 for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
235 twofish_enc_blk_3way(ctx->ctx, srcdst, srcdst);
236
237 nbytes %= bsize * 3;
238
239 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
240 twofish_enc_blk(ctx->ctx, srcdst, srcdst);
241}
242
243static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
244{
245 const unsigned int bsize = TF_BLOCK_SIZE;
246 struct crypt_priv *ctx = priv;
247 int i;
248
249 ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
250
251 if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
252 twofish_dec_blk_xway(ctx->ctx, srcdst, srcdst);
253 return;
254 }
255
256 for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
257 twofish_dec_blk_3way(ctx->ctx, srcdst, srcdst);
258
259 nbytes %= bsize * 3;
260
261 for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
262 twofish_dec_blk(ctx->ctx, srcdst, srcdst);
263}
264
265static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
266 struct scatterlist *src, unsigned int nbytes)
267{
268 struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
269 be128 buf[TWOFISH_PARALLEL_BLOCKS];
270 struct crypt_priv crypt_ctx = {
271 .ctx = &ctx->twofish_ctx,
272 .fpu_enabled = false,
273 };
274 struct lrw_crypt_req req = {
275 .tbuf = buf,
276 .tbuflen = sizeof(buf),
277
278 .table_ctx = &ctx->lrw_table,
279 .crypt_ctx = &crypt_ctx,
280 .crypt_fn = encrypt_callback,
281 };
282 int ret;
283
284 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
285 ret = lrw_crypt(desc, dst, src, nbytes, &req);
286 twofish_fpu_end(crypt_ctx.fpu_enabled);
287
288 return ret;
289}
290
291static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
292 struct scatterlist *src, unsigned int nbytes)
293{
294 struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
295 be128 buf[TWOFISH_PARALLEL_BLOCKS];
296 struct crypt_priv crypt_ctx = {
297 .ctx = &ctx->twofish_ctx,
298 .fpu_enabled = false,
299 };
300 struct lrw_crypt_req req = {
301 .tbuf = buf,
302 .tbuflen = sizeof(buf),
303
304 .table_ctx = &ctx->lrw_table,
305 .crypt_ctx = &crypt_ctx,
306 .crypt_fn = decrypt_callback,
307 };
308 int ret;
309
310 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
311 ret = lrw_crypt(desc, dst, src, nbytes, &req);
312 twofish_fpu_end(crypt_ctx.fpu_enabled);
313
314 return ret;
315}
316
317static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
318 struct scatterlist *src, unsigned int nbytes)
319{
320 struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
321 be128 buf[TWOFISH_PARALLEL_BLOCKS];
322 struct crypt_priv crypt_ctx = {
323 .ctx = &ctx->crypt_ctx,
324 .fpu_enabled = false,
325 };
326 struct xts_crypt_req req = {
327 .tbuf = buf,
328 .tbuflen = sizeof(buf),
329
330 .tweak_ctx = &ctx->tweak_ctx,
331 .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk),
332 .crypt_ctx = &crypt_ctx,
333 .crypt_fn = encrypt_callback,
334 };
335 int ret;
336
337 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
338 ret = xts_crypt(desc, dst, src, nbytes, &req);
339 twofish_fpu_end(crypt_ctx.fpu_enabled);
340
341 return ret;
342}
343
344static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
345 struct scatterlist *src, unsigned int nbytes)
346{
347 struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
348 be128 buf[TWOFISH_PARALLEL_BLOCKS];
349 struct crypt_priv crypt_ctx = {
350 .ctx = &ctx->crypt_ctx,
351 .fpu_enabled = false,
352 };
353 struct xts_crypt_req req = {
354 .tbuf = buf,
355 .tbuflen = sizeof(buf),
356
357 .tweak_ctx = &ctx->tweak_ctx,
358 .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk),
359 .crypt_ctx = &crypt_ctx,
360 .crypt_fn = decrypt_callback,
361 };
362 int ret;
363
364 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
365 ret = xts_crypt(desc, dst, src, nbytes, &req);
366 twofish_fpu_end(crypt_ctx.fpu_enabled);
367
368 return ret;
369}
370
371static struct crypto_alg twofish_algs[10] = { {
372 .cra_name = "__ecb-twofish-avx",
373 .cra_driver_name = "__driver-ecb-twofish-avx",
374 .cra_priority = 0,
375 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
376 .cra_blocksize = TF_BLOCK_SIZE,
377 .cra_ctxsize = sizeof(struct twofish_ctx),
378 .cra_alignmask = 0,
379 .cra_type = &crypto_blkcipher_type,
380 .cra_module = THIS_MODULE,
381 .cra_list = LIST_HEAD_INIT(twofish_algs[0].cra_list),
382 .cra_u = {
383 .blkcipher = {
384 .min_keysize = TF_MIN_KEY_SIZE,
385 .max_keysize = TF_MAX_KEY_SIZE,
386 .setkey = twofish_setkey,
387 .encrypt = ecb_encrypt,
388 .decrypt = ecb_decrypt,
389 },
390 },
391}, {
392 .cra_name = "__cbc-twofish-avx",
393 .cra_driver_name = "__driver-cbc-twofish-avx",
394 .cra_priority = 0,
395 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
396 .cra_blocksize = TF_BLOCK_SIZE,
397 .cra_ctxsize = sizeof(struct twofish_ctx),
398 .cra_alignmask = 0,
399 .cra_type = &crypto_blkcipher_type,
400 .cra_module = THIS_MODULE,
401 .cra_list = LIST_HEAD_INIT(twofish_algs[1].cra_list),
402 .cra_u = {
403 .blkcipher = {
404 .min_keysize = TF_MIN_KEY_SIZE,
405 .max_keysize = TF_MAX_KEY_SIZE,
406 .setkey = twofish_setkey,
407 .encrypt = cbc_encrypt,
408 .decrypt = cbc_decrypt,
409 },
410 },
411}, {
412 .cra_name = "__ctr-twofish-avx",
413 .cra_driver_name = "__driver-ctr-twofish-avx",
414 .cra_priority = 0,
415 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
416 .cra_blocksize = 1,
417 .cra_ctxsize = sizeof(struct twofish_ctx),
418 .cra_alignmask = 0,
419 .cra_type = &crypto_blkcipher_type,
420 .cra_module = THIS_MODULE,
421 .cra_list = LIST_HEAD_INIT(twofish_algs[2].cra_list),
422 .cra_u = {
423 .blkcipher = {
424 .min_keysize = TF_MIN_KEY_SIZE,
425 .max_keysize = TF_MAX_KEY_SIZE,
426 .ivsize = TF_BLOCK_SIZE,
427 .setkey = twofish_setkey,
428 .encrypt = ctr_crypt,
429 .decrypt = ctr_crypt,
430 },
431 },
432}, {
433 .cra_name = "__lrw-twofish-avx",
434 .cra_driver_name = "__driver-lrw-twofish-avx",
435 .cra_priority = 0,
436 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
437 .cra_blocksize = TF_BLOCK_SIZE,
438 .cra_ctxsize = sizeof(struct twofish_lrw_ctx),
439 .cra_alignmask = 0,
440 .cra_type = &crypto_blkcipher_type,
441 .cra_module = THIS_MODULE,
442 .cra_list = LIST_HEAD_INIT(twofish_algs[3].cra_list),
443 .cra_exit = lrw_twofish_exit_tfm,
444 .cra_u = {
445 .blkcipher = {
446 .min_keysize = TF_MIN_KEY_SIZE +
447 TF_BLOCK_SIZE,
448 .max_keysize = TF_MAX_KEY_SIZE +
449 TF_BLOCK_SIZE,
450 .ivsize = TF_BLOCK_SIZE,
451 .setkey = lrw_twofish_setkey,
452 .encrypt = lrw_encrypt,
453 .decrypt = lrw_decrypt,
454 },
455 },
456}, {
457 .cra_name = "__xts-twofish-avx",
458 .cra_driver_name = "__driver-xts-twofish-avx",
459 .cra_priority = 0,
460 .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
461 .cra_blocksize = TF_BLOCK_SIZE,
462 .cra_ctxsize = sizeof(struct twofish_xts_ctx),
463 .cra_alignmask = 0,
464 .cra_type = &crypto_blkcipher_type,
465 .cra_module = THIS_MODULE,
466 .cra_list = LIST_HEAD_INIT(twofish_algs[4].cra_list),
467 .cra_u = {
468 .blkcipher = {
469 .min_keysize = TF_MIN_KEY_SIZE * 2,
470 .max_keysize = TF_MAX_KEY_SIZE * 2,
471 .ivsize = TF_BLOCK_SIZE,
472 .setkey = xts_twofish_setkey,
473 .encrypt = xts_encrypt,
474 .decrypt = xts_decrypt,
475 },
476 },
477}, {
478 .cra_name = "ecb(twofish)",
479 .cra_driver_name = "ecb-twofish-avx",
480 .cra_priority = 400,
481 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
482 .cra_blocksize = TF_BLOCK_SIZE,
483 .cra_ctxsize = sizeof(struct async_helper_ctx),
484 .cra_alignmask = 0,
485 .cra_type = &crypto_ablkcipher_type,
486 .cra_module = THIS_MODULE,
487 .cra_list = LIST_HEAD_INIT(twofish_algs[5].cra_list),
488 .cra_init = ablk_init,
489 .cra_exit = ablk_exit,
490 .cra_u = {
491 .ablkcipher = {
492 .min_keysize = TF_MIN_KEY_SIZE,
493 .max_keysize = TF_MAX_KEY_SIZE,
494 .setkey = ablk_set_key,
495 .encrypt = ablk_encrypt,
496 .decrypt = ablk_decrypt,
497 },
498 },
499}, {
500 .cra_name = "cbc(twofish)",
501 .cra_driver_name = "cbc-twofish-avx",
502 .cra_priority = 400,
503 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
504 .cra_blocksize = TF_BLOCK_SIZE,
505 .cra_ctxsize = sizeof(struct async_helper_ctx),
506 .cra_alignmask = 0,
507 .cra_type = &crypto_ablkcipher_type,
508 .cra_module = THIS_MODULE,
509 .cra_list = LIST_HEAD_INIT(twofish_algs[6].cra_list),
510 .cra_init = ablk_init,
511 .cra_exit = ablk_exit,
512 .cra_u = {
513 .ablkcipher = {
514 .min_keysize = TF_MIN_KEY_SIZE,
515 .max_keysize = TF_MAX_KEY_SIZE,
516 .ivsize = TF_BLOCK_SIZE,
517 .setkey = ablk_set_key,
518 .encrypt = __ablk_encrypt,
519 .decrypt = ablk_decrypt,
520 },
521 },
522}, {
523 .cra_name = "ctr(twofish)",
524 .cra_driver_name = "ctr-twofish-avx",
525 .cra_priority = 400,
526 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
527 .cra_blocksize = 1,
528 .cra_ctxsize = sizeof(struct async_helper_ctx),
529 .cra_alignmask = 0,
530 .cra_type = &crypto_ablkcipher_type,
531 .cra_module = THIS_MODULE,
532 .cra_list = LIST_HEAD_INIT(twofish_algs[7].cra_list),
533 .cra_init = ablk_init,
534 .cra_exit = ablk_exit,
535 .cra_u = {
536 .ablkcipher = {
537 .min_keysize = TF_MIN_KEY_SIZE,
538 .max_keysize = TF_MAX_KEY_SIZE,
539 .ivsize = TF_BLOCK_SIZE,
540 .setkey = ablk_set_key,
541 .encrypt = ablk_encrypt,
542 .decrypt = ablk_encrypt,
543 .geniv = "chainiv",
544 },
545 },
546}, {
547 .cra_name = "lrw(twofish)",
548 .cra_driver_name = "lrw-twofish-avx",
549 .cra_priority = 400,
550 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
551 .cra_blocksize = TF_BLOCK_SIZE,
552 .cra_ctxsize = sizeof(struct async_helper_ctx),
553 .cra_alignmask = 0,
554 .cra_type = &crypto_ablkcipher_type,
555 .cra_module = THIS_MODULE,
556 .cra_list = LIST_HEAD_INIT(twofish_algs[8].cra_list),
557 .cra_init = ablk_init,
558 .cra_exit = ablk_exit,
559 .cra_u = {
560 .ablkcipher = {
561 .min_keysize = TF_MIN_KEY_SIZE +
562 TF_BLOCK_SIZE,
563 .max_keysize = TF_MAX_KEY_SIZE +
564 TF_BLOCK_SIZE,
565 .ivsize = TF_BLOCK_SIZE,
566 .setkey = ablk_set_key,
567 .encrypt = ablk_encrypt,
568 .decrypt = ablk_decrypt,
569 },
570 },
571}, {
572 .cra_name = "xts(twofish)",
573 .cra_driver_name = "xts-twofish-avx",
574 .cra_priority = 400,
575 .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
576 .cra_blocksize = TF_BLOCK_SIZE,
577 .cra_ctxsize = sizeof(struct async_helper_ctx),
578 .cra_alignmask = 0,
579 .cra_type = &crypto_ablkcipher_type,
580 .cra_module = THIS_MODULE,
581 .cra_list = LIST_HEAD_INIT(twofish_algs[9].cra_list),
582 .cra_init = ablk_init,
583 .cra_exit = ablk_exit,
584 .cra_u = {
585 .ablkcipher = {
586 .min_keysize = TF_MIN_KEY_SIZE * 2,
587 .max_keysize = TF_MAX_KEY_SIZE * 2,
588 .ivsize = TF_BLOCK_SIZE,
589 .setkey = ablk_set_key,
590 .encrypt = ablk_encrypt,
591 .decrypt = ablk_decrypt,
592 },
593 },
594} };
595
596static int __init twofish_init(void)
597{
598 u64 xcr0;
599
600 if (!cpu_has_avx || !cpu_has_osxsave) {
601 printk(KERN_INFO "AVX instructions are not detected.\n");
602 return -ENODEV;
603 }
604
605 xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
606 if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
607 printk(KERN_INFO "AVX detected but unusable.\n");
608 return -ENODEV;
609 }
610
611 return crypto_register_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
612}
613
614static void __exit twofish_exit(void)
615{
616 crypto_unregister_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
617}
618
619module_init(twofish_init);
620module_exit(twofish_exit);
621
622MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
623MODULE_LICENSE("GPL");
624MODULE_ALIAS("twofish");
diff --git a/arch/x86/crypto/twofish_glue_3way.c b/arch/x86/crypto/twofish_glue_3way.c
index 922ab24cce31..15f9347316c8 100644
--- a/arch/x86/crypto/twofish_glue_3way.c
+++ b/arch/x86/crypto/twofish_glue_3way.c
@@ -3,11 +3,6 @@
3 * 3 *
4 * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> 4 * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
5 * 5 *
6 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
7 * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
8 * CTR part based on code (crypto/ctr.c) by:
9 * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
10 *
11 * This program is free software; you can redistribute it and/or modify 6 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by 7 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or 8 * the Free Software Foundation; either version 2 of the License, or
@@ -33,20 +28,13 @@
33#include <crypto/algapi.h> 28#include <crypto/algapi.h>
34#include <crypto/twofish.h> 29#include <crypto/twofish.h>
35#include <crypto/b128ops.h> 30#include <crypto/b128ops.h>
31#include <asm/crypto/twofish.h>
32#include <asm/crypto/glue_helper.h>
36#include <crypto/lrw.h> 33#include <crypto/lrw.h>
37#include <crypto/xts.h> 34#include <crypto/xts.h>
38 35
39/* regular block cipher functions from twofish_x86_64 module */ 36EXPORT_SYMBOL_GPL(__twofish_enc_blk_3way);
40asmlinkage void twofish_enc_blk(struct twofish_ctx *ctx, u8 *dst, 37EXPORT_SYMBOL_GPL(twofish_dec_blk_3way);
41 const u8 *src);
42asmlinkage void twofish_dec_blk(struct twofish_ctx *ctx, u8 *dst,
43 const u8 *src);
44
45/* 3-way parallel cipher functions */
46asmlinkage void __twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
47 const u8 *src, bool xor);
48asmlinkage void twofish_dec_blk_3way(struct twofish_ctx *ctx, u8 *dst,
49 const u8 *src);
50 38
51static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst, 39static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
52 const u8 *src) 40 const u8 *src)
@@ -60,311 +48,139 @@ static inline void twofish_enc_blk_xor_3way(struct twofish_ctx *ctx, u8 *dst,
60 __twofish_enc_blk_3way(ctx, dst, src, true); 48 __twofish_enc_blk_3way(ctx, dst, src, true);
61} 49}
62 50
63static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk, 51void twofish_dec_blk_cbc_3way(void *ctx, u128 *dst, const u128 *src)
64 void (*fn)(struct twofish_ctx *, u8 *, const u8 *),
65 void (*fn_3way)(struct twofish_ctx *, u8 *, const u8 *))
66{
67 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
68 unsigned int bsize = TF_BLOCK_SIZE;
69 unsigned int nbytes;
70 int err;
71
72 err = blkcipher_walk_virt(desc, walk);
73
74 while ((nbytes = walk->nbytes)) {
75 u8 *wsrc = walk->src.virt.addr;
76 u8 *wdst = walk->dst.virt.addr;
77
78 /* Process three block batch */
79 if (nbytes >= bsize * 3) {
80 do {
81 fn_3way(ctx, wdst, wsrc);
82
83 wsrc += bsize * 3;
84 wdst += bsize * 3;
85 nbytes -= bsize * 3;
86 } while (nbytes >= bsize * 3);
87
88 if (nbytes < bsize)
89 goto done;
90 }
91
92 /* Handle leftovers */
93 do {
94 fn(ctx, wdst, wsrc);
95
96 wsrc += bsize;
97 wdst += bsize;
98 nbytes -= bsize;
99 } while (nbytes >= bsize);
100
101done:
102 err = blkcipher_walk_done(desc, walk, nbytes);
103 }
104
105 return err;
106}
107
108static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
109 struct scatterlist *src, unsigned int nbytes)
110{ 52{
111 struct blkcipher_walk walk; 53 u128 ivs[2];
112 54
113 blkcipher_walk_init(&walk, dst, src, nbytes); 55 ivs[0] = src[0];
114 return ecb_crypt(desc, &walk, twofish_enc_blk, twofish_enc_blk_3way); 56 ivs[1] = src[1];
115}
116 57
117static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 58 twofish_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src);
118 struct scatterlist *src, unsigned int nbytes)
119{
120 struct blkcipher_walk walk;
121 59
122 blkcipher_walk_init(&walk, dst, src, nbytes); 60 u128_xor(&dst[1], &dst[1], &ivs[0]);
123 return ecb_crypt(desc, &walk, twofish_dec_blk, twofish_dec_blk_3way); 61 u128_xor(&dst[2], &dst[2], &ivs[1]);
124} 62}
63EXPORT_SYMBOL_GPL(twofish_dec_blk_cbc_3way);
125 64
126static unsigned int __cbc_encrypt(struct blkcipher_desc *desc, 65void twofish_enc_blk_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv)
127 struct blkcipher_walk *walk)
128{
129 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
130 unsigned int bsize = TF_BLOCK_SIZE;
131 unsigned int nbytes = walk->nbytes;
132 u128 *src = (u128 *)walk->src.virt.addr;
133 u128 *dst = (u128 *)walk->dst.virt.addr;
134 u128 *iv = (u128 *)walk->iv;
135
136 do {
137 u128_xor(dst, src, iv);
138 twofish_enc_blk(ctx, (u8 *)dst, (u8 *)dst);
139 iv = dst;
140
141 src += 1;
142 dst += 1;
143 nbytes -= bsize;
144 } while (nbytes >= bsize);
145
146 u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv);
147 return nbytes;
148}
149
150static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
151 struct scatterlist *src, unsigned int nbytes)
152{ 66{
153 struct blkcipher_walk walk; 67 be128 ctrblk;
154 int err;
155 68
156 blkcipher_walk_init(&walk, dst, src, nbytes); 69 if (dst != src)
157 err = blkcipher_walk_virt(desc, &walk); 70 *dst = *src;
158 71
159 while ((nbytes = walk.nbytes)) { 72 u128_to_be128(&ctrblk, iv);
160 nbytes = __cbc_encrypt(desc, &walk); 73 u128_inc(iv);
161 err = blkcipher_walk_done(desc, &walk, nbytes);
162 }
163 74
164 return err; 75 twofish_enc_blk(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
76 u128_xor(dst, dst, (u128 *)&ctrblk);
165} 77}
78EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr);
166 79
167static unsigned int __cbc_decrypt(struct blkcipher_desc *desc, 80void twofish_enc_blk_ctr_3way(void *ctx, u128 *dst, const u128 *src,
168 struct blkcipher_walk *walk) 81 u128 *iv)
169{ 82{
170 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 83 be128 ctrblks[3];
171 unsigned int bsize = TF_BLOCK_SIZE;
172 unsigned int nbytes = walk->nbytes;
173 u128 *src = (u128 *)walk->src.virt.addr;
174 u128 *dst = (u128 *)walk->dst.virt.addr;
175 u128 ivs[3 - 1];
176 u128 last_iv;
177
178 /* Start of the last block. */
179 src += nbytes / bsize - 1;
180 dst += nbytes / bsize - 1;
181
182 last_iv = *src;
183
184 /* Process three block batch */
185 if (nbytes >= bsize * 3) {
186 do {
187 nbytes -= bsize * (3 - 1);
188 src -= 3 - 1;
189 dst -= 3 - 1;
190
191 ivs[0] = src[0];
192 ivs[1] = src[1];
193
194 twofish_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src);
195
196 u128_xor(dst + 1, dst + 1, ivs + 0);
197 u128_xor(dst + 2, dst + 2, ivs + 1);
198
199 nbytes -= bsize;
200 if (nbytes < bsize)
201 goto done;
202
203 u128_xor(dst, dst, src - 1);
204 src -= 1;
205 dst -= 1;
206 } while (nbytes >= bsize * 3);
207
208 if (nbytes < bsize)
209 goto done;
210 }
211
212 /* Handle leftovers */
213 for (;;) {
214 twofish_dec_blk(ctx, (u8 *)dst, (u8 *)src);
215
216 nbytes -= bsize;
217 if (nbytes < bsize)
218 break;
219 84
220 u128_xor(dst, dst, src - 1); 85 if (dst != src) {
221 src -= 1; 86 dst[0] = src[0];
222 dst -= 1; 87 dst[1] = src[1];
88 dst[2] = src[2];
223 } 89 }
224 90
225done: 91 u128_to_be128(&ctrblks[0], iv);
226 u128_xor(dst, dst, (u128 *)walk->iv); 92 u128_inc(iv);
227 *(u128 *)walk->iv = last_iv; 93 u128_to_be128(&ctrblks[1], iv);
94 u128_inc(iv);
95 u128_to_be128(&ctrblks[2], iv);
96 u128_inc(iv);
228 97
229 return nbytes; 98 twofish_enc_blk_xor_3way(ctx, (u8 *)dst, (u8 *)ctrblks);
230} 99}
100EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr_3way);
101
102static const struct common_glue_ctx twofish_enc = {
103 .num_funcs = 2,
104 .fpu_blocks_limit = -1,
105
106 .funcs = { {
107 .num_blocks = 3,
108 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
109 }, {
110 .num_blocks = 1,
111 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
112 } }
113};
231 114
232static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 115static const struct common_glue_ctx twofish_ctr = {
233 struct scatterlist *src, unsigned int nbytes) 116 .num_funcs = 2,
234{ 117 .fpu_blocks_limit = -1,
235 struct blkcipher_walk walk; 118
236 int err; 119 .funcs = { {
237 120 .num_blocks = 3,
238 blkcipher_walk_init(&walk, dst, src, nbytes); 121 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_ctr_3way) }
239 err = blkcipher_walk_virt(desc, &walk); 122 }, {
123 .num_blocks = 1,
124 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_ctr) }
125 } }
126};
240 127
241 while ((nbytes = walk.nbytes)) { 128static const struct common_glue_ctx twofish_dec = {
242 nbytes = __cbc_decrypt(desc, &walk); 129 .num_funcs = 2,
243 err = blkcipher_walk_done(desc, &walk, nbytes); 130 .fpu_blocks_limit = -1,
244 } 131
132 .funcs = { {
133 .num_blocks = 3,
134 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
135 }, {
136 .num_blocks = 1,
137 .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
138 } }
139};
245 140
246 return err; 141static const struct common_glue_ctx twofish_dec_cbc = {
247} 142 .num_funcs = 2,
143 .fpu_blocks_limit = -1,
144
145 .funcs = { {
146 .num_blocks = 3,
147 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
148 }, {
149 .num_blocks = 1,
150 .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
151 } }
152};
248 153
249static inline void u128_to_be128(be128 *dst, const u128 *src) 154static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
155 struct scatterlist *src, unsigned int nbytes)
250{ 156{
251 dst->a = cpu_to_be64(src->a); 157 return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes);
252 dst->b = cpu_to_be64(src->b);
253} 158}
254 159
255static inline void be128_to_u128(u128 *dst, const be128 *src) 160static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
161 struct scatterlist *src, unsigned int nbytes)
256{ 162{
257 dst->a = be64_to_cpu(src->a); 163 return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes);
258 dst->b = be64_to_cpu(src->b);
259} 164}
260 165
261static inline void u128_inc(u128 *i) 166static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
167 struct scatterlist *src, unsigned int nbytes)
262{ 168{
263 i->b++; 169 return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc,
264 if (!i->b) 170 dst, src, nbytes);
265 i->a++;
266} 171}
267 172
268static void ctr_crypt_final(struct blkcipher_desc *desc, 173static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
269 struct blkcipher_walk *walk) 174 struct scatterlist *src, unsigned int nbytes)
270{ 175{
271 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); 176 return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src,
272 u8 *ctrblk = walk->iv; 177 nbytes);
273 u8 keystream[TF_BLOCK_SIZE];
274 u8 *src = walk->src.virt.addr;
275 u8 *dst = walk->dst.virt.addr;
276 unsigned int nbytes = walk->nbytes;
277
278 twofish_enc_blk(ctx, keystream, ctrblk);
279 crypto_xor(keystream, src, nbytes);
280 memcpy(dst, keystream, nbytes);
281
282 crypto_inc(ctrblk, TF_BLOCK_SIZE);
283}
284
285static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
286 struct blkcipher_walk *walk)
287{
288 struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
289 unsigned int bsize = TF_BLOCK_SIZE;
290 unsigned int nbytes = walk->nbytes;
291 u128 *src = (u128 *)walk->src.virt.addr;
292 u128 *dst = (u128 *)walk->dst.virt.addr;
293 u128 ctrblk;
294 be128 ctrblocks[3];
295
296 be128_to_u128(&ctrblk, (be128 *)walk->iv);
297
298 /* Process three block batch */
299 if (nbytes >= bsize * 3) {
300 do {
301 if (dst != src) {
302 dst[0] = src[0];
303 dst[1] = src[1];
304 dst[2] = src[2];
305 }
306
307 /* create ctrblks for parallel encrypt */
308 u128_to_be128(&ctrblocks[0], &ctrblk);
309 u128_inc(&ctrblk);
310 u128_to_be128(&ctrblocks[1], &ctrblk);
311 u128_inc(&ctrblk);
312 u128_to_be128(&ctrblocks[2], &ctrblk);
313 u128_inc(&ctrblk);
314
315 twofish_enc_blk_xor_3way(ctx, (u8 *)dst,
316 (u8 *)ctrblocks);
317
318 src += 3;
319 dst += 3;
320 nbytes -= bsize * 3;
321 } while (nbytes >= bsize * 3);
322
323 if (nbytes < bsize)
324 goto done;
325 }
326
327 /* Handle leftovers */
328 do {
329 if (dst != src)
330 *dst = *src;
331
332 u128_to_be128(&ctrblocks[0], &ctrblk);
333 u128_inc(&ctrblk);
334
335 twofish_enc_blk(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks);
336 u128_xor(dst, dst, (u128 *)ctrblocks);
337
338 src += 1;
339 dst += 1;
340 nbytes -= bsize;
341 } while (nbytes >= bsize);
342
343done:
344 u128_to_be128((be128 *)walk->iv, &ctrblk);
345 return nbytes;
346} 178}
347 179
348static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, 180static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
349 struct scatterlist *src, unsigned int nbytes) 181 struct scatterlist *src, unsigned int nbytes)
350{ 182{
351 struct blkcipher_walk walk; 183 return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes);
352 int err;
353
354 blkcipher_walk_init(&walk, dst, src, nbytes);
355 err = blkcipher_walk_virt_block(desc, &walk, TF_BLOCK_SIZE);
356
357 while ((nbytes = walk.nbytes) >= TF_BLOCK_SIZE) {
358 nbytes = __ctr_crypt(desc, &walk);
359 err = blkcipher_walk_done(desc, &walk, nbytes);
360 }
361
362 if (walk.nbytes) {
363 ctr_crypt_final(desc, &walk);
364 err = blkcipher_walk_done(desc, &walk, 0);
365 }
366
367 return err;
368} 184}
369 185
370static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) 186static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
@@ -397,13 +213,8 @@ static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
397 twofish_dec_blk(ctx, srcdst, srcdst); 213 twofish_dec_blk(ctx, srcdst, srcdst);
398} 214}
399 215
400struct twofish_lrw_ctx { 216int lrw_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
401 struct lrw_table_ctx lrw_table; 217 unsigned int keylen)
402 struct twofish_ctx twofish_ctx;
403};
404
405static int lrw_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
406 unsigned int keylen)
407{ 218{
408 struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm); 219 struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
409 int err; 220 int err;
@@ -415,6 +226,7 @@ static int lrw_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
415 226
416 return lrw_init_table(&ctx->lrw_table, key + keylen - TF_BLOCK_SIZE); 227 return lrw_init_table(&ctx->lrw_table, key + keylen - TF_BLOCK_SIZE);
417} 228}
229EXPORT_SYMBOL_GPL(lrw_twofish_setkey);
418 230
419static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 231static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
420 struct scatterlist *src, unsigned int nbytes) 232 struct scatterlist *src, unsigned int nbytes)
@@ -450,20 +262,16 @@ static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
450 return lrw_crypt(desc, dst, src, nbytes, &req); 262 return lrw_crypt(desc, dst, src, nbytes, &req);
451} 263}
452 264
453static void lrw_exit_tfm(struct crypto_tfm *tfm) 265void lrw_twofish_exit_tfm(struct crypto_tfm *tfm)
454{ 266{
455 struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm); 267 struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
456 268
457 lrw_free_table(&ctx->lrw_table); 269 lrw_free_table(&ctx->lrw_table);
458} 270}
271EXPORT_SYMBOL_GPL(lrw_twofish_exit_tfm);
459 272
460struct twofish_xts_ctx { 273int xts_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
461 struct twofish_ctx tweak_ctx; 274 unsigned int keylen)
462 struct twofish_ctx crypt_ctx;
463};
464
465static int xts_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
466 unsigned int keylen)
467{ 275{
468 struct twofish_xts_ctx *ctx = crypto_tfm_ctx(tfm); 276 struct twofish_xts_ctx *ctx = crypto_tfm_ctx(tfm);
469 u32 *flags = &tfm->crt_flags; 277 u32 *flags = &tfm->crt_flags;
@@ -486,6 +294,7 @@ static int xts_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
486 return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2, 294 return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
487 flags); 295 flags);
488} 296}
297EXPORT_SYMBOL_GPL(xts_twofish_setkey);
489 298
490static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, 299static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
491 struct scatterlist *src, unsigned int nbytes) 300 struct scatterlist *src, unsigned int nbytes)
@@ -596,7 +405,7 @@ static struct crypto_alg tf_algs[5] = { {
596 .cra_type = &crypto_blkcipher_type, 405 .cra_type = &crypto_blkcipher_type,
597 .cra_module = THIS_MODULE, 406 .cra_module = THIS_MODULE,
598 .cra_list = LIST_HEAD_INIT(tf_algs[3].cra_list), 407 .cra_list = LIST_HEAD_INIT(tf_algs[3].cra_list),
599 .cra_exit = lrw_exit_tfm, 408 .cra_exit = lrw_twofish_exit_tfm,
600 .cra_u = { 409 .cra_u = {
601 .blkcipher = { 410 .blkcipher = {
602 .min_keysize = TF_MIN_KEY_SIZE + TF_BLOCK_SIZE, 411 .min_keysize = TF_MIN_KEY_SIZE + TF_BLOCK_SIZE,
diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h
index 3ea51a84a0e4..f34261296ffb 100644
--- a/arch/x86/include/asm/apic.h
+++ b/arch/x86/include/asm/apic.h
@@ -546,7 +546,7 @@ static inline const struct cpumask *online_target_cpus(void)
546 return cpu_online_mask; 546 return cpu_online_mask;
547} 547}
548 548
549DECLARE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid); 549DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid);
550 550
551 551
552static inline unsigned int read_apic_id(void) 552static inline unsigned int read_apic_id(void)
diff --git a/arch/x86/include/asm/bootparam.h b/arch/x86/include/asm/bootparam.h
index eb45aa6b1f27..2ad874cb661c 100644
--- a/arch/x86/include/asm/bootparam.h
+++ b/arch/x86/include/asm/bootparam.h
@@ -66,6 +66,7 @@ struct setup_header {
66 __u64 setup_data; 66 __u64 setup_data;
67 __u64 pref_address; 67 __u64 pref_address;
68 __u32 init_size; 68 __u32 init_size;
69 __u32 handover_offset;
69} __attribute__((packed)); 70} __attribute__((packed));
70 71
71struct sys_desc_table { 72struct sys_desc_table {
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index f91e80f4f180..6b7ee5ff6820 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -207,6 +207,8 @@
207#define X86_FEATURE_ERMS (9*32+ 9) /* Enhanced REP MOVSB/STOSB */ 207#define X86_FEATURE_ERMS (9*32+ 9) /* Enhanced REP MOVSB/STOSB */
208#define X86_FEATURE_INVPCID (9*32+10) /* Invalidate Processor Context ID */ 208#define X86_FEATURE_INVPCID (9*32+10) /* Invalidate Processor Context ID */
209#define X86_FEATURE_RTM (9*32+11) /* Restricted Transactional Memory */ 209#define X86_FEATURE_RTM (9*32+11) /* Restricted Transactional Memory */
210#define X86_FEATURE_RDSEED (9*32+18) /* The RDSEED instruction */
211#define X86_FEATURE_ADX (9*32+19) /* The ADCX and ADOX instructions */
210 212
211#if defined(__KERNEL__) && !defined(__ASSEMBLY__) 213#if defined(__KERNEL__) && !defined(__ASSEMBLY__)
212 214
diff --git a/arch/x86/include/asm/crypto/ablk_helper.h b/arch/x86/include/asm/crypto/ablk_helper.h
new file mode 100644
index 000000000000..4f93df50c23e
--- /dev/null
+++ b/arch/x86/include/asm/crypto/ablk_helper.h
@@ -0,0 +1,31 @@
1/*
2 * Shared async block cipher helpers
3 */
4
5#ifndef _CRYPTO_ABLK_HELPER_H
6#define _CRYPTO_ABLK_HELPER_H
7
8#include <linux/crypto.h>
9#include <linux/kernel.h>
10#include <crypto/cryptd.h>
11
12struct async_helper_ctx {
13 struct cryptd_ablkcipher *cryptd_tfm;
14};
15
16extern int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
17 unsigned int key_len);
18
19extern int __ablk_encrypt(struct ablkcipher_request *req);
20
21extern int ablk_encrypt(struct ablkcipher_request *req);
22
23extern int ablk_decrypt(struct ablkcipher_request *req);
24
25extern void ablk_exit(struct crypto_tfm *tfm);
26
27extern int ablk_init_common(struct crypto_tfm *tfm, const char *drv_name);
28
29extern int ablk_init(struct crypto_tfm *tfm);
30
31#endif /* _CRYPTO_ABLK_HELPER_H */
diff --git a/arch/x86/include/asm/aes.h b/arch/x86/include/asm/crypto/aes.h
index 80545a1cbe39..80545a1cbe39 100644
--- a/arch/x86/include/asm/aes.h
+++ b/arch/x86/include/asm/crypto/aes.h
diff --git a/arch/x86/include/asm/crypto/glue_helper.h b/arch/x86/include/asm/crypto/glue_helper.h
new file mode 100644
index 000000000000..3e408bddc96f
--- /dev/null
+++ b/arch/x86/include/asm/crypto/glue_helper.h
@@ -0,0 +1,115 @@
1/*
2 * Shared glue code for 128bit block ciphers
3 */
4
5#ifndef _CRYPTO_GLUE_HELPER_H
6#define _CRYPTO_GLUE_HELPER_H
7
8#include <linux/kernel.h>
9#include <linux/crypto.h>
10#include <asm/i387.h>
11#include <crypto/b128ops.h>
12
13typedef void (*common_glue_func_t)(void *ctx, u8 *dst, const u8 *src);
14typedef void (*common_glue_cbc_func_t)(void *ctx, u128 *dst, const u128 *src);
15typedef void (*common_glue_ctr_func_t)(void *ctx, u128 *dst, const u128 *src,
16 u128 *iv);
17
18#define GLUE_FUNC_CAST(fn) ((common_glue_func_t)(fn))
19#define GLUE_CBC_FUNC_CAST(fn) ((common_glue_cbc_func_t)(fn))
20#define GLUE_CTR_FUNC_CAST(fn) ((common_glue_ctr_func_t)(fn))
21
22struct common_glue_func_entry {
23 unsigned int num_blocks; /* number of blocks that @fn will process */
24 union {
25 common_glue_func_t ecb;
26 common_glue_cbc_func_t cbc;
27 common_glue_ctr_func_t ctr;
28 } fn_u;
29};
30
31struct common_glue_ctx {
32 unsigned int num_funcs;
33 int fpu_blocks_limit; /* -1 means fpu not needed at all */
34
35 /*
36 * First funcs entry must have largest num_blocks and last funcs entry
37 * must have num_blocks == 1!
38 */
39 struct common_glue_func_entry funcs[];
40};
41
42static inline bool glue_fpu_begin(unsigned int bsize, int fpu_blocks_limit,
43 struct blkcipher_desc *desc,
44 bool fpu_enabled, unsigned int nbytes)
45{
46 if (likely(fpu_blocks_limit < 0))
47 return false;
48
49 if (fpu_enabled)
50 return true;
51
52 /*
53 * Vector-registers are only used when chunk to be processed is large
54 * enough, so do not enable FPU until it is necessary.
55 */
56 if (nbytes < bsize * (unsigned int)fpu_blocks_limit)
57 return false;
58
59 if (desc) {
60 /* prevent sleeping if FPU is in use */
61 desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
62 }
63
64 kernel_fpu_begin();
65 return true;
66}
67
68static inline void glue_fpu_end(bool fpu_enabled)
69{
70 if (fpu_enabled)
71 kernel_fpu_end();
72}
73
74static inline void u128_to_be128(be128 *dst, const u128 *src)
75{
76 dst->a = cpu_to_be64(src->a);
77 dst->b = cpu_to_be64(src->b);
78}
79
80static inline void be128_to_u128(u128 *dst, const be128 *src)
81{
82 dst->a = be64_to_cpu(src->a);
83 dst->b = be64_to_cpu(src->b);
84}
85
86static inline void u128_inc(u128 *i)
87{
88 i->b++;
89 if (!i->b)
90 i->a++;
91}
92
93extern int glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx,
94 struct blkcipher_desc *desc,
95 struct scatterlist *dst,
96 struct scatterlist *src, unsigned int nbytes);
97
98extern int glue_cbc_encrypt_128bit(const common_glue_func_t fn,
99 struct blkcipher_desc *desc,
100 struct scatterlist *dst,
101 struct scatterlist *src,
102 unsigned int nbytes);
103
104extern int glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx,
105 struct blkcipher_desc *desc,
106 struct scatterlist *dst,
107 struct scatterlist *src,
108 unsigned int nbytes);
109
110extern int glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx,
111 struct blkcipher_desc *desc,
112 struct scatterlist *dst,
113 struct scatterlist *src, unsigned int nbytes);
114
115#endif /* _CRYPTO_GLUE_HELPER_H */
diff --git a/arch/x86/include/asm/crypto/serpent-avx.h b/arch/x86/include/asm/crypto/serpent-avx.h
new file mode 100644
index 000000000000..432deedd2945
--- /dev/null
+++ b/arch/x86/include/asm/crypto/serpent-avx.h
@@ -0,0 +1,32 @@
1#ifndef ASM_X86_SERPENT_AVX_H
2#define ASM_X86_SERPENT_AVX_H
3
4#include <linux/crypto.h>
5#include <crypto/serpent.h>
6
7#define SERPENT_PARALLEL_BLOCKS 8
8
9asmlinkage void __serpent_enc_blk_8way_avx(struct serpent_ctx *ctx, u8 *dst,
10 const u8 *src, bool xor);
11asmlinkage void serpent_dec_blk_8way_avx(struct serpent_ctx *ctx, u8 *dst,
12 const u8 *src);
13
14static inline void serpent_enc_blk_xway(struct serpent_ctx *ctx, u8 *dst,
15 const u8 *src)
16{
17 __serpent_enc_blk_8way_avx(ctx, dst, src, false);
18}
19
20static inline void serpent_enc_blk_xway_xor(struct serpent_ctx *ctx, u8 *dst,
21 const u8 *src)
22{
23 __serpent_enc_blk_8way_avx(ctx, dst, src, true);
24}
25
26static inline void serpent_dec_blk_xway(struct serpent_ctx *ctx, u8 *dst,
27 const u8 *src)
28{
29 serpent_dec_blk_8way_avx(ctx, dst, src);
30}
31
32#endif
diff --git a/arch/x86/include/asm/serpent.h b/arch/x86/include/asm/crypto/serpent-sse2.h
index d3ef63fe0c81..e6e77dffbdab 100644
--- a/arch/x86/include/asm/serpent.h
+++ b/arch/x86/include/asm/crypto/serpent-sse2.h
@@ -1,5 +1,5 @@
1#ifndef ASM_X86_SERPENT_H 1#ifndef ASM_X86_SERPENT_SSE2_H
2#define ASM_X86_SERPENT_H 2#define ASM_X86_SERPENT_SSE2_H
3 3
4#include <linux/crypto.h> 4#include <linux/crypto.h>
5#include <crypto/serpent.h> 5#include <crypto/serpent.h>
diff --git a/arch/x86/include/asm/crypto/twofish.h b/arch/x86/include/asm/crypto/twofish.h
new file mode 100644
index 000000000000..9d2c514bd5f9
--- /dev/null
+++ b/arch/x86/include/asm/crypto/twofish.h
@@ -0,0 +1,46 @@
1#ifndef ASM_X86_TWOFISH_H
2#define ASM_X86_TWOFISH_H
3
4#include <linux/crypto.h>
5#include <crypto/twofish.h>
6#include <crypto/lrw.h>
7#include <crypto/b128ops.h>
8
9struct twofish_lrw_ctx {
10 struct lrw_table_ctx lrw_table;
11 struct twofish_ctx twofish_ctx;
12};
13
14struct twofish_xts_ctx {
15 struct twofish_ctx tweak_ctx;
16 struct twofish_ctx crypt_ctx;
17};
18
19/* regular block cipher functions from twofish_x86_64 module */
20asmlinkage void twofish_enc_blk(struct twofish_ctx *ctx, u8 *dst,
21 const u8 *src);
22asmlinkage void twofish_dec_blk(struct twofish_ctx *ctx, u8 *dst,
23 const u8 *src);
24
25/* 3-way parallel cipher functions */
26asmlinkage void __twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
27 const u8 *src, bool xor);
28asmlinkage void twofish_dec_blk_3way(struct twofish_ctx *ctx, u8 *dst,
29 const u8 *src);
30
31/* helpers from twofish_x86_64-3way module */
32extern void twofish_dec_blk_cbc_3way(void *ctx, u128 *dst, const u128 *src);
33extern void twofish_enc_blk_ctr(void *ctx, u128 *dst, const u128 *src,
34 u128 *iv);
35extern void twofish_enc_blk_ctr_3way(void *ctx, u128 *dst, const u128 *src,
36 u128 *iv);
37
38extern int lrw_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
39 unsigned int keylen);
40
41extern void lrw_twofish_exit_tfm(struct crypto_tfm *tfm);
42
43extern int xts_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
44 unsigned int keylen);
45
46#endif /* ASM_X86_TWOFISH_H */
diff --git a/arch/x86/include/asm/entry_arch.h b/arch/x86/include/asm/entry_arch.h
index 0baa628e330c..40afa0005c69 100644
--- a/arch/x86/include/asm/entry_arch.h
+++ b/arch/x86/include/asm/entry_arch.h
@@ -15,15 +15,6 @@ BUILD_INTERRUPT(call_function_interrupt,CALL_FUNCTION_VECTOR)
15BUILD_INTERRUPT(call_function_single_interrupt,CALL_FUNCTION_SINGLE_VECTOR) 15BUILD_INTERRUPT(call_function_single_interrupt,CALL_FUNCTION_SINGLE_VECTOR)
16BUILD_INTERRUPT(irq_move_cleanup_interrupt,IRQ_MOVE_CLEANUP_VECTOR) 16BUILD_INTERRUPT(irq_move_cleanup_interrupt,IRQ_MOVE_CLEANUP_VECTOR)
17BUILD_INTERRUPT(reboot_interrupt,REBOOT_VECTOR) 17BUILD_INTERRUPT(reboot_interrupt,REBOOT_VECTOR)
18
19.irp idx,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, \
20 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
21.if NUM_INVALIDATE_TLB_VECTORS > \idx
22BUILD_INTERRUPT3(invalidate_interrupt\idx,
23 (INVALIDATE_TLB_VECTOR_START)+\idx,
24 smp_invalidate_interrupt)
25.endif
26.endr
27#endif 18#endif
28 19
29BUILD_INTERRUPT(x86_platform_ipi, X86_PLATFORM_IPI_VECTOR) 20BUILD_INTERRUPT(x86_platform_ipi, X86_PLATFORM_IPI_VECTOR)
diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h
index 4b4448761e88..1508e518c7e3 100644
--- a/arch/x86/include/asm/irq_vectors.h
+++ b/arch/x86/include/asm/irq_vectors.h
@@ -119,17 +119,6 @@
119 */ 119 */
120#define LOCAL_TIMER_VECTOR 0xef 120#define LOCAL_TIMER_VECTOR 0xef
121 121
122/* up to 32 vectors used for spreading out TLB flushes: */
123#if NR_CPUS <= 32
124# define NUM_INVALIDATE_TLB_VECTORS (NR_CPUS)
125#else
126# define NUM_INVALIDATE_TLB_VECTORS (32)
127#endif
128
129#define INVALIDATE_TLB_VECTOR_END (0xee)
130#define INVALIDATE_TLB_VECTOR_START \
131 (INVALIDATE_TLB_VECTOR_END-NUM_INVALIDATE_TLB_VECTORS+1)
132
133#define NR_VECTORS 256 122#define NR_VECTORS 256
134 123
135#define FPU_IRQ 13 124#define FPU_IRQ 13
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index 0b47ddb6f00b..a0facf3908d7 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -360,9 +360,10 @@ static inline void __flush_tlb_single(unsigned long addr)
360 360
361static inline void flush_tlb_others(const struct cpumask *cpumask, 361static inline void flush_tlb_others(const struct cpumask *cpumask,
362 struct mm_struct *mm, 362 struct mm_struct *mm,
363 unsigned long va) 363 unsigned long start,
364 unsigned long end)
364{ 365{
365 PVOP_VCALL3(pv_mmu_ops.flush_tlb_others, cpumask, mm, va); 366 PVOP_VCALL4(pv_mmu_ops.flush_tlb_others, cpumask, mm, start, end);
366} 367}
367 368
368static inline int paravirt_pgd_alloc(struct mm_struct *mm) 369static inline int paravirt_pgd_alloc(struct mm_struct *mm)
diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
index 8613cbb7ba41..142236ed83af 100644
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -248,7 +248,8 @@ struct pv_mmu_ops {
248 void (*flush_tlb_single)(unsigned long addr); 248 void (*flush_tlb_single)(unsigned long addr);
249 void (*flush_tlb_others)(const struct cpumask *cpus, 249 void (*flush_tlb_others)(const struct cpumask *cpus,
250 struct mm_struct *mm, 250 struct mm_struct *mm,
251 unsigned long va); 251 unsigned long start,
252 unsigned long end);
252 253
253 /* Hooks for allocating and freeing a pagetable top-level */ 254 /* Hooks for allocating and freeing a pagetable top-level */
254 int (*pgd_alloc)(struct mm_struct *mm); 255 int (*pgd_alloc)(struct mm_struct *mm);
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index d9b8e3f7f42a..1104afaba52b 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -551,6 +551,12 @@ DECLARE_PER_CPU(unsigned long, this_cpu_off);
551 { [0 ... NR_CPUS-1] = _initvalue }; \ 551 { [0 ... NR_CPUS-1] = _initvalue }; \
552 __typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map 552 __typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map
553 553
554#define DEFINE_EARLY_PER_CPU_READ_MOSTLY(_type, _name, _initvalue) \
555 DEFINE_PER_CPU_READ_MOSTLY(_type, _name) = _initvalue; \
556 __typeof__(_type) _name##_early_map[NR_CPUS] __initdata = \
557 { [0 ... NR_CPUS-1] = _initvalue }; \
558 __typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map
559
554#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \ 560#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \
555 EXPORT_PER_CPU_SYMBOL(_name) 561 EXPORT_PER_CPU_SYMBOL(_name)
556 562
@@ -559,6 +565,11 @@ DECLARE_PER_CPU(unsigned long, this_cpu_off);
559 extern __typeof__(_type) *_name##_early_ptr; \ 565 extern __typeof__(_type) *_name##_early_ptr; \
560 extern __typeof__(_type) _name##_early_map[] 566 extern __typeof__(_type) _name##_early_map[]
561 567
568#define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name) \
569 DECLARE_PER_CPU_READ_MOSTLY(_type, _name); \
570 extern __typeof__(_type) *_name##_early_ptr; \
571 extern __typeof__(_type) _name##_early_map[]
572
562#define early_per_cpu_ptr(_name) (_name##_early_ptr) 573#define early_per_cpu_ptr(_name) (_name##_early_ptr)
563#define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx]) 574#define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx])
564#define early_per_cpu(_name, _cpu) \ 575#define early_per_cpu(_name, _cpu) \
@@ -570,12 +581,18 @@ DECLARE_PER_CPU(unsigned long, this_cpu_off);
570#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \ 581#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \
571 DEFINE_PER_CPU(_type, _name) = _initvalue 582 DEFINE_PER_CPU(_type, _name) = _initvalue
572 583
584#define DEFINE_EARLY_PER_CPU_READ_MOSTLY(_type, _name, _initvalue) \
585 DEFINE_PER_CPU_READ_MOSTLY(_type, _name) = _initvalue
586
573#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \ 587#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \
574 EXPORT_PER_CPU_SYMBOL(_name) 588 EXPORT_PER_CPU_SYMBOL(_name)
575 589
576#define DECLARE_EARLY_PER_CPU(_type, _name) \ 590#define DECLARE_EARLY_PER_CPU(_type, _name) \
577 DECLARE_PER_CPU(_type, _name) 591 DECLARE_PER_CPU(_type, _name)
578 592
593#define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name) \
594 DECLARE_PER_CPU_READ_MOSTLY(_type, _name)
595
579#define early_per_cpu(_name, _cpu) per_cpu(_name, _cpu) 596#define early_per_cpu(_name, _cpu) per_cpu(_name, _cpu)
580#define early_per_cpu_ptr(_name) NULL 597#define early_per_cpu_ptr(_name) NULL
581/* no early_per_cpu_map() */ 598/* no early_per_cpu_map() */
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 39bc5777211a..d048cad9bcad 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -61,6 +61,19 @@ static inline void *current_text_addr(void)
61# define ARCH_MIN_MMSTRUCT_ALIGN 0 61# define ARCH_MIN_MMSTRUCT_ALIGN 0
62#endif 62#endif
63 63
64enum tlb_infos {
65 ENTRIES,
66 NR_INFO
67};
68
69extern u16 __read_mostly tlb_lli_4k[NR_INFO];
70extern u16 __read_mostly tlb_lli_2m[NR_INFO];
71extern u16 __read_mostly tlb_lli_4m[NR_INFO];
72extern u16 __read_mostly tlb_lld_4k[NR_INFO];
73extern u16 __read_mostly tlb_lld_2m[NR_INFO];
74extern u16 __read_mostly tlb_lld_4m[NR_INFO];
75extern s8 __read_mostly tlb_flushall_shift;
76
64/* 77/*
65 * CPU type and hardware bug flags. Kept separately for each CPU. 78 * CPU type and hardware bug flags. Kept separately for each CPU.
66 * Members of this structure are referenced in head.S, so think twice 79 * Members of this structure are referenced in head.S, so think twice
diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index 2ffa95dc2333..4f19a1526037 100644
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -31,12 +31,12 @@ static inline bool cpu_has_ht_siblings(void)
31 return has_siblings; 31 return has_siblings;
32} 32}
33 33
34DECLARE_PER_CPU(cpumask_var_t, cpu_sibling_map); 34DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);
35DECLARE_PER_CPU(cpumask_var_t, cpu_core_map); 35DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map);
36/* cpus sharing the last level cache: */ 36/* cpus sharing the last level cache: */
37DECLARE_PER_CPU(cpumask_var_t, cpu_llc_shared_map); 37DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map);
38DECLARE_PER_CPU(u16, cpu_llc_id); 38DECLARE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id);
39DECLARE_PER_CPU(int, cpu_number); 39DECLARE_PER_CPU_READ_MOSTLY(int, cpu_number);
40 40
41static inline struct cpumask *cpu_sibling_mask(int cpu) 41static inline struct cpumask *cpu_sibling_mask(int cpu)
42{ 42{
@@ -53,10 +53,10 @@ static inline struct cpumask *cpu_llc_shared_mask(int cpu)
53 return per_cpu(cpu_llc_shared_map, cpu); 53 return per_cpu(cpu_llc_shared_map, cpu);
54} 54}
55 55
56DECLARE_EARLY_PER_CPU(u16, x86_cpu_to_apicid); 56DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid);
57DECLARE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid); 57DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid);
58#if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86_32) 58#if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86_32)
59DECLARE_EARLY_PER_CPU(int, x86_cpu_to_logical_apicid); 59DECLARE_EARLY_PER_CPU_READ_MOSTLY(int, x86_cpu_to_logical_apicid);
60#endif 60#endif
61 61
62/* Static state in head.S used to set up a CPU */ 62/* Static state in head.S used to set up a CPU */
diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h
index 829215fef9ee..4fef20773b8f 100644
--- a/arch/x86/include/asm/tlb.h
+++ b/arch/x86/include/asm/tlb.h
@@ -4,7 +4,14 @@
4#define tlb_start_vma(tlb, vma) do { } while (0) 4#define tlb_start_vma(tlb, vma) do { } while (0)
5#define tlb_end_vma(tlb, vma) do { } while (0) 5#define tlb_end_vma(tlb, vma) do { } while (0)
6#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0) 6#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
7#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm) 7
8#define tlb_flush(tlb) \
9{ \
10 if (tlb->fullmm == 0) \
11 flush_tlb_mm_range(tlb->mm, tlb->start, tlb->end, 0UL); \
12 else \
13 flush_tlb_mm_range(tlb->mm, 0UL, TLB_FLUSH_ALL, 0UL); \
14}
8 15
9#include <asm-generic/tlb.h> 16#include <asm-generic/tlb.h>
10 17
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 36a1a2ab87d2..74a44333545a 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -73,14 +73,10 @@ static inline void __flush_tlb_one(unsigned long addr)
73 * - flush_tlb_page(vma, vmaddr) flushes one page 73 * - flush_tlb_page(vma, vmaddr) flushes one page
74 * - flush_tlb_range(vma, start, end) flushes a range of pages 74 * - flush_tlb_range(vma, start, end) flushes a range of pages
75 * - flush_tlb_kernel_range(start, end) flushes a range of kernel pages 75 * - flush_tlb_kernel_range(start, end) flushes a range of kernel pages
76 * - flush_tlb_others(cpumask, mm, va) flushes TLBs on other cpus 76 * - flush_tlb_others(cpumask, mm, start, end) flushes TLBs on other cpus
77 * 77 *
78 * ..but the i386 has somewhat limited tlb flushing capabilities, 78 * ..but the i386 has somewhat limited tlb flushing capabilities,
79 * and page-granular flushes are available only on i486 and up. 79 * and page-granular flushes are available only on i486 and up.
80 *
81 * x86-64 can only flush individual pages or full VMs. For a range flush
82 * we always do the full VM. Might be worth trying if for a small
83 * range a few INVLPGs in a row are a win.
84 */ 80 */
85 81
86#ifndef CONFIG_SMP 82#ifndef CONFIG_SMP
@@ -109,9 +105,17 @@ static inline void flush_tlb_range(struct vm_area_struct *vma,
109 __flush_tlb(); 105 __flush_tlb();
110} 106}
111 107
108static inline void flush_tlb_mm_range(struct mm_struct *mm,
109 unsigned long start, unsigned long end, unsigned long vmflag)
110{
111 if (mm == current->active_mm)
112 __flush_tlb();
113}
114
112static inline void native_flush_tlb_others(const struct cpumask *cpumask, 115static inline void native_flush_tlb_others(const struct cpumask *cpumask,
113 struct mm_struct *mm, 116 struct mm_struct *mm,
114 unsigned long va) 117 unsigned long start,
118 unsigned long end)
115{ 119{
116} 120}
117 121
@@ -119,27 +123,35 @@ static inline void reset_lazy_tlbstate(void)
119{ 123{
120} 124}
121 125
126static inline void flush_tlb_kernel_range(unsigned long start,
127 unsigned long end)
128{
129 flush_tlb_all();
130}
131
122#else /* SMP */ 132#else /* SMP */
123 133
124#include <asm/smp.h> 134#include <asm/smp.h>
125 135
126#define local_flush_tlb() __flush_tlb() 136#define local_flush_tlb() __flush_tlb()
127 137
138#define flush_tlb_mm(mm) flush_tlb_mm_range(mm, 0UL, TLB_FLUSH_ALL, 0UL)
139
140#define flush_tlb_range(vma, start, end) \
141 flush_tlb_mm_range(vma->vm_mm, start, end, vma->vm_flags)
142
128extern void flush_tlb_all(void); 143extern void flush_tlb_all(void);
129extern void flush_tlb_current_task(void); 144extern void flush_tlb_current_task(void);
130extern void flush_tlb_mm(struct mm_struct *);
131extern void flush_tlb_page(struct vm_area_struct *, unsigned long); 145extern void flush_tlb_page(struct vm_area_struct *, unsigned long);
146extern void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
147 unsigned long end, unsigned long vmflag);
148extern void flush_tlb_kernel_range(unsigned long start, unsigned long end);
132 149
133#define flush_tlb() flush_tlb_current_task() 150#define flush_tlb() flush_tlb_current_task()
134 151
135static inline void flush_tlb_range(struct vm_area_struct *vma,
136 unsigned long start, unsigned long end)
137{
138 flush_tlb_mm(vma->vm_mm);
139}
140
141void native_flush_tlb_others(const struct cpumask *cpumask, 152void native_flush_tlb_others(const struct cpumask *cpumask,
142 struct mm_struct *mm, unsigned long va); 153 struct mm_struct *mm,
154 unsigned long start, unsigned long end);
143 155
144#define TLBSTATE_OK 1 156#define TLBSTATE_OK 1
145#define TLBSTATE_LAZY 2 157#define TLBSTATE_LAZY 2
@@ -159,13 +171,8 @@ static inline void reset_lazy_tlbstate(void)
159#endif /* SMP */ 171#endif /* SMP */
160 172
161#ifndef CONFIG_PARAVIRT 173#ifndef CONFIG_PARAVIRT
162#define flush_tlb_others(mask, mm, va) native_flush_tlb_others(mask, mm, va) 174#define flush_tlb_others(mask, mm, start, end) \
175 native_flush_tlb_others(mask, mm, start, end)
163#endif 176#endif
164 177
165static inline void flush_tlb_kernel_range(unsigned long start,
166 unsigned long end)
167{
168 flush_tlb_all();
169}
170
171#endif /* _ASM_X86_TLBFLUSH_H */ 178#endif /* _ASM_X86_TLBFLUSH_H */
diff --git a/arch/x86/include/asm/uv/uv.h b/arch/x86/include/asm/uv/uv.h
index 3bb9491b7659..b47c2a82ff15 100644
--- a/arch/x86/include/asm/uv/uv.h
+++ b/arch/x86/include/asm/uv/uv.h
@@ -15,7 +15,8 @@ extern void uv_nmi_init(void);
15extern void uv_system_init(void); 15extern void uv_system_init(void);
16extern const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask, 16extern const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
17 struct mm_struct *mm, 17 struct mm_struct *mm,
18 unsigned long va, 18 unsigned long start,
19 unsigned end,
19 unsigned int cpu); 20 unsigned int cpu);
20 21
21#else /* X86_UV */ 22#else /* X86_UV */
@@ -26,7 +27,7 @@ static inline void uv_cpu_init(void) { }
26static inline void uv_system_init(void) { } 27static inline void uv_system_init(void) { }
27static inline const struct cpumask * 28static inline const struct cpumask *
28uv_flush_tlb_others(const struct cpumask *cpumask, struct mm_struct *mm, 29uv_flush_tlb_others(const struct cpumask *cpumask, struct mm_struct *mm,
29 unsigned long va, unsigned int cpu) 30 unsigned long start, unsigned long end, unsigned int cpu)
30{ return cpumask; } 31{ return cpumask; }
31 32
32#endif /* X86_UV */ 33#endif /* X86_UV */
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 98e24131ff3a..24deb3082328 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -75,8 +75,8 @@ physid_mask_t phys_cpu_present_map;
75/* 75/*
76 * Map cpu index to physical APIC ID 76 * Map cpu index to physical APIC ID
77 */ 77 */
78DEFINE_EARLY_PER_CPU(u16, x86_cpu_to_apicid, BAD_APICID); 78DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid, BAD_APICID);
79DEFINE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid, BAD_APICID); 79DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid, BAD_APICID);
80EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid); 80EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);
81EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid); 81EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
82 82
@@ -88,7 +88,7 @@ EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
88 * used for the mapping. This is where the behaviors of x86_64 and 32 88 * used for the mapping. This is where the behaviors of x86_64 and 32
89 * actually diverge. Let's keep it ugly for now. 89 * actually diverge. Let's keep it ugly for now.
90 */ 90 */
91DEFINE_EARLY_PER_CPU(int, x86_cpu_to_logical_apicid, BAD_APICID); 91DEFINE_EARLY_PER_CPU_READ_MOSTLY(int, x86_cpu_to_logical_apicid, BAD_APICID);
92 92
93/* 93/*
94 * Knob to control our willingness to enable the local APIC. 94 * Knob to control our willingness to enable the local APIC.
diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile
index bac4c3804cc7..d30a6a9a0121 100644
--- a/arch/x86/kernel/cpu/Makefile
+++ b/arch/x86/kernel/cpu/Makefile
@@ -14,7 +14,7 @@ CFLAGS_common.o := $(nostackp)
14 14
15obj-y := intel_cacheinfo.o scattered.o topology.o 15obj-y := intel_cacheinfo.o scattered.o topology.o
16obj-y += proc.o capflags.o powerflags.o common.o 16obj-y += proc.o capflags.o powerflags.o common.o
17obj-y += vmware.o hypervisor.o sched.o mshyperv.o 17obj-y += vmware.o hypervisor.o mshyperv.o
18obj-y += rdrand.o 18obj-y += rdrand.o
19obj-y += match.o 19obj-y += match.o
20 20
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 5bbc082c47ad..46d8786d655e 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -452,6 +452,35 @@ void __cpuinit cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
452 c->x86_cache_size = l2size; 452 c->x86_cache_size = l2size;
453} 453}
454 454
455u16 __read_mostly tlb_lli_4k[NR_INFO];
456u16 __read_mostly tlb_lli_2m[NR_INFO];
457u16 __read_mostly tlb_lli_4m[NR_INFO];
458u16 __read_mostly tlb_lld_4k[NR_INFO];
459u16 __read_mostly tlb_lld_2m[NR_INFO];
460u16 __read_mostly tlb_lld_4m[NR_INFO];
461
462/*
463 * tlb_flushall_shift shows the balance point in replacing cr3 write
464 * with multiple 'invlpg'. It will do this replacement when
465 * flush_tlb_lines <= active_lines/2^tlb_flushall_shift.
466 * If tlb_flushall_shift is -1, means the replacement will be disabled.
467 */
468s8 __read_mostly tlb_flushall_shift = -1;
469
470void __cpuinit cpu_detect_tlb(struct cpuinfo_x86 *c)
471{
472 if (this_cpu->c_detect_tlb)
473 this_cpu->c_detect_tlb(c);
474
475 printk(KERN_INFO "Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \
476 "Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \
477 "tlb_flushall_shift is 0x%x\n",
478 tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
479 tlb_lli_4m[ENTRIES], tlb_lld_4k[ENTRIES],
480 tlb_lld_2m[ENTRIES], tlb_lld_4m[ENTRIES],
481 tlb_flushall_shift);
482}
483
455void __cpuinit detect_ht(struct cpuinfo_x86 *c) 484void __cpuinit detect_ht(struct cpuinfo_x86 *c)
456{ 485{
457#ifdef CONFIG_X86_HT 486#ifdef CONFIG_X86_HT
@@ -911,6 +940,8 @@ void __init identify_boot_cpu(void)
911#else 940#else
912 vgetcpu_set_mode(); 941 vgetcpu_set_mode();
913#endif 942#endif
943 if (boot_cpu_data.cpuid_level >= 2)
944 cpu_detect_tlb(&boot_cpu_data);
914} 945}
915 946
916void __cpuinit identify_secondary_cpu(struct cpuinfo_x86 *c) 947void __cpuinit identify_secondary_cpu(struct cpuinfo_x86 *c)
diff --git a/arch/x86/kernel/cpu/cpu.h b/arch/x86/kernel/cpu/cpu.h
index 8bacc7826fb3..4041c24ae7db 100644
--- a/arch/x86/kernel/cpu/cpu.h
+++ b/arch/x86/kernel/cpu/cpu.h
@@ -20,10 +20,19 @@ struct cpu_dev {
20 void (*c_bsp_init)(struct cpuinfo_x86 *); 20 void (*c_bsp_init)(struct cpuinfo_x86 *);
21 void (*c_init)(struct cpuinfo_x86 *); 21 void (*c_init)(struct cpuinfo_x86 *);
22 void (*c_identify)(struct cpuinfo_x86 *); 22 void (*c_identify)(struct cpuinfo_x86 *);
23 void (*c_detect_tlb)(struct cpuinfo_x86 *);
23 unsigned int (*c_size_cache)(struct cpuinfo_x86 *, unsigned int); 24 unsigned int (*c_size_cache)(struct cpuinfo_x86 *, unsigned int);
24 int c_x86_vendor; 25 int c_x86_vendor;
25}; 26};
26 27
28struct _tlb_table {
29 unsigned char descriptor;
30 char tlb_type;
31 unsigned int entries;
32 /* unsigned int ways; */
33 char info[128];
34};
35
27#define cpu_dev_register(cpu_devX) \ 36#define cpu_dev_register(cpu_devX) \
28 static const struct cpu_dev *const __cpu_dev_##cpu_devX __used \ 37 static const struct cpu_dev *const __cpu_dev_##cpu_devX __used \
29 __attribute__((__section__(".x86_cpu_dev.init"))) = \ 38 __attribute__((__section__(".x86_cpu_dev.init"))) = \
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 3e6ff6cbf42a..0a4ce2980a5a 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -491,6 +491,181 @@ static unsigned int __cpuinit intel_size_cache(struct cpuinfo_x86 *c, unsigned i
491} 491}
492#endif 492#endif
493 493
494#define TLB_INST_4K 0x01
495#define TLB_INST_4M 0x02
496#define TLB_INST_2M_4M 0x03
497
498#define TLB_INST_ALL 0x05
499#define TLB_INST_1G 0x06
500
501#define TLB_DATA_4K 0x11
502#define TLB_DATA_4M 0x12
503#define TLB_DATA_2M_4M 0x13
504#define TLB_DATA_4K_4M 0x14
505
506#define TLB_DATA_1G 0x16
507
508#define TLB_DATA0_4K 0x21
509#define TLB_DATA0_4M 0x22
510#define TLB_DATA0_2M_4M 0x23
511
512#define STLB_4K 0x41
513
514static const struct _tlb_table intel_tlb_table[] __cpuinitconst = {
515 { 0x01, TLB_INST_4K, 32, " TLB_INST 4 KByte pages, 4-way set associative" },
516 { 0x02, TLB_INST_4M, 2, " TLB_INST 4 MByte pages, full associative" },
517 { 0x03, TLB_DATA_4K, 64, " TLB_DATA 4 KByte pages, 4-way set associative" },
518 { 0x04, TLB_DATA_4M, 8, " TLB_DATA 4 MByte pages, 4-way set associative" },
519 { 0x05, TLB_DATA_4M, 32, " TLB_DATA 4 MByte pages, 4-way set associative" },
520 { 0x0b, TLB_INST_4M, 4, " TLB_INST 4 MByte pages, 4-way set associative" },
521 { 0x4f, TLB_INST_4K, 32, " TLB_INST 4 KByte pages */" },
522 { 0x50, TLB_INST_ALL, 64, " TLB_INST 4 KByte and 2-MByte or 4-MByte pages" },
523 { 0x51, TLB_INST_ALL, 128, " TLB_INST 4 KByte and 2-MByte or 4-MByte pages" },
524 { 0x52, TLB_INST_ALL, 256, " TLB_INST 4 KByte and 2-MByte or 4-MByte pages" },
525 { 0x55, TLB_INST_2M_4M, 7, " TLB_INST 2-MByte or 4-MByte pages, fully associative" },
526 { 0x56, TLB_DATA0_4M, 16, " TLB_DATA0 4 MByte pages, 4-way set associative" },
527 { 0x57, TLB_DATA0_4K, 16, " TLB_DATA0 4 KByte pages, 4-way associative" },
528 { 0x59, TLB_DATA0_4K, 16, " TLB_DATA0 4 KByte pages, fully associative" },
529 { 0x5a, TLB_DATA0_2M_4M, 32, " TLB_DATA0 2-MByte or 4 MByte pages, 4-way set associative" },
530 { 0x5b, TLB_DATA_4K_4M, 64, " TLB_DATA 4 KByte and 4 MByte pages" },
531 { 0x5c, TLB_DATA_4K_4M, 128, " TLB_DATA 4 KByte and 4 MByte pages" },
532 { 0x5d, TLB_DATA_4K_4M, 256, " TLB_DATA 4 KByte and 4 MByte pages" },
533 { 0xb0, TLB_INST_4K, 128, " TLB_INST 4 KByte pages, 4-way set associative" },
534 { 0xb1, TLB_INST_2M_4M, 4, " TLB_INST 2M pages, 4-way, 8 entries or 4M pages, 4-way entries" },
535 { 0xb2, TLB_INST_4K, 64, " TLB_INST 4KByte pages, 4-way set associative" },
536 { 0xb3, TLB_DATA_4K, 128, " TLB_DATA 4 KByte pages, 4-way set associative" },
537 { 0xb4, TLB_DATA_4K, 256, " TLB_DATA 4 KByte pages, 4-way associative" },
538 { 0xba, TLB_DATA_4K, 64, " TLB_DATA 4 KByte pages, 4-way associative" },
539 { 0xc0, TLB_DATA_4K_4M, 8, " TLB_DATA 4 KByte and 4 MByte pages, 4-way associative" },
540 { 0xca, STLB_4K, 512, " STLB 4 KByte pages, 4-way associative" },
541 { 0x00, 0, 0 }
542};
543
544static void __cpuinit intel_tlb_lookup(const unsigned char desc)
545{
546 unsigned char k;
547 if (desc == 0)
548 return;
549
550 /* look up this descriptor in the table */
551 for (k = 0; intel_tlb_table[k].descriptor != desc && \
552 intel_tlb_table[k].descriptor != 0; k++)
553 ;
554
555 if (intel_tlb_table[k].tlb_type == 0)
556 return;
557
558 switch (intel_tlb_table[k].tlb_type) {
559 case STLB_4K:
560 if (tlb_lli_4k[ENTRIES] < intel_tlb_table[k].entries)
561 tlb_lli_4k[ENTRIES] = intel_tlb_table[k].entries;
562 if (tlb_lld_4k[ENTRIES] < intel_tlb_table[k].entries)
563 tlb_lld_4k[ENTRIES] = intel_tlb_table[k].entries;
564 break;
565 case TLB_INST_ALL:
566 if (tlb_lli_4k[ENTRIES] < intel_tlb_table[k].entries)
567 tlb_lli_4k[ENTRIES] = intel_tlb_table[k].entries;
568 if (tlb_lli_2m[ENTRIES] < intel_tlb_table[k].entries)
569 tlb_lli_2m[ENTRIES] = intel_tlb_table[k].entries;
570 if (tlb_lli_4m[ENTRIES] < intel_tlb_table[k].entries)
571 tlb_lli_4m[ENTRIES] = intel_tlb_table[k].entries;
572 break;
573 case TLB_INST_4K:
574 if (tlb_lli_4k[ENTRIES] < intel_tlb_table[k].entries)
575 tlb_lli_4k[ENTRIES] = intel_tlb_table[k].entries;
576 break;
577 case TLB_INST_4M:
578 if (tlb_lli_4m[ENTRIES] < intel_tlb_table[k].entries)
579 tlb_lli_4m[ENTRIES] = intel_tlb_table[k].entries;
580 break;
581 case TLB_INST_2M_4M:
582 if (tlb_lli_2m[ENTRIES] < intel_tlb_table[k].entries)
583 tlb_lli_2m[ENTRIES] = intel_tlb_table[k].entries;
584 if (tlb_lli_4m[ENTRIES] < intel_tlb_table[k].entries)
585 tlb_lli_4m[ENTRIES] = intel_tlb_table[k].entries;
586 break;
587 case TLB_DATA_4K:
588 case TLB_DATA0_4K:
589 if (tlb_lld_4k[ENTRIES] < intel_tlb_table[k].entries)
590 tlb_lld_4k[ENTRIES] = intel_tlb_table[k].entries;
591 break;
592 case TLB_DATA_4M:
593 case TLB_DATA0_4M:
594 if (tlb_lld_4m[ENTRIES] < intel_tlb_table[k].entries)
595 tlb_lld_4m[ENTRIES] = intel_tlb_table[k].entries;
596 break;
597 case TLB_DATA_2M_4M:
598 case TLB_DATA0_2M_4M:
599 if (tlb_lld_2m[ENTRIES] < intel_tlb_table[k].entries)
600 tlb_lld_2m[ENTRIES] = intel_tlb_table[k].entries;
601 if (tlb_lld_4m[ENTRIES] < intel_tlb_table[k].entries)
602 tlb_lld_4m[ENTRIES] = intel_tlb_table[k].entries;
603 break;
604 case TLB_DATA_4K_4M:
605 if (tlb_lld_4k[ENTRIES] < intel_tlb_table[k].entries)
606 tlb_lld_4k[ENTRIES] = intel_tlb_table[k].entries;
607 if (tlb_lld_4m[ENTRIES] < intel_tlb_table[k].entries)
608 tlb_lld_4m[ENTRIES] = intel_tlb_table[k].entries;
609 break;
610 }
611}
612
613static void __cpuinit intel_tlb_flushall_shift_set(struct cpuinfo_x86 *c)
614{
615 if (!cpu_has_invlpg) {
616 tlb_flushall_shift = -1;
617 return;
618 }
619 switch ((c->x86 << 8) + c->x86_model) {
620 case 0x60f: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
621 case 0x616: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
622 case 0x617: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
623 case 0x61d: /* six-core 45 nm xeon "Dunnington" */
624 tlb_flushall_shift = -1;
625 break;
626 case 0x61a: /* 45 nm nehalem, "Bloomfield" */
627 case 0x61e: /* 45 nm nehalem, "Lynnfield" */
628 case 0x625: /* 32 nm nehalem, "Clarkdale" */
629 case 0x62c: /* 32 nm nehalem, "Gulftown" */
630 case 0x62e: /* 45 nm nehalem-ex, "Beckton" */
631 case 0x62f: /* 32 nm Xeon E7 */
632 tlb_flushall_shift = 6;
633 break;
634 case 0x62a: /* SandyBridge */
635 case 0x62d: /* SandyBridge, "Romely-EP" */
636 tlb_flushall_shift = 5;
637 break;
638 case 0x63a: /* Ivybridge */
639 tlb_flushall_shift = 1;
640 break;
641 default:
642 tlb_flushall_shift = 6;
643 }
644}
645
646static void __cpuinit intel_detect_tlb(struct cpuinfo_x86 *c)
647{
648 int i, j, n;
649 unsigned int regs[4];
650 unsigned char *desc = (unsigned char *)regs;
651 /* Number of times to iterate */
652 n = cpuid_eax(2) & 0xFF;
653
654 for (i = 0 ; i < n ; i++) {
655 cpuid(2, &regs[0], &regs[1], &regs[2], &regs[3]);
656
657 /* If bit 31 is set, this is an unknown format */
658 for (j = 0 ; j < 3 ; j++)
659 if (regs[j] & (1 << 31))
660 regs[j] = 0;
661
662 /* Byte 0 is level count, not a descriptor */
663 for (j = 1 ; j < 16 ; j++)
664 intel_tlb_lookup(desc[j]);
665 }
666 intel_tlb_flushall_shift_set(c);
667}
668
494static const struct cpu_dev __cpuinitconst intel_cpu_dev = { 669static const struct cpu_dev __cpuinitconst intel_cpu_dev = {
495 .c_vendor = "Intel", 670 .c_vendor = "Intel",
496 .c_ident = { "GenuineIntel" }, 671 .c_ident = { "GenuineIntel" },
@@ -546,6 +721,7 @@ static const struct cpu_dev __cpuinitconst intel_cpu_dev = {
546 }, 721 },
547 .c_size_cache = intel_size_cache, 722 .c_size_cache = intel_size_cache,
548#endif 723#endif
724 .c_detect_tlb = intel_detect_tlb,
549 .c_early_init = early_init_intel, 725 .c_early_init = early_init_intel,
550 .c_init = init_intel, 726 .c_init = init_intel,
551 .c_x86_vendor = X86_VENDOR_INTEL, 727 .c_x86_vendor = X86_VENDOR_INTEL,
diff --git a/arch/x86/kernel/cpu/sched.c b/arch/x86/kernel/cpu/sched.c
deleted file mode 100644
index a640ae5ad201..000000000000
--- a/arch/x86/kernel/cpu/sched.c
+++ /dev/null
@@ -1,55 +0,0 @@
1#include <linux/sched.h>
2#include <linux/math64.h>
3#include <linux/percpu.h>
4#include <linux/irqflags.h>
5
6#include <asm/cpufeature.h>
7#include <asm/processor.h>
8
9#ifdef CONFIG_SMP
10
11static DEFINE_PER_CPU(struct aperfmperf, old_perf_sched);
12
13static unsigned long scale_aperfmperf(void)
14{
15 struct aperfmperf val, *old = &__get_cpu_var(old_perf_sched);
16 unsigned long ratio, flags;
17
18 local_irq_save(flags);
19 get_aperfmperf(&val);
20 local_irq_restore(flags);
21
22 ratio = calc_aperfmperf_ratio(old, &val);
23 *old = val;
24
25 return ratio;
26}
27
28unsigned long arch_scale_freq_power(struct sched_domain *sd, int cpu)
29{
30 /*
31 * do aperf/mperf on the cpu level because it includes things
32 * like turbo mode, which are relevant to full cores.
33 */
34 if (boot_cpu_has(X86_FEATURE_APERFMPERF))
35 return scale_aperfmperf();
36
37 /*
38 * maybe have something cpufreq here
39 */
40
41 return default_scale_freq_power(sd, cpu);
42}
43
44unsigned long arch_scale_smt_power(struct sched_domain *sd, int cpu)
45{
46 /*
47 * aperf/mperf already includes the smt gain
48 */
49 if (boot_cpu_has(X86_FEATURE_APERFMPERF))
50 return SCHED_LOAD_SCALE;
51
52 return default_scale_smt_power(sd, cpu);
53}
54
55#endif
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 111f6bbd8b38..69babd8c834f 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -1048,24 +1048,6 @@ apicinterrupt LOCAL_TIMER_VECTOR \
1048apicinterrupt X86_PLATFORM_IPI_VECTOR \ 1048apicinterrupt X86_PLATFORM_IPI_VECTOR \
1049 x86_platform_ipi smp_x86_platform_ipi 1049 x86_platform_ipi smp_x86_platform_ipi
1050 1050
1051#ifdef CONFIG_SMP
1052 ALIGN
1053 INTR_FRAME
1054.irp idx,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, \
1055 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
1056.if NUM_INVALIDATE_TLB_VECTORS > \idx
1057ENTRY(invalidate_interrupt\idx)
1058 pushq_cfi $~(INVALIDATE_TLB_VECTOR_START+\idx)
1059 jmp .Lcommon_invalidate_interrupt0
1060 CFI_ADJUST_CFA_OFFSET -8
1061END(invalidate_interrupt\idx)
1062.endif
1063.endr
1064 CFI_ENDPROC
1065apicinterrupt INVALIDATE_TLB_VECTOR_START, \
1066 invalidate_interrupt0, smp_invalidate_interrupt
1067#endif
1068
1069apicinterrupt THRESHOLD_APIC_VECTOR \ 1051apicinterrupt THRESHOLD_APIC_VECTOR \
1070 threshold_interrupt smp_threshold_interrupt 1052 threshold_interrupt smp_threshold_interrupt
1071apicinterrupt THERMAL_APIC_VECTOR \ 1053apicinterrupt THERMAL_APIC_VECTOR \
diff --git a/arch/x86/kernel/irqinit.c b/arch/x86/kernel/irqinit.c
index 252981afd6c4..6e03b0d69138 100644
--- a/arch/x86/kernel/irqinit.c
+++ b/arch/x86/kernel/irqinit.c
@@ -171,79 +171,6 @@ static void __init smp_intr_init(void)
171 */ 171 */
172 alloc_intr_gate(RESCHEDULE_VECTOR, reschedule_interrupt); 172 alloc_intr_gate(RESCHEDULE_VECTOR, reschedule_interrupt);
173 173
174 /* IPIs for invalidation */
175#define ALLOC_INVTLB_VEC(NR) \
176 alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+NR, \
177 invalidate_interrupt##NR)
178
179 switch (NUM_INVALIDATE_TLB_VECTORS) {
180 default:
181 ALLOC_INVTLB_VEC(31);
182 case 31:
183 ALLOC_INVTLB_VEC(30);
184 case 30:
185 ALLOC_INVTLB_VEC(29);
186 case 29:
187 ALLOC_INVTLB_VEC(28);
188 case 28:
189 ALLOC_INVTLB_VEC(27);
190 case 27:
191 ALLOC_INVTLB_VEC(26);
192 case 26:
193 ALLOC_INVTLB_VEC(25);
194 case 25:
195 ALLOC_INVTLB_VEC(24);
196 case 24:
197 ALLOC_INVTLB_VEC(23);
198 case 23:
199 ALLOC_INVTLB_VEC(22);
200 case 22:
201 ALLOC_INVTLB_VEC(21);
202 case 21:
203 ALLOC_INVTLB_VEC(20);
204 case 20:
205 ALLOC_INVTLB_VEC(19);
206 case 19:
207 ALLOC_INVTLB_VEC(18);
208 case 18:
209 ALLOC_INVTLB_VEC(17);
210 case 17:
211 ALLOC_INVTLB_VEC(16);
212 case 16:
213 ALLOC_INVTLB_VEC(15);
214 case 15:
215 ALLOC_INVTLB_VEC(14);
216 case 14:
217 ALLOC_INVTLB_VEC(13);
218 case 13:
219 ALLOC_INVTLB_VEC(12);
220 case 12:
221 ALLOC_INVTLB_VEC(11);
222 case 11:
223 ALLOC_INVTLB_VEC(10);
224 case 10:
225 ALLOC_INVTLB_VEC(9);
226 case 9:
227 ALLOC_INVTLB_VEC(8);
228 case 8:
229 ALLOC_INVTLB_VEC(7);
230 case 7:
231 ALLOC_INVTLB_VEC(6);
232 case 6:
233 ALLOC_INVTLB_VEC(5);
234 case 5:
235 ALLOC_INVTLB_VEC(4);
236 case 4:
237 ALLOC_INVTLB_VEC(3);
238 case 3:
239 ALLOC_INVTLB_VEC(2);
240 case 2:
241 ALLOC_INVTLB_VEC(1);
242 case 1:
243 ALLOC_INVTLB_VEC(0);
244 break;
245 }
246
247 /* IPI for generic function call */ 174 /* IPI for generic function call */
248 alloc_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt); 175 alloc_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt);
249 176
diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c
index 5a98aa272184..5cdff0357746 100644
--- a/arch/x86/kernel/setup_percpu.c
+++ b/arch/x86/kernel/setup_percpu.c
@@ -21,7 +21,7 @@
21#include <asm/cpu.h> 21#include <asm/cpu.h>
22#include <asm/stackprotector.h> 22#include <asm/stackprotector.h>
23 23
24DEFINE_PER_CPU(int, cpu_number); 24DEFINE_PER_CPU_READ_MOSTLY(int, cpu_number);
25EXPORT_PER_CPU_SYMBOL(cpu_number); 25EXPORT_PER_CPU_SYMBOL(cpu_number);
26 26
27#ifdef CONFIG_X86_64 27#ifdef CONFIG_X86_64
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index c1a310fb8309..7c5a8c314c02 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -106,17 +106,17 @@ int smp_num_siblings = 1;
106EXPORT_SYMBOL(smp_num_siblings); 106EXPORT_SYMBOL(smp_num_siblings);
107 107
108/* Last level cache ID of each logical CPU */ 108/* Last level cache ID of each logical CPU */
109DEFINE_PER_CPU(u16, cpu_llc_id) = BAD_APICID; 109DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id) = BAD_APICID;
110 110
111/* representing HT siblings of each logical CPU */ 111/* representing HT siblings of each logical CPU */
112DEFINE_PER_CPU(cpumask_var_t, cpu_sibling_map); 112DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);
113EXPORT_PER_CPU_SYMBOL(cpu_sibling_map); 113EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
114 114
115/* representing HT and core siblings of each logical CPU */ 115/* representing HT and core siblings of each logical CPU */
116DEFINE_PER_CPU(cpumask_var_t, cpu_core_map); 116DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map);
117EXPORT_PER_CPU_SYMBOL(cpu_core_map); 117EXPORT_PER_CPU_SYMBOL(cpu_core_map);
118 118
119DEFINE_PER_CPU(cpumask_var_t, cpu_llc_shared_map); 119DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map);
120 120
121/* Per CPU bogomips and other parameters */ 121/* Per CPU bogomips and other parameters */
122DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info); 122DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index a718e0d23503..931930a96160 100644
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -919,11 +919,13 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages,
919 919
920 /* 920 /*
921 * On success we use clflush, when the CPU supports it to 921 * On success we use clflush, when the CPU supports it to
922 * avoid the wbindv. If the CPU does not support it and in the 922 * avoid the wbindv. If the CPU does not support it, in the
923 * error case we fall back to cpa_flush_all (which uses 923 * error case, and during early boot (for EFI) we fall back
924 * wbindv): 924 * to cpa_flush_all (which uses wbinvd):
925 */ 925 */
926 if (!ret && cpu_has_clflush) { 926 if (early_boot_irqs_disabled)
927 __cpa_flush_all((void *)(long)cache);
928 else if (!ret && cpu_has_clflush) {
927 if (cpa.flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) { 929 if (cpa.flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) {
928 cpa_flush_array(addr, numpages, cache, 930 cpa_flush_array(addr, numpages, cache,
929 cpa.flags, pages); 931 cpa.flags, pages);
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 5e57e113b72c..613cd83e8c0c 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -12,6 +12,7 @@
12#include <asm/cache.h> 12#include <asm/cache.h>
13#include <asm/apic.h> 13#include <asm/apic.h>
14#include <asm/uv/uv.h> 14#include <asm/uv/uv.h>
15#include <linux/debugfs.h>
15 16
16DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate) 17DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate)
17 = { &init_mm, 0, }; 18 = { &init_mm, 0, };
@@ -27,33 +28,14 @@ DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate)
27 * 28 *
28 * More scalable flush, from Andi Kleen 29 * More scalable flush, from Andi Kleen
29 * 30 *
30 * To avoid global state use 8 different call vectors. 31 * Implement flush IPI by CALL_FUNCTION_VECTOR, Alex Shi
31 * Each CPU uses a specific vector to trigger flushes on other
32 * CPUs. Depending on the received vector the target CPUs look into
33 * the right array slot for the flush data.
34 *
35 * With more than 8 CPUs they are hashed to the 8 available
36 * vectors. The limited global vector space forces us to this right now.
37 * In future when interrupts are split into per CPU domains this could be
38 * fixed, at the cost of triggering multiple IPIs in some cases.
39 */ 32 */
40 33
41union smp_flush_state { 34struct flush_tlb_info {
42 struct { 35 struct mm_struct *flush_mm;
43 struct mm_struct *flush_mm; 36 unsigned long flush_start;
44 unsigned long flush_va; 37 unsigned long flush_end;
45 raw_spinlock_t tlbstate_lock; 38};
46 DECLARE_BITMAP(flush_cpumask, NR_CPUS);
47 };
48 char pad[INTERNODE_CACHE_BYTES];
49} ____cacheline_internodealigned_in_smp;
50
51/* State is put into the per CPU data section, but padded
52 to a full cache line because other CPUs can access it and we don't
53 want false sharing in the per cpu data segment. */
54static union smp_flush_state flush_state[NUM_INVALIDATE_TLB_VECTORS];
55
56static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);
57 39
58/* 40/*
59 * We cannot call mmdrop() because we are in interrupt context, 41 * We cannot call mmdrop() because we are in interrupt context,
@@ -72,28 +54,25 @@ void leave_mm(int cpu)
72EXPORT_SYMBOL_GPL(leave_mm); 54EXPORT_SYMBOL_GPL(leave_mm);
73 55
74/* 56/*
75 *
76 * The flush IPI assumes that a thread switch happens in this order: 57 * The flush IPI assumes that a thread switch happens in this order:
77 * [cpu0: the cpu that switches] 58 * [cpu0: the cpu that switches]
78 * 1) switch_mm() either 1a) or 1b) 59 * 1) switch_mm() either 1a) or 1b)
79 * 1a) thread switch to a different mm 60 * 1a) thread switch to a different mm
80 * 1a1) cpu_clear(cpu, old_mm->cpu_vm_mask); 61 * 1a1) set cpu_tlbstate to TLBSTATE_OK
81 * Stop ipi delivery for the old mm. This is not synchronized with 62 * Now the tlb flush NMI handler flush_tlb_func won't call leave_mm
82 * the other cpus, but smp_invalidate_interrupt ignore flush ipis 63 * if cpu0 was in lazy tlb mode.
83 * for the wrong mm, and in the worst case we perform a superfluous 64 * 1a2) update cpu active_mm
84 * tlb flush.
85 * 1a2) set cpu mmu_state to TLBSTATE_OK
86 * Now the smp_invalidate_interrupt won't call leave_mm if cpu0
87 * was in lazy tlb mode.
88 * 1a3) update cpu active_mm
89 * Now cpu0 accepts tlb flushes for the new mm. 65 * Now cpu0 accepts tlb flushes for the new mm.
90 * 1a4) cpu_set(cpu, new_mm->cpu_vm_mask); 66 * 1a3) cpu_set(cpu, new_mm->cpu_vm_mask);
91 * Now the other cpus will send tlb flush ipis. 67 * Now the other cpus will send tlb flush ipis.
92 * 1a4) change cr3. 68 * 1a4) change cr3.
69 * 1a5) cpu_clear(cpu, old_mm->cpu_vm_mask);
70 * Stop ipi delivery for the old mm. This is not synchronized with
71 * the other cpus, but flush_tlb_func ignore flush ipis for the wrong
72 * mm, and in the worst case we perform a superfluous tlb flush.
93 * 1b) thread switch without mm change 73 * 1b) thread switch without mm change
94 * cpu active_mm is correct, cpu0 already handles 74 * cpu active_mm is correct, cpu0 already handles flush ipis.
95 * flush ipis. 75 * 1b1) set cpu_tlbstate to TLBSTATE_OK
96 * 1b1) set cpu mmu_state to TLBSTATE_OK
97 * 1b2) test_and_set the cpu bit in cpu_vm_mask. 76 * 1b2) test_and_set the cpu bit in cpu_vm_mask.
98 * Atomically set the bit [other cpus will start sending flush ipis], 77 * Atomically set the bit [other cpus will start sending flush ipis],
99 * and test the bit. 78 * and test the bit.
@@ -106,174 +85,62 @@ EXPORT_SYMBOL_GPL(leave_mm);
106 * runs in kernel space, the cpu could load tlb entries for user space 85 * runs in kernel space, the cpu could load tlb entries for user space
107 * pages. 86 * pages.
108 * 87 *
109 * The good news is that cpu mmu_state is local to each cpu, no 88 * The good news is that cpu_tlbstate is local to each cpu, no
110 * write/read ordering problems. 89 * write/read ordering problems.
111 */ 90 */
112 91
113/* 92/*
114 * TLB flush IPI: 93 * TLB flush funcation:
115 *
116 * 1) Flush the tlb entries if the cpu uses the mm that's being flushed. 94 * 1) Flush the tlb entries if the cpu uses the mm that's being flushed.
117 * 2) Leave the mm if we are in the lazy tlb mode. 95 * 2) Leave the mm if we are in the lazy tlb mode.
118 *
119 * Interrupts are disabled.
120 */
121
122/*
123 * FIXME: use of asmlinkage is not consistent. On x86_64 it's noop
124 * but still used for documentation purpose but the usage is slightly
125 * inconsistent. On x86_32, asmlinkage is regparm(0) but interrupt
126 * entry calls in with the first parameter in %eax. Maybe define
127 * intrlinkage?
128 */ 96 */
129#ifdef CONFIG_X86_64 97static void flush_tlb_func(void *info)
130asmlinkage
131#endif
132void smp_invalidate_interrupt(struct pt_regs *regs)
133{ 98{
134 unsigned int cpu; 99 struct flush_tlb_info *f = info;
135 unsigned int sender;
136 union smp_flush_state *f;
137
138 cpu = smp_processor_id();
139 /*
140 * orig_rax contains the negated interrupt vector.
141 * Use that to determine where the sender put the data.
142 */
143 sender = ~regs->orig_ax - INVALIDATE_TLB_VECTOR_START;
144 f = &flush_state[sender];
145
146 if (!cpumask_test_cpu(cpu, to_cpumask(f->flush_cpumask)))
147 goto out;
148 /*
149 * This was a BUG() but until someone can quote me the
150 * line from the intel manual that guarantees an IPI to
151 * multiple CPUs is retried _only_ on the erroring CPUs
152 * its staying as a return
153 *
154 * BUG();
155 */
156
157 if (f->flush_mm == this_cpu_read(cpu_tlbstate.active_mm)) {
158 if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
159 if (f->flush_va == TLB_FLUSH_ALL)
160 local_flush_tlb();
161 else
162 __flush_tlb_one(f->flush_va);
163 } else
164 leave_mm(cpu);
165 }
166out:
167 ack_APIC_irq();
168 smp_mb__before_clear_bit();
169 cpumask_clear_cpu(cpu, to_cpumask(f->flush_cpumask));
170 smp_mb__after_clear_bit();
171 inc_irq_stat(irq_tlb_count);
172}
173 100
174static void flush_tlb_others_ipi(const struct cpumask *cpumask, 101 if (f->flush_mm != this_cpu_read(cpu_tlbstate.active_mm))
175 struct mm_struct *mm, unsigned long va) 102 return;
176{ 103
177 unsigned int sender; 104 if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
178 union smp_flush_state *f; 105 if (f->flush_end == TLB_FLUSH_ALL || !cpu_has_invlpg)
179 106 local_flush_tlb();
180 /* Caller has disabled preemption */ 107 else if (!f->flush_end)
181 sender = this_cpu_read(tlb_vector_offset); 108 __flush_tlb_single(f->flush_start);
182 f = &flush_state[sender]; 109 else {
183 110 unsigned long addr;
184 if (nr_cpu_ids > NUM_INVALIDATE_TLB_VECTORS) 111 addr = f->flush_start;
185 raw_spin_lock(&f->tlbstate_lock); 112 while (addr < f->flush_end) {
186 113 __flush_tlb_single(addr);
187 f->flush_mm = mm; 114 addr += PAGE_SIZE;
188 f->flush_va = va; 115 }
189 if (cpumask_andnot(to_cpumask(f->flush_cpumask), cpumask, cpumask_of(smp_processor_id()))) { 116 }
190 /* 117 } else
191 * We have to send the IPI only to 118 leave_mm(smp_processor_id());
192 * CPUs affected.
193 */
194 apic->send_IPI_mask(to_cpumask(f->flush_cpumask),
195 INVALIDATE_TLB_VECTOR_START + sender);
196
197 while (!cpumask_empty(to_cpumask(f->flush_cpumask)))
198 cpu_relax();
199 }
200 119
201 f->flush_mm = NULL;
202 f->flush_va = 0;
203 if (nr_cpu_ids > NUM_INVALIDATE_TLB_VECTORS)
204 raw_spin_unlock(&f->tlbstate_lock);
205} 120}
206 121
207void native_flush_tlb_others(const struct cpumask *cpumask, 122void native_flush_tlb_others(const struct cpumask *cpumask,
208 struct mm_struct *mm, unsigned long va) 123 struct mm_struct *mm, unsigned long start,
124 unsigned long end)
209{ 125{
126 struct flush_tlb_info info;
127 info.flush_mm = mm;
128 info.flush_start = start;
129 info.flush_end = end;
130
210 if (is_uv_system()) { 131 if (is_uv_system()) {
211 unsigned int cpu; 132 unsigned int cpu;
212 133
213 cpu = smp_processor_id(); 134 cpu = smp_processor_id();
214 cpumask = uv_flush_tlb_others(cpumask, mm, va, cpu); 135 cpumask = uv_flush_tlb_others(cpumask, mm, start, end, cpu);
215 if (cpumask) 136 if (cpumask)
216 flush_tlb_others_ipi(cpumask, mm, va); 137 smp_call_function_many(cpumask, flush_tlb_func,
138 &info, 1);
217 return; 139 return;
218 } 140 }
219 flush_tlb_others_ipi(cpumask, mm, va); 141 smp_call_function_many(cpumask, flush_tlb_func, &info, 1);
220} 142}
221 143
222static void __cpuinit calculate_tlb_offset(void)
223{
224 int cpu, node, nr_node_vecs, idx = 0;
225 /*
226 * we are changing tlb_vector_offset for each CPU in runtime, but this
227 * will not cause inconsistency, as the write is atomic under X86. we
228 * might see more lock contentions in a short time, but after all CPU's
229 * tlb_vector_offset are changed, everything should go normal
230 *
231 * Note: if NUM_INVALIDATE_TLB_VECTORS % nr_online_nodes !=0, we might
232 * waste some vectors.
233 **/
234 if (nr_online_nodes > NUM_INVALIDATE_TLB_VECTORS)
235 nr_node_vecs = 1;
236 else
237 nr_node_vecs = NUM_INVALIDATE_TLB_VECTORS/nr_online_nodes;
238
239 for_each_online_node(node) {
240 int node_offset = (idx % NUM_INVALIDATE_TLB_VECTORS) *
241 nr_node_vecs;
242 int cpu_offset = 0;
243 for_each_cpu(cpu, cpumask_of_node(node)) {
244 per_cpu(tlb_vector_offset, cpu) = node_offset +
245 cpu_offset;
246 cpu_offset++;
247 cpu_offset = cpu_offset % nr_node_vecs;
248 }
249 idx++;
250 }
251}
252
253static int __cpuinit tlb_cpuhp_notify(struct notifier_block *n,
254 unsigned long action, void *hcpu)
255{
256 switch (action & 0xf) {
257 case CPU_ONLINE:
258 case CPU_DEAD:
259 calculate_tlb_offset();
260 }
261 return NOTIFY_OK;
262}
263
264static int __cpuinit init_smp_flush(void)
265{
266 int i;
267
268 for (i = 0; i < ARRAY_SIZE(flush_state); i++)
269 raw_spin_lock_init(&flush_state[i].tlbstate_lock);
270
271 calculate_tlb_offset();
272 hotcpu_notifier(tlb_cpuhp_notify, 0);
273 return 0;
274}
275core_initcall(init_smp_flush);
276
277void flush_tlb_current_task(void) 144void flush_tlb_current_task(void)
278{ 145{
279 struct mm_struct *mm = current->mm; 146 struct mm_struct *mm = current->mm;
@@ -282,27 +149,91 @@ void flush_tlb_current_task(void)
282 149
283 local_flush_tlb(); 150 local_flush_tlb();
284 if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids) 151 if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
285 flush_tlb_others(mm_cpumask(mm), mm, TLB_FLUSH_ALL); 152 flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
286 preempt_enable(); 153 preempt_enable();
287} 154}
288 155
289void flush_tlb_mm(struct mm_struct *mm) 156/*
157 * It can find out the THP large page, or
158 * HUGETLB page in tlb_flush when THP disabled
159 */
160static inline unsigned long has_large_page(struct mm_struct *mm,
161 unsigned long start, unsigned long end)
162{
163 pgd_t *pgd;
164 pud_t *pud;
165 pmd_t *pmd;
166 unsigned long addr = ALIGN(start, HPAGE_SIZE);
167 for (; addr < end; addr += HPAGE_SIZE) {
168 pgd = pgd_offset(mm, addr);
169 if (likely(!pgd_none(*pgd))) {
170 pud = pud_offset(pgd, addr);
171 if (likely(!pud_none(*pud))) {
172 pmd = pmd_offset(pud, addr);
173 if (likely(!pmd_none(*pmd)))
174 if (pmd_large(*pmd))
175 return addr;
176 }
177 }
178 }
179 return 0;
180}
181
182void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
183 unsigned long end, unsigned long vmflag)
290{ 184{
185 unsigned long addr;
186 unsigned act_entries, tlb_entries = 0;
187
291 preempt_disable(); 188 preempt_disable();
189 if (current->active_mm != mm)
190 goto flush_all;
292 191
293 if (current->active_mm == mm) { 192 if (!current->mm) {
294 if (current->mm) 193 leave_mm(smp_processor_id());
194 goto flush_all;
195 }
196
197 if (end == TLB_FLUSH_ALL || tlb_flushall_shift == -1
198 || vmflag == VM_HUGETLB) {
199 local_flush_tlb();
200 goto flush_all;
201 }
202
203 /* In modern CPU, last level tlb used for both data/ins */
204 if (vmflag & VM_EXEC)
205 tlb_entries = tlb_lli_4k[ENTRIES];
206 else
207 tlb_entries = tlb_lld_4k[ENTRIES];
208 /* Assume all of TLB entries was occupied by this task */
209 act_entries = mm->total_vm > tlb_entries ? tlb_entries : mm->total_vm;
210
211 /* tlb_flushall_shift is on balance point, details in commit log */
212 if ((end - start) >> PAGE_SHIFT > act_entries >> tlb_flushall_shift)
213 local_flush_tlb();
214 else {
215 if (has_large_page(mm, start, end)) {
295 local_flush_tlb(); 216 local_flush_tlb();
296 else 217 goto flush_all;
297 leave_mm(smp_processor_id()); 218 }
219 /* flush range by one by one 'invlpg' */
220 for (addr = start; addr < end; addr += PAGE_SIZE)
221 __flush_tlb_single(addr);
222
223 if (cpumask_any_but(mm_cpumask(mm),
224 smp_processor_id()) < nr_cpu_ids)
225 flush_tlb_others(mm_cpumask(mm), mm, start, end);
226 preempt_enable();
227 return;
298 } 228 }
299 if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
300 flush_tlb_others(mm_cpumask(mm), mm, TLB_FLUSH_ALL);
301 229
230flush_all:
231 if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
232 flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
302 preempt_enable(); 233 preempt_enable();
303} 234}
304 235
305void flush_tlb_page(struct vm_area_struct *vma, unsigned long va) 236void flush_tlb_page(struct vm_area_struct *vma, unsigned long start)
306{ 237{
307 struct mm_struct *mm = vma->vm_mm; 238 struct mm_struct *mm = vma->vm_mm;
308 239
@@ -310,13 +241,13 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
310 241
311 if (current->active_mm == mm) { 242 if (current->active_mm == mm) {
312 if (current->mm) 243 if (current->mm)
313 __flush_tlb_one(va); 244 __flush_tlb_one(start);
314 else 245 else
315 leave_mm(smp_processor_id()); 246 leave_mm(smp_processor_id());
316 } 247 }
317 248
318 if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids) 249 if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
319 flush_tlb_others(mm_cpumask(mm), mm, va); 250 flush_tlb_others(mm_cpumask(mm), mm, start, 0UL);
320 251
321 preempt_enable(); 252 preempt_enable();
322} 253}
@@ -332,3 +263,83 @@ void flush_tlb_all(void)
332{ 263{
333 on_each_cpu(do_flush_tlb_all, NULL, 1); 264 on_each_cpu(do_flush_tlb_all, NULL, 1);
334} 265}
266
267static void do_kernel_range_flush(void *info)
268{
269 struct flush_tlb_info *f = info;
270 unsigned long addr;
271
272 /* flush range by one by one 'invlpg' */
273 for (addr = f->flush_start; addr < f->flush_end; addr += PAGE_SIZE)
274 __flush_tlb_single(addr);
275}
276
277void flush_tlb_kernel_range(unsigned long start, unsigned long end)
278{
279 unsigned act_entries;
280 struct flush_tlb_info info;
281
282 /* In modern CPU, last level tlb used for both data/ins */
283 act_entries = tlb_lld_4k[ENTRIES];
284
285 /* Balance as user space task's flush, a bit conservative */
286 if (end == TLB_FLUSH_ALL || tlb_flushall_shift == -1 ||
287 (end - start) >> PAGE_SHIFT > act_entries >> tlb_flushall_shift)
288
289 on_each_cpu(do_flush_tlb_all, NULL, 1);
290 else {
291 info.flush_start = start;
292 info.flush_end = end;
293 on_each_cpu(do_kernel_range_flush, &info, 1);
294 }
295}
296
297#ifdef CONFIG_DEBUG_TLBFLUSH
298static ssize_t tlbflush_read_file(struct file *file, char __user *user_buf,
299 size_t count, loff_t *ppos)
300{
301 char buf[32];
302 unsigned int len;
303
304 len = sprintf(buf, "%hd\n", tlb_flushall_shift);
305 return simple_read_from_buffer(user_buf, count, ppos, buf, len);
306}
307
308static ssize_t tlbflush_write_file(struct file *file,
309 const char __user *user_buf, size_t count, loff_t *ppos)
310{
311 char buf[32];
312 ssize_t len;
313 s8 shift;
314
315 len = min(count, sizeof(buf) - 1);
316 if (copy_from_user(buf, user_buf, len))
317 return -EFAULT;
318
319 buf[len] = '\0';
320 if (kstrtos8(buf, 0, &shift))
321 return -EINVAL;
322
323 if (shift > 64)
324 return -EINVAL;
325
326 tlb_flushall_shift = shift;
327 return count;
328}
329
330static const struct file_operations fops_tlbflush = {
331 .read = tlbflush_read_file,
332 .write = tlbflush_write_file,
333 .llseek = default_llseek,
334};
335
336static int __cpuinit create_tlb_flushall_shift(void)
337{
338 if (cpu_has_invlpg) {
339 debugfs_create_file("tlb_flushall_shift", S_IRUSR | S_IWUSR,
340 arch_debugfs_dir, NULL, &fops_tlbflush);
341 }
342 return 0;
343}
344late_initcall(create_tlb_flushall_shift);
345#endif
diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c
index 92660edaa1e7..2dc29f51e75a 100644
--- a/arch/x86/platform/efi/efi.c
+++ b/arch/x86/platform/efi/efi.c
@@ -234,22 +234,7 @@ static efi_status_t __init phys_efi_set_virtual_address_map(
234 return status; 234 return status;
235} 235}
236 236
237static efi_status_t __init phys_efi_get_time(efi_time_t *tm, 237static int efi_set_rtc_mmss(unsigned long nowtime)
238 efi_time_cap_t *tc)
239{
240 unsigned long flags;
241 efi_status_t status;
242
243 spin_lock_irqsave(&rtc_lock, flags);
244 efi_call_phys_prelog();
245 status = efi_call_phys2(efi_phys.get_time, virt_to_phys(tm),
246 virt_to_phys(tc));
247 efi_call_phys_epilog();
248 spin_unlock_irqrestore(&rtc_lock, flags);
249 return status;
250}
251
252int efi_set_rtc_mmss(unsigned long nowtime)
253{ 238{
254 int real_seconds, real_minutes; 239 int real_seconds, real_minutes;
255 efi_status_t status; 240 efi_status_t status;
@@ -278,7 +263,7 @@ int efi_set_rtc_mmss(unsigned long nowtime)
278 return 0; 263 return 0;
279} 264}
280 265
281unsigned long efi_get_time(void) 266static unsigned long efi_get_time(void)
282{ 267{
283 efi_status_t status; 268 efi_status_t status;
284 efi_time_t eft; 269 efi_time_t eft;
@@ -621,18 +606,13 @@ static int __init efi_runtime_init(void)
621 } 606 }
622 /* 607 /*
623 * We will only need *early* access to the following 608 * We will only need *early* access to the following
624 * two EFI runtime services before set_virtual_address_map 609 * EFI runtime service before set_virtual_address_map
625 * is invoked. 610 * is invoked.
626 */ 611 */
627 efi_phys.get_time = (efi_get_time_t *)runtime->get_time;
628 efi_phys.set_virtual_address_map = 612 efi_phys.set_virtual_address_map =
629 (efi_set_virtual_address_map_t *) 613 (efi_set_virtual_address_map_t *)
630 runtime->set_virtual_address_map; 614 runtime->set_virtual_address_map;
631 /* 615
632 * Make efi_get_time can be called before entering
633 * virtual mode.
634 */
635 efi.get_time = phys_efi_get_time;
636 early_iounmap(runtime, sizeof(efi_runtime_services_t)); 616 early_iounmap(runtime, sizeof(efi_runtime_services_t));
637 617
638 return 0; 618 return 0;
@@ -720,12 +700,10 @@ void __init efi_init(void)
720 efi_enabled = 0; 700 efi_enabled = 0;
721 return; 701 return;
722 } 702 }
723#ifdef CONFIG_X86_32
724 if (efi_native) { 703 if (efi_native) {
725 x86_platform.get_wallclock = efi_get_time; 704 x86_platform.get_wallclock = efi_get_time;
726 x86_platform.set_wallclock = efi_set_rtc_mmss; 705 x86_platform.set_wallclock = efi_set_rtc_mmss;
727 } 706 }
728#endif
729 707
730#if EFI_DEBUG 708#if EFI_DEBUG
731 print_efi_memmap(); 709 print_efi_memmap();
diff --git a/arch/x86/platform/uv/tlb_uv.c b/arch/x86/platform/uv/tlb_uv.c
index 71b5d5a07d7b..b8b3a37c80cd 100644
--- a/arch/x86/platform/uv/tlb_uv.c
+++ b/arch/x86/platform/uv/tlb_uv.c
@@ -1055,8 +1055,8 @@ static int set_distrib_bits(struct cpumask *flush_mask, struct bau_control *bcp,
1055 * done. The returned pointer is valid till preemption is re-enabled. 1055 * done. The returned pointer is valid till preemption is re-enabled.
1056 */ 1056 */
1057const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask, 1057const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
1058 struct mm_struct *mm, unsigned long va, 1058 struct mm_struct *mm, unsigned long start,
1059 unsigned int cpu) 1059 unsigned end, unsigned int cpu)
1060{ 1060{
1061 int locals = 0; 1061 int locals = 0;
1062 int remotes = 0; 1062 int remotes = 0;
@@ -1113,7 +1113,7 @@ const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
1113 1113
1114 record_send_statistics(stat, locals, hubs, remotes, bau_desc); 1114 record_send_statistics(stat, locals, hubs, remotes, bau_desc);
1115 1115
1116 bau_desc->payload.address = va; 1116 bau_desc->payload.address = start;
1117 bau_desc->payload.sending_cpu = cpu; 1117 bau_desc->payload.sending_cpu = cpu;
1118 /* 1118 /*
1119 * uv_flush_send_and_wait returns 0 if all cpu's were messaged, 1119 * uv_flush_send_and_wait returns 0 if all cpu's were messaged,
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 27336dfcda8e..b65a76133f4f 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1256,7 +1256,8 @@ static void xen_flush_tlb_single(unsigned long addr)
1256} 1256}
1257 1257
1258static void xen_flush_tlb_others(const struct cpumask *cpus, 1258static void xen_flush_tlb_others(const struct cpumask *cpus,
1259 struct mm_struct *mm, unsigned long va) 1259 struct mm_struct *mm, unsigned long start,
1260 unsigned long end)
1260{ 1261{
1261 struct { 1262 struct {
1262 struct mmuext_op op; 1263 struct mmuext_op op;
@@ -1268,7 +1269,7 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
1268 } *args; 1269 } *args;
1269 struct multicall_space mcs; 1270 struct multicall_space mcs;
1270 1271
1271 trace_xen_mmu_flush_tlb_others(cpus, mm, va); 1272 trace_xen_mmu_flush_tlb_others(cpus, mm, start, end);
1272 1273
1273 if (cpumask_empty(cpus)) 1274 if (cpumask_empty(cpus))
1274 return; /* nothing to do */ 1275 return; /* nothing to do */
@@ -1281,11 +1282,10 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
1281 cpumask_and(to_cpumask(args->mask), cpus, cpu_online_mask); 1282 cpumask_and(to_cpumask(args->mask), cpus, cpu_online_mask);
1282 cpumask_clear_cpu(smp_processor_id(), to_cpumask(args->mask)); 1283 cpumask_clear_cpu(smp_processor_id(), to_cpumask(args->mask));
1283 1284
1284 if (va == TLB_FLUSH_ALL) { 1285 args->op.cmd = MMUEXT_TLB_FLUSH_MULTI;
1285 args->op.cmd = MMUEXT_TLB_FLUSH_MULTI; 1286 if (start != TLB_FLUSH_ALL && (end - start) <= PAGE_SIZE) {
1286 } else {
1287 args->op.cmd = MMUEXT_INVLPG_MULTI; 1287 args->op.cmd = MMUEXT_INVLPG_MULTI;
1288 args->op.arg1.linear_addr = va; 1288 args->op.arg1.linear_addr = start;
1289 } 1289 }
1290 1290
1291 MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF); 1291 MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF);
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-27 12:42:26 -0500