aboutsummaryrefslogtreecommitdiffstats
path: root/arch
diff options
context:
space:
mode:
Diffstat (limited to 'arch')
-rw-r--r--arch/alpha/kernel/vmlinux.lds.S14
-rw-r--r--arch/arm/common/dmabounce.c3
-rw-r--r--arch/arm/mach-omap2/gpmc.c2
-rw-r--r--arch/arm/mach-realview/platsmp.c2
-rw-r--r--arch/arm/plat-omap/dma.c2
-rw-r--r--arch/blackfin/kernel/cplb-nompu/cplbinit.c6
-rw-r--r--arch/blackfin/kernel/irqchip.c5
-rw-r--r--arch/ia64/kernel/setup.c3
-rw-r--r--arch/ia64/kernel/time.c1
-rw-r--r--arch/mips/Kconfig2
-rw-r--r--arch/mips/kernel/cevt-txx9.c3
-rw-r--r--arch/mips/mm/c-r3k.c6
-rw-r--r--arch/mips/mm/page.c61
-rw-r--r--arch/mips/mm/sc-rm7k.c4
-rw-r--r--arch/mips/sgi-ip32/ip32-irq.c5
-rw-r--r--arch/mn10300/kernel/mn10300_ksyms.c5
-rw-r--r--arch/mn10300/kernel/process.c1
-rw-r--r--arch/mn10300/lib/Makefile2
-rw-r--r--arch/mn10300/lib/__ucmpdi2.S43
-rw-r--r--arch/powerpc/boot/Makefile3
-rw-r--r--arch/powerpc/kernel/legacy_serial.c10
-rw-r--r--arch/powerpc/kernel/of_platform.c2
-rw-r--r--arch/powerpc/platforms/52xx/lite5200_pm.c14
-rw-r--r--arch/um/Makefile1
-rw-r--r--arch/um/Makefile-i3867
-rw-r--r--arch/um/Makefile-x86_643
-rw-r--r--arch/x86/Kconfig4
-rw-r--r--arch/x86/Kconfig.debug32
-rw-r--r--arch/x86/kernel/acpi/realmode/wakeup.S38
-rw-r--r--arch/x86/kernel/acpi/realmode/wakeup.h5
-rw-r--r--arch/x86/kernel/acpi/sleep.c16
-rw-r--r--arch/x86/kernel/efi_32.c8
-rw-r--r--arch/x86/kernel/head_64.S2
-rw-r--r--arch/x86/kernel/i387.c4
-rw-r--r--arch/x86/kernel/smpboot.c1
-rw-r--r--arch/x86/mm/Makefile5
-rw-r--r--arch/x86/mm/fault.c57
-rw-r--r--arch/x86/mm/init_64.c7
-rw-r--r--arch/x86/mm/ioremap.c11
-rw-r--r--arch/x86/mm/kmmio.c510
-rw-r--r--arch/x86/mm/mmio-mod.c515
-rw-r--r--arch/x86/mm/pageattr.c1
-rw-r--r--arch/x86/mm/pf_in.c489
-rw-r--r--arch/x86/mm/pf_in.h39
-rw-r--r--arch/x86/mm/testmmiotrace.c71
-rw-r--r--arch/x86/pci/common.c8
-rw-r--r--arch/x86/xen/mmu.c4
47 files changed, 1904 insertions, 133 deletions
diff --git a/arch/alpha/kernel/vmlinux.lds.S b/arch/alpha/kernel/vmlinux.lds.S
index f13249be17c5..ef37fc1acaea 100644
--- a/arch/alpha/kernel/vmlinux.lds.S
+++ b/arch/alpha/kernel/vmlinux.lds.S
@@ -25,6 +25,13 @@ SECTIONS
25 } :kernel 25 } :kernel
26 _etext = .; /* End of text section */ 26 _etext = .; /* End of text section */
27 27
28 NOTES :kernel :note
29 .dummy : {
30 *(.dummy)
31 } :kernel
32
33 RODATA
34
28 /* Exception table */ 35 /* Exception table */
29 . = ALIGN(16); 36 . = ALIGN(16);
30 __ex_table : { 37 __ex_table : {
@@ -33,13 +40,6 @@ SECTIONS
33 __stop___ex_table = .; 40 __stop___ex_table = .;
34 } 41 }
35 42
36 NOTES :kernel :note
37 .dummy : {
38 *(.dummy)
39 } :kernel
40
41 RODATA
42
43 /* Will be freed after init */ 43 /* Will be freed after init */
44 . = ALIGN(PAGE_SIZE); 44 . = ALIGN(PAGE_SIZE);
45 /* Init code and data */ 45 /* Init code and data */
diff --git a/arch/arm/common/dmabounce.c b/arch/arm/common/dmabounce.c
index 52fc6a883281..2744673314b4 100644
--- a/arch/arm/common/dmabounce.c
+++ b/arch/arm/common/dmabounce.c
@@ -650,7 +650,8 @@ EXPORT_SYMBOL(dma_map_sg);
650EXPORT_SYMBOL(dma_unmap_sg); 650EXPORT_SYMBOL(dma_unmap_sg);
651EXPORT_SYMBOL(dma_sync_single_for_cpu); 651EXPORT_SYMBOL(dma_sync_single_for_cpu);
652EXPORT_SYMBOL(dma_sync_single_for_device); 652EXPORT_SYMBOL(dma_sync_single_for_device);
653EXPORT_SYMBOL(dma_sync_sg); 653EXPORT_SYMBOL(dma_sync_sg_for_cpu);
654EXPORT_SYMBOL(dma_sync_sg_for_device);
654EXPORT_SYMBOL(dmabounce_register_dev); 655EXPORT_SYMBOL(dmabounce_register_dev);
655EXPORT_SYMBOL(dmabounce_unregister_dev); 656EXPORT_SYMBOL(dmabounce_unregister_dev);
656 657
diff --git a/arch/arm/mach-omap2/gpmc.c b/arch/arm/mach-omap2/gpmc.c
index 02cede295e89..dbf68dc50ae2 100644
--- a/arch/arm/mach-omap2/gpmc.c
+++ b/arch/arm/mach-omap2/gpmc.c
@@ -42,7 +42,7 @@
42#define GPMC_STATUS 0x54 42#define GPMC_STATUS 0x54
43#define GPMC_PREFETCH_CONFIG1 0x1e0 43#define GPMC_PREFETCH_CONFIG1 0x1e0
44#define GPMC_PREFETCH_CONFIG2 0x1e4 44#define GPMC_PREFETCH_CONFIG2 0x1e4
45#define GPMC_PREFETCH_CONTROL 0x1e8 45#define GPMC_PREFETCH_CONTROL 0x1ec
46#define GPMC_PREFETCH_STATUS 0x1f0 46#define GPMC_PREFETCH_STATUS 0x1f0
47#define GPMC_ECC_CONFIG 0x1f4 47#define GPMC_ECC_CONFIG 0x1f4
48#define GPMC_ECC_CONTROL 0x1f8 48#define GPMC_ECC_CONTROL 0x1f8
diff --git a/arch/arm/mach-realview/platsmp.c b/arch/arm/mach-realview/platsmp.c
index 3e57428affee..8e813ed57519 100644
--- a/arch/arm/mach-realview/platsmp.c
+++ b/arch/arm/mach-realview/platsmp.c
@@ -74,6 +74,8 @@ static DEFINE_SPINLOCK(boot_lock);
74 74
75void __cpuinit platform_secondary_init(unsigned int cpu) 75void __cpuinit platform_secondary_init(unsigned int cpu)
76{ 76{
77 trace_hardirqs_off();
78
77 /* 79 /*
78 * the primary core may have used a "cross call" soft interrupt 80 * the primary core may have used a "cross call" soft interrupt
79 * to get this processor out of WFI in the BootMonitor - make 81 * to get this processor out of WFI in the BootMonitor - make
diff --git a/arch/arm/plat-omap/dma.c b/arch/arm/plat-omap/dma.c
index c00eda588cd8..39c637b0ffea 100644
--- a/arch/arm/plat-omap/dma.c
+++ b/arch/arm/plat-omap/dma.c
@@ -501,8 +501,6 @@ static inline void omap_enable_channel_irq(int lch)
501 501
502 /* Enable some nice interrupts. */ 502 /* Enable some nice interrupts. */
503 OMAP_DMA_CICR_REG(lch) = dma_chan[lch].enabled_irqs; 503 OMAP_DMA_CICR_REG(lch) = dma_chan[lch].enabled_irqs;
504
505 dma_chan[lch].flags |= OMAP_DMA_ACTIVE;
506} 504}
507 505
508static void omap_disable_channel_irq(int lch) 506static void omap_disable_channel_irq(int lch)
diff --git a/arch/blackfin/kernel/cplb-nompu/cplbinit.c b/arch/blackfin/kernel/cplb-nompu/cplbinit.c
index 917325bfbd84..6be0c50122e8 100644
--- a/arch/blackfin/kernel/cplb-nompu/cplbinit.c
+++ b/arch/blackfin/kernel/cplb-nompu/cplbinit.c
@@ -254,7 +254,8 @@ close_cplbtab(struct cplb_tab *table)
254} 254}
255 255
256/* helper function */ 256/* helper function */
257static void __fill_code_cplbtab(struct cplb_tab *t, int i, u32 a_start, u32 a_end) 257static void __init
258__fill_code_cplbtab(struct cplb_tab *t, int i, u32 a_start, u32 a_end)
258{ 259{
259 if (cplb_data[i].psize) { 260 if (cplb_data[i].psize) {
260 fill_cplbtab(t, 261 fill_cplbtab(t,
@@ -291,7 +292,8 @@ static void __fill_code_cplbtab(struct cplb_tab *t, int i, u32 a_start, u32 a_en
291 } 292 }
292} 293}
293 294
294static void __fill_data_cplbtab(struct cplb_tab *t, int i, u32 a_start, u32 a_end) 295static void __init
296__fill_data_cplbtab(struct cplb_tab *t, int i, u32 a_start, u32 a_end)
295{ 297{
296 if (cplb_data[i].psize) { 298 if (cplb_data[i].psize) {
297 fill_cplbtab(t, 299 fill_cplbtab(t,
diff --git a/arch/blackfin/kernel/irqchip.c b/arch/blackfin/kernel/irqchip.c
index 73647c158774..07402f57c9de 100644
--- a/arch/blackfin/kernel/irqchip.c
+++ b/arch/blackfin/kernel/irqchip.c
@@ -60,9 +60,14 @@ static struct irq_chip bad_chip = {
60}; 60};
61 61
62static struct irq_desc bad_irq_desc = { 62static struct irq_desc bad_irq_desc = {
63 .status = IRQ_DISABLED,
63 .chip = &bad_chip, 64 .chip = &bad_chip,
64 .handle_irq = handle_bad_irq, 65 .handle_irq = handle_bad_irq,
65 .depth = 1, 66 .depth = 1,
67 .lock = __SPIN_LOCK_UNLOCKED(irq_desc->lock),
68#ifdef CONFIG_SMP
69 .affinity = CPU_MASK_ALL
70#endif
66}; 71};
67 72
68int show_interrupts(struct seq_file *p, void *v) 73int show_interrupts(struct seq_file *p, void *v)
diff --git a/arch/ia64/kernel/setup.c b/arch/ia64/kernel/setup.c
index 4ae15c8c2488..632cda8f2e76 100644
--- a/arch/ia64/kernel/setup.c
+++ b/arch/ia64/kernel/setup.c
@@ -547,7 +547,8 @@ setup_arch (char **cmdline_p)
547# ifdef CONFIG_ACPI_NUMA 547# ifdef CONFIG_ACPI_NUMA
548 acpi_numa_init(); 548 acpi_numa_init();
549 per_cpu_scan_finalize((cpus_weight(early_cpu_possible_map) == 0 ? 549 per_cpu_scan_finalize((cpus_weight(early_cpu_possible_map) == 0 ?
550 32 : cpus_weight(early_cpu_possible_map)), additional_cpus); 550 32 : cpus_weight(early_cpu_possible_map)),
551 additional_cpus > 0 ? additional_cpus : 0);
551# endif 552# endif
552#else 553#else
553# ifdef CONFIG_SMP 554# ifdef CONFIG_SMP
diff --git a/arch/ia64/kernel/time.c b/arch/ia64/kernel/time.c
index 8c73643f2d66..aad1b7b1fff9 100644
--- a/arch/ia64/kernel/time.c
+++ b/arch/ia64/kernel/time.c
@@ -117,6 +117,7 @@ void account_system_vtime(struct task_struct *tsk)
117 117
118 local_irq_restore(flags); 118 local_irq_restore(flags);
119} 119}
120EXPORT_SYMBOL_GPL(account_system_vtime);
120 121
121/* 122/*
122 * Called from the timer interrupt handler to charge accumulated user time 123 * Called from the timer interrupt handler to charge accumulated user time
diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig
index e5a7c5d96364..24c5dee91768 100644
--- a/arch/mips/Kconfig
+++ b/arch/mips/Kconfig
@@ -1006,7 +1006,7 @@ config BOOT_ELF32
1006config MIPS_L1_CACHE_SHIFT 1006config MIPS_L1_CACHE_SHIFT
1007 int 1007 int
1008 default "4" if MACH_DECSTATION 1008 default "4" if MACH_DECSTATION
1009 default "7" if SGI_IP27 || SGI_IP28 || SNI_RM 1009 default "7" if SGI_IP22 || SGI_IP27 || SGI_IP28 || SNI_RM
1010 default "4" if PMC_MSP4200_EVAL 1010 default "4" if PMC_MSP4200_EVAL
1011 default "5" 1011 default "5"
1012 1012
diff --git a/arch/mips/kernel/cevt-txx9.c b/arch/mips/kernel/cevt-txx9.c
index 795cb8fb0d74..b5fc4eb412d2 100644
--- a/arch/mips/kernel/cevt-txx9.c
+++ b/arch/mips/kernel/cevt-txx9.c
@@ -161,6 +161,9 @@ void __init txx9_tmr_init(unsigned long baseaddr)
161 struct txx9_tmr_reg __iomem *tmrptr; 161 struct txx9_tmr_reg __iomem *tmrptr;
162 162
163 tmrptr = ioremap(baseaddr, sizeof(struct txx9_tmr_reg)); 163 tmrptr = ioremap(baseaddr, sizeof(struct txx9_tmr_reg));
164 /* Start once to make CounterResetEnable effective */
165 __raw_writel(TXx9_TMTCR_CRE | TXx9_TMTCR_TCE, &tmrptr->tcr);
166 /* Stop and reset the counter */
164 __raw_writel(TXx9_TMTCR_CRE, &tmrptr->tcr); 167 __raw_writel(TXx9_TMTCR_CRE, &tmrptr->tcr);
165 __raw_writel(0, &tmrptr->tisr); 168 __raw_writel(0, &tmrptr->tisr);
166 __raw_writel(0xffffffff, &tmrptr->cpra); 169 __raw_writel(0xffffffff, &tmrptr->cpra);
diff --git a/arch/mips/mm/c-r3k.c b/arch/mips/mm/c-r3k.c
index 76935e320214..27a5b466c85c 100644
--- a/arch/mips/mm/c-r3k.c
+++ b/arch/mips/mm/c-r3k.c
@@ -26,7 +26,7 @@
26static unsigned long icache_size, dcache_size; /* Size in bytes */ 26static unsigned long icache_size, dcache_size; /* Size in bytes */
27static unsigned long icache_lsize, dcache_lsize; /* Size in bytes */ 27static unsigned long icache_lsize, dcache_lsize; /* Size in bytes */
28 28
29unsigned long __init r3k_cache_size(unsigned long ca_flags) 29unsigned long __cpuinit r3k_cache_size(unsigned long ca_flags)
30{ 30{
31 unsigned long flags, status, dummy, size; 31 unsigned long flags, status, dummy, size;
32 volatile unsigned long *p; 32 volatile unsigned long *p;
@@ -61,7 +61,7 @@ unsigned long __init r3k_cache_size(unsigned long ca_flags)
61 return size * sizeof(*p); 61 return size * sizeof(*p);
62} 62}
63 63
64unsigned long __init r3k_cache_lsize(unsigned long ca_flags) 64unsigned long __cpuinit r3k_cache_lsize(unsigned long ca_flags)
65{ 65{
66 unsigned long flags, status, lsize, i; 66 unsigned long flags, status, lsize, i;
67 volatile unsigned long *p; 67 volatile unsigned long *p;
@@ -90,7 +90,7 @@ unsigned long __init r3k_cache_lsize(unsigned long ca_flags)
90 return lsize * sizeof(*p); 90 return lsize * sizeof(*p);
91} 91}
92 92
93static void __init r3k_probe_cache(void) 93static void __cpuinit r3k_probe_cache(void)
94{ 94{
95 dcache_size = r3k_cache_size(ST0_ISC); 95 dcache_size = r3k_cache_size(ST0_ISC);
96 if (dcache_size) 96 if (dcache_size)
diff --git a/arch/mips/mm/page.c b/arch/mips/mm/page.c
index 1edf0cbbeede..1417c6494858 100644
--- a/arch/mips/mm/page.c
+++ b/arch/mips/mm/page.c
@@ -235,13 +235,12 @@ static void __cpuinit set_prefetch_parameters(void)
235 } 235 }
236 /* 236 /*
237 * Too much unrolling will overflow the available space in 237 * Too much unrolling will overflow the available space in
238 * clear_space_array / copy_page_array. 8 words sounds generous, 238 * clear_space_array / copy_page_array.
239 * but a R4000 with 128 byte L2 line length can exceed even that.
240 */ 239 */
241 half_clear_loop_size = min(8 * clear_word_size, 240 half_clear_loop_size = min(16 * clear_word_size,
242 max(cache_line_size >> 1, 241 max(cache_line_size >> 1,
243 4 * clear_word_size)); 242 4 * clear_word_size));
244 half_copy_loop_size = min(8 * copy_word_size, 243 half_copy_loop_size = min(16 * copy_word_size,
245 max(cache_line_size >> 1, 244 max(cache_line_size >> 1,
246 4 * copy_word_size)); 245 4 * copy_word_size));
247} 246}
@@ -263,21 +262,23 @@ static inline void __cpuinit build_clear_pref(u32 **buf, int off)
263 if (pref_bias_clear_store) { 262 if (pref_bias_clear_store) {
264 uasm_i_pref(buf, pref_dst_mode, pref_bias_clear_store + off, 263 uasm_i_pref(buf, pref_dst_mode, pref_bias_clear_store + off,
265 A0); 264 A0);
266 } else if (cpu_has_cache_cdex_s) { 265 } else if (cache_line_size == (half_clear_loop_size << 1)) {
267 uasm_i_cache(buf, Create_Dirty_Excl_SD, off, A0); 266 if (cpu_has_cache_cdex_s) {
268 } else if (cpu_has_cache_cdex_p) { 267 uasm_i_cache(buf, Create_Dirty_Excl_SD, off, A0);
269 if (R4600_V1_HIT_CACHEOP_WAR && cpu_is_r4600_v1_x()) { 268 } else if (cpu_has_cache_cdex_p) {
270 uasm_i_nop(buf); 269 if (R4600_V1_HIT_CACHEOP_WAR && cpu_is_r4600_v1_x()) {
271 uasm_i_nop(buf); 270 uasm_i_nop(buf);
272 uasm_i_nop(buf); 271 uasm_i_nop(buf);
273 uasm_i_nop(buf); 272 uasm_i_nop(buf);
274 } 273 uasm_i_nop(buf);
274 }
275 275
276 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) 276 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())
277 uasm_i_lw(buf, ZERO, ZERO, AT); 277 uasm_i_lw(buf, ZERO, ZERO, AT);
278 278
279 uasm_i_cache(buf, Create_Dirty_Excl_D, off, A0); 279 uasm_i_cache(buf, Create_Dirty_Excl_D, off, A0);
280 } 280 }
281 }
281} 282}
282 283
283void __cpuinit build_clear_page(void) 284void __cpuinit build_clear_page(void)
@@ -403,20 +404,22 @@ static inline void build_copy_store_pref(u32 **buf, int off)
403 if (pref_bias_copy_store) { 404 if (pref_bias_copy_store) {
404 uasm_i_pref(buf, pref_dst_mode, pref_bias_copy_store + off, 405 uasm_i_pref(buf, pref_dst_mode, pref_bias_copy_store + off,
405 A0); 406 A0);
406 } else if (cpu_has_cache_cdex_s) { 407 } else if (cache_line_size == (half_copy_loop_size << 1)) {
407 uasm_i_cache(buf, Create_Dirty_Excl_SD, off, A0); 408 if (cpu_has_cache_cdex_s) {
408 } else if (cpu_has_cache_cdex_p) { 409 uasm_i_cache(buf, Create_Dirty_Excl_SD, off, A0);
409 if (R4600_V1_HIT_CACHEOP_WAR && cpu_is_r4600_v1_x()) { 410 } else if (cpu_has_cache_cdex_p) {
410 uasm_i_nop(buf); 411 if (R4600_V1_HIT_CACHEOP_WAR && cpu_is_r4600_v1_x()) {
411 uasm_i_nop(buf); 412 uasm_i_nop(buf);
412 uasm_i_nop(buf); 413 uasm_i_nop(buf);
413 uasm_i_nop(buf); 414 uasm_i_nop(buf);
414 } 415 uasm_i_nop(buf);
416 }
415 417
416 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x()) 418 if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())
417 uasm_i_lw(buf, ZERO, ZERO, AT); 419 uasm_i_lw(buf, ZERO, ZERO, AT);
418 420
419 uasm_i_cache(buf, Create_Dirty_Excl_D, off, A0); 421 uasm_i_cache(buf, Create_Dirty_Excl_D, off, A0);
422 }
420 } 423 }
421} 424}
422 425
diff --git a/arch/mips/mm/sc-rm7k.c b/arch/mips/mm/sc-rm7k.c
index fc227f3b1199..e3abfb2d7e86 100644
--- a/arch/mips/mm/sc-rm7k.c
+++ b/arch/mips/mm/sc-rm7k.c
@@ -86,7 +86,7 @@ static void rm7k_sc_inv(unsigned long addr, unsigned long size)
86/* 86/*
87 * This function is executed in uncached address space. 87 * This function is executed in uncached address space.
88 */ 88 */
89static __init void __rm7k_sc_enable(void) 89static __cpuinit void __rm7k_sc_enable(void)
90{ 90{
91 int i; 91 int i;
92 92
@@ -107,7 +107,7 @@ static __init void __rm7k_sc_enable(void)
107 } 107 }
108} 108}
109 109
110static __init void rm7k_sc_enable(void) 110static __cpuinit void rm7k_sc_enable(void)
111{ 111{
112 if (read_c0_config() & RM7K_CONF_SE) 112 if (read_c0_config() & RM7K_CONF_SE)
113 return; 113 return;
diff --git a/arch/mips/sgi-ip32/ip32-irq.c b/arch/mips/sgi-ip32/ip32-irq.c
index b0ea0e43ba48..0d6b6663d5f6 100644
--- a/arch/mips/sgi-ip32/ip32-irq.c
+++ b/arch/mips/sgi-ip32/ip32-irq.c
@@ -425,6 +425,11 @@ static void ip32_irq0(void)
425 BUILD_BUG_ON(MACEISA_SERIAL2_RDMAOR_IRQ - MACEISA_AUDIO_SW_IRQ != 31); 425 BUILD_BUG_ON(MACEISA_SERIAL2_RDMAOR_IRQ - MACEISA_AUDIO_SW_IRQ != 31);
426 426
427 crime_int = crime->istat & crime_mask; 427 crime_int = crime->istat & crime_mask;
428
429 /* crime sometime delivers spurious interrupts, ignore them */
430 if (unlikely(crime_int == 0))
431 return;
432
428 irq = MACE_VID_IN1_IRQ + __ffs(crime_int); 433 irq = MACE_VID_IN1_IRQ + __ffs(crime_int);
429 434
430 if (crime_int & CRIME_MACEISA_INT_MASK) { 435 if (crime_int & CRIME_MACEISA_INT_MASK) {
diff --git a/arch/mn10300/kernel/mn10300_ksyms.c b/arch/mn10300/kernel/mn10300_ksyms.c
index 6d19628634e3..f9eb9753a404 100644
--- a/arch/mn10300/kernel/mn10300_ksyms.c
+++ b/arch/mn10300/kernel/mn10300_ksyms.c
@@ -10,8 +10,11 @@
10 */ 10 */
11#include <linux/module.h> 11#include <linux/module.h>
12#include <asm/uaccess.h> 12#include <asm/uaccess.h>
13#include <asm/pgtable.h>
13 14
14 15
16EXPORT_SYMBOL(empty_zero_page);
17
15EXPORT_SYMBOL(change_bit); 18EXPORT_SYMBOL(change_bit);
16EXPORT_SYMBOL(test_and_change_bit); 19EXPORT_SYMBOL(test_and_change_bit);
17 20
@@ -31,7 +34,9 @@ extern u64 __ashrdi3(u64, unsigned);
31extern u64 __ashldi3(u64, unsigned); 34extern u64 __ashldi3(u64, unsigned);
32extern u64 __lshrdi3(u64, unsigned); 35extern u64 __lshrdi3(u64, unsigned);
33extern s64 __negdi2(s64); 36extern s64 __negdi2(s64);
37extern int __ucmpdi2(u64, u64);
34EXPORT_SYMBOL(__ashrdi3); 38EXPORT_SYMBOL(__ashrdi3);
35EXPORT_SYMBOL(__ashldi3); 39EXPORT_SYMBOL(__ashldi3);
36EXPORT_SYMBOL(__lshrdi3); 40EXPORT_SYMBOL(__lshrdi3);
37EXPORT_SYMBOL(__negdi2); 41EXPORT_SYMBOL(__negdi2);
42EXPORT_SYMBOL(__ucmpdi2);
diff --git a/arch/mn10300/kernel/process.c b/arch/mn10300/kernel/process.c
index 9c623c88387b..b28c9a60445b 100644
--- a/arch/mn10300/kernel/process.c
+++ b/arch/mn10300/kernel/process.c
@@ -153,6 +153,7 @@ int kernel_thread(int (*fn)(void *), void *arg, unsigned long flags)
153 return do_fork(flags | CLONE_VM | CLONE_UNTRACED, 0, &regs, 0, 153 return do_fork(flags | CLONE_VM | CLONE_UNTRACED, 0, &regs, 0,
154 NULL, NULL); 154 NULL, NULL);
155} 155}
156EXPORT_SYMBOL(kernel_thread);
156 157
157/* 158/*
158 * free current thread data structures etc.. 159 * free current thread data structures etc..
diff --git a/arch/mn10300/lib/Makefile b/arch/mn10300/lib/Makefile
index fdfa9ec5b5bb..0cd2346f4c13 100644
--- a/arch/mn10300/lib/Makefile
+++ b/arch/mn10300/lib/Makefile
@@ -4,4 +4,4 @@
4 4
5lib-y = delay.o usercopy.o checksum.o bitops.o memcpy.o memmove.o memset.o 5lib-y = delay.o usercopy.o checksum.o bitops.o memcpy.o memmove.o memset.o
6lib-y += do_csum.o 6lib-y += do_csum.o
7lib-y += __ashldi3.o __ashrdi3.o __lshrdi3.o negdi2.o 7lib-y += __ashldi3.o __ashrdi3.o __lshrdi3.o negdi2.o __ucmpdi2.o
diff --git a/arch/mn10300/lib/__ucmpdi2.S b/arch/mn10300/lib/__ucmpdi2.S
new file mode 100644
index 000000000000..60dcbdfe386c
--- /dev/null
+++ b/arch/mn10300/lib/__ucmpdi2.S
@@ -0,0 +1,43 @@
1/* __ucmpdi2.S: 64-bit unsigned compare
2 *
3 * Copyright (C) 2008 Red Hat, Inc. All Rights Reserved.
4 * Written by David Howells (dhowells@redhat.com)
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 */
11
12
13 .text
14 .p2align 4
15
16###############################################################################
17#
18# int __ucmpdi2(unsigned long long a [D0:D1],
19# unsigned long long b [(SP,12),(SP,16)])
20#
21# - returns 0, 1, or 2 as a <, =, > b respectively.
22#
23###############################################################################
24 .globl __ucmpdi2
25 .type __ucmpdi2,@function
26__ucmpdi2:
27 mov (12,sp),a0 # b.lsw
28 mov (16,sp),a1 # b.msw
29
30 sub a0,d0
31 subc a1,d1 # may clear Z, never sets it
32 bne __ucmpdi2_differ # a.msw != b.msw
33 mov +1,d0
34 rets
35
36__ucmpdi2_differ:
37 # C flag is set if LE, clear if GE
38 subc d0,d0 # -1 if LE, 0 if GE
39 add +1,d0 # 0 if LE, 1 if GE
40 add d0,d0 # 0 if LE, 2 if GE
41 rets
42
43 .size __ucmpdi2, .-__ucmpdi2
diff --git a/arch/powerpc/boot/Makefile b/arch/powerpc/boot/Makefile
index 1cee2f9fdf06..095e04db1c0e 100644
--- a/arch/powerpc/boot/Makefile
+++ b/arch/powerpc/boot/Makefile
@@ -273,7 +273,8 @@ endif
273initrd- := $(patsubst zImage%, zImage.initrd%, $(image-n) $(image-)) 273initrd- := $(patsubst zImage%, zImage.initrd%, $(image-n) $(image-))
274initrd-y := $(patsubst zImage%, zImage.initrd%, \ 274initrd-y := $(patsubst zImage%, zImage.initrd%, \
275 $(patsubst dtbImage%, dtbImage.initrd%, \ 275 $(patsubst dtbImage%, dtbImage.initrd%, \
276 $(patsubst treeImage%, treeImage.initrd%, $(image-y)))) 276 $(patsubst simpleImage%, simpleImage.initrd%, \
277 $(patsubst treeImage%, treeImage.initrd%, $(image-y)))))
277initrd-y := $(filter-out $(image-y), $(initrd-y)) 278initrd-y := $(filter-out $(image-y), $(initrd-y))
278targets += $(image-y) $(initrd-y) 279targets += $(image-y) $(initrd-y)
279 280
diff --git a/arch/powerpc/kernel/legacy_serial.c b/arch/powerpc/kernel/legacy_serial.c
index 61dd17449ddc..4d96e1db55ee 100644
--- a/arch/powerpc/kernel/legacy_serial.c
+++ b/arch/powerpc/kernel/legacy_serial.c
@@ -33,13 +33,14 @@ static struct legacy_serial_info {
33 phys_addr_t taddr; 33 phys_addr_t taddr;
34} legacy_serial_infos[MAX_LEGACY_SERIAL_PORTS]; 34} legacy_serial_infos[MAX_LEGACY_SERIAL_PORTS];
35 35
36static struct __initdata of_device_id parents[] = { 36static struct __initdata of_device_id legacy_serial_parents[] = {
37 {.type = "soc",}, 37 {.type = "soc",},
38 {.type = "tsi-bridge",}, 38 {.type = "tsi-bridge",},
39 {.type = "opb", }, 39 {.type = "opb", },
40 {.compatible = "ibm,opb",}, 40 {.compatible = "ibm,opb",},
41 {.compatible = "simple-bus",}, 41 {.compatible = "simple-bus",},
42 {.compatible = "wrs,epld-localbus",}, 42 {.compatible = "wrs,epld-localbus",},
43 {},
43}; 44};
44 45
45static unsigned int legacy_serial_count; 46static unsigned int legacy_serial_count;
@@ -136,6 +137,11 @@ static int __init add_legacy_soc_port(struct device_node *np,
136 if (of_get_property(np, "clock-frequency", NULL) == NULL) 137 if (of_get_property(np, "clock-frequency", NULL) == NULL)
137 return -1; 138 return -1;
138 139
140 /* if reg-shift or offset, don't try to use it */
141 if ((of_get_property(np, "reg-shift", NULL) != NULL) ||
142 (of_get_property(np, "reg-offset", NULL) != NULL))
143 return -1;
144
139 /* if rtas uses this device, don't try to use it as well */ 145 /* if rtas uses this device, don't try to use it as well */
140 if (of_get_property(np, "used-by-rtas", NULL) != NULL) 146 if (of_get_property(np, "used-by-rtas", NULL) != NULL)
141 return -1; 147 return -1;
@@ -322,7 +328,7 @@ void __init find_legacy_serial_ports(void)
322 struct device_node *parent = of_get_parent(np); 328 struct device_node *parent = of_get_parent(np);
323 if (!parent) 329 if (!parent)
324 continue; 330 continue;
325 if (of_match_node(parents, parent) != NULL) { 331 if (of_match_node(legacy_serial_parents, parent) != NULL) {
326 index = add_legacy_soc_port(np, np); 332 index = add_legacy_soc_port(np, np);
327 if (index >= 0 && np == stdout) 333 if (index >= 0 && np == stdout)
328 legacy_serial_console = index; 334 legacy_serial_console = index;
diff --git a/arch/powerpc/kernel/of_platform.c b/arch/powerpc/kernel/of_platform.c
index e79ad8afda07..3f37a6e62771 100644
--- a/arch/powerpc/kernel/of_platform.c
+++ b/arch/powerpc/kernel/of_platform.c
@@ -76,6 +76,8 @@ struct of_device* of_platform_device_create(struct device_node *np,
76 return NULL; 76 return NULL;
77 77
78 dev->dma_mask = 0xffffffffUL; 78 dev->dma_mask = 0xffffffffUL;
79 dev->dev.coherent_dma_mask = DMA_32BIT_MASK;
80
79 dev->dev.bus = &of_platform_bus_type; 81 dev->dev.bus = &of_platform_bus_type;
80 82
81 /* We do not fill the DMA ops for platform devices by default. 83 /* We do not fill the DMA ops for platform devices by default.
diff --git a/arch/powerpc/platforms/52xx/lite5200_pm.c b/arch/powerpc/platforms/52xx/lite5200_pm.c
index 41c7fd91e99e..fe92e65103ed 100644
--- a/arch/powerpc/platforms/52xx/lite5200_pm.c
+++ b/arch/powerpc/platforms/52xx/lite5200_pm.c
@@ -14,6 +14,7 @@ static struct mpc52xx_sdma __iomem *bes;
14static struct mpc52xx_xlb __iomem *xlb; 14static struct mpc52xx_xlb __iomem *xlb;
15static struct mpc52xx_gpio __iomem *gps; 15static struct mpc52xx_gpio __iomem *gps;
16static struct mpc52xx_gpio_wkup __iomem *gpw; 16static struct mpc52xx_gpio_wkup __iomem *gpw;
17static void __iomem *pci;
17static void __iomem *sram; 18static void __iomem *sram;
18static const int sram_size = 0x4000; /* 16 kBytes */ 19static const int sram_size = 0x4000; /* 16 kBytes */
19static void __iomem *mbar; 20static void __iomem *mbar;
@@ -50,6 +51,8 @@ static int lite5200_pm_prepare(void)
50 { .type = "builtin", .compatible = "mpc5200", }, /* efika */ 51 { .type = "builtin", .compatible = "mpc5200", }, /* efika */
51 {} 52 {}
52 }; 53 };
54 u64 regaddr64 = 0;
55 const u32 *regaddr_p;
53 56
54 /* deep sleep? let mpc52xx code handle that */ 57 /* deep sleep? let mpc52xx code handle that */
55 if (lite5200_pm_target_state == PM_SUSPEND_STANDBY) 58 if (lite5200_pm_target_state == PM_SUSPEND_STANDBY)
@@ -60,8 +63,12 @@ static int lite5200_pm_prepare(void)
60 63
61 /* map registers */ 64 /* map registers */
62 np = of_find_matching_node(NULL, immr_ids); 65 np = of_find_matching_node(NULL, immr_ids);
63 mbar = of_iomap(np, 0); 66 regaddr_p = of_get_address(np, 0, NULL, NULL);
67 if (regaddr_p)
68 regaddr64 = of_translate_address(np, regaddr_p);
64 of_node_put(np); 69 of_node_put(np);
70
71 mbar = ioremap((u32) regaddr64, 0xC000);
65 if (!mbar) { 72 if (!mbar) {
66 printk(KERN_ERR "%s:%i Error mapping registers\n", __func__, __LINE__); 73 printk(KERN_ERR "%s:%i Error mapping registers\n", __func__, __LINE__);
67 return -ENOSYS; 74 return -ENOSYS;
@@ -71,6 +78,7 @@ static int lite5200_pm_prepare(void)
71 pic = mbar + 0x500; 78 pic = mbar + 0x500;
72 gps = mbar + 0xb00; 79 gps = mbar + 0xb00;
73 gpw = mbar + 0xc00; 80 gpw = mbar + 0xc00;
81 pci = mbar + 0xd00;
74 bes = mbar + 0x1200; 82 bes = mbar + 0x1200;
75 xlb = mbar + 0x1f00; 83 xlb = mbar + 0x1f00;
76 sram = mbar + 0x8000; 84 sram = mbar + 0x8000;
@@ -85,6 +93,7 @@ static struct mpc52xx_sdma sbes;
85static struct mpc52xx_xlb sxlb; 93static struct mpc52xx_xlb sxlb;
86static struct mpc52xx_gpio sgps; 94static struct mpc52xx_gpio sgps;
87static struct mpc52xx_gpio_wkup sgpw; 95static struct mpc52xx_gpio_wkup sgpw;
96static char spci[0x200];
88 97
89static void lite5200_save_regs(void) 98static void lite5200_save_regs(void)
90{ 99{
@@ -94,6 +103,7 @@ static void lite5200_save_regs(void)
94 _memcpy_fromio(&sxlb, xlb, sizeof(*xlb)); 103 _memcpy_fromio(&sxlb, xlb, sizeof(*xlb));
95 _memcpy_fromio(&sgps, gps, sizeof(*gps)); 104 _memcpy_fromio(&sgps, gps, sizeof(*gps));
96 _memcpy_fromio(&sgpw, gpw, sizeof(*gpw)); 105 _memcpy_fromio(&sgpw, gpw, sizeof(*gpw));
106 _memcpy_fromio(spci, pci, 0x200);
97 107
98 _memcpy_fromio(saved_sram, sram, sram_size); 108 _memcpy_fromio(saved_sram, sram, sram_size);
99} 109}
@@ -103,6 +113,8 @@ static void lite5200_restore_regs(void)
103 int i; 113 int i;
104 _memcpy_toio(sram, saved_sram, sram_size); 114 _memcpy_toio(sram, saved_sram, sram_size);
105 115
116 /* PCI Configuration */
117 _memcpy_toio(pci, spci, 0x200);
106 118
107 /* 119 /*
108 * GPIOs. Interrupt Master Enable has higher address then other 120 * GPIOs. Interrupt Master Enable has higher address then other
diff --git a/arch/um/Makefile b/arch/um/Makefile
index dbeab15e7bb7..ca40397017b9 100644
--- a/arch/um/Makefile
+++ b/arch/um/Makefile
@@ -77,7 +77,6 @@ include $(srctree)/$(ARCH_DIR)/Makefile-os-$(OS)
77KERNEL_DEFINES = $(strip -Derrno=kernel_errno -Dsigprocmask=kernel_sigprocmask \ 77KERNEL_DEFINES = $(strip -Derrno=kernel_errno -Dsigprocmask=kernel_sigprocmask \
78 -Dmktime=kernel_mktime $(ARCH_KERNEL_DEFINES)) 78 -Dmktime=kernel_mktime $(ARCH_KERNEL_DEFINES))
79KBUILD_CFLAGS += $(KERNEL_DEFINES) 79KBUILD_CFLAGS += $(KERNEL_DEFINES)
80KBUILD_CFLAGS += $(call cc-option,-fno-unit-at-a-time,)
81 80
82PHONY += linux 81PHONY += linux
83 82
diff --git a/arch/um/Makefile-i386 b/arch/um/Makefile-i386
index 561e373bd850..302cbe504543 100644
--- a/arch/um/Makefile-i386
+++ b/arch/um/Makefile-i386
@@ -32,4 +32,11 @@ cflags-y += $(call cc-option,-mpreferred-stack-boundary=2)
32# an unresolved reference. 32# an unresolved reference.
33cflags-y += -ffreestanding 33cflags-y += -ffreestanding
34 34
35# Disable unit-at-a-time mode on pre-gcc-4.0 compilers, it makes gcc use
36# a lot more stack due to the lack of sharing of stacklots. Also, gcc
37# 4.3.0 needs -funit-at-a-time for extern inline functions.
38KBUILD_CFLAGS += $(shell if [ $(call cc-version) -lt 0400 ] ; then \
39 echo $(call cc-option,-fno-unit-at-a-time); \
40 else echo $(call cc-option,-funit-at-a-time); fi ;)
41
35KBUILD_CFLAGS += $(cflags-y) 42KBUILD_CFLAGS += $(cflags-y)
diff --git a/arch/um/Makefile-x86_64 b/arch/um/Makefile-x86_64
index 8ed362f93582..a9cd7e77a7ab 100644
--- a/arch/um/Makefile-x86_64
+++ b/arch/um/Makefile-x86_64
@@ -21,3 +21,6 @@ HEADER_ARCH := x86
21 21
22LINK-$(CONFIG_LD_SCRIPT_DYN) += -Wl,-rpath,/lib64 22LINK-$(CONFIG_LD_SCRIPT_DYN) += -Wl,-rpath,/lib64
23LINK-y += -m64 23LINK-y += -m64
24
25# Do unit-at-a-time unconditionally on x86_64, following the host
26KBUILD_CFLAGS += $(call cc-option,-funit-at-a-time)
diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
index 400135148555..c3a4c03c0800 100644
--- a/arch/x86/Kconfig
+++ b/arch/x86/Kconfig
@@ -968,8 +968,8 @@ config NUMA_EMU
968 number of nodes. This is only useful for debugging. 968 number of nodes. This is only useful for debugging.
969 969
970config NODES_SHIFT 970config NODES_SHIFT
971 int "Max num nodes shift(1-15)" 971 int "Max num nodes shift(1-9)"
972 range 1 15 if X86_64 972 range 1 9 if X86_64
973 default "6" if X86_64 973 default "6" if X86_64
974 default "4" if X86_NUMAQ 974 default "4" if X86_NUMAQ
975 default "3" 975 default "3"
diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug
index f395fd537c5c..f7169edfbeab 100644
--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -172,13 +172,33 @@ config IOMMU_LEAK
172 Add a simple leak tracer to the IOMMU code. This is useful when you 172 Add a simple leak tracer to the IOMMU code. This is useful when you
173 are debugging a buggy device driver that leaks IOMMU mappings. 173 are debugging a buggy device driver that leaks IOMMU mappings.
174 174
175config PAGE_FAULT_HANDLERS 175config MMIOTRACE_HOOKS
176 bool "Custom page fault handlers" 176 bool
177 depends on DEBUG_KERNEL 177
178config MMIOTRACE
179 bool "Memory mapped IO tracing"
180 depends on DEBUG_KERNEL && PCI
181 select TRACING
182 select MMIOTRACE_HOOKS
183 default y
184 help
185 Mmiotrace traces Memory Mapped I/O access and is meant for
186 debugging and reverse engineering. It is called from the ioremap
187 implementation and works via page faults. Tracing is disabled by
188 default and can be enabled at run-time.
189
190 See Documentation/tracers/mmiotrace.txt.
191 If you are not helping to develop drivers, say N.
192
193config MMIOTRACE_TEST
194 tristate "Test module for mmiotrace"
195 depends on MMIOTRACE && m
178 help 196 help
179 Allow the use of custom page fault handlers. A kernel module may 197 This is a dumb module for testing mmiotrace. It is very dangerous
180 register a function that is called on every page fault. Custom 198 as it will write garbage to IO memory starting at a given address.
181 handlers are used by some debugging and reverse engineering tools. 199 However, it should be safe to use on e.g. unused portion of VRAM.
200
201 Say N, unless you absolutely know what you are doing.
182 202
183# 203#
184# IO delay types: 204# IO delay types:
diff --git a/arch/x86/kernel/acpi/realmode/wakeup.S b/arch/x86/kernel/acpi/realmode/wakeup.S
index f9b77fb37e5b..3355973b12ac 100644
--- a/arch/x86/kernel/acpi/realmode/wakeup.S
+++ b/arch/x86/kernel/acpi/realmode/wakeup.S
@@ -5,6 +5,7 @@
5#include <asm/msr-index.h> 5#include <asm/msr-index.h>
6#include <asm/page.h> 6#include <asm/page.h>
7#include <asm/pgtable.h> 7#include <asm/pgtable.h>
8#include <asm/processor-flags.h>
8 9
9 .code16 10 .code16
10 .section ".header", "a" 11 .section ".header", "a"
@@ -24,6 +25,11 @@ pmode_gdt: .quad 0
24realmode_flags: .long 0 25realmode_flags: .long 0
25real_magic: .long 0 26real_magic: .long 0
26trampoline_segment: .word 0 27trampoline_segment: .word 0
28_pad1: .byte 0
29wakeup_jmp: .byte 0xea /* ljmpw */
30wakeup_jmp_off: .word 3f
31wakeup_jmp_seg: .word 0
32wakeup_gdt: .quad 0, 0, 0
27signature: .long 0x51ee1111 33signature: .long 0x51ee1111
28 34
29 .text 35 .text
@@ -34,11 +40,34 @@ _start:
34 cli 40 cli
35 cld 41 cld
36 42
43 /* Apparently some dimwit BIOS programmers don't know how to
44 program a PM to RM transition, and we might end up here with
45 junk in the data segment descriptor registers. The only way
46 to repair that is to go into PM and fix it ourselves... */
47 movw $16, %cx
48 lgdtl %cs:wakeup_gdt
49 movl %cr0, %eax
50 orb $X86_CR0_PE, %al
51 movl %eax, %cr0
52 jmp 1f
531: ljmpw $8, $2f
542:
55 movw %cx, %ds
56 movw %cx, %es
57 movw %cx, %ss
58 movw %cx, %fs
59 movw %cx, %gs
60
61 andb $~X86_CR0_PE, %al
62 movl %eax, %cr0
63 jmp wakeup_jmp
643:
37 /* Set up segments */ 65 /* Set up segments */
38 movw %cs, %ax 66 movw %cs, %ax
39 movw %ax, %ds 67 movw %ax, %ds
40 movw %ax, %es 68 movw %ax, %es
41 movw %ax, %ss 69 movw %ax, %ss
70 lidtl wakeup_idt
42 71
43 movl $wakeup_stack_end, %esp 72 movl $wakeup_stack_end, %esp
44 73
@@ -98,7 +127,14 @@ bogus_real_magic:
98 jmp 1b 127 jmp 1b
99 128
100 .data 129 .data
101 .balign 4 130 .balign 8
131
132 /* This is the standard real-mode IDT */
133wakeup_idt:
134 .word 0xffff /* limit */
135 .long 0 /* address */
136 .word 0
137
102 .globl HEAP, heap_end 138 .globl HEAP, heap_end
103HEAP: 139HEAP:
104 .long wakeup_heap 140 .long wakeup_heap
diff --git a/arch/x86/kernel/acpi/realmode/wakeup.h b/arch/x86/kernel/acpi/realmode/wakeup.h
index ef8166fe8020..69d38d0b2b64 100644
--- a/arch/x86/kernel/acpi/realmode/wakeup.h
+++ b/arch/x86/kernel/acpi/realmode/wakeup.h
@@ -24,6 +24,11 @@ struct wakeup_header {
24 u32 realmode_flags; 24 u32 realmode_flags;
25 u32 real_magic; 25 u32 real_magic;
26 u16 trampoline_segment; /* segment with trampoline code, 64-bit only */ 26 u16 trampoline_segment; /* segment with trampoline code, 64-bit only */
27 u8 _pad1;
28 u8 wakeup_jmp;
29 u16 wakeup_jmp_off;
30 u16 wakeup_jmp_seg;
31 u64 wakeup_gdt[3];
27 u32 signature; /* To check we have correct structure */ 32 u32 signature; /* To check we have correct structure */
28} __attribute__((__packed__)); 33} __attribute__((__packed__));
29 34
diff --git a/arch/x86/kernel/acpi/sleep.c b/arch/x86/kernel/acpi/sleep.c
index afc25ee9964b..36af01f029ed 100644
--- a/arch/x86/kernel/acpi/sleep.c
+++ b/arch/x86/kernel/acpi/sleep.c
@@ -50,6 +50,20 @@ int acpi_save_state_mem(void)
50 50
51 header->video_mode = saved_video_mode; 51 header->video_mode = saved_video_mode;
52 52
53 header->wakeup_jmp_seg = acpi_wakeup_address >> 4;
54 /* GDT[0]: GDT self-pointer */
55 header->wakeup_gdt[0] =
56 (u64)(sizeof(header->wakeup_gdt) - 1) +
57 ((u64)(acpi_wakeup_address +
58 ((char *)&header->wakeup_gdt - (char *)acpi_realmode))
59 << 16);
60 /* GDT[1]: real-mode-like code segment */
61 header->wakeup_gdt[1] = (0x009bULL << 40) +
62 ((u64)acpi_wakeup_address << 16) + 0xffff;
63 /* GDT[2]: real-mode-like data segment */
64 header->wakeup_gdt[2] = (0x0093ULL << 40) +
65 ((u64)acpi_wakeup_address << 16) + 0xffff;
66
53#ifndef CONFIG_64BIT 67#ifndef CONFIG_64BIT
54 store_gdt((struct desc_ptr *)&header->pmode_gdt); 68 store_gdt((struct desc_ptr *)&header->pmode_gdt);
55 69
@@ -111,7 +125,7 @@ void __init acpi_reserve_bootmem(void)
111 return; 125 return;
112 } 126 }
113 127
114 acpi_wakeup_address = acpi_realmode; 128 acpi_wakeup_address = virt_to_phys((void *)acpi_realmode);
115} 129}
116 130
117 131
diff --git a/arch/x86/kernel/efi_32.c b/arch/x86/kernel/efi_32.c
index 5d23d85624d4..4b63c8e1f13b 100644
--- a/arch/x86/kernel/efi_32.c
+++ b/arch/x86/kernel/efi_32.c
@@ -49,13 +49,13 @@ void efi_call_phys_prelog(void)
49 local_irq_save(efi_rt_eflags); 49 local_irq_save(efi_rt_eflags);
50 50
51 /* 51 /*
52 * If I don't have PSE, I should just duplicate two entries in page 52 * If I don't have PAE, I should just duplicate two entries in page
53 * directory. If I have PSE, I just need to duplicate one entry in 53 * directory. If I have PAE, I just need to duplicate one entry in
54 * page directory. 54 * page directory.
55 */ 55 */
56 cr4 = read_cr4(); 56 cr4 = read_cr4();
57 57
58 if (cr4 & X86_CR4_PSE) { 58 if (cr4 & X86_CR4_PAE) {
59 efi_bak_pg_dir_pointer[0].pgd = 59 efi_bak_pg_dir_pointer[0].pgd =
60 swapper_pg_dir[pgd_index(0)].pgd; 60 swapper_pg_dir[pgd_index(0)].pgd;
61 swapper_pg_dir[0].pgd = 61 swapper_pg_dir[0].pgd =
@@ -93,7 +93,7 @@ void efi_call_phys_epilog(void)
93 93
94 cr4 = read_cr4(); 94 cr4 = read_cr4();
95 95
96 if (cr4 & X86_CR4_PSE) { 96 if (cr4 & X86_CR4_PAE) {
97 swapper_pg_dir[pgd_index(0)].pgd = 97 swapper_pg_dir[pgd_index(0)].pgd =
98 efi_bak_pg_dir_pointer[0].pgd; 98 efi_bak_pg_dir_pointer[0].pgd;
99 } else { 99 } else {
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index 10a1955bb1d1..b817974ef942 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -128,7 +128,7 @@ ident_complete:
128 /* Fixup phys_base */ 128 /* Fixup phys_base */
129 addq %rbp, phys_base(%rip) 129 addq %rbp, phys_base(%rip)
130 130
131#ifdef CONFIG_SMP 131#ifdef CONFIG_X86_TRAMPOLINE
132 addq %rbp, trampoline_level4_pgt + 0(%rip) 132 addq %rbp, trampoline_level4_pgt + 0(%rip)
133 addq %rbp, trampoline_level4_pgt + (511*8)(%rip) 133 addq %rbp, trampoline_level4_pgt + (511*8)(%rip)
134#endif 134#endif
diff --git a/arch/x86/kernel/i387.c b/arch/x86/kernel/i387.c
index eb9ddd8efb82..95e80e5033c3 100644
--- a/arch/x86/kernel/i387.c
+++ b/arch/x86/kernel/i387.c
@@ -162,7 +162,7 @@ int xfpregs_get(struct task_struct *target, const struct user_regset *regset,
162 int ret; 162 int ret;
163 163
164 if (!cpu_has_fxsr) 164 if (!cpu_has_fxsr)
165 return -ENODEV; 165 return -EIO;
166 166
167 ret = init_fpu(target); 167 ret = init_fpu(target);
168 if (ret) 168 if (ret)
@@ -179,7 +179,7 @@ int xfpregs_set(struct task_struct *target, const struct user_regset *regset,
179 int ret; 179 int ret;
180 180
181 if (!cpu_has_fxsr) 181 if (!cpu_has_fxsr)
182 return -ENODEV; 182 return -EIO;
183 183
184 ret = init_fpu(target); 184 ret = init_fpu(target);
185 if (ret) 185 if (ret)
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index 56078d61c793..3e1cecedde42 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -996,7 +996,6 @@ do_rest:
996#endif 996#endif
997 cpu_clear(cpu, cpu_callout_map); /* was set by do_boot_cpu() */ 997 cpu_clear(cpu, cpu_callout_map); /* was set by do_boot_cpu() */
998 cpu_clear(cpu, cpu_initialized); /* was set by cpu_init() */ 998 cpu_clear(cpu, cpu_initialized); /* was set by cpu_init() */
999 cpu_clear(cpu, cpu_possible_map);
1000 cpu_clear(cpu, cpu_present_map); 999 cpu_clear(cpu, cpu_present_map);
1001 per_cpu(x86_cpu_to_apicid, cpu) = BAD_APICID; 1000 per_cpu(x86_cpu_to_apicid, cpu) = BAD_APICID;
1002 } 1001 }
diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile
index b7b3e4c7cfc9..07dab503c9e3 100644
--- a/arch/x86/mm/Makefile
+++ b/arch/x86/mm/Makefile
@@ -8,6 +8,11 @@ obj-$(CONFIG_X86_PTDUMP) += dump_pagetables.o
8 8
9obj-$(CONFIG_HIGHMEM) += highmem_32.o 9obj-$(CONFIG_HIGHMEM) += highmem_32.o
10 10
11obj-$(CONFIG_MMIOTRACE_HOOKS) += kmmio.o
12obj-$(CONFIG_MMIOTRACE) += mmiotrace.o
13mmiotrace-y := pf_in.o mmio-mod.o
14obj-$(CONFIG_MMIOTRACE_TEST) += testmmiotrace.o
15
11ifeq ($(CONFIG_X86_32),y) 16ifeq ($(CONFIG_X86_32),y)
12obj-$(CONFIG_NUMA) += discontig_32.o 17obj-$(CONFIG_NUMA) += discontig_32.o
13else 18else
diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
index 42394b353c6a..0a778e3c43ee 100644
--- a/arch/x86/mm/fault.c
+++ b/arch/x86/mm/fault.c
@@ -10,6 +10,7 @@
10#include <linux/string.h> 10#include <linux/string.h>
11#include <linux/types.h> 11#include <linux/types.h>
12#include <linux/ptrace.h> 12#include <linux/ptrace.h>
13#include <linux/mmiotrace.h>
13#include <linux/mman.h> 14#include <linux/mman.h>
14#include <linux/mm.h> 15#include <linux/mm.h>
15#include <linux/smp.h> 16#include <linux/smp.h>
@@ -49,58 +50,14 @@
49#define PF_RSVD (1<<3) 50#define PF_RSVD (1<<3)
50#define PF_INSTR (1<<4) 51#define PF_INSTR (1<<4)
51 52
52#ifdef CONFIG_PAGE_FAULT_HANDLERS 53static inline int kmmio_fault(struct pt_regs *regs, unsigned long addr)
53static HLIST_HEAD(pf_handlers); /* protected by RCU */
54static DEFINE_SPINLOCK(pf_handlers_writer);
55
56void register_page_fault_handler(struct pf_handler *new_pfh)
57{
58 unsigned long flags;
59 spin_lock_irqsave(&pf_handlers_writer, flags);
60 hlist_add_head_rcu(&new_pfh->hlist, &pf_handlers);
61 spin_unlock_irqrestore(&pf_handlers_writer, flags);
62}
63EXPORT_SYMBOL_GPL(register_page_fault_handler);
64
65/**
66 * unregister_page_fault_handler:
67 * The caller must ensure @old_pfh is not in use anymore before freeing it.
68 * This function does not guarantee it. The list of handlers is protected by
69 * RCU, so you can do this by e.g. calling synchronize_rcu().
70 */
71void unregister_page_fault_handler(struct pf_handler *old_pfh)
72{ 54{
73 unsigned long flags; 55#ifdef CONFIG_MMIOTRACE_HOOKS
74 spin_lock_irqsave(&pf_handlers_writer, flags); 56 if (unlikely(is_kmmio_active()))
75 hlist_del_rcu(&old_pfh->hlist); 57 if (kmmio_handler(regs, addr) == 1)
76 spin_unlock_irqrestore(&pf_handlers_writer, flags); 58 return -1;
77}
78EXPORT_SYMBOL_GPL(unregister_page_fault_handler);
79#endif 59#endif
80
81/* returns non-zero if do_page_fault() should return */
82static int handle_custom_pf(struct pt_regs *regs, unsigned long error_code,
83 unsigned long address)
84{
85#ifdef CONFIG_PAGE_FAULT_HANDLERS
86 int ret = 0;
87 struct pf_handler *cur;
88 struct hlist_node *ncur;
89
90 if (hlist_empty(&pf_handlers))
91 return 0;
92
93 rcu_read_lock();
94 hlist_for_each_entry_rcu(cur, ncur, &pf_handlers, hlist) {
95 ret = cur->handler(regs, error_code, address);
96 if (ret)
97 break;
98 }
99 rcu_read_unlock();
100 return ret;
101#else
102 return 0; 60 return 0;
103#endif
104} 61}
105 62
106static inline int notify_page_fault(struct pt_regs *regs) 63static inline int notify_page_fault(struct pt_regs *regs)
@@ -660,7 +617,7 @@ void __kprobes do_page_fault(struct pt_regs *regs, unsigned long error_code)
660 617
661 if (notify_page_fault(regs)) 618 if (notify_page_fault(regs))
662 return; 619 return;
663 if (handle_custom_pf(regs, error_code, address)) 620 if (unlikely(kmmio_fault(regs, address)))
664 return; 621 return;
665 622
666 /* 623 /*
diff --git a/arch/x86/mm/init_64.c b/arch/x86/mm/init_64.c
index a5fd2e06f5c9..17c0a6138a53 100644
--- a/arch/x86/mm/init_64.c
+++ b/arch/x86/mm/init_64.c
@@ -135,7 +135,7 @@ static __init void *spp_getpage(void)
135 return ptr; 135 return ptr;
136} 136}
137 137
138static void 138static __init void
139set_pte_phys(unsigned long vaddr, unsigned long phys, pgprot_t prot) 139set_pte_phys(unsigned long vaddr, unsigned long phys, pgprot_t prot)
140{ 140{
141 pgd_t *pgd; 141 pgd_t *pgd;
@@ -214,7 +214,7 @@ void __init cleanup_highmap(void)
214} 214}
215 215
216/* NOTE: this is meant to be run only at boot */ 216/* NOTE: this is meant to be run only at boot */
217void __set_fixmap(enum fixed_addresses idx, unsigned long phys, pgprot_t prot) 217void __init __set_fixmap(enum fixed_addresses idx, unsigned long phys, pgprot_t prot)
218{ 218{
219 unsigned long address = __fix_to_virt(idx); 219 unsigned long address = __fix_to_virt(idx);
220 220
@@ -526,7 +526,8 @@ static void __init early_memtest(unsigned long start, unsigned long end)
526 t_size = end - t_start; 526 t_size = end - t_start;
527 527
528 printk(KERN_CONT "\n %016llx - %016llx pattern %d", 528 printk(KERN_CONT "\n %016llx - %016llx pattern %d",
529 t_start, t_start + t_size, pattern); 529 (unsigned long long)t_start,
530 (unsigned long long)t_start + t_size, pattern);
530 531
531 memtest(t_start, t_size, pattern); 532 memtest(t_start, t_size, pattern);
532 533
diff --git a/arch/x86/mm/ioremap.c b/arch/x86/mm/ioremap.c
index 2b2bb3f9b683..e92aa461f4d6 100644
--- a/arch/x86/mm/ioremap.c
+++ b/arch/x86/mm/ioremap.c
@@ -12,6 +12,7 @@
12#include <linux/module.h> 12#include <linux/module.h>
13#include <linux/slab.h> 13#include <linux/slab.h>
14#include <linux/vmalloc.h> 14#include <linux/vmalloc.h>
15#include <linux/mmiotrace.h>
15 16
16#include <asm/cacheflush.h> 17#include <asm/cacheflush.h>
17#include <asm/e820.h> 18#include <asm/e820.h>
@@ -122,10 +123,13 @@ static void __iomem *__ioremap_caller(resource_size_t phys_addr,
122{ 123{
123 unsigned long pfn, offset, vaddr; 124 unsigned long pfn, offset, vaddr;
124 resource_size_t last_addr; 125 resource_size_t last_addr;
126 const resource_size_t unaligned_phys_addr = phys_addr;
127 const unsigned long unaligned_size = size;
125 struct vm_struct *area; 128 struct vm_struct *area;
126 unsigned long new_prot_val; 129 unsigned long new_prot_val;
127 pgprot_t prot; 130 pgprot_t prot;
128 int retval; 131 int retval;
132 void __iomem *ret_addr;
129 133
130 /* Don't allow wraparound or zero size */ 134 /* Don't allow wraparound or zero size */
131 last_addr = phys_addr + size - 1; 135 last_addr = phys_addr + size - 1;
@@ -233,7 +237,10 @@ static void __iomem *__ioremap_caller(resource_size_t phys_addr,
233 return NULL; 237 return NULL;
234 } 238 }
235 239
236 return (void __iomem *) (vaddr + offset); 240 ret_addr = (void __iomem *) (vaddr + offset);
241 mmiotrace_ioremap(unaligned_phys_addr, unaligned_size, ret_addr);
242
243 return ret_addr;
237} 244}
238 245
239/** 246/**
@@ -325,6 +332,8 @@ void iounmap(volatile void __iomem *addr)
325 addr = (volatile void __iomem *) 332 addr = (volatile void __iomem *)
326 (PAGE_MASK & (unsigned long __force)addr); 333 (PAGE_MASK & (unsigned long __force)addr);
327 334
335 mmiotrace_iounmap(addr);
336
328 /* Use the vm area unlocked, assuming the caller 337 /* Use the vm area unlocked, assuming the caller
329 ensures there isn't another iounmap for the same address 338 ensures there isn't another iounmap for the same address
330 in parallel. Reuse of the virtual address is prevented by 339 in parallel. Reuse of the virtual address is prevented by
diff --git a/arch/x86/mm/kmmio.c b/arch/x86/mm/kmmio.c
new file mode 100644
index 000000000000..93d82038af4b
--- /dev/null
+++ b/arch/x86/mm/kmmio.c
@@ -0,0 +1,510 @@
1/* Support for MMIO probes.
2 * Benfit many code from kprobes
3 * (C) 2002 Louis Zhuang <louis.zhuang@intel.com>.
4 * 2007 Alexander Eichner
5 * 2008 Pekka Paalanen <pq@iki.fi>
6 */
7
8#include <linux/list.h>
9#include <linux/rculist.h>
10#include <linux/spinlock.h>
11#include <linux/hash.h>
12#include <linux/init.h>
13#include <linux/module.h>
14#include <linux/kernel.h>
15#include <linux/uaccess.h>
16#include <linux/ptrace.h>
17#include <linux/preempt.h>
18#include <linux/percpu.h>
19#include <linux/kdebug.h>
20#include <linux/mutex.h>
21#include <linux/io.h>
22#include <asm/cacheflush.h>
23#include <asm/tlbflush.h>
24#include <linux/errno.h>
25#include <asm/debugreg.h>
26#include <linux/mmiotrace.h>
27
28#define KMMIO_PAGE_HASH_BITS 4
29#define KMMIO_PAGE_TABLE_SIZE (1 << KMMIO_PAGE_HASH_BITS)
30
31struct kmmio_fault_page {
32 struct list_head list;
33 struct kmmio_fault_page *release_next;
34 unsigned long page; /* location of the fault page */
35
36 /*
37 * Number of times this page has been registered as a part
38 * of a probe. If zero, page is disarmed and this may be freed.
39 * Used only by writers (RCU).
40 */
41 int count;
42};
43
44struct kmmio_delayed_release {
45 struct rcu_head rcu;
46 struct kmmio_fault_page *release_list;
47};
48
49struct kmmio_context {
50 struct kmmio_fault_page *fpage;
51 struct kmmio_probe *probe;
52 unsigned long saved_flags;
53 unsigned long addr;
54 int active;
55};
56
57static DEFINE_SPINLOCK(kmmio_lock);
58
59/* Protected by kmmio_lock */
60unsigned int kmmio_count;
61
62/* Read-protected by RCU, write-protected by kmmio_lock. */
63static struct list_head kmmio_page_table[KMMIO_PAGE_TABLE_SIZE];
64static LIST_HEAD(kmmio_probes);
65
66static struct list_head *kmmio_page_list(unsigned long page)
67{
68 return &kmmio_page_table[hash_long(page, KMMIO_PAGE_HASH_BITS)];
69}
70
71/* Accessed per-cpu */
72static DEFINE_PER_CPU(struct kmmio_context, kmmio_ctx);
73
74/*
75 * this is basically a dynamic stabbing problem:
76 * Could use the existing prio tree code or
77 * Possible better implementations:
78 * The Interval Skip List: A Data Structure for Finding All Intervals That
79 * Overlap a Point (might be simple)
80 * Space Efficient Dynamic Stabbing with Fast Queries - Mikkel Thorup
81 */
82/* Get the kmmio at this addr (if any). You must be holding RCU read lock. */
83static struct kmmio_probe *get_kmmio_probe(unsigned long addr)
84{
85 struct kmmio_probe *p;
86 list_for_each_entry_rcu(p, &kmmio_probes, list) {
87 if (addr >= p->addr && addr <= (p->addr + p->len))
88 return p;
89 }
90 return NULL;
91}
92
93/* You must be holding RCU read lock. */
94static struct kmmio_fault_page *get_kmmio_fault_page(unsigned long page)
95{
96 struct list_head *head;
97 struct kmmio_fault_page *p;
98
99 page &= PAGE_MASK;
100 head = kmmio_page_list(page);
101 list_for_each_entry_rcu(p, head, list) {
102 if (p->page == page)
103 return p;
104 }
105 return NULL;
106}
107
108static void set_page_present(unsigned long addr, bool present,
109 unsigned int *pglevel)
110{
111 pteval_t pteval;
112 pmdval_t pmdval;
113 unsigned int level;
114 pmd_t *pmd;
115 pte_t *pte = lookup_address(addr, &level);
116
117 if (!pte) {
118 pr_err("kmmio: no pte for page 0x%08lx\n", addr);
119 return;
120 }
121
122 if (pglevel)
123 *pglevel = level;
124
125 switch (level) {
126 case PG_LEVEL_2M:
127 pmd = (pmd_t *)pte;
128 pmdval = pmd_val(*pmd) & ~_PAGE_PRESENT;
129 if (present)
130 pmdval |= _PAGE_PRESENT;
131 set_pmd(pmd, __pmd(pmdval));
132 break;
133
134 case PG_LEVEL_4K:
135 pteval = pte_val(*pte) & ~_PAGE_PRESENT;
136 if (present)
137 pteval |= _PAGE_PRESENT;
138 set_pte_atomic(pte, __pte(pteval));
139 break;
140
141 default:
142 pr_err("kmmio: unexpected page level 0x%x.\n", level);
143 return;
144 }
145
146 __flush_tlb_one(addr);
147}
148
149/** Mark the given page as not present. Access to it will trigger a fault. */
150static void arm_kmmio_fault_page(unsigned long page, unsigned int *pglevel)
151{
152 set_page_present(page & PAGE_MASK, false, pglevel);
153}
154
155/** Mark the given page as present. */
156static void disarm_kmmio_fault_page(unsigned long page, unsigned int *pglevel)
157{
158 set_page_present(page & PAGE_MASK, true, pglevel);
159}
160
161/*
162 * This is being called from do_page_fault().
163 *
164 * We may be in an interrupt or a critical section. Also prefecthing may
165 * trigger a page fault. We may be in the middle of process switch.
166 * We cannot take any locks, because we could be executing especially
167 * within a kmmio critical section.
168 *
169 * Local interrupts are disabled, so preemption cannot happen.
170 * Do not enable interrupts, do not sleep, and watch out for other CPUs.
171 */
172/*
173 * Interrupts are disabled on entry as trap3 is an interrupt gate
174 * and they remain disabled thorough out this function.
175 */
176int kmmio_handler(struct pt_regs *regs, unsigned long addr)
177{
178 struct kmmio_context *ctx;
179 struct kmmio_fault_page *faultpage;
180 int ret = 0; /* default to fault not handled */
181
182 /*
183 * Preemption is now disabled to prevent process switch during
184 * single stepping. We can only handle one active kmmio trace
185 * per cpu, so ensure that we finish it before something else
186 * gets to run. We also hold the RCU read lock over single
187 * stepping to avoid looking up the probe and kmmio_fault_page
188 * again.
189 */
190 preempt_disable();
191 rcu_read_lock();
192
193 faultpage = get_kmmio_fault_page(addr);
194 if (!faultpage) {
195 /*
196 * Either this page fault is not caused by kmmio, or
197 * another CPU just pulled the kmmio probe from under
198 * our feet. The latter case should not be possible.
199 */
200 goto no_kmmio;
201 }
202
203 ctx = &get_cpu_var(kmmio_ctx);
204 if (ctx->active) {
205 disarm_kmmio_fault_page(faultpage->page, NULL);
206 if (addr == ctx->addr) {
207 /*
208 * On SMP we sometimes get recursive probe hits on the
209 * same address. Context is already saved, fall out.
210 */
211 pr_debug("kmmio: duplicate probe hit on CPU %d, for "
212 "address 0x%08lx.\n",
213 smp_processor_id(), addr);
214 ret = 1;
215 goto no_kmmio_ctx;
216 }
217 /*
218 * Prevent overwriting already in-flight context.
219 * This should not happen, let's hope disarming at least
220 * prevents a panic.
221 */
222 pr_emerg("kmmio: recursive probe hit on CPU %d, "
223 "for address 0x%08lx. Ignoring.\n",
224 smp_processor_id(), addr);
225 pr_emerg("kmmio: previous hit was at 0x%08lx.\n",
226 ctx->addr);
227 goto no_kmmio_ctx;
228 }
229 ctx->active++;
230
231 ctx->fpage = faultpage;
232 ctx->probe = get_kmmio_probe(addr);
233 ctx->saved_flags = (regs->flags & (X86_EFLAGS_TF | X86_EFLAGS_IF));
234 ctx->addr = addr;
235
236 if (ctx->probe && ctx->probe->pre_handler)
237 ctx->probe->pre_handler(ctx->probe, regs, addr);
238
239 /*
240 * Enable single-stepping and disable interrupts for the faulting
241 * context. Local interrupts must not get enabled during stepping.
242 */
243 regs->flags |= X86_EFLAGS_TF;
244 regs->flags &= ~X86_EFLAGS_IF;
245
246 /* Now we set present bit in PTE and single step. */
247 disarm_kmmio_fault_page(ctx->fpage->page, NULL);
248
249 /*
250 * If another cpu accesses the same page while we are stepping,
251 * the access will not be caught. It will simply succeed and the
252 * only downside is we lose the event. If this becomes a problem,
253 * the user should drop to single cpu before tracing.
254 */
255
256 put_cpu_var(kmmio_ctx);
257 return 1; /* fault handled */
258
259no_kmmio_ctx:
260 put_cpu_var(kmmio_ctx);
261no_kmmio:
262 rcu_read_unlock();
263 preempt_enable_no_resched();
264 return ret;
265}
266
267/*
268 * Interrupts are disabled on entry as trap1 is an interrupt gate
269 * and they remain disabled thorough out this function.
270 * This must always get called as the pair to kmmio_handler().
271 */
272static int post_kmmio_handler(unsigned long condition, struct pt_regs *regs)
273{
274 int ret = 0;
275 struct kmmio_context *ctx = &get_cpu_var(kmmio_ctx);
276
277 if (!ctx->active) {
278 pr_debug("kmmio: spurious debug trap on CPU %d.\n",
279 smp_processor_id());
280 goto out;
281 }
282
283 if (ctx->probe && ctx->probe->post_handler)
284 ctx->probe->post_handler(ctx->probe, condition, regs);
285
286 arm_kmmio_fault_page(ctx->fpage->page, NULL);
287
288 regs->flags &= ~X86_EFLAGS_TF;
289 regs->flags |= ctx->saved_flags;
290
291 /* These were acquired in kmmio_handler(). */
292 ctx->active--;
293 BUG_ON(ctx->active);
294 rcu_read_unlock();
295 preempt_enable_no_resched();
296
297 /*
298 * if somebody else is singlestepping across a probe point, flags
299 * will have TF set, in which case, continue the remaining processing
300 * of do_debug, as if this is not a probe hit.
301 */
302 if (!(regs->flags & X86_EFLAGS_TF))
303 ret = 1;
304out:
305 put_cpu_var(kmmio_ctx);
306 return ret;
307}
308
309/* You must be holding kmmio_lock. */
310static int add_kmmio_fault_page(unsigned long page)
311{
312 struct kmmio_fault_page *f;
313
314 page &= PAGE_MASK;
315 f = get_kmmio_fault_page(page);
316 if (f) {
317 if (!f->count)
318 arm_kmmio_fault_page(f->page, NULL);
319 f->count++;
320 return 0;
321 }
322
323 f = kmalloc(sizeof(*f), GFP_ATOMIC);
324 if (!f)
325 return -1;
326
327 f->count = 1;
328 f->page = page;
329 list_add_rcu(&f->list, kmmio_page_list(f->page));
330
331 arm_kmmio_fault_page(f->page, NULL);
332
333 return 0;
334}
335
336/* You must be holding kmmio_lock. */
337static void release_kmmio_fault_page(unsigned long page,
338 struct kmmio_fault_page **release_list)
339{
340 struct kmmio_fault_page *f;
341
342 page &= PAGE_MASK;
343 f = get_kmmio_fault_page(page);
344 if (!f)
345 return;
346
347 f->count--;
348 BUG_ON(f->count < 0);
349 if (!f->count) {
350 disarm_kmmio_fault_page(f->page, NULL);
351 f->release_next = *release_list;
352 *release_list = f;
353 }
354}
355
356/*
357 * With page-unaligned ioremaps, one or two armed pages may contain
358 * addresses from outside the intended mapping. Events for these addresses
359 * are currently silently dropped. The events may result only from programming
360 * mistakes by accessing addresses before the beginning or past the end of a
361 * mapping.
362 */
363int register_kmmio_probe(struct kmmio_probe *p)
364{
365 unsigned long flags;
366 int ret = 0;
367 unsigned long size = 0;
368 const unsigned long size_lim = p->len + (p->addr & ~PAGE_MASK);
369
370 spin_lock_irqsave(&kmmio_lock, flags);
371 if (get_kmmio_probe(p->addr)) {
372 ret = -EEXIST;
373 goto out;
374 }
375 kmmio_count++;
376 list_add_rcu(&p->list, &kmmio_probes);
377 while (size < size_lim) {
378 if (add_kmmio_fault_page(p->addr + size))
379 pr_err("kmmio: Unable to set page fault.\n");
380 size += PAGE_SIZE;
381 }
382out:
383 spin_unlock_irqrestore(&kmmio_lock, flags);
384 /*
385 * XXX: What should I do here?
386 * Here was a call to global_flush_tlb(), but it does not exist
387 * anymore. It seems it's not needed after all.
388 */
389 return ret;
390}
391EXPORT_SYMBOL(register_kmmio_probe);
392
393static void rcu_free_kmmio_fault_pages(struct rcu_head *head)
394{
395 struct kmmio_delayed_release *dr = container_of(
396 head,
397 struct kmmio_delayed_release,
398 rcu);
399 struct kmmio_fault_page *p = dr->release_list;
400 while (p) {
401 struct kmmio_fault_page *next = p->release_next;
402 BUG_ON(p->count);
403 kfree(p);
404 p = next;
405 }
406 kfree(dr);
407}
408
409static void remove_kmmio_fault_pages(struct rcu_head *head)
410{
411 struct kmmio_delayed_release *dr = container_of(
412 head,
413 struct kmmio_delayed_release,
414 rcu);
415 struct kmmio_fault_page *p = dr->release_list;
416 struct kmmio_fault_page **prevp = &dr->release_list;
417 unsigned long flags;
418 spin_lock_irqsave(&kmmio_lock, flags);
419 while (p) {
420 if (!p->count)
421 list_del_rcu(&p->list);
422 else
423 *prevp = p->release_next;
424 prevp = &p->release_next;
425 p = p->release_next;
426 }
427 spin_unlock_irqrestore(&kmmio_lock, flags);
428 /* This is the real RCU destroy call. */
429 call_rcu(&dr->rcu, rcu_free_kmmio_fault_pages);
430}
431
432/*
433 * Remove a kmmio probe. You have to synchronize_rcu() before you can be
434 * sure that the callbacks will not be called anymore. Only after that
435 * you may actually release your struct kmmio_probe.
436 *
437 * Unregistering a kmmio fault page has three steps:
438 * 1. release_kmmio_fault_page()
439 * Disarm the page, wait a grace period to let all faults finish.
440 * 2. remove_kmmio_fault_pages()
441 * Remove the pages from kmmio_page_table.
442 * 3. rcu_free_kmmio_fault_pages()
443 * Actally free the kmmio_fault_page structs as with RCU.
444 */
445void unregister_kmmio_probe(struct kmmio_probe *p)
446{
447 unsigned long flags;
448 unsigned long size = 0;
449 const unsigned long size_lim = p->len + (p->addr & ~PAGE_MASK);
450 struct kmmio_fault_page *release_list = NULL;
451 struct kmmio_delayed_release *drelease;
452
453 spin_lock_irqsave(&kmmio_lock, flags);
454 while (size < size_lim) {
455 release_kmmio_fault_page(p->addr + size, &release_list);
456 size += PAGE_SIZE;
457 }
458 list_del_rcu(&p->list);
459 kmmio_count--;
460 spin_unlock_irqrestore(&kmmio_lock, flags);
461
462 drelease = kmalloc(sizeof(*drelease), GFP_ATOMIC);
463 if (!drelease) {
464 pr_crit("kmmio: leaking kmmio_fault_page objects.\n");
465 return;
466 }
467 drelease->release_list = release_list;
468
469 /*
470 * This is not really RCU here. We have just disarmed a set of
471 * pages so that they cannot trigger page faults anymore. However,
472 * we cannot remove the pages from kmmio_page_table,
473 * because a probe hit might be in flight on another CPU. The
474 * pages are collected into a list, and they will be removed from
475 * kmmio_page_table when it is certain that no probe hit related to
476 * these pages can be in flight. RCU grace period sounds like a
477 * good choice.
478 *
479 * If we removed the pages too early, kmmio page fault handler might
480 * not find the respective kmmio_fault_page and determine it's not
481 * a kmmio fault, when it actually is. This would lead to madness.
482 */
483 call_rcu(&drelease->rcu, remove_kmmio_fault_pages);
484}
485EXPORT_SYMBOL(unregister_kmmio_probe);
486
487static int kmmio_die_notifier(struct notifier_block *nb, unsigned long val,
488 void *args)
489{
490 struct die_args *arg = args;
491
492 if (val == DIE_DEBUG && (arg->err & DR_STEP))
493 if (post_kmmio_handler(arg->err, arg->regs) == 1)
494 return NOTIFY_STOP;
495
496 return NOTIFY_DONE;
497}
498
499static struct notifier_block nb_die = {
500 .notifier_call = kmmio_die_notifier
501};
502
503static int __init init_kmmio(void)
504{
505 int i;
506 for (i = 0; i < KMMIO_PAGE_TABLE_SIZE; i++)
507 INIT_LIST_HEAD(&kmmio_page_table[i]);
508 return register_die_notifier(&nb_die);
509}
510fs_initcall(init_kmmio); /* should be before device_initcall() */
diff --git a/arch/x86/mm/mmio-mod.c b/arch/x86/mm/mmio-mod.c
new file mode 100644
index 000000000000..e7397e108beb
--- /dev/null
+++ b/arch/x86/mm/mmio-mod.c
@@ -0,0 +1,515 @@
1/*
2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License as published by
4 * the Free Software Foundation; either version 2 of the License, or
5 * (at your option) any later version.
6 *
7 * This program is distributed in the hope that it will be useful,
8 * but WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
10 * GNU General Public License for more details.
11 *
12 * You should have received a copy of the GNU General Public License
13 * along with this program; if not, write to the Free Software
14 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
15 *
16 * Copyright (C) IBM Corporation, 2005
17 * Jeff Muizelaar, 2006, 2007
18 * Pekka Paalanen, 2008 <pq@iki.fi>
19 *
20 * Derived from the read-mod example from relay-examples by Tom Zanussi.
21 */
22#define DEBUG 1
23
24#include <linux/module.h>
25#include <linux/debugfs.h>
26#include <linux/uaccess.h>
27#include <linux/io.h>
28#include <linux/version.h>
29#include <linux/kallsyms.h>
30#include <asm/pgtable.h>
31#include <linux/mmiotrace.h>
32#include <asm/e820.h> /* for ISA_START_ADDRESS */
33#include <asm/atomic.h>
34#include <linux/percpu.h>
35#include <linux/cpu.h>
36
37#include "pf_in.h"
38
39#define NAME "mmiotrace: "
40
41struct trap_reason {
42 unsigned long addr;
43 unsigned long ip;
44 enum reason_type type;
45 int active_traces;
46};
47
48struct remap_trace {
49 struct list_head list;
50 struct kmmio_probe probe;
51 resource_size_t phys;
52 unsigned long id;
53};
54
55/* Accessed per-cpu. */
56static DEFINE_PER_CPU(struct trap_reason, pf_reason);
57static DEFINE_PER_CPU(struct mmiotrace_rw, cpu_trace);
58
59#if 0 /* XXX: no way gather this info anymore */
60/* Access to this is not per-cpu. */
61static DEFINE_PER_CPU(atomic_t, dropped);
62#endif
63
64static struct dentry *marker_file;
65
66static DEFINE_MUTEX(mmiotrace_mutex);
67static DEFINE_SPINLOCK(trace_lock);
68static atomic_t mmiotrace_enabled;
69static LIST_HEAD(trace_list); /* struct remap_trace */
70
71/*
72 * Locking in this file:
73 * - mmiotrace_mutex enforces enable/disable_mmiotrace() critical sections.
74 * - mmiotrace_enabled may be modified only when holding mmiotrace_mutex
75 * and trace_lock.
76 * - Routines depending on is_enabled() must take trace_lock.
77 * - trace_list users must hold trace_lock.
78 * - is_enabled() guarantees that mmio_trace_record is allowed.
79 * - pre/post callbacks assume the effect of is_enabled() being true.
80 */
81
82/* module parameters */
83static unsigned long filter_offset;
84static int nommiotrace;
85static int trace_pc;
86
87module_param(filter_offset, ulong, 0);
88module_param(nommiotrace, bool, 0);
89module_param(trace_pc, bool, 0);
90
91MODULE_PARM_DESC(filter_offset, "Start address of traced mappings.");
92MODULE_PARM_DESC(nommiotrace, "Disable actual MMIO tracing.");
93MODULE_PARM_DESC(trace_pc, "Record address of faulting instructions.");
94
95static bool is_enabled(void)
96{
97 return atomic_read(&mmiotrace_enabled);
98}
99
100#if 0 /* XXX: needs rewrite */
101/*
102 * Write callback for the debugfs entry:
103 * Read a marker and write it to the mmio trace log
104 */
105static ssize_t write_marker(struct file *file, const char __user *buffer,
106 size_t count, loff_t *ppos)
107{
108 char *event = NULL;
109 struct mm_io_header *headp;
110 ssize_t len = (count > 65535) ? 65535 : count;
111
112 event = kzalloc(sizeof(*headp) + len, GFP_KERNEL);
113 if (!event)
114 return -ENOMEM;
115
116 headp = (struct mm_io_header *)event;
117 headp->type = MMIO_MAGIC | (MMIO_MARKER << MMIO_OPCODE_SHIFT);
118 headp->data_len = len;
119
120 if (copy_from_user(event + sizeof(*headp), buffer, len)) {
121 kfree(event);
122 return -EFAULT;
123 }
124
125 spin_lock_irq(&trace_lock);
126#if 0 /* XXX: convert this to use tracing */
127 if (is_enabled())
128 relay_write(chan, event, sizeof(*headp) + len);
129 else
130#endif
131 len = -EINVAL;
132 spin_unlock_irq(&trace_lock);
133 kfree(event);
134 return len;
135}
136#endif
137
138static void print_pte(unsigned long address)
139{
140 unsigned int level;
141 pte_t *pte = lookup_address(address, &level);
142
143 if (!pte) {
144 pr_err(NAME "Error in %s: no pte for page 0x%08lx\n",
145 __func__, address);
146 return;
147 }
148
149 if (level == PG_LEVEL_2M) {
150 pr_emerg(NAME "4MB pages are not currently supported: "
151 "0x%08lx\n", address);
152 BUG();
153 }
154 pr_info(NAME "pte for 0x%lx: 0x%llx 0x%llx\n", address,
155 (unsigned long long)pte_val(*pte),
156 (unsigned long long)pte_val(*pte) & _PAGE_PRESENT);
157}
158
159/*
160 * For some reason the pre/post pairs have been called in an
161 * unmatched order. Report and die.
162 */
163static void die_kmmio_nesting_error(struct pt_regs *regs, unsigned long addr)
164{
165 const struct trap_reason *my_reason = &get_cpu_var(pf_reason);
166 pr_emerg(NAME "unexpected fault for address: 0x%08lx, "
167 "last fault for address: 0x%08lx\n",
168 addr, my_reason->addr);
169 print_pte(addr);
170 print_symbol(KERN_EMERG "faulting IP is at %s\n", regs->ip);
171 print_symbol(KERN_EMERG "last faulting IP was at %s\n", my_reason->ip);
172#ifdef __i386__
173 pr_emerg("eax: %08lx ebx: %08lx ecx: %08lx edx: %08lx\n",
174 regs->ax, regs->bx, regs->cx, regs->dx);
175 pr_emerg("esi: %08lx edi: %08lx ebp: %08lx esp: %08lx\n",
176 regs->si, regs->di, regs->bp, regs->sp);
177#else
178 pr_emerg("rax: %016lx rcx: %016lx rdx: %016lx\n",
179 regs->ax, regs->cx, regs->dx);
180 pr_emerg("rsi: %016lx rdi: %016lx rbp: %016lx rsp: %016lx\n",
181 regs->si, regs->di, regs->bp, regs->sp);
182#endif
183 put_cpu_var(pf_reason);
184 BUG();
185}
186
187static void pre(struct kmmio_probe *p, struct pt_regs *regs,
188 unsigned long addr)
189{
190 struct trap_reason *my_reason = &get_cpu_var(pf_reason);
191 struct mmiotrace_rw *my_trace = &get_cpu_var(cpu_trace);
192 const unsigned long instptr = instruction_pointer(regs);
193 const enum reason_type type = get_ins_type(instptr);
194 struct remap_trace *trace = p->private;
195
196 /* it doesn't make sense to have more than one active trace per cpu */
197 if (my_reason->active_traces)
198 die_kmmio_nesting_error(regs, addr);
199 else
200 my_reason->active_traces++;
201
202 my_reason->type = type;
203 my_reason->addr = addr;
204 my_reason->ip = instptr;
205
206 my_trace->phys = addr - trace->probe.addr + trace->phys;
207 my_trace->map_id = trace->id;
208
209 /*
210 * Only record the program counter when requested.
211 * It may taint clean-room reverse engineering.
212 */
213 if (trace_pc)
214 my_trace->pc = instptr;
215 else
216 my_trace->pc = 0;
217
218 /*
219 * XXX: the timestamp recorded will be *after* the tracing has been
220 * done, not at the time we hit the instruction. SMP implications
221 * on event ordering?
222 */
223
224 switch (type) {
225 case REG_READ:
226 my_trace->opcode = MMIO_READ;
227 my_trace->width = get_ins_mem_width(instptr);
228 break;
229 case REG_WRITE:
230 my_trace->opcode = MMIO_WRITE;
231 my_trace->width = get_ins_mem_width(instptr);
232 my_trace->value = get_ins_reg_val(instptr, regs);
233 break;
234 case IMM_WRITE:
235 my_trace->opcode = MMIO_WRITE;
236 my_trace->width = get_ins_mem_width(instptr);
237 my_trace->value = get_ins_imm_val(instptr);
238 break;
239 default:
240 {
241 unsigned char *ip = (unsigned char *)instptr;
242 my_trace->opcode = MMIO_UNKNOWN_OP;
243 my_trace->width = 0;
244 my_trace->value = (*ip) << 16 | *(ip + 1) << 8 |
245 *(ip + 2);
246 }
247 }
248 put_cpu_var(cpu_trace);
249 put_cpu_var(pf_reason);
250}
251
252static void post(struct kmmio_probe *p, unsigned long condition,
253 struct pt_regs *regs)
254{
255 struct trap_reason *my_reason = &get_cpu_var(pf_reason);
256 struct mmiotrace_rw *my_trace = &get_cpu_var(cpu_trace);
257
258 /* this should always return the active_trace count to 0 */
259 my_reason->active_traces--;
260 if (my_reason->active_traces) {
261 pr_emerg(NAME "unexpected post handler");
262 BUG();
263 }
264
265 switch (my_reason->type) {
266 case REG_READ:
267 my_trace->value = get_ins_reg_val(my_reason->ip, regs);
268 break;
269 default:
270 break;
271 }
272
273 mmio_trace_rw(my_trace);
274 put_cpu_var(cpu_trace);
275 put_cpu_var(pf_reason);
276}
277
278static void ioremap_trace_core(resource_size_t offset, unsigned long size,
279 void __iomem *addr)
280{
281 static atomic_t next_id;
282 struct remap_trace *trace = kmalloc(sizeof(*trace), GFP_KERNEL);
283 /* These are page-unaligned. */
284 struct mmiotrace_map map = {
285 .phys = offset,
286 .virt = (unsigned long)addr,
287 .len = size,
288 .opcode = MMIO_PROBE
289 };
290
291 if (!trace) {
292 pr_err(NAME "kmalloc failed in ioremap\n");
293 return;
294 }
295
296 *trace = (struct remap_trace) {
297 .probe = {
298 .addr = (unsigned long)addr,
299 .len = size,
300 .pre_handler = pre,
301 .post_handler = post,
302 .private = trace
303 },
304 .phys = offset,
305 .id = atomic_inc_return(&next_id)
306 };
307 map.map_id = trace->id;
308
309 spin_lock_irq(&trace_lock);
310 if (!is_enabled())
311 goto not_enabled;
312
313 mmio_trace_mapping(&map);
314 list_add_tail(&trace->list, &trace_list);
315 if (!nommiotrace)
316 register_kmmio_probe(&trace->probe);
317
318not_enabled:
319 spin_unlock_irq(&trace_lock);
320}
321
322void mmiotrace_ioremap(resource_size_t offset, unsigned long size,
323 void __iomem *addr)
324{
325 if (!is_enabled()) /* recheck and proper locking in *_core() */
326 return;
327
328 pr_debug(NAME "ioremap_*(0x%llx, 0x%lx) = %p\n",
329 (unsigned long long)offset, size, addr);
330 if ((filter_offset) && (offset != filter_offset))
331 return;
332 ioremap_trace_core(offset, size, addr);
333}
334
335static void iounmap_trace_core(volatile void __iomem *addr)
336{
337 struct mmiotrace_map map = {
338 .phys = 0,
339 .virt = (unsigned long)addr,
340 .len = 0,
341 .opcode = MMIO_UNPROBE
342 };
343 struct remap_trace *trace;
344 struct remap_trace *tmp;
345 struct remap_trace *found_trace = NULL;
346
347 pr_debug(NAME "Unmapping %p.\n", addr);
348
349 spin_lock_irq(&trace_lock);
350 if (!is_enabled())
351 goto not_enabled;
352
353 list_for_each_entry_safe(trace, tmp, &trace_list, list) {
354 if ((unsigned long)addr == trace->probe.addr) {
355 if (!nommiotrace)
356 unregister_kmmio_probe(&trace->probe);
357 list_del(&trace->list);
358 found_trace = trace;
359 break;
360 }
361 }
362 map.map_id = (found_trace) ? found_trace->id : -1;
363 mmio_trace_mapping(&map);
364
365not_enabled:
366 spin_unlock_irq(&trace_lock);
367 if (found_trace) {
368 synchronize_rcu(); /* unregister_kmmio_probe() requirement */
369 kfree(found_trace);
370 }
371}
372
373void mmiotrace_iounmap(volatile void __iomem *addr)
374{
375 might_sleep();
376 if (is_enabled()) /* recheck and proper locking in *_core() */
377 iounmap_trace_core(addr);
378}
379
380static void clear_trace_list(void)
381{
382 struct remap_trace *trace;
383 struct remap_trace *tmp;
384
385 /*
386 * No locking required, because the caller ensures we are in a
387 * critical section via mutex, and is_enabled() is false,
388 * i.e. nothing can traverse or modify this list.
389 * Caller also ensures is_enabled() cannot change.
390 */
391 list_for_each_entry(trace, &trace_list, list) {
392 pr_notice(NAME "purging non-iounmapped "
393 "trace @0x%08lx, size 0x%lx.\n",
394 trace->probe.addr, trace->probe.len);
395 if (!nommiotrace)
396 unregister_kmmio_probe(&trace->probe);
397 }
398 synchronize_rcu(); /* unregister_kmmio_probe() requirement */
399
400 list_for_each_entry_safe(trace, tmp, &trace_list, list) {
401 list_del(&trace->list);
402 kfree(trace);
403 }
404}
405
406#ifdef CONFIG_HOTPLUG_CPU
407static cpumask_t downed_cpus;
408
409static void enter_uniprocessor(void)
410{
411 int cpu;
412 int err;
413
414 get_online_cpus();
415 downed_cpus = cpu_online_map;
416 cpu_clear(first_cpu(cpu_online_map), downed_cpus);
417 if (num_online_cpus() > 1)
418 pr_notice(NAME "Disabling non-boot CPUs...\n");
419 put_online_cpus();
420
421 for_each_cpu_mask(cpu, downed_cpus) {
422 err = cpu_down(cpu);
423 if (!err)
424 pr_info(NAME "CPU%d is down.\n", cpu);
425 else
426 pr_err(NAME "Error taking CPU%d down: %d\n", cpu, err);
427 }
428 if (num_online_cpus() > 1)
429 pr_warning(NAME "multiple CPUs still online, "
430 "may miss events.\n");
431}
432
433static void leave_uniprocessor(void)
434{
435 int cpu;
436 int err;
437
438 if (cpus_weight(downed_cpus) == 0)
439 return;
440 pr_notice(NAME "Re-enabling CPUs...\n");
441 for_each_cpu_mask(cpu, downed_cpus) {
442 err = cpu_up(cpu);
443 if (!err)
444 pr_info(NAME "enabled CPU%d.\n", cpu);
445 else
446 pr_err(NAME "cannot re-enable CPU%d: %d\n", cpu, err);
447 }
448}
449
450#else /* !CONFIG_HOTPLUG_CPU */
451static void enter_uniprocessor(void)
452{
453 if (num_online_cpus() > 1)
454 pr_warning(NAME "multiple CPUs are online, may miss events. "
455 "Suggest booting with maxcpus=1 kernel argument.\n");
456}
457
458static void leave_uniprocessor(void)
459{
460}
461#endif
462
463#if 0 /* XXX: out of order */
464static struct file_operations fops_marker = {
465 .owner = THIS_MODULE,
466 .write = write_marker
467};
468#endif
469
470void enable_mmiotrace(void)
471{
472 mutex_lock(&mmiotrace_mutex);
473 if (is_enabled())
474 goto out;
475
476#if 0 /* XXX: tracing does not support text entries */
477 marker_file = debugfs_create_file("marker", 0660, dir, NULL,
478 &fops_marker);
479 if (!marker_file)
480 pr_err(NAME "marker file creation failed.\n");
481#endif
482
483 if (nommiotrace)
484 pr_info(NAME "MMIO tracing disabled.\n");
485 enter_uniprocessor();
486 spin_lock_irq(&trace_lock);
487 atomic_inc(&mmiotrace_enabled);
488 spin_unlock_irq(&trace_lock);
489 pr_info(NAME "enabled.\n");
490out:
491 mutex_unlock(&mmiotrace_mutex);
492}
493
494void disable_mmiotrace(void)
495{
496 mutex_lock(&mmiotrace_mutex);
497 if (!is_enabled())
498 goto out;
499
500 spin_lock_irq(&trace_lock);
501 atomic_dec(&mmiotrace_enabled);
502 BUG_ON(is_enabled());
503 spin_unlock_irq(&trace_lock);
504
505 clear_trace_list(); /* guarantees: no more kmmio callbacks */
506 leave_uniprocessor();
507 if (marker_file) {
508 debugfs_remove(marker_file);
509 marker_file = NULL;
510 }
511
512 pr_info(NAME "disabled.\n");
513out:
514 mutex_unlock(&mmiotrace_mutex);
515}
diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index 60bcb5b6a37e..57970f2935c0 100644
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -227,6 +227,7 @@ pte_t *lookup_address(unsigned long address, unsigned int *level)
227 227
228 return pte_offset_kernel(pmd, address); 228 return pte_offset_kernel(pmd, address);
229} 229}
230EXPORT_SYMBOL_GPL(lookup_address);
230 231
231/* 232/*
232 * Set the new pmd in all the pgds we know about: 233 * Set the new pmd in all the pgds we know about:
diff --git a/arch/x86/mm/pf_in.c b/arch/x86/mm/pf_in.c
new file mode 100644
index 000000000000..efa1911e20ca
--- /dev/null
+++ b/arch/x86/mm/pf_in.c
@@ -0,0 +1,489 @@
1/*
2 * Fault Injection Test harness (FI)
3 * Copyright (C) Intel Crop.
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version 2
8 * of the License, or (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
18 * USA.
19 *
20 */
21
22/* Id: pf_in.c,v 1.1.1.1 2002/11/12 05:56:32 brlock Exp
23 * Copyright by Intel Crop., 2002
24 * Louis Zhuang (louis.zhuang@intel.com)
25 *
26 * Bjorn Steinbrink (B.Steinbrink@gmx.de), 2007
27 */
28
29#include <linux/module.h>
30#include <linux/ptrace.h> /* struct pt_regs */
31#include "pf_in.h"
32
33#ifdef __i386__
34/* IA32 Manual 3, 2-1 */
35static unsigned char prefix_codes[] = {
36 0xF0, 0xF2, 0xF3, 0x2E, 0x36, 0x3E, 0x26, 0x64,
37 0x65, 0x2E, 0x3E, 0x66, 0x67
38};
39/* IA32 Manual 3, 3-432*/
40static unsigned int reg_rop[] = {
41 0x8A, 0x8B, 0xB60F, 0xB70F, 0xBE0F, 0xBF0F
42};
43static unsigned int reg_wop[] = { 0x88, 0x89 };
44static unsigned int imm_wop[] = { 0xC6, 0xC7 };
45/* IA32 Manual 3, 3-432*/
46static unsigned int rw8[] = { 0x88, 0x8A, 0xC6 };
47static unsigned int rw32[] = {
48 0x89, 0x8B, 0xC7, 0xB60F, 0xB70F, 0xBE0F, 0xBF0F
49};
50static unsigned int mw8[] = { 0x88, 0x8A, 0xC6, 0xB60F, 0xBE0F };
51static unsigned int mw16[] = { 0xB70F, 0xBF0F };
52static unsigned int mw32[] = { 0x89, 0x8B, 0xC7 };
53static unsigned int mw64[] = {};
54#else /* not __i386__ */
55static unsigned char prefix_codes[] = {
56 0x66, 0x67, 0x2E, 0x3E, 0x26, 0x64, 0x65, 0x36,
57 0xF0, 0xF3, 0xF2,
58 /* REX Prefixes */
59 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47,
60 0x48, 0x49, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f
61};
62/* AMD64 Manual 3, Appendix A*/
63static unsigned int reg_rop[] = {
64 0x8A, 0x8B, 0xB60F, 0xB70F, 0xBE0F, 0xBF0F
65};
66static unsigned int reg_wop[] = { 0x88, 0x89 };
67static unsigned int imm_wop[] = { 0xC6, 0xC7 };
68static unsigned int rw8[] = { 0xC6, 0x88, 0x8A };
69static unsigned int rw32[] = {
70 0xC7, 0x89, 0x8B, 0xB60F, 0xB70F, 0xBE0F, 0xBF0F
71};
72/* 8 bit only */
73static unsigned int mw8[] = { 0xC6, 0x88, 0x8A, 0xB60F, 0xBE0F };
74/* 16 bit only */
75static unsigned int mw16[] = { 0xB70F, 0xBF0F };
76/* 16 or 32 bit */
77static unsigned int mw32[] = { 0xC7 };
78/* 16, 32 or 64 bit */
79static unsigned int mw64[] = { 0x89, 0x8B };
80#endif /* not __i386__ */
81
82static int skip_prefix(unsigned char *addr, int *shorted, int *enlarged,
83 int *rexr)
84{
85 int i;
86 unsigned char *p = addr;
87 *shorted = 0;
88 *enlarged = 0;
89 *rexr = 0;
90
91restart:
92 for (i = 0; i < ARRAY_SIZE(prefix_codes); i++) {
93 if (*p == prefix_codes[i]) {
94 if (*p == 0x66)
95 *shorted = 1;
96#ifdef __amd64__
97 if ((*p & 0xf8) == 0x48)
98 *enlarged = 1;
99 if ((*p & 0xf4) == 0x44)
100 *rexr = 1;
101#endif
102 p++;
103 goto restart;
104 }
105 }
106
107 return (p - addr);
108}
109
110static int get_opcode(unsigned char *addr, unsigned int *opcode)
111{
112 int len;
113
114 if (*addr == 0x0F) {
115 /* 0x0F is extension instruction */
116 *opcode = *(unsigned short *)addr;
117 len = 2;
118 } else {
119 *opcode = *addr;
120 len = 1;
121 }
122
123 return len;
124}
125
126#define CHECK_OP_TYPE(opcode, array, type) \
127 for (i = 0; i < ARRAY_SIZE(array); i++) { \
128 if (array[i] == opcode) { \
129 rv = type; \
130 goto exit; \
131 } \
132 }
133
134enum reason_type get_ins_type(unsigned long ins_addr)
135{
136 unsigned int opcode;
137 unsigned char *p;
138 int shorted, enlarged, rexr;
139 int i;
140 enum reason_type rv = OTHERS;
141
142 p = (unsigned char *)ins_addr;
143 p += skip_prefix(p, &shorted, &enlarged, &rexr);
144 p += get_opcode(p, &opcode);
145
146 CHECK_OP_TYPE(opcode, reg_rop, REG_READ);
147 CHECK_OP_TYPE(opcode, reg_wop, REG_WRITE);
148 CHECK_OP_TYPE(opcode, imm_wop, IMM_WRITE);
149
150exit:
151 return rv;
152}
153#undef CHECK_OP_TYPE
154
155static unsigned int get_ins_reg_width(unsigned long ins_addr)
156{
157 unsigned int opcode;
158 unsigned char *p;
159 int i, shorted, enlarged, rexr;
160
161 p = (unsigned char *)ins_addr;
162 p += skip_prefix(p, &shorted, &enlarged, &rexr);
163 p += get_opcode(p, &opcode);
164
165 for (i = 0; i < ARRAY_SIZE(rw8); i++)
166 if (rw8[i] == opcode)
167 return 1;
168
169 for (i = 0; i < ARRAY_SIZE(rw32); i++)
170 if (rw32[i] == opcode)
171 return (shorted ? 2 : (enlarged ? 8 : 4));
172
173 printk(KERN_ERR "mmiotrace: Unknown opcode 0x%02x\n", opcode);
174 return 0;
175}
176
177unsigned int get_ins_mem_width(unsigned long ins_addr)
178{
179 unsigned int opcode;
180 unsigned char *p;
181 int i, shorted, enlarged, rexr;
182
183 p = (unsigned char *)ins_addr;
184 p += skip_prefix(p, &shorted, &enlarged, &rexr);
185 p += get_opcode(p, &opcode);
186
187 for (i = 0; i < ARRAY_SIZE(mw8); i++)
188 if (mw8[i] == opcode)
189 return 1;
190
191 for (i = 0; i < ARRAY_SIZE(mw16); i++)
192 if (mw16[i] == opcode)
193 return 2;
194
195 for (i = 0; i < ARRAY_SIZE(mw32); i++)
196 if (mw32[i] == opcode)
197 return shorted ? 2 : 4;
198
199 for (i = 0; i < ARRAY_SIZE(mw64); i++)
200 if (mw64[i] == opcode)
201 return shorted ? 2 : (enlarged ? 8 : 4);
202
203 printk(KERN_ERR "mmiotrace: Unknown opcode 0x%02x\n", opcode);
204 return 0;
205}
206
207/*
208 * Define register ident in mod/rm byte.
209 * Note: these are NOT the same as in ptrace-abi.h.
210 */
211enum {
212 arg_AL = 0,
213 arg_CL = 1,
214 arg_DL = 2,
215 arg_BL = 3,
216 arg_AH = 4,
217 arg_CH = 5,
218 arg_DH = 6,
219 arg_BH = 7,
220
221 arg_AX = 0,
222 arg_CX = 1,
223 arg_DX = 2,
224 arg_BX = 3,
225 arg_SP = 4,
226 arg_BP = 5,
227 arg_SI = 6,
228 arg_DI = 7,
229#ifdef __amd64__
230 arg_R8 = 8,
231 arg_R9 = 9,
232 arg_R10 = 10,
233 arg_R11 = 11,
234 arg_R12 = 12,
235 arg_R13 = 13,
236 arg_R14 = 14,
237 arg_R15 = 15
238#endif
239};
240
241static unsigned char *get_reg_w8(int no, struct pt_regs *regs)
242{
243 unsigned char *rv = NULL;
244
245 switch (no) {
246 case arg_AL:
247 rv = (unsigned char *)&regs->ax;
248 break;
249 case arg_BL:
250 rv = (unsigned char *)&regs->bx;
251 break;
252 case arg_CL:
253 rv = (unsigned char *)&regs->cx;
254 break;
255 case arg_DL:
256 rv = (unsigned char *)&regs->dx;
257 break;
258 case arg_AH:
259 rv = 1 + (unsigned char *)&regs->ax;
260 break;
261 case arg_BH:
262 rv = 1 + (unsigned char *)&regs->bx;
263 break;
264 case arg_CH:
265 rv = 1 + (unsigned char *)&regs->cx;
266 break;
267 case arg_DH:
268 rv = 1 + (unsigned char *)&regs->dx;
269 break;
270#ifdef __amd64__
271 case arg_R8:
272 rv = (unsigned char *)&regs->r8;
273 break;
274 case arg_R9:
275 rv = (unsigned char *)&regs->r9;
276 break;
277 case arg_R10:
278 rv = (unsigned char *)&regs->r10;
279 break;
280 case arg_R11:
281 rv = (unsigned char *)&regs->r11;
282 break;
283 case arg_R12:
284 rv = (unsigned char *)&regs->r12;
285 break;
286 case arg_R13:
287 rv = (unsigned char *)&regs->r13;
288 break;
289 case arg_R14:
290 rv = (unsigned char *)&regs->r14;
291 break;
292 case arg_R15:
293 rv = (unsigned char *)&regs->r15;
294 break;
295#endif
296 default:
297 printk(KERN_ERR "mmiotrace: Error reg no# %d\n", no);
298 break;
299 }
300 return rv;
301}
302
303static unsigned long *get_reg_w32(int no, struct pt_regs *regs)
304{
305 unsigned long *rv = NULL;
306
307 switch (no) {
308 case arg_AX:
309 rv = &regs->ax;
310 break;
311 case arg_BX:
312 rv = &regs->bx;
313 break;
314 case arg_CX:
315 rv = &regs->cx;
316 break;
317 case arg_DX:
318 rv = &regs->dx;
319 break;
320 case arg_SP:
321 rv = &regs->sp;
322 break;
323 case arg_BP:
324 rv = &regs->bp;
325 break;
326 case arg_SI:
327 rv = &regs->si;
328 break;
329 case arg_DI:
330 rv = &regs->di;
331 break;
332#ifdef __amd64__
333 case arg_R8:
334 rv = &regs->r8;
335 break;
336 case arg_R9:
337 rv = &regs->r9;
338 break;
339 case arg_R10:
340 rv = &regs->r10;
341 break;
342 case arg_R11:
343 rv = &regs->r11;
344 break;
345 case arg_R12:
346 rv = &regs->r12;
347 break;
348 case arg_R13:
349 rv = &regs->r13;
350 break;
351 case arg_R14:
352 rv = &regs->r14;
353 break;
354 case arg_R15:
355 rv = &regs->r15;
356 break;
357#endif
358 default:
359 printk(KERN_ERR "mmiotrace: Error reg no# %d\n", no);
360 }
361
362 return rv;
363}
364
365unsigned long get_ins_reg_val(unsigned long ins_addr, struct pt_regs *regs)
366{
367 unsigned int opcode;
368 unsigned char mod_rm;
369 int reg;
370 unsigned char *p;
371 int i, shorted, enlarged, rexr;
372 unsigned long rv;
373
374 p = (unsigned char *)ins_addr;
375 p += skip_prefix(p, &shorted, &enlarged, &rexr);
376 p += get_opcode(p, &opcode);
377 for (i = 0; i < ARRAY_SIZE(reg_rop); i++)
378 if (reg_rop[i] == opcode) {
379 rv = REG_READ;
380 goto do_work;
381 }
382
383 for (i = 0; i < ARRAY_SIZE(reg_wop); i++)
384 if (reg_wop[i] == opcode) {
385 rv = REG_WRITE;
386 goto do_work;
387 }
388
389 printk(KERN_ERR "mmiotrace: Not a register instruction, opcode "
390 "0x%02x\n", opcode);
391 goto err;
392
393do_work:
394 mod_rm = *p;
395 reg = ((mod_rm >> 3) & 0x7) | (rexr << 3);
396 switch (get_ins_reg_width(ins_addr)) {
397 case 1:
398 return *get_reg_w8(reg, regs);
399
400 case 2:
401 return *(unsigned short *)get_reg_w32(reg, regs);
402
403 case 4:
404 return *(unsigned int *)get_reg_w32(reg, regs);
405
406#ifdef __amd64__
407 case 8:
408 return *(unsigned long *)get_reg_w32(reg, regs);
409#endif
410
411 default:
412 printk(KERN_ERR "mmiotrace: Error width# %d\n", reg);
413 }
414
415err:
416 return 0;
417}
418
419unsigned long get_ins_imm_val(unsigned long ins_addr)
420{
421 unsigned int opcode;
422 unsigned char mod_rm;
423 unsigned char mod;
424 unsigned char *p;
425 int i, shorted, enlarged, rexr;
426 unsigned long rv;
427
428 p = (unsigned char *)ins_addr;
429 p += skip_prefix(p, &shorted, &enlarged, &rexr);
430 p += get_opcode(p, &opcode);
431 for (i = 0; i < ARRAY_SIZE(imm_wop); i++)
432 if (imm_wop[i] == opcode) {
433 rv = IMM_WRITE;
434 goto do_work;
435 }
436
437 printk(KERN_ERR "mmiotrace: Not an immediate instruction, opcode "
438 "0x%02x\n", opcode);
439 goto err;
440
441do_work:
442 mod_rm = *p;
443 mod = mod_rm >> 6;
444 p++;
445 switch (mod) {
446 case 0:
447 /* if r/m is 5 we have a 32 disp (IA32 Manual 3, Table 2-2) */
448 /* AMD64: XXX Check for address size prefix? */
449 if ((mod_rm & 0x7) == 0x5)
450 p += 4;
451 break;
452
453 case 1:
454 p += 1;
455 break;
456
457 case 2:
458 p += 4;
459 break;
460
461 case 3:
462 default:
463 printk(KERN_ERR "mmiotrace: not a memory access instruction "
464 "at 0x%lx, rm_mod=0x%02x\n",
465 ins_addr, mod_rm);
466 }
467
468 switch (get_ins_reg_width(ins_addr)) {
469 case 1:
470 return *(unsigned char *)p;
471
472 case 2:
473 return *(unsigned short *)p;
474
475 case 4:
476 return *(unsigned int *)p;
477
478#ifdef __amd64__
479 case 8:
480 return *(unsigned long *)p;
481#endif
482
483 default:
484 printk(KERN_ERR "mmiotrace: Error: width.\n");
485 }
486
487err:
488 return 0;
489}
diff --git a/arch/x86/mm/pf_in.h b/arch/x86/mm/pf_in.h
new file mode 100644
index 000000000000..e05341a51a27
--- /dev/null
+++ b/arch/x86/mm/pf_in.h
@@ -0,0 +1,39 @@
1/*
2 * Fault Injection Test harness (FI)
3 * Copyright (C) Intel Crop.
4 *
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of the GNU General Public License
7 * as published by the Free Software Foundation; either version 2
8 * of the License, or (at your option) any later version.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program; if not, write to the Free Software
17 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
18 * USA.
19 *
20 */
21
22#ifndef __PF_H_
23#define __PF_H_
24
25enum reason_type {
26 NOT_ME, /* page fault is not in regions */
27 NOTHING, /* access others point in regions */
28 REG_READ, /* read from addr to reg */
29 REG_WRITE, /* write from reg to addr */
30 IMM_WRITE, /* write from imm to addr */
31 OTHERS /* Other instructions can not intercept */
32};
33
34enum reason_type get_ins_type(unsigned long ins_addr);
35unsigned int get_ins_mem_width(unsigned long ins_addr);
36unsigned long get_ins_reg_val(unsigned long ins_addr, struct pt_regs *regs);
37unsigned long get_ins_imm_val(unsigned long ins_addr);
38
39#endif /* __PF_H_ */
diff --git a/arch/x86/mm/testmmiotrace.c b/arch/x86/mm/testmmiotrace.c
new file mode 100644
index 000000000000..d877c5b423ef
--- /dev/null
+++ b/arch/x86/mm/testmmiotrace.c
@@ -0,0 +1,71 @@
1/*
2 * Written by Pekka Paalanen, 2008 <pq@iki.fi>
3 */
4#include <linux/module.h>
5#include <linux/io.h>
6
7#define MODULE_NAME "testmmiotrace"
8
9static unsigned long mmio_address;
10module_param(mmio_address, ulong, 0);
11MODULE_PARM_DESC(mmio_address, "Start address of the mapping of 16 kB.");
12
13static void do_write_test(void __iomem *p)
14{
15 unsigned int i;
16 for (i = 0; i < 256; i++)
17 iowrite8(i, p + i);
18 for (i = 1024; i < (5 * 1024); i += 2)
19 iowrite16(i * 12 + 7, p + i);
20 for (i = (5 * 1024); i < (16 * 1024); i += 4)
21 iowrite32(i * 212371 + 13, p + i);
22}
23
24static void do_read_test(void __iomem *p)
25{
26 unsigned int i;
27 for (i = 0; i < 256; i++)
28 ioread8(p + i);
29 for (i = 1024; i < (5 * 1024); i += 2)
30 ioread16(p + i);
31 for (i = (5 * 1024); i < (16 * 1024); i += 4)
32 ioread32(p + i);
33}
34
35static void do_test(void)
36{
37 void __iomem *p = ioremap_nocache(mmio_address, 0x4000);
38 if (!p) {
39 pr_err(MODULE_NAME ": could not ioremap, aborting.\n");
40 return;
41 }
42 do_write_test(p);
43 do_read_test(p);
44 iounmap(p);
45}
46
47static int __init init(void)
48{
49 if (mmio_address == 0) {
50 pr_err(MODULE_NAME ": you have to use the module argument "
51 "mmio_address.\n");
52 pr_err(MODULE_NAME ": DO NOT LOAD THIS MODULE UNLESS"
53 " YOU REALLY KNOW WHAT YOU ARE DOING!\n");
54 return -ENXIO;
55 }
56
57 pr_warning(MODULE_NAME ": WARNING: mapping 16 kB @ 0x%08lx "
58 "in PCI address space, and writing "
59 "rubbish in there.\n", mmio_address);
60 do_test();
61 return 0;
62}
63
64static void __exit cleanup(void)
65{
66 pr_debug(MODULE_NAME ": unloaded.\n");
67}
68
69module_init(init);
70module_exit(cleanup);
71MODULE_LICENSE("GPL");
diff --git a/arch/x86/pci/common.c b/arch/x86/pci/common.c
index 940185ecaeda..6e64aaf00d1d 100644
--- a/arch/x86/pci/common.c
+++ b/arch/x86/pci/common.c
@@ -328,18 +328,18 @@ static struct dmi_system_id __devinitdata pciprobe_dmi_table[] = {
328#endif 328#endif
329 { 329 {
330 .callback = set_bf_sort, 330 .callback = set_bf_sort,
331 .ident = "HP ProLiant DL360", 331 .ident = "HP ProLiant DL385 G2",
332 .matches = { 332 .matches = {
333 DMI_MATCH(DMI_SYS_VENDOR, "HP"), 333 DMI_MATCH(DMI_SYS_VENDOR, "HP"),
334 DMI_MATCH(DMI_PRODUCT_NAME, "ProLiant DL360"), 334 DMI_MATCH(DMI_PRODUCT_NAME, "ProLiant DL385 G2"),
335 }, 335 },
336 }, 336 },
337 { 337 {
338 .callback = set_bf_sort, 338 .callback = set_bf_sort,
339 .ident = "HP ProLiant DL380", 339 .ident = "HP ProLiant DL585 G2",
340 .matches = { 340 .matches = {
341 DMI_MATCH(DMI_SYS_VENDOR, "HP"), 341 DMI_MATCH(DMI_SYS_VENDOR, "HP"),
342 DMI_MATCH(DMI_PRODUCT_NAME, "ProLiant DL380"), 342 DMI_MATCH(DMI_PRODUCT_NAME, "ProLiant DL585 G2"),
343 }, 343 },
344 }, 344 },
345 {} 345 {}
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index df40bf74ea75..4e527e7893a8 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -185,7 +185,7 @@ static pteval_t pte_mfn_to_pfn(pteval_t val)
185 if (val & _PAGE_PRESENT) { 185 if (val & _PAGE_PRESENT) {
186 unsigned long mfn = (val & PTE_MASK) >> PAGE_SHIFT; 186 unsigned long mfn = (val & PTE_MASK) >> PAGE_SHIFT;
187 pteval_t flags = val & ~PTE_MASK; 187 pteval_t flags = val & ~PTE_MASK;
188 val = (mfn_to_pfn(mfn) << PAGE_SHIFT) | flags; 188 val = ((pteval_t)mfn_to_pfn(mfn) << PAGE_SHIFT) | flags;
189 } 189 }
190 190
191 return val; 191 return val;
@@ -196,7 +196,7 @@ static pteval_t pte_pfn_to_mfn(pteval_t val)
196 if (val & _PAGE_PRESENT) { 196 if (val & _PAGE_PRESENT) {
197 unsigned long pfn = (val & PTE_MASK) >> PAGE_SHIFT; 197 unsigned long pfn = (val & PTE_MASK) >> PAGE_SHIFT;
198 pteval_t flags = val & ~PTE_MASK; 198 pteval_t flags = val & ~PTE_MASK;
199 val = (pfn_to_mfn(pfn) << PAGE_SHIFT) | flags; 199 val = ((pteval_t)pfn_to_mfn(pfn) << PAGE_SHIFT) | flags;
200 } 200 }
201 201
202 return val; 202 return val;
generated by cgit v1.2.2 (git 2.25.0) at 2025-03-03 05:46:31 -0500