aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86_64/kernel
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86_64/kernel')
-rw-r--r--arch/x86_64/kernel/e820.c18
-rw-r--r--arch/x86_64/kernel/early-quirks.c17
-rw-r--r--arch/x86_64/kernel/early_printk.c2
-rw-r--r--arch/x86_64/kernel/entry.S10
-rw-r--r--arch/x86_64/kernel/genapic_flat.c2
-rw-r--r--arch/x86_64/kernel/i8259.c7
-rw-r--r--arch/x86_64/kernel/io_apic.c193
-rw-r--r--arch/x86_64/kernel/irq.c2
-rw-r--r--arch/x86_64/kernel/pci-calgary.c44
-rw-r--r--arch/x86_64/kernel/process.c7
-rw-r--r--arch/x86_64/kernel/smp.c3
-rw-r--r--arch/x86_64/kernel/smpboot.c7
-rw-r--r--arch/x86_64/kernel/time.c13
-rw-r--r--arch/x86_64/kernel/traps.c16
-rw-r--r--arch/x86_64/kernel/vmlinux.lds.S12
-rw-r--r--arch/x86_64/kernel/vsyscall.c47
16 files changed, 277 insertions, 123 deletions
diff --git a/arch/x86_64/kernel/e820.c b/arch/x86_64/kernel/e820.c
index b3f0908668e..6fe191c5808 100644
--- a/arch/x86_64/kernel/e820.c
+++ b/arch/x86_64/kernel/e820.c
@@ -54,13 +54,13 @@ static inline int bad_addr(unsigned long *addrp, unsigned long size)
54 54
55 /* various gunk below that needed for SMP startup */ 55 /* various gunk below that needed for SMP startup */
56 if (addr < 0x8000) { 56 if (addr < 0x8000) {
57 *addrp = 0x8000; 57 *addrp = PAGE_ALIGN(0x8000);
58 return 1; 58 return 1;
59 } 59 }
60 60
61 /* direct mapping tables of the kernel */ 61 /* direct mapping tables of the kernel */
62 if (last >= table_start<<PAGE_SHIFT && addr < table_end<<PAGE_SHIFT) { 62 if (last >= table_start<<PAGE_SHIFT && addr < table_end<<PAGE_SHIFT) {
63 *addrp = table_end << PAGE_SHIFT; 63 *addrp = PAGE_ALIGN(table_end << PAGE_SHIFT);
64 return 1; 64 return 1;
65 } 65 }
66 66
@@ -68,18 +68,18 @@ static inline int bad_addr(unsigned long *addrp, unsigned long size)
68#ifdef CONFIG_BLK_DEV_INITRD 68#ifdef CONFIG_BLK_DEV_INITRD
69 if (LOADER_TYPE && INITRD_START && last >= INITRD_START && 69 if (LOADER_TYPE && INITRD_START && last >= INITRD_START &&
70 addr < INITRD_START+INITRD_SIZE) { 70 addr < INITRD_START+INITRD_SIZE) {
71 *addrp = INITRD_START + INITRD_SIZE; 71 *addrp = PAGE_ALIGN(INITRD_START + INITRD_SIZE);
72 return 1; 72 return 1;
73 } 73 }
74#endif 74#endif
75 /* kernel code */ 75 /* kernel code */
76 if (last >= __pa_symbol(&_text) && last < __pa_symbol(&_end)) { 76 if (last >= __pa_symbol(&_text) && addr < __pa_symbol(&_end)) {
77 *addrp = __pa_symbol(&_end); 77 *addrp = PAGE_ALIGN(__pa_symbol(&_end));
78 return 1; 78 return 1;
79 } 79 }
80 80
81 if (last >= ebda_addr && addr < ebda_addr + ebda_size) { 81 if (last >= ebda_addr && addr < ebda_addr + ebda_size) {
82 *addrp = ebda_addr + ebda_size; 82 *addrp = PAGE_ALIGN(ebda_addr + ebda_size);
83 return 1; 83 return 1;
84 } 84 }
85 85
@@ -152,7 +152,7 @@ unsigned long __init find_e820_area(unsigned long start, unsigned long end, unsi
152 continue; 152 continue;
153 while (bad_addr(&addr, size) && addr+size <= ei->addr+ei->size) 153 while (bad_addr(&addr, size) && addr+size <= ei->addr+ei->size)
154 ; 154 ;
155 last = addr + size; 155 last = PAGE_ALIGN(addr) + size;
156 if (last > ei->addr + ei->size) 156 if (last > ei->addr + ei->size)
157 continue; 157 continue;
158 if (last > end) 158 if (last > end)
@@ -278,7 +278,7 @@ e820_register_active_regions(int nid, unsigned long start_pfn,
278 >> PAGE_SHIFT; 278 >> PAGE_SHIFT;
279 279
280 /* Skip map entries smaller than a page */ 280 /* Skip map entries smaller than a page */
281 if (ei_startpfn > ei_endpfn) 281 if (ei_startpfn >= ei_endpfn)
282 continue; 282 continue;
283 283
284 /* Check if end_pfn_map should be updated */ 284 /* Check if end_pfn_map should be updated */
@@ -594,7 +594,9 @@ static int __init parse_memmap_opt(char *p)
594 * size before original memory map is 594 * size before original memory map is
595 * reset. 595 * reset.
596 */ 596 */
597 e820_register_active_regions(0, 0, -1UL);
597 saved_max_pfn = e820_end_of_ram(); 598 saved_max_pfn = e820_end_of_ram();
599 remove_all_active_ranges();
598#endif 600#endif
599 end_pfn_map = 0; 601 end_pfn_map = 0;
600 e820.nr_map = 0; 602 e820.nr_map = 0;
diff --git a/arch/x86_64/kernel/early-quirks.c b/arch/x86_64/kernel/early-quirks.c
index 208e38a372c..68273bff58c 100644
--- a/arch/x86_64/kernel/early-quirks.c
+++ b/arch/x86_64/kernel/early-quirks.c
@@ -45,7 +45,13 @@ static void nvidia_bugs(void)
45 /* 45 /*
46 * All timer overrides on Nvidia are 46 * All timer overrides on Nvidia are
47 * wrong unless HPET is enabled. 47 * wrong unless HPET is enabled.
48 * Unfortunately that's not true on many Asus boards.
49 * We don't know yet how to detect this automatically, but
50 * at least allow a command line override.
48 */ 51 */
52 if (acpi_use_timer_override)
53 return;
54
49 nvidia_hpet_detected = 0; 55 nvidia_hpet_detected = 0;
50 acpi_table_parse(ACPI_HPET, nvidia_hpet_check); 56 acpi_table_parse(ACPI_HPET, nvidia_hpet_check);
51 if (nvidia_hpet_detected == 0) { 57 if (nvidia_hpet_detected == 0) {
@@ -53,6 +59,8 @@ static void nvidia_bugs(void)
53 printk(KERN_INFO "Nvidia board " 59 printk(KERN_INFO "Nvidia board "
54 "detected. Ignoring ACPI " 60 "detected. Ignoring ACPI "
55 "timer override.\n"); 61 "timer override.\n");
62 printk(KERN_INFO "If you got timer trouble "
63 "try acpi_use_timer_override\n");
56 } 64 }
57#endif 65#endif
58 /* RED-PEN skip them on mptables too? */ 66 /* RED-PEN skip them on mptables too? */
@@ -61,10 +69,11 @@ static void nvidia_bugs(void)
61 69
62static void ati_bugs(void) 70static void ati_bugs(void)
63{ 71{
64#if 1 /* for testing */ 72 if (timer_over_8254 == 1) {
65 printk("ATI board detected\n"); 73 timer_over_8254 = 0;
66#endif 74 printk(KERN_INFO
67 /* No bugs right now */ 75 "ATI board detected. Disabling timer routing over 8254.\n");
76 }
68} 77}
69 78
70struct chipset { 79struct chipset {
diff --git a/arch/x86_64/kernel/early_printk.c b/arch/x86_64/kernel/early_printk.c
index e22ecd54870..47b6d90349d 100644
--- a/arch/x86_64/kernel/early_printk.c
+++ b/arch/x86_64/kernel/early_printk.c
@@ -224,7 +224,7 @@ static int __init setup_early_printk(char *buf)
224 return 0; 224 return 0;
225 early_console_initialized = 1; 225 early_console_initialized = 1;
226 226
227 if (!strcmp(buf,"keep")) 227 if (strstr(buf, "keep"))
228 keep_early = 1; 228 keep_early = 1;
229 229
230 if (!strncmp(buf, "serial", 6)) { 230 if (!strncmp(buf, "serial", 6)) {
diff --git a/arch/x86_64/kernel/entry.S b/arch/x86_64/kernel/entry.S
index 38a7b2d528e..7d401b00d82 100644
--- a/arch/x86_64/kernel/entry.S
+++ b/arch/x86_64/kernel/entry.S
@@ -315,6 +315,8 @@ tracesys:
315 LOAD_ARGS ARGOFFSET /* reload args from stack in case ptrace changed it */ 315 LOAD_ARGS ARGOFFSET /* reload args from stack in case ptrace changed it */
316 RESTORE_REST 316 RESTORE_REST
317 cmpq $__NR_syscall_max,%rax 317 cmpq $__NR_syscall_max,%rax
318 movq $-ENOSYS,%rcx
319 cmova %rcx,%rax
318 ja 1f 320 ja 1f
319 movq %r10,%rcx /* fixup for C */ 321 movq %r10,%rcx /* fixup for C */
320 call *sys_call_table(,%rax,8) 322 call *sys_call_table(,%rax,8)
@@ -535,8 +537,6 @@ END(stub_rt_sigreturn)
5351: incl %gs:pda_irqcount 5371: incl %gs:pda_irqcount
536 cmoveq %gs:pda_irqstackptr,%rsp 538 cmoveq %gs:pda_irqstackptr,%rsp
537 push %rbp # backlink for old unwinder 539 push %rbp # backlink for old unwinder
538 CFI_ADJUST_CFA_OFFSET 8
539 CFI_REL_OFFSET rbp,0
540 /* 540 /*
541 * We entered an interrupt context - irqs are off: 541 * We entered an interrupt context - irqs are off:
542 */ 542 */
@@ -980,11 +980,6 @@ ENTRY(kernel_thread)
980 call do_fork 980 call do_fork
981 movq %rax,RAX(%rsp) 981 movq %rax,RAX(%rsp)
982 xorl %edi,%edi 982 xorl %edi,%edi
983 test %rax,%rax
984 jnz 1f
985 /* terminate stack in child */
986 movq %rdi,RIP(%rsp)
9871:
988 983
989 /* 984 /*
990 * It isn't worth to check for reschedule here, 985 * It isn't worth to check for reschedule here,
@@ -1176,7 +1171,6 @@ ENTRY(call_softirq)
1176 incl %gs:pda_irqcount 1171 incl %gs:pda_irqcount
1177 cmove %gs:pda_irqstackptr,%rsp 1172 cmove %gs:pda_irqstackptr,%rsp
1178 push %rbp # backlink for old unwinder 1173 push %rbp # backlink for old unwinder
1179 CFI_ADJUST_CFA_OFFSET 8
1180 call __do_softirq 1174 call __do_softirq
1181 leaveq 1175 leaveq
1182 CFI_DEF_CFA_REGISTER rsp 1176 CFI_DEF_CFA_REGISTER rsp
diff --git a/arch/x86_64/kernel/genapic_flat.c b/arch/x86_64/kernel/genapic_flat.c
index 0dfc223c183..7c01db8fa9d 100644
--- a/arch/x86_64/kernel/genapic_flat.c
+++ b/arch/x86_64/kernel/genapic_flat.c
@@ -153,7 +153,7 @@ struct genapic apic_flat = {
153 153
154static cpumask_t physflat_target_cpus(void) 154static cpumask_t physflat_target_cpus(void)
155{ 155{
156 return cpumask_of_cpu(0); 156 return cpu_online_map;
157} 157}
158 158
159static cpumask_t physflat_vector_allocation_domain(int cpu) 159static cpumask_t physflat_vector_allocation_domain(int cpu)
diff --git a/arch/x86_64/kernel/i8259.c b/arch/x86_64/kernel/i8259.c
index 0612a33bb89..c4ef801b765 100644
--- a/arch/x86_64/kernel/i8259.c
+++ b/arch/x86_64/kernel/i8259.c
@@ -178,7 +178,8 @@ void make_8259A_irq(unsigned int irq)
178{ 178{
179 disable_irq_nosync(irq); 179 disable_irq_nosync(irq);
180 io_apic_irqs &= ~(1<<irq); 180 io_apic_irqs &= ~(1<<irq);
181 set_irq_chip_and_handler(irq, &i8259A_chip, handle_level_irq); 181 set_irq_chip_and_handler_name(irq, &i8259A_chip, handle_level_irq,
182 "XT");
182 enable_irq(irq); 183 enable_irq(irq);
183} 184}
184 185
@@ -431,8 +432,8 @@ void __init init_ISA_irqs (void)
431 /* 432 /*
432 * 16 old-style INTA-cycle interrupts: 433 * 16 old-style INTA-cycle interrupts:
433 */ 434 */
434 set_irq_chip_and_handler(i, &i8259A_chip, 435 set_irq_chip_and_handler_name(i, &i8259A_chip,
435 handle_level_irq); 436 handle_level_irq, "XT");
436 } else { 437 } else {
437 /* 438 /*
438 * 'high' PCI IRQs filled in on demand 439 * 'high' PCI IRQs filled in on demand
diff --git a/arch/x86_64/kernel/io_apic.c b/arch/x86_64/kernel/io_apic.c
index 44b55f83387..c80081a6ba4 100644
--- a/arch/x86_64/kernel/io_apic.c
+++ b/arch/x86_64/kernel/io_apic.c
@@ -57,13 +57,13 @@ static int no_timer_check;
57 57
58static int disable_timer_pin_1 __initdata; 58static int disable_timer_pin_1 __initdata;
59 59
60int timer_over_8254 __initdata = 0; 60int timer_over_8254 __initdata = 1;
61 61
62/* Where if anywhere is the i8259 connect in external int mode */ 62/* Where if anywhere is the i8259 connect in external int mode */
63static struct { int pin, apic; } ioapic_i8259 = { -1, -1 }; 63static struct { int pin, apic; } ioapic_i8259 = { -1, -1 };
64 64
65static DEFINE_SPINLOCK(ioapic_lock); 65static DEFINE_SPINLOCK(ioapic_lock);
66static DEFINE_SPINLOCK(vector_lock); 66DEFINE_SPINLOCK(vector_lock);
67 67
68/* 68/*
69 * # of IRQ routing registers 69 * # of IRQ routing registers
@@ -88,6 +88,52 @@ static struct irq_pin_list {
88 short apic, pin, next; 88 short apic, pin, next;
89} irq_2_pin[PIN_MAP_SIZE]; 89} irq_2_pin[PIN_MAP_SIZE];
90 90
91struct io_apic {
92 unsigned int index;
93 unsigned int unused[3];
94 unsigned int data;
95};
96
97static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
98{
99 return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx)
100 + (mp_ioapics[idx].mpc_apicaddr & ~PAGE_MASK);
101}
102
103static inline unsigned int io_apic_read(unsigned int apic, unsigned int reg)
104{
105 struct io_apic __iomem *io_apic = io_apic_base(apic);
106 writel(reg, &io_apic->index);
107 return readl(&io_apic->data);
108}
109
110static inline void io_apic_write(unsigned int apic, unsigned int reg, unsigned int value)
111{
112 struct io_apic __iomem *io_apic = io_apic_base(apic);
113 writel(reg, &io_apic->index);
114 writel(value, &io_apic->data);
115}
116
117/*
118 * Re-write a value: to be used for read-modify-write
119 * cycles where the read already set up the index register.
120 */
121static inline void io_apic_modify(unsigned int apic, unsigned int value)
122{
123 struct io_apic __iomem *io_apic = io_apic_base(apic);
124 writel(value, &io_apic->data);
125}
126
127/*
128 * Synchronize the IO-APIC and the CPU by doing
129 * a dummy read from the IO-APIC
130 */
131static inline void io_apic_sync(unsigned int apic)
132{
133 struct io_apic __iomem *io_apic = io_apic_base(apic);
134 readl(&io_apic->data);
135}
136
91#define __DO_ACTION(R, ACTION, FINAL) \ 137#define __DO_ACTION(R, ACTION, FINAL) \
92 \ 138 \
93{ \ 139{ \
@@ -126,12 +172,34 @@ static struct IO_APIC_route_entry ioapic_read_entry(int apic, int pin)
126 return eu.entry; 172 return eu.entry;
127} 173}
128 174
175/*
176 * When we write a new IO APIC routing entry, we need to write the high
177 * word first! If the mask bit in the low word is clear, we will enable
178 * the interrupt, and we need to make sure the entry is fully populated
179 * before that happens.
180 */
129static void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e) 181static void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
130{ 182{
131 unsigned long flags; 183 unsigned long flags;
132 union entry_union eu; 184 union entry_union eu;
133 eu.entry = e; 185 eu.entry = e;
134 spin_lock_irqsave(&ioapic_lock, flags); 186 spin_lock_irqsave(&ioapic_lock, flags);
187 io_apic_write(apic, 0x11 + 2*pin, eu.w2);
188 io_apic_write(apic, 0x10 + 2*pin, eu.w1);
189 spin_unlock_irqrestore(&ioapic_lock, flags);
190}
191
192/*
193 * When we mask an IO APIC routing entry, we need to write the low
194 * word first, in order to set the mask bit before we change the
195 * high bits!
196 */
197static void ioapic_mask_entry(int apic, int pin)
198{
199 unsigned long flags;
200 union entry_union eu = { .entry.mask = 1 };
201
202 spin_lock_irqsave(&ioapic_lock, flags);
135 io_apic_write(apic, 0x10 + 2*pin, eu.w1); 203 io_apic_write(apic, 0x10 + 2*pin, eu.w1);
136 io_apic_write(apic, 0x11 + 2*pin, eu.w2); 204 io_apic_write(apic, 0x11 + 2*pin, eu.w2);
137 spin_unlock_irqrestore(&ioapic_lock, flags); 205 spin_unlock_irqrestore(&ioapic_lock, flags);
@@ -256,9 +324,7 @@ static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
256 /* 324 /*
257 * Disable it in the IO-APIC irq-routing table: 325 * Disable it in the IO-APIC irq-routing table:
258 */ 326 */
259 memset(&entry, 0, sizeof(entry)); 327 ioapic_mask_entry(apic, pin);
260 entry.mask = 1;
261 ioapic_write_entry(apic, pin, entry);
262} 328}
263 329
264static void clear_IO_APIC (void) 330static void clear_IO_APIC (void)
@@ -612,15 +678,15 @@ static int __assign_irq_vector(int irq, cpumask_t mask, cpumask_t *result)
612 * Also, we've got to be careful not to trash gate 678 * Also, we've got to be careful not to trash gate
613 * 0x80, because int 0x80 is hm, kind of importantish. ;) 679 * 0x80, because int 0x80 is hm, kind of importantish. ;)
614 */ 680 */
615 static struct { 681 static int current_vector = FIRST_DEVICE_VECTOR, current_offset = 0;
616 int vector;
617 int offset;
618 } pos[NR_CPUS] = { [ 0 ... NR_CPUS - 1] = {FIRST_DEVICE_VECTOR, 0} };
619 int old_vector = -1; 682 int old_vector = -1;
620 int cpu; 683 int cpu;
621 684
622 BUG_ON((unsigned)irq >= NR_IRQ_VECTORS); 685 BUG_ON((unsigned)irq >= NR_IRQ_VECTORS);
623 686
687 /* Only try and allocate irqs on cpus that are present */
688 cpus_and(mask, mask, cpu_online_map);
689
624 if (irq_vector[irq] > 0) 690 if (irq_vector[irq] > 0)
625 old_vector = irq_vector[irq]; 691 old_vector = irq_vector[irq];
626 if (old_vector > 0) { 692 if (old_vector > 0) {
@@ -630,15 +696,15 @@ static int __assign_irq_vector(int irq, cpumask_t mask, cpumask_t *result)
630 } 696 }
631 697
632 for_each_cpu_mask(cpu, mask) { 698 for_each_cpu_mask(cpu, mask) {
633 cpumask_t domain; 699 cpumask_t domain, new_mask;
634 int first, new_cpu; 700 int new_cpu;
635 int vector, offset; 701 int vector, offset;
636 702
637 domain = vector_allocation_domain(cpu); 703 domain = vector_allocation_domain(cpu);
638 first = first_cpu(domain); 704 cpus_and(new_mask, domain, cpu_online_map);
639 705
640 vector = pos[first].vector; 706 vector = current_vector;
641 offset = pos[first].offset; 707 offset = current_offset;
642next: 708next:
643 vector += 8; 709 vector += 8;
644 if (vector >= FIRST_SYSTEM_VECTOR) { 710 if (vector >= FIRST_SYSTEM_VECTOR) {
@@ -646,24 +712,24 @@ next:
646 offset = (offset + 1) % 8; 712 offset = (offset + 1) % 8;
647 vector = FIRST_DEVICE_VECTOR + offset; 713 vector = FIRST_DEVICE_VECTOR + offset;
648 } 714 }
649 if (unlikely(pos[first].vector == vector)) 715 if (unlikely(current_vector == vector))
650 continue; 716 continue;
651 if (vector == IA32_SYSCALL_VECTOR) 717 if (vector == IA32_SYSCALL_VECTOR)
652 goto next; 718 goto next;
653 for_each_cpu_mask(new_cpu, domain) 719 for_each_cpu_mask(new_cpu, new_mask)
654 if (per_cpu(vector_irq, cpu)[vector] != -1) 720 if (per_cpu(vector_irq, new_cpu)[vector] != -1)
655 goto next; 721 goto next;
656 /* Found one! */ 722 /* Found one! */
657 for_each_cpu_mask(new_cpu, domain) { 723 current_vector = vector;
658 pos[cpu].vector = vector; 724 current_offset = offset;
659 pos[cpu].offset = offset;
660 }
661 if (old_vector >= 0) { 725 if (old_vector >= 0) {
726 cpumask_t old_mask;
662 int old_cpu; 727 int old_cpu;
663 for_each_cpu_mask(old_cpu, irq_domain[irq]) 728 cpus_and(old_mask, irq_domain[irq], cpu_online_map);
729 for_each_cpu_mask(old_cpu, old_mask)
664 per_cpu(vector_irq, old_cpu)[old_vector] = -1; 730 per_cpu(vector_irq, old_cpu)[old_vector] = -1;
665 } 731 }
666 for_each_cpu_mask(new_cpu, domain) 732 for_each_cpu_mask(new_cpu, new_mask)
667 per_cpu(vector_irq, new_cpu)[vector] = irq; 733 per_cpu(vector_irq, new_cpu)[vector] = irq;
668 irq_vector[irq] = vector; 734 irq_vector[irq] = vector;
669 irq_domain[irq] = domain; 735 irq_domain[irq] = domain;
@@ -684,6 +750,30 @@ static int assign_irq_vector(int irq, cpumask_t mask, cpumask_t *result)
684 return vector; 750 return vector;
685} 751}
686 752
753void __setup_vector_irq(int cpu)
754{
755 /* Initialize vector_irq on a new cpu */
756 /* This function must be called with vector_lock held */
757 int irq, vector;
758
759 /* Mark the inuse vectors */
760 for (irq = 0; irq < NR_IRQ_VECTORS; ++irq) {
761 if (!cpu_isset(cpu, irq_domain[irq]))
762 continue;
763 vector = irq_vector[irq];
764 per_cpu(vector_irq, cpu)[vector] = irq;
765 }
766 /* Mark the free vectors */
767 for (vector = 0; vector < NR_VECTORS; ++vector) {
768 irq = per_cpu(vector_irq, cpu)[vector];
769 if (irq < 0)
770 continue;
771 if (!cpu_isset(cpu, irq_domain[irq]))
772 per_cpu(vector_irq, cpu)[vector] = -1;
773 }
774}
775
776
687extern void (*interrupt[NR_IRQS])(void); 777extern void (*interrupt[NR_IRQS])(void);
688 778
689static struct irq_chip ioapic_chip; 779static struct irq_chip ioapic_chip;
@@ -696,11 +786,13 @@ static void ioapic_register_intr(int irq, int vector, unsigned long trigger)
696{ 786{
697 if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) || 787 if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
698 trigger == IOAPIC_LEVEL) 788 trigger == IOAPIC_LEVEL)
699 set_irq_chip_and_handler(irq, &ioapic_chip, 789 set_irq_chip_and_handler_name(irq, &ioapic_chip,
700 handle_fasteoi_irq); 790 handle_fasteoi_irq, "fasteoi");
701 else 791 else {
702 set_irq_chip_and_handler(irq, &ioapic_chip, 792 irq_desc[irq].status |= IRQ_DELAYED_DISABLE;
703 handle_edge_irq); 793 set_irq_chip_and_handler_name(irq, &ioapic_chip,
794 handle_edge_irq, "edge");
795 }
704} 796}
705 797
706static void __init setup_IO_APIC_irqs(void) 798static void __init setup_IO_APIC_irqs(void)
@@ -806,7 +898,7 @@ static void __init setup_ExtINT_IRQ0_pin(unsigned int apic, unsigned int pin, in
806 * The timer IRQ doesn't have to know that behind the 898 * The timer IRQ doesn't have to know that behind the
807 * scene we have a 8259A-master in AEOI mode ... 899 * scene we have a 8259A-master in AEOI mode ...
808 */ 900 */
809 set_irq_chip_and_handler(0, &ioapic_chip, handle_edge_irq); 901 set_irq_chip_and_handler_name(0, &ioapic_chip, handle_edge_irq, "edge");
810 902
811 /* 903 /*
812 * Add it to the IO-APIC irq-routing table: 904 * Add it to the IO-APIC irq-routing table:
@@ -1255,12 +1347,15 @@ static int ioapic_retrigger_irq(unsigned int irq)
1255{ 1347{
1256 cpumask_t mask; 1348 cpumask_t mask;
1257 unsigned vector; 1349 unsigned vector;
1350 unsigned long flags;
1258 1351
1352 spin_lock_irqsave(&vector_lock, flags);
1259 vector = irq_vector[irq]; 1353 vector = irq_vector[irq];
1260 cpus_clear(mask); 1354 cpus_clear(mask);
1261 cpu_set(vector >> 8, mask); 1355 cpu_set(first_cpu(irq_domain[irq]), mask);
1262 1356
1263 send_IPI_mask(mask, vector & 0xff); 1357 send_IPI_mask(mask, vector);
1358 spin_unlock_irqrestore(&vector_lock, flags);
1264 1359
1265 return 1; 1360 return 1;
1266} 1361}
@@ -1839,7 +1934,7 @@ int arch_setup_msi_irq(unsigned int irq, struct pci_dev *dev)
1839 1934
1840 write_msi_msg(irq, &msg); 1935 write_msi_msg(irq, &msg);
1841 1936
1842 set_irq_chip_and_handler(irq, &msi_chip, handle_edge_irq); 1937 set_irq_chip_and_handler_name(irq, &msi_chip, handle_edge_irq, "edge");
1843 1938
1844 return 0; 1939 return 0;
1845} 1940}
@@ -1860,18 +1955,16 @@ void arch_teardown_msi_irq(unsigned int irq)
1860 1955
1861static void target_ht_irq(unsigned int irq, unsigned int dest, u8 vector) 1956static void target_ht_irq(unsigned int irq, unsigned int dest, u8 vector)
1862{ 1957{
1863 u32 low, high; 1958 struct ht_irq_msg msg;
1864 low = read_ht_irq_low(irq); 1959 fetch_ht_irq_msg(irq, &msg);
1865 high = read_ht_irq_high(irq);
1866 1960
1867 low &= ~(HT_IRQ_LOW_VECTOR_MASK | HT_IRQ_LOW_DEST_ID_MASK); 1961 msg.address_lo &= ~(HT_IRQ_LOW_VECTOR_MASK | HT_IRQ_LOW_DEST_ID_MASK);
1868 high &= ~(HT_IRQ_HIGH_DEST_ID_MASK); 1962 msg.address_hi &= ~(HT_IRQ_HIGH_DEST_ID_MASK);
1869 1963
1870 low |= HT_IRQ_LOW_VECTOR(vector) | HT_IRQ_LOW_DEST_ID(dest); 1964 msg.address_lo |= HT_IRQ_LOW_VECTOR(vector) | HT_IRQ_LOW_DEST_ID(dest);
1871 high |= HT_IRQ_HIGH_DEST_ID(dest); 1965 msg.address_hi |= HT_IRQ_HIGH_DEST_ID(dest);
1872 1966
1873 write_ht_irq_low(irq, low); 1967 write_ht_irq_msg(irq, &msg);
1874 write_ht_irq_high(irq, high);
1875} 1968}
1876 1969
1877static void set_ht_irq_affinity(unsigned int irq, cpumask_t mask) 1970static void set_ht_irq_affinity(unsigned int irq, cpumask_t mask)
@@ -1892,7 +1985,7 @@ static void set_ht_irq_affinity(unsigned int irq, cpumask_t mask)
1892 1985
1893 dest = cpu_mask_to_apicid(tmp); 1986 dest = cpu_mask_to_apicid(tmp);
1894 1987
1895 target_ht_irq(irq, dest, vector & 0xff); 1988 target_ht_irq(irq, dest, vector);
1896 set_native_irq_info(irq, mask); 1989 set_native_irq_info(irq, mask);
1897} 1990}
1898#endif 1991#endif
@@ -1915,14 +2008,15 @@ int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
1915 2008
1916 vector = assign_irq_vector(irq, TARGET_CPUS, &tmp); 2009 vector = assign_irq_vector(irq, TARGET_CPUS, &tmp);
1917 if (vector >= 0) { 2010 if (vector >= 0) {
1918 u32 low, high; 2011 struct ht_irq_msg msg;
1919 unsigned dest; 2012 unsigned dest;
1920 2013
1921 dest = cpu_mask_to_apicid(tmp); 2014 dest = cpu_mask_to_apicid(tmp);
1922 2015
1923 high = HT_IRQ_HIGH_DEST_ID(dest); 2016 msg.address_hi = HT_IRQ_HIGH_DEST_ID(dest);
1924 2017
1925 low = HT_IRQ_LOW_BASE | 2018 msg.address_lo =
2019 HT_IRQ_LOW_BASE |
1926 HT_IRQ_LOW_DEST_ID(dest) | 2020 HT_IRQ_LOW_DEST_ID(dest) |
1927 HT_IRQ_LOW_VECTOR(vector) | 2021 HT_IRQ_LOW_VECTOR(vector) |
1928 ((INT_DEST_MODE == 0) ? 2022 ((INT_DEST_MODE == 0) ?
@@ -1931,12 +2025,13 @@ int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
1931 HT_IRQ_LOW_RQEOI_EDGE | 2025 HT_IRQ_LOW_RQEOI_EDGE |
1932 ((INT_DELIVERY_MODE != dest_LowestPrio) ? 2026 ((INT_DELIVERY_MODE != dest_LowestPrio) ?
1933 HT_IRQ_LOW_MT_FIXED : 2027 HT_IRQ_LOW_MT_FIXED :
1934 HT_IRQ_LOW_MT_ARBITRATED); 2028 HT_IRQ_LOW_MT_ARBITRATED) |
2029 HT_IRQ_LOW_IRQ_MASKED;
1935 2030
1936 write_ht_irq_low(irq, low); 2031 write_ht_irq_msg(irq, &msg);
1937 write_ht_irq_high(irq, high);
1938 2032
1939 set_irq_chip_and_handler(irq, &ht_irq_chip, handle_edge_irq); 2033 set_irq_chip_and_handler_name(irq, &ht_irq_chip,
2034 handle_edge_irq, "edge");
1940 } 2035 }
1941 return vector; 2036 return vector;
1942} 2037}
diff --git a/arch/x86_64/kernel/irq.c b/arch/x86_64/kernel/irq.c
index dff68eb2b78..e46c55856d4 100644
--- a/arch/x86_64/kernel/irq.c
+++ b/arch/x86_64/kernel/irq.c
@@ -75,7 +75,7 @@ int show_interrupts(struct seq_file *p, void *v)
75 seq_printf(p, "%10u ", kstat_cpu(j).irqs[i]); 75 seq_printf(p, "%10u ", kstat_cpu(j).irqs[i]);
76#endif 76#endif
77 seq_printf(p, " %8s", irq_desc[i].chip->name); 77 seq_printf(p, " %8s", irq_desc[i].chip->name);
78 seq_printf(p, "-%s", handle_irq_name(irq_desc[i].handle_irq)); 78 seq_printf(p, "-%-8s", irq_desc[i].name);
79 79
80 seq_printf(p, " %s", action->name); 80 seq_printf(p, " %s", action->name);
81 for (action=action->next; action; action = action->next) 81 for (action=action->next; action; action = action->next)
diff --git a/arch/x86_64/kernel/pci-calgary.c b/arch/x86_64/kernel/pci-calgary.c
index b3296cc2f2f..37a770859e7 100644
--- a/arch/x86_64/kernel/pci-calgary.c
+++ b/arch/x86_64/kernel/pci-calgary.c
@@ -52,7 +52,8 @@
52#define ONE_BASED_CHASSIS_NUM 1 52#define ONE_BASED_CHASSIS_NUM 1
53 53
54/* register offsets inside the host bridge space */ 54/* register offsets inside the host bridge space */
55#define PHB_CSR_OFFSET 0x0110 55#define CALGARY_CONFIG_REG 0x0108
56#define PHB_CSR_OFFSET 0x0110 /* Channel Status */
56#define PHB_PLSSR_OFFSET 0x0120 57#define PHB_PLSSR_OFFSET 0x0120
57#define PHB_CONFIG_RW_OFFSET 0x0160 58#define PHB_CONFIG_RW_OFFSET 0x0160
58#define PHB_IOBASE_BAR_LOW 0x0170 59#define PHB_IOBASE_BAR_LOW 0x0170
@@ -83,6 +84,8 @@
83#define TAR_VALID 0x0000000000000008UL 84#define TAR_VALID 0x0000000000000008UL
84/* CSR (Channel/DMA Status Register) */ 85/* CSR (Channel/DMA Status Register) */
85#define CSR_AGENT_MASK 0xffe0ffff 86#define CSR_AGENT_MASK 0xffe0ffff
87/* CCR (Calgary Configuration Register) */
88#define CCR_2SEC_TIMEOUT 0x000000000000000EUL
86 89
87#define MAX_NUM_OF_PHBS 8 /* how many PHBs in total? */ 90#define MAX_NUM_OF_PHBS 8 /* how many PHBs in total? */
88#define MAX_NUM_CHASSIS 8 /* max number of chassis */ 91#define MAX_NUM_CHASSIS 8 /* max number of chassis */
@@ -732,6 +735,38 @@ static void calgary_watchdog(unsigned long data)
732 } 735 }
733} 736}
734 737
738static void __init calgary_increase_split_completion_timeout(void __iomem *bbar,
739 unsigned char busnum)
740{
741 u64 val64;
742 void __iomem *target;
743 unsigned long phb_shift = -1;
744 u64 mask;
745
746 switch (busno_to_phbid(busnum)) {
747 case 0: phb_shift = (63 - 19);
748 break;
749 case 1: phb_shift = (63 - 23);
750 break;
751 case 2: phb_shift = (63 - 27);
752 break;
753 case 3: phb_shift = (63 - 35);
754 break;
755 default:
756 BUG_ON(busno_to_phbid(busnum));
757 }
758
759 target = calgary_reg(bbar, CALGARY_CONFIG_REG);
760 val64 = be64_to_cpu(readq(target));
761
762 /* zero out this PHB's timer bits */
763 mask = ~(0xFUL << phb_shift);
764 val64 &= mask;
765 val64 |= (CCR_2SEC_TIMEOUT << phb_shift);
766 writeq(cpu_to_be64(val64), target);
767 readq(target); /* flush */
768}
769
735static void __init calgary_enable_translation(struct pci_dev *dev) 770static void __init calgary_enable_translation(struct pci_dev *dev)
736{ 771{
737 u32 val32; 772 u32 val32;
@@ -756,6 +791,13 @@ static void __init calgary_enable_translation(struct pci_dev *dev)
756 writel(cpu_to_be32(val32), target); 791 writel(cpu_to_be32(val32), target);
757 readl(target); /* flush */ 792 readl(target); /* flush */
758 793
794 /*
795 * Give split completion a longer timeout on bus 1 for aic94xx
796 * http://bugzilla.kernel.org/show_bug.cgi?id=7180
797 */
798 if (busnum == 1)
799 calgary_increase_split_completion_timeout(bbar, busnum);
800
759 init_timer(&tbl->watchdog_timer); 801 init_timer(&tbl->watchdog_timer);
760 tbl->watchdog_timer.function = &calgary_watchdog; 802 tbl->watchdog_timer.function = &calgary_watchdog;
761 tbl->watchdog_timer.data = (unsigned long)dev; 803 tbl->watchdog_timer.data = (unsigned long)dev;
diff --git a/arch/x86_64/kernel/process.c b/arch/x86_64/kernel/process.c
index 49f7fac6229..7451a4c43c1 100644
--- a/arch/x86_64/kernel/process.c
+++ b/arch/x86_64/kernel/process.c
@@ -88,9 +88,8 @@ void enter_idle(void)
88 88
89static void __exit_idle(void) 89static void __exit_idle(void)
90{ 90{
91 if (read_pda(isidle) == 0) 91 if (test_and_clear_bit_pda(0, isidle) == 0)
92 return; 92 return;
93 write_pda(isidle, 0);
94 atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL); 93 atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL);
95} 94}
96 95
@@ -145,7 +144,7 @@ static void poll_idle (void)
145void cpu_idle_wait(void) 144void cpu_idle_wait(void)
146{ 145{
147 unsigned int cpu, this_cpu = get_cpu(); 146 unsigned int cpu, this_cpu = get_cpu();
148 cpumask_t map; 147 cpumask_t map, tmp = current->cpus_allowed;
149 148
150 set_cpus_allowed(current, cpumask_of_cpu(this_cpu)); 149 set_cpus_allowed(current, cpumask_of_cpu(this_cpu));
151 put_cpu(); 150 put_cpu();
@@ -168,6 +167,8 @@ void cpu_idle_wait(void)
168 } 167 }
169 cpus_and(map, map, cpu_online_map); 168 cpus_and(map, map, cpu_online_map);
170 } while (!cpus_empty(map)); 169 } while (!cpus_empty(map));
170
171 set_cpus_allowed(current, tmp);
171} 172}
172EXPORT_SYMBOL_GPL(cpu_idle_wait); 173EXPORT_SYMBOL_GPL(cpu_idle_wait);
173 174
diff --git a/arch/x86_64/kernel/smp.c b/arch/x86_64/kernel/smp.c
index 4f67697f503..9f74c883568 100644
--- a/arch/x86_64/kernel/smp.c
+++ b/arch/x86_64/kernel/smp.c
@@ -376,9 +376,8 @@ int smp_call_function_single (int cpu, void (*func) (void *info), void *info,
376 /* prevent preemption and reschedule on another processor */ 376 /* prevent preemption and reschedule on another processor */
377 int me = get_cpu(); 377 int me = get_cpu();
378 if (cpu == me) { 378 if (cpu == me) {
379 WARN_ON(1);
380 put_cpu(); 379 put_cpu();
381 return -EBUSY; 380 return 0;
382 } 381 }
383 spin_lock_bh(&call_lock); 382 spin_lock_bh(&call_lock);
384 __smp_call_function_single(cpu, func, info, nonatomic, wait); 383 __smp_call_function_single(cpu, func, info, nonatomic, wait);
diff --git a/arch/x86_64/kernel/smpboot.c b/arch/x86_64/kernel/smpboot.c
index 7b7a6870288..62c2e747af5 100644
--- a/arch/x86_64/kernel/smpboot.c
+++ b/arch/x86_64/kernel/smpboot.c
@@ -581,12 +581,16 @@ void __cpuinit start_secondary(void)
581 * smp_call_function(). 581 * smp_call_function().
582 */ 582 */
583 lock_ipi_call_lock(); 583 lock_ipi_call_lock();
584 spin_lock(&vector_lock);
584 585
586 /* Setup the per cpu irq handling data structures */
587 __setup_vector_irq(smp_processor_id());
585 /* 588 /*
586 * Allow the master to continue. 589 * Allow the master to continue.
587 */ 590 */
588 cpu_set(smp_processor_id(), cpu_online_map); 591 cpu_set(smp_processor_id(), cpu_online_map);
589 per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE; 592 per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE;
593 spin_unlock(&vector_lock);
590 unlock_ipi_call_lock(); 594 unlock_ipi_call_lock();
591 595
592 cpu_idle(); 596 cpu_idle();
@@ -799,7 +803,6 @@ static int __cpuinit do_boot_cpu(int cpu, int apicid)
799 cpu, node); 803 cpu, node);
800 } 804 }
801 805
802
803 alternatives_smp_switch(1); 806 alternatives_smp_switch(1);
804 807
805 c_idle.idle = get_idle_for_cpu(cpu); 808 c_idle.idle = get_idle_for_cpu(cpu);
@@ -1246,8 +1249,10 @@ int __cpu_disable(void)
1246 local_irq_disable(); 1249 local_irq_disable();
1247 remove_siblinginfo(cpu); 1250 remove_siblinginfo(cpu);
1248 1251
1252 spin_lock(&vector_lock);
1249 /* It's now safe to remove this processor from the online map */ 1253 /* It's now safe to remove this processor from the online map */
1250 cpu_clear(cpu, cpu_online_map); 1254 cpu_clear(cpu, cpu_online_map);
1255 spin_unlock(&vector_lock);
1251 remove_cpu_from_maps(); 1256 remove_cpu_from_maps();
1252 fixup_irqs(cpu_online_map); 1257 fixup_irqs(cpu_online_map);
1253 return 0; 1258 return 0;
diff --git a/arch/x86_64/kernel/time.c b/arch/x86_64/kernel/time.c
index 1ba5a442ac3..e3ef544d2cf 100644
--- a/arch/x86_64/kernel/time.c
+++ b/arch/x86_64/kernel/time.c
@@ -876,15 +876,6 @@ static struct irqaction irq0 = {
876 timer_interrupt, IRQF_DISABLED, CPU_MASK_NONE, "timer", NULL, NULL 876 timer_interrupt, IRQF_DISABLED, CPU_MASK_NONE, "timer", NULL, NULL
877}; 877};
878 878
879static int __cpuinit
880time_cpu_notifier(struct notifier_block *nb, unsigned long action, void *hcpu)
881{
882 unsigned cpu = (unsigned long) hcpu;
883 if (action == CPU_ONLINE)
884 vsyscall_set_cpu(cpu);
885 return NOTIFY_DONE;
886}
887
888void __init time_init(void) 879void __init time_init(void)
889{ 880{
890 if (nohpet) 881 if (nohpet)
@@ -925,8 +916,6 @@ void __init time_init(void)
925 vxtime.last_tsc = get_cycles_sync(); 916 vxtime.last_tsc = get_cycles_sync();
926 set_cyc2ns_scale(cpu_khz); 917 set_cyc2ns_scale(cpu_khz);
927 setup_irq(0, &irq0); 918 setup_irq(0, &irq0);
928 hotcpu_notifier(time_cpu_notifier, 0);
929 time_cpu_notifier(NULL, CPU_ONLINE, (void *)(long)smp_processor_id());
930 919
931#ifndef CONFIG_SMP 920#ifndef CONFIG_SMP
932 time_init_gtod(); 921 time_init_gtod();
@@ -948,7 +937,7 @@ __cpuinit int unsynchronized_tsc(void)
948 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) { 937 if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL) {
949#ifdef CONFIG_ACPI 938#ifdef CONFIG_ACPI
950 /* But TSC doesn't tick in C3 so don't use it there */ 939 /* But TSC doesn't tick in C3 so don't use it there */
951 if (acpi_fadt.length > 0 && acpi_fadt.plvl3_lat < 100) 940 if (acpi_fadt.length > 0 && acpi_fadt.plvl3_lat < 1000)
952 return 1; 941 return 1;
953#endif 942#endif
954 return 0; 943 return 0;
diff --git a/arch/x86_64/kernel/traps.c b/arch/x86_64/kernel/traps.c
index 7819022a8db..0d65b22f229 100644
--- a/arch/x86_64/kernel/traps.c
+++ b/arch/x86_64/kernel/traps.c
@@ -242,12 +242,19 @@ static int dump_trace_unwind(struct unwind_frame_info *info, void *context)
242 * severe exception (double fault, nmi, stack fault, debug, mce) hardware stack 242 * severe exception (double fault, nmi, stack fault, debug, mce) hardware stack
243 */ 243 */
244 244
245static inline int valid_stack_ptr(struct thread_info *tinfo, void *p)
246{
247 void *t = (void *)tinfo;
248 return p > t && p < t + THREAD_SIZE - 3;
249}
250
245void dump_trace(struct task_struct *tsk, struct pt_regs *regs, unsigned long * stack, 251void dump_trace(struct task_struct *tsk, struct pt_regs *regs, unsigned long * stack,
246 struct stacktrace_ops *ops, void *data) 252 struct stacktrace_ops *ops, void *data)
247{ 253{
248 const unsigned cpu = smp_processor_id(); 254 const unsigned cpu = smp_processor_id();
249 unsigned long *irqstack_end = (unsigned long *)cpu_pda(cpu)->irqstackptr; 255 unsigned long *irqstack_end = (unsigned long *)cpu_pda(cpu)->irqstackptr;
250 unsigned used = 0; 256 unsigned used = 0;
257 struct thread_info *tinfo;
251 258
252 if (!tsk) 259 if (!tsk)
253 tsk = current; 260 tsk = current;
@@ -290,6 +297,12 @@ void dump_trace(struct task_struct *tsk, struct pt_regs *regs, unsigned long * s
290 if (tsk && tsk != current) 297 if (tsk && tsk != current)
291 stack = (unsigned long *)tsk->thread.rsp; 298 stack = (unsigned long *)tsk->thread.rsp;
292 } 299 }
300 /*
301 * Align the stack pointer on word boundary, later loops
302 * rely on that (and corruption / debug info bugs can cause
303 * unaligned values here):
304 */
305 stack = (unsigned long *)((unsigned long)stack & ~(sizeof(long)-1));
293 306
294 /* 307 /*
295 * Print function call entries within a stack. 'cond' is the 308 * Print function call entries within a stack. 'cond' is the
@@ -364,7 +377,8 @@ void dump_trace(struct task_struct *tsk, struct pt_regs *regs, unsigned long * s
364 /* 377 /*
365 * This handles the process stack: 378 * This handles the process stack:
366 */ 379 */
367 HANDLE_STACK (((long) stack & (THREAD_SIZE-1)) != 0); 380 tinfo = current_thread_info();
381 HANDLE_STACK (valid_stack_ptr(tinfo, stack));
368#undef HANDLE_STACK 382#undef HANDLE_STACK
369} 383}
370EXPORT_SYMBOL(dump_trace); 384EXPORT_SYMBOL(dump_trace);
diff --git a/arch/x86_64/kernel/vmlinux.lds.S b/arch/x86_64/kernel/vmlinux.lds.S
index b9df2ab6529..d9534e750d4 100644
--- a/arch/x86_64/kernel/vmlinux.lds.S
+++ b/arch/x86_64/kernel/vmlinux.lds.S
@@ -17,6 +17,7 @@ PHDRS {
17 text PT_LOAD FLAGS(5); /* R_E */ 17 text PT_LOAD FLAGS(5); /* R_E */
18 data PT_LOAD FLAGS(7); /* RWE */ 18 data PT_LOAD FLAGS(7); /* RWE */
19 user PT_LOAD FLAGS(7); /* RWE */ 19 user PT_LOAD FLAGS(7); /* RWE */
20 data.init PT_LOAD FLAGS(7); /* RWE */
20 note PT_NOTE FLAGS(4); /* R__ */ 21 note PT_NOTE FLAGS(4); /* R__ */
21} 22}
22SECTIONS 23SECTIONS
@@ -59,6 +60,7 @@ SECTIONS
59 } 60 }
60#endif 61#endif
61 62
63 . = ALIGN(PAGE_SIZE); /* Align data segment to page size boundary */
62 /* Data */ 64 /* Data */
63 .data : AT(ADDR(.data) - LOAD_OFFSET) { 65 .data : AT(ADDR(.data) - LOAD_OFFSET) {
64 *(.data) 66 *(.data)
@@ -131,7 +133,7 @@ SECTIONS
131 . = ALIGN(8192); /* init_task */ 133 . = ALIGN(8192); /* init_task */
132 .data.init_task : AT(ADDR(.data.init_task) - LOAD_OFFSET) { 134 .data.init_task : AT(ADDR(.data.init_task) - LOAD_OFFSET) {
133 *(.data.init_task) 135 *(.data.init_task)
134 } :data 136 }:data.init
135 137
136 . = ALIGN(4096); 138 . = ALIGN(4096);
137 .data.page_aligned : AT(ADDR(.data.page_aligned) - LOAD_OFFSET) { 139 .data.page_aligned : AT(ADDR(.data.page_aligned) - LOAD_OFFSET) {
@@ -174,13 +176,7 @@ SECTIONS
174 __setup_end = .; 176 __setup_end = .;
175 __initcall_start = .; 177 __initcall_start = .;
176 .initcall.init : AT(ADDR(.initcall.init) - LOAD_OFFSET) { 178 .initcall.init : AT(ADDR(.initcall.init) - LOAD_OFFSET) {
177 *(.initcall1.init) 179 INITCALLS
178 *(.initcall2.init)
179 *(.initcall3.init)
180 *(.initcall4.init)
181 *(.initcall5.init)
182 *(.initcall6.init)
183 *(.initcall7.init)
184 } 180 }
185 __initcall_end = .; 181 __initcall_end = .;
186 __con_initcall_start = .; 182 __con_initcall_start = .;
diff --git a/arch/x86_64/kernel/vsyscall.c b/arch/x86_64/kernel/vsyscall.c
index a98b460af6a..92546c1526f 100644
--- a/arch/x86_64/kernel/vsyscall.c
+++ b/arch/x86_64/kernel/vsyscall.c
@@ -27,6 +27,9 @@
27#include <linux/jiffies.h> 27#include <linux/jiffies.h>
28#include <linux/sysctl.h> 28#include <linux/sysctl.h>
29#include <linux/getcpu.h> 29#include <linux/getcpu.h>
30#include <linux/cpu.h>
31#include <linux/smp.h>
32#include <linux/notifier.h>
30 33
31#include <asm/vsyscall.h> 34#include <asm/vsyscall.h>
32#include <asm/pgtable.h> 35#include <asm/pgtable.h>
@@ -243,32 +246,17 @@ static ctl_table kernel_root_table2[] = {
243 246
244#endif 247#endif
245 248
246static void __cpuinit write_rdtscp_cb(void *info) 249/* Assume __initcall executes before all user space. Hopefully kmod
247{ 250 doesn't violate that. We'll find out if it does. */
248 write_rdtscp_aux((unsigned long)info); 251static void __cpuinit vsyscall_set_cpu(int cpu)
249}
250
251void __cpuinit vsyscall_set_cpu(int cpu)
252{ 252{
253 unsigned long *d; 253 unsigned long *d;
254 unsigned long node = 0; 254 unsigned long node = 0;
255#ifdef CONFIG_NUMA 255#ifdef CONFIG_NUMA
256 node = cpu_to_node[cpu]; 256 node = cpu_to_node[cpu];
257#endif 257#endif
258 if (cpu_has(&cpu_data[cpu], X86_FEATURE_RDTSCP)) { 258 if (cpu_has(&cpu_data[cpu], X86_FEATURE_RDTSCP))
259 void *info = (void *)((node << 12) | cpu); 259 write_rdtscp_aux((node << 12) | cpu);
260 /* Can happen on preemptive kernel */
261 if (get_cpu() == cpu)
262 write_rdtscp_cb(info);
263#ifdef CONFIG_SMP
264 else {
265 /* the notifier is unfortunately not executed on the
266 target CPU */
267 smp_call_function_single(cpu,write_rdtscp_cb,info,0,1);
268 }
269#endif
270 put_cpu();
271 }
272 260
273 /* Store cpu number in limit so that it can be loaded quickly 261 /* Store cpu number in limit so that it can be loaded quickly
274 in user space in vgetcpu. 262 in user space in vgetcpu.
@@ -280,6 +268,23 @@ void __cpuinit vsyscall_set_cpu(int cpu)
280 *d |= (node >> 4) << 48; 268 *d |= (node >> 4) << 48;
281} 269}
282 270
271static void __cpuinit cpu_vsyscall_init(void *arg)
272{
273 /* preemption should be already off */
274 vsyscall_set_cpu(raw_smp_processor_id());
275}
276
277#ifdef CONFIG_HOTPLUG_CPU
278static int __cpuinit
279cpu_vsyscall_notifier(struct notifier_block *n, unsigned long action, void *arg)
280{
281 long cpu = (long)arg;
282 if (action == CPU_ONLINE)
283 smp_call_function_single(cpu, cpu_vsyscall_init, NULL, 0, 1);
284 return NOTIFY_DONE;
285}
286#endif
287
283static void __init map_vsyscall(void) 288static void __init map_vsyscall(void)
284{ 289{
285 extern char __vsyscall_0; 290 extern char __vsyscall_0;
@@ -299,6 +304,8 @@ static int __init vsyscall_init(void)
299#ifdef CONFIG_SYSCTL 304#ifdef CONFIG_SYSCTL
300 register_sysctl_table(kernel_root_table2, 0); 305 register_sysctl_table(kernel_root_table2, 0);
301#endif 306#endif
307 on_each_cpu(cpu_vsyscall_init, NULL, 0, 1);
308 hotcpu_notifier(cpu_vsyscall_notifier, 0);
302 return 0; 309 return 0;
303} 310}
304 311