aboutsummaryrefslogtreecommitdiffstats
path: root/arch/ia64/kernel
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@woody.linux-foundation.org>2007-07-20 15:02:20 -0400
committerLinus Torvalds <torvalds@woody.linux-foundation.org>2007-07-20 15:02:20 -0400
commitefa7e8673c78cc6de2d6c367eb6f50449c57ed90 (patch)
tree2759535bd50e6d6cc4801a02106ee26f27aa610f /arch/ia64/kernel
parent02d6112cd7d75bf30685ee4b2e9dc0291b4d3133 (diff)
parentbd807f9c5b90c236c3d173a5815d620bc1b1316d (diff)
Merge branch 'release' of git://git.kernel.org/pub/scm/linux/kernel/git/aegl/linux-2.6
* 'release' of git://git.kernel.org/pub/scm/linux/kernel/git/aegl/linux-2.6: [IA64] Prevent people from directly including <asm/rwsem.h>. [IA64] remove time interpolator [IA64] Convert to generic timekeeping/clocksource [IA64] refresh some config files for 64K pagesize [IA64] Delete iosapic_free_rte() [IA64] fallocate system call [IA64] Enable percpu vector domain for IA64_DIG [IA64] Enable percpu vector domain for IA64_GENERIC [IA64] Support irq migration across domain [IA64] Add support for vector domain [IA64] Add mapping table between irq and vector [IA64] Check if irq is sharable [IA64] Fix invalid irq vector assumption for iosapic [IA64] Use dynamic irq for iosapic interrupts [IA64] Use per iosapic lock for indirect iosapic register access [IA64] Cleanup lock order in iosapic_register_intr [IA64] Remove duplicated members in iosapic_rte_info [IA64] Remove block structure for locking in iosapic.c
Diffstat (limited to 'arch/ia64/kernel')
-rw-r--r--arch/ia64/kernel/asm-offsets.c35
-rw-r--r--arch/ia64/kernel/cyclone.c46
-rw-r--r--arch/ia64/kernel/entry.S2
-rw-r--r--arch/ia64/kernel/fsys.S179
-rw-r--r--arch/ia64/kernel/fsyscall_gtod_data.h23
-rw-r--r--arch/ia64/kernel/iosapic.c652
-rw-r--r--arch/ia64/kernel/irq.c2
-rw-r--r--arch/ia64/kernel/irq_ia64.c317
-rw-r--r--arch/ia64/kernel/msi_ia64.c23
-rw-r--r--arch/ia64/kernel/smpboot.c4
-rw-r--r--arch/ia64/kernel/time.c96
11 files changed, 858 insertions, 521 deletions
diff --git a/arch/ia64/kernel/asm-offsets.c b/arch/ia64/kernel/asm-offsets.c
index 2236fabbb3c6..0aebc6f79e95 100644
--- a/arch/ia64/kernel/asm-offsets.c
+++ b/arch/ia64/kernel/asm-offsets.c
@@ -7,6 +7,7 @@
7#define ASM_OFFSETS_C 1 7#define ASM_OFFSETS_C 1
8 8
9#include <linux/sched.h> 9#include <linux/sched.h>
10#include <linux/clocksource.h>
10 11
11#include <asm-ia64/processor.h> 12#include <asm-ia64/processor.h>
12#include <asm-ia64/ptrace.h> 13#include <asm-ia64/ptrace.h>
@@ -15,6 +16,7 @@
15#include <asm-ia64/mca.h> 16#include <asm-ia64/mca.h>
16 17
17#include "../kernel/sigframe.h" 18#include "../kernel/sigframe.h"
19#include "../kernel/fsyscall_gtod_data.h"
18 20
19#define DEFINE(sym, val) \ 21#define DEFINE(sym, val) \
20 asm volatile("\n->" #sym " %0 " #val : : "i" (val)) 22 asm volatile("\n->" #sym " %0 " #val : : "i" (val))
@@ -256,17 +258,24 @@ void foo(void)
256 BLANK(); 258 BLANK();
257 259
258 /* used by fsys_gettimeofday in arch/ia64/kernel/fsys.S */ 260 /* used by fsys_gettimeofday in arch/ia64/kernel/fsys.S */
259 DEFINE(IA64_TIME_INTERPOLATOR_ADDRESS_OFFSET, offsetof (struct time_interpolator, addr)); 261 DEFINE(IA64_GTOD_LOCK_OFFSET,
260 DEFINE(IA64_TIME_INTERPOLATOR_SOURCE_OFFSET, offsetof (struct time_interpolator, source)); 262 offsetof (struct fsyscall_gtod_data_t, lock));
261 DEFINE(IA64_TIME_INTERPOLATOR_SHIFT_OFFSET, offsetof (struct time_interpolator, shift)); 263 DEFINE(IA64_GTOD_WALL_TIME_OFFSET,
262 DEFINE(IA64_TIME_INTERPOLATOR_NSEC_OFFSET, offsetof (struct time_interpolator, nsec_per_cyc)); 264 offsetof (struct fsyscall_gtod_data_t, wall_time));
263 DEFINE(IA64_TIME_INTERPOLATOR_OFFSET_OFFSET, offsetof (struct time_interpolator, offset)); 265 DEFINE(IA64_GTOD_MONO_TIME_OFFSET,
264 DEFINE(IA64_TIME_INTERPOLATOR_LAST_CYCLE_OFFSET, offsetof (struct time_interpolator, last_cycle)); 266 offsetof (struct fsyscall_gtod_data_t, monotonic_time));
265 DEFINE(IA64_TIME_INTERPOLATOR_LAST_COUNTER_OFFSET, offsetof (struct time_interpolator, last_counter)); 267 DEFINE(IA64_CLKSRC_MASK_OFFSET,
266 DEFINE(IA64_TIME_INTERPOLATOR_JITTER_OFFSET, offsetof (struct time_interpolator, jitter)); 268 offsetof (struct fsyscall_gtod_data_t, clk_mask));
267 DEFINE(IA64_TIME_INTERPOLATOR_MASK_OFFSET, offsetof (struct time_interpolator, mask)); 269 DEFINE(IA64_CLKSRC_MULT_OFFSET,
268 DEFINE(IA64_TIME_SOURCE_CPU, TIME_SOURCE_CPU); 270 offsetof (struct fsyscall_gtod_data_t, clk_mult));
269 DEFINE(IA64_TIME_SOURCE_MMIO64, TIME_SOURCE_MMIO64); 271 DEFINE(IA64_CLKSRC_SHIFT_OFFSET,
270 DEFINE(IA64_TIME_SOURCE_MMIO32, TIME_SOURCE_MMIO32); 272 offsetof (struct fsyscall_gtod_data_t, clk_shift));
271 DEFINE(IA64_TIMESPEC_TV_NSEC_OFFSET, offsetof (struct timespec, tv_nsec)); 273 DEFINE(IA64_CLKSRC_MMIO_OFFSET,
274 offsetof (struct fsyscall_gtod_data_t, clk_fsys_mmio));
275 DEFINE(IA64_CLKSRC_CYCLE_LAST_OFFSET,
276 offsetof (struct fsyscall_gtod_data_t, clk_cycle_last));
277 DEFINE(IA64_ITC_JITTER_OFFSET,
278 offsetof (struct itc_jitter_data_t, itc_jitter));
279 DEFINE(IA64_ITC_LASTCYCLE_OFFSET,
280 offsetof (struct itc_jitter_data_t, itc_lastcycle));
272} 281}
diff --git a/arch/ia64/kernel/cyclone.c b/arch/ia64/kernel/cyclone.c
index e00b21514f7c..2fd96d9062a1 100644
--- a/arch/ia64/kernel/cyclone.c
+++ b/arch/ia64/kernel/cyclone.c
@@ -3,6 +3,7 @@
3#include <linux/time.h> 3#include <linux/time.h>
4#include <linux/errno.h> 4#include <linux/errno.h>
5#include <linux/timex.h> 5#include <linux/timex.h>
6#include <linux/clocksource.h>
6#include <asm/io.h> 7#include <asm/io.h>
7 8
8/* IBM Summit (EXA) Cyclone counter code*/ 9/* IBM Summit (EXA) Cyclone counter code*/
@@ -18,13 +19,21 @@ void __init cyclone_setup(void)
18 use_cyclone = 1; 19 use_cyclone = 1;
19} 20}
20 21
22static void __iomem *cyclone_mc;
21 23
22struct time_interpolator cyclone_interpolator = { 24static cycle_t read_cyclone(void)
23 .source = TIME_SOURCE_MMIO64, 25{
24 .shift = 16, 26 return (cycle_t)readq((void __iomem *)cyclone_mc);
25 .frequency = CYCLONE_TIMER_FREQ, 27}
26 .drift = -100, 28
27 .mask = (1LL << 40) - 1 29static struct clocksource clocksource_cyclone = {
30 .name = "cyclone",
31 .rating = 300,
32 .read = read_cyclone,
33 .mask = (1LL << 40) - 1,
34 .mult = 0, /*to be caluclated*/
35 .shift = 16,
36 .flags = CLOCK_SOURCE_IS_CONTINUOUS,
28}; 37};
29 38
30int __init init_cyclone_clock(void) 39int __init init_cyclone_clock(void)
@@ -44,13 +53,15 @@ int __init init_cyclone_clock(void)
44 offset = (CYCLONE_CBAR_ADDR); 53 offset = (CYCLONE_CBAR_ADDR);
45 reg = (u64*)ioremap_nocache(offset, sizeof(u64)); 54 reg = (u64*)ioremap_nocache(offset, sizeof(u64));
46 if(!reg){ 55 if(!reg){
47 printk(KERN_ERR "Summit chipset: Could not find valid CBAR register.\n"); 56 printk(KERN_ERR "Summit chipset: Could not find valid CBAR"
57 " register.\n");
48 use_cyclone = 0; 58 use_cyclone = 0;
49 return -ENODEV; 59 return -ENODEV;
50 } 60 }
51 base = readq(reg); 61 base = readq(reg);
52 if(!base){ 62 if(!base){
53 printk(KERN_ERR "Summit chipset: Could not find valid CBAR value.\n"); 63 printk(KERN_ERR "Summit chipset: Could not find valid CBAR"
64 " value.\n");
54 use_cyclone = 0; 65 use_cyclone = 0;
55 return -ENODEV; 66 return -ENODEV;
56 } 67 }
@@ -60,7 +71,8 @@ int __init init_cyclone_clock(void)
60 offset = (base + CYCLONE_PMCC_OFFSET); 71 offset = (base + CYCLONE_PMCC_OFFSET);
61 reg = (u64*)ioremap_nocache(offset, sizeof(u64)); 72 reg = (u64*)ioremap_nocache(offset, sizeof(u64));
62 if(!reg){ 73 if(!reg){
63 printk(KERN_ERR "Summit chipset: Could not find valid PMCC register.\n"); 74 printk(KERN_ERR "Summit chipset: Could not find valid PMCC"
75 " register.\n");
64 use_cyclone = 0; 76 use_cyclone = 0;
65 return -ENODEV; 77 return -ENODEV;
66 } 78 }
@@ -71,7 +83,8 @@ int __init init_cyclone_clock(void)
71 offset = (base + CYCLONE_MPCS_OFFSET); 83 offset = (base + CYCLONE_MPCS_OFFSET);
72 reg = (u64*)ioremap_nocache(offset, sizeof(u64)); 84 reg = (u64*)ioremap_nocache(offset, sizeof(u64));
73 if(!reg){ 85 if(!reg){
74 printk(KERN_ERR "Summit chipset: Could not find valid MPCS register.\n"); 86 printk(KERN_ERR "Summit chipset: Could not find valid MPCS"
87 " register.\n");
75 use_cyclone = 0; 88 use_cyclone = 0;
76 return -ENODEV; 89 return -ENODEV;
77 } 90 }
@@ -82,7 +95,8 @@ int __init init_cyclone_clock(void)
82 offset = (base + CYCLONE_MPMC_OFFSET); 95 offset = (base + CYCLONE_MPMC_OFFSET);
83 cyclone_timer = (u32*)ioremap_nocache(offset, sizeof(u32)); 96 cyclone_timer = (u32*)ioremap_nocache(offset, sizeof(u32));
84 if(!cyclone_timer){ 97 if(!cyclone_timer){
85 printk(KERN_ERR "Summit chipset: Could not find valid MPMC register.\n"); 98 printk(KERN_ERR "Summit chipset: Could not find valid MPMC"
99 " register.\n");
86 use_cyclone = 0; 100 use_cyclone = 0;
87 return -ENODEV; 101 return -ENODEV;
88 } 102 }
@@ -93,7 +107,8 @@ int __init init_cyclone_clock(void)
93 int stall = 100; 107 int stall = 100;
94 while(stall--) barrier(); 108 while(stall--) barrier();
95 if(readl(cyclone_timer) == old){ 109 if(readl(cyclone_timer) == old){
96 printk(KERN_ERR "Summit chipset: Counter not counting! DISABLED\n"); 110 printk(KERN_ERR "Summit chipset: Counter not counting!"
111 " DISABLED\n");
97 iounmap(cyclone_timer); 112 iounmap(cyclone_timer);
98 cyclone_timer = 0; 113 cyclone_timer = 0;
99 use_cyclone = 0; 114 use_cyclone = 0;
@@ -101,8 +116,11 @@ int __init init_cyclone_clock(void)
101 } 116 }
102 } 117 }
103 /* initialize last tick */ 118 /* initialize last tick */
104 cyclone_interpolator.addr = cyclone_timer; 119 cyclone_mc = cyclone_timer;
105 register_time_interpolator(&cyclone_interpolator); 120 clocksource_cyclone.fsys_mmio = cyclone_timer;
121 clocksource_cyclone.mult = clocksource_hz2mult(CYCLONE_TIMER_FREQ,
122 clocksource_cyclone.shift);
123 clocksource_register(&clocksource_cyclone);
106 124
107 return 0; 125 return 0;
108} 126}
diff --git a/arch/ia64/kernel/entry.S b/arch/ia64/kernel/entry.S
index 95f517515235..c36f43c94600 100644
--- a/arch/ia64/kernel/entry.S
+++ b/arch/ia64/kernel/entry.S
@@ -1581,7 +1581,7 @@ sys_call_table:
1581 data8 sys_sync_file_range // 1300 1581 data8 sys_sync_file_range // 1300
1582 data8 sys_tee 1582 data8 sys_tee
1583 data8 sys_vmsplice 1583 data8 sys_vmsplice
1584 data8 sys_ni_syscall // reserved for move_pages 1584 data8 sys_fallocate
1585 data8 sys_getcpu 1585 data8 sys_getcpu
1586 data8 sys_epoll_pwait // 1305 1586 data8 sys_epoll_pwait // 1305
1587 data8 sys_utimensat 1587 data8 sys_utimensat
diff --git a/arch/ia64/kernel/fsys.S b/arch/ia64/kernel/fsys.S
index 3f926c2dc708..44841971f077 100644
--- a/arch/ia64/kernel/fsys.S
+++ b/arch/ia64/kernel/fsys.S
@@ -147,12 +147,11 @@ ENTRY(fsys_set_tid_address)
147 FSYS_RETURN 147 FSYS_RETURN
148END(fsys_set_tid_address) 148END(fsys_set_tid_address)
149 149
150/* 150#if IA64_GTOD_LOCK_OFFSET !=0
151 * Ensure that the time interpolator structure is compatible with the asm code 151#error fsys_gettimeofday incompatible with changes to struct fsyscall_gtod_data_t
152 */ 152#endif
153#if IA64_TIME_INTERPOLATOR_SOURCE_OFFSET !=0 || IA64_TIME_INTERPOLATOR_SHIFT_OFFSET != 2 \ 153#if IA64_ITC_JITTER_OFFSET !=0
154 || IA64_TIME_INTERPOLATOR_JITTER_OFFSET != 3 || IA64_TIME_INTERPOLATOR_NSEC_OFFSET != 4 154#error fsys_gettimeofday incompatible with changes to struct itc_jitter_data_t
155#error fsys_gettimeofday incompatible with changes to struct time_interpolator
156#endif 155#endif
157#define CLOCK_REALTIME 0 156#define CLOCK_REALTIME 0
158#define CLOCK_MONOTONIC 1 157#define CLOCK_MONOTONIC 1
@@ -179,126 +178,124 @@ ENTRY(fsys_gettimeofday)
179 // r11 = preserved: saved ar.pfs 178 // r11 = preserved: saved ar.pfs
180 // r12 = preserved: memory stack 179 // r12 = preserved: memory stack
181 // r13 = preserved: thread pointer 180 // r13 = preserved: thread pointer
182 // r14 = address of mask / mask 181 // r14 = address of mask / mask value
183 // r15 = preserved: system call number 182 // r15 = preserved: system call number
184 // r16 = preserved: current task pointer 183 // r16 = preserved: current task pointer
185 // r17 = wall to monotonic use 184 // r17 = (not used)
186 // r18 = time_interpolator->offset 185 // r18 = (not used)
187 // r19 = address of wall_to_monotonic 186 // r19 = address of itc_lastcycle
188 // r20 = pointer to struct time_interpolator / pointer to time_interpolator->address 187 // r20 = struct fsyscall_gtod_data (= address of gtod_lock.sequence)
189 // r21 = shift factor 188 // r21 = address of mmio_ptr
190 // r22 = address of time interpolator->last_counter 189 // r22 = address of wall_time or monotonic_time
191 // r23 = address of time_interpolator->last_cycle 190 // r23 = address of shift / value
192 // r24 = adress of time_interpolator->offset 191 // r24 = address mult factor / cycle_last value
193 // r25 = last_cycle value 192 // r25 = itc_lastcycle value
194 // r26 = last_counter value 193 // r26 = address clocksource cycle_last
195 // r27 = pointer to xtime 194 // r27 = (not used)
196 // r28 = sequence number at the beginning of critcal section 195 // r28 = sequence number at the beginning of critcal section
197 // r29 = address of seqlock 196 // r29 = address of itc_jitter
198 // r30 = time processing flags / memory address 197 // r30 = time processing flags / memory address
199 // r31 = pointer to result 198 // r31 = pointer to result
200 // Predicates 199 // Predicates
201 // p6,p7 short term use 200 // p6,p7 short term use
202 // p8 = timesource ar.itc 201 // p8 = timesource ar.itc
203 // p9 = timesource mmio64 202 // p9 = timesource mmio64
204 // p10 = timesource mmio32 203 // p10 = timesource mmio32 - not used
205 // p11 = timesource not to be handled by asm code 204 // p11 = timesource not to be handled by asm code
206 // p12 = memory time source ( = p9 | p10) 205 // p12 = memory time source ( = p9 | p10) - not used
207 // p13 = do cmpxchg with time_interpolator_last_cycle 206 // p13 = do cmpxchg with itc_lastcycle
208 // p14 = Divide by 1000 207 // p14 = Divide by 1000
209 // p15 = Add monotonic 208 // p15 = Add monotonic
210 // 209 //
211 // Note that instructions are optimized for McKinley. McKinley can process two 210 // Note that instructions are optimized for McKinley. McKinley can
212 // bundles simultaneously and therefore we continuously try to feed the CPU 211 // process two bundles simultaneously and therefore we continuously
213 // two bundles and then a stop. 212 // try to feed the CPU two bundles and then a stop.
214 tnat.nz p6,p0 = r31 // branch deferred since it does not fit into bundle structure 213 //
214 // Additional note that code has changed a lot. Optimization is TBD.
215 // Comments begin with "?" are maybe outdated.
216 tnat.nz p6,p0 = r31 // ? branch deferred to fit later bundle
215 mov pr = r30,0xc000 // Set predicates according to function 217 mov pr = r30,0xc000 // Set predicates according to function
216 add r2 = TI_FLAGS+IA64_TASK_SIZE,r16 218 add r2 = TI_FLAGS+IA64_TASK_SIZE,r16
217 movl r20 = time_interpolator 219 movl r20 = fsyscall_gtod_data // load fsyscall gettimeofday data address
218 ;; 220 ;;
219 ld8 r20 = [r20] // get pointer to time_interpolator structure 221 movl r29 = itc_jitter_data // itc_jitter
220 movl r29 = xtime_lock 222 add r22 = IA64_GTOD_WALL_TIME_OFFSET,r20 // wall_time
221 ld4 r2 = [r2] // process work pending flags 223 ld4 r2 = [r2] // process work pending flags
222 movl r27 = xtime 224 ;;
223 ;; // only one bundle here 225(p15) add r22 = IA64_GTOD_MONO_TIME_OFFSET,r20 // monotonic_time
224 ld8 r21 = [r20] // first quad with control information 226 add r21 = IA64_CLKSRC_MMIO_OFFSET,r20
227 add r19 = IA64_ITC_LASTCYCLE_OFFSET,r29
225 and r2 = TIF_ALLWORK_MASK,r2 228 and r2 = TIF_ALLWORK_MASK,r2
226(p6) br.cond.spnt.few .fail_einval // deferred branch 229(p6) br.cond.spnt.few .fail_einval // ? deferred branch
227 ;; 230 ;;
228 add r10 = IA64_TIME_INTERPOLATOR_ADDRESS_OFFSET,r20 231 add r26 = IA64_CLKSRC_CYCLE_LAST_OFFSET,r20 // clksrc_cycle_last
229 extr r3 = r21,32,32 // time_interpolator->nsec_per_cyc
230 extr r8 = r21,0,16 // time_interpolator->source
231 cmp.ne p6, p0 = 0, r2 // Fallback if work is scheduled 232 cmp.ne p6, p0 = 0, r2 // Fallback if work is scheduled
232(p6) br.cond.spnt.many fsys_fallback_syscall 233(p6) br.cond.spnt.many fsys_fallback_syscall
233 ;; 234 ;;
234 cmp.eq p8,p12 = 0,r8 // Check for cpu timer 235 // Begin critical section
235 cmp.eq p9,p0 = 1,r8 // MMIO64 ? 236.time_redo:
236 extr r2 = r21,24,8 // time_interpolator->jitter 237 ld4.acq r28 = [r20] // gtod_lock.sequence, Must take first
237 cmp.eq p10,p0 = 2,r8 // MMIO32 ? 238 ;;
238 cmp.ltu p11,p0 = 2,r8 // function or other clock 239 and r28 = ~1,r28 // And make sequence even to force retry if odd
239(p11) br.cond.spnt.many fsys_fallback_syscall
240 ;; 240 ;;
241 setf.sig f7 = r3 // Setup for scaling of counter 241 ld8 r30 = [r21] // clocksource->mmio_ptr
242(p15) movl r19 = wall_to_monotonic 242 add r24 = IA64_CLKSRC_MULT_OFFSET,r20
243(p12) ld8 r30 = [r10] 243 ld4 r2 = [r29] // itc_jitter value
244 cmp.ne p13,p0 = r2,r0 // need jitter compensation? 244 add r23 = IA64_CLKSRC_SHIFT_OFFSET,r20
245 extr r21 = r21,16,8 // shift factor 245 add r14 = IA64_CLKSRC_MASK_OFFSET,r20
246 ;; 246 ;;
247.time_redo: 247 ld4 r3 = [r24] // clocksource mult value
248 .pred.rel.mutex p8,p9,p10 248 ld8 r14 = [r14] // clocksource mask value
249 ld4.acq r28 = [r29] // xtime_lock.sequence. Must come first for locking purposes 249 cmp.eq p8,p9 = 0,r30 // use cpu timer if no mmio_ptr
250 ;; 250 ;;
251 and r28 = ~1,r28 // Make sequence even to force retry if odd 251 setf.sig f7 = r3 // Setup for mult scaling of counter
252(p8) cmp.ne p13,p0 = r2,r0 // need itc_jitter compensation, set p13
253 ld4 r23 = [r23] // clocksource shift value
254 ld8 r24 = [r26] // get clksrc_cycle_last value
255(p9) cmp.eq p13,p0 = 0,r30 // if mmio_ptr, clear p13 jitter control
252 ;; 256 ;;
257 .pred.rel.mutex p8,p9
253(p8) mov r2 = ar.itc // CPU_TIMER. 36 clocks latency!!! 258(p8) mov r2 = ar.itc // CPU_TIMER. 36 clocks latency!!!
254 add r22 = IA64_TIME_INTERPOLATOR_LAST_COUNTER_OFFSET,r20 259(p9) ld8 r2 = [r30] // MMIO_TIMER. Could also have latency issues..
255(p9) ld8 r2 = [r30] // readq(ti->address). Could also have latency issues.. 260(p13) ld8 r25 = [r19] // get itc_lastcycle value
256(p10) ld4 r2 = [r30] // readw(ti->address) 261 ;; // ? could be removed by moving the last add upward
257(p13) add r23 = IA64_TIME_INTERPOLATOR_LAST_CYCLE_OFFSET,r20 262 ld8 r9 = [r22],IA64_TIMESPEC_TV_NSEC_OFFSET // tv_sec
258 ;; // could be removed by moving the last add upward 263 ;;
259 ld8 r26 = [r22] // time_interpolator->last_counter 264 ld8 r8 = [r22],-IA64_TIMESPEC_TV_NSEC_OFFSET // tv_nsec
260(p13) ld8 r25 = [r23] // time interpolator->last_cycle 265(p13) sub r3 = r25,r2 // Diff needed before comparison (thanks davidm)
261 add r24 = IA64_TIME_INTERPOLATOR_OFFSET_OFFSET,r20 266 ;;
262(p15) ld8 r17 = [r19],IA64_TIMESPEC_TV_NSEC_OFFSET 267(p13) cmp.gt.unc p6,p7 = r3,r0 // check if it is less than last. p6,p7 cleared
263 ld8 r9 = [r27],IA64_TIMESPEC_TV_NSEC_OFFSET 268 sub r10 = r2,r24 // current_cycle - last_cycle
264 add r14 = IA64_TIME_INTERPOLATOR_MASK_OFFSET, r20 269 ;;
265 ;; 270(p6) sub r10 = r25,r24 // time we got was less than last_cycle
266 ld8 r18 = [r24] // time_interpolator->offset
267 ld8 r8 = [r27],-IA64_TIMESPEC_TV_NSEC_OFFSET // xtime.tv_nsec
268(p13) sub r3 = r25,r2 // Diff needed before comparison (thanks davidm)
269 ;;
270 ld8 r14 = [r14] // time_interpolator->mask
271(p13) cmp.gt.unc p6,p7 = r3,r0 // check if it is less than last. p6,p7 cleared
272 sub r10 = r2,r26 // current_counter - last_counter
273 ;;
274(p6) sub r10 = r25,r26 // time we got was less than last_cycle
275(p7) mov ar.ccv = r25 // more than last_cycle. Prep for cmpxchg 271(p7) mov ar.ccv = r25 // more than last_cycle. Prep for cmpxchg
276 ;; 272 ;;
273(p7) cmpxchg8.rel r3 = [r19],r2,ar.ccv
274 ;;
275(p7) cmp.ne p7,p0 = r25,r3 // if cmpxchg not successful
276 ;;
277(p7) sub r10 = r3,r24 // then use new last_cycle instead
278 ;;
277 and r10 = r10,r14 // Apply mask 279 and r10 = r10,r14 // Apply mask
278 ;; 280 ;;
279 setf.sig f8 = r10 281 setf.sig f8 = r10
280 nop.i 123 282 nop.i 123
281 ;; 283 ;;
282(p7) cmpxchg8.rel r3 = [r23],r2,ar.ccv 284 // fault check takes 5 cycles and we have spare time
283EX(.fail_efault, probe.w.fault r31, 3) // This takes 5 cycles and we have spare time 285EX(.fail_efault, probe.w.fault r31, 3)
284 xmpy.l f8 = f8,f7 // nsec_per_cyc*(counter-last_counter) 286 xmpy.l f8 = f8,f7 // nsec_per_cyc*(counter-last_counter)
285(p15) add r9 = r9,r17 // Add wall to monotonic.secs to result secs
286 ;; 287 ;;
287(p15) ld8 r17 = [r19],-IA64_TIMESPEC_TV_NSEC_OFFSET 288 // ? simulate tbit.nz.or p7,p0 = r28,0
288(p7) cmp.ne p7,p0 = r25,r3 // if cmpxchg not successful redo
289 // simulate tbit.nz.or p7,p0 = r28,0
290 getf.sig r2 = f8 289 getf.sig r2 = f8
291 mf 290 mf
292 add r8 = r8,r18 // Add time interpolator offset
293 ;; 291 ;;
294 ld4 r10 = [r29] // xtime_lock.sequence 292 ld4 r10 = [r20] // gtod_lock.sequence
295(p15) add r8 = r8, r17 // Add monotonic.nsecs to nsecs 293 shr.u r2 = r2,r23 // shift by factor
296 shr.u r2 = r2,r21 294 ;; // ? overloaded 3 bundles!
297 ;; // overloaded 3 bundles!
298 // End critical section.
299 add r8 = r8,r2 // Add xtime.nsecs 295 add r8 = r8,r2 // Add xtime.nsecs
300 cmp4.ne.or p7,p0 = r28,r10 296 cmp4.ne p7,p0 = r28,r10
301(p7) br.cond.dpnt.few .time_redo // sequence number changed ? 297(p7) br.cond.dpnt.few .time_redo // sequence number changed, redo
298 // End critical section.
302 // Now r8=tv->tv_nsec and r9=tv->tv_sec 299 // Now r8=tv->tv_nsec and r9=tv->tv_sec
303 mov r10 = r0 300 mov r10 = r0
304 movl r2 = 1000000000 301 movl r2 = 1000000000
@@ -308,19 +305,19 @@ EX(.fail_efault, probe.w.fault r31, 3) // This takes 5 cycles and we have spare
308.time_normalize: 305.time_normalize:
309 mov r21 = r8 306 mov r21 = r8
310 cmp.ge p6,p0 = r8,r2 307 cmp.ge p6,p0 = r8,r2
311(p14) shr.u r20 = r8, 3 // We can repeat this if necessary just wasting some time 308(p14) shr.u r20 = r8, 3 // We can repeat this if necessary just wasting time
312 ;; 309 ;;
313(p14) setf.sig f8 = r20 310(p14) setf.sig f8 = r20
314(p6) sub r8 = r8,r2 311(p6) sub r8 = r8,r2
315(p6) add r9 = 1,r9 // two nops before the branch. 312(p6) add r9 = 1,r9 // two nops before the branch.
316(p14) setf.sig f7 = r3 // Chances for repeats are 1 in 10000 for gettod 313(p14) setf.sig f7 = r3 // Chances for repeats are 1 in 10000 for gettod
317(p6) br.cond.dpnt.few .time_normalize 314(p6) br.cond.dpnt.few .time_normalize
318 ;; 315 ;;
319 // Divided by 8 though shift. Now divide by 125 316 // Divided by 8 though shift. Now divide by 125
320 // The compiler was able to do that with a multiply 317 // The compiler was able to do that with a multiply
321 // and a shift and we do the same 318 // and a shift and we do the same
322EX(.fail_efault, probe.w.fault r23, 3) // This also costs 5 cycles 319EX(.fail_efault, probe.w.fault r23, 3) // This also costs 5 cycles
323(p14) xmpy.hu f8 = f8, f7 // xmpy has 5 cycles latency so use it... 320(p14) xmpy.hu f8 = f8, f7 // xmpy has 5 cycles latency so use it
324 ;; 321 ;;
325 mov r8 = r0 322 mov r8 = r0
326(p14) getf.sig r2 = f8 323(p14) getf.sig r2 = f8
diff --git a/arch/ia64/kernel/fsyscall_gtod_data.h b/arch/ia64/kernel/fsyscall_gtod_data.h
new file mode 100644
index 000000000000..490dab55fba3
--- /dev/null
+++ b/arch/ia64/kernel/fsyscall_gtod_data.h
@@ -0,0 +1,23 @@
1/*
2 * (c) Copyright 2007 Hewlett-Packard Development Company, L.P.
3 * Contributed by Peter Keilty <peter.keilty@hp.com>
4 *
5 * fsyscall gettimeofday data
6 */
7
8struct fsyscall_gtod_data_t {
9 seqlock_t lock;
10 struct timespec wall_time;
11 struct timespec monotonic_time;
12 cycle_t clk_mask;
13 u32 clk_mult;
14 u32 clk_shift;
15 void *clk_fsys_mmio;
16 cycle_t clk_cycle_last;
17} __attribute__ ((aligned (L1_CACHE_BYTES)));
18
19struct itc_jitter_data_t {
20 int itc_jitter;
21 cycle_t itc_lastcycle;
22} __attribute__ ((aligned (L1_CACHE_BYTES)));
23
diff --git a/arch/ia64/kernel/iosapic.c b/arch/ia64/kernel/iosapic.c
index 37f46527d233..91e6dc1e7baf 100644
--- a/arch/ia64/kernel/iosapic.c
+++ b/arch/ia64/kernel/iosapic.c
@@ -118,15 +118,25 @@ static DEFINE_SPINLOCK(iosapic_lock);
118 * vector. 118 * vector.
119 */ 119 */
120 120
121struct iosapic_rte_info { 121#define NO_REF_RTE 0
122 struct list_head rte_list; /* node in list of RTEs sharing the 122
123 * same vector */ 123static struct iosapic {
124 char __iomem *addr; /* base address of IOSAPIC */ 124 char __iomem *addr; /* base address of IOSAPIC */
125 unsigned int gsi_base; /* first GSI assigned to this 125 unsigned int gsi_base; /* GSI base */
126 * IOSAPIC */ 126 unsigned short num_rte; /* # of RTEs on this IOSAPIC */
127 int rtes_inuse; /* # of RTEs in use on this IOSAPIC */
128#ifdef CONFIG_NUMA
129 unsigned short node; /* numa node association via pxm */
130#endif
131 spinlock_t lock; /* lock for indirect reg access */
132} iosapic_lists[NR_IOSAPICS];
133
134struct iosapic_rte_info {
135 struct list_head rte_list; /* RTEs sharing the same vector */
127 char rte_index; /* IOSAPIC RTE index */ 136 char rte_index; /* IOSAPIC RTE index */
128 int refcnt; /* reference counter */ 137 int refcnt; /* reference counter */
129 unsigned int flags; /* flags */ 138 unsigned int flags; /* flags */
139 struct iosapic *iosapic;
130} ____cacheline_aligned; 140} ____cacheline_aligned;
131 141
132static struct iosapic_intr_info { 142static struct iosapic_intr_info {
@@ -140,24 +150,23 @@ static struct iosapic_intr_info {
140 unsigned char polarity: 1; /* interrupt polarity 150 unsigned char polarity: 1; /* interrupt polarity
141 * (see iosapic.h) */ 151 * (see iosapic.h) */
142 unsigned char trigger : 1; /* trigger mode (see iosapic.h) */ 152 unsigned char trigger : 1; /* trigger mode (see iosapic.h) */
143} iosapic_intr_info[IA64_NUM_VECTORS]; 153} iosapic_intr_info[NR_IRQS];
144
145static struct iosapic {
146 char __iomem *addr; /* base address of IOSAPIC */
147 unsigned int gsi_base; /* first GSI assigned to this
148 * IOSAPIC */
149 unsigned short num_rte; /* # of RTEs on this IOSAPIC */
150 int rtes_inuse; /* # of RTEs in use on this IOSAPIC */
151#ifdef CONFIG_NUMA
152 unsigned short node; /* numa node association via pxm */
153#endif
154} iosapic_lists[NR_IOSAPICS];
155 154
156static unsigned char pcat_compat __devinitdata; /* 8259 compatibility flag */ 155static unsigned char pcat_compat __devinitdata; /* 8259 compatibility flag */
157 156
158static int iosapic_kmalloc_ok; 157static int iosapic_kmalloc_ok;
159static LIST_HEAD(free_rte_list); 158static LIST_HEAD(free_rte_list);
160 159
160static inline void
161iosapic_write(struct iosapic *iosapic, unsigned int reg, u32 val)
162{
163 unsigned long flags;
164
165 spin_lock_irqsave(&iosapic->lock, flags);
166 __iosapic_write(iosapic->addr, reg, val);
167 spin_unlock_irqrestore(&iosapic->lock, flags);
168}
169
161/* 170/*
162 * Find an IOSAPIC associated with a GSI 171 * Find an IOSAPIC associated with a GSI
163 */ 172 */
@@ -175,17 +184,18 @@ find_iosapic (unsigned int gsi)
175 return -1; 184 return -1;
176} 185}
177 186
178static inline int 187static inline int __gsi_to_irq(unsigned int gsi)
179_gsi_to_vector (unsigned int gsi)
180{ 188{
189 int irq;
181 struct iosapic_intr_info *info; 190 struct iosapic_intr_info *info;
182 struct iosapic_rte_info *rte; 191 struct iosapic_rte_info *rte;
183 192
184 for (info = iosapic_intr_info; info < 193 for (irq = 0; irq < NR_IRQS; irq++) {
185 iosapic_intr_info + IA64_NUM_VECTORS; ++info) 194 info = &iosapic_intr_info[irq];
186 list_for_each_entry(rte, &info->rtes, rte_list) 195 list_for_each_entry(rte, &info->rtes, rte_list)
187 if (rte->gsi_base + rte->rte_index == gsi) 196 if (rte->iosapic->gsi_base + rte->rte_index == gsi)
188 return info - iosapic_intr_info; 197 return irq;
198 }
189 return -1; 199 return -1;
190} 200}
191 201
@@ -196,7 +206,10 @@ _gsi_to_vector (unsigned int gsi)
196inline int 206inline int
197gsi_to_vector (unsigned int gsi) 207gsi_to_vector (unsigned int gsi)
198{ 208{
199 return _gsi_to_vector(gsi); 209 int irq = __gsi_to_irq(gsi);
210 if (check_irq_used(irq) < 0)
211 return -1;
212 return irq_to_vector(irq);
200} 213}
201 214
202int 215int
@@ -204,66 +217,48 @@ gsi_to_irq (unsigned int gsi)
204{ 217{
205 unsigned long flags; 218 unsigned long flags;
206 int irq; 219 int irq;
207 /* 220
208 * XXX fix me: this assumes an identity mapping between IA-64 vector
209 * and Linux irq numbers...
210 */
211 spin_lock_irqsave(&iosapic_lock, flags); 221 spin_lock_irqsave(&iosapic_lock, flags);
212 { 222 irq = __gsi_to_irq(gsi);
213 irq = _gsi_to_vector(gsi);
214 }
215 spin_unlock_irqrestore(&iosapic_lock, flags); 223 spin_unlock_irqrestore(&iosapic_lock, flags);
216
217 return irq; 224 return irq;
218} 225}
219 226
220static struct iosapic_rte_info *gsi_vector_to_rte(unsigned int gsi, 227static struct iosapic_rte_info *find_rte(unsigned int irq, unsigned int gsi)
221 unsigned int vec)
222{ 228{
223 struct iosapic_rte_info *rte; 229 struct iosapic_rte_info *rte;
224 230
225 list_for_each_entry(rte, &iosapic_intr_info[vec].rtes, rte_list) 231 list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list)
226 if (rte->gsi_base + rte->rte_index == gsi) 232 if (rte->iosapic->gsi_base + rte->rte_index == gsi)
227 return rte; 233 return rte;
228 return NULL; 234 return NULL;
229} 235}
230 236
231static void 237static void
232set_rte (unsigned int gsi, unsigned int vector, unsigned int dest, int mask) 238set_rte (unsigned int gsi, unsigned int irq, unsigned int dest, int mask)
233{ 239{
234 unsigned long pol, trigger, dmode; 240 unsigned long pol, trigger, dmode;
235 u32 low32, high32; 241 u32 low32, high32;
236 char __iomem *addr;
237 int rte_index; 242 int rte_index;
238 char redir; 243 char redir;
239 struct iosapic_rte_info *rte; 244 struct iosapic_rte_info *rte;
245 ia64_vector vector = irq_to_vector(irq);
240 246
241 DBG(KERN_DEBUG"IOSAPIC: routing vector %d to 0x%x\n", vector, dest); 247 DBG(KERN_DEBUG"IOSAPIC: routing vector %d to 0x%x\n", vector, dest);
242 248
243 rte = gsi_vector_to_rte(gsi, vector); 249 rte = find_rte(irq, gsi);
244 if (!rte) 250 if (!rte)
245 return; /* not an IOSAPIC interrupt */ 251 return; /* not an IOSAPIC interrupt */
246 252
247 rte_index = rte->rte_index; 253 rte_index = rte->rte_index;
248 addr = rte->addr; 254 pol = iosapic_intr_info[irq].polarity;
249 pol = iosapic_intr_info[vector].polarity; 255 trigger = iosapic_intr_info[irq].trigger;
250 trigger = iosapic_intr_info[vector].trigger; 256 dmode = iosapic_intr_info[irq].dmode;
251 dmode = iosapic_intr_info[vector].dmode;
252 257
253 redir = (dmode == IOSAPIC_LOWEST_PRIORITY) ? 1 : 0; 258 redir = (dmode == IOSAPIC_LOWEST_PRIORITY) ? 1 : 0;
254 259
255#ifdef CONFIG_SMP 260#ifdef CONFIG_SMP
256 { 261 set_irq_affinity_info(irq, (int)(dest & 0xffff), redir);
257 unsigned int irq;
258
259 for (irq = 0; irq < NR_IRQS; ++irq)
260 if (irq_to_vector(irq) == vector) {
261 set_irq_affinity_info(irq,
262 (int)(dest & 0xffff),
263 redir);
264 break;
265 }
266 }
267#endif 262#endif
268 263
269 low32 = ((pol << IOSAPIC_POLARITY_SHIFT) | 264 low32 = ((pol << IOSAPIC_POLARITY_SHIFT) |
@@ -275,10 +270,10 @@ set_rte (unsigned int gsi, unsigned int vector, unsigned int dest, int mask)
275 /* dest contains both id and eid */ 270 /* dest contains both id and eid */
276 high32 = (dest << IOSAPIC_DEST_SHIFT); 271 high32 = (dest << IOSAPIC_DEST_SHIFT);
277 272
278 iosapic_write(addr, IOSAPIC_RTE_HIGH(rte_index), high32); 273 iosapic_write(rte->iosapic, IOSAPIC_RTE_HIGH(rte_index), high32);
279 iosapic_write(addr, IOSAPIC_RTE_LOW(rte_index), low32); 274 iosapic_write(rte->iosapic, IOSAPIC_RTE_LOW(rte_index), low32);
280 iosapic_intr_info[vector].low32 = low32; 275 iosapic_intr_info[irq].low32 = low32;
281 iosapic_intr_info[vector].dest = dest; 276 iosapic_intr_info[irq].dest = dest;
282} 277}
283 278
284static void 279static void
@@ -294,15 +289,18 @@ kexec_disable_iosapic(void)
294{ 289{
295 struct iosapic_intr_info *info; 290 struct iosapic_intr_info *info;
296 struct iosapic_rte_info *rte; 291 struct iosapic_rte_info *rte;
297 u8 vec = 0; 292 ia64_vector vec;
298 for (info = iosapic_intr_info; info < 293 int irq;
299 iosapic_intr_info + IA64_NUM_VECTORS; ++info, ++vec) { 294
295 for (irq = 0; irq < NR_IRQS; irq++) {
296 info = &iosapic_intr_info[irq];
297 vec = irq_to_vector(irq);
300 list_for_each_entry(rte, &info->rtes, 298 list_for_each_entry(rte, &info->rtes,
301 rte_list) { 299 rte_list) {
302 iosapic_write(rte->addr, 300 iosapic_write(rte->iosapic,
303 IOSAPIC_RTE_LOW(rte->rte_index), 301 IOSAPIC_RTE_LOW(rte->rte_index),
304 IOSAPIC_MASK|vec); 302 IOSAPIC_MASK|vec);
305 iosapic_eoi(rte->addr, vec); 303 iosapic_eoi(rte->iosapic->addr, vec);
306 } 304 }
307 } 305 }
308} 306}
@@ -311,54 +309,36 @@ kexec_disable_iosapic(void)
311static void 309static void
312mask_irq (unsigned int irq) 310mask_irq (unsigned int irq)
313{ 311{
314 unsigned long flags;
315 char __iomem *addr;
316 u32 low32; 312 u32 low32;
317 int rte_index; 313 int rte_index;
318 ia64_vector vec = irq_to_vector(irq);
319 struct iosapic_rte_info *rte; 314 struct iosapic_rte_info *rte;
320 315
321 if (list_empty(&iosapic_intr_info[vec].rtes)) 316 if (list_empty(&iosapic_intr_info[irq].rtes))
322 return; /* not an IOSAPIC interrupt! */ 317 return; /* not an IOSAPIC interrupt! */
323 318
324 spin_lock_irqsave(&iosapic_lock, flags); 319 /* set only the mask bit */
325 { 320 low32 = iosapic_intr_info[irq].low32 |= IOSAPIC_MASK;
326 /* set only the mask bit */ 321 list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list) {
327 low32 = iosapic_intr_info[vec].low32 |= IOSAPIC_MASK; 322 rte_index = rte->rte_index;
328 list_for_each_entry(rte, &iosapic_intr_info[vec].rtes, 323 iosapic_write(rte->iosapic, IOSAPIC_RTE_LOW(rte_index), low32);
329 rte_list) {
330 addr = rte->addr;
331 rte_index = rte->rte_index;
332 iosapic_write(addr, IOSAPIC_RTE_LOW(rte_index), low32);
333 }
334 } 324 }
335 spin_unlock_irqrestore(&iosapic_lock, flags);
336} 325}
337 326
338static void 327static void
339unmask_irq (unsigned int irq) 328unmask_irq (unsigned int irq)
340{ 329{
341 unsigned long flags;
342 char __iomem *addr;
343 u32 low32; 330 u32 low32;
344 int rte_index; 331 int rte_index;
345 ia64_vector vec = irq_to_vector(irq);
346 struct iosapic_rte_info *rte; 332 struct iosapic_rte_info *rte;
347 333
348 if (list_empty(&iosapic_intr_info[vec].rtes)) 334 if (list_empty(&iosapic_intr_info[irq].rtes))
349 return; /* not an IOSAPIC interrupt! */ 335 return; /* not an IOSAPIC interrupt! */
350 336
351 spin_lock_irqsave(&iosapic_lock, flags); 337 low32 = iosapic_intr_info[irq].low32 &= ~IOSAPIC_MASK;
352 { 338 list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list) {
353 low32 = iosapic_intr_info[vec].low32 &= ~IOSAPIC_MASK; 339 rte_index = rte->rte_index;
354 list_for_each_entry(rte, &iosapic_intr_info[vec].rtes, 340 iosapic_write(rte->iosapic, IOSAPIC_RTE_LOW(rte_index), low32);
355 rte_list) {
356 addr = rte->addr;
357 rte_index = rte->rte_index;
358 iosapic_write(addr, IOSAPIC_RTE_LOW(rte_index), low32);
359 }
360 } 341 }
361 spin_unlock_irqrestore(&iosapic_lock, flags);
362} 342}
363 343
364 344
@@ -366,23 +346,24 @@ static void
366iosapic_set_affinity (unsigned int irq, cpumask_t mask) 346iosapic_set_affinity (unsigned int irq, cpumask_t mask)
367{ 347{
368#ifdef CONFIG_SMP 348#ifdef CONFIG_SMP
369 unsigned long flags;
370 u32 high32, low32; 349 u32 high32, low32;
371 int dest, rte_index; 350 int dest, rte_index;
372 char __iomem *addr;
373 int redir = (irq & IA64_IRQ_REDIRECTED) ? 1 : 0; 351 int redir = (irq & IA64_IRQ_REDIRECTED) ? 1 : 0;
374 ia64_vector vec;
375 struct iosapic_rte_info *rte; 352 struct iosapic_rte_info *rte;
353 struct iosapic *iosapic;
376 354
377 irq &= (~IA64_IRQ_REDIRECTED); 355 irq &= (~IA64_IRQ_REDIRECTED);
378 vec = irq_to_vector(irq);
379 356
357 cpus_and(mask, mask, cpu_online_map);
380 if (cpus_empty(mask)) 358 if (cpus_empty(mask))
381 return; 359 return;
382 360
361 if (reassign_irq_vector(irq, first_cpu(mask)))
362 return;
363
383 dest = cpu_physical_id(first_cpu(mask)); 364 dest = cpu_physical_id(first_cpu(mask));
384 365
385 if (list_empty(&iosapic_intr_info[vec].rtes)) 366 if (list_empty(&iosapic_intr_info[irq].rtes))
386 return; /* not an IOSAPIC interrupt */ 367 return; /* not an IOSAPIC interrupt */
387 368
388 set_irq_affinity_info(irq, dest, redir); 369 set_irq_affinity_info(irq, dest, redir);
@@ -390,31 +371,24 @@ iosapic_set_affinity (unsigned int irq, cpumask_t mask)
390 /* dest contains both id and eid */ 371 /* dest contains both id and eid */
391 high32 = dest << IOSAPIC_DEST_SHIFT; 372 high32 = dest << IOSAPIC_DEST_SHIFT;
392 373
393 spin_lock_irqsave(&iosapic_lock, flags); 374 low32 = iosapic_intr_info[irq].low32 & ~(7 << IOSAPIC_DELIVERY_SHIFT);
394 { 375 if (redir)
395 low32 = iosapic_intr_info[vec].low32 & 376 /* change delivery mode to lowest priority */
396 ~(7 << IOSAPIC_DELIVERY_SHIFT); 377 low32 |= (IOSAPIC_LOWEST_PRIORITY << IOSAPIC_DELIVERY_SHIFT);
397 378 else
398 if (redir) 379 /* change delivery mode to fixed */
399 /* change delivery mode to lowest priority */ 380 low32 |= (IOSAPIC_FIXED << IOSAPIC_DELIVERY_SHIFT);
400 low32 |= (IOSAPIC_LOWEST_PRIORITY << 381 low32 &= IOSAPIC_VECTOR_MASK;
401 IOSAPIC_DELIVERY_SHIFT); 382 low32 |= irq_to_vector(irq);
402 else 383
403 /* change delivery mode to fixed */ 384 iosapic_intr_info[irq].low32 = low32;
404 low32 |= (IOSAPIC_FIXED << IOSAPIC_DELIVERY_SHIFT); 385 iosapic_intr_info[irq].dest = dest;
405 386 list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list) {
406 iosapic_intr_info[vec].low32 = low32; 387 iosapic = rte->iosapic;
407 iosapic_intr_info[vec].dest = dest; 388 rte_index = rte->rte_index;
408 list_for_each_entry(rte, &iosapic_intr_info[vec].rtes, 389 iosapic_write(iosapic, IOSAPIC_RTE_HIGH(rte_index), high32);
409 rte_list) { 390 iosapic_write(iosapic, IOSAPIC_RTE_LOW(rte_index), low32);
410 addr = rte->addr;
411 rte_index = rte->rte_index;
412 iosapic_write(addr, IOSAPIC_RTE_HIGH(rte_index),
413 high32);
414 iosapic_write(addr, IOSAPIC_RTE_LOW(rte_index), low32);
415 }
416 } 391 }
417 spin_unlock_irqrestore(&iosapic_lock, flags);
418#endif 392#endif
419} 393}
420 394
@@ -434,10 +408,20 @@ iosapic_end_level_irq (unsigned int irq)
434{ 408{
435 ia64_vector vec = irq_to_vector(irq); 409 ia64_vector vec = irq_to_vector(irq);
436 struct iosapic_rte_info *rte; 410 struct iosapic_rte_info *rte;
411 int do_unmask_irq = 0;
437 412
438 move_native_irq(irq); 413 if (unlikely(irq_desc[irq].status & IRQ_MOVE_PENDING)) {
439 list_for_each_entry(rte, &iosapic_intr_info[vec].rtes, rte_list) 414 do_unmask_irq = 1;
440 iosapic_eoi(rte->addr, vec); 415 mask_irq(irq);
416 }
417
418 list_for_each_entry(rte, &iosapic_intr_info[irq].rtes, rte_list)
419 iosapic_eoi(rte->iosapic->addr, vec);
420
421 if (unlikely(do_unmask_irq)) {
422 move_masked_irq(irq);
423 unmask_irq(irq);
424 }
441} 425}
442 426
443#define iosapic_shutdown_level_irq mask_irq 427#define iosapic_shutdown_level_irq mask_irq
@@ -519,13 +503,12 @@ iosapic_version (char __iomem *addr)
519 * unsigned int reserved2 : 8; 503 * unsigned int reserved2 : 8;
520 * } 504 * }
521 */ 505 */
522 return iosapic_read(addr, IOSAPIC_VERSION); 506 return __iosapic_read(addr, IOSAPIC_VERSION);
523} 507}
524 508
525static int iosapic_find_sharable_vector (unsigned long trigger, 509static int iosapic_find_sharable_irq(unsigned long trigger, unsigned long pol)
526 unsigned long pol)
527{ 510{
528 int i, vector = -1, min_count = -1; 511 int i, irq = -ENOSPC, min_count = -1;
529 struct iosapic_intr_info *info; 512 struct iosapic_intr_info *info;
530 513
531 /* 514 /*
@@ -533,21 +516,21 @@ static int iosapic_find_sharable_vector (unsigned long trigger,
533 * supported yet 516 * supported yet
534 */ 517 */
535 if (trigger == IOSAPIC_EDGE) 518 if (trigger == IOSAPIC_EDGE)
536 return -1; 519 return -EINVAL;
537 520
538 for (i = IA64_FIRST_DEVICE_VECTOR; i <= IA64_LAST_DEVICE_VECTOR; i++) { 521 for (i = 0; i <= NR_IRQS; i++) {
539 info = &iosapic_intr_info[i]; 522 info = &iosapic_intr_info[i];
540 if (info->trigger == trigger && info->polarity == pol && 523 if (info->trigger == trigger && info->polarity == pol &&
541 (info->dmode == IOSAPIC_FIXED || info->dmode == 524 (info->dmode == IOSAPIC_FIXED ||
542 IOSAPIC_LOWEST_PRIORITY)) { 525 info->dmode == IOSAPIC_LOWEST_PRIORITY) &&
526 can_request_irq(i, IRQF_SHARED)) {
543 if (min_count == -1 || info->count < min_count) { 527 if (min_count == -1 || info->count < min_count) {
544 vector = i; 528 irq = i;
545 min_count = info->count; 529 min_count = info->count;
546 } 530 }
547 } 531 }
548 } 532 }
549 533 return irq;
550 return vector;
551} 534}
552 535
553/* 536/*
@@ -555,25 +538,25 @@ static int iosapic_find_sharable_vector (unsigned long trigger,
555 * assign a new vector for the other and make the vector available 538 * assign a new vector for the other and make the vector available
556 */ 539 */
557static void __init 540static void __init
558iosapic_reassign_vector (int vector) 541iosapic_reassign_vector (int irq)
559{ 542{
560 int new_vector; 543 int new_irq;
561 544
562 if (!list_empty(&iosapic_intr_info[vector].rtes)) { 545 if (!list_empty(&iosapic_intr_info[irq].rtes)) {
563 new_vector = assign_irq_vector(AUTO_ASSIGN); 546 new_irq = create_irq();
564 if (new_vector < 0) 547 if (new_irq < 0)
565 panic("%s: out of interrupt vectors!\n", __FUNCTION__); 548 panic("%s: out of interrupt vectors!\n", __FUNCTION__);
566 printk(KERN_INFO "Reassigning vector %d to %d\n", 549 printk(KERN_INFO "Reassigning vector %d to %d\n",
567 vector, new_vector); 550 irq_to_vector(irq), irq_to_vector(new_irq));
568 memcpy(&iosapic_intr_info[new_vector], &iosapic_intr_info[vector], 551 memcpy(&iosapic_intr_info[new_irq], &iosapic_intr_info[irq],
569 sizeof(struct iosapic_intr_info)); 552 sizeof(struct iosapic_intr_info));
570 INIT_LIST_HEAD(&iosapic_intr_info[new_vector].rtes); 553 INIT_LIST_HEAD(&iosapic_intr_info[new_irq].rtes);
571 list_move(iosapic_intr_info[vector].rtes.next, 554 list_move(iosapic_intr_info[irq].rtes.next,
572 &iosapic_intr_info[new_vector].rtes); 555 &iosapic_intr_info[new_irq].rtes);
573 memset(&iosapic_intr_info[vector], 0, 556 memset(&iosapic_intr_info[irq], 0,
574 sizeof(struct iosapic_intr_info)); 557 sizeof(struct iosapic_intr_info));
575 iosapic_intr_info[vector].low32 = IOSAPIC_MASK; 558 iosapic_intr_info[irq].low32 = IOSAPIC_MASK;
576 INIT_LIST_HEAD(&iosapic_intr_info[vector].rtes); 559 INIT_LIST_HEAD(&iosapic_intr_info[irq].rtes);
577 } 560 }
578} 561}
579 562
@@ -610,29 +593,18 @@ static struct iosapic_rte_info *iosapic_alloc_rte (void)
610 return rte; 593 return rte;
611} 594}
612 595
613static void iosapic_free_rte (struct iosapic_rte_info *rte) 596static inline int irq_is_shared (int irq)
614{ 597{
615 if (rte->flags & RTE_PREALLOCATED) 598 return (iosapic_intr_info[irq].count > 1);
616 list_add_tail(&rte->rte_list, &free_rte_list);
617 else
618 kfree(rte);
619}
620
621static inline int vector_is_shared (int vector)
622{
623 return (iosapic_intr_info[vector].count > 1);
624} 599}
625 600
626static int 601static int
627register_intr (unsigned int gsi, int vector, unsigned char delivery, 602register_intr (unsigned int gsi, int irq, unsigned char delivery,
628 unsigned long polarity, unsigned long trigger) 603 unsigned long polarity, unsigned long trigger)
629{ 604{
630 irq_desc_t *idesc; 605 irq_desc_t *idesc;
631 struct hw_interrupt_type *irq_type; 606 struct hw_interrupt_type *irq_type;
632 int rte_index;
633 int index; 607 int index;
634 unsigned long gsi_base;
635 void __iomem *iosapic_address;
636 struct iosapic_rte_info *rte; 608 struct iosapic_rte_info *rte;
637 609
638 index = find_iosapic(gsi); 610 index = find_iosapic(gsi);
@@ -642,10 +614,7 @@ register_intr (unsigned int gsi, int vector, unsigned char delivery,
642 return -ENODEV; 614 return -ENODEV;
643 } 615 }
644 616
645 iosapic_address = iosapic_lists[index].addr; 617 rte = find_rte(irq, gsi);
646 gsi_base = iosapic_lists[index].gsi_base;
647
648 rte = gsi_vector_to_rte(gsi, vector);
649 if (!rte) { 618 if (!rte) {
650 rte = iosapic_alloc_rte(); 619 rte = iosapic_alloc_rte();
651 if (!rte) { 620 if (!rte) {
@@ -654,40 +623,42 @@ register_intr (unsigned int gsi, int vector, unsigned char delivery,
654 return -ENOMEM; 623 return -ENOMEM;
655 } 624 }
656 625
657 rte_index = gsi - gsi_base; 626 rte->iosapic = &iosapic_lists[index];
658 rte->rte_index = rte_index; 627 rte->rte_index = gsi - rte->iosapic->gsi_base;
659 rte->addr = iosapic_address;
660 rte->gsi_base = gsi_base;
661 rte->refcnt++; 628 rte->refcnt++;
662 list_add_tail(&rte->rte_list, &iosapic_intr_info[vector].rtes); 629 list_add_tail(&rte->rte_list, &iosapic_intr_info[irq].rtes);
663 iosapic_intr_info[vector].count++; 630 iosapic_intr_info[irq].count++;
664 iosapic_lists[index].rtes_inuse++; 631 iosapic_lists[index].rtes_inuse++;
665 } 632 }
666 else if (vector_is_shared(vector)) { 633 else if (rte->refcnt == NO_REF_RTE) {
667 struct iosapic_intr_info *info = &iosapic_intr_info[vector]; 634 struct iosapic_intr_info *info = &iosapic_intr_info[irq];
668 if (info->trigger != trigger || info->polarity != polarity) { 635 if (info->count > 0 &&
636 (info->trigger != trigger || info->polarity != polarity)){
669 printk (KERN_WARNING 637 printk (KERN_WARNING
670 "%s: cannot override the interrupt\n", 638 "%s: cannot override the interrupt\n",
671 __FUNCTION__); 639 __FUNCTION__);
672 return -EINVAL; 640 return -EINVAL;
673 } 641 }
642 rte->refcnt++;
643 iosapic_intr_info[irq].count++;
644 iosapic_lists[index].rtes_inuse++;
674 } 645 }
675 646
676 iosapic_intr_info[vector].polarity = polarity; 647 iosapic_intr_info[irq].polarity = polarity;
677 iosapic_intr_info[vector].dmode = delivery; 648 iosapic_intr_info[irq].dmode = delivery;
678 iosapic_intr_info[vector].trigger = trigger; 649 iosapic_intr_info[irq].trigger = trigger;
679 650
680 if (trigger == IOSAPIC_EDGE) 651 if (trigger == IOSAPIC_EDGE)
681 irq_type = &irq_type_iosapic_edge; 652 irq_type = &irq_type_iosapic_edge;
682 else 653 else
683 irq_type = &irq_type_iosapic_level; 654 irq_type = &irq_type_iosapic_level;
684 655
685 idesc = irq_desc + vector; 656 idesc = irq_desc + irq;
686 if (idesc->chip != irq_type) { 657 if (idesc->chip != irq_type) {
687 if (idesc->chip != &no_irq_type) 658 if (idesc->chip != &no_irq_type)
688 printk(KERN_WARNING 659 printk(KERN_WARNING
689 "%s: changing vector %d from %s to %s\n", 660 "%s: changing vector %d from %s to %s\n",
690 __FUNCTION__, vector, 661 __FUNCTION__, irq_to_vector(irq),
691 idesc->chip->name, irq_type->name); 662 idesc->chip->name, irq_type->name);
692 idesc->chip = irq_type; 663 idesc->chip = irq_type;
693 } 664 }
@@ -695,18 +666,19 @@ register_intr (unsigned int gsi, int vector, unsigned char delivery,
695} 666}
696 667
697static unsigned int 668static unsigned int
698get_target_cpu (unsigned int gsi, int vector) 669get_target_cpu (unsigned int gsi, int irq)
699{ 670{
700#ifdef CONFIG_SMP 671#ifdef CONFIG_SMP
701 static int cpu = -1; 672 static int cpu = -1;
702 extern int cpe_vector; 673 extern int cpe_vector;
674 cpumask_t domain = irq_to_domain(irq);
703 675
704 /* 676 /*
705 * In case of vector shared by multiple RTEs, all RTEs that 677 * In case of vector shared by multiple RTEs, all RTEs that
706 * share the vector need to use the same destination CPU. 678 * share the vector need to use the same destination CPU.
707 */ 679 */
708 if (!list_empty(&iosapic_intr_info[vector].rtes)) 680 if (!list_empty(&iosapic_intr_info[irq].rtes))
709 return iosapic_intr_info[vector].dest; 681 return iosapic_intr_info[irq].dest;
710 682
711 /* 683 /*
712 * If the platform supports redirection via XTP, let it 684 * If the platform supports redirection via XTP, let it
@@ -723,7 +695,7 @@ get_target_cpu (unsigned int gsi, int vector)
723 return cpu_physical_id(smp_processor_id()); 695 return cpu_physical_id(smp_processor_id());
724 696
725#ifdef CONFIG_ACPI 697#ifdef CONFIG_ACPI
726 if (cpe_vector > 0 && vector == IA64_CPEP_VECTOR) 698 if (cpe_vector > 0 && irq_to_vector(irq) == IA64_CPEP_VECTOR)
727 return get_cpei_target_cpu(); 699 return get_cpei_target_cpu();
728#endif 700#endif
729 701
@@ -738,7 +710,7 @@ get_target_cpu (unsigned int gsi, int vector)
738 goto skip_numa_setup; 710 goto skip_numa_setup;
739 711
740 cpu_mask = node_to_cpumask(iosapic_lists[iosapic_index].node); 712 cpu_mask = node_to_cpumask(iosapic_lists[iosapic_index].node);
741 713 cpus_and(cpu_mask, cpu_mask, domain);
742 for_each_cpu_mask(numa_cpu, cpu_mask) { 714 for_each_cpu_mask(numa_cpu, cpu_mask) {
743 if (!cpu_online(numa_cpu)) 715 if (!cpu_online(numa_cpu))
744 cpu_clear(numa_cpu, cpu_mask); 716 cpu_clear(numa_cpu, cpu_mask);
@@ -749,8 +721,8 @@ get_target_cpu (unsigned int gsi, int vector)
749 if (!num_cpus) 721 if (!num_cpus)
750 goto skip_numa_setup; 722 goto skip_numa_setup;
751 723
752 /* Use vector assignment to distribute across cpus in node */ 724 /* Use irq assignment to distribute across cpus in node */
753 cpu_index = vector % num_cpus; 725 cpu_index = irq % num_cpus;
754 726
755 for (numa_cpu = first_cpu(cpu_mask) ; i < cpu_index ; i++) 727 for (numa_cpu = first_cpu(cpu_mask) ; i < cpu_index ; i++)
756 numa_cpu = next_cpu(numa_cpu, cpu_mask); 728 numa_cpu = next_cpu(numa_cpu, cpu_mask);
@@ -768,7 +740,7 @@ skip_numa_setup:
768 do { 740 do {
769 if (++cpu >= NR_CPUS) 741 if (++cpu >= NR_CPUS)
770 cpu = 0; 742 cpu = 0;
771 } while (!cpu_online(cpu)); 743 } while (!cpu_online(cpu) || !cpu_isset(cpu, domain));
772 744
773 return cpu_physical_id(cpu); 745 return cpu_physical_id(cpu);
774#else /* CONFIG_SMP */ 746#else /* CONFIG_SMP */
@@ -785,84 +757,72 @@ int
785iosapic_register_intr (unsigned int gsi, 757iosapic_register_intr (unsigned int gsi,
786 unsigned long polarity, unsigned long trigger) 758 unsigned long polarity, unsigned long trigger)
787{ 759{
788 int vector, mask = 1, err; 760 int irq, mask = 1, err;
789 unsigned int dest; 761 unsigned int dest;
790 unsigned long flags; 762 unsigned long flags;
791 struct iosapic_rte_info *rte; 763 struct iosapic_rte_info *rte;
792 u32 low32; 764 u32 low32;
793again: 765
794 /* 766 /*
795 * If this GSI has already been registered (i.e., it's a 767 * If this GSI has already been registered (i.e., it's a
796 * shared interrupt, or we lost a race to register it), 768 * shared interrupt, or we lost a race to register it),
797 * don't touch the RTE. 769 * don't touch the RTE.
798 */ 770 */
799 spin_lock_irqsave(&iosapic_lock, flags); 771 spin_lock_irqsave(&iosapic_lock, flags);
800 { 772 irq = __gsi_to_irq(gsi);
801 vector = gsi_to_vector(gsi); 773 if (irq > 0) {
802 if (vector > 0) { 774 rte = find_rte(irq, gsi);
803 rte = gsi_vector_to_rte(gsi, vector); 775 if(iosapic_intr_info[irq].count == 0) {
776 assign_irq_vector(irq);
777 dynamic_irq_init(irq);
778 } else if (rte->refcnt != NO_REF_RTE) {
804 rte->refcnt++; 779 rte->refcnt++;
805 spin_unlock_irqrestore(&iosapic_lock, flags); 780 goto unlock_iosapic_lock;
806 return vector;
807 } 781 }
808 } 782 } else
809 spin_unlock_irqrestore(&iosapic_lock, flags); 783 irq = create_irq();
810 784
811 /* If vector is running out, we try to find a sharable vector */ 785 /* If vector is running out, we try to find a sharable vector */
812 vector = assign_irq_vector(AUTO_ASSIGN); 786 if (irq < 0) {
813 if (vector < 0) { 787 irq = iosapic_find_sharable_irq(trigger, polarity);
814 vector = iosapic_find_sharable_vector(trigger, polarity); 788 if (irq < 0)
815 if (vector < 0) 789 goto unlock_iosapic_lock;
816 return -ENOSPC;
817 } 790 }
818 791
819 spin_lock_irqsave(&irq_desc[vector].lock, flags); 792 spin_lock(&irq_desc[irq].lock);
820 spin_lock(&iosapic_lock); 793 dest = get_target_cpu(gsi, irq);
821 { 794 err = register_intr(gsi, irq, IOSAPIC_LOWEST_PRIORITY,
822 if (gsi_to_vector(gsi) > 0) { 795 polarity, trigger);
823 if (list_empty(&iosapic_intr_info[vector].rtes)) 796 if (err < 0) {
824 free_irq_vector(vector); 797 irq = err;
825 spin_unlock(&iosapic_lock); 798 goto unlock_all;
826 spin_unlock_irqrestore(&irq_desc[vector].lock,
827 flags);
828 goto again;
829 }
830
831 dest = get_target_cpu(gsi, vector);
832 err = register_intr(gsi, vector, IOSAPIC_LOWEST_PRIORITY,
833 polarity, trigger);
834 if (err < 0) {
835 spin_unlock(&iosapic_lock);
836 spin_unlock_irqrestore(&irq_desc[vector].lock,
837 flags);
838 return err;
839 }
840
841 /*
842 * If the vector is shared and already unmasked for
843 * other interrupt sources, don't mask it.
844 */
845 low32 = iosapic_intr_info[vector].low32;
846 if (vector_is_shared(vector) && !(low32 & IOSAPIC_MASK))
847 mask = 0;
848 set_rte(gsi, vector, dest, mask);
849 } 799 }
850 spin_unlock(&iosapic_lock); 800
851 spin_unlock_irqrestore(&irq_desc[vector].lock, flags); 801 /*
802 * If the vector is shared and already unmasked for other
803 * interrupt sources, don't mask it.
804 */
805 low32 = iosapic_intr_info[irq].low32;
806 if (irq_is_shared(irq) && !(low32 & IOSAPIC_MASK))
807 mask = 0;
808 set_rte(gsi, irq, dest, mask);
852 809
853 printk(KERN_INFO "GSI %u (%s, %s) -> CPU %d (0x%04x) vector %d\n", 810 printk(KERN_INFO "GSI %u (%s, %s) -> CPU %d (0x%04x) vector %d\n",
854 gsi, (trigger == IOSAPIC_EDGE ? "edge" : "level"), 811 gsi, (trigger == IOSAPIC_EDGE ? "edge" : "level"),
855 (polarity == IOSAPIC_POL_HIGH ? "high" : "low"), 812 (polarity == IOSAPIC_POL_HIGH ? "high" : "low"),
856 cpu_logical_id(dest), dest, vector); 813 cpu_logical_id(dest), dest, irq_to_vector(irq));
857 814 unlock_all:
858 return vector; 815 spin_unlock(&irq_desc[irq].lock);
816 unlock_iosapic_lock:
817 spin_unlock_irqrestore(&iosapic_lock, flags);
818 return irq;
859} 819}
860 820
861void 821void
862iosapic_unregister_intr (unsigned int gsi) 822iosapic_unregister_intr (unsigned int gsi)
863{ 823{
864 unsigned long flags; 824 unsigned long flags;
865 int irq, vector, index; 825 int irq, index;
866 irq_desc_t *idesc; 826 irq_desc_t *idesc;
867 u32 low32; 827 u32 low32;
868 unsigned long trigger, polarity; 828 unsigned long trigger, polarity;
@@ -881,78 +841,56 @@ iosapic_unregister_intr (unsigned int gsi)
881 WARN_ON(1); 841 WARN_ON(1);
882 return; 842 return;
883 } 843 }
884 vector = irq_to_vector(irq);
885 844
886 idesc = irq_desc + irq; 845 spin_lock_irqsave(&iosapic_lock, flags);
887 spin_lock_irqsave(&idesc->lock, flags); 846 if ((rte = find_rte(irq, gsi)) == NULL) {
888 spin_lock(&iosapic_lock); 847 printk(KERN_ERR "iosapic_unregister_intr(%u) unbalanced\n",
889 { 848 gsi);
890 if ((rte = gsi_vector_to_rte(gsi, vector)) == NULL) { 849 WARN_ON(1);
891 printk(KERN_ERR 850 goto out;
892 "iosapic_unregister_intr(%u) unbalanced\n", 851 }
893 gsi);
894 WARN_ON(1);
895 goto out;
896 }
897 852
898 if (--rte->refcnt > 0) 853 if (--rte->refcnt > 0)
899 goto out; 854 goto out;
900 855
901 /* Mask the interrupt */ 856 idesc = irq_desc + irq;
902 low32 = iosapic_intr_info[vector].low32 | IOSAPIC_MASK; 857 rte->refcnt = NO_REF_RTE;
903 iosapic_write(rte->addr, IOSAPIC_RTE_LOW(rte->rte_index),
904 low32);
905 858
906 /* Remove the rte entry from the list */ 859 /* Mask the interrupt */
907 list_del(&rte->rte_list); 860 low32 = iosapic_intr_info[irq].low32 | IOSAPIC_MASK;
908 iosapic_intr_info[vector].count--; 861 iosapic_write(rte->iosapic, IOSAPIC_RTE_LOW(rte->rte_index), low32);
909 iosapic_free_rte(rte);
910 index = find_iosapic(gsi);
911 iosapic_lists[index].rtes_inuse--;
912 WARN_ON(iosapic_lists[index].rtes_inuse < 0);
913
914 trigger = iosapic_intr_info[vector].trigger;
915 polarity = iosapic_intr_info[vector].polarity;
916 dest = iosapic_intr_info[vector].dest;
917 printk(KERN_INFO
918 "GSI %u (%s, %s) -> CPU %d (0x%04x)"
919 " vector %d unregistered\n",
920 gsi, (trigger == IOSAPIC_EDGE ? "edge" : "level"),
921 (polarity == IOSAPIC_POL_HIGH ? "high" : "low"),
922 cpu_logical_id(dest), dest, vector);
923 862
924 if (list_empty(&iosapic_intr_info[vector].rtes)) { 863 iosapic_intr_info[irq].count--;
925 /* Sanity check */ 864 index = find_iosapic(gsi);
926 BUG_ON(iosapic_intr_info[vector].count); 865 iosapic_lists[index].rtes_inuse--;
866 WARN_ON(iosapic_lists[index].rtes_inuse < 0);
927 867
928 /* Clear the interrupt controller descriptor */ 868 trigger = iosapic_intr_info[irq].trigger;
929 idesc->chip = &no_irq_type; 869 polarity = iosapic_intr_info[irq].polarity;
870 dest = iosapic_intr_info[irq].dest;
871 printk(KERN_INFO
872 "GSI %u (%s, %s) -> CPU %d (0x%04x) vector %d unregistered\n",
873 gsi, (trigger == IOSAPIC_EDGE ? "edge" : "level"),
874 (polarity == IOSAPIC_POL_HIGH ? "high" : "low"),
875 cpu_logical_id(dest), dest, irq_to_vector(irq));
930 876
877 if (iosapic_intr_info[irq].count == 0) {
931#ifdef CONFIG_SMP 878#ifdef CONFIG_SMP
932 /* Clear affinity */ 879 /* Clear affinity */
933 cpus_setall(idesc->affinity); 880 cpus_setall(idesc->affinity);
934#endif 881#endif
935 882 /* Clear the interrupt information */
936 /* Clear the interrupt information */ 883 iosapic_intr_info[irq].dest = 0;
937 memset(&iosapic_intr_info[vector], 0, 884 iosapic_intr_info[irq].dmode = 0;
938 sizeof(struct iosapic_intr_info)); 885 iosapic_intr_info[irq].polarity = 0;
939 iosapic_intr_info[vector].low32 |= IOSAPIC_MASK; 886 iosapic_intr_info[irq].trigger = 0;
940 INIT_LIST_HEAD(&iosapic_intr_info[vector].rtes); 887 iosapic_intr_info[irq].low32 |= IOSAPIC_MASK;
941 888
942 if (idesc->action) { 889 /* Destroy and reserve IRQ */
943 printk(KERN_ERR 890 destroy_and_reserve_irq(irq);
944 "interrupt handlers still exist on"
945 "IRQ %u\n", irq);
946 WARN_ON(1);
947 }
948
949 /* Free the interrupt vector */
950 free_irq_vector(vector);
951 }
952 } 891 }
953 out: 892 out:
954 spin_unlock(&iosapic_lock); 893 spin_unlock_irqrestore(&iosapic_lock, flags);
955 spin_unlock_irqrestore(&idesc->lock, flags);
956} 894}
957 895
958/* 896/*
@@ -965,27 +903,30 @@ iosapic_register_platform_intr (u32 int_type, unsigned int gsi,
965{ 903{
966 static const char * const name[] = {"unknown", "PMI", "INIT", "CPEI"}; 904 static const char * const name[] = {"unknown", "PMI", "INIT", "CPEI"};
967 unsigned char delivery; 905 unsigned char delivery;
968 int vector, mask = 0; 906 int irq, vector, mask = 0;
969 unsigned int dest = ((id << 8) | eid) & 0xffff; 907 unsigned int dest = ((id << 8) | eid) & 0xffff;
970 908
971 switch (int_type) { 909 switch (int_type) {
972 case ACPI_INTERRUPT_PMI: 910 case ACPI_INTERRUPT_PMI:
973 vector = iosapic_vector; 911 irq = vector = iosapic_vector;
912 bind_irq_vector(irq, vector, CPU_MASK_ALL);
974 /* 913 /*
975 * since PMI vector is alloc'd by FW(ACPI) not by kernel, 914 * since PMI vector is alloc'd by FW(ACPI) not by kernel,
976 * we need to make sure the vector is available 915 * we need to make sure the vector is available
977 */ 916 */
978 iosapic_reassign_vector(vector); 917 iosapic_reassign_vector(irq);
979 delivery = IOSAPIC_PMI; 918 delivery = IOSAPIC_PMI;
980 break; 919 break;
981 case ACPI_INTERRUPT_INIT: 920 case ACPI_INTERRUPT_INIT:
982 vector = assign_irq_vector(AUTO_ASSIGN); 921 irq = create_irq();
983 if (vector < 0) 922 if (irq < 0)
984 panic("%s: out of interrupt vectors!\n", __FUNCTION__); 923 panic("%s: out of interrupt vectors!\n", __FUNCTION__);
924 vector = irq_to_vector(irq);
985 delivery = IOSAPIC_INIT; 925 delivery = IOSAPIC_INIT;
986 break; 926 break;
987 case ACPI_INTERRUPT_CPEI: 927 case ACPI_INTERRUPT_CPEI:
988 vector = IA64_CPE_VECTOR; 928 irq = vector = IA64_CPE_VECTOR;
929 BUG_ON(bind_irq_vector(irq, vector, CPU_MASK_ALL));
989 delivery = IOSAPIC_LOWEST_PRIORITY; 930 delivery = IOSAPIC_LOWEST_PRIORITY;
990 mask = 1; 931 mask = 1;
991 break; 932 break;
@@ -995,7 +936,7 @@ iosapic_register_platform_intr (u32 int_type, unsigned int gsi,
995 return -1; 936 return -1;
996 } 937 }
997 938
998 register_intr(gsi, vector, delivery, polarity, trigger); 939 register_intr(gsi, irq, delivery, polarity, trigger);
999 940
1000 printk(KERN_INFO 941 printk(KERN_INFO
1001 "PLATFORM int %s (0x%x): GSI %u (%s, %s) -> CPU %d (0x%04x)" 942 "PLATFORM int %s (0x%x): GSI %u (%s, %s) -> CPU %d (0x%04x)"
@@ -1005,7 +946,7 @@ iosapic_register_platform_intr (u32 int_type, unsigned int gsi,
1005 (polarity == IOSAPIC_POL_HIGH ? "high" : "low"), 946 (polarity == IOSAPIC_POL_HIGH ? "high" : "low"),
1006 cpu_logical_id(dest), dest, vector); 947 cpu_logical_id(dest), dest, vector);
1007 948
1008 set_rte(gsi, vector, dest, mask); 949 set_rte(gsi, irq, dest, mask);
1009 return vector; 950 return vector;
1010} 951}
1011 952
@@ -1017,30 +958,32 @@ iosapic_override_isa_irq (unsigned int isa_irq, unsigned int gsi,
1017 unsigned long polarity, 958 unsigned long polarity,
1018 unsigned long trigger) 959 unsigned long trigger)
1019{ 960{
1020 int vector; 961 int vector, irq;
1021 unsigned int dest = cpu_physical_id(smp_processor_id()); 962 unsigned int dest = cpu_physical_id(smp_processor_id());
1022 963
1023 vector = isa_irq_to_vector(isa_irq); 964 irq = vector = isa_irq_to_vector(isa_irq);
1024 965 BUG_ON(bind_irq_vector(irq, vector, CPU_MASK_ALL));
1025 register_intr(gsi, vector, IOSAPIC_LOWEST_PRIORITY, polarity, trigger); 966 register_intr(gsi, irq, IOSAPIC_LOWEST_PRIORITY, polarity, trigger);
1026 967
1027 DBG("ISA: IRQ %u -> GSI %u (%s,%s) -> CPU %d (0x%04x) vector %d\n", 968 DBG("ISA: IRQ %u -> GSI %u (%s,%s) -> CPU %d (0x%04x) vector %d\n",
1028 isa_irq, gsi, trigger == IOSAPIC_EDGE ? "edge" : "level", 969 isa_irq, gsi, trigger == IOSAPIC_EDGE ? "edge" : "level",
1029 polarity == IOSAPIC_POL_HIGH ? "high" : "low", 970 polarity == IOSAPIC_POL_HIGH ? "high" : "low",
1030 cpu_logical_id(dest), dest, vector); 971 cpu_logical_id(dest), dest, vector);
1031 972
1032 set_rte(gsi, vector, dest, 1); 973 set_rte(gsi, irq, dest, 1);
1033} 974}
1034 975
1035void __init 976void __init
1036iosapic_system_init (int system_pcat_compat) 977iosapic_system_init (int system_pcat_compat)
1037{ 978{
1038 int vector; 979 int irq;
1039 980
1040 for (vector = 0; vector < IA64_NUM_VECTORS; ++vector) { 981 for (irq = 0; irq < NR_IRQS; ++irq) {
1041 iosapic_intr_info[vector].low32 = IOSAPIC_MASK; 982 iosapic_intr_info[irq].low32 = IOSAPIC_MASK;
1042 /* mark as unused */ 983 /* mark as unused */
1043 INIT_LIST_HEAD(&iosapic_intr_info[vector].rtes); 984 INIT_LIST_HEAD(&iosapic_intr_info[irq].rtes);
985
986 iosapic_intr_info[irq].count = 0;
1044 } 987 }
1045 988
1046 pcat_compat = system_pcat_compat; 989 pcat_compat = system_pcat_compat;
@@ -1108,31 +1051,35 @@ iosapic_init (unsigned long phys_addr, unsigned int gsi_base)
1108 unsigned long flags; 1051 unsigned long flags;
1109 1052
1110 spin_lock_irqsave(&iosapic_lock, flags); 1053 spin_lock_irqsave(&iosapic_lock, flags);
1111 { 1054 index = find_iosapic(gsi_base);
1112 addr = ioremap(phys_addr, 0); 1055 if (index >= 0) {
1113 ver = iosapic_version(addr); 1056 spin_unlock_irqrestore(&iosapic_lock, flags);
1057 return -EBUSY;
1058 }
1114 1059
1115 if ((err = iosapic_check_gsi_range(gsi_base, ver))) { 1060 addr = ioremap(phys_addr, 0);
1116 iounmap(addr); 1061 ver = iosapic_version(addr);
1117 spin_unlock_irqrestore(&iosapic_lock, flags); 1062 if ((err = iosapic_check_gsi_range(gsi_base, ver))) {
1118 return err; 1063 iounmap(addr);
1119 } 1064 spin_unlock_irqrestore(&iosapic_lock, flags);
1065 return err;
1066 }
1120 1067
1121 /* 1068 /*
1122 * The MAX_REDIR register holds the highest input pin 1069 * The MAX_REDIR register holds the highest input pin number
1123 * number (starting from 0). 1070 * (starting from 0). We add 1 so that we can use it for
1124 * We add 1 so that we can use it for number of pins (= RTEs) 1071 * number of pins (= RTEs)
1125 */ 1072 */
1126 num_rte = ((ver >> 16) & 0xff) + 1; 1073 num_rte = ((ver >> 16) & 0xff) + 1;
1127 1074
1128 index = iosapic_alloc(); 1075 index = iosapic_alloc();
1129 iosapic_lists[index].addr = addr; 1076 iosapic_lists[index].addr = addr;
1130 iosapic_lists[index].gsi_base = gsi_base; 1077 iosapic_lists[index].gsi_base = gsi_base;
1131 iosapic_lists[index].num_rte = num_rte; 1078 iosapic_lists[index].num_rte = num_rte;
1132#ifdef CONFIG_NUMA 1079#ifdef CONFIG_NUMA
1133 iosapic_lists[index].node = MAX_NUMNODES; 1080 iosapic_lists[index].node = MAX_NUMNODES;
1134#endif 1081#endif
1135 } 1082 spin_lock_init(&iosapic_lists[index].lock);
1136 spin_unlock_irqrestore(&iosapic_lock, flags); 1083 spin_unlock_irqrestore(&iosapic_lock, flags);
1137 1084
1138 if ((gsi_base == 0) && pcat_compat) { 1085 if ((gsi_base == 0) && pcat_compat) {
@@ -1157,25 +1104,22 @@ iosapic_remove (unsigned int gsi_base)
1157 unsigned long flags; 1104 unsigned long flags;
1158 1105
1159 spin_lock_irqsave(&iosapic_lock, flags); 1106 spin_lock_irqsave(&iosapic_lock, flags);
1160 { 1107 index = find_iosapic(gsi_base);
1161 index = find_iosapic(gsi_base); 1108 if (index < 0) {
1162 if (index < 0) { 1109 printk(KERN_WARNING "%s: No IOSAPIC for GSI base %u\n",
1163 printk(KERN_WARNING "%s: No IOSAPIC for GSI base %u\n", 1110 __FUNCTION__, gsi_base);
1164 __FUNCTION__, gsi_base); 1111 goto out;
1165 goto out; 1112 }
1166 }
1167
1168 if (iosapic_lists[index].rtes_inuse) {
1169 err = -EBUSY;
1170 printk(KERN_WARNING
1171 "%s: IOSAPIC for GSI base %u is busy\n",
1172 __FUNCTION__, gsi_base);
1173 goto out;
1174 }
1175 1113
1176 iounmap(iosapic_lists[index].addr); 1114 if (iosapic_lists[index].rtes_inuse) {
1177 iosapic_free(index); 1115 err = -EBUSY;
1116 printk(KERN_WARNING "%s: IOSAPIC for GSI base %u is busy\n",
1117 __FUNCTION__, gsi_base);
1118 goto out;
1178 } 1119 }
1120
1121 iounmap(iosapic_lists[index].addr);
1122 iosapic_free(index);
1179 out: 1123 out:
1180 spin_unlock_irqrestore(&iosapic_lock, flags); 1124 spin_unlock_irqrestore(&iosapic_lock, flags);
1181 return err; 1125 return err;
diff --git a/arch/ia64/kernel/irq.c b/arch/ia64/kernel/irq.c
index 407b45870489..cc3ee4ef37af 100644
--- a/arch/ia64/kernel/irq.c
+++ b/arch/ia64/kernel/irq.c
@@ -35,7 +35,7 @@ void ack_bad_irq(unsigned int irq)
35#ifdef CONFIG_IA64_GENERIC 35#ifdef CONFIG_IA64_GENERIC
36unsigned int __ia64_local_vector_to_irq (ia64_vector vec) 36unsigned int __ia64_local_vector_to_irq (ia64_vector vec)
37{ 37{
38 return (unsigned int) vec; 38 return __get_cpu_var(vector_irq)[vec];
39} 39}
40#endif 40#endif
41 41
diff --git a/arch/ia64/kernel/irq_ia64.c b/arch/ia64/kernel/irq_ia64.c
index bc47049f060f..91797c111162 100644
--- a/arch/ia64/kernel/irq_ia64.c
+++ b/arch/ia64/kernel/irq_ia64.c
@@ -46,6 +46,12 @@
46 46
47#define IRQ_DEBUG 0 47#define IRQ_DEBUG 0
48 48
49#define IRQ_VECTOR_UNASSIGNED (0)
50
51#define IRQ_UNUSED (0)
52#define IRQ_USED (1)
53#define IRQ_RSVD (2)
54
49/* These can be overridden in platform_irq_init */ 55/* These can be overridden in platform_irq_init */
50int ia64_first_device_vector = IA64_DEF_FIRST_DEVICE_VECTOR; 56int ia64_first_device_vector = IA64_DEF_FIRST_DEVICE_VECTOR;
51int ia64_last_device_vector = IA64_DEF_LAST_DEVICE_VECTOR; 57int ia64_last_device_vector = IA64_DEF_LAST_DEVICE_VECTOR;
@@ -54,6 +60,8 @@ int ia64_last_device_vector = IA64_DEF_LAST_DEVICE_VECTOR;
54void __iomem *ipi_base_addr = ((void __iomem *) 60void __iomem *ipi_base_addr = ((void __iomem *)
55 (__IA64_UNCACHED_OFFSET | IA64_IPI_DEFAULT_BASE_ADDR)); 61 (__IA64_UNCACHED_OFFSET | IA64_IPI_DEFAULT_BASE_ADDR));
56 62
63static cpumask_t vector_allocation_domain(int cpu);
64
57/* 65/*
58 * Legacy IRQ to IA-64 vector translation table. 66 * Legacy IRQ to IA-64 vector translation table.
59 */ 67 */
@@ -64,46 +72,269 @@ __u8 isa_irq_to_vector_map[16] = {
64}; 72};
65EXPORT_SYMBOL(isa_irq_to_vector_map); 73EXPORT_SYMBOL(isa_irq_to_vector_map);
66 74
67static unsigned long ia64_vector_mask[BITS_TO_LONGS(IA64_MAX_DEVICE_VECTORS)]; 75DEFINE_SPINLOCK(vector_lock);
76
77struct irq_cfg irq_cfg[NR_IRQS] __read_mostly = {
78 [0 ... NR_IRQS - 1] = {
79 .vector = IRQ_VECTOR_UNASSIGNED,
80 .domain = CPU_MASK_NONE
81 }
82};
83
84DEFINE_PER_CPU(int[IA64_NUM_VECTORS], vector_irq) = {
85 [0 ... IA64_NUM_VECTORS - 1] = IA64_SPURIOUS_INT_VECTOR
86};
87
88static cpumask_t vector_table[IA64_MAX_DEVICE_VECTORS] = {
89 [0 ... IA64_MAX_DEVICE_VECTORS - 1] = CPU_MASK_NONE
90};
91
92static int irq_status[NR_IRQS] = {
93 [0 ... NR_IRQS -1] = IRQ_UNUSED
94};
95
96int check_irq_used(int irq)
97{
98 if (irq_status[irq] == IRQ_USED)
99 return 1;
100
101 return -1;
102}
103
104static void reserve_irq(unsigned int irq)
105{
106 unsigned long flags;
107
108 spin_lock_irqsave(&vector_lock, flags);
109 irq_status[irq] = IRQ_RSVD;
110 spin_unlock_irqrestore(&vector_lock, flags);
111}
112
113static inline int find_unassigned_irq(void)
114{
115 int irq;
116
117 for (irq = IA64_FIRST_DEVICE_VECTOR; irq < NR_IRQS; irq++)
118 if (irq_status[irq] == IRQ_UNUSED)
119 return irq;
120 return -ENOSPC;
121}
122
123static inline int find_unassigned_vector(cpumask_t domain)
124{
125 cpumask_t mask;
126 int pos;
127
128 cpus_and(mask, domain, cpu_online_map);
129 if (cpus_empty(mask))
130 return -EINVAL;
131
132 for (pos = 0; pos < IA64_NUM_DEVICE_VECTORS; pos++) {
133 cpus_and(mask, domain, vector_table[pos]);
134 if (!cpus_empty(mask))
135 continue;
136 return IA64_FIRST_DEVICE_VECTOR + pos;
137 }
138 return -ENOSPC;
139}
140
141static int __bind_irq_vector(int irq, int vector, cpumask_t domain)
142{
143 cpumask_t mask;
144 int cpu, pos;
145 struct irq_cfg *cfg = &irq_cfg[irq];
146
147 cpus_and(mask, domain, cpu_online_map);
148 if (cpus_empty(mask))
149 return -EINVAL;
150 if ((cfg->vector == vector) && cpus_equal(cfg->domain, domain))
151 return 0;
152 if (cfg->vector != IRQ_VECTOR_UNASSIGNED)
153 return -EBUSY;
154 for_each_cpu_mask(cpu, mask)
155 per_cpu(vector_irq, cpu)[vector] = irq;
156 cfg->vector = vector;
157 cfg->domain = domain;
158 irq_status[irq] = IRQ_USED;
159 pos = vector - IA64_FIRST_DEVICE_VECTOR;
160 cpus_or(vector_table[pos], vector_table[pos], domain);
161 return 0;
162}
163
164int bind_irq_vector(int irq, int vector, cpumask_t domain)
165{
166 unsigned long flags;
167 int ret;
168
169 spin_lock_irqsave(&vector_lock, flags);
170 ret = __bind_irq_vector(irq, vector, domain);
171 spin_unlock_irqrestore(&vector_lock, flags);
172 return ret;
173}
174
175static void __clear_irq_vector(int irq)
176{
177 int vector, cpu, pos;
178 cpumask_t mask;
179 cpumask_t domain;
180 struct irq_cfg *cfg = &irq_cfg[irq];
181
182 BUG_ON((unsigned)irq >= NR_IRQS);
183 BUG_ON(cfg->vector == IRQ_VECTOR_UNASSIGNED);
184 vector = cfg->vector;
185 domain = cfg->domain;
186 cpus_and(mask, cfg->domain, cpu_online_map);
187 for_each_cpu_mask(cpu, mask)
188 per_cpu(vector_irq, cpu)[vector] = IA64_SPURIOUS_INT_VECTOR;
189 cfg->vector = IRQ_VECTOR_UNASSIGNED;
190 cfg->domain = CPU_MASK_NONE;
191 irq_status[irq] = IRQ_UNUSED;
192 pos = vector - IA64_FIRST_DEVICE_VECTOR;
193 cpus_andnot(vector_table[pos], vector_table[pos], domain);
194}
195
196static void clear_irq_vector(int irq)
197{
198 unsigned long flags;
199
200 spin_lock_irqsave(&vector_lock, flags);
201 __clear_irq_vector(irq);
202 spin_unlock_irqrestore(&vector_lock, flags);
203}
68 204
69int 205int
70assign_irq_vector (int irq) 206assign_irq_vector (int irq)
71{ 207{
72 int pos, vector; 208 unsigned long flags;
73 again: 209 int vector, cpu;
74 pos = find_first_zero_bit(ia64_vector_mask, IA64_NUM_DEVICE_VECTORS); 210 cpumask_t domain;
75 vector = IA64_FIRST_DEVICE_VECTOR + pos; 211
76 if (vector > IA64_LAST_DEVICE_VECTOR) 212 vector = -ENOSPC;
77 return -ENOSPC; 213
78 if (test_and_set_bit(pos, ia64_vector_mask)) 214 spin_lock_irqsave(&vector_lock, flags);
79 goto again; 215 if (irq < 0) {
216 goto out;
217 }
218 for_each_online_cpu(cpu) {
219 domain = vector_allocation_domain(cpu);
220 vector = find_unassigned_vector(domain);
221 if (vector >= 0)
222 break;
223 }
224 if (vector < 0)
225 goto out;
226 BUG_ON(__bind_irq_vector(irq, vector, domain));
227 out:
228 spin_unlock_irqrestore(&vector_lock, flags);
80 return vector; 229 return vector;
81} 230}
82 231
83void 232void
84free_irq_vector (int vector) 233free_irq_vector (int vector)
85{ 234{
86 int pos; 235 if (vector < IA64_FIRST_DEVICE_VECTOR ||
87 236 vector > IA64_LAST_DEVICE_VECTOR)
88 if (vector < IA64_FIRST_DEVICE_VECTOR || vector > IA64_LAST_DEVICE_VECTOR)
89 return; 237 return;
90 238 clear_irq_vector(vector);
91 pos = vector - IA64_FIRST_DEVICE_VECTOR;
92 if (!test_and_clear_bit(pos, ia64_vector_mask))
93 printk(KERN_WARNING "%s: double free!\n", __FUNCTION__);
94} 239}
95 240
96int 241int
97reserve_irq_vector (int vector) 242reserve_irq_vector (int vector)
98{ 243{
99 int pos;
100
101 if (vector < IA64_FIRST_DEVICE_VECTOR || 244 if (vector < IA64_FIRST_DEVICE_VECTOR ||
102 vector > IA64_LAST_DEVICE_VECTOR) 245 vector > IA64_LAST_DEVICE_VECTOR)
103 return -EINVAL; 246 return -EINVAL;
247 return !!bind_irq_vector(vector, vector, CPU_MASK_ALL);
248}
104 249
105 pos = vector - IA64_FIRST_DEVICE_VECTOR; 250/*
106 return test_and_set_bit(pos, ia64_vector_mask); 251 * Initialize vector_irq on a new cpu. This function must be called
252 * with vector_lock held.
253 */
254void __setup_vector_irq(int cpu)
255{
256 int irq, vector;
257
258 /* Clear vector_irq */
259 for (vector = 0; vector < IA64_NUM_VECTORS; ++vector)
260 per_cpu(vector_irq, cpu)[vector] = IA64_SPURIOUS_INT_VECTOR;
261 /* Mark the inuse vectors */
262 for (irq = 0; irq < NR_IRQS; ++irq) {
263 if (!cpu_isset(cpu, irq_cfg[irq].domain))
264 continue;
265 vector = irq_to_vector(irq);
266 per_cpu(vector_irq, cpu)[vector] = irq;
267 }
268}
269
270#if defined(CONFIG_SMP) && (defined(CONFIG_IA64_GENERIC) || defined(CONFIG_IA64_DIG))
271static enum vector_domain_type {
272 VECTOR_DOMAIN_NONE,
273 VECTOR_DOMAIN_PERCPU
274} vector_domain_type = VECTOR_DOMAIN_NONE;
275
276static cpumask_t vector_allocation_domain(int cpu)
277{
278 if (vector_domain_type == VECTOR_DOMAIN_PERCPU)
279 return cpumask_of_cpu(cpu);
280 return CPU_MASK_ALL;
281}
282
283static int __init parse_vector_domain(char *arg)
284{
285 if (!arg)
286 return -EINVAL;
287 if (!strcmp(arg, "percpu")) {
288 vector_domain_type = VECTOR_DOMAIN_PERCPU;
289 no_int_routing = 1;
290 }
291 return 1;
292}
293early_param("vector", parse_vector_domain);
294#else
295static cpumask_t vector_allocation_domain(int cpu)
296{
297 return CPU_MASK_ALL;
298}
299#endif
300
301
302void destroy_and_reserve_irq(unsigned int irq)
303{
304 dynamic_irq_cleanup(irq);
305
306 clear_irq_vector(irq);
307 reserve_irq(irq);
308}
309
310static int __reassign_irq_vector(int irq, int cpu)
311{
312 struct irq_cfg *cfg = &irq_cfg[irq];
313 int vector;
314 cpumask_t domain;
315
316 if (cfg->vector == IRQ_VECTOR_UNASSIGNED || !cpu_online(cpu))
317 return -EINVAL;
318 if (cpu_isset(cpu, cfg->domain))
319 return 0;
320 domain = vector_allocation_domain(cpu);
321 vector = find_unassigned_vector(domain);
322 if (vector < 0)
323 return -ENOSPC;
324 __clear_irq_vector(irq);
325 BUG_ON(__bind_irq_vector(irq, vector, domain));
326 return 0;
327}
328
329int reassign_irq_vector(int irq, int cpu)
330{
331 unsigned long flags;
332 int ret;
333
334 spin_lock_irqsave(&vector_lock, flags);
335 ret = __reassign_irq_vector(irq, cpu);
336 spin_unlock_irqrestore(&vector_lock, flags);
337 return ret;
107} 338}
108 339
109/* 340/*
@@ -111,18 +342,35 @@ reserve_irq_vector (int vector)
111 */ 342 */
112int create_irq(void) 343int create_irq(void)
113{ 344{
114 int vector = assign_irq_vector(AUTO_ASSIGN); 345 unsigned long flags;
115 346 int irq, vector, cpu;
116 if (vector >= 0) 347 cpumask_t domain;
117 dynamic_irq_init(vector); 348
118 349 irq = vector = -ENOSPC;
119 return vector; 350 spin_lock_irqsave(&vector_lock, flags);
351 for_each_online_cpu(cpu) {
352 domain = vector_allocation_domain(cpu);
353 vector = find_unassigned_vector(domain);
354 if (vector >= 0)
355 break;
356 }
357 if (vector < 0)
358 goto out;
359 irq = find_unassigned_irq();
360 if (irq < 0)
361 goto out;
362 BUG_ON(__bind_irq_vector(irq, vector, domain));
363 out:
364 spin_unlock_irqrestore(&vector_lock, flags);
365 if (irq >= 0)
366 dynamic_irq_init(irq);
367 return irq;
120} 368}
121 369
122void destroy_irq(unsigned int irq) 370void destroy_irq(unsigned int irq)
123{ 371{
124 dynamic_irq_cleanup(irq); 372 dynamic_irq_cleanup(irq);
125 free_irq_vector(irq); 373 clear_irq_vector(irq);
126} 374}
127 375
128#ifdef CONFIG_SMP 376#ifdef CONFIG_SMP
@@ -301,14 +549,13 @@ register_percpu_irq (ia64_vector vec, struct irqaction *action)
301 irq_desc_t *desc; 549 irq_desc_t *desc;
302 unsigned int irq; 550 unsigned int irq;
303 551
304 for (irq = 0; irq < NR_IRQS; ++irq) 552 irq = vec;
305 if (irq_to_vector(irq) == vec) { 553 BUG_ON(bind_irq_vector(irq, vec, CPU_MASK_ALL));
306 desc = irq_desc + irq; 554 desc = irq_desc + irq;
307 desc->status |= IRQ_PER_CPU; 555 desc->status |= IRQ_PER_CPU;
308 desc->chip = &irq_type_ia64_lsapic; 556 desc->chip = &irq_type_ia64_lsapic;
309 if (action) 557 if (action)
310 setup_irq(irq, action); 558 setup_irq(irq, action);
311 }
312} 559}
313 560
314void __init 561void __init
diff --git a/arch/ia64/kernel/msi_ia64.c b/arch/ia64/kernel/msi_ia64.c
index c81080df70df..2fdbd5c3f213 100644
--- a/arch/ia64/kernel/msi_ia64.c
+++ b/arch/ia64/kernel/msi_ia64.c
@@ -13,6 +13,7 @@
13 13
14#define MSI_DATA_VECTOR_SHIFT 0 14#define MSI_DATA_VECTOR_SHIFT 0
15#define MSI_DATA_VECTOR(v) (((u8)v) << MSI_DATA_VECTOR_SHIFT) 15#define MSI_DATA_VECTOR(v) (((u8)v) << MSI_DATA_VECTOR_SHIFT)
16#define MSI_DATA_VECTOR_MASK 0xffffff00
16 17
17#define MSI_DATA_DELIVERY_SHIFT 8 18#define MSI_DATA_DELIVERY_SHIFT 8
18#define MSI_DATA_DELIVERY_FIXED (0 << MSI_DATA_DELIVERY_SHIFT) 19#define MSI_DATA_DELIVERY_FIXED (0 << MSI_DATA_DELIVERY_SHIFT)
@@ -50,17 +51,29 @@ static struct irq_chip ia64_msi_chip;
50static void ia64_set_msi_irq_affinity(unsigned int irq, cpumask_t cpu_mask) 51static void ia64_set_msi_irq_affinity(unsigned int irq, cpumask_t cpu_mask)
51{ 52{
52 struct msi_msg msg; 53 struct msi_msg msg;
53 u32 addr; 54 u32 addr, data;
55 int cpu = first_cpu(cpu_mask);
56
57 if (!cpu_online(cpu))
58 return;
59
60 if (reassign_irq_vector(irq, cpu))
61 return;
54 62
55 read_msi_msg(irq, &msg); 63 read_msi_msg(irq, &msg);
56 64
57 addr = msg.address_lo; 65 addr = msg.address_lo;
58 addr &= MSI_ADDR_DESTID_MASK; 66 addr &= MSI_ADDR_DESTID_MASK;
59 addr |= MSI_ADDR_DESTID_CPU(cpu_physical_id(first_cpu(cpu_mask))); 67 addr |= MSI_ADDR_DESTID_CPU(cpu_physical_id(cpu));
60 msg.address_lo = addr; 68 msg.address_lo = addr;
61 69
70 data = msg.data;
71 data &= MSI_DATA_VECTOR_MASK;
72 data |= MSI_DATA_VECTOR(irq_to_vector(irq));
73 msg.data = data;
74
62 write_msi_msg(irq, &msg); 75 write_msi_msg(irq, &msg);
63 irq_desc[irq].affinity = cpu_mask; 76 irq_desc[irq].affinity = cpumask_of_cpu(cpu);
64} 77}
65#endif /* CONFIG_SMP */ 78#endif /* CONFIG_SMP */
66 79
@@ -69,13 +82,15 @@ int ia64_setup_msi_irq(struct pci_dev *pdev, struct msi_desc *desc)
69 struct msi_msg msg; 82 struct msi_msg msg;
70 unsigned long dest_phys_id; 83 unsigned long dest_phys_id;
71 int irq, vector; 84 int irq, vector;
85 cpumask_t mask;
72 86
73 irq = create_irq(); 87 irq = create_irq();
74 if (irq < 0) 88 if (irq < 0)
75 return irq; 89 return irq;
76 90
77 set_irq_msi(irq, desc); 91 set_irq_msi(irq, desc);
78 dest_phys_id = cpu_physical_id(first_cpu(cpu_online_map)); 92 cpus_and(mask, irq_to_domain(irq), cpu_online_map);
93 dest_phys_id = cpu_physical_id(first_cpu(mask));
79 vector = irq_to_vector(irq); 94 vector = irq_to_vector(irq);
80 95
81 msg.address_hi = 0; 96 msg.address_hi = 0;
diff --git a/arch/ia64/kernel/smpboot.c b/arch/ia64/kernel/smpboot.c
index 3c9d8e6089cf..9f5c90b594b9 100644
--- a/arch/ia64/kernel/smpboot.c
+++ b/arch/ia64/kernel/smpboot.c
@@ -395,9 +395,13 @@ smp_callin (void)
395 fix_b0_for_bsp(); 395 fix_b0_for_bsp();
396 396
397 lock_ipi_calllock(); 397 lock_ipi_calllock();
398 spin_lock(&vector_lock);
399 /* Setup the per cpu irq handling data structures */
400 __setup_vector_irq(cpuid);
398 cpu_set(cpuid, cpu_online_map); 401 cpu_set(cpuid, cpu_online_map);
399 unlock_ipi_calllock(); 402 unlock_ipi_calllock();
400 per_cpu(cpu_state, cpuid) = CPU_ONLINE; 403 per_cpu(cpu_state, cpuid) = CPU_ONLINE;
404 spin_unlock(&vector_lock);
401 405
402 smp_setup_percpu_timer(); 406 smp_setup_percpu_timer();
403 407
diff --git a/arch/ia64/kernel/time.c b/arch/ia64/kernel/time.c
index 3486fe7d6e65..627785c48ea9 100644
--- a/arch/ia64/kernel/time.c
+++ b/arch/ia64/kernel/time.c
@@ -19,6 +19,7 @@
19#include <linux/interrupt.h> 19#include <linux/interrupt.h>
20#include <linux/efi.h> 20#include <linux/efi.h>
21#include <linux/timex.h> 21#include <linux/timex.h>
22#include <linux/clocksource.h>
22 23
23#include <asm/machvec.h> 24#include <asm/machvec.h>
24#include <asm/delay.h> 25#include <asm/delay.h>
@@ -28,6 +29,16 @@
28#include <asm/sections.h> 29#include <asm/sections.h>
29#include <asm/system.h> 30#include <asm/system.h>
30 31
32#include "fsyscall_gtod_data.h"
33
34static cycle_t itc_get_cycles(void);
35
36struct fsyscall_gtod_data_t fsyscall_gtod_data = {
37 .lock = SEQLOCK_UNLOCKED,
38};
39
40struct itc_jitter_data_t itc_jitter_data;
41
31volatile int time_keeper_id = 0; /* smp_processor_id() of time-keeper */ 42volatile int time_keeper_id = 0; /* smp_processor_id() of time-keeper */
32 43
33#ifdef CONFIG_IA64_DEBUG_IRQ 44#ifdef CONFIG_IA64_DEBUG_IRQ
@@ -37,11 +48,16 @@ EXPORT_SYMBOL(last_cli_ip);
37 48
38#endif 49#endif
39 50
40static struct time_interpolator itc_interpolator = { 51static struct clocksource clocksource_itc = {
41 .shift = 16, 52 .name = "itc",
42 .mask = 0xffffffffffffffffLL, 53 .rating = 350,
43 .source = TIME_SOURCE_CPU 54 .read = itc_get_cycles,
55 .mask = 0xffffffffffffffff,
56 .mult = 0, /*to be caluclated*/
57 .shift = 16,
58 .flags = CLOCK_SOURCE_IS_CONTINUOUS,
44}; 59};
60static struct clocksource *itc_clocksource;
45 61
46static irqreturn_t 62static irqreturn_t
47timer_interrupt (int irq, void *dev_id) 63timer_interrupt (int irq, void *dev_id)
@@ -210,8 +226,6 @@ ia64_init_itm (void)
210 + itc_freq/2)/itc_freq; 226 + itc_freq/2)/itc_freq;
211 227
212 if (!(sal_platform_features & IA64_SAL_PLATFORM_FEATURE_ITC_DRIFT)) { 228 if (!(sal_platform_features & IA64_SAL_PLATFORM_FEATURE_ITC_DRIFT)) {
213 itc_interpolator.frequency = local_cpu_data->itc_freq;
214 itc_interpolator.drift = itc_drift;
215#ifdef CONFIG_SMP 229#ifdef CONFIG_SMP
216 /* On IA64 in an SMP configuration ITCs are never accurately synchronized. 230 /* On IA64 in an SMP configuration ITCs are never accurately synchronized.
217 * Jitter compensation requires a cmpxchg which may limit 231 * Jitter compensation requires a cmpxchg which may limit
@@ -223,15 +237,50 @@ ia64_init_itm (void)
223 * even going backward) if the ITC offsets between the individual CPUs 237 * even going backward) if the ITC offsets between the individual CPUs
224 * are too large. 238 * are too large.
225 */ 239 */
226 if (!nojitter) itc_interpolator.jitter = 1; 240 if (!nojitter)
241 itc_jitter_data.itc_jitter = 1;
227#endif 242#endif
228 register_time_interpolator(&itc_interpolator);
229 } 243 }
230 244
231 /* Setup the CPU local timer tick */ 245 /* Setup the CPU local timer tick */
232 ia64_cpu_local_tick(); 246 ia64_cpu_local_tick();
247
248 if (!itc_clocksource) {
249 /* Sort out mult/shift values: */
250 clocksource_itc.mult =
251 clocksource_hz2mult(local_cpu_data->itc_freq,
252 clocksource_itc.shift);
253 clocksource_register(&clocksource_itc);
254 itc_clocksource = &clocksource_itc;
255 }
233} 256}
234 257
258static cycle_t itc_get_cycles()
259{
260 u64 lcycle, now, ret;
261
262 if (!itc_jitter_data.itc_jitter)
263 return get_cycles();
264
265 lcycle = itc_jitter_data.itc_lastcycle;
266 now = get_cycles();
267 if (lcycle && time_after(lcycle, now))
268 return lcycle;
269
270 /*
271 * Keep track of the last timer value returned.
272 * In an SMP environment, you could lose out in contention of
273 * cmpxchg. If so, your cmpxchg returns new value which the
274 * winner of contention updated to. Use the new value instead.
275 */
276 ret = cmpxchg(&itc_jitter_data.itc_lastcycle, lcycle, now);
277 if (unlikely(ret != lcycle))
278 return ret;
279
280 return now;
281}
282
283
235static struct irqaction timer_irqaction = { 284static struct irqaction timer_irqaction = {
236 .handler = timer_interrupt, 285 .handler = timer_interrupt,
237 .flags = IRQF_DISABLED | IRQF_IRQPOLL, 286 .flags = IRQF_DISABLED | IRQF_IRQPOLL,
@@ -307,3 +356,34 @@ ia64_setup_printk_clock(void)
307 if (!(sal_platform_features & IA64_SAL_PLATFORM_FEATURE_ITC_DRIFT)) 356 if (!(sal_platform_features & IA64_SAL_PLATFORM_FEATURE_ITC_DRIFT))
308 ia64_printk_clock = ia64_itc_printk_clock; 357 ia64_printk_clock = ia64_itc_printk_clock;
309} 358}
359
360void update_vsyscall(struct timespec *wall, struct clocksource *c)
361{
362 unsigned long flags;
363
364 write_seqlock_irqsave(&fsyscall_gtod_data.lock, flags);
365
366 /* copy fsyscall clock data */
367 fsyscall_gtod_data.clk_mask = c->mask;
368 fsyscall_gtod_data.clk_mult = c->mult;
369 fsyscall_gtod_data.clk_shift = c->shift;
370 fsyscall_gtod_data.clk_fsys_mmio = c->fsys_mmio;
371 fsyscall_gtod_data.clk_cycle_last = c->cycle_last;
372
373 /* copy kernel time structures */
374 fsyscall_gtod_data.wall_time.tv_sec = wall->tv_sec;
375 fsyscall_gtod_data.wall_time.tv_nsec = wall->tv_nsec;
376 fsyscall_gtod_data.monotonic_time.tv_sec = wall_to_monotonic.tv_sec
377 + wall->tv_sec;
378 fsyscall_gtod_data.monotonic_time.tv_nsec = wall_to_monotonic.tv_nsec
379 + wall->tv_nsec;
380
381 /* normalize */
382 while (fsyscall_gtod_data.monotonic_time.tv_nsec >= NSEC_PER_SEC) {
383 fsyscall_gtod_data.monotonic_time.tv_nsec -= NSEC_PER_SEC;
384 fsyscall_gtod_data.monotonic_time.tv_sec++;
385 }
386
387 write_sequnlock_irqrestore(&fsyscall_gtod_data.lock, flags);
388}
389
generated by cgit v1.2.2 (git 2.25.0) at 2025-02-28 02:09:53 -0500