aboutsummaryrefslogtreecommitdiffstats
path: root/arch/sparc64/kernel/irq.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/sparc64/kernel/irq.c')
-rw-r--r--arch/sparc64/kernel/irq.c1016
1 files changed, 339 insertions, 677 deletions
diff --git a/arch/sparc64/kernel/irq.c b/arch/sparc64/kernel/irq.c
index 11e645c9ec50..a8c9dc8d1958 100644
--- a/arch/sparc64/kernel/irq.c
+++ b/arch/sparc64/kernel/irq.c
@@ -22,6 +22,7 @@
22#include <linux/proc_fs.h> 22#include <linux/proc_fs.h>
23#include <linux/seq_file.h> 23#include <linux/seq_file.h>
24#include <linux/bootmem.h> 24#include <linux/bootmem.h>
25#include <linux/irq.h>
25 26
26#include <asm/ptrace.h> 27#include <asm/ptrace.h>
27#include <asm/processor.h> 28#include <asm/processor.h>
@@ -42,10 +43,6 @@
42#include <asm/auxio.h> 43#include <asm/auxio.h>
43#include <asm/head.h> 44#include <asm/head.h>
44 45
45#ifdef CONFIG_SMP
46static void distribute_irqs(void);
47#endif
48
49/* UPA nodes send interrupt packet to UltraSparc with first data reg 46/* UPA nodes send interrupt packet to UltraSparc with first data reg
50 * value low 5 (7 on Starfire) bits holding the IRQ identifier being 47 * value low 5 (7 on Starfire) bits holding the IRQ identifier being
51 * delivered. We must translate this into a non-vector IRQ so we can 48 * delivered. We must translate this into a non-vector IRQ so we can
@@ -57,10 +54,29 @@ static void distribute_irqs(void);
57 * The IVEC handler does not need to act atomically, the PIL dispatch 54 * The IVEC handler does not need to act atomically, the PIL dispatch
58 * code uses CAS to get an atomic snapshot of the list and clear it 55 * code uses CAS to get an atomic snapshot of the list and clear it
59 * at the same time. 56 * at the same time.
57 *
58 * If you make changes to ino_bucket, please update hand coded assembler
59 * of the vectored interrupt trap handler(s) in entry.S and sun4v_ivec.S
60 */ 60 */
61struct ino_bucket {
62 /* Next handler in per-CPU IRQ worklist. We know that
63 * bucket pointers have the high 32-bits clear, so to
64 * save space we only store the bits we need.
65 */
66/*0x00*/unsigned int irq_chain;
61 67
68 /* Virtual interrupt number assigned to this INO. */
69/*0x04*/unsigned int virt_irq;
70};
71
72#define NUM_IVECS (IMAP_INR + 1)
62struct ino_bucket ivector_table[NUM_IVECS] __attribute__ ((aligned (SMP_CACHE_BYTES))); 73struct ino_bucket ivector_table[NUM_IVECS] __attribute__ ((aligned (SMP_CACHE_BYTES)));
63 74
75#define __irq_ino(irq) \
76 (((struct ino_bucket *)(unsigned long)(irq)) - &ivector_table[0])
77#define __bucket(irq) ((struct ino_bucket *)(unsigned long)(irq))
78#define __irq(bucket) ((unsigned int)(unsigned long)(bucket))
79
64/* This has to be in the main kernel image, it cannot be 80/* This has to be in the main kernel image, it cannot be
65 * turned into per-cpu data. The reason is that the main 81 * turned into per-cpu data. The reason is that the main
66 * kernel image is locked into the TLB and this structure 82 * kernel image is locked into the TLB and this structure
@@ -68,71 +84,82 @@ struct ino_bucket ivector_table[NUM_IVECS] __attribute__ ((aligned (SMP_CACHE_BY
68 * access to this structure takes a TLB miss it could cause 84 * access to this structure takes a TLB miss it could cause
69 * the 5-level sparc v9 trap stack to overflow. 85 * the 5-level sparc v9 trap stack to overflow.
70 */ 86 */
71struct irq_work_struct { 87#define irq_work(__cpu) &(trap_block[(__cpu)].irq_worklist)
72 unsigned int irq_worklists[16];
73};
74struct irq_work_struct __irq_work[NR_CPUS];
75#define irq_work(__cpu, __pil) &(__irq_work[(__cpu)].irq_worklists[(__pil)])
76 88
77static struct irqaction *irq_action[NR_IRQS+1]; 89static unsigned int virt_to_real_irq_table[NR_IRQS];
90static unsigned char virt_irq_cur = 1;
78 91
79/* This only synchronizes entities which modify IRQ handler 92static unsigned char virt_irq_alloc(unsigned int real_irq)
80 * state and some selected user-level spots that want to 93{
81 * read things in the table. IRQ handler processing orders 94 unsigned char ent;
82 * its' accesses such that no locking is needed. 95
83 */ 96 BUILD_BUG_ON(NR_IRQS >= 256);
84static DEFINE_SPINLOCK(irq_action_lock); 97
98 ent = virt_irq_cur;
99 if (ent >= NR_IRQS) {
100 printk(KERN_ERR "IRQ: Out of virtual IRQs.\n");
101 return 0;
102 }
103
104 virt_irq_cur = ent + 1;
105 virt_to_real_irq_table[ent] = real_irq;
106
107 return ent;
108}
109
110#if 0 /* Currently unused. */
111static unsigned char real_to_virt_irq(unsigned int real_irq)
112{
113 struct ino_bucket *bucket = __bucket(real_irq);
85 114
86static void register_irq_proc (unsigned int irq); 115 return bucket->virt_irq;
116}
117#endif
118
119static unsigned int virt_to_real_irq(unsigned char virt_irq)
120{
121 return virt_to_real_irq_table[virt_irq];
122}
87 123
88/* 124/*
89 * Upper 2b of irqaction->flags holds the ino. 125 * /proc/interrupts printing:
90 * irqaction->mask holds the smp affinity information.
91 */ 126 */
92#define put_ino_in_irqaction(action, irq) \
93 action->flags &= 0xffffffffffffUL; \
94 if (__bucket(irq) == &pil0_dummy_bucket) \
95 action->flags |= 0xdeadUL << 48; \
96 else \
97 action->flags |= __irq_ino(irq) << 48;
98#define get_ino_in_irqaction(action) (action->flags >> 48)
99
100#define put_smpaff_in_irqaction(action, smpaff) (action)->mask = (smpaff)
101#define get_smpaff_in_irqaction(action) ((action)->mask)
102 127
103int show_interrupts(struct seq_file *p, void *v) 128int show_interrupts(struct seq_file *p, void *v)
104{ 129{
130 int i = *(loff_t *) v, j;
131 struct irqaction * action;
105 unsigned long flags; 132 unsigned long flags;
106 int i = *(loff_t *) v;
107 struct irqaction *action;
108#ifdef CONFIG_SMP
109 int j;
110#endif
111 133
112 spin_lock_irqsave(&irq_action_lock, flags); 134 if (i == 0) {
113 if (i <= NR_IRQS) { 135 seq_printf(p, " ");
114 if (!(action = *(i + irq_action))) 136 for_each_online_cpu(j)
115 goto out_unlock; 137 seq_printf(p, "CPU%d ",j);
116 seq_printf(p, "%3d: ", i); 138 seq_putc(p, '\n');
139 }
140
141 if (i < NR_IRQS) {
142 spin_lock_irqsave(&irq_desc[i].lock, flags);
143 action = irq_desc[i].action;
144 if (!action)
145 goto skip;
146 seq_printf(p, "%3d: ",i);
117#ifndef CONFIG_SMP 147#ifndef CONFIG_SMP
118 seq_printf(p, "%10u ", kstat_irqs(i)); 148 seq_printf(p, "%10u ", kstat_irqs(i));
119#else 149#else
120 for_each_online_cpu(j) { 150 for_each_online_cpu(j)
121 seq_printf(p, "%10u ", 151 seq_printf(p, "%10u ", kstat_cpu(j).irqs[i]);
122 kstat_cpu(j).irqs[i]);
123 }
124#endif 152#endif
125 seq_printf(p, " %s:%lx", action->name, 153 seq_printf(p, " %9s", irq_desc[i].handler->typename);
126 get_ino_in_irqaction(action)); 154 seq_printf(p, " %s", action->name);
127 for (action = action->next; action; action = action->next) { 155
128 seq_printf(p, ", %s:%lx", action->name, 156 for (action=action->next; action; action = action->next)
129 get_ino_in_irqaction(action)); 157 seq_printf(p, ", %s", action->name);
130 } 158
131 seq_putc(p, '\n'); 159 seq_putc(p, '\n');
160skip:
161 spin_unlock_irqrestore(&irq_desc[i].lock, flags);
132 } 162 }
133out_unlock:
134 spin_unlock_irqrestore(&irq_action_lock, flags);
135
136 return 0; 163 return 0;
137} 164}
138 165
@@ -173,556 +200,365 @@ static unsigned int sun4u_compute_tid(unsigned long imap, unsigned long cpuid)
173 return tid; 200 return tid;
174} 201}
175 202
176/* Now these are always passed a true fully specified sun4u INO. */ 203struct irq_handler_data {
177void enable_irq(unsigned int irq) 204 unsigned long iclr;
178{ 205 unsigned long imap;
179 struct ino_bucket *bucket = __bucket(irq);
180 unsigned long imap, cpuid;
181
182 imap = bucket->imap;
183 if (imap == 0UL)
184 return;
185
186 preempt_disable();
187
188 /* This gets the physical processor ID, even on uniprocessor,
189 * so we can always program the interrupt target correctly.
190 */
191 cpuid = real_hard_smp_processor_id();
192
193 if (tlb_type == hypervisor) {
194 unsigned int ino = __irq_ino(irq);
195 int err;
196 206
197 err = sun4v_intr_settarget(ino, cpuid); 207 void (*pre_handler)(unsigned int, void *, void *);
198 if (err != HV_EOK) 208 void *pre_handler_arg1;
199 printk("sun4v_intr_settarget(%x,%lu): err(%d)\n", 209 void *pre_handler_arg2;
200 ino, cpuid, err); 210};
201 err = sun4v_intr_setenabled(ino, HV_INTR_ENABLED);
202 if (err != HV_EOK)
203 printk("sun4v_intr_setenabled(%x): err(%d)\n",
204 ino, err);
205 } else {
206 unsigned int tid = sun4u_compute_tid(imap, cpuid);
207
208 /* NOTE NOTE NOTE, IGN and INO are read-only, IGN is a product
209 * of this SYSIO's preconfigured IGN in the SYSIO Control
210 * Register, the hardware just mirrors that value here.
211 * However for Graphics and UPA Slave devices the full
212 * IMAP_INR field can be set by the programmer here.
213 *
214 * Things like FFB can now be handled via the new IRQ
215 * mechanism.
216 */
217 upa_writel(tid | IMAP_VALID, imap);
218 }
219
220 preempt_enable();
221}
222 211
223/* This now gets passed true ino's as well. */ 212static inline struct ino_bucket *virt_irq_to_bucket(unsigned int virt_irq)
224void disable_irq(unsigned int irq)
225{ 213{
226 struct ino_bucket *bucket = __bucket(irq); 214 unsigned int real_irq = virt_to_real_irq(virt_irq);
227 unsigned long imap; 215 struct ino_bucket *bucket = NULL;
228
229 imap = bucket->imap;
230 if (imap != 0UL) {
231 if (tlb_type == hypervisor) {
232 unsigned int ino = __irq_ino(irq);
233 int err;
234
235 err = sun4v_intr_setenabled(ino, HV_INTR_DISABLED);
236 if (err != HV_EOK)
237 printk("sun4v_intr_setenabled(%x): "
238 "err(%d)\n", ino, err);
239 } else {
240 u32 tmp;
241
242 /* NOTE: We do not want to futz with the IRQ clear registers
243 * and move the state to IDLE, the SCSI code does call
244 * disable_irq() to assure atomicity in the queue cmd
245 * SCSI adapter driver code. Thus we'd lose interrupts.
246 */
247 tmp = upa_readl(imap);
248 tmp &= ~IMAP_VALID;
249 upa_writel(tmp, imap);
250 }
251 }
252}
253 216
254/* The timer is the one "weird" interrupt which is generated by 217 if (likely(real_irq))
255 * the CPU %tick register and not by some normal vectored interrupt 218 bucket = __bucket(real_irq);
256 * source. To handle this special case, we use this dummy INO bucket.
257 */
258static struct irq_desc pil0_dummy_desc;
259static struct ino_bucket pil0_dummy_bucket = {
260 .irq_info = &pil0_dummy_desc,
261};
262 219
263static void build_irq_error(const char *msg, unsigned int ino, int pil, int inofixup, 220 return bucket;
264 unsigned long iclr, unsigned long imap,
265 struct ino_bucket *bucket)
266{
267 prom_printf("IRQ: INO %04x (%d:%016lx:%016lx) --> "
268 "(%d:%d:%016lx:%016lx), halting...\n",
269 ino, bucket->pil, bucket->iclr, bucket->imap,
270 pil, inofixup, iclr, imap);
271 prom_halt();
272} 221}
273 222
274unsigned int build_irq(int pil, int inofixup, unsigned long iclr, unsigned long imap) 223#ifdef CONFIG_SMP
224static int irq_choose_cpu(unsigned int virt_irq)
275{ 225{
276 struct ino_bucket *bucket; 226 cpumask_t mask = irq_affinity[virt_irq];
277 int ino; 227 int cpuid;
278 228
279 if (pil == 0) { 229 if (cpus_equal(mask, CPU_MASK_ALL)) {
280 if (iclr != 0UL || imap != 0UL) { 230 static int irq_rover;
281 prom_printf("Invalid dummy bucket for PIL0 (%lx:%lx)\n", 231 static DEFINE_SPINLOCK(irq_rover_lock);
282 iclr, imap); 232 unsigned long flags;
283 prom_halt();
284 }
285 return __irq(&pil0_dummy_bucket);
286 }
287 233
288 BUG_ON(tlb_type == hypervisor); 234 /* Round-robin distribution... */
235 do_round_robin:
236 spin_lock_irqsave(&irq_rover_lock, flags);
289 237
290 /* RULE: Both must be specified in all other cases. */ 238 while (!cpu_online(irq_rover)) {
291 if (iclr == 0UL || imap == 0UL) { 239 if (++irq_rover >= NR_CPUS)
292 prom_printf("Invalid build_irq %d %d %016lx %016lx\n", 240 irq_rover = 0;
293 pil, inofixup, iclr, imap); 241 }
294 prom_halt(); 242 cpuid = irq_rover;
295 } 243 do {
296 244 if (++irq_rover >= NR_CPUS)
297 ino = (upa_readl(imap) & (IMAP_IGN | IMAP_INO)) + inofixup; 245 irq_rover = 0;
298 if (ino > NUM_IVECS) { 246 } while (!cpu_online(irq_rover));
299 prom_printf("Invalid INO %04x (%d:%d:%016lx:%016lx)\n",
300 ino, pil, inofixup, iclr, imap);
301 prom_halt();
302 }
303 247
304 bucket = &ivector_table[ino]; 248 spin_unlock_irqrestore(&irq_rover_lock, flags);
305 if (bucket->flags & IBF_ACTIVE) 249 } else {
306 build_irq_error("IRQ: Trying to build active INO bucket.\n", 250 cpumask_t tmp;
307 ino, pil, inofixup, iclr, imap, bucket);
308 251
309 if (bucket->irq_info) { 252 cpus_and(tmp, cpu_online_map, mask);
310 if (bucket->imap != imap || bucket->iclr != iclr)
311 build_irq_error("IRQ: Trying to reinit INO bucket.\n",
312 ino, pil, inofixup, iclr, imap, bucket);
313 253
314 goto out; 254 if (cpus_empty(tmp))
315 } 255 goto do_round_robin;
316 256
317 bucket->irq_info = kzalloc(sizeof(struct irq_desc), GFP_ATOMIC); 257 cpuid = first_cpu(tmp);
318 if (!bucket->irq_info) {
319 prom_printf("IRQ: Error, kmalloc(irq_desc) failed.\n");
320 prom_halt();
321 } 258 }
322 259
323 /* Ok, looks good, set it up. Don't touch the irq_chain or 260 return cpuid;
324 * the pending flag.
325 */
326 bucket->imap = imap;
327 bucket->iclr = iclr;
328 bucket->pil = pil;
329 bucket->flags = 0;
330
331out:
332 return __irq(bucket);
333} 261}
334 262#else
335unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino, int pil, unsigned char flags) 263static int irq_choose_cpu(unsigned int virt_irq)
336{ 264{
337 struct ino_bucket *bucket; 265 return real_hard_smp_processor_id();
338 unsigned long sysino; 266}
267#endif
339 268
340 sysino = sun4v_devino_to_sysino(devhandle, devino); 269static void sun4u_irq_enable(unsigned int virt_irq)
270{
271 irq_desc_t *desc = irq_desc + virt_irq;
272 struct irq_handler_data *data = desc->handler_data;
341 273
342 bucket = &ivector_table[sysino]; 274 if (likely(data)) {
275 unsigned long cpuid, imap;
276 unsigned int tid;
343 277
344 /* Catch accidental accesses to these things. IMAP/ICLR handling 278 cpuid = irq_choose_cpu(virt_irq);
345 * is done by hypervisor calls on sun4v platforms, not by direct 279 imap = data->imap;
346 * register accesses.
347 *
348 * But we need to make them look unique for the disable_irq() logic
349 * in free_irq().
350 */
351 bucket->imap = ~0UL - sysino;
352 bucket->iclr = ~0UL - sysino;
353 280
354 bucket->pil = pil; 281 tid = sun4u_compute_tid(imap, cpuid);
355 bucket->flags = flags;
356 282
357 bucket->irq_info = kzalloc(sizeof(struct irq_desc), GFP_ATOMIC); 283 upa_writel(tid | IMAP_VALID, imap);
358 if (!bucket->irq_info) {
359 prom_printf("IRQ: Error, kmalloc(irq_desc) failed.\n");
360 prom_halt();
361 } 284 }
362
363 return __irq(bucket);
364} 285}
365 286
366static void atomic_bucket_insert(struct ino_bucket *bucket) 287static void sun4u_irq_disable(unsigned int virt_irq)
367{ 288{
368 unsigned long pstate; 289 irq_desc_t *desc = irq_desc + virt_irq;
369 unsigned int *ent; 290 struct irq_handler_data *data = desc->handler_data;
370 291
371 __asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate)); 292 if (likely(data)) {
372 __asm__ __volatile__("wrpr %0, %1, %%pstate" 293 unsigned long imap = data->imap;
373 : : "r" (pstate), "i" (PSTATE_IE)); 294 u32 tmp = upa_readl(imap);
374 ent = irq_work(smp_processor_id(), bucket->pil);
375 bucket->irq_chain = *ent;
376 *ent = __irq(bucket);
377 __asm__ __volatile__("wrpr %0, 0x0, %%pstate" : : "r" (pstate));
378}
379 295
380static int check_irq_sharing(int pil, unsigned long irqflags) 296 tmp &= ~IMAP_VALID;
381{ 297 upa_writel(tmp, imap);
382 struct irqaction *action, *tmp;
383
384 action = *(irq_action + pil);
385 if (action) {
386 if ((action->flags & SA_SHIRQ) && (irqflags & SA_SHIRQ)) {
387 for (tmp = action; tmp->next; tmp = tmp->next)
388 ;
389 } else {
390 return -EBUSY;
391 }
392 } 298 }
393 return 0;
394} 299}
395 300
396static void append_irq_action(int pil, struct irqaction *action) 301static void sun4u_irq_end(unsigned int virt_irq)
397{ 302{
398 struct irqaction **pp = irq_action + pil; 303 irq_desc_t *desc = irq_desc + virt_irq;
304 struct irq_handler_data *data = desc->handler_data;
399 305
400 while (*pp) 306 if (likely(data))
401 pp = &((*pp)->next); 307 upa_writel(ICLR_IDLE, data->iclr);
402 *pp = action;
403} 308}
404 309
405static struct irqaction *get_action_slot(struct ino_bucket *bucket) 310static void sun4v_irq_enable(unsigned int virt_irq)
406{ 311{
407 struct irq_desc *desc = bucket->irq_info; 312 struct ino_bucket *bucket = virt_irq_to_bucket(virt_irq);
408 int max_irq, i; 313 unsigned int ino = bucket - &ivector_table[0];
409 314
410 max_irq = 1; 315 if (likely(bucket)) {
411 if (bucket->flags & IBF_PCI) 316 unsigned long cpuid;
412 max_irq = MAX_IRQ_DESC_ACTION; 317 int err;
413 for (i = 0; i < max_irq; i++) {
414 struct irqaction *p = &desc->action[i];
415 u32 mask = (1 << i);
416 318
417 if (desc->action_active_mask & mask) 319 cpuid = irq_choose_cpu(virt_irq);
418 continue;
419 320
420 desc->action_active_mask |= mask; 321 err = sun4v_intr_settarget(ino, cpuid);
421 return p; 322 if (err != HV_EOK)
323 printk("sun4v_intr_settarget(%x,%lu): err(%d)\n",
324 ino, cpuid, err);
325 err = sun4v_intr_setenabled(ino, HV_INTR_ENABLED);
326 if (err != HV_EOK)
327 printk("sun4v_intr_setenabled(%x): err(%d)\n",
328 ino, err);
422 } 329 }
423 return NULL;
424} 330}
425 331
426int request_irq(unsigned int irq, irqreturn_t (*handler)(int, void *, struct pt_regs *), 332static void sun4v_irq_disable(unsigned int virt_irq)
427 unsigned long irqflags, const char *name, void *dev_id)
428{ 333{
429 struct irqaction *action; 334 struct ino_bucket *bucket = virt_irq_to_bucket(virt_irq);
430 struct ino_bucket *bucket = __bucket(irq); 335 unsigned int ino = bucket - &ivector_table[0];
431 unsigned long flags;
432 int pending = 0;
433
434 if (unlikely(!handler))
435 return -EINVAL;
436
437 if (unlikely(!bucket->irq_info))
438 return -ENODEV;
439
440 if ((bucket != &pil0_dummy_bucket) && (irqflags & SA_SAMPLE_RANDOM)) {
441 /*
442 * This function might sleep, we want to call it first,
443 * outside of the atomic block. In SA_STATIC_ALLOC case,
444 * random driver's kmalloc will fail, but it is safe.
445 * If already initialized, random driver will not reinit.
446 * Yes, this might clear the entropy pool if the wrong
447 * driver is attempted to be loaded, without actually
448 * installing a new handler, but is this really a problem,
449 * only the sysadmin is able to do this.
450 */
451 rand_initialize_irq(irq);
452 }
453
454 spin_lock_irqsave(&irq_action_lock, flags);
455
456 if (check_irq_sharing(bucket->pil, irqflags)) {
457 spin_unlock_irqrestore(&irq_action_lock, flags);
458 return -EBUSY;
459 }
460 336
461 action = get_action_slot(bucket); 337 if (likely(bucket)) {
462 if (!action) { 338 int err;
463 spin_unlock_irqrestore(&irq_action_lock, flags);
464 return -ENOMEM;
465 }
466 339
467 bucket->flags |= IBF_ACTIVE; 340 err = sun4v_intr_setenabled(ino, HV_INTR_DISABLED);
468 pending = 0; 341 if (err != HV_EOK)
469 if (bucket != &pil0_dummy_bucket) { 342 printk("sun4v_intr_setenabled(%x): "
470 pending = bucket->pending; 343 "err(%d)\n", ino, err);
471 if (pending)
472 bucket->pending = 0;
473 } 344 }
345}
474 346
475 action->handler = handler; 347static void sun4v_irq_end(unsigned int virt_irq)
476 action->flags = irqflags; 348{
477 action->name = name; 349 struct ino_bucket *bucket = virt_irq_to_bucket(virt_irq);
478 action->next = NULL; 350 unsigned int ino = bucket - &ivector_table[0];
479 action->dev_id = dev_id;
480 put_ino_in_irqaction(action, irq);
481 put_smpaff_in_irqaction(action, CPU_MASK_NONE);
482
483 append_irq_action(bucket->pil, action);
484 351
485 enable_irq(irq); 352 if (likely(bucket)) {
353 int err;
486 354
487 /* We ate the IVEC already, this makes sure it does not get lost. */ 355 err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE);
488 if (pending) { 356 if (err != HV_EOK)
489 atomic_bucket_insert(bucket); 357 printk("sun4v_intr_setstate(%x): "
490 set_softint(1 << bucket->pil); 358 "err(%d)\n", ino, err);
491 } 359 }
360}
492 361
493 spin_unlock_irqrestore(&irq_action_lock, flags); 362static void run_pre_handler(unsigned int virt_irq)
494 363{
495 if (bucket != &pil0_dummy_bucket) 364 struct ino_bucket *bucket = virt_irq_to_bucket(virt_irq);
496 register_irq_proc(__irq_ino(irq)); 365 irq_desc_t *desc = irq_desc + virt_irq;
366 struct irq_handler_data *data = desc->handler_data;
497 367
498#ifdef CONFIG_SMP 368 if (likely(data->pre_handler)) {
499 distribute_irqs(); 369 data->pre_handler(__irq_ino(__irq(bucket)),
500#endif 370 data->pre_handler_arg1,
501 return 0; 371 data->pre_handler_arg2);
372 }
502} 373}
503 374
504EXPORT_SYMBOL(request_irq); 375static struct hw_interrupt_type sun4u_irq = {
376 .typename = "sun4u",
377 .enable = sun4u_irq_enable,
378 .disable = sun4u_irq_disable,
379 .end = sun4u_irq_end,
380};
505 381
506static struct irqaction *unlink_irq_action(unsigned int irq, void *dev_id) 382static struct hw_interrupt_type sun4u_irq_ack = {
507{ 383 .typename = "sun4u+ack",
508 struct ino_bucket *bucket = __bucket(irq); 384 .enable = sun4u_irq_enable,
509 struct irqaction *action, **pp; 385 .disable = sun4u_irq_disable,
386 .ack = run_pre_handler,
387 .end = sun4u_irq_end,
388};
510 389
511 pp = irq_action + bucket->pil; 390static struct hw_interrupt_type sun4v_irq = {
512 action = *pp; 391 .typename = "sun4v",
513 if (unlikely(!action)) 392 .enable = sun4v_irq_enable,
514 return NULL; 393 .disable = sun4v_irq_disable,
394 .end = sun4v_irq_end,
395};
515 396
516 if (unlikely(!action->handler)) { 397static struct hw_interrupt_type sun4v_irq_ack = {
517 printk("Freeing free IRQ %d\n", bucket->pil); 398 .typename = "sun4v+ack",
518 return NULL; 399 .enable = sun4v_irq_enable,
519 } 400 .disable = sun4v_irq_disable,
401 .ack = run_pre_handler,
402 .end = sun4v_irq_end,
403};
520 404
521 while (action && action->dev_id != dev_id) { 405void irq_install_pre_handler(int virt_irq,
522 pp = &action->next; 406 void (*func)(unsigned int, void *, void *),
523 action = *pp; 407 void *arg1, void *arg2)
524 } 408{
409 irq_desc_t *desc = irq_desc + virt_irq;
410 struct irq_handler_data *data = desc->handler_data;
525 411
526 if (likely(action)) 412 data->pre_handler = func;
527 *pp = action->next; 413 data->pre_handler_arg1 = arg1;
414 data->pre_handler_arg2 = arg2;
528 415
529 return action; 416 desc->handler = (desc->handler == &sun4u_irq ?
417 &sun4u_irq_ack : &sun4v_irq_ack);
530} 418}
531 419
532void free_irq(unsigned int irq, void *dev_id) 420unsigned int build_irq(int inofixup, unsigned long iclr, unsigned long imap)
533{ 421{
534 struct irqaction *action;
535 struct ino_bucket *bucket; 422 struct ino_bucket *bucket;
536 unsigned long flags; 423 struct irq_handler_data *data;
424 irq_desc_t *desc;
425 int ino;
537 426
538 spin_lock_irqsave(&irq_action_lock, flags); 427 BUG_ON(tlb_type == hypervisor);
539 428
540 action = unlink_irq_action(irq, dev_id); 429 ino = (upa_readl(imap) & (IMAP_IGN | IMAP_INO)) + inofixup;
430 bucket = &ivector_table[ino];
431 if (!bucket->virt_irq) {
432 bucket->virt_irq = virt_irq_alloc(__irq(bucket));
433 irq_desc[bucket->virt_irq].handler = &sun4u_irq;
434 }
541 435
542 spin_unlock_irqrestore(&irq_action_lock, flags); 436 desc = irq_desc + bucket->virt_irq;
437 if (unlikely(desc->handler_data))
438 goto out;
543 439
544 if (unlikely(!action)) 440 data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC);
545 return; 441 if (unlikely(!data)) {
442 prom_printf("IRQ: kzalloc(irq_handler_data) failed.\n");
443 prom_halt();
444 }
445 desc->handler_data = data;
546 446
547 synchronize_irq(irq); 447 data->imap = imap;
448 data->iclr = iclr;
548 449
549 spin_lock_irqsave(&irq_action_lock, flags); 450out:
451 return bucket->virt_irq;
452}
550 453
551 bucket = __bucket(irq); 454unsigned int sun4v_build_irq(u32 devhandle, unsigned int devino)
552 if (bucket != &pil0_dummy_bucket) { 455{
553 struct irq_desc *desc = bucket->irq_info; 456 struct ino_bucket *bucket;
554 int ent, i; 457 struct irq_handler_data *data;
458 unsigned long sysino;
459 irq_desc_t *desc;
555 460
556 for (i = 0; i < MAX_IRQ_DESC_ACTION; i++) { 461 BUG_ON(tlb_type != hypervisor);
557 struct irqaction *p = &desc->action[i];
558 462
559 if (p == action) { 463 sysino = sun4v_devino_to_sysino(devhandle, devino);
560 desc->action_active_mask &= ~(1 << i); 464 bucket = &ivector_table[sysino];
561 break; 465 if (!bucket->virt_irq) {
562 } 466 bucket->virt_irq = virt_irq_alloc(__irq(bucket));
563 } 467 irq_desc[bucket->virt_irq].handler = &sun4v_irq;
468 }
564 469
565 if (!desc->action_active_mask) { 470 desc = irq_desc + bucket->virt_irq;
566 unsigned long imap = bucket->imap; 471 if (unlikely(desc->handler_data))
567 472 goto out;
568 /* This unique interrupt source is now inactive. */
569 bucket->flags &= ~IBF_ACTIVE;
570
571 /* See if any other buckets share this bucket's IMAP
572 * and are still active.
573 */
574 for (ent = 0; ent < NUM_IVECS; ent++) {
575 struct ino_bucket *bp = &ivector_table[ent];
576 if (bp != bucket &&
577 bp->imap == imap &&
578 (bp->flags & IBF_ACTIVE) != 0)
579 break;
580 }
581 473
582 /* Only disable when no other sub-irq levels of 474 data = kzalloc(sizeof(struct irq_handler_data), GFP_ATOMIC);
583 * the same IMAP are active. 475 if (unlikely(!data)) {
584 */ 476 prom_printf("IRQ: kzalloc(irq_handler_data) failed.\n");
585 if (ent == NUM_IVECS) 477 prom_halt();
586 disable_irq(irq);
587 }
588 } 478 }
479 desc->handler_data = data;
589 480
590 spin_unlock_irqrestore(&irq_action_lock, flags); 481 /* Catch accidental accesses to these things. IMAP/ICLR handling
591} 482 * is done by hypervisor calls on sun4v platforms, not by direct
483 * register accesses.
484 */
485 data->imap = ~0UL;
486 data->iclr = ~0UL;
592 487
593EXPORT_SYMBOL(free_irq); 488out:
489 return bucket->virt_irq;
490}
594 491
595#ifdef CONFIG_SMP 492void hw_resend_irq(struct hw_interrupt_type *handler, unsigned int virt_irq)
596void synchronize_irq(unsigned int irq)
597{ 493{
598 struct ino_bucket *bucket = __bucket(irq); 494 struct ino_bucket *bucket = virt_irq_to_bucket(virt_irq);
599 495 unsigned long pstate;
600#if 0 496 unsigned int *ent;
601 /* The following is how I wish I could implement this. 497
602 * Unfortunately the ICLR registers are read-only, you can 498 __asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate));
603 * only write ICLR_foo values to them. To get the current 499 __asm__ __volatile__("wrpr %0, %1, %%pstate"
604 * IRQ status you would need to get at the IRQ diag registers 500 : : "r" (pstate), "i" (PSTATE_IE));
605 * in the PCI/SBUS controller and the layout of those vary 501 ent = irq_work(smp_processor_id());
606 * from one controller to the next, sigh... -DaveM 502 bucket->irq_chain = *ent;
607 */ 503 *ent = __irq(bucket);
608 unsigned long iclr = bucket->iclr; 504 set_softint(1 << PIL_DEVICE_IRQ);
609 505 __asm__ __volatile__("wrpr %0, 0x0, %%pstate" : : "r" (pstate));
610 while (1) {
611 u32 tmp = upa_readl(iclr);
612
613 if (tmp == ICLR_TRANSMIT ||
614 tmp == ICLR_PENDING) {
615 cpu_relax();
616 continue;
617 }
618 break;
619 }
620#else
621 /* So we have to do this with a INPROGRESS bit just like x86. */
622 while (bucket->flags & IBF_INPROGRESS)
623 cpu_relax();
624#endif
625} 506}
626#endif /* CONFIG_SMP */
627 507
628static void process_bucket(int irq, struct ino_bucket *bp, struct pt_regs *regs) 508void ack_bad_irq(unsigned int virt_irq)
629{ 509{
630 struct irq_desc *desc = bp->irq_info; 510 struct ino_bucket *bucket = virt_irq_to_bucket(virt_irq);
631 unsigned char flags = bp->flags; 511 unsigned int ino = 0xdeadbeef;
632 u32 action_mask, i;
633 int random;
634 512
635 bp->flags |= IBF_INPROGRESS; 513 if (bucket)
514 ino = bucket - &ivector_table[0];
636 515
637 if (unlikely(!(flags & IBF_ACTIVE))) { 516 printk(KERN_CRIT "Unexpected IRQ from ino[%x] virt_irq[%u]\n",
638 bp->pending = 1; 517 ino, virt_irq);
639 goto out; 518}
640 }
641
642 if (desc->pre_handler)
643 desc->pre_handler(bp,
644 desc->pre_handler_arg1,
645 desc->pre_handler_arg2);
646 519
647 action_mask = desc->action_active_mask; 520#ifndef CONFIG_SMP
648 random = 0; 521extern irqreturn_t timer_interrupt(int, void *, struct pt_regs *);
649 for (i = 0; i < MAX_IRQ_DESC_ACTION; i++) {
650 struct irqaction *p = &desc->action[i];
651 u32 mask = (1 << i);
652 522
653 if (!(action_mask & mask)) 523void timer_irq(int irq, struct pt_regs *regs)
654 continue; 524{
525 unsigned long clr_mask = 1 << irq;
526 unsigned long tick_mask = tick_ops->softint_mask;
655 527
656 action_mask &= ~mask; 528 if (get_softint() & tick_mask) {
529 irq = 0;
530 clr_mask = tick_mask;
531 }
532 clear_softint(clr_mask);
657 533
658 if (p->handler(__irq(bp), p->dev_id, regs) == IRQ_HANDLED) 534 irq_enter();
659 random |= p->flags;
660 535
661 if (!action_mask) 536 kstat_this_cpu.irqs[0]++;
662 break; 537 timer_interrupt(irq, NULL, regs);
663 }
664 if (bp->pil != 0) {
665 if (tlb_type == hypervisor) {
666 unsigned int ino = __irq_ino(bp);
667 int err;
668
669 err = sun4v_intr_setstate(ino, HV_INTR_STATE_IDLE);
670 if (err != HV_EOK)
671 printk("sun4v_intr_setstate(%x): "
672 "err(%d)\n", ino, err);
673 } else {
674 upa_writel(ICLR_IDLE, bp->iclr);
675 }
676 538
677 /* Test and add entropy */ 539 irq_exit();
678 if (random & SA_SAMPLE_RANDOM)
679 add_interrupt_randomness(irq);
680 }
681out:
682 bp->flags &= ~IBF_INPROGRESS;
683} 540}
541#endif
684 542
685void handler_irq(int irq, struct pt_regs *regs) 543void handler_irq(int irq, struct pt_regs *regs)
686{ 544{
687 struct ino_bucket *bp; 545 struct ino_bucket *bucket;
688 int cpu = smp_processor_id();
689
690#ifndef CONFIG_SMP
691 /*
692 * Check for TICK_INT on level 14 softint.
693 */
694 {
695 unsigned long clr_mask = 1 << irq;
696 unsigned long tick_mask = tick_ops->softint_mask;
697 546
698 if ((irq == 14) && (get_softint() & tick_mask)) {
699 irq = 0;
700 clr_mask = tick_mask;
701 }
702 clear_softint(clr_mask);
703 }
704#else
705 clear_softint(1 << irq); 547 clear_softint(1 << irq);
706#endif
707 548
708 irq_enter(); 549 irq_enter();
709 kstat_this_cpu.irqs[irq]++;
710 550
711 /* Sliiiick... */ 551 /* Sliiiick... */
712#ifndef CONFIG_SMP 552 bucket = __bucket(xchg32(irq_work(smp_processor_id()), 0));
713 bp = ((irq != 0) ? 553 while (bucket) {
714 __bucket(xchg32(irq_work(cpu, irq), 0)) : 554 struct ino_bucket *next = __bucket(bucket->irq_chain);
715 &pil0_dummy_bucket);
716#else
717 bp = __bucket(xchg32(irq_work(cpu, irq), 0));
718#endif
719 while (bp) {
720 struct ino_bucket *nbp = __bucket(bp->irq_chain);
721 555
722 bp->irq_chain = 0; 556 bucket->irq_chain = 0;
723 process_bucket(irq, bp, regs); 557 __do_IRQ(bucket->virt_irq, regs);
724 bp = nbp; 558
559 bucket = next;
725 } 560 }
561
726 irq_exit(); 562 irq_exit();
727} 563}
728 564
@@ -787,81 +623,6 @@ main_interrupt:
787EXPORT_SYMBOL(sparc_floppy_irq); 623EXPORT_SYMBOL(sparc_floppy_irq);
788#endif 624#endif
789 625
790/* We really don't need these at all on the Sparc. We only have
791 * stubs here because they are exported to modules.
792 */
793unsigned long probe_irq_on(void)
794{
795 return 0;
796}
797
798EXPORT_SYMBOL(probe_irq_on);
799
800int probe_irq_off(unsigned long mask)
801{
802 return 0;
803}
804
805EXPORT_SYMBOL(probe_irq_off);
806
807#ifdef CONFIG_SMP
808static int retarget_one_irq(struct irqaction *p, int goal_cpu)
809{
810 struct ino_bucket *bucket = get_ino_in_irqaction(p) + ivector_table;
811
812 while (!cpu_online(goal_cpu)) {
813 if (++goal_cpu >= NR_CPUS)
814 goal_cpu = 0;
815 }
816
817 if (tlb_type == hypervisor) {
818 unsigned int ino = __irq_ino(bucket);
819
820 sun4v_intr_settarget(ino, goal_cpu);
821 sun4v_intr_setenabled(ino, HV_INTR_ENABLED);
822 } else {
823 unsigned long imap = bucket->imap;
824 unsigned int tid = sun4u_compute_tid(imap, goal_cpu);
825
826 upa_writel(tid | IMAP_VALID, imap);
827 }
828
829 do {
830 if (++goal_cpu >= NR_CPUS)
831 goal_cpu = 0;
832 } while (!cpu_online(goal_cpu));
833
834 return goal_cpu;
835}
836
837/* Called from request_irq. */
838static void distribute_irqs(void)
839{
840 unsigned long flags;
841 int cpu, level;
842
843 spin_lock_irqsave(&irq_action_lock, flags);
844 cpu = 0;
845
846 /*
847 * Skip the timer at [0], and very rare error/power intrs at [15].
848 * Also level [12], it causes problems on Ex000 systems.
849 */
850 for (level = 1; level < NR_IRQS; level++) {
851 struct irqaction *p = irq_action[level];
852
853 if (level == 12)
854 continue;
855
856 while(p) {
857 cpu = retarget_one_irq(p, cpu);
858 p = p->next;
859 }
860 }
861 spin_unlock_irqrestore(&irq_action_lock, flags);
862}
863#endif
864
865struct sun5_timer { 626struct sun5_timer {
866 u64 count0; 627 u64 count0;
867 u64 limit0; 628 u64 limit0;
@@ -929,7 +690,7 @@ void init_irqwork_curcpu(void)
929{ 690{
930 int cpu = hard_smp_processor_id(); 691 int cpu = hard_smp_processor_id();
931 692
932 memset(__irq_work + cpu, 0, sizeof(struct irq_work_struct)); 693 trap_block[cpu].irq_worklist = 0;
933} 694}
934 695
935static void __cpuinit register_one_mondo(unsigned long paddr, unsigned long type) 696static void __cpuinit register_one_mondo(unsigned long paddr, unsigned long type)
@@ -1037,6 +798,10 @@ void __cpuinit sun4v_init_mondo_queues(int use_bootmem, int cpu, int alloc, int
1037 } 798 }
1038} 799}
1039 800
801static struct irqaction timer_irq_action = {
802 .name = "timer",
803};
804
1040/* Only invoked on boot processor. */ 805/* Only invoked on boot processor. */
1041void __init init_IRQ(void) 806void __init init_IRQ(void)
1042{ 807{
@@ -1064,109 +829,6 @@ void __init init_IRQ(void)
1064 : /* No outputs */ 829 : /* No outputs */
1065 : "i" (PSTATE_IE) 830 : "i" (PSTATE_IE)
1066 : "g1"); 831 : "g1");
1067}
1068 832
1069static struct proc_dir_entry * root_irq_dir; 833 irq_desc[0].action = &timer_irq_action;
1070static struct proc_dir_entry * irq_dir [NUM_IVECS];
1071
1072#ifdef CONFIG_SMP
1073
1074static int irq_affinity_read_proc (char *page, char **start, off_t off,
1075 int count, int *eof, void *data)
1076{
1077 struct ino_bucket *bp = ivector_table + (long)data;
1078 struct irq_desc *desc = bp->irq_info;
1079 struct irqaction *ap = desc->action;
1080 cpumask_t mask;
1081 int len;
1082
1083 mask = get_smpaff_in_irqaction(ap);
1084 if (cpus_empty(mask))
1085 mask = cpu_online_map;
1086
1087 len = cpumask_scnprintf(page, count, mask);
1088 if (count - len < 2)
1089 return -EINVAL;
1090 len += sprintf(page + len, "\n");
1091 return len;
1092}
1093
1094static inline void set_intr_affinity(int irq, cpumask_t hw_aff)
1095{
1096 struct ino_bucket *bp = ivector_table + irq;
1097 struct irq_desc *desc = bp->irq_info;
1098 struct irqaction *ap = desc->action;
1099
1100 /* Users specify affinity in terms of hw cpu ids.
1101 * As soon as we do this, handler_irq() might see and take action.
1102 */
1103 put_smpaff_in_irqaction(ap, hw_aff);
1104
1105 /* Migration is simply done by the next cpu to service this
1106 * interrupt.
1107 */
1108}
1109
1110static int irq_affinity_write_proc (struct file *file, const char __user *buffer,
1111 unsigned long count, void *data)
1112{
1113 int irq = (long) data, full_count = count, err;
1114 cpumask_t new_value;
1115
1116 err = cpumask_parse(buffer, count, new_value);
1117
1118 /*
1119 * Do not allow disabling IRQs completely - it's a too easy
1120 * way to make the system unusable accidentally :-) At least
1121 * one online CPU still has to be targeted.
1122 */
1123 cpus_and(new_value, new_value, cpu_online_map);
1124 if (cpus_empty(new_value))
1125 return -EINVAL;
1126
1127 set_intr_affinity(irq, new_value);
1128
1129 return full_count;
1130} 834}
1131
1132#endif
1133
1134#define MAX_NAMELEN 10
1135
1136static void register_irq_proc (unsigned int irq)
1137{
1138 char name [MAX_NAMELEN];
1139
1140 if (!root_irq_dir || irq_dir[irq])
1141 return;
1142
1143 memset(name, 0, MAX_NAMELEN);
1144 sprintf(name, "%x", irq);
1145
1146 /* create /proc/irq/1234 */
1147 irq_dir[irq] = proc_mkdir(name, root_irq_dir);
1148
1149#ifdef CONFIG_SMP
1150 /* XXX SMP affinity not supported on starfire yet. */
1151 if (this_is_starfire == 0) {
1152 struct proc_dir_entry *entry;
1153
1154 /* create /proc/irq/1234/smp_affinity */
1155 entry = create_proc_entry("smp_affinity", 0600, irq_dir[irq]);
1156
1157 if (entry) {
1158 entry->nlink = 1;
1159 entry->data = (void *)(long)irq;
1160 entry->read_proc = irq_affinity_read_proc;
1161 entry->write_proc = irq_affinity_write_proc;
1162 }
1163 }
1164#endif
1165}
1166
1167void init_irq_proc (void)
1168{
1169 /* create /proc/irq */
1170 root_irq_dir = proc_mkdir("irq", NULL);
1171}
1172
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-14 10:39:54 -0500