diff options
Diffstat (limited to 'arch/tile/kernel/irq.c')
-rw-r--r-- | arch/tile/kernel/irq.c | 259 |
1 files changed, 183 insertions, 76 deletions
diff --git a/arch/tile/kernel/irq.c b/arch/tile/kernel/irq.c index 24cc6b2abc2c..596c60086930 100644 --- a/arch/tile/kernel/irq.c +++ b/arch/tile/kernel/irq.c | |||
@@ -19,6 +19,11 @@ | |||
19 | #include <linux/kernel_stat.h> | 19 | #include <linux/kernel_stat.h> |
20 | #include <linux/uaccess.h> | 20 | #include <linux/uaccess.h> |
21 | #include <hv/drv_pcie_rc_intf.h> | 21 | #include <hv/drv_pcie_rc_intf.h> |
22 | #include <arch/spr_def.h> | ||
23 | #include <asm/traps.h> | ||
24 | |||
25 | /* Bit-flag stored in irq_desc->chip_data to indicate HW-cleared irqs. */ | ||
26 | #define IS_HW_CLEARED 1 | ||
22 | 27 | ||
23 | /* | 28 | /* |
24 | * The set of interrupts we enable for raw_local_irq_enable(). | 29 | * The set of interrupts we enable for raw_local_irq_enable(). |
@@ -31,30 +36,74 @@ DEFINE_PER_CPU(unsigned long long, interrupts_enabled_mask) = | |||
31 | INITIAL_INTERRUPTS_ENABLED; | 36 | INITIAL_INTERRUPTS_ENABLED; |
32 | EXPORT_PER_CPU_SYMBOL(interrupts_enabled_mask); | 37 | EXPORT_PER_CPU_SYMBOL(interrupts_enabled_mask); |
33 | 38 | ||
34 | /* Define per-tile device interrupt state */ | 39 | /* Define per-tile device interrupt statistics state. */ |
35 | DEFINE_PER_CPU(HV_IntrState, dev_intr_state); | ||
36 | |||
37 | DEFINE_PER_CPU(irq_cpustat_t, irq_stat) ____cacheline_internodealigned_in_smp; | 40 | DEFINE_PER_CPU(irq_cpustat_t, irq_stat) ____cacheline_internodealigned_in_smp; |
38 | EXPORT_PER_CPU_SYMBOL(irq_stat); | 41 | EXPORT_PER_CPU_SYMBOL(irq_stat); |
39 | 42 | ||
43 | /* | ||
44 | * Define per-tile irq disable mask; the hardware/HV only has a single | ||
45 | * mask that we use to implement both masking and disabling. | ||
46 | */ | ||
47 | static DEFINE_PER_CPU(unsigned long, irq_disable_mask) | ||
48 | ____cacheline_internodealigned_in_smp; | ||
49 | |||
50 | /* | ||
51 | * Per-tile IRQ nesting depth. Used to make sure we enable newly | ||
52 | * enabled IRQs before exiting the outermost interrupt. | ||
53 | */ | ||
54 | static DEFINE_PER_CPU(int, irq_depth); | ||
55 | |||
56 | /* State for allocating IRQs on Gx. */ | ||
57 | #if CHIP_HAS_IPI() | ||
58 | static unsigned long available_irqs = ~(1UL << IRQ_RESCHEDULE); | ||
59 | static DEFINE_SPINLOCK(available_irqs_lock); | ||
60 | #endif | ||
40 | 61 | ||
62 | #if CHIP_HAS_IPI() | ||
63 | /* Use SPRs to manipulate device interrupts. */ | ||
64 | #define mask_irqs(irq_mask) __insn_mtspr(SPR_IPI_MASK_SET_1, irq_mask) | ||
65 | #define unmask_irqs(irq_mask) __insn_mtspr(SPR_IPI_MASK_RESET_1, irq_mask) | ||
66 | #define clear_irqs(irq_mask) __insn_mtspr(SPR_IPI_EVENT_RESET_1, irq_mask) | ||
67 | #else | ||
68 | /* Use HV to manipulate device interrupts. */ | ||
69 | #define mask_irqs(irq_mask) hv_disable_intr(irq_mask) | ||
70 | #define unmask_irqs(irq_mask) hv_enable_intr(irq_mask) | ||
71 | #define clear_irqs(irq_mask) hv_clear_intr(irq_mask) | ||
72 | #endif | ||
41 | 73 | ||
42 | /* | 74 | /* |
43 | * Interrupt dispatcher, invoked upon a hypervisor device interrupt downcall | 75 | * The interrupt handling path, implemented in terms of HV interrupt |
76 | * emulation on TILE64 and TILEPro, and IPI hardware on TILE-Gx. | ||
44 | */ | 77 | */ |
45 | void tile_dev_intr(struct pt_regs *regs, int intnum) | 78 | void tile_dev_intr(struct pt_regs *regs, int intnum) |
46 | { | 79 | { |
47 | int irq; | 80 | int depth = __get_cpu_var(irq_depth)++; |
81 | unsigned long original_irqs; | ||
82 | unsigned long remaining_irqs; | ||
83 | struct pt_regs *old_regs; | ||
48 | 84 | ||
85 | #if CHIP_HAS_IPI() | ||
49 | /* | 86 | /* |
50 | * Get the device interrupt pending mask from where the hypervisor | 87 | * Pending interrupts are listed in an SPR. We might be |
51 | * has tucked it away for us. | 88 | * nested, so be sure to only handle irqs that weren't already |
89 | * masked by a previous interrupt. Then, mask out the ones | ||
90 | * we're going to handle. | ||
52 | */ | 91 | */ |
53 | unsigned long pending_dev_intr_mask = __insn_mfspr(SPR_SYSTEM_SAVE_1_3); | 92 | unsigned long masked = __insn_mfspr(SPR_IPI_MASK_1); |
54 | 93 | original_irqs = __insn_mfspr(SPR_IPI_EVENT_1) & ~masked; | |
94 | __insn_mtspr(SPR_IPI_MASK_SET_1, original_irqs); | ||
95 | #else | ||
96 | /* | ||
97 | * Hypervisor performs the equivalent of the Gx code above and | ||
98 | * then puts the pending interrupt mask into a system save reg | ||
99 | * for us to find. | ||
100 | */ | ||
101 | original_irqs = __insn_mfspr(SPR_SYSTEM_SAVE_1_3); | ||
102 | #endif | ||
103 | remaining_irqs = original_irqs; | ||
55 | 104 | ||
56 | /* Track time spent here in an interrupt context. */ | 105 | /* Track time spent here in an interrupt context. */ |
57 | struct pt_regs *old_regs = set_irq_regs(regs); | 106 | old_regs = set_irq_regs(regs); |
58 | irq_enter(); | 107 | irq_enter(); |
59 | 108 | ||
60 | #ifdef CONFIG_DEBUG_STACKOVERFLOW | 109 | #ifdef CONFIG_DEBUG_STACKOVERFLOW |
@@ -62,26 +111,35 @@ void tile_dev_intr(struct pt_regs *regs, int intnum) | |||
62 | { | 111 | { |
63 | long sp = stack_pointer - (long) current_thread_info(); | 112 | long sp = stack_pointer - (long) current_thread_info(); |
64 | if (unlikely(sp < (sizeof(struct thread_info) + STACK_WARN))) { | 113 | if (unlikely(sp < (sizeof(struct thread_info) + STACK_WARN))) { |
65 | printk(KERN_EMERG "tile_dev_intr: " | 114 | pr_emerg("tile_dev_intr: " |
66 | "stack overflow: %ld\n", | 115 | "stack overflow: %ld\n", |
67 | sp - sizeof(struct thread_info)); | 116 | sp - sizeof(struct thread_info)); |
68 | dump_stack(); | 117 | dump_stack(); |
69 | } | 118 | } |
70 | } | 119 | } |
71 | #endif | 120 | #endif |
121 | while (remaining_irqs) { | ||
122 | unsigned long irq = __ffs(remaining_irqs); | ||
123 | remaining_irqs &= ~(1UL << irq); | ||
72 | 124 | ||
73 | for (irq = 0; pending_dev_intr_mask; ++irq) { | 125 | /* Count device irqs; Linux IPIs are counted elsewhere. */ |
74 | if (pending_dev_intr_mask & 0x1) { | 126 | if (irq != IRQ_RESCHEDULE) |
75 | generic_handle_irq(irq); | 127 | __get_cpu_var(irq_stat).irq_dev_intr_count++; |
76 | 128 | ||
77 | /* Count device irqs; IPIs are counted elsewhere. */ | 129 | generic_handle_irq(irq); |
78 | if (irq > HV_MAX_IPI_INTERRUPT) | ||
79 | __get_cpu_var(irq_stat).irq_dev_intr_count++; | ||
80 | } | ||
81 | pending_dev_intr_mask >>= 1; | ||
82 | } | 130 | } |
83 | 131 | ||
84 | /* | 132 | /* |
133 | * If we weren't nested, turn on all enabled interrupts, | ||
134 | * including any that were reenabled during interrupt | ||
135 | * handling. | ||
136 | */ | ||
137 | if (depth == 0) | ||
138 | unmask_irqs(~__get_cpu_var(irq_disable_mask)); | ||
139 | |||
140 | __get_cpu_var(irq_depth)--; | ||
141 | |||
142 | /* | ||
85 | * Track time spent against the current process again and | 143 | * Track time spent against the current process again and |
86 | * process any softirqs if they are waiting. | 144 | * process any softirqs if they are waiting. |
87 | */ | 145 | */ |
@@ -90,97 +148,114 @@ void tile_dev_intr(struct pt_regs *regs, int intnum) | |||
90 | } | 148 | } |
91 | 149 | ||
92 | 150 | ||
151 | /* | ||
152 | * Remove an irq from the disabled mask. If we're in an interrupt | ||
153 | * context, defer enabling the HW interrupt until we leave. | ||
154 | */ | ||
155 | void enable_percpu_irq(unsigned int irq) | ||
156 | { | ||
157 | get_cpu_var(irq_disable_mask) &= ~(1UL << irq); | ||
158 | if (__get_cpu_var(irq_depth) == 0) | ||
159 | unmask_irqs(1UL << irq); | ||
160 | put_cpu_var(irq_disable_mask); | ||
161 | } | ||
162 | EXPORT_SYMBOL(enable_percpu_irq); | ||
163 | |||
164 | /* | ||
165 | * Add an irq to the disabled mask. We disable the HW interrupt | ||
166 | * immediately so that there's no possibility of it firing. If we're | ||
167 | * in an interrupt context, the return path is careful to avoid | ||
168 | * unmasking a newly disabled interrupt. | ||
169 | */ | ||
170 | void disable_percpu_irq(unsigned int irq) | ||
171 | { | ||
172 | get_cpu_var(irq_disable_mask) |= (1UL << irq); | ||
173 | mask_irqs(1UL << irq); | ||
174 | put_cpu_var(irq_disable_mask); | ||
175 | } | ||
176 | EXPORT_SYMBOL(disable_percpu_irq); | ||
177 | |||
93 | /* Mask an interrupt. */ | 178 | /* Mask an interrupt. */ |
94 | static void hv_dev_irq_mask(unsigned int irq) | 179 | static void tile_irq_chip_mask(unsigned int irq) |
95 | { | 180 | { |
96 | HV_IntrState *p_intr_state = &__get_cpu_var(dev_intr_state); | 181 | mask_irqs(1UL << irq); |
97 | hv_disable_intr(p_intr_state, 1 << irq); | ||
98 | } | 182 | } |
99 | 183 | ||
100 | /* Unmask an interrupt. */ | 184 | /* Unmask an interrupt. */ |
101 | static void hv_dev_irq_unmask(unsigned int irq) | 185 | static void tile_irq_chip_unmask(unsigned int irq) |
102 | { | 186 | { |
103 | /* Re-enable the hypervisor to generate interrupts. */ | 187 | unmask_irqs(1UL << irq); |
104 | HV_IntrState *p_intr_state = &__get_cpu_var(dev_intr_state); | ||
105 | hv_enable_intr(p_intr_state, 1 << irq); | ||
106 | } | 188 | } |
107 | 189 | ||
108 | /* | 190 | /* |
109 | * The HV doesn't latch incoming interrupts while an interrupt is | 191 | * Clear an interrupt before processing it so that any new assertions |
110 | * disabled, so we need to reenable interrupts before running the | 192 | * will trigger another irq. |
111 | * handler. | ||
112 | * | ||
113 | * ISSUE: Enabling the interrupt this early avoids any race conditions | ||
114 | * but introduces the possibility of nested interrupt stack overflow. | ||
115 | * An imminent change to the HV IRQ model will fix this. | ||
116 | */ | 193 | */ |
117 | static void hv_dev_irq_ack(unsigned int irq) | 194 | static void tile_irq_chip_ack(unsigned int irq) |
118 | { | 195 | { |
119 | hv_dev_irq_unmask(irq); | 196 | if ((unsigned long)get_irq_chip_data(irq) != IS_HW_CLEARED) |
197 | clear_irqs(1UL << irq); | ||
120 | } | 198 | } |
121 | 199 | ||
122 | /* | 200 | /* |
123 | * Since ack() reenables interrupts, there's nothing to do at eoi(). | 201 | * For per-cpu interrupts, we need to avoid unmasking any interrupts |
202 | * that we disabled via disable_percpu_irq(). | ||
124 | */ | 203 | */ |
125 | static void hv_dev_irq_eoi(unsigned int irq) | 204 | static void tile_irq_chip_eoi(unsigned int irq) |
126 | { | 205 | { |
206 | if (!(__get_cpu_var(irq_disable_mask) & (1UL << irq))) | ||
207 | unmask_irqs(1UL << irq); | ||
127 | } | 208 | } |
128 | 209 | ||
129 | static struct irq_chip hv_dev_irq_chip = { | 210 | static struct irq_chip tile_irq_chip = { |
130 | .typename = "hv_dev_irq_chip", | 211 | .typename = "tile_irq_chip", |
131 | .ack = hv_dev_irq_ack, | 212 | .ack = tile_irq_chip_ack, |
132 | .mask = hv_dev_irq_mask, | 213 | .eoi = tile_irq_chip_eoi, |
133 | .unmask = hv_dev_irq_unmask, | 214 | .mask = tile_irq_chip_mask, |
134 | .eoi = hv_dev_irq_eoi, | 215 | .unmask = tile_irq_chip_unmask, |
135 | }; | ||
136 | |||
137 | static struct irqaction resched_action = { | ||
138 | .handler = handle_reschedule_ipi, | ||
139 | .name = "resched", | ||
140 | .dev_id = handle_reschedule_ipi /* unique token */, | ||
141 | }; | 216 | }; |
142 | 217 | ||
143 | void __init init_IRQ(void) | 218 | void __init init_IRQ(void) |
144 | { | 219 | { |
145 | /* Bind IPI irqs. Does this belong somewhere else in init? */ | 220 | ipi_init(); |
146 | tile_irq_activate(IRQ_RESCHEDULE); | ||
147 | BUG_ON(setup_irq(IRQ_RESCHEDULE, &resched_action)); | ||
148 | } | 221 | } |
149 | 222 | ||
150 | void __cpuinit init_per_tile_IRQs(void) | 223 | void __cpuinit setup_irq_regs(void) |
151 | { | 224 | { |
152 | int rc; | 225 | /* Enable interrupt delivery. */ |
153 | 226 | unmask_irqs(~0UL); | |
154 | /* Set the pointer to the per-tile device interrupt state. */ | 227 | #if CHIP_HAS_IPI() |
155 | HV_IntrState *sv_ptr = &__get_cpu_var(dev_intr_state); | 228 | raw_local_irq_unmask(INT_IPI_1); |
156 | rc = hv_dev_register_intr_state(sv_ptr); | 229 | #endif |
157 | if (rc != HV_OK) | ||
158 | panic("hv_dev_register_intr_state: error %d", rc); | ||
159 | |||
160 | } | 230 | } |
161 | 231 | ||
162 | void tile_irq_activate(unsigned int irq) | 232 | void tile_irq_activate(unsigned int irq, int tile_irq_type) |
163 | { | 233 | { |
164 | /* | 234 | /* |
165 | * Paravirtualized drivers can call up to the HV to find out | 235 | * We use handle_level_irq() by default because the pending |
166 | * which irq they're associated with. The HV interface | 236 | * interrupt vector (whether modeled by the HV on TILE64 and |
167 | * doesn't provide a generic call for discovering all valid | 237 | * TILEPro or implemented in hardware on TILE-Gx) has |
168 | * IRQs, so drivers must call this method to initialize newly | 238 | * level-style semantics for each bit. An interrupt fires |
169 | * discovered IRQs. | 239 | * whenever a bit is high, not just at edges. |
170 | * | 240 | */ |
171 | * We could also just initialize all 32 IRQs at startup, but | 241 | irq_flow_handler_t handle = handle_level_irq; |
172 | * doing so would lead to a kernel fault if an unexpected | 242 | if (tile_irq_type == TILE_IRQ_PERCPU) |
173 | * interrupt fires and jumps to a NULL action. By defering | 243 | handle = handle_percpu_irq; |
174 | * the set_irq_chip_and_handler() call, unexpected IRQs are | 244 | set_irq_chip_and_handler(irq, &tile_irq_chip, handle); |
175 | * handled properly by handle_bad_irq(). | 245 | |
246 | /* | ||
247 | * Flag interrupts that are hardware-cleared so that ack() | ||
248 | * won't clear them. | ||
176 | */ | 249 | */ |
177 | hv_dev_irq_mask(irq); | 250 | if (tile_irq_type == TILE_IRQ_HW_CLEAR) |
178 | set_irq_chip_and_handler(irq, &hv_dev_irq_chip, handle_percpu_irq); | 251 | set_irq_chip_data(irq, (void *)IS_HW_CLEARED); |
179 | } | 252 | } |
253 | EXPORT_SYMBOL(tile_irq_activate); | ||
254 | |||
180 | 255 | ||
181 | void ack_bad_irq(unsigned int irq) | 256 | void ack_bad_irq(unsigned int irq) |
182 | { | 257 | { |
183 | printk(KERN_ERR "unexpected IRQ trap at vector %02x\n", irq); | 258 | pr_err("unexpected IRQ trap at vector %02x\n", irq); |
184 | } | 259 | } |
185 | 260 | ||
186 | /* | 261 | /* |
@@ -225,3 +300,35 @@ skip: | |||
225 | } | 300 | } |
226 | return 0; | 301 | return 0; |
227 | } | 302 | } |
303 | |||
304 | #if CHIP_HAS_IPI() | ||
305 | int create_irq(void) | ||
306 | { | ||
307 | unsigned long flags; | ||
308 | int result; | ||
309 | |||
310 | spin_lock_irqsave(&available_irqs_lock, flags); | ||
311 | if (available_irqs == 0) | ||
312 | result = -ENOMEM; | ||
313 | else { | ||
314 | result = __ffs(available_irqs); | ||
315 | available_irqs &= ~(1UL << result); | ||
316 | dynamic_irq_init(result); | ||
317 | } | ||
318 | spin_unlock_irqrestore(&available_irqs_lock, flags); | ||
319 | |||
320 | return result; | ||
321 | } | ||
322 | EXPORT_SYMBOL(create_irq); | ||
323 | |||
324 | void destroy_irq(unsigned int irq) | ||
325 | { | ||
326 | unsigned long flags; | ||
327 | |||
328 | spin_lock_irqsave(&available_irqs_lock, flags); | ||
329 | available_irqs |= (1UL << irq); | ||
330 | dynamic_irq_cleanup(irq); | ||
331 | spin_unlock_irqrestore(&available_irqs_lock, flags); | ||
332 | } | ||
333 | EXPORT_SYMBOL(destroy_irq); | ||
334 | #endif | ||