diff options
-rw-r--r-- | arch/x86/kernel/Makefile | 2 | ||||
-rw-r--r-- | arch/x86/kernel/entry_64.S | 4 | ||||
-rw-r--r-- | arch/x86/kernel/tlb_64.c | 5 | ||||
-rw-r--r-- | arch/x86/kernel/tlb_uv.c | 736 | ||||
-rw-r--r-- | include/asm-x86/uv/uv_bau.h | 331 |
5 files changed, 1077 insertions, 1 deletions
diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 605995e11ea7..4078e98f0125 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile | |||
@@ -94,7 +94,7 @@ obj-$(CONFIG_OLPC) += olpc.o | |||
94 | ### | 94 | ### |
95 | # 64 bit specific files | 95 | # 64 bit specific files |
96 | ifeq ($(CONFIG_X86_64),y) | 96 | ifeq ($(CONFIG_X86_64),y) |
97 | obj-y += genapic_64.o genapic_flat_64.o genx2apic_uv_x.o | 97 | obj-y += genapic_64.o genapic_flat_64.o genx2apic_uv_x.o tlb_uv.o |
98 | obj-$(CONFIG_X86_PM_TIMER) += pmtimer_64.o | 98 | obj-$(CONFIG_X86_PM_TIMER) += pmtimer_64.o |
99 | obj-$(CONFIG_AUDIT) += audit_64.o | 99 | obj-$(CONFIG_AUDIT) += audit_64.o |
100 | 100 | ||
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index 556a8df522a7..6fd1987466eb 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S | |||
@@ -720,6 +720,10 @@ ENTRY(apic_timer_interrupt) | |||
720 | apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt | 720 | apicinterrupt LOCAL_TIMER_VECTOR,smp_apic_timer_interrupt |
721 | END(apic_timer_interrupt) | 721 | END(apic_timer_interrupt) |
722 | 722 | ||
723 | ENTRY(uv_bau_message_intr1) | ||
724 | apicinterrupt 220,uv_bau_message_interrupt | ||
725 | END(uv_bau_message_intr1) | ||
726 | |||
723 | ENTRY(error_interrupt) | 727 | ENTRY(error_interrupt) |
724 | apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt | 728 | apicinterrupt ERROR_APIC_VECTOR,smp_error_interrupt |
725 | END(error_interrupt) | 729 | END(error_interrupt) |
diff --git a/arch/x86/kernel/tlb_64.c b/arch/x86/kernel/tlb_64.c index a1f07d793202..fc132113bdab 100644 --- a/arch/x86/kernel/tlb_64.c +++ b/arch/x86/kernel/tlb_64.c | |||
@@ -15,6 +15,8 @@ | |||
15 | #include <asm/proto.h> | 15 | #include <asm/proto.h> |
16 | #include <asm/apicdef.h> | 16 | #include <asm/apicdef.h> |
17 | #include <asm/idle.h> | 17 | #include <asm/idle.h> |
18 | #include <asm/uv/uv_hub.h> | ||
19 | #include <asm/uv/uv_bau.h> | ||
18 | 20 | ||
19 | #include <mach_ipi.h> | 21 | #include <mach_ipi.h> |
20 | /* | 22 | /* |
@@ -162,6 +164,9 @@ void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm, | |||
162 | union smp_flush_state *f; | 164 | union smp_flush_state *f; |
163 | cpumask_t cpumask = *cpumaskp; | 165 | cpumask_t cpumask = *cpumaskp; |
164 | 166 | ||
167 | if (is_uv_system() && uv_flush_tlb_others(&cpumask, mm, va)) | ||
168 | return; | ||
169 | |||
165 | /* Caller has disabled preemption */ | 170 | /* Caller has disabled preemption */ |
166 | sender = smp_processor_id() % NUM_INVALIDATE_TLB_VECTORS; | 171 | sender = smp_processor_id() % NUM_INVALIDATE_TLB_VECTORS; |
167 | f = &per_cpu(flush_state, sender); | 172 | f = &per_cpu(flush_state, sender); |
diff --git a/arch/x86/kernel/tlb_uv.c b/arch/x86/kernel/tlb_uv.c new file mode 100644 index 000000000000..28e7c68d9d78 --- /dev/null +++ b/arch/x86/kernel/tlb_uv.c | |||
@@ -0,0 +1,736 @@ | |||
1 | /* | ||
2 | * SGI UltraViolet TLB flush routines. | ||
3 | * | ||
4 | * (c) 2008 Cliff Wickman <cpw@sgi.com>, SGI. | ||
5 | * | ||
6 | * This code is released under the GNU General Public License version 2 or | ||
7 | * later. | ||
8 | */ | ||
9 | #include <linux/mc146818rtc.h> | ||
10 | #include <linux/proc_fs.h> | ||
11 | #include <linux/kernel.h> | ||
12 | |||
13 | #include <asm/mach-bigsmp/mach_apic.h> | ||
14 | #include <asm/mmu_context.h> | ||
15 | #include <asm/idle.h> | ||
16 | #include <asm/genapic.h> | ||
17 | #include <asm/uv/uv_hub.h> | ||
18 | #include <asm/uv/uv_mmrs.h> | ||
19 | #include <asm/uv/uv_bau.h> | ||
20 | |||
21 | struct bau_control **uv_bau_table_bases; | ||
22 | static int uv_bau_retry_limit; | ||
23 | static int uv_nshift; /* position of pnode (which is nasid>>1) */ | ||
24 | static unsigned long uv_mmask; | ||
25 | |||
26 | char *status_table[] = { | ||
27 | "IDLE", | ||
28 | "ACTIVE", | ||
29 | "DESTINATION TIMEOUT", | ||
30 | "SOURCE TIMEOUT" | ||
31 | }; | ||
32 | |||
33 | DEFINE_PER_CPU(struct ptc_stats, ptcstats); | ||
34 | DEFINE_PER_CPU(struct bau_control, bau_control); | ||
35 | |||
36 | /* | ||
37 | * Free a software acknowledge hardware resource by clearing its Pending | ||
38 | * bit. This will return a reply to the sender. | ||
39 | * If the message has timed out, a reply has already been sent by the | ||
40 | * hardware but the resource has not been released. In that case our | ||
41 | * clear of the Timeout bit (as well) will free the resource. No reply will | ||
42 | * be sent (the hardware will only do one reply per message). | ||
43 | */ | ||
44 | static void | ||
45 | uv_reply_to_message(int resource, | ||
46 | struct bau_payload_queue_entry *msg, | ||
47 | struct bau_msg_status *msp) | ||
48 | { | ||
49 | int fw; | ||
50 | |||
51 | fw = (1 << (resource + UV_SW_ACK_NPENDING)) | (1 << resource); | ||
52 | msg->replied_to = 1; | ||
53 | msg->sw_ack_vector = 0; | ||
54 | if (msp) | ||
55 | msp->seen_by.bits = 0; | ||
56 | uv_write_local_mmr(UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS, fw); | ||
57 | return; | ||
58 | } | ||
59 | |||
60 | /* | ||
61 | * Do all the things a cpu should do for a TLB shootdown message. | ||
62 | * Other cpu's may come here at the same time for this message. | ||
63 | */ | ||
64 | static void | ||
65 | uv_bau_process_message(struct bau_payload_queue_entry *msg, | ||
66 | int msg_slot, int sw_ack_slot) | ||
67 | { | ||
68 | int cpu; | ||
69 | unsigned long this_cpu_mask; | ||
70 | struct bau_msg_status *msp; | ||
71 | |||
72 | msp = __get_cpu_var(bau_control).msg_statuses + msg_slot; | ||
73 | cpu = uv_blade_processor_id(); | ||
74 | msg->number_of_cpus = | ||
75 | uv_blade_nr_online_cpus(uv_node_to_blade_id(numa_node_id())); | ||
76 | this_cpu_mask = (unsigned long)1 << cpu; | ||
77 | if (msp->seen_by.bits & this_cpu_mask) | ||
78 | return; | ||
79 | atomic_or_long(&msp->seen_by.bits, this_cpu_mask); | ||
80 | |||
81 | if (msg->replied_to == 1) | ||
82 | return; | ||
83 | |||
84 | if (msg->address == TLB_FLUSH_ALL) { | ||
85 | local_flush_tlb(); | ||
86 | __get_cpu_var(ptcstats).alltlb++; | ||
87 | } else { | ||
88 | __flush_tlb_one(msg->address); | ||
89 | __get_cpu_var(ptcstats).onetlb++; | ||
90 | } | ||
91 | |||
92 | __get_cpu_var(ptcstats).requestee++; | ||
93 | |||
94 | atomic_inc_short(&msg->acknowledge_count); | ||
95 | if (msg->number_of_cpus == msg->acknowledge_count) | ||
96 | uv_reply_to_message(sw_ack_slot, msg, msp); | ||
97 | return; | ||
98 | } | ||
99 | |||
100 | /* | ||
101 | * Examine the payload queue on all the distribution nodes to see | ||
102 | * which messages have not been seen, and which cpu(s) have not seen them. | ||
103 | * | ||
104 | * Returns the number of cpu's that have not responded. | ||
105 | */ | ||
106 | static int | ||
107 | uv_examine_destinations(struct bau_target_nodemask *distribution) | ||
108 | { | ||
109 | int sender; | ||
110 | int i; | ||
111 | int j; | ||
112 | int k; | ||
113 | int count = 0; | ||
114 | struct bau_control *bau_tablesp; | ||
115 | struct bau_payload_queue_entry *msg; | ||
116 | struct bau_msg_status *msp; | ||
117 | |||
118 | sender = smp_processor_id(); | ||
119 | for (i = 0; i < (sizeof(struct bau_target_nodemask) * BITSPERBYTE); | ||
120 | i++) { | ||
121 | if (bau_node_isset(i, distribution)) { | ||
122 | bau_tablesp = uv_bau_table_bases[i]; | ||
123 | for (msg = bau_tablesp->va_queue_first, j = 0; | ||
124 | j < DESTINATION_PAYLOAD_QUEUE_SIZE; msg++, j++) { | ||
125 | if ((msg->sending_cpu == sender) && | ||
126 | (!msg->replied_to)) { | ||
127 | msp = bau_tablesp->msg_statuses + j; | ||
128 | printk(KERN_DEBUG | ||
129 | "blade %d: address:%#lx %d of %d, not cpu(s): ", | ||
130 | i, msg->address, | ||
131 | msg->acknowledge_count, | ||
132 | msg->number_of_cpus); | ||
133 | for (k = 0; k < msg->number_of_cpus; | ||
134 | k++) { | ||
135 | if (!((long)1 << k & msp-> | ||
136 | seen_by.bits)) { | ||
137 | count++; | ||
138 | printk("%d ", k); | ||
139 | } | ||
140 | } | ||
141 | printk("\n"); | ||
142 | } | ||
143 | } | ||
144 | } | ||
145 | } | ||
146 | return count; | ||
147 | } | ||
148 | |||
149 | /** | ||
150 | * uv_flush_tlb_others - globally purge translation cache of a virtual | ||
151 | * address or all TLB's | ||
152 | * @cpumaskp: mask of all cpu's in which the address is to be removed | ||
153 | * @mm: mm_struct containing virtual address range | ||
154 | * @va: virtual address to be removed (or TLB_FLUSH_ALL for all TLB's on cpu) | ||
155 | * | ||
156 | * This is the entry point for initiating any UV global TLB shootdown. | ||
157 | * | ||
158 | * Purges the translation caches of all specified processors of the given | ||
159 | * virtual address, or purges all TLB's on specified processors. | ||
160 | * | ||
161 | * The caller has derived the cpumaskp from the mm_struct and has subtracted | ||
162 | * the local cpu from the mask. This function is called only if there | ||
163 | * are bits set in the mask. (e.g. flush_tlb_page()) | ||
164 | * | ||
165 | * The cpumaskp is converted into a nodemask of the nodes containing | ||
166 | * the cpus. | ||
167 | */ | ||
168 | int | ||
169 | uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm, unsigned long va) | ||
170 | { | ||
171 | int i; | ||
172 | int blade; | ||
173 | int cpu; | ||
174 | int bit; | ||
175 | int right_shift; | ||
176 | int this_blade; | ||
177 | int exams = 0; | ||
178 | int tries = 0; | ||
179 | long source_timeouts = 0; | ||
180 | long destination_timeouts = 0; | ||
181 | unsigned long index; | ||
182 | unsigned long mmr_offset; | ||
183 | unsigned long descriptor_status; | ||
184 | struct bau_activation_descriptor *bau_desc; | ||
185 | ktime_t time1, time2; | ||
186 | |||
187 | cpu = uv_blade_processor_id(); | ||
188 | this_blade = uv_numa_blade_id(); | ||
189 | bau_desc = __get_cpu_var(bau_control).descriptor_base; | ||
190 | bau_desc += (UV_ITEMS_PER_DESCRIPTOR * cpu); | ||
191 | |||
192 | bau_nodes_clear(&bau_desc->distribution, UV_DISTRIBUTION_SIZE); | ||
193 | |||
194 | i = 0; | ||
195 | for_each_cpu_mask(bit, *cpumaskp) { | ||
196 | blade = uv_cpu_to_blade_id(bit); | ||
197 | if (blade > (UV_DISTRIBUTION_SIZE - 1)) | ||
198 | BUG(); | ||
199 | if (blade == this_blade) | ||
200 | continue; | ||
201 | bau_node_set(blade, &bau_desc->distribution); | ||
202 | /* leave the bits for the remote cpu's in the mask until | ||
203 | success; on failure we fall back to the IPI method */ | ||
204 | i++; | ||
205 | } | ||
206 | if (i == 0) | ||
207 | goto none_to_flush; | ||
208 | __get_cpu_var(ptcstats).requestor++; | ||
209 | __get_cpu_var(ptcstats).ntargeted += i; | ||
210 | |||
211 | bau_desc->payload.address = va; | ||
212 | bau_desc->payload.sending_cpu = smp_processor_id(); | ||
213 | |||
214 | if (cpu < UV_CPUS_PER_ACT_STATUS) { | ||
215 | mmr_offset = UVH_LB_BAU_SB_ACTIVATION_STATUS_0; | ||
216 | right_shift = cpu * UV_ACT_STATUS_SIZE; | ||
217 | } else { | ||
218 | mmr_offset = UVH_LB_BAU_SB_ACTIVATION_STATUS_1; | ||
219 | right_shift = | ||
220 | ((cpu - UV_CPUS_PER_ACT_STATUS) * UV_ACT_STATUS_SIZE); | ||
221 | } | ||
222 | time1 = ktime_get(); | ||
223 | |||
224 | retry: | ||
225 | tries++; | ||
226 | index = ((unsigned long) | ||
227 | 1 << UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT) | cpu; | ||
228 | uv_write_local_mmr(UVH_LB_BAU_SB_ACTIVATION_CONTROL, index); | ||
229 | |||
230 | while ((descriptor_status = (((unsigned long) | ||
231 | uv_read_local_mmr(mmr_offset) >> | ||
232 | right_shift) & UV_ACT_STATUS_MASK)) != | ||
233 | DESC_STATUS_IDLE) { | ||
234 | if (descriptor_status == DESC_STATUS_SOURCE_TIMEOUT) { | ||
235 | source_timeouts++; | ||
236 | if (source_timeouts > SOURCE_TIMEOUT_LIMIT) | ||
237 | source_timeouts = 0; | ||
238 | __get_cpu_var(ptcstats).s_retry++; | ||
239 | goto retry; | ||
240 | } | ||
241 | /* spin here looking for progress at the destinations */ | ||
242 | if (descriptor_status == DESC_STATUS_DESTINATION_TIMEOUT) { | ||
243 | destination_timeouts++; | ||
244 | if (destination_timeouts > DESTINATION_TIMEOUT_LIMIT) { | ||
245 | /* returns # of cpus not responding */ | ||
246 | if (uv_examine_destinations | ||
247 | (&bau_desc->distribution) == 0) { | ||
248 | __get_cpu_var(ptcstats).d_retry++; | ||
249 | goto retry; | ||
250 | } | ||
251 | exams++; | ||
252 | if (exams >= uv_bau_retry_limit) { | ||
253 | printk(KERN_DEBUG | ||
254 | "uv_flush_tlb_others"); | ||
255 | printk("giving up on cpu %d\n", | ||
256 | smp_processor_id()); | ||
257 | goto unsuccessful; | ||
258 | } | ||
259 | /* delays can hang up the simulator | ||
260 | udelay(1000); | ||
261 | */ | ||
262 | destination_timeouts = 0; | ||
263 | } | ||
264 | } | ||
265 | } | ||
266 | if (tries > 1) | ||
267 | __get_cpu_var(ptcstats).retriesok++; | ||
268 | /* on success, clear the remote cpu's from the mask so we don't | ||
269 | use the IPI method of shootdown on them */ | ||
270 | for_each_cpu_mask(bit, *cpumaskp) { | ||
271 | blade = uv_cpu_to_blade_id(bit); | ||
272 | if (blade == this_blade) | ||
273 | continue; | ||
274 | cpu_clear(bit, *cpumaskp); | ||
275 | } | ||
276 | |||
277 | unsuccessful: | ||
278 | time2 = ktime_get(); | ||
279 | __get_cpu_var(ptcstats).sflush_ns += (time2.tv64 - time1.tv64); | ||
280 | |||
281 | none_to_flush: | ||
282 | if (cpus_empty(*cpumaskp)) | ||
283 | return 1; | ||
284 | |||
285 | /* Cause the caller to do an IPI-style TLB shootdown on | ||
286 | the cpu's still in the mask */ | ||
287 | __get_cpu_var(ptcstats).ptc_i++; | ||
288 | return 0; | ||
289 | } | ||
290 | |||
291 | /* | ||
292 | * The BAU message interrupt comes here. (registered by set_intr_gate) | ||
293 | * See entry_64.S | ||
294 | * | ||
295 | * We received a broadcast assist message. | ||
296 | * | ||
297 | * Interrupts may have been disabled; this interrupt could represent | ||
298 | * the receipt of several messages. | ||
299 | * | ||
300 | * All cores/threads on this node get this interrupt. | ||
301 | * The last one to see it does the s/w ack. | ||
302 | * (the resource will not be freed until noninterruptable cpus see this | ||
303 | * interrupt; hardware will timeout the s/w ack and reply ERROR) | ||
304 | */ | ||
305 | void | ||
306 | uv_bau_message_interrupt(struct pt_regs *regs) | ||
307 | { | ||
308 | struct bau_payload_queue_entry *pqp; | ||
309 | struct bau_payload_queue_entry *msg; | ||
310 | struct pt_regs *old_regs = set_irq_regs(regs); | ||
311 | ktime_t time1, time2; | ||
312 | int msg_slot; | ||
313 | int sw_ack_slot; | ||
314 | int fw; | ||
315 | int count = 0; | ||
316 | unsigned long local_pnode; | ||
317 | |||
318 | ack_APIC_irq(); | ||
319 | exit_idle(); | ||
320 | irq_enter(); | ||
321 | |||
322 | time1 = ktime_get(); | ||
323 | |||
324 | local_pnode = uv_blade_to_pnode(uv_numa_blade_id()); | ||
325 | |||
326 | pqp = __get_cpu_var(bau_control).va_queue_first; | ||
327 | msg = __get_cpu_var(bau_control).bau_msg_head; | ||
328 | while (msg->sw_ack_vector) { | ||
329 | count++; | ||
330 | fw = msg->sw_ack_vector; | ||
331 | msg_slot = msg - pqp; | ||
332 | sw_ack_slot = ffs(fw) - 1; | ||
333 | |||
334 | uv_bau_process_message(msg, msg_slot, sw_ack_slot); | ||
335 | |||
336 | msg++; | ||
337 | if (msg > __get_cpu_var(bau_control).va_queue_last) | ||
338 | msg = __get_cpu_var(bau_control).va_queue_first; | ||
339 | __get_cpu_var(bau_control).bau_msg_head = msg; | ||
340 | } | ||
341 | if (!count) | ||
342 | __get_cpu_var(ptcstats).nomsg++; | ||
343 | else if (count > 1) | ||
344 | __get_cpu_var(ptcstats).multmsg++; | ||
345 | |||
346 | time2 = ktime_get(); | ||
347 | __get_cpu_var(ptcstats).dflush_ns += (time2.tv64 - time1.tv64); | ||
348 | |||
349 | irq_exit(); | ||
350 | set_irq_regs(old_regs); | ||
351 | return; | ||
352 | } | ||
353 | |||
354 | static void | ||
355 | uv_enable_timeouts(void) | ||
356 | { | ||
357 | int i; | ||
358 | int blade; | ||
359 | int last_blade; | ||
360 | int pnode; | ||
361 | int cur_cpu = 0; | ||
362 | unsigned long apicid; | ||
363 | |||
364 | /* better if we had each_online_blade */ | ||
365 | last_blade = -1; | ||
366 | for_each_online_node(i) { | ||
367 | blade = uv_node_to_blade_id(i); | ||
368 | if (blade == last_blade) | ||
369 | continue; | ||
370 | last_blade = blade; | ||
371 | apicid = per_cpu(x86_cpu_to_apicid, cur_cpu); | ||
372 | pnode = uv_blade_to_pnode(blade); | ||
373 | cur_cpu += uv_blade_nr_possible_cpus(i); | ||
374 | } | ||
375 | return; | ||
376 | } | ||
377 | |||
378 | static void * | ||
379 | uv_ptc_seq_start(struct seq_file *file, loff_t *offset) | ||
380 | { | ||
381 | if (*offset < num_possible_cpus()) | ||
382 | return offset; | ||
383 | return NULL; | ||
384 | } | ||
385 | |||
386 | static void * | ||
387 | uv_ptc_seq_next(struct seq_file *file, void *data, loff_t *offset) | ||
388 | { | ||
389 | (*offset)++; | ||
390 | if (*offset < num_possible_cpus()) | ||
391 | return offset; | ||
392 | return NULL; | ||
393 | } | ||
394 | |||
395 | static void | ||
396 | uv_ptc_seq_stop(struct seq_file *file, void *data) | ||
397 | { | ||
398 | } | ||
399 | |||
400 | /* | ||
401 | * Display the statistics thru /proc | ||
402 | * data points to the cpu number | ||
403 | */ | ||
404 | static int | ||
405 | uv_ptc_seq_show(struct seq_file *file, void *data) | ||
406 | { | ||
407 | struct ptc_stats *stat; | ||
408 | int cpu; | ||
409 | |||
410 | cpu = *(loff_t *)data; | ||
411 | |||
412 | if (!cpu) { | ||
413 | seq_printf(file, | ||
414 | "# cpu requestor requestee one all sretry dretry ptc_i "); | ||
415 | seq_printf(file, | ||
416 | "sw_ack sflush_us dflush_us sok dnomsg dmult starget\n"); | ||
417 | } | ||
418 | if (cpu < num_possible_cpus() && cpu_online(cpu)) { | ||
419 | stat = &per_cpu(ptcstats, cpu); | ||
420 | seq_printf(file, "cpu %d %ld %ld %ld %ld %ld %ld %ld ", | ||
421 | cpu, stat->requestor, | ||
422 | stat->requestee, stat->onetlb, stat->alltlb, | ||
423 | stat->s_retry, stat->d_retry, stat->ptc_i); | ||
424 | seq_printf(file, "%lx %ld %ld %ld %ld %ld %ld\n", | ||
425 | uv_read_global_mmr64(uv_blade_to_pnode | ||
426 | (uv_cpu_to_blade_id(cpu)), | ||
427 | UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE), | ||
428 | stat->sflush_ns / 1000, stat->dflush_ns / 1000, | ||
429 | stat->retriesok, stat->nomsg, | ||
430 | stat->multmsg, stat->ntargeted); | ||
431 | } | ||
432 | |||
433 | return 0; | ||
434 | } | ||
435 | |||
436 | /* | ||
437 | * 0: display meaning of the statistics | ||
438 | * >0: retry limit | ||
439 | */ | ||
440 | static ssize_t | ||
441 | uv_ptc_proc_write(struct file *file, const char __user *user, | ||
442 | size_t count, loff_t *data) | ||
443 | { | ||
444 | long newmode; | ||
445 | char optstr[64]; | ||
446 | |||
447 | if (copy_from_user(optstr, user, count)) | ||
448 | return -EFAULT; | ||
449 | optstr[count - 1] = '\0'; | ||
450 | if (strict_strtoul(optstr, 10, &newmode) < 0) { | ||
451 | printk(KERN_DEBUG "%s is invalid\n", optstr); | ||
452 | return -EINVAL; | ||
453 | } | ||
454 | |||
455 | if (newmode == 0) { | ||
456 | printk(KERN_DEBUG "# cpu: cpu number\n"); | ||
457 | printk(KERN_DEBUG | ||
458 | "requestor: times this cpu was the flush requestor\n"); | ||
459 | printk(KERN_DEBUG | ||
460 | "requestee: times this cpu was requested to flush its TLBs\n"); | ||
461 | printk(KERN_DEBUG | ||
462 | "one: times requested to flush a single address\n"); | ||
463 | printk(KERN_DEBUG | ||
464 | "all: times requested to flush all TLB's\n"); | ||
465 | printk(KERN_DEBUG | ||
466 | "sretry: number of retries of source-side timeouts\n"); | ||
467 | printk(KERN_DEBUG | ||
468 | "dretry: number of retries of destination-side timeouts\n"); | ||
469 | printk(KERN_DEBUG | ||
470 | "ptc_i: times UV fell through to IPI-style flushes\n"); | ||
471 | printk(KERN_DEBUG | ||
472 | "sw_ack: image of UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE\n"); | ||
473 | printk(KERN_DEBUG | ||
474 | "sflush_us: microseconds spent in uv_flush_tlb_others()\n"); | ||
475 | printk(KERN_DEBUG | ||
476 | "dflush_us: microseconds spent in handling flush requests\n"); | ||
477 | printk(KERN_DEBUG "sok: successes on retry\n"); | ||
478 | printk(KERN_DEBUG "dnomsg: interrupts with no message\n"); | ||
479 | printk(KERN_DEBUG | ||
480 | "dmult: interrupts with multiple messages\n"); | ||
481 | printk(KERN_DEBUG "starget: nodes targeted\n"); | ||
482 | } else { | ||
483 | uv_bau_retry_limit = newmode; | ||
484 | printk(KERN_DEBUG "timeout retry limit:%d\n", | ||
485 | uv_bau_retry_limit); | ||
486 | } | ||
487 | |||
488 | return count; | ||
489 | } | ||
490 | |||
491 | static const struct seq_operations uv_ptc_seq_ops = { | ||
492 | .start = uv_ptc_seq_start, | ||
493 | .next = uv_ptc_seq_next, | ||
494 | .stop = uv_ptc_seq_stop, | ||
495 | .show = uv_ptc_seq_show | ||
496 | }; | ||
497 | |||
498 | static int | ||
499 | uv_ptc_proc_open(struct inode *inode, struct file *file) | ||
500 | { | ||
501 | return seq_open(file, &uv_ptc_seq_ops); | ||
502 | } | ||
503 | |||
504 | static const struct file_operations proc_uv_ptc_operations = { | ||
505 | .open = uv_ptc_proc_open, | ||
506 | .read = seq_read, | ||
507 | .write = uv_ptc_proc_write, | ||
508 | .llseek = seq_lseek, | ||
509 | .release = seq_release, | ||
510 | }; | ||
511 | |||
512 | static struct proc_dir_entry *proc_uv_ptc; | ||
513 | |||
514 | static int __init | ||
515 | uv_ptc_init(void) | ||
516 | { | ||
517 | static struct proc_dir_entry *sgi_proc_dir; | ||
518 | |||
519 | sgi_proc_dir = NULL; | ||
520 | |||
521 | if (!is_uv_system()) | ||
522 | return 0; | ||
523 | |||
524 | sgi_proc_dir = proc_mkdir("sgi_uv", NULL); | ||
525 | if (!sgi_proc_dir) | ||
526 | return -EINVAL; | ||
527 | |||
528 | proc_uv_ptc = create_proc_entry(UV_PTC_BASENAME, 0444, NULL); | ||
529 | if (!proc_uv_ptc) { | ||
530 | printk(KERN_ERR "unable to create %s proc entry\n", | ||
531 | UV_PTC_BASENAME); | ||
532 | return -EINVAL; | ||
533 | } | ||
534 | proc_uv_ptc->proc_fops = &proc_uv_ptc_operations; | ||
535 | return 0; | ||
536 | } | ||
537 | |||
538 | static void __exit | ||
539 | uv_ptc_exit(void) | ||
540 | { | ||
541 | remove_proc_entry(UV_PTC_BASENAME, NULL); | ||
542 | } | ||
543 | |||
544 | module_init(uv_ptc_init); | ||
545 | module_exit(uv_ptc_exit); | ||
546 | |||
547 | /* | ||
548 | * Initialization of BAU-related structures | ||
549 | */ | ||
550 | int __init | ||
551 | uv_bau_init(void) | ||
552 | { | ||
553 | int i; | ||
554 | int j; | ||
555 | int blade; | ||
556 | int nblades; | ||
557 | int *ip; | ||
558 | int pnode; | ||
559 | int last_blade; | ||
560 | int cur_cpu = 0; | ||
561 | unsigned long pa; | ||
562 | unsigned long n; | ||
563 | unsigned long m; | ||
564 | unsigned long mmr_image; | ||
565 | unsigned long apicid; | ||
566 | char *cp; | ||
567 | struct bau_control *bau_tablesp; | ||
568 | struct bau_activation_descriptor *adp, *ad2; | ||
569 | struct bau_payload_queue_entry *pqp; | ||
570 | struct bau_msg_status *msp; | ||
571 | struct bau_control *bcp; | ||
572 | |||
573 | if (!is_uv_system()) | ||
574 | return 0; | ||
575 | |||
576 | uv_bau_retry_limit = 1; | ||
577 | |||
578 | if ((sizeof(struct bau_local_cpumask) * BITSPERBYTE) < | ||
579 | MAX_CPUS_PER_NODE) { | ||
580 | printk(KERN_ERR | ||
581 | "uv_bau_init: bau_local_cpumask.bits too small\n"); | ||
582 | BUG(); | ||
583 | } | ||
584 | |||
585 | uv_nshift = uv_hub_info->n_val; | ||
586 | uv_mmask = ((unsigned long)1 << uv_hub_info->n_val) - 1; | ||
587 | nblades = 0; | ||
588 | last_blade = -1; | ||
589 | for_each_online_node(i) { | ||
590 | blade = uv_node_to_blade_id(i); | ||
591 | if (blade == last_blade) | ||
592 | continue; | ||
593 | last_blade = blade; | ||
594 | nblades++; | ||
595 | } | ||
596 | |||
597 | uv_bau_table_bases = (struct bau_control **) | ||
598 | kmalloc(nblades * sizeof(struct bau_control *), GFP_KERNEL); | ||
599 | if (!uv_bau_table_bases) | ||
600 | BUG(); | ||
601 | |||
602 | /* better if we had each_online_blade */ | ||
603 | last_blade = -1; | ||
604 | for_each_online_node(i) { | ||
605 | blade = uv_node_to_blade_id(i); | ||
606 | if (blade == last_blade) | ||
607 | continue; | ||
608 | last_blade = blade; | ||
609 | |||
610 | bau_tablesp = | ||
611 | kmalloc_node(sizeof(struct bau_control), GFP_KERNEL, i); | ||
612 | if (!bau_tablesp) | ||
613 | BUG(); | ||
614 | |||
615 | bau_tablesp->msg_statuses = | ||
616 | kmalloc_node(sizeof(struct bau_msg_status) * | ||
617 | DESTINATION_PAYLOAD_QUEUE_SIZE, GFP_KERNEL, i); | ||
618 | if (!bau_tablesp->msg_statuses) | ||
619 | BUG(); | ||
620 | for (j = 0, msp = bau_tablesp->msg_statuses; | ||
621 | j < DESTINATION_PAYLOAD_QUEUE_SIZE; j++, msp++) { | ||
622 | bau_cpubits_clear(&msp->seen_by, (int) | ||
623 | uv_blade_nr_possible_cpus(blade)); | ||
624 | } | ||
625 | |||
626 | bau_tablesp->watching = | ||
627 | kmalloc_node(sizeof(int) * DESTINATION_NUM_RESOURCES, | ||
628 | GFP_KERNEL, i); | ||
629 | if (!bau_tablesp->watching) | ||
630 | BUG(); | ||
631 | for (j = 0, ip = bau_tablesp->watching; | ||
632 | j < DESTINATION_PAYLOAD_QUEUE_SIZE; j++, ip++) { | ||
633 | *ip = 0; | ||
634 | } | ||
635 | |||
636 | uv_bau_table_bases[i] = bau_tablesp; | ||
637 | |||
638 | pnode = uv_blade_to_pnode(blade); | ||
639 | |||
640 | if (sizeof(struct bau_activation_descriptor) != 64) | ||
641 | BUG(); | ||
642 | |||
643 | adp = (struct bau_activation_descriptor *) | ||
644 | kmalloc_node(16384, GFP_KERNEL, i); | ||
645 | if (!adp) | ||
646 | BUG(); | ||
647 | if ((unsigned long)adp & 0xfff) | ||
648 | BUG(); | ||
649 | pa = __pa((unsigned long)adp); | ||
650 | n = pa >> uv_nshift; | ||
651 | m = pa & uv_mmask; | ||
652 | |||
653 | mmr_image = uv_read_global_mmr64(pnode, | ||
654 | UVH_LB_BAU_SB_DESCRIPTOR_BASE); | ||
655 | if (mmr_image) | ||
656 | uv_write_global_mmr64(pnode, (unsigned long) | ||
657 | UVH_LB_BAU_SB_DESCRIPTOR_BASE, | ||
658 | (n << UV_DESC_BASE_PNODE_SHIFT | | ||
659 | m)); | ||
660 | for (j = 0, ad2 = adp; j < UV_ACTIVATION_DESCRIPTOR_SIZE; | ||
661 | j++, ad2++) { | ||
662 | memset(ad2, 0, | ||
663 | sizeof(struct bau_activation_descriptor)); | ||
664 | ad2->header.sw_ack_flag = 1; | ||
665 | ad2->header.base_dest_nodeid = | ||
666 | uv_blade_to_pnode(uv_cpu_to_blade_id(0)); | ||
667 | ad2->header.command = UV_NET_ENDPOINT_INTD; | ||
668 | ad2->header.int_both = 1; | ||
669 | /* all others need to be set to zero: | ||
670 | fairness chaining multilevel count replied_to */ | ||
671 | } | ||
672 | |||
673 | pqp = (struct bau_payload_queue_entry *) | ||
674 | kmalloc_node((DESTINATION_PAYLOAD_QUEUE_SIZE + 1) * | ||
675 | sizeof(struct bau_payload_queue_entry), | ||
676 | GFP_KERNEL, i); | ||
677 | if (!pqp) | ||
678 | BUG(); | ||
679 | if (sizeof(struct bau_payload_queue_entry) != 32) | ||
680 | BUG(); | ||
681 | if ((unsigned long)(&((struct bau_payload_queue_entry *)0)-> | ||
682 | sw_ack_vector) != 15) | ||
683 | BUG(); | ||
684 | |||
685 | cp = (char *)pqp + 31; | ||
686 | pqp = (struct bau_payload_queue_entry *) | ||
687 | (((unsigned long)cp >> 5) << 5); | ||
688 | bau_tablesp->va_queue_first = pqp; | ||
689 | uv_write_global_mmr64(pnode, | ||
690 | UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST, | ||
691 | ((unsigned long)pnode << | ||
692 | UV_PAYLOADQ_PNODE_SHIFT) | | ||
693 | uv_physnodeaddr(pqp)); | ||
694 | uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL, | ||
695 | uv_physnodeaddr(pqp)); | ||
696 | bau_tablesp->va_queue_last = | ||
697 | pqp + (DESTINATION_PAYLOAD_QUEUE_SIZE - 1); | ||
698 | uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST, | ||
699 | (unsigned long) | ||
700 | uv_physnodeaddr(bau_tablesp-> | ||
701 | va_queue_last)); | ||
702 | memset(pqp, 0, sizeof(struct bau_payload_queue_entry) * | ||
703 | DESTINATION_PAYLOAD_QUEUE_SIZE); | ||
704 | |||
705 | /* this initialization can't be in firmware because the | ||
706 | messaging IRQ will be determined by the OS */ | ||
707 | apicid = per_cpu(x86_cpu_to_apicid, cur_cpu); | ||
708 | pa = uv_read_global_mmr64(pnode, UVH_BAU_DATA_CONFIG); | ||
709 | if ((pa & 0xff) != UV_BAU_MESSAGE) { | ||
710 | uv_write_global_mmr64(pnode, UVH_BAU_DATA_CONFIG, | ||
711 | ((apicid << 32) | | ||
712 | UV_BAU_MESSAGE)); | ||
713 | } | ||
714 | |||
715 | for (j = cur_cpu; j < (cur_cpu + uv_blade_nr_possible_cpus(i)); | ||
716 | j++) { | ||
717 | bcp = (struct bau_control *)&per_cpu(bau_control, j); | ||
718 | bcp->bau_msg_head = bau_tablesp->va_queue_first; | ||
719 | bcp->va_queue_first = bau_tablesp->va_queue_first; | ||
720 | |||
721 | bcp->va_queue_last = bau_tablesp->va_queue_last; | ||
722 | bcp->watching = bau_tablesp->watching; | ||
723 | bcp->msg_statuses = bau_tablesp->msg_statuses; | ||
724 | bcp->descriptor_base = adp; | ||
725 | } | ||
726 | cur_cpu += uv_blade_nr_possible_cpus(i); | ||
727 | } | ||
728 | |||
729 | set_intr_gate(UV_BAU_MESSAGE, uv_bau_message_intr1); | ||
730 | |||
731 | uv_enable_timeouts(); | ||
732 | |||
733 | return 0; | ||
734 | } | ||
735 | |||
736 | __initcall(uv_bau_init); | ||
diff --git a/include/asm-x86/uv/uv_bau.h b/include/asm-x86/uv/uv_bau.h new file mode 100644 index 000000000000..f125f86c89ac --- /dev/null +++ b/include/asm-x86/uv/uv_bau.h | |||
@@ -0,0 +1,331 @@ | |||
1 | /* | ||
2 | * This file is subject to the terms and conditions of the GNU General Public | ||
3 | * License. See the file "COPYING" in the main directory of this archive | ||
4 | * for more details. | ||
5 | * | ||
6 | * SGI UV Broadcast Assist Unit definitions | ||
7 | * | ||
8 | * Copyright (C) 2008 Silicon Graphics, Inc. All rights reserved. | ||
9 | */ | ||
10 | |||
11 | #ifndef __ASM_X86_UV_BAU__ | ||
12 | #define __ASM_X86_UV_BAU__ | ||
13 | |||
14 | #include <linux/bitmap.h> | ||
15 | #define BITSPERBYTE 8 | ||
16 | |||
17 | /* Broadcast Assist Unit messaging structures */ | ||
18 | |||
19 | /* | ||
20 | * Selective Broadcast activations are induced by software action | ||
21 | * specifying a particular 8-descriptor "set" via a 6-bit index written | ||
22 | * to an MMR. | ||
23 | * Thus there are 64 unique 512-byte sets of SB descriptors - one set for | ||
24 | * each 6-bit index value. These descriptor sets are mapped in sequence | ||
25 | * starting with set 0 located at the address specified in the | ||
26 | * BAU_SB_DESCRIPTOR_BASE register, set 1 is located at BASE + 512, | ||
27 | * set 2 is at BASE + 2*512, set 3 at BASE + 3*512, and so on. | ||
28 | * | ||
29 | * We will use 31 sets, one for sending BAU messages from each of the 32 | ||
30 | * cpu's on the node. | ||
31 | * | ||
32 | * TLB shootdown will use the first of the 8 descriptors of each set. | ||
33 | * Each of the descriptors is 64 bytes in size (8*64 = 512 bytes in a set). | ||
34 | */ | ||
35 | |||
36 | #define UV_ITEMS_PER_DESCRIPTOR 8 | ||
37 | #define UV_CPUS_PER_ACT_STATUS 32 | ||
38 | #define UV_ACT_STATUS_MASK 0x3 | ||
39 | #define UV_ACT_STATUS_SIZE 2 | ||
40 | #define UV_ACTIVATION_DESCRIPTOR_SIZE 32 | ||
41 | #define UV_DISTRIBUTION_SIZE 256 | ||
42 | #define UV_SW_ACK_NPENDING 8 | ||
43 | #define UV_BAU_MESSAGE 200 /* Messaging irq; see irq_64.h */ | ||
44 | /* and include/asm-x86/hw_irq_64.h */ | ||
45 | /* To be dynamically allocated in the future */ | ||
46 | #define UV_NET_ENDPOINT_INTD 0x38 | ||
47 | #define UV_DESC_BASE_PNODE_SHIFT 49 /* position of pnode (nasid>>1) in MMR */ | ||
48 | #define UV_PAYLOADQ_PNODE_SHIFT 49 | ||
49 | |||
50 | #define UV_PTC_BASENAME "sgi_uv/ptc_statistics" | ||
51 | #define uv_physnodeaddr(x) ((__pa((unsigned long)(x)) & uv_mmask)) | ||
52 | |||
53 | /* bits in UVH_LB_BAU_SB_ACTIVATION_STATUS_0/1 */ | ||
54 | #define DESC_STATUS_IDLE 0 | ||
55 | #define DESC_STATUS_ACTIVE 1 | ||
56 | #define DESC_STATUS_DESTINATION_TIMEOUT 2 | ||
57 | #define DESC_STATUS_SOURCE_TIMEOUT 3 | ||
58 | |||
59 | /* source side threshholds at which message retries print a warning */ | ||
60 | #define SOURCE_TIMEOUT_LIMIT 20 | ||
61 | #define DESTINATION_TIMEOUT_LIMIT 20 | ||
62 | |||
63 | /* number of entries in the destination side payload queue */ | ||
64 | #define DESTINATION_PAYLOAD_QUEUE_SIZE 17 | ||
65 | /* number of destination side software ack resources */ | ||
66 | #define DESTINATION_NUM_RESOURCES 8 | ||
67 | #define MAX_CPUS_PER_NODE 32 | ||
68 | |||
69 | /* Distribution: 32 bytes (256 bits) (bytes 0-0x1f of descriptor) */ | ||
70 | /* If the 'multilevel' flag in the header portion of the descriptor | ||
71 | * has been set to 0, then endpoint multi-unicast mode is selected. | ||
72 | * The distribution specification (32 bytes) is interpreted as a 256-bit | ||
73 | * distribution vector. Adjacent bits correspond to consecutive even numbered | ||
74 | * nodeIDs. The result of adding the index of a given bit to the 15-bit | ||
75 | * 'base_dest_nodeid' field of the header corresponds to the | ||
76 | * destination nodeID associated with that specified bit. */ | ||
77 | struct bau_target_nodemask { | ||
78 | unsigned long bits[BITS_TO_LONGS(256)]; | ||
79 | }; | ||
80 | |||
81 | /* mask of cpu's on a node */ | ||
82 | /* (during initialization we need to check that unsigned long has | ||
83 | enough bits for max. cpu's per node) */ | ||
84 | struct bau_local_cpumask { | ||
85 | unsigned long bits; | ||
86 | }; | ||
87 | |||
88 | /* | ||
89 | * Payload: 16 bytes (128 bits) (bytes 0x20-0x2f of descriptor) | ||
90 | * only 12 bytes (96 bits) of the payload area are usable. | ||
91 | * An additional 3 bytes (bits 27:4) of the header address are carried | ||
92 | * to the next bytes of the destination payload queue. | ||
93 | * And an additional 2 bytes of the header Suppl_A field are also | ||
94 | * carried to the destination payload queue. | ||
95 | * But the first byte of the Suppl_A becomes bits 127:120 (the 16th byte) | ||
96 | * of the destination payload queue, which is written by the hardware | ||
97 | * with the s/w ack resource bit vector. | ||
98 | * [ effective message contents (16 bytes (128 bits) maximum), not counting | ||
99 | * the s/w ack bit vector ] | ||
100 | */ | ||
101 | |||
102 | /* The payload is software-defined for INTD transactions */ | ||
103 | struct bau_msg_payload { | ||
104 | unsigned long address; /* signifies a page or all TLB's | ||
105 | of the cpu */ | ||
106 | /* 64 bits */ | ||
107 | unsigned short sending_cpu; /* filled in by sender */ | ||
108 | /* 16 bits */ | ||
109 | unsigned short acknowledge_count;/* filled in by destination */ | ||
110 | /* 16 bits */ | ||
111 | unsigned int reserved1:32; /* not usable */ | ||
112 | }; | ||
113 | |||
114 | |||
115 | /* Message header: 16 bytes (128 bits) (bytes 0x30-0x3f of descriptor) */ | ||
116 | /* see table 4.2.3.0.1 in broacast_assist spec. */ | ||
117 | struct bau_msg_header { | ||
118 | int dest_subnodeid:6; /* must be zero */ | ||
119 | /* bits 5:0 */ | ||
120 | int base_dest_nodeid:15; /* nasid>>1 (pnode) of first bit in node_map */ | ||
121 | /* bits 20:6 */ | ||
122 | int command:8; /* message type */ | ||
123 | /* bits 28:21 */ | ||
124 | /* 0x38: SN3net EndPoint Message */ | ||
125 | int rsvd_1:3; /* must be zero */ | ||
126 | /* bits 31:29 */ | ||
127 | /* int will align on 32 bits */ | ||
128 | int rsvd_2:9; /* must be zero */ | ||
129 | /* bits 40:32 */ | ||
130 | /* Suppl_A is 56-41 */ | ||
131 | int payload_2a:8; /* becomes byte 16 of msg */ | ||
132 | /* bits 48:41 */ /* not currently using */ | ||
133 | int payload_2b:8; /* becomes byte 17 of msg */ | ||
134 | /* bits 56:49 */ /* not currently using */ | ||
135 | /* Address field (96:57) is never used as an | ||
136 | address (these are address bits 42:3) */ | ||
137 | int rsvd_3:1; /* must be zero */ | ||
138 | /* bit 57 */ | ||
139 | /* address bits 27:4 are payload */ | ||
140 | /* these 24 bits become bytes 12-14 of msg */ | ||
141 | int replied_to:1; /* sent as 0 by the source to byte 12 */ | ||
142 | /* bit 58 */ | ||
143 | |||
144 | int payload_1a:5; /* not currently used */ | ||
145 | /* bits 63:59 */ | ||
146 | int payload_1b:8; /* not currently used */ | ||
147 | /* bits 71:64 */ | ||
148 | int payload_1c:8; /* not currently used */ | ||
149 | /* bits 79:72 */ | ||
150 | int payload_1d:2; /* not currently used */ | ||
151 | /* bits 81:80 */ | ||
152 | |||
153 | int rsvd_4:7; /* must be zero */ | ||
154 | /* bits 88:82 */ | ||
155 | int sw_ack_flag:1; /* software acknowledge flag */ | ||
156 | /* bit 89 */ | ||
157 | /* INTD trasactions at destination are to | ||
158 | wait for software acknowledge */ | ||
159 | int rsvd_5:6; /* must be zero */ | ||
160 | /* bits 95:90 */ | ||
161 | int rsvd_6:5; /* must be zero */ | ||
162 | /* bits 100:96 */ | ||
163 | int int_both:1; /* if 1, interrupt both sockets on the blade */ | ||
164 | /* bit 101*/ | ||
165 | int fairness:3; /* usually zero */ | ||
166 | /* bits 104:102 */ | ||
167 | int multilevel:1; /* multi-level multicast format */ | ||
168 | /* bit 105 */ | ||
169 | /* 0 for TLB: endpoint multi-unicast messages */ | ||
170 | int chaining:1; /* next descriptor is part of this activation*/ | ||
171 | /* bit 106 */ | ||
172 | int rsvd_7:21; /* must be zero */ | ||
173 | /* bits 127:107 */ | ||
174 | }; | ||
175 | |||
176 | /* The format of the message to send, plus all accompanying control */ | ||
177 | /* Should be 64 bytes */ | ||
178 | struct bau_activation_descriptor { | ||
179 | struct bau_target_nodemask distribution; | ||
180 | /* message template, consisting of header and payload: */ | ||
181 | struct bau_msg_header header; | ||
182 | struct bau_msg_payload payload; | ||
183 | }; | ||
184 | /* | ||
185 | * -payload-- ---------header------ | ||
186 | * bytes 0-11 bits 41-56 bits 58-81 | ||
187 | * A B (2) C (3) | ||
188 | * | ||
189 | * A/B/C are moved to: | ||
190 | * A C B | ||
191 | * bytes 0-11 bytes 12-14 bytes 16-17 (byte 15 filled in by hw as vector) | ||
192 | * ------------payload queue----------- | ||
193 | */ | ||
194 | |||
195 | /* | ||
196 | * The payload queue on the destination side is an array of these. | ||
197 | * With BAU_MISC_CONTROL set for software acknowledge mode, the messages | ||
198 | * are 32 bytes (2 micropackets) (256 bits) in length, but contain only 17 | ||
199 | * bytes of usable data, including the sw ack vector in byte 15 (bits 127:120) | ||
200 | * (12 bytes come from bau_msg_payload, 3 from payload_1, 2 from | ||
201 | * sw_ack_vector and payload_2) | ||
202 | * "Enabling Software Acknowledgment mode (see Section 4.3.3 Software | ||
203 | * Acknowledge Processing) also selects 32 byte (17 bytes usable) payload | ||
204 | * operation." | ||
205 | */ | ||
206 | struct bau_payload_queue_entry { | ||
207 | unsigned long address; /* signifies a page or all TLB's | ||
208 | of the cpu */ | ||
209 | /* 64 bits, bytes 0-7 */ | ||
210 | |||
211 | unsigned short sending_cpu; /* cpu that sent the message */ | ||
212 | /* 16 bits, bytes 8-9 */ | ||
213 | |||
214 | unsigned short acknowledge_count; /* filled in by destination */ | ||
215 | /* 16 bits, bytes 10-11 */ | ||
216 | |||
217 | unsigned short replied_to:1; /* sent as 0 by the source */ | ||
218 | /* 1 bit */ | ||
219 | unsigned short unused1:7; /* not currently using */ | ||
220 | /* 7 bits: byte 12) */ | ||
221 | |||
222 | unsigned char unused2[2]; /* not currently using */ | ||
223 | /* bytes 13-14 */ | ||
224 | |||
225 | unsigned char sw_ack_vector; /* filled in by the hardware */ | ||
226 | /* byte 15 (bits 127:120) */ | ||
227 | |||
228 | unsigned char unused4[3]; /* not currently using bytes 17-19 */ | ||
229 | /* bytes 17-19 */ | ||
230 | |||
231 | int number_of_cpus; /* filled in at destination */ | ||
232 | /* 32 bits, bytes 20-23 (aligned) */ | ||
233 | |||
234 | unsigned char unused5[8]; /* not using */ | ||
235 | /* bytes 24-31 */ | ||
236 | }; | ||
237 | |||
238 | /* one for every slot in the destination payload queue */ | ||
239 | struct bau_msg_status { | ||
240 | struct bau_local_cpumask seen_by; /* map of cpu's */ | ||
241 | }; | ||
242 | |||
243 | /* one for every slot in the destination software ack resources */ | ||
244 | struct bau_sw_ack_status { | ||
245 | struct bau_payload_queue_entry *msg; /* associated message */ | ||
246 | int watcher; /* cpu monitoring, or -1 */ | ||
247 | }; | ||
248 | |||
249 | /* one on every node and per-cpu; to locate the software tables */ | ||
250 | struct bau_control { | ||
251 | struct bau_activation_descriptor *descriptor_base; | ||
252 | struct bau_payload_queue_entry *bau_msg_head; | ||
253 | struct bau_payload_queue_entry *va_queue_first; | ||
254 | struct bau_payload_queue_entry *va_queue_last; | ||
255 | struct bau_msg_status *msg_statuses; | ||
256 | int *watching; /* pointer to array */ | ||
257 | }; | ||
258 | |||
259 | /* | ||
260 | * This structure is allocated per_cpu for UV TLB shootdown statistics. | ||
261 | */ | ||
262 | struct ptc_stats { | ||
263 | unsigned long ptc_i; /* number of IPI-style flushes */ | ||
264 | unsigned long requestor; /* number of nodes this cpu sent to */ | ||
265 | unsigned long requestee; /* times cpu was remotely requested */ | ||
266 | unsigned long alltlb; /* times all tlb's on this cpu were flushed */ | ||
267 | unsigned long onetlb; /* times just one tlb on this cpu was flushed */ | ||
268 | unsigned long s_retry; /* retries on source side timeouts */ | ||
269 | unsigned long d_retry; /* retries on destination side timeouts */ | ||
270 | unsigned long sflush_ns;/* nanoseconds spent in uv_flush_tlb_others */ | ||
271 | unsigned long dflush_ns;/* nanoseconds spent destination side */ | ||
272 | unsigned long retriesok; /* successes on retries */ | ||
273 | unsigned long nomsg; /* interrupts with no message */ | ||
274 | unsigned long multmsg; /* interrupts with multiple messages */ | ||
275 | unsigned long ntargeted;/* nodes targeted */ | ||
276 | }; | ||
277 | |||
278 | static inline int bau_node_isset(int node, struct bau_target_nodemask *dstp) | ||
279 | { | ||
280 | return constant_test_bit(node, &dstp->bits[0]); | ||
281 | } | ||
282 | static inline void bau_node_set(int node, struct bau_target_nodemask *dstp) | ||
283 | { | ||
284 | __set_bit(node, &dstp->bits[0]); | ||
285 | } | ||
286 | static inline void bau_nodes_clear(struct bau_target_nodemask *dstp, int nbits) | ||
287 | { | ||
288 | bitmap_zero(&dstp->bits[0], nbits); | ||
289 | } | ||
290 | |||
291 | static inline void bau_cpubits_clear(struct bau_local_cpumask *dstp, int nbits) | ||
292 | { | ||
293 | bitmap_zero(&dstp->bits, nbits); | ||
294 | } | ||
295 | |||
296 | /* | ||
297 | * atomic increment of a short integer | ||
298 | * (rather than using the __sync_add_and_fetch() intrinsic) | ||
299 | * | ||
300 | * returns the new value of the variable | ||
301 | */ | ||
302 | static inline short int atomic_inc_short(short int *v) | ||
303 | { | ||
304 | asm volatile("movw $1, %%cx\n" | ||
305 | "lock ; xaddw %%cx, %0\n" | ||
306 | : "+m" (*v) /* outputs */ | ||
307 | : : "%cx", "memory"); /* inputs : clobbereds */ | ||
308 | return *v; | ||
309 | } | ||
310 | |||
311 | /* | ||
312 | * atomic OR of two long integers | ||
313 | * (rather than using the __sync_or_and_fetch() intrinsic) | ||
314 | */ | ||
315 | static inline void atomic_or_long(unsigned long *v1, unsigned long v2) | ||
316 | { | ||
317 | asm volatile("movq %0, %%rax; lea %1, %%rdx\n" | ||
318 | "lock ; orq %%rax, %%rdx\n" | ||
319 | : "+m" (*v1) /* outputs */ | ||
320 | : "m" (v1), "m" (v2) /* inputs */ | ||
321 | : "memory"); /* clobbereds */ | ||
322 | } | ||
323 | |||
324 | #define cpubit_isset(cpu, bau_local_cpumask) \ | ||
325 | test_bit((cpu), (bau_local_cpumask).bits) | ||
326 | |||
327 | int uv_flush_tlb_others(cpumask_t *, struct mm_struct *, unsigned long); | ||
328 | void uv_bau_message_intr1(void); | ||
329 | void uv_bau_timeout_intr1(void); | ||
330 | |||
331 | #endif /* __ASM_X86_UV_BAU__ */ | ||