aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/kernel
diff options
context:
space:
mode:
authorCliff Wickman <cpw@sgi.com>2008-06-12 09:23:48 -0400
committerIngo Molnar <mingo@elte.hu>2008-07-08 06:23:24 -0400
commitb194b120507276b4f09e2e14f941884e777fc7c8 (patch)
tree2796da3608f770bae6382941ef95e90e889359d2 /arch/x86/kernel
parent73e991f45fe7644711c0c9dd357a1a2c6e222707 (diff)
SGI UV: TLB shootdown using broadcast assist unit, cleanups
TLB shootdown for SGI UV. v1: 6/2 original v2: 6/3 corrections/improvements per Ingo's review v3: 6/4 split atomic operations off to a separate patch (Jeremy's review) v4: 6/12 include <mach_apic.h> rather than <asm/mach-bigsmp/mach_apic.h> (fixes a !SMP build problem that Ingo found) fix the index on uv_table_bases[blade] Signed-off-by: Cliff Wickman <cpw@sgi.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
Diffstat (limited to 'arch/x86/kernel')
-rw-r--r--arch/x86/kernel/tlb_64.c2
-rw-r--r--arch/x86/kernel/tlb_uv.c704
2 files changed, 378 insertions, 328 deletions
diff --git a/arch/x86/kernel/tlb_64.c b/arch/x86/kernel/tlb_64.c
index fc132113bdab..5039d0f097a2 100644
--- a/arch/x86/kernel/tlb_64.c
+++ b/arch/x86/kernel/tlb_64.c
@@ -165,7 +165,7 @@ void native_flush_tlb_others(const cpumask_t *cpumaskp, struct mm_struct *mm,
165 cpumask_t cpumask = *cpumaskp; 165 cpumask_t cpumask = *cpumaskp;
166 166
167 if (is_uv_system() && uv_flush_tlb_others(&cpumask, mm, va)) 167 if (is_uv_system() && uv_flush_tlb_others(&cpumask, mm, va))
168 return; 168 return;
169 169
170 /* Caller has disabled preemption */ 170 /* Caller has disabled preemption */
171 sender = smp_processor_id() % NUM_INVALIDATE_TLB_VECTORS; 171 sender = smp_processor_id() % NUM_INVALIDATE_TLB_VECTORS;
diff --git a/arch/x86/kernel/tlb_uv.c b/arch/x86/kernel/tlb_uv.c
index 28e7c68d9d78..f7bc6a6fbe49 100644
--- a/arch/x86/kernel/tlb_uv.c
+++ b/arch/x86/kernel/tlb_uv.c
@@ -10,18 +10,20 @@
10#include <linux/proc_fs.h> 10#include <linux/proc_fs.h>
11#include <linux/kernel.h> 11#include <linux/kernel.h>
12 12
13#include <asm/mach-bigsmp/mach_apic.h>
14#include <asm/mmu_context.h> 13#include <asm/mmu_context.h>
15#include <asm/idle.h> 14#include <asm/idle.h>
16#include <asm/genapic.h> 15#include <asm/genapic.h>
17#include <asm/uv/uv_hub.h> 16#include <asm/uv/uv_hub.h>
18#include <asm/uv/uv_mmrs.h> 17#include <asm/uv/uv_mmrs.h>
19#include <asm/uv/uv_bau.h> 18#include <asm/uv/uv_bau.h>
19#include <asm/tsc.h>
20 20
21struct bau_control **uv_bau_table_bases; 21#include <mach_apic.h>
22static int uv_bau_retry_limit; 22
23static int uv_nshift; /* position of pnode (which is nasid>>1) */ 23static struct bau_control **uv_bau_table_bases __read_mostly;
24static unsigned long uv_mmask; 24static int uv_bau_retry_limit __read_mostly;
25static int uv_nshift __read_mostly; /* position of pnode (which is nasid>>1) */
26static unsigned long uv_mmask __read_mostly;
25 27
26char *status_table[] = { 28char *status_table[] = {
27 "IDLE", 29 "IDLE",
@@ -41,19 +43,18 @@ DEFINE_PER_CPU(struct bau_control, bau_control);
41 * clear of the Timeout bit (as well) will free the resource. No reply will 43 * clear of the Timeout bit (as well) will free the resource. No reply will
42 * be sent (the hardware will only do one reply per message). 44 * be sent (the hardware will only do one reply per message).
43 */ 45 */
44static void 46static void uv_reply_to_message(int resource,
45uv_reply_to_message(int resource,
46 struct bau_payload_queue_entry *msg, 47 struct bau_payload_queue_entry *msg,
47 struct bau_msg_status *msp) 48 struct bau_msg_status *msp)
48{ 49{
49 int fw; 50 unsigned long dw;
50 51
51 fw = (1 << (resource + UV_SW_ACK_NPENDING)) | (1 << resource); 52 dw = (1 << (resource + UV_SW_ACK_NPENDING)) | (1 << resource);
52 msg->replied_to = 1; 53 msg->replied_to = 1;
53 msg->sw_ack_vector = 0; 54 msg->sw_ack_vector = 0;
54 if (msp) 55 if (msp)
55 msp->seen_by.bits = 0; 56 msp->seen_by.bits = 0;
56 uv_write_local_mmr(UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS, fw); 57 uv_write_local_mmr(UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS, dw);
57 return; 58 return;
58} 59}
59 60
@@ -61,8 +62,7 @@ uv_reply_to_message(int resource,
61 * Do all the things a cpu should do for a TLB shootdown message. 62 * Do all the things a cpu should do for a TLB shootdown message.
62 * Other cpu's may come here at the same time for this message. 63 * Other cpu's may come here at the same time for this message.
63 */ 64 */
64static void 65static void uv_bau_process_message(struct bau_payload_queue_entry *msg,
65uv_bau_process_message(struct bau_payload_queue_entry *msg,
66 int msg_slot, int sw_ack_slot) 66 int msg_slot, int sw_ack_slot)
67{ 67{
68 int cpu; 68 int cpu;
@@ -103,8 +103,7 @@ uv_bau_process_message(struct bau_payload_queue_entry *msg,
103 * 103 *
104 * Returns the number of cpu's that have not responded. 104 * Returns the number of cpu's that have not responded.
105 */ 105 */
106static int 106static int uv_examine_destinations(struct bau_target_nodemask *distribution)
107uv_examine_destinations(struct bau_target_nodemask *distribution)
108{ 107{
109 int sender; 108 int sender;
110 int i; 109 int i;
@@ -118,34 +117,161 @@ uv_examine_destinations(struct bau_target_nodemask *distribution)
118 sender = smp_processor_id(); 117 sender = smp_processor_id();
119 for (i = 0; i < (sizeof(struct bau_target_nodemask) * BITSPERBYTE); 118 for (i = 0; i < (sizeof(struct bau_target_nodemask) * BITSPERBYTE);
120 i++) { 119 i++) {
121 if (bau_node_isset(i, distribution)) { 120 if (!bau_node_isset(i, distribution))
122 bau_tablesp = uv_bau_table_bases[i]; 121 continue;
123 for (msg = bau_tablesp->va_queue_first, j = 0; 122 bau_tablesp = uv_bau_table_bases[i];
124 j < DESTINATION_PAYLOAD_QUEUE_SIZE; msg++, j++) { 123 for (msg = bau_tablesp->va_queue_first, j = 0;
125 if ((msg->sending_cpu == sender) && 124 j < DESTINATION_PAYLOAD_QUEUE_SIZE; msg++, j++) {
126 (!msg->replied_to)) { 125 if ((msg->sending_cpu == sender) &&
127 msp = bau_tablesp->msg_statuses + j; 126 (!msg->replied_to)) {
128 printk(KERN_DEBUG 127 msp = bau_tablesp->msg_statuses + j;
128 printk(KERN_DEBUG
129 "blade %d: address:%#lx %d of %d, not cpu(s): ", 129 "blade %d: address:%#lx %d of %d, not cpu(s): ",
130 i, msg->address, 130 i, msg->address,
131 msg->acknowledge_count, 131 msg->acknowledge_count,
132 msg->number_of_cpus); 132 msg->number_of_cpus);
133 for (k = 0; k < msg->number_of_cpus; 133 for (k = 0; k < msg->number_of_cpus;
134 k++) { 134 k++) {
135 if (!((long)1 << k & msp-> 135 if (!((long)1 << k & msp->
136 seen_by.bits)) { 136 seen_by.bits)) {
137 count++; 137 count++;
138 printk("%d ", k); 138 printk("%d ", k);
139 }
140 } 139 }
141 printk("\n");
142 } 140 }
141 printk("\n");
143 } 142 }
144 } 143 }
145 } 144 }
146 return count; 145 return count;
147} 146}
148 147
148/*
149 * wait for completion of a broadcast message
150 *
151 * return COMPLETE, RETRY or GIVEUP
152 */
153static int uv_wait_completion(struct bau_activation_descriptor *bau_desc,
154 unsigned long mmr_offset, int right_shift)
155{
156 int exams = 0;
157 long destination_timeouts = 0;
158 long source_timeouts = 0;
159 unsigned long descriptor_status;
160
161 while ((descriptor_status = (((unsigned long)
162 uv_read_local_mmr(mmr_offset) >>
163 right_shift) & UV_ACT_STATUS_MASK)) !=
164 DESC_STATUS_IDLE) {
165 if (descriptor_status == DESC_STATUS_SOURCE_TIMEOUT) {
166 source_timeouts++;
167 if (source_timeouts > SOURCE_TIMEOUT_LIMIT)
168 source_timeouts = 0;
169 __get_cpu_var(ptcstats).s_retry++;
170 return FLUSH_RETRY;
171 }
172 /*
173 * spin here looking for progress at the destinations
174 */
175 if (descriptor_status == DESC_STATUS_DESTINATION_TIMEOUT) {
176 destination_timeouts++;
177 if (destination_timeouts > DESTINATION_TIMEOUT_LIMIT) {
178 /*
179 * returns number of cpus not responding
180 */
181 if (uv_examine_destinations
182 (&bau_desc->distribution) == 0) {
183 __get_cpu_var(ptcstats).d_retry++;
184 return FLUSH_RETRY;
185 }
186 exams++;
187 if (exams >= uv_bau_retry_limit) {
188 printk(KERN_DEBUG
189 "uv_flush_tlb_others");
190 printk("giving up on cpu %d\n",
191 smp_processor_id());
192 return FLUSH_GIVEUP;
193 }
194 /*
195 * delays can hang the simulator
196 udelay(1000);
197 */
198 destination_timeouts = 0;
199 }
200 }
201 }
202 return FLUSH_COMPLETE;
203}
204
205/**
206 * uv_flush_send_and_wait
207 *
208 * Send a broadcast and wait for a broadcast message to complete.
209 *
210 * The cpumaskp mask contains the cpus the broadcast was sent to.
211 *
212 * Returns 1 if all remote flushing was done. The mask is zeroed.
213 * Returns 0 if some remote flushing remains to be done. The mask is left
214 * unchanged.
215 */
216int uv_flush_send_and_wait(int cpu, int this_blade,
217 struct bau_activation_descriptor *bau_desc, cpumask_t *cpumaskp)
218{
219 int completion_status = 0;
220 int right_shift;
221 int bit;
222 int blade;
223 int tries = 0;
224 unsigned long index;
225 unsigned long mmr_offset;
226 cycles_t time1;
227 cycles_t time2;
228
229 if (cpu < UV_CPUS_PER_ACT_STATUS) {
230 mmr_offset = UVH_LB_BAU_SB_ACTIVATION_STATUS_0;
231 right_shift = cpu * UV_ACT_STATUS_SIZE;
232 } else {
233 mmr_offset = UVH_LB_BAU_SB_ACTIVATION_STATUS_1;
234 right_shift =
235 ((cpu - UV_CPUS_PER_ACT_STATUS) * UV_ACT_STATUS_SIZE);
236 }
237 time1 = get_cycles();
238 do {
239 tries++;
240 index = ((unsigned long)
241 1 << UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT) | cpu;
242 uv_write_local_mmr(UVH_LB_BAU_SB_ACTIVATION_CONTROL, index);
243 completion_status = uv_wait_completion(bau_desc, mmr_offset,
244 right_shift);
245 } while (completion_status == FLUSH_RETRY);
246 time2 = get_cycles();
247 __get_cpu_var(ptcstats).sflush += (time2 - time1);
248 if (tries > 1)
249 __get_cpu_var(ptcstats).retriesok++;
250
251 if (completion_status == FLUSH_GIVEUP) {
252 /*
253 * Cause the caller to do an IPI-style TLB shootdown on
254 * the cpu's, all of which are still in the mask.
255 */
256 __get_cpu_var(ptcstats).ptc_i++;
257 return 0;
258 }
259
260 /*
261 * Success, so clear the remote cpu's from the mask so we don't
262 * use the IPI method of shootdown on them.
263 */
264 for_each_cpu_mask(bit, *cpumaskp) {
265 blade = uv_cpu_to_blade_id(bit);
266 if (blade == this_blade)
267 continue;
268 cpu_clear(bit, *cpumaskp);
269 }
270 if (!cpus_empty(*cpumaskp))
271 return 0;
272 return 1;
273}
274
149/** 275/**
150 * uv_flush_tlb_others - globally purge translation cache of a virtual 276 * uv_flush_tlb_others - globally purge translation cache of a virtual
151 * address or all TLB's 277 * address or all TLB's
@@ -164,30 +290,25 @@ uv_examine_destinations(struct bau_target_nodemask *distribution)
164 * 290 *
165 * The cpumaskp is converted into a nodemask of the nodes containing 291 * The cpumaskp is converted into a nodemask of the nodes containing
166 * the cpus. 292 * the cpus.
293 *
294 * Returns 1 if all remote flushing was done.
295 * Returns 0 if some remote flushing remains to be done.
167 */ 296 */
168int 297int uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm,
169uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm, unsigned long va) 298 unsigned long va)
170{ 299{
171 int i; 300 int i;
301 int bit;
172 int blade; 302 int blade;
173 int cpu; 303 int cpu;
174 int bit;
175 int right_shift;
176 int this_blade; 304 int this_blade;
177 int exams = 0; 305 int locals = 0;
178 int tries = 0;
179 long source_timeouts = 0;
180 long destination_timeouts = 0;
181 unsigned long index;
182 unsigned long mmr_offset;
183 unsigned long descriptor_status;
184 struct bau_activation_descriptor *bau_desc; 306 struct bau_activation_descriptor *bau_desc;
185 ktime_t time1, time2;
186 307
187 cpu = uv_blade_processor_id(); 308 cpu = uv_blade_processor_id();
188 this_blade = uv_numa_blade_id(); 309 this_blade = uv_numa_blade_id();
189 bau_desc = __get_cpu_var(bau_control).descriptor_base; 310 bau_desc = __get_cpu_var(bau_control).descriptor_base;
190 bau_desc += (UV_ITEMS_PER_DESCRIPTOR * cpu); 311 bau_desc += UV_ITEMS_PER_DESCRIPTOR * cpu;
191 312
192 bau_nodes_clear(&bau_desc->distribution, UV_DISTRIBUTION_SIZE); 313 bau_nodes_clear(&bau_desc->distribution, UV_DISTRIBUTION_SIZE);
193 314
@@ -196,96 +317,29 @@ uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm, unsigned long va)
196 blade = uv_cpu_to_blade_id(bit); 317 blade = uv_cpu_to_blade_id(bit);
197 if (blade > (UV_DISTRIBUTION_SIZE - 1)) 318 if (blade > (UV_DISTRIBUTION_SIZE - 1))
198 BUG(); 319 BUG();
199 if (blade == this_blade) 320 if (blade == this_blade) {
321 locals++;
200 continue; 322 continue;
323 }
201 bau_node_set(blade, &bau_desc->distribution); 324 bau_node_set(blade, &bau_desc->distribution);
202 /* leave the bits for the remote cpu's in the mask until
203 success; on failure we fall back to the IPI method */
204 i++; 325 i++;
205 } 326 }
206 if (i == 0) 327 if (i == 0) {
207 goto none_to_flush; 328 /*
329 * no off_node flushing; return status for local node
330 */
331 if (locals)
332 return 0;
333 else
334 return 1;
335 }
208 __get_cpu_var(ptcstats).requestor++; 336 __get_cpu_var(ptcstats).requestor++;
209 __get_cpu_var(ptcstats).ntargeted += i; 337 __get_cpu_var(ptcstats).ntargeted += i;
210 338
211 bau_desc->payload.address = va; 339 bau_desc->payload.address = va;
212 bau_desc->payload.sending_cpu = smp_processor_id(); 340 bau_desc->payload.sending_cpu = smp_processor_id();
213 341
214 if (cpu < UV_CPUS_PER_ACT_STATUS) { 342 return uv_flush_send_and_wait(cpu, this_blade, bau_desc, cpumaskp);
215 mmr_offset = UVH_LB_BAU_SB_ACTIVATION_STATUS_0;
216 right_shift = cpu * UV_ACT_STATUS_SIZE;
217 } else {
218 mmr_offset = UVH_LB_BAU_SB_ACTIVATION_STATUS_1;
219 right_shift =
220 ((cpu - UV_CPUS_PER_ACT_STATUS) * UV_ACT_STATUS_SIZE);
221 }
222 time1 = ktime_get();
223
224retry:
225 tries++;
226 index = ((unsigned long)
227 1 << UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT) | cpu;
228 uv_write_local_mmr(UVH_LB_BAU_SB_ACTIVATION_CONTROL, index);
229
230 while ((descriptor_status = (((unsigned long)
231 uv_read_local_mmr(mmr_offset) >>
232 right_shift) & UV_ACT_STATUS_MASK)) !=
233 DESC_STATUS_IDLE) {
234 if (descriptor_status == DESC_STATUS_SOURCE_TIMEOUT) {
235 source_timeouts++;
236 if (source_timeouts > SOURCE_TIMEOUT_LIMIT)
237 source_timeouts = 0;
238 __get_cpu_var(ptcstats).s_retry++;
239 goto retry;
240 }
241 /* spin here looking for progress at the destinations */
242 if (descriptor_status == DESC_STATUS_DESTINATION_TIMEOUT) {
243 destination_timeouts++;
244 if (destination_timeouts > DESTINATION_TIMEOUT_LIMIT) {
245 /* returns # of cpus not responding */
246 if (uv_examine_destinations
247 (&bau_desc->distribution) == 0) {
248 __get_cpu_var(ptcstats).d_retry++;
249 goto retry;
250 }
251 exams++;
252 if (exams >= uv_bau_retry_limit) {
253 printk(KERN_DEBUG
254 "uv_flush_tlb_others");
255 printk("giving up on cpu %d\n",
256 smp_processor_id());
257 goto unsuccessful;
258 }
259 /* delays can hang up the simulator
260 udelay(1000);
261 */
262 destination_timeouts = 0;
263 }
264 }
265 }
266 if (tries > 1)
267 __get_cpu_var(ptcstats).retriesok++;
268 /* on success, clear the remote cpu's from the mask so we don't
269 use the IPI method of shootdown on them */
270 for_each_cpu_mask(bit, *cpumaskp) {
271 blade = uv_cpu_to_blade_id(bit);
272 if (blade == this_blade)
273 continue;
274 cpu_clear(bit, *cpumaskp);
275 }
276
277unsuccessful:
278 time2 = ktime_get();
279 __get_cpu_var(ptcstats).sflush_ns += (time2.tv64 - time1.tv64);
280
281none_to_flush:
282 if (cpus_empty(*cpumaskp))
283 return 1;
284
285 /* Cause the caller to do an IPI-style TLB shootdown on
286 the cpu's still in the mask */
287 __get_cpu_var(ptcstats).ptc_i++;
288 return 0;
289} 343}
290 344
291/* 345/*
@@ -302,13 +356,12 @@ none_to_flush:
302 * (the resource will not be freed until noninterruptable cpus see this 356 * (the resource will not be freed until noninterruptable cpus see this
303 * interrupt; hardware will timeout the s/w ack and reply ERROR) 357 * interrupt; hardware will timeout the s/w ack and reply ERROR)
304 */ 358 */
305void 359void uv_bau_message_interrupt(struct pt_regs *regs)
306uv_bau_message_interrupt(struct pt_regs *regs)
307{ 360{
308 struct bau_payload_queue_entry *pqp; 361 struct bau_payload_queue_entry *pqp;
309 struct bau_payload_queue_entry *msg; 362 struct bau_payload_queue_entry *msg;
310 struct pt_regs *old_regs = set_irq_regs(regs); 363 struct pt_regs *old_regs = set_irq_regs(regs);
311 ktime_t time1, time2; 364 cycles_t time1, time2;
312 int msg_slot; 365 int msg_slot;
313 int sw_ack_slot; 366 int sw_ack_slot;
314 int fw; 367 int fw;
@@ -319,7 +372,7 @@ uv_bau_message_interrupt(struct pt_regs *regs)
319 exit_idle(); 372 exit_idle();
320 irq_enter(); 373 irq_enter();
321 374
322 time1 = ktime_get(); 375 time1 = get_cycles();
323 376
324 local_pnode = uv_blade_to_pnode(uv_numa_blade_id()); 377 local_pnode = uv_blade_to_pnode(uv_numa_blade_id());
325 378
@@ -343,16 +396,15 @@ uv_bau_message_interrupt(struct pt_regs *regs)
343 else if (count > 1) 396 else if (count > 1)
344 __get_cpu_var(ptcstats).multmsg++; 397 __get_cpu_var(ptcstats).multmsg++;
345 398
346 time2 = ktime_get(); 399 time2 = get_cycles();
347 __get_cpu_var(ptcstats).dflush_ns += (time2.tv64 - time1.tv64); 400 __get_cpu_var(ptcstats).dflush += (time2 - time1);
348 401
349 irq_exit(); 402 irq_exit();
350 set_irq_regs(old_regs); 403 set_irq_regs(old_regs);
351 return; 404 return;
352} 405}
353 406
354static void 407static void uv_enable_timeouts(void)
355uv_enable_timeouts(void)
356{ 408{
357 int i; 409 int i;
358 int blade; 410 int blade;
@@ -361,7 +413,6 @@ uv_enable_timeouts(void)
361 int cur_cpu = 0; 413 int cur_cpu = 0;
362 unsigned long apicid; 414 unsigned long apicid;
363 415
364 /* better if we had each_online_blade */
365 last_blade = -1; 416 last_blade = -1;
366 for_each_online_node(i) { 417 for_each_online_node(i) {
367 blade = uv_node_to_blade_id(i); 418 blade = uv_node_to_blade_id(i);
@@ -375,16 +426,14 @@ uv_enable_timeouts(void)
375 return; 426 return;
376} 427}
377 428
378static void * 429static void *uv_ptc_seq_start(struct seq_file *file, loff_t *offset)
379uv_ptc_seq_start(struct seq_file *file, loff_t *offset)
380{ 430{
381 if (*offset < num_possible_cpus()) 431 if (*offset < num_possible_cpus())
382 return offset; 432 return offset;
383 return NULL; 433 return NULL;
384} 434}
385 435
386static void * 436static void *uv_ptc_seq_next(struct seq_file *file, void *data, loff_t *offset)
387uv_ptc_seq_next(struct seq_file *file, void *data, loff_t *offset)
388{ 437{
389 (*offset)++; 438 (*offset)++;
390 if (*offset < num_possible_cpus()) 439 if (*offset < num_possible_cpus())
@@ -392,8 +441,7 @@ uv_ptc_seq_next(struct seq_file *file, void *data, loff_t *offset)
392 return NULL; 441 return NULL;
393} 442}
394 443
395static void 444static void uv_ptc_seq_stop(struct seq_file *file, void *data)
396uv_ptc_seq_stop(struct seq_file *file, void *data)
397{ 445{
398} 446}
399 447
@@ -401,8 +449,7 @@ uv_ptc_seq_stop(struct seq_file *file, void *data)
401 * Display the statistics thru /proc 449 * Display the statistics thru /proc
402 * data points to the cpu number 450 * data points to the cpu number
403 */ 451 */
404static int 452static int uv_ptc_seq_show(struct seq_file *file, void *data)
405uv_ptc_seq_show(struct seq_file *file, void *data)
406{ 453{
407 struct ptc_stats *stat; 454 struct ptc_stats *stat;
408 int cpu; 455 int cpu;
@@ -413,7 +460,7 @@ uv_ptc_seq_show(struct seq_file *file, void *data)
413 seq_printf(file, 460 seq_printf(file,
414 "# cpu requestor requestee one all sretry dretry ptc_i "); 461 "# cpu requestor requestee one all sretry dretry ptc_i ");
415 seq_printf(file, 462 seq_printf(file,
416 "sw_ack sflush_us dflush_us sok dnomsg dmult starget\n"); 463 "sw_ack sflush dflush sok dnomsg dmult starget\n");
417 } 464 }
418 if (cpu < num_possible_cpus() && cpu_online(cpu)) { 465 if (cpu < num_possible_cpus() && cpu_online(cpu)) {
419 stat = &per_cpu(ptcstats, cpu); 466 stat = &per_cpu(ptcstats, cpu);
@@ -425,7 +472,7 @@ uv_ptc_seq_show(struct seq_file *file, void *data)
425 uv_read_global_mmr64(uv_blade_to_pnode 472 uv_read_global_mmr64(uv_blade_to_pnode
426 (uv_cpu_to_blade_id(cpu)), 473 (uv_cpu_to_blade_id(cpu)),
427 UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE), 474 UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE),
428 stat->sflush_ns / 1000, stat->dflush_ns / 1000, 475 stat->sflush, stat->dflush,
429 stat->retriesok, stat->nomsg, 476 stat->retriesok, stat->nomsg,
430 stat->multmsg, stat->ntargeted); 477 stat->multmsg, stat->ntargeted);
431 } 478 }
@@ -437,8 +484,7 @@ uv_ptc_seq_show(struct seq_file *file, void *data)
437 * 0: display meaning of the statistics 484 * 0: display meaning of the statistics
438 * >0: retry limit 485 * >0: retry limit
439 */ 486 */
440static ssize_t 487static ssize_t uv_ptc_proc_write(struct file *file, const char __user *user,
441uv_ptc_proc_write(struct file *file, const char __user *user,
442 size_t count, loff_t *data) 488 size_t count, loff_t *data)
443{ 489{
444 long newmode; 490 long newmode;
@@ -471,9 +517,9 @@ uv_ptc_proc_write(struct file *file, const char __user *user,
471 printk(KERN_DEBUG 517 printk(KERN_DEBUG
472 "sw_ack: image of UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE\n"); 518 "sw_ack: image of UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE\n");
473 printk(KERN_DEBUG 519 printk(KERN_DEBUG
474 "sflush_us: microseconds spent in uv_flush_tlb_others()\n"); 520 "sflush_us: cycles spent in uv_flush_tlb_others()\n");
475 printk(KERN_DEBUG 521 printk(KERN_DEBUG
476 "dflush_us: microseconds spent in handling flush requests\n"); 522 "dflush_us: cycles spent in handling flush requests\n");
477 printk(KERN_DEBUG "sok: successes on retry\n"); 523 printk(KERN_DEBUG "sok: successes on retry\n");
478 printk(KERN_DEBUG "dnomsg: interrupts with no message\n"); 524 printk(KERN_DEBUG "dnomsg: interrupts with no message\n");
479 printk(KERN_DEBUG 525 printk(KERN_DEBUG
@@ -489,40 +535,33 @@ uv_ptc_proc_write(struct file *file, const char __user *user,
489} 535}
490 536
491static const struct seq_operations uv_ptc_seq_ops = { 537static const struct seq_operations uv_ptc_seq_ops = {
492 .start = uv_ptc_seq_start, 538 .start = uv_ptc_seq_start,
493 .next = uv_ptc_seq_next, 539 .next = uv_ptc_seq_next,
494 .stop = uv_ptc_seq_stop, 540 .stop = uv_ptc_seq_stop,
495 .show = uv_ptc_seq_show 541 .show = uv_ptc_seq_show
496}; 542};
497 543
498static int 544static int uv_ptc_proc_open(struct inode *inode, struct file *file)
499uv_ptc_proc_open(struct inode *inode, struct file *file)
500{ 545{
501 return seq_open(file, &uv_ptc_seq_ops); 546 return seq_open(file, &uv_ptc_seq_ops);
502} 547}
503 548
504static const struct file_operations proc_uv_ptc_operations = { 549static const struct file_operations proc_uv_ptc_operations = {
505 .open = uv_ptc_proc_open, 550 .open = uv_ptc_proc_open,
506 .read = seq_read, 551 .read = seq_read,
507 .write = uv_ptc_proc_write, 552 .write = uv_ptc_proc_write,
508 .llseek = seq_lseek, 553 .llseek = seq_lseek,
509 .release = seq_release, 554 .release = seq_release,
510}; 555};
511 556
512static struct proc_dir_entry *proc_uv_ptc; 557static int __init uv_ptc_init(void)
513
514static int __init
515uv_ptc_init(void)
516{ 558{
517 static struct proc_dir_entry *sgi_proc_dir; 559 struct proc_dir_entry *proc_uv_ptc;
518
519 sgi_proc_dir = NULL;
520 560
521 if (!is_uv_system()) 561 if (!is_uv_system())
522 return 0; 562 return 0;
523 563
524 sgi_proc_dir = proc_mkdir("sgi_uv", NULL); 564 if (!proc_mkdir("sgi_uv", NULL))
525 if (!sgi_proc_dir)
526 return -EINVAL; 565 return -EINVAL;
527 566
528 proc_uv_ptc = create_proc_entry(UV_PTC_BASENAME, 0444, NULL); 567 proc_uv_ptc = create_proc_entry(UV_PTC_BASENAME, 0444, NULL);
@@ -535,202 +574,213 @@ uv_ptc_init(void)
535 return 0; 574 return 0;
536} 575}
537 576
538static void __exit 577/*
539uv_ptc_exit(void) 578 * begin the initialization of the per-blade control structures
579 */
580static struct bau_control * __init uv_table_bases_init(int blade, int node)
540{ 581{
541 remove_proc_entry(UV_PTC_BASENAME, NULL); 582 int i;
583 int *ip;
584 struct bau_msg_status *msp;
585 struct bau_control *bau_tablesp;
586
587 bau_tablesp =
588 kmalloc_node(sizeof(struct bau_control), GFP_KERNEL, node);
589 if (!bau_tablesp)
590 BUG();
591 bau_tablesp->msg_statuses =
592 kmalloc_node(sizeof(struct bau_msg_status) *
593 DESTINATION_PAYLOAD_QUEUE_SIZE, GFP_KERNEL, node);
594 if (!bau_tablesp->msg_statuses)
595 BUG();
596 for (i = 0, msp = bau_tablesp->msg_statuses;
597 i < DESTINATION_PAYLOAD_QUEUE_SIZE; i++, msp++) {
598 bau_cpubits_clear(&msp->seen_by, (int)
599 uv_blade_nr_possible_cpus(blade));
600 }
601 bau_tablesp->watching =
602 kmalloc_node(sizeof(int) * DESTINATION_NUM_RESOURCES,
603 GFP_KERNEL, node);
604 if (!bau_tablesp->watching)
605 BUG();
606 for (i = 0, ip = bau_tablesp->watching;
607 i < DESTINATION_PAYLOAD_QUEUE_SIZE; i++, ip++) {
608 *ip = 0;
609 }
610 uv_bau_table_bases[blade] = bau_tablesp;
611 return bau_tablesp;
542} 612}
543 613
544module_init(uv_ptc_init); 614/*
545module_exit(uv_ptc_exit); 615 * finish the initialization of the per-blade control structures
616 */
617static void __init uv_table_bases_finish(int blade, int node, int cur_cpu,
618 struct bau_control *bau_tablesp,
619 struct bau_activation_descriptor *adp)
620{
621 int i;
622 struct bau_control *bcp;
623
624 for (i = cur_cpu; i < (cur_cpu + uv_blade_nr_possible_cpus(blade));
625 i++) {
626 bcp = (struct bau_control *)&per_cpu(bau_control, i);
627 bcp->bau_msg_head = bau_tablesp->va_queue_first;
628 bcp->va_queue_first = bau_tablesp->va_queue_first;
629 bcp->va_queue_last = bau_tablesp->va_queue_last;
630 bcp->watching = bau_tablesp->watching;
631 bcp->msg_statuses = bau_tablesp->msg_statuses;
632 bcp->descriptor_base = adp;
633 }
634}
546 635
547/* 636/*
548 * Initialization of BAU-related structures 637 * initialize the sending side's sending buffers
549 */ 638 */
550int __init 639static struct bau_activation_descriptor * __init
551uv_bau_init(void) 640uv_activation_descriptor_init(int node, int pnode)
552{ 641{
553 int i; 642 int i;
554 int j;
555 int blade;
556 int nblades;
557 int *ip;
558 int pnode;
559 int last_blade;
560 int cur_cpu = 0;
561 unsigned long pa; 643 unsigned long pa;
562 unsigned long n;
563 unsigned long m; 644 unsigned long m;
645 unsigned long n;
564 unsigned long mmr_image; 646 unsigned long mmr_image;
565 unsigned long apicid; 647 struct bau_activation_descriptor *adp;
648 struct bau_activation_descriptor *ad2;
649
650 adp = (struct bau_activation_descriptor *)
651 kmalloc_node(16384, GFP_KERNEL, node);
652 if (!adp)
653 BUG();
654 pa = __pa((unsigned long)adp);
655 n = pa >> uv_nshift;
656 m = pa & uv_mmask;
657 mmr_image = uv_read_global_mmr64(pnode, UVH_LB_BAU_SB_DESCRIPTOR_BASE);
658 if (mmr_image)
659 uv_write_global_mmr64(pnode, (unsigned long)
660 UVH_LB_BAU_SB_DESCRIPTOR_BASE,
661 (n << UV_DESC_BASE_PNODE_SHIFT | m));
662 for (i = 0, ad2 = adp; i < UV_ACTIVATION_DESCRIPTOR_SIZE; i++, ad2++) {
663 memset(ad2, 0, sizeof(struct bau_activation_descriptor));
664 ad2->header.sw_ack_flag = 1;
665 ad2->header.base_dest_nodeid =
666 uv_blade_to_pnode(uv_cpu_to_blade_id(0));
667 ad2->header.command = UV_NET_ENDPOINT_INTD;
668 ad2->header.int_both = 1;
669 /*
670 * all others need to be set to zero:
671 * fairness chaining multilevel count replied_to
672 */
673 }
674 return adp;
675}
676
677/*
678 * initialize the destination side's receiving buffers
679 */
680static struct bau_payload_queue_entry * __init uv_payload_queue_init(int node,
681 int pnode, struct bau_control *bau_tablesp)
682{
566 char *cp; 683 char *cp;
567 struct bau_control *bau_tablesp;
568 struct bau_activation_descriptor *adp, *ad2;
569 struct bau_payload_queue_entry *pqp; 684 struct bau_payload_queue_entry *pqp;
570 struct bau_msg_status *msp;
571 struct bau_control *bcp;
572 685
573 if (!is_uv_system()) 686 pqp = (struct bau_payload_queue_entry *)
574 return 0; 687 kmalloc_node((DESTINATION_PAYLOAD_QUEUE_SIZE + 1) *
688 sizeof(struct bau_payload_queue_entry),
689 GFP_KERNEL, node);
690 if (!pqp)
691 BUG();
692 cp = (char *)pqp + 31;
693 pqp = (struct bau_payload_queue_entry *)(((unsigned long)cp >> 5) << 5);
694 bau_tablesp->va_queue_first = pqp;
695 uv_write_global_mmr64(pnode,
696 UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST,
697 ((unsigned long)pnode <<
698 UV_PAYLOADQ_PNODE_SHIFT) |
699 uv_physnodeaddr(pqp));
700 uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL,
701 uv_physnodeaddr(pqp));
702 bau_tablesp->va_queue_last =
703 pqp + (DESTINATION_PAYLOAD_QUEUE_SIZE - 1);
704 uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST,
705 (unsigned long)
706 uv_physnodeaddr(bau_tablesp->va_queue_last));
707 memset(pqp, 0, sizeof(struct bau_payload_queue_entry) *
708 DESTINATION_PAYLOAD_QUEUE_SIZE);
709 return pqp;
710}
575 711
576 uv_bau_retry_limit = 1; 712/*
713 * Initialization of each UV blade's structures
714 */
715static int __init uv_init_blade(int blade, int node, int cur_cpu)
716{
717 int pnode;
718 unsigned long pa;
719 unsigned long apicid;
720 struct bau_activation_descriptor *adp;
721 struct bau_payload_queue_entry *pqp;
722 struct bau_control *bau_tablesp;
577 723
578 if ((sizeof(struct bau_local_cpumask) * BITSPERBYTE) < 724 bau_tablesp = uv_table_bases_init(blade, node);
579 MAX_CPUS_PER_NODE) { 725 pnode = uv_blade_to_pnode(blade);
580 printk(KERN_ERR 726 adp = uv_activation_descriptor_init(node, pnode);
581 "uv_bau_init: bau_local_cpumask.bits too small\n"); 727 pqp = uv_payload_queue_init(node, pnode, bau_tablesp);
582 BUG(); 728 uv_table_bases_finish(blade, node, cur_cpu, bau_tablesp, adp);
729 /*
730 * the below initialization can't be in firmware because the
731 * messaging IRQ will be determined by the OS
732 */
733 apicid = per_cpu(x86_cpu_to_apicid, cur_cpu);
734 pa = uv_read_global_mmr64(pnode, UVH_BAU_DATA_CONFIG);
735 if ((pa & 0xff) != UV_BAU_MESSAGE) {
736 uv_write_global_mmr64(pnode, UVH_BAU_DATA_CONFIG,
737 ((apicid << 32) | UV_BAU_MESSAGE));
583 } 738 }
739 return 0;
740}
741
742/*
743 * Initialization of BAU-related structures
744 */
745static int __init uv_bau_init(void)
746{
747 int blade;
748 int node;
749 int nblades;
750 int last_blade;
751 int cur_cpu = 0;
752
753 if (!is_uv_system())
754 return 0;
584 755
756 uv_bau_retry_limit = 1;
585 uv_nshift = uv_hub_info->n_val; 757 uv_nshift = uv_hub_info->n_val;
586 uv_mmask = ((unsigned long)1 << uv_hub_info->n_val) - 1; 758 uv_mmask = ((unsigned long)1 << uv_hub_info->n_val) - 1;
587 nblades = 0; 759 nblades = 0;
588 last_blade = -1; 760 last_blade = -1;
589 for_each_online_node(i) { 761 for_each_online_node(node) {
590 blade = uv_node_to_blade_id(i); 762 blade = uv_node_to_blade_id(node);
591 if (blade == last_blade) 763 if (blade == last_blade)
592 continue; 764 continue;
593 last_blade = blade; 765 last_blade = blade;
594 nblades++; 766 nblades++;
595 } 767 }
596
597 uv_bau_table_bases = (struct bau_control **) 768 uv_bau_table_bases = (struct bau_control **)
598 kmalloc(nblades * sizeof(struct bau_control *), GFP_KERNEL); 769 kmalloc(nblades * sizeof(struct bau_control *), GFP_KERNEL);
599 if (!uv_bau_table_bases) 770 if (!uv_bau_table_bases)
600 BUG(); 771 BUG();
601
602 /* better if we had each_online_blade */
603 last_blade = -1; 772 last_blade = -1;
604 for_each_online_node(i) { 773 for_each_online_node(node) {
605 blade = uv_node_to_blade_id(i); 774 blade = uv_node_to_blade_id(node);
606 if (blade == last_blade) 775 if (blade == last_blade)
607 continue; 776 continue;
608 last_blade = blade; 777 last_blade = blade;
609 778 uv_init_blade(blade, node, cur_cpu);
610 bau_tablesp = 779 cur_cpu += uv_blade_nr_possible_cpus(blade);
611 kmalloc_node(sizeof(struct bau_control), GFP_KERNEL, i);
612 if (!bau_tablesp)
613 BUG();
614
615 bau_tablesp->msg_statuses =
616 kmalloc_node(sizeof(struct bau_msg_status) *
617 DESTINATION_PAYLOAD_QUEUE_SIZE, GFP_KERNEL, i);
618 if (!bau_tablesp->msg_statuses)
619 BUG();
620 for (j = 0, msp = bau_tablesp->msg_statuses;
621 j < DESTINATION_PAYLOAD_QUEUE_SIZE; j++, msp++) {
622 bau_cpubits_clear(&msp->seen_by, (int)
623 uv_blade_nr_possible_cpus(blade));
624 }
625
626 bau_tablesp->watching =
627 kmalloc_node(sizeof(int) * DESTINATION_NUM_RESOURCES,
628 GFP_KERNEL, i);
629 if (!bau_tablesp->watching)
630 BUG();
631 for (j = 0, ip = bau_tablesp->watching;
632 j < DESTINATION_PAYLOAD_QUEUE_SIZE; j++, ip++) {
633 *ip = 0;
634 }
635
636 uv_bau_table_bases[i] = bau_tablesp;
637
638 pnode = uv_blade_to_pnode(blade);
639
640 if (sizeof(struct bau_activation_descriptor) != 64)
641 BUG();
642
643 adp = (struct bau_activation_descriptor *)
644 kmalloc_node(16384, GFP_KERNEL, i);
645 if (!adp)
646 BUG();
647 if ((unsigned long)adp & 0xfff)
648 BUG();
649 pa = __pa((unsigned long)adp);
650 n = pa >> uv_nshift;
651 m = pa & uv_mmask;
652
653 mmr_image = uv_read_global_mmr64(pnode,
654 UVH_LB_BAU_SB_DESCRIPTOR_BASE);
655 if (mmr_image)
656 uv_write_global_mmr64(pnode, (unsigned long)
657 UVH_LB_BAU_SB_DESCRIPTOR_BASE,
658 (n << UV_DESC_BASE_PNODE_SHIFT |
659 m));
660 for (j = 0, ad2 = adp; j < UV_ACTIVATION_DESCRIPTOR_SIZE;
661 j++, ad2++) {
662 memset(ad2, 0,
663 sizeof(struct bau_activation_descriptor));
664 ad2->header.sw_ack_flag = 1;
665 ad2->header.base_dest_nodeid =
666 uv_blade_to_pnode(uv_cpu_to_blade_id(0));
667 ad2->header.command = UV_NET_ENDPOINT_INTD;
668 ad2->header.int_both = 1;
669 /* all others need to be set to zero:
670 fairness chaining multilevel count replied_to */
671 }
672
673 pqp = (struct bau_payload_queue_entry *)
674 kmalloc_node((DESTINATION_PAYLOAD_QUEUE_SIZE + 1) *
675 sizeof(struct bau_payload_queue_entry),
676 GFP_KERNEL, i);
677 if (!pqp)
678 BUG();
679 if (sizeof(struct bau_payload_queue_entry) != 32)
680 BUG();
681 if ((unsigned long)(&((struct bau_payload_queue_entry *)0)->
682 sw_ack_vector) != 15)
683 BUG();
684
685 cp = (char *)pqp + 31;
686 pqp = (struct bau_payload_queue_entry *)
687 (((unsigned long)cp >> 5) << 5);
688 bau_tablesp->va_queue_first = pqp;
689 uv_write_global_mmr64(pnode,
690 UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST,
691 ((unsigned long)pnode <<
692 UV_PAYLOADQ_PNODE_SHIFT) |
693 uv_physnodeaddr(pqp));
694 uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL,
695 uv_physnodeaddr(pqp));
696 bau_tablesp->va_queue_last =
697 pqp + (DESTINATION_PAYLOAD_QUEUE_SIZE - 1);
698 uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST,
699 (unsigned long)
700 uv_physnodeaddr(bau_tablesp->
701 va_queue_last));
702 memset(pqp, 0, sizeof(struct bau_payload_queue_entry) *
703 DESTINATION_PAYLOAD_QUEUE_SIZE);
704
705 /* this initialization can't be in firmware because the
706 messaging IRQ will be determined by the OS */
707 apicid = per_cpu(x86_cpu_to_apicid, cur_cpu);
708 pa = uv_read_global_mmr64(pnode, UVH_BAU_DATA_CONFIG);
709 if ((pa & 0xff) != UV_BAU_MESSAGE) {
710 uv_write_global_mmr64(pnode, UVH_BAU_DATA_CONFIG,
711 ((apicid << 32) |
712 UV_BAU_MESSAGE));
713 }
714
715 for (j = cur_cpu; j < (cur_cpu + uv_blade_nr_possible_cpus(i));
716 j++) {
717 bcp = (struct bau_control *)&per_cpu(bau_control, j);
718 bcp->bau_msg_head = bau_tablesp->va_queue_first;
719 bcp->va_queue_first = bau_tablesp->va_queue_first;
720
721 bcp->va_queue_last = bau_tablesp->va_queue_last;
722 bcp->watching = bau_tablesp->watching;
723 bcp->msg_statuses = bau_tablesp->msg_statuses;
724 bcp->descriptor_base = adp;
725 }
726 cur_cpu += uv_blade_nr_possible_cpus(i);
727 } 780 }
728
729 set_intr_gate(UV_BAU_MESSAGE, uv_bau_message_intr1); 781 set_intr_gate(UV_BAU_MESSAGE, uv_bau_message_intr1);
730
731 uv_enable_timeouts(); 782 uv_enable_timeouts();
732
733 return 0; 783 return 0;
734} 784}
735
736__initcall(uv_bau_init); 785__initcall(uv_bau_init);
786__initcall(uv_ptc_init);
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-20 06:06:02 -0500