aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/kernel/tlb_uv.c
blob: f396e61bcb344d9448d50350270e1ac987254119 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
/*
 *	SGI UltraViolet TLB flush routines.
 *
 *	(c) 2008 Cliff Wickman <cpw@sgi.com>, SGI.
 *
 *	This code is released under the GNU General Public License version 2 or
 *	later.
 */
#include <linux/seq_file.h>
#include <linux/proc_fs.h>
#include <linux/kernel.h>

#include <asm/mmu_context.h>
#include <asm/uv/uv.h>
#include <asm/uv/uv_mmrs.h>
#include <asm/uv/uv_hub.h>
#include <asm/uv/uv_bau.h>
#include <asm/genapic.h>
#include <asm/idle.h>
#include <asm/tsc.h>
#include <asm/irq_vectors.h>

#include <asm/genapic.h>

static struct bau_control	**uv_bau_table_bases __read_mostly;
static int			uv_bau_retry_limit __read_mostly;

/* position of pnode (which is nasid>>1): */
static int			uv_nshift __read_mostly;

static unsigned long		uv_mmask __read_mostly;

static DEFINE_PER_CPU(struct ptc_stats, ptcstats);
static DEFINE_PER_CPU(struct bau_control, bau_control);

/*
 * Free a software acknowledge hardware resource by clearing its Pending
 * bit. This will return a reply to the sender.
 * If the message has timed out, a reply has already been sent by the
 * hardware but the resource has not been released. In that case our
 * clear of the Timeout bit (as well) will free the resource. No reply will
 * be sent (the hardware will only do one reply per message).
 */
static void uv_reply_to_message(int resource,
				struct bau_payload_queue_entry *msg,
				struct bau_msg_status *msp)
{
	unsigned long dw;

	dw = (1 << (resource + UV_SW_ACK_NPENDING)) | (1 << resource);
	msg->replied_to = 1;
	msg->sw_ack_vector = 0;
	if (msp)
		msp->seen_by.bits = 0;
	uv_write_local_mmr(UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS, dw);
}

/*
 * Do all the things a cpu should do for a TLB shootdown message.
 * Other cpu's may come here at the same time for this message.
 */
static void uv_bau_process_message(struct bau_payload_queue_entry *msg,
				   int msg_slot, int sw_ack_slot)
{
	unsigned long this_cpu_mask;
	struct bau_msg_status *msp;
	int cpu;

	msp = __get_cpu_var(bau_control).msg_statuses + msg_slot;
	cpu = uv_blade_processor_id();
	msg->number_of_cpus =
	    uv_blade_nr_online_cpus(uv_node_to_blade_id(numa_node_id()));
	this_cpu_mask = 1UL << cpu;
	if (msp->seen_by.bits & this_cpu_mask)
		return;
	atomic_or_long(&msp->seen_by.bits, this_cpu_mask);

	if (msg->replied_to == 1)
		return;

	if (msg->address == TLB_FLUSH_ALL) {
		local_flush_tlb();
		__get_cpu_var(ptcstats).alltlb++;
	} else {
		__flush_tlb_one(msg->address);
		__get_cpu_var(ptcstats).onetlb++;
	}

	__get_cpu_var(ptcstats).requestee++;

	atomic_inc_short(&msg->acknowledge_count);
	if (msg->number_of_cpus == msg->acknowledge_count)
		uv_reply_to_message(sw_ack_slot, msg, msp);
}

/*
 * Examine the payload queue on one distribution node to see
 * which messages have not been seen, and which cpu(s) have not seen them.
 *
 * Returns the number of cpu's that have not responded.
 */
static int uv_examine_destination(struct bau_control *bau_tablesp, int sender)
{
	struct bau_payload_queue_entry *msg;
	struct bau_msg_status *msp;
	int count = 0;
	int i;
	int j;

	for (msg = bau_tablesp->va_queue_first, i = 0; i < DEST_Q_SIZE;
	     msg++, i++) {
		if ((msg->sending_cpu == sender) && (!msg->replied_to)) {
			msp = bau_tablesp->msg_statuses + i;
			printk(KERN_DEBUG
			       "blade %d: address:%#lx %d of %d, not cpu(s): ",
			       i, msg->address, msg->acknowledge_count,
			       msg->number_of_cpus);
			for (j = 0; j < msg->number_of_cpus; j++) {
				if (!((1L << j) & msp->seen_by.bits)) {
					count++;
					printk("%d ", j);
				}
			}
			printk("\n");
		}
	}
	return count;
}

/*
 * Examine the payload queue on all the distribution nodes to see
 * which messages have not been seen, and which cpu(s) have not seen them.
 *
 * Returns the number of cpu's that have not responded.
 */
static int uv_examine_destinations(struct bau_target_nodemask *distribution)
{
	int sender;
	int i;
	int count = 0;

	sender = smp_processor_id();
	for (i = 0; i < sizeof(struct bau_target_nodemask) * BITSPERBYTE; i++) {
		if (!bau_node_isset(i, distribution))
			continue;
		count += uv_examine_destination(uv_bau_table_bases[i], sender);
	}
	return count;
}

/*
 * wait for completion of a broadcast message
 *
 * return COMPLETE, RETRY or GIVEUP
 */
static int uv_wait_completion(struct bau_desc *bau_desc,
			      unsigned long mmr_offset, int right_shift)
{
	int exams = 0;
	long destination_timeouts = 0;
	long source_timeouts = 0;
	unsigned long descriptor_status;

	while ((descriptor_status = (((unsigned long)
		uv_read_local_mmr(mmr_offset) >>
			right_shift) & UV_ACT_STATUS_MASK)) !=
			DESC_STATUS_IDLE) {
		if (descriptor_status == DESC_STATUS_SOURCE_TIMEOUT) {
			source_timeouts++;
			if (source_timeouts > SOURCE_TIMEOUT_LIMIT)
				source_timeouts = 0;
			__get_cpu_var(ptcstats).s_retry++;
			return FLUSH_RETRY;
		}
		/*
		 * spin here looking for progress at the destinations
		 */
		if (descriptor_status == DESC_STATUS_DESTINATION_TIMEOUT) {
			destination_timeouts++;
			if (destination_timeouts > DESTINATION_TIMEOUT_LIMIT) {
				/*
				 * returns number of cpus not responding
				 */
				if (uv_examine_destinations
				    (&bau_desc->distribution) == 0) {
					__get_cpu_var(ptcstats).d_retry++;
					return FLUSH_RETRY;
				}
				exams++;
				if (exams >= uv_bau_retry_limit) {
					printk(KERN_DEBUG
					       "uv_flush_tlb_others");
					printk("giving up on cpu %d\n",
					       smp_processor_id());
					return FLUSH_GIVEUP;
				}
				/*
				 * delays can hang the simulator
				   udelay(1000);
				 */
				destination_timeouts = 0;
			}
		}
		cpu_relax();
	}
	return FLUSH_COMPLETE;
}

/**
 * uv_flush_send_and_wait
 *
 * Send a broadcast and wait for a broadcast message to complete.
 *
 * The flush_mask contains the cpus the broadcast was sent to.
 *
 * Returns NULL if all remote flushing was done. The mask is zeroed.
 * Returns @flush_mask if some remote flushing remains to be done. The
 * mask will have some bits still set.
 */
const struct cpumask *uv_flush_send_and_wait(int cpu, int this_blade,
					     struct bau_desc *bau_desc,
					     struct cpumask *flush_mask)
{
	int completion_status = 0;
	int right_shift;
	int tries = 0;
	int blade;
	int bit;
	unsigned long mmr_offset;
	unsigned long index;
	cycles_t time1;
	cycles_t time2;

	if (cpu < UV_CPUS_PER_ACT_STATUS) {
		mmr_offset = UVH_LB_BAU_SB_ACTIVATION_STATUS_0;
		right_shift = cpu * UV_ACT_STATUS_SIZE;
	} else {
		mmr_offset = UVH_LB_BAU_SB_ACTIVATION_STATUS_1;
		right_shift =
		    ((cpu - UV_CPUS_PER_ACT_STATUS) * UV_ACT_STATUS_SIZE);
	}
	time1 = get_cycles();
	do {
		tries++;
		index = (1UL << UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT) |
			cpu;
		uv_write_local_mmr(UVH_LB_BAU_SB_ACTIVATION_CONTROL, index);
		completion_status = uv_wait_completion(bau_desc, mmr_offset,
					right_shift);
	} while (completion_status == FLUSH_RETRY);
	time2 = get_cycles();
	__get_cpu_var(ptcstats).sflush += (time2 - time1);
	if (tries > 1)
		__get_cpu_var(ptcstats).retriesok++;

	if (completion_status == FLUSH_GIVEUP) {
		/*
		 * Cause the caller to do an IPI-style TLB shootdown on
		 * the cpu's, all of which are still in the mask.
		 */
		__get_cpu_var(ptcstats).ptc_i++;
		return flush_mask;
	}

	/*
	 * Success, so clear the remote cpu's from the mask so we don't
	 * use the IPI method of shootdown on them.
	 */
	for_each_cpu(bit, flush_mask) {
		blade = uv_cpu_to_blade_id(bit);
		if (blade == this_blade)
			continue;
		cpumask_clear_cpu(bit, flush_mask);
	}
	if (!cpumask_empty(flush_mask))
		return flush_mask;
	return NULL;
}

/**
 * uv_flush_tlb_others - globally purge translation cache of a virtual
 * address or all TLB's
 * @cpumask: mask of all cpu's in which the address is to be removed
 * @mm: mm_struct containing virtual address range
 * @va: virtual address to be removed (or TLB_FLUSH_ALL for all TLB's on cpu)
 * @cpu: the current cpu
 *
 * This is the entry point for initiating any UV global TLB shootdown.
 *
 * Purges the translation caches of all specified processors of the given
 * virtual address, or purges all TLB's on specified processors.
 *
 * The caller has derived the cpumask from the mm_struct.  This function
 * is called only if there are bits set in the mask. (e.g. flush_tlb_page())
 *
 * The cpumask is converted into a nodemask of the nodes containing
 * the cpus.
 *
 * Note that this function should be called with preemption disabled.
 *
 * Returns NULL if all remote flushing was done.
 * Returns pointer to cpumask if some remote flushing remains to be
 * done.  The returned pointer is valid till preemption is re-enabled.
 */
const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
					  struct mm_struct *mm,
					  unsigned long va, unsigned int cpu)
{
	static DEFINE_PER_CPU(cpumask_t, flush_tlb_mask);
	struct cpumask *flush_mask = &__get_cpu_var(flush_tlb_mask);
	int i;
	int bit;
	int blade;
	int uv_cpu;
	int this_blade;
	int locals = 0;
	struct bau_desc *bau_desc;

	WARN_ON(!in_atomic());

	cpumask_andnot(flush_mask, cpumask, cpumask_of(cpu));

	uv_cpu = uv_blade_processor_id();
	this_blade = uv_numa_blade_id();
	bau_desc = __get_cpu_var(bau_control).descriptor_base;
	bau_desc += UV_ITEMS_PER_DESCRIPTOR * uv_cpu;

	bau_nodes_clear(&bau_desc->distribution, UV_DISTRIBUTION_SIZE);

	i = 0;
	for_each_cpu(bit, flush_mask) {
		blade = uv_cpu_to_blade_id(bit);
		BUG_ON(blade > (UV_DISTRIBUTION_SIZE - 1));
		if (blade == this_blade) {
			locals++;
			continue;
		}
		bau_node_set(blade, &bau_desc->distribution);
		i++;
	}
	if (i == 0) {
		/*
		 * no off_node flushing; return status for local node
		 */
		if (locals)
			return flush_mask;
		else
			return NULL;
	}
	__get_cpu_var(ptcstats).requestor++;
	__get_cpu_var(ptcstats).ntargeted += i;

	bau_desc->payload.address = va;
	bau_desc->payload.sending_cpu = cpu;

	return uv_flush_send_and_wait(uv_cpu, this_blade, bau_desc, flush_mask);
}

/*
 * The BAU message interrupt comes here. (registered by set_intr_gate)
 * See entry_64.S
 *
 * We received a broadcast assist message.
 *
 * Interrupts may have been disabled; this interrupt could represent
 * the receipt of several messages.
 *
 * All cores/threads on this node get this interrupt.
 * The last one to see it does the s/w ack.
 * (the resource will not be freed until noninterruptable cpus see this
 *  interrupt; hardware will timeout the s/w ack and reply ERROR)
 */
void uv_bau_message_interrupt(struct pt_regs *regs)
{
	struct bau_payload_queue_entry *va_queue_first;
	struct bau_payload_queue_entry *va_queue_last;
	struct bau_payload_queue_entry *msg;
	struct pt_regs *old_regs = set_irq_regs(regs);
	cycles_t time1;
	cycles_t time2;
	int msg_slot;
	int sw_ack_slot;
	int fw;
	int count = 0;
	unsigned long local_pnode;

	ack_APIC_irq();
	exit_idle();
	irq_enter();

	time1 = get_cycles();

	local_pnode = uv_blade_to_pnode(uv_numa_blade_id());

	va_queue_first = __get_cpu_var(bau_control).va_queue_first;
	va_queue_last = __get_cpu_var(bau_control).va_queue_last;

	msg = __get_cpu_var(bau_control).bau_msg_head;
	while (msg->sw_ack_vector) {
		count++;
		fw = msg->sw_ack_vector;
		msg_slot = msg - va_queue_first;
		sw_ack_slot = ffs(fw) - 1;

		uv_bau_process_message(msg, msg_slot, sw_ack_slot);

		msg++;
		if (msg > va_queue_last)
			msg = va_queue_first;
		__get_cpu_var(bau_control).bau_msg_head = msg;
	}
	if (!count)
		__get_cpu_var(ptcstats).nomsg++;
	else if (count > 1)
		__get_cpu_var(ptcstats).multmsg++;

	time2 = get_cycles();
	__get_cpu_var(ptcstats).dflush += (time2 - time1);

	irq_exit();
	set_irq_regs(old_regs);
}

static void uv_enable_timeouts(void)
{
	int i;
	int blade;
	int last_blade;
	int pnode;
	int cur_cpu = 0;
	unsigned long apicid;

	last_blade = -1;
	for_each_online_node(i) {
		blade = uv_node_to_blade_id(i);
		if (blade == last_blade)
			continue;
		last_blade = blade;
		apicid = per_cpu(x86_cpu_to_apicid, cur_cpu);
		pnode = uv_blade_to_pnode(blade);
		cur_cpu += uv_blade_nr_possible_cpus(i);
	}
}

static void *uv_ptc_seq_start(struct seq_file *file, loff_t *offset)
{
	if (*offset < num_possible_cpus())
		return offset;
	return NULL;
}

static void *uv_ptc_seq_next(struct seq_file *file, void *data, loff_t *offset)
{
	(*offset)++;
	if (*offset < num_possible_cpus())
		return offset;
	return NULL;
}

static void uv_ptc_seq_stop(struct seq_file *file, void *data)
{
}

/*
 * Display the statistics thru /proc
 * data points to the cpu number
 */
static int uv_ptc_seq_show(struct seq_file *file, void *data)
{
	struct ptc_stats *stat;
	int cpu;

	cpu = *(loff_t *)data;

	if (!cpu) {
		seq_printf(file,
		"# cpu requestor requestee one all sretry dretry ptc_i ");
		seq_printf(file,
		"sw_ack sflush dflush sok dnomsg dmult starget\n");
	}
	if (cpu < num_possible_cpus() && cpu_online(cpu)) {
		stat = &per_cpu(ptcstats, cpu);
		seq_printf(file, "cpu %d %ld %ld %ld %ld %ld %ld %ld ",
			   cpu, stat->requestor,
			   stat->requestee, stat->onetlb, stat->alltlb,
			   stat->s_retry, stat->d_retry, stat->ptc_i);
		seq_printf(file, "%lx %ld %ld %ld %ld %ld %ld\n",
			   uv_read_global_mmr64(uv_blade_to_pnode
					(uv_cpu_to_blade_id(cpu)),
					UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE),
			   stat->sflush, stat->dflush,
			   stat->retriesok, stat->nomsg,
			   stat->multmsg, stat->ntargeted);
	}

	return 0;
}

/*
 *  0: display meaning of the statistics
 * >0: retry limit
 */
static ssize_t uv_ptc_proc_write(struct file *file, const char __user *user,
				 size_t count, loff_t *data)
{
	long newmode;
	char optstr[64];

	if (count == 0 || count > sizeof(optstr))
		return -EINVAL;
	if (copy_from_user(optstr, user, count))
		return -EFAULT;
	optstr[count - 1] = '\0';
	if (strict_strtoul(optstr, 10, &newmode) < 0) {
		printk(KERN_DEBUG "%s is invalid\n", optstr);
		return -EINVAL;
	}

	if (newmode == 0) {
		printk(KERN_DEBUG "# cpu:      cpu number\n");
		printk(KERN_DEBUG
		"requestor:  times this cpu was the flush requestor\n");
		printk(KERN_DEBUG
		"requestee:  times this cpu was requested to flush its TLBs\n");
		printk(KERN_DEBUG
		"one:        times requested to flush a single address\n");
		printk(KERN_DEBUG
		"all:        times requested to flush all TLB's\n");
		printk(KERN_DEBUG
		"sretry:     number of retries of source-side timeouts\n");
		printk(KERN_DEBUG
		"dretry:     number of retries of destination-side timeouts\n");
		printk(KERN_DEBUG
		"ptc_i:      times UV fell through to IPI-style flushes\n");
		printk(KERN_DEBUG
		"sw_ack:     image of UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE\n");
		printk(KERN_DEBUG
		"sflush_us:  cycles spent in uv_flush_tlb_others()\n");
		printk(KERN_DEBUG
		"dflush_us:  cycles spent in handling flush requests\n");
		printk(KERN_DEBUG "sok:        successes on retry\n");
		printk(KERN_DEBUG "dnomsg:     interrupts with no message\n");
		printk(KERN_DEBUG
		"dmult:      interrupts with multiple messages\n");
		printk(KERN_DEBUG "starget:    nodes targeted\n");
	} else {
		uv_bau_retry_limit = newmode;
		printk(KERN_DEBUG "timeout retry limit:%d\n",
		       uv_bau_retry_limit);
	}

	return count;
}

static const struct seq_operations uv_ptc_seq_ops = {
	.start		= uv_ptc_seq_start,
	.next		= uv_ptc_seq_next,
	.stop		= uv_ptc_seq_stop,
	.show		= uv_ptc_seq_show
};

static int uv_ptc_proc_open(struct inode *inode, struct file *file)
{
	return seq_open(file, &uv_ptc_seq_ops);
}

static const struct file_operations proc_uv_ptc_operations = {
	.open		= uv_ptc_proc_open,
	.read		= seq_read,
	.write		= uv_ptc_proc_write,
	.llseek		= seq_lseek,
	.release	= seq_release,
};

static int __init uv_ptc_init(void)
{
	struct proc_dir_entry *proc_uv_ptc;

	if (!is_uv_system())
		return 0;

	proc_uv_ptc = create_proc_entry(UV_PTC_BASENAME, 0444, NULL);
	if (!proc_uv_ptc) {
		printk(KERN_ERR "unable to create %s proc entry\n",
		       UV_PTC_BASENAME);
		return -EINVAL;
	}
	proc_uv_ptc->proc_fops = &proc_uv_ptc_operations;
	return 0;
}

/*
 * begin the initialization of the per-blade control structures
 */
static struct bau_control * __init uv_table_bases_init(int blade, int node)
{
	int i;
	struct bau_msg_status *msp;
	struct bau_control *bau_tabp;

	bau_tabp =
	    kmalloc_node(sizeof(struct bau_control), GFP_KERNEL, node);
	BUG_ON(!bau_tabp);

	bau_tabp->msg_statuses =
	    kmalloc_node(sizeof(struct bau_msg_status) *
			 DEST_Q_SIZE, GFP_KERNEL, node);
	BUG_ON(!bau_tabp->msg_statuses);

	for (i = 0, msp = bau_tabp->msg_statuses; i < DEST_Q_SIZE; i++, msp++)
		bau_cpubits_clear(&msp->seen_by, (int)
				  uv_blade_nr_possible_cpus(blade));

	uv_bau_table_bases[blade] = bau_tabp;

	return bau_tabp;
}

/*
 * finish the initialization of the per-blade control structures
 */
static void __init
uv_table_bases_finish(int blade, int node, int cur_cpu,
		      struct bau_control *bau_tablesp,
		      struct bau_desc *adp)
{
	struct bau_control *bcp;
	int i;

	for (i = cur_cpu; i < cur_cpu + uv_blade_nr_possible_cpus(blade); i++) {
		bcp = (struct bau_control *)&per_cpu(bau_control, i);

		bcp->bau_msg_head	= bau_tablesp->va_queue_first;
		bcp->va_queue_first	= bau_tablesp->va_queue_first;
		bcp->va_queue_last	= bau_tablesp->va_queue_last;
		bcp->msg_statuses	= bau_tablesp->msg_statuses;
		bcp->descriptor_base	= adp;
	}
}

/*
 * initialize the sending side's sending buffers
 */
static struct bau_desc * __init
uv_activation_descriptor_init(int node, int pnode)
{
	int i;
	unsigned long pa;
	unsigned long m;
	unsigned long n;
	unsigned long mmr_image;
	struct bau_desc *adp;
	struct bau_desc *ad2;

	adp = (struct bau_desc *)
	    kmalloc_node(16384, GFP_KERNEL, node);
	BUG_ON(!adp);

	pa = __pa((unsigned long)adp);
	n = pa >> uv_nshift;
	m = pa & uv_mmask;

	mmr_image = uv_read_global_mmr64(pnode, UVH_LB_BAU_SB_DESCRIPTOR_BASE);
	if (mmr_image) {
		uv_write_global_mmr64(pnode, (unsigned long)
				      UVH_LB_BAU_SB_DESCRIPTOR_BASE,
				      (n << UV_DESC_BASE_PNODE_SHIFT | m));
	}

	for (i = 0, ad2 = adp; i < UV_ACTIVATION_DESCRIPTOR_SIZE; i++, ad2++) {
		memset(ad2, 0, sizeof(struct bau_desc));
		ad2->header.sw_ack_flag = 1;
		ad2->header.base_dest_nodeid =
		    uv_blade_to_pnode(uv_cpu_to_blade_id(0));
		ad2->header.command = UV_NET_ENDPOINT_INTD;
		ad2->header.int_both = 1;
		/*
		 * all others need to be set to zero:
		 *   fairness chaining multilevel count replied_to
		 */
	}
	return adp;
}

/*
 * initialize the destination side's receiving buffers
 */
static struct bau_payload_queue_entry * __init
uv_payload_queue_init(int node, int pnode, struct bau_control *bau_tablesp)
{
	struct bau_payload_queue_entry *pqp;
	char *cp;

	pqp = (struct bau_payload_queue_entry *) kmalloc_node(
		(DEST_Q_SIZE + 1) * sizeof(struct bau_payload_queue_entry),
		GFP_KERNEL, node);
	BUG_ON(!pqp);

	cp = (char *)pqp + 31;
	pqp = (struct bau_payload_queue_entry *)(((unsigned long)cp >> 5) << 5);
	bau_tablesp->va_queue_first = pqp;
	uv_write_global_mmr64(pnode,
			      UVH_LB_BAU_INTD_PAYLOAD_QUEUE_FIRST,
			      ((unsigned long)pnode <<
			       UV_PAYLOADQ_PNODE_SHIFT) |
			      uv_physnodeaddr(pqp));
	uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL,
			      uv_physnodeaddr(pqp));
	bau_tablesp->va_queue_last = pqp + (DEST_Q_SIZE - 1);
	uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST,
			      (unsigned long)
			      uv_physnodeaddr(bau_tablesp->va_queue_last));
	memset(pqp, 0, sizeof(struct bau_payload_queue_entry) * DEST_Q_SIZE);

	return pqp;
}

/*
 * Initialization of each UV blade's structures
 */
static int __init uv_init_blade(int blade, int node, int cur_cpu)
{
	int pnode;
	unsigned long pa;
	unsigned long apicid;
	struct bau_desc *adp;
	struct bau_payload_queue_entry *pqp;
	struct bau_control *bau_tablesp;

	bau_tablesp = uv_table_bases_init(blade, node);
	pnode = uv_blade_to_pnode(blade);
	adp = uv_activation_descriptor_init(node, pnode);
	pqp = uv_payload_queue_init(node, pnode, bau_tablesp);
	uv_table_bases_finish(blade, node, cur_cpu, bau_tablesp, adp);
	/*
	 * the below initialization can't be in firmware because the
	 * messaging IRQ will be determined by the OS
	 */
	apicid = per_cpu(x86_cpu_to_apicid, cur_cpu);
	pa = uv_read_global_mmr64(pnode, UVH_BAU_DATA_CONFIG);
	if ((pa & 0xff) != UV_BAU_MESSAGE) {
		uv_write_global_mmr64(pnode, UVH_BAU_DATA_CONFIG,
				      ((apicid << 32) | UV_BAU_MESSAGE));
	}
	return 0;
}

/*
 * Initialization of BAU-related structures
 */
static int __init uv_bau_init(void)
{
	int blade;
	int node;
	int nblades;
	int last_blade;
	int cur_cpu = 0;

	if (!is_uv_system())
		return 0;

	uv_bau_retry_limit = 1;
	uv_nshift = uv_hub_info->n_val;
	uv_mmask = (1UL << uv_hub_info->n_val) - 1;
	nblades = 0;
	last_blade = -1;
	for_each_online_node(node) {
		blade = uv_node_to_blade_id(node);
		if (blade == last_blade)
			continue;
		last_blade = blade;
		nblades++;
	}
	uv_bau_table_bases = (struct bau_control **)
	    kmalloc(nblades * sizeof(struct bau_control *), GFP_KERNEL);
	BUG_ON(!uv_bau_table_bases);

	last_blade = -1;
	for_each_online_node(node) {
		blade = uv_node_to_blade_id(node);
		if (blade == last_blade)
			continue;
		last_blade = blade;
		uv_init_blade(blade, node, cur_cpu);
		cur_cpu += uv_blade_nr_possible_cpus(blade);
	}
	alloc_intr_gate(UV_BAU_MESSAGE, uv_bau_message_intr1);
	uv_enable_timeouts();

	return 0;
}
__initcall(uv_bau_init);
__initcall(uv_ptc_init);