aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/misc/sgi-xp/xpc_sn2.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/misc/sgi-xp/xpc_sn2.c')
-rw-r--r--drivers/misc/sgi-xp/xpc_sn2.c2404
1 files changed, 2404 insertions, 0 deletions
diff --git a/drivers/misc/sgi-xp/xpc_sn2.c b/drivers/misc/sgi-xp/xpc_sn2.c
new file mode 100644
index 000000000000..b4882ccf6344
--- /dev/null
+++ b/drivers/misc/sgi-xp/xpc_sn2.c
@@ -0,0 +1,2404 @@
1/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved.
7 */
8
9/*
10 * Cross Partition Communication (XPC) sn2-based functions.
11 *
12 * Architecture specific implementation of common functions.
13 *
14 */
15
16#include <linux/delay.h>
17#include <asm/uncached.h>
18#include <asm/sn/mspec.h>
19#include <asm/sn/sn_sal.h>
20#include "xpc.h"
21
22/*
23 * Define the number of u64s required to represent all the C-brick nasids
24 * as a bitmap. The cross-partition kernel modules deal only with
25 * C-brick nasids, thus the need for bitmaps which don't account for
26 * odd-numbered (non C-brick) nasids.
27 */
28#define XPC_MAX_PHYSNODES_SN2 (MAX_NUMALINK_NODES / 2)
29#define XP_NASID_MASK_BYTES_SN2 ((XPC_MAX_PHYSNODES_SN2 + 7) / 8)
30#define XP_NASID_MASK_WORDS_SN2 ((XPC_MAX_PHYSNODES_SN2 + 63) / 64)
31
32/*
33 * Memory for XPC's amo variables is allocated by the MSPEC driver. These
34 * pages are located in the lowest granule. The lowest granule uses 4k pages
35 * for cached references and an alternate TLB handler to never provide a
36 * cacheable mapping for the entire region. This will prevent speculative
37 * reading of cached copies of our lines from being issued which will cause
38 * a PI FSB Protocol error to be generated by the SHUB. For XPC, we need 64
39 * amo variables (based on XP_MAX_NPARTITIONS_SN2) to identify the senders of
40 * NOTIFY IRQs, 128 amo variables (based on XP_NASID_MASK_WORDS_SN2) to identify
41 * the senders of ACTIVATE IRQs, 1 amo variable to identify which remote
42 * partitions (i.e., XPCs) consider themselves currently engaged with the
43 * local XPC and 1 amo variable to request partition deactivation.
44 */
45#define XPC_NOTIFY_IRQ_AMOS_SN2 0
46#define XPC_ACTIVATE_IRQ_AMOS_SN2 (XPC_NOTIFY_IRQ_AMOS_SN2 + \
47 XP_MAX_NPARTITIONS_SN2)
48#define XPC_ENGAGED_PARTITIONS_AMO_SN2 (XPC_ACTIVATE_IRQ_AMOS_SN2 + \
49 XP_NASID_MASK_WORDS_SN2)
50#define XPC_DEACTIVATE_REQUEST_AMO_SN2 (XPC_ENGAGED_PARTITIONS_AMO_SN2 + 1)
51
52/*
53 * Buffer used to store a local copy of portions of a remote partition's
54 * reserved page (either its header and part_nasids mask, or its vars).
55 */
56static void *xpc_remote_copy_buffer_base_sn2;
57static char *xpc_remote_copy_buffer_sn2;
58
59static struct xpc_vars_sn2 *xpc_vars_sn2;
60static struct xpc_vars_part_sn2 *xpc_vars_part_sn2;
61
62static int
63xpc_setup_partitions_sn_sn2(void)
64{
65 /* nothing needs to be done */
66 return 0;
67}
68
69/* SH_IPI_ACCESS shub register value on startup */
70static u64 xpc_sh1_IPI_access_sn2;
71static u64 xpc_sh2_IPI_access0_sn2;
72static u64 xpc_sh2_IPI_access1_sn2;
73static u64 xpc_sh2_IPI_access2_sn2;
74static u64 xpc_sh2_IPI_access3_sn2;
75
76/*
77 * Change protections to allow IPI operations.
78 */
79static void
80xpc_allow_IPI_ops_sn2(void)
81{
82 int node;
83 int nasid;
84
85 /* !!! The following should get moved into SAL. */
86 if (is_shub2()) {
87 xpc_sh2_IPI_access0_sn2 =
88 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS0));
89 xpc_sh2_IPI_access1_sn2 =
90 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS1));
91 xpc_sh2_IPI_access2_sn2 =
92 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS2));
93 xpc_sh2_IPI_access3_sn2 =
94 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH2_IPI_ACCESS3));
95
96 for_each_online_node(node) {
97 nasid = cnodeid_to_nasid(node);
98 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0),
99 -1UL);
100 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1),
101 -1UL);
102 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2),
103 -1UL);
104 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3),
105 -1UL);
106 }
107 } else {
108 xpc_sh1_IPI_access_sn2 =
109 (u64)HUB_L((u64 *)LOCAL_MMR_ADDR(SH1_IPI_ACCESS));
110
111 for_each_online_node(node) {
112 nasid = cnodeid_to_nasid(node);
113 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS),
114 -1UL);
115 }
116 }
117}
118
119/*
120 * Restrict protections to disallow IPI operations.
121 */
122static void
123xpc_disallow_IPI_ops_sn2(void)
124{
125 int node;
126 int nasid;
127
128 /* !!! The following should get moved into SAL. */
129 if (is_shub2()) {
130 for_each_online_node(node) {
131 nasid = cnodeid_to_nasid(node);
132 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS0),
133 xpc_sh2_IPI_access0_sn2);
134 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS1),
135 xpc_sh2_IPI_access1_sn2);
136 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS2),
137 xpc_sh2_IPI_access2_sn2);
138 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH2_IPI_ACCESS3),
139 xpc_sh2_IPI_access3_sn2);
140 }
141 } else {
142 for_each_online_node(node) {
143 nasid = cnodeid_to_nasid(node);
144 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid, SH1_IPI_ACCESS),
145 xpc_sh1_IPI_access_sn2);
146 }
147 }
148}
149
150/*
151 * The following set of functions are used for the sending and receiving of
152 * IRQs (also known as IPIs). There are two flavors of IRQs, one that is
153 * associated with partition activity (SGI_XPC_ACTIVATE) and the other that
154 * is associated with channel activity (SGI_XPC_NOTIFY).
155 */
156
157static u64
158xpc_receive_IRQ_amo_sn2(struct amo *amo)
159{
160 return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_CLEAR);
161}
162
163static enum xp_retval
164xpc_send_IRQ_sn2(struct amo *amo, u64 flag, int nasid, int phys_cpuid,
165 int vector)
166{
167 int ret = 0;
168 unsigned long irq_flags;
169
170 local_irq_save(irq_flags);
171
172 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR, flag);
173 sn_send_IPI_phys(nasid, phys_cpuid, vector, 0);
174
175 /*
176 * We must always use the nofault function regardless of whether we
177 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
178 * didn't, we'd never know that the other partition is down and would
179 * keep sending IRQs and amos to it until the heartbeat times out.
180 */
181 ret = xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->variable),
182 xp_nofault_PIOR_target));
183
184 local_irq_restore(irq_flags);
185
186 return (ret == 0) ? xpSuccess : xpPioReadError;
187}
188
189static struct amo *
190xpc_init_IRQ_amo_sn2(int index)
191{
192 struct amo *amo = xpc_vars_sn2->amos_page + index;
193
194 (void)xpc_receive_IRQ_amo_sn2(amo); /* clear amo variable */
195 return amo;
196}
197
198/*
199 * Functions associated with SGI_XPC_ACTIVATE IRQ.
200 */
201
202/*
203 * Notify the heartbeat check thread that an activate IRQ has been received.
204 */
205static irqreturn_t
206xpc_handle_activate_IRQ_sn2(int irq, void *dev_id)
207{
208 unsigned long irq_flags;
209
210 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
211 xpc_activate_IRQ_rcvd++;
212 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
213
214 wake_up_interruptible(&xpc_activate_IRQ_wq);
215 return IRQ_HANDLED;
216}
217
218/*
219 * Flag the appropriate amo variable and send an IRQ to the specified node.
220 */
221static void
222xpc_send_activate_IRQ_sn2(unsigned long amos_page_pa, int from_nasid,
223 int to_nasid, int to_phys_cpuid)
224{
225 struct amo *amos = (struct amo *)__va(amos_page_pa +
226 (XPC_ACTIVATE_IRQ_AMOS_SN2 *
227 sizeof(struct amo)));
228
229 (void)xpc_send_IRQ_sn2(&amos[BIT_WORD(from_nasid / 2)],
230 BIT_MASK(from_nasid / 2), to_nasid,
231 to_phys_cpuid, SGI_XPC_ACTIVATE);
232}
233
234static void
235xpc_send_local_activate_IRQ_sn2(int from_nasid)
236{
237 unsigned long irq_flags;
238 struct amo *amos = (struct amo *)__va(xpc_vars_sn2->amos_page_pa +
239 (XPC_ACTIVATE_IRQ_AMOS_SN2 *
240 sizeof(struct amo)));
241
242 /* fake the sending and receipt of an activate IRQ from remote nasid */
243 FETCHOP_STORE_OP(TO_AMO((u64)&amos[BIT_WORD(from_nasid / 2)].variable),
244 FETCHOP_OR, BIT_MASK(from_nasid / 2));
245
246 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
247 xpc_activate_IRQ_rcvd++;
248 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
249
250 wake_up_interruptible(&xpc_activate_IRQ_wq);
251}
252
253/*
254 * Functions associated with SGI_XPC_NOTIFY IRQ.
255 */
256
257/*
258 * Check to see if any chctl flags were sent from the specified partition.
259 */
260static void
261xpc_check_for_sent_chctl_flags_sn2(struct xpc_partition *part)
262{
263 union xpc_channel_ctl_flags chctl;
264 unsigned long irq_flags;
265
266 chctl.all_flags = xpc_receive_IRQ_amo_sn2(part->sn.sn2.
267 local_chctl_amo_va);
268 if (chctl.all_flags == 0)
269 return;
270
271 spin_lock_irqsave(&part->chctl_lock, irq_flags);
272 part->chctl.all_flags |= chctl.all_flags;
273 spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
274
275 dev_dbg(xpc_chan, "received notify IRQ from partid=%d, chctl.all_flags="
276 "0x%lx\n", XPC_PARTID(part), chctl.all_flags);
277
278 xpc_wakeup_channel_mgr(part);
279}
280
281/*
282 * Handle the receipt of a SGI_XPC_NOTIFY IRQ by seeing whether the specified
283 * partition actually sent it. Since SGI_XPC_NOTIFY IRQs may be shared by more
284 * than one partition, we use an amo structure per partition to indicate
285 * whether a partition has sent an IRQ or not. If it has, then wake up the
286 * associated kthread to handle it.
287 *
288 * All SGI_XPC_NOTIFY IRQs received by XPC are the result of IRQs sent by XPC
289 * running on other partitions.
290 *
291 * Noteworthy Arguments:
292 *
293 * irq - Interrupt ReQuest number. NOT USED.
294 *
295 * dev_id - partid of IRQ's potential sender.
296 */
297static irqreturn_t
298xpc_handle_notify_IRQ_sn2(int irq, void *dev_id)
299{
300 short partid = (short)(u64)dev_id;
301 struct xpc_partition *part = &xpc_partitions[partid];
302
303 DBUG_ON(partid < 0 || partid >= XP_MAX_NPARTITIONS_SN2);
304
305 if (xpc_part_ref(part)) {
306 xpc_check_for_sent_chctl_flags_sn2(part);
307
308 xpc_part_deref(part);
309 }
310 return IRQ_HANDLED;
311}
312
313/*
314 * Check to see if xpc_handle_notify_IRQ_sn2() dropped any IRQs on the floor
315 * because the write to their associated amo variable completed after the IRQ
316 * was received.
317 */
318static void
319xpc_check_for_dropped_notify_IRQ_sn2(struct xpc_partition *part)
320{
321 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
322
323 if (xpc_part_ref(part)) {
324 xpc_check_for_sent_chctl_flags_sn2(part);
325
326 part_sn2->dropped_notify_IRQ_timer.expires = jiffies +
327 XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL;
328 add_timer(&part_sn2->dropped_notify_IRQ_timer);
329 xpc_part_deref(part);
330 }
331}
332
333/*
334 * Send a notify IRQ to the remote partition that is associated with the
335 * specified channel.
336 */
337static void
338xpc_send_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag,
339 char *chctl_flag_string, unsigned long *irq_flags)
340{
341 struct xpc_partition *part = &xpc_partitions[ch->partid];
342 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
343 union xpc_channel_ctl_flags chctl = { 0 };
344 enum xp_retval ret;
345
346 if (likely(part->act_state != XPC_P_AS_DEACTIVATING)) {
347 chctl.flags[ch->number] = chctl_flag;
348 ret = xpc_send_IRQ_sn2(part_sn2->remote_chctl_amo_va,
349 chctl.all_flags,
350 part_sn2->notify_IRQ_nasid,
351 part_sn2->notify_IRQ_phys_cpuid,
352 SGI_XPC_NOTIFY);
353 dev_dbg(xpc_chan, "%s sent to partid=%d, channel=%d, ret=%d\n",
354 chctl_flag_string, ch->partid, ch->number, ret);
355 if (unlikely(ret != xpSuccess)) {
356 if (irq_flags != NULL)
357 spin_unlock_irqrestore(&ch->lock, *irq_flags);
358 XPC_DEACTIVATE_PARTITION(part, ret);
359 if (irq_flags != NULL)
360 spin_lock_irqsave(&ch->lock, *irq_flags);
361 }
362 }
363}
364
365#define XPC_SEND_NOTIFY_IRQ_SN2(_ch, _ipi_f, _irq_f) \
366 xpc_send_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f, _irq_f)
367
368/*
369 * Make it look like the remote partition, which is associated with the
370 * specified channel, sent us a notify IRQ. This faked IRQ will be handled
371 * by xpc_check_for_dropped_notify_IRQ_sn2().
372 */
373static void
374xpc_send_local_notify_IRQ_sn2(struct xpc_channel *ch, u8 chctl_flag,
375 char *chctl_flag_string)
376{
377 struct xpc_partition *part = &xpc_partitions[ch->partid];
378 union xpc_channel_ctl_flags chctl = { 0 };
379
380 chctl.flags[ch->number] = chctl_flag;
381 FETCHOP_STORE_OP(TO_AMO((u64)&part->sn.sn2.local_chctl_amo_va->
382 variable), FETCHOP_OR, chctl.all_flags);
383 dev_dbg(xpc_chan, "%s sent local from partid=%d, channel=%d\n",
384 chctl_flag_string, ch->partid, ch->number);
385}
386
387#define XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(_ch, _ipi_f) \
388 xpc_send_local_notify_IRQ_sn2(_ch, _ipi_f, #_ipi_f)
389
390static void
391xpc_send_chctl_closerequest_sn2(struct xpc_channel *ch,
392 unsigned long *irq_flags)
393{
394 struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args;
395
396 args->reason = ch->reason;
397 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREQUEST, irq_flags);
398}
399
400static void
401xpc_send_chctl_closereply_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
402{
403 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_CLOSEREPLY, irq_flags);
404}
405
406static void
407xpc_send_chctl_openrequest_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
408{
409 struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args;
410
411 args->entry_size = ch->entry_size;
412 args->local_nentries = ch->local_nentries;
413 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREQUEST, irq_flags);
414}
415
416static void
417xpc_send_chctl_openreply_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
418{
419 struct xpc_openclose_args *args = ch->sn.sn2.local_openclose_args;
420
421 args->remote_nentries = ch->remote_nentries;
422 args->local_nentries = ch->local_nentries;
423 args->local_msgqueue_pa = xp_pa(ch->sn.sn2.local_msgqueue);
424 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_OPENREPLY, irq_flags);
425}
426
427static void
428xpc_send_chctl_msgrequest_sn2(struct xpc_channel *ch)
429{
430 XPC_SEND_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST, NULL);
431}
432
433static void
434xpc_send_chctl_local_msgrequest_sn2(struct xpc_channel *ch)
435{
436 XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST);
437}
438
439static void
440xpc_save_remote_msgqueue_pa_sn2(struct xpc_channel *ch,
441 unsigned long msgqueue_pa)
442{
443 ch->sn.sn2.remote_msgqueue_pa = msgqueue_pa;
444}
445
446/*
447 * This next set of functions are used to keep track of when a partition is
448 * potentially engaged in accessing memory belonging to another partition.
449 */
450
451static void
452xpc_indicate_partition_engaged_sn2(struct xpc_partition *part)
453{
454 unsigned long irq_flags;
455 struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa +
456 (XPC_ENGAGED_PARTITIONS_AMO_SN2 *
457 sizeof(struct amo)));
458
459 local_irq_save(irq_flags);
460
461 /* set bit corresponding to our partid in remote partition's amo */
462 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR,
463 BIT(sn_partition_id));
464
465 /*
466 * We must always use the nofault function regardless of whether we
467 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
468 * didn't, we'd never know that the other partition is down and would
469 * keep sending IRQs and amos to it until the heartbeat times out.
470 */
471 (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
472 variable),
473 xp_nofault_PIOR_target));
474
475 local_irq_restore(irq_flags);
476}
477
478static void
479xpc_indicate_partition_disengaged_sn2(struct xpc_partition *part)
480{
481 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
482 unsigned long irq_flags;
483 struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa +
484 (XPC_ENGAGED_PARTITIONS_AMO_SN2 *
485 sizeof(struct amo)));
486
487 local_irq_save(irq_flags);
488
489 /* clear bit corresponding to our partid in remote partition's amo */
490 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
491 ~BIT(sn_partition_id));
492
493 /*
494 * We must always use the nofault function regardless of whether we
495 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
496 * didn't, we'd never know that the other partition is down and would
497 * keep sending IRQs and amos to it until the heartbeat times out.
498 */
499 (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
500 variable),
501 xp_nofault_PIOR_target));
502
503 local_irq_restore(irq_flags);
504
505 /*
506 * Send activate IRQ to get other side to see that we've cleared our
507 * bit in their engaged partitions amo.
508 */
509 xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa,
510 cnodeid_to_nasid(0),
511 part_sn2->activate_IRQ_nasid,
512 part_sn2->activate_IRQ_phys_cpuid);
513}
514
515static void
516xpc_assume_partition_disengaged_sn2(short partid)
517{
518 struct amo *amo = xpc_vars_sn2->amos_page +
519 XPC_ENGAGED_PARTITIONS_AMO_SN2;
520
521 /* clear bit(s) based on partid mask in our partition's amo */
522 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
523 ~BIT(partid));
524}
525
526static int
527xpc_partition_engaged_sn2(short partid)
528{
529 struct amo *amo = xpc_vars_sn2->amos_page +
530 XPC_ENGAGED_PARTITIONS_AMO_SN2;
531
532 /* our partition's amo variable ANDed with partid mask */
533 return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) &
534 BIT(partid)) != 0;
535}
536
537static int
538xpc_any_partition_engaged_sn2(void)
539{
540 struct amo *amo = xpc_vars_sn2->amos_page +
541 XPC_ENGAGED_PARTITIONS_AMO_SN2;
542
543 /* our partition's amo variable */
544 return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) != 0;
545}
546
547/* original protection values for each node */
548static u64 xpc_prot_vec_sn2[MAX_NUMNODES];
549
550/*
551 * Change protections to allow amo operations on non-Shub 1.1 systems.
552 */
553static enum xp_retval
554xpc_allow_amo_ops_sn2(struct amo *amos_page)
555{
556 u64 nasid_array = 0;
557 int ret;
558
559 /*
560 * On SHUB 1.1, we cannot call sn_change_memprotect() since the BIST
561 * collides with memory operations. On those systems we call
562 * xpc_allow_amo_ops_shub_wars_1_1_sn2() instead.
563 */
564 if (!enable_shub_wars_1_1()) {
565 ret = sn_change_memprotect(ia64_tpa((u64)amos_page), PAGE_SIZE,
566 SN_MEMPROT_ACCESS_CLASS_1,
567 &nasid_array);
568 if (ret != 0)
569 return xpSalError;
570 }
571 return xpSuccess;
572}
573
574/*
575 * Change protections to allow amo operations on Shub 1.1 systems.
576 */
577static void
578xpc_allow_amo_ops_shub_wars_1_1_sn2(void)
579{
580 int node;
581 int nasid;
582
583 if (!enable_shub_wars_1_1())
584 return;
585
586 for_each_online_node(node) {
587 nasid = cnodeid_to_nasid(node);
588 /* save current protection values */
589 xpc_prot_vec_sn2[node] =
590 (u64)HUB_L((u64 *)GLOBAL_MMR_ADDR(nasid,
591 SH1_MD_DQLP_MMR_DIR_PRIVEC0));
592 /* open up everything */
593 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid,
594 SH1_MD_DQLP_MMR_DIR_PRIVEC0),
595 -1UL);
596 HUB_S((u64 *)GLOBAL_MMR_ADDR(nasid,
597 SH1_MD_DQRP_MMR_DIR_PRIVEC0),
598 -1UL);
599 }
600}
601
602static enum xp_retval
603xpc_get_partition_rsvd_page_pa_sn2(void *buf, u64 *cookie, unsigned long *rp_pa,
604 size_t *len)
605{
606 s64 status;
607 enum xp_retval ret;
608
609 status = sn_partition_reserved_page_pa((u64)buf, cookie, rp_pa, len);
610 if (status == SALRET_OK)
611 ret = xpSuccess;
612 else if (status == SALRET_MORE_PASSES)
613 ret = xpNeedMoreInfo;
614 else
615 ret = xpSalError;
616
617 return ret;
618}
619
620
621static int
622xpc_setup_rsvd_page_sn_sn2(struct xpc_rsvd_page *rp)
623{
624 struct amo *amos_page;
625 int i;
626 int ret;
627
628 xpc_vars_sn2 = XPC_RP_VARS(rp);
629
630 rp->sn.vars_pa = xp_pa(xpc_vars_sn2);
631
632 /* vars_part array follows immediately after vars */
633 xpc_vars_part_sn2 = (struct xpc_vars_part_sn2 *)((u8 *)XPC_RP_VARS(rp) +
634 XPC_RP_VARS_SIZE);
635
636 /*
637 * Before clearing xpc_vars_sn2, see if a page of amos had been
638 * previously allocated. If not we'll need to allocate one and set
639 * permissions so that cross-partition amos are allowed.
640 *
641 * The allocated amo page needs MCA reporting to remain disabled after
642 * XPC has unloaded. To make this work, we keep a copy of the pointer
643 * to this page (i.e., amos_page) in the struct xpc_vars_sn2 structure,
644 * which is pointed to by the reserved page, and re-use that saved copy
645 * on subsequent loads of XPC. This amo page is never freed, and its
646 * memory protections are never restricted.
647 */
648 amos_page = xpc_vars_sn2->amos_page;
649 if (amos_page == NULL) {
650 amos_page = (struct amo *)TO_AMO(uncached_alloc_page(0, 1));
651 if (amos_page == NULL) {
652 dev_err(xpc_part, "can't allocate page of amos\n");
653 return -ENOMEM;
654 }
655
656 /*
657 * Open up amo-R/W to cpu. This is done on Shub 1.1 systems
658 * when xpc_allow_amo_ops_shub_wars_1_1_sn2() is called.
659 */
660 ret = xpc_allow_amo_ops_sn2(amos_page);
661 if (ret != xpSuccess) {
662 dev_err(xpc_part, "can't allow amo operations\n");
663 uncached_free_page(__IA64_UNCACHED_OFFSET |
664 TO_PHYS((u64)amos_page), 1);
665 return -EPERM;
666 }
667 }
668
669 /* clear xpc_vars_sn2 */
670 memset(xpc_vars_sn2, 0, sizeof(struct xpc_vars_sn2));
671
672 xpc_vars_sn2->version = XPC_V_VERSION;
673 xpc_vars_sn2->activate_IRQ_nasid = cpuid_to_nasid(0);
674 xpc_vars_sn2->activate_IRQ_phys_cpuid = cpu_physical_id(0);
675 xpc_vars_sn2->vars_part_pa = xp_pa(xpc_vars_part_sn2);
676 xpc_vars_sn2->amos_page_pa = ia64_tpa((u64)amos_page);
677 xpc_vars_sn2->amos_page = amos_page; /* save for next load of XPC */
678
679 /* clear xpc_vars_part_sn2 */
680 memset((u64 *)xpc_vars_part_sn2, 0, sizeof(struct xpc_vars_part_sn2) *
681 XP_MAX_NPARTITIONS_SN2);
682
683 /* initialize the activate IRQ related amo variables */
684 for (i = 0; i < xpc_nasid_mask_nlongs; i++)
685 (void)xpc_init_IRQ_amo_sn2(XPC_ACTIVATE_IRQ_AMOS_SN2 + i);
686
687 /* initialize the engaged remote partitions related amo variables */
688 (void)xpc_init_IRQ_amo_sn2(XPC_ENGAGED_PARTITIONS_AMO_SN2);
689 (void)xpc_init_IRQ_amo_sn2(XPC_DEACTIVATE_REQUEST_AMO_SN2);
690
691 return 0;
692}
693
694static void
695xpc_increment_heartbeat_sn2(void)
696{
697 xpc_vars_sn2->heartbeat++;
698}
699
700static void
701xpc_offline_heartbeat_sn2(void)
702{
703 xpc_increment_heartbeat_sn2();
704 xpc_vars_sn2->heartbeat_offline = 1;
705}
706
707static void
708xpc_online_heartbeat_sn2(void)
709{
710 xpc_increment_heartbeat_sn2();
711 xpc_vars_sn2->heartbeat_offline = 0;
712}
713
714static void
715xpc_heartbeat_init_sn2(void)
716{
717 DBUG_ON(xpc_vars_sn2 == NULL);
718
719 bitmap_zero(xpc_vars_sn2->heartbeating_to_mask, XP_MAX_NPARTITIONS_SN2);
720 xpc_heartbeating_to_mask = &xpc_vars_sn2->heartbeating_to_mask[0];
721 xpc_online_heartbeat_sn2();
722}
723
724static void
725xpc_heartbeat_exit_sn2(void)
726{
727 xpc_offline_heartbeat_sn2();
728}
729
730static enum xp_retval
731xpc_get_remote_heartbeat_sn2(struct xpc_partition *part)
732{
733 struct xpc_vars_sn2 *remote_vars;
734 enum xp_retval ret;
735
736 remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2;
737
738 /* pull the remote vars structure that contains the heartbeat */
739 ret = xp_remote_memcpy(xp_pa(remote_vars),
740 part->sn.sn2.remote_vars_pa,
741 XPC_RP_VARS_SIZE);
742 if (ret != xpSuccess)
743 return ret;
744
745 dev_dbg(xpc_part, "partid=%d, heartbeat=%ld, last_heartbeat=%ld, "
746 "heartbeat_offline=%ld, HB_mask[0]=0x%lx\n", XPC_PARTID(part),
747 remote_vars->heartbeat, part->last_heartbeat,
748 remote_vars->heartbeat_offline,
749 remote_vars->heartbeating_to_mask[0]);
750
751 if ((remote_vars->heartbeat == part->last_heartbeat &&
752 remote_vars->heartbeat_offline == 0) ||
753 !xpc_hb_allowed(sn_partition_id,
754 &remote_vars->heartbeating_to_mask)) {
755 ret = xpNoHeartbeat;
756 } else {
757 part->last_heartbeat = remote_vars->heartbeat;
758 }
759
760 return ret;
761}
762
763/*
764 * Get a copy of the remote partition's XPC variables from the reserved page.
765 *
766 * remote_vars points to a buffer that is cacheline aligned for BTE copies and
767 * assumed to be of size XPC_RP_VARS_SIZE.
768 */
769static enum xp_retval
770xpc_get_remote_vars_sn2(unsigned long remote_vars_pa,
771 struct xpc_vars_sn2 *remote_vars)
772{
773 enum xp_retval ret;
774
775 if (remote_vars_pa == 0)
776 return xpVarsNotSet;
777
778 /* pull over the cross partition variables */
779 ret = xp_remote_memcpy(xp_pa(remote_vars), remote_vars_pa,
780 XPC_RP_VARS_SIZE);
781 if (ret != xpSuccess)
782 return ret;
783
784 if (XPC_VERSION_MAJOR(remote_vars->version) !=
785 XPC_VERSION_MAJOR(XPC_V_VERSION)) {
786 return xpBadVersion;
787 }
788
789 return xpSuccess;
790}
791
792static void
793xpc_request_partition_activation_sn2(struct xpc_rsvd_page *remote_rp,
794 unsigned long remote_rp_pa, int nasid)
795{
796 xpc_send_local_activate_IRQ_sn2(nasid);
797}
798
799static void
800xpc_request_partition_reactivation_sn2(struct xpc_partition *part)
801{
802 xpc_send_local_activate_IRQ_sn2(part->sn.sn2.activate_IRQ_nasid);
803}
804
805static void
806xpc_request_partition_deactivation_sn2(struct xpc_partition *part)
807{
808 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
809 unsigned long irq_flags;
810 struct amo *amo = (struct amo *)__va(part_sn2->remote_amos_page_pa +
811 (XPC_DEACTIVATE_REQUEST_AMO_SN2 *
812 sizeof(struct amo)));
813
814 local_irq_save(irq_flags);
815
816 /* set bit corresponding to our partid in remote partition's amo */
817 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR,
818 BIT(sn_partition_id));
819
820 /*
821 * We must always use the nofault function regardless of whether we
822 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
823 * didn't, we'd never know that the other partition is down and would
824 * keep sending IRQs and amos to it until the heartbeat times out.
825 */
826 (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
827 variable),
828 xp_nofault_PIOR_target));
829
830 local_irq_restore(irq_flags);
831
832 /*
833 * Send activate IRQ to get other side to see that we've set our
834 * bit in their deactivate request amo.
835 */
836 xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa,
837 cnodeid_to_nasid(0),
838 part_sn2->activate_IRQ_nasid,
839 part_sn2->activate_IRQ_phys_cpuid);
840}
841
842static void
843xpc_cancel_partition_deactivation_request_sn2(struct xpc_partition *part)
844{
845 unsigned long irq_flags;
846 struct amo *amo = (struct amo *)__va(part->sn.sn2.remote_amos_page_pa +
847 (XPC_DEACTIVATE_REQUEST_AMO_SN2 *
848 sizeof(struct amo)));
849
850 local_irq_save(irq_flags);
851
852 /* clear bit corresponding to our partid in remote partition's amo */
853 FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
854 ~BIT(sn_partition_id));
855
856 /*
857 * We must always use the nofault function regardless of whether we
858 * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
859 * didn't, we'd never know that the other partition is down and would
860 * keep sending IRQs and amos to it until the heartbeat times out.
861 */
862 (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
863 variable),
864 xp_nofault_PIOR_target));
865
866 local_irq_restore(irq_flags);
867}
868
869static int
870xpc_partition_deactivation_requested_sn2(short partid)
871{
872 struct amo *amo = xpc_vars_sn2->amos_page +
873 XPC_DEACTIVATE_REQUEST_AMO_SN2;
874
875 /* our partition's amo variable ANDed with partid mask */
876 return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) &
877 BIT(partid)) != 0;
878}
879
880/*
881 * Update the remote partition's info.
882 */
883static void
884xpc_update_partition_info_sn2(struct xpc_partition *part, u8 remote_rp_version,
885 unsigned long *remote_rp_ts_jiffies,
886 unsigned long remote_rp_pa,
887 unsigned long remote_vars_pa,
888 struct xpc_vars_sn2 *remote_vars)
889{
890 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
891
892 part->remote_rp_version = remote_rp_version;
893 dev_dbg(xpc_part, " remote_rp_version = 0x%016x\n",
894 part->remote_rp_version);
895
896 part->remote_rp_ts_jiffies = *remote_rp_ts_jiffies;
897 dev_dbg(xpc_part, " remote_rp_ts_jiffies = 0x%016lx\n",
898 part->remote_rp_ts_jiffies);
899
900 part->remote_rp_pa = remote_rp_pa;
901 dev_dbg(xpc_part, " remote_rp_pa = 0x%016lx\n", part->remote_rp_pa);
902
903 part_sn2->remote_vars_pa = remote_vars_pa;
904 dev_dbg(xpc_part, " remote_vars_pa = 0x%016lx\n",
905 part_sn2->remote_vars_pa);
906
907 part->last_heartbeat = remote_vars->heartbeat;
908 dev_dbg(xpc_part, " last_heartbeat = 0x%016lx\n",
909 part->last_heartbeat);
910
911 part_sn2->remote_vars_part_pa = remote_vars->vars_part_pa;
912 dev_dbg(xpc_part, " remote_vars_part_pa = 0x%016lx\n",
913 part_sn2->remote_vars_part_pa);
914
915 part_sn2->activate_IRQ_nasid = remote_vars->activate_IRQ_nasid;
916 dev_dbg(xpc_part, " activate_IRQ_nasid = 0x%x\n",
917 part_sn2->activate_IRQ_nasid);
918
919 part_sn2->activate_IRQ_phys_cpuid =
920 remote_vars->activate_IRQ_phys_cpuid;
921 dev_dbg(xpc_part, " activate_IRQ_phys_cpuid = 0x%x\n",
922 part_sn2->activate_IRQ_phys_cpuid);
923
924 part_sn2->remote_amos_page_pa = remote_vars->amos_page_pa;
925 dev_dbg(xpc_part, " remote_amos_page_pa = 0x%lx\n",
926 part_sn2->remote_amos_page_pa);
927
928 part_sn2->remote_vars_version = remote_vars->version;
929 dev_dbg(xpc_part, " remote_vars_version = 0x%x\n",
930 part_sn2->remote_vars_version);
931}
932
933/*
934 * Prior code has determined the nasid which generated a activate IRQ.
935 * Inspect that nasid to determine if its partition needs to be activated
936 * or deactivated.
937 *
938 * A partition is considered "awaiting activation" if our partition
939 * flags indicate it is not active and it has a heartbeat. A
940 * partition is considered "awaiting deactivation" if our partition
941 * flags indicate it is active but it has no heartbeat or it is not
942 * sending its heartbeat to us.
943 *
944 * To determine the heartbeat, the remote nasid must have a properly
945 * initialized reserved page.
946 */
947static void
948xpc_identify_activate_IRQ_req_sn2(int nasid)
949{
950 struct xpc_rsvd_page *remote_rp;
951 struct xpc_vars_sn2 *remote_vars;
952 unsigned long remote_rp_pa;
953 unsigned long remote_vars_pa;
954 int remote_rp_version;
955 int reactivate = 0;
956 unsigned long remote_rp_ts_jiffies = 0;
957 short partid;
958 struct xpc_partition *part;
959 struct xpc_partition_sn2 *part_sn2;
960 enum xp_retval ret;
961
962 /* pull over the reserved page structure */
963
964 remote_rp = (struct xpc_rsvd_page *)xpc_remote_copy_buffer_sn2;
965
966 ret = xpc_get_remote_rp(nasid, NULL, remote_rp, &remote_rp_pa);
967 if (ret != xpSuccess) {
968 dev_warn(xpc_part, "unable to get reserved page from nasid %d, "
969 "which sent interrupt, reason=%d\n", nasid, ret);
970 return;
971 }
972
973 remote_vars_pa = remote_rp->sn.vars_pa;
974 remote_rp_version = remote_rp->version;
975 remote_rp_ts_jiffies = remote_rp->ts_jiffies;
976
977 partid = remote_rp->SAL_partid;
978 part = &xpc_partitions[partid];
979 part_sn2 = &part->sn.sn2;
980
981 /* pull over the cross partition variables */
982
983 remote_vars = (struct xpc_vars_sn2 *)xpc_remote_copy_buffer_sn2;
984
985 ret = xpc_get_remote_vars_sn2(remote_vars_pa, remote_vars);
986 if (ret != xpSuccess) {
987 dev_warn(xpc_part, "unable to get XPC variables from nasid %d, "
988 "which sent interrupt, reason=%d\n", nasid, ret);
989
990 XPC_DEACTIVATE_PARTITION(part, ret);
991 return;
992 }
993
994 part->activate_IRQ_rcvd++;
995
996 dev_dbg(xpc_part, "partid for nasid %d is %d; IRQs = %d; HB = "
997 "%ld:0x%lx\n", (int)nasid, (int)partid, part->activate_IRQ_rcvd,
998 remote_vars->heartbeat, remote_vars->heartbeating_to_mask[0]);
999
1000 if (xpc_partition_disengaged(part) &&
1001 part->act_state == XPC_P_AS_INACTIVE) {
1002
1003 xpc_update_partition_info_sn2(part, remote_rp_version,
1004 &remote_rp_ts_jiffies,
1005 remote_rp_pa, remote_vars_pa,
1006 remote_vars);
1007
1008 if (xpc_partition_deactivation_requested_sn2(partid)) {
1009 /*
1010 * Other side is waiting on us to deactivate even though
1011 * we already have.
1012 */
1013 return;
1014 }
1015
1016 xpc_activate_partition(part);
1017 return;
1018 }
1019
1020 DBUG_ON(part->remote_rp_version == 0);
1021 DBUG_ON(part_sn2->remote_vars_version == 0);
1022
1023 if (remote_rp_ts_jiffies != part->remote_rp_ts_jiffies) {
1024
1025 /* the other side rebooted */
1026
1027 DBUG_ON(xpc_partition_engaged_sn2(partid));
1028 DBUG_ON(xpc_partition_deactivation_requested_sn2(partid));
1029
1030 xpc_update_partition_info_sn2(part, remote_rp_version,
1031 &remote_rp_ts_jiffies,
1032 remote_rp_pa, remote_vars_pa,
1033 remote_vars);
1034 reactivate = 1;
1035 }
1036
1037 if (part->disengage_timeout > 0 && !xpc_partition_disengaged(part)) {
1038 /* still waiting on other side to disengage from us */
1039 return;
1040 }
1041
1042 if (reactivate)
1043 XPC_DEACTIVATE_PARTITION(part, xpReactivating);
1044 else if (xpc_partition_deactivation_requested_sn2(partid))
1045 XPC_DEACTIVATE_PARTITION(part, xpOtherGoingDown);
1046}
1047
1048/*
1049 * Loop through the activation amo variables and process any bits
1050 * which are set. Each bit indicates a nasid sending a partition
1051 * activation or deactivation request.
1052 *
1053 * Return #of IRQs detected.
1054 */
1055int
1056xpc_identify_activate_IRQ_sender_sn2(void)
1057{
1058 int l;
1059 int b;
1060 unsigned long nasid_mask_long;
1061 u64 nasid; /* remote nasid */
1062 int n_IRQs_detected = 0;
1063 struct amo *act_amos;
1064
1065 act_amos = xpc_vars_sn2->amos_page + XPC_ACTIVATE_IRQ_AMOS_SN2;
1066
1067 /* scan through activate amo variables looking for non-zero entries */
1068 for (l = 0; l < xpc_nasid_mask_nlongs; l++) {
1069
1070 if (xpc_exiting)
1071 break;
1072
1073 nasid_mask_long = xpc_receive_IRQ_amo_sn2(&act_amos[l]);
1074
1075 b = find_first_bit(&nasid_mask_long, BITS_PER_LONG);
1076 if (b >= BITS_PER_LONG) {
1077 /* no IRQs from nasids in this amo variable */
1078 continue;
1079 }
1080
1081 dev_dbg(xpc_part, "amo[%d] gave back 0x%lx\n", l,
1082 nasid_mask_long);
1083
1084 /*
1085 * If this nasid has been added to the machine since
1086 * our partition was reset, this will retain the
1087 * remote nasid in our reserved pages machine mask.
1088 * This is used in the event of module reload.
1089 */
1090 xpc_mach_nasids[l] |= nasid_mask_long;
1091
1092 /* locate the nasid(s) which sent interrupts */
1093
1094 do {
1095 n_IRQs_detected++;
1096 nasid = (l * BITS_PER_LONG + b) * 2;
1097 dev_dbg(xpc_part, "interrupt from nasid %ld\n", nasid);
1098 xpc_identify_activate_IRQ_req_sn2(nasid);
1099
1100 b = find_next_bit(&nasid_mask_long, BITS_PER_LONG,
1101 b + 1);
1102 } while (b < BITS_PER_LONG);
1103 }
1104 return n_IRQs_detected;
1105}
1106
1107static void
1108xpc_process_activate_IRQ_rcvd_sn2(void)
1109{
1110 unsigned long irq_flags;
1111 int n_IRQs_expected;
1112 int n_IRQs_detected;
1113
1114 DBUG_ON(xpc_activate_IRQ_rcvd == 0);
1115
1116 spin_lock_irqsave(&xpc_activate_IRQ_rcvd_lock, irq_flags);
1117 n_IRQs_expected = xpc_activate_IRQ_rcvd;
1118 xpc_activate_IRQ_rcvd = 0;
1119 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
1120
1121 n_IRQs_detected = xpc_identify_activate_IRQ_sender_sn2();
1122 if (n_IRQs_detected < n_IRQs_expected) {
1123 /* retry once to help avoid missing amo */
1124 (void)xpc_identify_activate_IRQ_sender_sn2();
1125 }
1126}
1127
1128/*
1129 * Setup the channel structures that are sn2 specific.
1130 */
1131static enum xp_retval
1132xpc_setup_ch_structures_sn_sn2(struct xpc_partition *part)
1133{
1134 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
1135 struct xpc_channel_sn2 *ch_sn2;
1136 enum xp_retval retval;
1137 int ret;
1138 int cpuid;
1139 int ch_number;
1140 struct timer_list *timer;
1141 short partid = XPC_PARTID(part);
1142
1143 /* allocate all the required GET/PUT values */
1144
1145 part_sn2->local_GPs =
1146 xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE, GFP_KERNEL,
1147 &part_sn2->local_GPs_base);
1148 if (part_sn2->local_GPs == NULL) {
1149 dev_err(xpc_chan, "can't get memory for local get/put "
1150 "values\n");
1151 return xpNoMemory;
1152 }
1153
1154 part_sn2->remote_GPs =
1155 xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE, GFP_KERNEL,
1156 &part_sn2->remote_GPs_base);
1157 if (part_sn2->remote_GPs == NULL) {
1158 dev_err(xpc_chan, "can't get memory for remote get/put "
1159 "values\n");
1160 retval = xpNoMemory;
1161 goto out_1;
1162 }
1163
1164 part_sn2->remote_GPs_pa = 0;
1165
1166 /* allocate all the required open and close args */
1167
1168 part_sn2->local_openclose_args =
1169 xpc_kzalloc_cacheline_aligned(XPC_OPENCLOSE_ARGS_SIZE,
1170 GFP_KERNEL, &part_sn2->
1171 local_openclose_args_base);
1172 if (part_sn2->local_openclose_args == NULL) {
1173 dev_err(xpc_chan, "can't get memory for local connect args\n");
1174 retval = xpNoMemory;
1175 goto out_2;
1176 }
1177
1178 part_sn2->remote_openclose_args_pa = 0;
1179
1180 part_sn2->local_chctl_amo_va = xpc_init_IRQ_amo_sn2(partid);
1181
1182 part_sn2->notify_IRQ_nasid = 0;
1183 part_sn2->notify_IRQ_phys_cpuid = 0;
1184 part_sn2->remote_chctl_amo_va = NULL;
1185
1186 sprintf(part_sn2->notify_IRQ_owner, "xpc%02d", partid);
1187 ret = request_irq(SGI_XPC_NOTIFY, xpc_handle_notify_IRQ_sn2,
1188 IRQF_SHARED, part_sn2->notify_IRQ_owner,
1189 (void *)(u64)partid);
1190 if (ret != 0) {
1191 dev_err(xpc_chan, "can't register NOTIFY IRQ handler, "
1192 "errno=%d\n", -ret);
1193 retval = xpLackOfResources;
1194 goto out_3;
1195 }
1196
1197 /* Setup a timer to check for dropped notify IRQs */
1198 timer = &part_sn2->dropped_notify_IRQ_timer;
1199 init_timer(timer);
1200 timer->function =
1201 (void (*)(unsigned long))xpc_check_for_dropped_notify_IRQ_sn2;
1202 timer->data = (unsigned long)part;
1203 timer->expires = jiffies + XPC_DROPPED_NOTIFY_IRQ_WAIT_INTERVAL;
1204 add_timer(timer);
1205
1206 for (ch_number = 0; ch_number < part->nchannels; ch_number++) {
1207 ch_sn2 = &part->channels[ch_number].sn.sn2;
1208
1209 ch_sn2->local_GP = &part_sn2->local_GPs[ch_number];
1210 ch_sn2->local_openclose_args =
1211 &part_sn2->local_openclose_args[ch_number];
1212
1213 mutex_init(&ch_sn2->msg_to_pull_mutex);
1214 }
1215
1216 /*
1217 * Setup the per partition specific variables required by the
1218 * remote partition to establish channel connections with us.
1219 *
1220 * The setting of the magic # indicates that these per partition
1221 * specific variables are ready to be used.
1222 */
1223 xpc_vars_part_sn2[partid].GPs_pa = xp_pa(part_sn2->local_GPs);
1224 xpc_vars_part_sn2[partid].openclose_args_pa =
1225 xp_pa(part_sn2->local_openclose_args);
1226 xpc_vars_part_sn2[partid].chctl_amo_pa =
1227 xp_pa(part_sn2->local_chctl_amo_va);
1228 cpuid = raw_smp_processor_id(); /* any CPU in this partition will do */
1229 xpc_vars_part_sn2[partid].notify_IRQ_nasid = cpuid_to_nasid(cpuid);
1230 xpc_vars_part_sn2[partid].notify_IRQ_phys_cpuid =
1231 cpu_physical_id(cpuid);
1232 xpc_vars_part_sn2[partid].nchannels = part->nchannels;
1233 xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC1_SN2;
1234
1235 return xpSuccess;
1236
1237 /* setup of ch structures failed */
1238out_3:
1239 kfree(part_sn2->local_openclose_args_base);
1240 part_sn2->local_openclose_args = NULL;
1241out_2:
1242 kfree(part_sn2->remote_GPs_base);
1243 part_sn2->remote_GPs = NULL;
1244out_1:
1245 kfree(part_sn2->local_GPs_base);
1246 part_sn2->local_GPs = NULL;
1247 return retval;
1248}
1249
1250/*
1251 * Teardown the channel structures that are sn2 specific.
1252 */
1253static void
1254xpc_teardown_ch_structures_sn_sn2(struct xpc_partition *part)
1255{
1256 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
1257 short partid = XPC_PARTID(part);
1258
1259 /*
1260 * Indicate that the variables specific to the remote partition are no
1261 * longer available for its use.
1262 */
1263 xpc_vars_part_sn2[partid].magic = 0;
1264
1265 /* in case we've still got outstanding timers registered... */
1266 del_timer_sync(&part_sn2->dropped_notify_IRQ_timer);
1267 free_irq(SGI_XPC_NOTIFY, (void *)(u64)partid);
1268
1269 kfree(part_sn2->local_openclose_args_base);
1270 part_sn2->local_openclose_args = NULL;
1271 kfree(part_sn2->remote_GPs_base);
1272 part_sn2->remote_GPs = NULL;
1273 kfree(part_sn2->local_GPs_base);
1274 part_sn2->local_GPs = NULL;
1275 part_sn2->local_chctl_amo_va = NULL;
1276}
1277
1278/*
1279 * Create a wrapper that hides the underlying mechanism for pulling a cacheline
1280 * (or multiple cachelines) from a remote partition.
1281 *
1282 * src_pa must be a cacheline aligned physical address on the remote partition.
1283 * dst must be a cacheline aligned virtual address on this partition.
1284 * cnt must be cacheline sized
1285 */
1286/* ??? Replace this function by call to xp_remote_memcpy() or bte_copy()? */
1287static enum xp_retval
1288xpc_pull_remote_cachelines_sn2(struct xpc_partition *part, void *dst,
1289 const unsigned long src_pa, size_t cnt)
1290{
1291 enum xp_retval ret;
1292
1293 DBUG_ON(src_pa != L1_CACHE_ALIGN(src_pa));
1294 DBUG_ON((unsigned long)dst != L1_CACHE_ALIGN((unsigned long)dst));
1295 DBUG_ON(cnt != L1_CACHE_ALIGN(cnt));
1296
1297 if (part->act_state == XPC_P_AS_DEACTIVATING)
1298 return part->reason;
1299
1300 ret = xp_remote_memcpy(xp_pa(dst), src_pa, cnt);
1301 if (ret != xpSuccess) {
1302 dev_dbg(xpc_chan, "xp_remote_memcpy() from partition %d failed,"
1303 " ret=%d\n", XPC_PARTID(part), ret);
1304 }
1305 return ret;
1306}
1307
1308/*
1309 * Pull the remote per partition specific variables from the specified
1310 * partition.
1311 */
1312static enum xp_retval
1313xpc_pull_remote_vars_part_sn2(struct xpc_partition *part)
1314{
1315 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
1316 u8 buffer[L1_CACHE_BYTES * 2];
1317 struct xpc_vars_part_sn2 *pulled_entry_cacheline =
1318 (struct xpc_vars_part_sn2 *)L1_CACHE_ALIGN((u64)buffer);
1319 struct xpc_vars_part_sn2 *pulled_entry;
1320 unsigned long remote_entry_cacheline_pa;
1321 unsigned long remote_entry_pa;
1322 short partid = XPC_PARTID(part);
1323 enum xp_retval ret;
1324
1325 /* pull the cacheline that contains the variables we're interested in */
1326
1327 DBUG_ON(part_sn2->remote_vars_part_pa !=
1328 L1_CACHE_ALIGN(part_sn2->remote_vars_part_pa));
1329 DBUG_ON(sizeof(struct xpc_vars_part_sn2) != L1_CACHE_BYTES / 2);
1330
1331 remote_entry_pa = part_sn2->remote_vars_part_pa +
1332 sn_partition_id * sizeof(struct xpc_vars_part_sn2);
1333
1334 remote_entry_cacheline_pa = (remote_entry_pa & ~(L1_CACHE_BYTES - 1));
1335
1336 pulled_entry = (struct xpc_vars_part_sn2 *)((u64)pulled_entry_cacheline
1337 + (remote_entry_pa &
1338 (L1_CACHE_BYTES - 1)));
1339
1340 ret = xpc_pull_remote_cachelines_sn2(part, pulled_entry_cacheline,
1341 remote_entry_cacheline_pa,
1342 L1_CACHE_BYTES);
1343 if (ret != xpSuccess) {
1344 dev_dbg(xpc_chan, "failed to pull XPC vars_part from "
1345 "partition %d, ret=%d\n", partid, ret);
1346 return ret;
1347 }
1348
1349 /* see if they've been set up yet */
1350
1351 if (pulled_entry->magic != XPC_VP_MAGIC1_SN2 &&
1352 pulled_entry->magic != XPC_VP_MAGIC2_SN2) {
1353
1354 if (pulled_entry->magic != 0) {
1355 dev_dbg(xpc_chan, "partition %d's XPC vars_part for "
1356 "partition %d has bad magic value (=0x%lx)\n",
1357 partid, sn_partition_id, pulled_entry->magic);
1358 return xpBadMagic;
1359 }
1360
1361 /* they've not been initialized yet */
1362 return xpRetry;
1363 }
1364
1365 if (xpc_vars_part_sn2[partid].magic == XPC_VP_MAGIC1_SN2) {
1366
1367 /* validate the variables */
1368
1369 if (pulled_entry->GPs_pa == 0 ||
1370 pulled_entry->openclose_args_pa == 0 ||
1371 pulled_entry->chctl_amo_pa == 0) {
1372
1373 dev_err(xpc_chan, "partition %d's XPC vars_part for "
1374 "partition %d are not valid\n", partid,
1375 sn_partition_id);
1376 return xpInvalidAddress;
1377 }
1378
1379 /* the variables we imported look to be valid */
1380
1381 part_sn2->remote_GPs_pa = pulled_entry->GPs_pa;
1382 part_sn2->remote_openclose_args_pa =
1383 pulled_entry->openclose_args_pa;
1384 part_sn2->remote_chctl_amo_va =
1385 (struct amo *)__va(pulled_entry->chctl_amo_pa);
1386 part_sn2->notify_IRQ_nasid = pulled_entry->notify_IRQ_nasid;
1387 part_sn2->notify_IRQ_phys_cpuid =
1388 pulled_entry->notify_IRQ_phys_cpuid;
1389
1390 if (part->nchannels > pulled_entry->nchannels)
1391 part->nchannels = pulled_entry->nchannels;
1392
1393 /* let the other side know that we've pulled their variables */
1394
1395 xpc_vars_part_sn2[partid].magic = XPC_VP_MAGIC2_SN2;
1396 }
1397
1398 if (pulled_entry->magic == XPC_VP_MAGIC1_SN2)
1399 return xpRetry;
1400
1401 return xpSuccess;
1402}
1403
1404/*
1405 * Establish first contact with the remote partititon. This involves pulling
1406 * the XPC per partition variables from the remote partition and waiting for
1407 * the remote partition to pull ours.
1408 */
1409static enum xp_retval
1410xpc_make_first_contact_sn2(struct xpc_partition *part)
1411{
1412 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
1413 enum xp_retval ret;
1414
1415 /*
1416 * Register the remote partition's amos with SAL so it can handle
1417 * and cleanup errors within that address range should the remote
1418 * partition go down. We don't unregister this range because it is
1419 * difficult to tell when outstanding writes to the remote partition
1420 * are finished and thus when it is safe to unregister. This should
1421 * not result in wasted space in the SAL xp_addr_region table because
1422 * we should get the same page for remote_amos_page_pa after module
1423 * reloads and system reboots.
1424 */
1425 if (sn_register_xp_addr_region(part_sn2->remote_amos_page_pa,
1426 PAGE_SIZE, 1) < 0) {
1427 dev_warn(xpc_part, "xpc_activating(%d) failed to register "
1428 "xp_addr region\n", XPC_PARTID(part));
1429
1430 ret = xpPhysAddrRegFailed;
1431 XPC_DEACTIVATE_PARTITION(part, ret);
1432 return ret;
1433 }
1434
1435 /*
1436 * Send activate IRQ to get other side to activate if they've not
1437 * already begun to do so.
1438 */
1439 xpc_send_activate_IRQ_sn2(part_sn2->remote_amos_page_pa,
1440 cnodeid_to_nasid(0),
1441 part_sn2->activate_IRQ_nasid,
1442 part_sn2->activate_IRQ_phys_cpuid);
1443
1444 while ((ret = xpc_pull_remote_vars_part_sn2(part)) != xpSuccess) {
1445 if (ret != xpRetry) {
1446 XPC_DEACTIVATE_PARTITION(part, ret);
1447 return ret;
1448 }
1449
1450 dev_dbg(xpc_part, "waiting to make first contact with "
1451 "partition %d\n", XPC_PARTID(part));
1452
1453 /* wait a 1/4 of a second or so */
1454 (void)msleep_interruptible(250);
1455
1456 if (part->act_state == XPC_P_AS_DEACTIVATING)
1457 return part->reason;
1458 }
1459
1460 return xpSuccess;
1461}
1462
1463/*
1464 * Get the chctl flags and pull the openclose args and/or remote GPs as needed.
1465 */
1466static u64
1467xpc_get_chctl_all_flags_sn2(struct xpc_partition *part)
1468{
1469 struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
1470 unsigned long irq_flags;
1471 union xpc_channel_ctl_flags chctl;
1472 enum xp_retval ret;
1473
1474 /*
1475 * See if there are any chctl flags to be handled.
1476 */
1477
1478 spin_lock_irqsave(&part->chctl_lock, irq_flags);
1479 chctl = part->chctl;
1480 if (chctl.all_flags != 0)
1481 part->chctl.all_flags = 0;
1482
1483 spin_unlock_irqrestore(&part->chctl_lock, irq_flags);
1484
1485 if (xpc_any_openclose_chctl_flags_set(&chctl)) {
1486 ret = xpc_pull_remote_cachelines_sn2(part, part->
1487 remote_openclose_args,
1488 part_sn2->
1489 remote_openclose_args_pa,
1490 XPC_OPENCLOSE_ARGS_SIZE);
1491 if (ret != xpSuccess) {
1492 XPC_DEACTIVATE_PARTITION(part, ret);
1493
1494 dev_dbg(xpc_chan, "failed to pull openclose args from "
1495 "partition %d, ret=%d\n", XPC_PARTID(part),
1496 ret);
1497
1498 /* don't bother processing chctl flags anymore */
1499 chctl.all_flags = 0;
1500 }
1501 }
1502
1503 if (xpc_any_msg_chctl_flags_set(&chctl)) {
1504 ret = xpc_pull_remote_cachelines_sn2(part, part_sn2->remote_GPs,
1505 part_sn2->remote_GPs_pa,
1506 XPC_GP_SIZE);
1507 if (ret != xpSuccess) {
1508 XPC_DEACTIVATE_PARTITION(part, ret);
1509
1510 dev_dbg(xpc_chan, "failed to pull GPs from partition "
1511 "%d, ret=%d\n", XPC_PARTID(part), ret);
1512
1513 /* don't bother processing chctl flags anymore */
1514 chctl.all_flags = 0;
1515 }
1516 }
1517
1518 return chctl.all_flags;
1519}
1520
1521/*
1522 * Allocate the local message queue and the notify queue.
1523 */
1524static enum xp_retval
1525xpc_allocate_local_msgqueue_sn2(struct xpc_channel *ch)
1526{
1527 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1528 unsigned long irq_flags;
1529 int nentries;
1530 size_t nbytes;
1531
1532 for (nentries = ch->local_nentries; nentries > 0; nentries--) {
1533
1534 nbytes = nentries * ch->entry_size;
1535 ch_sn2->local_msgqueue =
1536 xpc_kzalloc_cacheline_aligned(nbytes, GFP_KERNEL,
1537 &ch_sn2->local_msgqueue_base);
1538 if (ch_sn2->local_msgqueue == NULL)
1539 continue;
1540
1541 nbytes = nentries * sizeof(struct xpc_notify_sn2);
1542 ch_sn2->notify_queue = kzalloc(nbytes, GFP_KERNEL);
1543 if (ch_sn2->notify_queue == NULL) {
1544 kfree(ch_sn2->local_msgqueue_base);
1545 ch_sn2->local_msgqueue = NULL;
1546 continue;
1547 }
1548
1549 spin_lock_irqsave(&ch->lock, irq_flags);
1550 if (nentries < ch->local_nentries) {
1551 dev_dbg(xpc_chan, "nentries=%d local_nentries=%d, "
1552 "partid=%d, channel=%d\n", nentries,
1553 ch->local_nentries, ch->partid, ch->number);
1554
1555 ch->local_nentries = nentries;
1556 }
1557 spin_unlock_irqrestore(&ch->lock, irq_flags);
1558 return xpSuccess;
1559 }
1560
1561 dev_dbg(xpc_chan, "can't get memory for local message queue and notify "
1562 "queue, partid=%d, channel=%d\n", ch->partid, ch->number);
1563 return xpNoMemory;
1564}
1565
1566/*
1567 * Allocate the cached remote message queue.
1568 */
1569static enum xp_retval
1570xpc_allocate_remote_msgqueue_sn2(struct xpc_channel *ch)
1571{
1572 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1573 unsigned long irq_flags;
1574 int nentries;
1575 size_t nbytes;
1576
1577 DBUG_ON(ch->remote_nentries <= 0);
1578
1579 for (nentries = ch->remote_nentries; nentries > 0; nentries--) {
1580
1581 nbytes = nentries * ch->entry_size;
1582 ch_sn2->remote_msgqueue =
1583 xpc_kzalloc_cacheline_aligned(nbytes, GFP_KERNEL, &ch_sn2->
1584 remote_msgqueue_base);
1585 if (ch_sn2->remote_msgqueue == NULL)
1586 continue;
1587
1588 spin_lock_irqsave(&ch->lock, irq_flags);
1589 if (nentries < ch->remote_nentries) {
1590 dev_dbg(xpc_chan, "nentries=%d remote_nentries=%d, "
1591 "partid=%d, channel=%d\n", nentries,
1592 ch->remote_nentries, ch->partid, ch->number);
1593
1594 ch->remote_nentries = nentries;
1595 }
1596 spin_unlock_irqrestore(&ch->lock, irq_flags);
1597 return xpSuccess;
1598 }
1599
1600 dev_dbg(xpc_chan, "can't get memory for cached remote message queue, "
1601 "partid=%d, channel=%d\n", ch->partid, ch->number);
1602 return xpNoMemory;
1603}
1604
1605/*
1606 * Allocate message queues and other stuff associated with a channel.
1607 *
1608 * Note: Assumes all of the channel sizes are filled in.
1609 */
1610static enum xp_retval
1611xpc_setup_msg_structures_sn2(struct xpc_channel *ch)
1612{
1613 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1614 enum xp_retval ret;
1615
1616 DBUG_ON(ch->flags & XPC_C_SETUP);
1617
1618 ret = xpc_allocate_local_msgqueue_sn2(ch);
1619 if (ret == xpSuccess) {
1620
1621 ret = xpc_allocate_remote_msgqueue_sn2(ch);
1622 if (ret != xpSuccess) {
1623 kfree(ch_sn2->local_msgqueue_base);
1624 ch_sn2->local_msgqueue = NULL;
1625 kfree(ch_sn2->notify_queue);
1626 ch_sn2->notify_queue = NULL;
1627 }
1628 }
1629 return ret;
1630}
1631
1632/*
1633 * Free up message queues and other stuff that were allocated for the specified
1634 * channel.
1635 */
1636static void
1637xpc_teardown_msg_structures_sn2(struct xpc_channel *ch)
1638{
1639 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1640
1641 DBUG_ON(!spin_is_locked(&ch->lock));
1642
1643 ch_sn2->remote_msgqueue_pa = 0;
1644
1645 ch_sn2->local_GP->get = 0;
1646 ch_sn2->local_GP->put = 0;
1647 ch_sn2->remote_GP.get = 0;
1648 ch_sn2->remote_GP.put = 0;
1649 ch_sn2->w_local_GP.get = 0;
1650 ch_sn2->w_local_GP.put = 0;
1651 ch_sn2->w_remote_GP.get = 0;
1652 ch_sn2->w_remote_GP.put = 0;
1653 ch_sn2->next_msg_to_pull = 0;
1654
1655 if (ch->flags & XPC_C_SETUP) {
1656 dev_dbg(xpc_chan, "ch->flags=0x%x, partid=%d, channel=%d\n",
1657 ch->flags, ch->partid, ch->number);
1658
1659 kfree(ch_sn2->local_msgqueue_base);
1660 ch_sn2->local_msgqueue = NULL;
1661 kfree(ch_sn2->remote_msgqueue_base);
1662 ch_sn2->remote_msgqueue = NULL;
1663 kfree(ch_sn2->notify_queue);
1664 ch_sn2->notify_queue = NULL;
1665 }
1666}
1667
1668/*
1669 * Notify those who wanted to be notified upon delivery of their message.
1670 */
1671static void
1672xpc_notify_senders_sn2(struct xpc_channel *ch, enum xp_retval reason, s64 put)
1673{
1674 struct xpc_notify_sn2 *notify;
1675 u8 notify_type;
1676 s64 get = ch->sn.sn2.w_remote_GP.get - 1;
1677
1678 while (++get < put && atomic_read(&ch->n_to_notify) > 0) {
1679
1680 notify = &ch->sn.sn2.notify_queue[get % ch->local_nentries];
1681
1682 /*
1683 * See if the notify entry indicates it was associated with
1684 * a message who's sender wants to be notified. It is possible
1685 * that it is, but someone else is doing or has done the
1686 * notification.
1687 */
1688 notify_type = notify->type;
1689 if (notify_type == 0 ||
1690 cmpxchg(&notify->type, notify_type, 0) != notify_type) {
1691 continue;
1692 }
1693
1694 DBUG_ON(notify_type != XPC_N_CALL);
1695
1696 atomic_dec(&ch->n_to_notify);
1697
1698 if (notify->func != NULL) {
1699 dev_dbg(xpc_chan, "notify->func() called, notify=0x%p "
1700 "msg_number=%ld partid=%d channel=%d\n",
1701 (void *)notify, get, ch->partid, ch->number);
1702
1703 notify->func(reason, ch->partid, ch->number,
1704 notify->key);
1705
1706 dev_dbg(xpc_chan, "notify->func() returned, notify=0x%p"
1707 " msg_number=%ld partid=%d channel=%d\n",
1708 (void *)notify, get, ch->partid, ch->number);
1709 }
1710 }
1711}
1712
1713static void
1714xpc_notify_senders_of_disconnect_sn2(struct xpc_channel *ch)
1715{
1716 xpc_notify_senders_sn2(ch, ch->reason, ch->sn.sn2.w_local_GP.put);
1717}
1718
1719/*
1720 * Clear some of the msg flags in the local message queue.
1721 */
1722static inline void
1723xpc_clear_local_msgqueue_flags_sn2(struct xpc_channel *ch)
1724{
1725 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1726 struct xpc_msg_sn2 *msg;
1727 s64 get;
1728
1729 get = ch_sn2->w_remote_GP.get;
1730 do {
1731 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->local_msgqueue +
1732 (get % ch->local_nentries) *
1733 ch->entry_size);
1734 msg->flags = 0;
1735 } while (++get < ch_sn2->remote_GP.get);
1736}
1737
1738/*
1739 * Clear some of the msg flags in the remote message queue.
1740 */
1741static inline void
1742xpc_clear_remote_msgqueue_flags_sn2(struct xpc_channel *ch)
1743{
1744 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1745 struct xpc_msg_sn2 *msg;
1746 s64 put;
1747
1748 put = ch_sn2->w_remote_GP.put;
1749 do {
1750 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue +
1751 (put % ch->remote_nentries) *
1752 ch->entry_size);
1753 msg->flags = 0;
1754 } while (++put < ch_sn2->remote_GP.put);
1755}
1756
1757static int
1758xpc_n_of_deliverable_payloads_sn2(struct xpc_channel *ch)
1759{
1760 return ch->sn.sn2.w_remote_GP.put - ch->sn.sn2.w_local_GP.get;
1761}
1762
1763static void
1764xpc_process_msg_chctl_flags_sn2(struct xpc_partition *part, int ch_number)
1765{
1766 struct xpc_channel *ch = &part->channels[ch_number];
1767 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1768 int npayloads_sent;
1769
1770 ch_sn2->remote_GP = part->sn.sn2.remote_GPs[ch_number];
1771
1772 /* See what, if anything, has changed for each connected channel */
1773
1774 xpc_msgqueue_ref(ch);
1775
1776 if (ch_sn2->w_remote_GP.get == ch_sn2->remote_GP.get &&
1777 ch_sn2->w_remote_GP.put == ch_sn2->remote_GP.put) {
1778 /* nothing changed since GPs were last pulled */
1779 xpc_msgqueue_deref(ch);
1780 return;
1781 }
1782
1783 if (!(ch->flags & XPC_C_CONNECTED)) {
1784 xpc_msgqueue_deref(ch);
1785 return;
1786 }
1787
1788 /*
1789 * First check to see if messages recently sent by us have been
1790 * received by the other side. (The remote GET value will have
1791 * changed since we last looked at it.)
1792 */
1793
1794 if (ch_sn2->w_remote_GP.get != ch_sn2->remote_GP.get) {
1795
1796 /*
1797 * We need to notify any senders that want to be notified
1798 * that their sent messages have been received by their
1799 * intended recipients. We need to do this before updating
1800 * w_remote_GP.get so that we don't allocate the same message
1801 * queue entries prematurely (see xpc_allocate_msg()).
1802 */
1803 if (atomic_read(&ch->n_to_notify) > 0) {
1804 /*
1805 * Notify senders that messages sent have been
1806 * received and delivered by the other side.
1807 */
1808 xpc_notify_senders_sn2(ch, xpMsgDelivered,
1809 ch_sn2->remote_GP.get);
1810 }
1811
1812 /*
1813 * Clear msg->flags in previously sent messages, so that
1814 * they're ready for xpc_allocate_msg().
1815 */
1816 xpc_clear_local_msgqueue_flags_sn2(ch);
1817
1818 ch_sn2->w_remote_GP.get = ch_sn2->remote_GP.get;
1819
1820 dev_dbg(xpc_chan, "w_remote_GP.get changed to %ld, partid=%d, "
1821 "channel=%d\n", ch_sn2->w_remote_GP.get, ch->partid,
1822 ch->number);
1823
1824 /*
1825 * If anyone was waiting for message queue entries to become
1826 * available, wake them up.
1827 */
1828 if (atomic_read(&ch->n_on_msg_allocate_wq) > 0)
1829 wake_up(&ch->msg_allocate_wq);
1830 }
1831
1832 /*
1833 * Now check for newly sent messages by the other side. (The remote
1834 * PUT value will have changed since we last looked at it.)
1835 */
1836
1837 if (ch_sn2->w_remote_GP.put != ch_sn2->remote_GP.put) {
1838 /*
1839 * Clear msg->flags in previously received messages, so that
1840 * they're ready for xpc_get_deliverable_payload_sn2().
1841 */
1842 xpc_clear_remote_msgqueue_flags_sn2(ch);
1843
1844 ch_sn2->w_remote_GP.put = ch_sn2->remote_GP.put;
1845
1846 dev_dbg(xpc_chan, "w_remote_GP.put changed to %ld, partid=%d, "
1847 "channel=%d\n", ch_sn2->w_remote_GP.put, ch->partid,
1848 ch->number);
1849
1850 npayloads_sent = xpc_n_of_deliverable_payloads_sn2(ch);
1851 if (npayloads_sent > 0) {
1852 dev_dbg(xpc_chan, "msgs waiting to be copied and "
1853 "delivered=%d, partid=%d, channel=%d\n",
1854 npayloads_sent, ch->partid, ch->number);
1855
1856 if (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE)
1857 xpc_activate_kthreads(ch, npayloads_sent);
1858 }
1859 }
1860
1861 xpc_msgqueue_deref(ch);
1862}
1863
1864static struct xpc_msg_sn2 *
1865xpc_pull_remote_msg_sn2(struct xpc_channel *ch, s64 get)
1866{
1867 struct xpc_partition *part = &xpc_partitions[ch->partid];
1868 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1869 unsigned long remote_msg_pa;
1870 struct xpc_msg_sn2 *msg;
1871 u32 msg_index;
1872 u32 nmsgs;
1873 u64 msg_offset;
1874 enum xp_retval ret;
1875
1876 if (mutex_lock_interruptible(&ch_sn2->msg_to_pull_mutex) != 0) {
1877 /* we were interrupted by a signal */
1878 return NULL;
1879 }
1880
1881 while (get >= ch_sn2->next_msg_to_pull) {
1882
1883 /* pull as many messages as are ready and able to be pulled */
1884
1885 msg_index = ch_sn2->next_msg_to_pull % ch->remote_nentries;
1886
1887 DBUG_ON(ch_sn2->next_msg_to_pull >= ch_sn2->w_remote_GP.put);
1888 nmsgs = ch_sn2->w_remote_GP.put - ch_sn2->next_msg_to_pull;
1889 if (msg_index + nmsgs > ch->remote_nentries) {
1890 /* ignore the ones that wrap the msg queue for now */
1891 nmsgs = ch->remote_nentries - msg_index;
1892 }
1893
1894 msg_offset = msg_index * ch->entry_size;
1895 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue +
1896 msg_offset);
1897 remote_msg_pa = ch_sn2->remote_msgqueue_pa + msg_offset;
1898
1899 ret = xpc_pull_remote_cachelines_sn2(part, msg, remote_msg_pa,
1900 nmsgs * ch->entry_size);
1901 if (ret != xpSuccess) {
1902
1903 dev_dbg(xpc_chan, "failed to pull %d msgs starting with"
1904 " msg %ld from partition %d, channel=%d, "
1905 "ret=%d\n", nmsgs, ch_sn2->next_msg_to_pull,
1906 ch->partid, ch->number, ret);
1907
1908 XPC_DEACTIVATE_PARTITION(part, ret);
1909
1910 mutex_unlock(&ch_sn2->msg_to_pull_mutex);
1911 return NULL;
1912 }
1913
1914 ch_sn2->next_msg_to_pull += nmsgs;
1915 }
1916
1917 mutex_unlock(&ch_sn2->msg_to_pull_mutex);
1918
1919 /* return the message we were looking for */
1920 msg_offset = (get % ch->remote_nentries) * ch->entry_size;
1921 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue + msg_offset);
1922
1923 return msg;
1924}
1925
1926/*
1927 * Get the next deliverable message's payload.
1928 */
1929static void *
1930xpc_get_deliverable_payload_sn2(struct xpc_channel *ch)
1931{
1932 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1933 struct xpc_msg_sn2 *msg;
1934 void *payload = NULL;
1935 s64 get;
1936
1937 do {
1938 if (ch->flags & XPC_C_DISCONNECTING)
1939 break;
1940
1941 get = ch_sn2->w_local_GP.get;
1942 rmb(); /* guarantee that .get loads before .put */
1943 if (get == ch_sn2->w_remote_GP.put)
1944 break;
1945
1946 /* There are messages waiting to be pulled and delivered.
1947 * We need to try to secure one for ourselves. We'll do this
1948 * by trying to increment w_local_GP.get and hope that no one
1949 * else beats us to it. If they do, we'll we'll simply have
1950 * to try again for the next one.
1951 */
1952
1953 if (cmpxchg(&ch_sn2->w_local_GP.get, get, get + 1) == get) {
1954 /* we got the entry referenced by get */
1955
1956 dev_dbg(xpc_chan, "w_local_GP.get changed to %ld, "
1957 "partid=%d, channel=%d\n", get + 1,
1958 ch->partid, ch->number);
1959
1960 /* pull the message from the remote partition */
1961
1962 msg = xpc_pull_remote_msg_sn2(ch, get);
1963
1964 DBUG_ON(msg != NULL && msg->number != get);
1965 DBUG_ON(msg != NULL && (msg->flags & XPC_M_SN2_DONE));
1966 DBUG_ON(msg != NULL && !(msg->flags & XPC_M_SN2_READY));
1967
1968 payload = &msg->payload;
1969 break;
1970 }
1971
1972 } while (1);
1973
1974 return payload;
1975}
1976
1977/*
1978 * Now we actually send the messages that are ready to be sent by advancing
1979 * the local message queue's Put value and then send a chctl msgrequest to the
1980 * recipient partition.
1981 */
1982static void
1983xpc_send_msgs_sn2(struct xpc_channel *ch, s64 initial_put)
1984{
1985 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1986 struct xpc_msg_sn2 *msg;
1987 s64 put = initial_put + 1;
1988 int send_msgrequest = 0;
1989
1990 while (1) {
1991
1992 while (1) {
1993 if (put == ch_sn2->w_local_GP.put)
1994 break;
1995
1996 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->
1997 local_msgqueue + (put %
1998 ch->local_nentries) *
1999 ch->entry_size);
2000
2001 if (!(msg->flags & XPC_M_SN2_READY))
2002 break;
2003
2004 put++;
2005 }
2006
2007 if (put == initial_put) {
2008 /* nothing's changed */
2009 break;
2010 }
2011
2012 if (cmpxchg_rel(&ch_sn2->local_GP->put, initial_put, put) !=
2013 initial_put) {
2014 /* someone else beat us to it */
2015 DBUG_ON(ch_sn2->local_GP->put < initial_put);
2016 break;
2017 }
2018
2019 /* we just set the new value of local_GP->put */
2020
2021 dev_dbg(xpc_chan, "local_GP->put changed to %ld, partid=%d, "
2022 "channel=%d\n", put, ch->partid, ch->number);
2023
2024 send_msgrequest = 1;
2025
2026 /*
2027 * We need to ensure that the message referenced by
2028 * local_GP->put is not XPC_M_SN2_READY or that local_GP->put
2029 * equals w_local_GP.put, so we'll go have a look.
2030 */
2031 initial_put = put;
2032 }
2033
2034 if (send_msgrequest)
2035 xpc_send_chctl_msgrequest_sn2(ch);
2036}
2037
2038/*
2039 * Allocate an entry for a message from the message queue associated with the
2040 * specified channel.
2041 */
2042static enum xp_retval
2043xpc_allocate_msg_sn2(struct xpc_channel *ch, u32 flags,
2044 struct xpc_msg_sn2 **address_of_msg)
2045{
2046 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
2047 struct xpc_msg_sn2 *msg;
2048 enum xp_retval ret;
2049 s64 put;
2050
2051 /*
2052 * Get the next available message entry from the local message queue.
2053 * If none are available, we'll make sure that we grab the latest
2054 * GP values.
2055 */
2056 ret = xpTimeout;
2057
2058 while (1) {
2059
2060 put = ch_sn2->w_local_GP.put;
2061 rmb(); /* guarantee that .put loads before .get */
2062 if (put - ch_sn2->w_remote_GP.get < ch->local_nentries) {
2063
2064 /* There are available message entries. We need to try
2065 * to secure one for ourselves. We'll do this by trying
2066 * to increment w_local_GP.put as long as someone else
2067 * doesn't beat us to it. If they do, we'll have to
2068 * try again.
2069 */
2070 if (cmpxchg(&ch_sn2->w_local_GP.put, put, put + 1) ==
2071 put) {
2072 /* we got the entry referenced by put */
2073 break;
2074 }
2075 continue; /* try again */
2076 }
2077
2078 /*
2079 * There aren't any available msg entries at this time.
2080 *
2081 * In waiting for a message entry to become available,
2082 * we set a timeout in case the other side is not sending
2083 * completion interrupts. This lets us fake a notify IRQ
2084 * that will cause the notify IRQ handler to fetch the latest
2085 * GP values as if an interrupt was sent by the other side.
2086 */
2087 if (ret == xpTimeout)
2088 xpc_send_chctl_local_msgrequest_sn2(ch);
2089
2090 if (flags & XPC_NOWAIT)
2091 return xpNoWait;
2092
2093 ret = xpc_allocate_msg_wait(ch);
2094 if (ret != xpInterrupted && ret != xpTimeout)
2095 return ret;
2096 }
2097
2098 /* get the message's address and initialize it */
2099 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->local_msgqueue +
2100 (put % ch->local_nentries) *
2101 ch->entry_size);
2102
2103 DBUG_ON(msg->flags != 0);
2104 msg->number = put;
2105
2106 dev_dbg(xpc_chan, "w_local_GP.put changed to %ld; msg=0x%p, "
2107 "msg_number=%ld, partid=%d, channel=%d\n", put + 1,
2108 (void *)msg, msg->number, ch->partid, ch->number);
2109
2110 *address_of_msg = msg;
2111 return xpSuccess;
2112}
2113
2114/*
2115 * Common code that does the actual sending of the message by advancing the
2116 * local message queue's Put value and sends a chctl msgrequest to the
2117 * partition the message is being sent to.
2118 */
2119static enum xp_retval
2120xpc_send_payload_sn2(struct xpc_channel *ch, u32 flags, void *payload,
2121 u16 payload_size, u8 notify_type, xpc_notify_func func,
2122 void *key)
2123{
2124 enum xp_retval ret = xpSuccess;
2125 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
2126 struct xpc_msg_sn2 *msg = msg;
2127 struct xpc_notify_sn2 *notify = notify;
2128 s64 msg_number;
2129 s64 put;
2130
2131 DBUG_ON(notify_type == XPC_N_CALL && func == NULL);
2132
2133 if (XPC_MSG_SIZE(payload_size) > ch->entry_size)
2134 return xpPayloadTooBig;
2135
2136 xpc_msgqueue_ref(ch);
2137
2138 if (ch->flags & XPC_C_DISCONNECTING) {
2139 ret = ch->reason;
2140 goto out_1;
2141 }
2142 if (!(ch->flags & XPC_C_CONNECTED)) {
2143 ret = xpNotConnected;
2144 goto out_1;
2145 }
2146
2147 ret = xpc_allocate_msg_sn2(ch, flags, &msg);
2148 if (ret != xpSuccess)
2149 goto out_1;
2150
2151 msg_number = msg->number;
2152
2153 if (notify_type != 0) {
2154 /*
2155 * Tell the remote side to send an ACK interrupt when the
2156 * message has been delivered.
2157 */
2158 msg->flags |= XPC_M_SN2_INTERRUPT;
2159
2160 atomic_inc(&ch->n_to_notify);
2161
2162 notify = &ch_sn2->notify_queue[msg_number % ch->local_nentries];
2163 notify->func = func;
2164 notify->key = key;
2165 notify->type = notify_type;
2166
2167 /* ??? Is a mb() needed here? */
2168
2169 if (ch->flags & XPC_C_DISCONNECTING) {
2170 /*
2171 * An error occurred between our last error check and
2172 * this one. We will try to clear the type field from
2173 * the notify entry. If we succeed then
2174 * xpc_disconnect_channel() didn't already process
2175 * the notify entry.
2176 */
2177 if (cmpxchg(&notify->type, notify_type, 0) ==
2178 notify_type) {
2179 atomic_dec(&ch->n_to_notify);
2180 ret = ch->reason;
2181 }
2182 goto out_1;
2183 }
2184 }
2185
2186 memcpy(&msg->payload, payload, payload_size);
2187
2188 msg->flags |= XPC_M_SN2_READY;
2189
2190 /*
2191 * The preceding store of msg->flags must occur before the following
2192 * load of local_GP->put.
2193 */
2194 mb();
2195
2196 /* see if the message is next in line to be sent, if so send it */
2197
2198 put = ch_sn2->local_GP->put;
2199 if (put == msg_number)
2200 xpc_send_msgs_sn2(ch, put);
2201
2202out_1:
2203 xpc_msgqueue_deref(ch);
2204 return ret;
2205}
2206
2207/*
2208 * Now we actually acknowledge the messages that have been delivered and ack'd
2209 * by advancing the cached remote message queue's Get value and if requested
2210 * send a chctl msgrequest to the message sender's partition.
2211 *
2212 * If a message has XPC_M_SN2_INTERRUPT set, send an interrupt to the partition
2213 * that sent the message.
2214 */
2215static void
2216xpc_acknowledge_msgs_sn2(struct xpc_channel *ch, s64 initial_get, u8 msg_flags)
2217{
2218 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
2219 struct xpc_msg_sn2 *msg;
2220 s64 get = initial_get + 1;
2221 int send_msgrequest = 0;
2222
2223 while (1) {
2224
2225 while (1) {
2226 if (get == ch_sn2->w_local_GP.get)
2227 break;
2228
2229 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->
2230 remote_msgqueue + (get %
2231 ch->remote_nentries) *
2232 ch->entry_size);
2233
2234 if (!(msg->flags & XPC_M_SN2_DONE))
2235 break;
2236
2237 msg_flags |= msg->flags;
2238 get++;
2239 }
2240
2241 if (get == initial_get) {
2242 /* nothing's changed */
2243 break;
2244 }
2245
2246 if (cmpxchg_rel(&ch_sn2->local_GP->get, initial_get, get) !=
2247 initial_get) {
2248 /* someone else beat us to it */
2249 DBUG_ON(ch_sn2->local_GP->get <= initial_get);
2250 break;
2251 }
2252
2253 /* we just set the new value of local_GP->get */
2254
2255 dev_dbg(xpc_chan, "local_GP->get changed to %ld, partid=%d, "
2256 "channel=%d\n", get, ch->partid, ch->number);
2257
2258 send_msgrequest = (msg_flags & XPC_M_SN2_INTERRUPT);
2259
2260 /*
2261 * We need to ensure that the message referenced by
2262 * local_GP->get is not XPC_M_SN2_DONE or that local_GP->get
2263 * equals w_local_GP.get, so we'll go have a look.
2264 */
2265 initial_get = get;
2266 }
2267
2268 if (send_msgrequest)
2269 xpc_send_chctl_msgrequest_sn2(ch);
2270}
2271
2272static void
2273xpc_received_payload_sn2(struct xpc_channel *ch, void *payload)
2274{
2275 struct xpc_msg_sn2 *msg;
2276 s64 msg_number;
2277 s64 get;
2278
2279 msg = container_of(payload, struct xpc_msg_sn2, payload);
2280 msg_number = msg->number;
2281
2282 dev_dbg(xpc_chan, "msg=0x%p, msg_number=%ld, partid=%d, channel=%d\n",
2283 (void *)msg, msg_number, ch->partid, ch->number);
2284
2285 DBUG_ON((((u64)msg - (u64)ch->remote_msgqueue) / ch->entry_size) !=
2286 msg_number % ch->remote_nentries);
2287 DBUG_ON(msg->flags & XPC_M_SN2_DONE);
2288
2289 msg->flags |= XPC_M_SN2_DONE;
2290
2291 /*
2292 * The preceding store of msg->flags must occur before the following
2293 * load of local_GP->get.
2294 */
2295 mb();
2296
2297 /*
2298 * See if this message is next in line to be acknowledged as having
2299 * been delivered.
2300 */
2301 get = ch->sn.sn2.local_GP->get;
2302 if (get == msg_number)
2303 xpc_acknowledge_msgs_sn2(ch, get, msg->flags);
2304}
2305
2306int
2307xpc_init_sn2(void)
2308{
2309 int ret;
2310 size_t buf_size;
2311
2312 xpc_setup_partitions_sn = xpc_setup_partitions_sn_sn2;
2313 xpc_get_partition_rsvd_page_pa = xpc_get_partition_rsvd_page_pa_sn2;
2314 xpc_setup_rsvd_page_sn = xpc_setup_rsvd_page_sn_sn2;
2315 xpc_increment_heartbeat = xpc_increment_heartbeat_sn2;
2316 xpc_offline_heartbeat = xpc_offline_heartbeat_sn2;
2317 xpc_online_heartbeat = xpc_online_heartbeat_sn2;
2318 xpc_heartbeat_init = xpc_heartbeat_init_sn2;
2319 xpc_heartbeat_exit = xpc_heartbeat_exit_sn2;
2320 xpc_get_remote_heartbeat = xpc_get_remote_heartbeat_sn2;
2321
2322 xpc_request_partition_activation = xpc_request_partition_activation_sn2;
2323 xpc_request_partition_reactivation =
2324 xpc_request_partition_reactivation_sn2;
2325 xpc_request_partition_deactivation =
2326 xpc_request_partition_deactivation_sn2;
2327 xpc_cancel_partition_deactivation_request =
2328 xpc_cancel_partition_deactivation_request_sn2;
2329
2330 xpc_process_activate_IRQ_rcvd = xpc_process_activate_IRQ_rcvd_sn2;
2331 xpc_setup_ch_structures_sn = xpc_setup_ch_structures_sn_sn2;
2332 xpc_teardown_ch_structures_sn = xpc_teardown_ch_structures_sn_sn2;
2333 xpc_make_first_contact = xpc_make_first_contact_sn2;
2334
2335 xpc_get_chctl_all_flags = xpc_get_chctl_all_flags_sn2;
2336 xpc_send_chctl_closerequest = xpc_send_chctl_closerequest_sn2;
2337 xpc_send_chctl_closereply = xpc_send_chctl_closereply_sn2;
2338 xpc_send_chctl_openrequest = xpc_send_chctl_openrequest_sn2;
2339 xpc_send_chctl_openreply = xpc_send_chctl_openreply_sn2;
2340
2341 xpc_save_remote_msgqueue_pa = xpc_save_remote_msgqueue_pa_sn2;
2342
2343 xpc_setup_msg_structures = xpc_setup_msg_structures_sn2;
2344 xpc_teardown_msg_structures = xpc_teardown_msg_structures_sn2;
2345
2346 xpc_notify_senders_of_disconnect = xpc_notify_senders_of_disconnect_sn2;
2347 xpc_process_msg_chctl_flags = xpc_process_msg_chctl_flags_sn2;
2348 xpc_n_of_deliverable_payloads = xpc_n_of_deliverable_payloads_sn2;
2349 xpc_get_deliverable_payload = xpc_get_deliverable_payload_sn2;
2350
2351 xpc_indicate_partition_engaged = xpc_indicate_partition_engaged_sn2;
2352 xpc_indicate_partition_disengaged =
2353 xpc_indicate_partition_disengaged_sn2;
2354 xpc_partition_engaged = xpc_partition_engaged_sn2;
2355 xpc_any_partition_engaged = xpc_any_partition_engaged_sn2;
2356 xpc_assume_partition_disengaged = xpc_assume_partition_disengaged_sn2;
2357
2358 xpc_send_payload = xpc_send_payload_sn2;
2359 xpc_received_payload = xpc_received_payload_sn2;
2360
2361 if (offsetof(struct xpc_msg_sn2, payload) > XPC_MSG_HDR_MAX_SIZE) {
2362 dev_err(xpc_part, "header portion of struct xpc_msg_sn2 is "
2363 "larger than %d\n", XPC_MSG_HDR_MAX_SIZE);
2364 return -E2BIG;
2365 }
2366
2367 buf_size = max(XPC_RP_VARS_SIZE,
2368 XPC_RP_HEADER_SIZE + XP_NASID_MASK_BYTES_SN2);
2369 xpc_remote_copy_buffer_sn2 = xpc_kmalloc_cacheline_aligned(buf_size,
2370 GFP_KERNEL,
2371 &xpc_remote_copy_buffer_base_sn2);
2372 if (xpc_remote_copy_buffer_sn2 == NULL) {
2373 dev_err(xpc_part, "can't get memory for remote copy buffer\n");
2374 return -ENOMEM;
2375 }
2376
2377 /* open up protections for IPI and [potentially] amo operations */
2378 xpc_allow_IPI_ops_sn2();
2379 xpc_allow_amo_ops_shub_wars_1_1_sn2();
2380
2381 /*
2382 * This is safe to do before the xpc_hb_checker thread has started
2383 * because the handler releases a wait queue. If an interrupt is
2384 * received before the thread is waiting, it will not go to sleep,
2385 * but rather immediately process the interrupt.
2386 */
2387 ret = request_irq(SGI_XPC_ACTIVATE, xpc_handle_activate_IRQ_sn2, 0,
2388 "xpc hb", NULL);
2389 if (ret != 0) {
2390 dev_err(xpc_part, "can't register ACTIVATE IRQ handler, "
2391 "errno=%d\n", -ret);
2392 xpc_disallow_IPI_ops_sn2();
2393 kfree(xpc_remote_copy_buffer_base_sn2);
2394 }
2395 return ret;
2396}
2397
2398void
2399xpc_exit_sn2(void)
2400{
2401 free_irq(SGI_XPC_ACTIVATE, NULL);
2402 xpc_disallow_IPI_ops_sn2();
2403 kfree(xpc_remote_copy_buffer_base_sn2);
2404}
generated by cgit v1.2.2 (git 2.25.0) at 2024-12-27 19:52:57 -0500