aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/misc
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/misc')
-rw-r--r--drivers/misc/Kconfig4
-rw-r--r--drivers/misc/eeprom/at24.c67
-rw-r--r--drivers/misc/eeprom/at25.c58
-rw-r--r--drivers/misc/sgi-gru/Makefile2
-rw-r--r--drivers/misc/sgi-gru/gru_instructions.h22
-rw-r--r--drivers/misc/sgi-gru/grufault.c130
-rw-r--r--drivers/misc/sgi-gru/grufile.c36
-rw-r--r--drivers/misc/sgi-gru/gruhandles.c183
-rw-r--r--drivers/misc/sgi-gru/gruhandles.h178
-rw-r--r--drivers/misc/sgi-gru/grukservices.c131
-rw-r--r--drivers/misc/sgi-gru/grukservices.h33
-rw-r--r--drivers/misc/sgi-gru/grumain.c84
-rw-r--r--drivers/misc/sgi-gru/gruprocfs.c45
-rw-r--r--drivers/misc/sgi-gru/grutables.h41
-rw-r--r--drivers/misc/sgi-gru/grutlbpurge.c7
-rw-r--r--drivers/misc/sgi-xp/xpc.h33
-rw-r--r--drivers/misc/sgi-xp/xpc_channel.c8
-rw-r--r--drivers/misc/sgi-xp/xpc_main.c6
-rw-r--r--drivers/misc/sgi-xp/xpc_sn2.c20
-rw-r--r--drivers/misc/sgi-xp/xpc_uv.c229
20 files changed, 899 insertions, 418 deletions
diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig
index 5f3bff434621..0b92b2f6ea68 100644
--- a/drivers/misc/Kconfig
+++ b/drivers/misc/Kconfig
@@ -165,7 +165,7 @@ config SGI_XP
165 depends on (IA64_GENERIC || IA64_SGI_SN2 || IA64_SGI_UV || X86_UV) && SMP 165 depends on (IA64_GENERIC || IA64_SGI_SN2 || IA64_SGI_UV || X86_UV) && SMP
166 select IA64_UNCACHED_ALLOCATOR if IA64_GENERIC || IA64_SGI_SN2 166 select IA64_UNCACHED_ALLOCATOR if IA64_GENERIC || IA64_SGI_SN2
167 select GENERIC_ALLOCATOR if IA64_GENERIC || IA64_SGI_SN2 167 select GENERIC_ALLOCATOR if IA64_GENERIC || IA64_SGI_SN2
168 select SGI_GRU if (IA64_GENERIC || IA64_SGI_UV || X86_64) && SMP 168 select SGI_GRU if X86_64 && SMP
169 ---help--- 169 ---help---
170 An SGI machine can be divided into multiple Single System 170 An SGI machine can be divided into multiple Single System
171 Images which act independently of each other and have 171 Images which act independently of each other and have
@@ -189,7 +189,7 @@ config HP_ILO
189 189
190config SGI_GRU 190config SGI_GRU
191 tristate "SGI GRU driver" 191 tristate "SGI GRU driver"
192 depends on (X86_UV || IA64_SGI_UV || IA64_GENERIC) && SMP 192 depends on X86_UV && SMP
193 default n 193 default n
194 select MMU_NOTIFIER 194 select MMU_NOTIFIER
195 ---help--- 195 ---help---
diff --git a/drivers/misc/eeprom/at24.c b/drivers/misc/eeprom/at24.c
index d4775528abc6..d184dfab9631 100644
--- a/drivers/misc/eeprom/at24.c
+++ b/drivers/misc/eeprom/at24.c
@@ -53,6 +53,7 @@
53 53
54struct at24_data { 54struct at24_data {
55 struct at24_platform_data chip; 55 struct at24_platform_data chip;
56 struct memory_accessor macc;
56 bool use_smbus; 57 bool use_smbus;
57 58
58 /* 59 /*
@@ -225,14 +226,11 @@ static ssize_t at24_eeprom_read(struct at24_data *at24, char *buf,
225 return status; 226 return status;
226} 227}
227 228
228static ssize_t at24_bin_read(struct kobject *kobj, struct bin_attribute *attr, 229static ssize_t at24_read(struct at24_data *at24,
229 char *buf, loff_t off, size_t count) 230 char *buf, loff_t off, size_t count)
230{ 231{
231 struct at24_data *at24;
232 ssize_t retval = 0; 232 ssize_t retval = 0;
233 233
234 at24 = dev_get_drvdata(container_of(kobj, struct device, kobj));
235
236 if (unlikely(!count)) 234 if (unlikely(!count))
237 return count; 235 return count;
238 236
@@ -262,12 +260,14 @@ static ssize_t at24_bin_read(struct kobject *kobj, struct bin_attribute *attr,
262 return retval; 260 return retval;
263} 261}
264 262
263static ssize_t at24_bin_read(struct kobject *kobj, struct bin_attribute *attr,
264 char *buf, loff_t off, size_t count)
265{
266 struct at24_data *at24;
265 267
266/* 268 at24 = dev_get_drvdata(container_of(kobj, struct device, kobj));
267 * REVISIT: export at24_bin{read,write}() to let other kernel code use 269 return at24_read(at24, buf, off, count);
268 * eeprom data. For example, it might hold a board's Ethernet address, or 270}
269 * board-specific calibration data generated on the manufacturing floor.
270 */
271 271
272 272
273/* 273/*
@@ -347,14 +347,11 @@ static ssize_t at24_eeprom_write(struct at24_data *at24, char *buf,
347 return -ETIMEDOUT; 347 return -ETIMEDOUT;
348} 348}
349 349
350static ssize_t at24_bin_write(struct kobject *kobj, struct bin_attribute *attr, 350static ssize_t at24_write(struct at24_data *at24,
351 char *buf, loff_t off, size_t count) 351 char *buf, loff_t off, size_t count)
352{ 352{
353 struct at24_data *at24;
354 ssize_t retval = 0; 353 ssize_t retval = 0;
355 354
356 at24 = dev_get_drvdata(container_of(kobj, struct device, kobj));
357
358 if (unlikely(!count)) 355 if (unlikely(!count))
359 return count; 356 return count;
360 357
@@ -384,6 +381,39 @@ static ssize_t at24_bin_write(struct kobject *kobj, struct bin_attribute *attr,
384 return retval; 381 return retval;
385} 382}
386 383
384static ssize_t at24_bin_write(struct kobject *kobj, struct bin_attribute *attr,
385 char *buf, loff_t off, size_t count)
386{
387 struct at24_data *at24;
388
389 at24 = dev_get_drvdata(container_of(kobj, struct device, kobj));
390 return at24_write(at24, buf, off, count);
391}
392
393/*-------------------------------------------------------------------------*/
394
395/*
396 * This lets other kernel code access the eeprom data. For example, it
397 * might hold a board's Ethernet address, or board-specific calibration
398 * data generated on the manufacturing floor.
399 */
400
401static ssize_t at24_macc_read(struct memory_accessor *macc, char *buf,
402 off_t offset, size_t count)
403{
404 struct at24_data *at24 = container_of(macc, struct at24_data, macc);
405
406 return at24_read(at24, buf, offset, count);
407}
408
409static ssize_t at24_macc_write(struct memory_accessor *macc, char *buf,
410 off_t offset, size_t count)
411{
412 struct at24_data *at24 = container_of(macc, struct at24_data, macc);
413
414 return at24_write(at24, buf, offset, count);
415}
416
387/*-------------------------------------------------------------------------*/ 417/*-------------------------------------------------------------------------*/
388 418
389static int at24_probe(struct i2c_client *client, const struct i2c_device_id *id) 419static int at24_probe(struct i2c_client *client, const struct i2c_device_id *id)
@@ -413,6 +443,9 @@ static int at24_probe(struct i2c_client *client, const struct i2c_device_id *id)
413 * is recommended anyhow. 443 * is recommended anyhow.
414 */ 444 */
415 chip.page_size = 1; 445 chip.page_size = 1;
446
447 chip.setup = NULL;
448 chip.context = NULL;
416 } 449 }
417 450
418 if (!is_power_of_2(chip.byte_len)) 451 if (!is_power_of_2(chip.byte_len))
@@ -463,6 +496,8 @@ static int at24_probe(struct i2c_client *client, const struct i2c_device_id *id)
463 at24->bin.read = at24_bin_read; 496 at24->bin.read = at24_bin_read;
464 at24->bin.size = chip.byte_len; 497 at24->bin.size = chip.byte_len;
465 498
499 at24->macc.read = at24_macc_read;
500
466 writable = !(chip.flags & AT24_FLAG_READONLY); 501 writable = !(chip.flags & AT24_FLAG_READONLY);
467 if (writable) { 502 if (writable) {
468 if (!use_smbus || i2c_check_functionality(client->adapter, 503 if (!use_smbus || i2c_check_functionality(client->adapter,
@@ -470,6 +505,8 @@ static int at24_probe(struct i2c_client *client, const struct i2c_device_id *id)
470 505
471 unsigned write_max = chip.page_size; 506 unsigned write_max = chip.page_size;
472 507
508 at24->macc.write = at24_macc_write;
509
473 at24->bin.write = at24_bin_write; 510 at24->bin.write = at24_bin_write;
474 at24->bin.attr.mode |= S_IWUSR; 511 at24->bin.attr.mode |= S_IWUSR;
475 512
@@ -520,6 +557,10 @@ static int at24_probe(struct i2c_client *client, const struct i2c_device_id *id)
520 at24->write_max, 557 at24->write_max,
521 use_smbus ? ", use_smbus" : ""); 558 use_smbus ? ", use_smbus" : "");
522 559
560 /* export data to kernel code */
561 if (chip.setup)
562 chip.setup(&at24->macc, chip.context);
563
523 return 0; 564 return 0;
524 565
525err_clients: 566err_clients:
diff --git a/drivers/misc/eeprom/at25.c b/drivers/misc/eeprom/at25.c
index 290dbe99647a..6bc0dac5c1e8 100644
--- a/drivers/misc/eeprom/at25.c
+++ b/drivers/misc/eeprom/at25.c
@@ -30,6 +30,7 @@
30 30
31struct at25_data { 31struct at25_data {
32 struct spi_device *spi; 32 struct spi_device *spi;
33 struct memory_accessor mem;
33 struct mutex lock; 34 struct mutex lock;
34 struct spi_eeprom chip; 35 struct spi_eeprom chip;
35 struct bin_attribute bin; 36 struct bin_attribute bin;
@@ -75,6 +76,13 @@ at25_ee_read(
75 struct spi_transfer t[2]; 76 struct spi_transfer t[2];
76 struct spi_message m; 77 struct spi_message m;
77 78
79 if (unlikely(offset >= at25->bin.size))
80 return 0;
81 if ((offset + count) > at25->bin.size)
82 count = at25->bin.size - offset;
83 if (unlikely(!count))
84 return count;
85
78 cp = command; 86 cp = command;
79 *cp++ = AT25_READ; 87 *cp++ = AT25_READ;
80 88
@@ -127,13 +135,6 @@ at25_bin_read(struct kobject *kobj, struct bin_attribute *bin_attr,
127 dev = container_of(kobj, struct device, kobj); 135 dev = container_of(kobj, struct device, kobj);
128 at25 = dev_get_drvdata(dev); 136 at25 = dev_get_drvdata(dev);
129 137
130 if (unlikely(off >= at25->bin.size))
131 return 0;
132 if ((off + count) > at25->bin.size)
133 count = at25->bin.size - off;
134 if (unlikely(!count))
135 return count;
136
137 return at25_ee_read(at25, buf, off, count); 138 return at25_ee_read(at25, buf, off, count);
138} 139}
139 140
@@ -146,6 +147,13 @@ at25_ee_write(struct at25_data *at25, char *buf, loff_t off, size_t count)
146 unsigned buf_size; 147 unsigned buf_size;
147 u8 *bounce; 148 u8 *bounce;
148 149
150 if (unlikely(off >= at25->bin.size))
151 return -EFBIG;
152 if ((off + count) > at25->bin.size)
153 count = at25->bin.size - off;
154 if (unlikely(!count))
155 return count;
156
149 /* Temp buffer starts with command and address */ 157 /* Temp buffer starts with command and address */
150 buf_size = at25->chip.page_size; 158 buf_size = at25->chip.page_size;
151 if (buf_size > io_limit) 159 if (buf_size > io_limit)
@@ -253,18 +261,31 @@ at25_bin_write(struct kobject *kobj, struct bin_attribute *bin_attr,
253 dev = container_of(kobj, struct device, kobj); 261 dev = container_of(kobj, struct device, kobj);
254 at25 = dev_get_drvdata(dev); 262 at25 = dev_get_drvdata(dev);
255 263
256 if (unlikely(off >= at25->bin.size))
257 return -EFBIG;
258 if ((off + count) > at25->bin.size)
259 count = at25->bin.size - off;
260 if (unlikely(!count))
261 return count;
262
263 return at25_ee_write(at25, buf, off, count); 264 return at25_ee_write(at25, buf, off, count);
264} 265}
265 266
266/*-------------------------------------------------------------------------*/ 267/*-------------------------------------------------------------------------*/
267 268
269/* Let in-kernel code access the eeprom data. */
270
271static ssize_t at25_mem_read(struct memory_accessor *mem, char *buf,
272 off_t offset, size_t count)
273{
274 struct at25_data *at25 = container_of(mem, struct at25_data, mem);
275
276 return at25_ee_read(at25, buf, offset, count);
277}
278
279static ssize_t at25_mem_write(struct memory_accessor *mem, char *buf,
280 off_t offset, size_t count)
281{
282 struct at25_data *at25 = container_of(mem, struct at25_data, mem);
283
284 return at25_ee_write(at25, buf, offset, count);
285}
286
287/*-------------------------------------------------------------------------*/
288
268static int at25_probe(struct spi_device *spi) 289static int at25_probe(struct spi_device *spi)
269{ 290{
270 struct at25_data *at25 = NULL; 291 struct at25_data *at25 = NULL;
@@ -317,6 +338,10 @@ static int at25_probe(struct spi_device *spi)
317 at25->addrlen = addrlen; 338 at25->addrlen = addrlen;
318 339
319 /* Export the EEPROM bytes through sysfs, since that's convenient. 340 /* Export the EEPROM bytes through sysfs, since that's convenient.
341 * And maybe to other kernel code; it might hold a board's Ethernet
342 * address, or board-specific calibration data generated on the
343 * manufacturing floor.
344 *
320 * Default to root-only access to the data; EEPROMs often hold data 345 * Default to root-only access to the data; EEPROMs often hold data
321 * that's sensitive for read and/or write, like ethernet addresses, 346 * that's sensitive for read and/or write, like ethernet addresses,
322 * security codes, board-specific manufacturing calibrations, etc. 347 * security codes, board-specific manufacturing calibrations, etc.
@@ -324,17 +349,22 @@ static int at25_probe(struct spi_device *spi)
324 at25->bin.attr.name = "eeprom"; 349 at25->bin.attr.name = "eeprom";
325 at25->bin.attr.mode = S_IRUSR; 350 at25->bin.attr.mode = S_IRUSR;
326 at25->bin.read = at25_bin_read; 351 at25->bin.read = at25_bin_read;
352 at25->mem.read = at25_mem_read;
327 353
328 at25->bin.size = at25->chip.byte_len; 354 at25->bin.size = at25->chip.byte_len;
329 if (!(chip->flags & EE_READONLY)) { 355 if (!(chip->flags & EE_READONLY)) {
330 at25->bin.write = at25_bin_write; 356 at25->bin.write = at25_bin_write;
331 at25->bin.attr.mode |= S_IWUSR; 357 at25->bin.attr.mode |= S_IWUSR;
358 at25->mem.write = at25_mem_write;
332 } 359 }
333 360
334 err = sysfs_create_bin_file(&spi->dev.kobj, &at25->bin); 361 err = sysfs_create_bin_file(&spi->dev.kobj, &at25->bin);
335 if (err) 362 if (err)
336 goto fail; 363 goto fail;
337 364
365 if (chip->setup)
366 chip->setup(&at25->mem, chip->context);
367
338 dev_info(&spi->dev, "%Zd %s %s eeprom%s, pagesize %u\n", 368 dev_info(&spi->dev, "%Zd %s %s eeprom%s, pagesize %u\n",
339 (at25->bin.size < 1024) 369 (at25->bin.size < 1024)
340 ? at25->bin.size 370 ? at25->bin.size
diff --git a/drivers/misc/sgi-gru/Makefile b/drivers/misc/sgi-gru/Makefile
index 9e9170b3599a..bcd8136d2f98 100644
--- a/drivers/misc/sgi-gru/Makefile
+++ b/drivers/misc/sgi-gru/Makefile
@@ -3,5 +3,5 @@ ifdef CONFIG_SGI_GRU_DEBUG
3endif 3endif
4 4
5obj-$(CONFIG_SGI_GRU) := gru.o 5obj-$(CONFIG_SGI_GRU) := gru.o
6gru-y := grufile.o grumain.o grufault.o grutlbpurge.o gruprocfs.o grukservices.o 6gru-y := grufile.o grumain.o grufault.o grutlbpurge.o gruprocfs.o grukservices.o gruhandles.o
7 7
diff --git a/drivers/misc/sgi-gru/gru_instructions.h b/drivers/misc/sgi-gru/gru_instructions.h
index 48762e7b98be..3fde33c1e8f3 100644
--- a/drivers/misc/sgi-gru/gru_instructions.h
+++ b/drivers/misc/sgi-gru/gru_instructions.h
@@ -19,8 +19,11 @@
19#ifndef __GRU_INSTRUCTIONS_H__ 19#ifndef __GRU_INSTRUCTIONS_H__
20#define __GRU_INSTRUCTIONS_H__ 20#define __GRU_INSTRUCTIONS_H__
21 21
22#define gru_flush_cache_hook(p) 22extern int gru_check_status_proc(void *cb);
23#define gru_emulator_wait_hook(p, w) 23extern int gru_wait_proc(void *cb);
24extern void gru_wait_abort_proc(void *cb);
25
26
24 27
25/* 28/*
26 * Architecture dependent functions 29 * Architecture dependent functions
@@ -29,16 +32,16 @@
29#if defined(CONFIG_IA64) 32#if defined(CONFIG_IA64)
30#include <linux/compiler.h> 33#include <linux/compiler.h>
31#include <asm/intrinsics.h> 34#include <asm/intrinsics.h>
32#define __flush_cache(p) ia64_fc(p) 35#define __flush_cache(p) ia64_fc((unsigned long)p)
33/* Use volatile on IA64 to ensure ordering via st4.rel */ 36/* Use volatile on IA64 to ensure ordering via st4.rel */
34#define gru_ordered_store_int(p,v) \ 37#define gru_ordered_store_int(p, v) \
35 do { \ 38 do { \
36 barrier(); \ 39 barrier(); \
37 *((volatile int *)(p)) = v; /* force st.rel */ \ 40 *((volatile int *)(p)) = v; /* force st.rel */ \
38 } while (0) 41 } while (0)
39#elif defined(CONFIG_X86_64) 42#elif defined(CONFIG_X86_64)
40#define __flush_cache(p) clflush(p) 43#define __flush_cache(p) clflush(p)
41#define gru_ordered_store_int(p,v) \ 44#define gru_ordered_store_int(p, v) \
42 do { \ 45 do { \
43 barrier(); \ 46 barrier(); \
44 *(int *)p = v; \ 47 *(int *)p = v; \
@@ -558,20 +561,19 @@ extern int gru_get_cb_exception_detail(void *cb,
558 561
559#define GRU_EXC_STR_SIZE 256 562#define GRU_EXC_STR_SIZE 256
560 563
561extern int gru_check_status_proc(void *cb);
562extern int gru_wait_proc(void *cb);
563extern void gru_wait_abort_proc(void *cb);
564 564
565/* 565/*
566 * Control block definition for checking status 566 * Control block definition for checking status
567 */ 567 */
568struct gru_control_block_status { 568struct gru_control_block_status {
569 unsigned int icmd :1; 569 unsigned int icmd :1;
570 unsigned int unused1 :31; 570 unsigned int ima :3;
571 unsigned int reserved0 :4;
572 unsigned int unused1 :24;
571 unsigned int unused2 :24; 573 unsigned int unused2 :24;
572 unsigned int istatus :2; 574 unsigned int istatus :2;
573 unsigned int isubstatus :4; 575 unsigned int isubstatus :4;
574 unsigned int inused3 :2; 576 unsigned int unused3 :2;
575}; 577};
576 578
577/* Get CB status */ 579/* Get CB status */
diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c
index 3ee698ad8599..ab118558552e 100644
--- a/drivers/misc/sgi-gru/grufault.c
+++ b/drivers/misc/sgi-gru/grufault.c
@@ -32,6 +32,7 @@
32#include <linux/device.h> 32#include <linux/device.h>
33#include <linux/io.h> 33#include <linux/io.h>
34#include <linux/uaccess.h> 34#include <linux/uaccess.h>
35#include <linux/security.h>
35#include <asm/pgtable.h> 36#include <asm/pgtable.h>
36#include "gru.h" 37#include "gru.h"
37#include "grutables.h" 38#include "grutables.h"
@@ -266,6 +267,44 @@ err:
266 return 1; 267 return 1;
267} 268}
268 269
270static int gru_vtop(struct gru_thread_state *gts, unsigned long vaddr,
271 int write, int atomic, unsigned long *gpa, int *pageshift)
272{
273 struct mm_struct *mm = gts->ts_mm;
274 struct vm_area_struct *vma;
275 unsigned long paddr;
276 int ret, ps;
277
278 vma = find_vma(mm, vaddr);
279 if (!vma)
280 goto inval;
281
282 /*
283 * Atomic lookup is faster & usually works even if called in non-atomic
284 * context.
285 */
286 rmb(); /* Must/check ms_range_active before loading PTEs */
287 ret = atomic_pte_lookup(vma, vaddr, write, &paddr, &ps);
288 if (ret) {
289 if (atomic)
290 goto upm;
291 if (non_atomic_pte_lookup(vma, vaddr, write, &paddr, &ps))
292 goto inval;
293 }
294 if (is_gru_paddr(paddr))
295 goto inval;
296 paddr = paddr & ~((1UL << ps) - 1);
297 *gpa = uv_soc_phys_ram_to_gpa(paddr);
298 *pageshift = ps;
299 return 0;
300
301inval:
302 return -1;
303upm:
304 return -2;
305}
306
307
269/* 308/*
270 * Drop a TLB entry into the GRU. The fault is described by info in an TFH. 309 * Drop a TLB entry into the GRU. The fault is described by info in an TFH.
271 * Input: 310 * Input:
@@ -280,10 +319,8 @@ static int gru_try_dropin(struct gru_thread_state *gts,
280 struct gru_tlb_fault_handle *tfh, 319 struct gru_tlb_fault_handle *tfh,
281 unsigned long __user *cb) 320 unsigned long __user *cb)
282{ 321{
283 struct mm_struct *mm = gts->ts_mm; 322 int pageshift = 0, asid, write, ret, atomic = !cb;
284 struct vm_area_struct *vma; 323 unsigned long gpa = 0, vaddr = 0;
285 int pageshift, asid, write, ret;
286 unsigned long paddr, gpa, vaddr;
287 324
288 /* 325 /*
289 * NOTE: The GRU contains magic hardware that eliminates races between 326 * NOTE: The GRU contains magic hardware that eliminates races between
@@ -317,28 +354,19 @@ static int gru_try_dropin(struct gru_thread_state *gts,
317 if (atomic_read(&gts->ts_gms->ms_range_active)) 354 if (atomic_read(&gts->ts_gms->ms_range_active))
318 goto failactive; 355 goto failactive;
319 356
320 vma = find_vma(mm, vaddr); 357 ret = gru_vtop(gts, vaddr, write, atomic, &gpa, &pageshift);
321 if (!vma) 358 if (ret == -1)
322 goto failinval; 359 goto failinval;
360 if (ret == -2)
361 goto failupm;
323 362
324 /* 363 if (!(gts->ts_sizeavail & GRU_SIZEAVAIL(pageshift))) {
325 * Atomic lookup is faster & usually works even if called in non-atomic 364 gts->ts_sizeavail |= GRU_SIZEAVAIL(pageshift);
326 * context. 365 if (atomic || !gru_update_cch(gts, 0)) {
327 */ 366 gts->ts_force_cch_reload = 1;
328 rmb(); /* Must/check ms_range_active before loading PTEs */
329 ret = atomic_pte_lookup(vma, vaddr, write, &paddr, &pageshift);
330 if (ret) {
331 if (!cb)
332 goto failupm; 367 goto failupm;
333 if (non_atomic_pte_lookup(vma, vaddr, write, &paddr, 368 }
334 &pageshift))
335 goto failinval;
336 } 369 }
337 if (is_gru_paddr(paddr))
338 goto failinval;
339
340 paddr = paddr & ~((1UL << pageshift) - 1);
341 gpa = uv_soc_phys_ram_to_gpa(paddr);
342 gru_cb_set_istatus_active(cb); 370 gru_cb_set_istatus_active(cb);
343 tfh_write_restart(tfh, gpa, GAA_RAM, vaddr, asid, write, 371 tfh_write_restart(tfh, gpa, GAA_RAM, vaddr, asid, write,
344 GRU_PAGESIZE(pageshift)); 372 GRU_PAGESIZE(pageshift));
@@ -368,6 +396,7 @@ failupm:
368 396
369failfmm: 397failfmm:
370 /* FMM state on UPM call */ 398 /* FMM state on UPM call */
399 gru_flush_cache(tfh);
371 STAT(tlb_dropin_fail_fmm); 400 STAT(tlb_dropin_fail_fmm);
372 gru_dbg(grudev, "FAILED fmm tfh: 0x%p, state %d\n", tfh, tfh->state); 401 gru_dbg(grudev, "FAILED fmm tfh: 0x%p, state %d\n", tfh, tfh->state);
373 return 0; 402 return 0;
@@ -448,6 +477,7 @@ irqreturn_t gru_intr(int irq, void *dev_id)
448 up_read(&gts->ts_mm->mmap_sem); 477 up_read(&gts->ts_mm->mmap_sem);
449 } else { 478 } else {
450 tfh_user_polling_mode(tfh); 479 tfh_user_polling_mode(tfh);
480 STAT(intr_mm_lock_failed);
451 } 481 }
452 } 482 }
453 return IRQ_HANDLED; 483 return IRQ_HANDLED;
@@ -497,10 +527,8 @@ int gru_handle_user_call_os(unsigned long cb)
497 if (!gts) 527 if (!gts)
498 return -EINVAL; 528 return -EINVAL;
499 529
500 if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE) { 530 if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE)
501 ret = -EINVAL;
502 goto exit; 531 goto exit;
503 }
504 532
505 /* 533 /*
506 * If force_unload is set, the UPM TLB fault is phony. The task 534 * If force_unload is set, the UPM TLB fault is phony. The task
@@ -508,6 +536,20 @@ int gru_handle_user_call_os(unsigned long cb)
508 * unload the context. The task will page fault and assign a new 536 * unload the context. The task will page fault and assign a new
509 * context. 537 * context.
510 */ 538 */
539 if (gts->ts_tgid_owner == current->tgid && gts->ts_blade >= 0 &&
540 gts->ts_blade != uv_numa_blade_id()) {
541 STAT(call_os_offnode_reference);
542 gts->ts_force_unload = 1;
543 }
544
545 /*
546 * CCH may contain stale data if ts_force_cch_reload is set.
547 */
548 if (gts->ts_gru && gts->ts_force_cch_reload) {
549 gru_update_cch(gts, 0);
550 gts->ts_force_cch_reload = 0;
551 }
552
511 ret = -EAGAIN; 553 ret = -EAGAIN;
512 cbrnum = thread_cbr_number(gts, ucbnum); 554 cbrnum = thread_cbr_number(gts, ucbnum);
513 if (gts->ts_force_unload) { 555 if (gts->ts_force_unload) {
@@ -541,11 +583,13 @@ int gru_get_exception_detail(unsigned long arg)
541 if (!gts) 583 if (!gts)
542 return -EINVAL; 584 return -EINVAL;
543 585
544 if (gts->ts_gru) { 586 ucbnum = get_cb_number((void *)excdet.cb);
545 ucbnum = get_cb_number((void *)excdet.cb); 587 if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE) {
588 ret = -EINVAL;
589 } else if (gts->ts_gru) {
546 cbrnum = thread_cbr_number(gts, ucbnum); 590 cbrnum = thread_cbr_number(gts, ucbnum);
547 cbe = get_cbe_by_index(gts->ts_gru, cbrnum); 591 cbe = get_cbe_by_index(gts->ts_gru, cbrnum);
548 prefetchw(cbe); /* Harmless on hardware, required for emulator */ 592 prefetchw(cbe);/* Harmless on hardware, required for emulator */
549 excdet.opc = cbe->opccpy; 593 excdet.opc = cbe->opccpy;
550 excdet.exopc = cbe->exopccpy; 594 excdet.exopc = cbe->exopccpy;
551 excdet.ecause = cbe->ecause; 595 excdet.ecause = cbe->ecause;
@@ -567,6 +611,31 @@ int gru_get_exception_detail(unsigned long arg)
567/* 611/*
568 * User request to unload a context. Content is saved for possible reload. 612 * User request to unload a context. Content is saved for possible reload.
569 */ 613 */
614static int gru_unload_all_contexts(void)
615{
616 struct gru_thread_state *gts;
617 struct gru_state *gru;
618 int gid, ctxnum;
619
620 if (!capable(CAP_SYS_ADMIN))
621 return -EPERM;
622 foreach_gid(gid) {
623 gru = GID_TO_GRU(gid);
624 spin_lock(&gru->gs_lock);
625 for (ctxnum = 0; ctxnum < GRU_NUM_CCH; ctxnum++) {
626 gts = gru->gs_gts[ctxnum];
627 if (gts && mutex_trylock(&gts->ts_ctxlock)) {
628 spin_unlock(&gru->gs_lock);
629 gru_unload_context(gts, 1);
630 gru_unlock_gts(gts);
631 spin_lock(&gru->gs_lock);
632 }
633 }
634 spin_unlock(&gru->gs_lock);
635 }
636 return 0;
637}
638
570int gru_user_unload_context(unsigned long arg) 639int gru_user_unload_context(unsigned long arg)
571{ 640{
572 struct gru_thread_state *gts; 641 struct gru_thread_state *gts;
@@ -578,6 +647,9 @@ int gru_user_unload_context(unsigned long arg)
578 647
579 gru_dbg(grudev, "gseg 0x%lx\n", req.gseg); 648 gru_dbg(grudev, "gseg 0x%lx\n", req.gseg);
580 649
650 if (!req.gseg)
651 return gru_unload_all_contexts();
652
581 gts = gru_find_lock_gts(req.gseg); 653 gts = gru_find_lock_gts(req.gseg);
582 if (!gts) 654 if (!gts)
583 return -EINVAL; 655 return -EINVAL;
@@ -609,7 +681,7 @@ int gru_user_flush_tlb(unsigned long arg)
609 if (!gts) 681 if (!gts)
610 return -EINVAL; 682 return -EINVAL;
611 683
612 gru_flush_tlb_range(gts->ts_gms, req.vaddr, req.vaddr + req.len); 684 gru_flush_tlb_range(gts->ts_gms, req.vaddr, req.len);
613 gru_unlock_gts(gts); 685 gru_unlock_gts(gts);
614 686
615 return 0; 687 return 0;
diff --git a/drivers/misc/sgi-gru/grufile.c b/drivers/misc/sgi-gru/grufile.c
index c67e4e8bd62c..3e6e42d2f01b 100644
--- a/drivers/misc/sgi-gru/grufile.c
+++ b/drivers/misc/sgi-gru/grufile.c
@@ -45,7 +45,9 @@
45#include <asm/uv/uv_mmrs.h> 45#include <asm/uv/uv_mmrs.h>
46 46
47struct gru_blade_state *gru_base[GRU_MAX_BLADES] __read_mostly; 47struct gru_blade_state *gru_base[GRU_MAX_BLADES] __read_mostly;
48unsigned long gru_start_paddr, gru_end_paddr __read_mostly; 48unsigned long gru_start_paddr __read_mostly;
49unsigned long gru_end_paddr __read_mostly;
50unsigned int gru_max_gids __read_mostly;
49struct gru_stats_s gru_stats; 51struct gru_stats_s gru_stats;
50 52
51/* Guaranteed user available resources on each node */ 53/* Guaranteed user available resources on each node */
@@ -101,7 +103,7 @@ static int gru_file_mmap(struct file *file, struct vm_area_struct *vma)
101 return -EPERM; 103 return -EPERM;
102 104
103 if (vma->vm_start & (GRU_GSEG_PAGESIZE - 1) || 105 if (vma->vm_start & (GRU_GSEG_PAGESIZE - 1) ||
104 vma->vm_end & (GRU_GSEG_PAGESIZE - 1)) 106 vma->vm_end & (GRU_GSEG_PAGESIZE - 1))
105 return -EINVAL; 107 return -EINVAL;
106 108
107 vma->vm_flags |= 109 vma->vm_flags |=
@@ -273,8 +275,11 @@ static void gru_init_chiplet(struct gru_state *gru, unsigned long paddr,
273 gru->gs_blade_id = bid; 275 gru->gs_blade_id = bid;
274 gru->gs_cbr_map = (GRU_CBR_AU == 64) ? ~0 : (1UL << GRU_CBR_AU) - 1; 276 gru->gs_cbr_map = (GRU_CBR_AU == 64) ? ~0 : (1UL << GRU_CBR_AU) - 1;
275 gru->gs_dsr_map = (1UL << GRU_DSR_AU) - 1; 277 gru->gs_dsr_map = (1UL << GRU_DSR_AU) - 1;
278 gru->gs_asid_limit = MAX_ASID;
276 gru_tgh_flush_init(gru); 279 gru_tgh_flush_init(gru);
277 gru_dbg(grudev, "bid %d, nid %d, gru %x, vaddr %p (0x%lx)\n", 280 if (gru->gs_gid >= gru_max_gids)
281 gru_max_gids = gru->gs_gid + 1;
282 gru_dbg(grudev, "bid %d, nid %d, gid %d, vaddr %p (0x%lx)\n",
278 bid, nid, gru->gs_gid, gru->gs_gru_base_vaddr, 283 bid, nid, gru->gs_gid, gru->gs_gru_base_vaddr,
279 gru->gs_gru_base_paddr); 284 gru->gs_gru_base_paddr);
280 gru_kservices_init(gru); 285 gru_kservices_init(gru);
@@ -295,7 +300,7 @@ static int gru_init_tables(unsigned long gru_base_paddr, void *gru_base_vaddr)
295 for_each_online_node(nid) { 300 for_each_online_node(nid) {
296 bid = uv_node_to_blade_id(nid); 301 bid = uv_node_to_blade_id(nid);
297 pnode = uv_node_to_pnode(nid); 302 pnode = uv_node_to_pnode(nid);
298 if (gru_base[bid]) 303 if (bid < 0 || gru_base[bid])
299 continue; 304 continue;
300 page = alloc_pages_node(nid, GFP_KERNEL, order); 305 page = alloc_pages_node(nid, GFP_KERNEL, order);
301 if (!page) 306 if (!page)
@@ -308,11 +313,11 @@ static int gru_init_tables(unsigned long gru_base_paddr, void *gru_base_vaddr)
308 dsrbytes = 0; 313 dsrbytes = 0;
309 cbrs = 0; 314 cbrs = 0;
310 for (gru = gru_base[bid]->bs_grus, chip = 0; 315 for (gru = gru_base[bid]->bs_grus, chip = 0;
311 chip < GRU_CHIPLETS_PER_BLADE; 316 chip < GRU_CHIPLETS_PER_BLADE;
312 chip++, gru++) { 317 chip++, gru++) {
313 paddr = gru_chiplet_paddr(gru_base_paddr, pnode, chip); 318 paddr = gru_chiplet_paddr(gru_base_paddr, pnode, chip);
314 vaddr = gru_chiplet_vaddr(gru_base_vaddr, pnode, chip); 319 vaddr = gru_chiplet_vaddr(gru_base_vaddr, pnode, chip);
315 gru_init_chiplet(gru, paddr, vaddr, bid, nid, chip); 320 gru_init_chiplet(gru, paddr, vaddr, nid, bid, chip);
316 n = hweight64(gru->gs_cbr_map) * GRU_CBR_AU_SIZE; 321 n = hweight64(gru->gs_cbr_map) * GRU_CBR_AU_SIZE;
317 cbrs = max(cbrs, n); 322 cbrs = max(cbrs, n);
318 n = hweight64(gru->gs_dsr_map) * GRU_DSR_AU_BYTES; 323 n = hweight64(gru->gs_dsr_map) * GRU_DSR_AU_BYTES;
@@ -370,26 +375,26 @@ static int __init gru_init(void)
370 void *gru_start_vaddr; 375 void *gru_start_vaddr;
371 376
372 if (!is_uv_system()) 377 if (!is_uv_system())
373 return 0; 378 return -ENODEV;
374 379
375#if defined CONFIG_IA64 380#if defined CONFIG_IA64
376 gru_start_paddr = 0xd000000000UL; /* ZZZZZZZZZZZZZZZZZZZ fixme */ 381 gru_start_paddr = 0xd000000000UL; /* ZZZZZZZZZZZZZZZZZZZ fixme */
377#else 382#else
378 gru_start_paddr = uv_read_local_mmr(UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR) & 383 gru_start_paddr = uv_read_local_mmr(UVH_RH_GAM_GRU_OVERLAY_CONFIG_MMR) &
379 0x7fffffffffffUL; 384 0x7fffffffffffUL;
380
381#endif 385#endif
382 gru_start_vaddr = __va(gru_start_paddr); 386 gru_start_vaddr = __va(gru_start_paddr);
383 gru_end_paddr = gru_start_paddr + MAX_NUMNODES * GRU_SIZE; 387 gru_end_paddr = gru_start_paddr + GRU_MAX_BLADES * GRU_SIZE;
384 printk(KERN_INFO "GRU space: 0x%lx - 0x%lx\n", 388 printk(KERN_INFO "GRU space: 0x%lx - 0x%lx\n",
385 gru_start_paddr, gru_end_paddr); 389 gru_start_paddr, gru_end_paddr);
386 irq = get_base_irq(); 390 irq = get_base_irq();
387 for (chip = 0; chip < GRU_CHIPLETS_PER_BLADE; chip++) { 391 for (chip = 0; chip < GRU_CHIPLETS_PER_BLADE; chip++) {
388 ret = request_irq(irq + chip, gru_intr, 0, id, NULL); 392 ret = request_irq(irq + chip, gru_intr, 0, id, NULL);
389 /* TODO: fix irq handling on x86. For now ignore failures because 393 /* TODO: fix irq handling on x86. For now ignore failure because
390 * interrupts are not required & not yet fully supported */ 394 * interrupts are not required & not yet fully supported */
391 if (ret) { 395 if (ret) {
392 printk("!!!WARNING: GRU ignoring request failure!!!\n"); 396 printk(KERN_WARNING
397 "!!!WARNING: GRU ignoring request failure!!!\n");
393 ret = 0; 398 ret = 0;
394 } 399 }
395 if (ret) { 400 if (ret) {
@@ -435,7 +440,7 @@ exit1:
435 440
436static void __exit gru_exit(void) 441static void __exit gru_exit(void)
437{ 442{
438 int i, bid; 443 int i, bid, gid;
439 int order = get_order(sizeof(struct gru_state) * 444 int order = get_order(sizeof(struct gru_state) *
440 GRU_CHIPLETS_PER_BLADE); 445 GRU_CHIPLETS_PER_BLADE);
441 446
@@ -445,6 +450,9 @@ static void __exit gru_exit(void)
445 for (i = 0; i < GRU_CHIPLETS_PER_BLADE; i++) 450 for (i = 0; i < GRU_CHIPLETS_PER_BLADE; i++)
446 free_irq(IRQ_GRU + i, NULL); 451 free_irq(IRQ_GRU + i, NULL);
447 452
453 foreach_gid(gid)
454 gru_kservices_exit(GID_TO_GRU(gid));
455
448 for (bid = 0; bid < GRU_MAX_BLADES; bid++) 456 for (bid = 0; bid < GRU_MAX_BLADES; bid++)
449 free_pages((unsigned long)gru_base[bid], order); 457 free_pages((unsigned long)gru_base[bid], order);
450 458
@@ -469,7 +477,11 @@ struct vm_operations_struct gru_vm_ops = {
469 .fault = gru_fault, 477 .fault = gru_fault,
470}; 478};
471 479
480#ifndef MODULE
472fs_initcall(gru_init); 481fs_initcall(gru_init);
482#else
483module_init(gru_init);
484#endif
473module_exit(gru_exit); 485module_exit(gru_exit);
474 486
475module_param(gru_options, ulong, 0644); 487module_param(gru_options, ulong, 0644);
diff --git a/drivers/misc/sgi-gru/gruhandles.c b/drivers/misc/sgi-gru/gruhandles.c
new file mode 100644
index 000000000000..9b7ccb328697
--- /dev/null
+++ b/drivers/misc/sgi-gru/gruhandles.c
@@ -0,0 +1,183 @@
1/*
2 * GRU KERNEL MCS INSTRUCTIONS
3 *
4 * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 */
20
21#include <linux/kernel.h>
22#include "gru.h"
23#include "grulib.h"
24#include "grutables.h"
25
26/* 10 sec */
27#ifdef CONFIG_IA64
28#include <asm/processor.h>
29#define GRU_OPERATION_TIMEOUT (((cycles_t) local_cpu_data->itc_freq)*10)
30#else
31#include <asm/tsc.h>
32#define GRU_OPERATION_TIMEOUT ((cycles_t) tsc_khz*10*1000)
33#endif
34
35/* Extract the status field from a kernel handle */
36#define GET_MSEG_HANDLE_STATUS(h) (((*(unsigned long *)(h)) >> 16) & 3)
37
38struct mcs_op_statistic mcs_op_statistics[mcsop_last];
39
40static void update_mcs_stats(enum mcs_op op, unsigned long clks)
41{
42 atomic_long_inc(&mcs_op_statistics[op].count);
43 atomic_long_add(clks, &mcs_op_statistics[op].total);
44 if (mcs_op_statistics[op].max < clks)
45 mcs_op_statistics[op].max = clks;
46}
47
48static void start_instruction(void *h)
49{
50 unsigned long *w0 = h;
51
52 wmb(); /* setting CMD bit must be last */
53 *w0 = *w0 | 1;
54 gru_flush_cache(h);
55}
56
57static int wait_instruction_complete(void *h, enum mcs_op opc)
58{
59 int status;
60 cycles_t start_time = get_cycles();
61
62 while (1) {
63 cpu_relax();
64 status = GET_MSEG_HANDLE_STATUS(h);
65 if (status != CCHSTATUS_ACTIVE)
66 break;
67 if (GRU_OPERATION_TIMEOUT < (get_cycles() - start_time))
68 panic("GRU %p is malfunctioning\n", h);
69 }
70 if (gru_options & OPT_STATS)
71 update_mcs_stats(opc, get_cycles() - start_time);
72 return status;
73}
74
75int cch_allocate(struct gru_context_configuration_handle *cch,
76 int asidval, int sizeavail, unsigned long cbrmap,
77 unsigned long dsrmap)
78{
79 int i;
80
81 for (i = 0; i < 8; i++) {
82 cch->asid[i] = (asidval++);
83 cch->sizeavail[i] = sizeavail;
84 }
85 cch->dsr_allocation_map = dsrmap;
86 cch->cbr_allocation_map = cbrmap;
87 cch->opc = CCHOP_ALLOCATE;
88 start_instruction(cch);
89 return wait_instruction_complete(cch, cchop_allocate);
90}
91
92int cch_start(struct gru_context_configuration_handle *cch)
93{
94 cch->opc = CCHOP_START;
95 start_instruction(cch);
96 return wait_instruction_complete(cch, cchop_start);
97}
98
99int cch_interrupt(struct gru_context_configuration_handle *cch)
100{
101 cch->opc = CCHOP_INTERRUPT;
102 start_instruction(cch);
103 return wait_instruction_complete(cch, cchop_interrupt);
104}
105
106int cch_deallocate(struct gru_context_configuration_handle *cch)
107{
108 cch->opc = CCHOP_DEALLOCATE;
109 start_instruction(cch);
110 return wait_instruction_complete(cch, cchop_deallocate);
111}
112
113int cch_interrupt_sync(struct gru_context_configuration_handle
114 *cch)
115{
116 cch->opc = CCHOP_INTERRUPT_SYNC;
117 start_instruction(cch);
118 return wait_instruction_complete(cch, cchop_interrupt_sync);
119}
120
121int tgh_invalidate(struct gru_tlb_global_handle *tgh,
122 unsigned long vaddr, unsigned long vaddrmask,
123 int asid, int pagesize, int global, int n,
124 unsigned short ctxbitmap)
125{
126 tgh->vaddr = vaddr;
127 tgh->asid = asid;
128 tgh->pagesize = pagesize;
129 tgh->n = n;
130 tgh->global = global;
131 tgh->vaddrmask = vaddrmask;
132 tgh->ctxbitmap = ctxbitmap;
133 tgh->opc = TGHOP_TLBINV;
134 start_instruction(tgh);
135 return wait_instruction_complete(tgh, tghop_invalidate);
136}
137
138void tfh_write_only(struct gru_tlb_fault_handle *tfh,
139 unsigned long pfn, unsigned long vaddr,
140 int asid, int dirty, int pagesize)
141{
142 tfh->fillasid = asid;
143 tfh->fillvaddr = vaddr;
144 tfh->pfn = pfn;
145 tfh->dirty = dirty;
146 tfh->pagesize = pagesize;
147 tfh->opc = TFHOP_WRITE_ONLY;
148 start_instruction(tfh);
149}
150
151void tfh_write_restart(struct gru_tlb_fault_handle *tfh,
152 unsigned long paddr, int gaa,
153 unsigned long vaddr, int asid, int dirty,
154 int pagesize)
155{
156 tfh->fillasid = asid;
157 tfh->fillvaddr = vaddr;
158 tfh->pfn = paddr >> GRU_PADDR_SHIFT;
159 tfh->gaa = gaa;
160 tfh->dirty = dirty;
161 tfh->pagesize = pagesize;
162 tfh->opc = TFHOP_WRITE_RESTART;
163 start_instruction(tfh);
164}
165
166void tfh_restart(struct gru_tlb_fault_handle *tfh)
167{
168 tfh->opc = TFHOP_RESTART;
169 start_instruction(tfh);
170}
171
172void tfh_user_polling_mode(struct gru_tlb_fault_handle *tfh)
173{
174 tfh->opc = TFHOP_USER_POLLING_MODE;
175 start_instruction(tfh);
176}
177
178void tfh_exception(struct gru_tlb_fault_handle *tfh)
179{
180 tfh->opc = TFHOP_EXCEPTION;
181 start_instruction(tfh);
182}
183
diff --git a/drivers/misc/sgi-gru/gruhandles.h b/drivers/misc/sgi-gru/gruhandles.h
index b63018d60fe1..1ed74d7508c8 100644
--- a/drivers/misc/sgi-gru/gruhandles.h
+++ b/drivers/misc/sgi-gru/gruhandles.h
@@ -489,170 +489,28 @@ enum gru_cbr_state {
489 * 64m 26 8 489 * 64m 26 8
490 * ... 490 * ...
491 */ 491 */
492#define GRU_PAGESIZE(sh) ((((sh) > 20 ? (sh) + 2: (sh)) >> 1) - 6) 492#define GRU_PAGESIZE(sh) ((((sh) > 20 ? (sh) + 2 : (sh)) >> 1) - 6)
493#define GRU_SIZEAVAIL(sh) (1UL << GRU_PAGESIZE(sh)) 493#define GRU_SIZEAVAIL(sh) (1UL << GRU_PAGESIZE(sh))
494 494
495/* minimum TLB purge count to ensure a full purge */ 495/* minimum TLB purge count to ensure a full purge */
496#define GRUMAXINVAL 1024UL 496#define GRUMAXINVAL 1024UL
497 497
498 498int cch_allocate(struct gru_context_configuration_handle *cch,
499/* Extract the status field from a kernel handle */ 499 int asidval, int sizeavail, unsigned long cbrmap, unsigned long dsrmap);
500#define GET_MSEG_HANDLE_STATUS(h) (((*(unsigned long *)(h)) >> 16) & 3) 500
501 501int cch_start(struct gru_context_configuration_handle *cch);
502static inline void start_instruction(void *h) 502int cch_interrupt(struct gru_context_configuration_handle *cch);
503{ 503int cch_deallocate(struct gru_context_configuration_handle *cch);
504 unsigned long *w0 = h; 504int cch_interrupt_sync(struct gru_context_configuration_handle *cch);
505 505int tgh_invalidate(struct gru_tlb_global_handle *tgh, unsigned long vaddr,
506 wmb(); /* setting CMD bit must be last */ 506 unsigned long vaddrmask, int asid, int pagesize, int global, int n,
507 *w0 = *w0 | 1; 507 unsigned short ctxbitmap);
508 gru_flush_cache(h); 508void tfh_write_only(struct gru_tlb_fault_handle *tfh, unsigned long pfn,
509} 509 unsigned long vaddr, int asid, int dirty, int pagesize);
510 510void tfh_write_restart(struct gru_tlb_fault_handle *tfh, unsigned long paddr,
511static inline int wait_instruction_complete(void *h) 511 int gaa, unsigned long vaddr, int asid, int dirty, int pagesize);
512{ 512void tfh_restart(struct gru_tlb_fault_handle *tfh);
513 int status; 513void tfh_user_polling_mode(struct gru_tlb_fault_handle *tfh);
514 514void tfh_exception(struct gru_tlb_fault_handle *tfh);
515 do {
516 cpu_relax();
517 barrier();
518 status = GET_MSEG_HANDLE_STATUS(h);
519 } while (status == CCHSTATUS_ACTIVE);
520 return status;
521}
522
523#if defined CONFIG_IA64
524static inline void cch_allocate_set_asids(
525 struct gru_context_configuration_handle *cch, int asidval)
526{
527 int i;
528
529 for (i = 0; i <= RGN_HPAGE; i++) { /* assume HPAGE is last region */
530 cch->asid[i] = (asidval++);
531#if 0
532 /* ZZZ hugepages not supported yet */
533 if (i == RGN_HPAGE)
534 cch->sizeavail[i] = GRU_SIZEAVAIL(hpage_shift);
535 else
536#endif
537 cch->sizeavail[i] = GRU_SIZEAVAIL(PAGE_SHIFT);
538 }
539}
540#elif defined CONFIG_X86_64
541static inline void cch_allocate_set_asids(
542 struct gru_context_configuration_handle *cch, int asidval)
543{
544 int i;
545
546 for (i = 0; i < 8; i++) {
547 cch->asid[i] = asidval++;
548 cch->sizeavail[i] = GRU_SIZEAVAIL(PAGE_SHIFT) |
549 GRU_SIZEAVAIL(21);
550 }
551}
552#endif
553
554static inline int cch_allocate(struct gru_context_configuration_handle *cch,
555 int asidval, unsigned long cbrmap,
556 unsigned long dsrmap)
557{
558 cch_allocate_set_asids(cch, asidval);
559 cch->dsr_allocation_map = dsrmap;
560 cch->cbr_allocation_map = cbrmap;
561 cch->opc = CCHOP_ALLOCATE;
562 start_instruction(cch);
563 return wait_instruction_complete(cch);
564}
565
566static inline int cch_start(struct gru_context_configuration_handle *cch)
567{
568 cch->opc = CCHOP_START;
569 start_instruction(cch);
570 return wait_instruction_complete(cch);
571}
572
573static inline int cch_interrupt(struct gru_context_configuration_handle *cch)
574{
575 cch->opc = CCHOP_INTERRUPT;
576 start_instruction(cch);
577 return wait_instruction_complete(cch);
578}
579
580static inline int cch_deallocate(struct gru_context_configuration_handle *cch)
581{
582 cch->opc = CCHOP_DEALLOCATE;
583 start_instruction(cch);
584 return wait_instruction_complete(cch);
585}
586
587static inline int cch_interrupt_sync(struct gru_context_configuration_handle
588 *cch)
589{
590 cch->opc = CCHOP_INTERRUPT_SYNC;
591 start_instruction(cch);
592 return wait_instruction_complete(cch);
593}
594
595static inline int tgh_invalidate(struct gru_tlb_global_handle *tgh,
596 unsigned long vaddr, unsigned long vaddrmask,
597 int asid, int pagesize, int global, int n,
598 unsigned short ctxbitmap)
599{
600 tgh->vaddr = vaddr;
601 tgh->asid = asid;
602 tgh->pagesize = pagesize;
603 tgh->n = n;
604 tgh->global = global;
605 tgh->vaddrmask = vaddrmask;
606 tgh->ctxbitmap = ctxbitmap;
607 tgh->opc = TGHOP_TLBINV;
608 start_instruction(tgh);
609 return wait_instruction_complete(tgh);
610}
611
612static inline void tfh_write_only(struct gru_tlb_fault_handle *tfh,
613 unsigned long pfn, unsigned long vaddr,
614 int asid, int dirty, int pagesize)
615{
616 tfh->fillasid = asid;
617 tfh->fillvaddr = vaddr;
618 tfh->pfn = pfn;
619 tfh->dirty = dirty;
620 tfh->pagesize = pagesize;
621 tfh->opc = TFHOP_WRITE_ONLY;
622 start_instruction(tfh);
623}
624
625static inline void tfh_write_restart(struct gru_tlb_fault_handle *tfh,
626 unsigned long paddr, int gaa,
627 unsigned long vaddr, int asid, int dirty,
628 int pagesize)
629{
630 tfh->fillasid = asid;
631 tfh->fillvaddr = vaddr;
632 tfh->pfn = paddr >> GRU_PADDR_SHIFT;
633 tfh->gaa = gaa;
634 tfh->dirty = dirty;
635 tfh->pagesize = pagesize;
636 tfh->opc = TFHOP_WRITE_RESTART;
637 start_instruction(tfh);
638}
639
640static inline void tfh_restart(struct gru_tlb_fault_handle *tfh)
641{
642 tfh->opc = TFHOP_RESTART;
643 start_instruction(tfh);
644}
645
646static inline void tfh_user_polling_mode(struct gru_tlb_fault_handle *tfh)
647{
648 tfh->opc = TFHOP_USER_POLLING_MODE;
649 start_instruction(tfh);
650}
651
652static inline void tfh_exception(struct gru_tlb_fault_handle *tfh)
653{
654 tfh->opc = TFHOP_EXCEPTION;
655 start_instruction(tfh);
656}
657 515
658#endif /* __GRUHANDLES_H__ */ 516#endif /* __GRUHANDLES_H__ */
diff --git a/drivers/misc/sgi-gru/grukservices.c b/drivers/misc/sgi-gru/grukservices.c
index 880c55dfb662..d8bd7d84a7cf 100644
--- a/drivers/misc/sgi-gru/grukservices.c
+++ b/drivers/misc/sgi-gru/grukservices.c
@@ -52,8 +52,10 @@
52 */ 52 */
53 53
54/* Blade percpu resources PERMANENTLY reserved for kernel use */ 54/* Blade percpu resources PERMANENTLY reserved for kernel use */
55#define GRU_NUM_KERNEL_CBR 1 55#define GRU_NUM_KERNEL_CBR 1
56#define GRU_NUM_KERNEL_DSR_BYTES 256 56#define GRU_NUM_KERNEL_DSR_BYTES 256
57#define GRU_NUM_KERNEL_DSR_CL (GRU_NUM_KERNEL_DSR_BYTES / \
58 GRU_CACHE_LINE_BYTES)
57#define KERNEL_CTXNUM 15 59#define KERNEL_CTXNUM 15
58 60
59/* GRU instruction attributes for all instructions */ 61/* GRU instruction attributes for all instructions */
@@ -94,7 +96,6 @@ struct message_header {
94 char fill; 96 char fill;
95}; 97};
96 98
97#define QLINES(mq) ((mq) + offsetof(struct message_queue, qlines))
98#define HSTATUS(mq, h) ((mq) + offsetof(struct message_queue, hstatus[h])) 99#define HSTATUS(mq, h) ((mq) + offsetof(struct message_queue, hstatus[h]))
99 100
100static int gru_get_cpu_resources(int dsr_bytes, void **cb, void **dsr) 101static int gru_get_cpu_resources(int dsr_bytes, void **cb, void **dsr)
@@ -122,7 +123,7 @@ int gru_get_cb_exception_detail(void *cb,
122 struct gru_control_block_extended *cbe; 123 struct gru_control_block_extended *cbe;
123 124
124 cbe = get_cbe(GRUBASE(cb), get_cb_number(cb)); 125 cbe = get_cbe(GRUBASE(cb), get_cb_number(cb));
125 prefetchw(cbe); /* Harmless on hardware, required for emulator */ 126 prefetchw(cbe); /* Harmless on hardware, required for emulator */
126 excdet->opc = cbe->opccpy; 127 excdet->opc = cbe->opccpy;
127 excdet->exopc = cbe->exopccpy; 128 excdet->exopc = cbe->exopccpy;
128 excdet->ecause = cbe->ecause; 129 excdet->ecause = cbe->ecause;
@@ -250,7 +251,8 @@ static inline void restore_present2(void *p, int val)
250 * Create a message queue. 251 * Create a message queue.
251 * qlines - message queue size in cache lines. Includes 2-line header. 252 * qlines - message queue size in cache lines. Includes 2-line header.
252 */ 253 */
253int gru_create_message_queue(void *p, unsigned int bytes) 254int gru_create_message_queue(struct gru_message_queue_desc *mqd,
255 void *p, unsigned int bytes, int nasid, int vector, int apicid)
254{ 256{
255 struct message_queue *mq = p; 257 struct message_queue *mq = p;
256 unsigned int qlines; 258 unsigned int qlines;
@@ -265,6 +267,12 @@ int gru_create_message_queue(void *p, unsigned int bytes)
265 mq->hstatus[0] = 0; 267 mq->hstatus[0] = 0;
266 mq->hstatus[1] = 1; 268 mq->hstatus[1] = 1;
267 mq->head = gru_mesq_head(2, qlines / 2 + 1); 269 mq->head = gru_mesq_head(2, qlines / 2 + 1);
270 mqd->mq = mq;
271 mqd->mq_gpa = uv_gpa(mq);
272 mqd->qlines = qlines;
273 mqd->interrupt_pnode = UV_NASID_TO_PNODE(nasid);
274 mqd->interrupt_vector = vector;
275 mqd->interrupt_apicid = apicid;
268 return 0; 276 return 0;
269} 277}
270EXPORT_SYMBOL_GPL(gru_create_message_queue); 278EXPORT_SYMBOL_GPL(gru_create_message_queue);
@@ -277,8 +285,8 @@ EXPORT_SYMBOL_GPL(gru_create_message_queue);
277 * -1 - if mesq sent successfully but queue not full 285 * -1 - if mesq sent successfully but queue not full
278 * >0 - unexpected error. MQE_xxx returned 286 * >0 - unexpected error. MQE_xxx returned
279 */ 287 */
280static int send_noop_message(void *cb, 288static int send_noop_message(void *cb, struct gru_message_queue_desc *mqd,
281 unsigned long mq, void *mesg) 289 void *mesg)
282{ 290{
283 const struct message_header noop_header = { 291 const struct message_header noop_header = {
284 .present = MQS_NOOP, .lines = 1}; 292 .present = MQS_NOOP, .lines = 1};
@@ -289,7 +297,7 @@ static int send_noop_message(void *cb,
289 STAT(mesq_noop); 297 STAT(mesq_noop);
290 save_mhdr = *mhdr; 298 save_mhdr = *mhdr;
291 *mhdr = noop_header; 299 *mhdr = noop_header;
292 gru_mesq(cb, mq, gru_get_tri(mhdr), 1, IMA); 300 gru_mesq(cb, mqd->mq_gpa, gru_get_tri(mhdr), 1, IMA);
293 ret = gru_wait(cb); 301 ret = gru_wait(cb);
294 302
295 if (ret) { 303 if (ret) {
@@ -313,7 +321,7 @@ static int send_noop_message(void *cb,
313 break; 321 break;
314 case CBSS_PUT_NACKED: 322 case CBSS_PUT_NACKED:
315 STAT(mesq_noop_put_nacked); 323 STAT(mesq_noop_put_nacked);
316 m = mq + (gru_get_amo_value_head(cb) << 6); 324 m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6);
317 gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, 1, 1, 325 gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, 1, 1,
318 IMA); 326 IMA);
319 if (gru_wait(cb) == CBS_IDLE) 327 if (gru_wait(cb) == CBS_IDLE)
@@ -333,30 +341,20 @@ static int send_noop_message(void *cb,
333/* 341/*
334 * Handle a gru_mesq full. 342 * Handle a gru_mesq full.
335 */ 343 */
336static int send_message_queue_full(void *cb, 344static int send_message_queue_full(void *cb, struct gru_message_queue_desc *mqd,
337 unsigned long mq, void *mesg, int lines) 345 void *mesg, int lines)
338{ 346{
339 union gru_mesqhead mqh; 347 union gru_mesqhead mqh;
340 unsigned int limit, head; 348 unsigned int limit, head;
341 unsigned long avalue; 349 unsigned long avalue;
342 int half, qlines, save; 350 int half, qlines;
343 351
344 /* Determine if switching to first/second half of q */ 352 /* Determine if switching to first/second half of q */
345 avalue = gru_get_amo_value(cb); 353 avalue = gru_get_amo_value(cb);
346 head = gru_get_amo_value_head(cb); 354 head = gru_get_amo_value_head(cb);
347 limit = gru_get_amo_value_limit(cb); 355 limit = gru_get_amo_value_limit(cb);
348 356
349 /* 357 qlines = mqd->qlines;
350 * Fetch "qlines" from the queue header. Since the queue may be
351 * in memory that can't be accessed using socket addresses, use
352 * the GRU to access the data. Use DSR space from the message.
353 */
354 save = *(int *)mesg;
355 gru_vload(cb, QLINES(mq), gru_get_tri(mesg), XTYPE_W, 1, 1, IMA);
356 if (gru_wait(cb) != CBS_IDLE)
357 goto cberr;
358 qlines = *(int *)mesg;
359 *(int *)mesg = save;
360 half = (limit != qlines); 358 half = (limit != qlines);
361 359
362 if (half) 360 if (half)
@@ -365,7 +363,7 @@ static int send_message_queue_full(void *cb,
365 mqh = gru_mesq_head(2, qlines / 2 + 1); 363 mqh = gru_mesq_head(2, qlines / 2 + 1);
366 364
367 /* Try to get lock for switching head pointer */ 365 /* Try to get lock for switching head pointer */
368 gru_gamir(cb, EOP_IR_CLR, HSTATUS(mq, half), XTYPE_DW, IMA); 366 gru_gamir(cb, EOP_IR_CLR, HSTATUS(mqd->mq_gpa, half), XTYPE_DW, IMA);
369 if (gru_wait(cb) != CBS_IDLE) 367 if (gru_wait(cb) != CBS_IDLE)
370 goto cberr; 368 goto cberr;
371 if (!gru_get_amo_value(cb)) { 369 if (!gru_get_amo_value(cb)) {
@@ -375,8 +373,8 @@ static int send_message_queue_full(void *cb,
375 373
376 /* Got the lock. Send optional NOP if queue not full, */ 374 /* Got the lock. Send optional NOP if queue not full, */
377 if (head != limit) { 375 if (head != limit) {
378 if (send_noop_message(cb, mq, mesg)) { 376 if (send_noop_message(cb, mqd, mesg)) {
379 gru_gamir(cb, EOP_IR_INC, HSTATUS(mq, half), 377 gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half),
380 XTYPE_DW, IMA); 378 XTYPE_DW, IMA);
381 if (gru_wait(cb) != CBS_IDLE) 379 if (gru_wait(cb) != CBS_IDLE)
382 goto cberr; 380 goto cberr;
@@ -387,14 +385,16 @@ static int send_message_queue_full(void *cb,
387 } 385 }
388 386
389 /* Then flip queuehead to other half of queue. */ 387 /* Then flip queuehead to other half of queue. */
390 gru_gamer(cb, EOP_ERR_CSWAP, mq, XTYPE_DW, mqh.val, avalue, IMA); 388 gru_gamer(cb, EOP_ERR_CSWAP, mqd->mq_gpa, XTYPE_DW, mqh.val, avalue,
389 IMA);
391 if (gru_wait(cb) != CBS_IDLE) 390 if (gru_wait(cb) != CBS_IDLE)
392 goto cberr; 391 goto cberr;
393 392
394 /* If not successfully in swapping queue head, clear the hstatus lock */ 393 /* If not successfully in swapping queue head, clear the hstatus lock */
395 if (gru_get_amo_value(cb) != avalue) { 394 if (gru_get_amo_value(cb) != avalue) {
396 STAT(mesq_qf_switch_head_failed); 395 STAT(mesq_qf_switch_head_failed);
397 gru_gamir(cb, EOP_IR_INC, HSTATUS(mq, half), XTYPE_DW, IMA); 396 gru_gamir(cb, EOP_IR_INC, HSTATUS(mqd->mq_gpa, half), XTYPE_DW,
397 IMA);
398 if (gru_wait(cb) != CBS_IDLE) 398 if (gru_wait(cb) != CBS_IDLE)
399 goto cberr; 399 goto cberr;
400 } 400 }
@@ -404,15 +404,25 @@ cberr:
404 return MQE_UNEXPECTED_CB_ERR; 404 return MQE_UNEXPECTED_CB_ERR;
405} 405}
406 406
407/*
408 * Send a cross-partition interrupt to the SSI that contains the target
409 * message queue. Normally, the interrupt is automatically delivered by hardware
410 * but some error conditions require explicit delivery.
411 */
412static void send_message_queue_interrupt(struct gru_message_queue_desc *mqd)
413{
414 if (mqd->interrupt_vector)
415 uv_hub_send_ipi(mqd->interrupt_pnode, mqd->interrupt_apicid,
416 mqd->interrupt_vector);
417}
418
407 419
408/* 420/*
409 * Handle a gru_mesq failure. Some of these failures are software recoverable 421 * Handle a gru_mesq failure. Some of these failures are software recoverable
410 * or retryable. 422 * or retryable.
411 */ 423 */
412static int send_message_failure(void *cb, 424static int send_message_failure(void *cb, struct gru_message_queue_desc *mqd,
413 unsigned long mq, 425 void *mesg, int lines)
414 void *mesg,
415 int lines)
416{ 426{
417 int substatus, ret = 0; 427 int substatus, ret = 0;
418 unsigned long m; 428 unsigned long m;
@@ -429,7 +439,7 @@ static int send_message_failure(void *cb,
429 break; 439 break;
430 case CBSS_QLIMIT_REACHED: 440 case CBSS_QLIMIT_REACHED:
431 STAT(mesq_send_qlimit_reached); 441 STAT(mesq_send_qlimit_reached);
432 ret = send_message_queue_full(cb, mq, mesg, lines); 442 ret = send_message_queue_full(cb, mqd, mesg, lines);
433 break; 443 break;
434 case CBSS_AMO_NACKED: 444 case CBSS_AMO_NACKED:
435 STAT(mesq_send_amo_nacked); 445 STAT(mesq_send_amo_nacked);
@@ -437,12 +447,14 @@ static int send_message_failure(void *cb,
437 break; 447 break;
438 case CBSS_PUT_NACKED: 448 case CBSS_PUT_NACKED:
439 STAT(mesq_send_put_nacked); 449 STAT(mesq_send_put_nacked);
440 m =mq + (gru_get_amo_value_head(cb) << 6); 450 m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6);
441 gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, lines, 1, IMA); 451 gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, lines, 1, IMA);
442 if (gru_wait(cb) == CBS_IDLE) 452 if (gru_wait(cb) == CBS_IDLE) {
443 ret = MQE_OK; 453 ret = MQE_OK;
444 else 454 send_message_queue_interrupt(mqd);
455 } else {
445 ret = MQE_UNEXPECTED_CB_ERR; 456 ret = MQE_UNEXPECTED_CB_ERR;
457 }
446 break; 458 break;
447 default: 459 default:
448 BUG(); 460 BUG();
@@ -452,12 +464,12 @@ static int send_message_failure(void *cb,
452 464
453/* 465/*
454 * Send a message to a message queue 466 * Send a message to a message queue
455 * cb GRU control block to use to send message 467 * mqd message queue descriptor
456 * mq message queue
457 * mesg message. ust be vaddr within a GSEG 468 * mesg message. ust be vaddr within a GSEG
458 * bytes message size (<= 2 CL) 469 * bytes message size (<= 2 CL)
459 */ 470 */
460int gru_send_message_gpa(unsigned long mq, void *mesg, unsigned int bytes) 471int gru_send_message_gpa(struct gru_message_queue_desc *mqd, void *mesg,
472 unsigned int bytes)
461{ 473{
462 struct message_header *mhdr; 474 struct message_header *mhdr;
463 void *cb; 475 void *cb;
@@ -481,10 +493,10 @@ int gru_send_message_gpa(unsigned long mq, void *mesg, unsigned int bytes)
481 493
482 do { 494 do {
483 ret = MQE_OK; 495 ret = MQE_OK;
484 gru_mesq(cb, mq, gru_get_tri(mhdr), clines, IMA); 496 gru_mesq(cb, mqd->mq_gpa, gru_get_tri(mhdr), clines, IMA);
485 istatus = gru_wait(cb); 497 istatus = gru_wait(cb);
486 if (istatus != CBS_IDLE) 498 if (istatus != CBS_IDLE)
487 ret = send_message_failure(cb, mq, dsr, clines); 499 ret = send_message_failure(cb, mqd, dsr, clines);
488 } while (ret == MQIE_AGAIN); 500 } while (ret == MQIE_AGAIN);
489 gru_free_cpu_resources(cb, dsr); 501 gru_free_cpu_resources(cb, dsr);
490 502
@@ -497,9 +509,9 @@ EXPORT_SYMBOL_GPL(gru_send_message_gpa);
497/* 509/*
498 * Advance the receive pointer for the queue to the next message. 510 * Advance the receive pointer for the queue to the next message.
499 */ 511 */
500void gru_free_message(void *rmq, void *mesg) 512void gru_free_message(struct gru_message_queue_desc *mqd, void *mesg)
501{ 513{
502 struct message_queue *mq = rmq; 514 struct message_queue *mq = mqd->mq;
503 struct message_header *mhdr = mq->next; 515 struct message_header *mhdr = mq->next;
504 void *next, *pnext; 516 void *next, *pnext;
505 int half = -1; 517 int half = -1;
@@ -529,16 +541,16 @@ EXPORT_SYMBOL_GPL(gru_free_message);
529 * present. User must call next_message() to move to next message. 541 * present. User must call next_message() to move to next message.
530 * rmq message queue 542 * rmq message queue
531 */ 543 */
532void *gru_get_next_message(void *rmq) 544void *gru_get_next_message(struct gru_message_queue_desc *mqd)
533{ 545{
534 struct message_queue *mq = rmq; 546 struct message_queue *mq = mqd->mq;
535 struct message_header *mhdr = mq->next; 547 struct message_header *mhdr = mq->next;
536 int present = mhdr->present; 548 int present = mhdr->present;
537 549
538 /* skip NOOP messages */ 550 /* skip NOOP messages */
539 STAT(mesq_receive); 551 STAT(mesq_receive);
540 while (present == MQS_NOOP) { 552 while (present == MQS_NOOP) {
541 gru_free_message(rmq, mhdr); 553 gru_free_message(mqd, mhdr);
542 mhdr = mq->next; 554 mhdr = mq->next;
543 present = mhdr->present; 555 present = mhdr->present;
544 } 556 }
@@ -576,7 +588,7 @@ int gru_copy_gpa(unsigned long dest_gpa, unsigned long src_gpa,
576 if (gru_get_cpu_resources(GRU_NUM_KERNEL_DSR_BYTES, &cb, &dsr)) 588 if (gru_get_cpu_resources(GRU_NUM_KERNEL_DSR_BYTES, &cb, &dsr))
577 return MQE_BUG_NO_RESOURCES; 589 return MQE_BUG_NO_RESOURCES;
578 gru_bcopy(cb, src_gpa, dest_gpa, gru_get_tri(dsr), 590 gru_bcopy(cb, src_gpa, dest_gpa, gru_get_tri(dsr),
579 XTYPE_B, bytes, GRU_NUM_KERNEL_DSR_BYTES, IMA); 591 XTYPE_B, bytes, GRU_NUM_KERNEL_DSR_CL, IMA);
580 ret = gru_wait(cb); 592 ret = gru_wait(cb);
581 gru_free_cpu_resources(cb, dsr); 593 gru_free_cpu_resources(cb, dsr);
582 return ret; 594 return ret;
@@ -611,7 +623,7 @@ static int quicktest(struct gru_state *gru)
611 623
612 if (word0 != word1 || word0 != MAGIC) { 624 if (word0 != word1 || word0 != MAGIC) {
613 printk 625 printk
614 ("GRU quicktest err: gru %d, found 0x%lx, expected 0x%lx\n", 626 ("GRU quicktest err: gid %d, found 0x%lx, expected 0x%lx\n",
615 gru->gs_gid, word1, MAGIC); 627 gru->gs_gid, word1, MAGIC);
616 BUG(); /* ZZZ should not be fatal */ 628 BUG(); /* ZZZ should not be fatal */
617 } 629 }
@@ -660,15 +672,15 @@ int gru_kservices_init(struct gru_state *gru)
660 cch->tlb_int_enable = 0; 672 cch->tlb_int_enable = 0;
661 cch->tfm_done_bit_enable = 0; 673 cch->tfm_done_bit_enable = 0;
662 cch->unmap_enable = 1; 674 cch->unmap_enable = 1;
663 err = cch_allocate(cch, 0, cbr_map, dsr_map); 675 err = cch_allocate(cch, 0, 0, cbr_map, dsr_map);
664 if (err) { 676 if (err) {
665 gru_dbg(grudev, 677 gru_dbg(grudev,
666 "Unable to allocate kernel CCH: gru %d, err %d\n", 678 "Unable to allocate kernel CCH: gid %d, err %d\n",
667 gru->gs_gid, err); 679 gru->gs_gid, err);
668 BUG(); 680 BUG();
669 } 681 }
670 if (cch_start(cch)) { 682 if (cch_start(cch)) {
671 gru_dbg(grudev, "Unable to start kernel CCH: gru %d, err %d\n", 683 gru_dbg(grudev, "Unable to start kernel CCH: gid %d, err %d\n",
672 gru->gs_gid, err); 684 gru->gs_gid, err);
673 BUG(); 685 BUG();
674 } 686 }
@@ -678,3 +690,22 @@ int gru_kservices_init(struct gru_state *gru)
678 quicktest(gru); 690 quicktest(gru);
679 return 0; 691 return 0;
680} 692}
693
694void gru_kservices_exit(struct gru_state *gru)
695{
696 struct gru_context_configuration_handle *cch;
697 struct gru_blade_state *bs;
698
699 bs = gru->gs_blade;
700 if (gru != &bs->bs_grus[1])
701 return;
702
703 cch = get_cch(gru->gs_gru_base_vaddr, KERNEL_CTXNUM);
704 lock_cch_handle(cch);
705 if (cch_interrupt_sync(cch))
706 BUG();
707 if (cch_deallocate(cch))
708 BUG();
709 unlock_cch_handle(cch);
710}
711
diff --git a/drivers/misc/sgi-gru/grukservices.h b/drivers/misc/sgi-gru/grukservices.h
index eb17e0a3ac61..747ed315d56f 100644
--- a/drivers/misc/sgi-gru/grukservices.h
+++ b/drivers/misc/sgi-gru/grukservices.h
@@ -41,6 +41,15 @@
41 * - gru_create_message_queue() needs interrupt vector info 41 * - gru_create_message_queue() needs interrupt vector info
42 */ 42 */
43 43
44struct gru_message_queue_desc {
45 void *mq; /* message queue vaddress */
46 unsigned long mq_gpa; /* global address of mq */
47 int qlines; /* queue size in CL */
48 int interrupt_vector; /* interrupt vector */
49 int interrupt_pnode; /* pnode for interrupt */
50 int interrupt_apicid; /* lapicid for interrupt */
51};
52
44/* 53/*
45 * Initialize a user allocated chunk of memory to be used as 54 * Initialize a user allocated chunk of memory to be used as
46 * a message queue. The caller must ensure that the queue is 55 * a message queue. The caller must ensure that the queue is
@@ -51,14 +60,19 @@
51 * to manage the queue. 60 * to manage the queue.
52 * 61 *
53 * Input: 62 * Input:
54 * p pointer to user allocated memory. 63 * mqd pointer to message queue descriptor
64 * p pointer to user allocated mesq memory.
55 * bytes size of message queue in bytes 65 * bytes size of message queue in bytes
66 * vector interrupt vector (zero if no interrupts)
67 * nasid nasid of blade where interrupt is delivered
68 * apicid apicid of cpu for interrupt
56 * 69 *
57 * Errors: 70 * Errors:
58 * 0 OK 71 * 0 OK
59 * >0 error 72 * >0 error
60 */ 73 */
61extern int gru_create_message_queue(void *p, unsigned int bytes); 74extern int gru_create_message_queue(struct gru_message_queue_desc *mqd,
75 void *p, unsigned int bytes, int nasid, int vector, int apicid);
62 76
63/* 77/*
64 * Send a message to a message queue. 78 * Send a message to a message queue.
@@ -68,7 +82,7 @@ extern int gru_create_message_queue(void *p, unsigned int bytes);
68 * 82 *
69 * 83 *
70 * Input: 84 * Input:
71 * xmq message queue - must be a UV global physical address 85 * mqd pointer to message queue descriptor
72 * mesg pointer to message. Must be 64-bit aligned 86 * mesg pointer to message. Must be 64-bit aligned
73 * bytes size of message in bytes 87 * bytes size of message in bytes
74 * 88 *
@@ -77,8 +91,8 @@ extern int gru_create_message_queue(void *p, unsigned int bytes);
77 * >0 Send failure - see error codes below 91 * >0 Send failure - see error codes below
78 * 92 *
79 */ 93 */
80extern int gru_send_message_gpa(unsigned long mq_gpa, void *mesg, 94extern int gru_send_message_gpa(struct gru_message_queue_desc *mqd,
81 unsigned int bytes); 95 void *mesg, unsigned int bytes);
82 96
83/* Status values for gru_send_message() */ 97/* Status values for gru_send_message() */
84#define MQE_OK 0 /* message sent successfully */ 98#define MQE_OK 0 /* message sent successfully */
@@ -94,10 +108,11 @@ extern int gru_send_message_gpa(unsigned long mq_gpa, void *mesg,
94 * API extensions may allow for out-of-order freeing. 108 * API extensions may allow for out-of-order freeing.
95 * 109 *
96 * Input 110 * Input
97 * mq message queue 111 * mqd pointer to message queue descriptor
98 * mesq message being freed 112 * mesq message being freed
99 */ 113 */
100extern void gru_free_message(void *mq, void *mesq); 114extern void gru_free_message(struct gru_message_queue_desc *mqd,
115 void *mesq);
101 116
102/* 117/*
103 * Get next message from message queue. Returns pointer to 118 * Get next message from message queue. Returns pointer to
@@ -106,13 +121,13 @@ extern void gru_free_message(void *mq, void *mesq);
106 * in order to move the queue pointers to next message. 121 * in order to move the queue pointers to next message.
107 * 122 *
108 * Input 123 * Input
109 * mq message queue 124 * mqd pointer to message queue descriptor
110 * 125 *
111 * Output: 126 * Output:
112 * p pointer to message 127 * p pointer to message
113 * NULL no message available 128 * NULL no message available
114 */ 129 */
115extern void *gru_get_next_message(void *mq); 130extern void *gru_get_next_message(struct gru_message_queue_desc *mqd);
116 131
117 132
118/* 133/*
diff --git a/drivers/misc/sgi-gru/grumain.c b/drivers/misc/sgi-gru/grumain.c
index 3d2fc216bae5..ec3f7a17d221 100644
--- a/drivers/misc/sgi-gru/grumain.c
+++ b/drivers/misc/sgi-gru/grumain.c
@@ -76,10 +76,9 @@ int gru_cpu_fault_map_id(void)
76/* Hit the asid limit. Start over */ 76/* Hit the asid limit. Start over */
77static int gru_wrap_asid(struct gru_state *gru) 77static int gru_wrap_asid(struct gru_state *gru)
78{ 78{
79 gru_dbg(grudev, "gru %p\n", gru); 79 gru_dbg(grudev, "gid %d\n", gru->gs_gid);
80 STAT(asid_wrap); 80 STAT(asid_wrap);
81 gru->gs_asid_gen++; 81 gru->gs_asid_gen++;
82 gru_flush_all_tlb(gru);
83 return MIN_ASID; 82 return MIN_ASID;
84} 83}
85 84
@@ -88,19 +87,21 @@ static int gru_reset_asid_limit(struct gru_state *gru, int asid)
88{ 87{
89 int i, gid, inuse_asid, limit; 88 int i, gid, inuse_asid, limit;
90 89
91 gru_dbg(grudev, "gru %p, asid 0x%x\n", gru, asid); 90 gru_dbg(grudev, "gid %d, asid 0x%x\n", gru->gs_gid, asid);
92 STAT(asid_next); 91 STAT(asid_next);
93 limit = MAX_ASID; 92 limit = MAX_ASID;
94 if (asid >= limit) 93 if (asid >= limit)
95 asid = gru_wrap_asid(gru); 94 asid = gru_wrap_asid(gru);
95 gru_flush_all_tlb(gru);
96 gid = gru->gs_gid; 96 gid = gru->gs_gid;
97again: 97again:
98 for (i = 0; i < GRU_NUM_CCH; i++) { 98 for (i = 0; i < GRU_NUM_CCH; i++) {
99 if (!gru->gs_gts[i]) 99 if (!gru->gs_gts[i])
100 continue; 100 continue;
101 inuse_asid = gru->gs_gts[i]->ts_gms->ms_asids[gid].mt_asid; 101 inuse_asid = gru->gs_gts[i]->ts_gms->ms_asids[gid].mt_asid;
102 gru_dbg(grudev, "gru %p, inuse_asid 0x%x, cxtnum %d, gts %p\n", 102 gru_dbg(grudev, "gid %d, gts %p, gms %p, inuse 0x%x, cxt %d\n",
103 gru, inuse_asid, i, gru->gs_gts[i]); 103 gru->gs_gid, gru->gs_gts[i], gru->gs_gts[i]->ts_gms,
104 inuse_asid, i);
104 if (inuse_asid == asid) { 105 if (inuse_asid == asid) {
105 asid += ASID_INC; 106 asid += ASID_INC;
106 if (asid >= limit) { 107 if (asid >= limit) {
@@ -120,8 +121,8 @@ again:
120 } 121 }
121 gru->gs_asid_limit = limit; 122 gru->gs_asid_limit = limit;
122 gru->gs_asid = asid; 123 gru->gs_asid = asid;
123 gru_dbg(grudev, "gru %p, new asid 0x%x, new_limit 0x%x\n", gru, asid, 124 gru_dbg(grudev, "gid %d, new asid 0x%x, new_limit 0x%x\n", gru->gs_gid,
124 limit); 125 asid, limit);
125 return asid; 126 return asid;
126} 127}
127 128
@@ -130,14 +131,12 @@ static int gru_assign_asid(struct gru_state *gru)
130{ 131{
131 int asid; 132 int asid;
132 133
133 spin_lock(&gru->gs_asid_lock);
134 gru->gs_asid += ASID_INC; 134 gru->gs_asid += ASID_INC;
135 asid = gru->gs_asid; 135 asid = gru->gs_asid;
136 if (asid >= gru->gs_asid_limit) 136 if (asid >= gru->gs_asid_limit)
137 asid = gru_reset_asid_limit(gru, asid); 137 asid = gru_reset_asid_limit(gru, asid);
138 spin_unlock(&gru->gs_asid_lock);
139 138
140 gru_dbg(grudev, "gru %p, asid 0x%x\n", gru, asid); 139 gru_dbg(grudev, "gid %d, asid 0x%x\n", gru->gs_gid, asid);
141 return asid; 140 return asid;
142} 141}
143 142
@@ -215,17 +214,20 @@ static int check_gru_resources(struct gru_state *gru, int cbr_au_count,
215 * TLB manangment requires tracking all GRU chiplets that have loaded a GSEG 214 * TLB manangment requires tracking all GRU chiplets that have loaded a GSEG
216 * context. 215 * context.
217 */ 216 */
218static int gru_load_mm_tracker(struct gru_state *gru, struct gru_mm_struct *gms, 217static int gru_load_mm_tracker(struct gru_state *gru,
219 int ctxnum) 218 struct gru_thread_state *gts)
220{ 219{
220 struct gru_mm_struct *gms = gts->ts_gms;
221 struct gru_mm_tracker *asids = &gms->ms_asids[gru->gs_gid]; 221 struct gru_mm_tracker *asids = &gms->ms_asids[gru->gs_gid];
222 unsigned short ctxbitmap = (1 << ctxnum); 222 unsigned short ctxbitmap = (1 << gts->ts_ctxnum);
223 int asid; 223 int asid;
224 224
225 spin_lock(&gms->ms_asid_lock); 225 spin_lock(&gms->ms_asid_lock);
226 asid = asids->mt_asid; 226 asid = asids->mt_asid;
227 227
228 if (asid == 0 || asids->mt_asid_gen != gru->gs_asid_gen) { 228 spin_lock(&gru->gs_asid_lock);
229 if (asid == 0 || (asids->mt_ctxbitmap == 0 && asids->mt_asid_gen !=
230 gru->gs_asid_gen)) {
229 asid = gru_assign_asid(gru); 231 asid = gru_assign_asid(gru);
230 asids->mt_asid = asid; 232 asids->mt_asid = asid;
231 asids->mt_asid_gen = gru->gs_asid_gen; 233 asids->mt_asid_gen = gru->gs_asid_gen;
@@ -233,6 +235,7 @@ static int gru_load_mm_tracker(struct gru_state *gru, struct gru_mm_struct *gms,
233 } else { 235 } else {
234 STAT(asid_reuse); 236 STAT(asid_reuse);
235 } 237 }
238 spin_unlock(&gru->gs_asid_lock);
236 239
237 BUG_ON(asids->mt_ctxbitmap & ctxbitmap); 240 BUG_ON(asids->mt_ctxbitmap & ctxbitmap);
238 asids->mt_ctxbitmap |= ctxbitmap; 241 asids->mt_ctxbitmap |= ctxbitmap;
@@ -241,24 +244,28 @@ static int gru_load_mm_tracker(struct gru_state *gru, struct gru_mm_struct *gms,
241 spin_unlock(&gms->ms_asid_lock); 244 spin_unlock(&gms->ms_asid_lock);
242 245
243 gru_dbg(grudev, 246 gru_dbg(grudev,
244 "gru %x, gms %p, ctxnum 0x%d, asid 0x%x, asidmap 0x%lx\n", 247 "gid %d, gts %p, gms %p, ctxnum %d, asid 0x%x, asidmap 0x%lx\n",
245 gru->gs_gid, gms, ctxnum, asid, gms->ms_asidmap[0]); 248 gru->gs_gid, gts, gms, gts->ts_ctxnum, asid,
249 gms->ms_asidmap[0]);
246 return asid; 250 return asid;
247} 251}
248 252
249static void gru_unload_mm_tracker(struct gru_state *gru, 253static void gru_unload_mm_tracker(struct gru_state *gru,
250 struct gru_mm_struct *gms, int ctxnum) 254 struct gru_thread_state *gts)
251{ 255{
256 struct gru_mm_struct *gms = gts->ts_gms;
252 struct gru_mm_tracker *asids; 257 struct gru_mm_tracker *asids;
253 unsigned short ctxbitmap; 258 unsigned short ctxbitmap;
254 259
255 asids = &gms->ms_asids[gru->gs_gid]; 260 asids = &gms->ms_asids[gru->gs_gid];
256 ctxbitmap = (1 << ctxnum); 261 ctxbitmap = (1 << gts->ts_ctxnum);
257 spin_lock(&gms->ms_asid_lock); 262 spin_lock(&gms->ms_asid_lock);
263 spin_lock(&gru->gs_asid_lock);
258 BUG_ON((asids->mt_ctxbitmap & ctxbitmap) != ctxbitmap); 264 BUG_ON((asids->mt_ctxbitmap & ctxbitmap) != ctxbitmap);
259 asids->mt_ctxbitmap ^= ctxbitmap; 265 asids->mt_ctxbitmap ^= ctxbitmap;
260 gru_dbg(grudev, "gru %x, gms %p, ctxnum 0x%d, asidmap 0x%lx\n", 266 gru_dbg(grudev, "gid %d, gts %p, gms %p, ctxnum 0x%d, asidmap 0x%lx\n",
261 gru->gs_gid, gms, ctxnum, gms->ms_asidmap[0]); 267 gru->gs_gid, gts, gms, gts->ts_ctxnum, gms->ms_asidmap[0]);
268 spin_unlock(&gru->gs_asid_lock);
262 spin_unlock(&gms->ms_asid_lock); 269 spin_unlock(&gms->ms_asid_lock);
263} 270}
264 271
@@ -319,6 +326,7 @@ static struct gru_thread_state *gru_alloc_gts(struct vm_area_struct *vma,
319 gts->ts_vma = vma; 326 gts->ts_vma = vma;
320 gts->ts_tlb_int_select = -1; 327 gts->ts_tlb_int_select = -1;
321 gts->ts_gms = gru_register_mmu_notifier(); 328 gts->ts_gms = gru_register_mmu_notifier();
329 gts->ts_sizeavail = GRU_SIZEAVAIL(PAGE_SHIFT);
322 if (!gts->ts_gms) 330 if (!gts->ts_gms)
323 goto err; 331 goto err;
324 332
@@ -399,7 +407,7 @@ static void gru_free_gru_context(struct gru_thread_state *gts)
399 struct gru_state *gru; 407 struct gru_state *gru;
400 408
401 gru = gts->ts_gru; 409 gru = gts->ts_gru;
402 gru_dbg(grudev, "gts %p, gru %p\n", gts, gru); 410 gru_dbg(grudev, "gts %p, gid %d\n", gts, gru->gs_gid);
403 411
404 spin_lock(&gru->gs_lock); 412 spin_lock(&gru->gs_lock);
405 gru->gs_gts[gts->ts_ctxnum] = NULL; 413 gru->gs_gts[gts->ts_ctxnum] = NULL;
@@ -408,6 +416,7 @@ static void gru_free_gru_context(struct gru_thread_state *gts)
408 __clear_bit(gts->ts_ctxnum, &gru->gs_context_map); 416 __clear_bit(gts->ts_ctxnum, &gru->gs_context_map);
409 gts->ts_ctxnum = NULLCTX; 417 gts->ts_ctxnum = NULLCTX;
410 gts->ts_gru = NULL; 418 gts->ts_gru = NULL;
419 gts->ts_blade = -1;
411 spin_unlock(&gru->gs_lock); 420 spin_unlock(&gru->gs_lock);
412 421
413 gts_drop(gts); 422 gts_drop(gts);
@@ -432,8 +441,8 @@ static inline long gru_copy_handle(void *d, void *s)
432 return GRU_HANDLE_BYTES; 441 return GRU_HANDLE_BYTES;
433} 442}
434 443
435static void gru_prefetch_context(void *gseg, void *cb, void *cbe, unsigned long cbrmap, 444static void gru_prefetch_context(void *gseg, void *cb, void *cbe,
436 unsigned long length) 445 unsigned long cbrmap, unsigned long length)
437{ 446{
438 int i, scr; 447 int i, scr;
439 448
@@ -500,12 +509,12 @@ void gru_unload_context(struct gru_thread_state *gts, int savestate)
500 zap_vma_ptes(gts->ts_vma, UGRUADDR(gts), GRU_GSEG_PAGESIZE); 509 zap_vma_ptes(gts->ts_vma, UGRUADDR(gts), GRU_GSEG_PAGESIZE);
501 cch = get_cch(gru->gs_gru_base_vaddr, ctxnum); 510 cch = get_cch(gru->gs_gru_base_vaddr, ctxnum);
502 511
512 gru_dbg(grudev, "gts %p\n", gts);
503 lock_cch_handle(cch); 513 lock_cch_handle(cch);
504 if (cch_interrupt_sync(cch)) 514 if (cch_interrupt_sync(cch))
505 BUG(); 515 BUG();
506 gru_dbg(grudev, "gts %p\n", gts);
507 516
508 gru_unload_mm_tracker(gru, gts->ts_gms, gts->ts_ctxnum); 517 gru_unload_mm_tracker(gru, gts);
509 if (savestate) 518 if (savestate)
510 gru_unload_context_data(gts->ts_gdata, gru->gs_gru_base_vaddr, 519 gru_unload_context_data(gts->ts_gdata, gru->gs_gru_base_vaddr,
511 ctxnum, gts->ts_cbr_map, 520 ctxnum, gts->ts_cbr_map,
@@ -534,7 +543,7 @@ static void gru_load_context(struct gru_thread_state *gts)
534 cch = get_cch(gru->gs_gru_base_vaddr, ctxnum); 543 cch = get_cch(gru->gs_gru_base_vaddr, ctxnum);
535 544
536 lock_cch_handle(cch); 545 lock_cch_handle(cch);
537 asid = gru_load_mm_tracker(gru, gts->ts_gms, gts->ts_ctxnum); 546 asid = gru_load_mm_tracker(gru, gts);
538 cch->tfm_fault_bit_enable = 547 cch->tfm_fault_bit_enable =
539 (gts->ts_user_options == GRU_OPT_MISS_FMM_POLL 548 (gts->ts_user_options == GRU_OPT_MISS_FMM_POLL
540 || gts->ts_user_options == GRU_OPT_MISS_FMM_INTR); 549 || gts->ts_user_options == GRU_OPT_MISS_FMM_INTR);
@@ -544,7 +553,8 @@ static void gru_load_context(struct gru_thread_state *gts)
544 cch->tlb_int_select = gts->ts_tlb_int_select; 553 cch->tlb_int_select = gts->ts_tlb_int_select;
545 } 554 }
546 cch->tfm_done_bit_enable = 0; 555 cch->tfm_done_bit_enable = 0;
547 err = cch_allocate(cch, asid, gts->ts_cbr_map, gts->ts_dsr_map); 556 err = cch_allocate(cch, asid, gts->ts_sizeavail, gts->ts_cbr_map,
557 gts->ts_dsr_map);
548 if (err) { 558 if (err) {
549 gru_dbg(grudev, 559 gru_dbg(grudev,
550 "err %d: cch %p, gts %p, cbr 0x%lx, dsr 0x%lx\n", 560 "err %d: cch %p, gts %p, cbr 0x%lx, dsr 0x%lx\n",
@@ -565,11 +575,12 @@ static void gru_load_context(struct gru_thread_state *gts)
565/* 575/*
566 * Update fields in an active CCH: 576 * Update fields in an active CCH:
567 * - retarget interrupts on local blade 577 * - retarget interrupts on local blade
578 * - update sizeavail mask
568 * - force a delayed context unload by clearing the CCH asids. This 579 * - force a delayed context unload by clearing the CCH asids. This
569 * forces TLB misses for new GRU instructions. The context is unloaded 580 * forces TLB misses for new GRU instructions. The context is unloaded
570 * when the next TLB miss occurs. 581 * when the next TLB miss occurs.
571 */ 582 */
572static int gru_update_cch(struct gru_thread_state *gts, int int_select) 583int gru_update_cch(struct gru_thread_state *gts, int force_unload)
573{ 584{
574 struct gru_context_configuration_handle *cch; 585 struct gru_context_configuration_handle *cch;
575 struct gru_state *gru = gts->ts_gru; 586 struct gru_state *gru = gts->ts_gru;
@@ -583,9 +594,11 @@ static int gru_update_cch(struct gru_thread_state *gts, int int_select)
583 goto exit; 594 goto exit;
584 if (cch_interrupt(cch)) 595 if (cch_interrupt(cch))
585 BUG(); 596 BUG();
586 if (int_select >= 0) { 597 if (!force_unload) {
587 gts->ts_tlb_int_select = int_select; 598 for (i = 0; i < 8; i++)
588 cch->tlb_int_select = int_select; 599 cch->sizeavail[i] = gts->ts_sizeavail;
600 gts->ts_tlb_int_select = gru_cpu_fault_map_id();
601 cch->tlb_int_select = gru_cpu_fault_map_id();
589 } else { 602 } else {
590 for (i = 0; i < 8; i++) 603 for (i = 0; i < 8; i++)
591 cch->asid[i] = 0; 604 cch->asid[i] = 0;
@@ -617,7 +630,7 @@ static int gru_retarget_intr(struct gru_thread_state *gts)
617 630
618 gru_dbg(grudev, "retarget from %d to %d\n", gts->ts_tlb_int_select, 631 gru_dbg(grudev, "retarget from %d to %d\n", gts->ts_tlb_int_select,
619 gru_cpu_fault_map_id()); 632 gru_cpu_fault_map_id());
620 return gru_update_cch(gts, gru_cpu_fault_map_id()); 633 return gru_update_cch(gts, 0);
621} 634}
622 635
623 636
@@ -688,7 +701,7 @@ static void gru_steal_context(struct gru_thread_state *gts)
688 STAT(steal_context_failed); 701 STAT(steal_context_failed);
689 } 702 }
690 gru_dbg(grudev, 703 gru_dbg(grudev,
691 "stole gru %x, ctxnum %d from gts %p. Need cb %d, ds %d;" 704 "stole gid %d, ctxnum %d from gts %p. Need cb %d, ds %d;"
692 " avail cb %ld, ds %ld\n", 705 " avail cb %ld, ds %ld\n",
693 gru->gs_gid, ctxnum, ngts, cbr, dsr, hweight64(gru->gs_cbr_map), 706 gru->gs_gid, ctxnum, ngts, cbr, dsr, hweight64(gru->gs_cbr_map),
694 hweight64(gru->gs_dsr_map)); 707 hweight64(gru->gs_dsr_map));
@@ -727,6 +740,7 @@ again:
727 } 740 }
728 reserve_gru_resources(gru, gts); 741 reserve_gru_resources(gru, gts);
729 gts->ts_gru = gru; 742 gts->ts_gru = gru;
743 gts->ts_blade = gru->gs_blade_id;
730 gts->ts_ctxnum = 744 gts->ts_ctxnum =
731 find_first_zero_bit(&gru->gs_context_map, GRU_NUM_CCH); 745 find_first_zero_bit(&gru->gs_context_map, GRU_NUM_CCH);
732 BUG_ON(gts->ts_ctxnum == GRU_NUM_CCH); 746 BUG_ON(gts->ts_ctxnum == GRU_NUM_CCH);
@@ -737,7 +751,7 @@ again:
737 751
738 STAT(assign_context); 752 STAT(assign_context);
739 gru_dbg(grudev, 753 gru_dbg(grudev,
740 "gseg %p, gts %p, gru %x, ctx %d, cbr %d, dsr %d\n", 754 "gseg %p, gts %p, gid %d, ctx %d, cbr %d, dsr %d\n",
741 gseg_virtual_address(gts->ts_gru, gts->ts_ctxnum), gts, 755 gseg_virtual_address(gts->ts_gru, gts->ts_ctxnum), gts,
742 gts->ts_gru->gs_gid, gts->ts_ctxnum, 756 gts->ts_gru->gs_gid, gts->ts_ctxnum,
743 gts->ts_cbr_au_count, gts->ts_dsr_au_count); 757 gts->ts_cbr_au_count, gts->ts_dsr_au_count);
@@ -773,8 +787,8 @@ int gru_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
773 return VM_FAULT_SIGBUS; 787 return VM_FAULT_SIGBUS;
774 788
775again: 789again:
776 preempt_disable();
777 mutex_lock(&gts->ts_ctxlock); 790 mutex_lock(&gts->ts_ctxlock);
791 preempt_disable();
778 if (gts->ts_gru) { 792 if (gts->ts_gru) {
779 if (gts->ts_gru->gs_blade_id != uv_numa_blade_id()) { 793 if (gts->ts_gru->gs_blade_id != uv_numa_blade_id()) {
780 STAT(migrated_nopfn_unload); 794 STAT(migrated_nopfn_unload);
diff --git a/drivers/misc/sgi-gru/gruprocfs.c b/drivers/misc/sgi-gru/gruprocfs.c
index 73b0ca061bb5..ee74821b171c 100644
--- a/drivers/misc/sgi-gru/gruprocfs.c
+++ b/drivers/misc/sgi-gru/gruprocfs.c
@@ -62,7 +62,9 @@ static int statistics_show(struct seq_file *s, void *p)
62 printstat(s, asid_wrap); 62 printstat(s, asid_wrap);
63 printstat(s, asid_reuse); 63 printstat(s, asid_reuse);
64 printstat(s, intr); 64 printstat(s, intr);
65 printstat(s, intr_mm_lock_failed);
65 printstat(s, call_os); 66 printstat(s, call_os);
67 printstat(s, call_os_offnode_reference);
66 printstat(s, call_os_check_for_bug); 68 printstat(s, call_os_check_for_bug);
67 printstat(s, call_os_wait_queue); 69 printstat(s, call_os_wait_queue);
68 printstat(s, user_flush_tlb); 70 printstat(s, user_flush_tlb);
@@ -120,6 +122,30 @@ static ssize_t statistics_write(struct file *file, const char __user *userbuf,
120 return count; 122 return count;
121} 123}
122 124
125static int mcs_statistics_show(struct seq_file *s, void *p)
126{
127 int op;
128 unsigned long total, count, max;
129 static char *id[] = {"cch_allocate", "cch_start", "cch_interrupt",
130 "cch_interrupt_sync", "cch_deallocate", "tgh_invalidate"};
131
132 for (op = 0; op < mcsop_last; op++) {
133 count = atomic_long_read(&mcs_op_statistics[op].count);
134 total = atomic_long_read(&mcs_op_statistics[op].total);
135 max = mcs_op_statistics[op].max;
136 seq_printf(s, "%-20s%12ld%12ld%12ld\n", id[op], count,
137 count ? total / count : 0, max);
138 }
139 return 0;
140}
141
142static ssize_t mcs_statistics_write(struct file *file,
143 const char __user *userbuf, size_t count, loff_t *data)
144{
145 memset(mcs_op_statistics, 0, sizeof(mcs_op_statistics));
146 return count;
147}
148
123static int options_show(struct seq_file *s, void *p) 149static int options_show(struct seq_file *s, void *p)
124{ 150{
125 seq_printf(s, "0x%lx\n", gru_options); 151 seq_printf(s, "0x%lx\n", gru_options);
@@ -135,6 +161,7 @@ static ssize_t options_write(struct file *file, const char __user *userbuf,
135 if (copy_from_user 161 if (copy_from_user
136 (buf, userbuf, count < sizeof(buf) ? count : sizeof(buf))) 162 (buf, userbuf, count < sizeof(buf) ? count : sizeof(buf)))
137 return -EFAULT; 163 return -EFAULT;
164 buf[count - 1] = '\0';
138 if (!strict_strtoul(buf, 10, &val)) 165 if (!strict_strtoul(buf, 10, &val))
139 gru_options = val; 166 gru_options = val;
140 167
@@ -199,7 +226,7 @@ static void seq_stop(struct seq_file *file, void *data)
199 226
200static void *seq_start(struct seq_file *file, loff_t *gid) 227static void *seq_start(struct seq_file *file, loff_t *gid)
201{ 228{
202 if (*gid < GRU_MAX_GRUS) 229 if (*gid < gru_max_gids)
203 return gid; 230 return gid;
204 return NULL; 231 return NULL;
205} 232}
@@ -207,7 +234,7 @@ static void *seq_start(struct seq_file *file, loff_t *gid)
207static void *seq_next(struct seq_file *file, void *data, loff_t *gid) 234static void *seq_next(struct seq_file *file, void *data, loff_t *gid)
208{ 235{
209 (*gid)++; 236 (*gid)++;
210 if (*gid < GRU_MAX_GRUS) 237 if (*gid < gru_max_gids)
211 return gid; 238 return gid;
212 return NULL; 239 return NULL;
213} 240}
@@ -231,6 +258,11 @@ static int statistics_open(struct inode *inode, struct file *file)
231 return single_open(file, statistics_show, NULL); 258 return single_open(file, statistics_show, NULL);
232} 259}
233 260
261static int mcs_statistics_open(struct inode *inode, struct file *file)
262{
263 return single_open(file, mcs_statistics_show, NULL);
264}
265
234static int options_open(struct inode *inode, struct file *file) 266static int options_open(struct inode *inode, struct file *file)
235{ 267{
236 return single_open(file, options_show, NULL); 268 return single_open(file, options_show, NULL);
@@ -255,6 +287,14 @@ static const struct file_operations statistics_fops = {
255 .release = single_release, 287 .release = single_release,
256}; 288};
257 289
290static const struct file_operations mcs_statistics_fops = {
291 .open = mcs_statistics_open,
292 .read = seq_read,
293 .write = mcs_statistics_write,
294 .llseek = seq_lseek,
295 .release = single_release,
296};
297
258static const struct file_operations options_fops = { 298static const struct file_operations options_fops = {
259 .open = options_open, 299 .open = options_open,
260 .read = seq_read, 300 .read = seq_read,
@@ -283,6 +323,7 @@ static struct proc_entry {
283 struct proc_dir_entry *entry; 323 struct proc_dir_entry *entry;
284} proc_files[] = { 324} proc_files[] = {
285 {"statistics", 0644, &statistics_fops}, 325 {"statistics", 0644, &statistics_fops},
326 {"mcs_statistics", 0644, &mcs_statistics_fops},
286 {"debug_options", 0644, &options_fops}, 327 {"debug_options", 0644, &options_fops},
287 {"cch_status", 0444, &cch_fops}, 328 {"cch_status", 0444, &cch_fops},
288 {"gru_status", 0444, &gru_fops}, 329 {"gru_status", 0444, &gru_fops},
diff --git a/drivers/misc/sgi-gru/grutables.h b/drivers/misc/sgi-gru/grutables.h
index a78f70deeb59..bf1eeb7553ed 100644
--- a/drivers/misc/sgi-gru/grutables.h
+++ b/drivers/misc/sgi-gru/grutables.h
@@ -153,6 +153,7 @@
153extern struct gru_stats_s gru_stats; 153extern struct gru_stats_s gru_stats;
154extern struct gru_blade_state *gru_base[]; 154extern struct gru_blade_state *gru_base[];
155extern unsigned long gru_start_paddr, gru_end_paddr; 155extern unsigned long gru_start_paddr, gru_end_paddr;
156extern unsigned int gru_max_gids;
156 157
157#define GRU_MAX_BLADES MAX_NUMNODES 158#define GRU_MAX_BLADES MAX_NUMNODES
158#define GRU_MAX_GRUS (GRU_MAX_BLADES * GRU_CHIPLETS_PER_BLADE) 159#define GRU_MAX_GRUS (GRU_MAX_BLADES * GRU_CHIPLETS_PER_BLADE)
@@ -184,7 +185,9 @@ struct gru_stats_s {
184 atomic_long_t asid_wrap; 185 atomic_long_t asid_wrap;
185 atomic_long_t asid_reuse; 186 atomic_long_t asid_reuse;
186 atomic_long_t intr; 187 atomic_long_t intr;
188 atomic_long_t intr_mm_lock_failed;
187 atomic_long_t call_os; 189 atomic_long_t call_os;
190 atomic_long_t call_os_offnode_reference;
188 atomic_long_t call_os_check_for_bug; 191 atomic_long_t call_os_check_for_bug;
189 atomic_long_t call_os_wait_queue; 192 atomic_long_t call_os_wait_queue;
190 atomic_long_t user_flush_tlb; 193 atomic_long_t user_flush_tlb;
@@ -237,6 +240,17 @@ struct gru_stats_s {
237 240
238}; 241};
239 242
243enum mcs_op {cchop_allocate, cchop_start, cchop_interrupt, cchop_interrupt_sync,
244 cchop_deallocate, tghop_invalidate, mcsop_last};
245
246struct mcs_op_statistic {
247 atomic_long_t count;
248 atomic_long_t total;
249 unsigned long max;
250};
251
252extern struct mcs_op_statistic mcs_op_statistics[mcsop_last];
253
240#define OPT_DPRINT 1 254#define OPT_DPRINT 1
241#define OPT_STATS 2 255#define OPT_STATS 2
242#define GRU_QUICKLOOK 4 256#define GRU_QUICKLOOK 4
@@ -278,13 +292,12 @@ struct gru_stats_s {
278/* Generate a GRU asid value from a GRU base asid & a virtual address. */ 292/* Generate a GRU asid value from a GRU base asid & a virtual address. */
279#if defined CONFIG_IA64 293#if defined CONFIG_IA64
280#define VADDR_HI_BIT 64 294#define VADDR_HI_BIT 64
281#define GRUREGION(addr) ((addr) >> (VADDR_HI_BIT - 3) & 3)
282#elif defined CONFIG_X86_64 295#elif defined CONFIG_X86_64
283#define VADDR_HI_BIT 48 296#define VADDR_HI_BIT 48
284#define GRUREGION(addr) (0) /* ZZZ could do better */
285#else 297#else
286#error "Unsupported architecture" 298#error "Unsupported architecture"
287#endif 299#endif
300#define GRUREGION(addr) ((addr) >> (VADDR_HI_BIT - 3) & 3)
288#define GRUASID(asid, addr) ((asid) + GRUREGION(addr)) 301#define GRUASID(asid, addr) ((asid) + GRUREGION(addr))
289 302
290/*------------------------------------------------------------------------------ 303/*------------------------------------------------------------------------------
@@ -297,12 +310,12 @@ struct gru_state;
297 * This structure is pointed to from the mmstruct via the notifier pointer. 310 * This structure is pointed to from the mmstruct via the notifier pointer.
298 * There is one of these per address space. 311 * There is one of these per address space.
299 */ 312 */
300struct gru_mm_tracker { 313struct gru_mm_tracker { /* pack to reduce size */
301 unsigned int mt_asid_gen; /* ASID wrap count */ 314 unsigned int mt_asid_gen:24; /* ASID wrap count */
302 int mt_asid; /* current base ASID for gru */ 315 unsigned int mt_asid:24; /* current base ASID for gru */
303 unsigned short mt_ctxbitmap; /* bitmap of contexts using 316 unsigned short mt_ctxbitmap:16;/* bitmap of contexts using
304 asid */ 317 asid */
305}; 318} __attribute__ ((packed));
306 319
307struct gru_mm_struct { 320struct gru_mm_struct {
308 struct mmu_notifier ms_notifier; 321 struct mmu_notifier ms_notifier;
@@ -348,6 +361,7 @@ struct gru_thread_state {
348 long ts_user_options;/* misc user option flags */ 361 long ts_user_options;/* misc user option flags */
349 pid_t ts_tgid_owner; /* task that is using the 362 pid_t ts_tgid_owner; /* task that is using the
350 context - for migration */ 363 context - for migration */
364 unsigned short ts_sizeavail; /* Pagesizes in use */
351 int ts_tsid; /* thread that owns the 365 int ts_tsid; /* thread that owns the
352 structure */ 366 structure */
353 int ts_tlb_int_select;/* target cpu if interrupts 367 int ts_tlb_int_select;/* target cpu if interrupts
@@ -359,6 +373,9 @@ struct gru_thread_state {
359 required for contest */ 373 required for contest */
360 unsigned char ts_cbr_au_count;/* Number of CBR resources 374 unsigned char ts_cbr_au_count;/* Number of CBR resources
361 required for contest */ 375 required for contest */
376 char ts_blade; /* If >= 0, migrate context if
377 ref from diferent blade */
378 char ts_force_cch_reload;
362 char ts_force_unload;/* force context to be unloaded 379 char ts_force_unload;/* force context to be unloaded
363 after migration */ 380 after migration */
364 char ts_cbr_idx[GRU_CBR_AU];/* CBR numbers of each 381 char ts_cbr_idx[GRU_CBR_AU];/* CBR numbers of each
@@ -392,12 +409,12 @@ struct gru_state {
392 gru segments (64) */ 409 gru segments (64) */
393 void *gs_gru_base_vaddr; /* Virtual address of 410 void *gs_gru_base_vaddr; /* Virtual address of
394 gru segments (64) */ 411 gru segments (64) */
395 unsigned char gs_gid; /* unique GRU number */ 412 unsigned short gs_gid; /* unique GRU number */
413 unsigned short gs_blade_id; /* blade of GRU */
396 unsigned char gs_tgh_local_shift; /* used to pick TGH for 414 unsigned char gs_tgh_local_shift; /* used to pick TGH for
397 local flush */ 415 local flush */
398 unsigned char gs_tgh_first_remote; /* starting TGH# for 416 unsigned char gs_tgh_first_remote; /* starting TGH# for
399 remote flush */ 417 remote flush */
400 unsigned short gs_blade_id; /* blade of GRU */
401 spinlock_t gs_asid_lock; /* lock used for 418 spinlock_t gs_asid_lock; /* lock used for
402 assigning asids */ 419 assigning asids */
403 spinlock_t gs_lock; /* lock used for 420 spinlock_t gs_lock; /* lock used for
@@ -492,6 +509,10 @@ struct gru_blade_state {
492 (i) < GRU_CHIPLETS_PER_BLADE; \ 509 (i) < GRU_CHIPLETS_PER_BLADE; \
493 (i)++, (gru)++) 510 (i)++, (gru)++)
494 511
512/* Scan all GRUs */
513#define foreach_gid(gid) \
514 for ((gid) = 0; (gid) < gru_max_gids; (gid)++)
515
495/* Scan all active GTSs on a gru. Note: must hold ss_lock to use this macro. */ 516/* Scan all active GTSs on a gru. Note: must hold ss_lock to use this macro. */
496#define for_each_gts_on_gru(gts, gru, ctxnum) \ 517#define for_each_gts_on_gru(gts, gru, ctxnum) \
497 for ((ctxnum) = 0; (ctxnum) < GRU_NUM_CCH; (ctxnum)++) \ 518 for ((ctxnum) = 0; (ctxnum) < GRU_NUM_CCH; (ctxnum)++) \
@@ -578,9 +599,11 @@ extern struct gru_thread_state *gru_find_thread_state(struct vm_area_struct
578extern struct gru_thread_state *gru_alloc_thread_state(struct vm_area_struct 599extern struct gru_thread_state *gru_alloc_thread_state(struct vm_area_struct
579 *vma, int tsid); 600 *vma, int tsid);
580extern void gru_unload_context(struct gru_thread_state *gts, int savestate); 601extern void gru_unload_context(struct gru_thread_state *gts, int savestate);
602extern int gru_update_cch(struct gru_thread_state *gts, int force_unload);
581extern void gts_drop(struct gru_thread_state *gts); 603extern void gts_drop(struct gru_thread_state *gts);
582extern void gru_tgh_flush_init(struct gru_state *gru); 604extern void gru_tgh_flush_init(struct gru_state *gru);
583extern int gru_kservices_init(struct gru_state *gru); 605extern int gru_kservices_init(struct gru_state *gru);
606extern void gru_kservices_exit(struct gru_state *gru);
584extern irqreturn_t gru_intr(int irq, void *dev_id); 607extern irqreturn_t gru_intr(int irq, void *dev_id);
585extern int gru_handle_user_call_os(unsigned long address); 608extern int gru_handle_user_call_os(unsigned long address);
586extern int gru_user_flush_tlb(unsigned long arg); 609extern int gru_user_flush_tlb(unsigned long arg);
diff --git a/drivers/misc/sgi-gru/grutlbpurge.c b/drivers/misc/sgi-gru/grutlbpurge.c
index c84496a77691..1d125091f5e7 100644
--- a/drivers/misc/sgi-gru/grutlbpurge.c
+++ b/drivers/misc/sgi-gru/grutlbpurge.c
@@ -187,7 +187,7 @@ void gru_flush_tlb_range(struct gru_mm_struct *gms, unsigned long start,
187 " FLUSH gruid %d, asid 0x%x, num %ld, cbmap 0x%x\n", 187 " FLUSH gruid %d, asid 0x%x, num %ld, cbmap 0x%x\n",
188 gid, asid, num, asids->mt_ctxbitmap); 188 gid, asid, num, asids->mt_ctxbitmap);
189 tgh = get_lock_tgh_handle(gru); 189 tgh = get_lock_tgh_handle(gru);
190 tgh_invalidate(tgh, start, 0, asid, grupagesize, 0, 190 tgh_invalidate(tgh, start, ~0, asid, grupagesize, 0,
191 num - 1, asids->mt_ctxbitmap); 191 num - 1, asids->mt_ctxbitmap);
192 get_unlock_tgh_handle(tgh); 192 get_unlock_tgh_handle(tgh);
193 } else { 193 } else {
@@ -210,11 +210,10 @@ void gru_flush_all_tlb(struct gru_state *gru)
210{ 210{
211 struct gru_tlb_global_handle *tgh; 211 struct gru_tlb_global_handle *tgh;
212 212
213 gru_dbg(grudev, "gru %p, gid %d\n", gru, gru->gs_gid); 213 gru_dbg(grudev, "gid %d\n", gru->gs_gid);
214 tgh = get_lock_tgh_handle(gru); 214 tgh = get_lock_tgh_handle(gru);
215 tgh_invalidate(tgh, 0, ~0, 0, 1, 1, GRUMAXINVAL - 1, 0); 215 tgh_invalidate(tgh, 0, ~0, 0, 1, 1, GRUMAXINVAL - 1, 0xffff);
216 get_unlock_tgh_handle(tgh); 216 get_unlock_tgh_handle(tgh);
217 preempt_enable();
218} 217}
219 218
220/* 219/*
diff --git a/drivers/misc/sgi-xp/xpc.h b/drivers/misc/sgi-xp/xpc.h
index 275b78896a73..114444cfd496 100644
--- a/drivers/misc/sgi-xp/xpc.h
+++ b/drivers/misc/sgi-xp/xpc.h
@@ -92,7 +92,9 @@ struct xpc_rsvd_page {
92 u8 pad1[3]; /* align to next u64 in 1st 64-byte cacheline */ 92 u8 pad1[3]; /* align to next u64 in 1st 64-byte cacheline */
93 union { 93 union {
94 unsigned long vars_pa; /* phys address of struct xpc_vars */ 94 unsigned long vars_pa; /* phys address of struct xpc_vars */
95 unsigned long activate_mq_gpa; /* gru phy addr of activate_mq */ 95 unsigned long activate_gru_mq_desc_gpa; /* phys addr of */
96 /* activate mq's */
97 /* gru mq descriptor */
96 } sn; 98 } sn;
97 unsigned long ts_jiffies; /* timestamp when rsvd pg was setup by XPC */ 99 unsigned long ts_jiffies; /* timestamp when rsvd pg was setup by XPC */
98 u64 pad2[10]; /* align to last u64 in 2nd 64-byte cacheline */ 100 u64 pad2[10]; /* align to last u64 in 2nd 64-byte cacheline */
@@ -189,7 +191,9 @@ struct xpc_gru_mq_uv {
189 int irq; /* irq raised when message is received in mq */ 191 int irq; /* irq raised when message is received in mq */
190 int mmr_blade; /* blade where watchlist was allocated from */ 192 int mmr_blade; /* blade where watchlist was allocated from */
191 unsigned long mmr_offset; /* offset of irq mmr located on mmr_blade */ 193 unsigned long mmr_offset; /* offset of irq mmr located on mmr_blade */
194 unsigned long mmr_value; /* value of irq mmr located on mmr_blade */
192 int watchlist_num; /* number of watchlist allocatd by BIOS */ 195 int watchlist_num; /* number of watchlist allocatd by BIOS */
196 void *gru_mq_desc; /* opaque structure used by the GRU driver */
193}; 197};
194 198
195/* 199/*
@@ -197,6 +201,7 @@ struct xpc_gru_mq_uv {
197 * heartbeat, partition active state, and channel state. This is UV only. 201 * heartbeat, partition active state, and channel state. This is UV only.
198 */ 202 */
199struct xpc_activate_mq_msghdr_uv { 203struct xpc_activate_mq_msghdr_uv {
204 unsigned int gru_msg_hdr; /* FOR GRU INTERNAL USE ONLY */
200 short partid; /* sender's partid */ 205 short partid; /* sender's partid */
201 u8 act_state; /* sender's act_state at time msg sent */ 206 u8 act_state; /* sender's act_state at time msg sent */
202 u8 type; /* message's type */ 207 u8 type; /* message's type */
@@ -232,7 +237,7 @@ struct xpc_activate_mq_msg_heartbeat_req_uv {
232struct xpc_activate_mq_msg_activate_req_uv { 237struct xpc_activate_mq_msg_activate_req_uv {
233 struct xpc_activate_mq_msghdr_uv hdr; 238 struct xpc_activate_mq_msghdr_uv hdr;
234 unsigned long rp_gpa; 239 unsigned long rp_gpa;
235 unsigned long activate_mq_gpa; 240 unsigned long activate_gru_mq_desc_gpa;
236}; 241};
237 242
238struct xpc_activate_mq_msg_deactivate_req_uv { 243struct xpc_activate_mq_msg_deactivate_req_uv {
@@ -263,7 +268,7 @@ struct xpc_activate_mq_msg_chctl_openreply_uv {
263 short ch_number; 268 short ch_number;
264 short remote_nentries; /* ??? Is this needed? What is? */ 269 short remote_nentries; /* ??? Is this needed? What is? */
265 short local_nentries; /* ??? Is this needed? What is? */ 270 short local_nentries; /* ??? Is this needed? What is? */
266 unsigned long local_notify_mq_gpa; 271 unsigned long notify_gru_mq_desc_gpa;
267}; 272};
268 273
269/* 274/*
@@ -510,8 +515,8 @@ struct xpc_channel_sn2 {
510}; 515};
511 516
512struct xpc_channel_uv { 517struct xpc_channel_uv {
513 unsigned long remote_notify_mq_gpa; /* gru phys address of remote */ 518 void *cached_notify_gru_mq_desc; /* remote partition's notify mq's */
514 /* partition's notify mq */ 519 /* gru mq descriptor */
515 520
516 struct xpc_send_msg_slot_uv *send_msg_slots; 521 struct xpc_send_msg_slot_uv *send_msg_slots;
517 void *recv_msg_slots; /* each slot will hold a xpc_notify_mq_msg_uv */ 522 void *recv_msg_slots; /* each slot will hold a xpc_notify_mq_msg_uv */
@@ -682,8 +687,12 @@ struct xpc_partition_sn2 {
682}; 687};
683 688
684struct xpc_partition_uv { 689struct xpc_partition_uv {
685 unsigned long remote_activate_mq_gpa; /* gru phys address of remote */ 690 unsigned long activate_gru_mq_desc_gpa; /* phys addr of parititon's */
686 /* partition's activate mq */ 691 /* activate mq's gru mq */
692 /* descriptor */
693 void *cached_activate_gru_mq_desc; /* cached copy of partition's */
694 /* activate mq's gru mq descriptor */
695 struct mutex cached_activate_gru_mq_desc_mutex;
687 spinlock_t flags_lock; /* protect updating of flags */ 696 spinlock_t flags_lock; /* protect updating of flags */
688 unsigned int flags; /* general flags */ 697 unsigned int flags; /* general flags */
689 u8 remote_act_state; /* remote partition's act_state */ 698 u8 remote_act_state; /* remote partition's act_state */
@@ -694,8 +703,9 @@ struct xpc_partition_uv {
694 703
695/* struct xpc_partition_uv flags */ 704/* struct xpc_partition_uv flags */
696 705
697#define XPC_P_HEARTBEAT_OFFLINE_UV 0x00000001 706#define XPC_P_HEARTBEAT_OFFLINE_UV 0x00000001
698#define XPC_P_ENGAGED_UV 0x00000002 707#define XPC_P_ENGAGED_UV 0x00000002
708#define XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV 0x00000004
699 709
700/* struct xpc_partition_uv act_state change requests */ 710/* struct xpc_partition_uv act_state change requests */
701 711
@@ -804,6 +814,7 @@ extern void xpc_activate_kthreads(struct xpc_channel *, int);
804extern void xpc_create_kthreads(struct xpc_channel *, int, int); 814extern void xpc_create_kthreads(struct xpc_channel *, int, int);
805extern void xpc_disconnect_wait(int); 815extern void xpc_disconnect_wait(int);
806extern int (*xpc_setup_partitions_sn) (void); 816extern int (*xpc_setup_partitions_sn) (void);
817extern void (*xpc_teardown_partitions_sn) (void);
807extern enum xp_retval (*xpc_get_partition_rsvd_page_pa) (void *, u64 *, 818extern enum xp_retval (*xpc_get_partition_rsvd_page_pa) (void *, u64 *,
808 unsigned long *, 819 unsigned long *,
809 size_t *); 820 size_t *);
@@ -846,8 +857,8 @@ extern void (*xpc_send_chctl_openrequest) (struct xpc_channel *,
846 unsigned long *); 857 unsigned long *);
847extern void (*xpc_send_chctl_openreply) (struct xpc_channel *, unsigned long *); 858extern void (*xpc_send_chctl_openreply) (struct xpc_channel *, unsigned long *);
848 859
849extern void (*xpc_save_remote_msgqueue_pa) (struct xpc_channel *, 860extern enum xp_retval (*xpc_save_remote_msgqueue_pa) (struct xpc_channel *,
850 unsigned long); 861 unsigned long);
851 862
852extern enum xp_retval (*xpc_send_payload) (struct xpc_channel *, u32, void *, 863extern enum xp_retval (*xpc_send_payload) (struct xpc_channel *, u32, void *,
853 u16, u8, xpc_notify_func, void *); 864 u16, u8, xpc_notify_func, void *);
diff --git a/drivers/misc/sgi-xp/xpc_channel.c b/drivers/misc/sgi-xp/xpc_channel.c
index 45fd653dbe31..99a2534c38a1 100644
--- a/drivers/misc/sgi-xp/xpc_channel.c
+++ b/drivers/misc/sgi-xp/xpc_channel.c
@@ -183,6 +183,7 @@ xpc_process_openclose_chctl_flags(struct xpc_partition *part, int ch_number,
183 &part->remote_openclose_args[ch_number]; 183 &part->remote_openclose_args[ch_number];
184 struct xpc_channel *ch = &part->channels[ch_number]; 184 struct xpc_channel *ch = &part->channels[ch_number];
185 enum xp_retval reason; 185 enum xp_retval reason;
186 enum xp_retval ret;
186 187
187 spin_lock_irqsave(&ch->lock, irq_flags); 188 spin_lock_irqsave(&ch->lock, irq_flags);
188 189
@@ -399,8 +400,13 @@ again:
399 DBUG_ON(args->local_nentries == 0); 400 DBUG_ON(args->local_nentries == 0);
400 DBUG_ON(args->remote_nentries == 0); 401 DBUG_ON(args->remote_nentries == 0);
401 402
403 ret = xpc_save_remote_msgqueue_pa(ch, args->local_msgqueue_pa);
404 if (ret != xpSuccess) {
405 XPC_DISCONNECT_CHANNEL(ch, ret, &irq_flags);
406 spin_unlock_irqrestore(&ch->lock, irq_flags);
407 return;
408 }
402 ch->flags |= XPC_C_ROPENREPLY; 409 ch->flags |= XPC_C_ROPENREPLY;
403 xpc_save_remote_msgqueue_pa(ch, args->local_msgqueue_pa);
404 410
405 if (args->local_nentries < ch->remote_nentries) { 411 if (args->local_nentries < ch->remote_nentries) {
406 dev_dbg(xpc_chan, "XPC_CHCTL_OPENREPLY: new " 412 dev_dbg(xpc_chan, "XPC_CHCTL_OPENREPLY: new "
diff --git a/drivers/misc/sgi-xp/xpc_main.c b/drivers/misc/sgi-xp/xpc_main.c
index 6576170de962..1ab9fda87fab 100644
--- a/drivers/misc/sgi-xp/xpc_main.c
+++ b/drivers/misc/sgi-xp/xpc_main.c
@@ -171,6 +171,7 @@ static struct notifier_block xpc_die_notifier = {
171}; 171};
172 172
173int (*xpc_setup_partitions_sn) (void); 173int (*xpc_setup_partitions_sn) (void);
174void (*xpc_teardown_partitions_sn) (void);
174enum xp_retval (*xpc_get_partition_rsvd_page_pa) (void *buf, u64 *cookie, 175enum xp_retval (*xpc_get_partition_rsvd_page_pa) (void *buf, u64 *cookie,
175 unsigned long *rp_pa, 176 unsigned long *rp_pa,
176 size_t *len); 177 size_t *len);
@@ -217,8 +218,8 @@ void (*xpc_send_chctl_openrequest) (struct xpc_channel *ch,
217void (*xpc_send_chctl_openreply) (struct xpc_channel *ch, 218void (*xpc_send_chctl_openreply) (struct xpc_channel *ch,
218 unsigned long *irq_flags); 219 unsigned long *irq_flags);
219 220
220void (*xpc_save_remote_msgqueue_pa) (struct xpc_channel *ch, 221enum xp_retval (*xpc_save_remote_msgqueue_pa) (struct xpc_channel *ch,
221 unsigned long msgqueue_pa); 222 unsigned long msgqueue_pa);
222 223
223enum xp_retval (*xpc_send_payload) (struct xpc_channel *ch, u32 flags, 224enum xp_retval (*xpc_send_payload) (struct xpc_channel *ch, u32 flags,
224 void *payload, u16 payload_size, 225 void *payload, u16 payload_size,
@@ -998,6 +999,7 @@ xpc_setup_partitions(void)
998static void 999static void
999xpc_teardown_partitions(void) 1000xpc_teardown_partitions(void)
1000{ 1001{
1002 xpc_teardown_partitions_sn();
1001 kfree(xpc_partitions); 1003 kfree(xpc_partitions);
1002} 1004}
1003 1005
diff --git a/drivers/misc/sgi-xp/xpc_sn2.c b/drivers/misc/sgi-xp/xpc_sn2.c
index 2e975762c32b..eaaa964942de 100644
--- a/drivers/misc/sgi-xp/xpc_sn2.c
+++ b/drivers/misc/sgi-xp/xpc_sn2.c
@@ -66,6 +66,12 @@ xpc_setup_partitions_sn_sn2(void)
66 return 0; 66 return 0;
67} 67}
68 68
69static void
70xpc_teardown_partitions_sn_sn2(void)
71{
72 /* nothing needs to be done */
73}
74
69/* SH_IPI_ACCESS shub register value on startup */ 75/* SH_IPI_ACCESS shub register value on startup */
70static u64 xpc_sh1_IPI_access_sn2; 76static u64 xpc_sh1_IPI_access_sn2;
71static u64 xpc_sh2_IPI_access0_sn2; 77static u64 xpc_sh2_IPI_access0_sn2;
@@ -436,11 +442,12 @@ xpc_send_chctl_local_msgrequest_sn2(struct xpc_channel *ch)
436 XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST); 442 XPC_SEND_LOCAL_NOTIFY_IRQ_SN2(ch, XPC_CHCTL_MSGREQUEST);
437} 443}
438 444
439static void 445static enum xp_retval
440xpc_save_remote_msgqueue_pa_sn2(struct xpc_channel *ch, 446xpc_save_remote_msgqueue_pa_sn2(struct xpc_channel *ch,
441 unsigned long msgqueue_pa) 447 unsigned long msgqueue_pa)
442{ 448{
443 ch->sn.sn2.remote_msgqueue_pa = msgqueue_pa; 449 ch->sn.sn2.remote_msgqueue_pa = msgqueue_pa;
450 return xpSuccess;
444} 451}
445 452
446/* 453/*
@@ -1737,20 +1744,20 @@ xpc_clear_remote_msgqueue_flags_sn2(struct xpc_channel *ch)
1737{ 1744{
1738 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2; 1745 struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
1739 struct xpc_msg_sn2 *msg; 1746 struct xpc_msg_sn2 *msg;
1740 s64 put; 1747 s64 put, remote_nentries = ch->remote_nentries;
1741 1748
1742 /* flags are zeroed when the buffer is allocated */ 1749 /* flags are zeroed when the buffer is allocated */
1743 if (ch_sn2->remote_GP.put < ch->remote_nentries) 1750 if (ch_sn2->remote_GP.put < remote_nentries)
1744 return; 1751 return;
1745 1752
1746 put = max(ch_sn2->w_remote_GP.put, ch->remote_nentries); 1753 put = max(ch_sn2->w_remote_GP.put, remote_nentries);
1747 do { 1754 do {
1748 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue + 1755 msg = (struct xpc_msg_sn2 *)((u64)ch_sn2->remote_msgqueue +
1749 (put % ch->remote_nentries) * 1756 (put % remote_nentries) *
1750 ch->entry_size); 1757 ch->entry_size);
1751 DBUG_ON(!(msg->flags & XPC_M_SN2_READY)); 1758 DBUG_ON(!(msg->flags & XPC_M_SN2_READY));
1752 DBUG_ON(!(msg->flags & XPC_M_SN2_DONE)); 1759 DBUG_ON(!(msg->flags & XPC_M_SN2_DONE));
1753 DBUG_ON(msg->number != put - ch->remote_nentries); 1760 DBUG_ON(msg->number != put - remote_nentries);
1754 msg->flags = 0; 1761 msg->flags = 0;
1755 } while (++put < ch_sn2->remote_GP.put); 1762 } while (++put < ch_sn2->remote_GP.put);
1756} 1763}
@@ -2315,6 +2322,7 @@ xpc_init_sn2(void)
2315 size_t buf_size; 2322 size_t buf_size;
2316 2323
2317 xpc_setup_partitions_sn = xpc_setup_partitions_sn_sn2; 2324 xpc_setup_partitions_sn = xpc_setup_partitions_sn_sn2;
2325 xpc_teardown_partitions_sn = xpc_teardown_partitions_sn_sn2;
2318 xpc_get_partition_rsvd_page_pa = xpc_get_partition_rsvd_page_pa_sn2; 2326 xpc_get_partition_rsvd_page_pa = xpc_get_partition_rsvd_page_pa_sn2;
2319 xpc_setup_rsvd_page_sn = xpc_setup_rsvd_page_sn_sn2; 2327 xpc_setup_rsvd_page_sn = xpc_setup_rsvd_page_sn_sn2;
2320 xpc_increment_heartbeat = xpc_increment_heartbeat_sn2; 2328 xpc_increment_heartbeat = xpc_increment_heartbeat_sn2;
diff --git a/drivers/misc/sgi-xp/xpc_uv.c b/drivers/misc/sgi-xp/xpc_uv.c
index 29c0502a96b2..f7fff4727edb 100644
--- a/drivers/misc/sgi-xp/xpc_uv.c
+++ b/drivers/misc/sgi-xp/xpc_uv.c
@@ -31,6 +31,21 @@
31#include "../sgi-gru/grukservices.h" 31#include "../sgi-gru/grukservices.h"
32#include "xpc.h" 32#include "xpc.h"
33 33
34#if defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV
35struct uv_IO_APIC_route_entry {
36 __u64 vector : 8,
37 delivery_mode : 3,
38 dest_mode : 1,
39 delivery_status : 1,
40 polarity : 1,
41 __reserved_1 : 1,
42 trigger : 1,
43 mask : 1,
44 __reserved_2 : 15,
45 dest : 32;
46};
47#endif
48
34static atomic64_t xpc_heartbeat_uv; 49static atomic64_t xpc_heartbeat_uv;
35static DECLARE_BITMAP(xpc_heartbeating_to_mask_uv, XP_MAX_NPARTITIONS_UV); 50static DECLARE_BITMAP(xpc_heartbeating_to_mask_uv, XP_MAX_NPARTITIONS_UV);
36 51
@@ -56,26 +71,52 @@ xpc_setup_partitions_sn_uv(void)
56 for (partid = 0; partid < XP_MAX_NPARTITIONS_UV; partid++) { 71 for (partid = 0; partid < XP_MAX_NPARTITIONS_UV; partid++) {
57 part_uv = &xpc_partitions[partid].sn.uv; 72 part_uv = &xpc_partitions[partid].sn.uv;
58 73
74 mutex_init(&part_uv->cached_activate_gru_mq_desc_mutex);
59 spin_lock_init(&part_uv->flags_lock); 75 spin_lock_init(&part_uv->flags_lock);
60 part_uv->remote_act_state = XPC_P_AS_INACTIVE; 76 part_uv->remote_act_state = XPC_P_AS_INACTIVE;
61 } 77 }
62 return 0; 78 return 0;
63} 79}
64 80
81static void
82xpc_teardown_partitions_sn_uv(void)
83{
84 short partid;
85 struct xpc_partition_uv *part_uv;
86 unsigned long irq_flags;
87
88 for (partid = 0; partid < XP_MAX_NPARTITIONS_UV; partid++) {
89 part_uv = &xpc_partitions[partid].sn.uv;
90
91 if (part_uv->cached_activate_gru_mq_desc != NULL) {
92 mutex_lock(&part_uv->cached_activate_gru_mq_desc_mutex);
93 spin_lock_irqsave(&part_uv->flags_lock, irq_flags);
94 part_uv->flags &= ~XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV;
95 spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags);
96 kfree(part_uv->cached_activate_gru_mq_desc);
97 part_uv->cached_activate_gru_mq_desc = NULL;
98 mutex_unlock(&part_uv->
99 cached_activate_gru_mq_desc_mutex);
100 }
101 }
102}
103
65static int 104static int
66xpc_get_gru_mq_irq_uv(struct xpc_gru_mq_uv *mq, int cpu, char *irq_name) 105xpc_get_gru_mq_irq_uv(struct xpc_gru_mq_uv *mq, int cpu, char *irq_name)
67{ 106{
107 int mmr_pnode = uv_blade_to_pnode(mq->mmr_blade);
108
68#if defined CONFIG_X86_64 109#if defined CONFIG_X86_64
69 mq->irq = uv_setup_irq(irq_name, cpu, mq->mmr_blade, mq->mmr_offset); 110 mq->irq = uv_setup_irq(irq_name, cpu, mq->mmr_blade, mq->mmr_offset);
70 if (mq->irq < 0) { 111 if (mq->irq < 0) {
71 dev_err(xpc_part, "uv_setup_irq() returned error=%d\n", 112 dev_err(xpc_part, "uv_setup_irq() returned error=%d\n",
72 mq->irq); 113 -mq->irq);
114 return mq->irq;
73 } 115 }
74 116
75#elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV 117 mq->mmr_value = uv_read_global_mmr64(mmr_pnode, mq->mmr_offset);
76 int mmr_pnode;
77 unsigned long mmr_value;
78 118
119#elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV
79 if (strcmp(irq_name, XPC_ACTIVATE_IRQ_NAME) == 0) 120 if (strcmp(irq_name, XPC_ACTIVATE_IRQ_NAME) == 0)
80 mq->irq = SGI_XPC_ACTIVATE; 121 mq->irq = SGI_XPC_ACTIVATE;
81 else if (strcmp(irq_name, XPC_NOTIFY_IRQ_NAME) == 0) 122 else if (strcmp(irq_name, XPC_NOTIFY_IRQ_NAME) == 0)
@@ -83,10 +124,8 @@ xpc_get_gru_mq_irq_uv(struct xpc_gru_mq_uv *mq, int cpu, char *irq_name)
83 else 124 else
84 return -EINVAL; 125 return -EINVAL;
85 126
86 mmr_pnode = uv_blade_to_pnode(mq->mmr_blade); 127 mq->mmr_value = (unsigned long)cpu_physical_id(cpu) << 32 | mq->irq;
87 mmr_value = (unsigned long)cpu_physical_id(cpu) << 32 | mq->irq; 128 uv_write_global_mmr64(mmr_pnode, mq->mmr_offset, mq->mmr_value);
88
89 uv_write_global_mmr64(mmr_pnode, mq->mmr_offset, mmr_value);
90#else 129#else
91 #error not a supported configuration 130 #error not a supported configuration
92#endif 131#endif
@@ -127,7 +166,7 @@ xpc_gru_mq_watchlist_alloc_uv(struct xpc_gru_mq_uv *mq)
127 return ret; 166 return ret;
128 } 167 }
129#elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV 168#elif defined CONFIG_IA64_GENERIC || defined CONFIG_IA64_SGI_UV
130 ret = sn_mq_watchlist_alloc(mq->mmr_blade, uv_gpa(mq->address), 169 ret = sn_mq_watchlist_alloc(mq->mmr_blade, (void *)uv_gpa(mq->address),
131 mq->order, &mq->mmr_offset); 170 mq->order, &mq->mmr_offset);
132 if (ret < 0) { 171 if (ret < 0) {
133 dev_err(xpc_part, "sn_mq_watchlist_alloc() failed, ret=%d\n", 172 dev_err(xpc_part, "sn_mq_watchlist_alloc() failed, ret=%d\n",
@@ -168,12 +207,22 @@ xpc_create_gru_mq_uv(unsigned int mq_size, int cpu, char *irq_name,
168 int pg_order; 207 int pg_order;
169 struct page *page; 208 struct page *page;
170 struct xpc_gru_mq_uv *mq; 209 struct xpc_gru_mq_uv *mq;
210 struct uv_IO_APIC_route_entry *mmr_value;
171 211
172 mq = kmalloc(sizeof(struct xpc_gru_mq_uv), GFP_KERNEL); 212 mq = kmalloc(sizeof(struct xpc_gru_mq_uv), GFP_KERNEL);
173 if (mq == NULL) { 213 if (mq == NULL) {
174 dev_err(xpc_part, "xpc_create_gru_mq_uv() failed to kmalloc() " 214 dev_err(xpc_part, "xpc_create_gru_mq_uv() failed to kmalloc() "
175 "a xpc_gru_mq_uv structure\n"); 215 "a xpc_gru_mq_uv structure\n");
176 ret = -ENOMEM; 216 ret = -ENOMEM;
217 goto out_0;
218 }
219
220 mq->gru_mq_desc = kzalloc(sizeof(struct gru_message_queue_desc),
221 GFP_KERNEL);
222 if (mq->gru_mq_desc == NULL) {
223 dev_err(xpc_part, "xpc_create_gru_mq_uv() failed to kmalloc() "
224 "a gru_message_queue_desc structure\n");
225 ret = -ENOMEM;
177 goto out_1; 226 goto out_1;
178 } 227 }
179 228
@@ -194,14 +243,6 @@ xpc_create_gru_mq_uv(unsigned int mq_size, int cpu, char *irq_name,
194 } 243 }
195 mq->address = page_address(page); 244 mq->address = page_address(page);
196 245
197 ret = gru_create_message_queue(mq->address, mq_size);
198 if (ret != 0) {
199 dev_err(xpc_part, "gru_create_message_queue() returned "
200 "error=%d\n", ret);
201 ret = -EINVAL;
202 goto out_3;
203 }
204
205 /* enable generation of irq when GRU mq operation occurs to this mq */ 246 /* enable generation of irq when GRU mq operation occurs to this mq */
206 ret = xpc_gru_mq_watchlist_alloc_uv(mq); 247 ret = xpc_gru_mq_watchlist_alloc_uv(mq);
207 if (ret != 0) 248 if (ret != 0)
@@ -214,10 +255,20 @@ xpc_create_gru_mq_uv(unsigned int mq_size, int cpu, char *irq_name,
214 ret = request_irq(mq->irq, irq_handler, 0, irq_name, NULL); 255 ret = request_irq(mq->irq, irq_handler, 0, irq_name, NULL);
215 if (ret != 0) { 256 if (ret != 0) {
216 dev_err(xpc_part, "request_irq(irq=%d) returned error=%d\n", 257 dev_err(xpc_part, "request_irq(irq=%d) returned error=%d\n",
217 mq->irq, ret); 258 mq->irq, -ret);
218 goto out_5; 259 goto out_5;
219 } 260 }
220 261
262 mmr_value = (struct uv_IO_APIC_route_entry *)&mq->mmr_value;
263 ret = gru_create_message_queue(mq->gru_mq_desc, mq->address, mq_size,
264 nid, mmr_value->vector, mmr_value->dest);
265 if (ret != 0) {
266 dev_err(xpc_part, "gru_create_message_queue() returned "
267 "error=%d\n", ret);
268 ret = -EINVAL;
269 goto out_6;
270 }
271
221 /* allow other partitions to access this GRU mq */ 272 /* allow other partitions to access this GRU mq */
222 xp_ret = xp_expand_memprotect(xp_pa(mq->address), mq_size); 273 xp_ret = xp_expand_memprotect(xp_pa(mq->address), mq_size);
223 if (xp_ret != xpSuccess) { 274 if (xp_ret != xpSuccess) {
@@ -237,8 +288,10 @@ out_4:
237out_3: 288out_3:
238 free_pages((unsigned long)mq->address, pg_order); 289 free_pages((unsigned long)mq->address, pg_order);
239out_2: 290out_2:
240 kfree(mq); 291 kfree(mq->gru_mq_desc);
241out_1: 292out_1:
293 kfree(mq);
294out_0:
242 return ERR_PTR(ret); 295 return ERR_PTR(ret);
243} 296}
244 297
@@ -268,13 +321,14 @@ xpc_destroy_gru_mq_uv(struct xpc_gru_mq_uv *mq)
268} 321}
269 322
270static enum xp_retval 323static enum xp_retval
271xpc_send_gru_msg(unsigned long mq_gpa, void *msg, size_t msg_size) 324xpc_send_gru_msg(struct gru_message_queue_desc *gru_mq_desc, void *msg,
325 size_t msg_size)
272{ 326{
273 enum xp_retval xp_ret; 327 enum xp_retval xp_ret;
274 int ret; 328 int ret;
275 329
276 while (1) { 330 while (1) {
277 ret = gru_send_message_gpa(mq_gpa, msg, msg_size); 331 ret = gru_send_message_gpa(gru_mq_desc, msg, msg_size);
278 if (ret == MQE_OK) { 332 if (ret == MQE_OK) {
279 xp_ret = xpSuccess; 333 xp_ret = xpSuccess;
280 break; 334 break;
@@ -421,7 +475,15 @@ xpc_handle_activate_mq_msg_uv(struct xpc_partition *part,
421 part_uv->act_state_req = XPC_P_ASR_ACTIVATE_UV; 475 part_uv->act_state_req = XPC_P_ASR_ACTIVATE_UV;
422 part->remote_rp_pa = msg->rp_gpa; /* !!! _pa is _gpa */ 476 part->remote_rp_pa = msg->rp_gpa; /* !!! _pa is _gpa */
423 part->remote_rp_ts_jiffies = msg_hdr->rp_ts_jiffies; 477 part->remote_rp_ts_jiffies = msg_hdr->rp_ts_jiffies;
424 part_uv->remote_activate_mq_gpa = msg->activate_mq_gpa; 478
479 if (msg->activate_gru_mq_desc_gpa !=
480 part_uv->activate_gru_mq_desc_gpa) {
481 spin_lock_irqsave(&part_uv->flags_lock, irq_flags);
482 part_uv->flags &= ~XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV;
483 spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags);
484 part_uv->activate_gru_mq_desc_gpa =
485 msg->activate_gru_mq_desc_gpa;
486 }
425 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags); 487 spin_unlock_irqrestore(&xpc_activate_IRQ_rcvd_lock, irq_flags);
426 488
427 (*wakeup_hb_checker)++; 489 (*wakeup_hb_checker)++;
@@ -498,7 +560,7 @@ xpc_handle_activate_mq_msg_uv(struct xpc_partition *part,
498 args = &part->remote_openclose_args[msg->ch_number]; 560 args = &part->remote_openclose_args[msg->ch_number];
499 args->remote_nentries = msg->remote_nentries; 561 args->remote_nentries = msg->remote_nentries;
500 args->local_nentries = msg->local_nentries; 562 args->local_nentries = msg->local_nentries;
501 args->local_msgqueue_pa = msg->local_notify_mq_gpa; 563 args->local_msgqueue_pa = msg->notify_gru_mq_desc_gpa;
502 564
503 spin_lock_irqsave(&part->chctl_lock, irq_flags); 565 spin_lock_irqsave(&part->chctl_lock, irq_flags);
504 part->chctl.flags[msg->ch_number] |= XPC_CHCTL_OPENREPLY; 566 part->chctl.flags[msg->ch_number] |= XPC_CHCTL_OPENREPLY;
@@ -558,9 +620,10 @@ xpc_handle_activate_IRQ_uv(int irq, void *dev_id)
558 short partid; 620 short partid;
559 struct xpc_partition *part; 621 struct xpc_partition *part;
560 int wakeup_hb_checker = 0; 622 int wakeup_hb_checker = 0;
623 int part_referenced;
561 624
562 while (1) { 625 while (1) {
563 msg_hdr = gru_get_next_message(xpc_activate_mq_uv->address); 626 msg_hdr = gru_get_next_message(xpc_activate_mq_uv->gru_mq_desc);
564 if (msg_hdr == NULL) 627 if (msg_hdr == NULL)
565 break; 628 break;
566 629
@@ -571,14 +634,15 @@ xpc_handle_activate_IRQ_uv(int irq, void *dev_id)
571 partid); 634 partid);
572 } else { 635 } else {
573 part = &xpc_partitions[partid]; 636 part = &xpc_partitions[partid];
574 if (xpc_part_ref(part)) { 637
575 xpc_handle_activate_mq_msg_uv(part, msg_hdr, 638 part_referenced = xpc_part_ref(part);
576 &wakeup_hb_checker); 639 xpc_handle_activate_mq_msg_uv(part, msg_hdr,
640 &wakeup_hb_checker);
641 if (part_referenced)
577 xpc_part_deref(part); 642 xpc_part_deref(part);
578 }
579 } 643 }
580 644
581 gru_free_message(xpc_activate_mq_uv->address, msg_hdr); 645 gru_free_message(xpc_activate_mq_uv->gru_mq_desc, msg_hdr);
582 } 646 }
583 647
584 if (wakeup_hb_checker) 648 if (wakeup_hb_checker)
@@ -588,21 +652,73 @@ xpc_handle_activate_IRQ_uv(int irq, void *dev_id)
588} 652}
589 653
590static enum xp_retval 654static enum xp_retval
655xpc_cache_remote_gru_mq_desc_uv(struct gru_message_queue_desc *gru_mq_desc,
656 unsigned long gru_mq_desc_gpa)
657{
658 enum xp_retval ret;
659
660 ret = xp_remote_memcpy(uv_gpa(gru_mq_desc), gru_mq_desc_gpa,
661 sizeof(struct gru_message_queue_desc));
662 if (ret == xpSuccess)
663 gru_mq_desc->mq = NULL;
664
665 return ret;
666}
667
668static enum xp_retval
591xpc_send_activate_IRQ_uv(struct xpc_partition *part, void *msg, size_t msg_size, 669xpc_send_activate_IRQ_uv(struct xpc_partition *part, void *msg, size_t msg_size,
592 int msg_type) 670 int msg_type)
593{ 671{
594 struct xpc_activate_mq_msghdr_uv *msg_hdr = msg; 672 struct xpc_activate_mq_msghdr_uv *msg_hdr = msg;
673 struct xpc_partition_uv *part_uv = &part->sn.uv;
674 struct gru_message_queue_desc *gru_mq_desc;
675 unsigned long irq_flags;
676 enum xp_retval ret;
595 677
596 DBUG_ON(msg_size > XPC_ACTIVATE_MSG_SIZE_UV); 678 DBUG_ON(msg_size > XPC_ACTIVATE_MSG_SIZE_UV);
597 679
598 msg_hdr->type = msg_type; 680 msg_hdr->type = msg_type;
599 msg_hdr->partid = XPC_PARTID(part); 681 msg_hdr->partid = xp_partition_id;
600 msg_hdr->act_state = part->act_state; 682 msg_hdr->act_state = part->act_state;
601 msg_hdr->rp_ts_jiffies = xpc_rsvd_page->ts_jiffies; 683 msg_hdr->rp_ts_jiffies = xpc_rsvd_page->ts_jiffies;
602 684
685 mutex_lock(&part_uv->cached_activate_gru_mq_desc_mutex);
686again:
687 if (!(part_uv->flags & XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV)) {
688 gru_mq_desc = part_uv->cached_activate_gru_mq_desc;
689 if (gru_mq_desc == NULL) {
690 gru_mq_desc = kmalloc(sizeof(struct
691 gru_message_queue_desc),
692 GFP_KERNEL);
693 if (gru_mq_desc == NULL) {
694 ret = xpNoMemory;
695 goto done;
696 }
697 part_uv->cached_activate_gru_mq_desc = gru_mq_desc;
698 }
699
700 ret = xpc_cache_remote_gru_mq_desc_uv(gru_mq_desc,
701 part_uv->
702 activate_gru_mq_desc_gpa);
703 if (ret != xpSuccess)
704 goto done;
705
706 spin_lock_irqsave(&part_uv->flags_lock, irq_flags);
707 part_uv->flags |= XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV;
708 spin_unlock_irqrestore(&part_uv->flags_lock, irq_flags);
709 }
710
603 /* ??? Is holding a spin_lock (ch->lock) during this call a bad idea? */ 711 /* ??? Is holding a spin_lock (ch->lock) during this call a bad idea? */
604 return xpc_send_gru_msg(part->sn.uv.remote_activate_mq_gpa, msg, 712 ret = xpc_send_gru_msg(part_uv->cached_activate_gru_mq_desc, msg,
605 msg_size); 713 msg_size);
714 if (ret != xpSuccess) {
715 smp_rmb(); /* ensure a fresh copy of part_uv->flags */
716 if (!(part_uv->flags & XPC_P_CACHED_ACTIVATE_GRU_MQ_DESC_UV))
717 goto again;
718 }
719done:
720 mutex_unlock(&part_uv->cached_activate_gru_mq_desc_mutex);
721 return ret;
606} 722}
607 723
608static void 724static void
@@ -620,7 +736,7 @@ static void
620xpc_send_activate_IRQ_ch_uv(struct xpc_channel *ch, unsigned long *irq_flags, 736xpc_send_activate_IRQ_ch_uv(struct xpc_channel *ch, unsigned long *irq_flags,
621 void *msg, size_t msg_size, int msg_type) 737 void *msg, size_t msg_size, int msg_type)
622{ 738{
623 struct xpc_partition *part = &xpc_partitions[ch->number]; 739 struct xpc_partition *part = &xpc_partitions[ch->partid];
624 enum xp_retval ret; 740 enum xp_retval ret;
625 741
626 ret = xpc_send_activate_IRQ_uv(part, msg, msg_size, msg_type); 742 ret = xpc_send_activate_IRQ_uv(part, msg, msg_size, msg_type);
@@ -692,7 +808,8 @@ xpc_get_partition_rsvd_page_pa_uv(void *buf, u64 *cookie, unsigned long *rp_pa,
692static int 808static int
693xpc_setup_rsvd_page_sn_uv(struct xpc_rsvd_page *rp) 809xpc_setup_rsvd_page_sn_uv(struct xpc_rsvd_page *rp)
694{ 810{
695 rp->sn.activate_mq_gpa = uv_gpa(xpc_activate_mq_uv->address); 811 rp->sn.activate_gru_mq_desc_gpa =
812 uv_gpa(xpc_activate_mq_uv->gru_mq_desc);
696 return 0; 813 return 0;
697} 814}
698 815
@@ -787,7 +904,8 @@ xpc_request_partition_activation_uv(struct xpc_rsvd_page *remote_rp,
787 904
788 part->remote_rp_pa = remote_rp_gpa; /* !!! _pa here is really _gpa */ 905 part->remote_rp_pa = remote_rp_gpa; /* !!! _pa here is really _gpa */
789 part->remote_rp_ts_jiffies = remote_rp->ts_jiffies; 906 part->remote_rp_ts_jiffies = remote_rp->ts_jiffies;
790 part->sn.uv.remote_activate_mq_gpa = remote_rp->sn.activate_mq_gpa; 907 part->sn.uv.activate_gru_mq_desc_gpa =
908 remote_rp->sn.activate_gru_mq_desc_gpa;
791 909
792 /* 910 /*
793 * ??? Is it a good idea to make this conditional on what is 911 * ??? Is it a good idea to make this conditional on what is
@@ -795,7 +913,8 @@ xpc_request_partition_activation_uv(struct xpc_rsvd_page *remote_rp,
795 */ 913 */
796 if (part->sn.uv.remote_act_state == XPC_P_AS_INACTIVE) { 914 if (part->sn.uv.remote_act_state == XPC_P_AS_INACTIVE) {
797 msg.rp_gpa = uv_gpa(xpc_rsvd_page); 915 msg.rp_gpa = uv_gpa(xpc_rsvd_page);
798 msg.activate_mq_gpa = xpc_rsvd_page->sn.activate_mq_gpa; 916 msg.activate_gru_mq_desc_gpa =
917 xpc_rsvd_page->sn.activate_gru_mq_desc_gpa;
799 xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg), 918 xpc_send_activate_IRQ_part_uv(part, &msg, sizeof(msg),
800 XPC_ACTIVATE_MQ_MSG_ACTIVATE_REQ_UV); 919 XPC_ACTIVATE_MQ_MSG_ACTIVATE_REQ_UV);
801 } 920 }
@@ -857,7 +976,8 @@ xpc_get_fifo_entry_uv(struct xpc_fifo_head_uv *head)
857 if (head->first == NULL) 976 if (head->first == NULL)
858 head->last = NULL; 977 head->last = NULL;
859 } 978 }
860 head->n_entries++; 979 head->n_entries--;
980 BUG_ON(head->n_entries < 0);
861 spin_unlock_irqrestore(&head->lock, irq_flags); 981 spin_unlock_irqrestore(&head->lock, irq_flags);
862 first->next = NULL; 982 first->next = NULL;
863 return first; 983 return first;
@@ -876,8 +996,7 @@ xpc_put_fifo_entry_uv(struct xpc_fifo_head_uv *head,
876 else 996 else
877 head->first = last; 997 head->first = last;
878 head->last = last; 998 head->last = last;
879 head->n_entries--; 999 head->n_entries++;
880 BUG_ON(head->n_entries < 0);
881 spin_unlock_irqrestore(&head->lock, irq_flags); 1000 spin_unlock_irqrestore(&head->lock, irq_flags);
882} 1001}
883 1002
@@ -1037,6 +1156,12 @@ xpc_setup_msg_structures_uv(struct xpc_channel *ch)
1037 1156
1038 DBUG_ON(ch->flags & XPC_C_SETUP); 1157 DBUG_ON(ch->flags & XPC_C_SETUP);
1039 1158
1159 ch_uv->cached_notify_gru_mq_desc = kmalloc(sizeof(struct
1160 gru_message_queue_desc),
1161 GFP_KERNEL);
1162 if (ch_uv->cached_notify_gru_mq_desc == NULL)
1163 return xpNoMemory;
1164
1040 ret = xpc_allocate_send_msg_slot_uv(ch); 1165 ret = xpc_allocate_send_msg_slot_uv(ch);
1041 if (ret == xpSuccess) { 1166 if (ret == xpSuccess) {
1042 1167
@@ -1060,7 +1185,8 @@ xpc_teardown_msg_structures_uv(struct xpc_channel *ch)
1060 1185
1061 DBUG_ON(!spin_is_locked(&ch->lock)); 1186 DBUG_ON(!spin_is_locked(&ch->lock));
1062 1187
1063 ch_uv->remote_notify_mq_gpa = 0; 1188 kfree(ch_uv->cached_notify_gru_mq_desc);
1189 ch_uv->cached_notify_gru_mq_desc = NULL;
1064 1190
1065 if (ch->flags & XPC_C_SETUP) { 1191 if (ch->flags & XPC_C_SETUP) {
1066 xpc_init_fifo_uv(&ch_uv->msg_slot_free_list); 1192 xpc_init_fifo_uv(&ch_uv->msg_slot_free_list);
@@ -1111,7 +1237,7 @@ xpc_send_chctl_openreply_uv(struct xpc_channel *ch, unsigned long *irq_flags)
1111 msg.ch_number = ch->number; 1237 msg.ch_number = ch->number;
1112 msg.local_nentries = ch->local_nentries; 1238 msg.local_nentries = ch->local_nentries;
1113 msg.remote_nentries = ch->remote_nentries; 1239 msg.remote_nentries = ch->remote_nentries;
1114 msg.local_notify_mq_gpa = uv_gpa(xpc_notify_mq_uv); 1240 msg.notify_gru_mq_desc_gpa = uv_gpa(xpc_notify_mq_uv->gru_mq_desc);
1115 xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg), 1241 xpc_send_activate_IRQ_ch_uv(ch, irq_flags, &msg, sizeof(msg),
1116 XPC_ACTIVATE_MQ_MSG_CHCTL_OPENREPLY_UV); 1242 XPC_ACTIVATE_MQ_MSG_CHCTL_OPENREPLY_UV);
1117} 1243}
@@ -1128,11 +1254,15 @@ xpc_send_chctl_local_msgrequest_uv(struct xpc_partition *part, int ch_number)
1128 xpc_wakeup_channel_mgr(part); 1254 xpc_wakeup_channel_mgr(part);
1129} 1255}
1130 1256
1131static void 1257static enum xp_retval
1132xpc_save_remote_msgqueue_pa_uv(struct xpc_channel *ch, 1258xpc_save_remote_msgqueue_pa_uv(struct xpc_channel *ch,
1133 unsigned long msgqueue_pa) 1259 unsigned long gru_mq_desc_gpa)
1134{ 1260{
1135 ch->sn.uv.remote_notify_mq_gpa = msgqueue_pa; 1261 struct xpc_channel_uv *ch_uv = &ch->sn.uv;
1262
1263 DBUG_ON(ch_uv->cached_notify_gru_mq_desc == NULL);
1264 return xpc_cache_remote_gru_mq_desc_uv(ch_uv->cached_notify_gru_mq_desc,
1265 gru_mq_desc_gpa);
1136} 1266}
1137 1267
1138static void 1268static void
@@ -1339,7 +1469,8 @@ xpc_handle_notify_IRQ_uv(int irq, void *dev_id)
1339 short partid; 1469 short partid;
1340 struct xpc_partition *part; 1470 struct xpc_partition *part;
1341 1471
1342 while ((msg = gru_get_next_message(xpc_notify_mq_uv)) != NULL) { 1472 while ((msg = gru_get_next_message(xpc_notify_mq_uv->gru_mq_desc)) !=
1473 NULL) {
1343 1474
1344 partid = msg->hdr.partid; 1475 partid = msg->hdr.partid;
1345 if (partid < 0 || partid >= XP_MAX_NPARTITIONS_UV) { 1476 if (partid < 0 || partid >= XP_MAX_NPARTITIONS_UV) {
@@ -1354,7 +1485,7 @@ xpc_handle_notify_IRQ_uv(int irq, void *dev_id)
1354 } 1485 }
1355 } 1486 }
1356 1487
1357 gru_free_message(xpc_notify_mq_uv, msg); 1488 gru_free_message(xpc_notify_mq_uv->gru_mq_desc, msg);
1358 } 1489 }
1359 1490
1360 return IRQ_HANDLED; 1491 return IRQ_HANDLED;
@@ -1438,7 +1569,8 @@ xpc_send_payload_uv(struct xpc_channel *ch, u32 flags, void *payload,
1438 msg->hdr.msg_slot_number = msg_slot->msg_slot_number; 1569 msg->hdr.msg_slot_number = msg_slot->msg_slot_number;
1439 memcpy(&msg->payload, payload, payload_size); 1570 memcpy(&msg->payload, payload, payload_size);
1440 1571
1441 ret = xpc_send_gru_msg(ch->sn.uv.remote_notify_mq_gpa, msg, msg_size); 1572 ret = xpc_send_gru_msg(ch->sn.uv.cached_notify_gru_mq_desc, msg,
1573 msg_size);
1442 if (ret == xpSuccess) 1574 if (ret == xpSuccess)
1443 goto out_1; 1575 goto out_1;
1444 1576
@@ -1529,7 +1661,7 @@ xpc_received_payload_uv(struct xpc_channel *ch, void *payload)
1529 msg->hdr.partid = xp_partition_id; 1661 msg->hdr.partid = xp_partition_id;
1530 msg->hdr.size = 0; /* size of zero indicates this is an ACK */ 1662 msg->hdr.size = 0; /* size of zero indicates this is an ACK */
1531 1663
1532 ret = xpc_send_gru_msg(ch->sn.uv.remote_notify_mq_gpa, msg, 1664 ret = xpc_send_gru_msg(ch->sn.uv.cached_notify_gru_mq_desc, msg,
1533 sizeof(struct xpc_notify_mq_msghdr_uv)); 1665 sizeof(struct xpc_notify_mq_msghdr_uv));
1534 if (ret != xpSuccess) 1666 if (ret != xpSuccess)
1535 XPC_DEACTIVATE_PARTITION(&xpc_partitions[ch->partid], ret); 1667 XPC_DEACTIVATE_PARTITION(&xpc_partitions[ch->partid], ret);
@@ -1541,6 +1673,7 @@ int
1541xpc_init_uv(void) 1673xpc_init_uv(void)
1542{ 1674{
1543 xpc_setup_partitions_sn = xpc_setup_partitions_sn_uv; 1675 xpc_setup_partitions_sn = xpc_setup_partitions_sn_uv;
1676 xpc_teardown_partitions_sn = xpc_teardown_partitions_sn_uv;
1544 xpc_process_activate_IRQ_rcvd = xpc_process_activate_IRQ_rcvd_uv; 1677 xpc_process_activate_IRQ_rcvd = xpc_process_activate_IRQ_rcvd_uv;
1545 xpc_get_partition_rsvd_page_pa = xpc_get_partition_rsvd_page_pa_uv; 1678 xpc_get_partition_rsvd_page_pa = xpc_get_partition_rsvd_page_pa_uv;
1546 xpc_setup_rsvd_page_sn = xpc_setup_rsvd_page_sn_uv; 1679 xpc_setup_rsvd_page_sn = xpc_setup_rsvd_page_sn_uv;
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-28 00:58:56 -0500