diff options
Diffstat (limited to 'drivers/misc/sgi-gru/grutables.h')
-rw-r--r-- | drivers/misc/sgi-gru/grutables.h | 609 |
1 files changed, 609 insertions, 0 deletions
diff --git a/drivers/misc/sgi-gru/grutables.h b/drivers/misc/sgi-gru/grutables.h new file mode 100644 index 000000000000..4251018f70ff --- /dev/null +++ b/drivers/misc/sgi-gru/grutables.h | |||
@@ -0,0 +1,609 @@ | |||
1 | /* | ||
2 | * SN Platform GRU Driver | ||
3 | * | ||
4 | * GRU DRIVER TABLES, MACROS, externs, etc | ||
5 | * | ||
6 | * Copyright (c) 2008 Silicon Graphics, Inc. All Rights Reserved. | ||
7 | * | ||
8 | * This program is free software; you can redistribute it and/or modify | ||
9 | * it under the terms of the GNU General Public License as published by | ||
10 | * the Free Software Foundation; either version 2 of the License, or | ||
11 | * (at your option) any later version. | ||
12 | * | ||
13 | * This program is distributed in the hope that it will be useful, | ||
14 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
16 | * GNU General Public License for more details. | ||
17 | * | ||
18 | * You should have received a copy of the GNU General Public License | ||
19 | * along with this program; if not, write to the Free Software | ||
20 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | ||
21 | */ | ||
22 | |||
23 | #ifndef __GRUTABLES_H__ | ||
24 | #define __GRUTABLES_H__ | ||
25 | |||
26 | /* | ||
27 | * GRU Chiplet: | ||
28 | * The GRU is a user addressible memory accelerator. It provides | ||
29 | * several forms of load, store, memset, bcopy instructions. In addition, it | ||
30 | * contains special instructions for AMOs, sending messages to message | ||
31 | * queues, etc. | ||
32 | * | ||
33 | * The GRU is an integral part of the node controller. It connects | ||
34 | * directly to the cpu socket. In its current implementation, there are 2 | ||
35 | * GRU chiplets in the node controller on each blade (~node). | ||
36 | * | ||
37 | * The entire GRU memory space is fully coherent and cacheable by the cpus. | ||
38 | * | ||
39 | * Each GRU chiplet has a physical memory map that looks like the following: | ||
40 | * | ||
41 | * +-----------------+ | ||
42 | * |/////////////////| | ||
43 | * |/////////////////| | ||
44 | * |/////////////////| | ||
45 | * |/////////////////| | ||
46 | * |/////////////////| | ||
47 | * |/////////////////| | ||
48 | * |/////////////////| | ||
49 | * |/////////////////| | ||
50 | * +-----------------+ | ||
51 | * | system control | | ||
52 | * +-----------------+ _______ +-------------+ | ||
53 | * |/////////////////| / | | | ||
54 | * |/////////////////| / | | | ||
55 | * |/////////////////| / | instructions| | ||
56 | * |/////////////////| / | | | ||
57 | * |/////////////////| / | | | ||
58 | * |/////////////////| / |-------------| | ||
59 | * |/////////////////| / | | | ||
60 | * +-----------------+ | | | ||
61 | * | context 15 | | data | | ||
62 | * +-----------------+ | | | ||
63 | * | ...... | \ | | | ||
64 | * +-----------------+ \____________ +-------------+ | ||
65 | * | context 1 | | ||
66 | * +-----------------+ | ||
67 | * | context 0 | | ||
68 | * +-----------------+ | ||
69 | * | ||
70 | * Each of the "contexts" is a chunk of memory that can be mmaped into user | ||
71 | * space. The context consists of 2 parts: | ||
72 | * | ||
73 | * - an instruction space that can be directly accessed by the user | ||
74 | * to issue GRU instructions and to check instruction status. | ||
75 | * | ||
76 | * - a data area that acts as normal RAM. | ||
77 | * | ||
78 | * User instructions contain virtual addresses of data to be accessed by the | ||
79 | * GRU. The GRU contains a TLB that is used to convert these user virtual | ||
80 | * addresses to physical addresses. | ||
81 | * | ||
82 | * The "system control" area of the GRU chiplet is used by the kernel driver | ||
83 | * to manage user contexts and to perform functions such as TLB dropin and | ||
84 | * purging. | ||
85 | * | ||
86 | * One context may be reserved for the kernel and used for cross-partition | ||
87 | * communication. The GRU will also be used to asynchronously zero out | ||
88 | * large blocks of memory (not currently implemented). | ||
89 | * | ||
90 | * | ||
91 | * Tables: | ||
92 | * | ||
93 | * VDATA-VMA Data - Holds a few parameters. Head of linked list of | ||
94 | * GTS tables for threads using the GSEG | ||
95 | * GTS - Gru Thread State - contains info for managing a GSEG context. A | ||
96 | * GTS is allocated for each thread accessing a | ||
97 | * GSEG. | ||
98 | * GTD - GRU Thread Data - contains shadow copy of GRU data when GSEG is | ||
99 | * not loaded into a GRU | ||
100 | * GMS - GRU Memory Struct - Used to manage TLB shootdowns. Tracks GRUs | ||
101 | * where a GSEG has been loaded. Similar to | ||
102 | * an mm_struct but for GRU. | ||
103 | * | ||
104 | * GS - GRU State - Used to manage the state of a GRU chiplet | ||
105 | * BS - Blade State - Used to manage state of all GRU chiplets | ||
106 | * on a blade | ||
107 | * | ||
108 | * | ||
109 | * Normal task tables for task using GRU. | ||
110 | * - 2 threads in process | ||
111 | * - 2 GSEGs open in process | ||
112 | * - GSEG1 is being used by both threads | ||
113 | * - GSEG2 is used only by thread 2 | ||
114 | * | ||
115 | * task -->| | ||
116 | * task ---+---> mm ->------ (notifier) -------+-> gms | ||
117 | * | | | ||
118 | * |--> vma -> vdata ---> gts--->| GSEG1 (thread1) | ||
119 | * | | | | ||
120 | * | +-> gts--->| GSEG1 (thread2) | ||
121 | * | | | ||
122 | * |--> vma -> vdata ---> gts--->| GSEG2 (thread2) | ||
123 | * . | ||
124 | * . | ||
125 | * | ||
126 | * GSEGs are marked DONTCOPY on fork | ||
127 | * | ||
128 | * At open | ||
129 | * file.private_data -> NULL | ||
130 | * | ||
131 | * At mmap, | ||
132 | * vma -> vdata | ||
133 | * | ||
134 | * After gseg reference | ||
135 | * vma -> vdata ->gts | ||
136 | * | ||
137 | * After fork | ||
138 | * parent | ||
139 | * vma -> vdata -> gts | ||
140 | * child | ||
141 | * (vma is not copied) | ||
142 | * | ||
143 | */ | ||
144 | |||
145 | #include <linux/rmap.h> | ||
146 | #include <linux/interrupt.h> | ||
147 | #include <linux/mutex.h> | ||
148 | #include <linux/wait.h> | ||
149 | #include <linux/mmu_notifier.h> | ||
150 | #include "gru.h" | ||
151 | #include "gruhandles.h" | ||
152 | |||
153 | extern struct gru_stats_s gru_stats; | ||
154 | extern struct gru_blade_state *gru_base[]; | ||
155 | extern unsigned long gru_start_paddr, gru_end_paddr; | ||
156 | |||
157 | #define GRU_MAX_BLADES MAX_NUMNODES | ||
158 | #define GRU_MAX_GRUS (GRU_MAX_BLADES * GRU_CHIPLETS_PER_BLADE) | ||
159 | |||
160 | #define GRU_DRIVER_ID_STR "SGI GRU Device Driver" | ||
161 | #define GRU_DRIVER_VERSION_STR "0.80" | ||
162 | |||
163 | /* | ||
164 | * GRU statistics. | ||
165 | */ | ||
166 | struct gru_stats_s { | ||
167 | atomic_long_t vdata_alloc; | ||
168 | atomic_long_t vdata_free; | ||
169 | atomic_long_t gts_alloc; | ||
170 | atomic_long_t gts_free; | ||
171 | atomic_long_t vdata_double_alloc; | ||
172 | atomic_long_t gts_double_allocate; | ||
173 | atomic_long_t assign_context; | ||
174 | atomic_long_t assign_context_failed; | ||
175 | atomic_long_t free_context; | ||
176 | atomic_long_t load_context; | ||
177 | atomic_long_t unload_context; | ||
178 | atomic_long_t steal_context; | ||
179 | atomic_long_t steal_context_failed; | ||
180 | atomic_long_t nopfn; | ||
181 | atomic_long_t break_cow; | ||
182 | atomic_long_t asid_new; | ||
183 | atomic_long_t asid_next; | ||
184 | atomic_long_t asid_wrap; | ||
185 | atomic_long_t asid_reuse; | ||
186 | atomic_long_t intr; | ||
187 | atomic_long_t call_os; | ||
188 | atomic_long_t call_os_check_for_bug; | ||
189 | atomic_long_t call_os_wait_queue; | ||
190 | atomic_long_t user_flush_tlb; | ||
191 | atomic_long_t user_unload_context; | ||
192 | atomic_long_t user_exception; | ||
193 | atomic_long_t set_task_slice; | ||
194 | atomic_long_t migrate_check; | ||
195 | atomic_long_t migrated_retarget; | ||
196 | atomic_long_t migrated_unload; | ||
197 | atomic_long_t migrated_unload_delay; | ||
198 | atomic_long_t migrated_nopfn_retarget; | ||
199 | atomic_long_t migrated_nopfn_unload; | ||
200 | atomic_long_t tlb_dropin; | ||
201 | atomic_long_t tlb_dropin_fail_no_asid; | ||
202 | atomic_long_t tlb_dropin_fail_upm; | ||
203 | atomic_long_t tlb_dropin_fail_invalid; | ||
204 | atomic_long_t tlb_dropin_fail_range_active; | ||
205 | atomic_long_t tlb_dropin_fail_idle; | ||
206 | atomic_long_t tlb_dropin_fail_fmm; | ||
207 | atomic_long_t mmu_invalidate_range; | ||
208 | atomic_long_t mmu_invalidate_page; | ||
209 | atomic_long_t mmu_clear_flush_young; | ||
210 | atomic_long_t flush_tlb; | ||
211 | atomic_long_t flush_tlb_gru; | ||
212 | atomic_long_t flush_tlb_gru_tgh; | ||
213 | atomic_long_t flush_tlb_gru_zero_asid; | ||
214 | |||
215 | atomic_long_t copy_gpa; | ||
216 | |||
217 | atomic_long_t mesq_receive; | ||
218 | atomic_long_t mesq_receive_none; | ||
219 | atomic_long_t mesq_send; | ||
220 | atomic_long_t mesq_send_failed; | ||
221 | atomic_long_t mesq_noop; | ||
222 | atomic_long_t mesq_send_unexpected_error; | ||
223 | atomic_long_t mesq_send_lb_overflow; | ||
224 | atomic_long_t mesq_send_qlimit_reached; | ||
225 | atomic_long_t mesq_send_amo_nacked; | ||
226 | atomic_long_t mesq_send_put_nacked; | ||
227 | atomic_long_t mesq_qf_not_full; | ||
228 | atomic_long_t mesq_qf_locked; | ||
229 | atomic_long_t mesq_qf_noop_not_full; | ||
230 | atomic_long_t mesq_qf_switch_head_failed; | ||
231 | atomic_long_t mesq_qf_unexpected_error; | ||
232 | atomic_long_t mesq_noop_unexpected_error; | ||
233 | atomic_long_t mesq_noop_lb_overflow; | ||
234 | atomic_long_t mesq_noop_qlimit_reached; | ||
235 | atomic_long_t mesq_noop_amo_nacked; | ||
236 | atomic_long_t mesq_noop_put_nacked; | ||
237 | |||
238 | }; | ||
239 | |||
240 | #define OPT_DPRINT 1 | ||
241 | #define OPT_STATS 2 | ||
242 | #define GRU_QUICKLOOK 4 | ||
243 | |||
244 | |||
245 | #define IRQ_GRU 110 /* Starting IRQ number for interrupts */ | ||
246 | |||
247 | /* Delay in jiffies between attempts to assign a GRU context */ | ||
248 | #define GRU_ASSIGN_DELAY ((HZ * 20) / 1000) | ||
249 | |||
250 | /* | ||
251 | * If a process has it's context stolen, min delay in jiffies before trying to | ||
252 | * steal a context from another process. | ||
253 | */ | ||
254 | #define GRU_STEAL_DELAY ((HZ * 200) / 1000) | ||
255 | |||
256 | #define STAT(id) do { \ | ||
257 | if (gru_options & OPT_STATS) \ | ||
258 | atomic_long_inc(&gru_stats.id); \ | ||
259 | } while (0) | ||
260 | |||
261 | #ifdef CONFIG_SGI_GRU_DEBUG | ||
262 | #define gru_dbg(dev, fmt, x...) \ | ||
263 | do { \ | ||
264 | if (gru_options & OPT_DPRINT) \ | ||
265 | dev_dbg(dev, "%s: " fmt, __func__, x); \ | ||
266 | } while (0) | ||
267 | #else | ||
268 | #define gru_dbg(x...) | ||
269 | #endif | ||
270 | |||
271 | /*----------------------------------------------------------------------------- | ||
272 | * ASID management | ||
273 | */ | ||
274 | #define MAX_ASID 0xfffff0 | ||
275 | #define MIN_ASID 8 | ||
276 | #define ASID_INC 8 /* number of regions */ | ||
277 | |||
278 | /* Generate a GRU asid value from a GRU base asid & a virtual address. */ | ||
279 | #if defined CONFIG_IA64 | ||
280 | #define VADDR_HI_BIT 64 | ||
281 | #define GRUREGION(addr) ((addr) >> (VADDR_HI_BIT - 3) & 3) | ||
282 | #elif defined __x86_64 | ||
283 | #define VADDR_HI_BIT 48 | ||
284 | #define GRUREGION(addr) (0) /* ZZZ could do better */ | ||
285 | #else | ||
286 | #error "Unsupported architecture" | ||
287 | #endif | ||
288 | #define GRUASID(asid, addr) ((asid) + GRUREGION(addr)) | ||
289 | |||
290 | /*------------------------------------------------------------------------------ | ||
291 | * File & VMS Tables | ||
292 | */ | ||
293 | |||
294 | struct gru_state; | ||
295 | |||
296 | /* | ||
297 | * This structure is pointed to from the mmstruct via the notifier pointer. | ||
298 | * There is one of these per address space. | ||
299 | */ | ||
300 | struct gru_mm_tracker { | ||
301 | unsigned int mt_asid_gen; /* ASID wrap count */ | ||
302 | int mt_asid; /* current base ASID for gru */ | ||
303 | unsigned short mt_ctxbitmap; /* bitmap of contexts using | ||
304 | asid */ | ||
305 | }; | ||
306 | |||
307 | struct gru_mm_struct { | ||
308 | struct mmu_notifier ms_notifier; | ||
309 | atomic_t ms_refcnt; | ||
310 | spinlock_t ms_asid_lock; /* protects ASID assignment */ | ||
311 | atomic_t ms_range_active;/* num range_invals active */ | ||
312 | char ms_released; | ||
313 | wait_queue_head_t ms_wait_queue; | ||
314 | DECLARE_BITMAP(ms_asidmap, GRU_MAX_GRUS); | ||
315 | struct gru_mm_tracker ms_asids[GRU_MAX_GRUS]; | ||
316 | }; | ||
317 | |||
318 | /* | ||
319 | * One of these structures is allocated when a GSEG is mmaped. The | ||
320 | * structure is pointed to by the vma->vm_private_data field in the vma struct. | ||
321 | */ | ||
322 | struct gru_vma_data { | ||
323 | spinlock_t vd_lock; /* Serialize access to vma */ | ||
324 | struct list_head vd_head; /* head of linked list of gts */ | ||
325 | long vd_user_options;/* misc user option flags */ | ||
326 | int vd_cbr_au_count; | ||
327 | int vd_dsr_au_count; | ||
328 | }; | ||
329 | |||
330 | /* | ||
331 | * One of these is allocated for each thread accessing a mmaped GRU. A linked | ||
332 | * list of these structure is hung off the struct gru_vma_data in the mm_struct. | ||
333 | */ | ||
334 | struct gru_thread_state { | ||
335 | struct list_head ts_next; /* list - head at vma-private */ | ||
336 | struct mutex ts_ctxlock; /* load/unload CTX lock */ | ||
337 | struct mm_struct *ts_mm; /* mm currently mapped to | ||
338 | context */ | ||
339 | struct vm_area_struct *ts_vma; /* vma of GRU context */ | ||
340 | struct gru_state *ts_gru; /* GRU where the context is | ||
341 | loaded */ | ||
342 | struct gru_mm_struct *ts_gms; /* asid & ioproc struct */ | ||
343 | unsigned long ts_cbr_map; /* map of allocated CBRs */ | ||
344 | unsigned long ts_dsr_map; /* map of allocated DATA | ||
345 | resources */ | ||
346 | unsigned long ts_steal_jiffies;/* jiffies when context last | ||
347 | stolen */ | ||
348 | long ts_user_options;/* misc user option flags */ | ||
349 | pid_t ts_tgid_owner; /* task that is using the | ||
350 | context - for migration */ | ||
351 | int ts_tsid; /* thread that owns the | ||
352 | structure */ | ||
353 | int ts_tlb_int_select;/* target cpu if interrupts | ||
354 | enabled */ | ||
355 | int ts_ctxnum; /* context number where the | ||
356 | context is loaded */ | ||
357 | atomic_t ts_refcnt; /* reference count GTS */ | ||
358 | unsigned char ts_dsr_au_count;/* Number of DSR resources | ||
359 | required for contest */ | ||
360 | unsigned char ts_cbr_au_count;/* Number of CBR resources | ||
361 | required for contest */ | ||
362 | char ts_force_unload;/* force context to be unloaded | ||
363 | after migration */ | ||
364 | char ts_cbr_idx[GRU_CBR_AU];/* CBR numbers of each | ||
365 | allocated CB */ | ||
366 | unsigned long ts_gdata[0]; /* save area for GRU data (CB, | ||
367 | DS, CBE) */ | ||
368 | }; | ||
369 | |||
370 | /* | ||
371 | * Threaded programs actually allocate an array of GSEGs when a context is | ||
372 | * created. Each thread uses a separate GSEG. TSID is the index into the GSEG | ||
373 | * array. | ||
374 | */ | ||
375 | #define TSID(a, v) (((a) - (v)->vm_start) / GRU_GSEG_PAGESIZE) | ||
376 | #define UGRUADDR(gts) ((gts)->ts_vma->vm_start + \ | ||
377 | (gts)->ts_tsid * GRU_GSEG_PAGESIZE) | ||
378 | |||
379 | #define NULLCTX (-1) /* if context not loaded into GRU */ | ||
380 | |||
381 | /*----------------------------------------------------------------------------- | ||
382 | * GRU State Tables | ||
383 | */ | ||
384 | |||
385 | /* | ||
386 | * One of these exists for each GRU chiplet. | ||
387 | */ | ||
388 | struct gru_state { | ||
389 | struct gru_blade_state *gs_blade; /* GRU state for entire | ||
390 | blade */ | ||
391 | unsigned long gs_gru_base_paddr; /* Physical address of | ||
392 | gru segments (64) */ | ||
393 | void *gs_gru_base_vaddr; /* Virtual address of | ||
394 | gru segments (64) */ | ||
395 | unsigned char gs_gid; /* unique GRU number */ | ||
396 | unsigned char gs_tgh_local_shift; /* used to pick TGH for | ||
397 | local flush */ | ||
398 | unsigned char gs_tgh_first_remote; /* starting TGH# for | ||
399 | remote flush */ | ||
400 | unsigned short gs_blade_id; /* blade of GRU */ | ||
401 | spinlock_t gs_asid_lock; /* lock used for | ||
402 | assigning asids */ | ||
403 | spinlock_t gs_lock; /* lock used for | ||
404 | assigning contexts */ | ||
405 | |||
406 | /* -- the following are protected by the gs_asid_lock spinlock ---- */ | ||
407 | unsigned int gs_asid; /* Next availe ASID */ | ||
408 | unsigned int gs_asid_limit; /* Limit of available | ||
409 | ASIDs */ | ||
410 | unsigned int gs_asid_gen; /* asid generation. | ||
411 | Inc on wrap */ | ||
412 | |||
413 | /* --- the following fields are protected by the gs_lock spinlock --- */ | ||
414 | unsigned long gs_context_map; /* bitmap to manage | ||
415 | contexts in use */ | ||
416 | unsigned long gs_cbr_map; /* bitmap to manage CB | ||
417 | resources */ | ||
418 | unsigned long gs_dsr_map; /* bitmap used to manage | ||
419 | DATA resources */ | ||
420 | unsigned int gs_reserved_cbrs; /* Number of kernel- | ||
421 | reserved cbrs */ | ||
422 | unsigned int gs_reserved_dsr_bytes; /* Bytes of kernel- | ||
423 | reserved dsrs */ | ||
424 | unsigned short gs_active_contexts; /* number of contexts | ||
425 | in use */ | ||
426 | struct gru_thread_state *gs_gts[GRU_NUM_CCH]; /* GTS currently using | ||
427 | the context */ | ||
428 | }; | ||
429 | |||
430 | /* | ||
431 | * This structure contains the GRU state for all the GRUs on a blade. | ||
432 | */ | ||
433 | struct gru_blade_state { | ||
434 | void *kernel_cb; /* First kernel | ||
435 | reserved cb */ | ||
436 | void *kernel_dsr; /* First kernel | ||
437 | reserved DSR */ | ||
438 | /* ---- the following are protected by the bs_lock spinlock ---- */ | ||
439 | spinlock_t bs_lock; /* lock used for | ||
440 | stealing contexts */ | ||
441 | int bs_lru_ctxnum; /* STEAL - last context | ||
442 | stolen */ | ||
443 | struct gru_state *bs_lru_gru; /* STEAL - last gru | ||
444 | stolen */ | ||
445 | |||
446 | struct gru_state bs_grus[GRU_CHIPLETS_PER_BLADE]; | ||
447 | }; | ||
448 | |||
449 | /*----------------------------------------------------------------------------- | ||
450 | * Address Primitives | ||
451 | */ | ||
452 | #define get_tfm_for_cpu(g, c) \ | ||
453 | ((struct gru_tlb_fault_map *)get_tfm((g)->gs_gru_base_vaddr, (c))) | ||
454 | #define get_tfh_by_index(g, i) \ | ||
455 | ((struct gru_tlb_fault_handle *)get_tfh((g)->gs_gru_base_vaddr, (i))) | ||
456 | #define get_tgh_by_index(g, i) \ | ||
457 | ((struct gru_tlb_global_handle *)get_tgh((g)->gs_gru_base_vaddr, (i))) | ||
458 | #define get_cbe_by_index(g, i) \ | ||
459 | ((struct gru_control_block_extended *)get_cbe((g)->gs_gru_base_vaddr,\ | ||
460 | (i))) | ||
461 | |||
462 | /*----------------------------------------------------------------------------- | ||
463 | * Useful Macros | ||
464 | */ | ||
465 | |||
466 | /* Given a blade# & chiplet#, get a pointer to the GRU */ | ||
467 | #define get_gru(b, c) (&gru_base[b]->bs_grus[c]) | ||
468 | |||
469 | /* Number of bytes to save/restore when unloading/loading GRU contexts */ | ||
470 | #define DSR_BYTES(dsr) ((dsr) * GRU_DSR_AU_BYTES) | ||
471 | #define CBR_BYTES(cbr) ((cbr) * GRU_HANDLE_BYTES * GRU_CBR_AU_SIZE * 2) | ||
472 | |||
473 | /* Convert a user CB number to the actual CBRNUM */ | ||
474 | #define thread_cbr_number(gts, n) ((gts)->ts_cbr_idx[(n) / GRU_CBR_AU_SIZE] \ | ||
475 | * GRU_CBR_AU_SIZE + (n) % GRU_CBR_AU_SIZE) | ||
476 | |||
477 | /* Convert a gid to a pointer to the GRU */ | ||
478 | #define GID_TO_GRU(gid) \ | ||
479 | (gru_base[(gid) / GRU_CHIPLETS_PER_BLADE] ? \ | ||
480 | (&gru_base[(gid) / GRU_CHIPLETS_PER_BLADE]-> \ | ||
481 | bs_grus[(gid) % GRU_CHIPLETS_PER_BLADE]) : \ | ||
482 | NULL) | ||
483 | |||
484 | /* Scan all active GRUs in a GRU bitmap */ | ||
485 | #define for_each_gru_in_bitmap(gid, map) \ | ||
486 | for ((gid) = find_first_bit((map), GRU_MAX_GRUS); (gid) < GRU_MAX_GRUS;\ | ||
487 | (gid)++, (gid) = find_next_bit((map), GRU_MAX_GRUS, (gid))) | ||
488 | |||
489 | /* Scan all active GRUs on a specific blade */ | ||
490 | #define for_each_gru_on_blade(gru, nid, i) \ | ||
491 | for ((gru) = gru_base[nid]->bs_grus, (i) = 0; \ | ||
492 | (i) < GRU_CHIPLETS_PER_BLADE; \ | ||
493 | (i)++, (gru)++) | ||
494 | |||
495 | /* Scan all active GTSs on a gru. Note: must hold ss_lock to use this macro. */ | ||
496 | #define for_each_gts_on_gru(gts, gru, ctxnum) \ | ||
497 | for ((ctxnum) = 0; (ctxnum) < GRU_NUM_CCH; (ctxnum)++) \ | ||
498 | if (((gts) = (gru)->gs_gts[ctxnum])) | ||
499 | |||
500 | /* Scan each CBR whose bit is set in a TFM (or copy of) */ | ||
501 | #define for_each_cbr_in_tfm(i, map) \ | ||
502 | for ((i) = find_first_bit(map, GRU_NUM_CBE); \ | ||
503 | (i) < GRU_NUM_CBE; \ | ||
504 | (i)++, (i) = find_next_bit(map, GRU_NUM_CBE, i)) | ||
505 | |||
506 | /* Scan each CBR in a CBR bitmap. Note: multiple CBRs in an allocation unit */ | ||
507 | #define for_each_cbr_in_allocation_map(i, map, k) \ | ||
508 | for ((k) = find_first_bit(map, GRU_CBR_AU); (k) < GRU_CBR_AU; \ | ||
509 | (k) = find_next_bit(map, GRU_CBR_AU, (k) + 1)) \ | ||
510 | for ((i) = (k)*GRU_CBR_AU_SIZE; \ | ||
511 | (i) < ((k) + 1) * GRU_CBR_AU_SIZE; (i)++) | ||
512 | |||
513 | /* Scan each DSR in a DSR bitmap. Note: multiple DSRs in an allocation unit */ | ||
514 | #define for_each_dsr_in_allocation_map(i, map, k) \ | ||
515 | for ((k) = find_first_bit((const unsigned long *)map, GRU_DSR_AU);\ | ||
516 | (k) < GRU_DSR_AU; \ | ||
517 | (k) = find_next_bit((const unsigned long *)map, \ | ||
518 | GRU_DSR_AU, (k) + 1)) \ | ||
519 | for ((i) = (k) * GRU_DSR_AU_CL; \ | ||
520 | (i) < ((k) + 1) * GRU_DSR_AU_CL; (i)++) | ||
521 | |||
522 | #define gseg_physical_address(gru, ctxnum) \ | ||
523 | ((gru)->gs_gru_base_paddr + ctxnum * GRU_GSEG_STRIDE) | ||
524 | #define gseg_virtual_address(gru, ctxnum) \ | ||
525 | ((gru)->gs_gru_base_vaddr + ctxnum * GRU_GSEG_STRIDE) | ||
526 | |||
527 | /*----------------------------------------------------------------------------- | ||
528 | * Lock / Unlock GRU handles | ||
529 | * Use the "delresp" bit in the handle as a "lock" bit. | ||
530 | */ | ||
531 | |||
532 | /* Lock hierarchy checking enabled only in emulator */ | ||
533 | |||
534 | static inline void __lock_handle(void *h) | ||
535 | { | ||
536 | while (test_and_set_bit(1, h)) | ||
537 | cpu_relax(); | ||
538 | } | ||
539 | |||
540 | static inline void __unlock_handle(void *h) | ||
541 | { | ||
542 | clear_bit(1, h); | ||
543 | } | ||
544 | |||
545 | static inline void lock_cch_handle(struct gru_context_configuration_handle *cch) | ||
546 | { | ||
547 | __lock_handle(cch); | ||
548 | } | ||
549 | |||
550 | static inline void unlock_cch_handle(struct gru_context_configuration_handle | ||
551 | *cch) | ||
552 | { | ||
553 | __unlock_handle(cch); | ||
554 | } | ||
555 | |||
556 | static inline void lock_tgh_handle(struct gru_tlb_global_handle *tgh) | ||
557 | { | ||
558 | __lock_handle(tgh); | ||
559 | } | ||
560 | |||
561 | static inline void unlock_tgh_handle(struct gru_tlb_global_handle *tgh) | ||
562 | { | ||
563 | __unlock_handle(tgh); | ||
564 | } | ||
565 | |||
566 | /*----------------------------------------------------------------------------- | ||
567 | * Function prototypes & externs | ||
568 | */ | ||
569 | struct gru_unload_context_req; | ||
570 | |||
571 | extern struct vm_operations_struct gru_vm_ops; | ||
572 | extern struct device *grudev; | ||
573 | |||
574 | extern struct gru_vma_data *gru_alloc_vma_data(struct vm_area_struct *vma, | ||
575 | int tsid); | ||
576 | extern struct gru_thread_state *gru_find_thread_state(struct vm_area_struct | ||
577 | *vma, int tsid); | ||
578 | extern struct gru_thread_state *gru_alloc_thread_state(struct vm_area_struct | ||
579 | *vma, int tsid); | ||
580 | extern void gru_unload_context(struct gru_thread_state *gts, int savestate); | ||
581 | extern void gts_drop(struct gru_thread_state *gts); | ||
582 | extern void gru_tgh_flush_init(struct gru_state *gru); | ||
583 | extern int gru_kservices_init(struct gru_state *gru); | ||
584 | extern irqreturn_t gru_intr(int irq, void *dev_id); | ||
585 | extern int gru_handle_user_call_os(unsigned long address); | ||
586 | extern int gru_user_flush_tlb(unsigned long arg); | ||
587 | extern int gru_user_unload_context(unsigned long arg); | ||
588 | extern int gru_get_exception_detail(unsigned long arg); | ||
589 | extern int gru_set_task_slice(long address); | ||
590 | extern int gru_cpu_fault_map_id(void); | ||
591 | extern struct vm_area_struct *gru_find_vma(unsigned long vaddr); | ||
592 | extern void gru_flush_all_tlb(struct gru_state *gru); | ||
593 | extern int gru_proc_init(void); | ||
594 | extern void gru_proc_exit(void); | ||
595 | |||
596 | extern unsigned long gru_reserve_cb_resources(struct gru_state *gru, | ||
597 | int cbr_au_count, char *cbmap); | ||
598 | extern unsigned long gru_reserve_ds_resources(struct gru_state *gru, | ||
599 | int dsr_au_count, char *dsmap); | ||
600 | extern int gru_fault(struct vm_area_struct *, struct vm_fault *vmf); | ||
601 | extern struct gru_mm_struct *gru_register_mmu_notifier(void); | ||
602 | extern void gru_drop_mmu_notifier(struct gru_mm_struct *gms); | ||
603 | |||
604 | extern void gru_flush_tlb_range(struct gru_mm_struct *gms, unsigned long start, | ||
605 | unsigned long len); | ||
606 | |||
607 | extern unsigned long gru_options; | ||
608 | |||
609 | #endif /* __GRUTABLES_H__ */ | ||