GRU Driver: hardware data structures

This series of patches adds a driver for the SGI UV GRU.  The driver is
still in development but it currently compiles for both x86_64 & IA64.
All simple regression tests pass on IA64.  Although features remain to be
added, I'd like to start the process of getting the driver into the
kernel.  Additional kernel drivers will depend on services provide by the
GRU driver.

The GRU is a hardware resource located in the system chipset.  The GRU
contains memory that is mmaped into the user address space.  This memory
is used to communicate with the GRU to perform functions such as
load/store, scatter/gather, bcopy, AMOs, etc.  The GRU is directly
accessed by user instructions using user virtual addresses.  GRU
instructions (ex., bcopy) use user virtual addresses for operands.

The GRU contains a large TLB that is functionally very similar to
processor TLBs.  Because the external contains a TLB with user virtual
address, it requires callouts from the core VM system when certain types
of changes are made to the process page tables.  There are several MMUOPS
patches currently being discussed but none has been accepted into the
kernel.  The GRU driver is built using version V18 from Andrea Arcangeli.

This patch:

Contains the definitions of the hardware GRU data structures that are used
by the driver to manage the GRU.

[akpm@linux-foundation;org: export hpage_shift]
Signed-off-by: Jack Steiner <steiner@sgi.com>
Cc: "Luck, Tony" <tony.luck@intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

1 files changed, 663 insertions, 0 deletions

diff --git a/drivers/misc/sgi-gru/gruhandles.h b/drivers/misc/sgi-gru/gruhandles.h
new file mode 100644
index 000000000000..d16031d62673
--- /dev/null
+++ b/drivers/misc/sgi-gru/gruhandles.h
@@ -0,0 +1,663 @@
+/*
+ * SN Platform GRU Driver
+ *
+ *              GRU HANDLE DEFINITION
+ *
+ *  Copyright (c) 2008 Silicon Graphics, Inc.  All Rights Reserved.
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
+ */
+
+#ifndef __GRUHANDLES_H__
+#define __GRUHANDLES_H__
+#include "gru_instructions.h"
+
+/*
+ * Manifest constants for GRU Memory Map
+ */
+#define GRU_GSEG0_BASE          0
+#define GRU_MCS_BASE            (64 * 1024 * 1024)
+#define GRU_SIZE                (128UL * 1024 * 1024)
+
+/* Handle & resource counts */
+#define GRU_NUM_CB              128
+#define GRU_NUM_DSR_BYTES       (32 * 1024)
+#define GRU_NUM_TFM             16
+#define GRU_NUM_TGH             24
+#define GRU_NUM_CBE             128
+#define GRU_NUM_TFH             128
+#define GRU_NUM_CCH             16
+#define GRU_NUM_GSH             1
+
+/* Maximum resource counts that can be reserved by user programs */
+#define GRU_NUM_USER_CBR        GRU_NUM_CBE
+#define GRU_NUM_USER_DSR_BYTES  GRU_NUM_DSR_BYTES
+
+/* Bytes per handle & handle stride. Code assumes all cb, tfh, cbe handles
+ * are the same */
+#define GRU_HANDLE_BYTES        64
+#define GRU_HANDLE_STRIDE       256
+
+/* Base addresses of handles */
+#define GRU_TFM_BASE            (GRU_MCS_BASE + 0x00000)
+#define GRU_TGH_BASE            (GRU_MCS_BASE + 0x08000)
+#define GRU_CBE_BASE            (GRU_MCS_BASE + 0x10000)
+#define GRU_TFH_BASE            (GRU_MCS_BASE + 0x18000)
+#define GRU_CCH_BASE            (GRU_MCS_BASE + 0x20000)
+#define GRU_GSH_BASE            (GRU_MCS_BASE + 0x30000)
+
+/* User gseg constants */
+#define GRU_GSEG_STRIDE         (4 * 1024 * 1024)
+#define GSEG_BASE(a)            ((a) & ~(GRU_GSEG_PAGESIZE - 1))
+
+/* Data segment constants */
+#define GRU_DSR_AU_BYTES        1024
+#define GRU_DSR_CL              (GRU_NUM_DSR_BYTES / GRU_CACHE_LINE_BYTES)
+#define GRU_DSR_AU_CL           (GRU_DSR_AU_BYTES / GRU_CACHE_LINE_BYTES)
+#define GRU_DSR_AU              (GRU_NUM_DSR_BYTES / GRU_DSR_AU_BYTES)
+
+/* Control block constants */
+#define GRU_CBR_AU_SIZE         2
+#define GRU_CBR_AU              (GRU_NUM_CBE / GRU_CBR_AU_SIZE)
+
+/* Convert resource counts to the number of AU */
+#define GRU_DS_BYTES_TO_AU(n)   DIV_ROUND_UP(n, GRU_DSR_AU_BYTES)
+#define GRU_CB_COUNT_TO_AU(n)   DIV_ROUND_UP(n, GRU_CBR_AU_SIZE)
+
+/* UV limits */
+#define GRU_CHIPLETS_PER_HUB    2
+#define GRU_HUBS_PER_BLADE      1
+#define GRU_CHIPLETS_PER_BLADE  (GRU_HUBS_PER_BLADE * GRU_CHIPLETS_PER_HUB)
+
+/* User GRU Gseg offsets */
+#define GRU_CB_BASE             0
+#define GRU_CB_LIMIT            (GRU_CB_BASE + GRU_HANDLE_STRIDE * GRU_NUM_CBE)
+#define GRU_DS_BASE             0x20000
+#define GRU_DS_LIMIT            (GRU_DS_BASE + GRU_NUM_DSR_BYTES)
+
+/* Convert a GRU physical address to the chiplet offset */
+#define GSEGPOFF(h)             ((h) & (GRU_SIZE - 1))
+
+/* Convert an arbitrary handle address to the beginning of the GRU segment */
+#ifndef __PLUGIN__
+#define GRUBASE(h)              ((void *)((unsigned long)(h) & ~(GRU_SIZE - 1)))
+#else
+extern void *gmu_grubase(void *h);
+#define GRUBASE(h)              gmu_grubase(h)
+#endif
+
+/* General addressing macros. */
+static inline void *get_gseg_base_address(void *base, int ctxnum)
+{
+        return (void *)(base + GRU_GSEG0_BASE + GRU_GSEG_STRIDE * ctxnum);
+}
+
+static inline void *get_gseg_base_address_cb(void *base, int ctxnum, int line)
+{
+        return (void *)(get_gseg_base_address(base, ctxnum) +
+                        GRU_CB_BASE + GRU_HANDLE_STRIDE * line);
+}
+
+static inline void *get_gseg_base_address_ds(void *base, int ctxnum, int line)
+{
+        return (void *)(get_gseg_base_address(base, ctxnum) + GRU_DS_BASE +
+                        GRU_CACHE_LINE_BYTES * line);
+}
+
+static inline struct gru_tlb_fault_map *get_tfm(void *base, int ctxnum)
+{
+        return (struct gru_tlb_fault_map *)(base + GRU_TFM_BASE +
+                                        ctxnum * GRU_HANDLE_STRIDE);
+}
+
+static inline struct gru_tlb_global_handle *get_tgh(void *base, int ctxnum)
+{
+        return (struct gru_tlb_global_handle *)(base + GRU_TGH_BASE +
+                                        ctxnum * GRU_HANDLE_STRIDE);
+}
+
+static inline struct gru_control_block_extended *get_cbe(void *base, int ctxnum)
+{
+        return (struct gru_control_block_extended *)(base + GRU_CBE_BASE +
+                                        ctxnum * GRU_HANDLE_STRIDE);
+}
+
+static inline struct gru_tlb_fault_handle *get_tfh(void *base, int ctxnum)
+{
+        return (struct gru_tlb_fault_handle *)(base + GRU_TFH_BASE +
+                                        ctxnum * GRU_HANDLE_STRIDE);
+}
+
+static inline struct gru_context_configuration_handle *get_cch(void *base,
+                                        int ctxnum)
+{
+        return (struct gru_context_configuration_handle *)(base +
+                                GRU_CCH_BASE + ctxnum * GRU_HANDLE_STRIDE);
+}
+
+static inline unsigned long get_cb_number(void *cb)
+{
+        return (((unsigned long)cb - GRU_CB_BASE) % GRU_GSEG_PAGESIZE) /
+                                        GRU_HANDLE_STRIDE;
+}
+
+/* byte offset to a specific GRU chiplet. (p=pnode, c=chiplet (0 or 1)*/
+static inline unsigned long gru_chiplet_paddr(unsigned long paddr, int pnode,
+                                                        int chiplet)
+{
+        return paddr + GRU_SIZE * (2 * pnode  + chiplet);
+}
+
+static inline void *gru_chiplet_vaddr(void *vaddr, int pnode, int chiplet)
+{
+        return vaddr + GRU_SIZE * (2 * pnode  + chiplet);
+}
+
+
+
+/*
+ * Global TLB Fault Map
+ *      Bitmap of outstanding TLB misses needing interrupt/polling service.
+ *
+ */
+struct gru_tlb_fault_map {
+        unsigned long fault_bits[BITS_TO_LONGS(GRU_NUM_CBE)];
+        unsigned long fill0[2];
+        unsigned long done_bits[BITS_TO_LONGS(GRU_NUM_CBE)];
+        unsigned long fill1[2];
+};
+
+/*
+ * TGH - TLB Global Handle
+ *      Used for TLB flushing.
+ *
+ */
+struct gru_tlb_global_handle {
+        unsigned int cmd:1;             /* DW 0 */
+        unsigned int delresp:1;
+        unsigned int opc:1;
+        unsigned int fill1:5;
+
+        unsigned int fill2:8;
+
+        unsigned int status:2;
+        unsigned long fill3:2;
+        unsigned int state:3;
+        unsigned long fill4:1;
+
+        unsigned int cause:3;
+        unsigned long fill5:37;
+
+        unsigned long vaddr:64;         /* DW 1 */
+
+        unsigned int asid:24;           /* DW 2 */
+        unsigned int fill6:8;
+
+        unsigned int pagesize:5;
+        unsigned int fill7:11;
+
+        unsigned int global:1;
+        unsigned int fill8:15;
+
+        unsigned long vaddrmask:39;     /* DW 3 */
+        unsigned int fill9:9;
+        unsigned int n:10;
+        unsigned int fill10:6;
+
+        unsigned int ctxbitmap:16;      /* DW4 */
+        unsigned long fill11[3];
+};
+
+enum gru_tgh_cmd {
+        TGHCMD_START
+};
+
+enum gru_tgh_opc {
+        TGHOP_TLBNOP,
+        TGHOP_TLBINV
+};
+
+enum gru_tgh_status {
+        TGHSTATUS_IDLE,
+        TGHSTATUS_EXCEPTION,
+        TGHSTATUS_ACTIVE
+};
+
+enum gru_tgh_state {
+        TGHSTATE_IDLE,
+        TGHSTATE_PE_INVAL,
+        TGHSTATE_INTERRUPT_INVAL,
+        TGHSTATE_WAITDONE,
+        TGHSTATE_RESTART_CTX,
+};
+
+/*
+ * TFH - TLB Global Handle
+ *      Used for TLB dropins into the GRU TLB.
+ *
+ */
+struct gru_tlb_fault_handle {
+        unsigned int cmd:1;             /* DW 0 - low 32*/
+        unsigned int delresp:1;
+        unsigned int fill0:2;
+        unsigned int opc:3;
+        unsigned int fill1:9;
+
+        unsigned int status:2;
+        unsigned int fill2:1;
+        unsigned int color:1;
+        unsigned int state:3;
+        unsigned int fill3:1;
+
+        unsigned int cause:7;           /* DW 0 - high 32 */
+        unsigned int fill4:1;
+
+        unsigned int indexway:12;
+        unsigned int fill5:4;
+
+        unsigned int ctxnum:4;
+        unsigned int fill6:12;
+
+        unsigned long missvaddr:64;     /* DW 1 */
+
+        unsigned int missasid:24;       /* DW 2 */
+        unsigned int fill7:8;
+        unsigned int fillasid:24;
+        unsigned int dirty:1;
+        unsigned int gaa:2;
+        unsigned long fill8:5;
+
+        unsigned long pfn:41;           /* DW 3 */
+        unsigned int fill9:7;
+        unsigned int pagesize:5;
+        unsigned int fill10:11;
+
+        unsigned long fillvaddr:64;     /* DW 4 */
+
+        unsigned long fill11[3];
+};
+
+enum gru_tfh_opc {
+        TFHOP_NOOP,
+        TFHOP_RESTART,
+        TFHOP_WRITE_ONLY,
+        TFHOP_WRITE_RESTART,
+        TFHOP_EXCEPTION,
+        TFHOP_USER_POLLING_MODE = 7,
+};
+
+enum tfh_status {
+        TFHSTATUS_IDLE,
+        TFHSTATUS_EXCEPTION,
+        TFHSTATUS_ACTIVE,
+};
+
+enum tfh_state {
+        TFHSTATE_INACTIVE,
+        TFHSTATE_IDLE,
+        TFHSTATE_MISS_UPM,
+        TFHSTATE_MISS_FMM,
+        TFHSTATE_HW_ERR,
+        TFHSTATE_WRITE_TLB,
+        TFHSTATE_RESTART_CBR,
+};
+
+/* TFH cause bits */
+enum tfh_cause {
+        TFHCAUSE_NONE,
+        TFHCAUSE_TLB_MISS,
+        TFHCAUSE_TLB_MOD,
+        TFHCAUSE_HW_ERROR_RR,
+        TFHCAUSE_HW_ERROR_MAIN_ARRAY,
+        TFHCAUSE_HW_ERROR_VALID,
+        TFHCAUSE_HW_ERROR_PAGESIZE,
+        TFHCAUSE_INSTRUCTION_EXCEPTION,
+        TFHCAUSE_UNCORRECTIBLE_ERROR,
+};
+
+/* GAA values */
+#define GAA_RAM                         0x0
+#define GAA_NCRAM                       0x2
+#define GAA_MMIO                        0x1
+#define GAA_REGISTER                    0x3
+
+/* GRU paddr shift for pfn. (NOTE: shift is NOT by actual pagesize) */
+#define GRU_PADDR_SHIFT                 12
+
+/*
+ * Context Configuration handle
+ *      Used to allocate resources to a GSEG context.
+ *
+ */
+struct gru_context_configuration_handle {
+        unsigned int cmd:1;                     /* DW0 */
+        unsigned int delresp:1;
+        unsigned int opc:3;
+        unsigned int unmap_enable:1;
+        unsigned int req_slice_set_enable:1;
+        unsigned int req_slice:2;
+        unsigned int cb_int_enable:1;
+        unsigned int tlb_int_enable:1;
+        unsigned int tfm_fault_bit_enable:1;
+        unsigned int tlb_int_select:4;
+
+        unsigned int status:2;
+        unsigned int state:2;
+        unsigned int reserved2:4;
+
+        unsigned int cause:4;
+        unsigned int tfm_done_bit_enable:1;
+        unsigned int unused:3;
+
+        unsigned int dsr_allocation_map;
+
+        unsigned long cbr_allocation_map;       /* DW1 */
+
+        unsigned int asid[8];                   /* DW 2 - 5 */
+        unsigned short sizeavail[8];            /* DW 6 - 7 */
+} __attribute__ ((packed));
+
+enum gru_cch_opc {
+        CCHOP_START = 1,
+        CCHOP_ALLOCATE,
+        CCHOP_INTERRUPT,
+        CCHOP_DEALLOCATE,
+        CCHOP_INTERRUPT_SYNC,
+};
+
+enum gru_cch_status {
+        CCHSTATUS_IDLE,
+        CCHSTATUS_EXCEPTION,
+        CCHSTATUS_ACTIVE,
+};
+
+enum gru_cch_state {
+        CCHSTATE_INACTIVE,
+        CCHSTATE_MAPPED,
+        CCHSTATE_ACTIVE,
+        CCHSTATE_INTERRUPTED,
+};
+
+/* CCH Exception cause */
+enum gru_cch_cause {
+        CCHCAUSE_REGION_REGISTER_WRITE_ERROR = 1,
+        CCHCAUSE_ILLEGAL_OPCODE = 2,
+        CCHCAUSE_INVALID_START_REQUEST = 3,
+        CCHCAUSE_INVALID_ALLOCATION_REQUEST = 4,
+        CCHCAUSE_INVALID_DEALLOCATION_REQUEST = 5,
+        CCHCAUSE_INVALID_INTERRUPT_REQUEST = 6,
+        CCHCAUSE_CCH_BUSY = 7,
+        CCHCAUSE_NO_CBRS_TO_ALLOCATE = 8,
+        CCHCAUSE_BAD_TFM_CONFIG = 9,
+        CCHCAUSE_CBR_RESOURCES_OVERSUBSCRIPED = 10,
+        CCHCAUSE_DSR_RESOURCES_OVERSUBSCRIPED = 11,
+        CCHCAUSE_CBR_DEALLOCATION_ERROR = 12,
+};
+/*
+ * CBE - Control Block Extended
+ *      Maintains internal GRU state for active CBs.
+ *
+ */
+struct gru_control_block_extended {
+        unsigned int reserved0:1;       /* DW 0  - low */
+        unsigned int imacpy:3;
+        unsigned int reserved1:4;
+        unsigned int xtypecpy:3;
+        unsigned int iaa0cpy:2;
+        unsigned int iaa1cpy:2;
+        unsigned int reserved2:1;
+        unsigned int opccpy:8;
+        unsigned int exopccpy:8;
+
+        unsigned int idef2cpy:22;       /* DW 0  - high */
+        unsigned int reserved3:10;
+
+        unsigned int idef4cpy:22;       /* DW 1 */
+        unsigned int reserved4:10;
+        unsigned int idef4upd:22;
+        unsigned int reserved5:10;
+
+        unsigned long idef1upd:64;      /* DW 2 */
+
+        unsigned long idef5cpy:64;      /* DW 3 */
+
+        unsigned long idef6cpy:64;      /* DW 4 */
+
+        unsigned long idef3upd:64;      /* DW 5 */
+
+        unsigned long idef5upd:64;      /* DW 6 */
+
+        unsigned int idef2upd:22;       /* DW 7 */
+        unsigned int reserved6:10;
+
+        unsigned int ecause:20;
+        unsigned int cbrstate:4;
+        unsigned int cbrexecstatus:8;
+};
+
+enum gru_cbr_state {
+        CBRSTATE_INACTIVE,
+        CBRSTATE_IDLE,
+        CBRSTATE_PE_CHECK,
+        CBRSTATE_QUEUED,
+        CBRSTATE_WAIT_RESPONSE,
+        CBRSTATE_INTERRUPTED,
+        CBRSTATE_INTERRUPTED_MISS_FMM,
+        CBRSTATE_BUSY_INTERRUPT_MISS_FMM,
+        CBRSTATE_INTERRUPTED_MISS_UPM,
+        CBRSTATE_BUSY_INTERRUPTED_MISS_UPM,
+        CBRSTATE_REQUEST_ISSUE,
+        CBRSTATE_BUSY_INTERRUPT,
+};
+
+/* CBE cbrexecstatus bits */
+#define CBR_EXS_ABORT_OCC_BIT                   0
+#define CBR_EXS_INT_OCC_BIT                     1
+#define CBR_EXS_PENDING_BIT                     2
+#define CBR_EXS_QUEUED_BIT                      3
+#define CBR_EXS_TLBHW_BIT                       4
+#define CBR_EXS_EXCEPTION_BIT                   5
+
+#define CBR_EXS_ABORT_OCC                       (1 << CBR_EXS_ABORT_OCC_BIT)
+#define CBR_EXS_INT_OCC                         (1 << CBR_EXS_INT_OCC_BIT)
+#define CBR_EXS_PENDING                         (1 << CBR_EXS_PENDING_BIT)
+#define CBR_EXS_QUEUED                          (1 << CBR_EXS_QUEUED_BIT)
+#define CBR_EXS_TLBHW                           (1 << CBR_EXS_TLBHW_BIT)
+#define CBR_EXS_EXCEPTION                       (1 << CBR_EXS_EXCEPTION_BIT)
+
+/* CBE ecause bits  - defined in gru_instructions.h */
+
+/*
+ * Convert a processor pagesize into the strange encoded pagesize used by the
+ * GRU. Processor pagesize is encoded as log of bytes per page. (or PAGE_SHIFT)
+ *      pagesize        log pagesize    grupagesize
+ *        4k                    12      0
+ *       16k                    14      1
+ *       64k                    16      2
+ *      256k                    18      3
+ *        1m                    20      4
+ *        2m                    21      5
+ *        4m                    22      6
+ *       16m                    24      7
+ *       64m                    26      8
+ *      ...
+ */
+#define GRU_PAGESIZE(sh)        ((((sh) > 20 ? (sh) + 2: (sh)) >> 1) - 6)
+#define GRU_SIZEAVAIL(sh)       (1UL << GRU_PAGESIZE(sh))
+
+/* minimum TLB purge count to ensure a full purge */
+#define GRUMAXINVAL             1024UL
+
+
+/* Extract the status field from a kernel handle */
+#define GET_MSEG_HANDLE_STATUS(h)       (((*(unsigned long *)(h)) >> 16) & 3)
+
+static inline void start_instruction(void *h)
+{
+        unsigned long *w0 = h;
+
+        wmb();          /* setting CMD bit must be last */
+        *w0 = *w0 | 1;
+        gru_flush_cache(h);
+}
+
+static inline int wait_instruction_complete(void *h)
+{
+        int status;
+
+        do {
+                cpu_relax();
+                barrier();
+                status = GET_MSEG_HANDLE_STATUS(h);
+        } while (status == CCHSTATUS_ACTIVE);
+        return status;
+}
+
+#if defined CONFIG_IA64
+static inline void cch_allocate_set_asids(
+                  struct gru_context_configuration_handle *cch, int asidval)
+{
+        int i;
+
+        for (i = 0; i <= RGN_HPAGE; i++) {  /*  assume HPAGE is last region */
+                cch->asid[i] = (asidval++);
+#if 0
+                /* ZZZ hugepages not supported yet */
+                if (i == RGN_HPAGE)
+                        cch->sizeavail[i] = GRU_SIZEAVAIL(hpage_shift);
+                else
+#endif
+                        cch->sizeavail[i] = GRU_SIZEAVAIL(PAGE_SHIFT);
+        }
+}
+#elif defined CONFIG_X86_64
+static inline void cch_allocate_set_asids(
+                  struct gru_context_configuration_handle *cch, int asidval)
+{
+        int i;
+
+        for (i = 0; i < 8; i++) {
+                cch->asid[i] = asidval++;
+                cch->sizeavail[i] = GRU_SIZEAVAIL(PAGE_SHIFT) |
+                        GRU_SIZEAVAIL(21);
+        }
+}
+#endif
+
+static inline int cch_allocate(struct gru_context_configuration_handle *cch,
+                               int asidval, unsigned long cbrmap,
+                               unsigned long dsrmap)
+{
+        cch_allocate_set_asids(cch, asidval);
+        cch->dsr_allocation_map = dsrmap;
+        cch->cbr_allocation_map = cbrmap;
+        cch->opc = CCHOP_ALLOCATE;
+        start_instruction(cch);
+        return wait_instruction_complete(cch);
+}
+
+static inline int cch_start(struct gru_context_configuration_handle *cch)
+{
+        cch->opc = CCHOP_START;
+        start_instruction(cch);
+        return wait_instruction_complete(cch);
+}
+
+static inline int cch_interrupt(struct gru_context_configuration_handle *cch)
+{
+        cch->opc = CCHOP_INTERRUPT;
+        start_instruction(cch);
+        return wait_instruction_complete(cch);
+}
+
+static inline int cch_deallocate(struct gru_context_configuration_handle *cch)
+{
+        cch->opc = CCHOP_DEALLOCATE;
+        start_instruction(cch);
+        return wait_instruction_complete(cch);
+}
+
+static inline int cch_interrupt_sync(struct gru_context_configuration_handle
+                                     *cch)
+{
+        cch->opc = CCHOP_INTERRUPT_SYNC;
+        start_instruction(cch);
+        return wait_instruction_complete(cch);
+}
+
+static inline int tgh_invalidate(struct gru_tlb_global_handle *tgh,
+                                 unsigned long vaddr, unsigned long vaddrmask,
+                                 int asid, int pagesize, int global, int n,
+                                 unsigned short ctxbitmap)
+{
+        tgh->vaddr = vaddr;
+        tgh->asid = asid;
+        tgh->pagesize = pagesize;
+        tgh->n = n;
+        tgh->global = global;
+        tgh->vaddrmask = vaddrmask;
+        tgh->ctxbitmap = ctxbitmap;
+        tgh->opc = TGHOP_TLBINV;
+        start_instruction(tgh);
+        return wait_instruction_complete(tgh);
+}
+
+static inline void tfh_write_only(struct gru_tlb_fault_handle *tfh,
+                                  unsigned long pfn, unsigned long vaddr,
+                                  int asid, int dirty, int pagesize)
+{
+        tfh->fillasid = asid;
+        tfh->fillvaddr = vaddr;
+        tfh->pfn = pfn;
+        tfh->dirty = dirty;
+        tfh->pagesize = pagesize;
+        tfh->opc = TFHOP_WRITE_ONLY;
+        start_instruction(tfh);
+}
+
+static inline void tfh_write_restart(struct gru_tlb_fault_handle *tfh,
+                                     unsigned long paddr, int gaa,
+                                     unsigned long vaddr, int asid, int dirty,
+                                     int pagesize)
+{
+        tfh->fillasid = asid;
+        tfh->fillvaddr = vaddr;
+        tfh->pfn = paddr >> GRU_PADDR_SHIFT;
+        tfh->gaa = gaa;
+        tfh->dirty = dirty;
+        tfh->pagesize = pagesize;
+        tfh->opc = TFHOP_WRITE_RESTART;
+        start_instruction(tfh);
+}
+
+static inline void tfh_restart(struct gru_tlb_fault_handle *tfh)
+{
+        tfh->opc = TFHOP_RESTART;
+        start_instruction(tfh);
+}
+
+static inline void tfh_user_polling_mode(struct gru_tlb_fault_handle *tfh)
+{
+        tfh->opc = TFHOP_USER_POLLING_MODE;
+        start_instruction(tfh);
+}
+
+static inline void tfh_exception(struct gru_tlb_fault_handle *tfh)
+{
+        tfh->opc = TFHOP_EXCEPTION;
+        start_instruction(tfh);
+}
+
+#endif /* __GRUHANDLES_H__ */