aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorIan Munsie <imunsie@au1.ibm.com>2014-10-08 04:55:02 -0400
committerMichael Ellerman <mpe@ellerman.id.au>2014-10-08 05:15:57 -0400
commitf204e0b8cedd7da1dfcfd05ed6b7692737e24029 (patch)
tree35ca15049345cdd5dbed38229a6b3add05610658
parent10542ca0156f60571ef41799d44d40dd4cb0a473 (diff)
cxl: Driver code for powernv PCIe based cards for userspace access
This is the core of the cxl driver. It adds support for using cxl cards in the powernv environment only (ie POWER8 bare metal). It allows access to cxl accelerators by userspace using the /dev/cxl/afuM.N char devices. The kernel driver has no knowledge of the function implemented by the accelerator. It provides services to userspace via the /dev/cxl/afuM.N devices. When a program opens this device and runs the start work IOCTL, the accelerator will have coherent access to that processes memory using the same virtual addresses. That process may mmap the device to access any MMIO space the accelerator provides. Also, reads on the device will allow interrupts to be received. These services are further documented in a later patch in Documentation/powerpc/cxl.txt. Documentation of the cxl hardware architecture and userspace API is provided in subsequent patches. Signed-off-by: Ian Munsie <imunsie@au1.ibm.com> Signed-off-by: Michael Neuling <mikey@neuling.org> Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Diffstat
-rw-r--r--drivers/misc/cxl/context.c193
-rw-r--r--drivers/misc/cxl/cxl.h629
-rw-r--r--drivers/misc/cxl/debugfs.c132
-rw-r--r--drivers/misc/cxl/fault.c291
-rw-r--r--drivers/misc/cxl/file.c508
-rw-r--r--drivers/misc/cxl/irq.c402
-rw-r--r--drivers/misc/cxl/main.c230
-rw-r--r--drivers/misc/cxl/native.c683
-rw-r--r--drivers/misc/cxl/pci.c1000
-rw-r--r--drivers/misc/cxl/sysfs.c385
10 files changed, 4453 insertions, 0 deletions
diff --git a/drivers/misc/cxl/context.c b/drivers/misc/cxl/context.c
new file mode 100644
index 000000000000..cca472109135
--- /dev/null
+++ b/drivers/misc/cxl/context.c
@@ -0,0 +1,193 @@
1/*
2 * Copyright 2014 IBM Corp.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */
9
10#include <linux/module.h>
11#include <linux/kernel.h>
12#include <linux/bitmap.h>
13#include <linux/sched.h>
14#include <linux/pid.h>
15#include <linux/fs.h>
16#include <linux/mm.h>
17#include <linux/debugfs.h>
18#include <linux/slab.h>
19#include <linux/idr.h>
20#include <asm/cputable.h>
21#include <asm/current.h>
22#include <asm/copro.h>
23
24#include "cxl.h"
25
26/*
27 * Allocates space for a CXL context.
28 */
29struct cxl_context *cxl_context_alloc(void)
30{
31 return kzalloc(sizeof(struct cxl_context), GFP_KERNEL);
32}
33
34/*
35 * Initialises a CXL context.
36 */
37int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master)
38{
39 int i;
40
41 spin_lock_init(&ctx->sste_lock);
42 ctx->afu = afu;
43 ctx->master = master;
44 ctx->pid = NULL; /* Set in start work ioctl */
45
46 /*
47 * Allocate the segment table before we put it in the IDR so that we
48 * can always access it when dereferenced from IDR. For the same
49 * reason, the segment table is only destroyed after the context is
50 * removed from the IDR. Access to this in the IOCTL is protected by
51 * Linux filesytem symantics (can't IOCTL until open is complete).
52 */
53 i = cxl_alloc_sst(ctx);
54 if (i)
55 return i;
56
57 INIT_WORK(&ctx->fault_work, cxl_handle_fault);
58
59 init_waitqueue_head(&ctx->wq);
60 spin_lock_init(&ctx->lock);
61
62 ctx->irq_bitmap = NULL;
63 ctx->pending_irq = false;
64 ctx->pending_fault = false;
65 ctx->pending_afu_err = false;
66
67 /*
68 * When we have to destroy all contexts in cxl_context_detach_all() we
69 * end up with afu_release_irqs() called from inside a
70 * idr_for_each_entry(). Hence we need to make sure that anything
71 * dereferenced from this IDR is ok before we allocate the IDR here.
72 * This clears out the IRQ ranges to ensure this.
73 */
74 for (i = 0; i < CXL_IRQ_RANGES; i++)
75 ctx->irqs.range[i] = 0;
76
77 mutex_init(&ctx->status_mutex);
78
79 ctx->status = OPENED;
80
81 /*
82 * Allocating IDR! We better make sure everything's setup that
83 * dereferences from it.
84 */
85 idr_preload(GFP_KERNEL);
86 spin_lock(&afu->contexts_lock);
87 i = idr_alloc(&ctx->afu->contexts_idr, ctx, 0,
88 ctx->afu->num_procs, GFP_NOWAIT);
89 spin_unlock(&afu->contexts_lock);
90 idr_preload_end();
91 if (i < 0)
92 return i;
93
94 ctx->pe = i;
95 ctx->elem = &ctx->afu->spa[i];
96 ctx->pe_inserted = false;
97 return 0;
98}
99
100/*
101 * Map a per-context mmio space into the given vma.
102 */
103int cxl_context_iomap(struct cxl_context *ctx, struct vm_area_struct *vma)
104{
105 u64 len = vma->vm_end - vma->vm_start;
106 len = min(len, ctx->psn_size);
107
108 if (ctx->afu->current_mode == CXL_MODE_DEDICATED) {
109 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
110 return vm_iomap_memory(vma, ctx->afu->psn_phys, ctx->afu->adapter->ps_size);
111 }
112
113 /* make sure there is a valid per process space for this AFU */
114 if ((ctx->master && !ctx->afu->psa) || (!ctx->afu->pp_psa)) {
115 pr_devel("AFU doesn't support mmio space\n");
116 return -EINVAL;
117 }
118
119 /* Can't mmap until the AFU is enabled */
120 if (!ctx->afu->enabled)
121 return -EBUSY;
122
123 pr_devel("%s: mmio physical: %llx pe: %i master:%i\n", __func__,
124 ctx->psn_phys, ctx->pe , ctx->master);
125
126 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
127 return vm_iomap_memory(vma, ctx->psn_phys, len);
128}
129
130/*
131 * Detach a context from the hardware. This disables interrupts and doesn't
132 * return until all outstanding interrupts for this context have completed. The
133 * hardware should no longer access *ctx after this has returned.
134 */
135static void __detach_context(struct cxl_context *ctx)
136{
137 enum cxl_context_status status;
138
139 mutex_lock(&ctx->status_mutex);
140 status = ctx->status;
141 ctx->status = CLOSED;
142 mutex_unlock(&ctx->status_mutex);
143 if (status != STARTED)
144 return;
145
146 WARN_ON(cxl_detach_process(ctx));
147 afu_release_irqs(ctx);
148 flush_work(&ctx->fault_work); /* Only needed for dedicated process */
149 wake_up_all(&ctx->wq);
150}
151
152/*
153 * Detach the given context from the AFU. This doesn't actually
154 * free the context but it should stop the context running in hardware
155 * (ie. prevent this context from generating any further interrupts
156 * so that it can be freed).
157 */
158void cxl_context_detach(struct cxl_context *ctx)
159{
160 __detach_context(ctx);
161}
162
163/*
164 * Detach all contexts on the given AFU.
165 */
166void cxl_context_detach_all(struct cxl_afu *afu)
167{
168 struct cxl_context *ctx;
169 int tmp;
170
171 rcu_read_lock();
172 idr_for_each_entry(&afu->contexts_idr, ctx, tmp)
173 /*
174 * Anything done in here needs to be setup before the IDR is
175 * created and torn down after the IDR removed
176 */
177 __detach_context(ctx);
178 rcu_read_unlock();
179}
180
181void cxl_context_free(struct cxl_context *ctx)
182{
183 spin_lock(&ctx->afu->contexts_lock);
184 idr_remove(&ctx->afu->contexts_idr, ctx->pe);
185 spin_unlock(&ctx->afu->contexts_lock);
186 synchronize_rcu();
187
188 free_page((u64)ctx->sstp);
189 ctx->sstp = NULL;
190
191 put_pid(ctx->pid);
192 kfree(ctx);
193}
diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h
new file mode 100644
index 000000000000..3d2b8677ec8a
--- /dev/null
+++ b/drivers/misc/cxl/cxl.h
@@ -0,0 +1,629 @@
1/*
2 * Copyright 2014 IBM Corp.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */
9
10#ifndef _CXL_H_
11#define _CXL_H_
12
13#include <linux/interrupt.h>
14#include <linux/semaphore.h>
15#include <linux/device.h>
16#include <linux/types.h>
17#include <linux/cdev.h>
18#include <linux/pid.h>
19#include <linux/io.h>
20#include <linux/pci.h>
21#include <asm/cputable.h>
22#include <asm/mmu.h>
23#include <asm/reg.h>
24#include <misc/cxl.h>
25
26#include <uapi/misc/cxl.h>
27
28extern uint cxl_verbose;
29
30#define CXL_TIMEOUT 5
31
32/*
33 * Bump version each time a user API change is made, whether it is
34 * backwards compatible ot not.
35 */
36#define CXL_API_VERSION 1
37#define CXL_API_VERSION_COMPATIBLE 1
38
39/*
40 * Opaque types to avoid accidentally passing registers for the wrong MMIO
41 *
42 * At the end of the day, I'm not married to using typedef here, but it might
43 * (and has!) help avoid bugs like mixing up CXL_PSL_CtxTime and
44 * CXL_PSL_CtxTime_An, or calling cxl_p1n_write instead of cxl_p1_write.
45 *
46 * I'm quite happy if these are changed back to #defines before upstreaming, it
47 * should be little more than a regexp search+replace operation in this file.
48 */
49typedef struct {
50 const int x;
51} cxl_p1_reg_t;
52typedef struct {
53 const int x;
54} cxl_p1n_reg_t;
55typedef struct {
56 const int x;
57} cxl_p2n_reg_t;
58#define cxl_reg_off(reg) \
59 (reg.x)
60
61/* Memory maps. Ref CXL Appendix A */
62
63/* PSL Privilege 1 Memory Map */
64/* Configuration and Control area */
65static const cxl_p1_reg_t CXL_PSL_CtxTime = {0x0000};
66static const cxl_p1_reg_t CXL_PSL_ErrIVTE = {0x0008};
67static const cxl_p1_reg_t CXL_PSL_KEY1 = {0x0010};
68static const cxl_p1_reg_t CXL_PSL_KEY2 = {0x0018};
69static const cxl_p1_reg_t CXL_PSL_Control = {0x0020};
70/* Downloading */
71static const cxl_p1_reg_t CXL_PSL_DLCNTL = {0x0060};
72static const cxl_p1_reg_t CXL_PSL_DLADDR = {0x0068};
73
74/* PSL Lookaside Buffer Management Area */
75static const cxl_p1_reg_t CXL_PSL_LBISEL = {0x0080};
76static const cxl_p1_reg_t CXL_PSL_SLBIE = {0x0088};
77static const cxl_p1_reg_t CXL_PSL_SLBIA = {0x0090};
78static const cxl_p1_reg_t CXL_PSL_TLBIE = {0x00A0};
79static const cxl_p1_reg_t CXL_PSL_TLBIA = {0x00A8};
80static const cxl_p1_reg_t CXL_PSL_AFUSEL = {0x00B0};
81
82/* 0x00C0:7EFF Implementation dependent area */
83static const cxl_p1_reg_t CXL_PSL_FIR1 = {0x0100};
84static const cxl_p1_reg_t CXL_PSL_FIR2 = {0x0108};
85static const cxl_p1_reg_t CXL_PSL_VERSION = {0x0118};
86static const cxl_p1_reg_t CXL_PSL_RESLCKTO = {0x0128};
87static const cxl_p1_reg_t CXL_PSL_FIR_CNTL = {0x0148};
88static const cxl_p1_reg_t CXL_PSL_DSNDCTL = {0x0150};
89static const cxl_p1_reg_t CXL_PSL_SNWRALLOC = {0x0158};
90static const cxl_p1_reg_t CXL_PSL_TRACE = {0x0170};
91/* 0x7F00:7FFF Reserved PCIe MSI-X Pending Bit Array area */
92/* 0x8000:FFFF Reserved PCIe MSI-X Table Area */
93
94/* PSL Slice Privilege 1 Memory Map */
95/* Configuration Area */
96static const cxl_p1n_reg_t CXL_PSL_SR_An = {0x00};
97static const cxl_p1n_reg_t CXL_PSL_LPID_An = {0x08};
98static const cxl_p1n_reg_t CXL_PSL_AMBAR_An = {0x10};
99static const cxl_p1n_reg_t CXL_PSL_SPOffset_An = {0x18};
100static const cxl_p1n_reg_t CXL_PSL_ID_An = {0x20};
101static const cxl_p1n_reg_t CXL_PSL_SERR_An = {0x28};
102/* Memory Management and Lookaside Buffer Management */
103static const cxl_p1n_reg_t CXL_PSL_SDR_An = {0x30};
104static const cxl_p1n_reg_t CXL_PSL_AMOR_An = {0x38};
105/* Pointer Area */
106static const cxl_p1n_reg_t CXL_HAURP_An = {0x80};
107static const cxl_p1n_reg_t CXL_PSL_SPAP_An = {0x88};
108static const cxl_p1n_reg_t CXL_PSL_LLCMD_An = {0x90};
109/* Control Area */
110static const cxl_p1n_reg_t CXL_PSL_SCNTL_An = {0xA0};
111static const cxl_p1n_reg_t CXL_PSL_CtxTime_An = {0xA8};
112static const cxl_p1n_reg_t CXL_PSL_IVTE_Offset_An = {0xB0};
113static const cxl_p1n_reg_t CXL_PSL_IVTE_Limit_An = {0xB8};
114/* 0xC0:FF Implementation Dependent Area */
115static const cxl_p1n_reg_t CXL_PSL_FIR_SLICE_An = {0xC0};
116static const cxl_p1n_reg_t CXL_AFU_DEBUG_An = {0xC8};
117static const cxl_p1n_reg_t CXL_PSL_APCALLOC_A = {0xD0};
118static const cxl_p1n_reg_t CXL_PSL_COALLOC_A = {0xD8};
119static const cxl_p1n_reg_t CXL_PSL_RXCTL_A = {0xE0};
120static const cxl_p1n_reg_t CXL_PSL_SLICE_TRACE = {0xE8};
121
122/* PSL Slice Privilege 2 Memory Map */
123/* Configuration and Control Area */
124static const cxl_p2n_reg_t CXL_PSL_PID_TID_An = {0x000};
125static const cxl_p2n_reg_t CXL_CSRP_An = {0x008};
126static const cxl_p2n_reg_t CXL_AURP0_An = {0x010};
127static const cxl_p2n_reg_t CXL_AURP1_An = {0x018};
128static const cxl_p2n_reg_t CXL_SSTP0_An = {0x020};
129static const cxl_p2n_reg_t CXL_SSTP1_An = {0x028};
130static const cxl_p2n_reg_t CXL_PSL_AMR_An = {0x030};
131/* Segment Lookaside Buffer Management */
132static const cxl_p2n_reg_t CXL_SLBIE_An = {0x040};
133static const cxl_p2n_reg_t CXL_SLBIA_An = {0x048};
134static const cxl_p2n_reg_t CXL_SLBI_Select_An = {0x050};
135/* Interrupt Registers */
136static const cxl_p2n_reg_t CXL_PSL_DSISR_An = {0x060};
137static const cxl_p2n_reg_t CXL_PSL_DAR_An = {0x068};
138static const cxl_p2n_reg_t CXL_PSL_DSR_An = {0x070};
139static const cxl_p2n_reg_t CXL_PSL_TFC_An = {0x078};
140static const cxl_p2n_reg_t CXL_PSL_PEHandle_An = {0x080};
141static const cxl_p2n_reg_t CXL_PSL_ErrStat_An = {0x088};
142/* AFU Registers */
143static const cxl_p2n_reg_t CXL_AFU_Cntl_An = {0x090};
144static const cxl_p2n_reg_t CXL_AFU_ERR_An = {0x098};
145/* Work Element Descriptor */
146static const cxl_p2n_reg_t CXL_PSL_WED_An = {0x0A0};
147/* 0x0C0:FFF Implementation Dependent Area */
148
149#define CXL_PSL_SPAP_Addr 0x0ffffffffffff000ULL
150#define CXL_PSL_SPAP_Size 0x0000000000000ff0ULL
151#define CXL_PSL_SPAP_Size_Shift 4
152#define CXL_PSL_SPAP_V 0x0000000000000001ULL
153
154/****** CXL_PSL_DLCNTL *****************************************************/
155#define CXL_PSL_DLCNTL_D (0x1ull << (63-28))
156#define CXL_PSL_DLCNTL_C (0x1ull << (63-29))
157#define CXL_PSL_DLCNTL_E (0x1ull << (63-30))
158#define CXL_PSL_DLCNTL_S (0x1ull << (63-31))
159#define CXL_PSL_DLCNTL_CE (CXL_PSL_DLCNTL_C | CXL_PSL_DLCNTL_E)
160#define CXL_PSL_DLCNTL_DCES (CXL_PSL_DLCNTL_D | CXL_PSL_DLCNTL_CE | CXL_PSL_DLCNTL_S)
161
162/****** CXL_PSL_SR_An ******************************************************/
163#define CXL_PSL_SR_An_SF MSR_SF /* 64bit */
164#define CXL_PSL_SR_An_TA (1ull << (63-1)) /* Tags active, GA1: 0 */
165#define CXL_PSL_SR_An_HV MSR_HV /* Hypervisor, GA1: 0 */
166#define CXL_PSL_SR_An_PR MSR_PR /* Problem state, GA1: 1 */
167#define CXL_PSL_SR_An_ISL (1ull << (63-53)) /* Ignore Segment Large Page */
168#define CXL_PSL_SR_An_TC (1ull << (63-54)) /* Page Table secondary hash */
169#define CXL_PSL_SR_An_US (1ull << (63-56)) /* User state, GA1: X */
170#define CXL_PSL_SR_An_SC (1ull << (63-58)) /* Segment Table secondary hash */
171#define CXL_PSL_SR_An_R MSR_DR /* Relocate, GA1: 1 */
172#define CXL_PSL_SR_An_MP (1ull << (63-62)) /* Master Process */
173#define CXL_PSL_SR_An_LE (1ull << (63-63)) /* Little Endian */
174
175/****** CXL_PSL_LLCMD_An ****************************************************/
176#define CXL_LLCMD_TERMINATE 0x0001000000000000ULL
177#define CXL_LLCMD_REMOVE 0x0002000000000000ULL
178#define CXL_LLCMD_SUSPEND 0x0003000000000000ULL
179#define CXL_LLCMD_RESUME 0x0004000000000000ULL
180#define CXL_LLCMD_ADD 0x0005000000000000ULL
181#define CXL_LLCMD_UPDATE 0x0006000000000000ULL
182#define CXL_LLCMD_HANDLE_MASK 0x000000000000ffffULL
183
184/****** CXL_PSL_ID_An ****************************************************/
185#define CXL_PSL_ID_An_F (1ull << (63-31))
186#define CXL_PSL_ID_An_L (1ull << (63-30))
187
188/****** CXL_PSL_SCNTL_An ****************************************************/
189#define CXL_PSL_SCNTL_An_CR (0x1ull << (63-15))
190/* Programming Modes: */
191#define CXL_PSL_SCNTL_An_PM_MASK (0xffffull << (63-31))
192#define CXL_PSL_SCNTL_An_PM_Shared (0x0000ull << (63-31))
193#define CXL_PSL_SCNTL_An_PM_OS (0x0001ull << (63-31))
194#define CXL_PSL_SCNTL_An_PM_Process (0x0002ull << (63-31))
195#define CXL_PSL_SCNTL_An_PM_AFU (0x0004ull << (63-31))
196#define CXL_PSL_SCNTL_An_PM_AFU_PBT (0x0104ull << (63-31))
197/* Purge Status (ro) */
198#define CXL_PSL_SCNTL_An_Ps_MASK (0x3ull << (63-39))
199#define CXL_PSL_SCNTL_An_Ps_Pending (0x1ull << (63-39))
200#define CXL_PSL_SCNTL_An_Ps_Complete (0x3ull << (63-39))
201/* Purge */
202#define CXL_PSL_SCNTL_An_Pc (0x1ull << (63-48))
203/* Suspend Status (ro) */
204#define CXL_PSL_SCNTL_An_Ss_MASK (0x3ull << (63-55))
205#define CXL_PSL_SCNTL_An_Ss_Pending (0x1ull << (63-55))
206#define CXL_PSL_SCNTL_An_Ss_Complete (0x3ull << (63-55))
207/* Suspend Control */
208#define CXL_PSL_SCNTL_An_Sc (0x1ull << (63-63))
209
210/* AFU Slice Enable Status (ro) */
211#define CXL_AFU_Cntl_An_ES_MASK (0x7ull << (63-2))
212#define CXL_AFU_Cntl_An_ES_Disabled (0x0ull << (63-2))
213#define CXL_AFU_Cntl_An_ES_Enabled (0x4ull << (63-2))
214/* AFU Slice Enable */
215#define CXL_AFU_Cntl_An_E (0x1ull << (63-3))
216/* AFU Slice Reset status (ro) */
217#define CXL_AFU_Cntl_An_RS_MASK (0x3ull << (63-5))
218#define CXL_AFU_Cntl_An_RS_Pending (0x1ull << (63-5))
219#define CXL_AFU_Cntl_An_RS_Complete (0x2ull << (63-5))
220/* AFU Slice Reset */
221#define CXL_AFU_Cntl_An_RA (0x1ull << (63-7))
222
223/****** CXL_SSTP0/1_An ******************************************************/
224/* These top bits are for the segment that CONTAINS the segment table */
225#define CXL_SSTP0_An_B_SHIFT SLB_VSID_SSIZE_SHIFT
226#define CXL_SSTP0_An_KS (1ull << (63-2))
227#define CXL_SSTP0_An_KP (1ull << (63-3))
228#define CXL_SSTP0_An_N (1ull << (63-4))
229#define CXL_SSTP0_An_L (1ull << (63-5))
230#define CXL_SSTP0_An_C (1ull << (63-6))
231#define CXL_SSTP0_An_TA (1ull << (63-7))
232#define CXL_SSTP0_An_LP_SHIFT (63-9) /* 2 Bits */
233/* And finally, the virtual address & size of the segment table: */
234#define CXL_SSTP0_An_SegTableSize_SHIFT (63-31) /* 12 Bits */
235#define CXL_SSTP0_An_SegTableSize_MASK \
236 (((1ull << 12) - 1) << CXL_SSTP0_An_SegTableSize_SHIFT)
237#define CXL_SSTP0_An_STVA_U_MASK ((1ull << (63-49))-1)
238#define CXL_SSTP1_An_STVA_L_MASK (~((1ull << (63-55))-1))
239#define CXL_SSTP1_An_V (1ull << (63-63))
240
241/****** CXL_PSL_SLBIE_[An] **************************************************/
242/* write: */
243#define CXL_SLBIE_C PPC_BIT(36) /* Class */
244#define CXL_SLBIE_SS PPC_BITMASK(37, 38) /* Segment Size */
245#define CXL_SLBIE_SS_SHIFT PPC_BITLSHIFT(38)
246#define CXL_SLBIE_TA PPC_BIT(38) /* Tags Active */
247/* read: */
248#define CXL_SLBIE_MAX PPC_BITMASK(24, 31)
249#define CXL_SLBIE_PENDING PPC_BITMASK(56, 63)
250
251/****** Common to all CXL_TLBIA/SLBIA_[An] **********************************/
252#define CXL_TLB_SLB_P (1ull) /* Pending (read) */
253
254/****** Common to all CXL_TLB/SLB_IA/IE_[An] registers **********************/
255#define CXL_TLB_SLB_IQ_ALL (0ull) /* Inv qualifier */
256#define CXL_TLB_SLB_IQ_LPID (1ull) /* Inv qualifier */
257#define CXL_TLB_SLB_IQ_LPIDPID (3ull) /* Inv qualifier */
258
259/****** CXL_PSL_AFUSEL ******************************************************/
260#define CXL_PSL_AFUSEL_A (1ull << (63-55)) /* Adapter wide invalidates affect all AFUs */
261
262/****** CXL_PSL_DSISR_An ****************************************************/
263#define CXL_PSL_DSISR_An_DS (1ull << (63-0)) /* Segment not found */
264#define CXL_PSL_DSISR_An_DM (1ull << (63-1)) /* PTE not found (See also: M) or protection fault */
265#define CXL_PSL_DSISR_An_ST (1ull << (63-2)) /* Segment Table PTE not found */
266#define CXL_PSL_DSISR_An_UR (1ull << (63-3)) /* AURP PTE not found */
267#define CXL_PSL_DSISR_TRANS (CXL_PSL_DSISR_An_DS | CXL_PSL_DSISR_An_DM | CXL_PSL_DSISR_An_ST | CXL_PSL_DSISR_An_UR)
268#define CXL_PSL_DSISR_An_PE (1ull << (63-4)) /* PSL Error (implementation specific) */
269#define CXL_PSL_DSISR_An_AE (1ull << (63-5)) /* AFU Error */
270#define CXL_PSL_DSISR_An_OC (1ull << (63-6)) /* OS Context Warning */
271/* NOTE: Bits 32:63 are undefined if DSISR[DS] = 1 */
272#define CXL_PSL_DSISR_An_M DSISR_NOHPTE /* PTE not found */
273#define CXL_PSL_DSISR_An_P DSISR_PROTFAULT /* Storage protection violation */
274#define CXL_PSL_DSISR_An_A (1ull << (63-37)) /* AFU lock access to write through or cache inhibited storage */
275#define CXL_PSL_DSISR_An_S DSISR_ISSTORE /* Access was afu_wr or afu_zero */
276#define CXL_PSL_DSISR_An_K DSISR_KEYFAULT /* Access not permitted by virtual page class key protection */
277
278/****** CXL_PSL_TFC_An ******************************************************/
279#define CXL_PSL_TFC_An_A (1ull << (63-28)) /* Acknowledge non-translation fault */
280#define CXL_PSL_TFC_An_C (1ull << (63-29)) /* Continue (abort transaction) */
281#define CXL_PSL_TFC_An_AE (1ull << (63-30)) /* Restart PSL with address error */
282#define CXL_PSL_TFC_An_R (1ull << (63-31)) /* Restart PSL transaction */
283
284/* cxl_process_element->software_status */
285#define CXL_PE_SOFTWARE_STATE_V (1ul << (31 - 0)) /* Valid */
286#define CXL_PE_SOFTWARE_STATE_C (1ul << (31 - 29)) /* Complete */
287#define CXL_PE_SOFTWARE_STATE_S (1ul << (31 - 30)) /* Suspend */
288#define CXL_PE_SOFTWARE_STATE_T (1ul << (31 - 31)) /* Terminate */
289
290/* SPA->sw_command_status */
291#define CXL_SPA_SW_CMD_MASK 0xffff000000000000ULL
292#define CXL_SPA_SW_CMD_TERMINATE 0x0001000000000000ULL
293#define CXL_SPA_SW_CMD_REMOVE 0x0002000000000000ULL
294#define CXL_SPA_SW_CMD_SUSPEND 0x0003000000000000ULL
295#define CXL_SPA_SW_CMD_RESUME 0x0004000000000000ULL
296#define CXL_SPA_SW_CMD_ADD 0x0005000000000000ULL
297#define CXL_SPA_SW_CMD_UPDATE 0x0006000000000000ULL
298#define CXL_SPA_SW_STATE_MASK 0x0000ffff00000000ULL
299#define CXL_SPA_SW_STATE_TERMINATED 0x0000000100000000ULL
300#define CXL_SPA_SW_STATE_REMOVED 0x0000000200000000ULL
301#define CXL_SPA_SW_STATE_SUSPENDED 0x0000000300000000ULL
302#define CXL_SPA_SW_STATE_RESUMED 0x0000000400000000ULL
303#define CXL_SPA_SW_STATE_ADDED 0x0000000500000000ULL
304#define CXL_SPA_SW_STATE_UPDATED 0x0000000600000000ULL
305#define CXL_SPA_SW_PSL_ID_MASK 0x00000000ffff0000ULL
306#define CXL_SPA_SW_LINK_MASK 0x000000000000ffffULL
307
308#define CXL_MAX_SLICES 4
309#define MAX_AFU_MMIO_REGS 3
310
311#define CXL_MODE_DEDICATED 0x1
312#define CXL_MODE_DIRECTED 0x2
313#define CXL_MODE_TIME_SLICED 0x4
314#define CXL_SUPPORTED_MODES (CXL_MODE_DEDICATED | CXL_MODE_DIRECTED)
315
316enum cxl_context_status {
317 CLOSED,
318 OPENED,
319 STARTED
320};
321
322enum prefault_modes {
323 CXL_PREFAULT_NONE,
324 CXL_PREFAULT_WED,
325 CXL_PREFAULT_ALL,
326};
327
328struct cxl_sste {
329 __be64 esid_data;
330 __be64 vsid_data;
331};
332
333#define to_cxl_adapter(d) container_of(d, struct cxl, dev)
334#define to_cxl_afu(d) container_of(d, struct cxl_afu, dev)
335
336struct cxl_afu {
337 irq_hw_number_t psl_hwirq;
338 irq_hw_number_t serr_hwirq;
339 unsigned int serr_virq;
340 void __iomem *p1n_mmio;
341 void __iomem *p2n_mmio;
342 phys_addr_t psn_phys;
343 u64 pp_offset;
344 u64 pp_size;
345 void __iomem *afu_desc_mmio;
346 struct cxl *adapter;
347 struct device dev;
348 struct cdev afu_cdev_s, afu_cdev_m, afu_cdev_d;
349 struct device *chardev_s, *chardev_m, *chardev_d;
350 struct idr contexts_idr;
351 struct dentry *debugfs;
352 spinlock_t contexts_lock;
353 struct mutex spa_mutex;
354 spinlock_t afu_cntl_lock;
355
356 /*
357 * Only the first part of the SPA is used for the process element
358 * linked list. The only other part that software needs to worry about
359 * is sw_command_status, which we store a separate pointer to.
360 * Everything else in the SPA is only used by hardware
361 */
362 struct cxl_process_element *spa;
363 __be64 *sw_command_status;
364 unsigned int spa_size;
365 int spa_order;
366 int spa_max_procs;
367 unsigned int psl_virq;
368
369 int pp_irqs;
370 int irqs_max;
371 int num_procs;
372 int max_procs_virtualised;
373 int slice;
374 int modes_supported;
375 int current_mode;
376 enum prefault_modes prefault_mode;
377 bool psa;
378 bool pp_psa;
379 bool enabled;
380};
381
382/*
383 * This is a cxl context. If the PSL is in dedicated mode, there will be one
384 * of these per AFU. If in AFU directed there can be lots of these.
385 */
386struct cxl_context {
387 struct cxl_afu *afu;
388
389 /* Problem state MMIO */
390 phys_addr_t psn_phys;
391 u64 psn_size;
392
393 spinlock_t sste_lock; /* Protects segment table entries */
394 struct cxl_sste *sstp;
395 u64 sstp0, sstp1;
396 unsigned int sst_size, sst_lru;
397
398 wait_queue_head_t wq;
399 struct pid *pid;
400 spinlock_t lock; /* Protects pending_irq_mask, pending_fault and fault_addr */
401 /* Only used in PR mode */
402 u64 process_token;
403
404 unsigned long *irq_bitmap; /* Accessed from IRQ context */
405 struct cxl_irq_ranges irqs;
406 u64 fault_addr;
407 u64 fault_dsisr;
408 u64 afu_err;
409
410 /*
411 * This status and it's lock pretects start and detach context
412 * from racing. It also prevents detach from racing with
413 * itself
414 */
415 enum cxl_context_status status;
416 struct mutex status_mutex;
417
418
419 /* XXX: Is it possible to need multiple work items at once? */
420 struct work_struct fault_work;
421 u64 dsisr;
422 u64 dar;
423
424 struct cxl_process_element *elem;
425
426 int pe; /* process element handle */
427 u32 irq_count;
428 bool pe_inserted;
429 bool master;
430 bool kernel;
431 bool pending_irq;
432 bool pending_fault;
433 bool pending_afu_err;
434};
435
436struct cxl {
437 void __iomem *p1_mmio;
438 void __iomem *p2_mmio;
439 irq_hw_number_t err_hwirq;
440 unsigned int err_virq;
441 spinlock_t afu_list_lock;
442 struct cxl_afu *afu[CXL_MAX_SLICES];
443 struct device dev;
444 struct dentry *trace;
445 struct dentry *psl_err_chk;
446 struct dentry *debugfs;
447 struct bin_attribute cxl_attr;
448 int adapter_num;
449 int user_irqs;
450 u64 afu_desc_off;
451 u64 afu_desc_size;
452 u64 ps_off;
453 u64 ps_size;
454 u16 psl_rev;
455 u16 base_image;
456 u8 vsec_status;
457 u8 caia_major;
458 u8 caia_minor;
459 u8 slices;
460 bool user_image_loaded;
461 bool perst_loads_image;
462 bool perst_select_user;
463};
464
465int cxl_alloc_one_irq(struct cxl *adapter);
466void cxl_release_one_irq(struct cxl *adapter, int hwirq);
467int cxl_alloc_irq_ranges(struct cxl_irq_ranges *irqs, struct cxl *adapter, unsigned int num);
468void cxl_release_irq_ranges(struct cxl_irq_ranges *irqs, struct cxl *adapter);
469int cxl_setup_irq(struct cxl *adapter, unsigned int hwirq, unsigned int virq);
470
471/* common == phyp + powernv */
472struct cxl_process_element_common {
473 __be32 tid;
474 __be32 pid;
475 __be64 csrp;
476 __be64 aurp0;
477 __be64 aurp1;
478 __be64 sstp0;
479 __be64 sstp1;
480 __be64 amr;
481 u8 reserved3[4];
482 __be64 wed;
483} __packed;
484
485/* just powernv */
486struct cxl_process_element {
487 __be64 sr;
488 __be64 SPOffset;
489 __be64 sdr;
490 __be64 haurp;
491 __be32 ctxtime;
492 __be16 ivte_offsets[4];
493 __be16 ivte_ranges[4];
494 __be32 lpid;
495 struct cxl_process_element_common common;
496 __be32 software_state;
497} __packed;
498
499static inline void __iomem *_cxl_p1_addr(struct cxl *cxl, cxl_p1_reg_t reg)
500{
501 WARN_ON(!cpu_has_feature(CPU_FTR_HVMODE));
502 return cxl->p1_mmio + cxl_reg_off(reg);
503}
504
505#define cxl_p1_write(cxl, reg, val) \
506 out_be64(_cxl_p1_addr(cxl, reg), val)
507#define cxl_p1_read(cxl, reg) \
508 in_be64(_cxl_p1_addr(cxl, reg))
509
510static inline void __iomem *_cxl_p1n_addr(struct cxl_afu *afu, cxl_p1n_reg_t reg)
511{
512 WARN_ON(!cpu_has_feature(CPU_FTR_HVMODE));
513 return afu->p1n_mmio + cxl_reg_off(reg);
514}
515
516#define cxl_p1n_write(afu, reg, val) \
517 out_be64(_cxl_p1n_addr(afu, reg), val)
518#define cxl_p1n_read(afu, reg) \
519 in_be64(_cxl_p1n_addr(afu, reg))
520
521static inline void __iomem *_cxl_p2n_addr(struct cxl_afu *afu, cxl_p2n_reg_t reg)
522{
523 return afu->p2n_mmio + cxl_reg_off(reg);
524}
525
526#define cxl_p2n_write(afu, reg, val) \
527 out_be64(_cxl_p2n_addr(afu, reg), val)
528#define cxl_p2n_read(afu, reg) \
529 in_be64(_cxl_p2n_addr(afu, reg))
530
531struct cxl_calls {
532 void (*cxl_slbia)(struct mm_struct *mm);
533 struct module *owner;
534};
535int register_cxl_calls(struct cxl_calls *calls);
536void unregister_cxl_calls(struct cxl_calls *calls);
537
538int cxl_alloc_adapter_nr(struct cxl *adapter);
539void cxl_remove_adapter_nr(struct cxl *adapter);
540
541int cxl_file_init(void);
542void cxl_file_exit(void);
543int cxl_register_adapter(struct cxl *adapter);
544int cxl_register_afu(struct cxl_afu *afu);
545int cxl_chardev_d_afu_add(struct cxl_afu *afu);
546int cxl_chardev_m_afu_add(struct cxl_afu *afu);
547int cxl_chardev_s_afu_add(struct cxl_afu *afu);
548void cxl_chardev_afu_remove(struct cxl_afu *afu);
549
550void cxl_context_detach_all(struct cxl_afu *afu);
551void cxl_context_free(struct cxl_context *ctx);
552void cxl_context_detach(struct cxl_context *ctx);
553
554int cxl_sysfs_adapter_add(struct cxl *adapter);
555void cxl_sysfs_adapter_remove(struct cxl *adapter);
556int cxl_sysfs_afu_add(struct cxl_afu *afu);
557void cxl_sysfs_afu_remove(struct cxl_afu *afu);
558int cxl_sysfs_afu_m_add(struct cxl_afu *afu);
559void cxl_sysfs_afu_m_remove(struct cxl_afu *afu);
560
561int cxl_afu_activate_mode(struct cxl_afu *afu, int mode);
562int _cxl_afu_deactivate_mode(struct cxl_afu *afu, int mode);
563int cxl_afu_deactivate_mode(struct cxl_afu *afu);
564int cxl_afu_select_best_mode(struct cxl_afu *afu);
565
566unsigned int cxl_map_irq(struct cxl *adapter, irq_hw_number_t hwirq,
567 irq_handler_t handler, void *cookie);
568void cxl_unmap_irq(unsigned int virq, void *cookie);
569int cxl_register_psl_irq(struct cxl_afu *afu);
570void cxl_release_psl_irq(struct cxl_afu *afu);
571int cxl_register_psl_err_irq(struct cxl *adapter);
572void cxl_release_psl_err_irq(struct cxl *adapter);
573int cxl_register_serr_irq(struct cxl_afu *afu);
574void cxl_release_serr_irq(struct cxl_afu *afu);
575int afu_register_irqs(struct cxl_context *ctx, u32 count);
576void afu_release_irqs(struct cxl_context *ctx);
577irqreturn_t cxl_slice_irq_err(int irq, void *data);
578
579int cxl_debugfs_init(void);
580void cxl_debugfs_exit(void);
581int cxl_debugfs_adapter_add(struct cxl *adapter);
582void cxl_debugfs_adapter_remove(struct cxl *adapter);
583int cxl_debugfs_afu_add(struct cxl_afu *afu);
584void cxl_debugfs_afu_remove(struct cxl_afu *afu);
585
586void cxl_handle_fault(struct work_struct *work);
587void cxl_prefault(struct cxl_context *ctx, u64 wed);
588
589struct cxl *get_cxl_adapter(int num);
590int cxl_alloc_sst(struct cxl_context *ctx);
591
592void init_cxl_native(void);
593
594struct cxl_context *cxl_context_alloc(void);
595int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master);
596void cxl_context_free(struct cxl_context *ctx);
597int cxl_context_iomap(struct cxl_context *ctx, struct vm_area_struct *vma);
598
599/* This matches the layout of the H_COLLECT_CA_INT_INFO retbuf */
600struct cxl_irq_info {
601 u64 dsisr;
602 u64 dar;
603 u64 dsr;
604 u32 pid;
605 u32 tid;
606 u64 afu_err;
607 u64 errstat;
608 u64 padding[3]; /* to match the expected retbuf size for plpar_hcall9 */
609};
610
611int cxl_attach_process(struct cxl_context *ctx, bool kernel, u64 wed,
612 u64 amr);
613int cxl_detach_process(struct cxl_context *ctx);
614
615int cxl_get_irq(struct cxl_context *ctx, struct cxl_irq_info *info);
616int cxl_ack_irq(struct cxl_context *ctx, u64 tfc, u64 psl_reset_mask);
617
618int cxl_check_error(struct cxl_afu *afu);
619int cxl_afu_slbia(struct cxl_afu *afu);
620int cxl_tlb_slb_invalidate(struct cxl *adapter);
621int cxl_afu_disable(struct cxl_afu *afu);
622int cxl_afu_reset(struct cxl_afu *afu);
623int cxl_psl_purge(struct cxl_afu *afu);
624
625void cxl_stop_trace(struct cxl *cxl);
626
627extern struct pci_driver cxl_pci_driver;
628
629#endif
diff --git a/drivers/misc/cxl/debugfs.c b/drivers/misc/cxl/debugfs.c
new file mode 100644
index 000000000000..825c412580bc
--- /dev/null
+++ b/drivers/misc/cxl/debugfs.c
@@ -0,0 +1,132 @@
1/*
2 * Copyright 2014 IBM Corp.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */
9
10#include <linux/debugfs.h>
11#include <linux/kernel.h>
12#include <linux/slab.h>
13
14#include "cxl.h"
15
16static struct dentry *cxl_debugfs;
17
18void cxl_stop_trace(struct cxl *adapter)
19{
20 int slice;
21
22 /* Stop the trace */
23 cxl_p1_write(adapter, CXL_PSL_TRACE, 0x8000000000000017LL);
24
25 /* Stop the slice traces */
26 spin_lock(&adapter->afu_list_lock);
27 for (slice = 0; slice < adapter->slices; slice++) {
28 if (adapter->afu[slice])
29 cxl_p1n_write(adapter->afu[slice], CXL_PSL_SLICE_TRACE, 0x8000000000000000LL);
30 }
31 spin_unlock(&adapter->afu_list_lock);
32}
33
34/* Helpers to export CXL mmaped IO registers via debugfs */
35static int debugfs_io_u64_get(void *data, u64 *val)
36{
37 *val = in_be64((u64 __iomem *)data);
38 return 0;
39}
40
41static int debugfs_io_u64_set(void *data, u64 val)
42{
43 out_be64((u64 __iomem *)data, val);
44 return 0;
45}
46DEFINE_SIMPLE_ATTRIBUTE(fops_io_x64, debugfs_io_u64_get, debugfs_io_u64_set, "0x%016llx\n");
47
48static struct dentry *debugfs_create_io_x64(const char *name, umode_t mode,
49 struct dentry *parent, u64 __iomem *value)
50{
51 return debugfs_create_file(name, mode, parent, (void *)value, &fops_io_x64);
52}
53
54int cxl_debugfs_adapter_add(struct cxl *adapter)
55{
56 struct dentry *dir;
57 char buf[32];
58
59 if (!cxl_debugfs)
60 return -ENODEV;
61
62 snprintf(buf, 32, "card%i", adapter->adapter_num);
63 dir = debugfs_create_dir(buf, cxl_debugfs);
64 if (IS_ERR(dir))
65 return PTR_ERR(dir);
66 adapter->debugfs = dir;
67
68 debugfs_create_io_x64("fir1", S_IRUSR, dir, _cxl_p1_addr(adapter, CXL_PSL_FIR1));
69 debugfs_create_io_x64("fir2", S_IRUSR, dir, _cxl_p1_addr(adapter, CXL_PSL_FIR2));
70 debugfs_create_io_x64("fir_cntl", S_IRUSR, dir, _cxl_p1_addr(adapter, CXL_PSL_FIR_CNTL));
71 debugfs_create_io_x64("err_ivte", S_IRUSR, dir, _cxl_p1_addr(adapter, CXL_PSL_ErrIVTE));
72
73 debugfs_create_io_x64("trace", S_IRUSR | S_IWUSR, dir, _cxl_p1_addr(adapter, CXL_PSL_TRACE));
74
75 return 0;
76}
77
78void cxl_debugfs_adapter_remove(struct cxl *adapter)
79{
80 debugfs_remove_recursive(adapter->debugfs);
81}
82
83int cxl_debugfs_afu_add(struct cxl_afu *afu)
84{
85 struct dentry *dir;
86 char buf[32];
87
88 if (!afu->adapter->debugfs)
89 return -ENODEV;
90
91 snprintf(buf, 32, "psl%i.%i", afu->adapter->adapter_num, afu->slice);
92 dir = debugfs_create_dir(buf, afu->adapter->debugfs);
93 if (IS_ERR(dir))
94 return PTR_ERR(dir);
95 afu->debugfs = dir;
96
97 debugfs_create_io_x64("fir", S_IRUSR, dir, _cxl_p1n_addr(afu, CXL_PSL_FIR_SLICE_An));
98 debugfs_create_io_x64("serr", S_IRUSR, dir, _cxl_p1n_addr(afu, CXL_PSL_SERR_An));
99 debugfs_create_io_x64("afu_debug", S_IRUSR, dir, _cxl_p1n_addr(afu, CXL_AFU_DEBUG_An));
100 debugfs_create_io_x64("sr", S_IRUSR, dir, _cxl_p1n_addr(afu, CXL_PSL_SR_An));
101
102 debugfs_create_io_x64("dsisr", S_IRUSR, dir, _cxl_p2n_addr(afu, CXL_PSL_DSISR_An));
103 debugfs_create_io_x64("dar", S_IRUSR, dir, _cxl_p2n_addr(afu, CXL_PSL_DAR_An));
104 debugfs_create_io_x64("sstp0", S_IRUSR, dir, _cxl_p2n_addr(afu, CXL_SSTP0_An));
105 debugfs_create_io_x64("sstp1", S_IRUSR, dir, _cxl_p2n_addr(afu, CXL_SSTP1_An));
106 debugfs_create_io_x64("err_status", S_IRUSR, dir, _cxl_p2n_addr(afu, CXL_PSL_ErrStat_An));
107
108 debugfs_create_io_x64("trace", S_IRUSR | S_IWUSR, dir, _cxl_p1n_addr(afu, CXL_PSL_SLICE_TRACE));
109
110 return 0;
111}
112
113void cxl_debugfs_afu_remove(struct cxl_afu *afu)
114{
115 debugfs_remove_recursive(afu->debugfs);
116}
117
118int __init cxl_debugfs_init(void)
119{
120 struct dentry *ent;
121 ent = debugfs_create_dir("cxl", NULL);
122 if (IS_ERR(ent))
123 return PTR_ERR(ent);
124 cxl_debugfs = ent;
125
126 return 0;
127}
128
129void cxl_debugfs_exit(void)
130{
131 debugfs_remove_recursive(cxl_debugfs);
132}
diff --git a/drivers/misc/cxl/fault.c b/drivers/misc/cxl/fault.c
new file mode 100644
index 000000000000..69506ebd4d07
--- /dev/null
+++ b/drivers/misc/cxl/fault.c
@@ -0,0 +1,291 @@
1/*
2 * Copyright 2014 IBM Corp.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */
9
10#include <linux/workqueue.h>
11#include <linux/sched.h>
12#include <linux/pid.h>
13#include <linux/mm.h>
14#include <linux/moduleparam.h>
15
16#undef MODULE_PARAM_PREFIX
17#define MODULE_PARAM_PREFIX "cxl" "."
18#include <asm/current.h>
19#include <asm/copro.h>
20#include <asm/mmu.h>
21
22#include "cxl.h"
23
24static struct cxl_sste* find_free_sste(struct cxl_sste *primary_group,
25 bool sec_hash,
26 struct cxl_sste *secondary_group,
27 unsigned int *lru)
28{
29 unsigned int i, entry;
30 struct cxl_sste *sste, *group = primary_group;
31
32 for (i = 0; i < 2; i++) {
33 for (entry = 0; entry < 8; entry++) {
34 sste = group + entry;
35 if (!(be64_to_cpu(sste->esid_data) & SLB_ESID_V))
36 return sste;
37 }
38 if (!sec_hash)
39 break;
40 group = secondary_group;
41 }
42 /* Nothing free, select an entry to cast out */
43 if (sec_hash && (*lru & 0x8))
44 sste = secondary_group + (*lru & 0x7);
45 else
46 sste = primary_group + (*lru & 0x7);
47 *lru = (*lru + 1) & 0xf;
48
49 return sste;
50}
51
52static void cxl_load_segment(struct cxl_context *ctx, struct copro_slb *slb)
53{
54 /* mask is the group index, we search primary and secondary here. */
55 unsigned int mask = (ctx->sst_size >> 7)-1; /* SSTP0[SegTableSize] */
56 bool sec_hash = 1;
57 struct cxl_sste *sste;
58 unsigned int hash;
59 unsigned long flags;
60
61
62 sec_hash = !!(cxl_p1n_read(ctx->afu, CXL_PSL_SR_An) & CXL_PSL_SR_An_SC);
63
64 if (slb->vsid & SLB_VSID_B_1T)
65 hash = (slb->esid >> SID_SHIFT_1T) & mask;
66 else /* 256M */
67 hash = (slb->esid >> SID_SHIFT) & mask;
68
69 spin_lock_irqsave(&ctx->sste_lock, flags);
70 sste = find_free_sste(ctx->sstp + (hash << 3), sec_hash,
71 ctx->sstp + ((~hash & mask) << 3), &ctx->sst_lru);
72
73 pr_devel("CXL Populating SST[%li]: %#llx %#llx\n",
74 sste - ctx->sstp, slb->vsid, slb->esid);
75
76 sste->vsid_data = cpu_to_be64(slb->vsid);
77 sste->esid_data = cpu_to_be64(slb->esid);
78 spin_unlock_irqrestore(&ctx->sste_lock, flags);
79}
80
81static int cxl_fault_segment(struct cxl_context *ctx, struct mm_struct *mm,
82 u64 ea)
83{
84 struct copro_slb slb = {0,0};
85 int rc;
86
87 if (!(rc = copro_calculate_slb(mm, ea, &slb))) {
88 cxl_load_segment(ctx, &slb);
89 }
90
91 return rc;
92}
93
94static void cxl_ack_ae(struct cxl_context *ctx)
95{
96 unsigned long flags;
97
98 cxl_ack_irq(ctx, CXL_PSL_TFC_An_AE, 0);
99
100 spin_lock_irqsave(&ctx->lock, flags);
101 ctx->pending_fault = true;
102 ctx->fault_addr = ctx->dar;
103 ctx->fault_dsisr = ctx->dsisr;
104 spin_unlock_irqrestore(&ctx->lock, flags);
105
106 wake_up_all(&ctx->wq);
107}
108
109static int cxl_handle_segment_miss(struct cxl_context *ctx,
110 struct mm_struct *mm, u64 ea)
111{
112 int rc;
113
114 pr_devel("CXL interrupt: Segment fault pe: %i ea: %#llx\n", ctx->pe, ea);
115
116 if ((rc = cxl_fault_segment(ctx, mm, ea)))
117 cxl_ack_ae(ctx);
118 else {
119
120 mb(); /* Order seg table write to TFC MMIO write */
121 cxl_ack_irq(ctx, CXL_PSL_TFC_An_R, 0);
122 }
123
124 return IRQ_HANDLED;
125}
126
127static void cxl_handle_page_fault(struct cxl_context *ctx,
128 struct mm_struct *mm, u64 dsisr, u64 dar)
129{
130 unsigned flt = 0;
131 int result;
132 unsigned long access, flags;
133
134 if ((result = copro_handle_mm_fault(mm, dar, dsisr, &flt))) {
135 pr_devel("copro_handle_mm_fault failed: %#x\n", result);
136 return cxl_ack_ae(ctx);
137 }
138
139 /*
140 * update_mmu_cache() will not have loaded the hash since current->trap
141 * is not a 0x400 or 0x300, so just call hash_page_mm() here.
142 */
143 access = _PAGE_PRESENT;
144 if (dsisr & CXL_PSL_DSISR_An_S)
145 access |= _PAGE_RW;
146 if ((!ctx->kernel) || ~(dar & (1ULL << 63)))
147 access |= _PAGE_USER;
148 local_irq_save(flags);
149 hash_page_mm(mm, dar, access, 0x300);
150 local_irq_restore(flags);
151
152 pr_devel("Page fault successfully handled for pe: %i!\n", ctx->pe);
153 cxl_ack_irq(ctx, CXL_PSL_TFC_An_R, 0);
154}
155
156void cxl_handle_fault(struct work_struct *fault_work)
157{
158 struct cxl_context *ctx =
159 container_of(fault_work, struct cxl_context, fault_work);
160 u64 dsisr = ctx->dsisr;
161 u64 dar = ctx->dar;
162 struct task_struct *task;
163 struct mm_struct *mm;
164
165 if (cxl_p2n_read(ctx->afu, CXL_PSL_DSISR_An) != dsisr ||
166 cxl_p2n_read(ctx->afu, CXL_PSL_DAR_An) != dar ||
167 cxl_p2n_read(ctx->afu, CXL_PSL_PEHandle_An) != ctx->pe) {
168 /* Most likely explanation is harmless - a dedicated process
169 * has detached and these were cleared by the PSL purge, but
170 * warn about it just in case */
171 dev_notice(&ctx->afu->dev, "cxl_handle_fault: Translation fault regs changed\n");
172 return;
173 }
174
175 pr_devel("CXL BOTTOM HALF handling fault for afu pe: %i. "
176 "DSISR: %#llx DAR: %#llx\n", ctx->pe, dsisr, dar);
177
178 if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) {
179 pr_devel("cxl_handle_fault unable to get task %i\n",
180 pid_nr(ctx->pid));
181 cxl_ack_ae(ctx);
182 return;
183 }
184 if (!(mm = get_task_mm(task))) {
185 pr_devel("cxl_handle_fault unable to get mm %i\n",
186 pid_nr(ctx->pid));
187 cxl_ack_ae(ctx);
188 goto out;
189 }
190
191 if (dsisr & CXL_PSL_DSISR_An_DS)
192 cxl_handle_segment_miss(ctx, mm, dar);
193 else if (dsisr & CXL_PSL_DSISR_An_DM)
194 cxl_handle_page_fault(ctx, mm, dsisr, dar);
195 else
196 WARN(1, "cxl_handle_fault has nothing to handle\n");
197
198 mmput(mm);
199out:
200 put_task_struct(task);
201}
202
203static void cxl_prefault_one(struct cxl_context *ctx, u64 ea)
204{
205 int rc;
206 struct task_struct *task;
207 struct mm_struct *mm;
208
209 if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) {
210 pr_devel("cxl_prefault_one unable to get task %i\n",
211 pid_nr(ctx->pid));
212 return;
213 }
214 if (!(mm = get_task_mm(task))) {
215 pr_devel("cxl_prefault_one unable to get mm %i\n",
216 pid_nr(ctx->pid));
217 put_task_struct(task);
218 return;
219 }
220
221 rc = cxl_fault_segment(ctx, mm, ea);
222
223 mmput(mm);
224 put_task_struct(task);
225}
226
227static u64 next_segment(u64 ea, u64 vsid)
228{
229 if (vsid & SLB_VSID_B_1T)
230 ea |= (1ULL << 40) - 1;
231 else
232 ea |= (1ULL << 28) - 1;
233
234 return ea + 1;
235}
236
237static void cxl_prefault_vma(struct cxl_context *ctx)
238{
239 u64 ea, last_esid = 0;
240 struct copro_slb slb;
241 struct vm_area_struct *vma;
242 int rc;
243 struct task_struct *task;
244 struct mm_struct *mm;
245
246 if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) {
247 pr_devel("cxl_prefault_vma unable to get task %i\n",
248 pid_nr(ctx->pid));
249 return;
250 }
251 if (!(mm = get_task_mm(task))) {
252 pr_devel("cxl_prefault_vm unable to get mm %i\n",
253 pid_nr(ctx->pid));
254 goto out1;
255 }
256
257 down_read(&mm->mmap_sem);
258 for (vma = mm->mmap; vma; vma = vma->vm_next) {
259 for (ea = vma->vm_start; ea < vma->vm_end;
260 ea = next_segment(ea, slb.vsid)) {
261 rc = copro_calculate_slb(mm, ea, &slb);
262 if (rc)
263 continue;
264
265 if (last_esid == slb.esid)
266 continue;
267
268 cxl_load_segment(ctx, &slb);
269 last_esid = slb.esid;
270 }
271 }
272 up_read(&mm->mmap_sem);
273
274 mmput(mm);
275out1:
276 put_task_struct(task);
277}
278
279void cxl_prefault(struct cxl_context *ctx, u64 wed)
280{
281 switch (ctx->afu->prefault_mode) {
282 case CXL_PREFAULT_WED:
283 cxl_prefault_one(ctx, wed);
284 break;
285 case CXL_PREFAULT_ALL:
286 cxl_prefault_vma(ctx);
287 break;
288 default:
289 break;
290 }
291}
diff --git a/drivers/misc/cxl/file.c b/drivers/misc/cxl/file.c
new file mode 100644
index 000000000000..847b7e646a7e
--- /dev/null
+++ b/drivers/misc/cxl/file.c
@@ -0,0 +1,508 @@
1/*
2 * Copyright 2014 IBM Corp.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */
9
10#include <linux/spinlock.h>
11#include <linux/module.h>
12#include <linux/export.h>
13#include <linux/kernel.h>
14#include <linux/bitmap.h>
15#include <linux/sched.h>
16#include <linux/poll.h>
17#include <linux/pid.h>
18#include <linux/fs.h>
19#include <linux/mm.h>
20#include <linux/slab.h>
21#include <asm/cputable.h>
22#include <asm/current.h>
23#include <asm/copro.h>
24
25#include "cxl.h"
26
27#define CXL_NUM_MINORS 256 /* Total to reserve */
28#define CXL_DEV_MINORS 13 /* 1 control + 4 AFUs * 3 (dedicated/master/shared) */
29
30#define CXL_CARD_MINOR(adapter) (adapter->adapter_num * CXL_DEV_MINORS)
31#define CXL_AFU_MINOR_D(afu) (CXL_CARD_MINOR(afu->adapter) + 1 + (3 * afu->slice))
32#define CXL_AFU_MINOR_M(afu) (CXL_AFU_MINOR_D(afu) + 1)
33#define CXL_AFU_MINOR_S(afu) (CXL_AFU_MINOR_D(afu) + 2)
34#define CXL_AFU_MKDEV_D(afu) MKDEV(MAJOR(cxl_dev), CXL_AFU_MINOR_D(afu))
35#define CXL_AFU_MKDEV_M(afu) MKDEV(MAJOR(cxl_dev), CXL_AFU_MINOR_M(afu))
36#define CXL_AFU_MKDEV_S(afu) MKDEV(MAJOR(cxl_dev), CXL_AFU_MINOR_S(afu))
37
38#define CXL_DEVT_ADAPTER(dev) (MINOR(dev) / CXL_DEV_MINORS)
39#define CXL_DEVT_AFU(dev) ((MINOR(dev) % CXL_DEV_MINORS - 1) / 3)
40
41#define CXL_DEVT_IS_CARD(dev) (MINOR(dev) % CXL_DEV_MINORS == 0)
42
43static dev_t cxl_dev;
44
45static struct class *cxl_class;
46
47static int __afu_open(struct inode *inode, struct file *file, bool master)
48{
49 struct cxl *adapter;
50 struct cxl_afu *afu;
51 struct cxl_context *ctx;
52 int adapter_num = CXL_DEVT_ADAPTER(inode->i_rdev);
53 int slice = CXL_DEVT_AFU(inode->i_rdev);
54 int rc = -ENODEV;
55
56 pr_devel("afu_open afu%i.%i\n", slice, adapter_num);
57
58 if (!(adapter = get_cxl_adapter(adapter_num)))
59 return -ENODEV;
60
61 if (slice > adapter->slices)
62 goto err_put_adapter;
63
64 spin_lock(&adapter->afu_list_lock);
65 if (!(afu = adapter->afu[slice])) {
66 spin_unlock(&adapter->afu_list_lock);
67 goto err_put_adapter;
68 }
69 get_device(&afu->dev);
70 spin_unlock(&adapter->afu_list_lock);
71
72 if (!afu->current_mode)
73 goto err_put_afu;
74
75 if (!(ctx = cxl_context_alloc())) {
76 rc = -ENOMEM;
77 goto err_put_afu;
78 }
79
80 if ((rc = cxl_context_init(ctx, afu, master)))
81 goto err_put_afu;
82
83 pr_devel("afu_open pe: %i\n", ctx->pe);
84 file->private_data = ctx;
85 cxl_ctx_get();
86
87 /* Our ref on the AFU will now hold the adapter */
88 put_device(&adapter->dev);
89
90 return 0;
91
92err_put_afu:
93 put_device(&afu->dev);
94err_put_adapter:
95 put_device(&adapter->dev);
96 return rc;
97}
98static int afu_open(struct inode *inode, struct file *file)
99{
100 return __afu_open(inode, file, false);
101}
102
103static int afu_master_open(struct inode *inode, struct file *file)
104{
105 return __afu_open(inode, file, true);
106}
107
108static int afu_release(struct inode *inode, struct file *file)
109{
110 struct cxl_context *ctx = file->private_data;
111
112 pr_devel("%s: closing cxl file descriptor. pe: %i\n",
113 __func__, ctx->pe);
114 cxl_context_detach(ctx);
115
116 put_device(&ctx->afu->dev);
117
118 /*
119 * At this this point all bottom halfs have finished and we should be
120 * getting no more IRQs from the hardware for this context. Once it's
121 * removed from the IDR (and RCU synchronised) it's safe to free the
122 * sstp and context.
123 */
124 cxl_context_free(ctx);
125
126 cxl_ctx_put();
127 return 0;
128}
129
130static long afu_ioctl_start_work(struct cxl_context *ctx,
131 struct cxl_ioctl_start_work __user *uwork)
132{
133 struct cxl_ioctl_start_work work;
134 u64 amr = 0;
135 int rc;
136
137 pr_devel("%s: pe: %i\n", __func__, ctx->pe);
138
139 mutex_lock(&ctx->status_mutex);
140 if (ctx->status != OPENED) {
141 rc = -EIO;
142 goto out;
143 }
144
145 if (copy_from_user(&work, uwork,
146 sizeof(struct cxl_ioctl_start_work))) {
147 rc = -EFAULT;
148 goto out;
149 }
150
151 /*
152 * if any of the reserved fields are set or any of the unused
153 * flags are set it's invalid
154 */
155 if (work.reserved1 || work.reserved2 || work.reserved3 ||
156 work.reserved4 || work.reserved5 || work.reserved6 ||
157 (work.flags & ~CXL_START_WORK_ALL)) {
158 rc = -EINVAL;
159 goto out;
160 }
161
162 if (!(work.flags & CXL_START_WORK_NUM_IRQS))
163 work.num_interrupts = ctx->afu->pp_irqs;
164 else if ((work.num_interrupts < ctx->afu->pp_irqs) ||
165 (work.num_interrupts > ctx->afu->irqs_max)) {
166 rc = -EINVAL;
167 goto out;
168 }
169 if ((rc = afu_register_irqs(ctx, work.num_interrupts)))
170 goto out;
171
172 if (work.flags & CXL_START_WORK_AMR)
173 amr = work.amr & mfspr(SPRN_UAMOR);
174
175 /*
176 * We grab the PID here and not in the file open to allow for the case
177 * where a process (master, some daemon, etc) has opened the chardev on
178 * behalf of another process, so the AFU's mm gets bound to the process
179 * that performs this ioctl and not the process that opened the file.
180 */
181 ctx->pid = get_pid(get_task_pid(current, PIDTYPE_PID));
182
183 if ((rc = cxl_attach_process(ctx, false, work.work_element_descriptor,
184 amr)))
185 goto out;
186
187 ctx->status = STARTED;
188 rc = 0;
189out:
190 mutex_unlock(&ctx->status_mutex);
191 return rc;
192}
193static long afu_ioctl_process_element(struct cxl_context *ctx,
194 int __user *upe)
195{
196 pr_devel("%s: pe: %i\n", __func__, ctx->pe);
197
198 if (copy_to_user(upe, &ctx->pe, sizeof(__u32)))
199 return -EFAULT;
200
201 return 0;
202}
203
204static long afu_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
205{
206 struct cxl_context *ctx = file->private_data;
207
208 if (ctx->status == CLOSED)
209 return -EIO;
210
211 pr_devel("afu_ioctl\n");
212 switch (cmd) {
213 case CXL_IOCTL_START_WORK:
214 return afu_ioctl_start_work(ctx, (struct cxl_ioctl_start_work __user *)arg);
215 case CXL_IOCTL_GET_PROCESS_ELEMENT:
216 return afu_ioctl_process_element(ctx, (__u32 __user *)arg);
217 }
218 return -EINVAL;
219}
220
221static long afu_compat_ioctl(struct file *file, unsigned int cmd,
222 unsigned long arg)
223{
224 return afu_ioctl(file, cmd, arg);
225}
226
227static int afu_mmap(struct file *file, struct vm_area_struct *vm)
228{
229 struct cxl_context *ctx = file->private_data;
230
231 /* AFU must be started before we can MMIO */
232 if (ctx->status != STARTED)
233 return -EIO;
234
235 return cxl_context_iomap(ctx, vm);
236}
237
238static unsigned int afu_poll(struct file *file, struct poll_table_struct *poll)
239{
240 struct cxl_context *ctx = file->private_data;
241 int mask = 0;
242 unsigned long flags;
243
244
245 poll_wait(file, &ctx->wq, poll);
246
247 pr_devel("afu_poll wait done pe: %i\n", ctx->pe);
248
249 spin_lock_irqsave(&ctx->lock, flags);
250 if (ctx->pending_irq || ctx->pending_fault ||
251 ctx->pending_afu_err)
252 mask |= POLLIN | POLLRDNORM;
253 else if (ctx->status == CLOSED)
254 /* Only error on closed when there are no futher events pending
255 */
256 mask |= POLLERR;
257 spin_unlock_irqrestore(&ctx->lock, flags);
258
259 pr_devel("afu_poll pe: %i returning %#x\n", ctx->pe, mask);
260
261 return mask;
262}
263
264static inline int ctx_event_pending(struct cxl_context *ctx)
265{
266 return (ctx->pending_irq || ctx->pending_fault ||
267 ctx->pending_afu_err || (ctx->status == CLOSED));
268}
269
270static ssize_t afu_read(struct file *file, char __user *buf, size_t count,
271 loff_t *off)
272{
273 struct cxl_context *ctx = file->private_data;
274 struct cxl_event event;
275 unsigned long flags;
276 DEFINE_WAIT(wait);
277
278 if (count < CXL_READ_MIN_SIZE)
279 return -EINVAL;
280
281 spin_lock_irqsave(&ctx->lock, flags);
282
283 for (;;) {
284 prepare_to_wait(&ctx->wq, &wait, TASK_INTERRUPTIBLE);
285 if (ctx_event_pending(ctx))
286 break;
287
288 spin_unlock_irqrestore(&ctx->lock, flags);
289 if (file->f_flags & O_NONBLOCK)
290 return -EAGAIN;
291
292 if (signal_pending(current))
293 return -ERESTARTSYS;
294
295 pr_devel("afu_read going to sleep...\n");
296 schedule();
297 pr_devel("afu_read woken up\n");
298 spin_lock_irqsave(&ctx->lock, flags);
299 }
300
301 finish_wait(&ctx->wq, &wait);
302
303 memset(&event, 0, sizeof(event));
304 event.header.process_element = ctx->pe;
305 event.header.size = sizeof(struct cxl_event_header);
306 if (ctx->pending_irq) {
307 pr_devel("afu_read delivering AFU interrupt\n");
308 event.header.size += sizeof(struct cxl_event_afu_interrupt);
309 event.header.type = CXL_EVENT_AFU_INTERRUPT;
310 event.irq.irq = find_first_bit(ctx->irq_bitmap, ctx->irq_count) + 1;
311 clear_bit(event.irq.irq - 1, ctx->irq_bitmap);
312 if (bitmap_empty(ctx->irq_bitmap, ctx->irq_count))
313 ctx->pending_irq = false;
314 } else if (ctx->pending_fault) {
315 pr_devel("afu_read delivering data storage fault\n");
316 event.header.size += sizeof(struct cxl_event_data_storage);
317 event.header.type = CXL_EVENT_DATA_STORAGE;
318 event.fault.addr = ctx->fault_addr;
319 event.fault.dsisr = ctx->fault_dsisr;
320 ctx->pending_fault = false;
321 } else if (ctx->pending_afu_err) {
322 pr_devel("afu_read delivering afu error\n");
323 event.header.size += sizeof(struct cxl_event_afu_error);
324 event.header.type = CXL_EVENT_AFU_ERROR;
325 event.afu_error.error = ctx->afu_err;
326 ctx->pending_afu_err = false;
327 } else if (ctx->status == CLOSED) {
328 pr_devel("afu_read fatal error\n");
329 spin_unlock_irqrestore(&ctx->lock, flags);
330 return -EIO;
331 } else
332 WARN(1, "afu_read must be buggy\n");
333
334 spin_unlock_irqrestore(&ctx->lock, flags);
335
336 if (copy_to_user(buf, &event, event.header.size))
337 return -EFAULT;
338 return event.header.size;
339}
340
341static const struct file_operations afu_fops = {
342 .owner = THIS_MODULE,
343 .open = afu_open,
344 .poll = afu_poll,
345 .read = afu_read,
346 .release = afu_release,
347 .unlocked_ioctl = afu_ioctl,
348 .compat_ioctl = afu_compat_ioctl,
349 .mmap = afu_mmap,
350};
351
352static const struct file_operations afu_master_fops = {
353 .owner = THIS_MODULE,
354 .open = afu_master_open,
355 .poll = afu_poll,
356 .read = afu_read,
357 .release = afu_release,
358 .unlocked_ioctl = afu_ioctl,
359 .compat_ioctl = afu_compat_ioctl,
360 .mmap = afu_mmap,
361};
362
363
364static char *cxl_devnode(struct device *dev, umode_t *mode)
365{
366 if (CXL_DEVT_IS_CARD(dev->devt)) {
367 /*
368 * These minor numbers will eventually be used to program the
369 * PSL and AFUs once we have dynamic reprogramming support
370 */
371 return NULL;
372 }
373 return kasprintf(GFP_KERNEL, "cxl/%s", dev_name(dev));
374}
375
376extern struct class *cxl_class;
377
378static int cxl_add_chardev(struct cxl_afu *afu, dev_t devt, struct cdev *cdev,
379 struct device **chardev, char *postfix, char *desc,
380 const struct file_operations *fops)
381{
382 struct device *dev;
383 int rc;
384
385 cdev_init(cdev, fops);
386 if ((rc = cdev_add(cdev, devt, 1))) {
387 dev_err(&afu->dev, "Unable to add %s chardev: %i\n", desc, rc);
388 return rc;
389 }
390
391 dev = device_create(cxl_class, &afu->dev, devt, afu,
392 "afu%i.%i%s", afu->adapter->adapter_num, afu->slice, postfix);
393 if (IS_ERR(dev)) {
394 dev_err(&afu->dev, "Unable to create %s chardev in sysfs: %i\n", desc, rc);
395 rc = PTR_ERR(dev);
396 goto err;
397 }
398
399 *chardev = dev;
400
401 return 0;
402err:
403 cdev_del(cdev);
404 return rc;
405}
406
407int cxl_chardev_d_afu_add(struct cxl_afu *afu)
408{
409 return cxl_add_chardev(afu, CXL_AFU_MKDEV_D(afu), &afu->afu_cdev_d,
410 &afu->chardev_d, "d", "dedicated",
411 &afu_master_fops); /* Uses master fops */
412}
413
414int cxl_chardev_m_afu_add(struct cxl_afu *afu)
415{
416 return cxl_add_chardev(afu, CXL_AFU_MKDEV_M(afu), &afu->afu_cdev_m,
417 &afu->chardev_m, "m", "master",
418 &afu_master_fops);
419}
420
421int cxl_chardev_s_afu_add(struct cxl_afu *afu)
422{
423 return cxl_add_chardev(afu, CXL_AFU_MKDEV_S(afu), &afu->afu_cdev_s,
424 &afu->chardev_s, "s", "shared",
425 &afu_fops);
426}
427
428void cxl_chardev_afu_remove(struct cxl_afu *afu)
429{
430 if (afu->chardev_d) {
431 cdev_del(&afu->afu_cdev_d);
432 device_unregister(afu->chardev_d);
433 afu->chardev_d = NULL;
434 }
435 if (afu->chardev_m) {
436 cdev_del(&afu->afu_cdev_m);
437 device_unregister(afu->chardev_m);
438 afu->chardev_m = NULL;
439 }
440 if (afu->chardev_s) {
441 cdev_del(&afu->afu_cdev_s);
442 device_unregister(afu->chardev_s);
443 afu->chardev_s = NULL;
444 }
445}
446
447int cxl_register_afu(struct cxl_afu *afu)
448{
449 afu->dev.class = cxl_class;
450
451 return device_register(&afu->dev);
452}
453
454int cxl_register_adapter(struct cxl *adapter)
455{
456 adapter->dev.class = cxl_class;
457
458 /*
459 * Future: When we support dynamically reprogramming the PSL & AFU we
460 * will expose the interface to do that via a chardev:
461 * adapter->dev.devt = CXL_CARD_MKDEV(adapter);
462 */
463
464 return device_register(&adapter->dev);
465}
466
467int __init cxl_file_init(void)
468{
469 int rc;
470
471 /*
472 * If these change we really need to update API. Either change some
473 * flags or update API version number CXL_API_VERSION.
474 */
475 BUILD_BUG_ON(CXL_API_VERSION != 1);
476 BUILD_BUG_ON(sizeof(struct cxl_ioctl_start_work) != 64);
477 BUILD_BUG_ON(sizeof(struct cxl_event_header) != 8);
478 BUILD_BUG_ON(sizeof(struct cxl_event_afu_interrupt) != 8);
479 BUILD_BUG_ON(sizeof(struct cxl_event_data_storage) != 32);
480 BUILD_BUG_ON(sizeof(struct cxl_event_afu_error) != 16);
481
482 if ((rc = alloc_chrdev_region(&cxl_dev, 0, CXL_NUM_MINORS, "cxl"))) {
483 pr_err("Unable to allocate CXL major number: %i\n", rc);
484 return rc;
485 }
486
487 pr_devel("CXL device allocated, MAJOR %i\n", MAJOR(cxl_dev));
488
489 cxl_class = class_create(THIS_MODULE, "cxl");
490 if (IS_ERR(cxl_class)) {
491 pr_err("Unable to create CXL class\n");
492 rc = PTR_ERR(cxl_class);
493 goto err;
494 }
495 cxl_class->devnode = cxl_devnode;
496
497 return 0;
498
499err:
500 unregister_chrdev_region(cxl_dev, CXL_NUM_MINORS);
501 return rc;
502}
503
504void cxl_file_exit(void)
505{
506 unregister_chrdev_region(cxl_dev, CXL_NUM_MINORS);
507 class_destroy(cxl_class);
508}
diff --git a/drivers/misc/cxl/irq.c b/drivers/misc/cxl/irq.c
new file mode 100644
index 000000000000..336020c8e1af
--- /dev/null
+++ b/drivers/misc/cxl/irq.c
@@ -0,0 +1,402 @@
1/*
2 * Copyright 2014 IBM Corp.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */
9
10#include <linux/interrupt.h>
11#include <linux/workqueue.h>
12#include <linux/sched.h>
13#include <linux/wait.h>
14#include <linux/slab.h>
15#include <linux/pid.h>
16#include <asm/cputable.h>
17#include <misc/cxl.h>
18
19#include "cxl.h"
20
21/* XXX: This is implementation specific */
22static irqreturn_t handle_psl_slice_error(struct cxl_context *ctx, u64 dsisr, u64 errstat)
23{
24 u64 fir1, fir2, fir_slice, serr, afu_debug;
25
26 fir1 = cxl_p1_read(ctx->afu->adapter, CXL_PSL_FIR1);
27 fir2 = cxl_p1_read(ctx->afu->adapter, CXL_PSL_FIR2);
28 fir_slice = cxl_p1n_read(ctx->afu, CXL_PSL_FIR_SLICE_An);
29 serr = cxl_p1n_read(ctx->afu, CXL_PSL_SERR_An);
30 afu_debug = cxl_p1n_read(ctx->afu, CXL_AFU_DEBUG_An);
31
32 dev_crit(&ctx->afu->dev, "PSL ERROR STATUS: 0x%.16llx\n", errstat);
33 dev_crit(&ctx->afu->dev, "PSL_FIR1: 0x%.16llx\n", fir1);
34 dev_crit(&ctx->afu->dev, "PSL_FIR2: 0x%.16llx\n", fir2);
35 dev_crit(&ctx->afu->dev, "PSL_SERR_An: 0x%.16llx\n", serr);
36 dev_crit(&ctx->afu->dev, "PSL_FIR_SLICE_An: 0x%.16llx\n", fir_slice);
37 dev_crit(&ctx->afu->dev, "CXL_PSL_AFU_DEBUG_An: 0x%.16llx\n", afu_debug);
38
39 dev_crit(&ctx->afu->dev, "STOPPING CXL TRACE\n");
40 cxl_stop_trace(ctx->afu->adapter);
41
42 return cxl_ack_irq(ctx, 0, errstat);
43}
44
45irqreturn_t cxl_slice_irq_err(int irq, void *data)
46{
47 struct cxl_afu *afu = data;
48 u64 fir_slice, errstat, serr, afu_debug;
49
50 WARN(irq, "CXL SLICE ERROR interrupt %i\n", irq);
51
52 serr = cxl_p1n_read(afu, CXL_PSL_SERR_An);
53 fir_slice = cxl_p1n_read(afu, CXL_PSL_FIR_SLICE_An);
54 errstat = cxl_p2n_read(afu, CXL_PSL_ErrStat_An);
55 afu_debug = cxl_p1n_read(afu, CXL_AFU_DEBUG_An);
56 dev_crit(&afu->dev, "PSL_SERR_An: 0x%.16llx\n", serr);
57 dev_crit(&afu->dev, "PSL_FIR_SLICE_An: 0x%.16llx\n", fir_slice);
58 dev_crit(&afu->dev, "CXL_PSL_ErrStat_An: 0x%.16llx\n", errstat);
59 dev_crit(&afu->dev, "CXL_PSL_AFU_DEBUG_An: 0x%.16llx\n", afu_debug);
60
61 cxl_p1n_write(afu, CXL_PSL_SERR_An, serr);
62
63 return IRQ_HANDLED;
64}
65
66static irqreturn_t cxl_irq_err(int irq, void *data)
67{
68 struct cxl *adapter = data;
69 u64 fir1, fir2, err_ivte;
70
71 WARN(1, "CXL ERROR interrupt %i\n", irq);
72
73 err_ivte = cxl_p1_read(adapter, CXL_PSL_ErrIVTE);
74 dev_crit(&adapter->dev, "PSL_ErrIVTE: 0x%.16llx\n", err_ivte);
75
76 dev_crit(&adapter->dev, "STOPPING CXL TRACE\n");
77 cxl_stop_trace(adapter);
78
79 fir1 = cxl_p1_read(adapter, CXL_PSL_FIR1);
80 fir2 = cxl_p1_read(adapter, CXL_PSL_FIR2);
81
82 dev_crit(&adapter->dev, "PSL_FIR1: 0x%.16llx\nPSL_FIR2: 0x%.16llx\n", fir1, fir2);
83
84 return IRQ_HANDLED;
85}
86
87static irqreturn_t schedule_cxl_fault(struct cxl_context *ctx, u64 dsisr, u64 dar)
88{
89 ctx->dsisr = dsisr;
90 ctx->dar = dar;
91 schedule_work(&ctx->fault_work);
92 return IRQ_HANDLED;
93}
94
95static irqreturn_t cxl_irq(int irq, void *data)
96{
97 struct cxl_context *ctx = data;
98 struct cxl_irq_info irq_info;
99 u64 dsisr, dar;
100 int result;
101
102 if ((result = cxl_get_irq(ctx, &irq_info))) {
103 WARN(1, "Unable to get CXL IRQ Info: %i\n", result);
104 return IRQ_HANDLED;
105 }
106
107 dsisr = irq_info.dsisr;
108 dar = irq_info.dar;
109
110 pr_devel("CXL interrupt %i for afu pe: %i DSISR: %#llx DAR: %#llx\n", irq, ctx->pe, dsisr, dar);
111
112 if (dsisr & CXL_PSL_DSISR_An_DS) {
113 /*
114 * We don't inherently need to sleep to handle this, but we do
115 * need to get a ref to the task's mm, which we can't do from
116 * irq context without the potential for a deadlock since it
117 * takes the task_lock. An alternate option would be to keep a
118 * reference to the task's mm the entire time it has cxl open,
119 * but to do that we need to solve the issue where we hold a
120 * ref to the mm, but the mm can hold a ref to the fd after an
121 * mmap preventing anything from being cleaned up.
122 */
123 pr_devel("Scheduling segment miss handling for later pe: %i\n", ctx->pe);
124 return schedule_cxl_fault(ctx, dsisr, dar);
125 }
126
127 if (dsisr & CXL_PSL_DSISR_An_M)
128 pr_devel("CXL interrupt: PTE not found\n");
129 if (dsisr & CXL_PSL_DSISR_An_P)
130 pr_devel("CXL interrupt: Storage protection violation\n");
131 if (dsisr & CXL_PSL_DSISR_An_A)
132 pr_devel("CXL interrupt: AFU lock access to write through or cache inhibited storage\n");
133 if (dsisr & CXL_PSL_DSISR_An_S)
134 pr_devel("CXL interrupt: Access was afu_wr or afu_zero\n");
135 if (dsisr & CXL_PSL_DSISR_An_K)
136 pr_devel("CXL interrupt: Access not permitted by virtual page class key protection\n");
137
138 if (dsisr & CXL_PSL_DSISR_An_DM) {
139 /*
140 * In some cases we might be able to handle the fault
141 * immediately if hash_page would succeed, but we still need
142 * the task's mm, which as above we can't get without a lock
143 */
144 pr_devel("Scheduling page fault handling for later pe: %i\n", ctx->pe);
145 return schedule_cxl_fault(ctx, dsisr, dar);
146 }
147 if (dsisr & CXL_PSL_DSISR_An_ST)
148 WARN(1, "CXL interrupt: Segment Table PTE not found\n");
149 if (dsisr & CXL_PSL_DSISR_An_UR)
150 pr_devel("CXL interrupt: AURP PTE not found\n");
151 if (dsisr & CXL_PSL_DSISR_An_PE)
152 return handle_psl_slice_error(ctx, dsisr, irq_info.errstat);
153 if (dsisr & CXL_PSL_DSISR_An_AE) {
154 pr_devel("CXL interrupt: AFU Error %.llx\n", irq_info.afu_err);
155
156 if (ctx->pending_afu_err) {
157 /*
158 * This shouldn't happen - the PSL treats these errors
159 * as fatal and will have reset the AFU, so there's not
160 * much point buffering multiple AFU errors.
161 * OTOH if we DO ever see a storm of these come in it's
162 * probably best that we log them somewhere:
163 */
164 dev_err_ratelimited(&ctx->afu->dev, "CXL AFU Error "
165 "undelivered to pe %i: %.llx\n",
166 ctx->pe, irq_info.afu_err);
167 } else {
168 spin_lock(&ctx->lock);
169 ctx->afu_err = irq_info.afu_err;
170 ctx->pending_afu_err = 1;
171 spin_unlock(&ctx->lock);
172
173 wake_up_all(&ctx->wq);
174 }
175
176 cxl_ack_irq(ctx, CXL_PSL_TFC_An_A, 0);
177 }
178 if (dsisr & CXL_PSL_DSISR_An_OC)
179 pr_devel("CXL interrupt: OS Context Warning\n");
180
181 WARN(1, "Unhandled CXL PSL IRQ\n");
182 return IRQ_HANDLED;
183}
184
185static irqreturn_t cxl_irq_multiplexed(int irq, void *data)
186{
187 struct cxl_afu *afu = data;
188 struct cxl_context *ctx;
189 int ph = cxl_p2n_read(afu, CXL_PSL_PEHandle_An) & 0xffff;
190 int ret;
191
192 rcu_read_lock();
193 ctx = idr_find(&afu->contexts_idr, ph);
194 if (ctx) {
195 ret = cxl_irq(irq, ctx);
196 rcu_read_unlock();
197 return ret;
198 }
199 rcu_read_unlock();
200
201 WARN(1, "Unable to demultiplex CXL PSL IRQ\n");
202 return IRQ_HANDLED;
203}
204
205static irqreturn_t cxl_irq_afu(int irq, void *data)
206{
207 struct cxl_context *ctx = data;
208 irq_hw_number_t hwirq = irqd_to_hwirq(irq_get_irq_data(irq));
209 int irq_off, afu_irq = 1;
210 __u16 range;
211 int r;
212
213 for (r = 1; r < CXL_IRQ_RANGES; r++) {
214 irq_off = hwirq - ctx->irqs.offset[r];
215 range = ctx->irqs.range[r];
216 if (irq_off >= 0 && irq_off < range) {
217 afu_irq += irq_off;
218 break;
219 }
220 afu_irq += range;
221 }
222 if (unlikely(r >= CXL_IRQ_RANGES)) {
223 WARN(1, "Recieved AFU IRQ out of range for pe %i (virq %i hwirq %lx)\n",
224 ctx->pe, irq, hwirq);
225 return IRQ_HANDLED;
226 }
227
228 pr_devel("Received AFU interrupt %i for pe: %i (virq %i hwirq %lx)\n",
229 afu_irq, ctx->pe, irq, hwirq);
230
231 if (unlikely(!ctx->irq_bitmap)) {
232 WARN(1, "Recieved AFU IRQ for context with no IRQ bitmap\n");
233 return IRQ_HANDLED;
234 }
235 spin_lock(&ctx->lock);
236 set_bit(afu_irq - 1, ctx->irq_bitmap);
237 ctx->pending_irq = true;
238 spin_unlock(&ctx->lock);
239
240 wake_up_all(&ctx->wq);
241
242 return IRQ_HANDLED;
243}
244
245unsigned int cxl_map_irq(struct cxl *adapter, irq_hw_number_t hwirq,
246 irq_handler_t handler, void *cookie)
247{
248 unsigned int virq;
249 int result;
250
251 /* IRQ Domain? */
252 virq = irq_create_mapping(NULL, hwirq);
253 if (!virq) {
254 dev_warn(&adapter->dev, "cxl_map_irq: irq_create_mapping failed\n");
255 return 0;
256 }
257
258 cxl_setup_irq(adapter, hwirq, virq);
259
260 pr_devel("hwirq %#lx mapped to virq %u\n", hwirq, virq);
261
262 result = request_irq(virq, handler, 0, "cxl", cookie);
263 if (result) {
264 dev_warn(&adapter->dev, "cxl_map_irq: request_irq failed: %i\n", result);
265 return 0;
266 }
267
268 return virq;
269}
270
271void cxl_unmap_irq(unsigned int virq, void *cookie)
272{
273 free_irq(virq, cookie);
274 irq_dispose_mapping(virq);
275}
276
277static int cxl_register_one_irq(struct cxl *adapter,
278 irq_handler_t handler,
279 void *cookie,
280 irq_hw_number_t *dest_hwirq,
281 unsigned int *dest_virq)
282{
283 int hwirq, virq;
284
285 if ((hwirq = cxl_alloc_one_irq(adapter)) < 0)
286 return hwirq;
287
288 if (!(virq = cxl_map_irq(adapter, hwirq, handler, cookie)))
289 goto err;
290
291 *dest_hwirq = hwirq;
292 *dest_virq = virq;
293
294 return 0;
295
296err:
297 cxl_release_one_irq(adapter, hwirq);
298 return -ENOMEM;
299}
300
301int cxl_register_psl_err_irq(struct cxl *adapter)
302{
303 int rc;
304
305 if ((rc = cxl_register_one_irq(adapter, cxl_irq_err, adapter,
306 &adapter->err_hwirq,
307 &adapter->err_virq)))
308 return rc;
309
310 cxl_p1_write(adapter, CXL_PSL_ErrIVTE, adapter->err_hwirq & 0xffff);
311
312 return 0;
313}
314
315void cxl_release_psl_err_irq(struct cxl *adapter)
316{
317 cxl_p1_write(adapter, CXL_PSL_ErrIVTE, 0x0000000000000000);
318 cxl_unmap_irq(adapter->err_virq, adapter);
319 cxl_release_one_irq(adapter, adapter->err_hwirq);
320}
321
322int cxl_register_serr_irq(struct cxl_afu *afu)
323{
324 u64 serr;
325 int rc;
326
327 if ((rc = cxl_register_one_irq(afu->adapter, cxl_slice_irq_err, afu,
328 &afu->serr_hwirq,
329 &afu->serr_virq)))
330 return rc;
331
332 serr = cxl_p1n_read(afu, CXL_PSL_SERR_An);
333 serr = (serr & 0x00ffffffffff0000ULL) | (afu->serr_hwirq & 0xffff);
334 cxl_p1n_write(afu, CXL_PSL_SERR_An, serr);
335
336 return 0;
337}
338
339void cxl_release_serr_irq(struct cxl_afu *afu)
340{
341 cxl_p1n_write(afu, CXL_PSL_SERR_An, 0x0000000000000000);
342 cxl_unmap_irq(afu->serr_virq, afu);
343 cxl_release_one_irq(afu->adapter, afu->serr_hwirq);
344}
345
346int cxl_register_psl_irq(struct cxl_afu *afu)
347{
348 return cxl_register_one_irq(afu->adapter, cxl_irq_multiplexed, afu,
349 &afu->psl_hwirq, &afu->psl_virq);
350}
351
352void cxl_release_psl_irq(struct cxl_afu *afu)
353{
354 cxl_unmap_irq(afu->psl_virq, afu);
355 cxl_release_one_irq(afu->adapter, afu->psl_hwirq);
356}
357
358int afu_register_irqs(struct cxl_context *ctx, u32 count)
359{
360 irq_hw_number_t hwirq;
361 int rc, r, i;
362
363 if ((rc = cxl_alloc_irq_ranges(&ctx->irqs, ctx->afu->adapter, count)))
364 return rc;
365
366 /* Multiplexed PSL Interrupt */
367 ctx->irqs.offset[0] = ctx->afu->psl_hwirq;
368 ctx->irqs.range[0] = 1;
369
370 ctx->irq_count = count;
371 ctx->irq_bitmap = kcalloc(BITS_TO_LONGS(count),
372 sizeof(*ctx->irq_bitmap), GFP_KERNEL);
373 if (!ctx->irq_bitmap)
374 return -ENOMEM;
375 for (r = 1; r < CXL_IRQ_RANGES; r++) {
376 hwirq = ctx->irqs.offset[r];
377 for (i = 0; i < ctx->irqs.range[r]; hwirq++, i++) {
378 cxl_map_irq(ctx->afu->adapter, hwirq,
379 cxl_irq_afu, ctx);
380 }
381 }
382
383 return 0;
384}
385
386void afu_release_irqs(struct cxl_context *ctx)
387{
388 irq_hw_number_t hwirq;
389 unsigned int virq;
390 int r, i;
391
392 for (r = 1; r < CXL_IRQ_RANGES; r++) {
393 hwirq = ctx->irqs.offset[r];
394 for (i = 0; i < ctx->irqs.range[r]; hwirq++, i++) {
395 virq = irq_find_mapping(NULL, hwirq);
396 if (virq)
397 cxl_unmap_irq(virq, ctx);
398 }
399 }
400
401 cxl_release_irq_ranges(&ctx->irqs, ctx->afu->adapter);
402}
diff --git a/drivers/misc/cxl/main.c b/drivers/misc/cxl/main.c
new file mode 100644
index 000000000000..4cde9b661642
--- /dev/null
+++ b/drivers/misc/cxl/main.c
@@ -0,0 +1,230 @@
1/*
2 * Copyright 2014 IBM Corp.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */
9
10#include <linux/spinlock.h>
11#include <linux/kernel.h>
12#include <linux/module.h>
13#include <linux/device.h>
14#include <linux/mutex.h>
15#include <linux/init.h>
16#include <linux/list.h>
17#include <linux/mm.h>
18#include <linux/of.h>
19#include <linux/slab.h>
20#include <linux/idr.h>
21#include <linux/pci.h>
22#include <asm/cputable.h>
23#include <misc/cxl.h>
24
25#include "cxl.h"
26
27static DEFINE_SPINLOCK(adapter_idr_lock);
28static DEFINE_IDR(cxl_adapter_idr);
29
30uint cxl_verbose;
31module_param_named(verbose, cxl_verbose, uint, 0600);
32MODULE_PARM_DESC(verbose, "Enable verbose dmesg output");
33
34static inline void _cxl_slbia(struct cxl_context *ctx, struct mm_struct *mm)
35{
36 struct task_struct *task;
37 unsigned long flags;
38 if (!(task = get_pid_task(ctx->pid, PIDTYPE_PID))) {
39 pr_devel("%s unable to get task %i\n",
40 __func__, pid_nr(ctx->pid));
41 return;
42 }
43
44 if (task->mm != mm)
45 goto out_put;
46
47 pr_devel("%s matched mm - card: %i afu: %i pe: %i\n", __func__,
48 ctx->afu->adapter->adapter_num, ctx->afu->slice, ctx->pe);
49
50 spin_lock_irqsave(&ctx->sste_lock, flags);
51 memset(ctx->sstp, 0, ctx->sst_size);
52 spin_unlock_irqrestore(&ctx->sste_lock, flags);
53 mb();
54 cxl_afu_slbia(ctx->afu);
55out_put:
56 put_task_struct(task);
57}
58
59static inline void cxl_slbia_core(struct mm_struct *mm)
60{
61 struct cxl *adapter;
62 struct cxl_afu *afu;
63 struct cxl_context *ctx;
64 int card, slice, id;
65
66 pr_devel("%s called\n", __func__);
67
68 spin_lock(&adapter_idr_lock);
69 idr_for_each_entry(&cxl_adapter_idr, adapter, card) {
70 /* XXX: Make this lookup faster with link from mm to ctx */
71 spin_lock(&adapter->afu_list_lock);
72 for (slice = 0; slice < adapter->slices; slice++) {
73 afu = adapter->afu[slice];
74 if (!afu->enabled)
75 continue;
76 rcu_read_lock();
77 idr_for_each_entry(&afu->contexts_idr, ctx, id)
78 _cxl_slbia(ctx, mm);
79 rcu_read_unlock();
80 }
81 spin_unlock(&adapter->afu_list_lock);
82 }
83 spin_unlock(&adapter_idr_lock);
84}
85
86static struct cxl_calls cxl_calls = {
87 .cxl_slbia = cxl_slbia_core,
88 .owner = THIS_MODULE,
89};
90
91int cxl_alloc_sst(struct cxl_context *ctx)
92{
93 unsigned long vsid;
94 u64 ea_mask, size, sstp0, sstp1;
95
96 sstp0 = 0;
97 sstp1 = 0;
98
99 ctx->sst_size = PAGE_SIZE;
100 ctx->sst_lru = 0;
101 ctx->sstp = (struct cxl_sste *)get_zeroed_page(GFP_KERNEL);
102 if (!ctx->sstp) {
103 pr_err("cxl_alloc_sst: Unable to allocate segment table\n");
104 return -ENOMEM;
105 }
106 pr_devel("SSTP allocated at 0x%p\n", ctx->sstp);
107
108 vsid = get_kernel_vsid((u64)ctx->sstp, mmu_kernel_ssize) << 12;
109
110 sstp0 |= (u64)mmu_kernel_ssize << CXL_SSTP0_An_B_SHIFT;
111 sstp0 |= (SLB_VSID_KERNEL | mmu_psize_defs[mmu_linear_psize].sllp) << 50;
112
113 size = (((u64)ctx->sst_size >> 8) - 1) << CXL_SSTP0_An_SegTableSize_SHIFT;
114 if (unlikely(size & ~CXL_SSTP0_An_SegTableSize_MASK)) {
115 WARN(1, "Impossible segment table size\n");
116 return -EINVAL;
117 }
118 sstp0 |= size;
119
120 if (mmu_kernel_ssize == MMU_SEGSIZE_256M)
121 ea_mask = 0xfffff00ULL;
122 else
123 ea_mask = 0xffffffff00ULL;
124
125 sstp0 |= vsid >> (50-14); /* Top 14 bits of VSID */
126 sstp1 |= (vsid << (64-(50-14))) & ~ea_mask;
127 sstp1 |= (u64)ctx->sstp & ea_mask;
128 sstp1 |= CXL_SSTP1_An_V;
129
130 pr_devel("Looked up %#llx: slbfee. %#llx (ssize: %x, vsid: %#lx), copied to SSTP0: %#llx, SSTP1: %#llx\n",
131 (u64)ctx->sstp, (u64)ctx->sstp & ESID_MASK, mmu_kernel_ssize, vsid, sstp0, sstp1);
132
133 /* Store calculated sstp hardware points for use later */
134 ctx->sstp0 = sstp0;
135 ctx->sstp1 = sstp1;
136
137 return 0;
138}
139
140/* Find a CXL adapter by it's number and increase it's refcount */
141struct cxl *get_cxl_adapter(int num)
142{
143 struct cxl *adapter;
144
145 spin_lock(&adapter_idr_lock);
146 if ((adapter = idr_find(&cxl_adapter_idr, num)))
147 get_device(&adapter->dev);
148 spin_unlock(&adapter_idr_lock);
149
150 return adapter;
151}
152
153int cxl_alloc_adapter_nr(struct cxl *adapter)
154{
155 int i;
156
157 idr_preload(GFP_KERNEL);
158 spin_lock(&adapter_idr_lock);
159 i = idr_alloc(&cxl_adapter_idr, adapter, 0, 0, GFP_NOWAIT);
160 spin_unlock(&adapter_idr_lock);
161 idr_preload_end();
162 if (i < 0)
163 return i;
164
165 adapter->adapter_num = i;
166
167 return 0;
168}
169
170void cxl_remove_adapter_nr(struct cxl *adapter)
171{
172 idr_remove(&cxl_adapter_idr, adapter->adapter_num);
173}
174
175int cxl_afu_select_best_mode(struct cxl_afu *afu)
176{
177 if (afu->modes_supported & CXL_MODE_DIRECTED)
178 return cxl_afu_activate_mode(afu, CXL_MODE_DIRECTED);
179
180 if (afu->modes_supported & CXL_MODE_DEDICATED)
181 return cxl_afu_activate_mode(afu, CXL_MODE_DEDICATED);
182
183 dev_warn(&afu->dev, "No supported programming modes available\n");
184 /* We don't fail this so the user can inspect sysfs */
185 return 0;
186}
187
188static int __init init_cxl(void)
189{
190 int rc = 0;
191
192 if (!cpu_has_feature(CPU_FTR_HVMODE))
193 return -EPERM;
194
195 if ((rc = cxl_file_init()))
196 return rc;
197
198 cxl_debugfs_init();
199
200 if ((rc = register_cxl_calls(&cxl_calls)))
201 goto err;
202
203 if ((rc = pci_register_driver(&cxl_pci_driver)))
204 goto err1;
205
206 return 0;
207err1:
208 unregister_cxl_calls(&cxl_calls);
209err:
210 cxl_debugfs_exit();
211 cxl_file_exit();
212
213 return rc;
214}
215
216static void exit_cxl(void)
217{
218 pci_unregister_driver(&cxl_pci_driver);
219
220 cxl_debugfs_exit();
221 cxl_file_exit();
222 unregister_cxl_calls(&cxl_calls);
223}
224
225module_init(init_cxl);
226module_exit(exit_cxl);
227
228MODULE_DESCRIPTION("IBM Coherent Accelerator");
229MODULE_AUTHOR("Ian Munsie <imunsie@au1.ibm.com>");
230MODULE_LICENSE("GPL");
diff --git a/drivers/misc/cxl/native.c b/drivers/misc/cxl/native.c
new file mode 100644
index 000000000000..623286a77114
--- /dev/null
+++ b/drivers/misc/cxl/native.c
@@ -0,0 +1,683 @@
1/*
2 * Copyright 2014 IBM Corp.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */
9
10#include <linux/spinlock.h>
11#include <linux/sched.h>
12#include <linux/slab.h>
13#include <linux/sched.h>
14#include <linux/mutex.h>
15#include <linux/mm.h>
16#include <linux/uaccess.h>
17#include <asm/synch.h>
18#include <misc/cxl.h>
19
20#include "cxl.h"
21
22static int afu_control(struct cxl_afu *afu, u64 command,
23 u64 result, u64 mask, bool enabled)
24{
25 u64 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
26 unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
27
28 spin_lock(&afu->afu_cntl_lock);
29 pr_devel("AFU command starting: %llx\n", command);
30
31 cxl_p2n_write(afu, CXL_AFU_Cntl_An, AFU_Cntl | command);
32
33 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
34 while ((AFU_Cntl & mask) != result) {
35 if (time_after_eq(jiffies, timeout)) {
36 dev_warn(&afu->dev, "WARNING: AFU control timed out!\n");
37 spin_unlock(&afu->afu_cntl_lock);
38 return -EBUSY;
39 }
40 pr_devel_ratelimited("AFU control... (0x%.16llx)\n",
41 AFU_Cntl | command);
42 cpu_relax();
43 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
44 };
45 pr_devel("AFU command complete: %llx\n", command);
46 afu->enabled = enabled;
47 spin_unlock(&afu->afu_cntl_lock);
48
49 return 0;
50}
51
52static int afu_enable(struct cxl_afu *afu)
53{
54 pr_devel("AFU enable request\n");
55
56 return afu_control(afu, CXL_AFU_Cntl_An_E,
57 CXL_AFU_Cntl_An_ES_Enabled,
58 CXL_AFU_Cntl_An_ES_MASK, true);
59}
60
61int cxl_afu_disable(struct cxl_afu *afu)
62{
63 pr_devel("AFU disable request\n");
64
65 return afu_control(afu, 0, CXL_AFU_Cntl_An_ES_Disabled,
66 CXL_AFU_Cntl_An_ES_MASK, false);
67}
68
69/* This will disable as well as reset */
70int cxl_afu_reset(struct cxl_afu *afu)
71{
72 pr_devel("AFU reset request\n");
73
74 return afu_control(afu, CXL_AFU_Cntl_An_RA,
75 CXL_AFU_Cntl_An_RS_Complete | CXL_AFU_Cntl_An_ES_Disabled,
76 CXL_AFU_Cntl_An_RS_MASK | CXL_AFU_Cntl_An_ES_MASK,
77 false);
78}
79
80static int afu_check_and_enable(struct cxl_afu *afu)
81{
82 if (afu->enabled)
83 return 0;
84 return afu_enable(afu);
85}
86
87int cxl_psl_purge(struct cxl_afu *afu)
88{
89 u64 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
90 u64 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
91 u64 dsisr, dar;
92 u64 start, end;
93 unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
94
95 pr_devel("PSL purge request\n");
96
97 if ((AFU_Cntl & CXL_AFU_Cntl_An_ES_MASK) != CXL_AFU_Cntl_An_ES_Disabled) {
98 WARN(1, "psl_purge request while AFU not disabled!\n");
99 cxl_afu_disable(afu);
100 }
101
102 cxl_p1n_write(afu, CXL_PSL_SCNTL_An,
103 PSL_CNTL | CXL_PSL_SCNTL_An_Pc);
104 start = local_clock();
105 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
106 while ((PSL_CNTL & CXL_PSL_SCNTL_An_Ps_MASK)
107 == CXL_PSL_SCNTL_An_Ps_Pending) {
108 if (time_after_eq(jiffies, timeout)) {
109 dev_warn(&afu->dev, "WARNING: PSL Purge timed out!\n");
110 return -EBUSY;
111 }
112 dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
113 pr_devel_ratelimited("PSL purging... PSL_CNTL: 0x%.16llx PSL_DSISR: 0x%.16llx\n", PSL_CNTL, dsisr);
114 if (dsisr & CXL_PSL_DSISR_TRANS) {
115 dar = cxl_p2n_read(afu, CXL_PSL_DAR_An);
116 dev_notice(&afu->dev, "PSL purge terminating pending translation, DSISR: 0x%.16llx, DAR: 0x%.16llx\n", dsisr, dar);
117 cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE);
118 } else if (dsisr) {
119 dev_notice(&afu->dev, "PSL purge acknowledging pending non-translation fault, DSISR: 0x%.16llx\n", dsisr);
120 cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A);
121 } else {
122 cpu_relax();
123 }
124 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An);
125 };
126 end = local_clock();
127 pr_devel("PSL purged in %lld ns\n", end - start);
128
129 cxl_p1n_write(afu, CXL_PSL_SCNTL_An,
130 PSL_CNTL & ~CXL_PSL_SCNTL_An_Pc);
131 return 0;
132}
133
134static int spa_max_procs(int spa_size)
135{
136 /*
137 * From the CAIA:
138 * end_of_SPA_area = SPA_Base + ((n+4) * 128) + (( ((n*8) + 127) >> 7) * 128) + 255
139 * Most of that junk is really just an overly-complicated way of saying
140 * the last 256 bytes are __aligned(128), so it's really:
141 * end_of_SPA_area = end_of_PSL_queue_area + __aligned(128) 255
142 * and
143 * end_of_PSL_queue_area = SPA_Base + ((n+4) * 128) + (n*8) - 1
144 * so
145 * sizeof(SPA) = ((n+4) * 128) + (n*8) + __aligned(128) 256
146 * Ignore the alignment (which is safe in this case as long as we are
147 * careful with our rounding) and solve for n:
148 */
149 return ((spa_size / 8) - 96) / 17;
150}
151
152static int alloc_spa(struct cxl_afu *afu)
153{
154 u64 spap;
155
156 /* Work out how many pages to allocate */
157 afu->spa_order = 0;
158 do {
159 afu->spa_order++;
160 afu->spa_size = (1 << afu->spa_order) * PAGE_SIZE;
161 afu->spa_max_procs = spa_max_procs(afu->spa_size);
162 } while (afu->spa_max_procs < afu->num_procs);
163
164 WARN_ON(afu->spa_size > 0x100000); /* Max size supported by the hardware */
165
166 if (!(afu->spa = (struct cxl_process_element *)
167 __get_free_pages(GFP_KERNEL | __GFP_ZERO, afu->spa_order))) {
168 pr_err("cxl_alloc_spa: Unable to allocate scheduled process area\n");
169 return -ENOMEM;
170 }
171 pr_devel("spa pages: %i afu->spa_max_procs: %i afu->num_procs: %i\n",
172 1<<afu->spa_order, afu->spa_max_procs, afu->num_procs);
173
174 afu->sw_command_status = (__be64 *)((char *)afu->spa +
175 ((afu->spa_max_procs + 3) * 128));
176
177 spap = virt_to_phys(afu->spa) & CXL_PSL_SPAP_Addr;
178 spap |= ((afu->spa_size >> (12 - CXL_PSL_SPAP_Size_Shift)) - 1) & CXL_PSL_SPAP_Size;
179 spap |= CXL_PSL_SPAP_V;
180 pr_devel("cxl: SPA allocated at 0x%p. Max processes: %i, sw_command_status: 0x%p CXL_PSL_SPAP_An=0x%016llx\n", afu->spa, afu->spa_max_procs, afu->sw_command_status, spap);
181 cxl_p1n_write(afu, CXL_PSL_SPAP_An, spap);
182
183 return 0;
184}
185
186static void release_spa(struct cxl_afu *afu)
187{
188 free_pages((unsigned long) afu->spa, afu->spa_order);
189}
190
191int cxl_tlb_slb_invalidate(struct cxl *adapter)
192{
193 unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
194
195 pr_devel("CXL adapter wide TLBIA & SLBIA\n");
196
197 cxl_p1_write(adapter, CXL_PSL_AFUSEL, CXL_PSL_AFUSEL_A);
198
199 cxl_p1_write(adapter, CXL_PSL_TLBIA, CXL_TLB_SLB_IQ_ALL);
200 while (cxl_p1_read(adapter, CXL_PSL_TLBIA) & CXL_TLB_SLB_P) {
201 if (time_after_eq(jiffies, timeout)) {
202 dev_warn(&adapter->dev, "WARNING: CXL adapter wide TLBIA timed out!\n");
203 return -EBUSY;
204 }
205 cpu_relax();
206 }
207
208 cxl_p1_write(adapter, CXL_PSL_SLBIA, CXL_TLB_SLB_IQ_ALL);
209 while (cxl_p1_read(adapter, CXL_PSL_SLBIA) & CXL_TLB_SLB_P) {
210 if (time_after_eq(jiffies, timeout)) {
211 dev_warn(&adapter->dev, "WARNING: CXL adapter wide SLBIA timed out!\n");
212 return -EBUSY;
213 }
214 cpu_relax();
215 }
216 return 0;
217}
218
219int cxl_afu_slbia(struct cxl_afu *afu)
220{
221 unsigned long timeout = jiffies + (HZ * CXL_TIMEOUT);
222
223 pr_devel("cxl_afu_slbia issuing SLBIA command\n");
224 cxl_p2n_write(afu, CXL_SLBIA_An, CXL_TLB_SLB_IQ_ALL);
225 while (cxl_p2n_read(afu, CXL_SLBIA_An) & CXL_TLB_SLB_P) {
226 if (time_after_eq(jiffies, timeout)) {
227 dev_warn(&afu->dev, "WARNING: CXL AFU SLBIA timed out!\n");
228 return -EBUSY;
229 }
230 cpu_relax();
231 }
232 return 0;
233}
234
235static int cxl_write_sstp(struct cxl_afu *afu, u64 sstp0, u64 sstp1)
236{
237 int rc;
238
239 /* 1. Disable SSTP by writing 0 to SSTP1[V] */
240 cxl_p2n_write(afu, CXL_SSTP1_An, 0);
241
242 /* 2. Invalidate all SLB entries */
243 if ((rc = cxl_afu_slbia(afu)))
244 return rc;
245
246 /* 3. Set SSTP0_An */
247 cxl_p2n_write(afu, CXL_SSTP0_An, sstp0);
248
249 /* 4. Set SSTP1_An */
250 cxl_p2n_write(afu, CXL_SSTP1_An, sstp1);
251
252 return 0;
253}
254
255/* Using per slice version may improve performance here. (ie. SLBIA_An) */
256static void slb_invalid(struct cxl_context *ctx)
257{
258 struct cxl *adapter = ctx->afu->adapter;
259 u64 slbia;
260
261 WARN_ON(!mutex_is_locked(&ctx->afu->spa_mutex));
262
263 cxl_p1_write(adapter, CXL_PSL_LBISEL,
264 ((u64)be32_to_cpu(ctx->elem->common.pid) << 32) |
265 be32_to_cpu(ctx->elem->lpid));
266 cxl_p1_write(adapter, CXL_PSL_SLBIA, CXL_TLB_SLB_IQ_LPIDPID);
267
268 while (1) {
269 slbia = cxl_p1_read(adapter, CXL_PSL_SLBIA);
270 if (!(slbia & CXL_TLB_SLB_P))
271 break;
272 cpu_relax();
273 }
274}
275
276static int do_process_element_cmd(struct cxl_context *ctx,
277 u64 cmd, u64 pe_state)
278{
279 u64 state;
280
281 WARN_ON(!ctx->afu->enabled);
282
283 ctx->elem->software_state = cpu_to_be32(pe_state);
284 smp_wmb();
285 *(ctx->afu->sw_command_status) = cpu_to_be64(cmd | 0 | ctx->pe);
286 smp_mb();
287 cxl_p1n_write(ctx->afu, CXL_PSL_LLCMD_An, cmd | ctx->pe);
288 while (1) {
289 state = be64_to_cpup(ctx->afu->sw_command_status);
290 if (state == ~0ULL) {
291 pr_err("cxl: Error adding process element to AFU\n");
292 return -1;
293 }
294 if ((state & (CXL_SPA_SW_CMD_MASK | CXL_SPA_SW_STATE_MASK | CXL_SPA_SW_LINK_MASK)) ==
295 (cmd | (cmd >> 16) | ctx->pe))
296 break;
297 /*
298 * The command won't finish in the PSL if there are
299 * outstanding DSIs. Hence we need to yield here in
300 * case there are outstanding DSIs that we need to
301 * service. Tuning possiblity: we could wait for a
302 * while before sched
303 */
304 schedule();
305
306 }
307 return 0;
308}
309
310static int add_process_element(struct cxl_context *ctx)
311{
312 int rc = 0;
313
314 mutex_lock(&ctx->afu->spa_mutex);
315 pr_devel("%s Adding pe: %i started\n", __func__, ctx->pe);
316 if (!(rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_ADD, CXL_PE_SOFTWARE_STATE_V)))
317 ctx->pe_inserted = true;
318 pr_devel("%s Adding pe: %i finished\n", __func__, ctx->pe);
319 mutex_unlock(&ctx->afu->spa_mutex);
320 return rc;
321}
322
323static int terminate_process_element(struct cxl_context *ctx)
324{
325 int rc = 0;
326
327 /* fast path terminate if it's already invalid */
328 if (!(ctx->elem->software_state & cpu_to_be32(CXL_PE_SOFTWARE_STATE_V)))
329 return rc;
330
331 mutex_lock(&ctx->afu->spa_mutex);
332 pr_devel("%s Terminate pe: %i started\n", __func__, ctx->pe);
333 rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_TERMINATE,
334 CXL_PE_SOFTWARE_STATE_V | CXL_PE_SOFTWARE_STATE_T);
335 ctx->elem->software_state = 0; /* Remove Valid bit */
336 pr_devel("%s Terminate pe: %i finished\n", __func__, ctx->pe);
337 mutex_unlock(&ctx->afu->spa_mutex);
338 return rc;
339}
340
341static int remove_process_element(struct cxl_context *ctx)
342{
343 int rc = 0;
344
345 mutex_lock(&ctx->afu->spa_mutex);
346 pr_devel("%s Remove pe: %i started\n", __func__, ctx->pe);
347 if (!(rc = do_process_element_cmd(ctx, CXL_SPA_SW_CMD_REMOVE, 0)))
348 ctx->pe_inserted = false;
349 slb_invalid(ctx);
350 pr_devel("%s Remove pe: %i finished\n", __func__, ctx->pe);
351 mutex_unlock(&ctx->afu->spa_mutex);
352
353 return rc;
354}
355
356
357static void assign_psn_space(struct cxl_context *ctx)
358{
359 if (!ctx->afu->pp_size || ctx->master) {
360 ctx->psn_phys = ctx->afu->psn_phys;
361 ctx->psn_size = ctx->afu->adapter->ps_size;
362 } else {
363 ctx->psn_phys = ctx->afu->psn_phys +
364 (ctx->afu->pp_offset + ctx->afu->pp_size * ctx->pe);
365 ctx->psn_size = ctx->afu->pp_size;
366 }
367}
368
369static int activate_afu_directed(struct cxl_afu *afu)
370{
371 int rc;
372
373 dev_info(&afu->dev, "Activating AFU directed mode\n");
374
375 if (alloc_spa(afu))
376 return -ENOMEM;
377
378 cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_AFU);
379 cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL);
380 cxl_p1n_write(afu, CXL_PSL_ID_An, CXL_PSL_ID_An_F | CXL_PSL_ID_An_L);
381
382 afu->current_mode = CXL_MODE_DIRECTED;
383 afu->num_procs = afu->max_procs_virtualised;
384
385 if ((rc = cxl_chardev_m_afu_add(afu)))
386 return rc;
387
388 if ((rc = cxl_sysfs_afu_m_add(afu)))
389 goto err;
390
391 if ((rc = cxl_chardev_s_afu_add(afu)))
392 goto err1;
393
394 return 0;
395err1:
396 cxl_sysfs_afu_m_remove(afu);
397err:
398 cxl_chardev_afu_remove(afu);
399 return rc;
400}
401
402#ifdef CONFIG_CPU_LITTLE_ENDIAN
403#define set_endian(sr) ((sr) |= CXL_PSL_SR_An_LE)
404#else
405#define set_endian(sr) ((sr) &= ~(CXL_PSL_SR_An_LE))
406#endif
407
408static int attach_afu_directed(struct cxl_context *ctx, u64 wed, u64 amr)
409{
410 u64 sr;
411 int r, result;
412
413 assign_psn_space(ctx);
414
415 ctx->elem->ctxtime = 0; /* disable */
416 ctx->elem->lpid = cpu_to_be32(mfspr(SPRN_LPID));
417 ctx->elem->haurp = 0; /* disable */
418 ctx->elem->sdr = cpu_to_be64(mfspr(SPRN_SDR1));
419
420 sr = CXL_PSL_SR_An_SC;
421 if (ctx->master)
422 sr |= CXL_PSL_SR_An_MP;
423 if (mfspr(SPRN_LPCR) & LPCR_TC)
424 sr |= CXL_PSL_SR_An_TC;
425 /* HV=0, PR=1, R=1 for userspace
426 * For kernel contexts: this would need to change
427 */
428 sr |= CXL_PSL_SR_An_PR | CXL_PSL_SR_An_R;
429 set_endian(sr);
430 sr &= ~(CXL_PSL_SR_An_HV);
431 if (!test_tsk_thread_flag(current, TIF_32BIT))
432 sr |= CXL_PSL_SR_An_SF;
433 ctx->elem->common.pid = cpu_to_be32(current->pid);
434 ctx->elem->common.tid = 0;
435 ctx->elem->sr = cpu_to_be64(sr);
436
437 ctx->elem->common.csrp = 0; /* disable */
438 ctx->elem->common.aurp0 = 0; /* disable */
439 ctx->elem->common.aurp1 = 0; /* disable */
440
441 cxl_prefault(ctx, wed);
442
443 ctx->elem->common.sstp0 = cpu_to_be64(ctx->sstp0);
444 ctx->elem->common.sstp1 = cpu_to_be64(ctx->sstp1);
445
446 for (r = 0; r < CXL_IRQ_RANGES; r++) {
447 ctx->elem->ivte_offsets[r] = cpu_to_be16(ctx->irqs.offset[r]);
448 ctx->elem->ivte_ranges[r] = cpu_to_be16(ctx->irqs.range[r]);
449 }
450
451 ctx->elem->common.amr = cpu_to_be64(amr);
452 ctx->elem->common.wed = cpu_to_be64(wed);
453
454 /* first guy needs to enable */
455 if ((result = afu_check_and_enable(ctx->afu)))
456 return result;
457
458 add_process_element(ctx);
459
460 return 0;
461}
462
463static int deactivate_afu_directed(struct cxl_afu *afu)
464{
465 dev_info(&afu->dev, "Deactivating AFU directed mode\n");
466
467 afu->current_mode = 0;
468 afu->num_procs = 0;
469
470 cxl_sysfs_afu_m_remove(afu);
471 cxl_chardev_afu_remove(afu);
472
473 cxl_afu_reset(afu);
474 cxl_afu_disable(afu);
475 cxl_psl_purge(afu);
476
477 release_spa(afu);
478
479 return 0;
480}
481
482static int activate_dedicated_process(struct cxl_afu *afu)
483{
484 dev_info(&afu->dev, "Activating dedicated process mode\n");
485
486 cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_Process);
487
488 cxl_p1n_write(afu, CXL_PSL_CtxTime_An, 0); /* disable */
489 cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0); /* disable */
490 cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL);
491 cxl_p1n_write(afu, CXL_PSL_LPID_An, mfspr(SPRN_LPID));
492 cxl_p1n_write(afu, CXL_HAURP_An, 0); /* disable */
493 cxl_p1n_write(afu, CXL_PSL_SDR_An, mfspr(SPRN_SDR1));
494
495 cxl_p2n_write(afu, CXL_CSRP_An, 0); /* disable */
496 cxl_p2n_write(afu, CXL_AURP0_An, 0); /* disable */
497 cxl_p2n_write(afu, CXL_AURP1_An, 0); /* disable */
498
499 afu->current_mode = CXL_MODE_DEDICATED;
500 afu->num_procs = 1;
501
502 return cxl_chardev_d_afu_add(afu);
503}
504
505static int attach_dedicated(struct cxl_context *ctx, u64 wed, u64 amr)
506{
507 struct cxl_afu *afu = ctx->afu;
508 u64 sr;
509 int rc;
510
511 sr = CXL_PSL_SR_An_SC;
512 set_endian(sr);
513 if (ctx->master)
514 sr |= CXL_PSL_SR_An_MP;
515 if (mfspr(SPRN_LPCR) & LPCR_TC)
516 sr |= CXL_PSL_SR_An_TC;
517 sr |= CXL_PSL_SR_An_PR | CXL_PSL_SR_An_R;
518 if (!test_tsk_thread_flag(current, TIF_32BIT))
519 sr |= CXL_PSL_SR_An_SF;
520 cxl_p2n_write(afu, CXL_PSL_PID_TID_An, (u64)current->pid << 32);
521 cxl_p1n_write(afu, CXL_PSL_SR_An, sr);
522
523 if ((rc = cxl_write_sstp(afu, ctx->sstp0, ctx->sstp1)))
524 return rc;
525
526 cxl_prefault(ctx, wed);
527
528 cxl_p1n_write(afu, CXL_PSL_IVTE_Offset_An,
529 (((u64)ctx->irqs.offset[0] & 0xffff) << 48) |
530 (((u64)ctx->irqs.offset[1] & 0xffff) << 32) |
531 (((u64)ctx->irqs.offset[2] & 0xffff) << 16) |
532 ((u64)ctx->irqs.offset[3] & 0xffff));
533 cxl_p1n_write(afu, CXL_PSL_IVTE_Limit_An, (u64)
534 (((u64)ctx->irqs.range[0] & 0xffff) << 48) |
535 (((u64)ctx->irqs.range[1] & 0xffff) << 32) |
536 (((u64)ctx->irqs.range[2] & 0xffff) << 16) |
537 ((u64)ctx->irqs.range[3] & 0xffff));
538
539 cxl_p2n_write(afu, CXL_PSL_AMR_An, amr);
540
541 /* master only context for dedicated */
542 assign_psn_space(ctx);
543
544 if ((rc = cxl_afu_reset(afu)))
545 return rc;
546
547 cxl_p2n_write(afu, CXL_PSL_WED_An, wed);
548
549 return afu_enable(afu);
550}
551
552static int deactivate_dedicated_process(struct cxl_afu *afu)
553{
554 dev_info(&afu->dev, "Deactivating dedicated process mode\n");
555
556 afu->current_mode = 0;
557 afu->num_procs = 0;
558
559 cxl_chardev_afu_remove(afu);
560
561 return 0;
562}
563
564int _cxl_afu_deactivate_mode(struct cxl_afu *afu, int mode)
565{
566 if (mode == CXL_MODE_DIRECTED)
567 return deactivate_afu_directed(afu);
568 if (mode == CXL_MODE_DEDICATED)
569 return deactivate_dedicated_process(afu);
570 return 0;
571}
572
573int cxl_afu_deactivate_mode(struct cxl_afu *afu)
574{
575 return _cxl_afu_deactivate_mode(afu, afu->current_mode);
576}
577
578int cxl_afu_activate_mode(struct cxl_afu *afu, int mode)
579{
580 if (!mode)
581 return 0;
582 if (!(mode & afu->modes_supported))
583 return -EINVAL;
584
585 if (mode == CXL_MODE_DIRECTED)
586 return activate_afu_directed(afu);
587 if (mode == CXL_MODE_DEDICATED)
588 return activate_dedicated_process(afu);
589
590 return -EINVAL;
591}
592
593int cxl_attach_process(struct cxl_context *ctx, bool kernel, u64 wed, u64 amr)
594{
595 ctx->kernel = kernel;
596 if (ctx->afu->current_mode == CXL_MODE_DIRECTED)
597 return attach_afu_directed(ctx, wed, amr);
598
599 if (ctx->afu->current_mode == CXL_MODE_DEDICATED)
600 return attach_dedicated(ctx, wed, amr);
601
602 return -EINVAL;
603}
604
605static inline int detach_process_native_dedicated(struct cxl_context *ctx)
606{
607 cxl_afu_reset(ctx->afu);
608 cxl_afu_disable(ctx->afu);
609 cxl_psl_purge(ctx->afu);
610 return 0;
611}
612
613/*
614 * TODO: handle case when this is called inside a rcu_read_lock() which may
615 * happen when we unbind the driver (ie. cxl_context_detach_all()) . Terminate
616 * & remove use a mutex lock and schedule which will not good with lock held.
617 * May need to write do_process_element_cmd() that handles outstanding page
618 * faults synchronously.
619 */
620static inline int detach_process_native_afu_directed(struct cxl_context *ctx)
621{
622 if (!ctx->pe_inserted)
623 return 0;
624 if (terminate_process_element(ctx))
625 return -1;
626 if (remove_process_element(ctx))
627 return -1;
628
629 return 0;
630}
631
632int cxl_detach_process(struct cxl_context *ctx)
633{
634 if (ctx->afu->current_mode == CXL_MODE_DEDICATED)
635 return detach_process_native_dedicated(ctx);
636
637 return detach_process_native_afu_directed(ctx);
638}
639
640int cxl_get_irq(struct cxl_context *ctx, struct cxl_irq_info *info)
641{
642 u64 pidtid;
643
644 info->dsisr = cxl_p2n_read(ctx->afu, CXL_PSL_DSISR_An);
645 info->dar = cxl_p2n_read(ctx->afu, CXL_PSL_DAR_An);
646 info->dsr = cxl_p2n_read(ctx->afu, CXL_PSL_DSR_An);
647 pidtid = cxl_p2n_read(ctx->afu, CXL_PSL_PID_TID_An);
648 info->pid = pidtid >> 32;
649 info->tid = pidtid & 0xffffffff;
650 info->afu_err = cxl_p2n_read(ctx->afu, CXL_AFU_ERR_An);
651 info->errstat = cxl_p2n_read(ctx->afu, CXL_PSL_ErrStat_An);
652
653 return 0;
654}
655
656static void recover_psl_err(struct cxl_afu *afu, u64 errstat)
657{
658 u64 dsisr;
659
660 pr_devel("RECOVERING FROM PSL ERROR... (0x%.16llx)\n", errstat);
661
662 /* Clear PSL_DSISR[PE] */
663 dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
664 cxl_p2n_write(afu, CXL_PSL_DSISR_An, dsisr & ~CXL_PSL_DSISR_An_PE);
665
666 /* Write 1s to clear error status bits */
667 cxl_p2n_write(afu, CXL_PSL_ErrStat_An, errstat);
668}
669
670int cxl_ack_irq(struct cxl_context *ctx, u64 tfc, u64 psl_reset_mask)
671{
672 if (tfc)
673 cxl_p2n_write(ctx->afu, CXL_PSL_TFC_An, tfc);
674 if (psl_reset_mask)
675 recover_psl_err(ctx->afu, psl_reset_mask);
676
677 return 0;
678}
679
680int cxl_check_error(struct cxl_afu *afu)
681{
682 return (cxl_p1n_read(afu, CXL_PSL_SCNTL_An) == ~0ULL);
683}
diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c
new file mode 100644
index 000000000000..10c98ab7f46e
--- /dev/null
+++ b/drivers/misc/cxl/pci.c
@@ -0,0 +1,1000 @@
1/*
2 * Copyright 2014 IBM Corp.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */
9
10#include <linux/pci_regs.h>
11#include <linux/pci_ids.h>
12#include <linux/device.h>
13#include <linux/module.h>
14#include <linux/kernel.h>
15#include <linux/slab.h>
16#include <linux/sort.h>
17#include <linux/pci.h>
18#include <linux/of.h>
19#include <linux/delay.h>
20#include <asm/opal.h>
21#include <asm/msi_bitmap.h>
22#include <asm/pci-bridge.h> /* for struct pci_controller */
23#include <asm/pnv-pci.h>
24
25#include "cxl.h"
26
27
28#define CXL_PCI_VSEC_ID 0x1280
29#define CXL_VSEC_MIN_SIZE 0x80
30
31#define CXL_READ_VSEC_LENGTH(dev, vsec, dest) \
32 { \
33 pci_read_config_word(dev, vsec + 0x6, dest); \
34 *dest >>= 4; \
35 }
36#define CXL_READ_VSEC_NAFUS(dev, vsec, dest) \
37 pci_read_config_byte(dev, vsec + 0x8, dest)
38
39#define CXL_READ_VSEC_STATUS(dev, vsec, dest) \
40 pci_read_config_byte(dev, vsec + 0x9, dest)
41#define CXL_STATUS_SECOND_PORT 0x80
42#define CXL_STATUS_MSI_X_FULL 0x40
43#define CXL_STATUS_MSI_X_SINGLE 0x20
44#define CXL_STATUS_FLASH_RW 0x08
45#define CXL_STATUS_FLASH_RO 0x04
46#define CXL_STATUS_LOADABLE_AFU 0x02
47#define CXL_STATUS_LOADABLE_PSL 0x01
48/* If we see these features we won't try to use the card */
49#define CXL_UNSUPPORTED_FEATURES \
50 (CXL_STATUS_MSI_X_FULL | CXL_STATUS_MSI_X_SINGLE)
51
52#define CXL_READ_VSEC_MODE_CONTROL(dev, vsec, dest) \
53 pci_read_config_byte(dev, vsec + 0xa, dest)
54#define CXL_WRITE_VSEC_MODE_CONTROL(dev, vsec, val) \
55 pci_write_config_byte(dev, vsec + 0xa, val)
56#define CXL_VSEC_PROTOCOL_MASK 0xe0
57#define CXL_VSEC_PROTOCOL_1024TB 0x80
58#define CXL_VSEC_PROTOCOL_512TB 0x40
59#define CXL_VSEC_PROTOCOL_256TB 0x20 /* Power 8 uses this */
60#define CXL_VSEC_PROTOCOL_ENABLE 0x01
61
62#define CXL_READ_VSEC_PSL_REVISION(dev, vsec, dest) \
63 pci_read_config_word(dev, vsec + 0xc, dest)
64#define CXL_READ_VSEC_CAIA_MINOR(dev, vsec, dest) \
65 pci_read_config_byte(dev, vsec + 0xe, dest)
66#define CXL_READ_VSEC_CAIA_MAJOR(dev, vsec, dest) \
67 pci_read_config_byte(dev, vsec + 0xf, dest)
68#define CXL_READ_VSEC_BASE_IMAGE(dev, vsec, dest) \
69 pci_read_config_word(dev, vsec + 0x10, dest)
70
71#define CXL_READ_VSEC_IMAGE_STATE(dev, vsec, dest) \
72 pci_read_config_byte(dev, vsec + 0x13, dest)
73#define CXL_WRITE_VSEC_IMAGE_STATE(dev, vsec, val) \
74 pci_write_config_byte(dev, vsec + 0x13, val)
75#define CXL_VSEC_USER_IMAGE_LOADED 0x80 /* RO */
76#define CXL_VSEC_PERST_LOADS_IMAGE 0x20 /* RW */
77#define CXL_VSEC_PERST_SELECT_USER 0x10 /* RW */
78
79#define CXL_READ_VSEC_AFU_DESC_OFF(dev, vsec, dest) \
80 pci_read_config_dword(dev, vsec + 0x20, dest)
81#define CXL_READ_VSEC_AFU_DESC_SIZE(dev, vsec, dest) \
82 pci_read_config_dword(dev, vsec + 0x24, dest)
83#define CXL_READ_VSEC_PS_OFF(dev, vsec, dest) \
84 pci_read_config_dword(dev, vsec + 0x28, dest)
85#define CXL_READ_VSEC_PS_SIZE(dev, vsec, dest) \
86 pci_read_config_dword(dev, vsec + 0x2c, dest)
87
88
89/* This works a little different than the p1/p2 register accesses to make it
90 * easier to pull out individual fields */
91#define AFUD_READ(afu, off) in_be64(afu->afu_desc_mmio + off)
92#define EXTRACT_PPC_BIT(val, bit) (!!(val & PPC_BIT(bit)))
93#define EXTRACT_PPC_BITS(val, bs, be) ((val & PPC_BITMASK(bs, be)) >> PPC_BITLSHIFT(be))
94
95#define AFUD_READ_INFO(afu) AFUD_READ(afu, 0x0)
96#define AFUD_NUM_INTS_PER_PROC(val) EXTRACT_PPC_BITS(val, 0, 15)
97#define AFUD_NUM_PROCS(val) EXTRACT_PPC_BITS(val, 16, 31)
98#define AFUD_NUM_CRS(val) EXTRACT_PPC_BITS(val, 32, 47)
99#define AFUD_MULTIMODE(val) EXTRACT_PPC_BIT(val, 48)
100#define AFUD_PUSH_BLOCK_TRANSFER(val) EXTRACT_PPC_BIT(val, 55)
101#define AFUD_DEDICATED_PROCESS(val) EXTRACT_PPC_BIT(val, 59)
102#define AFUD_AFU_DIRECTED(val) EXTRACT_PPC_BIT(val, 61)
103#define AFUD_TIME_SLICED(val) EXTRACT_PPC_BIT(val, 63)
104#define AFUD_READ_CR(afu) AFUD_READ(afu, 0x20)
105#define AFUD_CR_LEN(val) EXTRACT_PPC_BITS(val, 8, 63)
106#define AFUD_READ_CR_OFF(afu) AFUD_READ(afu, 0x28)
107#define AFUD_READ_PPPSA(afu) AFUD_READ(afu, 0x30)
108#define AFUD_PPPSA_PP(val) EXTRACT_PPC_BIT(val, 6)
109#define AFUD_PPPSA_PSA(val) EXTRACT_PPC_BIT(val, 7)
110#define AFUD_PPPSA_LEN(val) EXTRACT_PPC_BITS(val, 8, 63)
111#define AFUD_READ_PPPSA_OFF(afu) AFUD_READ(afu, 0x38)
112#define AFUD_READ_EB(afu) AFUD_READ(afu, 0x40)
113#define AFUD_EB_LEN(val) EXTRACT_PPC_BITS(val, 8, 63)
114#define AFUD_READ_EB_OFF(afu) AFUD_READ(afu, 0x48)
115
116static DEFINE_PCI_DEVICE_TABLE(cxl_pci_tbl) = {
117 { PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x0477), },
118 { PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x044b), },
119 { PCI_DEVICE(PCI_VENDOR_ID_IBM, 0x04cf), },
120 { PCI_DEVICE_CLASS(0x120000, ~0), },
121
122 { }
123};
124MODULE_DEVICE_TABLE(pci, cxl_pci_tbl);
125
126
127/*
128 * Mostly using these wrappers to avoid confusion:
129 * priv 1 is BAR2, while priv 2 is BAR0
130 */
131static inline resource_size_t p1_base(struct pci_dev *dev)
132{
133 return pci_resource_start(dev, 2);
134}
135
136static inline resource_size_t p1_size(struct pci_dev *dev)
137{
138 return pci_resource_len(dev, 2);
139}
140
141static inline resource_size_t p2_base(struct pci_dev *dev)
142{
143 return pci_resource_start(dev, 0);
144}
145
146static inline resource_size_t p2_size(struct pci_dev *dev)
147{
148 return pci_resource_len(dev, 0);
149}
150
151static int find_cxl_vsec(struct pci_dev *dev)
152{
153 int vsec = 0;
154 u16 val;
155
156 while ((vsec = pci_find_next_ext_capability(dev, vsec, PCI_EXT_CAP_ID_VNDR))) {
157 pci_read_config_word(dev, vsec + 0x4, &val);
158 if (val == CXL_PCI_VSEC_ID)
159 return vsec;
160 }
161 return 0;
162
163}
164
165static void dump_cxl_config_space(struct pci_dev *dev)
166{
167 int vsec;
168 u32 val;
169
170 dev_info(&dev->dev, "dump_cxl_config_space\n");
171
172 pci_read_config_dword(dev, PCI_BASE_ADDRESS_0, &val);
173 dev_info(&dev->dev, "BAR0: %#.8x\n", val);
174 pci_read_config_dword(dev, PCI_BASE_ADDRESS_1, &val);
175 dev_info(&dev->dev, "BAR1: %#.8x\n", val);
176 pci_read_config_dword(dev, PCI_BASE_ADDRESS_2, &val);
177 dev_info(&dev->dev, "BAR2: %#.8x\n", val);
178 pci_read_config_dword(dev, PCI_BASE_ADDRESS_3, &val);
179 dev_info(&dev->dev, "BAR3: %#.8x\n", val);
180 pci_read_config_dword(dev, PCI_BASE_ADDRESS_4, &val);
181 dev_info(&dev->dev, "BAR4: %#.8x\n", val);
182 pci_read_config_dword(dev, PCI_BASE_ADDRESS_5, &val);
183 dev_info(&dev->dev, "BAR5: %#.8x\n", val);
184
185 dev_info(&dev->dev, "p1 regs: %#llx, len: %#llx\n",
186 p1_base(dev), p1_size(dev));
187 dev_info(&dev->dev, "p2 regs: %#llx, len: %#llx\n",
188 p1_base(dev), p2_size(dev));
189 dev_info(&dev->dev, "BAR 4/5: %#llx, len: %#llx\n",
190 pci_resource_start(dev, 4), pci_resource_len(dev, 4));
191
192 if (!(vsec = find_cxl_vsec(dev)))
193 return;
194
195#define show_reg(name, what) \
196 dev_info(&dev->dev, "cxl vsec: %30s: %#x\n", name, what)
197
198 pci_read_config_dword(dev, vsec + 0x0, &val);
199 show_reg("Cap ID", (val >> 0) & 0xffff);
200 show_reg("Cap Ver", (val >> 16) & 0xf);
201 show_reg("Next Cap Ptr", (val >> 20) & 0xfff);
202 pci_read_config_dword(dev, vsec + 0x4, &val);
203 show_reg("VSEC ID", (val >> 0) & 0xffff);
204 show_reg("VSEC Rev", (val >> 16) & 0xf);
205 show_reg("VSEC Length", (val >> 20) & 0xfff);
206 pci_read_config_dword(dev, vsec + 0x8, &val);
207 show_reg("Num AFUs", (val >> 0) & 0xff);
208 show_reg("Status", (val >> 8) & 0xff);
209 show_reg("Mode Control", (val >> 16) & 0xff);
210 show_reg("Reserved", (val >> 24) & 0xff);
211 pci_read_config_dword(dev, vsec + 0xc, &val);
212 show_reg("PSL Rev", (val >> 0) & 0xffff);
213 show_reg("CAIA Ver", (val >> 16) & 0xffff);
214 pci_read_config_dword(dev, vsec + 0x10, &val);
215 show_reg("Base Image Rev", (val >> 0) & 0xffff);
216 show_reg("Reserved", (val >> 16) & 0x0fff);
217 show_reg("Image Control", (val >> 28) & 0x3);
218 show_reg("Reserved", (val >> 30) & 0x1);
219 show_reg("Image Loaded", (val >> 31) & 0x1);
220
221 pci_read_config_dword(dev, vsec + 0x14, &val);
222 show_reg("Reserved", val);
223 pci_read_config_dword(dev, vsec + 0x18, &val);
224 show_reg("Reserved", val);
225 pci_read_config_dword(dev, vsec + 0x1c, &val);
226 show_reg("Reserved", val);
227
228 pci_read_config_dword(dev, vsec + 0x20, &val);
229 show_reg("AFU Descriptor Offset", val);
230 pci_read_config_dword(dev, vsec + 0x24, &val);
231 show_reg("AFU Descriptor Size", val);
232 pci_read_config_dword(dev, vsec + 0x28, &val);
233 show_reg("Problem State Offset", val);
234 pci_read_config_dword(dev, vsec + 0x2c, &val);
235 show_reg("Problem State Size", val);
236
237 pci_read_config_dword(dev, vsec + 0x30, &val);
238 show_reg("Reserved", val);
239 pci_read_config_dword(dev, vsec + 0x34, &val);
240 show_reg("Reserved", val);
241 pci_read_config_dword(dev, vsec + 0x38, &val);
242 show_reg("Reserved", val);
243 pci_read_config_dword(dev, vsec + 0x3c, &val);
244 show_reg("Reserved", val);
245
246 pci_read_config_dword(dev, vsec + 0x40, &val);
247 show_reg("PSL Programming Port", val);
248 pci_read_config_dword(dev, vsec + 0x44, &val);
249 show_reg("PSL Programming Control", val);
250
251 pci_read_config_dword(dev, vsec + 0x48, &val);
252 show_reg("Reserved", val);
253 pci_read_config_dword(dev, vsec + 0x4c, &val);
254 show_reg("Reserved", val);
255
256 pci_read_config_dword(dev, vsec + 0x50, &val);
257 show_reg("Flash Address Register", val);
258 pci_read_config_dword(dev, vsec + 0x54, &val);
259 show_reg("Flash Size Register", val);
260 pci_read_config_dword(dev, vsec + 0x58, &val);
261 show_reg("Flash Status/Control Register", val);
262 pci_read_config_dword(dev, vsec + 0x58, &val);
263 show_reg("Flash Data Port", val);
264
265#undef show_reg
266}
267
268static void dump_afu_descriptor(struct cxl_afu *afu)
269{
270 u64 val;
271
272#define show_reg(name, what) \
273 dev_info(&afu->dev, "afu desc: %30s: %#llx\n", name, what)
274
275 val = AFUD_READ_INFO(afu);
276 show_reg("num_ints_per_process", AFUD_NUM_INTS_PER_PROC(val));
277 show_reg("num_of_processes", AFUD_NUM_PROCS(val));
278 show_reg("num_of_afu_CRs", AFUD_NUM_CRS(val));
279 show_reg("req_prog_mode", val & 0xffffULL);
280
281 val = AFUD_READ(afu, 0x8);
282 show_reg("Reserved", val);
283 val = AFUD_READ(afu, 0x10);
284 show_reg("Reserved", val);
285 val = AFUD_READ(afu, 0x18);
286 show_reg("Reserved", val);
287
288 val = AFUD_READ_CR(afu);
289 show_reg("Reserved", (val >> (63-7)) & 0xff);
290 show_reg("AFU_CR_len", AFUD_CR_LEN(val));
291
292 val = AFUD_READ_CR_OFF(afu);
293 show_reg("AFU_CR_offset", val);
294
295 val = AFUD_READ_PPPSA(afu);
296 show_reg("PerProcessPSA_control", (val >> (63-7)) & 0xff);
297 show_reg("PerProcessPSA Length", AFUD_PPPSA_LEN(val));
298
299 val = AFUD_READ_PPPSA_OFF(afu);
300 show_reg("PerProcessPSA_offset", val);
301
302 val = AFUD_READ_EB(afu);
303 show_reg("Reserved", (val >> (63-7)) & 0xff);
304 show_reg("AFU_EB_len", AFUD_EB_LEN(val));
305
306 val = AFUD_READ_EB_OFF(afu);
307 show_reg("AFU_EB_offset", val);
308
309#undef show_reg
310}
311
312static int init_implementation_adapter_regs(struct cxl *adapter, struct pci_dev *dev)
313{
314 struct device_node *np;
315 const __be32 *prop;
316 u64 psl_dsnctl;
317 u64 chipid;
318
319 if (!(np = pnv_pci_to_phb_node(dev)))
320 return -ENODEV;
321
322 while (np && !(prop = of_get_property(np, "ibm,chip-id", NULL)))
323 np = of_get_next_parent(np);
324 if (!np)
325 return -ENODEV;
326 chipid = be32_to_cpup(prop);
327 of_node_put(np);
328
329 /* Tell PSL where to route data to */
330 psl_dsnctl = 0x02E8900002000000ULL | (chipid << (63-5));
331 cxl_p1_write(adapter, CXL_PSL_DSNDCTL, psl_dsnctl);
332 cxl_p1_write(adapter, CXL_PSL_RESLCKTO, 0x20000000200ULL);
333 /* snoop write mask */
334 cxl_p1_write(adapter, CXL_PSL_SNWRALLOC, 0x00000000FFFFFFFFULL);
335 /* set fir_accum */
336 cxl_p1_write(adapter, CXL_PSL_FIR_CNTL, 0x0800000000000000ULL);
337 /* for debugging with trace arrays */
338 cxl_p1_write(adapter, CXL_PSL_TRACE, 0x0000FF7C00000000ULL);
339
340 return 0;
341}
342
343static int init_implementation_afu_regs(struct cxl_afu *afu)
344{
345 /* read/write masks for this slice */
346 cxl_p1n_write(afu, CXL_PSL_APCALLOC_A, 0xFFFFFFFEFEFEFEFEULL);
347 /* APC read/write masks for this slice */
348 cxl_p1n_write(afu, CXL_PSL_COALLOC_A, 0xFF000000FEFEFEFEULL);
349 /* for debugging with trace arrays */
350 cxl_p1n_write(afu, CXL_PSL_SLICE_TRACE, 0x0000FFFF00000000ULL);
351 cxl_p1n_write(afu, CXL_PSL_RXCTL_A, 0xF000000000000000ULL);
352
353 return 0;
354}
355
356int cxl_setup_irq(struct cxl *adapter, unsigned int hwirq,
357 unsigned int virq)
358{
359 struct pci_dev *dev = to_pci_dev(adapter->dev.parent);
360
361 return pnv_cxl_ioda_msi_setup(dev, hwirq, virq);
362}
363
364int cxl_alloc_one_irq(struct cxl *adapter)
365{
366 struct pci_dev *dev = to_pci_dev(adapter->dev.parent);
367
368 return pnv_cxl_alloc_hwirqs(dev, 1);
369}
370
371void cxl_release_one_irq(struct cxl *adapter, int hwirq)
372{
373 struct pci_dev *dev = to_pci_dev(adapter->dev.parent);
374
375 return pnv_cxl_release_hwirqs(dev, hwirq, 1);
376}
377
378int cxl_alloc_irq_ranges(struct cxl_irq_ranges *irqs, struct cxl *adapter, unsigned int num)
379{
380 struct pci_dev *dev = to_pci_dev(adapter->dev.parent);
381
382 return pnv_cxl_alloc_hwirq_ranges(irqs, dev, num);
383}
384
385void cxl_release_irq_ranges(struct cxl_irq_ranges *irqs, struct cxl *adapter)
386{
387 struct pci_dev *dev = to_pci_dev(adapter->dev.parent);
388
389 pnv_cxl_release_hwirq_ranges(irqs, dev);
390}
391
392static int setup_cxl_bars(struct pci_dev *dev)
393{
394 /* Safety check in case we get backported to < 3.17 without M64 */
395 if ((p1_base(dev) < 0x100000000ULL) ||
396 (p2_base(dev) < 0x100000000ULL)) {
397 dev_err(&dev->dev, "ABORTING: M32 BAR assignment incompatible with CXL\n");
398 return -ENODEV;
399 }
400
401 /*
402 * BAR 4/5 has a special meaning for CXL and must be programmed with a
403 * special value corresponding to the CXL protocol address range.
404 * For POWER 8 that means bits 48:49 must be set to 10
405 */
406 pci_write_config_dword(dev, PCI_BASE_ADDRESS_4, 0x00000000);
407 pci_write_config_dword(dev, PCI_BASE_ADDRESS_5, 0x00020000);
408
409 return 0;
410}
411
412/* pciex node: ibm,opal-m64-window = <0x3d058 0x0 0x3d058 0x0 0x8 0x0>; */
413static int switch_card_to_cxl(struct pci_dev *dev)
414{
415 int vsec;
416 u8 val;
417 int rc;
418
419 dev_info(&dev->dev, "switch card to CXL\n");
420
421 if (!(vsec = find_cxl_vsec(dev))) {
422 dev_err(&dev->dev, "ABORTING: CXL VSEC not found!\n");
423 return -ENODEV;
424 }
425
426 if ((rc = CXL_READ_VSEC_MODE_CONTROL(dev, vsec, &val))) {
427 dev_err(&dev->dev, "failed to read current mode control: %i", rc);
428 return rc;
429 }
430 val &= ~CXL_VSEC_PROTOCOL_MASK;
431 val |= CXL_VSEC_PROTOCOL_256TB | CXL_VSEC_PROTOCOL_ENABLE;
432 if ((rc = CXL_WRITE_VSEC_MODE_CONTROL(dev, vsec, val))) {
433 dev_err(&dev->dev, "failed to enable CXL protocol: %i", rc);
434 return rc;
435 }
436 /*
437 * The CAIA spec (v0.12 11.6 Bi-modal Device Support) states
438 * we must wait 100ms after this mode switch before touching
439 * PCIe config space.
440 */
441 msleep(100);
442
443 return 0;
444}
445
446static int cxl_map_slice_regs(struct cxl_afu *afu, struct cxl *adapter, struct pci_dev *dev)
447{
448 u64 p1n_base, p2n_base, afu_desc;
449 const u64 p1n_size = 0x100;
450 const u64 p2n_size = 0x1000;
451
452 p1n_base = p1_base(dev) + 0x10000 + (afu->slice * p1n_size);
453 p2n_base = p2_base(dev) + (afu->slice * p2n_size);
454 afu->psn_phys = p2_base(dev) + (adapter->ps_off + (afu->slice * adapter->ps_size));
455 afu_desc = p2_base(dev) + adapter->afu_desc_off + (afu->slice * adapter->afu_desc_size);
456
457 if (!(afu->p1n_mmio = ioremap(p1n_base, p1n_size)))
458 goto err;
459 if (!(afu->p2n_mmio = ioremap(p2n_base, p2n_size)))
460 goto err1;
461 if (afu_desc) {
462 if (!(afu->afu_desc_mmio = ioremap(afu_desc, adapter->afu_desc_size)))
463 goto err2;
464 }
465
466 return 0;
467err2:
468 iounmap(afu->p2n_mmio);
469err1:
470 iounmap(afu->p1n_mmio);
471err:
472 dev_err(&afu->dev, "Error mapping AFU MMIO regions\n");
473 return -ENOMEM;
474}
475
476static void cxl_unmap_slice_regs(struct cxl_afu *afu)
477{
478 if (afu->p1n_mmio)
479 iounmap(afu->p2n_mmio);
480 if (afu->p1n_mmio)
481 iounmap(afu->p1n_mmio);
482}
483
484static void cxl_release_afu(struct device *dev)
485{
486 struct cxl_afu *afu = to_cxl_afu(dev);
487
488 pr_devel("cxl_release_afu\n");
489
490 kfree(afu);
491}
492
493static struct cxl_afu *cxl_alloc_afu(struct cxl *adapter, int slice)
494{
495 struct cxl_afu *afu;
496
497 if (!(afu = kzalloc(sizeof(struct cxl_afu), GFP_KERNEL)))
498 return NULL;
499
500 afu->adapter = adapter;
501 afu->dev.parent = &adapter->dev;
502 afu->dev.release = cxl_release_afu;
503 afu->slice = slice;
504 idr_init(&afu->contexts_idr);
505 spin_lock_init(&afu->contexts_lock);
506 spin_lock_init(&afu->afu_cntl_lock);
507 mutex_init(&afu->spa_mutex);
508
509 afu->prefault_mode = CXL_PREFAULT_NONE;
510 afu->irqs_max = afu->adapter->user_irqs;
511
512 return afu;
513}
514
515/* Expects AFU struct to have recently been zeroed out */
516static int cxl_read_afu_descriptor(struct cxl_afu *afu)
517{
518 u64 val;
519
520 val = AFUD_READ_INFO(afu);
521 afu->pp_irqs = AFUD_NUM_INTS_PER_PROC(val);
522 afu->max_procs_virtualised = AFUD_NUM_PROCS(val);
523
524 if (AFUD_AFU_DIRECTED(val))
525 afu->modes_supported |= CXL_MODE_DIRECTED;
526 if (AFUD_DEDICATED_PROCESS(val))
527 afu->modes_supported |= CXL_MODE_DEDICATED;
528 if (AFUD_TIME_SLICED(val))
529 afu->modes_supported |= CXL_MODE_TIME_SLICED;
530
531 val = AFUD_READ_PPPSA(afu);
532 afu->pp_size = AFUD_PPPSA_LEN(val) * 4096;
533 afu->psa = AFUD_PPPSA_PSA(val);
534 if ((afu->pp_psa = AFUD_PPPSA_PP(val)))
535 afu->pp_offset = AFUD_READ_PPPSA_OFF(afu);
536
537 return 0;
538}
539
540static int cxl_afu_descriptor_looks_ok(struct cxl_afu *afu)
541{
542 if (afu->psa && afu->adapter->ps_size <
543 (afu->pp_offset + afu->pp_size*afu->max_procs_virtualised)) {
544 dev_err(&afu->dev, "per-process PSA can't fit inside the PSA!\n");
545 return -ENODEV;
546 }
547
548 if (afu->pp_psa && (afu->pp_size < PAGE_SIZE))
549 dev_warn(&afu->dev, "AFU uses < PAGE_SIZE per-process PSA!");
550
551 return 0;
552}
553
554static int sanitise_afu_regs(struct cxl_afu *afu)
555{
556 u64 reg;
557
558 /*
559 * Clear out any regs that contain either an IVTE or address or may be
560 * waiting on an acknowledgement to try to be a bit safer as we bring
561 * it online
562 */
563 reg = cxl_p2n_read(afu, CXL_AFU_Cntl_An);
564 if ((reg & CXL_AFU_Cntl_An_ES_MASK) != CXL_AFU_Cntl_An_ES_Disabled) {
565 dev_warn(&afu->dev, "WARNING: AFU was not disabled: %#.16llx\n", reg);
566 if (cxl_afu_reset(afu))
567 return -EIO;
568 if (cxl_afu_disable(afu))
569 return -EIO;
570 if (cxl_psl_purge(afu))
571 return -EIO;
572 }
573 cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0x0000000000000000);
574 cxl_p1n_write(afu, CXL_PSL_IVTE_Limit_An, 0x0000000000000000);
575 cxl_p1n_write(afu, CXL_PSL_IVTE_Offset_An, 0x0000000000000000);
576 cxl_p1n_write(afu, CXL_PSL_AMBAR_An, 0x0000000000000000);
577 cxl_p1n_write(afu, CXL_PSL_SPOffset_An, 0x0000000000000000);
578 cxl_p1n_write(afu, CXL_HAURP_An, 0x0000000000000000);
579 cxl_p2n_write(afu, CXL_CSRP_An, 0x0000000000000000);
580 cxl_p2n_write(afu, CXL_AURP1_An, 0x0000000000000000);
581 cxl_p2n_write(afu, CXL_AURP0_An, 0x0000000000000000);
582 cxl_p2n_write(afu, CXL_SSTP1_An, 0x0000000000000000);
583 cxl_p2n_write(afu, CXL_SSTP0_An, 0x0000000000000000);
584 reg = cxl_p2n_read(afu, CXL_PSL_DSISR_An);
585 if (reg) {
586 dev_warn(&afu->dev, "AFU had pending DSISR: %#.16llx\n", reg);
587 if (reg & CXL_PSL_DSISR_TRANS)
588 cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE);
589 else
590 cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A);
591 }
592 reg = cxl_p1n_read(afu, CXL_PSL_SERR_An);
593 if (reg) {
594 if (reg & ~0xffff)
595 dev_warn(&afu->dev, "AFU had pending SERR: %#.16llx\n", reg);
596 cxl_p1n_write(afu, CXL_PSL_SERR_An, reg & ~0xffff);
597 }
598 reg = cxl_p2n_read(afu, CXL_PSL_ErrStat_An);
599 if (reg) {
600 dev_warn(&afu->dev, "AFU had pending error status: %#.16llx\n", reg);
601 cxl_p2n_write(afu, CXL_PSL_ErrStat_An, reg);
602 }
603
604 return 0;
605}
606
607static int cxl_init_afu(struct cxl *adapter, int slice, struct pci_dev *dev)
608{
609 struct cxl_afu *afu;
610 bool free = true;
611 int rc;
612
613 if (!(afu = cxl_alloc_afu(adapter, slice)))
614 return -ENOMEM;
615
616 if ((rc = dev_set_name(&afu->dev, "afu%i.%i", adapter->adapter_num, slice)))
617 goto err1;
618
619 if ((rc = cxl_map_slice_regs(afu, adapter, dev)))
620 goto err1;
621
622 if ((rc = sanitise_afu_regs(afu)))
623 goto err2;
624
625 /* We need to reset the AFU before we can read the AFU descriptor */
626 if ((rc = cxl_afu_reset(afu)))
627 goto err2;
628
629 if (cxl_verbose)
630 dump_afu_descriptor(afu);
631
632 if ((rc = cxl_read_afu_descriptor(afu)))
633 goto err2;
634
635 if ((rc = cxl_afu_descriptor_looks_ok(afu)))
636 goto err2;
637
638 if ((rc = init_implementation_afu_regs(afu)))
639 goto err2;
640
641 if ((rc = cxl_register_serr_irq(afu)))
642 goto err2;
643
644 if ((rc = cxl_register_psl_irq(afu)))
645 goto err3;
646
647 /* Don't care if this fails */
648 cxl_debugfs_afu_add(afu);
649
650 /*
651 * After we call this function we must not free the afu directly, even
652 * if it returns an error!
653 */
654 if ((rc = cxl_register_afu(afu)))
655 goto err_put1;
656
657 if ((rc = cxl_sysfs_afu_add(afu)))
658 goto err_put1;
659
660
661 if ((rc = cxl_afu_select_best_mode(afu)))
662 goto err_put2;
663
664 adapter->afu[afu->slice] = afu;
665
666 return 0;
667
668err_put2:
669 cxl_sysfs_afu_remove(afu);
670err_put1:
671 device_unregister(&afu->dev);
672 free = false;
673 cxl_debugfs_afu_remove(afu);
674 cxl_release_psl_irq(afu);
675err3:
676 cxl_release_serr_irq(afu);
677err2:
678 cxl_unmap_slice_regs(afu);
679err1:
680 if (free)
681 kfree(afu);
682 return rc;
683}
684
685static void cxl_remove_afu(struct cxl_afu *afu)
686{
687 pr_devel("cxl_remove_afu\n");
688
689 if (!afu)
690 return;
691
692 cxl_sysfs_afu_remove(afu);
693 cxl_debugfs_afu_remove(afu);
694
695 spin_lock(&afu->adapter->afu_list_lock);
696 afu->adapter->afu[afu->slice] = NULL;
697 spin_unlock(&afu->adapter->afu_list_lock);
698
699 cxl_context_detach_all(afu);
700 cxl_afu_deactivate_mode(afu);
701
702 cxl_release_psl_irq(afu);
703 cxl_release_serr_irq(afu);
704 cxl_unmap_slice_regs(afu);
705
706 device_unregister(&afu->dev);
707}
708
709
710static int cxl_map_adapter_regs(struct cxl *adapter, struct pci_dev *dev)
711{
712 if (pci_request_region(dev, 2, "priv 2 regs"))
713 goto err1;
714 if (pci_request_region(dev, 0, "priv 1 regs"))
715 goto err2;
716
717 pr_devel("cxl_map_adapter_regs: p1: %#.16llx %#llx, p2: %#.16llx %#llx",
718 p1_base(dev), p1_size(dev), p2_base(dev), p2_size(dev));
719
720 if (!(adapter->p1_mmio = ioremap(p1_base(dev), p1_size(dev))))
721 goto err3;
722
723 if (!(adapter->p2_mmio = ioremap(p2_base(dev), p2_size(dev))))
724 goto err4;
725
726 return 0;
727
728err4:
729 iounmap(adapter->p1_mmio);
730 adapter->p1_mmio = NULL;
731err3:
732 pci_release_region(dev, 0);
733err2:
734 pci_release_region(dev, 2);
735err1:
736 return -ENOMEM;
737}
738
739static void cxl_unmap_adapter_regs(struct cxl *adapter)
740{
741 if (adapter->p1_mmio)
742 iounmap(adapter->p1_mmio);
743 if (adapter->p2_mmio)
744 iounmap(adapter->p2_mmio);
745}
746
747static int cxl_read_vsec(struct cxl *adapter, struct pci_dev *dev)
748{
749 int vsec;
750 u32 afu_desc_off, afu_desc_size;
751 u32 ps_off, ps_size;
752 u16 vseclen;
753 u8 image_state;
754
755 if (!(vsec = find_cxl_vsec(dev))) {
756 dev_err(&adapter->dev, "ABORTING: CXL VSEC not found!\n");
757 return -ENODEV;
758 }
759
760 CXL_READ_VSEC_LENGTH(dev, vsec, &vseclen);
761 if (vseclen < CXL_VSEC_MIN_SIZE) {
762 pr_err("ABORTING: CXL VSEC too short\n");
763 return -EINVAL;
764 }
765
766 CXL_READ_VSEC_STATUS(dev, vsec, &adapter->vsec_status);
767 CXL_READ_VSEC_PSL_REVISION(dev, vsec, &adapter->psl_rev);
768 CXL_READ_VSEC_CAIA_MAJOR(dev, vsec, &adapter->caia_major);
769 CXL_READ_VSEC_CAIA_MINOR(dev, vsec, &adapter->caia_minor);
770 CXL_READ_VSEC_BASE_IMAGE(dev, vsec, &adapter->base_image);
771 CXL_READ_VSEC_IMAGE_STATE(dev, vsec, &image_state);
772 adapter->user_image_loaded = !!(image_state & CXL_VSEC_USER_IMAGE_LOADED);
773 adapter->perst_loads_image = !!(image_state & CXL_VSEC_PERST_LOADS_IMAGE);
774 adapter->perst_select_user = !!(image_state & CXL_VSEC_PERST_SELECT_USER);
775
776 CXL_READ_VSEC_NAFUS(dev, vsec, &adapter->slices);
777 CXL_READ_VSEC_AFU_DESC_OFF(dev, vsec, &afu_desc_off);
778 CXL_READ_VSEC_AFU_DESC_SIZE(dev, vsec, &afu_desc_size);
779 CXL_READ_VSEC_PS_OFF(dev, vsec, &ps_off);
780 CXL_READ_VSEC_PS_SIZE(dev, vsec, &ps_size);
781
782 /* Convert everything to bytes, because there is NO WAY I'd look at the
783 * code a month later and forget what units these are in ;-) */
784 adapter->ps_off = ps_off * 64 * 1024;
785 adapter->ps_size = ps_size * 64 * 1024;
786 adapter->afu_desc_off = afu_desc_off * 64 * 1024;
787 adapter->afu_desc_size = afu_desc_size *64 * 1024;
788
789 /* Total IRQs - 1 PSL ERROR - #AFU*(1 slice error + 1 DSI) */
790 adapter->user_irqs = pnv_cxl_get_irq_count(dev) - 1 - 2*adapter->slices;
791
792 return 0;
793}
794
795static int cxl_vsec_looks_ok(struct cxl *adapter, struct pci_dev *dev)
796{
797 if (adapter->vsec_status & CXL_STATUS_SECOND_PORT)
798 return -EBUSY;
799
800 if (adapter->vsec_status & CXL_UNSUPPORTED_FEATURES) {
801 dev_err(&adapter->dev, "ABORTING: CXL requires unsupported features\n");
802 return -EINVAL;
803 }
804
805 if (!adapter->slices) {
806 /* Once we support dynamic reprogramming we can use the card if
807 * it supports loadable AFUs */
808 dev_err(&adapter->dev, "ABORTING: Device has no AFUs\n");
809 return -EINVAL;
810 }
811
812 if (!adapter->afu_desc_off || !adapter->afu_desc_size) {
813 dev_err(&adapter->dev, "ABORTING: VSEC shows no AFU descriptors\n");
814 return -EINVAL;
815 }
816
817 if (adapter->ps_size > p2_size(dev) - adapter->ps_off) {
818 dev_err(&adapter->dev, "ABORTING: Problem state size larger than "
819 "available in BAR2: 0x%llx > 0x%llx\n",
820 adapter->ps_size, p2_size(dev) - adapter->ps_off);
821 return -EINVAL;
822 }
823
824 return 0;
825}
826
827static void cxl_release_adapter(struct device *dev)
828{
829 struct cxl *adapter = to_cxl_adapter(dev);
830
831 pr_devel("cxl_release_adapter\n");
832
833 kfree(adapter);
834}
835
836static struct cxl *cxl_alloc_adapter(struct pci_dev *dev)
837{
838 struct cxl *adapter;
839
840 if (!(adapter = kzalloc(sizeof(struct cxl), GFP_KERNEL)))
841 return NULL;
842
843 adapter->dev.parent = &dev->dev;
844 adapter->dev.release = cxl_release_adapter;
845 pci_set_drvdata(dev, adapter);
846 spin_lock_init(&adapter->afu_list_lock);
847
848 return adapter;
849}
850
851static int sanitise_adapter_regs(struct cxl *adapter)
852{
853 cxl_p1_write(adapter, CXL_PSL_ErrIVTE, 0x0000000000000000);
854 return cxl_tlb_slb_invalidate(adapter);
855}
856
857static struct cxl *cxl_init_adapter(struct pci_dev *dev)
858{
859 struct cxl *adapter;
860 bool free = true;
861 int rc;
862
863
864 if (!(adapter = cxl_alloc_adapter(dev)))
865 return ERR_PTR(-ENOMEM);
866
867 if ((rc = switch_card_to_cxl(dev)))
868 goto err1;
869
870 if ((rc = cxl_alloc_adapter_nr(adapter)))
871 goto err1;
872
873 if ((rc = dev_set_name(&adapter->dev, "card%i", adapter->adapter_num)))
874 goto err2;
875
876 if ((rc = cxl_read_vsec(adapter, dev)))
877 goto err2;
878
879 if ((rc = cxl_vsec_looks_ok(adapter, dev)))
880 goto err2;
881
882 if ((rc = cxl_map_adapter_regs(adapter, dev)))
883 goto err2;
884
885 if ((rc = sanitise_adapter_regs(adapter)))
886 goto err2;
887
888 if ((rc = init_implementation_adapter_regs(adapter, dev)))
889 goto err3;
890
891 if ((rc = pnv_phb_to_cxl(dev)))
892 goto err3;
893
894 if ((rc = cxl_register_psl_err_irq(adapter)))
895 goto err3;
896
897 /* Don't care if this one fails: */
898 cxl_debugfs_adapter_add(adapter);
899
900 /*
901 * After we call this function we must not free the adapter directly,
902 * even if it returns an error!
903 */
904 if ((rc = cxl_register_adapter(adapter)))
905 goto err_put1;
906
907 if ((rc = cxl_sysfs_adapter_add(adapter)))
908 goto err_put1;
909
910 return adapter;
911
912err_put1:
913 device_unregister(&adapter->dev);
914 free = false;
915 cxl_debugfs_adapter_remove(adapter);
916 cxl_release_psl_err_irq(adapter);
917err3:
918 cxl_unmap_adapter_regs(adapter);
919err2:
920 cxl_remove_adapter_nr(adapter);
921err1:
922 if (free)
923 kfree(adapter);
924 return ERR_PTR(rc);
925}
926
927static void cxl_remove_adapter(struct cxl *adapter)
928{
929 struct pci_dev *pdev = to_pci_dev(adapter->dev.parent);
930
931 pr_devel("cxl_release_adapter\n");
932
933 cxl_sysfs_adapter_remove(adapter);
934 cxl_debugfs_adapter_remove(adapter);
935 cxl_release_psl_err_irq(adapter);
936 cxl_unmap_adapter_regs(adapter);
937 cxl_remove_adapter_nr(adapter);
938
939 device_unregister(&adapter->dev);
940
941 pci_release_region(pdev, 0);
942 pci_release_region(pdev, 2);
943 pci_disable_device(pdev);
944}
945
946static int cxl_probe(struct pci_dev *dev, const struct pci_device_id *id)
947{
948 struct cxl *adapter;
949 int slice;
950 int rc;
951
952 pci_dev_get(dev);
953
954 if (cxl_verbose)
955 dump_cxl_config_space(dev);
956
957 if ((rc = setup_cxl_bars(dev)))
958 return rc;
959
960 if ((rc = pci_enable_device(dev))) {
961 dev_err(&dev->dev, "pci_enable_device failed: %i\n", rc);
962 return rc;
963 }
964
965 adapter = cxl_init_adapter(dev);
966 if (IS_ERR(adapter)) {
967 dev_err(&dev->dev, "cxl_init_adapter failed: %li\n", PTR_ERR(adapter));
968 return PTR_ERR(adapter);
969 }
970
971 for (slice = 0; slice < adapter->slices; slice++) {
972 if ((rc = cxl_init_afu(adapter, slice, dev)))
973 dev_err(&dev->dev, "AFU %i failed to initialise: %i\n", slice, rc);
974 }
975
976 return 0;
977}
978
979static void cxl_remove(struct pci_dev *dev)
980{
981 struct cxl *adapter = pci_get_drvdata(dev);
982 int afu;
983
984 dev_warn(&dev->dev, "pci remove\n");
985
986 /*
987 * Lock to prevent someone grabbing a ref through the adapter list as
988 * we are removing it
989 */
990 for (afu = 0; afu < adapter->slices; afu++)
991 cxl_remove_afu(adapter->afu[afu]);
992 cxl_remove_adapter(adapter);
993}
994
995struct pci_driver cxl_pci_driver = {
996 .name = "cxl-pci",
997 .id_table = cxl_pci_tbl,
998 .probe = cxl_probe,
999 .remove = cxl_remove,
1000};
diff --git a/drivers/misc/cxl/sysfs.c b/drivers/misc/cxl/sysfs.c
new file mode 100644
index 000000000000..ce7ec06d87d1
--- /dev/null
+++ b/drivers/misc/cxl/sysfs.c
@@ -0,0 +1,385 @@
1/*
2 * Copyright 2014 IBM Corp.
3 *
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License
6 * as published by the Free Software Foundation; either version
7 * 2 of the License, or (at your option) any later version.
8 */
9
10#include <linux/kernel.h>
11#include <linux/device.h>
12#include <linux/sysfs.h>
13
14#include "cxl.h"
15
16#define to_afu_chardev_m(d) dev_get_drvdata(d)
17
18/********* Adapter attributes **********************************************/
19
20static ssize_t caia_version_show(struct device *device,
21 struct device_attribute *attr,
22 char *buf)
23{
24 struct cxl *adapter = to_cxl_adapter(device);
25
26 return scnprintf(buf, PAGE_SIZE, "%i.%i\n", adapter->caia_major,
27 adapter->caia_minor);
28}
29
30static ssize_t psl_revision_show(struct device *device,
31 struct device_attribute *attr,
32 char *buf)
33{
34 struct cxl *adapter = to_cxl_adapter(device);
35
36 return scnprintf(buf, PAGE_SIZE, "%i\n", adapter->psl_rev);
37}
38
39static ssize_t base_image_show(struct device *device,
40 struct device_attribute *attr,
41 char *buf)
42{
43 struct cxl *adapter = to_cxl_adapter(device);
44
45 return scnprintf(buf, PAGE_SIZE, "%i\n", adapter->base_image);
46}
47
48static ssize_t image_loaded_show(struct device *device,
49 struct device_attribute *attr,
50 char *buf)
51{
52 struct cxl *adapter = to_cxl_adapter(device);
53
54 if (adapter->user_image_loaded)
55 return scnprintf(buf, PAGE_SIZE, "user\n");
56 return scnprintf(buf, PAGE_SIZE, "factory\n");
57}
58
59static struct device_attribute adapter_attrs[] = {
60 __ATTR_RO(caia_version),
61 __ATTR_RO(psl_revision),
62 __ATTR_RO(base_image),
63 __ATTR_RO(image_loaded),
64};
65
66
67/********* AFU master specific attributes **********************************/
68
69static ssize_t mmio_size_show_master(struct device *device,
70 struct device_attribute *attr,
71 char *buf)
72{
73 struct cxl_afu *afu = to_afu_chardev_m(device);
74
75 return scnprintf(buf, PAGE_SIZE, "%llu\n", afu->adapter->ps_size);
76}
77
78static ssize_t pp_mmio_off_show(struct device *device,
79 struct device_attribute *attr,
80 char *buf)
81{
82 struct cxl_afu *afu = to_afu_chardev_m(device);
83
84 return scnprintf(buf, PAGE_SIZE, "%llu\n", afu->pp_offset);
85}
86
87static ssize_t pp_mmio_len_show(struct device *device,
88 struct device_attribute *attr,
89 char *buf)
90{
91 struct cxl_afu *afu = to_afu_chardev_m(device);
92
93 return scnprintf(buf, PAGE_SIZE, "%llu\n", afu->pp_size);
94}
95
96static struct device_attribute afu_master_attrs[] = {
97 __ATTR(mmio_size, S_IRUGO, mmio_size_show_master, NULL),
98 __ATTR_RO(pp_mmio_off),
99 __ATTR_RO(pp_mmio_len),
100};
101
102
103/********* AFU attributes **************************************************/
104
105static ssize_t mmio_size_show(struct device *device,
106 struct device_attribute *attr,
107 char *buf)
108{
109 struct cxl_afu *afu = to_cxl_afu(device);
110
111 if (afu->pp_size)
112 return scnprintf(buf, PAGE_SIZE, "%llu\n", afu->pp_size);
113 return scnprintf(buf, PAGE_SIZE, "%llu\n", afu->adapter->ps_size);
114}
115
116static ssize_t reset_store_afu(struct device *device,
117 struct device_attribute *attr,
118 const char *buf, size_t count)
119{
120 struct cxl_afu *afu = to_cxl_afu(device);
121 int rc;
122
123 /* Not safe to reset if it is currently in use */
124 spin_lock(&afu->contexts_lock);
125 if (!idr_is_empty(&afu->contexts_idr)) {
126 rc = -EBUSY;
127 goto err;
128 }
129
130 if ((rc = cxl_afu_reset(afu)))
131 goto err;
132
133 rc = count;
134err:
135 spin_unlock(&afu->contexts_lock);
136 return rc;
137}
138
139static ssize_t irqs_min_show(struct device *device,
140 struct device_attribute *attr,
141 char *buf)
142{
143 struct cxl_afu *afu = to_cxl_afu(device);
144
145 return scnprintf(buf, PAGE_SIZE, "%i\n", afu->pp_irqs);
146}
147
148static ssize_t irqs_max_show(struct device *device,
149 struct device_attribute *attr,
150 char *buf)
151{
152 struct cxl_afu *afu = to_cxl_afu(device);
153
154 return scnprintf(buf, PAGE_SIZE, "%i\n", afu->irqs_max);
155}
156
157static ssize_t irqs_max_store(struct device *device,
158 struct device_attribute *attr,
159 const char *buf, size_t count)
160{
161 struct cxl_afu *afu = to_cxl_afu(device);
162 ssize_t ret;
163 int irqs_max;
164
165 ret = sscanf(buf, "%i", &irqs_max);
166 if (ret != 1)
167 return -EINVAL;
168
169 if (irqs_max < afu->pp_irqs)
170 return -EINVAL;
171
172 if (irqs_max > afu->adapter->user_irqs)
173 return -EINVAL;
174
175 afu->irqs_max = irqs_max;
176 return count;
177}
178
179static ssize_t modes_supported_show(struct device *device,
180 struct device_attribute *attr, char *buf)
181{
182 struct cxl_afu *afu = to_cxl_afu(device);
183 char *p = buf, *end = buf + PAGE_SIZE;
184
185 if (afu->modes_supported & CXL_MODE_DEDICATED)
186 p += scnprintf(p, end - p, "dedicated_process\n");
187 if (afu->modes_supported & CXL_MODE_DIRECTED)
188 p += scnprintf(p, end - p, "afu_directed\n");
189 return (p - buf);
190}
191
192static ssize_t prefault_mode_show(struct device *device,
193 struct device_attribute *attr,
194 char *buf)
195{
196 struct cxl_afu *afu = to_cxl_afu(device);
197
198 switch (afu->prefault_mode) {
199 case CXL_PREFAULT_WED:
200 return scnprintf(buf, PAGE_SIZE, "work_element_descriptor\n");
201 case CXL_PREFAULT_ALL:
202 return scnprintf(buf, PAGE_SIZE, "all\n");
203 default:
204 return scnprintf(buf, PAGE_SIZE, "none\n");
205 }
206}
207
208static ssize_t prefault_mode_store(struct device *device,
209 struct device_attribute *attr,
210 const char *buf, size_t count)
211{
212 struct cxl_afu *afu = to_cxl_afu(device);
213 enum prefault_modes mode = -1;
214
215 if (!strncmp(buf, "work_element_descriptor", 23))
216 mode = CXL_PREFAULT_WED;
217 if (!strncmp(buf, "all", 3))
218 mode = CXL_PREFAULT_ALL;
219 if (!strncmp(buf, "none", 4))
220 mode = CXL_PREFAULT_NONE;
221
222 if (mode == -1)
223 return -EINVAL;
224
225 afu->prefault_mode = mode;
226 return count;
227}
228
229static ssize_t mode_show(struct device *device,
230 struct device_attribute *attr,
231 char *buf)
232{
233 struct cxl_afu *afu = to_cxl_afu(device);
234
235 if (afu->current_mode == CXL_MODE_DEDICATED)
236 return scnprintf(buf, PAGE_SIZE, "dedicated_process\n");
237 if (afu->current_mode == CXL_MODE_DIRECTED)
238 return scnprintf(buf, PAGE_SIZE, "afu_directed\n");
239 return scnprintf(buf, PAGE_SIZE, "none\n");
240}
241
242static ssize_t mode_store(struct device *device, struct device_attribute *attr,
243 const char *buf, size_t count)
244{
245 struct cxl_afu *afu = to_cxl_afu(device);
246 int old_mode, mode = -1;
247 int rc = -EBUSY;
248
249 /* can't change this if we have a user */
250 spin_lock(&afu->contexts_lock);
251 if (!idr_is_empty(&afu->contexts_idr))
252 goto err;
253
254 if (!strncmp(buf, "dedicated_process", 17))
255 mode = CXL_MODE_DEDICATED;
256 if (!strncmp(buf, "afu_directed", 12))
257 mode = CXL_MODE_DIRECTED;
258 if (!strncmp(buf, "none", 4))
259 mode = 0;
260
261 if (mode == -1) {
262 rc = -EINVAL;
263 goto err;
264 }
265
266 /*
267 * cxl_afu_deactivate_mode needs to be done outside the lock, prevent
268 * other contexts coming in before we are ready:
269 */
270 old_mode = afu->current_mode;
271 afu->current_mode = 0;
272 afu->num_procs = 0;
273
274 spin_unlock(&afu->contexts_lock);
275
276 if ((rc = _cxl_afu_deactivate_mode(afu, old_mode)))
277 return rc;
278 if ((rc = cxl_afu_activate_mode(afu, mode)))
279 return rc;
280
281 return count;
282err:
283 spin_unlock(&afu->contexts_lock);
284 return rc;
285}
286
287static ssize_t api_version_show(struct device *device,
288 struct device_attribute *attr,
289 char *buf)
290{
291 return scnprintf(buf, PAGE_SIZE, "%i\n", CXL_API_VERSION);
292}
293
294static ssize_t api_version_compatible_show(struct device *device,
295 struct device_attribute *attr,
296 char *buf)
297{
298 return scnprintf(buf, PAGE_SIZE, "%i\n", CXL_API_VERSION_COMPATIBLE);
299}
300
301static struct device_attribute afu_attrs[] = {
302 __ATTR_RO(mmio_size),
303 __ATTR_RO(irqs_min),
304 __ATTR_RW(irqs_max),
305 __ATTR_RO(modes_supported),
306 __ATTR_RW(mode),
307 __ATTR_RW(prefault_mode),
308 __ATTR_RO(api_version),
309 __ATTR_RO(api_version_compatible),
310 __ATTR(reset, S_IWUSR, NULL, reset_store_afu),
311};
312
313
314
315int cxl_sysfs_adapter_add(struct cxl *adapter)
316{
317 int i, rc;
318
319 for (i = 0; i < ARRAY_SIZE(adapter_attrs); i++) {
320 if ((rc = device_create_file(&adapter->dev, &adapter_attrs[i])))
321 goto err;
322 }
323 return 0;
324err:
325 for (i--; i >= 0; i--)
326 device_remove_file(&adapter->dev, &adapter_attrs[i]);
327 return rc;
328}
329void cxl_sysfs_adapter_remove(struct cxl *adapter)
330{
331 int i;
332
333 for (i = 0; i < ARRAY_SIZE(adapter_attrs); i++)
334 device_remove_file(&adapter->dev, &adapter_attrs[i]);
335}
336
337int cxl_sysfs_afu_add(struct cxl_afu *afu)
338{
339 int i, rc;
340
341 for (i = 0; i < ARRAY_SIZE(afu_attrs); i++) {
342 if ((rc = device_create_file(&afu->dev, &afu_attrs[i])))
343 goto err;
344 }
345
346 return 0;
347
348err:
349 for (i--; i >= 0; i--)
350 device_remove_file(&afu->dev, &afu_attrs[i]);
351 return rc;
352}
353
354void cxl_sysfs_afu_remove(struct cxl_afu *afu)
355{
356 int i;
357
358 for (i = 0; i < ARRAY_SIZE(afu_attrs); i++)
359 device_remove_file(&afu->dev, &afu_attrs[i]);
360}
361
362int cxl_sysfs_afu_m_add(struct cxl_afu *afu)
363{
364 int i, rc;
365
366 for (i = 0; i < ARRAY_SIZE(afu_master_attrs); i++) {
367 if ((rc = device_create_file(afu->chardev_m, &afu_master_attrs[i])))
368 goto err;
369 }
370
371 return 0;
372
373err:
374 for (i--; i >= 0; i--)
375 device_remove_file(afu->chardev_m, &afu_master_attrs[i]);
376 return rc;
377}
378
379void cxl_sysfs_afu_m_remove(struct cxl_afu *afu)
380{
381 int i;
382
383 for (i = 0; i < ARRAY_SIZE(afu_master_attrs); i++)
384 device_remove_file(afu->chardev_m, &afu_master_attrs[i]);
385}
generated by cgit v1.2.2 (git 2.25.0) at 2024-11-28 12:54:59 -0500