aboutsummaryrefslogtreecommitdiffstats
path: root/drivers/perf
diff options
context:
space:
mode:
authorRobin Murphy <robin.murphy@arm.com>2018-02-15 13:51:42 -0500
committerArnd Bergmann <arnd@arndb.de>2018-03-06 11:26:17 -0500
commit3de6be7a3dd8934e59d85fc60a170d4ab2f0a0f2 (patch)
tree57526a619d1fe827b191906ae2e455eb963afef8 /drivers/perf
parent1888d3ddc3d6a2511be86045cfb2e7ea5fc67c44 (diff)
drivers/bus: Split Arm CCI driver
The arm-cci driver is really two entirely separate drivers; one for MCPM port control and the other for the performance monitors. Since they are already relatively self-contained, let's take the plunge and move the PMU parts out to drivers/perf where they belong these days. For non-MCPM systems this leaves a small dependency on the remaining "bus" stub for initial probing and discovery, but we end up with something that still fits the general pattern of its fellow system PMU drivers to ease future maintenance. Moving code to a new file also offers a perfect excuse to modernise the license/copyright headers and clean up some funky linewraps on the way. Cc: Lorenzo Pieralisi <lorenzo.pieralisi@arm.com> Reviewed-by: Suzuki Poulose <suzuki.poulose@arm.com> Acked-by: Punit Agrawal <punit.agrawal@arm.com> Signed-off-by: Robin Murphy <robin.murphy@arm.com> Signed-off-by: Arnd Bergmann <arnd@arndb.de>
Diffstat (limited to 'drivers/perf')
-rw-r--r--drivers/perf/Kconfig26
-rw-r--r--drivers/perf/Makefile1
-rw-r--r--drivers/perf/arm-cci.c1747
3 files changed, 1774 insertions, 0 deletions
diff --git a/drivers/perf/Kconfig b/drivers/perf/Kconfig
index 331b6d992b5a..28bb5a029558 100644
--- a/drivers/perf/Kconfig
+++ b/drivers/perf/Kconfig
@@ -5,6 +5,32 @@
5menu "Performance monitor support" 5menu "Performance monitor support"
6 depends on PERF_EVENTS 6 depends on PERF_EVENTS
7 7
8config ARM_CCI_PMU
9 bool
10 select ARM_CCI
11
12config ARM_CCI400_PMU
13 bool "ARM CCI400 PMU support"
14 depends on (ARM && CPU_V7) || ARM64
15 select ARM_CCI400_COMMON
16 select ARM_CCI_PMU
17 help
18 Support for PMU events monitoring on the ARM CCI-400 (cache coherent
19 interconnect). CCI-400 supports counting events related to the
20 connected slave/master interfaces.
21
22config ARM_CCI5xx_PMU
23 bool "ARM CCI-500/CCI-550 PMU support"
24 depends on (ARM && CPU_V7) || ARM64
25 select ARM_CCI_PMU
26 help
27 Support for PMU events monitoring on the ARM CCI-500/CCI-550 cache
28 coherent interconnects. Both of them provide 8 independent event counters,
29 which can count events pertaining to the slave/master interfaces as well
30 as the internal events to the CCI.
31
32 If unsure, say Y
33
8config ARM_CCN 34config ARM_CCN
9 tristate "ARM CCN driver support" 35 tristate "ARM CCN driver support"
10 depends on ARM || ARM64 36 depends on ARM || ARM64
diff --git a/drivers/perf/Makefile b/drivers/perf/Makefile
index 5004abee0f3a..b3902bd37d53 100644
--- a/drivers/perf/Makefile
+++ b/drivers/perf/Makefile
@@ -1,4 +1,5 @@
1# SPDX-License-Identifier: GPL-2.0 1# SPDX-License-Identifier: GPL-2.0
2obj-$(CONFIG_ARM_CCI_PMU) += arm-cci.o
2obj-$(CONFIG_ARM_CCN) += arm-ccn.o 3obj-$(CONFIG_ARM_CCN) += arm-ccn.o
3obj-$(CONFIG_ARM_DSU_PMU) += arm_dsu_pmu.o 4obj-$(CONFIG_ARM_DSU_PMU) += arm_dsu_pmu.o
4obj-$(CONFIG_ARM_PMU) += arm_pmu.o arm_pmu_platform.o 5obj-$(CONFIG_ARM_PMU) += arm_pmu.o arm_pmu_platform.o
diff --git a/drivers/perf/arm-cci.c b/drivers/perf/arm-cci.c
new file mode 100644
index 000000000000..d5f8c750fd41
--- /dev/null
+++ b/drivers/perf/arm-cci.c
@@ -0,0 +1,1747 @@
1// SPDX-License-Identifier: GPL-2.0
2// CCI Cache Coherent Interconnect PMU driver
3// Copyright (C) 2013-2018 Arm Ltd.
4// Author: Punit Agrawal <punit.agrawal@arm.com>, Suzuki Poulose <suzuki.poulose@arm.com>
5
6#include <linux/arm-cci.h>
7#include <linux/io.h>
8#include <linux/interrupt.h>
9#include <linux/module.h>
10#include <linux/of_address.h>
11#include <linux/of_irq.h>
12#include <linux/of_platform.h>
13#include <linux/perf_event.h>
14#include <linux/platform_device.h>
15#include <linux/slab.h>
16#include <linux/spinlock.h>
17
18extern void __iomem *const cci_ctrl_base;
19
20#define DRIVER_NAME "ARM-CCI PMU"
21
22#define CCI_PMCR 0x0100
23#define CCI_PID2 0x0fe8
24
25#define CCI_PMCR_CEN 0x00000001
26#define CCI_PMCR_NCNT_MASK 0x0000f800
27#define CCI_PMCR_NCNT_SHIFT 11
28
29#define CCI_PID2_REV_MASK 0xf0
30#define CCI_PID2_REV_SHIFT 4
31
32#define CCI_PMU_EVT_SEL 0x000
33#define CCI_PMU_CNTR 0x004
34#define CCI_PMU_CNTR_CTRL 0x008
35#define CCI_PMU_OVRFLW 0x00c
36
37#define CCI_PMU_OVRFLW_FLAG 1
38
39#define CCI_PMU_CNTR_SIZE(model) ((model)->cntr_size)
40#define CCI_PMU_CNTR_BASE(model, idx) ((idx) * CCI_PMU_CNTR_SIZE(model))
41#define CCI_PMU_CNTR_MASK ((1ULL << 32) -1)
42#define CCI_PMU_CNTR_LAST(cci_pmu) (cci_pmu->num_cntrs - 1)
43
44#define CCI_PMU_MAX_HW_CNTRS(model) \
45 ((model)->num_hw_cntrs + (model)->fixed_hw_cntrs)
46
47/* Types of interfaces that can generate events */
48enum {
49 CCI_IF_SLAVE,
50 CCI_IF_MASTER,
51#ifdef CONFIG_ARM_CCI5xx_PMU
52 CCI_IF_GLOBAL,
53#endif
54 CCI_IF_MAX,
55};
56
57struct event_range {
58 u32 min;
59 u32 max;
60};
61
62struct cci_pmu_hw_events {
63 struct perf_event **events;
64 unsigned long *used_mask;
65 raw_spinlock_t pmu_lock;
66};
67
68struct cci_pmu;
69/*
70 * struct cci_pmu_model:
71 * @fixed_hw_cntrs - Number of fixed event counters
72 * @num_hw_cntrs - Maximum number of programmable event counters
73 * @cntr_size - Size of an event counter mapping
74 */
75struct cci_pmu_model {
76 char *name;
77 u32 fixed_hw_cntrs;
78 u32 num_hw_cntrs;
79 u32 cntr_size;
80 struct attribute **format_attrs;
81 struct attribute **event_attrs;
82 struct event_range event_ranges[CCI_IF_MAX];
83 int (*validate_hw_event)(struct cci_pmu *, unsigned long);
84 int (*get_event_idx)(struct cci_pmu *, struct cci_pmu_hw_events *, unsigned long);
85 void (*write_counters)(struct cci_pmu *, unsigned long *);
86};
87
88static struct cci_pmu_model cci_pmu_models[];
89
90struct cci_pmu {
91 void __iomem *base;
92 struct pmu pmu;
93 int nr_irqs;
94 int *irqs;
95 unsigned long active_irqs;
96 const struct cci_pmu_model *model;
97 struct cci_pmu_hw_events hw_events;
98 struct platform_device *plat_device;
99 int num_cntrs;
100 atomic_t active_events;
101 struct mutex reserve_mutex;
102 struct hlist_node node;
103 cpumask_t cpus;
104};
105
106#define to_cci_pmu(c) (container_of(c, struct cci_pmu, pmu))
107
108enum cci_models {
109#ifdef CONFIG_ARM_CCI400_PMU
110 CCI400_R0,
111 CCI400_R1,
112#endif
113#ifdef CONFIG_ARM_CCI5xx_PMU
114 CCI500_R0,
115 CCI550_R0,
116#endif
117 CCI_MODEL_MAX
118};
119
120static void pmu_write_counters(struct cci_pmu *cci_pmu,
121 unsigned long *mask);
122static ssize_t cci_pmu_format_show(struct device *dev,
123 struct device_attribute *attr, char *buf);
124static ssize_t cci_pmu_event_show(struct device *dev,
125 struct device_attribute *attr, char *buf);
126
127#define CCI_EXT_ATTR_ENTRY(_name, _func, _config) \
128 &((struct dev_ext_attribute[]) { \
129 { __ATTR(_name, S_IRUGO, _func, NULL), (void *)_config } \
130 })[0].attr.attr
131
132#define CCI_FORMAT_EXT_ATTR_ENTRY(_name, _config) \
133 CCI_EXT_ATTR_ENTRY(_name, cci_pmu_format_show, (char *)_config)
134#define CCI_EVENT_EXT_ATTR_ENTRY(_name, _config) \
135 CCI_EXT_ATTR_ENTRY(_name, cci_pmu_event_show, (unsigned long)_config)
136
137/* CCI400 PMU Specific definitions */
138
139#ifdef CONFIG_ARM_CCI400_PMU
140
141/* Port ids */
142#define CCI400_PORT_S0 0
143#define CCI400_PORT_S1 1
144#define CCI400_PORT_S2 2
145#define CCI400_PORT_S3 3
146#define CCI400_PORT_S4 4
147#define CCI400_PORT_M0 5
148#define CCI400_PORT_M1 6
149#define CCI400_PORT_M2 7
150
151#define CCI400_R1_PX 5
152
153/*
154 * Instead of an event id to monitor CCI cycles, a dedicated counter is
155 * provided. Use 0xff to represent CCI cycles and hope that no future revisions
156 * make use of this event in hardware.
157 */
158enum cci400_perf_events {
159 CCI400_PMU_CYCLES = 0xff
160};
161
162#define CCI400_PMU_CYCLE_CNTR_IDX 0
163#define CCI400_PMU_CNTR0_IDX 1
164
165/*
166 * CCI PMU event id is an 8-bit value made of two parts - bits 7:5 for one of 8
167 * ports and bits 4:0 are event codes. There are different event codes
168 * associated with each port type.
169 *
170 * Additionally, the range of events associated with the port types changed
171 * between Rev0 and Rev1.
172 *
173 * The constants below define the range of valid codes for each port type for
174 * the different revisions and are used to validate the event to be monitored.
175 */
176
177#define CCI400_PMU_EVENT_MASK 0xffUL
178#define CCI400_PMU_EVENT_SOURCE_SHIFT 5
179#define CCI400_PMU_EVENT_SOURCE_MASK 0x7
180#define CCI400_PMU_EVENT_CODE_SHIFT 0
181#define CCI400_PMU_EVENT_CODE_MASK 0x1f
182#define CCI400_PMU_EVENT_SOURCE(event) \
183 ((event >> CCI400_PMU_EVENT_SOURCE_SHIFT) & \
184 CCI400_PMU_EVENT_SOURCE_MASK)
185#define CCI400_PMU_EVENT_CODE(event) \
186 ((event >> CCI400_PMU_EVENT_CODE_SHIFT) & CCI400_PMU_EVENT_CODE_MASK)
187
188#define CCI400_R0_SLAVE_PORT_MIN_EV 0x00
189#define CCI400_R0_SLAVE_PORT_MAX_EV 0x13
190#define CCI400_R0_MASTER_PORT_MIN_EV 0x14
191#define CCI400_R0_MASTER_PORT_MAX_EV 0x1a
192
193#define CCI400_R1_SLAVE_PORT_MIN_EV 0x00
194#define CCI400_R1_SLAVE_PORT_MAX_EV 0x14
195#define CCI400_R1_MASTER_PORT_MIN_EV 0x00
196#define CCI400_R1_MASTER_PORT_MAX_EV 0x11
197
198#define CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(_name, _config) \
199 CCI_EXT_ATTR_ENTRY(_name, cci400_pmu_cycle_event_show, \
200 (unsigned long)_config)
201
202static ssize_t cci400_pmu_cycle_event_show(struct device *dev,
203 struct device_attribute *attr, char *buf);
204
205static struct attribute *cci400_pmu_format_attrs[] = {
206 CCI_FORMAT_EXT_ATTR_ENTRY(event, "config:0-4"),
207 CCI_FORMAT_EXT_ATTR_ENTRY(source, "config:5-7"),
208 NULL
209};
210
211static struct attribute *cci400_r0_pmu_event_attrs[] = {
212 /* Slave events */
213 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_any, 0x0),
214 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_device, 0x01),
215 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_normal_or_nonshareable, 0x2),
216 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_inner_or_outershareable, 0x3),
217 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maintenance, 0x4),
218 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_mem_barrier, 0x5),
219 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_sync_barrier, 0x6),
220 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg, 0x7),
221 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg_sync, 0x8),
222 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_tt_full, 0x9),
223 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_last_hs_snoop, 0xA),
224 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall_rvalids_h_rready_l, 0xB),
225 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_any, 0xC),
226 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_device, 0xD),
227 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_normal_or_nonshareable, 0xE),
228 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_inner_or_outershare_wback_wclean, 0xF),
229 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_unique, 0x10),
230 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_line_unique, 0x11),
231 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_evict, 0x12),
232 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall_tt_full, 0x13),
233 /* Master events */
234 CCI_EVENT_EXT_ATTR_ENTRY(mi_retry_speculative_fetch, 0x14),
235 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_addr_hazard, 0x15),
236 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_id_hazard, 0x16),
237 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_tt_full, 0x17),
238 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_barrier_hazard, 0x18),
239 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_barrier_hazard, 0x19),
240 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_tt_full, 0x1A),
241 /* Special event for cycles counter */
242 CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(cycles, 0xff),
243 NULL
244};
245
246static struct attribute *cci400_r1_pmu_event_attrs[] = {
247 /* Slave events */
248 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_any, 0x0),
249 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_device, 0x01),
250 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_normal_or_nonshareable, 0x2),
251 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_inner_or_outershareable, 0x3),
252 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maintenance, 0x4),
253 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_mem_barrier, 0x5),
254 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_sync_barrier, 0x6),
255 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg, 0x7),
256 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg_sync, 0x8),
257 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_tt_full, 0x9),
258 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_last_hs_snoop, 0xA),
259 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall_rvalids_h_rready_l, 0xB),
260 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_any, 0xC),
261 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_device, 0xD),
262 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_normal_or_nonshareable, 0xE),
263 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_inner_or_outershare_wback_wclean, 0xF),
264 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_unique, 0x10),
265 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_write_line_unique, 0x11),
266 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_evict, 0x12),
267 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall_tt_full, 0x13),
268 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_slave_id_hazard, 0x14),
269 /* Master events */
270 CCI_EVENT_EXT_ATTR_ENTRY(mi_retry_speculative_fetch, 0x0),
271 CCI_EVENT_EXT_ATTR_ENTRY(mi_stall_cycle_addr_hazard, 0x1),
272 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_master_id_hazard, 0x2),
273 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_hi_prio_rtq_full, 0x3),
274 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_barrier_hazard, 0x4),
275 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_barrier_hazard, 0x5),
276 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_wtq_full, 0x6),
277 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_low_prio_rtq_full, 0x7),
278 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_mid_prio_rtq_full, 0x8),
279 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn0, 0x9),
280 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn1, 0xA),
281 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn2, 0xB),
282 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall_qvn_vn3, 0xC),
283 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn0, 0xD),
284 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn1, 0xE),
285 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn2, 0xF),
286 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall_qvn_vn3, 0x10),
287 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_unique_or_line_unique_addr_hazard, 0x11),
288 /* Special event for cycles counter */
289 CCI400_CYCLE_EVENT_EXT_ATTR_ENTRY(cycles, 0xff),
290 NULL
291};
292
293static ssize_t cci400_pmu_cycle_event_show(struct device *dev,
294 struct device_attribute *attr, char *buf)
295{
296 struct dev_ext_attribute *eattr = container_of(attr,
297 struct dev_ext_attribute, attr);
298 return snprintf(buf, PAGE_SIZE, "config=0x%lx\n", (unsigned long)eattr->var);
299}
300
301static int cci400_get_event_idx(struct cci_pmu *cci_pmu,
302 struct cci_pmu_hw_events *hw,
303 unsigned long cci_event)
304{
305 int idx;
306
307 /* cycles event idx is fixed */
308 if (cci_event == CCI400_PMU_CYCLES) {
309 if (test_and_set_bit(CCI400_PMU_CYCLE_CNTR_IDX, hw->used_mask))
310 return -EAGAIN;
311
312 return CCI400_PMU_CYCLE_CNTR_IDX;
313 }
314
315 for (idx = CCI400_PMU_CNTR0_IDX; idx <= CCI_PMU_CNTR_LAST(cci_pmu); ++idx)
316 if (!test_and_set_bit(idx, hw->used_mask))
317 return idx;
318
319 /* No counters available */
320 return -EAGAIN;
321}
322
323static int cci400_validate_hw_event(struct cci_pmu *cci_pmu, unsigned long hw_event)
324{
325 u8 ev_source = CCI400_PMU_EVENT_SOURCE(hw_event);
326 u8 ev_code = CCI400_PMU_EVENT_CODE(hw_event);
327 int if_type;
328
329 if (hw_event & ~CCI400_PMU_EVENT_MASK)
330 return -ENOENT;
331
332 if (hw_event == CCI400_PMU_CYCLES)
333 return hw_event;
334
335 switch (ev_source) {
336 case CCI400_PORT_S0:
337 case CCI400_PORT_S1:
338 case CCI400_PORT_S2:
339 case CCI400_PORT_S3:
340 case CCI400_PORT_S4:
341 /* Slave Interface */
342 if_type = CCI_IF_SLAVE;
343 break;
344 case CCI400_PORT_M0:
345 case CCI400_PORT_M1:
346 case CCI400_PORT_M2:
347 /* Master Interface */
348 if_type = CCI_IF_MASTER;
349 break;
350 default:
351 return -ENOENT;
352 }
353
354 if (ev_code >= cci_pmu->model->event_ranges[if_type].min &&
355 ev_code <= cci_pmu->model->event_ranges[if_type].max)
356 return hw_event;
357
358 return -ENOENT;
359}
360
361static int probe_cci400_revision(void)
362{
363 int rev;
364 rev = readl_relaxed(cci_ctrl_base + CCI_PID2) & CCI_PID2_REV_MASK;
365 rev >>= CCI_PID2_REV_SHIFT;
366
367 if (rev < CCI400_R1_PX)
368 return CCI400_R0;
369 else
370 return CCI400_R1;
371}
372
373static const struct cci_pmu_model *probe_cci_model(struct platform_device *pdev)
374{
375 if (platform_has_secure_cci_access())
376 return &cci_pmu_models[probe_cci400_revision()];
377 return NULL;
378}
379#else /* !CONFIG_ARM_CCI400_PMU */
380static inline struct cci_pmu_model *probe_cci_model(struct platform_device *pdev)
381{
382 return NULL;
383}
384#endif /* CONFIG_ARM_CCI400_PMU */
385
386#ifdef CONFIG_ARM_CCI5xx_PMU
387
388/*
389 * CCI5xx PMU event id is an 9-bit value made of two parts.
390 * bits [8:5] - Source for the event
391 * bits [4:0] - Event code (specific to type of interface)
392 *
393 *
394 */
395
396/* Port ids */
397#define CCI5xx_PORT_S0 0x0
398#define CCI5xx_PORT_S1 0x1
399#define CCI5xx_PORT_S2 0x2
400#define CCI5xx_PORT_S3 0x3
401#define CCI5xx_PORT_S4 0x4
402#define CCI5xx_PORT_S5 0x5
403#define CCI5xx_PORT_S6 0x6
404
405#define CCI5xx_PORT_M0 0x8
406#define CCI5xx_PORT_M1 0x9
407#define CCI5xx_PORT_M2 0xa
408#define CCI5xx_PORT_M3 0xb
409#define CCI5xx_PORT_M4 0xc
410#define CCI5xx_PORT_M5 0xd
411#define CCI5xx_PORT_M6 0xe
412
413#define CCI5xx_PORT_GLOBAL 0xf
414
415#define CCI5xx_PMU_EVENT_MASK 0x1ffUL
416#define CCI5xx_PMU_EVENT_SOURCE_SHIFT 0x5
417#define CCI5xx_PMU_EVENT_SOURCE_MASK 0xf
418#define CCI5xx_PMU_EVENT_CODE_SHIFT 0x0
419#define CCI5xx_PMU_EVENT_CODE_MASK 0x1f
420
421#define CCI5xx_PMU_EVENT_SOURCE(event) \
422 ((event >> CCI5xx_PMU_EVENT_SOURCE_SHIFT) & CCI5xx_PMU_EVENT_SOURCE_MASK)
423#define CCI5xx_PMU_EVENT_CODE(event) \
424 ((event >> CCI5xx_PMU_EVENT_CODE_SHIFT) & CCI5xx_PMU_EVENT_CODE_MASK)
425
426#define CCI5xx_SLAVE_PORT_MIN_EV 0x00
427#define CCI5xx_SLAVE_PORT_MAX_EV 0x1f
428#define CCI5xx_MASTER_PORT_MIN_EV 0x00
429#define CCI5xx_MASTER_PORT_MAX_EV 0x06
430#define CCI5xx_GLOBAL_PORT_MIN_EV 0x00
431#define CCI5xx_GLOBAL_PORT_MAX_EV 0x0f
432
433
434#define CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(_name, _config) \
435 CCI_EXT_ATTR_ENTRY(_name, cci5xx_pmu_global_event_show, \
436 (unsigned long) _config)
437
438static ssize_t cci5xx_pmu_global_event_show(struct device *dev,
439 struct device_attribute *attr, char *buf);
440
441static struct attribute *cci5xx_pmu_format_attrs[] = {
442 CCI_FORMAT_EXT_ATTR_ENTRY(event, "config:0-4"),
443 CCI_FORMAT_EXT_ATTR_ENTRY(source, "config:5-8"),
444 NULL,
445};
446
447static struct attribute *cci5xx_pmu_event_attrs[] = {
448 /* Slave events */
449 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_arvalid, 0x0),
450 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_dev, 0x1),
451 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_nonshareable, 0x2),
452 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_shareable_non_alloc, 0x3),
453 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_shareable_alloc, 0x4),
454 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_invalidate, 0x5),
455 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_cache_maint, 0x6),
456 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_dvm_msg, 0x7),
457 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_rval, 0x8),
458 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_hs_rlast_snoop, 0x9),
459 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_hs_awalid, 0xA),
460 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_dev, 0xB),
461 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_non_shareable, 0xC),
462 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wb, 0xD),
463 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wlu, 0xE),
464 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_share_wunique, 0xF),
465 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_evict, 0x10),
466 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_wrevict, 0x11),
467 CCI_EVENT_EXT_ATTR_ENTRY(si_w_data_beat, 0x12),
468 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_acvalid, 0x13),
469 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_read, 0x14),
470 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_clean, 0x15),
471 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_data_transfer_low, 0x16),
472 CCI_EVENT_EXT_ATTR_ENTRY(si_rrq_stall_arvalid, 0x17),
473 CCI_EVENT_EXT_ATTR_ENTRY(si_r_data_stall, 0x18),
474 CCI_EVENT_EXT_ATTR_ENTRY(si_wrq_stall, 0x19),
475 CCI_EVENT_EXT_ATTR_ENTRY(si_w_data_stall, 0x1A),
476 CCI_EVENT_EXT_ATTR_ENTRY(si_w_resp_stall, 0x1B),
477 CCI_EVENT_EXT_ATTR_ENTRY(si_srq_stall, 0x1C),
478 CCI_EVENT_EXT_ATTR_ENTRY(si_s_data_stall, 0x1D),
479 CCI_EVENT_EXT_ATTR_ENTRY(si_rq_stall_ot_limit, 0x1E),
480 CCI_EVENT_EXT_ATTR_ENTRY(si_r_stall_arbit, 0x1F),
481
482 /* Master events */
483 CCI_EVENT_EXT_ATTR_ENTRY(mi_r_data_beat_any, 0x0),
484 CCI_EVENT_EXT_ATTR_ENTRY(mi_w_data_beat_any, 0x1),
485 CCI_EVENT_EXT_ATTR_ENTRY(mi_rrq_stall, 0x2),
486 CCI_EVENT_EXT_ATTR_ENTRY(mi_r_data_stall, 0x3),
487 CCI_EVENT_EXT_ATTR_ENTRY(mi_wrq_stall, 0x4),
488 CCI_EVENT_EXT_ATTR_ENTRY(mi_w_data_stall, 0x5),
489 CCI_EVENT_EXT_ATTR_ENTRY(mi_w_resp_stall, 0x6),
490
491 /* Global events */
492 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_0_1, 0x0),
493 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_2_3, 0x1),
494 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_4_5, 0x2),
495 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_filter_bank_6_7, 0x3),
496 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_0_1, 0x4),
497 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_2_3, 0x5),
498 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_4_5, 0x6),
499 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_access_miss_filter_bank_6_7, 0x7),
500 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_back_invalidation, 0x8),
501 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_stall_alloc_busy, 0x9),
502 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_stall_tt_full, 0xA),
503 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_wrq, 0xB),
504 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_cd_hs, 0xC),
505 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_rq_stall_addr_hazard, 0xD),
506 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_rq_stall_tt_full, 0xE),
507 CCI5xx_GLOBAL_EVENT_EXT_ATTR_ENTRY(cci_snoop_rq_tzmp1_prot, 0xF),
508 NULL
509};
510
511static ssize_t cci5xx_pmu_global_event_show(struct device *dev,
512 struct device_attribute *attr, char *buf)
513{
514 struct dev_ext_attribute *eattr = container_of(attr,
515 struct dev_ext_attribute, attr);
516 /* Global events have single fixed source code */
517 return snprintf(buf, PAGE_SIZE, "event=0x%lx,source=0x%x\n",
518 (unsigned long)eattr->var, CCI5xx_PORT_GLOBAL);
519}
520
521/*
522 * CCI500 provides 8 independent event counters that can count
523 * any of the events available.
524 * CCI500 PMU event source ids
525 * 0x0-0x6 - Slave interfaces
526 * 0x8-0xD - Master interfaces
527 * 0xf - Global Events
528 * 0x7,0xe - Reserved
529 */
530static int cci500_validate_hw_event(struct cci_pmu *cci_pmu,
531 unsigned long hw_event)
532{
533 u32 ev_source = CCI5xx_PMU_EVENT_SOURCE(hw_event);
534 u32 ev_code = CCI5xx_PMU_EVENT_CODE(hw_event);
535 int if_type;
536
537 if (hw_event & ~CCI5xx_PMU_EVENT_MASK)
538 return -ENOENT;
539
540 switch (ev_source) {
541 case CCI5xx_PORT_S0:
542 case CCI5xx_PORT_S1:
543 case CCI5xx_PORT_S2:
544 case CCI5xx_PORT_S3:
545 case CCI5xx_PORT_S4:
546 case CCI5xx_PORT_S5:
547 case CCI5xx_PORT_S6:
548 if_type = CCI_IF_SLAVE;
549 break;
550 case CCI5xx_PORT_M0:
551 case CCI5xx_PORT_M1:
552 case CCI5xx_PORT_M2:
553 case CCI5xx_PORT_M3:
554 case CCI5xx_PORT_M4:
555 case CCI5xx_PORT_M5:
556 if_type = CCI_IF_MASTER;
557 break;
558 case CCI5xx_PORT_GLOBAL:
559 if_type = CCI_IF_GLOBAL;
560 break;
561 default:
562 return -ENOENT;
563 }
564
565 if (ev_code >= cci_pmu->model->event_ranges[if_type].min &&
566 ev_code <= cci_pmu->model->event_ranges[if_type].max)
567 return hw_event;
568
569 return -ENOENT;
570}
571
572/*
573 * CCI550 provides 8 independent event counters that can count
574 * any of the events available.
575 * CCI550 PMU event source ids
576 * 0x0-0x6 - Slave interfaces
577 * 0x8-0xe - Master interfaces
578 * 0xf - Global Events
579 * 0x7 - Reserved
580 */
581static int cci550_validate_hw_event(struct cci_pmu *cci_pmu,
582 unsigned long hw_event)
583{
584 u32 ev_source = CCI5xx_PMU_EVENT_SOURCE(hw_event);
585 u32 ev_code = CCI5xx_PMU_EVENT_CODE(hw_event);
586 int if_type;
587
588 if (hw_event & ~CCI5xx_PMU_EVENT_MASK)
589 return -ENOENT;
590
591 switch (ev_source) {
592 case CCI5xx_PORT_S0:
593 case CCI5xx_PORT_S1:
594 case CCI5xx_PORT_S2:
595 case CCI5xx_PORT_S3:
596 case CCI5xx_PORT_S4:
597 case CCI5xx_PORT_S5:
598 case CCI5xx_PORT_S6:
599 if_type = CCI_IF_SLAVE;
600 break;
601 case CCI5xx_PORT_M0:
602 case CCI5xx_PORT_M1:
603 case CCI5xx_PORT_M2:
604 case CCI5xx_PORT_M3:
605 case CCI5xx_PORT_M4:
606 case CCI5xx_PORT_M5:
607 case CCI5xx_PORT_M6:
608 if_type = CCI_IF_MASTER;
609 break;
610 case CCI5xx_PORT_GLOBAL:
611 if_type = CCI_IF_GLOBAL;
612 break;
613 default:
614 return -ENOENT;
615 }
616
617 if (ev_code >= cci_pmu->model->event_ranges[if_type].min &&
618 ev_code <= cci_pmu->model->event_ranges[if_type].max)
619 return hw_event;
620
621 return -ENOENT;
622}
623
624#endif /* CONFIG_ARM_CCI5xx_PMU */
625
626/*
627 * Program the CCI PMU counters which have PERF_HES_ARCH set
628 * with the event period and mark them ready before we enable
629 * PMU.
630 */
631static void cci_pmu_sync_counters(struct cci_pmu *cci_pmu)
632{
633 int i;
634 struct cci_pmu_hw_events *cci_hw = &cci_pmu->hw_events;
635
636 DECLARE_BITMAP(mask, cci_pmu->num_cntrs);
637
638 bitmap_zero(mask, cci_pmu->num_cntrs);
639 for_each_set_bit(i, cci_pmu->hw_events.used_mask, cci_pmu->num_cntrs) {
640 struct perf_event *event = cci_hw->events[i];
641
642 if (WARN_ON(!event))
643 continue;
644
645 /* Leave the events which are not counting */
646 if (event->hw.state & PERF_HES_STOPPED)
647 continue;
648 if (event->hw.state & PERF_HES_ARCH) {
649 set_bit(i, mask);
650 event->hw.state &= ~PERF_HES_ARCH;
651 }
652 }
653
654 pmu_write_counters(cci_pmu, mask);
655}
656
657/* Should be called with cci_pmu->hw_events->pmu_lock held */
658static void __cci_pmu_enable_nosync(struct cci_pmu *cci_pmu)
659{
660 u32 val;
661
662 /* Enable all the PMU counters. */
663 val = readl_relaxed(cci_ctrl_base + CCI_PMCR) | CCI_PMCR_CEN;
664 writel(val, cci_ctrl_base + CCI_PMCR);
665}
666
667/* Should be called with cci_pmu->hw_events->pmu_lock held */
668static void __cci_pmu_enable_sync(struct cci_pmu *cci_pmu)
669{
670 cci_pmu_sync_counters(cci_pmu);
671 __cci_pmu_enable_nosync(cci_pmu);
672}
673
674/* Should be called with cci_pmu->hw_events->pmu_lock held */
675static void __cci_pmu_disable(void)
676{
677 u32 val;
678
679 /* Disable all the PMU counters. */
680 val = readl_relaxed(cci_ctrl_base + CCI_PMCR) & ~CCI_PMCR_CEN;
681 writel(val, cci_ctrl_base + CCI_PMCR);
682}
683
684static ssize_t cci_pmu_format_show(struct device *dev,
685 struct device_attribute *attr, char *buf)
686{
687 struct dev_ext_attribute *eattr = container_of(attr,
688 struct dev_ext_attribute, attr);
689 return snprintf(buf, PAGE_SIZE, "%s\n", (char *)eattr->var);
690}
691
692static ssize_t cci_pmu_event_show(struct device *dev,
693 struct device_attribute *attr, char *buf)
694{
695 struct dev_ext_attribute *eattr = container_of(attr,
696 struct dev_ext_attribute, attr);
697 /* source parameter is mandatory for normal PMU events */
698 return snprintf(buf, PAGE_SIZE, "source=?,event=0x%lx\n",
699 (unsigned long)eattr->var);
700}
701
702static int pmu_is_valid_counter(struct cci_pmu *cci_pmu, int idx)
703{
704 return 0 <= idx && idx <= CCI_PMU_CNTR_LAST(cci_pmu);
705}
706
707static u32 pmu_read_register(struct cci_pmu *cci_pmu, int idx, unsigned int offset)
708{
709 return readl_relaxed(cci_pmu->base +
710 CCI_PMU_CNTR_BASE(cci_pmu->model, idx) + offset);
711}
712
713static void pmu_write_register(struct cci_pmu *cci_pmu, u32 value,
714 int idx, unsigned int offset)
715{
716 writel_relaxed(value, cci_pmu->base +
717 CCI_PMU_CNTR_BASE(cci_pmu->model, idx) + offset);
718}
719
720static void pmu_disable_counter(struct cci_pmu *cci_pmu, int idx)
721{
722 pmu_write_register(cci_pmu, 0, idx, CCI_PMU_CNTR_CTRL);
723}
724
725static void pmu_enable_counter(struct cci_pmu *cci_pmu, int idx)
726{
727 pmu_write_register(cci_pmu, 1, idx, CCI_PMU_CNTR_CTRL);
728}
729
730static bool __maybe_unused
731pmu_counter_is_enabled(struct cci_pmu *cci_pmu, int idx)
732{
733 return (pmu_read_register(cci_pmu, idx, CCI_PMU_CNTR_CTRL) & 0x1) != 0;
734}
735
736static void pmu_set_event(struct cci_pmu *cci_pmu, int idx, unsigned long event)
737{
738 pmu_write_register(cci_pmu, event, idx, CCI_PMU_EVT_SEL);
739}
740
741/*
742 * For all counters on the CCI-PMU, disable any 'enabled' counters,
743 * saving the changed counters in the mask, so that we can restore
744 * it later using pmu_restore_counters. The mask is private to the
745 * caller. We cannot rely on the used_mask maintained by the CCI_PMU
746 * as it only tells us if the counter is assigned to perf_event or not.
747 * The state of the perf_event cannot be locked by the PMU layer, hence
748 * we check the individual counter status (which can be locked by
749 * cci_pm->hw_events->pmu_lock).
750 *
751 * @mask should be initialised to empty by the caller.
752 */
753static void __maybe_unused
754pmu_save_counters(struct cci_pmu *cci_pmu, unsigned long *mask)
755{
756 int i;
757
758 for (i = 0; i < cci_pmu->num_cntrs; i++) {
759 if (pmu_counter_is_enabled(cci_pmu, i)) {
760 set_bit(i, mask);
761 pmu_disable_counter(cci_pmu, i);
762 }
763 }
764}
765
766/*
767 * Restore the status of the counters. Reversal of the pmu_save_counters().
768 * For each counter set in the mask, enable the counter back.
769 */
770static void __maybe_unused
771pmu_restore_counters(struct cci_pmu *cci_pmu, unsigned long *mask)
772{
773 int i;
774
775 for_each_set_bit(i, mask, cci_pmu->num_cntrs)
776 pmu_enable_counter(cci_pmu, i);
777}
778
779/*
780 * Returns the number of programmable counters actually implemented
781 * by the cci
782 */
783static u32 pmu_get_max_counters(void)
784{
785 return (readl_relaxed(cci_ctrl_base + CCI_PMCR) &
786 CCI_PMCR_NCNT_MASK) >> CCI_PMCR_NCNT_SHIFT;
787}
788
789static int pmu_get_event_idx(struct cci_pmu_hw_events *hw, struct perf_event *event)
790{
791 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
792 unsigned long cci_event = event->hw.config_base;
793 int idx;
794
795 if (cci_pmu->model->get_event_idx)
796 return cci_pmu->model->get_event_idx(cci_pmu, hw, cci_event);
797
798 /* Generic code to find an unused idx from the mask */
799 for(idx = 0; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++)
800 if (!test_and_set_bit(idx, hw->used_mask))
801 return idx;
802
803 /* No counters available */
804 return -EAGAIN;
805}
806
807static int pmu_map_event(struct perf_event *event)
808{
809 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
810
811 if (event->attr.type < PERF_TYPE_MAX ||
812 !cci_pmu->model->validate_hw_event)
813 return -ENOENT;
814
815 return cci_pmu->model->validate_hw_event(cci_pmu, event->attr.config);
816}
817
818static int pmu_request_irq(struct cci_pmu *cci_pmu, irq_handler_t handler)
819{
820 int i;
821 struct platform_device *pmu_device = cci_pmu->plat_device;
822
823 if (unlikely(!pmu_device))
824 return -ENODEV;
825
826 if (cci_pmu->nr_irqs < 1) {
827 dev_err(&pmu_device->dev, "no irqs for CCI PMUs defined\n");
828 return -ENODEV;
829 }
830
831 /*
832 * Register all available CCI PMU interrupts. In the interrupt handler
833 * we iterate over the counters checking for interrupt source (the
834 * overflowing counter) and clear it.
835 *
836 * This should allow handling of non-unique interrupt for the counters.
837 */
838 for (i = 0; i < cci_pmu->nr_irqs; i++) {
839 int err = request_irq(cci_pmu->irqs[i], handler, IRQF_SHARED,
840 "arm-cci-pmu", cci_pmu);
841 if (err) {
842 dev_err(&pmu_device->dev, "unable to request IRQ%d for ARM CCI PMU counters\n",
843 cci_pmu->irqs[i]);
844 return err;
845 }
846
847 set_bit(i, &cci_pmu->active_irqs);
848 }
849
850 return 0;
851}
852
853static void pmu_free_irq(struct cci_pmu *cci_pmu)
854{
855 int i;
856
857 for (i = 0; i < cci_pmu->nr_irqs; i++) {
858 if (!test_and_clear_bit(i, &cci_pmu->active_irqs))
859 continue;
860
861 free_irq(cci_pmu->irqs[i], cci_pmu);
862 }
863}
864
865static u32 pmu_read_counter(struct perf_event *event)
866{
867 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
868 struct hw_perf_event *hw_counter = &event->hw;
869 int idx = hw_counter->idx;
870 u32 value;
871
872 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) {
873 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);
874 return 0;
875 }
876 value = pmu_read_register(cci_pmu, idx, CCI_PMU_CNTR);
877
878 return value;
879}
880
881static void pmu_write_counter(struct cci_pmu *cci_pmu, u32 value, int idx)
882{
883 pmu_write_register(cci_pmu, value, idx, CCI_PMU_CNTR);
884}
885
886static void __pmu_write_counters(struct cci_pmu *cci_pmu, unsigned long *mask)
887{
888 int i;
889 struct cci_pmu_hw_events *cci_hw = &cci_pmu->hw_events;
890
891 for_each_set_bit(i, mask, cci_pmu->num_cntrs) {
892 struct perf_event *event = cci_hw->events[i];
893
894 if (WARN_ON(!event))
895 continue;
896 pmu_write_counter(cci_pmu, local64_read(&event->hw.prev_count), i);
897 }
898}
899
900static void pmu_write_counters(struct cci_pmu *cci_pmu, unsigned long *mask)
901{
902 if (cci_pmu->model->write_counters)
903 cci_pmu->model->write_counters(cci_pmu, mask);
904 else
905 __pmu_write_counters(cci_pmu, mask);
906}
907
908#ifdef CONFIG_ARM_CCI5xx_PMU
909
910/*
911 * CCI-500/CCI-550 has advanced power saving policies, which could gate the
912 * clocks to the PMU counters, which makes the writes to them ineffective.
913 * The only way to write to those counters is when the global counters
914 * are enabled and the particular counter is enabled.
915 *
916 * So we do the following :
917 *
918 * 1) Disable all the PMU counters, saving their current state
919 * 2) Enable the global PMU profiling, now that all counters are
920 * disabled.
921 *
922 * For each counter to be programmed, repeat steps 3-7:
923 *
924 * 3) Write an invalid event code to the event control register for the
925 counter, so that the counters are not modified.
926 * 4) Enable the counter control for the counter.
927 * 5) Set the counter value
928 * 6) Disable the counter
929 * 7) Restore the event in the target counter
930 *
931 * 8) Disable the global PMU.
932 * 9) Restore the status of the rest of the counters.
933 *
934 * We choose an event which for CCI-5xx is guaranteed not to count.
935 * We use the highest possible event code (0x1f) for the master interface 0.
936 */
937#define CCI5xx_INVALID_EVENT ((CCI5xx_PORT_M0 << CCI5xx_PMU_EVENT_SOURCE_SHIFT) | \
938 (CCI5xx_PMU_EVENT_CODE_MASK << CCI5xx_PMU_EVENT_CODE_SHIFT))
939static void cci5xx_pmu_write_counters(struct cci_pmu *cci_pmu, unsigned long *mask)
940{
941 int i;
942 DECLARE_BITMAP(saved_mask, cci_pmu->num_cntrs);
943
944 bitmap_zero(saved_mask, cci_pmu->num_cntrs);
945 pmu_save_counters(cci_pmu, saved_mask);
946
947 /*
948 * Now that all the counters are disabled, we can safely turn the PMU on,
949 * without syncing the status of the counters
950 */
951 __cci_pmu_enable_nosync(cci_pmu);
952
953 for_each_set_bit(i, mask, cci_pmu->num_cntrs) {
954 struct perf_event *event = cci_pmu->hw_events.events[i];
955
956 if (WARN_ON(!event))
957 continue;
958
959 pmu_set_event(cci_pmu, i, CCI5xx_INVALID_EVENT);
960 pmu_enable_counter(cci_pmu, i);
961 pmu_write_counter(cci_pmu, local64_read(&event->hw.prev_count), i);
962 pmu_disable_counter(cci_pmu, i);
963 pmu_set_event(cci_pmu, i, event->hw.config_base);
964 }
965
966 __cci_pmu_disable();
967
968 pmu_restore_counters(cci_pmu, saved_mask);
969}
970
971#endif /* CONFIG_ARM_CCI5xx_PMU */
972
973static u64 pmu_event_update(struct perf_event *event)
974{
975 struct hw_perf_event *hwc = &event->hw;
976 u64 delta, prev_raw_count, new_raw_count;
977
978 do {
979 prev_raw_count = local64_read(&hwc->prev_count);
980 new_raw_count = pmu_read_counter(event);
981 } while (local64_cmpxchg(&hwc->prev_count, prev_raw_count,
982 new_raw_count) != prev_raw_count);
983
984 delta = (new_raw_count - prev_raw_count) & CCI_PMU_CNTR_MASK;
985
986 local64_add(delta, &event->count);
987
988 return new_raw_count;
989}
990
991static void pmu_read(struct perf_event *event)
992{
993 pmu_event_update(event);
994}
995
996static void pmu_event_set_period(struct perf_event *event)
997{
998 struct hw_perf_event *hwc = &event->hw;
999 /*
1000 * The CCI PMU counters have a period of 2^32. To account for the
1001 * possiblity of extreme interrupt latency we program for a period of
1002 * half that. Hopefully we can handle the interrupt before another 2^31
1003 * events occur and the counter overtakes its previous value.
1004 */
1005 u64 val = 1ULL << 31;
1006 local64_set(&hwc->prev_count, val);
1007
1008 /*
1009 * CCI PMU uses PERF_HES_ARCH to keep track of the counters, whose
1010 * values needs to be sync-ed with the s/w state before the PMU is
1011 * enabled.
1012 * Mark this counter for sync.
1013 */
1014 hwc->state |= PERF_HES_ARCH;
1015}
1016
1017static irqreturn_t pmu_handle_irq(int irq_num, void *dev)
1018{
1019 unsigned long flags;
1020 struct cci_pmu *cci_pmu = dev;
1021 struct cci_pmu_hw_events *events = &cci_pmu->hw_events;
1022 int idx, handled = IRQ_NONE;
1023
1024 raw_spin_lock_irqsave(&events->pmu_lock, flags);
1025
1026 /* Disable the PMU while we walk through the counters */
1027 __cci_pmu_disable();
1028 /*
1029 * Iterate over counters and update the corresponding perf events.
1030 * This should work regardless of whether we have per-counter overflow
1031 * interrupt or a combined overflow interrupt.
1032 */
1033 for (idx = 0; idx <= CCI_PMU_CNTR_LAST(cci_pmu); idx++) {
1034 struct perf_event *event = events->events[idx];
1035
1036 if (!event)
1037 continue;
1038
1039 /* Did this counter overflow? */
1040 if (!(pmu_read_register(cci_pmu, idx, CCI_PMU_OVRFLW) &
1041 CCI_PMU_OVRFLW_FLAG))
1042 continue;
1043
1044 pmu_write_register(cci_pmu, CCI_PMU_OVRFLW_FLAG, idx,
1045 CCI_PMU_OVRFLW);
1046
1047 pmu_event_update(event);
1048 pmu_event_set_period(event);
1049 handled = IRQ_HANDLED;
1050 }
1051
1052 /* Enable the PMU and sync possibly overflowed counters */
1053 __cci_pmu_enable_sync(cci_pmu);
1054 raw_spin_unlock_irqrestore(&events->pmu_lock, flags);
1055
1056 return IRQ_RETVAL(handled);
1057}
1058
1059static int cci_pmu_get_hw(struct cci_pmu *cci_pmu)
1060{
1061 int ret = pmu_request_irq(cci_pmu, pmu_handle_irq);
1062 if (ret) {
1063 pmu_free_irq(cci_pmu);
1064 return ret;
1065 }
1066 return 0;
1067}
1068
1069static void cci_pmu_put_hw(struct cci_pmu *cci_pmu)
1070{
1071 pmu_free_irq(cci_pmu);
1072}
1073
1074static void hw_perf_event_destroy(struct perf_event *event)
1075{
1076 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1077 atomic_t *active_events = &cci_pmu->active_events;
1078 struct mutex *reserve_mutex = &cci_pmu->reserve_mutex;
1079
1080 if (atomic_dec_and_mutex_lock(active_events, reserve_mutex)) {
1081 cci_pmu_put_hw(cci_pmu);
1082 mutex_unlock(reserve_mutex);
1083 }
1084}
1085
1086static void cci_pmu_enable(struct pmu *pmu)
1087{
1088 struct cci_pmu *cci_pmu = to_cci_pmu(pmu);
1089 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
1090 int enabled = bitmap_weight(hw_events->used_mask, cci_pmu->num_cntrs);
1091 unsigned long flags;
1092
1093 if (!enabled)
1094 return;
1095
1096 raw_spin_lock_irqsave(&hw_events->pmu_lock, flags);
1097 __cci_pmu_enable_sync(cci_pmu);
1098 raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags);
1099
1100}
1101
1102static void cci_pmu_disable(struct pmu *pmu)
1103{
1104 struct cci_pmu *cci_pmu = to_cci_pmu(pmu);
1105 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
1106 unsigned long flags;
1107
1108 raw_spin_lock_irqsave(&hw_events->pmu_lock, flags);
1109 __cci_pmu_disable();
1110 raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags);
1111}
1112
1113/*
1114 * Check if the idx represents a non-programmable counter.
1115 * All the fixed event counters are mapped before the programmable
1116 * counters.
1117 */
1118static bool pmu_fixed_hw_idx(struct cci_pmu *cci_pmu, int idx)
1119{
1120 return (idx >= 0) && (idx < cci_pmu->model->fixed_hw_cntrs);
1121}
1122
1123static void cci_pmu_start(struct perf_event *event, int pmu_flags)
1124{
1125 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1126 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
1127 struct hw_perf_event *hwc = &event->hw;
1128 int idx = hwc->idx;
1129 unsigned long flags;
1130
1131 /*
1132 * To handle interrupt latency, we always reprogram the period
1133 * regardlesss of PERF_EF_RELOAD.
1134 */
1135 if (pmu_flags & PERF_EF_RELOAD)
1136 WARN_ON_ONCE(!(hwc->state & PERF_HES_UPTODATE));
1137
1138 hwc->state = 0;
1139
1140 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) {
1141 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);
1142 return;
1143 }
1144
1145 raw_spin_lock_irqsave(&hw_events->pmu_lock, flags);
1146
1147 /* Configure the counter unless you are counting a fixed event */
1148 if (!pmu_fixed_hw_idx(cci_pmu, idx))
1149 pmu_set_event(cci_pmu, idx, hwc->config_base);
1150
1151 pmu_event_set_period(event);
1152 pmu_enable_counter(cci_pmu, idx);
1153
1154 raw_spin_unlock_irqrestore(&hw_events->pmu_lock, flags);
1155}
1156
1157static void cci_pmu_stop(struct perf_event *event, int pmu_flags)
1158{
1159 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1160 struct hw_perf_event *hwc = &event->hw;
1161 int idx = hwc->idx;
1162
1163 if (hwc->state & PERF_HES_STOPPED)
1164 return;
1165
1166 if (unlikely(!pmu_is_valid_counter(cci_pmu, idx))) {
1167 dev_err(&cci_pmu->plat_device->dev, "Invalid CCI PMU counter %d\n", idx);
1168 return;
1169 }
1170
1171 /*
1172 * We always reprogram the counter, so ignore PERF_EF_UPDATE. See
1173 * cci_pmu_start()
1174 */
1175 pmu_disable_counter(cci_pmu, idx);
1176 pmu_event_update(event);
1177 hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE;
1178}
1179
1180static int cci_pmu_add(struct perf_event *event, int flags)
1181{
1182 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1183 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
1184 struct hw_perf_event *hwc = &event->hw;
1185 int idx;
1186 int err = 0;
1187
1188 perf_pmu_disable(event->pmu);
1189
1190 /* If we don't have a space for the counter then finish early. */
1191 idx = pmu_get_event_idx(hw_events, event);
1192 if (idx < 0) {
1193 err = idx;
1194 goto out;
1195 }
1196
1197 event->hw.idx = idx;
1198 hw_events->events[idx] = event;
1199
1200 hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE;
1201 if (flags & PERF_EF_START)
1202 cci_pmu_start(event, PERF_EF_RELOAD);
1203
1204 /* Propagate our changes to the userspace mapping. */
1205 perf_event_update_userpage(event);
1206
1207out:
1208 perf_pmu_enable(event->pmu);
1209 return err;
1210}
1211
1212static void cci_pmu_del(struct perf_event *event, int flags)
1213{
1214 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1215 struct cci_pmu_hw_events *hw_events = &cci_pmu->hw_events;
1216 struct hw_perf_event *hwc = &event->hw;
1217 int idx = hwc->idx;
1218
1219 cci_pmu_stop(event, PERF_EF_UPDATE);
1220 hw_events->events[idx] = NULL;
1221 clear_bit(idx, hw_events->used_mask);
1222
1223 perf_event_update_userpage(event);
1224}
1225
1226static int validate_event(struct pmu *cci_pmu,
1227 struct cci_pmu_hw_events *hw_events,
1228 struct perf_event *event)
1229{
1230 if (is_software_event(event))
1231 return 1;
1232
1233 /*
1234 * Reject groups spanning multiple HW PMUs (e.g. CPU + CCI). The
1235 * core perf code won't check that the pmu->ctx == leader->ctx
1236 * until after pmu->event_init(event).
1237 */
1238 if (event->pmu != cci_pmu)
1239 return 0;
1240
1241 if (event->state < PERF_EVENT_STATE_OFF)
1242 return 1;
1243
1244 if (event->state == PERF_EVENT_STATE_OFF && !event->attr.enable_on_exec)
1245 return 1;
1246
1247 return pmu_get_event_idx(hw_events, event) >= 0;
1248}
1249
1250static int validate_group(struct perf_event *event)
1251{
1252 struct perf_event *sibling, *leader = event->group_leader;
1253 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1254 unsigned long mask[BITS_TO_LONGS(cci_pmu->num_cntrs)];
1255 struct cci_pmu_hw_events fake_pmu = {
1256 /*
1257 * Initialise the fake PMU. We only need to populate the
1258 * used_mask for the purposes of validation.
1259 */
1260 .used_mask = mask,
1261 };
1262 memset(mask, 0, BITS_TO_LONGS(cci_pmu->num_cntrs) * sizeof(unsigned long));
1263
1264 if (!validate_event(event->pmu, &fake_pmu, leader))
1265 return -EINVAL;
1266
1267 list_for_each_entry(sibling, &leader->sibling_list, group_entry) {
1268 if (!validate_event(event->pmu, &fake_pmu, sibling))
1269 return -EINVAL;
1270 }
1271
1272 if (!validate_event(event->pmu, &fake_pmu, event))
1273 return -EINVAL;
1274
1275 return 0;
1276}
1277
1278static int __hw_perf_event_init(struct perf_event *event)
1279{
1280 struct hw_perf_event *hwc = &event->hw;
1281 int mapping;
1282
1283 mapping = pmu_map_event(event);
1284
1285 if (mapping < 0) {
1286 pr_debug("event %x:%llx not supported\n", event->attr.type,
1287 event->attr.config);
1288 return mapping;
1289 }
1290
1291 /*
1292 * We don't assign an index until we actually place the event onto
1293 * hardware. Use -1 to signify that we haven't decided where to put it
1294 * yet.
1295 */
1296 hwc->idx = -1;
1297 hwc->config_base = 0;
1298 hwc->config = 0;
1299 hwc->event_base = 0;
1300
1301 /*
1302 * Store the event encoding into the config_base field.
1303 */
1304 hwc->config_base |= (unsigned long)mapping;
1305
1306 /*
1307 * Limit the sample_period to half of the counter width. That way, the
1308 * new counter value is far less likely to overtake the previous one
1309 * unless you have some serious IRQ latency issues.
1310 */
1311 hwc->sample_period = CCI_PMU_CNTR_MASK >> 1;
1312 hwc->last_period = hwc->sample_period;
1313 local64_set(&hwc->period_left, hwc->sample_period);
1314
1315 if (event->group_leader != event) {
1316 if (validate_group(event) != 0)
1317 return -EINVAL;
1318 }
1319
1320 return 0;
1321}
1322
1323static int cci_pmu_event_init(struct perf_event *event)
1324{
1325 struct cci_pmu *cci_pmu = to_cci_pmu(event->pmu);
1326 atomic_t *active_events = &cci_pmu->active_events;
1327 int err = 0;
1328 int cpu;
1329
1330 if (event->attr.type != event->pmu->type)
1331 return -ENOENT;
1332
1333 /* Shared by all CPUs, no meaningful state to sample */
1334 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
1335 return -EOPNOTSUPP;
1336
1337 /* We have no filtering of any kind */
1338 if (event->attr.exclude_user ||
1339 event->attr.exclude_kernel ||
1340 event->attr.exclude_hv ||
1341 event->attr.exclude_idle ||
1342 event->attr.exclude_host ||
1343 event->attr.exclude_guest)
1344 return -EINVAL;
1345
1346 /*
1347 * Following the example set by other "uncore" PMUs, we accept any CPU
1348 * and rewrite its affinity dynamically rather than having perf core
1349 * handle cpu == -1 and pid == -1 for this case.
1350 *
1351 * The perf core will pin online CPUs for the duration of this call and
1352 * the event being installed into its context, so the PMU's CPU can't
1353 * change under our feet.
1354 */
1355 cpu = cpumask_first(&cci_pmu->cpus);
1356 if (event->cpu < 0 || cpu < 0)
1357 return -EINVAL;
1358 event->cpu = cpu;
1359
1360 event->destroy = hw_perf_event_destroy;
1361 if (!atomic_inc_not_zero(active_events)) {
1362 mutex_lock(&cci_pmu->reserve_mutex);
1363 if (atomic_read(active_events) == 0)
1364 err = cci_pmu_get_hw(cci_pmu);
1365 if (!err)
1366 atomic_inc(active_events);
1367 mutex_unlock(&cci_pmu->reserve_mutex);
1368 }
1369 if (err)
1370 return err;
1371
1372 err = __hw_perf_event_init(event);
1373 if (err)
1374 hw_perf_event_destroy(event);
1375
1376 return err;
1377}
1378
1379static ssize_t pmu_cpumask_attr_show(struct device *dev,
1380 struct device_attribute *attr, char *buf)
1381{
1382 struct pmu *pmu = dev_get_drvdata(dev);
1383 struct cci_pmu *cci_pmu = to_cci_pmu(pmu);
1384
1385 int n = scnprintf(buf, PAGE_SIZE - 1, "%*pbl",
1386 cpumask_pr_args(&cci_pmu->cpus));
1387 buf[n++] = '\n';
1388 buf[n] = '\0';
1389 return n;
1390}
1391
1392static struct device_attribute pmu_cpumask_attr =
1393 __ATTR(cpumask, S_IRUGO, pmu_cpumask_attr_show, NULL);
1394
1395static struct attribute *pmu_attrs[] = {
1396 &pmu_cpumask_attr.attr,
1397 NULL,
1398};
1399
1400static struct attribute_group pmu_attr_group = {
1401 .attrs = pmu_attrs,
1402};
1403
1404static struct attribute_group pmu_format_attr_group = {
1405 .name = "format",
1406 .attrs = NULL, /* Filled in cci_pmu_init_attrs */
1407};
1408
1409static struct attribute_group pmu_event_attr_group = {
1410 .name = "events",
1411 .attrs = NULL, /* Filled in cci_pmu_init_attrs */
1412};
1413
1414static const struct attribute_group *pmu_attr_groups[] = {
1415 &pmu_attr_group,
1416 &pmu_format_attr_group,
1417 &pmu_event_attr_group,
1418 NULL
1419};
1420
1421static int cci_pmu_init(struct cci_pmu *cci_pmu, struct platform_device *pdev)
1422{
1423 const struct cci_pmu_model *model = cci_pmu->model;
1424 char *name = model->name;
1425 u32 num_cntrs;
1426
1427 pmu_event_attr_group.attrs = model->event_attrs;
1428 pmu_format_attr_group.attrs = model->format_attrs;
1429
1430 cci_pmu->pmu = (struct pmu) {
1431 .name = cci_pmu->model->name,
1432 .task_ctx_nr = perf_invalid_context,
1433 .pmu_enable = cci_pmu_enable,
1434 .pmu_disable = cci_pmu_disable,
1435 .event_init = cci_pmu_event_init,
1436 .add = cci_pmu_add,
1437 .del = cci_pmu_del,
1438 .start = cci_pmu_start,
1439 .stop = cci_pmu_stop,
1440 .read = pmu_read,
1441 .attr_groups = pmu_attr_groups,
1442 };
1443
1444 cci_pmu->plat_device = pdev;
1445 num_cntrs = pmu_get_max_counters();
1446 if (num_cntrs > cci_pmu->model->num_hw_cntrs) {
1447 dev_warn(&pdev->dev,
1448 "PMU implements more counters(%d) than supported by"
1449 " the model(%d), truncated.",
1450 num_cntrs, cci_pmu->model->num_hw_cntrs);
1451 num_cntrs = cci_pmu->model->num_hw_cntrs;
1452 }
1453 cci_pmu->num_cntrs = num_cntrs + cci_pmu->model->fixed_hw_cntrs;
1454
1455 return perf_pmu_register(&cci_pmu->pmu, name, -1);
1456}
1457
1458static int cci_pmu_offline_cpu(unsigned int cpu, struct hlist_node *node)
1459{
1460 struct cci_pmu *cci_pmu = hlist_entry_safe(node, struct cci_pmu, node);
1461 unsigned int target;
1462
1463 if (!cpumask_test_and_clear_cpu(cpu, &cci_pmu->cpus))
1464 return 0;
1465 target = cpumask_any_but(cpu_online_mask, cpu);
1466 if (target >= nr_cpu_ids)
1467 return 0;
1468 /*
1469 * TODO: migrate context once core races on event->ctx have
1470 * been fixed.
1471 */
1472 cpumask_set_cpu(target, &cci_pmu->cpus);
1473 return 0;
1474}
1475
1476static struct cci_pmu_model cci_pmu_models[] = {
1477#ifdef CONFIG_ARM_CCI400_PMU
1478 [CCI400_R0] = {
1479 .name = "CCI_400",
1480 .fixed_hw_cntrs = 1, /* Cycle counter */
1481 .num_hw_cntrs = 4,
1482 .cntr_size = SZ_4K,
1483 .format_attrs = cci400_pmu_format_attrs,
1484 .event_attrs = cci400_r0_pmu_event_attrs,
1485 .event_ranges = {
1486 [CCI_IF_SLAVE] = {
1487 CCI400_R0_SLAVE_PORT_MIN_EV,
1488 CCI400_R0_SLAVE_PORT_MAX_EV,
1489 },
1490 [CCI_IF_MASTER] = {
1491 CCI400_R0_MASTER_PORT_MIN_EV,
1492 CCI400_R0_MASTER_PORT_MAX_EV,
1493 },
1494 },
1495 .validate_hw_event = cci400_validate_hw_event,
1496 .get_event_idx = cci400_get_event_idx,
1497 },
1498 [CCI400_R1] = {
1499 .name = "CCI_400_r1",
1500 .fixed_hw_cntrs = 1, /* Cycle counter */
1501 .num_hw_cntrs = 4,
1502 .cntr_size = SZ_4K,
1503 .format_attrs = cci400_pmu_format_attrs,
1504 .event_attrs = cci400_r1_pmu_event_attrs,
1505 .event_ranges = {
1506 [CCI_IF_SLAVE] = {
1507 CCI400_R1_SLAVE_PORT_MIN_EV,
1508 CCI400_R1_SLAVE_PORT_MAX_EV,
1509 },
1510 [CCI_IF_MASTER] = {
1511 CCI400_R1_MASTER_PORT_MIN_EV,
1512 CCI400_R1_MASTER_PORT_MAX_EV,
1513 },
1514 },
1515 .validate_hw_event = cci400_validate_hw_event,
1516 .get_event_idx = cci400_get_event_idx,
1517 },
1518#endif
1519#ifdef CONFIG_ARM_CCI5xx_PMU
1520 [CCI500_R0] = {
1521 .name = "CCI_500",
1522 .fixed_hw_cntrs = 0,
1523 .num_hw_cntrs = 8,
1524 .cntr_size = SZ_64K,
1525 .format_attrs = cci5xx_pmu_format_attrs,
1526 .event_attrs = cci5xx_pmu_event_attrs,
1527 .event_ranges = {
1528 [CCI_IF_SLAVE] = {
1529 CCI5xx_SLAVE_PORT_MIN_EV,
1530 CCI5xx_SLAVE_PORT_MAX_EV,
1531 },
1532 [CCI_IF_MASTER] = {
1533 CCI5xx_MASTER_PORT_MIN_EV,
1534 CCI5xx_MASTER_PORT_MAX_EV,
1535 },
1536 [CCI_IF_GLOBAL] = {
1537 CCI5xx_GLOBAL_PORT_MIN_EV,
1538 CCI5xx_GLOBAL_PORT_MAX_EV,
1539 },
1540 },
1541 .validate_hw_event = cci500_validate_hw_event,
1542 .write_counters = cci5xx_pmu_write_counters,
1543 },
1544 [CCI550_R0] = {
1545 .name = "CCI_550",
1546 .fixed_hw_cntrs = 0,
1547 .num_hw_cntrs = 8,
1548 .cntr_size = SZ_64K,
1549 .format_attrs = cci5xx_pmu_format_attrs,
1550 .event_attrs = cci5xx_pmu_event_attrs,
1551 .event_ranges = {
1552 [CCI_IF_SLAVE] = {
1553 CCI5xx_SLAVE_PORT_MIN_EV,
1554 CCI5xx_SLAVE_PORT_MAX_EV,
1555 },
1556 [CCI_IF_MASTER] = {
1557 CCI5xx_MASTER_PORT_MIN_EV,
1558 CCI5xx_MASTER_PORT_MAX_EV,
1559 },
1560 [CCI_IF_GLOBAL] = {
1561 CCI5xx_GLOBAL_PORT_MIN_EV,
1562 CCI5xx_GLOBAL_PORT_MAX_EV,
1563 },
1564 },
1565 .validate_hw_event = cci550_validate_hw_event,
1566 .write_counters = cci5xx_pmu_write_counters,
1567 },
1568#endif
1569};
1570
1571static const struct of_device_id arm_cci_pmu_matches[] = {
1572#ifdef CONFIG_ARM_CCI400_PMU
1573 {
1574 .compatible = "arm,cci-400-pmu",
1575 .data = NULL,
1576 },
1577 {
1578 .compatible = "arm,cci-400-pmu,r0",
1579 .data = &cci_pmu_models[CCI400_R0],
1580 },
1581 {
1582 .compatible = "arm,cci-400-pmu,r1",
1583 .data = &cci_pmu_models[CCI400_R1],
1584 },
1585#endif
1586#ifdef CONFIG_ARM_CCI5xx_PMU
1587 {
1588 .compatible = "arm,cci-500-pmu,r0",
1589 .data = &cci_pmu_models[CCI500_R0],
1590 },
1591 {
1592 .compatible = "arm,cci-550-pmu,r0",
1593 .data = &cci_pmu_models[CCI550_R0],
1594 },
1595#endif
1596 {},
1597};
1598
1599static inline const struct cci_pmu_model *get_cci_model(struct platform_device *pdev)
1600{
1601 const struct of_device_id *match = of_match_node(arm_cci_pmu_matches,
1602 pdev->dev.of_node);
1603 if (!match)
1604 return NULL;
1605 if (match->data)
1606 return match->data;
1607
1608 dev_warn(&pdev->dev, "DEPRECATED compatible property,"
1609 "requires secure access to CCI registers");
1610 return probe_cci_model(pdev);
1611}
1612
1613static bool is_duplicate_irq(int irq, int *irqs, int nr_irqs)
1614{
1615 int i;
1616
1617 for (i = 0; i < nr_irqs; i++)
1618 if (irq == irqs[i])
1619 return true;
1620
1621 return false;
1622}
1623
1624static struct cci_pmu *cci_pmu_alloc(struct platform_device *pdev)
1625{
1626 struct cci_pmu *cci_pmu;
1627 const struct cci_pmu_model *model;
1628
1629 /*
1630 * All allocations are devm_* hence we don't have to free
1631 * them explicitly on an error, as it would end up in driver
1632 * detach.
1633 */
1634 model = get_cci_model(pdev);
1635 if (!model) {
1636 dev_warn(&pdev->dev, "CCI PMU version not supported\n");
1637 return ERR_PTR(-ENODEV);
1638 }
1639
1640 cci_pmu = devm_kzalloc(&pdev->dev, sizeof(*cci_pmu), GFP_KERNEL);
1641 if (!cci_pmu)
1642 return ERR_PTR(-ENOMEM);
1643
1644 cci_pmu->model = model;
1645 cci_pmu->irqs = devm_kcalloc(&pdev->dev, CCI_PMU_MAX_HW_CNTRS(model),
1646 sizeof(*cci_pmu->irqs), GFP_KERNEL);
1647 if (!cci_pmu->irqs)
1648 return ERR_PTR(-ENOMEM);
1649 cci_pmu->hw_events.events = devm_kcalloc(&pdev->dev,
1650 CCI_PMU_MAX_HW_CNTRS(model),
1651 sizeof(*cci_pmu->hw_events.events),
1652 GFP_KERNEL);
1653 if (!cci_pmu->hw_events.events)
1654 return ERR_PTR(-ENOMEM);
1655 cci_pmu->hw_events.used_mask = devm_kcalloc(&pdev->dev,
1656 BITS_TO_LONGS(CCI_PMU_MAX_HW_CNTRS(model)),
1657 sizeof(*cci_pmu->hw_events.used_mask),
1658 GFP_KERNEL);
1659 if (!cci_pmu->hw_events.used_mask)
1660 return ERR_PTR(-ENOMEM);
1661
1662 return cci_pmu;
1663}
1664
1665static int cci_pmu_probe(struct platform_device *pdev)
1666{
1667 struct resource *res;
1668 struct cci_pmu *cci_pmu;
1669 int i, ret, irq;
1670
1671 cci_pmu = cci_pmu_alloc(pdev);
1672 if (IS_ERR(cci_pmu))
1673 return PTR_ERR(cci_pmu);
1674
1675 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1676 cci_pmu->base = devm_ioremap_resource(&pdev->dev, res);
1677 if (IS_ERR(cci_pmu->base))
1678 return -ENOMEM;
1679
1680 /*
1681 * CCI PMU has one overflow interrupt per counter; but some may be tied
1682 * together to a common interrupt.
1683 */
1684 cci_pmu->nr_irqs = 0;
1685 for (i = 0; i < CCI_PMU_MAX_HW_CNTRS(cci_pmu->model); i++) {
1686 irq = platform_get_irq(pdev, i);
1687 if (irq < 0)
1688 break;
1689
1690 if (is_duplicate_irq(irq, cci_pmu->irqs, cci_pmu->nr_irqs))
1691 continue;
1692
1693 cci_pmu->irqs[cci_pmu->nr_irqs++] = irq;
1694 }
1695
1696 /*
1697 * Ensure that the device tree has as many interrupts as the number
1698 * of counters.
1699 */
1700 if (i < CCI_PMU_MAX_HW_CNTRS(cci_pmu->model)) {
1701 dev_warn(&pdev->dev, "In-correct number of interrupts: %d, should be %d\n",
1702 i, CCI_PMU_MAX_HW_CNTRS(cci_pmu->model));
1703 return -EINVAL;
1704 }
1705
1706 raw_spin_lock_init(&cci_pmu->hw_events.pmu_lock);
1707 mutex_init(&cci_pmu->reserve_mutex);
1708 atomic_set(&cci_pmu->active_events, 0);
1709 cpumask_set_cpu(get_cpu(), &cci_pmu->cpus);
1710
1711 ret = cci_pmu_init(cci_pmu, pdev);
1712 if (ret) {
1713 put_cpu();
1714 return ret;
1715 }
1716
1717 cpuhp_state_add_instance_nocalls(CPUHP_AP_PERF_ARM_CCI_ONLINE,
1718 &cci_pmu->node);
1719 put_cpu();
1720 pr_info("ARM %s PMU driver probed", cci_pmu->model->name);
1721 return 0;
1722}
1723
1724static struct platform_driver cci_pmu_driver = {
1725 .driver = {
1726 .name = DRIVER_NAME,
1727 .of_match_table = arm_cci_pmu_matches,
1728 },
1729 .probe = cci_pmu_probe,
1730};
1731
1732static int __init cci_platform_init(void)
1733{
1734 int ret;
1735
1736 ret = cpuhp_setup_state_multi(CPUHP_AP_PERF_ARM_CCI_ONLINE,
1737 "perf/arm/cci:online", NULL,
1738 cci_pmu_offline_cpu);
1739 if (ret)
1740 return ret;
1741
1742 return platform_driver_register(&cci_pmu_driver);
1743}
1744
1745device_initcall(cci_platform_init);
1746MODULE_LICENSE("GPL");
1747MODULE_DESCRIPTION("ARM CCI PMU support");
generated by cgit v1.2.2 (git 2.25.0) at 2025-08-27 22:20:19 -0400