aboutsummaryrefslogtreecommitdiffstats
path: root/arch/x86/kernel/cpu
diff options
context:
space:
mode:
Diffstat (limited to 'arch/x86/kernel/cpu')
-rw-r--r--arch/x86/kernel/cpu/Makefile6
-rw-r--r--arch/x86/kernel/cpu/mkcapflags.pl48
-rw-r--r--arch/x86/kernel/cpu/mkcapflags.sh41
-rw-r--r--arch/x86/kernel/cpu/perf_event.c89
-rw-r--r--arch/x86/kernel/cpu/perf_event.h56
-rw-r--r--arch/x86/kernel/cpu/perf_event_amd.c138
-rw-r--r--arch/x86/kernel/cpu/perf_event_amd_uncore.c547
-rw-r--r--arch/x86/kernel/cpu/perf_event_intel.c38
-rw-r--r--arch/x86/kernel/cpu/perf_event_intel_ds.c182
-rw-r--r--arch/x86/kernel/cpu/perf_event_intel_uncore.c876
-rw-r--r--arch/x86/kernel/cpu/perf_event_intel_uncore.h64
-rw-r--r--arch/x86/kernel/cpu/perf_event_p4.c9
12 files changed, 1779 insertions, 315 deletions
diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile
index a0e067d3d96c..deef0399fc78 100644
--- a/arch/x86/kernel/cpu/Makefile
+++ b/arch/x86/kernel/cpu/Makefile
@@ -31,7 +31,7 @@ obj-$(CONFIG_CPU_SUP_UMC_32) += umc.o
31obj-$(CONFIG_PERF_EVENTS) += perf_event.o 31obj-$(CONFIG_PERF_EVENTS) += perf_event.o
32 32
33ifdef CONFIG_PERF_EVENTS 33ifdef CONFIG_PERF_EVENTS
34obj-$(CONFIG_CPU_SUP_AMD) += perf_event_amd.o 34obj-$(CONFIG_CPU_SUP_AMD) += perf_event_amd.o perf_event_amd_uncore.o
35obj-$(CONFIG_CPU_SUP_INTEL) += perf_event_p6.o perf_event_knc.o perf_event_p4.o 35obj-$(CONFIG_CPU_SUP_INTEL) += perf_event_p6.o perf_event_knc.o perf_event_p4.o
36obj-$(CONFIG_CPU_SUP_INTEL) += perf_event_intel_lbr.o perf_event_intel_ds.o perf_event_intel.o 36obj-$(CONFIG_CPU_SUP_INTEL) += perf_event_intel_lbr.o perf_event_intel_ds.o perf_event_intel.o
37obj-$(CONFIG_CPU_SUP_INTEL) += perf_event_intel_uncore.o 37obj-$(CONFIG_CPU_SUP_INTEL) += perf_event_intel_uncore.o
@@ -43,10 +43,10 @@ obj-$(CONFIG_MTRR) += mtrr/
43obj-$(CONFIG_X86_LOCAL_APIC) += perfctr-watchdog.o perf_event_amd_ibs.o 43obj-$(CONFIG_X86_LOCAL_APIC) += perfctr-watchdog.o perf_event_amd_ibs.o
44 44
45quiet_cmd_mkcapflags = MKCAP $@ 45quiet_cmd_mkcapflags = MKCAP $@
46 cmd_mkcapflags = $(PERL) $(srctree)/$(src)/mkcapflags.pl $< $@ 46 cmd_mkcapflags = $(CONFIG_SHELL) $(srctree)/$(src)/mkcapflags.sh $< $@
47 47
48cpufeature = $(src)/../../include/asm/cpufeature.h 48cpufeature = $(src)/../../include/asm/cpufeature.h
49 49
50targets += capflags.c 50targets += capflags.c
51$(obj)/capflags.c: $(cpufeature) $(src)/mkcapflags.pl FORCE 51$(obj)/capflags.c: $(cpufeature) $(src)/mkcapflags.sh FORCE
52 $(call if_changed,mkcapflags) 52 $(call if_changed,mkcapflags)
diff --git a/arch/x86/kernel/cpu/mkcapflags.pl b/arch/x86/kernel/cpu/mkcapflags.pl
deleted file mode 100644
index 091972ef49de..000000000000
--- a/arch/x86/kernel/cpu/mkcapflags.pl
+++ /dev/null
@@ -1,48 +0,0 @@
1#!/usr/bin/perl -w
2#
3# Generate the x86_cap_flags[] array from include/asm-x86/cpufeature.h
4#
5
6($in, $out) = @ARGV;
7
8open(IN, "< $in\0") or die "$0: cannot open: $in: $!\n";
9open(OUT, "> $out\0") or die "$0: cannot create: $out: $!\n";
10
11print OUT "#ifndef _ASM_X86_CPUFEATURE_H\n";
12print OUT "#include <asm/cpufeature.h>\n";
13print OUT "#endif\n";
14print OUT "\n";
15print OUT "const char * const x86_cap_flags[NCAPINTS*32] = {\n";
16
17%features = ();
18$err = 0;
19
20while (defined($line = <IN>)) {
21 if ($line =~ /^\s*\#\s*define\s+(X86_FEATURE_(\S+))\s+(.*)$/) {
22 $macro = $1;
23 $feature = "\L$2";
24 $tail = $3;
25 if ($tail =~ /\/\*\s*\"([^"]*)\".*\*\//) {
26 $feature = "\L$1";
27 }
28
29 next if ($feature eq '');
30
31 if ($features{$feature}++) {
32 print STDERR "$in: duplicate feature name: $feature\n";
33 $err++;
34 }
35 printf OUT "\t%-32s = \"%s\",\n", "[$macro]", $feature;
36 }
37}
38print OUT "};\n";
39
40close(IN);
41close(OUT);
42
43if ($err) {
44 unlink($out);
45 exit(1);
46}
47
48exit(0);
diff --git a/arch/x86/kernel/cpu/mkcapflags.sh b/arch/x86/kernel/cpu/mkcapflags.sh
new file mode 100644
index 000000000000..2bf616505499
--- /dev/null
+++ b/arch/x86/kernel/cpu/mkcapflags.sh
@@ -0,0 +1,41 @@
1#!/bin/sh
2#
3# Generate the x86_cap_flags[] array from include/asm/cpufeature.h
4#
5
6IN=$1
7OUT=$2
8
9TABS="$(printf '\t\t\t\t\t')"
10trap 'rm "$OUT"' EXIT
11
12(
13 echo "#ifndef _ASM_X86_CPUFEATURE_H"
14 echo "#include <asm/cpufeature.h>"
15 echo "#endif"
16 echo ""
17 echo "const char * const x86_cap_flags[NCAPINTS*32] = {"
18
19 # Iterate through any input lines starting with #define X86_FEATURE_
20 sed -n -e 's/\t/ /g' -e 's/^ *# *define *X86_FEATURE_//p' $IN |
21 while read i
22 do
23 # Name is everything up to the first whitespace
24 NAME="$(echo "$i" | sed 's/ .*//')"
25
26 # If the /* comment */ starts with a quote string, grab that.
27 VALUE="$(echo "$i" | sed -n 's@.*/\* *\("[^"]*"\).*\*/@\1@p')"
28 [ -z "$VALUE" ] && VALUE="\"$NAME\""
29 [ "$VALUE" == '""' ] && continue
30
31 # Name is uppercase, VALUE is all lowercase
32 VALUE="$(echo "$VALUE" | tr A-Z a-z)"
33
34 TABCOUNT=$(( ( 5*8 - 14 - $(echo "$NAME" | wc -c) ) / 8 ))
35 printf "\t[%s]%.*s = %s,\n" \
36 "X86_FEATURE_$NAME" "$TABCOUNT" "$TABS" "$VALUE"
37 done
38 echo "};"
39) > $OUT
40
41trap - EXIT
diff --git a/arch/x86/kernel/cpu/perf_event.c b/arch/x86/kernel/cpu/perf_event.c
index bf0f01aea994..1025f3c99d20 100644
--- a/arch/x86/kernel/cpu/perf_event.c
+++ b/arch/x86/kernel/cpu/perf_event.c
@@ -180,8 +180,9 @@ static void release_pmc_hardware(void) {}
180 180
181static bool check_hw_exists(void) 181static bool check_hw_exists(void)
182{ 182{
183 u64 val, val_new = ~0; 183 u64 val, val_fail, val_new= ~0;
184 int i, reg, ret = 0; 184 int i, reg, reg_fail, ret = 0;
185 int bios_fail = 0;
185 186
186 /* 187 /*
187 * Check to see if the BIOS enabled any of the counters, if so 188 * Check to see if the BIOS enabled any of the counters, if so
@@ -192,8 +193,11 @@ static bool check_hw_exists(void)
192 ret = rdmsrl_safe(reg, &val); 193 ret = rdmsrl_safe(reg, &val);
193 if (ret) 194 if (ret)
194 goto msr_fail; 195 goto msr_fail;
195 if (val & ARCH_PERFMON_EVENTSEL_ENABLE) 196 if (val & ARCH_PERFMON_EVENTSEL_ENABLE) {
196 goto bios_fail; 197 bios_fail = 1;
198 val_fail = val;
199 reg_fail = reg;
200 }
197 } 201 }
198 202
199 if (x86_pmu.num_counters_fixed) { 203 if (x86_pmu.num_counters_fixed) {
@@ -202,8 +206,11 @@ static bool check_hw_exists(void)
202 if (ret) 206 if (ret)
203 goto msr_fail; 207 goto msr_fail;
204 for (i = 0; i < x86_pmu.num_counters_fixed; i++) { 208 for (i = 0; i < x86_pmu.num_counters_fixed; i++) {
205 if (val & (0x03 << i*4)) 209 if (val & (0x03 << i*4)) {
206 goto bios_fail; 210 bios_fail = 1;
211 val_fail = val;
212 reg_fail = reg;
213 }
207 } 214 }
208 } 215 }
209 216
@@ -221,14 +228,13 @@ static bool check_hw_exists(void)
221 if (ret || val != val_new) 228 if (ret || val != val_new)
222 goto msr_fail; 229 goto msr_fail;
223 230
224 return true;
225
226bios_fail:
227 /* 231 /*
228 * We still allow the PMU driver to operate: 232 * We still allow the PMU driver to operate:
229 */ 233 */
230 printk(KERN_CONT "Broken BIOS detected, complain to your hardware vendor.\n"); 234 if (bios_fail) {
231 printk(KERN_ERR FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n", reg, val); 235 printk(KERN_CONT "Broken BIOS detected, complain to your hardware vendor.\n");
236 printk(KERN_ERR FW_BUG "the BIOS has corrupted hw-PMU resources (MSR %x is %Lx)\n", reg_fail, val_fail);
237 }
232 238
233 return true; 239 return true;
234 240
@@ -1316,9 +1322,16 @@ static struct attribute_group x86_pmu_format_group = {
1316 */ 1322 */
1317static void __init filter_events(struct attribute **attrs) 1323static void __init filter_events(struct attribute **attrs)
1318{ 1324{
1325 struct device_attribute *d;
1326 struct perf_pmu_events_attr *pmu_attr;
1319 int i, j; 1327 int i, j;
1320 1328
1321 for (i = 0; attrs[i]; i++) { 1329 for (i = 0; attrs[i]; i++) {
1330 d = (struct device_attribute *)attrs[i];
1331 pmu_attr = container_of(d, struct perf_pmu_events_attr, attr);
1332 /* str trumps id */
1333 if (pmu_attr->event_str)
1334 continue;
1322 if (x86_pmu.event_map(i)) 1335 if (x86_pmu.event_map(i))
1323 continue; 1336 continue;
1324 1337
@@ -1330,22 +1343,45 @@ static void __init filter_events(struct attribute **attrs)
1330 } 1343 }
1331} 1344}
1332 1345
1333static ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr, 1346/* Merge two pointer arrays */
1347static __init struct attribute **merge_attr(struct attribute **a, struct attribute **b)
1348{
1349 struct attribute **new;
1350 int j, i;
1351
1352 for (j = 0; a[j]; j++)
1353 ;
1354 for (i = 0; b[i]; i++)
1355 j++;
1356 j++;
1357
1358 new = kmalloc(sizeof(struct attribute *) * j, GFP_KERNEL);
1359 if (!new)
1360 return NULL;
1361
1362 j = 0;
1363 for (i = 0; a[i]; i++)
1364 new[j++] = a[i];
1365 for (i = 0; b[i]; i++)
1366 new[j++] = b[i];
1367 new[j] = NULL;
1368
1369 return new;
1370}
1371
1372ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr,
1334 char *page) 1373 char *page)
1335{ 1374{
1336 struct perf_pmu_events_attr *pmu_attr = \ 1375 struct perf_pmu_events_attr *pmu_attr = \
1337 container_of(attr, struct perf_pmu_events_attr, attr); 1376 container_of(attr, struct perf_pmu_events_attr, attr);
1338
1339 u64 config = x86_pmu.event_map(pmu_attr->id); 1377 u64 config = x86_pmu.event_map(pmu_attr->id);
1340 return x86_pmu.events_sysfs_show(page, config);
1341}
1342 1378
1343#define EVENT_VAR(_id) event_attr_##_id 1379 /* string trumps id */
1344#define EVENT_PTR(_id) &event_attr_##_id.attr.attr 1380 if (pmu_attr->event_str)
1381 return sprintf(page, "%s", pmu_attr->event_str);
1345 1382
1346#define EVENT_ATTR(_name, _id) \ 1383 return x86_pmu.events_sysfs_show(page, config);
1347 PMU_EVENT_ATTR(_name, EVENT_VAR(_id), PERF_COUNT_HW_##_id, \ 1384}
1348 events_sysfs_show)
1349 1385
1350EVENT_ATTR(cpu-cycles, CPU_CYCLES ); 1386EVENT_ATTR(cpu-cycles, CPU_CYCLES );
1351EVENT_ATTR(instructions, INSTRUCTIONS ); 1387EVENT_ATTR(instructions, INSTRUCTIONS );
@@ -1459,16 +1495,27 @@ static int __init init_hw_perf_events(void)
1459 1495
1460 unconstrained = (struct event_constraint) 1496 unconstrained = (struct event_constraint)
1461 __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1, 1497 __EVENT_CONSTRAINT(0, (1ULL << x86_pmu.num_counters) - 1,
1462 0, x86_pmu.num_counters, 0); 1498 0, x86_pmu.num_counters, 0, 0);
1463 1499
1464 x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */ 1500 x86_pmu.attr_rdpmc = 1; /* enable userspace RDPMC usage by default */
1465 x86_pmu_format_group.attrs = x86_pmu.format_attrs; 1501 x86_pmu_format_group.attrs = x86_pmu.format_attrs;
1466 1502
1503 if (x86_pmu.event_attrs)
1504 x86_pmu_events_group.attrs = x86_pmu.event_attrs;
1505
1467 if (!x86_pmu.events_sysfs_show) 1506 if (!x86_pmu.events_sysfs_show)
1468 x86_pmu_events_group.attrs = &empty_attrs; 1507 x86_pmu_events_group.attrs = &empty_attrs;
1469 else 1508 else
1470 filter_events(x86_pmu_events_group.attrs); 1509 filter_events(x86_pmu_events_group.attrs);
1471 1510
1511 if (x86_pmu.cpu_events) {
1512 struct attribute **tmp;
1513
1514 tmp = merge_attr(x86_pmu_events_group.attrs, x86_pmu.cpu_events);
1515 if (!WARN_ON(!tmp))
1516 x86_pmu_events_group.attrs = tmp;
1517 }
1518
1472 pr_info("... version: %d\n", x86_pmu.version); 1519 pr_info("... version: %d\n", x86_pmu.version);
1473 pr_info("... bit width: %d\n", x86_pmu.cntval_bits); 1520 pr_info("... bit width: %d\n", x86_pmu.cntval_bits);
1474 pr_info("... generic registers: %d\n", x86_pmu.num_counters); 1521 pr_info("... generic registers: %d\n", x86_pmu.num_counters);
diff --git a/arch/x86/kernel/cpu/perf_event.h b/arch/x86/kernel/cpu/perf_event.h
index 7f5c75c2afdd..ba9aadfa683b 100644
--- a/arch/x86/kernel/cpu/perf_event.h
+++ b/arch/x86/kernel/cpu/perf_event.h
@@ -46,6 +46,7 @@ enum extra_reg_type {
46 EXTRA_REG_RSP_0 = 0, /* offcore_response_0 */ 46 EXTRA_REG_RSP_0 = 0, /* offcore_response_0 */
47 EXTRA_REG_RSP_1 = 1, /* offcore_response_1 */ 47 EXTRA_REG_RSP_1 = 1, /* offcore_response_1 */
48 EXTRA_REG_LBR = 2, /* lbr_select */ 48 EXTRA_REG_LBR = 2, /* lbr_select */
49 EXTRA_REG_LDLAT = 3, /* ld_lat_threshold */
49 50
50 EXTRA_REG_MAX /* number of entries needed */ 51 EXTRA_REG_MAX /* number of entries needed */
51}; 52};
@@ -59,7 +60,13 @@ struct event_constraint {
59 u64 cmask; 60 u64 cmask;
60 int weight; 61 int weight;
61 int overlap; 62 int overlap;
63 int flags;
62}; 64};
65/*
66 * struct event_constraint flags
67 */
68#define PERF_X86_EVENT_PEBS_LDLAT 0x1 /* ld+ldlat data address sampling */
69#define PERF_X86_EVENT_PEBS_ST 0x2 /* st data address sampling */
63 70
64struct amd_nb { 71struct amd_nb {
65 int nb_id; /* NorthBridge id */ 72 int nb_id; /* NorthBridge id */
@@ -170,16 +177,17 @@ struct cpu_hw_events {
170 void *kfree_on_online; 177 void *kfree_on_online;
171}; 178};
172 179
173#define __EVENT_CONSTRAINT(c, n, m, w, o) {\ 180#define __EVENT_CONSTRAINT(c, n, m, w, o, f) {\
174 { .idxmsk64 = (n) }, \ 181 { .idxmsk64 = (n) }, \
175 .code = (c), \ 182 .code = (c), \
176 .cmask = (m), \ 183 .cmask = (m), \
177 .weight = (w), \ 184 .weight = (w), \
178 .overlap = (o), \ 185 .overlap = (o), \
186 .flags = f, \
179} 187}
180 188
181#define EVENT_CONSTRAINT(c, n, m) \ 189#define EVENT_CONSTRAINT(c, n, m) \
182 __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 0) 190 __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 0, 0)
183 191
184/* 192/*
185 * The overlap flag marks event constraints with overlapping counter 193 * The overlap flag marks event constraints with overlapping counter
@@ -203,7 +211,7 @@ struct cpu_hw_events {
203 * and its counter masks must be kept at a minimum. 211 * and its counter masks must be kept at a minimum.
204 */ 212 */
205#define EVENT_CONSTRAINT_OVERLAP(c, n, m) \ 213#define EVENT_CONSTRAINT_OVERLAP(c, n, m) \
206 __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 1) 214 __EVENT_CONSTRAINT(c, n, m, HWEIGHT(n), 1, 0)
207 215
208/* 216/*
209 * Constraint on the Event code. 217 * Constraint on the Event code.
@@ -231,6 +239,14 @@ struct cpu_hw_events {
231#define INTEL_UEVENT_CONSTRAINT(c, n) \ 239#define INTEL_UEVENT_CONSTRAINT(c, n) \
232 EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK) 240 EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK)
233 241
242#define INTEL_PLD_CONSTRAINT(c, n) \
243 __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK, \
244 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_LDLAT)
245
246#define INTEL_PST_CONSTRAINT(c, n) \
247 __EVENT_CONSTRAINT(c, n, INTEL_ARCH_EVENT_MASK, \
248 HWEIGHT(n), 0, PERF_X86_EVENT_PEBS_ST)
249
234#define EVENT_CONSTRAINT_END \ 250#define EVENT_CONSTRAINT_END \
235 EVENT_CONSTRAINT(0, 0, 0) 251 EVENT_CONSTRAINT(0, 0, 0)
236 252
@@ -260,12 +276,22 @@ struct extra_reg {
260 .msr = (ms), \ 276 .msr = (ms), \
261 .config_mask = (m), \ 277 .config_mask = (m), \
262 .valid_mask = (vm), \ 278 .valid_mask = (vm), \
263 .idx = EXTRA_REG_##i \ 279 .idx = EXTRA_REG_##i, \
264 } 280 }
265 281
266#define INTEL_EVENT_EXTRA_REG(event, msr, vm, idx) \ 282#define INTEL_EVENT_EXTRA_REG(event, msr, vm, idx) \
267 EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT, vm, idx) 283 EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT, vm, idx)
268 284
285#define INTEL_UEVENT_EXTRA_REG(event, msr, vm, idx) \
286 EVENT_EXTRA_REG(event, msr, ARCH_PERFMON_EVENTSEL_EVENT | \
287 ARCH_PERFMON_EVENTSEL_UMASK, vm, idx)
288
289#define INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(c) \
290 INTEL_UEVENT_EXTRA_REG(c, \
291 MSR_PEBS_LD_LAT_THRESHOLD, \
292 0xffff, \
293 LDLAT)
294
269#define EVENT_EXTRA_END EVENT_EXTRA_REG(0, 0, 0, 0, RSP_0) 295#define EVENT_EXTRA_END EVENT_EXTRA_REG(0, 0, 0, 0, RSP_0)
270 296
271union perf_capabilities { 297union perf_capabilities {
@@ -355,8 +381,10 @@ struct x86_pmu {
355 */ 381 */
356 int attr_rdpmc; 382 int attr_rdpmc;
357 struct attribute **format_attrs; 383 struct attribute **format_attrs;
384 struct attribute **event_attrs;
358 385
359 ssize_t (*events_sysfs_show)(char *page, u64 config); 386 ssize_t (*events_sysfs_show)(char *page, u64 config);
387 struct attribute **cpu_events;
360 388
361 /* 389 /*
362 * CPU Hotplug hooks 390 * CPU Hotplug hooks
@@ -421,6 +449,23 @@ do { \
421#define ERF_NO_HT_SHARING 1 449#define ERF_NO_HT_SHARING 1
422#define ERF_HAS_RSP_1 2 450#define ERF_HAS_RSP_1 2
423 451
452#define EVENT_VAR(_id) event_attr_##_id
453#define EVENT_PTR(_id) &event_attr_##_id.attr.attr
454
455#define EVENT_ATTR(_name, _id) \
456static struct perf_pmu_events_attr EVENT_VAR(_id) = { \
457 .attr = __ATTR(_name, 0444, events_sysfs_show, NULL), \
458 .id = PERF_COUNT_HW_##_id, \
459 .event_str = NULL, \
460};
461
462#define EVENT_ATTR_STR(_name, v, str) \
463static struct perf_pmu_events_attr event_attr_##v = { \
464 .attr = __ATTR(_name, 0444, events_sysfs_show, NULL), \
465 .id = 0, \
466 .event_str = str, \
467};
468
424extern struct x86_pmu x86_pmu __read_mostly; 469extern struct x86_pmu x86_pmu __read_mostly;
425 470
426DECLARE_PER_CPU(struct cpu_hw_events, cpu_hw_events); 471DECLARE_PER_CPU(struct cpu_hw_events, cpu_hw_events);
@@ -628,6 +673,9 @@ int p6_pmu_init(void);
628 673
629int knc_pmu_init(void); 674int knc_pmu_init(void);
630 675
676ssize_t events_sysfs_show(struct device *dev, struct device_attribute *attr,
677 char *page);
678
631#else /* CONFIG_CPU_SUP_INTEL */ 679#else /* CONFIG_CPU_SUP_INTEL */
632 680
633static inline void reserve_ds_buffers(void) 681static inline void reserve_ds_buffers(void)
diff --git a/arch/x86/kernel/cpu/perf_event_amd.c b/arch/x86/kernel/cpu/perf_event_amd.c
index dfdab42aed27..7e28d9467bb4 100644
--- a/arch/x86/kernel/cpu/perf_event_amd.c
+++ b/arch/x86/kernel/cpu/perf_event_amd.c
@@ -132,14 +132,11 @@ static u64 amd_pmu_event_map(int hw_event)
132 return amd_perfmon_event_map[hw_event]; 132 return amd_perfmon_event_map[hw_event];
133} 133}
134 134
135static struct event_constraint *amd_nb_event_constraint;
136
137/* 135/*
138 * Previously calculated offsets 136 * Previously calculated offsets
139 */ 137 */
140static unsigned int event_offsets[X86_PMC_IDX_MAX] __read_mostly; 138static unsigned int event_offsets[X86_PMC_IDX_MAX] __read_mostly;
141static unsigned int count_offsets[X86_PMC_IDX_MAX] __read_mostly; 139static unsigned int count_offsets[X86_PMC_IDX_MAX] __read_mostly;
142static unsigned int rdpmc_indexes[X86_PMC_IDX_MAX] __read_mostly;
143 140
144/* 141/*
145 * Legacy CPUs: 142 * Legacy CPUs:
@@ -147,14 +144,10 @@ static unsigned int rdpmc_indexes[X86_PMC_IDX_MAX] __read_mostly;
147 * 144 *
148 * CPUs with core performance counter extensions: 145 * CPUs with core performance counter extensions:
149 * 6 counters starting at 0xc0010200 each offset by 2 146 * 6 counters starting at 0xc0010200 each offset by 2
150 *
151 * CPUs with north bridge performance counter extensions:
152 * 4 additional counters starting at 0xc0010240 each offset by 2
153 * (indexed right above either one of the above core counters)
154 */ 147 */
155static inline int amd_pmu_addr_offset(int index, bool eventsel) 148static inline int amd_pmu_addr_offset(int index, bool eventsel)
156{ 149{
157 int offset, first, base; 150 int offset;
158 151
159 if (!index) 152 if (!index)
160 return index; 153 return index;
@@ -167,23 +160,7 @@ static inline int amd_pmu_addr_offset(int index, bool eventsel)
167 if (offset) 160 if (offset)
168 return offset; 161 return offset;
169 162
170 if (amd_nb_event_constraint && 163 if (!cpu_has_perfctr_core)
171 test_bit(index, amd_nb_event_constraint->idxmsk)) {
172 /*
173 * calculate the offset of NB counters with respect to
174 * base eventsel or perfctr
175 */
176
177 first = find_first_bit(amd_nb_event_constraint->idxmsk,
178 X86_PMC_IDX_MAX);
179
180 if (eventsel)
181 base = MSR_F15H_NB_PERF_CTL - x86_pmu.eventsel;
182 else
183 base = MSR_F15H_NB_PERF_CTR - x86_pmu.perfctr;
184
185 offset = base + ((index - first) << 1);
186 } else if (!cpu_has_perfctr_core)
187 offset = index; 164 offset = index;
188 else 165 else
189 offset = index << 1; 166 offset = index << 1;
@@ -196,36 +173,6 @@ static inline int amd_pmu_addr_offset(int index, bool eventsel)
196 return offset; 173 return offset;
197} 174}
198 175
199static inline int amd_pmu_rdpmc_index(int index)
200{
201 int ret, first;
202
203 if (!index)
204 return index;
205
206 ret = rdpmc_indexes[index];
207
208 if (ret)
209 return ret;
210
211 if (amd_nb_event_constraint &&
212 test_bit(index, amd_nb_event_constraint->idxmsk)) {
213 /*
214 * according to the mnual, ECX value of the NB counters is
215 * the index of the NB counter (0, 1, 2 or 3) plus 6
216 */
217
218 first = find_first_bit(amd_nb_event_constraint->idxmsk,
219 X86_PMC_IDX_MAX);
220 ret = index - first + 6;
221 } else
222 ret = index;
223
224 rdpmc_indexes[index] = ret;
225
226 return ret;
227}
228
229static int amd_core_hw_config(struct perf_event *event) 176static int amd_core_hw_config(struct perf_event *event)
230{ 177{
231 if (event->attr.exclude_host && event->attr.exclude_guest) 178 if (event->attr.exclude_host && event->attr.exclude_guest)
@@ -245,34 +192,6 @@ static int amd_core_hw_config(struct perf_event *event)
245} 192}
246 193
247/* 194/*
248 * NB counters do not support the following event select bits:
249 * Host/Guest only
250 * Counter mask
251 * Invert counter mask
252 * Edge detect
253 * OS/User mode
254 */
255static int amd_nb_hw_config(struct perf_event *event)
256{
257 /* for NB, we only allow system wide counting mode */
258 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
259 return -EINVAL;
260
261 if (event->attr.exclude_user || event->attr.exclude_kernel ||
262 event->attr.exclude_host || event->attr.exclude_guest)
263 return -EINVAL;
264
265 event->hw.config &= ~(ARCH_PERFMON_EVENTSEL_USR |
266 ARCH_PERFMON_EVENTSEL_OS);
267
268 if (event->hw.config & ~(AMD64_RAW_EVENT_MASK_NB |
269 ARCH_PERFMON_EVENTSEL_INT))
270 return -EINVAL;
271
272 return 0;
273}
274
275/*
276 * AMD64 events are detected based on their event codes. 195 * AMD64 events are detected based on their event codes.
277 */ 196 */
278static inline unsigned int amd_get_event_code(struct hw_perf_event *hwc) 197static inline unsigned int amd_get_event_code(struct hw_perf_event *hwc)
@@ -285,11 +204,6 @@ static inline int amd_is_nb_event(struct hw_perf_event *hwc)
285 return (hwc->config & 0xe0) == 0xe0; 204 return (hwc->config & 0xe0) == 0xe0;
286} 205}
287 206
288static inline int amd_is_perfctr_nb_event(struct hw_perf_event *hwc)
289{
290 return amd_nb_event_constraint && amd_is_nb_event(hwc);
291}
292
293static inline int amd_has_nb(struct cpu_hw_events *cpuc) 207static inline int amd_has_nb(struct cpu_hw_events *cpuc)
294{ 208{
295 struct amd_nb *nb = cpuc->amd_nb; 209 struct amd_nb *nb = cpuc->amd_nb;
@@ -315,9 +229,6 @@ static int amd_pmu_hw_config(struct perf_event *event)
315 if (event->attr.type == PERF_TYPE_RAW) 229 if (event->attr.type == PERF_TYPE_RAW)
316 event->hw.config |= event->attr.config & AMD64_RAW_EVENT_MASK; 230 event->hw.config |= event->attr.config & AMD64_RAW_EVENT_MASK;
317 231
318 if (amd_is_perfctr_nb_event(&event->hw))
319 return amd_nb_hw_config(event);
320
321 return amd_core_hw_config(event); 232 return amd_core_hw_config(event);
322} 233}
323 234
@@ -341,19 +252,6 @@ static void __amd_put_nb_event_constraints(struct cpu_hw_events *cpuc,
341 } 252 }
342} 253}
343 254
344static void amd_nb_interrupt_hw_config(struct hw_perf_event *hwc)
345{
346 int core_id = cpu_data(smp_processor_id()).cpu_core_id;
347
348 /* deliver interrupts only to this core */
349 if (hwc->config & ARCH_PERFMON_EVENTSEL_INT) {
350 hwc->config |= AMD64_EVENTSEL_INT_CORE_ENABLE;
351 hwc->config &= ~AMD64_EVENTSEL_INT_CORE_SEL_MASK;
352 hwc->config |= (u64)(core_id) <<
353 AMD64_EVENTSEL_INT_CORE_SEL_SHIFT;
354 }
355}
356
357 /* 255 /*
358 * AMD64 NorthBridge events need special treatment because 256 * AMD64 NorthBridge events need special treatment because
359 * counter access needs to be synchronized across all cores 257 * counter access needs to be synchronized across all cores
@@ -441,9 +339,6 @@ __amd_get_nb_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *ev
441 if (new == -1) 339 if (new == -1)
442 return &emptyconstraint; 340 return &emptyconstraint;
443 341
444 if (amd_is_perfctr_nb_event(hwc))
445 amd_nb_interrupt_hw_config(hwc);
446
447 return &nb->event_constraints[new]; 342 return &nb->event_constraints[new];
448} 343}
449 344
@@ -543,8 +438,7 @@ amd_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
543 if (!(amd_has_nb(cpuc) && amd_is_nb_event(&event->hw))) 438 if (!(amd_has_nb(cpuc) && amd_is_nb_event(&event->hw)))
544 return &unconstrained; 439 return &unconstrained;
545 440
546 return __amd_get_nb_event_constraints(cpuc, event, 441 return __amd_get_nb_event_constraints(cpuc, event, NULL);
547 amd_nb_event_constraint);
548} 442}
549 443
550static void amd_put_event_constraints(struct cpu_hw_events *cpuc, 444static void amd_put_event_constraints(struct cpu_hw_events *cpuc,
@@ -643,9 +537,6 @@ static struct event_constraint amd_f15_PMC30 = EVENT_CONSTRAINT_OVERLAP(0, 0x09,
643static struct event_constraint amd_f15_PMC50 = EVENT_CONSTRAINT(0, 0x3F, 0); 537static struct event_constraint amd_f15_PMC50 = EVENT_CONSTRAINT(0, 0x3F, 0);
644static struct event_constraint amd_f15_PMC53 = EVENT_CONSTRAINT(0, 0x38, 0); 538static struct event_constraint amd_f15_PMC53 = EVENT_CONSTRAINT(0, 0x38, 0);
645 539
646static struct event_constraint amd_NBPMC96 = EVENT_CONSTRAINT(0, 0x3C0, 0);
647static struct event_constraint amd_NBPMC74 = EVENT_CONSTRAINT(0, 0xF0, 0);
648
649static struct event_constraint * 540static struct event_constraint *
650amd_get_event_constraints_f15h(struct cpu_hw_events *cpuc, struct perf_event *event) 541amd_get_event_constraints_f15h(struct cpu_hw_events *cpuc, struct perf_event *event)
651{ 542{
@@ -711,8 +602,8 @@ amd_get_event_constraints_f15h(struct cpu_hw_events *cpuc, struct perf_event *ev
711 return &amd_f15_PMC20; 602 return &amd_f15_PMC20;
712 } 603 }
713 case AMD_EVENT_NB: 604 case AMD_EVENT_NB:
714 return __amd_get_nb_event_constraints(cpuc, event, 605 /* moved to perf_event_amd_uncore.c */
715 amd_nb_event_constraint); 606 return &emptyconstraint;
716 default: 607 default:
717 return &emptyconstraint; 608 return &emptyconstraint;
718 } 609 }
@@ -738,7 +629,6 @@ static __initconst const struct x86_pmu amd_pmu = {
738 .eventsel = MSR_K7_EVNTSEL0, 629 .eventsel = MSR_K7_EVNTSEL0,
739 .perfctr = MSR_K7_PERFCTR0, 630 .perfctr = MSR_K7_PERFCTR0,
740 .addr_offset = amd_pmu_addr_offset, 631 .addr_offset = amd_pmu_addr_offset,
741 .rdpmc_index = amd_pmu_rdpmc_index,
742 .event_map = amd_pmu_event_map, 632 .event_map = amd_pmu_event_map,
743 .max_events = ARRAY_SIZE(amd_perfmon_event_map), 633 .max_events = ARRAY_SIZE(amd_perfmon_event_map),
744 .num_counters = AMD64_NUM_COUNTERS, 634 .num_counters = AMD64_NUM_COUNTERS,
@@ -790,23 +680,6 @@ static int setup_perfctr_core(void)
790 return 0; 680 return 0;
791} 681}
792 682
793static int setup_perfctr_nb(void)
794{
795 if (!cpu_has_perfctr_nb)
796 return -ENODEV;
797
798 x86_pmu.num_counters += AMD64_NUM_COUNTERS_NB;
799
800 if (cpu_has_perfctr_core)
801 amd_nb_event_constraint = &amd_NBPMC96;
802 else
803 amd_nb_event_constraint = &amd_NBPMC74;
804
805 printk(KERN_INFO "perf: AMD northbridge performance counters detected\n");
806
807 return 0;
808}
809
810__init int amd_pmu_init(void) 683__init int amd_pmu_init(void)
811{ 684{
812 /* Performance-monitoring supported from K7 and later: */ 685 /* Performance-monitoring supported from K7 and later: */
@@ -817,7 +690,6 @@ __init int amd_pmu_init(void)
817 690
818 setup_event_constraints(); 691 setup_event_constraints();
819 setup_perfctr_core(); 692 setup_perfctr_core();
820 setup_perfctr_nb();
821 693
822 /* Events are common for all AMDs */ 694 /* Events are common for all AMDs */
823 memcpy(hw_cache_event_ids, amd_hw_cache_event_ids, 695 memcpy(hw_cache_event_ids, amd_hw_cache_event_ids,
diff --git a/arch/x86/kernel/cpu/perf_event_amd_uncore.c b/arch/x86/kernel/cpu/perf_event_amd_uncore.c
new file mode 100644
index 000000000000..c0c661adf03e
--- /dev/null
+++ b/arch/x86/kernel/cpu/perf_event_amd_uncore.c
@@ -0,0 +1,547 @@
1/*
2 * Copyright (C) 2013 Advanced Micro Devices, Inc.
3 *
4 * Author: Jacob Shin <jacob.shin@amd.com>
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 */
10
11#include <linux/perf_event.h>
12#include <linux/percpu.h>
13#include <linux/types.h>
14#include <linux/slab.h>
15#include <linux/init.h>
16#include <linux/cpu.h>
17#include <linux/cpumask.h>
18
19#include <asm/cpufeature.h>
20#include <asm/perf_event.h>
21#include <asm/msr.h>
22
23#define NUM_COUNTERS_NB 4
24#define NUM_COUNTERS_L2 4
25#define MAX_COUNTERS NUM_COUNTERS_NB
26
27#define RDPMC_BASE_NB 6
28#define RDPMC_BASE_L2 10
29
30#define COUNTER_SHIFT 16
31
32struct amd_uncore {
33 int id;
34 int refcnt;
35 int cpu;
36 int num_counters;
37 int rdpmc_base;
38 u32 msr_base;
39 cpumask_t *active_mask;
40 struct pmu *pmu;
41 struct perf_event *events[MAX_COUNTERS];
42 struct amd_uncore *free_when_cpu_online;
43};
44
45static struct amd_uncore * __percpu *amd_uncore_nb;
46static struct amd_uncore * __percpu *amd_uncore_l2;
47
48static struct pmu amd_nb_pmu;
49static struct pmu amd_l2_pmu;
50
51static cpumask_t amd_nb_active_mask;
52static cpumask_t amd_l2_active_mask;
53
54static bool is_nb_event(struct perf_event *event)
55{
56 return event->pmu->type == amd_nb_pmu.type;
57}
58
59static bool is_l2_event(struct perf_event *event)
60{
61 return event->pmu->type == amd_l2_pmu.type;
62}
63
64static struct amd_uncore *event_to_amd_uncore(struct perf_event *event)
65{
66 if (is_nb_event(event) && amd_uncore_nb)
67 return *per_cpu_ptr(amd_uncore_nb, event->cpu);
68 else if (is_l2_event(event) && amd_uncore_l2)
69 return *per_cpu_ptr(amd_uncore_l2, event->cpu);
70
71 return NULL;
72}
73
74static void amd_uncore_read(struct perf_event *event)
75{
76 struct hw_perf_event *hwc = &event->hw;
77 u64 prev, new;
78 s64 delta;
79
80 /*
81 * since we do not enable counter overflow interrupts,
82 * we do not have to worry about prev_count changing on us
83 */
84
85 prev = local64_read(&hwc->prev_count);
86 rdpmcl(hwc->event_base_rdpmc, new);
87 local64_set(&hwc->prev_count, new);
88 delta = (new << COUNTER_SHIFT) - (prev << COUNTER_SHIFT);
89 delta >>= COUNTER_SHIFT;
90 local64_add(delta, &event->count);
91}
92
93static void amd_uncore_start(struct perf_event *event, int flags)
94{
95 struct hw_perf_event *hwc = &event->hw;
96
97 if (flags & PERF_EF_RELOAD)
98 wrmsrl(hwc->event_base, (u64)local64_read(&hwc->prev_count));
99
100 hwc->state = 0;
101 wrmsrl(hwc->config_base, (hwc->config | ARCH_PERFMON_EVENTSEL_ENABLE));
102 perf_event_update_userpage(event);
103}
104
105static void amd_uncore_stop(struct perf_event *event, int flags)
106{
107 struct hw_perf_event *hwc = &event->hw;
108
109 wrmsrl(hwc->config_base, hwc->config);
110 hwc->state |= PERF_HES_STOPPED;
111
112 if ((flags & PERF_EF_UPDATE) && !(hwc->state & PERF_HES_UPTODATE)) {
113 amd_uncore_read(event);
114 hwc->state |= PERF_HES_UPTODATE;
115 }
116}
117
118static int amd_uncore_add(struct perf_event *event, int flags)
119{
120 int i;
121 struct amd_uncore *uncore = event_to_amd_uncore(event);
122 struct hw_perf_event *hwc = &event->hw;
123
124 /* are we already assigned? */
125 if (hwc->idx != -1 && uncore->events[hwc->idx] == event)
126 goto out;
127
128 for (i = 0; i < uncore->num_counters; i++) {
129 if (uncore->events[i] == event) {
130 hwc->idx = i;
131 goto out;
132 }
133 }
134
135 /* if not, take the first available counter */
136 hwc->idx = -1;
137 for (i = 0; i < uncore->num_counters; i++) {
138 if (cmpxchg(&uncore->events[i], NULL, event) == NULL) {
139 hwc->idx = i;
140 break;
141 }
142 }
143
144out:
145 if (hwc->idx == -1)
146 return -EBUSY;
147
148 hwc->config_base = uncore->msr_base + (2 * hwc->idx);
149 hwc->event_base = uncore->msr_base + 1 + (2 * hwc->idx);
150 hwc->event_base_rdpmc = uncore->rdpmc_base + hwc->idx;
151 hwc->state = PERF_HES_UPTODATE | PERF_HES_STOPPED;
152
153 if (flags & PERF_EF_START)
154 amd_uncore_start(event, PERF_EF_RELOAD);
155
156 return 0;
157}
158
159static void amd_uncore_del(struct perf_event *event, int flags)
160{
161 int i;
162 struct amd_uncore *uncore = event_to_amd_uncore(event);
163 struct hw_perf_event *hwc = &event->hw;
164
165 amd_uncore_stop(event, PERF_EF_UPDATE);
166
167 for (i = 0; i < uncore->num_counters; i++) {
168 if (cmpxchg(&uncore->events[i], event, NULL) == event)
169 break;
170 }
171
172 hwc->idx = -1;
173}
174
175static int amd_uncore_event_init(struct perf_event *event)
176{
177 struct amd_uncore *uncore;
178 struct hw_perf_event *hwc = &event->hw;
179
180 if (event->attr.type != event->pmu->type)
181 return -ENOENT;
182
183 /*
184 * NB and L2 counters (MSRs) are shared across all cores that share the
185 * same NB / L2 cache. Interrupts can be directed to a single target
186 * core, however, event counts generated by processes running on other
187 * cores cannot be masked out. So we do not support sampling and
188 * per-thread events.
189 */
190 if (is_sampling_event(event) || event->attach_state & PERF_ATTACH_TASK)
191 return -EINVAL;
192
193 /* NB and L2 counters do not have usr/os/guest/host bits */
194 if (event->attr.exclude_user || event->attr.exclude_kernel ||
195 event->attr.exclude_host || event->attr.exclude_guest)
196 return -EINVAL;
197
198 /* and we do not enable counter overflow interrupts */
199 hwc->config = event->attr.config & AMD64_RAW_EVENT_MASK_NB;
200 hwc->idx = -1;
201
202 if (event->cpu < 0)
203 return -EINVAL;
204
205 uncore = event_to_amd_uncore(event);
206 if (!uncore)
207 return -ENODEV;
208
209 /*
210 * since request can come in to any of the shared cores, we will remap
211 * to a single common cpu.
212 */
213 event->cpu = uncore->cpu;
214
215 return 0;
216}
217
218static ssize_t amd_uncore_attr_show_cpumask(struct device *dev,
219 struct device_attribute *attr,
220 char *buf)
221{
222 int n;
223 cpumask_t *active_mask;
224 struct pmu *pmu = dev_get_drvdata(dev);
225
226 if (pmu->type == amd_nb_pmu.type)
227 active_mask = &amd_nb_active_mask;
228 else if (pmu->type == amd_l2_pmu.type)
229 active_mask = &amd_l2_active_mask;
230 else
231 return 0;
232
233 n = cpulist_scnprintf(buf, PAGE_SIZE - 2, active_mask);
234 buf[n++] = '\n';
235 buf[n] = '\0';
236 return n;
237}
238static DEVICE_ATTR(cpumask, S_IRUGO, amd_uncore_attr_show_cpumask, NULL);
239
240static struct attribute *amd_uncore_attrs[] = {
241 &dev_attr_cpumask.attr,
242 NULL,
243};
244
245static struct attribute_group amd_uncore_attr_group = {
246 .attrs = amd_uncore_attrs,
247};
248
249PMU_FORMAT_ATTR(event, "config:0-7,32-35");
250PMU_FORMAT_ATTR(umask, "config:8-15");
251
252static struct attribute *amd_uncore_format_attr[] = {
253 &format_attr_event.attr,
254 &format_attr_umask.attr,
255 NULL,
256};
257
258static struct attribute_group amd_uncore_format_group = {
259 .name = "format",
260 .attrs = amd_uncore_format_attr,
261};
262
263static const struct attribute_group *amd_uncore_attr_groups[] = {
264 &amd_uncore_attr_group,
265 &amd_uncore_format_group,
266 NULL,
267};
268
269static struct pmu amd_nb_pmu = {
270 .attr_groups = amd_uncore_attr_groups,
271 .name = "amd_nb",
272 .event_init = amd_uncore_event_init,
273 .add = amd_uncore_add,
274 .del = amd_uncore_del,
275 .start = amd_uncore_start,
276 .stop = amd_uncore_stop,
277 .read = amd_uncore_read,
278};
279
280static struct pmu amd_l2_pmu = {
281 .attr_groups = amd_uncore_attr_groups,
282 .name = "amd_l2",
283 .event_init = amd_uncore_event_init,
284 .add = amd_uncore_add,
285 .del = amd_uncore_del,
286 .start = amd_uncore_start,
287 .stop = amd_uncore_stop,
288 .read = amd_uncore_read,
289};
290
291static struct amd_uncore * __cpuinit amd_uncore_alloc(unsigned int cpu)
292{
293 return kzalloc_node(sizeof(struct amd_uncore), GFP_KERNEL,
294 cpu_to_node(cpu));
295}
296
297static void __cpuinit amd_uncore_cpu_up_prepare(unsigned int cpu)
298{
299 struct amd_uncore *uncore;
300
301 if (amd_uncore_nb) {
302 uncore = amd_uncore_alloc(cpu);
303 uncore->cpu = cpu;
304 uncore->num_counters = NUM_COUNTERS_NB;
305 uncore->rdpmc_base = RDPMC_BASE_NB;
306 uncore->msr_base = MSR_F15H_NB_PERF_CTL;
307 uncore->active_mask = &amd_nb_active_mask;
308 uncore->pmu = &amd_nb_pmu;
309 *per_cpu_ptr(amd_uncore_nb, cpu) = uncore;
310 }
311
312 if (amd_uncore_l2) {
313 uncore = amd_uncore_alloc(cpu);
314 uncore->cpu = cpu;
315 uncore->num_counters = NUM_COUNTERS_L2;
316 uncore->rdpmc_base = RDPMC_BASE_L2;
317 uncore->msr_base = MSR_F16H_L2I_PERF_CTL;
318 uncore->active_mask = &amd_l2_active_mask;
319 uncore->pmu = &amd_l2_pmu;
320 *per_cpu_ptr(amd_uncore_l2, cpu) = uncore;
321 }
322}
323
324static struct amd_uncore *
325__cpuinit amd_uncore_find_online_sibling(struct amd_uncore *this,
326 struct amd_uncore * __percpu *uncores)
327{
328 unsigned int cpu;
329 struct amd_uncore *that;
330
331 for_each_online_cpu(cpu) {
332 that = *per_cpu_ptr(uncores, cpu);
333
334 if (!that)
335 continue;
336
337 if (this == that)
338 continue;
339
340 if (this->id == that->id) {
341 that->free_when_cpu_online = this;
342 this = that;
343 break;
344 }
345 }
346
347 this->refcnt++;
348 return this;
349}
350
351static void __cpuinit amd_uncore_cpu_starting(unsigned int cpu)
352{
353 unsigned int eax, ebx, ecx, edx;
354 struct amd_uncore *uncore;
355
356 if (amd_uncore_nb) {
357 uncore = *per_cpu_ptr(amd_uncore_nb, cpu);
358 cpuid(0x8000001e, &eax, &ebx, &ecx, &edx);
359 uncore->id = ecx & 0xff;
360
361 uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_nb);
362 *per_cpu_ptr(amd_uncore_nb, cpu) = uncore;
363 }
364
365 if (amd_uncore_l2) {
366 unsigned int apicid = cpu_data(cpu).apicid;
367 unsigned int nshared;
368
369 uncore = *per_cpu_ptr(amd_uncore_l2, cpu);
370 cpuid_count(0x8000001d, 2, &eax, &ebx, &ecx, &edx);
371 nshared = ((eax >> 14) & 0xfff) + 1;
372 uncore->id = apicid - (apicid % nshared);
373
374 uncore = amd_uncore_find_online_sibling(uncore, amd_uncore_l2);
375 *per_cpu_ptr(amd_uncore_l2, cpu) = uncore;
376 }
377}
378
379static void __cpuinit uncore_online(unsigned int cpu,
380 struct amd_uncore * __percpu *uncores)
381{
382 struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu);
383
384 kfree(uncore->free_when_cpu_online);
385 uncore->free_when_cpu_online = NULL;
386
387 if (cpu == uncore->cpu)
388 cpumask_set_cpu(cpu, uncore->active_mask);
389}
390
391static void __cpuinit amd_uncore_cpu_online(unsigned int cpu)
392{
393 if (amd_uncore_nb)
394 uncore_online(cpu, amd_uncore_nb);
395
396 if (amd_uncore_l2)
397 uncore_online(cpu, amd_uncore_l2);
398}
399
400static void __cpuinit uncore_down_prepare(unsigned int cpu,
401 struct amd_uncore * __percpu *uncores)
402{
403 unsigned int i;
404 struct amd_uncore *this = *per_cpu_ptr(uncores, cpu);
405
406 if (this->cpu != cpu)
407 return;
408
409 /* this cpu is going down, migrate to a shared sibling if possible */
410 for_each_online_cpu(i) {
411 struct amd_uncore *that = *per_cpu_ptr(uncores, i);
412
413 if (cpu == i)
414 continue;
415
416 if (this == that) {
417 perf_pmu_migrate_context(this->pmu, cpu, i);
418 cpumask_clear_cpu(cpu, that->active_mask);
419 cpumask_set_cpu(i, that->active_mask);
420 that->cpu = i;
421 break;
422 }
423 }
424}
425
426static void __cpuinit amd_uncore_cpu_down_prepare(unsigned int cpu)
427{
428 if (amd_uncore_nb)
429 uncore_down_prepare(cpu, amd_uncore_nb);
430
431 if (amd_uncore_l2)
432 uncore_down_prepare(cpu, amd_uncore_l2);
433}
434
435static void __cpuinit uncore_dead(unsigned int cpu,
436 struct amd_uncore * __percpu *uncores)
437{
438 struct amd_uncore *uncore = *per_cpu_ptr(uncores, cpu);
439
440 if (cpu == uncore->cpu)
441 cpumask_clear_cpu(cpu, uncore->active_mask);
442
443 if (!--uncore->refcnt)
444 kfree(uncore);
445 *per_cpu_ptr(amd_uncore_nb, cpu) = NULL;
446}
447
448static void __cpuinit amd_uncore_cpu_dead(unsigned int cpu)
449{
450 if (amd_uncore_nb)
451 uncore_dead(cpu, amd_uncore_nb);
452
453 if (amd_uncore_l2)
454 uncore_dead(cpu, amd_uncore_l2);
455}
456
457static int __cpuinit
458amd_uncore_cpu_notifier(struct notifier_block *self, unsigned long action,
459 void *hcpu)
460{
461 unsigned int cpu = (long)hcpu;
462
463 switch (action & ~CPU_TASKS_FROZEN) {
464 case CPU_UP_PREPARE:
465 amd_uncore_cpu_up_prepare(cpu);
466 break;
467
468 case CPU_STARTING:
469 amd_uncore_cpu_starting(cpu);
470 break;
471
472 case CPU_ONLINE:
473 amd_uncore_cpu_online(cpu);
474 break;
475
476 case CPU_DOWN_PREPARE:
477 amd_uncore_cpu_down_prepare(cpu);
478 break;
479
480 case CPU_UP_CANCELED:
481 case CPU_DEAD:
482 amd_uncore_cpu_dead(cpu);
483 break;
484
485 default:
486 break;
487 }
488
489 return NOTIFY_OK;
490}
491
492static struct notifier_block amd_uncore_cpu_notifier_block __cpuinitdata = {
493 .notifier_call = amd_uncore_cpu_notifier,
494 .priority = CPU_PRI_PERF + 1,
495};
496
497static void __init init_cpu_already_online(void *dummy)
498{
499 unsigned int cpu = smp_processor_id();
500
501 amd_uncore_cpu_starting(cpu);
502 amd_uncore_cpu_online(cpu);
503}
504
505static int __init amd_uncore_init(void)
506{
507 unsigned int cpu;
508 int ret = -ENODEV;
509
510 if (boot_cpu_data.x86_vendor != X86_VENDOR_AMD)
511 return -ENODEV;
512
513 if (!cpu_has_topoext)
514 return -ENODEV;
515
516 if (cpu_has_perfctr_nb) {
517 amd_uncore_nb = alloc_percpu(struct amd_uncore *);
518 perf_pmu_register(&amd_nb_pmu, amd_nb_pmu.name, -1);
519
520 printk(KERN_INFO "perf: AMD NB counters detected\n");
521 ret = 0;
522 }
523
524 if (cpu_has_perfctr_l2) {
525 amd_uncore_l2 = alloc_percpu(struct amd_uncore *);
526 perf_pmu_register(&amd_l2_pmu, amd_l2_pmu.name, -1);
527
528 printk(KERN_INFO "perf: AMD L2I counters detected\n");
529 ret = 0;
530 }
531
532 if (ret)
533 return -ENODEV;
534
535 get_online_cpus();
536 /* init cpus already online before registering for hotplug notifier */
537 for_each_online_cpu(cpu) {
538 amd_uncore_cpu_up_prepare(cpu);
539 smp_call_function_single(cpu, init_cpu_already_online, NULL, 1);
540 }
541
542 register_cpu_notifier(&amd_uncore_cpu_notifier_block);
543 put_online_cpus();
544
545 return 0;
546}
547device_initcall(amd_uncore_init);
diff --git a/arch/x86/kernel/cpu/perf_event_intel.c b/arch/x86/kernel/cpu/perf_event_intel.c
index cc45deb791b0..ffd6050a1de4 100644
--- a/arch/x86/kernel/cpu/perf_event_intel.c
+++ b/arch/x86/kernel/cpu/perf_event_intel.c
@@ -81,6 +81,7 @@ static struct event_constraint intel_nehalem_event_constraints[] __read_mostly =
81static struct extra_reg intel_nehalem_extra_regs[] __read_mostly = 81static struct extra_reg intel_nehalem_extra_regs[] __read_mostly =
82{ 82{
83 INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff, RSP_0), 83 INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff, RSP_0),
84 INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x100b),
84 EVENT_EXTRA_END 85 EVENT_EXTRA_END
85}; 86};
86 87
@@ -108,6 +109,8 @@ static struct event_constraint intel_snb_event_constraints[] __read_mostly =
108 INTEL_EVENT_CONSTRAINT(0x48, 0x4), /* L1D_PEND_MISS.PENDING */ 109 INTEL_EVENT_CONSTRAINT(0x48, 0x4), /* L1D_PEND_MISS.PENDING */
109 INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PREC_DIST */ 110 INTEL_UEVENT_CONSTRAINT(0x01c0, 0x2), /* INST_RETIRED.PREC_DIST */
110 INTEL_EVENT_CONSTRAINT(0xcd, 0x8), /* MEM_TRANS_RETIRED.LOAD_LATENCY */ 111 INTEL_EVENT_CONSTRAINT(0xcd, 0x8), /* MEM_TRANS_RETIRED.LOAD_LATENCY */
112 INTEL_UEVENT_CONSTRAINT(0x04a3, 0xf), /* CYCLE_ACTIVITY.CYCLES_NO_DISPATCH */
113 INTEL_UEVENT_CONSTRAINT(0x02a3, 0x4), /* CYCLE_ACTIVITY.CYCLES_L1D_PENDING */
111 EVENT_CONSTRAINT_END 114 EVENT_CONSTRAINT_END
112}; 115};
113 116
@@ -136,6 +139,7 @@ static struct extra_reg intel_westmere_extra_regs[] __read_mostly =
136{ 139{
137 INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff, RSP_0), 140 INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff, RSP_0),
138 INTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0xffff, RSP_1), 141 INTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0xffff, RSP_1),
142 INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x100b),
139 EVENT_EXTRA_END 143 EVENT_EXTRA_END
140}; 144};
141 145
@@ -155,6 +159,8 @@ static struct event_constraint intel_gen_event_constraints[] __read_mostly =
155static struct extra_reg intel_snb_extra_regs[] __read_mostly = { 159static struct extra_reg intel_snb_extra_regs[] __read_mostly = {
156 INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0x3f807f8fffull, RSP_0), 160 INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0x3f807f8fffull, RSP_0),
157 INTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0x3f807f8fffull, RSP_1), 161 INTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0x3f807f8fffull, RSP_1),
162 INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
163 INTEL_UEVENT_PEBS_LDLAT_EXTRA_REG(0x01cd),
158 EVENT_EXTRA_END 164 EVENT_EXTRA_END
159}; 165};
160 166
@@ -164,6 +170,21 @@ static struct extra_reg intel_snbep_extra_regs[] __read_mostly = {
164 EVENT_EXTRA_END 170 EVENT_EXTRA_END
165}; 171};
166 172
173EVENT_ATTR_STR(mem-loads, mem_ld_nhm, "event=0x0b,umask=0x10,ldlat=3");
174EVENT_ATTR_STR(mem-loads, mem_ld_snb, "event=0xcd,umask=0x1,ldlat=3");
175EVENT_ATTR_STR(mem-stores, mem_st_snb, "event=0xcd,umask=0x2");
176
177struct attribute *nhm_events_attrs[] = {
178 EVENT_PTR(mem_ld_nhm),
179 NULL,
180};
181
182struct attribute *snb_events_attrs[] = {
183 EVENT_PTR(mem_ld_snb),
184 EVENT_PTR(mem_st_snb),
185 NULL,
186};
187
167static u64 intel_pmu_event_map(int hw_event) 188static u64 intel_pmu_event_map(int hw_event)
168{ 189{
169 return intel_perfmon_event_map[hw_event]; 190 return intel_perfmon_event_map[hw_event];
@@ -1398,8 +1419,11 @@ x86_get_event_constraints(struct cpu_hw_events *cpuc, struct perf_event *event)
1398 1419
1399 if (x86_pmu.event_constraints) { 1420 if (x86_pmu.event_constraints) {
1400 for_each_event_constraint(c, x86_pmu.event_constraints) { 1421 for_each_event_constraint(c, x86_pmu.event_constraints) {
1401 if ((event->hw.config & c->cmask) == c->code) 1422 if ((event->hw.config & c->cmask) == c->code) {
1423 /* hw.flags zeroed at initialization */
1424 event->hw.flags |= c->flags;
1402 return c; 1425 return c;
1426 }
1403 } 1427 }
1404 } 1428 }
1405 1429
@@ -1444,6 +1468,7 @@ intel_put_shared_regs_event_constraints(struct cpu_hw_events *cpuc,
1444static void intel_put_event_constraints(struct cpu_hw_events *cpuc, 1468static void intel_put_event_constraints(struct cpu_hw_events *cpuc,
1445 struct perf_event *event) 1469 struct perf_event *event)
1446{ 1470{
1471 event->hw.flags = 0;
1447 intel_put_shared_regs_event_constraints(cpuc, event); 1472 intel_put_shared_regs_event_constraints(cpuc, event);
1448} 1473}
1449 1474
@@ -1767,6 +1792,8 @@ static void intel_pmu_flush_branch_stack(void)
1767 1792
1768PMU_FORMAT_ATTR(offcore_rsp, "config1:0-63"); 1793PMU_FORMAT_ATTR(offcore_rsp, "config1:0-63");
1769 1794
1795PMU_FORMAT_ATTR(ldlat, "config1:0-15");
1796
1770static struct attribute *intel_arch3_formats_attr[] = { 1797static struct attribute *intel_arch3_formats_attr[] = {
1771 &format_attr_event.attr, 1798 &format_attr_event.attr,
1772 &format_attr_umask.attr, 1799 &format_attr_umask.attr,
@@ -1777,6 +1804,7 @@ static struct attribute *intel_arch3_formats_attr[] = {
1777 &format_attr_cmask.attr, 1804 &format_attr_cmask.attr,
1778 1805
1779 &format_attr_offcore_rsp.attr, /* XXX do NHM/WSM + SNB breakout */ 1806 &format_attr_offcore_rsp.attr, /* XXX do NHM/WSM + SNB breakout */
1807 &format_attr_ldlat.attr, /* PEBS load latency */
1780 NULL, 1808 NULL,
1781}; 1809};
1782 1810
@@ -2037,6 +2065,8 @@ __init int intel_pmu_init(void)
2037 x86_pmu.enable_all = intel_pmu_nhm_enable_all; 2065 x86_pmu.enable_all = intel_pmu_nhm_enable_all;
2038 x86_pmu.extra_regs = intel_nehalem_extra_regs; 2066 x86_pmu.extra_regs = intel_nehalem_extra_regs;
2039 2067
2068 x86_pmu.cpu_events = nhm_events_attrs;
2069
2040 /* UOPS_ISSUED.STALLED_CYCLES */ 2070 /* UOPS_ISSUED.STALLED_CYCLES */
2041 intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 2071 intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
2042 X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1); 2072 X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
@@ -2080,6 +2110,8 @@ __init int intel_pmu_init(void)
2080 x86_pmu.extra_regs = intel_westmere_extra_regs; 2110 x86_pmu.extra_regs = intel_westmere_extra_regs;
2081 x86_pmu.er_flags |= ERF_HAS_RSP_1; 2111 x86_pmu.er_flags |= ERF_HAS_RSP_1;
2082 2112
2113 x86_pmu.cpu_events = nhm_events_attrs;
2114
2083 /* UOPS_ISSUED.STALLED_CYCLES */ 2115 /* UOPS_ISSUED.STALLED_CYCLES */
2084 intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 2116 intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
2085 X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1); 2117 X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
@@ -2111,6 +2143,8 @@ __init int intel_pmu_init(void)
2111 x86_pmu.er_flags |= ERF_HAS_RSP_1; 2143 x86_pmu.er_flags |= ERF_HAS_RSP_1;
2112 x86_pmu.er_flags |= ERF_NO_HT_SHARING; 2144 x86_pmu.er_flags |= ERF_NO_HT_SHARING;
2113 2145
2146 x86_pmu.cpu_events = snb_events_attrs;
2147
2114 /* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */ 2148 /* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */
2115 intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 2149 intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
2116 X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1); 2150 X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
@@ -2140,6 +2174,8 @@ __init int intel_pmu_init(void)
2140 x86_pmu.er_flags |= ERF_HAS_RSP_1; 2174 x86_pmu.er_flags |= ERF_HAS_RSP_1;
2141 x86_pmu.er_flags |= ERF_NO_HT_SHARING; 2175 x86_pmu.er_flags |= ERF_NO_HT_SHARING;
2142 2176
2177 x86_pmu.cpu_events = snb_events_attrs;
2178
2143 /* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */ 2179 /* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */
2144 intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 2180 intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] =
2145 X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1); 2181 X86_CONFIG(.event=0x0e, .umask=0x01, .inv=1, .cmask=1);
diff --git a/arch/x86/kernel/cpu/perf_event_intel_ds.c b/arch/x86/kernel/cpu/perf_event_intel_ds.c
index 26830f3af0df..60250f687052 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_ds.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_ds.c
@@ -24,6 +24,130 @@ struct pebs_record_32 {
24 24
25 */ 25 */
26 26
27union intel_x86_pebs_dse {
28 u64 val;
29 struct {
30 unsigned int ld_dse:4;
31 unsigned int ld_stlb_miss:1;
32 unsigned int ld_locked:1;
33 unsigned int ld_reserved:26;
34 };
35 struct {
36 unsigned int st_l1d_hit:1;
37 unsigned int st_reserved1:3;
38 unsigned int st_stlb_miss:1;
39 unsigned int st_locked:1;
40 unsigned int st_reserved2:26;
41 };
42};
43
44
45/*
46 * Map PEBS Load Latency Data Source encodings to generic
47 * memory data source information
48 */
49#define P(a, b) PERF_MEM_S(a, b)
50#define OP_LH (P(OP, LOAD) | P(LVL, HIT))
51#define SNOOP_NONE_MISS (P(SNOOP, NONE) | P(SNOOP, MISS))
52
53static const u64 pebs_data_source[] = {
54 P(OP, LOAD) | P(LVL, MISS) | P(LVL, L3) | P(SNOOP, NA),/* 0x00:ukn L3 */
55 OP_LH | P(LVL, L1) | P(SNOOP, NONE), /* 0x01: L1 local */
56 OP_LH | P(LVL, LFB) | P(SNOOP, NONE), /* 0x02: LFB hit */
57 OP_LH | P(LVL, L2) | P(SNOOP, NONE), /* 0x03: L2 hit */
58 OP_LH | P(LVL, L3) | P(SNOOP, NONE), /* 0x04: L3 hit */
59 OP_LH | P(LVL, L3) | P(SNOOP, MISS), /* 0x05: L3 hit, snoop miss */
60 OP_LH | P(LVL, L3) | P(SNOOP, HIT), /* 0x06: L3 hit, snoop hit */
61 OP_LH | P(LVL, L3) | P(SNOOP, HITM), /* 0x07: L3 hit, snoop hitm */
62 OP_LH | P(LVL, REM_CCE1) | P(SNOOP, HIT), /* 0x08: L3 miss snoop hit */
63 OP_LH | P(LVL, REM_CCE1) | P(SNOOP, HITM), /* 0x09: L3 miss snoop hitm*/
64 OP_LH | P(LVL, LOC_RAM) | P(SNOOP, HIT), /* 0x0a: L3 miss, shared */
65 OP_LH | P(LVL, REM_RAM1) | P(SNOOP, HIT), /* 0x0b: L3 miss, shared */
66 OP_LH | P(LVL, LOC_RAM) | SNOOP_NONE_MISS,/* 0x0c: L3 miss, excl */
67 OP_LH | P(LVL, REM_RAM1) | SNOOP_NONE_MISS,/* 0x0d: L3 miss, excl */
68 OP_LH | P(LVL, IO) | P(SNOOP, NONE), /* 0x0e: I/O */
69 OP_LH | P(LVL, UNC) | P(SNOOP, NONE), /* 0x0f: uncached */
70};
71
72static u64 precise_store_data(u64 status)
73{
74 union intel_x86_pebs_dse dse;
75 u64 val = P(OP, STORE) | P(SNOOP, NA) | P(LVL, L1) | P(TLB, L2);
76
77 dse.val = status;
78
79 /*
80 * bit 4: TLB access
81 * 1 = stored missed 2nd level TLB
82 *
83 * so it either hit the walker or the OS
84 * otherwise hit 2nd level TLB
85 */
86 if (dse.st_stlb_miss)
87 val |= P(TLB, MISS);
88 else
89 val |= P(TLB, HIT);
90
91 /*
92 * bit 0: hit L1 data cache
93 * if not set, then all we know is that
94 * it missed L1D
95 */
96 if (dse.st_l1d_hit)
97 val |= P(LVL, HIT);
98 else
99 val |= P(LVL, MISS);
100
101 /*
102 * bit 5: Locked prefix
103 */
104 if (dse.st_locked)
105 val |= P(LOCK, LOCKED);
106
107 return val;
108}
109
110static u64 load_latency_data(u64 status)
111{
112 union intel_x86_pebs_dse dse;
113 u64 val;
114 int model = boot_cpu_data.x86_model;
115 int fam = boot_cpu_data.x86;
116
117 dse.val = status;
118
119 /*
120 * use the mapping table for bit 0-3
121 */
122 val = pebs_data_source[dse.ld_dse];
123
124 /*
125 * Nehalem models do not support TLB, Lock infos
126 */
127 if (fam == 0x6 && (model == 26 || model == 30
128 || model == 31 || model == 46)) {
129 val |= P(TLB, NA) | P(LOCK, NA);
130 return val;
131 }
132 /*
133 * bit 4: TLB access
134 * 0 = did not miss 2nd level TLB
135 * 1 = missed 2nd level TLB
136 */
137 if (dse.ld_stlb_miss)
138 val |= P(TLB, MISS) | P(TLB, L2);
139 else
140 val |= P(TLB, HIT) | P(TLB, L1) | P(TLB, L2);
141
142 /*
143 * bit 5: locked prefix
144 */
145 if (dse.ld_locked)
146 val |= P(LOCK, LOCKED);
147
148 return val;
149}
150
27struct pebs_record_core { 151struct pebs_record_core {
28 u64 flags, ip; 152 u64 flags, ip;
29 u64 ax, bx, cx, dx; 153 u64 ax, bx, cx, dx;
@@ -365,7 +489,7 @@ struct event_constraint intel_atom_pebs_event_constraints[] = {
365}; 489};
366 490
367struct event_constraint intel_nehalem_pebs_event_constraints[] = { 491struct event_constraint intel_nehalem_pebs_event_constraints[] = {
368 INTEL_EVENT_CONSTRAINT(0x0b, 0xf), /* MEM_INST_RETIRED.* */ 492 INTEL_PLD_CONSTRAINT(0x100b, 0xf), /* MEM_INST_RETIRED.* */
369 INTEL_EVENT_CONSTRAINT(0x0f, 0xf), /* MEM_UNCORE_RETIRED.* */ 493 INTEL_EVENT_CONSTRAINT(0x0f, 0xf), /* MEM_UNCORE_RETIRED.* */
370 INTEL_UEVENT_CONSTRAINT(0x010c, 0xf), /* MEM_STORE_RETIRED.DTLB_MISS */ 494 INTEL_UEVENT_CONSTRAINT(0x010c, 0xf), /* MEM_STORE_RETIRED.DTLB_MISS */
371 INTEL_EVENT_CONSTRAINT(0xc0, 0xf), /* INST_RETIRED.ANY */ 495 INTEL_EVENT_CONSTRAINT(0xc0, 0xf), /* INST_RETIRED.ANY */
@@ -380,7 +504,7 @@ struct event_constraint intel_nehalem_pebs_event_constraints[] = {
380}; 504};
381 505
382struct event_constraint intel_westmere_pebs_event_constraints[] = { 506struct event_constraint intel_westmere_pebs_event_constraints[] = {
383 INTEL_EVENT_CONSTRAINT(0x0b, 0xf), /* MEM_INST_RETIRED.* */ 507 INTEL_PLD_CONSTRAINT(0x100b, 0xf), /* MEM_INST_RETIRED.* */
384 INTEL_EVENT_CONSTRAINT(0x0f, 0xf), /* MEM_UNCORE_RETIRED.* */ 508 INTEL_EVENT_CONSTRAINT(0x0f, 0xf), /* MEM_UNCORE_RETIRED.* */
385 INTEL_UEVENT_CONSTRAINT(0x010c, 0xf), /* MEM_STORE_RETIRED.DTLB_MISS */ 509 INTEL_UEVENT_CONSTRAINT(0x010c, 0xf), /* MEM_STORE_RETIRED.DTLB_MISS */
386 INTEL_EVENT_CONSTRAINT(0xc0, 0xf), /* INSTR_RETIRED.* */ 510 INTEL_EVENT_CONSTRAINT(0xc0, 0xf), /* INSTR_RETIRED.* */
@@ -400,7 +524,8 @@ struct event_constraint intel_snb_pebs_event_constraints[] = {
400 INTEL_UEVENT_CONSTRAINT(0x02c2, 0xf), /* UOPS_RETIRED.RETIRE_SLOTS */ 524 INTEL_UEVENT_CONSTRAINT(0x02c2, 0xf), /* UOPS_RETIRED.RETIRE_SLOTS */
401 INTEL_EVENT_CONSTRAINT(0xc4, 0xf), /* BR_INST_RETIRED.* */ 525 INTEL_EVENT_CONSTRAINT(0xc4, 0xf), /* BR_INST_RETIRED.* */
402 INTEL_EVENT_CONSTRAINT(0xc5, 0xf), /* BR_MISP_RETIRED.* */ 526 INTEL_EVENT_CONSTRAINT(0xc5, 0xf), /* BR_MISP_RETIRED.* */
403 INTEL_EVENT_CONSTRAINT(0xcd, 0x8), /* MEM_TRANS_RETIRED.* */ 527 INTEL_PLD_CONSTRAINT(0x01cd, 0x8), /* MEM_TRANS_RETIRED.LAT_ABOVE_THR */
528 INTEL_PST_CONSTRAINT(0x02cd, 0x8), /* MEM_TRANS_RETIRED.PRECISE_STORES */
404 INTEL_EVENT_CONSTRAINT(0xd0, 0xf), /* MEM_UOP_RETIRED.* */ 529 INTEL_EVENT_CONSTRAINT(0xd0, 0xf), /* MEM_UOP_RETIRED.* */
405 INTEL_EVENT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */ 530 INTEL_EVENT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */
406 INTEL_EVENT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */ 531 INTEL_EVENT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */
@@ -414,7 +539,8 @@ struct event_constraint intel_ivb_pebs_event_constraints[] = {
414 INTEL_UEVENT_CONSTRAINT(0x02c2, 0xf), /* UOPS_RETIRED.RETIRE_SLOTS */ 539 INTEL_UEVENT_CONSTRAINT(0x02c2, 0xf), /* UOPS_RETIRED.RETIRE_SLOTS */
415 INTEL_EVENT_CONSTRAINT(0xc4, 0xf), /* BR_INST_RETIRED.* */ 540 INTEL_EVENT_CONSTRAINT(0xc4, 0xf), /* BR_INST_RETIRED.* */
416 INTEL_EVENT_CONSTRAINT(0xc5, 0xf), /* BR_MISP_RETIRED.* */ 541 INTEL_EVENT_CONSTRAINT(0xc5, 0xf), /* BR_MISP_RETIRED.* */
417 INTEL_EVENT_CONSTRAINT(0xcd, 0x8), /* MEM_TRANS_RETIRED.* */ 542 INTEL_PLD_CONSTRAINT(0x01cd, 0x8), /* MEM_TRANS_RETIRED.LAT_ABOVE_THR */
543 INTEL_PST_CONSTRAINT(0x02cd, 0x8), /* MEM_TRANS_RETIRED.PRECISE_STORES */
418 INTEL_EVENT_CONSTRAINT(0xd0, 0xf), /* MEM_UOP_RETIRED.* */ 544 INTEL_EVENT_CONSTRAINT(0xd0, 0xf), /* MEM_UOP_RETIRED.* */
419 INTEL_EVENT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */ 545 INTEL_EVENT_CONSTRAINT(0xd1, 0xf), /* MEM_LOAD_UOPS_RETIRED.* */
420 INTEL_EVENT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */ 546 INTEL_EVENT_CONSTRAINT(0xd2, 0xf), /* MEM_LOAD_UOPS_LLC_HIT_RETIRED.* */
@@ -431,8 +557,10 @@ struct event_constraint *intel_pebs_constraints(struct perf_event *event)
431 557
432 if (x86_pmu.pebs_constraints) { 558 if (x86_pmu.pebs_constraints) {
433 for_each_event_constraint(c, x86_pmu.pebs_constraints) { 559 for_each_event_constraint(c, x86_pmu.pebs_constraints) {
434 if ((event->hw.config & c->cmask) == c->code) 560 if ((event->hw.config & c->cmask) == c->code) {
561 event->hw.flags |= c->flags;
435 return c; 562 return c;
563 }
436 } 564 }
437 } 565 }
438 566
@@ -447,6 +575,11 @@ void intel_pmu_pebs_enable(struct perf_event *event)
447 hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT; 575 hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
448 576
449 cpuc->pebs_enabled |= 1ULL << hwc->idx; 577 cpuc->pebs_enabled |= 1ULL << hwc->idx;
578
579 if (event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT)
580 cpuc->pebs_enabled |= 1ULL << (hwc->idx + 32);
581 else if (event->hw.flags & PERF_X86_EVENT_PEBS_ST)
582 cpuc->pebs_enabled |= 1ULL << 63;
450} 583}
451 584
452void intel_pmu_pebs_disable(struct perf_event *event) 585void intel_pmu_pebs_disable(struct perf_event *event)
@@ -559,20 +692,51 @@ static void __intel_pmu_pebs_event(struct perf_event *event,
559 struct pt_regs *iregs, void *__pebs) 692 struct pt_regs *iregs, void *__pebs)
560{ 693{
561 /* 694 /*
562 * We cast to pebs_record_core since that is a subset of 695 * We cast to pebs_record_nhm to get the load latency data
563 * both formats and we don't use the other fields in this 696 * if extra_reg MSR_PEBS_LD_LAT_THRESHOLD used
564 * routine.
565 */ 697 */
566 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); 698 struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
567 struct pebs_record_core *pebs = __pebs; 699 struct pebs_record_nhm *pebs = __pebs;
568 struct perf_sample_data data; 700 struct perf_sample_data data;
569 struct pt_regs regs; 701 struct pt_regs regs;
702 u64 sample_type;
703 int fll, fst;
570 704
571 if (!intel_pmu_save_and_restart(event)) 705 if (!intel_pmu_save_and_restart(event))
572 return; 706 return;
573 707
708 fll = event->hw.flags & PERF_X86_EVENT_PEBS_LDLAT;
709 fst = event->hw.flags & PERF_X86_EVENT_PEBS_ST;
710
574 perf_sample_data_init(&data, 0, event->hw.last_period); 711 perf_sample_data_init(&data, 0, event->hw.last_period);
575 712
713 data.period = event->hw.last_period;
714 sample_type = event->attr.sample_type;
715
716 /*
717 * if PEBS-LL or PreciseStore
718 */
719 if (fll || fst) {
720 if (sample_type & PERF_SAMPLE_ADDR)
721 data.addr = pebs->dla;
722
723 /*
724 * Use latency for weight (only avail with PEBS-LL)
725 */
726 if (fll && (sample_type & PERF_SAMPLE_WEIGHT))
727 data.weight = pebs->lat;
728
729 /*
730 * data.data_src encodes the data source
731 */
732 if (sample_type & PERF_SAMPLE_DATA_SRC) {
733 if (fll)
734 data.data_src.val = load_latency_data(pebs->dse);
735 else
736 data.data_src.val = precise_store_data(pebs->dse);
737 }
738 }
739
576 /* 740 /*
577 * We use the interrupt regs as a base because the PEBS record 741 * We use the interrupt regs as a base because the PEBS record
578 * does not contain a full regs set, specifically it seems to 742 * does not contain a full regs set, specifically it seems to
diff --git a/arch/x86/kernel/cpu/perf_event_intel_uncore.c b/arch/x86/kernel/cpu/perf_event_intel_uncore.c
index b43200dbfe7e..d0f9e5aa2151 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_uncore.c
+++ b/arch/x86/kernel/cpu/perf_event_intel_uncore.c
@@ -17,6 +17,9 @@ static struct event_constraint constraint_fixed =
17static struct event_constraint constraint_empty = 17static struct event_constraint constraint_empty =
18 EVENT_CONSTRAINT(0, 0, 0); 18 EVENT_CONSTRAINT(0, 0, 0);
19 19
20#define __BITS_VALUE(x, i, n) ((typeof(x))(((x) >> ((i) * (n))) & \
21 ((1ULL << (n)) - 1)))
22
20DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7"); 23DEFINE_UNCORE_FORMAT_ATTR(event, event, "config:0-7");
21DEFINE_UNCORE_FORMAT_ATTR(event_ext, event, "config:0-7,21"); 24DEFINE_UNCORE_FORMAT_ATTR(event_ext, event, "config:0-7,21");
22DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15"); 25DEFINE_UNCORE_FORMAT_ATTR(umask, umask, "config:8-15");
@@ -31,9 +34,13 @@ DEFINE_UNCORE_FORMAT_ATTR(occ_sel, occ_sel, "config:14-15");
31DEFINE_UNCORE_FORMAT_ATTR(occ_invert, occ_invert, "config:30"); 34DEFINE_UNCORE_FORMAT_ATTR(occ_invert, occ_invert, "config:30");
32DEFINE_UNCORE_FORMAT_ATTR(occ_edge, occ_edge, "config:14-51"); 35DEFINE_UNCORE_FORMAT_ATTR(occ_edge, occ_edge, "config:14-51");
33DEFINE_UNCORE_FORMAT_ATTR(filter_tid, filter_tid, "config1:0-4"); 36DEFINE_UNCORE_FORMAT_ATTR(filter_tid, filter_tid, "config1:0-4");
37DEFINE_UNCORE_FORMAT_ATTR(filter_link, filter_link, "config1:5-8");
34DEFINE_UNCORE_FORMAT_ATTR(filter_nid, filter_nid, "config1:10-17"); 38DEFINE_UNCORE_FORMAT_ATTR(filter_nid, filter_nid, "config1:10-17");
39DEFINE_UNCORE_FORMAT_ATTR(filter_nid2, filter_nid, "config1:32-47");
35DEFINE_UNCORE_FORMAT_ATTR(filter_state, filter_state, "config1:18-22"); 40DEFINE_UNCORE_FORMAT_ATTR(filter_state, filter_state, "config1:18-22");
41DEFINE_UNCORE_FORMAT_ATTR(filter_state2, filter_state, "config1:17-22");
36DEFINE_UNCORE_FORMAT_ATTR(filter_opc, filter_opc, "config1:23-31"); 42DEFINE_UNCORE_FORMAT_ATTR(filter_opc, filter_opc, "config1:23-31");
43DEFINE_UNCORE_FORMAT_ATTR(filter_opc2, filter_opc, "config1:52-60");
37DEFINE_UNCORE_FORMAT_ATTR(filter_band0, filter_band0, "config1:0-7"); 44DEFINE_UNCORE_FORMAT_ATTR(filter_band0, filter_band0, "config1:0-7");
38DEFINE_UNCORE_FORMAT_ATTR(filter_band1, filter_band1, "config1:8-15"); 45DEFINE_UNCORE_FORMAT_ATTR(filter_band1, filter_band1, "config1:8-15");
39DEFINE_UNCORE_FORMAT_ATTR(filter_band2, filter_band2, "config1:16-23"); 46DEFINE_UNCORE_FORMAT_ATTR(filter_band2, filter_band2, "config1:16-23");
@@ -110,6 +117,21 @@ static void uncore_put_constraint(struct intel_uncore_box *box, struct perf_even
110 reg1->alloc = 0; 117 reg1->alloc = 0;
111} 118}
112 119
120static u64 uncore_shared_reg_config(struct intel_uncore_box *box, int idx)
121{
122 struct intel_uncore_extra_reg *er;
123 unsigned long flags;
124 u64 config;
125
126 er = &box->shared_regs[idx];
127
128 raw_spin_lock_irqsave(&er->lock, flags);
129 config = er->config;
130 raw_spin_unlock_irqrestore(&er->lock, flags);
131
132 return config;
133}
134
113/* Sandy Bridge-EP uncore support */ 135/* Sandy Bridge-EP uncore support */
114static struct intel_uncore_type snbep_uncore_cbox; 136static struct intel_uncore_type snbep_uncore_cbox;
115static struct intel_uncore_type snbep_uncore_pcu; 137static struct intel_uncore_type snbep_uncore_pcu;
@@ -205,7 +227,7 @@ static void snbep_uncore_msr_enable_event(struct intel_uncore_box *box, struct p
205 struct hw_perf_event_extra *reg1 = &hwc->extra_reg; 227 struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
206 228
207 if (reg1->idx != EXTRA_REG_NONE) 229 if (reg1->idx != EXTRA_REG_NONE)
208 wrmsrl(reg1->reg, reg1->config); 230 wrmsrl(reg1->reg, uncore_shared_reg_config(box, 0));
209 231
210 wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN); 232 wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
211} 233}
@@ -226,29 +248,6 @@ static void snbep_uncore_msr_init_box(struct intel_uncore_box *box)
226 wrmsrl(msr, SNBEP_PMON_BOX_CTL_INT); 248 wrmsrl(msr, SNBEP_PMON_BOX_CTL_INT);
227} 249}
228 250
229static int snbep_uncore_hw_config(struct intel_uncore_box *box, struct perf_event *event)
230{
231 struct hw_perf_event *hwc = &event->hw;
232 struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
233
234 if (box->pmu->type == &snbep_uncore_cbox) {
235 reg1->reg = SNBEP_C0_MSR_PMON_BOX_FILTER +
236 SNBEP_CBO_MSR_OFFSET * box->pmu->pmu_idx;
237 reg1->config = event->attr.config1 &
238 SNBEP_CB0_MSR_PMON_BOX_FILTER_MASK;
239 } else {
240 if (box->pmu->type == &snbep_uncore_pcu) {
241 reg1->reg = SNBEP_PCU_MSR_PMON_BOX_FILTER;
242 reg1->config = event->attr.config1 & SNBEP_PCU_MSR_PMON_BOX_FILTER_MASK;
243 } else {
244 return 0;
245 }
246 }
247 reg1->idx = 0;
248
249 return 0;
250}
251
252static struct attribute *snbep_uncore_formats_attr[] = { 251static struct attribute *snbep_uncore_formats_attr[] = {
253 &format_attr_event.attr, 252 &format_attr_event.attr,
254 &format_attr_umask.attr, 253 &format_attr_umask.attr,
@@ -345,16 +344,16 @@ static struct attribute_group snbep_uncore_qpi_format_group = {
345 .attrs = snbep_uncore_qpi_formats_attr, 344 .attrs = snbep_uncore_qpi_formats_attr,
346}; 345};
347 346
347#define SNBEP_UNCORE_MSR_OPS_COMMON_INIT() \
348 .init_box = snbep_uncore_msr_init_box, \
349 .disable_box = snbep_uncore_msr_disable_box, \
350 .enable_box = snbep_uncore_msr_enable_box, \
351 .disable_event = snbep_uncore_msr_disable_event, \
352 .enable_event = snbep_uncore_msr_enable_event, \
353 .read_counter = uncore_msr_read_counter
354
348static struct intel_uncore_ops snbep_uncore_msr_ops = { 355static struct intel_uncore_ops snbep_uncore_msr_ops = {
349 .init_box = snbep_uncore_msr_init_box, 356 SNBEP_UNCORE_MSR_OPS_COMMON_INIT(),
350 .disable_box = snbep_uncore_msr_disable_box,
351 .enable_box = snbep_uncore_msr_enable_box,
352 .disable_event = snbep_uncore_msr_disable_event,
353 .enable_event = snbep_uncore_msr_enable_event,
354 .read_counter = uncore_msr_read_counter,
355 .get_constraint = uncore_get_constraint,
356 .put_constraint = uncore_put_constraint,
357 .hw_config = snbep_uncore_hw_config,
358}; 357};
359 358
360static struct intel_uncore_ops snbep_uncore_pci_ops = { 359static struct intel_uncore_ops snbep_uncore_pci_ops = {
@@ -372,6 +371,7 @@ static struct event_constraint snbep_uncore_cbox_constraints[] = {
372 UNCORE_EVENT_CONSTRAINT(0x04, 0x3), 371 UNCORE_EVENT_CONSTRAINT(0x04, 0x3),
373 UNCORE_EVENT_CONSTRAINT(0x05, 0x3), 372 UNCORE_EVENT_CONSTRAINT(0x05, 0x3),
374 UNCORE_EVENT_CONSTRAINT(0x07, 0x3), 373 UNCORE_EVENT_CONSTRAINT(0x07, 0x3),
374 UNCORE_EVENT_CONSTRAINT(0x09, 0x3),
375 UNCORE_EVENT_CONSTRAINT(0x11, 0x1), 375 UNCORE_EVENT_CONSTRAINT(0x11, 0x1),
376 UNCORE_EVENT_CONSTRAINT(0x12, 0x3), 376 UNCORE_EVENT_CONSTRAINT(0x12, 0x3),
377 UNCORE_EVENT_CONSTRAINT(0x13, 0x3), 377 UNCORE_EVENT_CONSTRAINT(0x13, 0x3),
@@ -421,6 +421,14 @@ static struct event_constraint snbep_uncore_r3qpi_constraints[] = {
421 UNCORE_EVENT_CONSTRAINT(0x24, 0x3), 421 UNCORE_EVENT_CONSTRAINT(0x24, 0x3),
422 UNCORE_EVENT_CONSTRAINT(0x25, 0x3), 422 UNCORE_EVENT_CONSTRAINT(0x25, 0x3),
423 UNCORE_EVENT_CONSTRAINT(0x26, 0x3), 423 UNCORE_EVENT_CONSTRAINT(0x26, 0x3),
424 UNCORE_EVENT_CONSTRAINT(0x28, 0x3),
425 UNCORE_EVENT_CONSTRAINT(0x29, 0x3),
426 UNCORE_EVENT_CONSTRAINT(0x2a, 0x3),
427 UNCORE_EVENT_CONSTRAINT(0x2b, 0x3),
428 UNCORE_EVENT_CONSTRAINT(0x2c, 0x3),
429 UNCORE_EVENT_CONSTRAINT(0x2d, 0x3),
430 UNCORE_EVENT_CONSTRAINT(0x2e, 0x3),
431 UNCORE_EVENT_CONSTRAINT(0x2f, 0x3),
424 UNCORE_EVENT_CONSTRAINT(0x30, 0x3), 432 UNCORE_EVENT_CONSTRAINT(0x30, 0x3),
425 UNCORE_EVENT_CONSTRAINT(0x31, 0x3), 433 UNCORE_EVENT_CONSTRAINT(0x31, 0x3),
426 UNCORE_EVENT_CONSTRAINT(0x32, 0x3), 434 UNCORE_EVENT_CONSTRAINT(0x32, 0x3),
@@ -428,6 +436,8 @@ static struct event_constraint snbep_uncore_r3qpi_constraints[] = {
428 UNCORE_EVENT_CONSTRAINT(0x34, 0x3), 436 UNCORE_EVENT_CONSTRAINT(0x34, 0x3),
429 UNCORE_EVENT_CONSTRAINT(0x36, 0x3), 437 UNCORE_EVENT_CONSTRAINT(0x36, 0x3),
430 UNCORE_EVENT_CONSTRAINT(0x37, 0x3), 438 UNCORE_EVENT_CONSTRAINT(0x37, 0x3),
439 UNCORE_EVENT_CONSTRAINT(0x38, 0x3),
440 UNCORE_EVENT_CONSTRAINT(0x39, 0x3),
431 EVENT_CONSTRAINT_END 441 EVENT_CONSTRAINT_END
432}; 442};
433 443
@@ -446,6 +456,145 @@ static struct intel_uncore_type snbep_uncore_ubox = {
446 .format_group = &snbep_uncore_ubox_format_group, 456 .format_group = &snbep_uncore_ubox_format_group,
447}; 457};
448 458
459static struct extra_reg snbep_uncore_cbox_extra_regs[] = {
460 SNBEP_CBO_EVENT_EXTRA_REG(SNBEP_CBO_PMON_CTL_TID_EN,
461 SNBEP_CBO_PMON_CTL_TID_EN, 0x1),
462 SNBEP_CBO_EVENT_EXTRA_REG(0x0334, 0xffff, 0x4),
463 SNBEP_CBO_EVENT_EXTRA_REG(0x0534, 0xffff, 0x4),
464 SNBEP_CBO_EVENT_EXTRA_REG(0x0934, 0xffff, 0x4),
465 SNBEP_CBO_EVENT_EXTRA_REG(0x4134, 0xffff, 0x6),
466 SNBEP_CBO_EVENT_EXTRA_REG(0x0135, 0xffff, 0x8),
467 SNBEP_CBO_EVENT_EXTRA_REG(0x0335, 0xffff, 0x8),
468 SNBEP_CBO_EVENT_EXTRA_REG(0x4135, 0xffff, 0xc),
469 SNBEP_CBO_EVENT_EXTRA_REG(0x4335, 0xffff, 0xc),
470 SNBEP_CBO_EVENT_EXTRA_REG(0x4435, 0xffff, 0x2),
471 SNBEP_CBO_EVENT_EXTRA_REG(0x4835, 0xffff, 0x2),
472 SNBEP_CBO_EVENT_EXTRA_REG(0x4a35, 0xffff, 0x2),
473 SNBEP_CBO_EVENT_EXTRA_REG(0x5035, 0xffff, 0x2),
474 SNBEP_CBO_EVENT_EXTRA_REG(0x0136, 0xffff, 0x8),
475 SNBEP_CBO_EVENT_EXTRA_REG(0x0336, 0xffff, 0x8),
476 SNBEP_CBO_EVENT_EXTRA_REG(0x4136, 0xffff, 0xc),
477 SNBEP_CBO_EVENT_EXTRA_REG(0x4336, 0xffff, 0xc),
478 SNBEP_CBO_EVENT_EXTRA_REG(0x4436, 0xffff, 0x2),
479 SNBEP_CBO_EVENT_EXTRA_REG(0x4836, 0xffff, 0x2),
480 SNBEP_CBO_EVENT_EXTRA_REG(0x4a36, 0xffff, 0x2),
481 SNBEP_CBO_EVENT_EXTRA_REG(0x4037, 0x40ff, 0x2),
482 EVENT_EXTRA_END
483};
484
485static void snbep_cbox_put_constraint(struct intel_uncore_box *box, struct perf_event *event)
486{
487 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
488 struct intel_uncore_extra_reg *er = &box->shared_regs[0];
489 int i;
490
491 if (uncore_box_is_fake(box))
492 return;
493
494 for (i = 0; i < 5; i++) {
495 if (reg1->alloc & (0x1 << i))
496 atomic_sub(1 << (i * 6), &er->ref);
497 }
498 reg1->alloc = 0;
499}
500
501static struct event_constraint *
502__snbep_cbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event,
503 u64 (*cbox_filter_mask)(int fields))
504{
505 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
506 struct intel_uncore_extra_reg *er = &box->shared_regs[0];
507 int i, alloc = 0;
508 unsigned long flags;
509 u64 mask;
510
511 if (reg1->idx == EXTRA_REG_NONE)
512 return NULL;
513
514 raw_spin_lock_irqsave(&er->lock, flags);
515 for (i = 0; i < 5; i++) {
516 if (!(reg1->idx & (0x1 << i)))
517 continue;
518 if (!uncore_box_is_fake(box) && (reg1->alloc & (0x1 << i)))
519 continue;
520
521 mask = cbox_filter_mask(0x1 << i);
522 if (!__BITS_VALUE(atomic_read(&er->ref), i, 6) ||
523 !((reg1->config ^ er->config) & mask)) {
524 atomic_add(1 << (i * 6), &er->ref);
525 er->config &= ~mask;
526 er->config |= reg1->config & mask;
527 alloc |= (0x1 << i);
528 } else {
529 break;
530 }
531 }
532 raw_spin_unlock_irqrestore(&er->lock, flags);
533 if (i < 5)
534 goto fail;
535
536 if (!uncore_box_is_fake(box))
537 reg1->alloc |= alloc;
538
539 return 0;
540fail:
541 for (; i >= 0; i--) {
542 if (alloc & (0x1 << i))
543 atomic_sub(1 << (i * 6), &er->ref);
544 }
545 return &constraint_empty;
546}
547
548static u64 snbep_cbox_filter_mask(int fields)
549{
550 u64 mask = 0;
551
552 if (fields & 0x1)
553 mask |= SNBEP_CB0_MSR_PMON_BOX_FILTER_TID;
554 if (fields & 0x2)
555 mask |= SNBEP_CB0_MSR_PMON_BOX_FILTER_NID;
556 if (fields & 0x4)
557 mask |= SNBEP_CB0_MSR_PMON_BOX_FILTER_STATE;
558 if (fields & 0x8)
559 mask |= SNBEP_CB0_MSR_PMON_BOX_FILTER_OPC;
560
561 return mask;
562}
563
564static struct event_constraint *
565snbep_cbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
566{
567 return __snbep_cbox_get_constraint(box, event, snbep_cbox_filter_mask);
568}
569
570static int snbep_cbox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
571{
572 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
573 struct extra_reg *er;
574 int idx = 0;
575
576 for (er = snbep_uncore_cbox_extra_regs; er->msr; er++) {
577 if (er->event != (event->hw.config & er->config_mask))
578 continue;
579 idx |= er->idx;
580 }
581
582 if (idx) {
583 reg1->reg = SNBEP_C0_MSR_PMON_BOX_FILTER +
584 SNBEP_CBO_MSR_OFFSET * box->pmu->pmu_idx;
585 reg1->config = event->attr.config1 & snbep_cbox_filter_mask(idx);
586 reg1->idx = idx;
587 }
588 return 0;
589}
590
591static struct intel_uncore_ops snbep_uncore_cbox_ops = {
592 SNBEP_UNCORE_MSR_OPS_COMMON_INIT(),
593 .hw_config = snbep_cbox_hw_config,
594 .get_constraint = snbep_cbox_get_constraint,
595 .put_constraint = snbep_cbox_put_constraint,
596};
597
449static struct intel_uncore_type snbep_uncore_cbox = { 598static struct intel_uncore_type snbep_uncore_cbox = {
450 .name = "cbox", 599 .name = "cbox",
451 .num_counters = 4, 600 .num_counters = 4,
@@ -458,10 +607,104 @@ static struct intel_uncore_type snbep_uncore_cbox = {
458 .msr_offset = SNBEP_CBO_MSR_OFFSET, 607 .msr_offset = SNBEP_CBO_MSR_OFFSET,
459 .num_shared_regs = 1, 608 .num_shared_regs = 1,
460 .constraints = snbep_uncore_cbox_constraints, 609 .constraints = snbep_uncore_cbox_constraints,
461 .ops = &snbep_uncore_msr_ops, 610 .ops = &snbep_uncore_cbox_ops,
462 .format_group = &snbep_uncore_cbox_format_group, 611 .format_group = &snbep_uncore_cbox_format_group,
463}; 612};
464 613
614static u64 snbep_pcu_alter_er(struct perf_event *event, int new_idx, bool modify)
615{
616 struct hw_perf_event *hwc = &event->hw;
617 struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
618 u64 config = reg1->config;
619
620 if (new_idx > reg1->idx)
621 config <<= 8 * (new_idx - reg1->idx);
622 else
623 config >>= 8 * (reg1->idx - new_idx);
624
625 if (modify) {
626 hwc->config += new_idx - reg1->idx;
627 reg1->config = config;
628 reg1->idx = new_idx;
629 }
630 return config;
631}
632
633static struct event_constraint *
634snbep_pcu_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
635{
636 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
637 struct intel_uncore_extra_reg *er = &box->shared_regs[0];
638 unsigned long flags;
639 int idx = reg1->idx;
640 u64 mask, config1 = reg1->config;
641 bool ok = false;
642
643 if (reg1->idx == EXTRA_REG_NONE ||
644 (!uncore_box_is_fake(box) && reg1->alloc))
645 return NULL;
646again:
647 mask = 0xff << (idx * 8);
648 raw_spin_lock_irqsave(&er->lock, flags);
649 if (!__BITS_VALUE(atomic_read(&er->ref), idx, 8) ||
650 !((config1 ^ er->config) & mask)) {
651 atomic_add(1 << (idx * 8), &er->ref);
652 er->config &= ~mask;
653 er->config |= config1 & mask;
654 ok = true;
655 }
656 raw_spin_unlock_irqrestore(&er->lock, flags);
657
658 if (!ok) {
659 idx = (idx + 1) % 4;
660 if (idx != reg1->idx) {
661 config1 = snbep_pcu_alter_er(event, idx, false);
662 goto again;
663 }
664 return &constraint_empty;
665 }
666
667 if (!uncore_box_is_fake(box)) {
668 if (idx != reg1->idx)
669 snbep_pcu_alter_er(event, idx, true);
670 reg1->alloc = 1;
671 }
672 return NULL;
673}
674
675static void snbep_pcu_put_constraint(struct intel_uncore_box *box, struct perf_event *event)
676{
677 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
678 struct intel_uncore_extra_reg *er = &box->shared_regs[0];
679
680 if (uncore_box_is_fake(box) || !reg1->alloc)
681 return;
682
683 atomic_sub(1 << (reg1->idx * 8), &er->ref);
684 reg1->alloc = 0;
685}
686
687static int snbep_pcu_hw_config(struct intel_uncore_box *box, struct perf_event *event)
688{
689 struct hw_perf_event *hwc = &event->hw;
690 struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
691 int ev_sel = hwc->config & SNBEP_PMON_CTL_EV_SEL_MASK;
692
693 if (ev_sel >= 0xb && ev_sel <= 0xe) {
694 reg1->reg = SNBEP_PCU_MSR_PMON_BOX_FILTER;
695 reg1->idx = ev_sel - 0xb;
696 reg1->config = event->attr.config1 & (0xff << reg1->idx);
697 }
698 return 0;
699}
700
701static struct intel_uncore_ops snbep_uncore_pcu_ops = {
702 SNBEP_UNCORE_MSR_OPS_COMMON_INIT(),
703 .hw_config = snbep_pcu_hw_config,
704 .get_constraint = snbep_pcu_get_constraint,
705 .put_constraint = snbep_pcu_put_constraint,
706};
707
465static struct intel_uncore_type snbep_uncore_pcu = { 708static struct intel_uncore_type snbep_uncore_pcu = {
466 .name = "pcu", 709 .name = "pcu",
467 .num_counters = 4, 710 .num_counters = 4,
@@ -472,7 +715,7 @@ static struct intel_uncore_type snbep_uncore_pcu = {
472 .event_mask = SNBEP_PCU_MSR_PMON_RAW_EVENT_MASK, 715 .event_mask = SNBEP_PCU_MSR_PMON_RAW_EVENT_MASK,
473 .box_ctl = SNBEP_PCU_MSR_PMON_BOX_CTL, 716 .box_ctl = SNBEP_PCU_MSR_PMON_BOX_CTL,
474 .num_shared_regs = 1, 717 .num_shared_regs = 1,
475 .ops = &snbep_uncore_msr_ops, 718 .ops = &snbep_uncore_pcu_ops,
476 .format_group = &snbep_uncore_pcu_format_group, 719 .format_group = &snbep_uncore_pcu_format_group,
477}; 720};
478 721
@@ -544,55 +787,63 @@ static struct intel_uncore_type snbep_uncore_r3qpi = {
544 SNBEP_UNCORE_PCI_COMMON_INIT(), 787 SNBEP_UNCORE_PCI_COMMON_INIT(),
545}; 788};
546 789
790enum {
791 SNBEP_PCI_UNCORE_HA,
792 SNBEP_PCI_UNCORE_IMC,
793 SNBEP_PCI_UNCORE_QPI,
794 SNBEP_PCI_UNCORE_R2PCIE,
795 SNBEP_PCI_UNCORE_R3QPI,
796};
797
547static struct intel_uncore_type *snbep_pci_uncores[] = { 798static struct intel_uncore_type *snbep_pci_uncores[] = {
548 &snbep_uncore_ha, 799 [SNBEP_PCI_UNCORE_HA] = &snbep_uncore_ha,
549 &snbep_uncore_imc, 800 [SNBEP_PCI_UNCORE_IMC] = &snbep_uncore_imc,
550 &snbep_uncore_qpi, 801 [SNBEP_PCI_UNCORE_QPI] = &snbep_uncore_qpi,
551 &snbep_uncore_r2pcie, 802 [SNBEP_PCI_UNCORE_R2PCIE] = &snbep_uncore_r2pcie,
552 &snbep_uncore_r3qpi, 803 [SNBEP_PCI_UNCORE_R3QPI] = &snbep_uncore_r3qpi,
553 NULL, 804 NULL,
554}; 805};
555 806
556static DEFINE_PCI_DEVICE_TABLE(snbep_uncore_pci_ids) = { 807static DEFINE_PCI_DEVICE_TABLE(snbep_uncore_pci_ids) = {
557 { /* Home Agent */ 808 { /* Home Agent */
558 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_HA), 809 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_HA),
559 .driver_data = (unsigned long)&snbep_uncore_ha, 810 .driver_data = SNBEP_PCI_UNCORE_HA,
560 }, 811 },
561 { /* MC Channel 0 */ 812 { /* MC Channel 0 */
562 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC0), 813 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC0),
563 .driver_data = (unsigned long)&snbep_uncore_imc, 814 .driver_data = SNBEP_PCI_UNCORE_IMC,
564 }, 815 },
565 { /* MC Channel 1 */ 816 { /* MC Channel 1 */
566 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC1), 817 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC1),
567 .driver_data = (unsigned long)&snbep_uncore_imc, 818 .driver_data = SNBEP_PCI_UNCORE_IMC,
568 }, 819 },
569 { /* MC Channel 2 */ 820 { /* MC Channel 2 */
570 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC2), 821 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC2),
571 .driver_data = (unsigned long)&snbep_uncore_imc, 822 .driver_data = SNBEP_PCI_UNCORE_IMC,
572 }, 823 },
573 { /* MC Channel 3 */ 824 { /* MC Channel 3 */
574 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC3), 825 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_IMC3),
575 .driver_data = (unsigned long)&snbep_uncore_imc, 826 .driver_data = SNBEP_PCI_UNCORE_IMC,
576 }, 827 },
577 { /* QPI Port 0 */ 828 { /* QPI Port 0 */
578 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_QPI0), 829 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_QPI0),
579 .driver_data = (unsigned long)&snbep_uncore_qpi, 830 .driver_data = SNBEP_PCI_UNCORE_QPI,
580 }, 831 },
581 { /* QPI Port 1 */ 832 { /* QPI Port 1 */
582 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_QPI1), 833 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_QPI1),
583 .driver_data = (unsigned long)&snbep_uncore_qpi, 834 .driver_data = SNBEP_PCI_UNCORE_QPI,
584 }, 835 },
585 { /* P2PCIe */ 836 { /* R2PCIe */
586 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R2PCIE), 837 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R2PCIE),
587 .driver_data = (unsigned long)&snbep_uncore_r2pcie, 838 .driver_data = SNBEP_PCI_UNCORE_R2PCIE,
588 }, 839 },
589 { /* R3QPI Link 0 */ 840 { /* R3QPI Link 0 */
590 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R3QPI0), 841 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R3QPI0),
591 .driver_data = (unsigned long)&snbep_uncore_r3qpi, 842 .driver_data = SNBEP_PCI_UNCORE_R3QPI,
592 }, 843 },
593 { /* R3QPI Link 1 */ 844 { /* R3QPI Link 1 */
594 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R3QPI1), 845 PCI_DEVICE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_UNC_R3QPI1),
595 .driver_data = (unsigned long)&snbep_uncore_r3qpi, 846 .driver_data = SNBEP_PCI_UNCORE_R3QPI,
596 }, 847 },
597 { /* end: all zeroes */ } 848 { /* end: all zeroes */ }
598}; 849};
@@ -605,7 +856,7 @@ static struct pci_driver snbep_uncore_pci_driver = {
605/* 856/*
606 * build pci bus to socket mapping 857 * build pci bus to socket mapping
607 */ 858 */
608static int snbep_pci2phy_map_init(void) 859static int snbep_pci2phy_map_init(int devid)
609{ 860{
610 struct pci_dev *ubox_dev = NULL; 861 struct pci_dev *ubox_dev = NULL;
611 int i, bus, nodeid; 862 int i, bus, nodeid;
@@ -614,9 +865,7 @@ static int snbep_pci2phy_map_init(void)
614 865
615 while (1) { 866 while (1) {
616 /* find the UBOX device */ 867 /* find the UBOX device */
617 ubox_dev = pci_get_device(PCI_VENDOR_ID_INTEL, 868 ubox_dev = pci_get_device(PCI_VENDOR_ID_INTEL, devid, ubox_dev);
618 PCI_DEVICE_ID_INTEL_JAKETOWN_UBOX,
619 ubox_dev);
620 if (!ubox_dev) 869 if (!ubox_dev)
621 break; 870 break;
622 bus = ubox_dev->bus->number; 871 bus = ubox_dev->bus->number;
@@ -639,7 +888,7 @@ static int snbep_pci2phy_map_init(void)
639 break; 888 break;
640 } 889 }
641 } 890 }
642 }; 891 }
643 892
644 if (ubox_dev) 893 if (ubox_dev)
645 pci_dev_put(ubox_dev); 894 pci_dev_put(ubox_dev);
@@ -648,6 +897,440 @@ static int snbep_pci2phy_map_init(void)
648} 897}
649/* end of Sandy Bridge-EP uncore support */ 898/* end of Sandy Bridge-EP uncore support */
650 899
900/* IvyTown uncore support */
901static void ivt_uncore_msr_init_box(struct intel_uncore_box *box)
902{
903 unsigned msr = uncore_msr_box_ctl(box);
904 if (msr)
905 wrmsrl(msr, IVT_PMON_BOX_CTL_INT);
906}
907
908static void ivt_uncore_pci_init_box(struct intel_uncore_box *box)
909{
910 struct pci_dev *pdev = box->pci_dev;
911
912 pci_write_config_dword(pdev, SNBEP_PCI_PMON_BOX_CTL, IVT_PMON_BOX_CTL_INT);
913}
914
915#define IVT_UNCORE_MSR_OPS_COMMON_INIT() \
916 .init_box = ivt_uncore_msr_init_box, \
917 .disable_box = snbep_uncore_msr_disable_box, \
918 .enable_box = snbep_uncore_msr_enable_box, \
919 .disable_event = snbep_uncore_msr_disable_event, \
920 .enable_event = snbep_uncore_msr_enable_event, \
921 .read_counter = uncore_msr_read_counter
922
923static struct intel_uncore_ops ivt_uncore_msr_ops = {
924 IVT_UNCORE_MSR_OPS_COMMON_INIT(),
925};
926
927static struct intel_uncore_ops ivt_uncore_pci_ops = {
928 .init_box = ivt_uncore_pci_init_box,
929 .disable_box = snbep_uncore_pci_disable_box,
930 .enable_box = snbep_uncore_pci_enable_box,
931 .disable_event = snbep_uncore_pci_disable_event,
932 .enable_event = snbep_uncore_pci_enable_event,
933 .read_counter = snbep_uncore_pci_read_counter,
934};
935
936#define IVT_UNCORE_PCI_COMMON_INIT() \
937 .perf_ctr = SNBEP_PCI_PMON_CTR0, \
938 .event_ctl = SNBEP_PCI_PMON_CTL0, \
939 .event_mask = IVT_PMON_RAW_EVENT_MASK, \
940 .box_ctl = SNBEP_PCI_PMON_BOX_CTL, \
941 .ops = &ivt_uncore_pci_ops, \
942 .format_group = &ivt_uncore_format_group
943
944static struct attribute *ivt_uncore_formats_attr[] = {
945 &format_attr_event.attr,
946 &format_attr_umask.attr,
947 &format_attr_edge.attr,
948 &format_attr_inv.attr,
949 &format_attr_thresh8.attr,
950 NULL,
951};
952
953static struct attribute *ivt_uncore_ubox_formats_attr[] = {
954 &format_attr_event.attr,
955 &format_attr_umask.attr,
956 &format_attr_edge.attr,
957 &format_attr_inv.attr,
958 &format_attr_thresh5.attr,
959 NULL,
960};
961
962static struct attribute *ivt_uncore_cbox_formats_attr[] = {
963 &format_attr_event.attr,
964 &format_attr_umask.attr,
965 &format_attr_edge.attr,
966 &format_attr_tid_en.attr,
967 &format_attr_thresh8.attr,
968 &format_attr_filter_tid.attr,
969 &format_attr_filter_link.attr,
970 &format_attr_filter_state2.attr,
971 &format_attr_filter_nid2.attr,
972 &format_attr_filter_opc2.attr,
973 NULL,
974};
975
976static struct attribute *ivt_uncore_pcu_formats_attr[] = {
977 &format_attr_event_ext.attr,
978 &format_attr_occ_sel.attr,
979 &format_attr_edge.attr,
980 &format_attr_thresh5.attr,
981 &format_attr_occ_invert.attr,
982 &format_attr_occ_edge.attr,
983 &format_attr_filter_band0.attr,
984 &format_attr_filter_band1.attr,
985 &format_attr_filter_band2.attr,
986 &format_attr_filter_band3.attr,
987 NULL,
988};
989
990static struct attribute *ivt_uncore_qpi_formats_attr[] = {
991 &format_attr_event_ext.attr,
992 &format_attr_umask.attr,
993 &format_attr_edge.attr,
994 &format_attr_thresh8.attr,
995 NULL,
996};
997
998static struct attribute_group ivt_uncore_format_group = {
999 .name = "format",
1000 .attrs = ivt_uncore_formats_attr,
1001};
1002
1003static struct attribute_group ivt_uncore_ubox_format_group = {
1004 .name = "format",
1005 .attrs = ivt_uncore_ubox_formats_attr,
1006};
1007
1008static struct attribute_group ivt_uncore_cbox_format_group = {
1009 .name = "format",
1010 .attrs = ivt_uncore_cbox_formats_attr,
1011};
1012
1013static struct attribute_group ivt_uncore_pcu_format_group = {
1014 .name = "format",
1015 .attrs = ivt_uncore_pcu_formats_attr,
1016};
1017
1018static struct attribute_group ivt_uncore_qpi_format_group = {
1019 .name = "format",
1020 .attrs = ivt_uncore_qpi_formats_attr,
1021};
1022
1023static struct intel_uncore_type ivt_uncore_ubox = {
1024 .name = "ubox",
1025 .num_counters = 2,
1026 .num_boxes = 1,
1027 .perf_ctr_bits = 44,
1028 .fixed_ctr_bits = 48,
1029 .perf_ctr = SNBEP_U_MSR_PMON_CTR0,
1030 .event_ctl = SNBEP_U_MSR_PMON_CTL0,
1031 .event_mask = IVT_U_MSR_PMON_RAW_EVENT_MASK,
1032 .fixed_ctr = SNBEP_U_MSR_PMON_UCLK_FIXED_CTR,
1033 .fixed_ctl = SNBEP_U_MSR_PMON_UCLK_FIXED_CTL,
1034 .ops = &ivt_uncore_msr_ops,
1035 .format_group = &ivt_uncore_ubox_format_group,
1036};
1037
1038static struct extra_reg ivt_uncore_cbox_extra_regs[] = {
1039 SNBEP_CBO_EVENT_EXTRA_REG(SNBEP_CBO_PMON_CTL_TID_EN,
1040 SNBEP_CBO_PMON_CTL_TID_EN, 0x1),
1041 SNBEP_CBO_EVENT_EXTRA_REG(0x1031, 0x10ff, 0x2),
1042 SNBEP_CBO_EVENT_EXTRA_REG(0x0334, 0xffff, 0x4),
1043 SNBEP_CBO_EVENT_EXTRA_REG(0x0534, 0xffff, 0x4),
1044 SNBEP_CBO_EVENT_EXTRA_REG(0x0934, 0xffff, 0x4),
1045 SNBEP_CBO_EVENT_EXTRA_REG(0x4134, 0xffff, 0xc),
1046 SNBEP_CBO_EVENT_EXTRA_REG(0x0135, 0xffff, 0x10),
1047 SNBEP_CBO_EVENT_EXTRA_REG(0x0335, 0xffff, 0x10),
1048 SNBEP_CBO_EVENT_EXTRA_REG(0x2135, 0xffff, 0x10),
1049 SNBEP_CBO_EVENT_EXTRA_REG(0x2335, 0xffff, 0x10),
1050 SNBEP_CBO_EVENT_EXTRA_REG(0x4135, 0xffff, 0x18),
1051 SNBEP_CBO_EVENT_EXTRA_REG(0x4335, 0xffff, 0x18),
1052 SNBEP_CBO_EVENT_EXTRA_REG(0x4435, 0xffff, 0x8),
1053 SNBEP_CBO_EVENT_EXTRA_REG(0x4835, 0xffff, 0x8),
1054 SNBEP_CBO_EVENT_EXTRA_REG(0x4a35, 0xffff, 0x8),
1055 SNBEP_CBO_EVENT_EXTRA_REG(0x5035, 0xffff, 0x8),
1056 SNBEP_CBO_EVENT_EXTRA_REG(0x8135, 0xffff, 0x10),
1057 SNBEP_CBO_EVENT_EXTRA_REG(0x8335, 0xffff, 0x10),
1058 SNBEP_CBO_EVENT_EXTRA_REG(0x0136, 0xffff, 0x10),
1059 SNBEP_CBO_EVENT_EXTRA_REG(0x0336, 0xffff, 0x10),
1060 SNBEP_CBO_EVENT_EXTRA_REG(0x2336, 0xffff, 0x10),
1061 SNBEP_CBO_EVENT_EXTRA_REG(0x2336, 0xffff, 0x10),
1062 SNBEP_CBO_EVENT_EXTRA_REG(0x4136, 0xffff, 0x18),
1063 SNBEP_CBO_EVENT_EXTRA_REG(0x4336, 0xffff, 0x18),
1064 SNBEP_CBO_EVENT_EXTRA_REG(0x4436, 0xffff, 0x8),
1065 SNBEP_CBO_EVENT_EXTRA_REG(0x4836, 0xffff, 0x8),
1066 SNBEP_CBO_EVENT_EXTRA_REG(0x4a36, 0xffff, 0x8),
1067 SNBEP_CBO_EVENT_EXTRA_REG(0x5036, 0xffff, 0x8),
1068 SNBEP_CBO_EVENT_EXTRA_REG(0x8136, 0xffff, 0x10),
1069 SNBEP_CBO_EVENT_EXTRA_REG(0x8336, 0xffff, 0x10),
1070 SNBEP_CBO_EVENT_EXTRA_REG(0x4037, 0x40ff, 0x8),
1071 EVENT_EXTRA_END
1072};
1073
1074static u64 ivt_cbox_filter_mask(int fields)
1075{
1076 u64 mask = 0;
1077
1078 if (fields & 0x1)
1079 mask |= IVT_CB0_MSR_PMON_BOX_FILTER_TID;
1080 if (fields & 0x2)
1081 mask |= IVT_CB0_MSR_PMON_BOX_FILTER_LINK;
1082 if (fields & 0x4)
1083 mask |= IVT_CB0_MSR_PMON_BOX_FILTER_STATE;
1084 if (fields & 0x8)
1085 mask |= IVT_CB0_MSR_PMON_BOX_FILTER_NID;
1086 if (fields & 0x10)
1087 mask |= IVT_CB0_MSR_PMON_BOX_FILTER_OPC;
1088
1089 return mask;
1090}
1091
1092static struct event_constraint *
1093ivt_cbox_get_constraint(struct intel_uncore_box *box, struct perf_event *event)
1094{
1095 return __snbep_cbox_get_constraint(box, event, ivt_cbox_filter_mask);
1096}
1097
1098static int ivt_cbox_hw_config(struct intel_uncore_box *box, struct perf_event *event)
1099{
1100 struct hw_perf_event_extra *reg1 = &event->hw.extra_reg;
1101 struct extra_reg *er;
1102 int idx = 0;
1103
1104 for (er = ivt_uncore_cbox_extra_regs; er->msr; er++) {
1105 if (er->event != (event->hw.config & er->config_mask))
1106 continue;
1107 idx |= er->idx;
1108 }
1109
1110 if (idx) {
1111 reg1->reg = SNBEP_C0_MSR_PMON_BOX_FILTER +
1112 SNBEP_CBO_MSR_OFFSET * box->pmu->pmu_idx;
1113 reg1->config = event->attr.config1 & ivt_cbox_filter_mask(idx);
1114 reg1->idx = idx;
1115 }
1116 return 0;
1117}
1118
1119static void ivt_cbox_enable_event(struct intel_uncore_box *box, struct perf_event *event)
1120{
1121 struct hw_perf_event *hwc = &event->hw;
1122 struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
1123
1124 if (reg1->idx != EXTRA_REG_NONE) {
1125 u64 filter = uncore_shared_reg_config(box, 0);
1126 wrmsrl(reg1->reg, filter & 0xffffffff);
1127 wrmsrl(reg1->reg + 6, filter >> 32);
1128 }
1129
1130 wrmsrl(hwc->config_base, hwc->config | SNBEP_PMON_CTL_EN);
1131}
1132
1133static struct intel_uncore_ops ivt_uncore_cbox_ops = {
1134 .init_box = ivt_uncore_msr_init_box,
1135 .disable_box = snbep_uncore_msr_disable_box,
1136 .enable_box = snbep_uncore_msr_enable_box,
1137 .disable_event = snbep_uncore_msr_disable_event,
1138 .enable_event = ivt_cbox_enable_event,
1139 .read_counter = uncore_msr_read_counter,
1140 .hw_config = ivt_cbox_hw_config,
1141 .get_constraint = ivt_cbox_get_constraint,
1142 .put_constraint = snbep_cbox_put_constraint,
1143};
1144
1145static struct intel_uncore_type ivt_uncore_cbox = {
1146 .name = "cbox",
1147 .num_counters = 4,
1148 .num_boxes = 15,
1149 .perf_ctr_bits = 44,
1150 .event_ctl = SNBEP_C0_MSR_PMON_CTL0,
1151 .perf_ctr = SNBEP_C0_MSR_PMON_CTR0,
1152 .event_mask = IVT_CBO_MSR_PMON_RAW_EVENT_MASK,
1153 .box_ctl = SNBEP_C0_MSR_PMON_BOX_CTL,
1154 .msr_offset = SNBEP_CBO_MSR_OFFSET,
1155 .num_shared_regs = 1,
1156 .constraints = snbep_uncore_cbox_constraints,
1157 .ops = &ivt_uncore_cbox_ops,
1158 .format_group = &ivt_uncore_cbox_format_group,
1159};
1160
1161static struct intel_uncore_ops ivt_uncore_pcu_ops = {
1162 IVT_UNCORE_MSR_OPS_COMMON_INIT(),
1163 .hw_config = snbep_pcu_hw_config,
1164 .get_constraint = snbep_pcu_get_constraint,
1165 .put_constraint = snbep_pcu_put_constraint,
1166};
1167
1168static struct intel_uncore_type ivt_uncore_pcu = {
1169 .name = "pcu",
1170 .num_counters = 4,
1171 .num_boxes = 1,
1172 .perf_ctr_bits = 48,
1173 .perf_ctr = SNBEP_PCU_MSR_PMON_CTR0,
1174 .event_ctl = SNBEP_PCU_MSR_PMON_CTL0,
1175 .event_mask = IVT_PCU_MSR_PMON_RAW_EVENT_MASK,
1176 .box_ctl = SNBEP_PCU_MSR_PMON_BOX_CTL,
1177 .num_shared_regs = 1,
1178 .ops = &ivt_uncore_pcu_ops,
1179 .format_group = &ivt_uncore_pcu_format_group,
1180};
1181
1182static struct intel_uncore_type *ivt_msr_uncores[] = {
1183 &ivt_uncore_ubox,
1184 &ivt_uncore_cbox,
1185 &ivt_uncore_pcu,
1186 NULL,
1187};
1188
1189static struct intel_uncore_type ivt_uncore_ha = {
1190 .name = "ha",
1191 .num_counters = 4,
1192 .num_boxes = 2,
1193 .perf_ctr_bits = 48,
1194 IVT_UNCORE_PCI_COMMON_INIT(),
1195};
1196
1197static struct intel_uncore_type ivt_uncore_imc = {
1198 .name = "imc",
1199 .num_counters = 4,
1200 .num_boxes = 8,
1201 .perf_ctr_bits = 48,
1202 .fixed_ctr_bits = 48,
1203 .fixed_ctr = SNBEP_MC_CHy_PCI_PMON_FIXED_CTR,
1204 .fixed_ctl = SNBEP_MC_CHy_PCI_PMON_FIXED_CTL,
1205 IVT_UNCORE_PCI_COMMON_INIT(),
1206};
1207
1208static struct intel_uncore_type ivt_uncore_qpi = {
1209 .name = "qpi",
1210 .num_counters = 4,
1211 .num_boxes = 3,
1212 .perf_ctr_bits = 48,
1213 .perf_ctr = SNBEP_PCI_PMON_CTR0,
1214 .event_ctl = SNBEP_PCI_PMON_CTL0,
1215 .event_mask = IVT_QPI_PCI_PMON_RAW_EVENT_MASK,
1216 .box_ctl = SNBEP_PCI_PMON_BOX_CTL,
1217 .ops = &ivt_uncore_pci_ops,
1218 .format_group = &ivt_uncore_qpi_format_group,
1219};
1220
1221static struct intel_uncore_type ivt_uncore_r2pcie = {
1222 .name = "r2pcie",
1223 .num_counters = 4,
1224 .num_boxes = 1,
1225 .perf_ctr_bits = 44,
1226 .constraints = snbep_uncore_r2pcie_constraints,
1227 IVT_UNCORE_PCI_COMMON_INIT(),
1228};
1229
1230static struct intel_uncore_type ivt_uncore_r3qpi = {
1231 .name = "r3qpi",
1232 .num_counters = 3,
1233 .num_boxes = 2,
1234 .perf_ctr_bits = 44,
1235 .constraints = snbep_uncore_r3qpi_constraints,
1236 IVT_UNCORE_PCI_COMMON_INIT(),
1237};
1238
1239enum {
1240 IVT_PCI_UNCORE_HA,
1241 IVT_PCI_UNCORE_IMC,
1242 IVT_PCI_UNCORE_QPI,
1243 IVT_PCI_UNCORE_R2PCIE,
1244 IVT_PCI_UNCORE_R3QPI,
1245};
1246
1247static struct intel_uncore_type *ivt_pci_uncores[] = {
1248 [IVT_PCI_UNCORE_HA] = &ivt_uncore_ha,
1249 [IVT_PCI_UNCORE_IMC] = &ivt_uncore_imc,
1250 [IVT_PCI_UNCORE_QPI] = &ivt_uncore_qpi,
1251 [IVT_PCI_UNCORE_R2PCIE] = &ivt_uncore_r2pcie,
1252 [IVT_PCI_UNCORE_R3QPI] = &ivt_uncore_r3qpi,
1253 NULL,
1254};
1255
1256static DEFINE_PCI_DEVICE_TABLE(ivt_uncore_pci_ids) = {
1257 { /* Home Agent 0 */
1258 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe30),
1259 .driver_data = IVT_PCI_UNCORE_HA,
1260 },
1261 { /* Home Agent 1 */
1262 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe38),
1263 .driver_data = IVT_PCI_UNCORE_HA,
1264 },
1265 { /* MC0 Channel 0 */
1266 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb4),
1267 .driver_data = IVT_PCI_UNCORE_IMC,
1268 },
1269 { /* MC0 Channel 1 */
1270 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb5),
1271 .driver_data = IVT_PCI_UNCORE_IMC,
1272 },
1273 { /* MC0 Channel 3 */
1274 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb0),
1275 .driver_data = IVT_PCI_UNCORE_IMC,
1276 },
1277 { /* MC0 Channel 4 */
1278 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xeb1),
1279 .driver_data = IVT_PCI_UNCORE_IMC,
1280 },
1281 { /* MC1 Channel 0 */
1282 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef4),
1283 .driver_data = IVT_PCI_UNCORE_IMC,
1284 },
1285 { /* MC1 Channel 1 */
1286 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef5),
1287 .driver_data = IVT_PCI_UNCORE_IMC,
1288 },
1289 { /* MC1 Channel 3 */
1290 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef0),
1291 .driver_data = IVT_PCI_UNCORE_IMC,
1292 },
1293 { /* MC1 Channel 4 */
1294 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xef1),
1295 .driver_data = IVT_PCI_UNCORE_IMC,
1296 },
1297 { /* QPI0 Port 0 */
1298 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe32),
1299 .driver_data = IVT_PCI_UNCORE_QPI,
1300 },
1301 { /* QPI0 Port 1 */
1302 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe33),
1303 .driver_data = IVT_PCI_UNCORE_QPI,
1304 },
1305 { /* QPI1 Port 2 */
1306 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe3a),
1307 .driver_data = IVT_PCI_UNCORE_QPI,
1308 },
1309 { /* R2PCIe */
1310 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe34),
1311 .driver_data = IVT_PCI_UNCORE_R2PCIE,
1312 },
1313 { /* R3QPI0 Link 0 */
1314 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe36),
1315 .driver_data = IVT_PCI_UNCORE_R3QPI,
1316 },
1317 { /* R3QPI0 Link 1 */
1318 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe37),
1319 .driver_data = IVT_PCI_UNCORE_R3QPI,
1320 },
1321 { /* R3QPI1 Link 2 */
1322 PCI_DEVICE(PCI_VENDOR_ID_INTEL, 0xe3e),
1323 .driver_data = IVT_PCI_UNCORE_R3QPI,
1324 },
1325 { /* end: all zeroes */ }
1326};
1327
1328static struct pci_driver ivt_uncore_pci_driver = {
1329 .name = "ivt_uncore",
1330 .id_table = ivt_uncore_pci_ids,
1331};
1332/* end of IvyTown uncore support */
1333
651/* Sandy Bridge uncore support */ 1334/* Sandy Bridge uncore support */
652static void snb_uncore_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event) 1335static void snb_uncore_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
653{ 1336{
@@ -808,9 +1491,6 @@ static struct intel_uncore_type *nhm_msr_uncores[] = {
808/* end of Nehalem uncore support */ 1491/* end of Nehalem uncore support */
809 1492
810/* Nehalem-EX uncore support */ 1493/* Nehalem-EX uncore support */
811#define __BITS_VALUE(x, i, n) ((typeof(x))(((x) >> ((i) * (n))) & \
812 ((1ULL << (n)) - 1)))
813
814DEFINE_UNCORE_FORMAT_ATTR(event5, event, "config:1-5"); 1494DEFINE_UNCORE_FORMAT_ATTR(event5, event, "config:1-5");
815DEFINE_UNCORE_FORMAT_ATTR(counter, counter, "config:6-7"); 1495DEFINE_UNCORE_FORMAT_ATTR(counter, counter, "config:6-7");
816DEFINE_UNCORE_FORMAT_ATTR(match, match, "config1:0-63"); 1496DEFINE_UNCORE_FORMAT_ATTR(match, match, "config1:0-63");
@@ -1161,7 +1841,7 @@ static struct extra_reg nhmex_uncore_mbox_extra_regs[] = {
1161}; 1841};
1162 1842
1163/* Nehalem-EX or Westmere-EX ? */ 1843/* Nehalem-EX or Westmere-EX ? */
1164bool uncore_nhmex; 1844static bool uncore_nhmex;
1165 1845
1166static bool nhmex_mbox_get_shared_reg(struct intel_uncore_box *box, int idx, u64 config) 1846static bool nhmex_mbox_get_shared_reg(struct intel_uncore_box *box, int idx, u64 config)
1167{ 1847{
@@ -1239,7 +1919,7 @@ static void nhmex_mbox_put_shared_reg(struct intel_uncore_box *box, int idx)
1239 atomic_sub(1 << (idx * 8), &er->ref); 1919 atomic_sub(1 << (idx * 8), &er->ref);
1240} 1920}
1241 1921
1242u64 nhmex_mbox_alter_er(struct perf_event *event, int new_idx, bool modify) 1922static u64 nhmex_mbox_alter_er(struct perf_event *event, int new_idx, bool modify)
1243{ 1923{
1244 struct hw_perf_event *hwc = &event->hw; 1924 struct hw_perf_event *hwc = &event->hw;
1245 struct hw_perf_event_extra *reg1 = &hwc->extra_reg; 1925 struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
@@ -1554,7 +2234,7 @@ static struct intel_uncore_type nhmex_uncore_mbox = {
1554 .format_group = &nhmex_uncore_mbox_format_group, 2234 .format_group = &nhmex_uncore_mbox_format_group,
1555}; 2235};
1556 2236
1557void nhmex_rbox_alter_er(struct intel_uncore_box *box, struct perf_event *event) 2237static void nhmex_rbox_alter_er(struct intel_uncore_box *box, struct perf_event *event)
1558{ 2238{
1559 struct hw_perf_event *hwc = &event->hw; 2239 struct hw_perf_event *hwc = &event->hw;
1560 struct hw_perf_event_extra *reg1 = &hwc->extra_reg; 2240 struct hw_perf_event_extra *reg1 = &hwc->extra_reg;
@@ -1724,21 +2404,6 @@ static int nhmex_rbox_hw_config(struct intel_uncore_box *box, struct perf_event
1724 return 0; 2404 return 0;
1725} 2405}
1726 2406
1727static u64 nhmex_rbox_shared_reg_config(struct intel_uncore_box *box, int idx)
1728{
1729 struct intel_uncore_extra_reg *er;
1730 unsigned long flags;
1731 u64 config;
1732
1733 er = &box->shared_regs[idx];
1734
1735 raw_spin_lock_irqsave(&er->lock, flags);
1736 config = er->config;
1737 raw_spin_unlock_irqrestore(&er->lock, flags);
1738
1739 return config;
1740}
1741
1742static void nhmex_rbox_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event) 2407static void nhmex_rbox_msr_enable_event(struct intel_uncore_box *box, struct perf_event *event)
1743{ 2408{
1744 struct hw_perf_event *hwc = &event->hw; 2409 struct hw_perf_event *hwc = &event->hw;
@@ -1759,7 +2424,7 @@ static void nhmex_rbox_msr_enable_event(struct intel_uncore_box *box, struct per
1759 case 2: 2424 case 2:
1760 case 3: 2425 case 3:
1761 wrmsrl(NHMEX_R_MSR_PORTN_QLX_CFG(port), 2426 wrmsrl(NHMEX_R_MSR_PORTN_QLX_CFG(port),
1762 nhmex_rbox_shared_reg_config(box, 2 + (idx / 6) * 5)); 2427 uncore_shared_reg_config(box, 2 + (idx / 6) * 5));
1763 break; 2428 break;
1764 case 4: 2429 case 4:
1765 wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET1_MM_CFG(port), 2430 wrmsrl(NHMEX_R_MSR_PORTN_XBR_SET1_MM_CFG(port),
@@ -2285,7 +2950,7 @@ out:
2285 return ret; 2950 return ret;
2286} 2951}
2287 2952
2288int uncore_pmu_event_init(struct perf_event *event) 2953static int uncore_pmu_event_init(struct perf_event *event)
2289{ 2954{
2290 struct intel_uncore_pmu *pmu; 2955 struct intel_uncore_pmu *pmu;
2291 struct intel_uncore_box *box; 2956 struct intel_uncore_box *box;
@@ -2438,7 +3103,7 @@ static int __init uncore_type_init(struct intel_uncore_type *type)
2438 3103
2439 type->unconstrainted = (struct event_constraint) 3104 type->unconstrainted = (struct event_constraint)
2440 __EVENT_CONSTRAINT(0, (1ULL << type->num_counters) - 1, 3105 __EVENT_CONSTRAINT(0, (1ULL << type->num_counters) - 1,
2441 0, type->num_counters, 0); 3106 0, type->num_counters, 0, 0);
2442 3107
2443 for (i = 0; i < type->num_boxes; i++) { 3108 for (i = 0; i < type->num_boxes; i++) {
2444 pmus[i].func_id = -1; 3109 pmus[i].func_id = -1;
@@ -2556,6 +3221,8 @@ static void uncore_pci_remove(struct pci_dev *pdev)
2556 if (WARN_ON_ONCE(phys_id != box->phys_id)) 3221 if (WARN_ON_ONCE(phys_id != box->phys_id))
2557 return; 3222 return;
2558 3223
3224 pci_set_drvdata(pdev, NULL);
3225
2559 raw_spin_lock(&uncore_box_lock); 3226 raw_spin_lock(&uncore_box_lock);
2560 list_del(&box->list); 3227 list_del(&box->list);
2561 raw_spin_unlock(&uncore_box_lock); 3228 raw_spin_unlock(&uncore_box_lock);
@@ -2574,11 +3241,7 @@ static void uncore_pci_remove(struct pci_dev *pdev)
2574static int uncore_pci_probe(struct pci_dev *pdev, 3241static int uncore_pci_probe(struct pci_dev *pdev,
2575 const struct pci_device_id *id) 3242 const struct pci_device_id *id)
2576{ 3243{
2577 struct intel_uncore_type *type; 3244 return uncore_pci_add(pci_uncores[id->driver_data], pdev);
2578
2579 type = (struct intel_uncore_type *)id->driver_data;
2580
2581 return uncore_pci_add(type, pdev);
2582} 3245}
2583 3246
2584static int __init uncore_pci_init(void) 3247static int __init uncore_pci_init(void)
@@ -2587,12 +3250,19 @@ static int __init uncore_pci_init(void)
2587 3250
2588 switch (boot_cpu_data.x86_model) { 3251 switch (boot_cpu_data.x86_model) {
2589 case 45: /* Sandy Bridge-EP */ 3252 case 45: /* Sandy Bridge-EP */
2590 ret = snbep_pci2phy_map_init(); 3253 ret = snbep_pci2phy_map_init(0x3ce0);
2591 if (ret) 3254 if (ret)
2592 return ret; 3255 return ret;
2593 pci_uncores = snbep_pci_uncores; 3256 pci_uncores = snbep_pci_uncores;
2594 uncore_pci_driver = &snbep_uncore_pci_driver; 3257 uncore_pci_driver = &snbep_uncore_pci_driver;
2595 break; 3258 break;
3259 case 62: /* IvyTown */
3260 ret = snbep_pci2phy_map_init(0x0e1e);
3261 if (ret)
3262 return ret;
3263 pci_uncores = ivt_pci_uncores;
3264 uncore_pci_driver = &ivt_uncore_pci_driver;
3265 break;
2596 default: 3266 default:
2597 return 0; 3267 return 0;
2598 } 3268 }
@@ -2622,6 +3292,21 @@ static void __init uncore_pci_exit(void)
2622 } 3292 }
2623} 3293}
2624 3294
3295/* CPU hot plug/unplug are serialized by cpu_add_remove_lock mutex */
3296static LIST_HEAD(boxes_to_free);
3297
3298static void __cpuinit uncore_kfree_boxes(void)
3299{
3300 struct intel_uncore_box *box;
3301
3302 while (!list_empty(&boxes_to_free)) {
3303 box = list_entry(boxes_to_free.next,
3304 struct intel_uncore_box, list);
3305 list_del(&box->list);
3306 kfree(box);
3307 }
3308}
3309
2625static void __cpuinit uncore_cpu_dying(int cpu) 3310static void __cpuinit uncore_cpu_dying(int cpu)
2626{ 3311{
2627 struct intel_uncore_type *type; 3312 struct intel_uncore_type *type;
@@ -2636,7 +3321,7 @@ static void __cpuinit uncore_cpu_dying(int cpu)
2636 box = *per_cpu_ptr(pmu->box, cpu); 3321 box = *per_cpu_ptr(pmu->box, cpu);
2637 *per_cpu_ptr(pmu->box, cpu) = NULL; 3322 *per_cpu_ptr(pmu->box, cpu) = NULL;
2638 if (box && atomic_dec_and_test(&box->refcnt)) 3323 if (box && atomic_dec_and_test(&box->refcnt))
2639 kfree(box); 3324 list_add(&box->list, &boxes_to_free);
2640 } 3325 }
2641 } 3326 }
2642} 3327}
@@ -2666,8 +3351,11 @@ static int __cpuinit uncore_cpu_starting(int cpu)
2666 if (exist && exist->phys_id == phys_id) { 3351 if (exist && exist->phys_id == phys_id) {
2667 atomic_inc(&exist->refcnt); 3352 atomic_inc(&exist->refcnt);
2668 *per_cpu_ptr(pmu->box, cpu) = exist; 3353 *per_cpu_ptr(pmu->box, cpu) = exist;
2669 kfree(box); 3354 if (box) {
2670 box = NULL; 3355 list_add(&box->list,
3356 &boxes_to_free);
3357 box = NULL;
3358 }
2671 break; 3359 break;
2672 } 3360 }
2673 } 3361 }
@@ -2806,6 +3494,10 @@ static int
2806 case CPU_DYING: 3494 case CPU_DYING:
2807 uncore_cpu_dying(cpu); 3495 uncore_cpu_dying(cpu);
2808 break; 3496 break;
3497 case CPU_ONLINE:
3498 case CPU_DEAD:
3499 uncore_kfree_boxes();
3500 break;
2809 default: 3501 default:
2810 break; 3502 break;
2811 } 3503 }
@@ -2871,6 +3563,12 @@ static int __init uncore_cpu_init(void)
2871 nhmex_uncore_cbox.num_boxes = max_cores; 3563 nhmex_uncore_cbox.num_boxes = max_cores;
2872 msr_uncores = nhmex_msr_uncores; 3564 msr_uncores = nhmex_msr_uncores;
2873 break; 3565 break;
3566 case 62: /* IvyTown */
3567 if (ivt_uncore_cbox.num_boxes > max_cores)
3568 ivt_uncore_cbox.num_boxes = max_cores;
3569 msr_uncores = ivt_msr_uncores;
3570 break;
3571
2874 default: 3572 default:
2875 return 0; 3573 return 0;
2876 } 3574 }
diff --git a/arch/x86/kernel/cpu/perf_event_intel_uncore.h b/arch/x86/kernel/cpu/perf_event_intel_uncore.h
index e68a4550e952..f9528917f6e8 100644
--- a/arch/x86/kernel/cpu/perf_event_intel_uncore.h
+++ b/arch/x86/kernel/cpu/perf_event_intel_uncore.h
@@ -76,7 +76,7 @@
76#define SNBEP_PMON_CTL_UMASK_MASK 0x0000ff00 76#define SNBEP_PMON_CTL_UMASK_MASK 0x0000ff00
77#define SNBEP_PMON_CTL_RST (1 << 17) 77#define SNBEP_PMON_CTL_RST (1 << 17)
78#define SNBEP_PMON_CTL_EDGE_DET (1 << 18) 78#define SNBEP_PMON_CTL_EDGE_DET (1 << 18)
79#define SNBEP_PMON_CTL_EV_SEL_EXT (1 << 21) /* only for QPI */ 79#define SNBEP_PMON_CTL_EV_SEL_EXT (1 << 21)
80#define SNBEP_PMON_CTL_EN (1 << 22) 80#define SNBEP_PMON_CTL_EN (1 << 22)
81#define SNBEP_PMON_CTL_INVERT (1 << 23) 81#define SNBEP_PMON_CTL_INVERT (1 << 23)
82#define SNBEP_PMON_CTL_TRESH_MASK 0xff000000 82#define SNBEP_PMON_CTL_TRESH_MASK 0xff000000
@@ -148,9 +148,20 @@
148#define SNBEP_C0_MSR_PMON_CTL0 0xd10 148#define SNBEP_C0_MSR_PMON_CTL0 0xd10
149#define SNBEP_C0_MSR_PMON_BOX_CTL 0xd04 149#define SNBEP_C0_MSR_PMON_BOX_CTL 0xd04
150#define SNBEP_C0_MSR_PMON_BOX_FILTER 0xd14 150#define SNBEP_C0_MSR_PMON_BOX_FILTER 0xd14
151#define SNBEP_CB0_MSR_PMON_BOX_FILTER_MASK 0xfffffc1f
152#define SNBEP_CBO_MSR_OFFSET 0x20 151#define SNBEP_CBO_MSR_OFFSET 0x20
153 152
153#define SNBEP_CB0_MSR_PMON_BOX_FILTER_TID 0x1f
154#define SNBEP_CB0_MSR_PMON_BOX_FILTER_NID 0x3fc00
155#define SNBEP_CB0_MSR_PMON_BOX_FILTER_STATE 0x7c0000
156#define SNBEP_CB0_MSR_PMON_BOX_FILTER_OPC 0xff800000
157
158#define SNBEP_CBO_EVENT_EXTRA_REG(e, m, i) { \
159 .event = (e), \
160 .msr = SNBEP_C0_MSR_PMON_BOX_FILTER, \
161 .config_mask = (m), \
162 .idx = (i) \
163}
164
154/* SNB-EP PCU register */ 165/* SNB-EP PCU register */
155#define SNBEP_PCU_MSR_PMON_CTR0 0xc36 166#define SNBEP_PCU_MSR_PMON_CTR0 0xc36
156#define SNBEP_PCU_MSR_PMON_CTL0 0xc30 167#define SNBEP_PCU_MSR_PMON_CTL0 0xc30
@@ -160,6 +171,55 @@
160#define SNBEP_PCU_MSR_CORE_C3_CTR 0x3fc 171#define SNBEP_PCU_MSR_CORE_C3_CTR 0x3fc
161#define SNBEP_PCU_MSR_CORE_C6_CTR 0x3fd 172#define SNBEP_PCU_MSR_CORE_C6_CTR 0x3fd
162 173
174/* IVT event control */
175#define IVT_PMON_BOX_CTL_INT (SNBEP_PMON_BOX_CTL_RST_CTRL | \
176 SNBEP_PMON_BOX_CTL_RST_CTRS)
177#define IVT_PMON_RAW_EVENT_MASK (SNBEP_PMON_CTL_EV_SEL_MASK | \
178 SNBEP_PMON_CTL_UMASK_MASK | \
179 SNBEP_PMON_CTL_EDGE_DET | \
180 SNBEP_PMON_CTL_TRESH_MASK)
181/* IVT Ubox */
182#define IVT_U_MSR_PMON_GLOBAL_CTL 0xc00
183#define IVT_U_PMON_GLOBAL_FRZ_ALL (1 << 31)
184#define IVT_U_PMON_GLOBAL_UNFRZ_ALL (1 << 29)
185
186#define IVT_U_MSR_PMON_RAW_EVENT_MASK \
187 (SNBEP_PMON_CTL_EV_SEL_MASK | \
188 SNBEP_PMON_CTL_UMASK_MASK | \
189 SNBEP_PMON_CTL_EDGE_DET | \
190 SNBEP_U_MSR_PMON_CTL_TRESH_MASK)
191/* IVT Cbo */
192#define IVT_CBO_MSR_PMON_RAW_EVENT_MASK (IVT_PMON_RAW_EVENT_MASK | \
193 SNBEP_CBO_PMON_CTL_TID_EN)
194
195#define IVT_CB0_MSR_PMON_BOX_FILTER_TID (0x1fULL << 0)
196#define IVT_CB0_MSR_PMON_BOX_FILTER_LINK (0xfULL << 5)
197#define IVT_CB0_MSR_PMON_BOX_FILTER_STATE (0x3fULL << 17)
198#define IVT_CB0_MSR_PMON_BOX_FILTER_NID (0xffffULL << 32)
199#define IVT_CB0_MSR_PMON_BOX_FILTER_OPC (0x1ffULL << 52)
200#define IVT_CB0_MSR_PMON_BOX_FILTER_C6 (0x1ULL << 61)
201#define IVT_CB0_MSR_PMON_BOX_FILTER_NC (0x1ULL << 62)
202#define IVT_CB0_MSR_PMON_BOX_FILTER_IOSC (0x1ULL << 63)
203
204/* IVT home agent */
205#define IVT_HA_PCI_PMON_CTL_Q_OCC_RST (1 << 16)
206#define IVT_HA_PCI_PMON_RAW_EVENT_MASK \
207 (IVT_PMON_RAW_EVENT_MASK | \
208 IVT_HA_PCI_PMON_CTL_Q_OCC_RST)
209/* IVT PCU */
210#define IVT_PCU_MSR_PMON_RAW_EVENT_MASK \
211 (SNBEP_PMON_CTL_EV_SEL_MASK | \
212 SNBEP_PMON_CTL_EV_SEL_EXT | \
213 SNBEP_PCU_MSR_PMON_CTL_OCC_SEL_MASK | \
214 SNBEP_PMON_CTL_EDGE_DET | \
215 SNBEP_PCU_MSR_PMON_CTL_TRESH_MASK | \
216 SNBEP_PCU_MSR_PMON_CTL_OCC_INVERT | \
217 SNBEP_PCU_MSR_PMON_CTL_OCC_EDGE_DET)
218/* IVT QPI */
219#define IVT_QPI_PCI_PMON_RAW_EVENT_MASK \
220 (IVT_PMON_RAW_EVENT_MASK | \
221 SNBEP_PMON_CTL_EV_SEL_EXT)
222
163/* NHM-EX event control */ 223/* NHM-EX event control */
164#define NHMEX_PMON_CTL_EV_SEL_MASK 0x000000ff 224#define NHMEX_PMON_CTL_EV_SEL_MASK 0x000000ff
165#define NHMEX_PMON_CTL_UMASK_MASK 0x0000ff00 225#define NHMEX_PMON_CTL_UMASK_MASK 0x0000ff00
diff --git a/arch/x86/kernel/cpu/perf_event_p4.c b/arch/x86/kernel/cpu/perf_event_p4.c
index 92c7e39a079f..3486e6660357 100644
--- a/arch/x86/kernel/cpu/perf_event_p4.c
+++ b/arch/x86/kernel/cpu/perf_event_p4.c
@@ -895,8 +895,8 @@ static void p4_pmu_disable_pebs(void)
895 * So at moment let leave metrics turned on forever -- it's 895 * So at moment let leave metrics turned on forever -- it's
896 * ok for now but need to be revisited! 896 * ok for now but need to be revisited!
897 * 897 *
898 * (void)wrmsrl_safe(MSR_IA32_PEBS_ENABLE, (u64)0); 898 * (void)wrmsrl_safe(MSR_IA32_PEBS_ENABLE, 0);
899 * (void)wrmsrl_safe(MSR_P4_PEBS_MATRIX_VERT, (u64)0); 899 * (void)wrmsrl_safe(MSR_P4_PEBS_MATRIX_VERT, 0);
900 */ 900 */
901} 901}
902 902
@@ -910,8 +910,7 @@ static inline void p4_pmu_disable_event(struct perf_event *event)
910 * asserted again and again 910 * asserted again and again
911 */ 911 */
912 (void)wrmsrl_safe(hwc->config_base, 912 (void)wrmsrl_safe(hwc->config_base,
913 (u64)(p4_config_unpack_cccr(hwc->config)) & 913 p4_config_unpack_cccr(hwc->config) & ~P4_CCCR_ENABLE & ~P4_CCCR_OVF & ~P4_CCCR_RESERVED);
914 ~P4_CCCR_ENABLE & ~P4_CCCR_OVF & ~P4_CCCR_RESERVED);
915} 914}
916 915
917static void p4_pmu_disable_all(void) 916static void p4_pmu_disable_all(void)
@@ -957,7 +956,7 @@ static void p4_pmu_enable_event(struct perf_event *event)
957 u64 escr_addr, cccr; 956 u64 escr_addr, cccr;
958 957
959 bind = &p4_event_bind_map[idx]; 958 bind = &p4_event_bind_map[idx];
960 escr_addr = (u64)bind->escr_msr[thread]; 959 escr_addr = bind->escr_msr[thread];
961 960
962 /* 961 /*
963 * - we dont support cascaded counters yet 962 * - we dont support cascaded counters yet
generated by cgit v1.2.2 (git 2.25.0) at 2024-10-16 08:34:04 -0400