1 | // SPDX-License-Identifier: GPL-2.0 |
2 | #include <linux/perf_event.h> |
3 | #include <linux/sysfs.h> |
4 | #include <linux/nospec.h> |
5 | #include <asm/intel-family.h> |
6 | #include "probe.h" |
7 | |
8 | enum perf_msr_id { |
9 | PERF_MSR_TSC = 0, |
10 | PERF_MSR_APERF = 1, |
11 | PERF_MSR_MPERF = 2, |
12 | PERF_MSR_PPERF = 3, |
13 | PERF_MSR_SMI = 4, |
14 | PERF_MSR_PTSC = 5, |
15 | PERF_MSR_IRPERF = 6, |
16 | PERF_MSR_THERM = 7, |
17 | PERF_MSR_EVENT_MAX, |
18 | }; |
19 | |
20 | static bool test_aperfmperf(int idx, void *data) |
21 | { |
22 | return boot_cpu_has(X86_FEATURE_APERFMPERF); |
23 | } |
24 | |
25 | static bool test_ptsc(int idx, void *data) |
26 | { |
27 | return boot_cpu_has(X86_FEATURE_PTSC); |
28 | } |
29 | |
30 | static bool test_irperf(int idx, void *data) |
31 | { |
32 | return boot_cpu_has(X86_FEATURE_IRPERF); |
33 | } |
34 | |
35 | static bool test_therm_status(int idx, void *data) |
36 | { |
37 | return boot_cpu_has(X86_FEATURE_DTHERM); |
38 | } |
39 | |
40 | static bool test_intel(int idx, void *data) |
41 | { |
42 | if (boot_cpu_data.x86_vendor != X86_VENDOR_INTEL || |
43 | boot_cpu_data.x86 != 6) |
44 | return false; |
45 | |
46 | switch (boot_cpu_data.x86_model) { |
47 | case INTEL_FAM6_NEHALEM: |
48 | case INTEL_FAM6_NEHALEM_G: |
49 | case INTEL_FAM6_NEHALEM_EP: |
50 | case INTEL_FAM6_NEHALEM_EX: |
51 | |
52 | case INTEL_FAM6_WESTMERE: |
53 | case INTEL_FAM6_WESTMERE_EP: |
54 | case INTEL_FAM6_WESTMERE_EX: |
55 | |
56 | case INTEL_FAM6_SANDYBRIDGE: |
57 | case INTEL_FAM6_SANDYBRIDGE_X: |
58 | |
59 | case INTEL_FAM6_IVYBRIDGE: |
60 | case INTEL_FAM6_IVYBRIDGE_X: |
61 | |
62 | case INTEL_FAM6_HASWELL: |
63 | case INTEL_FAM6_HASWELL_X: |
64 | case INTEL_FAM6_HASWELL_L: |
65 | case INTEL_FAM6_HASWELL_G: |
66 | |
67 | case INTEL_FAM6_BROADWELL: |
68 | case INTEL_FAM6_BROADWELL_D: |
69 | case INTEL_FAM6_BROADWELL_G: |
70 | case INTEL_FAM6_BROADWELL_X: |
71 | case INTEL_FAM6_SAPPHIRERAPIDS_X: |
72 | case INTEL_FAM6_EMERALDRAPIDS_X: |
73 | case INTEL_FAM6_GRANITERAPIDS_X: |
74 | case INTEL_FAM6_GRANITERAPIDS_D: |
75 | |
76 | case INTEL_FAM6_ATOM_SILVERMONT: |
77 | case INTEL_FAM6_ATOM_SILVERMONT_D: |
78 | case INTEL_FAM6_ATOM_AIRMONT: |
79 | |
80 | case INTEL_FAM6_ATOM_GOLDMONT: |
81 | case INTEL_FAM6_ATOM_GOLDMONT_D: |
82 | case INTEL_FAM6_ATOM_GOLDMONT_PLUS: |
83 | case INTEL_FAM6_ATOM_TREMONT_D: |
84 | case INTEL_FAM6_ATOM_TREMONT: |
85 | case INTEL_FAM6_ATOM_TREMONT_L: |
86 | |
87 | case INTEL_FAM6_XEON_PHI_KNL: |
88 | case INTEL_FAM6_XEON_PHI_KNM: |
89 | if (idx == PERF_MSR_SMI) |
90 | return true; |
91 | break; |
92 | |
93 | case INTEL_FAM6_SKYLAKE_L: |
94 | case INTEL_FAM6_SKYLAKE: |
95 | case INTEL_FAM6_SKYLAKE_X: |
96 | case INTEL_FAM6_KABYLAKE_L: |
97 | case INTEL_FAM6_KABYLAKE: |
98 | case INTEL_FAM6_COMETLAKE_L: |
99 | case INTEL_FAM6_COMETLAKE: |
100 | case INTEL_FAM6_ICELAKE_L: |
101 | case INTEL_FAM6_ICELAKE: |
102 | case INTEL_FAM6_ICELAKE_X: |
103 | case INTEL_FAM6_ICELAKE_D: |
104 | case INTEL_FAM6_TIGERLAKE_L: |
105 | case INTEL_FAM6_TIGERLAKE: |
106 | case INTEL_FAM6_ROCKETLAKE: |
107 | case INTEL_FAM6_ALDERLAKE: |
108 | case INTEL_FAM6_ALDERLAKE_L: |
109 | case INTEL_FAM6_ATOM_GRACEMONT: |
110 | case INTEL_FAM6_RAPTORLAKE: |
111 | case INTEL_FAM6_RAPTORLAKE_P: |
112 | case INTEL_FAM6_RAPTORLAKE_S: |
113 | case INTEL_FAM6_METEORLAKE: |
114 | case INTEL_FAM6_METEORLAKE_L: |
115 | if (idx == PERF_MSR_SMI || idx == PERF_MSR_PPERF) |
116 | return true; |
117 | break; |
118 | } |
119 | |
120 | return false; |
121 | } |
122 | |
123 | PMU_EVENT_ATTR_STRING(tsc, attr_tsc, "event=0x00" ); |
124 | PMU_EVENT_ATTR_STRING(aperf, attr_aperf, "event=0x01" ); |
125 | PMU_EVENT_ATTR_STRING(mperf, attr_mperf, "event=0x02" ); |
126 | PMU_EVENT_ATTR_STRING(pperf, attr_pperf, "event=0x03" ); |
127 | PMU_EVENT_ATTR_STRING(smi, attr_smi, "event=0x04" ); |
128 | PMU_EVENT_ATTR_STRING(ptsc, attr_ptsc, "event=0x05" ); |
129 | PMU_EVENT_ATTR_STRING(irperf, attr_irperf, "event=0x06" ); |
130 | PMU_EVENT_ATTR_STRING(cpu_thermal_margin, attr_therm, "event=0x07" ); |
131 | PMU_EVENT_ATTR_STRING(cpu_thermal_margin.snapshot, attr_therm_snap, "1" ); |
132 | PMU_EVENT_ATTR_STRING(cpu_thermal_margin.unit, attr_therm_unit, "C" ); |
133 | |
134 | static unsigned long msr_mask; |
135 | |
136 | PMU_EVENT_GROUP(events, aperf); |
137 | PMU_EVENT_GROUP(events, mperf); |
138 | PMU_EVENT_GROUP(events, pperf); |
139 | PMU_EVENT_GROUP(events, smi); |
140 | PMU_EVENT_GROUP(events, ptsc); |
141 | PMU_EVENT_GROUP(events, irperf); |
142 | |
143 | static struct attribute *attrs_therm[] = { |
144 | &attr_therm.attr.attr, |
145 | &attr_therm_snap.attr.attr, |
146 | &attr_therm_unit.attr.attr, |
147 | NULL, |
148 | }; |
149 | |
150 | static struct attribute_group group_therm = { |
151 | .name = "events" , |
152 | .attrs = attrs_therm, |
153 | }; |
154 | |
155 | static struct perf_msr msr[] = { |
156 | [PERF_MSR_TSC] = { .no_check = true, }, |
157 | [PERF_MSR_APERF] = { MSR_IA32_APERF, .grp: &group_aperf, .test: test_aperfmperf, }, |
158 | [PERF_MSR_MPERF] = { MSR_IA32_MPERF, .grp: &group_mperf, .test: test_aperfmperf, }, |
159 | [PERF_MSR_PPERF] = { MSR_PPERF, .grp: &group_pperf, .test: test_intel, }, |
160 | [PERF_MSR_SMI] = { MSR_SMI_COUNT, .grp: &group_smi, .test: test_intel, }, |
161 | [PERF_MSR_PTSC] = { MSR_F15H_PTSC, .grp: &group_ptsc, .test: test_ptsc, }, |
162 | [PERF_MSR_IRPERF] = { MSR_F17H_IRPERF, .grp: &group_irperf, .test: test_irperf, }, |
163 | [PERF_MSR_THERM] = { MSR_IA32_THERM_STATUS, .grp: &group_therm, .test: test_therm_status, }, |
164 | }; |
165 | |
166 | static struct attribute *events_attrs[] = { |
167 | &attr_tsc.attr.attr, |
168 | NULL, |
169 | }; |
170 | |
171 | static struct attribute_group events_attr_group = { |
172 | .name = "events" , |
173 | .attrs = events_attrs, |
174 | }; |
175 | |
176 | PMU_FORMAT_ATTR(event, "config:0-63" ); |
177 | static struct attribute *format_attrs[] = { |
178 | &format_attr_event.attr, |
179 | NULL, |
180 | }; |
181 | static struct attribute_group format_attr_group = { |
182 | .name = "format" , |
183 | .attrs = format_attrs, |
184 | }; |
185 | |
186 | static const struct attribute_group *attr_groups[] = { |
187 | &events_attr_group, |
188 | &format_attr_group, |
189 | NULL, |
190 | }; |
191 | |
192 | static const struct attribute_group *attr_update[] = { |
193 | &group_aperf, |
194 | &group_mperf, |
195 | &group_pperf, |
196 | &group_smi, |
197 | &group_ptsc, |
198 | &group_irperf, |
199 | &group_therm, |
200 | NULL, |
201 | }; |
202 | |
203 | static int msr_event_init(struct perf_event *event) |
204 | { |
205 | u64 cfg = event->attr.config; |
206 | |
207 | if (event->attr.type != event->pmu->type) |
208 | return -ENOENT; |
209 | |
210 | /* unsupported modes and filters */ |
211 | if (event->attr.sample_period) /* no sampling */ |
212 | return -EINVAL; |
213 | |
214 | if (cfg >= PERF_MSR_EVENT_MAX) |
215 | return -EINVAL; |
216 | |
217 | cfg = array_index_nospec((unsigned long)cfg, PERF_MSR_EVENT_MAX); |
218 | |
219 | if (!(msr_mask & (1 << cfg))) |
220 | return -EINVAL; |
221 | |
222 | event->hw.idx = -1; |
223 | event->hw.event_base = msr[cfg].msr; |
224 | event->hw.config = cfg; |
225 | |
226 | return 0; |
227 | } |
228 | |
229 | static inline u64 msr_read_counter(struct perf_event *event) |
230 | { |
231 | u64 now; |
232 | |
233 | if (event->hw.event_base) |
234 | rdmsrl(event->hw.event_base, now); |
235 | else |
236 | now = rdtsc_ordered(); |
237 | |
238 | return now; |
239 | } |
240 | |
241 | static void msr_event_update(struct perf_event *event) |
242 | { |
243 | u64 prev, now; |
244 | s64 delta; |
245 | |
246 | /* Careful, an NMI might modify the previous event value: */ |
247 | prev = local64_read(&event->hw.prev_count); |
248 | do { |
249 | now = msr_read_counter(event); |
250 | } while (!local64_try_cmpxchg(l: &event->hw.prev_count, old: &prev, new: now)); |
251 | |
252 | delta = now - prev; |
253 | if (unlikely(event->hw.event_base == MSR_SMI_COUNT)) { |
254 | delta = sign_extend64(value: delta, index: 31); |
255 | local64_add(delta, &event->count); |
256 | } else if (unlikely(event->hw.event_base == MSR_IA32_THERM_STATUS)) { |
257 | /* If valid, extract digital readout, otherwise set to -1: */ |
258 | now = now & (1ULL << 31) ? (now >> 16) & 0x3f : -1; |
259 | local64_set(&event->count, now); |
260 | } else { |
261 | local64_add(delta, &event->count); |
262 | } |
263 | } |
264 | |
265 | static void msr_event_start(struct perf_event *event, int flags) |
266 | { |
267 | u64 now = msr_read_counter(event); |
268 | |
269 | local64_set(&event->hw.prev_count, now); |
270 | } |
271 | |
272 | static void msr_event_stop(struct perf_event *event, int flags) |
273 | { |
274 | msr_event_update(event); |
275 | } |
276 | |
277 | static void msr_event_del(struct perf_event *event, int flags) |
278 | { |
279 | msr_event_stop(event, PERF_EF_UPDATE); |
280 | } |
281 | |
282 | static int msr_event_add(struct perf_event *event, int flags) |
283 | { |
284 | if (flags & PERF_EF_START) |
285 | msr_event_start(event, flags); |
286 | |
287 | return 0; |
288 | } |
289 | |
290 | static struct pmu pmu_msr = { |
291 | .task_ctx_nr = perf_sw_context, |
292 | .attr_groups = attr_groups, |
293 | .event_init = msr_event_init, |
294 | .add = msr_event_add, |
295 | .del = msr_event_del, |
296 | .start = msr_event_start, |
297 | .stop = msr_event_stop, |
298 | .read = msr_event_update, |
299 | .capabilities = PERF_PMU_CAP_NO_INTERRUPT | PERF_PMU_CAP_NO_EXCLUDE, |
300 | .attr_update = attr_update, |
301 | }; |
302 | |
303 | static int __init msr_init(void) |
304 | { |
305 | if (!boot_cpu_has(X86_FEATURE_TSC)) { |
306 | pr_cont("no MSR PMU driver.\n" ); |
307 | return 0; |
308 | } |
309 | |
310 | msr_mask = perf_msr_probe(msr, cnt: PERF_MSR_EVENT_MAX, no_zero: true, NULL); |
311 | |
312 | perf_pmu_register(pmu: &pmu_msr, name: "msr" , type: -1); |
313 | |
314 | return 0; |
315 | } |
316 | device_initcall(msr_init); |
317 | |