1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Synopsys DesignWare PCIe PMU driver |
4 | * |
5 | * Copyright (C) 2021-2023 Alibaba Inc. |
6 | */ |
7 | |
8 | #include <linux/bitfield.h> |
9 | #include <linux/bitops.h> |
10 | #include <linux/cpuhotplug.h> |
11 | #include <linux/cpumask.h> |
12 | #include <linux/device.h> |
13 | #include <linux/errno.h> |
14 | #include <linux/kernel.h> |
15 | #include <linux/list.h> |
16 | #include <linux/perf_event.h> |
17 | #include <linux/pci.h> |
18 | #include <linux/platform_device.h> |
19 | #include <linux/smp.h> |
20 | #include <linux/sysfs.h> |
21 | #include <linux/types.h> |
22 | |
23 | #define DWC_PCIE_VSEC_RAS_DES_ID 0x02 |
24 | #define DWC_PCIE_EVENT_CNT_CTL 0x8 |
25 | |
26 | /* |
27 | * Event Counter Data Select includes two parts: |
28 | * - 27-24: Group number(4-bit: 0..0x7) |
29 | * - 23-16: Event number(8-bit: 0..0x13) within the Group |
30 | * |
31 | * Put them together as in TRM. |
32 | */ |
33 | #define DWC_PCIE_CNT_EVENT_SEL GENMASK(27, 16) |
34 | #define DWC_PCIE_CNT_LANE_SEL GENMASK(11, 8) |
35 | #define DWC_PCIE_CNT_STATUS BIT(7) |
36 | #define DWC_PCIE_CNT_ENABLE GENMASK(4, 2) |
37 | #define DWC_PCIE_PER_EVENT_OFF 0x1 |
38 | #define DWC_PCIE_PER_EVENT_ON 0x3 |
39 | #define DWC_PCIE_EVENT_CLEAR GENMASK(1, 0) |
40 | #define DWC_PCIE_EVENT_PER_CLEAR 0x1 |
41 | |
42 | #define DWC_PCIE_EVENT_CNT_DATA 0xC |
43 | |
44 | #define DWC_PCIE_TIME_BASED_ANAL_CTL 0x10 |
45 | #define DWC_PCIE_TIME_BASED_REPORT_SEL GENMASK(31, 24) |
46 | #define DWC_PCIE_TIME_BASED_DURATION_SEL GENMASK(15, 8) |
47 | #define DWC_PCIE_DURATION_MANUAL_CTL 0x0 |
48 | #define DWC_PCIE_DURATION_1MS 0x1 |
49 | #define DWC_PCIE_DURATION_10MS 0x2 |
50 | #define DWC_PCIE_DURATION_100MS 0x3 |
51 | #define DWC_PCIE_DURATION_1S 0x4 |
52 | #define DWC_PCIE_DURATION_2S 0x5 |
53 | #define DWC_PCIE_DURATION_4S 0x6 |
54 | #define DWC_PCIE_DURATION_4US 0xFF |
55 | #define DWC_PCIE_TIME_BASED_TIMER_START BIT(0) |
56 | #define DWC_PCIE_TIME_BASED_CNT_ENABLE 0x1 |
57 | |
58 | #define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW 0x14 |
59 | #define DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH 0x18 |
60 | |
61 | /* Event attributes */ |
62 | #define DWC_PCIE_CONFIG_EVENTID GENMASK(15, 0) |
63 | #define DWC_PCIE_CONFIG_TYPE GENMASK(19, 16) |
64 | #define DWC_PCIE_CONFIG_LANE GENMASK(27, 20) |
65 | |
66 | #define DWC_PCIE_EVENT_ID(event) FIELD_GET(DWC_PCIE_CONFIG_EVENTID, (event)->attr.config) |
67 | #define DWC_PCIE_EVENT_TYPE(event) FIELD_GET(DWC_PCIE_CONFIG_TYPE, (event)->attr.config) |
68 | #define DWC_PCIE_EVENT_LANE(event) FIELD_GET(DWC_PCIE_CONFIG_LANE, (event)->attr.config) |
69 | |
70 | enum dwc_pcie_event_type { |
71 | DWC_PCIE_TIME_BASE_EVENT, |
72 | DWC_PCIE_LANE_EVENT, |
73 | DWC_PCIE_EVENT_TYPE_MAX, |
74 | }; |
75 | |
76 | #define DWC_PCIE_LANE_EVENT_MAX_PERIOD GENMASK_ULL(31, 0) |
77 | #define DWC_PCIE_MAX_PERIOD GENMASK_ULL(63, 0) |
78 | |
79 | struct dwc_pcie_pmu { |
80 | struct pmu pmu; |
81 | struct pci_dev *pdev; /* Root Port device */ |
82 | u16 ras_des_offset; |
83 | u32 nr_lanes; |
84 | |
85 | struct list_head pmu_node; |
86 | struct hlist_node cpuhp_node; |
87 | struct perf_event *event[DWC_PCIE_EVENT_TYPE_MAX]; |
88 | int on_cpu; |
89 | }; |
90 | |
91 | #define to_dwc_pcie_pmu(p) (container_of(p, struct dwc_pcie_pmu, pmu)) |
92 | |
93 | static int dwc_pcie_pmu_hp_state; |
94 | static struct list_head dwc_pcie_dev_info_head = |
95 | LIST_HEAD_INIT(dwc_pcie_dev_info_head); |
96 | static bool notify; |
97 | |
98 | struct dwc_pcie_dev_info { |
99 | struct platform_device *plat_dev; |
100 | struct pci_dev *pdev; |
101 | struct list_head dev_node; |
102 | }; |
103 | |
104 | struct dwc_pcie_vendor_id { |
105 | int vendor_id; |
106 | }; |
107 | |
108 | static const struct dwc_pcie_vendor_id dwc_pcie_vendor_ids[] = { |
109 | {.vendor_id = PCI_VENDOR_ID_ALIBABA }, |
110 | {} /* terminator */ |
111 | }; |
112 | |
113 | static ssize_t cpumask_show(struct device *dev, |
114 | struct device_attribute *attr, |
115 | char *buf) |
116 | { |
117 | struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(dev_get_drvdata(dev)); |
118 | |
119 | return cpumap_print_to_pagebuf(list: true, buf, cpumask_of(pcie_pmu->on_cpu)); |
120 | } |
121 | static DEVICE_ATTR_RO(cpumask); |
122 | |
123 | static struct attribute *dwc_pcie_pmu_cpumask_attrs[] = { |
124 | &dev_attr_cpumask.attr, |
125 | NULL |
126 | }; |
127 | |
128 | static struct attribute_group dwc_pcie_cpumask_attr_group = { |
129 | .attrs = dwc_pcie_pmu_cpumask_attrs, |
130 | }; |
131 | |
132 | struct dwc_pcie_format_attr { |
133 | struct device_attribute attr; |
134 | u64 field; |
135 | int config; |
136 | }; |
137 | |
138 | PMU_FORMAT_ATTR(eventid, "config:0-15" ); |
139 | PMU_FORMAT_ATTR(type, "config:16-19" ); |
140 | PMU_FORMAT_ATTR(lane, "config:20-27" ); |
141 | |
142 | static struct attribute *dwc_pcie_format_attrs[] = { |
143 | &format_attr_type.attr, |
144 | &format_attr_eventid.attr, |
145 | &format_attr_lane.attr, |
146 | NULL, |
147 | }; |
148 | |
149 | static struct attribute_group dwc_pcie_format_attrs_group = { |
150 | .name = "format" , |
151 | .attrs = dwc_pcie_format_attrs, |
152 | }; |
153 | |
154 | struct dwc_pcie_event_attr { |
155 | struct device_attribute attr; |
156 | enum dwc_pcie_event_type type; |
157 | u16 eventid; |
158 | u8 lane; |
159 | }; |
160 | |
161 | static ssize_t dwc_pcie_event_show(struct device *dev, |
162 | struct device_attribute *attr, char *buf) |
163 | { |
164 | struct dwc_pcie_event_attr *eattr; |
165 | |
166 | eattr = container_of(attr, typeof(*eattr), attr); |
167 | |
168 | if (eattr->type == DWC_PCIE_LANE_EVENT) |
169 | return sysfs_emit(buf, fmt: "eventid=0x%x,type=0x%x,lane=?\n" , |
170 | eattr->eventid, eattr->type); |
171 | else if (eattr->type == DWC_PCIE_TIME_BASE_EVENT) |
172 | return sysfs_emit(buf, fmt: "eventid=0x%x,type=0x%x\n" , |
173 | eattr->eventid, eattr->type); |
174 | |
175 | return 0; |
176 | } |
177 | |
178 | #define DWC_PCIE_EVENT_ATTR(_name, _type, _eventid, _lane) \ |
179 | (&((struct dwc_pcie_event_attr[]) {{ \ |
180 | .attr = __ATTR(_name, 0444, dwc_pcie_event_show, NULL), \ |
181 | .type = _type, \ |
182 | .eventid = _eventid, \ |
183 | .lane = _lane, \ |
184 | }})[0].attr.attr) |
185 | |
186 | #define DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(_name, _eventid) \ |
187 | DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_TIME_BASE_EVENT, _eventid, 0) |
188 | #define DWC_PCIE_PMU_LANE_EVENT_ATTR(_name, _eventid) \ |
189 | DWC_PCIE_EVENT_ATTR(_name, DWC_PCIE_LANE_EVENT, _eventid, 0) |
190 | |
191 | static struct attribute *dwc_pcie_pmu_time_event_attrs[] = { |
192 | /* Group #0 */ |
193 | DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(one_cycle, 0x00), |
194 | DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_L0S, 0x01), |
195 | DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(RX_L0S, 0x02), |
196 | DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L0, 0x03), |
197 | DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1, 0x04), |
198 | DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_1, 0x05), |
199 | DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_2, 0x06), |
200 | DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(CFG_RCVRY, 0x07), |
201 | DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(TX_RX_L0S, 0x08), |
202 | DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(L1_AUX, 0x09), |
203 | |
204 | /* Group #1 */ |
205 | DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(Tx_PCIe_TLP_Data_Payload, 0x20), |
206 | DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(Rx_PCIe_TLP_Data_Payload, 0x21), |
207 | DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(Tx_CCIX_TLP_Data_Payload, 0x22), |
208 | DWC_PCIE_PMU_TIME_BASE_EVENT_ATTR(Rx_CCIX_TLP_Data_Payload, 0x23), |
209 | |
210 | /* |
211 | * Leave it to the user to specify the lane ID to avoid generating |
212 | * a list of hundreds of events. |
213 | */ |
214 | DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ack_dllp, 0x600), |
215 | DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_update_fc_dllp, 0x601), |
216 | DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ack_dllp, 0x602), |
217 | DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_update_fc_dllp, 0x603), |
218 | DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_nulified_tlp, 0x604), |
219 | DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_nulified_tlp, 0x605), |
220 | DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_duplicate_tl, 0x606), |
221 | DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_write, 0x700), |
222 | DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_memory_read, 0x701), |
223 | DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_write, 0x702), |
224 | DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_configuration_read, 0x703), |
225 | DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_write, 0x704), |
226 | DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_io_read, 0x705), |
227 | DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_without_data, 0x706), |
228 | DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_completion_with_data, 0x707), |
229 | DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_message_tlp, 0x708), |
230 | DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_atomic, 0x709), |
231 | DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_tlp_with_prefix, 0x70A), |
232 | DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_write, 0x70B), |
233 | DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_memory_read, 0x70C), |
234 | DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_write, 0x70F), |
235 | DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_io_read, 0x710), |
236 | DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_without_data, 0x711), |
237 | DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_completion_with_data, 0x712), |
238 | DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_message_tlp, 0x713), |
239 | DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_atomic, 0x714), |
240 | DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_tlp_with_prefix, 0x715), |
241 | DWC_PCIE_PMU_LANE_EVENT_ATTR(tx_ccix_tlp, 0x716), |
242 | DWC_PCIE_PMU_LANE_EVENT_ATTR(rx_ccix_tlp, 0x717), |
243 | NULL |
244 | }; |
245 | |
246 | static const struct attribute_group dwc_pcie_event_attrs_group = { |
247 | .name = "events" , |
248 | .attrs = dwc_pcie_pmu_time_event_attrs, |
249 | }; |
250 | |
251 | static const struct attribute_group *dwc_pcie_attr_groups[] = { |
252 | &dwc_pcie_event_attrs_group, |
253 | &dwc_pcie_format_attrs_group, |
254 | &dwc_pcie_cpumask_attr_group, |
255 | NULL |
256 | }; |
257 | |
258 | static void dwc_pcie_pmu_lane_event_enable(struct dwc_pcie_pmu *pcie_pmu, |
259 | bool enable) |
260 | { |
261 | struct pci_dev *pdev = pcie_pmu->pdev; |
262 | u16 ras_des_offset = pcie_pmu->ras_des_offset; |
263 | |
264 | if (enable) |
265 | pci_clear_and_set_config_dword(dev: pdev, |
266 | pos: ras_des_offset + DWC_PCIE_EVENT_CNT_CTL, |
267 | DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_ON); |
268 | else |
269 | pci_clear_and_set_config_dword(dev: pdev, |
270 | pos: ras_des_offset + DWC_PCIE_EVENT_CNT_CTL, |
271 | DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF); |
272 | } |
273 | |
274 | static void dwc_pcie_pmu_time_based_event_enable(struct dwc_pcie_pmu *pcie_pmu, |
275 | bool enable) |
276 | { |
277 | struct pci_dev *pdev = pcie_pmu->pdev; |
278 | u16 ras_des_offset = pcie_pmu->ras_des_offset; |
279 | |
280 | pci_clear_and_set_config_dword(dev: pdev, |
281 | pos: ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL, |
282 | DWC_PCIE_TIME_BASED_TIMER_START, set: enable); |
283 | } |
284 | |
285 | static u64 dwc_pcie_pmu_read_lane_event_counter(struct perf_event *event) |
286 | { |
287 | struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); |
288 | struct pci_dev *pdev = pcie_pmu->pdev; |
289 | u16 ras_des_offset = pcie_pmu->ras_des_offset; |
290 | u32 val; |
291 | |
292 | pci_read_config_dword(dev: pdev, where: ras_des_offset + DWC_PCIE_EVENT_CNT_DATA, val: &val); |
293 | |
294 | return val; |
295 | } |
296 | |
297 | static u64 dwc_pcie_pmu_read_time_based_counter(struct perf_event *event) |
298 | { |
299 | struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); |
300 | struct pci_dev *pdev = pcie_pmu->pdev; |
301 | int event_id = DWC_PCIE_EVENT_ID(event); |
302 | u16 ras_des_offset = pcie_pmu->ras_des_offset; |
303 | u32 lo, hi, ss; |
304 | u64 val; |
305 | |
306 | /* |
307 | * The 64-bit value of the data counter is spread across two |
308 | * registers that are not synchronized. In order to read them |
309 | * atomically, ensure that the high 32 bits match before and after |
310 | * reading the low 32 bits. |
311 | */ |
312 | pci_read_config_dword(dev: pdev, |
313 | where: ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH, val: &hi); |
314 | do { |
315 | /* snapshot the high 32 bits */ |
316 | ss = hi; |
317 | |
318 | pci_read_config_dword( |
319 | dev: pdev, where: ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_LOW, |
320 | val: &lo); |
321 | pci_read_config_dword( |
322 | dev: pdev, where: ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_DATA_REG_HIGH, |
323 | val: &hi); |
324 | } while (hi != ss); |
325 | |
326 | val = ((u64)hi << 32) | lo; |
327 | /* |
328 | * The Group#1 event measures the amount of data processed in 16-byte |
329 | * units. Simplify the end-user interface by multiplying the counter |
330 | * at the point of read. |
331 | */ |
332 | if (event_id >= 0x20 && event_id <= 0x23) |
333 | val *= 16; |
334 | |
335 | return val; |
336 | } |
337 | |
338 | static void dwc_pcie_pmu_event_update(struct perf_event *event) |
339 | { |
340 | struct hw_perf_event *hwc = &event->hw; |
341 | enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); |
342 | u64 delta, prev, now = 0; |
343 | |
344 | do { |
345 | prev = local64_read(&hwc->prev_count); |
346 | |
347 | if (type == DWC_PCIE_LANE_EVENT) |
348 | now = dwc_pcie_pmu_read_lane_event_counter(event); |
349 | else if (type == DWC_PCIE_TIME_BASE_EVENT) |
350 | now = dwc_pcie_pmu_read_time_based_counter(event); |
351 | |
352 | } while (local64_cmpxchg(l: &hwc->prev_count, old: prev, new: now) != prev); |
353 | |
354 | delta = (now - prev) & DWC_PCIE_MAX_PERIOD; |
355 | /* 32-bit counter for Lane Event Counting */ |
356 | if (type == DWC_PCIE_LANE_EVENT) |
357 | delta &= DWC_PCIE_LANE_EVENT_MAX_PERIOD; |
358 | |
359 | local64_add(delta, &event->count); |
360 | } |
361 | |
362 | static int dwc_pcie_pmu_event_init(struct perf_event *event) |
363 | { |
364 | struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); |
365 | enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); |
366 | struct perf_event *sibling; |
367 | u32 lane; |
368 | |
369 | if (event->attr.type != event->pmu->type) |
370 | return -ENOENT; |
371 | |
372 | /* We don't support sampling */ |
373 | if (is_sampling_event(event)) |
374 | return -EINVAL; |
375 | |
376 | /* We cannot support task bound events */ |
377 | if (event->cpu < 0 || event->attach_state & PERF_ATTACH_TASK) |
378 | return -EINVAL; |
379 | |
380 | if (event->group_leader != event && |
381 | !is_software_event(event: event->group_leader)) |
382 | return -EINVAL; |
383 | |
384 | for_each_sibling_event(sibling, event->group_leader) { |
385 | if (sibling->pmu != event->pmu && !is_software_event(event: sibling)) |
386 | return -EINVAL; |
387 | } |
388 | |
389 | if (type < 0 || type >= DWC_PCIE_EVENT_TYPE_MAX) |
390 | return -EINVAL; |
391 | |
392 | if (type == DWC_PCIE_LANE_EVENT) { |
393 | lane = DWC_PCIE_EVENT_LANE(event); |
394 | if (lane < 0 || lane >= pcie_pmu->nr_lanes) |
395 | return -EINVAL; |
396 | } |
397 | |
398 | event->cpu = pcie_pmu->on_cpu; |
399 | |
400 | return 0; |
401 | } |
402 | |
403 | static void dwc_pcie_pmu_event_start(struct perf_event *event, int flags) |
404 | { |
405 | struct hw_perf_event *hwc = &event->hw; |
406 | struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); |
407 | enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); |
408 | |
409 | hwc->state = 0; |
410 | local64_set(&hwc->prev_count, 0); |
411 | |
412 | if (type == DWC_PCIE_LANE_EVENT) |
413 | dwc_pcie_pmu_lane_event_enable(pcie_pmu, enable: true); |
414 | else if (type == DWC_PCIE_TIME_BASE_EVENT) |
415 | dwc_pcie_pmu_time_based_event_enable(pcie_pmu, enable: true); |
416 | } |
417 | |
418 | static void dwc_pcie_pmu_event_stop(struct perf_event *event, int flags) |
419 | { |
420 | struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); |
421 | enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); |
422 | struct hw_perf_event *hwc = &event->hw; |
423 | |
424 | if (event->hw.state & PERF_HES_STOPPED) |
425 | return; |
426 | |
427 | if (type == DWC_PCIE_LANE_EVENT) |
428 | dwc_pcie_pmu_lane_event_enable(pcie_pmu, enable: false); |
429 | else if (type == DWC_PCIE_TIME_BASE_EVENT) |
430 | dwc_pcie_pmu_time_based_event_enable(pcie_pmu, enable: false); |
431 | |
432 | dwc_pcie_pmu_event_update(event); |
433 | hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; |
434 | } |
435 | |
436 | static int dwc_pcie_pmu_event_add(struct perf_event *event, int flags) |
437 | { |
438 | struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); |
439 | struct pci_dev *pdev = pcie_pmu->pdev; |
440 | struct hw_perf_event *hwc = &event->hw; |
441 | enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); |
442 | int event_id = DWC_PCIE_EVENT_ID(event); |
443 | int lane = DWC_PCIE_EVENT_LANE(event); |
444 | u16 ras_des_offset = pcie_pmu->ras_des_offset; |
445 | u32 ctrl; |
446 | |
447 | /* one counter for each type and it is in use */ |
448 | if (pcie_pmu->event[type]) |
449 | return -ENOSPC; |
450 | |
451 | pcie_pmu->event[type] = event; |
452 | hwc->state = PERF_HES_STOPPED | PERF_HES_UPTODATE; |
453 | |
454 | if (type == DWC_PCIE_LANE_EVENT) { |
455 | /* EVENT_COUNTER_DATA_REG needs clear manually */ |
456 | ctrl = FIELD_PREP(DWC_PCIE_CNT_EVENT_SEL, event_id) | |
457 | FIELD_PREP(DWC_PCIE_CNT_LANE_SEL, lane) | |
458 | FIELD_PREP(DWC_PCIE_CNT_ENABLE, DWC_PCIE_PER_EVENT_OFF) | |
459 | FIELD_PREP(DWC_PCIE_EVENT_CLEAR, DWC_PCIE_EVENT_PER_CLEAR); |
460 | pci_write_config_dword(dev: pdev, where: ras_des_offset + DWC_PCIE_EVENT_CNT_CTL, |
461 | val: ctrl); |
462 | } else if (type == DWC_PCIE_TIME_BASE_EVENT) { |
463 | /* |
464 | * TIME_BASED_ANAL_DATA_REG is a 64 bit register, we can safely |
465 | * use it with any manually controlled duration. And it is |
466 | * cleared when next measurement starts. |
467 | */ |
468 | ctrl = FIELD_PREP(DWC_PCIE_TIME_BASED_REPORT_SEL, event_id) | |
469 | FIELD_PREP(DWC_PCIE_TIME_BASED_DURATION_SEL, |
470 | DWC_PCIE_DURATION_MANUAL_CTL) | |
471 | DWC_PCIE_TIME_BASED_CNT_ENABLE; |
472 | pci_write_config_dword( |
473 | dev: pdev, where: ras_des_offset + DWC_PCIE_TIME_BASED_ANAL_CTL, val: ctrl); |
474 | } |
475 | |
476 | if (flags & PERF_EF_START) |
477 | dwc_pcie_pmu_event_start(event, PERF_EF_RELOAD); |
478 | |
479 | perf_event_update_userpage(event); |
480 | |
481 | return 0; |
482 | } |
483 | |
484 | static void dwc_pcie_pmu_event_del(struct perf_event *event, int flags) |
485 | { |
486 | struct dwc_pcie_pmu *pcie_pmu = to_dwc_pcie_pmu(event->pmu); |
487 | enum dwc_pcie_event_type type = DWC_PCIE_EVENT_TYPE(event); |
488 | |
489 | dwc_pcie_pmu_event_stop(event, flags: flags | PERF_EF_UPDATE); |
490 | perf_event_update_userpage(event); |
491 | pcie_pmu->event[type] = NULL; |
492 | } |
493 | |
494 | static void dwc_pcie_pmu_remove_cpuhp_instance(void *hotplug_node) |
495 | { |
496 | cpuhp_state_remove_instance_nocalls(state: dwc_pcie_pmu_hp_state, node: hotplug_node); |
497 | } |
498 | |
499 | /* |
500 | * Find the binded DES capability device info of a PCI device. |
501 | * @pdev: The PCI device. |
502 | */ |
503 | static struct dwc_pcie_dev_info *dwc_pcie_find_dev_info(struct pci_dev *pdev) |
504 | { |
505 | struct dwc_pcie_dev_info *dev_info; |
506 | |
507 | list_for_each_entry(dev_info, &dwc_pcie_dev_info_head, dev_node) |
508 | if (dev_info->pdev == pdev) |
509 | return dev_info; |
510 | |
511 | return NULL; |
512 | } |
513 | |
514 | static void dwc_pcie_unregister_pmu(void *data) |
515 | { |
516 | struct dwc_pcie_pmu *pcie_pmu = data; |
517 | |
518 | perf_pmu_unregister(pmu: &pcie_pmu->pmu); |
519 | } |
520 | |
521 | static bool dwc_pcie_match_des_cap(struct pci_dev *pdev) |
522 | { |
523 | const struct dwc_pcie_vendor_id *vid; |
524 | u16 vsec = 0; |
525 | u32 val; |
526 | |
527 | if (!pci_is_pcie(dev: pdev) || !(pci_pcie_type(dev: pdev) == PCI_EXP_TYPE_ROOT_PORT)) |
528 | return false; |
529 | |
530 | for (vid = dwc_pcie_vendor_ids; vid->vendor_id; vid++) { |
531 | vsec = pci_find_vsec_capability(dev: pdev, vendor: vid->vendor_id, |
532 | DWC_PCIE_VSEC_RAS_DES_ID); |
533 | if (vsec) |
534 | break; |
535 | } |
536 | if (!vsec) |
537 | return false; |
538 | |
539 | pci_read_config_dword(dev: pdev, where: vsec + PCI_VNDR_HEADER, val: &val); |
540 | if (PCI_VNDR_HEADER_REV(val) != 0x04) |
541 | return false; |
542 | |
543 | pci_dbg(pdev, |
544 | "Detected PCIe Vendor-Specific Extended Capability RAS DES\n" ); |
545 | return true; |
546 | } |
547 | |
548 | static void dwc_pcie_unregister_dev(struct dwc_pcie_dev_info *dev_info) |
549 | { |
550 | platform_device_unregister(dev_info->plat_dev); |
551 | list_del(entry: &dev_info->dev_node); |
552 | kfree(objp: dev_info); |
553 | } |
554 | |
555 | static int dwc_pcie_register_dev(struct pci_dev *pdev) |
556 | { |
557 | struct platform_device *plat_dev; |
558 | struct dwc_pcie_dev_info *dev_info; |
559 | u32 bdf; |
560 | |
561 | bdf = PCI_DEVID(pdev->bus->number, pdev->devfn); |
562 | plat_dev = platform_device_register_data(NULL, name: "dwc_pcie_pmu" , id: bdf, |
563 | data: pdev, size: sizeof(*pdev)); |
564 | |
565 | if (IS_ERR(ptr: plat_dev)) |
566 | return PTR_ERR(ptr: plat_dev); |
567 | |
568 | dev_info = kzalloc(size: sizeof(*dev_info), GFP_KERNEL); |
569 | if (!dev_info) |
570 | return -ENOMEM; |
571 | |
572 | /* Cache platform device to handle pci device hotplug */ |
573 | dev_info->plat_dev = plat_dev; |
574 | dev_info->pdev = pdev; |
575 | list_add(new: &dev_info->dev_node, head: &dwc_pcie_dev_info_head); |
576 | |
577 | return 0; |
578 | } |
579 | |
580 | static int dwc_pcie_pmu_notifier(struct notifier_block *nb, |
581 | unsigned long action, void *data) |
582 | { |
583 | struct device *dev = data; |
584 | struct pci_dev *pdev = to_pci_dev(dev); |
585 | struct dwc_pcie_dev_info *dev_info; |
586 | |
587 | switch (action) { |
588 | case BUS_NOTIFY_ADD_DEVICE: |
589 | if (!dwc_pcie_match_des_cap(pdev)) |
590 | return NOTIFY_DONE; |
591 | if (dwc_pcie_register_dev(pdev)) |
592 | return NOTIFY_BAD; |
593 | break; |
594 | case BUS_NOTIFY_DEL_DEVICE: |
595 | dev_info = dwc_pcie_find_dev_info(pdev); |
596 | if (!dev_info) |
597 | return NOTIFY_DONE; |
598 | dwc_pcie_unregister_dev(dev_info); |
599 | break; |
600 | } |
601 | |
602 | return NOTIFY_OK; |
603 | } |
604 | |
605 | static struct notifier_block dwc_pcie_pmu_nb = { |
606 | .notifier_call = dwc_pcie_pmu_notifier, |
607 | }; |
608 | |
609 | static int dwc_pcie_pmu_probe(struct platform_device *plat_dev) |
610 | { |
611 | struct pci_dev *pdev = plat_dev->dev.platform_data; |
612 | struct dwc_pcie_pmu *pcie_pmu; |
613 | char *name; |
614 | u32 bdf, val; |
615 | u16 vsec; |
616 | int ret; |
617 | |
618 | vsec = pci_find_vsec_capability(dev: pdev, vendor: pdev->vendor, |
619 | DWC_PCIE_VSEC_RAS_DES_ID); |
620 | pci_read_config_dword(dev: pdev, where: vsec + PCI_VNDR_HEADER, val: &val); |
621 | bdf = PCI_DEVID(pdev->bus->number, pdev->devfn); |
622 | name = devm_kasprintf(dev: &plat_dev->dev, GFP_KERNEL, fmt: "dwc_rootport_%x" , bdf); |
623 | if (!name) |
624 | return -ENOMEM; |
625 | |
626 | pcie_pmu = devm_kzalloc(dev: &plat_dev->dev, size: sizeof(*pcie_pmu), GFP_KERNEL); |
627 | if (!pcie_pmu) |
628 | return -ENOMEM; |
629 | |
630 | pcie_pmu->pdev = pdev; |
631 | pcie_pmu->ras_des_offset = vsec; |
632 | pcie_pmu->nr_lanes = pcie_get_width_cap(dev: pdev); |
633 | pcie_pmu->on_cpu = -1; |
634 | pcie_pmu->pmu = (struct pmu){ |
635 | .name = name, |
636 | .parent = &pdev->dev, |
637 | .module = THIS_MODULE, |
638 | .attr_groups = dwc_pcie_attr_groups, |
639 | .capabilities = PERF_PMU_CAP_NO_EXCLUDE, |
640 | .task_ctx_nr = perf_invalid_context, |
641 | .event_init = dwc_pcie_pmu_event_init, |
642 | .add = dwc_pcie_pmu_event_add, |
643 | .del = dwc_pcie_pmu_event_del, |
644 | .start = dwc_pcie_pmu_event_start, |
645 | .stop = dwc_pcie_pmu_event_stop, |
646 | .read = dwc_pcie_pmu_event_update, |
647 | }; |
648 | |
649 | /* Add this instance to the list used by the offline callback */ |
650 | ret = cpuhp_state_add_instance(state: dwc_pcie_pmu_hp_state, |
651 | node: &pcie_pmu->cpuhp_node); |
652 | if (ret) { |
653 | pci_err(pdev, "Error %d registering hotplug @%x\n" , ret, bdf); |
654 | return ret; |
655 | } |
656 | |
657 | /* Unwind when platform driver removes */ |
658 | ret = devm_add_action_or_reset(&plat_dev->dev, |
659 | dwc_pcie_pmu_remove_cpuhp_instance, |
660 | &pcie_pmu->cpuhp_node); |
661 | if (ret) |
662 | return ret; |
663 | |
664 | ret = perf_pmu_register(pmu: &pcie_pmu->pmu, name, type: -1); |
665 | if (ret) { |
666 | pci_err(pdev, "Error %d registering PMU @%x\n" , ret, bdf); |
667 | return ret; |
668 | } |
669 | ret = devm_add_action_or_reset(&plat_dev->dev, dwc_pcie_unregister_pmu, |
670 | pcie_pmu); |
671 | if (ret) |
672 | return ret; |
673 | |
674 | return 0; |
675 | } |
676 | |
677 | static int dwc_pcie_pmu_online_cpu(unsigned int cpu, struct hlist_node *cpuhp_node) |
678 | { |
679 | struct dwc_pcie_pmu *pcie_pmu; |
680 | |
681 | pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node); |
682 | if (pcie_pmu->on_cpu == -1) |
683 | pcie_pmu->on_cpu = cpumask_local_spread( |
684 | i: 0, node: dev_to_node(dev: &pcie_pmu->pdev->dev)); |
685 | |
686 | return 0; |
687 | } |
688 | |
689 | static int dwc_pcie_pmu_offline_cpu(unsigned int cpu, struct hlist_node *cpuhp_node) |
690 | { |
691 | struct dwc_pcie_pmu *pcie_pmu; |
692 | struct pci_dev *pdev; |
693 | int node; |
694 | cpumask_t mask; |
695 | unsigned int target; |
696 | |
697 | pcie_pmu = hlist_entry_safe(cpuhp_node, struct dwc_pcie_pmu, cpuhp_node); |
698 | /* Nothing to do if this CPU doesn't own the PMU */ |
699 | if (cpu != pcie_pmu->on_cpu) |
700 | return 0; |
701 | |
702 | pcie_pmu->on_cpu = -1; |
703 | pdev = pcie_pmu->pdev; |
704 | node = dev_to_node(dev: &pdev->dev); |
705 | if (cpumask_and(dstp: &mask, src1p: cpumask_of_node(node), cpu_online_mask) && |
706 | cpumask_andnot(dstp: &mask, src1p: &mask, cpumask_of(cpu))) |
707 | target = cpumask_any(&mask); |
708 | else |
709 | target = cpumask_any_but(cpu_online_mask, cpu); |
710 | |
711 | if (target >= nr_cpu_ids) { |
712 | pci_err(pdev, "There is no CPU to set\n" ); |
713 | return 0; |
714 | } |
715 | |
716 | /* This PMU does NOT support interrupt, just migrate context. */ |
717 | perf_pmu_migrate_context(pmu: &pcie_pmu->pmu, src_cpu: cpu, dst_cpu: target); |
718 | pcie_pmu->on_cpu = target; |
719 | |
720 | return 0; |
721 | } |
722 | |
723 | static struct platform_driver dwc_pcie_pmu_driver = { |
724 | .probe = dwc_pcie_pmu_probe, |
725 | .driver = {.name = "dwc_pcie_pmu" ,}, |
726 | }; |
727 | |
728 | static int __init dwc_pcie_pmu_init(void) |
729 | { |
730 | struct pci_dev *pdev = NULL; |
731 | bool found = false; |
732 | int ret; |
733 | |
734 | for_each_pci_dev(pdev) { |
735 | if (!dwc_pcie_match_des_cap(pdev)) |
736 | continue; |
737 | |
738 | ret = dwc_pcie_register_dev(pdev); |
739 | if (ret) { |
740 | pci_dev_put(dev: pdev); |
741 | return ret; |
742 | } |
743 | |
744 | found = true; |
745 | } |
746 | if (!found) |
747 | return -ENODEV; |
748 | |
749 | ret = cpuhp_setup_state_multi(state: CPUHP_AP_ONLINE_DYN, |
750 | name: "perf/dwc_pcie_pmu:online" , |
751 | startup: dwc_pcie_pmu_online_cpu, |
752 | teardown: dwc_pcie_pmu_offline_cpu); |
753 | if (ret < 0) |
754 | return ret; |
755 | |
756 | dwc_pcie_pmu_hp_state = ret; |
757 | |
758 | ret = platform_driver_register(&dwc_pcie_pmu_driver); |
759 | if (ret) |
760 | goto platform_driver_register_err; |
761 | |
762 | ret = bus_register_notifier(bus: &pci_bus_type, nb: &dwc_pcie_pmu_nb); |
763 | if (ret) |
764 | goto platform_driver_register_err; |
765 | notify = true; |
766 | |
767 | return 0; |
768 | |
769 | platform_driver_register_err: |
770 | cpuhp_remove_multi_state(state: dwc_pcie_pmu_hp_state); |
771 | |
772 | return ret; |
773 | } |
774 | |
775 | static void __exit dwc_pcie_pmu_exit(void) |
776 | { |
777 | struct dwc_pcie_dev_info *dev_info, *tmp; |
778 | |
779 | if (notify) |
780 | bus_unregister_notifier(bus: &pci_bus_type, nb: &dwc_pcie_pmu_nb); |
781 | list_for_each_entry_safe(dev_info, tmp, &dwc_pcie_dev_info_head, dev_node) |
782 | dwc_pcie_unregister_dev(dev_info); |
783 | platform_driver_unregister(&dwc_pcie_pmu_driver); |
784 | cpuhp_remove_multi_state(state: dwc_pcie_pmu_hp_state); |
785 | } |
786 | |
787 | module_init(dwc_pcie_pmu_init); |
788 | module_exit(dwc_pcie_pmu_exit); |
789 | |
790 | MODULE_DESCRIPTION("PMU driver for DesignWare Cores PCI Express Controller" ); |
791 | MODULE_AUTHOR("Shuai Xue <xueshuai@linux.alibaba.com>" ); |
792 | MODULE_LICENSE("GPL v2" ); |
793 | |