1 | // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) |
2 | /* |
3 | * Copyright(c) 2019 Intel Corporation. |
4 | * |
5 | */ |
6 | |
7 | #include "aspm.h" |
8 | |
9 | /* Time after which the timer interrupt will re-enable ASPM */ |
10 | #define ASPM_TIMER_MS 1000 |
11 | /* Time for which interrupts are ignored after a timer has been scheduled */ |
12 | #define ASPM_RESCHED_TIMER_MS (ASPM_TIMER_MS / 2) |
13 | /* Two interrupts within this time trigger ASPM disable */ |
14 | #define ASPM_TRIGGER_MS 1 |
15 | #define ASPM_TRIGGER_NS (ASPM_TRIGGER_MS * 1000 * 1000ull) |
16 | #define ASPM_L1_SUPPORTED(reg) \ |
17 | ((((reg) & PCI_EXP_LNKCAP_ASPMS) >> 10) & 0x2) |
18 | |
19 | uint aspm_mode = ASPM_MODE_DISABLED; |
20 | module_param_named(aspm, aspm_mode, uint, 0444); |
21 | MODULE_PARM_DESC(aspm, "PCIe ASPM: 0: disable, 1: enable, 2: dynamic" ); |
22 | |
23 | static bool aspm_hw_l1_supported(struct hfi1_devdata *dd) |
24 | { |
25 | struct pci_dev *parent = dd->pcidev->bus->self; |
26 | u32 up, dn; |
27 | |
28 | /* |
29 | * If the driver does not have access to the upstream component, |
30 | * it cannot support ASPM L1 at all. |
31 | */ |
32 | if (!parent) |
33 | return false; |
34 | |
35 | pcie_capability_read_dword(dev: dd->pcidev, PCI_EXP_LNKCAP, val: &dn); |
36 | dn = ASPM_L1_SUPPORTED(dn); |
37 | |
38 | pcie_capability_read_dword(dev: parent, PCI_EXP_LNKCAP, val: &up); |
39 | up = ASPM_L1_SUPPORTED(up); |
40 | |
41 | /* ASPM works on A-step but is reported as not supported */ |
42 | return (!!dn || is_ax(dd)) && !!up; |
43 | } |
44 | |
45 | /* Set L1 entrance latency for slower entry to L1 */ |
46 | static void aspm_hw_set_l1_ent_latency(struct hfi1_devdata *dd) |
47 | { |
48 | u32 l1_ent_lat = 0x4u; |
49 | u32 reg32; |
50 | |
51 | pci_read_config_dword(dev: dd->pcidev, PCIE_CFG_REG_PL3, val: ®32); |
52 | reg32 &= ~PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SMASK; |
53 | reg32 |= l1_ent_lat << PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SHIFT; |
54 | pci_write_config_dword(dev: dd->pcidev, PCIE_CFG_REG_PL3, val: reg32); |
55 | } |
56 | |
57 | static void aspm_hw_enable_l1(struct hfi1_devdata *dd) |
58 | { |
59 | struct pci_dev *parent = dd->pcidev->bus->self; |
60 | |
61 | /* |
62 | * If the driver does not have access to the upstream component, |
63 | * it cannot support ASPM L1 at all. |
64 | */ |
65 | if (!parent) |
66 | return; |
67 | |
68 | /* Enable ASPM L1 first in upstream component and then downstream */ |
69 | pcie_capability_clear_and_set_word(dev: parent, PCI_EXP_LNKCTL, |
70 | PCI_EXP_LNKCTL_ASPMC, |
71 | PCI_EXP_LNKCTL_ASPM_L1); |
72 | pcie_capability_clear_and_set_word(dev: dd->pcidev, PCI_EXP_LNKCTL, |
73 | PCI_EXP_LNKCTL_ASPMC, |
74 | PCI_EXP_LNKCTL_ASPM_L1); |
75 | } |
76 | |
77 | void aspm_hw_disable_l1(struct hfi1_devdata *dd) |
78 | { |
79 | struct pci_dev *parent = dd->pcidev->bus->self; |
80 | |
81 | /* Disable ASPM L1 first in downstream component and then upstream */ |
82 | pcie_capability_clear_and_set_word(dev: dd->pcidev, PCI_EXP_LNKCTL, |
83 | PCI_EXP_LNKCTL_ASPMC, set: 0x0); |
84 | if (parent) |
85 | pcie_capability_clear_and_set_word(dev: parent, PCI_EXP_LNKCTL, |
86 | PCI_EXP_LNKCTL_ASPMC, set: 0x0); |
87 | } |
88 | |
89 | static void aspm_enable(struct hfi1_devdata *dd) |
90 | { |
91 | if (dd->aspm_enabled || aspm_mode == ASPM_MODE_DISABLED || |
92 | !dd->aspm_supported) |
93 | return; |
94 | |
95 | aspm_hw_enable_l1(dd); |
96 | dd->aspm_enabled = true; |
97 | } |
98 | |
99 | static void aspm_disable(struct hfi1_devdata *dd) |
100 | { |
101 | if (!dd->aspm_enabled || aspm_mode == ASPM_MODE_ENABLED) |
102 | return; |
103 | |
104 | aspm_hw_disable_l1(dd); |
105 | dd->aspm_enabled = false; |
106 | } |
107 | |
108 | static void aspm_disable_inc(struct hfi1_devdata *dd) |
109 | { |
110 | unsigned long flags; |
111 | |
112 | spin_lock_irqsave(&dd->aspm_lock, flags); |
113 | aspm_disable(dd); |
114 | atomic_inc(v: &dd->aspm_disabled_cnt); |
115 | spin_unlock_irqrestore(lock: &dd->aspm_lock, flags); |
116 | } |
117 | |
118 | static void aspm_enable_dec(struct hfi1_devdata *dd) |
119 | { |
120 | unsigned long flags; |
121 | |
122 | spin_lock_irqsave(&dd->aspm_lock, flags); |
123 | if (atomic_dec_and_test(v: &dd->aspm_disabled_cnt)) |
124 | aspm_enable(dd); |
125 | spin_unlock_irqrestore(lock: &dd->aspm_lock, flags); |
126 | } |
127 | |
128 | /* ASPM processing for each receive context interrupt */ |
129 | void __aspm_ctx_disable(struct hfi1_ctxtdata *rcd) |
130 | { |
131 | bool restart_timer; |
132 | bool close_interrupts; |
133 | unsigned long flags; |
134 | ktime_t now, prev; |
135 | |
136 | spin_lock_irqsave(&rcd->aspm_lock, flags); |
137 | /* PSM contexts are open */ |
138 | if (!rcd->aspm_intr_enable) |
139 | goto unlock; |
140 | |
141 | prev = rcd->aspm_ts_last_intr; |
142 | now = ktime_get(); |
143 | rcd->aspm_ts_last_intr = now; |
144 | |
145 | /* An interrupt pair close together in time */ |
146 | close_interrupts = ktime_to_ns(ktime_sub(now, prev)) < ASPM_TRIGGER_NS; |
147 | |
148 | /* Don't push out our timer till this much time has elapsed */ |
149 | restart_timer = ktime_to_ns(ktime_sub(now, rcd->aspm_ts_timer_sched)) > |
150 | ASPM_RESCHED_TIMER_MS * NSEC_PER_MSEC; |
151 | restart_timer = restart_timer && close_interrupts; |
152 | |
153 | /* Disable ASPM and schedule timer */ |
154 | if (rcd->aspm_enabled && close_interrupts) { |
155 | aspm_disable_inc(dd: rcd->dd); |
156 | rcd->aspm_enabled = false; |
157 | restart_timer = true; |
158 | } |
159 | |
160 | if (restart_timer) { |
161 | mod_timer(timer: &rcd->aspm_timer, |
162 | expires: jiffies + msecs_to_jiffies(ASPM_TIMER_MS)); |
163 | rcd->aspm_ts_timer_sched = now; |
164 | } |
165 | unlock: |
166 | spin_unlock_irqrestore(lock: &rcd->aspm_lock, flags); |
167 | } |
168 | |
169 | /* Timer function for re-enabling ASPM in the absence of interrupt activity */ |
170 | static void aspm_ctx_timer_function(struct timer_list *t) |
171 | { |
172 | struct hfi1_ctxtdata *rcd = from_timer(rcd, t, aspm_timer); |
173 | unsigned long flags; |
174 | |
175 | spin_lock_irqsave(&rcd->aspm_lock, flags); |
176 | aspm_enable_dec(dd: rcd->dd); |
177 | rcd->aspm_enabled = true; |
178 | spin_unlock_irqrestore(lock: &rcd->aspm_lock, flags); |
179 | } |
180 | |
181 | /* |
182 | * Disable interrupt processing for verbs contexts when PSM or VNIC contexts |
183 | * are open. |
184 | */ |
185 | void aspm_disable_all(struct hfi1_devdata *dd) |
186 | { |
187 | struct hfi1_ctxtdata *rcd; |
188 | unsigned long flags; |
189 | u16 i; |
190 | |
191 | for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) { |
192 | rcd = hfi1_rcd_get_by_index(dd, ctxt: i); |
193 | if (rcd) { |
194 | del_timer_sync(timer: &rcd->aspm_timer); |
195 | spin_lock_irqsave(&rcd->aspm_lock, flags); |
196 | rcd->aspm_intr_enable = false; |
197 | spin_unlock_irqrestore(lock: &rcd->aspm_lock, flags); |
198 | hfi1_rcd_put(rcd); |
199 | } |
200 | } |
201 | |
202 | aspm_disable(dd); |
203 | atomic_set(v: &dd->aspm_disabled_cnt, i: 0); |
204 | } |
205 | |
206 | /* Re-enable interrupt processing for verbs contexts */ |
207 | void aspm_enable_all(struct hfi1_devdata *dd) |
208 | { |
209 | struct hfi1_ctxtdata *rcd; |
210 | unsigned long flags; |
211 | u16 i; |
212 | |
213 | aspm_enable(dd); |
214 | |
215 | if (aspm_mode != ASPM_MODE_DYNAMIC) |
216 | return; |
217 | |
218 | for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) { |
219 | rcd = hfi1_rcd_get_by_index(dd, ctxt: i); |
220 | if (rcd) { |
221 | spin_lock_irqsave(&rcd->aspm_lock, flags); |
222 | rcd->aspm_intr_enable = true; |
223 | rcd->aspm_enabled = true; |
224 | spin_unlock_irqrestore(lock: &rcd->aspm_lock, flags); |
225 | hfi1_rcd_put(rcd); |
226 | } |
227 | } |
228 | } |
229 | |
230 | static void aspm_ctx_init(struct hfi1_ctxtdata *rcd) |
231 | { |
232 | spin_lock_init(&rcd->aspm_lock); |
233 | timer_setup(&rcd->aspm_timer, aspm_ctx_timer_function, 0); |
234 | rcd->aspm_intr_supported = rcd->dd->aspm_supported && |
235 | aspm_mode == ASPM_MODE_DYNAMIC && |
236 | rcd->ctxt < rcd->dd->first_dyn_alloc_ctxt; |
237 | } |
238 | |
239 | void aspm_init(struct hfi1_devdata *dd) |
240 | { |
241 | struct hfi1_ctxtdata *rcd; |
242 | u16 i; |
243 | |
244 | spin_lock_init(&dd->aspm_lock); |
245 | dd->aspm_supported = aspm_hw_l1_supported(dd); |
246 | |
247 | for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) { |
248 | rcd = hfi1_rcd_get_by_index(dd, ctxt: i); |
249 | if (rcd) |
250 | aspm_ctx_init(rcd); |
251 | hfi1_rcd_put(rcd); |
252 | } |
253 | |
254 | /* Start with ASPM disabled */ |
255 | aspm_hw_set_l1_ent_latency(dd); |
256 | dd->aspm_enabled = false; |
257 | aspm_hw_disable_l1(dd); |
258 | |
259 | /* Now turn on ASPM if configured */ |
260 | aspm_enable_all(dd); |
261 | } |
262 | |
263 | void aspm_exit(struct hfi1_devdata *dd) |
264 | { |
265 | aspm_disable_all(dd); |
266 | |
267 | /* Turn on ASPM on exit to conserve power */ |
268 | aspm_enable(dd); |
269 | } |
270 | |
271 | |