1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright 2019 NXP |
4 | */ |
5 | |
6 | #include <linux/mod_devicetable.h> |
7 | #include <linux/module.h> |
8 | #include <linux/device.h> |
9 | #include <linux/platform_device.h> |
10 | #include <linux/devfreq.h> |
11 | #include <linux/pm_opp.h> |
12 | #include <linux/clk.h> |
13 | #include <linux/clk-provider.h> |
14 | #include <linux/arm-smccc.h> |
15 | |
16 | #define IMX_SIP_DDR_DVFS 0xc2000004 |
17 | |
18 | /* Query available frequencies. */ |
19 | #define IMX_SIP_DDR_DVFS_GET_FREQ_COUNT 0x10 |
20 | #define IMX_SIP_DDR_DVFS_GET_FREQ_INFO 0x11 |
21 | |
22 | /* |
23 | * This should be in a 1:1 mapping with devicetree OPPs but |
24 | * firmware provides additional info. |
25 | */ |
26 | struct imx8m_ddrc_freq { |
27 | unsigned long rate; |
28 | unsigned long smcarg; |
29 | int dram_core_parent_index; |
30 | int dram_alt_parent_index; |
31 | int dram_apb_parent_index; |
32 | }; |
33 | |
34 | /* Hardware limitation */ |
35 | #define IMX8M_DDRC_MAX_FREQ_COUNT 4 |
36 | |
37 | /* |
38 | * i.MX8M DRAM Controller clocks have the following structure (abridged): |
39 | * |
40 | * +----------+ |\ +------+ |
41 | * | dram_pll |-------|M| dram_core | | |
42 | * +----------+ |U|---------->| D | |
43 | * /--|X| | D | |
44 | * dram_alt_root | |/ | R | |
45 | * | | C | |
46 | * +---------+ | | |
47 | * |FIX DIV/4| | | |
48 | * +---------+ | | |
49 | * composite: | | | |
50 | * +----------+ | | | |
51 | * | dram_alt |----/ | | |
52 | * +----------+ | | |
53 | * | dram_apb |-------------------->| | |
54 | * +----------+ +------+ |
55 | * |
56 | * The dram_pll is used for higher rates and dram_alt is used for lower rates. |
57 | * |
58 | * Frequency switching is implemented in TF-A (via SMC call) and can change the |
59 | * configuration of the clocks, including mux parents. The dram_alt and |
60 | * dram_apb clocks are "imx composite" and their parent can change too. |
61 | * |
62 | * We need to prepare/enable the new mux parents head of switching and update |
63 | * their information afterwards. |
64 | */ |
65 | struct imx8m_ddrc { |
66 | struct devfreq_dev_profile profile; |
67 | struct devfreq *devfreq; |
68 | |
69 | /* For frequency switching: */ |
70 | struct clk *dram_core; |
71 | struct clk *dram_pll; |
72 | struct clk *dram_alt; |
73 | struct clk *dram_apb; |
74 | |
75 | int freq_count; |
76 | struct imx8m_ddrc_freq freq_table[IMX8M_DDRC_MAX_FREQ_COUNT]; |
77 | }; |
78 | |
79 | static struct imx8m_ddrc_freq *imx8m_ddrc_find_freq(struct imx8m_ddrc *priv, |
80 | unsigned long rate) |
81 | { |
82 | struct imx8m_ddrc_freq *freq; |
83 | int i; |
84 | |
85 | /* |
86 | * Firmware reports values in MT/s, so we round-down from Hz |
87 | * Rounding is extra generous to ensure a match. |
88 | */ |
89 | rate = DIV_ROUND_CLOSEST(rate, 250000); |
90 | for (i = 0; i < priv->freq_count; ++i) { |
91 | freq = &priv->freq_table[i]; |
92 | if (freq->rate == rate || |
93 | freq->rate + 1 == rate || |
94 | freq->rate - 1 == rate) |
95 | return freq; |
96 | } |
97 | |
98 | return NULL; |
99 | } |
100 | |
101 | static void imx8m_ddrc_smc_set_freq(int target_freq) |
102 | { |
103 | struct arm_smccc_res res; |
104 | u32 online_cpus = 0; |
105 | int cpu; |
106 | |
107 | local_irq_disable(); |
108 | |
109 | for_each_online_cpu(cpu) |
110 | online_cpus |= (1 << (cpu * 8)); |
111 | |
112 | /* change the ddr freqency */ |
113 | arm_smccc_smc(IMX_SIP_DDR_DVFS, target_freq, online_cpus, |
114 | 0, 0, 0, 0, 0, &res); |
115 | |
116 | local_irq_enable(); |
117 | } |
118 | |
119 | static struct clk *clk_get_parent_by_index(struct clk *clk, int index) |
120 | { |
121 | struct clk_hw *hw; |
122 | |
123 | hw = clk_hw_get_parent_by_index(hw: __clk_get_hw(clk), index); |
124 | |
125 | return hw ? hw->clk : NULL; |
126 | } |
127 | |
128 | static int imx8m_ddrc_set_freq(struct device *dev, struct imx8m_ddrc_freq *freq) |
129 | { |
130 | struct imx8m_ddrc *priv = dev_get_drvdata(dev); |
131 | struct clk *new_dram_core_parent; |
132 | struct clk *new_dram_alt_parent; |
133 | struct clk *new_dram_apb_parent; |
134 | int ret; |
135 | |
136 | /* |
137 | * Fetch new parents |
138 | * |
139 | * new_dram_alt_parent and new_dram_apb_parent are optional but |
140 | * new_dram_core_parent is not. |
141 | */ |
142 | new_dram_core_parent = clk_get_parent_by_index( |
143 | clk: priv->dram_core, index: freq->dram_core_parent_index - 1); |
144 | if (!new_dram_core_parent) { |
145 | dev_err(dev, "failed to fetch new dram_core parent\n" ); |
146 | return -EINVAL; |
147 | } |
148 | if (freq->dram_alt_parent_index) { |
149 | new_dram_alt_parent = clk_get_parent_by_index( |
150 | clk: priv->dram_alt, |
151 | index: freq->dram_alt_parent_index - 1); |
152 | if (!new_dram_alt_parent) { |
153 | dev_err(dev, "failed to fetch new dram_alt parent\n" ); |
154 | return -EINVAL; |
155 | } |
156 | } else |
157 | new_dram_alt_parent = NULL; |
158 | |
159 | if (freq->dram_apb_parent_index) { |
160 | new_dram_apb_parent = clk_get_parent_by_index( |
161 | clk: priv->dram_apb, |
162 | index: freq->dram_apb_parent_index - 1); |
163 | if (!new_dram_apb_parent) { |
164 | dev_err(dev, "failed to fetch new dram_apb parent\n" ); |
165 | return -EINVAL; |
166 | } |
167 | } else |
168 | new_dram_apb_parent = NULL; |
169 | |
170 | /* increase reference counts and ensure clks are ON before switch */ |
171 | ret = clk_prepare_enable(clk: new_dram_core_parent); |
172 | if (ret) { |
173 | dev_err(dev, "failed to enable new dram_core parent: %d\n" , |
174 | ret); |
175 | goto out; |
176 | } |
177 | ret = clk_prepare_enable(clk: new_dram_alt_parent); |
178 | if (ret) { |
179 | dev_err(dev, "failed to enable new dram_alt parent: %d\n" , |
180 | ret); |
181 | goto out_disable_core_parent; |
182 | } |
183 | ret = clk_prepare_enable(clk: new_dram_apb_parent); |
184 | if (ret) { |
185 | dev_err(dev, "failed to enable new dram_apb parent: %d\n" , |
186 | ret); |
187 | goto out_disable_alt_parent; |
188 | } |
189 | |
190 | imx8m_ddrc_smc_set_freq(target_freq: freq->smcarg); |
191 | |
192 | /* update parents in clk tree after switch. */ |
193 | ret = clk_set_parent(clk: priv->dram_core, parent: new_dram_core_parent); |
194 | if (ret) |
195 | dev_warn(dev, "failed to set dram_core parent: %d\n" , ret); |
196 | if (new_dram_alt_parent) { |
197 | ret = clk_set_parent(clk: priv->dram_alt, parent: new_dram_alt_parent); |
198 | if (ret) |
199 | dev_warn(dev, "failed to set dram_alt parent: %d\n" , |
200 | ret); |
201 | } |
202 | if (new_dram_apb_parent) { |
203 | ret = clk_set_parent(clk: priv->dram_apb, parent: new_dram_apb_parent); |
204 | if (ret) |
205 | dev_warn(dev, "failed to set dram_apb parent: %d\n" , |
206 | ret); |
207 | } |
208 | |
209 | /* |
210 | * Explicitly refresh dram PLL rate. |
211 | * |
212 | * Even if it's marked with CLK_GET_RATE_NOCACHE the rate will not be |
213 | * automatically refreshed when clk_get_rate is called on children. |
214 | */ |
215 | clk_get_rate(clk: priv->dram_pll); |
216 | |
217 | /* |
218 | * clk_set_parent transfer the reference count from old parent. |
219 | * now we drop extra reference counts used during the switch |
220 | */ |
221 | clk_disable_unprepare(clk: new_dram_apb_parent); |
222 | out_disable_alt_parent: |
223 | clk_disable_unprepare(clk: new_dram_alt_parent); |
224 | out_disable_core_parent: |
225 | clk_disable_unprepare(clk: new_dram_core_parent); |
226 | out: |
227 | return ret; |
228 | } |
229 | |
230 | static int imx8m_ddrc_target(struct device *dev, unsigned long *freq, u32 flags) |
231 | { |
232 | struct imx8m_ddrc *priv = dev_get_drvdata(dev); |
233 | struct imx8m_ddrc_freq *freq_info; |
234 | struct dev_pm_opp *new_opp; |
235 | unsigned long old_freq, new_freq; |
236 | int ret; |
237 | |
238 | new_opp = devfreq_recommended_opp(dev, freq, flags); |
239 | if (IS_ERR(ptr: new_opp)) { |
240 | ret = PTR_ERR(ptr: new_opp); |
241 | dev_err(dev, "failed to get recommended opp: %d\n" , ret); |
242 | return ret; |
243 | } |
244 | dev_pm_opp_put(opp: new_opp); |
245 | |
246 | old_freq = clk_get_rate(clk: priv->dram_core); |
247 | if (*freq == old_freq) |
248 | return 0; |
249 | |
250 | freq_info = imx8m_ddrc_find_freq(priv, rate: *freq); |
251 | if (!freq_info) |
252 | return -EINVAL; |
253 | |
254 | /* |
255 | * Read back the clk rate to verify switch was correct and so that |
256 | * we can report it on all error paths. |
257 | */ |
258 | ret = imx8m_ddrc_set_freq(dev, freq: freq_info); |
259 | |
260 | new_freq = clk_get_rate(clk: priv->dram_core); |
261 | if (ret) |
262 | dev_err(dev, "ddrc failed freq switch to %lu from %lu: error %d. now at %lu\n" , |
263 | *freq, old_freq, ret, new_freq); |
264 | else if (*freq != new_freq) |
265 | dev_err(dev, "ddrc failed freq update to %lu from %lu, now at %lu\n" , |
266 | *freq, old_freq, new_freq); |
267 | else |
268 | dev_dbg(dev, "ddrc freq set to %lu (was %lu)\n" , |
269 | *freq, old_freq); |
270 | |
271 | return ret; |
272 | } |
273 | |
274 | static int imx8m_ddrc_get_cur_freq(struct device *dev, unsigned long *freq) |
275 | { |
276 | struct imx8m_ddrc *priv = dev_get_drvdata(dev); |
277 | |
278 | *freq = clk_get_rate(clk: priv->dram_core); |
279 | |
280 | return 0; |
281 | } |
282 | |
283 | static int imx8m_ddrc_init_freq_info(struct device *dev) |
284 | { |
285 | struct imx8m_ddrc *priv = dev_get_drvdata(dev); |
286 | struct arm_smccc_res res; |
287 | int index; |
288 | |
289 | /* An error here means DDR DVFS API not supported by firmware */ |
290 | arm_smccc_smc(IMX_SIP_DDR_DVFS, IMX_SIP_DDR_DVFS_GET_FREQ_COUNT, |
291 | 0, 0, 0, 0, 0, 0, &res); |
292 | priv->freq_count = res.a0; |
293 | if (priv->freq_count <= 0 || |
294 | priv->freq_count > IMX8M_DDRC_MAX_FREQ_COUNT) |
295 | return -ENODEV; |
296 | |
297 | for (index = 0; index < priv->freq_count; ++index) { |
298 | struct imx8m_ddrc_freq *freq = &priv->freq_table[index]; |
299 | |
300 | arm_smccc_smc(IMX_SIP_DDR_DVFS, IMX_SIP_DDR_DVFS_GET_FREQ_INFO, |
301 | index, 0, 0, 0, 0, 0, &res); |
302 | /* Result should be strictly positive */ |
303 | if ((long)res.a0 <= 0) |
304 | return -ENODEV; |
305 | |
306 | freq->rate = res.a0; |
307 | freq->smcarg = index; |
308 | freq->dram_core_parent_index = res.a1; |
309 | freq->dram_alt_parent_index = res.a2; |
310 | freq->dram_apb_parent_index = res.a3; |
311 | |
312 | /* dram_core has 2 options: dram_pll or dram_alt_root */ |
313 | if (freq->dram_core_parent_index != 1 && |
314 | freq->dram_core_parent_index != 2) |
315 | return -ENODEV; |
316 | /* dram_apb and dram_alt have exactly 8 possible parents */ |
317 | if (freq->dram_alt_parent_index > 8 || |
318 | freq->dram_apb_parent_index > 8) |
319 | return -ENODEV; |
320 | /* dram_core from alt requires explicit dram_alt parent */ |
321 | if (freq->dram_core_parent_index == 2 && |
322 | freq->dram_alt_parent_index == 0) |
323 | return -ENODEV; |
324 | } |
325 | |
326 | return 0; |
327 | } |
328 | |
329 | static int imx8m_ddrc_check_opps(struct device *dev) |
330 | { |
331 | struct imx8m_ddrc *priv = dev_get_drvdata(dev); |
332 | struct imx8m_ddrc_freq *freq_info; |
333 | struct dev_pm_opp *opp; |
334 | unsigned long freq; |
335 | int i, opp_count; |
336 | |
337 | /* Enumerate DT OPPs and disable those not supported by firmware */ |
338 | opp_count = dev_pm_opp_get_opp_count(dev); |
339 | if (opp_count < 0) |
340 | return opp_count; |
341 | for (i = 0, freq = 0; i < opp_count; ++i, ++freq) { |
342 | opp = dev_pm_opp_find_freq_ceil(dev, freq: &freq); |
343 | if (IS_ERR(ptr: opp)) { |
344 | dev_err(dev, "Failed enumerating OPPs: %ld\n" , |
345 | PTR_ERR(opp)); |
346 | return PTR_ERR(ptr: opp); |
347 | } |
348 | dev_pm_opp_put(opp); |
349 | |
350 | freq_info = imx8m_ddrc_find_freq(priv, rate: freq); |
351 | if (!freq_info) { |
352 | dev_info(dev, "Disable unsupported OPP %luHz %luMT/s\n" , |
353 | freq, DIV_ROUND_CLOSEST(freq, 250000)); |
354 | dev_pm_opp_disable(dev, freq); |
355 | } |
356 | } |
357 | |
358 | return 0; |
359 | } |
360 | |
361 | static void imx8m_ddrc_exit(struct device *dev) |
362 | { |
363 | dev_pm_opp_of_remove_table(dev); |
364 | } |
365 | |
366 | static int imx8m_ddrc_probe(struct platform_device *pdev) |
367 | { |
368 | struct device *dev = &pdev->dev; |
369 | struct imx8m_ddrc *priv; |
370 | const char *gov = DEVFREQ_GOV_USERSPACE; |
371 | int ret; |
372 | |
373 | priv = devm_kzalloc(dev, size: sizeof(*priv), GFP_KERNEL); |
374 | if (!priv) |
375 | return -ENOMEM; |
376 | |
377 | platform_set_drvdata(pdev, data: priv); |
378 | |
379 | ret = imx8m_ddrc_init_freq_info(dev); |
380 | if (ret) { |
381 | dev_err(dev, "failed to init firmware freq info: %d\n" , ret); |
382 | return ret; |
383 | } |
384 | |
385 | priv->dram_core = devm_clk_get(dev, id: "core" ); |
386 | if (IS_ERR(ptr: priv->dram_core)) { |
387 | ret = PTR_ERR(ptr: priv->dram_core); |
388 | dev_err(dev, "failed to fetch core clock: %d\n" , ret); |
389 | return ret; |
390 | } |
391 | priv->dram_pll = devm_clk_get(dev, id: "pll" ); |
392 | if (IS_ERR(ptr: priv->dram_pll)) { |
393 | ret = PTR_ERR(ptr: priv->dram_pll); |
394 | dev_err(dev, "failed to fetch pll clock: %d\n" , ret); |
395 | return ret; |
396 | } |
397 | priv->dram_alt = devm_clk_get(dev, id: "alt" ); |
398 | if (IS_ERR(ptr: priv->dram_alt)) { |
399 | ret = PTR_ERR(ptr: priv->dram_alt); |
400 | dev_err(dev, "failed to fetch alt clock: %d\n" , ret); |
401 | return ret; |
402 | } |
403 | priv->dram_apb = devm_clk_get(dev, id: "apb" ); |
404 | if (IS_ERR(ptr: priv->dram_apb)) { |
405 | ret = PTR_ERR(ptr: priv->dram_apb); |
406 | dev_err(dev, "failed to fetch apb clock: %d\n" , ret); |
407 | return ret; |
408 | } |
409 | |
410 | ret = dev_pm_opp_of_add_table(dev); |
411 | if (ret < 0) { |
412 | dev_err(dev, "failed to get OPP table\n" ); |
413 | return ret; |
414 | } |
415 | |
416 | ret = imx8m_ddrc_check_opps(dev); |
417 | if (ret < 0) |
418 | goto err; |
419 | |
420 | priv->profile.target = imx8m_ddrc_target; |
421 | priv->profile.exit = imx8m_ddrc_exit; |
422 | priv->profile.get_cur_freq = imx8m_ddrc_get_cur_freq; |
423 | priv->profile.initial_freq = clk_get_rate(clk: priv->dram_core); |
424 | |
425 | priv->devfreq = devm_devfreq_add_device(dev, profile: &priv->profile, |
426 | governor_name: gov, NULL); |
427 | if (IS_ERR(ptr: priv->devfreq)) { |
428 | ret = PTR_ERR(ptr: priv->devfreq); |
429 | dev_err(dev, "failed to add devfreq device: %d\n" , ret); |
430 | goto err; |
431 | } |
432 | |
433 | return 0; |
434 | |
435 | err: |
436 | dev_pm_opp_of_remove_table(dev); |
437 | return ret; |
438 | } |
439 | |
440 | static const struct of_device_id imx8m_ddrc_of_match[] = { |
441 | { .compatible = "fsl,imx8m-ddrc" , }, |
442 | { /* sentinel */ }, |
443 | }; |
444 | MODULE_DEVICE_TABLE(of, imx8m_ddrc_of_match); |
445 | |
446 | static struct platform_driver imx8m_ddrc_platdrv = { |
447 | .probe = imx8m_ddrc_probe, |
448 | .driver = { |
449 | .name = "imx8m-ddrc-devfreq" , |
450 | .of_match_table = imx8m_ddrc_of_match, |
451 | }, |
452 | }; |
453 | module_platform_driver(imx8m_ddrc_platdrv); |
454 | |
455 | MODULE_DESCRIPTION("i.MX8M DDR Controller frequency driver" ); |
456 | MODULE_AUTHOR("Leonard Crestez <leonard.crestez@nxp.com>" ); |
457 | MODULE_LICENSE("GPL v2" ); |
458 | |