1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Copyright (c) 2016, Fuzhou Rockchip Electronics Co., Ltd. |
4 | * Author: Lin Huang <hl@rock-chips.com> |
5 | */ |
6 | |
7 | #include <linux/arm-smccc.h> |
8 | #include <linux/bitfield.h> |
9 | #include <linux/clk.h> |
10 | #include <linux/delay.h> |
11 | #include <linux/devfreq.h> |
12 | #include <linux/devfreq-event.h> |
13 | #include <linux/interrupt.h> |
14 | #include <linux/mfd/syscon.h> |
15 | #include <linux/module.h> |
16 | #include <linux/of.h> |
17 | #include <linux/platform_device.h> |
18 | #include <linux/pm_opp.h> |
19 | #include <linux/regmap.h> |
20 | #include <linux/regulator/consumer.h> |
21 | #include <linux/rwsem.h> |
22 | #include <linux/suspend.h> |
23 | |
24 | #include <soc/rockchip/pm_domains.h> |
25 | #include <soc/rockchip/rockchip_grf.h> |
26 | #include <soc/rockchip/rk3399_grf.h> |
27 | #include <soc/rockchip/rockchip_sip.h> |
28 | |
29 | #define NS_TO_CYCLE(NS, MHz) (((NS) * (MHz)) / NSEC_PER_USEC) |
30 | |
31 | #define RK3399_SET_ODT_PD_0_SR_IDLE GENMASK(7, 0) |
32 | #define RK3399_SET_ODT_PD_0_SR_MC_GATE_IDLE GENMASK(15, 8) |
33 | #define RK3399_SET_ODT_PD_0_STANDBY_IDLE GENMASK(31, 16) |
34 | |
35 | #define RK3399_SET_ODT_PD_1_PD_IDLE GENMASK(11, 0) |
36 | #define RK3399_SET_ODT_PD_1_SRPD_LITE_IDLE GENMASK(27, 16) |
37 | |
38 | #define RK3399_SET_ODT_PD_2_ODT_ENABLE BIT(0) |
39 | |
40 | struct rk3399_dmcfreq { |
41 | struct device *dev; |
42 | struct devfreq *devfreq; |
43 | struct devfreq_dev_profile profile; |
44 | struct devfreq_simple_ondemand_data ondemand_data; |
45 | struct clk *dmc_clk; |
46 | struct devfreq_event_dev *edev; |
47 | struct mutex lock; |
48 | struct regulator *vdd_center; |
49 | struct regmap *regmap_pmu; |
50 | unsigned long rate, target_rate; |
51 | unsigned long volt, target_volt; |
52 | unsigned int odt_dis_freq; |
53 | |
54 | unsigned int pd_idle_ns; |
55 | unsigned int sr_idle_ns; |
56 | unsigned int sr_mc_gate_idle_ns; |
57 | unsigned int srpd_lite_idle_ns; |
58 | unsigned int standby_idle_ns; |
59 | unsigned int ddr3_odt_dis_freq; |
60 | unsigned int lpddr3_odt_dis_freq; |
61 | unsigned int lpddr4_odt_dis_freq; |
62 | |
63 | unsigned int pd_idle_dis_freq; |
64 | unsigned int sr_idle_dis_freq; |
65 | unsigned int sr_mc_gate_idle_dis_freq; |
66 | unsigned int srpd_lite_idle_dis_freq; |
67 | unsigned int standby_idle_dis_freq; |
68 | }; |
69 | |
70 | static int rk3399_dmcfreq_target(struct device *dev, unsigned long *freq, |
71 | u32 flags) |
72 | { |
73 | struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev); |
74 | struct dev_pm_opp *opp; |
75 | unsigned long old_clk_rate = dmcfreq->rate; |
76 | unsigned long target_volt, target_rate; |
77 | unsigned int ddrcon_mhz; |
78 | struct arm_smccc_res res; |
79 | int err; |
80 | |
81 | u32 odt_pd_arg0 = 0; |
82 | u32 odt_pd_arg1 = 0; |
83 | u32 odt_pd_arg2 = 0; |
84 | |
85 | opp = devfreq_recommended_opp(dev, freq, flags); |
86 | if (IS_ERR(ptr: opp)) |
87 | return PTR_ERR(ptr: opp); |
88 | |
89 | target_rate = dev_pm_opp_get_freq(opp); |
90 | target_volt = dev_pm_opp_get_voltage(opp); |
91 | dev_pm_opp_put(opp); |
92 | |
93 | if (dmcfreq->rate == target_rate) |
94 | return 0; |
95 | |
96 | mutex_lock(&dmcfreq->lock); |
97 | |
98 | /* |
99 | * Ensure power-domain transitions don't interfere with ARM Trusted |
100 | * Firmware power-domain idling. |
101 | */ |
102 | err = rockchip_pmu_block(); |
103 | if (err) { |
104 | dev_err(dev, "Failed to block PMU: %d\n" , err); |
105 | goto out_unlock; |
106 | } |
107 | |
108 | /* |
109 | * Some idle parameters may be based on the DDR controller clock, which |
110 | * is half of the DDR frequency. |
111 | * pd_idle and standby_idle are based on the controller clock cycle. |
112 | * sr_idle_cycle, sr_mc_gate_idle_cycle, and srpd_lite_idle_cycle |
113 | * are based on the 1024 controller clock cycle |
114 | */ |
115 | ddrcon_mhz = target_rate / USEC_PER_SEC / 2; |
116 | |
117 | u32p_replace_bits(p: &odt_pd_arg1, |
118 | NS_TO_CYCLE(dmcfreq->pd_idle_ns, ddrcon_mhz), |
119 | RK3399_SET_ODT_PD_1_PD_IDLE); |
120 | u32p_replace_bits(p: &odt_pd_arg0, |
121 | NS_TO_CYCLE(dmcfreq->standby_idle_ns, ddrcon_mhz), |
122 | RK3399_SET_ODT_PD_0_STANDBY_IDLE); |
123 | u32p_replace_bits(p: &odt_pd_arg0, |
124 | DIV_ROUND_UP(NS_TO_CYCLE(dmcfreq->sr_idle_ns, |
125 | ddrcon_mhz), 1024), |
126 | RK3399_SET_ODT_PD_0_SR_IDLE); |
127 | u32p_replace_bits(p: &odt_pd_arg0, |
128 | DIV_ROUND_UP(NS_TO_CYCLE(dmcfreq->sr_mc_gate_idle_ns, |
129 | ddrcon_mhz), 1024), |
130 | RK3399_SET_ODT_PD_0_SR_MC_GATE_IDLE); |
131 | u32p_replace_bits(p: &odt_pd_arg1, |
132 | DIV_ROUND_UP(NS_TO_CYCLE(dmcfreq->srpd_lite_idle_ns, |
133 | ddrcon_mhz), 1024), |
134 | RK3399_SET_ODT_PD_1_SRPD_LITE_IDLE); |
135 | |
136 | if (dmcfreq->regmap_pmu) { |
137 | if (target_rate >= dmcfreq->sr_idle_dis_freq) |
138 | odt_pd_arg0 &= ~RK3399_SET_ODT_PD_0_SR_IDLE; |
139 | |
140 | if (target_rate >= dmcfreq->sr_mc_gate_idle_dis_freq) |
141 | odt_pd_arg0 &= ~RK3399_SET_ODT_PD_0_SR_MC_GATE_IDLE; |
142 | |
143 | if (target_rate >= dmcfreq->standby_idle_dis_freq) |
144 | odt_pd_arg0 &= ~RK3399_SET_ODT_PD_0_STANDBY_IDLE; |
145 | |
146 | if (target_rate >= dmcfreq->pd_idle_dis_freq) |
147 | odt_pd_arg1 &= ~RK3399_SET_ODT_PD_1_PD_IDLE; |
148 | |
149 | if (target_rate >= dmcfreq->srpd_lite_idle_dis_freq) |
150 | odt_pd_arg1 &= ~RK3399_SET_ODT_PD_1_SRPD_LITE_IDLE; |
151 | |
152 | if (target_rate >= dmcfreq->odt_dis_freq) |
153 | odt_pd_arg2 |= RK3399_SET_ODT_PD_2_ODT_ENABLE; |
154 | |
155 | /* |
156 | * This makes a SMC call to the TF-A to set the DDR PD |
157 | * (power-down) timings and to enable or disable the |
158 | * ODT (on-die termination) resistors. |
159 | */ |
160 | arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, odt_pd_arg0, odt_pd_arg1, |
161 | ROCKCHIP_SIP_CONFIG_DRAM_SET_ODT_PD, odt_pd_arg2, |
162 | 0, 0, 0, &res); |
163 | } |
164 | |
165 | /* |
166 | * If frequency scaling from low to high, adjust voltage first. |
167 | * If frequency scaling from high to low, adjust frequency first. |
168 | */ |
169 | if (old_clk_rate < target_rate) { |
170 | err = regulator_set_voltage(regulator: dmcfreq->vdd_center, min_uV: target_volt, |
171 | max_uV: target_volt); |
172 | if (err) { |
173 | dev_err(dev, "Cannot set voltage %lu uV\n" , |
174 | target_volt); |
175 | goto out; |
176 | } |
177 | } |
178 | |
179 | err = clk_set_rate(clk: dmcfreq->dmc_clk, rate: target_rate); |
180 | if (err) { |
181 | dev_err(dev, "Cannot set frequency %lu (%d)\n" , target_rate, |
182 | err); |
183 | regulator_set_voltage(regulator: dmcfreq->vdd_center, min_uV: dmcfreq->volt, |
184 | max_uV: dmcfreq->volt); |
185 | goto out; |
186 | } |
187 | |
188 | /* |
189 | * Check the dpll rate, |
190 | * There only two result we will get, |
191 | * 1. Ddr frequency scaling fail, we still get the old rate. |
192 | * 2. Ddr frequency scaling sucessful, we get the rate we set. |
193 | */ |
194 | dmcfreq->rate = clk_get_rate(clk: dmcfreq->dmc_clk); |
195 | |
196 | /* If get the incorrect rate, set voltage to old value. */ |
197 | if (dmcfreq->rate != target_rate) { |
198 | dev_err(dev, "Got wrong frequency, Request %lu, Current %lu\n" , |
199 | target_rate, dmcfreq->rate); |
200 | regulator_set_voltage(regulator: dmcfreq->vdd_center, min_uV: dmcfreq->volt, |
201 | max_uV: dmcfreq->volt); |
202 | goto out; |
203 | } else if (old_clk_rate > target_rate) |
204 | err = regulator_set_voltage(regulator: dmcfreq->vdd_center, min_uV: target_volt, |
205 | max_uV: target_volt); |
206 | if (err) |
207 | dev_err(dev, "Cannot set voltage %lu uV\n" , target_volt); |
208 | |
209 | dmcfreq->rate = target_rate; |
210 | dmcfreq->volt = target_volt; |
211 | |
212 | out: |
213 | rockchip_pmu_unblock(); |
214 | out_unlock: |
215 | mutex_unlock(lock: &dmcfreq->lock); |
216 | return err; |
217 | } |
218 | |
219 | static int rk3399_dmcfreq_get_dev_status(struct device *dev, |
220 | struct devfreq_dev_status *stat) |
221 | { |
222 | struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev); |
223 | struct devfreq_event_data edata; |
224 | int ret = 0; |
225 | |
226 | ret = devfreq_event_get_event(edev: dmcfreq->edev, edata: &edata); |
227 | if (ret < 0) |
228 | return ret; |
229 | |
230 | stat->current_frequency = dmcfreq->rate; |
231 | stat->busy_time = edata.load_count; |
232 | stat->total_time = edata.total_count; |
233 | |
234 | return ret; |
235 | } |
236 | |
237 | static int rk3399_dmcfreq_get_cur_freq(struct device *dev, unsigned long *freq) |
238 | { |
239 | struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev); |
240 | |
241 | *freq = dmcfreq->rate; |
242 | |
243 | return 0; |
244 | } |
245 | |
246 | static __maybe_unused int rk3399_dmcfreq_suspend(struct device *dev) |
247 | { |
248 | struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev); |
249 | int ret = 0; |
250 | |
251 | ret = devfreq_event_disable_edev(edev: dmcfreq->edev); |
252 | if (ret < 0) { |
253 | dev_err(dev, "failed to disable the devfreq-event devices\n" ); |
254 | return ret; |
255 | } |
256 | |
257 | ret = devfreq_suspend_device(devfreq: dmcfreq->devfreq); |
258 | if (ret < 0) { |
259 | dev_err(dev, "failed to suspend the devfreq devices\n" ); |
260 | return ret; |
261 | } |
262 | |
263 | return 0; |
264 | } |
265 | |
266 | static __maybe_unused int rk3399_dmcfreq_resume(struct device *dev) |
267 | { |
268 | struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev); |
269 | int ret = 0; |
270 | |
271 | ret = devfreq_event_enable_edev(edev: dmcfreq->edev); |
272 | if (ret < 0) { |
273 | dev_err(dev, "failed to enable the devfreq-event devices\n" ); |
274 | return ret; |
275 | } |
276 | |
277 | ret = devfreq_resume_device(devfreq: dmcfreq->devfreq); |
278 | if (ret < 0) { |
279 | dev_err(dev, "failed to resume the devfreq devices\n" ); |
280 | return ret; |
281 | } |
282 | return ret; |
283 | } |
284 | |
285 | static SIMPLE_DEV_PM_OPS(rk3399_dmcfreq_pm, rk3399_dmcfreq_suspend, |
286 | rk3399_dmcfreq_resume); |
287 | |
288 | static int rk3399_dmcfreq_of_props(struct rk3399_dmcfreq *data, |
289 | struct device_node *np) |
290 | { |
291 | int ret = 0; |
292 | |
293 | /* |
294 | * These are all optional, and serve as minimum bounds. Give them large |
295 | * (i.e., never "disabled") values if the DT doesn't specify one. |
296 | */ |
297 | data->pd_idle_dis_freq = |
298 | data->sr_idle_dis_freq = |
299 | data->sr_mc_gate_idle_dis_freq = |
300 | data->srpd_lite_idle_dis_freq = |
301 | data->standby_idle_dis_freq = UINT_MAX; |
302 | |
303 | ret |= of_property_read_u32(np, propname: "rockchip,pd-idle-ns" , |
304 | out_value: &data->pd_idle_ns); |
305 | ret |= of_property_read_u32(np, propname: "rockchip,sr-idle-ns" , |
306 | out_value: &data->sr_idle_ns); |
307 | ret |= of_property_read_u32(np, propname: "rockchip,sr-mc-gate-idle-ns" , |
308 | out_value: &data->sr_mc_gate_idle_ns); |
309 | ret |= of_property_read_u32(np, propname: "rockchip,srpd-lite-idle-ns" , |
310 | out_value: &data->srpd_lite_idle_ns); |
311 | ret |= of_property_read_u32(np, propname: "rockchip,standby-idle-ns" , |
312 | out_value: &data->standby_idle_ns); |
313 | ret |= of_property_read_u32(np, propname: "rockchip,ddr3_odt_dis_freq" , |
314 | out_value: &data->ddr3_odt_dis_freq); |
315 | ret |= of_property_read_u32(np, propname: "rockchip,lpddr3_odt_dis_freq" , |
316 | out_value: &data->lpddr3_odt_dis_freq); |
317 | ret |= of_property_read_u32(np, propname: "rockchip,lpddr4_odt_dis_freq" , |
318 | out_value: &data->lpddr4_odt_dis_freq); |
319 | |
320 | ret |= of_property_read_u32(np, propname: "rockchip,pd-idle-dis-freq-hz" , |
321 | out_value: &data->pd_idle_dis_freq); |
322 | ret |= of_property_read_u32(np, propname: "rockchip,sr-idle-dis-freq-hz" , |
323 | out_value: &data->sr_idle_dis_freq); |
324 | ret |= of_property_read_u32(np, propname: "rockchip,sr-mc-gate-idle-dis-freq-hz" , |
325 | out_value: &data->sr_mc_gate_idle_dis_freq); |
326 | ret |= of_property_read_u32(np, propname: "rockchip,srpd-lite-idle-dis-freq-hz" , |
327 | out_value: &data->srpd_lite_idle_dis_freq); |
328 | ret |= of_property_read_u32(np, propname: "rockchip,standby-idle-dis-freq-hz" , |
329 | out_value: &data->standby_idle_dis_freq); |
330 | |
331 | return ret; |
332 | } |
333 | |
334 | static int rk3399_dmcfreq_probe(struct platform_device *pdev) |
335 | { |
336 | struct arm_smccc_res res; |
337 | struct device *dev = &pdev->dev; |
338 | struct device_node *np = pdev->dev.of_node, *node; |
339 | struct rk3399_dmcfreq *data; |
340 | int ret; |
341 | struct dev_pm_opp *opp; |
342 | u32 ddr_type; |
343 | u32 val; |
344 | |
345 | data = devm_kzalloc(dev, size: sizeof(struct rk3399_dmcfreq), GFP_KERNEL); |
346 | if (!data) |
347 | return -ENOMEM; |
348 | |
349 | mutex_init(&data->lock); |
350 | |
351 | data->vdd_center = devm_regulator_get(dev, id: "center" ); |
352 | if (IS_ERR(ptr: data->vdd_center)) |
353 | return dev_err_probe(dev, err: PTR_ERR(ptr: data->vdd_center), |
354 | fmt: "Cannot get the regulator \"center\"\n" ); |
355 | |
356 | data->dmc_clk = devm_clk_get(dev, id: "dmc_clk" ); |
357 | if (IS_ERR(ptr: data->dmc_clk)) |
358 | return dev_err_probe(dev, err: PTR_ERR(ptr: data->dmc_clk), |
359 | fmt: "Cannot get the clk dmc_clk\n" ); |
360 | |
361 | data->edev = devfreq_event_get_edev_by_phandle(dev, phandle_name: "devfreq-events" , index: 0); |
362 | if (IS_ERR(ptr: data->edev)) |
363 | return -EPROBE_DEFER; |
364 | |
365 | ret = devfreq_event_enable_edev(edev: data->edev); |
366 | if (ret < 0) { |
367 | dev_err(dev, "failed to enable devfreq-event devices\n" ); |
368 | return ret; |
369 | } |
370 | |
371 | rk3399_dmcfreq_of_props(data, np); |
372 | |
373 | node = of_parse_phandle(np, phandle_name: "rockchip,pmu" , index: 0); |
374 | if (!node) |
375 | goto no_pmu; |
376 | |
377 | data->regmap_pmu = syscon_node_to_regmap(np: node); |
378 | of_node_put(node); |
379 | if (IS_ERR(ptr: data->regmap_pmu)) { |
380 | ret = PTR_ERR(ptr: data->regmap_pmu); |
381 | goto err_edev; |
382 | } |
383 | |
384 | regmap_read(map: data->regmap_pmu, RK3399_PMUGRF_OS_REG2, val: &val); |
385 | ddr_type = FIELD_GET(RK3399_PMUGRF_OS_REG2_DDRTYPE, val); |
386 | |
387 | switch (ddr_type) { |
388 | case ROCKCHIP_DDRTYPE_DDR3: |
389 | data->odt_dis_freq = data->ddr3_odt_dis_freq; |
390 | break; |
391 | case ROCKCHIP_DDRTYPE_LPDDR3: |
392 | data->odt_dis_freq = data->lpddr3_odt_dis_freq; |
393 | break; |
394 | case ROCKCHIP_DDRTYPE_LPDDR4: |
395 | data->odt_dis_freq = data->lpddr4_odt_dis_freq; |
396 | break; |
397 | default: |
398 | ret = -EINVAL; |
399 | goto err_edev; |
400 | } |
401 | |
402 | no_pmu: |
403 | arm_smccc_smc(ROCKCHIP_SIP_DRAM_FREQ, 0, 0, |
404 | ROCKCHIP_SIP_CONFIG_DRAM_INIT, |
405 | 0, 0, 0, 0, &res); |
406 | |
407 | /* |
408 | * We add a devfreq driver to our parent since it has a device tree node |
409 | * with operating points. |
410 | */ |
411 | if (devm_pm_opp_of_add_table(dev)) { |
412 | dev_err(dev, "Invalid operating-points in device tree.\n" ); |
413 | ret = -EINVAL; |
414 | goto err_edev; |
415 | } |
416 | |
417 | data->ondemand_data.upthreshold = 25; |
418 | data->ondemand_data.downdifferential = 15; |
419 | |
420 | data->rate = clk_get_rate(clk: data->dmc_clk); |
421 | |
422 | opp = devfreq_recommended_opp(dev, freq: &data->rate, flags: 0); |
423 | if (IS_ERR(ptr: opp)) { |
424 | ret = PTR_ERR(ptr: opp); |
425 | goto err_edev; |
426 | } |
427 | |
428 | data->rate = dev_pm_opp_get_freq(opp); |
429 | data->volt = dev_pm_opp_get_voltage(opp); |
430 | dev_pm_opp_put(opp); |
431 | |
432 | data->profile = (struct devfreq_dev_profile) { |
433 | .polling_ms = 200, |
434 | .target = rk3399_dmcfreq_target, |
435 | .get_dev_status = rk3399_dmcfreq_get_dev_status, |
436 | .get_cur_freq = rk3399_dmcfreq_get_cur_freq, |
437 | .initial_freq = data->rate, |
438 | }; |
439 | |
440 | data->devfreq = devm_devfreq_add_device(dev, |
441 | profile: &data->profile, |
442 | DEVFREQ_GOV_SIMPLE_ONDEMAND, |
443 | data: &data->ondemand_data); |
444 | if (IS_ERR(ptr: data->devfreq)) { |
445 | ret = PTR_ERR(ptr: data->devfreq); |
446 | goto err_edev; |
447 | } |
448 | |
449 | devm_devfreq_register_opp_notifier(dev, devfreq: data->devfreq); |
450 | |
451 | data->dev = dev; |
452 | platform_set_drvdata(pdev, data); |
453 | |
454 | return 0; |
455 | |
456 | err_edev: |
457 | devfreq_event_disable_edev(edev: data->edev); |
458 | |
459 | return ret; |
460 | } |
461 | |
462 | static int rk3399_dmcfreq_remove(struct platform_device *pdev) |
463 | { |
464 | struct rk3399_dmcfreq *dmcfreq = dev_get_drvdata(dev: &pdev->dev); |
465 | |
466 | devfreq_event_disable_edev(edev: dmcfreq->edev); |
467 | |
468 | return 0; |
469 | } |
470 | |
471 | static const struct of_device_id rk3399dmc_devfreq_of_match[] = { |
472 | { .compatible = "rockchip,rk3399-dmc" }, |
473 | { }, |
474 | }; |
475 | MODULE_DEVICE_TABLE(of, rk3399dmc_devfreq_of_match); |
476 | |
477 | static struct platform_driver rk3399_dmcfreq_driver = { |
478 | .probe = rk3399_dmcfreq_probe, |
479 | .remove = rk3399_dmcfreq_remove, |
480 | .driver = { |
481 | .name = "rk3399-dmc-freq" , |
482 | .pm = &rk3399_dmcfreq_pm, |
483 | .of_match_table = rk3399dmc_devfreq_of_match, |
484 | }, |
485 | }; |
486 | module_platform_driver(rk3399_dmcfreq_driver); |
487 | |
488 | MODULE_LICENSE("GPL v2" ); |
489 | MODULE_AUTHOR("Lin Huang <hl@rock-chips.com>" ); |
490 | MODULE_DESCRIPTION("RK3399 dmcfreq driver with devfreq framework" ); |
491 | |