1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (c) 2019 Samsung Electronics Co., Ltd.
4 * Author: Lukasz Luba <l.luba@partner.samsung.com>
5 */
6
7#include <linux/clk.h>
8#include <linux/devfreq.h>
9#include <linux/devfreq-event.h>
10#include <linux/device.h>
11#include <linux/interrupt.h>
12#include <linux/io.h>
13#include <linux/mfd/syscon.h>
14#include <linux/module.h>
15#include <linux/moduleparam.h>
16#include <linux/of.h>
17#include <linux/pm_opp.h>
18#include <linux/platform_device.h>
19#include <linux/regmap.h>
20#include <linux/regulator/consumer.h>
21#include <linux/slab.h>
22#include "../jedec_ddr.h"
23#include "../of_memory.h"
24
25static int irqmode;
26module_param(irqmode, int, 0644);
27MODULE_PARM_DESC(irqmode, "Enable IRQ mode (0=off [default], 1=on)");
28
29#define EXYNOS5_DREXI_TIMINGAREF (0x0030)
30#define EXYNOS5_DREXI_TIMINGROW0 (0x0034)
31#define EXYNOS5_DREXI_TIMINGDATA0 (0x0038)
32#define EXYNOS5_DREXI_TIMINGPOWER0 (0x003C)
33#define EXYNOS5_DREXI_TIMINGROW1 (0x00E4)
34#define EXYNOS5_DREXI_TIMINGDATA1 (0x00E8)
35#define EXYNOS5_DREXI_TIMINGPOWER1 (0x00EC)
36#define CDREX_PAUSE (0x2091c)
37#define CDREX_LPDDR3PHY_CON3 (0x20a20)
38#define CDREX_LPDDR3PHY_CLKM_SRC (0x20700)
39#define EXYNOS5_TIMING_SET_SWI BIT(28)
40#define USE_MX_MSPLL_TIMINGS (1)
41#define USE_BPLL_TIMINGS (0)
42#define EXYNOS5_AREF_NORMAL (0x2e)
43
44#define DREX_PPCCLKCON (0x0130)
45#define DREX_PEREV2CONFIG (0x013c)
46#define DREX_PMNC_PPC (0xE000)
47#define DREX_CNTENS_PPC (0xE010)
48#define DREX_CNTENC_PPC (0xE020)
49#define DREX_INTENS_PPC (0xE030)
50#define DREX_INTENC_PPC (0xE040)
51#define DREX_FLAG_PPC (0xE050)
52#define DREX_PMCNT2_PPC (0xE130)
53
54/*
55 * A value for register DREX_PMNC_PPC which should be written to reset
56 * the cycle counter CCNT (a reference wall clock). It sets zero to the
57 * CCNT counter.
58 */
59#define CC_RESET BIT(2)
60
61/*
62 * A value for register DREX_PMNC_PPC which does the reset of all performance
63 * counters to zero.
64 */
65#define PPC_COUNTER_RESET BIT(1)
66
67/*
68 * Enables all configured counters (including cycle counter). The value should
69 * be written to the register DREX_PMNC_PPC.
70 */
71#define PPC_ENABLE BIT(0)
72
73/* A value for register DREX_PPCCLKCON which enables performance events clock.
74 * Must be written before first access to the performance counters register
75 * set, otherwise it could crash.
76 */
77#define PEREV_CLK_EN BIT(0)
78
79/*
80 * Values which are used to enable counters, interrupts or configure flags of
81 * the performance counters. They configure counter 2 and cycle counter.
82 */
83#define PERF_CNT2 BIT(2)
84#define PERF_CCNT BIT(31)
85
86/*
87 * Performance event types which are used for setting the preferred event
88 * to track in the counters.
89 * There is a set of different types, the values are from range 0 to 0x6f.
90 * These settings should be written to the configuration register which manages
91 * the type of the event (register DREX_PEREV2CONFIG).
92 */
93#define READ_TRANSFER_CH0 (0x6d)
94#define READ_TRANSFER_CH1 (0x6f)
95
96#define PERF_COUNTER_START_VALUE 0xff000000
97#define PERF_EVENT_UP_DOWN_THRESHOLD 900000000ULL
98
99/**
100 * struct dmc_opp_table - Operating level desciption
101 * @freq_hz: target frequency in Hz
102 * @volt_uv: target voltage in uV
103 *
104 * Covers frequency and voltage settings of the DMC operating mode.
105 */
106struct dmc_opp_table {
107 u32 freq_hz;
108 u32 volt_uv;
109};
110
111/**
112 * struct exynos5_dmc - main structure describing DMC device
113 * @dev: DMC device
114 * @df: devfreq device structure returned by devfreq framework
115 * @gov_data: configuration of devfreq governor
116 * @base_drexi0: DREX0 registers mapping
117 * @base_drexi1: DREX1 registers mapping
118 * @clk_regmap: regmap for clock controller registers
119 * @lock: protects curr_rate and frequency/voltage setting section
120 * @curr_rate: current frequency
121 * @curr_volt: current voltage
122 * @opp: OPP table
123 * @opp_count: number of 'opp' elements
124 * @timings_arr_size: number of 'timings' elements
125 * @timing_row: values for timing row register, for each OPP
126 * @timing_data: values for timing data register, for each OPP
127 * @timing_power: balues for timing power register, for each OPP
128 * @timings: DDR memory timings, from device tree
129 * @min_tck: DDR memory minimum timing values, from device tree
130 * @bypass_timing_row: value for timing row register for bypass timings
131 * @bypass_timing_data: value for timing data register for bypass timings
132 * @bypass_timing_power: value for timing power register for bypass
133 * timings
134 * @vdd_mif: Memory interface regulator
135 * @fout_spll: clock: SPLL
136 * @fout_bpll: clock: BPLL
137 * @mout_spll: clock: mux SPLL
138 * @mout_bpll: clock: mux BPLL
139 * @mout_mclk_cdrex: clock: mux mclk_cdrex
140 * @mout_mx_mspll_ccore: clock: mux mx_mspll_ccore
141 * @counter: devfreq events
142 * @num_counters: number of 'counter' elements
143 * @last_overflow_ts: time (in ns) of last overflow of each DREX
144 * @load: utilization in percents
145 * @total: total time between devfreq events
146 * @in_irq_mode: whether running in interrupt mode (true)
147 * or polling (false)
148 *
149 * The main structure for the Dynamic Memory Controller which covers clocks,
150 * memory regions, HW information, parameters and current operating mode.
151 */
152struct exynos5_dmc {
153 struct device *dev;
154 struct devfreq *df;
155 struct devfreq_simple_ondemand_data gov_data;
156 void __iomem *base_drexi0;
157 void __iomem *base_drexi1;
158 struct regmap *clk_regmap;
159 /* Protects curr_rate and frequency/voltage setting section */
160 struct mutex lock;
161 unsigned long curr_rate;
162 unsigned long curr_volt;
163 struct dmc_opp_table *opp;
164 int opp_count;
165 u32 timings_arr_size;
166 u32 *timing_row;
167 u32 *timing_data;
168 u32 *timing_power;
169 const struct lpddr3_timings *timings;
170 const struct lpddr3_min_tck *min_tck;
171 u32 bypass_timing_row;
172 u32 bypass_timing_data;
173 u32 bypass_timing_power;
174 struct regulator *vdd_mif;
175 struct clk *fout_spll;
176 struct clk *fout_bpll;
177 struct clk *mout_spll;
178 struct clk *mout_bpll;
179 struct clk *mout_mclk_cdrex;
180 struct clk *mout_mx_mspll_ccore;
181 struct devfreq_event_dev **counter;
182 int num_counters;
183 u64 last_overflow_ts[2];
184 unsigned long load;
185 unsigned long total;
186 bool in_irq_mode;
187};
188
189#define TIMING_FIELD(t_name, t_bit_beg, t_bit_end) \
190 { .name = t_name, .bit_beg = t_bit_beg, .bit_end = t_bit_end }
191
192#define TIMING_VAL2REG(timing, t_val) \
193({ \
194 u32 __val; \
195 __val = (t_val) << (timing)->bit_beg; \
196 __val; \
197})
198
199struct timing_reg {
200 char *name;
201 int bit_beg;
202 int bit_end;
203 unsigned int val;
204};
205
206static const struct timing_reg timing_row_reg_fields[] = {
207 TIMING_FIELD("tRFC", 24, 31),
208 TIMING_FIELD("tRRD", 20, 23),
209 TIMING_FIELD("tRP", 16, 19),
210 TIMING_FIELD("tRCD", 12, 15),
211 TIMING_FIELD("tRC", 6, 11),
212 TIMING_FIELD("tRAS", 0, 5),
213};
214
215static const struct timing_reg timing_data_reg_fields[] = {
216 TIMING_FIELD("tWTR", 28, 31),
217 TIMING_FIELD("tWR", 24, 27),
218 TIMING_FIELD("tRTP", 20, 23),
219 TIMING_FIELD("tW2W-C2C", 14, 14),
220 TIMING_FIELD("tR2R-C2C", 12, 12),
221 TIMING_FIELD("WL", 8, 11),
222 TIMING_FIELD("tDQSCK", 4, 7),
223 TIMING_FIELD("RL", 0, 3),
224};
225
226static const struct timing_reg timing_power_reg_fields[] = {
227 TIMING_FIELD("tFAW", 26, 31),
228 TIMING_FIELD("tXSR", 16, 25),
229 TIMING_FIELD("tXP", 8, 15),
230 TIMING_FIELD("tCKE", 4, 7),
231 TIMING_FIELD("tMRD", 0, 3),
232};
233
234#define TIMING_COUNT (ARRAY_SIZE(timing_row_reg_fields) + \
235 ARRAY_SIZE(timing_data_reg_fields) + \
236 ARRAY_SIZE(timing_power_reg_fields))
237
238static int exynos5_counters_set_event(struct exynos5_dmc *dmc)
239{
240 int i, ret;
241
242 for (i = 0; i < dmc->num_counters; i++) {
243 if (!dmc->counter[i])
244 continue;
245 ret = devfreq_event_set_event(edev: dmc->counter[i]);
246 if (ret < 0)
247 return ret;
248 }
249 return 0;
250}
251
252static int exynos5_counters_enable_edev(struct exynos5_dmc *dmc)
253{
254 int i, ret;
255
256 for (i = 0; i < dmc->num_counters; i++) {
257 if (!dmc->counter[i])
258 continue;
259 ret = devfreq_event_enable_edev(edev: dmc->counter[i]);
260 if (ret < 0)
261 return ret;
262 }
263 return 0;
264}
265
266static int exynos5_counters_disable_edev(struct exynos5_dmc *dmc)
267{
268 int i, ret;
269
270 for (i = 0; i < dmc->num_counters; i++) {
271 if (!dmc->counter[i])
272 continue;
273 ret = devfreq_event_disable_edev(edev: dmc->counter[i]);
274 if (ret < 0)
275 return ret;
276 }
277 return 0;
278}
279
280/**
281 * find_target_freq_idx() - Finds requested frequency in local DMC configuration
282 * @dmc: device for which the information is checked
283 * @target_rate: requested frequency in KHz
284 *
285 * Seeks in the local DMC driver structure for the requested frequency value
286 * and returns index or error value.
287 */
288static int find_target_freq_idx(struct exynos5_dmc *dmc,
289 unsigned long target_rate)
290{
291 int i;
292
293 for (i = dmc->opp_count - 1; i >= 0; i--)
294 if (dmc->opp[i].freq_hz <= target_rate)
295 return i;
296
297 return -EINVAL;
298}
299
300/**
301 * exynos5_switch_timing_regs() - Changes bank register set for DRAM timings
302 * @dmc: device for which the new settings is going to be applied
303 * @set: boolean variable passing set value
304 *
305 * Changes the register set, which holds timing parameters.
306 * There is two register sets: 0 and 1. The register set 0
307 * is used in normal operation when the clock is provided from main PLL.
308 * The bank register set 1 is used when the main PLL frequency is going to be
309 * changed and the clock is taken from alternative, stable source.
310 * This function switches between these banks according to the
311 * currently used clock source.
312 */
313static int exynos5_switch_timing_regs(struct exynos5_dmc *dmc, bool set)
314{
315 unsigned int reg;
316 int ret;
317
318 ret = regmap_read(map: dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, val: &reg);
319 if (ret)
320 return ret;
321
322 if (set)
323 reg |= EXYNOS5_TIMING_SET_SWI;
324 else
325 reg &= ~EXYNOS5_TIMING_SET_SWI;
326
327 regmap_write(map: dmc->clk_regmap, CDREX_LPDDR3PHY_CON3, val: reg);
328
329 return 0;
330}
331
332/**
333 * exynos5_init_freq_table() - Initialized PM OPP framework
334 * @dmc: DMC device for which the frequencies are used for OPP init
335 * @profile: devfreq device's profile
336 *
337 * Populate the devfreq device's OPP table based on current frequency, voltage.
338 */
339static int exynos5_init_freq_table(struct exynos5_dmc *dmc,
340 struct devfreq_dev_profile *profile)
341{
342 int i, ret;
343 int idx;
344 unsigned long freq;
345
346 ret = devm_pm_opp_of_add_table(dev: dmc->dev);
347 if (ret < 0) {
348 dev_err(dmc->dev, "Failed to get OPP table\n");
349 return ret;
350 }
351
352 dmc->opp_count = dev_pm_opp_get_opp_count(dev: dmc->dev);
353
354 dmc->opp = devm_kmalloc_array(dev: dmc->dev, n: dmc->opp_count,
355 size: sizeof(struct dmc_opp_table), GFP_KERNEL);
356 if (!dmc->opp)
357 return -ENOMEM;
358
359 idx = dmc->opp_count - 1;
360 for (i = 0, freq = ULONG_MAX; i < dmc->opp_count; i++, freq--) {
361 struct dev_pm_opp *opp;
362
363 opp = dev_pm_opp_find_freq_floor(dev: dmc->dev, freq: &freq);
364 if (IS_ERR(ptr: opp))
365 return PTR_ERR(ptr: opp);
366
367 dmc->opp[idx - i].freq_hz = freq;
368 dmc->opp[idx - i].volt_uv = dev_pm_opp_get_voltage(opp);
369
370 dev_pm_opp_put(opp);
371 }
372
373 return 0;
374}
375
376/**
377 * exynos5_set_bypass_dram_timings() - Low-level changes of the DRAM timings
378 * @dmc: device for which the new settings is going to be applied
379 *
380 * Low-level function for changing timings for DRAM memory clocking from
381 * 'bypass' clock source (fixed frequency @400MHz).
382 * It uses timing bank registers set 1.
383 */
384static void exynos5_set_bypass_dram_timings(struct exynos5_dmc *dmc)
385{
386 writel(EXYNOS5_AREF_NORMAL,
387 addr: dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGAREF);
388
389 writel(val: dmc->bypass_timing_row,
390 addr: dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGROW1);
391 writel(val: dmc->bypass_timing_row,
392 addr: dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGROW1);
393 writel(val: dmc->bypass_timing_data,
394 addr: dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGDATA1);
395 writel(val: dmc->bypass_timing_data,
396 addr: dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGDATA1);
397 writel(val: dmc->bypass_timing_power,
398 addr: dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGPOWER1);
399 writel(val: dmc->bypass_timing_power,
400 addr: dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGPOWER1);
401}
402
403/**
404 * exynos5_dram_change_timings() - Low-level changes of the DRAM final timings
405 * @dmc: device for which the new settings is going to be applied
406 * @target_rate: target frequency of the DMC
407 *
408 * Low-level function for changing timings for DRAM memory operating from main
409 * clock source (BPLL), which can have different frequencies. Thus, each
410 * frequency must have corresponding timings register values in order to keep
411 * the needed delays.
412 * It uses timing bank registers set 0.
413 */
414static int exynos5_dram_change_timings(struct exynos5_dmc *dmc,
415 unsigned long target_rate)
416{
417 int idx;
418
419 for (idx = dmc->opp_count - 1; idx >= 0; idx--)
420 if (dmc->opp[idx].freq_hz <= target_rate)
421 break;
422
423 if (idx < 0)
424 return -EINVAL;
425
426 writel(EXYNOS5_AREF_NORMAL,
427 addr: dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGAREF);
428
429 writel(val: dmc->timing_row[idx],
430 addr: dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGROW0);
431 writel(val: dmc->timing_row[idx],
432 addr: dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGROW0);
433 writel(val: dmc->timing_data[idx],
434 addr: dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGDATA0);
435 writel(val: dmc->timing_data[idx],
436 addr: dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGDATA0);
437 writel(val: dmc->timing_power[idx],
438 addr: dmc->base_drexi0 + EXYNOS5_DREXI_TIMINGPOWER0);
439 writel(val: dmc->timing_power[idx],
440 addr: dmc->base_drexi1 + EXYNOS5_DREXI_TIMINGPOWER0);
441
442 return 0;
443}
444
445/**
446 * exynos5_dmc_align_target_voltage() - Sets the final voltage for the DMC
447 * @dmc: device for which it is going to be set
448 * @target_volt: new voltage which is chosen to be final
449 *
450 * Function tries to align voltage to the safe level for 'normal' mode.
451 * It checks the need of higher voltage and changes the value. The target
452 * voltage might be lower that currently set and still the system will be
453 * stable.
454 */
455static int exynos5_dmc_align_target_voltage(struct exynos5_dmc *dmc,
456 unsigned long target_volt)
457{
458 int ret = 0;
459
460 if (dmc->curr_volt <= target_volt)
461 return 0;
462
463 ret = regulator_set_voltage(regulator: dmc->vdd_mif, min_uV: target_volt,
464 max_uV: target_volt);
465 if (!ret)
466 dmc->curr_volt = target_volt;
467
468 return ret;
469}
470
471/**
472 * exynos5_dmc_align_bypass_voltage() - Sets the voltage for the DMC
473 * @dmc: device for which it is going to be set
474 * @target_volt: new voltage which is chosen to be final
475 *
476 * Function tries to align voltage to the safe level for the 'bypass' mode.
477 * It checks the need of higher voltage and changes the value.
478 * The target voltage must not be less than currently needed, because
479 * for current frequency the device might become unstable.
480 */
481static int exynos5_dmc_align_bypass_voltage(struct exynos5_dmc *dmc,
482 unsigned long target_volt)
483{
484 int ret = 0;
485
486 if (dmc->curr_volt >= target_volt)
487 return 0;
488
489 ret = regulator_set_voltage(regulator: dmc->vdd_mif, min_uV: target_volt,
490 max_uV: target_volt);
491 if (!ret)
492 dmc->curr_volt = target_volt;
493
494 return ret;
495}
496
497/**
498 * exynos5_dmc_align_bypass_dram_timings() - Chooses and sets DRAM timings
499 * @dmc: device for which it is going to be set
500 * @target_rate: new frequency which is chosen to be final
501 *
502 * Function changes the DRAM timings for the temporary 'bypass' mode.
503 */
504static int exynos5_dmc_align_bypass_dram_timings(struct exynos5_dmc *dmc,
505 unsigned long target_rate)
506{
507 int idx = find_target_freq_idx(dmc, target_rate);
508
509 if (idx < 0)
510 return -EINVAL;
511
512 exynos5_set_bypass_dram_timings(dmc);
513
514 return 0;
515}
516
517/**
518 * exynos5_dmc_switch_to_bypass_configuration() - Switching to temporary clock
519 * @dmc: DMC device for which the switching is going to happen
520 * @target_rate: new frequency which is going to be set as a final
521 * @target_volt: new voltage which is going to be set as a final
522 *
523 * Function configures DMC and clocks for operating in temporary 'bypass' mode.
524 * This mode is used only temporary but if required, changes voltage and timings
525 * for DRAM chips. It switches the main clock to stable clock source for the
526 * period of the main PLL reconfiguration.
527 */
528static int
529exynos5_dmc_switch_to_bypass_configuration(struct exynos5_dmc *dmc,
530 unsigned long target_rate,
531 unsigned long target_volt)
532{
533 int ret;
534
535 /*
536 * Having higher voltage for a particular frequency does not harm
537 * the chip. Use it for the temporary frequency change when one
538 * voltage manipulation might be avoided.
539 */
540 ret = exynos5_dmc_align_bypass_voltage(dmc, target_volt);
541 if (ret)
542 return ret;
543
544 /*
545 * Longer delays for DRAM does not cause crash, the opposite does.
546 */
547 ret = exynos5_dmc_align_bypass_dram_timings(dmc, target_rate);
548 if (ret)
549 return ret;
550
551 /*
552 * Delays are long enough, so use them for the new coming clock.
553 */
554 ret = exynos5_switch_timing_regs(dmc, USE_MX_MSPLL_TIMINGS);
555
556 return ret;
557}
558
559/**
560 * exynos5_dmc_change_freq_and_volt() - Changes voltage and frequency of the DMC
561 * using safe procedure
562 * @dmc: device for which the frequency is going to be changed
563 * @target_rate: requested new frequency
564 * @target_volt: requested voltage which corresponds to the new frequency
565 *
566 * The DMC frequency change procedure requires a few steps.
567 * The main requirement is to change the clock source in the clk mux
568 * for the time of main clock PLL locking. The assumption is that the
569 * alternative clock source set as parent is stable.
570 * The second parent's clock frequency is fixed to 400MHz, it is named 'bypass'
571 * clock. This requires alignment in DRAM timing parameters for the new
572 * T-period. There is two bank sets for keeping DRAM
573 * timings: set 0 and set 1. The set 0 is used when main clock source is
574 * chosen. The 2nd set of regs is used for 'bypass' clock. Switching between
575 * the two bank sets is part of the process.
576 * The voltage must also be aligned to the minimum required level. There is
577 * this intermediate step with switching to 'bypass' parent clock source.
578 * if the old voltage is lower, it requires an increase of the voltage level.
579 * The complexity of the voltage manipulation is hidden in low level function.
580 * In this function there is last alignment of the voltage level at the end.
581 */
582static int
583exynos5_dmc_change_freq_and_volt(struct exynos5_dmc *dmc,
584 unsigned long target_rate,
585 unsigned long target_volt)
586{
587 int ret;
588
589 ret = exynos5_dmc_switch_to_bypass_configuration(dmc, target_rate,
590 target_volt);
591 if (ret)
592 return ret;
593
594 /*
595 * Voltage is set at least to a level needed for this frequency,
596 * so switching clock source is safe now.
597 */
598 clk_prepare_enable(clk: dmc->fout_spll);
599 clk_prepare_enable(clk: dmc->mout_spll);
600 clk_prepare_enable(clk: dmc->mout_mx_mspll_ccore);
601
602 ret = clk_set_parent(clk: dmc->mout_mclk_cdrex, parent: dmc->mout_mx_mspll_ccore);
603 if (ret)
604 goto disable_clocks;
605
606 /*
607 * We are safe to increase the timings for current bypass frequency.
608 * Thanks to this the settings will be ready for the upcoming clock
609 * source change.
610 */
611 exynos5_dram_change_timings(dmc, target_rate);
612
613 clk_set_rate(clk: dmc->fout_bpll, rate: target_rate);
614
615 ret = exynos5_switch_timing_regs(dmc, USE_BPLL_TIMINGS);
616 if (ret)
617 goto disable_clocks;
618
619 ret = clk_set_parent(clk: dmc->mout_mclk_cdrex, parent: dmc->mout_bpll);
620 if (ret)
621 goto disable_clocks;
622
623 /*
624 * Make sure if the voltage is not from 'bypass' settings and align to
625 * the right level for power efficiency.
626 */
627 ret = exynos5_dmc_align_target_voltage(dmc, target_volt);
628
629disable_clocks:
630 clk_disable_unprepare(clk: dmc->mout_mx_mspll_ccore);
631 clk_disable_unprepare(clk: dmc->mout_spll);
632 clk_disable_unprepare(clk: dmc->fout_spll);
633
634 return ret;
635}
636
637/**
638 * exynos5_dmc_get_volt_freq() - Gets the frequency and voltage from the OPP
639 * table.
640 * @dmc: device for which the frequency is going to be changed
641 * @freq: requested frequency in KHz
642 * @target_rate: returned frequency which is the same or lower than
643 * requested
644 * @target_volt: returned voltage which corresponds to the returned
645 * frequency
646 * @flags: devfreq flags provided for this frequency change request
647 *
648 * Function gets requested frequency and checks OPP framework for needed
649 * frequency and voltage. It populates the values 'target_rate' and
650 * 'target_volt' or returns error value when OPP framework fails.
651 */
652static int exynos5_dmc_get_volt_freq(struct exynos5_dmc *dmc,
653 unsigned long *freq,
654 unsigned long *target_rate,
655 unsigned long *target_volt, u32 flags)
656{
657 struct dev_pm_opp *opp;
658
659 opp = devfreq_recommended_opp(dev: dmc->dev, freq, flags);
660 if (IS_ERR(ptr: opp))
661 return PTR_ERR(ptr: opp);
662
663 *target_rate = dev_pm_opp_get_freq(opp);
664 *target_volt = dev_pm_opp_get_voltage(opp);
665 dev_pm_opp_put(opp);
666
667 return 0;
668}
669
670/**
671 * exynos5_dmc_target() - Function responsible for changing frequency of DMC
672 * @dev: device for which the frequency is going to be changed
673 * @freq: requested frequency in KHz
674 * @flags: flags provided for this frequency change request
675 *
676 * An entry function provided to the devfreq framework which provides frequency
677 * change of the DMC. The function gets the possible rate from OPP table based
678 * on requested frequency. It calls the next function responsible for the
679 * frequency and voltage change. In case of failure, does not set 'curr_rate'
680 * and returns error value to the framework.
681 */
682static int exynos5_dmc_target(struct device *dev, unsigned long *freq,
683 u32 flags)
684{
685 struct exynos5_dmc *dmc = dev_get_drvdata(dev);
686 unsigned long target_rate = 0;
687 unsigned long target_volt = 0;
688 int ret;
689
690 ret = exynos5_dmc_get_volt_freq(dmc, freq, target_rate: &target_rate, target_volt: &target_volt,
691 flags);
692
693 if (ret)
694 return ret;
695
696 if (target_rate == dmc->curr_rate)
697 return 0;
698
699 mutex_lock(&dmc->lock);
700
701 ret = exynos5_dmc_change_freq_and_volt(dmc, target_rate, target_volt);
702
703 if (ret) {
704 mutex_unlock(lock: &dmc->lock);
705 return ret;
706 }
707
708 dmc->curr_rate = target_rate;
709
710 mutex_unlock(lock: &dmc->lock);
711 return 0;
712}
713
714/**
715 * exynos5_counters_get() - Gets the performance counters values.
716 * @dmc: device for which the counters are going to be checked
717 * @load_count: variable which is populated with counter value
718 * @total_count: variable which is used as 'wall clock' reference
719 *
720 * Function which provides performance counters values. It sums up counters for
721 * two DMC channels. The 'total_count' is used as a reference and max value.
722 * The ratio 'load_count/total_count' shows the busy percentage [0%, 100%].
723 */
724static int exynos5_counters_get(struct exynos5_dmc *dmc,
725 unsigned long *load_count,
726 unsigned long *total_count)
727{
728 unsigned long total = 0;
729 struct devfreq_event_data event;
730 int ret, i;
731
732 *load_count = 0;
733
734 /* Take into account only read+write counters, but stop all */
735 for (i = 0; i < dmc->num_counters; i++) {
736 if (!dmc->counter[i])
737 continue;
738
739 ret = devfreq_event_get_event(edev: dmc->counter[i], edata: &event);
740 if (ret < 0)
741 return ret;
742
743 *load_count += event.load_count;
744
745 if (total < event.total_count)
746 total = event.total_count;
747 }
748
749 *total_count = total;
750
751 return 0;
752}
753
754/**
755 * exynos5_dmc_start_perf_events() - Setup and start performance event counters
756 * @dmc: device for which the counters are going to be checked
757 * @beg_value: initial value for the counter
758 *
759 * Function which enables needed counters, interrupts and sets initial values
760 * then starts the counters.
761 */
762static void exynos5_dmc_start_perf_events(struct exynos5_dmc *dmc,
763 u32 beg_value)
764{
765 /* Enable interrupts for counter 2 */
766 writel(PERF_CNT2, addr: dmc->base_drexi0 + DREX_INTENS_PPC);
767 writel(PERF_CNT2, addr: dmc->base_drexi1 + DREX_INTENS_PPC);
768
769 /* Enable counter 2 and CCNT */
770 writel(PERF_CNT2 | PERF_CCNT, addr: dmc->base_drexi0 + DREX_CNTENS_PPC);
771 writel(PERF_CNT2 | PERF_CCNT, addr: dmc->base_drexi1 + DREX_CNTENS_PPC);
772
773 /* Clear overflow flag for all counters */
774 writel(PERF_CNT2 | PERF_CCNT, addr: dmc->base_drexi0 + DREX_FLAG_PPC);
775 writel(PERF_CNT2 | PERF_CCNT, addr: dmc->base_drexi1 + DREX_FLAG_PPC);
776
777 /* Reset all counters */
778 writel(CC_RESET | PPC_COUNTER_RESET, addr: dmc->base_drexi0 + DREX_PMNC_PPC);
779 writel(CC_RESET | PPC_COUNTER_RESET, addr: dmc->base_drexi1 + DREX_PMNC_PPC);
780
781 /*
782 * Set start value for the counters, the number of samples that
783 * will be gathered is calculated as: 0xffffffff - beg_value
784 */
785 writel(val: beg_value, addr: dmc->base_drexi0 + DREX_PMCNT2_PPC);
786 writel(val: beg_value, addr: dmc->base_drexi1 + DREX_PMCNT2_PPC);
787
788 /* Start all counters */
789 writel(PPC_ENABLE, addr: dmc->base_drexi0 + DREX_PMNC_PPC);
790 writel(PPC_ENABLE, addr: dmc->base_drexi1 + DREX_PMNC_PPC);
791}
792
793/**
794 * exynos5_dmc_perf_events_calc() - Calculate utilization
795 * @dmc: device for which the counters are going to be checked
796 * @diff_ts: time between last interrupt and current one
797 *
798 * Function which calculates needed utilization for the devfreq governor.
799 * It prepares values for 'busy_time' and 'total_time' based on elapsed time
800 * between interrupts, which approximates utilization.
801 */
802static void exynos5_dmc_perf_events_calc(struct exynos5_dmc *dmc, u64 diff_ts)
803{
804 /*
805 * This is a simple algorithm for managing traffic on DMC.
806 * When there is almost no load the counters overflow every 4s,
807 * no mater the DMC frequency.
808 * The high load might be approximated using linear function.
809 * Knowing that, simple calculation can provide 'busy_time' and
810 * 'total_time' to the devfreq governor which picks up target
811 * frequency.
812 * We want a fast ramp up and slow decay in frequency change function.
813 */
814 if (diff_ts < PERF_EVENT_UP_DOWN_THRESHOLD) {
815 /*
816 * Set higher utilization for the simple_ondemand governor.
817 * The governor should increase the frequency of the DMC.
818 */
819 dmc->load = 70;
820 dmc->total = 100;
821 } else {
822 /*
823 * Set low utilization for the simple_ondemand governor.
824 * The governor should decrease the frequency of the DMC.
825 */
826 dmc->load = 35;
827 dmc->total = 100;
828 }
829
830 dev_dbg(dmc->dev, "diff_ts=%llu\n", diff_ts);
831}
832
833/**
834 * exynos5_dmc_perf_events_check() - Checks the status of the counters
835 * @dmc: device for which the counters are going to be checked
836 *
837 * Function which is called from threaded IRQ to check the counters state
838 * and to call approximation for the needed utilization.
839 */
840static void exynos5_dmc_perf_events_check(struct exynos5_dmc *dmc)
841{
842 u32 val;
843 u64 diff_ts, ts;
844
845 ts = ktime_get_ns();
846
847 /* Stop all counters */
848 writel(val: 0, addr: dmc->base_drexi0 + DREX_PMNC_PPC);
849 writel(val: 0, addr: dmc->base_drexi1 + DREX_PMNC_PPC);
850
851 /* Check the source in interrupt flag registers (which channel) */
852 val = readl(addr: dmc->base_drexi0 + DREX_FLAG_PPC);
853 if (val) {
854 diff_ts = ts - dmc->last_overflow_ts[0];
855 dmc->last_overflow_ts[0] = ts;
856 dev_dbg(dmc->dev, "drex0 0xE050 val= 0x%08x\n", val);
857 } else {
858 val = readl(addr: dmc->base_drexi1 + DREX_FLAG_PPC);
859 diff_ts = ts - dmc->last_overflow_ts[1];
860 dmc->last_overflow_ts[1] = ts;
861 dev_dbg(dmc->dev, "drex1 0xE050 val= 0x%08x\n", val);
862 }
863
864 exynos5_dmc_perf_events_calc(dmc, diff_ts);
865
866 exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE);
867}
868
869/**
870 * exynos5_dmc_enable_perf_events() - Enable performance events
871 * @dmc: device for which the counters are going to be checked
872 *
873 * Function which is setup needed environment and enables counters.
874 */
875static void exynos5_dmc_enable_perf_events(struct exynos5_dmc *dmc)
876{
877 u64 ts;
878
879 /* Enable Performance Event Clock */
880 writel(PEREV_CLK_EN, addr: dmc->base_drexi0 + DREX_PPCCLKCON);
881 writel(PEREV_CLK_EN, addr: dmc->base_drexi1 + DREX_PPCCLKCON);
882
883 /* Select read transfers as performance event2 */
884 writel(READ_TRANSFER_CH0, addr: dmc->base_drexi0 + DREX_PEREV2CONFIG);
885 writel(READ_TRANSFER_CH1, addr: dmc->base_drexi1 + DREX_PEREV2CONFIG);
886
887 ts = ktime_get_ns();
888 dmc->last_overflow_ts[0] = ts;
889 dmc->last_overflow_ts[1] = ts;
890
891 /* Devfreq shouldn't be faster than initialization, play safe though. */
892 dmc->load = 99;
893 dmc->total = 100;
894}
895
896/**
897 * exynos5_dmc_disable_perf_events() - Disable performance events
898 * @dmc: device for which the counters are going to be checked
899 *
900 * Function which stops, disables performance event counters and interrupts.
901 */
902static void exynos5_dmc_disable_perf_events(struct exynos5_dmc *dmc)
903{
904 /* Stop all counters */
905 writel(val: 0, addr: dmc->base_drexi0 + DREX_PMNC_PPC);
906 writel(val: 0, addr: dmc->base_drexi1 + DREX_PMNC_PPC);
907
908 /* Disable interrupts for counter 2 */
909 writel(PERF_CNT2, addr: dmc->base_drexi0 + DREX_INTENC_PPC);
910 writel(PERF_CNT2, addr: dmc->base_drexi1 + DREX_INTENC_PPC);
911
912 /* Disable counter 2 and CCNT */
913 writel(PERF_CNT2 | PERF_CCNT, addr: dmc->base_drexi0 + DREX_CNTENC_PPC);
914 writel(PERF_CNT2 | PERF_CCNT, addr: dmc->base_drexi1 + DREX_CNTENC_PPC);
915
916 /* Clear overflow flag for all counters */
917 writel(PERF_CNT2 | PERF_CCNT, addr: dmc->base_drexi0 + DREX_FLAG_PPC);
918 writel(PERF_CNT2 | PERF_CCNT, addr: dmc->base_drexi1 + DREX_FLAG_PPC);
919}
920
921/**
922 * exynos5_dmc_get_status() - Read current DMC performance statistics.
923 * @dev: device for which the statistics are requested
924 * @stat: structure which has statistic fields
925 *
926 * Function reads the DMC performance counters and calculates 'busy_time'
927 * and 'total_time'. To protect from overflow, the values are shifted right
928 * by 10. After read out the counters are setup to count again.
929 */
930static int exynos5_dmc_get_status(struct device *dev,
931 struct devfreq_dev_status *stat)
932{
933 struct exynos5_dmc *dmc = dev_get_drvdata(dev);
934 unsigned long load, total;
935 int ret;
936
937 if (dmc->in_irq_mode) {
938 mutex_lock(&dmc->lock);
939 stat->current_frequency = dmc->curr_rate;
940 mutex_unlock(lock: &dmc->lock);
941
942 stat->busy_time = dmc->load;
943 stat->total_time = dmc->total;
944 } else {
945 ret = exynos5_counters_get(dmc, load_count: &load, total_count: &total);
946 if (ret < 0)
947 return -EINVAL;
948
949 /* To protect from overflow, divide by 1024 */
950 stat->busy_time = load >> 10;
951 stat->total_time = total >> 10;
952
953 ret = exynos5_counters_set_event(dmc);
954 if (ret < 0) {
955 dev_err(dev, "could not set event counter\n");
956 return ret;
957 }
958 }
959
960 return 0;
961}
962
963/**
964 * exynos5_dmc_get_cur_freq() - Function returns current DMC frequency
965 * @dev: device for which the framework checks operating frequency
966 * @freq: returned frequency value
967 *
968 * It returns the currently used frequency of the DMC. The real operating
969 * frequency might be lower when the clock source value could not be divided
970 * to the requested value.
971 */
972static int exynos5_dmc_get_cur_freq(struct device *dev, unsigned long *freq)
973{
974 struct exynos5_dmc *dmc = dev_get_drvdata(dev);
975
976 mutex_lock(&dmc->lock);
977 *freq = dmc->curr_rate;
978 mutex_unlock(lock: &dmc->lock);
979
980 return 0;
981}
982
983/*
984 * exynos5_dmc_df_profile - Devfreq governor's profile structure
985 *
986 * It provides to the devfreq framework needed functions and polling period.
987 */
988static struct devfreq_dev_profile exynos5_dmc_df_profile = {
989 .timer = DEVFREQ_TIMER_DELAYED,
990 .target = exynos5_dmc_target,
991 .get_dev_status = exynos5_dmc_get_status,
992 .get_cur_freq = exynos5_dmc_get_cur_freq,
993};
994
995/**
996 * exynos5_dmc_align_init_freq() - Align initial frequency value
997 * @dmc: device for which the frequency is going to be set
998 * @bootloader_init_freq: initial frequency set by the bootloader in KHz
999 *
1000 * The initial bootloader frequency, which is present during boot, might be
1001 * different that supported frequency values in the driver. It is possible
1002 * due to different PLL settings or used PLL as a source.
1003 * This function provides the 'initial_freq' for the devfreq framework
1004 * statistics engine which supports only registered values. Thus, some alignment
1005 * must be made.
1006 */
1007static unsigned long
1008exynos5_dmc_align_init_freq(struct exynos5_dmc *dmc,
1009 unsigned long bootloader_init_freq)
1010{
1011 unsigned long aligned_freq;
1012 int idx;
1013
1014 idx = find_target_freq_idx(dmc, target_rate: bootloader_init_freq);
1015 if (idx >= 0)
1016 aligned_freq = dmc->opp[idx].freq_hz;
1017 else
1018 aligned_freq = dmc->opp[dmc->opp_count - 1].freq_hz;
1019
1020 return aligned_freq;
1021}
1022
1023/**
1024 * create_timings_aligned() - Create register values and align with standard
1025 * @dmc: device for which the frequency is going to be set
1026 * @reg_timing_row: array to fill with values for timing row register
1027 * @reg_timing_data: array to fill with values for timing data register
1028 * @reg_timing_power: array to fill with values for timing power register
1029 * @clk_period_ps: the period of the clock, known as tCK
1030 *
1031 * The function calculates timings and creates a register value ready for
1032 * a frequency transition. The register contains a few timings. They are
1033 * shifted by a known offset. The timing value is calculated based on memory
1034 * specyfication: minimal time required and minimal cycles required.
1035 */
1036static int create_timings_aligned(struct exynos5_dmc *dmc, u32 *reg_timing_row,
1037 u32 *reg_timing_data, u32 *reg_timing_power,
1038 u32 clk_period_ps)
1039{
1040 u32 val;
1041 const struct timing_reg *reg;
1042
1043 if (clk_period_ps == 0)
1044 return -EINVAL;
1045
1046 *reg_timing_row = 0;
1047 *reg_timing_data = 0;
1048 *reg_timing_power = 0;
1049
1050 val = dmc->timings->tRFC / clk_period_ps;
1051 val += dmc->timings->tRFC % clk_period_ps ? 1 : 0;
1052 val = max(val, dmc->min_tck->tRFC);
1053 reg = &timing_row_reg_fields[0];
1054 *reg_timing_row |= TIMING_VAL2REG(reg, val);
1055
1056 val = dmc->timings->tRRD / clk_period_ps;
1057 val += dmc->timings->tRRD % clk_period_ps ? 1 : 0;
1058 val = max(val, dmc->min_tck->tRRD);
1059 reg = &timing_row_reg_fields[1];
1060 *reg_timing_row |= TIMING_VAL2REG(reg, val);
1061
1062 val = dmc->timings->tRPab / clk_period_ps;
1063 val += dmc->timings->tRPab % clk_period_ps ? 1 : 0;
1064 val = max(val, dmc->min_tck->tRPab);
1065 reg = &timing_row_reg_fields[2];
1066 *reg_timing_row |= TIMING_VAL2REG(reg, val);
1067
1068 val = dmc->timings->tRCD / clk_period_ps;
1069 val += dmc->timings->tRCD % clk_period_ps ? 1 : 0;
1070 val = max(val, dmc->min_tck->tRCD);
1071 reg = &timing_row_reg_fields[3];
1072 *reg_timing_row |= TIMING_VAL2REG(reg, val);
1073
1074 val = dmc->timings->tRC / clk_period_ps;
1075 val += dmc->timings->tRC % clk_period_ps ? 1 : 0;
1076 val = max(val, dmc->min_tck->tRC);
1077 reg = &timing_row_reg_fields[4];
1078 *reg_timing_row |= TIMING_VAL2REG(reg, val);
1079
1080 val = dmc->timings->tRAS / clk_period_ps;
1081 val += dmc->timings->tRAS % clk_period_ps ? 1 : 0;
1082 val = max(val, dmc->min_tck->tRAS);
1083 reg = &timing_row_reg_fields[5];
1084 *reg_timing_row |= TIMING_VAL2REG(reg, val);
1085
1086 /* data related timings */
1087 val = dmc->timings->tWTR / clk_period_ps;
1088 val += dmc->timings->tWTR % clk_period_ps ? 1 : 0;
1089 val = max(val, dmc->min_tck->tWTR);
1090 reg = &timing_data_reg_fields[0];
1091 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1092
1093 val = dmc->timings->tWR / clk_period_ps;
1094 val += dmc->timings->tWR % clk_period_ps ? 1 : 0;
1095 val = max(val, dmc->min_tck->tWR);
1096 reg = &timing_data_reg_fields[1];
1097 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1098
1099 val = dmc->timings->tRTP / clk_period_ps;
1100 val += dmc->timings->tRTP % clk_period_ps ? 1 : 0;
1101 val = max(val, dmc->min_tck->tRTP);
1102 reg = &timing_data_reg_fields[2];
1103 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1104
1105 val = dmc->timings->tW2W_C2C / clk_period_ps;
1106 val += dmc->timings->tW2W_C2C % clk_period_ps ? 1 : 0;
1107 val = max(val, dmc->min_tck->tW2W_C2C);
1108 reg = &timing_data_reg_fields[3];
1109 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1110
1111 val = dmc->timings->tR2R_C2C / clk_period_ps;
1112 val += dmc->timings->tR2R_C2C % clk_period_ps ? 1 : 0;
1113 val = max(val, dmc->min_tck->tR2R_C2C);
1114 reg = &timing_data_reg_fields[4];
1115 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1116
1117 val = dmc->timings->tWL / clk_period_ps;
1118 val += dmc->timings->tWL % clk_period_ps ? 1 : 0;
1119 val = max(val, dmc->min_tck->tWL);
1120 reg = &timing_data_reg_fields[5];
1121 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1122
1123 val = dmc->timings->tDQSCK / clk_period_ps;
1124 val += dmc->timings->tDQSCK % clk_period_ps ? 1 : 0;
1125 val = max(val, dmc->min_tck->tDQSCK);
1126 reg = &timing_data_reg_fields[6];
1127 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1128
1129 val = dmc->timings->tRL / clk_period_ps;
1130 val += dmc->timings->tRL % clk_period_ps ? 1 : 0;
1131 val = max(val, dmc->min_tck->tRL);
1132 reg = &timing_data_reg_fields[7];
1133 *reg_timing_data |= TIMING_VAL2REG(reg, val);
1134
1135 /* power related timings */
1136 val = dmc->timings->tFAW / clk_period_ps;
1137 val += dmc->timings->tFAW % clk_period_ps ? 1 : 0;
1138 val = max(val, dmc->min_tck->tFAW);
1139 reg = &timing_power_reg_fields[0];
1140 *reg_timing_power |= TIMING_VAL2REG(reg, val);
1141
1142 val = dmc->timings->tXSR / clk_period_ps;
1143 val += dmc->timings->tXSR % clk_period_ps ? 1 : 0;
1144 val = max(val, dmc->min_tck->tXSR);
1145 reg = &timing_power_reg_fields[1];
1146 *reg_timing_power |= TIMING_VAL2REG(reg, val);
1147
1148 val = dmc->timings->tXP / clk_period_ps;
1149 val += dmc->timings->tXP % clk_period_ps ? 1 : 0;
1150 val = max(val, dmc->min_tck->tXP);
1151 reg = &timing_power_reg_fields[2];
1152 *reg_timing_power |= TIMING_VAL2REG(reg, val);
1153
1154 val = dmc->timings->tCKE / clk_period_ps;
1155 val += dmc->timings->tCKE % clk_period_ps ? 1 : 0;
1156 val = max(val, dmc->min_tck->tCKE);
1157 reg = &timing_power_reg_fields[3];
1158 *reg_timing_power |= TIMING_VAL2REG(reg, val);
1159
1160 val = dmc->timings->tMRD / clk_period_ps;
1161 val += dmc->timings->tMRD % clk_period_ps ? 1 : 0;
1162 val = max(val, dmc->min_tck->tMRD);
1163 reg = &timing_power_reg_fields[4];
1164 *reg_timing_power |= TIMING_VAL2REG(reg, val);
1165
1166 return 0;
1167}
1168
1169/**
1170 * of_get_dram_timings() - helper function for parsing DT settings for DRAM
1171 * @dmc: device for which the frequency is going to be set
1172 *
1173 * The function parses DT entries with DRAM information.
1174 */
1175static int of_get_dram_timings(struct exynos5_dmc *dmc)
1176{
1177 int ret = 0;
1178 int idx;
1179 struct device_node *np_ddr;
1180 u32 freq_mhz, clk_period_ps;
1181
1182 np_ddr = of_parse_phandle(np: dmc->dev->of_node, phandle_name: "device-handle", index: 0);
1183 if (!np_ddr) {
1184 dev_warn(dmc->dev, "could not find 'device-handle' in DT\n");
1185 return -EINVAL;
1186 }
1187
1188 dmc->timing_row = devm_kmalloc_array(dev: dmc->dev, TIMING_COUNT,
1189 size: sizeof(u32), GFP_KERNEL);
1190 if (!dmc->timing_row) {
1191 ret = -ENOMEM;
1192 goto put_node;
1193 }
1194
1195 dmc->timing_data = devm_kmalloc_array(dev: dmc->dev, TIMING_COUNT,
1196 size: sizeof(u32), GFP_KERNEL);
1197 if (!dmc->timing_data) {
1198 ret = -ENOMEM;
1199 goto put_node;
1200 }
1201
1202 dmc->timing_power = devm_kmalloc_array(dev: dmc->dev, TIMING_COUNT,
1203 size: sizeof(u32), GFP_KERNEL);
1204 if (!dmc->timing_power) {
1205 ret = -ENOMEM;
1206 goto put_node;
1207 }
1208
1209 dmc->timings = of_lpddr3_get_ddr_timings(np_ddr, dev: dmc->dev,
1210 DDR_TYPE_LPDDR3,
1211 nr_frequencies: &dmc->timings_arr_size);
1212 if (!dmc->timings) {
1213 dev_warn(dmc->dev, "could not get timings from DT\n");
1214 ret = -EINVAL;
1215 goto put_node;
1216 }
1217
1218 dmc->min_tck = of_lpddr3_get_min_tck(np: np_ddr, dev: dmc->dev);
1219 if (!dmc->min_tck) {
1220 dev_warn(dmc->dev, "could not get tck from DT\n");
1221 ret = -EINVAL;
1222 goto put_node;
1223 }
1224
1225 /* Sorted array of OPPs with frequency ascending */
1226 for (idx = 0; idx < dmc->opp_count; idx++) {
1227 freq_mhz = dmc->opp[idx].freq_hz / 1000000;
1228 clk_period_ps = 1000000 / freq_mhz;
1229
1230 ret = create_timings_aligned(dmc, reg_timing_row: &dmc->timing_row[idx],
1231 reg_timing_data: &dmc->timing_data[idx],
1232 reg_timing_power: &dmc->timing_power[idx],
1233 clk_period_ps);
1234 }
1235
1236
1237 /* Take the highest frequency's timings as 'bypass' */
1238 dmc->bypass_timing_row = dmc->timing_row[idx - 1];
1239 dmc->bypass_timing_data = dmc->timing_data[idx - 1];
1240 dmc->bypass_timing_power = dmc->timing_power[idx - 1];
1241
1242put_node:
1243 of_node_put(node: np_ddr);
1244 return ret;
1245}
1246
1247/**
1248 * exynos5_dmc_init_clks() - Initialize clocks needed for DMC operation.
1249 * @dmc: DMC structure containing needed fields
1250 *
1251 * Get the needed clocks defined in DT device, enable and set the right parents.
1252 * Read current frequency and initialize the initial rate for governor.
1253 */
1254static int exynos5_dmc_init_clks(struct exynos5_dmc *dmc)
1255{
1256 int ret;
1257 unsigned long target_volt = 0;
1258 unsigned long target_rate = 0;
1259 unsigned int tmp;
1260
1261 dmc->fout_spll = devm_clk_get(dev: dmc->dev, id: "fout_spll");
1262 if (IS_ERR(ptr: dmc->fout_spll))
1263 return PTR_ERR(ptr: dmc->fout_spll);
1264
1265 dmc->fout_bpll = devm_clk_get(dev: dmc->dev, id: "fout_bpll");
1266 if (IS_ERR(ptr: dmc->fout_bpll))
1267 return PTR_ERR(ptr: dmc->fout_bpll);
1268
1269 dmc->mout_mclk_cdrex = devm_clk_get(dev: dmc->dev, id: "mout_mclk_cdrex");
1270 if (IS_ERR(ptr: dmc->mout_mclk_cdrex))
1271 return PTR_ERR(ptr: dmc->mout_mclk_cdrex);
1272
1273 dmc->mout_bpll = devm_clk_get(dev: dmc->dev, id: "mout_bpll");
1274 if (IS_ERR(ptr: dmc->mout_bpll))
1275 return PTR_ERR(ptr: dmc->mout_bpll);
1276
1277 dmc->mout_mx_mspll_ccore = devm_clk_get(dev: dmc->dev,
1278 id: "mout_mx_mspll_ccore");
1279 if (IS_ERR(ptr: dmc->mout_mx_mspll_ccore))
1280 return PTR_ERR(ptr: dmc->mout_mx_mspll_ccore);
1281
1282 dmc->mout_spll = devm_clk_get(dev: dmc->dev, id: "ff_dout_spll2");
1283 if (IS_ERR(ptr: dmc->mout_spll)) {
1284 dmc->mout_spll = devm_clk_get(dev: dmc->dev, id: "mout_sclk_spll");
1285 if (IS_ERR(ptr: dmc->mout_spll))
1286 return PTR_ERR(ptr: dmc->mout_spll);
1287 }
1288
1289 /*
1290 * Convert frequency to KHz values and set it for the governor.
1291 */
1292 dmc->curr_rate = clk_get_rate(clk: dmc->mout_mclk_cdrex);
1293 dmc->curr_rate = exynos5_dmc_align_init_freq(dmc, bootloader_init_freq: dmc->curr_rate);
1294 exynos5_dmc_df_profile.initial_freq = dmc->curr_rate;
1295
1296 ret = exynos5_dmc_get_volt_freq(dmc, freq: &dmc->curr_rate, target_rate: &target_rate,
1297 target_volt: &target_volt, flags: 0);
1298 if (ret)
1299 return ret;
1300
1301 dmc->curr_volt = target_volt;
1302
1303 ret = clk_set_parent(clk: dmc->mout_mx_mspll_ccore, parent: dmc->mout_spll);
1304 if (ret)
1305 return ret;
1306
1307 clk_prepare_enable(clk: dmc->fout_bpll);
1308 clk_prepare_enable(clk: dmc->mout_bpll);
1309
1310 /*
1311 * Some bootloaders do not set clock routes correctly.
1312 * Stop one path in clocks to PHY.
1313 */
1314 regmap_read(map: dmc->clk_regmap, CDREX_LPDDR3PHY_CLKM_SRC, val: &tmp);
1315 tmp &= ~(BIT(1) | BIT(0));
1316 regmap_write(map: dmc->clk_regmap, CDREX_LPDDR3PHY_CLKM_SRC, val: tmp);
1317
1318 return 0;
1319}
1320
1321/**
1322 * exynos5_performance_counters_init() - Initializes performance DMC's counters
1323 * @dmc: DMC for which it does the setup
1324 *
1325 * Initialization of performance counters in DMC for estimating usage.
1326 * The counter's values are used for calculation of a memory bandwidth and based
1327 * on that the governor changes the frequency.
1328 * The counters are not used when the governor is GOVERNOR_USERSPACE.
1329 */
1330static int exynos5_performance_counters_init(struct exynos5_dmc *dmc)
1331{
1332 int ret, i;
1333
1334 dmc->num_counters = devfreq_event_get_edev_count(dev: dmc->dev,
1335 phandle_name: "devfreq-events");
1336 if (dmc->num_counters < 0) {
1337 dev_err(dmc->dev, "could not get devfreq-event counters\n");
1338 return dmc->num_counters;
1339 }
1340
1341 dmc->counter = devm_kcalloc(dev: dmc->dev, n: dmc->num_counters,
1342 size: sizeof(*dmc->counter), GFP_KERNEL);
1343 if (!dmc->counter)
1344 return -ENOMEM;
1345
1346 for (i = 0; i < dmc->num_counters; i++) {
1347 dmc->counter[i] =
1348 devfreq_event_get_edev_by_phandle(dev: dmc->dev,
1349 phandle_name: "devfreq-events", index: i);
1350 if (IS_ERR_OR_NULL(ptr: dmc->counter[i]))
1351 return -EPROBE_DEFER;
1352 }
1353
1354 ret = exynos5_counters_enable_edev(dmc);
1355 if (ret < 0) {
1356 dev_err(dmc->dev, "could not enable event counter\n");
1357 return ret;
1358 }
1359
1360 ret = exynos5_counters_set_event(dmc);
1361 if (ret < 0) {
1362 exynos5_counters_disable_edev(dmc);
1363 dev_err(dmc->dev, "could not set event counter\n");
1364 return ret;
1365 }
1366
1367 return 0;
1368}
1369
1370/**
1371 * exynos5_dmc_set_pause_on_switching() - Controls a pause feature in DMC
1372 * @dmc: device which is used for changing this feature
1373 *
1374 * There is a need of pausing DREX DMC when divider or MUX in clock tree
1375 * changes its configuration. In such situation access to the memory is blocked
1376 * in DMC automatically. This feature is used when clock frequency change
1377 * request appears and touches clock tree.
1378 */
1379static inline int exynos5_dmc_set_pause_on_switching(struct exynos5_dmc *dmc)
1380{
1381 unsigned int val;
1382 int ret;
1383
1384 ret = regmap_read(map: dmc->clk_regmap, CDREX_PAUSE, val: &val);
1385 if (ret)
1386 return ret;
1387
1388 val |= 1UL;
1389 regmap_write(map: dmc->clk_regmap, CDREX_PAUSE, val);
1390
1391 return 0;
1392}
1393
1394static irqreturn_t dmc_irq_thread(int irq, void *priv)
1395{
1396 int res;
1397 struct exynos5_dmc *dmc = priv;
1398
1399 mutex_lock(&dmc->df->lock);
1400 exynos5_dmc_perf_events_check(dmc);
1401 res = update_devfreq(devfreq: dmc->df);
1402 mutex_unlock(lock: &dmc->df->lock);
1403
1404 if (res)
1405 dev_warn(dmc->dev, "devfreq failed with %d\n", res);
1406
1407 return IRQ_HANDLED;
1408}
1409
1410/**
1411 * exynos5_dmc_probe() - Probe function for the DMC driver
1412 * @pdev: platform device for which the driver is going to be initialized
1413 *
1414 * Initialize basic components: clocks, regulators, performance counters, etc.
1415 * Read out product version and based on the information setup
1416 * internal structures for the controller (frequency and voltage) and for DRAM
1417 * memory parameters: timings for each operating frequency.
1418 * Register new devfreq device for controlling DVFS of the DMC.
1419 */
1420static int exynos5_dmc_probe(struct platform_device *pdev)
1421{
1422 int ret = 0;
1423 struct device *dev = &pdev->dev;
1424 struct device_node *np = dev->of_node;
1425 struct exynos5_dmc *dmc;
1426 int irq[2];
1427
1428 dmc = devm_kzalloc(dev, size: sizeof(*dmc), GFP_KERNEL);
1429 if (!dmc)
1430 return -ENOMEM;
1431
1432 mutex_init(&dmc->lock);
1433
1434 dmc->dev = dev;
1435 platform_set_drvdata(pdev, data: dmc);
1436
1437 dmc->base_drexi0 = devm_platform_ioremap_resource(pdev, index: 0);
1438 if (IS_ERR(ptr: dmc->base_drexi0))
1439 return PTR_ERR(ptr: dmc->base_drexi0);
1440
1441 dmc->base_drexi1 = devm_platform_ioremap_resource(pdev, index: 1);
1442 if (IS_ERR(ptr: dmc->base_drexi1))
1443 return PTR_ERR(ptr: dmc->base_drexi1);
1444
1445 dmc->clk_regmap = syscon_regmap_lookup_by_phandle(np,
1446 property: "samsung,syscon-clk");
1447 if (IS_ERR(ptr: dmc->clk_regmap))
1448 return PTR_ERR(ptr: dmc->clk_regmap);
1449
1450 ret = exynos5_init_freq_table(dmc, profile: &exynos5_dmc_df_profile);
1451 if (ret) {
1452 dev_warn(dev, "couldn't initialize frequency settings\n");
1453 return ret;
1454 }
1455
1456 dmc->vdd_mif = devm_regulator_get(dev, id: "vdd");
1457 if (IS_ERR(ptr: dmc->vdd_mif)) {
1458 ret = PTR_ERR(ptr: dmc->vdd_mif);
1459 return ret;
1460 }
1461
1462 ret = exynos5_dmc_init_clks(dmc);
1463 if (ret)
1464 return ret;
1465
1466 ret = of_get_dram_timings(dmc);
1467 if (ret) {
1468 dev_warn(dev, "couldn't initialize timings settings\n");
1469 goto remove_clocks;
1470 }
1471
1472 ret = exynos5_dmc_set_pause_on_switching(dmc);
1473 if (ret) {
1474 dev_warn(dev, "couldn't get access to PAUSE register\n");
1475 goto remove_clocks;
1476 }
1477
1478 /* There is two modes in which the driver works: polling or IRQ */
1479 irq[0] = platform_get_irq_byname(pdev, "drex_0");
1480 irq[1] = platform_get_irq_byname(pdev, "drex_1");
1481 if (irq[0] > 0 && irq[1] > 0 && irqmode) {
1482 ret = devm_request_threaded_irq(dev, irq: irq[0], NULL,
1483 thread_fn: dmc_irq_thread, IRQF_ONESHOT,
1484 devname: dev_name(dev), dev_id: dmc);
1485 if (ret) {
1486 dev_err(dev, "couldn't grab IRQ\n");
1487 goto remove_clocks;
1488 }
1489
1490 ret = devm_request_threaded_irq(dev, irq: irq[1], NULL,
1491 thread_fn: dmc_irq_thread, IRQF_ONESHOT,
1492 devname: dev_name(dev), dev_id: dmc);
1493 if (ret) {
1494 dev_err(dev, "couldn't grab IRQ\n");
1495 goto remove_clocks;
1496 }
1497
1498 /*
1499 * Setup default thresholds for the devfreq governor.
1500 * The values are chosen based on experiments.
1501 */
1502 dmc->gov_data.upthreshold = 55;
1503 dmc->gov_data.downdifferential = 5;
1504
1505 exynos5_dmc_enable_perf_events(dmc);
1506
1507 dmc->in_irq_mode = 1;
1508 } else {
1509 ret = exynos5_performance_counters_init(dmc);
1510 if (ret) {
1511 dev_warn(dev, "couldn't probe performance counters\n");
1512 goto remove_clocks;
1513 }
1514
1515 /*
1516 * Setup default thresholds for the devfreq governor.
1517 * The values are chosen based on experiments.
1518 */
1519 dmc->gov_data.upthreshold = 10;
1520 dmc->gov_data.downdifferential = 5;
1521
1522 exynos5_dmc_df_profile.polling_ms = 100;
1523 }
1524
1525 dmc->df = devm_devfreq_add_device(dev, profile: &exynos5_dmc_df_profile,
1526 DEVFREQ_GOV_SIMPLE_ONDEMAND,
1527 data: &dmc->gov_data);
1528
1529 if (IS_ERR(ptr: dmc->df)) {
1530 ret = PTR_ERR(ptr: dmc->df);
1531 goto err_devfreq_add;
1532 }
1533
1534 if (dmc->in_irq_mode)
1535 exynos5_dmc_start_perf_events(dmc, PERF_COUNTER_START_VALUE);
1536
1537 dev_info(dev, "DMC initialized, in irq mode: %d\n", dmc->in_irq_mode);
1538
1539 return 0;
1540
1541err_devfreq_add:
1542 if (dmc->in_irq_mode)
1543 exynos5_dmc_disable_perf_events(dmc);
1544 else
1545 exynos5_counters_disable_edev(dmc);
1546remove_clocks:
1547 clk_disable_unprepare(clk: dmc->mout_bpll);
1548 clk_disable_unprepare(clk: dmc->fout_bpll);
1549
1550 return ret;
1551}
1552
1553/**
1554 * exynos5_dmc_remove() - Remove function for the platform device
1555 * @pdev: platform device which is going to be removed
1556 *
1557 * The function relies on 'devm' framework function which automatically
1558 * clean the device's resources. It just calls explicitly disable function for
1559 * the performance counters.
1560 */
1561static void exynos5_dmc_remove(struct platform_device *pdev)
1562{
1563 struct exynos5_dmc *dmc = dev_get_drvdata(dev: &pdev->dev);
1564
1565 if (dmc->in_irq_mode)
1566 exynos5_dmc_disable_perf_events(dmc);
1567 else
1568 exynos5_counters_disable_edev(dmc);
1569
1570 clk_disable_unprepare(clk: dmc->mout_bpll);
1571 clk_disable_unprepare(clk: dmc->fout_bpll);
1572}
1573
1574static const struct of_device_id exynos5_dmc_of_match[] = {
1575 { .compatible = "samsung,exynos5422-dmc", },
1576 { },
1577};
1578MODULE_DEVICE_TABLE(of, exynos5_dmc_of_match);
1579
1580static struct platform_driver exynos5_dmc_platdrv = {
1581 .probe = exynos5_dmc_probe,
1582 .remove_new = exynos5_dmc_remove,
1583 .driver = {
1584 .name = "exynos5-dmc",
1585 .of_match_table = exynos5_dmc_of_match,
1586 },
1587};
1588module_platform_driver(exynos5_dmc_platdrv);
1589MODULE_DESCRIPTION("Driver for Exynos5422 Dynamic Memory Controller dynamic frequency and voltage change");
1590MODULE_LICENSE("GPL v2");
1591MODULE_AUTHOR("Lukasz Luba");
1592

source code of linux/drivers/memory/samsung/exynos5422-dmc.c