1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * drivers/clocksource/arm_global_timer.c |
4 | * |
5 | * Copyright (C) 2013 STMicroelectronics (R&D) Limited. |
6 | * Author: Stuart Menefy <stuart.menefy@st.com> |
7 | * Author: Srinivas Kandagatla <srinivas.kandagatla@st.com> |
8 | */ |
9 | |
10 | #include <linux/init.h> |
11 | #include <linux/interrupt.h> |
12 | #include <linux/clocksource.h> |
13 | #include <linux/clockchips.h> |
14 | #include <linux/cpu.h> |
15 | #include <linux/clk.h> |
16 | #include <linux/delay.h> |
17 | #include <linux/err.h> |
18 | #include <linux/io.h> |
19 | #include <linux/of.h> |
20 | #include <linux/of_irq.h> |
21 | #include <linux/of_address.h> |
22 | #include <linux/sched_clock.h> |
23 | |
24 | #include <asm/cputype.h> |
25 | |
26 | #define GT_COUNTER0 0x00 |
27 | #define GT_COUNTER1 0x04 |
28 | |
29 | #define GT_CONTROL 0x08 |
30 | #define GT_CONTROL_TIMER_ENABLE BIT(0) /* this bit is NOT banked */ |
31 | #define GT_CONTROL_COMP_ENABLE BIT(1) /* banked */ |
32 | #define GT_CONTROL_IRQ_ENABLE BIT(2) /* banked */ |
33 | #define GT_CONTROL_AUTO_INC BIT(3) /* banked */ |
34 | #define GT_CONTROL_PRESCALER_SHIFT 8 |
35 | #define GT_CONTROL_PRESCALER_MAX 0xF |
36 | #define GT_CONTROL_PRESCALER_MASK (GT_CONTROL_PRESCALER_MAX << \ |
37 | GT_CONTROL_PRESCALER_SHIFT) |
38 | |
39 | #define GT_INT_STATUS 0x0c |
40 | #define GT_INT_STATUS_EVENT_FLAG BIT(0) |
41 | |
42 | #define GT_COMP0 0x10 |
43 | #define GT_COMP1 0x14 |
44 | #define GT_AUTO_INC 0x18 |
45 | |
46 | #define MAX_F_ERR 50 |
47 | /* |
48 | * We are expecting to be clocked by the ARM peripheral clock. |
49 | * |
50 | * Note: it is assumed we are using a prescaler value of zero, so this is |
51 | * the units for all operations. |
52 | */ |
53 | static void __iomem *gt_base; |
54 | static struct notifier_block gt_clk_rate_change_nb; |
55 | static u32 gt_psv_new, gt_psv_bck, gt_target_rate; |
56 | static int gt_ppi; |
57 | static struct clock_event_device __percpu *gt_evt; |
58 | |
59 | /* |
60 | * To get the value from the Global Timer Counter register proceed as follows: |
61 | * 1. Read the upper 32-bit timer counter register |
62 | * 2. Read the lower 32-bit timer counter register |
63 | * 3. Read the upper 32-bit timer counter register again. If the value is |
64 | * different to the 32-bit upper value read previously, go back to step 2. |
65 | * Otherwise the 64-bit timer counter value is correct. |
66 | */ |
67 | static u64 notrace _gt_counter_read(void) |
68 | { |
69 | u64 counter; |
70 | u32 lower; |
71 | u32 upper, old_upper; |
72 | |
73 | upper = readl_relaxed(gt_base + GT_COUNTER1); |
74 | do { |
75 | old_upper = upper; |
76 | lower = readl_relaxed(gt_base + GT_COUNTER0); |
77 | upper = readl_relaxed(gt_base + GT_COUNTER1); |
78 | } while (upper != old_upper); |
79 | |
80 | counter = upper; |
81 | counter <<= 32; |
82 | counter |= lower; |
83 | return counter; |
84 | } |
85 | |
86 | static u64 gt_counter_read(void) |
87 | { |
88 | return _gt_counter_read(); |
89 | } |
90 | |
91 | /** |
92 | * To ensure that updates to comparator value register do not set the |
93 | * Interrupt Status Register proceed as follows: |
94 | * 1. Clear the Comp Enable bit in the Timer Control Register. |
95 | * 2. Write the lower 32-bit Comparator Value Register. |
96 | * 3. Write the upper 32-bit Comparator Value Register. |
97 | * 4. Set the Comp Enable bit and, if necessary, the IRQ enable bit. |
98 | */ |
99 | static void gt_compare_set(unsigned long delta, int periodic) |
100 | { |
101 | u64 counter = gt_counter_read(); |
102 | unsigned long ctrl; |
103 | |
104 | counter += delta; |
105 | ctrl = readl(addr: gt_base + GT_CONTROL); |
106 | ctrl &= ~(GT_CONTROL_COMP_ENABLE | GT_CONTROL_IRQ_ENABLE | |
107 | GT_CONTROL_AUTO_INC); |
108 | ctrl |= GT_CONTROL_TIMER_ENABLE; |
109 | writel_relaxed(ctrl, gt_base + GT_CONTROL); |
110 | writel_relaxed(lower_32_bits(counter), gt_base + GT_COMP0); |
111 | writel_relaxed(upper_32_bits(counter), gt_base + GT_COMP1); |
112 | |
113 | if (periodic) { |
114 | writel_relaxed(delta, gt_base + GT_AUTO_INC); |
115 | ctrl |= GT_CONTROL_AUTO_INC; |
116 | } |
117 | |
118 | ctrl |= GT_CONTROL_COMP_ENABLE | GT_CONTROL_IRQ_ENABLE; |
119 | writel_relaxed(ctrl, gt_base + GT_CONTROL); |
120 | } |
121 | |
122 | static int gt_clockevent_shutdown(struct clock_event_device *evt) |
123 | { |
124 | unsigned long ctrl; |
125 | |
126 | ctrl = readl(addr: gt_base + GT_CONTROL); |
127 | ctrl &= ~(GT_CONTROL_COMP_ENABLE | GT_CONTROL_IRQ_ENABLE | |
128 | GT_CONTROL_AUTO_INC); |
129 | writel(val: ctrl, addr: gt_base + GT_CONTROL); |
130 | return 0; |
131 | } |
132 | |
133 | static int gt_clockevent_set_periodic(struct clock_event_device *evt) |
134 | { |
135 | gt_compare_set(DIV_ROUND_CLOSEST(gt_target_rate, HZ), periodic: 1); |
136 | return 0; |
137 | } |
138 | |
139 | static int gt_clockevent_set_next_event(unsigned long evt, |
140 | struct clock_event_device *unused) |
141 | { |
142 | gt_compare_set(delta: evt, periodic: 0); |
143 | return 0; |
144 | } |
145 | |
146 | static irqreturn_t gt_clockevent_interrupt(int irq, void *dev_id) |
147 | { |
148 | struct clock_event_device *evt = dev_id; |
149 | |
150 | if (!(readl_relaxed(gt_base + GT_INT_STATUS) & |
151 | GT_INT_STATUS_EVENT_FLAG)) |
152 | return IRQ_NONE; |
153 | |
154 | /** |
155 | * ERRATA 740657( Global Timer can send 2 interrupts for |
156 | * the same event in single-shot mode) |
157 | * Workaround: |
158 | * Either disable single-shot mode. |
159 | * Or |
160 | * Modify the Interrupt Handler to avoid the |
161 | * offending sequence. This is achieved by clearing |
162 | * the Global Timer flag _after_ having incremented |
163 | * the Comparator register value to a higher value. |
164 | */ |
165 | if (clockevent_state_oneshot(dev: evt)) |
166 | gt_compare_set(ULONG_MAX, periodic: 0); |
167 | |
168 | writel_relaxed(GT_INT_STATUS_EVENT_FLAG, gt_base + GT_INT_STATUS); |
169 | evt->event_handler(evt); |
170 | |
171 | return IRQ_HANDLED; |
172 | } |
173 | |
174 | static int gt_starting_cpu(unsigned int cpu) |
175 | { |
176 | struct clock_event_device *clk = this_cpu_ptr(gt_evt); |
177 | |
178 | clk->name = "arm_global_timer" ; |
179 | clk->features = CLOCK_EVT_FEAT_PERIODIC | CLOCK_EVT_FEAT_ONESHOT | |
180 | CLOCK_EVT_FEAT_PERCPU; |
181 | clk->set_state_shutdown = gt_clockevent_shutdown; |
182 | clk->set_state_periodic = gt_clockevent_set_periodic; |
183 | clk->set_state_oneshot = gt_clockevent_shutdown; |
184 | clk->set_state_oneshot_stopped = gt_clockevent_shutdown; |
185 | clk->set_next_event = gt_clockevent_set_next_event; |
186 | clk->cpumask = cpumask_of(cpu); |
187 | clk->rating = 300; |
188 | clk->irq = gt_ppi; |
189 | clockevents_config_and_register(dev: clk, freq: gt_target_rate, |
190 | min_delta: 1, max_delta: 0xffffffff); |
191 | enable_percpu_irq(irq: clk->irq, type: IRQ_TYPE_NONE); |
192 | return 0; |
193 | } |
194 | |
195 | static int gt_dying_cpu(unsigned int cpu) |
196 | { |
197 | struct clock_event_device *clk = this_cpu_ptr(gt_evt); |
198 | |
199 | gt_clockevent_shutdown(evt: clk); |
200 | disable_percpu_irq(irq: clk->irq); |
201 | return 0; |
202 | } |
203 | |
204 | static u64 gt_clocksource_read(struct clocksource *cs) |
205 | { |
206 | return gt_counter_read(); |
207 | } |
208 | |
209 | static void gt_resume(struct clocksource *cs) |
210 | { |
211 | unsigned long ctrl; |
212 | |
213 | ctrl = readl(addr: gt_base + GT_CONTROL); |
214 | if (!(ctrl & GT_CONTROL_TIMER_ENABLE)) |
215 | /* re-enable timer on resume */ |
216 | writel(GT_CONTROL_TIMER_ENABLE, addr: gt_base + GT_CONTROL); |
217 | } |
218 | |
219 | static struct clocksource gt_clocksource = { |
220 | .name = "arm_global_timer" , |
221 | .rating = 300, |
222 | .read = gt_clocksource_read, |
223 | .mask = CLOCKSOURCE_MASK(64), |
224 | .flags = CLOCK_SOURCE_IS_CONTINUOUS, |
225 | .resume = gt_resume, |
226 | }; |
227 | |
228 | #ifdef CONFIG_CLKSRC_ARM_GLOBAL_TIMER_SCHED_CLOCK |
229 | static u64 notrace gt_sched_clock_read(void) |
230 | { |
231 | return _gt_counter_read(); |
232 | } |
233 | #endif |
234 | |
235 | static unsigned long gt_read_long(void) |
236 | { |
237 | return readl_relaxed(gt_base + GT_COUNTER0); |
238 | } |
239 | |
240 | static struct delay_timer gt_delay_timer = { |
241 | .read_current_timer = gt_read_long, |
242 | }; |
243 | |
244 | static void gt_write_presc(u32 psv) |
245 | { |
246 | u32 reg; |
247 | |
248 | reg = readl(addr: gt_base + GT_CONTROL); |
249 | reg &= ~GT_CONTROL_PRESCALER_MASK; |
250 | reg |= psv << GT_CONTROL_PRESCALER_SHIFT; |
251 | writel(val: reg, addr: gt_base + GT_CONTROL); |
252 | } |
253 | |
254 | static u32 gt_read_presc(void) |
255 | { |
256 | u32 reg; |
257 | |
258 | reg = readl(addr: gt_base + GT_CONTROL); |
259 | reg &= GT_CONTROL_PRESCALER_MASK; |
260 | return reg >> GT_CONTROL_PRESCALER_SHIFT; |
261 | } |
262 | |
263 | static void __init gt_delay_timer_init(void) |
264 | { |
265 | gt_delay_timer.freq = gt_target_rate; |
266 | register_current_timer_delay(>_delay_timer); |
267 | } |
268 | |
269 | static int __init gt_clocksource_init(void) |
270 | { |
271 | writel(val: 0, addr: gt_base + GT_CONTROL); |
272 | writel(val: 0, addr: gt_base + GT_COUNTER0); |
273 | writel(val: 0, addr: gt_base + GT_COUNTER1); |
274 | /* set prescaler and enable timer on all the cores */ |
275 | writel(((CONFIG_ARM_GT_INITIAL_PRESCALER_VAL - 1) << |
276 | GT_CONTROL_PRESCALER_SHIFT) |
277 | | GT_CONTROL_TIMER_ENABLE, gt_base + GT_CONTROL); |
278 | |
279 | #ifdef CONFIG_CLKSRC_ARM_GLOBAL_TIMER_SCHED_CLOCK |
280 | sched_clock_register(gt_sched_clock_read, 64, gt_target_rate); |
281 | #endif |
282 | return clocksource_register_hz(cs: >_clocksource, hz: gt_target_rate); |
283 | } |
284 | |
285 | static int gt_clk_rate_change_cb(struct notifier_block *nb, |
286 | unsigned long event, void *data) |
287 | { |
288 | struct clk_notifier_data *ndata = data; |
289 | |
290 | switch (event) { |
291 | case PRE_RATE_CHANGE: |
292 | { |
293 | int psv; |
294 | |
295 | psv = DIV_ROUND_CLOSEST(ndata->new_rate, |
296 | gt_target_rate); |
297 | |
298 | if (abs(gt_target_rate - (ndata->new_rate / psv)) > MAX_F_ERR) |
299 | return NOTIFY_BAD; |
300 | |
301 | psv--; |
302 | |
303 | /* prescaler within legal range? */ |
304 | if (psv < 0 || psv > GT_CONTROL_PRESCALER_MAX) |
305 | return NOTIFY_BAD; |
306 | |
307 | /* |
308 | * store timer clock ctrl register so we can restore it in case |
309 | * of an abort. |
310 | */ |
311 | gt_psv_bck = gt_read_presc(); |
312 | gt_psv_new = psv; |
313 | /* scale down: adjust divider in post-change notification */ |
314 | if (ndata->new_rate < ndata->old_rate) |
315 | return NOTIFY_DONE; |
316 | |
317 | /* scale up: adjust divider now - before frequency change */ |
318 | gt_write_presc(psv); |
319 | break; |
320 | } |
321 | case POST_RATE_CHANGE: |
322 | /* scale up: pre-change notification did the adjustment */ |
323 | if (ndata->new_rate > ndata->old_rate) |
324 | return NOTIFY_OK; |
325 | |
326 | /* scale down: adjust divider now - after frequency change */ |
327 | gt_write_presc(psv: gt_psv_new); |
328 | break; |
329 | |
330 | case ABORT_RATE_CHANGE: |
331 | /* we have to undo the adjustment in case we scale up */ |
332 | if (ndata->new_rate < ndata->old_rate) |
333 | return NOTIFY_OK; |
334 | |
335 | /* restore original register value */ |
336 | gt_write_presc(psv: gt_psv_bck); |
337 | break; |
338 | default: |
339 | return NOTIFY_DONE; |
340 | } |
341 | |
342 | return NOTIFY_DONE; |
343 | } |
344 | |
345 | static int __init global_timer_of_register(struct device_node *np) |
346 | { |
347 | struct clk *gt_clk; |
348 | static unsigned long gt_clk_rate; |
349 | int err = 0; |
350 | |
351 | /* |
352 | * In A9 r2p0 the comparators for each processor with the global timer |
353 | * fire when the timer value is greater than or equal to. In previous |
354 | * revisions the comparators fired when the timer value was equal to. |
355 | */ |
356 | if (read_cpuid_part() == ARM_CPU_PART_CORTEX_A9 |
357 | && (read_cpuid_id() & 0xf0000f) < 0x200000) { |
358 | pr_warn("global-timer: non support for this cpu version.\n" ); |
359 | return -ENOSYS; |
360 | } |
361 | |
362 | gt_ppi = irq_of_parse_and_map(node: np, index: 0); |
363 | if (!gt_ppi) { |
364 | pr_warn("global-timer: unable to parse irq\n" ); |
365 | return -EINVAL; |
366 | } |
367 | |
368 | gt_base = of_iomap(node: np, index: 0); |
369 | if (!gt_base) { |
370 | pr_warn("global-timer: invalid base address\n" ); |
371 | return -ENXIO; |
372 | } |
373 | |
374 | gt_clk = of_clk_get(np, index: 0); |
375 | if (!IS_ERR(ptr: gt_clk)) { |
376 | err = clk_prepare_enable(clk: gt_clk); |
377 | if (err) |
378 | goto out_unmap; |
379 | } else { |
380 | pr_warn("global-timer: clk not found\n" ); |
381 | err = -EINVAL; |
382 | goto out_unmap; |
383 | } |
384 | |
385 | gt_clk_rate = clk_get_rate(clk: gt_clk); |
386 | gt_target_rate = gt_clk_rate / CONFIG_ARM_GT_INITIAL_PRESCALER_VAL; |
387 | gt_clk_rate_change_nb.notifier_call = |
388 | gt_clk_rate_change_cb; |
389 | err = clk_notifier_register(clk: gt_clk, nb: >_clk_rate_change_nb); |
390 | if (err) { |
391 | pr_warn("Unable to register clock notifier\n" ); |
392 | goto out_clk; |
393 | } |
394 | |
395 | gt_evt = alloc_percpu(struct clock_event_device); |
396 | if (!gt_evt) { |
397 | pr_warn("global-timer: can't allocate memory\n" ); |
398 | err = -ENOMEM; |
399 | goto out_clk_nb; |
400 | } |
401 | |
402 | err = request_percpu_irq(irq: gt_ppi, handler: gt_clockevent_interrupt, |
403 | devname: "gt" , percpu_dev_id: gt_evt); |
404 | if (err) { |
405 | pr_warn("global-timer: can't register interrupt %d (%d)\n" , |
406 | gt_ppi, err); |
407 | goto out_free; |
408 | } |
409 | |
410 | /* Register and immediately configure the timer on the boot CPU */ |
411 | err = gt_clocksource_init(); |
412 | if (err) |
413 | goto out_irq; |
414 | |
415 | err = cpuhp_setup_state(state: CPUHP_AP_ARM_GLOBAL_TIMER_STARTING, |
416 | name: "clockevents/arm/global_timer:starting" , |
417 | startup: gt_starting_cpu, teardown: gt_dying_cpu); |
418 | if (err) |
419 | goto out_irq; |
420 | |
421 | gt_delay_timer_init(); |
422 | |
423 | return 0; |
424 | |
425 | out_irq: |
426 | free_percpu_irq(gt_ppi, gt_evt); |
427 | out_free: |
428 | free_percpu(pdata: gt_evt); |
429 | out_clk_nb: |
430 | clk_notifier_unregister(clk: gt_clk, nb: >_clk_rate_change_nb); |
431 | out_clk: |
432 | clk_disable_unprepare(clk: gt_clk); |
433 | out_unmap: |
434 | iounmap(addr: gt_base); |
435 | WARN(err, "ARM Global timer register failed (%d)\n" , err); |
436 | |
437 | return err; |
438 | } |
439 | |
440 | /* Only tested on r2p2 and r3p0 */ |
441 | TIMER_OF_DECLARE(arm_gt, "arm,cortex-a9-global-timer" , |
442 | global_timer_of_register); |
443 | |