1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * Copyright (C) 2015-2016 Socionext Inc. |
4 | * Author: Masahiro Yamada <yamada.masahiro@socionext.com> |
5 | */ |
6 | |
7 | #define pr_fmt(fmt) "uniphier: " fmt |
8 | |
9 | #include <linux/bitops.h> |
10 | #include <linux/init.h> |
11 | #include <linux/io.h> |
12 | #include <linux/log2.h> |
13 | #include <linux/of_address.h> |
14 | #include <linux/slab.h> |
15 | #include <asm/hardware/cache-uniphier.h> |
16 | #include <asm/outercache.h> |
17 | |
18 | /* control registers */ |
19 | #define UNIPHIER_SSCC 0x0 /* Control Register */ |
20 | #define UNIPHIER_SSCC_BST BIT(20) /* UCWG burst read */ |
21 | #define UNIPHIER_SSCC_ACT BIT(19) /* Inst-Data separate */ |
22 | #define UNIPHIER_SSCC_WTG BIT(18) /* WT gathering on */ |
23 | #define UNIPHIER_SSCC_PRD BIT(17) /* enable pre-fetch */ |
24 | #define UNIPHIER_SSCC_ON BIT(0) /* enable cache */ |
25 | #define UNIPHIER_SSCLPDAWCR 0x30 /* Unified/Data Active Way Control */ |
26 | #define UNIPHIER_SSCLPIAWCR 0x34 /* Instruction Active Way Control */ |
27 | |
28 | /* revision registers */ |
29 | #define UNIPHIER_SSCID 0x0 /* ID Register */ |
30 | |
31 | /* operation registers */ |
32 | #define UNIPHIER_SSCOPE 0x244 /* Cache Operation Primitive Entry */ |
33 | #define UNIPHIER_SSCOPE_CM_INV 0x0 /* invalidate */ |
34 | #define UNIPHIER_SSCOPE_CM_CLEAN 0x1 /* clean */ |
35 | #define UNIPHIER_SSCOPE_CM_FLUSH 0x2 /* flush */ |
36 | #define UNIPHIER_SSCOPE_CM_SYNC 0x8 /* sync (drain bufs) */ |
37 | #define UNIPHIER_SSCOPE_CM_FLUSH_PREFETCH 0x9 /* flush p-fetch buf */ |
38 | #define UNIPHIER_SSCOQM 0x248 /* Cache Operation Queue Mode */ |
39 | #define UNIPHIER_SSCOQM_S_MASK (0x3 << 17) |
40 | #define UNIPHIER_SSCOQM_S_RANGE (0x0 << 17) |
41 | #define UNIPHIER_SSCOQM_S_ALL (0x1 << 17) |
42 | #define UNIPHIER_SSCOQM_CE BIT(15) /* notify completion */ |
43 | #define UNIPHIER_SSCOQM_CM_INV 0x0 /* invalidate */ |
44 | #define UNIPHIER_SSCOQM_CM_CLEAN 0x1 /* clean */ |
45 | #define UNIPHIER_SSCOQM_CM_FLUSH 0x2 /* flush */ |
46 | #define UNIPHIER_SSCOQAD 0x24c /* Cache Operation Queue Address */ |
47 | #define UNIPHIER_SSCOQSZ 0x250 /* Cache Operation Queue Size */ |
48 | #define UNIPHIER_SSCOPPQSEF 0x25c /* Cache Operation Queue Set Complete*/ |
49 | #define UNIPHIER_SSCOPPQSEF_FE BIT(1) |
50 | #define UNIPHIER_SSCOPPQSEF_OE BIT(0) |
51 | #define UNIPHIER_SSCOLPQS 0x260 /* Cache Operation Queue Status */ |
52 | #define UNIPHIER_SSCOLPQS_EF BIT(2) |
53 | #define UNIPHIER_SSCOLPQS_EST BIT(1) |
54 | #define UNIPHIER_SSCOLPQS_QST BIT(0) |
55 | |
56 | /* Is the operation region specified by address range? */ |
57 | #define UNIPHIER_SSCOQM_S_IS_RANGE(op) \ |
58 | ((op & UNIPHIER_SSCOQM_S_MASK) == UNIPHIER_SSCOQM_S_RANGE) |
59 | |
60 | /** |
61 | * struct uniphier_cache_data - UniPhier outer cache specific data |
62 | * |
63 | * @ctrl_base: virtual base address of control registers |
64 | * @rev_base: virtual base address of revision registers |
65 | * @op_base: virtual base address of operation registers |
66 | * @way_ctrl_base: virtual address of the way control registers for this |
67 | * SoC revision |
68 | * @way_mask: each bit specifies if the way is present |
69 | * @nsets: number of associativity sets |
70 | * @line_size: line size in bytes |
71 | * @range_op_max_size: max size that can be handled by a single range operation |
72 | * @list: list node to include this level in the whole cache hierarchy |
73 | */ |
74 | struct uniphier_cache_data { |
75 | void __iomem *ctrl_base; |
76 | void __iomem *rev_base; |
77 | void __iomem *op_base; |
78 | void __iomem *way_ctrl_base; |
79 | u32 way_mask; |
80 | u32 nsets; |
81 | u32 line_size; |
82 | u32 range_op_max_size; |
83 | struct list_head list; |
84 | }; |
85 | |
86 | /* |
87 | * List of the whole outer cache hierarchy. This list is only modified during |
88 | * the early boot stage, so no mutex is taken for the access to the list. |
89 | */ |
90 | static LIST_HEAD(uniphier_cache_list); |
91 | |
92 | /** |
93 | * __uniphier_cache_sync - perform a sync point for a particular cache level |
94 | * |
95 | * @data: cache controller specific data |
96 | */ |
97 | static void __uniphier_cache_sync(struct uniphier_cache_data *data) |
98 | { |
99 | /* This sequence need not be atomic. Do not disable IRQ. */ |
100 | writel_relaxed(UNIPHIER_SSCOPE_CM_SYNC, |
101 | data->op_base + UNIPHIER_SSCOPE); |
102 | /* need a read back to confirm */ |
103 | readl_relaxed(data->op_base + UNIPHIER_SSCOPE); |
104 | } |
105 | |
106 | /** |
107 | * __uniphier_cache_maint_common - run a queue operation for a particular level |
108 | * |
109 | * @data: cache controller specific data |
110 | * @start: start address of range operation (don't care for "all" operation) |
111 | * @size: data size of range operation (don't care for "all" operation) |
112 | * @operation: flags to specify the desired cache operation |
113 | */ |
114 | static void __uniphier_cache_maint_common(struct uniphier_cache_data *data, |
115 | unsigned long start, |
116 | unsigned long size, |
117 | u32 operation) |
118 | { |
119 | unsigned long flags; |
120 | |
121 | /* |
122 | * No spin lock is necessary here because: |
123 | * |
124 | * [1] This outer cache controller is able to accept maintenance |
125 | * operations from multiple CPUs at a time in an SMP system; if a |
126 | * maintenance operation is under way and another operation is issued, |
127 | * the new one is stored in the queue. The controller performs one |
128 | * operation after another. If the queue is full, the status register, |
129 | * UNIPHIER_SSCOPPQSEF, indicates that the queue registration has |
130 | * failed. The status registers, UNIPHIER_{SSCOPPQSEF, SSCOLPQS}, have |
131 | * different instances for each CPU, i.e. each CPU can track the status |
132 | * of the maintenance operations triggered by itself. |
133 | * |
134 | * [2] The cache command registers, UNIPHIER_{SSCOQM, SSCOQAD, SSCOQSZ, |
135 | * SSCOQWN}, are shared between multiple CPUs, but the hardware still |
136 | * guarantees the registration sequence is atomic; the write access to |
137 | * them are arbitrated by the hardware. The first accessor to the |
138 | * register, UNIPHIER_SSCOQM, holds the access right and it is released |
139 | * by reading the status register, UNIPHIER_SSCOPPQSEF. While one CPU |
140 | * is holding the access right, other CPUs fail to register operations. |
141 | * One CPU should not hold the access right for a long time, so local |
142 | * IRQs should be disabled while the following sequence. |
143 | */ |
144 | local_irq_save(flags); |
145 | |
146 | /* clear the complete notification flag */ |
147 | writel_relaxed(UNIPHIER_SSCOLPQS_EF, data->op_base + UNIPHIER_SSCOLPQS); |
148 | |
149 | do { |
150 | /* set cache operation */ |
151 | writel_relaxed(UNIPHIER_SSCOQM_CE | operation, |
152 | data->op_base + UNIPHIER_SSCOQM); |
153 | |
154 | /* set address range if needed */ |
155 | if (likely(UNIPHIER_SSCOQM_S_IS_RANGE(operation))) { |
156 | writel_relaxed(start, data->op_base + UNIPHIER_SSCOQAD); |
157 | writel_relaxed(size, data->op_base + UNIPHIER_SSCOQSZ); |
158 | } |
159 | } while (unlikely(readl_relaxed(data->op_base + UNIPHIER_SSCOPPQSEF) & |
160 | (UNIPHIER_SSCOPPQSEF_FE | UNIPHIER_SSCOPPQSEF_OE))); |
161 | |
162 | /* wait until the operation is completed */ |
163 | while (likely(readl_relaxed(data->op_base + UNIPHIER_SSCOLPQS) != |
164 | UNIPHIER_SSCOLPQS_EF)) |
165 | cpu_relax(); |
166 | |
167 | local_irq_restore(flags); |
168 | } |
169 | |
170 | static void __uniphier_cache_maint_all(struct uniphier_cache_data *data, |
171 | u32 operation) |
172 | { |
173 | __uniphier_cache_maint_common(data, start: 0, size: 0, |
174 | UNIPHIER_SSCOQM_S_ALL | operation); |
175 | |
176 | __uniphier_cache_sync(data); |
177 | } |
178 | |
179 | static void __uniphier_cache_maint_range(struct uniphier_cache_data *data, |
180 | unsigned long start, unsigned long end, |
181 | u32 operation) |
182 | { |
183 | unsigned long size; |
184 | |
185 | /* |
186 | * If the start address is not aligned, |
187 | * perform a cache operation for the first cache-line |
188 | */ |
189 | start = start & ~(data->line_size - 1); |
190 | |
191 | size = end - start; |
192 | |
193 | if (unlikely(size >= (unsigned long)(-data->line_size))) { |
194 | /* this means cache operation for all range */ |
195 | __uniphier_cache_maint_all(data, operation); |
196 | return; |
197 | } |
198 | |
199 | /* |
200 | * If the end address is not aligned, |
201 | * perform a cache operation for the last cache-line |
202 | */ |
203 | size = ALIGN(size, data->line_size); |
204 | |
205 | while (size) { |
206 | unsigned long chunk_size = min_t(unsigned long, size, |
207 | data->range_op_max_size); |
208 | |
209 | __uniphier_cache_maint_common(data, start, size: chunk_size, |
210 | UNIPHIER_SSCOQM_S_RANGE | operation); |
211 | |
212 | start += chunk_size; |
213 | size -= chunk_size; |
214 | } |
215 | |
216 | __uniphier_cache_sync(data); |
217 | } |
218 | |
219 | static void __uniphier_cache_enable(struct uniphier_cache_data *data, bool on) |
220 | { |
221 | u32 val = 0; |
222 | |
223 | if (on) |
224 | val = UNIPHIER_SSCC_WTG | UNIPHIER_SSCC_PRD | UNIPHIER_SSCC_ON; |
225 | |
226 | writel_relaxed(val, data->ctrl_base + UNIPHIER_SSCC); |
227 | } |
228 | |
229 | static void __init __uniphier_cache_set_active_ways( |
230 | struct uniphier_cache_data *data) |
231 | { |
232 | unsigned int cpu; |
233 | |
234 | for_each_possible_cpu(cpu) |
235 | writel_relaxed(data->way_mask, data->way_ctrl_base + 4 * cpu); |
236 | } |
237 | |
238 | static void uniphier_cache_maint_range(unsigned long start, unsigned long end, |
239 | u32 operation) |
240 | { |
241 | struct uniphier_cache_data *data; |
242 | |
243 | list_for_each_entry(data, &uniphier_cache_list, list) |
244 | __uniphier_cache_maint_range(data, start, end, operation); |
245 | } |
246 | |
247 | static void uniphier_cache_maint_all(u32 operation) |
248 | { |
249 | struct uniphier_cache_data *data; |
250 | |
251 | list_for_each_entry(data, &uniphier_cache_list, list) |
252 | __uniphier_cache_maint_all(data, operation); |
253 | } |
254 | |
255 | static void uniphier_cache_inv_range(unsigned long start, unsigned long end) |
256 | { |
257 | uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_INV); |
258 | } |
259 | |
260 | static void uniphier_cache_clean_range(unsigned long start, unsigned long end) |
261 | { |
262 | uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_CLEAN); |
263 | } |
264 | |
265 | static void uniphier_cache_flush_range(unsigned long start, unsigned long end) |
266 | { |
267 | uniphier_cache_maint_range(start, end, UNIPHIER_SSCOQM_CM_FLUSH); |
268 | } |
269 | |
270 | static void __init uniphier_cache_inv_all(void) |
271 | { |
272 | uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_INV); |
273 | } |
274 | |
275 | static void uniphier_cache_flush_all(void) |
276 | { |
277 | uniphier_cache_maint_all(UNIPHIER_SSCOQM_CM_FLUSH); |
278 | } |
279 | |
280 | static void uniphier_cache_disable(void) |
281 | { |
282 | struct uniphier_cache_data *data; |
283 | |
284 | list_for_each_entry_reverse(data, &uniphier_cache_list, list) |
285 | __uniphier_cache_enable(data, on: false); |
286 | |
287 | uniphier_cache_flush_all(); |
288 | } |
289 | |
290 | static void __init uniphier_cache_enable(void) |
291 | { |
292 | struct uniphier_cache_data *data; |
293 | |
294 | uniphier_cache_inv_all(); |
295 | |
296 | list_for_each_entry(data, &uniphier_cache_list, list) { |
297 | __uniphier_cache_enable(data, on: true); |
298 | __uniphier_cache_set_active_ways(data); |
299 | } |
300 | } |
301 | |
302 | static void uniphier_cache_sync(void) |
303 | { |
304 | struct uniphier_cache_data *data; |
305 | |
306 | list_for_each_entry(data, &uniphier_cache_list, list) |
307 | __uniphier_cache_sync(data); |
308 | } |
309 | |
310 | static const struct of_device_id uniphier_cache_match[] __initconst = { |
311 | { .compatible = "socionext,uniphier-system-cache" }, |
312 | { /* sentinel */ } |
313 | }; |
314 | |
315 | static int __init __uniphier_cache_init(struct device_node *np, |
316 | unsigned int *cache_level) |
317 | { |
318 | struct uniphier_cache_data *data; |
319 | u32 level, cache_size; |
320 | struct device_node *next_np; |
321 | int ret = 0; |
322 | |
323 | if (!of_match_node(matches: uniphier_cache_match, node: np)) { |
324 | pr_err("L%d: not compatible with uniphier cache\n" , |
325 | *cache_level); |
326 | return -EINVAL; |
327 | } |
328 | |
329 | if (of_property_read_u32(np, propname: "cache-level" , out_value: &level)) { |
330 | pr_err("L%d: cache-level is not specified\n" , *cache_level); |
331 | return -EINVAL; |
332 | } |
333 | |
334 | if (level != *cache_level) { |
335 | pr_err("L%d: cache-level is unexpected value %d\n" , |
336 | *cache_level, level); |
337 | return -EINVAL; |
338 | } |
339 | |
340 | if (!of_property_read_bool(np, propname: "cache-unified" )) { |
341 | pr_err("L%d: cache-unified is not specified\n" , *cache_level); |
342 | return -EINVAL; |
343 | } |
344 | |
345 | data = kzalloc(size: sizeof(*data), GFP_KERNEL); |
346 | if (!data) |
347 | return -ENOMEM; |
348 | |
349 | if (of_property_read_u32(np, propname: "cache-line-size" , out_value: &data->line_size) || |
350 | !is_power_of_2(n: data->line_size)) { |
351 | pr_err("L%d: cache-line-size is unspecified or invalid\n" , |
352 | *cache_level); |
353 | ret = -EINVAL; |
354 | goto err; |
355 | } |
356 | |
357 | if (of_property_read_u32(np, propname: "cache-sets" , out_value: &data->nsets) || |
358 | !is_power_of_2(n: data->nsets)) { |
359 | pr_err("L%d: cache-sets is unspecified or invalid\n" , |
360 | *cache_level); |
361 | ret = -EINVAL; |
362 | goto err; |
363 | } |
364 | |
365 | if (of_property_read_u32(np, propname: "cache-size" , out_value: &cache_size) || |
366 | cache_size == 0 || cache_size % (data->nsets * data->line_size)) { |
367 | pr_err("L%d: cache-size is unspecified or invalid\n" , |
368 | *cache_level); |
369 | ret = -EINVAL; |
370 | goto err; |
371 | } |
372 | |
373 | data->way_mask = GENMASK(cache_size / data->nsets / data->line_size - 1, |
374 | 0); |
375 | |
376 | data->ctrl_base = of_iomap(node: np, index: 0); |
377 | if (!data->ctrl_base) { |
378 | pr_err("L%d: failed to map control register\n" , *cache_level); |
379 | ret = -ENOMEM; |
380 | goto err; |
381 | } |
382 | |
383 | data->rev_base = of_iomap(node: np, index: 1); |
384 | if (!data->rev_base) { |
385 | pr_err("L%d: failed to map revision register\n" , *cache_level); |
386 | ret = -ENOMEM; |
387 | goto err; |
388 | } |
389 | |
390 | data->op_base = of_iomap(node: np, index: 2); |
391 | if (!data->op_base) { |
392 | pr_err("L%d: failed to map operation register\n" , *cache_level); |
393 | ret = -ENOMEM; |
394 | goto err; |
395 | } |
396 | |
397 | data->way_ctrl_base = data->ctrl_base + 0xc00; |
398 | |
399 | if (*cache_level == 2) { |
400 | u32 revision = readl(addr: data->rev_base + UNIPHIER_SSCID); |
401 | /* |
402 | * The size of range operation is limited to (1 << 22) or less |
403 | * for PH-sLD8 or older SoCs. |
404 | */ |
405 | if (revision <= 0x16) |
406 | data->range_op_max_size = (u32)1 << 22; |
407 | |
408 | /* |
409 | * Unfortunatly, the offset address of active way control base |
410 | * varies from SoC to SoC. |
411 | */ |
412 | switch (revision) { |
413 | case 0x11: /* sLD3 */ |
414 | data->way_ctrl_base = data->ctrl_base + 0x870; |
415 | break; |
416 | case 0x12: /* LD4 */ |
417 | case 0x16: /* sld8 */ |
418 | data->way_ctrl_base = data->ctrl_base + 0x840; |
419 | break; |
420 | default: |
421 | break; |
422 | } |
423 | } |
424 | |
425 | data->range_op_max_size -= data->line_size; |
426 | |
427 | INIT_LIST_HEAD(list: &data->list); |
428 | list_add_tail(new: &data->list, head: &uniphier_cache_list); /* no mutex */ |
429 | |
430 | /* |
431 | * OK, this level has been successfully initialized. Look for the next |
432 | * level cache. Do not roll back even if the initialization of the |
433 | * next level cache fails because we want to continue with available |
434 | * cache levels. |
435 | */ |
436 | next_np = of_find_next_cache_node(np); |
437 | if (next_np) { |
438 | (*cache_level)++; |
439 | ret = __uniphier_cache_init(np: next_np, cache_level); |
440 | } |
441 | of_node_put(node: next_np); |
442 | |
443 | return ret; |
444 | err: |
445 | iounmap(addr: data->op_base); |
446 | iounmap(addr: data->rev_base); |
447 | iounmap(addr: data->ctrl_base); |
448 | kfree(objp: data); |
449 | |
450 | return ret; |
451 | } |
452 | |
453 | int __init uniphier_cache_init(void) |
454 | { |
455 | struct device_node *np = NULL; |
456 | unsigned int cache_level; |
457 | int ret = 0; |
458 | |
459 | /* look for level 2 cache */ |
460 | while ((np = of_find_matching_node(from: np, matches: uniphier_cache_match))) |
461 | if (!of_property_read_u32(np, propname: "cache-level" , out_value: &cache_level) && |
462 | cache_level == 2) |
463 | break; |
464 | |
465 | if (!np) |
466 | return -ENODEV; |
467 | |
468 | ret = __uniphier_cache_init(np, cache_level: &cache_level); |
469 | of_node_put(node: np); |
470 | |
471 | if (ret) { |
472 | /* |
473 | * Error out iif L2 initialization fails. Continue with any |
474 | * error on L3 or outer because they are optional. |
475 | */ |
476 | if (cache_level == 2) { |
477 | pr_err("failed to initialize L2 cache\n" ); |
478 | return ret; |
479 | } |
480 | |
481 | cache_level--; |
482 | ret = 0; |
483 | } |
484 | |
485 | outer_cache.inv_range = uniphier_cache_inv_range; |
486 | outer_cache.clean_range = uniphier_cache_clean_range; |
487 | outer_cache.flush_range = uniphier_cache_flush_range; |
488 | outer_cache.flush_all = uniphier_cache_flush_all; |
489 | outer_cache.disable = uniphier_cache_disable; |
490 | outer_cache.sync = uniphier_cache_sync; |
491 | |
492 | uniphier_cache_enable(); |
493 | |
494 | pr_info("enabled outer cache (cache level: %d)\n" , cache_level); |
495 | |
496 | return ret; |
497 | } |
498 | |