1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * SiFive composable cache controller Driver |
4 | * |
5 | * Copyright (C) 2018-2022 SiFive, Inc. |
6 | * |
7 | */ |
8 | |
9 | #define pr_fmt(fmt) "CCACHE: " fmt |
10 | |
11 | #include <linux/align.h> |
12 | #include <linux/debugfs.h> |
13 | #include <linux/interrupt.h> |
14 | #include <linux/of_irq.h> |
15 | #include <linux/of_address.h> |
16 | #include <linux/device.h> |
17 | #include <linux/bitfield.h> |
18 | #include <linux/platform_device.h> |
19 | #include <linux/property.h> |
20 | #include <asm/cacheflush.h> |
21 | #include <asm/cacheinfo.h> |
22 | #include <asm/dma-noncoherent.h> |
23 | #include <soc/sifive/sifive_ccache.h> |
24 | |
25 | #define SIFIVE_CCACHE_DIRECCFIX_LOW 0x100 |
26 | #define SIFIVE_CCACHE_DIRECCFIX_HIGH 0x104 |
27 | #define SIFIVE_CCACHE_DIRECCFIX_COUNT 0x108 |
28 | |
29 | #define SIFIVE_CCACHE_DIRECCFAIL_LOW 0x120 |
30 | #define SIFIVE_CCACHE_DIRECCFAIL_HIGH 0x124 |
31 | #define SIFIVE_CCACHE_DIRECCFAIL_COUNT 0x128 |
32 | |
33 | #define SIFIVE_CCACHE_DATECCFIX_LOW 0x140 |
34 | #define SIFIVE_CCACHE_DATECCFIX_HIGH 0x144 |
35 | #define SIFIVE_CCACHE_DATECCFIX_COUNT 0x148 |
36 | |
37 | #define SIFIVE_CCACHE_DATECCFAIL_LOW 0x160 |
38 | #define SIFIVE_CCACHE_DATECCFAIL_HIGH 0x164 |
39 | #define SIFIVE_CCACHE_DATECCFAIL_COUNT 0x168 |
40 | |
41 | #define SIFIVE_CCACHE_CONFIG 0x00 |
42 | #define SIFIVE_CCACHE_CONFIG_BANK_MASK GENMASK_ULL(7, 0) |
43 | #define SIFIVE_CCACHE_CONFIG_WAYS_MASK GENMASK_ULL(15, 8) |
44 | #define SIFIVE_CCACHE_CONFIG_SETS_MASK GENMASK_ULL(23, 16) |
45 | #define SIFIVE_CCACHE_CONFIG_BLKS_MASK GENMASK_ULL(31, 24) |
46 | |
47 | #define SIFIVE_CCACHE_FLUSH64 0x200 |
48 | #define SIFIVE_CCACHE_FLUSH32 0x240 |
49 | |
50 | #define SIFIVE_CCACHE_WAYENABLE 0x08 |
51 | #define SIFIVE_CCACHE_ECCINJECTERR 0x40 |
52 | |
53 | #define SIFIVE_CCACHE_MAX_ECCINTR 4 |
54 | #define SIFIVE_CCACHE_LINE_SIZE 64 |
55 | |
56 | static void __iomem *ccache_base; |
57 | static int g_irq[SIFIVE_CCACHE_MAX_ECCINTR]; |
58 | static struct riscv_cacheinfo_ops ccache_cache_ops; |
59 | static int level; |
60 | |
61 | enum { |
62 | DIR_CORR = 0, |
63 | DATA_CORR, |
64 | DATA_UNCORR, |
65 | DIR_UNCORR, |
66 | }; |
67 | |
68 | enum { |
69 | QUIRK_NONSTANDARD_CACHE_OPS = BIT(0), |
70 | QUIRK_BROKEN_DATA_UNCORR = BIT(1), |
71 | }; |
72 | |
73 | #ifdef CONFIG_DEBUG_FS |
74 | static struct dentry *sifive_test; |
75 | |
76 | static ssize_t ccache_write(struct file *file, const char __user *data, |
77 | size_t count, loff_t *ppos) |
78 | { |
79 | unsigned int val; |
80 | |
81 | if (kstrtouint_from_user(s: data, count, base: 0, res: &val)) |
82 | return -EINVAL; |
83 | if ((val < 0xFF) || (val >= 0x10000 && val < 0x100FF)) |
84 | writel(val, addr: ccache_base + SIFIVE_CCACHE_ECCINJECTERR); |
85 | else |
86 | return -EINVAL; |
87 | return count; |
88 | } |
89 | |
90 | static const struct file_operations ccache_fops = { |
91 | .owner = THIS_MODULE, |
92 | .open = simple_open, |
93 | .write = ccache_write |
94 | }; |
95 | |
96 | static void setup_sifive_debug(void) |
97 | { |
98 | sifive_test = debugfs_create_dir(name: "sifive_ccache_cache" , NULL); |
99 | |
100 | debugfs_create_file(name: "sifive_debug_inject_error" , mode: 0200, |
101 | parent: sifive_test, NULL, fops: &ccache_fops); |
102 | } |
103 | #endif |
104 | |
105 | static void ccache_config_read(void) |
106 | { |
107 | u32 cfg; |
108 | |
109 | cfg = readl(addr: ccache_base + SIFIVE_CCACHE_CONFIG); |
110 | pr_info("%llu banks, %llu ways, sets/bank=%llu, bytes/block=%llu\n" , |
111 | FIELD_GET(SIFIVE_CCACHE_CONFIG_BANK_MASK, cfg), |
112 | FIELD_GET(SIFIVE_CCACHE_CONFIG_WAYS_MASK, cfg), |
113 | BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_SETS_MASK, cfg)), |
114 | BIT_ULL(FIELD_GET(SIFIVE_CCACHE_CONFIG_BLKS_MASK, cfg))); |
115 | |
116 | cfg = readl(addr: ccache_base + SIFIVE_CCACHE_WAYENABLE); |
117 | pr_info("Index of the largest way enabled: %u\n" , cfg); |
118 | } |
119 | |
120 | static const struct of_device_id sifive_ccache_ids[] = { |
121 | { .compatible = "sifive,fu540-c000-ccache" }, |
122 | { .compatible = "sifive,fu740-c000-ccache" }, |
123 | { .compatible = "starfive,jh7100-ccache" , |
124 | .data = (void *)(QUIRK_NONSTANDARD_CACHE_OPS | QUIRK_BROKEN_DATA_UNCORR) }, |
125 | { .compatible = "sifive,ccache0" }, |
126 | { /* end of table */ } |
127 | }; |
128 | |
129 | static ATOMIC_NOTIFIER_HEAD(ccache_err_chain); |
130 | |
131 | int register_sifive_ccache_error_notifier(struct notifier_block *nb) |
132 | { |
133 | return atomic_notifier_chain_register(nh: &ccache_err_chain, nb); |
134 | } |
135 | EXPORT_SYMBOL_GPL(register_sifive_ccache_error_notifier); |
136 | |
137 | int unregister_sifive_ccache_error_notifier(struct notifier_block *nb) |
138 | { |
139 | return atomic_notifier_chain_unregister(nh: &ccache_err_chain, nb); |
140 | } |
141 | EXPORT_SYMBOL_GPL(unregister_sifive_ccache_error_notifier); |
142 | |
143 | #ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS |
144 | static void ccache_flush_range(phys_addr_t start, size_t len) |
145 | { |
146 | phys_addr_t end = start + len; |
147 | phys_addr_t line; |
148 | |
149 | if (!len) |
150 | return; |
151 | |
152 | mb(); |
153 | for (line = ALIGN_DOWN(start, SIFIVE_CCACHE_LINE_SIZE); line < end; |
154 | line += SIFIVE_CCACHE_LINE_SIZE) { |
155 | #ifdef CONFIG_32BIT |
156 | writel(line >> 4, ccache_base + SIFIVE_CCACHE_FLUSH32); |
157 | #else |
158 | writeq(line, ccache_base + SIFIVE_CCACHE_FLUSH64); |
159 | #endif |
160 | mb(); |
161 | } |
162 | } |
163 | |
164 | static const struct riscv_nonstd_cache_ops ccache_mgmt_ops __initconst = { |
165 | .wback = &ccache_flush_range, |
166 | .inv = &ccache_flush_range, |
167 | .wback_inv = &ccache_flush_range, |
168 | }; |
169 | #endif /* CONFIG_RISCV_NONSTANDARD_CACHE_OPS */ |
170 | |
171 | static int ccache_largest_wayenabled(void) |
172 | { |
173 | return readl(addr: ccache_base + SIFIVE_CCACHE_WAYENABLE) & 0xFF; |
174 | } |
175 | |
176 | static ssize_t number_of_ways_enabled_show(struct device *dev, |
177 | struct device_attribute *attr, |
178 | char *buf) |
179 | { |
180 | return sprintf(buf, fmt: "%u\n" , ccache_largest_wayenabled()); |
181 | } |
182 | |
183 | static DEVICE_ATTR_RO(number_of_ways_enabled); |
184 | |
185 | static struct attribute *priv_attrs[] = { |
186 | &dev_attr_number_of_ways_enabled.attr, |
187 | NULL, |
188 | }; |
189 | |
190 | static const struct attribute_group priv_attr_group = { |
191 | .attrs = priv_attrs, |
192 | }; |
193 | |
194 | static const struct attribute_group *ccache_get_priv_group(struct cacheinfo |
195 | *this_leaf) |
196 | { |
197 | /* We want to use private group for composable cache only */ |
198 | if (this_leaf->level == level) |
199 | return &priv_attr_group; |
200 | else |
201 | return NULL; |
202 | } |
203 | |
204 | static irqreturn_t ccache_int_handler(int irq, void *device) |
205 | { |
206 | unsigned int add_h, add_l; |
207 | |
208 | if (irq == g_irq[DIR_CORR]) { |
209 | add_h = readl(addr: ccache_base + SIFIVE_CCACHE_DIRECCFIX_HIGH); |
210 | add_l = readl(addr: ccache_base + SIFIVE_CCACHE_DIRECCFIX_LOW); |
211 | pr_err("DirError @ 0x%08X.%08X\n" , add_h, add_l); |
212 | /* Reading this register clears the DirError interrupt sig */ |
213 | readl(addr: ccache_base + SIFIVE_CCACHE_DIRECCFIX_COUNT); |
214 | atomic_notifier_call_chain(nh: &ccache_err_chain, |
215 | SIFIVE_CCACHE_ERR_TYPE_CE, |
216 | v: "DirECCFix" ); |
217 | } |
218 | if (irq == g_irq[DIR_UNCORR]) { |
219 | add_h = readl(addr: ccache_base + SIFIVE_CCACHE_DIRECCFAIL_HIGH); |
220 | add_l = readl(addr: ccache_base + SIFIVE_CCACHE_DIRECCFAIL_LOW); |
221 | /* Reading this register clears the DirFail interrupt sig */ |
222 | readl(addr: ccache_base + SIFIVE_CCACHE_DIRECCFAIL_COUNT); |
223 | atomic_notifier_call_chain(nh: &ccache_err_chain, |
224 | SIFIVE_CCACHE_ERR_TYPE_UE, |
225 | v: "DirECCFail" ); |
226 | panic(fmt: "CCACHE: DirFail @ 0x%08X.%08X\n" , add_h, add_l); |
227 | } |
228 | if (irq == g_irq[DATA_CORR]) { |
229 | add_h = readl(addr: ccache_base + SIFIVE_CCACHE_DATECCFIX_HIGH); |
230 | add_l = readl(addr: ccache_base + SIFIVE_CCACHE_DATECCFIX_LOW); |
231 | pr_err("DataError @ 0x%08X.%08X\n" , add_h, add_l); |
232 | /* Reading this register clears the DataError interrupt sig */ |
233 | readl(addr: ccache_base + SIFIVE_CCACHE_DATECCFIX_COUNT); |
234 | atomic_notifier_call_chain(nh: &ccache_err_chain, |
235 | SIFIVE_CCACHE_ERR_TYPE_CE, |
236 | v: "DatECCFix" ); |
237 | } |
238 | if (irq == g_irq[DATA_UNCORR]) { |
239 | add_h = readl(addr: ccache_base + SIFIVE_CCACHE_DATECCFAIL_HIGH); |
240 | add_l = readl(addr: ccache_base + SIFIVE_CCACHE_DATECCFAIL_LOW); |
241 | pr_err("DataFail @ 0x%08X.%08X\n" , add_h, add_l); |
242 | /* Reading this register clears the DataFail interrupt sig */ |
243 | readl(addr: ccache_base + SIFIVE_CCACHE_DATECCFAIL_COUNT); |
244 | atomic_notifier_call_chain(nh: &ccache_err_chain, |
245 | SIFIVE_CCACHE_ERR_TYPE_UE, |
246 | v: "DatECCFail" ); |
247 | } |
248 | |
249 | return IRQ_HANDLED; |
250 | } |
251 | |
252 | static int sifive_ccache_probe(struct platform_device *pdev) |
253 | { |
254 | struct device *dev = &pdev->dev; |
255 | unsigned long quirks; |
256 | int intr_num, rc; |
257 | |
258 | quirks = (unsigned long)device_get_match_data(dev); |
259 | |
260 | intr_num = platform_irq_count(pdev); |
261 | if (!intr_num) |
262 | return dev_err_probe(dev, err: -ENODEV, fmt: "No interrupts property\n" ); |
263 | |
264 | for (int i = 0; i < intr_num; i++) { |
265 | if (i == DATA_UNCORR && (quirks & QUIRK_BROKEN_DATA_UNCORR)) |
266 | continue; |
267 | |
268 | g_irq[i] = platform_get_irq(pdev, i); |
269 | if (g_irq[i] < 0) |
270 | return g_irq[i]; |
271 | |
272 | rc = devm_request_irq(dev, irq: g_irq[i], handler: ccache_int_handler, irqflags: 0, devname: "ccache_ecc" , NULL); |
273 | if (rc) |
274 | return dev_err_probe(dev, err: rc, fmt: "Could not request IRQ %d\n" , g_irq[i]); |
275 | } |
276 | |
277 | return 0; |
278 | } |
279 | |
280 | static struct platform_driver sifive_ccache_driver = { |
281 | .probe = sifive_ccache_probe, |
282 | .driver = { |
283 | .name = "sifive_ccache" , |
284 | .of_match_table = sifive_ccache_ids, |
285 | }, |
286 | }; |
287 | |
288 | static int __init sifive_ccache_init(void) |
289 | { |
290 | struct device_node *np; |
291 | struct resource res; |
292 | const struct of_device_id *match; |
293 | unsigned long quirks; |
294 | int rc; |
295 | |
296 | np = of_find_matching_node_and_match(NULL, matches: sifive_ccache_ids, match: &match); |
297 | if (!np) |
298 | return -ENODEV; |
299 | |
300 | quirks = (uintptr_t)match->data; |
301 | |
302 | if (of_address_to_resource(dev: np, index: 0, r: &res)) { |
303 | rc = -ENODEV; |
304 | goto err_node_put; |
305 | } |
306 | |
307 | ccache_base = ioremap(offset: res.start, size: resource_size(res: &res)); |
308 | if (!ccache_base) { |
309 | rc = -ENOMEM; |
310 | goto err_node_put; |
311 | } |
312 | |
313 | if (of_property_read_u32(np, propname: "cache-level" , out_value: &level)) { |
314 | rc = -ENOENT; |
315 | goto err_unmap; |
316 | } |
317 | |
318 | #ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS |
319 | if (quirks & QUIRK_NONSTANDARD_CACHE_OPS) { |
320 | riscv_cbom_block_size = SIFIVE_CCACHE_LINE_SIZE; |
321 | riscv_noncoherent_supported(); |
322 | riscv_noncoherent_register_cache_ops(&ccache_mgmt_ops); |
323 | } |
324 | #endif |
325 | |
326 | ccache_config_read(); |
327 | |
328 | ccache_cache_ops.get_priv_group = ccache_get_priv_group; |
329 | riscv_set_cacheinfo_ops(&ccache_cache_ops); |
330 | |
331 | #ifdef CONFIG_DEBUG_FS |
332 | setup_sifive_debug(); |
333 | #endif |
334 | |
335 | rc = platform_driver_register(&sifive_ccache_driver); |
336 | if (rc) |
337 | goto err_unmap; |
338 | |
339 | of_node_put(node: np); |
340 | |
341 | return 0; |
342 | |
343 | err_unmap: |
344 | iounmap(addr: ccache_base); |
345 | err_node_put: |
346 | of_node_put(node: np); |
347 | return rc; |
348 | } |
349 | |
350 | arch_initcall(sifive_ccache_init); |
351 | |