1 | // SPDX-License-Identifier: GPL-2.0-only |
2 | /* |
3 | * Apple DART (Device Address Resolution Table) IOMMU driver |
4 | * |
5 | * Copyright (C) 2021 The Asahi Linux Contributors |
6 | * |
7 | * Based on arm/arm-smmu/arm-ssmu.c and arm/arm-smmu-v3/arm-smmu-v3.c |
8 | * Copyright (C) 2013 ARM Limited |
9 | * Copyright (C) 2015 ARM Limited |
10 | * and on exynos-iommu.c |
11 | * Copyright (c) 2011,2016 Samsung Electronics Co., Ltd. |
12 | */ |
13 | |
14 | #include <linux/atomic.h> |
15 | #include <linux/bitfield.h> |
16 | #include <linux/clk.h> |
17 | #include <linux/dev_printk.h> |
18 | #include <linux/dma-mapping.h> |
19 | #include <linux/err.h> |
20 | #include <linux/interrupt.h> |
21 | #include <linux/io-pgtable.h> |
22 | #include <linux/iommu.h> |
23 | #include <linux/iopoll.h> |
24 | #include <linux/module.h> |
25 | #include <linux/of.h> |
26 | #include <linux/of_address.h> |
27 | #include <linux/of_iommu.h> |
28 | #include <linux/of_platform.h> |
29 | #include <linux/pci.h> |
30 | #include <linux/platform_device.h> |
31 | #include <linux/slab.h> |
32 | #include <linux/swab.h> |
33 | #include <linux/types.h> |
34 | |
35 | #include "dma-iommu.h" |
36 | |
37 | #define DART_MAX_STREAMS 256 |
38 | #define DART_MAX_TTBR 4 |
39 | #define MAX_DARTS_PER_DEVICE 2 |
40 | |
41 | /* Common registers */ |
42 | |
43 | #define DART_PARAMS1 0x00 |
44 | #define DART_PARAMS1_PAGE_SHIFT GENMASK(27, 24) |
45 | |
46 | #define DART_PARAMS2 0x04 |
47 | #define DART_PARAMS2_BYPASS_SUPPORT BIT(0) |
48 | |
49 | /* T8020/T6000 registers */ |
50 | |
51 | #define DART_T8020_STREAM_COMMAND 0x20 |
52 | #define DART_T8020_STREAM_COMMAND_BUSY BIT(2) |
53 | #define DART_T8020_STREAM_COMMAND_INVALIDATE BIT(20) |
54 | |
55 | #define DART_T8020_STREAM_SELECT 0x34 |
56 | |
57 | #define DART_T8020_ERROR 0x40 |
58 | #define DART_T8020_ERROR_STREAM GENMASK(27, 24) |
59 | #define DART_T8020_ERROR_CODE GENMASK(11, 0) |
60 | #define DART_T8020_ERROR_FLAG BIT(31) |
61 | |
62 | #define DART_T8020_ERROR_READ_FAULT BIT(4) |
63 | #define DART_T8020_ERROR_WRITE_FAULT BIT(3) |
64 | #define DART_T8020_ERROR_NO_PTE BIT(2) |
65 | #define DART_T8020_ERROR_NO_PMD BIT(1) |
66 | #define DART_T8020_ERROR_NO_TTBR BIT(0) |
67 | |
68 | #define DART_T8020_CONFIG 0x60 |
69 | #define DART_T8020_CONFIG_LOCK BIT(15) |
70 | |
71 | #define DART_STREAM_COMMAND_BUSY_TIMEOUT 100 |
72 | |
73 | #define DART_T8020_ERROR_ADDR_HI 0x54 |
74 | #define DART_T8020_ERROR_ADDR_LO 0x50 |
75 | |
76 | #define DART_T8020_STREAMS_ENABLE 0xfc |
77 | |
78 | #define DART_T8020_TCR 0x100 |
79 | #define DART_T8020_TCR_TRANSLATE_ENABLE BIT(7) |
80 | #define DART_T8020_TCR_BYPASS_DART BIT(8) |
81 | #define DART_T8020_TCR_BYPASS_DAPF BIT(12) |
82 | |
83 | #define DART_T8020_TTBR 0x200 |
84 | #define DART_T8020_USB4_TTBR 0x400 |
85 | #define DART_T8020_TTBR_VALID BIT(31) |
86 | #define DART_T8020_TTBR_ADDR_FIELD_SHIFT 0 |
87 | #define DART_T8020_TTBR_SHIFT 12 |
88 | |
89 | /* T8110 registers */ |
90 | |
91 | #define DART_T8110_PARAMS3 0x08 |
92 | #define DART_T8110_PARAMS3_PA_WIDTH GENMASK(29, 24) |
93 | #define DART_T8110_PARAMS3_VA_WIDTH GENMASK(21, 16) |
94 | #define DART_T8110_PARAMS3_VER_MAJ GENMASK(15, 8) |
95 | #define DART_T8110_PARAMS3_VER_MIN GENMASK(7, 0) |
96 | |
97 | #define DART_T8110_PARAMS4 0x0c |
98 | #define DART_T8110_PARAMS4_NUM_CLIENTS GENMASK(24, 16) |
99 | #define DART_T8110_PARAMS4_NUM_SIDS GENMASK(8, 0) |
100 | |
101 | #define DART_T8110_TLB_CMD 0x80 |
102 | #define DART_T8110_TLB_CMD_BUSY BIT(31) |
103 | #define DART_T8110_TLB_CMD_OP GENMASK(10, 8) |
104 | #define DART_T8110_TLB_CMD_OP_FLUSH_ALL 0 |
105 | #define DART_T8110_TLB_CMD_OP_FLUSH_SID 1 |
106 | #define DART_T8110_TLB_CMD_STREAM GENMASK(7, 0) |
107 | |
108 | #define DART_T8110_ERROR 0x100 |
109 | #define DART_T8110_ERROR_STREAM GENMASK(27, 20) |
110 | #define DART_T8110_ERROR_CODE GENMASK(14, 0) |
111 | #define DART_T8110_ERROR_FLAG BIT(31) |
112 | |
113 | #define DART_T8110_ERROR_MASK 0x104 |
114 | |
115 | #define DART_T8110_ERROR_READ_FAULT BIT(5) |
116 | #define DART_T8110_ERROR_WRITE_FAULT BIT(4) |
117 | #define DART_T8110_ERROR_NO_PTE BIT(3) |
118 | #define DART_T8110_ERROR_NO_PMD BIT(2) |
119 | #define DART_T8110_ERROR_NO_PGD BIT(1) |
120 | #define DART_T8110_ERROR_NO_TTBR BIT(0) |
121 | |
122 | #define DART_T8110_ERROR_ADDR_LO 0x170 |
123 | #define DART_T8110_ERROR_ADDR_HI 0x174 |
124 | |
125 | #define DART_T8110_PROTECT 0x200 |
126 | #define DART_T8110_UNPROTECT 0x204 |
127 | #define DART_T8110_PROTECT_LOCK 0x208 |
128 | #define DART_T8110_PROTECT_TTBR_TCR BIT(0) |
129 | |
130 | #define DART_T8110_ENABLE_STREAMS 0xc00 |
131 | #define DART_T8110_DISABLE_STREAMS 0xc20 |
132 | |
133 | #define DART_T8110_TCR 0x1000 |
134 | #define DART_T8110_TCR_REMAP GENMASK(11, 8) |
135 | #define DART_T8110_TCR_REMAP_EN BIT(7) |
136 | #define DART_T8110_TCR_BYPASS_DAPF BIT(2) |
137 | #define DART_T8110_TCR_BYPASS_DART BIT(1) |
138 | #define DART_T8110_TCR_TRANSLATE_ENABLE BIT(0) |
139 | |
140 | #define DART_T8110_TTBR 0x1400 |
141 | #define DART_T8110_TTBR_VALID BIT(0) |
142 | #define DART_T8110_TTBR_ADDR_FIELD_SHIFT 2 |
143 | #define DART_T8110_TTBR_SHIFT 14 |
144 | |
145 | #define DART_TCR(dart, sid) ((dart)->hw->tcr + ((sid) << 2)) |
146 | |
147 | #define DART_TTBR(dart, sid, idx) ((dart)->hw->ttbr + \ |
148 | (((dart)->hw->ttbr_count * (sid)) << 2) + \ |
149 | ((idx) << 2)) |
150 | |
151 | struct apple_dart_stream_map; |
152 | |
153 | enum dart_type { |
154 | DART_T8020, |
155 | DART_T6000, |
156 | DART_T8110, |
157 | }; |
158 | |
159 | struct apple_dart_hw { |
160 | enum dart_type type; |
161 | irqreturn_t (*irq_handler)(int irq, void *dev); |
162 | int (*invalidate_tlb)(struct apple_dart_stream_map *stream_map); |
163 | |
164 | u32 oas; |
165 | enum io_pgtable_fmt fmt; |
166 | |
167 | int max_sid_count; |
168 | |
169 | u64 lock; |
170 | u64 lock_bit; |
171 | |
172 | u64 error; |
173 | |
174 | u64 enable_streams; |
175 | |
176 | u64 tcr; |
177 | u64 tcr_enabled; |
178 | u64 tcr_disabled; |
179 | u64 tcr_bypass; |
180 | |
181 | u64 ttbr; |
182 | u64 ttbr_valid; |
183 | u64 ttbr_addr_field_shift; |
184 | u64 ttbr_shift; |
185 | int ttbr_count; |
186 | }; |
187 | |
188 | /* |
189 | * Private structure associated with each DART device. |
190 | * |
191 | * @dev: device struct |
192 | * @hw: SoC-specific hardware data |
193 | * @regs: mapped MMIO region |
194 | * @irq: interrupt number, can be shared with other DARTs |
195 | * @clks: clocks associated with this DART |
196 | * @num_clks: number of @clks |
197 | * @lock: lock for hardware operations involving this dart |
198 | * @pgsize: pagesize supported by this DART |
199 | * @supports_bypass: indicates if this DART supports bypass mode |
200 | * @sid2group: maps stream ids to iommu_groups |
201 | * @iommu: iommu core device |
202 | */ |
203 | struct apple_dart { |
204 | struct device *dev; |
205 | const struct apple_dart_hw *hw; |
206 | |
207 | void __iomem *regs; |
208 | |
209 | int irq; |
210 | struct clk_bulk_data *clks; |
211 | int num_clks; |
212 | |
213 | spinlock_t lock; |
214 | |
215 | u32 ias; |
216 | u32 oas; |
217 | u32 pgsize; |
218 | u32 num_streams; |
219 | u32 supports_bypass : 1; |
220 | |
221 | struct iommu_group *sid2group[DART_MAX_STREAMS]; |
222 | struct iommu_device iommu; |
223 | |
224 | u32 save_tcr[DART_MAX_STREAMS]; |
225 | u32 save_ttbr[DART_MAX_STREAMS][DART_MAX_TTBR]; |
226 | }; |
227 | |
228 | /* |
229 | * Convenience struct to identify streams. |
230 | * |
231 | * The normal variant is used inside apple_dart_master_cfg which isn't written |
232 | * to concurrently. |
233 | * The atomic variant is used inside apple_dart_domain where we have to guard |
234 | * against races from potential parallel calls to attach/detach_device. |
235 | * Note that even inside the atomic variant the apple_dart pointer is not |
236 | * protected: This pointer is initialized once under the domain init mutex |
237 | * and never changed again afterwards. Devices with different dart pointers |
238 | * cannot be attached to the same domain. |
239 | * |
240 | * @dart dart pointer |
241 | * @sid stream id bitmap |
242 | */ |
243 | struct apple_dart_stream_map { |
244 | struct apple_dart *dart; |
245 | DECLARE_BITMAP(sidmap, DART_MAX_STREAMS); |
246 | }; |
247 | struct apple_dart_atomic_stream_map { |
248 | struct apple_dart *dart; |
249 | atomic_long_t sidmap[BITS_TO_LONGS(DART_MAX_STREAMS)]; |
250 | }; |
251 | |
252 | /* |
253 | * This structure is attached to each iommu domain handled by a DART. |
254 | * |
255 | * @pgtbl_ops: pagetable ops allocated by io-pgtable |
256 | * @finalized: true if the domain has been completely initialized |
257 | * @init_lock: protects domain initialization |
258 | * @stream_maps: streams attached to this domain (valid for DMA/UNMANAGED only) |
259 | * @domain: core iommu domain pointer |
260 | */ |
261 | struct apple_dart_domain { |
262 | struct io_pgtable_ops *pgtbl_ops; |
263 | |
264 | bool finalized; |
265 | struct mutex init_lock; |
266 | struct apple_dart_atomic_stream_map stream_maps[MAX_DARTS_PER_DEVICE]; |
267 | |
268 | struct iommu_domain domain; |
269 | }; |
270 | |
271 | /* |
272 | * This structure is attached to devices with dev_iommu_priv_set() on of_xlate |
273 | * and contains a list of streams bound to this device. |
274 | * So far the worst case seen is a single device with two streams |
275 | * from different darts, such that this simple static array is enough. |
276 | * |
277 | * @streams: streams for this device |
278 | */ |
279 | struct apple_dart_master_cfg { |
280 | struct apple_dart_stream_map stream_maps[MAX_DARTS_PER_DEVICE]; |
281 | }; |
282 | |
283 | /* |
284 | * Helper macro to iterate over apple_dart_master_cfg.stream_maps and |
285 | * apple_dart_domain.stream_maps |
286 | * |
287 | * @i int used as loop variable |
288 | * @base pointer to base struct (apple_dart_master_cfg or apple_dart_domain) |
289 | * @stream pointer to the apple_dart_streams struct for each loop iteration |
290 | */ |
291 | #define for_each_stream_map(i, base, stream_map) \ |
292 | for (i = 0, stream_map = &(base)->stream_maps[0]; \ |
293 | i < MAX_DARTS_PER_DEVICE && stream_map->dart; \ |
294 | stream_map = &(base)->stream_maps[++i]) |
295 | |
296 | static struct platform_driver apple_dart_driver; |
297 | static const struct iommu_ops apple_dart_iommu_ops; |
298 | |
299 | static struct apple_dart_domain *to_dart_domain(struct iommu_domain *dom) |
300 | { |
301 | return container_of(dom, struct apple_dart_domain, domain); |
302 | } |
303 | |
304 | static void |
305 | apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map) |
306 | { |
307 | struct apple_dart *dart = stream_map->dart; |
308 | int sid; |
309 | |
310 | for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) |
311 | writel(val: dart->hw->tcr_enabled, addr: dart->regs + DART_TCR(dart, sid)); |
312 | } |
313 | |
314 | static void apple_dart_hw_disable_dma(struct apple_dart_stream_map *stream_map) |
315 | { |
316 | struct apple_dart *dart = stream_map->dart; |
317 | int sid; |
318 | |
319 | for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) |
320 | writel(val: dart->hw->tcr_disabled, addr: dart->regs + DART_TCR(dart, sid)); |
321 | } |
322 | |
323 | static void |
324 | apple_dart_hw_enable_bypass(struct apple_dart_stream_map *stream_map) |
325 | { |
326 | struct apple_dart *dart = stream_map->dart; |
327 | int sid; |
328 | |
329 | WARN_ON(!stream_map->dart->supports_bypass); |
330 | for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) |
331 | writel(val: dart->hw->tcr_bypass, |
332 | addr: dart->regs + DART_TCR(dart, sid)); |
333 | } |
334 | |
335 | static void apple_dart_hw_set_ttbr(struct apple_dart_stream_map *stream_map, |
336 | u8 idx, phys_addr_t paddr) |
337 | { |
338 | struct apple_dart *dart = stream_map->dart; |
339 | int sid; |
340 | |
341 | WARN_ON(paddr & ((1 << dart->hw->ttbr_shift) - 1)); |
342 | for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) |
343 | writel(val: dart->hw->ttbr_valid | |
344 | (paddr >> dart->hw->ttbr_shift) << dart->hw->ttbr_addr_field_shift, |
345 | addr: dart->regs + DART_TTBR(dart, sid, idx)); |
346 | } |
347 | |
348 | static void apple_dart_hw_clear_ttbr(struct apple_dart_stream_map *stream_map, |
349 | u8 idx) |
350 | { |
351 | struct apple_dart *dart = stream_map->dart; |
352 | int sid; |
353 | |
354 | for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) |
355 | writel(val: 0, addr: dart->regs + DART_TTBR(dart, sid, idx)); |
356 | } |
357 | |
358 | static void |
359 | apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map *stream_map) |
360 | { |
361 | int i; |
362 | |
363 | for (i = 0; i < stream_map->dart->hw->ttbr_count; ++i) |
364 | apple_dart_hw_clear_ttbr(stream_map, idx: i); |
365 | } |
366 | |
367 | static int |
368 | apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map, |
369 | u32 command) |
370 | { |
371 | unsigned long flags; |
372 | int ret, i; |
373 | u32 command_reg; |
374 | |
375 | spin_lock_irqsave(&stream_map->dart->lock, flags); |
376 | |
377 | for (i = 0; i < BITS_TO_U32(stream_map->dart->num_streams); i++) |
378 | writel(val: stream_map->sidmap[i], |
379 | addr: stream_map->dart->regs + DART_T8020_STREAM_SELECT + 4 * i); |
380 | writel(val: command, addr: stream_map->dart->regs + DART_T8020_STREAM_COMMAND); |
381 | |
382 | ret = readl_poll_timeout_atomic( |
383 | stream_map->dart->regs + DART_T8020_STREAM_COMMAND, command_reg, |
384 | !(command_reg & DART_T8020_STREAM_COMMAND_BUSY), 1, |
385 | DART_STREAM_COMMAND_BUSY_TIMEOUT); |
386 | |
387 | spin_unlock_irqrestore(lock: &stream_map->dart->lock, flags); |
388 | |
389 | if (ret) { |
390 | dev_err(stream_map->dart->dev, |
391 | "busy bit did not clear after command %x for streams %lx\n" , |
392 | command, stream_map->sidmap[0]); |
393 | return ret; |
394 | } |
395 | |
396 | return 0; |
397 | } |
398 | |
399 | static int |
400 | apple_dart_t8110_hw_tlb_command(struct apple_dart_stream_map *stream_map, |
401 | u32 command) |
402 | { |
403 | struct apple_dart *dart = stream_map->dart; |
404 | unsigned long flags; |
405 | int ret = 0; |
406 | int sid; |
407 | |
408 | spin_lock_irqsave(&dart->lock, flags); |
409 | |
410 | for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) { |
411 | u32 val = FIELD_PREP(DART_T8110_TLB_CMD_OP, command) | |
412 | FIELD_PREP(DART_T8110_TLB_CMD_STREAM, sid); |
413 | writel(val, addr: dart->regs + DART_T8110_TLB_CMD); |
414 | |
415 | ret = readl_poll_timeout_atomic( |
416 | dart->regs + DART_T8110_TLB_CMD, val, |
417 | !(val & DART_T8110_TLB_CMD_BUSY), 1, |
418 | DART_STREAM_COMMAND_BUSY_TIMEOUT); |
419 | |
420 | if (ret) |
421 | break; |
422 | |
423 | } |
424 | |
425 | spin_unlock_irqrestore(lock: &dart->lock, flags); |
426 | |
427 | if (ret) { |
428 | dev_err(stream_map->dart->dev, |
429 | "busy bit did not clear after command %x for stream %d\n" , |
430 | command, sid); |
431 | return ret; |
432 | } |
433 | |
434 | return 0; |
435 | } |
436 | |
437 | static int |
438 | apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) |
439 | { |
440 | return apple_dart_t8020_hw_stream_command( |
441 | stream_map, DART_T8020_STREAM_COMMAND_INVALIDATE); |
442 | } |
443 | |
444 | static int |
445 | apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map) |
446 | { |
447 | return apple_dart_t8110_hw_tlb_command( |
448 | stream_map, DART_T8110_TLB_CMD_OP_FLUSH_SID); |
449 | } |
450 | |
451 | static int apple_dart_hw_reset(struct apple_dart *dart) |
452 | { |
453 | u32 config; |
454 | struct apple_dart_stream_map stream_map; |
455 | int i; |
456 | |
457 | config = readl(addr: dart->regs + dart->hw->lock); |
458 | if (config & dart->hw->lock_bit) { |
459 | dev_err(dart->dev, "DART is locked down until reboot: %08x\n" , |
460 | config); |
461 | return -EINVAL; |
462 | } |
463 | |
464 | stream_map.dart = dart; |
465 | bitmap_zero(dst: stream_map.sidmap, DART_MAX_STREAMS); |
466 | bitmap_set(map: stream_map.sidmap, start: 0, nbits: dart->num_streams); |
467 | apple_dart_hw_disable_dma(stream_map: &stream_map); |
468 | apple_dart_hw_clear_all_ttbrs(stream_map: &stream_map); |
469 | |
470 | /* enable all streams globally since TCR is used to control isolation */ |
471 | for (i = 0; i < BITS_TO_U32(dart->num_streams); i++) |
472 | writel(U32_MAX, addr: dart->regs + dart->hw->enable_streams + 4 * i); |
473 | |
474 | /* clear any pending errors before the interrupt is unmasked */ |
475 | writel(readl(addr: dart->regs + dart->hw->error), addr: dart->regs + dart->hw->error); |
476 | |
477 | if (dart->hw->type == DART_T8110) |
478 | writel(val: 0, addr: dart->regs + DART_T8110_ERROR_MASK); |
479 | |
480 | return dart->hw->invalidate_tlb(&stream_map); |
481 | } |
482 | |
483 | static void apple_dart_domain_flush_tlb(struct apple_dart_domain *domain) |
484 | { |
485 | int i, j; |
486 | struct apple_dart_atomic_stream_map *domain_stream_map; |
487 | struct apple_dart_stream_map stream_map; |
488 | |
489 | for_each_stream_map(i, domain, domain_stream_map) { |
490 | stream_map.dart = domain_stream_map->dart; |
491 | |
492 | for (j = 0; j < BITS_TO_LONGS(stream_map.dart->num_streams); j++) |
493 | stream_map.sidmap[j] = atomic_long_read(v: &domain_stream_map->sidmap[j]); |
494 | |
495 | stream_map.dart->hw->invalidate_tlb(&stream_map); |
496 | } |
497 | } |
498 | |
499 | static void apple_dart_flush_iotlb_all(struct iommu_domain *domain) |
500 | { |
501 | apple_dart_domain_flush_tlb(domain: to_dart_domain(dom: domain)); |
502 | } |
503 | |
504 | static void apple_dart_iotlb_sync(struct iommu_domain *domain, |
505 | struct iommu_iotlb_gather *gather) |
506 | { |
507 | apple_dart_domain_flush_tlb(domain: to_dart_domain(dom: domain)); |
508 | } |
509 | |
510 | static int apple_dart_iotlb_sync_map(struct iommu_domain *domain, |
511 | unsigned long iova, size_t size) |
512 | { |
513 | apple_dart_domain_flush_tlb(domain: to_dart_domain(dom: domain)); |
514 | return 0; |
515 | } |
516 | |
517 | static phys_addr_t apple_dart_iova_to_phys(struct iommu_domain *domain, |
518 | dma_addr_t iova) |
519 | { |
520 | struct apple_dart_domain *dart_domain = to_dart_domain(dom: domain); |
521 | struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; |
522 | |
523 | if (!ops) |
524 | return 0; |
525 | |
526 | return ops->iova_to_phys(ops, iova); |
527 | } |
528 | |
529 | static int apple_dart_map_pages(struct iommu_domain *domain, unsigned long iova, |
530 | phys_addr_t paddr, size_t pgsize, |
531 | size_t pgcount, int prot, gfp_t gfp, |
532 | size_t *mapped) |
533 | { |
534 | struct apple_dart_domain *dart_domain = to_dart_domain(dom: domain); |
535 | struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; |
536 | |
537 | if (!ops) |
538 | return -ENODEV; |
539 | |
540 | return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, |
541 | mapped); |
542 | } |
543 | |
544 | static size_t apple_dart_unmap_pages(struct iommu_domain *domain, |
545 | unsigned long iova, size_t pgsize, |
546 | size_t pgcount, |
547 | struct iommu_iotlb_gather *gather) |
548 | { |
549 | struct apple_dart_domain *dart_domain = to_dart_domain(dom: domain); |
550 | struct io_pgtable_ops *ops = dart_domain->pgtbl_ops; |
551 | |
552 | return ops->unmap_pages(ops, iova, pgsize, pgcount, gather); |
553 | } |
554 | |
555 | static void |
556 | apple_dart_setup_translation(struct apple_dart_domain *domain, |
557 | struct apple_dart_stream_map *stream_map) |
558 | { |
559 | int i; |
560 | struct io_pgtable_cfg *pgtbl_cfg = |
561 | &io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg; |
562 | |
563 | for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i) |
564 | apple_dart_hw_set_ttbr(stream_map, idx: i, |
565 | paddr: pgtbl_cfg->apple_dart_cfg.ttbr[i]); |
566 | for (; i < stream_map->dart->hw->ttbr_count; ++i) |
567 | apple_dart_hw_clear_ttbr(stream_map, idx: i); |
568 | |
569 | apple_dart_hw_enable_translation(stream_map); |
570 | stream_map->dart->hw->invalidate_tlb(stream_map); |
571 | } |
572 | |
573 | static int apple_dart_finalize_domain(struct apple_dart_domain *dart_domain, |
574 | struct apple_dart_master_cfg *cfg) |
575 | { |
576 | struct apple_dart *dart = cfg->stream_maps[0].dart; |
577 | struct io_pgtable_cfg pgtbl_cfg; |
578 | int ret = 0; |
579 | int i, j; |
580 | |
581 | if (dart->pgsize > PAGE_SIZE) |
582 | return -EINVAL; |
583 | |
584 | mutex_lock(&dart_domain->init_lock); |
585 | |
586 | if (dart_domain->finalized) |
587 | goto done; |
588 | |
589 | for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { |
590 | dart_domain->stream_maps[i].dart = cfg->stream_maps[i].dart; |
591 | for (j = 0; j < BITS_TO_LONGS(dart->num_streams); j++) |
592 | atomic_long_set(v: &dart_domain->stream_maps[i].sidmap[j], |
593 | i: cfg->stream_maps[i].sidmap[j]); |
594 | } |
595 | |
596 | pgtbl_cfg = (struct io_pgtable_cfg){ |
597 | .pgsize_bitmap = dart->pgsize, |
598 | .ias = dart->ias, |
599 | .oas = dart->oas, |
600 | .coherent_walk = 1, |
601 | .iommu_dev = dart->dev, |
602 | }; |
603 | |
604 | dart_domain->pgtbl_ops = alloc_io_pgtable_ops(fmt: dart->hw->fmt, cfg: &pgtbl_cfg, |
605 | cookie: &dart_domain->domain); |
606 | if (!dart_domain->pgtbl_ops) { |
607 | ret = -ENOMEM; |
608 | goto done; |
609 | } |
610 | |
611 | dart_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; |
612 | dart_domain->domain.geometry.aperture_start = 0; |
613 | dart_domain->domain.geometry.aperture_end = |
614 | (dma_addr_t)DMA_BIT_MASK(dart->ias); |
615 | dart_domain->domain.geometry.force_aperture = true; |
616 | |
617 | dart_domain->finalized = true; |
618 | |
619 | done: |
620 | mutex_unlock(lock: &dart_domain->init_lock); |
621 | return ret; |
622 | } |
623 | |
624 | static int |
625 | apple_dart_mod_streams(struct apple_dart_atomic_stream_map *domain_maps, |
626 | struct apple_dart_stream_map *master_maps, |
627 | bool add_streams) |
628 | { |
629 | int i, j; |
630 | |
631 | for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { |
632 | if (domain_maps[i].dart != master_maps[i].dart) |
633 | return -EINVAL; |
634 | } |
635 | |
636 | for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { |
637 | if (!domain_maps[i].dart) |
638 | break; |
639 | for (j = 0; j < BITS_TO_LONGS(domain_maps[i].dart->num_streams); j++) { |
640 | if (add_streams) |
641 | atomic_long_or(i: master_maps[i].sidmap[j], |
642 | v: &domain_maps[i].sidmap[j]); |
643 | else |
644 | atomic_long_and(i: ~master_maps[i].sidmap[j], |
645 | v: &domain_maps[i].sidmap[j]); |
646 | } |
647 | } |
648 | |
649 | return 0; |
650 | } |
651 | |
652 | static int apple_dart_domain_add_streams(struct apple_dart_domain *domain, |
653 | struct apple_dart_master_cfg *cfg) |
654 | { |
655 | return apple_dart_mod_streams(domain_maps: domain->stream_maps, master_maps: cfg->stream_maps, |
656 | add_streams: true); |
657 | } |
658 | |
659 | static int apple_dart_attach_dev_paging(struct iommu_domain *domain, |
660 | struct device *dev) |
661 | { |
662 | int ret, i; |
663 | struct apple_dart_stream_map *stream_map; |
664 | struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); |
665 | struct apple_dart_domain *dart_domain = to_dart_domain(dom: domain); |
666 | |
667 | ret = apple_dart_finalize_domain(dart_domain, cfg); |
668 | if (ret) |
669 | return ret; |
670 | |
671 | ret = apple_dart_domain_add_streams(domain: dart_domain, cfg); |
672 | if (ret) |
673 | return ret; |
674 | |
675 | for_each_stream_map(i, cfg, stream_map) |
676 | apple_dart_setup_translation(domain: dart_domain, stream_map); |
677 | return 0; |
678 | } |
679 | |
680 | static int apple_dart_attach_dev_identity(struct iommu_domain *domain, |
681 | struct device *dev) |
682 | { |
683 | struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); |
684 | struct apple_dart_stream_map *stream_map; |
685 | int i; |
686 | |
687 | if (!cfg->stream_maps[0].dart->supports_bypass) |
688 | return -EINVAL; |
689 | |
690 | for_each_stream_map(i, cfg, stream_map) |
691 | apple_dart_hw_enable_bypass(stream_map); |
692 | return 0; |
693 | } |
694 | |
695 | static const struct iommu_domain_ops apple_dart_identity_ops = { |
696 | .attach_dev = apple_dart_attach_dev_identity, |
697 | }; |
698 | |
699 | static struct iommu_domain apple_dart_identity_domain = { |
700 | .type = IOMMU_DOMAIN_IDENTITY, |
701 | .ops = &apple_dart_identity_ops, |
702 | }; |
703 | |
704 | static int apple_dart_attach_dev_blocked(struct iommu_domain *domain, |
705 | struct device *dev) |
706 | { |
707 | struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); |
708 | struct apple_dart_stream_map *stream_map; |
709 | int i; |
710 | |
711 | for_each_stream_map(i, cfg, stream_map) |
712 | apple_dart_hw_disable_dma(stream_map); |
713 | return 0; |
714 | } |
715 | |
716 | static const struct iommu_domain_ops apple_dart_blocked_ops = { |
717 | .attach_dev = apple_dart_attach_dev_blocked, |
718 | }; |
719 | |
720 | static struct iommu_domain apple_dart_blocked_domain = { |
721 | .type = IOMMU_DOMAIN_BLOCKED, |
722 | .ops = &apple_dart_blocked_ops, |
723 | }; |
724 | |
725 | static struct iommu_device *apple_dart_probe_device(struct device *dev) |
726 | { |
727 | struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); |
728 | struct apple_dart_stream_map *stream_map; |
729 | int i; |
730 | |
731 | if (!cfg) |
732 | return ERR_PTR(error: -ENODEV); |
733 | |
734 | for_each_stream_map(i, cfg, stream_map) |
735 | device_link_add( |
736 | consumer: dev, supplier: stream_map->dart->dev, |
737 | DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER); |
738 | |
739 | return &cfg->stream_maps[0].dart->iommu; |
740 | } |
741 | |
742 | static void apple_dart_release_device(struct device *dev) |
743 | { |
744 | struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); |
745 | |
746 | kfree(objp: cfg); |
747 | } |
748 | |
749 | static struct iommu_domain *apple_dart_domain_alloc_paging(struct device *dev) |
750 | { |
751 | struct apple_dart_domain *dart_domain; |
752 | |
753 | dart_domain = kzalloc(size: sizeof(*dart_domain), GFP_KERNEL); |
754 | if (!dart_domain) |
755 | return NULL; |
756 | |
757 | mutex_init(&dart_domain->init_lock); |
758 | |
759 | if (dev) { |
760 | struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); |
761 | int ret; |
762 | |
763 | ret = apple_dart_finalize_domain(dart_domain, cfg); |
764 | if (ret) { |
765 | kfree(objp: dart_domain); |
766 | return ERR_PTR(error: ret); |
767 | } |
768 | } |
769 | return &dart_domain->domain; |
770 | } |
771 | |
772 | static void apple_dart_domain_free(struct iommu_domain *domain) |
773 | { |
774 | struct apple_dart_domain *dart_domain = to_dart_domain(dom: domain); |
775 | |
776 | if (dart_domain->pgtbl_ops) |
777 | free_io_pgtable_ops(ops: dart_domain->pgtbl_ops); |
778 | |
779 | kfree(objp: dart_domain); |
780 | } |
781 | |
782 | static int apple_dart_of_xlate(struct device *dev, |
783 | const struct of_phandle_args *args) |
784 | { |
785 | struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); |
786 | struct platform_device *iommu_pdev = of_find_device_by_node(np: args->np); |
787 | struct apple_dart *dart = platform_get_drvdata(pdev: iommu_pdev); |
788 | struct apple_dart *cfg_dart; |
789 | int i, sid; |
790 | |
791 | if (args->args_count != 1) |
792 | return -EINVAL; |
793 | sid = args->args[0]; |
794 | |
795 | if (!cfg) |
796 | cfg = kzalloc(size: sizeof(*cfg), GFP_KERNEL); |
797 | if (!cfg) |
798 | return -ENOMEM; |
799 | dev_iommu_priv_set(dev, priv: cfg); |
800 | |
801 | cfg_dart = cfg->stream_maps[0].dart; |
802 | if (cfg_dart) { |
803 | if (cfg_dart->supports_bypass != dart->supports_bypass) |
804 | return -EINVAL; |
805 | if (cfg_dart->pgsize != dart->pgsize) |
806 | return -EINVAL; |
807 | } |
808 | |
809 | for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { |
810 | if (cfg->stream_maps[i].dart == dart) { |
811 | set_bit(nr: sid, addr: cfg->stream_maps[i].sidmap); |
812 | return 0; |
813 | } |
814 | } |
815 | for (i = 0; i < MAX_DARTS_PER_DEVICE; ++i) { |
816 | if (!cfg->stream_maps[i].dart) { |
817 | cfg->stream_maps[i].dart = dart; |
818 | set_bit(nr: sid, addr: cfg->stream_maps[i].sidmap); |
819 | return 0; |
820 | } |
821 | } |
822 | |
823 | return -EINVAL; |
824 | } |
825 | |
826 | static DEFINE_MUTEX(apple_dart_groups_lock); |
827 | |
828 | static void apple_dart_release_group(void *iommu_data) |
829 | { |
830 | int i, sid; |
831 | struct apple_dart_stream_map *stream_map; |
832 | struct apple_dart_master_cfg *group_master_cfg = iommu_data; |
833 | |
834 | mutex_lock(&apple_dart_groups_lock); |
835 | |
836 | for_each_stream_map(i, group_master_cfg, stream_map) |
837 | for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) |
838 | stream_map->dart->sid2group[sid] = NULL; |
839 | |
840 | kfree(objp: iommu_data); |
841 | mutex_unlock(lock: &apple_dart_groups_lock); |
842 | } |
843 | |
844 | static int apple_dart_merge_master_cfg(struct apple_dart_master_cfg *dst, |
845 | struct apple_dart_master_cfg *src) |
846 | { |
847 | /* |
848 | * We know that this function is only called for groups returned from |
849 | * pci_device_group and that all Apple Silicon platforms never spread |
850 | * PCIe devices from the same bus across multiple DARTs such that we can |
851 | * just assume that both src and dst only have the same single DART. |
852 | */ |
853 | if (src->stream_maps[1].dart) |
854 | return -EINVAL; |
855 | if (dst->stream_maps[1].dart) |
856 | return -EINVAL; |
857 | if (src->stream_maps[0].dart != dst->stream_maps[0].dart) |
858 | return -EINVAL; |
859 | |
860 | bitmap_or(dst: dst->stream_maps[0].sidmap, |
861 | src1: dst->stream_maps[0].sidmap, |
862 | src2: src->stream_maps[0].sidmap, |
863 | nbits: dst->stream_maps[0].dart->num_streams); |
864 | return 0; |
865 | } |
866 | |
867 | static struct iommu_group *apple_dart_device_group(struct device *dev) |
868 | { |
869 | int i, sid; |
870 | struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); |
871 | struct apple_dart_stream_map *stream_map; |
872 | struct apple_dart_master_cfg *group_master_cfg; |
873 | struct iommu_group *group = NULL; |
874 | struct iommu_group *res = ERR_PTR(error: -EINVAL); |
875 | |
876 | mutex_lock(&apple_dart_groups_lock); |
877 | |
878 | for_each_stream_map(i, cfg, stream_map) { |
879 | for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) { |
880 | struct iommu_group *stream_group = |
881 | stream_map->dart->sid2group[sid]; |
882 | |
883 | if (group && group != stream_group) { |
884 | res = ERR_PTR(error: -EINVAL); |
885 | goto out; |
886 | } |
887 | |
888 | group = stream_group; |
889 | } |
890 | } |
891 | |
892 | if (group) { |
893 | res = iommu_group_ref_get(group); |
894 | goto out; |
895 | } |
896 | |
897 | #ifdef CONFIG_PCI |
898 | if (dev_is_pci(dev)) |
899 | group = pci_device_group(dev); |
900 | else |
901 | #endif |
902 | group = generic_device_group(dev); |
903 | |
904 | res = ERR_PTR(error: -ENOMEM); |
905 | if (!group) |
906 | goto out; |
907 | |
908 | group_master_cfg = iommu_group_get_iommudata(group); |
909 | if (group_master_cfg) { |
910 | int ret; |
911 | |
912 | ret = apple_dart_merge_master_cfg(dst: group_master_cfg, src: cfg); |
913 | if (ret) { |
914 | dev_err(dev, "Failed to merge DART IOMMU groups.\n" ); |
915 | iommu_group_put(group); |
916 | res = ERR_PTR(error: ret); |
917 | goto out; |
918 | } |
919 | } else { |
920 | group_master_cfg = kmemdup(p: cfg, size: sizeof(*group_master_cfg), |
921 | GFP_KERNEL); |
922 | if (!group_master_cfg) { |
923 | iommu_group_put(group); |
924 | goto out; |
925 | } |
926 | |
927 | iommu_group_set_iommudata(group, iommu_data: group_master_cfg, |
928 | release: apple_dart_release_group); |
929 | } |
930 | |
931 | for_each_stream_map(i, cfg, stream_map) |
932 | for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) |
933 | stream_map->dart->sid2group[sid] = group; |
934 | |
935 | res = group; |
936 | |
937 | out: |
938 | mutex_unlock(lock: &apple_dart_groups_lock); |
939 | return res; |
940 | } |
941 | |
942 | static int apple_dart_def_domain_type(struct device *dev) |
943 | { |
944 | struct apple_dart_master_cfg *cfg = dev_iommu_priv_get(dev); |
945 | |
946 | if (cfg->stream_maps[0].dart->pgsize > PAGE_SIZE) |
947 | return IOMMU_DOMAIN_IDENTITY; |
948 | if (!cfg->stream_maps[0].dart->supports_bypass) |
949 | return IOMMU_DOMAIN_DMA; |
950 | |
951 | return 0; |
952 | } |
953 | |
954 | #ifndef CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR |
955 | /* Keep things compiling when CONFIG_PCI_APPLE isn't selected */ |
956 | #define CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR 0 |
957 | #endif |
958 | #define DOORBELL_ADDR (CONFIG_PCIE_APPLE_MSI_DOORBELL_ADDR & PAGE_MASK) |
959 | |
960 | static void apple_dart_get_resv_regions(struct device *dev, |
961 | struct list_head *head) |
962 | { |
963 | if (IS_ENABLED(CONFIG_PCIE_APPLE) && dev_is_pci(dev)) { |
964 | struct iommu_resv_region *region; |
965 | int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; |
966 | |
967 | region = iommu_alloc_resv_region(DOORBELL_ADDR, |
968 | PAGE_SIZE, prot, |
969 | type: IOMMU_RESV_MSI, GFP_KERNEL); |
970 | if (!region) |
971 | return; |
972 | |
973 | list_add_tail(new: ®ion->list, head); |
974 | } |
975 | |
976 | iommu_dma_get_resv_regions(dev, list: head); |
977 | } |
978 | |
979 | static const struct iommu_ops apple_dart_iommu_ops = { |
980 | .identity_domain = &apple_dart_identity_domain, |
981 | .blocked_domain = &apple_dart_blocked_domain, |
982 | .domain_alloc_paging = apple_dart_domain_alloc_paging, |
983 | .probe_device = apple_dart_probe_device, |
984 | .release_device = apple_dart_release_device, |
985 | .device_group = apple_dart_device_group, |
986 | .of_xlate = apple_dart_of_xlate, |
987 | .def_domain_type = apple_dart_def_domain_type, |
988 | .get_resv_regions = apple_dart_get_resv_regions, |
989 | .pgsize_bitmap = -1UL, /* Restricted during dart probe */ |
990 | .owner = THIS_MODULE, |
991 | .default_domain_ops = &(const struct iommu_domain_ops) { |
992 | .attach_dev = apple_dart_attach_dev_paging, |
993 | .map_pages = apple_dart_map_pages, |
994 | .unmap_pages = apple_dart_unmap_pages, |
995 | .flush_iotlb_all = apple_dart_flush_iotlb_all, |
996 | .iotlb_sync = apple_dart_iotlb_sync, |
997 | .iotlb_sync_map = apple_dart_iotlb_sync_map, |
998 | .iova_to_phys = apple_dart_iova_to_phys, |
999 | .free = apple_dart_domain_free, |
1000 | } |
1001 | }; |
1002 | |
1003 | static irqreturn_t apple_dart_t8020_irq(int irq, void *dev) |
1004 | { |
1005 | struct apple_dart *dart = dev; |
1006 | const char *fault_name = NULL; |
1007 | u32 error = readl(addr: dart->regs + DART_T8020_ERROR); |
1008 | u32 error_code = FIELD_GET(DART_T8020_ERROR_CODE, error); |
1009 | u32 addr_lo = readl(addr: dart->regs + DART_T8020_ERROR_ADDR_LO); |
1010 | u32 addr_hi = readl(addr: dart->regs + DART_T8020_ERROR_ADDR_HI); |
1011 | u64 addr = addr_lo | (((u64)addr_hi) << 32); |
1012 | u8 stream_idx = FIELD_GET(DART_T8020_ERROR_STREAM, error); |
1013 | |
1014 | if (!(error & DART_T8020_ERROR_FLAG)) |
1015 | return IRQ_NONE; |
1016 | |
1017 | /* there should only be a single bit set but let's use == to be sure */ |
1018 | if (error_code == DART_T8020_ERROR_READ_FAULT) |
1019 | fault_name = "READ FAULT" ; |
1020 | else if (error_code == DART_T8020_ERROR_WRITE_FAULT) |
1021 | fault_name = "WRITE FAULT" ; |
1022 | else if (error_code == DART_T8020_ERROR_NO_PTE) |
1023 | fault_name = "NO PTE FOR IOVA" ; |
1024 | else if (error_code == DART_T8020_ERROR_NO_PMD) |
1025 | fault_name = "NO PMD FOR IOVA" ; |
1026 | else if (error_code == DART_T8020_ERROR_NO_TTBR) |
1027 | fault_name = "NO TTBR FOR IOVA" ; |
1028 | else |
1029 | fault_name = "unknown" ; |
1030 | |
1031 | dev_err_ratelimited( |
1032 | dart->dev, |
1033 | "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx" , |
1034 | error, stream_idx, error_code, fault_name, addr); |
1035 | |
1036 | writel(val: error, addr: dart->regs + DART_T8020_ERROR); |
1037 | return IRQ_HANDLED; |
1038 | } |
1039 | |
1040 | static irqreturn_t apple_dart_t8110_irq(int irq, void *dev) |
1041 | { |
1042 | struct apple_dart *dart = dev; |
1043 | const char *fault_name = NULL; |
1044 | u32 error = readl(addr: dart->regs + DART_T8110_ERROR); |
1045 | u32 error_code = FIELD_GET(DART_T8110_ERROR_CODE, error); |
1046 | u32 addr_lo = readl(addr: dart->regs + DART_T8110_ERROR_ADDR_LO); |
1047 | u32 addr_hi = readl(addr: dart->regs + DART_T8110_ERROR_ADDR_HI); |
1048 | u64 addr = addr_lo | (((u64)addr_hi) << 32); |
1049 | u8 stream_idx = FIELD_GET(DART_T8110_ERROR_STREAM, error); |
1050 | |
1051 | if (!(error & DART_T8110_ERROR_FLAG)) |
1052 | return IRQ_NONE; |
1053 | |
1054 | /* there should only be a single bit set but let's use == to be sure */ |
1055 | if (error_code == DART_T8110_ERROR_READ_FAULT) |
1056 | fault_name = "READ FAULT" ; |
1057 | else if (error_code == DART_T8110_ERROR_WRITE_FAULT) |
1058 | fault_name = "WRITE FAULT" ; |
1059 | else if (error_code == DART_T8110_ERROR_NO_PTE) |
1060 | fault_name = "NO PTE FOR IOVA" ; |
1061 | else if (error_code == DART_T8110_ERROR_NO_PMD) |
1062 | fault_name = "NO PMD FOR IOVA" ; |
1063 | else if (error_code == DART_T8110_ERROR_NO_PGD) |
1064 | fault_name = "NO PGD FOR IOVA" ; |
1065 | else if (error_code == DART_T8110_ERROR_NO_TTBR) |
1066 | fault_name = "NO TTBR FOR IOVA" ; |
1067 | else |
1068 | fault_name = "unknown" ; |
1069 | |
1070 | dev_err_ratelimited( |
1071 | dart->dev, |
1072 | "translation fault: status:0x%x stream:%d code:0x%x (%s) at 0x%llx" , |
1073 | error, stream_idx, error_code, fault_name, addr); |
1074 | |
1075 | writel(val: error, addr: dart->regs + DART_T8110_ERROR); |
1076 | return IRQ_HANDLED; |
1077 | } |
1078 | |
1079 | static int apple_dart_probe(struct platform_device *pdev) |
1080 | { |
1081 | int ret; |
1082 | u32 dart_params[4]; |
1083 | struct resource *res; |
1084 | struct apple_dart *dart; |
1085 | struct device *dev = &pdev->dev; |
1086 | |
1087 | dart = devm_kzalloc(dev, size: sizeof(*dart), GFP_KERNEL); |
1088 | if (!dart) |
1089 | return -ENOMEM; |
1090 | |
1091 | dart->dev = dev; |
1092 | dart->hw = of_device_get_match_data(dev); |
1093 | spin_lock_init(&dart->lock); |
1094 | |
1095 | dart->regs = devm_platform_get_and_ioremap_resource(pdev, index: 0, res: &res); |
1096 | if (IS_ERR(ptr: dart->regs)) |
1097 | return PTR_ERR(ptr: dart->regs); |
1098 | |
1099 | if (resource_size(res) < 0x4000) { |
1100 | dev_err(dev, "MMIO region too small (%pr)\n" , res); |
1101 | return -EINVAL; |
1102 | } |
1103 | |
1104 | dart->irq = platform_get_irq(pdev, 0); |
1105 | if (dart->irq < 0) |
1106 | return -ENODEV; |
1107 | |
1108 | ret = devm_clk_bulk_get_all(dev, clks: &dart->clks); |
1109 | if (ret < 0) |
1110 | return ret; |
1111 | dart->num_clks = ret; |
1112 | |
1113 | ret = clk_bulk_prepare_enable(num_clks: dart->num_clks, clks: dart->clks); |
1114 | if (ret) |
1115 | return ret; |
1116 | |
1117 | dart_params[0] = readl(addr: dart->regs + DART_PARAMS1); |
1118 | dart_params[1] = readl(addr: dart->regs + DART_PARAMS2); |
1119 | dart->pgsize = 1 << FIELD_GET(DART_PARAMS1_PAGE_SHIFT, dart_params[0]); |
1120 | dart->supports_bypass = dart_params[1] & DART_PARAMS2_BYPASS_SUPPORT; |
1121 | |
1122 | switch (dart->hw->type) { |
1123 | case DART_T8020: |
1124 | case DART_T6000: |
1125 | dart->ias = 32; |
1126 | dart->oas = dart->hw->oas; |
1127 | dart->num_streams = dart->hw->max_sid_count; |
1128 | break; |
1129 | |
1130 | case DART_T8110: |
1131 | dart_params[2] = readl(addr: dart->regs + DART_T8110_PARAMS3); |
1132 | dart_params[3] = readl(addr: dart->regs + DART_T8110_PARAMS4); |
1133 | dart->ias = FIELD_GET(DART_T8110_PARAMS3_VA_WIDTH, dart_params[2]); |
1134 | dart->oas = FIELD_GET(DART_T8110_PARAMS3_PA_WIDTH, dart_params[2]); |
1135 | dart->num_streams = FIELD_GET(DART_T8110_PARAMS4_NUM_SIDS, dart_params[3]); |
1136 | break; |
1137 | } |
1138 | |
1139 | if (dart->num_streams > DART_MAX_STREAMS) { |
1140 | dev_err(&pdev->dev, "Too many streams (%d > %d)\n" , |
1141 | dart->num_streams, DART_MAX_STREAMS); |
1142 | ret = -EINVAL; |
1143 | goto err_clk_disable; |
1144 | } |
1145 | |
1146 | ret = apple_dart_hw_reset(dart); |
1147 | if (ret) |
1148 | goto err_clk_disable; |
1149 | |
1150 | ret = request_irq(irq: dart->irq, handler: dart->hw->irq_handler, IRQF_SHARED, |
1151 | name: "apple-dart fault handler" , dev: dart); |
1152 | if (ret) |
1153 | goto err_clk_disable; |
1154 | |
1155 | platform_set_drvdata(pdev, data: dart); |
1156 | |
1157 | ret = iommu_device_sysfs_add(iommu: &dart->iommu, parent: dev, NULL, fmt: "apple-dart.%s" , |
1158 | dev_name(dev: &pdev->dev)); |
1159 | if (ret) |
1160 | goto err_free_irq; |
1161 | |
1162 | ret = iommu_device_register(iommu: &dart->iommu, ops: &apple_dart_iommu_ops, hwdev: dev); |
1163 | if (ret) |
1164 | goto err_sysfs_remove; |
1165 | |
1166 | dev_info( |
1167 | &pdev->dev, |
1168 | "DART [pagesize %x, %d streams, bypass support: %d, bypass forced: %d] initialized\n" , |
1169 | dart->pgsize, dart->num_streams, dart->supports_bypass, |
1170 | dart->pgsize > PAGE_SIZE); |
1171 | return 0; |
1172 | |
1173 | err_sysfs_remove: |
1174 | iommu_device_sysfs_remove(iommu: &dart->iommu); |
1175 | err_free_irq: |
1176 | free_irq(dart->irq, dart); |
1177 | err_clk_disable: |
1178 | clk_bulk_disable_unprepare(num_clks: dart->num_clks, clks: dart->clks); |
1179 | |
1180 | return ret; |
1181 | } |
1182 | |
1183 | static void apple_dart_remove(struct platform_device *pdev) |
1184 | { |
1185 | struct apple_dart *dart = platform_get_drvdata(pdev); |
1186 | |
1187 | apple_dart_hw_reset(dart); |
1188 | free_irq(dart->irq, dart); |
1189 | |
1190 | iommu_device_unregister(iommu: &dart->iommu); |
1191 | iommu_device_sysfs_remove(iommu: &dart->iommu); |
1192 | |
1193 | clk_bulk_disable_unprepare(num_clks: dart->num_clks, clks: dart->clks); |
1194 | } |
1195 | |
1196 | static const struct apple_dart_hw apple_dart_hw_t8103 = { |
1197 | .type = DART_T8020, |
1198 | .irq_handler = apple_dart_t8020_irq, |
1199 | .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb, |
1200 | .oas = 36, |
1201 | .fmt = APPLE_DART, |
1202 | .max_sid_count = 16, |
1203 | |
1204 | .enable_streams = DART_T8020_STREAMS_ENABLE, |
1205 | .lock = DART_T8020_CONFIG, |
1206 | .lock_bit = DART_T8020_CONFIG_LOCK, |
1207 | |
1208 | .error = DART_T8020_ERROR, |
1209 | |
1210 | .tcr = DART_T8020_TCR, |
1211 | .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE, |
1212 | .tcr_disabled = 0, |
1213 | .tcr_bypass = DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART, |
1214 | |
1215 | .ttbr = DART_T8020_TTBR, |
1216 | .ttbr_valid = DART_T8020_TTBR_VALID, |
1217 | .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT, |
1218 | .ttbr_shift = DART_T8020_TTBR_SHIFT, |
1219 | .ttbr_count = 4, |
1220 | }; |
1221 | |
1222 | static const struct apple_dart_hw apple_dart_hw_t8103_usb4 = { |
1223 | .type = DART_T8020, |
1224 | .irq_handler = apple_dart_t8020_irq, |
1225 | .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb, |
1226 | .oas = 36, |
1227 | .fmt = APPLE_DART, |
1228 | .max_sid_count = 64, |
1229 | |
1230 | .enable_streams = DART_T8020_STREAMS_ENABLE, |
1231 | .lock = DART_T8020_CONFIG, |
1232 | .lock_bit = DART_T8020_CONFIG_LOCK, |
1233 | |
1234 | .error = DART_T8020_ERROR, |
1235 | |
1236 | .tcr = DART_T8020_TCR, |
1237 | .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE, |
1238 | .tcr_disabled = 0, |
1239 | .tcr_bypass = 0, |
1240 | |
1241 | .ttbr = DART_T8020_USB4_TTBR, |
1242 | .ttbr_valid = DART_T8020_TTBR_VALID, |
1243 | .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT, |
1244 | .ttbr_shift = DART_T8020_TTBR_SHIFT, |
1245 | .ttbr_count = 4, |
1246 | }; |
1247 | |
1248 | static const struct apple_dart_hw apple_dart_hw_t6000 = { |
1249 | .type = DART_T6000, |
1250 | .irq_handler = apple_dart_t8020_irq, |
1251 | .invalidate_tlb = apple_dart_t8020_hw_invalidate_tlb, |
1252 | .oas = 42, |
1253 | .fmt = APPLE_DART2, |
1254 | .max_sid_count = 16, |
1255 | |
1256 | .enable_streams = DART_T8020_STREAMS_ENABLE, |
1257 | .lock = DART_T8020_CONFIG, |
1258 | .lock_bit = DART_T8020_CONFIG_LOCK, |
1259 | |
1260 | .error = DART_T8020_ERROR, |
1261 | |
1262 | .tcr = DART_T8020_TCR, |
1263 | .tcr_enabled = DART_T8020_TCR_TRANSLATE_ENABLE, |
1264 | .tcr_disabled = 0, |
1265 | .tcr_bypass = DART_T8020_TCR_BYPASS_DAPF | DART_T8020_TCR_BYPASS_DART, |
1266 | |
1267 | .ttbr = DART_T8020_TTBR, |
1268 | .ttbr_valid = DART_T8020_TTBR_VALID, |
1269 | .ttbr_addr_field_shift = DART_T8020_TTBR_ADDR_FIELD_SHIFT, |
1270 | .ttbr_shift = DART_T8020_TTBR_SHIFT, |
1271 | .ttbr_count = 4, |
1272 | }; |
1273 | |
1274 | static const struct apple_dart_hw apple_dart_hw_t8110 = { |
1275 | .type = DART_T8110, |
1276 | .irq_handler = apple_dart_t8110_irq, |
1277 | .invalidate_tlb = apple_dart_t8110_hw_invalidate_tlb, |
1278 | .fmt = APPLE_DART2, |
1279 | .max_sid_count = 256, |
1280 | |
1281 | .enable_streams = DART_T8110_ENABLE_STREAMS, |
1282 | .lock = DART_T8110_PROTECT, |
1283 | .lock_bit = DART_T8110_PROTECT_TTBR_TCR, |
1284 | |
1285 | .error = DART_T8110_ERROR, |
1286 | |
1287 | .tcr = DART_T8110_TCR, |
1288 | .tcr_enabled = DART_T8110_TCR_TRANSLATE_ENABLE, |
1289 | .tcr_disabled = 0, |
1290 | .tcr_bypass = DART_T8110_TCR_BYPASS_DAPF | DART_T8110_TCR_BYPASS_DART, |
1291 | |
1292 | .ttbr = DART_T8110_TTBR, |
1293 | .ttbr_valid = DART_T8110_TTBR_VALID, |
1294 | .ttbr_addr_field_shift = DART_T8110_TTBR_ADDR_FIELD_SHIFT, |
1295 | .ttbr_shift = DART_T8110_TTBR_SHIFT, |
1296 | .ttbr_count = 1, |
1297 | }; |
1298 | |
1299 | static __maybe_unused int apple_dart_suspend(struct device *dev) |
1300 | { |
1301 | struct apple_dart *dart = dev_get_drvdata(dev); |
1302 | unsigned int sid, idx; |
1303 | |
1304 | for (sid = 0; sid < dart->num_streams; sid++) { |
1305 | dart->save_tcr[sid] = readl(addr: dart->regs + DART_TCR(dart, sid)); |
1306 | for (idx = 0; idx < dart->hw->ttbr_count; idx++) |
1307 | dart->save_ttbr[sid][idx] = |
1308 | readl(addr: dart->regs + DART_TTBR(dart, sid, idx)); |
1309 | } |
1310 | |
1311 | return 0; |
1312 | } |
1313 | |
1314 | static __maybe_unused int apple_dart_resume(struct device *dev) |
1315 | { |
1316 | struct apple_dart *dart = dev_get_drvdata(dev); |
1317 | unsigned int sid, idx; |
1318 | int ret; |
1319 | |
1320 | ret = apple_dart_hw_reset(dart); |
1321 | if (ret) { |
1322 | dev_err(dev, "Failed to reset DART on resume\n" ); |
1323 | return ret; |
1324 | } |
1325 | |
1326 | for (sid = 0; sid < dart->num_streams; sid++) { |
1327 | for (idx = 0; idx < dart->hw->ttbr_count; idx++) |
1328 | writel(val: dart->save_ttbr[sid][idx], |
1329 | addr: dart->regs + DART_TTBR(dart, sid, idx)); |
1330 | writel(val: dart->save_tcr[sid], addr: dart->regs + DART_TCR(dart, sid)); |
1331 | } |
1332 | |
1333 | return 0; |
1334 | } |
1335 | |
1336 | static DEFINE_SIMPLE_DEV_PM_OPS(apple_dart_pm_ops, apple_dart_suspend, apple_dart_resume); |
1337 | |
1338 | static const struct of_device_id apple_dart_of_match[] = { |
1339 | { .compatible = "apple,t8103-dart" , .data = &apple_dart_hw_t8103 }, |
1340 | { .compatible = "apple,t8103-usb4-dart" , .data = &apple_dart_hw_t8103_usb4 }, |
1341 | { .compatible = "apple,t8110-dart" , .data = &apple_dart_hw_t8110 }, |
1342 | { .compatible = "apple,t6000-dart" , .data = &apple_dart_hw_t6000 }, |
1343 | {}, |
1344 | }; |
1345 | MODULE_DEVICE_TABLE(of, apple_dart_of_match); |
1346 | |
1347 | static struct platform_driver apple_dart_driver = { |
1348 | .driver = { |
1349 | .name = "apple-dart" , |
1350 | .of_match_table = apple_dart_of_match, |
1351 | .suppress_bind_attrs = true, |
1352 | .pm = pm_sleep_ptr(&apple_dart_pm_ops), |
1353 | }, |
1354 | .probe = apple_dart_probe, |
1355 | .remove_new = apple_dart_remove, |
1356 | }; |
1357 | |
1358 | module_platform_driver(apple_dart_driver); |
1359 | |
1360 | MODULE_DESCRIPTION("IOMMU API for Apple's DART" ); |
1361 | MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>" ); |
1362 | MODULE_LICENSE("GPL v2" ); |
1363 | |