1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * linux/arch/alpha/kernel/pci_iommu.c |
4 | */ |
5 | |
6 | #include <linux/kernel.h> |
7 | #include <linux/mm.h> |
8 | #include <linux/pci.h> |
9 | #include <linux/gfp.h> |
10 | #include <linux/memblock.h> |
11 | #include <linux/export.h> |
12 | #include <linux/scatterlist.h> |
13 | #include <linux/log2.h> |
14 | #include <linux/dma-map-ops.h> |
15 | #include <linux/iommu-helper.h> |
16 | |
17 | #include <asm/io.h> |
18 | #include <asm/hwrpb.h> |
19 | |
20 | #include "proto.h" |
21 | #include "pci_impl.h" |
22 | |
23 | |
24 | #define DEBUG_ALLOC 0 |
25 | #if DEBUG_ALLOC > 0 |
26 | # define DBGA(args...) printk(KERN_DEBUG args) |
27 | #else |
28 | # define DBGA(args...) |
29 | #endif |
30 | #if DEBUG_ALLOC > 1 |
31 | # define DBGA2(args...) printk(KERN_DEBUG args) |
32 | #else |
33 | # define DBGA2(args...) |
34 | #endif |
35 | |
36 | #define DEBUG_NODIRECT 0 |
37 | |
38 | #define ISA_DMA_MASK 0x00ffffff |
39 | |
40 | static inline unsigned long |
41 | mk_iommu_pte(unsigned long paddr) |
42 | { |
43 | return (paddr >> (PAGE_SHIFT-1)) | 1; |
44 | } |
45 | |
46 | /* Return the minimum of MAX or the first power of two larger |
47 | than main memory. */ |
48 | |
49 | unsigned long |
50 | size_for_memory(unsigned long max) |
51 | { |
52 | unsigned long mem = max_low_pfn << PAGE_SHIFT; |
53 | if (mem < max) |
54 | max = roundup_pow_of_two(mem); |
55 | return max; |
56 | } |
57 | |
58 | struct pci_iommu_arena * __init |
59 | iommu_arena_new_node(int nid, struct pci_controller *hose, dma_addr_t base, |
60 | unsigned long window_size, unsigned long align) |
61 | { |
62 | unsigned long mem_size; |
63 | struct pci_iommu_arena *arena; |
64 | |
65 | mem_size = window_size / (PAGE_SIZE / sizeof(unsigned long)); |
66 | |
67 | /* Note that the TLB lookup logic uses bitwise concatenation, |
68 | not addition, so the required arena alignment is based on |
69 | the size of the window. Retain the align parameter so that |
70 | particular systems can over-align the arena. */ |
71 | if (align < mem_size) |
72 | align = mem_size; |
73 | |
74 | arena = memblock_alloc(size: sizeof(*arena), SMP_CACHE_BYTES); |
75 | if (!arena) |
76 | panic(fmt: "%s: Failed to allocate %zu bytes\n" , __func__, |
77 | sizeof(*arena)); |
78 | arena->ptes = memblock_alloc(size: mem_size, align); |
79 | if (!arena->ptes) |
80 | panic(fmt: "%s: Failed to allocate %lu bytes align=0x%lx\n" , |
81 | __func__, mem_size, align); |
82 | |
83 | spin_lock_init(&arena->lock); |
84 | arena->hose = hose; |
85 | arena->dma_base = base; |
86 | arena->size = window_size; |
87 | arena->next_entry = 0; |
88 | |
89 | /* Align allocations to a multiple of a page size. Not needed |
90 | unless there are chip bugs. */ |
91 | arena->align_entry = 1; |
92 | |
93 | return arena; |
94 | } |
95 | |
96 | struct pci_iommu_arena * __init |
97 | iommu_arena_new(struct pci_controller *hose, dma_addr_t base, |
98 | unsigned long window_size, unsigned long align) |
99 | { |
100 | return iommu_arena_new_node(nid: 0, hose, base, window_size, align); |
101 | } |
102 | |
103 | /* Must be called with the arena lock held */ |
104 | static long |
105 | iommu_arena_find_pages(struct device *dev, struct pci_iommu_arena *arena, |
106 | long n, long mask) |
107 | { |
108 | unsigned long *ptes; |
109 | long i, p, nent; |
110 | int pass = 0; |
111 | unsigned long base; |
112 | unsigned long boundary_size; |
113 | |
114 | base = arena->dma_base >> PAGE_SHIFT; |
115 | boundary_size = dma_get_seg_boundary_nr_pages(dev, PAGE_SHIFT); |
116 | |
117 | /* Search forward for the first mask-aligned sequence of N free ptes */ |
118 | ptes = arena->ptes; |
119 | nent = arena->size >> PAGE_SHIFT; |
120 | p = ALIGN(arena->next_entry, mask + 1); |
121 | i = 0; |
122 | |
123 | again: |
124 | while (i < n && p+i < nent) { |
125 | if (!i && iommu_is_span_boundary(index: p, nr: n, shift: base, boundary_size)) { |
126 | p = ALIGN(p + 1, mask + 1); |
127 | goto again; |
128 | } |
129 | |
130 | if (ptes[p+i]) { |
131 | p = ALIGN(p + i + 1, mask + 1); |
132 | i = 0; |
133 | } else { |
134 | i = i + 1; |
135 | } |
136 | } |
137 | |
138 | if (i < n) { |
139 | if (pass < 1) { |
140 | /* |
141 | * Reached the end. Flush the TLB and restart |
142 | * the search from the beginning. |
143 | */ |
144 | alpha_mv.mv_pci_tbi(arena->hose, 0, -1); |
145 | |
146 | pass++; |
147 | p = 0; |
148 | i = 0; |
149 | goto again; |
150 | } else |
151 | return -1; |
152 | } |
153 | |
154 | /* Success. It's the responsibility of the caller to mark them |
155 | in use before releasing the lock */ |
156 | return p; |
157 | } |
158 | |
159 | static long |
160 | iommu_arena_alloc(struct device *dev, struct pci_iommu_arena *arena, long n, |
161 | unsigned int align) |
162 | { |
163 | unsigned long flags; |
164 | unsigned long *ptes; |
165 | long i, p, mask; |
166 | |
167 | spin_lock_irqsave(&arena->lock, flags); |
168 | |
169 | /* Search for N empty ptes */ |
170 | ptes = arena->ptes; |
171 | mask = max(align, arena->align_entry) - 1; |
172 | p = iommu_arena_find_pages(dev, arena, n, mask); |
173 | if (p < 0) { |
174 | spin_unlock_irqrestore(lock: &arena->lock, flags); |
175 | return -1; |
176 | } |
177 | |
178 | /* Success. Mark them all in use, ie not zero and invalid |
179 | for the iommu tlb that could load them from under us. |
180 | The chip specific bits will fill this in with something |
181 | kosher when we return. */ |
182 | for (i = 0; i < n; ++i) |
183 | ptes[p+i] = IOMMU_INVALID_PTE; |
184 | |
185 | arena->next_entry = p + n; |
186 | spin_unlock_irqrestore(lock: &arena->lock, flags); |
187 | |
188 | return p; |
189 | } |
190 | |
191 | static void |
192 | iommu_arena_free(struct pci_iommu_arena *arena, long ofs, long n) |
193 | { |
194 | unsigned long *p; |
195 | long i; |
196 | |
197 | p = arena->ptes + ofs; |
198 | for (i = 0; i < n; ++i) |
199 | p[i] = 0; |
200 | } |
201 | |
202 | /* |
203 | * True if the machine supports DAC addressing, and DEV can |
204 | * make use of it given MASK. |
205 | */ |
206 | static int pci_dac_dma_supported(struct pci_dev *dev, u64 mask) |
207 | { |
208 | dma_addr_t dac_offset = alpha_mv.pci_dac_offset; |
209 | int ok = 1; |
210 | |
211 | /* If this is not set, the machine doesn't support DAC at all. */ |
212 | if (dac_offset == 0) |
213 | ok = 0; |
214 | |
215 | /* The device has to be able to address our DAC bit. */ |
216 | if ((dac_offset & dev->dma_mask) != dac_offset) |
217 | ok = 0; |
218 | |
219 | /* If both conditions above are met, we are fine. */ |
220 | DBGA("pci_dac_dma_supported %s from %ps\n" , |
221 | ok ? "yes" : "no" , __builtin_return_address(0)); |
222 | |
223 | return ok; |
224 | } |
225 | |
226 | /* Map a single buffer of the indicated size for PCI DMA in streaming |
227 | mode. The 32-bit PCI bus mastering address to use is returned. |
228 | Once the device is given the dma address, the device owns this memory |
229 | until either pci_unmap_single or pci_dma_sync_single is performed. */ |
230 | |
231 | static dma_addr_t |
232 | pci_map_single_1(struct pci_dev *pdev, void *cpu_addr, size_t size, |
233 | int dac_allowed) |
234 | { |
235 | struct pci_controller *hose = pdev ? pdev->sysdata : pci_isa_hose; |
236 | dma_addr_t max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK; |
237 | struct pci_iommu_arena *arena; |
238 | long npages, dma_ofs, i; |
239 | unsigned long paddr; |
240 | dma_addr_t ret; |
241 | unsigned int align = 0; |
242 | struct device *dev = pdev ? &pdev->dev : NULL; |
243 | |
244 | paddr = __pa(cpu_addr); |
245 | |
246 | #if !DEBUG_NODIRECT |
247 | /* First check to see if we can use the direct map window. */ |
248 | if (paddr + size + __direct_map_base - 1 <= max_dma |
249 | && paddr + size <= __direct_map_size) { |
250 | ret = paddr + __direct_map_base; |
251 | |
252 | DBGA2("pci_map_single: [%p,%zx] -> direct %llx from %ps\n" , |
253 | cpu_addr, size, ret, __builtin_return_address(0)); |
254 | |
255 | return ret; |
256 | } |
257 | #endif |
258 | |
259 | /* Next, use DAC if selected earlier. */ |
260 | if (dac_allowed) { |
261 | ret = paddr + alpha_mv.pci_dac_offset; |
262 | |
263 | DBGA2("pci_map_single: [%p,%zx] -> DAC %llx from %ps\n" , |
264 | cpu_addr, size, ret, __builtin_return_address(0)); |
265 | |
266 | return ret; |
267 | } |
268 | |
269 | /* If the machine doesn't define a pci_tbi routine, we have to |
270 | assume it doesn't support sg mapping, and, since we tried to |
271 | use direct_map above, it now must be considered an error. */ |
272 | if (! alpha_mv.mv_pci_tbi) { |
273 | printk_once(KERN_WARNING "pci_map_single: no HW sg\n" ); |
274 | return DMA_MAPPING_ERROR; |
275 | } |
276 | |
277 | arena = hose->sg_pci; |
278 | if (!arena || arena->dma_base + arena->size - 1 > max_dma) |
279 | arena = hose->sg_isa; |
280 | |
281 | npages = iommu_num_pages(addr: paddr, len: size, PAGE_SIZE); |
282 | |
283 | /* Force allocation to 64KB boundary for ISA bridges. */ |
284 | if (pdev && pdev == isa_bridge) |
285 | align = 8; |
286 | dma_ofs = iommu_arena_alloc(dev, arena, n: npages, align); |
287 | if (dma_ofs < 0) { |
288 | printk(KERN_WARNING "pci_map_single failed: " |
289 | "could not allocate dma page tables\n" ); |
290 | return DMA_MAPPING_ERROR; |
291 | } |
292 | |
293 | paddr &= PAGE_MASK; |
294 | for (i = 0; i < npages; ++i, paddr += PAGE_SIZE) |
295 | arena->ptes[i + dma_ofs] = mk_iommu_pte(paddr); |
296 | |
297 | ret = arena->dma_base + dma_ofs * PAGE_SIZE; |
298 | ret += (unsigned long)cpu_addr & ~PAGE_MASK; |
299 | |
300 | DBGA2("pci_map_single: [%p,%zx] np %ld -> sg %llx from %ps\n" , |
301 | cpu_addr, size, npages, ret, __builtin_return_address(0)); |
302 | |
303 | return ret; |
304 | } |
305 | |
306 | /* Helper for generic DMA-mapping functions. */ |
307 | static struct pci_dev *alpha_gendev_to_pci(struct device *dev) |
308 | { |
309 | if (dev && dev_is_pci(dev)) |
310 | return to_pci_dev(dev); |
311 | |
312 | /* Assume that non-PCI devices asking for DMA are either ISA or EISA, |
313 | BUG() otherwise. */ |
314 | BUG_ON(!isa_bridge); |
315 | |
316 | /* Assume non-busmaster ISA DMA when dma_mask is not set (the ISA |
317 | bridge is bus master then). */ |
318 | if (!dev || !dev->dma_mask || !*dev->dma_mask) |
319 | return isa_bridge; |
320 | |
321 | /* For EISA bus masters, return isa_bridge (it might have smaller |
322 | dma_mask due to wiring limitations). */ |
323 | if (*dev->dma_mask >= isa_bridge->dma_mask) |
324 | return isa_bridge; |
325 | |
326 | /* This assumes ISA bus master with dma_mask 0xffffff. */ |
327 | return NULL; |
328 | } |
329 | |
330 | static dma_addr_t alpha_pci_map_page(struct device *dev, struct page *page, |
331 | unsigned long offset, size_t size, |
332 | enum dma_data_direction dir, |
333 | unsigned long attrs) |
334 | { |
335 | struct pci_dev *pdev = alpha_gendev_to_pci(dev); |
336 | int dac_allowed; |
337 | |
338 | BUG_ON(dir == DMA_NONE); |
339 | |
340 | dac_allowed = pdev ? pci_dac_dma_supported(dev: pdev, mask: pdev->dma_mask) : 0; |
341 | return pci_map_single_1(pdev, cpu_addr: (char *)page_address(page) + offset, |
342 | size, dac_allowed); |
343 | } |
344 | |
345 | /* Unmap a single streaming mode DMA translation. The DMA_ADDR and |
346 | SIZE must match what was provided for in a previous pci_map_single |
347 | call. All other usages are undefined. After this call, reads by |
348 | the cpu to the buffer are guaranteed to see whatever the device |
349 | wrote there. */ |
350 | |
351 | static void alpha_pci_unmap_page(struct device *dev, dma_addr_t dma_addr, |
352 | size_t size, enum dma_data_direction dir, |
353 | unsigned long attrs) |
354 | { |
355 | unsigned long flags; |
356 | struct pci_dev *pdev = alpha_gendev_to_pci(dev); |
357 | struct pci_controller *hose = pdev ? pdev->sysdata : pci_isa_hose; |
358 | struct pci_iommu_arena *arena; |
359 | long dma_ofs, npages; |
360 | |
361 | BUG_ON(dir == DMA_NONE); |
362 | |
363 | if (dma_addr >= __direct_map_base |
364 | && dma_addr < __direct_map_base + __direct_map_size) { |
365 | /* Nothing to do. */ |
366 | |
367 | DBGA2("pci_unmap_single: direct [%llx,%zx] from %ps\n" , |
368 | dma_addr, size, __builtin_return_address(0)); |
369 | |
370 | return; |
371 | } |
372 | |
373 | if (dma_addr > 0xffffffff) { |
374 | DBGA2("pci64_unmap_single: DAC [%llx,%zx] from %ps\n" , |
375 | dma_addr, size, __builtin_return_address(0)); |
376 | return; |
377 | } |
378 | |
379 | arena = hose->sg_pci; |
380 | if (!arena || dma_addr < arena->dma_base) |
381 | arena = hose->sg_isa; |
382 | |
383 | dma_ofs = (dma_addr - arena->dma_base) >> PAGE_SHIFT; |
384 | if (dma_ofs * PAGE_SIZE >= arena->size) { |
385 | printk(KERN_ERR "Bogus pci_unmap_single: dma_addr %llx " |
386 | " base %llx size %x\n" , |
387 | dma_addr, arena->dma_base, arena->size); |
388 | return; |
389 | BUG(); |
390 | } |
391 | |
392 | npages = iommu_num_pages(addr: dma_addr, len: size, PAGE_SIZE); |
393 | |
394 | spin_lock_irqsave(&arena->lock, flags); |
395 | |
396 | iommu_arena_free(arena, ofs: dma_ofs, n: npages); |
397 | |
398 | /* If we're freeing ptes above the `next_entry' pointer (they |
399 | may have snuck back into the TLB since the last wrap flush), |
400 | we need to flush the TLB before reallocating the latter. */ |
401 | if (dma_ofs >= arena->next_entry) |
402 | alpha_mv.mv_pci_tbi(hose, dma_addr, dma_addr + size - 1); |
403 | |
404 | spin_unlock_irqrestore(lock: &arena->lock, flags); |
405 | |
406 | DBGA2("pci_unmap_single: sg [%llx,%zx] np %ld from %ps\n" , |
407 | dma_addr, size, npages, __builtin_return_address(0)); |
408 | } |
409 | |
410 | /* Allocate and map kernel buffer using consistent mode DMA for PCI |
411 | device. Returns non-NULL cpu-view pointer to the buffer if |
412 | successful and sets *DMA_ADDRP to the pci side dma address as well, |
413 | else DMA_ADDRP is undefined. */ |
414 | |
415 | static void *alpha_pci_alloc_coherent(struct device *dev, size_t size, |
416 | dma_addr_t *dma_addrp, gfp_t gfp, |
417 | unsigned long attrs) |
418 | { |
419 | struct pci_dev *pdev = alpha_gendev_to_pci(dev); |
420 | void *cpu_addr; |
421 | long order = get_order(size); |
422 | |
423 | gfp &= ~GFP_DMA; |
424 | |
425 | try_again: |
426 | cpu_addr = (void *)__get_free_pages(gfp_mask: gfp | __GFP_ZERO, order); |
427 | if (! cpu_addr) { |
428 | printk(KERN_INFO "pci_alloc_consistent: " |
429 | "get_free_pages failed from %ps\n" , |
430 | __builtin_return_address(0)); |
431 | /* ??? Really atomic allocation? Otherwise we could play |
432 | with vmalloc and sg if we can't find contiguous memory. */ |
433 | return NULL; |
434 | } |
435 | memset(cpu_addr, 0, size); |
436 | |
437 | *dma_addrp = pci_map_single_1(pdev, cpu_addr, size, dac_allowed: 0); |
438 | if (*dma_addrp == DMA_MAPPING_ERROR) { |
439 | free_pages(addr: (unsigned long)cpu_addr, order); |
440 | if (alpha_mv.mv_pci_tbi || (gfp & GFP_DMA)) |
441 | return NULL; |
442 | /* The address doesn't fit required mask and we |
443 | do not have iommu. Try again with GFP_DMA. */ |
444 | gfp |= GFP_DMA; |
445 | goto try_again; |
446 | } |
447 | |
448 | DBGA2("pci_alloc_consistent: %zx -> [%p,%llx] from %ps\n" , |
449 | size, cpu_addr, *dma_addrp, __builtin_return_address(0)); |
450 | |
451 | return cpu_addr; |
452 | } |
453 | |
454 | /* Free and unmap a consistent DMA buffer. CPU_ADDR and DMA_ADDR must |
455 | be values that were returned from pci_alloc_consistent. SIZE must |
456 | be the same as what as passed into pci_alloc_consistent. |
457 | References to the memory and mappings associated with CPU_ADDR or |
458 | DMA_ADDR past this call are illegal. */ |
459 | |
460 | static void alpha_pci_free_coherent(struct device *dev, size_t size, |
461 | void *cpu_addr, dma_addr_t dma_addr, |
462 | unsigned long attrs) |
463 | { |
464 | struct pci_dev *pdev = alpha_gendev_to_pci(dev); |
465 | dma_unmap_single(&pdev->dev, dma_addr, size, DMA_BIDIRECTIONAL); |
466 | free_pages(addr: (unsigned long)cpu_addr, order: get_order(size)); |
467 | |
468 | DBGA2("pci_free_consistent: [%llx,%zx] from %ps\n" , |
469 | dma_addr, size, __builtin_return_address(0)); |
470 | } |
471 | |
472 | /* Classify the elements of the scatterlist. Write dma_address |
473 | of each element with: |
474 | 0 : Followers all physically adjacent. |
475 | 1 : Followers all virtually adjacent. |
476 | -1 : Not leader, physically adjacent to previous. |
477 | -2 : Not leader, virtually adjacent to previous. |
478 | Write dma_length of each leader with the combined lengths of |
479 | the mergable followers. */ |
480 | |
481 | #define SG_ENT_VIRT_ADDRESS(SG) (sg_virt((SG))) |
482 | #define SG_ENT_PHYS_ADDRESS(SG) __pa(SG_ENT_VIRT_ADDRESS(SG)) |
483 | |
484 | static void |
485 | sg_classify(struct device *dev, struct scatterlist *sg, struct scatterlist *end, |
486 | int virt_ok) |
487 | { |
488 | unsigned long next_paddr; |
489 | struct scatterlist *leader; |
490 | long leader_flag, leader_length; |
491 | unsigned int max_seg_size; |
492 | |
493 | leader = sg; |
494 | leader_flag = 0; |
495 | leader_length = leader->length; |
496 | next_paddr = SG_ENT_PHYS_ADDRESS(leader) + leader_length; |
497 | |
498 | /* we will not marge sg without device. */ |
499 | max_seg_size = dev ? dma_get_max_seg_size(dev) : 0; |
500 | for (++sg; sg < end; ++sg) { |
501 | unsigned long addr, len; |
502 | addr = SG_ENT_PHYS_ADDRESS(sg); |
503 | len = sg->length; |
504 | |
505 | if (leader_length + len > max_seg_size) |
506 | goto new_segment; |
507 | |
508 | if (next_paddr == addr) { |
509 | sg->dma_address = -1; |
510 | leader_length += len; |
511 | } else if (((next_paddr | addr) & ~PAGE_MASK) == 0 && virt_ok) { |
512 | sg->dma_address = -2; |
513 | leader_flag = 1; |
514 | leader_length += len; |
515 | } else { |
516 | new_segment: |
517 | leader->dma_address = leader_flag; |
518 | leader->dma_length = leader_length; |
519 | leader = sg; |
520 | leader_flag = 0; |
521 | leader_length = len; |
522 | } |
523 | |
524 | next_paddr = addr + len; |
525 | } |
526 | |
527 | leader->dma_address = leader_flag; |
528 | leader->dma_length = leader_length; |
529 | } |
530 | |
531 | /* Given a scatterlist leader, choose an allocation method and fill |
532 | in the blanks. */ |
533 | |
534 | static int |
535 | sg_fill(struct device *dev, struct scatterlist *leader, struct scatterlist *end, |
536 | struct scatterlist *out, struct pci_iommu_arena *arena, |
537 | dma_addr_t max_dma, int dac_allowed) |
538 | { |
539 | unsigned long paddr = SG_ENT_PHYS_ADDRESS(leader); |
540 | long size = leader->dma_length; |
541 | struct scatterlist *sg; |
542 | unsigned long *ptes; |
543 | long npages, dma_ofs, i; |
544 | |
545 | #if !DEBUG_NODIRECT |
546 | /* If everything is physically contiguous, and the addresses |
547 | fall into the direct-map window, use it. */ |
548 | if (leader->dma_address == 0 |
549 | && paddr + size + __direct_map_base - 1 <= max_dma |
550 | && paddr + size <= __direct_map_size) { |
551 | out->dma_address = paddr + __direct_map_base; |
552 | out->dma_length = size; |
553 | |
554 | DBGA(" sg_fill: [%p,%lx] -> direct %llx\n" , |
555 | __va(paddr), size, out->dma_address); |
556 | |
557 | return 0; |
558 | } |
559 | #endif |
560 | |
561 | /* If physically contiguous and DAC is available, use it. */ |
562 | if (leader->dma_address == 0 && dac_allowed) { |
563 | out->dma_address = paddr + alpha_mv.pci_dac_offset; |
564 | out->dma_length = size; |
565 | |
566 | DBGA(" sg_fill: [%p,%lx] -> DAC %llx\n" , |
567 | __va(paddr), size, out->dma_address); |
568 | |
569 | return 0; |
570 | } |
571 | |
572 | /* Otherwise, we'll use the iommu to make the pages virtually |
573 | contiguous. */ |
574 | |
575 | paddr &= ~PAGE_MASK; |
576 | npages = iommu_num_pages(addr: paddr, len: size, PAGE_SIZE); |
577 | dma_ofs = iommu_arena_alloc(dev, arena, n: npages, align: 0); |
578 | if (dma_ofs < 0) { |
579 | /* If we attempted a direct map above but failed, die. */ |
580 | if (leader->dma_address == 0) |
581 | return -1; |
582 | |
583 | /* Otherwise, break up the remaining virtually contiguous |
584 | hunks into individual direct maps and retry. */ |
585 | sg_classify(dev, sg: leader, end, virt_ok: 0); |
586 | return sg_fill(dev, leader, end, out, arena, max_dma, dac_allowed); |
587 | } |
588 | |
589 | out->dma_address = arena->dma_base + dma_ofs*PAGE_SIZE + paddr; |
590 | out->dma_length = size; |
591 | |
592 | DBGA(" sg_fill: [%p,%lx] -> sg %llx np %ld\n" , |
593 | __va(paddr), size, out->dma_address, npages); |
594 | |
595 | /* All virtually contiguous. We need to find the length of each |
596 | physically contiguous subsegment to fill in the ptes. */ |
597 | ptes = &arena->ptes[dma_ofs]; |
598 | sg = leader; |
599 | do { |
600 | #if DEBUG_ALLOC > 0 |
601 | struct scatterlist *last_sg = sg; |
602 | #endif |
603 | |
604 | size = sg->length; |
605 | paddr = SG_ENT_PHYS_ADDRESS(sg); |
606 | |
607 | while (sg+1 < end && (int) sg[1].dma_address == -1) { |
608 | size += sg[1].length; |
609 | sg = sg_next(sg); |
610 | } |
611 | |
612 | npages = iommu_num_pages(addr: paddr, len: size, PAGE_SIZE); |
613 | |
614 | paddr &= PAGE_MASK; |
615 | for (i = 0; i < npages; ++i, paddr += PAGE_SIZE) |
616 | *ptes++ = mk_iommu_pte(paddr); |
617 | |
618 | #if DEBUG_ALLOC > 0 |
619 | DBGA(" (%ld) [%p,%x] np %ld\n" , |
620 | last_sg - leader, SG_ENT_VIRT_ADDRESS(last_sg), |
621 | last_sg->length, npages); |
622 | while (++last_sg <= sg) { |
623 | DBGA(" (%ld) [%p,%x] cont\n" , |
624 | last_sg - leader, SG_ENT_VIRT_ADDRESS(last_sg), |
625 | last_sg->length); |
626 | } |
627 | #endif |
628 | } while (++sg < end && (int) sg->dma_address < 0); |
629 | |
630 | return 1; |
631 | } |
632 | |
633 | static int alpha_pci_map_sg(struct device *dev, struct scatterlist *sg, |
634 | int nents, enum dma_data_direction dir, |
635 | unsigned long attrs) |
636 | { |
637 | struct pci_dev *pdev = alpha_gendev_to_pci(dev); |
638 | struct scatterlist *start, *end, *out; |
639 | struct pci_controller *hose; |
640 | struct pci_iommu_arena *arena; |
641 | dma_addr_t max_dma; |
642 | int dac_allowed; |
643 | |
644 | BUG_ON(dir == DMA_NONE); |
645 | |
646 | dac_allowed = dev ? pci_dac_dma_supported(dev: pdev, mask: pdev->dma_mask) : 0; |
647 | |
648 | /* Fast path single entry scatterlists. */ |
649 | if (nents == 1) { |
650 | sg->dma_length = sg->length; |
651 | sg->dma_address |
652 | = pci_map_single_1(pdev, SG_ENT_VIRT_ADDRESS(sg), |
653 | size: sg->length, dac_allowed); |
654 | if (sg->dma_address == DMA_MAPPING_ERROR) |
655 | return -EIO; |
656 | return 1; |
657 | } |
658 | |
659 | start = sg; |
660 | end = sg + nents; |
661 | |
662 | /* First, prepare information about the entries. */ |
663 | sg_classify(dev, sg, end, virt_ok: alpha_mv.mv_pci_tbi != 0); |
664 | |
665 | /* Second, figure out where we're going to map things. */ |
666 | if (alpha_mv.mv_pci_tbi) { |
667 | hose = pdev ? pdev->sysdata : pci_isa_hose; |
668 | max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK; |
669 | arena = hose->sg_pci; |
670 | if (!arena || arena->dma_base + arena->size - 1 > max_dma) |
671 | arena = hose->sg_isa; |
672 | } else { |
673 | max_dma = -1; |
674 | arena = NULL; |
675 | hose = NULL; |
676 | } |
677 | |
678 | /* Third, iterate over the scatterlist leaders and allocate |
679 | dma space as needed. */ |
680 | for (out = sg; sg < end; ++sg) { |
681 | if ((int) sg->dma_address < 0) |
682 | continue; |
683 | if (sg_fill(dev, leader: sg, end, out, arena, max_dma, dac_allowed) < 0) |
684 | goto error; |
685 | out++; |
686 | } |
687 | |
688 | /* Mark the end of the list for pci_unmap_sg. */ |
689 | if (out < end) |
690 | out->dma_length = 0; |
691 | |
692 | if (out - start == 0) { |
693 | printk(KERN_WARNING "pci_map_sg failed: no entries?\n" ); |
694 | return -ENOMEM; |
695 | } |
696 | DBGA("pci_map_sg: %ld entries\n" , out - start); |
697 | |
698 | return out - start; |
699 | |
700 | error: |
701 | printk(KERN_WARNING "pci_map_sg failed: " |
702 | "could not allocate dma page tables\n" ); |
703 | |
704 | /* Some allocation failed while mapping the scatterlist |
705 | entries. Unmap them now. */ |
706 | if (out > start) |
707 | dma_unmap_sg(&pdev->dev, start, out - start, dir); |
708 | return -ENOMEM; |
709 | } |
710 | |
711 | /* Unmap a set of streaming mode DMA translations. Again, cpu read |
712 | rules concerning calls here are the same as for pci_unmap_single() |
713 | above. */ |
714 | |
715 | static void alpha_pci_unmap_sg(struct device *dev, struct scatterlist *sg, |
716 | int nents, enum dma_data_direction dir, |
717 | unsigned long attrs) |
718 | { |
719 | struct pci_dev *pdev = alpha_gendev_to_pci(dev); |
720 | unsigned long flags; |
721 | struct pci_controller *hose; |
722 | struct pci_iommu_arena *arena; |
723 | struct scatterlist *end; |
724 | dma_addr_t max_dma; |
725 | dma_addr_t fbeg, fend; |
726 | |
727 | BUG_ON(dir == DMA_NONE); |
728 | |
729 | if (! alpha_mv.mv_pci_tbi) |
730 | return; |
731 | |
732 | hose = pdev ? pdev->sysdata : pci_isa_hose; |
733 | max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK; |
734 | arena = hose->sg_pci; |
735 | if (!arena || arena->dma_base + arena->size - 1 > max_dma) |
736 | arena = hose->sg_isa; |
737 | |
738 | fbeg = -1, fend = 0; |
739 | |
740 | spin_lock_irqsave(&arena->lock, flags); |
741 | |
742 | for (end = sg + nents; sg < end; ++sg) { |
743 | dma_addr_t addr; |
744 | size_t size; |
745 | long npages, ofs; |
746 | dma_addr_t tend; |
747 | |
748 | addr = sg->dma_address; |
749 | size = sg->dma_length; |
750 | if (!size) |
751 | break; |
752 | |
753 | if (addr > 0xffffffff) { |
754 | /* It's a DAC address -- nothing to do. */ |
755 | DBGA(" (%ld) DAC [%llx,%zx]\n" , |
756 | sg - end + nents, addr, size); |
757 | continue; |
758 | } |
759 | |
760 | if (addr >= __direct_map_base |
761 | && addr < __direct_map_base + __direct_map_size) { |
762 | /* Nothing to do. */ |
763 | DBGA(" (%ld) direct [%llx,%zx]\n" , |
764 | sg - end + nents, addr, size); |
765 | continue; |
766 | } |
767 | |
768 | DBGA(" (%ld) sg [%llx,%zx]\n" , |
769 | sg - end + nents, addr, size); |
770 | |
771 | npages = iommu_num_pages(addr, len: size, PAGE_SIZE); |
772 | ofs = (addr - arena->dma_base) >> PAGE_SHIFT; |
773 | iommu_arena_free(arena, ofs, n: npages); |
774 | |
775 | tend = addr + size - 1; |
776 | if (fbeg > addr) fbeg = addr; |
777 | if (fend < tend) fend = tend; |
778 | } |
779 | |
780 | /* If we're freeing ptes above the `next_entry' pointer (they |
781 | may have snuck back into the TLB since the last wrap flush), |
782 | we need to flush the TLB before reallocating the latter. */ |
783 | if ((fend - arena->dma_base) >> PAGE_SHIFT >= arena->next_entry) |
784 | alpha_mv.mv_pci_tbi(hose, fbeg, fend); |
785 | |
786 | spin_unlock_irqrestore(lock: &arena->lock, flags); |
787 | |
788 | DBGA("pci_unmap_sg: %ld entries\n" , nents - (end - sg)); |
789 | } |
790 | |
791 | /* Return whether the given PCI device DMA address mask can be |
792 | supported properly. */ |
793 | |
794 | static int alpha_pci_supported(struct device *dev, u64 mask) |
795 | { |
796 | struct pci_dev *pdev = alpha_gendev_to_pci(dev); |
797 | struct pci_controller *hose; |
798 | struct pci_iommu_arena *arena; |
799 | |
800 | /* If there exists a direct map, and the mask fits either |
801 | the entire direct mapped space or the total system memory as |
802 | shifted by the map base */ |
803 | if (__direct_map_size != 0 |
804 | && (__direct_map_base + __direct_map_size - 1 <= mask || |
805 | __direct_map_base + (max_low_pfn << PAGE_SHIFT) - 1 <= mask)) |
806 | return 1; |
807 | |
808 | /* Check that we have a scatter-gather arena that fits. */ |
809 | hose = pdev ? pdev->sysdata : pci_isa_hose; |
810 | arena = hose->sg_isa; |
811 | if (arena && arena->dma_base + arena->size - 1 <= mask) |
812 | return 1; |
813 | arena = hose->sg_pci; |
814 | if (arena && arena->dma_base + arena->size - 1 <= mask) |
815 | return 1; |
816 | |
817 | /* As last resort try ZONE_DMA. */ |
818 | if (!__direct_map_base && MAX_DMA_ADDRESS - IDENT_ADDR - 1 <= mask) |
819 | return 1; |
820 | |
821 | return 0; |
822 | } |
823 | |
824 | |
825 | /* |
826 | * AGP GART extensions to the IOMMU |
827 | */ |
828 | int |
829 | iommu_reserve(struct pci_iommu_arena *arena, long pg_count, long align_mask) |
830 | { |
831 | unsigned long flags; |
832 | unsigned long *ptes; |
833 | long i, p; |
834 | |
835 | if (!arena) return -EINVAL; |
836 | |
837 | spin_lock_irqsave(&arena->lock, flags); |
838 | |
839 | /* Search for N empty ptes. */ |
840 | ptes = arena->ptes; |
841 | p = iommu_arena_find_pages(NULL, arena, n: pg_count, mask: align_mask); |
842 | if (p < 0) { |
843 | spin_unlock_irqrestore(lock: &arena->lock, flags); |
844 | return -1; |
845 | } |
846 | |
847 | /* Success. Mark them all reserved (ie not zero and invalid) |
848 | for the iommu tlb that could load them from under us. |
849 | They will be filled in with valid bits by _bind() */ |
850 | for (i = 0; i < pg_count; ++i) |
851 | ptes[p+i] = IOMMU_RESERVED_PTE; |
852 | |
853 | arena->next_entry = p + pg_count; |
854 | spin_unlock_irqrestore(lock: &arena->lock, flags); |
855 | |
856 | return p; |
857 | } |
858 | |
859 | int |
860 | iommu_release(struct pci_iommu_arena *arena, long pg_start, long pg_count) |
861 | { |
862 | unsigned long *ptes; |
863 | long i; |
864 | |
865 | if (!arena) return -EINVAL; |
866 | |
867 | ptes = arena->ptes; |
868 | |
869 | /* Make sure they're all reserved first... */ |
870 | for(i = pg_start; i < pg_start + pg_count; i++) |
871 | if (ptes[i] != IOMMU_RESERVED_PTE) |
872 | return -EBUSY; |
873 | |
874 | iommu_arena_free(arena, ofs: pg_start, n: pg_count); |
875 | return 0; |
876 | } |
877 | |
878 | int |
879 | iommu_bind(struct pci_iommu_arena *arena, long pg_start, long pg_count, |
880 | struct page **pages) |
881 | { |
882 | unsigned long flags; |
883 | unsigned long *ptes; |
884 | long i, j; |
885 | |
886 | if (!arena) return -EINVAL; |
887 | |
888 | spin_lock_irqsave(&arena->lock, flags); |
889 | |
890 | ptes = arena->ptes; |
891 | |
892 | for(j = pg_start; j < pg_start + pg_count; j++) { |
893 | if (ptes[j] != IOMMU_RESERVED_PTE) { |
894 | spin_unlock_irqrestore(lock: &arena->lock, flags); |
895 | return -EBUSY; |
896 | } |
897 | } |
898 | |
899 | for(i = 0, j = pg_start; i < pg_count; i++, j++) |
900 | ptes[j] = mk_iommu_pte(page_to_phys(pages[i])); |
901 | |
902 | spin_unlock_irqrestore(lock: &arena->lock, flags); |
903 | |
904 | return 0; |
905 | } |
906 | |
907 | int |
908 | iommu_unbind(struct pci_iommu_arena *arena, long pg_start, long pg_count) |
909 | { |
910 | unsigned long *p; |
911 | long i; |
912 | |
913 | if (!arena) return -EINVAL; |
914 | |
915 | p = arena->ptes + pg_start; |
916 | for(i = 0; i < pg_count; i++) |
917 | p[i] = IOMMU_RESERVED_PTE; |
918 | |
919 | return 0; |
920 | } |
921 | |
922 | const struct dma_map_ops alpha_pci_ops = { |
923 | .alloc = alpha_pci_alloc_coherent, |
924 | .free = alpha_pci_free_coherent, |
925 | .map_page = alpha_pci_map_page, |
926 | .unmap_page = alpha_pci_unmap_page, |
927 | .map_sg = alpha_pci_map_sg, |
928 | .unmap_sg = alpha_pci_unmap_sg, |
929 | .dma_supported = alpha_pci_supported, |
930 | .mmap = dma_common_mmap, |
931 | .get_sgtable = dma_common_get_sgtable, |
932 | .alloc_pages = dma_common_alloc_pages, |
933 | .free_pages = dma_common_free_pages, |
934 | }; |
935 | EXPORT_SYMBOL(alpha_pci_ops); |
936 | |