1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* Dynamic DMA mapping support.
4	*
5	* This implementation is a fallback for platforms that do not support
6	* I/O TLBs (aka DMA address translation hardware).
7	* Copyright (C) 2000 Asit Mallick <Asit.K.Mallick@intel.com>
8	* Copyright (C) 2000 Goutham Rao <goutham.rao@intel.com>
9	* Copyright (C) 2000, 2003 Hewlett-Packard Co
10	* David Mosberger-Tang <davidm@hpl.hp.com>
11	*
12	* 03/05/07 davidm Switch from PCI-DMA to generic device DMA API.
13	* 00/12/13 davidm Rename to swiotlb.c and add mark_clean() to avoid
14	* unnecessary i-cache flushing.
15	* 04/07/.. ak Better overflow handling. Assorted fixes.
16	* 05/09/10 linville Add support for syncing ranges, support syncing for
17	* DMA_BIDIRECTIONAL mappings, miscellaneous cleanup.
18	* 08/12/11 beckyb Add highmem support
19	*/
20
21	#define pr_fmt(fmt) "software IO TLB: " fmt
22
23	#include <linux/cache.h>
24	#include <linux/cc_platform.h>
25	#include <linux/ctype.h>
26	#include <linux/debugfs.h>
27	#include <linux/dma-direct.h>
28	#include <linux/dma-map-ops.h>
29	#include <linux/export.h>
30	#include <linux/gfp.h>
31	#include <linux/highmem.h>
32	#include <linux/io.h>
33	#include <linux/iommu-helper.h>
34	#include <linux/init.h>
35	#include <linux/memblock.h>
36	#include <linux/mm.h>
37	#include <linux/pfn.h>
38	#include <linux/rculist.h>
39	#include <linux/scatterlist.h>
40	#include <linux/set_memory.h>
41	#include <linux/spinlock.h>
42	#include <linux/string.h>
43	#include <linux/swiotlb.h>
44	#include <linux/types.h>
45	#ifdef CONFIG_DMA_RESTRICTED_POOL
46	#include <linux/of.h>
47	#include <linux/of_fdt.h>
48	#include <linux/of_reserved_mem.h>
49	#include <linux/slab.h>
50	#endif
51
52	#define CREATE_TRACE_POINTS
53	#include <trace/events/swiotlb.h>
54
55	#define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
56
57	/*
58	* Minimum IO TLB size to bother booting with. Systems with mainly
59	* 64bit capable cards will only lightly use the swiotlb. If we can't
60	* allocate a contiguous 1MB, we're probably in trouble anyway.
61	*/
62	#define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
63
64	#define INVALID_PHYS_ADDR (~(phys_addr_t)0)
65
66	/**
67	* struct io_tlb_slot - IO TLB slot descriptor
68	* @orig_addr: The original address corresponding to a mapped entry.
69	* @alloc_size: Size of the allocated buffer.
70	* @list: The free list describing the number of free entries available
71	* from each index.
72	*/
73	struct io_tlb_slot {
74	phys_addr_t orig_addr;
75	size_t alloc_size;
76	unsigned int list;
77	};
78
79	static bool swiotlb_force_bounce;
80	static bool swiotlb_force_disable;
81
82	#ifdef CONFIG_SWIOTLB_DYNAMIC
83
84	static void swiotlb_dyn_alloc(struct work_struct *work);
85
86	static struct io_tlb_mem io_tlb_default_mem = {
87	.lock = __SPIN_LOCK_UNLOCKED(io_tlb_default_mem.lock),
88	.pools = LIST_HEAD_INIT(io_tlb_default_mem.pools),
89	.dyn_alloc = __WORK_INITIALIZER(io_tlb_default_mem.dyn_alloc,
90	swiotlb_dyn_alloc),
91	};
92
93	#else /* !CONFIG_SWIOTLB_DYNAMIC */
94
95	static struct io_tlb_mem io_tlb_default_mem;
96
97	#endif /* CONFIG_SWIOTLB_DYNAMIC */
98
99	static unsigned long default_nslabs = IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT;
100	static unsigned long default_nareas;
101
102	/**
103	* struct io_tlb_area - IO TLB memory area descriptor
104	*
105	* This is a single area with a single lock.
106	*
107	* @used: The number of used IO TLB block.
108	* @index: The slot index to start searching in this area for next round.
109	* @lock: The lock to protect the above data structures in the map and
110	* unmap calls.
111	*/
112	struct io_tlb_area {
113	unsigned long used;
114	unsigned int index;
115	spinlock_t lock;
116	};
117
118	/*
119	* Round up number of slabs to the next power of 2. The last area is going
120	* be smaller than the rest if default_nslabs is not power of two.
121	* The number of slot in an area should be a multiple of IO_TLB_SEGSIZE,
122	* otherwise a segment may span two or more areas. It conflicts with free
123	* contiguous slots tracking: free slots are treated contiguous no matter
124	* whether they cross an area boundary.
125	*
126	* Return true if default_nslabs is rounded up.
127	*/
128	static bool round_up_default_nslabs(void)
129	{
130	if (!default_nareas)
131	return false;
132
133	if (default_nslabs < IO_TLB_SEGSIZE * default_nareas)
134	default_nslabs = IO_TLB_SEGSIZE * default_nareas;
135	else if (is_power_of_2(n: default_nslabs))
136	return false;
137	default_nslabs = roundup_pow_of_two(default_nslabs);
138	return true;
139	}
140
141	/**
142	* swiotlb_adjust_nareas() - adjust the number of areas and slots
143	* @nareas: Desired number of areas. Zero is treated as 1.
144	*
145	* Adjust the default number of areas in a memory pool.
146	* The default size of the memory pool may also change to meet minimum area
147	* size requirements.
148	*/
149	static void swiotlb_adjust_nareas(unsigned int nareas)
150	{
151	if (!nareas)
152	nareas = 1;
153	else if (!is_power_of_2(n: nareas))
154	nareas = roundup_pow_of_two(nareas);
155
156	default_nareas = nareas;
157
158	pr_info("area num %d.\n", nareas);
159	if (round_up_default_nslabs())
160	pr_info("SWIOTLB bounce buffer size roundup to %luMB",
161	(default_nslabs << IO_TLB_SHIFT) >> 20);
162	}
163
164	/**
165	* limit_nareas() - get the maximum number of areas for a given memory pool size
166	* @nareas: Desired number of areas.
167	* @nslots: Total number of slots in the memory pool.
168	*
169	* Limit the number of areas to the maximum possible number of areas in
170	* a memory pool of the given size.
171	*
172	* Return: Maximum possible number of areas.
173	*/
174	static unsigned int limit_nareas(unsigned int nareas, unsigned long nslots)
175	{
176	if (nslots < nareas * IO_TLB_SEGSIZE)
177	return nslots / IO_TLB_SEGSIZE;
178	return nareas;
179	}
180
181	static int __init
182	setup_io_tlb_npages(char *str)
183	{
184	if (isdigit(c: *str)) {
185	/* avoid tail segment of size < IO_TLB_SEGSIZE */
186	default_nslabs =
187	ALIGN(simple_strtoul(str, &str, 0), IO_TLB_SEGSIZE);
188	}
189	if (*str == ',')
190	++str;
191	if (isdigit(c: *str))
192	swiotlb_adjust_nareas(nareas: simple_strtoul(str, &str, 0));
193	if (*str == ',')
194	++str;
195	if (!strcmp(str, "force"))
196	swiotlb_force_bounce = true;
197	else if (!strcmp(str, "noforce"))
198	swiotlb_force_disable = true;
199
200	return 0;
201	}
202	early_param("swiotlb", setup_io_tlb_npages);
203
204	unsigned long swiotlb_size_or_default(void)
205	{
206	return default_nslabs << IO_TLB_SHIFT;
207	}
208
209	void __init swiotlb_adjust_size(unsigned long size)
210	{
211	/*
212	* If swiotlb parameter has not been specified, give a chance to
213	* architectures such as those supporting memory encryption to
214	* adjust/expand SWIOTLB size for their use.
215	*/
216	if (default_nslabs != IO_TLB_DEFAULT_SIZE >> IO_TLB_SHIFT)
217	return;
218
219	size = ALIGN(size, IO_TLB_SIZE);
220	default_nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
221	if (round_up_default_nslabs())
222	size = default_nslabs << IO_TLB_SHIFT;
223	pr_info("SWIOTLB bounce buffer size adjusted to %luMB", size >> 20);
224	}
225
226	void swiotlb_print_info(void)
227	{
228	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
229
230	if (!mem->nslabs) {
231	pr_warn("No low mem\n");
232	return;
233	}
234
235	pr_info("mapped [mem %pa-%pa] (%luMB)\n", &mem->start, &mem->end,
236	(mem->nslabs << IO_TLB_SHIFT) >> 20);
237	}
238
239	static inline unsigned long io_tlb_offset(unsigned long val)
240	{
241	return val & (IO_TLB_SEGSIZE - 1);
242	}
243
244	static inline unsigned long nr_slots(u64 val)
245	{
246	return DIV_ROUND_UP(val, IO_TLB_SIZE);
247	}
248
249	/*
250	* Early SWIOTLB allocation may be too early to allow an architecture to
251	* perform the desired operations. This function allows the architecture to
252	* call SWIOTLB when the operations are possible. It needs to be called
253	* before the SWIOTLB memory is used.
254	*/
255	void __init swiotlb_update_mem_attributes(void)
256	{
257	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
258	unsigned long bytes;
259
260	if (!mem->nslabs || mem->late_alloc)
261	return;
262	bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT);
263	set_memory_decrypted(addr: (unsigned long)mem->vaddr, numpages: bytes >> PAGE_SHIFT);
264	}
265
266	static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start,
267	unsigned long nslabs, bool late_alloc, unsigned int nareas)
268	{
269	void *vaddr = phys_to_virt(address: start);
270	unsigned long bytes = nslabs << IO_TLB_SHIFT, i;
271
272	mem->nslabs = nslabs;
273	mem->start = start;
274	mem->end = mem->start + bytes;
275	mem->late_alloc = late_alloc;
276	mem->nareas = nareas;
277	mem->area_nslabs = nslabs / mem->nareas;
278
279	for (i = 0; i < mem->nareas; i++) {
280	spin_lock_init(&mem->areas[i].lock);
281	mem->areas[i].index = 0;
282	mem->areas[i].used = 0;
283	}
284
285	for (i = 0; i < mem->nslabs; i++) {
286	mem->slots[i].list = IO_TLB_SEGSIZE - io_tlb_offset(val: i);
287	mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
288	mem->slots[i].alloc_size = 0;
289	}
290
291	memset(vaddr, 0, bytes);
292	mem->vaddr = vaddr;
293	return;
294	}
295
296	/**
297	* add_mem_pool() - add a memory pool to the allocator
298	* @mem: Software IO TLB allocator.
299	* @pool: Memory pool to be added.
300	*/
301	static void add_mem_pool(struct io_tlb_mem *mem, struct io_tlb_pool *pool)
302	{
303	#ifdef CONFIG_SWIOTLB_DYNAMIC
304	spin_lock(lock: &mem->lock);
305	list_add_rcu(new: &pool->node, head: &mem->pools);
306	mem->nslabs += pool->nslabs;
307	spin_unlock(lock: &mem->lock);
308	#else
309	mem->nslabs = pool->nslabs;
310	#endif
311	}
312
313	static void __init *swiotlb_memblock_alloc(unsigned long nslabs,
314	unsigned int flags,
315	int (*remap)(void *tlb, unsigned long nslabs))
316	{
317	size_t bytes = PAGE_ALIGN(nslabs << IO_TLB_SHIFT);
318	void *tlb;
319
320	/*
321	* By default allocate the bounce buffer memory from low memory, but
322	* allow to pick a location everywhere for hypervisors with guest
323	* memory encryption.
324	*/
325	if (flags & SWIOTLB_ANY)
326	tlb = memblock_alloc(size: bytes, PAGE_SIZE);
327	else
328	tlb = memblock_alloc_low(size: bytes, PAGE_SIZE);
329
330	if (!tlb) {
331	pr_warn("%s: Failed to allocate %zu bytes tlb structure\n",
332	__func__, bytes);
333	return NULL;
334	}
335
336	if (remap && remap(tlb, nslabs) < 0) {
337	memblock_free(ptr: tlb, PAGE_ALIGN(bytes));
338	pr_warn("%s: Failed to remap %zu bytes\n", __func__, bytes);
339	return NULL;
340	}
341
342	return tlb;
343	}
344
345	/*
346	* Statically reserve bounce buffer space and initialize bounce buffer data
347	* structures for the software IO TLB used to implement the DMA API.
348	*/
349	void __init swiotlb_init_remap(bool addressing_limit, unsigned int flags,
350	int (*remap)(void *tlb, unsigned long nslabs))
351	{
352	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
353	unsigned long nslabs;
354	unsigned int nareas;
355	size_t alloc_size;
356	void *tlb;
357
358	if (!addressing_limit && !swiotlb_force_bounce)
359	return;
360	if (swiotlb_force_disable)
361	return;
362
363	io_tlb_default_mem.force_bounce =
364	swiotlb_force_bounce || (flags & SWIOTLB_FORCE);
365
366	#ifdef CONFIG_SWIOTLB_DYNAMIC
367	if (!remap)
368	io_tlb_default_mem.can_grow = true;
369	if (flags & SWIOTLB_ANY)
370	io_tlb_default_mem.phys_limit = virt_to_phys(address: high_memory - 1);
371	else
372	io_tlb_default_mem.phys_limit = ARCH_LOW_ADDRESS_LIMIT;
373	#endif
374
375	if (!default_nareas)
376	swiotlb_adjust_nareas(num_possible_cpus());
377
378	nslabs = default_nslabs;
379	nareas = limit_nareas(nareas: default_nareas, nslots: nslabs);
380	while ((tlb = swiotlb_memblock_alloc(nslabs, flags, remap)) == NULL) {
381	if (nslabs <= IO_TLB_MIN_SLABS)
382	return;
383	nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
384	nareas = limit_nareas(nareas, nslots: nslabs);
385	}
386
387	if (default_nslabs != nslabs) {
388	pr_info("SWIOTLB bounce buffer size adjusted %lu -> %lu slabs",
389	default_nslabs, nslabs);
390	default_nslabs = nslabs;
391	}
392
393	alloc_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), nslabs));
394	mem->slots = memblock_alloc(size: alloc_size, PAGE_SIZE);
395	if (!mem->slots) {
396	pr_warn("%s: Failed to allocate %zu bytes align=0x%lx\n",
397	__func__, alloc_size, PAGE_SIZE);
398	return;
399	}
400
401	mem->areas = memblock_alloc(array_size(sizeof(struct io_tlb_area),
402	nareas), SMP_CACHE_BYTES);
403	if (!mem->areas) {
404	pr_warn("%s: Failed to allocate mem->areas.\n", __func__);
405	return;
406	}
407
408	swiotlb_init_io_tlb_pool(mem, __pa(tlb), nslabs, late_alloc: false, nareas);
409	add_mem_pool(mem: &io_tlb_default_mem, pool: mem);
410
411	if (flags & SWIOTLB_VERBOSE)
412	swiotlb_print_info();
413	}
414
415	void __init swiotlb_init(bool addressing_limit, unsigned int flags)
416	{
417	swiotlb_init_remap(addressing_limit, flags, NULL);
418	}
419
420	/*
421	* Systems with larger DMA zones (those that don't support ISA) can
422	* initialize the swiotlb later using the slab allocator if needed.
423	* This should be just like above, but with some error catching.
424	*/
425	int swiotlb_init_late(size_t size, gfp_t gfp_mask,
426	int (*remap)(void *tlb, unsigned long nslabs))
427	{
428	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
429	unsigned long nslabs = ALIGN(size >> IO_TLB_SHIFT, IO_TLB_SEGSIZE);
430	unsigned int nareas;
431	unsigned char *vstart = NULL;
432	unsigned int order, area_order;
433	bool retried = false;
434	int rc = 0;
435
436	if (io_tlb_default_mem.nslabs)
437	return 0;
438
439	if (swiotlb_force_disable)
440	return 0;
441
442	io_tlb_default_mem.force_bounce = swiotlb_force_bounce;
443
444	#ifdef CONFIG_SWIOTLB_DYNAMIC
445	if (!remap)
446	io_tlb_default_mem.can_grow = true;
447	if (IS_ENABLED(CONFIG_ZONE_DMA) && (gfp_mask & __GFP_DMA))
448	io_tlb_default_mem.phys_limit = DMA_BIT_MASK(zone_dma_bits);
449	else if (IS_ENABLED(CONFIG_ZONE_DMA32) && (gfp_mask & __GFP_DMA32))
450	io_tlb_default_mem.phys_limit = DMA_BIT_MASK(32);
451	else
452	io_tlb_default_mem.phys_limit = virt_to_phys(address: high_memory - 1);
453	#endif
454
455	if (!default_nareas)
456	swiotlb_adjust_nareas(num_possible_cpus());
457
458	retry:
459	order = get_order(size: nslabs << IO_TLB_SHIFT);
460	nslabs = SLABS_PER_PAGE << order;
461
462	while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
463	vstart = (void *)__get_free_pages(gfp_mask: gfp_mask | __GFP_NOWARN,
464	order);
465	if (vstart)
466	break;
467	order--;
468	nslabs = SLABS_PER_PAGE << order;
469	retried = true;
470	}
471
472	if (!vstart)
473	return -ENOMEM;
474
475	if (remap)
476	rc = remap(vstart, nslabs);
477	if (rc) {
478	free_pages(addr: (unsigned long)vstart, order);
479
480	nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
481	if (nslabs < IO_TLB_MIN_SLABS)
482	return rc;
483	retried = true;
484	goto retry;
485	}
486
487	if (retried) {
488	pr_warn("only able to allocate %ld MB\n",
489	(PAGE_SIZE << order) >> 20);
490	}
491
492	nareas = limit_nareas(nareas: default_nareas, nslots: nslabs);
493	area_order = get_order(array_size(sizeof(*mem->areas), nareas));
494	mem->areas = (struct io_tlb_area *)
495	__get_free_pages(GFP_KERNEL | __GFP_ZERO, order: area_order);
496	if (!mem->areas)
497	goto error_area;
498
499	mem->slots = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
500	order: get_order(array_size(sizeof(*mem->slots), nslabs)));
501	if (!mem->slots)
502	goto error_slots;
503
504	set_memory_decrypted(addr: (unsigned long)vstart,
505	numpages: (nslabs << IO_TLB_SHIFT) >> PAGE_SHIFT);
506	swiotlb_init_io_tlb_pool(mem, virt_to_phys(address: vstart), nslabs, late_alloc: true,
507	nareas);
508	add_mem_pool(mem: &io_tlb_default_mem, pool: mem);
509
510	swiotlb_print_info();
511	return 0;
512
513	error_slots:
514	free_pages(addr: (unsigned long)mem->areas, order: area_order);
515	error_area:
516	free_pages(addr: (unsigned long)vstart, order);
517	return -ENOMEM;
518	}
519
520	void __init swiotlb_exit(void)
521	{
522	struct io_tlb_pool *mem = &io_tlb_default_mem.defpool;
523	unsigned long tbl_vaddr;
524	size_t tbl_size, slots_size;
525	unsigned int area_order;
526
527	if (swiotlb_force_bounce)
528	return;
529
530	if (!mem->nslabs)
531	return;
532
533	pr_info("tearing down default memory pool\n");
534	tbl_vaddr = (unsigned long)phys_to_virt(address: mem->start);
535	tbl_size = PAGE_ALIGN(mem->end - mem->start);
536	slots_size = PAGE_ALIGN(array_size(sizeof(*mem->slots), mem->nslabs));
537
538	set_memory_encrypted(addr: tbl_vaddr, numpages: tbl_size >> PAGE_SHIFT);
539	if (mem->late_alloc) {
540	area_order = get_order(array_size(sizeof(*mem->areas),
541	mem->nareas));
542	free_pages(addr: (unsigned long)mem->areas, order: area_order);
543	free_pages(addr: tbl_vaddr, order: get_order(size: tbl_size));
544	free_pages(addr: (unsigned long)mem->slots, order: get_order(size: slots_size));
545	} else {
546	memblock_free_late(__pa(mem->areas),
547	array_size(sizeof(*mem->areas), mem->nareas));
548	memblock_free_late(base: mem->start, size: tbl_size);
549	memblock_free_late(__pa(mem->slots), size: slots_size);
550	}
551
552	memset(mem, 0, sizeof(*mem));
553	}
554
555	#ifdef CONFIG_SWIOTLB_DYNAMIC
556
557	/**
558	* alloc_dma_pages() - allocate pages to be used for DMA
559	* @gfp: GFP flags for the allocation.
560	* @bytes: Size of the buffer.
561	*
562	* Allocate pages from the buddy allocator. If successful, make the allocated
563	* pages decrypted that they can be used for DMA.
564	*
565	* Return: Decrypted pages, or %NULL on failure.
566	*/
567	static struct page *alloc_dma_pages(gfp_t gfp, size_t bytes)
568	{
569	unsigned int order = get_order(size: bytes);
570	struct page *page;
571	void *vaddr;
572
573	page = alloc_pages(gfp, order);
574	if (!page)
575	return NULL;
576
577	vaddr = page_address(page);
578	if (set_memory_decrypted(addr: (unsigned long)vaddr, PFN_UP(bytes)))
579	goto error;
580	return page;
581
582	error:
583	__free_pages(page, order);
584	return NULL;
585	}
586
587	/**
588	* swiotlb_alloc_tlb() - allocate a dynamic IO TLB buffer
589	* @dev: Device for which a memory pool is allocated.
590	* @bytes: Size of the buffer.
591	* @phys_limit: Maximum allowed physical address of the buffer.
592	* @gfp: GFP flags for the allocation.
593	*
594	* Return: Allocated pages, or %NULL on allocation failure.
595	*/
596	static struct page *swiotlb_alloc_tlb(struct device *dev, size_t bytes,
597	u64 phys_limit, gfp_t gfp)
598	{
599	struct page *page;
600
601	/*
602	* Allocate from the atomic pools if memory is encrypted and
603	* the allocation is atomic, because decrypting may block.
604	*/
605	if (!gfpflags_allow_blocking(gfp_flags: gfp) && dev && force_dma_unencrypted(dev)) {
606	void *vaddr;
607
608	if (!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))
609	return NULL;
610
611	return dma_alloc_from_pool(dev, size: bytes, cpu_addr: &vaddr, flags: gfp,
612	phys_addr_ok: dma_coherent_ok);
613	}
614
615	gfp &= ~GFP_ZONEMASK;
616	if (phys_limit <= DMA_BIT_MASK(zone_dma_bits))
617	gfp |= __GFP_DMA;
618	else if (phys_limit <= DMA_BIT_MASK(32))
619	gfp |= __GFP_DMA32;
620
621	while ((page = alloc_dma_pages(gfp, bytes)) &&
622	page_to_phys(page) + bytes - 1 > phys_limit) {
623	/* allocated, but too high */
624	__free_pages(page, order: get_order(size: bytes));
625
626	if (IS_ENABLED(CONFIG_ZONE_DMA32) &&
627	phys_limit < DMA_BIT_MASK(64) &&
628	!(gfp & (__GFP_DMA32 | __GFP_DMA)))
629	gfp |= __GFP_DMA32;
630	else if (IS_ENABLED(CONFIG_ZONE_DMA) &&
631	!(gfp & __GFP_DMA))
632	gfp = (gfp & ~__GFP_DMA32) | __GFP_DMA;
633	else
634	return NULL;
635	}
636
637	return page;
638	}
639
640	/**
641	* swiotlb_free_tlb() - free a dynamically allocated IO TLB buffer
642	* @vaddr: Virtual address of the buffer.
643	* @bytes: Size of the buffer.
644	*/
645	static void swiotlb_free_tlb(void *vaddr, size_t bytes)
646	{
647	if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) &&
648	dma_free_from_pool(NULL, start: vaddr, size: bytes))
649	return;
650
651	/* Intentional leak if pages cannot be encrypted again. */
652	if (!set_memory_encrypted(addr: (unsigned long)vaddr, PFN_UP(bytes)))
653	__free_pages(virt_to_page(vaddr), order: get_order(size: bytes));
654	}
655
656	/**
657	* swiotlb_alloc_pool() - allocate a new IO TLB memory pool
658	* @dev: Device for which a memory pool is allocated.
659	* @minslabs: Minimum number of slabs.
660	* @nslabs: Desired (maximum) number of slabs.
661	* @nareas: Number of areas.
662	* @phys_limit: Maximum DMA buffer physical address.
663	* @gfp: GFP flags for the allocations.
664	*
665	* Allocate and initialize a new IO TLB memory pool. The actual number of
666	* slabs may be reduced if allocation of @nslabs fails. If even
667	* @minslabs cannot be allocated, this function fails.
668	*
669	* Return: New memory pool, or %NULL on allocation failure.
670	*/
671	static struct io_tlb_pool *swiotlb_alloc_pool(struct device *dev,
672	unsigned long minslabs, unsigned long nslabs,
673	unsigned int nareas, u64 phys_limit, gfp_t gfp)
674	{
675	struct io_tlb_pool *pool;
676	unsigned int slot_order;
677	struct page *tlb;
678	size_t pool_size;
679	size_t tlb_size;
680
681	if (nslabs > SLABS_PER_PAGE << MAX_ORDER) {
682	nslabs = SLABS_PER_PAGE << MAX_ORDER;
683	nareas = limit_nareas(nareas, nslots: nslabs);
684	}
685
686	pool_size = sizeof(*pool) + array_size(sizeof(*pool->areas), nareas);
687	pool = kzalloc(size: pool_size, flags: gfp);
688	if (!pool)
689	goto error;
690	pool->areas = (void *)pool + sizeof(*pool);
691
692	tlb_size = nslabs << IO_TLB_SHIFT;
693	while (!(tlb = swiotlb_alloc_tlb(dev, bytes: tlb_size, phys_limit, gfp))) {
694	if (nslabs <= minslabs)
695	goto error_tlb;
696	nslabs = ALIGN(nslabs >> 1, IO_TLB_SEGSIZE);
697	nareas = limit_nareas(nareas, nslots: nslabs);
698	tlb_size = nslabs << IO_TLB_SHIFT;
699	}
700
701	slot_order = get_order(array_size(sizeof(*pool->slots), nslabs));
702	pool->slots = (struct io_tlb_slot *)
703	__get_free_pages(gfp_mask: gfp, order: slot_order);
704	if (!pool->slots)
705	goto error_slots;
706
707	swiotlb_init_io_tlb_pool(mem: pool, page_to_phys(tlb), nslabs, late_alloc: true, nareas);
708	return pool;
709
710	error_slots:
711	swiotlb_free_tlb(page_address(tlb), bytes: tlb_size);
712	error_tlb:
713	kfree(objp: pool);
714	error:
715	return NULL;
716	}
717
718	/**
719	* swiotlb_dyn_alloc() - dynamic memory pool allocation worker
720	* @work: Pointer to dyn_alloc in struct io_tlb_mem.
721	*/
722	static void swiotlb_dyn_alloc(struct work_struct *work)
723	{
724	struct io_tlb_mem *mem =
725	container_of(work, struct io_tlb_mem, dyn_alloc);
726	struct io_tlb_pool *pool;
727
728	pool = swiotlb_alloc_pool(NULL, IO_TLB_MIN_SLABS, nslabs: default_nslabs,
729	nareas: default_nareas, phys_limit: mem->phys_limit, GFP_KERNEL);
730	if (!pool) {
731	pr_warn_ratelimited("Failed to allocate new pool");
732	return;
733	}
734
735	add_mem_pool(mem, pool);
736	}
737
738	/**
739	* swiotlb_dyn_free() - RCU callback to free a memory pool
740	* @rcu: RCU head in the corresponding struct io_tlb_pool.
741	*/
742	static void swiotlb_dyn_free(struct rcu_head *rcu)
743	{
744	struct io_tlb_pool *pool = container_of(rcu, struct io_tlb_pool, rcu);
745	size_t slots_size = array_size(sizeof(*pool->slots), pool->nslabs);
746	size_t tlb_size = pool->end - pool->start;
747
748	free_pages(addr: (unsigned long)pool->slots, order: get_order(size: slots_size));
749	swiotlb_free_tlb(vaddr: pool->vaddr, bytes: tlb_size);
750	kfree(objp: pool);
751	}
752
753	/**
754	* swiotlb_find_pool() - find the IO TLB pool for a physical address
755	* @dev: Device which has mapped the DMA buffer.
756	* @paddr: Physical address within the DMA buffer.
757	*
758	* Find the IO TLB memory pool descriptor which contains the given physical
759	* address, if any.
760	*
761	* Return: Memory pool which contains @paddr, or %NULL if none.
762	*/
763	struct io_tlb_pool *swiotlb_find_pool(struct device *dev, phys_addr_t paddr)
764	{
765	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
766	struct io_tlb_pool *pool;
767
768	rcu_read_lock();
769	list_for_each_entry_rcu(pool, &mem->pools, node) {
770	if (paddr >= pool->start && paddr < pool->end)
771	goto out;
772	}
773
774	list_for_each_entry_rcu(pool, &dev->dma_io_tlb_pools, node) {
775	if (paddr >= pool->start && paddr < pool->end)
776	goto out;
777	}
778	pool = NULL;
779	out:
780	rcu_read_unlock();
781	return pool;
782	}
783
784	/**
785	* swiotlb_del_pool() - remove an IO TLB pool from a device
786	* @dev: Owning device.
787	* @pool: Memory pool to be removed.
788	*/
789	static void swiotlb_del_pool(struct device *dev, struct io_tlb_pool *pool)
790	{
791	unsigned long flags;
792
793	spin_lock_irqsave(&dev->dma_io_tlb_lock, flags);
794	list_del_rcu(entry: &pool->node);
795	spin_unlock_irqrestore(lock: &dev->dma_io_tlb_lock, flags);
796
797	call_rcu(head: &pool->rcu, func: swiotlb_dyn_free);
798	}
799
800	#endif /* CONFIG_SWIOTLB_DYNAMIC */
801
802	/**
803	* swiotlb_dev_init() - initialize swiotlb fields in &struct device
804	* @dev: Device to be initialized.
805	*/
806	void swiotlb_dev_init(struct device *dev)
807	{
808	dev->dma_io_tlb_mem = &io_tlb_default_mem;
809	#ifdef CONFIG_SWIOTLB_DYNAMIC
810	INIT_LIST_HEAD(list: &dev->dma_io_tlb_pools);
811	spin_lock_init(&dev->dma_io_tlb_lock);
812	dev->dma_uses_io_tlb = false;
813	#endif
814	}
815
816	/*
817	* Return the offset into a iotlb slot required to keep the device happy.
818	*/
819	static unsigned int swiotlb_align_offset(struct device *dev, u64 addr)
820	{
821	return addr & dma_get_min_align_mask(dev) & (IO_TLB_SIZE - 1);
822	}
823
824	/*
825	* Bounce: copy the swiotlb buffer from or back to the original dma location
826	*/
827	static void swiotlb_bounce(struct device *dev, phys_addr_t tlb_addr, size_t size,
828	enum dma_data_direction dir)
829	{
830	struct io_tlb_pool *mem = swiotlb_find_pool(dev, paddr: tlb_addr);
831	int index = (tlb_addr - mem->start) >> IO_TLB_SHIFT;
832	phys_addr_t orig_addr = mem->slots[index].orig_addr;
833	size_t alloc_size = mem->slots[index].alloc_size;
834	unsigned long pfn = PFN_DOWN(orig_addr);
835	unsigned char *vaddr = mem->vaddr + tlb_addr - mem->start;
836	unsigned int tlb_offset, orig_addr_offset;
837
838	if (orig_addr == INVALID_PHYS_ADDR)
839	return;
840
841	tlb_offset = tlb_addr & (IO_TLB_SIZE - 1);
842	orig_addr_offset = swiotlb_align_offset(dev, addr: orig_addr);
843	if (tlb_offset < orig_addr_offset) {
844	dev_WARN_ONCE(dev, 1,
845	"Access before mapping start detected. orig offset %u, requested offset %u.\n",
846	orig_addr_offset, tlb_offset);
847	return;
848	}
849
850	tlb_offset -= orig_addr_offset;
851	if (tlb_offset > alloc_size) {
852	dev_WARN_ONCE(dev, 1,
853	"Buffer overflow detected. Allocation size: %zu. Mapping size: %zu+%u.\n",
854	alloc_size, size, tlb_offset);
855	return;
856	}
857
858	orig_addr += tlb_offset;
859	alloc_size -= tlb_offset;
860
861	if (size > alloc_size) {
862	dev_WARN_ONCE(dev, 1,
863	"Buffer overflow detected. Allocation size: %zu. Mapping size: %zu.\n",
864	alloc_size, size);
865	size = alloc_size;
866	}
867
868	if (PageHighMem(pfn_to_page(pfn))) {
869	unsigned int offset = orig_addr & ~PAGE_MASK;
870	struct page *page;
871	unsigned int sz = 0;
872	unsigned long flags;
873
874	while (size) {
875	sz = min_t(size_t, PAGE_SIZE - offset, size);
876
877	local_irq_save(flags);
878	page = pfn_to_page(pfn);
879	if (dir == DMA_TO_DEVICE)
880	memcpy_from_page(to: vaddr, page, offset, len: sz);
881	else
882	memcpy_to_page(page, offset, from: vaddr, len: sz);
883	local_irq_restore(flags);
884
885	size -= sz;
886	pfn++;
887	vaddr += sz;
888	offset = 0;
889	}
890	} else if (dir == DMA_TO_DEVICE) {
891	memcpy(vaddr, phys_to_virt(orig_addr), size);
892	} else {
893	memcpy(phys_to_virt(orig_addr), vaddr, size);
894	}
895	}
896
897	static inline phys_addr_t slot_addr(phys_addr_t start, phys_addr_t idx)
898	{
899	return start + (idx << IO_TLB_SHIFT);
900	}
901
902	/*
903	* Carefully handle integer overflow which can occur when boundary_mask == ~0UL.
904	*/
905	static inline unsigned long get_max_slots(unsigned long boundary_mask)
906	{
907	return (boundary_mask >> IO_TLB_SHIFT) + 1;
908	}
909
910	static unsigned int wrap_area_index(struct io_tlb_pool *mem, unsigned int index)
911	{
912	if (index >= mem->area_nslabs)
913	return 0;
914	return index;
915	}
916
917	/*
918	* Track the total used slots with a global atomic value in order to have
919	* correct information to determine the high water mark. The mem_used()
920	* function gives imprecise results because there's no locking across
921	* multiple areas.
922	*/
923	#ifdef CONFIG_DEBUG_FS
924	static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots)
925	{
926	unsigned long old_hiwater, new_used;
927
928	new_used = atomic_long_add_return(i: nslots, v: &mem->total_used);
929	old_hiwater = atomic_long_read(v: &mem->used_hiwater);
930	do {
931	if (new_used <= old_hiwater)
932	break;
933	} while (!atomic_long_try_cmpxchg(v: &mem->used_hiwater,
934	old: &old_hiwater, new: new_used));
935	}
936
937	static void dec_used(struct io_tlb_mem *mem, unsigned int nslots)
938	{
939	atomic_long_sub(i: nslots, v: &mem->total_used);
940	}
941
942	#else /* !CONFIG_DEBUG_FS */
943	static void inc_used_and_hiwater(struct io_tlb_mem *mem, unsigned int nslots)
944	{
945	}
946	static void dec_used(struct io_tlb_mem *mem, unsigned int nslots)
947	{
948	}
949	#endif /* CONFIG_DEBUG_FS */
950
951	/**
952	* swiotlb_area_find_slots() - search for slots in one IO TLB memory area
953	* @dev: Device which maps the buffer.
954	* @pool: Memory pool to be searched.
955	* @area_index: Index of the IO TLB memory area to be searched.
956	* @orig_addr: Original (non-bounced) IO buffer address.
957	* @alloc_size: Total requested size of the bounce buffer,
958	* including initial alignment padding.
959	* @alloc_align_mask: Required alignment of the allocated buffer.
960	*
961	* Find a suitable sequence of IO TLB entries for the request and allocate
962	* a buffer from the given IO TLB memory area.
963	* This function takes care of locking.
964	*
965	* Return: Index of the first allocated slot, or -1 on error.
966	*/
967	static int swiotlb_area_find_slots(struct device *dev, struct io_tlb_pool *pool,
968	int area_index, phys_addr_t orig_addr, size_t alloc_size,
969	unsigned int alloc_align_mask)
970	{
971	struct io_tlb_area *area = pool->areas + area_index;
972	unsigned long boundary_mask = dma_get_seg_boundary(dev);
973	dma_addr_t tbl_dma_addr =
974	phys_to_dma_unencrypted(dev, paddr: pool->start) & boundary_mask;
975	unsigned long max_slots = get_max_slots(boundary_mask);
976	unsigned int iotlb_align_mask =
977	dma_get_min_align_mask(dev) | alloc_align_mask;
978	unsigned int nslots = nr_slots(val: alloc_size), stride;
979	unsigned int offset = swiotlb_align_offset(dev, addr: orig_addr);
980	unsigned int index, slots_checked, count = 0, i;
981	unsigned long flags;
982	unsigned int slot_base;
983	unsigned int slot_index;
984
985	BUG_ON(!nslots);
986	BUG_ON(area_index >= pool->nareas);
987
988	/*
989	* For allocations of PAGE_SIZE or larger only look for page aligned
990	* allocations.
991	*/
992	if (alloc_size >= PAGE_SIZE)
993	iotlb_align_mask |= ~PAGE_MASK;
994	iotlb_align_mask &= ~(IO_TLB_SIZE - 1);
995
996	/*
997	* For mappings with an alignment requirement don't bother looping to
998	* unaligned slots once we found an aligned one.
999	*/
1000	stride = (iotlb_align_mask >> IO_TLB_SHIFT) + 1;
1001
1002	spin_lock_irqsave(&area->lock, flags);
1003	if (unlikely(nslots > pool->area_nslabs - area->used))
1004	goto not_found;
1005
1006	slot_base = area_index * pool->area_nslabs;
1007	index = area->index;
1008
1009	for (slots_checked = 0; slots_checked < pool->area_nslabs; ) {
1010	slot_index = slot_base + index;
1011
1012	if (orig_addr &&
1013	(slot_addr(start: tbl_dma_addr, idx: slot_index) &
1014	iotlb_align_mask) != (orig_addr & iotlb_align_mask)) {
1015	index = wrap_area_index(mem: pool, index: index + 1);
1016	slots_checked++;
1017	continue;
1018	}
1019
1020	if (!iommu_is_span_boundary(index: slot_index, nr: nslots,
1021	shift: nr_slots(val: tbl_dma_addr),
1022	boundary_size: max_slots)) {
1023	if (pool->slots[slot_index].list >= nslots)
1024	goto found;
1025	}
1026	index = wrap_area_index(mem: pool, index: index + stride);
1027	slots_checked += stride;
1028	}
1029
1030	not_found:
1031	spin_unlock_irqrestore(lock: &area->lock, flags);
1032	return -1;
1033
1034	found:
1035	/*
1036	* If we find a slot that indicates we have 'nslots' number of
1037	* contiguous buffers, we allocate the buffers from that slot onwards
1038	* and set the list of free entries to '0' indicating unavailable.
1039	*/
1040	for (i = slot_index; i < slot_index + nslots; i++) {
1041	pool->slots[i].list = 0;
1042	pool->slots[i].alloc_size = alloc_size - (offset +
1043	((i - slot_index) << IO_TLB_SHIFT));
1044	}
1045	for (i = slot_index - 1;
1046	io_tlb_offset(val: i) != IO_TLB_SEGSIZE - 1 &&
1047	pool->slots[i].list; i--)
1048	pool->slots[i].list = ++count;
1049
1050	/*
1051	* Update the indices to avoid searching in the next round.
1052	*/
1053	area->index = wrap_area_index(mem: pool, index: index + nslots);
1054	area->used += nslots;
1055	spin_unlock_irqrestore(lock: &area->lock, flags);
1056
1057	inc_used_and_hiwater(mem: dev->dma_io_tlb_mem, nslots);
1058	return slot_index;
1059	}
1060
1061	/**
1062	* swiotlb_pool_find_slots() - search for slots in one memory pool
1063	* @dev: Device which maps the buffer.
1064	* @pool: Memory pool to be searched.
1065	* @orig_addr: Original (non-bounced) IO buffer address.
1066	* @alloc_size: Total requested size of the bounce buffer,
1067	* including initial alignment padding.
1068	* @alloc_align_mask: Required alignment of the allocated buffer.
1069	*
1070	* Search through one memory pool to find a sequence of slots that match the
1071	* allocation constraints.
1072	*
1073	* Return: Index of the first allocated slot, or -1 on error.
1074	*/
1075	static int swiotlb_pool_find_slots(struct device *dev, struct io_tlb_pool *pool,
1076	phys_addr_t orig_addr, size_t alloc_size,
1077	unsigned int alloc_align_mask)
1078	{
1079	int start = raw_smp_processor_id() & (pool->nareas - 1);
1080	int i = start, index;
1081
1082	do {
1083	index = swiotlb_area_find_slots(dev, pool, area_index: i, orig_addr,
1084	alloc_size, alloc_align_mask);
1085	if (index >= 0)
1086	return index;
1087	if (++i >= pool->nareas)
1088	i = 0;
1089	} while (i != start);
1090
1091	return -1;
1092	}
1093
1094	#ifdef CONFIG_SWIOTLB_DYNAMIC
1095
1096	/**
1097	* swiotlb_find_slots() - search for slots in the whole swiotlb
1098	* @dev: Device which maps the buffer.
1099	* @orig_addr: Original (non-bounced) IO buffer address.
1100	* @alloc_size: Total requested size of the bounce buffer,
1101	* including initial alignment padding.
1102	* @alloc_align_mask: Required alignment of the allocated buffer.
1103	* @retpool: Used memory pool, updated on return.
1104	*
1105	* Search through the whole software IO TLB to find a sequence of slots that
1106	* match the allocation constraints.
1107	*
1108	* Return: Index of the first allocated slot, or -1 on error.
1109	*/
1110	static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
1111	size_t alloc_size, unsigned int alloc_align_mask,
1112	struct io_tlb_pool **retpool)
1113	{
1114	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1115	struct io_tlb_pool *pool;
1116	unsigned long nslabs;
1117	unsigned long flags;
1118	u64 phys_limit;
1119	int index;
1120
1121	rcu_read_lock();
1122	list_for_each_entry_rcu(pool, &mem->pools, node) {
1123	index = swiotlb_pool_find_slots(dev, pool, orig_addr,
1124	alloc_size, alloc_align_mask);
1125	if (index >= 0) {
1126	rcu_read_unlock();
1127	goto found;
1128	}
1129	}
1130	rcu_read_unlock();
1131	if (!mem->can_grow)
1132	return -1;
1133
1134	schedule_work(work: &mem->dyn_alloc);
1135
1136	nslabs = nr_slots(val: alloc_size);
1137	phys_limit = min_not_zero(*dev->dma_mask, dev->bus_dma_limit);
1138	pool = swiotlb_alloc_pool(dev, minslabs: nslabs, nslabs, nareas: 1, phys_limit,
1139	GFP_NOWAIT | __GFP_NOWARN);
1140	if (!pool)
1141	return -1;
1142
1143	index = swiotlb_pool_find_slots(dev, pool, orig_addr,
1144	alloc_size, alloc_align_mask);
1145	if (index < 0) {
1146	swiotlb_dyn_free(rcu: &pool->rcu);
1147	return -1;
1148	}
1149
1150	pool->transient = true;
1151	spin_lock_irqsave(&dev->dma_io_tlb_lock, flags);
1152	list_add_rcu(new: &pool->node, head: &dev->dma_io_tlb_pools);
1153	spin_unlock_irqrestore(lock: &dev->dma_io_tlb_lock, flags);
1154
1155	found:
1156	WRITE_ONCE(dev->dma_uses_io_tlb, true);
1157
1158	/*
1159	* The general barrier orders reads and writes against a presumed store
1160	* of the SWIOTLB buffer address by a device driver (to a driver private
1161	* data structure). It serves two purposes.
1162	*
1163	* First, the store to dev->dma_uses_io_tlb must be ordered before the
1164	* presumed store. This guarantees that the returned buffer address
1165	* cannot be passed to another CPU before updating dev->dma_uses_io_tlb.
1166	*
1167	* Second, the load from mem->pools must be ordered before the same
1168	* presumed store. This guarantees that the returned buffer address
1169	* cannot be observed by another CPU before an update of the RCU list
1170	* that was made by swiotlb_dyn_alloc() on a third CPU (cf. multicopy
1171	* atomicity).
1172	*
1173	* See also the comment in is_swiotlb_buffer().
1174	*/
1175	smp_mb();
1176
1177	*retpool = pool;
1178	return index;
1179	}
1180
1181	#else /* !CONFIG_SWIOTLB_DYNAMIC */
1182
1183	static int swiotlb_find_slots(struct device *dev, phys_addr_t orig_addr,
1184	size_t alloc_size, unsigned int alloc_align_mask,
1185	struct io_tlb_pool **retpool)
1186	{
1187	*retpool = &dev->dma_io_tlb_mem->defpool;
1188	return swiotlb_pool_find_slots(dev, *retpool,
1189	orig_addr, alloc_size, alloc_align_mask);
1190	}
1191
1192	#endif /* CONFIG_SWIOTLB_DYNAMIC */
1193
1194	#ifdef CONFIG_DEBUG_FS
1195
1196	/**
1197	* mem_used() - get number of used slots in an allocator
1198	* @mem: Software IO TLB allocator.
1199	*
1200	* The result is accurate in this version of the function, because an atomic
1201	* counter is available if CONFIG_DEBUG_FS is set.
1202	*
1203	* Return: Number of used slots.
1204	*/
1205	static unsigned long mem_used(struct io_tlb_mem *mem)
1206	{
1207	return atomic_long_read(v: &mem->total_used);
1208	}
1209
1210	#else /* !CONFIG_DEBUG_FS */
1211
1212	/**
1213	* mem_pool_used() - get number of used slots in a memory pool
1214	* @pool: Software IO TLB memory pool.
1215	*
1216	* The result is not accurate, see mem_used().
1217	*
1218	* Return: Approximate number of used slots.
1219	*/
1220	static unsigned long mem_pool_used(struct io_tlb_pool *pool)
1221	{
1222	int i;
1223	unsigned long used = 0;
1224
1225	for (i = 0; i < pool->nareas; i++)
1226	used += pool->areas[i].used;
1227	return used;
1228	}
1229
1230	/**
1231	* mem_used() - get number of used slots in an allocator
1232	* @mem: Software IO TLB allocator.
1233	*
1234	* The result is not accurate, because there is no locking of individual
1235	* areas.
1236	*
1237	* Return: Approximate number of used slots.
1238	*/
1239	static unsigned long mem_used(struct io_tlb_mem *mem)
1240	{
1241	#ifdef CONFIG_SWIOTLB_DYNAMIC
1242	struct io_tlb_pool *pool;
1243	unsigned long used = 0;
1244
1245	rcu_read_lock();
1246	list_for_each_entry_rcu(pool, &mem->pools, node)
1247	used += mem_pool_used(pool);
1248	rcu_read_unlock();
1249
1250	return used;
1251	#else
1252	return mem_pool_used(&mem->defpool);
1253	#endif
1254	}
1255
1256	#endif /* CONFIG_DEBUG_FS */
1257
1258	phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
1259	size_t mapping_size, size_t alloc_size,
1260	unsigned int alloc_align_mask, enum dma_data_direction dir,
1261	unsigned long attrs)
1262	{
1263	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1264	unsigned int offset = swiotlb_align_offset(dev, addr: orig_addr);
1265	struct io_tlb_pool *pool;
1266	unsigned int i;
1267	int index;
1268	phys_addr_t tlb_addr;
1269
1270	if (!mem || !mem->nslabs) {
1271	dev_warn_ratelimited(dev,
1272	"Can not allocate SWIOTLB buffer earlier and can't now provide you with the DMA bounce buffer");
1273	return (phys_addr_t)DMA_MAPPING_ERROR;
1274	}
1275
1276	if (cc_platform_has(attr: CC_ATTR_MEM_ENCRYPT))
1277	pr_warn_once("Memory encryption is active and system is using DMA bounce buffers\n");
1278
1279	if (mapping_size > alloc_size) {
1280	dev_warn_once(dev, "Invalid sizes (mapping: %zd bytes, alloc: %zd bytes)",
1281	mapping_size, alloc_size);
1282	return (phys_addr_t)DMA_MAPPING_ERROR;
1283	}
1284
1285	index = swiotlb_find_slots(dev, orig_addr,
1286	alloc_size: alloc_size + offset, alloc_align_mask, retpool: &pool);
1287	if (index == -1) {
1288	if (!(attrs & DMA_ATTR_NO_WARN))
1289	dev_warn_ratelimited(dev,
1290	"swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n",
1291	alloc_size, mem->nslabs, mem_used(mem));
1292	return (phys_addr_t)DMA_MAPPING_ERROR;
1293	}
1294
1295	/*
1296	* Save away the mapping from the original address to the DMA address.
1297	* This is needed when we sync the memory. Then we sync the buffer if
1298	* needed.
1299	*/
1300	for (i = 0; i < nr_slots(val: alloc_size + offset); i++)
1301	pool->slots[index + i].orig_addr = slot_addr(start: orig_addr, idx: i);
1302	tlb_addr = slot_addr(start: pool->start, idx: index) + offset;
1303	/*
1304	* When the device is writing memory, i.e. dir == DMA_FROM_DEVICE, copy
1305	* the original buffer to the TLB buffer before initiating DMA in order
1306	* to preserve the original's data if the device does a partial write,
1307	* i.e. if the device doesn't overwrite the entire buffer. Preserving
1308	* the original data, even if it's garbage, is necessary to match
1309	* hardware behavior. Use of swiotlb is supposed to be transparent,
1310	* i.e. swiotlb must not corrupt memory by clobbering unwritten bytes.
1311	*/
1312	swiotlb_bounce(dev, tlb_addr, size: mapping_size, dir: DMA_TO_DEVICE);
1313	return tlb_addr;
1314	}
1315
1316	static void swiotlb_release_slots(struct device *dev, phys_addr_t tlb_addr)
1317	{
1318	struct io_tlb_pool *mem = swiotlb_find_pool(dev, paddr: tlb_addr);
1319	unsigned long flags;
1320	unsigned int offset = swiotlb_align_offset(dev, addr: tlb_addr);
1321	int index = (tlb_addr - offset - mem->start) >> IO_TLB_SHIFT;
1322	int nslots = nr_slots(val: mem->slots[index].alloc_size + offset);
1323	int aindex = index / mem->area_nslabs;
1324	struct io_tlb_area *area = &mem->areas[aindex];
1325	int count, i;
1326
1327	/*
1328	* Return the buffer to the free list by setting the corresponding
1329	* entries to indicate the number of contiguous entries available.
1330	* While returning the entries to the free list, we merge the entries
1331	* with slots below and above the pool being returned.
1332	*/
1333	BUG_ON(aindex >= mem->nareas);
1334
1335	spin_lock_irqsave(&area->lock, flags);
1336	if (index + nslots < ALIGN(index + 1, IO_TLB_SEGSIZE))
1337	count = mem->slots[index + nslots].list;
1338	else
1339	count = 0;
1340
1341	/*
1342	* Step 1: return the slots to the free list, merging the slots with
1343	* superceeding slots
1344	*/
1345	for (i = index + nslots - 1; i >= index; i--) {
1346	mem->slots[i].list = ++count;
1347	mem->slots[i].orig_addr = INVALID_PHYS_ADDR;
1348	mem->slots[i].alloc_size = 0;
1349	}
1350
1351	/*
1352	* Step 2: merge the returned slots with the preceding slots, if
1353	* available (non zero)
1354	*/
1355	for (i = index - 1;
1356	io_tlb_offset(val: i) != IO_TLB_SEGSIZE - 1 && mem->slots[i].list;
1357	i--)
1358	mem->slots[i].list = ++count;
1359	area->used -= nslots;
1360	spin_unlock_irqrestore(lock: &area->lock, flags);
1361
1362	dec_used(mem: dev->dma_io_tlb_mem, nslots);
1363	}
1364
1365	#ifdef CONFIG_SWIOTLB_DYNAMIC
1366
1367	/**
1368	* swiotlb_del_transient() - delete a transient memory pool
1369	* @dev: Device which mapped the buffer.
1370	* @tlb_addr: Physical address within a bounce buffer.
1371	*
1372	* Check whether the address belongs to a transient SWIOTLB memory pool.
1373	* If yes, then delete the pool.
1374	*
1375	* Return: %true if @tlb_addr belonged to a transient pool that was released.
1376	*/
1377	static bool swiotlb_del_transient(struct device *dev, phys_addr_t tlb_addr)
1378	{
1379	struct io_tlb_pool *pool;
1380
1381	pool = swiotlb_find_pool(dev, paddr: tlb_addr);
1382	if (!pool->transient)
1383	return false;
1384
1385	dec_used(mem: dev->dma_io_tlb_mem, nslots: pool->nslabs);
1386	swiotlb_del_pool(dev, pool);
1387	return true;
1388	}
1389
1390	#else /* !CONFIG_SWIOTLB_DYNAMIC */
1391
1392	static inline bool swiotlb_del_transient(struct device *dev,
1393	phys_addr_t tlb_addr)
1394	{
1395	return false;
1396	}
1397
1398	#endif /* CONFIG_SWIOTLB_DYNAMIC */
1399
1400	/*
1401	* tlb_addr is the physical address of the bounce buffer to unmap.
1402	*/
1403	void swiotlb_tbl_unmap_single(struct device *dev, phys_addr_t tlb_addr,
1404	size_t mapping_size, enum dma_data_direction dir,
1405	unsigned long attrs)
1406	{
1407	/*
1408	* First, sync the memory before unmapping the entry
1409	*/
1410	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
1411	(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
1412	swiotlb_bounce(dev, tlb_addr, size: mapping_size, dir: DMA_FROM_DEVICE);
1413
1414	if (swiotlb_del_transient(dev, tlb_addr))
1415	return;
1416	swiotlb_release_slots(dev, tlb_addr);
1417	}
1418
1419	void swiotlb_sync_single_for_device(struct device *dev, phys_addr_t tlb_addr,
1420	size_t size, enum dma_data_direction dir)
1421	{
1422	if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)
1423	swiotlb_bounce(dev, tlb_addr, size, dir: DMA_TO_DEVICE);
1424	else
1425	BUG_ON(dir != DMA_FROM_DEVICE);
1426	}
1427
1428	void swiotlb_sync_single_for_cpu(struct device *dev, phys_addr_t tlb_addr,
1429	size_t size, enum dma_data_direction dir)
1430	{
1431	if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
1432	swiotlb_bounce(dev, tlb_addr, size, dir: DMA_FROM_DEVICE);
1433	else
1434	BUG_ON(dir != DMA_TO_DEVICE);
1435	}
1436
1437	/*
1438	* Create a swiotlb mapping for the buffer at @paddr, and in case of DMAing
1439	* to the device copy the data into it as well.
1440	*/
1441	dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
1442	enum dma_data_direction dir, unsigned long attrs)
1443	{
1444	phys_addr_t swiotlb_addr;
1445	dma_addr_t dma_addr;
1446
1447	trace_swiotlb_bounced(dev, dev_addr: phys_to_dma(dev, paddr), size);
1448
1449	swiotlb_addr = swiotlb_tbl_map_single(dev, orig_addr: paddr, mapping_size: size, alloc_size: size, alloc_align_mask: 0, dir,
1450	attrs);
1451	if (swiotlb_addr == (phys_addr_t)DMA_MAPPING_ERROR)
1452	return DMA_MAPPING_ERROR;
1453
1454	/* Ensure that the address returned is DMA'ble */
1455	dma_addr = phys_to_dma_unencrypted(dev, paddr: swiotlb_addr);
1456	if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
1457	swiotlb_tbl_unmap_single(dev, tlb_addr: swiotlb_addr, mapping_size: size, dir,
1458	attrs: attrs | DMA_ATTR_SKIP_CPU_SYNC);
1459	dev_WARN_ONCE(dev, 1,
1460	"swiotlb addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",
1461	&dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);
1462	return DMA_MAPPING_ERROR;
1463	}
1464
1465	if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
1466	arch_sync_dma_for_device(paddr: swiotlb_addr, size, dir);
1467	return dma_addr;
1468	}
1469
1470	size_t swiotlb_max_mapping_size(struct device *dev)
1471	{
1472	int min_align_mask = dma_get_min_align_mask(dev);
1473	int min_align = 0;
1474
1475	/*
1476	* swiotlb_find_slots() skips slots according to
1477	* min align mask. This affects max mapping size.
1478	* Take it into acount here.
1479	*/
1480	if (min_align_mask)
1481	min_align = roundup(min_align_mask, IO_TLB_SIZE);
1482
1483	return ((size_t)IO_TLB_SIZE) * IO_TLB_SEGSIZE - min_align;
1484	}
1485
1486	/**
1487	* is_swiotlb_allocated() - check if the default software IO TLB is initialized
1488	*/
1489	bool is_swiotlb_allocated(void)
1490	{
1491	return io_tlb_default_mem.nslabs;
1492	}
1493
1494	bool is_swiotlb_active(struct device *dev)
1495	{
1496	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1497
1498	return mem && mem->nslabs;
1499	}
1500
1501	/**
1502	* default_swiotlb_base() - get the base address of the default SWIOTLB
1503	*
1504	* Get the lowest physical address used by the default software IO TLB pool.
1505	*/
1506	phys_addr_t default_swiotlb_base(void)
1507	{
1508	#ifdef CONFIG_SWIOTLB_DYNAMIC
1509	io_tlb_default_mem.can_grow = false;
1510	#endif
1511	return io_tlb_default_mem.defpool.start;
1512	}
1513
1514	/**
1515	* default_swiotlb_limit() - get the address limit of the default SWIOTLB
1516	*
1517	* Get the highest physical address used by the default software IO TLB pool.
1518	*/
1519	phys_addr_t default_swiotlb_limit(void)
1520	{
1521	#ifdef CONFIG_SWIOTLB_DYNAMIC
1522	return io_tlb_default_mem.phys_limit;
1523	#else
1524	return io_tlb_default_mem.defpool.end - 1;
1525	#endif
1526	}
1527
1528	#ifdef CONFIG_DEBUG_FS
1529
1530	static int io_tlb_used_get(void *data, u64 *val)
1531	{
1532	struct io_tlb_mem *mem = data;
1533
1534	*val = mem_used(mem);
1535	return 0;
1536	}
1537
1538	static int io_tlb_hiwater_get(void *data, u64 *val)
1539	{
1540	struct io_tlb_mem *mem = data;
1541
1542	*val = atomic_long_read(v: &mem->used_hiwater);
1543	return 0;
1544	}
1545
1546	static int io_tlb_hiwater_set(void *data, u64 val)
1547	{
1548	struct io_tlb_mem *mem = data;
1549
1550	/* Only allow setting to zero */
1551	if (val != 0)
1552	return -EINVAL;
1553
1554	atomic_long_set(v: &mem->used_hiwater, i: val);
1555	return 0;
1556	}
1557
1558	DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_used, io_tlb_used_get, NULL, "%llu\n");
1559	DEFINE_DEBUGFS_ATTRIBUTE(fops_io_tlb_hiwater, io_tlb_hiwater_get,
1560	io_tlb_hiwater_set, "%llu\n");
1561
1562	static void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
1563	const char *dirname)
1564	{
1565	atomic_long_set(v: &mem->total_used, i: 0);
1566	atomic_long_set(v: &mem->used_hiwater, i: 0);
1567
1568	mem->debugfs = debugfs_create_dir(name: dirname, parent: io_tlb_default_mem.debugfs);
1569	if (!mem->nslabs)
1570	return;
1571
1572	debugfs_create_ulong(name: "io_tlb_nslabs", mode: 0400, parent: mem->debugfs, value: &mem->nslabs);
1573	debugfs_create_file(name: "io_tlb_used", mode: 0400, parent: mem->debugfs, data: mem,
1574	fops: &fops_io_tlb_used);
1575	debugfs_create_file(name: "io_tlb_used_hiwater", mode: 0600, parent: mem->debugfs, data: mem,
1576	fops: &fops_io_tlb_hiwater);
1577	}
1578
1579	static int __init swiotlb_create_default_debugfs(void)
1580	{
1581	swiotlb_create_debugfs_files(mem: &io_tlb_default_mem, dirname: "swiotlb");
1582	return 0;
1583	}
1584
1585	late_initcall(swiotlb_create_default_debugfs);
1586
1587	#else /* !CONFIG_DEBUG_FS */
1588
1589	static inline void swiotlb_create_debugfs_files(struct io_tlb_mem *mem,
1590	const char *dirname)
1591	{
1592	}
1593
1594	#endif /* CONFIG_DEBUG_FS */
1595
1596	#ifdef CONFIG_DMA_RESTRICTED_POOL
1597
1598	struct page *swiotlb_alloc(struct device *dev, size_t size)
1599	{
1600	struct io_tlb_mem *mem = dev->dma_io_tlb_mem;
1601	struct io_tlb_pool *pool;
1602	phys_addr_t tlb_addr;
1603	int index;
1604
1605	if (!mem)
1606	return NULL;
1607
1608	index = swiotlb_find_slots(dev, orig_addr: 0, alloc_size: size, alloc_align_mask: 0, retpool: &pool);
1609	if (index == -1)
1610	return NULL;
1611
1612	tlb_addr = slot_addr(start: pool->start, idx: index);
1613
1614	return pfn_to_page(PFN_DOWN(tlb_addr));
1615	}
1616
1617	bool swiotlb_free(struct device *dev, struct page *page, size_t size)
1618	{
1619	phys_addr_t tlb_addr = page_to_phys(page);
1620
1621	if (!is_swiotlb_buffer(dev, paddr: tlb_addr))
1622	return false;
1623
1624	swiotlb_release_slots(dev, tlb_addr);
1625
1626	return true;
1627	}
1628
1629	static int rmem_swiotlb_device_init(struct reserved_mem *rmem,
1630	struct device *dev)
1631	{
1632	struct io_tlb_mem *mem = rmem->priv;
1633	unsigned long nslabs = rmem->size >> IO_TLB_SHIFT;
1634
1635	/* Set Per-device io tlb area to one */
1636	unsigned int nareas = 1;
1637
1638	if (PageHighMem(pfn_to_page(PHYS_PFN(rmem->base)))) {
1639	dev_err(dev, "Restricted DMA pool must be accessible within the linear mapping.");
1640	return -EINVAL;
1641	}
1642
1643	/*
1644	* Since multiple devices can share the same pool, the private data,
1645	* io_tlb_mem struct, will be initialized by the first device attached
1646	* to it.
1647	*/
1648	if (!mem) {
1649	struct io_tlb_pool *pool;
1650
1651	mem = kzalloc(size: sizeof(*mem), GFP_KERNEL);
1652	if (!mem)
1653	return -ENOMEM;
1654	pool = &mem->defpool;
1655
1656	pool->slots = kcalloc(n: nslabs, size: sizeof(*pool->slots), GFP_KERNEL);
1657	if (!pool->slots) {
1658	kfree(objp: mem);
1659	return -ENOMEM;
1660	}
1661
1662	pool->areas = kcalloc(n: nareas, size: sizeof(*pool->areas),
1663	GFP_KERNEL);
1664	if (!pool->areas) {
1665	kfree(objp: pool->slots);
1666	kfree(objp: mem);
1667	return -ENOMEM;
1668	}
1669
1670	set_memory_decrypted(addr: (unsigned long)phys_to_virt(address: rmem->base),
1671	numpages: rmem->size >> PAGE_SHIFT);
1672	swiotlb_init_io_tlb_pool(mem: pool, start: rmem->base, nslabs,
1673	late_alloc: false, nareas);
1674	mem->force_bounce = true;
1675	mem->for_alloc = true;
1676	#ifdef CONFIG_SWIOTLB_DYNAMIC
1677	spin_lock_init(&mem->lock);
1678	#endif
1679	add_mem_pool(mem, pool);
1680
1681	rmem->priv = mem;
1682
1683	swiotlb_create_debugfs_files(mem, dirname: rmem->name);
1684	}
1685
1686	dev->dma_io_tlb_mem = mem;
1687
1688	return 0;
1689	}
1690
1691	static void rmem_swiotlb_device_release(struct reserved_mem *rmem,
1692	struct device *dev)
1693	{
1694	dev->dma_io_tlb_mem = &io_tlb_default_mem;
1695	}
1696
1697	static const struct reserved_mem_ops rmem_swiotlb_ops = {
1698	.device_init = rmem_swiotlb_device_init,
1699	.device_release = rmem_swiotlb_device_release,
1700	};
1701
1702	static int __init rmem_swiotlb_setup(struct reserved_mem *rmem)
1703	{
1704	unsigned long node = rmem->fdt_node;
1705
1706	if (of_get_flat_dt_prop(node, name: "reusable", NULL) ||
1707	of_get_flat_dt_prop(node, name: "linux,cma-default", NULL) ||
1708	of_get_flat_dt_prop(node, name: "linux,dma-default", NULL) ||
1709	of_get_flat_dt_prop(node, name: "no-map", NULL))
1710	return -EINVAL;
1711
1712	rmem->ops = &rmem_swiotlb_ops;
1713	pr_info("Reserved memory: created restricted DMA pool at %pa, size %ld MiB\n",
1714	&rmem->base, (unsigned long)rmem->size / SZ_1M);
1715	return 0;
1716	}
1717
1718	RESERVEDMEM_OF_DECLARE(dma, "restricted-dma-pool", rmem_swiotlb_setup);
1719	#endif /* CONFIG_DMA_RESTRICTED_POOL */
1720