1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Copyright(C) 2016 Linaro Limited. All rights reserved.
4	* Author: Mathieu Poirier <mathieu.poirier@linaro.org>
5	*/
6
7	#include <linux/atomic.h>
8	#include <linux/coresight.h>
9	#include <linux/dma-mapping.h>
10	#include <linux/iommu.h>
11	#include <linux/idr.h>
12	#include <linux/mutex.h>
13	#include <linux/refcount.h>
14	#include <linux/slab.h>
15	#include <linux/types.h>
16	#include <linux/vmalloc.h>
17	#include "coresight-catu.h"
18	#include "coresight-etm-perf.h"
19	#include "coresight-priv.h"
20	#include "coresight-tmc.h"
21
22	struct etr_flat_buf {
23	struct device *dev;
24	dma_addr_t daddr;
25	void *vaddr;
26	size_t size;
27	};
28
29	/*
30	* etr_perf_buffer - Perf buffer used for ETR
31	* @drvdata - The ETR drvdaga this buffer has been allocated for.
32	* @etr_buf - Actual buffer used by the ETR
33	* @pid - The PID this etr_perf_buffer belongs to.
34	* @snaphost - Perf session mode
35	* @nr_pages - Number of pages in the ring buffer.
36	* @pages - Array of Pages in the ring buffer.
37	*/
38	struct etr_perf_buffer {
39	struct tmc_drvdata *drvdata;
40	struct etr_buf *etr_buf;
41	pid_t pid;
42	bool snapshot;
43	int nr_pages;
44	void **pages;
45	};
46
47	/* Convert the perf index to an offset within the ETR buffer */
48	#define PERF_IDX2OFF(idx, buf) \
49	((idx) % ((unsigned long)(buf)->nr_pages << PAGE_SHIFT))
50
51	/* Lower limit for ETR hardware buffer */
52	#define TMC_ETR_PERF_MIN_BUF_SIZE SZ_1M
53
54	/*
55	* The TMC ETR SG has a page size of 4K. The SG table contains pointers
56	* to 4KB buffers. However, the OS may use a PAGE_SIZE different from
57	* 4K (i.e, 16KB or 64KB). This implies that a single OS page could
58	* contain more than one SG buffer and tables.
59	*
60	* A table entry has the following format:
61	*
62	* ---Bit31------------Bit4-------Bit1-----Bit0--
63	* | Address[39:12] | SBZ | Entry Type |
64	* ----------------------------------------------
65	*
66	* Address: Bits [39:12] of a physical page address. Bits [11:0] are
67	* always zero.
68	*
69	* Entry type:
70	* b00 - Reserved.
71	* b01 - Last entry in the tables, points to 4K page buffer.
72	* b10 - Normal entry, points to 4K page buffer.
73	* b11 - Link. The address points to the base of next table.
74	*/
75
76	typedef u32 sgte_t;
77
78	#define ETR_SG_PAGE_SHIFT 12
79	#define ETR_SG_PAGE_SIZE (1UL << ETR_SG_PAGE_SHIFT)
80	#define ETR_SG_PAGES_PER_SYSPAGE (PAGE_SIZE / ETR_SG_PAGE_SIZE)
81	#define ETR_SG_PTRS_PER_PAGE (ETR_SG_PAGE_SIZE / sizeof(sgte_t))
82	#define ETR_SG_PTRS_PER_SYSPAGE (PAGE_SIZE / sizeof(sgte_t))
83
84	#define ETR_SG_ET_MASK 0x3
85	#define ETR_SG_ET_LAST 0x1
86	#define ETR_SG_ET_NORMAL 0x2
87	#define ETR_SG_ET_LINK 0x3
88
89	#define ETR_SG_ADDR_SHIFT 4
90
91	#define ETR_SG_ENTRY(addr, type) \
92	(sgte_t)((((addr) >> ETR_SG_PAGE_SHIFT) << ETR_SG_ADDR_SHIFT) | \
93	(type & ETR_SG_ET_MASK))
94
95	#define ETR_SG_ADDR(entry) \
96	(((dma_addr_t)(entry) >> ETR_SG_ADDR_SHIFT) << ETR_SG_PAGE_SHIFT)
97	#define ETR_SG_ET(entry) ((entry) & ETR_SG_ET_MASK)
98
99	/*
100	* struct etr_sg_table : ETR SG Table
101	* @sg_table: Generic SG Table holding the data/table pages.
102	* @hwaddr: hwaddress used by the TMC, which is the base
103	* address of the table.
104	*/
105	struct etr_sg_table {
106	struct tmc_sg_table *sg_table;
107	dma_addr_t hwaddr;
108	};
109
110	/*
111	* tmc_etr_sg_table_entries: Total number of table entries required to map
112	* @nr_pages system pages.
113	*
114	* We need to map @nr_pages * ETR_SG_PAGES_PER_SYSPAGE data pages.
115	* Each TMC page can map (ETR_SG_PTRS_PER_PAGE - 1) buffer pointers,
116	* with the last entry pointing to another page of table entries.
117	* If we spill over to a new page for mapping 1 entry, we could as
118	* well replace the link entry of the previous page with the last entry.
119	*/
120	static inline unsigned long __attribute_const__
121	tmc_etr_sg_table_entries(int nr_pages)
122	{
123	unsigned long nr_sgpages = nr_pages * ETR_SG_PAGES_PER_SYSPAGE;
124	unsigned long nr_sglinks = nr_sgpages / (ETR_SG_PTRS_PER_PAGE - 1);
125	/*
126	* If we spill over to a new page for 1 entry, we could as well
127	* make it the LAST entry in the previous page, skipping the Link
128	* address.
129	*/
130	if (nr_sglinks && (nr_sgpages % (ETR_SG_PTRS_PER_PAGE - 1) < 2))
131	nr_sglinks--;
132	return nr_sgpages + nr_sglinks;
133	}
134
135	/*
136	* tmc_pages_get_offset: Go through all the pages in the tmc_pages
137	* and map the device address @addr to an offset within the virtual
138	* contiguous buffer.
139	*/
140	static long
141	tmc_pages_get_offset(struct tmc_pages *tmc_pages, dma_addr_t addr)
142	{
143	int i;
144	dma_addr_t page_start;
145
146	for (i = 0; i < tmc_pages->nr_pages; i++) {
147	page_start = tmc_pages->daddrs[i];
148	if (addr >= page_start && addr < (page_start + PAGE_SIZE))
149	return i * PAGE_SIZE + (addr - page_start);
150	}
151
152	return -EINVAL;
153	}
154
155	/*
156	* tmc_pages_free : Unmap and free the pages used by tmc_pages.
157	* If the pages were not allocated in tmc_pages_alloc(), we would
158	* simply drop the refcount.
159	*/
160	static void tmc_pages_free(struct tmc_pages *tmc_pages,
161	struct device *dev, enum dma_data_direction dir)
162	{
163	int i;
164	struct device *real_dev = dev->parent;
165
166	for (i = 0; i < tmc_pages->nr_pages; i++) {
167	if (tmc_pages->daddrs && tmc_pages->daddrs[i])
168	dma_unmap_page(real_dev, tmc_pages->daddrs[i],
169	PAGE_SIZE, dir);
170	if (tmc_pages->pages && tmc_pages->pages[i])
171	__free_page(tmc_pages->pages[i]);
172	}
173
174	kfree(objp: tmc_pages->pages);
175	kfree(objp: tmc_pages->daddrs);
176	tmc_pages->pages = NULL;
177	tmc_pages->daddrs = NULL;
178	tmc_pages->nr_pages = 0;
179	}
180
181	/*
182	* tmc_pages_alloc : Allocate and map pages for a given @tmc_pages.
183	* If @pages is not NULL, the list of page virtual addresses are
184	* used as the data pages. The pages are then dma_map'ed for @dev
185	* with dma_direction @dir.
186	*
187	* Returns 0 upon success, else the error number.
188	*/
189	static int tmc_pages_alloc(struct tmc_pages *tmc_pages,
190	struct device *dev, int node,
191	enum dma_data_direction dir, void **pages)
192	{
193	int i, nr_pages;
194	dma_addr_t paddr;
195	struct page *page;
196	struct device *real_dev = dev->parent;
197
198	nr_pages = tmc_pages->nr_pages;
199	tmc_pages->daddrs = kcalloc(n: nr_pages, size: sizeof(*tmc_pages->daddrs),
200	GFP_KERNEL);
201	if (!tmc_pages->daddrs)
202	return -ENOMEM;
203	tmc_pages->pages = kcalloc(n: nr_pages, size: sizeof(*tmc_pages->pages),
204	GFP_KERNEL);
205	if (!tmc_pages->pages) {
206	kfree(objp: tmc_pages->daddrs);
207	tmc_pages->daddrs = NULL;
208	return -ENOMEM;
209	}
210
211	for (i = 0; i < nr_pages; i++) {
212	if (pages && pages[i]) {
213	page = virt_to_page(pages[i]);
214	/* Hold a refcount on the page */
215	get_page(page);
216	} else {
217	page = alloc_pages_node(nid: node,
218	GFP_KERNEL | __GFP_ZERO, order: 0);
219	if (!page)
220	goto err;
221	}
222	paddr = dma_map_page(real_dev, page, 0, PAGE_SIZE, dir);
223	if (dma_mapping_error(dev: real_dev, dma_addr: paddr))
224	goto err;
225	tmc_pages->daddrs[i] = paddr;
226	tmc_pages->pages[i] = page;
227	}
228	return 0;
229	err:
230	tmc_pages_free(tmc_pages, dev, dir);
231	return -ENOMEM;
232	}
233
234	static inline long
235	tmc_sg_get_data_page_offset(struct tmc_sg_table *sg_table, dma_addr_t addr)
236	{
237	return tmc_pages_get_offset(tmc_pages: &sg_table->data_pages, addr);
238	}
239
240	static inline void tmc_free_table_pages(struct tmc_sg_table *sg_table)
241	{
242	if (sg_table->table_vaddr)
243	vunmap(addr: sg_table->table_vaddr);
244	tmc_pages_free(tmc_pages: &sg_table->table_pages, dev: sg_table->dev, dir: DMA_TO_DEVICE);
245	}
246
247	static void tmc_free_data_pages(struct tmc_sg_table *sg_table)
248	{
249	if (sg_table->data_vaddr)
250	vunmap(addr: sg_table->data_vaddr);
251	tmc_pages_free(tmc_pages: &sg_table->data_pages, dev: sg_table->dev, dir: DMA_FROM_DEVICE);
252	}
253
254	void tmc_free_sg_table(struct tmc_sg_table *sg_table)
255	{
256	tmc_free_table_pages(sg_table);
257	tmc_free_data_pages(sg_table);
258	}
259	EXPORT_SYMBOL_GPL(tmc_free_sg_table);
260
261	/*
262	* Alloc pages for the table. Since this will be used by the device,
263	* allocate the pages closer to the device (i.e, dev_to_node(dev)
264	* rather than the CPU node).
265	*/
266	static int tmc_alloc_table_pages(struct tmc_sg_table *sg_table)
267	{
268	int rc;
269	struct tmc_pages *table_pages = &sg_table->table_pages;
270
271	rc = tmc_pages_alloc(tmc_pages: table_pages, dev: sg_table->dev,
272	node: dev_to_node(dev: sg_table->dev),
273	dir: DMA_TO_DEVICE, NULL);
274	if (rc)
275	return rc;
276	sg_table->table_vaddr = vmap(pages: table_pages->pages,
277	count: table_pages->nr_pages,
278	VM_MAP,
279	PAGE_KERNEL);
280	if (!sg_table->table_vaddr)
281	rc = -ENOMEM;
282	else
283	sg_table->table_daddr = table_pages->daddrs[0];
284	return rc;
285	}
286
287	static int tmc_alloc_data_pages(struct tmc_sg_table *sg_table, void **pages)
288	{
289	int rc;
290
291	/* Allocate data pages on the node requested by the caller */
292	rc = tmc_pages_alloc(tmc_pages: &sg_table->data_pages,
293	dev: sg_table->dev, node: sg_table->node,
294	dir: DMA_FROM_DEVICE, pages);
295	if (!rc) {
296	sg_table->data_vaddr = vmap(pages: sg_table->data_pages.pages,
297	count: sg_table->data_pages.nr_pages,
298	VM_MAP,
299	PAGE_KERNEL);
300	if (!sg_table->data_vaddr)
301	rc = -ENOMEM;
302	}
303	return rc;
304	}
305
306	/*
307	* tmc_alloc_sg_table: Allocate and setup dma pages for the TMC SG table
308	* and data buffers. TMC writes to the data buffers and reads from the SG
309	* Table pages.
310	*
311	* @dev - Coresight device to which page should be DMA mapped.
312	* @node - Numa node for mem allocations
313	* @nr_tpages - Number of pages for the table entries.
314	* @nr_dpages - Number of pages for Data buffer.
315	* @pages - Optional list of virtual address of pages.
316	*/
317	struct tmc_sg_table *tmc_alloc_sg_table(struct device *dev,
318	int node,
319	int nr_tpages,
320	int nr_dpages,
321	void **pages)
322	{
323	long rc;
324	struct tmc_sg_table *sg_table;
325
326	sg_table = kzalloc(size: sizeof(*sg_table), GFP_KERNEL);
327	if (!sg_table)
328	return ERR_PTR(error: -ENOMEM);
329	sg_table->data_pages.nr_pages = nr_dpages;
330	sg_table->table_pages.nr_pages = nr_tpages;
331	sg_table->node = node;
332	sg_table->dev = dev;
333
334	rc = tmc_alloc_data_pages(sg_table, pages);
335	if (!rc)
336	rc = tmc_alloc_table_pages(sg_table);
337	if (rc) {
338	tmc_free_sg_table(sg_table);
339	kfree(objp: sg_table);
340	return ERR_PTR(error: rc);
341	}
342
343	return sg_table;
344	}
345	EXPORT_SYMBOL_GPL(tmc_alloc_sg_table);
346
347	/*
348	* tmc_sg_table_sync_data_range: Sync the data buffer written
349	* by the device from @offset upto a @size bytes.
350	*/
351	void tmc_sg_table_sync_data_range(struct tmc_sg_table *table,
352	u64 offset, u64 size)
353	{
354	int i, index, start;
355	int npages = DIV_ROUND_UP(size, PAGE_SIZE);
356	struct device *real_dev = table->dev->parent;
357	struct tmc_pages *data = &table->data_pages;
358
359	start = offset >> PAGE_SHIFT;
360	for (i = start; i < (start + npages); i++) {
361	index = i % data->nr_pages;
362	dma_sync_single_for_cpu(dev: real_dev, addr: data->daddrs[index],
363	PAGE_SIZE, dir: DMA_FROM_DEVICE);
364	}
365	}
366	EXPORT_SYMBOL_GPL(tmc_sg_table_sync_data_range);
367
368	/* tmc_sg_sync_table: Sync the page table */
369	void tmc_sg_table_sync_table(struct tmc_sg_table *sg_table)
370	{
371	int i;
372	struct device *real_dev = sg_table->dev->parent;
373	struct tmc_pages *table_pages = &sg_table->table_pages;
374
375	for (i = 0; i < table_pages->nr_pages; i++)
376	dma_sync_single_for_device(dev: real_dev, addr: table_pages->daddrs[i],
377	PAGE_SIZE, dir: DMA_TO_DEVICE);
378	}
379	EXPORT_SYMBOL_GPL(tmc_sg_table_sync_table);
380
381	/*
382	* tmc_sg_table_get_data: Get the buffer pointer for data @offset
383	* in the SG buffer. The @bufpp is updated to point to the buffer.
384	* Returns :
385	* the length of linear data available at @offset.
386	* or
387	* <= 0 if no data is available.
388	*/
389	ssize_t tmc_sg_table_get_data(struct tmc_sg_table *sg_table,
390	u64 offset, size_t len, char **bufpp)
391	{
392	size_t size;
393	int pg_idx = offset >> PAGE_SHIFT;
394	int pg_offset = offset & (PAGE_SIZE - 1);
395	struct tmc_pages *data_pages = &sg_table->data_pages;
396
397	size = tmc_sg_table_buf_size(sg_table);
398	if (offset >= size)
399	return -EINVAL;
400
401	/* Make sure we don't go beyond the end */
402	len = (len < (size - offset)) ? len : size - offset;
403	/* Respect the page boundaries */
404	len = (len < (PAGE_SIZE - pg_offset)) ? len : (PAGE_SIZE - pg_offset);
405	if (len > 0)
406	*bufpp = page_address(data_pages->pages[pg_idx]) + pg_offset;
407	return len;
408	}
409	EXPORT_SYMBOL_GPL(tmc_sg_table_get_data);
410
411	#ifdef ETR_SG_DEBUG
412	/* Map a dma address to virtual address */
413	static unsigned long
414	tmc_sg_daddr_to_vaddr(struct tmc_sg_table *sg_table,
415	dma_addr_t addr, bool table)
416	{
417	long offset;
418	unsigned long base;
419	struct tmc_pages *tmc_pages;
420
421	if (table) {
422	tmc_pages = &sg_table->table_pages;
423	base = (unsigned long)sg_table->table_vaddr;
424	} else {
425	tmc_pages = &sg_table->data_pages;
426	base = (unsigned long)sg_table->data_vaddr;
427	}
428
429	offset = tmc_pages_get_offset(tmc_pages, addr);
430	if (offset < 0)
431	return 0;
432	return base + offset;
433	}
434
435	/* Dump the given sg_table */
436	static void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table)
437	{
438	sgte_t *ptr;
439	int i = 0;
440	dma_addr_t addr;
441	struct tmc_sg_table *sg_table = etr_table->sg_table;
442
443	ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
444	etr_table->hwaddr, true);
445	while (ptr) {
446	addr = ETR_SG_ADDR(*ptr);
447	switch (ETR_SG_ET(*ptr)) {
448	case ETR_SG_ET_NORMAL:
449	dev_dbg(sg_table->dev,
450	"%05d: %p\t:[N] 0x%llx\n", i, ptr, addr);
451	ptr++;
452	break;
453	case ETR_SG_ET_LINK:
454	dev_dbg(sg_table->dev,
455	"%05d: *** %p\t:{L} 0x%llx ***\n",
456	i, ptr, addr);
457	ptr = (sgte_t *)tmc_sg_daddr_to_vaddr(sg_table,
458	addr, true);
459	break;
460	case ETR_SG_ET_LAST:
461	dev_dbg(sg_table->dev,
462	"%05d: ### %p\t:[L] 0x%llx ###\n",
463	i, ptr, addr);
464	return;
465	default:
466	dev_dbg(sg_table->dev,
467	"%05d: xxx %p\t:[INVALID] 0x%llx xxx\n",
468	i, ptr, addr);
469	return;
470	}
471	i++;
472	}
473	dev_dbg(sg_table->dev, "******* End of Table *****\n");
474	}
475	#else
476	static inline void tmc_etr_sg_table_dump(struct etr_sg_table *etr_table) {}
477	#endif
478
479	/*
480	* Populate the SG Table page table entries from table/data
481	* pages allocated. Each Data page has ETR_SG_PAGES_PER_SYSPAGE SG pages.
482	* So does a Table page. So we keep track of indices of the tables
483	* in each system page and move the pointers accordingly.
484	*/
485	#define INC_IDX_ROUND(idx, size) ((idx) = ((idx) + 1) % (size))
486	static void tmc_etr_sg_table_populate(struct etr_sg_table *etr_table)
487	{
488	dma_addr_t paddr;
489	int i, type, nr_entries;
490	int tpidx = 0; /* index to the current system table_page */
491	int sgtidx = 0; /* index to the sg_table within the current syspage */
492	int sgtentry = 0; /* the entry within the sg_table */
493	int dpidx = 0; /* index to the current system data_page */
494	int spidx = 0; /* index to the SG page within the current data page */
495	sgte_t *ptr; /* pointer to the table entry to fill */
496	struct tmc_sg_table *sg_table = etr_table->sg_table;
497	dma_addr_t *table_daddrs = sg_table->table_pages.daddrs;
498	dma_addr_t *data_daddrs = sg_table->data_pages.daddrs;
499
500	nr_entries = tmc_etr_sg_table_entries(nr_pages: sg_table->data_pages.nr_pages);
501	/*
502	* Use the contiguous virtual address of the table to update entries.
503	*/
504	ptr = sg_table->table_vaddr;
505	/*
506	* Fill all the entries, except the last entry to avoid special
507	* checks within the loop.
508	*/
509	for (i = 0; i < nr_entries - 1; i++) {
510	if (sgtentry == ETR_SG_PTRS_PER_PAGE - 1) {
511	/*
512	* Last entry in a sg_table page is a link address to
513	* the next table page. If this sg_table is the last
514	* one in the system page, it links to the first
515	* sg_table in the next system page. Otherwise, it
516	* links to the next sg_table page within the system
517	* page.
518	*/
519	if (sgtidx == ETR_SG_PAGES_PER_SYSPAGE - 1) {
520	paddr = table_daddrs[tpidx + 1];
521	} else {
522	paddr = table_daddrs[tpidx] +
523	(ETR_SG_PAGE_SIZE * (sgtidx + 1));
524	}
525	type = ETR_SG_ET_LINK;
526	} else {
527	/*
528	* Update the indices to the data_pages to point to the
529	* next sg_page in the data buffer.
530	*/
531	type = ETR_SG_ET_NORMAL;
532	paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
533	if (!INC_IDX_ROUND(spidx, ETR_SG_PAGES_PER_SYSPAGE))
534	dpidx++;
535	}
536	*ptr++ = ETR_SG_ENTRY(paddr, type);
537	/*
538	* Move to the next table pointer, moving the table page index
539	* if necessary
540	*/
541	if (!INC_IDX_ROUND(sgtentry, ETR_SG_PTRS_PER_PAGE)) {
542	if (!INC_IDX_ROUND(sgtidx, ETR_SG_PAGES_PER_SYSPAGE))
543	tpidx++;
544	}
545	}
546
547	/* Set up the last entry, which is always a data pointer */
548	paddr = data_daddrs[dpidx] + spidx * ETR_SG_PAGE_SIZE;
549	*ptr++ = ETR_SG_ENTRY(paddr, ETR_SG_ET_LAST);
550	}
551
552	/*
553	* tmc_init_etr_sg_table: Allocate a TMC ETR SG table, data buffer of @size and
554	* populate the table.
555	*
556	* @dev - Device pointer for the TMC
557	* @node - NUMA node where the memory should be allocated
558	* @size - Total size of the data buffer
559	* @pages - Optional list of page virtual address
560	*/
561	static struct etr_sg_table *
562	tmc_init_etr_sg_table(struct device *dev, int node,
563	unsigned long size, void **pages)
564	{
565	int nr_entries, nr_tpages;
566	int nr_dpages = size >> PAGE_SHIFT;
567	struct tmc_sg_table *sg_table;
568	struct etr_sg_table *etr_table;
569
570	etr_table = kzalloc(size: sizeof(*etr_table), GFP_KERNEL);
571	if (!etr_table)
572	return ERR_PTR(error: -ENOMEM);
573	nr_entries = tmc_etr_sg_table_entries(nr_pages: nr_dpages);
574	nr_tpages = DIV_ROUND_UP(nr_entries, ETR_SG_PTRS_PER_SYSPAGE);
575
576	sg_table = tmc_alloc_sg_table(dev, node, nr_tpages, nr_dpages, pages);
577	if (IS_ERR(ptr: sg_table)) {
578	kfree(objp: etr_table);
579	return ERR_CAST(ptr: sg_table);
580	}
581
582	etr_table->sg_table = sg_table;
583	/* TMC should use table base address for DBA */
584	etr_table->hwaddr = sg_table->table_daddr;
585	tmc_etr_sg_table_populate(etr_table);
586	/* Sync the table pages for the HW */
587	tmc_sg_table_sync_table(sg_table);
588	tmc_etr_sg_table_dump(etr_table);
589
590	return etr_table;
591	}
592
593	/*
594	* tmc_etr_alloc_flat_buf: Allocate a contiguous DMA buffer.
595	*/
596	static int tmc_etr_alloc_flat_buf(struct tmc_drvdata *drvdata,
597	struct etr_buf *etr_buf, int node,
598	void **pages)
599	{
600	struct etr_flat_buf *flat_buf;
601	struct device *real_dev = drvdata->csdev->dev.parent;
602
603	/* We cannot reuse existing pages for flat buf */
604	if (pages)
605	return -EINVAL;
606
607	flat_buf = kzalloc(size: sizeof(*flat_buf), GFP_KERNEL);
608	if (!flat_buf)
609	return -ENOMEM;
610
611	flat_buf->vaddr = dma_alloc_noncoherent(dev: real_dev, size: etr_buf->size,
612	dma_handle: &flat_buf->daddr,
613	dir: DMA_FROM_DEVICE,
614	GFP_KERNEL | __GFP_NOWARN);
615	if (!flat_buf->vaddr) {
616	kfree(objp: flat_buf);
617	return -ENOMEM;
618	}
619
620	flat_buf->size = etr_buf->size;
621	flat_buf->dev = &drvdata->csdev->dev;
622	etr_buf->hwaddr = flat_buf->daddr;
623	etr_buf->mode = ETR_MODE_FLAT;
624	etr_buf->private = flat_buf;
625	return 0;
626	}
627
628	static void tmc_etr_free_flat_buf(struct etr_buf *etr_buf)
629	{
630	struct etr_flat_buf *flat_buf = etr_buf->private;
631
632	if (flat_buf && flat_buf->daddr) {
633	struct device *real_dev = flat_buf->dev->parent;
634
635	dma_free_noncoherent(dev: real_dev, size: etr_buf->size,
636	vaddr: flat_buf->vaddr, dma_handle: flat_buf->daddr,
637	dir: DMA_FROM_DEVICE);
638	}
639	kfree(objp: flat_buf);
640	}
641
642	static void tmc_etr_sync_flat_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
643	{
644	struct etr_flat_buf *flat_buf = etr_buf->private;
645	struct device *real_dev = flat_buf->dev->parent;
646
647	/*
648	* Adjust the buffer to point to the beginning of the trace data
649	* and update the available trace data.
650	*/
651	etr_buf->offset = rrp - etr_buf->hwaddr;
652	if (etr_buf->full)
653	etr_buf->len = etr_buf->size;
654	else
655	etr_buf->len = rwp - rrp;
656
657	/*
658	* The driver always starts tracing at the beginning of the buffer,
659	* the only reason why we would get a wrap around is when the buffer
660	* is full. Sync the entire buffer in one go for this case.
661	*/
662	if (etr_buf->offset + etr_buf->len > etr_buf->size)
663	dma_sync_single_for_cpu(dev: real_dev, addr: flat_buf->daddr,
664	size: etr_buf->size, dir: DMA_FROM_DEVICE);
665	else
666	dma_sync_single_for_cpu(dev: real_dev,
667	addr: flat_buf->daddr + etr_buf->offset,
668	size: etr_buf->len, dir: DMA_FROM_DEVICE);
669	}
670
671	static ssize_t tmc_etr_get_data_flat_buf(struct etr_buf *etr_buf,
672	u64 offset, size_t len, char **bufpp)
673	{
674	struct etr_flat_buf *flat_buf = etr_buf->private;
675
676	*bufpp = (char *)flat_buf->vaddr + offset;
677	/*
678	* tmc_etr_buf_get_data already adjusts the length to handle
679	* buffer wrapping around.
680	*/
681	return len;
682	}
683
684	static const struct etr_buf_operations etr_flat_buf_ops = {
685	.alloc = tmc_etr_alloc_flat_buf,
686	.free = tmc_etr_free_flat_buf,
687	.sync = tmc_etr_sync_flat_buf,
688	.get_data = tmc_etr_get_data_flat_buf,
689	};
690
691	/*
692	* tmc_etr_alloc_sg_buf: Allocate an SG buf @etr_buf. Setup the parameters
693	* appropriately.
694	*/
695	static int tmc_etr_alloc_sg_buf(struct tmc_drvdata *drvdata,
696	struct etr_buf *etr_buf, int node,
697	void **pages)
698	{
699	struct etr_sg_table *etr_table;
700	struct device *dev = &drvdata->csdev->dev;
701
702	etr_table = tmc_init_etr_sg_table(dev, node,
703	size: etr_buf->size, pages);
704	if (IS_ERR(ptr: etr_table))
705	return -ENOMEM;
706	etr_buf->hwaddr = etr_table->hwaddr;
707	etr_buf->mode = ETR_MODE_ETR_SG;
708	etr_buf->private = etr_table;
709	return 0;
710	}
711
712	static void tmc_etr_free_sg_buf(struct etr_buf *etr_buf)
713	{
714	struct etr_sg_table *etr_table = etr_buf->private;
715
716	if (etr_table) {
717	tmc_free_sg_table(etr_table->sg_table);
718	kfree(objp: etr_table);
719	}
720	}
721
722	static ssize_t tmc_etr_get_data_sg_buf(struct etr_buf *etr_buf, u64 offset,
723	size_t len, char **bufpp)
724	{
725	struct etr_sg_table *etr_table = etr_buf->private;
726
727	return tmc_sg_table_get_data(etr_table->sg_table, offset, len, bufpp);
728	}
729
730	static void tmc_etr_sync_sg_buf(struct etr_buf *etr_buf, u64 rrp, u64 rwp)
731	{
732	long r_offset, w_offset;
733	struct etr_sg_table *etr_table = etr_buf->private;
734	struct tmc_sg_table *table = etr_table->sg_table;
735
736	/* Convert hw address to offset in the buffer */
737	r_offset = tmc_sg_get_data_page_offset(sg_table: table, addr: rrp);
738	if (r_offset < 0) {
739	dev_warn(table->dev,
740	"Unable to map RRP %llx to offset\n", rrp);
741	etr_buf->len = 0;
742	return;
743	}
744
745	w_offset = tmc_sg_get_data_page_offset(sg_table: table, addr: rwp);
746	if (w_offset < 0) {
747	dev_warn(table->dev,
748	"Unable to map RWP %llx to offset\n", rwp);
749	etr_buf->len = 0;
750	return;
751	}
752
753	etr_buf->offset = r_offset;
754	if (etr_buf->full)
755	etr_buf->len = etr_buf->size;
756	else
757	etr_buf->len = ((w_offset < r_offset) ? etr_buf->size : 0) +
758	w_offset - r_offset;
759	tmc_sg_table_sync_data_range(table, r_offset, etr_buf->len);
760	}
761
762	static const struct etr_buf_operations etr_sg_buf_ops = {
763	.alloc = tmc_etr_alloc_sg_buf,
764	.free = tmc_etr_free_sg_buf,
765	.sync = tmc_etr_sync_sg_buf,
766	.get_data = tmc_etr_get_data_sg_buf,
767	};
768
769	/*
770	* TMC ETR could be connected to a CATU device, which can provide address
771	* translation service. This is represented by the Output port of the TMC
772	* (ETR) connected to the input port of the CATU.
773	*
774	* Returns : coresight_device ptr for the CATU device if a CATU is found.
775	* : NULL otherwise.
776	*/
777	struct coresight_device *
778	tmc_etr_get_catu_device(struct tmc_drvdata *drvdata)
779	{
780	struct coresight_device *etr = drvdata->csdev;
781	union coresight_dev_subtype catu_subtype = {
782	.helper_subtype = CORESIGHT_DEV_SUBTYPE_HELPER_CATU
783	};
784
785	if (!IS_ENABLED(CONFIG_CORESIGHT_CATU))
786	return NULL;
787
788	return coresight_find_output_type(pdata: etr->pdata, type: CORESIGHT_DEV_TYPE_HELPER,
789	subtype: catu_subtype);
790	}
791	EXPORT_SYMBOL_GPL(tmc_etr_get_catu_device);
792
793	static const struct etr_buf_operations *etr_buf_ops[] = {
794	[ETR_MODE_FLAT] = &etr_flat_buf_ops,
795	[ETR_MODE_ETR_SG] = &etr_sg_buf_ops,
796	[ETR_MODE_CATU] = NULL,
797	};
798
799	void tmc_etr_set_catu_ops(const struct etr_buf_operations *catu)
800	{
801	etr_buf_ops[ETR_MODE_CATU] = catu;
802	}
803	EXPORT_SYMBOL_GPL(tmc_etr_set_catu_ops);
804
805	void tmc_etr_remove_catu_ops(void)
806	{
807	etr_buf_ops[ETR_MODE_CATU] = NULL;
808	}
809	EXPORT_SYMBOL_GPL(tmc_etr_remove_catu_ops);
810
811	static inline int tmc_etr_mode_alloc_buf(int mode,
812	struct tmc_drvdata *drvdata,
813	struct etr_buf *etr_buf, int node,
814	void **pages)
815	{
816	int rc = -EINVAL;
817
818	switch (mode) {
819	case ETR_MODE_FLAT:
820	case ETR_MODE_ETR_SG:
821	case ETR_MODE_CATU:
822	if (etr_buf_ops[mode] && etr_buf_ops[mode]->alloc)
823	rc = etr_buf_ops[mode]->alloc(drvdata, etr_buf,
824	node, pages);
825	if (!rc)
826	etr_buf->ops = etr_buf_ops[mode];
827	return rc;
828	default:
829	return -EINVAL;
830	}
831	}
832
833	/*
834	* tmc_alloc_etr_buf: Allocate a buffer use by ETR.
835	* @drvdata : ETR device details.
836	* @size : size of the requested buffer.
837	* @flags : Required properties for the buffer.
838	* @node : Node for memory allocations.
839	* @pages : An optional list of pages.
840	*/
841	static struct etr_buf *tmc_alloc_etr_buf(struct tmc_drvdata *drvdata,
842	ssize_t size, int flags,
843	int node, void **pages)
844	{
845	int rc = -ENOMEM;
846	bool has_etr_sg, has_iommu;
847	bool has_sg, has_catu;
848	struct etr_buf *etr_buf;
849	struct device *dev = &drvdata->csdev->dev;
850
851	has_etr_sg = tmc_etr_has_cap(drvdata, TMC_ETR_SG);
852	has_iommu = iommu_get_domain_for_dev(dev: dev->parent);
853	has_catu = !!tmc_etr_get_catu_device(drvdata);
854
855	has_sg = has_catu || has_etr_sg;
856
857	etr_buf = kzalloc(size: sizeof(*etr_buf), GFP_KERNEL);
858	if (!etr_buf)
859	return ERR_PTR(error: -ENOMEM);
860
861	etr_buf->size = size;
862
863	/*
864	* If we have to use an existing list of pages, we cannot reliably
865	* use a contiguous DMA memory (even if we have an IOMMU). Otherwise,
866	* we use the contiguous DMA memory if at least one of the following
867	* conditions is true:
868	* a) The ETR cannot use Scatter-Gather.
869	* b) we have a backing IOMMU
870	* c) The requested memory size is smaller (< 1M).
871	*
872	* Fallback to available mechanisms.
873	*
874	*/
875	if (!pages &&
876	(!has_sg || has_iommu || size < SZ_1M))
877	rc = tmc_etr_mode_alloc_buf(mode: ETR_MODE_FLAT, drvdata,
878	etr_buf, node, pages);
879	if (rc && has_etr_sg)
880	rc = tmc_etr_mode_alloc_buf(mode: ETR_MODE_ETR_SG, drvdata,
881	etr_buf, node, pages);
882	if (rc && has_catu)
883	rc = tmc_etr_mode_alloc_buf(mode: ETR_MODE_CATU, drvdata,
884	etr_buf, node, pages);
885	if (rc) {
886	kfree(objp: etr_buf);
887	return ERR_PTR(error: rc);
888	}
889
890	refcount_set(r: &etr_buf->refcount, n: 1);
891	dev_dbg(dev, "allocated buffer of size %ldKB in mode %d\n",
892	(unsigned long)size >> 10, etr_buf->mode);
893	return etr_buf;
894	}
895
896	static void tmc_free_etr_buf(struct etr_buf *etr_buf)
897	{
898	WARN_ON(!etr_buf->ops || !etr_buf->ops->free);
899	etr_buf->ops->free(etr_buf);
900	kfree(objp: etr_buf);
901	}
902
903	/*
904	* tmc_etr_buf_get_data: Get the pointer the trace data at @offset
905	* with a maximum of @len bytes.
906	* Returns: The size of the linear data available @pos, with *bufpp
907	* updated to point to the buffer.
908	*/
909	static ssize_t tmc_etr_buf_get_data(struct etr_buf *etr_buf,
910	u64 offset, size_t len, char **bufpp)
911	{
912	/* Adjust the length to limit this transaction to end of buffer */
913	len = (len < (etr_buf->size - offset)) ? len : etr_buf->size - offset;
914
915	return etr_buf->ops->get_data(etr_buf, (u64)offset, len, bufpp);
916	}
917
918	static inline s64
919	tmc_etr_buf_insert_barrier_packet(struct etr_buf *etr_buf, u64 offset)
920	{
921	ssize_t len;
922	char *bufp;
923
924	len = tmc_etr_buf_get_data(etr_buf, offset,
925	CORESIGHT_BARRIER_PKT_SIZE, bufpp: &bufp);
926	if (WARN_ON(len < 0 || len < CORESIGHT_BARRIER_PKT_SIZE))
927	return -EINVAL;
928	coresight_insert_barrier_packet(buf: bufp);
929	return offset + CORESIGHT_BARRIER_PKT_SIZE;
930	}
931
932	/*
933	* tmc_sync_etr_buf: Sync the trace buffer availability with drvdata.
934	* Makes sure the trace data is synced to the memory for consumption.
935	* @etr_buf->offset will hold the offset to the beginning of the trace data
936	* within the buffer, with @etr_buf->len bytes to consume.
937	*/
938	static void tmc_sync_etr_buf(struct tmc_drvdata *drvdata)
939	{
940	struct etr_buf *etr_buf = drvdata->etr_buf;
941	u64 rrp, rwp;
942	u32 status;
943
944	rrp = tmc_read_rrp(drvdata);
945	rwp = tmc_read_rwp(drvdata);
946	status = readl_relaxed(drvdata->base + TMC_STS);
947
948	/*
949	* If there were memory errors in the session, truncate the
950	* buffer.
951	*/
952	if (WARN_ON_ONCE(status & TMC_STS_MEMERR)) {
953	dev_dbg(&drvdata->csdev->dev,
954	"tmc memory error detected, truncating buffer\n");
955	etr_buf->len = 0;
956	etr_buf->full = false;
957	return;
958	}
959
960	etr_buf->full = !!(status & TMC_STS_FULL);
961
962	WARN_ON(!etr_buf->ops || !etr_buf->ops->sync);
963
964	etr_buf->ops->sync(etr_buf, rrp, rwp);
965	}
966
967	static int __tmc_etr_enable_hw(struct tmc_drvdata *drvdata)
968	{
969	u32 axictl, sts;
970	struct etr_buf *etr_buf = drvdata->etr_buf;
971	int rc = 0;
972
973	CS_UNLOCK(addr: drvdata->base);
974
975	/* Wait for TMCSReady bit to be set */
976	rc = tmc_wait_for_tmcready(drvdata);
977	if (rc) {
978	dev_err(&drvdata->csdev->dev,
979	"Failed to enable : TMC not ready\n");
980	CS_LOCK(addr: drvdata->base);
981	return rc;
982	}
983
984	writel_relaxed(etr_buf->size / 4, drvdata->base + TMC_RSZ);
985	writel_relaxed(TMC_MODE_CIRCULAR_BUFFER, drvdata->base + TMC_MODE);
986
987	axictl = readl_relaxed(drvdata->base + TMC_AXICTL);
988	axictl &= ~TMC_AXICTL_CLEAR_MASK;
989	axictl |= TMC_AXICTL_PROT_CTL_B1;
990	axictl |= TMC_AXICTL_WR_BURST(drvdata->max_burst_size);
991	axictl |= TMC_AXICTL_AXCACHE_OS;
992
993	if (tmc_etr_has_cap(drvdata, TMC_ETR_AXI_ARCACHE)) {
994	axictl &= ~TMC_AXICTL_ARCACHE_MASK;
995	axictl |= TMC_AXICTL_ARCACHE_OS;
996	}
997
998	if (etr_buf->mode == ETR_MODE_ETR_SG)
999	axictl |= TMC_AXICTL_SCT_GAT_MODE;
1000
1001	writel_relaxed(axictl, drvdata->base + TMC_AXICTL);
1002	tmc_write_dba(drvdata, val: etr_buf->hwaddr);
1003	/*
1004	* If the TMC pointers must be programmed before the session,
1005	* we have to set it properly (i.e, RRP/RWP to base address and
1006	* STS to "not full").
1007	*/
1008	if (tmc_etr_has_cap(drvdata, TMC_ETR_SAVE_RESTORE)) {
1009	tmc_write_rrp(drvdata, val: etr_buf->hwaddr);
1010	tmc_write_rwp(drvdata, val: etr_buf->hwaddr);
1011	sts = readl_relaxed(drvdata->base + TMC_STS) & ~TMC_STS_FULL;
1012	writel_relaxed(sts, drvdata->base + TMC_STS);
1013	}
1014
1015	writel_relaxed(TMC_FFCR_EN_FMT | TMC_FFCR_EN_TI |
1016	TMC_FFCR_FON_FLIN | TMC_FFCR_FON_TRIG_EVT |
1017	TMC_FFCR_TRIGON_TRIGIN,
1018	drvdata->base + TMC_FFCR);
1019	writel_relaxed(drvdata->trigger_cntr, drvdata->base + TMC_TRG);
1020	tmc_enable_hw(drvdata);
1021
1022	CS_LOCK(addr: drvdata->base);
1023	return rc;
1024	}
1025
1026	static int tmc_etr_enable_hw(struct tmc_drvdata *drvdata,
1027	struct etr_buf *etr_buf)
1028	{
1029	int rc;
1030
1031	/* Callers should provide an appropriate buffer for use */
1032	if (WARN_ON(!etr_buf))
1033	return -EINVAL;
1034
1035	if ((etr_buf->mode == ETR_MODE_ETR_SG) &&
1036	WARN_ON(!tmc_etr_has_cap(drvdata, TMC_ETR_SG)))
1037	return -EINVAL;
1038
1039	if (WARN_ON(drvdata->etr_buf))
1040	return -EBUSY;
1041
1042	rc = coresight_claim_device(csdev: drvdata->csdev);
1043	if (!rc) {
1044	drvdata->etr_buf = etr_buf;
1045	rc = __tmc_etr_enable_hw(drvdata);
1046	if (rc) {
1047	drvdata->etr_buf = NULL;
1048	coresight_disclaim_device(csdev: drvdata->csdev);
1049	}
1050	}
1051
1052	return rc;
1053	}
1054
1055	/*
1056	* Return the available trace data in the buffer (starts at etr_buf->offset,
1057	* limited by etr_buf->len) from @pos, with a maximum limit of @len,
1058	* also updating the @bufpp on where to find it. Since the trace data
1059	* starts at anywhere in the buffer, depending on the RRP, we adjust the
1060	* @len returned to handle buffer wrapping around.
1061	*
1062	* We are protected here by drvdata->reading != 0, which ensures the
1063	* sysfs_buf stays alive.
1064	*/
1065	ssize_t tmc_etr_get_sysfs_trace(struct tmc_drvdata *drvdata,
1066	loff_t pos, size_t len, char **bufpp)
1067	{
1068	s64 offset;
1069	ssize_t actual = len;
1070	struct etr_buf *etr_buf = drvdata->sysfs_buf;
1071
1072	if (pos + actual > etr_buf->len)
1073	actual = etr_buf->len - pos;
1074	if (actual <= 0)
1075	return actual;
1076
1077	/* Compute the offset from which we read the data */
1078	offset = etr_buf->offset + pos;
1079	if (offset >= etr_buf->size)
1080	offset -= etr_buf->size;
1081	return tmc_etr_buf_get_data(etr_buf, offset, len: actual, bufpp);
1082	}
1083
1084	static struct etr_buf *
1085	tmc_etr_setup_sysfs_buf(struct tmc_drvdata *drvdata)
1086	{
1087	return tmc_alloc_etr_buf(drvdata, size: drvdata->size,
1088	flags: 0, cpu_to_node(cpu: 0), NULL);
1089	}
1090
1091	static void
1092	tmc_etr_free_sysfs_buf(struct etr_buf *buf)
1093	{
1094	if (buf)
1095	tmc_free_etr_buf(etr_buf: buf);
1096	}
1097
1098	static void tmc_etr_sync_sysfs_buf(struct tmc_drvdata *drvdata)
1099	{
1100	struct etr_buf *etr_buf = drvdata->etr_buf;
1101
1102	if (WARN_ON(drvdata->sysfs_buf != etr_buf)) {
1103	tmc_etr_free_sysfs_buf(buf: drvdata->sysfs_buf);
1104	drvdata->sysfs_buf = NULL;
1105	} else {
1106	tmc_sync_etr_buf(drvdata);
1107	/*
1108	* Insert barrier packets at the beginning, if there was
1109	* an overflow.
1110	*/
1111	if (etr_buf->full)
1112	tmc_etr_buf_insert_barrier_packet(etr_buf,
1113	offset: etr_buf->offset);
1114	}
1115	}
1116
1117	static void __tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1118	{
1119	CS_UNLOCK(addr: drvdata->base);
1120
1121	tmc_flush_and_stop(drvdata);
1122	/*
1123	* When operating in sysFS mode the content of the buffer needs to be
1124	* read before the TMC is disabled.
1125	*/
1126	if (drvdata->mode == CS_MODE_SYSFS)
1127	tmc_etr_sync_sysfs_buf(drvdata);
1128
1129	tmc_disable_hw(drvdata);
1130
1131	CS_LOCK(addr: drvdata->base);
1132
1133	}
1134
1135	void tmc_etr_disable_hw(struct tmc_drvdata *drvdata)
1136	{
1137	__tmc_etr_disable_hw(drvdata);
1138	coresight_disclaim_device(csdev: drvdata->csdev);
1139	/* Reset the ETR buf used by hardware */
1140	drvdata->etr_buf = NULL;
1141	}
1142
1143	static struct etr_buf *tmc_etr_get_sysfs_buffer(struct coresight_device *csdev)
1144	{
1145	int ret = 0;
1146	unsigned long flags;
1147	struct tmc_drvdata *drvdata = dev_get_drvdata(dev: csdev->dev.parent);
1148	struct etr_buf *sysfs_buf = NULL, *new_buf = NULL, *free_buf = NULL;
1149
1150	/*
1151	* If we are enabling the ETR from disabled state, we need to make
1152	* sure we have a buffer with the right size. The etr_buf is not reset
1153	* immediately after we stop the tracing in SYSFS mode as we wait for
1154	* the user to collect the data. We may be able to reuse the existing
1155	* buffer, provided the size matches. Any allocation has to be done
1156	* with the lock released.
1157	*/
1158	spin_lock_irqsave(&drvdata->spinlock, flags);
1159	sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1160	if (!sysfs_buf || (sysfs_buf->size != drvdata->size)) {
1161	spin_unlock_irqrestore(lock: &drvdata->spinlock, flags);
1162
1163	/* Allocate memory with the locks released */
1164	free_buf = new_buf = tmc_etr_setup_sysfs_buf(drvdata);
1165	if (IS_ERR(ptr: new_buf))
1166	return new_buf;
1167
1168	/* Let's try again */
1169	spin_lock_irqsave(&drvdata->spinlock, flags);
1170	}
1171
1172	if (drvdata->reading || drvdata->mode == CS_MODE_PERF) {
1173	ret = -EBUSY;
1174	goto out;
1175	}
1176
1177	/*
1178	* If we don't have a buffer or it doesn't match the requested size,
1179	* use the buffer allocated above. Otherwise reuse the existing buffer.
1180	*/
1181	sysfs_buf = READ_ONCE(drvdata->sysfs_buf);
1182	if (!sysfs_buf || (new_buf && sysfs_buf->size != new_buf->size)) {
1183	free_buf = sysfs_buf;
1184	drvdata->sysfs_buf = new_buf;
1185	}
1186
1187	out:
1188	spin_unlock_irqrestore(lock: &drvdata->spinlock, flags);
1189
1190	/* Free memory outside the spinlock if need be */
1191	if (free_buf)
1192	tmc_etr_free_sysfs_buf(buf: free_buf);
1193	return ret ? ERR_PTR(error: ret) : drvdata->sysfs_buf;
1194	}
1195
1196	static int tmc_enable_etr_sink_sysfs(struct coresight_device *csdev)
1197	{
1198	int ret = 0;
1199	unsigned long flags;
1200	struct tmc_drvdata *drvdata = dev_get_drvdata(dev: csdev->dev.parent);
1201	struct etr_buf *sysfs_buf = tmc_etr_get_sysfs_buffer(csdev);
1202
1203	if (IS_ERR(ptr: sysfs_buf))
1204	return PTR_ERR(ptr: sysfs_buf);
1205
1206	spin_lock_irqsave(&drvdata->spinlock, flags);
1207
1208	/*
1209	* In sysFS mode we can have multiple writers per sink. Since this
1210	* sink is already enabled no memory is needed and the HW need not be
1211	* touched, even if the buffer size has changed.
1212	*/
1213	if (drvdata->mode == CS_MODE_SYSFS) {
1214	atomic_inc(v: &csdev->refcnt);
1215	goto out;
1216	}
1217
1218	ret = tmc_etr_enable_hw(drvdata, etr_buf: sysfs_buf);
1219	if (!ret) {
1220	drvdata->mode = CS_MODE_SYSFS;
1221	atomic_inc(v: &csdev->refcnt);
1222	}
1223
1224	out:
1225	spin_unlock_irqrestore(lock: &drvdata->spinlock, flags);
1226
1227	if (!ret)
1228	dev_dbg(&csdev->dev, "TMC-ETR enabled\n");
1229
1230	return ret;
1231	}
1232
1233	struct etr_buf *tmc_etr_get_buffer(struct coresight_device *csdev,
1234	enum cs_mode mode, void *data)
1235	{
1236	struct perf_output_handle *handle = data;
1237	struct etr_perf_buffer *etr_perf;
1238
1239	switch (mode) {
1240	case CS_MODE_SYSFS:
1241	return tmc_etr_get_sysfs_buffer(csdev);
1242	case CS_MODE_PERF:
1243	etr_perf = etm_perf_sink_config(handle);
1244	if (WARN_ON(!etr_perf || !etr_perf->etr_buf))
1245	return ERR_PTR(error: -EINVAL);
1246	return etr_perf->etr_buf;
1247	default:
1248	return ERR_PTR(error: -EINVAL);
1249	}
1250	}
1251	EXPORT_SYMBOL_GPL(tmc_etr_get_buffer);
1252
1253	/*
1254	* alloc_etr_buf: Allocate ETR buffer for use by perf.
1255	* The size of the hardware buffer is dependent on the size configured
1256	* via sysfs and the perf ring buffer size. We prefer to allocate the
1257	* largest possible size, scaling down the size by half until it
1258	* reaches a minimum limit (1M), beyond which we give up.
1259	*/
1260	static struct etr_buf *
1261	alloc_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1262	int nr_pages, void **pages, bool snapshot)
1263	{
1264	int node;
1265	struct etr_buf *etr_buf;
1266	unsigned long size;
1267
1268	node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(cpu: event->cpu);
1269	/*
1270	* Try to match the perf ring buffer size if it is larger
1271	* than the size requested via sysfs.
1272	*/
1273	if ((nr_pages << PAGE_SHIFT) > drvdata->size) {
1274	etr_buf = tmc_alloc_etr_buf(drvdata, size: ((ssize_t)nr_pages << PAGE_SHIFT),
1275	flags: 0, node, NULL);
1276	if (!IS_ERR(ptr: etr_buf))
1277	goto done;
1278	}
1279
1280	/*
1281	* Else switch to configured size for this ETR
1282	* and scale down until we hit the minimum limit.
1283	*/
1284	size = drvdata->size;
1285	do {
1286	etr_buf = tmc_alloc_etr_buf(drvdata, size, flags: 0, node, NULL);
1287	if (!IS_ERR(ptr: etr_buf))
1288	goto done;
1289	size /= 2;
1290	} while (size >= TMC_ETR_PERF_MIN_BUF_SIZE);
1291
1292	return ERR_PTR(error: -ENOMEM);
1293
1294	done:
1295	return etr_buf;
1296	}
1297
1298	static struct etr_buf *
1299	get_perf_etr_buf_cpu_wide(struct tmc_drvdata *drvdata,
1300	struct perf_event *event, int nr_pages,
1301	void **pages, bool snapshot)
1302	{
1303	int ret;
1304	pid_t pid = task_pid_nr(tsk: event->owner);
1305	struct etr_buf *etr_buf;
1306
1307	retry:
1308	/*
1309	* An etr_perf_buffer is associated with an event and holds a reference
1310	* to the AUX ring buffer that was created for that event. In CPU-wide
1311	* N:1 mode multiple events (one per CPU), each with its own AUX ring
1312	* buffer, share a sink. As such an etr_perf_buffer is created for each
1313	* event but a single etr_buf associated with the ETR is shared between
1314	* them. The last event in a trace session will copy the content of the
1315	* etr_buf to its AUX ring buffer. Ring buffer associated to other
1316	* events are simply not used an freed as events are destoyed. We still
1317	* need to allocate a ring buffer for each event since we don't know
1318	* which event will be last.
1319	*/
1320
1321	/*
1322	* The first thing to do here is check if an etr_buf has already been
1323	* allocated for this session. If so it is shared with this event,
1324	* otherwise it is created.
1325	*/
1326	mutex_lock(&drvdata->idr_mutex);
1327	etr_buf = idr_find(&drvdata->idr, id: pid);
1328	if (etr_buf) {
1329	refcount_inc(r: &etr_buf->refcount);
1330	mutex_unlock(lock: &drvdata->idr_mutex);
1331	return etr_buf;
1332	}
1333
1334	/* If we made it here no buffer has been allocated, do so now. */
1335	mutex_unlock(lock: &drvdata->idr_mutex);
1336
1337	etr_buf = alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1338	if (IS_ERR(ptr: etr_buf))
1339	return etr_buf;
1340
1341	/* Now that we have a buffer, add it to the IDR. */
1342	mutex_lock(&drvdata->idr_mutex);
1343	ret = idr_alloc(&drvdata->idr, ptr: etr_buf, start: pid, end: pid + 1, GFP_KERNEL);
1344	mutex_unlock(lock: &drvdata->idr_mutex);
1345
1346	/* Another event with this session ID has allocated this buffer. */
1347	if (ret == -ENOSPC) {
1348	tmc_free_etr_buf(etr_buf);
1349	goto retry;
1350	}
1351
1352	/* The IDR can't allocate room for a new session, abandon ship. */
1353	if (ret == -ENOMEM) {
1354	tmc_free_etr_buf(etr_buf);
1355	return ERR_PTR(error: ret);
1356	}
1357
1358
1359	return etr_buf;
1360	}
1361
1362	static struct etr_buf *
1363	get_perf_etr_buf_per_thread(struct tmc_drvdata *drvdata,
1364	struct perf_event *event, int nr_pages,
1365	void **pages, bool snapshot)
1366	{
1367	/*
1368	* In per-thread mode the etr_buf isn't shared, so just go ahead
1369	* with memory allocation.
1370	*/
1371	return alloc_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1372	}
1373
1374	static struct etr_buf *
1375	get_perf_etr_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1376	int nr_pages, void **pages, bool snapshot)
1377	{
1378	if (event->cpu == -1)
1379	return get_perf_etr_buf_per_thread(drvdata, event, nr_pages,
1380	pages, snapshot);
1381
1382	return get_perf_etr_buf_cpu_wide(drvdata, event, nr_pages,
1383	pages, snapshot);
1384	}
1385
1386	static struct etr_perf_buffer *
1387	tmc_etr_setup_perf_buf(struct tmc_drvdata *drvdata, struct perf_event *event,
1388	int nr_pages, void **pages, bool snapshot)
1389	{
1390	int node;
1391	struct etr_buf *etr_buf;
1392	struct etr_perf_buffer *etr_perf;
1393
1394	node = (event->cpu == -1) ? NUMA_NO_NODE : cpu_to_node(cpu: event->cpu);
1395
1396	etr_perf = kzalloc_node(size: sizeof(*etr_perf), GFP_KERNEL, node);
1397	if (!etr_perf)
1398	return ERR_PTR(error: -ENOMEM);
1399
1400	etr_buf = get_perf_etr_buf(drvdata, event, nr_pages, pages, snapshot);
1401	if (!IS_ERR(ptr: etr_buf))
1402	goto done;
1403
1404	kfree(objp: etr_perf);
1405	return ERR_PTR(error: -ENOMEM);
1406
1407	done:
1408	/*
1409	* Keep a reference to the ETR this buffer has been allocated for
1410	* in order to have access to the IDR in tmc_free_etr_buffer().
1411	*/
1412	etr_perf->drvdata = drvdata;
1413	etr_perf->etr_buf = etr_buf;
1414
1415	return etr_perf;
1416	}
1417
1418
1419	static void *tmc_alloc_etr_buffer(struct coresight_device *csdev,
1420	struct perf_event *event, void **pages,
1421	int nr_pages, bool snapshot)
1422	{
1423	struct etr_perf_buffer *etr_perf;
1424	struct tmc_drvdata *drvdata = dev_get_drvdata(dev: csdev->dev.parent);
1425
1426	etr_perf = tmc_etr_setup_perf_buf(drvdata, event,
1427	nr_pages, pages, snapshot);
1428	if (IS_ERR(ptr: etr_perf)) {
1429	dev_dbg(&csdev->dev, "Unable to allocate ETR buffer\n");
1430	return NULL;
1431	}
1432
1433	etr_perf->pid = task_pid_nr(tsk: event->owner);
1434	etr_perf->snapshot = snapshot;
1435	etr_perf->nr_pages = nr_pages;
1436	etr_perf->pages = pages;
1437
1438	return etr_perf;
1439	}
1440
1441	static void tmc_free_etr_buffer(void *config)
1442	{
1443	struct etr_perf_buffer *etr_perf = config;
1444	struct tmc_drvdata *drvdata = etr_perf->drvdata;
1445	struct etr_buf *buf, *etr_buf = etr_perf->etr_buf;
1446
1447	if (!etr_buf)
1448	goto free_etr_perf_buffer;
1449
1450	mutex_lock(&drvdata->idr_mutex);
1451	/* If we are not the last one to use the buffer, don't touch it. */
1452	if (!refcount_dec_and_test(r: &etr_buf->refcount)) {
1453	mutex_unlock(lock: &drvdata->idr_mutex);
1454	goto free_etr_perf_buffer;
1455	}
1456
1457	/* We are the last one, remove from the IDR and free the buffer. */
1458	buf = idr_remove(&drvdata->idr, id: etr_perf->pid);
1459	mutex_unlock(lock: &drvdata->idr_mutex);
1460
1461	/*
1462	* Something went very wrong if the buffer associated with this ID
1463	* is not the same in the IDR. Leak to avoid use after free.
1464	*/
1465	if (buf && WARN_ON(buf != etr_buf))
1466	goto free_etr_perf_buffer;
1467
1468	tmc_free_etr_buf(etr_buf: etr_perf->etr_buf);
1469
1470	free_etr_perf_buffer:
1471	kfree(objp: etr_perf);
1472	}
1473
1474	/*
1475	* tmc_etr_sync_perf_buffer: Copy the actual trace data from the hardware
1476	* buffer to the perf ring buffer.
1477	*/
1478	static void tmc_etr_sync_perf_buffer(struct etr_perf_buffer *etr_perf,
1479	unsigned long head,
1480	unsigned long src_offset,
1481	unsigned long to_copy)
1482	{
1483	long bytes;
1484	long pg_idx, pg_offset;
1485	char **dst_pages, *src_buf;
1486	struct etr_buf *etr_buf = etr_perf->etr_buf;
1487
1488	head = PERF_IDX2OFF(head, etr_perf);
1489	pg_idx = head >> PAGE_SHIFT;
1490	pg_offset = head & (PAGE_SIZE - 1);
1491	dst_pages = (char **)etr_perf->pages;
1492
1493	while (to_copy > 0) {
1494	/*
1495	* In one iteration, we can copy minimum of :
1496	* 1) what is available in the source buffer,
1497	* 2) what is available in the source buffer, before it
1498	* wraps around.
1499	* 3) what is available in the destination page.
1500	* in one iteration.
1501	*/
1502	if (src_offset >= etr_buf->size)
1503	src_offset -= etr_buf->size;
1504	bytes = tmc_etr_buf_get_data(etr_buf, offset: src_offset, len: to_copy,
1505	bufpp: &src_buf);
1506	if (WARN_ON_ONCE(bytes <= 0))
1507	break;
1508	bytes = min(bytes, (long)(PAGE_SIZE - pg_offset));
1509
1510	memcpy(dst_pages[pg_idx] + pg_offset, src_buf, bytes);
1511
1512	to_copy -= bytes;
1513
1514	/* Move destination pointers */
1515	pg_offset += bytes;
1516	if (pg_offset == PAGE_SIZE) {
1517	pg_offset = 0;
1518	if (++pg_idx == etr_perf->nr_pages)
1519	pg_idx = 0;
1520	}
1521
1522	/* Move source pointers */
1523	src_offset += bytes;
1524	}
1525	}
1526
1527	/*
1528	* tmc_update_etr_buffer : Update the perf ring buffer with the
1529	* available trace data. We use software double buffering at the moment.
1530	*
1531	* TODO: Add support for reusing the perf ring buffer.
1532	*/
1533	static unsigned long
1534	tmc_update_etr_buffer(struct coresight_device *csdev,
1535	struct perf_output_handle *handle,
1536	void *config)
1537	{
1538	bool lost = false;
1539	unsigned long flags, offset, size = 0;
1540	struct tmc_drvdata *drvdata = dev_get_drvdata(dev: csdev->dev.parent);
1541	struct etr_perf_buffer *etr_perf = config;
1542	struct etr_buf *etr_buf = etr_perf->etr_buf;
1543
1544	spin_lock_irqsave(&drvdata->spinlock, flags);
1545
1546	/* Don't do anything if another tracer is using this sink */
1547	if (atomic_read(v: &csdev->refcnt) != 1) {
1548	spin_unlock_irqrestore(lock: &drvdata->spinlock, flags);
1549	goto out;
1550	}
1551
1552	if (WARN_ON(drvdata->perf_buf != etr_buf)) {
1553	lost = true;
1554	spin_unlock_irqrestore(lock: &drvdata->spinlock, flags);
1555	goto out;
1556	}
1557
1558	CS_UNLOCK(addr: drvdata->base);
1559
1560	tmc_flush_and_stop(drvdata);
1561	tmc_sync_etr_buf(drvdata);
1562
1563	CS_LOCK(addr: drvdata->base);
1564	spin_unlock_irqrestore(lock: &drvdata->spinlock, flags);
1565
1566	lost = etr_buf->full;
1567	offset = etr_buf->offset;
1568	size = etr_buf->len;
1569
1570	/*
1571	* The ETR buffer may be bigger than the space available in the
1572	* perf ring buffer (handle->size). If so advance the offset so that we
1573	* get the latest trace data. In snapshot mode none of that matters
1574	* since we are expected to clobber stale data in favour of the latest
1575	* traces.
1576	*/
1577	if (!etr_perf->snapshot && size > handle->size) {
1578	u32 mask = tmc_get_memwidth_mask(drvdata);
1579
1580	/*
1581	* Make sure the new size is aligned in accordance with the
1582	* requirement explained in function tmc_get_memwidth_mask().
1583	*/
1584	size = handle->size & mask;
1585	offset = etr_buf->offset + etr_buf->len - size;
1586
1587	if (offset >= etr_buf->size)
1588	offset -= etr_buf->size;
1589	lost = true;
1590	}
1591
1592	/* Insert barrier packets at the beginning, if there was an overflow */
1593	if (lost)
1594	tmc_etr_buf_insert_barrier_packet(etr_buf, offset);
1595	tmc_etr_sync_perf_buffer(etr_perf, head: handle->head, src_offset: offset, to_copy: size);
1596
1597	/*
1598	* In snapshot mode we simply increment the head by the number of byte
1599	* that were written. User space will figure out how many bytes to get
1600	* from the AUX buffer based on the position of the head.
1601	*/
1602	if (etr_perf->snapshot)
1603	handle->head += size;
1604
1605	/*
1606	* Ensure that the AUX trace data is visible before the aux_head
1607	* is updated via perf_aux_output_end(), as expected by the
1608	* perf ring buffer.
1609	*/
1610	smp_wmb();
1611
1612	out:
1613	/*
1614	* Don't set the TRUNCATED flag in snapshot mode because 1) the
1615	* captured buffer is expected to be truncated and 2) a full buffer
1616	* prevents the event from being re-enabled by the perf core,
1617	* resulting in stale data being send to user space.
1618	*/
1619	if (!etr_perf->snapshot && lost)
1620	perf_aux_output_flag(handle, PERF_AUX_FLAG_TRUNCATED);
1621	return size;
1622	}
1623
1624	static int tmc_enable_etr_sink_perf(struct coresight_device *csdev, void *data)
1625	{
1626	int rc = 0;
1627	pid_t pid;
1628	unsigned long flags;
1629	struct tmc_drvdata *drvdata = dev_get_drvdata(dev: csdev->dev.parent);
1630	struct perf_output_handle *handle = data;
1631	struct etr_perf_buffer *etr_perf = etm_perf_sink_config(handle);
1632
1633	spin_lock_irqsave(&drvdata->spinlock, flags);
1634	/* Don't use this sink if it is already claimed by sysFS */
1635	if (drvdata->mode == CS_MODE_SYSFS) {
1636	rc = -EBUSY;
1637	goto unlock_out;
1638	}
1639
1640	if (WARN_ON(!etr_perf || !etr_perf->etr_buf)) {
1641	rc = -EINVAL;
1642	goto unlock_out;
1643	}
1644
1645	/* Get a handle on the pid of the process to monitor */
1646	pid = etr_perf->pid;
1647
1648	/* Do not proceed if this device is associated with another session */
1649	if (drvdata->pid != -1 && drvdata->pid != pid) {
1650	rc = -EBUSY;
1651	goto unlock_out;
1652	}
1653
1654	/*
1655	* No HW configuration is needed if the sink is already in
1656	* use for this session.
1657	*/
1658	if (drvdata->pid == pid) {
1659	atomic_inc(v: &csdev->refcnt);
1660	goto unlock_out;
1661	}
1662
1663	rc = tmc_etr_enable_hw(drvdata, etr_buf: etr_perf->etr_buf);
1664	if (!rc) {
1665	/* Associate with monitored process. */
1666	drvdata->pid = pid;
1667	drvdata->mode = CS_MODE_PERF;
1668	drvdata->perf_buf = etr_perf->etr_buf;
1669	atomic_inc(v: &csdev->refcnt);
1670	}
1671
1672	unlock_out:
1673	spin_unlock_irqrestore(lock: &drvdata->spinlock, flags);
1674	return rc;
1675	}
1676
1677	static int tmc_enable_etr_sink(struct coresight_device *csdev,
1678	enum cs_mode mode, void *data)
1679	{
1680	switch (mode) {
1681	case CS_MODE_SYSFS:
1682	return tmc_enable_etr_sink_sysfs(csdev);
1683	case CS_MODE_PERF:
1684	return tmc_enable_etr_sink_perf(csdev, data);
1685	default:
1686	return -EINVAL;
1687	}
1688	}
1689
1690	static int tmc_disable_etr_sink(struct coresight_device *csdev)
1691	{
1692	unsigned long flags;
1693	struct tmc_drvdata *drvdata = dev_get_drvdata(dev: csdev->dev.parent);
1694
1695	spin_lock_irqsave(&drvdata->spinlock, flags);
1696
1697	if (drvdata->reading) {
1698	spin_unlock_irqrestore(lock: &drvdata->spinlock, flags);
1699	return -EBUSY;
1700	}
1701
1702	if (atomic_dec_return(v: &csdev->refcnt)) {
1703	spin_unlock_irqrestore(lock: &drvdata->spinlock, flags);
1704	return -EBUSY;
1705	}
1706
1707	/* Complain if we (somehow) got out of sync */
1708	WARN_ON_ONCE(drvdata->mode == CS_MODE_DISABLED);
1709	tmc_etr_disable_hw(drvdata);
1710	/* Dissociate from monitored process. */
1711	drvdata->pid = -1;
1712	drvdata->mode = CS_MODE_DISABLED;
1713	/* Reset perf specific data */
1714	drvdata->perf_buf = NULL;
1715
1716	spin_unlock_irqrestore(lock: &drvdata->spinlock, flags);
1717
1718	dev_dbg(&csdev->dev, "TMC-ETR disabled\n");
1719	return 0;
1720	}
1721
1722	static const struct coresight_ops_sink tmc_etr_sink_ops = {
1723	.enable = tmc_enable_etr_sink,
1724	.disable = tmc_disable_etr_sink,
1725	.alloc_buffer = tmc_alloc_etr_buffer,
1726	.update_buffer = tmc_update_etr_buffer,
1727	.free_buffer = tmc_free_etr_buffer,
1728	};
1729
1730	const struct coresight_ops tmc_etr_cs_ops = {
1731	.sink_ops = &tmc_etr_sink_ops,
1732	};
1733
1734	int tmc_read_prepare_etr(struct tmc_drvdata *drvdata)
1735	{
1736	int ret = 0;
1737	unsigned long flags;
1738
1739	/* config types are set a boot time and never change */
1740	if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1741	return -EINVAL;
1742
1743	spin_lock_irqsave(&drvdata->spinlock, flags);
1744	if (drvdata->reading) {
1745	ret = -EBUSY;
1746	goto out;
1747	}
1748
1749	/*
1750	* We can safely allow reads even if the ETR is operating in PERF mode,
1751	* since the sysfs session is captured in mode specific data.
1752	* If drvdata::sysfs_data is NULL the trace data has been read already.
1753	*/
1754	if (!drvdata->sysfs_buf) {
1755	ret = -EINVAL;
1756	goto out;
1757	}
1758
1759	/* Disable the TMC if we are trying to read from a running session. */
1760	if (drvdata->mode == CS_MODE_SYSFS)
1761	__tmc_etr_disable_hw(drvdata);
1762
1763	drvdata->reading = true;
1764	out:
1765	spin_unlock_irqrestore(lock: &drvdata->spinlock, flags);
1766
1767	return ret;
1768	}
1769
1770	int tmc_read_unprepare_etr(struct tmc_drvdata *drvdata)
1771	{
1772	unsigned long flags;
1773	struct etr_buf *sysfs_buf = NULL;
1774
1775	/* config types are set a boot time and never change */
1776	if (WARN_ON_ONCE(drvdata->config_type != TMC_CONFIG_TYPE_ETR))
1777	return -EINVAL;
1778
1779	spin_lock_irqsave(&drvdata->spinlock, flags);
1780
1781	/* RE-enable the TMC if need be */
1782	if (drvdata->mode == CS_MODE_SYSFS) {
1783	/*
1784	* The trace run will continue with the same allocated trace
1785	* buffer. Since the tracer is still enabled drvdata::buf can't
1786	* be NULL.
1787	*/
1788	__tmc_etr_enable_hw(drvdata);
1789	} else {
1790	/*
1791	* The ETR is not tracing and the buffer was just read.
1792	* As such prepare to free the trace buffer.
1793	*/
1794	sysfs_buf = drvdata->sysfs_buf;
1795	drvdata->sysfs_buf = NULL;
1796	}
1797
1798	drvdata->reading = false;
1799	spin_unlock_irqrestore(lock: &drvdata->spinlock, flags);
1800
1801	/* Free allocated memory out side of the spinlock */
1802	if (sysfs_buf)
1803	tmc_etr_free_sysfs_buf(buf: sysfs_buf);
1804
1805	return 0;
1806	}
1807