1	// SPDX-License-Identifier: GPL-2.0
2	// Copyright (C) 2018 Intel Corporation
3
4	#include <linux/device.h>
5	#include <linux/iopoll.h>
6	#include <linux/slab.h>
7
8	#include "ipu3.h"
9	#include "ipu3-css.h"
10	#include "ipu3-css-fw.h"
11	#include "ipu3-css-params.h"
12	#include "ipu3-dmamap.h"
13	#include "ipu3-tables.h"
14
15	/* IRQ configuration */
16	#define IMGU_IRQCTRL_IRQ_MASK (IMGU_IRQCTRL_IRQ_SP1 | \
17	IMGU_IRQCTRL_IRQ_SP2 | \
18	IMGU_IRQCTRL_IRQ_SW_PIN(0) | \
19	IMGU_IRQCTRL_IRQ_SW_PIN(1))
20
21	#define IPU3_CSS_FORMAT_BPP_DEN 50 /* Denominator */
22
23	/* Some sane limits for resolutions */
24	#define IPU3_CSS_MIN_RES 32
25	#define IPU3_CSS_MAX_H 3136
26	#define IPU3_CSS_MAX_W 4224
27
28	/* minimal envelope size(GDC in - out) should be 4 */
29	#define MIN_ENVELOPE 4
30
31	/*
32	* pre-allocated buffer size for CSS ABI, auxiliary frames
33	* after BDS and before GDC. Those values should be tuned
34	* to big enough to avoid buffer re-allocation when
35	* streaming to lower streaming latency.
36	*/
37	#define CSS_ABI_SIZE 136
38	#define CSS_BDS_SIZE (4480 * 3200 * 3)
39	#define CSS_GDC_SIZE (4224 * 3200 * 12 / 8)
40
41	#define IPU3_CSS_QUEUE_TO_FLAGS(q) (1 << (q))
42	#define IPU3_CSS_FORMAT_FL_IN \
43	IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_IN)
44	#define IPU3_CSS_FORMAT_FL_OUT \
45	IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_OUT)
46	#define IPU3_CSS_FORMAT_FL_VF \
47	IPU3_CSS_QUEUE_TO_FLAGS(IPU3_CSS_QUEUE_VF)
48
49	/* Formats supported by IPU3 Camera Sub System */
50	static const struct imgu_css_format imgu_css_formats[] = {
51	{
52	.pixelformat = V4L2_PIX_FMT_NV12,
53	.colorspace = V4L2_COLORSPACE_SRGB,
54	.frame_format = IMGU_ABI_FRAME_FORMAT_NV12,
55	.osys_format = IMGU_ABI_OSYS_FORMAT_NV12,
56	.osys_tiling = IMGU_ABI_OSYS_TILING_NONE,
57	.chroma_decim = 4,
58	.width_align = IPU3_UAPI_ISP_VEC_ELEMS,
59	.flags = IPU3_CSS_FORMAT_FL_OUT | IPU3_CSS_FORMAT_FL_VF,
60	}, {
61	/* Each 32 bytes contains 25 10-bit pixels */
62	.pixelformat = V4L2_PIX_FMT_IPU3_SBGGR10,
63	.colorspace = V4L2_COLORSPACE_RAW,
64	.frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
65	.bayer_order = IMGU_ABI_BAYER_ORDER_BGGR,
66	.bit_depth = 10,
67	.width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
68	.flags = IPU3_CSS_FORMAT_FL_IN,
69	}, {
70	.pixelformat = V4L2_PIX_FMT_IPU3_SGBRG10,
71	.colorspace = V4L2_COLORSPACE_RAW,
72	.frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
73	.bayer_order = IMGU_ABI_BAYER_ORDER_GBRG,
74	.bit_depth = 10,
75	.width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
76	.flags = IPU3_CSS_FORMAT_FL_IN,
77	}, {
78	.pixelformat = V4L2_PIX_FMT_IPU3_SGRBG10,
79	.colorspace = V4L2_COLORSPACE_RAW,
80	.frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
81	.bayer_order = IMGU_ABI_BAYER_ORDER_GRBG,
82	.bit_depth = 10,
83	.width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
84	.flags = IPU3_CSS_FORMAT_FL_IN,
85	}, {
86	.pixelformat = V4L2_PIX_FMT_IPU3_SRGGB10,
87	.colorspace = V4L2_COLORSPACE_RAW,
88	.frame_format = IMGU_ABI_FRAME_FORMAT_RAW_PACKED,
89	.bayer_order = IMGU_ABI_BAYER_ORDER_RGGB,
90	.bit_depth = 10,
91	.width_align = 2 * IPU3_UAPI_ISP_VEC_ELEMS,
92	.flags = IPU3_CSS_FORMAT_FL_IN,
93	},
94	};
95
96	static const struct {
97	enum imgu_abi_queue_id qid;
98	size_t ptr_ofs;
99	} imgu_css_queues[IPU3_CSS_QUEUES] = {
100	[IPU3_CSS_QUEUE_IN] = {
101	IMGU_ABI_QUEUE_C_ID,
102	offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
103	},
104	[IPU3_CSS_QUEUE_OUT] = {
105	IMGU_ABI_QUEUE_D_ID,
106	offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
107	},
108	[IPU3_CSS_QUEUE_VF] = {
109	IMGU_ABI_QUEUE_E_ID,
110	offsetof(struct imgu_abi_buffer, payload.frame.frame_data)
111	},
112	[IPU3_CSS_QUEUE_STAT_3A] = {
113	IMGU_ABI_QUEUE_F_ID,
114	offsetof(struct imgu_abi_buffer, payload.s3a.data_ptr)
115	},
116	};
117
118	/* Initialize queue based on given format, adjust format as needed */
119	static int imgu_css_queue_init(struct imgu_css_queue *queue,
120	struct v4l2_pix_format_mplane *fmt, u32 flags)
121	{
122	struct v4l2_pix_format_mplane *const f = &queue->fmt.mpix;
123	unsigned int i;
124	u32 sizeimage;
125
126	INIT_LIST_HEAD(list: &queue->bufs);
127
128	queue->css_fmt = NULL; /* Disable */
129	if (!fmt)
130	return 0;
131
132	for (i = 0; i < ARRAY_SIZE(imgu_css_formats); i++) {
133	if (!(imgu_css_formats[i].flags & flags))
134	continue;
135	queue->css_fmt = &imgu_css_formats[i];
136	if (imgu_css_formats[i].pixelformat == fmt->pixelformat)
137	break;
138	}
139	if (!queue->css_fmt)
140	return -EINVAL; /* Could not find any suitable format */
141
142	queue->fmt.mpix = *fmt;
143
144	f->width = ALIGN(clamp_t(u32, f->width,
145	IPU3_CSS_MIN_RES, IPU3_CSS_MAX_W), 2);
146	f->height = ALIGN(clamp_t(u32, f->height,
147	IPU3_CSS_MIN_RES, IPU3_CSS_MAX_H), 2);
148	queue->width_pad = ALIGN(f->width, queue->css_fmt->width_align);
149	f->plane_fmt[0].bytesperline =
150	imgu_bytesperline(width: f->width, frame_format: queue->css_fmt->frame_format);
151	sizeimage = f->height * f->plane_fmt[0].bytesperline;
152	if (queue->css_fmt->chroma_decim)
153	sizeimage += 2 * sizeimage / queue->css_fmt->chroma_decim;
154
155	f->plane_fmt[0].sizeimage = sizeimage;
156	f->field = V4L2_FIELD_NONE;
157	f->num_planes = 1;
158	f->colorspace = queue->css_fmt->colorspace;
159	f->flags = 0;
160	f->ycbcr_enc = V4L2_YCBCR_ENC_DEFAULT;
161	f->quantization = V4L2_QUANTIZATION_DEFAULT;
162	f->xfer_func = V4L2_XFER_FUNC_DEFAULT;
163	memset(f->reserved, 0, sizeof(f->reserved));
164
165	return 0;
166	}
167
168	static bool imgu_css_queue_enabled(struct imgu_css_queue *q)
169	{
170	return q->css_fmt;
171	}
172
173	/******************* css hw *******************/
174
175	/* In the style of writesl() defined in include/asm-generic/io.h */
176	static inline void writes(const void *mem, ssize_t count, void __iomem *addr)
177	{
178	if (count >= 4) {
179	const u32 *buf = mem;
180
181	count /= 4;
182	do {
183	writel(val: *buf++, addr);
184	addr += 4;
185	} while (--count);
186	}
187	}
188
189	/* Wait until register `reg', masked with `mask', becomes `cmp' */
190	static int imgu_hw_wait(void __iomem *base, int reg, u32 mask, u32 cmp)
191	{
192	u32 val;
193
194	return readl_poll_timeout(base + reg, val, (val & mask) == cmp,
195	1000, 100 * 1000);
196	}
197
198	/* Initialize the IPU3 CSS hardware and associated h/w blocks */
199
200	int imgu_css_set_powerup(struct device *dev, void __iomem *base,
201	unsigned int freq)
202	{
203	u32 pm_ctrl, state, val;
204
205	dev_dbg(dev, "%s with freq %u\n", __func__, freq);
206	/* Clear the CSS busy signal */
207	readl(addr: base + IMGU_REG_GP_BUSY);
208	writel(val: 0, addr: base + IMGU_REG_GP_BUSY);
209
210	/* Wait for idle signal */
211	if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
212	IMGU_STATE_IDLE_STS)) {
213	dev_err(dev, "failed to set CSS idle\n");
214	goto fail;
215	}
216
217	/* Reset the css */
218	writel(readl(addr: base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET,
219	addr: base + IMGU_REG_PM_CTRL);
220
221	usleep_range(min: 200, max: 300);
222
223	/** Prepare CSS */
224
225	pm_ctrl = readl(addr: base + IMGU_REG_PM_CTRL);
226	state = readl(addr: base + IMGU_REG_STATE);
227
228	dev_dbg(dev, "CSS pm_ctrl 0x%x state 0x%x (power %s)\n",
229	pm_ctrl, state, state & IMGU_STATE_POWER_DOWN ? "down" : "up");
230
231	/* Power up CSS using wrapper */
232	if (state & IMGU_STATE_POWER_DOWN) {
233	writel(IMGU_PM_CTRL_RACE_TO_HALT | IMGU_PM_CTRL_START,
234	addr: base + IMGU_REG_PM_CTRL);
235	if (imgu_hw_wait(base, IMGU_REG_PM_CTRL,
236	IMGU_PM_CTRL_START, cmp: 0)) {
237	dev_err(dev, "failed to power up CSS\n");
238	goto fail;
239	}
240	usleep_range(min: 2000, max: 3000);
241	} else {
242	writel(IMGU_PM_CTRL_RACE_TO_HALT, addr: base + IMGU_REG_PM_CTRL);
243	}
244
245	/* Set the busy bit */
246	writel(readl(addr: base + IMGU_REG_GP_BUSY) | 1, addr: base + IMGU_REG_GP_BUSY);
247
248	/* Set CSS clock frequency */
249	pm_ctrl = readl(addr: base + IMGU_REG_PM_CTRL);
250	val = pm_ctrl & ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
251	writel(val, addr: base + IMGU_REG_PM_CTRL);
252	writel(val: 0, addr: base + IMGU_REG_GP_BUSY);
253	if (imgu_hw_wait(base, IMGU_REG_STATE,
254	IMGU_STATE_PWRDNM_FSM_MASK, cmp: 0)) {
255	dev_err(dev, "failed to pwrdn CSS\n");
256	goto fail;
257	}
258	val = (freq / IMGU_SYSTEM_REQ_FREQ_DIVIDER) & IMGU_SYSTEM_REQ_FREQ_MASK;
259	writel(val, addr: base + IMGU_REG_SYSTEM_REQ);
260	writel(val: 1, addr: base + IMGU_REG_GP_BUSY);
261	writel(readl(addr: base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_HALT,
262	addr: base + IMGU_REG_PM_CTRL);
263	if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS,
264	IMGU_STATE_HALT_STS)) {
265	dev_err(dev, "failed to halt CSS\n");
266	goto fail;
267	}
268
269	writel(readl(addr: base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_START,
270	addr: base + IMGU_REG_PM_CTRL);
271	if (imgu_hw_wait(base, IMGU_REG_PM_CTRL, IMGU_PM_CTRL_START, cmp: 0)) {
272	dev_err(dev, "failed to start CSS\n");
273	goto fail;
274	}
275	writel(readl(addr: base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_UNHALT,
276	addr: base + IMGU_REG_PM_CTRL);
277
278	val = readl(addr: base + IMGU_REG_PM_CTRL); /* get pm_ctrl */
279	val &= ~(IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
280	val |= pm_ctrl & (IMGU_PM_CTRL_CSS_PWRDN | IMGU_PM_CTRL_RST_AT_EOF);
281	writel(val, addr: base + IMGU_REG_PM_CTRL);
282
283	return 0;
284
285	fail:
286	imgu_css_set_powerdown(dev, base);
287	return -EIO;
288	}
289
290	void imgu_css_set_powerdown(struct device *dev, void __iomem *base)
291	{
292	dev_dbg(dev, "%s\n", __func__);
293	/* wait for cio idle signal */
294	if (imgu_hw_wait(base, IMGU_REG_CIO_GATE_BURST_STATE,
295	IMGU_CIO_GATE_BURST_MASK, cmp: 0))
296	dev_warn(dev, "wait cio gate idle timeout");
297
298	/* wait for css idle signal */
299	if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
300	IMGU_STATE_IDLE_STS))
301	dev_warn(dev, "wait css idle timeout\n");
302
303	/* do halt-halted handshake with css */
304	writel(val: 1, addr: base + IMGU_REG_GP_HALT);
305	if (imgu_hw_wait(base, IMGU_REG_STATE, IMGU_STATE_HALT_STS,
306	IMGU_STATE_HALT_STS))
307	dev_warn(dev, "failed to halt css");
308
309	/* de-assert the busy bit */
310	writel(val: 0, addr: base + IMGU_REG_GP_BUSY);
311	}
312
313	static void imgu_css_hw_enable_irq(struct imgu_css *css)
314	{
315	void __iomem *const base = css->base;
316	u32 val, i;
317
318	/* Set up interrupts */
319
320	/*
321	* Enable IRQ on the SP which signals that SP goes to idle
322	* (aka ready state) and set trigger to pulse
323	*/
324	val = readl(addr: base + IMGU_REG_SP_CTRL(0)) | IMGU_CTRL_IRQ_READY;
325	writel(val, addr: base + IMGU_REG_SP_CTRL(0));
326	writel(val: val | IMGU_CTRL_IRQ_CLEAR, addr: base + IMGU_REG_SP_CTRL(0));
327
328	/* Enable IRQs from the IMGU wrapper */
329	writel(IMGU_REG_INT_CSS_IRQ, addr: base + IMGU_REG_INT_ENABLE);
330	/* Clear */
331	writel(IMGU_REG_INT_CSS_IRQ, addr: base + IMGU_REG_INT_STATUS);
332
333	/* Enable IRQs from main IRQ controller */
334	writel(val: ~0, addr: base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(IMGU_IRQCTRL_MAIN));
335	writel(val: 0, addr: base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN));
336	writel(IMGU_IRQCTRL_IRQ_MASK,
337	addr: base + IMGU_REG_IRQCTRL_EDGE(IMGU_IRQCTRL_MAIN));
338	writel(IMGU_IRQCTRL_IRQ_MASK,
339	addr: base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN));
340	writel(IMGU_IRQCTRL_IRQ_MASK,
341	addr: base + IMGU_REG_IRQCTRL_CLEAR(IMGU_IRQCTRL_MAIN));
342	writel(IMGU_IRQCTRL_IRQ_MASK,
343	addr: base + IMGU_REG_IRQCTRL_MASK(IMGU_IRQCTRL_MAIN));
344	/* Wait for write complete */
345	readl(addr: base + IMGU_REG_IRQCTRL_ENABLE(IMGU_IRQCTRL_MAIN));
346
347	/* Enable IRQs from SP0 and SP1 controllers */
348	for (i = IMGU_IRQCTRL_SP0; i <= IMGU_IRQCTRL_SP1; i++) {
349	writel(val: ~0, addr: base + IMGU_REG_IRQCTRL_EDGE_NOT_PULSE(i));
350	writel(val: 0, addr: base + IMGU_REG_IRQCTRL_MASK(i));
351	writel(IMGU_IRQCTRL_IRQ_MASK, addr: base + IMGU_REG_IRQCTRL_EDGE(i));
352	writel(IMGU_IRQCTRL_IRQ_MASK,
353	addr: base + IMGU_REG_IRQCTRL_ENABLE(i));
354	writel(IMGU_IRQCTRL_IRQ_MASK, addr: base + IMGU_REG_IRQCTRL_CLEAR(i));
355	writel(IMGU_IRQCTRL_IRQ_MASK, addr: base + IMGU_REG_IRQCTRL_MASK(i));
356	/* Wait for write complete */
357	readl(addr: base + IMGU_REG_IRQCTRL_ENABLE(i));
358	}
359	}
360
361	static int imgu_css_hw_init(struct imgu_css *css)
362	{
363	/* For checking that streaming monitor statuses are valid */
364	static const struct {
365	u32 reg;
366	u32 mask;
367	const char *name;
368	} stream_monitors[] = {
369	{
370	IMGU_REG_GP_SP1_STRMON_STAT,
371	IMGU_GP_STRMON_STAT_ISP_PORT_SP12ISP,
372	"ISP0 to SP0"
373	}, {
374	IMGU_REG_GP_ISP_STRMON_STAT,
375	IMGU_GP_STRMON_STAT_SP1_PORT_ISP2SP1,
376	"SP0 to ISP0"
377	}, {
378	IMGU_REG_GP_MOD_STRMON_STAT,
379	IMGU_GP_STRMON_STAT_MOD_PORT_ISP2DMA,
380	"ISP0 to DMA0"
381	}, {
382	IMGU_REG_GP_ISP_STRMON_STAT,
383	IMGU_GP_STRMON_STAT_ISP_PORT_DMA2ISP,
384	"DMA0 to ISP0"
385	}, {
386	IMGU_REG_GP_MOD_STRMON_STAT,
387	IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC,
388	"ISP0 to GDC0"
389	}, {
390	IMGU_REG_GP_MOD_STRMON_STAT,
391	IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS,
392	"GDC0 to ISP0"
393	}, {
394	IMGU_REG_GP_MOD_STRMON_STAT,
395	IMGU_GP_STRMON_STAT_MOD_PORT_SP12DMA,
396	"SP0 to DMA0"
397	}, {
398	IMGU_REG_GP_SP1_STRMON_STAT,
399	IMGU_GP_STRMON_STAT_SP1_PORT_DMA2SP1,
400	"DMA0 to SP0"
401	}, {
402	IMGU_REG_GP_MOD_STRMON_STAT,
403	IMGU_GP_STRMON_STAT_MOD_PORT_CELLS2GDC,
404	"SP0 to GDC0"
405	}, {
406	IMGU_REG_GP_MOD_STRMON_STAT,
407	IMGU_GP_STRMON_STAT_MOD_PORT_GDC2CELLS,
408	"GDC0 to SP0"
409	},
410	};
411
412	struct device *dev = css->dev;
413	void __iomem *const base = css->base;
414	u32 val, i;
415
416	/* Set instruction cache address and inv bit for ISP, SP, and SP1 */
417	for (i = 0; i < IMGU_NUM_SP; i++) {
418	struct imgu_fw_info *bi =
419	&css->fwp->binary_header[css->fw_sp[i]];
420
421	writel(val: css->binary[css->fw_sp[i]].daddr,
422	addr: base + IMGU_REG_SP_ICACHE_ADDR(bi->type));
423	writel(readl(addr: base + IMGU_REG_SP_CTRL(bi->type)) |
424	IMGU_CTRL_ICACHE_INV,
425	addr: base + IMGU_REG_SP_CTRL(bi->type));
426	}
427	writel(val: css->binary[css->fw_bl].daddr, addr: base + IMGU_REG_ISP_ICACHE_ADDR);
428	writel(readl(addr: base + IMGU_REG_ISP_CTRL) | IMGU_CTRL_ICACHE_INV,
429	addr: base + IMGU_REG_ISP_CTRL);
430
431	/* Check that IMGU hardware is ready */
432
433	if (!(readl(addr: base + IMGU_REG_SP_CTRL(0)) & IMGU_CTRL_IDLE)) {
434	dev_err(dev, "SP is not idle\n");
435	return -EIO;
436	}
437	if (!(readl(addr: base + IMGU_REG_ISP_CTRL) & IMGU_CTRL_IDLE)) {
438	dev_err(dev, "ISP is not idle\n");
439	return -EIO;
440	}
441
442	for (i = 0; i < ARRAY_SIZE(stream_monitors); i++) {
443	val = readl(addr: base + stream_monitors[i].reg);
444	if (val & stream_monitors[i].mask) {
445	dev_err(dev, "error: Stream monitor %s is valid\n",
446	stream_monitors[i].name);
447	return -EIO;
448	}
449	}
450
451	/* Initialize GDC with default values */
452
453	for (i = 0; i < ARRAY_SIZE(imgu_css_gdc_lut[0]); i++) {
454	u32 val0 = imgu_css_gdc_lut[0][i] & IMGU_GDC_LUT_MASK;
455	u32 val1 = imgu_css_gdc_lut[1][i] & IMGU_GDC_LUT_MASK;
456	u32 val2 = imgu_css_gdc_lut[2][i] & IMGU_GDC_LUT_MASK;
457	u32 val3 = imgu_css_gdc_lut[3][i] & IMGU_GDC_LUT_MASK;
458
459	writel(val: val0 | (val1 << 16),
460	addr: base + IMGU_REG_GDC_LUT_BASE + i * 8);
461	writel(val: val2 | (val3 << 16),
462	addr: base + IMGU_REG_GDC_LUT_BASE + i * 8 + 4);
463	}
464
465	return 0;
466	}
467
468	/* Boot the given IPU3 CSS SP */
469	static int imgu_css_hw_start_sp(struct imgu_css *css, int sp)
470	{
471	void __iomem *const base = css->base;
472	struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
473	struct imgu_abi_sp_init_dmem_cfg dmem_cfg = {
474	.ddr_data_addr = css->binary[css->fw_sp[sp]].daddr
475	+ bi->blob.data_source,
476	.dmem_data_addr = bi->blob.data_target,
477	.dmem_bss_addr = bi->blob.bss_target,
478	.data_size = bi->blob.data_size,
479	.bss_size = bi->blob.bss_size,
480	.sp_id = sp,
481	};
482
483	writes(mem: &dmem_cfg, count: sizeof(dmem_cfg), addr: base +
484	IMGU_REG_SP_DMEM_BASE(sp) + bi->info.sp.init_dmem_data);
485
486	writel(val: bi->info.sp.sp_entry, addr: base + IMGU_REG_SP_START_ADDR(sp));
487
488	writel(readl(addr: base + IMGU_REG_SP_CTRL(sp))
489	| IMGU_CTRL_START | IMGU_CTRL_RUN, addr: base + IMGU_REG_SP_CTRL(sp));
490
491	if (imgu_hw_wait(base: css->base, IMGU_REG_SP_DMEM_BASE(sp)
492	+ bi->info.sp.sw_state,
493	mask: ~0, cmp: IMGU_ABI_SP_SWSTATE_INITIALIZED))
494	return -EIO;
495
496	return 0;
497	}
498
499	/* Start the IPU3 CSS ImgU (Imaging Unit) and all the SPs */
500	static int imgu_css_hw_start(struct imgu_css *css)
501	{
502	static const u32 event_mask =
503	((1 << IMGU_ABI_EVTTYPE_OUT_FRAME_DONE) |
504	(1 << IMGU_ABI_EVTTYPE_2ND_OUT_FRAME_DONE) |
505	(1 << IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE) |
506	(1 << IMGU_ABI_EVTTYPE_2ND_VF_OUT_FRAME_DONE) |
507	(1 << IMGU_ABI_EVTTYPE_3A_STATS_DONE) |
508	(1 << IMGU_ABI_EVTTYPE_DIS_STATS_DONE) |
509	(1 << IMGU_ABI_EVTTYPE_PIPELINE_DONE) |
510	(1 << IMGU_ABI_EVTTYPE_FRAME_TAGGED) |
511	(1 << IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE) |
512	(1 << IMGU_ABI_EVTTYPE_METADATA_DONE) |
513	(1 << IMGU_ABI_EVTTYPE_ACC_STAGE_COMPLETE))
514	<< IMGU_ABI_SP_COMM_EVENT_IRQ_MASK_OR_SHIFT;
515
516	void __iomem *const base = css->base;
517	struct imgu_fw_info *bi, *bl = &css->fwp->binary_header[css->fw_bl];
518	unsigned int i;
519
520	writel(IMGU_TLB_INVALIDATE, addr: base + IMGU_REG_TLB_INVALIDATE);
521
522	/* Start bootloader */
523
524	writel(val: IMGU_ABI_BL_SWSTATE_BUSY,
525	addr: base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.sw_state);
526	writel(IMGU_NUM_SP,
527	addr: base + IMGU_REG_ISP_DMEM_BASE + bl->info.bl.num_dma_cmds);
528
529	for (i = 0; i < IMGU_NUM_SP; i++) {
530	int j = IMGU_NUM_SP - i - 1; /* load sp1 first, then sp0 */
531	struct imgu_fw_info *sp =
532	&css->fwp->binary_header[css->fw_sp[j]];
533	struct imgu_abi_bl_dma_cmd_entry dma_cmd = {
534	.src_addr = css->binary[css->fw_sp[j]].daddr
535	+ sp->blob.text_source,
536	.size = sp->blob.text_size,
537	.dst_type = IMGU_ABI_BL_DMACMD_TYPE_SP_PMEM,
538	.dst_addr = IMGU_SP_PMEM_BASE(j),
539	};
540
541	writes(mem: &dma_cmd, count: sizeof(dma_cmd),
542	addr: base + IMGU_REG_ISP_DMEM_BASE + i * sizeof(dma_cmd) +
543	bl->info.bl.dma_cmd_list);
544	}
545
546	writel(val: bl->info.bl.bl_entry, addr: base + IMGU_REG_ISP_START_ADDR);
547
548	writel(readl(addr: base + IMGU_REG_ISP_CTRL)
549	| IMGU_CTRL_START | IMGU_CTRL_RUN, addr: base + IMGU_REG_ISP_CTRL);
550	if (imgu_hw_wait(base: css->base, IMGU_REG_ISP_DMEM_BASE
551	+ bl->info.bl.sw_state, mask: ~0,
552	cmp: IMGU_ABI_BL_SWSTATE_OK)) {
553	dev_err(css->dev, "failed to start bootloader\n");
554	return -EIO;
555	}
556
557	/* Start ISP */
558
559	memset(css->xmem_sp_group_ptrs.vaddr, 0,
560	sizeof(struct imgu_abi_sp_group));
561
562	bi = &css->fwp->binary_header[css->fw_sp[0]];
563
564	writel(val: css->xmem_sp_group_ptrs.daddr,
565	addr: base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.per_frame_data);
566
567	writel(val: IMGU_ABI_SP_SWSTATE_TERMINATED,
568	addr: base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state);
569	writel(val: 1, addr: base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb);
570
571	if (imgu_css_hw_start_sp(css, sp: 0))
572	return -EIO;
573
574	writel(val: 0, addr: base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.isp_started);
575	writel(val: 0, addr: base + IMGU_REG_SP_DMEM_BASE(0) +
576	bi->info.sp.host_sp_queues_initialized);
577	writel(val: 0, addr: base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sleep_mode);
578	writel(val: 0, addr: base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.invalidate_tlb);
579	writel(IMGU_ABI_SP_COMM_COMMAND_READY, addr: base + IMGU_REG_SP_DMEM_BASE(0)
580	+ bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
581
582	/* Enable all events for all queues */
583
584	for (i = 0; i < IPU3_CSS_PIPE_ID_NUM; i++)
585	writel(val: event_mask, addr: base + IMGU_REG_SP_DMEM_BASE(0)
586	+ bi->info.sp.host_sp_com
587	+ IMGU_ABI_SP_COMM_EVENT_IRQ_MASK(i));
588	writel(val: 1, addr: base + IMGU_REG_SP_DMEM_BASE(0) +
589	bi->info.sp.host_sp_queues_initialized);
590
591	/* Start SP1 */
592
593	bi = &css->fwp->binary_header[css->fw_sp[1]];
594
595	writel(val: IMGU_ABI_SP_SWSTATE_TERMINATED,
596	addr: base + IMGU_REG_SP_DMEM_BASE(1) + bi->info.sp.sw_state);
597
598	if (imgu_css_hw_start_sp(css, sp: 1))
599	return -EIO;
600
601	writel(IMGU_ABI_SP_COMM_COMMAND_READY, addr: base + IMGU_REG_SP_DMEM_BASE(1)
602	+ bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
603
604	return 0;
605	}
606
607	static void imgu_css_hw_stop(struct imgu_css *css)
608	{
609	void __iomem *const base = css->base;
610	struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]];
611
612	/* Stop fw */
613	writel(IMGU_ABI_SP_COMM_COMMAND_TERMINATE,
614	addr: base + IMGU_REG_SP_DMEM_BASE(0) +
615	bi->info.sp.host_sp_com + IMGU_ABI_SP_COMM_COMMAND);
616	if (imgu_hw_wait(base: css->base, IMGU_REG_SP_CTRL(0),
617	IMGU_CTRL_IDLE, IMGU_CTRL_IDLE))
618	dev_err(css->dev, "wait sp0 idle timeout.\n");
619	if (readl(addr: base + IMGU_REG_SP_DMEM_BASE(0) + bi->info.sp.sw_state) !=
620	IMGU_ABI_SP_SWSTATE_TERMINATED)
621	dev_err(css->dev, "sp0 is not terminated.\n");
622	if (imgu_hw_wait(base: css->base, IMGU_REG_ISP_CTRL,
623	IMGU_CTRL_IDLE, IMGU_CTRL_IDLE))
624	dev_err(css->dev, "wait isp idle timeout\n");
625	}
626
627	static void imgu_css_hw_cleanup(struct imgu_css *css)
628	{
629	void __iomem *const base = css->base;
630
631	/** Reset CSS **/
632
633	/* Clear the CSS busy signal */
634	readl(addr: base + IMGU_REG_GP_BUSY);
635	writel(val: 0, addr: base + IMGU_REG_GP_BUSY);
636
637	/* Wait for idle signal */
638	if (imgu_hw_wait(base: css->base, IMGU_REG_STATE, IMGU_STATE_IDLE_STS,
639	IMGU_STATE_IDLE_STS))
640	dev_err(css->dev, "failed to shut down hw cleanly\n");
641
642	/* Reset the css */
643	writel(readl(addr: base + IMGU_REG_PM_CTRL) | IMGU_PM_CTRL_FORCE_RESET,
644	addr: base + IMGU_REG_PM_CTRL);
645
646	usleep_range(min: 200, max: 300);
647	}
648
649	static void imgu_css_pipeline_cleanup(struct imgu_css *css, unsigned int pipe)
650	{
651	struct imgu_device *imgu = dev_get_drvdata(dev: css->dev);
652	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
653	unsigned int i;
654
655	imgu_css_pool_cleanup(imgu, pool: &css_pipe->pool.parameter_set_info);
656	imgu_css_pool_cleanup(imgu, pool: &css_pipe->pool.acc);
657	imgu_css_pool_cleanup(imgu, pool: &css_pipe->pool.gdc);
658	imgu_css_pool_cleanup(imgu, pool: &css_pipe->pool.obgrid);
659
660	for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
661	imgu_css_pool_cleanup(imgu, pool: &css_pipe->pool.binary_params_p[i]);
662	}
663
664	/*
665	* This function initializes various stages of the
666	* IPU3 CSS ISP pipeline
667	*/
668	static int imgu_css_pipeline_init(struct imgu_css *css, unsigned int pipe)
669	{
670	static const int BYPC = 2; /* Bytes per component */
671	static const struct imgu_abi_buffer_sp buffer_sp_init = {
672	.buf_src = {.queue_id = IMGU_ABI_QUEUE_EVENT_ID},
673	.buf_type = IMGU_ABI_BUFFER_TYPE_INVALID,
674	};
675
676	struct imgu_abi_isp_iterator_config *cfg_iter;
677	struct imgu_abi_isp_ref_config *cfg_ref;
678	struct imgu_abi_isp_dvs_config *cfg_dvs;
679	struct imgu_abi_isp_tnr3_config *cfg_tnr;
680	struct imgu_abi_isp_ref_dmem_state *cfg_ref_state;
681	struct imgu_abi_isp_tnr3_dmem_state *cfg_tnr_state;
682
683	const int stage = 0;
684	unsigned int i, j;
685
686	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
687	struct imgu_css_queue *css_queue_in =
688	&css_pipe->queue[IPU3_CSS_QUEUE_IN];
689	struct imgu_css_queue *css_queue_out =
690	&css_pipe->queue[IPU3_CSS_QUEUE_OUT];
691	struct imgu_css_queue *css_queue_vf =
692	&css_pipe->queue[IPU3_CSS_QUEUE_VF];
693	const struct imgu_fw_info *bi =
694	&css->fwp->binary_header[css_pipe->bindex];
695	const unsigned int stripes = bi->info.isp.sp.iterator.num_stripes;
696
697	struct imgu_fw_config_memory_offsets *cofs = (void *)css->fwp +
698	bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_CONFIG];
699	struct imgu_fw_state_memory_offsets *sofs = (void *)css->fwp +
700	bi->blob.memory_offsets.offsets[IMGU_ABI_PARAM_CLASS_STATE];
701
702	struct imgu_abi_isp_stage *isp_stage;
703	struct imgu_abi_sp_stage *sp_stage;
704	struct imgu_abi_sp_group *sp_group;
705	struct imgu_abi_frames_sp *frames_sp;
706	struct imgu_abi_frame_sp *frame_sp;
707	struct imgu_abi_frame_sp_info *frame_sp_info;
708
709	const unsigned int bds_width_pad =
710	ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width,
711	2 * IPU3_UAPI_ISP_VEC_ELEMS);
712
713	const enum imgu_abi_memories m0 = IMGU_ABI_MEM_ISP_DMEM0;
714	enum imgu_abi_param_class cfg = IMGU_ABI_PARAM_CLASS_CONFIG;
715	void *vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr;
716
717	struct imgu_device *imgu = dev_get_drvdata(dev: css->dev);
718
719	dev_dbg(css->dev, "%s for pipe %d", __func__, pipe);
720
721	/* Configure iterator */
722
723	cfg_iter = imgu_css_fw_pipeline_params(css, pipe, cls: cfg, mem: m0,
724	par: &cofs->dmem.iterator,
725	par_size: sizeof(*cfg_iter), binary_params: vaddr);
726	if (!cfg_iter)
727	goto bad_firmware;
728
729	frame_sp_info = &cfg_iter->input_info;
730	frame_sp_info->res.width = css_queue_in->fmt.mpix.width;
731	frame_sp_info->res.height = css_queue_in->fmt.mpix.height;
732	frame_sp_info->padded_width = css_queue_in->width_pad;
733	frame_sp_info->format = css_queue_in->css_fmt->frame_format;
734	frame_sp_info->raw_bit_depth = css_queue_in->css_fmt->bit_depth;
735	frame_sp_info->raw_bayer_order = css_queue_in->css_fmt->bayer_order;
736	frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER;
737
738	frame_sp_info = &cfg_iter->internal_info;
739	frame_sp_info->res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width;
740	frame_sp_info->res.height = css_pipe->rect[IPU3_CSS_RECT_BDS].height;
741	frame_sp_info->padded_width = bds_width_pad;
742	frame_sp_info->format = css_queue_out->css_fmt->frame_format;
743	frame_sp_info->raw_bit_depth = css_queue_out->css_fmt->bit_depth;
744	frame_sp_info->raw_bayer_order = css_queue_out->css_fmt->bayer_order;
745	frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER;
746
747	frame_sp_info = &cfg_iter->output_info;
748	frame_sp_info->res.width = css_queue_out->fmt.mpix.width;
749	frame_sp_info->res.height = css_queue_out->fmt.mpix.height;
750	frame_sp_info->padded_width = css_queue_out->width_pad;
751	frame_sp_info->format = css_queue_out->css_fmt->frame_format;
752	frame_sp_info->raw_bit_depth = css_queue_out->css_fmt->bit_depth;
753	frame_sp_info->raw_bayer_order = css_queue_out->css_fmt->bayer_order;
754	frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER;
755
756	frame_sp_info = &cfg_iter->vf_info;
757	frame_sp_info->res.width = css_queue_vf->fmt.mpix.width;
758	frame_sp_info->res.height = css_queue_vf->fmt.mpix.height;
759	frame_sp_info->padded_width = css_queue_vf->width_pad;
760	frame_sp_info->format = css_queue_vf->css_fmt->frame_format;
761	frame_sp_info->raw_bit_depth = css_queue_vf->css_fmt->bit_depth;
762	frame_sp_info->raw_bayer_order = css_queue_vf->css_fmt->bayer_order;
763	frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER;
764
765	cfg_iter->dvs_envelope.width =
766	css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width;
767	cfg_iter->dvs_envelope.height =
768	css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height;
769
770	/* Configure reference (delay) frames */
771
772	cfg_ref = imgu_css_fw_pipeline_params(css, pipe, cls: cfg, mem: m0,
773	par: &cofs->dmem.ref,
774	par_size: sizeof(*cfg_ref), binary_params: vaddr);
775	if (!cfg_ref)
776	goto bad_firmware;
777
778	cfg_ref->port_b.crop = 0;
779	cfg_ref->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES / BYPC;
780	cfg_ref->port_b.width =
781	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width;
782	cfg_ref->port_b.stride =
783	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline;
784	cfg_ref->width_a_over_b =
785	IPU3_UAPI_ISP_VEC_ELEMS / cfg_ref->port_b.elems;
786	cfg_ref->dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1;
787	for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++) {
788	cfg_ref->ref_frame_addr_y[i] =
789	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr;
790	cfg_ref->ref_frame_addr_c[i] =
791	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i].daddr +
792	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline *
793	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height;
794	}
795	for (; i < IMGU_ABI_FRAMES_REF; i++) {
796	cfg_ref->ref_frame_addr_y[i] = 0;
797	cfg_ref->ref_frame_addr_c[i] = 0;
798	}
799
800	/* Configure DVS (digital video stabilization) */
801
802	cfg_dvs = imgu_css_fw_pipeline_params(css, pipe, cls: cfg, mem: m0,
803	par: &cofs->dmem.dvs, par_size: sizeof(*cfg_dvs),
804	binary_params: vaddr);
805	if (!cfg_dvs)
806	goto bad_firmware;
807
808	cfg_dvs->num_horizontal_blocks =
809	ALIGN(DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].width,
810	IMGU_DVS_BLOCK_W), 2);
811	cfg_dvs->num_vertical_blocks =
812	DIV_ROUND_UP(css_pipe->rect[IPU3_CSS_RECT_GDC].height,
813	IMGU_DVS_BLOCK_H);
814
815	/* Configure TNR (temporal noise reduction) */
816
817	if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) {
818	cfg_tnr = imgu_css_fw_pipeline_params(css, pipe, cls: cfg, mem: m0,
819	par: &cofs->dmem.tnr3,
820	par_size: sizeof(*cfg_tnr),
821	binary_params: vaddr);
822	if (!cfg_tnr)
823	goto bad_firmware;
824
825	cfg_tnr->port_b.crop = 0;
826	cfg_tnr->port_b.elems = IMGU_ABI_ISP_DDR_WORD_BYTES;
827	cfg_tnr->port_b.width =
828	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width;
829	cfg_tnr->port_b.stride =
830	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline;
831	cfg_tnr->width_a_over_b =
832	IPU3_UAPI_ISP_VEC_ELEMS / cfg_tnr->port_b.elems;
833	cfg_tnr->frame_height =
834	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height;
835	cfg_tnr->delay_frame = IPU3_CSS_AUX_FRAMES - 1;
836	for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
837	cfg_tnr->frame_addr[i] =
838	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR]
839	.mem[i].daddr;
840	for (; i < IMGU_ABI_FRAMES_TNR; i++)
841	cfg_tnr->frame_addr[i] = 0;
842	}
843
844	/* Configure ref dmem state parameters */
845
846	cfg = IMGU_ABI_PARAM_CLASS_STATE;
847	vaddr = css_pipe->binary_params_cs[cfg - 1][m0].vaddr;
848
849	cfg_ref_state = imgu_css_fw_pipeline_params(css, pipe, cls: cfg, mem: m0,
850	par: &sofs->dmem.ref,
851	par_size: sizeof(*cfg_ref_state),
852	binary_params: vaddr);
853	if (!cfg_ref_state)
854	goto bad_firmware;
855
856	cfg_ref_state->ref_in_buf_idx = 0;
857	cfg_ref_state->ref_out_buf_idx = 1;
858
859	/* Configure tnr dmem state parameters */
860	if (css_pipe->pipe_id == IPU3_CSS_PIPE_ID_VIDEO) {
861	cfg_tnr_state =
862	imgu_css_fw_pipeline_params(css, pipe, cls: cfg, mem: m0,
863	par: &sofs->dmem.tnr3,
864	par_size: sizeof(*cfg_tnr_state),
865	binary_params: vaddr);
866	if (!cfg_tnr_state)
867	goto bad_firmware;
868
869	cfg_tnr_state->in_bufidx = 0;
870	cfg_tnr_state->out_bufidx = 1;
871	cfg_tnr_state->bypass_filter = 0;
872	cfg_tnr_state->total_frame_counter = 0;
873	for (i = 0; i < IMGU_ABI_BUF_SETS_TNR; i++)
874	cfg_tnr_state->buffer_frame_counter[i] = 0;
875	}
876
877	/* Configure ISP stage */
878
879	isp_stage = css_pipe->xmem_isp_stage_ptrs[pipe][stage].vaddr;
880	memset(isp_stage, 0, sizeof(*isp_stage));
881	isp_stage->blob_info = bi->blob;
882	isp_stage->binary_info = bi->info.isp.sp;
883	strscpy(isp_stage->binary_name,
884	(char *)css->fwp + bi->blob.prog_name_offset,
885	sizeof(isp_stage->binary_name));
886	isp_stage->mem_initializers = bi->info.isp.sp.mem_initializers;
887	for (i = IMGU_ABI_PARAM_CLASS_CONFIG; i < IMGU_ABI_PARAM_CLASS_NUM; i++)
888	for (j = 0; j < IMGU_ABI_NUM_MEMORIES; j++)
889	isp_stage->mem_initializers.params[i][j].address =
890	css_pipe->binary_params_cs[i - 1][j].daddr;
891
892	/* Configure SP stage */
893
894	sp_stage = css_pipe->xmem_sp_stage_ptrs[pipe][stage].vaddr;
895	memset(sp_stage, 0, sizeof(*sp_stage));
896
897	frames_sp = &sp_stage->frames;
898	frames_sp->in.buf_attr = buffer_sp_init;
899	for (i = 0; i < IMGU_ABI_BINARY_MAX_OUTPUT_PORTS; i++)
900	frames_sp->out[i].buf_attr = buffer_sp_init;
901	frames_sp->out_vf.buf_attr = buffer_sp_init;
902	frames_sp->s3a_buf = buffer_sp_init;
903	frames_sp->dvs_buf = buffer_sp_init;
904
905	sp_stage->stage_type = IMGU_ABI_STAGE_TYPE_ISP;
906	sp_stage->num = stage;
907	sp_stage->isp_online = 0;
908	sp_stage->isp_copy_vf = 0;
909	sp_stage->isp_copy_output = 0;
910
911	sp_stage->enable.vf_output = css_pipe->vf_output_en;
912
913	frames_sp->effective_in_res.width =
914	css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].width;
915	frames_sp->effective_in_res.height =
916	css_pipe->rect[IPU3_CSS_RECT_EFFECTIVE].height;
917
918	frame_sp = &frames_sp->in;
919	frame_sp->info.res.width = css_queue_in->fmt.mpix.width;
920	frame_sp->info.res.height = css_queue_in->fmt.mpix.height;
921	frame_sp->info.padded_width = css_queue_in->width_pad;
922	frame_sp->info.format = css_queue_in->css_fmt->frame_format;
923	frame_sp->info.raw_bit_depth = css_queue_in->css_fmt->bit_depth;
924	frame_sp->info.raw_bayer_order = css_queue_in->css_fmt->bayer_order;
925	frame_sp->info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
926	frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_C_ID;
927	frame_sp->buf_attr.buf_type = IMGU_ABI_BUFFER_TYPE_INPUT_FRAME;
928
929	frame_sp = &frames_sp->out[0];
930	frame_sp->info.res.width = css_queue_out->fmt.mpix.width;
931	frame_sp->info.res.height = css_queue_out->fmt.mpix.height;
932	frame_sp->info.padded_width = css_queue_out->width_pad;
933	frame_sp->info.format = css_queue_out->css_fmt->frame_format;
934	frame_sp->info.raw_bit_depth = css_queue_out->css_fmt->bit_depth;
935	frame_sp->info.raw_bayer_order = css_queue_out->css_fmt->bayer_order;
936	frame_sp->info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
937	frame_sp->planes.nv.uv.offset = css_queue_out->width_pad *
938	css_queue_out->fmt.mpix.height;
939	frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_D_ID;
940	frame_sp->buf_attr.buf_type = IMGU_ABI_BUFFER_TYPE_OUTPUT_FRAME;
941
942	frame_sp = &frames_sp->out[1];
943	frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_EVENT_ID;
944
945	frame_sp_info = &frames_sp->internal_frame_info;
946	frame_sp_info->res.width = css_pipe->rect[IPU3_CSS_RECT_BDS].width;
947	frame_sp_info->res.height = css_pipe->rect[IPU3_CSS_RECT_BDS].height;
948	frame_sp_info->padded_width = bds_width_pad;
949	frame_sp_info->format = css_queue_out->css_fmt->frame_format;
950	frame_sp_info->raw_bit_depth = css_queue_out->css_fmt->bit_depth;
951	frame_sp_info->raw_bayer_order = css_queue_out->css_fmt->bayer_order;
952	frame_sp_info->raw_type = IMGU_ABI_RAW_TYPE_BAYER;
953
954	frame_sp = &frames_sp->out_vf;
955	frame_sp->info.res.width = css_queue_vf->fmt.mpix.width;
956	frame_sp->info.res.height = css_queue_vf->fmt.mpix.height;
957	frame_sp->info.padded_width = css_queue_vf->width_pad;
958	frame_sp->info.format = css_queue_vf->css_fmt->frame_format;
959	frame_sp->info.raw_bit_depth = css_queue_vf->css_fmt->bit_depth;
960	frame_sp->info.raw_bayer_order = css_queue_vf->css_fmt->bayer_order;
961	frame_sp->info.raw_type = IMGU_ABI_RAW_TYPE_BAYER;
962	frame_sp->planes.yuv.u.offset = css_queue_vf->width_pad *
963	css_queue_vf->fmt.mpix.height;
964	frame_sp->planes.yuv.v.offset = css_queue_vf->width_pad *
965	css_queue_vf->fmt.mpix.height * 5 / 4;
966	frame_sp->buf_attr.buf_src.queue_id = IMGU_ABI_QUEUE_E_ID;
967	frame_sp->buf_attr.buf_type = IMGU_ABI_BUFFER_TYPE_VF_OUTPUT_FRAME;
968
969	frames_sp->s3a_buf.buf_src.queue_id = IMGU_ABI_QUEUE_F_ID;
970	frames_sp->s3a_buf.buf_type = IMGU_ABI_BUFFER_TYPE_3A_STATISTICS;
971
972	frames_sp->dvs_buf.buf_src.queue_id = IMGU_ABI_QUEUE_G_ID;
973	frames_sp->dvs_buf.buf_type = IMGU_ABI_BUFFER_TYPE_DIS_STATISTICS;
974
975	sp_stage->dvs_envelope.width =
976	css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].width;
977	sp_stage->dvs_envelope.height =
978	css_pipe->rect[IPU3_CSS_RECT_ENVELOPE].height;
979
980	sp_stage->isp_pipe_version =
981	bi->info.isp.sp.pipeline.isp_pipe_version;
982	sp_stage->isp_deci_log_factor =
983	clamp(max(fls(css_pipe->rect[IPU3_CSS_RECT_BDS].width /
984	IMGU_MAX_BQ_GRID_WIDTH),
985	fls(css_pipe->rect[IPU3_CSS_RECT_BDS].height /
986	IMGU_MAX_BQ_GRID_HEIGHT)) - 1, 3, 5);
987	sp_stage->isp_vf_downscale_bits = 0;
988	sp_stage->if_config_index = 255;
989	sp_stage->sp_enable_xnr = 0;
990	sp_stage->num_stripes = stripes;
991	sp_stage->enable.s3a = 1;
992	sp_stage->enable.dvs_stats = 0;
993
994	sp_stage->xmem_bin_addr = css->binary[css_pipe->bindex].daddr;
995	sp_stage->xmem_map_addr = css_pipe->sp_ddr_ptrs.daddr;
996	sp_stage->isp_stage_addr =
997	css_pipe->xmem_isp_stage_ptrs[pipe][stage].daddr;
998
999	/* Configure SP group */
1000
1001	sp_group = css->xmem_sp_group_ptrs.vaddr;
1002	memset(&sp_group->pipe[pipe], 0, sizeof(struct imgu_abi_sp_pipeline));
1003
1004	sp_group->pipe[pipe].num_stages = 1;
1005	sp_group->pipe[pipe].pipe_id = css_pipe->pipe_id;
1006	sp_group->pipe[pipe].thread_id = pipe;
1007	sp_group->pipe[pipe].pipe_num = pipe;
1008	sp_group->pipe[pipe].num_execs = -1;
1009	sp_group->pipe[pipe].pipe_qos_config = -1;
1010	sp_group->pipe[pipe].required_bds_factor = 0;
1011	sp_group->pipe[pipe].dvs_frame_delay = IPU3_CSS_AUX_FRAMES - 1;
1012	sp_group->pipe[pipe].inout_port_config =
1013	IMGU_ABI_PORT_CONFIG_TYPE_INPUT_HOST |
1014	IMGU_ABI_PORT_CONFIG_TYPE_OUTPUT_HOST;
1015	sp_group->pipe[pipe].scaler_pp_lut = 0;
1016	sp_group->pipe[pipe].shading.internal_frame_origin_x_bqs_on_sctbl = 0;
1017	sp_group->pipe[pipe].shading.internal_frame_origin_y_bqs_on_sctbl = 0;
1018	sp_group->pipe[pipe].sp_stage_addr[stage] =
1019	css_pipe->xmem_sp_stage_ptrs[pipe][stage].daddr;
1020	sp_group->pipe[pipe].pipe_config =
1021	bi->info.isp.sp.enable.params ? (1 << pipe) : 0;
1022	sp_group->pipe[pipe].pipe_config |= IMGU_ABI_PIPE_CONFIG_ACQUIRE_ISP;
1023
1024	/* Initialize parameter pools */
1025
1026	if (imgu_css_pool_init(imgu, pool: &css_pipe->pool.parameter_set_info,
1027	size: sizeof(struct imgu_abi_parameter_set_info)) ||
1028	imgu_css_pool_init(imgu, pool: &css_pipe->pool.acc,
1029	size: sizeof(struct imgu_abi_acc_param)) ||
1030	imgu_css_pool_init(imgu, pool: &css_pipe->pool.gdc,
1031	size: sizeof(struct imgu_abi_gdc_warp_param) *
1032	3 * cfg_dvs->num_horizontal_blocks / 2 *
1033	cfg_dvs->num_vertical_blocks) ||
1034	imgu_css_pool_init(imgu, pool: &css_pipe->pool.obgrid,
1035	size: imgu_css_fw_obgrid_size(
1036	bi: &css->fwp->binary_header[css_pipe->bindex])))
1037	goto out_of_memory;
1038
1039	for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1040	if (imgu_css_pool_init(imgu,
1041	pool: &css_pipe->pool.binary_params_p[i],
1042	size: bi->info.isp.sp.mem_initializers.params
1043	[IMGU_ABI_PARAM_CLASS_PARAM][i].size))
1044	goto out_of_memory;
1045
1046	return 0;
1047
1048	bad_firmware:
1049	imgu_css_pipeline_cleanup(css, pipe);
1050	return -EPROTO;
1051
1052	out_of_memory:
1053	imgu_css_pipeline_cleanup(css, pipe);
1054	return -ENOMEM;
1055	}
1056
1057	static u8 imgu_css_queue_pos(struct imgu_css *css, int queue, int thread)
1058	{
1059	static const unsigned int sp;
1060	void __iomem *const base = css->base;
1061	struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1062	struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1063	bi->info.sp.host_sp_queue;
1064
1065	return queue >= 0 ? readb(addr: &q->host2sp_bufq_info[thread][queue].end) :
1066	readb(addr: &q->host2sp_evtq_info.end);
1067	}
1068
1069	/* Sent data to sp using given buffer queue, or if queue < 0, event queue. */
1070	static int imgu_css_queue_data(struct imgu_css *css,
1071	int queue, int thread, u32 data)
1072	{
1073	static const unsigned int sp;
1074	void __iomem *const base = css->base;
1075	struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1076	struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1077	bi->info.sp.host_sp_queue;
1078	u8 size, start, end, end2;
1079
1080	if (queue >= 0) {
1081	size = readb(addr: &q->host2sp_bufq_info[thread][queue].size);
1082	start = readb(addr: &q->host2sp_bufq_info[thread][queue].start);
1083	end = readb(addr: &q->host2sp_bufq_info[thread][queue].end);
1084	} else {
1085	size = readb(addr: &q->host2sp_evtq_info.size);
1086	start = readb(addr: &q->host2sp_evtq_info.start);
1087	end = readb(addr: &q->host2sp_evtq_info.end);
1088	}
1089
1090	if (size == 0)
1091	return -EIO;
1092
1093	end2 = (end + 1) % size;
1094	if (end2 == start)
1095	return -EBUSY; /* Queue full */
1096
1097	if (queue >= 0) {
1098	writel(val: data, addr: &q->host2sp_bufq[thread][queue][end]);
1099	writeb(val: end2, addr: &q->host2sp_bufq_info[thread][queue].end);
1100	} else {
1101	writel(val: data, addr: &q->host2sp_evtq[end]);
1102	writeb(val: end2, addr: &q->host2sp_evtq_info.end);
1103	}
1104
1105	return 0;
1106	}
1107
1108	/* Receive data using given buffer queue, or if queue < 0, event queue. */
1109	static int imgu_css_dequeue_data(struct imgu_css *css, int queue, u32 *data)
1110	{
1111	static const unsigned int sp;
1112	void __iomem *const base = css->base;
1113	struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[sp]];
1114	struct imgu_abi_queues __iomem *q = base + IMGU_REG_SP_DMEM_BASE(sp) +
1115	bi->info.sp.host_sp_queue;
1116	u8 size, start, end, start2;
1117
1118	if (queue >= 0) {
1119	size = readb(addr: &q->sp2host_bufq_info[queue].size);
1120	start = readb(addr: &q->sp2host_bufq_info[queue].start);
1121	end = readb(addr: &q->sp2host_bufq_info[queue].end);
1122	} else {
1123	size = readb(addr: &q->sp2host_evtq_info.size);
1124	start = readb(addr: &q->sp2host_evtq_info.start);
1125	end = readb(addr: &q->sp2host_evtq_info.end);
1126	}
1127
1128	if (size == 0)
1129	return -EIO;
1130
1131	if (end == start)
1132	return -EBUSY; /* Queue empty */
1133
1134	start2 = (start + 1) % size;
1135
1136	if (queue >= 0) {
1137	*data = readl(addr: &q->sp2host_bufq[queue][start]);
1138	writeb(val: start2, addr: &q->sp2host_bufq_info[queue].start);
1139	} else {
1140	int r;
1141
1142	*data = readl(addr: &q->sp2host_evtq[start]);
1143	writeb(val: start2, addr: &q->sp2host_evtq_info.start);
1144
1145	/* Acknowledge events dequeued from event queue */
1146	r = imgu_css_queue_data(css, queue, thread: 0,
1147	IMGU_ABI_EVENT_EVENT_DEQUEUED);
1148	if (r < 0)
1149	return r;
1150	}
1151
1152	return 0;
1153	}
1154
1155	/* Free binary-specific resources */
1156	static void imgu_css_binary_cleanup(struct imgu_css *css, unsigned int pipe)
1157	{
1158	struct imgu_device *imgu = dev_get_drvdata(dev: css->dev);
1159	unsigned int i, j;
1160
1161	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1162
1163	for (j = 0; j < IMGU_ABI_PARAM_CLASS_NUM - 1; j++)
1164	for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1165	imgu_dmamap_free(imgu,
1166	map: &css_pipe->binary_params_cs[j][i]);
1167
1168	j = IPU3_CSS_AUX_FRAME_REF;
1169	for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1170	imgu_dmamap_free(imgu,
1171	map: &css_pipe->aux_frames[j].mem[i]);
1172
1173	j = IPU3_CSS_AUX_FRAME_TNR;
1174	for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1175	imgu_dmamap_free(imgu,
1176	map: &css_pipe->aux_frames[j].mem[i]);
1177	}
1178
1179	static int imgu_css_binary_preallocate(struct imgu_css *css, unsigned int pipe)
1180	{
1181	struct imgu_device *imgu = dev_get_drvdata(dev: css->dev);
1182	unsigned int i, j;
1183
1184	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1185
1186	for (j = IMGU_ABI_PARAM_CLASS_CONFIG;
1187	j < IMGU_ABI_PARAM_CLASS_NUM; j++)
1188	for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++)
1189	if (!imgu_dmamap_alloc(imgu,
1190	map: &css_pipe->binary_params_cs[j - 1][i],
1191	CSS_ABI_SIZE))
1192	goto out_of_memory;
1193
1194	for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1195	if (!imgu_dmamap_alloc(imgu,
1196	map: &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i],
1197	CSS_BDS_SIZE))
1198	goto out_of_memory;
1199
1200	for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1201	if (!imgu_dmamap_alloc(imgu,
1202	map: &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].mem[i],
1203	CSS_GDC_SIZE))
1204	goto out_of_memory;
1205
1206	return 0;
1207
1208	out_of_memory:
1209	imgu_css_binary_cleanup(css, pipe);
1210	return -ENOMEM;
1211	}
1212
1213	/* allocate binary-specific resources */
1214	static int imgu_css_binary_setup(struct imgu_css *css, unsigned int pipe)
1215	{
1216	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1217	struct imgu_fw_info *bi = &css->fwp->binary_header[css_pipe->bindex];
1218	struct imgu_device *imgu = dev_get_drvdata(dev: css->dev);
1219	int i, j, size;
1220	static const int BYPC = 2; /* Bytes per component */
1221	unsigned int w, h;
1222
1223	/* Allocate parameter memory blocks for this binary */
1224
1225	for (j = IMGU_ABI_PARAM_CLASS_CONFIG; j < IMGU_ABI_PARAM_CLASS_NUM; j++)
1226	for (i = 0; i < IMGU_ABI_NUM_MEMORIES; i++) {
1227	if (imgu_css_dma_buffer_resize(
1228	imgu,
1229	map: &css_pipe->binary_params_cs[j - 1][i],
1230	size: bi->info.isp.sp.mem_initializers.params[j][i].size))
1231	goto out_of_memory;
1232	}
1233
1234	/* Allocate internal frame buffers */
1235
1236	/* Reference frames for DVS, FRAME_FORMAT_YUV420_16 */
1237	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel = BYPC;
1238	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].width =
1239	css_pipe->rect[IPU3_CSS_RECT_BDS].width;
1240	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height =
1241	ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].height,
1242	IMGU_DVS_BLOCK_H) + 2 * IMGU_GDC_BUF_Y;
1243	h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].height;
1244	w = ALIGN(css_pipe->rect[IPU3_CSS_RECT_BDS].width,
1245	2 * IPU3_UAPI_ISP_VEC_ELEMS) + 2 * IMGU_GDC_BUF_X;
1246	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperline =
1247	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].bytesperpixel * w;
1248	size = w * h * BYPC + (w / 2) * (h / 2) * BYPC * 2;
1249	for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1250	if (imgu_css_dma_buffer_resize(
1251	imgu,
1252	map: &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_REF].mem[i],
1253	size))
1254	goto out_of_memory;
1255
1256	/* TNR frames for temporal noise reduction, FRAME_FORMAT_YUV_LINE */
1257	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperpixel = 1;
1258	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width =
1259	roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].width,
1260	bi->info.isp.sp.block.block_width *
1261	IPU3_UAPI_ISP_VEC_ELEMS);
1262	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height =
1263	roundup(css_pipe->rect[IPU3_CSS_RECT_GDC].height,
1264	bi->info.isp.sp.block.output_block_height);
1265
1266	w = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].width;
1267	css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].bytesperline = w;
1268	h = css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].height;
1269	size = w * ALIGN(h * 3 / 2 + 3, 2); /* +3 for vf_pp prefetch */
1270	for (i = 0; i < IPU3_CSS_AUX_FRAMES; i++)
1271	if (imgu_css_dma_buffer_resize(
1272	imgu,
1273	map: &css_pipe->aux_frames[IPU3_CSS_AUX_FRAME_TNR].mem[i],
1274	size))
1275	goto out_of_memory;
1276
1277	return 0;
1278
1279	out_of_memory:
1280	imgu_css_binary_cleanup(css, pipe);
1281	return -ENOMEM;
1282	}
1283
1284	int imgu_css_start_streaming(struct imgu_css *css)
1285	{
1286	u32 data;
1287	int r, pipe;
1288
1289	if (css->streaming)
1290	return -EPROTO;
1291
1292	for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1293	r = imgu_css_binary_setup(css, pipe);
1294	if (r < 0)
1295	return r;
1296	}
1297
1298	r = imgu_css_hw_init(css);
1299	if (r < 0)
1300	return r;
1301
1302	r = imgu_css_hw_start(css);
1303	if (r < 0)
1304	goto fail;
1305
1306	for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1307	r = imgu_css_pipeline_init(css, pipe);
1308	if (r < 0)
1309	goto fail;
1310	}
1311
1312	css->streaming = true;
1313
1314	imgu_css_hw_enable_irq(css);
1315
1316	/* Initialize parameters to default */
1317	for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1318	r = imgu_css_set_parameters(css, pipe, NULL);
1319	if (r < 0)
1320	goto fail;
1321	}
1322
1323	while (!(r = imgu_css_dequeue_data(css, queue: IMGU_ABI_QUEUE_A_ID, data: &data)))
1324	;
1325	if (r != -EBUSY)
1326	goto fail;
1327
1328	while (!(r = imgu_css_dequeue_data(css, queue: IMGU_ABI_QUEUE_B_ID, data: &data)))
1329	;
1330	if (r != -EBUSY)
1331	goto fail;
1332
1333	for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1334	r = imgu_css_queue_data(css, queue: IMGU_ABI_QUEUE_EVENT_ID, thread: pipe,
1335	IMGU_ABI_EVENT_START_STREAM |
1336	pipe << 16);
1337	if (r < 0)
1338	goto fail;
1339	}
1340
1341	return 0;
1342
1343	fail:
1344	css->streaming = false;
1345	imgu_css_hw_cleanup(css);
1346	for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1347	imgu_css_pipeline_cleanup(css, pipe);
1348	imgu_css_binary_cleanup(css, pipe);
1349	}
1350
1351	return r;
1352	}
1353
1354	void imgu_css_stop_streaming(struct imgu_css *css)
1355	{
1356	struct imgu_css_buffer *b, *b0;
1357	int q, r, pipe;
1358
1359	for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1360	r = imgu_css_queue_data(css, queue: IMGU_ABI_QUEUE_EVENT_ID, thread: pipe,
1361	IMGU_ABI_EVENT_STOP_STREAM);
1362	if (r < 0)
1363	dev_warn(css->dev, "failed on stop stream event\n");
1364	}
1365
1366	if (!css->streaming)
1367	return;
1368
1369	imgu_css_hw_stop(css);
1370
1371	imgu_css_hw_cleanup(css);
1372
1373	for_each_set_bit(pipe, css->enabled_pipes, IMGU_MAX_PIPE_NUM) {
1374	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1375
1376	imgu_css_pipeline_cleanup(css, pipe);
1377
1378	spin_lock(lock: &css_pipe->qlock);
1379	for (q = 0; q < IPU3_CSS_QUEUES; q++)
1380	list_for_each_entry_safe(b, b0,
1381	&css_pipe->queue[q].bufs,
1382	list) {
1383	b->state = IPU3_CSS_BUFFER_FAILED;
1384	list_del(entry: &b->list);
1385	}
1386	spin_unlock(lock: &css_pipe->qlock);
1387	}
1388
1389	css->streaming = false;
1390	}
1391
1392	bool imgu_css_pipe_queue_empty(struct imgu_css *css, unsigned int pipe)
1393	{
1394	int q;
1395	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1396
1397	spin_lock(lock: &css_pipe->qlock);
1398	for (q = 0; q < IPU3_CSS_QUEUES; q++)
1399	if (!list_empty(head: &css_pipe->queue[q].bufs))
1400	break;
1401	spin_unlock(lock: &css_pipe->qlock);
1402	return (q == IPU3_CSS_QUEUES);
1403	}
1404
1405	bool imgu_css_queue_empty(struct imgu_css *css)
1406	{
1407	unsigned int pipe;
1408	bool ret = false;
1409
1410	for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++)
1411	ret &= imgu_css_pipe_queue_empty(css, pipe);
1412
1413	return ret;
1414	}
1415
1416	bool imgu_css_is_streaming(struct imgu_css *css)
1417	{
1418	return css->streaming;
1419	}
1420
1421	static int imgu_css_map_init(struct imgu_css *css, unsigned int pipe)
1422	{
1423	struct imgu_device *imgu = dev_get_drvdata(dev: css->dev);
1424	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1425	unsigned int p, q, i;
1426
1427	/* Allocate and map common structures with imgu hardware */
1428	for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++)
1429	for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) {
1430	if (!imgu_dmamap_alloc(imgu,
1431	map: &css_pipe->xmem_sp_stage_ptrs[p][i],
1432	len: sizeof(struct imgu_abi_sp_stage)))
1433	return -ENOMEM;
1434	if (!imgu_dmamap_alloc(imgu,
1435	map: &css_pipe->xmem_isp_stage_ptrs[p][i],
1436	len: sizeof(struct imgu_abi_isp_stage)))
1437	return -ENOMEM;
1438	}
1439
1440	if (!imgu_dmamap_alloc(imgu, map: &css_pipe->sp_ddr_ptrs,
1441	ALIGN(sizeof(struct imgu_abi_ddr_address_map),
1442	IMGU_ABI_ISP_DDR_WORD_BYTES)))
1443	return -ENOMEM;
1444
1445	for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1446	unsigned int abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]);
1447
1448	for (i = 0; i < abi_buf_num; i++)
1449	if (!imgu_dmamap_alloc(imgu,
1450	map: &css_pipe->abi_buffers[q][i],
1451	len: sizeof(struct imgu_abi_buffer)))
1452	return -ENOMEM;
1453	}
1454
1455	if (imgu_css_binary_preallocate(css, pipe)) {
1456	imgu_css_binary_cleanup(css, pipe);
1457	return -ENOMEM;
1458	}
1459
1460	return 0;
1461	}
1462
1463	static void imgu_css_pipe_cleanup(struct imgu_css *css, unsigned int pipe)
1464	{
1465	struct imgu_device *imgu = dev_get_drvdata(dev: css->dev);
1466	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1467	unsigned int p, q, i, abi_buf_num;
1468
1469	imgu_css_binary_cleanup(css, pipe);
1470
1471	for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1472	abi_buf_num = ARRAY_SIZE(css_pipe->abi_buffers[q]);
1473	for (i = 0; i < abi_buf_num; i++)
1474	imgu_dmamap_free(imgu, map: &css_pipe->abi_buffers[q][i]);
1475	}
1476
1477	for (p = 0; p < IPU3_CSS_PIPE_ID_NUM; p++)
1478	for (i = 0; i < IMGU_ABI_MAX_STAGES; i++) {
1479	imgu_dmamap_free(imgu,
1480	map: &css_pipe->xmem_sp_stage_ptrs[p][i]);
1481	imgu_dmamap_free(imgu,
1482	map: &css_pipe->xmem_isp_stage_ptrs[p][i]);
1483	}
1484
1485	imgu_dmamap_free(imgu, map: &css_pipe->sp_ddr_ptrs);
1486	}
1487
1488	void imgu_css_cleanup(struct imgu_css *css)
1489	{
1490	struct imgu_device *imgu = dev_get_drvdata(dev: css->dev);
1491	unsigned int pipe;
1492
1493	imgu_css_stop_streaming(css);
1494	for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++)
1495	imgu_css_pipe_cleanup(css, pipe);
1496	imgu_dmamap_free(imgu, map: &css->xmem_sp_group_ptrs);
1497	imgu_css_fw_cleanup(css);
1498	}
1499
1500	int imgu_css_init(struct device *dev, struct imgu_css *css,
1501	void __iomem *base, int length)
1502	{
1503	struct imgu_device *imgu = dev_get_drvdata(dev);
1504	int r, q, pipe;
1505
1506	/* Initialize main data structure */
1507	css->dev = dev;
1508	css->base = base;
1509	css->iomem_length = length;
1510
1511	for (pipe = 0; pipe < IMGU_MAX_PIPE_NUM; pipe++) {
1512	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1513
1514	css_pipe->vf_output_en = false;
1515	spin_lock_init(&css_pipe->qlock);
1516	css_pipe->bindex = IPU3_CSS_DEFAULT_BINARY;
1517	css_pipe->pipe_id = IPU3_CSS_PIPE_ID_VIDEO;
1518	for (q = 0; q < IPU3_CSS_QUEUES; q++) {
1519	r = imgu_css_queue_init(queue: &css_pipe->queue[q], NULL, flags: 0);
1520	if (r)
1521	return r;
1522	}
1523	r = imgu_css_map_init(css, pipe);
1524	if (r) {
1525	imgu_css_cleanup(css);
1526	return r;
1527	}
1528	}
1529	if (!imgu_dmamap_alloc(imgu, map: &css->xmem_sp_group_ptrs,
1530	len: sizeof(struct imgu_abi_sp_group)))
1531	return -ENOMEM;
1532
1533	r = imgu_css_fw_init(css);
1534	if (r)
1535	return r;
1536
1537	return 0;
1538	}
1539
1540	static u32 imgu_css_adjust(u32 res, u32 align)
1541	{
1542	u32 val = max_t(u32, IPU3_CSS_MIN_RES, res);
1543
1544	return DIV_ROUND_CLOSEST(val, align) * align;
1545	}
1546
1547	/* Select a binary matching the required resolutions and formats */
1548	static int imgu_css_find_binary(struct imgu_css *css,
1549	unsigned int pipe,
1550	struct imgu_css_queue queue[IPU3_CSS_QUEUES],
1551	struct v4l2_rect rects[IPU3_CSS_RECTS])
1552	{
1553	const int binary_nr = css->fwp->file_header.binary_nr;
1554	unsigned int binary_mode =
1555	(css->pipes[pipe].pipe_id == IPU3_CSS_PIPE_ID_CAPTURE) ?
1556	IA_CSS_BINARY_MODE_PRIMARY : IA_CSS_BINARY_MODE_VIDEO;
1557	const struct v4l2_pix_format_mplane *in =
1558	&queue[IPU3_CSS_QUEUE_IN].fmt.mpix;
1559	const struct v4l2_pix_format_mplane *out =
1560	&queue[IPU3_CSS_QUEUE_OUT].fmt.mpix;
1561	const struct v4l2_pix_format_mplane *vf =
1562	&queue[IPU3_CSS_QUEUE_VF].fmt.mpix;
1563	u32 stripe_w = 0, stripe_h = 0;
1564	const char *name;
1565	int i, j;
1566
1567	if (!imgu_css_queue_enabled(q: &queue[IPU3_CSS_QUEUE_IN]))
1568	return -EINVAL;
1569
1570	/* Find out the strip size boundary */
1571	for (i = 0; i < binary_nr; i++) {
1572	struct imgu_fw_info *bi = &css->fwp->binary_header[i];
1573
1574	u32 max_width = bi->info.isp.sp.output.max_width;
1575	u32 max_height = bi->info.isp.sp.output.max_height;
1576
1577	if (bi->info.isp.sp.iterator.num_stripes <= 1) {
1578	stripe_w = stripe_w ?
1579	min(stripe_w, max_width) : max_width;
1580	stripe_h = stripe_h ?
1581	min(stripe_h, max_height) : max_height;
1582	}
1583	}
1584
1585	for (i = 0; i < binary_nr; i++) {
1586	struct imgu_fw_info *bi = &css->fwp->binary_header[i];
1587	enum imgu_abi_frame_format q_fmt;
1588
1589	name = (void *)css->fwp + bi->blob.prog_name_offset;
1590
1591	/* Check that binary supports memory-to-memory processing */
1592	if (bi->info.isp.sp.input.source !=
1593	IMGU_ABI_BINARY_INPUT_SOURCE_MEMORY)
1594	continue;
1595
1596	/* Check that binary supports raw10 input */
1597	if (!bi->info.isp.sp.enable.input_feeder &&
1598	!bi->info.isp.sp.enable.input_raw)
1599	continue;
1600
1601	/* Check binary mode */
1602	if (bi->info.isp.sp.pipeline.mode != binary_mode)
1603	continue;
1604
1605	/* Since input is RGGB bayer, need to process colors */
1606	if (bi->info.isp.sp.enable.luma_only)
1607	continue;
1608
1609	if (in->width < bi->info.isp.sp.input.min_width ||
1610	in->width > bi->info.isp.sp.input.max_width ||
1611	in->height < bi->info.isp.sp.input.min_height ||
1612	in->height > bi->info.isp.sp.input.max_height)
1613	continue;
1614
1615	if (imgu_css_queue_enabled(q: &queue[IPU3_CSS_QUEUE_OUT])) {
1616	if (bi->info.isp.num_output_pins <= 0)
1617	continue;
1618
1619	q_fmt = queue[IPU3_CSS_QUEUE_OUT].css_fmt->frame_format;
1620	for (j = 0; j < bi->info.isp.num_output_formats; j++)
1621	if (bi->info.isp.output_formats[j] == q_fmt)
1622	break;
1623	if (j >= bi->info.isp.num_output_formats)
1624	continue;
1625
1626	if (out->width < bi->info.isp.sp.output.min_width ||
1627	out->width > bi->info.isp.sp.output.max_width ||
1628	out->height < bi->info.isp.sp.output.min_height ||
1629	out->height > bi->info.isp.sp.output.max_height)
1630	continue;
1631
1632	if (out->width > bi->info.isp.sp.internal.max_width ||
1633	out->height > bi->info.isp.sp.internal.max_height)
1634	continue;
1635	}
1636
1637	if (imgu_css_queue_enabled(q: &queue[IPU3_CSS_QUEUE_VF])) {
1638	if (bi->info.isp.num_output_pins <= 1)
1639	continue;
1640
1641	q_fmt = queue[IPU3_CSS_QUEUE_VF].css_fmt->frame_format;
1642	for (j = 0; j < bi->info.isp.num_output_formats; j++)
1643	if (bi->info.isp.output_formats[j] == q_fmt)
1644	break;
1645	if (j >= bi->info.isp.num_output_formats)
1646	continue;
1647
1648	if (vf->width < bi->info.isp.sp.output.min_width ||
1649	vf->width > bi->info.isp.sp.output.max_width ||
1650	vf->height < bi->info.isp.sp.output.min_height ||
1651	vf->height > bi->info.isp.sp.output.max_height)
1652	continue;
1653	}
1654
1655	/* All checks passed, select the binary */
1656	dev_dbg(css->dev, "using binary %s id = %u\n", name,
1657	bi->info.isp.sp.id);
1658	return i;
1659	}
1660
1661	/* Can not find suitable binary for these parameters */
1662	return -EINVAL;
1663	}
1664
1665	/*
1666	* Check that there is a binary matching requirements. Parameters may be
1667	* NULL indicating disabled input/output. Return negative if given
1668	* parameters can not be supported or on error, zero or positive indicating
1669	* found binary number. May modify the given parameters if not exact match
1670	* is found.
1671	*/
1672	int imgu_css_fmt_try(struct imgu_css *css,
1673	struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES],
1674	struct v4l2_rect *rects[IPU3_CSS_RECTS],
1675	unsigned int pipe)
1676	{
1677	static const u32 EFF_ALIGN_W = 2;
1678	static const u32 BDS_ALIGN_W = 4;
1679	static const u32 OUT_ALIGN_W = 8;
1680	static const u32 OUT_ALIGN_H = 4;
1681	static const u32 VF_ALIGN_W = 2;
1682	static const char *qnames[IPU3_CSS_QUEUES] = {
1683	[IPU3_CSS_QUEUE_IN] = "in",
1684	[IPU3_CSS_QUEUE_PARAMS] = "params",
1685	[IPU3_CSS_QUEUE_OUT] = "out",
1686	[IPU3_CSS_QUEUE_VF] = "vf",
1687	[IPU3_CSS_QUEUE_STAT_3A] = "3a",
1688	};
1689	static const char *rnames[IPU3_CSS_RECTS] = {
1690	[IPU3_CSS_RECT_EFFECTIVE] = "effective resolution",
1691	[IPU3_CSS_RECT_BDS] = "bayer-domain scaled resolution",
1692	[IPU3_CSS_RECT_ENVELOPE] = "DVS envelope size",
1693	[IPU3_CSS_RECT_GDC] = "GDC output res",
1694	};
1695	struct v4l2_rect r[IPU3_CSS_RECTS] = { };
1696	struct v4l2_rect *const eff = &r[IPU3_CSS_RECT_EFFECTIVE];
1697	struct v4l2_rect *const bds = &r[IPU3_CSS_RECT_BDS];
1698	struct v4l2_rect *const env = &r[IPU3_CSS_RECT_ENVELOPE];
1699	struct v4l2_rect *const gdc = &r[IPU3_CSS_RECT_GDC];
1700	struct imgu_css_queue *q;
1701	struct v4l2_pix_format_mplane *in, *out, *vf;
1702	int i, s, ret;
1703
1704	q = kcalloc(IPU3_CSS_QUEUES, size: sizeof(struct imgu_css_queue), GFP_KERNEL);
1705	if (!q)
1706	return -ENOMEM;
1707
1708	in = &q[IPU3_CSS_QUEUE_IN].fmt.mpix;
1709	out = &q[IPU3_CSS_QUEUE_OUT].fmt.mpix;
1710	vf = &q[IPU3_CSS_QUEUE_VF].fmt.mpix;
1711
1712	/* Adjust all formats, get statistics buffer sizes and formats */
1713	for (i = 0; i < IPU3_CSS_QUEUES; i++) {
1714	if (fmts[i])
1715	dev_dbg(css->dev, "%s %s: (%i,%i) fmt 0x%x\n", __func__,
1716	qnames[i], fmts[i]->width, fmts[i]->height,
1717	fmts[i]->pixelformat);
1718	else
1719	dev_dbg(css->dev, "%s %s: (not set)\n", __func__,
1720	qnames[i]);
1721	if (imgu_css_queue_init(queue: &q[i], fmt: fmts[i],
1722	IPU3_CSS_QUEUE_TO_FLAGS(i))) {
1723	dev_notice(css->dev, "can not initialize queue %s\n",
1724	qnames[i]);
1725	ret = -EINVAL;
1726	goto out;
1727	}
1728	}
1729	for (i = 0; i < IPU3_CSS_RECTS; i++) {
1730	if (rects[i]) {
1731	dev_dbg(css->dev, "%s %s: (%i,%i)\n", __func__,
1732	rnames[i], rects[i]->width, rects[i]->height);
1733	r[i].width = rects[i]->width;
1734	r[i].height = rects[i]->height;
1735	} else {
1736	dev_dbg(css->dev, "%s %s: (not set)\n", __func__,
1737	rnames[i]);
1738	}
1739	/* For now, force known good resolutions */
1740	r[i].left = 0;
1741	r[i].top = 0;
1742	}
1743
1744	/* Always require one input and vf only if out is also enabled */
1745	if (!imgu_css_queue_enabled(q: &q[IPU3_CSS_QUEUE_IN]) ||
1746	!imgu_css_queue_enabled(q: &q[IPU3_CSS_QUEUE_OUT])) {
1747	dev_warn(css->dev, "required queues are disabled\n");
1748	ret = -EINVAL;
1749	goto out;
1750	}
1751
1752	if (!imgu_css_queue_enabled(q: &q[IPU3_CSS_QUEUE_OUT])) {
1753	out->width = in->width;
1754	out->height = in->height;
1755	}
1756	if (eff->width <= 0 || eff->height <= 0) {
1757	eff->width = in->width;
1758	eff->height = in->height;
1759	}
1760	if (bds->width <= 0 || bds->height <= 0) {
1761	bds->width = out->width;
1762	bds->height = out->height;
1763	}
1764	if (gdc->width <= 0 || gdc->height <= 0) {
1765	gdc->width = out->width;
1766	gdc->height = out->height;
1767	}
1768
1769	in->width = imgu_css_adjust(res: in->width, align: 1);
1770	in->height = imgu_css_adjust(res: in->height, align: 1);
1771	eff->width = imgu_css_adjust(res: eff->width, align: EFF_ALIGN_W);
1772	eff->height = imgu_css_adjust(res: eff->height, align: 1);
1773	bds->width = imgu_css_adjust(res: bds->width, align: BDS_ALIGN_W);
1774	bds->height = imgu_css_adjust(res: bds->height, align: 1);
1775	gdc->width = imgu_css_adjust(res: gdc->width, align: OUT_ALIGN_W);
1776	gdc->height = imgu_css_adjust(res: gdc->height, align: OUT_ALIGN_H);
1777	out->width = imgu_css_adjust(res: out->width, align: OUT_ALIGN_W);
1778	out->height = imgu_css_adjust(res: out->height, align: OUT_ALIGN_H);
1779	vf->width = imgu_css_adjust(res: vf->width, align: VF_ALIGN_W);
1780	vf->height = imgu_css_adjust(res: vf->height, align: 1);
1781
1782	s = (bds->width - gdc->width) / 2;
1783	env->width = s < MIN_ENVELOPE ? MIN_ENVELOPE : s;
1784	s = (bds->height - gdc->height) / 2;
1785	env->height = s < MIN_ENVELOPE ? MIN_ENVELOPE : s;
1786
1787	ret = imgu_css_find_binary(css, pipe, queue: q, rects: r);
1788	if (ret < 0) {
1789	dev_err(css->dev, "failed to find suitable binary\n");
1790	ret = -EINVAL;
1791	goto out;
1792	}
1793	css->pipes[pipe].bindex = ret;
1794
1795	dev_dbg(css->dev, "Binary index %d for pipe %d found.",
1796	css->pipes[pipe].bindex, pipe);
1797
1798	/* Final adjustment and set back the queried formats */
1799	for (i = 0; i < IPU3_CSS_QUEUES; i++) {
1800	if (fmts[i]) {
1801	if (imgu_css_queue_init(queue: &q[i], fmt: &q[i].fmt.mpix,
1802	IPU3_CSS_QUEUE_TO_FLAGS(i))) {
1803	dev_err(css->dev,
1804	"final resolution adjustment failed\n");
1805	ret = -EINVAL;
1806	goto out;
1807	}
1808	*fmts[i] = q[i].fmt.mpix;
1809	}
1810	}
1811
1812	for (i = 0; i < IPU3_CSS_RECTS; i++)
1813	if (rects[i])
1814	*rects[i] = r[i];
1815
1816	dev_dbg(css->dev,
1817	"in(%u,%u) if(%u,%u) ds(%u,%u) gdc(%u,%u) out(%u,%u) vf(%u,%u)",
1818	in->width, in->height, eff->width, eff->height,
1819	bds->width, bds->height, gdc->width, gdc->height,
1820	out->width, out->height, vf->width, vf->height);
1821
1822	ret = 0;
1823	out:
1824	kfree(objp: q);
1825	return ret;
1826	}
1827
1828	int imgu_css_fmt_set(struct imgu_css *css,
1829	struct v4l2_pix_format_mplane *fmts[IPU3_CSS_QUEUES],
1830	struct v4l2_rect *rects[IPU3_CSS_RECTS],
1831	unsigned int pipe)
1832	{
1833	struct v4l2_rect rect_data[IPU3_CSS_RECTS];
1834	struct v4l2_rect *all_rects[IPU3_CSS_RECTS];
1835	int i, r;
1836	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1837
1838	for (i = 0; i < IPU3_CSS_RECTS; i++) {
1839	if (rects[i])
1840	rect_data[i] = *rects[i];
1841	else
1842	memset(&rect_data[i], 0, sizeof(rect_data[i]));
1843	all_rects[i] = &rect_data[i];
1844	}
1845	r = imgu_css_fmt_try(css, fmts, rects: all_rects, pipe);
1846	if (r < 0)
1847	return r;
1848
1849	for (i = 0; i < IPU3_CSS_QUEUES; i++)
1850	if (imgu_css_queue_init(queue: &css_pipe->queue[i], fmt: fmts[i],
1851	IPU3_CSS_QUEUE_TO_FLAGS(i)))
1852	return -EINVAL;
1853	for (i = 0; i < IPU3_CSS_RECTS; i++) {
1854	css_pipe->rect[i] = rect_data[i];
1855	if (rects[i])
1856	*rects[i] = rect_data[i];
1857	}
1858
1859	return 0;
1860	}
1861
1862	int imgu_css_meta_fmt_set(struct v4l2_meta_format *fmt)
1863	{
1864	switch (fmt->dataformat) {
1865	case V4L2_META_FMT_IPU3_PARAMS:
1866	fmt->buffersize = sizeof(struct ipu3_uapi_params);
1867
1868	/*
1869	* Sanity check for the parameter struct size. This must
1870	* not change!
1871	*/
1872	BUILD_BUG_ON(sizeof(struct ipu3_uapi_params) != 39328);
1873
1874	break;
1875	case V4L2_META_FMT_IPU3_STAT_3A:
1876	fmt->buffersize = sizeof(struct ipu3_uapi_stats_3a);
1877	break;
1878	default:
1879	return -EINVAL;
1880	}
1881
1882	return 0;
1883	}
1884
1885	/*
1886	* Queue given buffer to CSS. imgu_css_buf_prepare() must have been first
1887	* called for the buffer. May be called from interrupt context.
1888	* Returns 0 on success, -EBUSY if the buffer queue is full, or some other
1889	* code on error conditions.
1890	*/
1891	int imgu_css_buf_queue(struct imgu_css *css, unsigned int pipe,
1892	struct imgu_css_buffer *b)
1893	{
1894	struct imgu_abi_buffer *abi_buf;
1895	struct imgu_addr_t *buf_addr;
1896	u32 data;
1897	int r;
1898	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
1899
1900	if (!css->streaming)
1901	return -EPROTO; /* CSS or buffer in wrong state */
1902
1903	if (b->queue >= IPU3_CSS_QUEUES || !imgu_css_queues[b->queue].qid)
1904	return -EINVAL;
1905
1906	b->queue_pos = imgu_css_queue_pos(css, queue: imgu_css_queues[b->queue].qid,
1907	thread: pipe);
1908
1909	if (b->queue_pos >= ARRAY_SIZE(css->pipes[pipe].abi_buffers[b->queue]))
1910	return -EIO;
1911	abi_buf = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].vaddr;
1912
1913	/* Fill struct abi_buffer for firmware */
1914	memset(abi_buf, 0, sizeof(*abi_buf));
1915
1916	buf_addr = (void *)abi_buf + imgu_css_queues[b->queue].ptr_ofs;
1917	*(imgu_addr_t *)buf_addr = b->daddr;
1918
1919	if (b->queue == IPU3_CSS_QUEUE_STAT_3A)
1920	abi_buf->payload.s3a.data.dmem.s3a_tbl = b->daddr;
1921
1922	if (b->queue == IPU3_CSS_QUEUE_OUT)
1923	abi_buf->payload.frame.padded_width =
1924	css_pipe->queue[IPU3_CSS_QUEUE_OUT].width_pad;
1925
1926	if (b->queue == IPU3_CSS_QUEUE_VF)
1927	abi_buf->payload.frame.padded_width =
1928	css_pipe->queue[IPU3_CSS_QUEUE_VF].width_pad;
1929
1930	spin_lock(lock: &css_pipe->qlock);
1931	list_add_tail(new: &b->list, head: &css_pipe->queue[b->queue].bufs);
1932	spin_unlock(lock: &css_pipe->qlock);
1933	b->state = IPU3_CSS_BUFFER_QUEUED;
1934
1935	data = css->pipes[pipe].abi_buffers[b->queue][b->queue_pos].daddr;
1936	r = imgu_css_queue_data(css, queue: imgu_css_queues[b->queue].qid,
1937	thread: pipe, data);
1938	if (r < 0)
1939	goto queueing_failed;
1940
1941	data = IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe,
1942	imgu_css_queues[b->queue].qid);
1943	r = imgu_css_queue_data(css, queue: IMGU_ABI_QUEUE_EVENT_ID, thread: pipe, data);
1944	if (r < 0)
1945	goto queueing_failed;
1946
1947	dev_dbg(css->dev, "queued buffer %p to css queue %i in pipe %d\n",
1948	b, b->queue, pipe);
1949
1950	return 0;
1951
1952	queueing_failed:
1953	b->state = (r == -EBUSY || r == -EAGAIN) ?
1954	IPU3_CSS_BUFFER_NEW : IPU3_CSS_BUFFER_FAILED;
1955	list_del(entry: &b->list);
1956
1957	return r;
1958	}
1959
1960	/*
1961	* Get next ready CSS buffer. Returns -EAGAIN in which case the function
1962	* should be called again, or -EBUSY which means that there are no more
1963	* buffers available. May be called from interrupt context.
1964	*/
1965	struct imgu_css_buffer *imgu_css_buf_dequeue(struct imgu_css *css)
1966	{
1967	static const unsigned char evtype_to_queue[] = {
1968	[IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE] = IPU3_CSS_QUEUE_IN,
1969	[IMGU_ABI_EVTTYPE_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_OUT,
1970	[IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE] = IPU3_CSS_QUEUE_VF,
1971	[IMGU_ABI_EVTTYPE_3A_STATS_DONE] = IPU3_CSS_QUEUE_STAT_3A,
1972	};
1973	struct imgu_css_buffer *b = ERR_PTR(error: -EAGAIN);
1974	u32 event, daddr;
1975	int evtype, pipe, pipeid, queue, qid, r;
1976	struct imgu_css_pipe *css_pipe;
1977
1978	if (!css->streaming)
1979	return ERR_PTR(error: -EPROTO);
1980
1981	r = imgu_css_dequeue_data(css, queue: IMGU_ABI_QUEUE_EVENT_ID, data: &event);
1982	if (r < 0)
1983	return ERR_PTR(error: r);
1984
1985	evtype = (event & IMGU_ABI_EVTTYPE_EVENT_MASK) >>
1986	IMGU_ABI_EVTTYPE_EVENT_SHIFT;
1987
1988	switch (evtype) {
1989	case IMGU_ABI_EVTTYPE_OUT_FRAME_DONE:
1990	case IMGU_ABI_EVTTYPE_VF_OUT_FRAME_DONE:
1991	case IMGU_ABI_EVTTYPE_3A_STATS_DONE:
1992	case IMGU_ABI_EVTTYPE_INPUT_FRAME_DONE:
1993	pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >>
1994	IMGU_ABI_EVTTYPE_PIPE_SHIFT;
1995	pipeid = (event & IMGU_ABI_EVTTYPE_PIPEID_MASK) >>
1996	IMGU_ABI_EVTTYPE_PIPEID_SHIFT;
1997	queue = evtype_to_queue[evtype];
1998	qid = imgu_css_queues[queue].qid;
1999
2000	if (pipe >= IMGU_MAX_PIPE_NUM) {
2001	dev_err(css->dev, "Invalid pipe: %i\n", pipe);
2002	return ERR_PTR(error: -EIO);
2003	}
2004
2005	if (qid >= IMGU_ABI_QUEUE_NUM) {
2006	dev_err(css->dev, "Invalid qid: %i\n", qid);
2007	return ERR_PTR(error: -EIO);
2008	}
2009	css_pipe = &css->pipes[pipe];
2010	dev_dbg(css->dev,
2011	"event: buffer done 0x%x queue %i pipe %i pipeid %i\n",
2012	event, queue, pipe, pipeid);
2013
2014	r = imgu_css_dequeue_data(css, queue: qid, data: &daddr);
2015	if (r < 0) {
2016	dev_err(css->dev, "failed to dequeue buffer\n");
2017	/* Force real error, not -EBUSY */
2018	return ERR_PTR(error: -EIO);
2019	}
2020
2021	r = imgu_css_queue_data(css, queue: IMGU_ABI_QUEUE_EVENT_ID, thread: pipe,
2022	IMGU_ABI_EVENT_BUFFER_DEQUEUED(qid));
2023	if (r < 0) {
2024	dev_err(css->dev, "failed to queue event\n");
2025	return ERR_PTR(error: -EIO);
2026	}
2027
2028	spin_lock(lock: &css_pipe->qlock);
2029	if (list_empty(head: &css_pipe->queue[queue].bufs)) {
2030	spin_unlock(lock: &css_pipe->qlock);
2031	dev_err(css->dev, "event on empty queue\n");
2032	return ERR_PTR(error: -EIO);
2033	}
2034	b = list_first_entry(&css_pipe->queue[queue].bufs,
2035	struct imgu_css_buffer, list);
2036	if (queue != b->queue ||
2037	daddr != css_pipe->abi_buffers
2038	[b->queue][b->queue_pos].daddr) {
2039	spin_unlock(lock: &css_pipe->qlock);
2040	dev_err(css->dev, "dequeued bad buffer 0x%x\n", daddr);
2041	return ERR_PTR(error: -EIO);
2042	}
2043
2044	dev_dbg(css->dev, "buffer 0x%8x done from pipe %d\n", daddr, pipe);
2045	b->pipe = pipe;
2046	b->state = IPU3_CSS_BUFFER_DONE;
2047	list_del(entry: &b->list);
2048	spin_unlock(lock: &css_pipe->qlock);
2049	break;
2050	case IMGU_ABI_EVTTYPE_PIPELINE_DONE:
2051	pipe = (event & IMGU_ABI_EVTTYPE_PIPE_MASK) >>
2052	IMGU_ABI_EVTTYPE_PIPE_SHIFT;
2053	if (pipe >= IMGU_MAX_PIPE_NUM) {
2054	dev_err(css->dev, "Invalid pipe: %i\n", pipe);
2055	return ERR_PTR(error: -EIO);
2056	}
2057
2058	css_pipe = &css->pipes[pipe];
2059	dev_dbg(css->dev, "event: pipeline done 0x%8x for pipe %d\n",
2060	event, pipe);
2061	break;
2062	case IMGU_ABI_EVTTYPE_TIMER:
2063	r = imgu_css_dequeue_data(css, queue: IMGU_ABI_QUEUE_EVENT_ID, data: &event);
2064	if (r < 0)
2065	return ERR_PTR(error: r);
2066
2067	if ((event & IMGU_ABI_EVTTYPE_EVENT_MASK) >>
2068	IMGU_ABI_EVTTYPE_EVENT_SHIFT == IMGU_ABI_EVTTYPE_TIMER)
2069	dev_dbg(css->dev, "event: timer\n");
2070	else
2071	dev_warn(css->dev, "half of timer event missing\n");
2072	break;
2073	case IMGU_ABI_EVTTYPE_FW_WARNING:
2074	dev_warn(css->dev, "event: firmware warning 0x%x\n", event);
2075	break;
2076	case IMGU_ABI_EVTTYPE_FW_ASSERT:
2077	dev_err(css->dev,
2078	"event: firmware assert 0x%x module_id %i line_no %i\n",
2079	event,
2080	(event & IMGU_ABI_EVTTYPE_MODULEID_MASK) >>
2081	IMGU_ABI_EVTTYPE_MODULEID_SHIFT,
2082	swab16((event & IMGU_ABI_EVTTYPE_LINENO_MASK) >>
2083	IMGU_ABI_EVTTYPE_LINENO_SHIFT));
2084	break;
2085	default:
2086	dev_warn(css->dev, "received unknown event 0x%x\n", event);
2087	}
2088
2089	return b;
2090	}
2091
2092	/*
2093	* Get a new set of parameters from pool and initialize them based on
2094	* the parameters params, gdc, and obgrid. Any of these may be NULL,
2095	* in which case the previously set parameters are used.
2096	* If parameters haven't been set previously, initialize from scratch.
2097	*
2098	* Return index to css->parameter_set_info which has the newly created
2099	* parameters or negative value on error.
2100	*/
2101	int imgu_css_set_parameters(struct imgu_css *css, unsigned int pipe,
2102	struct ipu3_uapi_params *set_params)
2103	{
2104	static const unsigned int queue_id = IMGU_ABI_QUEUE_A_ID;
2105	struct imgu_css_pipe *css_pipe = &css->pipes[pipe];
2106	const int stage = 0;
2107	const struct imgu_fw_info *bi;
2108	int obgrid_size;
2109	unsigned int stripes, i;
2110	struct ipu3_uapi_flags *use = set_params ? &set_params->use : NULL;
2111
2112	/* Destination buffers which are filled here */
2113	struct imgu_abi_parameter_set_info *param_set;
2114	struct imgu_abi_acc_param *acc = NULL;
2115	struct imgu_abi_gdc_warp_param *gdc = NULL;
2116	struct ipu3_uapi_obgrid_param *obgrid = NULL;
2117	const struct imgu_css_map *map;
2118	void *vmem0 = NULL;
2119	void *dmem0 = NULL;
2120
2121	enum imgu_abi_memories m;
2122	int r = -EBUSY;
2123
2124	if (!css->streaming)
2125	return -EPROTO;
2126
2127	dev_dbg(css->dev, "%s for pipe %d", __func__, pipe);
2128
2129	bi = &css->fwp->binary_header[css_pipe->bindex];
2130	obgrid_size = imgu_css_fw_obgrid_size(bi);
2131	stripes = bi->info.isp.sp.iterator.num_stripes ? : 1;
2132
2133	imgu_css_pool_get(pool: &css_pipe->pool.parameter_set_info);
2134	param_set = imgu_css_pool_last(pool: &css_pipe->pool.parameter_set_info,
2135	last: 0)->vaddr;
2136
2137	/* Get a new acc only if new parameters given, or none yet */
2138	map = imgu_css_pool_last(pool: &css_pipe->pool.acc, last: 0);
2139	if (set_params || !map->vaddr) {
2140	imgu_css_pool_get(pool: &css_pipe->pool.acc);
2141	map = imgu_css_pool_last(pool: &css_pipe->pool.acc, last: 0);
2142	acc = map->vaddr;
2143	}
2144
2145	/* Get new VMEM0 only if needed, or none yet */
2146	m = IMGU_ABI_MEM_ISP_VMEM0;
2147	map = imgu_css_pool_last(pool: &css_pipe->pool.binary_params_p[m], last: 0);
2148	if (!map->vaddr || (set_params && (set_params->use.lin_vmem_params ||
2149	set_params->use.tnr3_vmem_params ||
2150	set_params->use.xnr3_vmem_params))) {
2151	imgu_css_pool_get(pool: &css_pipe->pool.binary_params_p[m]);
2152	map = imgu_css_pool_last(pool: &css_pipe->pool.binary_params_p[m], last: 0);
2153	vmem0 = map->vaddr;
2154	}
2155
2156	/* Get new DMEM0 only if needed, or none yet */
2157	m = IMGU_ABI_MEM_ISP_DMEM0;
2158	map = imgu_css_pool_last(pool: &css_pipe->pool.binary_params_p[m], last: 0);
2159	if (!map->vaddr || (set_params && (set_params->use.tnr3_dmem_params ||
2160	set_params->use.xnr3_dmem_params))) {
2161	imgu_css_pool_get(pool: &css_pipe->pool.binary_params_p[m]);
2162	map = imgu_css_pool_last(pool: &css_pipe->pool.binary_params_p[m], last: 0);
2163	dmem0 = map->vaddr;
2164	}
2165
2166	/* Configure acc parameter cluster */
2167	if (acc) {
2168	/* get acc_old */
2169	map = imgu_css_pool_last(pool: &css_pipe->pool.acc, last: 1);
2170	/* user acc */
2171	r = imgu_css_cfg_acc(css, pipe, use, acc, acc_old: map->vaddr,
2172	acc_user: set_params ? &set_params->acc_param : NULL);
2173	if (r < 0)
2174	goto fail;
2175	}
2176
2177	/* Configure late binding parameters */
2178	if (vmem0) {
2179	m = IMGU_ABI_MEM_ISP_VMEM0;
2180	map = imgu_css_pool_last(pool: &css_pipe->pool.binary_params_p[m], last: 1);
2181	r = imgu_css_cfg_vmem0(css, pipe, use, vmem0,
2182	vmem0_old: map->vaddr, user: set_params);
2183	if (r < 0)
2184	goto fail;
2185	}
2186
2187	if (dmem0) {
2188	m = IMGU_ABI_MEM_ISP_DMEM0;
2189	map = imgu_css_pool_last(pool: &css_pipe->pool.binary_params_p[m], last: 1);
2190	r = imgu_css_cfg_dmem0(css, pipe, use, dmem0,
2191	dmem0_old: map->vaddr, user: set_params);
2192	if (r < 0)
2193	goto fail;
2194	}
2195
2196	/* Get a new gdc only if a new gdc is given, or none yet */
2197	if (bi->info.isp.sp.enable.dvs_6axis) {
2198	unsigned int a = IPU3_CSS_AUX_FRAME_REF;
2199	unsigned int g = IPU3_CSS_RECT_GDC;
2200	unsigned int e = IPU3_CSS_RECT_ENVELOPE;
2201
2202	map = imgu_css_pool_last(pool: &css_pipe->pool.gdc, last: 0);
2203	if (!map->vaddr) {
2204	imgu_css_pool_get(pool: &css_pipe->pool.gdc);
2205	map = imgu_css_pool_last(pool: &css_pipe->pool.gdc, last: 0);
2206	gdc = map->vaddr;
2207	imgu_css_cfg_gdc_table(gdc: map->vaddr,
2208	frame_in_x: css_pipe->aux_frames[a].bytesperline /
2209	css_pipe->aux_frames[a].bytesperpixel,
2210	frame_in_y: css_pipe->aux_frames[a].height,
2211	frame_out_x: css_pipe->rect[g].width,
2212	frame_out_y: css_pipe->rect[g].height,
2213	env_w: css_pipe->rect[e].width,
2214	env_h: css_pipe->rect[e].height);
2215	}
2216	}
2217
2218	/* Get a new obgrid only if a new obgrid is given, or none yet */
2219	map = imgu_css_pool_last(pool: &css_pipe->pool.obgrid, last: 0);
2220	if (!map->vaddr || (set_params && set_params->use.obgrid_param)) {
2221	imgu_css_pool_get(pool: &css_pipe->pool.obgrid);
2222	map = imgu_css_pool_last(pool: &css_pipe->pool.obgrid, last: 0);
2223	obgrid = map->vaddr;
2224
2225	/* Configure optical black level grid (obgrid) */
2226	if (set_params && set_params->use.obgrid_param)
2227	for (i = 0; i < obgrid_size / sizeof(*obgrid); i++)
2228	obgrid[i] = set_params->obgrid_param;
2229	else
2230	memset(obgrid, 0, obgrid_size);
2231	}
2232
2233	/* Configure parameter set info, queued to `queue_id' */
2234
2235	memset(param_set, 0, sizeof(*param_set));
2236	map = imgu_css_pool_last(pool: &css_pipe->pool.acc, last: 0);
2237	param_set->mem_map.acc_cluster_params_for_sp = map->daddr;
2238
2239	map = imgu_css_pool_last(pool: &css_pipe->pool.gdc, last: 0);
2240	param_set->mem_map.dvs_6axis_params_y = map->daddr;
2241
2242	for (i = 0; i < stripes; i++) {
2243	map = imgu_css_pool_last(pool: &css_pipe->pool.obgrid, last: 0);
2244	param_set->mem_map.obgrid_tbl[i] =
2245	map->daddr + (obgrid_size / stripes) * i;
2246	}
2247
2248	for (m = 0; m < IMGU_ABI_NUM_MEMORIES; m++) {
2249	map = imgu_css_pool_last(pool: &css_pipe->pool.binary_params_p[m], last: 0);
2250	param_set->mem_map.isp_mem_param[stage][m] = map->daddr;
2251	}
2252
2253	/* Then queue the new parameter buffer */
2254	map = imgu_css_pool_last(pool: &css_pipe->pool.parameter_set_info, last: 0);
2255	r = imgu_css_queue_data(css, queue: queue_id, thread: pipe, data: map->daddr);
2256	if (r < 0)
2257	goto fail;
2258
2259	r = imgu_css_queue_data(css, queue: IMGU_ABI_QUEUE_EVENT_ID, thread: pipe,
2260	IMGU_ABI_EVENT_BUFFER_ENQUEUED(pipe,
2261	queue_id));
2262	if (r < 0)
2263	goto fail_no_put;
2264
2265	/* Finally dequeue all old parameter buffers */
2266
2267	do {
2268	u32 daddr;
2269
2270	r = imgu_css_dequeue_data(css, queue: queue_id, data: &daddr);
2271	if (r == -EBUSY)
2272	break;
2273	if (r)
2274	goto fail_no_put;
2275	r = imgu_css_queue_data(css, queue: IMGU_ABI_QUEUE_EVENT_ID, thread: pipe,
2276	IMGU_ABI_EVENT_BUFFER_DEQUEUED
2277	(queue_id));
2278	if (r < 0) {
2279	dev_err(css->dev, "failed to queue parameter event\n");
2280	goto fail_no_put;
2281	}
2282	} while (1);
2283
2284	return 0;
2285
2286	fail:
2287	/*
2288	* A failure, most likely the parameter queue was full.
2289	* Return error but continue streaming. User can try submitting new
2290	* parameters again later.
2291	*/
2292
2293	imgu_css_pool_put(pool: &css_pipe->pool.parameter_set_info);
2294	if (acc)
2295	imgu_css_pool_put(pool: &css_pipe->pool.acc);
2296	if (gdc)
2297	imgu_css_pool_put(pool: &css_pipe->pool.gdc);
2298	if (obgrid)
2299	imgu_css_pool_put(pool: &css_pipe->pool.obgrid);
2300	if (vmem0)
2301	imgu_css_pool_put(
2302	pool: &css_pipe->pool.binary_params_p
2303	[IMGU_ABI_MEM_ISP_VMEM0]);
2304	if (dmem0)
2305	imgu_css_pool_put(
2306	pool: &css_pipe->pool.binary_params_p
2307	[IMGU_ABI_MEM_ISP_DMEM0]);
2308
2309	fail_no_put:
2310	return r;
2311	}
2312
2313	int imgu_css_irq_ack(struct imgu_css *css)
2314	{
2315	static const int NUM_SWIRQS = 3;
2316	struct imgu_fw_info *bi = &css->fwp->binary_header[css->fw_sp[0]];
2317	void __iomem *const base = css->base;
2318	u32 irq_status[IMGU_IRQCTRL_NUM];
2319	int i;
2320
2321	u32 imgu_status = readl(addr: base + IMGU_REG_INT_STATUS);
2322
2323	writel(val: imgu_status, addr: base + IMGU_REG_INT_STATUS);
2324	for (i = 0; i < IMGU_IRQCTRL_NUM; i++)
2325	irq_status[i] = readl(addr: base + IMGU_REG_IRQCTRL_STATUS(i));
2326
2327	for (i = 0; i < NUM_SWIRQS; i++) {
2328	if (irq_status[IMGU_IRQCTRL_SP0] & IMGU_IRQCTRL_IRQ_SW_PIN(i)) {
2329	/* SP SW interrupt */
2330	u32 cnt = readl(addr: base + IMGU_REG_SP_DMEM_BASE(0) +
2331	bi->info.sp.output);
2332	u32 val = readl(addr: base + IMGU_REG_SP_DMEM_BASE(0) +
2333	bi->info.sp.output + 4 + 4 * i);
2334
2335	dev_dbg(css->dev, "%s: swirq %i cnt %i val 0x%x\n",
2336	__func__, i, cnt, val);
2337	}
2338	}
2339
2340	for (i = IMGU_IRQCTRL_NUM - 1; i >= 0; i--)
2341	if (irq_status[i]) {
2342	writel(val: irq_status[i], addr: base + IMGU_REG_IRQCTRL_CLEAR(i));
2343	/* Wait for write to complete */
2344	readl(addr: base + IMGU_REG_IRQCTRL_ENABLE(i));
2345	}
2346
2347	dev_dbg(css->dev, "%s: imgu 0x%x main 0x%x sp0 0x%x sp1 0x%x\n",
2348	__func__, imgu_status, irq_status[IMGU_IRQCTRL_MAIN],
2349	irq_status[IMGU_IRQCTRL_SP0], irq_status[IMGU_IRQCTRL_SP1]);
2350
2351	if (!imgu_status && !irq_status[IMGU_IRQCTRL_MAIN])
2352	return -ENOMSG;
2353
2354	return 0;
2355	}
2356