1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
4	*
5	* Copyright (C) 2010-2011 Texas Instruments Incorporated - https://www.ti.com
6	*
7	* Authors: Felipe Balbi <balbi@ti.com>,
8	* Sebastian Andrzej Siewior <bigeasy@linutronix.de>
9	*/
10
11	#include <linux/kernel.h>
12	#include <linux/delay.h>
13	#include <linux/slab.h>
14	#include <linux/spinlock.h>
15	#include <linux/platform_device.h>
16	#include <linux/pm_runtime.h>
17	#include <linux/interrupt.h>
18	#include <linux/io.h>
19	#include <linux/list.h>
20	#include <linux/dma-mapping.h>
21
22	#include <linux/usb/ch9.h>
23	#include <linux/usb/gadget.h>
24
25	#include "debug.h"
26	#include "core.h"
27	#include "gadget.h"
28	#include "io.h"
29
30	#define DWC3_ALIGN_FRAME(d, n) (((d)->frame_number + ((d)->interval * (n))) \
31	& ~((d)->interval - 1))
32
33	/**
34	* dwc3_gadget_set_test_mode - enables usb2 test modes
35	* @dwc: pointer to our context structure
36	* @mode: the mode to set (J, K SE0 NAK, Force Enable)
37	*
38	* Caller should take care of locking. This function will return 0 on
39	* success or -EINVAL if wrong Test Selector is passed.
40	*/
41	int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
42	{
43	u32 reg;
44
45	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
46	reg &= ~DWC3_DCTL_TSTCTRL_MASK;
47
48	switch (mode) {
49	case USB_TEST_J:
50	case USB_TEST_K:
51	case USB_TEST_SE0_NAK:
52	case USB_TEST_PACKET:
53	case USB_TEST_FORCE_ENABLE:
54	reg |= mode << 1;
55	break;
56	default:
57	return -EINVAL;
58	}
59
60	dwc3_gadget_dctl_write_safe(dwc, value: reg);
61
62	return 0;
63	}
64
65	/**
66	* dwc3_gadget_get_link_state - gets current state of usb link
67	* @dwc: pointer to our context structure
68	*
69	* Caller should take care of locking. This function will
70	* return the link state on success (>= 0) or -ETIMEDOUT.
71	*/
72	int dwc3_gadget_get_link_state(struct dwc3 *dwc)
73	{
74	u32 reg;
75
76	reg = dwc3_readl(base: dwc->regs, DWC3_DSTS);
77
78	return DWC3_DSTS_USBLNKST(reg);
79	}
80
81	/**
82	* dwc3_gadget_set_link_state - sets usb link to a particular state
83	* @dwc: pointer to our context structure
84	* @state: the state to put link into
85	*
86	* Caller should take care of locking. This function will
87	* return 0 on success or -ETIMEDOUT.
88	*/
89	int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
90	{
91	int retries = 10000;
92	u32 reg;
93
94	/*
95	* Wait until device controller is ready. Only applies to 1.94a and
96	* later RTL.
97	*/
98	if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) {
99	while (--retries) {
100	reg = dwc3_readl(base: dwc->regs, DWC3_DSTS);
101	if (reg & DWC3_DSTS_DCNRD)
102	udelay(5);
103	else
104	break;
105	}
106
107	if (retries <= 0)
108	return -ETIMEDOUT;
109	}
110
111	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
112	reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
113
114	/* set no action before sending new link state change */
115	dwc3_writel(base: dwc->regs, DWC3_DCTL, value: reg);
116
117	/* set requested state */
118	reg |= DWC3_DCTL_ULSTCHNGREQ(state);
119	dwc3_writel(base: dwc->regs, DWC3_DCTL, value: reg);
120
121	/*
122	* The following code is racy when called from dwc3_gadget_wakeup,
123	* and is not needed, at least on newer versions
124	*/
125	if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
126	return 0;
127
128	/* wait for a change in DSTS */
129	retries = 10000;
130	while (--retries) {
131	reg = dwc3_readl(base: dwc->regs, DWC3_DSTS);
132
133	if (DWC3_DSTS_USBLNKST(reg) == state)
134	return 0;
135
136	udelay(5);
137	}
138
139	return -ETIMEDOUT;
140	}
141
142	static void dwc3_ep0_reset_state(struct dwc3 *dwc)
143	{
144	unsigned int dir;
145
146	if (dwc->ep0state != EP0_SETUP_PHASE) {
147	dir = !!dwc->ep0_expect_in;
148	if (dwc->ep0state == EP0_DATA_PHASE)
149	dwc3_ep0_end_control_data(dwc, dep: dwc->eps[dir]);
150	else
151	dwc3_ep0_end_control_data(dwc, dep: dwc->eps[!dir]);
152
153	dwc->eps[0]->trb_enqueue = 0;
154	dwc->eps[1]->trb_enqueue = 0;
155
156	dwc3_ep0_stall_and_restart(dwc);
157	}
158	}
159
160	/**
161	* dwc3_ep_inc_trb - increment a trb index.
162	* @index: Pointer to the TRB index to increment.
163	*
164	* The index should never point to the link TRB. After incrementing,
165	* if it is point to the link TRB, wrap around to the beginning. The
166	* link TRB is always at the last TRB entry.
167	*/
168	static void dwc3_ep_inc_trb(u8 *index)
169	{
170	(*index)++;
171	if (*index == (DWC3_TRB_NUM - 1))
172	*index = 0;
173	}
174
175	/**
176	* dwc3_ep_inc_enq - increment endpoint's enqueue pointer
177	* @dep: The endpoint whose enqueue pointer we're incrementing
178	*/
179	static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
180	{
181	dwc3_ep_inc_trb(index: &dep->trb_enqueue);
182	}
183
184	/**
185	* dwc3_ep_inc_deq - increment endpoint's dequeue pointer
186	* @dep: The endpoint whose enqueue pointer we're incrementing
187	*/
188	static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
189	{
190	dwc3_ep_inc_trb(index: &dep->trb_dequeue);
191	}
192
193	static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
194	struct dwc3_request *req, int status)
195	{
196	struct dwc3 *dwc = dep->dwc;
197
198	list_del(entry: &req->list);
199	req->remaining = 0;
200	req->needs_extra_trb = false;
201	req->num_trbs = 0;
202
203	if (req->request.status == -EINPROGRESS)
204	req->request.status = status;
205
206	if (req->trb)
207	usb_gadget_unmap_request_by_dev(dev: dwc->sysdev,
208	req: &req->request, is_in: req->direction);
209
210	req->trb = NULL;
211	trace_dwc3_gadget_giveback(req);
212
213	if (dep->number > 1)
214	pm_runtime_put(dev: dwc->dev);
215	}
216
217	/**
218	* dwc3_gadget_giveback - call struct usb_request's ->complete callback
219	* @dep: The endpoint to whom the request belongs to
220	* @req: The request we're giving back
221	* @status: completion code for the request
222	*
223	* Must be called with controller's lock held and interrupts disabled. This
224	* function will unmap @req and call its ->complete() callback to notify upper
225	* layers that it has completed.
226	*/
227	void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
228	int status)
229	{
230	struct dwc3 *dwc = dep->dwc;
231
232	dwc3_gadget_del_and_unmap_request(dep, req, status);
233	req->status = DWC3_REQUEST_STATUS_COMPLETED;
234
235	spin_unlock(lock: &dwc->lock);
236	usb_gadget_giveback_request(ep: &dep->endpoint, req: &req->request);
237	spin_lock(lock: &dwc->lock);
238	}
239
240	/**
241	* dwc3_send_gadget_generic_command - issue a generic command for the controller
242	* @dwc: pointer to the controller context
243	* @cmd: the command to be issued
244	* @param: command parameter
245	*
246	* Caller should take care of locking. Issue @cmd with a given @param to @dwc
247	* and wait for its completion.
248	*/
249	int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned int cmd,
250	u32 param)
251	{
252	u32 timeout = 500;
253	int status = 0;
254	int ret = 0;
255	u32 reg;
256
257	dwc3_writel(base: dwc->regs, DWC3_DGCMDPAR, value: param);
258	dwc3_writel(base: dwc->regs, DWC3_DGCMD, value: cmd | DWC3_DGCMD_CMDACT);
259
260	do {
261	reg = dwc3_readl(base: dwc->regs, DWC3_DGCMD);
262	if (!(reg & DWC3_DGCMD_CMDACT)) {
263	status = DWC3_DGCMD_STATUS(reg);
264	if (status)
265	ret = -EINVAL;
266	break;
267	}
268	} while (--timeout);
269
270	if (!timeout) {
271	ret = -ETIMEDOUT;
272	status = -ETIMEDOUT;
273	}
274
275	trace_dwc3_gadget_generic_cmd(cmd, param, status);
276
277	return ret;
278	}
279
280	static int __dwc3_gadget_wakeup(struct dwc3 *dwc, bool async);
281
282	/**
283	* dwc3_send_gadget_ep_cmd - issue an endpoint command
284	* @dep: the endpoint to which the command is going to be issued
285	* @cmd: the command to be issued
286	* @params: parameters to the command
287	*
288	* Caller should handle locking. This function will issue @cmd with given
289	* @params to @dep and wait for its completion.
290	*/
291	int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd,
292	struct dwc3_gadget_ep_cmd_params *params)
293	{
294	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
295	struct dwc3 *dwc = dep->dwc;
296	u32 timeout = 5000;
297	u32 saved_config = 0;
298	u32 reg;
299
300	int cmd_status = 0;
301	int ret = -EINVAL;
302
303	/*
304	* When operating in USB 2.0 speeds (HS/FS), if GUSB2PHYCFG.ENBLSLPM or
305	* GUSB2PHYCFG.SUSPHY is set, it must be cleared before issuing an
306	* endpoint command.
307	*
308	* Save and clear both GUSB2PHYCFG.ENBLSLPM and GUSB2PHYCFG.SUSPHY
309	* settings. Restore them after the command is completed.
310	*
311	* DWC_usb3 3.30a and DWC_usb31 1.90a programming guide section 3.2.2
312	*/
313	if (dwc->gadget->speed <= USB_SPEED_HIGH ||
314	DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER) {
315	reg = dwc3_readl(base: dwc->regs, DWC3_GUSB2PHYCFG(0));
316	if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
317	saved_config |= DWC3_GUSB2PHYCFG_SUSPHY;
318	reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
319	}
320
321	if (reg & DWC3_GUSB2PHYCFG_ENBLSLPM) {
322	saved_config |= DWC3_GUSB2PHYCFG_ENBLSLPM;
323	reg &= ~DWC3_GUSB2PHYCFG_ENBLSLPM;
324	}
325
326	if (saved_config)
327	dwc3_writel(base: dwc->regs, DWC3_GUSB2PHYCFG(0), value: reg);
328	}
329
330	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
331	int link_state;
332
333	/*
334	* Initiate remote wakeup if the link state is in U3 when
335	* operating in SS/SSP or L1/L2 when operating in HS/FS. If the
336	* link state is in U1/U2, no remote wakeup is needed. The Start
337	* Transfer command will initiate the link recovery.
338	*/
339	link_state = dwc3_gadget_get_link_state(dwc);
340	switch (link_state) {
341	case DWC3_LINK_STATE_U2:
342	if (dwc->gadget->speed >= USB_SPEED_SUPER)
343	break;
344
345	fallthrough;
346	case DWC3_LINK_STATE_U3:
347	ret = __dwc3_gadget_wakeup(dwc, async: false);
348	dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
349	ret);
350	break;
351	}
352	}
353
354	/*
355	* For some commands such as Update Transfer command, DEPCMDPARn
356	* registers are reserved. Since the driver often sends Update Transfer
357	* command, don't write to DEPCMDPARn to avoid register write delays and
358	* improve performance.
359	*/
360	if (DWC3_DEPCMD_CMD(cmd) != DWC3_DEPCMD_UPDATETRANSFER) {
361	dwc3_writel(base: dep->regs, DWC3_DEPCMDPAR0, value: params->param0);
362	dwc3_writel(base: dep->regs, DWC3_DEPCMDPAR1, value: params->param1);
363	dwc3_writel(base: dep->regs, DWC3_DEPCMDPAR2, value: params->param2);
364	}
365
366	/*
367	* Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
368	* not relying on XferNotReady, we can make use of a special "No
369	* Response Update Transfer" command where we should clear both CmdAct
370	* and CmdIOC bits.
371	*
372	* With this, we don't need to wait for command completion and can
373	* straight away issue further commands to the endpoint.
374	*
375	* NOTICE: We're making an assumption that control endpoints will never
376	* make use of Update Transfer command. This is a safe assumption
377	* because we can never have more than one request at a time with
378	* Control Endpoints. If anybody changes that assumption, this chunk
379	* needs to be updated accordingly.
380	*/
381	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
382	!usb_endpoint_xfer_isoc(epd: desc))
383	cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
384	else
385	cmd |= DWC3_DEPCMD_CMDACT;
386
387	dwc3_writel(base: dep->regs, DWC3_DEPCMD, value: cmd);
388
389	if (!(cmd & DWC3_DEPCMD_CMDACT) ||
390	(DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER &&
391	!(cmd & DWC3_DEPCMD_CMDIOC))) {
392	ret = 0;
393	goto skip_status;
394	}
395
396	do {
397	reg = dwc3_readl(base: dep->regs, DWC3_DEPCMD);
398	if (!(reg & DWC3_DEPCMD_CMDACT)) {
399	cmd_status = DWC3_DEPCMD_STATUS(reg);
400
401	switch (cmd_status) {
402	case 0:
403	ret = 0;
404	break;
405	case DEPEVT_TRANSFER_NO_RESOURCE:
406	dev_WARN(dwc->dev, "No resource for %s\n",
407	dep->name);
408	ret = -EINVAL;
409	break;
410	case DEPEVT_TRANSFER_BUS_EXPIRY:
411	/*
412	* SW issues START TRANSFER command to
413	* isochronous ep with future frame interval. If
414	* future interval time has already passed when
415	* core receives the command, it will respond
416	* with an error status of 'Bus Expiry'.
417	*
418	* Instead of always returning -EINVAL, let's
419	* give a hint to the gadget driver that this is
420	* the case by returning -EAGAIN.
421	*/
422	ret = -EAGAIN;
423	break;
424	default:
425	dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
426	}
427
428	break;
429	}
430	} while (--timeout);
431
432	if (timeout == 0) {
433	ret = -ETIMEDOUT;
434	cmd_status = -ETIMEDOUT;
435	}
436
437	skip_status:
438	trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
439
440	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
441	if (ret == 0)
442	dep->flags |= DWC3_EP_TRANSFER_STARTED;
443
444	if (ret != -ETIMEDOUT)
445	dwc3_gadget_ep_get_transfer_index(dep);
446	}
447
448	if (saved_config) {
449	reg = dwc3_readl(base: dwc->regs, DWC3_GUSB2PHYCFG(0));
450	reg |= saved_config;
451	dwc3_writel(base: dwc->regs, DWC3_GUSB2PHYCFG(0), value: reg);
452	}
453
454	return ret;
455	}
456
457	static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
458	{
459	struct dwc3 *dwc = dep->dwc;
460	struct dwc3_gadget_ep_cmd_params params;
461	u32 cmd = DWC3_DEPCMD_CLEARSTALL;
462
463	/*
464	* As of core revision 2.60a the recommended programming model
465	* is to set the ClearPendIN bit when issuing a Clear Stall EP
466	* command for IN endpoints. This is to prevent an issue where
467	* some (non-compliant) hosts may not send ACK TPs for pending
468	* IN transfers due to a mishandled error condition. Synopsys
469	* STAR 9000614252.
470	*/
471	if (dep->direction &&
472	!DWC3_VER_IS_PRIOR(DWC3, 260A) &&
473	(dwc->gadget->speed >= USB_SPEED_SUPER))
474	cmd |= DWC3_DEPCMD_CLEARPENDIN;
475
476	memset(&params, 0, sizeof(params));
477
478	return dwc3_send_gadget_ep_cmd(dep, cmd, params: &params);
479	}
480
481	static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
482	struct dwc3_trb *trb)
483	{
484	u32 offset = (char *) trb - (char *) dep->trb_pool;
485
486	return dep->trb_pool_dma + offset;
487	}
488
489	static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
490	{
491	struct dwc3 *dwc = dep->dwc;
492
493	if (dep->trb_pool)
494	return 0;
495
496	dep->trb_pool = dma_alloc_coherent(dev: dwc->sysdev,
497	size: sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
498	dma_handle: &dep->trb_pool_dma, GFP_KERNEL);
499	if (!dep->trb_pool) {
500	dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
501	dep->name);
502	return -ENOMEM;
503	}
504
505	return 0;
506	}
507
508	static void dwc3_free_trb_pool(struct dwc3_ep *dep)
509	{
510	struct dwc3 *dwc = dep->dwc;
511
512	dma_free_coherent(dev: dwc->sysdev, size: sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
513	cpu_addr: dep->trb_pool, dma_handle: dep->trb_pool_dma);
514
515	dep->trb_pool = NULL;
516	dep->trb_pool_dma = 0;
517	}
518
519	static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep)
520	{
521	struct dwc3_gadget_ep_cmd_params params;
522
523	memset(&params, 0x00, sizeof(params));
524
525	params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
526
527	return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
528	params: &params);
529	}
530
531	/**
532	* dwc3_gadget_start_config - configure ep resources
533	* @dep: endpoint that is being enabled
534	*
535	* Issue a %DWC3_DEPCMD_DEPSTARTCFG command to @dep. After the command's
536	* completion, it will set Transfer Resource for all available endpoints.
537	*
538	* The assignment of transfer resources cannot perfectly follow the data book
539	* due to the fact that the controller driver does not have all knowledge of the
540	* configuration in advance. It is given this information piecemeal by the
541	* composite gadget framework after every SET_CONFIGURATION and
542	* SET_INTERFACE. Trying to follow the databook programming model in this
543	* scenario can cause errors. For two reasons:
544	*
545	* 1) The databook says to do %DWC3_DEPCMD_DEPSTARTCFG for every
546	* %USB_REQ_SET_CONFIGURATION and %USB_REQ_SET_INTERFACE (8.1.5). This is
547	* incorrect in the scenario of multiple interfaces.
548	*
549	* 2) The databook does not mention doing more %DWC3_DEPCMD_DEPXFERCFG for new
550	* endpoint on alt setting (8.1.6).
551	*
552	* The following simplified method is used instead:
553	*
554	* All hardware endpoints can be assigned a transfer resource and this setting
555	* will stay persistent until either a core reset or hibernation. So whenever we
556	* do a %DWC3_DEPCMD_DEPSTARTCFG(0) we can go ahead and do
557	* %DWC3_DEPCMD_DEPXFERCFG for every hardware endpoint as well. We are
558	* guaranteed that there are as many transfer resources as endpoints.
559	*
560	* This function is called for each endpoint when it is being enabled but is
561	* triggered only when called for EP0-out, which always happens first, and which
562	* should only happen in one of the above conditions.
563	*/
564	static int dwc3_gadget_start_config(struct dwc3_ep *dep)
565	{
566	struct dwc3_gadget_ep_cmd_params params;
567	struct dwc3 *dwc;
568	u32 cmd;
569	int i;
570	int ret;
571
572	if (dep->number)
573	return 0;
574
575	memset(&params, 0x00, sizeof(params));
576	cmd = DWC3_DEPCMD_DEPSTARTCFG;
577	dwc = dep->dwc;
578
579	ret = dwc3_send_gadget_ep_cmd(dep, cmd, params: &params);
580	if (ret)
581	return ret;
582
583	for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
584	struct dwc3_ep *dep = dwc->eps[i];
585
586	if (!dep)
587	continue;
588
589	ret = dwc3_gadget_set_xfer_resource(dep);
590	if (ret)
591	return ret;
592	}
593
594	return 0;
595	}
596
597	static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action)
598	{
599	const struct usb_ss_ep_comp_descriptor *comp_desc;
600	const struct usb_endpoint_descriptor *desc;
601	struct dwc3_gadget_ep_cmd_params params;
602	struct dwc3 *dwc = dep->dwc;
603
604	comp_desc = dep->endpoint.comp_desc;
605	desc = dep->endpoint.desc;
606
607	memset(&params, 0x00, sizeof(params));
608
609	params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
610	| DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
611
612	/* Burst size is only needed in SuperSpeed mode */
613	if (dwc->gadget->speed >= USB_SPEED_SUPER) {
614	u32 burst = dep->endpoint.maxburst;
615
616	params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
617	}
618
619	params.param0 |= action;
620	if (action == DWC3_DEPCFG_ACTION_RESTORE)
621	params.param2 |= dep->saved_state;
622
623	if (usb_endpoint_xfer_control(epd: desc))
624	params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
625
626	if (dep->number <= 1 || usb_endpoint_xfer_isoc(epd: desc))
627	params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
628
629	if (usb_ss_max_streams(comp: comp_desc) && usb_endpoint_xfer_bulk(epd: desc)) {
630	params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
631	| DWC3_DEPCFG_XFER_COMPLETE_EN
632	| DWC3_DEPCFG_STREAM_EVENT_EN;
633	dep->stream_capable = true;
634	}
635
636	if (!usb_endpoint_xfer_control(epd: desc))
637	params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
638
639	/*
640	* We are doing 1:1 mapping for endpoints, meaning
641	* Physical Endpoints 2 maps to Logical Endpoint 2 and
642	* so on. We consider the direction bit as part of the physical
643	* endpoint number. So USB endpoint 0x81 is 0x03.
644	*/
645	params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
646
647	/*
648	* We must use the lower 16 TX FIFOs even though
649	* HW might have more
650	*/
651	if (dep->direction)
652	params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
653
654	if (desc->bInterval) {
655	u8 bInterval_m1;
656
657	/*
658	* Valid range for DEPCFG.bInterval_m1 is from 0 to 13.
659	*
660	* NOTE: The programming guide incorrectly stated bInterval_m1
661	* must be set to 0 when operating in fullspeed. Internally the
662	* controller does not have this limitation. See DWC_usb3x
663	* programming guide section 3.2.2.1.
664	*/
665	bInterval_m1 = min_t(u8, desc->bInterval - 1, 13);
666
667	if (usb_endpoint_type(epd: desc) == USB_ENDPOINT_XFER_INT &&
668	dwc->gadget->speed == USB_SPEED_FULL)
669	dep->interval = desc->bInterval;
670	else
671	dep->interval = 1 << (desc->bInterval - 1);
672
673	params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(bInterval_m1);
674	}
675
676	return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, params: &params);
677	}
678
679	/**
680	* dwc3_gadget_calc_tx_fifo_size - calculates the txfifo size value
681	* @dwc: pointer to the DWC3 context
682	* @mult: multiplier to be used when calculating the fifo_size
683	*
684	* Calculates the size value based on the equation below:
685	*
686	* DWC3 revision 280A and prior:
687	* fifo_size = mult * (max_packet / mdwidth) + 1;
688	*
689	* DWC3 revision 290A and onwards:
690	* fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
691	*
692	* The max packet size is set to 1024, as the txfifo requirements mainly apply
693	* to super speed USB use cases. However, it is safe to overestimate the fifo
694	* allocations for other scenarios, i.e. high speed USB.
695	*/
696	static int dwc3_gadget_calc_tx_fifo_size(struct dwc3 *dwc, int mult)
697	{
698	int max_packet = 1024;
699	int fifo_size;
700	int mdwidth;
701
702	mdwidth = dwc3_mdwidth(dwc);
703
704	/* MDWIDTH is represented in bits, we need it in bytes */
705	mdwidth >>= 3;
706
707	if (DWC3_VER_IS_PRIOR(DWC3, 290A))
708	fifo_size = mult * (max_packet / mdwidth) + 1;
709	else
710	fifo_size = mult * ((max_packet + mdwidth) / mdwidth) + 1;
711	return fifo_size;
712	}
713
714	/**
715	* dwc3_gadget_clear_tx_fifos - Clears txfifo allocation
716	* @dwc: pointer to the DWC3 context
717	*
718	* Iterates through all the endpoint registers and clears the previous txfifo
719	* allocations.
720	*/
721	void dwc3_gadget_clear_tx_fifos(struct dwc3 *dwc)
722	{
723	struct dwc3_ep *dep;
724	int fifo_depth;
725	int size;
726	int num;
727
728	if (!dwc->do_fifo_resize)
729	return;
730
731	/* Read ep0IN related TXFIFO size */
732	dep = dwc->eps[1];
733	size = dwc3_readl(base: dwc->regs, DWC3_GTXFIFOSIZ(0));
734	if (DWC3_IP_IS(DWC3))
735	fifo_depth = DWC3_GTXFIFOSIZ_TXFDEP(size);
736	else
737	fifo_depth = DWC31_GTXFIFOSIZ_TXFDEP(size);
738
739	dwc->last_fifo_depth = fifo_depth;
740	/* Clear existing TXFIFO for all IN eps except ep0 */
741	for (num = 3; num < min_t(int, dwc->num_eps, DWC3_ENDPOINTS_NUM);
742	num += 2) {
743	dep = dwc->eps[num];
744	/* Don't change TXFRAMNUM on usb31 version */
745	size = DWC3_IP_IS(DWC3) ? 0 :
746	dwc3_readl(base: dwc->regs, DWC3_GTXFIFOSIZ(num >> 1)) &
747	DWC31_GTXFIFOSIZ_TXFRAMNUM;
748
749	dwc3_writel(base: dwc->regs, DWC3_GTXFIFOSIZ(num >> 1), value: size);
750	dep->flags &= ~DWC3_EP_TXFIFO_RESIZED;
751	}
752	dwc->num_ep_resized = 0;
753	}
754
755	/*
756	* dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case
757	* @dwc: pointer to our context structure
758	*
759	* This function will a best effort FIFO allocation in order
760	* to improve FIFO usage and throughput, while still allowing
761	* us to enable as many endpoints as possible.
762	*
763	* Keep in mind that this operation will be highly dependent
764	* on the configured size for RAM1 - which contains TxFifo -,
765	* the amount of endpoints enabled on coreConsultant tool, and
766	* the width of the Master Bus.
767	*
768	* In general, FIFO depths are represented with the following equation:
769	*
770	* fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
771	*
772	* In conjunction with dwc3_gadget_check_config(), this resizing logic will
773	* ensure that all endpoints will have enough internal memory for one max
774	* packet per endpoint.
775	*/
776	static int dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep)
777	{
778	struct dwc3 *dwc = dep->dwc;
779	int fifo_0_start;
780	int ram1_depth;
781	int fifo_size;
782	int min_depth;
783	int num_in_ep;
784	int remaining;
785	int num_fifos = 1;
786	int fifo;
787	int tmp;
788
789	if (!dwc->do_fifo_resize)
790	return 0;
791
792	/* resize IN endpoints except ep0 */
793	if (!usb_endpoint_dir_in(epd: dep->endpoint.desc) || dep->number <= 1)
794	return 0;
795
796	/* bail if already resized */
797	if (dep->flags & DWC3_EP_TXFIFO_RESIZED)
798	return 0;
799
800	ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
801
802	if ((dep->endpoint.maxburst > 1 &&
803	usb_endpoint_xfer_bulk(epd: dep->endpoint.desc)) ||
804	usb_endpoint_xfer_isoc(epd: dep->endpoint.desc))
805	num_fifos = 3;
806
807	if (dep->endpoint.maxburst > 6 &&
808	(usb_endpoint_xfer_bulk(epd: dep->endpoint.desc) ||
809	usb_endpoint_xfer_isoc(epd: dep->endpoint.desc)) && DWC3_IP_IS(DWC31))
810	num_fifos = dwc->tx_fifo_resize_max_num;
811
812	/* FIFO size for a single buffer */
813	fifo = dwc3_gadget_calc_tx_fifo_size(dwc, mult: 1);
814
815	/* Calculate the number of remaining EPs w/o any FIFO */
816	num_in_ep = dwc->max_cfg_eps;
817	num_in_ep -= dwc->num_ep_resized;
818
819	/* Reserve at least one FIFO for the number of IN EPs */
820	min_depth = num_in_ep * (fifo + 1);
821	remaining = ram1_depth - min_depth - dwc->last_fifo_depth;
822	remaining = max_t(int, 0, remaining);
823	/*
824	* We've already reserved 1 FIFO per EP, so check what we can fit in
825	* addition to it. If there is not enough remaining space, allocate
826	* all the remaining space to the EP.
827	*/
828	fifo_size = (num_fifos - 1) * fifo;
829	if (remaining < fifo_size)
830	fifo_size = remaining;
831
832	fifo_size += fifo;
833	/* Last increment according to the TX FIFO size equation */
834	fifo_size++;
835
836	/* Check if TXFIFOs start at non-zero addr */
837	tmp = dwc3_readl(base: dwc->regs, DWC3_GTXFIFOSIZ(0));
838	fifo_0_start = DWC3_GTXFIFOSIZ_TXFSTADDR(tmp);
839
840	fifo_size |= (fifo_0_start + (dwc->last_fifo_depth << 16));
841	if (DWC3_IP_IS(DWC3))
842	dwc->last_fifo_depth += DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
843	else
844	dwc->last_fifo_depth += DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
845
846	/* Check fifo size allocation doesn't exceed available RAM size. */
847	if (dwc->last_fifo_depth >= ram1_depth) {
848	dev_err(dwc->dev, "Fifosize(%d) > RAM size(%d) %s depth:%d\n",
849	dwc->last_fifo_depth, ram1_depth,
850	dep->endpoint.name, fifo_size);
851	if (DWC3_IP_IS(DWC3))
852	fifo_size = DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
853	else
854	fifo_size = DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
855
856	dwc->last_fifo_depth -= fifo_size;
857	return -ENOMEM;
858	}
859
860	dwc3_writel(base: dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), value: fifo_size);
861	dep->flags |= DWC3_EP_TXFIFO_RESIZED;
862	dwc->num_ep_resized++;
863
864	return 0;
865	}
866
867	/**
868	* __dwc3_gadget_ep_enable - initializes a hw endpoint
869	* @dep: endpoint to be initialized
870	* @action: one of INIT, MODIFY or RESTORE
871	*
872	* Caller should take care of locking. Execute all necessary commands to
873	* initialize a HW endpoint so it can be used by a gadget driver.
874	*/
875	static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action)
876	{
877	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
878	struct dwc3 *dwc = dep->dwc;
879
880	u32 reg;
881	int ret;
882
883	if (!(dep->flags & DWC3_EP_ENABLED)) {
884	ret = dwc3_gadget_resize_tx_fifos(dep);
885	if (ret)
886	return ret;
887
888	ret = dwc3_gadget_start_config(dep);
889	if (ret)
890	return ret;
891	}
892
893	ret = dwc3_gadget_set_ep_config(dep, action);
894	if (ret)
895	return ret;
896
897	if (!(dep->flags & DWC3_EP_ENABLED)) {
898	struct dwc3_trb *trb_st_hw;
899	struct dwc3_trb *trb_link;
900
901	dep->type = usb_endpoint_type(epd: desc);
902	dep->flags |= DWC3_EP_ENABLED;
903
904	reg = dwc3_readl(base: dwc->regs, DWC3_DALEPENA);
905	reg |= DWC3_DALEPENA_EP(dep->number);
906	dwc3_writel(base: dwc->regs, DWC3_DALEPENA, value: reg);
907
908	dep->trb_dequeue = 0;
909	dep->trb_enqueue = 0;
910
911	if (usb_endpoint_xfer_control(epd: desc))
912	goto out;
913
914	/* Initialize the TRB ring */
915	memset(dep->trb_pool, 0,
916	sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
917
918	/* Link TRB. The HWO bit is never reset */
919	trb_st_hw = &dep->trb_pool[0];
920
921	trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
922	trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
923	trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
924	trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
925	trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
926	}
927
928	/*
929	* Issue StartTransfer here with no-op TRB so we can always rely on No
930	* Response Update Transfer command.
931	*/
932	if (usb_endpoint_xfer_bulk(epd: desc) ||
933	usb_endpoint_xfer_int(epd: desc)) {
934	struct dwc3_gadget_ep_cmd_params params;
935	struct dwc3_trb *trb;
936	dma_addr_t trb_dma;
937	u32 cmd;
938
939	memset(&params, 0, sizeof(params));
940	trb = &dep->trb_pool[0];
941	trb_dma = dwc3_trb_dma_offset(dep, trb);
942
943	params.param0 = upper_32_bits(trb_dma);
944	params.param1 = lower_32_bits(trb_dma);
945
946	cmd = DWC3_DEPCMD_STARTTRANSFER;
947
948	ret = dwc3_send_gadget_ep_cmd(dep, cmd, params: &params);
949	if (ret < 0)
950	return ret;
951
952	if (dep->stream_capable) {
953	/*
954	* For streams, at start, there maybe a race where the
955	* host primes the endpoint before the function driver
956	* queues a request to initiate a stream. In that case,
957	* the controller will not see the prime to generate the
958	* ERDY and start stream. To workaround this, issue a
959	* no-op TRB as normal, but end it immediately. As a
960	* result, when the function driver queues the request,
961	* the next START_TRANSFER command will cause the
962	* controller to generate an ERDY to initiate the
963	* stream.
964	*/
965	dwc3_stop_active_transfer(dep, force: true, interrupt: true);
966
967	/*
968	* All stream eps will reinitiate stream on NoStream
969	* rejection until we can determine that the host can
970	* prime after the first transfer.
971	*
972	* However, if the controller is capable of
973	* TXF_FLUSH_BYPASS, then IN direction endpoints will
974	* automatically restart the stream without the driver
975	* initiation.
976	*/
977	if (!dep->direction ||
978	!(dwc->hwparams.hwparams9 &
979	DWC3_GHWPARAMS9_DEV_TXF_FLUSH_BYPASS))
980	dep->flags |= DWC3_EP_FORCE_RESTART_STREAM;
981	}
982	}
983
984	out:
985	trace_dwc3_gadget_ep_enable(dep);
986
987	return 0;
988	}
989
990	void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep, int status)
991	{
992	struct dwc3_request *req;
993
994	dwc3_stop_active_transfer(dep, force: true, interrupt: false);
995
996	/* If endxfer is delayed, avoid unmapping requests */
997	if (dep->flags & DWC3_EP_DELAY_STOP)
998	return;
999
1000	/* - giveback all requests to gadget driver */
1001	while (!list_empty(head: &dep->started_list)) {
1002	req = next_request(list: &dep->started_list);
1003
1004	dwc3_gadget_giveback(dep, req, status);
1005	}
1006
1007	while (!list_empty(head: &dep->pending_list)) {
1008	req = next_request(list: &dep->pending_list);
1009
1010	dwc3_gadget_giveback(dep, req, status);
1011	}
1012
1013	while (!list_empty(head: &dep->cancelled_list)) {
1014	req = next_request(list: &dep->cancelled_list);
1015
1016	dwc3_gadget_giveback(dep, req, status);
1017	}
1018	}
1019
1020	/**
1021	* __dwc3_gadget_ep_disable - disables a hw endpoint
1022	* @dep: the endpoint to disable
1023	*
1024	* This function undoes what __dwc3_gadget_ep_enable did and also removes
1025	* requests which are currently being processed by the hardware and those which
1026	* are not yet scheduled.
1027	*
1028	* Caller should take care of locking.
1029	*/
1030	static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
1031	{
1032	struct dwc3 *dwc = dep->dwc;
1033	u32 reg;
1034	u32 mask;
1035
1036	trace_dwc3_gadget_ep_disable(dep);
1037
1038	/* make sure HW endpoint isn't stalled */
1039	if (dep->flags & DWC3_EP_STALL)
1040	__dwc3_gadget_ep_set_halt(dep, value: 0, protocol: false);
1041
1042	reg = dwc3_readl(base: dwc->regs, DWC3_DALEPENA);
1043	reg &= ~DWC3_DALEPENA_EP(dep->number);
1044	dwc3_writel(base: dwc->regs, DWC3_DALEPENA, value: reg);
1045
1046	dwc3_remove_requests(dwc, dep, status: -ESHUTDOWN);
1047
1048	dep->stream_capable = false;
1049	dep->type = 0;
1050	mask = DWC3_EP_TXFIFO_RESIZED;
1051	/*
1052	* dwc3_remove_requests() can exit early if DWC3 EP delayed stop is
1053	* set. Do not clear DEP flags, so that the end transfer command will
1054	* be reattempted during the next SETUP stage.
1055	*/
1056	if (dep->flags & DWC3_EP_DELAY_STOP)
1057	mask |= (DWC3_EP_DELAY_STOP | DWC3_EP_TRANSFER_STARTED);
1058	dep->flags &= mask;
1059
1060	/* Clear out the ep descriptors for non-ep0 */
1061	if (dep->number > 1) {
1062	dep->endpoint.comp_desc = NULL;
1063	dep->endpoint.desc = NULL;
1064	}
1065
1066	return 0;
1067	}
1068
1069	/* -------------------------------------------------------------------------- */
1070
1071	static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
1072	const struct usb_endpoint_descriptor *desc)
1073	{
1074	return -EINVAL;
1075	}
1076
1077	static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
1078	{
1079	return -EINVAL;
1080	}
1081
1082	/* -------------------------------------------------------------------------- */
1083
1084	static int dwc3_gadget_ep_enable(struct usb_ep *ep,
1085	const struct usb_endpoint_descriptor *desc)
1086	{
1087	struct dwc3_ep *dep;
1088	struct dwc3 *dwc;
1089	unsigned long flags;
1090	int ret;
1091
1092	if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
1093	pr_debug("dwc3: invalid parameters\n");
1094	return -EINVAL;
1095	}
1096
1097	if (!desc->wMaxPacketSize) {
1098	pr_debug("dwc3: missing wMaxPacketSize\n");
1099	return -EINVAL;
1100	}
1101
1102	dep = to_dwc3_ep(ep);
1103	dwc = dep->dwc;
1104
1105	if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
1106	"%s is already enabled\n",
1107	dep->name))
1108	return 0;
1109
1110	spin_lock_irqsave(&dwc->lock, flags);
1111	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
1112	spin_unlock_irqrestore(lock: &dwc->lock, flags);
1113
1114	return ret;
1115	}
1116
1117	static int dwc3_gadget_ep_disable(struct usb_ep *ep)
1118	{
1119	struct dwc3_ep *dep;
1120	struct dwc3 *dwc;
1121	unsigned long flags;
1122	int ret;
1123
1124	if (!ep) {
1125	pr_debug("dwc3: invalid parameters\n");
1126	return -EINVAL;
1127	}
1128
1129	dep = to_dwc3_ep(ep);
1130	dwc = dep->dwc;
1131
1132	if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
1133	"%s is already disabled\n",
1134	dep->name))
1135	return 0;
1136
1137	spin_lock_irqsave(&dwc->lock, flags);
1138	ret = __dwc3_gadget_ep_disable(dep);
1139	spin_unlock_irqrestore(lock: &dwc->lock, flags);
1140
1141	return ret;
1142	}
1143
1144	static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
1145	gfp_t gfp_flags)
1146	{
1147	struct dwc3_request *req;
1148	struct dwc3_ep *dep = to_dwc3_ep(ep);
1149
1150	req = kzalloc(size: sizeof(*req), flags: gfp_flags);
1151	if (!req)
1152	return NULL;
1153
1154	req->direction = dep->direction;
1155	req->epnum = dep->number;
1156	req->dep = dep;
1157	req->status = DWC3_REQUEST_STATUS_UNKNOWN;
1158
1159	trace_dwc3_alloc_request(req);
1160
1161	return &req->request;
1162	}
1163
1164	static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
1165	struct usb_request *request)
1166	{
1167	struct dwc3_request *req = to_dwc3_request(request);
1168
1169	trace_dwc3_free_request(req);
1170	kfree(objp: req);
1171	}
1172
1173	/**
1174	* dwc3_ep_prev_trb - returns the previous TRB in the ring
1175	* @dep: The endpoint with the TRB ring
1176	* @index: The index of the current TRB in the ring
1177	*
1178	* Returns the TRB prior to the one pointed to by the index. If the
1179	* index is 0, we will wrap backwards, skip the link TRB, and return
1180	* the one just before that.
1181	*/
1182	static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
1183	{
1184	u8 tmp = index;
1185
1186	if (!tmp)
1187	tmp = DWC3_TRB_NUM - 1;
1188
1189	return &dep->trb_pool[tmp - 1];
1190	}
1191
1192	static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
1193	{
1194	u8 trbs_left;
1195
1196	/*
1197	* If the enqueue & dequeue are equal then the TRB ring is either full
1198	* or empty. It's considered full when there are DWC3_TRB_NUM-1 of TRBs
1199	* pending to be processed by the driver.
1200	*/
1201	if (dep->trb_enqueue == dep->trb_dequeue) {
1202	/*
1203	* If there is any request remained in the started_list at
1204	* this point, that means there is no TRB available.
1205	*/
1206	if (!list_empty(head: &dep->started_list))
1207	return 0;
1208
1209	return DWC3_TRB_NUM - 1;
1210	}
1211
1212	trbs_left = dep->trb_dequeue - dep->trb_enqueue;
1213	trbs_left &= (DWC3_TRB_NUM - 1);
1214
1215	if (dep->trb_dequeue < dep->trb_enqueue)
1216	trbs_left--;
1217
1218	return trbs_left;
1219	}
1220
1221	/**
1222	* dwc3_prepare_one_trb - setup one TRB from one request
1223	* @dep: endpoint for which this request is prepared
1224	* @req: dwc3_request pointer
1225	* @trb_length: buffer size of the TRB
1226	* @chain: should this TRB be chained to the next?
1227	* @node: only for isochronous endpoints. First TRB needs different type.
1228	* @use_bounce_buffer: set to use bounce buffer
1229	* @must_interrupt: set to interrupt on TRB completion
1230	*/
1231	static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
1232	struct dwc3_request *req, unsigned int trb_length,
1233	unsigned int chain, unsigned int node, bool use_bounce_buffer,
1234	bool must_interrupt)
1235	{
1236	struct dwc3_trb *trb;
1237	dma_addr_t dma;
1238	unsigned int stream_id = req->request.stream_id;
1239	unsigned int short_not_ok = req->request.short_not_ok;
1240	unsigned int no_interrupt = req->request.no_interrupt;
1241	unsigned int is_last = req->request.is_last;
1242	struct dwc3 *dwc = dep->dwc;
1243	struct usb_gadget *gadget = dwc->gadget;
1244	enum usb_device_speed speed = gadget->speed;
1245
1246	if (use_bounce_buffer)
1247	dma = dep->dwc->bounce_addr;
1248	else if (req->request.num_sgs > 0)
1249	dma = sg_dma_address(req->start_sg);
1250	else
1251	dma = req->request.dma;
1252
1253	trb = &dep->trb_pool[dep->trb_enqueue];
1254
1255	if (!req->trb) {
1256	dwc3_gadget_move_started_request(req);
1257	req->trb = trb;
1258	req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1259	}
1260
1261	req->num_trbs++;
1262
1263	trb->size = DWC3_TRB_SIZE_LENGTH(trb_length);
1264	trb->bpl = lower_32_bits(dma);
1265	trb->bph = upper_32_bits(dma);
1266
1267	switch (usb_endpoint_type(epd: dep->endpoint.desc)) {
1268	case USB_ENDPOINT_XFER_CONTROL:
1269	trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
1270	break;
1271
1272	case USB_ENDPOINT_XFER_ISOC:
1273	if (!node) {
1274	trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
1275
1276	/*
1277	* USB Specification 2.0 Section 5.9.2 states that: "If
1278	* there is only a single transaction in the microframe,
1279	* only a DATA0 data packet PID is used. If there are
1280	* two transactions per microframe, DATA1 is used for
1281	* the first transaction data packet and DATA0 is used
1282	* for the second transaction data packet. If there are
1283	* three transactions per microframe, DATA2 is used for
1284	* the first transaction data packet, DATA1 is used for
1285	* the second, and DATA0 is used for the third."
1286	*
1287	* IOW, we should satisfy the following cases:
1288	*
1289	* 1) length <= maxpacket
1290	* - DATA0
1291	*
1292	* 2) maxpacket < length <= (2 * maxpacket)
1293	* - DATA1, DATA0
1294	*
1295	* 3) (2 * maxpacket) < length <= (3 * maxpacket)
1296	* - DATA2, DATA1, DATA0
1297	*/
1298	if (speed == USB_SPEED_HIGH) {
1299	struct usb_ep *ep = &dep->endpoint;
1300	unsigned int mult = 2;
1301	unsigned int maxp = usb_endpoint_maxp(epd: ep->desc);
1302
1303	if (req->request.length <= (2 * maxp))
1304	mult--;
1305
1306	if (req->request.length <= maxp)
1307	mult--;
1308
1309	trb->size |= DWC3_TRB_SIZE_PCM1(mult);
1310	}
1311	} else {
1312	trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
1313	}
1314
1315	if (!no_interrupt && !chain)
1316	trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1317	break;
1318
1319	case USB_ENDPOINT_XFER_BULK:
1320	case USB_ENDPOINT_XFER_INT:
1321	trb->ctrl = DWC3_TRBCTL_NORMAL;
1322	break;
1323	default:
1324	/*
1325	* This is only possible with faulty memory because we
1326	* checked it already :)
1327	*/
1328	dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
1329	usb_endpoint_type(dep->endpoint.desc));
1330	}
1331
1332	/*
1333	* Enable Continue on Short Packet
1334	* when endpoint is not a stream capable
1335	*/
1336	if (usb_endpoint_dir_out(epd: dep->endpoint.desc)) {
1337	if (!dep->stream_capable)
1338	trb->ctrl |= DWC3_TRB_CTRL_CSP;
1339
1340	if (short_not_ok)
1341	trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1342	}
1343
1344	/* All TRBs setup for MST must set CSP=1 when LST=0 */
1345	if (dep->stream_capable && DWC3_MST_CAPABLE(&dwc->hwparams))
1346	trb->ctrl |= DWC3_TRB_CTRL_CSP;
1347
1348	if ((!no_interrupt && !chain) || must_interrupt)
1349	trb->ctrl |= DWC3_TRB_CTRL_IOC;
1350
1351	if (chain)
1352	trb->ctrl |= DWC3_TRB_CTRL_CHN;
1353	else if (dep->stream_capable && is_last &&
1354	!DWC3_MST_CAPABLE(&dwc->hwparams))
1355	trb->ctrl |= DWC3_TRB_CTRL_LST;
1356
1357	if (usb_endpoint_xfer_bulk(epd: dep->endpoint.desc) && dep->stream_capable)
1358	trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
1359
1360	/*
1361	* As per data book 4.2.3.2TRB Control Bit Rules section
1362	*
1363	* The controller autonomously checks the HWO field of a TRB to determine if the
1364	* entire TRB is valid. Therefore, software must ensure that the rest of the TRB
1365	* is valid before setting the HWO field to '1'. In most systems, this means that
1366	* software must update the fourth DWORD of a TRB last.
1367	*
1368	* However there is a possibility of CPU re-ordering here which can cause
1369	* controller to observe the HWO bit set prematurely.
1370	* Add a write memory barrier to prevent CPU re-ordering.
1371	*/
1372	wmb();
1373	trb->ctrl |= DWC3_TRB_CTRL_HWO;
1374
1375	dwc3_ep_inc_enq(dep);
1376
1377	trace_dwc3_prepare_trb(dep, trb);
1378	}
1379
1380	static bool dwc3_needs_extra_trb(struct dwc3_ep *dep, struct dwc3_request *req)
1381	{
1382	unsigned int maxp = usb_endpoint_maxp(epd: dep->endpoint.desc);
1383	unsigned int rem = req->request.length % maxp;
1384
1385	if ((req->request.length && req->request.zero && !rem &&
1386	!usb_endpoint_xfer_isoc(epd: dep->endpoint.desc)) ||
1387	(!req->direction && rem))
1388	return true;
1389
1390	return false;
1391	}
1392
1393	/**
1394	* dwc3_prepare_last_sg - prepare TRBs for the last SG entry
1395	* @dep: The endpoint that the request belongs to
1396	* @req: The request to prepare
1397	* @entry_length: The last SG entry size
1398	* @node: Indicates whether this is not the first entry (for isoc only)
1399	*
1400	* Return the number of TRBs prepared.
1401	*/
1402	static int dwc3_prepare_last_sg(struct dwc3_ep *dep,
1403	struct dwc3_request *req, unsigned int entry_length,
1404	unsigned int node)
1405	{
1406	unsigned int maxp = usb_endpoint_maxp(epd: dep->endpoint.desc);
1407	unsigned int rem = req->request.length % maxp;
1408	unsigned int num_trbs = 1;
1409
1410	if (dwc3_needs_extra_trb(dep, req))
1411	num_trbs++;
1412
1413	if (dwc3_calc_trbs_left(dep) < num_trbs)
1414	return 0;
1415
1416	req->needs_extra_trb = num_trbs > 1;
1417
1418	/* Prepare a normal TRB */
1419	if (req->direction || req->request.length)
1420	dwc3_prepare_one_trb(dep, req, trb_length: entry_length,
1421	chain: req->needs_extra_trb, node, use_bounce_buffer: false, must_interrupt: false);
1422
1423	/* Prepare extra TRBs for ZLP and MPS OUT transfer alignment */
1424	if ((!req->direction && !req->request.length) || req->needs_extra_trb)
1425	dwc3_prepare_one_trb(dep, req,
1426	trb_length: req->direction ? 0 : maxp - rem,
1427	chain: false, node: 1, use_bounce_buffer: true, must_interrupt: false);
1428
1429	return num_trbs;
1430	}
1431
1432	static int dwc3_prepare_trbs_sg(struct dwc3_ep *dep,
1433	struct dwc3_request *req)
1434	{
1435	struct scatterlist *sg = req->start_sg;
1436	struct scatterlist *s;
1437	int i;
1438	unsigned int length = req->request.length;
1439	unsigned int remaining = req->request.num_mapped_sgs
1440	- req->num_queued_sgs;
1441	unsigned int num_trbs = req->num_trbs;
1442	bool needs_extra_trb = dwc3_needs_extra_trb(dep, req);
1443
1444	/*
1445	* If we resume preparing the request, then get the remaining length of
1446	* the request and resume where we left off.
1447	*/
1448	for_each_sg(req->request.sg, s, req->num_queued_sgs, i)
1449	length -= sg_dma_len(s);
1450
1451	for_each_sg(sg, s, remaining, i) {
1452	unsigned int num_trbs_left = dwc3_calc_trbs_left(dep);
1453	unsigned int trb_length;
1454	bool must_interrupt = false;
1455	bool last_sg = false;
1456
1457	trb_length = min_t(unsigned int, length, sg_dma_len(s));
1458
1459	length -= trb_length;
1460
1461	/*
1462	* IOMMU driver is coalescing the list of sgs which shares a
1463	* page boundary into one and giving it to USB driver. With
1464	* this the number of sgs mapped is not equal to the number of
1465	* sgs passed. So mark the chain bit to false if it isthe last
1466	* mapped sg.
1467	*/
1468	if ((i == remaining - 1) || !length)
1469	last_sg = true;
1470
1471	if (!num_trbs_left)
1472	break;
1473
1474	if (last_sg) {
1475	if (!dwc3_prepare_last_sg(dep, req, entry_length: trb_length, node: i))
1476	break;
1477	} else {
1478	/*
1479	* Look ahead to check if we have enough TRBs for the
1480	* next SG entry. If not, set interrupt on this TRB to
1481	* resume preparing the next SG entry when more TRBs are
1482	* free.
1483	*/
1484	if (num_trbs_left == 1 || (needs_extra_trb &&
1485	num_trbs_left <= 2 &&
1486	sg_dma_len(sg_next(s)) >= length)) {
1487	struct dwc3_request *r;
1488
1489	/* Check if previous requests already set IOC */
1490	list_for_each_entry(r, &dep->started_list, list) {
1491	if (r != req && !r->request.no_interrupt)
1492	break;
1493
1494	if (r == req)
1495	must_interrupt = true;
1496	}
1497	}
1498
1499	dwc3_prepare_one_trb(dep, req, trb_length, chain: 1, node: i, use_bounce_buffer: false,
1500	must_interrupt);
1501	}
1502
1503	/*
1504	* There can be a situation where all sgs in sglist are not
1505	* queued because of insufficient trb number. To handle this
1506	* case, update start_sg to next sg to be queued, so that
1507	* we have free trbs we can continue queuing from where we
1508	* previously stopped
1509	*/
1510	if (!last_sg)
1511	req->start_sg = sg_next(s);
1512
1513	req->num_queued_sgs++;
1514	req->num_pending_sgs--;
1515
1516	/*
1517	* The number of pending SG entries may not correspond to the
1518	* number of mapped SG entries. If all the data are queued, then
1519	* don't include unused SG entries.
1520	*/
1521	if (length == 0) {
1522	req->num_pending_sgs = 0;
1523	break;
1524	}
1525
1526	if (must_interrupt)
1527	break;
1528	}
1529
1530	return req->num_trbs - num_trbs;
1531	}
1532
1533	static int dwc3_prepare_trbs_linear(struct dwc3_ep *dep,
1534	struct dwc3_request *req)
1535	{
1536	return dwc3_prepare_last_sg(dep, req, entry_length: req->request.length, node: 0);
1537	}
1538
1539	/*
1540	* dwc3_prepare_trbs - setup TRBs from requests
1541	* @dep: endpoint for which requests are being prepared
1542	*
1543	* The function goes through the requests list and sets up TRBs for the
1544	* transfers. The function returns once there are no more TRBs available or
1545	* it runs out of requests.
1546	*
1547	* Returns the number of TRBs prepared or negative errno.
1548	*/
1549	static int dwc3_prepare_trbs(struct dwc3_ep *dep)
1550	{
1551	struct dwc3_request *req, *n;
1552	int ret = 0;
1553
1554	BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1555
1556	/*
1557	* We can get in a situation where there's a request in the started list
1558	* but there weren't enough TRBs to fully kick it in the first time
1559	* around, so it has been waiting for more TRBs to be freed up.
1560	*
1561	* In that case, we should check if we have a request with pending_sgs
1562	* in the started list and prepare TRBs for that request first,
1563	* otherwise we will prepare TRBs completely out of order and that will
1564	* break things.
1565	*/
1566	list_for_each_entry(req, &dep->started_list, list) {
1567	if (req->num_pending_sgs > 0) {
1568	ret = dwc3_prepare_trbs_sg(dep, req);
1569	if (!ret || req->num_pending_sgs)
1570	return ret;
1571	}
1572
1573	if (!dwc3_calc_trbs_left(dep))
1574	return ret;
1575
1576	/*
1577	* Don't prepare beyond a transfer. In DWC_usb32, its transfer
1578	* burst capability may try to read and use TRBs beyond the
1579	* active transfer instead of stopping.
1580	*/
1581	if (dep->stream_capable && req->request.is_last &&
1582	!DWC3_MST_CAPABLE(&dep->dwc->hwparams))
1583	return ret;
1584	}
1585
1586	list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1587	struct dwc3 *dwc = dep->dwc;
1588
1589	ret = usb_gadget_map_request_by_dev(dev: dwc->sysdev, req: &req->request,
1590	is_in: dep->direction);
1591	if (ret)
1592	return ret;
1593
1594	req->sg = req->request.sg;
1595	req->start_sg = req->sg;
1596	req->num_queued_sgs = 0;
1597	req->num_pending_sgs = req->request.num_mapped_sgs;
1598
1599	if (req->num_pending_sgs > 0) {
1600	ret = dwc3_prepare_trbs_sg(dep, req);
1601	if (req->num_pending_sgs)
1602	return ret;
1603	} else {
1604	ret = dwc3_prepare_trbs_linear(dep, req);
1605	}
1606
1607	if (!ret || !dwc3_calc_trbs_left(dep))
1608	return ret;
1609
1610	/*
1611	* Don't prepare beyond a transfer. In DWC_usb32, its transfer
1612	* burst capability may try to read and use TRBs beyond the
1613	* active transfer instead of stopping.
1614	*/
1615	if (dep->stream_capable && req->request.is_last &&
1616	!DWC3_MST_CAPABLE(&dwc->hwparams))
1617	return ret;
1618	}
1619
1620	return ret;
1621	}
1622
1623	static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep);
1624
1625	static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1626	{
1627	struct dwc3_gadget_ep_cmd_params params;
1628	struct dwc3_request *req;
1629	int starting;
1630	int ret;
1631	u32 cmd;
1632
1633	/*
1634	* Note that it's normal to have no new TRBs prepared (i.e. ret == 0).
1635	* This happens when we need to stop and restart a transfer such as in
1636	* the case of reinitiating a stream or retrying an isoc transfer.
1637	*/
1638	ret = dwc3_prepare_trbs(dep);
1639	if (ret < 0)
1640	return ret;
1641
1642	starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
1643
1644	/*
1645	* If there's no new TRB prepared and we don't need to restart a
1646	* transfer, there's no need to update the transfer.
1647	*/
1648	if (!ret && !starting)
1649	return ret;
1650
1651	req = next_request(list: &dep->started_list);
1652	if (!req) {
1653	dep->flags |= DWC3_EP_PENDING_REQUEST;
1654	return 0;
1655	}
1656
1657	memset(&params, 0, sizeof(params));
1658
1659	if (starting) {
1660	params.param0 = upper_32_bits(req->trb_dma);
1661	params.param1 = lower_32_bits(req->trb_dma);
1662	cmd = DWC3_DEPCMD_STARTTRANSFER;
1663
1664	if (dep->stream_capable)
1665	cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id);
1666
1667	if (usb_endpoint_xfer_isoc(epd: dep->endpoint.desc))
1668	cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1669	} else {
1670	cmd = DWC3_DEPCMD_UPDATETRANSFER |
1671	DWC3_DEPCMD_PARAM(dep->resource_index);
1672	}
1673
1674	ret = dwc3_send_gadget_ep_cmd(dep, cmd, params: &params);
1675	if (ret < 0) {
1676	struct dwc3_request *tmp;
1677
1678	if (ret == -EAGAIN)
1679	return ret;
1680
1681	dwc3_stop_active_transfer(dep, force: true, interrupt: true);
1682
1683	list_for_each_entry_safe(req, tmp, &dep->started_list, list)
1684	dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_DEQUEUED);
1685
1686	/* If ep isn't started, then there's no end transfer pending */
1687	if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
1688	dwc3_gadget_ep_cleanup_cancelled_requests(dep);
1689
1690	return ret;
1691	}
1692
1693	if (dep->stream_capable && req->request.is_last &&
1694	!DWC3_MST_CAPABLE(&dep->dwc->hwparams))
1695	dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE;
1696
1697	return 0;
1698	}
1699
1700	static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1701	{
1702	u32 reg;
1703
1704	reg = dwc3_readl(base: dwc->regs, DWC3_DSTS);
1705	return DWC3_DSTS_SOFFN(reg);
1706	}
1707
1708	/**
1709	* __dwc3_stop_active_transfer - stop the current active transfer
1710	* @dep: isoc endpoint
1711	* @force: set forcerm bit in the command
1712	* @interrupt: command complete interrupt after End Transfer command
1713	*
1714	* When setting force, the ForceRM bit will be set. In that case
1715	* the controller won't update the TRB progress on command
1716	* completion. It also won't clear the HWO bit in the TRB.
1717	* The command will also not complete immediately in that case.
1718	*/
1719	static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt)
1720	{
1721	struct dwc3 *dwc = dep->dwc;
1722	struct dwc3_gadget_ep_cmd_params params;
1723	u32 cmd;
1724	int ret;
1725
1726	cmd = DWC3_DEPCMD_ENDTRANSFER;
1727	cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
1728	cmd |= interrupt ? DWC3_DEPCMD_CMDIOC : 0;
1729	cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
1730	memset(&params, 0, sizeof(params));
1731	ret = dwc3_send_gadget_ep_cmd(dep, cmd, params: &params);
1732	/*
1733	* If the End Transfer command was timed out while the device is
1734	* not in SETUP phase, it's possible that an incoming Setup packet
1735	* may prevent the command's completion. Let's retry when the
1736	* ep0state returns to EP0_SETUP_PHASE.
1737	*/
1738	if (ret == -ETIMEDOUT && dep->dwc->ep0state != EP0_SETUP_PHASE) {
1739	dep->flags |= DWC3_EP_DELAY_STOP;
1740	return 0;
1741	}
1742	WARN_ON_ONCE(ret);
1743	dep->resource_index = 0;
1744
1745	if (!interrupt) {
1746	if (!DWC3_IP_IS(DWC3) || DWC3_VER_IS_PRIOR(DWC3, 310A))
1747	mdelay(1);
1748	dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
1749	} else if (!ret) {
1750	dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
1751	}
1752
1753	dep->flags &= ~DWC3_EP_DELAY_STOP;
1754	return ret;
1755	}
1756
1757	/**
1758	* dwc3_gadget_start_isoc_quirk - workaround invalid frame number
1759	* @dep: isoc endpoint
1760	*
1761	* This function tests for the correct combination of BIT[15:14] from the 16-bit
1762	* microframe number reported by the XferNotReady event for the future frame
1763	* number to start the isoc transfer.
1764	*
1765	* In DWC_usb31 version 1.70a-ea06 and prior, for highspeed and fullspeed
1766	* isochronous IN, BIT[15:14] of the 16-bit microframe number reported by the
1767	* XferNotReady event are invalid. The driver uses this number to schedule the
1768	* isochronous transfer and passes it to the START TRANSFER command. Because
1769	* this number is invalid, the command may fail. If BIT[15:14] matches the
1770	* internal 16-bit microframe, the START TRANSFER command will pass and the
1771	* transfer will start at the scheduled time, if it is off by 1, the command
1772	* will still pass, but the transfer will start 2 seconds in the future. For all
1773	* other conditions, the START TRANSFER command will fail with bus-expiry.
1774	*
1775	* In order to workaround this issue, we can test for the correct combination of
1776	* BIT[15:14] by sending START TRANSFER commands with different values of
1777	* BIT[15:14]: 'b00, 'b01, 'b10, and 'b11. Each combination is 2^14 uframe apart
1778	* (or 2 seconds). 4 seconds into the future will result in a bus-expiry status.
1779	* As the result, within the 4 possible combinations for BIT[15:14], there will
1780	* be 2 successful and 2 failure START COMMAND status. One of the 2 successful
1781	* command status will result in a 2-second delay start. The smaller BIT[15:14]
1782	* value is the correct combination.
1783	*
1784	* Since there are only 4 outcomes and the results are ordered, we can simply
1785	* test 2 START TRANSFER commands with BIT[15:14] combinations 'b00 and 'b01 to
1786	* deduce the smaller successful combination.
1787	*
1788	* Let test0 = test status for combination 'b00 and test1 = test status for 'b01
1789	* of BIT[15:14]. The correct combination is as follow:
1790	*
1791	* if test0 fails and test1 passes, BIT[15:14] is 'b01
1792	* if test0 fails and test1 fails, BIT[15:14] is 'b10
1793	* if test0 passes and test1 fails, BIT[15:14] is 'b11
1794	* if test0 passes and test1 passes, BIT[15:14] is 'b00
1795	*
1796	* Synopsys STAR 9001202023: Wrong microframe number for isochronous IN
1797	* endpoints.
1798	*/
1799	static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep)
1800	{
1801	int cmd_status = 0;
1802	bool test0;
1803	bool test1;
1804
1805	while (dep->combo_num < 2) {
1806	struct dwc3_gadget_ep_cmd_params params;
1807	u32 test_frame_number;
1808	u32 cmd;
1809
1810	/*
1811	* Check if we can start isoc transfer on the next interval or
1812	* 4 uframes in the future with BIT[15:14] as dep->combo_num
1813	*/
1814	test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK;
1815	test_frame_number |= dep->combo_num << 14;
1816	test_frame_number += max_t(u32, 4, dep->interval);
1817
1818	params.param0 = upper_32_bits(dep->dwc->bounce_addr);
1819	params.param1 = lower_32_bits(dep->dwc->bounce_addr);
1820
1821	cmd = DWC3_DEPCMD_STARTTRANSFER;
1822	cmd |= DWC3_DEPCMD_PARAM(test_frame_number);
1823	cmd_status = dwc3_send_gadget_ep_cmd(dep, cmd, params: &params);
1824
1825	/* Redo if some other failure beside bus-expiry is received */
1826	if (cmd_status && cmd_status != -EAGAIN) {
1827	dep->start_cmd_status = 0;
1828	dep->combo_num = 0;
1829	return 0;
1830	}
1831
1832	/* Store the first test status */
1833	if (dep->combo_num == 0)
1834	dep->start_cmd_status = cmd_status;
1835
1836	dep->combo_num++;
1837
1838	/*
1839	* End the transfer if the START_TRANSFER command is successful
1840	* to wait for the next XferNotReady to test the command again
1841	*/
1842	if (cmd_status == 0) {
1843	dwc3_stop_active_transfer(dep, force: true, interrupt: true);
1844	return 0;
1845	}
1846	}
1847
1848	/* test0 and test1 are both completed at this point */
1849	test0 = (dep->start_cmd_status == 0);
1850	test1 = (cmd_status == 0);
1851
1852	if (!test0 && test1)
1853	dep->combo_num = 1;
1854	else if (!test0 && !test1)
1855	dep->combo_num = 2;
1856	else if (test0 && !test1)
1857	dep->combo_num = 3;
1858	else if (test0 && test1)
1859	dep->combo_num = 0;
1860
1861	dep->frame_number &= DWC3_FRNUMBER_MASK;
1862	dep->frame_number |= dep->combo_num << 14;
1863	dep->frame_number += max_t(u32, 4, dep->interval);
1864
1865	/* Reinitialize test variables */
1866	dep->start_cmd_status = 0;
1867	dep->combo_num = 0;
1868
1869	return __dwc3_gadget_kick_transfer(dep);
1870	}
1871
1872	static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
1873	{
1874	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
1875	struct dwc3 *dwc = dep->dwc;
1876	int ret;
1877	int i;
1878
1879	if (list_empty(head: &dep->pending_list) &&
1880	list_empty(head: &dep->started_list)) {
1881	dep->flags |= DWC3_EP_PENDING_REQUEST;
1882	return -EAGAIN;
1883	}
1884
1885	if (!dwc->dis_start_transfer_quirk &&
1886	(DWC3_VER_IS_PRIOR(DWC31, 170A) ||
1887	DWC3_VER_TYPE_IS_WITHIN(DWC31, 170A, EA01, EA06))) {
1888	if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction)
1889	return dwc3_gadget_start_isoc_quirk(dep);
1890	}
1891
1892	if (desc->bInterval <= 14 &&
1893	dwc->gadget->speed >= USB_SPEED_HIGH) {
1894	u32 frame = __dwc3_gadget_get_frame(dwc);
1895	bool rollover = frame <
1896	(dep->frame_number & DWC3_FRNUMBER_MASK);
1897
1898	/*
1899	* frame_number is set from XferNotReady and may be already
1900	* out of date. DSTS only provides the lower 14 bit of the
1901	* current frame number. So add the upper two bits of
1902	* frame_number and handle a possible rollover.
1903	* This will provide the correct frame_number unless more than
1904	* rollover has happened since XferNotReady.
1905	*/
1906
1907	dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) |
1908	frame;
1909	if (rollover)
1910	dep->frame_number += BIT(14);
1911	}
1912
1913	for (i = 0; i < DWC3_ISOC_MAX_RETRIES; i++) {
1914	int future_interval = i + 1;
1915
1916	/* Give the controller at least 500us to schedule transfers */
1917	if (desc->bInterval < 3)
1918	future_interval += 3 - desc->bInterval;
1919
1920	dep->frame_number = DWC3_ALIGN_FRAME(dep, future_interval);
1921
1922	ret = __dwc3_gadget_kick_transfer(dep);
1923	if (ret != -EAGAIN)
1924	break;
1925	}
1926
1927	/*
1928	* After a number of unsuccessful start attempts due to bus-expiry
1929	* status, issue END_TRANSFER command and retry on the next XferNotReady
1930	* event.
1931	*/
1932	if (ret == -EAGAIN)
1933	ret = __dwc3_stop_active_transfer(dep, force: false, interrupt: true);
1934
1935	return ret;
1936	}
1937
1938	static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1939	{
1940	struct dwc3 *dwc = dep->dwc;
1941
1942	if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) {
1943	dev_dbg(dwc->dev, "%s: can't queue to disabled endpoint\n",
1944	dep->name);
1945	return -ESHUTDOWN;
1946	}
1947
1948	if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1949	&req->request, req->dep->name))
1950	return -EINVAL;
1951
1952	if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED,
1953	"%s: request %pK already in flight\n",
1954	dep->name, &req->request))
1955	return -EINVAL;
1956
1957	pm_runtime_get(dev: dwc->dev);
1958
1959	req->request.actual = 0;
1960	req->request.status = -EINPROGRESS;
1961
1962	trace_dwc3_ep_queue(req);
1963
1964	list_add_tail(new: &req->list, head: &dep->pending_list);
1965	req->status = DWC3_REQUEST_STATUS_QUEUED;
1966
1967	if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)
1968	return 0;
1969
1970	/*
1971	* Start the transfer only after the END_TRANSFER is completed
1972	* and endpoint STALL is cleared.
1973	*/
1974	if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
1975	(dep->flags & DWC3_EP_WEDGE) ||
1976	(dep->flags & DWC3_EP_DELAY_STOP) ||
1977	(dep->flags & DWC3_EP_STALL)) {
1978	dep->flags |= DWC3_EP_DELAY_START;
1979	return 0;
1980	}
1981
1982	/*
1983	* NOTICE: Isochronous endpoints should NEVER be prestarted. We must
1984	* wait for a XferNotReady event so we will know what's the current
1985	* (micro-)frame number.
1986	*
1987	* Without this trick, we are very, very likely gonna get Bus Expiry
1988	* errors which will force us issue EndTransfer command.
1989	*/
1990	if (usb_endpoint_xfer_isoc(epd: dep->endpoint.desc)) {
1991	if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
1992	if ((dep->flags & DWC3_EP_PENDING_REQUEST))
1993	return __dwc3_gadget_start_isoc(dep);
1994
1995	return 0;
1996	}
1997	}
1998
1999	__dwc3_gadget_kick_transfer(dep);
2000
2001	return 0;
2002	}
2003
2004	static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
2005	gfp_t gfp_flags)
2006	{
2007	struct dwc3_request *req = to_dwc3_request(request);
2008	struct dwc3_ep *dep = to_dwc3_ep(ep);
2009	struct dwc3 *dwc = dep->dwc;
2010
2011	unsigned long flags;
2012
2013	int ret;
2014
2015	spin_lock_irqsave(&dwc->lock, flags);
2016	ret = __dwc3_gadget_ep_queue(dep, req);
2017	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2018
2019	return ret;
2020	}
2021
2022	static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *req)
2023	{
2024	int i;
2025
2026	/* If req->trb is not set, then the request has not started */
2027	if (!req->trb)
2028	return;
2029
2030	/*
2031	* If request was already started, this means we had to
2032	* stop the transfer. With that we also need to ignore
2033	* all TRBs used by the request, however TRBs can only
2034	* be modified after completion of END_TRANSFER
2035	* command. So what we do here is that we wait for
2036	* END_TRANSFER completion and only after that, we jump
2037	* over TRBs by clearing HWO and incrementing dequeue
2038	* pointer.
2039	*/
2040	for (i = 0; i < req->num_trbs; i++) {
2041	struct dwc3_trb *trb;
2042
2043	trb = &dep->trb_pool[dep->trb_dequeue];
2044	trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2045	dwc3_ep_inc_deq(dep);
2046	}
2047
2048	req->num_trbs = 0;
2049	}
2050
2051	static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep)
2052	{
2053	struct dwc3_request *req;
2054	struct dwc3 *dwc = dep->dwc;
2055
2056	while (!list_empty(head: &dep->cancelled_list)) {
2057	req = next_request(list: &dep->cancelled_list);
2058	dwc3_gadget_ep_skip_trbs(dep, req);
2059	switch (req->status) {
2060	case DWC3_REQUEST_STATUS_DISCONNECTED:
2061	dwc3_gadget_giveback(dep, req, status: -ESHUTDOWN);
2062	break;
2063	case DWC3_REQUEST_STATUS_DEQUEUED:
2064	dwc3_gadget_giveback(dep, req, status: -ECONNRESET);
2065	break;
2066	case DWC3_REQUEST_STATUS_STALLED:
2067	dwc3_gadget_giveback(dep, req, status: -EPIPE);
2068	break;
2069	default:
2070	dev_err(dwc->dev, "request cancelled with wrong reason:%d\n", req->status);
2071	dwc3_gadget_giveback(dep, req, status: -ECONNRESET);
2072	break;
2073	}
2074	/*
2075	* The endpoint is disabled, let the dwc3_remove_requests()
2076	* handle the cleanup.
2077	*/
2078	if (!dep->endpoint.desc)
2079	break;
2080	}
2081	}
2082
2083	static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
2084	struct usb_request *request)
2085	{
2086	struct dwc3_request *req = to_dwc3_request(request);
2087	struct dwc3_request *r = NULL;
2088
2089	struct dwc3_ep *dep = to_dwc3_ep(ep);
2090	struct dwc3 *dwc = dep->dwc;
2091
2092	unsigned long flags;
2093	int ret = 0;
2094
2095	trace_dwc3_ep_dequeue(req);
2096
2097	spin_lock_irqsave(&dwc->lock, flags);
2098
2099	list_for_each_entry(r, &dep->cancelled_list, list) {
2100	if (r == req)
2101	goto out;
2102	}
2103
2104	list_for_each_entry(r, &dep->pending_list, list) {
2105	if (r == req) {
2106	dwc3_gadget_giveback(dep, req, status: -ECONNRESET);
2107	goto out;
2108	}
2109	}
2110
2111	list_for_each_entry(r, &dep->started_list, list) {
2112	if (r == req) {
2113	struct dwc3_request *t;
2114
2115	/* wait until it is processed */
2116	dwc3_stop_active_transfer(dep, force: true, interrupt: true);
2117
2118	/*
2119	* Remove any started request if the transfer is
2120	* cancelled.
2121	*/
2122	list_for_each_entry_safe(r, t, &dep->started_list, list)
2123	dwc3_gadget_move_cancelled_request(req: r,
2124	DWC3_REQUEST_STATUS_DEQUEUED);
2125
2126	dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
2127
2128	goto out;
2129	}
2130	}
2131
2132	dev_err(dwc->dev, "request %pK was not queued to %s\n",
2133	request, ep->name);
2134	ret = -EINVAL;
2135	out:
2136	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2137
2138	return ret;
2139	}
2140
2141	int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
2142	{
2143	struct dwc3_gadget_ep_cmd_params params;
2144	struct dwc3 *dwc = dep->dwc;
2145	struct dwc3_request *req;
2146	struct dwc3_request *tmp;
2147	int ret;
2148
2149	if (usb_endpoint_xfer_isoc(epd: dep->endpoint.desc)) {
2150	dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
2151	return -EINVAL;
2152	}
2153
2154	memset(&params, 0x00, sizeof(params));
2155
2156	if (value) {
2157	struct dwc3_trb *trb;
2158
2159	unsigned int transfer_in_flight;
2160	unsigned int started;
2161
2162	if (dep->number > 1)
2163	trb = dwc3_ep_prev_trb(dep, index: dep->trb_enqueue);
2164	else
2165	trb = &dwc->ep0_trb[dep->trb_enqueue];
2166
2167	transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
2168	started = !list_empty(head: &dep->started_list);
2169
2170	if (!protocol && ((dep->direction && transfer_in_flight) ||
2171	(!dep->direction && started))) {
2172	return -EAGAIN;
2173	}
2174
2175	ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
2176	params: &params);
2177	if (ret)
2178	dev_err(dwc->dev, "failed to set STALL on %s\n",
2179	dep->name);
2180	else
2181	dep->flags |= DWC3_EP_STALL;
2182	} else {
2183	/*
2184	* Don't issue CLEAR_STALL command to control endpoints. The
2185	* controller automatically clears the STALL when it receives
2186	* the SETUP token.
2187	*/
2188	if (dep->number <= 1) {
2189	dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
2190	return 0;
2191	}
2192
2193	dwc3_stop_active_transfer(dep, force: true, interrupt: true);
2194
2195	list_for_each_entry_safe(req, tmp, &dep->started_list, list)
2196	dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_STALLED);
2197
2198	if (dep->flags & DWC3_EP_END_TRANSFER_PENDING ||
2199	(dep->flags & DWC3_EP_DELAY_STOP)) {
2200	dep->flags |= DWC3_EP_PENDING_CLEAR_STALL;
2201	if (protocol)
2202	dwc->clear_stall_protocol = dep->number;
2203
2204	return 0;
2205	}
2206
2207	dwc3_gadget_ep_cleanup_cancelled_requests(dep);
2208
2209	ret = dwc3_send_clear_stall_ep_cmd(dep);
2210	if (ret) {
2211	dev_err(dwc->dev, "failed to clear STALL on %s\n",
2212	dep->name);
2213	return ret;
2214	}
2215
2216	dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
2217
2218	if ((dep->flags & DWC3_EP_DELAY_START) &&
2219	!usb_endpoint_xfer_isoc(epd: dep->endpoint.desc))
2220	__dwc3_gadget_kick_transfer(dep);
2221
2222	dep->flags &= ~DWC3_EP_DELAY_START;
2223	}
2224
2225	return ret;
2226	}
2227
2228	static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2229	{
2230	struct dwc3_ep *dep = to_dwc3_ep(ep);
2231	struct dwc3 *dwc = dep->dwc;
2232
2233	unsigned long flags;
2234
2235	int ret;
2236
2237	spin_lock_irqsave(&dwc->lock, flags);
2238	ret = __dwc3_gadget_ep_set_halt(dep, value, protocol: false);
2239	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2240
2241	return ret;
2242	}
2243
2244	static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
2245	{
2246	struct dwc3_ep *dep = to_dwc3_ep(ep);
2247	struct dwc3 *dwc = dep->dwc;
2248	unsigned long flags;
2249	int ret;
2250
2251	spin_lock_irqsave(&dwc->lock, flags);
2252	dep->flags |= DWC3_EP_WEDGE;
2253
2254	if (dep->number == 0 || dep->number == 1)
2255	ret = __dwc3_gadget_ep0_set_halt(ep, value: 1);
2256	else
2257	ret = __dwc3_gadget_ep_set_halt(dep, value: 1, protocol: false);
2258	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2259
2260	return ret;
2261	}
2262
2263	/* -------------------------------------------------------------------------- */
2264
2265	static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
2266	.bLength = USB_DT_ENDPOINT_SIZE,
2267	.bDescriptorType = USB_DT_ENDPOINT,
2268	.bmAttributes = USB_ENDPOINT_XFER_CONTROL,
2269	};
2270
2271	static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
2272	.enable = dwc3_gadget_ep0_enable,
2273	.disable = dwc3_gadget_ep0_disable,
2274	.alloc_request = dwc3_gadget_ep_alloc_request,
2275	.free_request = dwc3_gadget_ep_free_request,
2276	.queue = dwc3_gadget_ep0_queue,
2277	.dequeue = dwc3_gadget_ep_dequeue,
2278	.set_halt = dwc3_gadget_ep0_set_halt,
2279	.set_wedge = dwc3_gadget_ep_set_wedge,
2280	};
2281
2282	static const struct usb_ep_ops dwc3_gadget_ep_ops = {
2283	.enable = dwc3_gadget_ep_enable,
2284	.disable = dwc3_gadget_ep_disable,
2285	.alloc_request = dwc3_gadget_ep_alloc_request,
2286	.free_request = dwc3_gadget_ep_free_request,
2287	.queue = dwc3_gadget_ep_queue,
2288	.dequeue = dwc3_gadget_ep_dequeue,
2289	.set_halt = dwc3_gadget_ep_set_halt,
2290	.set_wedge = dwc3_gadget_ep_set_wedge,
2291	};
2292
2293	/* -------------------------------------------------------------------------- */
2294
2295	static void dwc3_gadget_enable_linksts_evts(struct dwc3 *dwc, bool set)
2296	{
2297	u32 reg;
2298
2299	if (DWC3_VER_IS_PRIOR(DWC3, 250A))
2300	return;
2301
2302	reg = dwc3_readl(base: dwc->regs, DWC3_DEVTEN);
2303	if (set)
2304	reg |= DWC3_DEVTEN_ULSTCNGEN;
2305	else
2306	reg &= ~DWC3_DEVTEN_ULSTCNGEN;
2307
2308	dwc3_writel(base: dwc->regs, DWC3_DEVTEN, value: reg);
2309	}
2310
2311	static int dwc3_gadget_get_frame(struct usb_gadget *g)
2312	{
2313	struct dwc3 *dwc = gadget_to_dwc(g);
2314
2315	return __dwc3_gadget_get_frame(dwc);
2316	}
2317
2318	static int __dwc3_gadget_wakeup(struct dwc3 *dwc, bool async)
2319	{
2320	int retries;
2321
2322	int ret;
2323	u32 reg;
2324
2325	u8 link_state;
2326
2327	/*
2328	* According to the Databook Remote wakeup request should
2329	* be issued only when the device is in early suspend state.
2330	*
2331	* We can check that via USB Link State bits in DSTS register.
2332	*/
2333	reg = dwc3_readl(base: dwc->regs, DWC3_DSTS);
2334
2335	link_state = DWC3_DSTS_USBLNKST(reg);
2336
2337	switch (link_state) {
2338	case DWC3_LINK_STATE_RESET:
2339	case DWC3_LINK_STATE_RX_DET: /* in HS, means Early Suspend */
2340	case DWC3_LINK_STATE_U3: /* in HS, means SUSPEND */
2341	case DWC3_LINK_STATE_U2: /* in HS, means Sleep (L1) */
2342	case DWC3_LINK_STATE_U1:
2343	case DWC3_LINK_STATE_RESUME:
2344	break;
2345	default:
2346	return -EINVAL;
2347	}
2348
2349	if (async)
2350	dwc3_gadget_enable_linksts_evts(dwc, set: true);
2351
2352	ret = dwc3_gadget_set_link_state(dwc, state: DWC3_LINK_STATE_RECOV);
2353	if (ret < 0) {
2354	dev_err(dwc->dev, "failed to put link in Recovery\n");
2355	dwc3_gadget_enable_linksts_evts(dwc, set: false);
2356	return ret;
2357	}
2358
2359	/* Recent versions do this automatically */
2360	if (DWC3_VER_IS_PRIOR(DWC3, 194A)) {
2361	/* write zeroes to Link Change Request */
2362	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
2363	reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
2364	dwc3_writel(base: dwc->regs, DWC3_DCTL, value: reg);
2365	}
2366
2367	/*
2368	* Since link status change events are enabled we will receive
2369	* an U0 event when wakeup is successful. So bail out.
2370	*/
2371	if (async)
2372	return 0;
2373
2374	/* poll until Link State changes to ON */
2375	retries = 20000;
2376
2377	while (retries--) {
2378	reg = dwc3_readl(base: dwc->regs, DWC3_DSTS);
2379
2380	/* in HS, means ON */
2381	if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
2382	break;
2383	}
2384
2385	if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
2386	dev_err(dwc->dev, "failed to send remote wakeup\n");
2387	return -EINVAL;
2388	}
2389
2390	return 0;
2391	}
2392
2393	static int dwc3_gadget_wakeup(struct usb_gadget *g)
2394	{
2395	struct dwc3 *dwc = gadget_to_dwc(g);
2396	unsigned long flags;
2397	int ret;
2398
2399	if (!dwc->wakeup_configured) {
2400	dev_err(dwc->dev, "remote wakeup not configured\n");
2401	return -EINVAL;
2402	}
2403
2404	spin_lock_irqsave(&dwc->lock, flags);
2405	if (!dwc->gadget->wakeup_armed) {
2406	dev_err(dwc->dev, "not armed for remote wakeup\n");
2407	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2408	return -EINVAL;
2409	}
2410	ret = __dwc3_gadget_wakeup(dwc, async: true);
2411
2412	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2413
2414	return ret;
2415	}
2416
2417	static void dwc3_resume_gadget(struct dwc3 *dwc);
2418
2419	static int dwc3_gadget_func_wakeup(struct usb_gadget *g, int intf_id)
2420	{
2421	struct dwc3 *dwc = gadget_to_dwc(g);
2422	unsigned long flags;
2423	int ret;
2424	int link_state;
2425
2426	if (!dwc->wakeup_configured) {
2427	dev_err(dwc->dev, "remote wakeup not configured\n");
2428	return -EINVAL;
2429	}
2430
2431	spin_lock_irqsave(&dwc->lock, flags);
2432	/*
2433	* If the link is in U3, signal for remote wakeup and wait for the
2434	* link to transition to U0 before sending device notification.
2435	*/
2436	link_state = dwc3_gadget_get_link_state(dwc);
2437	if (link_state == DWC3_LINK_STATE_U3) {
2438	ret = __dwc3_gadget_wakeup(dwc, async: false);
2439	if (ret) {
2440	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2441	return -EINVAL;
2442	}
2443	dwc3_resume_gadget(dwc);
2444	dwc->suspended = false;
2445	dwc->link_state = DWC3_LINK_STATE_U0;
2446	}
2447
2448	ret = dwc3_send_gadget_generic_command(dwc, DWC3_DGCMD_DEV_NOTIFICATION,
2449	DWC3_DGCMDPAR_DN_FUNC_WAKE |
2450	DWC3_DGCMDPAR_INTF_SEL(intf_id));
2451	if (ret)
2452	dev_err(dwc->dev, "function remote wakeup failed, ret:%d\n", ret);
2453
2454	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2455
2456	return ret;
2457	}
2458
2459	static int dwc3_gadget_set_remote_wakeup(struct usb_gadget *g, int set)
2460	{
2461	struct dwc3 *dwc = gadget_to_dwc(g);
2462	unsigned long flags;
2463
2464	spin_lock_irqsave(&dwc->lock, flags);
2465	dwc->wakeup_configured = !!set;
2466	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2467
2468	return 0;
2469	}
2470
2471	static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
2472	int is_selfpowered)
2473	{
2474	struct dwc3 *dwc = gadget_to_dwc(g);
2475	unsigned long flags;
2476
2477	spin_lock_irqsave(&dwc->lock, flags);
2478	g->is_selfpowered = !!is_selfpowered;
2479	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2480
2481	return 0;
2482	}
2483
2484	static void dwc3_stop_active_transfers(struct dwc3 *dwc)
2485	{
2486	u32 epnum;
2487
2488	for (epnum = 2; epnum < dwc->num_eps; epnum++) {
2489	struct dwc3_ep *dep;
2490
2491	dep = dwc->eps[epnum];
2492	if (!dep)
2493	continue;
2494
2495	dwc3_remove_requests(dwc, dep, status: -ESHUTDOWN);
2496	}
2497	}
2498
2499	static void __dwc3_gadget_set_ssp_rate(struct dwc3 *dwc)
2500	{
2501	enum usb_ssp_rate ssp_rate = dwc->gadget_ssp_rate;
2502	u32 reg;
2503
2504	if (ssp_rate == USB_SSP_GEN_UNKNOWN)
2505	ssp_rate = dwc->max_ssp_rate;
2506
2507	reg = dwc3_readl(base: dwc->regs, DWC3_DCFG);
2508	reg &= ~DWC3_DCFG_SPEED_MASK;
2509	reg &= ~DWC3_DCFG_NUMLANES(~0);
2510
2511	if (ssp_rate == USB_SSP_GEN_1x2)
2512	reg |= DWC3_DCFG_SUPERSPEED;
2513	else if (dwc->max_ssp_rate != USB_SSP_GEN_1x2)
2514	reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2515
2516	if (ssp_rate != USB_SSP_GEN_2x1 &&
2517	dwc->max_ssp_rate != USB_SSP_GEN_2x1)
2518	reg |= DWC3_DCFG_NUMLANES(1);
2519
2520	dwc3_writel(base: dwc->regs, DWC3_DCFG, value: reg);
2521	}
2522
2523	static void __dwc3_gadget_set_speed(struct dwc3 *dwc)
2524	{
2525	enum usb_device_speed speed;
2526	u32 reg;
2527
2528	speed = dwc->gadget_max_speed;
2529	if (speed == USB_SPEED_UNKNOWN || speed > dwc->maximum_speed)
2530	speed = dwc->maximum_speed;
2531
2532	if (speed == USB_SPEED_SUPER_PLUS &&
2533	DWC3_IP_IS(DWC32)) {
2534	__dwc3_gadget_set_ssp_rate(dwc);
2535	return;
2536	}
2537
2538	reg = dwc3_readl(base: dwc->regs, DWC3_DCFG);
2539	reg &= ~(DWC3_DCFG_SPEED_MASK);
2540
2541	/*
2542	* WORKAROUND: DWC3 revision < 2.20a have an issue
2543	* which would cause metastability state on Run/Stop
2544	* bit if we try to force the IP to USB2-only mode.
2545	*
2546	* Because of that, we cannot configure the IP to any
2547	* speed other than the SuperSpeed
2548	*
2549	* Refers to:
2550	*
2551	* STAR#9000525659: Clock Domain Crossing on DCTL in
2552	* USB 2.0 Mode
2553	*/
2554	if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
2555	!dwc->dis_metastability_quirk) {
2556	reg |= DWC3_DCFG_SUPERSPEED;
2557	} else {
2558	switch (speed) {
2559	case USB_SPEED_FULL:
2560	reg |= DWC3_DCFG_FULLSPEED;
2561	break;
2562	case USB_SPEED_HIGH:
2563	reg |= DWC3_DCFG_HIGHSPEED;
2564	break;
2565	case USB_SPEED_SUPER:
2566	reg |= DWC3_DCFG_SUPERSPEED;
2567	break;
2568	case USB_SPEED_SUPER_PLUS:
2569	if (DWC3_IP_IS(DWC3))
2570	reg |= DWC3_DCFG_SUPERSPEED;
2571	else
2572	reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2573	break;
2574	default:
2575	dev_err(dwc->dev, "invalid speed (%d)\n", speed);
2576
2577	if (DWC3_IP_IS(DWC3))
2578	reg |= DWC3_DCFG_SUPERSPEED;
2579	else
2580	reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2581	}
2582	}
2583
2584	if (DWC3_IP_IS(DWC32) &&
2585	speed > USB_SPEED_UNKNOWN &&
2586	speed < USB_SPEED_SUPER_PLUS)
2587	reg &= ~DWC3_DCFG_NUMLANES(~0);
2588
2589	dwc3_writel(base: dwc->regs, DWC3_DCFG, value: reg);
2590	}
2591
2592	static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on)
2593	{
2594	u32 reg;
2595	u32 timeout = 2000;
2596
2597	if (pm_runtime_suspended(dev: dwc->dev))
2598	return 0;
2599
2600	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
2601	if (is_on) {
2602	if (DWC3_VER_IS_WITHIN(DWC3, ANY, 187A)) {
2603	reg &= ~DWC3_DCTL_TRGTULST_MASK;
2604	reg |= DWC3_DCTL_TRGTULST_RX_DET;
2605	}
2606
2607	if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
2608	reg &= ~DWC3_DCTL_KEEP_CONNECT;
2609	reg |= DWC3_DCTL_RUN_STOP;
2610
2611	__dwc3_gadget_set_speed(dwc);
2612	dwc->pullups_connected = true;
2613	} else {
2614	reg &= ~DWC3_DCTL_RUN_STOP;
2615
2616	dwc->pullups_connected = false;
2617	}
2618
2619	dwc3_gadget_dctl_write_safe(dwc, value: reg);
2620
2621	do {
2622	usleep_range(min: 1000, max: 2000);
2623	reg = dwc3_readl(base: dwc->regs, DWC3_DSTS);
2624	reg &= DWC3_DSTS_DEVCTRLHLT;
2625	} while (--timeout && !(!is_on ^ !reg));
2626
2627	if (!timeout)
2628	return -ETIMEDOUT;
2629
2630	return 0;
2631	}
2632
2633	static void dwc3_gadget_disable_irq(struct dwc3 *dwc);
2634	static void __dwc3_gadget_stop(struct dwc3 *dwc);
2635	static int __dwc3_gadget_start(struct dwc3 *dwc);
2636
2637	static int dwc3_gadget_soft_disconnect(struct dwc3 *dwc)
2638	{
2639	unsigned long flags;
2640	int ret;
2641
2642	spin_lock_irqsave(&dwc->lock, flags);
2643	dwc->connected = false;
2644
2645	/*
2646	* Attempt to end pending SETUP status phase, and not wait for the
2647	* function to do so.
2648	*/
2649	if (dwc->delayed_status)
2650	dwc3_ep0_send_delayed_status(dwc);
2651
2652	/*
2653	* In the Synopsys DesignWare Cores USB3 Databook Rev. 3.30a
2654	* Section 4.1.8 Table 4-7, it states that for a device-initiated
2655	* disconnect, the SW needs to ensure that it sends "a DEPENDXFER
2656	* command for any active transfers" before clearing the RunStop
2657	* bit.
2658	*/
2659	dwc3_stop_active_transfers(dwc);
2660	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2661
2662	/*
2663	* Per databook, when we want to stop the gadget, if a control transfer
2664	* is still in process, complete it and get the core into setup phase.
2665	* In case the host is unresponsive to a SETUP transaction, forcefully
2666	* stall the transfer, and move back to the SETUP phase, so that any
2667	* pending endxfers can be executed.
2668	*/
2669	if (dwc->ep0state != EP0_SETUP_PHASE) {
2670	reinit_completion(x: &dwc->ep0_in_setup);
2671
2672	ret = wait_for_completion_timeout(x: &dwc->ep0_in_setup,
2673	timeout: msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
2674	if (ret == 0) {
2675	dev_warn(dwc->dev, "wait for SETUP phase timed out\n");
2676	spin_lock_irqsave(&dwc->lock, flags);
2677	dwc3_ep0_reset_state(dwc);
2678	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2679	}
2680	}
2681
2682	/*
2683	* Note: if the GEVNTCOUNT indicates events in the event buffer, the
2684	* driver needs to acknowledge them before the controller can halt.
2685	* Simply let the interrupt handler acknowledges and handle the
2686	* remaining event generated by the controller while polling for
2687	* DSTS.DEVCTLHLT.
2688	*/
2689	ret = dwc3_gadget_run_stop(dwc, is_on: false);
2690
2691	/*
2692	* Stop the gadget after controller is halted, so that if needed, the
2693	* events to update EP0 state can still occur while the run/stop
2694	* routine polls for the halted state. DEVTEN is cleared as part of
2695	* gadget stop.
2696	*/
2697	spin_lock_irqsave(&dwc->lock, flags);
2698	__dwc3_gadget_stop(dwc);
2699	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2700
2701	return ret;
2702	}
2703
2704	static int dwc3_gadget_soft_connect(struct dwc3 *dwc)
2705	{
2706	int ret;
2707
2708	/*
2709	* In the Synopsys DWC_usb31 1.90a programming guide section
2710	* 4.1.9, it specifies that for a reconnect after a
2711	* device-initiated disconnect requires a core soft reset
2712	* (DCTL.CSftRst) before enabling the run/stop bit.
2713	*/
2714	ret = dwc3_core_soft_reset(dwc);
2715	if (ret)
2716	return ret;
2717
2718	dwc3_event_buffers_setup(dwc);
2719	__dwc3_gadget_start(dwc);
2720	return dwc3_gadget_run_stop(dwc, is_on: true);
2721	}
2722
2723	static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
2724	{
2725	struct dwc3 *dwc = gadget_to_dwc(g);
2726	int ret;
2727
2728	is_on = !!is_on;
2729
2730	dwc->softconnect = is_on;
2731
2732	/*
2733	* Avoid issuing a runtime resume if the device is already in the
2734	* suspended state during gadget disconnect. DWC3 gadget was already
2735	* halted/stopped during runtime suspend.
2736	*/
2737	if (!is_on) {
2738	pm_runtime_barrier(dev: dwc->dev);
2739	if (pm_runtime_suspended(dev: dwc->dev))
2740	return 0;
2741	}
2742
2743	/*
2744	* Check the return value for successful resume, or error. For a
2745	* successful resume, the DWC3 runtime PM resume routine will handle
2746	* the run stop sequence, so avoid duplicate operations here.
2747	*/
2748	ret = pm_runtime_get_sync(dev: dwc->dev);
2749	if (!ret || ret < 0) {
2750	pm_runtime_put(dev: dwc->dev);
2751	if (ret < 0)
2752	pm_runtime_set_suspended(dev: dwc->dev);
2753	return ret;
2754	}
2755
2756	if (dwc->pullups_connected == is_on) {
2757	pm_runtime_put(dev: dwc->dev);
2758	return 0;
2759	}
2760
2761	synchronize_irq(irq: dwc->irq_gadget);
2762
2763	if (!is_on)
2764	ret = dwc3_gadget_soft_disconnect(dwc);
2765	else
2766	ret = dwc3_gadget_soft_connect(dwc);
2767
2768	pm_runtime_put(dev: dwc->dev);
2769
2770	return ret;
2771	}
2772
2773	static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
2774	{
2775	u32 reg;
2776
2777	/* Enable all but Start and End of Frame IRQs */
2778	reg = (DWC3_DEVTEN_EVNTOVERFLOWEN |
2779	DWC3_DEVTEN_CMDCMPLTEN |
2780	DWC3_DEVTEN_ERRTICERREN |
2781	DWC3_DEVTEN_WKUPEVTEN |
2782	DWC3_DEVTEN_CONNECTDONEEN |
2783	DWC3_DEVTEN_USBRSTEN |
2784	DWC3_DEVTEN_DISCONNEVTEN);
2785
2786	if (DWC3_VER_IS_PRIOR(DWC3, 250A))
2787	reg |= DWC3_DEVTEN_ULSTCNGEN;
2788
2789	/* On 2.30a and above this bit enables U3/L2-L1 Suspend Events */
2790	if (!DWC3_VER_IS_PRIOR(DWC3, 230A))
2791	reg |= DWC3_DEVTEN_U3L2L1SUSPEN;
2792
2793	dwc3_writel(base: dwc->regs, DWC3_DEVTEN, value: reg);
2794	}
2795
2796	static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
2797	{
2798	/* mask all interrupts */
2799	dwc3_writel(base: dwc->regs, DWC3_DEVTEN, value: 0x00);
2800	}
2801
2802	static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
2803	static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
2804
2805	/**
2806	* dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
2807	* @dwc: pointer to our context structure
2808	*
2809	* The following looks like complex but it's actually very simple. In order to
2810	* calculate the number of packets we can burst at once on OUT transfers, we're
2811	* gonna use RxFIFO size.
2812	*
2813	* To calculate RxFIFO size we need two numbers:
2814	* MDWIDTH = size, in bits, of the internal memory bus
2815	* RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
2816	*
2817	* Given these two numbers, the formula is simple:
2818	*
2819	* RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
2820	*
2821	* 24 bytes is for 3x SETUP packets
2822	* 16 bytes is a clock domain crossing tolerance
2823	*
2824	* Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
2825	*/
2826	static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
2827	{
2828	u32 ram2_depth;
2829	u32 mdwidth;
2830	u32 nump;
2831	u32 reg;
2832
2833	ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
2834	mdwidth = dwc3_mdwidth(dwc);
2835
2836	nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
2837	nump = min_t(u32, nump, 16);
2838
2839	/* update NumP */
2840	reg = dwc3_readl(base: dwc->regs, DWC3_DCFG);
2841	reg &= ~DWC3_DCFG_NUMP_MASK;
2842	reg |= nump << DWC3_DCFG_NUMP_SHIFT;
2843	dwc3_writel(base: dwc->regs, DWC3_DCFG, value: reg);
2844	}
2845
2846	static int __dwc3_gadget_start(struct dwc3 *dwc)
2847	{
2848	struct dwc3_ep *dep;
2849	int ret = 0;
2850	u32 reg;
2851
2852	/*
2853	* Use IMOD if enabled via dwc->imod_interval. Otherwise, if
2854	* the core supports IMOD, disable it.
2855	*/
2856	if (dwc->imod_interval) {
2857	dwc3_writel(base: dwc->regs, DWC3_DEV_IMOD(0), value: dwc->imod_interval);
2858	dwc3_writel(base: dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
2859	} else if (dwc3_has_imod(dwc)) {
2860	dwc3_writel(base: dwc->regs, DWC3_DEV_IMOD(0), value: 0);
2861	}
2862
2863	/*
2864	* We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
2865	* field instead of letting dwc3 itself calculate that automatically.
2866	*
2867	* This way, we maximize the chances that we'll be able to get several
2868	* bursts of data without going through any sort of endpoint throttling.
2869	*/
2870	reg = dwc3_readl(base: dwc->regs, DWC3_GRXTHRCFG);
2871	if (DWC3_IP_IS(DWC3))
2872	reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
2873	else
2874	reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
2875
2876	dwc3_writel(base: dwc->regs, DWC3_GRXTHRCFG, value: reg);
2877
2878	dwc3_gadget_setup_nump(dwc);
2879
2880	/*
2881	* Currently the controller handles single stream only. So, Ignore
2882	* Packet Pending bit for stream selection and don't search for another
2883	* stream if the host sends Data Packet with PP=0 (for OUT direction) or
2884	* ACK with NumP=0 and PP=0 (for IN direction). This slightly improves
2885	* the stream performance.
2886	*/
2887	reg = dwc3_readl(base: dwc->regs, DWC3_DCFG);
2888	reg |= DWC3_DCFG_IGNSTRMPP;
2889	dwc3_writel(base: dwc->regs, DWC3_DCFG, value: reg);
2890
2891	/* Enable MST by default if the device is capable of MST */
2892	if (DWC3_MST_CAPABLE(&dwc->hwparams)) {
2893	reg = dwc3_readl(base: dwc->regs, DWC3_DCFG1);
2894	reg &= ~DWC3_DCFG1_DIS_MST_ENH;
2895	dwc3_writel(base: dwc->regs, DWC3_DCFG1, value: reg);
2896	}
2897
2898	/* Start with SuperSpeed Default */
2899	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2900
2901	dep = dwc->eps[0];
2902	dep->flags = 0;
2903	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2904	if (ret) {
2905	dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2906	goto err0;
2907	}
2908
2909	dep = dwc->eps[1];
2910	dep->flags = 0;
2911	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2912	if (ret) {
2913	dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2914	goto err1;
2915	}
2916
2917	/* begin to receive SETUP packets */
2918	dwc->ep0state = EP0_SETUP_PHASE;
2919	dwc->ep0_bounced = false;
2920	dwc->link_state = DWC3_LINK_STATE_SS_DIS;
2921	dwc->delayed_status = false;
2922	dwc3_ep0_out_start(dwc);
2923
2924	dwc3_gadget_enable_irq(dwc);
2925
2926	return 0;
2927
2928	err1:
2929	__dwc3_gadget_ep_disable(dep: dwc->eps[0]);
2930
2931	err0:
2932	return ret;
2933	}
2934
2935	static int dwc3_gadget_start(struct usb_gadget *g,
2936	struct usb_gadget_driver *driver)
2937	{
2938	struct dwc3 *dwc = gadget_to_dwc(g);
2939	unsigned long flags;
2940	int ret;
2941	int irq;
2942
2943	irq = dwc->irq_gadget;
2944	ret = request_threaded_irq(irq, handler: dwc3_interrupt, thread_fn: dwc3_thread_interrupt,
2945	IRQF_SHARED, name: "dwc3", dev: dwc->ev_buf);
2946	if (ret) {
2947	dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
2948	irq, ret);
2949	return ret;
2950	}
2951
2952	spin_lock_irqsave(&dwc->lock, flags);
2953	dwc->gadget_driver = driver;
2954	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2955
2956	return 0;
2957	}
2958
2959	static void __dwc3_gadget_stop(struct dwc3 *dwc)
2960	{
2961	dwc3_gadget_disable_irq(dwc);
2962	__dwc3_gadget_ep_disable(dep: dwc->eps[0]);
2963	__dwc3_gadget_ep_disable(dep: dwc->eps[1]);
2964	}
2965
2966	static int dwc3_gadget_stop(struct usb_gadget *g)
2967	{
2968	struct dwc3 *dwc = gadget_to_dwc(g);
2969	unsigned long flags;
2970
2971	spin_lock_irqsave(&dwc->lock, flags);
2972	dwc->gadget_driver = NULL;
2973	dwc->max_cfg_eps = 0;
2974	spin_unlock_irqrestore(lock: &dwc->lock, flags);
2975
2976	free_irq(dwc->irq_gadget, dwc->ev_buf);
2977
2978	return 0;
2979	}
2980
2981	static void dwc3_gadget_config_params(struct usb_gadget *g,
2982	struct usb_dcd_config_params *params)
2983	{
2984	struct dwc3 *dwc = gadget_to_dwc(g);
2985
2986	params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED;
2987	params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED;
2988
2989	/* Recommended BESL */
2990	if (!dwc->dis_enblslpm_quirk) {
2991	/*
2992	* If the recommended BESL baseline is 0 or if the BESL deep is
2993	* less than 2, Microsoft's Windows 10 host usb stack will issue
2994	* a usb reset immediately after it receives the extended BOS
2995	* descriptor and the enumeration will fail. To maintain
2996	* compatibility with the Windows' usb stack, let's set the
2997	* recommended BESL baseline to 1 and clamp the BESL deep to be
2998	* within 2 to 15.
2999	*/
3000	params->besl_baseline = 1;
3001	if (dwc->is_utmi_l1_suspend)
3002	params->besl_deep =
3003	clamp_t(u8, dwc->hird_threshold, 2, 15);
3004	}
3005
3006	/* U1 Device exit Latency */
3007	if (dwc->dis_u1_entry_quirk)
3008	params->bU1devExitLat = 0;
3009	else
3010	params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT;
3011
3012	/* U2 Device exit Latency */
3013	if (dwc->dis_u2_entry_quirk)
3014	params->bU2DevExitLat = 0;
3015	else
3016	params->bU2DevExitLat =
3017	cpu_to_le16(DWC3_DEFAULT_U2_DEV_EXIT_LAT);
3018	}
3019
3020	static void dwc3_gadget_set_speed(struct usb_gadget *g,
3021	enum usb_device_speed speed)
3022	{
3023	struct dwc3 *dwc = gadget_to_dwc(g);
3024	unsigned long flags;
3025
3026	spin_lock_irqsave(&dwc->lock, flags);
3027	dwc->gadget_max_speed = speed;
3028	spin_unlock_irqrestore(lock: &dwc->lock, flags);
3029	}
3030
3031	static void dwc3_gadget_set_ssp_rate(struct usb_gadget *g,
3032	enum usb_ssp_rate rate)
3033	{
3034	struct dwc3 *dwc = gadget_to_dwc(g);
3035	unsigned long flags;
3036
3037	spin_lock_irqsave(&dwc->lock, flags);
3038	dwc->gadget_max_speed = USB_SPEED_SUPER_PLUS;
3039	dwc->gadget_ssp_rate = rate;
3040	spin_unlock_irqrestore(lock: &dwc->lock, flags);
3041	}
3042
3043	static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA)
3044	{
3045	struct dwc3 *dwc = gadget_to_dwc(g);
3046	union power_supply_propval val = {0};
3047	int ret;
3048
3049	if (dwc->usb2_phy)
3050	return usb_phy_set_power(x: dwc->usb2_phy, mA);
3051
3052	if (!dwc->usb_psy)
3053	return -EOPNOTSUPP;
3054
3055	val.intval = 1000 * mA;
3056	ret = power_supply_set_property(psy: dwc->usb_psy, psp: POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, val: &val);
3057
3058	return ret;
3059	}
3060
3061	/**
3062	* dwc3_gadget_check_config - ensure dwc3 can support the USB configuration
3063	* @g: pointer to the USB gadget
3064	*
3065	* Used to record the maximum number of endpoints being used in a USB composite
3066	* device. (across all configurations) This is to be used in the calculation
3067	* of the TXFIFO sizes when resizing internal memory for individual endpoints.
3068	* It will help ensured that the resizing logic reserves enough space for at
3069	* least one max packet.
3070	*/
3071	static int dwc3_gadget_check_config(struct usb_gadget *g)
3072	{
3073	struct dwc3 *dwc = gadget_to_dwc(g);
3074	struct usb_ep *ep;
3075	int fifo_size = 0;
3076	int ram1_depth;
3077	int ep_num = 0;
3078
3079	if (!dwc->do_fifo_resize)
3080	return 0;
3081
3082	list_for_each_entry(ep, &g->ep_list, ep_list) {
3083	/* Only interested in the IN endpoints */
3084	if (ep->claimed && (ep->address & USB_DIR_IN))
3085	ep_num++;
3086	}
3087
3088	if (ep_num <= dwc->max_cfg_eps)
3089	return 0;
3090
3091	/* Update the max number of eps in the composition */
3092	dwc->max_cfg_eps = ep_num;
3093
3094	fifo_size = dwc3_gadget_calc_tx_fifo_size(dwc, mult: dwc->max_cfg_eps);
3095	/* Based on the equation, increment by one for every ep */
3096	fifo_size += dwc->max_cfg_eps;
3097
3098	/* Check if we can fit a single fifo per endpoint */
3099	ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
3100	if (fifo_size > ram1_depth)
3101	return -ENOMEM;
3102
3103	return 0;
3104	}
3105
3106	static void dwc3_gadget_async_callbacks(struct usb_gadget *g, bool enable)
3107	{
3108	struct dwc3 *dwc = gadget_to_dwc(g);
3109	unsigned long flags;
3110
3111	spin_lock_irqsave(&dwc->lock, flags);
3112	dwc->async_callbacks = enable;
3113	spin_unlock_irqrestore(lock: &dwc->lock, flags);
3114	}
3115
3116	static const struct usb_gadget_ops dwc3_gadget_ops = {
3117	.get_frame = dwc3_gadget_get_frame,
3118	.wakeup = dwc3_gadget_wakeup,
3119	.func_wakeup = dwc3_gadget_func_wakeup,
3120	.set_remote_wakeup = dwc3_gadget_set_remote_wakeup,
3121	.set_selfpowered = dwc3_gadget_set_selfpowered,
3122	.pullup = dwc3_gadget_pullup,
3123	.udc_start = dwc3_gadget_start,
3124	.udc_stop = dwc3_gadget_stop,
3125	.udc_set_speed = dwc3_gadget_set_speed,
3126	.udc_set_ssp_rate = dwc3_gadget_set_ssp_rate,
3127	.get_config_params = dwc3_gadget_config_params,
3128	.vbus_draw = dwc3_gadget_vbus_draw,
3129	.check_config = dwc3_gadget_check_config,
3130	.udc_async_callbacks = dwc3_gadget_async_callbacks,
3131	};
3132
3133	/* -------------------------------------------------------------------------- */
3134
3135	static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep)
3136	{
3137	struct dwc3 *dwc = dep->dwc;
3138
3139	usb_ep_set_maxpacket_limit(ep: &dep->endpoint, maxpacket_limit: 512);
3140	dep->endpoint.maxburst = 1;
3141	dep->endpoint.ops = &dwc3_gadget_ep0_ops;
3142	if (!dep->direction)
3143	dwc->gadget->ep0 = &dep->endpoint;
3144
3145	dep->endpoint.caps.type_control = true;
3146
3147	return 0;
3148	}
3149
3150	static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
3151	{
3152	struct dwc3 *dwc = dep->dwc;
3153	u32 mdwidth;
3154	int size;
3155	int maxpacket;
3156
3157	mdwidth = dwc3_mdwidth(dwc);
3158
3159	/* MDWIDTH is represented in bits, we need it in bytes */
3160	mdwidth /= 8;
3161
3162	size = dwc3_readl(base: dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
3163	if (DWC3_IP_IS(DWC3))
3164	size = DWC3_GTXFIFOSIZ_TXFDEP(size);
3165	else
3166	size = DWC31_GTXFIFOSIZ_TXFDEP(size);
3167
3168	/*
3169	* maxpacket size is determined as part of the following, after assuming
3170	* a mult value of one maxpacket:
3171	* DWC3 revision 280A and prior:
3172	* fifo_size = mult * (max_packet / mdwidth) + 1;
3173	* maxpacket = mdwidth * (fifo_size - 1);
3174	*
3175	* DWC3 revision 290A and onwards:
3176	* fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
3177	* maxpacket = mdwidth * ((fifo_size - 1) - 1) - mdwidth;
3178	*/
3179	if (DWC3_VER_IS_PRIOR(DWC3, 290A))
3180	maxpacket = mdwidth * (size - 1);
3181	else
3182	maxpacket = mdwidth * ((size - 1) - 1) - mdwidth;
3183
3184	/* Functionally, space for one max packet is sufficient */
3185	size = min_t(int, maxpacket, 1024);
3186	usb_ep_set_maxpacket_limit(ep: &dep->endpoint, maxpacket_limit: size);
3187
3188	dep->endpoint.max_streams = 16;
3189	dep->endpoint.ops = &dwc3_gadget_ep_ops;
3190	list_add_tail(new: &dep->endpoint.ep_list,
3191	head: &dwc->gadget->ep_list);
3192	dep->endpoint.caps.type_iso = true;
3193	dep->endpoint.caps.type_bulk = true;
3194	dep->endpoint.caps.type_int = true;
3195
3196	return dwc3_alloc_trb_pool(dep);
3197	}
3198
3199	static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
3200	{
3201	struct dwc3 *dwc = dep->dwc;
3202	u32 mdwidth;
3203	int size;
3204
3205	mdwidth = dwc3_mdwidth(dwc);
3206
3207	/* MDWIDTH is represented in bits, convert to bytes */
3208	mdwidth /= 8;
3209
3210	/* All OUT endpoints share a single RxFIFO space */
3211	size = dwc3_readl(base: dwc->regs, DWC3_GRXFIFOSIZ(0));
3212	if (DWC3_IP_IS(DWC3))
3213	size = DWC3_GRXFIFOSIZ_RXFDEP(size);
3214	else
3215	size = DWC31_GRXFIFOSIZ_RXFDEP(size);
3216
3217	/* FIFO depth is in MDWDITH bytes */
3218	size *= mdwidth;
3219
3220	/*
3221	* To meet performance requirement, a minimum recommended RxFIFO size
3222	* is defined as follow:
3223	* RxFIFO size >= (3 x MaxPacketSize) +
3224	* (3 x 8 bytes setup packets size) + (16 bytes clock crossing margin)
3225	*
3226	* Then calculate the max packet limit as below.
3227	*/
3228	size -= (3 * 8) + 16;
3229	if (size < 0)
3230	size = 0;
3231	else
3232	size /= 3;
3233
3234	usb_ep_set_maxpacket_limit(ep: &dep->endpoint, maxpacket_limit: size);
3235	dep->endpoint.max_streams = 16;
3236	dep->endpoint.ops = &dwc3_gadget_ep_ops;
3237	list_add_tail(new: &dep->endpoint.ep_list,
3238	head: &dwc->gadget->ep_list);
3239	dep->endpoint.caps.type_iso = true;
3240	dep->endpoint.caps.type_bulk = true;
3241	dep->endpoint.caps.type_int = true;
3242
3243	return dwc3_alloc_trb_pool(dep);
3244	}
3245
3246	static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum)
3247	{
3248	struct dwc3_ep *dep;
3249	bool direction = epnum & 1;
3250	int ret;
3251	u8 num = epnum >> 1;
3252
3253	dep = kzalloc(size: sizeof(*dep), GFP_KERNEL);
3254	if (!dep)
3255	return -ENOMEM;
3256
3257	dep->dwc = dwc;
3258	dep->number = epnum;
3259	dep->direction = direction;
3260	dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
3261	dwc->eps[epnum] = dep;
3262	dep->combo_num = 0;
3263	dep->start_cmd_status = 0;
3264
3265	snprintf(buf: dep->name, size: sizeof(dep->name), fmt: "ep%u%s", num,
3266	direction ? "in" : "out");
3267
3268	dep->endpoint.name = dep->name;
3269
3270	if (!(dep->number > 1)) {
3271	dep->endpoint.desc = &dwc3_gadget_ep0_desc;
3272	dep->endpoint.comp_desc = NULL;
3273	}
3274
3275	if (num == 0)
3276	ret = dwc3_gadget_init_control_endpoint(dep);
3277	else if (direction)
3278	ret = dwc3_gadget_init_in_endpoint(dep);
3279	else
3280	ret = dwc3_gadget_init_out_endpoint(dep);
3281
3282	if (ret)
3283	return ret;
3284
3285	dep->endpoint.caps.dir_in = direction;
3286	dep->endpoint.caps.dir_out = !direction;
3287
3288	INIT_LIST_HEAD(list: &dep->pending_list);
3289	INIT_LIST_HEAD(list: &dep->started_list);
3290	INIT_LIST_HEAD(list: &dep->cancelled_list);
3291
3292	dwc3_debugfs_create_endpoint_dir(dep);
3293
3294	return 0;
3295	}
3296
3297	static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
3298	{
3299	u8 epnum;
3300
3301	INIT_LIST_HEAD(list: &dwc->gadget->ep_list);
3302
3303	for (epnum = 0; epnum < total; epnum++) {
3304	int ret;
3305
3306	ret = dwc3_gadget_init_endpoint(dwc, epnum);
3307	if (ret)
3308	return ret;
3309	}
3310
3311	return 0;
3312	}
3313
3314	static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
3315	{
3316	struct dwc3_ep *dep;
3317	u8 epnum;
3318
3319	for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
3320	dep = dwc->eps[epnum];
3321	if (!dep)
3322	continue;
3323	/*
3324	* Physical endpoints 0 and 1 are special; they form the
3325	* bi-directional USB endpoint 0.
3326	*
3327	* For those two physical endpoints, we don't allocate a TRB
3328	* pool nor do we add them the endpoints list. Due to that, we
3329	* shouldn't do these two operations otherwise we would end up
3330	* with all sorts of bugs when removing dwc3.ko.
3331	*/
3332	if (epnum != 0 && epnum != 1) {
3333	dwc3_free_trb_pool(dep);
3334	list_del(entry: &dep->endpoint.ep_list);
3335	}
3336
3337	dwc3_debugfs_remove_endpoint_dir(dep);
3338	kfree(objp: dep);
3339	}
3340	}
3341
3342	/* -------------------------------------------------------------------------- */
3343
3344	static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep,
3345	struct dwc3_request *req, struct dwc3_trb *trb,
3346	const struct dwc3_event_depevt *event, int status, int chain)
3347	{
3348	unsigned int count;
3349
3350	dwc3_ep_inc_deq(dep);
3351
3352	trace_dwc3_complete_trb(dep, trb);
3353	req->num_trbs--;
3354
3355	/*
3356	* If we're in the middle of series of chained TRBs and we
3357	* receive a short transfer along the way, DWC3 will skip
3358	* through all TRBs including the last TRB in the chain (the
3359	* where CHN bit is zero. DWC3 will also avoid clearing HWO
3360	* bit and SW has to do it manually.
3361	*
3362	* We're going to do that here to avoid problems of HW trying
3363	* to use bogus TRBs for transfers.
3364	*/
3365	if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
3366	trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
3367
3368	/*
3369	* For isochronous transfers, the first TRB in a service interval must
3370	* have the Isoc-First type. Track and report its interval frame number.
3371	*/
3372	if (usb_endpoint_xfer_isoc(epd: dep->endpoint.desc) &&
3373	(trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) {
3374	unsigned int frame_number;
3375
3376	frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl);
3377	frame_number &= ~(dep->interval - 1);
3378	req->request.frame_number = frame_number;
3379	}
3380
3381	/*
3382	* We use bounce buffer for requests that needs extra TRB or OUT ZLP. If
3383	* this TRB points to the bounce buffer address, it's a MPS alignment
3384	* TRB. Don't add it to req->remaining calculation.
3385	*/
3386	if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) &&
3387	trb->bph == upper_32_bits(dep->dwc->bounce_addr)) {
3388	trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
3389	return 1;
3390	}
3391
3392	count = trb->size & DWC3_TRB_SIZE_MASK;
3393	req->remaining += count;
3394
3395	if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
3396	return 1;
3397
3398	if (event->status & DEPEVT_STATUS_SHORT && !chain)
3399	return 1;
3400
3401	if ((trb->ctrl & DWC3_TRB_CTRL_ISP_IMI) &&
3402	DWC3_TRB_SIZE_TRBSTS(trb->size) == DWC3_TRBSTS_MISSED_ISOC)
3403	return 1;
3404
3405	if ((trb->ctrl & DWC3_TRB_CTRL_IOC) ||
3406	(trb->ctrl & DWC3_TRB_CTRL_LST))
3407	return 1;
3408
3409	return 0;
3410	}
3411
3412	static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep,
3413	struct dwc3_request *req, const struct dwc3_event_depevt *event,
3414	int status)
3415	{
3416	struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
3417	struct scatterlist *sg = req->sg;
3418	struct scatterlist *s;
3419	unsigned int num_queued = req->num_queued_sgs;
3420	unsigned int i;
3421	int ret = 0;
3422
3423	for_each_sg(sg, s, num_queued, i) {
3424	trb = &dep->trb_pool[dep->trb_dequeue];
3425
3426	req->sg = sg_next(s);
3427	req->num_queued_sgs--;
3428
3429	ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req,
3430	trb, event, status, chain: true);
3431	if (ret)
3432	break;
3433	}
3434
3435	return ret;
3436	}
3437
3438	static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep,
3439	struct dwc3_request *req, const struct dwc3_event_depevt *event,
3440	int status)
3441	{
3442	struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
3443
3444	return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb,
3445	event, status, chain: false);
3446	}
3447
3448	static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req)
3449	{
3450	return req->num_pending_sgs == 0 && req->num_queued_sgs == 0;
3451	}
3452
3453	static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
3454	const struct dwc3_event_depevt *event,
3455	struct dwc3_request *req, int status)
3456	{
3457	int request_status;
3458	int ret;
3459
3460	if (req->request.num_mapped_sgs)
3461	ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event,
3462	status);
3463	else
3464	ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
3465	status);
3466
3467	req->request.actual = req->request.length - req->remaining;
3468
3469	if (!dwc3_gadget_ep_request_completed(req))
3470	goto out;
3471
3472	if (req->needs_extra_trb) {
3473	ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
3474	status);
3475	req->needs_extra_trb = false;
3476	}
3477
3478	/*
3479	* The event status only reflects the status of the TRB with IOC set.
3480	* For the requests that don't set interrupt on completion, the driver
3481	* needs to check and return the status of the completed TRBs associated
3482	* with the request. Use the status of the last TRB of the request.
3483	*/
3484	if (req->request.no_interrupt) {
3485	struct dwc3_trb *trb;
3486
3487	trb = dwc3_ep_prev_trb(dep, index: dep->trb_dequeue);
3488	switch (DWC3_TRB_SIZE_TRBSTS(trb->size)) {
3489	case DWC3_TRBSTS_MISSED_ISOC:
3490	/* Isoc endpoint only */
3491	request_status = -EXDEV;
3492	break;
3493	case DWC3_TRB_STS_XFER_IN_PROG:
3494	/* Applicable when End Transfer with ForceRM=0 */
3495	case DWC3_TRBSTS_SETUP_PENDING:
3496	/* Control endpoint only */
3497	case DWC3_TRBSTS_OK:
3498	default:
3499	request_status = 0;
3500	break;
3501	}
3502	} else {
3503	request_status = status;
3504	}
3505
3506	dwc3_gadget_giveback(dep, req, status: request_status);
3507
3508	out:
3509	return ret;
3510	}
3511
3512	static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep,
3513	const struct dwc3_event_depevt *event, int status)
3514	{
3515	struct dwc3_request *req;
3516
3517	while (!list_empty(head: &dep->started_list)) {
3518	int ret;
3519
3520	req = next_request(list: &dep->started_list);
3521	ret = dwc3_gadget_ep_cleanup_completed_request(dep, event,
3522	req, status);
3523	if (ret)
3524	break;
3525	/*
3526	* The endpoint is disabled, let the dwc3_remove_requests()
3527	* handle the cleanup.
3528	*/
3529	if (!dep->endpoint.desc)
3530	break;
3531	}
3532	}
3533
3534	static bool dwc3_gadget_ep_should_continue(struct dwc3_ep *dep)
3535	{
3536	struct dwc3_request *req;
3537	struct dwc3 *dwc = dep->dwc;
3538
3539	if (!dep->endpoint.desc || !dwc->pullups_connected ||
3540	!dwc->connected)
3541	return false;
3542
3543	if (!list_empty(head: &dep->pending_list))
3544	return true;
3545
3546	/*
3547	* We only need to check the first entry of the started list. We can
3548	* assume the completed requests are removed from the started list.
3549	*/
3550	req = next_request(list: &dep->started_list);
3551	if (!req)
3552	return false;
3553
3554	return !dwc3_gadget_ep_request_completed(req);
3555	}
3556
3557	static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep,
3558	const struct dwc3_event_depevt *event)
3559	{
3560	dep->frame_number = event->parameters;
3561	}
3562
3563	static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep,
3564	const struct dwc3_event_depevt *event, int status)
3565	{
3566	struct dwc3 *dwc = dep->dwc;
3567	bool no_started_trb = true;
3568
3569	dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
3570
3571	if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3572	goto out;
3573
3574	if (!dep->endpoint.desc)
3575	return no_started_trb;
3576
3577	if (usb_endpoint_xfer_isoc(epd: dep->endpoint.desc) &&
3578	list_empty(head: &dep->started_list) &&
3579	(list_empty(head: &dep->pending_list) || status == -EXDEV))
3580	dwc3_stop_active_transfer(dep, force: true, interrupt: true);
3581	else if (dwc3_gadget_ep_should_continue(dep))
3582	if (__dwc3_gadget_kick_transfer(dep) == 0)
3583	no_started_trb = false;
3584
3585	out:
3586	/*
3587	* WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
3588	* See dwc3_gadget_linksts_change_interrupt() for 1st half.
3589	*/
3590	if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
3591	u32 reg;
3592	int i;
3593
3594	for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
3595	dep = dwc->eps[i];
3596
3597	if (!(dep->flags & DWC3_EP_ENABLED))
3598	continue;
3599
3600	if (!list_empty(head: &dep->started_list))
3601	return no_started_trb;
3602	}
3603
3604	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
3605	reg |= dwc->u1u2;
3606	dwc3_writel(base: dwc->regs, DWC3_DCTL, value: reg);
3607
3608	dwc->u1u2 = 0;
3609	}
3610
3611	return no_started_trb;
3612	}
3613
3614	static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
3615	const struct dwc3_event_depevt *event)
3616	{
3617	int status = 0;
3618
3619	if (!dep->endpoint.desc)
3620	return;
3621
3622	if (usb_endpoint_xfer_isoc(epd: dep->endpoint.desc))
3623	dwc3_gadget_endpoint_frame_from_event(dep, event);
3624
3625	if (event->status & DEPEVT_STATUS_BUSERR)
3626	status = -ECONNRESET;
3627
3628	if (event->status & DEPEVT_STATUS_MISSED_ISOC)
3629	status = -EXDEV;
3630
3631	dwc3_gadget_endpoint_trbs_complete(dep, event, status);
3632	}
3633
3634	static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep,
3635	const struct dwc3_event_depevt *event)
3636	{
3637	int status = 0;
3638
3639	dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3640
3641	if (event->status & DEPEVT_STATUS_BUSERR)
3642	status = -ECONNRESET;
3643
3644	if (dwc3_gadget_endpoint_trbs_complete(dep, event, status))
3645	dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
3646	}
3647
3648	static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
3649	const struct dwc3_event_depevt *event)
3650	{
3651	dwc3_gadget_endpoint_frame_from_event(dep, event);
3652
3653	/*
3654	* The XferNotReady event is generated only once before the endpoint
3655	* starts. It will be generated again when END_TRANSFER command is
3656	* issued. For some controller versions, the XferNotReady event may be
3657	* generated while the END_TRANSFER command is still in process. Ignore
3658	* it and wait for the next XferNotReady event after the command is
3659	* completed.
3660	*/
3661	if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3662	return;
3663
3664	(void) __dwc3_gadget_start_isoc(dep);
3665	}
3666
3667	static void dwc3_gadget_endpoint_command_complete(struct dwc3_ep *dep,
3668	const struct dwc3_event_depevt *event)
3669	{
3670	u8 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
3671
3672	if (cmd != DWC3_DEPCMD_ENDTRANSFER)
3673	return;
3674
3675	/*
3676	* The END_TRANSFER command will cause the controller to generate a
3677	* NoStream Event, and it's not due to the host DP NoStream rejection.
3678	* Ignore the next NoStream event.
3679	*/
3680	if (dep->stream_capable)
3681	dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM;
3682
3683	dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
3684	dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3685	dwc3_gadget_ep_cleanup_cancelled_requests(dep);
3686
3687	if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) {
3688	struct dwc3 *dwc = dep->dwc;
3689
3690	dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL;
3691	if (dwc3_send_clear_stall_ep_cmd(dep)) {
3692	struct usb_ep *ep0 = &dwc->eps[0]->endpoint;
3693
3694	dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name);
3695	if (dwc->delayed_status)
3696	__dwc3_gadget_ep0_set_halt(ep: ep0, value: 1);
3697	return;
3698	}
3699
3700	dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
3701	if (dwc->clear_stall_protocol == dep->number)
3702	dwc3_ep0_send_delayed_status(dwc);
3703	}
3704
3705	if ((dep->flags & DWC3_EP_DELAY_START) &&
3706	!usb_endpoint_xfer_isoc(epd: dep->endpoint.desc))
3707	__dwc3_gadget_kick_transfer(dep);
3708
3709	dep->flags &= ~DWC3_EP_DELAY_START;
3710	}
3711
3712	static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep,
3713	const struct dwc3_event_depevt *event)
3714	{
3715	struct dwc3 *dwc = dep->dwc;
3716
3717	if (event->status == DEPEVT_STREAMEVT_FOUND) {
3718	dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3719	goto out;
3720	}
3721
3722	/* Note: NoStream rejection event param value is 0 and not 0xFFFF */
3723	switch (event->parameters) {
3724	case DEPEVT_STREAM_PRIME:
3725	/*
3726	* If the host can properly transition the endpoint state from
3727	* idle to prime after a NoStream rejection, there's no need to
3728	* force restarting the endpoint to reinitiate the stream. To
3729	* simplify the check, assume the host follows the USB spec if
3730	* it primed the endpoint more than once.
3731	*/
3732	if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) {
3733	if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED)
3734	dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM;
3735	else
3736	dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3737	}
3738
3739	break;
3740	case DEPEVT_STREAM_NOSTREAM:
3741	if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) ||
3742	!(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) ||
3743	(!DWC3_MST_CAPABLE(&dwc->hwparams) &&
3744	!(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)))
3745	break;
3746
3747	/*
3748	* If the host rejects a stream due to no active stream, by the
3749	* USB and xHCI spec, the endpoint will be put back to idle
3750	* state. When the host is ready (buffer added/updated), it will
3751	* prime the endpoint to inform the usb device controller. This
3752	* triggers the device controller to issue ERDY to restart the
3753	* stream. However, some hosts don't follow this and keep the
3754	* endpoint in the idle state. No prime will come despite host
3755	* streams are updated, and the device controller will not be
3756	* triggered to generate ERDY to move the next stream data. To
3757	* workaround this and maintain compatibility with various
3758	* hosts, force to reinitiate the stream until the host is ready
3759	* instead of waiting for the host to prime the endpoint.
3760	*/
3761	if (DWC3_VER_IS_WITHIN(DWC32, 100A, ANY)) {
3762	unsigned int cmd = DWC3_DGCMD_SET_ENDPOINT_PRIME;
3763
3764	dwc3_send_gadget_generic_command(dwc, cmd, param: dep->number);
3765	} else {
3766	dep->flags |= DWC3_EP_DELAY_START;
3767	dwc3_stop_active_transfer(dep, force: true, interrupt: true);
3768	return;
3769	}
3770	break;
3771	}
3772
3773	out:
3774	dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM;
3775	}
3776
3777	static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
3778	const struct dwc3_event_depevt *event)
3779	{
3780	struct dwc3_ep *dep;
3781	u8 epnum = event->endpoint_number;
3782
3783	dep = dwc->eps[epnum];
3784
3785	if (!(dep->flags & DWC3_EP_ENABLED)) {
3786	if ((epnum > 1) && !(dep->flags & DWC3_EP_TRANSFER_STARTED))
3787	return;
3788
3789	/* Handle only EPCMDCMPLT when EP disabled */
3790	if ((event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT) &&
3791	!(epnum <= 1 && event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE))
3792	return;
3793	}
3794
3795	if (epnum == 0 || epnum == 1) {
3796	dwc3_ep0_interrupt(dwc, event);
3797	return;
3798	}
3799
3800	switch (event->endpoint_event) {
3801	case DWC3_DEPEVT_XFERINPROGRESS:
3802	dwc3_gadget_endpoint_transfer_in_progress(dep, event);
3803	break;
3804	case DWC3_DEPEVT_XFERNOTREADY:
3805	dwc3_gadget_endpoint_transfer_not_ready(dep, event);
3806	break;
3807	case DWC3_DEPEVT_EPCMDCMPLT:
3808	dwc3_gadget_endpoint_command_complete(dep, event);
3809	break;
3810	case DWC3_DEPEVT_XFERCOMPLETE:
3811	dwc3_gadget_endpoint_transfer_complete(dep, event);
3812	break;
3813	case DWC3_DEPEVT_STREAMEVT:
3814	dwc3_gadget_endpoint_stream_event(dep, event);
3815	break;
3816	case DWC3_DEPEVT_RXTXFIFOEVT:
3817	break;
3818	default:
3819	dev_err(dwc->dev, "unknown endpoint event %d\n", event->endpoint_event);
3820	break;
3821	}
3822	}
3823
3824	static void dwc3_disconnect_gadget(struct dwc3 *dwc)
3825	{
3826	if (dwc->async_callbacks && dwc->gadget_driver->disconnect) {
3827	spin_unlock(lock: &dwc->lock);
3828	dwc->gadget_driver->disconnect(dwc->gadget);
3829	spin_lock(lock: &dwc->lock);
3830	}
3831	}
3832
3833	static void dwc3_suspend_gadget(struct dwc3 *dwc)
3834	{
3835	if (dwc->async_callbacks && dwc->gadget_driver->suspend) {
3836	spin_unlock(lock: &dwc->lock);
3837	dwc->gadget_driver->suspend(dwc->gadget);
3838	spin_lock(lock: &dwc->lock);
3839	}
3840	}
3841
3842	static void dwc3_resume_gadget(struct dwc3 *dwc)
3843	{
3844	if (dwc->async_callbacks && dwc->gadget_driver->resume) {
3845	spin_unlock(lock: &dwc->lock);
3846	dwc->gadget_driver->resume(dwc->gadget);
3847	spin_lock(lock: &dwc->lock);
3848	}
3849	}
3850
3851	static void dwc3_reset_gadget(struct dwc3 *dwc)
3852	{
3853	if (!dwc->gadget_driver)
3854	return;
3855
3856	if (dwc->async_callbacks && dwc->gadget->speed != USB_SPEED_UNKNOWN) {
3857	spin_unlock(lock: &dwc->lock);
3858	usb_gadget_udc_reset(gadget: dwc->gadget, driver: dwc->gadget_driver);
3859	spin_lock(lock: &dwc->lock);
3860	}
3861	}
3862
3863	void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
3864	bool interrupt)
3865	{
3866	struct dwc3 *dwc = dep->dwc;
3867
3868	/*
3869	* Only issue End Transfer command to the control endpoint of a started
3870	* Data Phase. Typically we should only do so in error cases such as
3871	* invalid/unexpected direction as described in the control transfer
3872	* flow of the programming guide.
3873	*/
3874	if (dep->number <= 1 && dwc->ep0state != EP0_DATA_PHASE)
3875	return;
3876
3877	if (interrupt && (dep->flags & DWC3_EP_DELAY_STOP))
3878	return;
3879
3880	if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) ||
3881	(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
3882	return;
3883
3884	/*
3885	* If a Setup packet is received but yet to DMA out, the controller will
3886	* not process the End Transfer command of any endpoint. Polling of its
3887	* DEPCMD.CmdAct may block setting up TRB for Setup packet, causing a
3888	* timeout. Delay issuing the End Transfer command until the Setup TRB is
3889	* prepared.
3890	*/
3891	if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status) {
3892	dep->flags |= DWC3_EP_DELAY_STOP;
3893	return;
3894	}
3895
3896	/*
3897	* NOTICE: We are violating what the Databook says about the
3898	* EndTransfer command. Ideally we would _always_ wait for the
3899	* EndTransfer Command Completion IRQ, but that's causing too
3900	* much trouble synchronizing between us and gadget driver.
3901	*
3902	* We have discussed this with the IP Provider and it was
3903	* suggested to giveback all requests here.
3904	*
3905	* Note also that a similar handling was tested by Synopsys
3906	* (thanks a lot Paul) and nothing bad has come out of it.
3907	* In short, what we're doing is issuing EndTransfer with
3908	* CMDIOC bit set and delay kicking transfer until the
3909	* EndTransfer command had completed.
3910	*
3911	* As of IP version 3.10a of the DWC_usb3 IP, the controller
3912	* supports a mode to work around the above limitation. The
3913	* software can poll the CMDACT bit in the DEPCMD register
3914	* after issuing a EndTransfer command. This mode is enabled
3915	* by writing GUCTL2[14]. This polling is already done in the
3916	* dwc3_send_gadget_ep_cmd() function so if the mode is
3917	* enabled, the EndTransfer command will have completed upon
3918	* returning from this function.
3919	*
3920	* This mode is NOT available on the DWC_usb31 IP. In this
3921	* case, if the IOC bit is not set, then delay by 1ms
3922	* after issuing the EndTransfer command. This allows for the
3923	* controller to handle the command completely before DWC3
3924	* remove requests attempts to unmap USB request buffers.
3925	*/
3926
3927	__dwc3_stop_active_transfer(dep, force, interrupt);
3928	}
3929
3930	static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
3931	{
3932	u32 epnum;
3933
3934	for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
3935	struct dwc3_ep *dep;
3936	int ret;
3937
3938	dep = dwc->eps[epnum];
3939	if (!dep)
3940	continue;
3941
3942	if (!(dep->flags & DWC3_EP_STALL))
3943	continue;
3944
3945	dep->flags &= ~DWC3_EP_STALL;
3946
3947	ret = dwc3_send_clear_stall_ep_cmd(dep);
3948	WARN_ON_ONCE(ret);
3949	}
3950	}
3951
3952	static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
3953	{
3954	int reg;
3955
3956	dwc->suspended = false;
3957
3958	dwc3_gadget_set_link_state(dwc, state: DWC3_LINK_STATE_RX_DET);
3959
3960	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
3961	reg &= ~DWC3_DCTL_INITU1ENA;
3962	reg &= ~DWC3_DCTL_INITU2ENA;
3963	dwc3_gadget_dctl_write_safe(dwc, value: reg);
3964
3965	dwc->connected = false;
3966
3967	dwc3_disconnect_gadget(dwc);
3968
3969	dwc->gadget->speed = USB_SPEED_UNKNOWN;
3970	dwc->setup_packet_pending = false;
3971	dwc->gadget->wakeup_armed = false;
3972	dwc3_gadget_enable_linksts_evts(dwc, set: false);
3973	usb_gadget_set_state(gadget: dwc->gadget, state: USB_STATE_NOTATTACHED);
3974
3975	dwc3_ep0_reset_state(dwc);
3976	}
3977
3978	static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
3979	{
3980	u32 reg;
3981
3982	dwc->suspended = false;
3983
3984	/*
3985	* Ideally, dwc3_reset_gadget() would trigger the function
3986	* drivers to stop any active transfers through ep disable.
3987	* However, for functions which defer ep disable, such as mass
3988	* storage, we will need to rely on the call to stop active
3989	* transfers here, and avoid allowing of request queuing.
3990	*/
3991	dwc->connected = false;
3992
3993	/*
3994	* WORKAROUND: DWC3 revisions <1.88a have an issue which
3995	* would cause a missing Disconnect Event if there's a
3996	* pending Setup Packet in the FIFO.
3997	*
3998	* There's no suggested workaround on the official Bug
3999	* report, which states that "unless the driver/application
4000	* is doing any special handling of a disconnect event,
4001	* there is no functional issue".
4002	*
4003	* Unfortunately, it turns out that we _do_ some special
4004	* handling of a disconnect event, namely complete all
4005	* pending transfers, notify gadget driver of the
4006	* disconnection, and so on.
4007	*
4008	* Our suggested workaround is to follow the Disconnect
4009	* Event steps here, instead, based on a setup_packet_pending
4010	* flag. Such flag gets set whenever we have a SETUP_PENDING
4011	* status for EP0 TRBs and gets cleared on XferComplete for the
4012	* same endpoint.
4013	*
4014	* Refers to:
4015	*
4016	* STAR#9000466709: RTL: Device : Disconnect event not
4017	* generated if setup packet pending in FIFO
4018	*/
4019	if (DWC3_VER_IS_PRIOR(DWC3, 188A)) {
4020	if (dwc->setup_packet_pending)
4021	dwc3_gadget_disconnect_interrupt(dwc);
4022	}
4023
4024	dwc3_reset_gadget(dwc);
4025
4026	/*
4027	* From SNPS databook section 8.1.2, the EP0 should be in setup
4028	* phase. So ensure that EP0 is in setup phase by issuing a stall
4029	* and restart if EP0 is not in setup phase.
4030	*/
4031	dwc3_ep0_reset_state(dwc);
4032
4033	/*
4034	* In the Synopsis DesignWare Cores USB3 Databook Rev. 3.30a
4035	* Section 4.1.2 Table 4-2, it states that during a USB reset, the SW
4036	* needs to ensure that it sends "a DEPENDXFER command for any active
4037	* transfers."
4038	*/
4039	dwc3_stop_active_transfers(dwc);
4040	dwc->connected = true;
4041
4042	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
4043	reg &= ~DWC3_DCTL_TSTCTRL_MASK;
4044	dwc3_gadget_dctl_write_safe(dwc, value: reg);
4045	dwc->test_mode = false;
4046	dwc->gadget->wakeup_armed = false;
4047	dwc3_gadget_enable_linksts_evts(dwc, set: false);
4048	dwc3_clear_stall_all_ep(dwc);
4049
4050	/* Reset device address to zero */
4051	reg = dwc3_readl(base: dwc->regs, DWC3_DCFG);
4052	reg &= ~(DWC3_DCFG_DEVADDR_MASK);
4053	dwc3_writel(base: dwc->regs, DWC3_DCFG, value: reg);
4054	}
4055
4056	static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
4057	{
4058	struct dwc3_ep *dep;
4059	int ret;
4060	u32 reg;
4061	u8 lanes = 1;
4062	u8 speed;
4063
4064	if (!dwc->softconnect)
4065	return;
4066
4067	reg = dwc3_readl(base: dwc->regs, DWC3_DSTS);
4068	speed = reg & DWC3_DSTS_CONNECTSPD;
4069	dwc->speed = speed;
4070
4071	if (DWC3_IP_IS(DWC32))
4072	lanes = DWC3_DSTS_CONNLANES(reg) + 1;
4073
4074	dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN;
4075
4076	/*
4077	* RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
4078	* each time on Connect Done.
4079	*
4080	* Currently we always use the reset value. If any platform
4081	* wants to set this to a different value, we need to add a
4082	* setting and update GCTL.RAMCLKSEL here.
4083	*/
4084
4085	switch (speed) {
4086	case DWC3_DSTS_SUPERSPEED_PLUS:
4087	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
4088	dwc->gadget->ep0->maxpacket = 512;
4089	dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
4090
4091	if (lanes > 1)
4092	dwc->gadget->ssp_rate = USB_SSP_GEN_2x2;
4093	else
4094	dwc->gadget->ssp_rate = USB_SSP_GEN_2x1;
4095	break;
4096	case DWC3_DSTS_SUPERSPEED:
4097	/*
4098	* WORKAROUND: DWC3 revisions <1.90a have an issue which
4099	* would cause a missing USB3 Reset event.
4100	*
4101	* In such situations, we should force a USB3 Reset
4102	* event by calling our dwc3_gadget_reset_interrupt()
4103	* routine.
4104	*
4105	* Refers to:
4106	*
4107	* STAR#9000483510: RTL: SS : USB3 reset event may
4108	* not be generated always when the link enters poll
4109	*/
4110	if (DWC3_VER_IS_PRIOR(DWC3, 190A))
4111	dwc3_gadget_reset_interrupt(dwc);
4112
4113	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
4114	dwc->gadget->ep0->maxpacket = 512;
4115	dwc->gadget->speed = USB_SPEED_SUPER;
4116
4117	if (lanes > 1) {
4118	dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
4119	dwc->gadget->ssp_rate = USB_SSP_GEN_1x2;
4120	}
4121	break;
4122	case DWC3_DSTS_HIGHSPEED:
4123	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
4124	dwc->gadget->ep0->maxpacket = 64;
4125	dwc->gadget->speed = USB_SPEED_HIGH;
4126	break;
4127	case DWC3_DSTS_FULLSPEED:
4128	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
4129	dwc->gadget->ep0->maxpacket = 64;
4130	dwc->gadget->speed = USB_SPEED_FULL;
4131	break;
4132	}
4133
4134	dwc->eps[1]->endpoint.maxpacket = dwc->gadget->ep0->maxpacket;
4135
4136	/* Enable USB2 LPM Capability */
4137
4138	if (!DWC3_VER_IS_WITHIN(DWC3, ANY, 194A) &&
4139	!dwc->usb2_gadget_lpm_disable &&
4140	(speed != DWC3_DSTS_SUPERSPEED) &&
4141	(speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
4142	reg = dwc3_readl(base: dwc->regs, DWC3_DCFG);
4143	reg |= DWC3_DCFG_LPM_CAP;
4144	dwc3_writel(base: dwc->regs, DWC3_DCFG, value: reg);
4145
4146	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
4147	reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
4148
4149	reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold |
4150	(dwc->is_utmi_l1_suspend << 4));
4151
4152	/*
4153	* When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
4154	* DCFG.LPMCap is set, core responses with an ACK and the
4155	* BESL value in the LPM token is less than or equal to LPM
4156	* NYET threshold.
4157	*/
4158	WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum,
4159	"LPM Erratum not available on dwc3 revisions < 2.40a\n");
4160
4161	if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A))
4162	reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold);
4163
4164	dwc3_gadget_dctl_write_safe(dwc, value: reg);
4165	} else {
4166	if (dwc->usb2_gadget_lpm_disable) {
4167	reg = dwc3_readl(base: dwc->regs, DWC3_DCFG);
4168	reg &= ~DWC3_DCFG_LPM_CAP;
4169	dwc3_writel(base: dwc->regs, DWC3_DCFG, value: reg);
4170	}
4171
4172	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
4173	reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
4174	dwc3_gadget_dctl_write_safe(dwc, value: reg);
4175	}
4176
4177	dep = dwc->eps[0];
4178	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
4179	if (ret) {
4180	dev_err(dwc->dev, "failed to enable %s\n", dep->name);
4181	return;
4182	}
4183
4184	dep = dwc->eps[1];
4185	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
4186	if (ret) {
4187	dev_err(dwc->dev, "failed to enable %s\n", dep->name);
4188	return;
4189	}
4190
4191	/*
4192	* Configure PHY via GUSB3PIPECTLn if required.
4193	*
4194	* Update GTXFIFOSIZn
4195	*
4196	* In both cases reset values should be sufficient.
4197	*/
4198	}
4199
4200	static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc, unsigned int evtinfo)
4201	{
4202	dwc->suspended = false;
4203
4204	/*
4205	* TODO take core out of low power mode when that's
4206	* implemented.
4207	*/
4208
4209	if (dwc->async_callbacks && dwc->gadget_driver->resume) {
4210	spin_unlock(lock: &dwc->lock);
4211	dwc->gadget_driver->resume(dwc->gadget);
4212	spin_lock(lock: &dwc->lock);
4213	}
4214
4215	dwc->link_state = evtinfo & DWC3_LINK_STATE_MASK;
4216	}
4217
4218	static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
4219	unsigned int evtinfo)
4220	{
4221	enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
4222	unsigned int pwropt;
4223
4224	/*
4225	* WORKAROUND: DWC3 < 2.50a have an issue when configured without
4226	* Hibernation mode enabled which would show up when device detects
4227	* host-initiated U3 exit.
4228	*
4229	* In that case, device will generate a Link State Change Interrupt
4230	* from U3 to RESUME which is only necessary if Hibernation is
4231	* configured in.
4232	*
4233	* There are no functional changes due to such spurious event and we
4234	* just need to ignore it.
4235	*
4236	* Refers to:
4237	*
4238	* STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
4239	* operational mode
4240	*/
4241	pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
4242	if (DWC3_VER_IS_PRIOR(DWC3, 250A) &&
4243	(pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
4244	if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
4245	(next == DWC3_LINK_STATE_RESUME)) {
4246	return;
4247	}
4248	}
4249
4250	/*
4251	* WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
4252	* on the link partner, the USB session might do multiple entry/exit
4253	* of low power states before a transfer takes place.
4254	*
4255	* Due to this problem, we might experience lower throughput. The
4256	* suggested workaround is to disable DCTL[12:9] bits if we're
4257	* transitioning from U1/U2 to U0 and enable those bits again
4258	* after a transfer completes and there are no pending transfers
4259	* on any of the enabled endpoints.
4260	*
4261	* This is the first half of that workaround.
4262	*
4263	* Refers to:
4264	*
4265	* STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
4266	* core send LGO_Ux entering U0
4267	*/
4268	if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
4269	if (next == DWC3_LINK_STATE_U0) {
4270	u32 u1u2;
4271	u32 reg;
4272
4273	switch (dwc->link_state) {
4274	case DWC3_LINK_STATE_U1:
4275	case DWC3_LINK_STATE_U2:
4276	reg = dwc3_readl(base: dwc->regs, DWC3_DCTL);
4277	u1u2 = reg & (DWC3_DCTL_INITU2ENA
4278	| DWC3_DCTL_ACCEPTU2ENA
4279	| DWC3_DCTL_INITU1ENA
4280	| DWC3_DCTL_ACCEPTU1ENA);
4281
4282	if (!dwc->u1u2)
4283	dwc->u1u2 = reg & u1u2;
4284
4285	reg &= ~u1u2;
4286
4287	dwc3_gadget_dctl_write_safe(dwc, value: reg);
4288	break;
4289	default:
4290	/* do nothing */
4291	break;
4292	}
4293	}
4294	}
4295
4296	switch (next) {
4297	case DWC3_LINK_STATE_U0:
4298	if (dwc->gadget->wakeup_armed) {
4299	dwc3_gadget_enable_linksts_evts(dwc, set: false);
4300	dwc3_resume_gadget(dwc);
4301	dwc->suspended = false;
4302	}
4303	break;
4304	case DWC3_LINK_STATE_U1:
4305	if (dwc->speed == USB_SPEED_SUPER)
4306	dwc3_suspend_gadget(dwc);
4307	break;
4308	case DWC3_LINK_STATE_U2:
4309	case DWC3_LINK_STATE_U3:
4310	dwc3_suspend_gadget(dwc);
4311	break;
4312	case DWC3_LINK_STATE_RESUME:
4313	dwc3_resume_gadget(dwc);
4314	break;
4315	default:
4316	/* do nothing */
4317	break;
4318	}
4319
4320	dwc->link_state = next;
4321	}
4322
4323	static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
4324	unsigned int evtinfo)
4325	{
4326	enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
4327
4328	if (!dwc->suspended && next == DWC3_LINK_STATE_U3) {
4329	dwc->suspended = true;
4330	dwc3_suspend_gadget(dwc);
4331	}
4332
4333	dwc->link_state = next;
4334	}
4335
4336	static void dwc3_gadget_interrupt(struct dwc3 *dwc,
4337	const struct dwc3_event_devt *event)
4338	{
4339	switch (event->type) {
4340	case DWC3_DEVICE_EVENT_DISCONNECT:
4341	dwc3_gadget_disconnect_interrupt(dwc);
4342	break;
4343	case DWC3_DEVICE_EVENT_RESET:
4344	dwc3_gadget_reset_interrupt(dwc);
4345	break;
4346	case DWC3_DEVICE_EVENT_CONNECT_DONE:
4347	dwc3_gadget_conndone_interrupt(dwc);
4348	break;
4349	case DWC3_DEVICE_EVENT_WAKEUP:
4350	dwc3_gadget_wakeup_interrupt(dwc, evtinfo: event->event_info);
4351	break;
4352	case DWC3_DEVICE_EVENT_HIBER_REQ:
4353	dev_WARN_ONCE(dwc->dev, true, "unexpected hibernation event\n");
4354	break;
4355	case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
4356	dwc3_gadget_linksts_change_interrupt(dwc, evtinfo: event->event_info);
4357	break;
4358	case DWC3_DEVICE_EVENT_SUSPEND:
4359	/* It changed to be suspend event for version 2.30a and above */
4360	if (!DWC3_VER_IS_PRIOR(DWC3, 230A))
4361	dwc3_gadget_suspend_interrupt(dwc, evtinfo: event->event_info);
4362	break;
4363	case DWC3_DEVICE_EVENT_SOF:
4364	case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
4365	case DWC3_DEVICE_EVENT_CMD_CMPL:
4366	case DWC3_DEVICE_EVENT_OVERFLOW:
4367	break;
4368	default:
4369	dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
4370	}
4371	}
4372
4373	static void dwc3_process_event_entry(struct dwc3 *dwc,
4374	const union dwc3_event *event)
4375	{
4376	trace_dwc3_event(event: event->raw, dwc);
4377
4378	if (!event->type.is_devspec)
4379	dwc3_endpoint_interrupt(dwc, event: &event->depevt);
4380	else if (event->type.type == DWC3_EVENT_TYPE_DEV)
4381	dwc3_gadget_interrupt(dwc, event: &event->devt);
4382	else
4383	dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
4384	}
4385
4386	static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
4387	{
4388	struct dwc3 *dwc = evt->dwc;
4389	irqreturn_t ret = IRQ_NONE;
4390	int left;
4391
4392	left = evt->count;
4393
4394	if (!(evt->flags & DWC3_EVENT_PENDING))
4395	return IRQ_NONE;
4396
4397	while (left > 0) {
4398	union dwc3_event event;
4399
4400	event.raw = *(u32 *) (evt->cache + evt->lpos);
4401
4402	dwc3_process_event_entry(dwc, event: &event);
4403
4404	/*
4405	* FIXME we wrap around correctly to the next entry as
4406	* almost all entries are 4 bytes in size. There is one
4407	* entry which has 12 bytes which is a regular entry
4408	* followed by 8 bytes data. ATM I don't know how
4409	* things are organized if we get next to the a
4410	* boundary so I worry about that once we try to handle
4411	* that.
4412	*/
4413	evt->lpos = (evt->lpos + 4) % evt->length;
4414	left -= 4;
4415	}
4416
4417	evt->count = 0;
4418	ret = IRQ_HANDLED;
4419
4420	/* Unmask interrupt */
4421	dwc3_writel(base: dwc->regs, DWC3_GEVNTSIZ(0),
4422	DWC3_GEVNTSIZ_SIZE(evt->length));
4423
4424	if (dwc->imod_interval) {
4425	dwc3_writel(base: dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
4426	dwc3_writel(base: dwc->regs, DWC3_DEV_IMOD(0), value: dwc->imod_interval);
4427	}
4428
4429	/* Keep the clearing of DWC3_EVENT_PENDING at the end */
4430	evt->flags &= ~DWC3_EVENT_PENDING;
4431
4432	return ret;
4433	}
4434
4435	static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
4436	{
4437	struct dwc3_event_buffer *evt = _evt;
4438	struct dwc3 *dwc = evt->dwc;
4439	unsigned long flags;
4440	irqreturn_t ret = IRQ_NONE;
4441
4442	local_bh_disable();
4443	spin_lock_irqsave(&dwc->lock, flags);
4444	ret = dwc3_process_event_buf(evt);
4445	spin_unlock_irqrestore(lock: &dwc->lock, flags);
4446	local_bh_enable();
4447
4448	return ret;
4449	}
4450
4451	static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
4452	{
4453	struct dwc3 *dwc = evt->dwc;
4454	u32 amount;
4455	u32 count;
4456
4457	if (pm_runtime_suspended(dev: dwc->dev)) {
4458	dwc->pending_events = true;
4459	/*
4460	* Trigger runtime resume. The get() function will be balanced
4461	* after processing the pending events in dwc3_process_pending
4462	* events().
4463	*/
4464	pm_runtime_get(dev: dwc->dev);
4465	disable_irq_nosync(irq: dwc->irq_gadget);
4466	return IRQ_HANDLED;
4467	}
4468
4469	/*
4470	* With PCIe legacy interrupt, test shows that top-half irq handler can
4471	* be called again after HW interrupt deassertion. Check if bottom-half
4472	* irq event handler completes before caching new event to prevent
4473	* losing events.
4474	*/
4475	if (evt->flags & DWC3_EVENT_PENDING)
4476	return IRQ_HANDLED;
4477
4478	count = dwc3_readl(base: dwc->regs, DWC3_GEVNTCOUNT(0));
4479	count &= DWC3_GEVNTCOUNT_MASK;
4480	if (!count)
4481	return IRQ_NONE;
4482
4483	evt->count = count;
4484	evt->flags |= DWC3_EVENT_PENDING;
4485
4486	/* Mask interrupt */
4487	dwc3_writel(base: dwc->regs, DWC3_GEVNTSIZ(0),
4488	DWC3_GEVNTSIZ_INTMASK | DWC3_GEVNTSIZ_SIZE(evt->length));
4489
4490	amount = min(count, evt->length - evt->lpos);
4491	memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
4492
4493	if (amount < count)
4494	memcpy(evt->cache, evt->buf, count - amount);
4495
4496	dwc3_writel(base: dwc->regs, DWC3_GEVNTCOUNT(0), value: count);
4497
4498	return IRQ_WAKE_THREAD;
4499	}
4500
4501	static irqreturn_t dwc3_interrupt(int irq, void *_evt)
4502	{
4503	struct dwc3_event_buffer *evt = _evt;
4504
4505	return dwc3_check_event_buf(evt);
4506	}
4507
4508	static int dwc3_gadget_get_irq(struct dwc3 *dwc)
4509	{
4510	struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
4511	int irq;
4512
4513	irq = platform_get_irq_byname_optional(dev: dwc3_pdev, name: "peripheral");
4514	if (irq > 0)
4515	goto out;
4516
4517	if (irq == -EPROBE_DEFER)
4518	goto out;
4519
4520	irq = platform_get_irq_byname_optional(dev: dwc3_pdev, name: "dwc_usb3");
4521	if (irq > 0)
4522	goto out;
4523
4524	if (irq == -EPROBE_DEFER)
4525	goto out;
4526
4527	irq = platform_get_irq(dwc3_pdev, 0);
4528
4529	out:
4530	return irq;
4531	}
4532
4533	static void dwc_gadget_release(struct device *dev)
4534	{
4535	struct usb_gadget *gadget = container_of(dev, struct usb_gadget, dev);
4536
4537	kfree(objp: gadget);
4538	}
4539
4540	/**
4541	* dwc3_gadget_init - initializes gadget related registers
4542	* @dwc: pointer to our controller context structure
4543	*
4544	* Returns 0 on success otherwise negative errno.
4545	*/
4546	int dwc3_gadget_init(struct dwc3 *dwc)
4547	{
4548	int ret;
4549	int irq;
4550	struct device *dev;
4551
4552	irq = dwc3_gadget_get_irq(dwc);
4553	if (irq < 0) {
4554	ret = irq;
4555	goto err0;
4556	}
4557
4558	dwc->irq_gadget = irq;
4559
4560	dwc->ep0_trb = dma_alloc_coherent(dev: dwc->sysdev,
4561	size: sizeof(*dwc->ep0_trb) * 2,
4562	dma_handle: &dwc->ep0_trb_addr, GFP_KERNEL);
4563	if (!dwc->ep0_trb) {
4564	dev_err(dwc->dev, "failed to allocate ep0 trb\n");
4565	ret = -ENOMEM;
4566	goto err0;
4567	}
4568
4569	dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
4570	if (!dwc->setup_buf) {
4571	ret = -ENOMEM;
4572	goto err1;
4573	}
4574
4575	dwc->bounce = dma_alloc_coherent(dev: dwc->sysdev, DWC3_BOUNCE_SIZE,
4576	dma_handle: &dwc->bounce_addr, GFP_KERNEL);
4577	if (!dwc->bounce) {
4578	ret = -ENOMEM;
4579	goto err2;
4580	}
4581
4582	init_completion(x: &dwc->ep0_in_setup);
4583	dwc->gadget = kzalloc(size: sizeof(struct usb_gadget), GFP_KERNEL);
4584	if (!dwc->gadget) {
4585	ret = -ENOMEM;
4586	goto err3;
4587	}
4588
4589
4590	usb_initialize_gadget(parent: dwc->dev, gadget: dwc->gadget, release: dwc_gadget_release);
4591	dev = &dwc->gadget->dev;
4592	dev->platform_data = dwc;
4593	dwc->gadget->ops = &dwc3_gadget_ops;
4594	dwc->gadget->speed = USB_SPEED_UNKNOWN;
4595	dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN;
4596	dwc->gadget->sg_supported = true;
4597	dwc->gadget->name = "dwc3-gadget";
4598	dwc->gadget->lpm_capable = !dwc->usb2_gadget_lpm_disable;
4599	dwc->gadget->wakeup_capable = true;
4600
4601	/*
4602	* FIXME We might be setting max_speed to <SUPER, however versions
4603	* <2.20a of dwc3 have an issue with metastability (documented
4604	* elsewhere in this driver) which tells us we can't set max speed to
4605	* anything lower than SUPER.
4606	*
4607	* Because gadget.max_speed is only used by composite.c and function
4608	* drivers (i.e. it won't go into dwc3's registers) we are allowing this
4609	* to happen so we avoid sending SuperSpeed Capability descriptor
4610	* together with our BOS descriptor as that could confuse host into
4611	* thinking we can handle super speed.
4612	*
4613	* Note that, in fact, we won't even support GetBOS requests when speed
4614	* is less than super speed because we don't have means, yet, to tell
4615	* composite.c that we are USB 2.0 + LPM ECN.
4616	*/
4617	if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
4618	!dwc->dis_metastability_quirk)
4619	dev_info(dwc->dev, "changing max_speed on rev %08x\n",
4620	dwc->revision);
4621
4622	dwc->gadget->max_speed = dwc->maximum_speed;
4623	dwc->gadget->max_ssp_rate = dwc->max_ssp_rate;
4624
4625	/*
4626	* REVISIT: Here we should clear all pending IRQs to be
4627	* sure we're starting from a well known location.
4628	*/
4629
4630	ret = dwc3_gadget_init_endpoints(dwc, total: dwc->num_eps);
4631	if (ret)
4632	goto err4;
4633
4634	ret = usb_add_gadget(gadget: dwc->gadget);
4635	if (ret) {
4636	dev_err(dwc->dev, "failed to add gadget\n");
4637	goto err5;
4638	}
4639
4640	if (DWC3_IP_IS(DWC32) && dwc->maximum_speed == USB_SPEED_SUPER_PLUS)
4641	dwc3_gadget_set_ssp_rate(g: dwc->gadget, rate: dwc->max_ssp_rate);
4642	else
4643	dwc3_gadget_set_speed(g: dwc->gadget, speed: dwc->maximum_speed);
4644
4645	return 0;
4646
4647	err5:
4648	dwc3_gadget_free_endpoints(dwc);
4649	err4:
4650	usb_put_gadget(gadget: dwc->gadget);
4651	dwc->gadget = NULL;
4652	err3:
4653	dma_free_coherent(dev: dwc->sysdev, DWC3_BOUNCE_SIZE, cpu_addr: dwc->bounce,
4654	dma_handle: dwc->bounce_addr);
4655
4656	err2:
4657	kfree(objp: dwc->setup_buf);
4658
4659	err1:
4660	dma_free_coherent(dev: dwc->sysdev, size: sizeof(*dwc->ep0_trb) * 2,
4661	cpu_addr: dwc->ep0_trb, dma_handle: dwc->ep0_trb_addr);
4662
4663	err0:
4664	return ret;
4665	}
4666
4667	/* -------------------------------------------------------------------------- */
4668
4669	void dwc3_gadget_exit(struct dwc3 *dwc)
4670	{
4671	if (!dwc->gadget)
4672	return;
4673
4674	usb_del_gadget(gadget: dwc->gadget);
4675	dwc3_gadget_free_endpoints(dwc);
4676	usb_put_gadget(gadget: dwc->gadget);
4677	dma_free_coherent(dev: dwc->sysdev, DWC3_BOUNCE_SIZE, cpu_addr: dwc->bounce,
4678	dma_handle: dwc->bounce_addr);
4679	kfree(objp: dwc->setup_buf);
4680	dma_free_coherent(dev: dwc->sysdev, size: sizeof(*dwc->ep0_trb) * 2,
4681	cpu_addr: dwc->ep0_trb, dma_handle: dwc->ep0_trb_addr);
4682	}
4683
4684	int dwc3_gadget_suspend(struct dwc3 *dwc)
4685	{
4686	unsigned long flags;
4687	int ret;
4688
4689	if (!dwc->gadget_driver)
4690	return 0;
4691
4692	ret = dwc3_gadget_soft_disconnect(dwc);
4693	if (ret)
4694	goto err;
4695
4696	spin_lock_irqsave(&dwc->lock, flags);
4697	dwc3_disconnect_gadget(dwc);
4698	spin_unlock_irqrestore(lock: &dwc->lock, flags);
4699
4700	return 0;
4701
4702	err:
4703	/*
4704	* Attempt to reset the controller's state. Likely no
4705	* communication can be established until the host
4706	* performs a port reset.
4707	*/
4708	if (dwc->softconnect)
4709	dwc3_gadget_soft_connect(dwc);
4710
4711	return ret;
4712	}
4713
4714	int dwc3_gadget_resume(struct dwc3 *dwc)
4715	{
4716	if (!dwc->gadget_driver || !dwc->softconnect)
4717	return 0;
4718
4719	return dwc3_gadget_soft_connect(dwc);
4720	}
4721
4722	void dwc3_gadget_process_pending_events(struct dwc3 *dwc)
4723	{
4724	if (dwc->pending_events) {
4725	dwc3_interrupt(irq: dwc->irq_gadget, evt: dwc->ev_buf);
4726	dwc3_thread_interrupt(irq: dwc->irq_gadget, evt: dwc->ev_buf);
4727	pm_runtime_put(dev: dwc->dev);
4728	dwc->pending_events = false;
4729	enable_irq(irq: dwc->irq_gadget);
4730	}
4731	}
4732