1	// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2	/*
3	* hcd_intr.c - DesignWare HS OTG Controller host-mode interrupt handling
4	*
5	* Copyright (C) 2004-2013 Synopsys, Inc.
6	*/
7
8	/*
9	* This file contains the interrupt handlers for Host mode
10	*/
11	#include <linux/kernel.h>
12	#include <linux/module.h>
13	#include <linux/spinlock.h>
14	#include <linux/interrupt.h>
15	#include <linux/dma-mapping.h>
16	#include <linux/io.h>
17	#include <linux/slab.h>
18	#include <linux/usb.h>
19
20	#include <linux/usb/hcd.h>
21	#include <linux/usb/ch11.h>
22
23	#include "core.h"
24	#include "hcd.h"
25
26	/*
27	* If we get this many NAKs on a split transaction we'll slow down
28	* retransmission. A 1 here means delay after the first NAK.
29	*/
30	#define DWC2_NAKS_BEFORE_DELAY 3
31
32	/* This function is for debug only */
33	static void dwc2_track_missed_sofs(struct dwc2_hsotg *hsotg)
34	{
35	u16 curr_frame_number = hsotg->frame_number;
36	u16 expected = dwc2_frame_num_inc(frame: hsotg->last_frame_num, inc: 1);
37
38	if (expected != curr_frame_number)
39	dwc2_sch_vdbg(hsotg, "MISSED SOF %04x != %04x\n",
40	expected, curr_frame_number);
41
42	#ifdef CONFIG_USB_DWC2_TRACK_MISSED_SOFS
43	if (hsotg->frame_num_idx < FRAME_NUM_ARRAY_SIZE) {
44	if (expected != curr_frame_number) {
45	hsotg->frame_num_array[hsotg->frame_num_idx] =
46	curr_frame_number;
47	hsotg->last_frame_num_array[hsotg->frame_num_idx] =
48	hsotg->last_frame_num;
49	hsotg->frame_num_idx++;
50	}
51	} else if (!hsotg->dumped_frame_num_array) {
52	int i;
53
54	dev_info(hsotg->dev, "Frame Last Frame\n");
55	dev_info(hsotg->dev, "----- ----------\n");
56	for (i = 0; i < FRAME_NUM_ARRAY_SIZE; i++) {
57	dev_info(hsotg->dev, "0x%04x 0x%04x\n",
58	hsotg->frame_num_array[i],
59	hsotg->last_frame_num_array[i]);
60	}
61	hsotg->dumped_frame_num_array = 1;
62	}
63	#endif
64	hsotg->last_frame_num = curr_frame_number;
65	}
66
67	static void dwc2_hc_handle_tt_clear(struct dwc2_hsotg *hsotg,
68	struct dwc2_host_chan *chan,
69	struct dwc2_qtd *qtd)
70	{
71	struct usb_device *root_hub = dwc2_hsotg_to_hcd(hsotg)->self.root_hub;
72	struct urb *usb_urb;
73
74	if (!chan->qh)
75	return;
76
77	if (chan->qh->dev_speed == USB_SPEED_HIGH)
78	return;
79
80	if (!qtd->urb)
81	return;
82
83	usb_urb = qtd->urb->priv;
84	if (!usb_urb || !usb_urb->dev || !usb_urb->dev->tt)
85	return;
86
87	/*
88	* The root hub doesn't really have a TT, but Linux thinks it
89	* does because how could you have a "high speed hub" that
90	* directly talks directly to low speed devices without a TT?
91	* It's all lies. Lies, I tell you.
92	*/
93	if (usb_urb->dev->tt->hub == root_hub)
94	return;
95
96	if (qtd->urb->status != -EPIPE && qtd->urb->status != -EREMOTEIO) {
97	chan->qh->tt_buffer_dirty = 1;
98	if (usb_hub_clear_tt_buffer(urb: usb_urb))
99	/* Clear failed; let's hope things work anyway */
100	chan->qh->tt_buffer_dirty = 0;
101	}
102	}
103
104	/*
105	* Handles the start-of-frame interrupt in host mode. Non-periodic
106	* transactions may be queued to the DWC_otg controller for the current
107	* (micro)frame. Periodic transactions may be queued to the controller
108	* for the next (micro)frame.
109	*/
110	static void dwc2_sof_intr(struct dwc2_hsotg *hsotg)
111	{
112	struct list_head *qh_entry;
113	struct dwc2_qh *qh;
114	enum dwc2_transaction_type tr_type;
115
116	/* Clear interrupt */
117	dwc2_writel(hsotg, GINTSTS_SOF, GINTSTS);
118
119	#ifdef DEBUG_SOF
120	dev_vdbg(hsotg->dev, "--Start of Frame Interrupt--\n");
121	#endif
122
123	hsotg->frame_number = dwc2_hcd_get_frame_number(hsotg);
124
125	dwc2_track_missed_sofs(hsotg);
126
127	/* Determine whether any periodic QHs should be executed */
128	qh_entry = hsotg->periodic_sched_inactive.next;
129	while (qh_entry != &hsotg->periodic_sched_inactive) {
130	qh = list_entry(qh_entry, struct dwc2_qh, qh_list_entry);
131	qh_entry = qh_entry->next;
132	if (dwc2_frame_num_le(frame1: qh->next_active_frame,
133	frame2: hsotg->frame_number)) {
134	dwc2_sch_vdbg(hsotg, "QH=%p ready fn=%04x, nxt=%04x\n",
135	qh, hsotg->frame_number,
136	qh->next_active_frame);
137
138	/*
139	* Move QH to the ready list to be executed next
140	* (micro)frame
141	*/
142	list_move_tail(list: &qh->qh_list_entry,
143	head: &hsotg->periodic_sched_ready);
144	}
145	}
146	tr_type = dwc2_hcd_select_transactions(hsotg);
147	if (tr_type != DWC2_TRANSACTION_NONE)
148	dwc2_hcd_queue_transactions(hsotg, tr_type);
149	}
150
151	/*
152	* Handles the Rx FIFO Level Interrupt, which indicates that there is
153	* at least one packet in the Rx FIFO. The packets are moved from the FIFO to
154	* memory if the DWC_otg controller is operating in Slave mode.
155	*/
156	static void dwc2_rx_fifo_level_intr(struct dwc2_hsotg *hsotg)
157	{
158	u32 grxsts, chnum, bcnt, dpid, pktsts;
159	struct dwc2_host_chan *chan;
160
161	if (dbg_perio())
162	dev_vdbg(hsotg->dev, "--RxFIFO Level Interrupt--\n");
163
164	grxsts = dwc2_readl(hsotg, GRXSTSP);
165	chnum = (grxsts & GRXSTS_HCHNUM_MASK) >> GRXSTS_HCHNUM_SHIFT;
166	chan = hsotg->hc_ptr_array[chnum];
167	if (!chan) {
168	dev_err(hsotg->dev, "Unable to get corresponding channel\n");
169	return;
170	}
171
172	bcnt = (grxsts & GRXSTS_BYTECNT_MASK) >> GRXSTS_BYTECNT_SHIFT;
173	dpid = (grxsts & GRXSTS_DPID_MASK) >> GRXSTS_DPID_SHIFT;
174	pktsts = (grxsts & GRXSTS_PKTSTS_MASK) >> GRXSTS_PKTSTS_SHIFT;
175
176	/* Packet Status */
177	if (dbg_perio()) {
178	dev_vdbg(hsotg->dev, " Ch num = %d\n", chnum);
179	dev_vdbg(hsotg->dev, " Count = %d\n", bcnt);
180	dev_vdbg(hsotg->dev, " DPID = %d, chan.dpid = %d\n", dpid,
181	chan->data_pid_start);
182	dev_vdbg(hsotg->dev, " PStatus = %d\n", pktsts);
183	}
184
185	switch (pktsts) {
186	case GRXSTS_PKTSTS_HCHIN:
187	/* Read the data into the host buffer */
188	if (bcnt > 0) {
189	dwc2_read_packet(hsotg, dest: chan->xfer_buf, bytes: bcnt);
190
191	/* Update the HC fields for the next packet received */
192	chan->xfer_count += bcnt;
193	chan->xfer_buf += bcnt;
194	}
195	break;
196	case GRXSTS_PKTSTS_HCHIN_XFER_COMP:
197	case GRXSTS_PKTSTS_DATATOGGLEERR:
198	case GRXSTS_PKTSTS_HCHHALTED:
199	/* Handled in interrupt, just ignore data */
200	break;
201	default:
202	dev_err(hsotg->dev,
203	"RxFIFO Level Interrupt: Unknown status %d\n", pktsts);
204	break;
205	}
206	}
207
208	/*
209	* This interrupt occurs when the non-periodic Tx FIFO is half-empty. More
210	* data packets may be written to the FIFO for OUT transfers. More requests
211	* may be written to the non-periodic request queue for IN transfers. This
212	* interrupt is enabled only in Slave mode.
213	*/
214	static void dwc2_np_tx_fifo_empty_intr(struct dwc2_hsotg *hsotg)
215	{
216	dev_vdbg(hsotg->dev, "--Non-Periodic TxFIFO Empty Interrupt--\n");
217	dwc2_hcd_queue_transactions(hsotg, tr_type: DWC2_TRANSACTION_NON_PERIODIC);
218	}
219
220	/*
221	* This interrupt occurs when the periodic Tx FIFO is half-empty. More data
222	* packets may be written to the FIFO for OUT transfers. More requests may be
223	* written to the periodic request queue for IN transfers. This interrupt is
224	* enabled only in Slave mode.
225	*/
226	static void dwc2_perio_tx_fifo_empty_intr(struct dwc2_hsotg *hsotg)
227	{
228	if (dbg_perio())
229	dev_vdbg(hsotg->dev, "--Periodic TxFIFO Empty Interrupt--\n");
230	dwc2_hcd_queue_transactions(hsotg, tr_type: DWC2_TRANSACTION_PERIODIC);
231	}
232
233	static void dwc2_hprt0_enable(struct dwc2_hsotg *hsotg, u32 hprt0,
234	u32 *hprt0_modify)
235	{
236	struct dwc2_core_params *params = &hsotg->params;
237	int do_reset = 0;
238	u32 usbcfg;
239	u32 prtspd;
240	u32 hcfg;
241	u32 fslspclksel;
242	u32 hfir;
243
244	dev_vdbg(hsotg->dev, "%s(%p)\n", __func__, hsotg);
245
246	/* Every time when port enables calculate HFIR.FrInterval */
247	hfir = dwc2_readl(hsotg, HFIR);
248	hfir &= ~HFIR_FRINT_MASK;
249	hfir |= dwc2_calc_frame_interval(hsotg) << HFIR_FRINT_SHIFT &
250	HFIR_FRINT_MASK;
251	dwc2_writel(hsotg, value: hfir, HFIR);
252
253	/* Check if we need to adjust the PHY clock speed for low power */
254	if (!params->host_support_fs_ls_low_power) {
255	/* Port has been enabled, set the reset change flag */
256	hsotg->flags.b.port_reset_change = 1;
257	return;
258	}
259
260	usbcfg = dwc2_readl(hsotg, GUSBCFG);
261	prtspd = (hprt0 & HPRT0_SPD_MASK) >> HPRT0_SPD_SHIFT;
262
263	if (prtspd == HPRT0_SPD_LOW_SPEED || prtspd == HPRT0_SPD_FULL_SPEED) {
264	/* Low power */
265	if (!(usbcfg & GUSBCFG_PHY_LP_CLK_SEL)) {
266	/* Set PHY low power clock select for FS/LS devices */
267	usbcfg |= GUSBCFG_PHY_LP_CLK_SEL;
268	dwc2_writel(hsotg, value: usbcfg, GUSBCFG);
269	do_reset = 1;
270	}
271
272	hcfg = dwc2_readl(hsotg, HCFG);
273	fslspclksel = (hcfg & HCFG_FSLSPCLKSEL_MASK) >>
274	HCFG_FSLSPCLKSEL_SHIFT;
275
276	if (prtspd == HPRT0_SPD_LOW_SPEED &&
277	params->host_ls_low_power_phy_clk) {
278	/* 6 MHZ */
279	dev_vdbg(hsotg->dev,
280	"FS_PHY programming HCFG to 6 MHz\n");
281	if (fslspclksel != HCFG_FSLSPCLKSEL_6_MHZ) {
282	fslspclksel = HCFG_FSLSPCLKSEL_6_MHZ;
283	hcfg &= ~HCFG_FSLSPCLKSEL_MASK;
284	hcfg |= fslspclksel << HCFG_FSLSPCLKSEL_SHIFT;
285	dwc2_writel(hsotg, value: hcfg, HCFG);
286	do_reset = 1;
287	}
288	} else {
289	/* 48 MHZ */
290	dev_vdbg(hsotg->dev,
291	"FS_PHY programming HCFG to 48 MHz\n");
292	if (fslspclksel != HCFG_FSLSPCLKSEL_48_MHZ) {
293	fslspclksel = HCFG_FSLSPCLKSEL_48_MHZ;
294	hcfg &= ~HCFG_FSLSPCLKSEL_MASK;
295	hcfg |= fslspclksel << HCFG_FSLSPCLKSEL_SHIFT;
296	dwc2_writel(hsotg, value: hcfg, HCFG);
297	do_reset = 1;
298	}
299	}
300	} else {
301	/* Not low power */
302	if (usbcfg & GUSBCFG_PHY_LP_CLK_SEL) {
303	usbcfg &= ~GUSBCFG_PHY_LP_CLK_SEL;
304	dwc2_writel(hsotg, value: usbcfg, GUSBCFG);
305	do_reset = 1;
306	}
307	}
308
309	if (do_reset) {
310	*hprt0_modify |= HPRT0_RST;
311	dwc2_writel(hsotg, value: *hprt0_modify, HPRT0);
312	queue_delayed_work(wq: hsotg->wq_otg, dwork: &hsotg->reset_work,
313	delay: msecs_to_jiffies(m: 60));
314	} else {
315	/* Port has been enabled, set the reset change flag */
316	hsotg->flags.b.port_reset_change = 1;
317	}
318	}
319
320	/*
321	* There are multiple conditions that can cause a port interrupt. This function
322	* determines which interrupt conditions have occurred and handles them
323	* appropriately.
324	*/
325	static void dwc2_port_intr(struct dwc2_hsotg *hsotg)
326	{
327	u32 hprt0;
328	u32 hprt0_modify;
329
330	dev_vdbg(hsotg->dev, "--Port Interrupt--\n");
331
332	hprt0 = dwc2_readl(hsotg, HPRT0);
333	hprt0_modify = hprt0;
334
335	/*
336	* Clear appropriate bits in HPRT0 to clear the interrupt bit in
337	* GINTSTS
338	*/
339	hprt0_modify &= ~(HPRT0_ENA | HPRT0_CONNDET | HPRT0_ENACHG |
340	HPRT0_OVRCURRCHG);
341
342	/*
343	* Port Connect Detected
344	* Set flag and clear if detected
345	*/
346	if (hprt0 & HPRT0_CONNDET) {
347	dwc2_writel(hsotg, value: hprt0_modify | HPRT0_CONNDET, HPRT0);
348
349	dev_vdbg(hsotg->dev,
350	"--Port Interrupt HPRT0=0x%08x Port Connect Detected--\n",
351	hprt0);
352	dwc2_hcd_connect(hsotg);
353
354	/*
355	* The Hub driver asserts a reset when it sees port connect
356	* status change flag
357	*/
358	}
359
360	/*
361	* Port Enable Changed
362	* Clear if detected - Set internal flag if disabled
363	*/
364	if (hprt0 & HPRT0_ENACHG) {
365	dwc2_writel(hsotg, value: hprt0_modify | HPRT0_ENACHG, HPRT0);
366	dev_vdbg(hsotg->dev,
367	" --Port Interrupt HPRT0=0x%08x Port Enable Changed (now %d)--\n",
368	hprt0, !!(hprt0 & HPRT0_ENA));
369	if (hprt0 & HPRT0_ENA) {
370	hsotg->new_connection = true;
371	dwc2_hprt0_enable(hsotg, hprt0, hprt0_modify: &hprt0_modify);
372	} else {
373	hsotg->flags.b.port_enable_change = 1;
374	if (hsotg->params.dma_desc_fs_enable) {
375	u32 hcfg;
376
377	hsotg->params.dma_desc_enable = false;
378	hsotg->new_connection = false;
379	hcfg = dwc2_readl(hsotg, HCFG);
380	hcfg &= ~HCFG_DESCDMA;
381	dwc2_writel(hsotg, value: hcfg, HCFG);
382	}
383	}
384	}
385
386	/* Overcurrent Change Interrupt */
387	if (hprt0 & HPRT0_OVRCURRCHG) {
388	dwc2_writel(hsotg, value: hprt0_modify | HPRT0_OVRCURRCHG,
389	HPRT0);
390	dev_vdbg(hsotg->dev,
391	" --Port Interrupt HPRT0=0x%08x Port Overcurrent Changed--\n",
392	hprt0);
393	hsotg->flags.b.port_over_current_change = 1;
394	}
395	}
396
397	/*
398	* Gets the actual length of a transfer after the transfer halts. halt_status
399	* holds the reason for the halt.
400	*
401	* For IN transfers where halt_status is DWC2_HC_XFER_COMPLETE, *short_read
402	* is set to 1 upon return if less than the requested number of bytes were
403	* transferred. short_read may also be NULL on entry, in which case it remains
404	* unchanged.
405	*/
406	static u32 dwc2_get_actual_xfer_length(struct dwc2_hsotg *hsotg,
407	struct dwc2_host_chan *chan, int chnum,
408	struct dwc2_qtd *qtd,
409	enum dwc2_halt_status halt_status,
410	int *short_read)
411	{
412	u32 hctsiz, count, length;
413
414	hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
415
416	if (halt_status == DWC2_HC_XFER_COMPLETE) {
417	if (chan->ep_is_in) {
418	count = (hctsiz & TSIZ_XFERSIZE_MASK) >>
419	TSIZ_XFERSIZE_SHIFT;
420	length = chan->xfer_len - count;
421	if (short_read)
422	*short_read = (count != 0);
423	} else if (chan->qh->do_split) {
424	length = qtd->ssplit_out_xfer_count;
425	} else {
426	length = chan->xfer_len;
427	}
428	} else {
429	/*
430	* Must use the hctsiz.pktcnt field to determine how much data
431	* has been transferred. This field reflects the number of
432	* packets that have been transferred via the USB. This is
433	* always an integral number of packets if the transfer was
434	* halted before its normal completion. (Can't use the
435	* hctsiz.xfersize field because that reflects the number of
436	* bytes transferred via the AHB, not the USB).
437	*/
438	count = (hctsiz & TSIZ_PKTCNT_MASK) >> TSIZ_PKTCNT_SHIFT;
439	length = (chan->start_pkt_count - count) * chan->max_packet;
440	}
441
442	return length;
443	}
444
445	/**
446	* dwc2_update_urb_state() - Updates the state of the URB after a Transfer
447	* Complete interrupt on the host channel. Updates the actual_length field
448	* of the URB based on the number of bytes transferred via the host channel.
449	* Sets the URB status if the data transfer is finished.
450	*
451	* @hsotg: Programming view of the DWC_otg controller
452	* @chan: Programming view of host channel
453	* @chnum: Channel number
454	* @urb: Processing URB
455	* @qtd: Queue transfer descriptor
456	*
457	* Return: 1 if the data transfer specified by the URB is completely finished,
458	* 0 otherwise
459	*/
460	static int dwc2_update_urb_state(struct dwc2_hsotg *hsotg,
461	struct dwc2_host_chan *chan, int chnum,
462	struct dwc2_hcd_urb *urb,
463	struct dwc2_qtd *qtd)
464	{
465	u32 hctsiz;
466	int xfer_done = 0;
467	int short_read = 0;
468	int xfer_length = dwc2_get_actual_xfer_length(hsotg, chan, chnum, qtd,
469	halt_status: DWC2_HC_XFER_COMPLETE,
470	short_read: &short_read);
471
472	if (urb->actual_length + xfer_length > urb->length) {
473	dev_dbg(hsotg->dev, "%s(): trimming xfer length\n", __func__);
474	xfer_length = urb->length - urb->actual_length;
475	}
476
477	dev_vdbg(hsotg->dev, "urb->actual_length=%d xfer_length=%d\n",
478	urb->actual_length, xfer_length);
479	urb->actual_length += xfer_length;
480
481	if (xfer_length && chan->ep_type == USB_ENDPOINT_XFER_BULK &&
482	(urb->flags & URB_SEND_ZERO_PACKET) &&
483	urb->actual_length >= urb->length &&
484	!(urb->length % chan->max_packet)) {
485	xfer_done = 0;
486	} else if (short_read || urb->actual_length >= urb->length) {
487	xfer_done = 1;
488	urb->status = 0;
489	}
490
491	hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
492	dev_vdbg(hsotg->dev, "DWC_otg: %s: %s, channel %d\n",
493	__func__, (chan->ep_is_in ? "IN" : "OUT"), chnum);
494	dev_vdbg(hsotg->dev, " chan->xfer_len %d\n", chan->xfer_len);
495	dev_vdbg(hsotg->dev, " hctsiz.xfersize %d\n",
496	(hctsiz & TSIZ_XFERSIZE_MASK) >> TSIZ_XFERSIZE_SHIFT);
497	dev_vdbg(hsotg->dev, " urb->transfer_buffer_length %d\n", urb->length);
498	dev_vdbg(hsotg->dev, " urb->actual_length %d\n", urb->actual_length);
499	dev_vdbg(hsotg->dev, " short_read %d, xfer_done %d\n", short_read,
500	xfer_done);
501
502	return xfer_done;
503	}
504
505	/*
506	* Save the starting data toggle for the next transfer. The data toggle is
507	* saved in the QH for non-control transfers and it's saved in the QTD for
508	* control transfers.
509	*/
510	void dwc2_hcd_save_data_toggle(struct dwc2_hsotg *hsotg,
511	struct dwc2_host_chan *chan, int chnum,
512	struct dwc2_qtd *qtd)
513	{
514	u32 hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
515	u32 pid = (hctsiz & TSIZ_SC_MC_PID_MASK) >> TSIZ_SC_MC_PID_SHIFT;
516
517	if (chan->ep_type != USB_ENDPOINT_XFER_CONTROL) {
518	if (WARN(!chan || !chan->qh,
519	"chan->qh must be specified for non-control eps\n"))
520	return;
521
522	if (pid == TSIZ_SC_MC_PID_DATA0)
523	chan->qh->data_toggle = DWC2_HC_PID_DATA0;
524	else
525	chan->qh->data_toggle = DWC2_HC_PID_DATA1;
526	} else {
527	if (WARN(!qtd,
528	"qtd must be specified for control eps\n"))
529	return;
530
531	if (pid == TSIZ_SC_MC_PID_DATA0)
532	qtd->data_toggle = DWC2_HC_PID_DATA0;
533	else
534	qtd->data_toggle = DWC2_HC_PID_DATA1;
535	}
536	}
537
538	/**
539	* dwc2_update_isoc_urb_state() - Updates the state of an Isochronous URB when
540	* the transfer is stopped for any reason. The fields of the current entry in
541	* the frame descriptor array are set based on the transfer state and the input
542	* halt_status. Completes the Isochronous URB if all the URB frames have been
543	* completed.
544	*
545	* @hsotg: Programming view of the DWC_otg controller
546	* @chan: Programming view of host channel
547	* @chnum: Channel number
548	* @halt_status: Reason for halting a host channel
549	* @qtd: Queue transfer descriptor
550	*
551	* Return: DWC2_HC_XFER_COMPLETE if there are more frames remaining to be
552	* transferred in the URB. Otherwise return DWC2_HC_XFER_URB_COMPLETE.
553	*/
554	static enum dwc2_halt_status dwc2_update_isoc_urb_state(
555	struct dwc2_hsotg *hsotg, struct dwc2_host_chan *chan,
556	int chnum, struct dwc2_qtd *qtd,
557	enum dwc2_halt_status halt_status)
558	{
559	struct dwc2_hcd_iso_packet_desc *frame_desc;
560	struct dwc2_hcd_urb *urb = qtd->urb;
561
562	if (!urb)
563	return DWC2_HC_XFER_NO_HALT_STATUS;
564
565	frame_desc = &urb->iso_descs[qtd->isoc_frame_index];
566
567	switch (halt_status) {
568	case DWC2_HC_XFER_COMPLETE:
569	frame_desc->status = 0;
570	frame_desc->actual_length = dwc2_get_actual_xfer_length(hsotg,
571	chan, chnum, qtd, halt_status, NULL);
572	break;
573	case DWC2_HC_XFER_FRAME_OVERRUN:
574	urb->error_count++;
575	if (chan->ep_is_in)
576	frame_desc->status = -ENOSR;
577	else
578	frame_desc->status = -ECOMM;
579	frame_desc->actual_length = 0;
580	break;
581	case DWC2_HC_XFER_BABBLE_ERR:
582	urb->error_count++;
583	frame_desc->status = -EOVERFLOW;
584	/* Don't need to update actual_length in this case */
585	break;
586	case DWC2_HC_XFER_XACT_ERR:
587	urb->error_count++;
588	frame_desc->status = -EPROTO;
589	frame_desc->actual_length = dwc2_get_actual_xfer_length(hsotg,
590	chan, chnum, qtd, halt_status, NULL);
591
592	/* Skip whole frame */
593	if (chan->qh->do_split &&
594	chan->ep_type == USB_ENDPOINT_XFER_ISOC && chan->ep_is_in &&
595	hsotg->params.host_dma) {
596	qtd->complete_split = 0;
597	qtd->isoc_split_offset = 0;
598	}
599
600	break;
601	default:
602	dev_err(hsotg->dev, "Unhandled halt_status (%d)\n",
603	halt_status);
604	break;
605	}
606
607	if (++qtd->isoc_frame_index == urb->packet_count) {
608	/*
609	* urb->status is not used for isoc transfers. The individual
610	* frame_desc statuses are used instead.
611	*/
612	dwc2_host_complete(hsotg, qtd, status: 0);
613	halt_status = DWC2_HC_XFER_URB_COMPLETE;
614	} else {
615	halt_status = DWC2_HC_XFER_COMPLETE;
616	}
617
618	return halt_status;
619	}
620
621	/*
622	* Frees the first QTD in the QH's list if free_qtd is 1. For non-periodic
623	* QHs, removes the QH from the active non-periodic schedule. If any QTDs are
624	* still linked to the QH, the QH is added to the end of the inactive
625	* non-periodic schedule. For periodic QHs, removes the QH from the periodic
626	* schedule if no more QTDs are linked to the QH.
627	*/
628	static void dwc2_deactivate_qh(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,
629	int free_qtd)
630	{
631	int continue_split = 0;
632	struct dwc2_qtd *qtd;
633
634	if (dbg_qh(qh))
635	dev_vdbg(hsotg->dev, " %s(%p,%p,%d)\n", __func__,
636	hsotg, qh, free_qtd);
637
638	if (list_empty(head: &qh->qtd_list)) {
639	dev_dbg(hsotg->dev, "## QTD list empty ##\n");
640	goto no_qtd;
641	}
642
643	qtd = list_first_entry(&qh->qtd_list, struct dwc2_qtd, qtd_list_entry);
644
645	if (qtd->complete_split)
646	continue_split = 1;
647	else if (qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_MID ||
648	qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_END)
649	continue_split = 1;
650
651	if (free_qtd) {
652	dwc2_hcd_qtd_unlink_and_free(hsotg, qtd, qh);
653	continue_split = 0;
654	}
655
656	no_qtd:
657	qh->channel = NULL;
658	dwc2_hcd_qh_deactivate(hsotg, qh, sched_csplit: continue_split);
659	}
660
661	/**
662	* dwc2_release_channel() - Releases a host channel for use by other transfers
663	*
664	* @hsotg: The HCD state structure
665	* @chan: The host channel to release
666	* @qtd: The QTD associated with the host channel. This QTD may be
667	* freed if the transfer is complete or an error has occurred.
668	* @halt_status: Reason the channel is being released. This status
669	* determines the actions taken by this function.
670	*
671	* Also attempts to select and queue more transactions since at least one host
672	* channel is available.
673	*/
674	static void dwc2_release_channel(struct dwc2_hsotg *hsotg,
675	struct dwc2_host_chan *chan,
676	struct dwc2_qtd *qtd,
677	enum dwc2_halt_status halt_status)
678	{
679	enum dwc2_transaction_type tr_type;
680	u32 haintmsk;
681	int free_qtd = 0;
682
683	if (dbg_hc(hc: chan))
684	dev_vdbg(hsotg->dev, " %s: channel %d, halt_status %d\n",
685	__func__, chan->hc_num, halt_status);
686
687	switch (halt_status) {
688	case DWC2_HC_XFER_URB_COMPLETE:
689	free_qtd = 1;
690	break;
691	case DWC2_HC_XFER_AHB_ERR:
692	case DWC2_HC_XFER_STALL:
693	case DWC2_HC_XFER_BABBLE_ERR:
694	free_qtd = 1;
695	break;
696	case DWC2_HC_XFER_XACT_ERR:
697	if (qtd && qtd->error_count >= 3) {
698	dev_vdbg(hsotg->dev,
699	" Complete URB with transaction error\n");
700	free_qtd = 1;
701	dwc2_host_complete(hsotg, qtd, status: -EPROTO);
702	}
703	break;
704	case DWC2_HC_XFER_URB_DEQUEUE:
705	/*
706	* The QTD has already been removed and the QH has been
707	* deactivated. Don't want to do anything except release the
708	* host channel and try to queue more transfers.
709	*/
710	goto cleanup;
711	case DWC2_HC_XFER_PERIODIC_INCOMPLETE:
712	dev_vdbg(hsotg->dev, " Complete URB with I/O error\n");
713	free_qtd = 1;
714	dwc2_host_complete(hsotg, qtd, status: -EIO);
715	break;
716	case DWC2_HC_XFER_NO_HALT_STATUS:
717	default:
718	break;
719	}
720
721	dwc2_deactivate_qh(hsotg, qh: chan->qh, free_qtd);
722
723	cleanup:
724	/*
725	* Release the host channel for use by other transfers. The cleanup
726	* function clears the channel interrupt enables and conditions, so
727	* there's no need to clear the Channel Halted interrupt separately.
728	*/
729	if (!list_empty(head: &chan->hc_list_entry))
730	list_del(entry: &chan->hc_list_entry);
731	dwc2_hc_cleanup(hsotg, chan);
732	list_add_tail(new: &chan->hc_list_entry, head: &hsotg->free_hc_list);
733
734	if (hsotg->params.uframe_sched) {
735	hsotg->available_host_channels++;
736	} else {
737	switch (chan->ep_type) {
738	case USB_ENDPOINT_XFER_CONTROL:
739	case USB_ENDPOINT_XFER_BULK:
740	hsotg->non_periodic_channels--;
741	break;
742	default:
743	/*
744	* Don't release reservations for periodic channels
745	* here. That's done when a periodic transfer is
746	* descheduled (i.e. when the QH is removed from the
747	* periodic schedule).
748	*/
749	break;
750	}
751	}
752
753	haintmsk = dwc2_readl(hsotg, HAINTMSK);
754	haintmsk &= ~(1 << chan->hc_num);
755	dwc2_writel(hsotg, value: haintmsk, HAINTMSK);
756
757	/* Try to queue more transfers now that there's a free channel */
758	tr_type = dwc2_hcd_select_transactions(hsotg);
759	if (tr_type != DWC2_TRANSACTION_NONE)
760	dwc2_hcd_queue_transactions(hsotg, tr_type);
761	}
762
763	/*
764	* Halts a host channel. If the channel cannot be halted immediately because
765	* the request queue is full, this function ensures that the FIFO empty
766	* interrupt for the appropriate queue is enabled so that the halt request can
767	* be queued when there is space in the request queue.
768	*
769	* This function may also be called in DMA mode. In that case, the channel is
770	* simply released since the core always halts the channel automatically in
771	* DMA mode.
772	*/
773	static void dwc2_halt_channel(struct dwc2_hsotg *hsotg,
774	struct dwc2_host_chan *chan, struct dwc2_qtd *qtd,
775	enum dwc2_halt_status halt_status)
776	{
777	if (dbg_hc(hc: chan))
778	dev_vdbg(hsotg->dev, "%s()\n", __func__);
779
780	if (hsotg->params.host_dma) {
781	if (dbg_hc(hc: chan))
782	dev_vdbg(hsotg->dev, "DMA enabled\n");
783	dwc2_release_channel(hsotg, chan, qtd, halt_status);
784	return;
785	}
786
787	/* Slave mode processing */
788	dwc2_hc_halt(hsotg, chan, halt_status);
789
790	if (chan->halt_on_queue) {
791	u32 gintmsk;
792
793	dev_vdbg(hsotg->dev, "Halt on queue\n");
794	if (chan->ep_type == USB_ENDPOINT_XFER_CONTROL ||
795	chan->ep_type == USB_ENDPOINT_XFER_BULK) {
796	dev_vdbg(hsotg->dev, "control/bulk\n");
797	/*
798	* Make sure the Non-periodic Tx FIFO empty interrupt
799	* is enabled so that the non-periodic schedule will
800	* be processed
801	*/
802	gintmsk = dwc2_readl(hsotg, GINTMSK);
803	gintmsk |= GINTSTS_NPTXFEMP;
804	dwc2_writel(hsotg, value: gintmsk, GINTMSK);
805	} else {
806	dev_vdbg(hsotg->dev, "isoc/intr\n");
807	/*
808	* Move the QH from the periodic queued schedule to
809	* the periodic assigned schedule. This allows the
810	* halt to be queued when the periodic schedule is
811	* processed.
812	*/
813	list_move_tail(list: &chan->qh->qh_list_entry,
814	head: &hsotg->periodic_sched_assigned);
815
816	/*
817	* Make sure the Periodic Tx FIFO Empty interrupt is
818	* enabled so that the periodic schedule will be
819	* processed
820	*/
821	gintmsk = dwc2_readl(hsotg, GINTMSK);
822	gintmsk |= GINTSTS_PTXFEMP;
823	dwc2_writel(hsotg, value: gintmsk, GINTMSK);
824	}
825	}
826	}
827
828	/*
829	* Performs common cleanup for non-periodic transfers after a Transfer
830	* Complete interrupt. This function should be called after any endpoint type
831	* specific handling is finished to release the host channel.
832	*/
833	static void dwc2_complete_non_periodic_xfer(struct dwc2_hsotg *hsotg,
834	struct dwc2_host_chan *chan,
835	int chnum, struct dwc2_qtd *qtd,
836	enum dwc2_halt_status halt_status)
837	{
838	dev_vdbg(hsotg->dev, "%s()\n", __func__);
839
840	qtd->error_count = 0;
841
842	if (chan->hcint & HCINTMSK_NYET) {
843	/*
844	* Got a NYET on the last transaction of the transfer. This
845	* means that the endpoint should be in the PING state at the
846	* beginning of the next transfer.
847	*/
848	dev_vdbg(hsotg->dev, "got NYET\n");
849	chan->qh->ping_state = 1;
850	}
851
852	/*
853	* Always halt and release the host channel to make it available for
854	* more transfers. There may still be more phases for a control
855	* transfer or more data packets for a bulk transfer at this point,
856	* but the host channel is still halted. A channel will be reassigned
857	* to the transfer when the non-periodic schedule is processed after
858	* the channel is released. This allows transactions to be queued
859	* properly via dwc2_hcd_queue_transactions, which also enables the
860	* Tx FIFO Empty interrupt if necessary.
861	*/
862	if (chan->ep_is_in) {
863	/*
864	* IN transfers in Slave mode require an explicit disable to
865	* halt the channel. (In DMA mode, this call simply releases
866	* the channel.)
867	*/
868	dwc2_halt_channel(hsotg, chan, qtd, halt_status);
869	} else {
870	/*
871	* The channel is automatically disabled by the core for OUT
872	* transfers in Slave mode
873	*/
874	dwc2_release_channel(hsotg, chan, qtd, halt_status);
875	}
876	}
877
878	/*
879	* Performs common cleanup for periodic transfers after a Transfer Complete
880	* interrupt. This function should be called after any endpoint type specific
881	* handling is finished to release the host channel.
882	*/
883	static void dwc2_complete_periodic_xfer(struct dwc2_hsotg *hsotg,
884	struct dwc2_host_chan *chan, int chnum,
885	struct dwc2_qtd *qtd,
886	enum dwc2_halt_status halt_status)
887	{
888	u32 hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
889
890	qtd->error_count = 0;
891
892	if (!chan->ep_is_in || (hctsiz & TSIZ_PKTCNT_MASK) == 0)
893	/* Core halts channel in these cases */
894	dwc2_release_channel(hsotg, chan, qtd, halt_status);
895	else
896	/* Flush any outstanding requests from the Tx queue */
897	dwc2_halt_channel(hsotg, chan, qtd, halt_status);
898	}
899
900	static int dwc2_xfercomp_isoc_split_in(struct dwc2_hsotg *hsotg,
901	struct dwc2_host_chan *chan, int chnum,
902	struct dwc2_qtd *qtd)
903	{
904	struct dwc2_hcd_iso_packet_desc *frame_desc;
905	u32 len;
906	u32 hctsiz;
907	u32 pid;
908
909	if (!qtd->urb)
910	return 0;
911
912	frame_desc = &qtd->urb->iso_descs[qtd->isoc_frame_index];
913	len = dwc2_get_actual_xfer_length(hsotg, chan, chnum, qtd,
914	halt_status: DWC2_HC_XFER_COMPLETE, NULL);
915	if (!len && !qtd->isoc_split_offset) {
916	qtd->complete_split = 0;
917	return 0;
918	}
919
920	frame_desc->actual_length += len;
921
922	if (chan->align_buf) {
923	dev_vdbg(hsotg->dev, "non-aligned buffer\n");
924	dma_unmap_single(hsotg->dev, chan->qh->dw_align_buf_dma,
925	DWC2_KMEM_UNALIGNED_BUF_SIZE, DMA_FROM_DEVICE);
926	memcpy(qtd->urb->buf + (chan->xfer_dma - qtd->urb->dma),
927	chan->qh->dw_align_buf, len);
928	}
929
930	qtd->isoc_split_offset += len;
931
932	hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
933	pid = (hctsiz & TSIZ_SC_MC_PID_MASK) >> TSIZ_SC_MC_PID_SHIFT;
934
935	if (frame_desc->actual_length >= frame_desc->length || pid == 0) {
936	frame_desc->status = 0;
937	qtd->isoc_frame_index++;
938	qtd->complete_split = 0;
939	qtd->isoc_split_offset = 0;
940	}
941
942	if (qtd->isoc_frame_index == qtd->urb->packet_count) {
943	dwc2_host_complete(hsotg, qtd, status: 0);
944	dwc2_release_channel(hsotg, chan, qtd,
945	halt_status: DWC2_HC_XFER_URB_COMPLETE);
946	} else {
947	dwc2_release_channel(hsotg, chan, qtd,
948	halt_status: DWC2_HC_XFER_NO_HALT_STATUS);
949	}
950
951	return 1; /* Indicates that channel released */
952	}
953
954	/*
955	* Handles a host channel Transfer Complete interrupt. This handler may be
956	* called in either DMA mode or Slave mode.
957	*/
958	static void dwc2_hc_xfercomp_intr(struct dwc2_hsotg *hsotg,
959	struct dwc2_host_chan *chan, int chnum,
960	struct dwc2_qtd *qtd)
961	{
962	struct dwc2_hcd_urb *urb = qtd->urb;
963	enum dwc2_halt_status halt_status = DWC2_HC_XFER_COMPLETE;
964	int pipe_type;
965	int urb_xfer_done;
966
967	if (dbg_hc(hc: chan))
968	dev_vdbg(hsotg->dev,
969	"--Host Channel %d Interrupt: Transfer Complete--\n",
970	chnum);
971
972	if (!urb)
973	goto handle_xfercomp_done;
974
975	pipe_type = dwc2_hcd_get_pipe_type(pipe: &urb->pipe_info);
976
977	if (hsotg->params.dma_desc_enable) {
978	dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum, halt_status);
979	if (pipe_type == USB_ENDPOINT_XFER_ISOC)
980	/* Do not disable the interrupt, just clear it */
981	return;
982	goto handle_xfercomp_done;
983	}
984
985	/* Handle xfer complete on CSPLIT */
986	if (chan->qh->do_split) {
987	if (chan->ep_type == USB_ENDPOINT_XFER_ISOC && chan->ep_is_in &&
988	hsotg->params.host_dma) {
989	if (qtd->complete_split &&
990	dwc2_xfercomp_isoc_split_in(hsotg, chan, chnum,
991	qtd))
992	goto handle_xfercomp_done;
993	} else {
994	qtd->complete_split = 0;
995	}
996	}
997
998	/* Update the QTD and URB states */
999	switch (pipe_type) {
1000	case USB_ENDPOINT_XFER_CONTROL:
1001	switch (qtd->control_phase) {
1002	case DWC2_CONTROL_SETUP:
1003	if (urb->length > 0)
1004	qtd->control_phase = DWC2_CONTROL_DATA;
1005	else
1006	qtd->control_phase = DWC2_CONTROL_STATUS;
1007	dev_vdbg(hsotg->dev,
1008	" Control setup transaction done\n");
1009	halt_status = DWC2_HC_XFER_COMPLETE;
1010	break;
1011	case DWC2_CONTROL_DATA:
1012	urb_xfer_done = dwc2_update_urb_state(hsotg, chan,
1013	chnum, urb, qtd);
1014	if (urb_xfer_done) {
1015	qtd->control_phase = DWC2_CONTROL_STATUS;
1016	dev_vdbg(hsotg->dev,
1017	" Control data transfer done\n");
1018	} else {
1019	dwc2_hcd_save_data_toggle(hsotg, chan, chnum,
1020	qtd);
1021	}
1022	halt_status = DWC2_HC_XFER_COMPLETE;
1023	break;
1024	case DWC2_CONTROL_STATUS:
1025	dev_vdbg(hsotg->dev, " Control transfer complete\n");
1026	if (urb->status == -EINPROGRESS)
1027	urb->status = 0;
1028	dwc2_host_complete(hsotg, qtd, status: urb->status);
1029	halt_status = DWC2_HC_XFER_URB_COMPLETE;
1030	break;
1031	}
1032
1033	dwc2_complete_non_periodic_xfer(hsotg, chan, chnum, qtd,
1034	halt_status);
1035	break;
1036	case USB_ENDPOINT_XFER_BULK:
1037	dev_vdbg(hsotg->dev, " Bulk transfer complete\n");
1038	urb_xfer_done = dwc2_update_urb_state(hsotg, chan, chnum, urb,
1039	qtd);
1040	if (urb_xfer_done) {
1041	dwc2_host_complete(hsotg, qtd, status: urb->status);
1042	halt_status = DWC2_HC_XFER_URB_COMPLETE;
1043	} else {
1044	halt_status = DWC2_HC_XFER_COMPLETE;
1045	}
1046
1047	dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1048	dwc2_complete_non_periodic_xfer(hsotg, chan, chnum, qtd,
1049	halt_status);
1050	break;
1051	case USB_ENDPOINT_XFER_INT:
1052	dev_vdbg(hsotg->dev, " Interrupt transfer complete\n");
1053	urb_xfer_done = dwc2_update_urb_state(hsotg, chan, chnum, urb,
1054	qtd);
1055
1056	/*
1057	* Interrupt URB is done on the first transfer complete
1058	* interrupt
1059	*/
1060	if (urb_xfer_done) {
1061	dwc2_host_complete(hsotg, qtd, status: urb->status);
1062	halt_status = DWC2_HC_XFER_URB_COMPLETE;
1063	} else {
1064	halt_status = DWC2_HC_XFER_COMPLETE;
1065	}
1066
1067	dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1068	dwc2_complete_periodic_xfer(hsotg, chan, chnum, qtd,
1069	halt_status);
1070	break;
1071	case USB_ENDPOINT_XFER_ISOC:
1072	if (dbg_perio())
1073	dev_vdbg(hsotg->dev, " Isochronous transfer complete\n");
1074	if (qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_ALL)
1075	halt_status = dwc2_update_isoc_urb_state(hsotg, chan,
1076	chnum, qtd,
1077	halt_status: DWC2_HC_XFER_COMPLETE);
1078	dwc2_complete_periodic_xfer(hsotg, chan, chnum, qtd,
1079	halt_status);
1080	break;
1081	}
1082
1083	handle_xfercomp_done:
1084	disable_hc_int(hsotg, chnum, HCINTMSK_XFERCOMPL);
1085	}
1086
1087	/*
1088	* Handles a host channel STALL interrupt. This handler may be called in
1089	* either DMA mode or Slave mode.
1090	*/
1091	static void dwc2_hc_stall_intr(struct dwc2_hsotg *hsotg,
1092	struct dwc2_host_chan *chan, int chnum,
1093	struct dwc2_qtd *qtd)
1094	{
1095	struct dwc2_hcd_urb *urb = qtd->urb;
1096	int pipe_type;
1097
1098	dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: STALL Received--\n",
1099	chnum);
1100
1101	if (hsotg->params.dma_desc_enable) {
1102	dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1103	halt_status: DWC2_HC_XFER_STALL);
1104	goto handle_stall_done;
1105	}
1106
1107	if (!urb)
1108	goto handle_stall_halt;
1109
1110	pipe_type = dwc2_hcd_get_pipe_type(pipe: &urb->pipe_info);
1111
1112	if (pipe_type == USB_ENDPOINT_XFER_CONTROL)
1113	dwc2_host_complete(hsotg, qtd, status: -EPIPE);
1114
1115	if (pipe_type == USB_ENDPOINT_XFER_BULK ||
1116	pipe_type == USB_ENDPOINT_XFER_INT) {
1117	dwc2_host_complete(hsotg, qtd, status: -EPIPE);
1118	/*
1119	* USB protocol requires resetting the data toggle for bulk
1120	* and interrupt endpoints when a CLEAR_FEATURE(ENDPOINT_HALT)
1121	* setup command is issued to the endpoint. Anticipate the
1122	* CLEAR_FEATURE command since a STALL has occurred and reset
1123	* the data toggle now.
1124	*/
1125	chan->qh->data_toggle = 0;
1126	}
1127
1128	handle_stall_halt:
1129	dwc2_halt_channel(hsotg, chan, qtd, halt_status: DWC2_HC_XFER_STALL);
1130
1131	handle_stall_done:
1132	disable_hc_int(hsotg, chnum, HCINTMSK_STALL);
1133	}
1134
1135	/*
1136	* Updates the state of the URB when a transfer has been stopped due to an
1137	* abnormal condition before the transfer completes. Modifies the
1138	* actual_length field of the URB to reflect the number of bytes that have
1139	* actually been transferred via the host channel.
1140	*/
1141	static void dwc2_update_urb_state_abn(struct dwc2_hsotg *hsotg,
1142	struct dwc2_host_chan *chan, int chnum,
1143	struct dwc2_hcd_urb *urb,
1144	struct dwc2_qtd *qtd,
1145	enum dwc2_halt_status halt_status)
1146	{
1147	u32 xfer_length = dwc2_get_actual_xfer_length(hsotg, chan, chnum,
1148	qtd, halt_status, NULL);
1149	u32 hctsiz;
1150
1151	if (urb->actual_length + xfer_length > urb->length) {
1152	dev_warn(hsotg->dev, "%s(): trimming xfer length\n", __func__);
1153	xfer_length = urb->length - urb->actual_length;
1154	}
1155
1156	urb->actual_length += xfer_length;
1157
1158	hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
1159	dev_vdbg(hsotg->dev, "DWC_otg: %s: %s, channel %d\n",
1160	__func__, (chan->ep_is_in ? "IN" : "OUT"), chnum);
1161	dev_vdbg(hsotg->dev, " chan->start_pkt_count %d\n",
1162	chan->start_pkt_count);
1163	dev_vdbg(hsotg->dev, " hctsiz.pktcnt %d\n",
1164	(hctsiz & TSIZ_PKTCNT_MASK) >> TSIZ_PKTCNT_SHIFT);
1165	dev_vdbg(hsotg->dev, " chan->max_packet %d\n", chan->max_packet);
1166	dev_vdbg(hsotg->dev, " bytes_transferred %d\n",
1167	xfer_length);
1168	dev_vdbg(hsotg->dev, " urb->actual_length %d\n",
1169	urb->actual_length);
1170	dev_vdbg(hsotg->dev, " urb->transfer_buffer_length %d\n",
1171	urb->length);
1172	}
1173
1174	/*
1175	* Handles a host channel NAK interrupt. This handler may be called in either
1176	* DMA mode or Slave mode.
1177	*/
1178	static void dwc2_hc_nak_intr(struct dwc2_hsotg *hsotg,
1179	struct dwc2_host_chan *chan, int chnum,
1180	struct dwc2_qtd *qtd)
1181	{
1182	if (!qtd) {
1183	dev_dbg(hsotg->dev, "%s: qtd is NULL\n", __func__);
1184	return;
1185	}
1186
1187	if (!qtd->urb) {
1188	dev_dbg(hsotg->dev, "%s: qtd->urb is NULL\n", __func__);
1189	return;
1190	}
1191
1192	if (dbg_hc(hc: chan))
1193	dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: NAK Received--\n",
1194	chnum);
1195
1196	/*
1197	* Handle NAK for IN/OUT SSPLIT/CSPLIT transfers, bulk, control, and
1198	* interrupt. Re-start the SSPLIT transfer.
1199	*
1200	* Normally for non-periodic transfers we'll retry right away, but to
1201	* avoid interrupt storms we'll wait before retrying if we've got
1202	* several NAKs. If we didn't do this we'd retry directly from the
1203	* interrupt handler and could end up quickly getting another
1204	* interrupt (another NAK), which we'd retry. Note that we do not
1205	* delay retries for IN parts of control requests, as those are expected
1206	* to complete fairly quickly, and if we delay them we risk confusing
1207	* the device and cause it issue STALL.
1208	*
1209	* Note that in DMA mode software only gets involved to re-send NAKed
1210	* transfers for split transactions, so we only need to apply this
1211	* delaying logic when handling splits. In non-DMA mode presumably we
1212	* might want a similar delay if someone can demonstrate this problem
1213	* affects that code path too.
1214	*/
1215	if (chan->do_split) {
1216	if (chan->complete_split)
1217	qtd->error_count = 0;
1218	qtd->complete_split = 0;
1219	qtd->num_naks++;
1220	qtd->qh->want_wait = qtd->num_naks >= DWC2_NAKS_BEFORE_DELAY &&
1221	!(chan->ep_type == USB_ENDPOINT_XFER_CONTROL &&
1222	chan->ep_is_in);
1223	dwc2_halt_channel(hsotg, chan, qtd, halt_status: DWC2_HC_XFER_NAK);
1224	goto handle_nak_done;
1225	}
1226
1227	switch (dwc2_hcd_get_pipe_type(pipe: &qtd->urb->pipe_info)) {
1228	case USB_ENDPOINT_XFER_CONTROL:
1229	case USB_ENDPOINT_XFER_BULK:
1230	if (hsotg->params.host_dma && chan->ep_is_in) {
1231	/*
1232	* NAK interrupts are enabled on bulk/control IN
1233	* transfers in DMA mode for the sole purpose of
1234	* resetting the error count after a transaction error
1235	* occurs. The core will continue transferring data.
1236	*/
1237	qtd->error_count = 0;
1238	break;
1239	}
1240
1241	/*
1242	* NAK interrupts normally occur during OUT transfers in DMA
1243	* or Slave mode. For IN transfers, more requests will be
1244	* queued as request queue space is available.
1245	*/
1246	qtd->error_count = 0;
1247
1248	if (!chan->qh->ping_state) {
1249	dwc2_update_urb_state_abn(hsotg, chan, chnum, urb: qtd->urb,
1250	qtd, halt_status: DWC2_HC_XFER_NAK);
1251	dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1252
1253	if (chan->speed == USB_SPEED_HIGH)
1254	chan->qh->ping_state = 1;
1255	}
1256
1257	/*
1258	* Halt the channel so the transfer can be re-started from
1259	* the appropriate point or the PING protocol will
1260	* start/continue
1261	*/
1262	dwc2_halt_channel(hsotg, chan, qtd, halt_status: DWC2_HC_XFER_NAK);
1263	break;
1264	case USB_ENDPOINT_XFER_INT:
1265	qtd->error_count = 0;
1266	dwc2_halt_channel(hsotg, chan, qtd, halt_status: DWC2_HC_XFER_NAK);
1267	break;
1268	case USB_ENDPOINT_XFER_ISOC:
1269	/* Should never get called for isochronous transfers */
1270	dev_err(hsotg->dev, "NACK interrupt for ISOC transfer\n");
1271	break;
1272	}
1273
1274	handle_nak_done:
1275	disable_hc_int(hsotg, chnum, HCINTMSK_NAK);
1276	}
1277
1278	/*
1279	* Handles a host channel ACK interrupt. This interrupt is enabled when
1280	* performing the PING protocol in Slave mode, when errors occur during
1281	* either Slave mode or DMA mode, and during Start Split transactions.
1282	*/
1283	static void dwc2_hc_ack_intr(struct dwc2_hsotg *hsotg,
1284	struct dwc2_host_chan *chan, int chnum,
1285	struct dwc2_qtd *qtd)
1286	{
1287	struct dwc2_hcd_iso_packet_desc *frame_desc;
1288
1289	if (dbg_hc(hc: chan))
1290	dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: ACK Received--\n",
1291	chnum);
1292
1293	if (chan->do_split) {
1294	/* Handle ACK on SSPLIT. ACK should not occur in CSPLIT. */
1295	if (!chan->ep_is_in &&
1296	chan->data_pid_start != DWC2_HC_PID_SETUP)
1297	qtd->ssplit_out_xfer_count = chan->xfer_len;
1298
1299	if (chan->ep_type != USB_ENDPOINT_XFER_ISOC || chan->ep_is_in) {
1300	qtd->complete_split = 1;
1301	dwc2_halt_channel(hsotg, chan, qtd, halt_status: DWC2_HC_XFER_ACK);
1302	} else {
1303	/* ISOC OUT */
1304	switch (chan->xact_pos) {
1305	case DWC2_HCSPLT_XACTPOS_ALL:
1306	break;
1307	case DWC2_HCSPLT_XACTPOS_END:
1308	qtd->isoc_split_pos = DWC2_HCSPLT_XACTPOS_ALL;
1309	qtd->isoc_split_offset = 0;
1310	break;
1311	case DWC2_HCSPLT_XACTPOS_BEGIN:
1312	case DWC2_HCSPLT_XACTPOS_MID:
1313	/*
1314	* For BEGIN or MID, calculate the length for
1315	* the next microframe to determine the correct
1316	* SSPLIT token, either MID or END
1317	*/
1318	frame_desc = &qtd->urb->iso_descs[
1319	qtd->isoc_frame_index];
1320	qtd->isoc_split_offset += 188;
1321
1322	if (frame_desc->length - qtd->isoc_split_offset
1323	<= 188)
1324	qtd->isoc_split_pos =
1325	DWC2_HCSPLT_XACTPOS_END;
1326	else
1327	qtd->isoc_split_pos =
1328	DWC2_HCSPLT_XACTPOS_MID;
1329	break;
1330	}
1331	}
1332	} else {
1333	qtd->error_count = 0;
1334
1335	if (chan->qh->ping_state) {
1336	chan->qh->ping_state = 0;
1337	/*
1338	* Halt the channel so the transfer can be re-started
1339	* from the appropriate point. This only happens in
1340	* Slave mode. In DMA mode, the ping_state is cleared
1341	* when the transfer is started because the core
1342	* automatically executes the PING, then the transfer.
1343	*/
1344	dwc2_halt_channel(hsotg, chan, qtd, halt_status: DWC2_HC_XFER_ACK);
1345	}
1346	}
1347
1348	/*
1349	* If the ACK occurred when _not_ in the PING state, let the channel
1350	* continue transferring data after clearing the error count
1351	*/
1352	disable_hc_int(hsotg, chnum, HCINTMSK_ACK);
1353	}
1354
1355	/*
1356	* Handles a host channel NYET interrupt. This interrupt should only occur on
1357	* Bulk and Control OUT endpoints and for complete split transactions. If a
1358	* NYET occurs at the same time as a Transfer Complete interrupt, it is
1359	* handled in the xfercomp interrupt handler, not here. This handler may be
1360	* called in either DMA mode or Slave mode.
1361	*/
1362	static void dwc2_hc_nyet_intr(struct dwc2_hsotg *hsotg,
1363	struct dwc2_host_chan *chan, int chnum,
1364	struct dwc2_qtd *qtd)
1365	{
1366	if (dbg_hc(hc: chan))
1367	dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: NYET Received--\n",
1368	chnum);
1369
1370	/*
1371	* NYET on CSPLIT
1372	* re-do the CSPLIT immediately on non-periodic
1373	*/
1374	if (chan->do_split && chan->complete_split) {
1375	if (chan->ep_is_in && chan->ep_type == USB_ENDPOINT_XFER_ISOC &&
1376	hsotg->params.host_dma) {
1377	qtd->complete_split = 0;
1378	qtd->isoc_split_offset = 0;
1379	qtd->isoc_frame_index++;
1380	if (qtd->urb &&
1381	qtd->isoc_frame_index == qtd->urb->packet_count) {
1382	dwc2_host_complete(hsotg, qtd, status: 0);
1383	dwc2_release_channel(hsotg, chan, qtd,
1384	halt_status: DWC2_HC_XFER_URB_COMPLETE);
1385	} else {
1386	dwc2_release_channel(hsotg, chan, qtd,
1387	halt_status: DWC2_HC_XFER_NO_HALT_STATUS);
1388	}
1389	goto handle_nyet_done;
1390	}
1391
1392	if (chan->ep_type == USB_ENDPOINT_XFER_INT ||
1393	chan->ep_type == USB_ENDPOINT_XFER_ISOC) {
1394	struct dwc2_qh *qh = chan->qh;
1395	bool past_end;
1396
1397	if (!hsotg->params.uframe_sched) {
1398	int frnum = dwc2_hcd_get_frame_number(hsotg);
1399
1400	/* Don't have num_hs_transfers; simple logic */
1401	past_end = dwc2_full_frame_num(frame: frnum) !=
1402	dwc2_full_frame_num(frame: qh->next_active_frame);
1403	} else {
1404	int end_frnum;
1405
1406	/*
1407	* Figure out the end frame based on
1408	* schedule.
1409	*
1410	* We don't want to go on trying again
1411	* and again forever. Let's stop when
1412	* we've done all the transfers that
1413	* were scheduled.
1414	*
1415	* We're going to be comparing
1416	* start_active_frame and
1417	* next_active_frame, both of which
1418	* are 1 before the time the packet
1419	* goes on the wire, so that cancels
1420	* out. Basically if had 1 transfer
1421	* and we saw 1 NYET then we're done.
1422	* We're getting a NYET here so if
1423	* next >= (start + num_transfers)
1424	* we're done. The complexity is that
1425	* for all but ISOC_OUT we skip one
1426	* slot.
1427	*/
1428	end_frnum = dwc2_frame_num_inc(
1429	frame: qh->start_active_frame,
1430	inc: qh->num_hs_transfers);
1431
1432	if (qh->ep_type != USB_ENDPOINT_XFER_ISOC ||
1433	qh->ep_is_in)
1434	end_frnum =
1435	dwc2_frame_num_inc(frame: end_frnum, inc: 1);
1436
1437	past_end = dwc2_frame_num_le(
1438	frame1: end_frnum, frame2: qh->next_active_frame);
1439	}
1440
1441	if (past_end) {
1442	/* Treat this as a transaction error. */
1443	#if 0
1444	/*
1445	* Todo: Fix system performance so this can
1446	* be treated as an error. Right now complete
1447	* splits cannot be scheduled precisely enough
1448	* due to other system activity, so this error
1449	* occurs regularly in Slave mode.
1450	*/
1451	qtd->error_count++;
1452	#endif
1453	qtd->complete_split = 0;
1454	dwc2_halt_channel(hsotg, chan, qtd,
1455	halt_status: DWC2_HC_XFER_XACT_ERR);
1456	/* Todo: add support for isoc release */
1457	goto handle_nyet_done;
1458	}
1459	}
1460
1461	dwc2_halt_channel(hsotg, chan, qtd, halt_status: DWC2_HC_XFER_NYET);
1462	goto handle_nyet_done;
1463	}
1464
1465	chan->qh->ping_state = 1;
1466	qtd->error_count = 0;
1467
1468	dwc2_update_urb_state_abn(hsotg, chan, chnum, urb: qtd->urb, qtd,
1469	halt_status: DWC2_HC_XFER_NYET);
1470	dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1471
1472	/*
1473	* Halt the channel and re-start the transfer so the PING protocol
1474	* will start
1475	*/
1476	dwc2_halt_channel(hsotg, chan, qtd, halt_status: DWC2_HC_XFER_NYET);
1477
1478	handle_nyet_done:
1479	disable_hc_int(hsotg, chnum, HCINTMSK_NYET);
1480	}
1481
1482	/*
1483	* Handles a host channel babble interrupt. This handler may be called in
1484	* either DMA mode or Slave mode.
1485	*/
1486	static void dwc2_hc_babble_intr(struct dwc2_hsotg *hsotg,
1487	struct dwc2_host_chan *chan, int chnum,
1488	struct dwc2_qtd *qtd)
1489	{
1490	dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: Babble Error--\n",
1491	chnum);
1492
1493	dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1494
1495	if (hsotg->params.dma_desc_enable) {
1496	dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1497	halt_status: DWC2_HC_XFER_BABBLE_ERR);
1498	goto disable_int;
1499	}
1500
1501	if (chan->ep_type != USB_ENDPOINT_XFER_ISOC) {
1502	dwc2_host_complete(hsotg, qtd, status: -EOVERFLOW);
1503	dwc2_halt_channel(hsotg, chan, qtd, halt_status: DWC2_HC_XFER_BABBLE_ERR);
1504	} else {
1505	enum dwc2_halt_status halt_status;
1506
1507	halt_status = dwc2_update_isoc_urb_state(hsotg, chan, chnum,
1508	qtd, halt_status: DWC2_HC_XFER_BABBLE_ERR);
1509	dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1510	}
1511
1512	disable_int:
1513	disable_hc_int(hsotg, chnum, HCINTMSK_BBLERR);
1514	}
1515
1516	/*
1517	* Handles a host channel AHB error interrupt. This handler is only called in
1518	* DMA mode.
1519	*/
1520	static void dwc2_hc_ahberr_intr(struct dwc2_hsotg *hsotg,
1521	struct dwc2_host_chan *chan, int chnum,
1522	struct dwc2_qtd *qtd)
1523	{
1524	struct dwc2_hcd_urb *urb = qtd->urb;
1525	char *pipetype, *speed;
1526	u32 hcchar;
1527	u32 hcsplt;
1528	u32 hctsiz;
1529	u32 hc_dma;
1530
1531	dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: AHB Error--\n",
1532	chnum);
1533
1534	if (!urb)
1535	goto handle_ahberr_halt;
1536
1537	dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1538
1539	hcchar = dwc2_readl(hsotg, HCCHAR(chnum));
1540	hcsplt = dwc2_readl(hsotg, HCSPLT(chnum));
1541	hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
1542	hc_dma = dwc2_readl(hsotg, HCDMA(chnum));
1543
1544	dev_err(hsotg->dev, "AHB ERROR, Channel %d\n", chnum);
1545	dev_err(hsotg->dev, " hcchar 0x%08x, hcsplt 0x%08x\n", hcchar, hcsplt);
1546	dev_err(hsotg->dev, " hctsiz 0x%08x, hc_dma 0x%08x\n", hctsiz, hc_dma);
1547	dev_err(hsotg->dev, " Device address: %d\n",
1548	dwc2_hcd_get_dev_addr(&urb->pipe_info));
1549	dev_err(hsotg->dev, " Endpoint: %d, %s\n",
1550	dwc2_hcd_get_ep_num(&urb->pipe_info),
1551	dwc2_hcd_is_pipe_in(&urb->pipe_info) ? "IN" : "OUT");
1552
1553	switch (dwc2_hcd_get_pipe_type(pipe: &urb->pipe_info)) {
1554	case USB_ENDPOINT_XFER_CONTROL:
1555	pipetype = "CONTROL";
1556	break;
1557	case USB_ENDPOINT_XFER_BULK:
1558	pipetype = "BULK";
1559	break;
1560	case USB_ENDPOINT_XFER_INT:
1561	pipetype = "INTERRUPT";
1562	break;
1563	case USB_ENDPOINT_XFER_ISOC:
1564	pipetype = "ISOCHRONOUS";
1565	break;
1566	default:
1567	pipetype = "UNKNOWN";
1568	break;
1569	}
1570
1571	dev_err(hsotg->dev, " Endpoint type: %s\n", pipetype);
1572
1573	switch (chan->speed) {
1574	case USB_SPEED_HIGH:
1575	speed = "HIGH";
1576	break;
1577	case USB_SPEED_FULL:
1578	speed = "FULL";
1579	break;
1580	case USB_SPEED_LOW:
1581	speed = "LOW";
1582	break;
1583	default:
1584	speed = "UNKNOWN";
1585	break;
1586	}
1587
1588	dev_err(hsotg->dev, " Speed: %s\n", speed);
1589
1590	dev_err(hsotg->dev, " Max packet size: %d (mult %d)\n",
1591	dwc2_hcd_get_maxp(&urb->pipe_info),
1592	dwc2_hcd_get_maxp_mult(&urb->pipe_info));
1593	dev_err(hsotg->dev, " Data buffer length: %d\n", urb->length);
1594	dev_err(hsotg->dev, " Transfer buffer: %p, Transfer DMA: %08lx\n",
1595	urb->buf, (unsigned long)urb->dma);
1596	dev_err(hsotg->dev, " Setup buffer: %p, Setup DMA: %08lx\n",
1597	urb->setup_packet, (unsigned long)urb->setup_dma);
1598	dev_err(hsotg->dev, " Interval: %d\n", urb->interval);
1599
1600	/* Core halts the channel for Descriptor DMA mode */
1601	if (hsotg->params.dma_desc_enable) {
1602	dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1603	halt_status: DWC2_HC_XFER_AHB_ERR);
1604	goto handle_ahberr_done;
1605	}
1606
1607	dwc2_host_complete(hsotg, qtd, status: -EIO);
1608
1609	handle_ahberr_halt:
1610	/*
1611	* Force a channel halt. Don't call dwc2_halt_channel because that won't
1612	* write to the HCCHARn register in DMA mode to force the halt.
1613	*/
1614	dwc2_hc_halt(hsotg, chan, halt_status: DWC2_HC_XFER_AHB_ERR);
1615
1616	handle_ahberr_done:
1617	disable_hc_int(hsotg, chnum, HCINTMSK_AHBERR);
1618	}
1619
1620	/*
1621	* Handles a host channel transaction error interrupt. This handler may be
1622	* called in either DMA mode or Slave mode.
1623	*/
1624	static void dwc2_hc_xacterr_intr(struct dwc2_hsotg *hsotg,
1625	struct dwc2_host_chan *chan, int chnum,
1626	struct dwc2_qtd *qtd)
1627	{
1628	dev_dbg(hsotg->dev,
1629	"--Host Channel %d Interrupt: Transaction Error--\n", chnum);
1630
1631	dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1632
1633	if (hsotg->params.dma_desc_enable) {
1634	dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1635	halt_status: DWC2_HC_XFER_XACT_ERR);
1636	goto handle_xacterr_done;
1637	}
1638
1639	switch (dwc2_hcd_get_pipe_type(pipe: &qtd->urb->pipe_info)) {
1640	case USB_ENDPOINT_XFER_CONTROL:
1641	case USB_ENDPOINT_XFER_BULK:
1642	qtd->error_count++;
1643	if (!chan->qh->ping_state) {
1644	dwc2_update_urb_state_abn(hsotg, chan, chnum, urb: qtd->urb,
1645	qtd, halt_status: DWC2_HC_XFER_XACT_ERR);
1646	dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1647	if (!chan->ep_is_in && chan->speed == USB_SPEED_HIGH)
1648	chan->qh->ping_state = 1;
1649	}
1650
1651	/*
1652	* Halt the channel so the transfer can be re-started from
1653	* the appropriate point or the PING protocol will start
1654	*/
1655	dwc2_halt_channel(hsotg, chan, qtd, halt_status: DWC2_HC_XFER_XACT_ERR);
1656	break;
1657	case USB_ENDPOINT_XFER_INT:
1658	qtd->error_count++;
1659	if (chan->do_split && chan->complete_split)
1660	qtd->complete_split = 0;
1661	dwc2_halt_channel(hsotg, chan, qtd, halt_status: DWC2_HC_XFER_XACT_ERR);
1662	break;
1663	case USB_ENDPOINT_XFER_ISOC:
1664	{
1665	enum dwc2_halt_status halt_status;
1666
1667	halt_status = dwc2_update_isoc_urb_state(hsotg, chan,
1668	chnum, qtd, halt_status: DWC2_HC_XFER_XACT_ERR);
1669	dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1670	}
1671	break;
1672	}
1673
1674	handle_xacterr_done:
1675	disable_hc_int(hsotg, chnum, HCINTMSK_XACTERR);
1676	}
1677
1678	/*
1679	* Handles a host channel frame overrun interrupt. This handler may be called
1680	* in either DMA mode or Slave mode.
1681	*/
1682	static void dwc2_hc_frmovrun_intr(struct dwc2_hsotg *hsotg,
1683	struct dwc2_host_chan *chan, int chnum,
1684	struct dwc2_qtd *qtd)
1685	{
1686	enum dwc2_halt_status halt_status;
1687
1688	if (dbg_hc(hc: chan))
1689	dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: Frame Overrun--\n",
1690	chnum);
1691
1692	dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1693
1694	switch (dwc2_hcd_get_pipe_type(pipe: &qtd->urb->pipe_info)) {
1695	case USB_ENDPOINT_XFER_CONTROL:
1696	case USB_ENDPOINT_XFER_BULK:
1697	break;
1698	case USB_ENDPOINT_XFER_INT:
1699	dwc2_halt_channel(hsotg, chan, qtd, halt_status: DWC2_HC_XFER_FRAME_OVERRUN);
1700	break;
1701	case USB_ENDPOINT_XFER_ISOC:
1702	halt_status = dwc2_update_isoc_urb_state(hsotg, chan, chnum,
1703	qtd, halt_status: DWC2_HC_XFER_FRAME_OVERRUN);
1704	dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1705	break;
1706	}
1707
1708	disable_hc_int(hsotg, chnum, HCINTMSK_FRMOVRUN);
1709	}
1710
1711	/*
1712	* Handles a host channel data toggle error interrupt. This handler may be
1713	* called in either DMA mode or Slave mode.
1714	*/
1715	static void dwc2_hc_datatglerr_intr(struct dwc2_hsotg *hsotg,
1716	struct dwc2_host_chan *chan, int chnum,
1717	struct dwc2_qtd *qtd)
1718	{
1719	dev_dbg(hsotg->dev,
1720	"--Host Channel %d Interrupt: Data Toggle Error--\n", chnum);
1721
1722	if (chan->ep_is_in)
1723	qtd->error_count = 0;
1724	else
1725	dev_err(hsotg->dev,
1726	"Data Toggle Error on OUT transfer, channel %d\n",
1727	chnum);
1728
1729	dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1730	disable_hc_int(hsotg, chnum, HCINTMSK_DATATGLERR);
1731	}
1732
1733	/*
1734	* For debug only. It checks that a valid halt status is set and that
1735	* HCCHARn.chdis is clear. If there's a problem, corrective action is
1736	* taken and a warning is issued.
1737	*
1738	* Return: true if halt status is ok, false otherwise
1739	*/
1740	static bool dwc2_halt_status_ok(struct dwc2_hsotg *hsotg,
1741	struct dwc2_host_chan *chan, int chnum,
1742	struct dwc2_qtd *qtd)
1743	{
1744	#ifdef DEBUG
1745	u32 hcchar;
1746	u32 hctsiz;
1747	u32 hcintmsk;
1748	u32 hcsplt;
1749
1750	if (chan->halt_status == DWC2_HC_XFER_NO_HALT_STATUS) {
1751	/*
1752	* This code is here only as a check. This condition should
1753	* never happen. Ignore the halt if it does occur.
1754	*/
1755	hcchar = dwc2_readl(hsotg, HCCHAR(chnum));
1756	hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
1757	hcintmsk = dwc2_readl(hsotg, HCINTMSK(chnum));
1758	hcsplt = dwc2_readl(hsotg, HCSPLT(chnum));
1759	dev_dbg(hsotg->dev,
1760	"%s: chan->halt_status DWC2_HC_XFER_NO_HALT_STATUS,\n",
1761	__func__);
1762	dev_dbg(hsotg->dev,
1763	"channel %d, hcchar 0x%08x, hctsiz 0x%08x,\n",
1764	chnum, hcchar, hctsiz);
1765	dev_dbg(hsotg->dev,
1766	"hcint 0x%08x, hcintmsk 0x%08x, hcsplt 0x%08x,\n",
1767	chan->hcint, hcintmsk, hcsplt);
1768	if (qtd)
1769	dev_dbg(hsotg->dev, "qtd->complete_split %d\n",
1770	qtd->complete_split);
1771	dev_warn(hsotg->dev,
1772	"%s: no halt status, channel %d, ignoring interrupt\n",
1773	__func__, chnum);
1774	return false;
1775	}
1776
1777	/*
1778	* This code is here only as a check. hcchar.chdis should never be set
1779	* when the halt interrupt occurs. Halt the channel again if it does
1780	* occur.
1781	*/
1782	hcchar = dwc2_readl(hsotg, HCCHAR(chnum));
1783	if (hcchar & HCCHAR_CHDIS) {
1784	dev_warn(hsotg->dev,
1785	"%s: hcchar.chdis set unexpectedly, hcchar 0x%08x, trying to halt again\n",
1786	__func__, hcchar);
1787	chan->halt_pending = 0;
1788	dwc2_halt_channel(hsotg, chan, qtd, halt_status: chan->halt_status);
1789	return false;
1790	}
1791	#endif
1792
1793	return true;
1794	}
1795
1796	/*
1797	* Handles a host Channel Halted interrupt in DMA mode. This handler
1798	* determines the reason the channel halted and proceeds accordingly.
1799	*/
1800	static void dwc2_hc_chhltd_intr_dma(struct dwc2_hsotg *hsotg,
1801	struct dwc2_host_chan *chan, int chnum,
1802	struct dwc2_qtd *qtd)
1803	{
1804	u32 hcintmsk;
1805	int out_nak_enh = 0;
1806
1807	if (dbg_hc(hc: chan))
1808	dev_vdbg(hsotg->dev,
1809	"--Host Channel %d Interrupt: DMA Channel Halted--\n",
1810	chnum);
1811
1812	/*
1813	* For core with OUT NAK enhancement, the flow for high-speed
1814	* CONTROL/BULK OUT is handled a little differently
1815	*/
1816	if (hsotg->hw_params.snpsid >= DWC2_CORE_REV_2_71a) {
1817	if (chan->speed == USB_SPEED_HIGH && !chan->ep_is_in &&
1818	(chan->ep_type == USB_ENDPOINT_XFER_CONTROL ||
1819	chan->ep_type == USB_ENDPOINT_XFER_BULK)) {
1820	out_nak_enh = 1;
1821	}
1822	}
1823
1824	if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE ||
1825	(chan->halt_status == DWC2_HC_XFER_AHB_ERR &&
1826	!hsotg->params.dma_desc_enable)) {
1827	if (hsotg->params.dma_desc_enable)
1828	dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1829	halt_status: chan->halt_status);
1830	else
1831	/*
1832	* Just release the channel. A dequeue can happen on a
1833	* transfer timeout. In the case of an AHB Error, the
1834	* channel was forced to halt because there's no way to
1835	* gracefully recover.
1836	*/
1837	dwc2_release_channel(hsotg, chan, qtd,
1838	halt_status: chan->halt_status);
1839	return;
1840	}
1841
1842	hcintmsk = dwc2_readl(hsotg, HCINTMSK(chnum));
1843
1844	if (chan->hcint & HCINTMSK_XFERCOMPL) {
1845	/*
1846	* Todo: This is here because of a possible hardware bug. Spec
1847	* says that on SPLIT-ISOC OUT transfers in DMA mode that a HALT
1848	* interrupt w/ACK bit set should occur, but I only see the
1849	* XFERCOMP bit, even with it masked out. This is a workaround
1850	* for that behavior. Should fix this when hardware is fixed.
1851	*/
1852	if (chan->ep_type == USB_ENDPOINT_XFER_ISOC && !chan->ep_is_in)
1853	dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
1854	dwc2_hc_xfercomp_intr(hsotg, chan, chnum, qtd);
1855	} else if (chan->hcint & HCINTMSK_STALL) {
1856	dwc2_hc_stall_intr(hsotg, chan, chnum, qtd);
1857	} else if ((chan->hcint & HCINTMSK_XACTERR) &&
1858	!hsotg->params.dma_desc_enable) {
1859	if (out_nak_enh) {
1860	if (chan->hcint &
1861	(HCINTMSK_NYET | HCINTMSK_NAK | HCINTMSK_ACK)) {
1862	dev_vdbg(hsotg->dev,
1863	"XactErr with NYET/NAK/ACK\n");
1864	qtd->error_count = 0;
1865	} else {
1866	dev_vdbg(hsotg->dev,
1867	"XactErr without NYET/NAK/ACK\n");
1868	}
1869	}
1870
1871	/*
1872	* Must handle xacterr before nak or ack. Could get a xacterr
1873	* at the same time as either of these on a BULK/CONTROL OUT
1874	* that started with a PING. The xacterr takes precedence.
1875	*/
1876	dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
1877	} else if ((chan->hcint & HCINTMSK_XCS_XACT) &&
1878	hsotg->params.dma_desc_enable) {
1879	dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
1880	} else if ((chan->hcint & HCINTMSK_AHBERR) &&
1881	hsotg->params.dma_desc_enable) {
1882	dwc2_hc_ahberr_intr(hsotg, chan, chnum, qtd);
1883	} else if (chan->hcint & HCINTMSK_BBLERR) {
1884	dwc2_hc_babble_intr(hsotg, chan, chnum, qtd);
1885	} else if (chan->hcint & HCINTMSK_FRMOVRUN) {
1886	dwc2_hc_frmovrun_intr(hsotg, chan, chnum, qtd);
1887	} else if (!out_nak_enh) {
1888	if (chan->hcint & HCINTMSK_NYET) {
1889	/*
1890	* Must handle nyet before nak or ack. Could get a nyet
1891	* at the same time as either of those on a BULK/CONTROL
1892	* OUT that started with a PING. The nyet takes
1893	* precedence.
1894	*/
1895	dwc2_hc_nyet_intr(hsotg, chan, chnum, qtd);
1896	} else if ((chan->hcint & HCINTMSK_NAK) &&
1897	!(hcintmsk & HCINTMSK_NAK)) {
1898	/*
1899	* If nak is not masked, it's because a non-split IN
1900	* transfer is in an error state. In that case, the nak
1901	* is handled by the nak interrupt handler, not here.
1902	* Handle nak here for BULK/CONTROL OUT transfers, which
1903	* halt on a NAK to allow rewinding the buffer pointer.
1904	*/
1905	dwc2_hc_nak_intr(hsotg, chan, chnum, qtd);
1906	} else if ((chan->hcint & HCINTMSK_ACK) &&
1907	!(hcintmsk & HCINTMSK_ACK)) {
1908	/*
1909	* If ack is not masked, it's because a non-split IN
1910	* transfer is in an error state. In that case, the ack
1911	* is handled by the ack interrupt handler, not here.
1912	* Handle ack here for split transfers. Start splits
1913	* halt on ACK.
1914	*/
1915	dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
1916	} else {
1917	if (chan->ep_type == USB_ENDPOINT_XFER_INT ||
1918	chan->ep_type == USB_ENDPOINT_XFER_ISOC) {
1919	/*
1920	* A periodic transfer halted with no other
1921	* channel interrupts set. Assume it was halted
1922	* by the core because it could not be completed
1923	* in its scheduled (micro)frame.
1924	*/
1925	dev_dbg(hsotg->dev,
1926	"%s: Halt channel %d (assume incomplete periodic transfer)\n",
1927	__func__, chnum);
1928	dwc2_halt_channel(hsotg, chan, qtd,
1929	halt_status: DWC2_HC_XFER_PERIODIC_INCOMPLETE);
1930	} else {
1931	dev_err(hsotg->dev,
1932	"%s: Channel %d - ChHltd set, but reason is unknown\n",
1933	__func__, chnum);
1934	dev_err(hsotg->dev,
1935	"hcint 0x%08x, intsts 0x%08x\n",
1936	chan->hcint,
1937	dwc2_readl(hsotg, GINTSTS));
1938	goto error;
1939	}
1940	}
1941	} else {
1942	dev_info(hsotg->dev,
1943	"NYET/NAK/ACK/other in non-error case, 0x%08x\n",
1944	chan->hcint);
1945	error:
1946	/* Failthrough: use 3-strikes rule */
1947	qtd->error_count++;
1948	dwc2_update_urb_state_abn(hsotg, chan, chnum, urb: qtd->urb,
1949	qtd, halt_status: DWC2_HC_XFER_XACT_ERR);
1950	/*
1951	* We can get here after a completed transaction
1952	* (urb->actual_length >= urb->length) which was not reported
1953	* as completed. If that is the case, and we do not abort
1954	* the transfer, a transfer of size 0 will be enqueued
1955	* subsequently. If urb->actual_length is not DMA-aligned,
1956	* the buffer will then point to an unaligned address, and
1957	* the resulting behavior is undefined. Bail out in that
1958	* situation.
1959	*/
1960	if (qtd->urb->actual_length >= qtd->urb->length)
1961	qtd->error_count = 3;
1962	dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1963	dwc2_halt_channel(hsotg, chan, qtd, halt_status: DWC2_HC_XFER_XACT_ERR);
1964	}
1965	}
1966
1967	/*
1968	* Handles a host channel Channel Halted interrupt
1969	*
1970	* In slave mode, this handler is called only when the driver specifically
1971	* requests a halt. This occurs during handling other host channel interrupts
1972	* (e.g. nak, xacterr, stall, nyet, etc.).
1973	*
1974	* In DMA mode, this is the interrupt that occurs when the core has finished
1975	* processing a transfer on a channel. Other host channel interrupts (except
1976	* ahberr) are disabled in DMA mode.
1977	*/
1978	static void dwc2_hc_chhltd_intr(struct dwc2_hsotg *hsotg,
1979	struct dwc2_host_chan *chan, int chnum,
1980	struct dwc2_qtd *qtd)
1981	{
1982	if (dbg_hc(hc: chan))
1983	dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: Channel Halted--\n",
1984	chnum);
1985
1986	if (hsotg->params.host_dma) {
1987	dwc2_hc_chhltd_intr_dma(hsotg, chan, chnum, qtd);
1988	} else {
1989	if (!dwc2_halt_status_ok(hsotg, chan, chnum, qtd))
1990	return;
1991	dwc2_release_channel(hsotg, chan, qtd, halt_status: chan->halt_status);
1992	}
1993	}
1994
1995	/*
1996	* Check if the given qtd is still the top of the list (and thus valid).
1997	*
1998	* If dwc2_hcd_qtd_unlink_and_free() has been called since we grabbed
1999	* the qtd from the top of the list, this will return false (otherwise true).
2000	*/
2001	static bool dwc2_check_qtd_still_ok(struct dwc2_qtd *qtd, struct dwc2_qh *qh)
2002	{
2003	struct dwc2_qtd *cur_head;
2004
2005	if (!qh)
2006	return false;
2007
2008	cur_head = list_first_entry(&qh->qtd_list, struct dwc2_qtd,
2009	qtd_list_entry);
2010	return (cur_head == qtd);
2011	}
2012
2013	/* Handles interrupt for a specific Host Channel */
2014	static void dwc2_hc_n_intr(struct dwc2_hsotg *hsotg, int chnum)
2015	{
2016	struct dwc2_qtd *qtd;
2017	struct dwc2_host_chan *chan;
2018	u32 hcint, hcintmsk;
2019
2020	chan = hsotg->hc_ptr_array[chnum];
2021
2022	hcint = dwc2_readl(hsotg, HCINT(chnum));
2023	hcintmsk = dwc2_readl(hsotg, HCINTMSK(chnum));
2024	if (!chan) {
2025	dev_err(hsotg->dev, "## hc_ptr_array for channel is NULL ##\n");
2026	dwc2_writel(hsotg, value: hcint, HCINT(chnum));
2027	return;
2028	}
2029
2030	if (dbg_hc(hc: chan)) {
2031	dev_vdbg(hsotg->dev, "--Host Channel Interrupt--, Channel %d\n",
2032	chnum);
2033	dev_vdbg(hsotg->dev,
2034	" hcint 0x%08x, hcintmsk 0x%08x, hcint&hcintmsk 0x%08x\n",
2035	hcint, hcintmsk, hcint & hcintmsk);
2036	}
2037
2038	dwc2_writel(hsotg, value: hcint, HCINT(chnum));
2039
2040	/*
2041	* If we got an interrupt after someone called
2042	* dwc2_hcd_endpoint_disable() we don't want to crash below
2043	*/
2044	if (!chan->qh) {
2045	dev_warn(hsotg->dev, "Interrupt on disabled channel\n");
2046	return;
2047	}
2048
2049	chan->hcint = hcint;
2050	hcint &= hcintmsk;
2051
2052	/*
2053	* If the channel was halted due to a dequeue, the qtd list might
2054	* be empty or at least the first entry will not be the active qtd.
2055	* In this case, take a shortcut and just release the channel.
2056	*/
2057	if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE) {
2058	/*
2059	* If the channel was halted, this should be the only
2060	* interrupt unmasked
2061	*/
2062	WARN_ON(hcint != HCINTMSK_CHHLTD);
2063	if (hsotg->params.dma_desc_enable)
2064	dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
2065	halt_status: chan->halt_status);
2066	else
2067	dwc2_release_channel(hsotg, chan, NULL,
2068	halt_status: chan->halt_status);
2069	return;
2070	}
2071
2072	if (list_empty(head: &chan->qh->qtd_list)) {
2073	/*
2074	* TODO: Will this ever happen with the
2075	* DWC2_HC_XFER_URB_DEQUEUE handling above?
2076	*/
2077	dev_dbg(hsotg->dev, "## no QTD queued for channel %d ##\n",
2078	chnum);
2079	dev_dbg(hsotg->dev,
2080	" hcint 0x%08x, hcintmsk 0x%08x, hcint&hcintmsk 0x%08x\n",
2081	chan->hcint, hcintmsk, hcint);
2082	chan->halt_status = DWC2_HC_XFER_NO_HALT_STATUS;
2083	disable_hc_int(hsotg, chnum, HCINTMSK_CHHLTD);
2084	chan->hcint = 0;
2085	return;
2086	}
2087
2088	qtd = list_first_entry(&chan->qh->qtd_list, struct dwc2_qtd,
2089	qtd_list_entry);
2090
2091	if (!hsotg->params.host_dma) {
2092	if ((hcint & HCINTMSK_CHHLTD) && hcint != HCINTMSK_CHHLTD)
2093	hcint &= ~HCINTMSK_CHHLTD;
2094	}
2095
2096	if (hcint & HCINTMSK_XFERCOMPL) {
2097	dwc2_hc_xfercomp_intr(hsotg, chan, chnum, qtd);
2098	/*
2099	* If NYET occurred at same time as Xfer Complete, the NYET is
2100	* handled by the Xfer Complete interrupt handler. Don't want
2101	* to call the NYET interrupt handler in this case.
2102	*/
2103	hcint &= ~HCINTMSK_NYET;
2104	}
2105
2106	if (hcint & HCINTMSK_CHHLTD) {
2107	dwc2_hc_chhltd_intr(hsotg, chan, chnum, qtd);
2108	if (!dwc2_check_qtd_still_ok(qtd, qh: chan->qh))
2109	goto exit;
2110	}
2111	if (hcint & HCINTMSK_AHBERR) {
2112	dwc2_hc_ahberr_intr(hsotg, chan, chnum, qtd);
2113	if (!dwc2_check_qtd_still_ok(qtd, qh: chan->qh))
2114	goto exit;
2115	}
2116	if (hcint & HCINTMSK_STALL) {
2117	dwc2_hc_stall_intr(hsotg, chan, chnum, qtd);
2118	if (!dwc2_check_qtd_still_ok(qtd, qh: chan->qh))
2119	goto exit;
2120	}
2121	if (hcint & HCINTMSK_NAK) {
2122	dwc2_hc_nak_intr(hsotg, chan, chnum, qtd);
2123	if (!dwc2_check_qtd_still_ok(qtd, qh: chan->qh))
2124	goto exit;
2125	}
2126	if (hcint & HCINTMSK_ACK) {
2127	dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
2128	if (!dwc2_check_qtd_still_ok(qtd, qh: chan->qh))
2129	goto exit;
2130	}
2131	if (hcint & HCINTMSK_NYET) {
2132	dwc2_hc_nyet_intr(hsotg, chan, chnum, qtd);
2133	if (!dwc2_check_qtd_still_ok(qtd, qh: chan->qh))
2134	goto exit;
2135	}
2136	if (hcint & HCINTMSK_XACTERR) {
2137	dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
2138	if (!dwc2_check_qtd_still_ok(qtd, qh: chan->qh))
2139	goto exit;
2140	}
2141	if (hcint & HCINTMSK_BBLERR) {
2142	dwc2_hc_babble_intr(hsotg, chan, chnum, qtd);
2143	if (!dwc2_check_qtd_still_ok(qtd, qh: chan->qh))
2144	goto exit;
2145	}
2146	if (hcint & HCINTMSK_FRMOVRUN) {
2147	dwc2_hc_frmovrun_intr(hsotg, chan, chnum, qtd);
2148	if (!dwc2_check_qtd_still_ok(qtd, qh: chan->qh))
2149	goto exit;
2150	}
2151	if (hcint & HCINTMSK_DATATGLERR) {
2152	dwc2_hc_datatglerr_intr(hsotg, chan, chnum, qtd);
2153	if (!dwc2_check_qtd_still_ok(qtd, qh: chan->qh))
2154	goto exit;
2155	}
2156
2157	exit:
2158	chan->hcint = 0;
2159	}
2160
2161	/*
2162	* This interrupt indicates that one or more host channels has a pending
2163	* interrupt. There are multiple conditions that can cause each host channel
2164	* interrupt. This function determines which conditions have occurred for each
2165	* host channel interrupt and handles them appropriately.
2166	*/
2167	static void dwc2_hc_intr(struct dwc2_hsotg *hsotg)
2168	{
2169	u32 haint;
2170	int i;
2171	struct dwc2_host_chan *chan, *chan_tmp;
2172
2173	haint = dwc2_readl(hsotg, HAINT);
2174	if (dbg_perio()) {
2175	dev_vdbg(hsotg->dev, "%s()\n", __func__);
2176
2177	dev_vdbg(hsotg->dev, "HAINT=%08x\n", haint);
2178	}
2179
2180	/*
2181	* According to USB 2.0 spec section 11.18.8, a host must
2182	* issue complete-split transactions in a microframe for a
2183	* set of full-/low-speed endpoints in the same relative
2184	* order as the start-splits were issued in a microframe for.
2185	*/
2186	list_for_each_entry_safe(chan, chan_tmp, &hsotg->split_order,
2187	split_order_list_entry) {
2188	int hc_num = chan->hc_num;
2189
2190	if (haint & (1 << hc_num)) {
2191	dwc2_hc_n_intr(hsotg, chnum: hc_num);
2192	haint &= ~(1 << hc_num);
2193	}
2194	}
2195
2196	for (i = 0; i < hsotg->params.host_channels; i++) {
2197	if (haint & (1 << i))
2198	dwc2_hc_n_intr(hsotg, chnum: i);
2199	}
2200	}
2201
2202	/* This function handles interrupts for the HCD */
2203	irqreturn_t dwc2_handle_hcd_intr(struct dwc2_hsotg *hsotg)
2204	{
2205	u32 gintsts, dbg_gintsts;
2206	irqreturn_t retval = IRQ_HANDLED;
2207
2208	if (!dwc2_is_controller_alive(hsotg)) {
2209	dev_warn(hsotg->dev, "Controller is dead\n");
2210	return retval;
2211	} else {
2212	retval = IRQ_NONE;
2213	}
2214
2215	spin_lock(lock: &hsotg->lock);
2216
2217	/* Check if HOST Mode */
2218	if (dwc2_is_host_mode(hsotg)) {
2219	gintsts = dwc2_read_core_intr(hsotg);
2220	if (!gintsts) {
2221	spin_unlock(lock: &hsotg->lock);
2222	return retval;
2223	}
2224
2225	retval = IRQ_HANDLED;
2226
2227	dbg_gintsts = gintsts;
2228	#ifndef DEBUG_SOF
2229	dbg_gintsts &= ~GINTSTS_SOF;
2230	#endif
2231	if (!dbg_perio())
2232	dbg_gintsts &= ~(GINTSTS_HCHINT | GINTSTS_RXFLVL |
2233	GINTSTS_PTXFEMP);
2234
2235	/* Only print if there are any non-suppressed interrupts left */
2236	if (dbg_gintsts)
2237	dev_vdbg(hsotg->dev,
2238	"DWC OTG HCD Interrupt Detected gintsts&gintmsk=0x%08x\n",
2239	gintsts);
2240
2241	if (gintsts & GINTSTS_SOF)
2242	dwc2_sof_intr(hsotg);
2243	if (gintsts & GINTSTS_RXFLVL)
2244	dwc2_rx_fifo_level_intr(hsotg);
2245	if (gintsts & GINTSTS_NPTXFEMP)
2246	dwc2_np_tx_fifo_empty_intr(hsotg);
2247	if (gintsts & GINTSTS_PRTINT)
2248	dwc2_port_intr(hsotg);
2249	if (gintsts & GINTSTS_HCHINT)
2250	dwc2_hc_intr(hsotg);
2251	if (gintsts & GINTSTS_PTXFEMP)
2252	dwc2_perio_tx_fifo_empty_intr(hsotg);
2253
2254	if (dbg_gintsts) {
2255	dev_vdbg(hsotg->dev,
2256	"DWC OTG HCD Finished Servicing Interrupts\n");
2257	dev_vdbg(hsotg->dev,
2258	"DWC OTG HCD gintsts=0x%08x gintmsk=0x%08x\n",
2259	dwc2_readl(hsotg, GINTSTS),
2260	dwc2_readl(hsotg, GINTMSK));
2261	}
2262	}
2263
2264	spin_unlock(lock: &hsotg->lock);
2265
2266	return retval;
2267	}
2268