1	// SPDX-License-Identifier: GPL-2.0+
2	/*
3	* Copyright (C) 2011 Marvell International Ltd. All rights reserved.
4	* Author: Chao Xie <chao.xie@marvell.com>
5	* Neil Zhang <zhangwm@marvell.com>
6	*/
7
8	#include <linux/module.h>
9	#include <linux/pci.h>
10	#include <linux/dma-mapping.h>
11	#include <linux/dmapool.h>
12	#include <linux/kernel.h>
13	#include <linux/delay.h>
14	#include <linux/ioport.h>
15	#include <linux/sched.h>
16	#include <linux/slab.h>
17	#include <linux/errno.h>
18	#include <linux/err.h>
19	#include <linux/timer.h>
20	#include <linux/list.h>
21	#include <linux/interrupt.h>
22	#include <linux/moduleparam.h>
23	#include <linux/device.h>
24	#include <linux/usb/ch9.h>
25	#include <linux/usb/gadget.h>
26	#include <linux/usb/otg.h>
27	#include <linux/pm.h>
28	#include <linux/io.h>
29	#include <linux/irq.h>
30	#include <linux/platform_device.h>
31	#include <linux/clk.h>
32	#include <linux/platform_data/mv_usb.h>
33	#include <asm/unaligned.h>
34
35	#include "mv_udc.h"
36
37	#define DRIVER_DESC "Marvell PXA USB Device Controller driver"
38
39	#define ep_dir(ep) (((ep)->ep_num == 0) ? \
40	((ep)->udc->ep0_dir) : ((ep)->direction))
41
42	/* timeout value -- usec */
43	#define RESET_TIMEOUT 10000
44	#define FLUSH_TIMEOUT 10000
45	#define EPSTATUS_TIMEOUT 10000
46	#define PRIME_TIMEOUT 10000
47	#define READSAFE_TIMEOUT 1000
48
49	#define LOOPS_USEC_SHIFT 1
50	#define LOOPS_USEC (1 << LOOPS_USEC_SHIFT)
51	#define LOOPS(timeout) ((timeout) >> LOOPS_USEC_SHIFT)
52
53	static DECLARE_COMPLETION(release_done);
54
55	static const char driver_name[] = "mv_udc";
56
57	static void nuke(struct mv_ep *ep, int status);
58	static void stop_activity(struct mv_udc *udc, struct usb_gadget_driver *driver);
59
60	/* for endpoint 0 operations */
61	static const struct usb_endpoint_descriptor mv_ep0_desc = {
62	.bLength = USB_DT_ENDPOINT_SIZE,
63	.bDescriptorType = USB_DT_ENDPOINT,
64	.bEndpointAddress = 0,
65	.bmAttributes = USB_ENDPOINT_XFER_CONTROL,
66	.wMaxPacketSize = EP0_MAX_PKT_SIZE,
67	};
68
69	static void ep0_reset(struct mv_udc *udc)
70	{
71	struct mv_ep *ep;
72	u32 epctrlx;
73	int i = 0;
74
75	/* ep0 in and out */
76	for (i = 0; i < 2; i++) {
77	ep = &udc->eps[i];
78	ep->udc = udc;
79
80	/* ep0 dQH */
81	ep->dqh = &udc->ep_dqh[i];
82
83	/* configure ep0 endpoint capabilities in dQH */
84	ep->dqh->max_packet_length =
85	(EP0_MAX_PKT_SIZE << EP_QUEUE_HEAD_MAX_PKT_LEN_POS)
86	| EP_QUEUE_HEAD_IOS;
87
88	ep->dqh->next_dtd_ptr = EP_QUEUE_HEAD_NEXT_TERMINATE;
89
90	epctrlx = readl(addr: &udc->op_regs->epctrlx[0]);
91	if (i) { /* TX */
92	epctrlx |= EPCTRL_TX_ENABLE
93	| (USB_ENDPOINT_XFER_CONTROL
94	<< EPCTRL_TX_EP_TYPE_SHIFT);
95
96	} else { /* RX */
97	epctrlx |= EPCTRL_RX_ENABLE
98	| (USB_ENDPOINT_XFER_CONTROL
99	<< EPCTRL_RX_EP_TYPE_SHIFT);
100	}
101
102	writel(val: epctrlx, addr: &udc->op_regs->epctrlx[0]);
103	}
104	}
105
106	/* protocol ep0 stall, will automatically be cleared on new transaction */
107	static void ep0_stall(struct mv_udc *udc)
108	{
109	u32 epctrlx;
110
111	/* set TX and RX to stall */
112	epctrlx = readl(addr: &udc->op_regs->epctrlx[0]);
113	epctrlx |= EPCTRL_RX_EP_STALL | EPCTRL_TX_EP_STALL;
114	writel(val: epctrlx, addr: &udc->op_regs->epctrlx[0]);
115
116	/* update ep0 state */
117	udc->ep0_state = WAIT_FOR_SETUP;
118	udc->ep0_dir = EP_DIR_OUT;
119	}
120
121	static int process_ep_req(struct mv_udc *udc, int index,
122	struct mv_req *curr_req)
123	{
124	struct mv_dtd *curr_dtd;
125	struct mv_dqh *curr_dqh;
126	int actual, remaining_length;
127	int i, direction;
128	int retval = 0;
129	u32 errors;
130	u32 bit_pos;
131
132	curr_dqh = &udc->ep_dqh[index];
133	direction = index % 2;
134
135	curr_dtd = curr_req->head;
136	actual = curr_req->req.length;
137
138	for (i = 0; i < curr_req->dtd_count; i++) {
139	if (curr_dtd->size_ioc_sts & DTD_STATUS_ACTIVE) {
140	dev_dbg(&udc->dev->dev, "%s, dTD not completed\n",
141	udc->eps[index].name);
142	return 1;
143	}
144
145	errors = curr_dtd->size_ioc_sts & DTD_ERROR_MASK;
146	if (!errors) {
147	remaining_length =
148	(curr_dtd->size_ioc_sts & DTD_PACKET_SIZE)
149	>> DTD_LENGTH_BIT_POS;
150	actual -= remaining_length;
151
152	if (remaining_length) {
153	if (direction) {
154	dev_dbg(&udc->dev->dev,
155	"TX dTD remains data\n");
156	retval = -EPROTO;
157	break;
158	} else
159	break;
160	}
161	} else {
162	dev_info(&udc->dev->dev,
163	"complete_tr error: ep=%d %s: error = 0x%x\n",
164	index >> 1, direction ? "SEND" : "RECV",
165	errors);
166	if (errors & DTD_STATUS_HALTED) {
167	/* Clear the errors and Halt condition */
168	curr_dqh->size_ioc_int_sts &= ~errors;
169	retval = -EPIPE;
170	} else if (errors & DTD_STATUS_DATA_BUFF_ERR) {
171	retval = -EPROTO;
172	} else if (errors & DTD_STATUS_TRANSACTION_ERR) {
173	retval = -EILSEQ;
174	}
175	}
176	if (i != curr_req->dtd_count - 1)
177	curr_dtd = (struct mv_dtd *)curr_dtd->next_dtd_virt;
178	}
179	if (retval)
180	return retval;
181
182	if (direction == EP_DIR_OUT)
183	bit_pos = 1 << curr_req->ep->ep_num;
184	else
185	bit_pos = 1 << (16 + curr_req->ep->ep_num);
186
187	while (curr_dqh->curr_dtd_ptr == curr_dtd->td_dma) {
188	if (curr_dtd->dtd_next == EP_QUEUE_HEAD_NEXT_TERMINATE) {
189	while (readl(addr: &udc->op_regs->epstatus) & bit_pos)
190	udelay(1);
191	break;
192	}
193	udelay(1);
194	}
195
196	curr_req->req.actual = actual;
197
198	return 0;
199	}
200
201	/*
202	* done() - retire a request; caller blocked irqs
203	* @status : request status to be set, only works when
204	* request is still in progress.
205	*/
206	static void done(struct mv_ep *ep, struct mv_req *req, int status)
207	__releases(&ep->udc->lock)
208	__acquires(&ep->udc->lock)
209	{
210	struct mv_udc *udc = NULL;
211	unsigned char stopped = ep->stopped;
212	struct mv_dtd *curr_td, *next_td;
213	int j;
214
215	udc = (struct mv_udc *)ep->udc;
216	/* Removed the req from fsl_ep->queue */
217	list_del_init(entry: &req->queue);
218
219	/* req.status should be set as -EINPROGRESS in ep_queue() */
220	if (req->req.status == -EINPROGRESS)
221	req->req.status = status;
222	else
223	status = req->req.status;
224
225	/* Free dtd for the request */
226	next_td = req->head;
227	for (j = 0; j < req->dtd_count; j++) {
228	curr_td = next_td;
229	if (j != req->dtd_count - 1)
230	next_td = curr_td->next_dtd_virt;
231	dma_pool_free(pool: udc->dtd_pool, vaddr: curr_td, addr: curr_td->td_dma);
232	}
233
234	usb_gadget_unmap_request(gadget: &udc->gadget, req: &req->req, ep_dir(ep));
235
236	if (status && (status != -ESHUTDOWN))
237	dev_info(&udc->dev->dev, "complete %s req %p stat %d len %u/%u",
238	ep->ep.name, &req->req, status,
239	req->req.actual, req->req.length);
240
241	ep->stopped = 1;
242
243	spin_unlock(lock: &ep->udc->lock);
244
245	usb_gadget_giveback_request(ep: &ep->ep, req: &req->req);
246
247	spin_lock(lock: &ep->udc->lock);
248	ep->stopped = stopped;
249	}
250
251	static int queue_dtd(struct mv_ep *ep, struct mv_req *req)
252	{
253	struct mv_udc *udc;
254	struct mv_dqh *dqh;
255	u32 bit_pos, direction;
256	u32 usbcmd, epstatus;
257	unsigned int loops;
258	int retval = 0;
259
260	udc = ep->udc;
261	direction = ep_dir(ep);
262	dqh = &(udc->ep_dqh[ep->ep_num * 2 + direction]);
263	bit_pos = 1 << (((direction == EP_DIR_OUT) ? 0 : 16) + ep->ep_num);
264
265	/* check if the pipe is empty */
266	if (!(list_empty(head: &ep->queue))) {
267	struct mv_req *lastreq;
268	lastreq = list_entry(ep->queue.prev, struct mv_req, queue);
269	lastreq->tail->dtd_next =
270	req->head->td_dma & EP_QUEUE_HEAD_NEXT_POINTER_MASK;
271
272	wmb();
273
274	if (readl(addr: &udc->op_regs->epprime) & bit_pos)
275	goto done;
276
277	loops = LOOPS(READSAFE_TIMEOUT);
278	while (1) {
279	/* start with setting the semaphores */
280	usbcmd = readl(addr: &udc->op_regs->usbcmd);
281	usbcmd |= USBCMD_ATDTW_TRIPWIRE_SET;
282	writel(val: usbcmd, addr: &udc->op_regs->usbcmd);
283
284	/* read the endpoint status */
285	epstatus = readl(addr: &udc->op_regs->epstatus) & bit_pos;
286
287	/*
288	* Reread the ATDTW semaphore bit to check if it is
289	* cleared. When hardware see a hazard, it will clear
290	* the bit or else we remain set to 1 and we can
291	* proceed with priming of endpoint if not already
292	* primed.
293	*/
294	if (readl(addr: &udc->op_regs->usbcmd)
295	& USBCMD_ATDTW_TRIPWIRE_SET)
296	break;
297
298	loops--;
299	if (loops == 0) {
300	dev_err(&udc->dev->dev,
301	"Timeout for ATDTW_TRIPWIRE...\n");
302	retval = -ETIME;
303	goto done;
304	}
305	udelay(LOOPS_USEC);
306	}
307
308	/* Clear the semaphore */
309	usbcmd = readl(addr: &udc->op_regs->usbcmd);
310	usbcmd &= USBCMD_ATDTW_TRIPWIRE_CLEAR;
311	writel(val: usbcmd, addr: &udc->op_regs->usbcmd);
312
313	if (epstatus)
314	goto done;
315	}
316
317	/* Write dQH next pointer and terminate bit to 0 */
318	dqh->next_dtd_ptr = req->head->td_dma
319	& EP_QUEUE_HEAD_NEXT_POINTER_MASK;
320
321	/* clear active and halt bit, in case set from a previous error */
322	dqh->size_ioc_int_sts &= ~(DTD_STATUS_ACTIVE | DTD_STATUS_HALTED);
323
324	/* Ensure that updates to the QH will occur before priming. */
325	wmb();
326
327	/* Prime the Endpoint */
328	writel(val: bit_pos, addr: &udc->op_regs->epprime);
329
330	done:
331	return retval;
332	}
333
334	static struct mv_dtd *build_dtd(struct mv_req *req, unsigned *length,
335	dma_addr_t *dma, int *is_last)
336	{
337	struct mv_dtd *dtd;
338	struct mv_udc *udc;
339	struct mv_dqh *dqh;
340	u32 temp, mult = 0;
341
342	/* how big will this transfer be? */
343	if (usb_endpoint_xfer_isoc(epd: req->ep->ep.desc)) {
344	dqh = req->ep->dqh;
345	mult = (dqh->max_packet_length >> EP_QUEUE_HEAD_MULT_POS)
346	& 0x3;
347	*length = min(req->req.length - req->req.actual,
348	(unsigned)(mult * req->ep->ep.maxpacket));
349	} else
350	*length = min(req->req.length - req->req.actual,
351	(unsigned)EP_MAX_LENGTH_TRANSFER);
352
353	udc = req->ep->udc;
354
355	/*
356	* Be careful that no _GFP_HIGHMEM is set,
357	* or we can not use dma_to_virt
358	*/
359	dtd = dma_pool_alloc(pool: udc->dtd_pool, GFP_ATOMIC, handle: dma);
360	if (dtd == NULL)
361	return dtd;
362
363	dtd->td_dma = *dma;
364	/* initialize buffer page pointers */
365	temp = (u32)(req->req.dma + req->req.actual);
366	dtd->buff_ptr0 = cpu_to_le32(temp);
367	temp &= ~0xFFF;
368	dtd->buff_ptr1 = cpu_to_le32(temp + 0x1000);
369	dtd->buff_ptr2 = cpu_to_le32(temp + 0x2000);
370	dtd->buff_ptr3 = cpu_to_le32(temp + 0x3000);
371	dtd->buff_ptr4 = cpu_to_le32(temp + 0x4000);
372
373	req->req.actual += *length;
374
375	/* zlp is needed if req->req.zero is set */
376	if (req->req.zero) {
377	if (*length == 0 || (*length % req->ep->ep.maxpacket) != 0)
378	*is_last = 1;
379	else
380	*is_last = 0;
381	} else if (req->req.length == req->req.actual)
382	*is_last = 1;
383	else
384	*is_last = 0;
385
386	/* Fill in the transfer size; set active bit */
387	temp = ((*length << DTD_LENGTH_BIT_POS) | DTD_STATUS_ACTIVE);
388
389	/* Enable interrupt for the last dtd of a request */
390	if (*is_last && !req->req.no_interrupt)
391	temp |= DTD_IOC;
392
393	temp |= mult << 10;
394
395	dtd->size_ioc_sts = temp;
396
397	mb();
398
399	return dtd;
400	}
401
402	/* generate dTD linked list for a request */
403	static int req_to_dtd(struct mv_req *req)
404	{
405	unsigned count;
406	int is_last, is_first = 1;
407	struct mv_dtd *dtd, *last_dtd = NULL;
408	dma_addr_t dma;
409
410	do {
411	dtd = build_dtd(req, length: &count, dma: &dma, is_last: &is_last);
412	if (dtd == NULL)
413	return -ENOMEM;
414
415	if (is_first) {
416	is_first = 0;
417	req->head = dtd;
418	} else {
419	last_dtd->dtd_next = dma;
420	last_dtd->next_dtd_virt = dtd;
421	}
422	last_dtd = dtd;
423	req->dtd_count++;
424	} while (!is_last);
425
426	/* set terminate bit to 1 for the last dTD */
427	dtd->dtd_next = DTD_NEXT_TERMINATE;
428
429	req->tail = dtd;
430
431	return 0;
432	}
433
434	static int mv_ep_enable(struct usb_ep *_ep,
435	const struct usb_endpoint_descriptor *desc)
436	{
437	struct mv_udc *udc;
438	struct mv_ep *ep;
439	struct mv_dqh *dqh;
440	u16 max = 0;
441	u32 bit_pos, epctrlx, direction;
442	const unsigned char zlt = 1;
443	unsigned char ios, mult;
444	unsigned long flags;
445
446	ep = container_of(_ep, struct mv_ep, ep);
447	udc = ep->udc;
448
449	if (!_ep || !desc
450	|| desc->bDescriptorType != USB_DT_ENDPOINT)
451	return -EINVAL;
452
453	if (!udc->driver || udc->gadget.speed == USB_SPEED_UNKNOWN)
454	return -ESHUTDOWN;
455
456	direction = ep_dir(ep);
457	max = usb_endpoint_maxp(epd: desc);
458
459	/*
460	* disable HW zero length termination select
461	* driver handles zero length packet through req->req.zero
462	*/
463	bit_pos = 1 << ((direction == EP_DIR_OUT ? 0 : 16) + ep->ep_num);
464
465	/* Check if the Endpoint is Primed */
466	if ((readl(addr: &udc->op_regs->epprime) & bit_pos)
467	|| (readl(addr: &udc->op_regs->epstatus) & bit_pos)) {
468	dev_info(&udc->dev->dev,
469	"ep=%d %s: Init ERROR: ENDPTPRIME=0x%x,"
470	" ENDPTSTATUS=0x%x, bit_pos=0x%x\n",
471	(unsigned)ep->ep_num, direction ? "SEND" : "RECV",
472	(unsigned)readl(&udc->op_regs->epprime),
473	(unsigned)readl(&udc->op_regs->epstatus),
474	(unsigned)bit_pos);
475	goto en_done;
476	}
477
478	/* Set the max packet length, interrupt on Setup and Mult fields */
479	ios = 0;
480	mult = 0;
481	switch (desc->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK) {
482	case USB_ENDPOINT_XFER_BULK:
483	case USB_ENDPOINT_XFER_INT:
484	break;
485	case USB_ENDPOINT_XFER_CONTROL:
486	ios = 1;
487	break;
488	case USB_ENDPOINT_XFER_ISOC:
489	/* Calculate transactions needed for high bandwidth iso */
490	mult = usb_endpoint_maxp_mult(epd: desc);
491	/* 3 transactions at most */
492	if (mult > 3)
493	goto en_done;
494	break;
495	default:
496	goto en_done;
497	}
498
499	spin_lock_irqsave(&udc->lock, flags);
500	/* Get the endpoint queue head address */
501	dqh = ep->dqh;
502	dqh->max_packet_length = (max << EP_QUEUE_HEAD_MAX_PKT_LEN_POS)
503	| (mult << EP_QUEUE_HEAD_MULT_POS)
504	| (zlt ? EP_QUEUE_HEAD_ZLT_SEL : 0)
505	| (ios ? EP_QUEUE_HEAD_IOS : 0);
506	dqh->next_dtd_ptr = 1;
507	dqh->size_ioc_int_sts = 0;
508
509	ep->ep.maxpacket = max;
510	ep->ep.desc = desc;
511	ep->stopped = 0;
512
513	/* Enable the endpoint for Rx or Tx and set the endpoint type */
514	epctrlx = readl(addr: &udc->op_regs->epctrlx[ep->ep_num]);
515	if (direction == EP_DIR_IN) {
516	epctrlx &= ~EPCTRL_TX_ALL_MASK;
517	epctrlx |= EPCTRL_TX_ENABLE | EPCTRL_TX_DATA_TOGGLE_RST
518	| ((desc->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK)
519	<< EPCTRL_TX_EP_TYPE_SHIFT);
520	} else {
521	epctrlx &= ~EPCTRL_RX_ALL_MASK;
522	epctrlx |= EPCTRL_RX_ENABLE | EPCTRL_RX_DATA_TOGGLE_RST
523	| ((desc->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK)
524	<< EPCTRL_RX_EP_TYPE_SHIFT);
525	}
526	writel(val: epctrlx, addr: &udc->op_regs->epctrlx[ep->ep_num]);
527
528	/*
529	* Implement Guideline (GL# USB-7) The unused endpoint type must
530	* be programmed to bulk.
531	*/
532	epctrlx = readl(addr: &udc->op_regs->epctrlx[ep->ep_num]);
533	if ((epctrlx & EPCTRL_RX_ENABLE) == 0) {
534	epctrlx |= (USB_ENDPOINT_XFER_BULK
535	<< EPCTRL_RX_EP_TYPE_SHIFT);
536	writel(val: epctrlx, addr: &udc->op_regs->epctrlx[ep->ep_num]);
537	}
538
539	epctrlx = readl(addr: &udc->op_regs->epctrlx[ep->ep_num]);
540	if ((epctrlx & EPCTRL_TX_ENABLE) == 0) {
541	epctrlx |= (USB_ENDPOINT_XFER_BULK
542	<< EPCTRL_TX_EP_TYPE_SHIFT);
543	writel(val: epctrlx, addr: &udc->op_regs->epctrlx[ep->ep_num]);
544	}
545
546	spin_unlock_irqrestore(lock: &udc->lock, flags);
547
548	return 0;
549	en_done:
550	return -EINVAL;
551	}
552
553	static int mv_ep_disable(struct usb_ep *_ep)
554	{
555	struct mv_udc *udc;
556	struct mv_ep *ep;
557	struct mv_dqh *dqh;
558	u32 epctrlx, direction;
559	unsigned long flags;
560
561	ep = container_of(_ep, struct mv_ep, ep);
562	if ((_ep == NULL) || !ep->ep.desc)
563	return -EINVAL;
564
565	udc = ep->udc;
566
567	/* Get the endpoint queue head address */
568	dqh = ep->dqh;
569
570	spin_lock_irqsave(&udc->lock, flags);
571
572	direction = ep_dir(ep);
573
574	/* Reset the max packet length and the interrupt on Setup */
575	dqh->max_packet_length = 0;
576
577	/* Disable the endpoint for Rx or Tx and reset the endpoint type */
578	epctrlx = readl(addr: &udc->op_regs->epctrlx[ep->ep_num]);
579	epctrlx &= ~((direction == EP_DIR_IN)
580	? (EPCTRL_TX_ENABLE | EPCTRL_TX_TYPE)
581	: (EPCTRL_RX_ENABLE | EPCTRL_RX_TYPE));
582	writel(val: epctrlx, addr: &udc->op_regs->epctrlx[ep->ep_num]);
583
584	/* nuke all pending requests (does flush) */
585	nuke(ep, status: -ESHUTDOWN);
586
587	ep->ep.desc = NULL;
588	ep->stopped = 1;
589
590	spin_unlock_irqrestore(lock: &udc->lock, flags);
591
592	return 0;
593	}
594
595	static struct usb_request *
596	mv_alloc_request(struct usb_ep *_ep, gfp_t gfp_flags)
597	{
598	struct mv_req *req;
599
600	req = kzalloc(size: sizeof *req, flags: gfp_flags);
601	if (!req)
602	return NULL;
603
604	req->req.dma = DMA_ADDR_INVALID;
605	INIT_LIST_HEAD(list: &req->queue);
606
607	return &req->req;
608	}
609
610	static void mv_free_request(struct usb_ep *_ep, struct usb_request *_req)
611	{
612	struct mv_req *req = NULL;
613
614	req = container_of(_req, struct mv_req, req);
615
616	if (_req)
617	kfree(objp: req);
618	}
619
620	static void mv_ep_fifo_flush(struct usb_ep *_ep)
621	{
622	struct mv_udc *udc;
623	u32 bit_pos, direction;
624	struct mv_ep *ep;
625	unsigned int loops;
626
627	if (!_ep)
628	return;
629
630	ep = container_of(_ep, struct mv_ep, ep);
631	if (!ep->ep.desc)
632	return;
633
634	udc = ep->udc;
635	direction = ep_dir(ep);
636
637	if (ep->ep_num == 0)
638	bit_pos = (1 << 16) | 1;
639	else if (direction == EP_DIR_OUT)
640	bit_pos = 1 << ep->ep_num;
641	else
642	bit_pos = 1 << (16 + ep->ep_num);
643
644	loops = LOOPS(EPSTATUS_TIMEOUT);
645	do {
646	unsigned int inter_loops;
647
648	if (loops == 0) {
649	dev_err(&udc->dev->dev,
650	"TIMEOUT for ENDPTSTATUS=0x%x, bit_pos=0x%x\n",
651	(unsigned)readl(&udc->op_regs->epstatus),
652	(unsigned)bit_pos);
653	return;
654	}
655	/* Write 1 to the Flush register */
656	writel(val: bit_pos, addr: &udc->op_regs->epflush);
657
658	/* Wait until flushing completed */
659	inter_loops = LOOPS(FLUSH_TIMEOUT);
660	while (readl(addr: &udc->op_regs->epflush)) {
661	/*
662	* ENDPTFLUSH bit should be cleared to indicate this
663	* operation is complete
664	*/
665	if (inter_loops == 0) {
666	dev_err(&udc->dev->dev,
667	"TIMEOUT for ENDPTFLUSH=0x%x,"
668	"bit_pos=0x%x\n",
669	(unsigned)readl(&udc->op_regs->epflush),
670	(unsigned)bit_pos);
671	return;
672	}
673	inter_loops--;
674	udelay(LOOPS_USEC);
675	}
676	loops--;
677	} while (readl(addr: &udc->op_regs->epstatus) & bit_pos);
678	}
679
680	/* queues (submits) an I/O request to an endpoint */
681	static int
682	mv_ep_queue(struct usb_ep *_ep, struct usb_request *_req, gfp_t gfp_flags)
683	{
684	struct mv_ep *ep = container_of(_ep, struct mv_ep, ep);
685	struct mv_req *req = container_of(_req, struct mv_req, req);
686	struct mv_udc *udc = ep->udc;
687	unsigned long flags;
688	int retval;
689
690	/* catch various bogus parameters */
691	if (!_req || !req->req.complete || !req->req.buf
692	|| !list_empty(head: &req->queue)) {
693	dev_err(&udc->dev->dev, "%s, bad params", __func__);
694	return -EINVAL;
695	}
696	if (unlikely(!_ep || !ep->ep.desc)) {
697	dev_err(&udc->dev->dev, "%s, bad ep", __func__);
698	return -EINVAL;
699	}
700
701	udc = ep->udc;
702	if (!udc->driver || udc->gadget.speed == USB_SPEED_UNKNOWN)
703	return -ESHUTDOWN;
704
705	req->ep = ep;
706
707	/* map virtual address to hardware */
708	retval = usb_gadget_map_request(gadget: &udc->gadget, req: _req, ep_dir(ep));
709	if (retval)
710	return retval;
711
712	req->req.status = -EINPROGRESS;
713	req->req.actual = 0;
714	req->dtd_count = 0;
715
716	spin_lock_irqsave(&udc->lock, flags);
717
718	/* build dtds and push them to device queue */
719	if (!req_to_dtd(req)) {
720	retval = queue_dtd(ep, req);
721	if (retval) {
722	spin_unlock_irqrestore(lock: &udc->lock, flags);
723	dev_err(&udc->dev->dev, "Failed to queue dtd\n");
724	goto err_unmap_dma;
725	}
726	} else {
727	spin_unlock_irqrestore(lock: &udc->lock, flags);
728	dev_err(&udc->dev->dev, "Failed to dma_pool_alloc\n");
729	retval = -ENOMEM;
730	goto err_unmap_dma;
731	}
732
733	/* Update ep0 state */
734	if (ep->ep_num == 0)
735	udc->ep0_state = DATA_STATE_XMIT;
736
737	/* irq handler advances the queue */
738	list_add_tail(new: &req->queue, head: &ep->queue);
739	spin_unlock_irqrestore(lock: &udc->lock, flags);
740
741	return 0;
742
743	err_unmap_dma:
744	usb_gadget_unmap_request(gadget: &udc->gadget, req: _req, ep_dir(ep));
745
746	return retval;
747	}
748
749	static void mv_prime_ep(struct mv_ep *ep, struct mv_req *req)
750	{
751	struct mv_dqh *dqh = ep->dqh;
752	u32 bit_pos;
753
754	/* Write dQH next pointer and terminate bit to 0 */
755	dqh->next_dtd_ptr = req->head->td_dma
756	& EP_QUEUE_HEAD_NEXT_POINTER_MASK;
757
758	/* clear active and halt bit, in case set from a previous error */
759	dqh->size_ioc_int_sts &= ~(DTD_STATUS_ACTIVE | DTD_STATUS_HALTED);
760
761	/* Ensure that updates to the QH will occure before priming. */
762	wmb();
763
764	bit_pos = 1 << (((ep_dir(ep) == EP_DIR_OUT) ? 0 : 16) + ep->ep_num);
765
766	/* Prime the Endpoint */
767	writel(val: bit_pos, addr: &ep->udc->op_regs->epprime);
768	}
769
770	/* dequeues (cancels, unlinks) an I/O request from an endpoint */
771	static int mv_ep_dequeue(struct usb_ep *_ep, struct usb_request *_req)
772	{
773	struct mv_ep *ep = container_of(_ep, struct mv_ep, ep);
774	struct mv_req *req = NULL, *iter;
775	struct mv_udc *udc = ep->udc;
776	unsigned long flags;
777	int stopped, ret = 0;
778	u32 epctrlx;
779
780	if (!_ep || !_req)
781	return -EINVAL;
782
783	spin_lock_irqsave(&ep->udc->lock, flags);
784	stopped = ep->stopped;
785
786	/* Stop the ep before we deal with the queue */
787	ep->stopped = 1;
788	epctrlx = readl(addr: &udc->op_regs->epctrlx[ep->ep_num]);
789	if (ep_dir(ep) == EP_DIR_IN)
790	epctrlx &= ~EPCTRL_TX_ENABLE;
791	else
792	epctrlx &= ~EPCTRL_RX_ENABLE;
793	writel(val: epctrlx, addr: &udc->op_regs->epctrlx[ep->ep_num]);
794
795	/* make sure it's actually queued on this endpoint */
796	list_for_each_entry(iter, &ep->queue, queue) {
797	if (&iter->req != _req)
798	continue;
799	req = iter;
800	break;
801	}
802	if (!req) {
803	ret = -EINVAL;
804	goto out;
805	}
806
807	/* The request is in progress, or completed but not dequeued */
808	if (ep->queue.next == &req->queue) {
809	_req->status = -ECONNRESET;
810	mv_ep_fifo_flush(_ep); /* flush current transfer */
811
812	/* The request isn't the last request in this ep queue */
813	if (req->queue.next != &ep->queue) {
814	struct mv_req *next_req;
815
816	next_req = list_entry(req->queue.next,
817	struct mv_req, queue);
818
819	/* Point the QH to the first TD of next request */
820	mv_prime_ep(ep, req: next_req);
821	} else {
822	struct mv_dqh *qh;
823
824	qh = ep->dqh;
825	qh->next_dtd_ptr = 1;
826	qh->size_ioc_int_sts = 0;
827	}
828
829	/* The request hasn't been processed, patch up the TD chain */
830	} else {
831	struct mv_req *prev_req;
832
833	prev_req = list_entry(req->queue.prev, struct mv_req, queue);
834	writel(readl(addr: &req->tail->dtd_next),
835	addr: &prev_req->tail->dtd_next);
836
837	}
838
839	done(ep, req, status: -ECONNRESET);
840
841	/* Enable EP */
842	out:
843	epctrlx = readl(addr: &udc->op_regs->epctrlx[ep->ep_num]);
844	if (ep_dir(ep) == EP_DIR_IN)
845	epctrlx |= EPCTRL_TX_ENABLE;
846	else
847	epctrlx |= EPCTRL_RX_ENABLE;
848	writel(val: epctrlx, addr: &udc->op_regs->epctrlx[ep->ep_num]);
849	ep->stopped = stopped;
850
851	spin_unlock_irqrestore(lock: &ep->udc->lock, flags);
852	return ret;
853	}
854
855	static void ep_set_stall(struct mv_udc *udc, u8 ep_num, u8 direction, int stall)
856	{
857	u32 epctrlx;
858
859	epctrlx = readl(addr: &udc->op_regs->epctrlx[ep_num]);
860
861	if (stall) {
862	if (direction == EP_DIR_IN)
863	epctrlx |= EPCTRL_TX_EP_STALL;
864	else
865	epctrlx |= EPCTRL_RX_EP_STALL;
866	} else {
867	if (direction == EP_DIR_IN) {
868	epctrlx &= ~EPCTRL_TX_EP_STALL;
869	epctrlx |= EPCTRL_TX_DATA_TOGGLE_RST;
870	} else {
871	epctrlx &= ~EPCTRL_RX_EP_STALL;
872	epctrlx |= EPCTRL_RX_DATA_TOGGLE_RST;
873	}
874	}
875	writel(val: epctrlx, addr: &udc->op_regs->epctrlx[ep_num]);
876	}
877
878	static int ep_is_stall(struct mv_udc *udc, u8 ep_num, u8 direction)
879	{
880	u32 epctrlx;
881
882	epctrlx = readl(addr: &udc->op_regs->epctrlx[ep_num]);
883
884	if (direction == EP_DIR_OUT)
885	return (epctrlx & EPCTRL_RX_EP_STALL) ? 1 : 0;
886	else
887	return (epctrlx & EPCTRL_TX_EP_STALL) ? 1 : 0;
888	}
889
890	static int mv_ep_set_halt_wedge(struct usb_ep *_ep, int halt, int wedge)
891	{
892	struct mv_ep *ep;
893	unsigned long flags;
894	int status = 0;
895	struct mv_udc *udc;
896
897	ep = container_of(_ep, struct mv_ep, ep);
898	udc = ep->udc;
899	if (!_ep || !ep->ep.desc) {
900	status = -EINVAL;
901	goto out;
902	}
903
904	if (ep->ep.desc->bmAttributes == USB_ENDPOINT_XFER_ISOC) {
905	status = -EOPNOTSUPP;
906	goto out;
907	}
908
909	/*
910	* Attempt to halt IN ep will fail if any transfer requests
911	* are still queue
912	*/
913	if (halt && (ep_dir(ep) == EP_DIR_IN) && !list_empty(head: &ep->queue)) {
914	status = -EAGAIN;
915	goto out;
916	}
917
918	spin_lock_irqsave(&ep->udc->lock, flags);
919	ep_set_stall(udc, ep_num: ep->ep_num, ep_dir(ep), stall: halt);
920	if (halt && wedge)
921	ep->wedge = 1;
922	else if (!halt)
923	ep->wedge = 0;
924	spin_unlock_irqrestore(lock: &ep->udc->lock, flags);
925
926	if (ep->ep_num == 0) {
927	udc->ep0_state = WAIT_FOR_SETUP;
928	udc->ep0_dir = EP_DIR_OUT;
929	}
930	out:
931	return status;
932	}
933
934	static int mv_ep_set_halt(struct usb_ep *_ep, int halt)
935	{
936	return mv_ep_set_halt_wedge(_ep, halt, wedge: 0);
937	}
938
939	static int mv_ep_set_wedge(struct usb_ep *_ep)
940	{
941	return mv_ep_set_halt_wedge(_ep, halt: 1, wedge: 1);
942	}
943
944	static const struct usb_ep_ops mv_ep_ops = {
945	.enable = mv_ep_enable,
946	.disable = mv_ep_disable,
947
948	.alloc_request = mv_alloc_request,
949	.free_request = mv_free_request,
950
951	.queue = mv_ep_queue,
952	.dequeue = mv_ep_dequeue,
953
954	.set_wedge = mv_ep_set_wedge,
955	.set_halt = mv_ep_set_halt,
956	.fifo_flush = mv_ep_fifo_flush, /* flush fifo */
957	};
958
959	static int udc_clock_enable(struct mv_udc *udc)
960	{
961	return clk_prepare_enable(clk: udc->clk);
962	}
963
964	static void udc_clock_disable(struct mv_udc *udc)
965	{
966	clk_disable_unprepare(clk: udc->clk);
967	}
968
969	static void udc_stop(struct mv_udc *udc)
970	{
971	u32 tmp;
972
973	/* Disable interrupts */
974	tmp = readl(addr: &udc->op_regs->usbintr);
975	tmp &= ~(USBINTR_INT_EN | USBINTR_ERR_INT_EN |
976	USBINTR_PORT_CHANGE_DETECT_EN | USBINTR_RESET_EN);
977	writel(val: tmp, addr: &udc->op_regs->usbintr);
978
979	udc->stopped = 1;
980
981	/* Reset the Run the bit in the command register to stop VUSB */
982	tmp = readl(addr: &udc->op_regs->usbcmd);
983	tmp &= ~USBCMD_RUN_STOP;
984	writel(val: tmp, addr: &udc->op_regs->usbcmd);
985	}
986
987	static void udc_start(struct mv_udc *udc)
988	{
989	u32 usbintr;
990
991	usbintr = USBINTR_INT_EN | USBINTR_ERR_INT_EN
992	| USBINTR_PORT_CHANGE_DETECT_EN
993	| USBINTR_RESET_EN | USBINTR_DEVICE_SUSPEND;
994	/* Enable interrupts */
995	writel(val: usbintr, addr: &udc->op_regs->usbintr);
996
997	udc->stopped = 0;
998
999	/* Set the Run bit in the command register */
1000	writel(USBCMD_RUN_STOP, addr: &udc->op_regs->usbcmd);
1001	}
1002
1003	static int udc_reset(struct mv_udc *udc)
1004	{
1005	unsigned int loops;
1006	u32 tmp, portsc;
1007
1008	/* Stop the controller */
1009	tmp = readl(addr: &udc->op_regs->usbcmd);
1010	tmp &= ~USBCMD_RUN_STOP;
1011	writel(val: tmp, addr: &udc->op_regs->usbcmd);
1012
1013	/* Reset the controller to get default values */
1014	writel(USBCMD_CTRL_RESET, addr: &udc->op_regs->usbcmd);
1015
1016	/* wait for reset to complete */
1017	loops = LOOPS(RESET_TIMEOUT);
1018	while (readl(addr: &udc->op_regs->usbcmd) & USBCMD_CTRL_RESET) {
1019	if (loops == 0) {
1020	dev_err(&udc->dev->dev,
1021	"Wait for RESET completed TIMEOUT\n");
1022	return -ETIMEDOUT;
1023	}
1024	loops--;
1025	udelay(LOOPS_USEC);
1026	}
1027
1028	/* set controller to device mode */
1029	tmp = readl(addr: &udc->op_regs->usbmode);
1030	tmp |= USBMODE_CTRL_MODE_DEVICE;
1031
1032	/* turn setup lockout off, require setup tripwire in usbcmd */
1033	tmp |= USBMODE_SETUP_LOCK_OFF;
1034
1035	writel(val: tmp, addr: &udc->op_regs->usbmode);
1036
1037	writel(val: 0x0, addr: &udc->op_regs->epsetupstat);
1038
1039	/* Configure the Endpoint List Address */
1040	writel(val: udc->ep_dqh_dma & USB_EP_LIST_ADDRESS_MASK,
1041	addr: &udc->op_regs->eplistaddr);
1042
1043	portsc = readl(addr: &udc->op_regs->portsc[0]);
1044	if (readl(addr: &udc->cap_regs->hcsparams) & HCSPARAMS_PPC)
1045	portsc &= (~PORTSCX_W1C_BITS | ~PORTSCX_PORT_POWER);
1046
1047	if (udc->force_fs)
1048	portsc |= PORTSCX_FORCE_FULL_SPEED_CONNECT;
1049	else
1050	portsc &= (~PORTSCX_FORCE_FULL_SPEED_CONNECT);
1051
1052	writel(val: portsc, addr: &udc->op_regs->portsc[0]);
1053
1054	tmp = readl(addr: &udc->op_regs->epctrlx[0]);
1055	tmp &= ~(EPCTRL_TX_EP_STALL | EPCTRL_RX_EP_STALL);
1056	writel(val: tmp, addr: &udc->op_regs->epctrlx[0]);
1057
1058	return 0;
1059	}
1060
1061	static int mv_udc_enable_internal(struct mv_udc *udc)
1062	{
1063	int retval;
1064
1065	if (udc->active)
1066	return 0;
1067
1068	dev_dbg(&udc->dev->dev, "enable udc\n");
1069	retval = udc_clock_enable(udc);
1070	if (retval)
1071	return retval;
1072
1073	if (udc->pdata->phy_init) {
1074	retval = udc->pdata->phy_init(udc->phy_regs);
1075	if (retval) {
1076	dev_err(&udc->dev->dev,
1077	"init phy error %d\n", retval);
1078	udc_clock_disable(udc);
1079	return retval;
1080	}
1081	}
1082	udc->active = 1;
1083
1084	return 0;
1085	}
1086
1087	static int mv_udc_enable(struct mv_udc *udc)
1088	{
1089	if (udc->clock_gating)
1090	return mv_udc_enable_internal(udc);
1091
1092	return 0;
1093	}
1094
1095	static void mv_udc_disable_internal(struct mv_udc *udc)
1096	{
1097	if (udc->active) {
1098	dev_dbg(&udc->dev->dev, "disable udc\n");
1099	if (udc->pdata->phy_deinit)
1100	udc->pdata->phy_deinit(udc->phy_regs);
1101	udc_clock_disable(udc);
1102	udc->active = 0;
1103	}
1104	}
1105
1106	static void mv_udc_disable(struct mv_udc *udc)
1107	{
1108	if (udc->clock_gating)
1109	mv_udc_disable_internal(udc);
1110	}
1111
1112	static int mv_udc_get_frame(struct usb_gadget *gadget)
1113	{
1114	struct mv_udc *udc;
1115	u16 retval;
1116
1117	if (!gadget)
1118	return -ENODEV;
1119
1120	udc = container_of(gadget, struct mv_udc, gadget);
1121
1122	retval = readl(addr: &udc->op_regs->frindex) & USB_FRINDEX_MASKS;
1123
1124	return retval;
1125	}
1126
1127	/* Tries to wake up the host connected to this gadget */
1128	static int mv_udc_wakeup(struct usb_gadget *gadget)
1129	{
1130	struct mv_udc *udc = container_of(gadget, struct mv_udc, gadget);
1131	u32 portsc;
1132
1133	/* Remote wakeup feature not enabled by host */
1134	if (!udc->remote_wakeup)
1135	return -ENOTSUPP;
1136
1137	portsc = readl(addr: &udc->op_regs->portsc);
1138	/* not suspended? */
1139	if (!(portsc & PORTSCX_PORT_SUSPEND))
1140	return 0;
1141	/* trigger force resume */
1142	portsc |= PORTSCX_PORT_FORCE_RESUME;
1143	writel(val: portsc, addr: &udc->op_regs->portsc[0]);
1144	return 0;
1145	}
1146
1147	static int mv_udc_vbus_session(struct usb_gadget *gadget, int is_active)
1148	{
1149	struct mv_udc *udc;
1150	unsigned long flags;
1151	int retval = 0;
1152
1153	udc = container_of(gadget, struct mv_udc, gadget);
1154	spin_lock_irqsave(&udc->lock, flags);
1155
1156	udc->vbus_active = (is_active != 0);
1157
1158	dev_dbg(&udc->dev->dev, "%s: softconnect %d, vbus_active %d\n",
1159	__func__, udc->softconnect, udc->vbus_active);
1160
1161	if (udc->driver && udc->softconnect && udc->vbus_active) {
1162	retval = mv_udc_enable(udc);
1163	if (retval == 0) {
1164	/* Clock is disabled, need re-init registers */
1165	udc_reset(udc);
1166	ep0_reset(udc);
1167	udc_start(udc);
1168	}
1169	} else if (udc->driver && udc->softconnect) {
1170	if (!udc->active)
1171	goto out;
1172
1173	/* stop all the transfer in queue*/
1174	stop_activity(udc, driver: udc->driver);
1175	udc_stop(udc);
1176	mv_udc_disable(udc);
1177	}
1178
1179	out:
1180	spin_unlock_irqrestore(lock: &udc->lock, flags);
1181	return retval;
1182	}
1183
1184	static int mv_udc_pullup(struct usb_gadget *gadget, int is_on)
1185	{
1186	struct mv_udc *udc;
1187	unsigned long flags;
1188	int retval = 0;
1189
1190	udc = container_of(gadget, struct mv_udc, gadget);
1191	spin_lock_irqsave(&udc->lock, flags);
1192
1193	udc->softconnect = (is_on != 0);
1194
1195	dev_dbg(&udc->dev->dev, "%s: softconnect %d, vbus_active %d\n",
1196	__func__, udc->softconnect, udc->vbus_active);
1197
1198	if (udc->driver && udc->softconnect && udc->vbus_active) {
1199	retval = mv_udc_enable(udc);
1200	if (retval == 0) {
1201	/* Clock is disabled, need re-init registers */
1202	udc_reset(udc);
1203	ep0_reset(udc);
1204	udc_start(udc);
1205	}
1206	} else if (udc->driver && udc->vbus_active) {
1207	/* stop all the transfer in queue*/
1208	stop_activity(udc, driver: udc->driver);
1209	udc_stop(udc);
1210	mv_udc_disable(udc);
1211	}
1212
1213	spin_unlock_irqrestore(lock: &udc->lock, flags);
1214	return retval;
1215	}
1216
1217	static int mv_udc_start(struct usb_gadget *, struct usb_gadget_driver *);
1218	static int mv_udc_stop(struct usb_gadget *);
1219	/* device controller usb_gadget_ops structure */
1220	static const struct usb_gadget_ops mv_ops = {
1221
1222	/* returns the current frame number */
1223	.get_frame = mv_udc_get_frame,
1224
1225	/* tries to wake up the host connected to this gadget */
1226	.wakeup = mv_udc_wakeup,
1227
1228	/* notify controller that VBUS is powered or not */
1229	.vbus_session = mv_udc_vbus_session,
1230
1231	/* D+ pullup, software-controlled connect/disconnect to USB host */
1232	.pullup = mv_udc_pullup,
1233	.udc_start = mv_udc_start,
1234	.udc_stop = mv_udc_stop,
1235	};
1236
1237	static int eps_init(struct mv_udc *udc)
1238	{
1239	struct mv_ep *ep;
1240	char name[14];
1241	int i;
1242
1243	/* initialize ep0 */
1244	ep = &udc->eps[0];
1245	ep->udc = udc;
1246	strncpy(p: ep->name, q: "ep0", size: sizeof(ep->name));
1247	ep->ep.name = ep->name;
1248	ep->ep.ops = &mv_ep_ops;
1249	ep->wedge = 0;
1250	ep->stopped = 0;
1251	usb_ep_set_maxpacket_limit(ep: &ep->ep, EP0_MAX_PKT_SIZE);
1252	ep->ep.caps.type_control = true;
1253	ep->ep.caps.dir_in = true;
1254	ep->ep.caps.dir_out = true;
1255	ep->ep_num = 0;
1256	ep->ep.desc = &mv_ep0_desc;
1257	INIT_LIST_HEAD(list: &ep->queue);
1258
1259	ep->ep_type = USB_ENDPOINT_XFER_CONTROL;
1260
1261	/* initialize other endpoints */
1262	for (i = 2; i < udc->max_eps * 2; i++) {
1263	ep = &udc->eps[i];
1264	if (i % 2) {
1265	snprintf(buf: name, size: sizeof(name), fmt: "ep%din", i / 2);
1266	ep->direction = EP_DIR_IN;
1267	ep->ep.caps.dir_in = true;
1268	} else {
1269	snprintf(buf: name, size: sizeof(name), fmt: "ep%dout", i / 2);
1270	ep->direction = EP_DIR_OUT;
1271	ep->ep.caps.dir_out = true;
1272	}
1273	ep->udc = udc;
1274	strncpy(p: ep->name, q: name, size: sizeof(ep->name));
1275	ep->ep.name = ep->name;
1276
1277	ep->ep.caps.type_iso = true;
1278	ep->ep.caps.type_bulk = true;
1279	ep->ep.caps.type_int = true;
1280
1281	ep->ep.ops = &mv_ep_ops;
1282	ep->stopped = 0;
1283	usb_ep_set_maxpacket_limit(ep: &ep->ep, maxpacket_limit: (unsigned short) ~0);
1284	ep->ep_num = i / 2;
1285
1286	INIT_LIST_HEAD(list: &ep->queue);
1287	list_add_tail(new: &ep->ep.ep_list, head: &udc->gadget.ep_list);
1288
1289	ep->dqh = &udc->ep_dqh[i];
1290	}
1291
1292	return 0;
1293	}
1294
1295	/* delete all endpoint requests, called with spinlock held */
1296	static void nuke(struct mv_ep *ep, int status)
1297	{
1298	/* called with spinlock held */
1299	ep->stopped = 1;
1300
1301	/* endpoint fifo flush */
1302	mv_ep_fifo_flush(ep: &ep->ep);
1303
1304	while (!list_empty(head: &ep->queue)) {
1305	struct mv_req *req = NULL;
1306	req = list_entry(ep->queue.next, struct mv_req, queue);
1307	done(ep, req, status);
1308	}
1309	}
1310
1311	static void gadget_reset(struct mv_udc *udc, struct usb_gadget_driver *driver)
1312	{
1313	struct mv_ep *ep;
1314
1315	nuke(ep: &udc->eps[0], status: -ESHUTDOWN);
1316
1317	list_for_each_entry(ep, &udc->gadget.ep_list, ep.ep_list) {
1318	nuke(ep, status: -ESHUTDOWN);
1319	}
1320
1321	/* report reset; the driver is already quiesced */
1322	if (driver) {
1323	spin_unlock(lock: &udc->lock);
1324	usb_gadget_udc_reset(gadget: &udc->gadget, driver);
1325	spin_lock(lock: &udc->lock);
1326	}
1327	}
1328	/* stop all USB activities */
1329	static void stop_activity(struct mv_udc *udc, struct usb_gadget_driver *driver)
1330	{
1331	struct mv_ep *ep;
1332
1333	nuke(ep: &udc->eps[0], status: -ESHUTDOWN);
1334
1335	list_for_each_entry(ep, &udc->gadget.ep_list, ep.ep_list) {
1336	nuke(ep, status: -ESHUTDOWN);
1337	}
1338
1339	/* report disconnect; the driver is already quiesced */
1340	if (driver) {
1341	spin_unlock(lock: &udc->lock);
1342	driver->disconnect(&udc->gadget);
1343	spin_lock(lock: &udc->lock);
1344	}
1345	}
1346
1347	static int mv_udc_start(struct usb_gadget *gadget,
1348	struct usb_gadget_driver *driver)
1349	{
1350	struct mv_udc *udc;
1351	int retval = 0;
1352	unsigned long flags;
1353
1354	udc = container_of(gadget, struct mv_udc, gadget);
1355
1356	if (udc->driver)
1357	return -EBUSY;
1358
1359	spin_lock_irqsave(&udc->lock, flags);
1360
1361	/* hook up the driver ... */
1362	udc->driver = driver;
1363
1364	udc->usb_state = USB_STATE_ATTACHED;
1365	udc->ep0_state = WAIT_FOR_SETUP;
1366	udc->ep0_dir = EP_DIR_OUT;
1367
1368	spin_unlock_irqrestore(lock: &udc->lock, flags);
1369
1370	if (udc->transceiver) {
1371	retval = otg_set_peripheral(otg: udc->transceiver->otg,
1372	periph: &udc->gadget);
1373	if (retval) {
1374	dev_err(&udc->dev->dev,
1375	"unable to register peripheral to otg\n");
1376	udc->driver = NULL;
1377	return retval;
1378	}
1379	}
1380
1381	/* When boot with cable attached, there will be no vbus irq occurred */
1382	if (udc->qwork)
1383	queue_work(wq: udc->qwork, work: &udc->vbus_work);
1384
1385	return 0;
1386	}
1387
1388	static int mv_udc_stop(struct usb_gadget *gadget)
1389	{
1390	struct mv_udc *udc;
1391	unsigned long flags;
1392
1393	udc = container_of(gadget, struct mv_udc, gadget);
1394
1395	spin_lock_irqsave(&udc->lock, flags);
1396
1397	mv_udc_enable(udc);
1398	udc_stop(udc);
1399
1400	/* stop all usb activities */
1401	udc->gadget.speed = USB_SPEED_UNKNOWN;
1402	stop_activity(udc, NULL);
1403	mv_udc_disable(udc);
1404
1405	spin_unlock_irqrestore(lock: &udc->lock, flags);
1406
1407	/* unbind gadget driver */
1408	udc->driver = NULL;
1409
1410	return 0;
1411	}
1412
1413	static void mv_set_ptc(struct mv_udc *udc, u32 mode)
1414	{
1415	u32 portsc;
1416
1417	portsc = readl(addr: &udc->op_regs->portsc[0]);
1418	portsc |= mode << 16;
1419	writel(val: portsc, addr: &udc->op_regs->portsc[0]);
1420	}
1421
1422	static void prime_status_complete(struct usb_ep *ep, struct usb_request *_req)
1423	{
1424	struct mv_ep *mvep = container_of(ep, struct mv_ep, ep);
1425	struct mv_req *req = container_of(_req, struct mv_req, req);
1426	struct mv_udc *udc;
1427	unsigned long flags;
1428
1429	udc = mvep->udc;
1430
1431	dev_info(&udc->dev->dev, "switch to test mode %d\n", req->test_mode);
1432
1433	spin_lock_irqsave(&udc->lock, flags);
1434	if (req->test_mode) {
1435	mv_set_ptc(udc, mode: req->test_mode);
1436	req->test_mode = 0;
1437	}
1438	spin_unlock_irqrestore(lock: &udc->lock, flags);
1439	}
1440
1441	static int
1442	udc_prime_status(struct mv_udc *udc, u8 direction, u16 status, bool empty)
1443	{
1444	int retval = 0;
1445	struct mv_req *req;
1446	struct mv_ep *ep;
1447
1448	ep = &udc->eps[0];
1449	udc->ep0_dir = direction;
1450	udc->ep0_state = WAIT_FOR_OUT_STATUS;
1451
1452	req = udc->status_req;
1453
1454	/* fill in the request structure */
1455	if (empty == false) {
1456	*((u16 *) req->req.buf) = cpu_to_le16(status);
1457	req->req.length = 2;
1458	} else
1459	req->req.length = 0;
1460
1461	req->ep = ep;
1462	req->req.status = -EINPROGRESS;
1463	req->req.actual = 0;
1464	if (udc->test_mode) {
1465	req->req.complete = prime_status_complete;
1466	req->test_mode = udc->test_mode;
1467	udc->test_mode = 0;
1468	} else
1469	req->req.complete = NULL;
1470	req->dtd_count = 0;
1471
1472	if (req->req.dma == DMA_ADDR_INVALID) {
1473	req->req.dma = dma_map_single(ep->udc->gadget.dev.parent,
1474	req->req.buf, req->req.length,
1475	ep_dir(ep) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
1476	req->mapped = 1;
1477	}
1478
1479	/* prime the data phase */
1480	if (!req_to_dtd(req)) {
1481	retval = queue_dtd(ep, req);
1482	if (retval) {
1483	dev_err(&udc->dev->dev,
1484	"Failed to queue dtd when prime status\n");
1485	goto out;
1486	}
1487	} else{ /* no mem */
1488	retval = -ENOMEM;
1489	dev_err(&udc->dev->dev,
1490	"Failed to dma_pool_alloc when prime status\n");
1491	goto out;
1492	}
1493
1494	list_add_tail(new: &req->queue, head: &ep->queue);
1495
1496	return 0;
1497	out:
1498	usb_gadget_unmap_request(gadget: &udc->gadget, req: &req->req, ep_dir(ep));
1499
1500	return retval;
1501	}
1502
1503	static void mv_udc_testmode(struct mv_udc *udc, u16 index)
1504	{
1505	if (index <= USB_TEST_FORCE_ENABLE) {
1506	udc->test_mode = index;
1507	if (udc_prime_status(udc, EP_DIR_IN, status: 0, empty: true))
1508	ep0_stall(udc);
1509	} else
1510	dev_err(&udc->dev->dev,
1511	"This test mode(%d) is not supported\n", index);
1512	}
1513
1514	static void ch9setaddress(struct mv_udc *udc, struct usb_ctrlrequest *setup)
1515	{
1516	udc->dev_addr = (u8)setup->wValue;
1517
1518	/* update usb state */
1519	udc->usb_state = USB_STATE_ADDRESS;
1520
1521	if (udc_prime_status(udc, EP_DIR_IN, status: 0, empty: true))
1522	ep0_stall(udc);
1523	}
1524
1525	static void ch9getstatus(struct mv_udc *udc, u8 ep_num,
1526	struct usb_ctrlrequest *setup)
1527	{
1528	u16 status = 0;
1529	int retval;
1530
1531	if ((setup->bRequestType & (USB_DIR_IN | USB_TYPE_MASK))
1532	!= (USB_DIR_IN | USB_TYPE_STANDARD))
1533	return;
1534
1535	if ((setup->bRequestType & USB_RECIP_MASK) == USB_RECIP_DEVICE) {
1536	status = 1 << USB_DEVICE_SELF_POWERED;
1537	status |= udc->remote_wakeup << USB_DEVICE_REMOTE_WAKEUP;
1538	} else if ((setup->bRequestType & USB_RECIP_MASK)
1539	== USB_RECIP_INTERFACE) {
1540	/* get interface status */
1541	status = 0;
1542	} else if ((setup->bRequestType & USB_RECIP_MASK)
1543	== USB_RECIP_ENDPOINT) {
1544	u8 ep_num, direction;
1545
1546	ep_num = setup->wIndex & USB_ENDPOINT_NUMBER_MASK;
1547	direction = (setup->wIndex & USB_ENDPOINT_DIR_MASK)
1548	? EP_DIR_IN : EP_DIR_OUT;
1549	status = ep_is_stall(udc, ep_num, direction)
1550	<< USB_ENDPOINT_HALT;
1551	}
1552
1553	retval = udc_prime_status(udc, EP_DIR_IN, status, empty: false);
1554	if (retval)
1555	ep0_stall(udc);
1556	else
1557	udc->ep0_state = DATA_STATE_XMIT;
1558	}
1559
1560	static void ch9clearfeature(struct mv_udc *udc, struct usb_ctrlrequest *setup)
1561	{
1562	u8 ep_num;
1563	u8 direction;
1564	struct mv_ep *ep;
1565
1566	if ((setup->bRequestType & (USB_TYPE_MASK | USB_RECIP_MASK))
1567	== ((USB_TYPE_STANDARD | USB_RECIP_DEVICE))) {
1568	switch (setup->wValue) {
1569	case USB_DEVICE_REMOTE_WAKEUP:
1570	udc->remote_wakeup = 0;
1571	break;
1572	default:
1573	goto out;
1574	}
1575	} else if ((setup->bRequestType & (USB_TYPE_MASK | USB_RECIP_MASK))
1576	== ((USB_TYPE_STANDARD | USB_RECIP_ENDPOINT))) {
1577	switch (setup->wValue) {
1578	case USB_ENDPOINT_HALT:
1579	ep_num = setup->wIndex & USB_ENDPOINT_NUMBER_MASK;
1580	direction = (setup->wIndex & USB_ENDPOINT_DIR_MASK)
1581	? EP_DIR_IN : EP_DIR_OUT;
1582	if (setup->wValue != 0 || setup->wLength != 0
1583	|| ep_num > udc->max_eps)
1584	goto out;
1585	ep = &udc->eps[ep_num * 2 + direction];
1586	if (ep->wedge == 1)
1587	break;
1588	spin_unlock(lock: &udc->lock);
1589	ep_set_stall(udc, ep_num, direction, stall: 0);
1590	spin_lock(lock: &udc->lock);
1591	break;
1592	default:
1593	goto out;
1594	}
1595	} else
1596	goto out;
1597
1598	if (udc_prime_status(udc, EP_DIR_IN, status: 0, empty: true))
1599	ep0_stall(udc);
1600	out:
1601	return;
1602	}
1603
1604	static void ch9setfeature(struct mv_udc *udc, struct usb_ctrlrequest *setup)
1605	{
1606	u8 ep_num;
1607	u8 direction;
1608
1609	if ((setup->bRequestType & (USB_TYPE_MASK | USB_RECIP_MASK))
1610	== ((USB_TYPE_STANDARD | USB_RECIP_DEVICE))) {
1611	switch (setup->wValue) {
1612	case USB_DEVICE_REMOTE_WAKEUP:
1613	udc->remote_wakeup = 1;
1614	break;
1615	case USB_DEVICE_TEST_MODE:
1616	if (setup->wIndex & 0xFF
1617	|| udc->gadget.speed != USB_SPEED_HIGH)
1618	ep0_stall(udc);
1619
1620	if (udc->usb_state != USB_STATE_CONFIGURED
1621	&& udc->usb_state != USB_STATE_ADDRESS
1622	&& udc->usb_state != USB_STATE_DEFAULT)
1623	ep0_stall(udc);
1624
1625	mv_udc_testmode(udc, index: (setup->wIndex >> 8));
1626	goto out;
1627	default:
1628	goto out;
1629	}
1630	} else if ((setup->bRequestType & (USB_TYPE_MASK | USB_RECIP_MASK))
1631	== ((USB_TYPE_STANDARD | USB_RECIP_ENDPOINT))) {
1632	switch (setup->wValue) {
1633	case USB_ENDPOINT_HALT:
1634	ep_num = setup->wIndex & USB_ENDPOINT_NUMBER_MASK;
1635	direction = (setup->wIndex & USB_ENDPOINT_DIR_MASK)
1636	? EP_DIR_IN : EP_DIR_OUT;
1637	if (setup->wValue != 0 || setup->wLength != 0
1638	|| ep_num > udc->max_eps)
1639	goto out;
1640	spin_unlock(lock: &udc->lock);
1641	ep_set_stall(udc, ep_num, direction, stall: 1);
1642	spin_lock(lock: &udc->lock);
1643	break;
1644	default:
1645	goto out;
1646	}
1647	} else
1648	goto out;
1649
1650	if (udc_prime_status(udc, EP_DIR_IN, status: 0, empty: true))
1651	ep0_stall(udc);
1652	out:
1653	return;
1654	}
1655
1656	static void handle_setup_packet(struct mv_udc *udc, u8 ep_num,
1657	struct usb_ctrlrequest *setup)
1658	__releases(&ep->udc->lock)
1659	__acquires(&ep->udc->lock)
1660	{
1661	bool delegate = false;
1662
1663	nuke(ep: &udc->eps[ep_num * 2 + EP_DIR_OUT], status: -ESHUTDOWN);
1664
1665	dev_dbg(&udc->dev->dev, "SETUP %02x.%02x v%04x i%04x l%04x\n",
1666	setup->bRequestType, setup->bRequest,
1667	setup->wValue, setup->wIndex, setup->wLength);
1668	/* We process some standard setup requests here */
1669	if ((setup->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD) {
1670	switch (setup->bRequest) {
1671	case USB_REQ_GET_STATUS:
1672	ch9getstatus(udc, ep_num, setup);
1673	break;
1674
1675	case USB_REQ_SET_ADDRESS:
1676	ch9setaddress(udc, setup);
1677	break;
1678
1679	case USB_REQ_CLEAR_FEATURE:
1680	ch9clearfeature(udc, setup);
1681	break;
1682
1683	case USB_REQ_SET_FEATURE:
1684	ch9setfeature(udc, setup);
1685	break;
1686
1687	default:
1688	delegate = true;
1689	}
1690	} else
1691	delegate = true;
1692
1693	/* delegate USB standard requests to the gadget driver */
1694	if (delegate == true) {
1695	/* USB requests handled by gadget */
1696	if (setup->wLength) {
1697	/* DATA phase from gadget, STATUS phase from udc */
1698	udc->ep0_dir = (setup->bRequestType & USB_DIR_IN)
1699	? EP_DIR_IN : EP_DIR_OUT;
1700	spin_unlock(lock: &udc->lock);
1701	if (udc->driver->setup(&udc->gadget,
1702	&udc->local_setup_buff) < 0)
1703	ep0_stall(udc);
1704	spin_lock(lock: &udc->lock);
1705	udc->ep0_state = (setup->bRequestType & USB_DIR_IN)
1706	? DATA_STATE_XMIT : DATA_STATE_RECV;
1707	} else {
1708	/* no DATA phase, IN STATUS phase from gadget */
1709	udc->ep0_dir = EP_DIR_IN;
1710	spin_unlock(lock: &udc->lock);
1711	if (udc->driver->setup(&udc->gadget,
1712	&udc->local_setup_buff) < 0)
1713	ep0_stall(udc);
1714	spin_lock(lock: &udc->lock);
1715	udc->ep0_state = WAIT_FOR_OUT_STATUS;
1716	}
1717	}
1718	}
1719
1720	/* complete DATA or STATUS phase of ep0 prime status phase if needed */
1721	static void ep0_req_complete(struct mv_udc *udc,
1722	struct mv_ep *ep0, struct mv_req *req)
1723	{
1724	u32 new_addr;
1725
1726	if (udc->usb_state == USB_STATE_ADDRESS) {
1727	/* set the new address */
1728	new_addr = (u32)udc->dev_addr;
1729	writel(val: new_addr << USB_DEVICE_ADDRESS_BIT_SHIFT,
1730	addr: &udc->op_regs->deviceaddr);
1731	}
1732
1733	done(ep: ep0, req, status: 0);
1734
1735	switch (udc->ep0_state) {
1736	case DATA_STATE_XMIT:
1737	/* receive status phase */
1738	if (udc_prime_status(udc, EP_DIR_OUT, status: 0, empty: true))
1739	ep0_stall(udc);
1740	break;
1741	case DATA_STATE_RECV:
1742	/* send status phase */
1743	if (udc_prime_status(udc, EP_DIR_IN, status: 0 , empty: true))
1744	ep0_stall(udc);
1745	break;
1746	case WAIT_FOR_OUT_STATUS:
1747	udc->ep0_state = WAIT_FOR_SETUP;
1748	break;
1749	case WAIT_FOR_SETUP:
1750	dev_err(&udc->dev->dev, "unexpect ep0 packets\n");
1751	break;
1752	default:
1753	ep0_stall(udc);
1754	break;
1755	}
1756	}
1757
1758	static void get_setup_data(struct mv_udc *udc, u8 ep_num, u8 *buffer_ptr)
1759	{
1760	u32 temp;
1761	struct mv_dqh *dqh;
1762
1763	dqh = &udc->ep_dqh[ep_num * 2 + EP_DIR_OUT];
1764
1765	/* Clear bit in ENDPTSETUPSTAT */
1766	writel(val: (1 << ep_num), addr: &udc->op_regs->epsetupstat);
1767
1768	/* while a hazard exists when setup package arrives */
1769	do {
1770	/* Set Setup Tripwire */
1771	temp = readl(addr: &udc->op_regs->usbcmd);
1772	writel(val: temp | USBCMD_SETUP_TRIPWIRE_SET, addr: &udc->op_regs->usbcmd);
1773
1774	/* Copy the setup packet to local buffer */
1775	memcpy(buffer_ptr, (u8 *) dqh->setup_buffer, 8);
1776	} while (!(readl(addr: &udc->op_regs->usbcmd) & USBCMD_SETUP_TRIPWIRE_SET));
1777
1778	/* Clear Setup Tripwire */
1779	temp = readl(addr: &udc->op_regs->usbcmd);
1780	writel(val: temp & ~USBCMD_SETUP_TRIPWIRE_SET, addr: &udc->op_regs->usbcmd);
1781	}
1782
1783	static void irq_process_tr_complete(struct mv_udc *udc)
1784	{
1785	u32 tmp, bit_pos;
1786	int i, ep_num = 0, direction = 0;
1787	struct mv_ep *curr_ep;
1788	struct mv_req *curr_req, *temp_req;
1789	int status;
1790
1791	/*
1792	* We use separate loops for ENDPTSETUPSTAT and ENDPTCOMPLETE
1793	* because the setup packets are to be read ASAP
1794	*/
1795
1796	/* Process all Setup packet received interrupts */
1797	tmp = readl(addr: &udc->op_regs->epsetupstat);
1798
1799	if (tmp) {
1800	for (i = 0; i < udc->max_eps; i++) {
1801	if (tmp & (1 << i)) {
1802	get_setup_data(udc, ep_num: i,
1803	buffer_ptr: (u8 *)(&udc->local_setup_buff));
1804	handle_setup_packet(udc, ep_num: i,
1805	setup: &udc->local_setup_buff);
1806	}
1807	}
1808	}
1809
1810	/* Don't clear the endpoint setup status register here.
1811	* It is cleared as a setup packet is read out of the buffer
1812	*/
1813
1814	/* Process non-setup transaction complete interrupts */
1815	tmp = readl(addr: &udc->op_regs->epcomplete);
1816
1817	if (!tmp)
1818	return;
1819
1820	writel(val: tmp, addr: &udc->op_regs->epcomplete);
1821
1822	for (i = 0; i < udc->max_eps * 2; i++) {
1823	ep_num = i >> 1;
1824	direction = i % 2;
1825
1826	bit_pos = 1 << (ep_num + 16 * direction);
1827
1828	if (!(bit_pos & tmp))
1829	continue;
1830
1831	if (i == 1)
1832	curr_ep = &udc->eps[0];
1833	else
1834	curr_ep = &udc->eps[i];
1835	/* process the req queue until an uncomplete request */
1836	list_for_each_entry_safe(curr_req, temp_req,
1837	&curr_ep->queue, queue) {
1838	status = process_ep_req(udc, index: i, curr_req);
1839	if (status)
1840	break;
1841
1842	/* write back status to req */
1843	curr_req->req.status = status;
1844
1845	/* ep0 request completion */
1846	if (ep_num == 0) {
1847	ep0_req_complete(udc, ep0: curr_ep, req: curr_req);
1848	break;
1849	} else {
1850	done(ep: curr_ep, req: curr_req, status);
1851	}
1852	}
1853	}
1854	}
1855
1856	static void irq_process_reset(struct mv_udc *udc)
1857	{
1858	u32 tmp;
1859	unsigned int loops;
1860
1861	udc->ep0_dir = EP_DIR_OUT;
1862	udc->ep0_state = WAIT_FOR_SETUP;
1863	udc->remote_wakeup = 0; /* default to 0 on reset */
1864
1865	/* The address bits are past bit 25-31. Set the address */
1866	tmp = readl(addr: &udc->op_regs->deviceaddr);
1867	tmp &= ~(USB_DEVICE_ADDRESS_MASK);
1868	writel(val: tmp, addr: &udc->op_regs->deviceaddr);
1869
1870	/* Clear all the setup token semaphores */
1871	tmp = readl(addr: &udc->op_regs->epsetupstat);
1872	writel(val: tmp, addr: &udc->op_regs->epsetupstat);
1873
1874	/* Clear all the endpoint complete status bits */
1875	tmp = readl(addr: &udc->op_regs->epcomplete);
1876	writel(val: tmp, addr: &udc->op_regs->epcomplete);
1877
1878	/* wait until all endptprime bits cleared */
1879	loops = LOOPS(PRIME_TIMEOUT);
1880	while (readl(addr: &udc->op_regs->epprime) & 0xFFFFFFFF) {
1881	if (loops == 0) {
1882	dev_err(&udc->dev->dev,
1883	"Timeout for ENDPTPRIME = 0x%x\n",
1884	readl(&udc->op_regs->epprime));
1885	break;
1886	}
1887	loops--;
1888	udelay(LOOPS_USEC);
1889	}
1890
1891	/* Write 1s to the Flush register */
1892	writel(val: (u32)~0, addr: &udc->op_regs->epflush);
1893
1894	if (readl(addr: &udc->op_regs->portsc[0]) & PORTSCX_PORT_RESET) {
1895	dev_info(&udc->dev->dev, "usb bus reset\n");
1896	udc->usb_state = USB_STATE_DEFAULT;
1897	/* reset all the queues, stop all USB activities */
1898	gadget_reset(udc, driver: udc->driver);
1899	} else {
1900	dev_info(&udc->dev->dev, "USB reset portsc 0x%x\n",
1901	readl(&udc->op_regs->portsc));
1902
1903	/*
1904	* re-initialize
1905	* controller reset
1906	*/
1907	udc_reset(udc);
1908
1909	/* reset all the queues, stop all USB activities */
1910	stop_activity(udc, driver: udc->driver);
1911
1912	/* reset ep0 dQH and endptctrl */
1913	ep0_reset(udc);
1914
1915	/* enable interrupt and set controller to run state */
1916	udc_start(udc);
1917
1918	udc->usb_state = USB_STATE_ATTACHED;
1919	}
1920	}
1921
1922	static void handle_bus_resume(struct mv_udc *udc)
1923	{
1924	udc->usb_state = udc->resume_state;
1925	udc->resume_state = 0;
1926
1927	/* report resume to the driver */
1928	if (udc->driver) {
1929	if (udc->driver->resume) {
1930	spin_unlock(lock: &udc->lock);
1931	udc->driver->resume(&udc->gadget);
1932	spin_lock(lock: &udc->lock);
1933	}
1934	}
1935	}
1936
1937	static void irq_process_suspend(struct mv_udc *udc)
1938	{
1939	udc->resume_state = udc->usb_state;
1940	udc->usb_state = USB_STATE_SUSPENDED;
1941
1942	if (udc->driver->suspend) {
1943	spin_unlock(lock: &udc->lock);
1944	udc->driver->suspend(&udc->gadget);
1945	spin_lock(lock: &udc->lock);
1946	}
1947	}
1948
1949	static void irq_process_port_change(struct mv_udc *udc)
1950	{
1951	u32 portsc;
1952
1953	portsc = readl(addr: &udc->op_regs->portsc[0]);
1954	if (!(portsc & PORTSCX_PORT_RESET)) {
1955	/* Get the speed */
1956	u32 speed = portsc & PORTSCX_PORT_SPEED_MASK;
1957	switch (speed) {
1958	case PORTSCX_PORT_SPEED_HIGH:
1959	udc->gadget.speed = USB_SPEED_HIGH;
1960	break;
1961	case PORTSCX_PORT_SPEED_FULL:
1962	udc->gadget.speed = USB_SPEED_FULL;
1963	break;
1964	case PORTSCX_PORT_SPEED_LOW:
1965	udc->gadget.speed = USB_SPEED_LOW;
1966	break;
1967	default:
1968	udc->gadget.speed = USB_SPEED_UNKNOWN;
1969	break;
1970	}
1971	}
1972
1973	if (portsc & PORTSCX_PORT_SUSPEND) {
1974	udc->resume_state = udc->usb_state;
1975	udc->usb_state = USB_STATE_SUSPENDED;
1976	if (udc->driver->suspend) {
1977	spin_unlock(lock: &udc->lock);
1978	udc->driver->suspend(&udc->gadget);
1979	spin_lock(lock: &udc->lock);
1980	}
1981	}
1982
1983	if (!(portsc & PORTSCX_PORT_SUSPEND)
1984	&& udc->usb_state == USB_STATE_SUSPENDED) {
1985	handle_bus_resume(udc);
1986	}
1987
1988	if (!udc->resume_state)
1989	udc->usb_state = USB_STATE_DEFAULT;
1990	}
1991
1992	static void irq_process_error(struct mv_udc *udc)
1993	{
1994	/* Increment the error count */
1995	udc->errors++;
1996	}
1997
1998	static irqreturn_t mv_udc_irq(int irq, void *dev)
1999	{
2000	struct mv_udc *udc = (struct mv_udc *)dev;
2001	u32 status, intr;
2002
2003	/* Disable ISR when stopped bit is set */
2004	if (udc->stopped)
2005	return IRQ_NONE;
2006
2007	spin_lock(lock: &udc->lock);
2008
2009	status = readl(addr: &udc->op_regs->usbsts);
2010	intr = readl(addr: &udc->op_regs->usbintr);
2011	status &= intr;
2012
2013	if (status == 0) {
2014	spin_unlock(lock: &udc->lock);
2015	return IRQ_NONE;
2016	}
2017
2018	/* Clear all the interrupts occurred */
2019	writel(val: status, addr: &udc->op_regs->usbsts);
2020
2021	if (status & USBSTS_ERR)
2022	irq_process_error(udc);
2023
2024	if (status & USBSTS_RESET)
2025	irq_process_reset(udc);
2026
2027	if (status & USBSTS_PORT_CHANGE)
2028	irq_process_port_change(udc);
2029
2030	if (status & USBSTS_INT)
2031	irq_process_tr_complete(udc);
2032
2033	if (status & USBSTS_SUSPEND)
2034	irq_process_suspend(udc);
2035
2036	spin_unlock(lock: &udc->lock);
2037
2038	return IRQ_HANDLED;
2039	}
2040
2041	static irqreturn_t mv_udc_vbus_irq(int irq, void *dev)
2042	{
2043	struct mv_udc *udc = (struct mv_udc *)dev;
2044
2045	/* polling VBUS and init phy may cause too much time*/
2046	if (udc->qwork)
2047	queue_work(wq: udc->qwork, work: &udc->vbus_work);
2048
2049	return IRQ_HANDLED;
2050	}
2051
2052	static void mv_udc_vbus_work(struct work_struct *work)
2053	{
2054	struct mv_udc *udc;
2055	unsigned int vbus;
2056
2057	udc = container_of(work, struct mv_udc, vbus_work);
2058	if (!udc->pdata->vbus)
2059	return;
2060
2061	vbus = udc->pdata->vbus->poll();
2062	dev_info(&udc->dev->dev, "vbus is %d\n", vbus);
2063
2064	if (vbus == VBUS_HIGH)
2065	mv_udc_vbus_session(gadget: &udc->gadget, is_active: 1);
2066	else if (vbus == VBUS_LOW)
2067	mv_udc_vbus_session(gadget: &udc->gadget, is_active: 0);
2068	}
2069
2070	/* release device structure */
2071	static void gadget_release(struct device *_dev)
2072	{
2073	struct mv_udc *udc;
2074
2075	udc = dev_get_drvdata(dev: _dev);
2076
2077	complete(udc->done);
2078	}
2079
2080	static void mv_udc_remove(struct platform_device *pdev)
2081	{
2082	struct mv_udc *udc;
2083
2084	udc = platform_get_drvdata(pdev);
2085
2086	usb_del_gadget_udc(gadget: &udc->gadget);
2087
2088	if (udc->qwork)
2089	destroy_workqueue(wq: udc->qwork);
2090
2091	/* free memory allocated in probe */
2092	dma_pool_destroy(pool: udc->dtd_pool);
2093
2094	if (udc->ep_dqh)
2095	dma_free_coherent(dev: &pdev->dev, size: udc->ep_dqh_size,
2096	cpu_addr: udc->ep_dqh, dma_handle: udc->ep_dqh_dma);
2097
2098	mv_udc_disable(udc);
2099
2100	/* free dev, wait for the release() finished */
2101	wait_for_completion(udc->done);
2102	}
2103
2104	static int mv_udc_probe(struct platform_device *pdev)
2105	{
2106	struct mv_usb_platform_data *pdata = dev_get_platdata(dev: &pdev->dev);
2107	struct mv_udc *udc;
2108	int retval = 0;
2109	struct resource *r;
2110	size_t size;
2111
2112	if (pdata == NULL) {
2113	dev_err(&pdev->dev, "missing platform_data\n");
2114	return -ENODEV;
2115	}
2116
2117	udc = devm_kzalloc(dev: &pdev->dev, size: sizeof(*udc), GFP_KERNEL);
2118	if (udc == NULL)
2119	return -ENOMEM;
2120
2121	udc->done = &release_done;
2122	udc->pdata = dev_get_platdata(dev: &pdev->dev);
2123	spin_lock_init(&udc->lock);
2124
2125	udc->dev = pdev;
2126
2127	if (pdata->mode == MV_USB_MODE_OTG) {
2128	udc->transceiver = devm_usb_get_phy(dev: &pdev->dev,
2129	type: USB_PHY_TYPE_USB2);
2130	if (IS_ERR(ptr: udc->transceiver)) {
2131	retval = PTR_ERR(ptr: udc->transceiver);
2132
2133	if (retval == -ENXIO)
2134	return retval;
2135
2136	udc->transceiver = NULL;
2137	return -EPROBE_DEFER;
2138	}
2139	}
2140
2141	/* udc only have one sysclk. */
2142	udc->clk = devm_clk_get(dev: &pdev->dev, NULL);
2143	if (IS_ERR(ptr: udc->clk))
2144	return PTR_ERR(ptr: udc->clk);
2145
2146	r = platform_get_resource_byname(udc->dev, IORESOURCE_MEM, "capregs");
2147	if (r == NULL) {
2148	dev_err(&pdev->dev, "no I/O memory resource defined\n");
2149	return -ENODEV;
2150	}
2151
2152	udc->cap_regs = (struct mv_cap_regs __iomem *)
2153	devm_ioremap(dev: &pdev->dev, offset: r->start, size: resource_size(res: r));
2154	if (udc->cap_regs == NULL) {
2155	dev_err(&pdev->dev, "failed to map I/O memory\n");
2156	return -EBUSY;
2157	}
2158
2159	r = platform_get_resource_byname(udc->dev, IORESOURCE_MEM, "phyregs");
2160	if (r == NULL) {
2161	dev_err(&pdev->dev, "no phy I/O memory resource defined\n");
2162	return -ENODEV;
2163	}
2164
2165	udc->phy_regs = devm_ioremap(dev: &pdev->dev, offset: r->start, size: resource_size(res: r));
2166	if (udc->phy_regs == NULL) {
2167	dev_err(&pdev->dev, "failed to map phy I/O memory\n");
2168	return -EBUSY;
2169	}
2170
2171	/* we will acces controller register, so enable the clk */
2172	retval = mv_udc_enable_internal(udc);
2173	if (retval)
2174	return retval;
2175
2176	udc->op_regs =
2177	(struct mv_op_regs __iomem *)((unsigned long)udc->cap_regs
2178	+ (readl(addr: &udc->cap_regs->caplength_hciversion)
2179	& CAPLENGTH_MASK));
2180	udc->max_eps = readl(addr: &udc->cap_regs->dccparams) & DCCPARAMS_DEN_MASK;
2181
2182	/*
2183	* some platform will use usb to download image, it may not disconnect
2184	* usb gadget before loading kernel. So first stop udc here.
2185	*/
2186	udc_stop(udc);
2187	writel(val: 0xFFFFFFFF, addr: &udc->op_regs->usbsts);
2188
2189	size = udc->max_eps * sizeof(struct mv_dqh) *2;
2190	size = (size + DQH_ALIGNMENT - 1) & ~(DQH_ALIGNMENT - 1);
2191	udc->ep_dqh = dma_alloc_coherent(dev: &pdev->dev, size,
2192	dma_handle: &udc->ep_dqh_dma, GFP_KERNEL);
2193
2194	if (udc->ep_dqh == NULL) {
2195	dev_err(&pdev->dev, "allocate dQH memory failed\n");
2196	retval = -ENOMEM;
2197	goto err_disable_clock;
2198	}
2199	udc->ep_dqh_size = size;
2200
2201	/* create dTD dma_pool resource */
2202	udc->dtd_pool = dma_pool_create(name: "mv_dtd",
2203	dev: &pdev->dev,
2204	size: sizeof(struct mv_dtd),
2205	DTD_ALIGNMENT,
2206	DMA_BOUNDARY);
2207
2208	if (!udc->dtd_pool) {
2209	retval = -ENOMEM;
2210	goto err_free_dma;
2211	}
2212
2213	size = udc->max_eps * sizeof(struct mv_ep) *2;
2214	udc->eps = devm_kzalloc(dev: &pdev->dev, size, GFP_KERNEL);
2215	if (udc->eps == NULL) {
2216	retval = -ENOMEM;
2217	goto err_destroy_dma;
2218	}
2219
2220	/* initialize ep0 status request structure */
2221	udc->status_req = devm_kzalloc(dev: &pdev->dev, size: sizeof(struct mv_req),
2222	GFP_KERNEL);
2223	if (!udc->status_req) {
2224	retval = -ENOMEM;
2225	goto err_destroy_dma;
2226	}
2227	INIT_LIST_HEAD(list: &udc->status_req->queue);
2228
2229	/* allocate a small amount of memory to get valid address */
2230	udc->status_req->req.buf = devm_kzalloc(dev: &pdev->dev, size: 8, GFP_KERNEL);
2231	if (!udc->status_req->req.buf) {
2232	retval = -ENOMEM;
2233	goto err_destroy_dma;
2234	}
2235	udc->status_req->req.dma = DMA_ADDR_INVALID;
2236
2237	udc->resume_state = USB_STATE_NOTATTACHED;
2238	udc->usb_state = USB_STATE_POWERED;
2239	udc->ep0_dir = EP_DIR_OUT;
2240	udc->remote_wakeup = 0;
2241
2242	r = platform_get_resource(udc->dev, IORESOURCE_IRQ, 0);
2243	if (r == NULL) {
2244	dev_err(&pdev->dev, "no IRQ resource defined\n");
2245	retval = -ENODEV;
2246	goto err_destroy_dma;
2247	}
2248	udc->irq = r->start;
2249	if (devm_request_irq(dev: &pdev->dev, irq: udc->irq, handler: mv_udc_irq,
2250	IRQF_SHARED, devname: driver_name, dev_id: udc)) {
2251	dev_err(&pdev->dev, "Request irq %d for UDC failed\n",
2252	udc->irq);
2253	retval = -ENODEV;
2254	goto err_destroy_dma;
2255	}
2256
2257	/* initialize gadget structure */
2258	udc->gadget.ops = &mv_ops; /* usb_gadget_ops */
2259	udc->gadget.ep0 = &udc->eps[0].ep; /* gadget ep0 */
2260	INIT_LIST_HEAD(list: &udc->gadget.ep_list); /* ep_list */
2261	udc->gadget.speed = USB_SPEED_UNKNOWN; /* speed */
2262	udc->gadget.max_speed = USB_SPEED_HIGH; /* support dual speed */
2263
2264	/* the "gadget" abstracts/virtualizes the controller */
2265	udc->gadget.name = driver_name; /* gadget name */
2266
2267	eps_init(udc);
2268
2269	/* VBUS detect: we can disable/enable clock on demand.*/
2270	if (udc->transceiver)
2271	udc->clock_gating = 1;
2272	else if (pdata->vbus) {
2273	udc->clock_gating = 1;
2274	retval = devm_request_threaded_irq(dev: &pdev->dev,
2275	irq: pdata->vbus->irq, NULL,
2276	thread_fn: mv_udc_vbus_irq, IRQF_ONESHOT, devname: "vbus", dev_id: udc);
2277	if (retval) {
2278	dev_info(&pdev->dev,
2279	"Can not request irq for VBUS, "
2280	"disable clock gating\n");
2281	udc->clock_gating = 0;
2282	}
2283
2284	udc->qwork = create_singlethread_workqueue("mv_udc_queue");
2285	if (!udc->qwork) {
2286	dev_err(&pdev->dev, "cannot create workqueue\n");
2287	retval = -ENOMEM;
2288	goto err_destroy_dma;
2289	}
2290
2291	INIT_WORK(&udc->vbus_work, mv_udc_vbus_work);
2292	}
2293
2294	/*
2295	* When clock gating is supported, we can disable clk and phy.
2296	* If not, it means that VBUS detection is not supported, we
2297	* have to enable vbus active all the time to let controller work.
2298	*/
2299	if (udc->clock_gating)
2300	mv_udc_disable_internal(udc);
2301	else
2302	udc->vbus_active = 1;
2303
2304	retval = usb_add_gadget_udc_release(parent: &pdev->dev, gadget: &udc->gadget,
2305	release: gadget_release);
2306	if (retval)
2307	goto err_create_workqueue;
2308
2309	platform_set_drvdata(pdev, data: udc);
2310	dev_info(&pdev->dev, "successful probe UDC device %s clock gating.\n",
2311	udc->clock_gating ? "with" : "without");
2312
2313	return 0;
2314
2315	err_create_workqueue:
2316	if (udc->qwork)
2317	destroy_workqueue(wq: udc->qwork);
2318	err_destroy_dma:
2319	dma_pool_destroy(pool: udc->dtd_pool);
2320	err_free_dma:
2321	dma_free_coherent(dev: &pdev->dev, size: udc->ep_dqh_size,
2322	cpu_addr: udc->ep_dqh, dma_handle: udc->ep_dqh_dma);
2323	err_disable_clock:
2324	mv_udc_disable_internal(udc);
2325
2326	return retval;
2327	}
2328
2329	#ifdef CONFIG_PM
2330	static int mv_udc_suspend(struct device *dev)
2331	{
2332	struct mv_udc *udc;
2333
2334	udc = dev_get_drvdata(dev);
2335
2336	/* if OTG is enabled, the following will be done in OTG driver*/
2337	if (udc->transceiver)
2338	return 0;
2339
2340	if (udc->pdata->vbus && udc->pdata->vbus->poll)
2341	if (udc->pdata->vbus->poll() == VBUS_HIGH) {
2342	dev_info(&udc->dev->dev, "USB cable is connected!\n");
2343	return -EAGAIN;
2344	}
2345
2346	/*
2347	* only cable is unplugged, udc can suspend.
2348	* So do not care about clock_gating == 1.
2349	*/
2350	if (!udc->clock_gating) {
2351	udc_stop(udc);
2352
2353	spin_lock_irq(lock: &udc->lock);
2354	/* stop all usb activities */
2355	stop_activity(udc, driver: udc->driver);
2356	spin_unlock_irq(lock: &udc->lock);
2357
2358	mv_udc_disable_internal(udc);
2359	}
2360
2361	return 0;
2362	}
2363
2364	static int mv_udc_resume(struct device *dev)
2365	{
2366	struct mv_udc *udc;
2367	int retval;
2368
2369	udc = dev_get_drvdata(dev);
2370
2371	/* if OTG is enabled, the following will be done in OTG driver*/
2372	if (udc->transceiver)
2373	return 0;
2374
2375	if (!udc->clock_gating) {
2376	retval = mv_udc_enable_internal(udc);
2377	if (retval)
2378	return retval;
2379
2380	if (udc->driver && udc->softconnect) {
2381	udc_reset(udc);
2382	ep0_reset(udc);
2383	udc_start(udc);
2384	}
2385	}
2386
2387	return 0;
2388	}
2389
2390	static const struct dev_pm_ops mv_udc_pm_ops = {
2391	.suspend = mv_udc_suspend,
2392	.resume = mv_udc_resume,
2393	};
2394	#endif
2395
2396	static void mv_udc_shutdown(struct platform_device *pdev)
2397	{
2398	struct mv_udc *udc;
2399	u32 mode;
2400
2401	udc = platform_get_drvdata(pdev);
2402	/* reset controller mode to IDLE */
2403	mv_udc_enable(udc);
2404	mode = readl(addr: &udc->op_regs->usbmode);
2405	mode &= ~3;
2406	writel(val: mode, addr: &udc->op_regs->usbmode);
2407	mv_udc_disable(udc);
2408	}
2409
2410	static struct platform_driver udc_driver = {
2411	.probe = mv_udc_probe,
2412	.remove_new = mv_udc_remove,
2413	.shutdown = mv_udc_shutdown,
2414	.driver = {
2415	.name = "mv-udc",
2416	#ifdef CONFIG_PM
2417	.pm = &mv_udc_pm_ops,
2418	#endif
2419	},
2420	};
2421
2422	module_platform_driver(udc_driver);
2423	MODULE_ALIAS("platform:mv-udc");
2424	MODULE_DESCRIPTION(DRIVER_DESC);
2425	MODULE_AUTHOR("Chao Xie <chao.xie@marvell.com>");
2426	MODULE_LICENSE("GPL");
2427