1	// SPDX-License-Identifier: GPL-2.0+
2	/*
3	* Xilinx USB peripheral controller driver
4	*
5	* Copyright (C) 2004 by Thomas Rathbone
6	* Copyright (C) 2005 by HP Labs
7	* Copyright (C) 2005 by David Brownell
8	* Copyright (C) 2010 - 2014 Xilinx, Inc.
9	*
10	* Some parts of this driver code is based on the driver for at91-series
11	* USB peripheral controller (at91_udc.c).
12	*/
13
14	#include <linux/clk.h>
15	#include <linux/delay.h>
16	#include <linux/device.h>
17	#include <linux/dma-mapping.h>
18	#include <linux/interrupt.h>
19	#include <linux/io.h>
20	#include <linux/module.h>
21	#include <linux/of.h>
22	#include <linux/platform_device.h>
23	#include <linux/prefetch.h>
24	#include <linux/usb/ch9.h>
25	#include <linux/usb/gadget.h>
26
27	/* Register offsets for the USB device.*/
28	#define XUSB_EP0_CONFIG_OFFSET 0x0000 /* EP0 Config Reg Offset */
29	#define XUSB_SETUP_PKT_ADDR_OFFSET 0x0080 /* Setup Packet Address */
30	#define XUSB_ADDRESS_OFFSET 0x0100 /* Address Register */
31	#define XUSB_CONTROL_OFFSET 0x0104 /* Control Register */
32	#define XUSB_STATUS_OFFSET 0x0108 /* Status Register */
33	#define XUSB_FRAMENUM_OFFSET 0x010C /* Frame Number Register */
34	#define XUSB_IER_OFFSET 0x0110 /* Interrupt Enable Register */
35	#define XUSB_BUFFREADY_OFFSET 0x0114 /* Buffer Ready Register */
36	#define XUSB_TESTMODE_OFFSET 0x0118 /* Test Mode Register */
37	#define XUSB_DMA_RESET_OFFSET 0x0200 /* DMA Soft Reset Register */
38	#define XUSB_DMA_CONTROL_OFFSET 0x0204 /* DMA Control Register */
39	#define XUSB_DMA_DSAR_ADDR_OFFSET 0x0208 /* DMA source Address Reg */
40	#define XUSB_DMA_DDAR_ADDR_OFFSET 0x020C /* DMA destination Addr Reg */
41	#define XUSB_DMA_LENGTH_OFFSET 0x0210 /* DMA Length Register */
42	#define XUSB_DMA_STATUS_OFFSET 0x0214 /* DMA Status Register */
43
44	/* Endpoint Configuration Space offsets */
45	#define XUSB_EP_CFGSTATUS_OFFSET 0x00 /* Endpoint Config Status */
46	#define XUSB_EP_BUF0COUNT_OFFSET 0x08 /* Buffer 0 Count */
47	#define XUSB_EP_BUF1COUNT_OFFSET 0x0C /* Buffer 1 Count */
48
49	#define XUSB_CONTROL_USB_READY_MASK 0x80000000 /* USB ready Mask */
50	#define XUSB_CONTROL_USB_RMTWAKE_MASK 0x40000000 /* Remote wake up mask */
51
52	/* Interrupt register related masks.*/
53	#define XUSB_STATUS_GLOBAL_INTR_MASK 0x80000000 /* Global Intr Enable */
54	#define XUSB_STATUS_DMADONE_MASK 0x04000000 /* DMA done Mask */
55	#define XUSB_STATUS_DMAERR_MASK 0x02000000 /* DMA Error Mask */
56	#define XUSB_STATUS_DMABUSY_MASK 0x80000000 /* DMA Error Mask */
57	#define XUSB_STATUS_RESUME_MASK 0x01000000 /* USB Resume Mask */
58	#define XUSB_STATUS_RESET_MASK 0x00800000 /* USB Reset Mask */
59	#define XUSB_STATUS_SUSPEND_MASK 0x00400000 /* USB Suspend Mask */
60	#define XUSB_STATUS_DISCONNECT_MASK 0x00200000 /* USB Disconnect Mask */
61	#define XUSB_STATUS_FIFO_BUFF_RDY_MASK 0x00100000 /* FIFO Buff Ready Mask */
62	#define XUSB_STATUS_FIFO_BUFF_FREE_MASK 0x00080000 /* FIFO Buff Free Mask */
63	#define XUSB_STATUS_SETUP_PACKET_MASK 0x00040000 /* Setup packet received */
64	#define XUSB_STATUS_EP1_BUFF2_COMP_MASK 0x00000200 /* EP 1 Buff 2 Processed */
65	#define XUSB_STATUS_EP1_BUFF1_COMP_MASK 0x00000002 /* EP 1 Buff 1 Processed */
66	#define XUSB_STATUS_EP0_BUFF2_COMP_MASK 0x00000100 /* EP 0 Buff 2 Processed */
67	#define XUSB_STATUS_EP0_BUFF1_COMP_MASK 0x00000001 /* EP 0 Buff 1 Processed */
68	#define XUSB_STATUS_HIGH_SPEED_MASK 0x00010000 /* USB Speed Mask */
69	/* Suspend,Reset,Suspend and Disconnect Mask */
70	#define XUSB_STATUS_INTR_EVENT_MASK 0x01E00000
71	/* Buffers completion Mask */
72	#define XUSB_STATUS_INTR_BUFF_COMP_ALL_MASK 0x0000FEFF
73	/* Mask for buffer 0 and buffer 1 completion for all Endpoints */
74	#define XUSB_STATUS_INTR_BUFF_COMP_SHIFT_MASK 0x00000101
75	#define XUSB_STATUS_EP_BUFF2_SHIFT 8 /* EP buffer offset */
76
77	/* Endpoint Configuration Status Register */
78	#define XUSB_EP_CFG_VALID_MASK 0x80000000 /* Endpoint Valid bit */
79	#define XUSB_EP_CFG_STALL_MASK 0x40000000 /* Endpoint Stall bit */
80	#define XUSB_EP_CFG_DATA_TOGGLE_MASK 0x08000000 /* Endpoint Data toggle */
81
82	/* USB device specific global configuration constants.*/
83	#define XUSB_MAX_ENDPOINTS 8 /* Maximum End Points */
84	#define XUSB_EP_NUMBER_ZERO 0 /* End point Zero */
85	/* DPRAM is the source address for DMA transfer */
86	#define XUSB_DMA_READ_FROM_DPRAM 0x80000000
87	#define XUSB_DMA_DMASR_BUSY 0x80000000 /* DMA busy */
88	#define XUSB_DMA_DMASR_ERROR 0x40000000 /* DMA Error */
89	/*
90	* When this bit is set, the DMA buffer ready bit is set by hardware upon
91	* DMA transfer completion.
92	*/
93	#define XUSB_DMA_BRR_CTRL 0x40000000 /* DMA bufready ctrl bit */
94	/* Phase States */
95	#define SETUP_PHASE 0x0000 /* Setup Phase */
96	#define DATA_PHASE 0x0001 /* Data Phase */
97	#define STATUS_PHASE 0x0002 /* Status Phase */
98
99	#define EP0_MAX_PACKET 64 /* Endpoint 0 maximum packet length */
100	#define STATUSBUFF_SIZE 2 /* Buffer size for GET_STATUS command */
101	#define EPNAME_SIZE 4 /* Buffer size for endpoint name */
102
103	/* container_of helper macros */
104	#define to_udc(g) container_of((g), struct xusb_udc, gadget)
105	#define to_xusb_ep(ep) container_of((ep), struct xusb_ep, ep_usb)
106	#define to_xusb_req(req) container_of((req), struct xusb_req, usb_req)
107
108	/**
109	* struct xusb_req - Xilinx USB device request structure
110	* @usb_req: Linux usb request structure
111	* @queue: usb device request queue
112	* @ep: pointer to xusb_endpoint structure
113	*/
114	struct xusb_req {
115	struct usb_request usb_req;
116	struct list_head queue;
117	struct xusb_ep *ep;
118	};
119
120	/**
121	* struct xusb_ep - USB end point structure.
122	* @ep_usb: usb endpoint instance
123	* @queue: endpoint message queue
124	* @udc: xilinx usb peripheral driver instance pointer
125	* @desc: pointer to the usb endpoint descriptor
126	* @rambase: the endpoint buffer address
127	* @offset: the endpoint register offset value
128	* @name: name of the endpoint
129	* @epnumber: endpoint number
130	* @maxpacket: maximum packet size the endpoint can store
131	* @buffer0count: the size of the packet recieved in the first buffer
132	* @buffer1count: the size of the packet received in the second buffer
133	* @curbufnum: current buffer of endpoint that will be processed next
134	* @buffer0ready: the busy state of first buffer
135	* @buffer1ready: the busy state of second buffer
136	* @is_in: endpoint direction (IN or OUT)
137	* @is_iso: endpoint type(isochronous or non isochronous)
138	*/
139	struct xusb_ep {
140	struct usb_ep ep_usb;
141	struct list_head queue;
142	struct xusb_udc *udc;
143	const struct usb_endpoint_descriptor *desc;
144	u32 rambase;
145	u32 offset;
146	char name[4];
147	u16 epnumber;
148	u16 maxpacket;
149	u16 buffer0count;
150	u16 buffer1count;
151	u8 curbufnum;
152	bool buffer0ready;
153	bool buffer1ready;
154	bool is_in;
155	bool is_iso;
156	};
157
158	/**
159	* struct xusb_udc - USB peripheral driver structure
160	* @gadget: USB gadget driver instance
161	* @ep: an array of endpoint structures
162	* @driver: pointer to the usb gadget driver instance
163	* @setup: usb_ctrlrequest structure for control requests
164	* @req: pointer to dummy request for get status command
165	* @dev: pointer to device structure in gadget
166	* @usb_state: device in suspended state or not
167	* @remote_wkp: remote wakeup enabled by host
168	* @setupseqtx: tx status
169	* @setupseqrx: rx status
170	* @addr: the usb device base address
171	* @lock: instance of spinlock
172	* @dma_enabled: flag indicating whether the dma is included in the system
173	* @clk: pointer to struct clk
174	* @read_fn: function pointer to read device registers
175	* @write_fn: function pointer to write to device registers
176	*/
177	struct xusb_udc {
178	struct usb_gadget gadget;
179	struct xusb_ep ep[8];
180	struct usb_gadget_driver *driver;
181	struct usb_ctrlrequest setup;
182	struct xusb_req *req;
183	struct device *dev;
184	u32 usb_state;
185	u32 remote_wkp;
186	u32 setupseqtx;
187	u32 setupseqrx;
188	void __iomem *addr;
189	spinlock_t lock;
190	bool dma_enabled;
191	struct clk *clk;
192
193	unsigned int (*read_fn)(void __iomem *reg);
194	void (*write_fn)(void __iomem *, u32, u32);
195	};
196
197	/* Endpoint buffer start addresses in the core */
198	static u32 rambase[8] = { 0x22, 0x1000, 0x1100, 0x1200, 0x1300, 0x1400, 0x1500,
199	0x1600 };
200
201	static const char driver_name[] = "xilinx-udc";
202	static const char ep0name[] = "ep0";
203
204	/* Control endpoint configuration.*/
205	static const struct usb_endpoint_descriptor config_bulk_out_desc = {
206	.bLength = USB_DT_ENDPOINT_SIZE,
207	.bDescriptorType = USB_DT_ENDPOINT,
208	.bEndpointAddress = USB_DIR_OUT,
209	.bmAttributes = USB_ENDPOINT_XFER_BULK,
210	.wMaxPacketSize = cpu_to_le16(EP0_MAX_PACKET),
211	};
212
213	/**
214	* xudc_write32 - little endian write to device registers
215	* @addr: base addr of device registers
216	* @offset: register offset
217	* @val: data to be written
218	*/
219	static void xudc_write32(void __iomem *addr, u32 offset, u32 val)
220	{
221	iowrite32(val, addr + offset);
222	}
223
224	/**
225	* xudc_read32 - little endian read from device registers
226	* @addr: addr of device register
227	* Return: value at addr
228	*/
229	static unsigned int xudc_read32(void __iomem *addr)
230	{
231	return ioread32(addr);
232	}
233
234	/**
235	* xudc_write32_be - big endian write to device registers
236	* @addr: base addr of device registers
237	* @offset: register offset
238	* @val: data to be written
239	*/
240	static void xudc_write32_be(void __iomem *addr, u32 offset, u32 val)
241	{
242	iowrite32be(val, addr + offset);
243	}
244
245	/**
246	* xudc_read32_be - big endian read from device registers
247	* @addr: addr of device register
248	* Return: value at addr
249	*/
250	static unsigned int xudc_read32_be(void __iomem *addr)
251	{
252	return ioread32be(addr);
253	}
254
255	/**
256	* xudc_wrstatus - Sets up the usb device status stages.
257	* @udc: pointer to the usb device controller structure.
258	*/
259	static void xudc_wrstatus(struct xusb_udc *udc)
260	{
261	struct xusb_ep *ep0 = &udc->ep[XUSB_EP_NUMBER_ZERO];
262	u32 epcfgreg;
263
264	epcfgreg = udc->read_fn(udc->addr + ep0->offset)|
265	XUSB_EP_CFG_DATA_TOGGLE_MASK;
266	udc->write_fn(udc->addr, ep0->offset, epcfgreg);
267	udc->write_fn(udc->addr, ep0->offset + XUSB_EP_BUF0COUNT_OFFSET, 0);
268	udc->write_fn(udc->addr, XUSB_BUFFREADY_OFFSET, 1);
269	}
270
271	/**
272	* xudc_epconfig - Configures the given endpoint.
273	* @ep: pointer to the usb device endpoint structure.
274	* @udc: pointer to the usb peripheral controller structure.
275	*
276	* This function configures a specific endpoint with the given configuration
277	* data.
278	*/
279	static void xudc_epconfig(struct xusb_ep *ep, struct xusb_udc *udc)
280	{
281	u32 epcfgreg;
282
283	/*
284	* Configure the end point direction, type, Max Packet Size and the
285	* EP buffer location.
286	*/
287	epcfgreg = ((ep->is_in << 29) | (ep->is_iso << 28) |
288	(ep->ep_usb.maxpacket << 15) | (ep->rambase));
289	udc->write_fn(udc->addr, ep->offset, epcfgreg);
290
291	/* Set the Buffer count and the Buffer ready bits.*/
292	udc->write_fn(udc->addr, ep->offset + XUSB_EP_BUF0COUNT_OFFSET,
293	ep->buffer0count);
294	udc->write_fn(udc->addr, ep->offset + XUSB_EP_BUF1COUNT_OFFSET,
295	ep->buffer1count);
296	if (ep->buffer0ready)
297	udc->write_fn(udc->addr, XUSB_BUFFREADY_OFFSET,
298	1 << ep->epnumber);
299	if (ep->buffer1ready)
300	udc->write_fn(udc->addr, XUSB_BUFFREADY_OFFSET,
301	1 << (ep->epnumber + XUSB_STATUS_EP_BUFF2_SHIFT));
302	}
303
304	/**
305	* xudc_start_dma - Starts DMA transfer.
306	* @ep: pointer to the usb device endpoint structure.
307	* @src: DMA source address.
308	* @dst: DMA destination address.
309	* @length: number of bytes to transfer.
310	*
311	* Return: 0 on success, error code on failure
312	*
313	* This function starts DMA transfer by writing to DMA source,
314	* destination and lenth registers.
315	*/
316	static int xudc_start_dma(struct xusb_ep *ep, dma_addr_t src,
317	dma_addr_t dst, u32 length)
318	{
319	struct xusb_udc *udc = ep->udc;
320	int rc = 0;
321	u32 timeout = 500;
322	u32 reg;
323
324	/*
325	* Set the addresses in the DMA source and
326	* destination registers and then set the length
327	* into the DMA length register.
328	*/
329	udc->write_fn(udc->addr, XUSB_DMA_DSAR_ADDR_OFFSET, src);
330	udc->write_fn(udc->addr, XUSB_DMA_DDAR_ADDR_OFFSET, dst);
331	udc->write_fn(udc->addr, XUSB_DMA_LENGTH_OFFSET, length);
332
333	/*
334	* Wait till DMA transaction is complete and
335	* check whether the DMA transaction was
336	* successful.
337	*/
338	do {
339	reg = udc->read_fn(udc->addr + XUSB_DMA_STATUS_OFFSET);
340	if (!(reg & XUSB_DMA_DMASR_BUSY))
341	break;
342
343	/*
344	* We can't sleep here, because it's also called from
345	* interrupt context.
346	*/
347	timeout--;
348	if (!timeout) {
349	dev_err(udc->dev, "DMA timeout\n");
350	return -ETIMEDOUT;
351	}
352	udelay(1);
353	} while (1);
354
355	if ((udc->read_fn(udc->addr + XUSB_DMA_STATUS_OFFSET) &
356	XUSB_DMA_DMASR_ERROR) == XUSB_DMA_DMASR_ERROR){
357	dev_err(udc->dev, "DMA Error\n");
358	rc = -EINVAL;
359	}
360
361	return rc;
362	}
363
364	/**
365	* xudc_dma_send - Sends IN data using DMA.
366	* @ep: pointer to the usb device endpoint structure.
367	* @req: pointer to the usb request structure.
368	* @buffer: pointer to data to be sent.
369	* @length: number of bytes to send.
370	*
371	* Return: 0 on success, -EAGAIN if no buffer is free and error
372	* code on failure.
373	*
374	* This function sends data using DMA.
375	*/
376	static int xudc_dma_send(struct xusb_ep *ep, struct xusb_req *req,
377	u8 *buffer, u32 length)
378	{
379	u32 *eprambase;
380	dma_addr_t src;
381	dma_addr_t dst;
382	struct xusb_udc *udc = ep->udc;
383
384	src = req->usb_req.dma + req->usb_req.actual;
385	if (req->usb_req.length)
386	dma_sync_single_for_device(dev: udc->dev, addr: src,
387	size: length, dir: DMA_TO_DEVICE);
388	if (!ep->curbufnum && !ep->buffer0ready) {
389	/* Get the Buffer address and copy the transmit data.*/
390	eprambase = (u32 __force *)(udc->addr + ep->rambase);
391	dst = virt_to_phys(address: eprambase);
392	udc->write_fn(udc->addr, ep->offset +
393	XUSB_EP_BUF0COUNT_OFFSET, length);
394	udc->write_fn(udc->addr, XUSB_DMA_CONTROL_OFFSET,
395	XUSB_DMA_BRR_CTRL | (1 << ep->epnumber));
396	ep->buffer0ready = 1;
397	ep->curbufnum = 1;
398	} else if (ep->curbufnum && !ep->buffer1ready) {
399	/* Get the Buffer address and copy the transmit data.*/
400	eprambase = (u32 __force *)(udc->addr + ep->rambase +
401	ep->ep_usb.maxpacket);
402	dst = virt_to_phys(address: eprambase);
403	udc->write_fn(udc->addr, ep->offset +
404	XUSB_EP_BUF1COUNT_OFFSET, length);
405	udc->write_fn(udc->addr, XUSB_DMA_CONTROL_OFFSET,
406	XUSB_DMA_BRR_CTRL | (1 << (ep->epnumber +
407	XUSB_STATUS_EP_BUFF2_SHIFT)));
408	ep->buffer1ready = 1;
409	ep->curbufnum = 0;
410	} else {
411	/* None of ping pong buffers are ready currently .*/
412	return -EAGAIN;
413	}
414
415	return xudc_start_dma(ep, src, dst, length);
416	}
417
418	/**
419	* xudc_dma_receive - Receives OUT data using DMA.
420	* @ep: pointer to the usb device endpoint structure.
421	* @req: pointer to the usb request structure.
422	* @buffer: pointer to storage buffer of received data.
423	* @length: number of bytes to receive.
424	*
425	* Return: 0 on success, -EAGAIN if no buffer is free and error
426	* code on failure.
427	*
428	* This function receives data using DMA.
429	*/
430	static int xudc_dma_receive(struct xusb_ep *ep, struct xusb_req *req,
431	u8 *buffer, u32 length)
432	{
433	u32 *eprambase;
434	dma_addr_t src;
435	dma_addr_t dst;
436	struct xusb_udc *udc = ep->udc;
437
438	dst = req->usb_req.dma + req->usb_req.actual;
439	if (!ep->curbufnum && !ep->buffer0ready) {
440	/* Get the Buffer address and copy the transmit data */
441	eprambase = (u32 __force *)(udc->addr + ep->rambase);
442	src = virt_to_phys(address: eprambase);
443	udc->write_fn(udc->addr, XUSB_DMA_CONTROL_OFFSET,
444	XUSB_DMA_BRR_CTRL | XUSB_DMA_READ_FROM_DPRAM |
445	(1 << ep->epnumber));
446	ep->buffer0ready = 1;
447	ep->curbufnum = 1;
448	} else if (ep->curbufnum && !ep->buffer1ready) {
449	/* Get the Buffer address and copy the transmit data */
450	eprambase = (u32 __force *)(udc->addr +
451	ep->rambase + ep->ep_usb.maxpacket);
452	src = virt_to_phys(address: eprambase);
453	udc->write_fn(udc->addr, XUSB_DMA_CONTROL_OFFSET,
454	XUSB_DMA_BRR_CTRL | XUSB_DMA_READ_FROM_DPRAM |
455	(1 << (ep->epnumber +
456	XUSB_STATUS_EP_BUFF2_SHIFT)));
457	ep->buffer1ready = 1;
458	ep->curbufnum = 0;
459	} else {
460	/* None of the ping-pong buffers are ready currently */
461	return -EAGAIN;
462	}
463
464	return xudc_start_dma(ep, src, dst, length);
465	}
466
467	/**
468	* xudc_eptxrx - Transmits or receives data to or from an endpoint.
469	* @ep: pointer to the usb endpoint configuration structure.
470	* @req: pointer to the usb request structure.
471	* @bufferptr: pointer to buffer containing the data to be sent.
472	* @bufferlen: The number of data bytes to be sent.
473	*
474	* Return: 0 on success, -EAGAIN if no buffer is free.
475	*
476	* This function copies the transmit/receive data to/from the end point buffer
477	* and enables the buffer for transmission/reception.
478	*/
479	static int xudc_eptxrx(struct xusb_ep *ep, struct xusb_req *req,
480	u8 *bufferptr, u32 bufferlen)
481	{
482	u32 *eprambase;
483	u32 bytestosend;
484	int rc = 0;
485	struct xusb_udc *udc = ep->udc;
486
487	bytestosend = bufferlen;
488	if (udc->dma_enabled) {
489	if (ep->is_in)
490	rc = xudc_dma_send(ep, req, buffer: bufferptr, length: bufferlen);
491	else
492	rc = xudc_dma_receive(ep, req, buffer: bufferptr, length: bufferlen);
493	return rc;
494	}
495	/* Put the transmit buffer into the correct ping-pong buffer.*/
496	if (!ep->curbufnum && !ep->buffer0ready) {
497	/* Get the Buffer address and copy the transmit data.*/
498	eprambase = (u32 __force *)(udc->addr + ep->rambase);
499	if (ep->is_in) {
500	memcpy_toio((void __iomem *)eprambase, bufferptr,
501	bytestosend);
502	udc->write_fn(udc->addr, ep->offset +
503	XUSB_EP_BUF0COUNT_OFFSET, bufferlen);
504	} else {
505	memcpy_toio((void __iomem *)bufferptr, eprambase,
506	bytestosend);
507	}
508	/*
509	* Enable the buffer for transmission.
510	*/
511	udc->write_fn(udc->addr, XUSB_BUFFREADY_OFFSET,
512	1 << ep->epnumber);
513	ep->buffer0ready = 1;
514	ep->curbufnum = 1;
515	} else if (ep->curbufnum && !ep->buffer1ready) {
516	/* Get the Buffer address and copy the transmit data.*/
517	eprambase = (u32 __force *)(udc->addr + ep->rambase +
518	ep->ep_usb.maxpacket);
519	if (ep->is_in) {
520	memcpy_toio((void __iomem *)eprambase, bufferptr,
521	bytestosend);
522	udc->write_fn(udc->addr, ep->offset +
523	XUSB_EP_BUF1COUNT_OFFSET, bufferlen);
524	} else {
525	memcpy_toio((void __iomem *)bufferptr, eprambase,
526	bytestosend);
527	}
528	/*
529	* Enable the buffer for transmission.
530	*/
531	udc->write_fn(udc->addr, XUSB_BUFFREADY_OFFSET,
532	1 << (ep->epnumber + XUSB_STATUS_EP_BUFF2_SHIFT));
533	ep->buffer1ready = 1;
534	ep->curbufnum = 0;
535	} else {
536	/* None of the ping-pong buffers are ready currently */
537	return -EAGAIN;
538	}
539	return rc;
540	}
541
542	/**
543	* xudc_done - Exeutes the endpoint data transfer completion tasks.
544	* @ep: pointer to the usb device endpoint structure.
545	* @req: pointer to the usb request structure.
546	* @status: Status of the data transfer.
547	*
548	* Deletes the message from the queue and updates data transfer completion
549	* status.
550	*/
551	static void xudc_done(struct xusb_ep *ep, struct xusb_req *req, int status)
552	{
553	struct xusb_udc *udc = ep->udc;
554
555	list_del_init(entry: &req->queue);
556
557	if (req->usb_req.status == -EINPROGRESS)
558	req->usb_req.status = status;
559	else
560	status = req->usb_req.status;
561
562	if (status && status != -ESHUTDOWN)
563	dev_dbg(udc->dev, "%s done %p, status %d\n",
564	ep->ep_usb.name, req, status);
565	/* unmap request if DMA is present*/
566	if (udc->dma_enabled && ep->epnumber && req->usb_req.length)
567	usb_gadget_unmap_request(gadget: &udc->gadget, req: &req->usb_req,
568	is_in: ep->is_in);
569
570	if (req->usb_req.complete) {
571	spin_unlock(lock: &udc->lock);
572	req->usb_req.complete(&ep->ep_usb, &req->usb_req);
573	spin_lock(lock: &udc->lock);
574	}
575	}
576
577	/**
578	* xudc_read_fifo - Reads the data from the given endpoint buffer.
579	* @ep: pointer to the usb device endpoint structure.
580	* @req: pointer to the usb request structure.
581	*
582	* Return: 0 if request is completed and -EAGAIN if not completed.
583	*
584	* Pulls OUT packet data from the endpoint buffer.
585	*/
586	static int xudc_read_fifo(struct xusb_ep *ep, struct xusb_req *req)
587	{
588	u8 *buf;
589	u32 is_short, count, bufferspace;
590	u8 bufoffset;
591	u8 two_pkts = 0;
592	int ret;
593	int retval = -EAGAIN;
594	struct xusb_udc *udc = ep->udc;
595
596	if (ep->buffer0ready && ep->buffer1ready) {
597	dev_dbg(udc->dev, "Packet NOT ready!\n");
598	return retval;
599	}
600	top:
601	if (ep->curbufnum)
602	bufoffset = XUSB_EP_BUF1COUNT_OFFSET;
603	else
604	bufoffset = XUSB_EP_BUF0COUNT_OFFSET;
605
606	count = udc->read_fn(udc->addr + ep->offset + bufoffset);
607
608	if (!ep->buffer0ready && !ep->buffer1ready)
609	two_pkts = 1;
610
611	buf = req->usb_req.buf + req->usb_req.actual;
612	prefetchw(x: buf);
613	bufferspace = req->usb_req.length - req->usb_req.actual;
614	is_short = count < ep->ep_usb.maxpacket;
615
616	if (unlikely(!bufferspace)) {
617	/*
618	* This happens when the driver's buffer
619	* is smaller than what the host sent.
620	* discard the extra data.
621	*/
622	if (req->usb_req.status != -EOVERFLOW)
623	dev_dbg(udc->dev, "%s overflow %d\n",
624	ep->ep_usb.name, count);
625	req->usb_req.status = -EOVERFLOW;
626	xudc_done(ep, req, status: -EOVERFLOW);
627	return 0;
628	}
629
630	ret = xudc_eptxrx(ep, req, bufferptr: buf, bufferlen: count);
631	switch (ret) {
632	case 0:
633	req->usb_req.actual += min(count, bufferspace);
634	dev_dbg(udc->dev, "read %s, %d bytes%s req %p %d/%d\n",
635	ep->ep_usb.name, count, is_short ? "/S" : "", req,
636	req->usb_req.actual, req->usb_req.length);
637
638	/* Completion */
639	if ((req->usb_req.actual == req->usb_req.length) || is_short) {
640	if (udc->dma_enabled && req->usb_req.length)
641	dma_sync_single_for_cpu(dev: udc->dev,
642	addr: req->usb_req.dma,
643	size: req->usb_req.actual,
644	dir: DMA_FROM_DEVICE);
645	xudc_done(ep, req, status: 0);
646	return 0;
647	}
648	if (two_pkts) {
649	two_pkts = 0;
650	goto top;
651	}
652	break;
653	case -EAGAIN:
654	dev_dbg(udc->dev, "receive busy\n");
655	break;
656	case -EINVAL:
657	case -ETIMEDOUT:
658	/* DMA error, dequeue the request */
659	xudc_done(ep, req, status: -ECONNRESET);
660	retval = 0;
661	break;
662	}
663
664	return retval;
665	}
666
667	/**
668	* xudc_write_fifo - Writes data into the given endpoint buffer.
669	* @ep: pointer to the usb device endpoint structure.
670	* @req: pointer to the usb request structure.
671	*
672	* Return: 0 if request is completed and -EAGAIN if not completed.
673	*
674	* Loads endpoint buffer for an IN packet.
675	*/
676	static int xudc_write_fifo(struct xusb_ep *ep, struct xusb_req *req)
677	{
678	u32 max;
679	u32 length;
680	int ret;
681	int retval = -EAGAIN;
682	struct xusb_udc *udc = ep->udc;
683	int is_last, is_short = 0;
684	u8 *buf;
685
686	max = le16_to_cpu(ep->desc->wMaxPacketSize);
687	buf = req->usb_req.buf + req->usb_req.actual;
688	prefetch(buf);
689	length = req->usb_req.length - req->usb_req.actual;
690	length = min(length, max);
691
692	ret = xudc_eptxrx(ep, req, bufferptr: buf, bufferlen: length);
693	switch (ret) {
694	case 0:
695	req->usb_req.actual += length;
696	if (unlikely(length != max)) {
697	is_last = is_short = 1;
698	} else {
699	if (likely(req->usb_req.length !=
700	req->usb_req.actual) || req->usb_req.zero)
701	is_last = 0;
702	else
703	is_last = 1;
704	}
705	dev_dbg(udc->dev, "%s: wrote %s %d bytes%s%s %d left %p\n",
706	__func__, ep->ep_usb.name, length, is_last ? "/L" : "",
707	is_short ? "/S" : "",
708	req->usb_req.length - req->usb_req.actual, req);
709	/* completion */
710	if (is_last) {
711	xudc_done(ep, req, status: 0);
712	retval = 0;
713	}
714	break;
715	case -EAGAIN:
716	dev_dbg(udc->dev, "Send busy\n");
717	break;
718	case -EINVAL:
719	case -ETIMEDOUT:
720	/* DMA error, dequeue the request */
721	xudc_done(ep, req, status: -ECONNRESET);
722	retval = 0;
723	break;
724	}
725
726	return retval;
727	}
728
729	/**
730	* xudc_nuke - Cleans up the data transfer message list.
731	* @ep: pointer to the usb device endpoint structure.
732	* @status: Status of the data transfer.
733	*/
734	static void xudc_nuke(struct xusb_ep *ep, int status)
735	{
736	struct xusb_req *req;
737
738	while (!list_empty(head: &ep->queue)) {
739	req = list_first_entry(&ep->queue, struct xusb_req, queue);
740	xudc_done(ep, req, status);
741	}
742	}
743
744	/**
745	* xudc_ep_set_halt - Stalls/unstalls the given endpoint.
746	* @_ep: pointer to the usb device endpoint structure.
747	* @value: value to indicate stall/unstall.
748	*
749	* Return: 0 for success and error value on failure
750	*/
751	static int xudc_ep_set_halt(struct usb_ep *_ep, int value)
752	{
753	struct xusb_ep *ep = to_xusb_ep(_ep);
754	struct xusb_udc *udc;
755	unsigned long flags;
756	u32 epcfgreg;
757
758	if (!_ep || (!ep->desc && ep->epnumber)) {
759	pr_debug("%s: bad ep or descriptor\n", __func__);
760	return -EINVAL;
761	}
762	udc = ep->udc;
763
764	if (ep->is_in && (!list_empty(head: &ep->queue)) && value) {
765	dev_dbg(udc->dev, "requests pending can't halt\n");
766	return -EAGAIN;
767	}
768
769	if (ep->buffer0ready || ep->buffer1ready) {
770	dev_dbg(udc->dev, "HW buffers busy can't halt\n");
771	return -EAGAIN;
772	}
773
774	spin_lock_irqsave(&udc->lock, flags);
775
776	if (value) {
777	/* Stall the device.*/
778	epcfgreg = udc->read_fn(udc->addr + ep->offset);
779	epcfgreg |= XUSB_EP_CFG_STALL_MASK;
780	udc->write_fn(udc->addr, ep->offset, epcfgreg);
781	} else {
782	/* Unstall the device.*/
783	epcfgreg = udc->read_fn(udc->addr + ep->offset);
784	epcfgreg &= ~XUSB_EP_CFG_STALL_MASK;
785	udc->write_fn(udc->addr, ep->offset, epcfgreg);
786	if (ep->epnumber) {
787	/* Reset the toggle bit.*/
788	epcfgreg = udc->read_fn(ep->udc->addr + ep->offset);
789	epcfgreg &= ~XUSB_EP_CFG_DATA_TOGGLE_MASK;
790	udc->write_fn(udc->addr, ep->offset, epcfgreg);
791	}
792	}
793
794	spin_unlock_irqrestore(lock: &udc->lock, flags);
795	return 0;
796	}
797
798	/**
799	* __xudc_ep_enable - Enables the given endpoint.
800	* @ep: pointer to the xusb endpoint structure.
801	* @desc: pointer to usb endpoint descriptor.
802	*
803	* Return: 0 for success and error value on failure
804	*/
805	static int __xudc_ep_enable(struct xusb_ep *ep,
806	const struct usb_endpoint_descriptor *desc)
807	{
808	struct xusb_udc *udc = ep->udc;
809	u32 tmp;
810	u32 epcfg;
811	u32 ier;
812	u16 maxpacket;
813
814	ep->is_in = ((desc->bEndpointAddress & USB_DIR_IN) != 0);
815	/* Bit 3...0:endpoint number */
816	ep->epnumber = (desc->bEndpointAddress & 0x0f);
817	ep->desc = desc;
818	ep->ep_usb.desc = desc;
819	tmp = desc->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK;
820	ep->ep_usb.maxpacket = maxpacket = le16_to_cpu(desc->wMaxPacketSize);
821
822	switch (tmp) {
823	case USB_ENDPOINT_XFER_CONTROL:
824	dev_dbg(udc->dev, "only one control endpoint\n");
825	/* NON- ISO */
826	ep->is_iso = 0;
827	return -EINVAL;
828	case USB_ENDPOINT_XFER_INT:
829	/* NON- ISO */
830	ep->is_iso = 0;
831	if (maxpacket > 64) {
832	dev_dbg(udc->dev, "bogus maxpacket %d\n", maxpacket);
833	return -EINVAL;
834	}
835	break;
836	case USB_ENDPOINT_XFER_BULK:
837	/* NON- ISO */
838	ep->is_iso = 0;
839	if (!(is_power_of_2(n: maxpacket) && maxpacket >= 8 &&
840	maxpacket <= 512)) {
841	dev_dbg(udc->dev, "bogus maxpacket %d\n", maxpacket);
842	return -EINVAL;
843	}
844	break;
845	case USB_ENDPOINT_XFER_ISOC:
846	/* ISO */
847	ep->is_iso = 1;
848	break;
849	}
850
851	ep->buffer0ready = false;
852	ep->buffer1ready = false;
853	ep->curbufnum = 0;
854	ep->rambase = rambase[ep->epnumber];
855	xudc_epconfig(ep, udc);
856
857	dev_dbg(udc->dev, "Enable Endpoint %d max pkt is %d\n",
858	ep->epnumber, maxpacket);
859
860	/* Enable the End point.*/
861	epcfg = udc->read_fn(udc->addr + ep->offset);
862	epcfg |= XUSB_EP_CFG_VALID_MASK;
863	udc->write_fn(udc->addr, ep->offset, epcfg);
864	if (ep->epnumber)
865	ep->rambase <<= 2;
866
867	/* Enable buffer completion interrupts for endpoint */
868	ier = udc->read_fn(udc->addr + XUSB_IER_OFFSET);
869	ier |= (XUSB_STATUS_INTR_BUFF_COMP_SHIFT_MASK << ep->epnumber);
870	udc->write_fn(udc->addr, XUSB_IER_OFFSET, ier);
871
872	/* for OUT endpoint set buffers ready to receive */
873	if (ep->epnumber && !ep->is_in) {
874	udc->write_fn(udc->addr, XUSB_BUFFREADY_OFFSET,
875	1 << ep->epnumber);
876	ep->buffer0ready = true;
877	udc->write_fn(udc->addr, XUSB_BUFFREADY_OFFSET,
878	(1 << (ep->epnumber +
879	XUSB_STATUS_EP_BUFF2_SHIFT)));
880	ep->buffer1ready = true;
881	}
882
883	return 0;
884	}
885
886	/**
887	* xudc_ep_enable - Enables the given endpoint.
888	* @_ep: pointer to the usb endpoint structure.
889	* @desc: pointer to usb endpoint descriptor.
890	*
891	* Return: 0 for success and error value on failure
892	*/
893	static int xudc_ep_enable(struct usb_ep *_ep,
894	const struct usb_endpoint_descriptor *desc)
895	{
896	struct xusb_ep *ep;
897	struct xusb_udc *udc;
898	unsigned long flags;
899	int ret;
900
901	if (!_ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
902	pr_debug("%s: bad ep or descriptor\n", __func__);
903	return -EINVAL;
904	}
905
906	ep = to_xusb_ep(_ep);
907	udc = ep->udc;
908
909	if (!udc->driver || udc->gadget.speed == USB_SPEED_UNKNOWN) {
910	dev_dbg(udc->dev, "bogus device state\n");
911	return -ESHUTDOWN;
912	}
913
914	spin_lock_irqsave(&udc->lock, flags);
915	ret = __xudc_ep_enable(ep, desc);
916	spin_unlock_irqrestore(lock: &udc->lock, flags);
917
918	return ret;
919	}
920
921	/**
922	* xudc_ep_disable - Disables the given endpoint.
923	* @_ep: pointer to the usb endpoint structure.
924	*
925	* Return: 0 for success and error value on failure
926	*/
927	static int xudc_ep_disable(struct usb_ep *_ep)
928	{
929	struct xusb_ep *ep;
930	unsigned long flags;
931	u32 epcfg;
932	struct xusb_udc *udc;
933
934	if (!_ep) {
935	pr_debug("%s: invalid ep\n", __func__);
936	return -EINVAL;
937	}
938
939	ep = to_xusb_ep(_ep);
940	udc = ep->udc;
941
942	spin_lock_irqsave(&udc->lock, flags);
943
944	xudc_nuke(ep, status: -ESHUTDOWN);
945
946	/* Restore the endpoint's pristine config */
947	ep->desc = NULL;
948	ep->ep_usb.desc = NULL;
949
950	dev_dbg(udc->dev, "USB Ep %d disable\n ", ep->epnumber);
951	/* Disable the endpoint.*/
952	epcfg = udc->read_fn(udc->addr + ep->offset);
953	epcfg &= ~XUSB_EP_CFG_VALID_MASK;
954	udc->write_fn(udc->addr, ep->offset, epcfg);
955
956	spin_unlock_irqrestore(lock: &udc->lock, flags);
957	return 0;
958	}
959
960	/**
961	* xudc_ep_alloc_request - Initializes the request queue.
962	* @_ep: pointer to the usb endpoint structure.
963	* @gfp_flags: Flags related to the request call.
964	*
965	* Return: pointer to request structure on success and a NULL on failure.
966	*/
967	static struct usb_request *xudc_ep_alloc_request(struct usb_ep *_ep,
968	gfp_t gfp_flags)
969	{
970	struct xusb_ep *ep = to_xusb_ep(_ep);
971	struct xusb_req *req;
972
973	req = kzalloc(size: sizeof(*req), flags: gfp_flags);
974	if (!req)
975	return NULL;
976
977	req->ep = ep;
978	INIT_LIST_HEAD(list: &req->queue);
979	return &req->usb_req;
980	}
981
982	/**
983	* xudc_free_request - Releases the request from queue.
984	* @_ep: pointer to the usb device endpoint structure.
985	* @_req: pointer to the usb request structure.
986	*/
987	static void xudc_free_request(struct usb_ep *_ep, struct usb_request *_req)
988	{
989	struct xusb_req *req = to_xusb_req(_req);
990
991	kfree(objp: req);
992	}
993
994	/**
995	* __xudc_ep0_queue - Adds the request to endpoint 0 queue.
996	* @ep0: pointer to the xusb endpoint 0 structure.
997	* @req: pointer to the xusb request structure.
998	*
999	* Return: 0 for success and error value on failure
1000	*/
1001	static int __xudc_ep0_queue(struct xusb_ep *ep0, struct xusb_req *req)
1002	{
1003	struct xusb_udc *udc = ep0->udc;
1004	u32 length;
1005	u8 *corebuf;
1006
1007	if (!udc->driver || udc->gadget.speed == USB_SPEED_UNKNOWN) {
1008	dev_dbg(udc->dev, "%s, bogus device state\n", __func__);
1009	return -EINVAL;
1010	}
1011	if (!list_empty(head: &ep0->queue)) {
1012	dev_dbg(udc->dev, "%s:ep0 busy\n", __func__);
1013	return -EBUSY;
1014	}
1015
1016	req->usb_req.status = -EINPROGRESS;
1017	req->usb_req.actual = 0;
1018
1019	list_add_tail(new: &req->queue, head: &ep0->queue);
1020
1021	if (udc->setup.bRequestType & USB_DIR_IN) {
1022	prefetch(req->usb_req.buf);
1023	length = req->usb_req.length;
1024	corebuf = (void __force *) ((ep0->rambase << 2) +
1025	udc->addr);
1026	length = req->usb_req.actual = min_t(u32, length,
1027	EP0_MAX_PACKET);
1028	memcpy_toio((void __iomem *)corebuf, req->usb_req.buf, length);
1029	udc->write_fn(udc->addr, XUSB_EP_BUF0COUNT_OFFSET, length);
1030	udc->write_fn(udc->addr, XUSB_BUFFREADY_OFFSET, 1);
1031	} else {
1032	if (udc->setup.wLength) {
1033	/* Enable EP0 buffer to receive data */
1034	udc->write_fn(udc->addr, XUSB_EP_BUF0COUNT_OFFSET, 0);
1035	udc->write_fn(udc->addr, XUSB_BUFFREADY_OFFSET, 1);
1036	} else {
1037	xudc_wrstatus(udc);
1038	}
1039	}
1040
1041	return 0;
1042	}
1043
1044	/**
1045	* xudc_ep0_queue - Adds the request to endpoint 0 queue.
1046	* @_ep: pointer to the usb endpoint 0 structure.
1047	* @_req: pointer to the usb request structure.
1048	* @gfp_flags: Flags related to the request call.
1049	*
1050	* Return: 0 for success and error value on failure
1051	*/
1052	static int xudc_ep0_queue(struct usb_ep *_ep, struct usb_request *_req,
1053	gfp_t gfp_flags)
1054	{
1055	struct xusb_req *req = to_xusb_req(_req);
1056	struct xusb_ep *ep0 = to_xusb_ep(_ep);
1057	struct xusb_udc *udc = ep0->udc;
1058	unsigned long flags;
1059	int ret;
1060
1061	spin_lock_irqsave(&udc->lock, flags);
1062	ret = __xudc_ep0_queue(ep0, req);
1063	spin_unlock_irqrestore(lock: &udc->lock, flags);
1064
1065	return ret;
1066	}
1067
1068	/**
1069	* xudc_ep_queue - Adds the request to endpoint queue.
1070	* @_ep: pointer to the usb endpoint structure.
1071	* @_req: pointer to the usb request structure.
1072	* @gfp_flags: Flags related to the request call.
1073	*
1074	* Return: 0 for success and error value on failure
1075	*/
1076	static int xudc_ep_queue(struct usb_ep *_ep, struct usb_request *_req,
1077	gfp_t gfp_flags)
1078	{
1079	struct xusb_req *req = to_xusb_req(_req);
1080	struct xusb_ep *ep = to_xusb_ep(_ep);
1081	struct xusb_udc *udc = ep->udc;
1082	int ret;
1083	unsigned long flags;
1084
1085	if (!ep->desc) {
1086	dev_dbg(udc->dev, "%s: queuing request to disabled %s\n",
1087	__func__, ep->name);
1088	return -ESHUTDOWN;
1089	}
1090
1091	if (!udc->driver || udc->gadget.speed == USB_SPEED_UNKNOWN) {
1092	dev_dbg(udc->dev, "%s, bogus device state\n", __func__);
1093	return -EINVAL;
1094	}
1095
1096	spin_lock_irqsave(&udc->lock, flags);
1097
1098	_req->status = -EINPROGRESS;
1099	_req->actual = 0;
1100
1101	if (udc->dma_enabled) {
1102	ret = usb_gadget_map_request(gadget: &udc->gadget, req: &req->usb_req,
1103	is_in: ep->is_in);
1104	if (ret) {
1105	dev_dbg(udc->dev, "gadget_map failed ep%d\n",
1106	ep->epnumber);
1107	spin_unlock_irqrestore(lock: &udc->lock, flags);
1108	return -EAGAIN;
1109	}
1110	}
1111
1112	if (list_empty(head: &ep->queue)) {
1113	if (ep->is_in) {
1114	dev_dbg(udc->dev, "xudc_write_fifo from ep_queue\n");
1115	if (!xudc_write_fifo(ep, req))
1116	req = NULL;
1117	} else {
1118	dev_dbg(udc->dev, "xudc_read_fifo from ep_queue\n");
1119	if (!xudc_read_fifo(ep, req))
1120	req = NULL;
1121	}
1122	}
1123
1124	if (req != NULL)
1125	list_add_tail(new: &req->queue, head: &ep->queue);
1126
1127	spin_unlock_irqrestore(lock: &udc->lock, flags);
1128	return 0;
1129	}
1130
1131	/**
1132	* xudc_ep_dequeue - Removes the request from the queue.
1133	* @_ep: pointer to the usb device endpoint structure.
1134	* @_req: pointer to the usb request structure.
1135	*
1136	* Return: 0 for success and error value on failure
1137	*/
1138	static int xudc_ep_dequeue(struct usb_ep *_ep, struct usb_request *_req)
1139	{
1140	struct xusb_ep *ep = to_xusb_ep(_ep);
1141	struct xusb_req *req = NULL;
1142	struct xusb_req *iter;
1143	struct xusb_udc *udc = ep->udc;
1144	unsigned long flags;
1145
1146	spin_lock_irqsave(&udc->lock, flags);
1147	/* Make sure it's actually queued on this endpoint */
1148	list_for_each_entry(iter, &ep->queue, queue) {
1149	if (&iter->usb_req != _req)
1150	continue;
1151	req = iter;
1152	break;
1153	}
1154	if (!req) {
1155	spin_unlock_irqrestore(lock: &udc->lock, flags);
1156	return -EINVAL;
1157	}
1158	xudc_done(ep, req, status: -ECONNRESET);
1159	spin_unlock_irqrestore(lock: &udc->lock, flags);
1160
1161	return 0;
1162	}
1163
1164	/**
1165	* xudc_ep0_enable - Enables the given endpoint.
1166	* @ep: pointer to the usb endpoint structure.
1167	* @desc: pointer to usb endpoint descriptor.
1168	*
1169	* Return: error always.
1170	*
1171	* endpoint 0 enable should not be called by gadget layer.
1172	*/
1173	static int xudc_ep0_enable(struct usb_ep *ep,
1174	const struct usb_endpoint_descriptor *desc)
1175	{
1176	return -EINVAL;
1177	}
1178
1179	/**
1180	* xudc_ep0_disable - Disables the given endpoint.
1181	* @ep: pointer to the usb endpoint structure.
1182	*
1183	* Return: error always.
1184	*
1185	* endpoint 0 disable should not be called by gadget layer.
1186	*/
1187	static int xudc_ep0_disable(struct usb_ep *ep)
1188	{
1189	return -EINVAL;
1190	}
1191
1192	static const struct usb_ep_ops xusb_ep0_ops = {
1193	.enable = xudc_ep0_enable,
1194	.disable = xudc_ep0_disable,
1195	.alloc_request = xudc_ep_alloc_request,
1196	.free_request = xudc_free_request,
1197	.queue = xudc_ep0_queue,
1198	.dequeue = xudc_ep_dequeue,
1199	.set_halt = xudc_ep_set_halt,
1200	};
1201
1202	static const struct usb_ep_ops xusb_ep_ops = {
1203	.enable = xudc_ep_enable,
1204	.disable = xudc_ep_disable,
1205	.alloc_request = xudc_ep_alloc_request,
1206	.free_request = xudc_free_request,
1207	.queue = xudc_ep_queue,
1208	.dequeue = xudc_ep_dequeue,
1209	.set_halt = xudc_ep_set_halt,
1210	};
1211
1212	/**
1213	* xudc_get_frame - Reads the current usb frame number.
1214	* @gadget: pointer to the usb gadget structure.
1215	*
1216	* Return: current frame number for success and error value on failure.
1217	*/
1218	static int xudc_get_frame(struct usb_gadget *gadget)
1219	{
1220	struct xusb_udc *udc;
1221	int frame;
1222
1223	if (!gadget)
1224	return -ENODEV;
1225
1226	udc = to_udc(gadget);
1227	frame = udc->read_fn(udc->addr + XUSB_FRAMENUM_OFFSET);
1228	return frame;
1229	}
1230
1231	/**
1232	* xudc_wakeup - Send remote wakeup signal to host
1233	* @gadget: pointer to the usb gadget structure.
1234	*
1235	* Return: 0 on success and error on failure
1236	*/
1237	static int xudc_wakeup(struct usb_gadget *gadget)
1238	{
1239	struct xusb_udc *udc = to_udc(gadget);
1240	u32 crtlreg;
1241	int status = -EINVAL;
1242	unsigned long flags;
1243
1244	spin_lock_irqsave(&udc->lock, flags);
1245
1246	/* Remote wake up not enabled by host */
1247	if (!udc->remote_wkp)
1248	goto done;
1249
1250	crtlreg = udc->read_fn(udc->addr + XUSB_CONTROL_OFFSET);
1251	crtlreg |= XUSB_CONTROL_USB_RMTWAKE_MASK;
1252	/* set remote wake up bit */
1253	udc->write_fn(udc->addr, XUSB_CONTROL_OFFSET, crtlreg);
1254	/*
1255	* wait for a while and reset remote wake up bit since this bit
1256	* is not cleared by HW after sending remote wakeup to host.
1257	*/
1258	mdelay(2);
1259
1260	crtlreg &= ~XUSB_CONTROL_USB_RMTWAKE_MASK;
1261	udc->write_fn(udc->addr, XUSB_CONTROL_OFFSET, crtlreg);
1262	status = 0;
1263	done:
1264	spin_unlock_irqrestore(lock: &udc->lock, flags);
1265	return status;
1266	}
1267
1268	/**
1269	* xudc_pullup - start/stop USB traffic
1270	* @gadget: pointer to the usb gadget structure.
1271	* @is_on: flag to start or stop
1272	*
1273	* Return: 0 always
1274	*
1275	* This function starts/stops SIE engine of IP based on is_on.
1276	*/
1277	static int xudc_pullup(struct usb_gadget *gadget, int is_on)
1278	{
1279	struct xusb_udc *udc = to_udc(gadget);
1280	unsigned long flags;
1281	u32 crtlreg;
1282
1283	spin_lock_irqsave(&udc->lock, flags);
1284
1285	crtlreg = udc->read_fn(udc->addr + XUSB_CONTROL_OFFSET);
1286	if (is_on)
1287	crtlreg |= XUSB_CONTROL_USB_READY_MASK;
1288	else
1289	crtlreg &= ~XUSB_CONTROL_USB_READY_MASK;
1290
1291	udc->write_fn(udc->addr, XUSB_CONTROL_OFFSET, crtlreg);
1292
1293	spin_unlock_irqrestore(lock: &udc->lock, flags);
1294
1295	return 0;
1296	}
1297
1298	/**
1299	* xudc_eps_init - initialize endpoints.
1300	* @udc: pointer to the usb device controller structure.
1301	*/
1302	static void xudc_eps_init(struct xusb_udc *udc)
1303	{
1304	u32 ep_number;
1305
1306	INIT_LIST_HEAD(list: &udc->gadget.ep_list);
1307
1308	for (ep_number = 0; ep_number < XUSB_MAX_ENDPOINTS; ep_number++) {
1309	struct xusb_ep *ep = &udc->ep[ep_number];
1310
1311	if (ep_number) {
1312	list_add_tail(new: &ep->ep_usb.ep_list,
1313	head: &udc->gadget.ep_list);
1314	usb_ep_set_maxpacket_limit(ep: &ep->ep_usb,
1315	maxpacket_limit: (unsigned short) ~0);
1316	snprintf(buf: ep->name, EPNAME_SIZE, fmt: "ep%d", ep_number);
1317	ep->ep_usb.name = ep->name;
1318	ep->ep_usb.ops = &xusb_ep_ops;
1319
1320	ep->ep_usb.caps.type_iso = true;
1321	ep->ep_usb.caps.type_bulk = true;
1322	ep->ep_usb.caps.type_int = true;
1323	} else {
1324	ep->ep_usb.name = ep0name;
1325	usb_ep_set_maxpacket_limit(ep: &ep->ep_usb, EP0_MAX_PACKET);
1326	ep->ep_usb.ops = &xusb_ep0_ops;
1327
1328	ep->ep_usb.caps.type_control = true;
1329	}
1330
1331	ep->ep_usb.caps.dir_in = true;
1332	ep->ep_usb.caps.dir_out = true;
1333
1334	ep->udc = udc;
1335	ep->epnumber = ep_number;
1336	ep->desc = NULL;
1337	/*
1338	* The configuration register address offset between
1339	* each endpoint is 0x10.
1340	*/
1341	ep->offset = XUSB_EP0_CONFIG_OFFSET + (ep_number * 0x10);
1342	ep->is_in = 0;
1343	ep->is_iso = 0;
1344	ep->maxpacket = 0;
1345	xudc_epconfig(ep, udc);
1346
1347	/* Initialize one queue per endpoint */
1348	INIT_LIST_HEAD(list: &ep->queue);
1349	}
1350	}
1351
1352	/**
1353	* xudc_stop_activity - Stops any further activity on the device.
1354	* @udc: pointer to the usb device controller structure.
1355	*/
1356	static void xudc_stop_activity(struct xusb_udc *udc)
1357	{
1358	int i;
1359	struct xusb_ep *ep;
1360
1361	for (i = 0; i < XUSB_MAX_ENDPOINTS; i++) {
1362	ep = &udc->ep[i];
1363	xudc_nuke(ep, status: -ESHUTDOWN);
1364	}
1365	}
1366
1367	/**
1368	* xudc_start - Starts the device.
1369	* @gadget: pointer to the usb gadget structure
1370	* @driver: pointer to gadget driver structure
1371	*
1372	* Return: zero on success and error on failure
1373	*/
1374	static int xudc_start(struct usb_gadget *gadget,
1375	struct usb_gadget_driver *driver)
1376	{
1377	struct xusb_udc *udc = to_udc(gadget);
1378	struct xusb_ep *ep0 = &udc->ep[XUSB_EP_NUMBER_ZERO];
1379	const struct usb_endpoint_descriptor *desc = &config_bulk_out_desc;
1380	unsigned long flags;
1381	int ret = 0;
1382
1383	spin_lock_irqsave(&udc->lock, flags);
1384
1385	if (udc->driver) {
1386	dev_err(udc->dev, "%s is already bound to %s\n",
1387	udc->gadget.name, udc->driver->driver.name);
1388	ret = -EBUSY;
1389	goto err;
1390	}
1391
1392	/* hook up the driver */
1393	udc->driver = driver;
1394	udc->gadget.speed = driver->max_speed;
1395
1396	/* Enable the control endpoint. */
1397	ret = __xudc_ep_enable(ep: ep0, desc);
1398
1399	/* Set device address and remote wakeup to 0 */
1400	udc->write_fn(udc->addr, XUSB_ADDRESS_OFFSET, 0);
1401	udc->remote_wkp = 0;
1402	err:
1403	spin_unlock_irqrestore(lock: &udc->lock, flags);
1404	return ret;
1405	}
1406
1407	/**
1408	* xudc_stop - stops the device.
1409	* @gadget: pointer to the usb gadget structure
1410	*
1411	* Return: zero always
1412	*/
1413	static int xudc_stop(struct usb_gadget *gadget)
1414	{
1415	struct xusb_udc *udc = to_udc(gadget);
1416	unsigned long flags;
1417
1418	spin_lock_irqsave(&udc->lock, flags);
1419
1420	udc->gadget.speed = USB_SPEED_UNKNOWN;
1421	udc->driver = NULL;
1422
1423	/* Set device address and remote wakeup to 0 */
1424	udc->write_fn(udc->addr, XUSB_ADDRESS_OFFSET, 0);
1425	udc->remote_wkp = 0;
1426
1427	xudc_stop_activity(udc);
1428
1429	spin_unlock_irqrestore(lock: &udc->lock, flags);
1430
1431	return 0;
1432	}
1433
1434	static const struct usb_gadget_ops xusb_udc_ops = {
1435	.get_frame = xudc_get_frame,
1436	.wakeup = xudc_wakeup,
1437	.pullup = xudc_pullup,
1438	.udc_start = xudc_start,
1439	.udc_stop = xudc_stop,
1440	};
1441
1442	/**
1443	* xudc_clear_stall_all_ep - clears stall of every endpoint.
1444	* @udc: pointer to the udc structure.
1445	*/
1446	static void xudc_clear_stall_all_ep(struct xusb_udc *udc)
1447	{
1448	struct xusb_ep *ep;
1449	u32 epcfgreg;
1450	int i;
1451
1452	for (i = 0; i < XUSB_MAX_ENDPOINTS; i++) {
1453	ep = &udc->ep[i];
1454	epcfgreg = udc->read_fn(udc->addr + ep->offset);
1455	epcfgreg &= ~XUSB_EP_CFG_STALL_MASK;
1456	udc->write_fn(udc->addr, ep->offset, epcfgreg);
1457	if (ep->epnumber) {
1458	/* Reset the toggle bit.*/
1459	epcfgreg = udc->read_fn(udc->addr + ep->offset);
1460	epcfgreg &= ~XUSB_EP_CFG_DATA_TOGGLE_MASK;
1461	udc->write_fn(udc->addr, ep->offset, epcfgreg);
1462	}
1463	}
1464	}
1465
1466	/**
1467	* xudc_startup_handler - The usb device controller interrupt handler.
1468	* @udc: pointer to the udc structure.
1469	* @intrstatus: The mask value containing the interrupt sources.
1470	*
1471	* This function handles the RESET,SUSPEND,RESUME and DISCONNECT interrupts.
1472	*/
1473	static void xudc_startup_handler(struct xusb_udc *udc, u32 intrstatus)
1474	{
1475	u32 intrreg;
1476
1477	if (intrstatus & XUSB_STATUS_RESET_MASK) {
1478
1479	dev_dbg(udc->dev, "Reset\n");
1480
1481	if (intrstatus & XUSB_STATUS_HIGH_SPEED_MASK)
1482	udc->gadget.speed = USB_SPEED_HIGH;
1483	else
1484	udc->gadget.speed = USB_SPEED_FULL;
1485
1486	xudc_stop_activity(udc);
1487	xudc_clear_stall_all_ep(udc);
1488	udc->write_fn(udc->addr, XUSB_TESTMODE_OFFSET, 0);
1489
1490	/* Set device address and remote wakeup to 0 */
1491	udc->write_fn(udc->addr, XUSB_ADDRESS_OFFSET, 0);
1492	udc->remote_wkp = 0;
1493
1494	/* Enable the suspend, resume and disconnect */
1495	intrreg = udc->read_fn(udc->addr + XUSB_IER_OFFSET);
1496	intrreg |= XUSB_STATUS_SUSPEND_MASK | XUSB_STATUS_RESUME_MASK |
1497	XUSB_STATUS_DISCONNECT_MASK;
1498	udc->write_fn(udc->addr, XUSB_IER_OFFSET, intrreg);
1499	}
1500	if (intrstatus & XUSB_STATUS_SUSPEND_MASK) {
1501
1502	dev_dbg(udc->dev, "Suspend\n");
1503
1504	/* Enable the reset, resume and disconnect */
1505	intrreg = udc->read_fn(udc->addr + XUSB_IER_OFFSET);
1506	intrreg |= XUSB_STATUS_RESET_MASK | XUSB_STATUS_RESUME_MASK |
1507	XUSB_STATUS_DISCONNECT_MASK;
1508	udc->write_fn(udc->addr, XUSB_IER_OFFSET, intrreg);
1509
1510	udc->usb_state = USB_STATE_SUSPENDED;
1511
1512	if (udc->driver->suspend) {
1513	spin_unlock(lock: &udc->lock);
1514	udc->driver->suspend(&udc->gadget);
1515	spin_lock(lock: &udc->lock);
1516	}
1517	}
1518	if (intrstatus & XUSB_STATUS_RESUME_MASK) {
1519	bool condition = (udc->usb_state != USB_STATE_SUSPENDED);
1520
1521	dev_WARN_ONCE(udc->dev, condition,
1522	"Resume IRQ while not suspended\n");
1523
1524	dev_dbg(udc->dev, "Resume\n");
1525
1526	/* Enable the reset, suspend and disconnect */
1527	intrreg = udc->read_fn(udc->addr + XUSB_IER_OFFSET);
1528	intrreg |= XUSB_STATUS_RESET_MASK | XUSB_STATUS_SUSPEND_MASK |
1529	XUSB_STATUS_DISCONNECT_MASK;
1530	udc->write_fn(udc->addr, XUSB_IER_OFFSET, intrreg);
1531
1532	udc->usb_state = 0;
1533
1534	if (udc->driver->resume) {
1535	spin_unlock(lock: &udc->lock);
1536	udc->driver->resume(&udc->gadget);
1537	spin_lock(lock: &udc->lock);
1538	}
1539	}
1540	if (intrstatus & XUSB_STATUS_DISCONNECT_MASK) {
1541
1542	dev_dbg(udc->dev, "Disconnect\n");
1543
1544	/* Enable the reset, resume and suspend */
1545	intrreg = udc->read_fn(udc->addr + XUSB_IER_OFFSET);
1546	intrreg |= XUSB_STATUS_RESET_MASK | XUSB_STATUS_RESUME_MASK |
1547	XUSB_STATUS_SUSPEND_MASK;
1548	udc->write_fn(udc->addr, XUSB_IER_OFFSET, intrreg);
1549
1550	if (udc->driver && udc->driver->disconnect) {
1551	spin_unlock(lock: &udc->lock);
1552	udc->driver->disconnect(&udc->gadget);
1553	spin_lock(lock: &udc->lock);
1554	}
1555	}
1556	}
1557
1558	/**
1559	* xudc_ep0_stall - Stall endpoint zero.
1560	* @udc: pointer to the udc structure.
1561	*
1562	* This function stalls endpoint zero.
1563	*/
1564	static void xudc_ep0_stall(struct xusb_udc *udc)
1565	{
1566	u32 epcfgreg;
1567	struct xusb_ep *ep0 = &udc->ep[XUSB_EP_NUMBER_ZERO];
1568
1569	epcfgreg = udc->read_fn(udc->addr + ep0->offset);
1570	epcfgreg |= XUSB_EP_CFG_STALL_MASK;
1571	udc->write_fn(udc->addr, ep0->offset, epcfgreg);
1572	}
1573
1574	/**
1575	* xudc_setaddress - executes SET_ADDRESS command
1576	* @udc: pointer to the udc structure.
1577	*
1578	* This function executes USB SET_ADDRESS command
1579	*/
1580	static void xudc_setaddress(struct xusb_udc *udc)
1581	{
1582	struct xusb_ep *ep0 = &udc->ep[0];
1583	struct xusb_req *req = udc->req;
1584	int ret;
1585
1586	req->usb_req.length = 0;
1587	ret = __xudc_ep0_queue(ep0, req);
1588	if (ret == 0)
1589	return;
1590
1591	dev_err(udc->dev, "Can't respond to SET ADDRESS request\n");
1592	xudc_ep0_stall(udc);
1593	}
1594
1595	/**
1596	* xudc_getstatus - executes GET_STATUS command
1597	* @udc: pointer to the udc structure.
1598	*
1599	* This function executes USB GET_STATUS command
1600	*/
1601	static void xudc_getstatus(struct xusb_udc *udc)
1602	{
1603	struct xusb_ep *ep0 = &udc->ep[0];
1604	struct xusb_req *req = udc->req;
1605	struct xusb_ep *target_ep;
1606	u16 status = 0;
1607	u32 epcfgreg;
1608	int epnum;
1609	u32 halt;
1610	int ret;
1611
1612	switch (udc->setup.bRequestType & USB_RECIP_MASK) {
1613	case USB_RECIP_DEVICE:
1614	/* Get device status */
1615	status = 1 << USB_DEVICE_SELF_POWERED;
1616	if (udc->remote_wkp)
1617	status |= (1 << USB_DEVICE_REMOTE_WAKEUP);
1618	break;
1619	case USB_RECIP_INTERFACE:
1620	break;
1621	case USB_RECIP_ENDPOINT:
1622	epnum = le16_to_cpu(udc->setup.wIndex) & USB_ENDPOINT_NUMBER_MASK;
1623	if (epnum >= XUSB_MAX_ENDPOINTS)
1624	goto stall;
1625	target_ep = &udc->ep[epnum];
1626	epcfgreg = udc->read_fn(udc->addr + target_ep->offset);
1627	halt = epcfgreg & XUSB_EP_CFG_STALL_MASK;
1628	if (le16_to_cpu(udc->setup.wIndex) & USB_DIR_IN) {
1629	if (!target_ep->is_in)
1630	goto stall;
1631	} else {
1632	if (target_ep->is_in)
1633	goto stall;
1634	}
1635	if (halt)
1636	status = 1 << USB_ENDPOINT_HALT;
1637	break;
1638	default:
1639	goto stall;
1640	}
1641
1642	req->usb_req.length = 2;
1643	*(__le16 *)req->usb_req.buf = cpu_to_le16(status);
1644	ret = __xudc_ep0_queue(ep0, req);
1645	if (ret == 0)
1646	return;
1647	stall:
1648	dev_err(udc->dev, "Can't respond to getstatus request\n");
1649	xudc_ep0_stall(udc);
1650	}
1651
1652	/**
1653	* xudc_set_clear_feature - Executes the set feature and clear feature commands.
1654	* @udc: pointer to the usb device controller structure.
1655	*
1656	* Processes the SET_FEATURE and CLEAR_FEATURE commands.
1657	*/
1658	static void xudc_set_clear_feature(struct xusb_udc *udc)
1659	{
1660	struct xusb_ep *ep0 = &udc->ep[0];
1661	struct xusb_req *req = udc->req;
1662	struct xusb_ep *target_ep;
1663	u8 endpoint;
1664	u8 outinbit;
1665	u32 epcfgreg;
1666	int flag = (udc->setup.bRequest == USB_REQ_SET_FEATURE ? 1 : 0);
1667	int ret;
1668
1669	switch (udc->setup.bRequestType) {
1670	case USB_RECIP_DEVICE:
1671	switch (le16_to_cpu(udc->setup.wValue)) {
1672	case USB_DEVICE_TEST_MODE:
1673	/*
1674	* The Test Mode will be executed
1675	* after the status phase.
1676	*/
1677	break;
1678	case USB_DEVICE_REMOTE_WAKEUP:
1679	if (flag)
1680	udc->remote_wkp = 1;
1681	else
1682	udc->remote_wkp = 0;
1683	break;
1684	default:
1685	xudc_ep0_stall(udc);
1686	break;
1687	}
1688	break;
1689	case USB_RECIP_ENDPOINT:
1690	if (!udc->setup.wValue) {
1691	endpoint = le16_to_cpu(udc->setup.wIndex) &
1692	USB_ENDPOINT_NUMBER_MASK;
1693	if (endpoint >= XUSB_MAX_ENDPOINTS) {
1694	xudc_ep0_stall(udc);
1695	return;
1696	}
1697	target_ep = &udc->ep[endpoint];
1698	outinbit = le16_to_cpu(udc->setup.wIndex) &
1699	USB_ENDPOINT_DIR_MASK;
1700	outinbit = outinbit >> 7;
1701
1702	/* Make sure direction matches.*/
1703	if (outinbit != target_ep->is_in) {
1704	xudc_ep0_stall(udc);
1705	return;
1706	}
1707	epcfgreg = udc->read_fn(udc->addr + target_ep->offset);
1708	if (!endpoint) {
1709	/* Clear the stall.*/
1710	epcfgreg &= ~XUSB_EP_CFG_STALL_MASK;
1711	udc->write_fn(udc->addr,
1712	target_ep->offset, epcfgreg);
1713	} else {
1714	if (flag) {
1715	epcfgreg |= XUSB_EP_CFG_STALL_MASK;
1716	udc->write_fn(udc->addr,
1717	target_ep->offset,
1718	epcfgreg);
1719	} else {
1720	/* Unstall the endpoint.*/
1721	epcfgreg &= ~(XUSB_EP_CFG_STALL_MASK |
1722	XUSB_EP_CFG_DATA_TOGGLE_MASK);
1723	udc->write_fn(udc->addr,
1724	target_ep->offset,
1725	epcfgreg);
1726	}
1727	}
1728	}
1729	break;
1730	default:
1731	xudc_ep0_stall(udc);
1732	return;
1733	}
1734
1735	req->usb_req.length = 0;
1736	ret = __xudc_ep0_queue(ep0, req);
1737	if (ret == 0)
1738	return;
1739
1740	dev_err(udc->dev, "Can't respond to SET/CLEAR FEATURE\n");
1741	xudc_ep0_stall(udc);
1742	}
1743
1744	/**
1745	* xudc_handle_setup - Processes the setup packet.
1746	* @udc: pointer to the usb device controller structure.
1747	*
1748	* Process setup packet and delegate to gadget layer.
1749	*/
1750	static void xudc_handle_setup(struct xusb_udc *udc)
1751	__must_hold(&udc->lock)
1752	{
1753	struct xusb_ep *ep0 = &udc->ep[0];
1754	struct usb_ctrlrequest setup;
1755	u32 *ep0rambase;
1756
1757	/* Load up the chapter 9 command buffer.*/
1758	ep0rambase = (u32 __force *) (udc->addr + XUSB_SETUP_PKT_ADDR_OFFSET);
1759	memcpy_toio((void __iomem *)&setup, ep0rambase, 8);
1760
1761	udc->setup = setup;
1762	udc->setup.wValue = cpu_to_le16((u16 __force)setup.wValue);
1763	udc->setup.wIndex = cpu_to_le16((u16 __force)setup.wIndex);
1764	udc->setup.wLength = cpu_to_le16((u16 __force)setup.wLength);
1765
1766	/* Clear previous requests */
1767	xudc_nuke(ep: ep0, status: -ECONNRESET);
1768
1769	if (udc->setup.bRequestType & USB_DIR_IN) {
1770	/* Execute the get command.*/
1771	udc->setupseqrx = STATUS_PHASE;
1772	udc->setupseqtx = DATA_PHASE;
1773	} else {
1774	/* Execute the put command.*/
1775	udc->setupseqrx = DATA_PHASE;
1776	udc->setupseqtx = STATUS_PHASE;
1777	}
1778
1779	switch (udc->setup.bRequest) {
1780	case USB_REQ_GET_STATUS:
1781	/* Data+Status phase form udc */
1782	if ((udc->setup.bRequestType &
1783	(USB_DIR_IN | USB_TYPE_MASK)) !=
1784	(USB_DIR_IN | USB_TYPE_STANDARD))
1785	break;
1786	xudc_getstatus(udc);
1787	return;
1788	case USB_REQ_SET_ADDRESS:
1789	/* Status phase from udc */
1790	if (udc->setup.bRequestType != (USB_DIR_OUT |
1791	USB_TYPE_STANDARD | USB_RECIP_DEVICE))
1792	break;
1793	xudc_setaddress(udc);
1794	return;
1795	case USB_REQ_CLEAR_FEATURE:
1796	case USB_REQ_SET_FEATURE:
1797	/* Requests with no data phase, status phase from udc */
1798	if ((udc->setup.bRequestType & USB_TYPE_MASK)
1799	!= USB_TYPE_STANDARD)
1800	break;
1801	xudc_set_clear_feature(udc);
1802	return;
1803	default:
1804	break;
1805	}
1806
1807	spin_unlock(lock: &udc->lock);
1808	if (udc->driver->setup(&udc->gadget, &setup) < 0)
1809	xudc_ep0_stall(udc);
1810	spin_lock(lock: &udc->lock);
1811	}
1812
1813	/**
1814	* xudc_ep0_out - Processes the endpoint 0 OUT token.
1815	* @udc: pointer to the usb device controller structure.
1816	*/
1817	static void xudc_ep0_out(struct xusb_udc *udc)
1818	{
1819	struct xusb_ep *ep0 = &udc->ep[0];
1820	struct xusb_req *req;
1821	u8 *ep0rambase;
1822	unsigned int bytes_to_rx;
1823	void *buffer;
1824
1825	req = list_first_entry(&ep0->queue, struct xusb_req, queue);
1826
1827	switch (udc->setupseqrx) {
1828	case STATUS_PHASE:
1829	/*
1830	* This resets both state machines for the next
1831	* Setup packet.
1832	*/
1833	udc->setupseqrx = SETUP_PHASE;
1834	udc->setupseqtx = SETUP_PHASE;
1835	req->usb_req.actual = req->usb_req.length;
1836	xudc_done(ep: ep0, req, status: 0);
1837	break;
1838	case DATA_PHASE:
1839	bytes_to_rx = udc->read_fn(udc->addr +
1840	XUSB_EP_BUF0COUNT_OFFSET);
1841	/* Copy the data to be received from the DPRAM. */
1842	ep0rambase = (u8 __force *) (udc->addr +
1843	(ep0->rambase << 2));
1844	buffer = req->usb_req.buf + req->usb_req.actual;
1845	req->usb_req.actual = req->usb_req.actual + bytes_to_rx;
1846	memcpy_toio((void __iomem *)buffer, ep0rambase, bytes_to_rx);
1847
1848	if (req->usb_req.length == req->usb_req.actual) {
1849	/* Data transfer completed get ready for Status stage */
1850	xudc_wrstatus(udc);
1851	} else {
1852	/* Enable EP0 buffer to receive data */
1853	udc->write_fn(udc->addr, XUSB_EP_BUF0COUNT_OFFSET, 0);
1854	udc->write_fn(udc->addr, XUSB_BUFFREADY_OFFSET, 1);
1855	}
1856	break;
1857	default:
1858	break;
1859	}
1860	}
1861
1862	/**
1863	* xudc_ep0_in - Processes the endpoint 0 IN token.
1864	* @udc: pointer to the usb device controller structure.
1865	*/
1866	static void xudc_ep0_in(struct xusb_udc *udc)
1867	{
1868	struct xusb_ep *ep0 = &udc->ep[0];
1869	struct xusb_req *req;
1870	unsigned int bytes_to_tx;
1871	void *buffer;
1872	u32 epcfgreg;
1873	u16 count = 0;
1874	u16 length;
1875	u8 *ep0rambase;
1876	u8 test_mode = le16_to_cpu(udc->setup.wIndex) >> 8;
1877
1878	req = list_first_entry(&ep0->queue, struct xusb_req, queue);
1879	bytes_to_tx = req->usb_req.length - req->usb_req.actual;
1880
1881	switch (udc->setupseqtx) {
1882	case STATUS_PHASE:
1883	switch (udc->setup.bRequest) {
1884	case USB_REQ_SET_ADDRESS:
1885	/* Set the address of the device.*/
1886	udc->write_fn(udc->addr, XUSB_ADDRESS_OFFSET,
1887	le16_to_cpu(udc->setup.wValue));
1888	break;
1889	case USB_REQ_SET_FEATURE:
1890	if (udc->setup.bRequestType ==
1891	USB_RECIP_DEVICE) {
1892	if (le16_to_cpu(udc->setup.wValue) ==
1893	USB_DEVICE_TEST_MODE)
1894	udc->write_fn(udc->addr,
1895	XUSB_TESTMODE_OFFSET,
1896	test_mode);
1897	}
1898	break;
1899	}
1900	req->usb_req.actual = req->usb_req.length;
1901	xudc_done(ep: ep0, req, status: 0);
1902	break;
1903	case DATA_PHASE:
1904	if (!bytes_to_tx) {
1905	/*
1906	* We're done with data transfer, next
1907	* will be zero length OUT with data toggle of
1908	* 1. Setup data_toggle.
1909	*/
1910	epcfgreg = udc->read_fn(udc->addr + ep0->offset);
1911	epcfgreg |= XUSB_EP_CFG_DATA_TOGGLE_MASK;
1912	udc->write_fn(udc->addr, ep0->offset, epcfgreg);
1913	udc->setupseqtx = STATUS_PHASE;
1914	} else {
1915	length = count = min_t(u32, bytes_to_tx,
1916	EP0_MAX_PACKET);
1917	/* Copy the data to be transmitted into the DPRAM. */
1918	ep0rambase = (u8 __force *) (udc->addr +
1919	(ep0->rambase << 2));
1920	buffer = req->usb_req.buf + req->usb_req.actual;
1921	req->usb_req.actual = req->usb_req.actual + length;
1922	memcpy_toio((void __iomem *)ep0rambase, buffer, length);
1923	}
1924	udc->write_fn(udc->addr, XUSB_EP_BUF0COUNT_OFFSET, count);
1925	udc->write_fn(udc->addr, XUSB_BUFFREADY_OFFSET, 1);
1926	break;
1927	default:
1928	break;
1929	}
1930	}
1931
1932	/**
1933	* xudc_ctrl_ep_handler - Endpoint 0 interrupt handler.
1934	* @udc: pointer to the udc structure.
1935	* @intrstatus: It's the mask value for the interrupt sources on endpoint 0.
1936	*
1937	* Processes the commands received during enumeration phase.
1938	*/
1939	static void xudc_ctrl_ep_handler(struct xusb_udc *udc, u32 intrstatus)
1940	{
1941
1942	if (intrstatus & XUSB_STATUS_SETUP_PACKET_MASK) {
1943	xudc_handle_setup(udc);
1944	} else {
1945	if (intrstatus & XUSB_STATUS_FIFO_BUFF_RDY_MASK)
1946	xudc_ep0_out(udc);
1947	else if (intrstatus & XUSB_STATUS_FIFO_BUFF_FREE_MASK)
1948	xudc_ep0_in(udc);
1949	}
1950	}
1951
1952	/**
1953	* xudc_nonctrl_ep_handler - Non control endpoint interrupt handler.
1954	* @udc: pointer to the udc structure.
1955	* @epnum: End point number for which the interrupt is to be processed
1956	* @intrstatus: mask value for interrupt sources of endpoints other
1957	* than endpoint 0.
1958	*
1959	* Processes the buffer completion interrupts.
1960	*/
1961	static void xudc_nonctrl_ep_handler(struct xusb_udc *udc, u8 epnum,
1962	u32 intrstatus)
1963	{
1964
1965	struct xusb_req *req;
1966	struct xusb_ep *ep;
1967
1968	ep = &udc->ep[epnum];
1969	/* Process the End point interrupts.*/
1970	if (intrstatus & (XUSB_STATUS_EP0_BUFF1_COMP_MASK << epnum))
1971	ep->buffer0ready = 0;
1972	if (intrstatus & (XUSB_STATUS_EP0_BUFF2_COMP_MASK << epnum))
1973	ep->buffer1ready = false;
1974
1975	if (list_empty(head: &ep->queue))
1976	return;
1977
1978	req = list_first_entry(&ep->queue, struct xusb_req, queue);
1979
1980	if (ep->is_in)
1981	xudc_write_fifo(ep, req);
1982	else
1983	xudc_read_fifo(ep, req);
1984	}
1985
1986	/**
1987	* xudc_irq - The main interrupt handler.
1988	* @irq: The interrupt number.
1989	* @_udc: pointer to the usb device controller structure.
1990	*
1991	* Return: IRQ_HANDLED after the interrupt is handled.
1992	*/
1993	static irqreturn_t xudc_irq(int irq, void *_udc)
1994	{
1995	struct xusb_udc *udc = _udc;
1996	u32 intrstatus;
1997	u32 ier;
1998	u8 index;
1999	u32 bufintr;
2000	unsigned long flags;
2001
2002	spin_lock_irqsave(&udc->lock, flags);
2003
2004	/*
2005	* Event interrupts are level sensitive hence first disable
2006	* IER, read ISR and figure out active interrupts.
2007	*/
2008	ier = udc->read_fn(udc->addr + XUSB_IER_OFFSET);
2009	ier &= ~XUSB_STATUS_INTR_EVENT_MASK;
2010	udc->write_fn(udc->addr, XUSB_IER_OFFSET, ier);
2011
2012	/* Read the Interrupt Status Register.*/
2013	intrstatus = udc->read_fn(udc->addr + XUSB_STATUS_OFFSET);
2014
2015	/* Call the handler for the event interrupt.*/
2016	if (intrstatus & XUSB_STATUS_INTR_EVENT_MASK) {
2017	/*
2018	* Check if there is any action to be done for :
2019	* - USB Reset received {XUSB_STATUS_RESET_MASK}
2020	* - USB Suspend received {XUSB_STATUS_SUSPEND_MASK}
2021	* - USB Resume received {XUSB_STATUS_RESUME_MASK}
2022	* - USB Disconnect received {XUSB_STATUS_DISCONNECT_MASK}
2023	*/
2024	xudc_startup_handler(udc, intrstatus);
2025	}
2026
2027	/* Check the buffer completion interrupts */
2028	if (intrstatus & XUSB_STATUS_INTR_BUFF_COMP_ALL_MASK) {
2029	/* Enable Reset, Suspend, Resume and Disconnect */
2030	ier = udc->read_fn(udc->addr + XUSB_IER_OFFSET);
2031	ier |= XUSB_STATUS_INTR_EVENT_MASK;
2032	udc->write_fn(udc->addr, XUSB_IER_OFFSET, ier);
2033
2034	if (intrstatus & XUSB_STATUS_EP0_BUFF1_COMP_MASK)
2035	xudc_ctrl_ep_handler(udc, intrstatus);
2036
2037	for (index = 1; index < 8; index++) {
2038	bufintr = ((intrstatus &
2039	(XUSB_STATUS_EP1_BUFF1_COMP_MASK <<
2040	(index - 1))) || (intrstatus &
2041	(XUSB_STATUS_EP1_BUFF2_COMP_MASK <<
2042	(index - 1))));
2043	if (bufintr) {
2044	xudc_nonctrl_ep_handler(udc, epnum: index,
2045	intrstatus);
2046	}
2047	}
2048	}
2049
2050	spin_unlock_irqrestore(lock: &udc->lock, flags);
2051	return IRQ_HANDLED;
2052	}
2053
2054	/**
2055	* xudc_probe - The device probe function for driver initialization.
2056	* @pdev: pointer to the platform device structure.
2057	*
2058	* Return: 0 for success and error value on failure
2059	*/
2060	static int xudc_probe(struct platform_device *pdev)
2061	{
2062	struct device_node *np = pdev->dev.of_node;
2063	struct resource *res;
2064	struct xusb_udc *udc;
2065	int irq;
2066	int ret;
2067	u32 ier;
2068	u8 *buff;
2069
2070	udc = devm_kzalloc(dev: &pdev->dev, size: sizeof(*udc), GFP_KERNEL);
2071	if (!udc)
2072	return -ENOMEM;
2073
2074	/* Create a dummy request for GET_STATUS, SET_ADDRESS */
2075	udc->req = devm_kzalloc(dev: &pdev->dev, size: sizeof(struct xusb_req),
2076	GFP_KERNEL);
2077	if (!udc->req)
2078	return -ENOMEM;
2079
2080	buff = devm_kzalloc(dev: &pdev->dev, STATUSBUFF_SIZE, GFP_KERNEL);
2081	if (!buff)
2082	return -ENOMEM;
2083
2084	udc->req->usb_req.buf = buff;
2085
2086	/* Map the registers */
2087	udc->addr = devm_platform_get_and_ioremap_resource(pdev, index: 0, res: &res);
2088	if (IS_ERR(ptr: udc->addr))
2089	return PTR_ERR(ptr: udc->addr);
2090
2091	irq = platform_get_irq(pdev, 0);
2092	if (irq < 0)
2093	return irq;
2094	ret = devm_request_irq(dev: &pdev->dev, irq, handler: xudc_irq, irqflags: 0,
2095	devname: dev_name(dev: &pdev->dev), dev_id: udc);
2096	if (ret < 0) {
2097	dev_dbg(&pdev->dev, "unable to request irq %d", irq);
2098	goto fail;
2099	}
2100
2101	udc->dma_enabled = of_property_read_bool(np, propname: "xlnx,has-builtin-dma");
2102
2103	/* Setup gadget structure */
2104	udc->gadget.ops = &xusb_udc_ops;
2105	udc->gadget.max_speed = USB_SPEED_HIGH;
2106	udc->gadget.speed = USB_SPEED_UNKNOWN;
2107	udc->gadget.ep0 = &udc->ep[XUSB_EP_NUMBER_ZERO].ep_usb;
2108	udc->gadget.name = driver_name;
2109
2110	udc->clk = devm_clk_get(dev: &pdev->dev, id: "s_axi_aclk");
2111	if (IS_ERR(ptr: udc->clk)) {
2112	if (PTR_ERR(ptr: udc->clk) != -ENOENT) {
2113	ret = PTR_ERR(ptr: udc->clk);
2114	goto fail;
2115	}
2116
2117	/*
2118	* Clock framework support is optional, continue on,
2119	* anyways if we don't find a matching clock
2120	*/
2121	dev_warn(&pdev->dev, "s_axi_aclk clock property is not found\n");
2122	udc->clk = NULL;
2123	}
2124
2125	ret = clk_prepare_enable(clk: udc->clk);
2126	if (ret) {
2127	dev_err(&pdev->dev, "Unable to enable clock.\n");
2128	return ret;
2129	}
2130
2131	spin_lock_init(&udc->lock);
2132
2133	/* Check for IP endianness */
2134	udc->write_fn = xudc_write32_be;
2135	udc->read_fn = xudc_read32_be;
2136	udc->write_fn(udc->addr, XUSB_TESTMODE_OFFSET, USB_TEST_J);
2137	if ((udc->read_fn(udc->addr + XUSB_TESTMODE_OFFSET))
2138	!= USB_TEST_J) {
2139	udc->write_fn = xudc_write32;
2140	udc->read_fn = xudc_read32;
2141	}
2142	udc->write_fn(udc->addr, XUSB_TESTMODE_OFFSET, 0);
2143
2144	xudc_eps_init(udc);
2145
2146	/* Set device address to 0.*/
2147	udc->write_fn(udc->addr, XUSB_ADDRESS_OFFSET, 0);
2148
2149	ret = usb_add_gadget_udc(parent: &pdev->dev, gadget: &udc->gadget);
2150	if (ret)
2151	goto err_disable_unprepare_clk;
2152
2153	udc->dev = &udc->gadget.dev;
2154
2155	/* Enable the interrupts.*/
2156	ier = XUSB_STATUS_GLOBAL_INTR_MASK | XUSB_STATUS_INTR_EVENT_MASK |
2157	XUSB_STATUS_FIFO_BUFF_RDY_MASK | XUSB_STATUS_FIFO_BUFF_FREE_MASK |
2158	XUSB_STATUS_SETUP_PACKET_MASK |
2159	XUSB_STATUS_INTR_BUFF_COMP_ALL_MASK;
2160
2161	udc->write_fn(udc->addr, XUSB_IER_OFFSET, ier);
2162
2163	platform_set_drvdata(pdev, data: udc);
2164
2165	dev_vdbg(&pdev->dev, "%s at 0x%08X mapped to %p %s\n",
2166	driver_name, (u32)res->start, udc->addr,
2167	udc->dma_enabled ? "with DMA" : "without DMA");
2168
2169	return 0;
2170
2171	err_disable_unprepare_clk:
2172	clk_disable_unprepare(clk: udc->clk);
2173	fail:
2174	dev_err(&pdev->dev, "probe failed, %d\n", ret);
2175	return ret;
2176	}
2177
2178	/**
2179	* xudc_remove - Releases the resources allocated during the initialization.
2180	* @pdev: pointer to the platform device structure.
2181	*
2182	* Return: 0 always
2183	*/
2184	static void xudc_remove(struct platform_device *pdev)
2185	{
2186	struct xusb_udc *udc = platform_get_drvdata(pdev);
2187
2188	usb_del_gadget_udc(gadget: &udc->gadget);
2189	clk_disable_unprepare(clk: udc->clk);
2190	}
2191
2192	#ifdef CONFIG_PM_SLEEP
2193	static int xudc_suspend(struct device *dev)
2194	{
2195	struct xusb_udc *udc;
2196	u32 crtlreg;
2197	unsigned long flags;
2198
2199	udc = dev_get_drvdata(dev);
2200
2201	spin_lock_irqsave(&udc->lock, flags);
2202
2203	crtlreg = udc->read_fn(udc->addr + XUSB_CONTROL_OFFSET);
2204	crtlreg &= ~XUSB_CONTROL_USB_READY_MASK;
2205
2206	udc->write_fn(udc->addr, XUSB_CONTROL_OFFSET, crtlreg);
2207
2208	spin_unlock_irqrestore(lock: &udc->lock, flags);
2209	if (udc->driver && udc->driver->suspend)
2210	udc->driver->suspend(&udc->gadget);
2211
2212	clk_disable(clk: udc->clk);
2213
2214	return 0;
2215	}
2216
2217	static int xudc_resume(struct device *dev)
2218	{
2219	struct xusb_udc *udc;
2220	u32 crtlreg;
2221	unsigned long flags;
2222	int ret;
2223
2224	udc = dev_get_drvdata(dev);
2225
2226	ret = clk_enable(clk: udc->clk);
2227	if (ret < 0)
2228	return ret;
2229
2230	spin_lock_irqsave(&udc->lock, flags);
2231
2232	crtlreg = udc->read_fn(udc->addr + XUSB_CONTROL_OFFSET);
2233	crtlreg |= XUSB_CONTROL_USB_READY_MASK;
2234
2235	udc->write_fn(udc->addr, XUSB_CONTROL_OFFSET, crtlreg);
2236
2237	spin_unlock_irqrestore(lock: &udc->lock, flags);
2238
2239	return 0;
2240	}
2241	#endif /* CONFIG_PM_SLEEP */
2242
2243	static const struct dev_pm_ops xudc_pm_ops = {
2244	SET_SYSTEM_SLEEP_PM_OPS(xudc_suspend, xudc_resume)
2245	};
2246
2247	/* Match table for of_platform binding */
2248	static const struct of_device_id usb_of_match[] = {
2249	{ .compatible = "xlnx,usb2-device-4.00.a", },
2250	{ /* end of list */ },
2251	};
2252	MODULE_DEVICE_TABLE(of, usb_of_match);
2253
2254	static struct platform_driver xudc_driver = {
2255	.driver = {
2256	.name = driver_name,
2257	.of_match_table = usb_of_match,
2258	.pm = &xudc_pm_ops,
2259	},
2260	.probe = xudc_probe,
2261	.remove_new = xudc_remove,
2262	};
2263
2264	module_platform_driver(xudc_driver);
2265
2266	MODULE_DESCRIPTION("Xilinx udc driver");
2267	MODULE_AUTHOR("Xilinx, Inc");
2268	MODULE_LICENSE("GPL");
2269