1	// SPDX-License-Identifier: GPL-2.0+
2	/*
3	* Copyright (c) 2008 Rodolfo Giometti <giometti@linux.it>
4	* Copyright (c) 2008 Eurotech S.p.A. <info@eurtech.it>
5	*
6	* This code is *strongly* based on EHCI-HCD code by David Brownell since
7	* the chip is a quasi-EHCI compatible.
8	*/
9
10	#include <linux/module.h>
11	#include <linux/pci.h>
12	#include <linux/dmapool.h>
13	#include <linux/kernel.h>
14	#include <linux/delay.h>
15	#include <linux/ioport.h>
16	#include <linux/sched.h>
17	#include <linux/slab.h>
18	#include <linux/errno.h>
19	#include <linux/timer.h>
20	#include <linux/list.h>
21	#include <linux/interrupt.h>
22	#include <linux/usb.h>
23	#include <linux/usb/hcd.h>
24	#include <linux/moduleparam.h>
25	#include <linux/dma-mapping.h>
26	#include <linux/io.h>
27	#include <linux/iopoll.h>
28
29	#include <asm/irq.h>
30	#include <asm/unaligned.h>
31
32	#include <linux/irq.h>
33	#include <linux/platform_device.h>
34
35	#define DRIVER_VERSION "0.0.50"
36
37	#define OXU_DEVICEID 0x00
38	#define OXU_REV_MASK 0xffff0000
39	#define OXU_REV_SHIFT 16
40	#define OXU_REV_2100 0x2100
41	#define OXU_BO_SHIFT 8
42	#define OXU_BO_MASK (0x3 << OXU_BO_SHIFT)
43	#define OXU_MAJ_REV_SHIFT 4
44	#define OXU_MAJ_REV_MASK (0xf << OXU_MAJ_REV_SHIFT)
45	#define OXU_MIN_REV_SHIFT 0
46	#define OXU_MIN_REV_MASK (0xf << OXU_MIN_REV_SHIFT)
47	#define OXU_HOSTIFCONFIG 0x04
48	#define OXU_SOFTRESET 0x08
49	#define OXU_SRESET (1 << 0)
50
51	#define OXU_PIOBURSTREADCTRL 0x0C
52
53	#define OXU_CHIPIRQSTATUS 0x10
54	#define OXU_CHIPIRQEN_SET 0x14
55	#define OXU_CHIPIRQEN_CLR 0x18
56	#define OXU_USBSPHLPWUI 0x00000080
57	#define OXU_USBOTGLPWUI 0x00000040
58	#define OXU_USBSPHI 0x00000002
59	#define OXU_USBOTGI 0x00000001
60
61	#define OXU_CLKCTRL_SET 0x1C
62	#define OXU_SYSCLKEN 0x00000008
63	#define OXU_USBSPHCLKEN 0x00000002
64	#define OXU_USBOTGCLKEN 0x00000001
65
66	#define OXU_ASO 0x68
67	#define OXU_SPHPOEN 0x00000100
68	#define OXU_OVRCCURPUPDEN 0x00000800
69	#define OXU_ASO_OP (1 << 10)
70	#define OXU_COMPARATOR 0x000004000
71
72	#define OXU_USBMODE 0x1A8
73	#define OXU_VBPS 0x00000020
74	#define OXU_ES_LITTLE 0x00000000
75	#define OXU_CM_HOST_ONLY 0x00000003
76
77	/*
78	* Proper EHCI structs & defines
79	*/
80
81	/* Magic numbers that can affect system performance */
82	#define EHCI_TUNE_CERR 3 /* 0-3 qtd retries; 0 == don't stop */
83	#define EHCI_TUNE_RL_HS 4 /* nak throttle; see 4.9 */
84	#define EHCI_TUNE_RL_TT 0
85	#define EHCI_TUNE_MULT_HS 1 /* 1-3 transactions/uframe; 4.10.3 */
86	#define EHCI_TUNE_MULT_TT 1
87	#define EHCI_TUNE_FLS 2 /* (small) 256 frame schedule */
88
89	struct oxu_hcd;
90
91	/* EHCI register interface, corresponds to EHCI Revision 0.95 specification */
92
93	/* Section 2.2 Host Controller Capability Registers */
94	struct ehci_caps {
95	/* these fields are specified as 8 and 16 bit registers,
96	* but some hosts can't perform 8 or 16 bit PCI accesses.
97	*/
98	u32 hc_capbase;
99	#define HC_LENGTH(p) (((p)>>00)&0x00ff) /* bits 7:0 */
100	#define HC_VERSION(p) (((p)>>16)&0xffff) /* bits 31:16 */
101	u32 hcs_params; /* HCSPARAMS - offset 0x4 */
102	#define HCS_DEBUG_PORT(p) (((p)>>20)&0xf) /* bits 23:20, debug port? */
103	#define HCS_INDICATOR(p) ((p)&(1 << 16)) /* true: has port indicators */
104	#define HCS_N_CC(p) (((p)>>12)&0xf) /* bits 15:12, #companion HCs */
105	#define HCS_N_PCC(p) (((p)>>8)&0xf) /* bits 11:8, ports per CC */
106	#define HCS_PORTROUTED(p) ((p)&(1 << 7)) /* true: port routing */
107	#define HCS_PPC(p) ((p)&(1 << 4)) /* true: port power control */
108	#define HCS_N_PORTS(p) (((p)>>0)&0xf) /* bits 3:0, ports on HC */
109
110	u32 hcc_params; /* HCCPARAMS - offset 0x8 */
111	#define HCC_EXT_CAPS(p) (((p)>>8)&0xff) /* for pci extended caps */
112	#define HCC_ISOC_CACHE(p) ((p)&(1 << 7)) /* true: can cache isoc frame */
113	#define HCC_ISOC_THRES(p) (((p)>>4)&0x7) /* bits 6:4, uframes cached */
114	#define HCC_CANPARK(p) ((p)&(1 << 2)) /* true: can park on async qh */
115	#define HCC_PGM_FRAMELISTLEN(p) ((p)&(1 << 1)) /* true: periodic_size changes*/
116	#define HCC_64BIT_ADDR(p) ((p)&(1)) /* true: can use 64-bit addr */
117	u8 portroute[8]; /* nibbles for routing - offset 0xC */
118	} __packed;
119
120
121	/* Section 2.3 Host Controller Operational Registers */
122	struct ehci_regs {
123	/* USBCMD: offset 0x00 */
124	u32 command;
125	/* 23:16 is r/w intr rate, in microframes; default "8" == 1/msec */
126	#define CMD_PARK (1<<11) /* enable "park" on async qh */
127	#define CMD_PARK_CNT(c) (((c)>>8)&3) /* how many transfers to park for */
128	#define CMD_LRESET (1<<7) /* partial reset (no ports, etc) */
129	#define CMD_IAAD (1<<6) /* "doorbell" interrupt async advance */
130	#define CMD_ASE (1<<5) /* async schedule enable */
131	#define CMD_PSE (1<<4) /* periodic schedule enable */
132	/* 3:2 is periodic frame list size */
133	#define CMD_RESET (1<<1) /* reset HC not bus */
134	#define CMD_RUN (1<<0) /* start/stop HC */
135
136	/* USBSTS: offset 0x04 */
137	u32 status;
138	#define STS_ASS (1<<15) /* Async Schedule Status */
139	#define STS_PSS (1<<14) /* Periodic Schedule Status */
140	#define STS_RECL (1<<13) /* Reclamation */
141	#define STS_HALT (1<<12) /* Not running (any reason) */
142	/* some bits reserved */
143	/* these STS_* flags are also intr_enable bits (USBINTR) */
144	#define STS_IAA (1<<5) /* Interrupted on async advance */
145	#define STS_FATAL (1<<4) /* such as some PCI access errors */
146	#define STS_FLR (1<<3) /* frame list rolled over */
147	#define STS_PCD (1<<2) /* port change detect */
148	#define STS_ERR (1<<1) /* "error" completion (overflow, ...) */
149	#define STS_INT (1<<0) /* "normal" completion (short, ...) */
150
151	#define INTR_MASK (STS_IAA | STS_FATAL | STS_PCD | STS_ERR | STS_INT)
152
153	/* USBINTR: offset 0x08 */
154	u32 intr_enable;
155
156	/* FRINDEX: offset 0x0C */
157	u32 frame_index; /* current microframe number */
158	/* CTRLDSSEGMENT: offset 0x10 */
159	u32 segment; /* address bits 63:32 if needed */
160	/* PERIODICLISTBASE: offset 0x14 */
161	u32 frame_list; /* points to periodic list */
162	/* ASYNCLISTADDR: offset 0x18 */
163	u32 async_next; /* address of next async queue head */
164
165	u32 reserved[9];
166
167	/* CONFIGFLAG: offset 0x40 */
168	u32 configured_flag;
169	#define FLAG_CF (1<<0) /* true: we'll support "high speed" */
170
171	/* PORTSC: offset 0x44 */
172	u32 port_status[]; /* up to N_PORTS */
173	/* 31:23 reserved */
174	#define PORT_WKOC_E (1<<22) /* wake on overcurrent (enable) */
175	#define PORT_WKDISC_E (1<<21) /* wake on disconnect (enable) */
176	#define PORT_WKCONN_E (1<<20) /* wake on connect (enable) */
177	/* 19:16 for port testing */
178	#define PORT_LED_OFF (0<<14)
179	#define PORT_LED_AMBER (1<<14)
180	#define PORT_LED_GREEN (2<<14)
181	#define PORT_LED_MASK (3<<14)
182	#define PORT_OWNER (1<<13) /* true: companion hc owns this port */
183	#define PORT_POWER (1<<12) /* true: has power (see PPC) */
184	#define PORT_USB11(x) (((x)&(3<<10)) == (1<<10)) /* USB 1.1 device */
185	/* 11:10 for detecting lowspeed devices (reset vs release ownership) */
186	/* 9 reserved */
187	#define PORT_RESET (1<<8) /* reset port */
188	#define PORT_SUSPEND (1<<7) /* suspend port */
189	#define PORT_RESUME (1<<6) /* resume it */
190	#define PORT_OCC (1<<5) /* over current change */
191	#define PORT_OC (1<<4) /* over current active */
192	#define PORT_PEC (1<<3) /* port enable change */
193	#define PORT_PE (1<<2) /* port enable */
194	#define PORT_CSC (1<<1) /* connect status change */
195	#define PORT_CONNECT (1<<0) /* device connected */
196	#define PORT_RWC_BITS (PORT_CSC | PORT_PEC | PORT_OCC)
197	} __packed;
198
199	/* Appendix C, Debug port ... intended for use with special "debug devices"
200	* that can help if there's no serial console. (nonstandard enumeration.)
201	*/
202	struct ehci_dbg_port {
203	u32 control;
204	#define DBGP_OWNER (1<<30)
205	#define DBGP_ENABLED (1<<28)
206	#define DBGP_DONE (1<<16)
207	#define DBGP_INUSE (1<<10)
208	#define DBGP_ERRCODE(x) (((x)>>7)&0x07)
209	# define DBGP_ERR_BAD 1
210	# define DBGP_ERR_SIGNAL 2
211	#define DBGP_ERROR (1<<6)
212	#define DBGP_GO (1<<5)
213	#define DBGP_OUT (1<<4)
214	#define DBGP_LEN(x) (((x)>>0)&0x0f)
215	u32 pids;
216	#define DBGP_PID_GET(x) (((x)>>16)&0xff)
217	#define DBGP_PID_SET(data, tok) (((data)<<8)|(tok))
218	u32 data03;
219	u32 data47;
220	u32 address;
221	#define DBGP_EPADDR(dev, ep) (((dev)<<8)|(ep))
222	} __packed;
223
224	#define QTD_NEXT(dma) cpu_to_le32((u32)dma)
225
226	/*
227	* EHCI Specification 0.95 Section 3.5
228	* QTD: describe data transfer components (buffer, direction, ...)
229	* See Fig 3-6 "Queue Element Transfer Descriptor Block Diagram".
230	*
231	* These are associated only with "QH" (Queue Head) structures,
232	* used with control, bulk, and interrupt transfers.
233	*/
234	struct ehci_qtd {
235	/* first part defined by EHCI spec */
236	__le32 hw_next; /* see EHCI 3.5.1 */
237	__le32 hw_alt_next; /* see EHCI 3.5.2 */
238	__le32 hw_token; /* see EHCI 3.5.3 */
239	#define QTD_TOGGLE (1 << 31) /* data toggle */
240	#define QTD_LENGTH(tok) (((tok)>>16) & 0x7fff)
241	#define QTD_IOC (1 << 15) /* interrupt on complete */
242	#define QTD_CERR(tok) (((tok)>>10) & 0x3)
243	#define QTD_PID(tok) (((tok)>>8) & 0x3)
244	#define QTD_STS_ACTIVE (1 << 7) /* HC may execute this */
245	#define QTD_STS_HALT (1 << 6) /* halted on error */
246	#define QTD_STS_DBE (1 << 5) /* data buffer error (in HC) */
247	#define QTD_STS_BABBLE (1 << 4) /* device was babbling (qtd halted) */
248	#define QTD_STS_XACT (1 << 3) /* device gave illegal response */
249	#define QTD_STS_MMF (1 << 2) /* incomplete split transaction */
250	#define QTD_STS_STS (1 << 1) /* split transaction state */
251	#define QTD_STS_PING (1 << 0) /* issue PING? */
252	__le32 hw_buf[5]; /* see EHCI 3.5.4 */
253	__le32 hw_buf_hi[5]; /* Appendix B */
254
255	/* the rest is HCD-private */
256	dma_addr_t qtd_dma; /* qtd address */
257	struct list_head qtd_list; /* sw qtd list */
258	struct urb *urb; /* qtd's urb */
259	size_t length; /* length of buffer */
260
261	u32 qtd_buffer_len;
262	void *buffer;
263	dma_addr_t buffer_dma;
264	void *transfer_buffer;
265	void *transfer_dma;
266	} __aligned(32);
267
268	/* mask NakCnt+T in qh->hw_alt_next */
269	#define QTD_MASK cpu_to_le32 (~0x1f)
270
271	#define IS_SHORT_READ(token) (QTD_LENGTH(token) != 0 && QTD_PID(token) == 1)
272
273	/* Type tag from {qh, itd, sitd, fstn}->hw_next */
274	#define Q_NEXT_TYPE(dma) ((dma) & cpu_to_le32 (3 << 1))
275
276	/* values for that type tag */
277	#define Q_TYPE_QH cpu_to_le32 (1 << 1)
278
279	/* next async queue entry, or pointer to interrupt/periodic QH */
280	#define QH_NEXT(dma) (cpu_to_le32(((u32)dma)&~0x01f)|Q_TYPE_QH)
281
282	/* for periodic/async schedules and qtd lists, mark end of list */
283	#define EHCI_LIST_END cpu_to_le32(1) /* "null pointer" to hw */
284
285	/*
286	* Entries in periodic shadow table are pointers to one of four kinds
287	* of data structure. That's dictated by the hardware; a type tag is
288	* encoded in the low bits of the hardware's periodic schedule. Use
289	* Q_NEXT_TYPE to get the tag.
290	*
291	* For entries in the async schedule, the type tag always says "qh".
292	*/
293	union ehci_shadow {
294	struct ehci_qh *qh; /* Q_TYPE_QH */
295	__le32 *hw_next; /* (all types) */
296	void *ptr;
297	};
298
299	/*
300	* EHCI Specification 0.95 Section 3.6
301	* QH: describes control/bulk/interrupt endpoints
302	* See Fig 3-7 "Queue Head Structure Layout".
303	*
304	* These appear in both the async and (for interrupt) periodic schedules.
305	*/
306
307	struct ehci_qh {
308	/* first part defined by EHCI spec */
309	__le32 hw_next; /* see EHCI 3.6.1 */
310	__le32 hw_info1; /* see EHCI 3.6.2 */
311	#define QH_HEAD 0x00008000
312	__le32 hw_info2; /* see EHCI 3.6.2 */
313	#define QH_SMASK 0x000000ff
314	#define QH_CMASK 0x0000ff00
315	#define QH_HUBADDR 0x007f0000
316	#define QH_HUBPORT 0x3f800000
317	#define QH_MULT 0xc0000000
318	__le32 hw_current; /* qtd list - see EHCI 3.6.4 */
319
320	/* qtd overlay (hardware parts of a struct ehci_qtd) */
321	__le32 hw_qtd_next;
322	__le32 hw_alt_next;
323	__le32 hw_token;
324	__le32 hw_buf[5];
325	__le32 hw_buf_hi[5];
326
327	/* the rest is HCD-private */
328	dma_addr_t qh_dma; /* address of qh */
329	union ehci_shadow qh_next; /* ptr to qh; or periodic */
330	struct list_head qtd_list; /* sw qtd list */
331	struct ehci_qtd *dummy;
332	struct ehci_qh *reclaim; /* next to reclaim */
333
334	struct oxu_hcd *oxu;
335	struct kref kref;
336	unsigned int stamp;
337
338	u8 qh_state;
339	#define QH_STATE_LINKED 1 /* HC sees this */
340	#define QH_STATE_UNLINK 2 /* HC may still see this */
341	#define QH_STATE_IDLE 3 /* HC doesn't see this */
342	#define QH_STATE_UNLINK_WAIT 4 /* LINKED and on reclaim q */
343	#define QH_STATE_COMPLETING 5 /* don't touch token.HALT */
344
345	/* periodic schedule info */
346	u8 usecs; /* intr bandwidth */
347	u8 gap_uf; /* uframes split/csplit gap */
348	u8 c_usecs; /* ... split completion bw */
349	u16 tt_usecs; /* tt downstream bandwidth */
350	unsigned short period; /* polling interval */
351	unsigned short start; /* where polling starts */
352	#define NO_FRAME ((unsigned short)~0) /* pick new start */
353	struct usb_device *dev; /* access to TT */
354	} __aligned(32);
355
356	/*
357	* Proper OXU210HP structs
358	*/
359
360	#define OXU_OTG_CORE_OFFSET 0x00400
361	#define OXU_OTG_CAP_OFFSET (OXU_OTG_CORE_OFFSET + 0x100)
362	#define OXU_SPH_CORE_OFFSET 0x00800
363	#define OXU_SPH_CAP_OFFSET (OXU_SPH_CORE_OFFSET + 0x100)
364
365	#define OXU_OTG_MEM 0xE000
366	#define OXU_SPH_MEM 0x16000
367
368	/* Only how many elements & element structure are specifies here. */
369	/* 2 host controllers are enabled - total size <= 28 kbytes */
370	#define DEFAULT_I_TDPS 1024
371	#define QHEAD_NUM 16
372	#define QTD_NUM 32
373	#define SITD_NUM 8
374	#define MURB_NUM 8
375
376	#define BUFFER_NUM 8
377	#define BUFFER_SIZE 512
378
379	struct oxu_info {
380	struct usb_hcd *hcd[2];
381	};
382
383	struct oxu_buf {
384	u8 buffer[BUFFER_SIZE];
385	} __aligned(BUFFER_SIZE);
386
387	struct oxu_onchip_mem {
388	struct oxu_buf db_pool[BUFFER_NUM];
389
390	u32 frame_list[DEFAULT_I_TDPS];
391	struct ehci_qh qh_pool[QHEAD_NUM];
392	struct ehci_qtd qtd_pool[QTD_NUM];
393	} __aligned(4 << 10);
394
395	#define EHCI_MAX_ROOT_PORTS 15 /* see HCS_N_PORTS */
396
397	struct oxu_murb {
398	struct urb urb;
399	struct urb *main;
400	u8 last;
401	};
402
403	struct oxu_hcd { /* one per controller */
404	unsigned int is_otg:1;
405
406	u8 qh_used[QHEAD_NUM];
407	u8 qtd_used[QTD_NUM];
408	u8 db_used[BUFFER_NUM];
409	u8 murb_used[MURB_NUM];
410
411	struct oxu_onchip_mem __iomem *mem;
412	spinlock_t mem_lock;
413
414	struct timer_list urb_timer;
415
416	struct ehci_caps __iomem *caps;
417	struct ehci_regs __iomem *regs;
418
419	u32 hcs_params; /* cached register copy */
420	spinlock_t lock;
421
422	/* async schedule support */
423	struct ehci_qh *async;
424	struct ehci_qh *reclaim;
425	unsigned int reclaim_ready:1;
426	unsigned int scanning:1;
427
428	/* periodic schedule support */
429	unsigned int periodic_size;
430	__le32 *periodic; /* hw periodic table */
431	dma_addr_t periodic_dma;
432	unsigned int i_thresh; /* uframes HC might cache */
433
434	union ehci_shadow *pshadow; /* mirror hw periodic table */
435	int next_uframe; /* scan periodic, start here */
436	unsigned int periodic_sched; /* periodic activity count */
437
438	/* per root hub port */
439	unsigned long reset_done[EHCI_MAX_ROOT_PORTS];
440	/* bit vectors (one bit per port) */
441	unsigned long bus_suspended; /* which ports were
442	* already suspended at the
443	* start of a bus suspend
444	*/
445	unsigned long companion_ports;/* which ports are dedicated
446	* to the companion controller
447	*/
448
449	struct timer_list watchdog;
450	unsigned long actions;
451	unsigned int stamp;
452	unsigned long next_statechange;
453	u32 command;
454
455	/* SILICON QUIRKS */
456	struct list_head urb_list; /* this is the head to urb
457	* queue that didn't get enough
458	* resources
459	*/
460	struct oxu_murb *murb_pool; /* murb per split big urb */
461	unsigned int urb_len;
462
463	u8 sbrn; /* packed release number */
464	};
465
466	#define EHCI_IAA_JIFFIES (HZ/100) /* arbitrary; ~10 msec */
467	#define EHCI_IO_JIFFIES (HZ/10) /* io watchdog > irq_thresh */
468	#define EHCI_ASYNC_JIFFIES (HZ/20) /* async idle timeout */
469	#define EHCI_SHRINK_JIFFIES (HZ/200) /* async qh unlink delay */
470
471	enum ehci_timer_action {
472	TIMER_IO_WATCHDOG,
473	TIMER_IAA_WATCHDOG,
474	TIMER_ASYNC_SHRINK,
475	TIMER_ASYNC_OFF,
476	};
477
478	/*
479	* Main defines
480	*/
481
482	#define oxu_dbg(oxu, fmt, args...) \
483	dev_dbg(oxu_to_hcd(oxu)->self.controller , fmt , ## args)
484	#define oxu_err(oxu, fmt, args...) \
485	dev_err(oxu_to_hcd(oxu)->self.controller , fmt , ## args)
486	#define oxu_info(oxu, fmt, args...) \
487	dev_info(oxu_to_hcd(oxu)->self.controller , fmt , ## args)
488
489	#ifdef CONFIG_DYNAMIC_DEBUG
490	#define DEBUG
491	#endif
492
493	static inline struct usb_hcd *oxu_to_hcd(struct oxu_hcd *oxu)
494	{
495	return container_of((void *) oxu, struct usb_hcd, hcd_priv);
496	}
497
498	static inline struct oxu_hcd *hcd_to_oxu(struct usb_hcd *hcd)
499	{
500	return (struct oxu_hcd *) (hcd->hcd_priv);
501	}
502
503	/*
504	* Debug stuff
505	*/
506
507	#undef OXU_URB_TRACE
508	#undef OXU_VERBOSE_DEBUG
509
510	#ifdef OXU_VERBOSE_DEBUG
511	#define oxu_vdbg oxu_dbg
512	#else
513	#define oxu_vdbg(oxu, fmt, args...) /* Nop */
514	#endif
515
516	#ifdef DEBUG
517
518	static int __attribute__((__unused__))
519	dbg_status_buf(char *buf, unsigned len, const char *label, u32 status)
520	{
521	return scnprintf(buf, size: len, fmt: "%s%sstatus %04x%s%s%s%s%s%s%s%s%s%s",
522	label, label[0] ? " " : "", status,
523	(status & STS_ASS) ? " Async" : "",
524	(status & STS_PSS) ? " Periodic" : "",
525	(status & STS_RECL) ? " Recl" : "",
526	(status & STS_HALT) ? " Halt" : "",
527	(status & STS_IAA) ? " IAA" : "",
528	(status & STS_FATAL) ? " FATAL" : "",
529	(status & STS_FLR) ? " FLR" : "",
530	(status & STS_PCD) ? " PCD" : "",
531	(status & STS_ERR) ? " ERR" : "",
532	(status & STS_INT) ? " INT" : ""
533	);
534	}
535
536	static int __attribute__((__unused__))
537	dbg_intr_buf(char *buf, unsigned len, const char *label, u32 enable)
538	{
539	return scnprintf(buf, size: len, fmt: "%s%sintrenable %02x%s%s%s%s%s%s",
540	label, label[0] ? " " : "", enable,
541	(enable & STS_IAA) ? " IAA" : "",
542	(enable & STS_FATAL) ? " FATAL" : "",
543	(enable & STS_FLR) ? " FLR" : "",
544	(enable & STS_PCD) ? " PCD" : "",
545	(enable & STS_ERR) ? " ERR" : "",
546	(enable & STS_INT) ? " INT" : ""
547	);
548	}
549
550	static const char *const fls_strings[] =
551	{ "1024", "512", "256", "??" };
552
553	static int dbg_command_buf(char *buf, unsigned len,
554	const char *label, u32 command)
555	{
556	return scnprintf(buf, size: len,
557	fmt: "%s%scommand %06x %s=%d ithresh=%d%s%s%s%s period=%s%s %s",
558	label, label[0] ? " " : "", command,
559	(command & CMD_PARK) ? "park" : "(park)",
560	CMD_PARK_CNT(command),
561	(command >> 16) & 0x3f,
562	(command & CMD_LRESET) ? " LReset" : "",
563	(command & CMD_IAAD) ? " IAAD" : "",
564	(command & CMD_ASE) ? " Async" : "",
565	(command & CMD_PSE) ? " Periodic" : "",
566	fls_strings[(command >> 2) & 0x3],
567	(command & CMD_RESET) ? " Reset" : "",
568	(command & CMD_RUN) ? "RUN" : "HALT"
569	);
570	}
571
572	static int dbg_port_buf(char *buf, unsigned len, const char *label,
573	int port, u32 status)
574	{
575	char *sig;
576
577	/* signaling state */
578	switch (status & (3 << 10)) {
579	case 0 << 10:
580	sig = "se0";
581	break;
582	case 1 << 10:
583	sig = "k"; /* low speed */
584	break;
585	case 2 << 10:
586	sig = "j";
587	break;
588	default:
589	sig = "?";
590	break;
591	}
592
593	return scnprintf(buf, size: len,
594	fmt: "%s%sport %d status %06x%s%s sig=%s%s%s%s%s%s%s%s%s%s",
595	label, label[0] ? " " : "", port, status,
596	(status & PORT_POWER) ? " POWER" : "",
597	(status & PORT_OWNER) ? " OWNER" : "",
598	sig,
599	(status & PORT_RESET) ? " RESET" : "",
600	(status & PORT_SUSPEND) ? " SUSPEND" : "",
601	(status & PORT_RESUME) ? " RESUME" : "",
602	(status & PORT_OCC) ? " OCC" : "",
603	(status & PORT_OC) ? " OC" : "",
604	(status & PORT_PEC) ? " PEC" : "",
605	(status & PORT_PE) ? " PE" : "",
606	(status & PORT_CSC) ? " CSC" : "",
607	(status & PORT_CONNECT) ? " CONNECT" : ""
608	);
609	}
610
611	#else
612
613	static inline int __attribute__((__unused__))
614	dbg_status_buf(char *buf, unsigned len, const char *label, u32 status)
615	{ return 0; }
616
617	static inline int __attribute__((__unused__))
618	dbg_command_buf(char *buf, unsigned len, const char *label, u32 command)
619	{ return 0; }
620
621	static inline int __attribute__((__unused__))
622	dbg_intr_buf(char *buf, unsigned len, const char *label, u32 enable)
623	{ return 0; }
624
625	static inline int __attribute__((__unused__))
626	dbg_port_buf(char *buf, unsigned len, const char *label, int port, u32 status)
627	{ return 0; }
628
629	#endif /* DEBUG */
630
631	/* functions have the "wrong" filename when they're output... */
632	#define dbg_status(oxu, label, status) { \
633	char _buf[80]; \
634	dbg_status_buf(_buf, sizeof _buf, label, status); \
635	oxu_dbg(oxu, "%s\n", _buf); \
636	}
637
638	#define dbg_cmd(oxu, label, command) { \
639	char _buf[80]; \
640	dbg_command_buf(_buf, sizeof _buf, label, command); \
641	oxu_dbg(oxu, "%s\n", _buf); \
642	}
643
644	#define dbg_port(oxu, label, port, status) { \
645	char _buf[80]; \
646	dbg_port_buf(_buf, sizeof _buf, label, port, status); \
647	oxu_dbg(oxu, "%s\n", _buf); \
648	}
649
650	/*
651	* Module parameters
652	*/
653
654	/* Initial IRQ latency: faster than hw default */
655	static int log2_irq_thresh; /* 0 to 6 */
656	module_param(log2_irq_thresh, int, S_IRUGO);
657	MODULE_PARM_DESC(log2_irq_thresh, "log2 IRQ latency, 1-64 microframes");
658
659	/* Initial park setting: slower than hw default */
660	static unsigned park;
661	module_param(park, uint, S_IRUGO);
662	MODULE_PARM_DESC(park, "park setting; 1-3 back-to-back async packets");
663
664	/* For flakey hardware, ignore overcurrent indicators */
665	static bool ignore_oc;
666	module_param(ignore_oc, bool, S_IRUGO);
667	MODULE_PARM_DESC(ignore_oc, "ignore bogus hardware overcurrent indications");
668
669
670	static void ehci_work(struct oxu_hcd *oxu);
671	static int oxu_hub_control(struct usb_hcd *hcd,
672	u16 typeReq, u16 wValue, u16 wIndex,
673	char *buf, u16 wLength);
674
675	/*
676	* Local functions
677	*/
678
679	/* Low level read/write registers functions */
680	static inline u32 oxu_readl(void __iomem *base, u32 reg)
681	{
682	return readl(addr: base + reg);
683	}
684
685	static inline void oxu_writel(void __iomem *base, u32 reg, u32 val)
686	{
687	writel(val, addr: base + reg);
688	}
689
690	static inline void timer_action_done(struct oxu_hcd *oxu,
691	enum ehci_timer_action action)
692	{
693	clear_bit(nr: action, addr: &oxu->actions);
694	}
695
696	static inline void timer_action(struct oxu_hcd *oxu,
697	enum ehci_timer_action action)
698	{
699	if (!test_and_set_bit(nr: action, addr: &oxu->actions)) {
700	unsigned long t;
701
702	switch (action) {
703	case TIMER_IAA_WATCHDOG:
704	t = EHCI_IAA_JIFFIES;
705	break;
706	case TIMER_IO_WATCHDOG:
707	t = EHCI_IO_JIFFIES;
708	break;
709	case TIMER_ASYNC_OFF:
710	t = EHCI_ASYNC_JIFFIES;
711	break;
712	case TIMER_ASYNC_SHRINK:
713	default:
714	t = EHCI_SHRINK_JIFFIES;
715	break;
716	}
717	t += jiffies;
718	/* all timings except IAA watchdog can be overridden.
719	* async queue SHRINK often precedes IAA. while it's ready
720	* to go OFF neither can matter, and afterwards the IO
721	* watchdog stops unless there's still periodic traffic.
722	*/
723	if (action != TIMER_IAA_WATCHDOG
724	&& t > oxu->watchdog.expires
725	&& timer_pending(timer: &oxu->watchdog))
726	return;
727	mod_timer(timer: &oxu->watchdog, expires: t);
728	}
729	}
730
731	/*
732	* handshake - spin reading hc until handshake completes or fails
733	* @ptr: address of hc register to be read
734	* @mask: bits to look at in result of read
735	* @done: value of those bits when handshake succeeds
736	* @usec: timeout in microseconds
737	*
738	* Returns negative errno, or zero on success
739	*
740	* Success happens when the "mask" bits have the specified value (hardware
741	* handshake done). There are two failure modes: "usec" have passed (major
742	* hardware flakeout), or the register reads as all-ones (hardware removed).
743	*
744	* That last failure should_only happen in cases like physical cardbus eject
745	* before driver shutdown. But it also seems to be caused by bugs in cardbus
746	* bridge shutdown: shutting down the bridge before the devices using it.
747	*/
748	static int handshake(struct oxu_hcd *oxu, void __iomem *ptr,
749	u32 mask, u32 done, int usec)
750	{
751	u32 result;
752	int ret;
753
754	ret = readl_poll_timeout_atomic(ptr, result,
755	((result & mask) == done ||
756	result == U32_MAX),
757	1, usec);
758	if (result == U32_MAX) /* card removed */
759	return -ENODEV;
760
761	return ret;
762	}
763
764	/* Force HC to halt state from unknown (EHCI spec section 2.3) */
765	static int ehci_halt(struct oxu_hcd *oxu)
766	{
767	u32 temp = readl(addr: &oxu->regs->status);
768
769	/* disable any irqs left enabled by previous code */
770	writel(val: 0, addr: &oxu->regs->intr_enable);
771
772	if ((temp & STS_HALT) != 0)
773	return 0;
774
775	temp = readl(addr: &oxu->regs->command);
776	temp &= ~CMD_RUN;
777	writel(val: temp, addr: &oxu->regs->command);
778	return handshake(oxu, ptr: &oxu->regs->status,
779	STS_HALT, STS_HALT, usec: 16 * 125);
780	}
781
782	/* Put TDI/ARC silicon into EHCI mode */
783	static void tdi_reset(struct oxu_hcd *oxu)
784	{
785	u32 __iomem *reg_ptr;
786	u32 tmp;
787
788	reg_ptr = (u32 __iomem *)(((u8 __iomem *)oxu->regs) + 0x68);
789	tmp = readl(addr: reg_ptr);
790	tmp |= 0x3;
791	writel(val: tmp, addr: reg_ptr);
792	}
793
794	/* Reset a non-running (STS_HALT == 1) controller */
795	static int ehci_reset(struct oxu_hcd *oxu)
796	{
797	int retval;
798	u32 command = readl(addr: &oxu->regs->command);
799
800	command |= CMD_RESET;
801	dbg_cmd(oxu, "reset", command);
802	writel(val: command, addr: &oxu->regs->command);
803	oxu_to_hcd(oxu)->state = HC_STATE_HALT;
804	oxu->next_statechange = jiffies;
805	retval = handshake(oxu, ptr: &oxu->regs->command,
806	CMD_RESET, done: 0, usec: 250 * 1000);
807
808	if (retval)
809	return retval;
810
811	tdi_reset(oxu);
812
813	return retval;
814	}
815
816	/* Idle the controller (from running) */
817	static void ehci_quiesce(struct oxu_hcd *oxu)
818	{
819	u32 temp;
820
821	#ifdef DEBUG
822	BUG_ON(!HC_IS_RUNNING(oxu_to_hcd(oxu)->state));
823	#endif
824
825	/* wait for any schedule enables/disables to take effect */
826	temp = readl(addr: &oxu->regs->command) << 10;
827	temp &= STS_ASS | STS_PSS;
828	if (handshake(oxu, ptr: &oxu->regs->status, STS_ASS | STS_PSS,
829	done: temp, usec: 16 * 125) != 0) {
830	oxu_to_hcd(oxu)->state = HC_STATE_HALT;
831	return;
832	}
833
834	/* then disable anything that's still active */
835	temp = readl(addr: &oxu->regs->command);
836	temp &= ~(CMD_ASE | CMD_IAAD | CMD_PSE);
837	writel(val: temp, addr: &oxu->regs->command);
838
839	/* hardware can take 16 microframes to turn off ... */
840	if (handshake(oxu, ptr: &oxu->regs->status, STS_ASS | STS_PSS,
841	done: 0, usec: 16 * 125) != 0) {
842	oxu_to_hcd(oxu)->state = HC_STATE_HALT;
843	return;
844	}
845	}
846
847	static int check_reset_complete(struct oxu_hcd *oxu, int index,
848	u32 __iomem *status_reg, int port_status)
849	{
850	if (!(port_status & PORT_CONNECT)) {
851	oxu->reset_done[index] = 0;
852	return port_status;
853	}
854
855	/* if reset finished and it's still not enabled -- handoff */
856	if (!(port_status & PORT_PE)) {
857	oxu_dbg(oxu, "Failed to enable port %d on root hub TT\n",
858	index+1);
859	return port_status;
860	} else
861	oxu_dbg(oxu, "port %d high speed\n", index + 1);
862
863	return port_status;
864	}
865
866	static void ehci_hub_descriptor(struct oxu_hcd *oxu,
867	struct usb_hub_descriptor *desc)
868	{
869	int ports = HCS_N_PORTS(oxu->hcs_params);
870	u16 temp;
871
872	desc->bDescriptorType = USB_DT_HUB;
873	desc->bPwrOn2PwrGood = 10; /* oxu 1.0, 2.3.9 says 20ms max */
874	desc->bHubContrCurrent = 0;
875
876	desc->bNbrPorts = ports;
877	temp = 1 + (ports / 8);
878	desc->bDescLength = 7 + 2 * temp;
879
880	/* ports removable, and usb 1.0 legacy PortPwrCtrlMask */
881	memset(&desc->u.hs.DeviceRemovable[0], 0, temp);
882	memset(&desc->u.hs.DeviceRemovable[temp], 0xff, temp);
883
884	temp = HUB_CHAR_INDV_PORT_OCPM; /* per-port overcurrent reporting */
885	if (HCS_PPC(oxu->hcs_params))
886	temp |= HUB_CHAR_INDV_PORT_LPSM; /* per-port power control */
887	else
888	temp |= HUB_CHAR_NO_LPSM; /* no power switching */
889	desc->wHubCharacteristics = (__force __u16)cpu_to_le16(temp);
890	}
891
892
893	/* Allocate an OXU210HP on-chip memory data buffer
894	*
895	* An on-chip memory data buffer is required for each OXU210HP USB transfer.
896	* Each transfer descriptor has one or more on-chip memory data buffers.
897	*
898	* Data buffers are allocated from a fix sized pool of data blocks.
899	* To minimise fragmentation and give reasonable memory utlisation,
900	* data buffers are allocated with sizes the power of 2 multiples of
901	* the block size, starting on an address a multiple of the allocated size.
902	*
903	* FIXME: callers of this function require a buffer to be allocated for
904	* len=0. This is a waste of on-chip memory and should be fix. Then this
905	* function should be changed to not allocate a buffer for len=0.
906	*/
907	static int oxu_buf_alloc(struct oxu_hcd *oxu, struct ehci_qtd *qtd, int len)
908	{
909	int n_blocks; /* minium blocks needed to hold len */
910	int a_blocks; /* blocks allocated */
911	int i, j;
912
913	/* Don't allocte bigger than supported */
914	if (len > BUFFER_SIZE * BUFFER_NUM) {
915	oxu_err(oxu, "buffer too big (%d)\n", len);
916	return -ENOMEM;
917	}
918
919	spin_lock(lock: &oxu->mem_lock);
920
921	/* Number of blocks needed to hold len */
922	n_blocks = (len + BUFFER_SIZE - 1) / BUFFER_SIZE;
923
924	/* Round the number of blocks up to the power of 2 */
925	for (a_blocks = 1; a_blocks < n_blocks; a_blocks <<= 1)
926	;
927
928	/* Find a suitable available data buffer */
929	for (i = 0; i < BUFFER_NUM;
930	i += max(a_blocks, (int)oxu->db_used[i])) {
931
932	/* Check all the required blocks are available */
933	for (j = 0; j < a_blocks; j++)
934	if (oxu->db_used[i + j])
935	break;
936
937	if (j != a_blocks)
938	continue;
939
940	/* Allocate blocks found! */
941	qtd->buffer = (void *) &oxu->mem->db_pool[i];
942	qtd->buffer_dma = virt_to_phys(address: qtd->buffer);
943
944	qtd->qtd_buffer_len = BUFFER_SIZE * a_blocks;
945	oxu->db_used[i] = a_blocks;
946
947	spin_unlock(lock: &oxu->mem_lock);
948
949	return 0;
950	}
951
952	/* Failed */
953
954	spin_unlock(lock: &oxu->mem_lock);
955
956	return -ENOMEM;
957	}
958
959	static void oxu_buf_free(struct oxu_hcd *oxu, struct ehci_qtd *qtd)
960	{
961	int index;
962
963	spin_lock(lock: &oxu->mem_lock);
964
965	index = (qtd->buffer - (void *) &oxu->mem->db_pool[0])
966	/ BUFFER_SIZE;
967	oxu->db_used[index] = 0;
968	qtd->qtd_buffer_len = 0;
969	qtd->buffer_dma = 0;
970	qtd->buffer = NULL;
971
972	spin_unlock(lock: &oxu->mem_lock);
973	}
974
975	static inline void ehci_qtd_init(struct ehci_qtd *qtd, dma_addr_t dma)
976	{
977	memset(qtd, 0, sizeof *qtd);
978	qtd->qtd_dma = dma;
979	qtd->hw_token = cpu_to_le32(QTD_STS_HALT);
980	qtd->hw_next = EHCI_LIST_END;
981	qtd->hw_alt_next = EHCI_LIST_END;
982	INIT_LIST_HEAD(list: &qtd->qtd_list);
983	}
984
985	static inline void oxu_qtd_free(struct oxu_hcd *oxu, struct ehci_qtd *qtd)
986	{
987	int index;
988
989	if (qtd->buffer)
990	oxu_buf_free(oxu, qtd);
991
992	spin_lock(lock: &oxu->mem_lock);
993
994	index = qtd - &oxu->mem->qtd_pool[0];
995	oxu->qtd_used[index] = 0;
996
997	spin_unlock(lock: &oxu->mem_lock);
998	}
999
1000	static struct ehci_qtd *ehci_qtd_alloc(struct oxu_hcd *oxu)
1001	{
1002	int i;
1003	struct ehci_qtd *qtd = NULL;
1004
1005	spin_lock(lock: &oxu->mem_lock);
1006
1007	for (i = 0; i < QTD_NUM; i++)
1008	if (!oxu->qtd_used[i])
1009	break;
1010
1011	if (i < QTD_NUM) {
1012	qtd = (struct ehci_qtd *) &oxu->mem->qtd_pool[i];
1013	memset(qtd, 0, sizeof *qtd);
1014
1015	qtd->hw_token = cpu_to_le32(QTD_STS_HALT);
1016	qtd->hw_next = EHCI_LIST_END;
1017	qtd->hw_alt_next = EHCI_LIST_END;
1018	INIT_LIST_HEAD(list: &qtd->qtd_list);
1019
1020	qtd->qtd_dma = virt_to_phys(address: qtd);
1021
1022	oxu->qtd_used[i] = 1;
1023	}
1024
1025	spin_unlock(lock: &oxu->mem_lock);
1026
1027	return qtd;
1028	}
1029
1030	static void oxu_qh_free(struct oxu_hcd *oxu, struct ehci_qh *qh)
1031	{
1032	int index;
1033
1034	spin_lock(lock: &oxu->mem_lock);
1035
1036	index = qh - &oxu->mem->qh_pool[0];
1037	oxu->qh_used[index] = 0;
1038
1039	spin_unlock(lock: &oxu->mem_lock);
1040	}
1041
1042	static void qh_destroy(struct kref *kref)
1043	{
1044	struct ehci_qh *qh = container_of(kref, struct ehci_qh, kref);
1045	struct oxu_hcd *oxu = qh->oxu;
1046
1047	/* clean qtds first, and know this is not linked */
1048	if (!list_empty(head: &qh->qtd_list) || qh->qh_next.ptr) {
1049	oxu_dbg(oxu, "unused qh not empty!\n");
1050	BUG();
1051	}
1052	if (qh->dummy)
1053	oxu_qtd_free(oxu, qtd: qh->dummy);
1054	oxu_qh_free(oxu, qh);
1055	}
1056
1057	static struct ehci_qh *oxu_qh_alloc(struct oxu_hcd *oxu)
1058	{
1059	int i;
1060	struct ehci_qh *qh = NULL;
1061
1062	spin_lock(lock: &oxu->mem_lock);
1063
1064	for (i = 0; i < QHEAD_NUM; i++)
1065	if (!oxu->qh_used[i])
1066	break;
1067
1068	if (i < QHEAD_NUM) {
1069	qh = (struct ehci_qh *) &oxu->mem->qh_pool[i];
1070	memset(qh, 0, sizeof *qh);
1071
1072	kref_init(kref: &qh->kref);
1073	qh->oxu = oxu;
1074	qh->qh_dma = virt_to_phys(address: qh);
1075	INIT_LIST_HEAD(list: &qh->qtd_list);
1076
1077	/* dummy td enables safe urb queuing */
1078	qh->dummy = ehci_qtd_alloc(oxu);
1079	if (qh->dummy == NULL) {
1080	oxu_dbg(oxu, "no dummy td\n");
1081	oxu->qh_used[i] = 0;
1082	qh = NULL;
1083	goto unlock;
1084	}
1085
1086	oxu->qh_used[i] = 1;
1087	}
1088	unlock:
1089	spin_unlock(lock: &oxu->mem_lock);
1090
1091	return qh;
1092	}
1093
1094	/* to share a qh (cpu threads, or hc) */
1095	static inline struct ehci_qh *qh_get(struct ehci_qh *qh)
1096	{
1097	kref_get(kref: &qh->kref);
1098	return qh;
1099	}
1100
1101	static inline void qh_put(struct ehci_qh *qh)
1102	{
1103	kref_put(kref: &qh->kref, release: qh_destroy);
1104	}
1105
1106	static void oxu_murb_free(struct oxu_hcd *oxu, struct oxu_murb *murb)
1107	{
1108	int index;
1109
1110	spin_lock(lock: &oxu->mem_lock);
1111
1112	index = murb - &oxu->murb_pool[0];
1113	oxu->murb_used[index] = 0;
1114
1115	spin_unlock(lock: &oxu->mem_lock);
1116	}
1117
1118	static struct oxu_murb *oxu_murb_alloc(struct oxu_hcd *oxu)
1119
1120	{
1121	int i;
1122	struct oxu_murb *murb = NULL;
1123
1124	spin_lock(lock: &oxu->mem_lock);
1125
1126	for (i = 0; i < MURB_NUM; i++)
1127	if (!oxu->murb_used[i])
1128	break;
1129
1130	if (i < MURB_NUM) {
1131	murb = &(oxu->murb_pool)[i];
1132
1133	oxu->murb_used[i] = 1;
1134	}
1135
1136	spin_unlock(lock: &oxu->mem_lock);
1137
1138	return murb;
1139	}
1140
1141	/* The queue heads and transfer descriptors are managed from pools tied
1142	* to each of the "per device" structures.
1143	* This is the initialisation and cleanup code.
1144	*/
1145	static void ehci_mem_cleanup(struct oxu_hcd *oxu)
1146	{
1147	kfree(objp: oxu->murb_pool);
1148	oxu->murb_pool = NULL;
1149
1150	if (oxu->async)
1151	qh_put(qh: oxu->async);
1152	oxu->async = NULL;
1153
1154	del_timer(timer: &oxu->urb_timer);
1155
1156	oxu->periodic = NULL;
1157
1158	/* shadow periodic table */
1159	kfree(objp: oxu->pshadow);
1160	oxu->pshadow = NULL;
1161	}
1162
1163	/* Remember to add cleanup code (above) if you add anything here.
1164	*/
1165	static int ehci_mem_init(struct oxu_hcd *oxu, gfp_t flags)
1166	{
1167	int i;
1168
1169	for (i = 0; i < oxu->periodic_size; i++)
1170	oxu->mem->frame_list[i] = EHCI_LIST_END;
1171	for (i = 0; i < QHEAD_NUM; i++)
1172	oxu->qh_used[i] = 0;
1173	for (i = 0; i < QTD_NUM; i++)
1174	oxu->qtd_used[i] = 0;
1175
1176	oxu->murb_pool = kcalloc(MURB_NUM, size: sizeof(struct oxu_murb), flags);
1177	if (!oxu->murb_pool)
1178	goto fail;
1179
1180	for (i = 0; i < MURB_NUM; i++)
1181	oxu->murb_used[i] = 0;
1182
1183	oxu->async = oxu_qh_alloc(oxu);
1184	if (!oxu->async)
1185	goto fail;
1186
1187	oxu->periodic = (__le32 *) &oxu->mem->frame_list;
1188	oxu->periodic_dma = virt_to_phys(address: oxu->periodic);
1189
1190	for (i = 0; i < oxu->periodic_size; i++)
1191	oxu->periodic[i] = EHCI_LIST_END;
1192
1193	/* software shadow of hardware table */
1194	oxu->pshadow = kcalloc(n: oxu->periodic_size, size: sizeof(void *), flags);
1195	if (oxu->pshadow != NULL)
1196	return 0;
1197
1198	fail:
1199	oxu_dbg(oxu, "couldn't init memory\n");
1200	ehci_mem_cleanup(oxu);
1201	return -ENOMEM;
1202	}
1203
1204	/* Fill a qtd, returning how much of the buffer we were able to queue up.
1205	*/
1206	static int qtd_fill(struct ehci_qtd *qtd, dma_addr_t buf, size_t len,
1207	int token, int maxpacket)
1208	{
1209	int i, count;
1210	u64 addr = buf;
1211
1212	/* one buffer entry per 4K ... first might be short or unaligned */
1213	qtd->hw_buf[0] = cpu_to_le32((u32)addr);
1214	qtd->hw_buf_hi[0] = cpu_to_le32((u32)(addr >> 32));
1215	count = 0x1000 - (buf & 0x0fff); /* rest of that page */
1216	if (likely(len < count)) /* ... iff needed */
1217	count = len;
1218	else {
1219	buf += 0x1000;
1220	buf &= ~0x0fff;
1221
1222	/* per-qtd limit: from 16K to 20K (best alignment) */
1223	for (i = 1; count < len && i < 5; i++) {
1224	addr = buf;
1225	qtd->hw_buf[i] = cpu_to_le32((u32)addr);
1226	qtd->hw_buf_hi[i] = cpu_to_le32((u32)(addr >> 32));
1227	buf += 0x1000;
1228	if ((count + 0x1000) < len)
1229	count += 0x1000;
1230	else
1231	count = len;
1232	}
1233
1234	/* short packets may only terminate transfers */
1235	if (count != len)
1236	count -= (count % maxpacket);
1237	}
1238	qtd->hw_token = cpu_to_le32((count << 16) | token);
1239	qtd->length = count;
1240
1241	return count;
1242	}
1243
1244	static inline void qh_update(struct oxu_hcd *oxu,
1245	struct ehci_qh *qh, struct ehci_qtd *qtd)
1246	{
1247	/* writes to an active overlay are unsafe */
1248	BUG_ON(qh->qh_state != QH_STATE_IDLE);
1249
1250	qh->hw_qtd_next = QTD_NEXT(qtd->qtd_dma);
1251	qh->hw_alt_next = EHCI_LIST_END;
1252
1253	/* Except for control endpoints, we make hardware maintain data
1254	* toggle (like OHCI) ... here (re)initialize the toggle in the QH,
1255	* and set the pseudo-toggle in udev. Only usb_clear_halt() will
1256	* ever clear it.
1257	*/
1258	if (!(qh->hw_info1 & cpu_to_le32(1 << 14))) {
1259	unsigned is_out, epnum;
1260
1261	is_out = !(qtd->hw_token & cpu_to_le32(1 << 8));
1262	epnum = (le32_to_cpup(p: &qh->hw_info1) >> 8) & 0x0f;
1263	if (unlikely(!usb_gettoggle(qh->dev, epnum, is_out))) {
1264	qh->hw_token &= ~cpu_to_le32(QTD_TOGGLE);
1265	usb_settoggle(qh->dev, epnum, is_out, 1);
1266	}
1267	}
1268
1269	/* HC must see latest qtd and qh data before we clear ACTIVE+HALT */
1270	wmb();
1271	qh->hw_token &= cpu_to_le32(QTD_TOGGLE | QTD_STS_PING);
1272	}
1273
1274	/* If it weren't for a common silicon quirk (writing the dummy into the qh
1275	* overlay, so qh->hw_token wrongly becomes inactive/halted), only fault
1276	* recovery (including urb dequeue) would need software changes to a QH...
1277	*/
1278	static void qh_refresh(struct oxu_hcd *oxu, struct ehci_qh *qh)
1279	{
1280	struct ehci_qtd *qtd;
1281
1282	if (list_empty(head: &qh->qtd_list))
1283	qtd = qh->dummy;
1284	else {
1285	qtd = list_entry(qh->qtd_list.next,
1286	struct ehci_qtd, qtd_list);
1287	/* first qtd may already be partially processed */
1288	if (cpu_to_le32(qtd->qtd_dma) == qh->hw_current)
1289	qtd = NULL;
1290	}
1291
1292	if (qtd)
1293	qh_update(oxu, qh, qtd);
1294	}
1295
1296	static void qtd_copy_status(struct oxu_hcd *oxu, struct urb *urb,
1297	size_t length, u32 token)
1298	{
1299	/* count IN/OUT bytes, not SETUP (even short packets) */
1300	if (likely(QTD_PID(token) != 2))
1301	urb->actual_length += length - QTD_LENGTH(token);
1302
1303	/* don't modify error codes */
1304	if (unlikely(urb->status != -EINPROGRESS))
1305	return;
1306
1307	/* force cleanup after short read; not always an error */
1308	if (unlikely(IS_SHORT_READ(token)))
1309	urb->status = -EREMOTEIO;
1310
1311	/* serious "can't proceed" faults reported by the hardware */
1312	if (token & QTD_STS_HALT) {
1313	if (token & QTD_STS_BABBLE) {
1314	/* FIXME "must" disable babbling device's port too */
1315	urb->status = -EOVERFLOW;
1316	} else if (token & QTD_STS_MMF) {
1317	/* fs/ls interrupt xfer missed the complete-split */
1318	urb->status = -EPROTO;
1319	} else if (token & QTD_STS_DBE) {
1320	urb->status = (QTD_PID(token) == 1) /* IN ? */
1321	? -ENOSR /* hc couldn't read data */
1322	: -ECOMM; /* hc couldn't write data */
1323	} else if (token & QTD_STS_XACT) {
1324	/* timeout, bad crc, wrong PID, etc; retried */
1325	if (QTD_CERR(token))
1326	urb->status = -EPIPE;
1327	else {
1328	oxu_dbg(oxu, "devpath %s ep%d%s 3strikes\n",
1329	urb->dev->devpath,
1330	usb_pipeendpoint(urb->pipe),
1331	usb_pipein(urb->pipe) ? "in" : "out");
1332	urb->status = -EPROTO;
1333	}
1334	/* CERR nonzero + no errors + halt --> stall */
1335	} else if (QTD_CERR(token))
1336	urb->status = -EPIPE;
1337	else /* unknown */
1338	urb->status = -EPROTO;
1339
1340	oxu_vdbg(oxu, "dev%d ep%d%s qtd token %08x --> status %d\n",
1341	usb_pipedevice(urb->pipe),
1342	usb_pipeendpoint(urb->pipe),
1343	usb_pipein(urb->pipe) ? "in" : "out",
1344	token, urb->status);
1345	}
1346	}
1347
1348	static void ehci_urb_done(struct oxu_hcd *oxu, struct urb *urb)
1349	__releases(oxu->lock)
1350	__acquires(oxu->lock)
1351	{
1352	if (likely(urb->hcpriv != NULL)) {
1353	struct ehci_qh *qh = (struct ehci_qh *) urb->hcpriv;
1354
1355	/* S-mask in a QH means it's an interrupt urb */
1356	if ((qh->hw_info2 & cpu_to_le32(QH_SMASK)) != 0) {
1357
1358	/* ... update hc-wide periodic stats (for usbfs) */
1359	oxu_to_hcd(oxu)->self.bandwidth_int_reqs--;
1360	}
1361	qh_put(qh);
1362	}
1363
1364	urb->hcpriv = NULL;
1365	switch (urb->status) {
1366	case -EINPROGRESS: /* success */
1367	urb->status = 0;
1368	break;
1369	default: /* fault */
1370	break;
1371	case -EREMOTEIO: /* fault or normal */
1372	if (!(urb->transfer_flags & URB_SHORT_NOT_OK))
1373	urb->status = 0;
1374	break;
1375	case -ECONNRESET: /* canceled */
1376	case -ENOENT:
1377	break;
1378	}
1379
1380	#ifdef OXU_URB_TRACE
1381	oxu_dbg(oxu, "%s %s urb %p ep%d%s status %d len %d/%d\n",
1382	__func__, urb->dev->devpath, urb,
1383	usb_pipeendpoint(urb->pipe),
1384	usb_pipein(urb->pipe) ? "in" : "out",
1385	urb->status,
1386	urb->actual_length, urb->transfer_buffer_length);
1387	#endif
1388
1389	/* complete() can reenter this HCD */
1390	spin_unlock(lock: &oxu->lock);
1391	usb_hcd_giveback_urb(hcd: oxu_to_hcd(oxu), urb, status: urb->status);
1392	spin_lock(lock: &oxu->lock);
1393	}
1394
1395	static void start_unlink_async(struct oxu_hcd *oxu, struct ehci_qh *qh);
1396	static void unlink_async(struct oxu_hcd *oxu, struct ehci_qh *qh);
1397
1398	static void intr_deschedule(struct oxu_hcd *oxu, struct ehci_qh *qh);
1399	static int qh_schedule(struct oxu_hcd *oxu, struct ehci_qh *qh);
1400
1401	#define HALT_BIT cpu_to_le32(QTD_STS_HALT)
1402
1403	/* Process and free completed qtds for a qh, returning URBs to drivers.
1404	* Chases up to qh->hw_current. Returns number of completions called,
1405	* indicating how much "real" work we did.
1406	*/
1407	static unsigned qh_completions(struct oxu_hcd *oxu, struct ehci_qh *qh)
1408	{
1409	struct ehci_qtd *last = NULL, *end = qh->dummy;
1410	struct ehci_qtd *qtd, *tmp;
1411	int stopped;
1412	unsigned count = 0;
1413	int do_status = 0;
1414	u8 state;
1415	struct oxu_murb *murb = NULL;
1416
1417	if (unlikely(list_empty(&qh->qtd_list)))
1418	return count;
1419
1420	/* completions (or tasks on other cpus) must never clobber HALT
1421	* till we've gone through and cleaned everything up, even when
1422	* they add urbs to this qh's queue or mark them for unlinking.
1423	*
1424	* NOTE: unlinking expects to be done in queue order.
1425	*/
1426	state = qh->qh_state;
1427	qh->qh_state = QH_STATE_COMPLETING;
1428	stopped = (state == QH_STATE_IDLE);
1429
1430	/* remove de-activated QTDs from front of queue.
1431	* after faults (including short reads), cleanup this urb
1432	* then let the queue advance.
1433	* if queue is stopped, handles unlinks.
1434	*/
1435	list_for_each_entry_safe(qtd, tmp, &qh->qtd_list, qtd_list) {
1436	struct urb *urb;
1437	u32 token = 0;
1438
1439	urb = qtd->urb;
1440
1441	/* Clean up any state from previous QTD ...*/
1442	if (last) {
1443	if (likely(last->urb != urb)) {
1444	if (last->urb->complete == NULL) {
1445	murb = (struct oxu_murb *) last->urb;
1446	last->urb = murb->main;
1447	if (murb->last) {
1448	ehci_urb_done(oxu, urb: last->urb);
1449	count++;
1450	}
1451	oxu_murb_free(oxu, murb);
1452	} else {
1453	ehci_urb_done(oxu, urb: last->urb);
1454	count++;
1455	}
1456	}
1457	oxu_qtd_free(oxu, qtd: last);
1458	last = NULL;
1459	}
1460
1461	/* ignore urbs submitted during completions we reported */
1462	if (qtd == end)
1463	break;
1464
1465	/* hardware copies qtd out of qh overlay */
1466	rmb();
1467	token = le32_to_cpu(qtd->hw_token);
1468
1469	/* always clean up qtds the hc de-activated */
1470	if ((token & QTD_STS_ACTIVE) == 0) {
1471
1472	if ((token & QTD_STS_HALT) != 0) {
1473	stopped = 1;
1474
1475	/* magic dummy for some short reads; qh won't advance.
1476	* that silicon quirk can kick in with this dummy too.
1477	*/
1478	} else if (IS_SHORT_READ(token) &&
1479	!(qtd->hw_alt_next & EHCI_LIST_END)) {
1480	stopped = 1;
1481	goto halt;
1482	}
1483
1484	/* stop scanning when we reach qtds the hc is using */
1485	} else if (likely(!stopped &&
1486	HC_IS_RUNNING(oxu_to_hcd(oxu)->state))) {
1487	break;
1488
1489	} else {
1490	stopped = 1;
1491
1492	if (unlikely(!HC_IS_RUNNING(oxu_to_hcd(oxu)->state)))
1493	urb->status = -ESHUTDOWN;
1494
1495	/* ignore active urbs unless some previous qtd
1496	* for the urb faulted (including short read) or
1497	* its urb was canceled. we may patch qh or qtds.
1498	*/
1499	if (likely(urb->status == -EINPROGRESS))
1500	continue;
1501
1502	/* issue status after short control reads */
1503	if (unlikely(do_status != 0)
1504	&& QTD_PID(token) == 0 /* OUT */) {
1505	do_status = 0;
1506	continue;
1507	}
1508
1509	/* token in overlay may be most current */
1510	if (state == QH_STATE_IDLE
1511	&& cpu_to_le32(qtd->qtd_dma)
1512	== qh->hw_current)
1513	token = le32_to_cpu(qh->hw_token);
1514
1515	/* force halt for unlinked or blocked qh, so we'll
1516	* patch the qh later and so that completions can't
1517	* activate it while we "know" it's stopped.
1518	*/
1519	if ((HALT_BIT & qh->hw_token) == 0) {
1520	halt:
1521	qh->hw_token |= HALT_BIT;
1522	wmb();
1523	}
1524	}
1525
1526	/* Remove it from the queue */
1527	qtd_copy_status(oxu, urb: urb->complete ?
1528	urb : ((struct oxu_murb *) urb)->main,
1529	length: qtd->length, token);
1530	if ((usb_pipein(qtd->urb->pipe)) &&
1531	(NULL != qtd->transfer_buffer))
1532	memcpy(qtd->transfer_buffer, qtd->buffer, qtd->length);
1533	do_status = (urb->status == -EREMOTEIO)
1534	&& usb_pipecontrol(urb->pipe);
1535
1536	if (stopped && qtd->qtd_list.prev != &qh->qtd_list) {
1537	last = list_entry(qtd->qtd_list.prev,
1538	struct ehci_qtd, qtd_list);
1539	last->hw_next = qtd->hw_next;
1540	}
1541	list_del(entry: &qtd->qtd_list);
1542	last = qtd;
1543	}
1544
1545	/* last urb's completion might still need calling */
1546	if (likely(last != NULL)) {
1547	if (last->urb->complete == NULL) {
1548	murb = (struct oxu_murb *) last->urb;
1549	last->urb = murb->main;
1550	if (murb->last) {
1551	ehci_urb_done(oxu, urb: last->urb);
1552	count++;
1553	}
1554	oxu_murb_free(oxu, murb);
1555	} else {
1556	ehci_urb_done(oxu, urb: last->urb);
1557	count++;
1558	}
1559	oxu_qtd_free(oxu, qtd: last);
1560	}
1561
1562	/* restore original state; caller must unlink or relink */
1563	qh->qh_state = state;
1564
1565	/* be sure the hardware's done with the qh before refreshing
1566	* it after fault cleanup, or recovering from silicon wrongly
1567	* overlaying the dummy qtd (which reduces DMA chatter).
1568	*/
1569	if (stopped != 0 || qh->hw_qtd_next == EHCI_LIST_END) {
1570	switch (state) {
1571	case QH_STATE_IDLE:
1572	qh_refresh(oxu, qh);
1573	break;
1574	case QH_STATE_LINKED:
1575	/* should be rare for periodic transfers,
1576	* except maybe high bandwidth ...
1577	*/
1578	if ((cpu_to_le32(QH_SMASK)
1579	& qh->hw_info2) != 0) {
1580	intr_deschedule(oxu, qh);
1581	(void) qh_schedule(oxu, qh);
1582	} else
1583	unlink_async(oxu, qh);
1584	break;
1585	/* otherwise, unlink already started */
1586	}
1587	}
1588
1589	return count;
1590	}
1591
1592	/* High bandwidth multiplier, as encoded in highspeed endpoint descriptors */
1593	#define hb_mult(wMaxPacketSize) (1 + (((wMaxPacketSize) >> 11) & 0x03))
1594	/* ... and packet size, for any kind of endpoint descriptor */
1595	#define max_packet(wMaxPacketSize) ((wMaxPacketSize) & 0x07ff)
1596
1597	/* Reverse of qh_urb_transaction: free a list of TDs.
1598	* used for cleanup after errors, before HC sees an URB's TDs.
1599	*/
1600	static void qtd_list_free(struct oxu_hcd *oxu,
1601	struct urb *urb, struct list_head *head)
1602	{
1603	struct ehci_qtd *qtd, *temp;
1604
1605	list_for_each_entry_safe(qtd, temp, head, qtd_list) {
1606	list_del(entry: &qtd->qtd_list);
1607	oxu_qtd_free(oxu, qtd);
1608	}
1609	}
1610
1611	/* Create a list of filled qtds for this URB; won't link into qh.
1612	*/
1613	static struct list_head *qh_urb_transaction(struct oxu_hcd *oxu,
1614	struct urb *urb,
1615	struct list_head *head,
1616	gfp_t flags)
1617	{
1618	struct ehci_qtd *qtd, *qtd_prev;
1619	dma_addr_t buf;
1620	int len, maxpacket;
1621	int is_input;
1622	u32 token;
1623	void *transfer_buf = NULL;
1624	int ret;
1625
1626	/*
1627	* URBs map to sequences of QTDs: one logical transaction
1628	*/
1629	qtd = ehci_qtd_alloc(oxu);
1630	if (unlikely(!qtd))
1631	return NULL;
1632	list_add_tail(new: &qtd->qtd_list, head);
1633	qtd->urb = urb;
1634
1635	token = QTD_STS_ACTIVE;
1636	token |= (EHCI_TUNE_CERR << 10);
1637	/* for split transactions, SplitXState initialized to zero */
1638
1639	len = urb->transfer_buffer_length;
1640	is_input = usb_pipein(urb->pipe);
1641	if (!urb->transfer_buffer && urb->transfer_buffer_length && is_input)
1642	urb->transfer_buffer = phys_to_virt(address: urb->transfer_dma);
1643
1644	if (usb_pipecontrol(urb->pipe)) {
1645	/* SETUP pid */
1646	ret = oxu_buf_alloc(oxu, qtd, len: sizeof(struct usb_ctrlrequest));
1647	if (ret)
1648	goto cleanup;
1649
1650	qtd_fill(qtd, buf: qtd->buffer_dma, len: sizeof(struct usb_ctrlrequest),
1651	token: token | (2 /* "setup" */ << 8), maxpacket: 8);
1652	memcpy(qtd->buffer, qtd->urb->setup_packet,
1653	sizeof(struct usb_ctrlrequest));
1654
1655	/* ... and always at least one more pid */
1656	token ^= QTD_TOGGLE;
1657	qtd_prev = qtd;
1658	qtd = ehci_qtd_alloc(oxu);
1659	if (unlikely(!qtd))
1660	goto cleanup;
1661	qtd->urb = urb;
1662	qtd_prev->hw_next = QTD_NEXT(qtd->qtd_dma);
1663	list_add_tail(new: &qtd->qtd_list, head);
1664
1665	/* for zero length DATA stages, STATUS is always IN */
1666	if (len == 0)
1667	token |= (1 /* "in" */ << 8);
1668	}
1669
1670	/*
1671	* Data transfer stage: buffer setup
1672	*/
1673
1674	ret = oxu_buf_alloc(oxu, qtd, len);
1675	if (ret)
1676	goto cleanup;
1677
1678	buf = qtd->buffer_dma;
1679	transfer_buf = urb->transfer_buffer;
1680
1681	if (!is_input)
1682	memcpy(qtd->buffer, qtd->urb->transfer_buffer, len);
1683
1684	if (is_input)
1685	token |= (1 /* "in" */ << 8);
1686	/* else it's already initted to "out" pid (0 << 8) */
1687
1688	maxpacket = usb_maxpacket(udev: urb->dev, pipe: urb->pipe);
1689
1690	/*
1691	* buffer gets wrapped in one or more qtds;
1692	* last one may be "short" (including zero len)
1693	* and may serve as a control status ack
1694	*/
1695	for (;;) {
1696	int this_qtd_len;
1697
1698	this_qtd_len = qtd_fill(qtd, buf, len, token, maxpacket);
1699	qtd->transfer_buffer = transfer_buf;
1700	len -= this_qtd_len;
1701	buf += this_qtd_len;
1702	transfer_buf += this_qtd_len;
1703	if (is_input)
1704	qtd->hw_alt_next = oxu->async->hw_alt_next;
1705
1706	/* qh makes control packets use qtd toggle; maybe switch it */
1707	if ((maxpacket & (this_qtd_len + (maxpacket - 1))) == 0)
1708	token ^= QTD_TOGGLE;
1709
1710	if (likely(len <= 0))
1711	break;
1712
1713	qtd_prev = qtd;
1714	qtd = ehci_qtd_alloc(oxu);
1715	if (unlikely(!qtd))
1716	goto cleanup;
1717	if (likely(len > 0)) {
1718	ret = oxu_buf_alloc(oxu, qtd, len);
1719	if (ret)
1720	goto cleanup;
1721	}
1722	qtd->urb = urb;
1723	qtd_prev->hw_next = QTD_NEXT(qtd->qtd_dma);
1724	list_add_tail(new: &qtd->qtd_list, head);
1725	}
1726
1727	/* unless the bulk/interrupt caller wants a chance to clean
1728	* up after short reads, hc should advance qh past this urb
1729	*/
1730	if (likely((urb->transfer_flags & URB_SHORT_NOT_OK) == 0
1731	|| usb_pipecontrol(urb->pipe)))
1732	qtd->hw_alt_next = EHCI_LIST_END;
1733
1734	/*
1735	* control requests may need a terminating data "status" ack;
1736	* bulk ones may need a terminating short packet (zero length).
1737	*/
1738	if (likely(urb->transfer_buffer_length != 0)) {
1739	int one_more = 0;
1740
1741	if (usb_pipecontrol(urb->pipe)) {
1742	one_more = 1;
1743	token ^= 0x0100; /* "in" <--> "out" */
1744	token |= QTD_TOGGLE; /* force DATA1 */
1745	} else if (usb_pipebulk(urb->pipe)
1746	&& (urb->transfer_flags & URB_ZERO_PACKET)
1747	&& !(urb->transfer_buffer_length % maxpacket)) {
1748	one_more = 1;
1749	}
1750	if (one_more) {
1751	qtd_prev = qtd;
1752	qtd = ehci_qtd_alloc(oxu);
1753	if (unlikely(!qtd))
1754	goto cleanup;
1755	qtd->urb = urb;
1756	qtd_prev->hw_next = QTD_NEXT(qtd->qtd_dma);
1757	list_add_tail(new: &qtd->qtd_list, head);
1758
1759	/* never any data in such packets */
1760	qtd_fill(qtd, buf: 0, len: 0, token, maxpacket: 0);
1761	}
1762	}
1763
1764	/* by default, enable interrupt on urb completion */
1765	qtd->hw_token |= cpu_to_le32(QTD_IOC);
1766	return head;
1767
1768	cleanup:
1769	qtd_list_free(oxu, urb, head);
1770	return NULL;
1771	}
1772
1773	/* Each QH holds a qtd list; a QH is used for everything except iso.
1774	*
1775	* For interrupt urbs, the scheduler must set the microframe scheduling
1776	* mask(s) each time the QH gets scheduled. For highspeed, that's
1777	* just one microframe in the s-mask. For split interrupt transactions
1778	* there are additional complications: c-mask, maybe FSTNs.
1779	*/
1780	static struct ehci_qh *qh_make(struct oxu_hcd *oxu,
1781	struct urb *urb, gfp_t flags)
1782	{
1783	struct ehci_qh *qh = oxu_qh_alloc(oxu);
1784	u32 info1 = 0, info2 = 0;
1785	int is_input, type;
1786	int maxp = 0;
1787
1788	if (!qh)
1789	return qh;
1790
1791	/*
1792	* init endpoint/device data for this QH
1793	*/
1794	info1 |= usb_pipeendpoint(urb->pipe) << 8;
1795	info1 |= usb_pipedevice(urb->pipe) << 0;
1796
1797	is_input = usb_pipein(urb->pipe);
1798	type = usb_pipetype(urb->pipe);
1799	maxp = usb_maxpacket(udev: urb->dev, pipe: urb->pipe);
1800
1801	/* Compute interrupt scheduling parameters just once, and save.
1802	* - allowing for high bandwidth, how many nsec/uframe are used?
1803	* - split transactions need a second CSPLIT uframe; same question
1804	* - splits also need a schedule gap (for full/low speed I/O)
1805	* - qh has a polling interval
1806	*
1807	* For control/bulk requests, the HC or TT handles these.
1808	*/
1809	if (type == PIPE_INTERRUPT) {
1810	qh->usecs = NS_TO_US(usb_calc_bus_time(USB_SPEED_HIGH,
1811	is_input, 0,
1812	hb_mult(maxp) * max_packet(maxp)));
1813	qh->start = NO_FRAME;
1814
1815	if (urb->dev->speed == USB_SPEED_HIGH) {
1816	qh->c_usecs = 0;
1817	qh->gap_uf = 0;
1818
1819	qh->period = urb->interval >> 3;
1820	if (qh->period == 0 && urb->interval != 1) {
1821	/* NOTE interval 2 or 4 uframes could work.
1822	* But interval 1 scheduling is simpler, and
1823	* includes high bandwidth.
1824	*/
1825	oxu_dbg(oxu, "intr period %d uframes, NYET!\n",
1826	urb->interval);
1827	goto done;
1828	}
1829	} else {
1830	struct usb_tt *tt = urb->dev->tt;
1831	int think_time;
1832
1833	/* gap is f(FS/LS transfer times) */
1834	qh->gap_uf = 1 + usb_calc_bus_time(speed: urb->dev->speed,
1835	is_input, isoc: 0, bytecount: maxp) / (125 * 1000);
1836
1837	/* FIXME this just approximates SPLIT/CSPLIT times */
1838	if (is_input) { /* SPLIT, gap, CSPLIT+DATA */
1839	qh->c_usecs = qh->usecs + HS_USECS(0);
1840	qh->usecs = HS_USECS(1);
1841	} else { /* SPLIT+DATA, gap, CSPLIT */
1842	qh->usecs += HS_USECS(1);
1843	qh->c_usecs = HS_USECS(0);
1844	}
1845
1846	think_time = tt ? tt->think_time : 0;
1847	qh->tt_usecs = NS_TO_US(think_time +
1848	usb_calc_bus_time(urb->dev->speed,
1849	is_input, 0, max_packet(maxp)));
1850	qh->period = urb->interval;
1851	}
1852	}
1853
1854	/* support for tt scheduling, and access to toggles */
1855	qh->dev = urb->dev;
1856
1857	/* using TT? */
1858	switch (urb->dev->speed) {
1859	case USB_SPEED_LOW:
1860	info1 |= (1 << 12); /* EPS "low" */
1861	fallthrough;
1862
1863	case USB_SPEED_FULL:
1864	/* EPS 0 means "full" */
1865	if (type != PIPE_INTERRUPT)
1866	info1 |= (EHCI_TUNE_RL_TT << 28);
1867	if (type == PIPE_CONTROL) {
1868	info1 |= (1 << 27); /* for TT */
1869	info1 |= 1 << 14; /* toggle from qtd */
1870	}
1871	info1 |= maxp << 16;
1872
1873	info2 |= (EHCI_TUNE_MULT_TT << 30);
1874	info2 |= urb->dev->ttport << 23;
1875
1876	/* NOTE: if (PIPE_INTERRUPT) { scheduler sets c-mask } */
1877
1878	break;
1879
1880	case USB_SPEED_HIGH: /* no TT involved */
1881	info1 |= (2 << 12); /* EPS "high" */
1882	if (type == PIPE_CONTROL) {
1883	info1 |= (EHCI_TUNE_RL_HS << 28);
1884	info1 |= 64 << 16; /* usb2 fixed maxpacket */
1885	info1 |= 1 << 14; /* toggle from qtd */
1886	info2 |= (EHCI_TUNE_MULT_HS << 30);
1887	} else if (type == PIPE_BULK) {
1888	info1 |= (EHCI_TUNE_RL_HS << 28);
1889	info1 |= 512 << 16; /* usb2 fixed maxpacket */
1890	info2 |= (EHCI_TUNE_MULT_HS << 30);
1891	} else { /* PIPE_INTERRUPT */
1892	info1 |= max_packet(maxp) << 16;
1893	info2 |= hb_mult(maxp) << 30;
1894	}
1895	break;
1896	default:
1897	oxu_dbg(oxu, "bogus dev %p speed %d\n", urb->dev, urb->dev->speed);
1898	done:
1899	qh_put(qh);
1900	return NULL;
1901	}
1902
1903	/* NOTE: if (PIPE_INTERRUPT) { scheduler sets s-mask } */
1904
1905	/* init as live, toggle clear, advance to dummy */
1906	qh->qh_state = QH_STATE_IDLE;
1907	qh->hw_info1 = cpu_to_le32(info1);
1908	qh->hw_info2 = cpu_to_le32(info2);
1909	usb_settoggle(urb->dev, usb_pipeendpoint(urb->pipe), !is_input, 1);
1910	qh_refresh(oxu, qh);
1911	return qh;
1912	}
1913
1914	/* Move qh (and its qtds) onto async queue; maybe enable queue.
1915	*/
1916	static void qh_link_async(struct oxu_hcd *oxu, struct ehci_qh *qh)
1917	{
1918	__le32 dma = QH_NEXT(qh->qh_dma);
1919	struct ehci_qh *head;
1920
1921	/* (re)start the async schedule? */
1922	head = oxu->async;
1923	timer_action_done(oxu, action: TIMER_ASYNC_OFF);
1924	if (!head->qh_next.qh) {
1925	u32 cmd = readl(addr: &oxu->regs->command);
1926
1927	if (!(cmd & CMD_ASE)) {
1928	/* in case a clear of CMD_ASE didn't take yet */
1929	(void)handshake(oxu, ptr: &oxu->regs->status,
1930	STS_ASS, done: 0, usec: 150);
1931	cmd |= CMD_ASE | CMD_RUN;
1932	writel(val: cmd, addr: &oxu->regs->command);
1933	oxu_to_hcd(oxu)->state = HC_STATE_RUNNING;
1934	/* posted write need not be known to HC yet ... */
1935	}
1936	}
1937
1938	/* clear halt and/or toggle; and maybe recover from silicon quirk */
1939	if (qh->qh_state == QH_STATE_IDLE)
1940	qh_refresh(oxu, qh);
1941
1942	/* splice right after start */
1943	qh->qh_next = head->qh_next;
1944	qh->hw_next = head->hw_next;
1945	wmb();
1946
1947	head->qh_next.qh = qh;
1948	head->hw_next = dma;
1949
1950	qh->qh_state = QH_STATE_LINKED;
1951	/* qtd completions reported later by interrupt */
1952	}
1953
1954	#define QH_ADDR_MASK cpu_to_le32(0x7f)
1955
1956	/*
1957	* For control/bulk/interrupt, return QH with these TDs appended.
1958	* Allocates and initializes the QH if necessary.
1959	* Returns null if it can't allocate a QH it needs to.
1960	* If the QH has TDs (urbs) already, that's great.
1961	*/
1962	static struct ehci_qh *qh_append_tds(struct oxu_hcd *oxu,
1963	struct urb *urb, struct list_head *qtd_list,
1964	int epnum, void **ptr)
1965	{
1966	struct ehci_qh *qh = NULL;
1967
1968	qh = (struct ehci_qh *) *ptr;
1969	if (unlikely(qh == NULL)) {
1970	/* can't sleep here, we have oxu->lock... */
1971	qh = qh_make(oxu, urb, GFP_ATOMIC);
1972	*ptr = qh;
1973	}
1974	if (likely(qh != NULL)) {
1975	struct ehci_qtd *qtd;
1976
1977	if (unlikely(list_empty(qtd_list)))
1978	qtd = NULL;
1979	else
1980	qtd = list_entry(qtd_list->next, struct ehci_qtd,
1981	qtd_list);
1982
1983	/* control qh may need patching ... */
1984	if (unlikely(epnum == 0)) {
1985
1986	/* usb_reset_device() briefly reverts to address 0 */
1987	if (usb_pipedevice(urb->pipe) == 0)
1988	qh->hw_info1 &= ~QH_ADDR_MASK;
1989	}
1990
1991	/* just one way to queue requests: swap with the dummy qtd.
1992	* only hc or qh_refresh() ever modify the overlay.
1993	*/
1994	if (likely(qtd != NULL)) {
1995	struct ehci_qtd *dummy;
1996	dma_addr_t dma;
1997	__le32 token;
1998
1999	/* to avoid racing the HC, use the dummy td instead of
2000	* the first td of our list (becomes new dummy). both
2001	* tds stay deactivated until we're done, when the
2002	* HC is allowed to fetch the old dummy (4.10.2).
2003	*/
2004	token = qtd->hw_token;
2005	qtd->hw_token = HALT_BIT;
2006	wmb();
2007	dummy = qh->dummy;
2008
2009	dma = dummy->qtd_dma;
2010	*dummy = *qtd;
2011	dummy->qtd_dma = dma;
2012
2013	list_del(entry: &qtd->qtd_list);
2014	list_add(new: &dummy->qtd_list, head: qtd_list);
2015	list_splice(list: qtd_list, head: qh->qtd_list.prev);
2016
2017	ehci_qtd_init(qtd, dma: qtd->qtd_dma);
2018	qh->dummy = qtd;
2019
2020	/* hc must see the new dummy at list end */
2021	dma = qtd->qtd_dma;
2022	qtd = list_entry(qh->qtd_list.prev,
2023	struct ehci_qtd, qtd_list);
2024	qtd->hw_next = QTD_NEXT(dma);
2025
2026	/* let the hc process these next qtds */
2027	dummy->hw_token = (token & ~(0x80));
2028	wmb();
2029	dummy->hw_token = token;
2030
2031	urb->hcpriv = qh_get(qh);
2032	}
2033	}
2034	return qh;
2035	}
2036
2037	static int submit_async(struct oxu_hcd *oxu, struct urb *urb,
2038	struct list_head *qtd_list, gfp_t mem_flags)
2039	{
2040	int epnum = urb->ep->desc.bEndpointAddress;
2041	unsigned long flags;
2042	struct ehci_qh *qh = NULL;
2043	int rc = 0;
2044	#ifdef OXU_URB_TRACE
2045	struct ehci_qtd *qtd;
2046
2047	qtd = list_entry(qtd_list->next, struct ehci_qtd, qtd_list);
2048
2049	oxu_dbg(oxu, "%s %s urb %p ep%d%s len %d, qtd %p [qh %p]\n",
2050	__func__, urb->dev->devpath, urb,
2051	epnum & 0x0f, (epnum & USB_DIR_IN) ? "in" : "out",
2052	urb->transfer_buffer_length,
2053	qtd, urb->ep->hcpriv);
2054	#endif
2055
2056	spin_lock_irqsave(&oxu->lock, flags);
2057	if (unlikely(!HCD_HW_ACCESSIBLE(oxu_to_hcd(oxu)))) {
2058	rc = -ESHUTDOWN;
2059	goto done;
2060	}
2061
2062	qh = qh_append_tds(oxu, urb, qtd_list, epnum, ptr: &urb->ep->hcpriv);
2063	if (unlikely(qh == NULL)) {
2064	rc = -ENOMEM;
2065	goto done;
2066	}
2067
2068	/* Control/bulk operations through TTs don't need scheduling,
2069	* the HC and TT handle it when the TT has a buffer ready.
2070	*/
2071	if (likely(qh->qh_state == QH_STATE_IDLE))
2072	qh_link_async(oxu, qh: qh_get(qh));
2073	done:
2074	spin_unlock_irqrestore(lock: &oxu->lock, flags);
2075	if (unlikely(qh == NULL))
2076	qtd_list_free(oxu, urb, head: qtd_list);
2077	return rc;
2078	}
2079
2080	/* The async qh for the qtds being reclaimed are now unlinked from the HC */
2081
2082	static void end_unlink_async(struct oxu_hcd *oxu)
2083	{
2084	struct ehci_qh *qh = oxu->reclaim;
2085	struct ehci_qh *next;
2086
2087	timer_action_done(oxu, action: TIMER_IAA_WATCHDOG);
2088
2089	qh->qh_state = QH_STATE_IDLE;
2090	qh->qh_next.qh = NULL;
2091	qh_put(qh); /* refcount from reclaim */
2092
2093	/* other unlink(s) may be pending (in QH_STATE_UNLINK_WAIT) */
2094	next = qh->reclaim;
2095	oxu->reclaim = next;
2096	oxu->reclaim_ready = 0;
2097	qh->reclaim = NULL;
2098
2099	qh_completions(oxu, qh);
2100
2101	if (!list_empty(head: &qh->qtd_list)
2102	&& HC_IS_RUNNING(oxu_to_hcd(oxu)->state))
2103	qh_link_async(oxu, qh);
2104	else {
2105	qh_put(qh); /* refcount from async list */
2106
2107	/* it's not free to turn the async schedule on/off; leave it
2108	* active but idle for a while once it empties.
2109	*/
2110	if (HC_IS_RUNNING(oxu_to_hcd(oxu)->state)
2111	&& oxu->async->qh_next.qh == NULL)
2112	timer_action(oxu, action: TIMER_ASYNC_OFF);
2113	}
2114
2115	if (next) {
2116	oxu->reclaim = NULL;
2117	start_unlink_async(oxu, qh: next);
2118	}
2119	}
2120
2121	/* makes sure the async qh will become idle */
2122	/* caller must own oxu->lock */
2123
2124	static void start_unlink_async(struct oxu_hcd *oxu, struct ehci_qh *qh)
2125	{
2126	int cmd = readl(addr: &oxu->regs->command);
2127	struct ehci_qh *prev;
2128
2129	#ifdef DEBUG
2130	assert_spin_locked(&oxu->lock);
2131	BUG_ON(oxu->reclaim || (qh->qh_state != QH_STATE_LINKED
2132	&& qh->qh_state != QH_STATE_UNLINK_WAIT));
2133	#endif
2134
2135	/* stop async schedule right now? */
2136	if (unlikely(qh == oxu->async)) {
2137	/* can't get here without STS_ASS set */
2138	if (oxu_to_hcd(oxu)->state != HC_STATE_HALT
2139	&& !oxu->reclaim) {
2140	/* ... and CMD_IAAD clear */
2141	writel(val: cmd & ~CMD_ASE, addr: &oxu->regs->command);
2142	wmb();
2143	/* handshake later, if we need to */
2144	timer_action_done(oxu, action: TIMER_ASYNC_OFF);
2145	}
2146	return;
2147	}
2148
2149	qh->qh_state = QH_STATE_UNLINK;
2150	oxu->reclaim = qh = qh_get(qh);
2151
2152	prev = oxu->async;
2153	while (prev->qh_next.qh != qh)
2154	prev = prev->qh_next.qh;
2155
2156	prev->hw_next = qh->hw_next;
2157	prev->qh_next = qh->qh_next;
2158	wmb();
2159
2160	if (unlikely(oxu_to_hcd(oxu)->state == HC_STATE_HALT)) {
2161	/* if (unlikely(qh->reclaim != 0))
2162	* this will recurse, probably not much
2163	*/
2164	end_unlink_async(oxu);
2165	return;
2166	}
2167
2168	oxu->reclaim_ready = 0;
2169	cmd |= CMD_IAAD;
2170	writel(val: cmd, addr: &oxu->regs->command);
2171	(void) readl(addr: &oxu->regs->command);
2172	timer_action(oxu, action: TIMER_IAA_WATCHDOG);
2173	}
2174
2175	static void scan_async(struct oxu_hcd *oxu)
2176	{
2177	struct ehci_qh *qh;
2178	enum ehci_timer_action action = TIMER_IO_WATCHDOG;
2179
2180	if (!++(oxu->stamp))
2181	oxu->stamp++;
2182	timer_action_done(oxu, action: TIMER_ASYNC_SHRINK);
2183	rescan:
2184	qh = oxu->async->qh_next.qh;
2185	if (likely(qh != NULL)) {
2186	do {
2187	/* clean any finished work for this qh */
2188	if (!list_empty(head: &qh->qtd_list)
2189	&& qh->stamp != oxu->stamp) {
2190	int temp;
2191
2192	/* unlinks could happen here; completion
2193	* reporting drops the lock. rescan using
2194	* the latest schedule, but don't rescan
2195	* qhs we already finished (no looping).
2196	*/
2197	qh = qh_get(qh);
2198	qh->stamp = oxu->stamp;
2199	temp = qh_completions(oxu, qh);
2200	qh_put(qh);
2201	if (temp != 0)
2202	goto rescan;
2203	}
2204
2205	/* unlink idle entries, reducing HC PCI usage as well
2206	* as HCD schedule-scanning costs. delay for any qh
2207	* we just scanned, there's a not-unusual case that it
2208	* doesn't stay idle for long.
2209	* (plus, avoids some kind of re-activation race.)
2210	*/
2211	if (list_empty(head: &qh->qtd_list)) {
2212	if (qh->stamp == oxu->stamp)
2213	action = TIMER_ASYNC_SHRINK;
2214	else if (!oxu->reclaim
2215	&& qh->qh_state == QH_STATE_LINKED)
2216	start_unlink_async(oxu, qh);
2217	}
2218
2219	qh = qh->qh_next.qh;
2220	} while (qh);
2221	}
2222	if (action == TIMER_ASYNC_SHRINK)
2223	timer_action(oxu, action: TIMER_ASYNC_SHRINK);
2224	}
2225
2226	/*
2227	* periodic_next_shadow - return "next" pointer on shadow list
2228	* @periodic: host pointer to qh/itd/sitd
2229	* @tag: hardware tag for type of this record
2230	*/
2231	static union ehci_shadow *periodic_next_shadow(union ehci_shadow *periodic,
2232	__le32 tag)
2233	{
2234	switch (tag) {
2235	default:
2236	case Q_TYPE_QH:
2237	return &periodic->qh->qh_next;
2238	}
2239	}
2240
2241	/* caller must hold oxu->lock */
2242	static void periodic_unlink(struct oxu_hcd *oxu, unsigned frame, void *ptr)
2243	{
2244	union ehci_shadow *prev_p = &oxu->pshadow[frame];
2245	__le32 *hw_p = &oxu->periodic[frame];
2246	union ehci_shadow here = *prev_p;
2247
2248	/* find predecessor of "ptr"; hw and shadow lists are in sync */
2249	while (here.ptr && here.ptr != ptr) {
2250	prev_p = periodic_next_shadow(periodic: prev_p, Q_NEXT_TYPE(*hw_p));
2251	hw_p = here.hw_next;
2252	here = *prev_p;
2253	}
2254	/* an interrupt entry (at list end) could have been shared */
2255	if (!here.ptr)
2256	return;
2257
2258	/* update shadow and hardware lists ... the old "next" pointers
2259	* from ptr may still be in use, the caller updates them.
2260	*/
2261	*prev_p = *periodic_next_shadow(periodic: &here, Q_NEXT_TYPE(*hw_p));
2262	*hw_p = *here.hw_next;
2263	}
2264
2265	/* how many of the uframe's 125 usecs are allocated? */
2266	static unsigned short periodic_usecs(struct oxu_hcd *oxu,
2267	unsigned frame, unsigned uframe)
2268	{
2269	__le32 *hw_p = &oxu->periodic[frame];
2270	union ehci_shadow *q = &oxu->pshadow[frame];
2271	unsigned usecs = 0;
2272
2273	while (q->ptr) {
2274	switch (Q_NEXT_TYPE(*hw_p)) {
2275	case Q_TYPE_QH:
2276	default:
2277	/* is it in the S-mask? */
2278	if (q->qh->hw_info2 & cpu_to_le32(1 << uframe))
2279	usecs += q->qh->usecs;
2280	/* ... or C-mask? */
2281	if (q->qh->hw_info2 & cpu_to_le32(1 << (8 + uframe)))
2282	usecs += q->qh->c_usecs;
2283	hw_p = &q->qh->hw_next;
2284	q = &q->qh->qh_next;
2285	break;
2286	}
2287	}
2288	#ifdef DEBUG
2289	if (usecs > 100)
2290	oxu_err(oxu, "uframe %d sched overrun: %d usecs\n",
2291	frame * 8 + uframe, usecs);
2292	#endif
2293	return usecs;
2294	}
2295
2296	static int enable_periodic(struct oxu_hcd *oxu)
2297	{
2298	u32 cmd;
2299	int status;
2300
2301	/* did clearing PSE did take effect yet?
2302	* takes effect only at frame boundaries...
2303	*/
2304	status = handshake(oxu, ptr: &oxu->regs->status, STS_PSS, done: 0, usec: 9 * 125);
2305	if (status != 0) {
2306	oxu_to_hcd(oxu)->state = HC_STATE_HALT;
2307	usb_hc_died(hcd: oxu_to_hcd(oxu));
2308	return status;
2309	}
2310
2311	cmd = readl(addr: &oxu->regs->command) | CMD_PSE;
2312	writel(val: cmd, addr: &oxu->regs->command);
2313	/* posted write ... PSS happens later */
2314	oxu_to_hcd(oxu)->state = HC_STATE_RUNNING;
2315
2316	/* make sure ehci_work scans these */
2317	oxu->next_uframe = readl(addr: &oxu->regs->frame_index)
2318	% (oxu->periodic_size << 3);
2319	return 0;
2320	}
2321
2322	static int disable_periodic(struct oxu_hcd *oxu)
2323	{
2324	u32 cmd;
2325	int status;
2326
2327	/* did setting PSE not take effect yet?
2328	* takes effect only at frame boundaries...
2329	*/
2330	status = handshake(oxu, ptr: &oxu->regs->status, STS_PSS, STS_PSS, usec: 9 * 125);
2331	if (status != 0) {
2332	oxu_to_hcd(oxu)->state = HC_STATE_HALT;
2333	usb_hc_died(hcd: oxu_to_hcd(oxu));
2334	return status;
2335	}
2336
2337	cmd = readl(addr: &oxu->regs->command) & ~CMD_PSE;
2338	writel(val: cmd, addr: &oxu->regs->command);
2339	/* posted write ... */
2340
2341	oxu->next_uframe = -1;
2342	return 0;
2343	}
2344
2345	/* periodic schedule slots have iso tds (normal or split) first, then a
2346	* sparse tree for active interrupt transfers.
2347	*
2348	* this just links in a qh; caller guarantees uframe masks are set right.
2349	* no FSTN support (yet; oxu 0.96+)
2350	*/
2351	static int qh_link_periodic(struct oxu_hcd *oxu, struct ehci_qh *qh)
2352	{
2353	unsigned i;
2354	unsigned period = qh->period;
2355
2356	dev_dbg(&qh->dev->dev,
2357	"link qh%d-%04x/%p start %d [%d/%d us]\n",
2358	period, le32_to_cpup(&qh->hw_info2) & (QH_CMASK | QH_SMASK),
2359	qh, qh->start, qh->usecs, qh->c_usecs);
2360
2361	/* high bandwidth, or otherwise every microframe */
2362	if (period == 0)
2363	period = 1;
2364
2365	for (i = qh->start; i < oxu->periodic_size; i += period) {
2366	union ehci_shadow *prev = &oxu->pshadow[i];
2367	__le32 *hw_p = &oxu->periodic[i];
2368	union ehci_shadow here = *prev;
2369	__le32 type = 0;
2370
2371	/* skip the iso nodes at list head */
2372	while (here.ptr) {
2373	type = Q_NEXT_TYPE(*hw_p);
2374	if (type == Q_TYPE_QH)
2375	break;
2376	prev = periodic_next_shadow(periodic: prev, tag: type);
2377	hw_p = &here.qh->hw_next;
2378	here = *prev;
2379	}
2380
2381	/* sorting each branch by period (slow-->fast)
2382	* enables sharing interior tree nodes
2383	*/
2384	while (here.ptr && qh != here.qh) {
2385	if (qh->period > here.qh->period)
2386	break;
2387	prev = &here.qh->qh_next;
2388	hw_p = &here.qh->hw_next;
2389	here = *prev;
2390	}
2391	/* link in this qh, unless some earlier pass did that */
2392	if (qh != here.qh) {
2393	qh->qh_next = here;
2394	if (here.qh)
2395	qh->hw_next = *hw_p;
2396	wmb();
2397	prev->qh = qh;
2398	*hw_p = QH_NEXT(qh->qh_dma);
2399	}
2400	}
2401	qh->qh_state = QH_STATE_LINKED;
2402	qh_get(qh);
2403
2404	/* update per-qh bandwidth for usbfs */
2405	oxu_to_hcd(oxu)->self.bandwidth_allocated += qh->period
2406	? ((qh->usecs + qh->c_usecs) / qh->period)
2407	: (qh->usecs * 8);
2408
2409	/* maybe enable periodic schedule processing */
2410	if (!oxu->periodic_sched++)
2411	return enable_periodic(oxu);
2412
2413	return 0;
2414	}
2415
2416	static void qh_unlink_periodic(struct oxu_hcd *oxu, struct ehci_qh *qh)
2417	{
2418	unsigned i;
2419	unsigned period;
2420
2421	/* FIXME:
2422	* IF this isn't high speed
2423	* and this qh is active in the current uframe
2424	* (and overlay token SplitXstate is false?)
2425	* THEN
2426	* qh->hw_info1 |= cpu_to_le32(1 << 7 "ignore");
2427	*/
2428
2429	/* high bandwidth, or otherwise part of every microframe */
2430	period = qh->period;
2431	if (period == 0)
2432	period = 1;
2433
2434	for (i = qh->start; i < oxu->periodic_size; i += period)
2435	periodic_unlink(oxu, frame: i, ptr: qh);
2436
2437	/* update per-qh bandwidth for usbfs */
2438	oxu_to_hcd(oxu)->self.bandwidth_allocated -= qh->period
2439	? ((qh->usecs + qh->c_usecs) / qh->period)
2440	: (qh->usecs * 8);
2441
2442	dev_dbg(&qh->dev->dev,
2443	"unlink qh%d-%04x/%p start %d [%d/%d us]\n",
2444	qh->period,
2445	le32_to_cpup(&qh->hw_info2) & (QH_CMASK | QH_SMASK),
2446	qh, qh->start, qh->usecs, qh->c_usecs);
2447
2448	/* qh->qh_next still "live" to HC */
2449	qh->qh_state = QH_STATE_UNLINK;
2450	qh->qh_next.ptr = NULL;
2451	qh_put(qh);
2452
2453	/* maybe turn off periodic schedule */
2454	oxu->periodic_sched--;
2455	if (!oxu->periodic_sched)
2456	(void) disable_periodic(oxu);
2457	}
2458
2459	static void intr_deschedule(struct oxu_hcd *oxu, struct ehci_qh *qh)
2460	{
2461	unsigned wait;
2462
2463	qh_unlink_periodic(oxu, qh);
2464
2465	/* simple/paranoid: always delay, expecting the HC needs to read
2466	* qh->hw_next or finish a writeback after SPLIT/CSPLIT ... and
2467	* expect hub_wq to clean up after any CSPLITs we won't issue.
2468	* active high speed queues may need bigger delays...
2469	*/
2470	if (list_empty(head: &qh->qtd_list)
2471	|| (cpu_to_le32(QH_CMASK) & qh->hw_info2) != 0)
2472	wait = 2;
2473	else
2474	wait = 55; /* worst case: 3 * 1024 */
2475
2476	udelay(wait);
2477	qh->qh_state = QH_STATE_IDLE;
2478	qh->hw_next = EHCI_LIST_END;
2479	wmb();
2480	}
2481
2482	static int check_period(struct oxu_hcd *oxu,
2483	unsigned frame, unsigned uframe,
2484	unsigned period, unsigned usecs)
2485	{
2486	int claimed;
2487
2488	/* complete split running into next frame?
2489	* given FSTN support, we could sometimes check...
2490	*/
2491	if (uframe >= 8)
2492	return 0;
2493
2494	/*
2495	* 80% periodic == 100 usec/uframe available
2496	* convert "usecs we need" to "max already claimed"
2497	*/
2498	usecs = 100 - usecs;
2499
2500	/* we "know" 2 and 4 uframe intervals were rejected; so
2501	* for period 0, check _every_ microframe in the schedule.
2502	*/
2503	if (unlikely(period == 0)) {
2504	do {
2505	for (uframe = 0; uframe < 7; uframe++) {
2506	claimed = periodic_usecs(oxu, frame, uframe);
2507	if (claimed > usecs)
2508	return 0;
2509	}
2510	} while ((frame += 1) < oxu->periodic_size);
2511
2512	/* just check the specified uframe, at that period */
2513	} else {
2514	do {
2515	claimed = periodic_usecs(oxu, frame, uframe);
2516	if (claimed > usecs)
2517	return 0;
2518	} while ((frame += period) < oxu->periodic_size);
2519	}
2520
2521	return 1;
2522	}
2523
2524	static int check_intr_schedule(struct oxu_hcd *oxu,
2525	unsigned frame, unsigned uframe,
2526	const struct ehci_qh *qh, __le32 *c_maskp)
2527	{
2528	int retval = -ENOSPC;
2529
2530	if (qh->c_usecs && uframe >= 6) /* FSTN territory? */
2531	goto done;
2532
2533	if (!check_period(oxu, frame, uframe, period: qh->period, usecs: qh->usecs))
2534	goto done;
2535	if (!qh->c_usecs) {
2536	retval = 0;
2537	*c_maskp = 0;
2538	goto done;
2539	}
2540
2541	done:
2542	return retval;
2543	}
2544
2545	/* "first fit" scheduling policy used the first time through,
2546	* or when the previous schedule slot can't be re-used.
2547	*/
2548	static int qh_schedule(struct oxu_hcd *oxu, struct ehci_qh *qh)
2549	{
2550	int status;
2551	unsigned uframe;
2552	__le32 c_mask;
2553	unsigned frame; /* 0..(qh->period - 1), or NO_FRAME */
2554
2555	qh_refresh(oxu, qh);
2556	qh->hw_next = EHCI_LIST_END;
2557	frame = qh->start;
2558
2559	/* reuse the previous schedule slots, if we can */
2560	if (frame < qh->period) {
2561	uframe = ffs(le32_to_cpup(&qh->hw_info2) & QH_SMASK);
2562	status = check_intr_schedule(oxu, frame, uframe: --uframe,
2563	qh, c_maskp: &c_mask);
2564	} else {
2565	uframe = 0;
2566	c_mask = 0;
2567	status = -ENOSPC;
2568	}
2569
2570	/* else scan the schedule to find a group of slots such that all
2571	* uframes have enough periodic bandwidth available.
2572	*/
2573	if (status) {
2574	/* "normal" case, uframing flexible except with splits */
2575	if (qh->period) {
2576	frame = qh->period - 1;
2577	do {
2578	for (uframe = 0; uframe < 8; uframe++) {
2579	status = check_intr_schedule(oxu,
2580	frame, uframe, qh,
2581	c_maskp: &c_mask);
2582	if (status == 0)
2583	break;
2584	}
2585	} while (status && frame--);
2586
2587	/* qh->period == 0 means every uframe */
2588	} else {
2589	frame = 0;
2590	status = check_intr_schedule(oxu, frame: 0, uframe: 0, qh, c_maskp: &c_mask);
2591	}
2592	if (status)
2593	goto done;
2594	qh->start = frame;
2595
2596	/* reset S-frame and (maybe) C-frame masks */
2597	qh->hw_info2 &= cpu_to_le32(~(QH_CMASK | QH_SMASK));
2598	qh->hw_info2 |= qh->period
2599	? cpu_to_le32(1 << uframe)
2600	: cpu_to_le32(QH_SMASK);
2601	qh->hw_info2 |= c_mask;
2602	} else
2603	oxu_dbg(oxu, "reused qh %p schedule\n", qh);
2604
2605	/* stuff into the periodic schedule */
2606	status = qh_link_periodic(oxu, qh);
2607	done:
2608	return status;
2609	}
2610
2611	static int intr_submit(struct oxu_hcd *oxu, struct urb *urb,
2612	struct list_head *qtd_list, gfp_t mem_flags)
2613	{
2614	unsigned epnum;
2615	unsigned long flags;
2616	struct ehci_qh *qh;
2617	int status = 0;
2618	struct list_head empty;
2619
2620	/* get endpoint and transfer/schedule data */
2621	epnum = urb->ep->desc.bEndpointAddress;
2622
2623	spin_lock_irqsave(&oxu->lock, flags);
2624
2625	if (unlikely(!HCD_HW_ACCESSIBLE(oxu_to_hcd(oxu)))) {
2626	status = -ESHUTDOWN;
2627	goto done;
2628	}
2629
2630	/* get qh and force any scheduling errors */
2631	INIT_LIST_HEAD(list: &empty);
2632	qh = qh_append_tds(oxu, urb, qtd_list: &empty, epnum, ptr: &urb->ep->hcpriv);
2633	if (qh == NULL) {
2634	status = -ENOMEM;
2635	goto done;
2636	}
2637	if (qh->qh_state == QH_STATE_IDLE) {
2638	status = qh_schedule(oxu, qh);
2639	if (status != 0)
2640	goto done;
2641	}
2642
2643	/* then queue the urb's tds to the qh */
2644	qh = qh_append_tds(oxu, urb, qtd_list, epnum, ptr: &urb->ep->hcpriv);
2645	BUG_ON(qh == NULL);
2646
2647	/* ... update usbfs periodic stats */
2648	oxu_to_hcd(oxu)->self.bandwidth_int_reqs++;
2649
2650	done:
2651	spin_unlock_irqrestore(lock: &oxu->lock, flags);
2652	if (status)
2653	qtd_list_free(oxu, urb, head: qtd_list);
2654
2655	return status;
2656	}
2657
2658	static inline int itd_submit(struct oxu_hcd *oxu, struct urb *urb,
2659	gfp_t mem_flags)
2660	{
2661	oxu_dbg(oxu, "iso support is missing!\n");
2662	return -ENOSYS;
2663	}
2664
2665	static inline int sitd_submit(struct oxu_hcd *oxu, struct urb *urb,
2666	gfp_t mem_flags)
2667	{
2668	oxu_dbg(oxu, "split iso support is missing!\n");
2669	return -ENOSYS;
2670	}
2671
2672	static void scan_periodic(struct oxu_hcd *oxu)
2673	{
2674	unsigned frame, clock, now_uframe, mod;
2675	unsigned modified;
2676
2677	mod = oxu->periodic_size << 3;
2678
2679	/*
2680	* When running, scan from last scan point up to "now"
2681	* else clean up by scanning everything that's left.
2682	* Touches as few pages as possible: cache-friendly.
2683	*/
2684	now_uframe = oxu->next_uframe;
2685	if (HC_IS_RUNNING(oxu_to_hcd(oxu)->state))
2686	clock = readl(addr: &oxu->regs->frame_index);
2687	else
2688	clock = now_uframe + mod - 1;
2689	clock %= mod;
2690
2691	for (;;) {
2692	union ehci_shadow q, *q_p;
2693	__le32 type, *hw_p;
2694
2695	/* don't scan past the live uframe */
2696	frame = now_uframe >> 3;
2697	if (frame != (clock >> 3)) {
2698	/* safe to scan the whole frame at once */
2699	now_uframe |= 0x07;
2700	}
2701
2702	restart:
2703	/* scan each element in frame's queue for completions */
2704	q_p = &oxu->pshadow[frame];
2705	hw_p = &oxu->periodic[frame];
2706	q.ptr = q_p->ptr;
2707	type = Q_NEXT_TYPE(*hw_p);
2708	modified = 0;
2709
2710	while (q.ptr != NULL) {
2711	union ehci_shadow temp;
2712
2713	switch (type) {
2714	case Q_TYPE_QH:
2715	/* handle any completions */
2716	temp.qh = qh_get(qh: q.qh);
2717	type = Q_NEXT_TYPE(q.qh->hw_next);
2718	q = q.qh->qh_next;
2719	modified = qh_completions(oxu, qh: temp.qh);
2720	if (unlikely(list_empty(&temp.qh->qtd_list)))
2721	intr_deschedule(oxu, qh: temp.qh);
2722	qh_put(qh: temp.qh);
2723	break;
2724	default:
2725	oxu_dbg(oxu, "corrupt type %d frame %d shadow %p\n",
2726	type, frame, q.ptr);
2727	q.ptr = NULL;
2728	}
2729
2730	/* assume completion callbacks modify the queue */
2731	if (unlikely(modified))
2732	goto restart;
2733	}
2734
2735	/* Stop when we catch up to the HC */
2736
2737	/* FIXME: this assumes we won't get lapped when
2738	* latencies climb; that should be rare, but...
2739	* detect it, and just go all the way around.
2740	* FLR might help detect this case, so long as latencies
2741	* don't exceed periodic_size msec (default 1.024 sec).
2742	*/
2743
2744	/* FIXME: likewise assumes HC doesn't halt mid-scan */
2745
2746	if (now_uframe == clock) {
2747	unsigned now;
2748
2749	if (!HC_IS_RUNNING(oxu_to_hcd(oxu)->state))
2750	break;
2751	oxu->next_uframe = now_uframe;
2752	now = readl(addr: &oxu->regs->frame_index) % mod;
2753	if (now_uframe == now)
2754	break;
2755
2756	/* rescan the rest of this frame, then ... */
2757	clock = now;
2758	} else {
2759	now_uframe++;
2760	now_uframe %= mod;
2761	}
2762	}
2763	}
2764
2765	/* On some systems, leaving remote wakeup enabled prevents system shutdown.
2766	* The firmware seems to think that powering off is a wakeup event!
2767	* This routine turns off remote wakeup and everything else, on all ports.
2768	*/
2769	static void ehci_turn_off_all_ports(struct oxu_hcd *oxu)
2770	{
2771	int port = HCS_N_PORTS(oxu->hcs_params);
2772
2773	while (port--)
2774	writel(PORT_RWC_BITS, addr: &oxu->regs->port_status[port]);
2775	}
2776
2777	static void ehci_port_power(struct oxu_hcd *oxu, int is_on)
2778	{
2779	unsigned port;
2780
2781	if (!HCS_PPC(oxu->hcs_params))
2782	return;
2783
2784	oxu_dbg(oxu, "...power%s ports...\n", is_on ? "up" : "down");
2785	for (port = HCS_N_PORTS(oxu->hcs_params); port > 0; ) {
2786	if (is_on)
2787	oxu_hub_control(hcd: oxu_to_hcd(oxu), SetPortFeature,
2788	USB_PORT_FEAT_POWER, wIndex: port--, NULL, wLength: 0);
2789	else
2790	oxu_hub_control(hcd: oxu_to_hcd(oxu), ClearPortFeature,
2791	USB_PORT_FEAT_POWER, wIndex: port--, NULL, wLength: 0);
2792	}
2793
2794	msleep(msecs: 20);
2795	}
2796
2797	/* Called from some interrupts, timers, and so on.
2798	* It calls driver completion functions, after dropping oxu->lock.
2799	*/
2800	static void ehci_work(struct oxu_hcd *oxu)
2801	{
2802	timer_action_done(oxu, action: TIMER_IO_WATCHDOG);
2803	if (oxu->reclaim_ready)
2804	end_unlink_async(oxu);
2805
2806	/* another CPU may drop oxu->lock during a schedule scan while
2807	* it reports urb completions. this flag guards against bogus
2808	* attempts at re-entrant schedule scanning.
2809	*/
2810	if (oxu->scanning)
2811	return;
2812	oxu->scanning = 1;
2813	scan_async(oxu);
2814	if (oxu->next_uframe != -1)
2815	scan_periodic(oxu);
2816	oxu->scanning = 0;
2817
2818	/* the IO watchdog guards against hardware or driver bugs that
2819	* misplace IRQs, and should let us run completely without IRQs.
2820	* such lossage has been observed on both VT6202 and VT8235.
2821	*/
2822	if (HC_IS_RUNNING(oxu_to_hcd(oxu)->state) &&
2823	(oxu->async->qh_next.ptr != NULL ||
2824	oxu->periodic_sched != 0))
2825	timer_action(oxu, action: TIMER_IO_WATCHDOG);
2826	}
2827
2828	static void unlink_async(struct oxu_hcd *oxu, struct ehci_qh *qh)
2829	{
2830	/* if we need to use IAA and it's busy, defer */
2831	if (qh->qh_state == QH_STATE_LINKED
2832	&& oxu->reclaim
2833	&& HC_IS_RUNNING(oxu_to_hcd(oxu)->state)) {
2834	struct ehci_qh *last;
2835
2836	for (last = oxu->reclaim;
2837	last->reclaim;
2838	last = last->reclaim)
2839	continue;
2840	qh->qh_state = QH_STATE_UNLINK_WAIT;
2841	last->reclaim = qh;
2842
2843	/* bypass IAA if the hc can't care */
2844	} else if (!HC_IS_RUNNING(oxu_to_hcd(oxu)->state) && oxu->reclaim)
2845	end_unlink_async(oxu);
2846
2847	/* something else might have unlinked the qh by now */
2848	if (qh->qh_state == QH_STATE_LINKED)
2849	start_unlink_async(oxu, qh);
2850	}
2851
2852	/*
2853	* USB host controller methods
2854	*/
2855
2856	static irqreturn_t oxu210_hcd_irq(struct usb_hcd *hcd)
2857	{
2858	struct oxu_hcd *oxu = hcd_to_oxu(hcd);
2859	u32 status, pcd_status = 0;
2860	int bh;
2861
2862	spin_lock(lock: &oxu->lock);
2863
2864	status = readl(addr: &oxu->regs->status);
2865
2866	/* e.g. cardbus physical eject */
2867	if (status == ~(u32) 0) {
2868	oxu_dbg(oxu, "device removed\n");
2869	goto dead;
2870	}
2871
2872	/* Shared IRQ? */
2873	status &= INTR_MASK;
2874	if (!status || unlikely(hcd->state == HC_STATE_HALT)) {
2875	spin_unlock(lock: &oxu->lock);
2876	return IRQ_NONE;
2877	}
2878
2879	/* clear (just) interrupts */
2880	writel(val: status, addr: &oxu->regs->status);
2881	readl(addr: &oxu->regs->command); /* unblock posted write */
2882	bh = 0;
2883
2884	#ifdef OXU_VERBOSE_DEBUG
2885	/* unrequested/ignored: Frame List Rollover */
2886	dbg_status(oxu, "irq", status);
2887	#endif
2888
2889	/* INT, ERR, and IAA interrupt rates can be throttled */
2890
2891	/* normal [4.15.1.2] or error [4.15.1.1] completion */
2892	if (likely((status & (STS_INT|STS_ERR)) != 0))
2893	bh = 1;
2894
2895	/* complete the unlinking of some qh [4.15.2.3] */
2896	if (status & STS_IAA) {
2897	oxu->reclaim_ready = 1;
2898	bh = 1;
2899	}
2900
2901	/* remote wakeup [4.3.1] */
2902	if (status & STS_PCD) {
2903	unsigned i = HCS_N_PORTS(oxu->hcs_params);
2904	pcd_status = status;
2905
2906	/* resume root hub? */
2907	if (!(readl(addr: &oxu->regs->command) & CMD_RUN))
2908	usb_hcd_resume_root_hub(hcd);
2909
2910	while (i--) {
2911	int pstatus = readl(addr: &oxu->regs->port_status[i]);
2912
2913	if (pstatus & PORT_OWNER)
2914	continue;
2915	if (!(pstatus & PORT_RESUME)
2916	|| oxu->reset_done[i] != 0)
2917	continue;
2918
2919	/* start USB_RESUME_TIMEOUT resume signaling from this
2920	* port, and make hub_wq collect PORT_STAT_C_SUSPEND to
2921	* stop that signaling.
2922	*/
2923	oxu->reset_done[i] = jiffies +
2924	msecs_to_jiffies(USB_RESUME_TIMEOUT);
2925	oxu_dbg(oxu, "port %d remote wakeup\n", i + 1);
2926	mod_timer(timer: &hcd->rh_timer, expires: oxu->reset_done[i]);
2927	}
2928	}
2929
2930	/* PCI errors [4.15.2.4] */
2931	if (unlikely((status & STS_FATAL) != 0)) {
2932	/* bogus "fatal" IRQs appear on some chips... why? */
2933	status = readl(addr: &oxu->regs->status);
2934	dbg_cmd(oxu, "fatal", readl(&oxu->regs->command));
2935	dbg_status(oxu, "fatal", status);
2936	if (status & STS_HALT) {
2937	oxu_err(oxu, "fatal error\n");
2938	dead:
2939	ehci_reset(oxu);
2940	writel(val: 0, addr: &oxu->regs->configured_flag);
2941	usb_hc_died(hcd);
2942	/* generic layer kills/unlinks all urbs, then
2943	* uses oxu_stop to clean up the rest
2944	*/
2945	bh = 1;
2946	}
2947	}
2948
2949	if (bh)
2950	ehci_work(oxu);
2951	spin_unlock(lock: &oxu->lock);
2952	if (pcd_status & STS_PCD)
2953	usb_hcd_poll_rh_status(hcd);
2954	return IRQ_HANDLED;
2955	}
2956
2957	static irqreturn_t oxu_irq(struct usb_hcd *hcd)
2958	{
2959	struct oxu_hcd *oxu = hcd_to_oxu(hcd);
2960	int ret = IRQ_HANDLED;
2961
2962	u32 status = oxu_readl(base: hcd->regs, OXU_CHIPIRQSTATUS);
2963	u32 enable = oxu_readl(base: hcd->regs, OXU_CHIPIRQEN_SET);
2964
2965	/* Disable all interrupt */
2966	oxu_writel(base: hcd->regs, OXU_CHIPIRQEN_CLR, val: enable);
2967
2968	if ((oxu->is_otg && (status & OXU_USBOTGI)) ||
2969	(!oxu->is_otg && (status & OXU_USBSPHI)))
2970	oxu210_hcd_irq(hcd);
2971	else
2972	ret = IRQ_NONE;
2973
2974	/* Enable all interrupt back */
2975	oxu_writel(base: hcd->regs, OXU_CHIPIRQEN_SET, val: enable);
2976
2977	return ret;
2978	}
2979
2980	static void oxu_watchdog(struct timer_list *t)
2981	{
2982	struct oxu_hcd *oxu = from_timer(oxu, t, watchdog);
2983	unsigned long flags;
2984
2985	spin_lock_irqsave(&oxu->lock, flags);
2986
2987	/* lost IAA irqs wedge things badly; seen with a vt8235 */
2988	if (oxu->reclaim) {
2989	u32 status = readl(addr: &oxu->regs->status);
2990	if (status & STS_IAA) {
2991	oxu_vdbg(oxu, "lost IAA\n");
2992	writel(STS_IAA, addr: &oxu->regs->status);
2993	oxu->reclaim_ready = 1;
2994	}
2995	}
2996
2997	/* stop async processing after it's idled a bit */
2998	if (test_bit(TIMER_ASYNC_OFF, &oxu->actions))
2999	start_unlink_async(oxu, qh: oxu->async);
3000
3001	/* oxu could run by timer, without IRQs ... */
3002	ehci_work(oxu);
3003
3004	spin_unlock_irqrestore(lock: &oxu->lock, flags);
3005	}
3006
3007	/* One-time init, only for memory state.
3008	*/
3009	static int oxu_hcd_init(struct usb_hcd *hcd)
3010	{
3011	struct oxu_hcd *oxu = hcd_to_oxu(hcd);
3012	u32 temp;
3013	int retval;
3014	u32 hcc_params;
3015
3016	spin_lock_init(&oxu->lock);
3017
3018	timer_setup(&oxu->watchdog, oxu_watchdog, 0);
3019
3020	/*
3021	* hw default: 1K periodic list heads, one per frame.
3022	* periodic_size can shrink by USBCMD update if hcc_params allows.
3023	*/
3024	oxu->periodic_size = DEFAULT_I_TDPS;
3025	retval = ehci_mem_init(oxu, GFP_KERNEL);
3026	if (retval < 0)
3027	return retval;
3028
3029	/* controllers may cache some of the periodic schedule ... */
3030	hcc_params = readl(addr: &oxu->caps->hcc_params);
3031	if (HCC_ISOC_CACHE(hcc_params)) /* full frame cache */
3032	oxu->i_thresh = 8;
3033	else /* N microframes cached */
3034	oxu->i_thresh = 2 + HCC_ISOC_THRES(hcc_params);
3035
3036	oxu->reclaim = NULL;
3037	oxu->reclaim_ready = 0;
3038	oxu->next_uframe = -1;
3039
3040	/*
3041	* dedicate a qh for the async ring head, since we couldn't unlink
3042	* a 'real' qh without stopping the async schedule [4.8]. use it
3043	* as the 'reclamation list head' too.
3044	* its dummy is used in hw_alt_next of many tds, to prevent the qh
3045	* from automatically advancing to the next td after short reads.
3046	*/
3047	oxu->async->qh_next.qh = NULL;
3048	oxu->async->hw_next = QH_NEXT(oxu->async->qh_dma);
3049	oxu->async->hw_info1 = cpu_to_le32(QH_HEAD);
3050	oxu->async->hw_token = cpu_to_le32(QTD_STS_HALT);
3051	oxu->async->hw_qtd_next = EHCI_LIST_END;
3052	oxu->async->qh_state = QH_STATE_LINKED;
3053	oxu->async->hw_alt_next = QTD_NEXT(oxu->async->dummy->qtd_dma);
3054
3055	/* clear interrupt enables, set irq latency */
3056	if (log2_irq_thresh < 0 || log2_irq_thresh > 6)
3057	log2_irq_thresh = 0;
3058	temp = 1 << (16 + log2_irq_thresh);
3059	if (HCC_CANPARK(hcc_params)) {
3060	/* HW default park == 3, on hardware that supports it (like
3061	* NVidia and ALI silicon), maximizes throughput on the async
3062	* schedule by avoiding QH fetches between transfers.
3063	*
3064	* With fast usb storage devices and NForce2, "park" seems to
3065	* make problems: throughput reduction (!), data errors...
3066	*/
3067	if (park) {
3068	park = min(park, (unsigned) 3);
3069	temp |= CMD_PARK;
3070	temp |= park << 8;
3071	}
3072	oxu_dbg(oxu, "park %d\n", park);
3073	}
3074	if (HCC_PGM_FRAMELISTLEN(hcc_params)) {
3075	/* periodic schedule size can be smaller than default */
3076	temp &= ~(3 << 2);
3077	temp |= (EHCI_TUNE_FLS << 2);
3078	}
3079	oxu->command = temp;
3080
3081	return 0;
3082	}
3083
3084	/* Called during probe() after chip reset completes.
3085	*/
3086	static int oxu_reset(struct usb_hcd *hcd)
3087	{
3088	struct oxu_hcd *oxu = hcd_to_oxu(hcd);
3089
3090	spin_lock_init(&oxu->mem_lock);
3091	INIT_LIST_HEAD(list: &oxu->urb_list);
3092	oxu->urb_len = 0;
3093
3094	if (oxu->is_otg) {
3095	oxu->caps = hcd->regs + OXU_OTG_CAP_OFFSET;
3096	oxu->regs = hcd->regs + OXU_OTG_CAP_OFFSET + \
3097	HC_LENGTH(readl(&oxu->caps->hc_capbase));
3098
3099	oxu->mem = hcd->regs + OXU_SPH_MEM;
3100	} else {
3101	oxu->caps = hcd->regs + OXU_SPH_CAP_OFFSET;
3102	oxu->regs = hcd->regs + OXU_SPH_CAP_OFFSET + \
3103	HC_LENGTH(readl(&oxu->caps->hc_capbase));
3104
3105	oxu->mem = hcd->regs + OXU_OTG_MEM;
3106	}
3107
3108	oxu->hcs_params = readl(addr: &oxu->caps->hcs_params);
3109	oxu->sbrn = 0x20;
3110
3111	return oxu_hcd_init(hcd);
3112	}
3113
3114	static int oxu_run(struct usb_hcd *hcd)
3115	{
3116	struct oxu_hcd *oxu = hcd_to_oxu(hcd);
3117	int retval;
3118	u32 temp, hcc_params;
3119
3120	hcd->uses_new_polling = 1;
3121
3122	/* EHCI spec section 4.1 */
3123	retval = ehci_reset(oxu);
3124	if (retval != 0) {
3125	ehci_mem_cleanup(oxu);
3126	return retval;
3127	}
3128	writel(val: oxu->periodic_dma, addr: &oxu->regs->frame_list);
3129	writel(val: (u32) oxu->async->qh_dma, addr: &oxu->regs->async_next);
3130
3131	/* hcc_params controls whether oxu->regs->segment must (!!!)
3132	* be used; it constrains QH/ITD/SITD and QTD locations.
3133	* dma_pool consistent memory always uses segment zero.
3134	* streaming mappings for I/O buffers, like dma_map_single(),
3135	* can return segments above 4GB, if the device allows.
3136	*
3137	* NOTE: the dma mask is visible through dev->dma_mask, so
3138	* drivers can pass this info along ... like NETIF_F_HIGHDMA,
3139	* Scsi_Host.highmem_io, and so forth. It's readonly to all
3140	* host side drivers though.
3141	*/
3142	hcc_params = readl(addr: &oxu->caps->hcc_params);
3143	if (HCC_64BIT_ADDR(hcc_params))
3144	writel(val: 0, addr: &oxu->regs->segment);
3145
3146	oxu->command &= ~(CMD_LRESET | CMD_IAAD | CMD_PSE |
3147	CMD_ASE | CMD_RESET);
3148	oxu->command |= CMD_RUN;
3149	writel(val: oxu->command, addr: &oxu->regs->command);
3150	dbg_cmd(oxu, "init", oxu->command);
3151
3152	/*
3153	* Start, enabling full USB 2.0 functionality ... usb 1.1 devices
3154	* are explicitly handed to companion controller(s), so no TT is
3155	* involved with the root hub. (Except where one is integrated,
3156	* and there's no companion controller unless maybe for USB OTG.)
3157	*/
3158	hcd->state = HC_STATE_RUNNING;
3159	writel(FLAG_CF, addr: &oxu->regs->configured_flag);
3160	readl(addr: &oxu->regs->command); /* unblock posted writes */
3161
3162	temp = HC_VERSION(readl(&oxu->caps->hc_capbase));
3163	oxu_info(oxu, "USB %x.%x started, quasi-EHCI %x.%02x, driver %s%s\n",
3164	((oxu->sbrn & 0xf0)>>4), (oxu->sbrn & 0x0f),
3165	temp >> 8, temp & 0xff, DRIVER_VERSION,
3166	ignore_oc ? ", overcurrent ignored" : "");
3167
3168	writel(INTR_MASK, addr: &oxu->regs->intr_enable); /* Turn On Interrupts */
3169
3170	return 0;
3171	}
3172
3173	static void oxu_stop(struct usb_hcd *hcd)
3174	{
3175	struct oxu_hcd *oxu = hcd_to_oxu(hcd);
3176
3177	/* Turn off port power on all root hub ports. */
3178	ehci_port_power(oxu, is_on: 0);
3179
3180	/* no more interrupts ... */
3181	del_timer_sync(timer: &oxu->watchdog);
3182
3183	spin_lock_irq(lock: &oxu->lock);
3184	if (HC_IS_RUNNING(hcd->state))
3185	ehci_quiesce(oxu);
3186
3187	ehci_reset(oxu);
3188	writel(val: 0, addr: &oxu->regs->intr_enable);
3189	spin_unlock_irq(lock: &oxu->lock);
3190
3191	/* let companion controllers work when we aren't */
3192	writel(val: 0, addr: &oxu->regs->configured_flag);
3193
3194	/* root hub is shut down separately (first, when possible) */
3195	spin_lock_irq(lock: &oxu->lock);
3196	if (oxu->async)
3197	ehci_work(oxu);
3198	spin_unlock_irq(lock: &oxu->lock);
3199	ehci_mem_cleanup(oxu);
3200
3201	dbg_status(oxu, "oxu_stop completed", readl(&oxu->regs->status));
3202	}
3203
3204	/* Kick in for silicon on any bus (not just pci, etc).
3205	* This forcibly disables dma and IRQs, helping kexec and other cases
3206	* where the next system software may expect clean state.
3207	*/
3208	static void oxu_shutdown(struct usb_hcd *hcd)
3209	{
3210	struct oxu_hcd *oxu = hcd_to_oxu(hcd);
3211
3212	(void) ehci_halt(oxu);
3213	ehci_turn_off_all_ports(oxu);
3214
3215	/* make BIOS/etc use companion controller during reboot */
3216	writel(val: 0, addr: &oxu->regs->configured_flag);
3217
3218	/* unblock posted writes */
3219	readl(addr: &oxu->regs->configured_flag);
3220	}
3221
3222	/* Non-error returns are a promise to giveback() the urb later
3223	* we drop ownership so next owner (or urb unlink) can get it
3224	*
3225	* urb + dev is in hcd.self.controller.urb_list
3226	* we're queueing TDs onto software and hardware lists
3227	*
3228	* hcd-specific init for hcpriv hasn't been done yet
3229	*
3230	* NOTE: control, bulk, and interrupt share the same code to append TDs
3231	* to a (possibly active) QH, and the same QH scanning code.
3232	*/
3233	static int __oxu_urb_enqueue(struct usb_hcd *hcd, struct urb *urb,
3234	gfp_t mem_flags)
3235	{
3236	struct oxu_hcd *oxu = hcd_to_oxu(hcd);
3237	struct list_head qtd_list;
3238
3239	INIT_LIST_HEAD(list: &qtd_list);
3240
3241	switch (usb_pipetype(urb->pipe)) {
3242	case PIPE_CONTROL:
3243	case PIPE_BULK:
3244	default:
3245	if (!qh_urb_transaction(oxu, urb, head: &qtd_list, flags: mem_flags))
3246	return -ENOMEM;
3247	return submit_async(oxu, urb, qtd_list: &qtd_list, mem_flags);
3248
3249	case PIPE_INTERRUPT:
3250	if (!qh_urb_transaction(oxu, urb, head: &qtd_list, flags: mem_flags))
3251	return -ENOMEM;
3252	return intr_submit(oxu, urb, qtd_list: &qtd_list, mem_flags);
3253
3254	case PIPE_ISOCHRONOUS:
3255	if (urb->dev->speed == USB_SPEED_HIGH)
3256	return itd_submit(oxu, urb, mem_flags);
3257	else
3258	return sitd_submit(oxu, urb, mem_flags);
3259	}
3260	}
3261
3262	/* This function is responsible for breaking URBs with big data size
3263	* into smaller size and processing small urbs in sequence.
3264	*/
3265	static int oxu_urb_enqueue(struct usb_hcd *hcd, struct urb *urb,
3266	gfp_t mem_flags)
3267	{
3268	struct oxu_hcd *oxu = hcd_to_oxu(hcd);
3269	int num, rem;
3270	void *transfer_buffer;
3271	struct urb *murb;
3272	int i, ret;
3273
3274	/* If not bulk pipe just enqueue the URB */
3275	if (!usb_pipebulk(urb->pipe))
3276	return __oxu_urb_enqueue(hcd, urb, mem_flags);
3277
3278	/* Otherwise we should verify the USB transfer buffer size! */
3279	transfer_buffer = urb->transfer_buffer;
3280
3281	num = urb->transfer_buffer_length / 4096;
3282	rem = urb->transfer_buffer_length % 4096;
3283	if (rem != 0)
3284	num++;
3285
3286	/* If URB is smaller than 4096 bytes just enqueue it! */
3287	if (num == 1)
3288	return __oxu_urb_enqueue(hcd, urb, mem_flags);
3289
3290	/* Ok, we have more job to do! :) */
3291
3292	for (i = 0; i < num - 1; i++) {
3293	/* Get free micro URB poll till a free urb is received */
3294
3295	do {
3296	murb = (struct urb *) oxu_murb_alloc(oxu);
3297	if (!murb)
3298	schedule();
3299	} while (!murb);
3300
3301	/* Coping the urb */
3302	memcpy(murb, urb, sizeof(struct urb));
3303
3304	murb->transfer_buffer_length = 4096;
3305	murb->transfer_buffer = transfer_buffer + i * 4096;
3306
3307	/* Null pointer for the encodes that this is a micro urb */
3308	murb->complete = NULL;
3309
3310	((struct oxu_murb *) murb)->main = urb;
3311	((struct oxu_murb *) murb)->last = 0;
3312
3313	/* This loop is to guarantee urb to be processed when there's
3314	* not enough resources at a particular time by retrying.
3315	*/
3316	do {
3317	ret = __oxu_urb_enqueue(hcd, urb: murb, mem_flags);
3318	if (ret)
3319	schedule();
3320	} while (ret);
3321	}
3322
3323	/* Last urb requires special handling */
3324
3325	/* Get free micro URB poll till a free urb is received */
3326	do {
3327	murb = (struct urb *) oxu_murb_alloc(oxu);
3328	if (!murb)
3329	schedule();
3330	} while (!murb);
3331
3332	/* Coping the urb */
3333	memcpy(murb, urb, sizeof(struct urb));
3334
3335	murb->transfer_buffer_length = rem > 0 ? rem : 4096;
3336	murb->transfer_buffer = transfer_buffer + (num - 1) * 4096;
3337
3338	/* Null pointer for the encodes that this is a micro urb */
3339	murb->complete = NULL;
3340
3341	((struct oxu_murb *) murb)->main = urb;
3342	((struct oxu_murb *) murb)->last = 1;
3343
3344	do {
3345	ret = __oxu_urb_enqueue(hcd, urb: murb, mem_flags);
3346	if (ret)
3347	schedule();
3348	} while (ret);
3349
3350	return ret;
3351	}
3352
3353	/* Remove from hardware lists.
3354	* Completions normally happen asynchronously
3355	*/
3356	static int oxu_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
3357	{
3358	struct oxu_hcd *oxu = hcd_to_oxu(hcd);
3359	struct ehci_qh *qh;
3360	unsigned long flags;
3361
3362	spin_lock_irqsave(&oxu->lock, flags);
3363	switch (usb_pipetype(urb->pipe)) {
3364	case PIPE_CONTROL:
3365	case PIPE_BULK:
3366	default:
3367	qh = (struct ehci_qh *) urb->hcpriv;
3368	if (!qh)
3369	break;
3370	unlink_async(oxu, qh);
3371	break;
3372
3373	case PIPE_INTERRUPT:
3374	qh = (struct ehci_qh *) urb->hcpriv;
3375	if (!qh)
3376	break;
3377	switch (qh->qh_state) {
3378	case QH_STATE_LINKED:
3379	intr_deschedule(oxu, qh);
3380	fallthrough;
3381	case QH_STATE_IDLE:
3382	qh_completions(oxu, qh);
3383	break;
3384	default:
3385	oxu_dbg(oxu, "bogus qh %p state %d\n",
3386	qh, qh->qh_state);
3387	goto done;
3388	}
3389
3390	/* reschedule QH iff another request is queued */
3391	if (!list_empty(head: &qh->qtd_list)
3392	&& HC_IS_RUNNING(hcd->state)) {
3393	int status;
3394
3395	status = qh_schedule(oxu, qh);
3396	spin_unlock_irqrestore(lock: &oxu->lock, flags);
3397
3398	if (status != 0) {
3399	/* shouldn't happen often, but ...
3400	* FIXME kill those tds' urbs
3401	*/
3402	dev_err(hcd->self.controller,
3403	"can't reschedule qh %p, err %d\n", qh,
3404	status);
3405	}
3406	return status;
3407	}
3408	break;
3409	}
3410	done:
3411	spin_unlock_irqrestore(lock: &oxu->lock, flags);
3412	return 0;
3413	}
3414
3415	/* Bulk qh holds the data toggle */
3416	static void oxu_endpoint_disable(struct usb_hcd *hcd,
3417	struct usb_host_endpoint *ep)
3418	{
3419	struct oxu_hcd *oxu = hcd_to_oxu(hcd);
3420	unsigned long flags;
3421	struct ehci_qh *qh, *tmp;
3422
3423	/* ASSERT: any requests/urbs are being unlinked */
3424	/* ASSERT: nobody can be submitting urbs for this any more */
3425
3426	rescan:
3427	spin_lock_irqsave(&oxu->lock, flags);
3428	qh = ep->hcpriv;
3429	if (!qh)
3430	goto done;
3431
3432	/* endpoints can be iso streams. for now, we don't
3433	* accelerate iso completions ... so spin a while.
3434	*/
3435	if (qh->hw_info1 == 0) {
3436	oxu_vdbg(oxu, "iso delay\n");
3437	goto idle_timeout;
3438	}
3439
3440	if (!HC_IS_RUNNING(hcd->state))
3441	qh->qh_state = QH_STATE_IDLE;
3442	switch (qh->qh_state) {
3443	case QH_STATE_LINKED:
3444	for (tmp = oxu->async->qh_next.qh;
3445	tmp && tmp != qh;
3446	tmp = tmp->qh_next.qh)
3447	continue;
3448	/* periodic qh self-unlinks on empty */
3449	if (!tmp)
3450	goto nogood;
3451	unlink_async(oxu, qh);
3452	fallthrough;
3453	case QH_STATE_UNLINK: /* wait for hw to finish? */
3454	idle_timeout:
3455	spin_unlock_irqrestore(lock: &oxu->lock, flags);
3456	schedule_timeout_uninterruptible(timeout: 1);
3457	goto rescan;
3458	case QH_STATE_IDLE: /* fully unlinked */
3459	if (list_empty(head: &qh->qtd_list)) {
3460	qh_put(qh);
3461	break;
3462	}
3463	fallthrough;
3464	default:
3465	nogood:
3466	/* caller was supposed to have unlinked any requests;
3467	* that's not our job. just leak this memory.
3468	*/
3469	oxu_err(oxu, "qh %p (#%02x) state %d%s\n",
3470	qh, ep->desc.bEndpointAddress, qh->qh_state,
3471	list_empty(&qh->qtd_list) ? "" : "(has tds)");
3472	break;
3473	}
3474	ep->hcpriv = NULL;
3475	done:
3476	spin_unlock_irqrestore(lock: &oxu->lock, flags);
3477	}
3478
3479	static int oxu_get_frame(struct usb_hcd *hcd)
3480	{
3481	struct oxu_hcd *oxu = hcd_to_oxu(hcd);
3482
3483	return (readl(addr: &oxu->regs->frame_index) >> 3) %
3484	oxu->periodic_size;
3485	}
3486
3487	/* Build "status change" packet (one or two bytes) from HC registers */
3488	static int oxu_hub_status_data(struct usb_hcd *hcd, char *buf)
3489	{
3490	struct oxu_hcd *oxu = hcd_to_oxu(hcd);
3491	u32 temp, mask, status = 0;
3492	int ports, i, retval = 1;
3493	unsigned long flags;
3494
3495	/* if !PM, root hub timers won't get shut down ... */
3496	if (!HC_IS_RUNNING(hcd->state))
3497	return 0;
3498
3499	/* init status to no-changes */
3500	buf[0] = 0;
3501	ports = HCS_N_PORTS(oxu->hcs_params);
3502	if (ports > 7) {
3503	buf[1] = 0;
3504	retval++;
3505	}
3506
3507	/* Some boards (mostly VIA?) report bogus overcurrent indications,
3508	* causing massive log spam unless we completely ignore them. It
3509	* may be relevant that VIA VT8235 controllers, where PORT_POWER is
3510	* always set, seem to clear PORT_OCC and PORT_CSC when writing to
3511	* PORT_POWER; that's surprising, but maybe within-spec.
3512	*/
3513	if (!ignore_oc)
3514	mask = PORT_CSC | PORT_PEC | PORT_OCC;
3515	else
3516	mask = PORT_CSC | PORT_PEC;
3517
3518	/* no hub change reports (bit 0) for now (power, ...) */
3519
3520	/* port N changes (bit N)? */
3521	spin_lock_irqsave(&oxu->lock, flags);
3522	for (i = 0; i < ports; i++) {
3523	temp = readl(addr: &oxu->regs->port_status[i]);
3524
3525	/*
3526	* Return status information even for ports with OWNER set.
3527	* Otherwise hub_wq wouldn't see the disconnect event when a
3528	* high-speed device is switched over to the companion
3529	* controller by the user.
3530	*/
3531
3532	if (!(temp & PORT_CONNECT))
3533	oxu->reset_done[i] = 0;
3534	if ((temp & mask) != 0 || ((temp & PORT_RESUME) != 0 &&
3535	time_after_eq(jiffies, oxu->reset_done[i]))) {
3536	if (i < 7)
3537	buf[0] |= 1 << (i + 1);
3538	else
3539	buf[1] |= 1 << (i - 7);
3540	status = STS_PCD;
3541	}
3542	}
3543	/* FIXME autosuspend idle root hubs */
3544	spin_unlock_irqrestore(lock: &oxu->lock, flags);
3545	return status ? retval : 0;
3546	}
3547
3548	/* Returns the speed of a device attached to a port on the root hub. */
3549	static inline unsigned int oxu_port_speed(struct oxu_hcd *oxu,
3550	unsigned int portsc)
3551	{
3552	switch ((portsc >> 26) & 3) {
3553	case 0:
3554	return 0;
3555	case 1:
3556	return USB_PORT_STAT_LOW_SPEED;
3557	case 2:
3558	default:
3559	return USB_PORT_STAT_HIGH_SPEED;
3560	}
3561	}
3562
3563	#define PORT_WAKE_BITS (PORT_WKOC_E|PORT_WKDISC_E|PORT_WKCONN_E)
3564	static int oxu_hub_control(struct usb_hcd *hcd, u16 typeReq,
3565	u16 wValue, u16 wIndex, char *buf, u16 wLength)
3566	{
3567	struct oxu_hcd *oxu = hcd_to_oxu(hcd);
3568	int ports = HCS_N_PORTS(oxu->hcs_params);
3569	u32 __iomem *status_reg = &oxu->regs->port_status[wIndex - 1];
3570	u32 temp, status;
3571	unsigned long flags;
3572	int retval = 0;
3573	unsigned selector;
3574
3575	/*
3576	* FIXME: support SetPortFeatures USB_PORT_FEAT_INDICATOR.
3577	* HCS_INDICATOR may say we can change LEDs to off/amber/green.
3578	* (track current state ourselves) ... blink for diagnostics,
3579	* power, "this is the one", etc. EHCI spec supports this.
3580	*/
3581
3582	spin_lock_irqsave(&oxu->lock, flags);
3583	switch (typeReq) {
3584	case ClearHubFeature:
3585	switch (wValue) {
3586	case C_HUB_LOCAL_POWER:
3587	case C_HUB_OVER_CURRENT:
3588	/* no hub-wide feature/status flags */
3589	break;
3590	default:
3591	goto error;
3592	}
3593	break;
3594	case ClearPortFeature:
3595	if (!wIndex || wIndex > ports)
3596	goto error;
3597	wIndex--;
3598	temp = readl(addr: status_reg);
3599
3600	/*
3601	* Even if OWNER is set, so the port is owned by the
3602	* companion controller, hub_wq needs to be able to clear
3603	* the port-change status bits (especially
3604	* USB_PORT_STAT_C_CONNECTION).
3605	*/
3606
3607	switch (wValue) {
3608	case USB_PORT_FEAT_ENABLE:
3609	writel(val: temp & ~PORT_PE, addr: status_reg);
3610	break;
3611	case USB_PORT_FEAT_C_ENABLE:
3612	writel(val: (temp & ~PORT_RWC_BITS) | PORT_PEC, addr: status_reg);
3613	break;
3614	case USB_PORT_FEAT_SUSPEND:
3615	if (temp & PORT_RESET)
3616	goto error;
3617	if (temp & PORT_SUSPEND) {
3618	if ((temp & PORT_PE) == 0)
3619	goto error;
3620	/* resume signaling for 20 msec */
3621	temp &= ~(PORT_RWC_BITS | PORT_WAKE_BITS);
3622	writel(val: temp | PORT_RESUME, addr: status_reg);
3623	oxu->reset_done[wIndex] = jiffies
3624	+ msecs_to_jiffies(m: 20);
3625	}
3626	break;
3627	case USB_PORT_FEAT_C_SUSPEND:
3628	/* we auto-clear this feature */
3629	break;
3630	case USB_PORT_FEAT_POWER:
3631	if (HCS_PPC(oxu->hcs_params))
3632	writel(val: temp & ~(PORT_RWC_BITS | PORT_POWER),
3633	addr: status_reg);
3634	break;
3635	case USB_PORT_FEAT_C_CONNECTION:
3636	writel(val: (temp & ~PORT_RWC_BITS) | PORT_CSC, addr: status_reg);
3637	break;
3638	case USB_PORT_FEAT_C_OVER_CURRENT:
3639	writel(val: (temp & ~PORT_RWC_BITS) | PORT_OCC, addr: status_reg);
3640	break;
3641	case USB_PORT_FEAT_C_RESET:
3642	/* GetPortStatus clears reset */
3643	break;
3644	default:
3645	goto error;
3646	}
3647	readl(addr: &oxu->regs->command); /* unblock posted write */
3648	break;
3649	case GetHubDescriptor:
3650	ehci_hub_descriptor(oxu, desc: (struct usb_hub_descriptor *)
3651	buf);
3652	break;
3653	case GetHubStatus:
3654	/* no hub-wide feature/status flags */
3655	memset(buf, 0, 4);
3656	break;
3657	case GetPortStatus:
3658	if (!wIndex || wIndex > ports)
3659	goto error;
3660	wIndex--;
3661	status = 0;
3662	temp = readl(addr: status_reg);
3663
3664	/* wPortChange bits */
3665	if (temp & PORT_CSC)
3666	status |= USB_PORT_STAT_C_CONNECTION << 16;
3667	if (temp & PORT_PEC)
3668	status |= USB_PORT_STAT_C_ENABLE << 16;
3669	if ((temp & PORT_OCC) && !ignore_oc)
3670	status |= USB_PORT_STAT_C_OVERCURRENT << 16;
3671
3672	/* whoever resumes must GetPortStatus to complete it!! */
3673	if (temp & PORT_RESUME) {
3674
3675	/* Remote Wakeup received? */
3676	if (!oxu->reset_done[wIndex]) {
3677	/* resume signaling for 20 msec */
3678	oxu->reset_done[wIndex] = jiffies
3679	+ msecs_to_jiffies(m: 20);
3680	/* check the port again */
3681	mod_timer(timer: &oxu_to_hcd(oxu)->rh_timer,
3682	expires: oxu->reset_done[wIndex]);
3683	}
3684
3685	/* resume completed? */
3686	else if (time_after_eq(jiffies,
3687	oxu->reset_done[wIndex])) {
3688	status |= USB_PORT_STAT_C_SUSPEND << 16;
3689	oxu->reset_done[wIndex] = 0;
3690
3691	/* stop resume signaling */
3692	temp = readl(addr: status_reg);
3693	writel(val: temp & ~(PORT_RWC_BITS | PORT_RESUME),
3694	addr: status_reg);
3695	retval = handshake(oxu, ptr: status_reg,
3696	PORT_RESUME, done: 0, usec: 2000 /* 2msec */);
3697	if (retval != 0) {
3698	oxu_err(oxu,
3699	"port %d resume error %d\n",
3700	wIndex + 1, retval);
3701	goto error;
3702	}
3703	temp &= ~(PORT_SUSPEND|PORT_RESUME|(3<<10));
3704	}
3705	}
3706
3707	/* whoever resets must GetPortStatus to complete it!! */
3708	if ((temp & PORT_RESET)
3709	&& time_after_eq(jiffies,
3710	oxu->reset_done[wIndex])) {
3711	status |= USB_PORT_STAT_C_RESET << 16;
3712	oxu->reset_done[wIndex] = 0;
3713
3714	/* force reset to complete */
3715	writel(val: temp & ~(PORT_RWC_BITS | PORT_RESET),
3716	addr: status_reg);
3717	/* REVISIT: some hardware needs 550+ usec to clear
3718	* this bit; seems too long to spin routinely...
3719	*/
3720	retval = handshake(oxu, ptr: status_reg,
3721	PORT_RESET, done: 0, usec: 750);
3722	if (retval != 0) {
3723	oxu_err(oxu, "port %d reset error %d\n",
3724	wIndex + 1, retval);
3725	goto error;
3726	}
3727
3728	/* see what we found out */
3729	temp = check_reset_complete(oxu, index: wIndex, status_reg,
3730	readl(addr: status_reg));
3731	}
3732
3733	/* transfer dedicated ports to the companion hc */
3734	if ((temp & PORT_CONNECT) &&
3735	test_bit(wIndex, &oxu->companion_ports)) {
3736	temp &= ~PORT_RWC_BITS;
3737	temp |= PORT_OWNER;
3738	writel(val: temp, addr: status_reg);
3739	oxu_dbg(oxu, "port %d --> companion\n", wIndex + 1);
3740	temp = readl(addr: status_reg);
3741	}
3742
3743	/*
3744	* Even if OWNER is set, there's no harm letting hub_wq
3745	* see the wPortStatus values (they should all be 0 except
3746	* for PORT_POWER anyway).
3747	*/
3748
3749	if (temp & PORT_CONNECT) {
3750	status |= USB_PORT_STAT_CONNECTION;
3751	/* status may be from integrated TT */
3752	status |= oxu_port_speed(oxu, portsc: temp);
3753	}
3754	if (temp & PORT_PE)
3755	status |= USB_PORT_STAT_ENABLE;
3756	if (temp & (PORT_SUSPEND|PORT_RESUME))
3757	status |= USB_PORT_STAT_SUSPEND;
3758	if (temp & PORT_OC)
3759	status |= USB_PORT_STAT_OVERCURRENT;
3760	if (temp & PORT_RESET)
3761	status |= USB_PORT_STAT_RESET;
3762	if (temp & PORT_POWER)
3763	status |= USB_PORT_STAT_POWER;
3764
3765	#ifndef OXU_VERBOSE_DEBUG
3766	if (status & ~0xffff) /* only if wPortChange is interesting */
3767	#endif
3768	dbg_port(oxu, "GetStatus", wIndex + 1, temp);
3769	put_unaligned(cpu_to_le32(status), (__le32 *) buf);
3770	break;
3771	case SetHubFeature:
3772	switch (wValue) {
3773	case C_HUB_LOCAL_POWER:
3774	case C_HUB_OVER_CURRENT:
3775	/* no hub-wide feature/status flags */
3776	break;
3777	default:
3778	goto error;
3779	}
3780	break;
3781	case SetPortFeature:
3782	selector = wIndex >> 8;
3783	wIndex &= 0xff;
3784	if (!wIndex || wIndex > ports)
3785	goto error;
3786	wIndex--;
3787	temp = readl(addr: status_reg);
3788	if (temp & PORT_OWNER)
3789	break;
3790
3791	temp &= ~PORT_RWC_BITS;
3792	switch (wValue) {
3793	case USB_PORT_FEAT_SUSPEND:
3794	if ((temp & PORT_PE) == 0
3795	|| (temp & PORT_RESET) != 0)
3796	goto error;
3797	if (device_may_wakeup(dev: &hcd->self.root_hub->dev))
3798	temp |= PORT_WAKE_BITS;
3799	writel(val: temp | PORT_SUSPEND, addr: status_reg);
3800	break;
3801	case USB_PORT_FEAT_POWER:
3802	if (HCS_PPC(oxu->hcs_params))
3803	writel(val: temp | PORT_POWER, addr: status_reg);
3804	break;
3805	case USB_PORT_FEAT_RESET:
3806	if (temp & PORT_RESUME)
3807	goto error;
3808	/* line status bits may report this as low speed,
3809	* which can be fine if this root hub has a
3810	* transaction translator built in.
3811	*/
3812	oxu_vdbg(oxu, "port %d reset\n", wIndex + 1);
3813	temp |= PORT_RESET;
3814	temp &= ~PORT_PE;
3815
3816	/*
3817	* caller must wait, then call GetPortStatus
3818	* usb 2.0 spec says 50 ms resets on root
3819	*/
3820	oxu->reset_done[wIndex] = jiffies
3821	+ msecs_to_jiffies(m: 50);
3822	writel(val: temp, addr: status_reg);
3823	break;
3824
3825	/* For downstream facing ports (these): one hub port is put
3826	* into test mode according to USB2 11.24.2.13, then the hub
3827	* must be reset (which for root hub now means rmmod+modprobe,
3828	* or else system reboot). See EHCI 2.3.9 and 4.14 for info
3829	* about the EHCI-specific stuff.
3830	*/
3831	case USB_PORT_FEAT_TEST:
3832	if (!selector || selector > 5)
3833	goto error;
3834	ehci_quiesce(oxu);
3835	ehci_halt(oxu);
3836	temp |= selector << 16;
3837	writel(val: temp, addr: status_reg);
3838	break;
3839
3840	default:
3841	goto error;
3842	}
3843	readl(addr: &oxu->regs->command); /* unblock posted writes */
3844	break;
3845
3846	default:
3847	error:
3848	/* "stall" on error */
3849	retval = -EPIPE;
3850	}
3851	spin_unlock_irqrestore(lock: &oxu->lock, flags);
3852	return retval;
3853	}
3854
3855	#ifdef CONFIG_PM
3856
3857	static int oxu_bus_suspend(struct usb_hcd *hcd)
3858	{
3859	struct oxu_hcd *oxu = hcd_to_oxu(hcd);
3860	int port;
3861	int mask;
3862
3863	oxu_dbg(oxu, "suspend root hub\n");
3864
3865	if (time_before(jiffies, oxu->next_statechange))
3866	msleep(msecs: 5);
3867
3868	port = HCS_N_PORTS(oxu->hcs_params);
3869	spin_lock_irq(lock: &oxu->lock);
3870
3871	/* stop schedules, clean any completed work */
3872	if (HC_IS_RUNNING(hcd->state)) {
3873	ehci_quiesce(oxu);
3874	hcd->state = HC_STATE_QUIESCING;
3875	}
3876	oxu->command = readl(addr: &oxu->regs->command);
3877	if (oxu->reclaim)
3878	oxu->reclaim_ready = 1;
3879	ehci_work(oxu);
3880
3881	/* Unlike other USB host controller types, EHCI doesn't have
3882	* any notion of "global" or bus-wide suspend. The driver has
3883	* to manually suspend all the active unsuspended ports, and
3884	* then manually resume them in the bus_resume() routine.
3885	*/
3886	oxu->bus_suspended = 0;
3887	while (port--) {
3888	u32 __iomem *reg = &oxu->regs->port_status[port];
3889	u32 t1 = readl(addr: reg) & ~PORT_RWC_BITS;
3890	u32 t2 = t1;
3891
3892	/* keep track of which ports we suspend */
3893	if ((t1 & PORT_PE) && !(t1 & PORT_OWNER) &&
3894	!(t1 & PORT_SUSPEND)) {
3895	t2 |= PORT_SUSPEND;
3896	set_bit(nr: port, addr: &oxu->bus_suspended);
3897	}
3898
3899	/* enable remote wakeup on all ports */
3900	if (device_may_wakeup(dev: &hcd->self.root_hub->dev))
3901	t2 |= PORT_WKOC_E|PORT_WKDISC_E|PORT_WKCONN_E;
3902	else
3903	t2 &= ~(PORT_WKOC_E|PORT_WKDISC_E|PORT_WKCONN_E);
3904
3905	if (t1 != t2) {
3906	oxu_vdbg(oxu, "port %d, %08x -> %08x\n",
3907	port + 1, t1, t2);
3908	writel(val: t2, addr: reg);
3909	}
3910	}
3911
3912	spin_unlock_irq(lock: &oxu->lock);
3913	/* turn off now-idle HC */
3914	del_timer_sync(timer: &oxu->watchdog);
3915	spin_lock_irq(lock: &oxu->lock);
3916	ehci_halt(oxu);
3917	hcd->state = HC_STATE_SUSPENDED;
3918
3919	/* allow remote wakeup */
3920	mask = INTR_MASK;
3921	if (!device_may_wakeup(dev: &hcd->self.root_hub->dev))
3922	mask &= ~STS_PCD;
3923	writel(val: mask, addr: &oxu->regs->intr_enable);
3924	readl(addr: &oxu->regs->intr_enable);
3925
3926	oxu->next_statechange = jiffies + msecs_to_jiffies(m: 10);
3927	spin_unlock_irq(lock: &oxu->lock);
3928	return 0;
3929	}
3930
3931	/* Caller has locked the root hub, and should reset/reinit on error */
3932	static int oxu_bus_resume(struct usb_hcd *hcd)
3933	{
3934	struct oxu_hcd *oxu = hcd_to_oxu(hcd);
3935	u32 temp;
3936	int i;
3937
3938	if (time_before(jiffies, oxu->next_statechange))
3939	msleep(msecs: 5);
3940	spin_lock_irq(lock: &oxu->lock);
3941
3942	/* Ideally and we've got a real resume here, and no port's power
3943	* was lost. (For PCI, that means Vaux was maintained.) But we
3944	* could instead be restoring a swsusp snapshot -- so that BIOS was
3945	* the last user of the controller, not reset/pm hardware keeping
3946	* state we gave to it.
3947	*/
3948	temp = readl(addr: &oxu->regs->intr_enable);
3949	oxu_dbg(oxu, "resume root hub%s\n", temp ? "" : " after power loss");
3950
3951	/* at least some APM implementations will try to deliver
3952	* IRQs right away, so delay them until we're ready.
3953	*/
3954	writel(val: 0, addr: &oxu->regs->intr_enable);
3955
3956	/* re-init operational registers */
3957	writel(val: 0, addr: &oxu->regs->segment);
3958	writel(val: oxu->periodic_dma, addr: &oxu->regs->frame_list);
3959	writel(val: (u32) oxu->async->qh_dma, addr: &oxu->regs->async_next);
3960
3961	/* restore CMD_RUN, framelist size, and irq threshold */
3962	writel(val: oxu->command, addr: &oxu->regs->command);
3963
3964	/* Some controller/firmware combinations need a delay during which
3965	* they set up the port statuses. See Bugzilla #8190. */
3966	mdelay(8);
3967
3968	/* manually resume the ports we suspended during bus_suspend() */
3969	i = HCS_N_PORTS(oxu->hcs_params);
3970	while (i--) {
3971	temp = readl(addr: &oxu->regs->port_status[i]);
3972	temp &= ~(PORT_RWC_BITS
3973	| PORT_WKOC_E | PORT_WKDISC_E | PORT_WKCONN_E);
3974	if (test_bit(i, &oxu->bus_suspended) && (temp & PORT_SUSPEND)) {
3975	oxu->reset_done[i] = jiffies + msecs_to_jiffies(m: 20);
3976	temp |= PORT_RESUME;
3977	}
3978	writel(val: temp, addr: &oxu->regs->port_status[i]);
3979	}
3980	i = HCS_N_PORTS(oxu->hcs_params);
3981	mdelay(20);
3982	while (i--) {
3983	temp = readl(addr: &oxu->regs->port_status[i]);
3984	if (test_bit(i, &oxu->bus_suspended) && (temp & PORT_SUSPEND)) {
3985	temp &= ~(PORT_RWC_BITS | PORT_RESUME);
3986	writel(val: temp, addr: &oxu->regs->port_status[i]);
3987	oxu_vdbg(oxu, "resumed port %d\n", i + 1);
3988	}
3989	}
3990	(void) readl(addr: &oxu->regs->command);
3991
3992	/* maybe re-activate the schedule(s) */
3993	temp = 0;
3994	if (oxu->async->qh_next.qh)
3995	temp |= CMD_ASE;
3996	if (oxu->periodic_sched)
3997	temp |= CMD_PSE;
3998	if (temp) {
3999	oxu->command |= temp;
4000	writel(val: oxu->command, addr: &oxu->regs->command);
4001	}
4002
4003	oxu->next_statechange = jiffies + msecs_to_jiffies(m: 5);
4004	hcd->state = HC_STATE_RUNNING;
4005
4006	/* Now we can safely re-enable irqs */
4007	writel(INTR_MASK, addr: &oxu->regs->intr_enable);
4008
4009	spin_unlock_irq(lock: &oxu->lock);
4010	return 0;
4011	}
4012
4013	#else
4014
4015	static int oxu_bus_suspend(struct usb_hcd *hcd)
4016	{
4017	return 0;
4018	}
4019
4020	static int oxu_bus_resume(struct usb_hcd *hcd)
4021	{
4022	return 0;
4023	}
4024
4025	#endif /* CONFIG_PM */
4026
4027	static const struct hc_driver oxu_hc_driver = {
4028	.description = "oxu210hp_hcd",
4029	.product_desc = "oxu210hp HCD",
4030	.hcd_priv_size = sizeof(struct oxu_hcd),
4031
4032	/*
4033	* Generic hardware linkage
4034	*/
4035	.irq = oxu_irq,
4036	.flags = HCD_MEMORY | HCD_USB2,
4037
4038	/*
4039	* Basic lifecycle operations
4040	*/
4041	.reset = oxu_reset,
4042	.start = oxu_run,
4043	.stop = oxu_stop,
4044	.shutdown = oxu_shutdown,
4045
4046	/*
4047	* Managing i/o requests and associated device resources
4048	*/
4049	.urb_enqueue = oxu_urb_enqueue,
4050	.urb_dequeue = oxu_urb_dequeue,
4051	.endpoint_disable = oxu_endpoint_disable,
4052
4053	/*
4054	* Scheduling support
4055	*/
4056	.get_frame_number = oxu_get_frame,
4057
4058	/*
4059	* Root hub support
4060	*/
4061	.hub_status_data = oxu_hub_status_data,
4062	.hub_control = oxu_hub_control,
4063	.bus_suspend = oxu_bus_suspend,
4064	.bus_resume = oxu_bus_resume,
4065	};
4066
4067	/*
4068	* Module stuff
4069	*/
4070
4071	static void oxu_configuration(struct platform_device *pdev, void __iomem *base)
4072	{
4073	u32 tmp;
4074
4075	/* Initialize top level registers.
4076	* First write ever
4077	*/
4078	oxu_writel(base, OXU_HOSTIFCONFIG, val: 0x0000037D);
4079	oxu_writel(base, OXU_SOFTRESET, OXU_SRESET);
4080	oxu_writel(base, OXU_HOSTIFCONFIG, val: 0x0000037D);
4081
4082	tmp = oxu_readl(base, OXU_PIOBURSTREADCTRL);
4083	oxu_writel(base, OXU_PIOBURSTREADCTRL, val: tmp | 0x0040);
4084
4085	oxu_writel(base, OXU_ASO, OXU_SPHPOEN | OXU_OVRCCURPUPDEN |
4086	OXU_COMPARATOR | OXU_ASO_OP);
4087
4088	tmp = oxu_readl(base, OXU_CLKCTRL_SET);
4089	oxu_writel(base, OXU_CLKCTRL_SET, val: tmp | OXU_SYSCLKEN | OXU_USBOTGCLKEN);
4090
4091	/* Clear all top interrupt enable */
4092	oxu_writel(base, OXU_CHIPIRQEN_CLR, val: 0xff);
4093
4094	/* Clear all top interrupt status */
4095	oxu_writel(base, OXU_CHIPIRQSTATUS, val: 0xff);
4096
4097	/* Enable all needed top interrupt except OTG SPH core */
4098	oxu_writel(base, OXU_CHIPIRQEN_SET, OXU_USBSPHLPWUI | OXU_USBOTGLPWUI);
4099	}
4100
4101	static int oxu_verify_id(struct platform_device *pdev, void __iomem *base)
4102	{
4103	u32 id;
4104	static const char * const bo[] = {
4105	"reserved",
4106	"128-pin LQFP",
4107	"84-pin TFBGA",
4108	"reserved",
4109	};
4110
4111	/* Read controller signature register to find a match */
4112	id = oxu_readl(base, OXU_DEVICEID);
4113	dev_info(&pdev->dev, "device ID %x\n", id);
4114	if ((id & OXU_REV_MASK) != (OXU_REV_2100 << OXU_REV_SHIFT))
4115	return -1;
4116
4117	dev_info(&pdev->dev, "found device %x %s (%04x:%04x)\n",
4118	id >> OXU_REV_SHIFT,
4119	bo[(id & OXU_BO_MASK) >> OXU_BO_SHIFT],
4120	(id & OXU_MAJ_REV_MASK) >> OXU_MAJ_REV_SHIFT,
4121	(id & OXU_MIN_REV_MASK) >> OXU_MIN_REV_SHIFT);
4122
4123	return 0;
4124	}
4125
4126	static const struct hc_driver oxu_hc_driver;
4127	static struct usb_hcd *oxu_create(struct platform_device *pdev,
4128	unsigned long memstart, unsigned long memlen,
4129	void __iomem *base, int irq, int otg)
4130	{
4131	struct device *dev = &pdev->dev;
4132
4133	struct usb_hcd *hcd;
4134	struct oxu_hcd *oxu;
4135	int ret;
4136
4137	/* Set endian mode and host mode */
4138	oxu_writel(base: base + (otg ? OXU_OTG_CORE_OFFSET : OXU_SPH_CORE_OFFSET),
4139	OXU_USBMODE,
4140	OXU_CM_HOST_ONLY | OXU_ES_LITTLE | OXU_VBPS);
4141
4142	hcd = usb_create_hcd(driver: &oxu_hc_driver, dev,
4143	bus_name: otg ? "oxu210hp_otg" : "oxu210hp_sph");
4144	if (!hcd)
4145	return ERR_PTR(error: -ENOMEM);
4146
4147	hcd->rsrc_start = memstart;
4148	hcd->rsrc_len = memlen;
4149	hcd->regs = base;
4150	hcd->irq = irq;
4151	hcd->state = HC_STATE_HALT;
4152
4153	oxu = hcd_to_oxu(hcd);
4154	oxu->is_otg = otg;
4155
4156	ret = usb_add_hcd(hcd, irqnum: irq, IRQF_SHARED);
4157	if (ret < 0) {
4158	usb_put_hcd(hcd);
4159	return ERR_PTR(error: ret);
4160	}
4161
4162	device_wakeup_enable(dev: hcd->self.controller);
4163	return hcd;
4164	}
4165
4166	static int oxu_init(struct platform_device *pdev,
4167	unsigned long memstart, unsigned long memlen,
4168	void __iomem *base, int irq)
4169	{
4170	struct oxu_info *info = platform_get_drvdata(pdev);
4171	struct usb_hcd *hcd;
4172	int ret;
4173
4174	/* First time configuration at start up */
4175	oxu_configuration(pdev, base);
4176
4177	ret = oxu_verify_id(pdev, base);
4178	if (ret) {
4179	dev_err(&pdev->dev, "no devices found!\n");
4180	return -ENODEV;
4181	}
4182
4183	/* Create the OTG controller */
4184	hcd = oxu_create(pdev, memstart, memlen, base, irq, otg: 1);
4185	if (IS_ERR(ptr: hcd)) {
4186	dev_err(&pdev->dev, "cannot create OTG controller!\n");
4187	ret = PTR_ERR(ptr: hcd);
4188	goto error_create_otg;
4189	}
4190	info->hcd[0] = hcd;
4191
4192	/* Create the SPH host controller */
4193	hcd = oxu_create(pdev, memstart, memlen, base, irq, otg: 0);
4194	if (IS_ERR(ptr: hcd)) {
4195	dev_err(&pdev->dev, "cannot create SPH controller!\n");
4196	ret = PTR_ERR(ptr: hcd);
4197	goto error_create_sph;
4198	}
4199	info->hcd[1] = hcd;
4200
4201	oxu_writel(base, OXU_CHIPIRQEN_SET,
4202	val: oxu_readl(base, OXU_CHIPIRQEN_SET) | 3);
4203
4204	return 0;
4205
4206	error_create_sph:
4207	usb_remove_hcd(hcd: info->hcd[0]);
4208	usb_put_hcd(hcd: info->hcd[0]);
4209
4210	error_create_otg:
4211	return ret;
4212	}
4213
4214	static int oxu_drv_probe(struct platform_device *pdev)
4215	{
4216	struct resource *res;
4217	void __iomem *base;
4218	unsigned long memstart, memlen;
4219	int irq, ret;
4220	struct oxu_info *info;
4221
4222	if (usb_disabled())
4223	return -ENODEV;
4224
4225	/*
4226	* Get the platform resources
4227	*/
4228	irq = platform_get_irq(pdev, 0);
4229	if (irq < 0)
4230	return irq;
4231	dev_dbg(&pdev->dev, "IRQ resource %d\n", irq);
4232
4233	base = devm_platform_get_and_ioremap_resource(pdev, index: 0, res: &res);
4234	if (IS_ERR(ptr: base)) {
4235	ret = PTR_ERR(ptr: base);
4236	goto error;
4237	}
4238	memstart = res->start;
4239	memlen = resource_size(res);
4240
4241	ret = irq_set_irq_type(irq, IRQF_TRIGGER_FALLING);
4242	if (ret) {
4243	dev_err(&pdev->dev, "error setting irq type\n");
4244	ret = -EFAULT;
4245	goto error;
4246	}
4247
4248	/* Allocate a driver data struct to hold useful info for both
4249	* SPH & OTG devices
4250	*/
4251	info = devm_kzalloc(dev: &pdev->dev, size: sizeof(struct oxu_info), GFP_KERNEL);
4252	if (!info) {
4253	ret = -EFAULT;
4254	goto error;
4255	}
4256	platform_set_drvdata(pdev, data: info);
4257
4258	ret = oxu_init(pdev, memstart, memlen, base, irq);
4259	if (ret < 0) {
4260	dev_dbg(&pdev->dev, "cannot init USB devices\n");
4261	goto error;
4262	}
4263
4264	dev_info(&pdev->dev, "devices enabled and running\n");
4265	platform_set_drvdata(pdev, data: info);
4266
4267	return 0;
4268
4269	error:
4270	dev_err(&pdev->dev, "init %s fail, %d\n", dev_name(&pdev->dev), ret);
4271	return ret;
4272	}
4273
4274	static void oxu_remove(struct platform_device *pdev, struct usb_hcd *hcd)
4275	{
4276	usb_remove_hcd(hcd);
4277	usb_put_hcd(hcd);
4278	}
4279
4280	static void oxu_drv_remove(struct platform_device *pdev)
4281	{
4282	struct oxu_info *info = platform_get_drvdata(pdev);
4283
4284	oxu_remove(pdev, hcd: info->hcd[0]);
4285	oxu_remove(pdev, hcd: info->hcd[1]);
4286	}
4287
4288	static void oxu_drv_shutdown(struct platform_device *pdev)
4289	{
4290	oxu_drv_remove(pdev);
4291	}
4292
4293	#if 0
4294	/* FIXME: TODO */
4295	static int oxu_drv_suspend(struct device *dev)
4296	{
4297	struct platform_device *pdev = to_platform_device(dev);
4298	struct usb_hcd *hcd = dev_get_drvdata(dev);
4299
4300	return 0;
4301	}
4302
4303	static int oxu_drv_resume(struct device *dev)
4304	{
4305	struct platform_device *pdev = to_platform_device(dev);
4306	struct usb_hcd *hcd = dev_get_drvdata(dev);
4307
4308	return 0;
4309	}
4310	#else
4311	#define oxu_drv_suspend NULL
4312	#define oxu_drv_resume NULL
4313	#endif
4314
4315	static struct platform_driver oxu_driver = {
4316	.probe = oxu_drv_probe,
4317	.remove_new = oxu_drv_remove,
4318	.shutdown = oxu_drv_shutdown,
4319	.suspend = oxu_drv_suspend,
4320	.resume = oxu_drv_resume,
4321	.driver = {
4322	.name = "oxu210hp-hcd",
4323	.bus = &platform_bus_type
4324	}
4325	};
4326
4327	module_platform_driver(oxu_driver);
4328
4329	MODULE_DESCRIPTION("Oxford OXU210HP HCD driver - ver. " DRIVER_VERSION);
4330	MODULE_AUTHOR("Rodolfo Giometti <giometti@linux.it>");
4331	MODULE_LICENSE("GPL");
4332