1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* xHCI host controller driver
4	*
5	* Copyright (C) 2008 Intel Corp.
6	*
7	* Author: Sarah Sharp
8	* Some code borrowed from the Linux EHCI driver.
9	*/
10
11	#include <linux/usb.h>
12	#include <linux/overflow.h>
13	#include <linux/pci.h>
14	#include <linux/slab.h>
15	#include <linux/dmapool.h>
16	#include <linux/dma-mapping.h>
17
18	#include "xhci.h"
19	#include "xhci-trace.h"
20	#include "xhci-debugfs.h"
21
22	/*
23	* Allocates a generic ring segment from the ring pool, sets the dma address,
24	* initializes the segment to zero, and sets the private next pointer to NULL.
25	*
26	* Section 4.11.1.1:
27	* "All components of all Command and Transfer TRBs shall be initialized to '0'"
28	*/
29	static struct xhci_segment *xhci_segment_alloc(struct xhci_hcd *xhci,
30	unsigned int cycle_state,
31	unsigned int max_packet,
32	unsigned int num,
33	gfp_t flags)
34	{
35	struct xhci_segment *seg;
36	dma_addr_t dma;
37	int i;
38	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
39
40	seg = kzalloc_node(size: sizeof(*seg), flags, node: dev_to_node(dev));
41	if (!seg)
42	return NULL;
43
44	seg->trbs = dma_pool_zalloc(pool: xhci->segment_pool, mem_flags: flags, handle: &dma);
45	if (!seg->trbs) {
46	kfree(objp: seg);
47	return NULL;
48	}
49
50	if (max_packet) {
51	seg->bounce_buf = kzalloc_node(size: max_packet, flags,
52	node: dev_to_node(dev));
53	if (!seg->bounce_buf) {
54	dma_pool_free(pool: xhci->segment_pool, vaddr: seg->trbs, addr: dma);
55	kfree(objp: seg);
56	return NULL;
57	}
58	}
59	/* If the cycle state is 0, set the cycle bit to 1 for all the TRBs */
60	if (cycle_state == 0) {
61	for (i = 0; i < TRBS_PER_SEGMENT; i++)
62	seg->trbs[i].link.control = cpu_to_le32(TRB_CYCLE);
63	}
64	seg->num = num;
65	seg->dma = dma;
66	seg->next = NULL;
67
68	return seg;
69	}
70
71	static void xhci_segment_free(struct xhci_hcd *xhci, struct xhci_segment *seg)
72	{
73	if (seg->trbs) {
74	dma_pool_free(pool: xhci->segment_pool, vaddr: seg->trbs, addr: seg->dma);
75	seg->trbs = NULL;
76	}
77	kfree(objp: seg->bounce_buf);
78	kfree(objp: seg);
79	}
80
81	static void xhci_free_segments_for_ring(struct xhci_hcd *xhci,
82	struct xhci_segment *first)
83	{
84	struct xhci_segment *seg;
85
86	seg = first->next;
87	while (seg != first) {
88	struct xhci_segment *next = seg->next;
89	xhci_segment_free(xhci, seg);
90	seg = next;
91	}
92	xhci_segment_free(xhci, seg: first);
93	}
94
95	/*
96	* Make the prev segment point to the next segment.
97	*
98	* Change the last TRB in the prev segment to be a Link TRB which points to the
99	* DMA address of the next segment. The caller needs to set any Link TRB
100	* related flags, such as End TRB, Toggle Cycle, and no snoop.
101	*/
102	static void xhci_link_segments(struct xhci_segment *prev,
103	struct xhci_segment *next,
104	enum xhci_ring_type type, bool chain_links)
105	{
106	u32 val;
107
108	if (!prev || !next)
109	return;
110	prev->next = next;
111	if (type != TYPE_EVENT) {
112	prev->trbs[TRBS_PER_SEGMENT-1].link.segment_ptr =
113	cpu_to_le64(next->dma);
114
115	/* Set the last TRB in the segment to have a TRB type ID of Link TRB */
116	val = le32_to_cpu(prev->trbs[TRBS_PER_SEGMENT-1].link.control);
117	val &= ~TRB_TYPE_BITMASK;
118	val |= TRB_TYPE(TRB_LINK);
119	if (chain_links)
120	val |= TRB_CHAIN;
121	prev->trbs[TRBS_PER_SEGMENT-1].link.control = cpu_to_le32(val);
122	}
123	}
124
125	/*
126	* Link the ring to the new segments.
127	* Set Toggle Cycle for the new ring if needed.
128	*/
129	static void xhci_link_rings(struct xhci_hcd *xhci, struct xhci_ring *ring,
130	struct xhci_segment *first, struct xhci_segment *last,
131	unsigned int num_segs)
132	{
133	struct xhci_segment *next, *seg;
134	bool chain_links;
135
136	if (!ring || !first || !last)
137	return;
138
139	/* Set chain bit for 0.95 hosts, and for isoc rings on AMD 0.96 host */
140	chain_links = !!(xhci_link_trb_quirk(xhci) ||
141	(ring->type == TYPE_ISOC &&
142	(xhci->quirks & XHCI_AMD_0x96_HOST)));
143
144	next = ring->enq_seg->next;
145	xhci_link_segments(prev: ring->enq_seg, next: first, type: ring->type, chain_links);
146	xhci_link_segments(prev: last, next, type: ring->type, chain_links);
147	ring->num_segs += num_segs;
148
149	if (ring->enq_seg == ring->last_seg) {
150	if (ring->type != TYPE_EVENT) {
151	ring->last_seg->trbs[TRBS_PER_SEGMENT-1].link.control
152	&= ~cpu_to_le32(LINK_TOGGLE);
153	last->trbs[TRBS_PER_SEGMENT-1].link.control
154	|= cpu_to_le32(LINK_TOGGLE);
155	}
156	ring->last_seg = last;
157	}
158
159	for (seg = last; seg != ring->last_seg; seg = seg->next)
160	seg->next->num = seg->num + 1;
161	}
162
163	/*
164	* We need a radix tree for mapping physical addresses of TRBs to which stream
165	* ID they belong to. We need to do this because the host controller won't tell
166	* us which stream ring the TRB came from. We could store the stream ID in an
167	* event data TRB, but that doesn't help us for the cancellation case, since the
168	* endpoint may stop before it reaches that event data TRB.
169	*
170	* The radix tree maps the upper portion of the TRB DMA address to a ring
171	* segment that has the same upper portion of DMA addresses. For example, say I
172	* have segments of size 1KB, that are always 1KB aligned. A segment may
173	* start at 0x10c91000 and end at 0x10c913f0. If I use the upper 10 bits, the
174	* key to the stream ID is 0x43244. I can use the DMA address of the TRB to
175	* pass the radix tree a key to get the right stream ID:
176	*
177	* 0x10c90fff >> 10 = 0x43243
178	* 0x10c912c0 >> 10 = 0x43244
179	* 0x10c91400 >> 10 = 0x43245
180	*
181	* Obviously, only those TRBs with DMA addresses that are within the segment
182	* will make the radix tree return the stream ID for that ring.
183	*
184	* Caveats for the radix tree:
185	*
186	* The radix tree uses an unsigned long as a key pair. On 32-bit systems, an
187	* unsigned long will be 32-bits; on a 64-bit system an unsigned long will be
188	* 64-bits. Since we only request 32-bit DMA addresses, we can use that as the
189	* key on 32-bit or 64-bit systems (it would also be fine if we asked for 64-bit
190	* PCI DMA addresses on a 64-bit system). There might be a problem on 32-bit
191	* extended systems (where the DMA address can be bigger than 32-bits),
192	* if we allow the PCI dma mask to be bigger than 32-bits. So don't do that.
193	*/
194	static int xhci_insert_segment_mapping(struct radix_tree_root *trb_address_map,
195	struct xhci_ring *ring,
196	struct xhci_segment *seg,
197	gfp_t mem_flags)
198	{
199	unsigned long key;
200	int ret;
201
202	key = (unsigned long)(seg->dma >> TRB_SEGMENT_SHIFT);
203	/* Skip any segments that were already added. */
204	if (radix_tree_lookup(trb_address_map, key))
205	return 0;
206
207	ret = radix_tree_maybe_preload(gfp_mask: mem_flags);
208	if (ret)
209	return ret;
210	ret = radix_tree_insert(trb_address_map,
211	index: key, ring);
212	radix_tree_preload_end();
213	return ret;
214	}
215
216	static void xhci_remove_segment_mapping(struct radix_tree_root *trb_address_map,
217	struct xhci_segment *seg)
218	{
219	unsigned long key;
220
221	key = (unsigned long)(seg->dma >> TRB_SEGMENT_SHIFT);
222	if (radix_tree_lookup(trb_address_map, key))
223	radix_tree_delete(trb_address_map, key);
224	}
225
226	static int xhci_update_stream_segment_mapping(
227	struct radix_tree_root *trb_address_map,
228	struct xhci_ring *ring,
229	struct xhci_segment *first_seg,
230	struct xhci_segment *last_seg,
231	gfp_t mem_flags)
232	{
233	struct xhci_segment *seg;
234	struct xhci_segment *failed_seg;
235	int ret;
236
237	if (WARN_ON_ONCE(trb_address_map == NULL))
238	return 0;
239
240	seg = first_seg;
241	do {
242	ret = xhci_insert_segment_mapping(trb_address_map,
243	ring, seg, mem_flags);
244	if (ret)
245	goto remove_streams;
246	if (seg == last_seg)
247	return 0;
248	seg = seg->next;
249	} while (seg != first_seg);
250
251	return 0;
252
253	remove_streams:
254	failed_seg = seg;
255	seg = first_seg;
256	do {
257	xhci_remove_segment_mapping(trb_address_map, seg);
258	if (seg == failed_seg)
259	return ret;
260	seg = seg->next;
261	} while (seg != first_seg);
262
263	return ret;
264	}
265
266	static void xhci_remove_stream_mapping(struct xhci_ring *ring)
267	{
268	struct xhci_segment *seg;
269
270	if (WARN_ON_ONCE(ring->trb_address_map == NULL))
271	return;
272
273	seg = ring->first_seg;
274	do {
275	xhci_remove_segment_mapping(trb_address_map: ring->trb_address_map, seg);
276	seg = seg->next;
277	} while (seg != ring->first_seg);
278	}
279
280	static int xhci_update_stream_mapping(struct xhci_ring *ring, gfp_t mem_flags)
281	{
282	return xhci_update_stream_segment_mapping(trb_address_map: ring->trb_address_map, ring,
283	first_seg: ring->first_seg, last_seg: ring->last_seg, mem_flags);
284	}
285
286	/* XXX: Do we need the hcd structure in all these functions? */
287	void xhci_ring_free(struct xhci_hcd *xhci, struct xhci_ring *ring)
288	{
289	if (!ring)
290	return;
291
292	trace_xhci_ring_free(ring);
293
294	if (ring->first_seg) {
295	if (ring->type == TYPE_STREAM)
296	xhci_remove_stream_mapping(ring);
297	xhci_free_segments_for_ring(xhci, first: ring->first_seg);
298	}
299
300	kfree(objp: ring);
301	}
302
303	void xhci_initialize_ring_info(struct xhci_ring *ring,
304	unsigned int cycle_state)
305	{
306	/* The ring is empty, so the enqueue pointer == dequeue pointer */
307	ring->enqueue = ring->first_seg->trbs;
308	ring->enq_seg = ring->first_seg;
309	ring->dequeue = ring->enqueue;
310	ring->deq_seg = ring->first_seg;
311	/* The ring is initialized to 0. The producer must write 1 to the cycle
312	* bit to handover ownership of the TRB, so PCS = 1. The consumer must
313	* compare CCS to the cycle bit to check ownership, so CCS = 1.
314	*
315	* New rings are initialized with cycle state equal to 1; if we are
316	* handling ring expansion, set the cycle state equal to the old ring.
317	*/
318	ring->cycle_state = cycle_state;
319
320	/*
321	* Each segment has a link TRB, and leave an extra TRB for SW
322	* accounting purpose
323	*/
324	ring->num_trbs_free = ring->num_segs * (TRBS_PER_SEGMENT - 1) - 1;
325	}
326
327	/* Allocate segments and link them for a ring */
328	static int xhci_alloc_segments_for_ring(struct xhci_hcd *xhci,
329	struct xhci_segment **first, struct xhci_segment **last,
330	unsigned int num_segs, unsigned int num,
331	unsigned int cycle_state, enum xhci_ring_type type,
332	unsigned int max_packet, gfp_t flags)
333	{
334	struct xhci_segment *prev;
335	bool chain_links;
336
337	/* Set chain bit for 0.95 hosts, and for isoc rings on AMD 0.96 host */
338	chain_links = !!(xhci_link_trb_quirk(xhci) ||
339	(type == TYPE_ISOC &&
340	(xhci->quirks & XHCI_AMD_0x96_HOST)));
341
342	prev = xhci_segment_alloc(xhci, cycle_state, max_packet, num, flags);
343	if (!prev)
344	return -ENOMEM;
345	num++;
346
347	*first = prev;
348	while (num < num_segs) {
349	struct xhci_segment *next;
350
351	next = xhci_segment_alloc(xhci, cycle_state, max_packet, num,
352	flags);
353	if (!next) {
354	prev = *first;
355	while (prev) {
356	next = prev->next;
357	xhci_segment_free(xhci, seg: prev);
358	prev = next;
359	}
360	return -ENOMEM;
361	}
362	xhci_link_segments(prev, next, type, chain_links);
363
364	prev = next;
365	num++;
366	}
367	xhci_link_segments(prev, next: *first, type, chain_links);
368	*last = prev;
369
370	return 0;
371	}
372
373	/*
374	* Create a new ring with zero or more segments.
375	*
376	* Link each segment together into a ring.
377	* Set the end flag and the cycle toggle bit on the last segment.
378	* See section 4.9.1 and figures 15 and 16.
379	*/
380	struct xhci_ring *xhci_ring_alloc(struct xhci_hcd *xhci,
381	unsigned int num_segs, unsigned int cycle_state,
382	enum xhci_ring_type type, unsigned int max_packet, gfp_t flags)
383	{
384	struct xhci_ring *ring;
385	int ret;
386	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
387
388	ring = kzalloc_node(size: sizeof(*ring), flags, node: dev_to_node(dev));
389	if (!ring)
390	return NULL;
391
392	ring->num_segs = num_segs;
393	ring->bounce_buf_len = max_packet;
394	INIT_LIST_HEAD(list: &ring->td_list);
395	ring->type = type;
396	if (num_segs == 0)
397	return ring;
398
399	ret = xhci_alloc_segments_for_ring(xhci, first: &ring->first_seg,
400	last: &ring->last_seg, num_segs, num: 0, cycle_state, type,
401	max_packet, flags);
402	if (ret)
403	goto fail;
404
405	/* Only event ring does not use link TRB */
406	if (type != TYPE_EVENT) {
407	/* See section 4.9.2.1 and 6.4.4.1 */
408	ring->last_seg->trbs[TRBS_PER_SEGMENT - 1].link.control |=
409	cpu_to_le32(LINK_TOGGLE);
410	}
411	xhci_initialize_ring_info(ring, cycle_state);
412	trace_xhci_ring_alloc(ring);
413	return ring;
414
415	fail:
416	kfree(objp: ring);
417	return NULL;
418	}
419
420	void xhci_free_endpoint_ring(struct xhci_hcd *xhci,
421	struct xhci_virt_device *virt_dev,
422	unsigned int ep_index)
423	{
424	xhci_ring_free(xhci, ring: virt_dev->eps[ep_index].ring);
425	virt_dev->eps[ep_index].ring = NULL;
426	}
427
428	/*
429	* Expand an existing ring.
430	* Allocate a new ring which has same segment numbers and link the two rings.
431	*/
432	int xhci_ring_expansion(struct xhci_hcd *xhci, struct xhci_ring *ring,
433	unsigned int num_new_segs, gfp_t flags)
434	{
435	struct xhci_segment *first;
436	struct xhci_segment *last;
437	int ret;
438
439	ret = xhci_alloc_segments_for_ring(xhci, first: &first, last: &last,
440	num_segs: num_new_segs, num: ring->enq_seg->num + 1,
441	cycle_state: ring->cycle_state, type: ring->type,
442	max_packet: ring->bounce_buf_len, flags);
443	if (ret)
444	return -ENOMEM;
445
446	if (ring->type == TYPE_STREAM)
447	ret = xhci_update_stream_segment_mapping(trb_address_map: ring->trb_address_map,
448	ring, first_seg: first, last_seg: last, mem_flags: flags);
449	if (ret) {
450	struct xhci_segment *next;
451	do {
452	next = first->next;
453	xhci_segment_free(xhci, seg: first);
454	if (first == last)
455	break;
456	first = next;
457	} while (true);
458	return ret;
459	}
460
461	xhci_link_rings(xhci, ring, first, last, num_segs: num_new_segs);
462	trace_xhci_ring_expansion(ring);
463	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_ring_expansion,
464	fmt: "ring expansion succeed, now has %d segments",
465	ring->num_segs);
466
467	return 0;
468	}
469
470	struct xhci_container_ctx *xhci_alloc_container_ctx(struct xhci_hcd *xhci,
471	int type, gfp_t flags)
472	{
473	struct xhci_container_ctx *ctx;
474	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
475
476	if ((type != XHCI_CTX_TYPE_DEVICE) && (type != XHCI_CTX_TYPE_INPUT))
477	return NULL;
478
479	ctx = kzalloc_node(size: sizeof(*ctx), flags, node: dev_to_node(dev));
480	if (!ctx)
481	return NULL;
482
483	ctx->type = type;
484	ctx->size = HCC_64BYTE_CONTEXT(xhci->hcc_params) ? 2048 : 1024;
485	if (type == XHCI_CTX_TYPE_INPUT)
486	ctx->size += CTX_SIZE(xhci->hcc_params);
487
488	ctx->bytes = dma_pool_zalloc(pool: xhci->device_pool, mem_flags: flags, handle: &ctx->dma);
489	if (!ctx->bytes) {
490	kfree(objp: ctx);
491	return NULL;
492	}
493	return ctx;
494	}
495
496	void xhci_free_container_ctx(struct xhci_hcd *xhci,
497	struct xhci_container_ctx *ctx)
498	{
499	if (!ctx)
500	return;
501	dma_pool_free(pool: xhci->device_pool, vaddr: ctx->bytes, addr: ctx->dma);
502	kfree(objp: ctx);
503	}
504
505	struct xhci_input_control_ctx *xhci_get_input_control_ctx(
506	struct xhci_container_ctx *ctx)
507	{
508	if (ctx->type != XHCI_CTX_TYPE_INPUT)
509	return NULL;
510
511	return (struct xhci_input_control_ctx *)ctx->bytes;
512	}
513
514	struct xhci_slot_ctx *xhci_get_slot_ctx(struct xhci_hcd *xhci,
515	struct xhci_container_ctx *ctx)
516	{
517	if (ctx->type == XHCI_CTX_TYPE_DEVICE)
518	return (struct xhci_slot_ctx *)ctx->bytes;
519
520	return (struct xhci_slot_ctx *)
521	(ctx->bytes + CTX_SIZE(xhci->hcc_params));
522	}
523
524	struct xhci_ep_ctx *xhci_get_ep_ctx(struct xhci_hcd *xhci,
525	struct xhci_container_ctx *ctx,
526	unsigned int ep_index)
527	{
528	/* increment ep index by offset of start of ep ctx array */
529	ep_index++;
530	if (ctx->type == XHCI_CTX_TYPE_INPUT)
531	ep_index++;
532
533	return (struct xhci_ep_ctx *)
534	(ctx->bytes + (ep_index * CTX_SIZE(xhci->hcc_params)));
535	}
536	EXPORT_SYMBOL_GPL(xhci_get_ep_ctx);
537
538	/***************** Streams structures manipulation *************************/
539
540	static void xhci_free_stream_ctx(struct xhci_hcd *xhci,
541	unsigned int num_stream_ctxs,
542	struct xhci_stream_ctx *stream_ctx, dma_addr_t dma)
543	{
544	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
545	size_t size = sizeof(struct xhci_stream_ctx) * num_stream_ctxs;
546
547	if (size > MEDIUM_STREAM_ARRAY_SIZE)
548	dma_free_coherent(dev, size, cpu_addr: stream_ctx, dma_handle: dma);
549	else if (size > SMALL_STREAM_ARRAY_SIZE)
550	dma_pool_free(pool: xhci->medium_streams_pool, vaddr: stream_ctx, addr: dma);
551	else
552	dma_pool_free(pool: xhci->small_streams_pool, vaddr: stream_ctx, addr: dma);
553	}
554
555	/*
556	* The stream context array for each endpoint with bulk streams enabled can
557	* vary in size, based on:
558	* - how many streams the endpoint supports,
559	* - the maximum primary stream array size the host controller supports,
560	* - and how many streams the device driver asks for.
561	*
562	* The stream context array must be a power of 2, and can be as small as
563	* 64 bytes or as large as 1MB.
564	*/
565	static struct xhci_stream_ctx *xhci_alloc_stream_ctx(struct xhci_hcd *xhci,
566	unsigned int num_stream_ctxs, dma_addr_t *dma,
567	gfp_t mem_flags)
568	{
569	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
570	size_t size = size_mul(factor1: sizeof(struct xhci_stream_ctx), factor2: num_stream_ctxs);
571
572	if (size > MEDIUM_STREAM_ARRAY_SIZE)
573	return dma_alloc_coherent(dev, size, dma_handle: dma, gfp: mem_flags);
574	if (size > SMALL_STREAM_ARRAY_SIZE)
575	return dma_pool_zalloc(pool: xhci->medium_streams_pool, mem_flags, handle: dma);
576	else
577	return dma_pool_zalloc(pool: xhci->small_streams_pool, mem_flags, handle: dma);
578	}
579
580	struct xhci_ring *xhci_dma_to_transfer_ring(
581	struct xhci_virt_ep *ep,
582	u64 address)
583	{
584	if (ep->ep_state & EP_HAS_STREAMS)
585	return radix_tree_lookup(&ep->stream_info->trb_address_map,
586	address >> TRB_SEGMENT_SHIFT);
587	return ep->ring;
588	}
589
590	/*
591	* Change an endpoint's internal structure so it supports stream IDs. The
592	* number of requested streams includes stream 0, which cannot be used by device
593	* drivers.
594	*
595	* The number of stream contexts in the stream context array may be bigger than
596	* the number of streams the driver wants to use. This is because the number of
597	* stream context array entries must be a power of two.
598	*/
599	struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci,
600	unsigned int num_stream_ctxs,
601	unsigned int num_streams,
602	unsigned int max_packet, gfp_t mem_flags)
603	{
604	struct xhci_stream_info *stream_info;
605	u32 cur_stream;
606	struct xhci_ring *cur_ring;
607	u64 addr;
608	int ret;
609	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
610
611	xhci_dbg(xhci, "Allocating %u streams and %u stream context array entries.\n",
612	num_streams, num_stream_ctxs);
613	if (xhci->cmd_ring_reserved_trbs == MAX_RSVD_CMD_TRBS) {
614	xhci_dbg(xhci, "Command ring has no reserved TRBs available\n");
615	return NULL;
616	}
617	xhci->cmd_ring_reserved_trbs++;
618
619	stream_info = kzalloc_node(size: sizeof(*stream_info), flags: mem_flags,
620	node: dev_to_node(dev));
621	if (!stream_info)
622	goto cleanup_trbs;
623
624	stream_info->num_streams = num_streams;
625	stream_info->num_stream_ctxs = num_stream_ctxs;
626
627	/* Initialize the array of virtual pointers to stream rings. */
628	stream_info->stream_rings = kcalloc_node(
629	n: num_streams, size: sizeof(struct xhci_ring *), flags: mem_flags,
630	node: dev_to_node(dev));
631	if (!stream_info->stream_rings)
632	goto cleanup_info;
633
634	/* Initialize the array of DMA addresses for stream rings for the HW. */
635	stream_info->stream_ctx_array = xhci_alloc_stream_ctx(xhci,
636	num_stream_ctxs, dma: &stream_info->ctx_array_dma,
637	mem_flags);
638	if (!stream_info->stream_ctx_array)
639	goto cleanup_ring_array;
640
641	/* Allocate everything needed to free the stream rings later */
642	stream_info->free_streams_command =
643	xhci_alloc_command_with_ctx(xhci, allocate_completion: true, mem_flags);
644	if (!stream_info->free_streams_command)
645	goto cleanup_ctx;
646
647	INIT_RADIX_TREE(&stream_info->trb_address_map, GFP_ATOMIC);
648
649	/* Allocate rings for all the streams that the driver will use,
650	* and add their segment DMA addresses to the radix tree.
651	* Stream 0 is reserved.
652	*/
653
654	for (cur_stream = 1; cur_stream < num_streams; cur_stream++) {
655	stream_info->stream_rings[cur_stream] =
656	xhci_ring_alloc(xhci, num_segs: 2, cycle_state: 1, type: TYPE_STREAM, max_packet,
657	flags: mem_flags);
658	cur_ring = stream_info->stream_rings[cur_stream];
659	if (!cur_ring)
660	goto cleanup_rings;
661	cur_ring->stream_id = cur_stream;
662	cur_ring->trb_address_map = &stream_info->trb_address_map;
663	/* Set deq ptr, cycle bit, and stream context type */
664	addr = cur_ring->first_seg->dma |
665	SCT_FOR_CTX(SCT_PRI_TR) |
666	cur_ring->cycle_state;
667	stream_info->stream_ctx_array[cur_stream].stream_ring =
668	cpu_to_le64(addr);
669	xhci_dbg(xhci, "Setting stream %d ring ptr to 0x%08llx\n", cur_stream, addr);
670
671	ret = xhci_update_stream_mapping(ring: cur_ring, mem_flags);
672	if (ret) {
673	xhci_ring_free(xhci, ring: cur_ring);
674	stream_info->stream_rings[cur_stream] = NULL;
675	goto cleanup_rings;
676	}
677	}
678	/* Leave the other unused stream ring pointers in the stream context
679	* array initialized to zero. This will cause the xHC to give us an
680	* error if the device asks for a stream ID we don't have setup (if it
681	* was any other way, the host controller would assume the ring is
682	* "empty" and wait forever for data to be queued to that stream ID).
683	*/
684
685	return stream_info;
686
687	cleanup_rings:
688	for (cur_stream = 1; cur_stream < num_streams; cur_stream++) {
689	cur_ring = stream_info->stream_rings[cur_stream];
690	if (cur_ring) {
691	xhci_ring_free(xhci, ring: cur_ring);
692	stream_info->stream_rings[cur_stream] = NULL;
693	}
694	}
695	xhci_free_command(xhci, command: stream_info->free_streams_command);
696	cleanup_ctx:
697	xhci_free_stream_ctx(xhci,
698	num_stream_ctxs: stream_info->num_stream_ctxs,
699	stream_ctx: stream_info->stream_ctx_array,
700	dma: stream_info->ctx_array_dma);
701	cleanup_ring_array:
702	kfree(objp: stream_info->stream_rings);
703	cleanup_info:
704	kfree(objp: stream_info);
705	cleanup_trbs:
706	xhci->cmd_ring_reserved_trbs--;
707	return NULL;
708	}
709	/*
710	* Sets the MaxPStreams field and the Linear Stream Array field.
711	* Sets the dequeue pointer to the stream context array.
712	*/
713	void xhci_setup_streams_ep_input_ctx(struct xhci_hcd *xhci,
714	struct xhci_ep_ctx *ep_ctx,
715	struct xhci_stream_info *stream_info)
716	{
717	u32 max_primary_streams;
718	/* MaxPStreams is the number of stream context array entries, not the
719	* number we're actually using. Must be in 2^(MaxPstreams + 1) format.
720	* fls(0) = 0, fls(0x1) = 1, fls(0x10) = 2, fls(0x100) = 3, etc.
721	*/
722	max_primary_streams = fls(x: stream_info->num_stream_ctxs) - 2;
723	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_context_change,
724	fmt: "Setting number of stream ctx array entries to %u",
725	1 << (max_primary_streams + 1));
726	ep_ctx->ep_info &= cpu_to_le32(~EP_MAXPSTREAMS_MASK);
727	ep_ctx->ep_info |= cpu_to_le32(EP_MAXPSTREAMS(max_primary_streams)
728	| EP_HAS_LSA);
729	ep_ctx->deq = cpu_to_le64(stream_info->ctx_array_dma);
730	}
731
732	/*
733	* Sets the MaxPStreams field and the Linear Stream Array field to 0.
734	* Reinstalls the "normal" endpoint ring (at its previous dequeue mark,
735	* not at the beginning of the ring).
736	*/
737	void xhci_setup_no_streams_ep_input_ctx(struct xhci_ep_ctx *ep_ctx,
738	struct xhci_virt_ep *ep)
739	{
740	dma_addr_t addr;
741	ep_ctx->ep_info &= cpu_to_le32(~(EP_MAXPSTREAMS_MASK | EP_HAS_LSA));
742	addr = xhci_trb_virt_to_dma(seg: ep->ring->deq_seg, trb: ep->ring->dequeue);
743	ep_ctx->deq = cpu_to_le64(addr | ep->ring->cycle_state);
744	}
745
746	/* Frees all stream contexts associated with the endpoint,
747	*
748	* Caller should fix the endpoint context streams fields.
749	*/
750	void xhci_free_stream_info(struct xhci_hcd *xhci,
751	struct xhci_stream_info *stream_info)
752	{
753	int cur_stream;
754	struct xhci_ring *cur_ring;
755
756	if (!stream_info)
757	return;
758
759	for (cur_stream = 1; cur_stream < stream_info->num_streams;
760	cur_stream++) {
761	cur_ring = stream_info->stream_rings[cur_stream];
762	if (cur_ring) {
763	xhci_ring_free(xhci, ring: cur_ring);
764	stream_info->stream_rings[cur_stream] = NULL;
765	}
766	}
767	xhci_free_command(xhci, command: stream_info->free_streams_command);
768	xhci->cmd_ring_reserved_trbs--;
769	if (stream_info->stream_ctx_array)
770	xhci_free_stream_ctx(xhci,
771	num_stream_ctxs: stream_info->num_stream_ctxs,
772	stream_ctx: stream_info->stream_ctx_array,
773	dma: stream_info->ctx_array_dma);
774
775	kfree(objp: stream_info->stream_rings);
776	kfree(objp: stream_info);
777	}
778
779
780	/***************** Device context manipulation *************************/
781
782	static void xhci_free_tt_info(struct xhci_hcd *xhci,
783	struct xhci_virt_device *virt_dev,
784	int slot_id)
785	{
786	struct list_head *tt_list_head;
787	struct xhci_tt_bw_info *tt_info, *next;
788	bool slot_found = false;
789
790	/* If the device never made it past the Set Address stage,
791	* it may not have the real_port set correctly.
792	*/
793	if (virt_dev->real_port == 0 ||
794	virt_dev->real_port > HCS_MAX_PORTS(xhci->hcs_params1)) {
795	xhci_dbg(xhci, "Bad real port.\n");
796	return;
797	}
798
799	tt_list_head = &(xhci->rh_bw[virt_dev->real_port - 1].tts);
800	list_for_each_entry_safe(tt_info, next, tt_list_head, tt_list) {
801	/* Multi-TT hubs will have more than one entry */
802	if (tt_info->slot_id == slot_id) {
803	slot_found = true;
804	list_del(entry: &tt_info->tt_list);
805	kfree(objp: tt_info);
806	} else if (slot_found) {
807	break;
808	}
809	}
810	}
811
812	int xhci_alloc_tt_info(struct xhci_hcd *xhci,
813	struct xhci_virt_device *virt_dev,
814	struct usb_device *hdev,
815	struct usb_tt *tt, gfp_t mem_flags)
816	{
817	struct xhci_tt_bw_info *tt_info;
818	unsigned int num_ports;
819	int i, j;
820	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
821
822	if (!tt->multi)
823	num_ports = 1;
824	else
825	num_ports = hdev->maxchild;
826
827	for (i = 0; i < num_ports; i++, tt_info++) {
828	struct xhci_interval_bw_table *bw_table;
829
830	tt_info = kzalloc_node(size: sizeof(*tt_info), flags: mem_flags,
831	node: dev_to_node(dev));
832	if (!tt_info)
833	goto free_tts;
834	INIT_LIST_HEAD(list: &tt_info->tt_list);
835	list_add(new: &tt_info->tt_list,
836	head: &xhci->rh_bw[virt_dev->real_port - 1].tts);
837	tt_info->slot_id = virt_dev->udev->slot_id;
838	if (tt->multi)
839	tt_info->ttport = i+1;
840	bw_table = &tt_info->bw_table;
841	for (j = 0; j < XHCI_MAX_INTERVAL; j++)
842	INIT_LIST_HEAD(list: &bw_table->interval_bw[j].endpoints);
843	}
844	return 0;
845
846	free_tts:
847	xhci_free_tt_info(xhci, virt_dev, slot_id: virt_dev->udev->slot_id);
848	return -ENOMEM;
849	}
850
851
852	/* All the xhci_tds in the ring's TD list should be freed at this point.
853	* Should be called with xhci->lock held if there is any chance the TT lists
854	* will be manipulated by the configure endpoint, allocate device, or update
855	* hub functions while this function is removing the TT entries from the list.
856	*/
857	void xhci_free_virt_device(struct xhci_hcd *xhci, int slot_id)
858	{
859	struct xhci_virt_device *dev;
860	int i;
861	int old_active_eps = 0;
862
863	/* Slot ID 0 is reserved */
864	if (slot_id == 0 || !xhci->devs[slot_id])
865	return;
866
867	dev = xhci->devs[slot_id];
868
869	xhci->dcbaa->dev_context_ptrs[slot_id] = 0;
870	if (!dev)
871	return;
872
873	trace_xhci_free_virt_device(vdev: dev);
874
875	if (dev->tt_info)
876	old_active_eps = dev->tt_info->active_eps;
877
878	for (i = 0; i < 31; i++) {
879	if (dev->eps[i].ring)
880	xhci_ring_free(xhci, ring: dev->eps[i].ring);
881	if (dev->eps[i].stream_info)
882	xhci_free_stream_info(xhci,
883	stream_info: dev->eps[i].stream_info);
884	/*
885	* Endpoints are normally deleted from the bandwidth list when
886	* endpoints are dropped, before device is freed.
887	* If host is dying or being removed then endpoints aren't
888	* dropped cleanly, so delete the endpoint from list here.
889	* Only applicable for hosts with software bandwidth checking.
890	*/
891
892	if (!list_empty(head: &dev->eps[i].bw_endpoint_list)) {
893	list_del_init(entry: &dev->eps[i].bw_endpoint_list);
894	xhci_dbg(xhci, "Slot %u endpoint %u not removed from BW list!\n",
895	slot_id, i);
896	}
897	}
898	/* If this is a hub, free the TT(s) from the TT list */
899	xhci_free_tt_info(xhci, virt_dev: dev, slot_id);
900	/* If necessary, update the number of active TTs on this root port */
901	xhci_update_tt_active_eps(xhci, virt_dev: dev, old_active_eps);
902
903	if (dev->in_ctx)
904	xhci_free_container_ctx(xhci, ctx: dev->in_ctx);
905	if (dev->out_ctx)
906	xhci_free_container_ctx(xhci, ctx: dev->out_ctx);
907
908	if (dev->udev && dev->udev->slot_id)
909	dev->udev->slot_id = 0;
910	kfree(objp: xhci->devs[slot_id]);
911	xhci->devs[slot_id] = NULL;
912	}
913
914	/*
915	* Free a virt_device structure.
916	* If the virt_device added a tt_info (a hub) and has children pointing to
917	* that tt_info, then free the child first. Recursive.
918	* We can't rely on udev at this point to find child-parent relationships.
919	*/
920	static void xhci_free_virt_devices_depth_first(struct xhci_hcd *xhci, int slot_id)
921	{
922	struct xhci_virt_device *vdev;
923	struct list_head *tt_list_head;
924	struct xhci_tt_bw_info *tt_info, *next;
925	int i;
926
927	vdev = xhci->devs[slot_id];
928	if (!vdev)
929	return;
930
931	if (vdev->real_port == 0 ||
932	vdev->real_port > HCS_MAX_PORTS(xhci->hcs_params1)) {
933	xhci_dbg(xhci, "Bad vdev->real_port.\n");
934	goto out;
935	}
936
937	tt_list_head = &(xhci->rh_bw[vdev->real_port - 1].tts);
938	list_for_each_entry_safe(tt_info, next, tt_list_head, tt_list) {
939	/* is this a hub device that added a tt_info to the tts list */
940	if (tt_info->slot_id == slot_id) {
941	/* are any devices using this tt_info? */
942	for (i = 1; i < HCS_MAX_SLOTS(xhci->hcs_params1); i++) {
943	vdev = xhci->devs[i];
944	if (vdev && (vdev->tt_info == tt_info))
945	xhci_free_virt_devices_depth_first(
946	xhci, slot_id: i);
947	}
948	}
949	}
950	out:
951	/* we are now at a leaf device */
952	xhci_debugfs_remove_slot(xhci, slot_id);
953	xhci_free_virt_device(xhci, slot_id);
954	}
955
956	int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id,
957	struct usb_device *udev, gfp_t flags)
958	{
959	struct xhci_virt_device *dev;
960	int i;
961
962	/* Slot ID 0 is reserved */
963	if (slot_id == 0 || xhci->devs[slot_id]) {
964	xhci_warn(xhci, "Bad Slot ID %d\n", slot_id);
965	return 0;
966	}
967
968	dev = kzalloc(size: sizeof(*dev), flags);
969	if (!dev)
970	return 0;
971
972	dev->slot_id = slot_id;
973
974	/* Allocate the (output) device context that will be used in the HC. */
975	dev->out_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_DEVICE, flags);
976	if (!dev->out_ctx)
977	goto fail;
978
979	xhci_dbg(xhci, "Slot %d output ctx = 0x%pad (dma)\n", slot_id, &dev->out_ctx->dma);
980
981	/* Allocate the (input) device context for address device command */
982	dev->in_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT, flags);
983	if (!dev->in_ctx)
984	goto fail;
985
986	xhci_dbg(xhci, "Slot %d input ctx = 0x%pad (dma)\n", slot_id, &dev->in_ctx->dma);
987
988	/* Initialize the cancellation and bandwidth list for each ep */
989	for (i = 0; i < 31; i++) {
990	dev->eps[i].ep_index = i;
991	dev->eps[i].vdev = dev;
992	dev->eps[i].xhci = xhci;
993	INIT_LIST_HEAD(list: &dev->eps[i].cancelled_td_list);
994	INIT_LIST_HEAD(list: &dev->eps[i].bw_endpoint_list);
995	}
996
997	/* Allocate endpoint 0 ring */
998	dev->eps[0].ring = xhci_ring_alloc(xhci, num_segs: 2, cycle_state: 1, type: TYPE_CTRL, max_packet: 0, flags);
999	if (!dev->eps[0].ring)
1000	goto fail;
1001
1002	dev->udev = udev;
1003
1004	/* Point to output device context in dcbaa. */
1005	xhci->dcbaa->dev_context_ptrs[slot_id] = cpu_to_le64(dev->out_ctx->dma);
1006	xhci_dbg(xhci, "Set slot id %d dcbaa entry %p to 0x%llx\n",
1007	slot_id,
1008	&xhci->dcbaa->dev_context_ptrs[slot_id],
1009	le64_to_cpu(xhci->dcbaa->dev_context_ptrs[slot_id]));
1010
1011	trace_xhci_alloc_virt_device(vdev: dev);
1012
1013	xhci->devs[slot_id] = dev;
1014
1015	return 1;
1016	fail:
1017
1018	if (dev->in_ctx)
1019	xhci_free_container_ctx(xhci, ctx: dev->in_ctx);
1020	if (dev->out_ctx)
1021	xhci_free_container_ctx(xhci, ctx: dev->out_ctx);
1022	kfree(objp: dev);
1023
1024	return 0;
1025	}
1026
1027	void xhci_copy_ep0_dequeue_into_input_ctx(struct xhci_hcd *xhci,
1028	struct usb_device *udev)
1029	{
1030	struct xhci_virt_device *virt_dev;
1031	struct xhci_ep_ctx *ep0_ctx;
1032	struct xhci_ring *ep_ring;
1033
1034	virt_dev = xhci->devs[udev->slot_id];
1035	ep0_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, 0);
1036	ep_ring = virt_dev->eps[0].ring;
1037	/*
1038	* FIXME we don't keep track of the dequeue pointer very well after a
1039	* Set TR dequeue pointer, so we're setting the dequeue pointer of the
1040	* host to our enqueue pointer. This should only be called after a
1041	* configured device has reset, so all control transfers should have
1042	* been completed or cancelled before the reset.
1043	*/
1044	ep0_ctx->deq = cpu_to_le64(xhci_trb_virt_to_dma(ep_ring->enq_seg,
1045	ep_ring->enqueue)
1046	| ep_ring->cycle_state);
1047	}
1048
1049	/*
1050	* The xHCI roothub may have ports of differing speeds in any order in the port
1051	* status registers.
1052	*
1053	* The xHCI hardware wants to know the roothub port number that the USB device
1054	* is attached to (or the roothub port its ancestor hub is attached to). All we
1055	* know is the index of that port under either the USB 2.0 or the USB 3.0
1056	* roothub, but that doesn't give us the real index into the HW port status
1057	* registers. Call xhci_find_raw_port_number() to get real index.
1058	*/
1059	static u32 xhci_find_real_port_number(struct xhci_hcd *xhci,
1060	struct usb_device *udev)
1061	{
1062	struct usb_device *top_dev;
1063	struct usb_hcd *hcd;
1064
1065	if (udev->speed >= USB_SPEED_SUPER)
1066	hcd = xhci_get_usb3_hcd(xhci);
1067	else
1068	hcd = xhci->main_hcd;
1069
1070	for (top_dev = udev; top_dev->parent && top_dev->parent->parent;
1071	top_dev = top_dev->parent)
1072	/* Found device below root hub */;
1073
1074	return xhci_find_raw_port_number(hcd, port1: top_dev->portnum);
1075	}
1076
1077	/* Setup an xHCI virtual device for a Set Address command */
1078	int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *udev)
1079	{
1080	struct xhci_virt_device *dev;
1081	struct xhci_ep_ctx *ep0_ctx;
1082	struct xhci_slot_ctx *slot_ctx;
1083	u32 port_num;
1084	u32 max_packets;
1085	struct usb_device *top_dev;
1086
1087	dev = xhci->devs[udev->slot_id];
1088	/* Slot ID 0 is reserved */
1089	if (udev->slot_id == 0 || !dev) {
1090	xhci_warn(xhci, "Slot ID %d is not assigned to this device\n",
1091	udev->slot_id);
1092	return -EINVAL;
1093	}
1094	ep0_ctx = xhci_get_ep_ctx(xhci, dev->in_ctx, 0);
1095	slot_ctx = xhci_get_slot_ctx(xhci, ctx: dev->in_ctx);
1096
1097	/* 3) Only the control endpoint is valid - one endpoint context */
1098	slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1) | udev->route);
1099	switch (udev->speed) {
1100	case USB_SPEED_SUPER_PLUS:
1101	slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_SSP);
1102	max_packets = MAX_PACKET(512);
1103	break;
1104	case USB_SPEED_SUPER:
1105	slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_SS);
1106	max_packets = MAX_PACKET(512);
1107	break;
1108	case USB_SPEED_HIGH:
1109	slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_HS);
1110	max_packets = MAX_PACKET(64);
1111	break;
1112	/* USB core guesses at a 64-byte max packet first for FS devices */
1113	case USB_SPEED_FULL:
1114	slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_FS);
1115	max_packets = MAX_PACKET(64);
1116	break;
1117	case USB_SPEED_LOW:
1118	slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_LS);
1119	max_packets = MAX_PACKET(8);
1120	break;
1121	default:
1122	/* Speed was set earlier, this shouldn't happen. */
1123	return -EINVAL;
1124	}
1125	/* Find the root hub port this device is under */
1126	port_num = xhci_find_real_port_number(xhci, udev);
1127	if (!port_num)
1128	return -EINVAL;
1129	slot_ctx->dev_info2 |= cpu_to_le32(ROOT_HUB_PORT(port_num));
1130	/* Set the port number in the virtual_device to the faked port number */
1131	for (top_dev = udev; top_dev->parent && top_dev->parent->parent;
1132	top_dev = top_dev->parent)
1133	/* Found device below root hub */;
1134	dev->fake_port = top_dev->portnum;
1135	dev->real_port = port_num;
1136	xhci_dbg(xhci, "Set root hub portnum to %d\n", port_num);
1137	xhci_dbg(xhci, "Set fake root hub portnum to %d\n", dev->fake_port);
1138
1139	/* Find the right bandwidth table that this device will be a part of.
1140	* If this is a full speed device attached directly to a root port (or a
1141	* decendent of one), it counts as a primary bandwidth domain, not a
1142	* secondary bandwidth domain under a TT. An xhci_tt_info structure
1143	* will never be created for the HS root hub.
1144	*/
1145	if (!udev->tt || !udev->tt->hub->parent) {
1146	dev->bw_table = &xhci->rh_bw[port_num - 1].bw_table;
1147	} else {
1148	struct xhci_root_port_bw_info *rh_bw;
1149	struct xhci_tt_bw_info *tt_bw;
1150
1151	rh_bw = &xhci->rh_bw[port_num - 1];
1152	/* Find the right TT. */
1153	list_for_each_entry(tt_bw, &rh_bw->tts, tt_list) {
1154	if (tt_bw->slot_id != udev->tt->hub->slot_id)
1155	continue;
1156
1157	if (!dev->udev->tt->multi ||
1158	(udev->tt->multi &&
1159	tt_bw->ttport == dev->udev->ttport)) {
1160	dev->bw_table = &tt_bw->bw_table;
1161	dev->tt_info = tt_bw;
1162	break;
1163	}
1164	}
1165	if (!dev->tt_info)
1166	xhci_warn(xhci, "WARN: Didn't find a matching TT\n");
1167	}
1168
1169	/* Is this a LS/FS device under an external HS hub? */
1170	if (udev->tt && udev->tt->hub->parent) {
1171	slot_ctx->tt_info = cpu_to_le32(udev->tt->hub->slot_id |
1172	(udev->ttport << 8));
1173	if (udev->tt->multi)
1174	slot_ctx->dev_info |= cpu_to_le32(DEV_MTT);
1175	}
1176	xhci_dbg(xhci, "udev->tt = %p\n", udev->tt);
1177	xhci_dbg(xhci, "udev->ttport = 0x%x\n", udev->ttport);
1178
1179	/* Step 4 - ring already allocated */
1180	/* Step 5 */
1181	ep0_ctx->ep_info2 = cpu_to_le32(EP_TYPE(CTRL_EP));
1182
1183	/* EP 0 can handle "burst" sizes of 1, so Max Burst Size field is 0 */
1184	ep0_ctx->ep_info2 |= cpu_to_le32(MAX_BURST(0) | ERROR_COUNT(3) |
1185	max_packets);
1186
1187	ep0_ctx->deq = cpu_to_le64(dev->eps[0].ring->first_seg->dma |
1188	dev->eps[0].ring->cycle_state);
1189
1190	trace_xhci_setup_addressable_virt_device(vdev: dev);
1191
1192	/* Steps 7 and 8 were done in xhci_alloc_virt_device() */
1193
1194	return 0;
1195	}
1196
1197	/*
1198	* Convert interval expressed as 2^(bInterval - 1) == interval into
1199	* straight exponent value 2^n == interval.
1200	*
1201	*/
1202	static unsigned int xhci_parse_exponent_interval(struct usb_device *udev,
1203	struct usb_host_endpoint *ep)
1204	{
1205	unsigned int interval;
1206
1207	interval = clamp_val(ep->desc.bInterval, 1, 16) - 1;
1208	if (interval != ep->desc.bInterval - 1)
1209	dev_warn(&udev->dev,
1210	"ep %#x - rounding interval to %d %sframes\n",
1211	ep->desc.bEndpointAddress,
1212	1 << interval,
1213	udev->speed == USB_SPEED_FULL ? "" : "micro");
1214
1215	if (udev->speed == USB_SPEED_FULL) {
1216	/*
1217	* Full speed isoc endpoints specify interval in frames,
1218	* not microframes. We are using microframes everywhere,
1219	* so adjust accordingly.
1220	*/
1221	interval += 3; /* 1 frame = 2^3 uframes */
1222	}
1223
1224	return interval;
1225	}
1226
1227	/*
1228	* Convert bInterval expressed in microframes (in 1-255 range) to exponent of
1229	* microframes, rounded down to nearest power of 2.
1230	*/
1231	static unsigned int xhci_microframes_to_exponent(struct usb_device *udev,
1232	struct usb_host_endpoint *ep, unsigned int desc_interval,
1233	unsigned int min_exponent, unsigned int max_exponent)
1234	{
1235	unsigned int interval;
1236
1237	interval = fls(x: desc_interval) - 1;
1238	interval = clamp_val(interval, min_exponent, max_exponent);
1239	if ((1 << interval) != desc_interval)
1240	dev_dbg(&udev->dev,
1241	"ep %#x - rounding interval to %d microframes, ep desc says %d microframes\n",
1242	ep->desc.bEndpointAddress,
1243	1 << interval,
1244	desc_interval);
1245
1246	return interval;
1247	}
1248
1249	static unsigned int xhci_parse_microframe_interval(struct usb_device *udev,
1250	struct usb_host_endpoint *ep)
1251	{
1252	if (ep->desc.bInterval == 0)
1253	return 0;
1254	return xhci_microframes_to_exponent(udev, ep,
1255	desc_interval: ep->desc.bInterval, min_exponent: 0, max_exponent: 15);
1256	}
1257
1258
1259	static unsigned int xhci_parse_frame_interval(struct usb_device *udev,
1260	struct usb_host_endpoint *ep)
1261	{
1262	return xhci_microframes_to_exponent(udev, ep,
1263	desc_interval: ep->desc.bInterval * 8, min_exponent: 3, max_exponent: 10);
1264	}
1265
1266	/* Return the polling or NAK interval.
1267	*
1268	* The polling interval is expressed in "microframes". If xHCI's Interval field
1269	* is set to N, it will service the endpoint every 2^(Interval)*125us.
1270	*
1271	* The NAK interval is one NAK per 1 to 255 microframes, or no NAKs if interval
1272	* is set to 0.
1273	*/
1274	static unsigned int xhci_get_endpoint_interval(struct usb_device *udev,
1275	struct usb_host_endpoint *ep)
1276	{
1277	unsigned int interval = 0;
1278
1279	switch (udev->speed) {
1280	case USB_SPEED_HIGH:
1281	/* Max NAK rate */
1282	if (usb_endpoint_xfer_control(epd: &ep->desc) ||
1283	usb_endpoint_xfer_bulk(epd: &ep->desc)) {
1284	interval = xhci_parse_microframe_interval(udev, ep);
1285	break;
1286	}
1287	fallthrough; /* SS and HS isoc/int have same decoding */
1288
1289	case USB_SPEED_SUPER_PLUS:
1290	case USB_SPEED_SUPER:
1291	if (usb_endpoint_xfer_int(epd: &ep->desc) ||
1292	usb_endpoint_xfer_isoc(epd: &ep->desc)) {
1293	interval = xhci_parse_exponent_interval(udev, ep);
1294	}
1295	break;
1296
1297	case USB_SPEED_FULL:
1298	if (usb_endpoint_xfer_isoc(epd: &ep->desc)) {
1299	interval = xhci_parse_exponent_interval(udev, ep);
1300	break;
1301	}
1302	/*
1303	* Fall through for interrupt endpoint interval decoding
1304	* since it uses the same rules as low speed interrupt
1305	* endpoints.
1306	*/
1307	fallthrough;
1308
1309	case USB_SPEED_LOW:
1310	if (usb_endpoint_xfer_int(epd: &ep->desc) ||
1311	usb_endpoint_xfer_isoc(epd: &ep->desc)) {
1312
1313	interval = xhci_parse_frame_interval(udev, ep);
1314	}
1315	break;
1316
1317	default:
1318	BUG();
1319	}
1320	return interval;
1321	}
1322
1323	/* The "Mult" field in the endpoint context is only set for SuperSpeed isoc eps.
1324	* High speed endpoint descriptors can define "the number of additional
1325	* transaction opportunities per microframe", but that goes in the Max Burst
1326	* endpoint context field.
1327	*/
1328	static u32 xhci_get_endpoint_mult(struct usb_device *udev,
1329	struct usb_host_endpoint *ep)
1330	{
1331	if (udev->speed < USB_SPEED_SUPER ||
1332	!usb_endpoint_xfer_isoc(epd: &ep->desc))
1333	return 0;
1334	return ep->ss_ep_comp.bmAttributes;
1335	}
1336
1337	static u32 xhci_get_endpoint_max_burst(struct usb_device *udev,
1338	struct usb_host_endpoint *ep)
1339	{
1340	/* Super speed and Plus have max burst in ep companion desc */
1341	if (udev->speed >= USB_SPEED_SUPER)
1342	return ep->ss_ep_comp.bMaxBurst;
1343
1344	if (udev->speed == USB_SPEED_HIGH &&
1345	(usb_endpoint_xfer_isoc(epd: &ep->desc) ||
1346	usb_endpoint_xfer_int(epd: &ep->desc)))
1347	return usb_endpoint_maxp_mult(epd: &ep->desc) - 1;
1348
1349	return 0;
1350	}
1351
1352	static u32 xhci_get_endpoint_type(struct usb_host_endpoint *ep)
1353	{
1354	int in;
1355
1356	in = usb_endpoint_dir_in(epd: &ep->desc);
1357
1358	switch (usb_endpoint_type(epd: &ep->desc)) {
1359	case USB_ENDPOINT_XFER_CONTROL:
1360	return CTRL_EP;
1361	case USB_ENDPOINT_XFER_BULK:
1362	return in ? BULK_IN_EP : BULK_OUT_EP;
1363	case USB_ENDPOINT_XFER_ISOC:
1364	return in ? ISOC_IN_EP : ISOC_OUT_EP;
1365	case USB_ENDPOINT_XFER_INT:
1366	return in ? INT_IN_EP : INT_OUT_EP;
1367	}
1368	return 0;
1369	}
1370
1371	/* Return the maximum endpoint service interval time (ESIT) payload.
1372	* Basically, this is the maxpacket size, multiplied by the burst size
1373	* and mult size.
1374	*/
1375	static u32 xhci_get_max_esit_payload(struct usb_device *udev,
1376	struct usb_host_endpoint *ep)
1377	{
1378	int max_burst;
1379	int max_packet;
1380
1381	/* Only applies for interrupt or isochronous endpoints */
1382	if (usb_endpoint_xfer_control(epd: &ep->desc) ||
1383	usb_endpoint_xfer_bulk(epd: &ep->desc))
1384	return 0;
1385
1386	/* SuperSpeedPlus Isoc ep sending over 48k per esit */
1387	if ((udev->speed >= USB_SPEED_SUPER_PLUS) &&
1388	USB_SS_SSP_ISOC_COMP(ep->ss_ep_comp.bmAttributes))
1389	return le32_to_cpu(ep->ssp_isoc_ep_comp.dwBytesPerInterval);
1390
1391	/* SuperSpeed or SuperSpeedPlus Isoc ep with less than 48k per esit */
1392	if (udev->speed >= USB_SPEED_SUPER)
1393	return le16_to_cpu(ep->ss_ep_comp.wBytesPerInterval);
1394
1395	max_packet = usb_endpoint_maxp(epd: &ep->desc);
1396	max_burst = usb_endpoint_maxp_mult(epd: &ep->desc);
1397	/* A 0 in max burst means 1 transfer per ESIT */
1398	return max_packet * max_burst;
1399	}
1400
1401	/* Set up an endpoint with one ring segment. Do not allocate stream rings.
1402	* Drivers will have to call usb_alloc_streams() to do that.
1403	*/
1404	int xhci_endpoint_init(struct xhci_hcd *xhci,
1405	struct xhci_virt_device *virt_dev,
1406	struct usb_device *udev,
1407	struct usb_host_endpoint *ep,
1408	gfp_t mem_flags)
1409	{
1410	unsigned int ep_index;
1411	struct xhci_ep_ctx *ep_ctx;
1412	struct xhci_ring *ep_ring;
1413	unsigned int max_packet;
1414	enum xhci_ring_type ring_type;
1415	u32 max_esit_payload;
1416	u32 endpoint_type;
1417	unsigned int max_burst;
1418	unsigned int interval;
1419	unsigned int mult;
1420	unsigned int avg_trb_len;
1421	unsigned int err_count = 0;
1422
1423	ep_index = xhci_get_endpoint_index(desc: &ep->desc);
1424	ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index);
1425
1426	endpoint_type = xhci_get_endpoint_type(ep);
1427	if (!endpoint_type)
1428	return -EINVAL;
1429
1430	ring_type = usb_endpoint_type(epd: &ep->desc);
1431
1432	/*
1433	* Get values to fill the endpoint context, mostly from ep descriptor.
1434	* The average TRB buffer lengt for bulk endpoints is unclear as we
1435	* have no clue on scatter gather list entry size. For Isoc and Int,
1436	* set it to max available. See xHCI 1.1 spec 4.14.1.1 for details.
1437	*/
1438	max_esit_payload = xhci_get_max_esit_payload(udev, ep);
1439	interval = xhci_get_endpoint_interval(udev, ep);
1440
1441	/* Periodic endpoint bInterval limit quirk */
1442	if (usb_endpoint_xfer_int(epd: &ep->desc) ||
1443	usb_endpoint_xfer_isoc(epd: &ep->desc)) {
1444	if ((xhci->quirks & XHCI_LIMIT_ENDPOINT_INTERVAL_7) &&
1445	udev->speed >= USB_SPEED_HIGH &&
1446	interval >= 7) {
1447	interval = 6;
1448	}
1449	}
1450
1451	mult = xhci_get_endpoint_mult(udev, ep);
1452	max_packet = usb_endpoint_maxp(epd: &ep->desc);
1453	max_burst = xhci_get_endpoint_max_burst(udev, ep);
1454	avg_trb_len = max_esit_payload;
1455
1456	/* FIXME dig Mult and streams info out of ep companion desc */
1457
1458	/* Allow 3 retries for everything but isoc, set CErr = 3 */
1459	if (!usb_endpoint_xfer_isoc(epd: &ep->desc))
1460	err_count = 3;
1461	/* HS bulk max packet should be 512, FS bulk supports 8, 16, 32 or 64 */
1462	if (usb_endpoint_xfer_bulk(epd: &ep->desc)) {
1463	if (udev->speed == USB_SPEED_HIGH)
1464	max_packet = 512;
1465	if (udev->speed == USB_SPEED_FULL) {
1466	max_packet = rounddown_pow_of_two(max_packet);
1467	max_packet = clamp_val(max_packet, 8, 64);
1468	}
1469	}
1470	/* xHCI 1.0 and 1.1 indicates that ctrl ep avg TRB Length should be 8 */
1471	if (usb_endpoint_xfer_control(epd: &ep->desc) && xhci->hci_version >= 0x100)
1472	avg_trb_len = 8;
1473	/* xhci 1.1 with LEC support doesn't use mult field, use RsvdZ */
1474	if ((xhci->hci_version > 0x100) && HCC2_LEC(xhci->hcc_params2))
1475	mult = 0;
1476
1477	/* Set up the endpoint ring */
1478	virt_dev->eps[ep_index].new_ring =
1479	xhci_ring_alloc(xhci, num_segs: 2, cycle_state: 1, type: ring_type, max_packet, flags: mem_flags);
1480	if (!virt_dev->eps[ep_index].new_ring)
1481	return -ENOMEM;
1482
1483	virt_dev->eps[ep_index].skip = false;
1484	ep_ring = virt_dev->eps[ep_index].new_ring;
1485
1486	/* Fill the endpoint context */
1487	ep_ctx->ep_info = cpu_to_le32(EP_MAX_ESIT_PAYLOAD_HI(max_esit_payload) |
1488	EP_INTERVAL(interval) |
1489	EP_MULT(mult));
1490	ep_ctx->ep_info2 = cpu_to_le32(EP_TYPE(endpoint_type) |
1491	MAX_PACKET(max_packet) |
1492	MAX_BURST(max_burst) |
1493	ERROR_COUNT(err_count));
1494	ep_ctx->deq = cpu_to_le64(ep_ring->first_seg->dma |
1495	ep_ring->cycle_state);
1496
1497	ep_ctx->tx_info = cpu_to_le32(EP_MAX_ESIT_PAYLOAD_LO(max_esit_payload) |
1498	EP_AVG_TRB_LENGTH(avg_trb_len));
1499
1500	return 0;
1501	}
1502
1503	void xhci_endpoint_zero(struct xhci_hcd *xhci,
1504	struct xhci_virt_device *virt_dev,
1505	struct usb_host_endpoint *ep)
1506	{
1507	unsigned int ep_index;
1508	struct xhci_ep_ctx *ep_ctx;
1509
1510	ep_index = xhci_get_endpoint_index(desc: &ep->desc);
1511	ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index);
1512
1513	ep_ctx->ep_info = 0;
1514	ep_ctx->ep_info2 = 0;
1515	ep_ctx->deq = 0;
1516	ep_ctx->tx_info = 0;
1517	/* Don't free the endpoint ring until the set interface or configuration
1518	* request succeeds.
1519	*/
1520	}
1521
1522	void xhci_clear_endpoint_bw_info(struct xhci_bw_info *bw_info)
1523	{
1524	bw_info->ep_interval = 0;
1525	bw_info->mult = 0;
1526	bw_info->num_packets = 0;
1527	bw_info->max_packet_size = 0;
1528	bw_info->type = 0;
1529	bw_info->max_esit_payload = 0;
1530	}
1531
1532	void xhci_update_bw_info(struct xhci_hcd *xhci,
1533	struct xhci_container_ctx *in_ctx,
1534	struct xhci_input_control_ctx *ctrl_ctx,
1535	struct xhci_virt_device *virt_dev)
1536	{
1537	struct xhci_bw_info *bw_info;
1538	struct xhci_ep_ctx *ep_ctx;
1539	unsigned int ep_type;
1540	int i;
1541
1542	for (i = 1; i < 31; i++) {
1543	bw_info = &virt_dev->eps[i].bw_info;
1544
1545	/* We can't tell what endpoint type is being dropped, but
1546	* unconditionally clearing the bandwidth info for non-periodic
1547	* endpoints should be harmless because the info will never be
1548	* set in the first place.
1549	*/
1550	if (!EP_IS_ADDED(ctrl_ctx, i) && EP_IS_DROPPED(ctrl_ctx, i)) {
1551	/* Dropped endpoint */
1552	xhci_clear_endpoint_bw_info(bw_info);
1553	continue;
1554	}
1555
1556	if (EP_IS_ADDED(ctrl_ctx, i)) {
1557	ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, i);
1558	ep_type = CTX_TO_EP_TYPE(le32_to_cpu(ep_ctx->ep_info2));
1559
1560	/* Ignore non-periodic endpoints */
1561	if (ep_type != ISOC_OUT_EP && ep_type != INT_OUT_EP &&
1562	ep_type != ISOC_IN_EP &&
1563	ep_type != INT_IN_EP)
1564	continue;
1565
1566	/* Added or changed endpoint */
1567	bw_info->ep_interval = CTX_TO_EP_INTERVAL(
1568	le32_to_cpu(ep_ctx->ep_info));
1569	/* Number of packets and mult are zero-based in the
1570	* input context, but we want one-based for the
1571	* interval table.
1572	*/
1573	bw_info->mult = CTX_TO_EP_MULT(
1574	le32_to_cpu(ep_ctx->ep_info)) + 1;
1575	bw_info->num_packets = CTX_TO_MAX_BURST(
1576	le32_to_cpu(ep_ctx->ep_info2)) + 1;
1577	bw_info->max_packet_size = MAX_PACKET_DECODED(
1578	le32_to_cpu(ep_ctx->ep_info2));
1579	bw_info->type = ep_type;
1580	bw_info->max_esit_payload = CTX_TO_MAX_ESIT_PAYLOAD(
1581	le32_to_cpu(ep_ctx->tx_info));
1582	}
1583	}
1584	}
1585
1586	/* Copy output xhci_ep_ctx to the input xhci_ep_ctx copy.
1587	* Useful when you want to change one particular aspect of the endpoint and then
1588	* issue a configure endpoint command.
1589	*/
1590	void xhci_endpoint_copy(struct xhci_hcd *xhci,
1591	struct xhci_container_ctx *in_ctx,
1592	struct xhci_container_ctx *out_ctx,
1593	unsigned int ep_index)
1594	{
1595	struct xhci_ep_ctx *out_ep_ctx;
1596	struct xhci_ep_ctx *in_ep_ctx;
1597
1598	out_ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
1599	in_ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, ep_index);
1600
1601	in_ep_ctx->ep_info = out_ep_ctx->ep_info;
1602	in_ep_ctx->ep_info2 = out_ep_ctx->ep_info2;
1603	in_ep_ctx->deq = out_ep_ctx->deq;
1604	in_ep_ctx->tx_info = out_ep_ctx->tx_info;
1605	if (xhci->quirks & XHCI_MTK_HOST) {
1606	in_ep_ctx->reserved[0] = out_ep_ctx->reserved[0];
1607	in_ep_ctx->reserved[1] = out_ep_ctx->reserved[1];
1608	}
1609	}
1610
1611	/* Copy output xhci_slot_ctx to the input xhci_slot_ctx.
1612	* Useful when you want to change one particular aspect of the endpoint and then
1613	* issue a configure endpoint command. Only the context entries field matters,
1614	* but we'll copy the whole thing anyway.
1615	*/
1616	void xhci_slot_copy(struct xhci_hcd *xhci,
1617	struct xhci_container_ctx *in_ctx,
1618	struct xhci_container_ctx *out_ctx)
1619	{
1620	struct xhci_slot_ctx *in_slot_ctx;
1621	struct xhci_slot_ctx *out_slot_ctx;
1622
1623	in_slot_ctx = xhci_get_slot_ctx(xhci, ctx: in_ctx);
1624	out_slot_ctx = xhci_get_slot_ctx(xhci, ctx: out_ctx);
1625
1626	in_slot_ctx->dev_info = out_slot_ctx->dev_info;
1627	in_slot_ctx->dev_info2 = out_slot_ctx->dev_info2;
1628	in_slot_ctx->tt_info = out_slot_ctx->tt_info;
1629	in_slot_ctx->dev_state = out_slot_ctx->dev_state;
1630	}
1631
1632	/* Set up the scratchpad buffer array and scratchpad buffers, if needed. */
1633	static int scratchpad_alloc(struct xhci_hcd *xhci, gfp_t flags)
1634	{
1635	int i;
1636	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
1637	int num_sp = HCS_MAX_SCRATCHPAD(xhci->hcs_params2);
1638
1639	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_init,
1640	fmt: "Allocating %d scratchpad buffers", num_sp);
1641
1642	if (!num_sp)
1643	return 0;
1644
1645	xhci->scratchpad = kzalloc_node(size: sizeof(*xhci->scratchpad), flags,
1646	node: dev_to_node(dev));
1647	if (!xhci->scratchpad)
1648	goto fail_sp;
1649
1650	xhci->scratchpad->sp_array = dma_alloc_coherent(dev,
1651	size: size_mul(factor1: sizeof(u64), factor2: num_sp),
1652	dma_handle: &xhci->scratchpad->sp_dma, gfp: flags);
1653	if (!xhci->scratchpad->sp_array)
1654	goto fail_sp2;
1655
1656	xhci->scratchpad->sp_buffers = kcalloc_node(n: num_sp, size: sizeof(void *),
1657	flags, node: dev_to_node(dev));
1658	if (!xhci->scratchpad->sp_buffers)
1659	goto fail_sp3;
1660
1661	xhci->dcbaa->dev_context_ptrs[0] = cpu_to_le64(xhci->scratchpad->sp_dma);
1662	for (i = 0; i < num_sp; i++) {
1663	dma_addr_t dma;
1664	void *buf = dma_alloc_coherent(dev, size: xhci->page_size, dma_handle: &dma,
1665	gfp: flags);
1666	if (!buf)
1667	goto fail_sp4;
1668
1669	xhci->scratchpad->sp_array[i] = dma;
1670	xhci->scratchpad->sp_buffers[i] = buf;
1671	}
1672
1673	return 0;
1674
1675	fail_sp4:
1676	while (i--)
1677	dma_free_coherent(dev, size: xhci->page_size,
1678	cpu_addr: xhci->scratchpad->sp_buffers[i],
1679	dma_handle: xhci->scratchpad->sp_array[i]);
1680
1681	kfree(objp: xhci->scratchpad->sp_buffers);
1682
1683	fail_sp3:
1684	dma_free_coherent(dev, size: num_sp * sizeof(u64),
1685	cpu_addr: xhci->scratchpad->sp_array,
1686	dma_handle: xhci->scratchpad->sp_dma);
1687
1688	fail_sp2:
1689	kfree(objp: xhci->scratchpad);
1690	xhci->scratchpad = NULL;
1691
1692	fail_sp:
1693	return -ENOMEM;
1694	}
1695
1696	static void scratchpad_free(struct xhci_hcd *xhci)
1697	{
1698	int num_sp;
1699	int i;
1700	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
1701
1702	if (!xhci->scratchpad)
1703	return;
1704
1705	num_sp = HCS_MAX_SCRATCHPAD(xhci->hcs_params2);
1706
1707	for (i = 0; i < num_sp; i++) {
1708	dma_free_coherent(dev, size: xhci->page_size,
1709	cpu_addr: xhci->scratchpad->sp_buffers[i],
1710	dma_handle: xhci->scratchpad->sp_array[i]);
1711	}
1712	kfree(objp: xhci->scratchpad->sp_buffers);
1713	dma_free_coherent(dev, size: num_sp * sizeof(u64),
1714	cpu_addr: xhci->scratchpad->sp_array,
1715	dma_handle: xhci->scratchpad->sp_dma);
1716	kfree(objp: xhci->scratchpad);
1717	xhci->scratchpad = NULL;
1718	}
1719
1720	struct xhci_command *xhci_alloc_command(struct xhci_hcd *xhci,
1721	bool allocate_completion, gfp_t mem_flags)
1722	{
1723	struct xhci_command *command;
1724	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
1725
1726	command = kzalloc_node(size: sizeof(*command), flags: mem_flags, node: dev_to_node(dev));
1727	if (!command)
1728	return NULL;
1729
1730	if (allocate_completion) {
1731	command->completion =
1732	kzalloc_node(size: sizeof(struct completion), flags: mem_flags,
1733	node: dev_to_node(dev));
1734	if (!command->completion) {
1735	kfree(objp: command);
1736	return NULL;
1737	}
1738	init_completion(x: command->completion);
1739	}
1740
1741	command->status = 0;
1742	INIT_LIST_HEAD(list: &command->cmd_list);
1743	return command;
1744	}
1745
1746	struct xhci_command *xhci_alloc_command_with_ctx(struct xhci_hcd *xhci,
1747	bool allocate_completion, gfp_t mem_flags)
1748	{
1749	struct xhci_command *command;
1750
1751	command = xhci_alloc_command(xhci, allocate_completion, mem_flags);
1752	if (!command)
1753	return NULL;
1754
1755	command->in_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT,
1756	flags: mem_flags);
1757	if (!command->in_ctx) {
1758	kfree(objp: command->completion);
1759	kfree(objp: command);
1760	return NULL;
1761	}
1762	return command;
1763	}
1764
1765	void xhci_urb_free_priv(struct urb_priv *urb_priv)
1766	{
1767	kfree(objp: urb_priv);
1768	}
1769
1770	void xhci_free_command(struct xhci_hcd *xhci,
1771	struct xhci_command *command)
1772	{
1773	xhci_free_container_ctx(xhci,
1774	ctx: command->in_ctx);
1775	kfree(objp: command->completion);
1776	kfree(objp: command);
1777	}
1778
1779	static int xhci_alloc_erst(struct xhci_hcd *xhci,
1780	struct xhci_ring *evt_ring,
1781	struct xhci_erst *erst,
1782	gfp_t flags)
1783	{
1784	size_t size;
1785	unsigned int val;
1786	struct xhci_segment *seg;
1787	struct xhci_erst_entry *entry;
1788
1789	size = size_mul(factor1: sizeof(struct xhci_erst_entry), factor2: evt_ring->num_segs);
1790	erst->entries = dma_alloc_coherent(dev: xhci_to_hcd(xhci)->self.sysdev,
1791	size, dma_handle: &erst->erst_dma_addr, gfp: flags);
1792	if (!erst->entries)
1793	return -ENOMEM;
1794
1795	erst->num_entries = evt_ring->num_segs;
1796
1797	seg = evt_ring->first_seg;
1798	for (val = 0; val < evt_ring->num_segs; val++) {
1799	entry = &erst->entries[val];
1800	entry->seg_addr = cpu_to_le64(seg->dma);
1801	entry->seg_size = cpu_to_le32(TRBS_PER_SEGMENT);
1802	entry->rsvd = 0;
1803	seg = seg->next;
1804	}
1805
1806	return 0;
1807	}
1808
1809	static void
1810	xhci_remove_interrupter(struct xhci_hcd *xhci, struct xhci_interrupter *ir)
1811	{
1812	u32 tmp;
1813
1814	if (!ir)
1815	return;
1816
1817	/*
1818	* Clean out interrupter registers except ERSTBA. Clearing either the
1819	* low or high 32 bits of ERSTBA immediately causes the controller to
1820	* dereference the partially cleared 64 bit address, causing IOMMU error.
1821	*/
1822	if (ir->ir_set) {
1823	tmp = readl(addr: &ir->ir_set->erst_size);
1824	tmp &= ERST_SIZE_MASK;
1825	writel(val: tmp, addr: &ir->ir_set->erst_size);
1826
1827	xhci_write_64(xhci, ERST_EHB, regs: &ir->ir_set->erst_dequeue);
1828	}
1829	}
1830
1831	static void
1832	xhci_free_interrupter(struct xhci_hcd *xhci, struct xhci_interrupter *ir)
1833	{
1834	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
1835	size_t erst_size;
1836
1837	if (!ir)
1838	return;
1839
1840	erst_size = sizeof(struct xhci_erst_entry) * ir->erst.num_entries;
1841	if (ir->erst.entries)
1842	dma_free_coherent(dev, size: erst_size,
1843	cpu_addr: ir->erst.entries,
1844	dma_handle: ir->erst.erst_dma_addr);
1845	ir->erst.entries = NULL;
1846
1847	/* free interrupter event ring */
1848	if (ir->event_ring)
1849	xhci_ring_free(xhci, ring: ir->event_ring);
1850
1851	ir->event_ring = NULL;
1852
1853	kfree(objp: ir);
1854	}
1855
1856	void xhci_mem_cleanup(struct xhci_hcd *xhci)
1857	{
1858	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
1859	int i, j, num_ports;
1860
1861	cancel_delayed_work_sync(dwork: &xhci->cmd_timer);
1862
1863	xhci_remove_interrupter(xhci, ir: xhci->interrupter);
1864	xhci_free_interrupter(xhci, ir: xhci->interrupter);
1865	xhci->interrupter = NULL;
1866	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_init, fmt: "Freed primary event ring");
1867
1868	if (xhci->cmd_ring)
1869	xhci_ring_free(xhci, ring: xhci->cmd_ring);
1870	xhci->cmd_ring = NULL;
1871	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_init, fmt: "Freed command ring");
1872	xhci_cleanup_command_queue(xhci);
1873
1874	num_ports = HCS_MAX_PORTS(xhci->hcs_params1);
1875	for (i = 0; i < num_ports && xhci->rh_bw; i++) {
1876	struct xhci_interval_bw_table *bwt = &xhci->rh_bw[i].bw_table;
1877	for (j = 0; j < XHCI_MAX_INTERVAL; j++) {
1878	struct list_head *ep = &bwt->interval_bw[j].endpoints;
1879	while (!list_empty(head: ep))
1880	list_del_init(entry: ep->next);
1881	}
1882	}
1883
1884	for (i = HCS_MAX_SLOTS(xhci->hcs_params1); i > 0; i--)
1885	xhci_free_virt_devices_depth_first(xhci, slot_id: i);
1886
1887	dma_pool_destroy(pool: xhci->segment_pool);
1888	xhci->segment_pool = NULL;
1889	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_init, fmt: "Freed segment pool");
1890
1891	dma_pool_destroy(pool: xhci->device_pool);
1892	xhci->device_pool = NULL;
1893	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_init, fmt: "Freed device context pool");
1894
1895	dma_pool_destroy(pool: xhci->small_streams_pool);
1896	xhci->small_streams_pool = NULL;
1897	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_init,
1898	fmt: "Freed small stream array pool");
1899
1900	dma_pool_destroy(pool: xhci->medium_streams_pool);
1901	xhci->medium_streams_pool = NULL;
1902	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_init,
1903	fmt: "Freed medium stream array pool");
1904
1905	if (xhci->dcbaa)
1906	dma_free_coherent(dev, size: sizeof(*xhci->dcbaa),
1907	cpu_addr: xhci->dcbaa, dma_handle: xhci->dcbaa->dma);
1908	xhci->dcbaa = NULL;
1909
1910	scratchpad_free(xhci);
1911
1912	if (!xhci->rh_bw)
1913	goto no_bw;
1914
1915	for (i = 0; i < num_ports; i++) {
1916	struct xhci_tt_bw_info *tt, *n;
1917	list_for_each_entry_safe(tt, n, &xhci->rh_bw[i].tts, tt_list) {
1918	list_del(entry: &tt->tt_list);
1919	kfree(objp: tt);
1920	}
1921	}
1922
1923	no_bw:
1924	xhci->cmd_ring_reserved_trbs = 0;
1925	xhci->usb2_rhub.num_ports = 0;
1926	xhci->usb3_rhub.num_ports = 0;
1927	xhci->num_active_eps = 0;
1928	kfree(objp: xhci->usb2_rhub.ports);
1929	kfree(objp: xhci->usb3_rhub.ports);
1930	kfree(objp: xhci->hw_ports);
1931	kfree(objp: xhci->rh_bw);
1932	kfree(objp: xhci->ext_caps);
1933	for (i = 0; i < xhci->num_port_caps; i++)
1934	kfree(objp: xhci->port_caps[i].psi);
1935	kfree(objp: xhci->port_caps);
1936	xhci->num_port_caps = 0;
1937
1938	xhci->usb2_rhub.ports = NULL;
1939	xhci->usb3_rhub.ports = NULL;
1940	xhci->hw_ports = NULL;
1941	xhci->rh_bw = NULL;
1942	xhci->ext_caps = NULL;
1943	xhci->port_caps = NULL;
1944
1945	xhci->page_size = 0;
1946	xhci->page_shift = 0;
1947	xhci->usb2_rhub.bus_state.bus_suspended = 0;
1948	xhci->usb3_rhub.bus_state.bus_suspended = 0;
1949	}
1950
1951	static void xhci_set_hc_event_deq(struct xhci_hcd *xhci, struct xhci_interrupter *ir)
1952	{
1953	dma_addr_t deq;
1954
1955	deq = xhci_trb_virt_to_dma(seg: ir->event_ring->deq_seg,
1956	trb: ir->event_ring->dequeue);
1957	if (!deq)
1958	xhci_warn(xhci, "WARN something wrong with SW event ring dequeue ptr.\n");
1959	/* Update HC event ring dequeue pointer */
1960	/* Don't clear the EHB bit (which is RW1C) because
1961	* there might be more events to service.
1962	*/
1963	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_init,
1964	fmt: "// Write event ring dequeue pointer, preserving EHB bit");
1965	xhci_write_64(xhci, val: deq & ERST_PTR_MASK, regs: &ir->ir_set->erst_dequeue);
1966	}
1967
1968	static void xhci_add_in_port(struct xhci_hcd *xhci, unsigned int num_ports,
1969	__le32 __iomem *addr, int max_caps)
1970	{
1971	u32 temp, port_offset, port_count;
1972	int i;
1973	u8 major_revision, minor_revision, tmp_minor_revision;
1974	struct xhci_hub *rhub;
1975	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
1976	struct xhci_port_cap *port_cap;
1977
1978	temp = readl(addr);
1979	major_revision = XHCI_EXT_PORT_MAJOR(temp);
1980	minor_revision = XHCI_EXT_PORT_MINOR(temp);
1981
1982	if (major_revision == 0x03) {
1983	rhub = &xhci->usb3_rhub;
1984	/*
1985	* Some hosts incorrectly use sub-minor version for minor
1986	* version (i.e. 0x02 instead of 0x20 for bcdUSB 0x320 and 0x01
1987	* for bcdUSB 0x310). Since there is no USB release with sub
1988	* minor version 0x301 to 0x309, we can assume that they are
1989	* incorrect and fix it here.
1990	*/
1991	if (minor_revision > 0x00 && minor_revision < 0x10)
1992	minor_revision <<= 4;
1993	/*
1994	* Some zhaoxin's xHCI controller that follow usb3.1 spec
1995	* but only support Gen1.
1996	*/
1997	if (xhci->quirks & XHCI_ZHAOXIN_HOST) {
1998	tmp_minor_revision = minor_revision;
1999	minor_revision = 0;
2000	}
2001
2002	} else if (major_revision <= 0x02) {
2003	rhub = &xhci->usb2_rhub;
2004	} else {
2005	xhci_warn(xhci, "Ignoring unknown port speed, Ext Cap %p, revision = 0x%x\n",
2006	addr, major_revision);
2007	/* Ignoring port protocol we can't understand. FIXME */
2008	return;
2009	}
2010
2011	/* Port offset and count in the third dword, see section 7.2 */
2012	temp = readl(addr: addr + 2);
2013	port_offset = XHCI_EXT_PORT_OFF(temp);
2014	port_count = XHCI_EXT_PORT_COUNT(temp);
2015	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_init,
2016	fmt: "Ext Cap %p, port offset = %u, count = %u, revision = 0x%x",
2017	addr, port_offset, port_count, major_revision);
2018	/* Port count includes the current port offset */
2019	if (port_offset == 0 || (port_offset + port_count - 1) > num_ports)
2020	/* WTF? "Valid values are ‘1’ to MaxPorts" */
2021	return;
2022
2023	port_cap = &xhci->port_caps[xhci->num_port_caps++];
2024	if (xhci->num_port_caps > max_caps)
2025	return;
2026
2027	port_cap->psi_count = XHCI_EXT_PORT_PSIC(temp);
2028
2029	if (port_cap->psi_count) {
2030	port_cap->psi = kcalloc_node(n: port_cap->psi_count,
2031	size: sizeof(*port_cap->psi),
2032	GFP_KERNEL, node: dev_to_node(dev));
2033	if (!port_cap->psi)
2034	port_cap->psi_count = 0;
2035
2036	port_cap->psi_uid_count++;
2037	for (i = 0; i < port_cap->psi_count; i++) {
2038	port_cap->psi[i] = readl(addr: addr + 4 + i);
2039
2040	/* count unique ID values, two consecutive entries can
2041	* have the same ID if link is assymetric
2042	*/
2043	if (i && (XHCI_EXT_PORT_PSIV(port_cap->psi[i]) !=
2044	XHCI_EXT_PORT_PSIV(port_cap->psi[i - 1])))
2045	port_cap->psi_uid_count++;
2046
2047	if (xhci->quirks & XHCI_ZHAOXIN_HOST &&
2048	major_revision == 0x03 &&
2049	XHCI_EXT_PORT_PSIV(port_cap->psi[i]) >= 5)
2050	minor_revision = tmp_minor_revision;
2051
2052	xhci_dbg(xhci, "PSIV:%d PSIE:%d PLT:%d PFD:%d LP:%d PSIM:%d\n",
2053	XHCI_EXT_PORT_PSIV(port_cap->psi[i]),
2054	XHCI_EXT_PORT_PSIE(port_cap->psi[i]),
2055	XHCI_EXT_PORT_PLT(port_cap->psi[i]),
2056	XHCI_EXT_PORT_PFD(port_cap->psi[i]),
2057	XHCI_EXT_PORT_LP(port_cap->psi[i]),
2058	XHCI_EXT_PORT_PSIM(port_cap->psi[i]));
2059	}
2060	}
2061
2062	rhub->maj_rev = major_revision;
2063
2064	if (rhub->min_rev < minor_revision)
2065	rhub->min_rev = minor_revision;
2066
2067	port_cap->maj_rev = major_revision;
2068	port_cap->min_rev = minor_revision;
2069
2070	/* cache usb2 port capabilities */
2071	if (major_revision < 0x03 && xhci->num_ext_caps < max_caps)
2072	xhci->ext_caps[xhci->num_ext_caps++] = temp;
2073
2074	if ((xhci->hci_version >= 0x100) && (major_revision != 0x03) &&
2075	(temp & XHCI_HLC)) {
2076	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_init,
2077	fmt: "xHCI 1.0: support USB2 hardware lpm");
2078	xhci->hw_lpm_support = 1;
2079	}
2080
2081	port_offset--;
2082	for (i = port_offset; i < (port_offset + port_count); i++) {
2083	struct xhci_port *hw_port = &xhci->hw_ports[i];
2084	/* Duplicate entry. Ignore the port if the revisions differ. */
2085	if (hw_port->rhub) {
2086	xhci_warn(xhci, "Duplicate port entry, Ext Cap %p, port %u\n", addr, i);
2087	xhci_warn(xhci, "Port was marked as USB %u, duplicated as USB %u\n",
2088	hw_port->rhub->maj_rev, major_revision);
2089	/* Only adjust the roothub port counts if we haven't
2090	* found a similar duplicate.
2091	*/
2092	if (hw_port->rhub != rhub &&
2093	hw_port->hcd_portnum != DUPLICATE_ENTRY) {
2094	hw_port->rhub->num_ports--;
2095	hw_port->hcd_portnum = DUPLICATE_ENTRY;
2096	}
2097	continue;
2098	}
2099	hw_port->rhub = rhub;
2100	hw_port->port_cap = port_cap;
2101	rhub->num_ports++;
2102	}
2103	/* FIXME: Should we disable ports not in the Extended Capabilities? */
2104	}
2105
2106	static void xhci_create_rhub_port_array(struct xhci_hcd *xhci,
2107	struct xhci_hub *rhub, gfp_t flags)
2108	{
2109	int port_index = 0;
2110	int i;
2111	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
2112
2113	if (!rhub->num_ports)
2114	return;
2115	rhub->ports = kcalloc_node(n: rhub->num_ports, size: sizeof(*rhub->ports),
2116	flags, node: dev_to_node(dev));
2117	if (!rhub->ports)
2118	return;
2119
2120	for (i = 0; i < HCS_MAX_PORTS(xhci->hcs_params1); i++) {
2121	if (xhci->hw_ports[i].rhub != rhub ||
2122	xhci->hw_ports[i].hcd_portnum == DUPLICATE_ENTRY)
2123	continue;
2124	xhci->hw_ports[i].hcd_portnum = port_index;
2125	rhub->ports[port_index] = &xhci->hw_ports[i];
2126	port_index++;
2127	if (port_index == rhub->num_ports)
2128	break;
2129	}
2130	}
2131
2132	/*
2133	* Scan the Extended Capabilities for the "Supported Protocol Capabilities" that
2134	* specify what speeds each port is supposed to be. We can't count on the port
2135	* speed bits in the PORTSC register being correct until a device is connected,
2136	* but we need to set up the two fake roothubs with the correct number of USB
2137	* 3.0 and USB 2.0 ports at host controller initialization time.
2138	*/
2139	static int xhci_setup_port_arrays(struct xhci_hcd *xhci, gfp_t flags)
2140	{
2141	void __iomem *base;
2142	u32 offset;
2143	unsigned int num_ports;
2144	int i, j;
2145	int cap_count = 0;
2146	u32 cap_start;
2147	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
2148
2149	num_ports = HCS_MAX_PORTS(xhci->hcs_params1);
2150	xhci->hw_ports = kcalloc_node(n: num_ports, size: sizeof(*xhci->hw_ports),
2151	flags, node: dev_to_node(dev));
2152	if (!xhci->hw_ports)
2153	return -ENOMEM;
2154
2155	for (i = 0; i < num_ports; i++) {
2156	xhci->hw_ports[i].addr = &xhci->op_regs->port_status_base +
2157	NUM_PORT_REGS * i;
2158	xhci->hw_ports[i].hw_portnum = i;
2159
2160	init_completion(x: &xhci->hw_ports[i].rexit_done);
2161	init_completion(x: &xhci->hw_ports[i].u3exit_done);
2162	}
2163
2164	xhci->rh_bw = kcalloc_node(n: num_ports, size: sizeof(*xhci->rh_bw), flags,
2165	node: dev_to_node(dev));
2166	if (!xhci->rh_bw)
2167	return -ENOMEM;
2168	for (i = 0; i < num_ports; i++) {
2169	struct xhci_interval_bw_table *bw_table;
2170
2171	INIT_LIST_HEAD(list: &xhci->rh_bw[i].tts);
2172	bw_table = &xhci->rh_bw[i].bw_table;
2173	for (j = 0; j < XHCI_MAX_INTERVAL; j++)
2174	INIT_LIST_HEAD(list: &bw_table->interval_bw[j].endpoints);
2175	}
2176	base = &xhci->cap_regs->hc_capbase;
2177
2178	cap_start = xhci_find_next_ext_cap(base, start: 0, XHCI_EXT_CAPS_PROTOCOL);
2179	if (!cap_start) {
2180	xhci_err(xhci, "No Extended Capability registers, unable to set up roothub\n");
2181	return -ENODEV;
2182	}
2183
2184	offset = cap_start;
2185	/* count extended protocol capability entries for later caching */
2186	while (offset) {
2187	cap_count++;
2188	offset = xhci_find_next_ext_cap(base, start: offset,
2189	XHCI_EXT_CAPS_PROTOCOL);
2190	}
2191
2192	xhci->ext_caps = kcalloc_node(n: cap_count, size: sizeof(*xhci->ext_caps),
2193	flags, node: dev_to_node(dev));
2194	if (!xhci->ext_caps)
2195	return -ENOMEM;
2196
2197	xhci->port_caps = kcalloc_node(n: cap_count, size: sizeof(*xhci->port_caps),
2198	flags, node: dev_to_node(dev));
2199	if (!xhci->port_caps)
2200	return -ENOMEM;
2201
2202	offset = cap_start;
2203
2204	while (offset) {
2205	xhci_add_in_port(xhci, num_ports, addr: base + offset, max_caps: cap_count);
2206	if (xhci->usb2_rhub.num_ports + xhci->usb3_rhub.num_ports ==
2207	num_ports)
2208	break;
2209	offset = xhci_find_next_ext_cap(base, start: offset,
2210	XHCI_EXT_CAPS_PROTOCOL);
2211	}
2212	if (xhci->usb2_rhub.num_ports == 0 && xhci->usb3_rhub.num_ports == 0) {
2213	xhci_warn(xhci, "No ports on the roothubs?\n");
2214	return -ENODEV;
2215	}
2216	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_init,
2217	fmt: "Found %u USB 2.0 ports and %u USB 3.0 ports.",
2218	xhci->usb2_rhub.num_ports, xhci->usb3_rhub.num_ports);
2219
2220	/* Place limits on the number of roothub ports so that the hub
2221	* descriptors aren't longer than the USB core will allocate.
2222	*/
2223	if (xhci->usb3_rhub.num_ports > USB_SS_MAXPORTS) {
2224	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_init,
2225	fmt: "Limiting USB 3.0 roothub ports to %u.",
2226	USB_SS_MAXPORTS);
2227	xhci->usb3_rhub.num_ports = USB_SS_MAXPORTS;
2228	}
2229	if (xhci->usb2_rhub.num_ports > USB_MAXCHILDREN) {
2230	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_init,
2231	fmt: "Limiting USB 2.0 roothub ports to %u.",
2232	USB_MAXCHILDREN);
2233	xhci->usb2_rhub.num_ports = USB_MAXCHILDREN;
2234	}
2235
2236	if (!xhci->usb2_rhub.num_ports)
2237	xhci_info(xhci, "USB2 root hub has no ports\n");
2238
2239	if (!xhci->usb3_rhub.num_ports)
2240	xhci_info(xhci, "USB3 root hub has no ports\n");
2241
2242	xhci_create_rhub_port_array(xhci, rhub: &xhci->usb2_rhub, flags);
2243	xhci_create_rhub_port_array(xhci, rhub: &xhci->usb3_rhub, flags);
2244
2245	return 0;
2246	}
2247
2248	static struct xhci_interrupter *
2249	xhci_alloc_interrupter(struct xhci_hcd *xhci, gfp_t flags)
2250	{
2251	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
2252	struct xhci_interrupter *ir;
2253	unsigned int num_segs;
2254	int ret;
2255
2256	ir = kzalloc_node(size: sizeof(*ir), flags, node: dev_to_node(dev));
2257	if (!ir)
2258	return NULL;
2259
2260	num_segs = min_t(unsigned int, 1 << HCS_ERST_MAX(xhci->hcs_params2),
2261	ERST_MAX_SEGS);
2262
2263	ir->event_ring = xhci_ring_alloc(xhci, num_segs, cycle_state: 1, type: TYPE_EVENT, max_packet: 0,
2264	flags);
2265	if (!ir->event_ring) {
2266	xhci_warn(xhci, "Failed to allocate interrupter event ring\n");
2267	kfree(objp: ir);
2268	return NULL;
2269	}
2270
2271	ret = xhci_alloc_erst(xhci, evt_ring: ir->event_ring, erst: &ir->erst, flags);
2272	if (ret) {
2273	xhci_warn(xhci, "Failed to allocate interrupter erst\n");
2274	xhci_ring_free(xhci, ring: ir->event_ring);
2275	kfree(objp: ir);
2276	return NULL;
2277	}
2278
2279	return ir;
2280	}
2281
2282	static int
2283	xhci_add_interrupter(struct xhci_hcd *xhci, struct xhci_interrupter *ir,
2284	unsigned int intr_num)
2285	{
2286	u64 erst_base;
2287	u32 erst_size;
2288
2289	if (intr_num > xhci->max_interrupters) {
2290	xhci_warn(xhci, "Can't add interrupter %d, max interrupters %d\n",
2291	intr_num, xhci->max_interrupters);
2292	return -EINVAL;
2293	}
2294
2295	ir->ir_set = &xhci->run_regs->ir_set[intr_num];
2296
2297	/* set ERST count with the number of entries in the segment table */
2298	erst_size = readl(addr: &ir->ir_set->erst_size);
2299	erst_size &= ERST_SIZE_MASK;
2300	erst_size |= ir->event_ring->num_segs;
2301	writel(val: erst_size, addr: &ir->ir_set->erst_size);
2302
2303	erst_base = xhci_read_64(xhci, regs: &ir->ir_set->erst_base);
2304	erst_base &= ERST_BASE_RSVDP;
2305	erst_base |= ir->erst.erst_dma_addr & ~ERST_BASE_RSVDP;
2306	xhci_write_64(xhci, val: erst_base, regs: &ir->ir_set->erst_base);
2307
2308	/* Set the event ring dequeue address of this interrupter */
2309	xhci_set_hc_event_deq(xhci, ir);
2310
2311	return 0;
2312	}
2313
2314	int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags)
2315	{
2316	dma_addr_t dma;
2317	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
2318	unsigned int val, val2;
2319	u64 val_64;
2320	u32 page_size, temp;
2321	int i;
2322
2323	INIT_LIST_HEAD(list: &xhci->cmd_list);
2324
2325	/* init command timeout work */
2326	INIT_DELAYED_WORK(&xhci->cmd_timer, xhci_handle_command_timeout);
2327	init_completion(x: &xhci->cmd_ring_stop_completion);
2328
2329	page_size = readl(addr: &xhci->op_regs->page_size);
2330	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_init,
2331	fmt: "Supported page size register = 0x%x", page_size);
2332	i = ffs(page_size);
2333	if (i < 16)
2334	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_init,
2335	fmt: "Supported page size of %iK", (1 << (i+12)) / 1024);
2336	else
2337	xhci_warn(xhci, "WARN: no supported page size\n");
2338	/* Use 4K pages, since that's common and the minimum the HC supports */
2339	xhci->page_shift = 12;
2340	xhci->page_size = 1 << xhci->page_shift;
2341	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_init,
2342	fmt: "HCD page size set to %iK", xhci->page_size / 1024);
2343
2344	/*
2345	* Program the Number of Device Slots Enabled field in the CONFIG
2346	* register with the max value of slots the HC can handle.
2347	*/
2348	val = HCS_MAX_SLOTS(readl(&xhci->cap_regs->hcs_params1));
2349	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_init,
2350	fmt: "// xHC can handle at most %d device slots.", val);
2351	val2 = readl(addr: &xhci->op_regs->config_reg);
2352	val |= (val2 & ~HCS_SLOTS_MASK);
2353	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_init,
2354	fmt: "// Setting Max device slots reg = 0x%x.", val);
2355	writel(val, addr: &xhci->op_regs->config_reg);
2356
2357	/*
2358	* xHCI section 5.4.6 - Device Context array must be
2359	* "physically contiguous and 64-byte (cache line) aligned".
2360	*/
2361	xhci->dcbaa = dma_alloc_coherent(dev, size: sizeof(*xhci->dcbaa), dma_handle: &dma,
2362	gfp: flags);
2363	if (!xhci->dcbaa)
2364	goto fail;
2365	xhci->dcbaa->dma = dma;
2366	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_init,
2367	fmt: "// Device context base array address = 0x%pad (DMA), %p (virt)",
2368	&xhci->dcbaa->dma, xhci->dcbaa);
2369	xhci_write_64(xhci, val: dma, regs: &xhci->op_regs->dcbaa_ptr);
2370
2371	/*
2372	* Initialize the ring segment pool. The ring must be a contiguous
2373	* structure comprised of TRBs. The TRBs must be 16 byte aligned,
2374	* however, the command ring segment needs 64-byte aligned segments
2375	* and our use of dma addresses in the trb_address_map radix tree needs
2376	* TRB_SEGMENT_SIZE alignment, so we pick the greater alignment need.
2377	*/
2378	if (xhci->quirks & XHCI_ZHAOXIN_TRB_FETCH)
2379	xhci->segment_pool = dma_pool_create(name: "xHCI ring segments", dev,
2380	TRB_SEGMENT_SIZE * 2, TRB_SEGMENT_SIZE * 2, allocation: xhci->page_size * 2);
2381	else
2382	xhci->segment_pool = dma_pool_create(name: "xHCI ring segments", dev,
2383	TRB_SEGMENT_SIZE, TRB_SEGMENT_SIZE, allocation: xhci->page_size);
2384
2385	/* See Table 46 and Note on Figure 55 */
2386	xhci->device_pool = dma_pool_create(name: "xHCI input/output contexts", dev,
2387	size: 2112, align: 64, allocation: xhci->page_size);
2388	if (!xhci->segment_pool || !xhci->device_pool)
2389	goto fail;
2390
2391	/* Linear stream context arrays don't have any boundary restrictions,
2392	* and only need to be 16-byte aligned.
2393	*/
2394	xhci->small_streams_pool =
2395	dma_pool_create(name: "xHCI 256 byte stream ctx arrays",
2396	dev, SMALL_STREAM_ARRAY_SIZE, align: 16, allocation: 0);
2397	xhci->medium_streams_pool =
2398	dma_pool_create(name: "xHCI 1KB stream ctx arrays",
2399	dev, MEDIUM_STREAM_ARRAY_SIZE, align: 16, allocation: 0);
2400	/* Any stream context array bigger than MEDIUM_STREAM_ARRAY_SIZE
2401	* will be allocated with dma_alloc_coherent()
2402	*/
2403
2404	if (!xhci->small_streams_pool || !xhci->medium_streams_pool)
2405	goto fail;
2406
2407	/* Set up the command ring to have one segments for now. */
2408	xhci->cmd_ring = xhci_ring_alloc(xhci, num_segs: 1, cycle_state: 1, type: TYPE_COMMAND, max_packet: 0, flags);
2409	if (!xhci->cmd_ring)
2410	goto fail;
2411	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_init,
2412	fmt: "Allocated command ring at %p", xhci->cmd_ring);
2413	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_init, fmt: "First segment DMA is 0x%pad",
2414	&xhci->cmd_ring->first_seg->dma);
2415
2416	/* Set the address in the Command Ring Control register */
2417	val_64 = xhci_read_64(xhci, regs: &xhci->op_regs->cmd_ring);
2418	val_64 = (val_64 & (u64) CMD_RING_RSVD_BITS) |
2419	(xhci->cmd_ring->first_seg->dma & (u64) ~CMD_RING_RSVD_BITS) |
2420	xhci->cmd_ring->cycle_state;
2421	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_init,
2422	fmt: "// Setting command ring address to 0x%016llx", val_64);
2423	xhci_write_64(xhci, val: val_64, regs: &xhci->op_regs->cmd_ring);
2424
2425	/* Reserve one command ring TRB for disabling LPM.
2426	* Since the USB core grabs the shared usb_bus bandwidth mutex before
2427	* disabling LPM, we only need to reserve one TRB for all devices.
2428	*/
2429	xhci->cmd_ring_reserved_trbs++;
2430
2431	val = readl(addr: &xhci->cap_regs->db_off);
2432	val &= DBOFF_MASK;
2433	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_init,
2434	fmt: "// Doorbell array is located at offset 0x%x from cap regs base addr",
2435	val);
2436	xhci->dba = (void __iomem *) xhci->cap_regs + val;
2437
2438	/* Allocate and set up primary interrupter 0 with an event ring. */
2439	xhci_dbg_trace(xhci, trace: trace_xhci_dbg_init,
2440	fmt: "Allocating primary event ring");
2441	xhci->interrupter = xhci_alloc_interrupter(xhci, flags);
2442	if (!xhci->interrupter)
2443	goto fail;
2444
2445	if (xhci_add_interrupter(xhci, ir: xhci->interrupter, intr_num: 0))
2446	goto fail;
2447
2448	xhci->isoc_bei_interval = AVOID_BEI_INTERVAL_MAX;
2449
2450	/*
2451	* XXX: Might need to set the Interrupter Moderation Register to
2452	* something other than the default (~1ms minimum between interrupts).
2453	* See section 5.5.1.2.
2454	*/
2455	for (i = 0; i < MAX_HC_SLOTS; i++)
2456	xhci->devs[i] = NULL;
2457
2458	if (scratchpad_alloc(xhci, flags))
2459	goto fail;
2460	if (xhci_setup_port_arrays(xhci, flags))
2461	goto fail;
2462
2463	/* Enable USB 3.0 device notifications for function remote wake, which
2464	* is necessary for allowing USB 3.0 devices to do remote wakeup from
2465	* U3 (device suspend).
2466	*/
2467	temp = readl(addr: &xhci->op_regs->dev_notification);
2468	temp &= ~DEV_NOTE_MASK;
2469	temp |= DEV_NOTE_FWAKE;
2470	writel(val: temp, addr: &xhci->op_regs->dev_notification);
2471
2472	return 0;
2473
2474	fail:
2475	xhci_halt(xhci);
2476	xhci_reset(xhci, XHCI_RESET_SHORT_USEC);
2477	xhci_mem_cleanup(xhci);
2478	return -ENOMEM;
2479	}
2480