1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* PCI detection and setup code
4	*/
5
6	#include <linux/kernel.h>
7	#include <linux/delay.h>
8	#include <linux/init.h>
9	#include <linux/pci.h>
10	#include <linux/msi.h>
11	#include <linux/of_pci.h>
12	#include <linux/pci_hotplug.h>
13	#include <linux/slab.h>
14	#include <linux/module.h>
15	#include <linux/cpumask.h>
16	#include <linux/aer.h>
17	#include <linux/acpi.h>
18	#include <linux/hypervisor.h>
19	#include <linux/irqdomain.h>
20	#include <linux/pm_runtime.h>
21	#include <linux/bitfield.h>
22	#include "pci.h"
23
24	#define CARDBUS_LATENCY_TIMER 176 /* secondary latency timer */
25	#define CARDBUS_RESERVE_BUSNR 3
26
27	static struct resource busn_resource = {
28	.name = "PCI busn",
29	.start = 0,
30	.end = 255,
31	.flags = IORESOURCE_BUS,
32	};
33
34	/* Ugh. Need to stop exporting this to modules. */
35	LIST_HEAD(pci_root_buses);
36	EXPORT_SYMBOL(pci_root_buses);
37
38	static LIST_HEAD(pci_domain_busn_res_list);
39
40	struct pci_domain_busn_res {
41	struct list_head list;
42	struct resource res;
43	int domain_nr;
44	};
45
46	static struct resource *get_pci_domain_busn_res(int domain_nr)
47	{
48	struct pci_domain_busn_res *r;
49
50	list_for_each_entry(r, &pci_domain_busn_res_list, list)
51	if (r->domain_nr == domain_nr)
52	return &r->res;
53
54	r = kzalloc(size: sizeof(*r), GFP_KERNEL);
55	if (!r)
56	return NULL;
57
58	r->domain_nr = domain_nr;
59	r->res.start = 0;
60	r->res.end = 0xff;
61	r->res.flags = IORESOURCE_BUS | IORESOURCE_PCI_FIXED;
62
63	list_add_tail(new: &r->list, head: &pci_domain_busn_res_list);
64
65	return &r->res;
66	}
67
68	/*
69	* Some device drivers need know if PCI is initiated.
70	* Basically, we think PCI is not initiated when there
71	* is no device to be found on the pci_bus_type.
72	*/
73	int no_pci_devices(void)
74	{
75	struct device *dev;
76	int no_devices;
77
78	dev = bus_find_next_device(bus: &pci_bus_type, NULL);
79	no_devices = (dev == NULL);
80	put_device(dev);
81	return no_devices;
82	}
83	EXPORT_SYMBOL(no_pci_devices);
84
85	/*
86	* PCI Bus Class
87	*/
88	static void release_pcibus_dev(struct device *dev)
89	{
90	struct pci_bus *pci_bus = to_pci_bus(dev);
91
92	put_device(dev: pci_bus->bridge);
93	pci_bus_remove_resources(bus: pci_bus);
94	pci_release_bus_of_node(bus: pci_bus);
95	kfree(objp: pci_bus);
96	}
97
98	static struct class pcibus_class = {
99	.name = "pci_bus",
100	.dev_release = &release_pcibus_dev,
101	.dev_groups = pcibus_groups,
102	};
103
104	static int __init pcibus_class_init(void)
105	{
106	return class_register(class: &pcibus_class);
107	}
108	postcore_initcall(pcibus_class_init);
109
110	static u64 pci_size(u64 base, u64 maxbase, u64 mask)
111	{
112	u64 size = mask & maxbase; /* Find the significant bits */
113	if (!size)
114	return 0;
115
116	/*
117	* Get the lowest of them to find the decode size, and from that
118	* the extent.
119	*/
120	size = size & ~(size-1);
121
122	/*
123	* base == maxbase can be valid only if the BAR has already been
124	* programmed with all 1s.
125	*/
126	if (base == maxbase && ((base | (size - 1)) & mask) != mask)
127	return 0;
128
129	return size;
130	}
131
132	static inline unsigned long decode_bar(struct pci_dev *dev, u32 bar)
133	{
134	u32 mem_type;
135	unsigned long flags;
136
137	if ((bar & PCI_BASE_ADDRESS_SPACE) == PCI_BASE_ADDRESS_SPACE_IO) {
138	flags = bar & ~PCI_BASE_ADDRESS_IO_MASK;
139	flags |= IORESOURCE_IO;
140	return flags;
141	}
142
143	flags = bar & ~PCI_BASE_ADDRESS_MEM_MASK;
144	flags |= IORESOURCE_MEM;
145	if (flags & PCI_BASE_ADDRESS_MEM_PREFETCH)
146	flags |= IORESOURCE_PREFETCH;
147
148	mem_type = bar & PCI_BASE_ADDRESS_MEM_TYPE_MASK;
149	switch (mem_type) {
150	case PCI_BASE_ADDRESS_MEM_TYPE_32:
151	break;
152	case PCI_BASE_ADDRESS_MEM_TYPE_1M:
153	/* 1M mem BAR treated as 32-bit BAR */
154	break;
155	case PCI_BASE_ADDRESS_MEM_TYPE_64:
156	flags |= IORESOURCE_MEM_64;
157	break;
158	default:
159	/* mem unknown type treated as 32-bit BAR */
160	break;
161	}
162	return flags;
163	}
164
165	#define PCI_COMMAND_DECODE_ENABLE (PCI_COMMAND_MEMORY | PCI_COMMAND_IO)
166
167	/**
168	* __pci_read_base - Read a PCI BAR
169	* @dev: the PCI device
170	* @type: type of the BAR
171	* @res: resource buffer to be filled in
172	* @pos: BAR position in the config space
173	*
174	* Returns 1 if the BAR is 64-bit, or 0 if 32-bit.
175	*/
176	int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type,
177	struct resource *res, unsigned int pos)
178	{
179	u32 l = 0, sz = 0, mask;
180	u64 l64, sz64, mask64;
181	u16 orig_cmd;
182	struct pci_bus_region region, inverted_region;
183
184	mask = type ? PCI_ROM_ADDRESS_MASK : ~0;
185
186	/* No printks while decoding is disabled! */
187	if (!dev->mmio_always_on) {
188	pci_read_config_word(dev, PCI_COMMAND, val: &orig_cmd);
189	if (orig_cmd & PCI_COMMAND_DECODE_ENABLE) {
190	pci_write_config_word(dev, PCI_COMMAND,
191	val: orig_cmd & ~PCI_COMMAND_DECODE_ENABLE);
192	}
193	}
194
195	res->name = pci_name(pdev: dev);
196
197	pci_read_config_dword(dev, where: pos, val: &l);
198	pci_write_config_dword(dev, where: pos, val: l | mask);
199	pci_read_config_dword(dev, where: pos, val: &sz);
200	pci_write_config_dword(dev, where: pos, val: l);
201
202	/*
203	* All bits set in sz means the device isn't working properly.
204	* If the BAR isn't implemented, all bits must be 0. If it's a
205	* memory BAR or a ROM, bit 0 must be clear; if it's an io BAR, bit
206	* 1 must be clear.
207	*/
208	if (PCI_POSSIBLE_ERROR(sz))
209	sz = 0;
210
211	/*
212	* I don't know how l can have all bits set. Copied from old code.
213	* Maybe it fixes a bug on some ancient platform.
214	*/
215	if (PCI_POSSIBLE_ERROR(l))
216	l = 0;
217
218	if (type == pci_bar_unknown) {
219	res->flags = decode_bar(dev, bar: l);
220	res->flags |= IORESOURCE_SIZEALIGN;
221	if (res->flags & IORESOURCE_IO) {
222	l64 = l & PCI_BASE_ADDRESS_IO_MASK;
223	sz64 = sz & PCI_BASE_ADDRESS_IO_MASK;
224	mask64 = PCI_BASE_ADDRESS_IO_MASK & (u32)IO_SPACE_LIMIT;
225	} else {
226	l64 = l & PCI_BASE_ADDRESS_MEM_MASK;
227	sz64 = sz & PCI_BASE_ADDRESS_MEM_MASK;
228	mask64 = (u32)PCI_BASE_ADDRESS_MEM_MASK;
229	}
230	} else {
231	if (l & PCI_ROM_ADDRESS_ENABLE)
232	res->flags |= IORESOURCE_ROM_ENABLE;
233	l64 = l & PCI_ROM_ADDRESS_MASK;
234	sz64 = sz & PCI_ROM_ADDRESS_MASK;
235	mask64 = PCI_ROM_ADDRESS_MASK;
236	}
237
238	if (res->flags & IORESOURCE_MEM_64) {
239	pci_read_config_dword(dev, where: pos + 4, val: &l);
240	pci_write_config_dword(dev, where: pos + 4, val: ~0);
241	pci_read_config_dword(dev, where: pos + 4, val: &sz);
242	pci_write_config_dword(dev, where: pos + 4, val: l);
243
244	l64 |= ((u64)l << 32);
245	sz64 |= ((u64)sz << 32);
246	mask64 |= ((u64)~0 << 32);
247	}
248
249	if (!dev->mmio_always_on && (orig_cmd & PCI_COMMAND_DECODE_ENABLE))
250	pci_write_config_word(dev, PCI_COMMAND, val: orig_cmd);
251
252	if (!sz64)
253	goto fail;
254
255	sz64 = pci_size(base: l64, maxbase: sz64, mask: mask64);
256	if (!sz64) {
257	pci_info(dev, FW_BUG "reg 0x%x: invalid BAR (can't size)\n",
258	pos);
259	goto fail;
260	}
261
262	if (res->flags & IORESOURCE_MEM_64) {
263	if ((sizeof(pci_bus_addr_t) < 8 || sizeof(resource_size_t) < 8)
264	&& sz64 > 0x100000000ULL) {
265	res->flags |= IORESOURCE_UNSET | IORESOURCE_DISABLED;
266	res->start = 0;
267	res->end = 0;
268	pci_err(dev, "reg 0x%x: can't handle BAR larger than 4GB (size %#010llx)\n",
269	pos, (unsigned long long)sz64);
270	goto out;
271	}
272
273	if ((sizeof(pci_bus_addr_t) < 8) && l) {
274	/* Above 32-bit boundary; try to reallocate */
275	res->flags |= IORESOURCE_UNSET;
276	res->start = 0;
277	res->end = sz64 - 1;
278	pci_info(dev, "reg 0x%x: can't handle BAR above 4GB (bus address %#010llx)\n",
279	pos, (unsigned long long)l64);
280	goto out;
281	}
282	}
283
284	region.start = l64;
285	region.end = l64 + sz64 - 1;
286
287	pcibios_bus_to_resource(bus: dev->bus, res, region: &region);
288	pcibios_resource_to_bus(bus: dev->bus, region: &inverted_region, res);
289
290	/*
291	* If "A" is a BAR value (a bus address), "bus_to_resource(A)" is
292	* the corresponding resource address (the physical address used by
293	* the CPU. Converting that resource address back to a bus address
294	* should yield the original BAR value:
295	*
296	* resource_to_bus(bus_to_resource(A)) == A
297	*
298	* If it doesn't, CPU accesses to "bus_to_resource(A)" will not
299	* be claimed by the device.
300	*/
301	if (inverted_region.start != region.start) {
302	res->flags |= IORESOURCE_UNSET;
303	res->start = 0;
304	res->end = region.end - region.start;
305	pci_info(dev, "reg 0x%x: initial BAR value %#010llx invalid\n",
306	pos, (unsigned long long)region.start);
307	}
308
309	goto out;
310
311
312	fail:
313	res->flags = 0;
314	out:
315	if (res->flags)
316	pci_info(dev, "reg 0x%x: %pR\n", pos, res);
317
318	return (res->flags & IORESOURCE_MEM_64) ? 1 : 0;
319	}
320
321	static void pci_read_bases(struct pci_dev *dev, unsigned int howmany, int rom)
322	{
323	unsigned int pos, reg;
324
325	if (dev->non_compliant_bars)
326	return;
327
328	/* Per PCIe r4.0, sec 9.3.4.1.11, the VF BARs are all RO Zero */
329	if (dev->is_virtfn)
330	return;
331
332	for (pos = 0; pos < howmany; pos++) {
333	struct resource *res = &dev->resource[pos];
334	reg = PCI_BASE_ADDRESS_0 + (pos << 2);
335	pos += __pci_read_base(dev, type: pci_bar_unknown, res, pos: reg);
336	}
337
338	if (rom) {
339	struct resource *res = &dev->resource[PCI_ROM_RESOURCE];
340	dev->rom_base_reg = rom;
341	res->flags = IORESOURCE_MEM | IORESOURCE_PREFETCH |
342	IORESOURCE_READONLY | IORESOURCE_SIZEALIGN;
343	__pci_read_base(dev, type: pci_bar_mem32, res, pos: rom);
344	}
345	}
346
347	static void pci_read_bridge_windows(struct pci_dev *bridge)
348	{
349	u16 io;
350	u32 pmem, tmp;
351
352	pci_read_config_word(dev: bridge, PCI_IO_BASE, val: &io);
353	if (!io) {
354	pci_write_config_word(dev: bridge, PCI_IO_BASE, val: 0xe0f0);
355	pci_read_config_word(dev: bridge, PCI_IO_BASE, val: &io);
356	pci_write_config_word(dev: bridge, PCI_IO_BASE, val: 0x0);
357	}
358	if (io)
359	bridge->io_window = 1;
360
361	/*
362	* DECchip 21050 pass 2 errata: the bridge may miss an address
363	* disconnect boundary by one PCI data phase. Workaround: do not
364	* use prefetching on this device.
365	*/
366	if (bridge->vendor == PCI_VENDOR_ID_DEC && bridge->device == 0x0001)
367	return;
368
369	pci_read_config_dword(dev: bridge, PCI_PREF_MEMORY_BASE, val: &pmem);
370	if (!pmem) {
371	pci_write_config_dword(dev: bridge, PCI_PREF_MEMORY_BASE,
372	val: 0xffe0fff0);
373	pci_read_config_dword(dev: bridge, PCI_PREF_MEMORY_BASE, val: &pmem);
374	pci_write_config_dword(dev: bridge, PCI_PREF_MEMORY_BASE, val: 0x0);
375	}
376	if (!pmem)
377	return;
378
379	bridge->pref_window = 1;
380
381	if ((pmem & PCI_PREF_RANGE_TYPE_MASK) == PCI_PREF_RANGE_TYPE_64) {
382
383	/*
384	* Bridge claims to have a 64-bit prefetchable memory
385	* window; verify that the upper bits are actually
386	* writable.
387	*/
388	pci_read_config_dword(dev: bridge, PCI_PREF_BASE_UPPER32, val: &pmem);
389	pci_write_config_dword(dev: bridge, PCI_PREF_BASE_UPPER32,
390	val: 0xffffffff);
391	pci_read_config_dword(dev: bridge, PCI_PREF_BASE_UPPER32, val: &tmp);
392	pci_write_config_dword(dev: bridge, PCI_PREF_BASE_UPPER32, val: pmem);
393	if (tmp)
394	bridge->pref_64_window = 1;
395	}
396	}
397
398	static void pci_read_bridge_io(struct pci_bus *child)
399	{
400	struct pci_dev *dev = child->self;
401	u8 io_base_lo, io_limit_lo;
402	unsigned long io_mask, io_granularity, base, limit;
403	struct pci_bus_region region;
404	struct resource *res;
405
406	io_mask = PCI_IO_RANGE_MASK;
407	io_granularity = 0x1000;
408	if (dev->io_window_1k) {
409	/* Support 1K I/O space granularity */
410	io_mask = PCI_IO_1K_RANGE_MASK;
411	io_granularity = 0x400;
412	}
413
414	res = child->resource[0];
415	pci_read_config_byte(dev, PCI_IO_BASE, val: &io_base_lo);
416	pci_read_config_byte(dev, PCI_IO_LIMIT, val: &io_limit_lo);
417	base = (io_base_lo & io_mask) << 8;
418	limit = (io_limit_lo & io_mask) << 8;
419
420	if ((io_base_lo & PCI_IO_RANGE_TYPE_MASK) == PCI_IO_RANGE_TYPE_32) {
421	u16 io_base_hi, io_limit_hi;
422
423	pci_read_config_word(dev, PCI_IO_BASE_UPPER16, val: &io_base_hi);
424	pci_read_config_word(dev, PCI_IO_LIMIT_UPPER16, val: &io_limit_hi);
425	base |= ((unsigned long) io_base_hi << 16);
426	limit |= ((unsigned long) io_limit_hi << 16);
427	}
428
429	if (base <= limit) {
430	res->flags = (io_base_lo & PCI_IO_RANGE_TYPE_MASK) | IORESOURCE_IO;
431	region.start = base;
432	region.end = limit + io_granularity - 1;
433	pcibios_bus_to_resource(bus: dev->bus, res, region: &region);
434	pci_info(dev, " bridge window %pR\n", res);
435	}
436	}
437
438	static void pci_read_bridge_mmio(struct pci_bus *child)
439	{
440	struct pci_dev *dev = child->self;
441	u16 mem_base_lo, mem_limit_lo;
442	unsigned long base, limit;
443	struct pci_bus_region region;
444	struct resource *res;
445
446	res = child->resource[1];
447	pci_read_config_word(dev, PCI_MEMORY_BASE, val: &mem_base_lo);
448	pci_read_config_word(dev, PCI_MEMORY_LIMIT, val: &mem_limit_lo);
449	base = ((unsigned long) mem_base_lo & PCI_MEMORY_RANGE_MASK) << 16;
450	limit = ((unsigned long) mem_limit_lo & PCI_MEMORY_RANGE_MASK) << 16;
451	if (base <= limit) {
452	res->flags = (mem_base_lo & PCI_MEMORY_RANGE_TYPE_MASK) | IORESOURCE_MEM;
453	region.start = base;
454	region.end = limit + 0xfffff;
455	pcibios_bus_to_resource(bus: dev->bus, res, region: &region);
456	pci_info(dev, " bridge window %pR\n", res);
457	}
458	}
459
460	static void pci_read_bridge_mmio_pref(struct pci_bus *child)
461	{
462	struct pci_dev *dev = child->self;
463	u16 mem_base_lo, mem_limit_lo;
464	u64 base64, limit64;
465	pci_bus_addr_t base, limit;
466	struct pci_bus_region region;
467	struct resource *res;
468
469	res = child->resource[2];
470	pci_read_config_word(dev, PCI_PREF_MEMORY_BASE, val: &mem_base_lo);
471	pci_read_config_word(dev, PCI_PREF_MEMORY_LIMIT, val: &mem_limit_lo);
472	base64 = (mem_base_lo & PCI_PREF_RANGE_MASK) << 16;
473	limit64 = (mem_limit_lo & PCI_PREF_RANGE_MASK) << 16;
474
475	if ((mem_base_lo & PCI_PREF_RANGE_TYPE_MASK) == PCI_PREF_RANGE_TYPE_64) {
476	u32 mem_base_hi, mem_limit_hi;
477
478	pci_read_config_dword(dev, PCI_PREF_BASE_UPPER32, val: &mem_base_hi);
479	pci_read_config_dword(dev, PCI_PREF_LIMIT_UPPER32, val: &mem_limit_hi);
480
481	/*
482	* Some bridges set the base > limit by default, and some
483	* (broken) BIOSes do not initialize them. If we find
484	* this, just assume they are not being used.
485	*/
486	if (mem_base_hi <= mem_limit_hi) {
487	base64 |= (u64) mem_base_hi << 32;
488	limit64 |= (u64) mem_limit_hi << 32;
489	}
490	}
491
492	base = (pci_bus_addr_t) base64;
493	limit = (pci_bus_addr_t) limit64;
494
495	if (base != base64) {
496	pci_err(dev, "can't handle bridge window above 4GB (bus address %#010llx)\n",
497	(unsigned long long) base64);
498	return;
499	}
500
501	if (base <= limit) {
502	res->flags = (mem_base_lo & PCI_PREF_RANGE_TYPE_MASK) |
503	IORESOURCE_MEM | IORESOURCE_PREFETCH;
504	if (res->flags & PCI_PREF_RANGE_TYPE_64)
505	res->flags |= IORESOURCE_MEM_64;
506	region.start = base;
507	region.end = limit + 0xfffff;
508	pcibios_bus_to_resource(bus: dev->bus, res, region: &region);
509	pci_info(dev, " bridge window %pR\n", res);
510	}
511	}
512
513	void pci_read_bridge_bases(struct pci_bus *child)
514	{
515	struct pci_dev *dev = child->self;
516	struct resource *res;
517	int i;
518
519	if (pci_is_root_bus(pbus: child)) /* It's a host bus, nothing to read */
520	return;
521
522	pci_info(dev, "PCI bridge to %pR%s\n",
523	&child->busn_res,
524	dev->transparent ? " (subtractive decode)" : "");
525
526	pci_bus_remove_resources(bus: child);
527	for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++)
528	child->resource[i] = &dev->resource[PCI_BRIDGE_RESOURCES+i];
529
530	pci_read_bridge_io(child);
531	pci_read_bridge_mmio(child);
532	pci_read_bridge_mmio_pref(child);
533
534	if (dev->transparent) {
535	pci_bus_for_each_resource(child->parent, res) {
536	if (res && res->flags) {
537	pci_bus_add_resource(bus: child, res,
538	PCI_SUBTRACTIVE_DECODE);
539	pci_info(dev, " bridge window %pR (subtractive decode)\n",
540	res);
541	}
542	}
543	}
544	}
545
546	static struct pci_bus *pci_alloc_bus(struct pci_bus *parent)
547	{
548	struct pci_bus *b;
549
550	b = kzalloc(size: sizeof(*b), GFP_KERNEL);
551	if (!b)
552	return NULL;
553
554	INIT_LIST_HEAD(list: &b->node);
555	INIT_LIST_HEAD(list: &b->children);
556	INIT_LIST_HEAD(list: &b->devices);
557	INIT_LIST_HEAD(list: &b->slots);
558	INIT_LIST_HEAD(list: &b->resources);
559	b->max_bus_speed = PCI_SPEED_UNKNOWN;
560	b->cur_bus_speed = PCI_SPEED_UNKNOWN;
561	#ifdef CONFIG_PCI_DOMAINS_GENERIC
562	if (parent)
563	b->domain_nr = parent->domain_nr;
564	#endif
565	return b;
566	}
567
568	static void pci_release_host_bridge_dev(struct device *dev)
569	{
570	struct pci_host_bridge *bridge = to_pci_host_bridge(dev);
571
572	if (bridge->release_fn)
573	bridge->release_fn(bridge);
574
575	pci_free_resource_list(resources: &bridge->windows);
576	pci_free_resource_list(resources: &bridge->dma_ranges);
577	kfree(objp: bridge);
578	}
579
580	static void pci_init_host_bridge(struct pci_host_bridge *bridge)
581	{
582	INIT_LIST_HEAD(list: &bridge->windows);
583	INIT_LIST_HEAD(list: &bridge->dma_ranges);
584
585	/*
586	* We assume we can manage these PCIe features. Some systems may
587	* reserve these for use by the platform itself, e.g., an ACPI BIOS
588	* may implement its own AER handling and use _OSC to prevent the
589	* OS from interfering.
590	*/
591	bridge->native_aer = 1;
592	bridge->native_pcie_hotplug = 1;
593	bridge->native_shpc_hotplug = 1;
594	bridge->native_pme = 1;
595	bridge->native_ltr = 1;
596	bridge->native_dpc = 1;
597	bridge->domain_nr = PCI_DOMAIN_NR_NOT_SET;
598	bridge->native_cxl_error = 1;
599
600	device_initialize(dev: &bridge->dev);
601	}
602
603	struct pci_host_bridge *pci_alloc_host_bridge(size_t priv)
604	{
605	struct pci_host_bridge *bridge;
606
607	bridge = kzalloc(size: sizeof(*bridge) + priv, GFP_KERNEL);
608	if (!bridge)
609	return NULL;
610
611	pci_init_host_bridge(bridge);
612	bridge->dev.release = pci_release_host_bridge_dev;
613
614	return bridge;
615	}
616	EXPORT_SYMBOL(pci_alloc_host_bridge);
617
618	static void devm_pci_alloc_host_bridge_release(void *data)
619	{
620	pci_free_host_bridge(bridge: data);
621	}
622
623	struct pci_host_bridge *devm_pci_alloc_host_bridge(struct device *dev,
624	size_t priv)
625	{
626	int ret;
627	struct pci_host_bridge *bridge;
628
629	bridge = pci_alloc_host_bridge(priv);
630	if (!bridge)
631	return NULL;
632
633	bridge->dev.parent = dev;
634
635	ret = devm_add_action_or_reset(dev, devm_pci_alloc_host_bridge_release,
636	bridge);
637	if (ret)
638	return NULL;
639
640	ret = devm_of_pci_bridge_init(dev, bridge);
641	if (ret)
642	return NULL;
643
644	return bridge;
645	}
646	EXPORT_SYMBOL(devm_pci_alloc_host_bridge);
647
648	void pci_free_host_bridge(struct pci_host_bridge *bridge)
649	{
650	put_device(dev: &bridge->dev);
651	}
652	EXPORT_SYMBOL(pci_free_host_bridge);
653
654	/* Indexed by PCI_X_SSTATUS_FREQ (secondary bus mode and frequency) */
655	static const unsigned char pcix_bus_speed[] = {
656	PCI_SPEED_UNKNOWN, /* 0 */
657	PCI_SPEED_66MHz_PCIX, /* 1 */
658	PCI_SPEED_100MHz_PCIX, /* 2 */
659	PCI_SPEED_133MHz_PCIX, /* 3 */
660	PCI_SPEED_UNKNOWN, /* 4 */
661	PCI_SPEED_66MHz_PCIX_ECC, /* 5 */
662	PCI_SPEED_100MHz_PCIX_ECC, /* 6 */
663	PCI_SPEED_133MHz_PCIX_ECC, /* 7 */
664	PCI_SPEED_UNKNOWN, /* 8 */
665	PCI_SPEED_66MHz_PCIX_266, /* 9 */
666	PCI_SPEED_100MHz_PCIX_266, /* A */
667	PCI_SPEED_133MHz_PCIX_266, /* B */
668	PCI_SPEED_UNKNOWN, /* C */
669	PCI_SPEED_66MHz_PCIX_533, /* D */
670	PCI_SPEED_100MHz_PCIX_533, /* E */
671	PCI_SPEED_133MHz_PCIX_533 /* F */
672	};
673
674	/* Indexed by PCI_EXP_LNKCAP_SLS, PCI_EXP_LNKSTA_CLS */
675	const unsigned char pcie_link_speed[] = {
676	PCI_SPEED_UNKNOWN, /* 0 */
677	PCIE_SPEED_2_5GT, /* 1 */
678	PCIE_SPEED_5_0GT, /* 2 */
679	PCIE_SPEED_8_0GT, /* 3 */
680	PCIE_SPEED_16_0GT, /* 4 */
681	PCIE_SPEED_32_0GT, /* 5 */
682	PCIE_SPEED_64_0GT, /* 6 */
683	PCI_SPEED_UNKNOWN, /* 7 */
684	PCI_SPEED_UNKNOWN, /* 8 */
685	PCI_SPEED_UNKNOWN, /* 9 */
686	PCI_SPEED_UNKNOWN, /* A */
687	PCI_SPEED_UNKNOWN, /* B */
688	PCI_SPEED_UNKNOWN, /* C */
689	PCI_SPEED_UNKNOWN, /* D */
690	PCI_SPEED_UNKNOWN, /* E */
691	PCI_SPEED_UNKNOWN /* F */
692	};
693	EXPORT_SYMBOL_GPL(pcie_link_speed);
694
695	const char *pci_speed_string(enum pci_bus_speed speed)
696	{
697	/* Indexed by the pci_bus_speed enum */
698	static const char *speed_strings[] = {
699	"33 MHz PCI", /* 0x00 */
700	"66 MHz PCI", /* 0x01 */
701	"66 MHz PCI-X", /* 0x02 */
702	"100 MHz PCI-X", /* 0x03 */
703	"133 MHz PCI-X", /* 0x04 */
704	NULL, /* 0x05 */
705	NULL, /* 0x06 */
706	NULL, /* 0x07 */
707	NULL, /* 0x08 */
708	"66 MHz PCI-X 266", /* 0x09 */
709	"100 MHz PCI-X 266", /* 0x0a */
710	"133 MHz PCI-X 266", /* 0x0b */
711	"Unknown AGP", /* 0x0c */
712	"1x AGP", /* 0x0d */
713	"2x AGP", /* 0x0e */
714	"4x AGP", /* 0x0f */
715	"8x AGP", /* 0x10 */
716	"66 MHz PCI-X 533", /* 0x11 */
717	"100 MHz PCI-X 533", /* 0x12 */
718	"133 MHz PCI-X 533", /* 0x13 */
719	"2.5 GT/s PCIe", /* 0x14 */
720	"5.0 GT/s PCIe", /* 0x15 */
721	"8.0 GT/s PCIe", /* 0x16 */
722	"16.0 GT/s PCIe", /* 0x17 */
723	"32.0 GT/s PCIe", /* 0x18 */
724	"64.0 GT/s PCIe", /* 0x19 */
725	};
726
727	if (speed < ARRAY_SIZE(speed_strings))
728	return speed_strings[speed];
729	return "Unknown";
730	}
731	EXPORT_SYMBOL_GPL(pci_speed_string);
732
733	void pcie_update_link_speed(struct pci_bus *bus, u16 linksta)
734	{
735	bus->cur_bus_speed = pcie_link_speed[linksta & PCI_EXP_LNKSTA_CLS];
736	}
737	EXPORT_SYMBOL_GPL(pcie_update_link_speed);
738
739	static unsigned char agp_speeds[] = {
740	AGP_UNKNOWN,
741	AGP_1X,
742	AGP_2X,
743	AGP_4X,
744	AGP_8X
745	};
746
747	static enum pci_bus_speed agp_speed(int agp3, int agpstat)
748	{
749	int index = 0;
750
751	if (agpstat & 4)
752	index = 3;
753	else if (agpstat & 2)
754	index = 2;
755	else if (agpstat & 1)
756	index = 1;
757	else
758	goto out;
759
760	if (agp3) {
761	index += 2;
762	if (index == 5)
763	index = 0;
764	}
765
766	out:
767	return agp_speeds[index];
768	}
769
770	static void pci_set_bus_speed(struct pci_bus *bus)
771	{
772	struct pci_dev *bridge = bus->self;
773	int pos;
774
775	pos = pci_find_capability(dev: bridge, PCI_CAP_ID_AGP);
776	if (!pos)
777	pos = pci_find_capability(dev: bridge, PCI_CAP_ID_AGP3);
778	if (pos) {
779	u32 agpstat, agpcmd;
780
781	pci_read_config_dword(dev: bridge, where: pos + PCI_AGP_STATUS, val: &agpstat);
782	bus->max_bus_speed = agp_speed(agp3: agpstat & 8, agpstat: agpstat & 7);
783
784	pci_read_config_dword(dev: bridge, where: pos + PCI_AGP_COMMAND, val: &agpcmd);
785	bus->cur_bus_speed = agp_speed(agp3: agpstat & 8, agpstat: agpcmd & 7);
786	}
787
788	pos = pci_find_capability(dev: bridge, PCI_CAP_ID_PCIX);
789	if (pos) {
790	u16 status;
791	enum pci_bus_speed max;
792
793	pci_read_config_word(dev: bridge, where: pos + PCI_X_BRIDGE_SSTATUS,
794	val: &status);
795
796	if (status & PCI_X_SSTATUS_533MHZ) {
797	max = PCI_SPEED_133MHz_PCIX_533;
798	} else if (status & PCI_X_SSTATUS_266MHZ) {
799	max = PCI_SPEED_133MHz_PCIX_266;
800	} else if (status & PCI_X_SSTATUS_133MHZ) {
801	if ((status & PCI_X_SSTATUS_VERS) == PCI_X_SSTATUS_V2)
802	max = PCI_SPEED_133MHz_PCIX_ECC;
803	else
804	max = PCI_SPEED_133MHz_PCIX;
805	} else {
806	max = PCI_SPEED_66MHz_PCIX;
807	}
808
809	bus->max_bus_speed = max;
810	bus->cur_bus_speed =
811	pcix_bus_speed[FIELD_GET(PCI_X_SSTATUS_FREQ, status)];
812
813	return;
814	}
815
816	if (pci_is_pcie(dev: bridge)) {
817	u32 linkcap;
818	u16 linksta;
819
820	pcie_capability_read_dword(dev: bridge, PCI_EXP_LNKCAP, val: &linkcap);
821	bus->max_bus_speed = pcie_link_speed[linkcap & PCI_EXP_LNKCAP_SLS];
822
823	pcie_capability_read_word(dev: bridge, PCI_EXP_LNKSTA, val: &linksta);
824	pcie_update_link_speed(bus, linksta);
825	}
826	}
827
828	static struct irq_domain *pci_host_bridge_msi_domain(struct pci_bus *bus)
829	{
830	struct irq_domain *d;
831
832	/* If the host bridge driver sets a MSI domain of the bridge, use it */
833	d = dev_get_msi_domain(dev: bus->bridge);
834
835	/*
836	* Any firmware interface that can resolve the msi_domain
837	* should be called from here.
838	*/
839	if (!d)
840	d = pci_host_bridge_of_msi_domain(bus);
841	if (!d)
842	d = pci_host_bridge_acpi_msi_domain(bus);
843
844	/*
845	* If no IRQ domain was found via the OF tree, try looking it up
846	* directly through the fwnode_handle.
847	*/
848	if (!d) {
849	struct fwnode_handle *fwnode = pci_root_bus_fwnode(bus);
850
851	if (fwnode)
852	d = irq_find_matching_fwnode(fwnode,
853	bus_token: DOMAIN_BUS_PCI_MSI);
854	}
855
856	return d;
857	}
858
859	static void pci_set_bus_msi_domain(struct pci_bus *bus)
860	{
861	struct irq_domain *d;
862	struct pci_bus *b;
863
864	/*
865	* The bus can be a root bus, a subordinate bus, or a virtual bus
866	* created by an SR-IOV device. Walk up to the first bridge device
867	* found or derive the domain from the host bridge.
868	*/
869	for (b = bus, d = NULL; !d && !pci_is_root_bus(pbus: b); b = b->parent) {
870	if (b->self)
871	d = dev_get_msi_domain(dev: &b->self->dev);
872	}
873
874	if (!d)
875	d = pci_host_bridge_msi_domain(bus: b);
876
877	dev_set_msi_domain(dev: &bus->dev, d);
878	}
879
880	static int pci_register_host_bridge(struct pci_host_bridge *bridge)
881	{
882	struct device *parent = bridge->dev.parent;
883	struct resource_entry *window, *next, *n;
884	struct pci_bus *bus, *b;
885	resource_size_t offset, next_offset;
886	LIST_HEAD(resources);
887	struct resource *res, *next_res;
888	char addr[64], *fmt;
889	const char *name;
890	int err;
891
892	bus = pci_alloc_bus(NULL);
893	if (!bus)
894	return -ENOMEM;
895
896	bridge->bus = bus;
897
898	bus->sysdata = bridge->sysdata;
899	bus->ops = bridge->ops;
900	bus->number = bus->busn_res.start = bridge->busnr;
901	#ifdef CONFIG_PCI_DOMAINS_GENERIC
902	if (bridge->domain_nr == PCI_DOMAIN_NR_NOT_SET)
903	bus->domain_nr = pci_bus_find_domain_nr(bus, parent);
904	else
905	bus->domain_nr = bridge->domain_nr;
906	if (bus->domain_nr < 0) {
907	err = bus->domain_nr;
908	goto free;
909	}
910	#endif
911
912	b = pci_find_bus(domain: pci_domain_nr(bus), busnr: bridge->busnr);
913	if (b) {
914	/* Ignore it if we already got here via a different bridge */
915	dev_dbg(&b->dev, "bus already known\n");
916	err = -EEXIST;
917	goto free;
918	}
919
920	dev_set_name(dev: &bridge->dev, name: "pci%04x:%02x", pci_domain_nr(bus),
921	bridge->busnr);
922
923	err = pcibios_root_bridge_prepare(bridge);
924	if (err)
925	goto free;
926
927	/* Temporarily move resources off the list */
928	list_splice_init(list: &bridge->windows, head: &resources);
929	err = device_add(dev: &bridge->dev);
930	if (err) {
931	put_device(dev: &bridge->dev);
932	goto free;
933	}
934	bus->bridge = get_device(dev: &bridge->dev);
935	device_enable_async_suspend(dev: bus->bridge);
936	pci_set_bus_of_node(bus);
937	pci_set_bus_msi_domain(bus);
938	if (bridge->msi_domain && !dev_get_msi_domain(dev: &bus->dev) &&
939	!pci_host_of_has_msi_map(dev: parent))
940	bus->bus_flags |= PCI_BUS_FLAGS_NO_MSI;
941
942	if (!parent)
943	set_dev_node(dev: bus->bridge, pcibus_to_node(bus));
944
945	bus->dev.class = &pcibus_class;
946	bus->dev.parent = bus->bridge;
947
948	dev_set_name(dev: &bus->dev, name: "%04x:%02x", pci_domain_nr(bus), bus->number);
949	name = dev_name(dev: &bus->dev);
950
951	err = device_register(dev: &bus->dev);
952	if (err)
953	goto unregister;
954
955	pcibios_add_bus(bus);
956
957	if (bus->ops->add_bus) {
958	err = bus->ops->add_bus(bus);
959	if (WARN_ON(err < 0))
960	dev_err(&bus->dev, "failed to add bus: %d\n", err);
961	}
962
963	/* Create legacy_io and legacy_mem files for this bus */
964	pci_create_legacy_files(bus);
965
966	if (parent)
967	dev_info(parent, "PCI host bridge to bus %s\n", name);
968	else
969	pr_info("PCI host bridge to bus %s\n", name);
970
971	if (nr_node_ids > 1 && pcibus_to_node(bus) == NUMA_NO_NODE)
972	dev_warn(&bus->dev, "Unknown NUMA node; performance will be reduced\n");
973
974	/* Coalesce contiguous windows */
975	resource_list_for_each_entry_safe(window, n, &resources) {
976	if (list_is_last(list: &window->node, head: &resources))
977	break;
978
979	next = list_next_entry(window, node);
980	offset = window->offset;
981	res = window->res;
982	next_offset = next->offset;
983	next_res = next->res;
984
985	if (res->flags != next_res->flags || offset != next_offset)
986	continue;
987
988	if (res->end + 1 == next_res->start) {
989	next_res->start = res->start;
990	res->flags = res->start = res->end = 0;
991	}
992	}
993
994	/* Add initial resources to the bus */
995	resource_list_for_each_entry_safe(window, n, &resources) {
996	offset = window->offset;
997	res = window->res;
998	if (!res->flags && !res->start && !res->end) {
999	release_resource(new: res);
1000	resource_list_destroy_entry(entry: window);
1001	continue;
1002	}
1003
1004	list_move_tail(list: &window->node, head: &bridge->windows);
1005
1006	if (res->flags & IORESOURCE_BUS)
1007	pci_bus_insert_busn_res(b: bus, bus: bus->number, busmax: res->end);
1008	else
1009	pci_bus_add_resource(bus, res, flags: 0);
1010
1011	if (offset) {
1012	if (resource_type(res) == IORESOURCE_IO)
1013	fmt = " (bus address [%#06llx-%#06llx])";
1014	else
1015	fmt = " (bus address [%#010llx-%#010llx])";
1016
1017	snprintf(buf: addr, size: sizeof(addr), fmt,
1018	(unsigned long long)(res->start - offset),
1019	(unsigned long long)(res->end - offset));
1020	} else
1021	addr[0] = '\0';
1022
1023	dev_info(&bus->dev, "root bus resource %pR%s\n", res, addr);
1024	}
1025
1026	down_write(sem: &pci_bus_sem);
1027	list_add_tail(new: &bus->node, head: &pci_root_buses);
1028	up_write(sem: &pci_bus_sem);
1029
1030	return 0;
1031
1032	unregister:
1033	put_device(dev: &bridge->dev);
1034	device_del(dev: &bridge->dev);
1035
1036	free:
1037	#ifdef CONFIG_PCI_DOMAINS_GENERIC
1038	pci_bus_release_domain_nr(bus, parent);
1039	#endif
1040	kfree(objp: bus);
1041	return err;
1042	}
1043
1044	static bool pci_bridge_child_ext_cfg_accessible(struct pci_dev *bridge)
1045	{
1046	int pos;
1047	u32 status;
1048
1049	/*
1050	* If extended config space isn't accessible on a bridge's primary
1051	* bus, we certainly can't access it on the secondary bus.
1052	*/
1053	if (bridge->bus->bus_flags & PCI_BUS_FLAGS_NO_EXTCFG)
1054	return false;
1055
1056	/*
1057	* PCIe Root Ports and switch ports are PCIe on both sides, so if
1058	* extended config space is accessible on the primary, it's also
1059	* accessible on the secondary.
1060	*/
1061	if (pci_is_pcie(dev: bridge) &&
1062	(pci_pcie_type(dev: bridge) == PCI_EXP_TYPE_ROOT_PORT ||
1063	pci_pcie_type(dev: bridge) == PCI_EXP_TYPE_UPSTREAM ||
1064	pci_pcie_type(dev: bridge) == PCI_EXP_TYPE_DOWNSTREAM))
1065	return true;
1066
1067	/*
1068	* For the other bridge types:
1069	* - PCI-to-PCI bridges
1070	* - PCIe-to-PCI/PCI-X forward bridges
1071	* - PCI/PCI-X-to-PCIe reverse bridges
1072	* extended config space on the secondary side is only accessible
1073	* if the bridge supports PCI-X Mode 2.
1074	*/
1075	pos = pci_find_capability(dev: bridge, PCI_CAP_ID_PCIX);
1076	if (!pos)
1077	return false;
1078
1079	pci_read_config_dword(dev: bridge, where: pos + PCI_X_STATUS, val: &status);
1080	return status & (PCI_X_STATUS_266MHZ | PCI_X_STATUS_533MHZ);
1081	}
1082
1083	static struct pci_bus *pci_alloc_child_bus(struct pci_bus *parent,
1084	struct pci_dev *bridge, int busnr)
1085	{
1086	struct pci_bus *child;
1087	struct pci_host_bridge *host;
1088	int i;
1089	int ret;
1090
1091	/* Allocate a new bus and inherit stuff from the parent */
1092	child = pci_alloc_bus(parent);
1093	if (!child)
1094	return NULL;
1095
1096	child->parent = parent;
1097	child->sysdata = parent->sysdata;
1098	child->bus_flags = parent->bus_flags;
1099
1100	host = pci_find_host_bridge(bus: parent);
1101	if (host->child_ops)
1102	child->ops = host->child_ops;
1103	else
1104	child->ops = parent->ops;
1105
1106	/*
1107	* Initialize some portions of the bus device, but don't register
1108	* it now as the parent is not properly set up yet.
1109	*/
1110	child->dev.class = &pcibus_class;
1111	dev_set_name(dev: &child->dev, name: "%04x:%02x", pci_domain_nr(bus: child), busnr);
1112
1113	/* Set up the primary, secondary and subordinate bus numbers */
1114	child->number = child->busn_res.start = busnr;
1115	child->primary = parent->busn_res.start;
1116	child->busn_res.end = 0xff;
1117
1118	if (!bridge) {
1119	child->dev.parent = parent->bridge;
1120	goto add_dev;
1121	}
1122
1123	child->self = bridge;
1124	child->bridge = get_device(dev: &bridge->dev);
1125	child->dev.parent = child->bridge;
1126	pci_set_bus_of_node(bus: child);
1127	pci_set_bus_speed(bus: child);
1128
1129	/*
1130	* Check whether extended config space is accessible on the child
1131	* bus. Note that we currently assume it is always accessible on
1132	* the root bus.
1133	*/
1134	if (!pci_bridge_child_ext_cfg_accessible(bridge)) {
1135	child->bus_flags |= PCI_BUS_FLAGS_NO_EXTCFG;
1136	pci_info(child, "extended config space not accessible\n");
1137	}
1138
1139	/* Set up default resource pointers and names */
1140	for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++) {
1141	child->resource[i] = &bridge->resource[PCI_BRIDGE_RESOURCES+i];
1142	child->resource[i]->name = child->name;
1143	}
1144	bridge->subordinate = child;
1145
1146	add_dev:
1147	pci_set_bus_msi_domain(bus: child);
1148	ret = device_register(dev: &child->dev);
1149	WARN_ON(ret < 0);
1150
1151	pcibios_add_bus(bus: child);
1152
1153	if (child->ops->add_bus) {
1154	ret = child->ops->add_bus(child);
1155	if (WARN_ON(ret < 0))
1156	dev_err(&child->dev, "failed to add bus: %d\n", ret);
1157	}
1158
1159	/* Create legacy_io and legacy_mem files for this bus */
1160	pci_create_legacy_files(bus: child);
1161
1162	return child;
1163	}
1164
1165	struct pci_bus *pci_add_new_bus(struct pci_bus *parent, struct pci_dev *dev,
1166	int busnr)
1167	{
1168	struct pci_bus *child;
1169
1170	child = pci_alloc_child_bus(parent, bridge: dev, busnr);
1171	if (child) {
1172	down_write(sem: &pci_bus_sem);
1173	list_add_tail(new: &child->node, head: &parent->children);
1174	up_write(sem: &pci_bus_sem);
1175	}
1176	return child;
1177	}
1178	EXPORT_SYMBOL(pci_add_new_bus);
1179
1180	static void pci_enable_crs(struct pci_dev *pdev)
1181	{
1182	u16 root_cap = 0;
1183
1184	/* Enable CRS Software Visibility if supported */
1185	pcie_capability_read_word(dev: pdev, PCI_EXP_RTCAP, val: &root_cap);
1186	if (root_cap & PCI_EXP_RTCAP_CRSVIS)
1187	pcie_capability_set_word(dev: pdev, PCI_EXP_RTCTL,
1188	PCI_EXP_RTCTL_CRSSVE);
1189	}
1190
1191	static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus,
1192	unsigned int available_buses);
1193	/**
1194	* pci_ea_fixed_busnrs() - Read fixed Secondary and Subordinate bus
1195	* numbers from EA capability.
1196	* @dev: Bridge
1197	* @sec: updated with secondary bus number from EA
1198	* @sub: updated with subordinate bus number from EA
1199	*
1200	* If @dev is a bridge with EA capability that specifies valid secondary
1201	* and subordinate bus numbers, return true with the bus numbers in @sec
1202	* and @sub. Otherwise return false.
1203	*/
1204	static bool pci_ea_fixed_busnrs(struct pci_dev *dev, u8 *sec, u8 *sub)
1205	{
1206	int ea, offset;
1207	u32 dw;
1208	u8 ea_sec, ea_sub;
1209
1210	if (dev->hdr_type != PCI_HEADER_TYPE_BRIDGE)
1211	return false;
1212
1213	/* find PCI EA capability in list */
1214	ea = pci_find_capability(dev, PCI_CAP_ID_EA);
1215	if (!ea)
1216	return false;
1217
1218	offset = ea + PCI_EA_FIRST_ENT;
1219	pci_read_config_dword(dev, where: offset, val: &dw);
1220	ea_sec = FIELD_GET(PCI_EA_SEC_BUS_MASK, dw);
1221	ea_sub = FIELD_GET(PCI_EA_SUB_BUS_MASK, dw);
1222	if (ea_sec == 0 || ea_sub < ea_sec)
1223	return false;
1224
1225	*sec = ea_sec;
1226	*sub = ea_sub;
1227	return true;
1228	}
1229
1230	/*
1231	* pci_scan_bridge_extend() - Scan buses behind a bridge
1232	* @bus: Parent bus the bridge is on
1233	* @dev: Bridge itself
1234	* @max: Starting subordinate number of buses behind this bridge
1235	* @available_buses: Total number of buses available for this bridge and
1236	* the devices below. After the minimal bus space has
1237	* been allocated the remaining buses will be
1238	* distributed equally between hotplug-capable bridges.
1239	* @pass: Either %0 (scan already configured bridges) or %1 (scan bridges
1240	* that need to be reconfigured.
1241	*
1242	* If it's a bridge, configure it and scan the bus behind it.
1243	* For CardBus bridges, we don't scan behind as the devices will
1244	* be handled by the bridge driver itself.
1245	*
1246	* We need to process bridges in two passes -- first we scan those
1247	* already configured by the BIOS and after we are done with all of
1248	* them, we proceed to assigning numbers to the remaining buses in
1249	* order to avoid overlaps between old and new bus numbers.
1250	*
1251	* Return: New subordinate number covering all buses behind this bridge.
1252	*/
1253	static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,
1254	int max, unsigned int available_buses,
1255	int pass)
1256	{
1257	struct pci_bus *child;
1258	int is_cardbus = (dev->hdr_type == PCI_HEADER_TYPE_CARDBUS);
1259	u32 buses, i, j = 0;
1260	u16 bctl;
1261	u8 primary, secondary, subordinate;
1262	int broken = 0;
1263	bool fixed_buses;
1264	u8 fixed_sec, fixed_sub;
1265	int next_busnr;
1266
1267	/*
1268	* Make sure the bridge is powered on to be able to access config
1269	* space of devices below it.
1270	*/
1271	pm_runtime_get_sync(dev: &dev->dev);
1272
1273	pci_read_config_dword(dev, PCI_PRIMARY_BUS, val: &buses);
1274	primary = buses & 0xFF;
1275	secondary = (buses >> 8) & 0xFF;
1276	subordinate = (buses >> 16) & 0xFF;
1277
1278	pci_dbg(dev, "scanning [bus %02x-%02x] behind bridge, pass %d\n",
1279	secondary, subordinate, pass);
1280
1281	if (!primary && (primary != bus->number) && secondary && subordinate) {
1282	pci_warn(dev, "Primary bus is hard wired to 0\n");
1283	primary = bus->number;
1284	}
1285
1286	/* Check if setup is sensible at all */
1287	if (!pass &&
1288	(primary != bus->number || secondary <= bus->number ||
1289	secondary > subordinate)) {
1290	pci_info(dev, "bridge configuration invalid ([bus %02x-%02x]), reconfiguring\n",
1291	secondary, subordinate);
1292	broken = 1;
1293	}
1294
1295	/*
1296	* Disable Master-Abort Mode during probing to avoid reporting of
1297	* bus errors in some architectures.
1298	*/
1299	pci_read_config_word(dev, PCI_BRIDGE_CONTROL, val: &bctl);
1300	pci_write_config_word(dev, PCI_BRIDGE_CONTROL,
1301	val: bctl & ~PCI_BRIDGE_CTL_MASTER_ABORT);
1302
1303	pci_enable_crs(pdev: dev);
1304
1305	if ((secondary || subordinate) && !pcibios_assign_all_busses() &&
1306	!is_cardbus && !broken) {
1307	unsigned int cmax, buses;
1308
1309	/*
1310	* Bus already configured by firmware, process it in the
1311	* first pass and just note the configuration.
1312	*/
1313	if (pass)
1314	goto out;
1315
1316	/*
1317	* The bus might already exist for two reasons: Either we
1318	* are rescanning the bus or the bus is reachable through
1319	* more than one bridge. The second case can happen with
1320	* the i450NX chipset.
1321	*/
1322	child = pci_find_bus(domain: pci_domain_nr(bus), busnr: secondary);
1323	if (!child) {
1324	child = pci_add_new_bus(bus, dev, secondary);
1325	if (!child)
1326	goto out;
1327	child->primary = primary;
1328	pci_bus_insert_busn_res(b: child, bus: secondary, busmax: subordinate);
1329	child->bridge_ctl = bctl;
1330	}
1331
1332	buses = subordinate - secondary;
1333	cmax = pci_scan_child_bus_extend(bus: child, available_buses: buses);
1334	if (cmax > subordinate)
1335	pci_warn(dev, "bridge has subordinate %02x but max busn %02x\n",
1336	subordinate, cmax);
1337
1338	/* Subordinate should equal child->busn_res.end */
1339	if (subordinate > max)
1340	max = subordinate;
1341	} else {
1342
1343	/*
1344	* We need to assign a number to this bus which we always
1345	* do in the second pass.
1346	*/
1347	if (!pass) {
1348	if (pcibios_assign_all_busses() || broken || is_cardbus)
1349
1350	/*
1351	* Temporarily disable forwarding of the
1352	* configuration cycles on all bridges in
1353	* this bus segment to avoid possible
1354	* conflicts in the second pass between two
1355	* bridges programmed with overlapping bus
1356	* ranges.
1357	*/
1358	pci_write_config_dword(dev, PCI_PRIMARY_BUS,
1359	val: buses & ~0xffffff);
1360	goto out;
1361	}
1362
1363	/* Clear errors */
1364	pci_write_config_word(dev, PCI_STATUS, val: 0xffff);
1365
1366	/* Read bus numbers from EA Capability (if present) */
1367	fixed_buses = pci_ea_fixed_busnrs(dev, sec: &fixed_sec, sub: &fixed_sub);
1368	if (fixed_buses)
1369	next_busnr = fixed_sec;
1370	else
1371	next_busnr = max + 1;
1372
1373	/*
1374	* Prevent assigning a bus number that already exists.
1375	* This can happen when a bridge is hot-plugged, so in this
1376	* case we only re-scan this bus.
1377	*/
1378	child = pci_find_bus(domain: pci_domain_nr(bus), busnr: next_busnr);
1379	if (!child) {
1380	child = pci_add_new_bus(bus, dev, next_busnr);
1381	if (!child)
1382	goto out;
1383	pci_bus_insert_busn_res(b: child, bus: next_busnr,
1384	busmax: bus->busn_res.end);
1385	}
1386	max++;
1387	if (available_buses)
1388	available_buses--;
1389
1390	buses = (buses & 0xff000000)
1391	| ((unsigned int)(child->primary) << 0)
1392	| ((unsigned int)(child->busn_res.start) << 8)
1393	| ((unsigned int)(child->busn_res.end) << 16);
1394
1395	/*
1396	* yenta.c forces a secondary latency timer of 176.
1397	* Copy that behaviour here.
1398	*/
1399	if (is_cardbus) {
1400	buses &= ~0xff000000;
1401	buses |= CARDBUS_LATENCY_TIMER << 24;
1402	}
1403
1404	/* We need to blast all three values with a single write */
1405	pci_write_config_dword(dev, PCI_PRIMARY_BUS, val: buses);
1406
1407	if (!is_cardbus) {
1408	child->bridge_ctl = bctl;
1409	max = pci_scan_child_bus_extend(bus: child, available_buses);
1410	} else {
1411
1412	/*
1413	* For CardBus bridges, we leave 4 bus numbers as
1414	* cards with a PCI-to-PCI bridge can be inserted
1415	* later.
1416	*/
1417	for (i = 0; i < CARDBUS_RESERVE_BUSNR; i++) {
1418	struct pci_bus *parent = bus;
1419	if (pci_find_bus(domain: pci_domain_nr(bus),
1420	busnr: max+i+1))
1421	break;
1422	while (parent->parent) {
1423	if ((!pcibios_assign_all_busses()) &&
1424	(parent->busn_res.end > max) &&
1425	(parent->busn_res.end <= max+i)) {
1426	j = 1;
1427	}
1428	parent = parent->parent;
1429	}
1430	if (j) {
1431
1432	/*
1433	* Often, there are two CardBus
1434	* bridges -- try to leave one
1435	* valid bus number for each one.
1436	*/
1437	i /= 2;
1438	break;
1439	}
1440	}
1441	max += i;
1442	}
1443
1444	/*
1445	* Set subordinate bus number to its real value.
1446	* If fixed subordinate bus number exists from EA
1447	* capability then use it.
1448	*/
1449	if (fixed_buses)
1450	max = fixed_sub;
1451	pci_bus_update_busn_res_end(b: child, busmax: max);
1452	pci_write_config_byte(dev, PCI_SUBORDINATE_BUS, val: max);
1453	}
1454
1455	sprintf(buf: child->name,
1456	fmt: (is_cardbus ? "PCI CardBus %04x:%02x" : "PCI Bus %04x:%02x"),
1457	pci_domain_nr(bus), child->number);
1458
1459	/* Check that all devices are accessible */
1460	while (bus->parent) {
1461	if ((child->busn_res.end > bus->busn_res.end) ||
1462	(child->number > bus->busn_res.end) ||
1463	(child->number < bus->number) ||
1464	(child->busn_res.end < bus->number)) {
1465	dev_info(&dev->dev, "devices behind bridge are unusable because %pR cannot be assigned for them\n",
1466	&child->busn_res);
1467	break;
1468	}
1469	bus = bus->parent;
1470	}
1471
1472	out:
1473	pci_write_config_word(dev, PCI_BRIDGE_CONTROL, val: bctl);
1474
1475	pm_runtime_put(dev: &dev->dev);
1476
1477	return max;
1478	}
1479
1480	/*
1481	* pci_scan_bridge() - Scan buses behind a bridge
1482	* @bus: Parent bus the bridge is on
1483	* @dev: Bridge itself
1484	* @max: Starting subordinate number of buses behind this bridge
1485	* @pass: Either %0 (scan already configured bridges) or %1 (scan bridges
1486	* that need to be reconfigured.
1487	*
1488	* If it's a bridge, configure it and scan the bus behind it.
1489	* For CardBus bridges, we don't scan behind as the devices will
1490	* be handled by the bridge driver itself.
1491	*
1492	* We need to process bridges in two passes -- first we scan those
1493	* already configured by the BIOS and after we are done with all of
1494	* them, we proceed to assigning numbers to the remaining buses in
1495	* order to avoid overlaps between old and new bus numbers.
1496	*
1497	* Return: New subordinate number covering all buses behind this bridge.
1498	*/
1499	int pci_scan_bridge(struct pci_bus *bus, struct pci_dev *dev, int max, int pass)
1500	{
1501	return pci_scan_bridge_extend(bus, dev, max, available_buses: 0, pass);
1502	}
1503	EXPORT_SYMBOL(pci_scan_bridge);
1504
1505	/*
1506	* Read interrupt line and base address registers.
1507	* The architecture-dependent code can tweak these, of course.
1508	*/
1509	static void pci_read_irq(struct pci_dev *dev)
1510	{
1511	unsigned char irq;
1512
1513	/* VFs are not allowed to use INTx, so skip the config reads */
1514	if (dev->is_virtfn) {
1515	dev->pin = 0;
1516	dev->irq = 0;
1517	return;
1518	}
1519
1520	pci_read_config_byte(dev, PCI_INTERRUPT_PIN, val: &irq);
1521	dev->pin = irq;
1522	if (irq)
1523	pci_read_config_byte(dev, PCI_INTERRUPT_LINE, val: &irq);
1524	dev->irq = irq;
1525	}
1526
1527	void set_pcie_port_type(struct pci_dev *pdev)
1528	{
1529	int pos;
1530	u16 reg16;
1531	u32 reg32;
1532	int type;
1533	struct pci_dev *parent;
1534
1535	pos = pci_find_capability(dev: pdev, PCI_CAP_ID_EXP);
1536	if (!pos)
1537	return;
1538
1539	pdev->pcie_cap = pos;
1540	pci_read_config_word(dev: pdev, where: pos + PCI_EXP_FLAGS, val: &reg16);
1541	pdev->pcie_flags_reg = reg16;
1542	pci_read_config_dword(dev: pdev, where: pos + PCI_EXP_DEVCAP, val: &pdev->devcap);
1543	pdev->pcie_mpss = FIELD_GET(PCI_EXP_DEVCAP_PAYLOAD, pdev->devcap);
1544
1545	pcie_capability_read_dword(dev: pdev, PCI_EXP_LNKCAP, val: &reg32);
1546	if (reg32 & PCI_EXP_LNKCAP_DLLLARC)
1547	pdev->link_active_reporting = 1;
1548
1549	parent = pci_upstream_bridge(dev: pdev);
1550	if (!parent)
1551	return;
1552
1553	/*
1554	* Some systems do not identify their upstream/downstream ports
1555	* correctly so detect impossible configurations here and correct
1556	* the port type accordingly.
1557	*/
1558	type = pci_pcie_type(dev: pdev);
1559	if (type == PCI_EXP_TYPE_DOWNSTREAM) {
1560	/*
1561	* If pdev claims to be downstream port but the parent
1562	* device is also downstream port assume pdev is actually
1563	* upstream port.
1564	*/
1565	if (pcie_downstream_port(dev: parent)) {
1566	pci_info(pdev, "claims to be downstream port but is acting as upstream port, correcting type\n");
1567	pdev->pcie_flags_reg &= ~PCI_EXP_FLAGS_TYPE;
1568	pdev->pcie_flags_reg |= PCI_EXP_TYPE_UPSTREAM;
1569	}
1570	} else if (type == PCI_EXP_TYPE_UPSTREAM) {
1571	/*
1572	* If pdev claims to be upstream port but the parent
1573	* device is also upstream port assume pdev is actually
1574	* downstream port.
1575	*/
1576	if (pci_pcie_type(dev: parent) == PCI_EXP_TYPE_UPSTREAM) {
1577	pci_info(pdev, "claims to be upstream port but is acting as downstream port, correcting type\n");
1578	pdev->pcie_flags_reg &= ~PCI_EXP_FLAGS_TYPE;
1579	pdev->pcie_flags_reg |= PCI_EXP_TYPE_DOWNSTREAM;
1580	}
1581	}
1582	}
1583
1584	void set_pcie_hotplug_bridge(struct pci_dev *pdev)
1585	{
1586	u32 reg32;
1587
1588	pcie_capability_read_dword(dev: pdev, PCI_EXP_SLTCAP, val: &reg32);
1589	if (reg32 & PCI_EXP_SLTCAP_HPC)
1590	pdev->is_hotplug_bridge = 1;
1591	}
1592
1593	static void set_pcie_thunderbolt(struct pci_dev *dev)
1594	{
1595	u16 vsec;
1596
1597	/* Is the device part of a Thunderbolt controller? */
1598	vsec = pci_find_vsec_capability(dev, PCI_VENDOR_ID_INTEL, PCI_VSEC_ID_INTEL_TBT);
1599	if (vsec)
1600	dev->is_thunderbolt = 1;
1601	}
1602
1603	static void set_pcie_untrusted(struct pci_dev *dev)
1604	{
1605	struct pci_dev *parent;
1606
1607	/*
1608	* If the upstream bridge is untrusted we treat this device
1609	* untrusted as well.
1610	*/
1611	parent = pci_upstream_bridge(dev);
1612	if (parent && (parent->untrusted || parent->external_facing))
1613	dev->untrusted = true;
1614	}
1615
1616	static void pci_set_removable(struct pci_dev *dev)
1617	{
1618	struct pci_dev *parent = pci_upstream_bridge(dev);
1619
1620	/*
1621	* We (only) consider everything downstream from an external_facing
1622	* device to be removable by the user. We're mainly concerned with
1623	* consumer platforms with user accessible thunderbolt ports that are
1624	* vulnerable to DMA attacks, and we expect those ports to be marked by
1625	* the firmware as external_facing. Devices in traditional hotplug
1626	* slots can technically be removed, but the expectation is that unless
1627	* the port is marked with external_facing, such devices are less
1628	* accessible to user / may not be removed by end user, and thus not
1629	* exposed as "removable" to userspace.
1630	*/
1631	if (parent &&
1632	(parent->external_facing || dev_is_removable(dev: &parent->dev)))
1633	dev_set_removable(dev: &dev->dev, removable: DEVICE_REMOVABLE);
1634	}
1635
1636	/**
1637	* pci_ext_cfg_is_aliased - Is ext config space just an alias of std config?
1638	* @dev: PCI device
1639	*
1640	* PCI Express to PCI/PCI-X Bridge Specification, rev 1.0, 4.1.4 says that
1641	* when forwarding a type1 configuration request the bridge must check that
1642	* the extended register address field is zero. The bridge is not permitted
1643	* to forward the transactions and must handle it as an Unsupported Request.
1644	* Some bridges do not follow this rule and simply drop the extended register
1645	* bits, resulting in the standard config space being aliased, every 256
1646	* bytes across the entire configuration space. Test for this condition by
1647	* comparing the first dword of each potential alias to the vendor/device ID.
1648	* Known offenders:
1649	* ASM1083/1085 PCIe-to-PCI Reversible Bridge (1b21:1080, rev 01 & 03)
1650	* AMD/ATI SBx00 PCI to PCI Bridge (1002:4384, rev 40)
1651	*/
1652	static bool pci_ext_cfg_is_aliased(struct pci_dev *dev)
1653	{
1654	#ifdef CONFIG_PCI_QUIRKS
1655	int pos, ret;
1656	u32 header, tmp;
1657
1658	pci_read_config_dword(dev, PCI_VENDOR_ID, val: &header);
1659
1660	for (pos = PCI_CFG_SPACE_SIZE;
1661	pos < PCI_CFG_SPACE_EXP_SIZE; pos += PCI_CFG_SPACE_SIZE) {
1662	ret = pci_read_config_dword(dev, where: pos, val: &tmp);
1663	if ((ret != PCIBIOS_SUCCESSFUL) || (header != tmp))
1664	return false;
1665	}
1666
1667	return true;
1668	#else
1669	return false;
1670	#endif
1671	}
1672
1673	/**
1674	* pci_cfg_space_size_ext - Get the configuration space size of the PCI device
1675	* @dev: PCI device
1676	*
1677	* Regular PCI devices have 256 bytes, but PCI-X 2 and PCI Express devices
1678	* have 4096 bytes. Even if the device is capable, that doesn't mean we can
1679	* access it. Maybe we don't have a way to generate extended config space
1680	* accesses, or the device is behind a reverse Express bridge. So we try
1681	* reading the dword at 0x100 which must either be 0 or a valid extended
1682	* capability header.
1683	*/
1684	static int pci_cfg_space_size_ext(struct pci_dev *dev)
1685	{
1686	u32 status;
1687	int pos = PCI_CFG_SPACE_SIZE;
1688
1689	if (pci_read_config_dword(dev, where: pos, val: &status) != PCIBIOS_SUCCESSFUL)
1690	return PCI_CFG_SPACE_SIZE;
1691	if (PCI_POSSIBLE_ERROR(status) || pci_ext_cfg_is_aliased(dev))
1692	return PCI_CFG_SPACE_SIZE;
1693
1694	return PCI_CFG_SPACE_EXP_SIZE;
1695	}
1696
1697	int pci_cfg_space_size(struct pci_dev *dev)
1698	{
1699	int pos;
1700	u32 status;
1701	u16 class;
1702
1703	#ifdef CONFIG_PCI_IOV
1704	/*
1705	* Per the SR-IOV specification (rev 1.1, sec 3.5), VFs are required to
1706	* implement a PCIe capability and therefore must implement extended
1707	* config space. We can skip the NO_EXTCFG test below and the
1708	* reachability/aliasing test in pci_cfg_space_size_ext() by virtue of
1709	* the fact that the SR-IOV capability on the PF resides in extended
1710	* config space and must be accessible and non-aliased to have enabled
1711	* support for this VF. This is a micro performance optimization for
1712	* systems supporting many VFs.
1713	*/
1714	if (dev->is_virtfn)
1715	return PCI_CFG_SPACE_EXP_SIZE;
1716	#endif
1717
1718	if (dev->bus->bus_flags & PCI_BUS_FLAGS_NO_EXTCFG)
1719	return PCI_CFG_SPACE_SIZE;
1720
1721	class = dev->class >> 8;
1722	if (class == PCI_CLASS_BRIDGE_HOST)
1723	return pci_cfg_space_size_ext(dev);
1724
1725	if (pci_is_pcie(dev))
1726	return pci_cfg_space_size_ext(dev);
1727
1728	pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
1729	if (!pos)
1730	return PCI_CFG_SPACE_SIZE;
1731
1732	pci_read_config_dword(dev, where: pos + PCI_X_STATUS, val: &status);
1733	if (status & (PCI_X_STATUS_266MHZ | PCI_X_STATUS_533MHZ))
1734	return pci_cfg_space_size_ext(dev);
1735
1736	return PCI_CFG_SPACE_SIZE;
1737	}
1738
1739	static u32 pci_class(struct pci_dev *dev)
1740	{
1741	u32 class;
1742
1743	#ifdef CONFIG_PCI_IOV
1744	if (dev->is_virtfn)
1745	return dev->physfn->sriov->class;
1746	#endif
1747	pci_read_config_dword(dev, PCI_CLASS_REVISION, val: &class);
1748	return class;
1749	}
1750
1751	static void pci_subsystem_ids(struct pci_dev *dev, u16 *vendor, u16 *device)
1752	{
1753	#ifdef CONFIG_PCI_IOV
1754	if (dev->is_virtfn) {
1755	*vendor = dev->physfn->sriov->subsystem_vendor;
1756	*device = dev->physfn->sriov->subsystem_device;
1757	return;
1758	}
1759	#endif
1760	pci_read_config_word(dev, PCI_SUBSYSTEM_VENDOR_ID, val: vendor);
1761	pci_read_config_word(dev, PCI_SUBSYSTEM_ID, val: device);
1762	}
1763
1764	static u8 pci_hdr_type(struct pci_dev *dev)
1765	{
1766	u8 hdr_type;
1767
1768	#ifdef CONFIG_PCI_IOV
1769	if (dev->is_virtfn)
1770	return dev->physfn->sriov->hdr_type;
1771	#endif
1772	pci_read_config_byte(dev, PCI_HEADER_TYPE, val: &hdr_type);
1773	return hdr_type;
1774	}
1775
1776	#define LEGACY_IO_RESOURCE (IORESOURCE_IO | IORESOURCE_PCI_FIXED)
1777
1778	/**
1779	* pci_intx_mask_broken - Test PCI_COMMAND_INTX_DISABLE writability
1780	* @dev: PCI device
1781	*
1782	* Test whether PCI_COMMAND_INTX_DISABLE is writable for @dev. Check this
1783	* at enumeration-time to avoid modifying PCI_COMMAND at run-time.
1784	*/
1785	static int pci_intx_mask_broken(struct pci_dev *dev)
1786	{
1787	u16 orig, toggle, new;
1788
1789	pci_read_config_word(dev, PCI_COMMAND, val: &orig);
1790	toggle = orig ^ PCI_COMMAND_INTX_DISABLE;
1791	pci_write_config_word(dev, PCI_COMMAND, val: toggle);
1792	pci_read_config_word(dev, PCI_COMMAND, val: &new);
1793
1794	pci_write_config_word(dev, PCI_COMMAND, val: orig);
1795
1796	/*
1797	* PCI_COMMAND_INTX_DISABLE was reserved and read-only prior to PCI
1798	* r2.3, so strictly speaking, a device is not *broken* if it's not
1799	* writable. But we'll live with the misnomer for now.
1800	*/
1801	if (new != toggle)
1802	return 1;
1803	return 0;
1804	}
1805
1806	static void early_dump_pci_device(struct pci_dev *pdev)
1807	{
1808	u32 value[256 / 4];
1809	int i;
1810
1811	pci_info(pdev, "config space:\n");
1812
1813	for (i = 0; i < 256; i += 4)
1814	pci_read_config_dword(dev: pdev, where: i, val: &value[i / 4]);
1815
1816	print_hex_dump(KERN_INFO, prefix_str: "", prefix_type: DUMP_PREFIX_OFFSET, rowsize: 16, groupsize: 1,
1817	buf: value, len: 256, ascii: false);
1818	}
1819
1820	/**
1821	* pci_setup_device - Fill in class and map information of a device
1822	* @dev: the device structure to fill
1823	*
1824	* Initialize the device structure with information about the device's
1825	* vendor,class,memory and IO-space addresses, IRQ lines etc.
1826	* Called at initialisation of the PCI subsystem and by CardBus services.
1827	* Returns 0 on success and negative if unknown type of device (not normal,
1828	* bridge or CardBus).
1829	*/
1830	int pci_setup_device(struct pci_dev *dev)
1831	{
1832	u32 class;
1833	u16 cmd;
1834	u8 hdr_type;
1835	int err, pos = 0;
1836	struct pci_bus_region region;
1837	struct resource *res;
1838
1839	hdr_type = pci_hdr_type(dev);
1840
1841	dev->sysdata = dev->bus->sysdata;
1842	dev->dev.parent = dev->bus->bridge;
1843	dev->dev.bus = &pci_bus_type;
1844	dev->hdr_type = hdr_type & 0x7f;
1845	dev->multifunction = !!(hdr_type & 0x80);
1846	dev->error_state = pci_channel_io_normal;
1847	set_pcie_port_type(dev);
1848
1849	err = pci_set_of_node(dev);
1850	if (err)
1851	return err;
1852	pci_set_acpi_fwnode(dev);
1853
1854	pci_dev_assign_slot(dev);
1855
1856	/*
1857	* Assume 32-bit PCI; let 64-bit PCI cards (which are far rarer)
1858	* set this higher, assuming the system even supports it.
1859	*/
1860	dev->dma_mask = 0xffffffff;
1861
1862	dev_set_name(dev: &dev->dev, name: "%04x:%02x:%02x.%d", pci_domain_nr(bus: dev->bus),
1863	dev->bus->number, PCI_SLOT(dev->devfn),
1864	PCI_FUNC(dev->devfn));
1865
1866	class = pci_class(dev);
1867
1868	dev->revision = class & 0xff;
1869	dev->class = class >> 8; /* upper 3 bytes */
1870
1871	if (pci_early_dump)
1872	early_dump_pci_device(pdev: dev);
1873
1874	/* Need to have dev->class ready */
1875	dev->cfg_size = pci_cfg_space_size(dev);
1876
1877	/* Need to have dev->cfg_size ready */
1878	set_pcie_thunderbolt(dev);
1879
1880	set_pcie_untrusted(dev);
1881
1882	/* "Unknown power state" */
1883	dev->current_state = PCI_UNKNOWN;
1884
1885	/* Early fixups, before probing the BARs */
1886	pci_fixup_device(pass: pci_fixup_early, dev);
1887
1888	pci_set_removable(dev);
1889
1890	pci_info(dev, "[%04x:%04x] type %02x class %#08x\n",
1891	dev->vendor, dev->device, dev->hdr_type, dev->class);
1892
1893	/* Device class may be changed after fixup */
1894	class = dev->class >> 8;
1895
1896	if (dev->non_compliant_bars && !dev->mmio_always_on) {
1897	pci_read_config_word(dev, PCI_COMMAND, val: &cmd);
1898	if (cmd & (PCI_COMMAND_IO | PCI_COMMAND_MEMORY)) {
1899	pci_info(dev, "device has non-compliant BARs; disabling IO/MEM decoding\n");
1900	cmd &= ~PCI_COMMAND_IO;
1901	cmd &= ~PCI_COMMAND_MEMORY;
1902	pci_write_config_word(dev, PCI_COMMAND, val: cmd);
1903	}
1904	}
1905
1906	dev->broken_intx_masking = pci_intx_mask_broken(dev);
1907
1908	switch (dev->hdr_type) { /* header type */
1909	case PCI_HEADER_TYPE_NORMAL: /* standard header */
1910	if (class == PCI_CLASS_BRIDGE_PCI)
1911	goto bad;
1912	pci_read_irq(dev);
1913	pci_read_bases(dev, howmany: 6, PCI_ROM_ADDRESS);
1914
1915	pci_subsystem_ids(dev, vendor: &dev->subsystem_vendor, device: &dev->subsystem_device);
1916
1917	/*
1918	* Do the ugly legacy mode stuff here rather than broken chip
1919	* quirk code. Legacy mode ATA controllers have fixed
1920	* addresses. These are not always echoed in BAR0-3, and
1921	* BAR0-3 in a few cases contain junk!
1922	*/
1923	if (class == PCI_CLASS_STORAGE_IDE) {
1924	u8 progif;
1925	pci_read_config_byte(dev, PCI_CLASS_PROG, val: &progif);
1926	if ((progif & 1) == 0) {
1927	region.start = 0x1F0;
1928	region.end = 0x1F7;
1929	res = &dev->resource[0];
1930	res->flags = LEGACY_IO_RESOURCE;
1931	pcibios_bus_to_resource(bus: dev->bus, res, region: &region);
1932	pci_info(dev, "legacy IDE quirk: reg 0x10: %pR\n",
1933	res);
1934	region.start = 0x3F6;
1935	region.end = 0x3F6;
1936	res = &dev->resource[1];
1937	res->flags = LEGACY_IO_RESOURCE;
1938	pcibios_bus_to_resource(bus: dev->bus, res, region: &region);
1939	pci_info(dev, "legacy IDE quirk: reg 0x14: %pR\n",
1940	res);
1941	}
1942	if ((progif & 4) == 0) {
1943	region.start = 0x170;
1944	region.end = 0x177;
1945	res = &dev->resource[2];
1946	res->flags = LEGACY_IO_RESOURCE;
1947	pcibios_bus_to_resource(bus: dev->bus, res, region: &region);
1948	pci_info(dev, "legacy IDE quirk: reg 0x18: %pR\n",
1949	res);
1950	region.start = 0x376;
1951	region.end = 0x376;
1952	res = &dev->resource[3];
1953	res->flags = LEGACY_IO_RESOURCE;
1954	pcibios_bus_to_resource(bus: dev->bus, res, region: &region);
1955	pci_info(dev, "legacy IDE quirk: reg 0x1c: %pR\n",
1956	res);
1957	}
1958	}
1959	break;
1960
1961	case PCI_HEADER_TYPE_BRIDGE: /* bridge header */
1962	/*
1963	* The PCI-to-PCI bridge spec requires that subtractive
1964	* decoding (i.e. transparent) bridge must have programming
1965	* interface code of 0x01.
1966	*/
1967	pci_read_irq(dev);
1968	dev->transparent = ((dev->class & 0xff) == 1);
1969	pci_read_bases(dev, howmany: 2, PCI_ROM_ADDRESS1);
1970	pci_read_bridge_windows(bridge: dev);
1971	set_pcie_hotplug_bridge(dev);
1972	pos = pci_find_capability(dev, PCI_CAP_ID_SSVID);
1973	if (pos) {
1974	pci_read_config_word(dev, where: pos + PCI_SSVID_VENDOR_ID, val: &dev->subsystem_vendor);
1975	pci_read_config_word(dev, where: pos + PCI_SSVID_DEVICE_ID, val: &dev->subsystem_device);
1976	}
1977	break;
1978
1979	case PCI_HEADER_TYPE_CARDBUS: /* CardBus bridge header */
1980	if (class != PCI_CLASS_BRIDGE_CARDBUS)
1981	goto bad;
1982	pci_read_irq(dev);
1983	pci_read_bases(dev, howmany: 1, rom: 0);
1984	pci_read_config_word(dev, PCI_CB_SUBSYSTEM_VENDOR_ID, val: &dev->subsystem_vendor);
1985	pci_read_config_word(dev, PCI_CB_SUBSYSTEM_ID, val: &dev->subsystem_device);
1986	break;
1987
1988	default: /* unknown header */
1989	pci_err(dev, "unknown header type %02x, ignoring device\n",
1990	dev->hdr_type);
1991	pci_release_of_node(dev);
1992	return -EIO;
1993
1994	bad:
1995	pci_err(dev, "ignoring class %#08x (doesn't match header type %02x)\n",
1996	dev->class, dev->hdr_type);
1997	dev->class = PCI_CLASS_NOT_DEFINED << 8;
1998	}
1999
2000	/* We found a fine healthy device, go go go... */
2001	return 0;
2002	}
2003
2004	static void pci_configure_mps(struct pci_dev *dev)
2005	{
2006	struct pci_dev *bridge = pci_upstream_bridge(dev);
2007	int mps, mpss, p_mps, rc;
2008
2009	if (!pci_is_pcie(dev))
2010	return;
2011
2012	/* MPS and MRRS fields are of type 'RsvdP' for VFs, short-circuit out */
2013	if (dev->is_virtfn)
2014	return;
2015
2016	/*
2017	* For Root Complex Integrated Endpoints, program the maximum
2018	* supported value unless limited by the PCIE_BUS_PEER2PEER case.
2019	*/
2020	if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END) {
2021	if (pcie_bus_config == PCIE_BUS_PEER2PEER)
2022	mps = 128;
2023	else
2024	mps = 128 << dev->pcie_mpss;
2025	rc = pcie_set_mps(dev, mps);
2026	if (rc) {
2027	pci_warn(dev, "can't set Max Payload Size to %d; if necessary, use \"pci=pcie_bus_safe\" and report a bug\n",
2028	mps);
2029	}
2030	return;
2031	}
2032
2033	if (!bridge || !pci_is_pcie(dev: bridge))
2034	return;
2035
2036	mps = pcie_get_mps(dev);
2037	p_mps = pcie_get_mps(dev: bridge);
2038
2039	if (mps == p_mps)
2040	return;
2041
2042	if (pcie_bus_config == PCIE_BUS_TUNE_OFF) {
2043	pci_warn(dev, "Max Payload Size %d, but upstream %s set to %d; if necessary, use \"pci=pcie_bus_safe\" and report a bug\n",
2044	mps, pci_name(bridge), p_mps);
2045	return;
2046	}
2047
2048	/*
2049	* Fancier MPS configuration is done later by
2050	* pcie_bus_configure_settings()
2051	*/
2052	if (pcie_bus_config != PCIE_BUS_DEFAULT)
2053	return;
2054
2055	mpss = 128 << dev->pcie_mpss;
2056	if (mpss < p_mps && pci_pcie_type(dev: bridge) == PCI_EXP_TYPE_ROOT_PORT) {
2057	pcie_set_mps(dev: bridge, mps: mpss);
2058	pci_info(dev, "Upstream bridge's Max Payload Size set to %d (was %d, max %d)\n",
2059	mpss, p_mps, 128 << bridge->pcie_mpss);
2060	p_mps = pcie_get_mps(dev: bridge);
2061	}
2062
2063	rc = pcie_set_mps(dev, mps: p_mps);
2064	if (rc) {
2065	pci_warn(dev, "can't set Max Payload Size to %d; if necessary, use \"pci=pcie_bus_safe\" and report a bug\n",
2066	p_mps);
2067	return;
2068	}
2069
2070	pci_info(dev, "Max Payload Size set to %d (was %d, max %d)\n",
2071	p_mps, mps, mpss);
2072	}
2073
2074	int pci_configure_extended_tags(struct pci_dev *dev, void *ign)
2075	{
2076	struct pci_host_bridge *host;
2077	u32 cap;
2078	u16 ctl;
2079	int ret;
2080
2081	if (!pci_is_pcie(dev))
2082	return 0;
2083
2084	ret = pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, val: &cap);
2085	if (ret)
2086	return 0;
2087
2088	if (!(cap & PCI_EXP_DEVCAP_EXT_TAG))
2089	return 0;
2090
2091	ret = pcie_capability_read_word(dev, PCI_EXP_DEVCTL, val: &ctl);
2092	if (ret)
2093	return 0;
2094
2095	host = pci_find_host_bridge(bus: dev->bus);
2096	if (!host)
2097	return 0;
2098
2099	/*
2100	* If some device in the hierarchy doesn't handle Extended Tags
2101	* correctly, make sure they're disabled.
2102	*/
2103	if (host->no_ext_tags) {
2104	if (ctl & PCI_EXP_DEVCTL_EXT_TAG) {
2105	pci_info(dev, "disabling Extended Tags\n");
2106	pcie_capability_clear_word(dev, PCI_EXP_DEVCTL,
2107	PCI_EXP_DEVCTL_EXT_TAG);
2108	}
2109	return 0;
2110	}
2111
2112	if (!(ctl & PCI_EXP_DEVCTL_EXT_TAG)) {
2113	pci_info(dev, "enabling Extended Tags\n");
2114	pcie_capability_set_word(dev, PCI_EXP_DEVCTL,
2115	PCI_EXP_DEVCTL_EXT_TAG);
2116	}
2117	return 0;
2118	}
2119
2120	/**
2121	* pcie_relaxed_ordering_enabled - Probe for PCIe relaxed ordering enable
2122	* @dev: PCI device to query
2123	*
2124	* Returns true if the device has enabled relaxed ordering attribute.
2125	*/
2126	bool pcie_relaxed_ordering_enabled(struct pci_dev *dev)
2127	{
2128	u16 v;
2129
2130	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, val: &v);
2131
2132	return !!(v & PCI_EXP_DEVCTL_RELAX_EN);
2133	}
2134	EXPORT_SYMBOL(pcie_relaxed_ordering_enabled);
2135
2136	static void pci_configure_relaxed_ordering(struct pci_dev *dev)
2137	{
2138	struct pci_dev *root;
2139
2140	/* PCI_EXP_DEVCTL_RELAX_EN is RsvdP in VFs */
2141	if (dev->is_virtfn)
2142	return;
2143
2144	if (!pcie_relaxed_ordering_enabled(dev))
2145	return;
2146
2147	/*
2148	* For now, we only deal with Relaxed Ordering issues with Root
2149	* Ports. Peer-to-Peer DMA is another can of worms.
2150	*/
2151	root = pcie_find_root_port(dev);
2152	if (!root)
2153	return;
2154
2155	if (root->dev_flags & PCI_DEV_FLAGS_NO_RELAXED_ORDERING) {
2156	pcie_capability_clear_word(dev, PCI_EXP_DEVCTL,
2157	PCI_EXP_DEVCTL_RELAX_EN);
2158	pci_info(dev, "Relaxed Ordering disabled because the Root Port didn't support it\n");
2159	}
2160	}
2161
2162	static void pci_configure_ltr(struct pci_dev *dev)
2163	{
2164	#ifdef CONFIG_PCIEASPM
2165	struct pci_host_bridge *host = pci_find_host_bridge(bus: dev->bus);
2166	struct pci_dev *bridge;
2167	u32 cap, ctl;
2168
2169	if (!pci_is_pcie(dev))
2170	return;
2171
2172	/* Read L1 PM substate capabilities */
2173	dev->l1ss = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_L1SS);
2174
2175	pcie_capability_read_dword(dev, PCI_EXP_DEVCAP2, val: &cap);
2176	if (!(cap & PCI_EXP_DEVCAP2_LTR))
2177	return;
2178
2179	pcie_capability_read_dword(dev, PCI_EXP_DEVCTL2, val: &ctl);
2180	if (ctl & PCI_EXP_DEVCTL2_LTR_EN) {
2181	if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) {
2182	dev->ltr_path = 1;
2183	return;
2184	}
2185
2186	bridge = pci_upstream_bridge(dev);
2187	if (bridge && bridge->ltr_path)
2188	dev->ltr_path = 1;
2189
2190	return;
2191	}
2192
2193	if (!host->native_ltr)
2194	return;
2195
2196	/*
2197	* Software must not enable LTR in an Endpoint unless the Root
2198	* Complex and all intermediate Switches indicate support for LTR.
2199	* PCIe r4.0, sec 6.18.
2200	*/
2201	if (pci_pcie_type(dev) == PCI_EXP_TYPE_ROOT_PORT) {
2202	pcie_capability_set_word(dev, PCI_EXP_DEVCTL2,
2203	PCI_EXP_DEVCTL2_LTR_EN);
2204	dev->ltr_path = 1;
2205	return;
2206	}
2207
2208	/*
2209	* If we're configuring a hot-added device, LTR was likely
2210	* disabled in the upstream bridge, so re-enable it before enabling
2211	* it in the new device.
2212	*/
2213	bridge = pci_upstream_bridge(dev);
2214	if (bridge && bridge->ltr_path) {
2215	pci_bridge_reconfigure_ltr(dev);
2216	pcie_capability_set_word(dev, PCI_EXP_DEVCTL2,
2217	PCI_EXP_DEVCTL2_LTR_EN);
2218	dev->ltr_path = 1;
2219	}
2220	#endif
2221	}
2222
2223	static void pci_configure_eetlp_prefix(struct pci_dev *dev)
2224	{
2225	#ifdef CONFIG_PCI_PASID
2226	struct pci_dev *bridge;
2227	int pcie_type;
2228	u32 cap;
2229
2230	if (!pci_is_pcie(dev))
2231	return;
2232
2233	pcie_capability_read_dword(dev, PCI_EXP_DEVCAP2, val: &cap);
2234	if (!(cap & PCI_EXP_DEVCAP2_EE_PREFIX))
2235	return;
2236
2237	pcie_type = pci_pcie_type(dev);
2238	if (pcie_type == PCI_EXP_TYPE_ROOT_PORT ||
2239	pcie_type == PCI_EXP_TYPE_RC_END)
2240	dev->eetlp_prefix_path = 1;
2241	else {
2242	bridge = pci_upstream_bridge(dev);
2243	if (bridge && bridge->eetlp_prefix_path)
2244	dev->eetlp_prefix_path = 1;
2245	}
2246	#endif
2247	}
2248
2249	static void pci_configure_serr(struct pci_dev *dev)
2250	{
2251	u16 control;
2252
2253	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
2254
2255	/*
2256	* A bridge will not forward ERR_ messages coming from an
2257	* endpoint unless SERR# forwarding is enabled.
2258	*/
2259	pci_read_config_word(dev, PCI_BRIDGE_CONTROL, val: &control);
2260	if (!(control & PCI_BRIDGE_CTL_SERR)) {
2261	control |= PCI_BRIDGE_CTL_SERR;
2262	pci_write_config_word(dev, PCI_BRIDGE_CONTROL, val: control);
2263	}
2264	}
2265	}
2266
2267	static void pci_configure_device(struct pci_dev *dev)
2268	{
2269	pci_configure_mps(dev);
2270	pci_configure_extended_tags(dev, NULL);
2271	pci_configure_relaxed_ordering(dev);
2272	pci_configure_ltr(dev);
2273	pci_configure_eetlp_prefix(dev);
2274	pci_configure_serr(dev);
2275
2276	pci_acpi_program_hp_params(dev);
2277	}
2278
2279	static void pci_release_capabilities(struct pci_dev *dev)
2280	{
2281	pci_aer_exit(dev);
2282	pci_rcec_exit(dev);
2283	pci_iov_release(dev);
2284	pci_free_cap_save_buffers(dev);
2285	}
2286
2287	/**
2288	* pci_release_dev - Free a PCI device structure when all users of it are
2289	* finished
2290	* @dev: device that's been disconnected
2291	*
2292	* Will be called only by the device core when all users of this PCI device are
2293	* done.
2294	*/
2295	static void pci_release_dev(struct device *dev)
2296	{
2297	struct pci_dev *pci_dev;
2298
2299	pci_dev = to_pci_dev(dev);
2300	pci_release_capabilities(dev: pci_dev);
2301	pci_release_of_node(dev: pci_dev);
2302	pcibios_release_device(dev: pci_dev);
2303	pci_bus_put(bus: pci_dev->bus);
2304	kfree(objp: pci_dev->driver_override);
2305	bitmap_free(bitmap: pci_dev->dma_alias_mask);
2306	dev_dbg(dev, "device released\n");
2307	kfree(objp: pci_dev);
2308	}
2309
2310	struct pci_dev *pci_alloc_dev(struct pci_bus *bus)
2311	{
2312	struct pci_dev *dev;
2313
2314	dev = kzalloc(size: sizeof(struct pci_dev), GFP_KERNEL);
2315	if (!dev)
2316	return NULL;
2317
2318	INIT_LIST_HEAD(list: &dev->bus_list);
2319	dev->dev.type = &pci_dev_type;
2320	dev->bus = pci_bus_get(bus);
2321	dev->driver_exclusive_resource = (struct resource) {
2322	.name = "PCI Exclusive",
2323	.start = 0,
2324	.end = -1,
2325	};
2326
2327	spin_lock_init(&dev->pcie_cap_lock);
2328	#ifdef CONFIG_PCI_MSI
2329	raw_spin_lock_init(&dev->msi_lock);
2330	#endif
2331	return dev;
2332	}
2333	EXPORT_SYMBOL(pci_alloc_dev);
2334
2335	static bool pci_bus_crs_vendor_id(u32 l)
2336	{
2337	return (l & 0xffff) == PCI_VENDOR_ID_PCI_SIG;
2338	}
2339
2340	static bool pci_bus_wait_crs(struct pci_bus *bus, int devfn, u32 *l,
2341	int timeout)
2342	{
2343	int delay = 1;
2344
2345	if (!pci_bus_crs_vendor_id(l: *l))
2346	return true; /* not a CRS completion */
2347
2348	if (!timeout)
2349	return false; /* CRS, but caller doesn't want to wait */
2350
2351	/*
2352	* We got the reserved Vendor ID that indicates a completion with
2353	* Configuration Request Retry Status (CRS). Retry until we get a
2354	* valid Vendor ID or we time out.
2355	*/
2356	while (pci_bus_crs_vendor_id(l: *l)) {
2357	if (delay > timeout) {
2358	pr_warn("pci %04x:%02x:%02x.%d: not ready after %dms; giving up\n",
2359	pci_domain_nr(bus), bus->number,
2360	PCI_SLOT(devfn), PCI_FUNC(devfn), delay - 1);
2361
2362	return false;
2363	}
2364	if (delay >= 1000)
2365	pr_info("pci %04x:%02x:%02x.%d: not ready after %dms; waiting\n",
2366	pci_domain_nr(bus), bus->number,
2367	PCI_SLOT(devfn), PCI_FUNC(devfn), delay - 1);
2368
2369	msleep(msecs: delay);
2370	delay *= 2;
2371
2372	if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, val: l))
2373	return false;
2374	}
2375
2376	if (delay >= 1000)
2377	pr_info("pci %04x:%02x:%02x.%d: ready after %dms\n",
2378	pci_domain_nr(bus), bus->number,
2379	PCI_SLOT(devfn), PCI_FUNC(devfn), delay - 1);
2380
2381	return true;
2382	}
2383
2384	bool pci_bus_generic_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
2385	int timeout)
2386	{
2387	if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, val: l))
2388	return false;
2389
2390	/* Some broken boards return 0 or ~0 (PCI_ERROR_RESPONSE) if a slot is empty: */
2391	if (PCI_POSSIBLE_ERROR(*l) || *l == 0x00000000 ||
2392	*l == 0x0000ffff || *l == 0xffff0000)
2393	return false;
2394
2395	if (pci_bus_crs_vendor_id(l: *l))
2396	return pci_bus_wait_crs(bus, devfn, l, timeout);
2397
2398	return true;
2399	}
2400
2401	bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
2402	int timeout)
2403	{
2404	#ifdef CONFIG_PCI_QUIRKS
2405	struct pci_dev *bridge = bus->self;
2406
2407	/*
2408	* Certain IDT switches have an issue where they improperly trigger
2409	* ACS Source Validation errors on completions for config reads.
2410	*/
2411	if (bridge && bridge->vendor == PCI_VENDOR_ID_IDT &&
2412	bridge->device == 0x80b5)
2413	return pci_idt_bus_quirk(bus, devfn, pl: l, crs_timeout: timeout);
2414	#endif
2415
2416	return pci_bus_generic_read_dev_vendor_id(bus, devfn, l, timeout);
2417	}
2418	EXPORT_SYMBOL(pci_bus_read_dev_vendor_id);
2419
2420	/*
2421	* Read the config data for a PCI device, sanity-check it,
2422	* and fill in the dev structure.
2423	*/
2424	static struct pci_dev *pci_scan_device(struct pci_bus *bus, int devfn)
2425	{
2426	struct pci_dev *dev;
2427	u32 l;
2428
2429	if (!pci_bus_read_dev_vendor_id(bus, devfn, &l, 60*1000))
2430	return NULL;
2431
2432	dev = pci_alloc_dev(bus);
2433	if (!dev)
2434	return NULL;
2435
2436	dev->devfn = devfn;
2437	dev->vendor = l & 0xffff;
2438	dev->device = (l >> 16) & 0xffff;
2439
2440	if (pci_setup_device(dev)) {
2441	pci_bus_put(bus: dev->bus);
2442	kfree(objp: dev);
2443	return NULL;
2444	}
2445
2446	return dev;
2447	}
2448
2449	void pcie_report_downtraining(struct pci_dev *dev)
2450	{
2451	if (!pci_is_pcie(dev))
2452	return;
2453
2454	/* Look from the device up to avoid downstream ports with no devices */
2455	if ((pci_pcie_type(dev) != PCI_EXP_TYPE_ENDPOINT) &&
2456	(pci_pcie_type(dev) != PCI_EXP_TYPE_LEG_END) &&
2457	(pci_pcie_type(dev) != PCI_EXP_TYPE_UPSTREAM))
2458	return;
2459
2460	/* Multi-function PCIe devices share the same link/status */
2461	if (PCI_FUNC(dev->devfn) != 0 || dev->is_virtfn)
2462	return;
2463
2464	/* Print link status only if the device is constrained by the fabric */
2465	__pcie_print_link_status(dev, verbose: false);
2466	}
2467
2468	static void pci_init_capabilities(struct pci_dev *dev)
2469	{
2470	pci_ea_init(dev); /* Enhanced Allocation */
2471	pci_msi_init(dev); /* Disable MSI */
2472	pci_msix_init(dev); /* Disable MSI-X */
2473
2474	/* Buffers for saving PCIe and PCI-X capabilities */
2475	pci_allocate_cap_save_buffers(dev);
2476
2477	pci_pm_init(dev); /* Power Management */
2478	pci_vpd_init(dev); /* Vital Product Data */
2479	pci_configure_ari(dev); /* Alternative Routing-ID Forwarding */
2480	pci_iov_init(dev); /* Single Root I/O Virtualization */
2481	pci_ats_init(dev); /* Address Translation Services */
2482	pci_pri_init(dev); /* Page Request Interface */
2483	pci_pasid_init(dev); /* Process Address Space ID */
2484	pci_acs_init(dev); /* Access Control Services */
2485	pci_ptm_init(dev); /* Precision Time Measurement */
2486	pci_aer_init(dev); /* Advanced Error Reporting */
2487	pci_dpc_init(pdev: dev); /* Downstream Port Containment */
2488	pci_rcec_init(dev); /* Root Complex Event Collector */
2489	pci_doe_init(pdev: dev); /* Data Object Exchange */
2490
2491	pcie_report_downtraining(dev);
2492	pci_init_reset_methods(dev);
2493	}
2494
2495	/*
2496	* This is the equivalent of pci_host_bridge_msi_domain() that acts on
2497	* devices. Firmware interfaces that can select the MSI domain on a
2498	* per-device basis should be called from here.
2499	*/
2500	static struct irq_domain *pci_dev_msi_domain(struct pci_dev *dev)
2501	{
2502	struct irq_domain *d;
2503
2504	/*
2505	* If a domain has been set through the pcibios_device_add()
2506	* callback, then this is the one (platform code knows best).
2507	*/
2508	d = dev_get_msi_domain(dev: &dev->dev);
2509	if (d)
2510	return d;
2511
2512	/*
2513	* Let's see if we have a firmware interface able to provide
2514	* the domain.
2515	*/
2516	d = pci_msi_get_device_domain(pdev: dev);
2517	if (d)
2518	return d;
2519
2520	return NULL;
2521	}
2522
2523	static void pci_set_msi_domain(struct pci_dev *dev)
2524	{
2525	struct irq_domain *d;
2526
2527	/*
2528	* If the platform or firmware interfaces cannot supply a
2529	* device-specific MSI domain, then inherit the default domain
2530	* from the host bridge itself.
2531	*/
2532	d = pci_dev_msi_domain(dev);
2533	if (!d)
2534	d = dev_get_msi_domain(dev: &dev->bus->dev);
2535
2536	dev_set_msi_domain(dev: &dev->dev, d);
2537	}
2538
2539	void pci_device_add(struct pci_dev *dev, struct pci_bus *bus)
2540	{
2541	int ret;
2542
2543	pci_configure_device(dev);
2544
2545	device_initialize(dev: &dev->dev);
2546	dev->dev.release = pci_release_dev;
2547
2548	set_dev_node(dev: &dev->dev, pcibus_to_node(bus));
2549	dev->dev.dma_mask = &dev->dma_mask;
2550	dev->dev.dma_parms = &dev->dma_parms;
2551	dev->dev.coherent_dma_mask = 0xffffffffull;
2552
2553	dma_set_max_seg_size(dev: &dev->dev, size: 65536);
2554	dma_set_seg_boundary(dev: &dev->dev, mask: 0xffffffff);
2555
2556	pcie_failed_link_retrain(dev);
2557
2558	/* Fix up broken headers */
2559	pci_fixup_device(pass: pci_fixup_header, dev);
2560
2561	pci_reassigndev_resource_alignment(dev);
2562
2563	dev->state_saved = false;
2564
2565	pci_init_capabilities(dev);
2566
2567	/*
2568	* Add the device to our list of discovered devices
2569	* and the bus list for fixup functions, etc.
2570	*/
2571	down_write(sem: &pci_bus_sem);
2572	list_add_tail(new: &dev->bus_list, head: &bus->devices);
2573	up_write(sem: &pci_bus_sem);
2574
2575	ret = pcibios_device_add(dev);
2576	WARN_ON(ret < 0);
2577
2578	/* Set up MSI IRQ domain */
2579	pci_set_msi_domain(dev);
2580
2581	/* Notifier could use PCI capabilities */
2582	dev->match_driver = false;
2583	ret = device_add(dev: &dev->dev);
2584	WARN_ON(ret < 0);
2585	}
2586
2587	struct pci_dev *pci_scan_single_device(struct pci_bus *bus, int devfn)
2588	{
2589	struct pci_dev *dev;
2590
2591	dev = pci_get_slot(bus, devfn);
2592	if (dev) {
2593	pci_dev_put(dev);
2594	return dev;
2595	}
2596
2597	dev = pci_scan_device(bus, devfn);
2598	if (!dev)
2599	return NULL;
2600
2601	pci_device_add(dev, bus);
2602
2603	return dev;
2604	}
2605	EXPORT_SYMBOL(pci_scan_single_device);
2606
2607	static int next_ari_fn(struct pci_bus *bus, struct pci_dev *dev, int fn)
2608	{
2609	int pos;
2610	u16 cap = 0;
2611	unsigned int next_fn;
2612
2613	if (!dev)
2614	return -ENODEV;
2615
2616	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI);
2617	if (!pos)
2618	return -ENODEV;
2619
2620	pci_read_config_word(dev, where: pos + PCI_ARI_CAP, val: &cap);
2621	next_fn = PCI_ARI_CAP_NFN(cap);
2622	if (next_fn <= fn)
2623	return -ENODEV; /* protect against malformed list */
2624
2625	return next_fn;
2626	}
2627
2628	static int next_fn(struct pci_bus *bus, struct pci_dev *dev, int fn)
2629	{
2630	if (pci_ari_enabled(bus))
2631	return next_ari_fn(bus, dev, fn);
2632
2633	if (fn >= 7)
2634	return -ENODEV;
2635	/* only multifunction devices may have more functions */
2636	if (dev && !dev->multifunction)
2637	return -ENODEV;
2638
2639	return fn + 1;
2640	}
2641
2642	static int only_one_child(struct pci_bus *bus)
2643	{
2644	struct pci_dev *bridge = bus->self;
2645
2646	/*
2647	* Systems with unusual topologies set PCI_SCAN_ALL_PCIE_DEVS so
2648	* we scan for all possible devices, not just Device 0.
2649	*/
2650	if (pci_has_flag(flag: PCI_SCAN_ALL_PCIE_DEVS))
2651	return 0;
2652
2653	/*
2654	* A PCIe Downstream Port normally leads to a Link with only Device
2655	* 0 on it (PCIe spec r3.1, sec 7.3.1). As an optimization, scan
2656	* only for Device 0 in that situation.
2657	*/
2658	if (bridge && pci_is_pcie(dev: bridge) && pcie_downstream_port(dev: bridge))
2659	return 1;
2660
2661	return 0;
2662	}
2663
2664	/**
2665	* pci_scan_slot - Scan a PCI slot on a bus for devices
2666	* @bus: PCI bus to scan
2667	* @devfn: slot number to scan (must have zero function)
2668	*
2669	* Scan a PCI slot on the specified PCI bus for devices, adding
2670	* discovered devices to the @bus->devices list. New devices
2671	* will not have is_added set.
2672	*
2673	* Returns the number of new devices found.
2674	*/
2675	int pci_scan_slot(struct pci_bus *bus, int devfn)
2676	{
2677	struct pci_dev *dev;
2678	int fn = 0, nr = 0;
2679
2680	if (only_one_child(bus) && (devfn > 0))
2681	return 0; /* Already scanned the entire slot */
2682
2683	do {
2684	dev = pci_scan_single_device(bus, devfn + fn);
2685	if (dev) {
2686	if (!pci_dev_is_added(dev))
2687	nr++;
2688	if (fn > 0)
2689	dev->multifunction = 1;
2690	} else if (fn == 0) {
2691	/*
2692	* Function 0 is required unless we are running on
2693	* a hypervisor that passes through individual PCI
2694	* functions.
2695	*/
2696	if (!hypervisor_isolated_pci_functions())
2697	break;
2698	}
2699	fn = next_fn(bus, dev, fn);
2700	} while (fn >= 0);
2701
2702	/* Only one slot has PCIe device */
2703	if (bus->self && nr)
2704	pcie_aspm_init_link_state(pdev: bus->self);
2705
2706	return nr;
2707	}
2708	EXPORT_SYMBOL(pci_scan_slot);
2709
2710	static int pcie_find_smpss(struct pci_dev *dev, void *data)
2711	{
2712	u8 *smpss = data;
2713
2714	if (!pci_is_pcie(dev))
2715	return 0;
2716
2717	/*
2718	* We don't have a way to change MPS settings on devices that have
2719	* drivers attached. A hot-added device might support only the minimum
2720	* MPS setting (MPS=128). Therefore, if the fabric contains a bridge
2721	* where devices may be hot-added, we limit the fabric MPS to 128 so
2722	* hot-added devices will work correctly.
2723	*
2724	* However, if we hot-add a device to a slot directly below a Root
2725	* Port, it's impossible for there to be other existing devices below
2726	* the port. We don't limit the MPS in this case because we can
2727	* reconfigure MPS on both the Root Port and the hot-added device,
2728	* and there are no other devices involved.
2729	*
2730	* Note that this PCIE_BUS_SAFE path assumes no peer-to-peer DMA.
2731	*/
2732	if (dev->is_hotplug_bridge &&
2733	pci_pcie_type(dev) != PCI_EXP_TYPE_ROOT_PORT)
2734	*smpss = 0;
2735
2736	if (*smpss > dev->pcie_mpss)
2737	*smpss = dev->pcie_mpss;
2738
2739	return 0;
2740	}
2741
2742	static void pcie_write_mps(struct pci_dev *dev, int mps)
2743	{
2744	int rc;
2745
2746	if (pcie_bus_config == PCIE_BUS_PERFORMANCE) {
2747	mps = 128 << dev->pcie_mpss;
2748
2749	if (pci_pcie_type(dev) != PCI_EXP_TYPE_ROOT_PORT &&
2750	dev->bus->self)
2751
2752	/*
2753	* For "Performance", the assumption is made that
2754	* downstream communication will never be larger than
2755	* the MRRS. So, the MPS only needs to be configured
2756	* for the upstream communication. This being the case,
2757	* walk from the top down and set the MPS of the child
2758	* to that of the parent bus.
2759	*
2760	* Configure the device MPS with the smaller of the
2761	* device MPSS or the bridge MPS (which is assumed to be
2762	* properly configured at this point to the largest
2763	* allowable MPS based on its parent bus).
2764	*/
2765	mps = min(mps, pcie_get_mps(dev->bus->self));
2766	}
2767
2768	rc = pcie_set_mps(dev, mps);
2769	if (rc)
2770	pci_err(dev, "Failed attempting to set the MPS\n");
2771	}
2772
2773	static void pcie_write_mrrs(struct pci_dev *dev)
2774	{
2775	int rc, mrrs;
2776
2777	/*
2778	* In the "safe" case, do not configure the MRRS. There appear to be
2779	* issues with setting MRRS to 0 on a number of devices.
2780	*/
2781	if (pcie_bus_config != PCIE_BUS_PERFORMANCE)
2782	return;
2783
2784	/*
2785	* For max performance, the MRRS must be set to the largest supported
2786	* value. However, it cannot be configured larger than the MPS the
2787	* device or the bus can support. This should already be properly
2788	* configured by a prior call to pcie_write_mps().
2789	*/
2790	mrrs = pcie_get_mps(dev);
2791
2792	/*
2793	* MRRS is a R/W register. Invalid values can be written, but a
2794	* subsequent read will verify if the value is acceptable or not.
2795	* If the MRRS value provided is not acceptable (e.g., too large),
2796	* shrink the value until it is acceptable to the HW.
2797	*/
2798	while (mrrs != pcie_get_readrq(dev) && mrrs >= 128) {
2799	rc = pcie_set_readrq(dev, rq: mrrs);
2800	if (!rc)
2801	break;
2802
2803	pci_warn(dev, "Failed attempting to set the MRRS\n");
2804	mrrs /= 2;
2805	}
2806
2807	if (mrrs < 128)
2808	pci_err(dev, "MRRS was unable to be configured with a safe value. If problems are experienced, try running with pci=pcie_bus_safe\n");
2809	}
2810
2811	static int pcie_bus_configure_set(struct pci_dev *dev, void *data)
2812	{
2813	int mps, orig_mps;
2814
2815	if (!pci_is_pcie(dev))
2816	return 0;
2817
2818	if (pcie_bus_config == PCIE_BUS_TUNE_OFF ||
2819	pcie_bus_config == PCIE_BUS_DEFAULT)
2820	return 0;
2821
2822	mps = 128 << *(u8 *)data;
2823	orig_mps = pcie_get_mps(dev);
2824
2825	pcie_write_mps(dev, mps);
2826	pcie_write_mrrs(dev);
2827
2828	pci_info(dev, "Max Payload Size set to %4d/%4d (was %4d), Max Read Rq %4d\n",
2829	pcie_get_mps(dev), 128 << dev->pcie_mpss,
2830	orig_mps, pcie_get_readrq(dev));
2831
2832	return 0;
2833	}
2834
2835	/*
2836	* pcie_bus_configure_settings() requires that pci_walk_bus work in a top-down,
2837	* parents then children fashion. If this changes, then this code will not
2838	* work as designed.
2839	*/
2840	void pcie_bus_configure_settings(struct pci_bus *bus)
2841	{
2842	u8 smpss = 0;
2843
2844	if (!bus->self)
2845	return;
2846
2847	if (!pci_is_pcie(dev: bus->self))
2848	return;
2849
2850	/*
2851	* FIXME - Peer to peer DMA is possible, though the endpoint would need
2852	* to be aware of the MPS of the destination. To work around this,
2853	* simply force the MPS of the entire system to the smallest possible.
2854	*/
2855	if (pcie_bus_config == PCIE_BUS_PEER2PEER)
2856	smpss = 0;
2857
2858	if (pcie_bus_config == PCIE_BUS_SAFE) {
2859	smpss = bus->self->pcie_mpss;
2860
2861	pcie_find_smpss(dev: bus->self, data: &smpss);
2862	pci_walk_bus(top: bus, cb: pcie_find_smpss, userdata: &smpss);
2863	}
2864
2865	pcie_bus_configure_set(dev: bus->self, data: &smpss);
2866	pci_walk_bus(top: bus, cb: pcie_bus_configure_set, userdata: &smpss);
2867	}
2868	EXPORT_SYMBOL_GPL(pcie_bus_configure_settings);
2869
2870	/*
2871	* Called after each bus is probed, but before its children are examined. This
2872	* is marked as __weak because multiple architectures define it.
2873	*/
2874	void __weak pcibios_fixup_bus(struct pci_bus *bus)
2875	{
2876	/* nothing to do, expected to be removed in the future */
2877	}
2878
2879	/**
2880	* pci_scan_child_bus_extend() - Scan devices below a bus
2881	* @bus: Bus to scan for devices
2882	* @available_buses: Total number of buses available (%0 does not try to
2883	* extend beyond the minimal)
2884	*
2885	* Scans devices below @bus including subordinate buses. Returns new
2886	* subordinate number including all the found devices. Passing
2887	* @available_buses causes the remaining bus space to be distributed
2888	* equally between hotplug-capable bridges to allow future extension of the
2889	* hierarchy.
2890	*/
2891	static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus,
2892	unsigned int available_buses)
2893	{
2894	unsigned int used_buses, normal_bridges = 0, hotplug_bridges = 0;
2895	unsigned int start = bus->busn_res.start;
2896	unsigned int devfn, cmax, max = start;
2897	struct pci_dev *dev;
2898
2899	dev_dbg(&bus->dev, "scanning bus\n");
2900
2901	/* Go find them, Rover! */
2902	for (devfn = 0; devfn < 256; devfn += 8)
2903	pci_scan_slot(bus, devfn);
2904
2905	/* Reserve buses for SR-IOV capability */
2906	used_buses = pci_iov_bus_range(bus);
2907	max += used_buses;
2908
2909	/*
2910	* After performing arch-dependent fixup of the bus, look behind
2911	* all PCI-to-PCI bridges on this bus.
2912	*/
2913	if (!bus->is_added) {
2914	dev_dbg(&bus->dev, "fixups for bus\n");
2915	pcibios_fixup_bus(bus);
2916	bus->is_added = 1;
2917	}
2918
2919	/*
2920	* Calculate how many hotplug bridges and normal bridges there
2921	* are on this bus. We will distribute the additional available
2922	* buses between hotplug bridges.
2923	*/
2924	for_each_pci_bridge(dev, bus) {
2925	if (dev->is_hotplug_bridge)
2926	hotplug_bridges++;
2927	else
2928	normal_bridges++;
2929	}
2930
2931	/*
2932	* Scan bridges that are already configured. We don't touch them
2933	* unless they are misconfigured (which will be done in the second
2934	* scan below).
2935	*/
2936	for_each_pci_bridge(dev, bus) {
2937	cmax = max;
2938	max = pci_scan_bridge_extend(bus, dev, max, available_buses: 0, pass: 0);
2939
2940	/*
2941	* Reserve one bus for each bridge now to avoid extending
2942	* hotplug bridges too much during the second scan below.
2943	*/
2944	used_buses++;
2945	if (max - cmax > 1)
2946	used_buses += max - cmax - 1;
2947	}
2948
2949	/* Scan bridges that need to be reconfigured */
2950	for_each_pci_bridge(dev, bus) {
2951	unsigned int buses = 0;
2952
2953	if (!hotplug_bridges && normal_bridges == 1) {
2954	/*
2955	* There is only one bridge on the bus (upstream
2956	* port) so it gets all available buses which it
2957	* can then distribute to the possible hotplug
2958	* bridges below.
2959	*/
2960	buses = available_buses;
2961	} else if (dev->is_hotplug_bridge) {
2962	/*
2963	* Distribute the extra buses between hotplug
2964	* bridges if any.
2965	*/
2966	buses = available_buses / hotplug_bridges;
2967	buses = min(buses, available_buses - used_buses + 1);
2968	}
2969
2970	cmax = max;
2971	max = pci_scan_bridge_extend(bus, dev, max: cmax, available_buses: buses, pass: 1);
2972	/* One bus is already accounted so don't add it again */
2973	if (max - cmax > 1)
2974	used_buses += max - cmax - 1;
2975	}
2976
2977	/*
2978	* Make sure a hotplug bridge has at least the minimum requested
2979	* number of buses but allow it to grow up to the maximum available
2980	* bus number if there is room.
2981	*/
2982	if (bus->self && bus->self->is_hotplug_bridge) {
2983	used_buses = max_t(unsigned int, available_buses,
2984	pci_hotplug_bus_size - 1);
2985	if (max - start < used_buses) {
2986	max = start + used_buses;
2987
2988	/* Do not allocate more buses than we have room left */
2989	if (max > bus->busn_res.end)
2990	max = bus->busn_res.end;
2991
2992	dev_dbg(&bus->dev, "%pR extended by %#02x\n",
2993	&bus->busn_res, max - start);
2994	}
2995	}
2996
2997	/*
2998	* We've scanned the bus and so we know all about what's on
2999	* the other side of any bridges that may be on this bus plus
3000	* any devices.
3001	*
3002	* Return how far we've got finding sub-buses.
3003	*/
3004	dev_dbg(&bus->dev, "bus scan returning with max=%02x\n", max);
3005	return max;
3006	}
3007
3008	/**
3009	* pci_scan_child_bus() - Scan devices below a bus
3010	* @bus: Bus to scan for devices
3011	*
3012	* Scans devices below @bus including subordinate buses. Returns new
3013	* subordinate number including all the found devices.
3014	*/
3015	unsigned int pci_scan_child_bus(struct pci_bus *bus)
3016	{
3017	return pci_scan_child_bus_extend(bus, available_buses: 0);
3018	}
3019	EXPORT_SYMBOL_GPL(pci_scan_child_bus);
3020
3021	/**
3022	* pcibios_root_bridge_prepare - Platform-specific host bridge setup
3023	* @bridge: Host bridge to set up
3024	*
3025	* Default empty implementation. Replace with an architecture-specific setup
3026	* routine, if necessary.
3027	*/
3028	int __weak pcibios_root_bridge_prepare(struct pci_host_bridge *bridge)
3029	{
3030	return 0;
3031	}
3032
3033	void __weak pcibios_add_bus(struct pci_bus *bus)
3034	{
3035	}
3036
3037	void __weak pcibios_remove_bus(struct pci_bus *bus)
3038	{
3039	}
3040
3041	struct pci_bus *pci_create_root_bus(struct device *parent, int bus,
3042	struct pci_ops *ops, void *sysdata, struct list_head *resources)
3043	{
3044	int error;
3045	struct pci_host_bridge *bridge;
3046
3047	bridge = pci_alloc_host_bridge(0);
3048	if (!bridge)
3049	return NULL;
3050
3051	bridge->dev.parent = parent;
3052
3053	list_splice_init(list: resources, head: &bridge->windows);
3054	bridge->sysdata = sysdata;
3055	bridge->busnr = bus;
3056	bridge->ops = ops;
3057
3058	error = pci_register_host_bridge(bridge);
3059	if (error < 0)
3060	goto err_out;
3061
3062	return bridge->bus;
3063
3064	err_out:
3065	put_device(dev: &bridge->dev);
3066	return NULL;
3067	}
3068	EXPORT_SYMBOL_GPL(pci_create_root_bus);
3069
3070	int pci_host_probe(struct pci_host_bridge *bridge)
3071	{
3072	struct pci_bus *bus, *child;
3073	int ret;
3074
3075	ret = pci_scan_root_bus_bridge(bridge);
3076	if (ret < 0) {
3077	dev_err(bridge->dev.parent, "Scanning root bridge failed");
3078	return ret;
3079	}
3080
3081	bus = bridge->bus;
3082
3083	/*
3084	* We insert PCI resources into the iomem_resource and
3085	* ioport_resource trees in either pci_bus_claim_resources()
3086	* or pci_bus_assign_resources().
3087	*/
3088	if (pci_has_flag(flag: PCI_PROBE_ONLY)) {
3089	pci_bus_claim_resources(bus);
3090	} else {
3091	pci_bus_size_bridges(bus);
3092	pci_bus_assign_resources(bus);
3093
3094	list_for_each_entry(child, &bus->children, node)
3095	pcie_bus_configure_settings(child);
3096	}
3097
3098	pci_bus_add_devices(bus);
3099	return 0;
3100	}
3101	EXPORT_SYMBOL_GPL(pci_host_probe);
3102
3103	int pci_bus_insert_busn_res(struct pci_bus *b, int bus, int bus_max)
3104	{
3105	struct resource *res = &b->busn_res;
3106	struct resource *parent_res, *conflict;
3107
3108	res->start = bus;
3109	res->end = bus_max;
3110	res->flags = IORESOURCE_BUS;
3111
3112	if (!pci_is_root_bus(pbus: b))
3113	parent_res = &b->parent->busn_res;
3114	else {
3115	parent_res = get_pci_domain_busn_res(domain_nr: pci_domain_nr(bus: b));
3116	res->flags |= IORESOURCE_PCI_FIXED;
3117	}
3118
3119	conflict = request_resource_conflict(root: parent_res, new: res);
3120
3121	if (conflict)
3122	dev_info(&b->dev,
3123	"busn_res: can not insert %pR under %s%pR (conflicts with %s %pR)\n",
3124	res, pci_is_root_bus(b) ? "domain " : "",
3125	parent_res, conflict->name, conflict);
3126
3127	return conflict == NULL;
3128	}
3129
3130	int pci_bus_update_busn_res_end(struct pci_bus *b, int bus_max)
3131	{
3132	struct resource *res = &b->busn_res;
3133	struct resource old_res = *res;
3134	resource_size_t size;
3135	int ret;
3136
3137	if (res->start > bus_max)
3138	return -EINVAL;
3139
3140	size = bus_max - res->start + 1;
3141	ret = adjust_resource(res, start: res->start, size);
3142	dev_info(&b->dev, "busn_res: %pR end %s updated to %02x\n",
3143	&old_res, ret ? "can not be" : "is", bus_max);
3144
3145	if (!ret && !res->parent)
3146	pci_bus_insert_busn_res(b, bus: res->start, bus_max: res->end);
3147
3148	return ret;
3149	}
3150
3151	void pci_bus_release_busn_res(struct pci_bus *b)
3152	{
3153	struct resource *res = &b->busn_res;
3154	int ret;
3155
3156	if (!res->flags || !res->parent)
3157	return;
3158
3159	ret = release_resource(new: res);
3160	dev_info(&b->dev, "busn_res: %pR %s released\n",
3161	res, ret ? "can not be" : "is");
3162	}
3163
3164	int pci_scan_root_bus_bridge(struct pci_host_bridge *bridge)
3165	{
3166	struct resource_entry *window;
3167	bool found = false;
3168	struct pci_bus *b;
3169	int max, bus, ret;
3170
3171	if (!bridge)
3172	return -EINVAL;
3173
3174	resource_list_for_each_entry(window, &bridge->windows)
3175	if (window->res->flags & IORESOURCE_BUS) {
3176	bridge->busnr = window->res->start;
3177	found = true;
3178	break;
3179	}
3180
3181	ret = pci_register_host_bridge(bridge);
3182	if (ret < 0)
3183	return ret;
3184
3185	b = bridge->bus;
3186	bus = bridge->busnr;
3187
3188	if (!found) {
3189	dev_info(&b->dev,
3190	"No busn resource found for root bus, will use [bus %02x-ff]\n",
3191	bus);
3192	pci_bus_insert_busn_res(b, bus, bus_max: 255);
3193	}
3194
3195	max = pci_scan_child_bus(b);
3196
3197	if (!found)
3198	pci_bus_update_busn_res_end(b, bus_max: max);
3199
3200	return 0;
3201	}
3202	EXPORT_SYMBOL(pci_scan_root_bus_bridge);
3203
3204	struct pci_bus *pci_scan_root_bus(struct device *parent, int bus,
3205	struct pci_ops *ops, void *sysdata, struct list_head *resources)
3206	{
3207	struct resource_entry *window;
3208	bool found = false;
3209	struct pci_bus *b;
3210	int max;
3211
3212	resource_list_for_each_entry(window, resources)
3213	if (window->res->flags & IORESOURCE_BUS) {
3214	found = true;
3215	break;
3216	}
3217
3218	b = pci_create_root_bus(parent, bus, ops, sysdata, resources);
3219	if (!b)
3220	return NULL;
3221
3222	if (!found) {
3223	dev_info(&b->dev,
3224	"No busn resource found for root bus, will use [bus %02x-ff]\n",
3225	bus);
3226	pci_bus_insert_busn_res(b, bus, bus_max: 255);
3227	}
3228
3229	max = pci_scan_child_bus(b);
3230
3231	if (!found)
3232	pci_bus_update_busn_res_end(b, bus_max: max);
3233
3234	return b;
3235	}
3236	EXPORT_SYMBOL(pci_scan_root_bus);
3237
3238	struct pci_bus *pci_scan_bus(int bus, struct pci_ops *ops,
3239	void *sysdata)
3240	{
3241	LIST_HEAD(resources);
3242	struct pci_bus *b;
3243
3244	pci_add_resource(resources: &resources, res: &ioport_resource);
3245	pci_add_resource(resources: &resources, res: &iomem_resource);
3246	pci_add_resource(resources: &resources, res: &busn_resource);
3247	b = pci_create_root_bus(NULL, bus, ops, sysdata, &resources);
3248	if (b) {
3249	pci_scan_child_bus(b);
3250	} else {
3251	pci_free_resource_list(resources: &resources);
3252	}
3253	return b;
3254	}
3255	EXPORT_SYMBOL(pci_scan_bus);
3256
3257	/**
3258	* pci_rescan_bus_bridge_resize - Scan a PCI bus for devices
3259	* @bridge: PCI bridge for the bus to scan
3260	*
3261	* Scan a PCI bus and child buses for new devices, add them,
3262	* and enable them, resizing bridge mmio/io resource if necessary
3263	* and possible. The caller must ensure the child devices are already
3264	* removed for resizing to occur.
3265	*
3266	* Returns the max number of subordinate bus discovered.
3267	*/
3268	unsigned int pci_rescan_bus_bridge_resize(struct pci_dev *bridge)
3269	{
3270	unsigned int max;
3271	struct pci_bus *bus = bridge->subordinate;
3272
3273	max = pci_scan_child_bus(bus);
3274
3275	pci_assign_unassigned_bridge_resources(bridge);
3276
3277	pci_bus_add_devices(bus);
3278
3279	return max;
3280	}
3281
3282	/**
3283	* pci_rescan_bus - Scan a PCI bus for devices
3284	* @bus: PCI bus to scan
3285	*
3286	* Scan a PCI bus and child buses for new devices, add them,
3287	* and enable them.
3288	*
3289	* Returns the max number of subordinate bus discovered.
3290	*/
3291	unsigned int pci_rescan_bus(struct pci_bus *bus)
3292	{
3293	unsigned int max;
3294
3295	max = pci_scan_child_bus(bus);
3296	pci_assign_unassigned_bus_resources(bus);
3297	pci_bus_add_devices(bus);
3298
3299	return max;
3300	}
3301	EXPORT_SYMBOL_GPL(pci_rescan_bus);
3302
3303	/*
3304	* pci_rescan_bus(), pci_rescan_bus_bridge_resize() and PCI device removal
3305	* routines should always be executed under this mutex.
3306	*/
3307	static DEFINE_MUTEX(pci_rescan_remove_lock);
3308
3309	void pci_lock_rescan_remove(void)
3310	{
3311	mutex_lock(&pci_rescan_remove_lock);
3312	}
3313	EXPORT_SYMBOL_GPL(pci_lock_rescan_remove);
3314
3315	void pci_unlock_rescan_remove(void)
3316	{
3317	mutex_unlock(lock: &pci_rescan_remove_lock);
3318	}
3319	EXPORT_SYMBOL_GPL(pci_unlock_rescan_remove);
3320
3321	static int __init pci_sort_bf_cmp(const struct device *d_a,
3322	const struct device *d_b)
3323	{
3324	const struct pci_dev *a = to_pci_dev(d_a);
3325	const struct pci_dev *b = to_pci_dev(d_b);
3326
3327	if (pci_domain_nr(bus: a->bus) < pci_domain_nr(bus: b->bus)) return -1;
3328	else if (pci_domain_nr(bus: a->bus) > pci_domain_nr(bus: b->bus)) return 1;
3329
3330	if (a->bus->number < b->bus->number) return -1;
3331	else if (a->bus->number > b->bus->number) return 1;
3332
3333	if (a->devfn < b->devfn) return -1;
3334	else if (a->devfn > b->devfn) return 1;
3335
3336	return 0;
3337	}
3338
3339	void __init pci_sort_breadthfirst(void)
3340	{
3341	bus_sort_breadthfirst(bus: &pci_bus_type, compare: &pci_sort_bf_cmp);
3342	}
3343
3344	int pci_hp_add_bridge(struct pci_dev *dev)
3345	{
3346	struct pci_bus *parent = dev->bus;
3347	int busnr, start = parent->busn_res.start;
3348	unsigned int available_buses = 0;
3349	int end = parent->busn_res.end;
3350
3351	for (busnr = start; busnr <= end; busnr++) {
3352	if (!pci_find_bus(domain: pci_domain_nr(bus: parent), busnr))
3353	break;
3354	}
3355	if (busnr-- > end) {
3356	pci_err(dev, "No bus number available for hot-added bridge\n");
3357	return -1;
3358	}
3359
3360	/* Scan bridges that are already configured */
3361	busnr = pci_scan_bridge(parent, dev, busnr, 0);
3362
3363	/*
3364	* Distribute the available bus numbers between hotplug-capable
3365	* bridges to make extending the chain later possible.
3366	*/
3367	available_buses = end - busnr;
3368
3369	/* Scan bridges that need to be reconfigured */
3370	pci_scan_bridge_extend(bus: parent, dev, max: busnr, available_buses, pass: 1);
3371
3372	if (!dev->subordinate)
3373	return -1;
3374
3375	return 0;
3376	}
3377	EXPORT_SYMBOL_GPL(pci_hp_add_bridge);
3378