1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* PCI detection and setup code
4	*/
5
6	#include <linux/kernel.h>
7	#include <linux/delay.h>
8	#include <linux/init.h>
9	#include <linux/pci.h>
10	#include <linux/msi.h>
11	#include <linux/of_pci.h>
12	#include <linux/pci_hotplug.h>
13	#include <linux/slab.h>
14	#include <linux/module.h>
15	#include <linux/cpumask.h>
16	#include <linux/aer.h>
17	#include <linux/acpi.h>
18	#include <linux/hypervisor.h>
19	#include <linux/irqdomain.h>
20	#include <linux/pm_runtime.h>
21	#include <linux/bitfield.h>
22	#include "pci.h"
23
24	#define CARDBUS_LATENCY_TIMER 176 /* secondary latency timer */
25	#define CARDBUS_RESERVE_BUSNR 3
26
27	static struct resource busn_resource = {
28	.name = "PCI busn",
29	.start = 0,
30	.end = 255,
31	.flags = IORESOURCE_BUS,
32	};
33
34	/* Ugh. Need to stop exporting this to modules. */
35	LIST_HEAD(pci_root_buses);
36	EXPORT_SYMBOL(pci_root_buses);
37
38	static LIST_HEAD(pci_domain_busn_res_list);
39
40	struct pci_domain_busn_res {
41	struct list_head list;
42	struct resource res;
43	int domain_nr;
44	};
45
46	static struct resource *get_pci_domain_busn_res(int domain_nr)
47	{
48	struct pci_domain_busn_res *r;
49
50	list_for_each_entry(r, &pci_domain_busn_res_list, list)
51	if (r->domain_nr == domain_nr)
52	return &r->res;
53
54	r = kzalloc(size: sizeof(*r), GFP_KERNEL);
55	if (!r)
56	return NULL;
57
58	r->domain_nr = domain_nr;
59	r->res.start = 0;
60	r->res.end = 0xff;
61	r->res.flags = IORESOURCE_BUS | IORESOURCE_PCI_FIXED;
62
63	list_add_tail(new: &r->list, head: &pci_domain_busn_res_list);
64
65	return &r->res;
66	}
67
68	/*
69	* Some device drivers need know if PCI is initiated.
70	* Basically, we think PCI is not initiated when there
71	* is no device to be found on the pci_bus_type.
72	*/
73	int no_pci_devices(void)
74	{
75	struct device *dev;
76	int no_devices;
77
78	dev = bus_find_next_device(bus: &pci_bus_type, NULL);
79	no_devices = (dev == NULL);
80	put_device(dev);
81	return no_devices;
82	}
83	EXPORT_SYMBOL(no_pci_devices);
84
85	/*
86	* PCI Bus Class
87	*/
88	static void release_pcibus_dev(struct device *dev)
89	{
90	struct pci_bus *pci_bus = to_pci_bus(dev);
91
92	put_device(dev: pci_bus->bridge);
93	pci_bus_remove_resources(bus: pci_bus);
94	pci_release_bus_of_node(bus: pci_bus);
95	kfree(objp: pci_bus);
96	}
97
98	static struct class pcibus_class = {
99	.name = "pci_bus",
100	.dev_release = &release_pcibus_dev,
101	.dev_groups = pcibus_groups,
102	};
103
104	static int __init pcibus_class_init(void)
105	{
106	return class_register(class: &pcibus_class);
107	}
108	postcore_initcall(pcibus_class_init);
109
110	static u64 pci_size(u64 base, u64 maxbase, u64 mask)
111	{
112	u64 size = mask & maxbase; /* Find the significant bits */
113	if (!size)
114	return 0;
115
116	/*
117	* Get the lowest of them to find the decode size, and from that
118	* the extent.
119	*/
120	size = size & ~(size-1);
121
122	/*
123	* base == maxbase can be valid only if the BAR has already been
124	* programmed with all 1s.
125	*/
126	if (base == maxbase && ((base | (size - 1)) & mask) != mask)
127	return 0;
128
129	return size;
130	}
131
132	static inline unsigned long decode_bar(struct pci_dev *dev, u32 bar)
133	{
134	u32 mem_type;
135	unsigned long flags;
136
137	if ((bar & PCI_BASE_ADDRESS_SPACE) == PCI_BASE_ADDRESS_SPACE_IO) {
138	flags = bar & ~PCI_BASE_ADDRESS_IO_MASK;
139	flags |= IORESOURCE_IO;
140	return flags;
141	}
142
143	flags = bar & ~PCI_BASE_ADDRESS_MEM_MASK;
144	flags |= IORESOURCE_MEM;
145	if (flags & PCI_BASE_ADDRESS_MEM_PREFETCH)
146	flags |= IORESOURCE_PREFETCH;
147
148	mem_type = bar & PCI_BASE_ADDRESS_MEM_TYPE_MASK;
149	switch (mem_type) {
150	case PCI_BASE_ADDRESS_MEM_TYPE_32:
151	break;
152	case PCI_BASE_ADDRESS_MEM_TYPE_1M:
153	/* 1M mem BAR treated as 32-bit BAR */
154	break;
155	case PCI_BASE_ADDRESS_MEM_TYPE_64:
156	flags |= IORESOURCE_MEM_64;
157	break;
158	default:
159	/* mem unknown type treated as 32-bit BAR */
160	break;
161	}
162	return flags;
163	}
164
165	#define PCI_COMMAND_DECODE_ENABLE (PCI_COMMAND_MEMORY | PCI_COMMAND_IO)
166
167	/**
168	* __pci_read_base - Read a PCI BAR
169	* @dev: the PCI device
170	* @type: type of the BAR
171	* @res: resource buffer to be filled in
172	* @pos: BAR position in the config space
173	*
174	* Returns 1 if the BAR is 64-bit, or 0 if 32-bit.
175	*/
176	int __pci_read_base(struct pci_dev *dev, enum pci_bar_type type,
177	struct resource *res, unsigned int pos)
178	{
179	u32 l = 0, sz = 0, mask;
180	u64 l64, sz64, mask64;
181	u16 orig_cmd;
182	struct pci_bus_region region, inverted_region;
183	const char *res_name = pci_resource_name(dev, i: res - dev->resource);
184
185	mask = type ? PCI_ROM_ADDRESS_MASK : ~0;
186
187	/* No printks while decoding is disabled! */
188	if (!dev->mmio_always_on) {
189	pci_read_config_word(dev, PCI_COMMAND, val: &orig_cmd);
190	if (orig_cmd & PCI_COMMAND_DECODE_ENABLE) {
191	pci_write_config_word(dev, PCI_COMMAND,
192	val: orig_cmd & ~PCI_COMMAND_DECODE_ENABLE);
193	}
194	}
195
196	res->name = pci_name(pdev: dev);
197
198	pci_read_config_dword(dev, where: pos, val: &l);
199	pci_write_config_dword(dev, where: pos, val: l | mask);
200	pci_read_config_dword(dev, where: pos, val: &sz);
201	pci_write_config_dword(dev, where: pos, val: l);
202
203	/*
204	* All bits set in sz means the device isn't working properly.
205	* If the BAR isn't implemented, all bits must be 0. If it's a
206	* memory BAR or a ROM, bit 0 must be clear; if it's an io BAR, bit
207	* 1 must be clear.
208	*/
209	if (PCI_POSSIBLE_ERROR(sz))
210	sz = 0;
211
212	/*
213	* I don't know how l can have all bits set. Copied from old code.
214	* Maybe it fixes a bug on some ancient platform.
215	*/
216	if (PCI_POSSIBLE_ERROR(l))
217	l = 0;
218
219	if (type == pci_bar_unknown) {
220	res->flags = decode_bar(dev, bar: l);
221	res->flags |= IORESOURCE_SIZEALIGN;
222	if (res->flags & IORESOURCE_IO) {
223	l64 = l & PCI_BASE_ADDRESS_IO_MASK;
224	sz64 = sz & PCI_BASE_ADDRESS_IO_MASK;
225	mask64 = PCI_BASE_ADDRESS_IO_MASK & (u32)IO_SPACE_LIMIT;
226	} else {
227	l64 = l & PCI_BASE_ADDRESS_MEM_MASK;
228	sz64 = sz & PCI_BASE_ADDRESS_MEM_MASK;
229	mask64 = (u32)PCI_BASE_ADDRESS_MEM_MASK;
230	}
231	} else {
232	if (l & PCI_ROM_ADDRESS_ENABLE)
233	res->flags |= IORESOURCE_ROM_ENABLE;
234	l64 = l & PCI_ROM_ADDRESS_MASK;
235	sz64 = sz & PCI_ROM_ADDRESS_MASK;
236	mask64 = PCI_ROM_ADDRESS_MASK;
237	}
238
239	if (res->flags & IORESOURCE_MEM_64) {
240	pci_read_config_dword(dev, where: pos + 4, val: &l);
241	pci_write_config_dword(dev, where: pos + 4, val: ~0);
242	pci_read_config_dword(dev, where: pos + 4, val: &sz);
243	pci_write_config_dword(dev, where: pos + 4, val: l);
244
245	l64 |= ((u64)l << 32);
246	sz64 |= ((u64)sz << 32);
247	mask64 |= ((u64)~0 << 32);
248	}
249
250	if (!dev->mmio_always_on && (orig_cmd & PCI_COMMAND_DECODE_ENABLE))
251	pci_write_config_word(dev, PCI_COMMAND, val: orig_cmd);
252
253	if (!sz64)
254	goto fail;
255
256	sz64 = pci_size(base: l64, maxbase: sz64, mask: mask64);
257	if (!sz64) {
258	pci_info(dev, FW_BUG "%s: invalid; can't size\n", res_name);
259	goto fail;
260	}
261
262	if (res->flags & IORESOURCE_MEM_64) {
263	if ((sizeof(pci_bus_addr_t) < 8 || sizeof(resource_size_t) < 8)
264	&& sz64 > 0x100000000ULL) {
265	res->flags |= IORESOURCE_UNSET | IORESOURCE_DISABLED;
266	res->start = 0;
267	res->end = 0;
268	pci_err(dev, "%s: can't handle BAR larger than 4GB (size %#010llx)\n",
269	res_name, (unsigned long long)sz64);
270	goto out;
271	}
272
273	if ((sizeof(pci_bus_addr_t) < 8) && l) {
274	/* Above 32-bit boundary; try to reallocate */
275	res->flags |= IORESOURCE_UNSET;
276	res->start = 0;
277	res->end = sz64 - 1;
278	pci_info(dev, "%s: can't handle BAR above 4GB (bus address %#010llx)\n",
279	res_name, (unsigned long long)l64);
280	goto out;
281	}
282	}
283
284	region.start = l64;
285	region.end = l64 + sz64 - 1;
286
287	pcibios_bus_to_resource(bus: dev->bus, res, region: &region);
288	pcibios_resource_to_bus(bus: dev->bus, region: &inverted_region, res);
289
290	/*
291	* If "A" is a BAR value (a bus address), "bus_to_resource(A)" is
292	* the corresponding resource address (the physical address used by
293	* the CPU. Converting that resource address back to a bus address
294	* should yield the original BAR value:
295	*
296	* resource_to_bus(bus_to_resource(A)) == A
297	*
298	* If it doesn't, CPU accesses to "bus_to_resource(A)" will not
299	* be claimed by the device.
300	*/
301	if (inverted_region.start != region.start) {
302	res->flags |= IORESOURCE_UNSET;
303	res->start = 0;
304	res->end = region.end - region.start;
305	pci_info(dev, "%s: initial BAR value %#010llx invalid\n",
306	res_name, (unsigned long long)region.start);
307	}
308
309	goto out;
310
311
312	fail:
313	res->flags = 0;
314	out:
315	if (res->flags)
316	pci_info(dev, "%s %pR\n", res_name, res);
317
318	return (res->flags & IORESOURCE_MEM_64) ? 1 : 0;
319	}
320
321	static void pci_read_bases(struct pci_dev *dev, unsigned int howmany, int rom)
322	{
323	unsigned int pos, reg;
324
325	if (dev->non_compliant_bars)
326	return;
327
328	/* Per PCIe r4.0, sec 9.3.4.1.11, the VF BARs are all RO Zero */
329	if (dev->is_virtfn)
330	return;
331
332	for (pos = 0; pos < howmany; pos++) {
333	struct resource *res = &dev->resource[pos];
334	reg = PCI_BASE_ADDRESS_0 + (pos << 2);
335	pos += __pci_read_base(dev, type: pci_bar_unknown, res, pos: reg);
336	}
337
338	if (rom) {
339	struct resource *res = &dev->resource[PCI_ROM_RESOURCE];
340	dev->rom_base_reg = rom;
341	res->flags = IORESOURCE_MEM | IORESOURCE_PREFETCH |
342	IORESOURCE_READONLY | IORESOURCE_SIZEALIGN;
343	__pci_read_base(dev, type: pci_bar_mem32, res, pos: rom);
344	}
345	}
346
347	static void pci_read_bridge_io(struct pci_dev *dev, struct resource *res,
348	bool log)
349	{
350	u8 io_base_lo, io_limit_lo;
351	unsigned long io_mask, io_granularity, base, limit;
352	struct pci_bus_region region;
353
354	io_mask = PCI_IO_RANGE_MASK;
355	io_granularity = 0x1000;
356	if (dev->io_window_1k) {
357	/* Support 1K I/O space granularity */
358	io_mask = PCI_IO_1K_RANGE_MASK;
359	io_granularity = 0x400;
360	}
361
362	pci_read_config_byte(dev, PCI_IO_BASE, val: &io_base_lo);
363	pci_read_config_byte(dev, PCI_IO_LIMIT, val: &io_limit_lo);
364	base = (io_base_lo & io_mask) << 8;
365	limit = (io_limit_lo & io_mask) << 8;
366
367	if ((io_base_lo & PCI_IO_RANGE_TYPE_MASK) == PCI_IO_RANGE_TYPE_32) {
368	u16 io_base_hi, io_limit_hi;
369
370	pci_read_config_word(dev, PCI_IO_BASE_UPPER16, val: &io_base_hi);
371	pci_read_config_word(dev, PCI_IO_LIMIT_UPPER16, val: &io_limit_hi);
372	base |= ((unsigned long) io_base_hi << 16);
373	limit |= ((unsigned long) io_limit_hi << 16);
374	}
375
376	if (base <= limit) {
377	res->flags = (io_base_lo & PCI_IO_RANGE_TYPE_MASK) | IORESOURCE_IO;
378	region.start = base;
379	region.end = limit + io_granularity - 1;
380	pcibios_bus_to_resource(bus: dev->bus, res, region: &region);
381	if (log)
382	pci_info(dev, " bridge window %pR\n", res);
383	}
384	}
385
386	static void pci_read_bridge_mmio(struct pci_dev *dev, struct resource *res,
387	bool log)
388	{
389	u16 mem_base_lo, mem_limit_lo;
390	unsigned long base, limit;
391	struct pci_bus_region region;
392
393	pci_read_config_word(dev, PCI_MEMORY_BASE, val: &mem_base_lo);
394	pci_read_config_word(dev, PCI_MEMORY_LIMIT, val: &mem_limit_lo);
395	base = ((unsigned long) mem_base_lo & PCI_MEMORY_RANGE_MASK) << 16;
396	limit = ((unsigned long) mem_limit_lo & PCI_MEMORY_RANGE_MASK) << 16;
397	if (base <= limit) {
398	res->flags = (mem_base_lo & PCI_MEMORY_RANGE_TYPE_MASK) | IORESOURCE_MEM;
399	region.start = base;
400	region.end = limit + 0xfffff;
401	pcibios_bus_to_resource(bus: dev->bus, res, region: &region);
402	if (log)
403	pci_info(dev, " bridge window %pR\n", res);
404	}
405	}
406
407	static void pci_read_bridge_mmio_pref(struct pci_dev *dev, struct resource *res,
408	bool log)
409	{
410	u16 mem_base_lo, mem_limit_lo;
411	u64 base64, limit64;
412	pci_bus_addr_t base, limit;
413	struct pci_bus_region region;
414
415	pci_read_config_word(dev, PCI_PREF_MEMORY_BASE, val: &mem_base_lo);
416	pci_read_config_word(dev, PCI_PREF_MEMORY_LIMIT, val: &mem_limit_lo);
417	base64 = (mem_base_lo & PCI_PREF_RANGE_MASK) << 16;
418	limit64 = (mem_limit_lo & PCI_PREF_RANGE_MASK) << 16;
419
420	if ((mem_base_lo & PCI_PREF_RANGE_TYPE_MASK) == PCI_PREF_RANGE_TYPE_64) {
421	u32 mem_base_hi, mem_limit_hi;
422
423	pci_read_config_dword(dev, PCI_PREF_BASE_UPPER32, val: &mem_base_hi);
424	pci_read_config_dword(dev, PCI_PREF_LIMIT_UPPER32, val: &mem_limit_hi);
425
426	/*
427	* Some bridges set the base > limit by default, and some
428	* (broken) BIOSes do not initialize them. If we find
429	* this, just assume they are not being used.
430	*/
431	if (mem_base_hi <= mem_limit_hi) {
432	base64 |= (u64) mem_base_hi << 32;
433	limit64 |= (u64) mem_limit_hi << 32;
434	}
435	}
436
437	base = (pci_bus_addr_t) base64;
438	limit = (pci_bus_addr_t) limit64;
439
440	if (base != base64) {
441	pci_err(dev, "can't handle bridge window above 4GB (bus address %#010llx)\n",
442	(unsigned long long) base64);
443	return;
444	}
445
446	if (base <= limit) {
447	res->flags = (mem_base_lo & PCI_PREF_RANGE_TYPE_MASK) |
448	IORESOURCE_MEM | IORESOURCE_PREFETCH;
449	if (res->flags & PCI_PREF_RANGE_TYPE_64)
450	res->flags |= IORESOURCE_MEM_64;
451	region.start = base;
452	region.end = limit + 0xfffff;
453	pcibios_bus_to_resource(bus: dev->bus, res, region: &region);
454	if (log)
455	pci_info(dev, " bridge window %pR\n", res);
456	}
457	}
458
459	static void pci_read_bridge_windows(struct pci_dev *bridge)
460	{
461	u32 buses;
462	u16 io;
463	u32 pmem, tmp;
464	struct resource res;
465
466	pci_read_config_dword(dev: bridge, PCI_PRIMARY_BUS, val: &buses);
467	res.flags = IORESOURCE_BUS;
468	res.start = (buses >> 8) & 0xff;
469	res.end = (buses >> 16) & 0xff;
470	pci_info(bridge, "PCI bridge to %pR%s\n", &res,
471	bridge->transparent ? " (subtractive decode)" : "");
472
473	pci_read_config_word(dev: bridge, PCI_IO_BASE, val: &io);
474	if (!io) {
475	pci_write_config_word(dev: bridge, PCI_IO_BASE, val: 0xe0f0);
476	pci_read_config_word(dev: bridge, PCI_IO_BASE, val: &io);
477	pci_write_config_word(dev: bridge, PCI_IO_BASE, val: 0x0);
478	}
479	if (io) {
480	bridge->io_window = 1;
481	pci_read_bridge_io(dev: bridge, res: &res, log: true);
482	}
483
484	pci_read_bridge_mmio(dev: bridge, res: &res, log: true);
485
486	/*
487	* DECchip 21050 pass 2 errata: the bridge may miss an address
488	* disconnect boundary by one PCI data phase. Workaround: do not
489	* use prefetching on this device.
490	*/
491	if (bridge->vendor == PCI_VENDOR_ID_DEC && bridge->device == 0x0001)
492	return;
493
494	pci_read_config_dword(dev: bridge, PCI_PREF_MEMORY_BASE, val: &pmem);
495	if (!pmem) {
496	pci_write_config_dword(dev: bridge, PCI_PREF_MEMORY_BASE,
497	val: 0xffe0fff0);
498	pci_read_config_dword(dev: bridge, PCI_PREF_MEMORY_BASE, val: &pmem);
499	pci_write_config_dword(dev: bridge, PCI_PREF_MEMORY_BASE, val: 0x0);
500	}
501	if (!pmem)
502	return;
503
504	bridge->pref_window = 1;
505
506	if ((pmem & PCI_PREF_RANGE_TYPE_MASK) == PCI_PREF_RANGE_TYPE_64) {
507
508	/*
509	* Bridge claims to have a 64-bit prefetchable memory
510	* window; verify that the upper bits are actually
511	* writable.
512	*/
513	pci_read_config_dword(dev: bridge, PCI_PREF_BASE_UPPER32, val: &pmem);
514	pci_write_config_dword(dev: bridge, PCI_PREF_BASE_UPPER32,
515	val: 0xffffffff);
516	pci_read_config_dword(dev: bridge, PCI_PREF_BASE_UPPER32, val: &tmp);
517	pci_write_config_dword(dev: bridge, PCI_PREF_BASE_UPPER32, val: pmem);
518	if (tmp)
519	bridge->pref_64_window = 1;
520	}
521
522	pci_read_bridge_mmio_pref(dev: bridge, res: &res, log: true);
523	}
524
525	void pci_read_bridge_bases(struct pci_bus *child)
526	{
527	struct pci_dev *dev = child->self;
528	struct resource *res;
529	int i;
530
531	if (pci_is_root_bus(pbus: child)) /* It's a host bus, nothing to read */
532	return;
533
534	pci_info(dev, "PCI bridge to %pR%s\n",
535	&child->busn_res,
536	dev->transparent ? " (subtractive decode)" : "");
537
538	pci_bus_remove_resources(bus: child);
539	for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++)
540	child->resource[i] = &dev->resource[PCI_BRIDGE_RESOURCES+i];
541
542	pci_read_bridge_io(dev: child->self, res: child->resource[0], log: false);
543	pci_read_bridge_mmio(dev: child->self, res: child->resource[1], log: false);
544	pci_read_bridge_mmio_pref(dev: child->self, res: child->resource[2], log: false);
545
546	if (dev->transparent) {
547	pci_bus_for_each_resource(child->parent, res) {
548	if (res && res->flags) {
549	pci_bus_add_resource(bus: child, res,
550	PCI_SUBTRACTIVE_DECODE);
551	pci_info(dev, " bridge window %pR (subtractive decode)\n",
552	res);
553	}
554	}
555	}
556	}
557
558	static struct pci_bus *pci_alloc_bus(struct pci_bus *parent)
559	{
560	struct pci_bus *b;
561
562	b = kzalloc(size: sizeof(*b), GFP_KERNEL);
563	if (!b)
564	return NULL;
565
566	INIT_LIST_HEAD(list: &b->node);
567	INIT_LIST_HEAD(list: &b->children);
568	INIT_LIST_HEAD(list: &b->devices);
569	INIT_LIST_HEAD(list: &b->slots);
570	INIT_LIST_HEAD(list: &b->resources);
571	b->max_bus_speed = PCI_SPEED_UNKNOWN;
572	b->cur_bus_speed = PCI_SPEED_UNKNOWN;
573	#ifdef CONFIG_PCI_DOMAINS_GENERIC
574	if (parent)
575	b->domain_nr = parent->domain_nr;
576	#endif
577	return b;
578	}
579
580	static void pci_release_host_bridge_dev(struct device *dev)
581	{
582	struct pci_host_bridge *bridge = to_pci_host_bridge(dev);
583
584	if (bridge->release_fn)
585	bridge->release_fn(bridge);
586
587	pci_free_resource_list(resources: &bridge->windows);
588	pci_free_resource_list(resources: &bridge->dma_ranges);
589	kfree(objp: bridge);
590	}
591
592	static void pci_init_host_bridge(struct pci_host_bridge *bridge)
593	{
594	INIT_LIST_HEAD(list: &bridge->windows);
595	INIT_LIST_HEAD(list: &bridge->dma_ranges);
596
597	/*
598	* We assume we can manage these PCIe features. Some systems may
599	* reserve these for use by the platform itself, e.g., an ACPI BIOS
600	* may implement its own AER handling and use _OSC to prevent the
601	* OS from interfering.
602	*/
603	bridge->native_aer = 1;
604	bridge->native_pcie_hotplug = 1;
605	bridge->native_shpc_hotplug = 1;
606	bridge->native_pme = 1;
607	bridge->native_ltr = 1;
608	bridge->native_dpc = 1;
609	bridge->domain_nr = PCI_DOMAIN_NR_NOT_SET;
610	bridge->native_cxl_error = 1;
611
612	device_initialize(dev: &bridge->dev);
613	}
614
615	struct pci_host_bridge *pci_alloc_host_bridge(size_t priv)
616	{
617	struct pci_host_bridge *bridge;
618
619	bridge = kzalloc(size: sizeof(*bridge) + priv, GFP_KERNEL);
620	if (!bridge)
621	return NULL;
622
623	pci_init_host_bridge(bridge);
624	bridge->dev.release = pci_release_host_bridge_dev;
625
626	return bridge;
627	}
628	EXPORT_SYMBOL(pci_alloc_host_bridge);
629
630	static void devm_pci_alloc_host_bridge_release(void *data)
631	{
632	pci_free_host_bridge(bridge: data);
633	}
634
635	struct pci_host_bridge *devm_pci_alloc_host_bridge(struct device *dev,
636	size_t priv)
637	{
638	int ret;
639	struct pci_host_bridge *bridge;
640
641	bridge = pci_alloc_host_bridge(priv);
642	if (!bridge)
643	return NULL;
644
645	bridge->dev.parent = dev;
646
647	ret = devm_add_action_or_reset(dev, devm_pci_alloc_host_bridge_release,
648	bridge);
649	if (ret)
650	return NULL;
651
652	ret = devm_of_pci_bridge_init(dev, bridge);
653	if (ret)
654	return NULL;
655
656	return bridge;
657	}
658	EXPORT_SYMBOL(devm_pci_alloc_host_bridge);
659
660	void pci_free_host_bridge(struct pci_host_bridge *bridge)
661	{
662	put_device(dev: &bridge->dev);
663	}
664	EXPORT_SYMBOL(pci_free_host_bridge);
665
666	/* Indexed by PCI_X_SSTATUS_FREQ (secondary bus mode and frequency) */
667	static const unsigned char pcix_bus_speed[] = {
668	PCI_SPEED_UNKNOWN, /* 0 */
669	PCI_SPEED_66MHz_PCIX, /* 1 */
670	PCI_SPEED_100MHz_PCIX, /* 2 */
671	PCI_SPEED_133MHz_PCIX, /* 3 */
672	PCI_SPEED_UNKNOWN, /* 4 */
673	PCI_SPEED_66MHz_PCIX_ECC, /* 5 */
674	PCI_SPEED_100MHz_PCIX_ECC, /* 6 */
675	PCI_SPEED_133MHz_PCIX_ECC, /* 7 */
676	PCI_SPEED_UNKNOWN, /* 8 */
677	PCI_SPEED_66MHz_PCIX_266, /* 9 */
678	PCI_SPEED_100MHz_PCIX_266, /* A */
679	PCI_SPEED_133MHz_PCIX_266, /* B */
680	PCI_SPEED_UNKNOWN, /* C */
681	PCI_SPEED_66MHz_PCIX_533, /* D */
682	PCI_SPEED_100MHz_PCIX_533, /* E */
683	PCI_SPEED_133MHz_PCIX_533 /* F */
684	};
685
686	/* Indexed by PCI_EXP_LNKCAP_SLS, PCI_EXP_LNKSTA_CLS */
687	const unsigned char pcie_link_speed[] = {
688	PCI_SPEED_UNKNOWN, /* 0 */
689	PCIE_SPEED_2_5GT, /* 1 */
690	PCIE_SPEED_5_0GT, /* 2 */
691	PCIE_SPEED_8_0GT, /* 3 */
692	PCIE_SPEED_16_0GT, /* 4 */
693	PCIE_SPEED_32_0GT, /* 5 */
694	PCIE_SPEED_64_0GT, /* 6 */
695	PCI_SPEED_UNKNOWN, /* 7 */
696	PCI_SPEED_UNKNOWN, /* 8 */
697	PCI_SPEED_UNKNOWN, /* 9 */
698	PCI_SPEED_UNKNOWN, /* A */
699	PCI_SPEED_UNKNOWN, /* B */
700	PCI_SPEED_UNKNOWN, /* C */
701	PCI_SPEED_UNKNOWN, /* D */
702	PCI_SPEED_UNKNOWN, /* E */
703	PCI_SPEED_UNKNOWN /* F */
704	};
705	EXPORT_SYMBOL_GPL(pcie_link_speed);
706
707	const char *pci_speed_string(enum pci_bus_speed speed)
708	{
709	/* Indexed by the pci_bus_speed enum */
710	static const char *speed_strings[] = {
711	"33 MHz PCI", /* 0x00 */
712	"66 MHz PCI", /* 0x01 */
713	"66 MHz PCI-X", /* 0x02 */
714	"100 MHz PCI-X", /* 0x03 */
715	"133 MHz PCI-X", /* 0x04 */
716	NULL, /* 0x05 */
717	NULL, /* 0x06 */
718	NULL, /* 0x07 */
719	NULL, /* 0x08 */
720	"66 MHz PCI-X 266", /* 0x09 */
721	"100 MHz PCI-X 266", /* 0x0a */
722	"133 MHz PCI-X 266", /* 0x0b */
723	"Unknown AGP", /* 0x0c */
724	"1x AGP", /* 0x0d */
725	"2x AGP", /* 0x0e */
726	"4x AGP", /* 0x0f */
727	"8x AGP", /* 0x10 */
728	"66 MHz PCI-X 533", /* 0x11 */
729	"100 MHz PCI-X 533", /* 0x12 */
730	"133 MHz PCI-X 533", /* 0x13 */
731	"2.5 GT/s PCIe", /* 0x14 */
732	"5.0 GT/s PCIe", /* 0x15 */
733	"8.0 GT/s PCIe", /* 0x16 */
734	"16.0 GT/s PCIe", /* 0x17 */
735	"32.0 GT/s PCIe", /* 0x18 */
736	"64.0 GT/s PCIe", /* 0x19 */
737	};
738
739	if (speed < ARRAY_SIZE(speed_strings))
740	return speed_strings[speed];
741	return "Unknown";
742	}
743	EXPORT_SYMBOL_GPL(pci_speed_string);
744
745	void pcie_update_link_speed(struct pci_bus *bus, u16 linksta)
746	{
747	bus->cur_bus_speed = pcie_link_speed[linksta & PCI_EXP_LNKSTA_CLS];
748	}
749	EXPORT_SYMBOL_GPL(pcie_update_link_speed);
750
751	static unsigned char agp_speeds[] = {
752	AGP_UNKNOWN,
753	AGP_1X,
754	AGP_2X,
755	AGP_4X,
756	AGP_8X
757	};
758
759	static enum pci_bus_speed agp_speed(int agp3, int agpstat)
760	{
761	int index = 0;
762
763	if (agpstat & 4)
764	index = 3;
765	else if (agpstat & 2)
766	index = 2;
767	else if (agpstat & 1)
768	index = 1;
769	else
770	goto out;
771
772	if (agp3) {
773	index += 2;
774	if (index == 5)
775	index = 0;
776	}
777
778	out:
779	return agp_speeds[index];
780	}
781
782	static void pci_set_bus_speed(struct pci_bus *bus)
783	{
784	struct pci_dev *bridge = bus->self;
785	int pos;
786
787	pos = pci_find_capability(dev: bridge, PCI_CAP_ID_AGP);
788	if (!pos)
789	pos = pci_find_capability(dev: bridge, PCI_CAP_ID_AGP3);
790	if (pos) {
791	u32 agpstat, agpcmd;
792
793	pci_read_config_dword(dev: bridge, where: pos + PCI_AGP_STATUS, val: &agpstat);
794	bus->max_bus_speed = agp_speed(agp3: agpstat & 8, agpstat: agpstat & 7);
795
796	pci_read_config_dword(dev: bridge, where: pos + PCI_AGP_COMMAND, val: &agpcmd);
797	bus->cur_bus_speed = agp_speed(agp3: agpstat & 8, agpstat: agpcmd & 7);
798	}
799
800	pos = pci_find_capability(dev: bridge, PCI_CAP_ID_PCIX);
801	if (pos) {
802	u16 status;
803	enum pci_bus_speed max;
804
805	pci_read_config_word(dev: bridge, where: pos + PCI_X_BRIDGE_SSTATUS,
806	val: &status);
807
808	if (status & PCI_X_SSTATUS_533MHZ) {
809	max = PCI_SPEED_133MHz_PCIX_533;
810	} else if (status & PCI_X_SSTATUS_266MHZ) {
811	max = PCI_SPEED_133MHz_PCIX_266;
812	} else if (status & PCI_X_SSTATUS_133MHZ) {
813	if ((status & PCI_X_SSTATUS_VERS) == PCI_X_SSTATUS_V2)
814	max = PCI_SPEED_133MHz_PCIX_ECC;
815	else
816	max = PCI_SPEED_133MHz_PCIX;
817	} else {
818	max = PCI_SPEED_66MHz_PCIX;
819	}
820
821	bus->max_bus_speed = max;
822	bus->cur_bus_speed =
823	pcix_bus_speed[FIELD_GET(PCI_X_SSTATUS_FREQ, status)];
824
825	return;
826	}
827
828	if (pci_is_pcie(dev: bridge)) {
829	u32 linkcap;
830	u16 linksta;
831
832	pcie_capability_read_dword(dev: bridge, PCI_EXP_LNKCAP, val: &linkcap);
833	bus->max_bus_speed = pcie_link_speed[linkcap & PCI_EXP_LNKCAP_SLS];
834
835	pcie_capability_read_word(dev: bridge, PCI_EXP_LNKSTA, val: &linksta);
836	pcie_update_link_speed(bus, linksta);
837	}
838	}
839
840	static struct irq_domain *pci_host_bridge_msi_domain(struct pci_bus *bus)
841	{
842	struct irq_domain *d;
843
844	/* If the host bridge driver sets a MSI domain of the bridge, use it */
845	d = dev_get_msi_domain(dev: bus->bridge);
846
847	/*
848	* Any firmware interface that can resolve the msi_domain
849	* should be called from here.
850	*/
851	if (!d)
852	d = pci_host_bridge_of_msi_domain(bus);
853	if (!d)
854	d = pci_host_bridge_acpi_msi_domain(bus);
855
856	/*
857	* If no IRQ domain was found via the OF tree, try looking it up
858	* directly through the fwnode_handle.
859	*/
860	if (!d) {
861	struct fwnode_handle *fwnode = pci_root_bus_fwnode(bus);
862
863	if (fwnode)
864	d = irq_find_matching_fwnode(fwnode,
865	bus_token: DOMAIN_BUS_PCI_MSI);
866	}
867
868	return d;
869	}
870
871	static void pci_set_bus_msi_domain(struct pci_bus *bus)
872	{
873	struct irq_domain *d;
874	struct pci_bus *b;
875
876	/*
877	* The bus can be a root bus, a subordinate bus, or a virtual bus
878	* created by an SR-IOV device. Walk up to the first bridge device
879	* found or derive the domain from the host bridge.
880	*/
881	for (b = bus, d = NULL; !d && !pci_is_root_bus(pbus: b); b = b->parent) {
882	if (b->self)
883	d = dev_get_msi_domain(dev: &b->self->dev);
884	}
885
886	if (!d)
887	d = pci_host_bridge_msi_domain(bus: b);
888
889	dev_set_msi_domain(dev: &bus->dev, d);
890	}
891
892	static int pci_register_host_bridge(struct pci_host_bridge *bridge)
893	{
894	struct device *parent = bridge->dev.parent;
895	struct resource_entry *window, *next, *n;
896	struct pci_bus *bus, *b;
897	resource_size_t offset, next_offset;
898	LIST_HEAD(resources);
899	struct resource *res, *next_res;
900	char addr[64], *fmt;
901	const char *name;
902	int err;
903
904	bus = pci_alloc_bus(NULL);
905	if (!bus)
906	return -ENOMEM;
907
908	bridge->bus = bus;
909
910	bus->sysdata = bridge->sysdata;
911	bus->ops = bridge->ops;
912	bus->number = bus->busn_res.start = bridge->busnr;
913	#ifdef CONFIG_PCI_DOMAINS_GENERIC
914	if (bridge->domain_nr == PCI_DOMAIN_NR_NOT_SET)
915	bus->domain_nr = pci_bus_find_domain_nr(bus, parent);
916	else
917	bus->domain_nr = bridge->domain_nr;
918	if (bus->domain_nr < 0) {
919	err = bus->domain_nr;
920	goto free;
921	}
922	#endif
923
924	b = pci_find_bus(domain: pci_domain_nr(bus), busnr: bridge->busnr);
925	if (b) {
926	/* Ignore it if we already got here via a different bridge */
927	dev_dbg(&b->dev, "bus already known\n");
928	err = -EEXIST;
929	goto free;
930	}
931
932	dev_set_name(dev: &bridge->dev, name: "pci%04x:%02x", pci_domain_nr(bus),
933	bridge->busnr);
934
935	err = pcibios_root_bridge_prepare(bridge);
936	if (err)
937	goto free;
938
939	/* Temporarily move resources off the list */
940	list_splice_init(list: &bridge->windows, head: &resources);
941	err = device_add(dev: &bridge->dev);
942	if (err) {
943	put_device(dev: &bridge->dev);
944	goto free;
945	}
946	bus->bridge = get_device(dev: &bridge->dev);
947	device_enable_async_suspend(dev: bus->bridge);
948	pci_set_bus_of_node(bus);
949	pci_set_bus_msi_domain(bus);
950	if (bridge->msi_domain && !dev_get_msi_domain(dev: &bus->dev) &&
951	!pci_host_of_has_msi_map(dev: parent))
952	bus->bus_flags |= PCI_BUS_FLAGS_NO_MSI;
953
954	if (!parent)
955	set_dev_node(dev: bus->bridge, pcibus_to_node(bus));
956
957	bus->dev.class = &pcibus_class;
958	bus->dev.parent = bus->bridge;
959
960	dev_set_name(dev: &bus->dev, name: "%04x:%02x", pci_domain_nr(bus), bus->number);
961	name = dev_name(dev: &bus->dev);
962
963	err = device_register(dev: &bus->dev);
964	if (err)
965	goto unregister;
966
967	pcibios_add_bus(bus);
968
969	if (bus->ops->add_bus) {
970	err = bus->ops->add_bus(bus);
971	if (WARN_ON(err < 0))
972	dev_err(&bus->dev, "failed to add bus: %d\n", err);
973	}
974
975	/* Create legacy_io and legacy_mem files for this bus */
976	pci_create_legacy_files(bus);
977
978	if (parent)
979	dev_info(parent, "PCI host bridge to bus %s\n", name);
980	else
981	pr_info("PCI host bridge to bus %s\n", name);
982
983	if (nr_node_ids > 1 && pcibus_to_node(bus) == NUMA_NO_NODE)
984	dev_warn(&bus->dev, "Unknown NUMA node; performance will be reduced\n");
985
986	/* Coalesce contiguous windows */
987	resource_list_for_each_entry_safe(window, n, &resources) {
988	if (list_is_last(list: &window->node, head: &resources))
989	break;
990
991	next = list_next_entry(window, node);
992	offset = window->offset;
993	res = window->res;
994	next_offset = next->offset;
995	next_res = next->res;
996
997	if (res->flags != next_res->flags || offset != next_offset)
998	continue;
999
1000	if (res->end + 1 == next_res->start) {
1001	next_res->start = res->start;
1002	res->flags = res->start = res->end = 0;
1003	}
1004	}
1005
1006	/* Add initial resources to the bus */
1007	resource_list_for_each_entry_safe(window, n, &resources) {
1008	offset = window->offset;
1009	res = window->res;
1010	if (!res->flags && !res->start && !res->end) {
1011	release_resource(new: res);
1012	resource_list_destroy_entry(entry: window);
1013	continue;
1014	}
1015
1016	list_move_tail(list: &window->node, head: &bridge->windows);
1017
1018	if (res->flags & IORESOURCE_BUS)
1019	pci_bus_insert_busn_res(b: bus, bus: bus->number, busmax: res->end);
1020	else
1021	pci_bus_add_resource(bus, res, flags: 0);
1022
1023	if (offset) {
1024	if (resource_type(res) == IORESOURCE_IO)
1025	fmt = " (bus address [%#06llx-%#06llx])";
1026	else
1027	fmt = " (bus address [%#010llx-%#010llx])";
1028
1029	snprintf(buf: addr, size: sizeof(addr), fmt,
1030	(unsigned long long)(res->start - offset),
1031	(unsigned long long)(res->end - offset));
1032	} else
1033	addr[0] = '\0';
1034
1035	dev_info(&bus->dev, "root bus resource %pR%s\n", res, addr);
1036	}
1037
1038	down_write(sem: &pci_bus_sem);
1039	list_add_tail(new: &bus->node, head: &pci_root_buses);
1040	up_write(sem: &pci_bus_sem);
1041
1042	return 0;
1043
1044	unregister:
1045	put_device(dev: &bridge->dev);
1046	device_del(dev: &bridge->dev);
1047
1048	free:
1049	#ifdef CONFIG_PCI_DOMAINS_GENERIC
1050	pci_bus_release_domain_nr(bus, parent);
1051	#endif
1052	kfree(objp: bus);
1053	return err;
1054	}
1055
1056	static bool pci_bridge_child_ext_cfg_accessible(struct pci_dev *bridge)
1057	{
1058	int pos;
1059	u32 status;
1060
1061	/*
1062	* If extended config space isn't accessible on a bridge's primary
1063	* bus, we certainly can't access it on the secondary bus.
1064	*/
1065	if (bridge->bus->bus_flags & PCI_BUS_FLAGS_NO_EXTCFG)
1066	return false;
1067
1068	/*
1069	* PCIe Root Ports and switch ports are PCIe on both sides, so if
1070	* extended config space is accessible on the primary, it's also
1071	* accessible on the secondary.
1072	*/
1073	if (pci_is_pcie(dev: bridge) &&
1074	(pci_pcie_type(dev: bridge) == PCI_EXP_TYPE_ROOT_PORT ||
1075	pci_pcie_type(dev: bridge) == PCI_EXP_TYPE_UPSTREAM ||
1076	pci_pcie_type(dev: bridge) == PCI_EXP_TYPE_DOWNSTREAM))
1077	return true;
1078
1079	/*
1080	* For the other bridge types:
1081	* - PCI-to-PCI bridges
1082	* - PCIe-to-PCI/PCI-X forward bridges
1083	* - PCI/PCI-X-to-PCIe reverse bridges
1084	* extended config space on the secondary side is only accessible
1085	* if the bridge supports PCI-X Mode 2.
1086	*/
1087	pos = pci_find_capability(dev: bridge, PCI_CAP_ID_PCIX);
1088	if (!pos)
1089	return false;
1090
1091	pci_read_config_dword(dev: bridge, where: pos + PCI_X_STATUS, val: &status);
1092	return status & (PCI_X_STATUS_266MHZ | PCI_X_STATUS_533MHZ);
1093	}
1094
1095	static struct pci_bus *pci_alloc_child_bus(struct pci_bus *parent,
1096	struct pci_dev *bridge, int busnr)
1097	{
1098	struct pci_bus *child;
1099	struct pci_host_bridge *host;
1100	int i;
1101	int ret;
1102
1103	/* Allocate a new bus and inherit stuff from the parent */
1104	child = pci_alloc_bus(parent);
1105	if (!child)
1106	return NULL;
1107
1108	child->parent = parent;
1109	child->sysdata = parent->sysdata;
1110	child->bus_flags = parent->bus_flags;
1111
1112	host = pci_find_host_bridge(bus: parent);
1113	if (host->child_ops)
1114	child->ops = host->child_ops;
1115	else
1116	child->ops = parent->ops;
1117
1118	/*
1119	* Initialize some portions of the bus device, but don't register
1120	* it now as the parent is not properly set up yet.
1121	*/
1122	child->dev.class = &pcibus_class;
1123	dev_set_name(dev: &child->dev, name: "%04x:%02x", pci_domain_nr(bus: child), busnr);
1124
1125	/* Set up the primary, secondary and subordinate bus numbers */
1126	child->number = child->busn_res.start = busnr;
1127	child->primary = parent->busn_res.start;
1128	child->busn_res.end = 0xff;
1129
1130	if (!bridge) {
1131	child->dev.parent = parent->bridge;
1132	goto add_dev;
1133	}
1134
1135	child->self = bridge;
1136	child->bridge = get_device(dev: &bridge->dev);
1137	child->dev.parent = child->bridge;
1138	pci_set_bus_of_node(bus: child);
1139	pci_set_bus_speed(bus: child);
1140
1141	/*
1142	* Check whether extended config space is accessible on the child
1143	* bus. Note that we currently assume it is always accessible on
1144	* the root bus.
1145	*/
1146	if (!pci_bridge_child_ext_cfg_accessible(bridge)) {
1147	child->bus_flags |= PCI_BUS_FLAGS_NO_EXTCFG;
1148	pci_info(child, "extended config space not accessible\n");
1149	}
1150
1151	/* Set up default resource pointers and names */
1152	for (i = 0; i < PCI_BRIDGE_RESOURCE_NUM; i++) {
1153	child->resource[i] = &bridge->resource[PCI_BRIDGE_RESOURCES+i];
1154	child->resource[i]->name = child->name;
1155	}
1156	bridge->subordinate = child;
1157
1158	add_dev:
1159	pci_set_bus_msi_domain(bus: child);
1160	ret = device_register(dev: &child->dev);
1161	WARN_ON(ret < 0);
1162
1163	pcibios_add_bus(bus: child);
1164
1165	if (child->ops->add_bus) {
1166	ret = child->ops->add_bus(child);
1167	if (WARN_ON(ret < 0))
1168	dev_err(&child->dev, "failed to add bus: %d\n", ret);
1169	}
1170
1171	/* Create legacy_io and legacy_mem files for this bus */
1172	pci_create_legacy_files(bus: child);
1173
1174	return child;
1175	}
1176
1177	struct pci_bus *pci_add_new_bus(struct pci_bus *parent, struct pci_dev *dev,
1178	int busnr)
1179	{
1180	struct pci_bus *child;
1181
1182	child = pci_alloc_child_bus(parent, bridge: dev, busnr);
1183	if (child) {
1184	down_write(sem: &pci_bus_sem);
1185	list_add_tail(new: &child->node, head: &parent->children);
1186	up_write(sem: &pci_bus_sem);
1187	}
1188	return child;
1189	}
1190	EXPORT_SYMBOL(pci_add_new_bus);
1191
1192	static void pci_enable_crs(struct pci_dev *pdev)
1193	{
1194	u16 root_cap = 0;
1195
1196	/* Enable CRS Software Visibility if supported */
1197	pcie_capability_read_word(dev: pdev, PCI_EXP_RTCAP, val: &root_cap);
1198	if (root_cap & PCI_EXP_RTCAP_CRSVIS)
1199	pcie_capability_set_word(dev: pdev, PCI_EXP_RTCTL,
1200	PCI_EXP_RTCTL_CRSSVE);
1201	}
1202
1203	static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus,
1204	unsigned int available_buses);
1205	/**
1206	* pci_ea_fixed_busnrs() - Read fixed Secondary and Subordinate bus
1207	* numbers from EA capability.
1208	* @dev: Bridge
1209	* @sec: updated with secondary bus number from EA
1210	* @sub: updated with subordinate bus number from EA
1211	*
1212	* If @dev is a bridge with EA capability that specifies valid secondary
1213	* and subordinate bus numbers, return true with the bus numbers in @sec
1214	* and @sub. Otherwise return false.
1215	*/
1216	static bool pci_ea_fixed_busnrs(struct pci_dev *dev, u8 *sec, u8 *sub)
1217	{
1218	int ea, offset;
1219	u32 dw;
1220	u8 ea_sec, ea_sub;
1221
1222	if (dev->hdr_type != PCI_HEADER_TYPE_BRIDGE)
1223	return false;
1224
1225	/* find PCI EA capability in list */
1226	ea = pci_find_capability(dev, PCI_CAP_ID_EA);
1227	if (!ea)
1228	return false;
1229
1230	offset = ea + PCI_EA_FIRST_ENT;
1231	pci_read_config_dword(dev, where: offset, val: &dw);
1232	ea_sec = FIELD_GET(PCI_EA_SEC_BUS_MASK, dw);
1233	ea_sub = FIELD_GET(PCI_EA_SUB_BUS_MASK, dw);
1234	if (ea_sec == 0 || ea_sub < ea_sec)
1235	return false;
1236
1237	*sec = ea_sec;
1238	*sub = ea_sub;
1239	return true;
1240	}
1241
1242	/*
1243	* pci_scan_bridge_extend() - Scan buses behind a bridge
1244	* @bus: Parent bus the bridge is on
1245	* @dev: Bridge itself
1246	* @max: Starting subordinate number of buses behind this bridge
1247	* @available_buses: Total number of buses available for this bridge and
1248	* the devices below. After the minimal bus space has
1249	* been allocated the remaining buses will be
1250	* distributed equally between hotplug-capable bridges.
1251	* @pass: Either %0 (scan already configured bridges) or %1 (scan bridges
1252	* that need to be reconfigured.
1253	*
1254	* If it's a bridge, configure it and scan the bus behind it.
1255	* For CardBus bridges, we don't scan behind as the devices will
1256	* be handled by the bridge driver itself.
1257	*
1258	* We need to process bridges in two passes -- first we scan those
1259	* already configured by the BIOS and after we are done with all of
1260	* them, we proceed to assigning numbers to the remaining buses in
1261	* order to avoid overlaps between old and new bus numbers.
1262	*
1263	* Return: New subordinate number covering all buses behind this bridge.
1264	*/
1265	static int pci_scan_bridge_extend(struct pci_bus *bus, struct pci_dev *dev,
1266	int max, unsigned int available_buses,
1267	int pass)
1268	{
1269	struct pci_bus *child;
1270	int is_cardbus = (dev->hdr_type == PCI_HEADER_TYPE_CARDBUS);
1271	u32 buses, i, j = 0;
1272	u16 bctl;
1273	u8 primary, secondary, subordinate;
1274	int broken = 0;
1275	bool fixed_buses;
1276	u8 fixed_sec, fixed_sub;
1277	int next_busnr;
1278
1279	/*
1280	* Make sure the bridge is powered on to be able to access config
1281	* space of devices below it.
1282	*/
1283	pm_runtime_get_sync(dev: &dev->dev);
1284
1285	pci_read_config_dword(dev, PCI_PRIMARY_BUS, val: &buses);
1286	primary = buses & 0xFF;
1287	secondary = (buses >> 8) & 0xFF;
1288	subordinate = (buses >> 16) & 0xFF;
1289
1290	pci_dbg(dev, "scanning [bus %02x-%02x] behind bridge, pass %d\n",
1291	secondary, subordinate, pass);
1292
1293	if (!primary && (primary != bus->number) && secondary && subordinate) {
1294	pci_warn(dev, "Primary bus is hard wired to 0\n");
1295	primary = bus->number;
1296	}
1297
1298	/* Check if setup is sensible at all */
1299	if (!pass &&
1300	(primary != bus->number || secondary <= bus->number ||
1301	secondary > subordinate)) {
1302	pci_info(dev, "bridge configuration invalid ([bus %02x-%02x]), reconfiguring\n",
1303	secondary, subordinate);
1304	broken = 1;
1305	}
1306
1307	/*
1308	* Disable Master-Abort Mode during probing to avoid reporting of
1309	* bus errors in some architectures.
1310	*/
1311	pci_read_config_word(dev, PCI_BRIDGE_CONTROL, val: &bctl);
1312	pci_write_config_word(dev, PCI_BRIDGE_CONTROL,
1313	val: bctl & ~PCI_BRIDGE_CTL_MASTER_ABORT);
1314
1315	pci_enable_crs(pdev: dev);
1316
1317	if ((secondary || subordinate) && !pcibios_assign_all_busses() &&
1318	!is_cardbus && !broken) {
1319	unsigned int cmax, buses;
1320
1321	/*
1322	* Bus already configured by firmware, process it in the
1323	* first pass and just note the configuration.
1324	*/
1325	if (pass)
1326	goto out;
1327
1328	/*
1329	* The bus might already exist for two reasons: Either we
1330	* are rescanning the bus or the bus is reachable through
1331	* more than one bridge. The second case can happen with
1332	* the i450NX chipset.
1333	*/
1334	child = pci_find_bus(domain: pci_domain_nr(bus), busnr: secondary);
1335	if (!child) {
1336	child = pci_add_new_bus(bus, dev, secondary);
1337	if (!child)
1338	goto out;
1339	child->primary = primary;
1340	pci_bus_insert_busn_res(b: child, bus: secondary, busmax: subordinate);
1341	child->bridge_ctl = bctl;
1342	}
1343
1344	buses = subordinate - secondary;
1345	cmax = pci_scan_child_bus_extend(bus: child, available_buses: buses);
1346	if (cmax > subordinate)
1347	pci_warn(dev, "bridge has subordinate %02x but max busn %02x\n",
1348	subordinate, cmax);
1349
1350	/* Subordinate should equal child->busn_res.end */
1351	if (subordinate > max)
1352	max = subordinate;
1353	} else {
1354
1355	/*
1356	* We need to assign a number to this bus which we always
1357	* do in the second pass.
1358	*/
1359	if (!pass) {
1360	if (pcibios_assign_all_busses() || broken || is_cardbus)
1361
1362	/*
1363	* Temporarily disable forwarding of the
1364	* configuration cycles on all bridges in
1365	* this bus segment to avoid possible
1366	* conflicts in the second pass between two
1367	* bridges programmed with overlapping bus
1368	* ranges.
1369	*/
1370	pci_write_config_dword(dev, PCI_PRIMARY_BUS,
1371	val: buses & ~0xffffff);
1372	goto out;
1373	}
1374
1375	/* Clear errors */
1376	pci_write_config_word(dev, PCI_STATUS, val: 0xffff);
1377
1378	/* Read bus numbers from EA Capability (if present) */
1379	fixed_buses = pci_ea_fixed_busnrs(dev, sec: &fixed_sec, sub: &fixed_sub);
1380	if (fixed_buses)
1381	next_busnr = fixed_sec;
1382	else
1383	next_busnr = max + 1;
1384
1385	/*
1386	* Prevent assigning a bus number that already exists.
1387	* This can happen when a bridge is hot-plugged, so in this
1388	* case we only re-scan this bus.
1389	*/
1390	child = pci_find_bus(domain: pci_domain_nr(bus), busnr: next_busnr);
1391	if (!child) {
1392	child = pci_add_new_bus(bus, dev, next_busnr);
1393	if (!child)
1394	goto out;
1395	pci_bus_insert_busn_res(b: child, bus: next_busnr,
1396	busmax: bus->busn_res.end);
1397	}
1398	max++;
1399	if (available_buses)
1400	available_buses--;
1401
1402	buses = (buses & 0xff000000)
1403	| ((unsigned int)(child->primary) << 0)
1404	| ((unsigned int)(child->busn_res.start) << 8)
1405	| ((unsigned int)(child->busn_res.end) << 16);
1406
1407	/*
1408	* yenta.c forces a secondary latency timer of 176.
1409	* Copy that behaviour here.
1410	*/
1411	if (is_cardbus) {
1412	buses &= ~0xff000000;
1413	buses |= CARDBUS_LATENCY_TIMER << 24;
1414	}
1415
1416	/* We need to blast all three values with a single write */
1417	pci_write_config_dword(dev, PCI_PRIMARY_BUS, val: buses);
1418
1419	if (!is_cardbus) {
1420	child->bridge_ctl = bctl;
1421	max = pci_scan_child_bus_extend(bus: child, available_buses);
1422	} else {
1423
1424	/*
1425	* For CardBus bridges, we leave 4 bus numbers as
1426	* cards with a PCI-to-PCI bridge can be inserted
1427	* later.
1428	*/
1429	for (i = 0; i < CARDBUS_RESERVE_BUSNR; i++) {
1430	struct pci_bus *parent = bus;
1431	if (pci_find_bus(domain: pci_domain_nr(bus),
1432	busnr: max+i+1))
1433	break;
1434	while (parent->parent) {
1435	if ((!pcibios_assign_all_busses()) &&
1436	(parent->busn_res.end > max) &&
1437	(parent->busn_res.end <= max+i)) {
1438	j = 1;
1439	}
1440	parent = parent->parent;
1441	}
1442	if (j) {
1443
1444	/*
1445	* Often, there are two CardBus
1446	* bridges -- try to leave one
1447	* valid bus number for each one.
1448	*/
1449	i /= 2;
1450	break;
1451	}
1452	}
1453	max += i;
1454	}
1455
1456	/*
1457	* Set subordinate bus number to its real value.
1458	* If fixed subordinate bus number exists from EA
1459	* capability then use it.
1460	*/
1461	if (fixed_buses)
1462	max = fixed_sub;
1463	pci_bus_update_busn_res_end(b: child, busmax: max);
1464	pci_write_config_byte(dev, PCI_SUBORDINATE_BUS, val: max);
1465	}
1466
1467	sprintf(buf: child->name,
1468	fmt: (is_cardbus ? "PCI CardBus %04x:%02x" : "PCI Bus %04x:%02x"),
1469	pci_domain_nr(bus), child->number);
1470
1471	/* Check that all devices are accessible */
1472	while (bus->parent) {
1473	if ((child->busn_res.end > bus->busn_res.end) ||
1474	(child->number > bus->busn_res.end) ||
1475	(child->number < bus->number) ||
1476	(child->busn_res.end < bus->number)) {
1477	dev_info(&dev->dev, "devices behind bridge are unusable because %pR cannot be assigned for them\n",
1478	&child->busn_res);
1479	break;
1480	}
1481	bus = bus->parent;
1482	}
1483
1484	out:
1485	pci_write_config_word(dev, PCI_BRIDGE_CONTROL, val: bctl);
1486
1487	pm_runtime_put(dev: &dev->dev);
1488
1489	return max;
1490	}
1491
1492	/*
1493	* pci_scan_bridge() - Scan buses behind a bridge
1494	* @bus: Parent bus the bridge is on
1495	* @dev: Bridge itself
1496	* @max: Starting subordinate number of buses behind this bridge
1497	* @pass: Either %0 (scan already configured bridges) or %1 (scan bridges
1498	* that need to be reconfigured.
1499	*
1500	* If it's a bridge, configure it and scan the bus behind it.
1501	* For CardBus bridges, we don't scan behind as the devices will
1502	* be handled by the bridge driver itself.
1503	*
1504	* We need to process bridges in two passes -- first we scan those
1505	* already configured by the BIOS and after we are done with all of
1506	* them, we proceed to assigning numbers to the remaining buses in
1507	* order to avoid overlaps between old and new bus numbers.
1508	*
1509	* Return: New subordinate number covering all buses behind this bridge.
1510	*/
1511	int pci_scan_bridge(struct pci_bus *bus, struct pci_dev *dev, int max, int pass)
1512	{
1513	return pci_scan_bridge_extend(bus, dev, max, available_buses: 0, pass);
1514	}
1515	EXPORT_SYMBOL(pci_scan_bridge);
1516
1517	/*
1518	* Read interrupt line and base address registers.
1519	* The architecture-dependent code can tweak these, of course.
1520	*/
1521	static void pci_read_irq(struct pci_dev *dev)
1522	{
1523	unsigned char irq;
1524
1525	/* VFs are not allowed to use INTx, so skip the config reads */
1526	if (dev->is_virtfn) {
1527	dev->pin = 0;
1528	dev->irq = 0;
1529	return;
1530	}
1531
1532	pci_read_config_byte(dev, PCI_INTERRUPT_PIN, val: &irq);
1533	dev->pin = irq;
1534	if (irq)
1535	pci_read_config_byte(dev, PCI_INTERRUPT_LINE, val: &irq);
1536	dev->irq = irq;
1537	}
1538
1539	void set_pcie_port_type(struct pci_dev *pdev)
1540	{
1541	int pos;
1542	u16 reg16;
1543	u32 reg32;
1544	int type;
1545	struct pci_dev *parent;
1546
1547	pos = pci_find_capability(dev: pdev, PCI_CAP_ID_EXP);
1548	if (!pos)
1549	return;
1550
1551	pdev->pcie_cap = pos;
1552	pci_read_config_word(dev: pdev, where: pos + PCI_EXP_FLAGS, val: &reg16);
1553	pdev->pcie_flags_reg = reg16;
1554	pci_read_config_dword(dev: pdev, where: pos + PCI_EXP_DEVCAP, val: &pdev->devcap);
1555	pdev->pcie_mpss = FIELD_GET(PCI_EXP_DEVCAP_PAYLOAD, pdev->devcap);
1556
1557	pcie_capability_read_dword(dev: pdev, PCI_EXP_LNKCAP, val: &reg32);
1558	if (reg32 & PCI_EXP_LNKCAP_DLLLARC)
1559	pdev->link_active_reporting = 1;
1560
1561	parent = pci_upstream_bridge(dev: pdev);
1562	if (!parent)
1563	return;
1564
1565	/*
1566	* Some systems do not identify their upstream/downstream ports
1567	* correctly so detect impossible configurations here and correct
1568	* the port type accordingly.
1569	*/
1570	type = pci_pcie_type(dev: pdev);
1571	if (type == PCI_EXP_TYPE_DOWNSTREAM) {
1572	/*
1573	* If pdev claims to be downstream port but the parent
1574	* device is also downstream port assume pdev is actually
1575	* upstream port.
1576	*/
1577	if (pcie_downstream_port(dev: parent)) {
1578	pci_info(pdev, "claims to be downstream port but is acting as upstream port, correcting type\n");
1579	pdev->pcie_flags_reg &= ~PCI_EXP_FLAGS_TYPE;
1580	pdev->pcie_flags_reg |= PCI_EXP_TYPE_UPSTREAM;
1581	}
1582	} else if (type == PCI_EXP_TYPE_UPSTREAM) {
1583	/*
1584	* If pdev claims to be upstream port but the parent
1585	* device is also upstream port assume pdev is actually
1586	* downstream port.
1587	*/
1588	if (pci_pcie_type(dev: parent) == PCI_EXP_TYPE_UPSTREAM) {
1589	pci_info(pdev, "claims to be upstream port but is acting as downstream port, correcting type\n");
1590	pdev->pcie_flags_reg &= ~PCI_EXP_FLAGS_TYPE;
1591	pdev->pcie_flags_reg |= PCI_EXP_TYPE_DOWNSTREAM;
1592	}
1593	}
1594	}
1595
1596	void set_pcie_hotplug_bridge(struct pci_dev *pdev)
1597	{
1598	u32 reg32;
1599
1600	pcie_capability_read_dword(dev: pdev, PCI_EXP_SLTCAP, val: &reg32);
1601	if (reg32 & PCI_EXP_SLTCAP_HPC)
1602	pdev->is_hotplug_bridge = 1;
1603	}
1604
1605	static void set_pcie_thunderbolt(struct pci_dev *dev)
1606	{
1607	u16 vsec;
1608
1609	/* Is the device part of a Thunderbolt controller? */
1610	vsec = pci_find_vsec_capability(dev, PCI_VENDOR_ID_INTEL, PCI_VSEC_ID_INTEL_TBT);
1611	if (vsec)
1612	dev->is_thunderbolt = 1;
1613	}
1614
1615	static void set_pcie_untrusted(struct pci_dev *dev)
1616	{
1617	struct pci_dev *parent;
1618
1619	/*
1620	* If the upstream bridge is untrusted we treat this device
1621	* untrusted as well.
1622	*/
1623	parent = pci_upstream_bridge(dev);
1624	if (parent && (parent->untrusted || parent->external_facing))
1625	dev->untrusted = true;
1626	}
1627
1628	static void pci_set_removable(struct pci_dev *dev)
1629	{
1630	struct pci_dev *parent = pci_upstream_bridge(dev);
1631
1632	/*
1633	* We (only) consider everything downstream from an external_facing
1634	* device to be removable by the user. We're mainly concerned with
1635	* consumer platforms with user accessible thunderbolt ports that are
1636	* vulnerable to DMA attacks, and we expect those ports to be marked by
1637	* the firmware as external_facing. Devices in traditional hotplug
1638	* slots can technically be removed, but the expectation is that unless
1639	* the port is marked with external_facing, such devices are less
1640	* accessible to user / may not be removed by end user, and thus not
1641	* exposed as "removable" to userspace.
1642	*/
1643	if (parent &&
1644	(parent->external_facing || dev_is_removable(dev: &parent->dev)))
1645	dev_set_removable(dev: &dev->dev, removable: DEVICE_REMOVABLE);
1646	}
1647
1648	/**
1649	* pci_ext_cfg_is_aliased - Is ext config space just an alias of std config?
1650	* @dev: PCI device
1651	*
1652	* PCI Express to PCI/PCI-X Bridge Specification, rev 1.0, 4.1.4 says that
1653	* when forwarding a type1 configuration request the bridge must check that
1654	* the extended register address field is zero. The bridge is not permitted
1655	* to forward the transactions and must handle it as an Unsupported Request.
1656	* Some bridges do not follow this rule and simply drop the extended register
1657	* bits, resulting in the standard config space being aliased, every 256
1658	* bytes across the entire configuration space. Test for this condition by
1659	* comparing the first dword of each potential alias to the vendor/device ID.
1660	* Known offenders:
1661	* ASM1083/1085 PCIe-to-PCI Reversible Bridge (1b21:1080, rev 01 & 03)
1662	* AMD/ATI SBx00 PCI to PCI Bridge (1002:4384, rev 40)
1663	*/
1664	static bool pci_ext_cfg_is_aliased(struct pci_dev *dev)
1665	{
1666	#ifdef CONFIG_PCI_QUIRKS
1667	int pos, ret;
1668	u32 header, tmp;
1669
1670	pci_read_config_dword(dev, PCI_VENDOR_ID, val: &header);
1671
1672	for (pos = PCI_CFG_SPACE_SIZE;
1673	pos < PCI_CFG_SPACE_EXP_SIZE; pos += PCI_CFG_SPACE_SIZE) {
1674	ret = pci_read_config_dword(dev, where: pos, val: &tmp);
1675	if ((ret != PCIBIOS_SUCCESSFUL) || (header != tmp))
1676	return false;
1677	}
1678
1679	return true;
1680	#else
1681	return false;
1682	#endif
1683	}
1684
1685	/**
1686	* pci_cfg_space_size_ext - Get the configuration space size of the PCI device
1687	* @dev: PCI device
1688	*
1689	* Regular PCI devices have 256 bytes, but PCI-X 2 and PCI Express devices
1690	* have 4096 bytes. Even if the device is capable, that doesn't mean we can
1691	* access it. Maybe we don't have a way to generate extended config space
1692	* accesses, or the device is behind a reverse Express bridge. So we try
1693	* reading the dword at 0x100 which must either be 0 or a valid extended
1694	* capability header.
1695	*/
1696	static int pci_cfg_space_size_ext(struct pci_dev *dev)
1697	{
1698	u32 status;
1699	int pos = PCI_CFG_SPACE_SIZE;
1700
1701	if (pci_read_config_dword(dev, where: pos, val: &status) != PCIBIOS_SUCCESSFUL)
1702	return PCI_CFG_SPACE_SIZE;
1703	if (PCI_POSSIBLE_ERROR(status) || pci_ext_cfg_is_aliased(dev))
1704	return PCI_CFG_SPACE_SIZE;
1705
1706	return PCI_CFG_SPACE_EXP_SIZE;
1707	}
1708
1709	int pci_cfg_space_size(struct pci_dev *dev)
1710	{
1711	int pos;
1712	u32 status;
1713	u16 class;
1714
1715	#ifdef CONFIG_PCI_IOV
1716	/*
1717	* Per the SR-IOV specification (rev 1.1, sec 3.5), VFs are required to
1718	* implement a PCIe capability and therefore must implement extended
1719	* config space. We can skip the NO_EXTCFG test below and the
1720	* reachability/aliasing test in pci_cfg_space_size_ext() by virtue of
1721	* the fact that the SR-IOV capability on the PF resides in extended
1722	* config space and must be accessible and non-aliased to have enabled
1723	* support for this VF. This is a micro performance optimization for
1724	* systems supporting many VFs.
1725	*/
1726	if (dev->is_virtfn)
1727	return PCI_CFG_SPACE_EXP_SIZE;
1728	#endif
1729
1730	if (dev->bus->bus_flags & PCI_BUS_FLAGS_NO_EXTCFG)
1731	return PCI_CFG_SPACE_SIZE;
1732
1733	class = dev->class >> 8;
1734	if (class == PCI_CLASS_BRIDGE_HOST)
1735	return pci_cfg_space_size_ext(dev);
1736
1737	if (pci_is_pcie(dev))
1738	return pci_cfg_space_size_ext(dev);
1739
1740	pos = pci_find_capability(dev, PCI_CAP_ID_PCIX);
1741	if (!pos)
1742	return PCI_CFG_SPACE_SIZE;
1743
1744	pci_read_config_dword(dev, where: pos + PCI_X_STATUS, val: &status);
1745	if (status & (PCI_X_STATUS_266MHZ | PCI_X_STATUS_533MHZ))
1746	return pci_cfg_space_size_ext(dev);
1747
1748	return PCI_CFG_SPACE_SIZE;
1749	}
1750
1751	static u32 pci_class(struct pci_dev *dev)
1752	{
1753	u32 class;
1754
1755	#ifdef CONFIG_PCI_IOV
1756	if (dev->is_virtfn)
1757	return dev->physfn->sriov->class;
1758	#endif
1759	pci_read_config_dword(dev, PCI_CLASS_REVISION, val: &class);
1760	return class;
1761	}
1762
1763	static void pci_subsystem_ids(struct pci_dev *dev, u16 *vendor, u16 *device)
1764	{
1765	#ifdef CONFIG_PCI_IOV
1766	if (dev->is_virtfn) {
1767	*vendor = dev->physfn->sriov->subsystem_vendor;
1768	*device = dev->physfn->sriov->subsystem_device;
1769	return;
1770	}
1771	#endif
1772	pci_read_config_word(dev, PCI_SUBSYSTEM_VENDOR_ID, val: vendor);
1773	pci_read_config_word(dev, PCI_SUBSYSTEM_ID, val: device);
1774	}
1775
1776	static u8 pci_hdr_type(struct pci_dev *dev)
1777	{
1778	u8 hdr_type;
1779
1780	#ifdef CONFIG_PCI_IOV
1781	if (dev->is_virtfn)
1782	return dev->physfn->sriov->hdr_type;
1783	#endif
1784	pci_read_config_byte(dev, PCI_HEADER_TYPE, val: &hdr_type);
1785	return hdr_type;
1786	}
1787
1788	#define LEGACY_IO_RESOURCE (IORESOURCE_IO | IORESOURCE_PCI_FIXED)
1789
1790	/**
1791	* pci_intx_mask_broken - Test PCI_COMMAND_INTX_DISABLE writability
1792	* @dev: PCI device
1793	*
1794	* Test whether PCI_COMMAND_INTX_DISABLE is writable for @dev. Check this
1795	* at enumeration-time to avoid modifying PCI_COMMAND at run-time.
1796	*/
1797	static int pci_intx_mask_broken(struct pci_dev *dev)
1798	{
1799	u16 orig, toggle, new;
1800
1801	pci_read_config_word(dev, PCI_COMMAND, val: &orig);
1802	toggle = orig ^ PCI_COMMAND_INTX_DISABLE;
1803	pci_write_config_word(dev, PCI_COMMAND, val: toggle);
1804	pci_read_config_word(dev, PCI_COMMAND, val: &new);
1805
1806	pci_write_config_word(dev, PCI_COMMAND, val: orig);
1807
1808	/*
1809	* PCI_COMMAND_INTX_DISABLE was reserved and read-only prior to PCI
1810	* r2.3, so strictly speaking, a device is not *broken* if it's not
1811	* writable. But we'll live with the misnomer for now.
1812	*/
1813	if (new != toggle)
1814	return 1;
1815	return 0;
1816	}
1817
1818	static void early_dump_pci_device(struct pci_dev *pdev)
1819	{
1820	u32 value[256 / 4];
1821	int i;
1822
1823	pci_info(pdev, "config space:\n");
1824
1825	for (i = 0; i < 256; i += 4)
1826	pci_read_config_dword(dev: pdev, where: i, val: &value[i / 4]);
1827
1828	print_hex_dump(KERN_INFO, prefix_str: "", prefix_type: DUMP_PREFIX_OFFSET, rowsize: 16, groupsize: 1,
1829	buf: value, len: 256, ascii: false);
1830	}
1831
1832	static const char *pci_type_str(struct pci_dev *dev)
1833	{
1834	static const char * const str[] = {
1835	"PCIe Endpoint",
1836	"PCIe Legacy Endpoint",
1837	"PCIe unknown",
1838	"PCIe unknown",
1839	"PCIe Root Port",
1840	"PCIe Switch Upstream Port",
1841	"PCIe Switch Downstream Port",
1842	"PCIe to PCI/PCI-X bridge",
1843	"PCI/PCI-X to PCIe bridge",
1844	"PCIe Root Complex Integrated Endpoint",
1845	"PCIe Root Complex Event Collector",
1846	};
1847	int type;
1848
1849	if (pci_is_pcie(dev)) {
1850	type = pci_pcie_type(dev);
1851	if (type < ARRAY_SIZE(str))
1852	return str[type];
1853
1854	return "PCIe unknown";
1855	}
1856
1857	switch (dev->hdr_type) {
1858	case PCI_HEADER_TYPE_NORMAL:
1859	return "conventional PCI endpoint";
1860	case PCI_HEADER_TYPE_BRIDGE:
1861	return "conventional PCI bridge";
1862	case PCI_HEADER_TYPE_CARDBUS:
1863	return "CardBus bridge";
1864	default:
1865	return "conventional PCI";
1866	}
1867	}
1868
1869	/**
1870	* pci_setup_device - Fill in class and map information of a device
1871	* @dev: the device structure to fill
1872	*
1873	* Initialize the device structure with information about the device's
1874	* vendor,class,memory and IO-space addresses, IRQ lines etc.
1875	* Called at initialisation of the PCI subsystem and by CardBus services.
1876	* Returns 0 on success and negative if unknown type of device (not normal,
1877	* bridge or CardBus).
1878	*/
1879	int pci_setup_device(struct pci_dev *dev)
1880	{
1881	u32 class;
1882	u16 cmd;
1883	u8 hdr_type;
1884	int err, pos = 0;
1885	struct pci_bus_region region;
1886	struct resource *res;
1887
1888	hdr_type = pci_hdr_type(dev);
1889
1890	dev->sysdata = dev->bus->sysdata;
1891	dev->dev.parent = dev->bus->bridge;
1892	dev->dev.bus = &pci_bus_type;
1893	dev->hdr_type = hdr_type & 0x7f;
1894	dev->multifunction = !!(hdr_type & 0x80);
1895	dev->error_state = pci_channel_io_normal;
1896	set_pcie_port_type(dev);
1897
1898	err = pci_set_of_node(dev);
1899	if (err)
1900	return err;
1901	pci_set_acpi_fwnode(dev);
1902
1903	pci_dev_assign_slot(dev);
1904
1905	/*
1906	* Assume 32-bit PCI; let 64-bit PCI cards (which are far rarer)
1907	* set this higher, assuming the system even supports it.
1908	*/
1909	dev->dma_mask = 0xffffffff;
1910
1911	dev_set_name(dev: &dev->dev, name: "%04x:%02x:%02x.%d", pci_domain_nr(bus: dev->bus),
1912	dev->bus->number, PCI_SLOT(dev->devfn),
1913	PCI_FUNC(dev->devfn));
1914
1915	class = pci_class(dev);
1916
1917	dev->revision = class & 0xff;
1918	dev->class = class >> 8; /* upper 3 bytes */
1919
1920	if (pci_early_dump)
1921	early_dump_pci_device(pdev: dev);
1922
1923	/* Need to have dev->class ready */
1924	dev->cfg_size = pci_cfg_space_size(dev);
1925
1926	/* Need to have dev->cfg_size ready */
1927	set_pcie_thunderbolt(dev);
1928
1929	set_pcie_untrusted(dev);
1930
1931	/* "Unknown power state" */
1932	dev->current_state = PCI_UNKNOWN;
1933
1934	/* Early fixups, before probing the BARs */
1935	pci_fixup_device(pass: pci_fixup_early, dev);
1936
1937	pci_set_removable(dev);
1938
1939	pci_info(dev, "[%04x:%04x] type %02x class %#08x %s\n",
1940	dev->vendor, dev->device, dev->hdr_type, dev->class,
1941	pci_type_str(dev));
1942
1943	/* Device class may be changed after fixup */
1944	class = dev->class >> 8;
1945
1946	if (dev->non_compliant_bars && !dev->mmio_always_on) {
1947	pci_read_config_word(dev, PCI_COMMAND, val: &cmd);
1948	if (cmd & (PCI_COMMAND_IO | PCI_COMMAND_MEMORY)) {
1949	pci_info(dev, "device has non-compliant BARs; disabling IO/MEM decoding\n");
1950	cmd &= ~PCI_COMMAND_IO;
1951	cmd &= ~PCI_COMMAND_MEMORY;
1952	pci_write_config_word(dev, PCI_COMMAND, val: cmd);
1953	}
1954	}
1955
1956	dev->broken_intx_masking = pci_intx_mask_broken(dev);
1957
1958	switch (dev->hdr_type) { /* header type */
1959	case PCI_HEADER_TYPE_NORMAL: /* standard header */
1960	if (class == PCI_CLASS_BRIDGE_PCI)
1961	goto bad;
1962	pci_read_irq(dev);
1963	pci_read_bases(dev, howmany: 6, PCI_ROM_ADDRESS);
1964
1965	pci_subsystem_ids(dev, vendor: &dev->subsystem_vendor, device: &dev->subsystem_device);
1966
1967	/*
1968	* Do the ugly legacy mode stuff here rather than broken chip
1969	* quirk code. Legacy mode ATA controllers have fixed
1970	* addresses. These are not always echoed in BAR0-3, and
1971	* BAR0-3 in a few cases contain junk!
1972	*/
1973	if (class == PCI_CLASS_STORAGE_IDE) {
1974	u8 progif;
1975	pci_read_config_byte(dev, PCI_CLASS_PROG, val: &progif);
1976	if ((progif & 1) == 0) {
1977	region.start = 0x1F0;
1978	region.end = 0x1F7;
1979	res = &dev->resource[0];
1980	res->flags = LEGACY_IO_RESOURCE;
1981	pcibios_bus_to_resource(bus: dev->bus, res, region: &region);
1982	pci_info(dev, "BAR 0 %pR: legacy IDE quirk\n",
1983	res);
1984	region.start = 0x3F6;
1985	region.end = 0x3F6;
1986	res = &dev->resource[1];
1987	res->flags = LEGACY_IO_RESOURCE;
1988	pcibios_bus_to_resource(bus: dev->bus, res, region: &region);
1989	pci_info(dev, "BAR 1 %pR: legacy IDE quirk\n",
1990	res);
1991	}
1992	if ((progif & 4) == 0) {
1993	region.start = 0x170;
1994	region.end = 0x177;
1995	res = &dev->resource[2];
1996	res->flags = LEGACY_IO_RESOURCE;
1997	pcibios_bus_to_resource(bus: dev->bus, res, region: &region);
1998	pci_info(dev, "BAR 2 %pR: legacy IDE quirk\n",
1999	res);
2000	region.start = 0x376;
2001	region.end = 0x376;
2002	res = &dev->resource[3];
2003	res->flags = LEGACY_IO_RESOURCE;
2004	pcibios_bus_to_resource(bus: dev->bus, res, region: &region);
2005	pci_info(dev, "BAR 3 %pR: legacy IDE quirk\n",
2006	res);
2007	}
2008	}
2009	break;
2010
2011	case PCI_HEADER_TYPE_BRIDGE: /* bridge header */
2012	/*
2013	* The PCI-to-PCI bridge spec requires that subtractive
2014	* decoding (i.e. transparent) bridge must have programming
2015	* interface code of 0x01.
2016	*/
2017	pci_read_irq(dev);
2018	dev->transparent = ((dev->class & 0xff) == 1);
2019	pci_read_bases(dev, howmany: 2, PCI_ROM_ADDRESS1);
2020	pci_read_bridge_windows(bridge: dev);
2021	set_pcie_hotplug_bridge(dev);
2022	pos = pci_find_capability(dev, PCI_CAP_ID_SSVID);
2023	if (pos) {
2024	pci_read_config_word(dev, where: pos + PCI_SSVID_VENDOR_ID, val: &dev->subsystem_vendor);
2025	pci_read_config_word(dev, where: pos + PCI_SSVID_DEVICE_ID, val: &dev->subsystem_device);
2026	}
2027	break;
2028
2029	case PCI_HEADER_TYPE_CARDBUS: /* CardBus bridge header */
2030	if (class != PCI_CLASS_BRIDGE_CARDBUS)
2031	goto bad;
2032	pci_read_irq(dev);
2033	pci_read_bases(dev, howmany: 1, rom: 0);
2034	pci_read_config_word(dev, PCI_CB_SUBSYSTEM_VENDOR_ID, val: &dev->subsystem_vendor);
2035	pci_read_config_word(dev, PCI_CB_SUBSYSTEM_ID, val: &dev->subsystem_device);
2036	break;
2037
2038	default: /* unknown header */
2039	pci_err(dev, "unknown header type %02x, ignoring device\n",
2040	dev->hdr_type);
2041	pci_release_of_node(dev);
2042	return -EIO;
2043
2044	bad:
2045	pci_err(dev, "ignoring class %#08x (doesn't match header type %02x)\n",
2046	dev->class, dev->hdr_type);
2047	dev->class = PCI_CLASS_NOT_DEFINED << 8;
2048	}
2049
2050	/* We found a fine healthy device, go go go... */
2051	return 0;
2052	}
2053
2054	static void pci_configure_mps(struct pci_dev *dev)
2055	{
2056	struct pci_dev *bridge = pci_upstream_bridge(dev);
2057	int mps, mpss, p_mps, rc;
2058
2059	if (!pci_is_pcie(dev))
2060	return;
2061
2062	/* MPS and MRRS fields are of type 'RsvdP' for VFs, short-circuit out */
2063	if (dev->is_virtfn)
2064	return;
2065
2066	/*
2067	* For Root Complex Integrated Endpoints, program the maximum
2068	* supported value unless limited by the PCIE_BUS_PEER2PEER case.
2069	*/
2070	if (pci_pcie_type(dev) == PCI_EXP_TYPE_RC_END) {
2071	if (pcie_bus_config == PCIE_BUS_PEER2PEER)
2072	mps = 128;
2073	else
2074	mps = 128 << dev->pcie_mpss;
2075	rc = pcie_set_mps(dev, mps);
2076	if (rc) {
2077	pci_warn(dev, "can't set Max Payload Size to %d; if necessary, use \"pci=pcie_bus_safe\" and report a bug\n",
2078	mps);
2079	}
2080	return;
2081	}
2082
2083	if (!bridge || !pci_is_pcie(dev: bridge))
2084	return;
2085
2086	mps = pcie_get_mps(dev);
2087	p_mps = pcie_get_mps(dev: bridge);
2088
2089	if (mps == p_mps)
2090	return;
2091
2092	if (pcie_bus_config == PCIE_BUS_TUNE_OFF) {
2093	pci_warn(dev, "Max Payload Size %d, but upstream %s set to %d; if necessary, use \"pci=pcie_bus_safe\" and report a bug\n",
2094	mps, pci_name(bridge), p_mps);
2095	return;
2096	}
2097
2098	/*
2099	* Fancier MPS configuration is done later by
2100	* pcie_bus_configure_settings()
2101	*/
2102	if (pcie_bus_config != PCIE_BUS_DEFAULT)
2103	return;
2104
2105	mpss = 128 << dev->pcie_mpss;
2106	if (mpss < p_mps && pci_pcie_type(dev: bridge) == PCI_EXP_TYPE_ROOT_PORT) {
2107	pcie_set_mps(dev: bridge, mps: mpss);
2108	pci_info(dev, "Upstream bridge's Max Payload Size set to %d (was %d, max %d)\n",
2109	mpss, p_mps, 128 << bridge->pcie_mpss);
2110	p_mps = pcie_get_mps(dev: bridge);
2111	}
2112
2113	rc = pcie_set_mps(dev, mps: p_mps);
2114	if (rc) {
2115	pci_warn(dev, "can't set Max Payload Size to %d; if necessary, use \"pci=pcie_bus_safe\" and report a bug\n",
2116	p_mps);
2117	return;
2118	}
2119
2120	pci_info(dev, "Max Payload Size set to %d (was %d, max %d)\n",
2121	p_mps, mps, mpss);
2122	}
2123
2124	int pci_configure_extended_tags(struct pci_dev *dev, void *ign)
2125	{
2126	struct pci_host_bridge *host;
2127	u32 cap;
2128	u16 ctl;
2129	int ret;
2130
2131	if (!pci_is_pcie(dev))
2132	return 0;
2133
2134	ret = pcie_capability_read_dword(dev, PCI_EXP_DEVCAP, val: &cap);
2135	if (ret)
2136	return 0;
2137
2138	if (!(cap & PCI_EXP_DEVCAP_EXT_TAG))
2139	return 0;
2140
2141	ret = pcie_capability_read_word(dev, PCI_EXP_DEVCTL, val: &ctl);
2142	if (ret)
2143	return 0;
2144
2145	host = pci_find_host_bridge(bus: dev->bus);
2146	if (!host)
2147	return 0;
2148
2149	/*
2150	* If some device in the hierarchy doesn't handle Extended Tags
2151	* correctly, make sure they're disabled.
2152	*/
2153	if (host->no_ext_tags) {
2154	if (ctl & PCI_EXP_DEVCTL_EXT_TAG) {
2155	pci_info(dev, "disabling Extended Tags\n");
2156	pcie_capability_clear_word(dev, PCI_EXP_DEVCTL,
2157	PCI_EXP_DEVCTL_EXT_TAG);
2158	}
2159	return 0;
2160	}
2161
2162	if (!(ctl & PCI_EXP_DEVCTL_EXT_TAG)) {
2163	pci_info(dev, "enabling Extended Tags\n");
2164	pcie_capability_set_word(dev, PCI_EXP_DEVCTL,
2165	PCI_EXP_DEVCTL_EXT_TAG);
2166	}
2167	return 0;
2168	}
2169
2170	/**
2171	* pcie_relaxed_ordering_enabled - Probe for PCIe relaxed ordering enable
2172	* @dev: PCI device to query
2173	*
2174	* Returns true if the device has enabled relaxed ordering attribute.
2175	*/
2176	bool pcie_relaxed_ordering_enabled(struct pci_dev *dev)
2177	{
2178	u16 v;
2179
2180	pcie_capability_read_word(dev, PCI_EXP_DEVCTL, val: &v);
2181
2182	return !!(v & PCI_EXP_DEVCTL_RELAX_EN);
2183	}
2184	EXPORT_SYMBOL(pcie_relaxed_ordering_enabled);
2185
2186	static void pci_configure_relaxed_ordering(struct pci_dev *dev)
2187	{
2188	struct pci_dev *root;
2189
2190	/* PCI_EXP_DEVCTL_RELAX_EN is RsvdP in VFs */
2191	if (dev->is_virtfn)
2192	return;
2193
2194	if (!pcie_relaxed_ordering_enabled(dev))
2195	return;
2196
2197	/*
2198	* For now, we only deal with Relaxed Ordering issues with Root
2199	* Ports. Peer-to-Peer DMA is another can of worms.
2200	*/
2201	root = pcie_find_root_port(dev);
2202	if (!root)
2203	return;
2204
2205	if (root->dev_flags & PCI_DEV_FLAGS_NO_RELAXED_ORDERING) {
2206	pcie_capability_clear_word(dev, PCI_EXP_DEVCTL,
2207	PCI_EXP_DEVCTL_RELAX_EN);
2208	pci_info(dev, "Relaxed Ordering disabled because the Root Port didn't support it\n");
2209	}
2210	}
2211
2212	static void pci_configure_eetlp_prefix(struct pci_dev *dev)
2213	{
2214	#ifdef CONFIG_PCI_PASID
2215	struct pci_dev *bridge;
2216	int pcie_type;
2217	u32 cap;
2218
2219	if (!pci_is_pcie(dev))
2220	return;
2221
2222	pcie_capability_read_dword(dev, PCI_EXP_DEVCAP2, val: &cap);
2223	if (!(cap & PCI_EXP_DEVCAP2_EE_PREFIX))
2224	return;
2225
2226	pcie_type = pci_pcie_type(dev);
2227	if (pcie_type == PCI_EXP_TYPE_ROOT_PORT ||
2228	pcie_type == PCI_EXP_TYPE_RC_END)
2229	dev->eetlp_prefix_path = 1;
2230	else {
2231	bridge = pci_upstream_bridge(dev);
2232	if (bridge && bridge->eetlp_prefix_path)
2233	dev->eetlp_prefix_path = 1;
2234	}
2235	#endif
2236	}
2237
2238	static void pci_configure_serr(struct pci_dev *dev)
2239	{
2240	u16 control;
2241
2242	if (dev->hdr_type == PCI_HEADER_TYPE_BRIDGE) {
2243
2244	/*
2245	* A bridge will not forward ERR_ messages coming from an
2246	* endpoint unless SERR# forwarding is enabled.
2247	*/
2248	pci_read_config_word(dev, PCI_BRIDGE_CONTROL, val: &control);
2249	if (!(control & PCI_BRIDGE_CTL_SERR)) {
2250	control |= PCI_BRIDGE_CTL_SERR;
2251	pci_write_config_word(dev, PCI_BRIDGE_CONTROL, val: control);
2252	}
2253	}
2254	}
2255
2256	static void pci_configure_device(struct pci_dev *dev)
2257	{
2258	pci_configure_mps(dev);
2259	pci_configure_extended_tags(dev, NULL);
2260	pci_configure_relaxed_ordering(dev);
2261	pci_configure_ltr(pdev: dev);
2262	pci_configure_aspm_l1ss(dev);
2263	pci_configure_eetlp_prefix(dev);
2264	pci_configure_serr(dev);
2265
2266	pci_acpi_program_hp_params(dev);
2267	}
2268
2269	static void pci_release_capabilities(struct pci_dev *dev)
2270	{
2271	pci_aer_exit(dev);
2272	pci_rcec_exit(dev);
2273	pci_iov_release(dev);
2274	pci_free_cap_save_buffers(dev);
2275	}
2276
2277	/**
2278	* pci_release_dev - Free a PCI device structure when all users of it are
2279	* finished
2280	* @dev: device that's been disconnected
2281	*
2282	* Will be called only by the device core when all users of this PCI device are
2283	* done.
2284	*/
2285	static void pci_release_dev(struct device *dev)
2286	{
2287	struct pci_dev *pci_dev;
2288
2289	pci_dev = to_pci_dev(dev);
2290	pci_release_capabilities(dev: pci_dev);
2291	pci_release_of_node(dev: pci_dev);
2292	pcibios_release_device(dev: pci_dev);
2293	pci_bus_put(bus: pci_dev->bus);
2294	kfree(objp: pci_dev->driver_override);
2295	bitmap_free(bitmap: pci_dev->dma_alias_mask);
2296	dev_dbg(dev, "device released\n");
2297	kfree(objp: pci_dev);
2298	}
2299
2300	static const struct device_type pci_dev_type = {
2301	.groups = pci_dev_attr_groups,
2302	};
2303
2304	struct pci_dev *pci_alloc_dev(struct pci_bus *bus)
2305	{
2306	struct pci_dev *dev;
2307
2308	dev = kzalloc(size: sizeof(struct pci_dev), GFP_KERNEL);
2309	if (!dev)
2310	return NULL;
2311
2312	INIT_LIST_HEAD(list: &dev->bus_list);
2313	dev->dev.type = &pci_dev_type;
2314	dev->bus = pci_bus_get(bus);
2315	dev->driver_exclusive_resource = (struct resource) {
2316	.name = "PCI Exclusive",
2317	.start = 0,
2318	.end = -1,
2319	};
2320
2321	spin_lock_init(&dev->pcie_cap_lock);
2322	#ifdef CONFIG_PCI_MSI
2323	raw_spin_lock_init(&dev->msi_lock);
2324	#endif
2325	return dev;
2326	}
2327	EXPORT_SYMBOL(pci_alloc_dev);
2328
2329	static bool pci_bus_crs_vendor_id(u32 l)
2330	{
2331	return (l & 0xffff) == PCI_VENDOR_ID_PCI_SIG;
2332	}
2333
2334	static bool pci_bus_wait_crs(struct pci_bus *bus, int devfn, u32 *l,
2335	int timeout)
2336	{
2337	int delay = 1;
2338
2339	if (!pci_bus_crs_vendor_id(l: *l))
2340	return true; /* not a CRS completion */
2341
2342	if (!timeout)
2343	return false; /* CRS, but caller doesn't want to wait */
2344
2345	/*
2346	* We got the reserved Vendor ID that indicates a completion with
2347	* Configuration Request Retry Status (CRS). Retry until we get a
2348	* valid Vendor ID or we time out.
2349	*/
2350	while (pci_bus_crs_vendor_id(l: *l)) {
2351	if (delay > timeout) {
2352	pr_warn("pci %04x:%02x:%02x.%d: not ready after %dms; giving up\n",
2353	pci_domain_nr(bus), bus->number,
2354	PCI_SLOT(devfn), PCI_FUNC(devfn), delay - 1);
2355
2356	return false;
2357	}
2358	if (delay >= 1000)
2359	pr_info("pci %04x:%02x:%02x.%d: not ready after %dms; waiting\n",
2360	pci_domain_nr(bus), bus->number,
2361	PCI_SLOT(devfn), PCI_FUNC(devfn), delay - 1);
2362
2363	msleep(msecs: delay);
2364	delay *= 2;
2365
2366	if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, val: l))
2367	return false;
2368	}
2369
2370	if (delay >= 1000)
2371	pr_info("pci %04x:%02x:%02x.%d: ready after %dms\n",
2372	pci_domain_nr(bus), bus->number,
2373	PCI_SLOT(devfn), PCI_FUNC(devfn), delay - 1);
2374
2375	return true;
2376	}
2377
2378	bool pci_bus_generic_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
2379	int timeout)
2380	{
2381	if (pci_bus_read_config_dword(bus, devfn, PCI_VENDOR_ID, val: l))
2382	return false;
2383
2384	/* Some broken boards return 0 or ~0 (PCI_ERROR_RESPONSE) if a slot is empty: */
2385	if (PCI_POSSIBLE_ERROR(*l) || *l == 0x00000000 ||
2386	*l == 0x0000ffff || *l == 0xffff0000)
2387	return false;
2388
2389	if (pci_bus_crs_vendor_id(l: *l))
2390	return pci_bus_wait_crs(bus, devfn, l, timeout);
2391
2392	return true;
2393	}
2394
2395	bool pci_bus_read_dev_vendor_id(struct pci_bus *bus, int devfn, u32 *l,
2396	int timeout)
2397	{
2398	#ifdef CONFIG_PCI_QUIRKS
2399	struct pci_dev *bridge = bus->self;
2400
2401	/*
2402	* Certain IDT switches have an issue where they improperly trigger
2403	* ACS Source Validation errors on completions for config reads.
2404	*/
2405	if (bridge && bridge->vendor == PCI_VENDOR_ID_IDT &&
2406	bridge->device == 0x80b5)
2407	return pci_idt_bus_quirk(bus, devfn, pl: l, crs_timeout: timeout);
2408	#endif
2409
2410	return pci_bus_generic_read_dev_vendor_id(bus, devfn, l, timeout);
2411	}
2412	EXPORT_SYMBOL(pci_bus_read_dev_vendor_id);
2413
2414	/*
2415	* Read the config data for a PCI device, sanity-check it,
2416	* and fill in the dev structure.
2417	*/
2418	static struct pci_dev *pci_scan_device(struct pci_bus *bus, int devfn)
2419	{
2420	struct pci_dev *dev;
2421	u32 l;
2422
2423	if (!pci_bus_read_dev_vendor_id(bus, devfn, &l, 60*1000))
2424	return NULL;
2425
2426	dev = pci_alloc_dev(bus);
2427	if (!dev)
2428	return NULL;
2429
2430	dev->devfn = devfn;
2431	dev->vendor = l & 0xffff;
2432	dev->device = (l >> 16) & 0xffff;
2433
2434	if (pci_setup_device(dev)) {
2435	pci_bus_put(bus: dev->bus);
2436	kfree(objp: dev);
2437	return NULL;
2438	}
2439
2440	return dev;
2441	}
2442
2443	void pcie_report_downtraining(struct pci_dev *dev)
2444	{
2445	if (!pci_is_pcie(dev))
2446	return;
2447
2448	/* Look from the device up to avoid downstream ports with no devices */
2449	if ((pci_pcie_type(dev) != PCI_EXP_TYPE_ENDPOINT) &&
2450	(pci_pcie_type(dev) != PCI_EXP_TYPE_LEG_END) &&
2451	(pci_pcie_type(dev) != PCI_EXP_TYPE_UPSTREAM))
2452	return;
2453
2454	/* Multi-function PCIe devices share the same link/status */
2455	if (PCI_FUNC(dev->devfn) != 0 || dev->is_virtfn)
2456	return;
2457
2458	/* Print link status only if the device is constrained by the fabric */
2459	__pcie_print_link_status(dev, verbose: false);
2460	}
2461
2462	static void pci_init_capabilities(struct pci_dev *dev)
2463	{
2464	pci_ea_init(dev); /* Enhanced Allocation */
2465	pci_msi_init(dev); /* Disable MSI */
2466	pci_msix_init(dev); /* Disable MSI-X */
2467
2468	/* Buffers for saving PCIe and PCI-X capabilities */
2469	pci_allocate_cap_save_buffers(dev);
2470
2471	pci_pm_init(dev); /* Power Management */
2472	pci_vpd_init(dev); /* Vital Product Data */
2473	pci_configure_ari(dev); /* Alternative Routing-ID Forwarding */
2474	pci_iov_init(dev); /* Single Root I/O Virtualization */
2475	pci_ats_init(dev); /* Address Translation Services */
2476	pci_pri_init(dev); /* Page Request Interface */
2477	pci_pasid_init(dev); /* Process Address Space ID */
2478	pci_acs_init(dev); /* Access Control Services */
2479	pci_ptm_init(dev); /* Precision Time Measurement */
2480	pci_aer_init(dev); /* Advanced Error Reporting */
2481	pci_dpc_init(pdev: dev); /* Downstream Port Containment */
2482	pci_rcec_init(dev); /* Root Complex Event Collector */
2483	pci_doe_init(pdev: dev); /* Data Object Exchange */
2484
2485	pcie_report_downtraining(dev);
2486	pci_init_reset_methods(dev);
2487	}
2488
2489	/*
2490	* This is the equivalent of pci_host_bridge_msi_domain() that acts on
2491	* devices. Firmware interfaces that can select the MSI domain on a
2492	* per-device basis should be called from here.
2493	*/
2494	static struct irq_domain *pci_dev_msi_domain(struct pci_dev *dev)
2495	{
2496	struct irq_domain *d;
2497
2498	/*
2499	* If a domain has been set through the pcibios_device_add()
2500	* callback, then this is the one (platform code knows best).
2501	*/
2502	d = dev_get_msi_domain(dev: &dev->dev);
2503	if (d)
2504	return d;
2505
2506	/*
2507	* Let's see if we have a firmware interface able to provide
2508	* the domain.
2509	*/
2510	d = pci_msi_get_device_domain(pdev: dev);
2511	if (d)
2512	return d;
2513
2514	return NULL;
2515	}
2516
2517	static void pci_set_msi_domain(struct pci_dev *dev)
2518	{
2519	struct irq_domain *d;
2520
2521	/*
2522	* If the platform or firmware interfaces cannot supply a
2523	* device-specific MSI domain, then inherit the default domain
2524	* from the host bridge itself.
2525	*/
2526	d = pci_dev_msi_domain(dev);
2527	if (!d)
2528	d = dev_get_msi_domain(dev: &dev->bus->dev);
2529
2530	dev_set_msi_domain(dev: &dev->dev, d);
2531	}
2532
2533	void pci_device_add(struct pci_dev *dev, struct pci_bus *bus)
2534	{
2535	int ret;
2536
2537	pci_configure_device(dev);
2538
2539	device_initialize(dev: &dev->dev);
2540	dev->dev.release = pci_release_dev;
2541
2542	set_dev_node(dev: &dev->dev, pcibus_to_node(bus));
2543	dev->dev.dma_mask = &dev->dma_mask;
2544	dev->dev.dma_parms = &dev->dma_parms;
2545	dev->dev.coherent_dma_mask = 0xffffffffull;
2546
2547	dma_set_max_seg_size(dev: &dev->dev, size: 65536);
2548	dma_set_seg_boundary(dev: &dev->dev, mask: 0xffffffff);
2549
2550	pcie_failed_link_retrain(dev);
2551
2552	/* Fix up broken headers */
2553	pci_fixup_device(pass: pci_fixup_header, dev);
2554
2555	pci_reassigndev_resource_alignment(dev);
2556
2557	dev->state_saved = false;
2558
2559	pci_init_capabilities(dev);
2560
2561	/*
2562	* Add the device to our list of discovered devices
2563	* and the bus list for fixup functions, etc.
2564	*/
2565	down_write(sem: &pci_bus_sem);
2566	list_add_tail(new: &dev->bus_list, head: &bus->devices);
2567	up_write(sem: &pci_bus_sem);
2568
2569	ret = pcibios_device_add(dev);
2570	WARN_ON(ret < 0);
2571
2572	/* Set up MSI IRQ domain */
2573	pci_set_msi_domain(dev);
2574
2575	/* Notifier could use PCI capabilities */
2576	dev->match_driver = false;
2577	ret = device_add(dev: &dev->dev);
2578	WARN_ON(ret < 0);
2579	}
2580
2581	struct pci_dev *pci_scan_single_device(struct pci_bus *bus, int devfn)
2582	{
2583	struct pci_dev *dev;
2584
2585	dev = pci_get_slot(bus, devfn);
2586	if (dev) {
2587	pci_dev_put(dev);
2588	return dev;
2589	}
2590
2591	dev = pci_scan_device(bus, devfn);
2592	if (!dev)
2593	return NULL;
2594
2595	pci_device_add(dev, bus);
2596
2597	return dev;
2598	}
2599	EXPORT_SYMBOL(pci_scan_single_device);
2600
2601	static int next_ari_fn(struct pci_bus *bus, struct pci_dev *dev, int fn)
2602	{
2603	int pos;
2604	u16 cap = 0;
2605	unsigned int next_fn;
2606
2607	if (!dev)
2608	return -ENODEV;
2609
2610	pos = pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ARI);
2611	if (!pos)
2612	return -ENODEV;
2613
2614	pci_read_config_word(dev, where: pos + PCI_ARI_CAP, val: &cap);
2615	next_fn = PCI_ARI_CAP_NFN(cap);
2616	if (next_fn <= fn)
2617	return -ENODEV; /* protect against malformed list */
2618
2619	return next_fn;
2620	}
2621
2622	static int next_fn(struct pci_bus *bus, struct pci_dev *dev, int fn)
2623	{
2624	if (pci_ari_enabled(bus))
2625	return next_ari_fn(bus, dev, fn);
2626
2627	if (fn >= 7)
2628	return -ENODEV;
2629	/* only multifunction devices may have more functions */
2630	if (dev && !dev->multifunction)
2631	return -ENODEV;
2632
2633	return fn + 1;
2634	}
2635
2636	static int only_one_child(struct pci_bus *bus)
2637	{
2638	struct pci_dev *bridge = bus->self;
2639
2640	/*
2641	* Systems with unusual topologies set PCI_SCAN_ALL_PCIE_DEVS so
2642	* we scan for all possible devices, not just Device 0.
2643	*/
2644	if (pci_has_flag(flag: PCI_SCAN_ALL_PCIE_DEVS))
2645	return 0;
2646
2647	/*
2648	* A PCIe Downstream Port normally leads to a Link with only Device
2649	* 0 on it (PCIe spec r3.1, sec 7.3.1). As an optimization, scan
2650	* only for Device 0 in that situation.
2651	*/
2652	if (bridge && pci_is_pcie(dev: bridge) && pcie_downstream_port(dev: bridge))
2653	return 1;
2654
2655	return 0;
2656	}
2657
2658	/**
2659	* pci_scan_slot - Scan a PCI slot on a bus for devices
2660	* @bus: PCI bus to scan
2661	* @devfn: slot number to scan (must have zero function)
2662	*
2663	* Scan a PCI slot on the specified PCI bus for devices, adding
2664	* discovered devices to the @bus->devices list. New devices
2665	* will not have is_added set.
2666	*
2667	* Returns the number of new devices found.
2668	*/
2669	int pci_scan_slot(struct pci_bus *bus, int devfn)
2670	{
2671	struct pci_dev *dev;
2672	int fn = 0, nr = 0;
2673
2674	if (only_one_child(bus) && (devfn > 0))
2675	return 0; /* Already scanned the entire slot */
2676
2677	do {
2678	dev = pci_scan_single_device(bus, devfn + fn);
2679	if (dev) {
2680	if (!pci_dev_is_added(dev))
2681	nr++;
2682	if (fn > 0)
2683	dev->multifunction = 1;
2684	} else if (fn == 0) {
2685	/*
2686	* Function 0 is required unless we are running on
2687	* a hypervisor that passes through individual PCI
2688	* functions.
2689	*/
2690	if (!hypervisor_isolated_pci_functions())
2691	break;
2692	}
2693	fn = next_fn(bus, dev, fn);
2694	} while (fn >= 0);
2695
2696	/* Only one slot has PCIe device */
2697	if (bus->self && nr)
2698	pcie_aspm_init_link_state(pdev: bus->self);
2699
2700	return nr;
2701	}
2702	EXPORT_SYMBOL(pci_scan_slot);
2703
2704	static int pcie_find_smpss(struct pci_dev *dev, void *data)
2705	{
2706	u8 *smpss = data;
2707
2708	if (!pci_is_pcie(dev))
2709	return 0;
2710
2711	/*
2712	* We don't have a way to change MPS settings on devices that have
2713	* drivers attached. A hot-added device might support only the minimum
2714	* MPS setting (MPS=128). Therefore, if the fabric contains a bridge
2715	* where devices may be hot-added, we limit the fabric MPS to 128 so
2716	* hot-added devices will work correctly.
2717	*
2718	* However, if we hot-add a device to a slot directly below a Root
2719	* Port, it's impossible for there to be other existing devices below
2720	* the port. We don't limit the MPS in this case because we can
2721	* reconfigure MPS on both the Root Port and the hot-added device,
2722	* and there are no other devices involved.
2723	*
2724	* Note that this PCIE_BUS_SAFE path assumes no peer-to-peer DMA.
2725	*/
2726	if (dev->is_hotplug_bridge &&
2727	pci_pcie_type(dev) != PCI_EXP_TYPE_ROOT_PORT)
2728	*smpss = 0;
2729
2730	if (*smpss > dev->pcie_mpss)
2731	*smpss = dev->pcie_mpss;
2732
2733	return 0;
2734	}
2735
2736	static void pcie_write_mps(struct pci_dev *dev, int mps)
2737	{
2738	int rc;
2739
2740	if (pcie_bus_config == PCIE_BUS_PERFORMANCE) {
2741	mps = 128 << dev->pcie_mpss;
2742
2743	if (pci_pcie_type(dev) != PCI_EXP_TYPE_ROOT_PORT &&
2744	dev->bus->self)
2745
2746	/*
2747	* For "Performance", the assumption is made that
2748	* downstream communication will never be larger than
2749	* the MRRS. So, the MPS only needs to be configured
2750	* for the upstream communication. This being the case,
2751	* walk from the top down and set the MPS of the child
2752	* to that of the parent bus.
2753	*
2754	* Configure the device MPS with the smaller of the
2755	* device MPSS or the bridge MPS (which is assumed to be
2756	* properly configured at this point to the largest
2757	* allowable MPS based on its parent bus).
2758	*/
2759	mps = min(mps, pcie_get_mps(dev->bus->self));
2760	}
2761
2762	rc = pcie_set_mps(dev, mps);
2763	if (rc)
2764	pci_err(dev, "Failed attempting to set the MPS\n");
2765	}
2766
2767	static void pcie_write_mrrs(struct pci_dev *dev)
2768	{
2769	int rc, mrrs;
2770
2771	/*
2772	* In the "safe" case, do not configure the MRRS. There appear to be
2773	* issues with setting MRRS to 0 on a number of devices.
2774	*/
2775	if (pcie_bus_config != PCIE_BUS_PERFORMANCE)
2776	return;
2777
2778	/*
2779	* For max performance, the MRRS must be set to the largest supported
2780	* value. However, it cannot be configured larger than the MPS the
2781	* device or the bus can support. This should already be properly
2782	* configured by a prior call to pcie_write_mps().
2783	*/
2784	mrrs = pcie_get_mps(dev);
2785
2786	/*
2787	* MRRS is a R/W register. Invalid values can be written, but a
2788	* subsequent read will verify if the value is acceptable or not.
2789	* If the MRRS value provided is not acceptable (e.g., too large),
2790	* shrink the value until it is acceptable to the HW.
2791	*/
2792	while (mrrs != pcie_get_readrq(dev) && mrrs >= 128) {
2793	rc = pcie_set_readrq(dev, rq: mrrs);
2794	if (!rc)
2795	break;
2796
2797	pci_warn(dev, "Failed attempting to set the MRRS\n");
2798	mrrs /= 2;
2799	}
2800
2801	if (mrrs < 128)
2802	pci_err(dev, "MRRS was unable to be configured with a safe value. If problems are experienced, try running with pci=pcie_bus_safe\n");
2803	}
2804
2805	static int pcie_bus_configure_set(struct pci_dev *dev, void *data)
2806	{
2807	int mps, orig_mps;
2808
2809	if (!pci_is_pcie(dev))
2810	return 0;
2811
2812	if (pcie_bus_config == PCIE_BUS_TUNE_OFF ||
2813	pcie_bus_config == PCIE_BUS_DEFAULT)
2814	return 0;
2815
2816	mps = 128 << *(u8 *)data;
2817	orig_mps = pcie_get_mps(dev);
2818
2819	pcie_write_mps(dev, mps);
2820	pcie_write_mrrs(dev);
2821
2822	pci_info(dev, "Max Payload Size set to %4d/%4d (was %4d), Max Read Rq %4d\n",
2823	pcie_get_mps(dev), 128 << dev->pcie_mpss,
2824	orig_mps, pcie_get_readrq(dev));
2825
2826	return 0;
2827	}
2828
2829	/*
2830	* pcie_bus_configure_settings() requires that pci_walk_bus work in a top-down,
2831	* parents then children fashion. If this changes, then this code will not
2832	* work as designed.
2833	*/
2834	void pcie_bus_configure_settings(struct pci_bus *bus)
2835	{
2836	u8 smpss = 0;
2837
2838	if (!bus->self)
2839	return;
2840
2841	if (!pci_is_pcie(dev: bus->self))
2842	return;
2843
2844	/*
2845	* FIXME - Peer to peer DMA is possible, though the endpoint would need
2846	* to be aware of the MPS of the destination. To work around this,
2847	* simply force the MPS of the entire system to the smallest possible.
2848	*/
2849	if (pcie_bus_config == PCIE_BUS_PEER2PEER)
2850	smpss = 0;
2851
2852	if (pcie_bus_config == PCIE_BUS_SAFE) {
2853	smpss = bus->self->pcie_mpss;
2854
2855	pcie_find_smpss(dev: bus->self, data: &smpss);
2856	pci_walk_bus(top: bus, cb: pcie_find_smpss, userdata: &smpss);
2857	}
2858
2859	pcie_bus_configure_set(dev: bus->self, data: &smpss);
2860	pci_walk_bus(top: bus, cb: pcie_bus_configure_set, userdata: &smpss);
2861	}
2862	EXPORT_SYMBOL_GPL(pcie_bus_configure_settings);
2863
2864	/*
2865	* Called after each bus is probed, but before its children are examined. This
2866	* is marked as __weak because multiple architectures define it.
2867	*/
2868	void __weak pcibios_fixup_bus(struct pci_bus *bus)
2869	{
2870	/* nothing to do, expected to be removed in the future */
2871	}
2872
2873	/**
2874	* pci_scan_child_bus_extend() - Scan devices below a bus
2875	* @bus: Bus to scan for devices
2876	* @available_buses: Total number of buses available (%0 does not try to
2877	* extend beyond the minimal)
2878	*
2879	* Scans devices below @bus including subordinate buses. Returns new
2880	* subordinate number including all the found devices. Passing
2881	* @available_buses causes the remaining bus space to be distributed
2882	* equally between hotplug-capable bridges to allow future extension of the
2883	* hierarchy.
2884	*/
2885	static unsigned int pci_scan_child_bus_extend(struct pci_bus *bus,
2886	unsigned int available_buses)
2887	{
2888	unsigned int used_buses, normal_bridges = 0, hotplug_bridges = 0;
2889	unsigned int start = bus->busn_res.start;
2890	unsigned int devfn, cmax, max = start;
2891	struct pci_dev *dev;
2892
2893	dev_dbg(&bus->dev, "scanning bus\n");
2894
2895	/* Go find them, Rover! */
2896	for (devfn = 0; devfn < 256; devfn += 8)
2897	pci_scan_slot(bus, devfn);
2898
2899	/* Reserve buses for SR-IOV capability */
2900	used_buses = pci_iov_bus_range(bus);
2901	max += used_buses;
2902
2903	/*
2904	* After performing arch-dependent fixup of the bus, look behind
2905	* all PCI-to-PCI bridges on this bus.
2906	*/
2907	if (!bus->is_added) {
2908	dev_dbg(&bus->dev, "fixups for bus\n");
2909	pcibios_fixup_bus(bus);
2910	bus->is_added = 1;
2911	}
2912
2913	/*
2914	* Calculate how many hotplug bridges and normal bridges there
2915	* are on this bus. We will distribute the additional available
2916	* buses between hotplug bridges.
2917	*/
2918	for_each_pci_bridge(dev, bus) {
2919	if (dev->is_hotplug_bridge)
2920	hotplug_bridges++;
2921	else
2922	normal_bridges++;
2923	}
2924
2925	/*
2926	* Scan bridges that are already configured. We don't touch them
2927	* unless they are misconfigured (which will be done in the second
2928	* scan below).
2929	*/
2930	for_each_pci_bridge(dev, bus) {
2931	cmax = max;
2932	max = pci_scan_bridge_extend(bus, dev, max, available_buses: 0, pass: 0);
2933
2934	/*
2935	* Reserve one bus for each bridge now to avoid extending
2936	* hotplug bridges too much during the second scan below.
2937	*/
2938	used_buses++;
2939	if (max - cmax > 1)
2940	used_buses += max - cmax - 1;
2941	}
2942
2943	/* Scan bridges that need to be reconfigured */
2944	for_each_pci_bridge(dev, bus) {
2945	unsigned int buses = 0;
2946
2947	if (!hotplug_bridges && normal_bridges == 1) {
2948	/*
2949	* There is only one bridge on the bus (upstream
2950	* port) so it gets all available buses which it
2951	* can then distribute to the possible hotplug
2952	* bridges below.
2953	*/
2954	buses = available_buses;
2955	} else if (dev->is_hotplug_bridge) {
2956	/*
2957	* Distribute the extra buses between hotplug
2958	* bridges if any.
2959	*/
2960	buses = available_buses / hotplug_bridges;
2961	buses = min(buses, available_buses - used_buses + 1);
2962	}
2963
2964	cmax = max;
2965	max = pci_scan_bridge_extend(bus, dev, max: cmax, available_buses: buses, pass: 1);
2966	/* One bus is already accounted so don't add it again */
2967	if (max - cmax > 1)
2968	used_buses += max - cmax - 1;
2969	}
2970
2971	/*
2972	* Make sure a hotplug bridge has at least the minimum requested
2973	* number of buses but allow it to grow up to the maximum available
2974	* bus number if there is room.
2975	*/
2976	if (bus->self && bus->self->is_hotplug_bridge) {
2977	used_buses = max_t(unsigned int, available_buses,
2978	pci_hotplug_bus_size - 1);
2979	if (max - start < used_buses) {
2980	max = start + used_buses;
2981
2982	/* Do not allocate more buses than we have room left */
2983	if (max > bus->busn_res.end)
2984	max = bus->busn_res.end;
2985
2986	dev_dbg(&bus->dev, "%pR extended by %#02x\n",
2987	&bus->busn_res, max - start);
2988	}
2989	}
2990
2991	/*
2992	* We've scanned the bus and so we know all about what's on
2993	* the other side of any bridges that may be on this bus plus
2994	* any devices.
2995	*
2996	* Return how far we've got finding sub-buses.
2997	*/
2998	dev_dbg(&bus->dev, "bus scan returning with max=%02x\n", max);
2999	return max;
3000	}
3001
3002	/**
3003	* pci_scan_child_bus() - Scan devices below a bus
3004	* @bus: Bus to scan for devices
3005	*
3006	* Scans devices below @bus including subordinate buses. Returns new
3007	* subordinate number including all the found devices.
3008	*/
3009	unsigned int pci_scan_child_bus(struct pci_bus *bus)
3010	{
3011	return pci_scan_child_bus_extend(bus, available_buses: 0);
3012	}
3013	EXPORT_SYMBOL_GPL(pci_scan_child_bus);
3014
3015	/**
3016	* pcibios_root_bridge_prepare - Platform-specific host bridge setup
3017	* @bridge: Host bridge to set up
3018	*
3019	* Default empty implementation. Replace with an architecture-specific setup
3020	* routine, if necessary.
3021	*/
3022	int __weak pcibios_root_bridge_prepare(struct pci_host_bridge *bridge)
3023	{
3024	return 0;
3025	}
3026
3027	void __weak pcibios_add_bus(struct pci_bus *bus)
3028	{
3029	}
3030
3031	void __weak pcibios_remove_bus(struct pci_bus *bus)
3032	{
3033	}
3034
3035	struct pci_bus *pci_create_root_bus(struct device *parent, int bus,
3036	struct pci_ops *ops, void *sysdata, struct list_head *resources)
3037	{
3038	int error;
3039	struct pci_host_bridge *bridge;
3040
3041	bridge = pci_alloc_host_bridge(0);
3042	if (!bridge)
3043	return NULL;
3044
3045	bridge->dev.parent = parent;
3046
3047	list_splice_init(list: resources, head: &bridge->windows);
3048	bridge->sysdata = sysdata;
3049	bridge->busnr = bus;
3050	bridge->ops = ops;
3051
3052	error = pci_register_host_bridge(bridge);
3053	if (error < 0)
3054	goto err_out;
3055
3056	return bridge->bus;
3057
3058	err_out:
3059	put_device(dev: &bridge->dev);
3060	return NULL;
3061	}
3062	EXPORT_SYMBOL_GPL(pci_create_root_bus);
3063
3064	int pci_host_probe(struct pci_host_bridge *bridge)
3065	{
3066	struct pci_bus *bus, *child;
3067	int ret;
3068
3069	ret = pci_scan_root_bus_bridge(bridge);
3070	if (ret < 0) {
3071	dev_err(bridge->dev.parent, "Scanning root bridge failed");
3072	return ret;
3073	}
3074
3075	bus = bridge->bus;
3076
3077	/*
3078	* We insert PCI resources into the iomem_resource and
3079	* ioport_resource trees in either pci_bus_claim_resources()
3080	* or pci_bus_assign_resources().
3081	*/
3082	if (pci_has_flag(flag: PCI_PROBE_ONLY)) {
3083	pci_bus_claim_resources(bus);
3084	} else {
3085	pci_bus_size_bridges(bus);
3086	pci_bus_assign_resources(bus);
3087
3088	list_for_each_entry(child, &bus->children, node)
3089	pcie_bus_configure_settings(child);
3090	}
3091
3092	pci_bus_add_devices(bus);
3093	return 0;
3094	}
3095	EXPORT_SYMBOL_GPL(pci_host_probe);
3096
3097	int pci_bus_insert_busn_res(struct pci_bus *b, int bus, int bus_max)
3098	{
3099	struct resource *res = &b->busn_res;
3100	struct resource *parent_res, *conflict;
3101
3102	res->start = bus;
3103	res->end = bus_max;
3104	res->flags = IORESOURCE_BUS;
3105
3106	if (!pci_is_root_bus(pbus: b))
3107	parent_res = &b->parent->busn_res;
3108	else {
3109	parent_res = get_pci_domain_busn_res(domain_nr: pci_domain_nr(bus: b));
3110	res->flags |= IORESOURCE_PCI_FIXED;
3111	}
3112
3113	conflict = request_resource_conflict(root: parent_res, new: res);
3114
3115	if (conflict)
3116	dev_info(&b->dev,
3117	"busn_res: can not insert %pR under %s%pR (conflicts with %s %pR)\n",
3118	res, pci_is_root_bus(b) ? "domain " : "",
3119	parent_res, conflict->name, conflict);
3120
3121	return conflict == NULL;
3122	}
3123
3124	int pci_bus_update_busn_res_end(struct pci_bus *b, int bus_max)
3125	{
3126	struct resource *res = &b->busn_res;
3127	struct resource old_res = *res;
3128	resource_size_t size;
3129	int ret;
3130
3131	if (res->start > bus_max)
3132	return -EINVAL;
3133
3134	size = bus_max - res->start + 1;
3135	ret = adjust_resource(res, start: res->start, size);
3136	dev_info(&b->dev, "busn_res: %pR end %s updated to %02x\n",
3137	&old_res, ret ? "can not be" : "is", bus_max);
3138
3139	if (!ret && !res->parent)
3140	pci_bus_insert_busn_res(b, bus: res->start, bus_max: res->end);
3141
3142	return ret;
3143	}
3144
3145	void pci_bus_release_busn_res(struct pci_bus *b)
3146	{
3147	struct resource *res = &b->busn_res;
3148	int ret;
3149
3150	if (!res->flags || !res->parent)
3151	return;
3152
3153	ret = release_resource(new: res);
3154	dev_info(&b->dev, "busn_res: %pR %s released\n",
3155	res, ret ? "can not be" : "is");
3156	}
3157
3158	int pci_scan_root_bus_bridge(struct pci_host_bridge *bridge)
3159	{
3160	struct resource_entry *window;
3161	bool found = false;
3162	struct pci_bus *b;
3163	int max, bus, ret;
3164
3165	if (!bridge)
3166	return -EINVAL;
3167
3168	resource_list_for_each_entry(window, &bridge->windows)
3169	if (window->res->flags & IORESOURCE_BUS) {
3170	bridge->busnr = window->res->start;
3171	found = true;
3172	break;
3173	}
3174
3175	ret = pci_register_host_bridge(bridge);
3176	if (ret < 0)
3177	return ret;
3178
3179	b = bridge->bus;
3180	bus = bridge->busnr;
3181
3182	if (!found) {
3183	dev_info(&b->dev,
3184	"No busn resource found for root bus, will use [bus %02x-ff]\n",
3185	bus);
3186	pci_bus_insert_busn_res(b, bus, bus_max: 255);
3187	}
3188
3189	max = pci_scan_child_bus(b);
3190
3191	if (!found)
3192	pci_bus_update_busn_res_end(b, bus_max: max);
3193
3194	return 0;
3195	}
3196	EXPORT_SYMBOL(pci_scan_root_bus_bridge);
3197
3198	struct pci_bus *pci_scan_root_bus(struct device *parent, int bus,
3199	struct pci_ops *ops, void *sysdata, struct list_head *resources)
3200	{
3201	struct resource_entry *window;
3202	bool found = false;
3203	struct pci_bus *b;
3204	int max;
3205
3206	resource_list_for_each_entry(window, resources)
3207	if (window->res->flags & IORESOURCE_BUS) {
3208	found = true;
3209	break;
3210	}
3211
3212	b = pci_create_root_bus(parent, bus, ops, sysdata, resources);
3213	if (!b)
3214	return NULL;
3215
3216	if (!found) {
3217	dev_info(&b->dev,
3218	"No busn resource found for root bus, will use [bus %02x-ff]\n",
3219	bus);
3220	pci_bus_insert_busn_res(b, bus, bus_max: 255);
3221	}
3222
3223	max = pci_scan_child_bus(b);
3224
3225	if (!found)
3226	pci_bus_update_busn_res_end(b, bus_max: max);
3227
3228	return b;
3229	}
3230	EXPORT_SYMBOL(pci_scan_root_bus);
3231
3232	struct pci_bus *pci_scan_bus(int bus, struct pci_ops *ops,
3233	void *sysdata)
3234	{
3235	LIST_HEAD(resources);
3236	struct pci_bus *b;
3237
3238	pci_add_resource(resources: &resources, res: &ioport_resource);
3239	pci_add_resource(resources: &resources, res: &iomem_resource);
3240	pci_add_resource(resources: &resources, res: &busn_resource);
3241	b = pci_create_root_bus(NULL, bus, ops, sysdata, &resources);
3242	if (b) {
3243	pci_scan_child_bus(b);
3244	} else {
3245	pci_free_resource_list(resources: &resources);
3246	}
3247	return b;
3248	}
3249	EXPORT_SYMBOL(pci_scan_bus);
3250
3251	/**
3252	* pci_rescan_bus_bridge_resize - Scan a PCI bus for devices
3253	* @bridge: PCI bridge for the bus to scan
3254	*
3255	* Scan a PCI bus and child buses for new devices, add them,
3256	* and enable them, resizing bridge mmio/io resource if necessary
3257	* and possible. The caller must ensure the child devices are already
3258	* removed for resizing to occur.
3259	*
3260	* Returns the max number of subordinate bus discovered.
3261	*/
3262	unsigned int pci_rescan_bus_bridge_resize(struct pci_dev *bridge)
3263	{
3264	unsigned int max;
3265	struct pci_bus *bus = bridge->subordinate;
3266
3267	max = pci_scan_child_bus(bus);
3268
3269	pci_assign_unassigned_bridge_resources(bridge);
3270
3271	pci_bus_add_devices(bus);
3272
3273	return max;
3274	}
3275
3276	/**
3277	* pci_rescan_bus - Scan a PCI bus for devices
3278	* @bus: PCI bus to scan
3279	*
3280	* Scan a PCI bus and child buses for new devices, add them,
3281	* and enable them.
3282	*
3283	* Returns the max number of subordinate bus discovered.
3284	*/
3285	unsigned int pci_rescan_bus(struct pci_bus *bus)
3286	{
3287	unsigned int max;
3288
3289	max = pci_scan_child_bus(bus);
3290	pci_assign_unassigned_bus_resources(bus);
3291	pci_bus_add_devices(bus);
3292
3293	return max;
3294	}
3295	EXPORT_SYMBOL_GPL(pci_rescan_bus);
3296
3297	/*
3298	* pci_rescan_bus(), pci_rescan_bus_bridge_resize() and PCI device removal
3299	* routines should always be executed under this mutex.
3300	*/
3301	static DEFINE_MUTEX(pci_rescan_remove_lock);
3302
3303	void pci_lock_rescan_remove(void)
3304	{
3305	mutex_lock(&pci_rescan_remove_lock);
3306	}
3307	EXPORT_SYMBOL_GPL(pci_lock_rescan_remove);
3308
3309	void pci_unlock_rescan_remove(void)
3310	{
3311	mutex_unlock(lock: &pci_rescan_remove_lock);
3312	}
3313	EXPORT_SYMBOL_GPL(pci_unlock_rescan_remove);
3314
3315	static int __init pci_sort_bf_cmp(const struct device *d_a,
3316	const struct device *d_b)
3317	{
3318	const struct pci_dev *a = to_pci_dev(d_a);
3319	const struct pci_dev *b = to_pci_dev(d_b);
3320
3321	if (pci_domain_nr(bus: a->bus) < pci_domain_nr(bus: b->bus)) return -1;
3322	else if (pci_domain_nr(bus: a->bus) > pci_domain_nr(bus: b->bus)) return 1;
3323
3324	if (a->bus->number < b->bus->number) return -1;
3325	else if (a->bus->number > b->bus->number) return 1;
3326
3327	if (a->devfn < b->devfn) return -1;
3328	else if (a->devfn > b->devfn) return 1;
3329
3330	return 0;
3331	}
3332
3333	void __init pci_sort_breadthfirst(void)
3334	{
3335	bus_sort_breadthfirst(bus: &pci_bus_type, compare: &pci_sort_bf_cmp);
3336	}
3337
3338	int pci_hp_add_bridge(struct pci_dev *dev)
3339	{
3340	struct pci_bus *parent = dev->bus;
3341	int busnr, start = parent->busn_res.start;
3342	unsigned int available_buses = 0;
3343	int end = parent->busn_res.end;
3344
3345	for (busnr = start; busnr <= end; busnr++) {
3346	if (!pci_find_bus(domain: pci_domain_nr(bus: parent), busnr))
3347	break;
3348	}
3349	if (busnr-- > end) {
3350	pci_err(dev, "No bus number available for hot-added bridge\n");
3351	return -1;
3352	}
3353
3354	/* Scan bridges that are already configured */
3355	busnr = pci_scan_bridge(parent, dev, busnr, 0);
3356
3357	/*
3358	* Distribute the available bus numbers between hotplug-capable
3359	* bridges to make extending the chain later possible.
3360	*/
3361	available_buses = end - busnr;
3362
3363	/* Scan bridges that need to be reconfigured */
3364	pci_scan_bridge_extend(bus: parent, dev, max: busnr, available_buses, pass: 1);
3365
3366	if (!dev->subordinate)
3367	return -1;
3368
3369	return 0;
3370	}
3371	EXPORT_SYMBOL_GPL(pci_hp_add_bridge);
3372