1	// SPDX-License-Identifier: GPL-2.0-only OR MIT
2	/*
3	* Bluetooth HCI driver for Broadcom 4377/4378/4387 devices attached via PCIe
4	*
5	* Copyright (C) The Asahi Linux Contributors
6	*/
7
8	#include <linux/async.h>
9	#include <linux/bitfield.h>
10	#include <linux/completion.h>
11	#include <linux/dma-mapping.h>
12	#include <linux/dmi.h>
13	#include <linux/firmware.h>
14	#include <linux/module.h>
15	#include <linux/msi.h>
16	#include <linux/of.h>
17	#include <linux/pci.h>
18	#include <linux/printk.h>
19
20	#include <asm/unaligned.h>
21
22	#include <net/bluetooth/bluetooth.h>
23	#include <net/bluetooth/hci_core.h>
24
25	enum bcm4377_chip {
26	BCM4377 = 0,
27	BCM4378,
28	BCM4387,
29	};
30
31	#define BCM4377_DEVICE_ID 0x5fa0
32	#define BCM4378_DEVICE_ID 0x5f69
33	#define BCM4387_DEVICE_ID 0x5f71
34
35	#define BCM4377_TIMEOUT 1000
36
37	/*
38	* These devices only support DMA transactions inside a 32bit window
39	* (possibly to avoid 64 bit arithmetic). The window size cannot exceed
40	* 0xffffffff but is always aligned down to the previous 0x200 byte boundary
41	* which effectively limits the window to [start, start+0xfffffe00].
42	* We just limit the DMA window to [0, 0xfffffe00] to make sure we don't
43	* run into this limitation.
44	*/
45	#define BCM4377_DMA_MASK 0xfffffe00
46
47	#define BCM4377_PCIECFG_BAR0_WINDOW1 0x80
48	#define BCM4377_PCIECFG_BAR0_WINDOW2 0x70
49	#define BCM4377_PCIECFG_BAR0_CORE2_WINDOW1 0x74
50	#define BCM4377_PCIECFG_BAR0_CORE2_WINDOW2 0x78
51	#define BCM4377_PCIECFG_BAR2_WINDOW 0x84
52
53	#define BCM4377_PCIECFG_BAR0_CORE2_WINDOW1_DEFAULT 0x18011000
54	#define BCM4377_PCIECFG_BAR2_WINDOW_DEFAULT 0x19000000
55
56	#define BCM4377_PCIECFG_SUBSYSTEM_CTRL 0x88
57
58	#define BCM4377_BAR0_FW_DOORBELL 0x140
59	#define BCM4377_BAR0_RTI_CONTROL 0x144
60
61	#define BCM4377_BAR0_SLEEP_CONTROL 0x150
62	#define BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE 0
63	#define BCM4377_BAR0_SLEEP_CONTROL_AWAKE 2
64	#define BCM4377_BAR0_SLEEP_CONTROL_QUIESCE 3
65
66	#define BCM4377_BAR0_DOORBELL 0x174
67	#define BCM4377_BAR0_DOORBELL_VALUE GENMASK(31, 16)
68	#define BCM4377_BAR0_DOORBELL_IDX GENMASK(15, 8)
69	#define BCM4377_BAR0_DOORBELL_RING BIT(5)
70
71	#define BCM4377_BAR0_HOST_WINDOW_LO 0x590
72	#define BCM4377_BAR0_HOST_WINDOW_HI 0x594
73	#define BCM4377_BAR0_HOST_WINDOW_SIZE 0x598
74
75	#define BCM4377_BAR2_BOOTSTAGE 0x200454
76
77	#define BCM4377_BAR2_FW_LO 0x200478
78	#define BCM4377_BAR2_FW_HI 0x20047c
79	#define BCM4377_BAR2_FW_SIZE 0x200480
80
81	#define BCM4377_BAR2_CONTEXT_ADDR_LO 0x20048c
82	#define BCM4377_BAR2_CONTEXT_ADDR_HI 0x200450
83
84	#define BCM4377_BAR2_RTI_STATUS 0x20045c
85	#define BCM4377_BAR2_RTI_WINDOW_LO 0x200494
86	#define BCM4377_BAR2_RTI_WINDOW_HI 0x200498
87	#define BCM4377_BAR2_RTI_WINDOW_SIZE 0x20049c
88
89	#define BCM4377_OTP_SIZE 0xe0
90	#define BCM4377_OTP_SYS_VENDOR 0x15
91	#define BCM4377_OTP_CIS 0x80
92	#define BCM4377_OTP_VENDOR_HDR 0x00000008
93	#define BCM4377_OTP_MAX_PARAM_LEN 16
94
95	#define BCM4377_N_TRANSFER_RINGS 9
96	#define BCM4377_N_COMPLETION_RINGS 6
97
98	#define BCM4377_MAX_RING_SIZE 256
99
100	#define BCM4377_MSGID_GENERATION GENMASK(15, 8)
101	#define BCM4377_MSGID_ID GENMASK(7, 0)
102
103	#define BCM4377_RING_N_ENTRIES 128
104
105	#define BCM4377_CONTROL_MSG_SIZE 0x34
106	#define BCM4377_XFER_RING_MAX_INPLACE_PAYLOAD_SIZE (4 * 0xff)
107
108	#define MAX_ACL_PAYLOAD_SIZE (HCI_MAX_FRAME_SIZE + HCI_ACL_HDR_SIZE)
109	#define MAX_SCO_PAYLOAD_SIZE (HCI_MAX_SCO_SIZE + HCI_SCO_HDR_SIZE)
110	#define MAX_EVENT_PAYLOAD_SIZE (HCI_MAX_EVENT_SIZE + HCI_EVENT_HDR_SIZE)
111
112	enum bcm4377_otp_params_type {
113	BCM4377_OTP_BOARD_PARAMS,
114	BCM4377_OTP_CHIP_PARAMS
115	};
116
117	enum bcm4377_transfer_ring_id {
118	BCM4377_XFER_RING_CONTROL = 0,
119	BCM4377_XFER_RING_HCI_H2D = 1,
120	BCM4377_XFER_RING_HCI_D2H = 2,
121	BCM4377_XFER_RING_SCO_H2D = 3,
122	BCM4377_XFER_RING_SCO_D2H = 4,
123	BCM4377_XFER_RING_ACL_H2D = 5,
124	BCM4377_XFER_RING_ACL_D2H = 6,
125	};
126
127	enum bcm4377_completion_ring_id {
128	BCM4377_ACK_RING_CONTROL = 0,
129	BCM4377_ACK_RING_HCI_ACL = 1,
130	BCM4377_EVENT_RING_HCI_ACL = 2,
131	BCM4377_ACK_RING_SCO = 3,
132	BCM4377_EVENT_RING_SCO = 4,
133	};
134
135	enum bcm4377_doorbell {
136	BCM4377_DOORBELL_CONTROL = 0,
137	BCM4377_DOORBELL_HCI_H2D = 1,
138	BCM4377_DOORBELL_HCI_D2H = 2,
139	BCM4377_DOORBELL_ACL_H2D = 3,
140	BCM4377_DOORBELL_ACL_D2H = 4,
141	BCM4377_DOORBELL_SCO = 6,
142	};
143
144	/*
145	* Transfer ring entry
146	*
147	* flags: Flags to indicate if the payload is appended or mapped
148	* len: Payload length
149	* payload: Optional payload DMA address
150	* id: Message id to recognize the answer in the completion ring entry
151	*/
152	struct bcm4377_xfer_ring_entry {
153	#define BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED BIT(0)
154	#define BCM4377_XFER_RING_FLAG_PAYLOAD_IN_FOOTER BIT(1)
155	u8 flags;
156	__le16 len;
157	u8 _unk0;
158	__le64 payload;
159	__le16 id;
160	u8 _unk1[2];
161	} __packed;
162	static_assert(sizeof(struct bcm4377_xfer_ring_entry) == 0x10);
163
164	/*
165	* Completion ring entry
166	*
167	* flags: Flags to indicate if the payload is appended or mapped. If the payload
168	* is mapped it can be found in the buffer of the corresponding transfer
169	* ring message.
170	* ring_id: Transfer ring ID which required this message
171	* msg_id: Message ID specified in transfer ring entry
172	* len: Payload length
173	*/
174	struct bcm4377_completion_ring_entry {
175	u8 flags;
176	u8 _unk0;
177	__le16 ring_id;
178	__le16 msg_id;
179	__le32 len;
180	u8 _unk1[6];
181	} __packed;
182	static_assert(sizeof(struct bcm4377_completion_ring_entry) == 0x10);
183
184	enum bcm4377_control_message_type {
185	BCM4377_CONTROL_MSG_CREATE_XFER_RING = 1,
186	BCM4377_CONTROL_MSG_CREATE_COMPLETION_RING = 2,
187	BCM4377_CONTROL_MSG_DESTROY_XFER_RING = 3,
188	BCM4377_CONTROL_MSG_DESTROY_COMPLETION_RING = 4,
189	};
190
191	/*
192	* Control message used to create a completion ring
193	*
194	* msg_type: Must be BCM4377_CONTROL_MSG_CREATE_COMPLETION_RING
195	* header_size: Unknown, but probably reserved space in front of the entry
196	* footer_size: Number of 32 bit words reserved for payloads after the entry
197	* id/id_again: Completion ring index
198	* ring_iova: DMA address of the ring buffer
199	* n_elements: Number of elements inside the ring buffer
200	* msi: MSI index, doesn't work for all rings though and should be zero
201	* intmod_delay: Unknown delay
202	* intmod_bytes: Unknown
203	*/
204	struct bcm4377_create_completion_ring_msg {
205	u8 msg_type;
206	u8 header_size;
207	u8 footer_size;
208	u8 _unk0;
209	__le16 id;
210	__le16 id_again;
211	__le64 ring_iova;
212	__le16 n_elements;
213	__le32 unk;
214	u8 _unk1[6];
215	__le16 msi;
216	__le16 intmod_delay;
217	__le32 intmod_bytes;
218	__le16 _unk2;
219	__le32 _unk3;
220	u8 _unk4[10];
221	} __packed;
222	static_assert(sizeof(struct bcm4377_create_completion_ring_msg) ==
223	BCM4377_CONTROL_MSG_SIZE);
224
225	/*
226	* Control ring message used to destroy a completion ring
227	*
228	* msg_type: Must be BCM4377_CONTROL_MSG_DESTROY_COMPLETION_RING
229	* ring_id: Completion ring to be destroyed
230	*/
231	struct bcm4377_destroy_completion_ring_msg {
232	u8 msg_type;
233	u8 _pad0;
234	__le16 ring_id;
235	u8 _pad1[48];
236	} __packed;
237	static_assert(sizeof(struct bcm4377_destroy_completion_ring_msg) ==
238	BCM4377_CONTROL_MSG_SIZE);
239
240	/*
241	* Control message used to create a transfer ring
242	*
243	* msg_type: Must be BCM4377_CONTROL_MSG_CREATE_XFER_RING
244	* header_size: Number of 32 bit words reserved for unknown content before the
245	* entry
246	* footer_size: Number of 32 bit words reserved for payloads after the entry
247	* ring_id/ring_id_again: Transfer ring index
248	* ring_iova: DMA address of the ring buffer
249	* n_elements: Number of elements inside the ring buffer
250	* completion_ring_id: Completion ring index for acknowledgements and events
251	* doorbell: Doorbell index used to notify device of new entries
252	* flags: Transfer ring flags
253	* - virtual: set if there is no associated shared memory and only the
254	* corresponding completion ring is used
255	* - sync: only set for the SCO rings
256	*/
257	struct bcm4377_create_transfer_ring_msg {
258	u8 msg_type;
259	u8 header_size;
260	u8 footer_size;
261	u8 _unk0;
262	__le16 ring_id;
263	__le16 ring_id_again;
264	__le64 ring_iova;
265	u8 _unk1[8];
266	__le16 n_elements;
267	__le16 completion_ring_id;
268	__le16 doorbell;
269	#define BCM4377_XFER_RING_FLAG_VIRTUAL BIT(7)
270	#define BCM4377_XFER_RING_FLAG_SYNC BIT(8)
271	__le16 flags;
272	u8 _unk2[20];
273	} __packed;
274	static_assert(sizeof(struct bcm4377_create_transfer_ring_msg) ==
275	BCM4377_CONTROL_MSG_SIZE);
276
277	/*
278	* Control ring message used to destroy a transfer ring
279	*
280	* msg_type: Must be BCM4377_CONTROL_MSG_DESTROY_XFER_RING
281	* ring_id: Transfer ring to be destroyed
282	*/
283	struct bcm4377_destroy_transfer_ring_msg {
284	u8 msg_type;
285	u8 _pad0;
286	__le16 ring_id;
287	u8 _pad1[48];
288	} __packed;
289	static_assert(sizeof(struct bcm4377_destroy_transfer_ring_msg) ==
290	BCM4377_CONTROL_MSG_SIZE);
291
292	/*
293	* "Converged IPC" context struct used to make the device aware of all other
294	* shared memory structures. A pointer to this structure is configured inside a
295	* MMIO register.
296	*
297	* version: Protocol version, must be 2.
298	* size: Size of this structure, must be 0x68.
299	* enabled_caps: Enabled capabilities. Unknown bitfield but should be 2.
300	* peripheral_info_addr: DMA address for a 0x20 buffer to which the device will
301	* write unknown contents
302	* {completion,xfer}_ring_{tails,heads}_addr: DMA pointers to ring heads/tails
303	* n_completion_rings: Number of completion rings, the firmware only works if
304	* this is set to BCM4377_N_COMPLETION_RINGS.
305	* n_xfer_rings: Number of transfer rings, the firmware only works if
306	* this is set to BCM4377_N_TRANSFER_RINGS.
307	* control_completion_ring_addr: Control completion ring buffer DMA address
308	* control_xfer_ring_addr: Control transfer ring buffer DMA address
309	* control_xfer_ring_n_entries: Number of control transfer ring entries
310	* control_completion_ring_n_entries: Number of control completion ring entries
311	* control_xfer_ring_doorbell: Control transfer ring doorbell
312	* control_completion_ring_doorbell: Control completion ring doorbell,
313	* must be set to 0xffff
314	* control_xfer_ring_msi: Control completion ring MSI index, must be 0
315	* control_completion_ring_msi: Control completion ring MSI index, must be 0.
316	* control_xfer_ring_header_size: Number of 32 bit words reserved in front of
317	* every control transfer ring entry
318	* control_xfer_ring_footer_size: Number of 32 bit words reserved after every
319	* control transfer ring entry
320	* control_completion_ring_header_size: Number of 32 bit words reserved in front
321	* of every control completion ring entry
322	* control_completion_ring_footer_size: Number of 32 bit words reserved after
323	* every control completion ring entry
324	* scratch_pad: Optional scratch pad DMA address
325	* scratch_pad_size: Scratch pad size
326	*/
327	struct bcm4377_context {
328	__le16 version;
329	__le16 size;
330	__le32 enabled_caps;
331
332	__le64 peripheral_info_addr;
333
334	/* ring heads and tails */
335	__le64 completion_ring_heads_addr;
336	__le64 xfer_ring_tails_addr;
337	__le64 completion_ring_tails_addr;
338	__le64 xfer_ring_heads_addr;
339	__le16 n_completion_rings;
340	__le16 n_xfer_rings;
341
342	/* control ring configuration */
343	__le64 control_completion_ring_addr;
344	__le64 control_xfer_ring_addr;
345	__le16 control_xfer_ring_n_entries;
346	__le16 control_completion_ring_n_entries;
347	__le16 control_xfer_ring_doorbell;
348	__le16 control_completion_ring_doorbell;
349	__le16 control_xfer_ring_msi;
350	__le16 control_completion_ring_msi;
351	u8 control_xfer_ring_header_size;
352	u8 control_xfer_ring_footer_size;
353	u8 control_completion_ring_header_size;
354	u8 control_completion_ring_footer_size;
355
356	__le16 _unk0;
357	__le16 _unk1;
358
359	__le64 scratch_pad;
360	__le32 scratch_pad_size;
361
362	__le32 _unk3;
363	} __packed;
364	static_assert(sizeof(struct bcm4377_context) == 0x68);
365
366	#define BCM4378_CALIBRATION_CHUNK_SIZE 0xe6
367	struct bcm4378_hci_send_calibration_cmd {
368	u8 unk;
369	__le16 blocks_left;
370	u8 data[BCM4378_CALIBRATION_CHUNK_SIZE];
371	} __packed;
372
373	#define BCM4378_PTB_CHUNK_SIZE 0xcf
374	struct bcm4378_hci_send_ptb_cmd {
375	__le16 blocks_left;
376	u8 data[BCM4378_PTB_CHUNK_SIZE];
377	} __packed;
378
379	/*
380	* Shared memory structure used to store the ring head and tail pointers.
381	*/
382	struct bcm4377_ring_state {
383	__le16 completion_ring_head[BCM4377_N_COMPLETION_RINGS];
384	__le16 completion_ring_tail[BCM4377_N_COMPLETION_RINGS];
385	__le16 xfer_ring_head[BCM4377_N_TRANSFER_RINGS];
386	__le16 xfer_ring_tail[BCM4377_N_TRANSFER_RINGS];
387	};
388
389	/*
390	* A transfer ring can be used in two configurations:
391	* 1) Send control or HCI messages to the device which are then acknowledged
392	* in the corresponding completion ring
393	* 2) Receiving HCI frames from the devices. In this case the transfer ring
394	* itself contains empty messages that are acknowledged once data is
395	* available from the device. If the payloads fit inside the footers
396	* of the completion ring the transfer ring can be configured to be
397	* virtual such that it has no ring buffer.
398	*
399	* ring_id: ring index hardcoded in the firmware
400	* doorbell: doorbell index to notify device of new entries
401	* payload_size: optional in-place payload size
402	* mapped_payload_size: optional out-of-place payload size
403	* completion_ring: index of corresponding completion ring
404	* n_entries: number of entries inside this ring
405	* generation: ring generation; incremented on hci_open to detect stale messages
406	* sync: set to true for SCO rings
407	* virtual: set to true if this ring has no entries and is just required to
408	* setup a corresponding completion ring for device->host messages
409	* d2h_buffers_only: set to true if this ring is only used to provide large
410	* buffers used by device->host messages in the completion
411	* ring
412	* allow_wait: allow to wait for messages to be acknowledged
413	* enabled: true once the ring has been created and can be used
414	* ring: ring buffer for entries (struct bcm4377_xfer_ring_entry)
415	* ring_dma: DMA address for ring entry buffer
416	* payloads: payload buffer for mapped_payload_size payloads
417	* payloads_dma:DMA address for payload buffer
418	* events: pointer to array of completions if waiting is allowed
419	* msgids: bitmap to keep track of used message ids
420	* lock: Spinlock to protect access to ring structurs used in the irq handler
421	*/
422	struct bcm4377_transfer_ring {
423	enum bcm4377_transfer_ring_id ring_id;
424	enum bcm4377_doorbell doorbell;
425	size_t payload_size;
426	size_t mapped_payload_size;
427	u8 completion_ring;
428	u16 n_entries;
429	u8 generation;
430
431	bool sync;
432	bool virtual;
433	bool d2h_buffers_only;
434	bool allow_wait;
435	bool enabled;
436
437	void *ring;
438	dma_addr_t ring_dma;
439
440	void *payloads;
441	dma_addr_t payloads_dma;
442
443	struct completion **events;
444	DECLARE_BITMAP(msgids, BCM4377_MAX_RING_SIZE);
445	spinlock_t lock;
446	};
447
448	/*
449	* A completion ring can be either used to either acknowledge messages sent in
450	* the corresponding transfer ring or to receive messages associated with the
451	* transfer ring. When used to receive messages the transfer ring either
452	* has no ring buffer and is only advanced ("virtual transfer ring") or it
453	* only contains empty DMA buffers to be used for the payloads.
454	*
455	* ring_id: completion ring id, hardcoded in firmware
456	* payload_size: optional payload size after each entry
457	* delay: unknown delay
458	* n_entries: number of entries in this ring
459	* enabled: true once the ring has been created and can be used
460	* ring: ring buffer for entries (struct bcm4377_completion_ring_entry)
461	* ring_dma: DMA address of ring buffer
462	* transfer_rings: bitmap of corresponding transfer ring ids
463	*/
464	struct bcm4377_completion_ring {
465	enum bcm4377_completion_ring_id ring_id;
466	u16 payload_size;
467	u16 delay;
468	u16 n_entries;
469	bool enabled;
470
471	void *ring;
472	dma_addr_t ring_dma;
473
474	unsigned long transfer_rings;
475	};
476
477	struct bcm4377_data;
478
479	/*
480	* Chip-specific configuration struct
481	*
482	* id: Chip id (e.g. 0x4377 for BCM4377)
483	* otp_offset: Offset to the start of the OTP inside BAR0
484	* bar0_window1: Backplane address mapped to the first window in BAR0
485	* bar0_window2: Backplane address mapped to the second window in BAR0
486	* bar0_core2_window2: Optional backplane address mapped to the second core's
487	* second window in BAR0
488	* has_bar0_core2_window2: Set to true if this chip requires the second core's
489	* second window to be configured
490	* clear_pciecfg_subsystem_ctrl_bit19: Set to true if bit 19 in the
491	* vendor-specific subsystem control
492	* register has to be cleared
493	* disable_aspm: Set to true if ASPM must be disabled due to hardware errata
494	* broken_ext_scan: Set to true if the chip erroneously claims to support
495	* extended scanning
496	* broken_mws_transport_config: Set to true if the chip erroneously claims to
497	* support MWS Transport Configuration
498	* send_calibration: Optional callback to send calibration data
499	* send_ptb: Callback to send "PTB" regulatory/calibration data
500	*/
501	struct bcm4377_hw {
502	unsigned int id;
503
504	u32 otp_offset;
505
506	u32 bar0_window1;
507	u32 bar0_window2;
508	u32 bar0_core2_window2;
509
510	unsigned long has_bar0_core2_window2 : 1;
511	unsigned long clear_pciecfg_subsystem_ctrl_bit19 : 1;
512	unsigned long disable_aspm : 1;
513	unsigned long broken_ext_scan : 1;
514	unsigned long broken_mws_transport_config : 1;
515	unsigned long broken_le_coded : 1;
516
517	int (*send_calibration)(struct bcm4377_data *bcm4377);
518	int (*send_ptb)(struct bcm4377_data *bcm4377,
519	const struct firmware *fw);
520	};
521
522	static const struct bcm4377_hw bcm4377_hw_variants[];
523	static const struct dmi_system_id bcm4377_dmi_board_table[];
524
525	/*
526	* Private struct associated with each device containing global state
527	*
528	* pdev: Pointer to associated struct pci_dev
529	* hdev: Pointer to associated strucy hci_dev
530	* bar0: iomem pointing to BAR0
531	* bar1: iomem pointing to BAR2
532	* bootstage: Current value of the bootstage
533	* rti_status: Current "RTI" status value
534	* hw: Pointer to chip-specific struct bcm4377_hw
535	* taurus_cal_blob: "Taurus" calibration blob used for some chips
536	* taurus_cal_size: "Taurus" calibration blob size
537	* taurus_beamforming_cal_blob: "Taurus" beamforming calibration blob used for
538	* some chips
539	* taurus_beamforming_cal_size: "Taurus" beamforming calibration blob size
540	* stepping: Chip stepping read from OTP; used for firmware selection
541	* vendor: Antenna vendor read from OTP; used for firmware selection
542	* board_type: Board type from FDT or DMI match; used for firmware selection
543	* event: Event for changed bootstage or rti_status; used for booting firmware
544	* ctx: "Converged IPC" context
545	* ctx_dma: "Converged IPC" context DMA address
546	* ring_state: Shared memory buffer containing ring head and tail indexes
547	* ring_state_dma: DMA address for ring_state
548	* {control,hci_acl,sco}_ack_ring: Completion rings used to acknowledge messages
549	* {hci_acl,sco}_event_ring: Completion rings used for device->host messages
550	* control_h2d_ring: Transfer ring used for control messages
551	* {hci,sco,acl}_h2d_ring: Transfer ring used to transfer HCI frames
552	* {hci,sco,acl}_d2h_ring: Transfer ring used to receive HCI frames in the
553	* corresponding completion ring
554	*/
555	struct bcm4377_data {
556	struct pci_dev *pdev;
557	struct hci_dev *hdev;
558
559	void __iomem *bar0;
560	void __iomem *bar2;
561
562	u32 bootstage;
563	u32 rti_status;
564
565	const struct bcm4377_hw *hw;
566
567	const void *taurus_cal_blob;
568	int taurus_cal_size;
569	const void *taurus_beamforming_cal_blob;
570	int taurus_beamforming_cal_size;
571
572	char stepping[BCM4377_OTP_MAX_PARAM_LEN];
573	char vendor[BCM4377_OTP_MAX_PARAM_LEN];
574	const char *board_type;
575
576	struct completion event;
577
578	struct bcm4377_context *ctx;
579	dma_addr_t ctx_dma;
580
581	struct bcm4377_ring_state *ring_state;
582	dma_addr_t ring_state_dma;
583
584	/*
585	* The HCI and ACL rings have to be merged because this structure is
586	* hardcoded in the firmware.
587	*/
588	struct bcm4377_completion_ring control_ack_ring;
589	struct bcm4377_completion_ring hci_acl_ack_ring;
590	struct bcm4377_completion_ring hci_acl_event_ring;
591	struct bcm4377_completion_ring sco_ack_ring;
592	struct bcm4377_completion_ring sco_event_ring;
593
594	struct bcm4377_transfer_ring control_h2d_ring;
595	struct bcm4377_transfer_ring hci_h2d_ring;
596	struct bcm4377_transfer_ring hci_d2h_ring;
597	struct bcm4377_transfer_ring sco_h2d_ring;
598	struct bcm4377_transfer_ring sco_d2h_ring;
599	struct bcm4377_transfer_ring acl_h2d_ring;
600	struct bcm4377_transfer_ring acl_d2h_ring;
601	};
602
603	static void bcm4377_ring_doorbell(struct bcm4377_data *bcm4377, u8 doorbell,
604	u16 val)
605	{
606	u32 db = 0;
607
608	db |= FIELD_PREP(BCM4377_BAR0_DOORBELL_VALUE, val);
609	db |= FIELD_PREP(BCM4377_BAR0_DOORBELL_IDX, doorbell);
610	db |= BCM4377_BAR0_DOORBELL_RING;
611
612	dev_dbg(&bcm4377->pdev->dev, "write %d to doorbell #%d (0x%x)\n", val,
613	doorbell, db);
614	iowrite32(db, bcm4377->bar0 + BCM4377_BAR0_DOORBELL);
615	}
616
617	static int bcm4377_extract_msgid(struct bcm4377_data *bcm4377,
618	struct bcm4377_transfer_ring *ring,
619	u16 raw_msgid, u8 *msgid)
620	{
621	u8 generation = FIELD_GET(BCM4377_MSGID_GENERATION, raw_msgid);
622	*msgid = FIELD_GET(BCM4377_MSGID_ID, raw_msgid);
623
624	if (generation != ring->generation) {
625	dev_warn(
626	&bcm4377->pdev->dev,
627	"invalid message generation %d should be %d in entry for ring %d\n",
628	generation, ring->generation, ring->ring_id);
629	return -EINVAL;
630	}
631
632	if (*msgid >= ring->n_entries) {
633	dev_warn(&bcm4377->pdev->dev,
634	"invalid message id in entry for ring %d: %d > %d\n",
635	ring->ring_id, *msgid, ring->n_entries);
636	return -EINVAL;
637	}
638
639	return 0;
640	}
641
642	static void bcm4377_handle_event(struct bcm4377_data *bcm4377,
643	struct bcm4377_transfer_ring *ring,
644	u16 raw_msgid, u8 entry_flags, u8 type,
645	void *payload, size_t len)
646	{
647	struct sk_buff *skb;
648	u16 head;
649	u8 msgid;
650	unsigned long flags;
651
652	spin_lock_irqsave(&ring->lock, flags);
653	if (!ring->enabled) {
654	dev_warn(&bcm4377->pdev->dev,
655	"event for disabled transfer ring %d\n",
656	ring->ring_id);
657	goto out;
658	}
659
660	if (ring->d2h_buffers_only &&
661	entry_flags & BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED) {
662	if (bcm4377_extract_msgid(bcm4377, ring, raw_msgid, msgid: &msgid))
663	goto out;
664
665	if (len > ring->mapped_payload_size) {
666	dev_warn(
667	&bcm4377->pdev->dev,
668	"invalid payload len in event for ring %d: %zu > %zu\n",
669	ring->ring_id, len, ring->mapped_payload_size);
670	goto out;
671	}
672
673	payload = ring->payloads + msgid * ring->mapped_payload_size;
674	}
675
676	skb = bt_skb_alloc(len, GFP_ATOMIC);
677	if (!skb)
678	goto out;
679
680	memcpy(skb_put(skb, len), payload, len);
681	hci_skb_pkt_type(skb) = type;
682	hci_recv_frame(hdev: bcm4377->hdev, skb);
683
684	out:
685	head = le16_to_cpu(bcm4377->ring_state->xfer_ring_head[ring->ring_id]);
686	head = (head + 1) % ring->n_entries;
687	bcm4377->ring_state->xfer_ring_head[ring->ring_id] = cpu_to_le16(head);
688
689	bcm4377_ring_doorbell(bcm4377, doorbell: ring->doorbell, val: head);
690
691	spin_unlock_irqrestore(lock: &ring->lock, flags);
692	}
693
694	static void bcm4377_handle_ack(struct bcm4377_data *bcm4377,
695	struct bcm4377_transfer_ring *ring,
696	u16 raw_msgid)
697	{
698	unsigned long flags;
699	u8 msgid;
700
701	spin_lock_irqsave(&ring->lock, flags);
702
703	if (bcm4377_extract_msgid(bcm4377, ring, raw_msgid, msgid: &msgid))
704	goto unlock;
705
706	if (!test_bit(msgid, ring->msgids)) {
707	dev_warn(
708	&bcm4377->pdev->dev,
709	"invalid message id in ack for ring %d: %d is not used\n",
710	ring->ring_id, msgid);
711	goto unlock;
712	}
713
714	if (ring->allow_wait && ring->events[msgid]) {
715	complete(ring->events[msgid]);
716	ring->events[msgid] = NULL;
717	}
718
719	bitmap_release_region(bitmap: ring->msgids, pos: msgid, order: ring->n_entries);
720
721	unlock:
722	spin_unlock_irqrestore(lock: &ring->lock, flags);
723	}
724
725	static void bcm4377_handle_completion(struct bcm4377_data *bcm4377,
726	struct bcm4377_completion_ring *ring,
727	u16 pos)
728	{
729	struct bcm4377_completion_ring_entry *entry;
730	u16 msg_id, transfer_ring;
731	size_t entry_size, data_len;
732	void *data;
733
734	if (pos >= ring->n_entries) {
735	dev_warn(&bcm4377->pdev->dev,
736	"invalid offset %d for completion ring %d\n", pos,
737	ring->ring_id);
738	return;
739	}
740
741	entry_size = sizeof(*entry) + ring->payload_size;
742	entry = ring->ring + pos * entry_size;
743	data = ring->ring + pos * entry_size + sizeof(*entry);
744	data_len = le32_to_cpu(entry->len);
745	msg_id = le16_to_cpu(entry->msg_id);
746	transfer_ring = le16_to_cpu(entry->ring_id);
747
748	if ((ring->transfer_rings & BIT(transfer_ring)) == 0) {
749	dev_warn(
750	&bcm4377->pdev->dev,
751	"invalid entry at offset %d for transfer ring %d in completion ring %d\n",
752	pos, transfer_ring, ring->ring_id);
753	return;
754	}
755
756	dev_dbg(&bcm4377->pdev->dev,
757	"entry in completion ring %d for transfer ring %d with msg_id %d\n",
758	ring->ring_id, transfer_ring, msg_id);
759
760	switch (transfer_ring) {
761	case BCM4377_XFER_RING_CONTROL:
762	bcm4377_handle_ack(bcm4377, ring: &bcm4377->control_h2d_ring, raw_msgid: msg_id);
763	break;
764	case BCM4377_XFER_RING_HCI_H2D:
765	bcm4377_handle_ack(bcm4377, ring: &bcm4377->hci_h2d_ring, raw_msgid: msg_id);
766	break;
767	case BCM4377_XFER_RING_SCO_H2D:
768	bcm4377_handle_ack(bcm4377, ring: &bcm4377->sco_h2d_ring, raw_msgid: msg_id);
769	break;
770	case BCM4377_XFER_RING_ACL_H2D:
771	bcm4377_handle_ack(bcm4377, ring: &bcm4377->acl_h2d_ring, raw_msgid: msg_id);
772	break;
773
774	case BCM4377_XFER_RING_HCI_D2H:
775	bcm4377_handle_event(bcm4377, ring: &bcm4377->hci_d2h_ring, raw_msgid: msg_id,
776	entry_flags: entry->flags, HCI_EVENT_PKT, payload: data,
777	len: data_len);
778	break;
779	case BCM4377_XFER_RING_SCO_D2H:
780	bcm4377_handle_event(bcm4377, ring: &bcm4377->sco_d2h_ring, raw_msgid: msg_id,
781	entry_flags: entry->flags, HCI_SCODATA_PKT, payload: data,
782	len: data_len);
783	break;
784	case BCM4377_XFER_RING_ACL_D2H:
785	bcm4377_handle_event(bcm4377, ring: &bcm4377->acl_d2h_ring, raw_msgid: msg_id,
786	entry_flags: entry->flags, HCI_ACLDATA_PKT, payload: data,
787	len: data_len);
788	break;
789
790	default:
791	dev_warn(
792	&bcm4377->pdev->dev,
793	"entry in completion ring %d for unknown transfer ring %d with msg_id %d\n",
794	ring->ring_id, transfer_ring, msg_id);
795	}
796	}
797
798	static void bcm4377_poll_completion_ring(struct bcm4377_data *bcm4377,
799	struct bcm4377_completion_ring *ring)
800	{
801	u16 tail;
802	__le16 *heads = bcm4377->ring_state->completion_ring_head;
803	__le16 *tails = bcm4377->ring_state->completion_ring_tail;
804
805	if (!ring->enabled)
806	return;
807
808	tail = le16_to_cpu(tails[ring->ring_id]);
809	dev_dbg(&bcm4377->pdev->dev,
810	"completion ring #%d: head: %d, tail: %d\n", ring->ring_id,
811	le16_to_cpu(heads[ring->ring_id]), tail);
812
813	while (tail != le16_to_cpu(READ_ONCE(heads[ring->ring_id]))) {
814	/*
815	* ensure the CPU doesn't speculate through the comparison.
816	* otherwise it might already read the (empty) queue entry
817	* before the updated head has been loaded and checked.
818	*/
819	dma_rmb();
820
821	bcm4377_handle_completion(bcm4377, ring, pos: tail);
822
823	tail = (tail + 1) % ring->n_entries;
824	tails[ring->ring_id] = cpu_to_le16(tail);
825	}
826	}
827
828	static irqreturn_t bcm4377_irq(int irq, void *data)
829	{
830	struct bcm4377_data *bcm4377 = data;
831	u32 bootstage, rti_status;
832
833	bootstage = ioread32(bcm4377->bar2 + BCM4377_BAR2_BOOTSTAGE);
834	rti_status = ioread32(bcm4377->bar2 + BCM4377_BAR2_RTI_STATUS);
835
836	if (bootstage != bcm4377->bootstage ||
837	rti_status != bcm4377->rti_status) {
838	dev_dbg(&bcm4377->pdev->dev,
839	"bootstage = %d -> %d, rti state = %d -> %d\n",
840	bcm4377->bootstage, bootstage, bcm4377->rti_status,
841	rti_status);
842	complete(&bcm4377->event);
843	bcm4377->bootstage = bootstage;
844	bcm4377->rti_status = rti_status;
845	}
846
847	if (rti_status > 2)
848	dev_err(&bcm4377->pdev->dev, "RTI status is %d\n", rti_status);
849
850	bcm4377_poll_completion_ring(bcm4377, ring: &bcm4377->control_ack_ring);
851	bcm4377_poll_completion_ring(bcm4377, ring: &bcm4377->hci_acl_event_ring);
852	bcm4377_poll_completion_ring(bcm4377, ring: &bcm4377->hci_acl_ack_ring);
853	bcm4377_poll_completion_ring(bcm4377, ring: &bcm4377->sco_ack_ring);
854	bcm4377_poll_completion_ring(bcm4377, ring: &bcm4377->sco_event_ring);
855
856	return IRQ_HANDLED;
857	}
858
859	static int bcm4377_enqueue(struct bcm4377_data *bcm4377,
860	struct bcm4377_transfer_ring *ring, void *data,
861	size_t len, bool wait)
862	{
863	unsigned long flags;
864	struct bcm4377_xfer_ring_entry *entry;
865	void *payload;
866	size_t offset;
867	u16 head, tail, new_head;
868	u16 raw_msgid;
869	int ret, msgid;
870	DECLARE_COMPLETION_ONSTACK(event);
871
872	if (len > ring->payload_size && len > ring->mapped_payload_size) {
873	dev_warn(
874	&bcm4377->pdev->dev,
875	"payload len %zu is too large for ring %d (max is %zu or %zu)\n",
876	len, ring->ring_id, ring->payload_size,
877	ring->mapped_payload_size);
878	return -EINVAL;
879	}
880	if (wait && !ring->allow_wait)
881	return -EINVAL;
882	if (ring->virtual)
883	return -EINVAL;
884
885	spin_lock_irqsave(&ring->lock, flags);
886
887	head = le16_to_cpu(bcm4377->ring_state->xfer_ring_head[ring->ring_id]);
888	tail = le16_to_cpu(bcm4377->ring_state->xfer_ring_tail[ring->ring_id]);
889
890	new_head = (head + 1) % ring->n_entries;
891
892	if (new_head == tail) {
893	dev_warn(&bcm4377->pdev->dev,
894	"can't send message because ring %d is full\n",
895	ring->ring_id);
896	ret = -EINVAL;
897	goto out;
898	}
899
900	msgid = bitmap_find_free_region(bitmap: ring->msgids, bits: ring->n_entries, order: 0);
901	if (msgid < 0) {
902	dev_warn(&bcm4377->pdev->dev,
903	"can't find message id for ring %d\n", ring->ring_id);
904	ret = -EINVAL;
905	goto out;
906	}
907
908	raw_msgid = FIELD_PREP(BCM4377_MSGID_GENERATION, ring->generation);
909	raw_msgid |= FIELD_PREP(BCM4377_MSGID_ID, msgid);
910
911	offset = head * (sizeof(*entry) + ring->payload_size);
912	entry = ring->ring + offset;
913
914	memset(entry, 0, sizeof(*entry));
915	entry->id = cpu_to_le16(raw_msgid);
916	entry->len = cpu_to_le16(len);
917
918	if (len <= ring->payload_size) {
919	entry->flags = BCM4377_XFER_RING_FLAG_PAYLOAD_IN_FOOTER;
920	payload = ring->ring + offset + sizeof(*entry);
921	} else {
922	entry->flags = BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED;
923	entry->payload = cpu_to_le64(ring->payloads_dma +
924	msgid * ring->mapped_payload_size);
925	payload = ring->payloads + msgid * ring->mapped_payload_size;
926	}
927
928	memcpy(payload, data, len);
929
930	if (wait)
931	ring->events[msgid] = &event;
932
933	/*
934	* The 4377 chips stop responding to any commands as soon as they
935	* have been idle for a while. Poking the sleep control register here
936	* makes them come alive again.
937	*/
938	iowrite32(BCM4377_BAR0_SLEEP_CONTROL_AWAKE,
939	bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL);
940
941	dev_dbg(&bcm4377->pdev->dev,
942	"updating head for transfer queue #%d to %d\n", ring->ring_id,
943	new_head);
944	bcm4377->ring_state->xfer_ring_head[ring->ring_id] =
945	cpu_to_le16(new_head);
946
947	if (!ring->sync)
948	bcm4377_ring_doorbell(bcm4377, doorbell: ring->doorbell, val: new_head);
949	ret = 0;
950
951	out:
952	spin_unlock_irqrestore(lock: &ring->lock, flags);
953
954	if (ret == 0 && wait) {
955	ret = wait_for_completion_interruptible_timeout(
956	x: &event, BCM4377_TIMEOUT);
957	if (ret == 0)
958	ret = -ETIMEDOUT;
959	else if (ret > 0)
960	ret = 0;
961
962	spin_lock_irqsave(&ring->lock, flags);
963	ring->events[msgid] = NULL;
964	spin_unlock_irqrestore(lock: &ring->lock, flags);
965	}
966
967	return ret;
968	}
969
970	static int bcm4377_create_completion_ring(struct bcm4377_data *bcm4377,
971	struct bcm4377_completion_ring *ring)
972	{
973	struct bcm4377_create_completion_ring_msg msg;
974	int ret;
975
976	if (ring->enabled) {
977	dev_warn(&bcm4377->pdev->dev,
978	"completion ring %d already enabled\n", ring->ring_id);
979	return 0;
980	}
981
982	memset(ring->ring, 0,
983	ring->n_entries * (sizeof(struct bcm4377_completion_ring_entry) +
984	ring->payload_size));
985	memset(&msg, 0, sizeof(msg));
986	msg.msg_type = BCM4377_CONTROL_MSG_CREATE_COMPLETION_RING;
987	msg.id = cpu_to_le16(ring->ring_id);
988	msg.id_again = cpu_to_le16(ring->ring_id);
989	msg.ring_iova = cpu_to_le64(ring->ring_dma);
990	msg.n_elements = cpu_to_le16(ring->n_entries);
991	msg.intmod_bytes = cpu_to_le32(0xffffffff);
992	msg.unk = cpu_to_le32(0xffffffff);
993	msg.intmod_delay = cpu_to_le16(ring->delay);
994	msg.footer_size = ring->payload_size / 4;
995
996	ret = bcm4377_enqueue(bcm4377, ring: &bcm4377->control_h2d_ring, data: &msg,
997	len: sizeof(msg), wait: true);
998	if (!ret)
999	ring->enabled = true;
1000
1001	return ret;
1002	}
1003
1004	static int bcm4377_destroy_completion_ring(struct bcm4377_data *bcm4377,
1005	struct bcm4377_completion_ring *ring)
1006	{
1007	struct bcm4377_destroy_completion_ring_msg msg;
1008	int ret;
1009
1010	memset(&msg, 0, sizeof(msg));
1011	msg.msg_type = BCM4377_CONTROL_MSG_DESTROY_COMPLETION_RING;
1012	msg.ring_id = cpu_to_le16(ring->ring_id);
1013
1014	ret = bcm4377_enqueue(bcm4377, ring: &bcm4377->control_h2d_ring, data: &msg,
1015	len: sizeof(msg), wait: true);
1016	if (ret)
1017	dev_warn(&bcm4377->pdev->dev,
1018	"failed to destroy completion ring %d\n",
1019	ring->ring_id);
1020
1021	ring->enabled = false;
1022	return ret;
1023	}
1024
1025	static int bcm4377_create_transfer_ring(struct bcm4377_data *bcm4377,
1026	struct bcm4377_transfer_ring *ring)
1027	{
1028	struct bcm4377_create_transfer_ring_msg msg;
1029	u16 flags = 0;
1030	int ret, i;
1031	unsigned long spinlock_flags;
1032
1033	if (ring->virtual)
1034	flags |= BCM4377_XFER_RING_FLAG_VIRTUAL;
1035	if (ring->sync)
1036	flags |= BCM4377_XFER_RING_FLAG_SYNC;
1037
1038	spin_lock_irqsave(&ring->lock, spinlock_flags);
1039	memset(&msg, 0, sizeof(msg));
1040	msg.msg_type = BCM4377_CONTROL_MSG_CREATE_XFER_RING;
1041	msg.ring_id = cpu_to_le16(ring->ring_id);
1042	msg.ring_id_again = cpu_to_le16(ring->ring_id);
1043	msg.ring_iova = cpu_to_le64(ring->ring_dma);
1044	msg.n_elements = cpu_to_le16(ring->n_entries);
1045	msg.completion_ring_id = cpu_to_le16(ring->completion_ring);
1046	msg.doorbell = cpu_to_le16(ring->doorbell);
1047	msg.flags = cpu_to_le16(flags);
1048	msg.footer_size = ring->payload_size / 4;
1049
1050	bcm4377->ring_state->xfer_ring_head[ring->ring_id] = 0;
1051	bcm4377->ring_state->xfer_ring_tail[ring->ring_id] = 0;
1052	ring->generation++;
1053	spin_unlock_irqrestore(lock: &ring->lock, flags: spinlock_flags);
1054
1055	ret = bcm4377_enqueue(bcm4377, ring: &bcm4377->control_h2d_ring, data: &msg,
1056	len: sizeof(msg), wait: true);
1057
1058	spin_lock_irqsave(&ring->lock, spinlock_flags);
1059
1060	if (ring->d2h_buffers_only) {
1061	for (i = 0; i < ring->n_entries; ++i) {
1062	struct bcm4377_xfer_ring_entry *entry =
1063	ring->ring + i * sizeof(*entry);
1064	u16 raw_msgid = FIELD_PREP(BCM4377_MSGID_GENERATION,
1065	ring->generation);
1066	raw_msgid |= FIELD_PREP(BCM4377_MSGID_ID, i);
1067
1068	memset(entry, 0, sizeof(*entry));
1069	entry->id = cpu_to_le16(raw_msgid);
1070	entry->len = cpu_to_le16(ring->mapped_payload_size);
1071	entry->flags = BCM4377_XFER_RING_FLAG_PAYLOAD_MAPPED;
1072	entry->payload =
1073	cpu_to_le64(ring->payloads_dma +
1074	i * ring->mapped_payload_size);
1075	}
1076	}
1077
1078	/*
1079	* send some messages if this is a device->host ring to allow the device
1080	* to reply by acknowledging them in the completion ring
1081	*/
1082	if (ring->virtual || ring->d2h_buffers_only) {
1083	bcm4377->ring_state->xfer_ring_head[ring->ring_id] =
1084	cpu_to_le16(0xf);
1085	bcm4377_ring_doorbell(bcm4377, doorbell: ring->doorbell, val: 0xf);
1086	}
1087
1088	ring->enabled = true;
1089	spin_unlock_irqrestore(lock: &ring->lock, flags: spinlock_flags);
1090
1091	return ret;
1092	}
1093
1094	static int bcm4377_destroy_transfer_ring(struct bcm4377_data *bcm4377,
1095	struct bcm4377_transfer_ring *ring)
1096	{
1097	struct bcm4377_destroy_transfer_ring_msg msg;
1098	int ret;
1099
1100	memset(&msg, 0, sizeof(msg));
1101	msg.msg_type = BCM4377_CONTROL_MSG_DESTROY_XFER_RING;
1102	msg.ring_id = cpu_to_le16(ring->ring_id);
1103
1104	ret = bcm4377_enqueue(bcm4377, ring: &bcm4377->control_h2d_ring, data: &msg,
1105	len: sizeof(msg), wait: true);
1106	if (ret)
1107	dev_warn(&bcm4377->pdev->dev,
1108	"failed to destroy transfer ring %d\n", ring->ring_id);
1109
1110	ring->enabled = false;
1111	return ret;
1112	}
1113
1114	static int __bcm4378_send_calibration_chunk(struct bcm4377_data *bcm4377,
1115	const void *data, size_t data_len,
1116	u16 blocks_left)
1117	{
1118	struct bcm4378_hci_send_calibration_cmd cmd;
1119	struct sk_buff *skb;
1120
1121	if (data_len > sizeof(cmd.data))
1122	return -EINVAL;
1123
1124	memset(&cmd, 0, sizeof(cmd));
1125	cmd.unk = 0x03;
1126	cmd.blocks_left = cpu_to_le16(blocks_left);
1127	memcpy(cmd.data, data, data_len);
1128
1129	skb = __hci_cmd_sync(hdev: bcm4377->hdev, opcode: 0xfd97, plen: sizeof(cmd), param: &cmd,
1130	HCI_INIT_TIMEOUT);
1131	if (IS_ERR(ptr: skb))
1132	return PTR_ERR(ptr: skb);
1133
1134	kfree_skb(skb);
1135	return 0;
1136	}
1137
1138	static int __bcm4378_send_calibration(struct bcm4377_data *bcm4377,
1139	const void *data, size_t data_size)
1140	{
1141	int ret;
1142	size_t i, left, transfer_len;
1143	size_t blocks =
1144	DIV_ROUND_UP(data_size, (size_t)BCM4378_CALIBRATION_CHUNK_SIZE);
1145
1146	if (!data) {
1147	dev_err(&bcm4377->pdev->dev,
1148	"no calibration data available.\n");
1149	return -ENOENT;
1150	}
1151
1152	for (i = 0, left = data_size; i < blocks; ++i, left -= transfer_len) {
1153	transfer_len =
1154	min_t(size_t, left, BCM4378_CALIBRATION_CHUNK_SIZE);
1155
1156	ret = __bcm4378_send_calibration_chunk(
1157	bcm4377, data: data + i * BCM4378_CALIBRATION_CHUNK_SIZE,
1158	data_len: transfer_len, blocks_left: blocks - i - 1);
1159	if (ret) {
1160	dev_err(&bcm4377->pdev->dev,
1161	"send calibration chunk failed with %d\n", ret);
1162	return ret;
1163	}
1164	}
1165
1166	return 0;
1167	}
1168
1169	static int bcm4378_send_calibration(struct bcm4377_data *bcm4377)
1170	{
1171	if ((strcmp(bcm4377->stepping, "b1") == 0) ||
1172	strcmp(bcm4377->stepping, "b3") == 0)
1173	return __bcm4378_send_calibration(
1174	bcm4377, data: bcm4377->taurus_beamforming_cal_blob,
1175	data_size: bcm4377->taurus_beamforming_cal_size);
1176	else
1177	return __bcm4378_send_calibration(bcm4377,
1178	data: bcm4377->taurus_cal_blob,
1179	data_size: bcm4377->taurus_cal_size);
1180	}
1181
1182	static int bcm4387_send_calibration(struct bcm4377_data *bcm4377)
1183	{
1184	if (strcmp(bcm4377->stepping, "c2") == 0)
1185	return __bcm4378_send_calibration(
1186	bcm4377, data: bcm4377->taurus_beamforming_cal_blob,
1187	data_size: bcm4377->taurus_beamforming_cal_size);
1188	else
1189	return __bcm4378_send_calibration(bcm4377,
1190	data: bcm4377->taurus_cal_blob,
1191	data_size: bcm4377->taurus_cal_size);
1192	}
1193
1194	static const struct firmware *bcm4377_request_blob(struct bcm4377_data *bcm4377,
1195	const char *suffix)
1196	{
1197	const struct firmware *fw;
1198	char name0[64], name1[64];
1199	int ret;
1200
1201	snprintf(buf: name0, size: sizeof(name0), fmt: "brcm/brcmbt%04x%s-%s-%s.%s",
1202	bcm4377->hw->id, bcm4377->stepping, bcm4377->board_type,
1203	bcm4377->vendor, suffix);
1204	snprintf(buf: name1, size: sizeof(name1), fmt: "brcm/brcmbt%04x%s-%s.%s",
1205	bcm4377->hw->id, bcm4377->stepping, bcm4377->board_type,
1206	suffix);
1207	dev_dbg(&bcm4377->pdev->dev, "Trying to load firmware: '%s' or '%s'\n",
1208	name0, name1);
1209
1210	ret = firmware_request_nowarn(fw: &fw, name: name0, device: &bcm4377->pdev->dev);
1211	if (!ret)
1212	return fw;
1213	ret = firmware_request_nowarn(fw: &fw, name: name1, device: &bcm4377->pdev->dev);
1214	if (!ret)
1215	return fw;
1216
1217	dev_err(&bcm4377->pdev->dev,
1218	"Unable to load firmware; tried '%s' and '%s'\n", name0, name1);
1219	return NULL;
1220	}
1221
1222	static int bcm4377_send_ptb(struct bcm4377_data *bcm4377,
1223	const struct firmware *fw)
1224	{
1225	struct sk_buff *skb;
1226
1227	skb = __hci_cmd_sync(hdev: bcm4377->hdev, opcode: 0xfd98, plen: fw->size, param: fw->data,
1228	HCI_INIT_TIMEOUT);
1229	/*
1230	* This command seems to always fail on more recent firmware versions
1231	* (even in traces taken from the macOS driver). It's unclear why this
1232	* happens but because the PTB file contains calibration and/or
1233	* regulatory data and may be required on older firmware we still try to
1234	* send it here just in case and just ignore if it fails.
1235	*/
1236	if (!IS_ERR(ptr: skb))
1237	kfree_skb(skb);
1238	return 0;
1239	}
1240
1241	static int bcm4378_send_ptb_chunk(struct bcm4377_data *bcm4377,
1242	const void *data, size_t data_len,
1243	u16 blocks_left)
1244	{
1245	struct bcm4378_hci_send_ptb_cmd cmd;
1246	struct sk_buff *skb;
1247
1248	if (data_len > BCM4378_PTB_CHUNK_SIZE)
1249	return -EINVAL;
1250
1251	memset(&cmd, 0, sizeof(cmd));
1252	cmd.blocks_left = cpu_to_le16(blocks_left);
1253	memcpy(cmd.data, data, data_len);
1254
1255	skb = __hci_cmd_sync(hdev: bcm4377->hdev, opcode: 0xfe0d, plen: sizeof(cmd), param: &cmd,
1256	HCI_INIT_TIMEOUT);
1257	if (IS_ERR(ptr: skb))
1258	return PTR_ERR(ptr: skb);
1259
1260	kfree_skb(skb);
1261	return 0;
1262	}
1263
1264	static int bcm4378_send_ptb(struct bcm4377_data *bcm4377,
1265	const struct firmware *fw)
1266	{
1267	size_t chunks = DIV_ROUND_UP(fw->size, (size_t)BCM4378_PTB_CHUNK_SIZE);
1268	size_t i, left, transfer_len;
1269	int ret;
1270
1271	for (i = 0, left = fw->size; i < chunks; ++i, left -= transfer_len) {
1272	transfer_len = min_t(size_t, left, BCM4378_PTB_CHUNK_SIZE);
1273
1274	dev_dbg(&bcm4377->pdev->dev, "sending ptb chunk %zu/%zu\n",
1275	i + 1, chunks);
1276	ret = bcm4378_send_ptb_chunk(
1277	bcm4377, data: fw->data + i * BCM4378_PTB_CHUNK_SIZE,
1278	data_len: transfer_len, blocks_left: chunks - i - 1);
1279	if (ret) {
1280	dev_err(&bcm4377->pdev->dev,
1281	"sending ptb chunk %zu failed (%d)", i, ret);
1282	return ret;
1283	}
1284	}
1285
1286	return 0;
1287	}
1288
1289	static int bcm4377_hci_open(struct hci_dev *hdev)
1290	{
1291	struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1292	int ret;
1293
1294	dev_dbg(&bcm4377->pdev->dev, "creating rings\n");
1295
1296	ret = bcm4377_create_completion_ring(bcm4377,
1297	ring: &bcm4377->hci_acl_ack_ring);
1298	if (ret)
1299	return ret;
1300	ret = bcm4377_create_completion_ring(bcm4377,
1301	ring: &bcm4377->hci_acl_event_ring);
1302	if (ret)
1303	goto destroy_hci_acl_ack;
1304	ret = bcm4377_create_completion_ring(bcm4377, ring: &bcm4377->sco_ack_ring);
1305	if (ret)
1306	goto destroy_hci_acl_event;
1307	ret = bcm4377_create_completion_ring(bcm4377, ring: &bcm4377->sco_event_ring);
1308	if (ret)
1309	goto destroy_sco_ack;
1310	dev_dbg(&bcm4377->pdev->dev,
1311	"all completion rings successfully created!\n");
1312
1313	ret = bcm4377_create_transfer_ring(bcm4377, ring: &bcm4377->hci_h2d_ring);
1314	if (ret)
1315	goto destroy_sco_event;
1316	ret = bcm4377_create_transfer_ring(bcm4377, ring: &bcm4377->hci_d2h_ring);
1317	if (ret)
1318	goto destroy_hci_h2d;
1319	ret = bcm4377_create_transfer_ring(bcm4377, ring: &bcm4377->sco_h2d_ring);
1320	if (ret)
1321	goto destroy_hci_d2h;
1322	ret = bcm4377_create_transfer_ring(bcm4377, ring: &bcm4377->sco_d2h_ring);
1323	if (ret)
1324	goto destroy_sco_h2d;
1325	ret = bcm4377_create_transfer_ring(bcm4377, ring: &bcm4377->acl_h2d_ring);
1326	if (ret)
1327	goto destroy_sco_d2h;
1328	ret = bcm4377_create_transfer_ring(bcm4377, ring: &bcm4377->acl_d2h_ring);
1329	if (ret)
1330	goto destroy_acl_h2d;
1331	dev_dbg(&bcm4377->pdev->dev,
1332	"all transfer rings successfully created!\n");
1333
1334	return 0;
1335
1336	destroy_acl_h2d:
1337	bcm4377_destroy_transfer_ring(bcm4377, ring: &bcm4377->acl_h2d_ring);
1338	destroy_sco_d2h:
1339	bcm4377_destroy_transfer_ring(bcm4377, ring: &bcm4377->sco_d2h_ring);
1340	destroy_sco_h2d:
1341	bcm4377_destroy_transfer_ring(bcm4377, ring: &bcm4377->sco_h2d_ring);
1342	destroy_hci_d2h:
1343	bcm4377_destroy_transfer_ring(bcm4377, ring: &bcm4377->hci_h2d_ring);
1344	destroy_hci_h2d:
1345	bcm4377_destroy_transfer_ring(bcm4377, ring: &bcm4377->hci_d2h_ring);
1346	destroy_sco_event:
1347	bcm4377_destroy_completion_ring(bcm4377, ring: &bcm4377->sco_event_ring);
1348	destroy_sco_ack:
1349	bcm4377_destroy_completion_ring(bcm4377, ring: &bcm4377->sco_ack_ring);
1350	destroy_hci_acl_event:
1351	bcm4377_destroy_completion_ring(bcm4377, ring: &bcm4377->hci_acl_event_ring);
1352	destroy_hci_acl_ack:
1353	bcm4377_destroy_completion_ring(bcm4377, ring: &bcm4377->hci_acl_ack_ring);
1354
1355	dev_err(&bcm4377->pdev->dev, "Creating rings failed with %d\n", ret);
1356	return ret;
1357	}
1358
1359	static int bcm4377_hci_close(struct hci_dev *hdev)
1360	{
1361	struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1362
1363	dev_dbg(&bcm4377->pdev->dev, "destroying rings in hci_close\n");
1364
1365	bcm4377_destroy_transfer_ring(bcm4377, ring: &bcm4377->acl_d2h_ring);
1366	bcm4377_destroy_transfer_ring(bcm4377, ring: &bcm4377->acl_h2d_ring);
1367	bcm4377_destroy_transfer_ring(bcm4377, ring: &bcm4377->sco_d2h_ring);
1368	bcm4377_destroy_transfer_ring(bcm4377, ring: &bcm4377->sco_h2d_ring);
1369	bcm4377_destroy_transfer_ring(bcm4377, ring: &bcm4377->hci_d2h_ring);
1370	bcm4377_destroy_transfer_ring(bcm4377, ring: &bcm4377->hci_h2d_ring);
1371
1372	bcm4377_destroy_completion_ring(bcm4377, ring: &bcm4377->sco_event_ring);
1373	bcm4377_destroy_completion_ring(bcm4377, ring: &bcm4377->sco_ack_ring);
1374	bcm4377_destroy_completion_ring(bcm4377, ring: &bcm4377->hci_acl_event_ring);
1375	bcm4377_destroy_completion_ring(bcm4377, ring: &bcm4377->hci_acl_ack_ring);
1376
1377	return 0;
1378	}
1379
1380	static bool bcm4377_is_valid_bdaddr(struct bcm4377_data *bcm4377,
1381	bdaddr_t *addr)
1382	{
1383	if (addr->b[0] != 0x93)
1384	return true;
1385	if (addr->b[1] != 0x76)
1386	return true;
1387	if (addr->b[2] != 0x00)
1388	return true;
1389	if (addr->b[4] != (bcm4377->hw->id & 0xff))
1390	return true;
1391	if (addr->b[5] != (bcm4377->hw->id >> 8))
1392	return true;
1393	return false;
1394	}
1395
1396	static int bcm4377_check_bdaddr(struct bcm4377_data *bcm4377)
1397	{
1398	struct hci_rp_read_bd_addr *bda;
1399	struct sk_buff *skb;
1400
1401	skb = __hci_cmd_sync(hdev: bcm4377->hdev, HCI_OP_READ_BD_ADDR, plen: 0, NULL,
1402	HCI_INIT_TIMEOUT);
1403	if (IS_ERR(ptr: skb)) {
1404	int err = PTR_ERR(ptr: skb);
1405
1406	dev_err(&bcm4377->pdev->dev, "HCI_OP_READ_BD_ADDR failed (%d)",
1407	err);
1408	return err;
1409	}
1410
1411	if (skb->len != sizeof(*bda)) {
1412	dev_err(&bcm4377->pdev->dev,
1413	"HCI_OP_READ_BD_ADDR reply length invalid");
1414	kfree_skb(skb);
1415	return -EIO;
1416	}
1417
1418	bda = (struct hci_rp_read_bd_addr *)skb->data;
1419	if (!bcm4377_is_valid_bdaddr(bcm4377, addr: &bda->bdaddr))
1420	set_bit(nr: HCI_QUIRK_USE_BDADDR_PROPERTY, addr: &bcm4377->hdev->quirks);
1421
1422	kfree_skb(skb);
1423	return 0;
1424	}
1425
1426	static int bcm4377_hci_setup(struct hci_dev *hdev)
1427	{
1428	struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1429	const struct firmware *fw;
1430	int ret;
1431
1432	if (bcm4377->hw->send_calibration) {
1433	ret = bcm4377->hw->send_calibration(bcm4377);
1434	if (ret)
1435	return ret;
1436	}
1437
1438	fw = bcm4377_request_blob(bcm4377, suffix: "ptb");
1439	if (!fw) {
1440	dev_err(&bcm4377->pdev->dev, "failed to load PTB data");
1441	return -ENOENT;
1442	}
1443
1444	ret = bcm4377->hw->send_ptb(bcm4377, fw);
1445	release_firmware(fw);
1446	if (ret)
1447	return ret;
1448
1449	return bcm4377_check_bdaddr(bcm4377);
1450	}
1451
1452	static int bcm4377_hci_send_frame(struct hci_dev *hdev, struct sk_buff *skb)
1453	{
1454	struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1455	struct bcm4377_transfer_ring *ring;
1456	int ret;
1457
1458	switch (hci_skb_pkt_type(skb)) {
1459	case HCI_COMMAND_PKT:
1460	hdev->stat.cmd_tx++;
1461	ring = &bcm4377->hci_h2d_ring;
1462	break;
1463
1464	case HCI_ACLDATA_PKT:
1465	hdev->stat.acl_tx++;
1466	ring = &bcm4377->acl_h2d_ring;
1467	break;
1468
1469	case HCI_SCODATA_PKT:
1470	hdev->stat.sco_tx++;
1471	ring = &bcm4377->sco_h2d_ring;
1472	break;
1473
1474	default:
1475	return -EILSEQ;
1476	}
1477
1478	ret = bcm4377_enqueue(bcm4377, ring, data: skb->data, len: skb->len, wait: false);
1479	if (ret < 0) {
1480	hdev->stat.err_tx++;
1481	return ret;
1482	}
1483
1484	hdev->stat.byte_tx += skb->len;
1485	kfree_skb(skb);
1486	return ret;
1487	}
1488
1489	static int bcm4377_hci_set_bdaddr(struct hci_dev *hdev, const bdaddr_t *bdaddr)
1490	{
1491	struct bcm4377_data *bcm4377 = hci_get_drvdata(hdev);
1492	struct sk_buff *skb;
1493	int err;
1494
1495	skb = __hci_cmd_sync(hdev, opcode: 0xfc01, plen: 6, param: bdaddr, HCI_INIT_TIMEOUT);
1496	if (IS_ERR(ptr: skb)) {
1497	err = PTR_ERR(ptr: skb);
1498	dev_err(&bcm4377->pdev->dev,
1499	"Change address command failed (%d)", err);
1500	return err;
1501	}
1502	kfree_skb(skb);
1503
1504	return 0;
1505	}
1506
1507	static int bcm4377_alloc_transfer_ring(struct bcm4377_data *bcm4377,
1508	struct bcm4377_transfer_ring *ring)
1509	{
1510	size_t entry_size;
1511
1512	spin_lock_init(&ring->lock);
1513	ring->payload_size = ALIGN(ring->payload_size, 4);
1514	ring->mapped_payload_size = ALIGN(ring->mapped_payload_size, 4);
1515
1516	if (ring->payload_size > BCM4377_XFER_RING_MAX_INPLACE_PAYLOAD_SIZE)
1517	return -EINVAL;
1518	if (ring->n_entries > BCM4377_MAX_RING_SIZE)
1519	return -EINVAL;
1520	if (ring->virtual && ring->allow_wait)
1521	return -EINVAL;
1522
1523	if (ring->d2h_buffers_only) {
1524	if (ring->virtual)
1525	return -EINVAL;
1526	if (ring->payload_size)
1527	return -EINVAL;
1528	if (!ring->mapped_payload_size)
1529	return -EINVAL;
1530	}
1531	if (ring->virtual)
1532	return 0;
1533
1534	entry_size =
1535	ring->payload_size + sizeof(struct bcm4377_xfer_ring_entry);
1536	ring->ring = dmam_alloc_coherent(dev: &bcm4377->pdev->dev,
1537	size: ring->n_entries * entry_size,
1538	dma_handle: &ring->ring_dma, GFP_KERNEL);
1539	if (!ring->ring)
1540	return -ENOMEM;
1541
1542	if (ring->allow_wait) {
1543	ring->events = devm_kcalloc(dev: &bcm4377->pdev->dev,
1544	n: ring->n_entries,
1545	size: sizeof(*ring->events), GFP_KERNEL);
1546	if (!ring->events)
1547	return -ENOMEM;
1548	}
1549
1550	if (ring->mapped_payload_size) {
1551	ring->payloads = dmam_alloc_coherent(
1552	dev: &bcm4377->pdev->dev,
1553	size: ring->n_entries * ring->mapped_payload_size,
1554	dma_handle: &ring->payloads_dma, GFP_KERNEL);
1555	if (!ring->payloads)
1556	return -ENOMEM;
1557	}
1558
1559	return 0;
1560	}
1561
1562	static int bcm4377_alloc_completion_ring(struct bcm4377_data *bcm4377,
1563	struct bcm4377_completion_ring *ring)
1564	{
1565	size_t entry_size;
1566
1567	ring->payload_size = ALIGN(ring->payload_size, 4);
1568	if (ring->payload_size > BCM4377_XFER_RING_MAX_INPLACE_PAYLOAD_SIZE)
1569	return -EINVAL;
1570	if (ring->n_entries > BCM4377_MAX_RING_SIZE)
1571	return -EINVAL;
1572
1573	entry_size = ring->payload_size +
1574	sizeof(struct bcm4377_completion_ring_entry);
1575
1576	ring->ring = dmam_alloc_coherent(dev: &bcm4377->pdev->dev,
1577	size: ring->n_entries * entry_size,
1578	dma_handle: &ring->ring_dma, GFP_KERNEL);
1579	if (!ring->ring)
1580	return -ENOMEM;
1581	return 0;
1582	}
1583
1584	static int bcm4377_init_context(struct bcm4377_data *bcm4377)
1585	{
1586	struct device *dev = &bcm4377->pdev->dev;
1587	dma_addr_t peripheral_info_dma;
1588
1589	bcm4377->ctx = dmam_alloc_coherent(dev, size: sizeof(*bcm4377->ctx),
1590	dma_handle: &bcm4377->ctx_dma, GFP_KERNEL);
1591	if (!bcm4377->ctx)
1592	return -ENOMEM;
1593	memset(bcm4377->ctx, 0, sizeof(*bcm4377->ctx));
1594
1595	bcm4377->ring_state =
1596	dmam_alloc_coherent(dev, size: sizeof(*bcm4377->ring_state),
1597	dma_handle: &bcm4377->ring_state_dma, GFP_KERNEL);
1598	if (!bcm4377->ring_state)
1599	return -ENOMEM;
1600	memset(bcm4377->ring_state, 0, sizeof(*bcm4377->ring_state));
1601
1602	bcm4377->ctx->version = cpu_to_le16(1);
1603	bcm4377->ctx->size = cpu_to_le16(sizeof(*bcm4377->ctx));
1604	bcm4377->ctx->enabled_caps = cpu_to_le32(2);
1605
1606	/*
1607	* The BT device will write 0x20 bytes of data to this buffer but
1608	* the exact contents are unknown. It only needs to exist for BT
1609	* to work such that we can just allocate and then ignore it.
1610	*/
1611	if (!dmam_alloc_coherent(dev: &bcm4377->pdev->dev, size: 0x20,
1612	dma_handle: &peripheral_info_dma, GFP_KERNEL))
1613	return -ENOMEM;
1614	bcm4377->ctx->peripheral_info_addr = cpu_to_le64(peripheral_info_dma);
1615
1616	bcm4377->ctx->xfer_ring_heads_addr = cpu_to_le64(
1617	bcm4377->ring_state_dma +
1618	offsetof(struct bcm4377_ring_state, xfer_ring_head));
1619	bcm4377->ctx->xfer_ring_tails_addr = cpu_to_le64(
1620	bcm4377->ring_state_dma +
1621	offsetof(struct bcm4377_ring_state, xfer_ring_tail));
1622	bcm4377->ctx->completion_ring_heads_addr = cpu_to_le64(
1623	bcm4377->ring_state_dma +
1624	offsetof(struct bcm4377_ring_state, completion_ring_head));
1625	bcm4377->ctx->completion_ring_tails_addr = cpu_to_le64(
1626	bcm4377->ring_state_dma +
1627	offsetof(struct bcm4377_ring_state, completion_ring_tail));
1628
1629	bcm4377->ctx->n_completion_rings =
1630	cpu_to_le16(BCM4377_N_COMPLETION_RINGS);
1631	bcm4377->ctx->n_xfer_rings = cpu_to_le16(BCM4377_N_TRANSFER_RINGS);
1632
1633	bcm4377->ctx->control_completion_ring_addr =
1634	cpu_to_le64(bcm4377->control_ack_ring.ring_dma);
1635	bcm4377->ctx->control_completion_ring_n_entries =
1636	cpu_to_le16(bcm4377->control_ack_ring.n_entries);
1637	bcm4377->ctx->control_completion_ring_doorbell = cpu_to_le16(0xffff);
1638	bcm4377->ctx->control_completion_ring_msi = 0;
1639	bcm4377->ctx->control_completion_ring_header_size = 0;
1640	bcm4377->ctx->control_completion_ring_footer_size = 0;
1641
1642	bcm4377->ctx->control_xfer_ring_addr =
1643	cpu_to_le64(bcm4377->control_h2d_ring.ring_dma);
1644	bcm4377->ctx->control_xfer_ring_n_entries =
1645	cpu_to_le16(bcm4377->control_h2d_ring.n_entries);
1646	bcm4377->ctx->control_xfer_ring_doorbell =
1647	cpu_to_le16(bcm4377->control_h2d_ring.doorbell);
1648	bcm4377->ctx->control_xfer_ring_msi = 0;
1649	bcm4377->ctx->control_xfer_ring_header_size = 0;
1650	bcm4377->ctx->control_xfer_ring_footer_size =
1651	bcm4377->control_h2d_ring.payload_size / 4;
1652
1653	dev_dbg(&bcm4377->pdev->dev, "context initialized at IOVA %pad",
1654	&bcm4377->ctx_dma);
1655
1656	return 0;
1657	}
1658
1659	static int bcm4377_prepare_rings(struct bcm4377_data *bcm4377)
1660	{
1661	int ret;
1662
1663	/*
1664	* Even though many of these settings appear to be configurable
1665	* when sending the "create ring" messages most of these are
1666	* actually hardcoded in some (and quite possibly all) firmware versions
1667	* and changing them on the host has no effect.
1668	* Specifically, this applies to at least the doorbells, the transfer
1669	* and completion ring ids and their mapping (e.g. both HCI and ACL
1670	* entries will always be queued in completion rings 1 and 2 no matter
1671	* what we configure here).
1672	*/
1673	bcm4377->control_ack_ring.ring_id = BCM4377_ACK_RING_CONTROL;
1674	bcm4377->control_ack_ring.n_entries = 32;
1675	bcm4377->control_ack_ring.transfer_rings =
1676	BIT(BCM4377_XFER_RING_CONTROL);
1677
1678	bcm4377->hci_acl_ack_ring.ring_id = BCM4377_ACK_RING_HCI_ACL;
1679	bcm4377->hci_acl_ack_ring.n_entries = 2 * BCM4377_RING_N_ENTRIES;
1680	bcm4377->hci_acl_ack_ring.transfer_rings =
1681	BIT(BCM4377_XFER_RING_HCI_H2D) | BIT(BCM4377_XFER_RING_ACL_H2D);
1682	bcm4377->hci_acl_ack_ring.delay = 1000;
1683
1684	/*
1685	* A payload size of MAX_EVENT_PAYLOAD_SIZE is enough here since large
1686	* ACL packets will be transmitted inside buffers mapped via
1687	* acl_d2h_ring anyway.
1688	*/
1689	bcm4377->hci_acl_event_ring.ring_id = BCM4377_EVENT_RING_HCI_ACL;
1690	bcm4377->hci_acl_event_ring.payload_size = MAX_EVENT_PAYLOAD_SIZE;
1691	bcm4377->hci_acl_event_ring.n_entries = 2 * BCM4377_RING_N_ENTRIES;
1692	bcm4377->hci_acl_event_ring.transfer_rings =
1693	BIT(BCM4377_XFER_RING_HCI_D2H) | BIT(BCM4377_XFER_RING_ACL_D2H);
1694	bcm4377->hci_acl_event_ring.delay = 1000;
1695
1696	bcm4377->sco_ack_ring.ring_id = BCM4377_ACK_RING_SCO;
1697	bcm4377->sco_ack_ring.n_entries = BCM4377_RING_N_ENTRIES;
1698	bcm4377->sco_ack_ring.transfer_rings = BIT(BCM4377_XFER_RING_SCO_H2D);
1699
1700	bcm4377->sco_event_ring.ring_id = BCM4377_EVENT_RING_SCO;
1701	bcm4377->sco_event_ring.payload_size = MAX_SCO_PAYLOAD_SIZE;
1702	bcm4377->sco_event_ring.n_entries = BCM4377_RING_N_ENTRIES;
1703	bcm4377->sco_event_ring.transfer_rings = BIT(BCM4377_XFER_RING_SCO_D2H);
1704
1705	bcm4377->control_h2d_ring.ring_id = BCM4377_XFER_RING_CONTROL;
1706	bcm4377->control_h2d_ring.doorbell = BCM4377_DOORBELL_CONTROL;
1707	bcm4377->control_h2d_ring.payload_size = BCM4377_CONTROL_MSG_SIZE;
1708	bcm4377->control_h2d_ring.completion_ring = BCM4377_ACK_RING_CONTROL;
1709	bcm4377->control_h2d_ring.allow_wait = true;
1710	bcm4377->control_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1711
1712	bcm4377->hci_h2d_ring.ring_id = BCM4377_XFER_RING_HCI_H2D;
1713	bcm4377->hci_h2d_ring.doorbell = BCM4377_DOORBELL_HCI_H2D;
1714	bcm4377->hci_h2d_ring.payload_size = MAX_EVENT_PAYLOAD_SIZE;
1715	bcm4377->hci_h2d_ring.completion_ring = BCM4377_ACK_RING_HCI_ACL;
1716	bcm4377->hci_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1717
1718	bcm4377->hci_d2h_ring.ring_id = BCM4377_XFER_RING_HCI_D2H;
1719	bcm4377->hci_d2h_ring.doorbell = BCM4377_DOORBELL_HCI_D2H;
1720	bcm4377->hci_d2h_ring.completion_ring = BCM4377_EVENT_RING_HCI_ACL;
1721	bcm4377->hci_d2h_ring.virtual = true;
1722	bcm4377->hci_d2h_ring.n_entries = BCM4377_RING_N_ENTRIES;
1723
1724	bcm4377->sco_h2d_ring.ring_id = BCM4377_XFER_RING_SCO_H2D;
1725	bcm4377->sco_h2d_ring.doorbell = BCM4377_DOORBELL_SCO;
1726	bcm4377->sco_h2d_ring.payload_size = MAX_SCO_PAYLOAD_SIZE;
1727	bcm4377->sco_h2d_ring.completion_ring = BCM4377_ACK_RING_SCO;
1728	bcm4377->sco_h2d_ring.sync = true;
1729	bcm4377->sco_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1730
1731	bcm4377->sco_d2h_ring.ring_id = BCM4377_XFER_RING_SCO_D2H;
1732	bcm4377->sco_d2h_ring.doorbell = BCM4377_DOORBELL_SCO;
1733	bcm4377->sco_d2h_ring.completion_ring = BCM4377_EVENT_RING_SCO;
1734	bcm4377->sco_d2h_ring.virtual = true;
1735	bcm4377->sco_d2h_ring.sync = true;
1736	bcm4377->sco_d2h_ring.n_entries = BCM4377_RING_N_ENTRIES;
1737
1738	/*
1739	* This ring has to use mapped_payload_size because the largest ACL
1740	* packet doesn't fit inside the largest possible footer
1741	*/
1742	bcm4377->acl_h2d_ring.ring_id = BCM4377_XFER_RING_ACL_H2D;
1743	bcm4377->acl_h2d_ring.doorbell = BCM4377_DOORBELL_ACL_H2D;
1744	bcm4377->acl_h2d_ring.mapped_payload_size = MAX_ACL_PAYLOAD_SIZE;
1745	bcm4377->acl_h2d_ring.completion_ring = BCM4377_ACK_RING_HCI_ACL;
1746	bcm4377->acl_h2d_ring.n_entries = BCM4377_RING_N_ENTRIES;
1747
1748	/*
1749	* This ring only contains empty buffers to be used by incoming
1750	* ACL packets that do not fit inside the footer of hci_acl_event_ring
1751	*/
1752	bcm4377->acl_d2h_ring.ring_id = BCM4377_XFER_RING_ACL_D2H;
1753	bcm4377->acl_d2h_ring.doorbell = BCM4377_DOORBELL_ACL_D2H;
1754	bcm4377->acl_d2h_ring.completion_ring = BCM4377_EVENT_RING_HCI_ACL;
1755	bcm4377->acl_d2h_ring.d2h_buffers_only = true;
1756	bcm4377->acl_d2h_ring.mapped_payload_size = MAX_ACL_PAYLOAD_SIZE;
1757	bcm4377->acl_d2h_ring.n_entries = BCM4377_RING_N_ENTRIES;
1758
1759	/*
1760	* no need for any cleanup since this is only called from _probe
1761	* and only devres-managed allocations are used
1762	*/
1763	ret = bcm4377_alloc_transfer_ring(bcm4377, ring: &bcm4377->control_h2d_ring);
1764	if (ret)
1765	return ret;
1766	ret = bcm4377_alloc_transfer_ring(bcm4377, ring: &bcm4377->hci_h2d_ring);
1767	if (ret)
1768	return ret;
1769	ret = bcm4377_alloc_transfer_ring(bcm4377, ring: &bcm4377->hci_d2h_ring);
1770	if (ret)
1771	return ret;
1772	ret = bcm4377_alloc_transfer_ring(bcm4377, ring: &bcm4377->sco_h2d_ring);
1773	if (ret)
1774	return ret;
1775	ret = bcm4377_alloc_transfer_ring(bcm4377, ring: &bcm4377->sco_d2h_ring);
1776	if (ret)
1777	return ret;
1778	ret = bcm4377_alloc_transfer_ring(bcm4377, ring: &bcm4377->acl_h2d_ring);
1779	if (ret)
1780	return ret;
1781	ret = bcm4377_alloc_transfer_ring(bcm4377, ring: &bcm4377->acl_d2h_ring);
1782	if (ret)
1783	return ret;
1784
1785	ret = bcm4377_alloc_completion_ring(bcm4377,
1786	ring: &bcm4377->control_ack_ring);
1787	if (ret)
1788	return ret;
1789	ret = bcm4377_alloc_completion_ring(bcm4377,
1790	ring: &bcm4377->hci_acl_ack_ring);
1791	if (ret)
1792	return ret;
1793	ret = bcm4377_alloc_completion_ring(bcm4377,
1794	ring: &bcm4377->hci_acl_event_ring);
1795	if (ret)
1796	return ret;
1797	ret = bcm4377_alloc_completion_ring(bcm4377, ring: &bcm4377->sco_ack_ring);
1798	if (ret)
1799	return ret;
1800	ret = bcm4377_alloc_completion_ring(bcm4377, ring: &bcm4377->sco_event_ring);
1801	if (ret)
1802	return ret;
1803
1804	dev_dbg(&bcm4377->pdev->dev, "all rings allocated and prepared\n");
1805
1806	return 0;
1807	}
1808
1809	static int bcm4377_boot(struct bcm4377_data *bcm4377)
1810	{
1811	const struct firmware *fw;
1812	void *bfr;
1813	dma_addr_t fw_dma;
1814	int ret = 0;
1815	u32 bootstage, rti_status;
1816
1817	bootstage = ioread32(bcm4377->bar2 + BCM4377_BAR2_BOOTSTAGE);
1818	rti_status = ioread32(bcm4377->bar2 + BCM4377_BAR2_RTI_STATUS);
1819
1820	if (bootstage != 0) {
1821	dev_err(&bcm4377->pdev->dev, "bootstage is %d and not 0\n",
1822	bootstage);
1823	return -EINVAL;
1824	}
1825
1826	if (rti_status != 0) {
1827	dev_err(&bcm4377->pdev->dev, "RTI status is %d and not 0\n",
1828	rti_status);
1829	return -EINVAL;
1830	}
1831
1832	fw = bcm4377_request_blob(bcm4377, suffix: "bin");
1833	if (!fw) {
1834	dev_err(&bcm4377->pdev->dev, "Failed to load firmware\n");
1835	return -ENOENT;
1836	}
1837
1838	bfr = dma_alloc_coherent(dev: &bcm4377->pdev->dev, size: fw->size, dma_handle: &fw_dma,
1839	GFP_KERNEL);
1840	if (!bfr) {
1841	ret = -ENOMEM;
1842	goto out_release_fw;
1843	}
1844
1845	memcpy(bfr, fw->data, fw->size);
1846
1847	iowrite32(0, bcm4377->bar0 + BCM4377_BAR0_HOST_WINDOW_LO);
1848	iowrite32(0, bcm4377->bar0 + BCM4377_BAR0_HOST_WINDOW_HI);
1849	iowrite32(BCM4377_DMA_MASK,
1850	bcm4377->bar0 + BCM4377_BAR0_HOST_WINDOW_SIZE);
1851
1852	iowrite32(lower_32_bits(fw_dma), bcm4377->bar2 + BCM4377_BAR2_FW_LO);
1853	iowrite32(upper_32_bits(fw_dma), bcm4377->bar2 + BCM4377_BAR2_FW_HI);
1854	iowrite32(fw->size, bcm4377->bar2 + BCM4377_BAR2_FW_SIZE);
1855	iowrite32(0, bcm4377->bar0 + BCM4377_BAR0_FW_DOORBELL);
1856
1857	dev_dbg(&bcm4377->pdev->dev, "waiting for firmware to boot\n");
1858
1859	ret = wait_for_completion_interruptible_timeout(x: &bcm4377->event,
1860	BCM4377_TIMEOUT);
1861	if (ret == 0) {
1862	ret = -ETIMEDOUT;
1863	goto out_dma_free;
1864	} else if (ret < 0) {
1865	goto out_dma_free;
1866	}
1867
1868	if (bcm4377->bootstage != 2) {
1869	dev_err(&bcm4377->pdev->dev, "boostage %d != 2\n",
1870	bcm4377->bootstage);
1871	ret = -ENXIO;
1872	goto out_dma_free;
1873	}
1874
1875	dev_dbg(&bcm4377->pdev->dev, "firmware has booted (stage = %x)\n",
1876	bcm4377->bootstage);
1877	ret = 0;
1878
1879	out_dma_free:
1880	dma_free_coherent(dev: &bcm4377->pdev->dev, size: fw->size, cpu_addr: bfr, dma_handle: fw_dma);
1881	out_release_fw:
1882	release_firmware(fw);
1883	return ret;
1884	}
1885
1886	static int bcm4377_setup_rti(struct bcm4377_data *bcm4377)
1887	{
1888	int ret;
1889
1890	dev_dbg(&bcm4377->pdev->dev, "starting RTI\n");
1891	iowrite32(1, bcm4377->bar0 + BCM4377_BAR0_RTI_CONTROL);
1892
1893	ret = wait_for_completion_interruptible_timeout(x: &bcm4377->event,
1894	BCM4377_TIMEOUT);
1895	if (ret == 0) {
1896	dev_err(&bcm4377->pdev->dev,
1897	"timed out while waiting for RTI to transition to state 1");
1898	return -ETIMEDOUT;
1899	} else if (ret < 0) {
1900	return ret;
1901	}
1902
1903	if (bcm4377->rti_status != 1) {
1904	dev_err(&bcm4377->pdev->dev, "RTI did not ack state 1 (%d)\n",
1905	bcm4377->rti_status);
1906	return -ENODEV;
1907	}
1908	dev_dbg(&bcm4377->pdev->dev, "RTI is in state 1\n");
1909
1910	/* allow access to the entire IOVA space again */
1911	iowrite32(0, bcm4377->bar2 + BCM4377_BAR2_RTI_WINDOW_LO);
1912	iowrite32(0, bcm4377->bar2 + BCM4377_BAR2_RTI_WINDOW_HI);
1913	iowrite32(BCM4377_DMA_MASK,
1914	bcm4377->bar2 + BCM4377_BAR2_RTI_WINDOW_SIZE);
1915
1916	/* setup "Converged IPC" context */
1917	iowrite32(lower_32_bits(bcm4377->ctx_dma),
1918	bcm4377->bar2 + BCM4377_BAR2_CONTEXT_ADDR_LO);
1919	iowrite32(upper_32_bits(bcm4377->ctx_dma),
1920	bcm4377->bar2 + BCM4377_BAR2_CONTEXT_ADDR_HI);
1921	iowrite32(2, bcm4377->bar0 + BCM4377_BAR0_RTI_CONTROL);
1922
1923	ret = wait_for_completion_interruptible_timeout(x: &bcm4377->event,
1924	BCM4377_TIMEOUT);
1925	if (ret == 0) {
1926	dev_err(&bcm4377->pdev->dev,
1927	"timed out while waiting for RTI to transition to state 2");
1928	return -ETIMEDOUT;
1929	} else if (ret < 0) {
1930	return ret;
1931	}
1932
1933	if (bcm4377->rti_status != 2) {
1934	dev_err(&bcm4377->pdev->dev, "RTI did not ack state 2 (%d)\n",
1935	bcm4377->rti_status);
1936	return -ENODEV;
1937	}
1938
1939	dev_dbg(&bcm4377->pdev->dev,
1940	"RTI is in state 2; control ring is ready\n");
1941	bcm4377->control_ack_ring.enabled = true;
1942
1943	return 0;
1944	}
1945
1946	static int bcm4377_parse_otp_board_params(struct bcm4377_data *bcm4377,
1947	char tag, const char *val, size_t len)
1948	{
1949	if (tag != 'V')
1950	return 0;
1951	if (len >= sizeof(bcm4377->vendor))
1952	return -EINVAL;
1953
1954	strscpy(bcm4377->vendor, val, len + 1);
1955	return 0;
1956	}
1957
1958	static int bcm4377_parse_otp_chip_params(struct bcm4377_data *bcm4377, char tag,
1959	const char *val, size_t len)
1960	{
1961	size_t idx = 0;
1962
1963	if (tag != 's')
1964	return 0;
1965	if (len >= sizeof(bcm4377->stepping))
1966	return -EINVAL;
1967
1968	while (len != 0) {
1969	bcm4377->stepping[idx] = tolower(val[idx]);
1970	if (val[idx] == '\0')
1971	return 0;
1972
1973	idx++;
1974	len--;
1975	}
1976
1977	bcm4377->stepping[idx] = '\0';
1978	return 0;
1979	}
1980
1981	static int bcm4377_parse_otp_str(struct bcm4377_data *bcm4377, const u8 *str,
1982	enum bcm4377_otp_params_type type)
1983	{
1984	const char *p;
1985	int ret;
1986
1987	p = skip_spaces(str);
1988	while (*p) {
1989	char tag = *p++;
1990	const char *end;
1991	size_t len;
1992
1993	if (*p++ != '=') /* implicit NUL check */
1994	return -EINVAL;
1995
1996	/* *p might be NUL here, if so end == p and len == 0 */
1997	end = strchrnul(p, ' ');
1998	len = end - p;
1999
2000	/* leave 1 byte for NUL in destination string */
2001	if (len > (BCM4377_OTP_MAX_PARAM_LEN - 1))
2002	return -EINVAL;
2003
2004	switch (type) {
2005	case BCM4377_OTP_BOARD_PARAMS:
2006	ret = bcm4377_parse_otp_board_params(bcm4377, tag, val: p,
2007	len);
2008	break;
2009	case BCM4377_OTP_CHIP_PARAMS:
2010	ret = bcm4377_parse_otp_chip_params(bcm4377, tag, val: p,
2011	len);
2012	break;
2013	default:
2014	ret = -EINVAL;
2015	break;
2016	}
2017
2018	if (ret)
2019	return ret;
2020
2021	/* Skip to next arg, if any */
2022	p = skip_spaces(end);
2023	}
2024
2025	return 0;
2026	}
2027
2028	static int bcm4377_parse_otp_sys_vendor(struct bcm4377_data *bcm4377, u8 *otp,
2029	size_t size)
2030	{
2031	int idx = 4;
2032	const char *chip_params;
2033	const char *board_params;
2034	int ret;
2035
2036	/* 4-byte header and two empty strings */
2037	if (size < 6)
2038	return -EINVAL;
2039
2040	if (get_unaligned_le32(p: otp) != BCM4377_OTP_VENDOR_HDR)
2041	return -EINVAL;
2042
2043	chip_params = &otp[idx];
2044
2045	/* Skip first string, including terminator */
2046	idx += strnlen(p: chip_params, maxlen: size - idx) + 1;
2047	if (idx >= size)
2048	return -EINVAL;
2049
2050	board_params = &otp[idx];
2051
2052	/* Skip to terminator of second string */
2053	idx += strnlen(p: board_params, maxlen: size - idx);
2054	if (idx >= size)
2055	return -EINVAL;
2056
2057	/* At this point both strings are guaranteed NUL-terminated */
2058	dev_dbg(&bcm4377->pdev->dev,
2059	"OTP: chip_params='%s' board_params='%s'\n", chip_params,
2060	board_params);
2061
2062	ret = bcm4377_parse_otp_str(bcm4377, str: chip_params,
2063	type: BCM4377_OTP_CHIP_PARAMS);
2064	if (ret)
2065	return ret;
2066
2067	ret = bcm4377_parse_otp_str(bcm4377, str: board_params,
2068	type: BCM4377_OTP_BOARD_PARAMS);
2069	if (ret)
2070	return ret;
2071
2072	if (!bcm4377->stepping[0] || !bcm4377->vendor[0])
2073	return -EINVAL;
2074
2075	dev_dbg(&bcm4377->pdev->dev, "OTP: stepping=%s, vendor=%s\n",
2076	bcm4377->stepping, bcm4377->vendor);
2077	return 0;
2078	}
2079
2080	static int bcm4377_parse_otp(struct bcm4377_data *bcm4377)
2081	{
2082	u8 *otp;
2083	int i;
2084	int ret = -ENOENT;
2085
2086	otp = kzalloc(BCM4377_OTP_SIZE, GFP_KERNEL);
2087	if (!otp)
2088	return -ENOMEM;
2089
2090	for (i = 0; i < BCM4377_OTP_SIZE; ++i)
2091	otp[i] = ioread8(bcm4377->bar0 + bcm4377->hw->otp_offset + i);
2092
2093	i = 0;
2094	while (i < (BCM4377_OTP_SIZE - 1)) {
2095	u8 type = otp[i];
2096	u8 length = otp[i + 1];
2097
2098	if (type == 0)
2099	break;
2100
2101	if ((i + 2 + length) > BCM4377_OTP_SIZE)
2102	break;
2103
2104	switch (type) {
2105	case BCM4377_OTP_SYS_VENDOR:
2106	dev_dbg(&bcm4377->pdev->dev,
2107	"OTP @ 0x%x (%d): SYS_VENDOR", i, length);
2108	ret = bcm4377_parse_otp_sys_vendor(bcm4377, otp: &otp[i + 2],
2109	size: length);
2110	break;
2111	case BCM4377_OTP_CIS:
2112	dev_dbg(&bcm4377->pdev->dev, "OTP @ 0x%x (%d): CIS", i,
2113	length);
2114	break;
2115	default:
2116	dev_dbg(&bcm4377->pdev->dev, "OTP @ 0x%x (%d): unknown",
2117	i, length);
2118	break;
2119	}
2120
2121	i += 2 + length;
2122	}
2123
2124	kfree(objp: otp);
2125	return ret;
2126	}
2127
2128	static int bcm4377_init_cfg(struct bcm4377_data *bcm4377)
2129	{
2130	int ret;
2131	u32 ctrl;
2132
2133	ret = pci_write_config_dword(dev: bcm4377->pdev,
2134	BCM4377_PCIECFG_BAR0_WINDOW1,
2135	val: bcm4377->hw->bar0_window1);
2136	if (ret)
2137	return ret;
2138
2139	ret = pci_write_config_dword(dev: bcm4377->pdev,
2140	BCM4377_PCIECFG_BAR0_WINDOW2,
2141	val: bcm4377->hw->bar0_window2);
2142	if (ret)
2143	return ret;
2144
2145	ret = pci_write_config_dword(
2146	dev: bcm4377->pdev, BCM4377_PCIECFG_BAR0_CORE2_WINDOW1,
2147	BCM4377_PCIECFG_BAR0_CORE2_WINDOW1_DEFAULT);
2148	if (ret)
2149	return ret;
2150
2151	if (bcm4377->hw->has_bar0_core2_window2) {
2152	ret = pci_write_config_dword(dev: bcm4377->pdev,
2153	BCM4377_PCIECFG_BAR0_CORE2_WINDOW2,
2154	val: bcm4377->hw->bar0_core2_window2);
2155	if (ret)
2156	return ret;
2157	}
2158
2159	ret = pci_write_config_dword(dev: bcm4377->pdev, BCM4377_PCIECFG_BAR2_WINDOW,
2160	BCM4377_PCIECFG_BAR2_WINDOW_DEFAULT);
2161	if (ret)
2162	return ret;
2163
2164	ret = pci_read_config_dword(dev: bcm4377->pdev,
2165	BCM4377_PCIECFG_SUBSYSTEM_CTRL, val: &ctrl);
2166	if (ret)
2167	return ret;
2168
2169	if (bcm4377->hw->clear_pciecfg_subsystem_ctrl_bit19)
2170	ctrl &= ~BIT(19);
2171	ctrl |= BIT(16);
2172
2173	return pci_write_config_dword(dev: bcm4377->pdev,
2174	BCM4377_PCIECFG_SUBSYSTEM_CTRL, val: ctrl);
2175	}
2176
2177	static int bcm4377_probe_dmi(struct bcm4377_data *bcm4377)
2178	{
2179	const struct dmi_system_id *board_type_dmi_id;
2180
2181	board_type_dmi_id = dmi_first_match(list: bcm4377_dmi_board_table);
2182	if (board_type_dmi_id && board_type_dmi_id->driver_data) {
2183	bcm4377->board_type = board_type_dmi_id->driver_data;
2184	dev_dbg(&bcm4377->pdev->dev,
2185	"found board type via DMI match: %s\n",
2186	bcm4377->board_type);
2187	}
2188
2189	return 0;
2190	}
2191
2192	static int bcm4377_probe_of(struct bcm4377_data *bcm4377)
2193	{
2194	struct device_node *np = bcm4377->pdev->dev.of_node;
2195	int ret;
2196
2197	if (!np)
2198	return 0;
2199
2200	ret = of_property_read_string(np, propname: "brcm,board-type",
2201	out_string: &bcm4377->board_type);
2202	if (ret) {
2203	dev_err(&bcm4377->pdev->dev, "no brcm,board-type property\n");
2204	return ret;
2205	}
2206
2207	bcm4377->taurus_beamforming_cal_blob =
2208	of_get_property(node: np, name: "brcm,taurus-bf-cal-blob",
2209	lenp: &bcm4377->taurus_beamforming_cal_size);
2210	if (!bcm4377->taurus_beamforming_cal_blob) {
2211	dev_err(&bcm4377->pdev->dev,
2212	"no brcm,taurus-bf-cal-blob property\n");
2213	return -ENOENT;
2214	}
2215	bcm4377->taurus_cal_blob = of_get_property(node: np, name: "brcm,taurus-cal-blob",
2216	lenp: &bcm4377->taurus_cal_size);
2217	if (!bcm4377->taurus_cal_blob) {
2218	dev_err(&bcm4377->pdev->dev,
2219	"no brcm,taurus-cal-blob property\n");
2220	return -ENOENT;
2221	}
2222
2223	return 0;
2224	}
2225
2226	static void bcm4377_disable_aspm(struct bcm4377_data *bcm4377)
2227	{
2228	pci_disable_link_state(pdev: bcm4377->pdev,
2229	PCIE_LINK_STATE_L0S | PCIE_LINK_STATE_L1);
2230
2231	/*
2232	* pci_disable_link_state can fail if either CONFIG_PCIEASPM is disabled
2233	* or if the BIOS hasn't handed over control to us. We must *always*
2234	* disable ASPM for this device due to hardware errata though.
2235	*/
2236	pcie_capability_clear_word(dev: bcm4377->pdev, PCI_EXP_LNKCTL,
2237	PCI_EXP_LNKCTL_ASPMC);
2238	}
2239
2240	static void bcm4377_pci_free_irq_vectors(void *data)
2241	{
2242	pci_free_irq_vectors(dev: data);
2243	}
2244
2245	static void bcm4377_hci_free_dev(void *data)
2246	{
2247	hci_free_dev(hdev: data);
2248	}
2249
2250	static void bcm4377_hci_unregister_dev(void *data)
2251	{
2252	hci_unregister_dev(hdev: data);
2253	}
2254
2255	static int bcm4377_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2256	{
2257	struct bcm4377_data *bcm4377;
2258	struct hci_dev *hdev;
2259	int ret, irq;
2260
2261	ret = dma_set_mask_and_coherent(dev: &pdev->dev, BCM4377_DMA_MASK);
2262	if (ret)
2263	return ret;
2264
2265	bcm4377 = devm_kzalloc(dev: &pdev->dev, size: sizeof(*bcm4377), GFP_KERNEL);
2266	if (!bcm4377)
2267	return -ENOMEM;
2268
2269	bcm4377->pdev = pdev;
2270	bcm4377->hw = &bcm4377_hw_variants[id->driver_data];
2271	init_completion(x: &bcm4377->event);
2272
2273	ret = bcm4377_prepare_rings(bcm4377);
2274	if (ret)
2275	return ret;
2276
2277	ret = bcm4377_init_context(bcm4377);
2278	if (ret)
2279	return ret;
2280
2281	ret = bcm4377_probe_dmi(bcm4377);
2282	if (ret)
2283	return ret;
2284	ret = bcm4377_probe_of(bcm4377);
2285	if (ret)
2286	return ret;
2287	if (!bcm4377->board_type) {
2288	dev_err(&pdev->dev, "unable to determine board type\n");
2289	return -ENODEV;
2290	}
2291
2292	if (bcm4377->hw->disable_aspm)
2293	bcm4377_disable_aspm(bcm4377);
2294
2295	ret = pci_reset_function_locked(dev: pdev);
2296	if (ret)
2297	dev_warn(
2298	&pdev->dev,
2299	"function level reset failed with %d; trying to continue anyway\n",
2300	ret);
2301
2302	/*
2303	* If this number is too low and we try to access any BAR too
2304	* early the device will crash. Experiments have shown that
2305	* approximately 50 msec is the minimum amount we have to wait.
2306	* Let's double that to be safe.
2307	*/
2308	msleep(msecs: 100);
2309
2310	ret = pcim_enable_device(pdev);
2311	if (ret)
2312	return ret;
2313	pci_set_master(dev: pdev);
2314
2315	ret = bcm4377_init_cfg(bcm4377);
2316	if (ret)
2317	return ret;
2318
2319	bcm4377->bar0 = pcim_iomap(pdev, bar: 0, maxlen: 0);
2320	if (!bcm4377->bar0)
2321	return -EBUSY;
2322	bcm4377->bar2 = pcim_iomap(pdev, bar: 2, maxlen: 0);
2323	if (!bcm4377->bar2)
2324	return -EBUSY;
2325
2326	ret = bcm4377_parse_otp(bcm4377);
2327	if (ret) {
2328	dev_err(&pdev->dev, "Reading OTP failed with %d\n", ret);
2329	return ret;
2330	}
2331
2332	/*
2333	* Legacy interrupts result in an IRQ storm because we don't know where
2334	* the interrupt mask and status registers for these chips are.
2335	* MSIs are acked automatically instead.
2336	*/
2337	ret = pci_alloc_irq_vectors(dev: pdev, min_vecs: 1, max_vecs: 1, PCI_IRQ_MSI);
2338	if (ret < 0)
2339	return -ENODEV;
2340	ret = devm_add_action_or_reset(&pdev->dev, bcm4377_pci_free_irq_vectors,
2341	pdev);
2342	if (ret)
2343	return ret;
2344
2345	irq = pci_irq_vector(dev: pdev, nr: 0);
2346	if (irq <= 0)
2347	return -ENODEV;
2348
2349	ret = devm_request_irq(dev: &pdev->dev, irq, handler: bcm4377_irq, irqflags: 0, devname: "bcm4377",
2350	dev_id: bcm4377);
2351	if (ret)
2352	return ret;
2353
2354	hdev = hci_alloc_dev();
2355	if (!hdev)
2356	return -ENOMEM;
2357	ret = devm_add_action_or_reset(&pdev->dev, bcm4377_hci_free_dev, hdev);
2358	if (ret)
2359	return ret;
2360
2361	bcm4377->hdev = hdev;
2362
2363	hdev->bus = HCI_PCI;
2364	hdev->dev_type = HCI_PRIMARY;
2365	hdev->open = bcm4377_hci_open;
2366	hdev->close = bcm4377_hci_close;
2367	hdev->send = bcm4377_hci_send_frame;
2368	hdev->set_bdaddr = bcm4377_hci_set_bdaddr;
2369	hdev->setup = bcm4377_hci_setup;
2370
2371	if (bcm4377->hw->broken_mws_transport_config)
2372	set_bit(nr: HCI_QUIRK_BROKEN_MWS_TRANSPORT_CONFIG, addr: &hdev->quirks);
2373	if (bcm4377->hw->broken_ext_scan)
2374	set_bit(nr: HCI_QUIRK_BROKEN_EXT_SCAN, addr: &hdev->quirks);
2375	if (bcm4377->hw->broken_le_coded)
2376	set_bit(nr: HCI_QUIRK_BROKEN_LE_CODED, addr: &hdev->quirks);
2377
2378	pci_set_drvdata(pdev, data: bcm4377);
2379	hci_set_drvdata(hdev, data: bcm4377);
2380	SET_HCIDEV_DEV(hdev, &pdev->dev);
2381
2382	ret = bcm4377_boot(bcm4377);
2383	if (ret)
2384	return ret;
2385
2386	ret = bcm4377_setup_rti(bcm4377);
2387	if (ret)
2388	return ret;
2389
2390	ret = hci_register_dev(hdev);
2391	if (ret)
2392	return ret;
2393	return devm_add_action_or_reset(&pdev->dev, bcm4377_hci_unregister_dev,
2394	hdev);
2395	}
2396
2397	static int bcm4377_suspend(struct pci_dev *pdev, pm_message_t state)
2398	{
2399	struct bcm4377_data *bcm4377 = pci_get_drvdata(pdev);
2400	int ret;
2401
2402	ret = hci_suspend_dev(hdev: bcm4377->hdev);
2403	if (ret)
2404	return ret;
2405
2406	iowrite32(BCM4377_BAR0_SLEEP_CONTROL_QUIESCE,
2407	bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL);
2408
2409	return 0;
2410	}
2411
2412	static int bcm4377_resume(struct pci_dev *pdev)
2413	{
2414	struct bcm4377_data *bcm4377 = pci_get_drvdata(pdev);
2415
2416	iowrite32(BCM4377_BAR0_SLEEP_CONTROL_UNQUIESCE,
2417	bcm4377->bar0 + BCM4377_BAR0_SLEEP_CONTROL);
2418
2419	return hci_resume_dev(hdev: bcm4377->hdev);
2420	}
2421
2422	static const struct dmi_system_id bcm4377_dmi_board_table[] = {
2423	{
2424	.matches = {
2425	DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
2426	DMI_MATCH(DMI_PRODUCT_NAME, "MacBookAir9,1"),
2427	},
2428	.driver_data = "apple,formosa",
2429	},
2430	{
2431	.matches = {
2432	DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
2433	DMI_MATCH(DMI_PRODUCT_NAME, "MacBookPro15,4"),
2434	},
2435	.driver_data = "apple,formosa",
2436	},
2437	{
2438	.matches = {
2439	DMI_MATCH(DMI_BOARD_VENDOR, "Apple Inc."),
2440	DMI_MATCH(DMI_PRODUCT_NAME, "MacBookPro16,3"),
2441	},
2442	.driver_data = "apple,formosa",
2443	},
2444	{}
2445	};
2446
2447	static const struct bcm4377_hw bcm4377_hw_variants[] = {
2448	[BCM4377] = {
2449	.id = 0x4377,
2450	.otp_offset = 0x4120,
2451	.bar0_window1 = 0x1800b000,
2452	.bar0_window2 = 0x1810c000,
2453	.disable_aspm = true,
2454	.broken_ext_scan = true,
2455	.send_ptb = bcm4377_send_ptb,
2456	},
2457
2458	[BCM4378] = {
2459	.id = 0x4378,
2460	.otp_offset = 0x4120,
2461	.bar0_window1 = 0x18002000,
2462	.bar0_window2 = 0x1810a000,
2463	.bar0_core2_window2 = 0x18107000,
2464	.has_bar0_core2_window2 = true,
2465	.broken_mws_transport_config = true,
2466	.broken_le_coded = true,
2467	.send_calibration = bcm4378_send_calibration,
2468	.send_ptb = bcm4378_send_ptb,
2469	},
2470
2471	[BCM4387] = {
2472	.id = 0x4387,
2473	.otp_offset = 0x413c,
2474	.bar0_window1 = 0x18002000,
2475	.bar0_window2 = 0x18109000,
2476	.bar0_core2_window2 = 0x18106000,
2477	.has_bar0_core2_window2 = true,
2478	.clear_pciecfg_subsystem_ctrl_bit19 = true,
2479	.broken_mws_transport_config = true,
2480	.broken_le_coded = true,
2481	.send_calibration = bcm4387_send_calibration,
2482	.send_ptb = bcm4378_send_ptb,
2483	},
2484	};
2485
2486	#define BCM4377_DEVID_ENTRY(id) \
2487	{ \
2488	PCI_VENDOR_ID_BROADCOM, BCM##id##_DEVICE_ID, PCI_ANY_ID, \
2489	PCI_ANY_ID, PCI_CLASS_NETWORK_OTHER << 8, 0xffff00, \
2490	BCM##id \
2491	}
2492
2493	static const struct pci_device_id bcm4377_devid_table[] = {
2494	BCM4377_DEVID_ENTRY(4377),
2495	BCM4377_DEVID_ENTRY(4378),
2496	BCM4377_DEVID_ENTRY(4387),
2497	{},
2498	};
2499	MODULE_DEVICE_TABLE(pci, bcm4377_devid_table);
2500
2501	static struct pci_driver bcm4377_pci_driver = {
2502	.name = "hci_bcm4377",
2503	.id_table = bcm4377_devid_table,
2504	.probe = bcm4377_probe,
2505	.suspend = bcm4377_suspend,
2506	.resume = bcm4377_resume,
2507	};
2508	module_pci_driver(bcm4377_pci_driver);
2509
2510	MODULE_AUTHOR("Sven Peter <sven@svenpeter.dev>");
2511	MODULE_DESCRIPTION("Bluetooth support for Broadcom 4377/4378/4387 devices");
2512	MODULE_LICENSE("Dual MIT/GPL");
2513	MODULE_FIRMWARE("brcm/brcmbt4377*.bin");
2514	MODULE_FIRMWARE("brcm/brcmbt4377*.ptb");
2515	MODULE_FIRMWARE("brcm/brcmbt4378*.bin");
2516	MODULE_FIRMWARE("brcm/brcmbt4378*.ptb");
2517	MODULE_FIRMWARE("brcm/brcmbt4387*.bin");
2518	MODULE_FIRMWARE("brcm/brcmbt4387*.ptb");
2519