1	// SPDX-License-Identifier: GPL-2.0
2	/* Copyright (C) 2021, Intel Corporation. */
3
4	#include "ice.h"
5	#include "ice_lib.h"
6	#include "ice_trace.h"
7
8	#define E810_OUT_PROP_DELAY_NS 1
9
10	#define UNKNOWN_INCVAL_E82X 0x100000000ULL
11
12	static const struct ptp_pin_desc ice_pin_desc_e810t[] = {
13	/* name idx func chan */
14	{ "GNSS", GNSS, PTP_PF_EXTTS, 0, { 0, } },
15	{ "SMA1", SMA1, PTP_PF_NONE, 1, { 0, } },
16	{ "U.FL1", UFL1, PTP_PF_NONE, 1, { 0, } },
17	{ "SMA2", SMA2, PTP_PF_NONE, 2, { 0, } },
18	{ "U.FL2", UFL2, PTP_PF_NONE, 2, { 0, } },
19	};
20
21	/**
22	* ice_get_sma_config_e810t
23	* @hw: pointer to the hw struct
24	* @ptp_pins: pointer to the ptp_pin_desc struture
25	*
26	* Read the configuration of the SMA control logic and put it into the
27	* ptp_pin_desc structure
28	*/
29	static int
30	ice_get_sma_config_e810t(struct ice_hw *hw, struct ptp_pin_desc *ptp_pins)
31	{
32	u8 data, i;
33	int status;
34
35	/* Read initial pin state */
36	status = ice_read_sma_ctrl_e810t(hw, data: &data);
37	if (status)
38	return status;
39
40	/* initialize with defaults */
41	for (i = 0; i < NUM_PTP_PINS_E810T; i++) {
42	strscpy(ptp_pins[i].name, ice_pin_desc_e810t[i].name,
43	sizeof(ptp_pins[i].name));
44	ptp_pins[i].index = ice_pin_desc_e810t[i].index;
45	ptp_pins[i].func = ice_pin_desc_e810t[i].func;
46	ptp_pins[i].chan = ice_pin_desc_e810t[i].chan;
47	}
48
49	/* Parse SMA1/UFL1 */
50	switch (data & ICE_SMA1_MASK_E810T) {
51	case ICE_SMA1_MASK_E810T:
52	default:
53	ptp_pins[SMA1].func = PTP_PF_NONE;
54	ptp_pins[UFL1].func = PTP_PF_NONE;
55	break;
56	case ICE_SMA1_DIR_EN_E810T:
57	ptp_pins[SMA1].func = PTP_PF_PEROUT;
58	ptp_pins[UFL1].func = PTP_PF_NONE;
59	break;
60	case ICE_SMA1_TX_EN_E810T:
61	ptp_pins[SMA1].func = PTP_PF_EXTTS;
62	ptp_pins[UFL1].func = PTP_PF_NONE;
63	break;
64	case 0:
65	ptp_pins[SMA1].func = PTP_PF_EXTTS;
66	ptp_pins[UFL1].func = PTP_PF_PEROUT;
67	break;
68	}
69
70	/* Parse SMA2/UFL2 */
71	switch (data & ICE_SMA2_MASK_E810T) {
72	case ICE_SMA2_MASK_E810T:
73	default:
74	ptp_pins[SMA2].func = PTP_PF_NONE;
75	ptp_pins[UFL2].func = PTP_PF_NONE;
76	break;
77	case (ICE_SMA2_TX_EN_E810T | ICE_SMA2_UFL2_RX_DIS_E810T):
78	ptp_pins[SMA2].func = PTP_PF_EXTTS;
79	ptp_pins[UFL2].func = PTP_PF_NONE;
80	break;
81	case (ICE_SMA2_DIR_EN_E810T | ICE_SMA2_UFL2_RX_DIS_E810T):
82	ptp_pins[SMA2].func = PTP_PF_PEROUT;
83	ptp_pins[UFL2].func = PTP_PF_NONE;
84	break;
85	case (ICE_SMA2_DIR_EN_E810T | ICE_SMA2_TX_EN_E810T):
86	ptp_pins[SMA2].func = PTP_PF_NONE;
87	ptp_pins[UFL2].func = PTP_PF_EXTTS;
88	break;
89	case ICE_SMA2_DIR_EN_E810T:
90	ptp_pins[SMA2].func = PTP_PF_PEROUT;
91	ptp_pins[UFL2].func = PTP_PF_EXTTS;
92	break;
93	}
94
95	return 0;
96	}
97
98	/**
99	* ice_ptp_set_sma_config_e810t
100	* @hw: pointer to the hw struct
101	* @ptp_pins: pointer to the ptp_pin_desc struture
102	*
103	* Set the configuration of the SMA control logic based on the configuration in
104	* num_pins parameter
105	*/
106	static int
107	ice_ptp_set_sma_config_e810t(struct ice_hw *hw,
108	const struct ptp_pin_desc *ptp_pins)
109	{
110	int status;
111	u8 data;
112
113	/* SMA1 and UFL1 cannot be set to TX at the same time */
114	if (ptp_pins[SMA1].func == PTP_PF_PEROUT &&
115	ptp_pins[UFL1].func == PTP_PF_PEROUT)
116	return -EINVAL;
117
118	/* SMA2 and UFL2 cannot be set to RX at the same time */
119	if (ptp_pins[SMA2].func == PTP_PF_EXTTS &&
120	ptp_pins[UFL2].func == PTP_PF_EXTTS)
121	return -EINVAL;
122
123	/* Read initial pin state value */
124	status = ice_read_sma_ctrl_e810t(hw, data: &data);
125	if (status)
126	return status;
127
128	/* Set the right sate based on the desired configuration */
129	data &= ~ICE_SMA1_MASK_E810T;
130	if (ptp_pins[SMA1].func == PTP_PF_NONE &&
131	ptp_pins[UFL1].func == PTP_PF_NONE) {
132	dev_info(ice_hw_to_dev(hw), "SMA1 + U.FL1 disabled");
133	data |= ICE_SMA1_MASK_E810T;
134	} else if (ptp_pins[SMA1].func == PTP_PF_EXTTS &&
135	ptp_pins[UFL1].func == PTP_PF_NONE) {
136	dev_info(ice_hw_to_dev(hw), "SMA1 RX");
137	data |= ICE_SMA1_TX_EN_E810T;
138	} else if (ptp_pins[SMA1].func == PTP_PF_NONE &&
139	ptp_pins[UFL1].func == PTP_PF_PEROUT) {
140	/* U.FL 1 TX will always enable SMA 1 RX */
141	dev_info(ice_hw_to_dev(hw), "SMA1 RX + U.FL1 TX");
142	} else if (ptp_pins[SMA1].func == PTP_PF_EXTTS &&
143	ptp_pins[UFL1].func == PTP_PF_PEROUT) {
144	dev_info(ice_hw_to_dev(hw), "SMA1 RX + U.FL1 TX");
145	} else if (ptp_pins[SMA1].func == PTP_PF_PEROUT &&
146	ptp_pins[UFL1].func == PTP_PF_NONE) {
147	dev_info(ice_hw_to_dev(hw), "SMA1 TX");
148	data |= ICE_SMA1_DIR_EN_E810T;
149	}
150
151	data &= ~ICE_SMA2_MASK_E810T;
152	if (ptp_pins[SMA2].func == PTP_PF_NONE &&
153	ptp_pins[UFL2].func == PTP_PF_NONE) {
154	dev_info(ice_hw_to_dev(hw), "SMA2 + U.FL2 disabled");
155	data |= ICE_SMA2_MASK_E810T;
156	} else if (ptp_pins[SMA2].func == PTP_PF_EXTTS &&
157	ptp_pins[UFL2].func == PTP_PF_NONE) {
158	dev_info(ice_hw_to_dev(hw), "SMA2 RX");
159	data |= (ICE_SMA2_TX_EN_E810T |
160	ICE_SMA2_UFL2_RX_DIS_E810T);
161	} else if (ptp_pins[SMA2].func == PTP_PF_NONE &&
162	ptp_pins[UFL2].func == PTP_PF_EXTTS) {
163	dev_info(ice_hw_to_dev(hw), "UFL2 RX");
164	data |= (ICE_SMA2_DIR_EN_E810T | ICE_SMA2_TX_EN_E810T);
165	} else if (ptp_pins[SMA2].func == PTP_PF_PEROUT &&
166	ptp_pins[UFL2].func == PTP_PF_NONE) {
167	dev_info(ice_hw_to_dev(hw), "SMA2 TX");
168	data |= (ICE_SMA2_DIR_EN_E810T |
169	ICE_SMA2_UFL2_RX_DIS_E810T);
170	} else if (ptp_pins[SMA2].func == PTP_PF_PEROUT &&
171	ptp_pins[UFL2].func == PTP_PF_EXTTS) {
172	dev_info(ice_hw_to_dev(hw), "SMA2 TX + U.FL2 RX");
173	data |= ICE_SMA2_DIR_EN_E810T;
174	}
175
176	return ice_write_sma_ctrl_e810t(hw, data);
177	}
178
179	/**
180	* ice_ptp_set_sma_e810t
181	* @info: the driver's PTP info structure
182	* @pin: pin index in kernel structure
183	* @func: Pin function to be set (PTP_PF_NONE, PTP_PF_EXTTS or PTP_PF_PEROUT)
184	*
185	* Set the configuration of a single SMA pin
186	*/
187	static int
188	ice_ptp_set_sma_e810t(struct ptp_clock_info *info, unsigned int pin,
189	enum ptp_pin_function func)
190	{
191	struct ptp_pin_desc ptp_pins[NUM_PTP_PINS_E810T];
192	struct ice_pf *pf = ptp_info_to_pf(info);
193	struct ice_hw *hw = &pf->hw;
194	int err;
195
196	if (pin < SMA1 || func > PTP_PF_PEROUT)
197	return -EOPNOTSUPP;
198
199	err = ice_get_sma_config_e810t(hw, ptp_pins);
200	if (err)
201	return err;
202
203	/* Disable the same function on the other pin sharing the channel */
204	if (pin == SMA1 && ptp_pins[UFL1].func == func)
205	ptp_pins[UFL1].func = PTP_PF_NONE;
206	if (pin == UFL1 && ptp_pins[SMA1].func == func)
207	ptp_pins[SMA1].func = PTP_PF_NONE;
208
209	if (pin == SMA2 && ptp_pins[UFL2].func == func)
210	ptp_pins[UFL2].func = PTP_PF_NONE;
211	if (pin == UFL2 && ptp_pins[SMA2].func == func)
212	ptp_pins[SMA2].func = PTP_PF_NONE;
213
214	/* Set up new pin function in the temp table */
215	ptp_pins[pin].func = func;
216
217	return ice_ptp_set_sma_config_e810t(hw, ptp_pins);
218	}
219
220	/**
221	* ice_verify_pin_e810t
222	* @info: the driver's PTP info structure
223	* @pin: Pin index
224	* @func: Assigned function
225	* @chan: Assigned channel
226	*
227	* Verify if pin supports requested pin function. If the Check pins consistency.
228	* Reconfigure the SMA logic attached to the given pin to enable its
229	* desired functionality
230	*/
231	static int
232	ice_verify_pin_e810t(struct ptp_clock_info *info, unsigned int pin,
233	enum ptp_pin_function func, unsigned int chan)
234	{
235	/* Don't allow channel reassignment */
236	if (chan != ice_pin_desc_e810t[pin].chan)
237	return -EOPNOTSUPP;
238
239	/* Check if functions are properly assigned */
240	switch (func) {
241	case PTP_PF_NONE:
242	break;
243	case PTP_PF_EXTTS:
244	if (pin == UFL1)
245	return -EOPNOTSUPP;
246	break;
247	case PTP_PF_PEROUT:
248	if (pin == UFL2 || pin == GNSS)
249	return -EOPNOTSUPP;
250	break;
251	case PTP_PF_PHYSYNC:
252	return -EOPNOTSUPP;
253	}
254
255	return ice_ptp_set_sma_e810t(info, pin, func);
256	}
257
258	/**
259	* ice_ptp_cfg_tx_interrupt - Configure Tx timestamp interrupt for the device
260	* @pf: Board private structure
261	*
262	* Program the device to respond appropriately to the Tx timestamp interrupt
263	* cause.
264	*/
265	static void ice_ptp_cfg_tx_interrupt(struct ice_pf *pf)
266	{
267	struct ice_hw *hw = &pf->hw;
268	bool enable;
269	u32 val;
270
271	switch (pf->ptp.tx_interrupt_mode) {
272	case ICE_PTP_TX_INTERRUPT_ALL:
273	/* React to interrupts across all quads. */
274	wr32(hw, PFINT_TSYN_MSK + (0x4 * hw->pf_id), (u32)0x1f);
275	enable = true;
276	break;
277	case ICE_PTP_TX_INTERRUPT_NONE:
278	/* Do not react to interrupts on any quad. */
279	wr32(hw, PFINT_TSYN_MSK + (0x4 * hw->pf_id), (u32)0x0);
280	enable = false;
281	break;
282	case ICE_PTP_TX_INTERRUPT_SELF:
283	default:
284	enable = pf->ptp.tstamp_config.tx_type == HWTSTAMP_TX_ON;
285	break;
286	}
287
288	/* Configure the Tx timestamp interrupt */
289	val = rd32(hw, PFINT_OICR_ENA);
290	if (enable)
291	val |= PFINT_OICR_TSYN_TX_M;
292	else
293	val &= ~PFINT_OICR_TSYN_TX_M;
294	wr32(hw, PFINT_OICR_ENA, val);
295	}
296
297	/**
298	* ice_set_rx_tstamp - Enable or disable Rx timestamping
299	* @pf: The PF pointer to search in
300	* @on: bool value for whether timestamps are enabled or disabled
301	*/
302	static void ice_set_rx_tstamp(struct ice_pf *pf, bool on)
303	{
304	struct ice_vsi *vsi;
305	u16 i;
306
307	vsi = ice_get_main_vsi(pf);
308	if (!vsi || !vsi->rx_rings)
309	return;
310
311	/* Set the timestamp flag for all the Rx rings */
312	ice_for_each_rxq(vsi, i) {
313	if (!vsi->rx_rings[i])
314	continue;
315	vsi->rx_rings[i]->ptp_rx = on;
316	}
317	}
318
319	/**
320	* ice_ptp_disable_timestamp_mode - Disable current timestamp mode
321	* @pf: Board private structure
322	*
323	* Called during preparation for reset to temporarily disable timestamping on
324	* the device. Called during remove to disable timestamping while cleaning up
325	* driver resources.
326	*/
327	static void ice_ptp_disable_timestamp_mode(struct ice_pf *pf)
328	{
329	struct ice_hw *hw = &pf->hw;
330	u32 val;
331
332	val = rd32(hw, PFINT_OICR_ENA);
333	val &= ~PFINT_OICR_TSYN_TX_M;
334	wr32(hw, PFINT_OICR_ENA, val);
335
336	ice_set_rx_tstamp(pf, on: false);
337	}
338
339	/**
340	* ice_ptp_restore_timestamp_mode - Restore timestamp configuration
341	* @pf: Board private structure
342	*
343	* Called at the end of rebuild to restore timestamp configuration after
344	* a device reset.
345	*/
346	void ice_ptp_restore_timestamp_mode(struct ice_pf *pf)
347	{
348	struct ice_hw *hw = &pf->hw;
349	bool enable_rx;
350
351	ice_ptp_cfg_tx_interrupt(pf);
352
353	enable_rx = pf->ptp.tstamp_config.rx_filter == HWTSTAMP_FILTER_ALL;
354	ice_set_rx_tstamp(pf, on: enable_rx);
355
356	/* Trigger an immediate software interrupt to ensure that timestamps
357	* which occurred during reset are handled now.
358	*/
359	wr32(hw, PFINT_OICR, PFINT_OICR_TSYN_TX_M);
360	ice_flush(hw);
361	}
362
363	/**
364	* ice_ptp_read_src_clk_reg - Read the source clock register
365	* @pf: Board private structure
366	* @sts: Optional parameter for holding a pair of system timestamps from
367	* the system clock. Will be ignored if NULL is given.
368	*/
369	static u64
370	ice_ptp_read_src_clk_reg(struct ice_pf *pf, struct ptp_system_timestamp *sts)
371	{
372	struct ice_hw *hw = &pf->hw;
373	u32 hi, lo, lo2;
374	u8 tmr_idx;
375
376	tmr_idx = ice_get_ptp_src_clock_index(hw);
377	/* Read the system timestamp pre PHC read */
378	ptp_read_system_prets(sts);
379
380	lo = rd32(hw, GLTSYN_TIME_L(tmr_idx));
381
382	/* Read the system timestamp post PHC read */
383	ptp_read_system_postts(sts);
384
385	hi = rd32(hw, GLTSYN_TIME_H(tmr_idx));
386	lo2 = rd32(hw, GLTSYN_TIME_L(tmr_idx));
387
388	if (lo2 < lo) {
389	/* if TIME_L rolled over read TIME_L again and update
390	* system timestamps
391	*/
392	ptp_read_system_prets(sts);
393	lo = rd32(hw, GLTSYN_TIME_L(tmr_idx));
394	ptp_read_system_postts(sts);
395	hi = rd32(hw, GLTSYN_TIME_H(tmr_idx));
396	}
397
398	return ((u64)hi << 32) | lo;
399	}
400
401	/**
402	* ice_ptp_extend_32b_ts - Convert a 32b nanoseconds timestamp to 64b
403	* @cached_phc_time: recently cached copy of PHC time
404	* @in_tstamp: Ingress/egress 32b nanoseconds timestamp value
405	*
406	* Hardware captures timestamps which contain only 32 bits of nominal
407	* nanoseconds, as opposed to the 64bit timestamps that the stack expects.
408	* Note that the captured timestamp values may be 40 bits, but the lower
409	* 8 bits are sub-nanoseconds and generally discarded.
410	*
411	* Extend the 32bit nanosecond timestamp using the following algorithm and
412	* assumptions:
413	*
414	* 1) have a recently cached copy of the PHC time
415	* 2) assume that the in_tstamp was captured 2^31 nanoseconds (~2.1
416	* seconds) before or after the PHC time was captured.
417	* 3) calculate the delta between the cached time and the timestamp
418	* 4) if the delta is smaller than 2^31 nanoseconds, then the timestamp was
419	* captured after the PHC time. In this case, the full timestamp is just
420	* the cached PHC time plus the delta.
421	* 5) otherwise, if the delta is larger than 2^31 nanoseconds, then the
422	* timestamp was captured *before* the PHC time, i.e. because the PHC
423	* cache was updated after the timestamp was captured by hardware. In this
424	* case, the full timestamp is the cached time minus the inverse delta.
425	*
426	* This algorithm works even if the PHC time was updated after a Tx timestamp
427	* was requested, but before the Tx timestamp event was reported from
428	* hardware.
429	*
430	* This calculation primarily relies on keeping the cached PHC time up to
431	* date. If the timestamp was captured more than 2^31 nanoseconds after the
432	* PHC time, it is possible that the lower 32bits of PHC time have
433	* overflowed more than once, and we might generate an incorrect timestamp.
434	*
435	* This is prevented by (a) periodically updating the cached PHC time once
436	* a second, and (b) discarding any Tx timestamp packet if it has waited for
437	* a timestamp for more than one second.
438	*/
439	static u64 ice_ptp_extend_32b_ts(u64 cached_phc_time, u32 in_tstamp)
440	{
441	u32 delta, phc_time_lo;
442	u64 ns;
443
444	/* Extract the lower 32 bits of the PHC time */
445	phc_time_lo = (u32)cached_phc_time;
446
447	/* Calculate the delta between the lower 32bits of the cached PHC
448	* time and the in_tstamp value
449	*/
450	delta = (in_tstamp - phc_time_lo);
451
452	/* Do not assume that the in_tstamp is always more recent than the
453	* cached PHC time. If the delta is large, it indicates that the
454	* in_tstamp was taken in the past, and should be converted
455	* forward.
456	*/
457	if (delta > (U32_MAX / 2)) {
458	/* reverse the delta calculation here */
459	delta = (phc_time_lo - in_tstamp);
460	ns = cached_phc_time - delta;
461	} else {
462	ns = cached_phc_time + delta;
463	}
464
465	return ns;
466	}
467
468	/**
469	* ice_ptp_extend_40b_ts - Convert a 40b timestamp to 64b nanoseconds
470	* @pf: Board private structure
471	* @in_tstamp: Ingress/egress 40b timestamp value
472	*
473	* The Tx and Rx timestamps are 40 bits wide, including 32 bits of nominal
474	* nanoseconds, 7 bits of sub-nanoseconds, and a valid bit.
475	*
476	* *--------------------------------------------------------------*
477	* | 32 bits of nanoseconds | 7 high bits of sub ns underflow | v |
478	* *--------------------------------------------------------------*
479	*
480	* The low bit is an indicator of whether the timestamp is valid. The next
481	* 7 bits are a capture of the upper 7 bits of the sub-nanosecond underflow,
482	* and the remaining 32 bits are the lower 32 bits of the PHC timer.
483	*
484	* It is assumed that the caller verifies the timestamp is valid prior to
485	* calling this function.
486	*
487	* Extract the 32bit nominal nanoseconds and extend them. Use the cached PHC
488	* time stored in the device private PTP structure as the basis for timestamp
489	* extension.
490	*
491	* See ice_ptp_extend_32b_ts for a detailed explanation of the extension
492	* algorithm.
493	*/
494	static u64 ice_ptp_extend_40b_ts(struct ice_pf *pf, u64 in_tstamp)
495	{
496	const u64 mask = GENMASK_ULL(31, 0);
497	unsigned long discard_time;
498
499	/* Discard the hardware timestamp if the cached PHC time is too old */
500	discard_time = pf->ptp.cached_phc_jiffies + msecs_to_jiffies(m: 2000);
501	if (time_is_before_jiffies(discard_time)) {
502	pf->ptp.tx_hwtstamp_discarded++;
503	return 0;
504	}
505
506	return ice_ptp_extend_32b_ts(cached_phc_time: pf->ptp.cached_phc_time,
507	in_tstamp: (in_tstamp >> 8) & mask);
508	}
509
510	/**
511	* ice_ptp_is_tx_tracker_up - Check if Tx tracker is ready for new timestamps
512	* @tx: the PTP Tx timestamp tracker to check
513	*
514	* Check that a given PTP Tx timestamp tracker is up, i.e. that it is ready
515	* to accept new timestamp requests.
516	*
517	* Assumes the tx->lock spinlock is already held.
518	*/
519	static bool
520	ice_ptp_is_tx_tracker_up(struct ice_ptp_tx *tx)
521	{
522	lockdep_assert_held(&tx->lock);
523
524	return tx->init && !tx->calibrating;
525	}
526
527	/**
528	* ice_ptp_req_tx_single_tstamp - Request Tx timestamp for a port from FW
529	* @tx: the PTP Tx timestamp tracker
530	* @idx: index of the timestamp to request
531	*/
532	void ice_ptp_req_tx_single_tstamp(struct ice_ptp_tx *tx, u8 idx)
533	{
534	struct ice_ptp_port *ptp_port;
535	struct sk_buff *skb;
536	struct ice_pf *pf;
537
538	if (!tx->init)
539	return;
540
541	ptp_port = container_of(tx, struct ice_ptp_port, tx);
542	pf = ptp_port_to_pf(ptp_port);
543
544	/* Drop packets which have waited for more than 2 seconds */
545	if (time_is_before_jiffies(tx->tstamps[idx].start + 2 * HZ)) {
546	/* Count the number of Tx timestamps that timed out */
547	pf->ptp.tx_hwtstamp_timeouts++;
548
549	skb = tx->tstamps[idx].skb;
550	tx->tstamps[idx].skb = NULL;
551	clear_bit(nr: idx, addr: tx->in_use);
552
553	dev_kfree_skb_any(skb);
554	return;
555	}
556
557	ice_trace(tx_tstamp_fw_req, tx->tstamps[idx].skb, idx);
558
559	/* Write TS index to read to the PF register so the FW can read it */
560	wr32(&pf->hw, PF_SB_ATQBAL,
561	TS_LL_READ_TS_INTR | FIELD_PREP(TS_LL_READ_TS_IDX, idx) |
562	TS_LL_READ_TS);
563	tx->last_ll_ts_idx_read = idx;
564	}
565
566	/**
567	* ice_ptp_complete_tx_single_tstamp - Complete Tx timestamp for a port
568	* @tx: the PTP Tx timestamp tracker
569	*/
570	void ice_ptp_complete_tx_single_tstamp(struct ice_ptp_tx *tx)
571	{
572	struct skb_shared_hwtstamps shhwtstamps = {};
573	u8 idx = tx->last_ll_ts_idx_read;
574	struct ice_ptp_port *ptp_port;
575	u64 raw_tstamp, tstamp;
576	bool drop_ts = false;
577	struct sk_buff *skb;
578	struct ice_pf *pf;
579	u32 val;
580
581	if (!tx->init || tx->last_ll_ts_idx_read < 0)
582	return;
583
584	ptp_port = container_of(tx, struct ice_ptp_port, tx);
585	pf = ptp_port_to_pf(ptp_port);
586
587	ice_trace(tx_tstamp_fw_done, tx->tstamps[idx].skb, idx);
588
589	val = rd32(&pf->hw, PF_SB_ATQBAL);
590
591	/* When the bit is cleared, the TS is ready in the register */
592	if (val & TS_LL_READ_TS) {
593	dev_err(ice_pf_to_dev(pf), "Failed to get the Tx tstamp - FW not ready");
594	return;
595	}
596
597	/* High 8 bit value of the TS is on the bits 16:23 */
598	raw_tstamp = FIELD_GET(TS_LL_READ_TS_HIGH, val);
599	raw_tstamp <<= 32;
600
601	/* Read the low 32 bit value */
602	raw_tstamp |= (u64)rd32(&pf->hw, PF_SB_ATQBAH);
603
604	/* Devices using this interface always verify the timestamp differs
605	* relative to the last cached timestamp value.
606	*/
607	if (raw_tstamp == tx->tstamps[idx].cached_tstamp)
608	return;
609
610	tx->tstamps[idx].cached_tstamp = raw_tstamp;
611	clear_bit(nr: idx, addr: tx->in_use);
612	skb = tx->tstamps[idx].skb;
613	tx->tstamps[idx].skb = NULL;
614	if (test_and_clear_bit(nr: idx, addr: tx->stale))
615	drop_ts = true;
616
617	if (!skb)
618	return;
619
620	if (drop_ts) {
621	dev_kfree_skb_any(skb);
622	return;
623	}
624
625	/* Extend the timestamp using cached PHC time */
626	tstamp = ice_ptp_extend_40b_ts(pf, in_tstamp: raw_tstamp);
627	if (tstamp) {
628	shhwtstamps.hwtstamp = ns_to_ktime(ns: tstamp);
629	ice_trace(tx_tstamp_complete, skb, idx);
630	}
631
632	skb_tstamp_tx(orig_skb: skb, hwtstamps: &shhwtstamps);
633	dev_kfree_skb_any(skb);
634	}
635
636	/**
637	* ice_ptp_process_tx_tstamp - Process Tx timestamps for a port
638	* @tx: the PTP Tx timestamp tracker
639	*
640	* Process timestamps captured by the PHY associated with this port. To do
641	* this, loop over each index with a waiting skb.
642	*
643	* If a given index has a valid timestamp, perform the following steps:
644	*
645	* 1) check that the timestamp request is not stale
646	* 2) check that a timestamp is ready and available in the PHY memory bank
647	* 3) read and copy the timestamp out of the PHY register
648	* 4) unlock the index by clearing the associated in_use bit
649	* 5) check if the timestamp is stale, and discard if so
650	* 6) extend the 40 bit timestamp value to get a 64 bit timestamp value
651	* 7) send this 64 bit timestamp to the stack
652	*
653	* Note that we do not hold the tracking lock while reading the Tx timestamp.
654	* This is because reading the timestamp requires taking a mutex that might
655	* sleep.
656	*
657	* The only place where we set in_use is when a new timestamp is initiated
658	* with a slot index. This is only called in the hard xmit routine where an
659	* SKB has a request flag set. The only places where we clear this bit is this
660	* function, or during teardown when the Tx timestamp tracker is being
661	* removed. A timestamp index will never be re-used until the in_use bit for
662	* that index is cleared.
663	*
664	* If a Tx thread starts a new timestamp, we might not begin processing it
665	* right away but we will notice it at the end when we re-queue the task.
666	*
667	* If a Tx thread starts a new timestamp just after this function exits, the
668	* interrupt for that timestamp should re-trigger this function once
669	* a timestamp is ready.
670	*
671	* In cases where the PTP hardware clock was directly adjusted, some
672	* timestamps may not be able to safely use the timestamp extension math. In
673	* this case, software will set the stale bit for any outstanding Tx
674	* timestamps when the clock is adjusted. Then this function will discard
675	* those captured timestamps instead of sending them to the stack.
676	*
677	* If a Tx packet has been waiting for more than 2 seconds, it is not possible
678	* to correctly extend the timestamp using the cached PHC time. It is
679	* extremely unlikely that a packet will ever take this long to timestamp. If
680	* we detect a Tx timestamp request that has waited for this long we assume
681	* the packet will never be sent by hardware and discard it without reading
682	* the timestamp register.
683	*/
684	static void ice_ptp_process_tx_tstamp(struct ice_ptp_tx *tx)
685	{
686	struct ice_ptp_port *ptp_port;
687	unsigned long flags;
688	struct ice_pf *pf;
689	struct ice_hw *hw;
690	u64 tstamp_ready;
691	bool link_up;
692	int err;
693	u8 idx;
694
695	ptp_port = container_of(tx, struct ice_ptp_port, tx);
696	pf = ptp_port_to_pf(ptp_port);
697	hw = &pf->hw;
698
699	/* Read the Tx ready status first */
700	if (tx->has_ready_bitmap) {
701	err = ice_get_phy_tx_tstamp_ready(hw, block: tx->block, tstamp_ready: &tstamp_ready);
702	if (err)
703	return;
704	}
705
706	/* Drop packets if the link went down */
707	link_up = ptp_port->link_up;
708
709	for_each_set_bit(idx, tx->in_use, tx->len) {
710	struct skb_shared_hwtstamps shhwtstamps = {};
711	u8 phy_idx = idx + tx->offset;
712	u64 raw_tstamp = 0, tstamp;
713	bool drop_ts = !link_up;
714	struct sk_buff *skb;
715
716	/* Drop packets which have waited for more than 2 seconds */
717	if (time_is_before_jiffies(tx->tstamps[idx].start + 2 * HZ)) {
718	drop_ts = true;
719
720	/* Count the number of Tx timestamps that timed out */
721	pf->ptp.tx_hwtstamp_timeouts++;
722	}
723
724	/* Only read a timestamp from the PHY if its marked as ready
725	* by the tstamp_ready register. This avoids unnecessary
726	* reading of timestamps which are not yet valid. This is
727	* important as we must read all timestamps which are valid
728	* and only timestamps which are valid during each interrupt.
729	* If we do not, the hardware logic for generating a new
730	* interrupt can get stuck on some devices.
731	*/
732	if (tx->has_ready_bitmap &&
733	!(tstamp_ready & BIT_ULL(phy_idx))) {
734	if (drop_ts)
735	goto skip_ts_read;
736
737	continue;
738	}
739
740	ice_trace(tx_tstamp_fw_req, tx->tstamps[idx].skb, idx);
741
742	err = ice_read_phy_tstamp(hw, block: tx->block, idx: phy_idx, tstamp: &raw_tstamp);
743	if (err && !drop_ts)
744	continue;
745
746	ice_trace(tx_tstamp_fw_done, tx->tstamps[idx].skb, idx);
747
748	/* For PHYs which don't implement a proper timestamp ready
749	* bitmap, verify that the timestamp value is different
750	* from the last cached timestamp. If it is not, skip this for
751	* now assuming it hasn't yet been captured by hardware.
752	*/
753	if (!drop_ts && !tx->has_ready_bitmap &&
754	raw_tstamp == tx->tstamps[idx].cached_tstamp)
755	continue;
756
757	/* Discard any timestamp value without the valid bit set */
758	if (!(raw_tstamp & ICE_PTP_TS_VALID))
759	drop_ts = true;
760
761	skip_ts_read:
762	spin_lock_irqsave(&tx->lock, flags);
763	if (!tx->has_ready_bitmap && raw_tstamp)
764	tx->tstamps[idx].cached_tstamp = raw_tstamp;
765	clear_bit(nr: idx, addr: tx->in_use);
766	skb = tx->tstamps[idx].skb;
767	tx->tstamps[idx].skb = NULL;
768	if (test_and_clear_bit(nr: idx, addr: tx->stale))
769	drop_ts = true;
770	spin_unlock_irqrestore(lock: &tx->lock, flags);
771
772	/* It is unlikely but possible that the SKB will have been
773	* flushed at this point due to link change or teardown.
774	*/
775	if (!skb)
776	continue;
777
778	if (drop_ts) {
779	dev_kfree_skb_any(skb);
780	continue;
781	}
782
783	/* Extend the timestamp using cached PHC time */
784	tstamp = ice_ptp_extend_40b_ts(pf, in_tstamp: raw_tstamp);
785	if (tstamp) {
786	shhwtstamps.hwtstamp = ns_to_ktime(ns: tstamp);
787	ice_trace(tx_tstamp_complete, skb, idx);
788	}
789
790	skb_tstamp_tx(orig_skb: skb, hwtstamps: &shhwtstamps);
791	dev_kfree_skb_any(skb);
792	}
793	}
794
795	/**
796	* ice_ptp_tx_tstamp_owner - Process Tx timestamps for all ports on the device
797	* @pf: Board private structure
798	*/
799	static enum ice_tx_tstamp_work ice_ptp_tx_tstamp_owner(struct ice_pf *pf)
800	{
801	struct ice_ptp_port *port;
802	unsigned int i;
803
804	mutex_lock(&pf->ptp.ports_owner.lock);
805	list_for_each_entry(port, &pf->ptp.ports_owner.ports, list_member) {
806	struct ice_ptp_tx *tx = &port->tx;
807
808	if (!tx || !tx->init)
809	continue;
810
811	ice_ptp_process_tx_tstamp(tx);
812	}
813	mutex_unlock(lock: &pf->ptp.ports_owner.lock);
814
815	for (i = 0; i < ICE_MAX_QUAD; i++) {
816	u64 tstamp_ready;
817	int err;
818
819	/* Read the Tx ready status first */
820	err = ice_get_phy_tx_tstamp_ready(hw: &pf->hw, block: i, tstamp_ready: &tstamp_ready);
821	if (err)
822	break;
823	else if (tstamp_ready)
824	return ICE_TX_TSTAMP_WORK_PENDING;
825	}
826
827	return ICE_TX_TSTAMP_WORK_DONE;
828	}
829
830	/**
831	* ice_ptp_tx_tstamp - Process Tx timestamps for this function.
832	* @tx: Tx tracking structure to initialize
833	*
834	* Returns: ICE_TX_TSTAMP_WORK_PENDING if there are any outstanding incomplete
835	* Tx timestamps, or ICE_TX_TSTAMP_WORK_DONE otherwise.
836	*/
837	static enum ice_tx_tstamp_work ice_ptp_tx_tstamp(struct ice_ptp_tx *tx)
838	{
839	bool more_timestamps;
840	unsigned long flags;
841
842	if (!tx->init)
843	return ICE_TX_TSTAMP_WORK_DONE;
844
845	/* Process the Tx timestamp tracker */
846	ice_ptp_process_tx_tstamp(tx);
847
848	/* Check if there are outstanding Tx timestamps */
849	spin_lock_irqsave(&tx->lock, flags);
850	more_timestamps = tx->init && !bitmap_empty(src: tx->in_use, nbits: tx->len);
851	spin_unlock_irqrestore(lock: &tx->lock, flags);
852
853	if (more_timestamps)
854	return ICE_TX_TSTAMP_WORK_PENDING;
855
856	return ICE_TX_TSTAMP_WORK_DONE;
857	}
858
859	/**
860	* ice_ptp_alloc_tx_tracker - Initialize tracking for Tx timestamps
861	* @tx: Tx tracking structure to initialize
862	*
863	* Assumes that the length has already been initialized. Do not call directly,
864	* use the ice_ptp_init_tx_* instead.
865	*/
866	static int
867	ice_ptp_alloc_tx_tracker(struct ice_ptp_tx *tx)
868	{
869	unsigned long *in_use, *stale;
870	struct ice_tx_tstamp *tstamps;
871
872	tstamps = kcalloc(n: tx->len, size: sizeof(*tstamps), GFP_KERNEL);
873	in_use = bitmap_zalloc(nbits: tx->len, GFP_KERNEL);
874	stale = bitmap_zalloc(nbits: tx->len, GFP_KERNEL);
875
876	if (!tstamps || !in_use || !stale) {
877	kfree(objp: tstamps);
878	bitmap_free(bitmap: in_use);
879	bitmap_free(bitmap: stale);
880
881	return -ENOMEM;
882	}
883
884	tx->tstamps = tstamps;
885	tx->in_use = in_use;
886	tx->stale = stale;
887	tx->init = 1;
888	tx->last_ll_ts_idx_read = -1;
889
890	spin_lock_init(&tx->lock);
891
892	return 0;
893	}
894
895	/**
896	* ice_ptp_flush_tx_tracker - Flush any remaining timestamps from the tracker
897	* @pf: Board private structure
898	* @tx: the tracker to flush
899	*
900	* Called during teardown when a Tx tracker is being removed.
901	*/
902	static void
903	ice_ptp_flush_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx)
904	{
905	struct ice_hw *hw = &pf->hw;
906	unsigned long flags;
907	u64 tstamp_ready;
908	int err;
909	u8 idx;
910
911	err = ice_get_phy_tx_tstamp_ready(hw, block: tx->block, tstamp_ready: &tstamp_ready);
912	if (err) {
913	dev_dbg(ice_pf_to_dev(pf), "Failed to get the Tx tstamp ready bitmap for block %u, err %d\n",
914	tx->block, err);
915
916	/* If we fail to read the Tx timestamp ready bitmap just
917	* skip clearing the PHY timestamps.
918	*/
919	tstamp_ready = 0;
920	}
921
922	for_each_set_bit(idx, tx->in_use, tx->len) {
923	u8 phy_idx = idx + tx->offset;
924	struct sk_buff *skb;
925
926	/* In case this timestamp is ready, we need to clear it. */
927	if (!hw->reset_ongoing && (tstamp_ready & BIT_ULL(phy_idx)))
928	ice_clear_phy_tstamp(hw, block: tx->block, idx: phy_idx);
929
930	spin_lock_irqsave(&tx->lock, flags);
931	skb = tx->tstamps[idx].skb;
932	tx->tstamps[idx].skb = NULL;
933	clear_bit(nr: idx, addr: tx->in_use);
934	clear_bit(nr: idx, addr: tx->stale);
935	spin_unlock_irqrestore(lock: &tx->lock, flags);
936
937	/* Count the number of Tx timestamps flushed */
938	pf->ptp.tx_hwtstamp_flushed++;
939
940	/* Free the SKB after we've cleared the bit */
941	dev_kfree_skb_any(skb);
942	}
943	}
944
945	/**
946	* ice_ptp_mark_tx_tracker_stale - Mark unfinished timestamps as stale
947	* @tx: the tracker to mark
948	*
949	* Mark currently outstanding Tx timestamps as stale. This prevents sending
950	* their timestamp value to the stack. This is required to prevent extending
951	* the 40bit hardware timestamp incorrectly.
952	*
953	* This should be called when the PTP clock is modified such as after a set
954	* time request.
955	*/
956	static void
957	ice_ptp_mark_tx_tracker_stale(struct ice_ptp_tx *tx)
958	{
959	unsigned long flags;
960
961	spin_lock_irqsave(&tx->lock, flags);
962	bitmap_or(dst: tx->stale, src1: tx->stale, src2: tx->in_use, nbits: tx->len);
963	spin_unlock_irqrestore(lock: &tx->lock, flags);
964	}
965
966	/**
967	* ice_ptp_flush_all_tx_tracker - Flush all timestamp trackers on this clock
968	* @pf: Board private structure
969	*
970	* Called by the clock owner to flush all the Tx timestamp trackers associated
971	* with the clock.
972	*/
973	static void
974	ice_ptp_flush_all_tx_tracker(struct ice_pf *pf)
975	{
976	struct ice_ptp_port *port;
977
978	list_for_each_entry(port, &pf->ptp.ports_owner.ports, list_member)
979	ice_ptp_flush_tx_tracker(ptp_port_to_pf(port), tx: &port->tx);
980	}
981
982	/**
983	* ice_ptp_release_tx_tracker - Release allocated memory for Tx tracker
984	* @pf: Board private structure
985	* @tx: Tx tracking structure to release
986	*
987	* Free memory associated with the Tx timestamp tracker.
988	*/
989	static void
990	ice_ptp_release_tx_tracker(struct ice_pf *pf, struct ice_ptp_tx *tx)
991	{
992	unsigned long flags;
993
994	spin_lock_irqsave(&tx->lock, flags);
995	tx->init = 0;
996	spin_unlock_irqrestore(lock: &tx->lock, flags);
997
998	/* wait for potentially outstanding interrupt to complete */
999	synchronize_irq(irq: pf->oicr_irq.virq);
1000
1001	ice_ptp_flush_tx_tracker(pf, tx);
1002
1003	kfree(objp: tx->tstamps);
1004	tx->tstamps = NULL;
1005
1006	bitmap_free(bitmap: tx->in_use);
1007	tx->in_use = NULL;
1008
1009	bitmap_free(bitmap: tx->stale);
1010	tx->stale = NULL;
1011
1012	tx->len = 0;
1013	}
1014
1015	/**
1016	* ice_ptp_init_tx_e82x - Initialize tracking for Tx timestamps
1017	* @pf: Board private structure
1018	* @tx: the Tx tracking structure to initialize
1019	* @port: the port this structure tracks
1020	*
1021	* Initialize the Tx timestamp tracker for this port. For generic MAC devices,
1022	* the timestamp block is shared for all ports in the same quad. To avoid
1023	* ports using the same timestamp index, logically break the block of
1024	* registers into chunks based on the port number.
1025	*/
1026	static int
1027	ice_ptp_init_tx_e82x(struct ice_pf *pf, struct ice_ptp_tx *tx, u8 port)
1028	{
1029	tx->block = port / ICE_PORTS_PER_QUAD;
1030	tx->offset = (port % ICE_PORTS_PER_QUAD) * INDEX_PER_PORT_E82X;
1031	tx->len = INDEX_PER_PORT_E82X;
1032	tx->has_ready_bitmap = 1;
1033
1034	return ice_ptp_alloc_tx_tracker(tx);
1035	}
1036
1037	/**
1038	* ice_ptp_init_tx_e810 - Initialize tracking for Tx timestamps
1039	* @pf: Board private structure
1040	* @tx: the Tx tracking structure to initialize
1041	*
1042	* Initialize the Tx timestamp tracker for this PF. For E810 devices, each
1043	* port has its own block of timestamps, independent of the other ports.
1044	*/
1045	static int
1046	ice_ptp_init_tx_e810(struct ice_pf *pf, struct ice_ptp_tx *tx)
1047	{
1048	tx->block = pf->hw.port_info->lport;
1049	tx->offset = 0;
1050	tx->len = INDEX_PER_PORT_E810;
1051	/* The E810 PHY does not provide a timestamp ready bitmap. Instead,
1052	* verify new timestamps against cached copy of the last read
1053	* timestamp.
1054	*/
1055	tx->has_ready_bitmap = 0;
1056
1057	return ice_ptp_alloc_tx_tracker(tx);
1058	}
1059
1060	/**
1061	* ice_ptp_update_cached_phctime - Update the cached PHC time values
1062	* @pf: Board specific private structure
1063	*
1064	* This function updates the system time values which are cached in the PF
1065	* structure and the Rx rings.
1066	*
1067	* This function must be called periodically to ensure that the cached value
1068	* is never more than 2 seconds old.
1069	*
1070	* Note that the cached copy in the PF PTP structure is always updated, even
1071	* if we can't update the copy in the Rx rings.
1072	*
1073	* Return:
1074	* * 0 - OK, successfully updated
1075	* * -EAGAIN - PF was busy, need to reschedule the update
1076	*/
1077	static int ice_ptp_update_cached_phctime(struct ice_pf *pf)
1078	{
1079	struct device *dev = ice_pf_to_dev(pf);
1080	unsigned long update_before;
1081	u64 systime;
1082	int i;
1083
1084	update_before = pf->ptp.cached_phc_jiffies + msecs_to_jiffies(m: 2000);
1085	if (pf->ptp.cached_phc_time &&
1086	time_is_before_jiffies(update_before)) {
1087	unsigned long time_taken = jiffies - pf->ptp.cached_phc_jiffies;
1088
1089	dev_warn(dev, "%u msecs passed between update to cached PHC time\n",
1090	jiffies_to_msecs(time_taken));
1091	pf->ptp.late_cached_phc_updates++;
1092	}
1093
1094	/* Read the current PHC time */
1095	systime = ice_ptp_read_src_clk_reg(pf, NULL);
1096
1097	/* Update the cached PHC time stored in the PF structure */
1098	WRITE_ONCE(pf->ptp.cached_phc_time, systime);
1099	WRITE_ONCE(pf->ptp.cached_phc_jiffies, jiffies);
1100
1101	if (test_and_set_bit(nr: ICE_CFG_BUSY, addr: pf->state))
1102	return -EAGAIN;
1103
1104	ice_for_each_vsi(pf, i) {
1105	struct ice_vsi *vsi = pf->vsi[i];
1106	int j;
1107
1108	if (!vsi)
1109	continue;
1110
1111	if (vsi->type != ICE_VSI_PF)
1112	continue;
1113
1114	ice_for_each_rxq(vsi, j) {
1115	if (!vsi->rx_rings[j])
1116	continue;
1117	WRITE_ONCE(vsi->rx_rings[j]->cached_phctime, systime);
1118	}
1119	}
1120	clear_bit(nr: ICE_CFG_BUSY, addr: pf->state);
1121
1122	return 0;
1123	}
1124
1125	/**
1126	* ice_ptp_reset_cached_phctime - Reset cached PHC time after an update
1127	* @pf: Board specific private structure
1128	*
1129	* This function must be called when the cached PHC time is no longer valid,
1130	* such as after a time adjustment. It marks any currently outstanding Tx
1131	* timestamps as stale and updates the cached PHC time for both the PF and Rx
1132	* rings.
1133	*
1134	* If updating the PHC time cannot be done immediately, a warning message is
1135	* logged and the work item is scheduled immediately to minimize the window
1136	* with a wrong cached timestamp.
1137	*/
1138	static void ice_ptp_reset_cached_phctime(struct ice_pf *pf)
1139	{
1140	struct device *dev = ice_pf_to_dev(pf);
1141	int err;
1142
1143	/* Update the cached PHC time immediately if possible, otherwise
1144	* schedule the work item to execute soon.
1145	*/
1146	err = ice_ptp_update_cached_phctime(pf);
1147	if (err) {
1148	/* If another thread is updating the Rx rings, we won't
1149	* properly reset them here. This could lead to reporting of
1150	* invalid timestamps, but there isn't much we can do.
1151	*/
1152	dev_warn(dev, "%s: ICE_CFG_BUSY, unable to immediately update cached PHC time\n",
1153	__func__);
1154
1155	/* Queue the work item to update the Rx rings when possible */
1156	kthread_queue_delayed_work(worker: pf->ptp.kworker, dwork: &pf->ptp.work,
1157	delay: msecs_to_jiffies(m: 10));
1158	}
1159
1160	/* Mark any outstanding timestamps as stale, since they might have
1161	* been captured in hardware before the time update. This could lead
1162	* to us extending them with the wrong cached value resulting in
1163	* incorrect timestamp values.
1164	*/
1165	ice_ptp_mark_tx_tracker_stale(tx: &pf->ptp.port.tx);
1166	}
1167
1168	/**
1169	* ice_ptp_read_time - Read the time from the device
1170	* @pf: Board private structure
1171	* @ts: timespec structure to hold the current time value
1172	* @sts: Optional parameter for holding a pair of system timestamps from
1173	* the system clock. Will be ignored if NULL is given.
1174	*
1175	* This function reads the source clock registers and stores them in a timespec.
1176	* However, since the registers are 64 bits of nanoseconds, we must convert the
1177	* result to a timespec before we can return.
1178	*/
1179	static void
1180	ice_ptp_read_time(struct ice_pf *pf, struct timespec64 *ts,
1181	struct ptp_system_timestamp *sts)
1182	{
1183	u64 time_ns = ice_ptp_read_src_clk_reg(pf, sts);
1184
1185	*ts = ns_to_timespec64(nsec: time_ns);
1186	}
1187
1188	/**
1189	* ice_ptp_write_init - Set PHC time to provided value
1190	* @pf: Board private structure
1191	* @ts: timespec structure that holds the new time value
1192	*
1193	* Set the PHC time to the specified time provided in the timespec.
1194	*/
1195	static int ice_ptp_write_init(struct ice_pf *pf, struct timespec64 *ts)
1196	{
1197	u64 ns = timespec64_to_ns(ts);
1198	struct ice_hw *hw = &pf->hw;
1199
1200	return ice_ptp_init_time(hw, time: ns);
1201	}
1202
1203	/**
1204	* ice_ptp_write_adj - Adjust PHC clock time atomically
1205	* @pf: Board private structure
1206	* @adj: Adjustment in nanoseconds
1207	*
1208	* Perform an atomic adjustment of the PHC time by the specified number of
1209	* nanoseconds.
1210	*/
1211	static int ice_ptp_write_adj(struct ice_pf *pf, s32 adj)
1212	{
1213	struct ice_hw *hw = &pf->hw;
1214
1215	return ice_ptp_adj_clock(hw, adj);
1216	}
1217
1218	/**
1219	* ice_base_incval - Get base timer increment value
1220	* @pf: Board private structure
1221	*
1222	* Look up the base timer increment value for this device. The base increment
1223	* value is used to define the nominal clock tick rate. This increment value
1224	* is programmed during device initialization. It is also used as the basis
1225	* for calculating adjustments using scaled_ppm.
1226	*/
1227	static u64 ice_base_incval(struct ice_pf *pf)
1228	{
1229	struct ice_hw *hw = &pf->hw;
1230	u64 incval;
1231
1232	if (ice_is_e810(hw))
1233	incval = ICE_PTP_NOMINAL_INCVAL_E810;
1234	else if (ice_e82x_time_ref(hw) < NUM_ICE_TIME_REF_FREQ)
1235	incval = ice_e82x_nominal_incval(time_ref: ice_e82x_time_ref(hw));
1236	else
1237	incval = UNKNOWN_INCVAL_E82X;
1238
1239	dev_dbg(ice_pf_to_dev(pf), "PTP: using base increment value of 0x%016llx\n",
1240	incval);
1241
1242	return incval;
1243	}
1244
1245	/**
1246	* ice_ptp_check_tx_fifo - Check whether Tx FIFO is in an OK state
1247	* @port: PTP port for which Tx FIFO is checked
1248	*/
1249	static int ice_ptp_check_tx_fifo(struct ice_ptp_port *port)
1250	{
1251	int quad = port->port_num / ICE_PORTS_PER_QUAD;
1252	int offs = port->port_num % ICE_PORTS_PER_QUAD;
1253	struct ice_pf *pf;
1254	struct ice_hw *hw;
1255	u32 val, phy_sts;
1256	int err;
1257
1258	pf = ptp_port_to_pf(port);
1259	hw = &pf->hw;
1260
1261	if (port->tx_fifo_busy_cnt == FIFO_OK)
1262	return 0;
1263
1264	/* need to read FIFO state */
1265	if (offs == 0 || offs == 1)
1266	err = ice_read_quad_reg_e82x(hw, quad, Q_REG_FIFO01_STATUS,
1267	val: &val);
1268	else
1269	err = ice_read_quad_reg_e82x(hw, quad, Q_REG_FIFO23_STATUS,
1270	val: &val);
1271
1272	if (err) {
1273	dev_err(ice_pf_to_dev(pf), "PTP failed to check port %d Tx FIFO, err %d\n",
1274	port->port_num, err);
1275	return err;
1276	}
1277
1278	if (offs & 0x1)
1279	phy_sts = FIELD_GET(Q_REG_FIFO13_M, val);
1280	else
1281	phy_sts = FIELD_GET(Q_REG_FIFO02_M, val);
1282
1283	if (phy_sts & FIFO_EMPTY) {
1284	port->tx_fifo_busy_cnt = FIFO_OK;
1285	return 0;
1286	}
1287
1288	port->tx_fifo_busy_cnt++;
1289
1290	dev_dbg(ice_pf_to_dev(pf), "Try %d, port %d FIFO not empty\n",
1291	port->tx_fifo_busy_cnt, port->port_num);
1292
1293	if (port->tx_fifo_busy_cnt == ICE_PTP_FIFO_NUM_CHECKS) {
1294	dev_dbg(ice_pf_to_dev(pf),
1295	"Port %d Tx FIFO still not empty; resetting quad %d\n",
1296	port->port_num, quad);
1297	ice_ptp_reset_ts_memory_quad_e82x(hw, quad);
1298	port->tx_fifo_busy_cnt = FIFO_OK;
1299	return 0;
1300	}
1301
1302	return -EAGAIN;
1303	}
1304
1305	/**
1306	* ice_ptp_wait_for_offsets - Check for valid Tx and Rx offsets
1307	* @work: Pointer to the kthread_work structure for this task
1308	*
1309	* Check whether hardware has completed measuring the Tx and Rx offset values
1310	* used to configure and enable vernier timestamp calibration.
1311	*
1312	* Once the offset in either direction is measured, configure the associated
1313	* registers with the calibrated offset values and enable timestamping. The Tx
1314	* and Rx directions are configured independently as soon as their associated
1315	* offsets are known.
1316	*
1317	* This function reschedules itself until both Tx and Rx calibration have
1318	* completed.
1319	*/
1320	static void ice_ptp_wait_for_offsets(struct kthread_work *work)
1321	{
1322	struct ice_ptp_port *port;
1323	struct ice_pf *pf;
1324	struct ice_hw *hw;
1325	int tx_err;
1326	int rx_err;
1327
1328	port = container_of(work, struct ice_ptp_port, ov_work.work);
1329	pf = ptp_port_to_pf(port);
1330	hw = &pf->hw;
1331
1332	if (ice_is_reset_in_progress(state: pf->state)) {
1333	/* wait for device driver to complete reset */
1334	kthread_queue_delayed_work(worker: pf->ptp.kworker,
1335	dwork: &port->ov_work,
1336	delay: msecs_to_jiffies(m: 100));
1337	return;
1338	}
1339
1340	tx_err = ice_ptp_check_tx_fifo(port);
1341	if (!tx_err)
1342	tx_err = ice_phy_cfg_tx_offset_e82x(hw, port: port->port_num);
1343	rx_err = ice_phy_cfg_rx_offset_e82x(hw, port: port->port_num);
1344	if (tx_err || rx_err) {
1345	/* Tx and/or Rx offset not yet configured, try again later */
1346	kthread_queue_delayed_work(worker: pf->ptp.kworker,
1347	dwork: &port->ov_work,
1348	delay: msecs_to_jiffies(m: 100));
1349	return;
1350	}
1351	}
1352
1353	/**
1354	* ice_ptp_port_phy_stop - Stop timestamping for a PHY port
1355	* @ptp_port: PTP port to stop
1356	*/
1357	static int
1358	ice_ptp_port_phy_stop(struct ice_ptp_port *ptp_port)
1359	{
1360	struct ice_pf *pf = ptp_port_to_pf(ptp_port);
1361	u8 port = ptp_port->port_num;
1362	struct ice_hw *hw = &pf->hw;
1363	int err;
1364
1365	if (ice_is_e810(hw))
1366	return 0;
1367
1368	mutex_lock(&ptp_port->ps_lock);
1369
1370	kthread_cancel_delayed_work_sync(work: &ptp_port->ov_work);
1371
1372	err = ice_stop_phy_timer_e82x(hw, port, soft_reset: true);
1373	if (err)
1374	dev_err(ice_pf_to_dev(pf), "PTP failed to set PHY port %d down, err %d\n",
1375	port, err);
1376
1377	mutex_unlock(lock: &ptp_port->ps_lock);
1378
1379	return err;
1380	}
1381
1382	/**
1383	* ice_ptp_port_phy_restart - (Re)start and calibrate PHY timestamping
1384	* @ptp_port: PTP port for which the PHY start is set
1385	*
1386	* Start the PHY timestamping block, and initiate Vernier timestamping
1387	* calibration. If timestamping cannot be calibrated (such as if link is down)
1388	* then disable the timestamping block instead.
1389	*/
1390	static int
1391	ice_ptp_port_phy_restart(struct ice_ptp_port *ptp_port)
1392	{
1393	struct ice_pf *pf = ptp_port_to_pf(ptp_port);
1394	u8 port = ptp_port->port_num;
1395	struct ice_hw *hw = &pf->hw;
1396	unsigned long flags;
1397	int err;
1398
1399	if (ice_is_e810(hw))
1400	return 0;
1401
1402	if (!ptp_port->link_up)
1403	return ice_ptp_port_phy_stop(ptp_port);
1404
1405	mutex_lock(&ptp_port->ps_lock);
1406
1407	kthread_cancel_delayed_work_sync(work: &ptp_port->ov_work);
1408
1409	/* temporarily disable Tx timestamps while calibrating PHY offset */
1410	spin_lock_irqsave(&ptp_port->tx.lock, flags);
1411	ptp_port->tx.calibrating = true;
1412	spin_unlock_irqrestore(lock: &ptp_port->tx.lock, flags);
1413	ptp_port->tx_fifo_busy_cnt = 0;
1414
1415	/* Start the PHY timer in Vernier mode */
1416	err = ice_start_phy_timer_e82x(hw, port);
1417	if (err)
1418	goto out_unlock;
1419
1420	/* Enable Tx timestamps right away */
1421	spin_lock_irqsave(&ptp_port->tx.lock, flags);
1422	ptp_port->tx.calibrating = false;
1423	spin_unlock_irqrestore(lock: &ptp_port->tx.lock, flags);
1424
1425	kthread_queue_delayed_work(worker: pf->ptp.kworker, dwork: &ptp_port->ov_work, delay: 0);
1426
1427	out_unlock:
1428	if (err)
1429	dev_err(ice_pf_to_dev(pf), "PTP failed to set PHY port %d up, err %d\n",
1430	port, err);
1431
1432	mutex_unlock(lock: &ptp_port->ps_lock);
1433
1434	return err;
1435	}
1436
1437	/**
1438	* ice_ptp_link_change - Reconfigure PTP after link status change
1439	* @pf: Board private structure
1440	* @port: Port for which the PHY start is set
1441	* @linkup: Link is up or down
1442	*/
1443	void ice_ptp_link_change(struct ice_pf *pf, u8 port, bool linkup)
1444	{
1445	struct ice_ptp_port *ptp_port;
1446	struct ice_hw *hw = &pf->hw;
1447
1448	if (pf->ptp.state != ICE_PTP_READY)
1449	return;
1450
1451	if (WARN_ON_ONCE(port >= ICE_NUM_EXTERNAL_PORTS))
1452	return;
1453
1454	ptp_port = &pf->ptp.port;
1455	if (WARN_ON_ONCE(ptp_port->port_num != port))
1456	return;
1457
1458	/* Update cached link status for this port immediately */
1459	ptp_port->link_up = linkup;
1460
1461	switch (hw->phy_model) {
1462	case ICE_PHY_E810:
1463	/* Do not reconfigure E810 PHY */
1464	return;
1465	case ICE_PHY_E82X:
1466	ice_ptp_port_phy_restart(ptp_port);
1467	return;
1468	default:
1469	dev_warn(ice_pf_to_dev(pf), "%s: Unknown PHY type\n", __func__);
1470	}
1471	}
1472
1473	/**
1474	* ice_ptp_cfg_phy_interrupt - Configure PHY interrupt settings
1475	* @pf: PF private structure
1476	* @ena: bool value to enable or disable interrupt
1477	* @threshold: Minimum number of packets at which intr is triggered
1478	*
1479	* Utility function to enable or disable Tx timestamp interrupt and threshold
1480	*/
1481	static int ice_ptp_cfg_phy_interrupt(struct ice_pf *pf, bool ena, u32 threshold)
1482	{
1483	struct ice_hw *hw = &pf->hw;
1484	int err = 0;
1485	int quad;
1486	u32 val;
1487
1488	ice_ptp_reset_ts_memory(hw);
1489
1490	for (quad = 0; quad < ICE_MAX_QUAD; quad++) {
1491	err = ice_read_quad_reg_e82x(hw, quad, Q_REG_TX_MEM_GBL_CFG,
1492	val: &val);
1493	if (err)
1494	break;
1495
1496	if (ena) {
1497	val |= Q_REG_TX_MEM_GBL_CFG_INTR_ENA_M;
1498	val &= ~Q_REG_TX_MEM_GBL_CFG_INTR_THR_M;
1499	val |= FIELD_PREP(Q_REG_TX_MEM_GBL_CFG_INTR_THR_M,
1500	threshold);
1501	} else {
1502	val &= ~Q_REG_TX_MEM_GBL_CFG_INTR_ENA_M;
1503	}
1504
1505	err = ice_write_quad_reg_e82x(hw, quad, Q_REG_TX_MEM_GBL_CFG,
1506	val);
1507	if (err)
1508	break;
1509	}
1510
1511	if (err)
1512	dev_err(ice_pf_to_dev(pf), "PTP failed in intr ena, err %d\n",
1513	err);
1514	return err;
1515	}
1516
1517	/**
1518	* ice_ptp_reset_phy_timestamping - Reset PHY timestamping block
1519	* @pf: Board private structure
1520	*/
1521	static void ice_ptp_reset_phy_timestamping(struct ice_pf *pf)
1522	{
1523	ice_ptp_port_phy_restart(ptp_port: &pf->ptp.port);
1524	}
1525
1526	/**
1527	* ice_ptp_restart_all_phy - Restart all PHYs to recalibrate timestamping
1528	* @pf: Board private structure
1529	*/
1530	static void ice_ptp_restart_all_phy(struct ice_pf *pf)
1531	{
1532	struct list_head *entry;
1533
1534	list_for_each(entry, &pf->ptp.ports_owner.ports) {
1535	struct ice_ptp_port *port = list_entry(entry,
1536	struct ice_ptp_port,
1537	list_member);
1538
1539	if (port->link_up)
1540	ice_ptp_port_phy_restart(ptp_port: port);
1541	}
1542	}
1543
1544	/**
1545	* ice_ptp_adjfine - Adjust clock increment rate
1546	* @info: the driver's PTP info structure
1547	* @scaled_ppm: Parts per million with 16-bit fractional field
1548	*
1549	* Adjust the frequency of the clock by the indicated scaled ppm from the
1550	* base frequency.
1551	*/
1552	static int ice_ptp_adjfine(struct ptp_clock_info *info, long scaled_ppm)
1553	{
1554	struct ice_pf *pf = ptp_info_to_pf(info);
1555	struct ice_hw *hw = &pf->hw;
1556	u64 incval;
1557	int err;
1558
1559	incval = adjust_by_scaled_ppm(base: ice_base_incval(pf), scaled_ppm);
1560	err = ice_ptp_write_incval_locked(hw, incval);
1561	if (err) {
1562	dev_err(ice_pf_to_dev(pf), "PTP failed to set incval, err %d\n",
1563	err);
1564	return -EIO;
1565	}
1566
1567	return 0;
1568	}
1569
1570	/**
1571	* ice_ptp_extts_event - Process PTP external clock event
1572	* @pf: Board private structure
1573	*/
1574	void ice_ptp_extts_event(struct ice_pf *pf)
1575	{
1576	struct ptp_clock_event event;
1577	struct ice_hw *hw = &pf->hw;
1578	u8 chan, tmr_idx;
1579	u32 hi, lo;
1580
1581	tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned;
1582	/* Event time is captured by one of the two matched registers
1583	* GLTSYN_EVNT_L: 32 LSB of sampled time event
1584	* GLTSYN_EVNT_H: 32 MSB of sampled time event
1585	* Event is defined in GLTSYN_EVNT_0 register
1586	*/
1587	for (chan = 0; chan < GLTSYN_EVNT_H_IDX_MAX; chan++) {
1588	/* Check if channel is enabled */
1589	if (pf->ptp.ext_ts_irq & (1 << chan)) {
1590	lo = rd32(hw, GLTSYN_EVNT_L(chan, tmr_idx));
1591	hi = rd32(hw, GLTSYN_EVNT_H(chan, tmr_idx));
1592	event.timestamp = (((u64)hi) << 32) | lo;
1593	event.type = PTP_CLOCK_EXTTS;
1594	event.index = chan;
1595
1596	/* Fire event */
1597	ptp_clock_event(ptp: pf->ptp.clock, event: &event);
1598	pf->ptp.ext_ts_irq &= ~(1 << chan);
1599	}
1600	}
1601	}
1602
1603	/**
1604	* ice_ptp_cfg_extts - Configure EXTTS pin and channel
1605	* @pf: Board private structure
1606	* @ena: true to enable; false to disable
1607	* @chan: GPIO channel (0-3)
1608	* @gpio_pin: GPIO pin
1609	* @extts_flags: request flags from the ptp_extts_request.flags
1610	*/
1611	static int
1612	ice_ptp_cfg_extts(struct ice_pf *pf, bool ena, unsigned int chan, u32 gpio_pin,
1613	unsigned int extts_flags)
1614	{
1615	u32 func, aux_reg, gpio_reg, irq_reg;
1616	struct ice_hw *hw = &pf->hw;
1617	u8 tmr_idx;
1618
1619	if (chan > (unsigned int)pf->ptp.info.n_ext_ts)
1620	return -EINVAL;
1621
1622	tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned;
1623
1624	irq_reg = rd32(hw, PFINT_OICR_ENA);
1625
1626	if (ena) {
1627	/* Enable the interrupt */
1628	irq_reg |= PFINT_OICR_TSYN_EVNT_M;
1629	aux_reg = GLTSYN_AUX_IN_0_INT_ENA_M;
1630
1631	#define GLTSYN_AUX_IN_0_EVNTLVL_RISING_EDGE BIT(0)
1632	#define GLTSYN_AUX_IN_0_EVNTLVL_FALLING_EDGE BIT(1)
1633
1634	/* set event level to requested edge */
1635	if (extts_flags & PTP_FALLING_EDGE)
1636	aux_reg |= GLTSYN_AUX_IN_0_EVNTLVL_FALLING_EDGE;
1637	if (extts_flags & PTP_RISING_EDGE)
1638	aux_reg |= GLTSYN_AUX_IN_0_EVNTLVL_RISING_EDGE;
1639
1640	/* Write GPIO CTL reg.
1641	* 0x1 is input sampled by EVENT register(channel)
1642	* + num_in_channels * tmr_idx
1643	*/
1644	func = 1 + chan + (tmr_idx * 3);
1645	gpio_reg = FIELD_PREP(GLGEN_GPIO_CTL_PIN_FUNC_M, func);
1646	pf->ptp.ext_ts_chan |= (1 << chan);
1647	} else {
1648	/* clear the values we set to reset defaults */
1649	aux_reg = 0;
1650	gpio_reg = 0;
1651	pf->ptp.ext_ts_chan &= ~(1 << chan);
1652	if (!pf->ptp.ext_ts_chan)
1653	irq_reg &= ~PFINT_OICR_TSYN_EVNT_M;
1654	}
1655
1656	wr32(hw, PFINT_OICR_ENA, irq_reg);
1657	wr32(hw, GLTSYN_AUX_IN(chan, tmr_idx), aux_reg);
1658	wr32(hw, GLGEN_GPIO_CTL(gpio_pin), gpio_reg);
1659
1660	return 0;
1661	}
1662
1663	/**
1664	* ice_ptp_cfg_clkout - Configure clock to generate periodic wave
1665	* @pf: Board private structure
1666	* @chan: GPIO channel (0-3)
1667	* @config: desired periodic clk configuration. NULL will disable channel
1668	* @store: If set to true the values will be stored
1669	*
1670	* Configure the internal clock generator modules to generate the clock wave of
1671	* specified period.
1672	*/
1673	static int ice_ptp_cfg_clkout(struct ice_pf *pf, unsigned int chan,
1674	struct ice_perout_channel *config, bool store)
1675	{
1676	u64 current_time, period, start_time, phase;
1677	struct ice_hw *hw = &pf->hw;
1678	u32 func, val, gpio_pin;
1679	u8 tmr_idx;
1680
1681	tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned;
1682
1683	/* 0. Reset mode & out_en in AUX_OUT */
1684	wr32(hw, GLTSYN_AUX_OUT(chan, tmr_idx), 0);
1685
1686	/* If we're disabling the output, clear out CLKO and TGT and keep
1687	* output level low
1688	*/
1689	if (!config || !config->ena) {
1690	wr32(hw, GLTSYN_CLKO(chan, tmr_idx), 0);
1691	wr32(hw, GLTSYN_TGT_L(chan, tmr_idx), 0);
1692	wr32(hw, GLTSYN_TGT_H(chan, tmr_idx), 0);
1693
1694	val = GLGEN_GPIO_CTL_PIN_DIR_M;
1695	gpio_pin = pf->ptp.perout_channels[chan].gpio_pin;
1696	wr32(hw, GLGEN_GPIO_CTL(gpio_pin), val);
1697
1698	/* Store the value if requested */
1699	if (store)
1700	memset(&pf->ptp.perout_channels[chan], 0,
1701	sizeof(struct ice_perout_channel));
1702
1703	return 0;
1704	}
1705	period = config->period;
1706	start_time = config->start_time;
1707	div64_u64_rem(dividend: start_time, divisor: period, remainder: &phase);
1708	gpio_pin = config->gpio_pin;
1709
1710	/* 1. Write clkout with half of required period value */
1711	if (period & 0x1) {
1712	dev_err(ice_pf_to_dev(pf), "CLK Period must be an even value\n");
1713	goto err;
1714	}
1715
1716	period >>= 1;
1717
1718	/* For proper operation, the GLTSYN_CLKO must be larger than clock tick
1719	*/
1720	#define MIN_PULSE 3
1721	if (period <= MIN_PULSE || period > U32_MAX) {
1722	dev_err(ice_pf_to_dev(pf), "CLK Period must be > %d && < 2^33",
1723	MIN_PULSE * 2);
1724	goto err;
1725	}
1726
1727	wr32(hw, GLTSYN_CLKO(chan, tmr_idx), lower_32_bits(period));
1728
1729	/* Allow time for programming before start_time is hit */
1730	current_time = ice_ptp_read_src_clk_reg(pf, NULL);
1731
1732	/* if start time is in the past start the timer at the nearest second
1733	* maintaining phase
1734	*/
1735	if (start_time < current_time)
1736	start_time = div64_u64(dividend: current_time + NSEC_PER_SEC - 1,
1737	NSEC_PER_SEC) * NSEC_PER_SEC + phase;
1738
1739	if (ice_is_e810(hw))
1740	start_time -= E810_OUT_PROP_DELAY_NS;
1741	else
1742	start_time -= ice_e82x_pps_delay(time_ref: ice_e82x_time_ref(hw));
1743
1744	/* 2. Write TARGET time */
1745	wr32(hw, GLTSYN_TGT_L(chan, tmr_idx), lower_32_bits(start_time));
1746	wr32(hw, GLTSYN_TGT_H(chan, tmr_idx), upper_32_bits(start_time));
1747
1748	/* 3. Write AUX_OUT register */
1749	val = GLTSYN_AUX_OUT_0_OUT_ENA_M | GLTSYN_AUX_OUT_0_OUTMOD_M;
1750	wr32(hw, GLTSYN_AUX_OUT(chan, tmr_idx), val);
1751
1752	/* 4. write GPIO CTL reg */
1753	func = 8 + chan + (tmr_idx * 4);
1754	val = GLGEN_GPIO_CTL_PIN_DIR_M |
1755	FIELD_PREP(GLGEN_GPIO_CTL_PIN_FUNC_M, func);
1756	wr32(hw, GLGEN_GPIO_CTL(gpio_pin), val);
1757
1758	/* Store the value if requested */
1759	if (store) {
1760	memcpy(&pf->ptp.perout_channels[chan], config,
1761	sizeof(struct ice_perout_channel));
1762	pf->ptp.perout_channels[chan].start_time = phase;
1763	}
1764
1765	return 0;
1766	err:
1767	dev_err(ice_pf_to_dev(pf), "PTP failed to cfg per_clk\n");
1768	return -EFAULT;
1769	}
1770
1771	/**
1772	* ice_ptp_disable_all_clkout - Disable all currently configured outputs
1773	* @pf: pointer to the PF structure
1774	*
1775	* Disable all currently configured clock outputs. This is necessary before
1776	* certain changes to the PTP hardware clock. Use ice_ptp_enable_all_clkout to
1777	* re-enable the clocks again.
1778	*/
1779	static void ice_ptp_disable_all_clkout(struct ice_pf *pf)
1780	{
1781	uint i;
1782
1783	for (i = 0; i < pf->ptp.info.n_per_out; i++)
1784	if (pf->ptp.perout_channels[i].ena)
1785	ice_ptp_cfg_clkout(pf, chan: i, NULL, store: false);
1786	}
1787
1788	/**
1789	* ice_ptp_enable_all_clkout - Enable all configured periodic clock outputs
1790	* @pf: pointer to the PF structure
1791	*
1792	* Enable all currently configured clock outputs. Use this after
1793	* ice_ptp_disable_all_clkout to reconfigure the output signals according to
1794	* their configuration.
1795	*/
1796	static void ice_ptp_enable_all_clkout(struct ice_pf *pf)
1797	{
1798	uint i;
1799
1800	for (i = 0; i < pf->ptp.info.n_per_out; i++)
1801	if (pf->ptp.perout_channels[i].ena)
1802	ice_ptp_cfg_clkout(pf, chan: i, config: &pf->ptp.perout_channels[i],
1803	store: false);
1804	}
1805
1806	/**
1807	* ice_ptp_gpio_enable_e810 - Enable/disable ancillary features of PHC
1808	* @info: the driver's PTP info structure
1809	* @rq: The requested feature to change
1810	* @on: Enable/disable flag
1811	*/
1812	static int
1813	ice_ptp_gpio_enable_e810(struct ptp_clock_info *info,
1814	struct ptp_clock_request *rq, int on)
1815	{
1816	struct ice_pf *pf = ptp_info_to_pf(info);
1817	struct ice_perout_channel clk_cfg = {0};
1818	bool sma_pres = false;
1819	unsigned int chan;
1820	u32 gpio_pin;
1821	int err;
1822
1823	if (ice_is_feature_supported(pf, f: ICE_F_SMA_CTRL))
1824	sma_pres = true;
1825
1826	switch (rq->type) {
1827	case PTP_CLK_REQ_PEROUT:
1828	chan = rq->perout.index;
1829	if (sma_pres) {
1830	if (chan == ice_pin_desc_e810t[SMA1].chan)
1831	clk_cfg.gpio_pin = GPIO_20;
1832	else if (chan == ice_pin_desc_e810t[SMA2].chan)
1833	clk_cfg.gpio_pin = GPIO_22;
1834	else
1835	return -1;
1836	} else if (ice_is_e810t(hw: &pf->hw)) {
1837	if (chan == 0)
1838	clk_cfg.gpio_pin = GPIO_20;
1839	else
1840	clk_cfg.gpio_pin = GPIO_22;
1841	} else if (chan == PPS_CLK_GEN_CHAN) {
1842	clk_cfg.gpio_pin = PPS_PIN_INDEX;
1843	} else {
1844	clk_cfg.gpio_pin = chan;
1845	}
1846
1847	clk_cfg.period = ((rq->perout.period.sec * NSEC_PER_SEC) +
1848	rq->perout.period.nsec);
1849	clk_cfg.start_time = ((rq->perout.start.sec * NSEC_PER_SEC) +
1850	rq->perout.start.nsec);
1851	clk_cfg.ena = !!on;
1852
1853	err = ice_ptp_cfg_clkout(pf, chan, config: &clk_cfg, store: true);
1854	break;
1855	case PTP_CLK_REQ_EXTTS:
1856	chan = rq->extts.index;
1857	if (sma_pres) {
1858	if (chan < ice_pin_desc_e810t[SMA2].chan)
1859	gpio_pin = GPIO_21;
1860	else
1861	gpio_pin = GPIO_23;
1862	} else if (ice_is_e810t(hw: &pf->hw)) {
1863	if (chan == 0)
1864	gpio_pin = GPIO_21;
1865	else
1866	gpio_pin = GPIO_23;
1867	} else {
1868	gpio_pin = chan;
1869	}
1870
1871	err = ice_ptp_cfg_extts(pf, ena: !!on, chan, gpio_pin,
1872	extts_flags: rq->extts.flags);
1873	break;
1874	default:
1875	return -EOPNOTSUPP;
1876	}
1877
1878	return err;
1879	}
1880
1881	/**
1882	* ice_ptp_gpio_enable_e823 - Enable/disable ancillary features of PHC
1883	* @info: the driver's PTP info structure
1884	* @rq: The requested feature to change
1885	* @on: Enable/disable flag
1886	*/
1887	static int ice_ptp_gpio_enable_e823(struct ptp_clock_info *info,
1888	struct ptp_clock_request *rq, int on)
1889	{
1890	struct ice_pf *pf = ptp_info_to_pf(info);
1891	struct ice_perout_channel clk_cfg = {0};
1892	int err;
1893
1894	switch (rq->type) {
1895	case PTP_CLK_REQ_PPS:
1896	clk_cfg.gpio_pin = PPS_PIN_INDEX;
1897	clk_cfg.period = NSEC_PER_SEC;
1898	clk_cfg.ena = !!on;
1899
1900	err = ice_ptp_cfg_clkout(pf, PPS_CLK_GEN_CHAN, config: &clk_cfg, store: true);
1901	break;
1902	case PTP_CLK_REQ_EXTTS:
1903	err = ice_ptp_cfg_extts(pf, ena: !!on, chan: rq->extts.index,
1904	TIME_SYNC_PIN_INDEX, extts_flags: rq->extts.flags);
1905	break;
1906	default:
1907	return -EOPNOTSUPP;
1908	}
1909
1910	return err;
1911	}
1912
1913	/**
1914	* ice_ptp_gettimex64 - Get the time of the clock
1915	* @info: the driver's PTP info structure
1916	* @ts: timespec64 structure to hold the current time value
1917	* @sts: Optional parameter for holding a pair of system timestamps from
1918	* the system clock. Will be ignored if NULL is given.
1919	*
1920	* Read the device clock and return the correct value on ns, after converting it
1921	* into a timespec struct.
1922	*/
1923	static int
1924	ice_ptp_gettimex64(struct ptp_clock_info *info, struct timespec64 *ts,
1925	struct ptp_system_timestamp *sts)
1926	{
1927	struct ice_pf *pf = ptp_info_to_pf(info);
1928	struct ice_hw *hw = &pf->hw;
1929
1930	if (!ice_ptp_lock(hw)) {
1931	dev_err(ice_pf_to_dev(pf), "PTP failed to get time\n");
1932	return -EBUSY;
1933	}
1934
1935	ice_ptp_read_time(pf, ts, sts);
1936	ice_ptp_unlock(hw);
1937
1938	return 0;
1939	}
1940
1941	/**
1942	* ice_ptp_settime64 - Set the time of the clock
1943	* @info: the driver's PTP info structure
1944	* @ts: timespec64 structure that holds the new time value
1945	*
1946	* Set the device clock to the user input value. The conversion from timespec
1947	* to ns happens in the write function.
1948	*/
1949	static int
1950	ice_ptp_settime64(struct ptp_clock_info *info, const struct timespec64 *ts)
1951	{
1952	struct ice_pf *pf = ptp_info_to_pf(info);
1953	struct timespec64 ts64 = *ts;
1954	struct ice_hw *hw = &pf->hw;
1955	int err;
1956
1957	/* For Vernier mode, we need to recalibrate after new settime
1958	* Start with disabling timestamp block
1959	*/
1960	if (pf->ptp.port.link_up)
1961	ice_ptp_port_phy_stop(ptp_port: &pf->ptp.port);
1962
1963	if (!ice_ptp_lock(hw)) {
1964	err = -EBUSY;
1965	goto exit;
1966	}
1967
1968	/* Disable periodic outputs */
1969	ice_ptp_disable_all_clkout(pf);
1970
1971	err = ice_ptp_write_init(pf, ts: &ts64);
1972	ice_ptp_unlock(hw);
1973
1974	if (!err)
1975	ice_ptp_reset_cached_phctime(pf);
1976
1977	/* Reenable periodic outputs */
1978	ice_ptp_enable_all_clkout(pf);
1979
1980	/* Recalibrate and re-enable timestamp blocks for E822/E823 */
1981	if (hw->phy_model == ICE_PHY_E82X)
1982	ice_ptp_restart_all_phy(pf);
1983	exit:
1984	if (err) {
1985	dev_err(ice_pf_to_dev(pf), "PTP failed to set time %d\n", err);
1986	return err;
1987	}
1988
1989	return 0;
1990	}
1991
1992	/**
1993	* ice_ptp_adjtime_nonatomic - Do a non-atomic clock adjustment
1994	* @info: the driver's PTP info structure
1995	* @delta: Offset in nanoseconds to adjust the time by
1996	*/
1997	static int ice_ptp_adjtime_nonatomic(struct ptp_clock_info *info, s64 delta)
1998	{
1999	struct timespec64 now, then;
2000	int ret;
2001
2002	then = ns_to_timespec64(nsec: delta);
2003	ret = ice_ptp_gettimex64(info, ts: &now, NULL);
2004	if (ret)
2005	return ret;
2006	now = timespec64_add(lhs: now, rhs: then);
2007
2008	return ice_ptp_settime64(info, ts: (const struct timespec64 *)&now);
2009	}
2010
2011	/**
2012	* ice_ptp_adjtime - Adjust the time of the clock by the indicated delta
2013	* @info: the driver's PTP info structure
2014	* @delta: Offset in nanoseconds to adjust the time by
2015	*/
2016	static int ice_ptp_adjtime(struct ptp_clock_info *info, s64 delta)
2017	{
2018	struct ice_pf *pf = ptp_info_to_pf(info);
2019	struct ice_hw *hw = &pf->hw;
2020	struct device *dev;
2021	int err;
2022
2023	dev = ice_pf_to_dev(pf);
2024
2025	/* Hardware only supports atomic adjustments using signed 32-bit
2026	* integers. For any adjustment outside this range, perform
2027	* a non-atomic get->adjust->set flow.
2028	*/
2029	if (delta > S32_MAX || delta < S32_MIN) {
2030	dev_dbg(dev, "delta = %lld, adjtime non-atomic\n", delta);
2031	return ice_ptp_adjtime_nonatomic(info, delta);
2032	}
2033
2034	if (!ice_ptp_lock(hw)) {
2035	dev_err(dev, "PTP failed to acquire semaphore in adjtime\n");
2036	return -EBUSY;
2037	}
2038
2039	/* Disable periodic outputs */
2040	ice_ptp_disable_all_clkout(pf);
2041
2042	err = ice_ptp_write_adj(pf, adj: delta);
2043
2044	/* Reenable periodic outputs */
2045	ice_ptp_enable_all_clkout(pf);
2046
2047	ice_ptp_unlock(hw);
2048
2049	if (err) {
2050	dev_err(dev, "PTP failed to adjust time, err %d\n", err);
2051	return err;
2052	}
2053
2054	ice_ptp_reset_cached_phctime(pf);
2055
2056	return 0;
2057	}
2058
2059	#ifdef CONFIG_ICE_HWTS
2060	/**
2061	* ice_ptp_get_syncdevicetime - Get the cross time stamp info
2062	* @device: Current device time
2063	* @system: System counter value read synchronously with device time
2064	* @ctx: Context provided by timekeeping code
2065	*
2066	* Read device and system (ART) clock simultaneously and return the corrected
2067	* clock values in ns.
2068	*/
2069	static int
2070	ice_ptp_get_syncdevicetime(ktime_t *device,
2071	struct system_counterval_t *system,
2072	void *ctx)
2073	{
2074	struct ice_pf *pf = (struct ice_pf *)ctx;
2075	struct ice_hw *hw = &pf->hw;
2076	u32 hh_lock, hh_art_ctl;
2077	int i;
2078
2079	#define MAX_HH_HW_LOCK_TRIES 5
2080	#define MAX_HH_CTL_LOCK_TRIES 100
2081
2082	for (i = 0; i < MAX_HH_HW_LOCK_TRIES; i++) {
2083	/* Get the HW lock */
2084	hh_lock = rd32(hw, PFHH_SEM + (PFTSYN_SEM_BYTES * hw->pf_id));
2085	if (hh_lock & PFHH_SEM_BUSY_M) {
2086	usleep_range(min: 10000, max: 15000);
2087	continue;
2088	}
2089	break;
2090	}
2091	if (hh_lock & PFHH_SEM_BUSY_M) {
2092	dev_err(ice_pf_to_dev(pf), "PTP failed to get hh lock\n");
2093	return -EBUSY;
2094	}
2095
2096	/* Program cmd to master timer */
2097	ice_ptp_src_cmd(hw, cmd: ICE_PTP_READ_TIME);
2098
2099	/* Start the ART and device clock sync sequence */
2100	hh_art_ctl = rd32(hw, GLHH_ART_CTL);
2101	hh_art_ctl = hh_art_ctl | GLHH_ART_CTL_ACTIVE_M;
2102	wr32(hw, GLHH_ART_CTL, hh_art_ctl);
2103
2104	for (i = 0; i < MAX_HH_CTL_LOCK_TRIES; i++) {
2105	/* Wait for sync to complete */
2106	hh_art_ctl = rd32(hw, GLHH_ART_CTL);
2107	if (hh_art_ctl & GLHH_ART_CTL_ACTIVE_M) {
2108	udelay(1);
2109	continue;
2110	} else {
2111	u32 hh_ts_lo, hh_ts_hi, tmr_idx;
2112	u64 hh_ts;
2113
2114	tmr_idx = hw->func_caps.ts_func_info.tmr_index_assoc;
2115	/* Read ART time */
2116	hh_ts_lo = rd32(hw, GLHH_ART_TIME_L);
2117	hh_ts_hi = rd32(hw, GLHH_ART_TIME_H);
2118	hh_ts = ((u64)hh_ts_hi << 32) | hh_ts_lo;
2119	*system = convert_art_ns_to_tsc(art_ns: hh_ts);
2120	/* Read Device source clock time */
2121	hh_ts_lo = rd32(hw, GLTSYN_HHTIME_L(tmr_idx));
2122	hh_ts_hi = rd32(hw, GLTSYN_HHTIME_H(tmr_idx));
2123	hh_ts = ((u64)hh_ts_hi << 32) | hh_ts_lo;
2124	*device = ns_to_ktime(ns: hh_ts);
2125	break;
2126	}
2127	}
2128
2129	/* Clear the master timer */
2130	ice_ptp_src_cmd(hw, cmd: ICE_PTP_NOP);
2131
2132	/* Release HW lock */
2133	hh_lock = rd32(hw, PFHH_SEM + (PFTSYN_SEM_BYTES * hw->pf_id));
2134	hh_lock = hh_lock & ~PFHH_SEM_BUSY_M;
2135	wr32(hw, PFHH_SEM + (PFTSYN_SEM_BYTES * hw->pf_id), hh_lock);
2136
2137	if (i == MAX_HH_CTL_LOCK_TRIES)
2138	return -ETIMEDOUT;
2139
2140	return 0;
2141	}
2142
2143	/**
2144	* ice_ptp_getcrosststamp_e82x - Capture a device cross timestamp
2145	* @info: the driver's PTP info structure
2146	* @cts: The memory to fill the cross timestamp info
2147	*
2148	* Capture a cross timestamp between the ART and the device PTP hardware
2149	* clock. Fill the cross timestamp information and report it back to the
2150	* caller.
2151	*
2152	* This is only valid for E822 and E823 devices which have support for
2153	* generating the cross timestamp via PCIe PTM.
2154	*
2155	* In order to correctly correlate the ART timestamp back to the TSC time, the
2156	* CPU must have X86_FEATURE_TSC_KNOWN_FREQ.
2157	*/
2158	static int
2159	ice_ptp_getcrosststamp_e82x(struct ptp_clock_info *info,
2160	struct system_device_crosststamp *cts)
2161	{
2162	struct ice_pf *pf = ptp_info_to_pf(info);
2163
2164	return get_device_system_crosststamp(get_time_fn: ice_ptp_get_syncdevicetime,
2165	ctx: pf, NULL, xtstamp: cts);
2166	}
2167	#endif /* CONFIG_ICE_HWTS */
2168
2169	/**
2170	* ice_ptp_get_ts_config - ioctl interface to read the timestamping config
2171	* @pf: Board private structure
2172	* @ifr: ioctl data
2173	*
2174	* Copy the timestamping config to user buffer
2175	*/
2176	int ice_ptp_get_ts_config(struct ice_pf *pf, struct ifreq *ifr)
2177	{
2178	struct hwtstamp_config *config;
2179
2180	if (pf->ptp.state != ICE_PTP_READY)
2181	return -EIO;
2182
2183	config = &pf->ptp.tstamp_config;
2184
2185	return copy_to_user(to: ifr->ifr_data, from: config, n: sizeof(*config)) ?
2186	-EFAULT : 0;
2187	}
2188
2189	/**
2190	* ice_ptp_set_timestamp_mode - Setup driver for requested timestamp mode
2191	* @pf: Board private structure
2192	* @config: hwtstamp settings requested or saved
2193	*/
2194	static int
2195	ice_ptp_set_timestamp_mode(struct ice_pf *pf, struct hwtstamp_config *config)
2196	{
2197	switch (config->tx_type) {
2198	case HWTSTAMP_TX_OFF:
2199	pf->ptp.tstamp_config.tx_type = HWTSTAMP_TX_OFF;
2200	break;
2201	case HWTSTAMP_TX_ON:
2202	pf->ptp.tstamp_config.tx_type = HWTSTAMP_TX_ON;
2203	break;
2204	default:
2205	return -ERANGE;
2206	}
2207
2208	switch (config->rx_filter) {
2209	case HWTSTAMP_FILTER_NONE:
2210	pf->ptp.tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE;
2211	break;
2212	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
2213	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
2214	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
2215	case HWTSTAMP_FILTER_PTP_V2_EVENT:
2216	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
2217	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
2218	case HWTSTAMP_FILTER_PTP_V2_SYNC:
2219	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
2220	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
2221	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
2222	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
2223	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
2224	case HWTSTAMP_FILTER_NTP_ALL:
2225	case HWTSTAMP_FILTER_ALL:
2226	pf->ptp.tstamp_config.rx_filter = HWTSTAMP_FILTER_ALL;
2227	break;
2228	default:
2229	return -ERANGE;
2230	}
2231
2232	/* Immediately update the device timestamping mode */
2233	ice_ptp_restore_timestamp_mode(pf);
2234
2235	return 0;
2236	}
2237
2238	/**
2239	* ice_ptp_set_ts_config - ioctl interface to control the timestamping
2240	* @pf: Board private structure
2241	* @ifr: ioctl data
2242	*
2243	* Get the user config and store it
2244	*/
2245	int ice_ptp_set_ts_config(struct ice_pf *pf, struct ifreq *ifr)
2246	{
2247	struct hwtstamp_config config;
2248	int err;
2249
2250	if (pf->ptp.state != ICE_PTP_READY)
2251	return -EAGAIN;
2252
2253	if (copy_from_user(to: &config, from: ifr->ifr_data, n: sizeof(config)))
2254	return -EFAULT;
2255
2256	err = ice_ptp_set_timestamp_mode(pf, config: &config);
2257	if (err)
2258	return err;
2259
2260	/* Return the actual configuration set */
2261	config = pf->ptp.tstamp_config;
2262
2263	return copy_to_user(to: ifr->ifr_data, from: &config, n: sizeof(config)) ?
2264	-EFAULT : 0;
2265	}
2266
2267	/**
2268	* ice_ptp_get_rx_hwts - Get packet Rx timestamp in ns
2269	* @rx_desc: Receive descriptor
2270	* @pkt_ctx: Packet context to get the cached time
2271	*
2272	* The driver receives a notification in the receive descriptor with timestamp.
2273	*/
2274	u64 ice_ptp_get_rx_hwts(const union ice_32b_rx_flex_desc *rx_desc,
2275	const struct ice_pkt_ctx *pkt_ctx)
2276	{
2277	u64 ts_ns, cached_time;
2278	u32 ts_high;
2279
2280	if (!(rx_desc->wb.time_stamp_low & ICE_PTP_TS_VALID))
2281	return 0;
2282
2283	cached_time = READ_ONCE(pkt_ctx->cached_phctime);
2284
2285	/* Do not report a timestamp if we don't have a cached PHC time */
2286	if (!cached_time)
2287	return 0;
2288
2289	/* Use ice_ptp_extend_32b_ts directly, using the ring-specific cached
2290	* PHC value, rather than accessing the PF. This also allows us to
2291	* simply pass the upper 32bits of nanoseconds directly. Calling
2292	* ice_ptp_extend_40b_ts is unnecessary as it would just discard these
2293	* bits itself.
2294	*/
2295	ts_high = le32_to_cpu(rx_desc->wb.flex_ts.ts_high);
2296	ts_ns = ice_ptp_extend_32b_ts(cached_phc_time: cached_time, in_tstamp: ts_high);
2297
2298	return ts_ns;
2299	}
2300
2301	/**
2302	* ice_ptp_disable_sma_pins_e810t - Disable E810-T SMA pins
2303	* @pf: pointer to the PF structure
2304	* @info: PTP clock info structure
2305	*
2306	* Disable the OS access to the SMA pins. Called to clear out the OS
2307	* indications of pin support when we fail to setup the E810-T SMA control
2308	* register.
2309	*/
2310	static void
2311	ice_ptp_disable_sma_pins_e810t(struct ice_pf *pf, struct ptp_clock_info *info)
2312	{
2313	struct device *dev = ice_pf_to_dev(pf);
2314
2315	dev_warn(dev, "Failed to configure E810-T SMA pin control\n");
2316
2317	info->enable = NULL;
2318	info->verify = NULL;
2319	info->n_pins = 0;
2320	info->n_ext_ts = 0;
2321	info->n_per_out = 0;
2322	}
2323
2324	/**
2325	* ice_ptp_setup_sma_pins_e810t - Setup the SMA pins
2326	* @pf: pointer to the PF structure
2327	* @info: PTP clock info structure
2328	*
2329	* Finish setting up the SMA pins by allocating pin_config, and setting it up
2330	* according to the current status of the SMA. On failure, disable all of the
2331	* extended SMA pin support.
2332	*/
2333	static void
2334	ice_ptp_setup_sma_pins_e810t(struct ice_pf *pf, struct ptp_clock_info *info)
2335	{
2336	struct device *dev = ice_pf_to_dev(pf);
2337	int err;
2338
2339	/* Allocate memory for kernel pins interface */
2340	info->pin_config = devm_kcalloc(dev, n: info->n_pins,
2341	size: sizeof(*info->pin_config), GFP_KERNEL);
2342	if (!info->pin_config) {
2343	ice_ptp_disable_sma_pins_e810t(pf, info);
2344	return;
2345	}
2346
2347	/* Read current SMA status */
2348	err = ice_get_sma_config_e810t(hw: &pf->hw, ptp_pins: info->pin_config);
2349	if (err)
2350	ice_ptp_disable_sma_pins_e810t(pf, info);
2351	}
2352
2353	/**
2354	* ice_ptp_setup_pins_e810 - Setup PTP pins in sysfs
2355	* @pf: pointer to the PF instance
2356	* @info: PTP clock capabilities
2357	*/
2358	static void
2359	ice_ptp_setup_pins_e810(struct ice_pf *pf, struct ptp_clock_info *info)
2360	{
2361	if (ice_is_feature_supported(pf, f: ICE_F_SMA_CTRL)) {
2362	info->n_ext_ts = N_EXT_TS_E810;
2363	info->n_per_out = N_PER_OUT_E810T;
2364	info->n_pins = NUM_PTP_PINS_E810T;
2365	info->verify = ice_verify_pin_e810t;
2366
2367	/* Complete setup of the SMA pins */
2368	ice_ptp_setup_sma_pins_e810t(pf, info);
2369	} else if (ice_is_e810t(hw: &pf->hw)) {
2370	info->n_ext_ts = N_EXT_TS_NO_SMA_E810T;
2371	info->n_per_out = N_PER_OUT_NO_SMA_E810T;
2372	} else {
2373	info->n_per_out = N_PER_OUT_E810;
2374	info->n_ext_ts = N_EXT_TS_E810;
2375	}
2376	}
2377
2378	/**
2379	* ice_ptp_setup_pins_e823 - Setup PTP pins in sysfs
2380	* @pf: pointer to the PF instance
2381	* @info: PTP clock capabilities
2382	*/
2383	static void
2384	ice_ptp_setup_pins_e823(struct ice_pf *pf, struct ptp_clock_info *info)
2385	{
2386	info->pps = 1;
2387	info->n_per_out = 0;
2388	info->n_ext_ts = 1;
2389	}
2390
2391	/**
2392	* ice_ptp_set_funcs_e82x - Set specialized functions for E82x support
2393	* @pf: Board private structure
2394	* @info: PTP info to fill
2395	*
2396	* Assign functions to the PTP capabiltiies structure for E82x devices.
2397	* Functions which operate across all device families should be set directly
2398	* in ice_ptp_set_caps. Only add functions here which are distinct for E82x
2399	* devices.
2400	*/
2401	static void
2402	ice_ptp_set_funcs_e82x(struct ice_pf *pf, struct ptp_clock_info *info)
2403	{
2404	#ifdef CONFIG_ICE_HWTS
2405	if (boot_cpu_has(X86_FEATURE_ART) &&
2406	boot_cpu_has(X86_FEATURE_TSC_KNOWN_FREQ))
2407	info->getcrosststamp = ice_ptp_getcrosststamp_e82x;
2408	#endif /* CONFIG_ICE_HWTS */
2409	}
2410
2411	/**
2412	* ice_ptp_set_funcs_e810 - Set specialized functions for E810 support
2413	* @pf: Board private structure
2414	* @info: PTP info to fill
2415	*
2416	* Assign functions to the PTP capabiltiies structure for E810 devices.
2417	* Functions which operate across all device families should be set directly
2418	* in ice_ptp_set_caps. Only add functions here which are distinct for e810
2419	* devices.
2420	*/
2421	static void
2422	ice_ptp_set_funcs_e810(struct ice_pf *pf, struct ptp_clock_info *info)
2423	{
2424	info->enable = ice_ptp_gpio_enable_e810;
2425	ice_ptp_setup_pins_e810(pf, info);
2426	}
2427
2428	/**
2429	* ice_ptp_set_funcs_e823 - Set specialized functions for E823 support
2430	* @pf: Board private structure
2431	* @info: PTP info to fill
2432	*
2433	* Assign functions to the PTP capabiltiies structure for E823 devices.
2434	* Functions which operate across all device families should be set directly
2435	* in ice_ptp_set_caps. Only add functions here which are distinct for e823
2436	* devices.
2437	*/
2438	static void
2439	ice_ptp_set_funcs_e823(struct ice_pf *pf, struct ptp_clock_info *info)
2440	{
2441	ice_ptp_set_funcs_e82x(pf, info);
2442
2443	info->enable = ice_ptp_gpio_enable_e823;
2444	ice_ptp_setup_pins_e823(pf, info);
2445	}
2446
2447	/**
2448	* ice_ptp_set_caps - Set PTP capabilities
2449	* @pf: Board private structure
2450	*/
2451	static void ice_ptp_set_caps(struct ice_pf *pf)
2452	{
2453	struct ptp_clock_info *info = &pf->ptp.info;
2454	struct device *dev = ice_pf_to_dev(pf);
2455
2456	snprintf(buf: info->name, size: sizeof(info->name) - 1, fmt: "%s-%s-clk",
2457	dev_driver_string(dev), dev_name(dev));
2458	info->owner = THIS_MODULE;
2459	info->max_adj = 100000000;
2460	info->adjtime = ice_ptp_adjtime;
2461	info->adjfine = ice_ptp_adjfine;
2462	info->gettimex64 = ice_ptp_gettimex64;
2463	info->settime64 = ice_ptp_settime64;
2464
2465	if (ice_is_e810(hw: &pf->hw))
2466	ice_ptp_set_funcs_e810(pf, info);
2467	else if (ice_is_e823(hw: &pf->hw))
2468	ice_ptp_set_funcs_e823(pf, info);
2469	else
2470	ice_ptp_set_funcs_e82x(pf, info);
2471	}
2472
2473	/**
2474	* ice_ptp_create_clock - Create PTP clock device for userspace
2475	* @pf: Board private structure
2476	*
2477	* This function creates a new PTP clock device. It only creates one if we
2478	* don't already have one. Will return error if it can't create one, but success
2479	* if we already have a device. Should be used by ice_ptp_init to create clock
2480	* initially, and prevent global resets from creating new clock devices.
2481	*/
2482	static long ice_ptp_create_clock(struct ice_pf *pf)
2483	{
2484	struct ptp_clock_info *info;
2485	struct device *dev;
2486
2487	/* No need to create a clock device if we already have one */
2488	if (pf->ptp.clock)
2489	return 0;
2490
2491	ice_ptp_set_caps(pf);
2492
2493	info = &pf->ptp.info;
2494	dev = ice_pf_to_dev(pf);
2495
2496	/* Attempt to register the clock before enabling the hardware. */
2497	pf->ptp.clock = ptp_clock_register(info, parent: dev);
2498	if (IS_ERR(ptr: pf->ptp.clock)) {
2499	dev_err(ice_pf_to_dev(pf), "Failed to register PTP clock device");
2500	return PTR_ERR(ptr: pf->ptp.clock);
2501	}
2502
2503	return 0;
2504	}
2505
2506	/**
2507	* ice_ptp_request_ts - Request an available Tx timestamp index
2508	* @tx: the PTP Tx timestamp tracker to request from
2509	* @skb: the SKB to associate with this timestamp request
2510	*/
2511	s8 ice_ptp_request_ts(struct ice_ptp_tx *tx, struct sk_buff *skb)
2512	{
2513	unsigned long flags;
2514	u8 idx;
2515
2516	spin_lock_irqsave(&tx->lock, flags);
2517
2518	/* Check that this tracker is accepting new timestamp requests */
2519	if (!ice_ptp_is_tx_tracker_up(tx)) {
2520	spin_unlock_irqrestore(lock: &tx->lock, flags);
2521	return -1;
2522	}
2523
2524	/* Find and set the first available index */
2525	idx = find_next_zero_bit(addr: tx->in_use, size: tx->len,
2526	offset: tx->last_ll_ts_idx_read + 1);
2527	if (idx == tx->len)
2528	idx = find_first_zero_bit(addr: tx->in_use, size: tx->len);
2529
2530	if (idx < tx->len) {
2531	/* We got a valid index that no other thread could have set. Store
2532	* a reference to the skb and the start time to allow discarding old
2533	* requests.
2534	*/
2535	set_bit(nr: idx, addr: tx->in_use);
2536	clear_bit(nr: idx, addr: tx->stale);
2537	tx->tstamps[idx].start = jiffies;
2538	tx->tstamps[idx].skb = skb_get(skb);
2539	skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
2540	ice_trace(tx_tstamp_request, skb, idx);
2541	}
2542
2543	spin_unlock_irqrestore(lock: &tx->lock, flags);
2544
2545	/* return the appropriate PHY timestamp register index, -1 if no
2546	* indexes were available.
2547	*/
2548	if (idx >= tx->len)
2549	return -1;
2550	else
2551	return idx + tx->offset;
2552	}
2553
2554	/**
2555	* ice_ptp_process_ts - Process the PTP Tx timestamps
2556	* @pf: Board private structure
2557	*
2558	* Returns: ICE_TX_TSTAMP_WORK_PENDING if there are any outstanding Tx
2559	* timestamps that need processing, and ICE_TX_TSTAMP_WORK_DONE otherwise.
2560	*/
2561	enum ice_tx_tstamp_work ice_ptp_process_ts(struct ice_pf *pf)
2562	{
2563	switch (pf->ptp.tx_interrupt_mode) {
2564	case ICE_PTP_TX_INTERRUPT_NONE:
2565	/* This device has the clock owner handle timestamps for it */
2566	return ICE_TX_TSTAMP_WORK_DONE;
2567	case ICE_PTP_TX_INTERRUPT_SELF:
2568	/* This device handles its own timestamps */
2569	return ice_ptp_tx_tstamp(tx: &pf->ptp.port.tx);
2570	case ICE_PTP_TX_INTERRUPT_ALL:
2571	/* This device handles timestamps for all ports */
2572	return ice_ptp_tx_tstamp_owner(pf);
2573	default:
2574	WARN_ONCE(1, "Unexpected Tx timestamp interrupt mode %u\n",
2575	pf->ptp.tx_interrupt_mode);
2576	return ICE_TX_TSTAMP_WORK_DONE;
2577	}
2578	}
2579
2580	/**
2581	* ice_ptp_maybe_trigger_tx_interrupt - Trigger Tx timstamp interrupt
2582	* @pf: Board private structure
2583	*
2584	* The device PHY issues Tx timestamp interrupts to the driver for processing
2585	* timestamp data from the PHY. It will not interrupt again until all
2586	* current timestamp data is read. In rare circumstances, it is possible that
2587	* the driver fails to read all outstanding data.
2588	*
2589	* To avoid getting permanently stuck, periodically check if the PHY has
2590	* outstanding timestamp data. If so, trigger an interrupt from software to
2591	* process this data.
2592	*/
2593	static void ice_ptp_maybe_trigger_tx_interrupt(struct ice_pf *pf)
2594	{
2595	struct device *dev = ice_pf_to_dev(pf);
2596	struct ice_hw *hw = &pf->hw;
2597	bool trigger_oicr = false;
2598	unsigned int i;
2599
2600	if (ice_is_e810(hw))
2601	return;
2602
2603	if (!ice_pf_src_tmr_owned(pf))
2604	return;
2605
2606	for (i = 0; i < ICE_MAX_QUAD; i++) {
2607	u64 tstamp_ready;
2608	int err;
2609
2610	err = ice_get_phy_tx_tstamp_ready(hw: &pf->hw, block: i, tstamp_ready: &tstamp_ready);
2611	if (!err && tstamp_ready) {
2612	trigger_oicr = true;
2613	break;
2614	}
2615	}
2616
2617	if (trigger_oicr) {
2618	/* Trigger a software interrupt, to ensure this data
2619	* gets processed.
2620	*/
2621	dev_dbg(dev, "PTP periodic task detected waiting timestamps. Triggering Tx timestamp interrupt now.\n");
2622
2623	wr32(hw, PFINT_OICR, PFINT_OICR_TSYN_TX_M);
2624	ice_flush(hw);
2625	}
2626	}
2627
2628	static void ice_ptp_periodic_work(struct kthread_work *work)
2629	{
2630	struct ice_ptp *ptp = container_of(work, struct ice_ptp, work.work);
2631	struct ice_pf *pf = container_of(ptp, struct ice_pf, ptp);
2632	int err;
2633
2634	if (pf->ptp.state != ICE_PTP_READY)
2635	return;
2636
2637	err = ice_ptp_update_cached_phctime(pf);
2638
2639	ice_ptp_maybe_trigger_tx_interrupt(pf);
2640
2641	/* Run twice a second or reschedule if phc update failed */
2642	kthread_queue_delayed_work(worker: ptp->kworker, dwork: &ptp->work,
2643	delay: msecs_to_jiffies(m: err ? 10 : 500));
2644	}
2645
2646	/**
2647	* ice_ptp_prepare_for_reset - Prepare PTP for reset
2648	* @pf: Board private structure
2649	* @reset_type: the reset type being performed
2650	*/
2651	void ice_ptp_prepare_for_reset(struct ice_pf *pf, enum ice_reset_req reset_type)
2652	{
2653	struct ice_ptp *ptp = &pf->ptp;
2654	u8 src_tmr;
2655
2656	if (ptp->state != ICE_PTP_READY)
2657	return;
2658
2659	ptp->state = ICE_PTP_RESETTING;
2660
2661	/* Disable timestamping for both Tx and Rx */
2662	ice_ptp_disable_timestamp_mode(pf);
2663
2664	kthread_cancel_delayed_work_sync(work: &ptp->work);
2665
2666	if (reset_type == ICE_RESET_PFR)
2667	return;
2668
2669	ice_ptp_release_tx_tracker(pf, tx: &pf->ptp.port.tx);
2670
2671	/* Disable periodic outputs */
2672	ice_ptp_disable_all_clkout(pf);
2673
2674	src_tmr = ice_get_ptp_src_clock_index(hw: &pf->hw);
2675
2676	/* Disable source clock */
2677	wr32(&pf->hw, GLTSYN_ENA(src_tmr), (u32)~GLTSYN_ENA_TSYN_ENA_M);
2678
2679	/* Acquire PHC and system timer to restore after reset */
2680	ptp->reset_time = ktime_get_real_ns();
2681	}
2682
2683	/**
2684	* ice_ptp_rebuild_owner - Initialize PTP clock owner after reset
2685	* @pf: Board private structure
2686	*
2687	* Companion function for ice_ptp_rebuild() which handles tasks that only the
2688	* PTP clock owner instance should perform.
2689	*/
2690	static int ice_ptp_rebuild_owner(struct ice_pf *pf)
2691	{
2692	struct ice_ptp *ptp = &pf->ptp;
2693	struct ice_hw *hw = &pf->hw;
2694	struct timespec64 ts;
2695	u64 time_diff;
2696	int err;
2697
2698	err = ice_ptp_init_phc(hw);
2699	if (err)
2700	return err;
2701
2702	/* Acquire the global hardware lock */
2703	if (!ice_ptp_lock(hw)) {
2704	err = -EBUSY;
2705	return err;
2706	}
2707
2708	/* Write the increment time value to PHY and LAN */
2709	err = ice_ptp_write_incval(hw, incval: ice_base_incval(pf));
2710	if (err) {
2711	ice_ptp_unlock(hw);
2712	return err;
2713	}
2714
2715	/* Write the initial Time value to PHY and LAN using the cached PHC
2716	* time before the reset and time difference between stopping and
2717	* starting the clock.
2718	*/
2719	if (ptp->cached_phc_time) {
2720	time_diff = ktime_get_real_ns() - ptp->reset_time;
2721	ts = ns_to_timespec64(nsec: ptp->cached_phc_time + time_diff);
2722	} else {
2723	ts = ktime_to_timespec64(ktime_get_real());
2724	}
2725	err = ice_ptp_write_init(pf, ts: &ts);
2726	if (err) {
2727	ice_ptp_unlock(hw);
2728	return err;
2729	}
2730
2731	/* Release the global hardware lock */
2732	ice_ptp_unlock(hw);
2733
2734	/* Flush software tracking of any outstanding timestamps since we're
2735	* about to flush the PHY timestamp block.
2736	*/
2737	ice_ptp_flush_all_tx_tracker(pf);
2738
2739	if (!ice_is_e810(hw)) {
2740	/* Enable quad interrupts */
2741	err = ice_ptp_cfg_phy_interrupt(pf, ena: true, threshold: 1);
2742	if (err)
2743	return err;
2744
2745	ice_ptp_restart_all_phy(pf);
2746	}
2747
2748	return 0;
2749	}
2750
2751	/**
2752	* ice_ptp_rebuild - Initialize PTP hardware clock support after reset
2753	* @pf: Board private structure
2754	* @reset_type: the reset type being performed
2755	*/
2756	void ice_ptp_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type)
2757	{
2758	struct ice_ptp *ptp = &pf->ptp;
2759	int err;
2760
2761	if (ptp->state == ICE_PTP_READY) {
2762	ice_ptp_prepare_for_reset(pf, reset_type);
2763	} else if (ptp->state != ICE_PTP_RESETTING) {
2764	err = -EINVAL;
2765	dev_err(ice_pf_to_dev(pf), "PTP was not initialized\n");
2766	goto err;
2767	}
2768
2769	if (ice_pf_src_tmr_owned(pf) && reset_type != ICE_RESET_PFR) {
2770	err = ice_ptp_rebuild_owner(pf);
2771	if (err)
2772	goto err;
2773	}
2774
2775	ptp->state = ICE_PTP_READY;
2776
2777	/* Start periodic work going */
2778	kthread_queue_delayed_work(worker: ptp->kworker, dwork: &ptp->work, delay: 0);
2779
2780	dev_info(ice_pf_to_dev(pf), "PTP reset successful\n");
2781	return;
2782
2783	err:
2784	ptp->state = ICE_PTP_ERROR;
2785	dev_err(ice_pf_to_dev(pf), "PTP reset failed %d\n", err);
2786	}
2787
2788	/**
2789	* ice_ptp_aux_dev_to_aux_pf - Get auxiliary PF handle for the auxiliary device
2790	* @aux_dev: auxiliary device to get the auxiliary PF for
2791	*/
2792	static struct ice_pf *
2793	ice_ptp_aux_dev_to_aux_pf(struct auxiliary_device *aux_dev)
2794	{
2795	struct ice_ptp_port *aux_port;
2796	struct ice_ptp *aux_ptp;
2797
2798	aux_port = container_of(aux_dev, struct ice_ptp_port, aux_dev);
2799	aux_ptp = container_of(aux_port, struct ice_ptp, port);
2800
2801	return container_of(aux_ptp, struct ice_pf, ptp);
2802	}
2803
2804	/**
2805	* ice_ptp_aux_dev_to_owner_pf - Get PF handle for the auxiliary device
2806	* @aux_dev: auxiliary device to get the PF for
2807	*/
2808	static struct ice_pf *
2809	ice_ptp_aux_dev_to_owner_pf(struct auxiliary_device *aux_dev)
2810	{
2811	struct ice_ptp_port_owner *ports_owner;
2812	struct auxiliary_driver *aux_drv;
2813	struct ice_ptp *owner_ptp;
2814
2815	if (!aux_dev->dev.driver)
2816	return NULL;
2817
2818	aux_drv = to_auxiliary_drv(drv: aux_dev->dev.driver);
2819	ports_owner = container_of(aux_drv, struct ice_ptp_port_owner,
2820	aux_driver);
2821	owner_ptp = container_of(ports_owner, struct ice_ptp, ports_owner);
2822	return container_of(owner_ptp, struct ice_pf, ptp);
2823	}
2824
2825	/**
2826	* ice_ptp_auxbus_probe - Probe auxiliary devices
2827	* @aux_dev: PF's auxiliary device
2828	* @id: Auxiliary device ID
2829	*/
2830	static int ice_ptp_auxbus_probe(struct auxiliary_device *aux_dev,
2831	const struct auxiliary_device_id *id)
2832	{
2833	struct ice_pf *owner_pf = ice_ptp_aux_dev_to_owner_pf(aux_dev);
2834	struct ice_pf *aux_pf = ice_ptp_aux_dev_to_aux_pf(aux_dev);
2835
2836	if (WARN_ON(!owner_pf))
2837	return -ENODEV;
2838
2839	INIT_LIST_HEAD(list: &aux_pf->ptp.port.list_member);
2840	mutex_lock(&owner_pf->ptp.ports_owner.lock);
2841	list_add(new: &aux_pf->ptp.port.list_member,
2842	head: &owner_pf->ptp.ports_owner.ports);
2843	mutex_unlock(lock: &owner_pf->ptp.ports_owner.lock);
2844
2845	return 0;
2846	}
2847
2848	/**
2849	* ice_ptp_auxbus_remove - Remove auxiliary devices from the bus
2850	* @aux_dev: PF's auxiliary device
2851	*/
2852	static void ice_ptp_auxbus_remove(struct auxiliary_device *aux_dev)
2853	{
2854	struct ice_pf *owner_pf = ice_ptp_aux_dev_to_owner_pf(aux_dev);
2855	struct ice_pf *aux_pf = ice_ptp_aux_dev_to_aux_pf(aux_dev);
2856
2857	mutex_lock(&owner_pf->ptp.ports_owner.lock);
2858	list_del(entry: &aux_pf->ptp.port.list_member);
2859	mutex_unlock(lock: &owner_pf->ptp.ports_owner.lock);
2860	}
2861
2862	/**
2863	* ice_ptp_auxbus_shutdown
2864	* @aux_dev: PF's auxiliary device
2865	*/
2866	static void ice_ptp_auxbus_shutdown(struct auxiliary_device *aux_dev)
2867	{
2868	/* Doing nothing here, but handle to auxbus driver must be satisfied */
2869	}
2870
2871	/**
2872	* ice_ptp_auxbus_suspend
2873	* @aux_dev: PF's auxiliary device
2874	* @state: power management state indicator
2875	*/
2876	static int
2877	ice_ptp_auxbus_suspend(struct auxiliary_device *aux_dev, pm_message_t state)
2878	{
2879	/* Doing nothing here, but handle to auxbus driver must be satisfied */
2880	return 0;
2881	}
2882
2883	/**
2884	* ice_ptp_auxbus_resume
2885	* @aux_dev: PF's auxiliary device
2886	*/
2887	static int ice_ptp_auxbus_resume(struct auxiliary_device *aux_dev)
2888	{
2889	/* Doing nothing here, but handle to auxbus driver must be satisfied */
2890	return 0;
2891	}
2892
2893	/**
2894	* ice_ptp_auxbus_create_id_table - Create auxiliary device ID table
2895	* @pf: Board private structure
2896	* @name: auxiliary bus driver name
2897	*/
2898	static struct auxiliary_device_id *
2899	ice_ptp_auxbus_create_id_table(struct ice_pf *pf, const char *name)
2900	{
2901	struct auxiliary_device_id *ids;
2902
2903	/* Second id left empty to terminate the array */
2904	ids = devm_kcalloc(ice_pf_to_dev(pf), n: 2,
2905	size: sizeof(struct auxiliary_device_id), GFP_KERNEL);
2906	if (!ids)
2907	return NULL;
2908
2909	snprintf(buf: ids[0].name, size: sizeof(ids[0].name), fmt: "ice.%s", name);
2910
2911	return ids;
2912	}
2913
2914	/**
2915	* ice_ptp_register_auxbus_driver - Register PTP auxiliary bus driver
2916	* @pf: Board private structure
2917	*/
2918	static int ice_ptp_register_auxbus_driver(struct ice_pf *pf)
2919	{
2920	struct auxiliary_driver *aux_driver;
2921	struct ice_ptp *ptp;
2922	struct device *dev;
2923	char *name;
2924	int err;
2925
2926	ptp = &pf->ptp;
2927	dev = ice_pf_to_dev(pf);
2928	aux_driver = &ptp->ports_owner.aux_driver;
2929	INIT_LIST_HEAD(list: &ptp->ports_owner.ports);
2930	mutex_init(&ptp->ports_owner.lock);
2931	name = devm_kasprintf(dev, GFP_KERNEL, fmt: "ptp_aux_dev_%u_%u_clk%u",
2932	pf->pdev->bus->number, PCI_SLOT(pf->pdev->devfn),
2933	ice_get_ptp_src_clock_index(hw: &pf->hw));
2934	if (!name)
2935	return -ENOMEM;
2936
2937	aux_driver->name = name;
2938	aux_driver->shutdown = ice_ptp_auxbus_shutdown;
2939	aux_driver->suspend = ice_ptp_auxbus_suspend;
2940	aux_driver->remove = ice_ptp_auxbus_remove;
2941	aux_driver->resume = ice_ptp_auxbus_resume;
2942	aux_driver->probe = ice_ptp_auxbus_probe;
2943	aux_driver->id_table = ice_ptp_auxbus_create_id_table(pf, name);
2944	if (!aux_driver->id_table)
2945	return -ENOMEM;
2946
2947	err = auxiliary_driver_register(aux_driver);
2948	if (err) {
2949	devm_kfree(dev, p: aux_driver->id_table);
2950	dev_err(dev, "Failed registering aux_driver, name <%s>\n",
2951	name);
2952	}
2953
2954	return err;
2955	}
2956
2957	/**
2958	* ice_ptp_unregister_auxbus_driver - Unregister PTP auxiliary bus driver
2959	* @pf: Board private structure
2960	*/
2961	static void ice_ptp_unregister_auxbus_driver(struct ice_pf *pf)
2962	{
2963	struct auxiliary_driver *aux_driver = &pf->ptp.ports_owner.aux_driver;
2964
2965	auxiliary_driver_unregister(auxdrv: aux_driver);
2966	devm_kfree(ice_pf_to_dev(pf), p: aux_driver->id_table);
2967
2968	mutex_destroy(lock: &pf->ptp.ports_owner.lock);
2969	}
2970
2971	/**
2972	* ice_ptp_clock_index - Get the PTP clock index for this device
2973	* @pf: Board private structure
2974	*
2975	* Returns: the PTP clock index associated with this PF, or -1 if no PTP clock
2976	* is associated.
2977	*/
2978	int ice_ptp_clock_index(struct ice_pf *pf)
2979	{
2980	struct auxiliary_device *aux_dev;
2981	struct ice_pf *owner_pf;
2982	struct ptp_clock *clock;
2983
2984	aux_dev = &pf->ptp.port.aux_dev;
2985	owner_pf = ice_ptp_aux_dev_to_owner_pf(aux_dev);
2986	if (!owner_pf)
2987	return -1;
2988	clock = owner_pf->ptp.clock;
2989
2990	return clock ? ptp_clock_index(ptp: clock) : -1;
2991	}
2992
2993	/**
2994	* ice_ptp_init_owner - Initialize PTP_1588_CLOCK device
2995	* @pf: Board private structure
2996	*
2997	* Setup and initialize a PTP clock device that represents the device hardware
2998	* clock. Save the clock index for other functions connected to the same
2999	* hardware resource.
3000	*/
3001	static int ice_ptp_init_owner(struct ice_pf *pf)
3002	{
3003	struct ice_hw *hw = &pf->hw;
3004	struct timespec64 ts;
3005	int err;
3006
3007	err = ice_ptp_init_phc(hw);
3008	if (err) {
3009	dev_err(ice_pf_to_dev(pf), "Failed to initialize PHC, err %d\n",
3010	err);
3011	return err;
3012	}
3013
3014	/* Acquire the global hardware lock */
3015	if (!ice_ptp_lock(hw)) {
3016	err = -EBUSY;
3017	goto err_exit;
3018	}
3019
3020	/* Write the increment time value to PHY and LAN */
3021	err = ice_ptp_write_incval(hw, incval: ice_base_incval(pf));
3022	if (err) {
3023	ice_ptp_unlock(hw);
3024	goto err_exit;
3025	}
3026
3027	ts = ktime_to_timespec64(ktime_get_real());
3028	/* Write the initial Time value to PHY and LAN */
3029	err = ice_ptp_write_init(pf, ts: &ts);
3030	if (err) {
3031	ice_ptp_unlock(hw);
3032	goto err_exit;
3033	}
3034
3035	/* Release the global hardware lock */
3036	ice_ptp_unlock(hw);
3037
3038	if (!ice_is_e810(hw)) {
3039	/* Enable quad interrupts */
3040	err = ice_ptp_cfg_phy_interrupt(pf, ena: true, threshold: 1);
3041	if (err)
3042	goto err_exit;
3043	}
3044
3045	/* Ensure we have a clock device */
3046	err = ice_ptp_create_clock(pf);
3047	if (err)
3048	goto err_clk;
3049
3050	err = ice_ptp_register_auxbus_driver(pf);
3051	if (err) {
3052	dev_err(ice_pf_to_dev(pf), "Failed to register PTP auxbus driver");
3053	goto err_aux;
3054	}
3055
3056	return 0;
3057	err_aux:
3058	ptp_clock_unregister(ptp: pf->ptp.clock);
3059	err_clk:
3060	pf->ptp.clock = NULL;
3061	err_exit:
3062	return err;
3063	}
3064
3065	/**
3066	* ice_ptp_init_work - Initialize PTP work threads
3067	* @pf: Board private structure
3068	* @ptp: PF PTP structure
3069	*/
3070	static int ice_ptp_init_work(struct ice_pf *pf, struct ice_ptp *ptp)
3071	{
3072	struct kthread_worker *kworker;
3073
3074	/* Initialize work functions */
3075	kthread_init_delayed_work(&ptp->work, ice_ptp_periodic_work);
3076
3077	/* Allocate a kworker for handling work required for the ports
3078	* connected to the PTP hardware clock.
3079	*/
3080	kworker = kthread_create_worker(flags: 0, namefmt: "ice-ptp-%s",
3081	dev_name(ice_pf_to_dev(pf)));
3082	if (IS_ERR(ptr: kworker))
3083	return PTR_ERR(ptr: kworker);
3084
3085	ptp->kworker = kworker;
3086
3087	/* Start periodic work going */
3088	kthread_queue_delayed_work(worker: ptp->kworker, dwork: &ptp->work, delay: 0);
3089
3090	return 0;
3091	}
3092
3093	/**
3094	* ice_ptp_init_port - Initialize PTP port structure
3095	* @pf: Board private structure
3096	* @ptp_port: PTP port structure
3097	*/
3098	static int ice_ptp_init_port(struct ice_pf *pf, struct ice_ptp_port *ptp_port)
3099	{
3100	struct ice_hw *hw = &pf->hw;
3101
3102	mutex_init(&ptp_port->ps_lock);
3103
3104	switch (hw->phy_model) {
3105	case ICE_PHY_E810:
3106	return ice_ptp_init_tx_e810(pf, tx: &ptp_port->tx);
3107	case ICE_PHY_E82X:
3108	kthread_init_delayed_work(&ptp_port->ov_work,
3109	ice_ptp_wait_for_offsets);
3110
3111	return ice_ptp_init_tx_e82x(pf, tx: &ptp_port->tx,
3112	port: ptp_port->port_num);
3113	default:
3114	return -ENODEV;
3115	}
3116	}
3117
3118	/**
3119	* ice_ptp_release_auxbus_device
3120	* @dev: device that utilizes the auxbus
3121	*/
3122	static void ice_ptp_release_auxbus_device(struct device *dev)
3123	{
3124	/* Doing nothing here, but handle to auxbux device must be satisfied */
3125	}
3126
3127	/**
3128	* ice_ptp_create_auxbus_device - Create PTP auxiliary bus device
3129	* @pf: Board private structure
3130	*/
3131	static int ice_ptp_create_auxbus_device(struct ice_pf *pf)
3132	{
3133	struct auxiliary_device *aux_dev;
3134	struct ice_ptp *ptp;
3135	struct device *dev;
3136	char *name;
3137	int err;
3138	u32 id;
3139
3140	ptp = &pf->ptp;
3141	id = ptp->port.port_num;
3142	dev = ice_pf_to_dev(pf);
3143
3144	aux_dev = &ptp->port.aux_dev;
3145
3146	name = devm_kasprintf(dev, GFP_KERNEL, fmt: "ptp_aux_dev_%u_%u_clk%u",
3147	pf->pdev->bus->number, PCI_SLOT(pf->pdev->devfn),
3148	ice_get_ptp_src_clock_index(hw: &pf->hw));
3149	if (!name)
3150	return -ENOMEM;
3151
3152	aux_dev->name = name;
3153	aux_dev->id = id;
3154	aux_dev->dev.release = ice_ptp_release_auxbus_device;
3155	aux_dev->dev.parent = dev;
3156
3157	err = auxiliary_device_init(auxdev: aux_dev);
3158	if (err)
3159	goto aux_err;
3160
3161	err = auxiliary_device_add(aux_dev);
3162	if (err) {
3163	auxiliary_device_uninit(auxdev: aux_dev);
3164	goto aux_err;
3165	}
3166
3167	return 0;
3168	aux_err:
3169	dev_err(dev, "Failed to create PTP auxiliary bus device <%s>\n", name);
3170	devm_kfree(dev, p: name);
3171	return err;
3172	}
3173
3174	/**
3175	* ice_ptp_remove_auxbus_device - Remove PTP auxiliary bus device
3176	* @pf: Board private structure
3177	*/
3178	static void ice_ptp_remove_auxbus_device(struct ice_pf *pf)
3179	{
3180	struct auxiliary_device *aux_dev = &pf->ptp.port.aux_dev;
3181
3182	auxiliary_device_delete(auxdev: aux_dev);
3183	auxiliary_device_uninit(auxdev: aux_dev);
3184
3185	memset(aux_dev, 0, sizeof(*aux_dev));
3186	}
3187
3188	/**
3189	* ice_ptp_init_tx_interrupt_mode - Initialize device Tx interrupt mode
3190	* @pf: Board private structure
3191	*
3192	* Initialize the Tx timestamp interrupt mode for this device. For most device
3193	* types, each PF processes the interrupt and manages its own timestamps. For
3194	* E822-based devices, only the clock owner processes the timestamps. Other
3195	* PFs disable the interrupt and do not process their own timestamps.
3196	*/
3197	static void ice_ptp_init_tx_interrupt_mode(struct ice_pf *pf)
3198	{
3199	switch (pf->hw.phy_model) {
3200	case ICE_PHY_E82X:
3201	/* E822 based PHY has the clock owner process the interrupt
3202	* for all ports.
3203	*/
3204	if (ice_pf_src_tmr_owned(pf))
3205	pf->ptp.tx_interrupt_mode = ICE_PTP_TX_INTERRUPT_ALL;
3206	else
3207	pf->ptp.tx_interrupt_mode = ICE_PTP_TX_INTERRUPT_NONE;
3208	break;
3209	default:
3210	/* other PHY types handle their own Tx interrupt */
3211	pf->ptp.tx_interrupt_mode = ICE_PTP_TX_INTERRUPT_SELF;
3212	}
3213	}
3214
3215	/**
3216	* ice_ptp_init - Initialize PTP hardware clock support
3217	* @pf: Board private structure
3218	*
3219	* Set up the device for interacting with the PTP hardware clock for all
3220	* functions, both the function that owns the clock hardware, and the
3221	* functions connected to the clock hardware.
3222	*
3223	* The clock owner will allocate and register a ptp_clock with the
3224	* PTP_1588_CLOCK infrastructure. All functions allocate a kthread and work
3225	* items used for asynchronous work such as Tx timestamps and periodic work.
3226	*/
3227	void ice_ptp_init(struct ice_pf *pf)
3228	{
3229	struct ice_ptp *ptp = &pf->ptp;
3230	struct ice_hw *hw = &pf->hw;
3231	int err;
3232
3233	ptp->state = ICE_PTP_INITIALIZING;
3234
3235	ice_ptp_init_phy_model(hw);
3236
3237	ice_ptp_init_tx_interrupt_mode(pf);
3238
3239	/* If this function owns the clock hardware, it must allocate and
3240	* configure the PTP clock device to represent it.
3241	*/
3242	if (ice_pf_src_tmr_owned(pf)) {
3243	err = ice_ptp_init_owner(pf);
3244	if (err)
3245	goto err;
3246	}
3247
3248	ptp->port.port_num = hw->pf_id;
3249	err = ice_ptp_init_port(pf, ptp_port: &ptp->port);
3250	if (err)
3251	goto err;
3252
3253	/* Start the PHY timestamping block */
3254	ice_ptp_reset_phy_timestamping(pf);
3255
3256	/* Configure initial Tx interrupt settings */
3257	ice_ptp_cfg_tx_interrupt(pf);
3258
3259	err = ice_ptp_create_auxbus_device(pf);
3260	if (err)
3261	goto err;
3262
3263	ptp->state = ICE_PTP_READY;
3264
3265	err = ice_ptp_init_work(pf, ptp);
3266	if (err)
3267	goto err;
3268
3269	dev_info(ice_pf_to_dev(pf), "PTP init successful\n");
3270	return;
3271
3272	err:
3273	/* If we registered a PTP clock, release it */
3274	if (pf->ptp.clock) {
3275	ptp_clock_unregister(ptp: ptp->clock);
3276	pf->ptp.clock = NULL;
3277	}
3278	ptp->state = ICE_PTP_ERROR;
3279	dev_err(ice_pf_to_dev(pf), "PTP failed %d\n", err);
3280	}
3281
3282	/**
3283	* ice_ptp_release - Disable the driver/HW support and unregister the clock
3284	* @pf: Board private structure
3285	*
3286	* This function handles the cleanup work required from the initialization by
3287	* clearing out the important information and unregistering the clock
3288	*/
3289	void ice_ptp_release(struct ice_pf *pf)
3290	{
3291	if (pf->ptp.state != ICE_PTP_READY)
3292	return;
3293
3294	pf->ptp.state = ICE_PTP_UNINIT;
3295
3296	/* Disable timestamping for both Tx and Rx */
3297	ice_ptp_disable_timestamp_mode(pf);
3298
3299	ice_ptp_remove_auxbus_device(pf);
3300
3301	ice_ptp_release_tx_tracker(pf, tx: &pf->ptp.port.tx);
3302
3303	kthread_cancel_delayed_work_sync(work: &pf->ptp.work);
3304
3305	ice_ptp_port_phy_stop(ptp_port: &pf->ptp.port);
3306	mutex_destroy(lock: &pf->ptp.port.ps_lock);
3307	if (pf->ptp.kworker) {
3308	kthread_destroy_worker(worker: pf->ptp.kworker);
3309	pf->ptp.kworker = NULL;
3310	}
3311
3312	if (ice_pf_src_tmr_owned(pf))
3313	ice_ptp_unregister_auxbus_driver(pf);
3314
3315	if (!pf->ptp.clock)
3316	return;
3317
3318	/* Disable periodic outputs */
3319	ice_ptp_disable_all_clkout(pf);
3320
3321	ptp_clock_unregister(ptp: pf->ptp.clock);
3322	pf->ptp.clock = NULL;
3323
3324	dev_info(ice_pf_to_dev(pf), "Removed PTP clock\n");
3325	}
3326