1	// SPDX-License-Identifier: GPL-2.0
2	/*
3	* Copyright (c) 2016-2017, Linaro Ltd
4	*/
5
6	#include <linux/idr.h>
7	#include <linux/interrupt.h>
8	#include <linux/io.h>
9	#include <linux/list.h>
10	#include <linux/mfd/syscon.h>
11	#include <linux/module.h>
12	#include <linux/of.h>
13	#include <linux/of_address.h>
14	#include <linux/platform_device.h>
15	#include <linux/regmap.h>
16	#include <linux/rpmsg.h>
17	#include <linux/sizes.h>
18	#include <linux/slab.h>
19	#include <linux/wait.h>
20	#include <linux/workqueue.h>
21	#include <linux/mailbox_client.h>
22
23	#include "rpmsg_internal.h"
24	#include "qcom_glink_native.h"
25
26	#define GLINK_NAME_SIZE 32
27	#define GLINK_VERSION_1 1
28
29	#define RPM_GLINK_CID_MIN 1
30	#define RPM_GLINK_CID_MAX 65536
31
32	struct glink_msg {
33	__le16 cmd;
34	__le16 param1;
35	__le32 param2;
36	u8 data[];
37	} __packed;
38
39	/**
40	* struct glink_defer_cmd - deferred incoming control message
41	* @node: list node
42	* @msg: message header
43	* @data: payload of the message
44	*
45	* Copy of a received control message, to be added to @rx_queue and processed
46	* by @rx_work of @qcom_glink.
47	*/
48	struct glink_defer_cmd {
49	struct list_head node;
50
51	struct glink_msg msg;
52	u8 data[];
53	};
54
55	/**
56	* struct glink_core_rx_intent - RX intent
57	* RX intent
58	*
59	* @data: pointer to the data (may be NULL for zero-copy)
60	* @id: remote or local intent ID
61	* @size: size of the original intent (do not modify)
62	* @reuse: To mark if the intent can be reused after first use
63	* @in_use: To mark if intent is already in use for the channel
64	* @offset: next write offset (initially 0)
65	* @node: list node
66	*/
67	struct glink_core_rx_intent {
68	void *data;
69	u32 id;
70	size_t size;
71	bool reuse;
72	bool in_use;
73	u32 offset;
74
75	struct list_head node;
76	};
77
78	/**
79	* struct qcom_glink - driver context, relates to one remote subsystem
80	* @dev: reference to the associated struct device
81	* @rx_pipe: pipe object for receive FIFO
82	* @tx_pipe: pipe object for transmit FIFO
83	* @rx_work: worker for handling received control messages
84	* @rx_lock: protects the @rx_queue
85	* @rx_queue: queue of received control messages to be processed in @rx_work
86	* @tx_lock: synchronizes operations on the tx fifo
87	* @idr_lock: synchronizes @lcids and @rcids modifications
88	* @lcids: idr of all channels with a known local channel id
89	* @rcids: idr of all channels with a known remote channel id
90	* @features: remote features
91	* @intentless: flag to indicate that there is no intent
92	* @tx_avail_notify: Waitqueue for pending tx tasks
93	* @sent_read_notify: flag to check cmd sent or not
94	* @abort_tx: flag indicating that all tx attempts should fail
95	*/
96	struct qcom_glink {
97	struct device *dev;
98
99	struct qcom_glink_pipe *rx_pipe;
100	struct qcom_glink_pipe *tx_pipe;
101
102	struct work_struct rx_work;
103	spinlock_t rx_lock;
104	struct list_head rx_queue;
105
106	spinlock_t tx_lock;
107
108	spinlock_t idr_lock;
109	struct idr lcids;
110	struct idr rcids;
111	unsigned long features;
112
113	bool intentless;
114	wait_queue_head_t tx_avail_notify;
115	bool sent_read_notify;
116
117	bool abort_tx;
118	};
119
120	enum {
121	GLINK_STATE_CLOSED,
122	GLINK_STATE_OPENING,
123	GLINK_STATE_OPEN,
124	GLINK_STATE_CLOSING,
125	};
126
127	/**
128	* struct glink_channel - internal representation of a channel
129	* @rpdev: rpdev reference, only used for primary endpoints
130	* @ept: rpmsg endpoint this channel is associated with
131	* @glink: qcom_glink context handle
132	* @refcount: refcount for the channel object
133	* @recv_lock: guard for @ept.cb
134	* @name: unique channel name/identifier
135	* @lcid: channel id, in local space
136	* @rcid: channel id, in remote space
137	* @intent_lock: lock for protection of @liids, @riids
138	* @liids: idr of all local intents
139	* @riids: idr of all remote intents
140	* @intent_work: worker responsible for transmitting rx_done packets
141	* @done_intents: list of intents that needs to be announced rx_done
142	* @buf: receive buffer, for gathering fragments
143	* @buf_offset: write offset in @buf
144	* @buf_size: size of current @buf
145	* @open_ack: completed once remote has acked the open-request
146	* @open_req: completed once open-request has been received
147	* @intent_req_lock: Synchronises multiple intent requests
148	* @intent_req_result: Result of intent request
149	* @intent_received: flag indicating that an intent has been received
150	* @intent_req_wq: wait queue for intent_req signalling
151	*/
152	struct glink_channel {
153	struct rpmsg_endpoint ept;
154
155	struct rpmsg_device *rpdev;
156	struct qcom_glink *glink;
157
158	struct kref refcount;
159
160	spinlock_t recv_lock;
161
162	char *name;
163	unsigned int lcid;
164	unsigned int rcid;
165
166	spinlock_t intent_lock;
167	struct idr liids;
168	struct idr riids;
169	struct work_struct intent_work;
170	struct list_head done_intents;
171
172	struct glink_core_rx_intent *buf;
173	int buf_offset;
174	int buf_size;
175
176	struct completion open_ack;
177	struct completion open_req;
178
179	struct mutex intent_req_lock;
180	int intent_req_result;
181	bool intent_received;
182	wait_queue_head_t intent_req_wq;
183	};
184
185	#define to_glink_channel(_ept) container_of(_ept, struct glink_channel, ept)
186
187	static const struct rpmsg_endpoint_ops glink_endpoint_ops;
188
189	#define GLINK_CMD_VERSION 0
190	#define GLINK_CMD_VERSION_ACK 1
191	#define GLINK_CMD_OPEN 2
192	#define GLINK_CMD_CLOSE 3
193	#define GLINK_CMD_OPEN_ACK 4
194	#define GLINK_CMD_INTENT 5
195	#define GLINK_CMD_RX_DONE 6
196	#define GLINK_CMD_RX_INTENT_REQ 7
197	#define GLINK_CMD_RX_INTENT_REQ_ACK 8
198	#define GLINK_CMD_TX_DATA 9
199	#define GLINK_CMD_CLOSE_ACK 11
200	#define GLINK_CMD_TX_DATA_CONT 12
201	#define GLINK_CMD_READ_NOTIF 13
202	#define GLINK_CMD_RX_DONE_W_REUSE 14
203	#define GLINK_CMD_SIGNALS 15
204
205	#define GLINK_FEATURE_INTENTLESS BIT(1)
206
207	#define NATIVE_DTR_SIG NATIVE_DSR_SIG
208	#define NATIVE_DSR_SIG BIT(31)
209	#define NATIVE_RTS_SIG NATIVE_CTS_SIG
210	#define NATIVE_CTS_SIG BIT(30)
211
212	static void qcom_glink_rx_done_work(struct work_struct *work);
213
214	static struct glink_channel *qcom_glink_alloc_channel(struct qcom_glink *glink,
215	const char *name)
216	{
217	struct glink_channel *channel;
218
219	channel = kzalloc(size: sizeof(*channel), GFP_KERNEL);
220	if (!channel)
221	return ERR_PTR(error: -ENOMEM);
222
223	/* Setup glink internal glink_channel data */
224	spin_lock_init(&channel->recv_lock);
225	spin_lock_init(&channel->intent_lock);
226	mutex_init(&channel->intent_req_lock);
227
228	channel->glink = glink;
229	channel->name = kstrdup(s: name, GFP_KERNEL);
230	if (!channel->name) {
231	kfree(objp: channel);
232	return ERR_PTR(error: -ENOMEM);
233	}
234
235	init_completion(x: &channel->open_req);
236	init_completion(x: &channel->open_ack);
237	init_waitqueue_head(&channel->intent_req_wq);
238
239	INIT_LIST_HEAD(list: &channel->done_intents);
240	INIT_WORK(&channel->intent_work, qcom_glink_rx_done_work);
241
242	idr_init(idr: &channel->liids);
243	idr_init(idr: &channel->riids);
244	kref_init(kref: &channel->refcount);
245
246	return channel;
247	}
248
249	static void qcom_glink_channel_release(struct kref *ref)
250	{
251	struct glink_channel *channel = container_of(ref, struct glink_channel,
252	refcount);
253	struct glink_core_rx_intent *intent;
254	struct glink_core_rx_intent *tmp;
255	unsigned long flags;
256	int iid;
257
258	/* cancel pending rx_done work */
259	cancel_work_sync(work: &channel->intent_work);
260
261	spin_lock_irqsave(&channel->intent_lock, flags);
262	/* Free all non-reuse intents pending rx_done work */
263	list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) {
264	if (!intent->reuse) {
265	kfree(objp: intent->data);
266	kfree(objp: intent);
267	}
268	}
269
270	idr_for_each_entry(&channel->liids, tmp, iid) {
271	kfree(objp: tmp->data);
272	kfree(objp: tmp);
273	}
274	idr_destroy(&channel->liids);
275
276	idr_for_each_entry(&channel->riids, tmp, iid)
277	kfree(objp: tmp);
278	idr_destroy(&channel->riids);
279	spin_unlock_irqrestore(lock: &channel->intent_lock, flags);
280
281	kfree(objp: channel->name);
282	kfree(objp: channel);
283	}
284
285	static size_t qcom_glink_rx_avail(struct qcom_glink *glink)
286	{
287	return glink->rx_pipe->avail(glink->rx_pipe);
288	}
289
290	static void qcom_glink_rx_peek(struct qcom_glink *glink,
291	void *data, unsigned int offset, size_t count)
292	{
293	glink->rx_pipe->peek(glink->rx_pipe, data, offset, count);
294	}
295
296	static void qcom_glink_rx_advance(struct qcom_glink *glink, size_t count)
297	{
298	glink->rx_pipe->advance(glink->rx_pipe, count);
299	}
300
301	static size_t qcom_glink_tx_avail(struct qcom_glink *glink)
302	{
303	return glink->tx_pipe->avail(glink->tx_pipe);
304	}
305
306	static void qcom_glink_tx_write(struct qcom_glink *glink,
307	const void *hdr, size_t hlen,
308	const void *data, size_t dlen)
309	{
310	glink->tx_pipe->write(glink->tx_pipe, hdr, hlen, data, dlen);
311	}
312
313	static void qcom_glink_tx_kick(struct qcom_glink *glink)
314	{
315	glink->tx_pipe->kick(glink->tx_pipe);
316	}
317
318	static void qcom_glink_send_read_notify(struct qcom_glink *glink)
319	{
320	struct glink_msg msg;
321
322	msg.cmd = cpu_to_le16(GLINK_CMD_READ_NOTIF);
323	msg.param1 = 0;
324	msg.param2 = 0;
325
326	qcom_glink_tx_write(glink, hdr: &msg, hlen: sizeof(msg), NULL, dlen: 0);
327
328	qcom_glink_tx_kick(glink);
329	}
330
331	static int qcom_glink_tx(struct qcom_glink *glink,
332	const void *hdr, size_t hlen,
333	const void *data, size_t dlen, bool wait)
334	{
335	unsigned int tlen = hlen + dlen;
336	unsigned long flags;
337	int ret = 0;
338
339	/* Reject packets that are too big */
340	if (tlen >= glink->tx_pipe->length)
341	return -EINVAL;
342
343	spin_lock_irqsave(&glink->tx_lock, flags);
344
345	if (glink->abort_tx) {
346	ret = -EIO;
347	goto out;
348	}
349
350	while (qcom_glink_tx_avail(glink) < tlen) {
351	if (!wait) {
352	ret = -EAGAIN;
353	goto out;
354	}
355
356	if (glink->abort_tx) {
357	ret = -EIO;
358	goto out;
359	}
360
361	if (!glink->sent_read_notify) {
362	glink->sent_read_notify = true;
363	qcom_glink_send_read_notify(glink);
364	}
365
366	/* Wait without holding the tx_lock */
367	spin_unlock_irqrestore(lock: &glink->tx_lock, flags);
368
369	wait_event_timeout(glink->tx_avail_notify,
370	qcom_glink_tx_avail(glink) >= tlen, 10 * HZ);
371
372	spin_lock_irqsave(&glink->tx_lock, flags);
373
374	if (qcom_glink_tx_avail(glink) >= tlen)
375	glink->sent_read_notify = false;
376	}
377
378	qcom_glink_tx_write(glink, hdr, hlen, data, dlen);
379	qcom_glink_tx_kick(glink);
380
381	out:
382	spin_unlock_irqrestore(lock: &glink->tx_lock, flags);
383
384	return ret;
385	}
386
387	static int qcom_glink_send_version(struct qcom_glink *glink)
388	{
389	struct glink_msg msg;
390
391	msg.cmd = cpu_to_le16(GLINK_CMD_VERSION);
392	msg.param1 = cpu_to_le16(GLINK_VERSION_1);
393	msg.param2 = cpu_to_le32(glink->features);
394
395	return qcom_glink_tx(glink, hdr: &msg, hlen: sizeof(msg), NULL, dlen: 0, wait: true);
396	}
397
398	static void qcom_glink_send_version_ack(struct qcom_glink *glink)
399	{
400	struct glink_msg msg;
401
402	msg.cmd = cpu_to_le16(GLINK_CMD_VERSION_ACK);
403	msg.param1 = cpu_to_le16(GLINK_VERSION_1);
404	msg.param2 = cpu_to_le32(glink->features);
405
406	qcom_glink_tx(glink, hdr: &msg, hlen: sizeof(msg), NULL, dlen: 0, wait: true);
407	}
408
409	static void qcom_glink_send_open_ack(struct qcom_glink *glink,
410	struct glink_channel *channel)
411	{
412	struct glink_msg msg;
413
414	msg.cmd = cpu_to_le16(GLINK_CMD_OPEN_ACK);
415	msg.param1 = cpu_to_le16(channel->rcid);
416	msg.param2 = cpu_to_le32(0);
417
418	qcom_glink_tx(glink, hdr: &msg, hlen: sizeof(msg), NULL, dlen: 0, wait: true);
419	}
420
421	static void qcom_glink_handle_intent_req_ack(struct qcom_glink *glink,
422	unsigned int cid, bool granted)
423	{
424	struct glink_channel *channel;
425	unsigned long flags;
426
427	spin_lock_irqsave(&glink->idr_lock, flags);
428	channel = idr_find(&glink->rcids, id: cid);
429	spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
430	if (!channel) {
431	dev_err(glink->dev, "unable to find channel\n");
432	return;
433	}
434
435	WRITE_ONCE(channel->intent_req_result, granted);
436	wake_up_all(&channel->intent_req_wq);
437	}
438
439	static void qcom_glink_intent_req_abort(struct glink_channel *channel)
440	{
441	WRITE_ONCE(channel->intent_req_result, 0);
442	wake_up_all(&channel->intent_req_wq);
443	}
444
445	/**
446	* qcom_glink_send_open_req() - send a GLINK_CMD_OPEN request to the remote
447	* @glink: Ptr to the glink edge
448	* @channel: Ptr to the channel that the open req is sent
449	*
450	* Allocates a local channel id and sends a GLINK_CMD_OPEN message to the remote.
451	* Will return with refcount held, regardless of outcome.
452	*
453	* Return: 0 on success, negative errno otherwise.
454	*/
455	static int qcom_glink_send_open_req(struct qcom_glink *glink,
456	struct glink_channel *channel)
457	{
458	struct {
459	struct glink_msg msg;
460	u8 name[GLINK_NAME_SIZE];
461	} __packed req;
462	int name_len = strlen(channel->name) + 1;
463	int req_len = ALIGN(sizeof(req.msg) + name_len, 8);
464	int ret;
465	unsigned long flags;
466
467	kref_get(kref: &channel->refcount);
468
469	spin_lock_irqsave(&glink->idr_lock, flags);
470	ret = idr_alloc_cyclic(&glink->lcids, ptr: channel,
471	RPM_GLINK_CID_MIN, RPM_GLINK_CID_MAX,
472	GFP_ATOMIC);
473	spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
474	if (ret < 0)
475	return ret;
476
477	channel->lcid = ret;
478
479	req.msg.cmd = cpu_to_le16(GLINK_CMD_OPEN);
480	req.msg.param1 = cpu_to_le16(channel->lcid);
481	req.msg.param2 = cpu_to_le32(name_len);
482	strcpy(p: req.name, q: channel->name);
483
484	ret = qcom_glink_tx(glink, hdr: &req, hlen: req_len, NULL, dlen: 0, wait: true);
485	if (ret)
486	goto remove_idr;
487
488	return 0;
489
490	remove_idr:
491	spin_lock_irqsave(&glink->idr_lock, flags);
492	idr_remove(&glink->lcids, id: channel->lcid);
493	channel->lcid = 0;
494	spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
495
496	return ret;
497	}
498
499	static void qcom_glink_send_close_req(struct qcom_glink *glink,
500	struct glink_channel *channel)
501	{
502	struct glink_msg req;
503
504	req.cmd = cpu_to_le16(GLINK_CMD_CLOSE);
505	req.param1 = cpu_to_le16(channel->lcid);
506	req.param2 = 0;
507
508	qcom_glink_tx(glink, hdr: &req, hlen: sizeof(req), NULL, dlen: 0, wait: true);
509	}
510
511	static void qcom_glink_send_close_ack(struct qcom_glink *glink,
512	unsigned int rcid)
513	{
514	struct glink_msg req;
515
516	req.cmd = cpu_to_le16(GLINK_CMD_CLOSE_ACK);
517	req.param1 = cpu_to_le16(rcid);
518	req.param2 = 0;
519
520	qcom_glink_tx(glink, hdr: &req, hlen: sizeof(req), NULL, dlen: 0, wait: true);
521	}
522
523	static void qcom_glink_rx_done_work(struct work_struct *work)
524	{
525	struct glink_channel *channel = container_of(work, struct glink_channel,
526	intent_work);
527	struct qcom_glink *glink = channel->glink;
528	struct glink_core_rx_intent *intent, *tmp;
529	struct {
530	u16 id;
531	u16 lcid;
532	u32 liid;
533	} __packed cmd;
534
535	unsigned int cid = channel->lcid;
536	unsigned int iid;
537	bool reuse;
538	unsigned long flags;
539
540	spin_lock_irqsave(&channel->intent_lock, flags);
541	list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) {
542	list_del(entry: &intent->node);
543	spin_unlock_irqrestore(lock: &channel->intent_lock, flags);
544	iid = intent->id;
545	reuse = intent->reuse;
546
547	cmd.id = reuse ? GLINK_CMD_RX_DONE_W_REUSE : GLINK_CMD_RX_DONE;
548	cmd.lcid = cid;
549	cmd.liid = iid;
550
551	qcom_glink_tx(glink, hdr: &cmd, hlen: sizeof(cmd), NULL, dlen: 0, wait: true);
552	if (!reuse) {
553	kfree(objp: intent->data);
554	kfree(objp: intent);
555	}
556	spin_lock_irqsave(&channel->intent_lock, flags);
557	}
558	spin_unlock_irqrestore(lock: &channel->intent_lock, flags);
559	}
560
561	static void qcom_glink_rx_done(struct qcom_glink *glink,
562	struct glink_channel *channel,
563	struct glink_core_rx_intent *intent)
564	{
565	/* We don't send RX_DONE to intentless systems */
566	if (glink->intentless) {
567	kfree(objp: intent->data);
568	kfree(objp: intent);
569	return;
570	}
571
572	/* Take it off the tree of receive intents */
573	if (!intent->reuse) {
574	spin_lock(lock: &channel->intent_lock);
575	idr_remove(&channel->liids, id: intent->id);
576	spin_unlock(lock: &channel->intent_lock);
577	}
578
579	/* Schedule the sending of a rx_done indication */
580	spin_lock(lock: &channel->intent_lock);
581	list_add_tail(new: &intent->node, head: &channel->done_intents);
582	spin_unlock(lock: &channel->intent_lock);
583
584	schedule_work(work: &channel->intent_work);
585	}
586
587	/**
588	* qcom_glink_receive_version() - receive version/features from remote system
589	*
590	* @glink: pointer to transport interface
591	* @version: remote version
592	* @features: remote features
593	*
594	* This function is called in response to a remote-initiated version/feature
595	* negotiation sequence.
596	*/
597	static void qcom_glink_receive_version(struct qcom_glink *glink,
598	u32 version,
599	u32 features)
600	{
601	switch (version) {
602	case 0:
603	break;
604	case GLINK_VERSION_1:
605	glink->features &= features;
606	fallthrough;
607	default:
608	qcom_glink_send_version_ack(glink);
609	break;
610	}
611	}
612
613	/**
614	* qcom_glink_receive_version_ack() - receive negotiation ack from remote system
615	*
616	* @glink: pointer to transport interface
617	* @version: remote version response
618	* @features: remote features response
619	*
620	* This function is called in response to a local-initiated version/feature
621	* negotiation sequence and is the counter-offer from the remote side based
622	* upon the initial version and feature set requested.
623	*/
624	static void qcom_glink_receive_version_ack(struct qcom_glink *glink,
625	u32 version,
626	u32 features)
627	{
628	switch (version) {
629	case 0:
630	/* Version negotiation failed */
631	break;
632	case GLINK_VERSION_1:
633	if (features == glink->features)
634	break;
635
636	glink->features &= features;
637	fallthrough;
638	default:
639	qcom_glink_send_version(glink);
640	break;
641	}
642	}
643
644	/**
645	* qcom_glink_send_intent_req_ack() - convert an rx intent request ack cmd to
646	* wire format and transmit
647	* @glink: The transport to transmit on.
648	* @channel: The glink channel
649	* @granted: The request response to encode.
650	*
651	* Return: 0 on success or standard Linux error code.
652	*/
653	static int qcom_glink_send_intent_req_ack(struct qcom_glink *glink,
654	struct glink_channel *channel,
655	bool granted)
656	{
657	struct glink_msg msg;
658
659	msg.cmd = cpu_to_le16(GLINK_CMD_RX_INTENT_REQ_ACK);
660	msg.param1 = cpu_to_le16(channel->lcid);
661	msg.param2 = cpu_to_le32(granted);
662
663	qcom_glink_tx(glink, hdr: &msg, hlen: sizeof(msg), NULL, dlen: 0, wait: true);
664
665	return 0;
666	}
667
668	/**
669	* qcom_glink_advertise_intent - convert an rx intent cmd to wire format and
670	* transmit
671	* @glink: The transport to transmit on.
672	* @channel: The local channel
673	* @intent: The intent to pass on to remote.
674	*
675	* Return: 0 on success or standard Linux error code.
676	*/
677	static int qcom_glink_advertise_intent(struct qcom_glink *glink,
678	struct glink_channel *channel,
679	struct glink_core_rx_intent *intent)
680	{
681	struct command {
682	__le16 id;
683	__le16 lcid;
684	__le32 count;
685	__le32 size;
686	__le32 liid;
687	} __packed;
688	struct command cmd;
689
690	cmd.id = cpu_to_le16(GLINK_CMD_INTENT);
691	cmd.lcid = cpu_to_le16(channel->lcid);
692	cmd.count = cpu_to_le32(1);
693	cmd.size = cpu_to_le32(intent->size);
694	cmd.liid = cpu_to_le32(intent->id);
695
696	qcom_glink_tx(glink, hdr: &cmd, hlen: sizeof(cmd), NULL, dlen: 0, wait: true);
697
698	return 0;
699	}
700
701	static struct glink_core_rx_intent *
702	qcom_glink_alloc_intent(struct qcom_glink *glink,
703	struct glink_channel *channel,
704	size_t size,
705	bool reuseable)
706	{
707	struct glink_core_rx_intent *intent;
708	int ret;
709	unsigned long flags;
710
711	intent = kzalloc(size: sizeof(*intent), GFP_KERNEL);
712	if (!intent)
713	return NULL;
714
715	intent->data = kzalloc(size, GFP_KERNEL);
716	if (!intent->data)
717	goto free_intent;
718
719	spin_lock_irqsave(&channel->intent_lock, flags);
720	ret = idr_alloc_cyclic(&channel->liids, ptr: intent, start: 1, end: -1, GFP_ATOMIC);
721	if (ret < 0) {
722	spin_unlock_irqrestore(lock: &channel->intent_lock, flags);
723	goto free_data;
724	}
725	spin_unlock_irqrestore(lock: &channel->intent_lock, flags);
726
727	intent->id = ret;
728	intent->size = size;
729	intent->reuse = reuseable;
730
731	return intent;
732
733	free_data:
734	kfree(objp: intent->data);
735	free_intent:
736	kfree(objp: intent);
737	return NULL;
738	}
739
740	static void qcom_glink_handle_rx_done(struct qcom_glink *glink,
741	u32 cid, uint32_t iid,
742	bool reuse)
743	{
744	struct glink_core_rx_intent *intent;
745	struct glink_channel *channel;
746	unsigned long flags;
747
748	spin_lock_irqsave(&glink->idr_lock, flags);
749	channel = idr_find(&glink->rcids, id: cid);
750	spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
751	if (!channel) {
752	dev_err(glink->dev, "invalid channel id received\n");
753	return;
754	}
755
756	spin_lock_irqsave(&channel->intent_lock, flags);
757	intent = idr_find(&channel->riids, id: iid);
758
759	if (!intent) {
760	spin_unlock_irqrestore(lock: &channel->intent_lock, flags);
761	dev_err(glink->dev, "invalid intent id received\n");
762	return;
763	}
764
765	intent->in_use = false;
766
767	if (!reuse) {
768	idr_remove(&channel->riids, id: intent->id);
769	kfree(objp: intent);
770	}
771	spin_unlock_irqrestore(lock: &channel->intent_lock, flags);
772
773	if (reuse) {
774	WRITE_ONCE(channel->intent_received, true);
775	wake_up_all(&channel->intent_req_wq);
776	}
777	}
778
779	/**
780	* qcom_glink_handle_intent_req() - Receive a request for rx_intent
781	* from remote side
782	* @glink: Pointer to the transport interface
783	* @cid: Remote channel ID
784	* @size: size of the intent
785	*
786	* The function searches for the local channel to which the request for
787	* rx_intent has arrived and allocates and notifies the remote back
788	*/
789	static void qcom_glink_handle_intent_req(struct qcom_glink *glink,
790	u32 cid, size_t size)
791	{
792	struct glink_core_rx_intent *intent;
793	struct glink_channel *channel;
794	unsigned long flags;
795
796	spin_lock_irqsave(&glink->idr_lock, flags);
797	channel = idr_find(&glink->rcids, id: cid);
798	spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
799
800	if (!channel) {
801	pr_err("%s channel not found for cid %d\n", __func__, cid);
802	return;
803	}
804
805	intent = qcom_glink_alloc_intent(glink, channel, size, reuseable: false);
806	if (intent)
807	qcom_glink_advertise_intent(glink, channel, intent);
808
809	qcom_glink_send_intent_req_ack(glink, channel, granted: !!intent);
810	}
811
812	static int qcom_glink_rx_defer(struct qcom_glink *glink, size_t extra)
813	{
814	struct glink_defer_cmd *dcmd;
815
816	extra = ALIGN(extra, 8);
817
818	if (qcom_glink_rx_avail(glink) < sizeof(struct glink_msg) + extra) {
819	dev_dbg(glink->dev, "Insufficient data in rx fifo");
820	return -ENXIO;
821	}
822
823	dcmd = kzalloc(struct_size(dcmd, data, extra), GFP_ATOMIC);
824	if (!dcmd)
825	return -ENOMEM;
826
827	INIT_LIST_HEAD(list: &dcmd->node);
828
829	qcom_glink_rx_peek(glink, data: &dcmd->msg, offset: 0, count: sizeof(dcmd->msg) + extra);
830
831	spin_lock(lock: &glink->rx_lock);
832	list_add_tail(new: &dcmd->node, head: &glink->rx_queue);
833	spin_unlock(lock: &glink->rx_lock);
834
835	schedule_work(work: &glink->rx_work);
836	qcom_glink_rx_advance(glink, count: sizeof(dcmd->msg) + extra);
837
838	return 0;
839	}
840
841	static int qcom_glink_rx_data(struct qcom_glink *glink, size_t avail)
842	{
843	struct glink_core_rx_intent *intent;
844	struct glink_channel *channel;
845	struct {
846	struct glink_msg msg;
847	__le32 chunk_size;
848	__le32 left_size;
849	} __packed hdr;
850	unsigned int chunk_size;
851	unsigned int left_size;
852	unsigned int rcid;
853	unsigned int liid;
854	int ret = 0;
855	unsigned long flags;
856
857	if (avail < sizeof(hdr)) {
858	dev_dbg(glink->dev, "Not enough data in fifo\n");
859	return -EAGAIN;
860	}
861
862	qcom_glink_rx_peek(glink, data: &hdr, offset: 0, count: sizeof(hdr));
863	chunk_size = le32_to_cpu(hdr.chunk_size);
864	left_size = le32_to_cpu(hdr.left_size);
865
866	if (avail < sizeof(hdr) + chunk_size) {
867	dev_dbg(glink->dev, "Payload not yet in fifo\n");
868	return -EAGAIN;
869	}
870
871	rcid = le16_to_cpu(hdr.msg.param1);
872	spin_lock_irqsave(&glink->idr_lock, flags);
873	channel = idr_find(&glink->rcids, id: rcid);
874	spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
875	if (!channel) {
876	dev_dbg(glink->dev, "Data on non-existing channel\n");
877
878	/* Drop the message */
879	goto advance_rx;
880	}
881
882	if (glink->intentless) {
883	/* Might have an ongoing, fragmented, message to append */
884	if (!channel->buf) {
885	intent = kzalloc(size: sizeof(*intent), GFP_ATOMIC);
886	if (!intent)
887	return -ENOMEM;
888
889	intent->data = kmalloc(size: chunk_size + left_size,
890	GFP_ATOMIC);
891	if (!intent->data) {
892	kfree(objp: intent);
893	return -ENOMEM;
894	}
895
896	intent->id = 0xbabababa;
897	intent->size = chunk_size + left_size;
898	intent->offset = 0;
899
900	channel->buf = intent;
901	} else {
902	intent = channel->buf;
903	}
904	} else {
905	liid = le32_to_cpu(hdr.msg.param2);
906
907	spin_lock_irqsave(&channel->intent_lock, flags);
908	intent = idr_find(&channel->liids, id: liid);
909	spin_unlock_irqrestore(lock: &channel->intent_lock, flags);
910
911	if (!intent) {
912	dev_err(glink->dev,
913	"no intent found for channel %s intent %d",
914	channel->name, liid);
915	ret = -ENOENT;
916	goto advance_rx;
917	}
918	}
919
920	if (intent->size - intent->offset < chunk_size) {
921	dev_err(glink->dev, "Insufficient space in intent\n");
922
923	/* The packet header lied, drop payload */
924	goto advance_rx;
925	}
926
927	qcom_glink_rx_peek(glink, data: intent->data + intent->offset,
928	offset: sizeof(hdr), count: chunk_size);
929	intent->offset += chunk_size;
930
931	/* Handle message when no fragments remain to be received */
932	if (!left_size) {
933	spin_lock(lock: &channel->recv_lock);
934	if (channel->ept.cb) {
935	channel->ept.cb(channel->ept.rpdev,
936	intent->data,
937	intent->offset,
938	channel->ept.priv,
939	RPMSG_ADDR_ANY);
940	}
941	spin_unlock(lock: &channel->recv_lock);
942
943	intent->offset = 0;
944	channel->buf = NULL;
945
946	qcom_glink_rx_done(glink, channel, intent);
947	}
948
949	advance_rx:
950	qcom_glink_rx_advance(glink, ALIGN(sizeof(hdr) + chunk_size, 8));
951
952	return ret;
953	}
954
955	static void qcom_glink_handle_intent(struct qcom_glink *glink,
956	unsigned int cid,
957	unsigned int count,
958	size_t avail)
959	{
960	struct glink_core_rx_intent *intent;
961	struct glink_channel *channel;
962	struct intent_pair {
963	__le32 size;
964	__le32 iid;
965	};
966
967	struct {
968	struct glink_msg msg;
969	struct intent_pair intents[];
970	} __packed * msg;
971
972	const size_t msglen = struct_size(msg, intents, count);
973	int ret;
974	int i;
975	unsigned long flags;
976
977	if (avail < msglen) {
978	dev_dbg(glink->dev, "Not enough data in fifo\n");
979	return;
980	}
981
982	spin_lock_irqsave(&glink->idr_lock, flags);
983	channel = idr_find(&glink->rcids, id: cid);
984	spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
985	if (!channel) {
986	dev_err(glink->dev, "intents for non-existing channel\n");
987	qcom_glink_rx_advance(glink, ALIGN(msglen, 8));
988	return;
989	}
990
991	msg = kmalloc(size: msglen, GFP_ATOMIC);
992	if (!msg)
993	return;
994
995	qcom_glink_rx_peek(glink, data: msg, offset: 0, count: msglen);
996
997	for (i = 0; i < count; ++i) {
998	intent = kzalloc(size: sizeof(*intent), GFP_ATOMIC);
999	if (!intent)
1000	break;
1001
1002	intent->id = le32_to_cpu(msg->intents[i].iid);
1003	intent->size = le32_to_cpu(msg->intents[i].size);
1004
1005	spin_lock_irqsave(&channel->intent_lock, flags);
1006	ret = idr_alloc(&channel->riids, ptr: intent,
1007	start: intent->id, end: intent->id + 1, GFP_ATOMIC);
1008	spin_unlock_irqrestore(lock: &channel->intent_lock, flags);
1009
1010	if (ret < 0)
1011	dev_err(glink->dev, "failed to store remote intent\n");
1012	}
1013
1014	WRITE_ONCE(channel->intent_received, true);
1015	wake_up_all(&channel->intent_req_wq);
1016
1017	kfree(objp: msg);
1018	qcom_glink_rx_advance(glink, ALIGN(msglen, 8));
1019	}
1020
1021	static int qcom_glink_rx_open_ack(struct qcom_glink *glink, unsigned int lcid)
1022	{
1023	struct glink_channel *channel;
1024
1025	spin_lock(lock: &glink->idr_lock);
1026	channel = idr_find(&glink->lcids, id: lcid);
1027	spin_unlock(lock: &glink->idr_lock);
1028	if (!channel) {
1029	dev_err(glink->dev, "Invalid open ack packet\n");
1030	return -EINVAL;
1031	}
1032
1033	complete_all(&channel->open_ack);
1034
1035	return 0;
1036	}
1037
1038	/**
1039	* qcom_glink_set_flow_control() - convert a signal cmd to wire format and transmit
1040	* @ept: Rpmsg endpoint for channel.
1041	* @pause: Pause transmission
1042	* @dst: destination address of the endpoint
1043	*
1044	* Return: 0 on success or standard Linux error code.
1045	*/
1046	static int qcom_glink_set_flow_control(struct rpmsg_endpoint *ept, bool pause, u32 dst)
1047	{
1048	struct glink_channel *channel = to_glink_channel(ept);
1049	struct qcom_glink *glink = channel->glink;
1050	struct glink_msg msg;
1051	u32 sigs = 0;
1052
1053	if (pause)
1054	sigs |= NATIVE_DTR_SIG | NATIVE_RTS_SIG;
1055
1056	msg.cmd = cpu_to_le16(GLINK_CMD_SIGNALS);
1057	msg.param1 = cpu_to_le16(channel->lcid);
1058	msg.param2 = cpu_to_le32(sigs);
1059
1060	return qcom_glink_tx(glink, hdr: &msg, hlen: sizeof(msg), NULL, dlen: 0, wait: true);
1061	}
1062
1063	static void qcom_glink_handle_signals(struct qcom_glink *glink,
1064	unsigned int rcid, unsigned int sigs)
1065	{
1066	struct glink_channel *channel;
1067	unsigned long flags;
1068	bool enable;
1069
1070	spin_lock_irqsave(&glink->idr_lock, flags);
1071	channel = idr_find(&glink->rcids, id: rcid);
1072	spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1073	if (!channel) {
1074	dev_err(glink->dev, "signal for non-existing channel\n");
1075	return;
1076	}
1077
1078	enable = sigs & NATIVE_DSR_SIG || sigs & NATIVE_CTS_SIG;
1079
1080	if (channel->ept.flow_cb)
1081	channel->ept.flow_cb(channel->ept.rpdev, channel->ept.priv, enable);
1082	}
1083
1084	void qcom_glink_native_rx(struct qcom_glink *glink)
1085	{
1086	struct glink_msg msg;
1087	unsigned int param1;
1088	unsigned int param2;
1089	unsigned int avail;
1090	unsigned int cmd;
1091	int ret = 0;
1092
1093	/* To wakeup any blocking writers */
1094	wake_up_all(&glink->tx_avail_notify);
1095
1096	for (;;) {
1097	avail = qcom_glink_rx_avail(glink);
1098	if (avail < sizeof(msg))
1099	break;
1100
1101	qcom_glink_rx_peek(glink, data: &msg, offset: 0, count: sizeof(msg));
1102
1103	cmd = le16_to_cpu(msg.cmd);
1104	param1 = le16_to_cpu(msg.param1);
1105	param2 = le32_to_cpu(msg.param2);
1106
1107	switch (cmd) {
1108	case GLINK_CMD_VERSION:
1109	case GLINK_CMD_VERSION_ACK:
1110	case GLINK_CMD_CLOSE:
1111	case GLINK_CMD_CLOSE_ACK:
1112	case GLINK_CMD_RX_INTENT_REQ:
1113	ret = qcom_glink_rx_defer(glink, extra: 0);
1114	break;
1115	case GLINK_CMD_OPEN_ACK:
1116	ret = qcom_glink_rx_open_ack(glink, lcid: param1);
1117	qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1118	break;
1119	case GLINK_CMD_OPEN:
1120	ret = qcom_glink_rx_defer(glink, extra: param2);
1121	break;
1122	case GLINK_CMD_TX_DATA:
1123	case GLINK_CMD_TX_DATA_CONT:
1124	ret = qcom_glink_rx_data(glink, avail);
1125	break;
1126	case GLINK_CMD_READ_NOTIF:
1127	qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1128	qcom_glink_tx_kick(glink);
1129	break;
1130	case GLINK_CMD_INTENT:
1131	qcom_glink_handle_intent(glink, cid: param1, count: param2, avail);
1132	break;
1133	case GLINK_CMD_RX_DONE:
1134	qcom_glink_handle_rx_done(glink, cid: param1, iid: param2, reuse: false);
1135	qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1136	break;
1137	case GLINK_CMD_RX_DONE_W_REUSE:
1138	qcom_glink_handle_rx_done(glink, cid: param1, iid: param2, reuse: true);
1139	qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1140	break;
1141	case GLINK_CMD_RX_INTENT_REQ_ACK:
1142	qcom_glink_handle_intent_req_ack(glink, cid: param1, granted: param2);
1143	qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1144	break;
1145	case GLINK_CMD_SIGNALS:
1146	qcom_glink_handle_signals(glink, rcid: param1, sigs: param2);
1147	qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8));
1148	break;
1149	default:
1150	dev_err(glink->dev, "unhandled rx cmd: %d\n", cmd);
1151	ret = -EINVAL;
1152	break;
1153	}
1154
1155	if (ret)
1156	break;
1157	}
1158	}
1159	EXPORT_SYMBOL(qcom_glink_native_rx);
1160
1161	/* Locally initiated rpmsg_create_ept */
1162	static struct glink_channel *qcom_glink_create_local(struct qcom_glink *glink,
1163	const char *name)
1164	{
1165	struct glink_channel *channel;
1166	int ret;
1167	unsigned long flags;
1168
1169	channel = qcom_glink_alloc_channel(glink, name);
1170	if (IS_ERR(ptr: channel))
1171	return ERR_CAST(ptr: channel);
1172
1173	ret = qcom_glink_send_open_req(glink, channel);
1174	if (ret)
1175	goto release_channel;
1176
1177	ret = wait_for_completion_timeout(x: &channel->open_ack, timeout: 5 * HZ);
1178	if (!ret)
1179	goto err_timeout;
1180
1181	ret = wait_for_completion_timeout(x: &channel->open_req, timeout: 5 * HZ);
1182	if (!ret)
1183	goto err_timeout;
1184
1185	qcom_glink_send_open_ack(glink, channel);
1186
1187	return channel;
1188
1189	err_timeout:
1190	/* qcom_glink_send_open_req() did register the channel in lcids*/
1191	spin_lock_irqsave(&glink->idr_lock, flags);
1192	idr_remove(&glink->lcids, id: channel->lcid);
1193	spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1194
1195	release_channel:
1196	/* Release qcom_glink_send_open_req() reference */
1197	kref_put(kref: &channel->refcount, release: qcom_glink_channel_release);
1198	/* Release qcom_glink_alloc_channel() reference */
1199	kref_put(kref: &channel->refcount, release: qcom_glink_channel_release);
1200
1201	return ERR_PTR(error: -ETIMEDOUT);
1202	}
1203
1204	/* Remote initiated rpmsg_create_ept */
1205	static int qcom_glink_create_remote(struct qcom_glink *glink,
1206	struct glink_channel *channel)
1207	{
1208	int ret;
1209
1210	qcom_glink_send_open_ack(glink, channel);
1211
1212	ret = qcom_glink_send_open_req(glink, channel);
1213	if (ret)
1214	goto close_link;
1215
1216	ret = wait_for_completion_timeout(x: &channel->open_ack, timeout: 5 * HZ);
1217	if (!ret) {
1218	ret = -ETIMEDOUT;
1219	goto close_link;
1220	}
1221
1222	return 0;
1223
1224	close_link:
1225	/*
1226	* Send a close request to "undo" our open-ack. The close-ack will
1227	* release qcom_glink_send_open_req() reference and the last reference
1228	* will be relesed after receiving remote_close or transport unregister
1229	* by calling qcom_glink_native_remove().
1230	*/
1231	qcom_glink_send_close_req(glink, channel);
1232
1233	return ret;
1234	}
1235
1236	static struct rpmsg_endpoint *qcom_glink_create_ept(struct rpmsg_device *rpdev,
1237	rpmsg_rx_cb_t cb,
1238	void *priv,
1239	struct rpmsg_channel_info
1240	chinfo)
1241	{
1242	struct glink_channel *parent = to_glink_channel(rpdev->ept);
1243	struct glink_channel *channel;
1244	struct qcom_glink *glink = parent->glink;
1245	struct rpmsg_endpoint *ept;
1246	const char *name = chinfo.name;
1247	int cid;
1248	int ret;
1249	unsigned long flags;
1250
1251	spin_lock_irqsave(&glink->idr_lock, flags);
1252	idr_for_each_entry(&glink->rcids, channel, cid) {
1253	if (!strcmp(channel->name, name))
1254	break;
1255	}
1256	spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1257
1258	if (!channel) {
1259	channel = qcom_glink_create_local(glink, name);
1260	if (IS_ERR(ptr: channel))
1261	return NULL;
1262	} else {
1263	ret = qcom_glink_create_remote(glink, channel);
1264	if (ret)
1265	return NULL;
1266	}
1267
1268	ept = &channel->ept;
1269	ept->rpdev = rpdev;
1270	ept->cb = cb;
1271	ept->priv = priv;
1272	ept->ops = &glink_endpoint_ops;
1273
1274	return ept;
1275	}
1276
1277	static int qcom_glink_announce_create(struct rpmsg_device *rpdev)
1278	{
1279	struct glink_channel *channel = to_glink_channel(rpdev->ept);
1280	struct device_node *np = rpdev->dev.of_node;
1281	struct qcom_glink *glink = channel->glink;
1282	struct glink_core_rx_intent *intent;
1283	const struct property *prop = NULL;
1284	__be32 defaults[] = { cpu_to_be32(SZ_1K), cpu_to_be32(5) };
1285	int num_intents;
1286	int num_groups = 1;
1287	__be32 *val = defaults;
1288	int size;
1289
1290	if (glink->intentless || !completion_done(x: &channel->open_ack))
1291	return 0;
1292
1293	prop = of_find_property(np, name: "qcom,intents", NULL);
1294	if (prop) {
1295	val = prop->value;
1296	num_groups = prop->length / sizeof(u32) / 2;
1297	}
1298
1299	/* Channel is now open, advertise base set of intents */
1300	while (num_groups--) {
1301	size = be32_to_cpup(p: val++);
1302	num_intents = be32_to_cpup(p: val++);
1303	while (num_intents--) {
1304	intent = qcom_glink_alloc_intent(glink, channel, size,
1305	reuseable: true);
1306	if (!intent)
1307	break;
1308
1309	qcom_glink_advertise_intent(glink, channel, intent);
1310	}
1311	}
1312	return 0;
1313	}
1314
1315	static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept)
1316	{
1317	struct glink_channel *channel = to_glink_channel(ept);
1318	struct qcom_glink *glink = channel->glink;
1319	unsigned long flags;
1320
1321	spin_lock_irqsave(&channel->recv_lock, flags);
1322	channel->ept.cb = NULL;
1323	spin_unlock_irqrestore(lock: &channel->recv_lock, flags);
1324
1325	/* Decouple the potential rpdev from the channel */
1326	channel->rpdev = NULL;
1327
1328	qcom_glink_send_close_req(glink, channel);
1329	}
1330
1331	static int qcom_glink_request_intent(struct qcom_glink *glink,
1332	struct glink_channel *channel,
1333	size_t size)
1334	{
1335	struct {
1336	u16 id;
1337	u16 cid;
1338	u32 size;
1339	} __packed cmd;
1340
1341	int ret;
1342
1343	mutex_lock(&channel->intent_req_lock);
1344
1345	WRITE_ONCE(channel->intent_req_result, -1);
1346	WRITE_ONCE(channel->intent_received, false);
1347
1348	cmd.id = GLINK_CMD_RX_INTENT_REQ;
1349	cmd.cid = channel->lcid;
1350	cmd.size = size;
1351
1352	ret = qcom_glink_tx(glink, hdr: &cmd, hlen: sizeof(cmd), NULL, dlen: 0, wait: true);
1353	if (ret)
1354	goto unlock;
1355
1356	ret = wait_event_timeout(channel->intent_req_wq,
1357	READ_ONCE(channel->intent_req_result) >= 0 &&
1358	READ_ONCE(channel->intent_received),
1359	10 * HZ);
1360	if (!ret) {
1361	dev_err(glink->dev, "intent request timed out\n");
1362	ret = -ETIMEDOUT;
1363	} else {
1364	ret = READ_ONCE(channel->intent_req_result) ? 0 : -ECANCELED;
1365	}
1366
1367	unlock:
1368	mutex_unlock(lock: &channel->intent_req_lock);
1369	return ret;
1370	}
1371
1372	static int __qcom_glink_send(struct glink_channel *channel,
1373	void *data, int len, bool wait)
1374	{
1375	struct qcom_glink *glink = channel->glink;
1376	struct glink_core_rx_intent *intent = NULL;
1377	struct glink_core_rx_intent *tmp;
1378	int iid = 0;
1379	struct {
1380	struct glink_msg msg;
1381	__le32 chunk_size;
1382	__le32 left_size;
1383	} __packed req;
1384	int ret;
1385	unsigned long flags;
1386	int chunk_size = len;
1387	size_t offset = 0;
1388
1389	if (!glink->intentless) {
1390	while (!intent) {
1391	spin_lock_irqsave(&channel->intent_lock, flags);
1392	idr_for_each_entry(&channel->riids, tmp, iid) {
1393	if (tmp->size >= len && !tmp->in_use) {
1394	if (!intent)
1395	intent = tmp;
1396	else if (intent->size > tmp->size)
1397	intent = tmp;
1398	if (intent->size == len)
1399	break;
1400	}
1401	}
1402	if (intent)
1403	intent->in_use = true;
1404	spin_unlock_irqrestore(lock: &channel->intent_lock, flags);
1405
1406	/* We found an available intent */
1407	if (intent)
1408	break;
1409
1410	if (!wait)
1411	return -EBUSY;
1412
1413	ret = qcom_glink_request_intent(glink, channel, size: len);
1414	if (ret < 0)
1415	return ret;
1416	}
1417
1418	iid = intent->id;
1419	}
1420
1421	while (offset < len) {
1422	chunk_size = len - offset;
1423	if (chunk_size > SZ_8K && wait)
1424	chunk_size = SZ_8K;
1425
1426	req.msg.cmd = cpu_to_le16(offset == 0 ? GLINK_CMD_TX_DATA : GLINK_CMD_TX_DATA_CONT);
1427	req.msg.param1 = cpu_to_le16(channel->lcid);
1428	req.msg.param2 = cpu_to_le32(iid);
1429	req.chunk_size = cpu_to_le32(chunk_size);
1430	req.left_size = cpu_to_le32(len - offset - chunk_size);
1431
1432	ret = qcom_glink_tx(glink, hdr: &req, hlen: sizeof(req), data: data + offset, dlen: chunk_size, wait);
1433	if (ret) {
1434	/* Mark intent available if we failed */
1435	if (intent)
1436	intent->in_use = false;
1437	return ret;
1438	}
1439
1440	offset += chunk_size;
1441	}
1442
1443	return 0;
1444	}
1445
1446	static int qcom_glink_send(struct rpmsg_endpoint *ept, void *data, int len)
1447	{
1448	struct glink_channel *channel = to_glink_channel(ept);
1449
1450	return __qcom_glink_send(channel, data, len, wait: true);
1451	}
1452
1453	static int qcom_glink_trysend(struct rpmsg_endpoint *ept, void *data, int len)
1454	{
1455	struct glink_channel *channel = to_glink_channel(ept);
1456
1457	return __qcom_glink_send(channel, data, len, wait: false);
1458	}
1459
1460	static int qcom_glink_sendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst)
1461	{
1462	struct glink_channel *channel = to_glink_channel(ept);
1463
1464	return __qcom_glink_send(channel, data, len, wait: true);
1465	}
1466
1467	static int qcom_glink_trysendto(struct rpmsg_endpoint *ept, void *data, int len, u32 dst)
1468	{
1469	struct glink_channel *channel = to_glink_channel(ept);
1470
1471	return __qcom_glink_send(channel, data, len, wait: false);
1472	}
1473
1474	/*
1475	* Finds the device_node for the glink child interested in this channel.
1476	*/
1477	static struct device_node *qcom_glink_match_channel(struct device_node *node,
1478	const char *channel)
1479	{
1480	struct device_node *child;
1481	const char *name;
1482	const char *key;
1483	int ret;
1484
1485	for_each_available_child_of_node(node, child) {
1486	key = "qcom,glink-channels";
1487	ret = of_property_read_string(np: child, propname: key, out_string: &name);
1488	if (ret)
1489	continue;
1490
1491	if (strcmp(name, channel) == 0)
1492	return child;
1493	}
1494
1495	return NULL;
1496	}
1497
1498	static const struct rpmsg_device_ops glink_device_ops = {
1499	.create_ept = qcom_glink_create_ept,
1500	.announce_create = qcom_glink_announce_create,
1501	};
1502
1503	static const struct rpmsg_endpoint_ops glink_endpoint_ops = {
1504	.destroy_ept = qcom_glink_destroy_ept,
1505	.send = qcom_glink_send,
1506	.sendto = qcom_glink_sendto,
1507	.trysend = qcom_glink_trysend,
1508	.trysendto = qcom_glink_trysendto,
1509	.set_flow_control = qcom_glink_set_flow_control,
1510	};
1511
1512	static void qcom_glink_rpdev_release(struct device *dev)
1513	{
1514	struct rpmsg_device *rpdev = to_rpmsg_device(dev);
1515
1516	kfree(objp: rpdev->driver_override);
1517	kfree(objp: rpdev);
1518	}
1519
1520	static int qcom_glink_rx_open(struct qcom_glink *glink, unsigned int rcid,
1521	char *name)
1522	{
1523	struct glink_channel *channel;
1524	struct rpmsg_device *rpdev;
1525	bool create_device = false;
1526	struct device_node *node;
1527	int lcid;
1528	int ret;
1529	unsigned long flags;
1530
1531	spin_lock_irqsave(&glink->idr_lock, flags);
1532	idr_for_each_entry(&glink->lcids, channel, lcid) {
1533	if (!strcmp(channel->name, name))
1534	break;
1535	}
1536	spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1537
1538	if (!channel) {
1539	channel = qcom_glink_alloc_channel(glink, name);
1540	if (IS_ERR(ptr: channel))
1541	return PTR_ERR(ptr: channel);
1542
1543	/* The opening dance was initiated by the remote */
1544	create_device = true;
1545	}
1546
1547	spin_lock_irqsave(&glink->idr_lock, flags);
1548	ret = idr_alloc(&glink->rcids, ptr: channel, start: rcid, end: rcid + 1, GFP_ATOMIC);
1549	if (ret < 0) {
1550	dev_err(glink->dev, "Unable to insert channel into rcid list\n");
1551	spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1552	goto free_channel;
1553	}
1554	channel->rcid = ret;
1555	spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1556
1557	complete_all(&channel->open_req);
1558
1559	if (create_device) {
1560	rpdev = kzalloc(size: sizeof(*rpdev), GFP_KERNEL);
1561	if (!rpdev) {
1562	ret = -ENOMEM;
1563	goto rcid_remove;
1564	}
1565
1566	rpdev->ept = &channel->ept;
1567	strscpy_pad(dest: rpdev->id.name, src: name, RPMSG_NAME_SIZE);
1568	rpdev->src = RPMSG_ADDR_ANY;
1569	rpdev->dst = RPMSG_ADDR_ANY;
1570	rpdev->ops = &glink_device_ops;
1571
1572	node = qcom_glink_match_channel(node: glink->dev->of_node, channel: name);
1573	rpdev->dev.of_node = node;
1574	rpdev->dev.parent = glink->dev;
1575	rpdev->dev.release = qcom_glink_rpdev_release;
1576
1577	ret = rpmsg_register_device(rpdev);
1578	if (ret)
1579	goto rcid_remove;
1580
1581	channel->rpdev = rpdev;
1582	}
1583
1584	return 0;
1585
1586	rcid_remove:
1587	spin_lock_irqsave(&glink->idr_lock, flags);
1588	idr_remove(&glink->rcids, id: channel->rcid);
1589	channel->rcid = 0;
1590	spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1591	free_channel:
1592	/* Release the reference, iff we took it */
1593	if (create_device)
1594	kref_put(kref: &channel->refcount, release: qcom_glink_channel_release);
1595
1596	return ret;
1597	}
1598
1599	static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid)
1600	{
1601	struct rpmsg_channel_info chinfo;
1602	struct glink_channel *channel;
1603	unsigned long flags;
1604
1605	spin_lock_irqsave(&glink->idr_lock, flags);
1606	channel = idr_find(&glink->rcids, id: rcid);
1607	spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1608	if (WARN(!channel, "close request on unknown channel\n"))
1609	return;
1610
1611	/* cancel pending rx_done work */
1612	cancel_work_sync(work: &channel->intent_work);
1613
1614	if (channel->rpdev) {
1615	strscpy_pad(dest: chinfo.name, src: channel->name, count: sizeof(chinfo.name));
1616	chinfo.src = RPMSG_ADDR_ANY;
1617	chinfo.dst = RPMSG_ADDR_ANY;
1618
1619	rpmsg_unregister_device(parent: glink->dev, chinfo: &chinfo);
1620	}
1621	channel->rpdev = NULL;
1622
1623	qcom_glink_send_close_ack(glink, rcid: channel->rcid);
1624
1625	spin_lock_irqsave(&glink->idr_lock, flags);
1626	idr_remove(&glink->rcids, id: channel->rcid);
1627	channel->rcid = 0;
1628	spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1629
1630	kref_put(kref: &channel->refcount, release: qcom_glink_channel_release);
1631	}
1632
1633	static void qcom_glink_rx_close_ack(struct qcom_glink *glink, unsigned int lcid)
1634	{
1635	struct rpmsg_channel_info chinfo;
1636	struct glink_channel *channel;
1637	unsigned long flags;
1638
1639	/* To wakeup any blocking writers */
1640	wake_up_all(&glink->tx_avail_notify);
1641
1642	spin_lock_irqsave(&glink->idr_lock, flags);
1643	channel = idr_find(&glink->lcids, id: lcid);
1644	if (WARN(!channel, "close ack on unknown channel\n")) {
1645	spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1646	return;
1647	}
1648
1649	idr_remove(&glink->lcids, id: channel->lcid);
1650	channel->lcid = 0;
1651	spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1652
1653	/* Decouple the potential rpdev from the channel */
1654	if (channel->rpdev) {
1655	strscpy(p: chinfo.name, q: channel->name, size: sizeof(chinfo.name));
1656	chinfo.src = RPMSG_ADDR_ANY;
1657	chinfo.dst = RPMSG_ADDR_ANY;
1658
1659	rpmsg_unregister_device(parent: glink->dev, chinfo: &chinfo);
1660	}
1661	channel->rpdev = NULL;
1662
1663	kref_put(kref: &channel->refcount, release: qcom_glink_channel_release);
1664	}
1665
1666	static void qcom_glink_work(struct work_struct *work)
1667	{
1668	struct qcom_glink *glink = container_of(work, struct qcom_glink,
1669	rx_work);
1670	struct glink_defer_cmd *dcmd;
1671	struct glink_msg *msg;
1672	unsigned long flags;
1673	unsigned int param1;
1674	unsigned int param2;
1675	unsigned int cmd;
1676
1677	for (;;) {
1678	spin_lock_irqsave(&glink->rx_lock, flags);
1679	if (list_empty(head: &glink->rx_queue)) {
1680	spin_unlock_irqrestore(lock: &glink->rx_lock, flags);
1681	break;
1682	}
1683	dcmd = list_first_entry(&glink->rx_queue,
1684	struct glink_defer_cmd, node);
1685	list_del(entry: &dcmd->node);
1686	spin_unlock_irqrestore(lock: &glink->rx_lock, flags);
1687
1688	msg = &dcmd->msg;
1689	cmd = le16_to_cpu(msg->cmd);
1690	param1 = le16_to_cpu(msg->param1);
1691	param2 = le32_to_cpu(msg->param2);
1692
1693	switch (cmd) {
1694	case GLINK_CMD_VERSION:
1695	qcom_glink_receive_version(glink, version: param1, features: param2);
1696	break;
1697	case GLINK_CMD_VERSION_ACK:
1698	qcom_glink_receive_version_ack(glink, version: param1, features: param2);
1699	break;
1700	case GLINK_CMD_OPEN:
1701	qcom_glink_rx_open(glink, rcid: param1, name: msg->data);
1702	break;
1703	case GLINK_CMD_CLOSE:
1704	qcom_glink_rx_close(glink, rcid: param1);
1705	break;
1706	case GLINK_CMD_CLOSE_ACK:
1707	qcom_glink_rx_close_ack(glink, lcid: param1);
1708	break;
1709	case GLINK_CMD_RX_INTENT_REQ:
1710	qcom_glink_handle_intent_req(glink, cid: param1, size: param2);
1711	break;
1712	default:
1713	WARN(1, "Unknown defer object %d\n", cmd);
1714	break;
1715	}
1716
1717	kfree(objp: dcmd);
1718	}
1719	}
1720
1721	static void qcom_glink_cancel_rx_work(struct qcom_glink *glink)
1722	{
1723	struct glink_defer_cmd *dcmd;
1724	struct glink_defer_cmd *tmp;
1725
1726	/* cancel any pending deferred rx_work */
1727	cancel_work_sync(work: &glink->rx_work);
1728
1729	list_for_each_entry_safe(dcmd, tmp, &glink->rx_queue, node)
1730	kfree(objp: dcmd);
1731	}
1732
1733	static ssize_t rpmsg_name_show(struct device *dev,
1734	struct device_attribute *attr, char *buf)
1735	{
1736	int ret = 0;
1737	const char *name;
1738
1739	ret = of_property_read_string(np: dev->of_node, propname: "label", out_string: &name);
1740	if (ret < 0)
1741	name = dev->of_node->name;
1742
1743	return sysfs_emit(buf, fmt: "%s\n", name);
1744	}
1745	static DEVICE_ATTR_RO(rpmsg_name);
1746
1747	static struct attribute *qcom_glink_attrs[] = {
1748	&dev_attr_rpmsg_name.attr,
1749	NULL
1750	};
1751	ATTRIBUTE_GROUPS(qcom_glink);
1752
1753	static void qcom_glink_device_release(struct device *dev)
1754	{
1755	struct rpmsg_device *rpdev = to_rpmsg_device(dev);
1756	struct glink_channel *channel = to_glink_channel(rpdev->ept);
1757
1758	/* Release qcom_glink_alloc_channel() reference */
1759	kref_put(kref: &channel->refcount, release: qcom_glink_channel_release);
1760	kfree(objp: rpdev->driver_override);
1761	kfree(objp: rpdev);
1762	}
1763
1764	static int qcom_glink_create_chrdev(struct qcom_glink *glink)
1765	{
1766	struct rpmsg_device *rpdev;
1767	struct glink_channel *channel;
1768
1769	rpdev = kzalloc(size: sizeof(*rpdev), GFP_KERNEL);
1770	if (!rpdev)
1771	return -ENOMEM;
1772
1773	channel = qcom_glink_alloc_channel(glink, name: "rpmsg_chrdev");
1774	if (IS_ERR(ptr: channel)) {
1775	kfree(objp: rpdev);
1776	return PTR_ERR(ptr: channel);
1777	}
1778	channel->rpdev = rpdev;
1779
1780	rpdev->ept = &channel->ept;
1781	rpdev->ops = &glink_device_ops;
1782	rpdev->dev.parent = glink->dev;
1783	rpdev->dev.release = qcom_glink_device_release;
1784
1785	return rpmsg_ctrldev_register_device(rpdev);
1786	}
1787
1788	struct qcom_glink *qcom_glink_native_probe(struct device *dev,
1789	unsigned long features,
1790	struct qcom_glink_pipe *rx,
1791	struct qcom_glink_pipe *tx,
1792	bool intentless)
1793	{
1794	int ret;
1795	struct qcom_glink *glink;
1796
1797	glink = devm_kzalloc(dev, size: sizeof(*glink), GFP_KERNEL);
1798	if (!glink)
1799	return ERR_PTR(error: -ENOMEM);
1800
1801	glink->dev = dev;
1802	glink->tx_pipe = tx;
1803	glink->rx_pipe = rx;
1804
1805	glink->features = features;
1806	glink->intentless = intentless;
1807
1808	spin_lock_init(&glink->tx_lock);
1809	spin_lock_init(&glink->rx_lock);
1810	INIT_LIST_HEAD(list: &glink->rx_queue);
1811	INIT_WORK(&glink->rx_work, qcom_glink_work);
1812	init_waitqueue_head(&glink->tx_avail_notify);
1813
1814	spin_lock_init(&glink->idr_lock);
1815	idr_init(idr: &glink->lcids);
1816	idr_init(idr: &glink->rcids);
1817
1818	glink->dev->groups = qcom_glink_groups;
1819
1820	ret = device_add_groups(dev, groups: qcom_glink_groups);
1821	if (ret)
1822	dev_err(dev, "failed to add groups\n");
1823
1824	ret = qcom_glink_send_version(glink);
1825	if (ret)
1826	return ERR_PTR(error: ret);
1827
1828	ret = qcom_glink_create_chrdev(glink);
1829	if (ret)
1830	dev_err(glink->dev, "failed to register chrdev\n");
1831
1832	return glink;
1833	}
1834	EXPORT_SYMBOL_GPL(qcom_glink_native_probe);
1835
1836	static int qcom_glink_remove_device(struct device *dev, void *data)
1837	{
1838	device_unregister(dev);
1839
1840	return 0;
1841	}
1842
1843	void qcom_glink_native_remove(struct qcom_glink *glink)
1844	{
1845	struct glink_channel *channel;
1846	unsigned long flags;
1847	int cid;
1848	int ret;
1849
1850	qcom_glink_cancel_rx_work(glink);
1851
1852	/* Fail all attempts at sending messages */
1853	spin_lock_irqsave(&glink->tx_lock, flags);
1854	glink->abort_tx = true;
1855	wake_up_all(&glink->tx_avail_notify);
1856	spin_unlock_irqrestore(lock: &glink->tx_lock, flags);
1857
1858	/* Abort any senders waiting for intent requests */
1859	spin_lock_irqsave(&glink->idr_lock, flags);
1860	idr_for_each_entry(&glink->lcids, channel, cid)
1861	qcom_glink_intent_req_abort(channel);
1862	spin_unlock_irqrestore(lock: &glink->idr_lock, flags);
1863
1864	ret = device_for_each_child(dev: glink->dev, NULL, fn: qcom_glink_remove_device);
1865	if (ret)
1866	dev_warn(glink->dev, "Can't remove GLINK devices: %d\n", ret);
1867
1868	/* Release any defunct local channels, waiting for close-ack */
1869	idr_for_each_entry(&glink->lcids, channel, cid)
1870	kref_put(kref: &channel->refcount, release: qcom_glink_channel_release);
1871
1872	/* Release any defunct local channels, waiting for close-req */
1873	idr_for_each_entry(&glink->rcids, channel, cid)
1874	kref_put(kref: &channel->refcount, release: qcom_glink_channel_release);
1875
1876	idr_destroy(&glink->lcids);
1877	idr_destroy(&glink->rcids);
1878	}
1879	EXPORT_SYMBOL_GPL(qcom_glink_native_remove);
1880
1881	MODULE_DESCRIPTION("Qualcomm GLINK driver");
1882	MODULE_LICENSE("GPL v2");
1883