1	// SPDX-License-Identifier: GPL-2.0+
2	/*
3	* IBM Power Systems Virtual Management Channel Support.
4	*
5	* Copyright (c) 2004, 2018 IBM Corp.
6	* Dave Engebretsen engebret@us.ibm.com
7	* Steven Royer seroyer@linux.vnet.ibm.com
8	* Adam Reznechek adreznec@linux.vnet.ibm.com
9	* Bryant G. Ly <bryantly@linux.vnet.ibm.com>
10	*/
11
12	#include <linux/module.h>
13	#include <linux/kernel.h>
14	#include <linux/kthread.h>
15	#include <linux/major.h>
16	#include <linux/string.h>
17	#include <linux/fcntl.h>
18	#include <linux/slab.h>
19	#include <linux/poll.h>
20	#include <linux/init.h>
21	#include <linux/fs.h>
22	#include <linux/interrupt.h>
23	#include <linux/spinlock.h>
24	#include <linux/percpu.h>
25	#include <linux/delay.h>
26	#include <linux/uaccess.h>
27	#include <linux/io.h>
28	#include <linux/miscdevice.h>
29	#include <linux/sched/signal.h>
30
31	#include <asm/byteorder.h>
32	#include <asm/irq.h>
33	#include <asm/vio.h>
34
35	#include "ibmvmc.h"
36
37	#define IBMVMC_DRIVER_VERSION "1.0"
38
39	/*
40	* Static global variables
41	*/
42	static DECLARE_WAIT_QUEUE_HEAD(ibmvmc_read_wait);
43
44	static const char ibmvmc_driver_name[] = "ibmvmc";
45
46	static struct ibmvmc_struct ibmvmc;
47	static struct ibmvmc_hmc hmcs[MAX_HMCS];
48	static struct crq_server_adapter ibmvmc_adapter;
49
50	static int ibmvmc_max_buf_pool_size = DEFAULT_BUF_POOL_SIZE;
51	static int ibmvmc_max_hmcs = DEFAULT_HMCS;
52	static int ibmvmc_max_mtu = DEFAULT_MTU;
53
54	static inline long h_copy_rdma(s64 length, u64 sliobn, u64 slioba,
55	u64 dliobn, u64 dlioba)
56	{
57	long rc = 0;
58
59	/* Ensure all writes to source memory are visible before hcall */
60	dma_wmb();
61	pr_debug("ibmvmc: h_copy_rdma(0x%llx, 0x%llx, 0x%llx, 0x%llx, 0x%llx\n",
62	length, sliobn, slioba, dliobn, dlioba);
63	rc = plpar_hcall_norets(H_COPY_RDMA, length, sliobn, slioba,
64	dliobn, dlioba);
65	pr_debug("ibmvmc: h_copy_rdma rc = 0x%lx\n", rc);
66
67	return rc;
68	}
69
70	static inline void h_free_crq(uint32_t unit_address)
71	{
72	long rc = 0;
73
74	do {
75	if (H_IS_LONG_BUSY(rc))
76	msleep(msecs: get_longbusy_msecs(rc));
77
78	rc = plpar_hcall_norets(H_FREE_CRQ, unit_address);
79	} while ((rc == H_BUSY) || (H_IS_LONG_BUSY(rc)));
80	}
81
82	/**
83	* h_request_vmc: - request a hypervisor virtual management channel device
84	* @vmc_index: drc index of the vmc device created
85	*
86	* Requests the hypervisor create a new virtual management channel device,
87	* allowing this partition to send hypervisor virtualization control
88	* commands.
89	*
90	* Return:
91	* 0 - Success
92	* Non-zero - Failure
93	*/
94	static inline long h_request_vmc(u32 *vmc_index)
95	{
96	long rc = 0;
97	unsigned long retbuf[PLPAR_HCALL_BUFSIZE];
98
99	do {
100	if (H_IS_LONG_BUSY(rc))
101	msleep(msecs: get_longbusy_msecs(rc));
102
103	/* Call to request the VMC device from phyp */
104	rc = plpar_hcall(H_REQUEST_VMC, retbuf);
105	pr_debug("ibmvmc: %s rc = 0x%lx\n", __func__, rc);
106	*vmc_index = retbuf[0];
107	} while ((rc == H_BUSY) || (H_IS_LONG_BUSY(rc)));
108
109	return rc;
110	}
111
112	/* routines for managing a command/response queue */
113	/**
114	* ibmvmc_handle_event: - Interrupt handler for crq events
115	* @irq: number of irq to handle, not used
116	* @dev_instance: crq_server_adapter that received interrupt
117	*
118	* Disables interrupts and schedules ibmvmc_task
119	*
120	* Always returns IRQ_HANDLED
121	*/
122	static irqreturn_t ibmvmc_handle_event(int irq, void *dev_instance)
123	{
124	struct crq_server_adapter *adapter =
125	(struct crq_server_adapter *)dev_instance;
126
127	vio_disable_interrupts(to_vio_dev(adapter->dev));
128	tasklet_schedule(t: &adapter->work_task);
129
130	return IRQ_HANDLED;
131	}
132
133	/**
134	* ibmvmc_release_crq_queue - Release CRQ Queue
135	*
136	* @adapter: crq_server_adapter struct
137	*
138	* Return:
139	* 0 - Success
140	* Non-Zero - Failure
141	*/
142	static void ibmvmc_release_crq_queue(struct crq_server_adapter *adapter)
143	{
144	struct vio_dev *vdev = to_vio_dev(adapter->dev);
145	struct crq_queue *queue = &adapter->queue;
146
147	free_irq(vdev->irq, (void *)adapter);
148	tasklet_kill(t: &adapter->work_task);
149
150	if (adapter->reset_task)
151	kthread_stop(k: adapter->reset_task);
152
153	h_free_crq(unit_address: vdev->unit_address);
154	dma_unmap_single(adapter->dev,
155	queue->msg_token,
156	queue->size * sizeof(*queue->msgs), DMA_BIDIRECTIONAL);
157	free_page((unsigned long)queue->msgs);
158	}
159
160	/**
161	* ibmvmc_reset_crq_queue - Reset CRQ Queue
162	*
163	* @adapter: crq_server_adapter struct
164	*
165	* This function calls h_free_crq and then calls H_REG_CRQ and does all the
166	* bookkeeping to get us back to where we can communicate.
167	*
168	* Return:
169	* 0 - Success
170	* Non-Zero - Failure
171	*/
172	static int ibmvmc_reset_crq_queue(struct crq_server_adapter *adapter)
173	{
174	struct vio_dev *vdev = to_vio_dev(adapter->dev);
175	struct crq_queue *queue = &adapter->queue;
176	int rc = 0;
177
178	/* Close the CRQ */
179	h_free_crq(unit_address: vdev->unit_address);
180
181	/* Clean out the queue */
182	memset(queue->msgs, 0x00, PAGE_SIZE);
183	queue->cur = 0;
184
185	/* And re-open it again */
186	rc = plpar_hcall_norets(H_REG_CRQ,
187	vdev->unit_address,
188	queue->msg_token, PAGE_SIZE);
189	if (rc == 2)
190	/* Adapter is good, but other end is not ready */
191	dev_warn(adapter->dev, "Partner adapter not ready\n");
192	else if (rc != 0)
193	dev_err(adapter->dev, "couldn't register crq--rc 0x%x\n", rc);
194
195	return rc;
196	}
197
198	/**
199	* crq_queue_next_crq: - Returns the next entry in message queue
200	* @queue: crq_queue to use
201	*
202	* Returns pointer to next entry in queue, or NULL if there are no new
203	* entried in the CRQ.
204	*/
205	static struct ibmvmc_crq_msg *crq_queue_next_crq(struct crq_queue *queue)
206	{
207	struct ibmvmc_crq_msg *crq;
208	unsigned long flags;
209
210	spin_lock_irqsave(&queue->lock, flags);
211	crq = &queue->msgs[queue->cur];
212	if (crq->valid & 0x80) {
213	if (++queue->cur == queue->size)
214	queue->cur = 0;
215
216	/* Ensure the read of the valid bit occurs before reading any
217	* other bits of the CRQ entry
218	*/
219	dma_rmb();
220	} else {
221	crq = NULL;
222	}
223
224	spin_unlock_irqrestore(lock: &queue->lock, flags);
225
226	return crq;
227	}
228
229	/**
230	* ibmvmc_send_crq - Send CRQ
231	*
232	* @adapter: crq_server_adapter struct
233	* @word1: Word1 Data field
234	* @word2: Word2 Data field
235	*
236	* Return:
237	* 0 - Success
238	* Non-Zero - Failure
239	*/
240	static long ibmvmc_send_crq(struct crq_server_adapter *adapter,
241	u64 word1, u64 word2)
242	{
243	struct vio_dev *vdev = to_vio_dev(adapter->dev);
244	long rc = 0;
245
246	dev_dbg(adapter->dev, "(0x%x, 0x%016llx, 0x%016llx)\n",
247	vdev->unit_address, word1, word2);
248
249	/*
250	* Ensure the command buffer is flushed to memory before handing it
251	* over to the other side to prevent it from fetching any stale data.
252	*/
253	dma_wmb();
254	rc = plpar_hcall_norets(H_SEND_CRQ, vdev->unit_address, word1, word2);
255	dev_dbg(adapter->dev, "rc = 0x%lx\n", rc);
256
257	return rc;
258	}
259
260	/**
261	* alloc_dma_buffer - Create DMA Buffer
262	*
263	* @vdev: vio_dev struct
264	* @size: Size field
265	* @dma_handle: DMA address field
266	*
267	* Allocates memory for the command queue and maps remote memory into an
268	* ioba.
269	*
270	* Returns a pointer to the buffer
271	*/
272	static void *alloc_dma_buffer(struct vio_dev *vdev, size_t size,
273	dma_addr_t *dma_handle)
274	{
275	/* allocate memory */
276	void *buffer = kzalloc(size, GFP_ATOMIC);
277
278	if (!buffer) {
279	*dma_handle = 0;
280	return NULL;
281	}
282
283	/* DMA map */
284	*dma_handle = dma_map_single(&vdev->dev, buffer, size,
285	DMA_BIDIRECTIONAL);
286
287	if (dma_mapping_error(&vdev->dev, *dma_handle)) {
288	*dma_handle = 0;
289	kfree_sensitive(objp: buffer);
290	return NULL;
291	}
292
293	return buffer;
294	}
295
296	/**
297	* free_dma_buffer - Free DMA Buffer
298	*
299	* @vdev: vio_dev struct
300	* @size: Size field
301	* @vaddr: Address field
302	* @dma_handle: DMA address field
303	*
304	* Releases memory for a command queue and unmaps mapped remote memory.
305	*/
306	static void free_dma_buffer(struct vio_dev *vdev, size_t size, void *vaddr,
307	dma_addr_t dma_handle)
308	{
309	/* DMA unmap */
310	dma_unmap_single(&vdev->dev, dma_handle, size, DMA_BIDIRECTIONAL);
311
312	/* deallocate memory */
313	kfree_sensitive(objp: vaddr);
314	}
315
316	/**
317	* ibmvmc_get_valid_hmc_buffer - Retrieve Valid HMC Buffer
318	*
319	* @hmc_index: HMC Index Field
320	*
321	* Return:
322	* Pointer to ibmvmc_buffer
323	*/
324	static struct ibmvmc_buffer *ibmvmc_get_valid_hmc_buffer(u8 hmc_index)
325	{
326	struct ibmvmc_buffer *buffer;
327	struct ibmvmc_buffer *ret_buf = NULL;
328	unsigned long i;
329
330	if (hmc_index > ibmvmc.max_hmc_index)
331	return NULL;
332
333	buffer = hmcs[hmc_index].buffer;
334
335	for (i = 0; i < ibmvmc_max_buf_pool_size; i++) {
336	if (buffer[i].valid && buffer[i].free &&
337	buffer[i].owner == VMC_BUF_OWNER_ALPHA) {
338	buffer[i].free = 0;
339	ret_buf = &buffer[i];
340	break;
341	}
342	}
343
344	return ret_buf;
345	}
346
347	/**
348	* ibmvmc_get_free_hmc_buffer - Get Free HMC Buffer
349	*
350	* @adapter: crq_server_adapter struct
351	* @hmc_index: Hmc Index field
352	*
353	* Return:
354	* Pointer to ibmvmc_buffer
355	*/
356	static struct ibmvmc_buffer *ibmvmc_get_free_hmc_buffer(struct crq_server_adapter *adapter,
357	u8 hmc_index)
358	{
359	struct ibmvmc_buffer *buffer;
360	struct ibmvmc_buffer *ret_buf = NULL;
361	unsigned long i;
362
363	if (hmc_index > ibmvmc.max_hmc_index) {
364	dev_info(adapter->dev, "get_free_hmc_buffer: invalid hmc_index=0x%x\n",
365	hmc_index);
366	return NULL;
367	}
368
369	buffer = hmcs[hmc_index].buffer;
370
371	for (i = 0; i < ibmvmc_max_buf_pool_size; i++) {
372	if (buffer[i].free &&
373	buffer[i].owner == VMC_BUF_OWNER_ALPHA) {
374	buffer[i].free = 0;
375	ret_buf = &buffer[i];
376	break;
377	}
378	}
379
380	return ret_buf;
381	}
382
383	/**
384	* ibmvmc_free_hmc_buffer - Free an HMC Buffer
385	*
386	* @hmc: ibmvmc_hmc struct
387	* @buffer: ibmvmc_buffer struct
388	*
389	*/
390	static void ibmvmc_free_hmc_buffer(struct ibmvmc_hmc *hmc,
391	struct ibmvmc_buffer *buffer)
392	{
393	unsigned long flags;
394
395	spin_lock_irqsave(&hmc->lock, flags);
396	buffer->free = 1;
397	spin_unlock_irqrestore(lock: &hmc->lock, flags);
398	}
399
400	/**
401	* ibmvmc_count_hmc_buffers - Count HMC Buffers
402	*
403	* @hmc_index: HMC Index field
404	* @valid: Valid number of buffers field
405	* @free: Free number of buffers field
406	*
407	*/
408	static void ibmvmc_count_hmc_buffers(u8 hmc_index, unsigned int *valid,
409	unsigned int *free)
410	{
411	struct ibmvmc_buffer *buffer;
412	unsigned long i;
413	unsigned long flags;
414
415	if (hmc_index > ibmvmc.max_hmc_index)
416	return;
417
418	if (!valid || !free)
419	return;
420
421	*valid = 0; *free = 0;
422
423	buffer = hmcs[hmc_index].buffer;
424	spin_lock_irqsave(&hmcs[hmc_index].lock, flags);
425
426	for (i = 0; i < ibmvmc_max_buf_pool_size; i++) {
427	if (buffer[i].valid) {
428	*valid = *valid + 1;
429	if (buffer[i].free)
430	*free = *free + 1;
431	}
432	}
433
434	spin_unlock_irqrestore(lock: &hmcs[hmc_index].lock, flags);
435	}
436
437	/**
438	* ibmvmc_get_free_hmc - Get Free HMC
439	*
440	* Return:
441	* Pointer to an available HMC Connection
442	* Null otherwise
443	*/
444	static struct ibmvmc_hmc *ibmvmc_get_free_hmc(void)
445	{
446	unsigned long i;
447	unsigned long flags;
448
449	/*
450	* Find an available HMC connection.
451	*/
452	for (i = 0; i <= ibmvmc.max_hmc_index; i++) {
453	spin_lock_irqsave(&hmcs[i].lock, flags);
454	if (hmcs[i].state == ibmhmc_state_free) {
455	hmcs[i].index = i;
456	hmcs[i].state = ibmhmc_state_initial;
457	spin_unlock_irqrestore(lock: &hmcs[i].lock, flags);
458	return &hmcs[i];
459	}
460	spin_unlock_irqrestore(lock: &hmcs[i].lock, flags);
461	}
462
463	return NULL;
464	}
465
466	/**
467	* ibmvmc_return_hmc - Return an HMC Connection
468	*
469	* @hmc: ibmvmc_hmc struct
470	* @release_readers: Number of readers connected to session
471	*
472	* This function releases the HMC connections back into the pool.
473	*
474	* Return:
475	* 0 - Success
476	* Non-zero - Failure
477	*/
478	static int ibmvmc_return_hmc(struct ibmvmc_hmc *hmc, bool release_readers)
479	{
480	struct ibmvmc_buffer *buffer;
481	struct crq_server_adapter *adapter;
482	struct vio_dev *vdev;
483	unsigned long i;
484	unsigned long flags;
485
486	if (!hmc || !hmc->adapter)
487	return -EIO;
488
489	if (release_readers) {
490	if (hmc->file_session) {
491	struct ibmvmc_file_session *session = hmc->file_session;
492
493	session->valid = 0;
494	wake_up_interruptible(&ibmvmc_read_wait);
495	}
496	}
497
498	adapter = hmc->adapter;
499	vdev = to_vio_dev(adapter->dev);
500
501	spin_lock_irqsave(&hmc->lock, flags);
502	hmc->index = 0;
503	hmc->state = ibmhmc_state_free;
504	hmc->queue_head = 0;
505	hmc->queue_tail = 0;
506	buffer = hmc->buffer;
507	for (i = 0; i < ibmvmc_max_buf_pool_size; i++) {
508	if (buffer[i].valid) {
509	free_dma_buffer(vdev,
510	size: ibmvmc.max_mtu,
511	vaddr: buffer[i].real_addr_local,
512	dma_handle: buffer[i].dma_addr_local);
513	dev_dbg(adapter->dev, "Forgot buffer id 0x%lx\n", i);
514	}
515	memset(&buffer[i], 0, sizeof(struct ibmvmc_buffer));
516
517	hmc->queue_outbound_msgs[i] = VMC_INVALID_BUFFER_ID;
518	}
519
520	spin_unlock_irqrestore(lock: &hmc->lock, flags);
521
522	return 0;
523	}
524
525	/**
526	* ibmvmc_send_open - Interface Open
527	* @buffer: Pointer to ibmvmc_buffer struct
528	* @hmc: Pointer to ibmvmc_hmc struct
529	*
530	* This command is sent by the management partition as the result of a
531	* management partition device request. It causes the hypervisor to
532	* prepare a set of data buffers for the management application connection
533	* indicated HMC idx. A unique HMC Idx would be used if multiple management
534	* applications running concurrently were desired. Before responding to this
535	* command, the hypervisor must provide the management partition with at
536	* least one of these new buffers via the Add Buffer. This indicates whether
537	* the messages are inbound or outbound from the hypervisor.
538	*
539	* Return:
540	* 0 - Success
541	* Non-zero - Failure
542	*/
543	static int ibmvmc_send_open(struct ibmvmc_buffer *buffer,
544	struct ibmvmc_hmc *hmc)
545	{
546	struct ibmvmc_crq_msg crq_msg;
547	struct crq_server_adapter *adapter;
548	__be64 *crq_as_u64 = (__be64 *)&crq_msg;
549	int rc = 0;
550
551	if (!hmc || !hmc->adapter)
552	return -EIO;
553
554	adapter = hmc->adapter;
555
556	dev_dbg(adapter->dev, "send_open: 0x%lx 0x%lx 0x%lx 0x%lx 0x%lx\n",
557	(unsigned long)buffer->size, (unsigned long)adapter->liobn,
558	(unsigned long)buffer->dma_addr_local,
559	(unsigned long)adapter->riobn,
560	(unsigned long)buffer->dma_addr_remote);
561
562	rc = h_copy_rdma(length: buffer->size,
563	sliobn: adapter->liobn,
564	slioba: buffer->dma_addr_local,
565	dliobn: adapter->riobn,
566	dlioba: buffer->dma_addr_remote);
567	if (rc) {
568	dev_err(adapter->dev, "Error: In send_open, h_copy_rdma rc 0x%x\n",
569	rc);
570	return -EIO;
571	}
572
573	hmc->state = ibmhmc_state_opening;
574
575	crq_msg.valid = 0x80;
576	crq_msg.type = VMC_MSG_OPEN;
577	crq_msg.status = 0;
578	crq_msg.var1.rsvd = 0;
579	crq_msg.hmc_session = hmc->session;
580	crq_msg.hmc_index = hmc->index;
581	crq_msg.var2.buffer_id = cpu_to_be16(buffer->id);
582	crq_msg.rsvd = 0;
583	crq_msg.var3.rsvd = 0;
584
585	ibmvmc_send_crq(adapter, be64_to_cpu(crq_as_u64[0]),
586	be64_to_cpu(crq_as_u64[1]));
587
588	return rc;
589	}
590
591	/**
592	* ibmvmc_send_close - Interface Close
593	* @hmc: Pointer to ibmvmc_hmc struct
594	*
595	* This command is sent by the management partition to terminate a
596	* management application to hypervisor connection. When this command is
597	* sent, the management partition has quiesced all I/O operations to all
598	* buffers associated with this management application connection, and
599	* has freed any storage for these buffers.
600	*
601	* Return:
602	* 0 - Success
603	* Non-zero - Failure
604	*/
605	static int ibmvmc_send_close(struct ibmvmc_hmc *hmc)
606	{
607	struct ibmvmc_crq_msg crq_msg;
608	struct crq_server_adapter *adapter;
609	__be64 *crq_as_u64 = (__be64 *)&crq_msg;
610	int rc = 0;
611
612	if (!hmc || !hmc->adapter)
613	return -EIO;
614
615	adapter = hmc->adapter;
616
617	dev_info(adapter->dev, "CRQ send: close\n");
618
619	crq_msg.valid = 0x80;
620	crq_msg.type = VMC_MSG_CLOSE;
621	crq_msg.status = 0;
622	crq_msg.var1.rsvd = 0;
623	crq_msg.hmc_session = hmc->session;
624	crq_msg.hmc_index = hmc->index;
625	crq_msg.var2.rsvd = 0;
626	crq_msg.rsvd = 0;
627	crq_msg.var3.rsvd = 0;
628
629	ibmvmc_send_crq(adapter, be64_to_cpu(crq_as_u64[0]),
630	be64_to_cpu(crq_as_u64[1]));
631
632	return rc;
633	}
634
635	/**
636	* ibmvmc_send_capabilities - Send VMC Capabilities
637	*
638	* @adapter: crq_server_adapter struct
639	*
640	* The capabilities message is an administrative message sent after the CRQ
641	* initialization sequence of messages and is used to exchange VMC capabilities
642	* between the management partition and the hypervisor. The management
643	* partition must send this message and the hypervisor must respond with VMC
644	* capabilities Response message before HMC interface message can begin. Any
645	* HMC interface messages received before the exchange of capabilities has
646	* complete are dropped.
647	*
648	* Return:
649	* 0 - Success
650	*/
651	static int ibmvmc_send_capabilities(struct crq_server_adapter *adapter)
652	{
653	struct ibmvmc_admin_crq_msg crq_msg;
654	__be64 *crq_as_u64 = (__be64 *)&crq_msg;
655
656	dev_dbg(adapter->dev, "ibmvmc: CRQ send: capabilities\n");
657	crq_msg.valid = 0x80;
658	crq_msg.type = VMC_MSG_CAP;
659	crq_msg.status = 0;
660	crq_msg.rsvd[0] = 0;
661	crq_msg.rsvd[1] = 0;
662	crq_msg.max_hmc = ibmvmc_max_hmcs;
663	crq_msg.max_mtu = cpu_to_be32(ibmvmc_max_mtu);
664	crq_msg.pool_size = cpu_to_be16(ibmvmc_max_buf_pool_size);
665	crq_msg.crq_size = cpu_to_be16(adapter->queue.size);
666	crq_msg.version = cpu_to_be16(IBMVMC_PROTOCOL_VERSION);
667
668	ibmvmc_send_crq(adapter, be64_to_cpu(crq_as_u64[0]),
669	be64_to_cpu(crq_as_u64[1]));
670
671	ibmvmc.state = ibmvmc_state_capabilities;
672
673	return 0;
674	}
675
676	/**
677	* ibmvmc_send_add_buffer_resp - Add Buffer Response
678	*
679	* @adapter: crq_server_adapter struct
680	* @status: Status field
681	* @hmc_session: HMC Session field
682	* @hmc_index: HMC Index field
683	* @buffer_id: Buffer Id field
684	*
685	* This command is sent by the management partition to the hypervisor in
686	* response to the Add Buffer message. The Status field indicates the result of
687	* the command.
688	*
689	* Return:
690	* 0 - Success
691	*/
692	static int ibmvmc_send_add_buffer_resp(struct crq_server_adapter *adapter,
693	u8 status, u8 hmc_session,
694	u8 hmc_index, u16 buffer_id)
695	{
696	struct ibmvmc_crq_msg crq_msg;
697	__be64 *crq_as_u64 = (__be64 *)&crq_msg;
698
699	dev_dbg(adapter->dev, "CRQ send: add_buffer_resp\n");
700	crq_msg.valid = 0x80;
701	crq_msg.type = VMC_MSG_ADD_BUF_RESP;
702	crq_msg.status = status;
703	crq_msg.var1.rsvd = 0;
704	crq_msg.hmc_session = hmc_session;
705	crq_msg.hmc_index = hmc_index;
706	crq_msg.var2.buffer_id = cpu_to_be16(buffer_id);
707	crq_msg.rsvd = 0;
708	crq_msg.var3.rsvd = 0;
709
710	ibmvmc_send_crq(adapter, be64_to_cpu(crq_as_u64[0]),
711	be64_to_cpu(crq_as_u64[1]));
712
713	return 0;
714	}
715
716	/**
717	* ibmvmc_send_rem_buffer_resp - Remove Buffer Response
718	*
719	* @adapter: crq_server_adapter struct
720	* @status: Status field
721	* @hmc_session: HMC Session field
722	* @hmc_index: HMC Index field
723	* @buffer_id: Buffer Id field
724	*
725	* This command is sent by the management partition to the hypervisor in
726	* response to the Remove Buffer message. The Buffer ID field indicates
727	* which buffer the management partition selected to remove. The Status
728	* field indicates the result of the command.
729	*
730	* Return:
731	* 0 - Success
732	*/
733	static int ibmvmc_send_rem_buffer_resp(struct crq_server_adapter *adapter,
734	u8 status, u8 hmc_session,
735	u8 hmc_index, u16 buffer_id)
736	{
737	struct ibmvmc_crq_msg crq_msg;
738	__be64 *crq_as_u64 = (__be64 *)&crq_msg;
739
740	dev_dbg(adapter->dev, "CRQ send: rem_buffer_resp\n");
741	crq_msg.valid = 0x80;
742	crq_msg.type = VMC_MSG_REM_BUF_RESP;
743	crq_msg.status = status;
744	crq_msg.var1.rsvd = 0;
745	crq_msg.hmc_session = hmc_session;
746	crq_msg.hmc_index = hmc_index;
747	crq_msg.var2.buffer_id = cpu_to_be16(buffer_id);
748	crq_msg.rsvd = 0;
749	crq_msg.var3.rsvd = 0;
750
751	ibmvmc_send_crq(adapter, be64_to_cpu(crq_as_u64[0]),
752	be64_to_cpu(crq_as_u64[1]));
753
754	return 0;
755	}
756
757	/**
758	* ibmvmc_send_msg - Signal Message
759	*
760	* @adapter: crq_server_adapter struct
761	* @buffer: ibmvmc_buffer struct
762	* @hmc: ibmvmc_hmc struct
763	* @msg_len: message length field
764	*
765	* This command is sent between the management partition and the hypervisor
766	* in order to signal the arrival of an HMC protocol message. The command
767	* can be sent by both the management partition and the hypervisor. It is
768	* used for all traffic between the management application and the hypervisor,
769	* regardless of who initiated the communication.
770	*
771	* There is no response to this message.
772	*
773	* Return:
774	* 0 - Success
775	* Non-zero - Failure
776	*/
777	static int ibmvmc_send_msg(struct crq_server_adapter *adapter,
778	struct ibmvmc_buffer *buffer,
779	struct ibmvmc_hmc *hmc, int msg_len)
780	{
781	struct ibmvmc_crq_msg crq_msg;
782	__be64 *crq_as_u64 = (__be64 *)&crq_msg;
783	int rc = 0;
784
785	dev_dbg(adapter->dev, "CRQ send: rdma to HV\n");
786	rc = h_copy_rdma(length: msg_len,
787	sliobn: adapter->liobn,
788	slioba: buffer->dma_addr_local,
789	dliobn: adapter->riobn,
790	dlioba: buffer->dma_addr_remote);
791	if (rc) {
792	dev_err(adapter->dev, "Error in send_msg, h_copy_rdma rc 0x%x\n",
793	rc);
794	return rc;
795	}
796
797	crq_msg.valid = 0x80;
798	crq_msg.type = VMC_MSG_SIGNAL;
799	crq_msg.status = 0;
800	crq_msg.var1.rsvd = 0;
801	crq_msg.hmc_session = hmc->session;
802	crq_msg.hmc_index = hmc->index;
803	crq_msg.var2.buffer_id = cpu_to_be16(buffer->id);
804	crq_msg.var3.msg_len = cpu_to_be32(msg_len);
805	dev_dbg(adapter->dev, "CRQ send: msg to HV 0x%llx 0x%llx\n",
806	be64_to_cpu(crq_as_u64[0]), be64_to_cpu(crq_as_u64[1]));
807
808	buffer->owner = VMC_BUF_OWNER_HV;
809	ibmvmc_send_crq(adapter, be64_to_cpu(crq_as_u64[0]),
810	be64_to_cpu(crq_as_u64[1]));
811
812	return rc;
813	}
814
815	/**
816	* ibmvmc_open - Open Session
817	*
818	* @inode: inode struct
819	* @file: file struct
820	*
821	* Return:
822	* 0 - Success
823	* Non-zero - Failure
824	*/
825	static int ibmvmc_open(struct inode *inode, struct file *file)
826	{
827	struct ibmvmc_file_session *session;
828
829	pr_debug("%s: inode = 0x%lx, file = 0x%lx, state = 0x%x\n", __func__,
830	(unsigned long)inode, (unsigned long)file,
831	ibmvmc.state);
832
833	session = kzalloc(size: sizeof(*session), GFP_KERNEL);
834	if (!session)
835	return -ENOMEM;
836
837	session->file = file;
838	file->private_data = session;
839
840	return 0;
841	}
842
843	/**
844	* ibmvmc_close - Close Session
845	*
846	* @inode: inode struct
847	* @file: file struct
848	*
849	* Return:
850	* 0 - Success
851	* Non-zero - Failure
852	*/
853	static int ibmvmc_close(struct inode *inode, struct file *file)
854	{
855	struct ibmvmc_file_session *session;
856	struct ibmvmc_hmc *hmc;
857	int rc = 0;
858	unsigned long flags;
859
860	pr_debug("%s: file = 0x%lx, state = 0x%x\n", __func__,
861	(unsigned long)file, ibmvmc.state);
862
863	session = file->private_data;
864	if (!session)
865	return -EIO;
866
867	hmc = session->hmc;
868	if (hmc) {
869	if (!hmc->adapter)
870	return -EIO;
871
872	if (ibmvmc.state == ibmvmc_state_failed) {
873	dev_warn(hmc->adapter->dev, "close: state_failed\n");
874	return -EIO;
875	}
876
877	spin_lock_irqsave(&hmc->lock, flags);
878	if (hmc->state >= ibmhmc_state_opening) {
879	rc = ibmvmc_send_close(hmc);
880	if (rc)
881	dev_warn(hmc->adapter->dev, "close: send_close failed.\n");
882	}
883	spin_unlock_irqrestore(lock: &hmc->lock, flags);
884	}
885
886	kfree_sensitive(objp: session);
887
888	return rc;
889	}
890
891	/**
892	* ibmvmc_read - Read
893	*
894	* @file: file struct
895	* @buf: Character buffer
896	* @nbytes: Size in bytes
897	* @ppos: Offset
898	*
899	* Return:
900	* 0 - Success
901	* Non-zero - Failure
902	*/
903	static ssize_t ibmvmc_read(struct file *file, char *buf, size_t nbytes,
904	loff_t *ppos)
905	{
906	struct ibmvmc_file_session *session;
907	struct ibmvmc_hmc *hmc;
908	struct crq_server_adapter *adapter;
909	struct ibmvmc_buffer *buffer;
910	ssize_t n;
911	ssize_t retval = 0;
912	unsigned long flags;
913	DEFINE_WAIT(wait);
914
915	pr_debug("ibmvmc: read: file = 0x%lx, buf = 0x%lx, nbytes = 0x%lx\n",
916	(unsigned long)file, (unsigned long)buf,
917	(unsigned long)nbytes);
918
919	if (nbytes == 0)
920	return 0;
921
922	if (nbytes > ibmvmc.max_mtu) {
923	pr_warn("ibmvmc: read: nbytes invalid 0x%x\n",
924	(unsigned int)nbytes);
925	return -EINVAL;
926	}
927
928	session = file->private_data;
929	if (!session) {
930	pr_warn("ibmvmc: read: no session\n");
931	return -EIO;
932	}
933
934	hmc = session->hmc;
935	if (!hmc) {
936	pr_warn("ibmvmc: read: no hmc\n");
937	return -EIO;
938	}
939
940	adapter = hmc->adapter;
941	if (!adapter) {
942	pr_warn("ibmvmc: read: no adapter\n");
943	return -EIO;
944	}
945
946	do {
947	prepare_to_wait(wq_head: &ibmvmc_read_wait, wq_entry: &wait, TASK_INTERRUPTIBLE);
948
949	spin_lock_irqsave(&hmc->lock, flags);
950	if (hmc->queue_tail != hmc->queue_head)
951	/* Data is available */
952	break;
953
954	spin_unlock_irqrestore(lock: &hmc->lock, flags);
955
956	if (!session->valid) {
957	retval = -EBADFD;
958	goto out;
959	}
960	if (file->f_flags & O_NONBLOCK) {
961	retval = -EAGAIN;
962	goto out;
963	}
964
965	schedule();
966
967	if (signal_pending(current)) {
968	retval = -ERESTARTSYS;
969	goto out;
970	}
971	} while (1);
972
973	buffer = &(hmc->buffer[hmc->queue_outbound_msgs[hmc->queue_tail]]);
974	hmc->queue_tail++;
975	if (hmc->queue_tail == ibmvmc_max_buf_pool_size)
976	hmc->queue_tail = 0;
977	spin_unlock_irqrestore(lock: &hmc->lock, flags);
978
979	nbytes = min_t(size_t, nbytes, buffer->msg_len);
980	n = copy_to_user(to: (void *)buf, from: buffer->real_addr_local, n: nbytes);
981	dev_dbg(adapter->dev, "read: copy to user nbytes = 0x%lx.\n", nbytes);
982	ibmvmc_free_hmc_buffer(hmc, buffer);
983	retval = nbytes;
984
985	if (n) {
986	dev_warn(adapter->dev, "read: copy to user failed.\n");
987	retval = -EFAULT;
988	}
989
990	out:
991	finish_wait(wq_head: &ibmvmc_read_wait, wq_entry: &wait);
992	dev_dbg(adapter->dev, "read: out %ld\n", retval);
993	return retval;
994	}
995
996	/**
997	* ibmvmc_poll - Poll
998	*
999	* @file: file struct
1000	* @wait: Poll Table
1001	*
1002	* Return:
1003	* poll.h return values
1004	*/
1005	static unsigned int ibmvmc_poll(struct file *file, poll_table *wait)
1006	{
1007	struct ibmvmc_file_session *session;
1008	struct ibmvmc_hmc *hmc;
1009	unsigned int mask = 0;
1010
1011	session = file->private_data;
1012	if (!session)
1013	return 0;
1014
1015	hmc = session->hmc;
1016	if (!hmc)
1017	return 0;
1018
1019	poll_wait(filp: file, wait_address: &ibmvmc_read_wait, p: wait);
1020
1021	if (hmc->queue_head != hmc->queue_tail)
1022	mask |= POLLIN | POLLRDNORM;
1023
1024	return mask;
1025	}
1026
1027	/**
1028	* ibmvmc_write - Write
1029	*
1030	* @file: file struct
1031	* @buffer: Character buffer
1032	* @count: Count field
1033	* @ppos: Offset
1034	*
1035	* Return:
1036	* 0 - Success
1037	* Non-zero - Failure
1038	*/
1039	static ssize_t ibmvmc_write(struct file *file, const char *buffer,
1040	size_t count, loff_t *ppos)
1041	{
1042	struct inode *inode;
1043	struct ibmvmc_buffer *vmc_buffer;
1044	struct ibmvmc_file_session *session;
1045	struct crq_server_adapter *adapter;
1046	struct ibmvmc_hmc *hmc;
1047	unsigned char *buf;
1048	unsigned long flags;
1049	size_t bytes;
1050	const char *p = buffer;
1051	size_t c = count;
1052	int ret = 0;
1053
1054	session = file->private_data;
1055	if (!session)
1056	return -EIO;
1057
1058	hmc = session->hmc;
1059	if (!hmc)
1060	return -EIO;
1061
1062	spin_lock_irqsave(&hmc->lock, flags);
1063	if (hmc->state == ibmhmc_state_free) {
1064	/* HMC connection is not valid (possibly was reset under us). */
1065	ret = -EIO;
1066	goto out;
1067	}
1068
1069	adapter = hmc->adapter;
1070	if (!adapter) {
1071	ret = -EIO;
1072	goto out;
1073	}
1074
1075	if (count > ibmvmc.max_mtu) {
1076	dev_warn(adapter->dev, "invalid buffer size 0x%lx\n",
1077	(unsigned long)count);
1078	ret = -EIO;
1079	goto out;
1080	}
1081
1082	/* Waiting for the open resp message to the ioctl(1) - retry */
1083	if (hmc->state == ibmhmc_state_opening) {
1084	ret = -EBUSY;
1085	goto out;
1086	}
1087
1088	/* Make sure the ioctl() was called & the open msg sent, and that
1089	* the HMC connection has not failed.
1090	*/
1091	if (hmc->state != ibmhmc_state_ready) {
1092	ret = -EIO;
1093	goto out;
1094	}
1095
1096	vmc_buffer = ibmvmc_get_valid_hmc_buffer(hmc_index: hmc->index);
1097	if (!vmc_buffer) {
1098	/* No buffer available for the msg send, or we have not yet
1099	* completed the open/open_resp sequence. Retry until this is
1100	* complete.
1101	*/
1102	ret = -EBUSY;
1103	goto out;
1104	}
1105	if (!vmc_buffer->real_addr_local) {
1106	dev_err(adapter->dev, "no buffer storage assigned\n");
1107	ret = -EIO;
1108	goto out;
1109	}
1110	buf = vmc_buffer->real_addr_local;
1111
1112	while (c > 0) {
1113	bytes = min_t(size_t, c, vmc_buffer->size);
1114
1115	bytes -= copy_from_user(to: buf, from: p, n: bytes);
1116	if (!bytes) {
1117	ret = -EFAULT;
1118	goto out;
1119	}
1120	c -= bytes;
1121	p += bytes;
1122	}
1123	if (p == buffer)
1124	goto out;
1125
1126	inode = file_inode(f: file);
1127	inode_set_mtime_to_ts(inode, ts: inode_set_ctime_current(inode));
1128	mark_inode_dirty(inode);
1129
1130	dev_dbg(adapter->dev, "write: file = 0x%lx, count = 0x%lx\n",
1131	(unsigned long)file, (unsigned long)count);
1132
1133	ibmvmc_send_msg(adapter, buffer: vmc_buffer, hmc, msg_len: count);
1134	ret = p - buffer;
1135	out:
1136	spin_unlock_irqrestore(lock: &hmc->lock, flags);
1137	return (ssize_t)(ret);
1138	}
1139
1140	/**
1141	* ibmvmc_setup_hmc - Setup the HMC
1142	*
1143	* @session: ibmvmc_file_session struct
1144	*
1145	* Return:
1146	* 0 - Success
1147	* Non-zero - Failure
1148	*/
1149	static long ibmvmc_setup_hmc(struct ibmvmc_file_session *session)
1150	{
1151	struct ibmvmc_hmc *hmc;
1152	unsigned int valid, free, index;
1153
1154	if (ibmvmc.state == ibmvmc_state_failed) {
1155	pr_warn("ibmvmc: Reserve HMC: state_failed\n");
1156	return -EIO;
1157	}
1158
1159	if (ibmvmc.state < ibmvmc_state_ready) {
1160	pr_warn("ibmvmc: Reserve HMC: not state_ready\n");
1161	return -EAGAIN;
1162	}
1163
1164	/* Device is busy until capabilities have been exchanged and we
1165	* have a generic buffer for each possible HMC connection.
1166	*/
1167	for (index = 0; index <= ibmvmc.max_hmc_index; index++) {
1168	valid = 0;
1169	ibmvmc_count_hmc_buffers(hmc_index: index, valid: &valid, free: &free);
1170	if (valid == 0) {
1171	pr_warn("ibmvmc: buffers not ready for index %d\n",
1172	index);
1173	return -ENOBUFS;
1174	}
1175	}
1176
1177	/* Get an hmc object, and transition to ibmhmc_state_initial */
1178	hmc = ibmvmc_get_free_hmc();
1179	if (!hmc) {
1180	pr_warn("%s: free hmc not found\n", __func__);
1181	return -EBUSY;
1182	}
1183
1184	hmc->session = hmc->session + 1;
1185	if (hmc->session == 0xff)
1186	hmc->session = 1;
1187
1188	session->hmc = hmc;
1189	hmc->adapter = &ibmvmc_adapter;
1190	hmc->file_session = session;
1191	session->valid = 1;
1192
1193	return 0;
1194	}
1195
1196	/**
1197	* ibmvmc_ioctl_sethmcid - IOCTL Set HMC ID
1198	*
1199	* @session: ibmvmc_file_session struct
1200	* @new_hmc_id: HMC id field
1201	*
1202	* IOCTL command to setup the hmc id
1203	*
1204	* Return:
1205	* 0 - Success
1206	* Non-zero - Failure
1207	*/
1208	static long ibmvmc_ioctl_sethmcid(struct ibmvmc_file_session *session,
1209	unsigned char __user *new_hmc_id)
1210	{
1211	struct ibmvmc_hmc *hmc;
1212	struct ibmvmc_buffer *buffer;
1213	size_t bytes;
1214	char print_buffer[HMC_ID_LEN + 1];
1215	unsigned long flags;
1216	long rc = 0;
1217
1218	/* Reserve HMC session */
1219	hmc = session->hmc;
1220	if (!hmc) {
1221	rc = ibmvmc_setup_hmc(session);
1222	if (rc)
1223	return rc;
1224
1225	hmc = session->hmc;
1226	if (!hmc) {
1227	pr_err("ibmvmc: setup_hmc success but no hmc\n");
1228	return -EIO;
1229	}
1230	}
1231
1232	if (hmc->state != ibmhmc_state_initial) {
1233	pr_warn("ibmvmc: sethmcid: invalid state to send open 0x%x\n",
1234	hmc->state);
1235	return -EIO;
1236	}
1237
1238	bytes = copy_from_user(to: hmc->hmc_id, from: new_hmc_id, HMC_ID_LEN);
1239	if (bytes)
1240	return -EFAULT;
1241
1242	/* Send Open Session command */
1243	spin_lock_irqsave(&hmc->lock, flags);
1244	buffer = ibmvmc_get_valid_hmc_buffer(hmc_index: hmc->index);
1245	spin_unlock_irqrestore(lock: &hmc->lock, flags);
1246
1247	if (!buffer || !buffer->real_addr_local) {
1248	pr_warn("ibmvmc: sethmcid: no buffer available\n");
1249	return -EIO;
1250	}
1251
1252	strscpy(p: print_buffer, q: hmc->hmc_id, size: sizeof(print_buffer));
1253	pr_info("ibmvmc: sethmcid: Set HMC ID: \"%s\"\n", print_buffer);
1254
1255	memcpy(buffer->real_addr_local, hmc->hmc_id, HMC_ID_LEN);
1256	/* RDMA over ID, send open msg, change state to ibmhmc_state_opening */
1257	rc = ibmvmc_send_open(buffer, hmc);
1258
1259	return rc;
1260	}
1261
1262	/**
1263	* ibmvmc_ioctl_query - IOCTL Query
1264	*
1265	* @session: ibmvmc_file_session struct
1266	* @ret_struct: ibmvmc_query_struct
1267	*
1268	* Return:
1269	* 0 - Success
1270	* Non-zero - Failure
1271	*/
1272	static long ibmvmc_ioctl_query(struct ibmvmc_file_session *session,
1273	struct ibmvmc_query_struct __user *ret_struct)
1274	{
1275	struct ibmvmc_query_struct query_struct;
1276	size_t bytes;
1277
1278	memset(&query_struct, 0, sizeof(query_struct));
1279	query_struct.have_vmc = (ibmvmc.state > ibmvmc_state_initial);
1280	query_struct.state = ibmvmc.state;
1281	query_struct.vmc_drc_index = ibmvmc.vmc_drc_index;
1282
1283	bytes = copy_to_user(to: ret_struct, from: &query_struct,
1284	n: sizeof(query_struct));
1285	if (bytes)
1286	return -EFAULT;
1287
1288	return 0;
1289	}
1290
1291	/**
1292	* ibmvmc_ioctl_requestvmc - IOCTL Request VMC
1293	*
1294	* @session: ibmvmc_file_session struct
1295	* @ret_vmc_index: VMC Index
1296	*
1297	* Return:
1298	* 0 - Success
1299	* Non-zero - Failure
1300	*/
1301	static long ibmvmc_ioctl_requestvmc(struct ibmvmc_file_session *session,
1302	u32 __user *ret_vmc_index)
1303	{
1304	/* TODO: (adreznec) Add locking to control multiple process access */
1305	size_t bytes;
1306	long rc;
1307	u32 vmc_drc_index;
1308
1309	/* Call to request the VMC device from phyp*/
1310	rc = h_request_vmc(vmc_index: &vmc_drc_index);
1311	pr_debug("ibmvmc: requestvmc: H_REQUEST_VMC rc = 0x%lx\n", rc);
1312
1313	if (rc == H_SUCCESS) {
1314	rc = 0;
1315	} else if (rc == H_FUNCTION) {
1316	pr_err("ibmvmc: requestvmc: h_request_vmc not supported\n");
1317	return -EPERM;
1318	} else if (rc == H_AUTHORITY) {
1319	pr_err("ibmvmc: requestvmc: hypervisor denied vmc request\n");
1320	return -EPERM;
1321	} else if (rc == H_HARDWARE) {
1322	pr_err("ibmvmc: requestvmc: hypervisor hardware fault\n");
1323	return -EIO;
1324	} else if (rc == H_RESOURCE) {
1325	pr_err("ibmvmc: requestvmc: vmc resource unavailable\n");
1326	return -ENODEV;
1327	} else if (rc == H_NOT_AVAILABLE) {
1328	pr_err("ibmvmc: requestvmc: system cannot be vmc managed\n");
1329	return -EPERM;
1330	} else if (rc == H_PARAMETER) {
1331	pr_err("ibmvmc: requestvmc: invalid parameter\n");
1332	return -EINVAL;
1333	}
1334
1335	/* Success, set the vmc index in global struct */
1336	ibmvmc.vmc_drc_index = vmc_drc_index;
1337
1338	bytes = copy_to_user(to: ret_vmc_index, from: &vmc_drc_index,
1339	n: sizeof(*ret_vmc_index));
1340	if (bytes) {
1341	pr_warn("ibmvmc: requestvmc: copy to user failed.\n");
1342	return -EFAULT;
1343	}
1344	return rc;
1345	}
1346
1347	/**
1348	* ibmvmc_ioctl - IOCTL
1349	*
1350	* @file: file information
1351	* @cmd: cmd field
1352	* @arg: Argument field
1353	*
1354	* Return:
1355	* 0 - Success
1356	* Non-zero - Failure
1357	*/
1358	static long ibmvmc_ioctl(struct file *file,
1359	unsigned int cmd, unsigned long arg)
1360	{
1361	struct ibmvmc_file_session *session = file->private_data;
1362
1363	pr_debug("ibmvmc: ioctl file=0x%lx, cmd=0x%x, arg=0x%lx, ses=0x%lx\n",
1364	(unsigned long)file, cmd, arg,
1365	(unsigned long)session);
1366
1367	if (!session) {
1368	pr_warn("ibmvmc: ioctl: no session\n");
1369	return -EIO;
1370	}
1371
1372	switch (cmd) {
1373	case VMC_IOCTL_SETHMCID:
1374	return ibmvmc_ioctl_sethmcid(session,
1375	new_hmc_id: (unsigned char __user *)arg);
1376	case VMC_IOCTL_QUERY:
1377	return ibmvmc_ioctl_query(session,
1378	ret_struct: (struct ibmvmc_query_struct __user *)arg);
1379	case VMC_IOCTL_REQUESTVMC:
1380	return ibmvmc_ioctl_requestvmc(session,
1381	ret_vmc_index: (unsigned int __user *)arg);
1382	default:
1383	pr_warn("ibmvmc: unknown ioctl 0x%x\n", cmd);
1384	return -EINVAL;
1385	}
1386	}
1387
1388	static const struct file_operations ibmvmc_fops = {
1389	.owner = THIS_MODULE,
1390	.read = ibmvmc_read,
1391	.write = ibmvmc_write,
1392	.poll = ibmvmc_poll,
1393	.unlocked_ioctl = ibmvmc_ioctl,
1394	.open = ibmvmc_open,
1395	.release = ibmvmc_close,
1396	};
1397
1398	/**
1399	* ibmvmc_add_buffer - Add Buffer
1400	*
1401	* @adapter: crq_server_adapter struct
1402	* @crq: ibmvmc_crq_msg struct
1403	*
1404	* This message transfers a buffer from hypervisor ownership to management
1405	* partition ownership. The LIOBA is obtained from the virtual TCE table
1406	* associated with the hypervisor side of the VMC device, and points to a
1407	* buffer of size MTU (as established in the capabilities exchange).
1408	*
1409	* Typical flow for ading buffers:
1410	* 1. A new management application connection is opened by the management
1411	* partition.
1412	* 2. The hypervisor assigns new buffers for the traffic associated with
1413	* that connection.
1414	* 3. The hypervisor sends VMC Add Buffer messages to the management
1415	* partition, informing it of the new buffers.
1416	* 4. The hypervisor sends an HMC protocol message (to the management
1417	* application) notifying it of the new buffers. This informs the
1418	* application that it has buffers available for sending HMC
1419	* commands.
1420	*
1421	* Return:
1422	* 0 - Success
1423	* Non-zero - Failure
1424	*/
1425	static int ibmvmc_add_buffer(struct crq_server_adapter *adapter,
1426	struct ibmvmc_crq_msg *crq)
1427	{
1428	struct ibmvmc_buffer *buffer;
1429	u8 hmc_index;
1430	u8 hmc_session;
1431	u16 buffer_id;
1432	unsigned long flags;
1433	int rc = 0;
1434
1435	if (!crq)
1436	return -1;
1437
1438	hmc_session = crq->hmc_session;
1439	hmc_index = crq->hmc_index;
1440	buffer_id = be16_to_cpu(crq->var2.buffer_id);
1441
1442	if (hmc_index > ibmvmc.max_hmc_index) {
1443	dev_err(adapter->dev, "add_buffer: invalid hmc_index = 0x%x\n",
1444	hmc_index);
1445	ibmvmc_send_add_buffer_resp(adapter, VMC_MSG_INVALID_HMC_INDEX,
1446	hmc_session, hmc_index, buffer_id);
1447	return -1;
1448	}
1449
1450	if (buffer_id >= ibmvmc.max_buffer_pool_size) {
1451	dev_err(adapter->dev, "add_buffer: invalid buffer_id = 0x%x\n",
1452	buffer_id);
1453	ibmvmc_send_add_buffer_resp(adapter, VMC_MSG_INVALID_BUFFER_ID,
1454	hmc_session, hmc_index, buffer_id);
1455	return -1;
1456	}
1457
1458	spin_lock_irqsave(&hmcs[hmc_index].lock, flags);
1459	buffer = &hmcs[hmc_index].buffer[buffer_id];
1460
1461	if (buffer->real_addr_local || buffer->dma_addr_local) {
1462	dev_warn(adapter->dev, "add_buffer: already allocated id = 0x%lx\n",
1463	(unsigned long)buffer_id);
1464	spin_unlock_irqrestore(lock: &hmcs[hmc_index].lock, flags);
1465	ibmvmc_send_add_buffer_resp(adapter, VMC_MSG_INVALID_BUFFER_ID,
1466	hmc_session, hmc_index, buffer_id);
1467	return -1;
1468	}
1469
1470	buffer->real_addr_local = alloc_dma_buffer(vdev: to_vio_dev(adapter->dev),
1471	size: ibmvmc.max_mtu,
1472	dma_handle: &buffer->dma_addr_local);
1473
1474	if (!buffer->real_addr_local) {
1475	dev_err(adapter->dev, "add_buffer: alloc_dma_buffer failed.\n");
1476	spin_unlock_irqrestore(lock: &hmcs[hmc_index].lock, flags);
1477	ibmvmc_send_add_buffer_resp(adapter, VMC_MSG_INTERFACE_FAILURE,
1478	hmc_session, hmc_index, buffer_id);
1479	return -1;
1480	}
1481
1482	buffer->dma_addr_remote = be32_to_cpu(crq->var3.lioba);
1483	buffer->size = ibmvmc.max_mtu;
1484	buffer->owner = crq->var1.owner;
1485	buffer->free = 1;
1486	/* Must ensure valid==1 is observable only after all other fields are */
1487	dma_wmb();
1488	buffer->valid = 1;
1489	buffer->id = buffer_id;
1490
1491	dev_dbg(adapter->dev, "add_buffer: successfully added a buffer:\n");
1492	dev_dbg(adapter->dev, " index: %d, session: %d, buffer: 0x%x, owner: %d\n",
1493	hmc_index, hmc_session, buffer_id, buffer->owner);
1494	dev_dbg(adapter->dev, " local: 0x%x, remote: 0x%x\n",
1495	(u32)buffer->dma_addr_local,
1496	(u32)buffer->dma_addr_remote);
1497	spin_unlock_irqrestore(lock: &hmcs[hmc_index].lock, flags);
1498
1499	ibmvmc_send_add_buffer_resp(adapter, VMC_MSG_SUCCESS, hmc_session,
1500	hmc_index, buffer_id);
1501
1502	return rc;
1503	}
1504
1505	/**
1506	* ibmvmc_rem_buffer - Remove Buffer
1507	*
1508	* @adapter: crq_server_adapter struct
1509	* @crq: ibmvmc_crq_msg struct
1510	*
1511	* This message requests an HMC buffer to be transferred from management
1512	* partition ownership to hypervisor ownership. The management partition may
1513	* not be able to satisfy the request at a particular point in time if all its
1514	* buffers are in use. The management partition requires a depth of at least
1515	* one inbound buffer to allow management application commands to flow to the
1516	* hypervisor. It is, therefore, an interface error for the hypervisor to
1517	* attempt to remove the management partition's last buffer.
1518	*
1519	* The hypervisor is expected to manage buffer usage with the management
1520	* application directly and inform the management partition when buffers may be
1521	* removed. The typical flow for removing buffers:
1522	*
1523	* 1. The management application no longer needs a communication path to a
1524	* particular hypervisor function. That function is closed.
1525	* 2. The hypervisor and the management application quiesce all traffic to that
1526	* function. The hypervisor requests a reduction in buffer pool size.
1527	* 3. The management application acknowledges the reduction in buffer pool size.
1528	* 4. The hypervisor sends a Remove Buffer message to the management partition,
1529	* informing it of the reduction in buffers.
1530	* 5. The management partition verifies it can remove the buffer. This is
1531	* possible if buffers have been quiesced.
1532	*
1533	* Return:
1534	* 0 - Success
1535	* Non-zero - Failure
1536	*/
1537	/*
1538	* The hypervisor requested that we pick an unused buffer, and return it.
1539	* Before sending the buffer back, we free any storage associated with the
1540	* buffer.
1541	*/
1542	static int ibmvmc_rem_buffer(struct crq_server_adapter *adapter,
1543	struct ibmvmc_crq_msg *crq)
1544	{
1545	struct ibmvmc_buffer *buffer;
1546	u8 hmc_index;
1547	u8 hmc_session;
1548	u16 buffer_id = 0;
1549	unsigned long flags;
1550	int rc = 0;
1551
1552	if (!crq)
1553	return -1;
1554
1555	hmc_session = crq->hmc_session;
1556	hmc_index = crq->hmc_index;
1557
1558	if (hmc_index > ibmvmc.max_hmc_index) {
1559	dev_warn(adapter->dev, "rem_buffer: invalid hmc_index = 0x%x\n",
1560	hmc_index);
1561	ibmvmc_send_rem_buffer_resp(adapter, VMC_MSG_INVALID_HMC_INDEX,
1562	hmc_session, hmc_index, buffer_id);
1563	return -1;
1564	}
1565
1566	spin_lock_irqsave(&hmcs[hmc_index].lock, flags);
1567	buffer = ibmvmc_get_free_hmc_buffer(adapter, hmc_index);
1568	if (!buffer) {
1569	dev_info(adapter->dev, "rem_buffer: no buffer to remove\n");
1570	spin_unlock_irqrestore(lock: &hmcs[hmc_index].lock, flags);
1571	ibmvmc_send_rem_buffer_resp(adapter, VMC_MSG_NO_BUFFER,
1572	hmc_session, hmc_index,
1573	VMC_INVALID_BUFFER_ID);
1574	return -1;
1575	}
1576
1577	buffer_id = buffer->id;
1578
1579	if (buffer->valid)
1580	free_dma_buffer(vdev: to_vio_dev(adapter->dev),
1581	size: ibmvmc.max_mtu,
1582	vaddr: buffer->real_addr_local,
1583	dma_handle: buffer->dma_addr_local);
1584
1585	memset(buffer, 0, sizeof(struct ibmvmc_buffer));
1586	spin_unlock_irqrestore(lock: &hmcs[hmc_index].lock, flags);
1587
1588	dev_dbg(adapter->dev, "rem_buffer: removed buffer 0x%x.\n", buffer_id);
1589	ibmvmc_send_rem_buffer_resp(adapter, VMC_MSG_SUCCESS, hmc_session,
1590	hmc_index, buffer_id);
1591
1592	return rc;
1593	}
1594
1595	static int ibmvmc_recv_msg(struct crq_server_adapter *adapter,
1596	struct ibmvmc_crq_msg *crq)
1597	{
1598	struct ibmvmc_buffer *buffer;
1599	struct ibmvmc_hmc *hmc;
1600	unsigned long msg_len;
1601	u8 hmc_index;
1602	u8 hmc_session;
1603	u16 buffer_id;
1604	unsigned long flags;
1605	int rc = 0;
1606
1607	if (!crq)
1608	return -1;
1609
1610	/* Hypervisor writes CRQs directly into our memory in big endian */
1611	dev_dbg(adapter->dev, "Recv_msg: msg from HV 0x%016llx 0x%016llx\n",
1612	be64_to_cpu(*((unsigned long *)crq)),
1613	be64_to_cpu(*(((unsigned long *)crq) + 1)));
1614
1615	hmc_session = crq->hmc_session;
1616	hmc_index = crq->hmc_index;
1617	buffer_id = be16_to_cpu(crq->var2.buffer_id);
1618	msg_len = be32_to_cpu(crq->var3.msg_len);
1619
1620	if (hmc_index > ibmvmc.max_hmc_index) {
1621	dev_err(adapter->dev, "Recv_msg: invalid hmc_index = 0x%x\n",
1622	hmc_index);
1623	ibmvmc_send_add_buffer_resp(adapter, VMC_MSG_INVALID_HMC_INDEX,
1624	hmc_session, hmc_index, buffer_id);
1625	return -1;
1626	}
1627
1628	if (buffer_id >= ibmvmc.max_buffer_pool_size) {
1629	dev_err(adapter->dev, "Recv_msg: invalid buffer_id = 0x%x\n",
1630	buffer_id);
1631	ibmvmc_send_add_buffer_resp(adapter, VMC_MSG_INVALID_BUFFER_ID,
1632	hmc_session, hmc_index, buffer_id);
1633	return -1;
1634	}
1635
1636	hmc = &hmcs[hmc_index];
1637	spin_lock_irqsave(&hmc->lock, flags);
1638
1639	if (hmc->state == ibmhmc_state_free) {
1640	dev_err(adapter->dev, "Recv_msg: invalid hmc state = 0x%x\n",
1641	hmc->state);
1642	/* HMC connection is not valid (possibly was reset under us). */
1643	spin_unlock_irqrestore(lock: &hmc->lock, flags);
1644	return -1;
1645	}
1646
1647	buffer = &hmc->buffer[buffer_id];
1648
1649	if (buffer->valid == 0 || buffer->owner == VMC_BUF_OWNER_ALPHA) {
1650	dev_err(adapter->dev, "Recv_msg: not valid, or not HV. 0x%x 0x%x\n",
1651	buffer->valid, buffer->owner);
1652	spin_unlock_irqrestore(lock: &hmc->lock, flags);
1653	return -1;
1654	}
1655
1656	/* RDMA the data into the partition. */
1657	rc = h_copy_rdma(length: msg_len,
1658	sliobn: adapter->riobn,
1659	slioba: buffer->dma_addr_remote,
1660	dliobn: adapter->liobn,
1661	dlioba: buffer->dma_addr_local);
1662
1663	dev_dbg(adapter->dev, "Recv_msg: msg_len = 0x%x, buffer_id = 0x%x, queue_head = 0x%x, hmc_idx = 0x%x\n",
1664	(unsigned int)msg_len, (unsigned int)buffer_id,
1665	(unsigned int)hmc->queue_head, (unsigned int)hmc_index);
1666	buffer->msg_len = msg_len;
1667	buffer->free = 0;
1668	buffer->owner = VMC_BUF_OWNER_ALPHA;
1669
1670	if (rc) {
1671	dev_err(adapter->dev, "Failure in recv_msg: h_copy_rdma = 0x%x\n",
1672	rc);
1673	spin_unlock_irqrestore(lock: &hmc->lock, flags);
1674	return -1;
1675	}
1676
1677	/* Must be locked because read operates on the same data */
1678	hmc->queue_outbound_msgs[hmc->queue_head] = buffer_id;
1679	hmc->queue_head++;
1680	if (hmc->queue_head == ibmvmc_max_buf_pool_size)
1681	hmc->queue_head = 0;
1682
1683	if (hmc->queue_head == hmc->queue_tail)
1684	dev_err(adapter->dev, "outbound buffer queue wrapped.\n");
1685
1686	spin_unlock_irqrestore(lock: &hmc->lock, flags);
1687
1688	wake_up_interruptible(&ibmvmc_read_wait);
1689
1690	return 0;
1691	}
1692
1693	/**
1694	* ibmvmc_process_capabilities - Process Capabilities
1695	*
1696	* @adapter: crq_server_adapter struct
1697	* @crqp: ibmvmc_crq_msg struct
1698	*
1699	*/
1700	static void ibmvmc_process_capabilities(struct crq_server_adapter *adapter,
1701	struct ibmvmc_crq_msg *crqp)
1702	{
1703	struct ibmvmc_admin_crq_msg *crq = (struct ibmvmc_admin_crq_msg *)crqp;
1704
1705	if ((be16_to_cpu(crq->version) >> 8) !=
1706	(IBMVMC_PROTOCOL_VERSION >> 8)) {
1707	dev_err(adapter->dev, "init failed, incompatible versions 0x%x 0x%x\n",
1708	be16_to_cpu(crq->version),
1709	IBMVMC_PROTOCOL_VERSION);
1710	ibmvmc.state = ibmvmc_state_failed;
1711	return;
1712	}
1713
1714	ibmvmc.max_mtu = min_t(u32, ibmvmc_max_mtu, be32_to_cpu(crq->max_mtu));
1715	ibmvmc.max_buffer_pool_size = min_t(u16, ibmvmc_max_buf_pool_size,
1716	be16_to_cpu(crq->pool_size));
1717	ibmvmc.max_hmc_index = min_t(u8, ibmvmc_max_hmcs, crq->max_hmc) - 1;
1718	ibmvmc.state = ibmvmc_state_ready;
1719
1720	dev_info(adapter->dev, "Capabilities: mtu=0x%x, pool_size=0x%x, max_hmc=0x%x\n",
1721	ibmvmc.max_mtu, ibmvmc.max_buffer_pool_size,
1722	ibmvmc.max_hmc_index);
1723	}
1724
1725	/**
1726	* ibmvmc_validate_hmc_session - Validate HMC Session
1727	*
1728	* @adapter: crq_server_adapter struct
1729	* @crq: ibmvmc_crq_msg struct
1730	*
1731	* Return:
1732	* 0 - Success
1733	* Non-zero - Failure
1734	*/
1735	static int ibmvmc_validate_hmc_session(struct crq_server_adapter *adapter,
1736	struct ibmvmc_crq_msg *crq)
1737	{
1738	unsigned char hmc_index;
1739
1740	hmc_index = crq->hmc_index;
1741
1742	if (crq->hmc_session == 0)
1743	return 0;
1744
1745	if (hmc_index > ibmvmc.max_hmc_index)
1746	return -1;
1747
1748	if (hmcs[hmc_index].session != crq->hmc_session) {
1749	dev_warn(adapter->dev, "Drop, bad session: expected 0x%x, recv 0x%x\n",
1750	hmcs[hmc_index].session, crq->hmc_session);
1751	return -1;
1752	}
1753
1754	return 0;
1755	}
1756
1757	/**
1758	* ibmvmc_reset - Reset
1759	*
1760	* @adapter: crq_server_adapter struct
1761	* @xport_event: export_event field
1762	*
1763	* Closes all HMC sessions and conditionally schedules a CRQ reset.
1764	* @xport_event: If true, the partner closed their CRQ; we don't need to reset.
1765	* If false, we need to schedule a CRQ reset.
1766	*/
1767	static void ibmvmc_reset(struct crq_server_adapter *adapter, bool xport_event)
1768	{
1769	int i;
1770
1771	if (ibmvmc.state != ibmvmc_state_sched_reset) {
1772	dev_info(adapter->dev, "*** Reset to initial state.\n");
1773	for (i = 0; i < ibmvmc_max_hmcs; i++)
1774	ibmvmc_return_hmc(hmc: &hmcs[i], release_readers: xport_event);
1775
1776	if (xport_event) {
1777	/* CRQ was closed by the partner. We don't need to do
1778	* anything except set ourself to the correct state to
1779	* handle init msgs.
1780	*/
1781	ibmvmc.state = ibmvmc_state_crqinit;
1782	} else {
1783	/* The partner did not close their CRQ - instead, we're
1784	* closing the CRQ on our end. Need to schedule this
1785	* for process context, because CRQ reset may require a
1786	* sleep.
1787	*
1788	* Setting ibmvmc.state here immediately prevents
1789	* ibmvmc_open from completing until the reset
1790	* completes in process context.
1791	*/
1792	ibmvmc.state = ibmvmc_state_sched_reset;
1793	dev_dbg(adapter->dev, "Device reset scheduled");
1794	wake_up_interruptible(&adapter->reset_wait_queue);
1795	}
1796	}
1797	}
1798
1799	/**
1800	* ibmvmc_reset_task - Reset Task
1801	*
1802	* @data: Data field
1803	*
1804	* Performs a CRQ reset of the VMC device in process context.
1805	* NOTE: This function should not be called directly, use ibmvmc_reset.
1806	*/
1807	static int ibmvmc_reset_task(void *data)
1808	{
1809	struct crq_server_adapter *adapter = data;
1810	int rc;
1811
1812	set_user_nice(current, nice: -20);
1813
1814	while (!kthread_should_stop()) {
1815	wait_event_interruptible(adapter->reset_wait_queue,
1816	(ibmvmc.state == ibmvmc_state_sched_reset) ||
1817	kthread_should_stop());
1818
1819	if (kthread_should_stop())
1820	break;
1821
1822	dev_dbg(adapter->dev, "CRQ resetting in process context");
1823	tasklet_disable(t: &adapter->work_task);
1824
1825	rc = ibmvmc_reset_crq_queue(adapter);
1826
1827	if (rc != H_SUCCESS && rc != H_RESOURCE) {
1828	dev_err(adapter->dev, "Error initializing CRQ. rc = 0x%x\n",
1829	rc);
1830	ibmvmc.state = ibmvmc_state_failed;
1831	} else {
1832	ibmvmc.state = ibmvmc_state_crqinit;
1833
1834	if (ibmvmc_send_crq(adapter, word1: 0xC001000000000000LL, word2: 0)
1835	!= 0 && rc != H_RESOURCE)
1836	dev_warn(adapter->dev, "Failed to send initialize CRQ message\n");
1837	}
1838
1839	vio_enable_interrupts(to_vio_dev(adapter->dev));
1840	tasklet_enable(t: &adapter->work_task);
1841	}
1842
1843	return 0;
1844	}
1845
1846	/**
1847	* ibmvmc_process_open_resp - Process Open Response
1848	*
1849	* @crq: ibmvmc_crq_msg struct
1850	* @adapter: crq_server_adapter struct
1851	*
1852	* This command is sent by the hypervisor in response to the Interface
1853	* Open message. When this message is received, the indicated buffer is
1854	* again available for management partition use.
1855	*/
1856	static void ibmvmc_process_open_resp(struct ibmvmc_crq_msg *crq,
1857	struct crq_server_adapter *adapter)
1858	{
1859	unsigned char hmc_index;
1860	unsigned short buffer_id;
1861
1862	hmc_index = crq->hmc_index;
1863	if (hmc_index > ibmvmc.max_hmc_index) {
1864	/* Why would PHYP give an index > max negotiated? */
1865	ibmvmc_reset(adapter, xport_event: false);
1866	return;
1867	}
1868
1869	if (crq->status) {
1870	dev_warn(adapter->dev, "open_resp: failed - status 0x%x\n",
1871	crq->status);
1872	ibmvmc_return_hmc(hmc: &hmcs[hmc_index], release_readers: false);
1873	return;
1874	}
1875
1876	if (hmcs[hmc_index].state == ibmhmc_state_opening) {
1877	buffer_id = be16_to_cpu(crq->var2.buffer_id);
1878	if (buffer_id >= ibmvmc.max_buffer_pool_size) {
1879	dev_err(adapter->dev, "open_resp: invalid buffer_id = 0x%x\n",
1880	buffer_id);
1881	hmcs[hmc_index].state = ibmhmc_state_failed;
1882	} else {
1883	ibmvmc_free_hmc_buffer(hmc: &hmcs[hmc_index],
1884	buffer: &hmcs[hmc_index].buffer[buffer_id]);
1885	hmcs[hmc_index].state = ibmhmc_state_ready;
1886	dev_dbg(adapter->dev, "open_resp: set hmc state = ready\n");
1887	}
1888	} else {
1889	dev_warn(adapter->dev, "open_resp: invalid hmc state (0x%x)\n",
1890	hmcs[hmc_index].state);
1891	}
1892	}
1893
1894	/**
1895	* ibmvmc_process_close_resp - Process Close Response
1896	*
1897	* @crq: ibmvmc_crq_msg struct
1898	* @adapter: crq_server_adapter struct
1899	*
1900	* This command is sent by the hypervisor in response to the managemant
1901	* application Interface Close message.
1902	*
1903	* If the close fails, simply reset the entire driver as the state of the VMC
1904	* must be in tough shape.
1905	*/
1906	static void ibmvmc_process_close_resp(struct ibmvmc_crq_msg *crq,
1907	struct crq_server_adapter *adapter)
1908	{
1909	unsigned char hmc_index;
1910
1911	hmc_index = crq->hmc_index;
1912	if (hmc_index > ibmvmc.max_hmc_index) {
1913	ibmvmc_reset(adapter, xport_event: false);
1914	return;
1915	}
1916
1917	if (crq->status) {
1918	dev_warn(adapter->dev, "close_resp: failed - status 0x%x\n",
1919	crq->status);
1920	ibmvmc_reset(adapter, xport_event: false);
1921	return;
1922	}
1923
1924	ibmvmc_return_hmc(hmc: &hmcs[hmc_index], release_readers: false);
1925	}
1926
1927	/**
1928	* ibmvmc_crq_process - Process CRQ
1929	*
1930	* @adapter: crq_server_adapter struct
1931	* @crq: ibmvmc_crq_msg struct
1932	*
1933	* Process the CRQ message based upon the type of message received.
1934	*
1935	*/
1936	static void ibmvmc_crq_process(struct crq_server_adapter *adapter,
1937	struct ibmvmc_crq_msg *crq)
1938	{
1939	switch (crq->type) {
1940	case VMC_MSG_CAP_RESP:
1941	dev_dbg(adapter->dev, "CRQ recv: capabilities resp (0x%x)\n",
1942	crq->type);
1943	if (ibmvmc.state == ibmvmc_state_capabilities)
1944	ibmvmc_process_capabilities(adapter, crqp: crq);
1945	else
1946	dev_warn(adapter->dev, "caps msg invalid in state 0x%x\n",
1947	ibmvmc.state);
1948	break;
1949	case VMC_MSG_OPEN_RESP:
1950	dev_dbg(adapter->dev, "CRQ recv: open resp (0x%x)\n",
1951	crq->type);
1952	if (ibmvmc_validate_hmc_session(adapter, crq) == 0)
1953	ibmvmc_process_open_resp(crq, adapter);
1954	break;
1955	case VMC_MSG_ADD_BUF:
1956	dev_dbg(adapter->dev, "CRQ recv: add buf (0x%x)\n",
1957	crq->type);
1958	if (ibmvmc_validate_hmc_session(adapter, crq) == 0)
1959	ibmvmc_add_buffer(adapter, crq);
1960	break;
1961	case VMC_MSG_REM_BUF:
1962	dev_dbg(adapter->dev, "CRQ recv: rem buf (0x%x)\n",
1963	crq->type);
1964	if (ibmvmc_validate_hmc_session(adapter, crq) == 0)
1965	ibmvmc_rem_buffer(adapter, crq);
1966	break;
1967	case VMC_MSG_SIGNAL:
1968	dev_dbg(adapter->dev, "CRQ recv: signal msg (0x%x)\n",
1969	crq->type);
1970	if (ibmvmc_validate_hmc_session(adapter, crq) == 0)
1971	ibmvmc_recv_msg(adapter, crq);
1972	break;
1973	case VMC_MSG_CLOSE_RESP:
1974	dev_dbg(adapter->dev, "CRQ recv: close resp (0x%x)\n",
1975	crq->type);
1976	if (ibmvmc_validate_hmc_session(adapter, crq) == 0)
1977	ibmvmc_process_close_resp(crq, adapter);
1978	break;
1979	case VMC_MSG_CAP:
1980	case VMC_MSG_OPEN:
1981	case VMC_MSG_CLOSE:
1982	case VMC_MSG_ADD_BUF_RESP:
1983	case VMC_MSG_REM_BUF_RESP:
1984	dev_warn(adapter->dev, "CRQ recv: unexpected msg (0x%x)\n",
1985	crq->type);
1986	break;
1987	default:
1988	dev_warn(adapter->dev, "CRQ recv: unknown msg (0x%x)\n",
1989	crq->type);
1990	break;
1991	}
1992	}
1993
1994	/**
1995	* ibmvmc_handle_crq_init - Handle CRQ Init
1996	*
1997	* @crq: ibmvmc_crq_msg struct
1998	* @adapter: crq_server_adapter struct
1999	*
2000	* Handle the type of crq initialization based on whether
2001	* it is a message or a response.
2002	*
2003	*/
2004	static void ibmvmc_handle_crq_init(struct ibmvmc_crq_msg *crq,
2005	struct crq_server_adapter *adapter)
2006	{
2007	switch (crq->type) {
2008	case 0x01: /* Initialization message */
2009	dev_dbg(adapter->dev, "CRQ recv: CRQ init msg - state 0x%x\n",
2010	ibmvmc.state);
2011	if (ibmvmc.state == ibmvmc_state_crqinit) {
2012	/* Send back a response */
2013	if (ibmvmc_send_crq(adapter, word1: 0xC002000000000000,
2014	word2: 0) == 0)
2015	ibmvmc_send_capabilities(adapter);
2016	else
2017	dev_err(adapter->dev, " Unable to send init rsp\n");
2018	} else {
2019	dev_err(adapter->dev, "Invalid state 0x%x mtu = 0x%x\n",
2020	ibmvmc.state, ibmvmc.max_mtu);
2021	}
2022
2023	break;
2024	case 0x02: /* Initialization response */
2025	dev_dbg(adapter->dev, "CRQ recv: initialization resp msg - state 0x%x\n",
2026	ibmvmc.state);
2027	if (ibmvmc.state == ibmvmc_state_crqinit)
2028	ibmvmc_send_capabilities(adapter);
2029	break;
2030	default:
2031	dev_warn(adapter->dev, "Unknown crq message type 0x%lx\n",
2032	(unsigned long)crq->type);
2033	}
2034	}
2035
2036	/**
2037	* ibmvmc_handle_crq - Handle CRQ
2038	*
2039	* @crq: ibmvmc_crq_msg struct
2040	* @adapter: crq_server_adapter struct
2041	*
2042	* Read the command elements from the command queue and execute the
2043	* requests based upon the type of crq message.
2044	*
2045	*/
2046	static void ibmvmc_handle_crq(struct ibmvmc_crq_msg *crq,
2047	struct crq_server_adapter *adapter)
2048	{
2049	switch (crq->valid) {
2050	case 0xC0: /* initialization */
2051	ibmvmc_handle_crq_init(crq, adapter);
2052	break;
2053	case 0xFF: /* Hypervisor telling us the connection is closed */
2054	dev_warn(adapter->dev, "CRQ recv: virtual adapter failed - resetting.\n");
2055	ibmvmc_reset(adapter, xport_event: true);
2056	break;
2057	case 0x80: /* real payload */
2058	ibmvmc_crq_process(adapter, crq);
2059	break;
2060	default:
2061	dev_warn(adapter->dev, "CRQ recv: unknown msg 0x%02x.\n",
2062	crq->valid);
2063	break;
2064	}
2065	}
2066
2067	static void ibmvmc_task(unsigned long data)
2068	{
2069	struct crq_server_adapter *adapter =
2070	(struct crq_server_adapter *)data;
2071	struct vio_dev *vdev = to_vio_dev(adapter->dev);
2072	struct ibmvmc_crq_msg *crq;
2073	int done = 0;
2074
2075	while (!done) {
2076	/* Pull all the valid messages off the CRQ */
2077	while ((crq = crq_queue_next_crq(queue: &adapter->queue)) != NULL) {
2078	ibmvmc_handle_crq(crq, adapter);
2079	crq->valid = 0x00;
2080	/* CRQ reset was requested, stop processing CRQs.
2081	* Interrupts will be re-enabled by the reset task.
2082	*/
2083	if (ibmvmc.state == ibmvmc_state_sched_reset)
2084	return;
2085	}
2086
2087	vio_enable_interrupts(vdev);
2088	crq = crq_queue_next_crq(queue: &adapter->queue);
2089	if (crq) {
2090	vio_disable_interrupts(vdev);
2091	ibmvmc_handle_crq(crq, adapter);
2092	crq->valid = 0x00;
2093	/* CRQ reset was requested, stop processing CRQs.
2094	* Interrupts will be re-enabled by the reset task.
2095	*/
2096	if (ibmvmc.state == ibmvmc_state_sched_reset)
2097	return;
2098	} else {
2099	done = 1;
2100	}
2101	}
2102	}
2103
2104	/**
2105	* ibmvmc_init_crq_queue - Init CRQ Queue
2106	*
2107	* @adapter: crq_server_adapter struct
2108	*
2109	* Return:
2110	* 0 - Success
2111	* Non-zero - Failure
2112	*/
2113	static int ibmvmc_init_crq_queue(struct crq_server_adapter *adapter)
2114	{
2115	struct vio_dev *vdev = to_vio_dev(adapter->dev);
2116	struct crq_queue *queue = &adapter->queue;
2117	int rc = 0;
2118	int retrc = 0;
2119
2120	queue->msgs = (struct ibmvmc_crq_msg *)get_zeroed_page(GFP_KERNEL);
2121
2122	if (!queue->msgs)
2123	goto malloc_failed;
2124
2125	queue->size = PAGE_SIZE / sizeof(*queue->msgs);
2126
2127	queue->msg_token = dma_map_single(adapter->dev, queue->msgs,
2128	queue->size * sizeof(*queue->msgs),
2129	DMA_BIDIRECTIONAL);
2130
2131	if (dma_mapping_error(adapter->dev, queue->msg_token))
2132	goto map_failed;
2133
2134	retrc = plpar_hcall_norets(H_REG_CRQ,
2135	vdev->unit_address,
2136	queue->msg_token, PAGE_SIZE);
2137	rc = retrc;
2138
2139	if (rc == H_RESOURCE)
2140	rc = ibmvmc_reset_crq_queue(adapter);
2141
2142	if (rc == 2) {
2143	dev_warn(adapter->dev, "Partner adapter not ready\n");
2144	retrc = 0;
2145	} else if (rc != 0) {
2146	dev_err(adapter->dev, "Error %d opening adapter\n", rc);
2147	goto reg_crq_failed;
2148	}
2149
2150	queue->cur = 0;
2151	spin_lock_init(&queue->lock);
2152
2153	tasklet_init(t: &adapter->work_task, func: ibmvmc_task, data: (unsigned long)adapter);
2154
2155	if (request_irq(irq: vdev->irq,
2156	handler: ibmvmc_handle_event,
2157	flags: 0, name: "ibmvmc", dev: (void *)adapter) != 0) {
2158	dev_err(adapter->dev, "couldn't register irq 0x%x\n",
2159	vdev->irq);
2160	goto req_irq_failed;
2161	}
2162
2163	rc = vio_enable_interrupts(vdev);
2164	if (rc != 0) {
2165	dev_err(adapter->dev, "Error %d enabling interrupts!!!\n", rc);
2166	goto req_irq_failed;
2167	}
2168
2169	return retrc;
2170
2171	req_irq_failed:
2172	/* Cannot have any work since we either never got our IRQ registered,
2173	* or never got interrupts enabled
2174	*/
2175	tasklet_kill(t: &adapter->work_task);
2176	h_free_crq(unit_address: vdev->unit_address);
2177	reg_crq_failed:
2178	dma_unmap_single(adapter->dev,
2179	queue->msg_token,
2180	queue->size * sizeof(*queue->msgs), DMA_BIDIRECTIONAL);
2181	map_failed:
2182	free_page((unsigned long)queue->msgs);
2183	malloc_failed:
2184	return -ENOMEM;
2185	}
2186
2187	/* Fill in the liobn and riobn fields on the adapter */
2188	static int read_dma_window(struct vio_dev *vdev,
2189	struct crq_server_adapter *adapter)
2190	{
2191	const __be32 *dma_window;
2192	const __be32 *prop;
2193
2194	/* TODO Using of_parse_dma_window would be better, but it doesn't give
2195	* a way to read multiple windows without already knowing the size of
2196	* a window or the number of windows
2197	*/
2198	dma_window =
2199	(const __be32 *)vio_get_attribute(vdev, "ibm,my-dma-window",
2200	NULL);
2201	if (!dma_window) {
2202	dev_warn(adapter->dev, "Couldn't find ibm,my-dma-window property\n");
2203	return -1;
2204	}
2205
2206	adapter->liobn = be32_to_cpu(*dma_window);
2207	dma_window++;
2208
2209	prop = (const __be32 *)vio_get_attribute(vdev, "ibm,#dma-address-cells",
2210	NULL);
2211	if (!prop) {
2212	dev_warn(adapter->dev, "Couldn't find ibm,#dma-address-cells property\n");
2213	dma_window++;
2214	} else {
2215	dma_window += be32_to_cpu(*prop);
2216	}
2217
2218	prop = (const __be32 *)vio_get_attribute(vdev, "ibm,#dma-size-cells",
2219	NULL);
2220	if (!prop) {
2221	dev_warn(adapter->dev, "Couldn't find ibm,#dma-size-cells property\n");
2222	dma_window++;
2223	} else {
2224	dma_window += be32_to_cpu(*prop);
2225	}
2226
2227	/* dma_window should point to the second window now */
2228	adapter->riobn = be32_to_cpu(*dma_window);
2229
2230	return 0;
2231	}
2232
2233	static int ibmvmc_probe(struct vio_dev *vdev, const struct vio_device_id *id)
2234	{
2235	struct crq_server_adapter *adapter = &ibmvmc_adapter;
2236	int rc;
2237
2238	dev_set_drvdata(dev: &vdev->dev, NULL);
2239	memset(adapter, 0, sizeof(*adapter));
2240	adapter->dev = &vdev->dev;
2241
2242	dev_info(adapter->dev, "Probe for UA 0x%x\n", vdev->unit_address);
2243
2244	rc = read_dma_window(vdev, adapter);
2245	if (rc != 0) {
2246	ibmvmc.state = ibmvmc_state_failed;
2247	return -1;
2248	}
2249
2250	dev_dbg(adapter->dev, "Probe: liobn 0x%x, riobn 0x%x\n",
2251	adapter->liobn, adapter->riobn);
2252
2253	init_waitqueue_head(&adapter->reset_wait_queue);
2254	adapter->reset_task = kthread_run(ibmvmc_reset_task, adapter, "ibmvmc");
2255	if (IS_ERR(ptr: adapter->reset_task)) {
2256	dev_err(adapter->dev, "Failed to start reset thread\n");
2257	ibmvmc.state = ibmvmc_state_failed;
2258	rc = PTR_ERR(ptr: adapter->reset_task);
2259	adapter->reset_task = NULL;
2260	return rc;
2261	}
2262
2263	rc = ibmvmc_init_crq_queue(adapter);
2264	if (rc != 0 && rc != H_RESOURCE) {
2265	dev_err(adapter->dev, "Error initializing CRQ. rc = 0x%x\n",
2266	rc);
2267	ibmvmc.state = ibmvmc_state_failed;
2268	goto crq_failed;
2269	}
2270
2271	ibmvmc.state = ibmvmc_state_crqinit;
2272
2273	/* Try to send an initialization message. Note that this is allowed
2274	* to fail if the other end is not acive. In that case we just wait
2275	* for the other side to initialize.
2276	*/
2277	if (ibmvmc_send_crq(adapter, word1: 0xC001000000000000LL, word2: 0) != 0 &&
2278	rc != H_RESOURCE)
2279	dev_warn(adapter->dev, "Failed to send initialize CRQ message\n");
2280
2281	dev_set_drvdata(dev: &vdev->dev, data: adapter);
2282
2283	return 0;
2284
2285	crq_failed:
2286	kthread_stop(k: adapter->reset_task);
2287	adapter->reset_task = NULL;
2288	return -EPERM;
2289	}
2290
2291	static void ibmvmc_remove(struct vio_dev *vdev)
2292	{
2293	struct crq_server_adapter *adapter = dev_get_drvdata(dev: &vdev->dev);
2294
2295	dev_info(adapter->dev, "Entering remove for UA 0x%x\n",
2296	vdev->unit_address);
2297	ibmvmc_release_crq_queue(adapter);
2298	}
2299
2300	static struct vio_device_id ibmvmc_device_table[] = {
2301	{ "ibm,vmc", "IBM,vmc" },
2302	{ "", "" }
2303	};
2304	MODULE_DEVICE_TABLE(vio, ibmvmc_device_table);
2305
2306	static struct vio_driver ibmvmc_driver = {
2307	.name = ibmvmc_driver_name,
2308	.id_table = ibmvmc_device_table,
2309	.probe = ibmvmc_probe,
2310	.remove = ibmvmc_remove,
2311	};
2312
2313	static void __init ibmvmc_scrub_module_parms(void)
2314	{
2315	if (ibmvmc_max_mtu > MAX_MTU) {
2316	pr_warn("ibmvmc: Max MTU reduced to %d\n", MAX_MTU);
2317	ibmvmc_max_mtu = MAX_MTU;
2318	} else if (ibmvmc_max_mtu < MIN_MTU) {
2319	pr_warn("ibmvmc: Max MTU increased to %d\n", MIN_MTU);
2320	ibmvmc_max_mtu = MIN_MTU;
2321	}
2322
2323	if (ibmvmc_max_buf_pool_size > MAX_BUF_POOL_SIZE) {
2324	pr_warn("ibmvmc: Max buffer pool size reduced to %d\n",
2325	MAX_BUF_POOL_SIZE);
2326	ibmvmc_max_buf_pool_size = MAX_BUF_POOL_SIZE;
2327	} else if (ibmvmc_max_buf_pool_size < MIN_BUF_POOL_SIZE) {
2328	pr_warn("ibmvmc: Max buffer pool size increased to %d\n",
2329	MIN_BUF_POOL_SIZE);
2330	ibmvmc_max_buf_pool_size = MIN_BUF_POOL_SIZE;
2331	}
2332
2333	if (ibmvmc_max_hmcs > MAX_HMCS) {
2334	pr_warn("ibmvmc: Max HMCs reduced to %d\n", MAX_HMCS);
2335	ibmvmc_max_hmcs = MAX_HMCS;
2336	} else if (ibmvmc_max_hmcs < MIN_HMCS) {
2337	pr_warn("ibmvmc: Max HMCs increased to %d\n", MIN_HMCS);
2338	ibmvmc_max_hmcs = MIN_HMCS;
2339	}
2340	}
2341
2342	static struct miscdevice ibmvmc_miscdev = {
2343	.name = ibmvmc_driver_name,
2344	.minor = MISC_DYNAMIC_MINOR,
2345	.fops = &ibmvmc_fops,
2346	};
2347
2348	static int __init ibmvmc_module_init(void)
2349	{
2350	int rc, i, j;
2351
2352	ibmvmc.state = ibmvmc_state_initial;
2353	pr_info("ibmvmc: version %s\n", IBMVMC_DRIVER_VERSION);
2354
2355	rc = misc_register(misc: &ibmvmc_miscdev);
2356	if (rc) {
2357	pr_err("ibmvmc: misc registration failed\n");
2358	goto misc_register_failed;
2359	}
2360	pr_info("ibmvmc: node %d:%d\n", MISC_MAJOR,
2361	ibmvmc_miscdev.minor);
2362
2363	/* Initialize data structures */
2364	memset(hmcs, 0, sizeof(struct ibmvmc_hmc) * MAX_HMCS);
2365	for (i = 0; i < MAX_HMCS; i++) {
2366	spin_lock_init(&hmcs[i].lock);
2367	hmcs[i].state = ibmhmc_state_free;
2368	for (j = 0; j < MAX_BUF_POOL_SIZE; j++)
2369	hmcs[i].queue_outbound_msgs[j] = VMC_INVALID_BUFFER_ID;
2370	}
2371
2372	/* Sanity check module parms */
2373	ibmvmc_scrub_module_parms();
2374
2375	/*
2376	* Initialize some reasonable values. Might be negotiated smaller
2377	* values during the capabilities exchange.
2378	*/
2379	ibmvmc.max_mtu = ibmvmc_max_mtu;
2380	ibmvmc.max_buffer_pool_size = ibmvmc_max_buf_pool_size;
2381	ibmvmc.max_hmc_index = ibmvmc_max_hmcs - 1;
2382
2383	rc = vio_register_driver(&ibmvmc_driver);
2384
2385	if (rc) {
2386	pr_err("ibmvmc: rc %d from vio_register_driver\n", rc);
2387	goto vio_reg_failed;
2388	}
2389
2390	return 0;
2391
2392	vio_reg_failed:
2393	misc_deregister(misc: &ibmvmc_miscdev);
2394	misc_register_failed:
2395	return rc;
2396	}
2397
2398	static void __exit ibmvmc_module_exit(void)
2399	{
2400	pr_info("ibmvmc: module exit\n");
2401	vio_unregister_driver(&ibmvmc_driver);
2402	misc_deregister(misc: &ibmvmc_miscdev);
2403	}
2404
2405	module_init(ibmvmc_module_init);
2406	module_exit(ibmvmc_module_exit);
2407
2408	module_param_named(buf_pool_size, ibmvmc_max_buf_pool_size,
2409	int, 0644);
2410	MODULE_PARM_DESC(buf_pool_size, "Buffer pool size");
2411	module_param_named(max_hmcs, ibmvmc_max_hmcs, int, 0644);
2412	MODULE_PARM_DESC(max_hmcs, "Max HMCs");
2413	module_param_named(max_mtu, ibmvmc_max_mtu, int, 0644);
2414	MODULE_PARM_DESC(max_mtu, "Max MTU");
2415
2416	MODULE_AUTHOR("Steven Royer <seroyer@linux.vnet.ibm.com>");
2417	MODULE_DESCRIPTION("IBM VMC");
2418	MODULE_VERSION(IBMVMC_DRIVER_VERSION);
2419	MODULE_LICENSE("GPL v2");
2420