1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* linux/drivers/misc/xillybus_core.c
4	*
5	* Copyright 2011 Xillybus Ltd, http://xillybus.com
6	*
7	* Driver for the Xillybus FPGA/host framework.
8	*
9	* This driver interfaces with a special IP core in an FPGA, setting up
10	* a pipe between a hardware FIFO in the programmable logic and a device
11	* file in the host. The number of such pipes and their attributes are
12	* set up on the logic. This driver detects these automatically and
13	* creates the device files accordingly.
14	*/
15
16	#include <linux/list.h>
17	#include <linux/device.h>
18	#include <linux/module.h>
19	#include <linux/io.h>
20	#include <linux/dma-mapping.h>
21	#include <linux/interrupt.h>
22	#include <linux/sched.h>
23	#include <linux/fs.h>
24	#include <linux/spinlock.h>
25	#include <linux/mutex.h>
26	#include <linux/crc32.h>
27	#include <linux/poll.h>
28	#include <linux/delay.h>
29	#include <linux/slab.h>
30	#include <linux/workqueue.h>
31	#include "xillybus.h"
32	#include "xillybus_class.h"
33
34	MODULE_DESCRIPTION("Xillybus core functions");
35	MODULE_AUTHOR("Eli Billauer, Xillybus Ltd.");
36	MODULE_ALIAS("xillybus_core");
37	MODULE_LICENSE("GPL v2");
38
39	/* General timeout is 100 ms, rx timeout is 10 ms */
40	#define XILLY_RX_TIMEOUT (10*HZ/1000)
41	#define XILLY_TIMEOUT (100*HZ/1000)
42
43	#define fpga_msg_ctrl_reg 0x0008
44	#define fpga_dma_control_reg 0x0020
45	#define fpga_dma_bufno_reg 0x0024
46	#define fpga_dma_bufaddr_lowaddr_reg 0x0028
47	#define fpga_dma_bufaddr_highaddr_reg 0x002c
48	#define fpga_buf_ctrl_reg 0x0030
49	#define fpga_buf_offset_reg 0x0034
50	#define fpga_endian_reg 0x0040
51
52	#define XILLYMSG_OPCODE_RELEASEBUF 1
53	#define XILLYMSG_OPCODE_QUIESCEACK 2
54	#define XILLYMSG_OPCODE_FIFOEOF 3
55	#define XILLYMSG_OPCODE_FATAL_ERROR 4
56	#define XILLYMSG_OPCODE_NONEMPTY 5
57
58	static const char xillyname[] = "xillybus";
59
60	static struct workqueue_struct *xillybus_wq;
61
62	/*
63	* Locking scheme: Mutexes protect invocations of character device methods.
64	* If both locks are taken, wr_mutex is taken first, rd_mutex second.
65	*
66	* wr_spinlock protects wr_*_buf_idx, wr_empty, wr_sleepy, wr_ready and the
67	* buffers' end_offset fields against changes made by IRQ handler (and in
68	* theory, other file request handlers, but the mutex handles that). Nothing
69	* else.
70	* They are held for short direct memory manipulations. Needless to say,
71	* no mutex locking is allowed when a spinlock is held.
72	*
73	* rd_spinlock does the same with rd_*_buf_idx, rd_empty and end_offset.
74	*
75	* register_mutex is endpoint-specific, and is held when non-atomic
76	* register operations are performed. wr_mutex and rd_mutex may be
77	* held when register_mutex is taken, but none of the spinlocks. Note that
78	* register_mutex doesn't protect against sporadic buf_ctrl_reg writes
79	* which are unrelated to buf_offset_reg, since they are harmless.
80	*
81	* Blocking on the wait queues is allowed with mutexes held, but not with
82	* spinlocks.
83	*
84	* Only interruptible blocking is allowed on mutexes and wait queues.
85	*
86	* All in all, the locking order goes (with skips allowed, of course):
87	* wr_mutex -> rd_mutex -> register_mutex -> wr_spinlock -> rd_spinlock
88	*/
89
90	static void malformed_message(struct xilly_endpoint *endpoint, u32 *buf)
91	{
92	int opcode;
93	int msg_channel, msg_bufno, msg_data, msg_dir;
94
95	opcode = (buf[0] >> 24) & 0xff;
96	msg_dir = buf[0] & 1;
97	msg_channel = (buf[0] >> 1) & 0x7ff;
98	msg_bufno = (buf[0] >> 12) & 0x3ff;
99	msg_data = buf[1] & 0xfffffff;
100
101	dev_warn(endpoint->dev,
102	"Malformed message (skipping): opcode=%d, channel=%03x, dir=%d, bufno=%03x, data=%07x\n",
103	opcode, msg_channel, msg_dir, msg_bufno, msg_data);
104	}
105
106	/*
107	* xillybus_isr assumes the interrupt is allocated exclusively to it,
108	* which is the natural case MSI and several other hardware-oriented
109	* interrupts. Sharing is not allowed.
110	*/
111
112	irqreturn_t xillybus_isr(int irq, void *data)
113	{
114	struct xilly_endpoint *ep = data;
115	u32 *buf;
116	unsigned int buf_size;
117	int i;
118	int opcode;
119	unsigned int msg_channel, msg_bufno, msg_data, msg_dir;
120	struct xilly_channel *channel;
121
122	buf = ep->msgbuf_addr;
123	buf_size = ep->msg_buf_size/sizeof(u32);
124
125	dma_sync_single_for_cpu(dev: ep->dev, addr: ep->msgbuf_dma_addr,
126	size: ep->msg_buf_size, dir: DMA_FROM_DEVICE);
127
128	for (i = 0; i < buf_size; i += 2) {
129	if (((buf[i+1] >> 28) & 0xf) != ep->msg_counter) {
130	malformed_message(endpoint: ep, buf: &buf[i]);
131	dev_warn(ep->dev,
132	"Sending a NACK on counter %x (instead of %x) on entry %d\n",
133	((buf[i+1] >> 28) & 0xf),
134	ep->msg_counter,
135	i/2);
136
137	if (++ep->failed_messages > 10) {
138	dev_err(ep->dev,
139	"Lost sync with interrupt messages. Stopping.\n");
140	} else {
141	dma_sync_single_for_device(dev: ep->dev,
142	addr: ep->msgbuf_dma_addr,
143	size: ep->msg_buf_size,
144	dir: DMA_FROM_DEVICE);
145
146	iowrite32(0x01, /* Message NACK */
147	ep->registers + fpga_msg_ctrl_reg);
148	}
149	return IRQ_HANDLED;
150	} else if (buf[i] & (1 << 22)) /* Last message */
151	break;
152	}
153
154	if (i >= buf_size) {
155	dev_err(ep->dev, "Bad interrupt message. Stopping.\n");
156	return IRQ_HANDLED;
157	}
158
159	buf_size = i + 2;
160
161	for (i = 0; i < buf_size; i += 2) { /* Scan through messages */
162	opcode = (buf[i] >> 24) & 0xff;
163
164	msg_dir = buf[i] & 1;
165	msg_channel = (buf[i] >> 1) & 0x7ff;
166	msg_bufno = (buf[i] >> 12) & 0x3ff;
167	msg_data = buf[i+1] & 0xfffffff;
168
169	switch (opcode) {
170	case XILLYMSG_OPCODE_RELEASEBUF:
171	if ((msg_channel > ep->num_channels) ||
172	(msg_channel == 0)) {
173	malformed_message(endpoint: ep, buf: &buf[i]);
174	break;
175	}
176
177	channel = ep->channels[msg_channel];
178
179	if (msg_dir) { /* Write channel */
180	if (msg_bufno >= channel->num_wr_buffers) {
181	malformed_message(endpoint: ep, buf: &buf[i]);
182	break;
183	}
184	spin_lock(lock: &channel->wr_spinlock);
185	channel->wr_buffers[msg_bufno]->end_offset =
186	msg_data;
187	channel->wr_fpga_buf_idx = msg_bufno;
188	channel->wr_empty = 0;
189	channel->wr_sleepy = 0;
190	spin_unlock(lock: &channel->wr_spinlock);
191
192	wake_up_interruptible(&channel->wr_wait);
193
194	} else {
195	/* Read channel */
196
197	if (msg_bufno >= channel->num_rd_buffers) {
198	malformed_message(endpoint: ep, buf: &buf[i]);
199	break;
200	}
201
202	spin_lock(lock: &channel->rd_spinlock);
203	channel->rd_fpga_buf_idx = msg_bufno;
204	channel->rd_full = 0;
205	spin_unlock(lock: &channel->rd_spinlock);
206
207	wake_up_interruptible(&channel->rd_wait);
208	if (!channel->rd_synchronous)
209	queue_delayed_work(
210	wq: xillybus_wq,
211	dwork: &channel->rd_workitem,
212	XILLY_RX_TIMEOUT);
213	}
214
215	break;
216	case XILLYMSG_OPCODE_NONEMPTY:
217	if ((msg_channel > ep->num_channels) ||
218	(msg_channel == 0) || (!msg_dir) ||
219	!ep->channels[msg_channel]->wr_supports_nonempty) {
220	malformed_message(endpoint: ep, buf: &buf[i]);
221	break;
222	}
223
224	channel = ep->channels[msg_channel];
225
226	if (msg_bufno >= channel->num_wr_buffers) {
227	malformed_message(endpoint: ep, buf: &buf[i]);
228	break;
229	}
230	spin_lock(lock: &channel->wr_spinlock);
231	if (msg_bufno == channel->wr_host_buf_idx)
232	channel->wr_ready = 1;
233	spin_unlock(lock: &channel->wr_spinlock);
234
235	wake_up_interruptible(&channel->wr_ready_wait);
236
237	break;
238	case XILLYMSG_OPCODE_QUIESCEACK:
239	ep->idtlen = msg_data;
240	wake_up_interruptible(&ep->ep_wait);
241
242	break;
243	case XILLYMSG_OPCODE_FIFOEOF:
244	if ((msg_channel > ep->num_channels) ||
245	(msg_channel == 0) || (!msg_dir) ||
246	!ep->channels[msg_channel]->num_wr_buffers) {
247	malformed_message(endpoint: ep, buf: &buf[i]);
248	break;
249	}
250	channel = ep->channels[msg_channel];
251	spin_lock(lock: &channel->wr_spinlock);
252	channel->wr_eof = msg_bufno;
253	channel->wr_sleepy = 0;
254
255	channel->wr_hangup = channel->wr_empty &&
256	(channel->wr_host_buf_idx == msg_bufno);
257
258	spin_unlock(lock: &channel->wr_spinlock);
259
260	wake_up_interruptible(&channel->wr_wait);
261
262	break;
263	case XILLYMSG_OPCODE_FATAL_ERROR:
264	ep->fatal_error = 1;
265	wake_up_interruptible(&ep->ep_wait); /* For select() */
266	dev_err(ep->dev,
267	"FPGA reported a fatal error. This means that the low-level communication with the device has failed. This hardware problem is most likely unrelated to Xillybus (neither kernel module nor FPGA core), but reports are still welcome. All I/O is aborted.\n");
268	break;
269	default:
270	malformed_message(endpoint: ep, buf: &buf[i]);
271	break;
272	}
273	}
274
275	dma_sync_single_for_device(dev: ep->dev, addr: ep->msgbuf_dma_addr,
276	size: ep->msg_buf_size, dir: DMA_FROM_DEVICE);
277
278	ep->msg_counter = (ep->msg_counter + 1) & 0xf;
279	ep->failed_messages = 0;
280	iowrite32(0x03, ep->registers + fpga_msg_ctrl_reg); /* Message ACK */
281
282	return IRQ_HANDLED;
283	}
284	EXPORT_SYMBOL(xillybus_isr);
285
286	/*
287	* A few trivial memory management functions.
288	* NOTE: These functions are used only on probe and remove, and therefore
289	* no locks are applied!
290	*/
291
292	static void xillybus_autoflush(struct work_struct *work);
293
294	struct xilly_alloc_state {
295	void *salami;
296	int left_of_salami;
297	int nbuffer;
298	enum dma_data_direction direction;
299	u32 regdirection;
300	};
301
302	static void xilly_unmap(void *ptr)
303	{
304	struct xilly_mapping *data = ptr;
305
306	dma_unmap_single(data->device, data->dma_addr,
307	data->size, data->direction);
308
309	kfree(objp: ptr);
310	}
311
312	static int xilly_map_single(struct xilly_endpoint *ep,
313	void *ptr,
314	size_t size,
315	int direction,
316	dma_addr_t *ret_dma_handle
317	)
318	{
319	dma_addr_t addr;
320	struct xilly_mapping *this;
321
322	this = kzalloc(size: sizeof(*this), GFP_KERNEL);
323	if (!this)
324	return -ENOMEM;
325
326	addr = dma_map_single(ep->dev, ptr, size, direction);
327
328	if (dma_mapping_error(dev: ep->dev, dma_addr: addr)) {
329	kfree(objp: this);
330	return -ENODEV;
331	}
332
333	this->device = ep->dev;
334	this->dma_addr = addr;
335	this->size = size;
336	this->direction = direction;
337
338	*ret_dma_handle = addr;
339
340	return devm_add_action_or_reset(ep->dev, xilly_unmap, this);
341	}
342
343	static int xilly_get_dma_buffers(struct xilly_endpoint *ep,
344	struct xilly_alloc_state *s,
345	struct xilly_buffer **buffers,
346	int bufnum, int bytebufsize)
347	{
348	int i, rc;
349	dma_addr_t dma_addr;
350	struct device *dev = ep->dev;
351	struct xilly_buffer *this_buffer = NULL; /* Init to silence warning */
352
353	if (buffers) { /* Not the message buffer */
354	this_buffer = devm_kcalloc(dev, n: bufnum,
355	size: sizeof(struct xilly_buffer),
356	GFP_KERNEL);
357	if (!this_buffer)
358	return -ENOMEM;
359	}
360
361	for (i = 0; i < bufnum; i++) {
362	/*
363	* Buffers are expected in descending size order, so there
364	* is either enough space for this buffer or none at all.
365	*/
366
367	if ((s->left_of_salami < bytebufsize) &&
368	(s->left_of_salami > 0)) {
369	dev_err(ep->dev,
370	"Corrupt buffer allocation in IDT. Aborting.\n");
371	return -ENODEV;
372	}
373
374	if (s->left_of_salami == 0) {
375	int allocorder, allocsize;
376
377	allocsize = PAGE_SIZE;
378	allocorder = 0;
379	while (bytebufsize > allocsize) {
380	allocsize *= 2;
381	allocorder++;
382	}
383
384	s->salami = (void *) devm_get_free_pages(
385	dev,
386	GFP_KERNEL | __GFP_DMA32 | __GFP_ZERO,
387	order: allocorder);
388	if (!s->salami)
389	return -ENOMEM;
390
391	s->left_of_salami = allocsize;
392	}
393
394	rc = xilly_map_single(ep, ptr: s->salami,
395	size: bytebufsize, direction: s->direction,
396	ret_dma_handle: &dma_addr);
397	if (rc)
398	return rc;
399
400	iowrite32((u32) (dma_addr & 0xffffffff),
401	ep->registers + fpga_dma_bufaddr_lowaddr_reg);
402	iowrite32(((u32) ((((u64) dma_addr) >> 32) & 0xffffffff)),
403	ep->registers + fpga_dma_bufaddr_highaddr_reg);
404
405	if (buffers) { /* Not the message buffer */
406	this_buffer->addr = s->salami;
407	this_buffer->dma_addr = dma_addr;
408	buffers[i] = this_buffer++;
409
410	iowrite32(s->regdirection | s->nbuffer++,
411	ep->registers + fpga_dma_bufno_reg);
412	} else {
413	ep->msgbuf_addr = s->salami;
414	ep->msgbuf_dma_addr = dma_addr;
415	ep->msg_buf_size = bytebufsize;
416
417	iowrite32(s->regdirection,
418	ep->registers + fpga_dma_bufno_reg);
419	}
420
421	s->left_of_salami -= bytebufsize;
422	s->salami += bytebufsize;
423	}
424	return 0;
425	}
426
427	static int xilly_setupchannels(struct xilly_endpoint *ep,
428	unsigned char *chandesc,
429	int entries)
430	{
431	struct device *dev = ep->dev;
432	int i, entry, rc;
433	struct xilly_channel *channel;
434	int channelnum, bufnum, bufsize, format, is_writebuf;
435	int bytebufsize;
436	int synchronous, allowpartial, exclusive_open, seekable;
437	int supports_nonempty;
438	int msg_buf_done = 0;
439
440	struct xilly_alloc_state rd_alloc = {
441	.salami = NULL,
442	.left_of_salami = 0,
443	.nbuffer = 1,
444	.direction = DMA_TO_DEVICE,
445	.regdirection = 0,
446	};
447
448	struct xilly_alloc_state wr_alloc = {
449	.salami = NULL,
450	.left_of_salami = 0,
451	.nbuffer = 1,
452	.direction = DMA_FROM_DEVICE,
453	.regdirection = 0x80000000,
454	};
455
456	channel = devm_kcalloc(dev, n: ep->num_channels,
457	size: sizeof(struct xilly_channel), GFP_KERNEL);
458	if (!channel)
459	return -ENOMEM;
460
461	ep->channels = devm_kcalloc(dev, n: ep->num_channels + 1,
462	size: sizeof(struct xilly_channel *),
463	GFP_KERNEL);
464	if (!ep->channels)
465	return -ENOMEM;
466
467	ep->channels[0] = NULL; /* Channel 0 is message buf. */
468
469	/* Initialize all channels with defaults */
470
471	for (i = 1; i <= ep->num_channels; i++) {
472	channel->wr_buffers = NULL;
473	channel->rd_buffers = NULL;
474	channel->num_wr_buffers = 0;
475	channel->num_rd_buffers = 0;
476	channel->wr_fpga_buf_idx = -1;
477	channel->wr_host_buf_idx = 0;
478	channel->wr_host_buf_pos = 0;
479	channel->wr_empty = 1;
480	channel->wr_ready = 0;
481	channel->wr_sleepy = 1;
482	channel->rd_fpga_buf_idx = 0;
483	channel->rd_host_buf_idx = 0;
484	channel->rd_host_buf_pos = 0;
485	channel->rd_full = 0;
486	channel->wr_ref_count = 0;
487	channel->rd_ref_count = 0;
488
489	spin_lock_init(&channel->wr_spinlock);
490	spin_lock_init(&channel->rd_spinlock);
491	mutex_init(&channel->wr_mutex);
492	mutex_init(&channel->rd_mutex);
493	init_waitqueue_head(&channel->rd_wait);
494	init_waitqueue_head(&channel->wr_wait);
495	init_waitqueue_head(&channel->wr_ready_wait);
496
497	INIT_DELAYED_WORK(&channel->rd_workitem, xillybus_autoflush);
498
499	channel->endpoint = ep;
500	channel->chan_num = i;
501
502	channel->log2_element_size = 0;
503
504	ep->channels[i] = channel++;
505	}
506
507	for (entry = 0; entry < entries; entry++, chandesc += 4) {
508	struct xilly_buffer **buffers = NULL;
509
510	is_writebuf = chandesc[0] & 0x01;
511	channelnum = (chandesc[0] >> 1) | ((chandesc[1] & 0x0f) << 7);
512	format = (chandesc[1] >> 4) & 0x03;
513	allowpartial = (chandesc[1] >> 6) & 0x01;
514	synchronous = (chandesc[1] >> 7) & 0x01;
515	bufsize = 1 << (chandesc[2] & 0x1f);
516	bufnum = 1 << (chandesc[3] & 0x0f);
517	exclusive_open = (chandesc[2] >> 7) & 0x01;
518	seekable = (chandesc[2] >> 6) & 0x01;
519	supports_nonempty = (chandesc[2] >> 5) & 0x01;
520
521	if ((channelnum > ep->num_channels) ||
522	((channelnum == 0) && !is_writebuf)) {
523	dev_err(ep->dev,
524	"IDT requests channel out of range. Aborting.\n");
525	return -ENODEV;
526	}
527
528	channel = ep->channels[channelnum]; /* NULL for msg channel */
529
530	if (!is_writebuf || channelnum > 0) {
531	channel->log2_element_size = ((format > 2) ?
532	2 : format);
533
534	bytebufsize = bufsize *
535	(1 << channel->log2_element_size);
536
537	buffers = devm_kcalloc(dev, n: bufnum,
538	size: sizeof(struct xilly_buffer *),
539	GFP_KERNEL);
540	if (!buffers)
541	return -ENOMEM;
542	} else {
543	bytebufsize = bufsize << 2;
544	}
545
546	if (!is_writebuf) {
547	channel->num_rd_buffers = bufnum;
548	channel->rd_buf_size = bytebufsize;
549	channel->rd_allow_partial = allowpartial;
550	channel->rd_synchronous = synchronous;
551	channel->rd_exclusive_open = exclusive_open;
552	channel->seekable = seekable;
553
554	channel->rd_buffers = buffers;
555	rc = xilly_get_dma_buffers(ep, s: &rd_alloc, buffers,
556	bufnum, bytebufsize);
557	} else if (channelnum > 0) {
558	channel->num_wr_buffers = bufnum;
559	channel->wr_buf_size = bytebufsize;
560
561	channel->seekable = seekable;
562	channel->wr_supports_nonempty = supports_nonempty;
563
564	channel->wr_allow_partial = allowpartial;
565	channel->wr_synchronous = synchronous;
566	channel->wr_exclusive_open = exclusive_open;
567
568	channel->wr_buffers = buffers;
569	rc = xilly_get_dma_buffers(ep, s: &wr_alloc, buffers,
570	bufnum, bytebufsize);
571	} else {
572	rc = xilly_get_dma_buffers(ep, s: &wr_alloc, NULL,
573	bufnum, bytebufsize);
574	msg_buf_done++;
575	}
576
577	if (rc)
578	return -ENOMEM;
579	}
580
581	if (!msg_buf_done) {
582	dev_err(ep->dev,
583	"Corrupt IDT: No message buffer. Aborting.\n");
584	return -ENODEV;
585	}
586	return 0;
587	}
588
589	static int xilly_scan_idt(struct xilly_endpoint *endpoint,
590	struct xilly_idt_handle *idt_handle)
591	{
592	int count = 0;
593	unsigned char *idt = endpoint->channels[1]->wr_buffers[0]->addr;
594	unsigned char *end_of_idt = idt + endpoint->idtlen - 4;
595	unsigned char *scan;
596	int len;
597
598	scan = idt + 1;
599	idt_handle->names = scan;
600
601	while ((scan <= end_of_idt) && *scan) {
602	while ((scan <= end_of_idt) && *scan++)
603	/* Do nothing, just scan thru string */;
604	count++;
605	}
606
607	idt_handle->names_len = scan - idt_handle->names;
608
609	scan++;
610
611	if (scan > end_of_idt) {
612	dev_err(endpoint->dev,
613	"IDT device name list overflow. Aborting.\n");
614	return -ENODEV;
615	}
616	idt_handle->chandesc = scan;
617
618	len = endpoint->idtlen - (3 + ((int) (scan - idt)));
619
620	if (len & 0x03) {
621	dev_err(endpoint->dev,
622	"Corrupt IDT device name list. Aborting.\n");
623	return -ENODEV;
624	}
625
626	idt_handle->entries = len >> 2;
627	endpoint->num_channels = count;
628
629	return 0;
630	}
631
632	static int xilly_obtain_idt(struct xilly_endpoint *endpoint)
633	{
634	struct xilly_channel *channel;
635	unsigned char *version;
636	long t;
637
638	channel = endpoint->channels[1]; /* This should be generated ad-hoc */
639
640	channel->wr_sleepy = 1;
641
642	iowrite32(1 |
643	(3 << 24), /* Opcode 3 for channel 0 = Send IDT */
644	endpoint->registers + fpga_buf_ctrl_reg);
645
646	t = wait_event_interruptible_timeout(channel->wr_wait,
647	(!channel->wr_sleepy),
648	XILLY_TIMEOUT);
649
650	if (t <= 0) {
651	dev_err(endpoint->dev, "Failed to obtain IDT. Aborting.\n");
652
653	if (endpoint->fatal_error)
654	return -EIO;
655
656	return -ENODEV;
657	}
658
659	dma_sync_single_for_cpu(dev: channel->endpoint->dev,
660	addr: channel->wr_buffers[0]->dma_addr,
661	size: channel->wr_buf_size,
662	dir: DMA_FROM_DEVICE);
663
664	if (channel->wr_buffers[0]->end_offset != endpoint->idtlen) {
665	dev_err(endpoint->dev,
666	"IDT length mismatch (%d != %d). Aborting.\n",
667	channel->wr_buffers[0]->end_offset, endpoint->idtlen);
668	return -ENODEV;
669	}
670
671	if (crc32_le(crc: ~0, p: channel->wr_buffers[0]->addr,
672	len: endpoint->idtlen+1) != 0) {
673	dev_err(endpoint->dev, "IDT failed CRC check. Aborting.\n");
674	return -ENODEV;
675	}
676
677	version = channel->wr_buffers[0]->addr;
678
679	/* Check version number. Reject anything above 0x82. */
680	if (*version > 0x82) {
681	dev_err(endpoint->dev,
682	"No support for IDT version 0x%02x. Maybe the xillybus driver needs an upgrade. Aborting.\n",
683	*version);
684	return -ENODEV;
685	}
686
687	return 0;
688	}
689
690	static ssize_t xillybus_read(struct file *filp, char __user *userbuf,
691	size_t count, loff_t *f_pos)
692	{
693	ssize_t rc;
694	unsigned long flags;
695	int bytes_done = 0;
696	int no_time_left = 0;
697	long deadline, left_to_sleep;
698	struct xilly_channel *channel = filp->private_data;
699
700	int empty, reached_eof, exhausted, ready;
701	/* Initializations are there only to silence warnings */
702
703	int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0;
704	int waiting_bufidx;
705
706	if (channel->endpoint->fatal_error)
707	return -EIO;
708
709	deadline = jiffies + 1 + XILLY_RX_TIMEOUT;
710
711	rc = mutex_lock_interruptible(&channel->wr_mutex);
712	if (rc)
713	return rc;
714
715	while (1) { /* Note that we may drop mutex within this loop */
716	int bytes_to_do = count - bytes_done;
717
718	spin_lock_irqsave(&channel->wr_spinlock, flags);
719
720	empty = channel->wr_empty;
721	ready = !empty || channel->wr_ready;
722
723	if (!empty) {
724	bufidx = channel->wr_host_buf_idx;
725	bufpos = channel->wr_host_buf_pos;
726	howmany = ((channel->wr_buffers[bufidx]->end_offset
727	+ 1) << channel->log2_element_size)
728	- bufpos;
729
730	/* Update wr_host_* to its post-operation state */
731	if (howmany > bytes_to_do) {
732	bufferdone = 0;
733
734	howmany = bytes_to_do;
735	channel->wr_host_buf_pos += howmany;
736	} else {
737	bufferdone = 1;
738
739	channel->wr_host_buf_pos = 0;
740
741	if (bufidx == channel->wr_fpga_buf_idx) {
742	channel->wr_empty = 1;
743	channel->wr_sleepy = 1;
744	channel->wr_ready = 0;
745	}
746
747	if (bufidx >= (channel->num_wr_buffers - 1))
748	channel->wr_host_buf_idx = 0;
749	else
750	channel->wr_host_buf_idx++;
751	}
752	}
753
754	/*
755	* Marking our situation after the possible changes above,
756	* for use after releasing the spinlock.
757	*
758	* empty = empty before change
759	* exhasted = empty after possible change
760	*/
761
762	reached_eof = channel->wr_empty &&
763	(channel->wr_host_buf_idx == channel->wr_eof);
764	channel->wr_hangup = reached_eof;
765	exhausted = channel->wr_empty;
766	waiting_bufidx = channel->wr_host_buf_idx;
767
768	spin_unlock_irqrestore(lock: &channel->wr_spinlock, flags);
769
770	if (!empty) { /* Go on, now without the spinlock */
771
772	if (bufpos == 0) /* Position zero means it's virgin */
773	dma_sync_single_for_cpu(dev: channel->endpoint->dev,
774	addr: channel->wr_buffers[bufidx]->dma_addr,
775	size: channel->wr_buf_size,
776	dir: DMA_FROM_DEVICE);
777
778	if (copy_to_user(
779	to: userbuf,
780	from: channel->wr_buffers[bufidx]->addr
781	+ bufpos, n: howmany))
782	rc = -EFAULT;
783
784	userbuf += howmany;
785	bytes_done += howmany;
786
787	if (bufferdone) {
788	dma_sync_single_for_device(dev: channel->endpoint->dev,
789	addr: channel->wr_buffers[bufidx]->dma_addr,
790	size: channel->wr_buf_size,
791	dir: DMA_FROM_DEVICE);
792
793	/*
794	* Tell FPGA the buffer is done with. It's an
795	* atomic operation to the FPGA, so what
796	* happens with other channels doesn't matter,
797	* and the certain channel is protected with
798	* the channel-specific mutex.
799	*/
800
801	iowrite32(1 | (channel->chan_num << 1) |
802	(bufidx << 12),
803	channel->endpoint->registers +
804	fpga_buf_ctrl_reg);
805	}
806
807	if (rc) {
808	mutex_unlock(lock: &channel->wr_mutex);
809	return rc;
810	}
811	}
812
813	/* This includes a zero-count return = EOF */
814	if ((bytes_done >= count) || reached_eof)
815	break;
816
817	if (!exhausted)
818	continue; /* More in RAM buffer(s)? Just go on. */
819
820	if ((bytes_done > 0) &&
821	(no_time_left ||
822	(channel->wr_synchronous && channel->wr_allow_partial)))
823	break;
824
825	/*
826	* Nonblocking read: The "ready" flag tells us that the FPGA
827	* has data to send. In non-blocking mode, if it isn't on,
828	* just return. But if there is, we jump directly to the point
829	* where we ask for the FPGA to send all it has, and wait
830	* until that data arrives. So in a sense, we *do* block in
831	* nonblocking mode, but only for a very short time.
832	*/
833
834	if (!no_time_left && (filp->f_flags & O_NONBLOCK)) {
835	if (bytes_done > 0)
836	break;
837
838	if (ready)
839	goto desperate;
840
841	rc = -EAGAIN;
842	break;
843	}
844
845	if (!no_time_left || (bytes_done > 0)) {
846	/*
847	* Note that in case of an element-misaligned read
848	* request, offsetlimit will include the last element,
849	* which will be partially read from.
850	*/
851	int offsetlimit = ((count - bytes_done) - 1) >>
852	channel->log2_element_size;
853	int buf_elements = channel->wr_buf_size >>
854	channel->log2_element_size;
855
856	/*
857	* In synchronous mode, always send an offset limit.
858	* Just don't send a value too big.
859	*/
860
861	if (channel->wr_synchronous) {
862	/* Don't request more than one buffer */
863	if (channel->wr_allow_partial &&
864	(offsetlimit >= buf_elements))
865	offsetlimit = buf_elements - 1;
866
867	/* Don't request more than all buffers */
868	if (!channel->wr_allow_partial &&
869	(offsetlimit >=
870	(buf_elements * channel->num_wr_buffers)))
871	offsetlimit = buf_elements *
872	channel->num_wr_buffers - 1;
873	}
874
875	/*
876	* In asynchronous mode, force early flush of a buffer
877	* only if that will allow returning a full count. The
878	* "offsetlimit < ( ... )" rather than "<=" excludes
879	* requesting a full buffer, which would obviously
880	* cause a buffer transmission anyhow
881	*/
882
883	if (channel->wr_synchronous ||
884	(offsetlimit < (buf_elements - 1))) {
885	mutex_lock(&channel->endpoint->register_mutex);
886
887	iowrite32(offsetlimit,
888	channel->endpoint->registers +
889	fpga_buf_offset_reg);
890
891	iowrite32(1 | (channel->chan_num << 1) |
892	(2 << 24) | /* 2 = offset limit */
893	(waiting_bufidx << 12),
894	channel->endpoint->registers +
895	fpga_buf_ctrl_reg);
896
897	mutex_unlock(lock: &channel->endpoint->
898	register_mutex);
899	}
900	}
901
902	/*
903	* If partial completion is disallowed, there is no point in
904	* timeout sleeping. Neither if no_time_left is set and
905	* there's no data.
906	*/
907
908	if (!channel->wr_allow_partial ||
909	(no_time_left && (bytes_done == 0))) {
910	/*
911	* This do-loop will run more than once if another
912	* thread reasserted wr_sleepy before we got the mutex
913	* back, so we try again.
914	*/
915
916	do {
917	mutex_unlock(lock: &channel->wr_mutex);
918
919	if (wait_event_interruptible(
920	channel->wr_wait,
921	(!channel->wr_sleepy)))
922	goto interrupted;
923
924	if (mutex_lock_interruptible(
925	&channel->wr_mutex))
926	goto interrupted;
927	} while (channel->wr_sleepy);
928
929	continue;
930
931	interrupted: /* Mutex is not held if got here */
932	if (channel->endpoint->fatal_error)
933	return -EIO;
934	if (bytes_done)
935	return bytes_done;
936	if (filp->f_flags & O_NONBLOCK)
937	return -EAGAIN; /* Don't admit snoozing */
938	return -EINTR;
939	}
940
941	left_to_sleep = deadline - ((long) jiffies);
942
943	/*
944	* If our time is out, skip the waiting. We may miss wr_sleepy
945	* being deasserted but hey, almost missing the train is like
946	* missing it.
947	*/
948
949	if (left_to_sleep > 0) {
950	left_to_sleep =
951	wait_event_interruptible_timeout(
952	channel->wr_wait,
953	(!channel->wr_sleepy),
954	left_to_sleep);
955
956	if (left_to_sleep > 0) /* wr_sleepy deasserted */
957	continue;
958
959	if (left_to_sleep < 0) { /* Interrupt */
960	mutex_unlock(lock: &channel->wr_mutex);
961	if (channel->endpoint->fatal_error)
962	return -EIO;
963	if (bytes_done)
964	return bytes_done;
965	return -EINTR;
966	}
967	}
968
969	desperate:
970	no_time_left = 1; /* We're out of sleeping time. Desperate! */
971
972	if (bytes_done == 0) {
973	/*
974	* Reaching here means that we allow partial return,
975	* that we've run out of time, and that we have
976	* nothing to return.
977	* So tell the FPGA to send anything it has or gets.
978	*/
979
980	iowrite32(1 | (channel->chan_num << 1) |
981	(3 << 24) | /* Opcode 3, flush it all! */
982	(waiting_bufidx << 12),
983	channel->endpoint->registers +
984	fpga_buf_ctrl_reg);
985	}
986
987	/*
988	* Reaching here means that we *do* have data in the buffer,
989	* but the "partial" flag disallows returning less than
990	* required. And we don't have as much. So loop again,
991	* which is likely to end up blocking indefinitely until
992	* enough data has arrived.
993	*/
994	}
995
996	mutex_unlock(lock: &channel->wr_mutex);
997
998	if (channel->endpoint->fatal_error)
999	return -EIO;
1000
1001	if (rc)
1002	return rc;
1003
1004	return bytes_done;
1005	}
1006
1007	/*
1008	* The timeout argument takes values as follows:
1009	* >0 : Flush with timeout
1010	* ==0 : Flush, and wait idefinitely for the flush to complete
1011	* <0 : Autoflush: Flush only if there's a single buffer occupied
1012	*/
1013
1014	static int xillybus_myflush(struct xilly_channel *channel, long timeout)
1015	{
1016	int rc;
1017	unsigned long flags;
1018
1019	int end_offset_plus1;
1020	int bufidx, bufidx_minus1;
1021	int i;
1022	int empty;
1023	int new_rd_host_buf_pos;
1024
1025	if (channel->endpoint->fatal_error)
1026	return -EIO;
1027	rc = mutex_lock_interruptible(&channel->rd_mutex);
1028	if (rc)
1029	return rc;
1030
1031	/*
1032	* Don't flush a closed channel. This can happen when the work queued
1033	* autoflush thread fires off after the file has closed. This is not
1034	* an error, just something to dismiss.
1035	*/
1036
1037	if (!channel->rd_ref_count)
1038	goto done;
1039
1040	bufidx = channel->rd_host_buf_idx;
1041
1042	bufidx_minus1 = (bufidx == 0) ?
1043	channel->num_rd_buffers - 1 :
1044	bufidx - 1;
1045
1046	end_offset_plus1 = channel->rd_host_buf_pos >>
1047	channel->log2_element_size;
1048
1049	new_rd_host_buf_pos = channel->rd_host_buf_pos -
1050	(end_offset_plus1 << channel->log2_element_size);
1051
1052	/* Submit the current buffer if it's nonempty */
1053	if (end_offset_plus1) {
1054	unsigned char *tail = channel->rd_buffers[bufidx]->addr +
1055	(end_offset_plus1 << channel->log2_element_size);
1056
1057	/* Copy unflushed data, so we can put it in next buffer */
1058	for (i = 0; i < new_rd_host_buf_pos; i++)
1059	channel->rd_leftovers[i] = *tail++;
1060
1061	spin_lock_irqsave(&channel->rd_spinlock, flags);
1062
1063	/* Autoflush only if a single buffer is occupied */
1064
1065	if ((timeout < 0) &&
1066	(channel->rd_full ||
1067	(bufidx_minus1 != channel->rd_fpga_buf_idx))) {
1068	spin_unlock_irqrestore(lock: &channel->rd_spinlock, flags);
1069	/*
1070	* A new work item may be queued by the ISR exactly
1071	* now, since the execution of a work item allows the
1072	* queuing of a new one while it's running.
1073	*/
1074	goto done;
1075	}
1076
1077	/* The 4th element is never needed for data, so it's a flag */
1078	channel->rd_leftovers[3] = (new_rd_host_buf_pos != 0);
1079
1080	/* Set up rd_full to reflect a certain moment's state */
1081
1082	if (bufidx == channel->rd_fpga_buf_idx)
1083	channel->rd_full = 1;
1084	spin_unlock_irqrestore(lock: &channel->rd_spinlock, flags);
1085
1086	if (bufidx >= (channel->num_rd_buffers - 1))
1087	channel->rd_host_buf_idx = 0;
1088	else
1089	channel->rd_host_buf_idx++;
1090
1091	dma_sync_single_for_device(dev: channel->endpoint->dev,
1092	addr: channel->rd_buffers[bufidx]->dma_addr,
1093	size: channel->rd_buf_size,
1094	dir: DMA_TO_DEVICE);
1095
1096	mutex_lock(&channel->endpoint->register_mutex);
1097
1098	iowrite32(end_offset_plus1 - 1,
1099	channel->endpoint->registers + fpga_buf_offset_reg);
1100
1101	iowrite32((channel->chan_num << 1) | /* Channel ID */
1102	(2 << 24) | /* Opcode 2, submit buffer */
1103	(bufidx << 12),
1104	channel->endpoint->registers + fpga_buf_ctrl_reg);
1105
1106	mutex_unlock(lock: &channel->endpoint->register_mutex);
1107	} else if (bufidx == 0) {
1108	bufidx = channel->num_rd_buffers - 1;
1109	} else {
1110	bufidx--;
1111	}
1112
1113	channel->rd_host_buf_pos = new_rd_host_buf_pos;
1114
1115	if (timeout < 0)
1116	goto done; /* Autoflush */
1117
1118	/*
1119	* bufidx is now the last buffer written to (or equal to
1120	* rd_fpga_buf_idx if buffer was never written to), and
1121	* channel->rd_host_buf_idx the one after it.
1122	*
1123	* If bufidx == channel->rd_fpga_buf_idx we're either empty or full.
1124	*/
1125
1126	while (1) { /* Loop waiting for draining of buffers */
1127	spin_lock_irqsave(&channel->rd_spinlock, flags);
1128
1129	if (bufidx != channel->rd_fpga_buf_idx)
1130	channel->rd_full = 1; /*
1131	* Not really full,
1132	* but needs waiting.
1133	*/
1134
1135	empty = !channel->rd_full;
1136
1137	spin_unlock_irqrestore(lock: &channel->rd_spinlock, flags);
1138
1139	if (empty)
1140	break;
1141
1142	/*
1143	* Indefinite sleep with mutex taken. With data waiting for
1144	* flushing user should not be surprised if open() for write
1145	* sleeps.
1146	*/
1147	if (timeout == 0)
1148	wait_event_interruptible(channel->rd_wait,
1149	(!channel->rd_full));
1150
1151	else if (wait_event_interruptible_timeout(
1152	channel->rd_wait,
1153	(!channel->rd_full),
1154	timeout) == 0) {
1155	dev_warn(channel->endpoint->dev,
1156	"Timed out while flushing. Output data may be lost.\n");
1157
1158	rc = -ETIMEDOUT;
1159	break;
1160	}
1161
1162	if (channel->rd_full) {
1163	rc = -EINTR;
1164	break;
1165	}
1166	}
1167
1168	done:
1169	mutex_unlock(lock: &channel->rd_mutex);
1170
1171	if (channel->endpoint->fatal_error)
1172	return -EIO;
1173
1174	return rc;
1175	}
1176
1177	static int xillybus_flush(struct file *filp, fl_owner_t id)
1178	{
1179	if (!(filp->f_mode & FMODE_WRITE))
1180	return 0;
1181
1182	return xillybus_myflush(channel: filp->private_data, HZ); /* 1 second timeout */
1183	}
1184
1185	static void xillybus_autoflush(struct work_struct *work)
1186	{
1187	struct delayed_work *workitem = container_of(
1188	work, struct delayed_work, work);
1189	struct xilly_channel *channel = container_of(
1190	workitem, struct xilly_channel, rd_workitem);
1191	int rc;
1192
1193	rc = xillybus_myflush(channel, timeout: -1);
1194	if (rc == -EINTR)
1195	dev_warn(channel->endpoint->dev,
1196	"Autoflush failed because work queue thread got a signal.\n");
1197	else if (rc)
1198	dev_err(channel->endpoint->dev,
1199	"Autoflush failed under weird circumstances.\n");
1200	}
1201
1202	static ssize_t xillybus_write(struct file *filp, const char __user *userbuf,
1203	size_t count, loff_t *f_pos)
1204	{
1205	ssize_t rc;
1206	unsigned long flags;
1207	int bytes_done = 0;
1208	struct xilly_channel *channel = filp->private_data;
1209
1210	int full, exhausted;
1211	/* Initializations are there only to silence warnings */
1212
1213	int howmany = 0, bufpos = 0, bufidx = 0, bufferdone = 0;
1214	int end_offset_plus1 = 0;
1215
1216	if (channel->endpoint->fatal_error)
1217	return -EIO;
1218
1219	rc = mutex_lock_interruptible(&channel->rd_mutex);
1220	if (rc)
1221	return rc;
1222
1223	while (1) {
1224	int bytes_to_do = count - bytes_done;
1225
1226	spin_lock_irqsave(&channel->rd_spinlock, flags);
1227
1228	full = channel->rd_full;
1229
1230	if (!full) {
1231	bufidx = channel->rd_host_buf_idx;
1232	bufpos = channel->rd_host_buf_pos;
1233	howmany = channel->rd_buf_size - bufpos;
1234
1235	/*
1236	* Update rd_host_* to its state after this operation.
1237	* count=0 means committing the buffer immediately,
1238	* which is like flushing, but not necessarily block.
1239	*/
1240
1241	if ((howmany > bytes_to_do) &&
1242	(count ||
1243	((bufpos >> channel->log2_element_size) == 0))) {
1244	bufferdone = 0;
1245
1246	howmany = bytes_to_do;
1247	channel->rd_host_buf_pos += howmany;
1248	} else {
1249	bufferdone = 1;
1250
1251	if (count) {
1252	end_offset_plus1 =
1253	channel->rd_buf_size >>
1254	channel->log2_element_size;
1255	channel->rd_host_buf_pos = 0;
1256	} else {
1257	unsigned char *tail;
1258	int i;
1259
1260	howmany = 0;
1261
1262	end_offset_plus1 = bufpos >>
1263	channel->log2_element_size;
1264
1265	channel->rd_host_buf_pos -=
1266	end_offset_plus1 <<
1267	channel->log2_element_size;
1268
1269	tail = channel->
1270	rd_buffers[bufidx]->addr +
1271	(end_offset_plus1 <<
1272	channel->log2_element_size);
1273
1274	for (i = 0;
1275	i < channel->rd_host_buf_pos;
1276	i++)
1277	channel->rd_leftovers[i] =
1278	*tail++;
1279	}
1280
1281	if (bufidx == channel->rd_fpga_buf_idx)
1282	channel->rd_full = 1;
1283
1284	if (bufidx >= (channel->num_rd_buffers - 1))
1285	channel->rd_host_buf_idx = 0;
1286	else
1287	channel->rd_host_buf_idx++;
1288	}
1289	}
1290
1291	/*
1292	* Marking our situation after the possible changes above,
1293	* for use after releasing the spinlock.
1294	*
1295	* full = full before change
1296	* exhasted = full after possible change
1297	*/
1298
1299	exhausted = channel->rd_full;
1300
1301	spin_unlock_irqrestore(lock: &channel->rd_spinlock, flags);
1302
1303	if (!full) { /* Go on, now without the spinlock */
1304	unsigned char *head =
1305	channel->rd_buffers[bufidx]->addr;
1306	int i;
1307
1308	if ((bufpos == 0) || /* Zero means it's virgin */
1309	(channel->rd_leftovers[3] != 0)) {
1310	dma_sync_single_for_cpu(dev: channel->endpoint->dev,
1311	addr: channel->rd_buffers[bufidx]->dma_addr,
1312	size: channel->rd_buf_size,
1313	dir: DMA_TO_DEVICE);
1314
1315	/* Virgin, but leftovers are due */
1316	for (i = 0; i < bufpos; i++)
1317	*head++ = channel->rd_leftovers[i];
1318
1319	channel->rd_leftovers[3] = 0; /* Clear flag */
1320	}
1321
1322	if (copy_from_user(
1323	to: channel->rd_buffers[bufidx]->addr + bufpos,
1324	from: userbuf, n: howmany))
1325	rc = -EFAULT;
1326
1327	userbuf += howmany;
1328	bytes_done += howmany;
1329
1330	if (bufferdone) {
1331	dma_sync_single_for_device(dev: channel->endpoint->dev,
1332	addr: channel->rd_buffers[bufidx]->dma_addr,
1333	size: channel->rd_buf_size,
1334	dir: DMA_TO_DEVICE);
1335
1336	mutex_lock(&channel->endpoint->register_mutex);
1337
1338	iowrite32(end_offset_plus1 - 1,
1339	channel->endpoint->registers +
1340	fpga_buf_offset_reg);
1341
1342	iowrite32((channel->chan_num << 1) |
1343	(2 << 24) | /* 2 = submit buffer */
1344	(bufidx << 12),
1345	channel->endpoint->registers +
1346	fpga_buf_ctrl_reg);
1347
1348	mutex_unlock(lock: &channel->endpoint->
1349	register_mutex);
1350
1351	channel->rd_leftovers[3] =
1352	(channel->rd_host_buf_pos != 0);
1353	}
1354
1355	if (rc) {
1356	mutex_unlock(lock: &channel->rd_mutex);
1357
1358	if (channel->endpoint->fatal_error)
1359	return -EIO;
1360
1361	if (!channel->rd_synchronous)
1362	queue_delayed_work(
1363	wq: xillybus_wq,
1364	dwork: &channel->rd_workitem,
1365	XILLY_RX_TIMEOUT);
1366
1367	return rc;
1368	}
1369	}
1370
1371	if (bytes_done >= count)
1372	break;
1373
1374	if (!exhausted)
1375	continue; /* If there's more space, just go on */
1376
1377	if ((bytes_done > 0) && channel->rd_allow_partial)
1378	break;
1379
1380	/*
1381	* Indefinite sleep with mutex taken. With data waiting for
1382	* flushing, user should not be surprised if open() for write
1383	* sleeps.
1384	*/
1385
1386	if (filp->f_flags & O_NONBLOCK) {
1387	rc = -EAGAIN;
1388	break;
1389	}
1390
1391	if (wait_event_interruptible(channel->rd_wait,
1392	(!channel->rd_full))) {
1393	mutex_unlock(lock: &channel->rd_mutex);
1394
1395	if (channel->endpoint->fatal_error)
1396	return -EIO;
1397
1398	if (bytes_done)
1399	return bytes_done;
1400	return -EINTR;
1401	}
1402	}
1403
1404	mutex_unlock(lock: &channel->rd_mutex);
1405
1406	if (!channel->rd_synchronous)
1407	queue_delayed_work(wq: xillybus_wq,
1408	dwork: &channel->rd_workitem,
1409	XILLY_RX_TIMEOUT);
1410
1411	if (channel->endpoint->fatal_error)
1412	return -EIO;
1413
1414	if (rc)
1415	return rc;
1416
1417	if ((channel->rd_synchronous) && (bytes_done > 0)) {
1418	rc = xillybus_myflush(channel: filp->private_data, timeout: 0); /* No timeout */
1419
1420	if (rc && (rc != -EINTR))
1421	return rc;
1422	}
1423
1424	return bytes_done;
1425	}
1426
1427	static int xillybus_open(struct inode *inode, struct file *filp)
1428	{
1429	int rc;
1430	unsigned long flags;
1431	struct xilly_endpoint *endpoint;
1432	struct xilly_channel *channel;
1433	int index;
1434
1435	rc = xillybus_find_inode(inode, private_data: (void **)&endpoint, index: &index);
1436	if (rc)
1437	return rc;
1438
1439	if (endpoint->fatal_error)
1440	return -EIO;
1441
1442	channel = endpoint->channels[1 + index];
1443	filp->private_data = channel;
1444
1445	/*
1446	* It gets complicated because:
1447	* 1. We don't want to take a mutex we don't have to
1448	* 2. We don't want to open one direction if the other will fail.
1449	*/
1450
1451	if ((filp->f_mode & FMODE_READ) && (!channel->num_wr_buffers))
1452	return -ENODEV;
1453
1454	if ((filp->f_mode & FMODE_WRITE) && (!channel->num_rd_buffers))
1455	return -ENODEV;
1456
1457	if ((filp->f_mode & FMODE_READ) && (filp->f_flags & O_NONBLOCK) &&
1458	(channel->wr_synchronous || !channel->wr_allow_partial ||
1459	!channel->wr_supports_nonempty)) {
1460	dev_err(endpoint->dev,
1461	"open() failed: O_NONBLOCK not allowed for read on this device\n");
1462	return -ENODEV;
1463	}
1464
1465	if ((filp->f_mode & FMODE_WRITE) && (filp->f_flags & O_NONBLOCK) &&
1466	(channel->rd_synchronous || !channel->rd_allow_partial)) {
1467	dev_err(endpoint->dev,
1468	"open() failed: O_NONBLOCK not allowed for write on this device\n");
1469	return -ENODEV;
1470	}
1471
1472	/*
1473	* Note: open() may block on getting mutexes despite O_NONBLOCK.
1474	* This shouldn't occur normally, since multiple open of the same
1475	* file descriptor is almost always prohibited anyhow
1476	* (*_exclusive_open is normally set in real-life systems).
1477	*/
1478
1479	if (filp->f_mode & FMODE_READ) {
1480	rc = mutex_lock_interruptible(&channel->wr_mutex);
1481	if (rc)
1482	return rc;
1483	}
1484
1485	if (filp->f_mode & FMODE_WRITE) {
1486	rc = mutex_lock_interruptible(&channel->rd_mutex);
1487	if (rc)
1488	goto unlock_wr;
1489	}
1490
1491	if ((filp->f_mode & FMODE_READ) &&
1492	(channel->wr_ref_count != 0) &&
1493	(channel->wr_exclusive_open)) {
1494	rc = -EBUSY;
1495	goto unlock;
1496	}
1497
1498	if ((filp->f_mode & FMODE_WRITE) &&
1499	(channel->rd_ref_count != 0) &&
1500	(channel->rd_exclusive_open)) {
1501	rc = -EBUSY;
1502	goto unlock;
1503	}
1504
1505	if (filp->f_mode & FMODE_READ) {
1506	if (channel->wr_ref_count == 0) { /* First open of file */
1507	/* Move the host to first buffer */
1508	spin_lock_irqsave(&channel->wr_spinlock, flags);
1509	channel->wr_host_buf_idx = 0;
1510	channel->wr_host_buf_pos = 0;
1511	channel->wr_fpga_buf_idx = -1;
1512	channel->wr_empty = 1;
1513	channel->wr_ready = 0;
1514	channel->wr_sleepy = 1;
1515	channel->wr_eof = -1;
1516	channel->wr_hangup = 0;
1517
1518	spin_unlock_irqrestore(lock: &channel->wr_spinlock, flags);
1519
1520	iowrite32(1 | (channel->chan_num << 1) |
1521	(4 << 24) | /* Opcode 4, open channel */
1522	((channel->wr_synchronous & 1) << 23),
1523	channel->endpoint->registers +
1524	fpga_buf_ctrl_reg);
1525	}
1526
1527	channel->wr_ref_count++;
1528	}
1529
1530	if (filp->f_mode & FMODE_WRITE) {
1531	if (channel->rd_ref_count == 0) { /* First open of file */
1532	/* Move the host to first buffer */
1533	spin_lock_irqsave(&channel->rd_spinlock, flags);
1534	channel->rd_host_buf_idx = 0;
1535	channel->rd_host_buf_pos = 0;
1536	channel->rd_leftovers[3] = 0; /* No leftovers. */
1537	channel->rd_fpga_buf_idx = channel->num_rd_buffers - 1;
1538	channel->rd_full = 0;
1539
1540	spin_unlock_irqrestore(lock: &channel->rd_spinlock, flags);
1541
1542	iowrite32((channel->chan_num << 1) |
1543	(4 << 24), /* Opcode 4, open channel */
1544	channel->endpoint->registers +
1545	fpga_buf_ctrl_reg);
1546	}
1547
1548	channel->rd_ref_count++;
1549	}
1550
1551	unlock:
1552	if (filp->f_mode & FMODE_WRITE)
1553	mutex_unlock(lock: &channel->rd_mutex);
1554	unlock_wr:
1555	if (filp->f_mode & FMODE_READ)
1556	mutex_unlock(lock: &channel->wr_mutex);
1557
1558	if (!rc && (!channel->seekable))
1559	return nonseekable_open(inode, filp);
1560
1561	return rc;
1562	}
1563
1564	static int xillybus_release(struct inode *inode, struct file *filp)
1565	{
1566	unsigned long flags;
1567	struct xilly_channel *channel = filp->private_data;
1568
1569	int buf_idx;
1570	int eof;
1571
1572	if (channel->endpoint->fatal_error)
1573	return -EIO;
1574
1575	if (filp->f_mode & FMODE_WRITE) {
1576	mutex_lock(&channel->rd_mutex);
1577
1578	channel->rd_ref_count--;
1579
1580	if (channel->rd_ref_count == 0) {
1581	/*
1582	* We rely on the kernel calling flush()
1583	* before we get here.
1584	*/
1585
1586	iowrite32((channel->chan_num << 1) | /* Channel ID */
1587	(5 << 24), /* Opcode 5, close channel */
1588	channel->endpoint->registers +
1589	fpga_buf_ctrl_reg);
1590	}
1591	mutex_unlock(lock: &channel->rd_mutex);
1592	}
1593
1594	if (filp->f_mode & FMODE_READ) {
1595	mutex_lock(&channel->wr_mutex);
1596
1597	channel->wr_ref_count--;
1598
1599	if (channel->wr_ref_count == 0) {
1600	iowrite32(1 | (channel->chan_num << 1) |
1601	(5 << 24), /* Opcode 5, close channel */
1602	channel->endpoint->registers +
1603	fpga_buf_ctrl_reg);
1604
1605	/*
1606	* This is crazily cautious: We make sure that not
1607	* only that we got an EOF (be it because we closed
1608	* the channel or because of a user's EOF), but verify
1609	* that it's one beyond the last buffer arrived, so
1610	* we have no leftover buffers pending before wrapping
1611	* up (which can only happen in asynchronous channels,
1612	* BTW)
1613	*/
1614
1615	while (1) {
1616	spin_lock_irqsave(&channel->wr_spinlock,
1617	flags);
1618	buf_idx = channel->wr_fpga_buf_idx;
1619	eof = channel->wr_eof;
1620	channel->wr_sleepy = 1;
1621	spin_unlock_irqrestore(lock: &channel->wr_spinlock,
1622	flags);
1623
1624	/*
1625	* Check if eof points at the buffer after
1626	* the last one the FPGA submitted. Note that
1627	* no EOF is marked by negative eof.
1628	*/
1629
1630	buf_idx++;
1631	if (buf_idx == channel->num_wr_buffers)
1632	buf_idx = 0;
1633
1634	if (buf_idx == eof)
1635	break;
1636
1637	/*
1638	* Steal extra 100 ms if awaken by interrupt.
1639	* This is a simple workaround for an
1640	* interrupt pending when entering, which would
1641	* otherwise result in declaring the hardware
1642	* non-responsive.
1643	*/
1644
1645	if (wait_event_interruptible(
1646	channel->wr_wait,
1647	(!channel->wr_sleepy)))
1648	msleep(msecs: 100);
1649
1650	if (channel->wr_sleepy) {
1651	mutex_unlock(lock: &channel->wr_mutex);
1652	dev_warn(channel->endpoint->dev,
1653	"Hardware failed to respond to close command, therefore left in messy state.\n");
1654	return -EINTR;
1655	}
1656	}
1657	}
1658
1659	mutex_unlock(lock: &channel->wr_mutex);
1660	}
1661
1662	return 0;
1663	}
1664
1665	static loff_t xillybus_llseek(struct file *filp, loff_t offset, int whence)
1666	{
1667	struct xilly_channel *channel = filp->private_data;
1668	loff_t pos = filp->f_pos;
1669	int rc = 0;
1670
1671	/*
1672	* Take both mutexes not allowing interrupts, since it seems like
1673	* common applications don't expect an -EINTR here. Besides, multiple
1674	* access to a single file descriptor on seekable devices is a mess
1675	* anyhow.
1676	*/
1677
1678	if (channel->endpoint->fatal_error)
1679	return -EIO;
1680
1681	mutex_lock(&channel->wr_mutex);
1682	mutex_lock(&channel->rd_mutex);
1683
1684	switch (whence) {
1685	case SEEK_SET:
1686	pos = offset;
1687	break;
1688	case SEEK_CUR:
1689	pos += offset;
1690	break;
1691	case SEEK_END:
1692	pos = offset; /* Going to the end => to the beginning */
1693	break;
1694	default:
1695	rc = -EINVAL;
1696	goto end;
1697	}
1698
1699	/* In any case, we must finish on an element boundary */
1700	if (pos & ((1 << channel->log2_element_size) - 1)) {
1701	rc = -EINVAL;
1702	goto end;
1703	}
1704
1705	mutex_lock(&channel->endpoint->register_mutex);
1706
1707	iowrite32(pos >> channel->log2_element_size,
1708	channel->endpoint->registers + fpga_buf_offset_reg);
1709
1710	iowrite32((channel->chan_num << 1) |
1711	(6 << 24), /* Opcode 6, set address */
1712	channel->endpoint->registers + fpga_buf_ctrl_reg);
1713
1714	mutex_unlock(lock: &channel->endpoint->register_mutex);
1715
1716	end:
1717	mutex_unlock(lock: &channel->rd_mutex);
1718	mutex_unlock(lock: &channel->wr_mutex);
1719
1720	if (rc) /* Return error after releasing mutexes */
1721	return rc;
1722
1723	filp->f_pos = pos;
1724
1725	/*
1726	* Since seekable devices are allowed only when the channel is
1727	* synchronous, we assume that there is no data pending in either
1728	* direction (which holds true as long as no concurrent access on the
1729	* file descriptor takes place).
1730	* The only thing we may need to throw away is leftovers from partial
1731	* write() flush.
1732	*/
1733
1734	channel->rd_leftovers[3] = 0;
1735
1736	return pos;
1737	}
1738
1739	static __poll_t xillybus_poll(struct file *filp, poll_table *wait)
1740	{
1741	struct xilly_channel *channel = filp->private_data;
1742	__poll_t mask = 0;
1743	unsigned long flags;
1744
1745	poll_wait(filp, wait_address: &channel->endpoint->ep_wait, p: wait);
1746
1747	/*
1748	* poll() won't play ball regarding read() channels which
1749	* aren't asynchronous and support the nonempty message. Allowing
1750	* that will create situations where data has been delivered at
1751	* the FPGA, and users expecting select() to wake up, which it may
1752	* not.
1753	*/
1754
1755	if (!channel->wr_synchronous && channel->wr_supports_nonempty) {
1756	poll_wait(filp, wait_address: &channel->wr_wait, p: wait);
1757	poll_wait(filp, wait_address: &channel->wr_ready_wait, p: wait);
1758
1759	spin_lock_irqsave(&channel->wr_spinlock, flags);
1760	if (!channel->wr_empty || channel->wr_ready)
1761	mask |= EPOLLIN | EPOLLRDNORM;
1762
1763	if (channel->wr_hangup)
1764	/*
1765	* Not EPOLLHUP, because its behavior is in the
1766	* mist, and EPOLLIN does what we want: Wake up
1767	* the read file descriptor so it sees EOF.
1768	*/
1769	mask |= EPOLLIN | EPOLLRDNORM;
1770	spin_unlock_irqrestore(lock: &channel->wr_spinlock, flags);
1771	}
1772
1773	/*
1774	* If partial data write is disallowed on a write() channel,
1775	* it's pointless to ever signal OK to write, because is could
1776	* block despite some space being available.
1777	*/
1778
1779	if (channel->rd_allow_partial) {
1780	poll_wait(filp, wait_address: &channel->rd_wait, p: wait);
1781
1782	spin_lock_irqsave(&channel->rd_spinlock, flags);
1783	if (!channel->rd_full)
1784	mask |= EPOLLOUT | EPOLLWRNORM;
1785	spin_unlock_irqrestore(lock: &channel->rd_spinlock, flags);
1786	}
1787
1788	if (channel->endpoint->fatal_error)
1789	mask |= EPOLLERR;
1790
1791	return mask;
1792	}
1793
1794	static const struct file_operations xillybus_fops = {
1795	.owner = THIS_MODULE,
1796	.read = xillybus_read,
1797	.write = xillybus_write,
1798	.open = xillybus_open,
1799	.flush = xillybus_flush,
1800	.release = xillybus_release,
1801	.llseek = xillybus_llseek,
1802	.poll = xillybus_poll,
1803	};
1804
1805	struct xilly_endpoint *xillybus_init_endpoint(struct device *dev)
1806	{
1807	struct xilly_endpoint *endpoint;
1808
1809	endpoint = devm_kzalloc(dev, size: sizeof(*endpoint), GFP_KERNEL);
1810	if (!endpoint)
1811	return NULL;
1812
1813	endpoint->dev = dev;
1814	endpoint->msg_counter = 0x0b;
1815	endpoint->failed_messages = 0;
1816	endpoint->fatal_error = 0;
1817
1818	init_waitqueue_head(&endpoint->ep_wait);
1819	mutex_init(&endpoint->register_mutex);
1820
1821	return endpoint;
1822	}
1823	EXPORT_SYMBOL(xillybus_init_endpoint);
1824
1825	static int xilly_quiesce(struct xilly_endpoint *endpoint)
1826	{
1827	long t;
1828
1829	endpoint->idtlen = -1;
1830
1831	iowrite32((u32) (endpoint->dma_using_dac & 0x0001),
1832	endpoint->registers + fpga_dma_control_reg);
1833
1834	t = wait_event_interruptible_timeout(endpoint->ep_wait,
1835	(endpoint->idtlen >= 0),
1836	XILLY_TIMEOUT);
1837	if (t <= 0) {
1838	dev_err(endpoint->dev,
1839	"Failed to quiesce the device on exit.\n");
1840	return -ENODEV;
1841	}
1842	return 0;
1843	}
1844
1845	int xillybus_endpoint_discovery(struct xilly_endpoint *endpoint)
1846	{
1847	int rc;
1848	long t;
1849
1850	void *bootstrap_resources;
1851	int idtbuffersize = (1 << PAGE_SHIFT);
1852	struct device *dev = endpoint->dev;
1853
1854	/*
1855	* The bogus IDT is used during bootstrap for allocating the initial
1856	* message buffer, and then the message buffer and space for the IDT
1857	* itself. The initial message buffer is of a single page's size, but
1858	* it's soon replaced with a more modest one (and memory is freed).
1859	*/
1860
1861	unsigned char bogus_idt[8] = { 1, 224, (PAGE_SHIFT)-2, 0,
1862	3, 192, PAGE_SHIFT, 0 };
1863	struct xilly_idt_handle idt_handle;
1864
1865	/*
1866	* Writing the value 0x00000001 to Endianness register signals which
1867	* endianness this processor is using, so the FPGA can swap words as
1868	* necessary.
1869	*/
1870
1871	iowrite32(1, endpoint->registers + fpga_endian_reg);
1872
1873	/* Bootstrap phase I: Allocate temporary message buffer */
1874
1875	bootstrap_resources = devres_open_group(dev, NULL, GFP_KERNEL);
1876	if (!bootstrap_resources)
1877	return -ENOMEM;
1878
1879	endpoint->num_channels = 0;
1880
1881	rc = xilly_setupchannels(ep: endpoint, chandesc: bogus_idt, entries: 1);
1882	if (rc)
1883	return rc;
1884
1885	/* Clear the message subsystem (and counter in particular) */
1886	iowrite32(0x04, endpoint->registers + fpga_msg_ctrl_reg);
1887
1888	endpoint->idtlen = -1;
1889
1890	/*
1891	* Set DMA 32/64 bit mode, quiesce the device (?!) and get IDT
1892	* buffer size.
1893	*/
1894	iowrite32((u32) (endpoint->dma_using_dac & 0x0001),
1895	endpoint->registers + fpga_dma_control_reg);
1896
1897	t = wait_event_interruptible_timeout(endpoint->ep_wait,
1898	(endpoint->idtlen >= 0),
1899	XILLY_TIMEOUT);
1900	if (t <= 0) {
1901	dev_err(endpoint->dev, "No response from FPGA. Aborting.\n");
1902	return -ENODEV;
1903	}
1904
1905	/* Enable DMA */
1906	iowrite32((u32) (0x0002 | (endpoint->dma_using_dac & 0x0001)),
1907	endpoint->registers + fpga_dma_control_reg);
1908
1909	/* Bootstrap phase II: Allocate buffer for IDT and obtain it */
1910	while (endpoint->idtlen >= idtbuffersize) {
1911	idtbuffersize *= 2;
1912	bogus_idt[6]++;
1913	}
1914
1915	endpoint->num_channels = 1;
1916
1917	rc = xilly_setupchannels(ep: endpoint, chandesc: bogus_idt, entries: 2);
1918	if (rc)
1919	goto failed_idt;
1920
1921	rc = xilly_obtain_idt(endpoint);
1922	if (rc)
1923	goto failed_idt;
1924
1925	rc = xilly_scan_idt(endpoint, idt_handle: &idt_handle);
1926	if (rc)
1927	goto failed_idt;
1928
1929	devres_close_group(dev, id: bootstrap_resources);
1930
1931	/* Bootstrap phase III: Allocate buffers according to IDT */
1932
1933	rc = xilly_setupchannels(ep: endpoint,
1934	chandesc: idt_handle.chandesc,
1935	entries: idt_handle.entries);
1936	if (rc)
1937	goto failed_idt;
1938
1939	rc = xillybus_init_chrdev(dev, fops: &xillybus_fops,
1940	owner: endpoint->owner, private_data: endpoint,
1941	idt: idt_handle.names,
1942	len: idt_handle.names_len,
1943	num_nodes: endpoint->num_channels,
1944	prefix: xillyname, enumerate: false);
1945
1946	if (rc)
1947	goto failed_idt;
1948
1949	devres_release_group(dev, id: bootstrap_resources);
1950
1951	return 0;
1952
1953	failed_idt:
1954	xilly_quiesce(endpoint);
1955	flush_workqueue(xillybus_wq);
1956
1957	return rc;
1958	}
1959	EXPORT_SYMBOL(xillybus_endpoint_discovery);
1960
1961	void xillybus_endpoint_remove(struct xilly_endpoint *endpoint)
1962	{
1963	xillybus_cleanup_chrdev(private_data: endpoint, dev: endpoint->dev);
1964
1965	xilly_quiesce(endpoint);
1966
1967	/*
1968	* Flushing is done upon endpoint release to prevent access to memory
1969	* just about to be released. This makes the quiesce complete.
1970	*/
1971	flush_workqueue(xillybus_wq);
1972	}
1973	EXPORT_SYMBOL(xillybus_endpoint_remove);
1974
1975	static int __init xillybus_init(void)
1976	{
1977	xillybus_wq = alloc_workqueue(fmt: xillyname, flags: 0, max_active: 0);
1978	if (!xillybus_wq)
1979	return -ENOMEM;
1980
1981	return 0;
1982	}
1983
1984	static void __exit xillybus_exit(void)
1985	{
1986	/* flush_workqueue() was called for each endpoint released */
1987	destroy_workqueue(wq: xillybus_wq);
1988	}
1989
1990	module_init(xillybus_init);
1991	module_exit(xillybus_exit);
1992