1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Tty buffer allocation management |
4 | */ |
5 | |
6 | #include <linux/types.h> |
7 | #include <linux/errno.h> |
8 | #include <linux/minmax.h> |
9 | #include <linux/tty.h> |
10 | #include <linux/tty_driver.h> |
11 | #include <linux/tty_flip.h> |
12 | #include <linux/timer.h> |
13 | #include <linux/string.h> |
14 | #include <linux/slab.h> |
15 | #include <linux/sched.h> |
16 | #include <linux/wait.h> |
17 | #include <linux/bitops.h> |
18 | #include <linux/delay.h> |
19 | #include <linux/module.h> |
20 | #include <linux/ratelimit.h> |
21 | #include "tty.h" |
22 | |
23 | #define MIN_TTYB_SIZE 256 |
24 | #define TTYB_ALIGN_MASK 0xff |
25 | |
26 | /* |
27 | * Byte threshold to limit memory consumption for flip buffers. |
28 | * The actual memory limit is > 2x this amount. |
29 | */ |
30 | #define TTYB_DEFAULT_MEM_LIMIT (640 * 1024UL) |
31 | |
32 | /* |
33 | * We default to dicing tty buffer allocations to this many characters |
34 | * in order to avoid multiple page allocations. We know the size of |
35 | * tty_buffer itself but it must also be taken into account that the |
36 | * buffer is 256 byte aligned. See tty_buffer_find for the allocation |
37 | * logic this must match. |
38 | */ |
39 | |
40 | #define TTY_BUFFER_PAGE (((PAGE_SIZE - sizeof(struct tty_buffer)) / 2) & ~TTYB_ALIGN_MASK) |
41 | |
42 | /** |
43 | * tty_buffer_lock_exclusive - gain exclusive access to buffer |
44 | * @port: tty port owning the flip buffer |
45 | * |
46 | * Guarantees safe use of the &tty_ldisc_ops.receive_buf() method by excluding |
47 | * the buffer work and any pending flush from using the flip buffer. Data can |
48 | * continue to be added concurrently to the flip buffer from the driver side. |
49 | * |
50 | * See also tty_buffer_unlock_exclusive(). |
51 | */ |
52 | void tty_buffer_lock_exclusive(struct tty_port *port) |
53 | { |
54 | struct tty_bufhead *buf = &port->buf; |
55 | |
56 | atomic_inc(v: &buf->priority); |
57 | mutex_lock(&buf->lock); |
58 | } |
59 | EXPORT_SYMBOL_GPL(tty_buffer_lock_exclusive); |
60 | |
61 | /** |
62 | * tty_buffer_unlock_exclusive - release exclusive access |
63 | * @port: tty port owning the flip buffer |
64 | * |
65 | * The buffer work is restarted if there is data in the flip buffer. |
66 | * |
67 | * See also tty_buffer_lock_exclusive(). |
68 | */ |
69 | void tty_buffer_unlock_exclusive(struct tty_port *port) |
70 | { |
71 | struct tty_bufhead *buf = &port->buf; |
72 | bool restart = buf->head->commit != buf->head->read; |
73 | |
74 | atomic_dec(v: &buf->priority); |
75 | mutex_unlock(lock: &buf->lock); |
76 | |
77 | if (restart) |
78 | queue_work(wq: system_unbound_wq, work: &buf->work); |
79 | } |
80 | EXPORT_SYMBOL_GPL(tty_buffer_unlock_exclusive); |
81 | |
82 | /** |
83 | * tty_buffer_space_avail - return unused buffer space |
84 | * @port: tty port owning the flip buffer |
85 | * |
86 | * Returns: the # of bytes which can be written by the driver without reaching |
87 | * the buffer limit. |
88 | * |
89 | * Note: this does not guarantee that memory is available to write the returned |
90 | * # of bytes (use tty_prepare_flip_string() to pre-allocate if memory |
91 | * guarantee is required). |
92 | */ |
93 | unsigned int tty_buffer_space_avail(struct tty_port *port) |
94 | { |
95 | int space = port->buf.mem_limit - atomic_read(v: &port->buf.mem_used); |
96 | |
97 | return max(space, 0); |
98 | } |
99 | EXPORT_SYMBOL_GPL(tty_buffer_space_avail); |
100 | |
101 | static void tty_buffer_reset(struct tty_buffer *p, size_t size) |
102 | { |
103 | p->used = 0; |
104 | p->size = size; |
105 | p->next = NULL; |
106 | p->commit = 0; |
107 | p->lookahead = 0; |
108 | p->read = 0; |
109 | p->flags = true; |
110 | } |
111 | |
112 | /** |
113 | * tty_buffer_free_all - free buffers used by a tty |
114 | * @port: tty port to free from |
115 | * |
116 | * Remove all the buffers pending on a tty whether queued with data or in the |
117 | * free ring. Must be called when the tty is no longer in use. |
118 | */ |
119 | void tty_buffer_free_all(struct tty_port *port) |
120 | { |
121 | struct tty_bufhead *buf = &port->buf; |
122 | struct tty_buffer *p, *next; |
123 | struct llist_node *llist; |
124 | unsigned int freed = 0; |
125 | int still_used; |
126 | |
127 | while ((p = buf->head) != NULL) { |
128 | buf->head = p->next; |
129 | freed += p->size; |
130 | if (p->size > 0) |
131 | kfree(objp: p); |
132 | } |
133 | llist = llist_del_all(head: &buf->free); |
134 | llist_for_each_entry_safe(p, next, llist, free) |
135 | kfree(objp: p); |
136 | |
137 | tty_buffer_reset(p: &buf->sentinel, size: 0); |
138 | buf->head = &buf->sentinel; |
139 | buf->tail = &buf->sentinel; |
140 | |
141 | still_used = atomic_xchg(v: &buf->mem_used, new: 0); |
142 | WARN(still_used != freed, "we still have not freed %d bytes!" , |
143 | still_used - freed); |
144 | } |
145 | |
146 | /** |
147 | * tty_buffer_alloc - allocate a tty buffer |
148 | * @port: tty port |
149 | * @size: desired size (characters) |
150 | * |
151 | * Allocate a new tty buffer to hold the desired number of characters. We |
152 | * round our buffers off in 256 character chunks to get better allocation |
153 | * behaviour. |
154 | * |
155 | * Returns: %NULL if out of memory or the allocation would exceed the per |
156 | * device queue. |
157 | */ |
158 | static struct tty_buffer *tty_buffer_alloc(struct tty_port *port, size_t size) |
159 | { |
160 | struct llist_node *free; |
161 | struct tty_buffer *p; |
162 | |
163 | /* Round the buffer size out */ |
164 | size = __ALIGN_MASK(size, TTYB_ALIGN_MASK); |
165 | |
166 | if (size <= MIN_TTYB_SIZE) { |
167 | free = llist_del_first(head: &port->buf.free); |
168 | if (free) { |
169 | p = llist_entry(free, struct tty_buffer, free); |
170 | goto found; |
171 | } |
172 | } |
173 | |
174 | /* Should possibly check if this fails for the largest buffer we |
175 | * have queued and recycle that ? |
176 | */ |
177 | if (atomic_read(v: &port->buf.mem_used) > port->buf.mem_limit) |
178 | return NULL; |
179 | p = kmalloc(struct_size(p, data, 2 * size), GFP_ATOMIC | __GFP_NOWARN); |
180 | if (p == NULL) |
181 | return NULL; |
182 | |
183 | found: |
184 | tty_buffer_reset(p, size); |
185 | atomic_add(i: size, v: &port->buf.mem_used); |
186 | return p; |
187 | } |
188 | |
189 | /** |
190 | * tty_buffer_free - free a tty buffer |
191 | * @port: tty port owning the buffer |
192 | * @b: the buffer to free |
193 | * |
194 | * Free a tty buffer, or add it to the free list according to our internal |
195 | * strategy. |
196 | */ |
197 | static void tty_buffer_free(struct tty_port *port, struct tty_buffer *b) |
198 | { |
199 | struct tty_bufhead *buf = &port->buf; |
200 | |
201 | /* Dumb strategy for now - should keep some stats */ |
202 | WARN_ON(atomic_sub_return(b->size, &buf->mem_used) < 0); |
203 | |
204 | if (b->size > MIN_TTYB_SIZE) |
205 | kfree(objp: b); |
206 | else if (b->size > 0) |
207 | llist_add(new: &b->free, head: &buf->free); |
208 | } |
209 | |
210 | /** |
211 | * tty_buffer_flush - flush full tty buffers |
212 | * @tty: tty to flush |
213 | * @ld: optional ldisc ptr (must be referenced) |
214 | * |
215 | * Flush all the buffers containing receive data. If @ld != %NULL, flush the |
216 | * ldisc input buffer. |
217 | * |
218 | * Locking: takes buffer lock to ensure single-threaded flip buffer 'consumer'. |
219 | */ |
220 | void tty_buffer_flush(struct tty_struct *tty, struct tty_ldisc *ld) |
221 | { |
222 | struct tty_port *port = tty->port; |
223 | struct tty_bufhead *buf = &port->buf; |
224 | struct tty_buffer *next; |
225 | |
226 | atomic_inc(v: &buf->priority); |
227 | |
228 | mutex_lock(&buf->lock); |
229 | /* paired w/ release in __tty_buffer_request_room; ensures there are |
230 | * no pending memory accesses to the freed buffer |
231 | */ |
232 | while ((next = smp_load_acquire(&buf->head->next)) != NULL) { |
233 | tty_buffer_free(port, b: buf->head); |
234 | buf->head = next; |
235 | } |
236 | buf->head->read = buf->head->commit; |
237 | buf->head->lookahead = buf->head->read; |
238 | |
239 | if (ld && ld->ops->flush_buffer) |
240 | ld->ops->flush_buffer(tty); |
241 | |
242 | atomic_dec(v: &buf->priority); |
243 | mutex_unlock(lock: &buf->lock); |
244 | } |
245 | |
246 | /** |
247 | * __tty_buffer_request_room - grow tty buffer if needed |
248 | * @port: tty port |
249 | * @size: size desired |
250 | * @flags: buffer has to store flags along character data |
251 | * |
252 | * Make at least @size bytes of linear space available for the tty buffer. |
253 | * |
254 | * Will change over to a new buffer if the current buffer is encoded as |
255 | * %TTY_NORMAL (so has no flags buffer) and the new buffer requires a flags |
256 | * buffer. |
257 | * |
258 | * Returns: the size we managed to find. |
259 | */ |
260 | static int __tty_buffer_request_room(struct tty_port *port, size_t size, |
261 | bool flags) |
262 | { |
263 | struct tty_bufhead *buf = &port->buf; |
264 | struct tty_buffer *n, *b = buf->tail; |
265 | size_t left = (b->flags ? 1 : 2) * b->size - b->used; |
266 | bool change = !b->flags && flags; |
267 | |
268 | if (!change && left >= size) |
269 | return size; |
270 | |
271 | /* This is the slow path - looking for new buffers to use */ |
272 | n = tty_buffer_alloc(port, size); |
273 | if (n == NULL) |
274 | return change ? 0 : left; |
275 | |
276 | n->flags = flags; |
277 | buf->tail = n; |
278 | /* |
279 | * Paired w/ acquire in flush_to_ldisc() and lookahead_bufs() |
280 | * ensures they see all buffer data. |
281 | */ |
282 | smp_store_release(&b->commit, b->used); |
283 | /* |
284 | * Paired w/ acquire in flush_to_ldisc() and lookahead_bufs() |
285 | * ensures the latest commit value can be read before the head |
286 | * is advanced to the next buffer. |
287 | */ |
288 | smp_store_release(&b->next, n); |
289 | |
290 | return size; |
291 | } |
292 | |
293 | int tty_buffer_request_room(struct tty_port *port, size_t size) |
294 | { |
295 | return __tty_buffer_request_room(port, size, flags: true); |
296 | } |
297 | EXPORT_SYMBOL_GPL(tty_buffer_request_room); |
298 | |
299 | size_t __tty_insert_flip_string_flags(struct tty_port *port, const u8 *chars, |
300 | const u8 *flags, bool mutable_flags, |
301 | size_t size) |
302 | { |
303 | bool need_flags = mutable_flags || flags[0] != TTY_NORMAL; |
304 | size_t copied = 0; |
305 | |
306 | do { |
307 | size_t goal = min_t(size_t, size - copied, TTY_BUFFER_PAGE); |
308 | size_t space = __tty_buffer_request_room(port, size: goal, flags: need_flags); |
309 | struct tty_buffer *tb = port->buf.tail; |
310 | |
311 | if (unlikely(space == 0)) |
312 | break; |
313 | |
314 | memcpy(char_buf_ptr(tb, tb->used), chars, space); |
315 | |
316 | if (mutable_flags) { |
317 | memcpy(flag_buf_ptr(tb, tb->used), flags, space); |
318 | flags += space; |
319 | } else if (tb->flags) { |
320 | memset(flag_buf_ptr(tb, tb->used), flags[0], space); |
321 | } else { |
322 | /* tb->flags should be available once requested */ |
323 | WARN_ON_ONCE(need_flags); |
324 | } |
325 | |
326 | tb->used += space; |
327 | copied += space; |
328 | chars += space; |
329 | |
330 | /* There is a small chance that we need to split the data over |
331 | * several buffers. If this is the case we must loop. |
332 | */ |
333 | } while (unlikely(size > copied)); |
334 | |
335 | return copied; |
336 | } |
337 | EXPORT_SYMBOL(__tty_insert_flip_string_flags); |
338 | |
339 | /** |
340 | * tty_prepare_flip_string - make room for characters |
341 | * @port: tty port |
342 | * @chars: return pointer for character write area |
343 | * @size: desired size |
344 | * |
345 | * Prepare a block of space in the buffer for data. |
346 | * |
347 | * This is used for drivers that need their own block copy routines into the |
348 | * buffer. There is no guarantee the buffer is a DMA target! |
349 | * |
350 | * Returns: the length available and buffer pointer (@chars) to the space which |
351 | * is now allocated and accounted for as ready for normal characters. |
352 | */ |
353 | size_t tty_prepare_flip_string(struct tty_port *port, u8 **chars, size_t size) |
354 | { |
355 | size_t space = __tty_buffer_request_room(port, size, flags: false); |
356 | |
357 | if (likely(space)) { |
358 | struct tty_buffer *tb = port->buf.tail; |
359 | |
360 | *chars = char_buf_ptr(b: tb, ofs: tb->used); |
361 | if (tb->flags) |
362 | memset(flag_buf_ptr(tb, tb->used), TTY_NORMAL, space); |
363 | tb->used += space; |
364 | } |
365 | |
366 | return space; |
367 | } |
368 | EXPORT_SYMBOL_GPL(tty_prepare_flip_string); |
369 | |
370 | /** |
371 | * tty_ldisc_receive_buf - forward data to line discipline |
372 | * @ld: line discipline to process input |
373 | * @p: char buffer |
374 | * @f: %TTY_NORMAL, %TTY_BREAK, etc. flags buffer |
375 | * @count: number of bytes to process |
376 | * |
377 | * Callers other than flush_to_ldisc() need to exclude the kworker from |
378 | * concurrent use of the line discipline, see paste_selection(). |
379 | * |
380 | * Returns: the number of bytes processed. |
381 | */ |
382 | size_t tty_ldisc_receive_buf(struct tty_ldisc *ld, const u8 *p, const u8 *f, |
383 | size_t count) |
384 | { |
385 | if (ld->ops->receive_buf2) |
386 | count = ld->ops->receive_buf2(ld->tty, p, f, count); |
387 | else { |
388 | count = min_t(size_t, count, ld->tty->receive_room); |
389 | if (count && ld->ops->receive_buf) |
390 | ld->ops->receive_buf(ld->tty, p, f, count); |
391 | } |
392 | return count; |
393 | } |
394 | EXPORT_SYMBOL_GPL(tty_ldisc_receive_buf); |
395 | |
396 | static void lookahead_bufs(struct tty_port *port, struct tty_buffer *head) |
397 | { |
398 | head->lookahead = max(head->lookahead, head->read); |
399 | |
400 | while (head) { |
401 | struct tty_buffer *next; |
402 | unsigned int count; |
403 | |
404 | /* |
405 | * Paired w/ release in __tty_buffer_request_room(); |
406 | * ensures commit value read is not stale if the head |
407 | * is advancing to the next buffer. |
408 | */ |
409 | next = smp_load_acquire(&head->next); |
410 | /* |
411 | * Paired w/ release in __tty_buffer_request_room() or in |
412 | * tty_buffer_flush(); ensures we see the committed buffer data. |
413 | */ |
414 | count = smp_load_acquire(&head->commit) - head->lookahead; |
415 | if (!count) { |
416 | head = next; |
417 | continue; |
418 | } |
419 | |
420 | if (port->client_ops->lookahead_buf) { |
421 | u8 *p, *f = NULL; |
422 | |
423 | p = char_buf_ptr(b: head, ofs: head->lookahead); |
424 | if (head->flags) |
425 | f = flag_buf_ptr(b: head, ofs: head->lookahead); |
426 | |
427 | port->client_ops->lookahead_buf(port, p, f, count); |
428 | } |
429 | |
430 | head->lookahead += count; |
431 | } |
432 | } |
433 | |
434 | static size_t |
435 | receive_buf(struct tty_port *port, struct tty_buffer *head, size_t count) |
436 | { |
437 | u8 *p = char_buf_ptr(b: head, ofs: head->read); |
438 | const u8 *f = NULL; |
439 | size_t n; |
440 | |
441 | if (head->flags) |
442 | f = flag_buf_ptr(b: head, ofs: head->read); |
443 | |
444 | n = port->client_ops->receive_buf(port, p, f, count); |
445 | if (n > 0) |
446 | memset(p, 0, n); |
447 | return n; |
448 | } |
449 | |
450 | /** |
451 | * flush_to_ldisc - flush data from buffer to ldisc |
452 | * @work: tty structure passed from work queue. |
453 | * |
454 | * This routine is called out of the software interrupt to flush data from the |
455 | * buffer chain to the line discipline. |
456 | * |
457 | * The receive_buf() method is single threaded for each tty instance. |
458 | * |
459 | * Locking: takes buffer lock to ensure single-threaded flip buffer 'consumer'. |
460 | */ |
461 | static void flush_to_ldisc(struct work_struct *work) |
462 | { |
463 | struct tty_port *port = container_of(work, struct tty_port, buf.work); |
464 | struct tty_bufhead *buf = &port->buf; |
465 | |
466 | mutex_lock(&buf->lock); |
467 | |
468 | while (1) { |
469 | struct tty_buffer *head = buf->head; |
470 | struct tty_buffer *next; |
471 | size_t count, rcvd; |
472 | |
473 | /* Ldisc or user is trying to gain exclusive access */ |
474 | if (atomic_read(v: &buf->priority)) |
475 | break; |
476 | |
477 | /* paired w/ release in __tty_buffer_request_room(); |
478 | * ensures commit value read is not stale if the head |
479 | * is advancing to the next buffer |
480 | */ |
481 | next = smp_load_acquire(&head->next); |
482 | /* paired w/ release in __tty_buffer_request_room() or in |
483 | * tty_buffer_flush(); ensures we see the committed buffer data |
484 | */ |
485 | count = smp_load_acquire(&head->commit) - head->read; |
486 | if (!count) { |
487 | if (next == NULL) |
488 | break; |
489 | buf->head = next; |
490 | tty_buffer_free(port, b: head); |
491 | continue; |
492 | } |
493 | |
494 | rcvd = receive_buf(port, head, count); |
495 | head->read += rcvd; |
496 | if (rcvd < count) |
497 | lookahead_bufs(port, head); |
498 | if (!rcvd) |
499 | break; |
500 | |
501 | if (need_resched()) |
502 | cond_resched(); |
503 | } |
504 | |
505 | mutex_unlock(lock: &buf->lock); |
506 | |
507 | } |
508 | |
509 | static inline void tty_flip_buffer_commit(struct tty_buffer *tail) |
510 | { |
511 | /* |
512 | * Paired w/ acquire in flush_to_ldisc(); ensures flush_to_ldisc() sees |
513 | * buffer data. |
514 | */ |
515 | smp_store_release(&tail->commit, tail->used); |
516 | } |
517 | |
518 | /** |
519 | * tty_flip_buffer_push - push terminal buffers |
520 | * @port: tty port to push |
521 | * |
522 | * Queue a push of the terminal flip buffers to the line discipline. Can be |
523 | * called from IRQ/atomic context. |
524 | * |
525 | * In the event of the queue being busy for flipping the work will be held off |
526 | * and retried later. |
527 | */ |
528 | void tty_flip_buffer_push(struct tty_port *port) |
529 | { |
530 | struct tty_bufhead *buf = &port->buf; |
531 | |
532 | tty_flip_buffer_commit(tail: buf->tail); |
533 | queue_work(wq: system_unbound_wq, work: &buf->work); |
534 | } |
535 | EXPORT_SYMBOL(tty_flip_buffer_push); |
536 | |
537 | /** |
538 | * tty_insert_flip_string_and_push_buffer - add characters to the tty buffer and |
539 | * push |
540 | * @port: tty port |
541 | * @chars: characters |
542 | * @size: size |
543 | * |
544 | * The function combines tty_insert_flip_string() and tty_flip_buffer_push() |
545 | * with the exception of properly holding the @port->lock. |
546 | * |
547 | * To be used only internally (by pty currently). |
548 | * |
549 | * Returns: the number added. |
550 | */ |
551 | int tty_insert_flip_string_and_push_buffer(struct tty_port *port, |
552 | const u8 *chars, size_t size) |
553 | { |
554 | struct tty_bufhead *buf = &port->buf; |
555 | unsigned long flags; |
556 | |
557 | spin_lock_irqsave(&port->lock, flags); |
558 | size = tty_insert_flip_string(port, chars, size); |
559 | if (size) |
560 | tty_flip_buffer_commit(tail: buf->tail); |
561 | spin_unlock_irqrestore(lock: &port->lock, flags); |
562 | |
563 | queue_work(wq: system_unbound_wq, work: &buf->work); |
564 | |
565 | return size; |
566 | } |
567 | |
568 | /** |
569 | * tty_buffer_init - prepare a tty buffer structure |
570 | * @port: tty port to initialise |
571 | * |
572 | * Set up the initial state of the buffer management for a tty device. Must be |
573 | * called before the other tty buffer functions are used. |
574 | */ |
575 | void tty_buffer_init(struct tty_port *port) |
576 | { |
577 | struct tty_bufhead *buf = &port->buf; |
578 | |
579 | mutex_init(&buf->lock); |
580 | tty_buffer_reset(p: &buf->sentinel, size: 0); |
581 | buf->head = &buf->sentinel; |
582 | buf->tail = &buf->sentinel; |
583 | init_llist_head(list: &buf->free); |
584 | atomic_set(v: &buf->mem_used, i: 0); |
585 | atomic_set(v: &buf->priority, i: 0); |
586 | INIT_WORK(&buf->work, flush_to_ldisc); |
587 | buf->mem_limit = TTYB_DEFAULT_MEM_LIMIT; |
588 | } |
589 | |
590 | /** |
591 | * tty_buffer_set_limit - change the tty buffer memory limit |
592 | * @port: tty port to change |
593 | * @limit: memory limit to set |
594 | * |
595 | * Change the tty buffer memory limit. |
596 | * |
597 | * Must be called before the other tty buffer functions are used. |
598 | */ |
599 | int tty_buffer_set_limit(struct tty_port *port, int limit) |
600 | { |
601 | if (limit < MIN_TTYB_SIZE) |
602 | return -EINVAL; |
603 | port->buf.mem_limit = limit; |
604 | return 0; |
605 | } |
606 | EXPORT_SYMBOL_GPL(tty_buffer_set_limit); |
607 | |
608 | /* slave ptys can claim nested buffer lock when handling BRK and INTR */ |
609 | void tty_buffer_set_lock_subclass(struct tty_port *port) |
610 | { |
611 | lockdep_set_subclass(&port->buf.lock, TTY_LOCK_SLAVE); |
612 | } |
613 | |
614 | bool tty_buffer_restart_work(struct tty_port *port) |
615 | { |
616 | return queue_work(wq: system_unbound_wq, work: &port->buf.work); |
617 | } |
618 | |
619 | bool tty_buffer_cancel_work(struct tty_port *port) |
620 | { |
621 | return cancel_work_sync(work: &port->buf.work); |
622 | } |
623 | |
624 | void tty_buffer_flush_work(struct tty_port *port) |
625 | { |
626 | flush_work(work: &port->buf.work); |
627 | } |
628 | |