1// SPDX-License-Identifier: GPL-2.0 OR MIT
2/**************************************************************************
3 *
4 * Copyright 2017 VMware, Inc., Palo Alto, CA., USA
5 * All Rights Reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
17 * of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
22 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
23 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
24 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
25 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 *
27 **************************************************************************/
28
29#include "vmwgfx_drv.h"
30#include <linux/highmem.h>
31
32/*
33 * Template that implements find_first_diff() for a generic
34 * unsigned integer type. @size and return value are in bytes.
35 */
36#define VMW_FIND_FIRST_DIFF(_type) \
37static size_t vmw_find_first_diff_ ## _type \
38 (const _type * dst, const _type * src, size_t size)\
39{ \
40 size_t i; \
41 \
42 for (i = 0; i < size; i += sizeof(_type)) { \
43 if (*dst++ != *src++) \
44 break; \
45 } \
46 \
47 return i; \
48}
49
50
51/*
52 * Template that implements find_last_diff() for a generic
53 * unsigned integer type. Pointers point to the item following the
54 * *end* of the area to be examined. @size and return value are in
55 * bytes.
56 */
57#define VMW_FIND_LAST_DIFF(_type) \
58static ssize_t vmw_find_last_diff_ ## _type( \
59 const _type * dst, const _type * src, size_t size) \
60{ \
61 while (size) { \
62 if (*--dst != *--src) \
63 break; \
64 \
65 size -= sizeof(_type); \
66 } \
67 return size; \
68}
69
70
71/*
72 * Instantiate find diff functions for relevant unsigned integer sizes,
73 * assuming that wider integers are faster (including aligning) up to the
74 * architecture native width, which is assumed to be 32 bit unless
75 * CONFIG_64BIT is defined.
76 */
77VMW_FIND_FIRST_DIFF(u8);
78VMW_FIND_LAST_DIFF(u8);
79
80VMW_FIND_FIRST_DIFF(u16);
81VMW_FIND_LAST_DIFF(u16);
82
83VMW_FIND_FIRST_DIFF(u32);
84VMW_FIND_LAST_DIFF(u32);
85
86#ifdef CONFIG_64BIT
87VMW_FIND_FIRST_DIFF(u64);
88VMW_FIND_LAST_DIFF(u64);
89#endif
90
91
92/* We use size aligned copies. This computes (addr - align(addr)) */
93#define SPILL(_var, _type) ((unsigned long) _var & (sizeof(_type) - 1))
94
95
96/*
97 * Template to compute find_first_diff() for a certain integer type
98 * including a head copy for alignment, and adjustment of parameters
99 * for tail find or increased resolution find using an unsigned integer find
100 * of smaller width. If finding is complete, and resolution is sufficient,
101 * the macro executes a return statement. Otherwise it falls through.
102 */
103#define VMW_TRY_FIND_FIRST_DIFF(_type) \
104do { \
105 unsigned int spill = SPILL(dst, _type); \
106 size_t diff_offs; \
107 \
108 if (spill && spill == SPILL(src, _type) && \
109 sizeof(_type) - spill <= size) { \
110 spill = sizeof(_type) - spill; \
111 diff_offs = vmw_find_first_diff_u8(dst, src, spill); \
112 if (diff_offs < spill) \
113 return round_down(offset + diff_offs, granularity); \
114 \
115 dst += spill; \
116 src += spill; \
117 size -= spill; \
118 offset += spill; \
119 spill = 0; \
120 } \
121 if (!spill && !SPILL(src, _type)) { \
122 size_t to_copy = size & ~(sizeof(_type) - 1); \
123 \
124 diff_offs = vmw_find_first_diff_ ## _type \
125 ((_type *) dst, (_type *) src, to_copy); \
126 if (diff_offs >= size || granularity == sizeof(_type)) \
127 return (offset + diff_offs); \
128 \
129 dst += diff_offs; \
130 src += diff_offs; \
131 size -= diff_offs; \
132 offset += diff_offs; \
133 } \
134} while (0) \
135
136
137/**
138 * vmw_find_first_diff - find the first difference between dst and src
139 *
140 * @dst: The destination address
141 * @src: The source address
142 * @size: Number of bytes to compare
143 * @granularity: The granularity needed for the return value in bytes.
144 * return: The offset from find start where the first difference was
145 * encountered in bytes. If no difference was found, the function returns
146 * a value >= @size.
147 */
148static size_t vmw_find_first_diff(const u8 *dst, const u8 *src, size_t size,
149 size_t granularity)
150{
151 size_t offset = 0;
152
153 /*
154 * Try finding with large integers if alignment allows, or we can
155 * fix it. Fall through if we need better resolution or alignment
156 * was bad.
157 */
158#ifdef CONFIG_64BIT
159 VMW_TRY_FIND_FIRST_DIFF(u64);
160#endif
161 VMW_TRY_FIND_FIRST_DIFF(u32);
162 VMW_TRY_FIND_FIRST_DIFF(u16);
163
164 return round_down(offset + vmw_find_first_diff_u8(dst, src, size),
165 granularity);
166}
167
168
169/*
170 * Template to compute find_last_diff() for a certain integer type
171 * including a tail copy for alignment, and adjustment of parameters
172 * for head find or increased resolution find using an unsigned integer find
173 * of smaller width. If finding is complete, and resolution is sufficient,
174 * the macro executes a return statement. Otherwise it falls through.
175 */
176#define VMW_TRY_FIND_LAST_DIFF(_type) \
177do { \
178 unsigned int spill = SPILL(dst, _type); \
179 ssize_t location; \
180 ssize_t diff_offs; \
181 \
182 if (spill && spill <= size && spill == SPILL(src, _type)) { \
183 diff_offs = vmw_find_last_diff_u8(dst, src, spill); \
184 if (diff_offs) { \
185 location = size - spill + diff_offs - 1; \
186 return round_down(location, granularity); \
187 } \
188 \
189 dst -= spill; \
190 src -= spill; \
191 size -= spill; \
192 spill = 0; \
193 } \
194 if (!spill && !SPILL(src, _type)) { \
195 size_t to_copy = round_down(size, sizeof(_type)); \
196 \
197 diff_offs = vmw_find_last_diff_ ## _type \
198 ((_type *) dst, (_type *) src, to_copy); \
199 location = size - to_copy + diff_offs - sizeof(_type); \
200 if (location < 0 || granularity == sizeof(_type)) \
201 return location; \
202 \
203 dst -= to_copy - diff_offs; \
204 src -= to_copy - diff_offs; \
205 size -= to_copy - diff_offs; \
206 } \
207} while (0)
208
209
210/**
211 * vmw_find_last_diff - find the last difference between dst and src
212 *
213 * @dst: The destination address
214 * @src: The source address
215 * @size: Number of bytes to compare
216 * @granularity: The granularity needed for the return value in bytes.
217 * return: The offset from find start where the last difference was
218 * encountered in bytes, or a negative value if no difference was found.
219 */
220static ssize_t vmw_find_last_diff(const u8 *dst, const u8 *src, size_t size,
221 size_t granularity)
222{
223 dst += size;
224 src += size;
225
226#ifdef CONFIG_64BIT
227 VMW_TRY_FIND_LAST_DIFF(u64);
228#endif
229 VMW_TRY_FIND_LAST_DIFF(u32);
230 VMW_TRY_FIND_LAST_DIFF(u16);
231
232 return round_down(vmw_find_last_diff_u8(dst, src, size) - 1,
233 granularity);
234}
235
236
237/**
238 * vmw_memcpy - A wrapper around kernel memcpy with allowing to plug it into a
239 * struct vmw_diff_cpy.
240 *
241 * @diff: The struct vmw_diff_cpy closure argument (unused).
242 * @dest: The copy destination.
243 * @src: The copy source.
244 * @n: Number of bytes to copy.
245 */
246void vmw_memcpy(struct vmw_diff_cpy *diff, u8 *dest, const u8 *src, size_t n)
247{
248 memcpy(dest, src, n);
249}
250
251
252/**
253 * vmw_adjust_rect - Adjust rectangle coordinates for newly found difference
254 *
255 * @diff: The struct vmw_diff_cpy used to track the modified bounding box.
256 * @diff_offs: The offset from @diff->line_offset where the difference was
257 * found.
258 */
259static void vmw_adjust_rect(struct vmw_diff_cpy *diff, size_t diff_offs)
260{
261 size_t offs = (diff_offs + diff->line_offset) / diff->cpp;
262 struct drm_rect *rect = &diff->rect;
263
264 rect->x1 = min_t(int, rect->x1, offs);
265 rect->x2 = max_t(int, rect->x2, offs + 1);
266 rect->y1 = min_t(int, rect->y1, diff->line);
267 rect->y2 = max_t(int, rect->y2, diff->line + 1);
268}
269
270/**
271 * vmw_diff_memcpy - memcpy that creates a bounding box of modified content.
272 *
273 * @diff: The struct vmw_diff_cpy used to track the modified bounding box.
274 * @dest: The copy destination.
275 * @src: The copy source.
276 * @n: Number of bytes to copy.
277 *
278 * In order to correctly track the modified content, the field @diff->line must
279 * be pre-loaded with the current line number, the field @diff->line_offset must
280 * be pre-loaded with the line offset in bytes where the copy starts, and
281 * finally the field @diff->cpp need to be preloaded with the number of bytes
282 * per unit in the horizontal direction of the area we're examining.
283 * Typically bytes per pixel.
284 * This is needed to know the needed granularity of the difference computing
285 * operations. A higher cpp generally leads to faster execution at the cost of
286 * bounding box width precision.
287 */
288void vmw_diff_memcpy(struct vmw_diff_cpy *diff, u8 *dest, const u8 *src,
289 size_t n)
290{
291 ssize_t csize, byte_len;
292
293 if (WARN_ON_ONCE(round_down(n, diff->cpp) != n))
294 return;
295
296 /* TODO: Possibly use a single vmw_find_first_diff per line? */
297 csize = vmw_find_first_diff(dst: dest, src, size: n, granularity: diff->cpp);
298 if (csize < n) {
299 vmw_adjust_rect(diff, diff_offs: csize);
300 byte_len = diff->cpp;
301
302 /*
303 * Starting from where first difference was found, find
304 * location of last difference, and then copy.
305 */
306 diff->line_offset += csize;
307 dest += csize;
308 src += csize;
309 n -= csize;
310 csize = vmw_find_last_diff(dst: dest, src, size: n, granularity: diff->cpp);
311 if (csize >= 0) {
312 byte_len += csize;
313 vmw_adjust_rect(diff, diff_offs: csize);
314 }
315 memcpy(dest, src, byte_len);
316 }
317 diff->line_offset += n;
318}
319
320/**
321 * struct vmw_bo_blit_line_data - Convenience argument to vmw_bo_cpu_blit_line
322 *
323 * @mapped_dst: Already mapped destination page index in @dst_pages.
324 * @dst_addr: Kernel virtual address of mapped destination page.
325 * @dst_pages: Array of destination bo pages.
326 * @dst_num_pages: Number of destination bo pages.
327 * @dst_prot: Destination bo page protection.
328 * @mapped_src: Already mapped source page index in @dst_pages.
329 * @src_addr: Kernel virtual address of mapped source page.
330 * @src_pages: Array of source bo pages.
331 * @src_num_pages: Number of source bo pages.
332 * @src_prot: Source bo page protection.
333 * @diff: Struct vmw_diff_cpy, in the end forwarded to the memcpy routine.
334 */
335struct vmw_bo_blit_line_data {
336 u32 mapped_dst;
337 u8 *dst_addr;
338 struct page **dst_pages;
339 u32 dst_num_pages;
340 pgprot_t dst_prot;
341 u32 mapped_src;
342 u8 *src_addr;
343 struct page **src_pages;
344 u32 src_num_pages;
345 pgprot_t src_prot;
346 struct vmw_diff_cpy *diff;
347};
348
349/**
350 * vmw_bo_cpu_blit_line - Blit part of a line from one bo to another.
351 *
352 * @d: Blit data as described above.
353 * @dst_offset: Destination copy start offset from start of bo.
354 * @src_offset: Source copy start offset from start of bo.
355 * @bytes_to_copy: Number of bytes to copy in this line.
356 */
357static int vmw_bo_cpu_blit_line(struct vmw_bo_blit_line_data *d,
358 u32 dst_offset,
359 u32 src_offset,
360 u32 bytes_to_copy)
361{
362 struct vmw_diff_cpy *diff = d->diff;
363
364 while (bytes_to_copy) {
365 u32 copy_size = bytes_to_copy;
366 u32 dst_page = dst_offset >> PAGE_SHIFT;
367 u32 src_page = src_offset >> PAGE_SHIFT;
368 u32 dst_page_offset = dst_offset & ~PAGE_MASK;
369 u32 src_page_offset = src_offset & ~PAGE_MASK;
370 bool unmap_dst = d->dst_addr && dst_page != d->mapped_dst;
371 bool unmap_src = d->src_addr && (src_page != d->mapped_src ||
372 unmap_dst);
373
374 copy_size = min_t(u32, copy_size, PAGE_SIZE - dst_page_offset);
375 copy_size = min_t(u32, copy_size, PAGE_SIZE - src_page_offset);
376
377 if (unmap_src) {
378 kunmap_atomic(d->src_addr);
379 d->src_addr = NULL;
380 }
381
382 if (unmap_dst) {
383 kunmap_atomic(d->dst_addr);
384 d->dst_addr = NULL;
385 }
386
387 if (!d->dst_addr) {
388 if (WARN_ON_ONCE(dst_page >= d->dst_num_pages))
389 return -EINVAL;
390
391 d->dst_addr =
392 kmap_atomic_prot(page: d->dst_pages[dst_page],
393 prot: d->dst_prot);
394 if (!d->dst_addr)
395 return -ENOMEM;
396
397 d->mapped_dst = dst_page;
398 }
399
400 if (!d->src_addr) {
401 if (WARN_ON_ONCE(src_page >= d->src_num_pages))
402 return -EINVAL;
403
404 d->src_addr =
405 kmap_atomic_prot(page: d->src_pages[src_page],
406 prot: d->src_prot);
407 if (!d->src_addr)
408 return -ENOMEM;
409
410 d->mapped_src = src_page;
411 }
412 diff->do_cpy(diff, d->dst_addr + dst_page_offset,
413 d->src_addr + src_page_offset, copy_size);
414
415 bytes_to_copy -= copy_size;
416 dst_offset += copy_size;
417 src_offset += copy_size;
418 }
419
420 return 0;
421}
422
423/**
424 * vmw_bo_cpu_blit - in-kernel cpu blit.
425 *
426 * @dst: Destination buffer object.
427 * @dst_offset: Destination offset of blit start in bytes.
428 * @dst_stride: Destination stride in bytes.
429 * @src: Source buffer object.
430 * @src_offset: Source offset of blit start in bytes.
431 * @src_stride: Source stride in bytes.
432 * @w: Width of blit.
433 * @h: Height of blit.
434 * @diff: The struct vmw_diff_cpy used to track the modified bounding box.
435 * return: Zero on success. Negative error value on failure. Will print out
436 * kernel warnings on caller bugs.
437 *
438 * Performs a CPU blit from one buffer object to another avoiding a full
439 * bo vmap which may exhaust- or fragment vmalloc space.
440 * On supported architectures (x86), we're using kmap_atomic which avoids
441 * cross-processor TLB- and cache flushes and may, on non-HIGHMEM systems
442 * reference already set-up mappings.
443 *
444 * Neither of the buffer objects may be placed in PCI memory
445 * (Fixed memory in TTM terminology) when using this function.
446 */
447int vmw_bo_cpu_blit(struct ttm_buffer_object *dst,
448 u32 dst_offset, u32 dst_stride,
449 struct ttm_buffer_object *src,
450 u32 src_offset, u32 src_stride,
451 u32 w, u32 h,
452 struct vmw_diff_cpy *diff)
453{
454 struct ttm_operation_ctx ctx = {
455 .interruptible = false,
456 .no_wait_gpu = false
457 };
458 u32 j, initial_line = dst_offset / dst_stride;
459 struct vmw_bo_blit_line_data d = {0};
460 int ret = 0;
461 struct page **dst_pages = NULL;
462 struct page **src_pages = NULL;
463
464 /* Buffer objects need to be either pinned or reserved: */
465 if (!(dst->pin_count))
466 dma_resv_assert_held(dst->base.resv);
467 if (!(src->pin_count))
468 dma_resv_assert_held(src->base.resv);
469
470 if (!ttm_tt_is_populated(tt: dst->ttm)) {
471 ret = dst->bdev->funcs->ttm_tt_populate(dst->bdev, dst->ttm, &ctx);
472 if (ret)
473 return ret;
474 }
475
476 if (!ttm_tt_is_populated(tt: src->ttm)) {
477 ret = src->bdev->funcs->ttm_tt_populate(src->bdev, src->ttm, &ctx);
478 if (ret)
479 return ret;
480 }
481
482 if (!src->ttm->pages && src->ttm->sg) {
483 src_pages = kvmalloc_array(n: src->ttm->num_pages,
484 size: sizeof(struct page *), GFP_KERNEL);
485 if (!src_pages)
486 return -ENOMEM;
487 ret = drm_prime_sg_to_page_array(sgt: src->ttm->sg, pages: src_pages,
488 max_pages: src->ttm->num_pages);
489 if (ret)
490 goto out;
491 }
492 if (!dst->ttm->pages && dst->ttm->sg) {
493 dst_pages = kvmalloc_array(n: dst->ttm->num_pages,
494 size: sizeof(struct page *), GFP_KERNEL);
495 if (!dst_pages) {
496 ret = -ENOMEM;
497 goto out;
498 }
499 ret = drm_prime_sg_to_page_array(sgt: dst->ttm->sg, pages: dst_pages,
500 max_pages: dst->ttm->num_pages);
501 if (ret)
502 goto out;
503 }
504
505 d.mapped_dst = 0;
506 d.mapped_src = 0;
507 d.dst_addr = NULL;
508 d.src_addr = NULL;
509 d.dst_pages = dst->ttm->pages ? dst->ttm->pages : dst_pages;
510 d.src_pages = src->ttm->pages ? src->ttm->pages : src_pages;
511 d.dst_num_pages = PFN_UP(dst->resource->size);
512 d.src_num_pages = PFN_UP(src->resource->size);
513 d.dst_prot = ttm_io_prot(bo: dst, res: dst->resource, PAGE_KERNEL);
514 d.src_prot = ttm_io_prot(bo: src, res: src->resource, PAGE_KERNEL);
515 d.diff = diff;
516
517 for (j = 0; j < h; ++j) {
518 diff->line = j + initial_line;
519 diff->line_offset = dst_offset % dst_stride;
520 ret = vmw_bo_cpu_blit_line(d: &d, dst_offset, src_offset, bytes_to_copy: w);
521 if (ret)
522 goto out;
523
524 dst_offset += dst_stride;
525 src_offset += src_stride;
526 }
527out:
528 if (d.src_addr)
529 kunmap_atomic(d.src_addr);
530 if (d.dst_addr)
531 kunmap_atomic(d.dst_addr);
532 if (src_pages)
533 kvfree(addr: src_pages);
534 if (dst_pages)
535 kvfree(addr: dst_pages);
536
537 return ret;
538}
539

source code of linux/drivers/gpu/drm/vmwgfx/vmwgfx_blit.c