1	/* strcpy with SSSE3
2	Copyright (C) 2011-2022 Free Software Foundation, Inc.
3	This file is part of the GNU C Library.
4
5	The GNU C Library is free software; you can redistribute it and/or
6	modify it under the terms of the GNU Lesser General Public
7	License as published by the Free Software Foundation; either
8	version 2.1 of the License, or (at your option) any later version.
9
10	The GNU C Library is distributed in the hope that it will be useful,
11	but WITHOUT ANY WARRANTY; without even the implied warranty of
12	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13	Lesser General Public License for more details.
14
15	You should have received a copy of the GNU Lesser General Public
16	License along with the GNU C Library; if not, see
17	<https://www.gnu.org/licenses/>. */
18
19	#if IS_IN (libc)
20
21	# ifndef USE_AS_STRCAT
22	# include <sysdep.h>
23
24	# ifndef STRCPY
25	# define STRCPY __strcpy_ssse3
26	# endif
27
28	.section .text.ssse3,"ax",@progbits
29	ENTRY (STRCPY)
30
31	mov %rsi, %rcx
32	# ifdef USE_AS_STRNCPY
33	mov %RDX_LP, %R8_LP
34	# endif
35	mov %rdi, %rdx
36	# ifdef USE_AS_STRNCPY
37	test %R8_LP, %R8_LP
38	jz L(Exit0)
39	cmp $8, %R8_LP
40	jbe L(StrncpyExit8Bytes)
41	# endif
42	cmpb $0, (%rcx)
43	jz L(Exit1)
44	cmpb $0, 1(%rcx)
45	jz L(Exit2)
46	cmpb $0, 2(%rcx)
47	jz L(Exit3)
48	cmpb $0, 3(%rcx)
49	jz L(Exit4)
50	cmpb $0, 4(%rcx)
51	jz L(Exit5)
52	cmpb $0, 5(%rcx)
53	jz L(Exit6)
54	cmpb $0, 6(%rcx)
55	jz L(Exit7)
56	cmpb $0, 7(%rcx)
57	jz L(Exit8)
58	# ifdef USE_AS_STRNCPY
59	cmp $16, %r8
60	jb L(StrncpyExit15Bytes)
61	# endif
62	cmpb $0, 8(%rcx)
63	jz L(Exit9)
64	cmpb $0, 9(%rcx)
65	jz L(Exit10)
66	cmpb $0, 10(%rcx)
67	jz L(Exit11)
68	cmpb $0, 11(%rcx)
69	jz L(Exit12)
70	cmpb $0, 12(%rcx)
71	jz L(Exit13)
72	cmpb $0, 13(%rcx)
73	jz L(Exit14)
74	cmpb $0, 14(%rcx)
75	jz L(Exit15)
76	# ifdef USE_AS_STRNCPY
77	cmp $16, %r8
78	je L(Exit16)
79	# endif
80	cmpb $0, 15(%rcx)
81	jz L(Exit16)
82	# endif
83
84	# ifdef USE_AS_STRNCPY
85	mov %rcx, %rsi
86	sub $16, %r8
87	and $0xf, %rsi
88
89	/* add 16 bytes rcx_offset to r8 */
90
91	add %rsi, %r8
92	# endif
93	lea 16(%rcx), %rsi
94	and $-16, %rsi
95	pxor %xmm0, %xmm0
96	mov (%rcx), %r9
97	mov %r9, (%rdx)
98	pcmpeqb (%rsi), %xmm0
99	mov 8(%rcx), %r9
100	mov %r9, 8(%rdx)
101
102	/* convert byte mask in xmm0 to bit mask */
103
104	pmovmskb %xmm0, %rax
105	sub %rcx, %rsi
106
107	# ifdef USE_AS_STRNCPY
108	sub $16, %r8
109	jbe L(CopyFrom1To16BytesCase2OrCase3)
110	# endif
111	test %rax, %rax
112	jnz L(CopyFrom1To16Bytes)
113
114	mov %rdx, %rax
115	lea 16(%rdx), %rdx
116	and $-16, %rdx
117	sub %rdx, %rax
118
119	# ifdef USE_AS_STRNCPY
120	add %rax, %rsi
121	lea -1(%rsi), %rsi
122	and $1<<31, %esi
123	test %rsi, %rsi
124	jnz L(ContinueCopy)
125	lea 16(%r8), %r8
126
127	L(ContinueCopy):
128	# endif
129	sub %rax, %rcx
130	mov %rcx, %rax
131	and $0xf, %rax
132	mov $0, %rsi
133
134	/* case: rcx_offset == rdx_offset */
135
136	jz L(Align16Both)
137
138	cmp $8, %rax
139	jae L(ShlHigh8)
140	cmp $1, %rax
141	je L(Shl1)
142	cmp $2, %rax
143	je L(Shl2)
144	cmp $3, %rax
145	je L(Shl3)
146	cmp $4, %rax
147	je L(Shl4)
148	cmp $5, %rax
149	je L(Shl5)
150	cmp $6, %rax
151	je L(Shl6)
152	jmp L(Shl7)
153
154	L(ShlHigh8):
155	je L(Shl8)
156	cmp $9, %rax
157	je L(Shl9)
158	cmp $10, %rax
159	je L(Shl10)
160	cmp $11, %rax
161	je L(Shl11)
162	cmp $12, %rax
163	je L(Shl12)
164	cmp $13, %rax
165	je L(Shl13)
166	cmp $14, %rax
167	je L(Shl14)
168	jmp L(Shl15)
169
170	L(Align16Both):
171	movaps (%rcx), %xmm1
172	movaps 16(%rcx), %xmm2
173	movaps %xmm1, (%rdx)
174	pcmpeqb %xmm2, %xmm0
175	pmovmskb %xmm0, %rax
176	lea 16(%rsi), %rsi
177	# ifdef USE_AS_STRNCPY
178	sub $16, %r8
179	jbe L(CopyFrom1To16BytesCase2OrCase3)
180	# endif
181	test %rax, %rax
182	jnz L(CopyFrom1To16Bytes)
183
184	movaps 16(%rcx, %rsi), %xmm3
185	movaps %xmm2, (%rdx, %rsi)
186	pcmpeqb %xmm3, %xmm0
187	pmovmskb %xmm0, %rax
188	lea 16(%rsi), %rsi
189	# ifdef USE_AS_STRNCPY
190	sub $16, %r8
191	jbe L(CopyFrom1To16BytesCase2OrCase3)
192	# endif
193	test %rax, %rax
194	jnz L(CopyFrom1To16Bytes)
195
196	movaps 16(%rcx, %rsi), %xmm4
197	movaps %xmm3, (%rdx, %rsi)
198	pcmpeqb %xmm4, %xmm0
199	pmovmskb %xmm0, %rax
200	lea 16(%rsi), %rsi
201	# ifdef USE_AS_STRNCPY
202	sub $16, %r8
203	jbe L(CopyFrom1To16BytesCase2OrCase3)
204	# endif
205	test %rax, %rax
206	jnz L(CopyFrom1To16Bytes)
207
208	movaps 16(%rcx, %rsi), %xmm1
209	movaps %xmm4, (%rdx, %rsi)
210	pcmpeqb %xmm1, %xmm0
211	pmovmskb %xmm0, %rax
212	lea 16(%rsi), %rsi
213	# ifdef USE_AS_STRNCPY
214	sub $16, %r8
215	jbe L(CopyFrom1To16BytesCase2OrCase3)
216	# endif
217	test %rax, %rax
218	jnz L(CopyFrom1To16Bytes)
219
220	movaps 16(%rcx, %rsi), %xmm2
221	movaps %xmm1, (%rdx, %rsi)
222	pcmpeqb %xmm2, %xmm0
223	pmovmskb %xmm0, %rax
224	lea 16(%rsi), %rsi
225	# ifdef USE_AS_STRNCPY
226	sub $16, %r8
227	jbe L(CopyFrom1To16BytesCase2OrCase3)
228	# endif
229	test %rax, %rax
230	jnz L(CopyFrom1To16Bytes)
231
232	movaps 16(%rcx, %rsi), %xmm3
233	movaps %xmm2, (%rdx, %rsi)
234	pcmpeqb %xmm3, %xmm0
235	pmovmskb %xmm0, %rax
236	lea 16(%rsi), %rsi
237	# ifdef USE_AS_STRNCPY
238	sub $16, %r8
239	jbe L(CopyFrom1To16BytesCase2OrCase3)
240	# endif
241	test %rax, %rax
242	jnz L(CopyFrom1To16Bytes)
243
244	movaps %xmm3, (%rdx, %rsi)
245	mov %rcx, %rax
246	lea 16(%rcx, %rsi), %rcx
247	and $-0x40, %rcx
248	sub %rcx, %rax
249	sub %rax, %rdx
250	# ifdef USE_AS_STRNCPY
251	lea 112(%r8, %rax), %r8
252	# endif
253	mov $-0x40, %rsi
254
255	.p2align 4
256	L(Aligned64Loop):
257	movaps (%rcx), %xmm2
258	movaps %xmm2, %xmm4
259	movaps 16(%rcx), %xmm5
260	movaps 32(%rcx), %xmm3
261	movaps %xmm3, %xmm6
262	movaps 48(%rcx), %xmm7
263	pminub %xmm5, %xmm2
264	pminub %xmm7, %xmm3
265	pminub %xmm2, %xmm3
266	pcmpeqb %xmm0, %xmm3
267	pmovmskb %xmm3, %rax
268	lea 64(%rdx), %rdx
269	lea 64(%rcx), %rcx
270	# ifdef USE_AS_STRNCPY
271	sub $64, %r8
272	jbe L(StrncpyLeaveCase2OrCase3)
273	# endif
274	test %rax, %rax
275	jnz L(Aligned64Leave)
276	movaps %xmm4, -64(%rdx)
277	movaps %xmm5, -48(%rdx)
278	movaps %xmm6, -32(%rdx)
279	movaps %xmm7, -16(%rdx)
280	jmp L(Aligned64Loop)
281
282	L(Aligned64Leave):
283	# ifdef USE_AS_STRNCPY
284	lea 48(%r8), %r8
285	# endif
286	pcmpeqb %xmm4, %xmm0
287	pmovmskb %xmm0, %rax
288	test %rax, %rax
289	jnz L(CopyFrom1To16Bytes)
290
291	pcmpeqb %xmm5, %xmm0
292	# ifdef USE_AS_STRNCPY
293	lea -16(%r8), %r8
294	# endif
295	pmovmskb %xmm0, %rax
296	movaps %xmm4, -64(%rdx)
297	test %rax, %rax
298	lea 16(%rsi), %rsi
299	jnz L(CopyFrom1To16Bytes)
300
301	pcmpeqb %xmm6, %xmm0
302	# ifdef USE_AS_STRNCPY
303	lea -16(%r8), %r8
304	# endif
305	pmovmskb %xmm0, %rax
306	movaps %xmm5, -48(%rdx)
307	test %rax, %rax
308	lea 16(%rsi), %rsi
309	jnz L(CopyFrom1To16Bytes)
310
311	movaps %xmm6, -32(%rdx)
312	pcmpeqb %xmm7, %xmm0
313	# ifdef USE_AS_STRNCPY
314	lea -16(%r8), %r8
315	# endif
316	pmovmskb %xmm0, %rax
317	lea 16(%rsi), %rsi
318	jmp L(CopyFrom1To16Bytes)
319
320	.p2align 4
321	L(Shl1):
322	movaps -1(%rcx), %xmm1
323	movaps 15(%rcx), %xmm2
324	L(Shl1Start):
325	pcmpeqb %xmm2, %xmm0
326	pmovmskb %xmm0, %rax
327	movaps %xmm2, %xmm3
328	# ifdef USE_AS_STRNCPY
329	sub $16, %r8
330	jbe L(StrncpyExit1Case2OrCase3)
331	# endif
332	test %rax, %rax
333	jnz L(Shl1LoopExit)
334
335	palignr $1, %xmm1, %xmm2
336	movaps %xmm2, (%rdx)
337	movaps 31(%rcx), %xmm2
338
339	pcmpeqb %xmm2, %xmm0
340	lea 16(%rdx), %rdx
341	pmovmskb %xmm0, %rax
342	lea 16(%rcx), %rcx
343	movaps %xmm2, %xmm1
344	# ifdef USE_AS_STRNCPY
345	sub $16, %r8
346	jbe L(StrncpyExit1Case2OrCase3)
347	# endif
348	test %rax, %rax
349	jnz L(Shl1LoopExit)
350
351	palignr $1, %xmm3, %xmm2
352	movaps %xmm2, (%rdx)
353	movaps 31(%rcx), %xmm2
354
355	pcmpeqb %xmm2, %xmm0
356	lea 16(%rdx), %rdx
357	pmovmskb %xmm0, %rax
358	lea 16(%rcx), %rcx
359	movaps %xmm2, %xmm3
360	# ifdef USE_AS_STRNCPY
361	sub $16, %r8
362	jbe L(StrncpyExit1Case2OrCase3)
363	# endif
364	test %rax, %rax
365	jnz L(Shl1LoopExit)
366
367	palignr $1, %xmm1, %xmm2
368	movaps %xmm2, (%rdx)
369	movaps 31(%rcx), %xmm2
370
371	pcmpeqb %xmm2, %xmm0
372	lea 16(%rdx), %rdx
373	pmovmskb %xmm0, %rax
374	lea 16(%rcx), %rcx
375	# ifdef USE_AS_STRNCPY
376	sub $16, %r8
377	jbe L(StrncpyExit1Case2OrCase3)
378	# endif
379	test %rax, %rax
380	jnz L(Shl1LoopExit)
381
382	palignr $1, %xmm3, %xmm2
383	movaps %xmm2, (%rdx)
384	lea 31(%rcx), %rcx
385	lea 16(%rdx), %rdx
386
387	mov %rcx, %rax
388	and $-0x40, %rcx
389	sub %rcx, %rax
390	lea -15(%rcx), %rcx
391	sub %rax, %rdx
392	# ifdef USE_AS_STRNCPY
393	add %rax, %r8
394	# endif
395	movaps -1(%rcx), %xmm1
396
397	/* 64 bytes loop */
398	.p2align 4
399	L(Shl1LoopStart):
400	movaps 15(%rcx), %xmm2
401	movaps 31(%rcx), %xmm3
402	movaps %xmm3, %xmm6
403	movaps 47(%rcx), %xmm4
404	movaps %xmm4, %xmm7
405	movaps 63(%rcx), %xmm5
406	pminub %xmm2, %xmm6
407	pminub %xmm5, %xmm7
408	pminub %xmm6, %xmm7
409	pcmpeqb %xmm0, %xmm7
410	pmovmskb %xmm7, %rax
411	movaps %xmm5, %xmm7
412	palignr $1, %xmm4, %xmm5
413	test %rax, %rax
414	palignr $1, %xmm3, %xmm4
415	jnz L(Shl1Start)
416	# ifdef USE_AS_STRNCPY
417	sub $64, %r8
418	jbe L(StrncpyLeave1)
419	# endif
420	palignr $1, %xmm2, %xmm3
421	lea 64(%rcx), %rcx
422	palignr $1, %xmm1, %xmm2
423	movaps %xmm7, %xmm1
424	movaps %xmm5, 48(%rdx)
425	movaps %xmm4, 32(%rdx)
426	movaps %xmm3, 16(%rdx)
427	movaps %xmm2, (%rdx)
428	lea 64(%rdx), %rdx
429	jmp L(Shl1LoopStart)
430
431	L(Shl1LoopExit):
432	movdqu -1(%rcx), %xmm1
433	mov $15, %rsi
434	movdqu %xmm1, -1(%rdx)
435	jmp L(CopyFrom1To16Bytes)
436
437	.p2align 4
438	L(Shl2):
439	movaps -2(%rcx), %xmm1
440	movaps 14(%rcx), %xmm2
441	L(Shl2Start):
442	pcmpeqb %xmm2, %xmm0
443	pmovmskb %xmm0, %rax
444	movaps %xmm2, %xmm3
445	# ifdef USE_AS_STRNCPY
446	sub $16, %r8
447	jbe L(StrncpyExit2Case2OrCase3)
448	# endif
449	test %rax, %rax
450	jnz L(Shl2LoopExit)
451
452	palignr $2, %xmm1, %xmm2
453	movaps %xmm2, (%rdx)
454	movaps 30(%rcx), %xmm2
455
456	pcmpeqb %xmm2, %xmm0
457	lea 16(%rdx), %rdx
458	pmovmskb %xmm0, %rax
459	lea 16(%rcx), %rcx
460	movaps %xmm2, %xmm1
461	# ifdef USE_AS_STRNCPY
462	sub $16, %r8
463	jbe L(StrncpyExit2Case2OrCase3)
464	# endif
465	test %rax, %rax
466	jnz L(Shl2LoopExit)
467
468	palignr $2, %xmm3, %xmm2
469	movaps %xmm2, (%rdx)
470	movaps 30(%rcx), %xmm2
471
472	pcmpeqb %xmm2, %xmm0
473	lea 16(%rdx), %rdx
474	pmovmskb %xmm0, %rax
475	lea 16(%rcx), %rcx
476	movaps %xmm2, %xmm3
477	# ifdef USE_AS_STRNCPY
478	sub $16, %r8
479	jbe L(StrncpyExit2Case2OrCase3)
480	# endif
481	test %rax, %rax
482	jnz L(Shl2LoopExit)
483
484	palignr $2, %xmm1, %xmm2
485	movaps %xmm2, (%rdx)
486	movaps 30(%rcx), %xmm2
487
488	pcmpeqb %xmm2, %xmm0
489	lea 16(%rdx), %rdx
490	pmovmskb %xmm0, %rax
491	lea 16(%rcx), %rcx
492	# ifdef USE_AS_STRNCPY
493	sub $16, %r8
494	jbe L(StrncpyExit2Case2OrCase3)
495	# endif
496	test %rax, %rax
497	jnz L(Shl2LoopExit)
498
499	palignr $2, %xmm3, %xmm2
500	movaps %xmm2, (%rdx)
501	lea 30(%rcx), %rcx
502	lea 16(%rdx), %rdx
503
504	mov %rcx, %rax
505	and $-0x40, %rcx
506	sub %rcx, %rax
507	lea -14(%rcx), %rcx
508	sub %rax, %rdx
509	# ifdef USE_AS_STRNCPY
510	add %rax, %r8
511	# endif
512	movaps -2(%rcx), %xmm1
513
514	/* 64 bytes loop */
515	.p2align 4
516	L(Shl2LoopStart):
517	movaps 14(%rcx), %xmm2
518	movaps 30(%rcx), %xmm3
519	movaps %xmm3, %xmm6
520	movaps 46(%rcx), %xmm4
521	movaps %xmm4, %xmm7
522	movaps 62(%rcx), %xmm5
523	pminub %xmm2, %xmm6
524	pminub %xmm5, %xmm7
525	pminub %xmm6, %xmm7
526	pcmpeqb %xmm0, %xmm7
527	pmovmskb %xmm7, %rax
528	movaps %xmm5, %xmm7
529	palignr $2, %xmm4, %xmm5
530	test %rax, %rax
531	palignr $2, %xmm3, %xmm4
532	jnz L(Shl2Start)
533	# ifdef USE_AS_STRNCPY
534	sub $64, %r8
535	jbe L(StrncpyLeave2)
536	# endif
537	palignr $2, %xmm2, %xmm3
538	lea 64(%rcx), %rcx
539	palignr $2, %xmm1, %xmm2
540	movaps %xmm7, %xmm1
541	movaps %xmm5, 48(%rdx)
542	movaps %xmm4, 32(%rdx)
543	movaps %xmm3, 16(%rdx)
544	movaps %xmm2, (%rdx)
545	lea 64(%rdx), %rdx
546	jmp L(Shl2LoopStart)
547
548	L(Shl2LoopExit):
549	movdqu -2(%rcx), %xmm1
550	mov $14, %rsi
551	movdqu %xmm1, -2(%rdx)
552	jmp L(CopyFrom1To16Bytes)
553
554	.p2align 4
555	L(Shl3):
556	movaps -3(%rcx), %xmm1
557	movaps 13(%rcx), %xmm2
558	L(Shl3Start):
559	pcmpeqb %xmm2, %xmm0
560	pmovmskb %xmm0, %rax
561	movaps %xmm2, %xmm3
562	# ifdef USE_AS_STRNCPY
563	sub $16, %r8
564	jbe L(StrncpyExit3Case2OrCase3)
565	# endif
566	test %rax, %rax
567	jnz L(Shl3LoopExit)
568
569	palignr $3, %xmm1, %xmm2
570	movaps %xmm2, (%rdx)
571	movaps 29(%rcx), %xmm2
572
573	pcmpeqb %xmm2, %xmm0
574	lea 16(%rdx), %rdx
575	pmovmskb %xmm0, %rax
576	lea 16(%rcx), %rcx
577	movaps %xmm2, %xmm1
578	# ifdef USE_AS_STRNCPY
579	sub $16, %r8
580	jbe L(StrncpyExit3Case2OrCase3)
581	# endif
582	test %rax, %rax
583	jnz L(Shl3LoopExit)
584
585	palignr $3, %xmm3, %xmm2
586	movaps %xmm2, (%rdx)
587	movaps 29(%rcx), %xmm2
588
589	pcmpeqb %xmm2, %xmm0
590	lea 16(%rdx), %rdx
591	pmovmskb %xmm0, %rax
592	lea 16(%rcx), %rcx
593	movaps %xmm2, %xmm3
594	# ifdef USE_AS_STRNCPY
595	sub $16, %r8
596	jbe L(StrncpyExit3Case2OrCase3)
597	# endif
598	test %rax, %rax
599	jnz L(Shl3LoopExit)
600
601	palignr $3, %xmm1, %xmm2
602	movaps %xmm2, (%rdx)
603	movaps 29(%rcx), %xmm2
604
605	pcmpeqb %xmm2, %xmm0
606	lea 16(%rdx), %rdx
607	pmovmskb %xmm0, %rax
608	lea 16(%rcx), %rcx
609	# ifdef USE_AS_STRNCPY
610	sub $16, %r8
611	jbe L(StrncpyExit3Case2OrCase3)
612	# endif
613	test %rax, %rax
614	jnz L(Shl3LoopExit)
615
616	palignr $3, %xmm3, %xmm2
617	movaps %xmm2, (%rdx)
618	lea 29(%rcx), %rcx
619	lea 16(%rdx), %rdx
620
621	mov %rcx, %rax
622	and $-0x40, %rcx
623	sub %rcx, %rax
624	lea -13(%rcx), %rcx
625	sub %rax, %rdx
626	# ifdef USE_AS_STRNCPY
627	add %rax, %r8
628	# endif
629	movaps -3(%rcx), %xmm1
630
631	/* 64 bytes loop */
632	.p2align 4
633	L(Shl3LoopStart):
634	movaps 13(%rcx), %xmm2
635	movaps 29(%rcx), %xmm3
636	movaps %xmm3, %xmm6
637	movaps 45(%rcx), %xmm4
638	movaps %xmm4, %xmm7
639	movaps 61(%rcx), %xmm5
640	pminub %xmm2, %xmm6
641	pminub %xmm5, %xmm7
642	pminub %xmm6, %xmm7
643	pcmpeqb %xmm0, %xmm7
644	pmovmskb %xmm7, %rax
645	movaps %xmm5, %xmm7
646	palignr $3, %xmm4, %xmm5
647	test %rax, %rax
648	palignr $3, %xmm3, %xmm4
649	jnz L(Shl3Start)
650	# ifdef USE_AS_STRNCPY
651	sub $64, %r8
652	jbe L(StrncpyLeave3)
653	# endif
654	palignr $3, %xmm2, %xmm3
655	lea 64(%rcx), %rcx
656	palignr $3, %xmm1, %xmm2
657	movaps %xmm7, %xmm1
658	movaps %xmm5, 48(%rdx)
659	movaps %xmm4, 32(%rdx)
660	movaps %xmm3, 16(%rdx)
661	movaps %xmm2, (%rdx)
662	lea 64(%rdx), %rdx
663	jmp L(Shl3LoopStart)
664
665	L(Shl3LoopExit):
666	movdqu -3(%rcx), %xmm1
667	mov $13, %rsi
668	movdqu %xmm1, -3(%rdx)
669	jmp L(CopyFrom1To16Bytes)
670
671	.p2align 4
672	L(Shl4):
673	movaps -4(%rcx), %xmm1
674	movaps 12(%rcx), %xmm2
675	L(Shl4Start):
676	pcmpeqb %xmm2, %xmm0
677	pmovmskb %xmm0, %rax
678	movaps %xmm2, %xmm3
679	# ifdef USE_AS_STRNCPY
680	sub $16, %r8
681	jbe L(StrncpyExit4Case2OrCase3)
682	# endif
683	test %rax, %rax
684	jnz L(Shl4LoopExit)
685
686	palignr $4, %xmm1, %xmm2
687	movaps %xmm2, (%rdx)
688	movaps 28(%rcx), %xmm2
689
690	pcmpeqb %xmm2, %xmm0
691	lea 16(%rdx), %rdx
692	pmovmskb %xmm0, %rax
693	lea 16(%rcx), %rcx
694	movaps %xmm2, %xmm1
695	# ifdef USE_AS_STRNCPY
696	sub $16, %r8
697	jbe L(StrncpyExit4Case2OrCase3)
698	# endif
699	test %rax, %rax
700	jnz L(Shl4LoopExit)
701
702	palignr $4, %xmm3, %xmm2
703	movaps %xmm2, (%rdx)
704	movaps 28(%rcx), %xmm2
705
706	pcmpeqb %xmm2, %xmm0
707	lea 16(%rdx), %rdx
708	pmovmskb %xmm0, %rax
709	lea 16(%rcx), %rcx
710	movaps %xmm2, %xmm3
711	# ifdef USE_AS_STRNCPY
712	sub $16, %r8
713	jbe L(StrncpyExit4Case2OrCase3)
714	# endif
715	test %rax, %rax
716	jnz L(Shl4LoopExit)
717
718	palignr $4, %xmm1, %xmm2
719	movaps %xmm2, (%rdx)
720	movaps 28(%rcx), %xmm2
721
722	pcmpeqb %xmm2, %xmm0
723	lea 16(%rdx), %rdx
724	pmovmskb %xmm0, %rax
725	lea 16(%rcx), %rcx
726	# ifdef USE_AS_STRNCPY
727	sub $16, %r8
728	jbe L(StrncpyExit4Case2OrCase3)
729	# endif
730	test %rax, %rax
731	jnz L(Shl4LoopExit)
732
733	palignr $4, %xmm3, %xmm2
734	movaps %xmm2, (%rdx)
735	lea 28(%rcx), %rcx
736	lea 16(%rdx), %rdx
737
738	mov %rcx, %rax
739	and $-0x40, %rcx
740	sub %rcx, %rax
741	lea -12(%rcx), %rcx
742	sub %rax, %rdx
743	# ifdef USE_AS_STRNCPY
744	add %rax, %r8
745	# endif
746	movaps -4(%rcx), %xmm1
747
748	/* 64 bytes loop */
749	.p2align 4
750	L(Shl4LoopStart):
751	movaps 12(%rcx), %xmm2
752	movaps 28(%rcx), %xmm3
753	movaps %xmm3, %xmm6
754	movaps 44(%rcx), %xmm4
755	movaps %xmm4, %xmm7
756	movaps 60(%rcx), %xmm5
757	pminub %xmm2, %xmm6
758	pminub %xmm5, %xmm7
759	pminub %xmm6, %xmm7
760	pcmpeqb %xmm0, %xmm7
761	pmovmskb %xmm7, %rax
762	movaps %xmm5, %xmm7
763	palignr $4, %xmm4, %xmm5
764	test %rax, %rax
765	palignr $4, %xmm3, %xmm4
766	jnz L(Shl4Start)
767	# ifdef USE_AS_STRNCPY
768	sub $64, %r8
769	jbe L(StrncpyLeave4)
770	# endif
771	palignr $4, %xmm2, %xmm3
772	lea 64(%rcx), %rcx
773	palignr $4, %xmm1, %xmm2
774	movaps %xmm7, %xmm1
775	movaps %xmm5, 48(%rdx)
776	movaps %xmm4, 32(%rdx)
777	movaps %xmm3, 16(%rdx)
778	movaps %xmm2, (%rdx)
779	lea 64(%rdx), %rdx
780	jmp L(Shl4LoopStart)
781
782	L(Shl4LoopExit):
783	movdqu -4(%rcx), %xmm1
784	mov $12, %rsi
785	movdqu %xmm1, -4(%rdx)
786	jmp L(CopyFrom1To16Bytes)
787
788	.p2align 4
789	L(Shl5):
790	movaps -5(%rcx), %xmm1
791	movaps 11(%rcx), %xmm2
792	L(Shl5Start):
793	pcmpeqb %xmm2, %xmm0
794	pmovmskb %xmm0, %rax
795	movaps %xmm2, %xmm3
796	# ifdef USE_AS_STRNCPY
797	sub $16, %r8
798	jbe L(StrncpyExit5Case2OrCase3)
799	# endif
800	test %rax, %rax
801	jnz L(Shl5LoopExit)
802
803	palignr $5, %xmm1, %xmm2
804	movaps %xmm2, (%rdx)
805	movaps 27(%rcx), %xmm2
806
807	pcmpeqb %xmm2, %xmm0
808	lea 16(%rdx), %rdx
809	pmovmskb %xmm0, %rax
810	lea 16(%rcx), %rcx
811	movaps %xmm2, %xmm1
812	# ifdef USE_AS_STRNCPY
813	sub $16, %r8
814	jbe L(StrncpyExit5Case2OrCase3)
815	# endif
816	test %rax, %rax
817	jnz L(Shl5LoopExit)
818
819	palignr $5, %xmm3, %xmm2
820	movaps %xmm2, (%rdx)
821	movaps 27(%rcx), %xmm2
822
823	pcmpeqb %xmm2, %xmm0
824	lea 16(%rdx), %rdx
825	pmovmskb %xmm0, %rax
826	lea 16(%rcx), %rcx
827	movaps %xmm2, %xmm3
828	# ifdef USE_AS_STRNCPY
829	sub $16, %r8
830	jbe L(StrncpyExit5Case2OrCase3)
831	# endif
832	test %rax, %rax
833	jnz L(Shl5LoopExit)
834
835	palignr $5, %xmm1, %xmm2
836	movaps %xmm2, (%rdx)
837	movaps 27(%rcx), %xmm2
838
839	pcmpeqb %xmm2, %xmm0
840	lea 16(%rdx), %rdx
841	pmovmskb %xmm0, %rax
842	lea 16(%rcx), %rcx
843	# ifdef USE_AS_STRNCPY
844	sub $16, %r8
845	jbe L(StrncpyExit5Case2OrCase3)
846	# endif
847	test %rax, %rax
848	jnz L(Shl5LoopExit)
849
850	palignr $5, %xmm3, %xmm2
851	movaps %xmm2, (%rdx)
852	lea 27(%rcx), %rcx
853	lea 16(%rdx), %rdx
854
855	mov %rcx, %rax
856	and $-0x40, %rcx
857	sub %rcx, %rax
858	lea -11(%rcx), %rcx
859	sub %rax, %rdx
860	# ifdef USE_AS_STRNCPY
861	add %rax, %r8
862	# endif
863	movaps -5(%rcx), %xmm1
864
865	/* 64 bytes loop */
866	.p2align 4
867	L(Shl5LoopStart):
868	movaps 11(%rcx), %xmm2
869	movaps 27(%rcx), %xmm3
870	movaps %xmm3, %xmm6
871	movaps 43(%rcx), %xmm4
872	movaps %xmm4, %xmm7
873	movaps 59(%rcx), %xmm5
874	pminub %xmm2, %xmm6
875	pminub %xmm5, %xmm7
876	pminub %xmm6, %xmm7
877	pcmpeqb %xmm0, %xmm7
878	pmovmskb %xmm7, %rax
879	movaps %xmm5, %xmm7
880	palignr $5, %xmm4, %xmm5
881	test %rax, %rax
882	palignr $5, %xmm3, %xmm4
883	jnz L(Shl5Start)
884	# ifdef USE_AS_STRNCPY
885	sub $64, %r8
886	jbe L(StrncpyLeave5)
887	# endif
888	palignr $5, %xmm2, %xmm3
889	lea 64(%rcx), %rcx
890	palignr $5, %xmm1, %xmm2
891	movaps %xmm7, %xmm1
892	movaps %xmm5, 48(%rdx)
893	movaps %xmm4, 32(%rdx)
894	movaps %xmm3, 16(%rdx)
895	movaps %xmm2, (%rdx)
896	lea 64(%rdx), %rdx
897	jmp L(Shl5LoopStart)
898
899	L(Shl5LoopExit):
900	movdqu -5(%rcx), %xmm1
901	mov $11, %rsi
902	movdqu %xmm1, -5(%rdx)
903	jmp L(CopyFrom1To16Bytes)
904
905	.p2align 4
906	L(Shl6):
907	movaps -6(%rcx), %xmm1
908	movaps 10(%rcx), %xmm2
909	L(Shl6Start):
910	pcmpeqb %xmm2, %xmm0
911	pmovmskb %xmm0, %rax
912	movaps %xmm2, %xmm3
913	# ifdef USE_AS_STRNCPY
914	sub $16, %r8
915	jbe L(StrncpyExit6Case2OrCase3)
916	# endif
917	test %rax, %rax
918	jnz L(Shl6LoopExit)
919
920	palignr $6, %xmm1, %xmm2
921	movaps %xmm2, (%rdx)
922	movaps 26(%rcx), %xmm2
923
924	pcmpeqb %xmm2, %xmm0
925	lea 16(%rdx), %rdx
926	pmovmskb %xmm0, %rax
927	lea 16(%rcx), %rcx
928	movaps %xmm2, %xmm1
929	# ifdef USE_AS_STRNCPY
930	sub $16, %r8
931	jbe L(StrncpyExit6Case2OrCase3)
932	# endif
933	test %rax, %rax
934	jnz L(Shl6LoopExit)
935
936	palignr $6, %xmm3, %xmm2
937	movaps %xmm2, (%rdx)
938	movaps 26(%rcx), %xmm2
939
940	pcmpeqb %xmm2, %xmm0
941	lea 16(%rdx), %rdx
942	pmovmskb %xmm0, %rax
943	lea 16(%rcx), %rcx
944	movaps %xmm2, %xmm3
945	# ifdef USE_AS_STRNCPY
946	sub $16, %r8
947	jbe L(StrncpyExit6Case2OrCase3)
948	# endif
949	test %rax, %rax
950	jnz L(Shl6LoopExit)
951
952	palignr $6, %xmm1, %xmm2
953	movaps %xmm2, (%rdx)
954	movaps 26(%rcx), %xmm2
955
956	pcmpeqb %xmm2, %xmm0
957	lea 16(%rdx), %rdx
958	pmovmskb %xmm0, %rax
959	lea 16(%rcx), %rcx
960	# ifdef USE_AS_STRNCPY
961	sub $16, %r8
962	jbe L(StrncpyExit6Case2OrCase3)
963	# endif
964	test %rax, %rax
965	jnz L(Shl6LoopExit)
966
967	palignr $6, %xmm3, %xmm2
968	movaps %xmm2, (%rdx)
969	lea 26(%rcx), %rcx
970	lea 16(%rdx), %rdx
971
972	mov %rcx, %rax
973	and $-0x40, %rcx
974	sub %rcx, %rax
975	lea -10(%rcx), %rcx
976	sub %rax, %rdx
977	# ifdef USE_AS_STRNCPY
978	add %rax, %r8
979	# endif
980	movaps -6(%rcx), %xmm1
981
982	/* 64 bytes loop */
983	.p2align 4
984	L(Shl6LoopStart):
985	movaps 10(%rcx), %xmm2
986	movaps 26(%rcx), %xmm3
987	movaps %xmm3, %xmm6
988	movaps 42(%rcx), %xmm4
989	movaps %xmm4, %xmm7
990	movaps 58(%rcx), %xmm5
991	pminub %xmm2, %xmm6
992	pminub %xmm5, %xmm7
993	pminub %xmm6, %xmm7
994	pcmpeqb %xmm0, %xmm7
995	pmovmskb %xmm7, %rax
996	movaps %xmm5, %xmm7
997	palignr $6, %xmm4, %xmm5
998	test %rax, %rax
999	palignr $6, %xmm3, %xmm4
1000	jnz L(Shl6Start)
1001	# ifdef USE_AS_STRNCPY
1002	sub $64, %r8
1003	jbe L(StrncpyLeave6)
1004	# endif
1005	palignr $6, %xmm2, %xmm3
1006	lea 64(%rcx), %rcx
1007	palignr $6, %xmm1, %xmm2
1008	movaps %xmm7, %xmm1
1009	movaps %xmm5, 48(%rdx)
1010	movaps %xmm4, 32(%rdx)
1011	movaps %xmm3, 16(%rdx)
1012	movaps %xmm2, (%rdx)
1013	lea 64(%rdx), %rdx
1014	jmp L(Shl6LoopStart)
1015
1016	L(Shl6LoopExit):
1017	mov (%rcx), %r9
1018	mov 6(%rcx), %esi
1019	mov %r9, (%rdx)
1020	mov %esi, 6(%rdx)
1021	mov $10, %rsi
1022	jmp L(CopyFrom1To16Bytes)
1023
1024	.p2align 4
1025	L(Shl7):
1026	movaps -7(%rcx), %xmm1
1027	movaps 9(%rcx), %xmm2
1028	L(Shl7Start):
1029	pcmpeqb %xmm2, %xmm0
1030	pmovmskb %xmm0, %rax
1031	movaps %xmm2, %xmm3
1032	# ifdef USE_AS_STRNCPY
1033	sub $16, %r8
1034	jbe L(StrncpyExit7Case2OrCase3)
1035	# endif
1036	test %rax, %rax
1037	jnz L(Shl7LoopExit)
1038
1039	palignr $7, %xmm1, %xmm2
1040	movaps %xmm2, (%rdx)
1041	movaps 25(%rcx), %xmm2
1042
1043	pcmpeqb %xmm2, %xmm0
1044	lea 16(%rdx), %rdx
1045	pmovmskb %xmm0, %rax
1046	lea 16(%rcx), %rcx
1047	movaps %xmm2, %xmm1
1048	# ifdef USE_AS_STRNCPY
1049	sub $16, %r8
1050	jbe L(StrncpyExit7Case2OrCase3)
1051	# endif
1052	test %rax, %rax
1053	jnz L(Shl7LoopExit)
1054
1055	palignr $7, %xmm3, %xmm2
1056	movaps %xmm2, (%rdx)
1057	movaps 25(%rcx), %xmm2
1058
1059	pcmpeqb %xmm2, %xmm0
1060	lea 16(%rdx), %rdx
1061	pmovmskb %xmm0, %rax
1062	lea 16(%rcx), %rcx
1063	movaps %xmm2, %xmm3
1064	# ifdef USE_AS_STRNCPY
1065	sub $16, %r8
1066	jbe L(StrncpyExit7Case2OrCase3)
1067	# endif
1068	test %rax, %rax
1069	jnz L(Shl7LoopExit)
1070
1071	palignr $7, %xmm1, %xmm2
1072	movaps %xmm2, (%rdx)
1073	movaps 25(%rcx), %xmm2
1074
1075	pcmpeqb %xmm2, %xmm0
1076	lea 16(%rdx), %rdx
1077	pmovmskb %xmm0, %rax
1078	lea 16(%rcx), %rcx
1079	# ifdef USE_AS_STRNCPY
1080	sub $16, %r8
1081	jbe L(StrncpyExit7Case2OrCase3)
1082	# endif
1083	test %rax, %rax
1084	jnz L(Shl7LoopExit)
1085
1086	palignr $7, %xmm3, %xmm2
1087	movaps %xmm2, (%rdx)
1088	lea 25(%rcx), %rcx
1089	lea 16(%rdx), %rdx
1090
1091	mov %rcx, %rax
1092	and $-0x40, %rcx
1093	sub %rcx, %rax
1094	lea -9(%rcx), %rcx
1095	sub %rax, %rdx
1096	# ifdef USE_AS_STRNCPY
1097	add %rax, %r8
1098	# endif
1099	movaps -7(%rcx), %xmm1
1100
1101	/* 64 bytes loop */
1102	.p2align 4
1103	L(Shl7LoopStart):
1104	movaps 9(%rcx), %xmm2
1105	movaps 25(%rcx), %xmm3
1106	movaps %xmm3, %xmm6
1107	movaps 41(%rcx), %xmm4
1108	movaps %xmm4, %xmm7
1109	movaps 57(%rcx), %xmm5
1110	pminub %xmm2, %xmm6
1111	pminub %xmm5, %xmm7
1112	pminub %xmm6, %xmm7
1113	pcmpeqb %xmm0, %xmm7
1114	pmovmskb %xmm7, %rax
1115	movaps %xmm5, %xmm7
1116	palignr $7, %xmm4, %xmm5
1117	test %rax, %rax
1118	palignr $7, %xmm3, %xmm4
1119	jnz L(Shl7Start)
1120	# ifdef USE_AS_STRNCPY
1121	sub $64, %r8
1122	jbe L(StrncpyLeave7)
1123	# endif
1124	palignr $7, %xmm2, %xmm3
1125	lea 64(%rcx), %rcx
1126	palignr $7, %xmm1, %xmm2
1127	movaps %xmm7, %xmm1
1128	movaps %xmm5, 48(%rdx)
1129	movaps %xmm4, 32(%rdx)
1130	movaps %xmm3, 16(%rdx)
1131	movaps %xmm2, (%rdx)
1132	lea 64(%rdx), %rdx
1133	jmp L(Shl7LoopStart)
1134
1135	L(Shl7LoopExit):
1136	mov (%rcx), %r9
1137	mov 5(%rcx), %esi
1138	mov %r9, (%rdx)
1139	mov %esi, 5(%rdx)
1140	mov $9, %rsi
1141	jmp L(CopyFrom1To16Bytes)
1142
1143	.p2align 4
1144	L(Shl8):
1145	movaps -8(%rcx), %xmm1
1146	movaps 8(%rcx), %xmm2
1147	L(Shl8Start):
1148	pcmpeqb %xmm2, %xmm0
1149	pmovmskb %xmm0, %rax
1150	movaps %xmm2, %xmm3
1151	# ifdef USE_AS_STRNCPY
1152	sub $16, %r8
1153	jbe L(StrncpyExit8Case2OrCase3)
1154	# endif
1155	test %rax, %rax
1156	jnz L(Shl8LoopExit)
1157
1158	palignr $8, %xmm1, %xmm2
1159	movaps %xmm2, (%rdx)
1160	movaps 24(%rcx), %xmm2
1161
1162	pcmpeqb %xmm2, %xmm0
1163	lea 16(%rdx), %rdx
1164	pmovmskb %xmm0, %rax
1165	lea 16(%rcx), %rcx
1166	movaps %xmm2, %xmm1
1167	# ifdef USE_AS_STRNCPY
1168	sub $16, %r8
1169	jbe L(StrncpyExit8Case2OrCase3)
1170	# endif
1171	test %rax, %rax
1172	jnz L(Shl8LoopExit)
1173
1174	palignr $8, %xmm3, %xmm2
1175	movaps %xmm2, (%rdx)
1176	movaps 24(%rcx), %xmm2
1177
1178	pcmpeqb %xmm2, %xmm0
1179	lea 16(%rdx), %rdx
1180	pmovmskb %xmm0, %rax
1181	lea 16(%rcx), %rcx
1182	movaps %xmm2, %xmm3
1183	# ifdef USE_AS_STRNCPY
1184	sub $16, %r8
1185	jbe L(StrncpyExit8Case2OrCase3)
1186	# endif
1187	test %rax, %rax
1188	jnz L(Shl8LoopExit)
1189
1190	palignr $8, %xmm1, %xmm2
1191	movaps %xmm2, (%rdx)
1192	movaps 24(%rcx), %xmm2
1193
1194	pcmpeqb %xmm2, %xmm0
1195	lea 16(%rdx), %rdx
1196	pmovmskb %xmm0, %rax
1197	lea 16(%rcx), %rcx
1198	# ifdef USE_AS_STRNCPY
1199	sub $16, %r8
1200	jbe L(StrncpyExit8Case2OrCase3)
1201	# endif
1202	test %rax, %rax
1203	jnz L(Shl8LoopExit)
1204
1205	palignr $8, %xmm3, %xmm2
1206	movaps %xmm2, (%rdx)
1207	lea 24(%rcx), %rcx
1208	lea 16(%rdx), %rdx
1209
1210	mov %rcx, %rax
1211	and $-0x40, %rcx
1212	sub %rcx, %rax
1213	lea -8(%rcx), %rcx
1214	sub %rax, %rdx
1215	# ifdef USE_AS_STRNCPY
1216	add %rax, %r8
1217	# endif
1218	movaps -8(%rcx), %xmm1
1219
1220	/* 64 bytes loop */
1221	.p2align 4
1222	L(Shl8LoopStart):
1223	movaps 8(%rcx), %xmm2
1224	movaps 24(%rcx), %xmm3
1225	movaps %xmm3, %xmm6
1226	movaps 40(%rcx), %xmm4
1227	movaps %xmm4, %xmm7
1228	movaps 56(%rcx), %xmm5
1229	pminub %xmm2, %xmm6
1230	pminub %xmm5, %xmm7
1231	pminub %xmm6, %xmm7
1232	pcmpeqb %xmm0, %xmm7
1233	pmovmskb %xmm7, %rax
1234	movaps %xmm5, %xmm7
1235	palignr $8, %xmm4, %xmm5
1236	test %rax, %rax
1237	palignr $8, %xmm3, %xmm4
1238	jnz L(Shl8Start)
1239	# ifdef USE_AS_STRNCPY
1240	sub $64, %r8
1241	jbe L(StrncpyLeave8)
1242	# endif
1243	palignr $8, %xmm2, %xmm3
1244	lea 64(%rcx), %rcx
1245	palignr $8, %xmm1, %xmm2
1246	movaps %xmm7, %xmm1
1247	movaps %xmm5, 48(%rdx)
1248	movaps %xmm4, 32(%rdx)
1249	movaps %xmm3, 16(%rdx)
1250	movaps %xmm2, (%rdx)
1251	lea 64(%rdx), %rdx
1252	jmp L(Shl8LoopStart)
1253
1254	L(Shl8LoopExit):
1255	mov (%rcx), %r9
1256	mov $8, %rsi
1257	mov %r9, (%rdx)
1258	jmp L(CopyFrom1To16Bytes)
1259
1260	.p2align 4
1261	L(Shl9):
1262	movaps -9(%rcx), %xmm1
1263	movaps 7(%rcx), %xmm2
1264	L(Shl9Start):
1265	pcmpeqb %xmm2, %xmm0
1266	pmovmskb %xmm0, %rax
1267	movaps %xmm2, %xmm3
1268	# ifdef USE_AS_STRNCPY
1269	sub $16, %r8
1270	jbe L(StrncpyExit9Case2OrCase3)
1271	# endif
1272	test %rax, %rax
1273	jnz L(Shl9LoopExit)
1274
1275	palignr $9, %xmm1, %xmm2
1276	movaps %xmm2, (%rdx)
1277	movaps 23(%rcx), %xmm2
1278
1279	pcmpeqb %xmm2, %xmm0
1280	lea 16(%rdx), %rdx
1281	pmovmskb %xmm0, %rax
1282	lea 16(%rcx), %rcx
1283	movaps %xmm2, %xmm1
1284	# ifdef USE_AS_STRNCPY
1285	sub $16, %r8
1286	jbe L(StrncpyExit9Case2OrCase3)
1287	# endif
1288	test %rax, %rax
1289	jnz L(Shl9LoopExit)
1290
1291	palignr $9, %xmm3, %xmm2
1292	movaps %xmm2, (%rdx)
1293	movaps 23(%rcx), %xmm2
1294
1295	pcmpeqb %xmm2, %xmm0
1296	lea 16(%rdx), %rdx
1297	pmovmskb %xmm0, %rax
1298	lea 16(%rcx), %rcx
1299	movaps %xmm2, %xmm3
1300	# ifdef USE_AS_STRNCPY
1301	sub $16, %r8
1302	jbe L(StrncpyExit9Case2OrCase3)
1303	# endif
1304	test %rax, %rax
1305	jnz L(Shl9LoopExit)
1306
1307	palignr $9, %xmm1, %xmm2
1308	movaps %xmm2, (%rdx)
1309	movaps 23(%rcx), %xmm2
1310
1311	pcmpeqb %xmm2, %xmm0
1312	lea 16(%rdx), %rdx
1313	pmovmskb %xmm0, %rax
1314	lea 16(%rcx), %rcx
1315	# ifdef USE_AS_STRNCPY
1316	sub $16, %r8
1317	jbe L(StrncpyExit9Case2OrCase3)
1318	# endif
1319	test %rax, %rax
1320	jnz L(Shl9LoopExit)
1321
1322	palignr $9, %xmm3, %xmm2
1323	movaps %xmm2, (%rdx)
1324	lea 23(%rcx), %rcx
1325	lea 16(%rdx), %rdx
1326
1327	mov %rcx, %rax
1328	and $-0x40, %rcx
1329	sub %rcx, %rax
1330	lea -7(%rcx), %rcx
1331	sub %rax, %rdx
1332	# ifdef USE_AS_STRNCPY
1333	add %rax, %r8
1334	# endif
1335	movaps -9(%rcx), %xmm1
1336
1337	/* 64 bytes loop */
1338	.p2align 4
1339	L(Shl9LoopStart):
1340	movaps 7(%rcx), %xmm2
1341	movaps 23(%rcx), %xmm3
1342	movaps %xmm3, %xmm6
1343	movaps 39(%rcx), %xmm4
1344	movaps %xmm4, %xmm7
1345	movaps 55(%rcx), %xmm5
1346	pminub %xmm2, %xmm6
1347	pminub %xmm5, %xmm7
1348	pminub %xmm6, %xmm7
1349	pcmpeqb %xmm0, %xmm7
1350	pmovmskb %xmm7, %rax
1351	movaps %xmm5, %xmm7
1352	palignr $9, %xmm4, %xmm5
1353	test %rax, %rax
1354	palignr $9, %xmm3, %xmm4
1355	jnz L(Shl9Start)
1356	# ifdef USE_AS_STRNCPY
1357	sub $64, %r8
1358	jbe L(StrncpyLeave9)
1359	# endif
1360	palignr $9, %xmm2, %xmm3
1361	lea 64(%rcx), %rcx
1362	palignr $9, %xmm1, %xmm2
1363	movaps %xmm7, %xmm1
1364	movaps %xmm5, 48(%rdx)
1365	movaps %xmm4, 32(%rdx)
1366	movaps %xmm3, 16(%rdx)
1367	movaps %xmm2, (%rdx)
1368	lea 64(%rdx), %rdx
1369	jmp L(Shl9LoopStart)
1370
1371	L(Shl9LoopExit):
1372	mov -1(%rcx), %r9
1373	mov $7, %rsi
1374	mov %r9, -1(%rdx)
1375	jmp L(CopyFrom1To16Bytes)
1376
1377	.p2align 4
1378	L(Shl10):
1379	movaps -10(%rcx), %xmm1
1380	movaps 6(%rcx), %xmm2
1381	L(Shl10Start):
1382	pcmpeqb %xmm2, %xmm0
1383	pmovmskb %xmm0, %rax
1384	movaps %xmm2, %xmm3
1385	# ifdef USE_AS_STRNCPY
1386	sub $16, %r8
1387	jbe L(StrncpyExit10Case2OrCase3)
1388	# endif
1389	test %rax, %rax
1390	jnz L(Shl10LoopExit)
1391
1392	palignr $10, %xmm1, %xmm2
1393	movaps %xmm2, (%rdx)
1394	movaps 22(%rcx), %xmm2
1395
1396	pcmpeqb %xmm2, %xmm0
1397	lea 16(%rdx), %rdx
1398	pmovmskb %xmm0, %rax
1399	lea 16(%rcx), %rcx
1400	movaps %xmm2, %xmm1
1401	# ifdef USE_AS_STRNCPY
1402	sub $16, %r8
1403	jbe L(StrncpyExit10Case2OrCase3)
1404	# endif
1405	test %rax, %rax
1406	jnz L(Shl10LoopExit)
1407
1408	palignr $10, %xmm3, %xmm2
1409	movaps %xmm2, (%rdx)
1410	movaps 22(%rcx), %xmm2
1411
1412	pcmpeqb %xmm2, %xmm0
1413	lea 16(%rdx), %rdx
1414	pmovmskb %xmm0, %rax
1415	lea 16(%rcx), %rcx
1416	movaps %xmm2, %xmm3
1417	# ifdef USE_AS_STRNCPY
1418	sub $16, %r8
1419	jbe L(StrncpyExit10Case2OrCase3)
1420	# endif
1421	test %rax, %rax
1422	jnz L(Shl10LoopExit)
1423
1424	palignr $10, %xmm1, %xmm2
1425	movaps %xmm2, (%rdx)
1426	movaps 22(%rcx), %xmm2
1427
1428	pcmpeqb %xmm2, %xmm0
1429	lea 16(%rdx), %rdx
1430	pmovmskb %xmm0, %rax
1431	lea 16(%rcx), %rcx
1432	# ifdef USE_AS_STRNCPY
1433	sub $16, %r8
1434	jbe L(StrncpyExit10Case2OrCase3)
1435	# endif
1436	test %rax, %rax
1437	jnz L(Shl10LoopExit)
1438
1439	palignr $10, %xmm3, %xmm2
1440	movaps %xmm2, (%rdx)
1441	lea 22(%rcx), %rcx
1442	lea 16(%rdx), %rdx
1443
1444	mov %rcx, %rax
1445	and $-0x40, %rcx
1446	sub %rcx, %rax
1447	lea -6(%rcx), %rcx
1448	sub %rax, %rdx
1449	# ifdef USE_AS_STRNCPY
1450	add %rax, %r8
1451	# endif
1452	movaps -10(%rcx), %xmm1
1453
1454	/* 64 bytes loop */
1455	.p2align 4
1456	L(Shl10LoopStart):
1457	movaps 6(%rcx), %xmm2
1458	movaps 22(%rcx), %xmm3
1459	movaps %xmm3, %xmm6
1460	movaps 38(%rcx), %xmm4
1461	movaps %xmm4, %xmm7
1462	movaps 54(%rcx), %xmm5
1463	pminub %xmm2, %xmm6
1464	pminub %xmm5, %xmm7
1465	pminub %xmm6, %xmm7
1466	pcmpeqb %xmm0, %xmm7
1467	pmovmskb %xmm7, %rax
1468	movaps %xmm5, %xmm7
1469	palignr $10, %xmm4, %xmm5
1470	test %rax, %rax
1471	palignr $10, %xmm3, %xmm4
1472	jnz L(Shl10Start)
1473	# ifdef USE_AS_STRNCPY
1474	sub $64, %r8
1475	jbe L(StrncpyLeave10)
1476	# endif
1477	palignr $10, %xmm2, %xmm3
1478	lea 64(%rcx), %rcx
1479	palignr $10, %xmm1, %xmm2
1480	movaps %xmm7, %xmm1
1481	movaps %xmm5, 48(%rdx)
1482	movaps %xmm4, 32(%rdx)
1483	movaps %xmm3, 16(%rdx)
1484	movaps %xmm2, (%rdx)
1485	lea 64(%rdx), %rdx
1486	jmp L(Shl10LoopStart)
1487
1488	L(Shl10LoopExit):
1489	mov -2(%rcx), %r9
1490	mov $6, %rsi
1491	mov %r9, -2(%rdx)
1492	jmp L(CopyFrom1To16Bytes)
1493
1494	.p2align 4
1495	L(Shl11):
1496	movaps -11(%rcx), %xmm1
1497	movaps 5(%rcx), %xmm2
1498	L(Shl11Start):
1499	pcmpeqb %xmm2, %xmm0
1500	pmovmskb %xmm0, %rax
1501	movaps %xmm2, %xmm3
1502	# ifdef USE_AS_STRNCPY
1503	sub $16, %r8
1504	jbe L(StrncpyExit11Case2OrCase3)
1505	# endif
1506	test %rax, %rax
1507	jnz L(Shl11LoopExit)
1508
1509	palignr $11, %xmm1, %xmm2
1510	movaps %xmm2, (%rdx)
1511	movaps 21(%rcx), %xmm2
1512
1513	pcmpeqb %xmm2, %xmm0
1514	lea 16(%rdx), %rdx
1515	pmovmskb %xmm0, %rax
1516	lea 16(%rcx), %rcx
1517	movaps %xmm2, %xmm1
1518	# ifdef USE_AS_STRNCPY
1519	sub $16, %r8
1520	jbe L(StrncpyExit11Case2OrCase3)
1521	# endif
1522	test %rax, %rax
1523	jnz L(Shl11LoopExit)
1524
1525	palignr $11, %xmm3, %xmm2
1526	movaps %xmm2, (%rdx)
1527	movaps 21(%rcx), %xmm2
1528
1529	pcmpeqb %xmm2, %xmm0
1530	lea 16(%rdx), %rdx
1531	pmovmskb %xmm0, %rax
1532	lea 16(%rcx), %rcx
1533	movaps %xmm2, %xmm3
1534	# ifdef USE_AS_STRNCPY
1535	sub $16, %r8
1536	jbe L(StrncpyExit11Case2OrCase3)
1537	# endif
1538	test %rax, %rax
1539	jnz L(Shl11LoopExit)
1540
1541	palignr $11, %xmm1, %xmm2
1542	movaps %xmm2, (%rdx)
1543	movaps 21(%rcx), %xmm2
1544
1545	pcmpeqb %xmm2, %xmm0
1546	lea 16(%rdx), %rdx
1547	pmovmskb %xmm0, %rax
1548	lea 16(%rcx), %rcx
1549	# ifdef USE_AS_STRNCPY
1550	sub $16, %r8
1551	jbe L(StrncpyExit11Case2OrCase3)
1552	# endif
1553	test %rax, %rax
1554	jnz L(Shl11LoopExit)
1555
1556	palignr $11, %xmm3, %xmm2
1557	movaps %xmm2, (%rdx)
1558	lea 21(%rcx), %rcx
1559	lea 16(%rdx), %rdx
1560
1561	mov %rcx, %rax
1562	and $-0x40, %rcx
1563	sub %rcx, %rax
1564	lea -5(%rcx), %rcx
1565	sub %rax, %rdx
1566	# ifdef USE_AS_STRNCPY
1567	add %rax, %r8
1568	# endif
1569	movaps -11(%rcx), %xmm1
1570
1571	/* 64 bytes loop */
1572	.p2align 4
1573	L(Shl11LoopStart):
1574	movaps 5(%rcx), %xmm2
1575	movaps 21(%rcx), %xmm3
1576	movaps %xmm3, %xmm6
1577	movaps 37(%rcx), %xmm4
1578	movaps %xmm4, %xmm7
1579	movaps 53(%rcx), %xmm5
1580	pminub %xmm2, %xmm6
1581	pminub %xmm5, %xmm7
1582	pminub %xmm6, %xmm7
1583	pcmpeqb %xmm0, %xmm7
1584	pmovmskb %xmm7, %rax
1585	movaps %xmm5, %xmm7
1586	palignr $11, %xmm4, %xmm5
1587	test %rax, %rax
1588	palignr $11, %xmm3, %xmm4
1589	jnz L(Shl11Start)
1590	# ifdef USE_AS_STRNCPY
1591	sub $64, %r8
1592	jbe L(StrncpyLeave11)
1593	# endif
1594	palignr $11, %xmm2, %xmm3
1595	lea 64(%rcx), %rcx
1596	palignr $11, %xmm1, %xmm2
1597	movaps %xmm7, %xmm1
1598	movaps %xmm5, 48(%rdx)
1599	movaps %xmm4, 32(%rdx)
1600	movaps %xmm3, 16(%rdx)
1601	movaps %xmm2, (%rdx)
1602	lea 64(%rdx), %rdx
1603	jmp L(Shl11LoopStart)
1604
1605	L(Shl11LoopExit):
1606	mov -3(%rcx), %r9
1607	mov $5, %rsi
1608	mov %r9, -3(%rdx)
1609	jmp L(CopyFrom1To16Bytes)
1610
1611	.p2align 4
1612	L(Shl12):
1613	movaps -12(%rcx), %xmm1
1614	movaps 4(%rcx), %xmm2
1615	L(Shl12Start):
1616	pcmpeqb %xmm2, %xmm0
1617	pmovmskb %xmm0, %rax
1618	movaps %xmm2, %xmm3
1619	# ifdef USE_AS_STRNCPY
1620	sub $16, %r8
1621	jbe L(StrncpyExit12Case2OrCase3)
1622	# endif
1623	test %rax, %rax
1624	jnz L(Shl12LoopExit)
1625
1626	palignr $12, %xmm1, %xmm2
1627	movaps %xmm2, (%rdx)
1628	movaps 20(%rcx), %xmm2
1629
1630	pcmpeqb %xmm2, %xmm0
1631	lea 16(%rdx), %rdx
1632	pmovmskb %xmm0, %rax
1633	lea 16(%rcx), %rcx
1634	movaps %xmm2, %xmm1
1635	# ifdef USE_AS_STRNCPY
1636	sub $16, %r8
1637	jbe L(StrncpyExit12Case2OrCase3)
1638	# endif
1639	test %rax, %rax
1640	jnz L(Shl12LoopExit)
1641
1642	palignr $12, %xmm3, %xmm2
1643	movaps %xmm2, (%rdx)
1644	movaps 20(%rcx), %xmm2
1645
1646	pcmpeqb %xmm2, %xmm0
1647	lea 16(%rdx), %rdx
1648	pmovmskb %xmm0, %rax
1649	lea 16(%rcx), %rcx
1650	movaps %xmm2, %xmm3
1651	# ifdef USE_AS_STRNCPY
1652	sub $16, %r8
1653	jbe L(StrncpyExit12Case2OrCase3)
1654	# endif
1655	test %rax, %rax
1656	jnz L(Shl12LoopExit)
1657
1658	palignr $12, %xmm1, %xmm2
1659	movaps %xmm2, (%rdx)
1660	movaps 20(%rcx), %xmm2
1661
1662	pcmpeqb %xmm2, %xmm0
1663	lea 16(%rdx), %rdx
1664	pmovmskb %xmm0, %rax
1665	lea 16(%rcx), %rcx
1666	# ifdef USE_AS_STRNCPY
1667	sub $16, %r8
1668	jbe L(StrncpyExit12Case2OrCase3)
1669	# endif
1670	test %rax, %rax
1671	jnz L(Shl12LoopExit)
1672
1673	palignr $12, %xmm3, %xmm2
1674	movaps %xmm2, (%rdx)
1675	lea 20(%rcx), %rcx
1676	lea 16(%rdx), %rdx
1677
1678	mov %rcx, %rax
1679	and $-0x40, %rcx
1680	sub %rcx, %rax
1681	lea -4(%rcx), %rcx
1682	sub %rax, %rdx
1683	# ifdef USE_AS_STRNCPY
1684	add %rax, %r8
1685	# endif
1686	movaps -12(%rcx), %xmm1
1687
1688	/* 64 bytes loop */
1689	.p2align 4
1690	L(Shl12LoopStart):
1691	movaps 4(%rcx), %xmm2
1692	movaps 20(%rcx), %xmm3
1693	movaps %xmm3, %xmm6
1694	movaps 36(%rcx), %xmm4
1695	movaps %xmm4, %xmm7
1696	movaps 52(%rcx), %xmm5
1697	pminub %xmm2, %xmm6
1698	pminub %xmm5, %xmm7
1699	pminub %xmm6, %xmm7
1700	pcmpeqb %xmm0, %xmm7
1701	pmovmskb %xmm7, %rax
1702	movaps %xmm5, %xmm7
1703	palignr $12, %xmm4, %xmm5
1704	test %rax, %rax
1705	palignr $12, %xmm3, %xmm4
1706	jnz L(Shl12Start)
1707	# ifdef USE_AS_STRNCPY
1708	sub $64, %r8
1709	jbe L(StrncpyLeave12)
1710	# endif
1711	palignr $12, %xmm2, %xmm3
1712	lea 64(%rcx), %rcx
1713	palignr $12, %xmm1, %xmm2
1714	movaps %xmm7, %xmm1
1715	movaps %xmm5, 48(%rdx)
1716	movaps %xmm4, 32(%rdx)
1717	movaps %xmm3, 16(%rdx)
1718	movaps %xmm2, (%rdx)
1719	lea 64(%rdx), %rdx
1720	jmp L(Shl12LoopStart)
1721
1722	L(Shl12LoopExit):
1723	mov (%rcx), %r9d
1724	mov $4, %rsi
1725	mov %r9d, (%rdx)
1726	jmp L(CopyFrom1To16Bytes)
1727
1728	.p2align 4
1729	L(Shl13):
1730	movaps -13(%rcx), %xmm1
1731	movaps 3(%rcx), %xmm2
1732	L(Shl13Start):
1733	pcmpeqb %xmm2, %xmm0
1734	pmovmskb %xmm0, %rax
1735	movaps %xmm2, %xmm3
1736	# ifdef USE_AS_STRNCPY
1737	sub $16, %r8
1738	jbe L(StrncpyExit13Case2OrCase3)
1739	# endif
1740	test %rax, %rax
1741	jnz L(Shl13LoopExit)
1742
1743	palignr $13, %xmm1, %xmm2
1744	movaps %xmm2, (%rdx)
1745	movaps 19(%rcx), %xmm2
1746
1747	pcmpeqb %xmm2, %xmm0
1748	lea 16(%rdx), %rdx
1749	pmovmskb %xmm0, %rax
1750	lea 16(%rcx), %rcx
1751	movaps %xmm2, %xmm1
1752	# ifdef USE_AS_STRNCPY
1753	sub $16, %r8
1754	jbe L(StrncpyExit13Case2OrCase3)
1755	# endif
1756	test %rax, %rax
1757	jnz L(Shl13LoopExit)
1758
1759	palignr $13, %xmm3, %xmm2
1760	movaps %xmm2, (%rdx)
1761	movaps 19(%rcx), %xmm2
1762
1763	pcmpeqb %xmm2, %xmm0
1764	lea 16(%rdx), %rdx
1765	pmovmskb %xmm0, %rax
1766	lea 16(%rcx), %rcx
1767	movaps %xmm2, %xmm3
1768	# ifdef USE_AS_STRNCPY
1769	sub $16, %r8
1770	jbe L(StrncpyExit13Case2OrCase3)
1771	# endif
1772	test %rax, %rax
1773	jnz L(Shl13LoopExit)
1774
1775	palignr $13, %xmm1, %xmm2
1776	movaps %xmm2, (%rdx)
1777	movaps 19(%rcx), %xmm2
1778
1779	pcmpeqb %xmm2, %xmm0
1780	lea 16(%rdx), %rdx
1781	pmovmskb %xmm0, %rax
1782	lea 16(%rcx), %rcx
1783	# ifdef USE_AS_STRNCPY
1784	sub $16, %r8
1785	jbe L(StrncpyExit13Case2OrCase3)
1786	# endif
1787	test %rax, %rax
1788	jnz L(Shl13LoopExit)
1789
1790	palignr $13, %xmm3, %xmm2
1791	movaps %xmm2, (%rdx)
1792	lea 19(%rcx), %rcx
1793	lea 16(%rdx), %rdx
1794
1795	mov %rcx, %rax
1796	and $-0x40, %rcx
1797	sub %rcx, %rax
1798	lea -3(%rcx), %rcx
1799	sub %rax, %rdx
1800	# ifdef USE_AS_STRNCPY
1801	add %rax, %r8
1802	# endif
1803	movaps -13(%rcx), %xmm1
1804
1805	/* 64 bytes loop */
1806	.p2align 4
1807	L(Shl13LoopStart):
1808	movaps 3(%rcx), %xmm2
1809	movaps 19(%rcx), %xmm3
1810	movaps %xmm3, %xmm6
1811	movaps 35(%rcx), %xmm4
1812	movaps %xmm4, %xmm7
1813	movaps 51(%rcx), %xmm5
1814	pminub %xmm2, %xmm6
1815	pminub %xmm5, %xmm7
1816	pminub %xmm6, %xmm7
1817	pcmpeqb %xmm0, %xmm7
1818	pmovmskb %xmm7, %rax
1819	movaps %xmm5, %xmm7
1820	palignr $13, %xmm4, %xmm5
1821	test %rax, %rax
1822	palignr $13, %xmm3, %xmm4
1823	jnz L(Shl13Start)
1824	# ifdef USE_AS_STRNCPY
1825	sub $64, %r8
1826	jbe L(StrncpyLeave13)
1827	# endif
1828	palignr $13, %xmm2, %xmm3
1829	lea 64(%rcx), %rcx
1830	palignr $13, %xmm1, %xmm2
1831	movaps %xmm7, %xmm1
1832	movaps %xmm5, 48(%rdx)
1833	movaps %xmm4, 32(%rdx)
1834	movaps %xmm3, 16(%rdx)
1835	movaps %xmm2, (%rdx)
1836	lea 64(%rdx), %rdx
1837	jmp L(Shl13LoopStart)
1838
1839	L(Shl13LoopExit):
1840	mov -1(%rcx), %r9d
1841	mov $3, %rsi
1842	mov %r9d, -1(%rdx)
1843	jmp L(CopyFrom1To16Bytes)
1844
1845	.p2align 4
1846	L(Shl14):
1847	movaps -14(%rcx), %xmm1
1848	movaps 2(%rcx), %xmm2
1849	L(Shl14Start):
1850	pcmpeqb %xmm2, %xmm0
1851	pmovmskb %xmm0, %rax
1852	movaps %xmm2, %xmm3
1853	# ifdef USE_AS_STRNCPY
1854	sub $16, %r8
1855	jbe L(StrncpyExit14Case2OrCase3)
1856	# endif
1857	test %rax, %rax
1858	jnz L(Shl14LoopExit)
1859
1860	palignr $14, %xmm1, %xmm2
1861	movaps %xmm2, (%rdx)
1862	movaps 18(%rcx), %xmm2
1863
1864	pcmpeqb %xmm2, %xmm0
1865	lea 16(%rdx), %rdx
1866	pmovmskb %xmm0, %rax
1867	lea 16(%rcx), %rcx
1868	movaps %xmm2, %xmm1
1869	# ifdef USE_AS_STRNCPY
1870	sub $16, %r8
1871	jbe L(StrncpyExit14Case2OrCase3)
1872	# endif
1873	test %rax, %rax
1874	jnz L(Shl14LoopExit)
1875
1876	palignr $14, %xmm3, %xmm2
1877	movaps %xmm2, (%rdx)
1878	movaps 18(%rcx), %xmm2
1879
1880	pcmpeqb %xmm2, %xmm0
1881	lea 16(%rdx), %rdx
1882	pmovmskb %xmm0, %rax
1883	lea 16(%rcx), %rcx
1884	movaps %xmm2, %xmm3
1885	# ifdef USE_AS_STRNCPY
1886	sub $16, %r8
1887	jbe L(StrncpyExit14Case2OrCase3)
1888	# endif
1889	test %rax, %rax
1890	jnz L(Shl14LoopExit)
1891
1892	palignr $14, %xmm1, %xmm2
1893	movaps %xmm2, (%rdx)
1894	movaps 18(%rcx), %xmm2
1895
1896	pcmpeqb %xmm2, %xmm0
1897	lea 16(%rdx), %rdx
1898	pmovmskb %xmm0, %rax
1899	lea 16(%rcx), %rcx
1900	# ifdef USE_AS_STRNCPY
1901	sub $16, %r8
1902	jbe L(StrncpyExit14Case2OrCase3)
1903	# endif
1904	test %rax, %rax
1905	jnz L(Shl14LoopExit)
1906
1907	palignr $14, %xmm3, %xmm2
1908	movaps %xmm2, (%rdx)
1909	lea 18(%rcx), %rcx
1910	lea 16(%rdx), %rdx
1911
1912	mov %rcx, %rax
1913	and $-0x40, %rcx
1914	sub %rcx, %rax
1915	lea -2(%rcx), %rcx
1916	sub %rax, %rdx
1917	# ifdef USE_AS_STRNCPY
1918	add %rax, %r8
1919	# endif
1920	movaps -14(%rcx), %xmm1
1921
1922	/* 64 bytes loop */
1923	.p2align 4
1924	L(Shl14LoopStart):
1925	movaps 2(%rcx), %xmm2
1926	movaps 18(%rcx), %xmm3
1927	movaps %xmm3, %xmm6
1928	movaps 34(%rcx), %xmm4
1929	movaps %xmm4, %xmm7
1930	movaps 50(%rcx), %xmm5
1931	pminub %xmm2, %xmm6
1932	pminub %xmm5, %xmm7
1933	pminub %xmm6, %xmm7
1934	pcmpeqb %xmm0, %xmm7
1935	pmovmskb %xmm7, %rax
1936	movaps %xmm5, %xmm7
1937	palignr $14, %xmm4, %xmm5
1938	test %rax, %rax
1939	palignr $14, %xmm3, %xmm4
1940	jnz L(Shl14Start)
1941	# ifdef USE_AS_STRNCPY
1942	sub $64, %r8
1943	jbe L(StrncpyLeave14)
1944	# endif
1945	palignr $14, %xmm2, %xmm3
1946	lea 64(%rcx), %rcx
1947	palignr $14, %xmm1, %xmm2
1948	movaps %xmm7, %xmm1
1949	movaps %xmm5, 48(%rdx)
1950	movaps %xmm4, 32(%rdx)
1951	movaps %xmm3, 16(%rdx)
1952	movaps %xmm2, (%rdx)
1953	lea 64(%rdx), %rdx
1954	jmp L(Shl14LoopStart)
1955
1956	L(Shl14LoopExit):
1957	mov -2(%rcx), %r9d
1958	mov $2, %rsi
1959	mov %r9d, -2(%rdx)
1960	jmp L(CopyFrom1To16Bytes)
1961
1962	.p2align 4
1963	L(Shl15):
1964	movaps -15(%rcx), %xmm1
1965	movaps 1(%rcx), %xmm2
1966	L(Shl15Start):
1967	pcmpeqb %xmm2, %xmm0
1968	pmovmskb %xmm0, %rax
1969	movaps %xmm2, %xmm3
1970	# ifdef USE_AS_STRNCPY
1971	sub $16, %r8
1972	jbe L(StrncpyExit15Case2OrCase3)
1973	# endif
1974	test %rax, %rax
1975	jnz L(Shl15LoopExit)
1976
1977	palignr $15, %xmm1, %xmm2
1978	movaps %xmm2, (%rdx)
1979	movaps 17(%rcx), %xmm2
1980
1981	pcmpeqb %xmm2, %xmm0
1982	lea 16(%rdx), %rdx
1983	pmovmskb %xmm0, %rax
1984	lea 16(%rcx), %rcx
1985	movaps %xmm2, %xmm1
1986	# ifdef USE_AS_STRNCPY
1987	sub $16, %r8
1988	jbe L(StrncpyExit15Case2OrCase3)
1989	# endif
1990	test %rax, %rax
1991	jnz L(Shl15LoopExit)
1992
1993	palignr $15, %xmm3, %xmm2
1994	movaps %xmm2, (%rdx)
1995	movaps 17(%rcx), %xmm2
1996
1997	pcmpeqb %xmm2, %xmm0
1998	lea 16(%rdx), %rdx
1999	pmovmskb %xmm0, %rax
2000	lea 16(%rcx), %rcx
2001	movaps %xmm2, %xmm3
2002	# ifdef USE_AS_STRNCPY
2003	sub $16, %r8
2004	jbe L(StrncpyExit15Case2OrCase3)
2005	# endif
2006	test %rax, %rax
2007	jnz L(Shl15LoopExit)
2008
2009	palignr $15, %xmm1, %xmm2
2010	movaps %xmm2, (%rdx)
2011	movaps 17(%rcx), %xmm2
2012
2013	pcmpeqb %xmm2, %xmm0
2014	lea 16(%rdx), %rdx
2015	pmovmskb %xmm0, %rax
2016	lea 16(%rcx), %rcx
2017	# ifdef USE_AS_STRNCPY
2018	sub $16, %r8
2019	jbe L(StrncpyExit15Case2OrCase3)
2020	# endif
2021	test %rax, %rax
2022	jnz L(Shl15LoopExit)
2023
2024	palignr $15, %xmm3, %xmm2
2025	movaps %xmm2, (%rdx)
2026	lea 17(%rcx), %rcx
2027	lea 16(%rdx), %rdx
2028
2029	mov %rcx, %rax
2030	and $-0x40, %rcx
2031	sub %rcx, %rax
2032	lea -1(%rcx), %rcx
2033	sub %rax, %rdx
2034	# ifdef USE_AS_STRNCPY
2035	add %rax, %r8
2036	# endif
2037	movaps -15(%rcx), %xmm1
2038
2039	/* 64 bytes loop */
2040	.p2align 4
2041	L(Shl15LoopStart):
2042	movaps 1(%rcx), %xmm2
2043	movaps 17(%rcx), %xmm3
2044	movaps %xmm3, %xmm6
2045	movaps 33(%rcx), %xmm4
2046	movaps %xmm4, %xmm7
2047	movaps 49(%rcx), %xmm5
2048	pminub %xmm2, %xmm6
2049	pminub %xmm5, %xmm7
2050	pminub %xmm6, %xmm7
2051	pcmpeqb %xmm0, %xmm7
2052	pmovmskb %xmm7, %rax
2053	movaps %xmm5, %xmm7
2054	palignr $15, %xmm4, %xmm5
2055	test %rax, %rax
2056	palignr $15, %xmm3, %xmm4
2057	jnz L(Shl15Start)
2058	# ifdef USE_AS_STRNCPY
2059	sub $64, %r8
2060	jbe L(StrncpyLeave15)
2061	# endif
2062	palignr $15, %xmm2, %xmm3
2063	lea 64(%rcx), %rcx
2064	palignr $15, %xmm1, %xmm2
2065	movaps %xmm7, %xmm1
2066	movaps %xmm5, 48(%rdx)
2067	movaps %xmm4, 32(%rdx)
2068	movaps %xmm3, 16(%rdx)
2069	movaps %xmm2, (%rdx)
2070	lea 64(%rdx), %rdx
2071	jmp L(Shl15LoopStart)
2072
2073	L(Shl15LoopExit):
2074	mov -3(%rcx), %r9d
2075	mov $1, %rsi
2076	mov %r9d, -3(%rdx)
2077	# ifdef USE_AS_STRCAT
2078	jmp L(CopyFrom1To16Bytes)
2079	# endif
2080
2081	# ifndef USE_AS_STRCAT
2082
2083	.p2align 4
2084	L(CopyFrom1To16Bytes):
2085	# ifdef USE_AS_STRNCPY
2086	add $16, %r8
2087	# endif
2088	add %rsi, %rdx
2089	add %rsi, %rcx
2090
2091	test %al, %al
2092	jz L(ExitHigh)
2093	test $0x01, %al
2094	jnz L(Exit1)
2095	test $0x02, %al
2096	jnz L(Exit2)
2097	test $0x04, %al
2098	jnz L(Exit3)
2099	test $0x08, %al
2100	jnz L(Exit4)
2101	test $0x10, %al
2102	jnz L(Exit5)
2103	test $0x20, %al
2104	jnz L(Exit6)
2105	test $0x40, %al
2106	jnz L(Exit7)
2107
2108	.p2align 4
2109	L(Exit8):
2110	mov (%rcx), %rax
2111	mov %rax, (%rdx)
2112	# ifdef USE_AS_STPCPY
2113	lea 7(%rdx), %rax
2114	# else
2115	mov %rdi, %rax
2116	# endif
2117	# ifdef USE_AS_STRNCPY
2118	sub $8, %r8
2119	lea 8(%rdx), %rcx
2120	jnz L(StrncpyFillTailWithZero1)
2121	# ifdef USE_AS_STPCPY
2122	cmpb $1, (%rax)
2123	sbb $-1, %rax
2124	# endif
2125	# endif
2126	ret
2127
2128	.p2align 4
2129	L(ExitHigh):
2130	test $0x01, %ah
2131	jnz L(Exit9)
2132	test $0x02, %ah
2133	jnz L(Exit10)
2134	test $0x04, %ah
2135	jnz L(Exit11)
2136	test $0x08, %ah
2137	jnz L(Exit12)
2138	test $0x10, %ah
2139	jnz L(Exit13)
2140	test $0x20, %ah
2141	jnz L(Exit14)
2142	test $0x40, %ah
2143	jnz L(Exit15)
2144
2145	.p2align 4
2146	L(Exit16):
2147	mov (%rcx), %rax
2148	mov %rax, (%rdx)
2149	mov 8(%rcx), %rax
2150	mov %rax, 8(%rdx)
2151	# ifdef USE_AS_STPCPY
2152	lea 15(%rdx), %rax
2153	# else
2154	mov %rdi, %rax
2155	# endif
2156	# ifdef USE_AS_STRNCPY
2157	sub $16, %r8
2158	lea 16(%rdx), %rcx
2159	jnz L(StrncpyFillTailWithZero1)
2160	# ifdef USE_AS_STPCPY
2161	cmpb $1, (%rax)
2162	sbb $-1, %rax
2163	# endif
2164	# endif
2165	ret
2166
2167	# ifdef USE_AS_STRNCPY
2168
2169	.p2align 4
2170	L(CopyFrom1To16BytesCase2):
2171	add $16, %r8
2172	add %rsi, %rcx
2173	lea (%rsi, %rdx), %rsi
2174	lea -9(%r8), %rdx
2175	and $1<<7, %dh
2176	or %al, %dh
2177	test %dh, %dh
2178	lea (%rsi), %rdx
2179	jz L(ExitHighCase2)
2180
2181	cmp $1, %r8
2182	je L(Exit1)
2183	test $0x01, %al
2184	jnz L(Exit1)
2185	cmp $2, %r8
2186	je L(Exit2)
2187	test $0x02, %al
2188	jnz L(Exit2)
2189	cmp $3, %r8
2190	je L(Exit3)
2191	test $0x04, %al
2192	jnz L(Exit3)
2193	cmp $4, %r8
2194	je L(Exit4)
2195	test $0x08, %al
2196	jnz L(Exit4)
2197	cmp $5, %r8
2198	je L(Exit5)
2199	test $0x10, %al
2200	jnz L(Exit5)
2201	cmp $6, %r8
2202	je L(Exit6)
2203	test $0x20, %al
2204	jnz L(Exit6)
2205	cmp $7, %r8
2206	je L(Exit7)
2207	test $0x40, %al
2208	jnz L(Exit7)
2209	jmp L(Exit8)
2210
2211	.p2align 4
2212	L(ExitHighCase2):
2213	cmp $9, %r8
2214	je L(Exit9)
2215	test $0x01, %ah
2216	jnz L(Exit9)
2217	cmp $10, %r8
2218	je L(Exit10)
2219	test $0x02, %ah
2220	jnz L(Exit10)
2221	cmp $11, %r8
2222	je L(Exit11)
2223	test $0x04, %ah
2224	jnz L(Exit11)
2225	cmp $12, %r8
2226	je L(Exit12)
2227	test $0x8, %ah
2228	jnz L(Exit12)
2229	cmp $13, %r8
2230	je L(Exit13)
2231	test $0x10, %ah
2232	jnz L(Exit13)
2233	cmp $14, %r8
2234	je L(Exit14)
2235	test $0x20, %ah
2236	jnz L(Exit14)
2237	cmp $15, %r8
2238	je L(Exit15)
2239	test $0x40, %ah
2240	jnz L(Exit15)
2241	jmp L(Exit16)
2242
2243	L(CopyFrom1To16BytesCase2OrCase3):
2244	test %rax, %rax
2245	jnz L(CopyFrom1To16BytesCase2)
2246
2247	.p2align 4
2248	L(CopyFrom1To16BytesCase3):
2249	add $16, %r8
2250	add %rsi, %rdx
2251	add %rsi, %rcx
2252
2253	cmp $16, %r8
2254	je L(Exit16)
2255	cmp $8, %r8
2256	je L(Exit8)
2257	jg L(More8Case3)
2258	cmp $4, %r8
2259	je L(Exit4)
2260	jg L(More4Case3)
2261	cmp $2, %r8
2262	jl L(Exit1)
2263	je L(Exit2)
2264	jg L(Exit3)
2265	L(More8Case3): /* but less than 16 */
2266	cmp $12, %r8
2267	je L(Exit12)
2268	jl L(Less12Case3)
2269	cmp $14, %r8
2270	jl L(Exit13)
2271	je L(Exit14)
2272	jg L(Exit15)
2273	L(More4Case3): /* but less than 8 */
2274	cmp $6, %r8
2275	jl L(Exit5)
2276	je L(Exit6)
2277	jg L(Exit7)
2278	L(Less12Case3): /* but more than 8 */
2279	cmp $10, %r8
2280	jl L(Exit9)
2281	je L(Exit10)
2282	jg L(Exit11)
2283	# endif
2284
2285	.p2align 4
2286	L(Exit1):
2287	movb (%rcx), %al
2288	movb %al, (%rdx)
2289	# ifdef USE_AS_STPCPY
2290	lea (%rdx), %rax
2291	# else
2292	mov %rdi, %rax
2293	# endif
2294	# ifdef USE_AS_STRNCPY
2295	sub $1, %r8
2296	lea 1(%rdx), %rcx
2297	jnz L(StrncpyFillTailWithZero1)
2298	# ifdef USE_AS_STPCPY
2299	cmpb $1, (%rax)
2300	sbb $-1, %rax
2301	# endif
2302	# endif
2303	ret
2304
2305	.p2align 4
2306	L(Exit2):
2307	movw (%rcx), %ax
2308	movw %ax, (%rdx)
2309	# ifdef USE_AS_STPCPY
2310	lea 1(%rdx), %rax
2311	# else
2312	mov %rdi, %rax
2313	# endif
2314	# ifdef USE_AS_STRNCPY
2315	sub $2, %r8
2316	lea 2(%rdx), %rcx
2317	jnz L(StrncpyFillTailWithZero1)
2318	# ifdef USE_AS_STPCPY
2319	cmpb $1, (%rax)
2320	sbb $-1, %rax
2321	# endif
2322	# endif
2323	ret
2324
2325	.p2align 4
2326	L(Exit3):
2327	movw (%rcx), %ax
2328	movw %ax, (%rdx)
2329	movb 2(%rcx), %al
2330	movb %al, 2(%rdx)
2331	# ifdef USE_AS_STPCPY
2332	lea 2(%rdx), %rax
2333	# else
2334	mov %rdi, %rax
2335	# endif
2336	# ifdef USE_AS_STRNCPY
2337	sub $3, %r8
2338	lea 3(%rdx), %rcx
2339	jnz L(StrncpyFillTailWithZero1)
2340	# ifdef USE_AS_STPCPY
2341	cmpb $1, (%rax)
2342	sbb $-1, %rax
2343	# endif
2344	# endif
2345	ret
2346
2347	.p2align 4
2348	L(Exit4):
2349	movl (%rcx), %eax
2350	movl %eax, (%rdx)
2351	# ifdef USE_AS_STPCPY
2352	lea 3(%rdx), %rax
2353	# else
2354	mov %rdi, %rax
2355	# endif
2356	# ifdef USE_AS_STRNCPY
2357	sub $4, %r8
2358	lea 4(%rdx), %rcx
2359	jnz L(StrncpyFillTailWithZero1)
2360	# ifdef USE_AS_STPCPY
2361	cmpb $1, (%rax)
2362	sbb $-1, %rax
2363	# endif
2364	# endif
2365	ret
2366
2367	.p2align 4
2368	L(Exit5):
2369	movl (%rcx), %eax
2370	movl %eax, (%rdx)
2371	movb 4(%rcx), %al
2372	movb %al, 4(%rdx)
2373	# ifdef USE_AS_STPCPY
2374	lea 4(%rdx), %rax
2375	# else
2376	mov %rdi, %rax
2377	# endif
2378	# ifdef USE_AS_STRNCPY
2379	sub $5, %r8
2380	lea 5(%rdx), %rcx
2381	jnz L(StrncpyFillTailWithZero1)
2382	# ifdef USE_AS_STPCPY
2383	cmpb $1, (%rax)
2384	sbb $-1, %rax
2385	# endif
2386	# endif
2387	ret
2388
2389	.p2align 4
2390	L(Exit6):
2391	movl (%rcx), %eax
2392	movl %eax, (%rdx)
2393	movw 4(%rcx), %ax
2394	movw %ax, 4(%rdx)
2395	# ifdef USE_AS_STPCPY
2396	lea 5(%rdx), %rax
2397	# else
2398	mov %rdi, %rax
2399	# endif
2400	# ifdef USE_AS_STRNCPY
2401	sub $6, %r8
2402	lea 6(%rdx), %rcx
2403	jnz L(StrncpyFillTailWithZero1)
2404	# ifdef USE_AS_STPCPY
2405	cmpb $1, (%rax)
2406	sbb $-1, %rax
2407	# endif
2408	# endif
2409	ret
2410
2411	.p2align 4
2412	L(Exit7):
2413	movl (%rcx), %eax
2414	movl %eax, (%rdx)
2415	movl 3(%rcx), %eax
2416	movl %eax, 3(%rdx)
2417	# ifdef USE_AS_STPCPY
2418	lea 6(%rdx), %rax
2419	# else
2420	mov %rdi, %rax
2421	# endif
2422	# ifdef USE_AS_STRNCPY
2423	sub $7, %r8
2424	lea 7(%rdx), %rcx
2425	jnz L(StrncpyFillTailWithZero1)
2426	# ifdef USE_AS_STPCPY
2427	cmpb $1, (%rax)
2428	sbb $-1, %rax
2429	# endif
2430	# endif
2431	ret
2432
2433	.p2align 4
2434	L(Exit9):
2435	mov (%rcx), %rax
2436	mov %rax, (%rdx)
2437	mov 5(%rcx), %eax
2438	mov %eax, 5(%rdx)
2439	# ifdef USE_AS_STPCPY
2440	lea 8(%rdx), %rax
2441	# else
2442	mov %rdi, %rax
2443	# endif
2444	# ifdef USE_AS_STRNCPY
2445	sub $9, %r8
2446	lea 9(%rdx), %rcx
2447	jnz L(StrncpyFillTailWithZero1)
2448	# ifdef USE_AS_STPCPY
2449	cmpb $1, (%rax)
2450	sbb $-1, %rax
2451	# endif
2452	# endif
2453	ret
2454
2455	.p2align 4
2456	L(Exit10):
2457	mov (%rcx), %rax
2458	mov %rax, (%rdx)
2459	mov 6(%rcx), %eax
2460	mov %eax, 6(%rdx)
2461	# ifdef USE_AS_STPCPY
2462	lea 9(%rdx), %rax
2463	# else
2464	mov %rdi, %rax
2465	# endif
2466	# ifdef USE_AS_STRNCPY
2467	sub $10, %r8
2468	lea 10(%rdx), %rcx
2469	jnz L(StrncpyFillTailWithZero1)
2470	# ifdef USE_AS_STPCPY
2471	cmpb $1, (%rax)
2472	sbb $-1, %rax
2473	# endif
2474	# endif
2475	ret
2476
2477	.p2align 4
2478	L(Exit11):
2479	mov (%rcx), %rax
2480	mov %rax, (%rdx)
2481	mov 7(%rcx), %eax
2482	mov %eax, 7(%rdx)
2483	# ifdef USE_AS_STPCPY
2484	lea 10(%rdx), %rax
2485	# else
2486	mov %rdi, %rax
2487	# endif
2488	# ifdef USE_AS_STRNCPY
2489	sub $11, %r8
2490	lea 11(%rdx), %rcx
2491	jnz L(StrncpyFillTailWithZero1)
2492	# ifdef USE_AS_STPCPY
2493	cmpb $1, (%rax)
2494	sbb $-1, %rax
2495	# endif
2496	# endif
2497	ret
2498
2499	.p2align 4
2500	L(Exit12):
2501	mov (%rcx), %rax
2502	mov %rax, (%rdx)
2503	mov 8(%rcx), %eax
2504	mov %eax, 8(%rdx)
2505	# ifdef USE_AS_STPCPY
2506	lea 11(%rdx), %rax
2507	# else
2508	mov %rdi, %rax
2509	# endif
2510	# ifdef USE_AS_STRNCPY
2511	sub $12, %r8
2512	lea 12(%rdx), %rcx
2513	jnz L(StrncpyFillTailWithZero1)
2514	# ifdef USE_AS_STPCPY
2515	cmpb $1, (%rax)
2516	sbb $-1, %rax
2517	# endif
2518	# endif
2519	ret
2520
2521	.p2align 4
2522	L(Exit13):
2523	mov (%rcx), %rax
2524	mov %rax, (%rdx)
2525	mov 5(%rcx), %rax
2526	mov %rax, 5(%rdx)
2527	# ifdef USE_AS_STPCPY
2528	lea 12(%rdx), %rax
2529	# else
2530	mov %rdi, %rax
2531	# endif
2532	# ifdef USE_AS_STRNCPY
2533	sub $13, %r8
2534	lea 13(%rdx), %rcx
2535	jnz L(StrncpyFillTailWithZero1)
2536	# ifdef USE_AS_STPCPY
2537	cmpb $1, (%rax)
2538	sbb $-1, %rax
2539	# endif
2540	# endif
2541	ret
2542
2543	.p2align 4
2544	L(Exit14):
2545	mov (%rcx), %rax
2546	mov %rax, (%rdx)
2547	mov 6(%rcx), %rax
2548	mov %rax, 6(%rdx)
2549	# ifdef USE_AS_STPCPY
2550	lea 13(%rdx), %rax
2551	# else
2552	mov %rdi, %rax
2553	# endif
2554	# ifdef USE_AS_STRNCPY
2555	sub $14, %r8
2556	lea 14(%rdx), %rcx
2557	jnz L(StrncpyFillTailWithZero1)
2558	# ifdef USE_AS_STPCPY
2559	cmpb $1, (%rax)
2560	sbb $-1, %rax
2561	# endif
2562	# endif
2563	ret
2564
2565	.p2align 4
2566	L(Exit15):
2567	mov (%rcx), %rax
2568	mov %rax, (%rdx)
2569	mov 7(%rcx), %rax
2570	mov %rax, 7(%rdx)
2571	# ifdef USE_AS_STPCPY
2572	lea 14(%rdx), %rax
2573	# else
2574	mov %rdi, %rax
2575	# endif
2576	# ifdef USE_AS_STRNCPY
2577	sub $15, %r8
2578	lea 15(%rdx), %rcx
2579	jnz L(StrncpyFillTailWithZero1)
2580	# ifdef USE_AS_STPCPY
2581	cmpb $1, (%rax)
2582	sbb $-1, %rax
2583	# endif
2584	# endif
2585	ret
2586
2587	# ifdef USE_AS_STRNCPY
2588	.p2align 4
2589	L(Fill0):
2590	ret
2591
2592	.p2align 4
2593	L(Fill1):
2594	movb %dl, (%rcx)
2595	ret
2596
2597	.p2align 4
2598	L(Fill2):
2599	movw %dx, (%rcx)
2600	ret
2601
2602	.p2align 4
2603	L(Fill3):
2604	movw %dx, (%rcx)
2605	movb %dl, 2(%rcx)
2606	ret
2607
2608	.p2align 4
2609	L(Fill4):
2610	movl %edx, (%rcx)
2611	ret
2612
2613	.p2align 4
2614	L(Fill5):
2615	movl %edx, (%rcx)
2616	movb %dl, 4(%rcx)
2617	ret
2618
2619	.p2align 4
2620	L(Fill6):
2621	movl %edx, (%rcx)
2622	movw %dx, 4(%rcx)
2623	ret
2624
2625	.p2align 4
2626	L(Fill7):
2627	movl %edx, (%rcx)
2628	movl %edx, 3(%rcx)
2629	ret
2630
2631	.p2align 4
2632	L(Fill8):
2633	mov %rdx, (%rcx)
2634	ret
2635
2636	.p2align 4
2637	L(Fill9):
2638	mov %rdx, (%rcx)
2639	movb %dl, 8(%rcx)
2640	ret
2641
2642	.p2align 4
2643	L(Fill10):
2644	mov %rdx, (%rcx)
2645	movw %dx, 8(%rcx)
2646	ret
2647
2648	.p2align 4
2649	L(Fill11):
2650	mov %rdx, (%rcx)
2651	movl %edx, 7(%rcx)
2652	ret
2653
2654	.p2align 4
2655	L(Fill12):
2656	mov %rdx, (%rcx)
2657	movl %edx, 8(%rcx)
2658	ret
2659
2660	.p2align 4
2661	L(Fill13):
2662	mov %rdx, (%rcx)
2663	mov %rdx, 5(%rcx)
2664	ret
2665
2666	.p2align 4
2667	L(Fill14):
2668	mov %rdx, (%rcx)
2669	mov %rdx, 6(%rcx)
2670	ret
2671
2672	.p2align 4
2673	L(Fill15):
2674	mov %rdx, (%rcx)
2675	mov %rdx, 7(%rcx)
2676	ret
2677
2678	.p2align 4
2679	L(Fill16):
2680	mov %rdx, (%rcx)
2681	mov %rdx, 8(%rcx)
2682	ret
2683
2684	.p2align 4
2685	L(StrncpyFillExit1):
2686	lea 16(%r8), %r8
2687	L(FillFrom1To16Bytes):
2688	test %r8, %r8
2689	jz L(Fill0)
2690	cmp $16, %r8
2691	je L(Fill16)
2692	cmp $8, %r8
2693	je L(Fill8)
2694	jg L(FillMore8)
2695	cmp $4, %r8
2696	je L(Fill4)
2697	jg L(FillMore4)
2698	cmp $2, %r8
2699	jl L(Fill1)
2700	je L(Fill2)
2701	jg L(Fill3)
2702	L(FillMore8): /* but less than 16 */
2703	cmp $12, %r8
2704	je L(Fill12)
2705	jl L(FillLess12)
2706	cmp $14, %r8
2707	jl L(Fill13)
2708	je L(Fill14)
2709	jg L(Fill15)
2710	L(FillMore4): /* but less than 8 */
2711	cmp $6, %r8
2712	jl L(Fill5)
2713	je L(Fill6)
2714	jg L(Fill7)
2715	L(FillLess12): /* but more than 8 */
2716	cmp $10, %r8
2717	jl L(Fill9)
2718	je L(Fill10)
2719	jmp L(Fill11)
2720
2721	.p2align 4
2722	L(StrncpyFillTailWithZero1):
2723	xor %rdx, %rdx
2724	sub $16, %r8
2725	jbe L(StrncpyFillExit1)
2726
2727	pxor %xmm0, %xmm0
2728	mov %rdx, (%rcx)
2729	mov %rdx, 8(%rcx)
2730
2731	lea 16(%rcx), %rcx
2732
2733	mov %rcx, %rdx
2734	and $0xf, %rdx
2735	sub %rdx, %rcx
2736	add %rdx, %r8
2737	xor %rdx, %rdx
2738	sub $64, %r8
2739	jb L(StrncpyFillLess64)
2740
2741	L(StrncpyFillLoopMovdqa):
2742	movdqa %xmm0, (%rcx)
2743	movdqa %xmm0, 16(%rcx)
2744	movdqa %xmm0, 32(%rcx)
2745	movdqa %xmm0, 48(%rcx)
2746	lea 64(%rcx), %rcx
2747	sub $64, %r8
2748	jae L(StrncpyFillLoopMovdqa)
2749
2750	L(StrncpyFillLess64):
2751	add $32, %r8
2752	jl L(StrncpyFillLess32)
2753	movdqa %xmm0, (%rcx)
2754	movdqa %xmm0, 16(%rcx)
2755	lea 32(%rcx), %rcx
2756	sub $16, %r8
2757	jl L(StrncpyFillExit1)
2758	movdqa %xmm0, (%rcx)
2759	lea 16(%rcx), %rcx
2760	jmp L(FillFrom1To16Bytes)
2761
2762	L(StrncpyFillLess32):
2763	add $16, %r8
2764	jl L(StrncpyFillExit1)
2765	movdqa %xmm0, (%rcx)
2766	lea 16(%rcx), %rcx
2767	jmp L(FillFrom1To16Bytes)
2768
2769	.p2align 4
2770	L(Exit0):
2771	mov %rdx, %rax
2772	ret
2773
2774	.p2align 4
2775	L(StrncpyExit15Bytes):
2776	cmp $9, %r8
2777	je L(Exit9)
2778	cmpb $0, 8(%rcx)
2779	jz L(Exit9)
2780	cmp $10, %r8
2781	je L(Exit10)
2782	cmpb $0, 9(%rcx)
2783	jz L(Exit10)
2784	cmp $11, %r8
2785	je L(Exit11)
2786	cmpb $0, 10(%rcx)
2787	jz L(Exit11)
2788	cmp $12, %r8
2789	je L(Exit12)
2790	cmpb $0, 11(%rcx)
2791	jz L(Exit12)
2792	cmp $13, %r8
2793	je L(Exit13)
2794	cmpb $0, 12(%rcx)
2795	jz L(Exit13)
2796	cmp $14, %r8
2797	je L(Exit14)
2798	cmpb $0, 13(%rcx)
2799	jz L(Exit14)
2800	mov (%rcx), %rax
2801	mov %rax, (%rdx)
2802	mov 7(%rcx), %rax
2803	mov %rax, 7(%rdx)
2804	# ifdef USE_AS_STPCPY
2805	lea 14(%rdx), %rax
2806	cmpb $1, (%rax)
2807	sbb $-1, %rax
2808	# else
2809	mov %rdi, %rax
2810	# endif
2811	ret
2812
2813	.p2align 4
2814	L(StrncpyExit8Bytes):
2815	cmp $1, %r8
2816	je L(Exit1)
2817	cmpb $0, (%rcx)
2818	jz L(Exit1)
2819	cmp $2, %r8
2820	je L(Exit2)
2821	cmpb $0, 1(%rcx)
2822	jz L(Exit2)
2823	cmp $3, %r8
2824	je L(Exit3)
2825	cmpb $0, 2(%rcx)
2826	jz L(Exit3)
2827	cmp $4, %r8
2828	je L(Exit4)
2829	cmpb $0, 3(%rcx)
2830	jz L(Exit4)
2831	cmp $5, %r8
2832	je L(Exit5)
2833	cmpb $0, 4(%rcx)
2834	jz L(Exit5)
2835	cmp $6, %r8
2836	je L(Exit6)
2837	cmpb $0, 5(%rcx)
2838	jz L(Exit6)
2839	cmp $7, %r8
2840	je L(Exit7)
2841	cmpb $0, 6(%rcx)
2842	jz L(Exit7)
2843	mov (%rcx), %rax
2844	mov %rax, (%rdx)
2845	# ifdef USE_AS_STPCPY
2846	lea 7(%rdx), %rax
2847	cmpb $1, (%rax)
2848	sbb $-1, %rax
2849	# else
2850	mov %rdi, %rax
2851	# endif
2852	ret
2853
2854	# endif
2855	# endif
2856
2857	# ifdef USE_AS_STRNCPY
2858	.p2align 4
2859	L(StrncpyLeaveCase2OrCase3):
2860	test %rax, %rax
2861	jnz L(Aligned64LeaveCase2)
2862
2863	L(Aligned64LeaveCase3):
2864	lea 64(%r8), %r8
2865	sub $16, %r8
2866	jbe L(CopyFrom1To16BytesCase3)
2867	movaps %xmm4, -64(%rdx)
2868	lea 16(%rsi), %rsi
2869	sub $16, %r8
2870	jbe L(CopyFrom1To16BytesCase3)
2871	movaps %xmm5, -48(%rdx)
2872	lea 16(%rsi), %rsi
2873	sub $16, %r8
2874	jbe L(CopyFrom1To16BytesCase3)
2875	movaps %xmm6, -32(%rdx)
2876	lea 16(%rsi), %rsi
2877	lea -16(%r8), %r8
2878	jmp L(CopyFrom1To16BytesCase3)
2879
2880	L(Aligned64LeaveCase2):
2881	pcmpeqb %xmm4, %xmm0
2882	pmovmskb %xmm0, %rax
2883	add $48, %r8
2884	jle L(CopyFrom1To16BytesCase2OrCase3)
2885	test %rax, %rax
2886	jnz L(CopyFrom1To16Bytes)
2887
2888	pcmpeqb %xmm5, %xmm0
2889	pmovmskb %xmm0, %rax
2890	movaps %xmm4, -64(%rdx)
2891	lea 16(%rsi), %rsi
2892	sub $16, %r8
2893	jbe L(CopyFrom1To16BytesCase2OrCase3)
2894	test %rax, %rax
2895	jnz L(CopyFrom1To16Bytes)
2896
2897	pcmpeqb %xmm6, %xmm0
2898	pmovmskb %xmm0, %rax
2899	movaps %xmm5, -48(%rdx)
2900	lea 16(%rsi), %rsi
2901	sub $16, %r8
2902	jbe L(CopyFrom1To16BytesCase2OrCase3)
2903	test %rax, %rax
2904	jnz L(CopyFrom1To16Bytes)
2905
2906	pcmpeqb %xmm7, %xmm0
2907	pmovmskb %xmm0, %rax
2908	movaps %xmm6, -32(%rdx)
2909	lea 16(%rsi), %rsi
2910	lea -16(%r8), %r8
2911	jmp L(CopyFrom1To16BytesCase2)
2912	/*--------------------------------------------------*/
2913	.p2align 4
2914	L(StrncpyExit1Case2OrCase3):
2915	movdqu -1(%rcx), %xmm0
2916	movdqu %xmm0, -1(%rdx)
2917	mov $15, %rsi
2918	test %rax, %rax
2919	jnz L(CopyFrom1To16BytesCase2)
2920	jmp L(CopyFrom1To16BytesCase3)
2921
2922	.p2align 4
2923	L(StrncpyExit2Case2OrCase3):
2924	movdqu -2(%rcx), %xmm0
2925	movdqu %xmm0, -2(%rdx)
2926	mov $14, %rsi
2927	test %rax, %rax
2928	jnz L(CopyFrom1To16BytesCase2)
2929	jmp L(CopyFrom1To16BytesCase3)
2930
2931	.p2align 4
2932	L(StrncpyExit3Case2OrCase3):
2933	movdqu -3(%rcx), %xmm0
2934	movdqu %xmm0, -3(%rdx)
2935	mov $13, %rsi
2936	test %rax, %rax
2937	jnz L(CopyFrom1To16BytesCase2)
2938	jmp L(CopyFrom1To16BytesCase3)
2939
2940	.p2align 4
2941	L(StrncpyExit4Case2OrCase3):
2942	movdqu -4(%rcx), %xmm0
2943	movdqu %xmm0, -4(%rdx)
2944	mov $12, %rsi
2945	test %rax, %rax
2946	jnz L(CopyFrom1To16BytesCase2)
2947	jmp L(CopyFrom1To16BytesCase3)
2948
2949	.p2align 4
2950	L(StrncpyExit5Case2OrCase3):
2951	movdqu -5(%rcx), %xmm0
2952	movdqu %xmm0, -5(%rdx)
2953	mov $11, %rsi
2954	test %rax, %rax
2955	jnz L(CopyFrom1To16BytesCase2)
2956	jmp L(CopyFrom1To16BytesCase3)
2957
2958	.p2align 4
2959	L(StrncpyExit6Case2OrCase3):
2960	mov (%rcx), %rsi
2961	mov 6(%rcx), %r9d
2962	mov %r9d, 6(%rdx)
2963	mov %rsi, (%rdx)
2964	test %rax, %rax
2965	mov $10, %rsi
2966	jnz L(CopyFrom1To16BytesCase2)
2967	jmp L(CopyFrom1To16BytesCase3)
2968
2969	.p2align 4
2970	L(StrncpyExit7Case2OrCase3):
2971	mov (%rcx), %rsi
2972	mov 5(%rcx), %r9d
2973	mov %r9d, 5(%rdx)
2974	mov %rsi, (%rdx)
2975	test %rax, %rax
2976	mov $9, %rsi
2977	jnz L(CopyFrom1To16BytesCase2)
2978	jmp L(CopyFrom1To16BytesCase3)
2979
2980	.p2align 4
2981	L(StrncpyExit8Case2OrCase3):
2982	mov (%rcx), %r9
2983	mov $8, %rsi
2984	mov %r9, (%rdx)
2985	test %rax, %rax
2986	jnz L(CopyFrom1To16BytesCase2)
2987	jmp L(CopyFrom1To16BytesCase3)
2988
2989	.p2align 4
2990	L(StrncpyExit9Case2OrCase3):
2991	mov -1(%rcx), %r9
2992	mov $7, %rsi
2993	mov %r9, -1(%rdx)
2994	test %rax, %rax
2995	jnz L(CopyFrom1To16BytesCase2)
2996	jmp L(CopyFrom1To16BytesCase3)
2997
2998	.p2align 4
2999	L(StrncpyExit10Case2OrCase3):
3000	mov -2(%rcx), %r9
3001	mov $6, %rsi
3002	mov %r9, -2(%rdx)
3003	test %rax, %rax
3004	jnz L(CopyFrom1To16BytesCase2)
3005	jmp L(CopyFrom1To16BytesCase3)
3006
3007	.p2align 4
3008	L(StrncpyExit11Case2OrCase3):
3009	mov -3(%rcx), %r9
3010	mov $5, %rsi
3011	mov %r9, -3(%rdx)
3012	test %rax, %rax
3013	jnz L(CopyFrom1To16BytesCase2)
3014	jmp L(CopyFrom1To16BytesCase3)
3015
3016	.p2align 4
3017	L(StrncpyExit12Case2OrCase3):
3018	mov (%rcx), %r9d
3019	mov $4, %rsi
3020	mov %r9d, (%rdx)
3021	test %rax, %rax
3022	jnz L(CopyFrom1To16BytesCase2)
3023	jmp L(CopyFrom1To16BytesCase3)
3024
3025	.p2align 4
3026	L(StrncpyExit13Case2OrCase3):
3027	mov -1(%rcx), %r9d
3028	mov $3, %rsi
3029	mov %r9d, -1(%rdx)
3030	test %rax, %rax
3031	jnz L(CopyFrom1To16BytesCase2)
3032	jmp L(CopyFrom1To16BytesCase3)
3033
3034	.p2align 4
3035	L(StrncpyExit14Case2OrCase3):
3036	mov -2(%rcx), %r9d
3037	mov $2, %rsi
3038	mov %r9d, -2(%rdx)
3039	test %rax, %rax
3040	jnz L(CopyFrom1To16BytesCase2)
3041	jmp L(CopyFrom1To16BytesCase3)
3042
3043	.p2align 4
3044	L(StrncpyExit15Case2OrCase3):
3045	mov -3(%rcx), %r9d
3046	mov $1, %rsi
3047	mov %r9d, -3(%rdx)
3048	test %rax, %rax
3049	jnz L(CopyFrom1To16BytesCase2)
3050	jmp L(CopyFrom1To16BytesCase3)
3051
3052	.p2align 4
3053	L(StrncpyLeave1):
3054	movaps %xmm2, %xmm3
3055	add $48, %r8
3056	jle L(StrncpyExit1)
3057	palignr $1, %xmm1, %xmm2
3058	movaps %xmm2, (%rdx)
3059	movaps 31(%rcx), %xmm2
3060	lea 16(%rsi), %rsi
3061	sub $16, %r8
3062	jbe L(StrncpyExit1)
3063	palignr $1, %xmm3, %xmm2
3064	movaps %xmm2, 16(%rdx)
3065	lea 16(%rsi), %rsi
3066	sub $16, %r8
3067	jbe L(StrncpyExit1)
3068	movaps %xmm4, 32(%rdx)
3069	lea 16(%rsi), %rsi
3070	sub $16, %r8
3071	jbe L(StrncpyExit1)
3072	movaps %xmm5, 48(%rdx)
3073	lea 16(%rsi), %rsi
3074	lea -16(%r8), %r8
3075
3076	L(StrncpyExit1):
3077	lea 15(%rdx, %rsi), %rdx
3078	lea 15(%rcx, %rsi), %rcx
3079	mov -15(%rcx), %rsi
3080	mov -8(%rcx), %rax
3081	mov %rsi, -15(%rdx)
3082	mov %rax, -8(%rdx)
3083	xor %rsi, %rsi
3084	jmp L(CopyFrom1To16BytesCase3)
3085
3086	.p2align 4
3087	L(StrncpyLeave2):
3088	movaps %xmm2, %xmm3
3089	add $48, %r8
3090	jle L(StrncpyExit2)
3091	palignr $2, %xmm1, %xmm2
3092	movaps %xmm2, (%rdx)
3093	movaps 30(%rcx), %xmm2
3094	lea 16(%rsi), %rsi
3095	sub $16, %r8
3096	jbe L(StrncpyExit2)
3097	palignr $2, %xmm3, %xmm2
3098	movaps %xmm2, 16(%rdx)
3099	lea 16(%rsi), %rsi
3100	sub $16, %r8
3101	jbe L(StrncpyExit2)
3102	movaps %xmm4, 32(%rdx)
3103	lea 16(%rsi), %rsi
3104	sub $16, %r8
3105	jbe L(StrncpyExit2)
3106	movaps %xmm5, 48(%rdx)
3107	lea 16(%rsi), %rsi
3108	lea -16(%r8), %r8
3109
3110	L(StrncpyExit2):
3111	lea 14(%rdx, %rsi), %rdx
3112	lea 14(%rcx, %rsi), %rcx
3113	mov -14(%rcx), %rsi
3114	mov -8(%rcx), %rax
3115	mov %rsi, -14(%rdx)
3116	mov %rax, -8(%rdx)
3117	xor %rsi, %rsi
3118	jmp L(CopyFrom1To16BytesCase3)
3119
3120	.p2align 4
3121	L(StrncpyLeave3):
3122	movaps %xmm2, %xmm3
3123	add $48, %r8
3124	jle L(StrncpyExit3)
3125	palignr $3, %xmm1, %xmm2
3126	movaps %xmm2, (%rdx)
3127	movaps 29(%rcx), %xmm2
3128	lea 16(%rsi), %rsi
3129	sub $16, %r8
3130	jbe L(StrncpyExit3)
3131	palignr $3, %xmm3, %xmm2
3132	movaps %xmm2, 16(%rdx)
3133	lea 16(%rsi), %rsi
3134	sub $16, %r8
3135	jbe L(StrncpyExit3)
3136	movaps %xmm4, 32(%rdx)
3137	lea 16(%rsi), %rsi
3138	sub $16, %r8
3139	jbe L(StrncpyExit3)
3140	movaps %xmm5, 48(%rdx)
3141	lea 16(%rsi), %rsi
3142	lea -16(%r8), %r8
3143
3144	L(StrncpyExit3):
3145	lea 13(%rdx, %rsi), %rdx
3146	lea 13(%rcx, %rsi), %rcx
3147	mov -13(%rcx), %rsi
3148	mov -8(%rcx), %rax
3149	mov %rsi, -13(%rdx)
3150	mov %rax, -8(%rdx)
3151	xor %rsi, %rsi
3152	jmp L(CopyFrom1To16BytesCase3)
3153
3154	.p2align 4
3155	L(StrncpyLeave4):
3156	movaps %xmm2, %xmm3
3157	add $48, %r8
3158	jle L(StrncpyExit4)
3159	palignr $4, %xmm1, %xmm2
3160	movaps %xmm2, (%rdx)
3161	movaps 28(%rcx), %xmm2
3162	lea 16(%rsi), %rsi
3163	sub $16, %r8
3164	jbe L(StrncpyExit4)
3165	palignr $4, %xmm3, %xmm2
3166	movaps %xmm2, 16(%rdx)
3167	lea 16(%rsi), %rsi
3168	sub $16, %r8
3169	jbe L(StrncpyExit4)
3170	movaps %xmm4, 32(%rdx)
3171	lea 16(%rsi), %rsi
3172	sub $16, %r8
3173	jbe L(StrncpyExit4)
3174	movaps %xmm5, 48(%rdx)
3175	lea 16(%rsi), %rsi
3176	lea -16(%r8), %r8
3177
3178	L(StrncpyExit4):
3179	lea 12(%rdx, %rsi), %rdx
3180	lea 12(%rcx, %rsi), %rcx
3181	mov -12(%rcx), %rsi
3182	mov -4(%rcx), %eax
3183	mov %rsi, -12(%rdx)
3184	mov %eax, -4(%rdx)
3185	xor %rsi, %rsi
3186	jmp L(CopyFrom1To16BytesCase3)
3187
3188	.p2align 4
3189	L(StrncpyLeave5):
3190	movaps %xmm2, %xmm3
3191	add $48, %r8
3192	jle L(StrncpyExit5)
3193	palignr $5, %xmm1, %xmm2
3194	movaps %xmm2, (%rdx)
3195	movaps 27(%rcx), %xmm2
3196	lea 16(%rsi), %rsi
3197	sub $16, %r8
3198	jbe L(StrncpyExit5)
3199	palignr $5, %xmm3, %xmm2
3200	movaps %xmm2, 16(%rdx)
3201	lea 16(%rsi), %rsi
3202	sub $16, %r8
3203	jbe L(StrncpyExit5)
3204	movaps %xmm4, 32(%rdx)
3205	lea 16(%rsi), %rsi
3206	sub $16, %r8
3207	jbe L(StrncpyExit5)
3208	movaps %xmm5, 48(%rdx)
3209	lea 16(%rsi), %rsi
3210	lea -16(%r8), %r8
3211
3212	L(StrncpyExit5):
3213	lea 11(%rdx, %rsi), %rdx
3214	lea 11(%rcx, %rsi), %rcx
3215	mov -11(%rcx), %rsi
3216	mov -4(%rcx), %eax
3217	mov %rsi, -11(%rdx)
3218	mov %eax, -4(%rdx)
3219	xor %rsi, %rsi
3220	jmp L(CopyFrom1To16BytesCase3)
3221
3222	.p2align 4
3223	L(StrncpyLeave6):
3224	movaps %xmm2, %xmm3
3225	add $48, %r8
3226	jle L(StrncpyExit6)
3227	palignr $6, %xmm1, %xmm2
3228	movaps %xmm2, (%rdx)
3229	movaps 26(%rcx), %xmm2
3230	lea 16(%rsi), %rsi
3231	sub $16, %r8
3232	jbe L(StrncpyExit6)
3233	palignr $6, %xmm3, %xmm2
3234	movaps %xmm2, 16(%rdx)
3235	lea 16(%rsi), %rsi
3236	sub $16, %r8
3237	jbe L(StrncpyExit6)
3238	movaps %xmm4, 32(%rdx)
3239	lea 16(%rsi), %rsi
3240	sub $16, %r8
3241	jbe L(StrncpyExit6)
3242	movaps %xmm5, 48(%rdx)
3243	lea 16(%rsi), %rsi
3244	lea -16(%r8), %r8
3245
3246	L(StrncpyExit6):
3247	lea 10(%rdx, %rsi), %rdx
3248	lea 10(%rcx, %rsi), %rcx
3249	mov -10(%rcx), %rsi
3250	movw -2(%rcx), %ax
3251	mov %rsi, -10(%rdx)
3252	movw %ax, -2(%rdx)
3253	xor %rsi, %rsi
3254	jmp L(CopyFrom1To16BytesCase3)
3255
3256	.p2align 4
3257	L(StrncpyLeave7):
3258	movaps %xmm2, %xmm3
3259	add $48, %r8
3260	jle L(StrncpyExit7)
3261	palignr $7, %xmm1, %xmm2
3262	movaps %xmm2, (%rdx)
3263	movaps 25(%rcx), %xmm2
3264	lea 16(%rsi), %rsi
3265	sub $16, %r8
3266	jbe L(StrncpyExit7)
3267	palignr $7, %xmm3, %xmm2
3268	movaps %xmm2, 16(%rdx)
3269	lea 16(%rsi), %rsi
3270	sub $16, %r8
3271	jbe L(StrncpyExit7)
3272	movaps %xmm4, 32(%rdx)
3273	lea 16(%rsi), %rsi
3274	sub $16, %r8
3275	jbe L(StrncpyExit7)
3276	movaps %xmm5, 48(%rdx)
3277	lea 16(%rsi), %rsi
3278	lea -16(%r8), %r8
3279
3280	L(StrncpyExit7):
3281	lea 9(%rdx, %rsi), %rdx
3282	lea 9(%rcx, %rsi), %rcx
3283	mov -9(%rcx), %rsi
3284	movb -1(%rcx), %ah
3285	mov %rsi, -9(%rdx)
3286	movb %ah, -1(%rdx)
3287	xor %rsi, %rsi
3288	jmp L(CopyFrom1To16BytesCase3)
3289
3290	.p2align 4
3291	L(StrncpyLeave8):
3292	movaps %xmm2, %xmm3
3293	add $48, %r8
3294	jle L(StrncpyExit8)
3295	palignr $8, %xmm1, %xmm2
3296	movaps %xmm2, (%rdx)
3297	movaps 24(%rcx), %xmm2
3298	lea 16(%rsi), %rsi
3299	sub $16, %r8
3300	jbe L(StrncpyExit8)
3301	palignr $8, %xmm3, %xmm2
3302	movaps %xmm2, 16(%rdx)
3303	lea 16(%rsi), %rsi
3304	sub $16, %r8
3305	jbe L(StrncpyExit8)
3306	movaps %xmm4, 32(%rdx)
3307	lea 16(%rsi), %rsi
3308	sub $16, %r8
3309	jbe L(StrncpyExit8)
3310	movaps %xmm5, 48(%rdx)
3311	lea 16(%rsi), %rsi
3312	lea -16(%r8), %r8
3313
3314	L(StrncpyExit8):
3315	lea 8(%rdx, %rsi), %rdx
3316	lea 8(%rcx, %rsi), %rcx
3317	mov -8(%rcx), %rax
3318	xor %rsi, %rsi
3319	mov %rax, -8(%rdx)
3320	jmp L(CopyFrom1To16BytesCase3)
3321
3322	.p2align 4
3323	L(StrncpyLeave9):
3324	movaps %xmm2, %xmm3
3325	add $48, %r8
3326	jle L(StrncpyExit9)
3327	palignr $9, %xmm1, %xmm2
3328	movaps %xmm2, (%rdx)
3329	movaps 23(%rcx), %xmm2
3330	lea 16(%rsi), %rsi
3331	sub $16, %r8
3332	jbe L(StrncpyExit9)
3333	palignr $9, %xmm3, %xmm2
3334	movaps %xmm2, 16(%rdx)
3335	lea 16(%rsi), %rsi
3336	sub $16, %r8
3337	jbe L(StrncpyExit9)
3338	movaps %xmm4, 32(%rdx)
3339	lea 16(%rsi), %rsi
3340	sub $16, %r8
3341	jbe L(StrncpyExit9)
3342	movaps %xmm5, 48(%rdx)
3343	lea 16(%rsi), %rsi
3344	lea -16(%r8), %r8
3345
3346	L(StrncpyExit9):
3347	lea 7(%rdx, %rsi), %rdx
3348	lea 7(%rcx, %rsi), %rcx
3349	mov -8(%rcx), %rax
3350	xor %rsi, %rsi
3351	mov %rax, -8(%rdx)
3352	jmp L(CopyFrom1To16BytesCase3)
3353
3354	.p2align 4
3355	L(StrncpyLeave10):
3356	movaps %xmm2, %xmm3
3357	add $48, %r8
3358	jle L(StrncpyExit10)
3359	palignr $10, %xmm1, %xmm2
3360	movaps %xmm2, (%rdx)
3361	movaps 22(%rcx), %xmm2
3362	lea 16(%rsi), %rsi
3363	sub $16, %r8
3364	jbe L(StrncpyExit10)
3365	palignr $10, %xmm3, %xmm2
3366	movaps %xmm2, 16(%rdx)
3367	lea 16(%rsi), %rsi
3368	sub $16, %r8
3369	jbe L(StrncpyExit10)
3370	movaps %xmm4, 32(%rdx)
3371	lea 16(%rsi), %rsi
3372	sub $16, %r8
3373	jbe L(StrncpyExit10)
3374	movaps %xmm5, 48(%rdx)
3375	lea 16(%rsi), %rsi
3376	lea -16(%r8), %r8
3377
3378	L(StrncpyExit10):
3379	lea 6(%rdx, %rsi), %rdx
3380	lea 6(%rcx, %rsi), %rcx
3381	mov -8(%rcx), %rax
3382	xor %rsi, %rsi
3383	mov %rax, -8(%rdx)
3384	jmp L(CopyFrom1To16BytesCase3)
3385
3386	.p2align 4
3387	L(StrncpyLeave11):
3388	movaps %xmm2, %xmm3
3389	add $48, %r8
3390	jle L(StrncpyExit11)
3391	palignr $11, %xmm1, %xmm2
3392	movaps %xmm2, (%rdx)
3393	movaps 21(%rcx), %xmm2
3394	lea 16(%rsi), %rsi
3395	sub $16, %r8
3396	jbe L(StrncpyExit11)
3397	palignr $11, %xmm3, %xmm2
3398	movaps %xmm2, 16(%rdx)
3399	lea 16(%rsi), %rsi
3400	sub $16, %r8
3401	jbe L(StrncpyExit11)
3402	movaps %xmm4, 32(%rdx)
3403	lea 16(%rsi), %rsi
3404	sub $16, %r8
3405	jbe L(StrncpyExit11)
3406	movaps %xmm5, 48(%rdx)
3407	lea 16(%rsi), %rsi
3408	lea -16(%r8), %r8
3409
3410	L(StrncpyExit11):
3411	lea 5(%rdx, %rsi), %rdx
3412	lea 5(%rcx, %rsi), %rcx
3413	mov -8(%rcx), %rax
3414	xor %rsi, %rsi
3415	mov %rax, -8(%rdx)
3416	jmp L(CopyFrom1To16BytesCase3)
3417
3418	.p2align 4
3419	L(StrncpyLeave12):
3420	movaps %xmm2, %xmm3
3421	add $48, %r8
3422	jle L(StrncpyExit12)
3423	palignr $12, %xmm1, %xmm2
3424	movaps %xmm2, (%rdx)
3425	movaps 20(%rcx), %xmm2
3426	lea 16(%rsi), %rsi
3427	sub $16, %r8
3428	jbe L(StrncpyExit12)
3429	palignr $12, %xmm3, %xmm2
3430	movaps %xmm2, 16(%rdx)
3431	lea 16(%rsi), %rsi
3432	sub $16, %r8
3433	jbe L(StrncpyExit12)
3434	movaps %xmm4, 32(%rdx)
3435	lea 16(%rsi), %rsi
3436	sub $16, %r8
3437	jbe L(StrncpyExit12)
3438	movaps %xmm5, 48(%rdx)
3439	lea 16(%rsi), %rsi
3440	lea -16(%r8), %r8
3441
3442	L(StrncpyExit12):
3443	lea 4(%rdx, %rsi), %rdx
3444	lea 4(%rcx, %rsi), %rcx
3445	mov -4(%rcx), %eax
3446	xor %rsi, %rsi
3447	mov %eax, -4(%rdx)
3448	jmp L(CopyFrom1To16BytesCase3)
3449
3450	.p2align 4
3451	L(StrncpyLeave13):
3452	movaps %xmm2, %xmm3
3453	add $48, %r8
3454	jle L(StrncpyExit13)
3455	palignr $13, %xmm1, %xmm2
3456	movaps %xmm2, (%rdx)
3457	movaps 19(%rcx), %xmm2
3458	lea 16(%rsi), %rsi
3459	sub $16, %r8
3460	jbe L(StrncpyExit13)
3461	palignr $13, %xmm3, %xmm2
3462	movaps %xmm2, 16(%rdx)
3463	lea 16(%rsi), %rsi
3464	sub $16, %r8
3465	jbe L(StrncpyExit13)
3466	movaps %xmm4, 32(%rdx)
3467	lea 16(%rsi), %rsi
3468	sub $16, %r8
3469	jbe L(StrncpyExit13)
3470	movaps %xmm5, 48(%rdx)
3471	lea 16(%rsi), %rsi
3472	lea -16(%r8), %r8
3473
3474	L(StrncpyExit13):
3475	lea 3(%rdx, %rsi), %rdx
3476	lea 3(%rcx, %rsi), %rcx
3477	mov -4(%rcx), %eax
3478	xor %rsi, %rsi
3479	mov %eax, -4(%rdx)
3480	jmp L(CopyFrom1To16BytesCase3)
3481
3482	.p2align 4
3483	L(StrncpyLeave14):
3484	movaps %xmm2, %xmm3
3485	add $48, %r8
3486	jle L(StrncpyExit14)
3487	palignr $14, %xmm1, %xmm2
3488	movaps %xmm2, (%rdx)
3489	movaps 18(%rcx), %xmm2
3490	lea 16(%rsi), %rsi
3491	sub $16, %r8
3492	jbe L(StrncpyExit14)
3493	palignr $14, %xmm3, %xmm2
3494	movaps %xmm2, 16(%rdx)
3495	lea 16(%rsi), %rsi
3496	sub $16, %r8
3497	jbe L(StrncpyExit14)
3498	movaps %xmm4, 32(%rdx)
3499	lea 16(%rsi), %rsi
3500	sub $16, %r8
3501	jbe L(StrncpyExit14)
3502	movaps %xmm5, 48(%rdx)
3503	lea 16(%rsi), %rsi
3504	lea -16(%r8), %r8
3505
3506	L(StrncpyExit14):
3507	lea 2(%rdx, %rsi), %rdx
3508	lea 2(%rcx, %rsi), %rcx
3509	movw -2(%rcx), %ax
3510	xor %rsi, %rsi
3511	movw %ax, -2(%rdx)
3512	jmp L(CopyFrom1To16BytesCase3)
3513
3514	.p2align 4
3515	L(StrncpyLeave15):
3516	movaps %xmm2, %xmm3
3517	add $48, %r8
3518	jle L(StrncpyExit15)
3519	palignr $15, %xmm1, %xmm2
3520	movaps %xmm2, (%rdx)
3521	movaps 17(%rcx), %xmm2
3522	lea 16(%rsi), %rsi
3523	sub $16, %r8
3524	jbe L(StrncpyExit15)
3525	palignr $15, %xmm3, %xmm2
3526	movaps %xmm2, 16(%rdx)
3527	lea 16(%rsi), %rsi
3528	sub $16, %r8
3529	jbe L(StrncpyExit15)
3530	movaps %xmm4, 32(%rdx)
3531	lea 16(%rsi), %rsi
3532	sub $16, %r8
3533	jbe L(StrncpyExit15)
3534	movaps %xmm5, 48(%rdx)
3535	lea 16(%rsi), %rsi
3536	lea -16(%r8), %r8
3537
3538	L(StrncpyExit15):
3539	lea 1(%rdx, %rsi), %rdx
3540	lea 1(%rcx, %rsi), %rcx
3541	movb -1(%rcx), %ah
3542	xor %rsi, %rsi
3543	movb %ah, -1(%rdx)
3544	jmp L(CopyFrom1To16BytesCase3)
3545
3546	# endif
3547	# ifndef USE_AS_STRCAT
3548	END (STRCPY)
3549	# endif
3550	#endif
3551