1/* Optimized version of the standard memchr() function.
2 This file is part of the GNU C Library.
3 Copyright (C) 2000-2022 Free Software Foundation, Inc.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
18
19/* Return: the address of the first occurence of chr in str or NULL
20
21 Inputs:
22 in0: str
23 in1: chr
24 in2: byte count
25
26 This implementation assumes little endian mode. For big endian mode,
27 the instruction czx1.r should be replaced by czx1.l.
28
29 The algorithm is fairly straightforward: search byte by byte until we
30 we get to a word aligned address, then search word by word as much as
31 possible; the remaining few bytes are searched one at a time.
32
33 The word by word search is performed by xor-ing the word with a word
34 containing chr in every byte. If there is a hit, the result will
35 contain a zero byte in the corresponding position. The presence and
36 position of that zero byte is detected with a czx instruction.
37
38 All the loops in this function could have had the internal branch removed
39 if br.ctop and br.cloop could be predicated :-(. */
40
41#include <sysdep.h>
42#undef ret
43
44#define saved_pr r15
45#define saved_lc r16
46#define chr r17
47#define len r18
48#define last r20
49#define val r21
50#define tmp r24
51#define chrx8 r25
52#define loopcnt r30
53
54#define str in0
55
56ENTRY(__memchr)
57 .prologue
58 alloc r2 = ar.pfs, 3, 0, 29, 32
59#include "softpipe.h"
60 .rotr value[MEMLAT+1], addr[MEMLAT+3], aux[2], poschr[2]
61 .rotp p[MEMLAT+3]
62 .save ar.lc, saved_lc
63 mov saved_lc = ar.lc // save the loop counter
64 .save pr, saved_pr
65 mov saved_pr = pr // save the predicates
66 .body
67 mov ret0 = str
68 add last = str, in2 // last byte
69 ;;
70 cmp.ltu p6, p0 = last, str
71 ;;
72(p6) mov last = -1
73 and tmp = 7, str // tmp = str % 8
74 cmp.ne p7, p0 = r0, r0 // clear p7
75 extr.u chr = in1, 0, 8 // chr = (unsigned char) in1
76 mov len = in2
77 cmp.gtu p6, p0 = 16, in2 // use a simple loop for short
78(p6) br.cond.spnt .srchfew ;; // searches
79 sub loopcnt = 8, tmp // loopcnt = 8 - tmp
80 cmp.eq p6, p0 = tmp, r0
81(p6) br.cond.sptk .str_aligned;;
82 sub len = len, loopcnt
83 adds loopcnt = -1, loopcnt;;
84 mov ar.lc = loopcnt
85.l1:
86 ld1 val = [ret0], 1
87 ;;
88 cmp.eq p6, p0 = val, chr
89(p6) br.cond.spnt .foundit
90 br.cloop.sptk .l1 ;;
91.str_aligned:
92 cmp.ne p6, p0 = r0, r0 // clear p6
93 shr.u loopcnt = len, 3 // loopcnt = len / 8
94 and len = 7, len ;; // remaining len = len & 7
95 adds loopcnt = -1, loopcnt
96 mov ar.ec = MEMLAT + 3
97 mux1 chrx8 = chr, @brcst ;; // get a word full of chr
98 mov ar.lc = loopcnt
99 mov pr.rot = 1 << 16 ;;
100.l2:
101(p[0]) mov addr[0] = ret0
102(p[0]) ld8.s value[0] = [ret0], 8 // speculative load
103(p[MEMLAT]) chk.s value[MEMLAT], .recovery // check and recovery
104(p[MEMLAT]) xor aux[0] = value[MEMLAT], chrx8
105(p[MEMLAT+1]) czx1.r poschr[0] = aux[1]
106(p[MEMLAT+2]) cmp.ne p7, p0 = 8, poschr[1]
107(p7) br.cond.dpnt .foundit
108 br.ctop.dptk .l2
109.srchfew:
110 adds loopcnt = -1, len
111 cmp.eq p6, p0 = len, r0
112(p6) br.cond.spnt .notfound ;;
113 mov ar.lc = loopcnt
114.l3:
115 ld1 val = [ret0], 1
116 ;;
117 cmp.eq p6, p0 = val, chr
118(p6) br.cond.dpnt .foundit
119 br.cloop.sptk .l3 ;;
120.notfound:
121 cmp.ne p6, p0 = r0, r0 // clear p6 (p7 was already 0 when we got here)
122 mov ret0 = r0 ;; // return NULL
123.foundit:
124 .pred.rel "mutex" p6, p7
125(p6) adds ret0 = -1, ret0 // if we got here from l1 or l3
126(p7) add ret0 = addr[MEMLAT+2], poschr[1] // if we got here from l2
127 mov pr = saved_pr, -1
128 mov ar.lc = saved_lc
129 br.ret.sptk.many b0
130
131.recovery:
132#if MEMLAT != 6
133# error "MEMLAT must be 6!"
134#endif
135(p[MEMLAT-6]) add ret0 = -8, ret0;;
136(p[MEMLAT-5]) add ret0 = -8, ret0;;
137(p[MEMLAT-4]) add ret0 = -8, ret0;;
138(p[MEMLAT-3]) add ret0 = -8, ret0;;
139(p[MEMLAT-2]) add ret0 = -8, ret0;;
140(p[MEMLAT-1]) add ret0 = -8, ret0;;
141(p[MEMLAT]) add ret0 = -8, ret0;;
142(p[MEMLAT+1]) add ret0 = -8, ret0;;
143(p[MEMLAT+2]) add ret0 = -8, ret0;;
144.l4:
145 mov addr[MEMLAT+2] = ret0
146 ld8 tmp = [ret0];; // load the first unchecked 8byte
147 xor aux[1] = tmp, chrx8;;
148 czx1.r poschr[1] = aux[1];;
149 cmp.ne p7, p0 = 8, poschr[1];;
150(p7) add ret0 = addr[MEMLAT+2], poschr[1];;
151(p7) cmp.geu p6, p7 = ret0, last // don't go over the last byte
152(p6) br.cond.spnt .notfound;;
153(p7) br.cond.spnt .foundit;;
154 adds ret0 = 8, ret0 // load the next unchecked 8byte
155 br.sptk .l4;;
156
157END(__memchr)
158
159weak_alias (__memchr, memchr)
160libc_hidden_builtin_def (memchr)
161

source code of glibc/sysdeps/ia64/memchr.S