| 1 | /* SPDX-License-Identifier: GPL-2.0-or-later */ |
|---|
| 2 | /* |
|---|
| 3 | * Author: Anton Blanchard <anton@au.ibm.com> |
|---|
| 4 | * Copyright 2015 IBM Corporation. |
|---|
| 5 | */ |
|---|
| 6 | #include <linux/export.h> |
|---|
| 7 | #include <asm/ppc_asm.h> |
|---|
| 8 | #include <asm/ppc-opcode.h> |
|---|
| 9 | |
|---|
| 10 | #define off8 r6 |
|---|
| 11 | #define off16 r7 |
|---|
| 12 | #define off24 r8 |
|---|
| 13 | |
|---|
| 14 | #define rA r9 |
|---|
| 15 | #define rB r10 |
|---|
| 16 | #define rC r11 |
|---|
| 17 | #define rD r27 |
|---|
| 18 | #define rE r28 |
|---|
| 19 | #define rF r29 |
|---|
| 20 | #define rG r30 |
|---|
| 21 | #define rH r31 |
|---|
| 22 | |
|---|
| 23 | #ifdef __LITTLE_ENDIAN__ |
|---|
| 24 | #define LH lhbrx |
|---|
| 25 | #define LW lwbrx |
|---|
| 26 | #define LD ldbrx |
|---|
| 27 | #define LVS lvsr |
|---|
| 28 | #define VPERM(_VRT,_VRA,_VRB,_VRC) \ |
|---|
| 29 | vperm _VRT,_VRB,_VRA,_VRC |
|---|
| 30 | #else |
|---|
| 31 | #define LH lhzx |
|---|
| 32 | #define LW lwzx |
|---|
| 33 | #define LD ldx |
|---|
| 34 | #define LVS lvsl |
|---|
| 35 | #define VPERM(_VRT,_VRA,_VRB,_VRC) \ |
|---|
| 36 | vperm _VRT,_VRA,_VRB,_VRC |
|---|
| 37 | #endif |
|---|
| 38 | |
|---|
| 39 | #define VMX_THRESH 4096 |
|---|
| 40 | #define ENTER_VMX_OPS \ |
|---|
| 41 | mflr r0; \ |
|---|
| 42 | std r3,-STACKFRAMESIZE+STK_REG(R31)(r1); \ |
|---|
| 43 | std r4,-STACKFRAMESIZE+STK_REG(R30)(r1); \ |
|---|
| 44 | std r5,-STACKFRAMESIZE+STK_REG(R29)(r1); \ |
|---|
| 45 | std r0,16(r1); \ |
|---|
| 46 | stdu r1,-STACKFRAMESIZE(r1); \ |
|---|
| 47 | bl CFUNC(enter_vmx_ops); \ |
|---|
| 48 | cmpwi cr1,r3,0; \ |
|---|
| 49 | ld r0,STACKFRAMESIZE+16(r1); \ |
|---|
| 50 | ld r3,STK_REG(R31)(r1); \ |
|---|
| 51 | ld r4,STK_REG(R30)(r1); \ |
|---|
| 52 | ld r5,STK_REG(R29)(r1); \ |
|---|
| 53 | addi r1,r1,STACKFRAMESIZE; \ |
|---|
| 54 | mtlr r0 |
|---|
| 55 | |
|---|
| 56 | #define EXIT_VMX_OPS \ |
|---|
| 57 | mflr r0; \ |
|---|
| 58 | std r3,-STACKFRAMESIZE+STK_REG(R31)(r1); \ |
|---|
| 59 | std r4,-STACKFRAMESIZE+STK_REG(R30)(r1); \ |
|---|
| 60 | std r5,-STACKFRAMESIZE+STK_REG(R29)(r1); \ |
|---|
| 61 | std r0,16(r1); \ |
|---|
| 62 | stdu r1,-STACKFRAMESIZE(r1); \ |
|---|
| 63 | bl CFUNC(exit_vmx_ops); \ |
|---|
| 64 | ld r0,STACKFRAMESIZE+16(r1); \ |
|---|
| 65 | ld r3,STK_REG(R31)(r1); \ |
|---|
| 66 | ld r4,STK_REG(R30)(r1); \ |
|---|
| 67 | ld r5,STK_REG(R29)(r1); \ |
|---|
| 68 | addi r1,r1,STACKFRAMESIZE; \ |
|---|
| 69 | mtlr r0 |
|---|
| 70 | |
|---|
| 71 | /* |
|---|
| 72 | * LD_VSR_CROSS16B load the 2nd 16 bytes for _vaddr which is unaligned with |
|---|
| 73 | * 16 bytes boundary and permute the result with the 1st 16 bytes. |
|---|
| 74 | |
|---|
| 75 | * | y y y y y y y y y y y y y 0 1 2 | 3 4 5 6 7 8 9 a b c d e f z z z | |
|---|
| 76 | * ^ ^ ^ |
|---|
| 77 | * 0xbbbb10 0xbbbb20 0xbbb30 |
|---|
| 78 | * ^ |
|---|
| 79 | * _vaddr |
|---|
| 80 | * |
|---|
| 81 | * |
|---|
| 82 | * _vmask is the mask generated by LVS |
|---|
| 83 | * _v1st_qw is the 1st aligned QW of current addr which is already loaded. |
|---|
| 84 | * for example: 0xyyyyyyyyyyyyy012 for big endian |
|---|
| 85 | * _v2nd_qw is the 2nd aligned QW of cur _vaddr to be loaded. |
|---|
| 86 | * for example: 0x3456789abcdefzzz for big endian |
|---|
| 87 | * The permute result is saved in _v_res. |
|---|
| 88 | * for example: 0x0123456789abcdef for big endian. |
|---|
| 89 | */ |
|---|
| 90 | #define LD_VSR_CROSS16B(_vaddr,_vmask,_v1st_qw,_v2nd_qw,_v_res) \ |
|---|
| 91 | lvx _v2nd_qw,_vaddr,off16; \ |
|---|
| 92 | VPERM(_v_res,_v1st_qw,_v2nd_qw,_vmask) |
|---|
| 93 | |
|---|
| 94 | /* |
|---|
| 95 | * There are 2 categories for memcmp: |
|---|
| 96 | * 1) src/dst has the same offset to the 8 bytes boundary. The handlers |
|---|
| 97 | * are named like .Lsameoffset_xxxx |
|---|
| 98 | * 2) src/dst has different offset to the 8 bytes boundary. The handlers |
|---|
| 99 | * are named like .Ldiffoffset_xxxx |
|---|
| 100 | */ |
|---|
| 101 | _GLOBAL_TOC(memcmp) |
|---|
| 102 | cmpdi cr1,r5,0 |
|---|
| 103 | |
|---|
| 104 | /* Use the short loop if the src/dst addresses are not |
|---|
| 105 | * with the same offset of 8 bytes align boundary. |
|---|
| 106 | */ |
|---|
| 107 | xor r6,r3,r4 |
|---|
| 108 | andi. r6,r6,7 |
|---|
| 109 | |
|---|
| 110 | /* Fall back to short loop if compare at aligned addrs |
|---|
| 111 | * with less than 8 bytes. |
|---|
| 112 | */ |
|---|
| 113 | cmpdi cr6,r5,7 |
|---|
| 114 | |
|---|
| 115 | beq cr1,.Lzero |
|---|
| 116 | bgt cr6,.Lno_short |
|---|
| 117 | |
|---|
| 118 | .Lshort: |
|---|
| 119 | mtctr r5 |
|---|
| 120 | 1: lbz rA,0(r3) |
|---|
| 121 | lbz rB,0(r4) |
|---|
| 122 | subf. rC,rB,rA |
|---|
| 123 | bne .Lnon_zero |
|---|
| 124 | bdz .Lzero |
|---|
| 125 | |
|---|
| 126 | lbz rA,1(r3) |
|---|
| 127 | lbz rB,1(r4) |
|---|
| 128 | subf. rC,rB,rA |
|---|
| 129 | bne .Lnon_zero |
|---|
| 130 | bdz .Lzero |
|---|
| 131 | |
|---|
| 132 | lbz rA,2(r3) |
|---|
| 133 | lbz rB,2(r4) |
|---|
| 134 | subf. rC,rB,rA |
|---|
| 135 | bne .Lnon_zero |
|---|
| 136 | bdz .Lzero |
|---|
| 137 | |
|---|
| 138 | lbz rA,3(r3) |
|---|
| 139 | lbz rB,3(r4) |
|---|
| 140 | subf. rC,rB,rA |
|---|
| 141 | bne .Lnon_zero |
|---|
| 142 | |
|---|
| 143 | addi r3,r3,4 |
|---|
| 144 | addi r4,r4,4 |
|---|
| 145 | |
|---|
| 146 | bdnz 1b |
|---|
| 147 | |
|---|
| 148 | .Lzero: |
|---|
| 149 | li r3,0 |
|---|
| 150 | blr |
|---|
| 151 | |
|---|
| 152 | .Lno_short: |
|---|
| 153 | dcbt 0,r3 |
|---|
| 154 | dcbt 0,r4 |
|---|
| 155 | bne .Ldiffoffset_8bytes_make_align_start |
|---|
| 156 | |
|---|
| 157 | |
|---|
| 158 | .Lsameoffset_8bytes_make_align_start: |
|---|
| 159 | /* attempt to compare bytes not aligned with 8 bytes so that |
|---|
| 160 | * rest comparison can run based on 8 bytes alignment. |
|---|
| 161 | */ |
|---|
| 162 | andi. r6,r3,7 |
|---|
| 163 | |
|---|
| 164 | /* Try to compare the first double word which is not 8 bytes aligned: |
|---|
| 165 | * load the first double word at (src & ~7UL) and shift left appropriate |
|---|
| 166 | * bits before comparision. |
|---|
| 167 | */ |
|---|
| 168 | rlwinm r6,r3,3,26,28 |
|---|
| 169 | beq .Lsameoffset_8bytes_aligned |
|---|
| 170 | clrrdi r3,r3,3 |
|---|
| 171 | clrrdi r4,r4,3 |
|---|
| 172 | LD rA,0,r3 |
|---|
| 173 | LD rB,0,r4 |
|---|
| 174 | sld rA,rA,r6 |
|---|
| 175 | sld rB,rB,r6 |
|---|
| 176 | cmpld cr0,rA,rB |
|---|
| 177 | srwi r6,r6,3 |
|---|
| 178 | bne cr0,.LcmpAB_lightweight |
|---|
| 179 | subfic r6,r6,8 |
|---|
| 180 | subf. r5,r6,r5 |
|---|
| 181 | addi r3,r3,8 |
|---|
| 182 | addi r4,r4,8 |
|---|
| 183 | beq .Lzero |
|---|
| 184 | |
|---|
| 185 | .Lsameoffset_8bytes_aligned: |
|---|
| 186 | /* now we are aligned with 8 bytes. |
|---|
| 187 | * Use .Llong loop if left cmp bytes are equal or greater than 32B. |
|---|
| 188 | */ |
|---|
| 189 | cmpdi cr6,r5,31 |
|---|
| 190 | bgt cr6,.Llong |
|---|
| 191 | |
|---|
| 192 | .Lcmp_lt32bytes: |
|---|
| 193 | /* compare 1 ~ 31 bytes, at least r3 addr is 8 bytes aligned now */ |
|---|
| 194 | cmpdi cr5,r5,7 |
|---|
| 195 | srdi r0,r5,3 |
|---|
| 196 | ble cr5,.Lcmp_rest_lt8bytes |
|---|
| 197 | |
|---|
| 198 | /* handle 8 ~ 31 bytes */ |
|---|
| 199 | clrldi r5,r5,61 |
|---|
| 200 | mtctr r0 |
|---|
| 201 | 2: |
|---|
| 202 | LD rA,0,r3 |
|---|
| 203 | LD rB,0,r4 |
|---|
| 204 | cmpld cr0,rA,rB |
|---|
| 205 | addi r3,r3,8 |
|---|
| 206 | addi r4,r4,8 |
|---|
| 207 | bne cr0,.LcmpAB_lightweight |
|---|
| 208 | bdnz 2b |
|---|
| 209 | |
|---|
| 210 | cmpwi r5,0 |
|---|
| 211 | beq .Lzero |
|---|
| 212 | |
|---|
| 213 | .Lcmp_rest_lt8bytes: |
|---|
| 214 | /* |
|---|
| 215 | * Here we have less than 8 bytes to compare. At least s1 is aligned to |
|---|
| 216 | * 8 bytes, but s2 may not be. We must make sure s2 + 7 doesn't cross a |
|---|
| 217 | * page boundary, otherwise we might read past the end of the buffer and |
|---|
| 218 | * trigger a page fault. We use 4K as the conservative minimum page |
|---|
| 219 | * size. If we detect that case we go to the byte-by-byte loop. |
|---|
| 220 | * |
|---|
| 221 | * Otherwise the next double word is loaded from s1 and s2, and shifted |
|---|
| 222 | * right to compare the appropriate bits. |
|---|
| 223 | */ |
|---|
| 224 | clrldi r6,r4,(64-12) // r6 = r4 & 0xfff |
|---|
| 225 | cmpdi r6,0xff8 |
|---|
| 226 | bgt .Lshort |
|---|
| 227 | |
|---|
| 228 | subfic r6,r5,8 |
|---|
| 229 | slwi r6,r6,3 |
|---|
| 230 | LD rA,0,r3 |
|---|
| 231 | LD rB,0,r4 |
|---|
| 232 | srd rA,rA,r6 |
|---|
| 233 | srd rB,rB,r6 |
|---|
| 234 | cmpld cr0,rA,rB |
|---|
| 235 | bne cr0,.LcmpAB_lightweight |
|---|
| 236 | b .Lzero |
|---|
| 237 | |
|---|
| 238 | .Lnon_zero: |
|---|
| 239 | mr r3,rC |
|---|
| 240 | blr |
|---|
| 241 | |
|---|
| 242 | .Llong: |
|---|
| 243 | #ifdef CONFIG_ALTIVEC |
|---|
| 244 | BEGIN_FTR_SECTION |
|---|
| 245 | /* Try to use vmx loop if length is equal or greater than 4K */ |
|---|
| 246 | cmpldi cr6,r5,VMX_THRESH |
|---|
| 247 | bge cr6,.Lsameoffset_vmx_cmp |
|---|
| 248 | END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) |
|---|
| 249 | |
|---|
| 250 | .Llong_novmx_cmp: |
|---|
| 251 | #endif |
|---|
| 252 | /* At least s1 addr is aligned with 8 bytes */ |
|---|
| 253 | li off8,8 |
|---|
| 254 | li off16,16 |
|---|
| 255 | li off24,24 |
|---|
| 256 | |
|---|
| 257 | std r31,-8(r1) |
|---|
| 258 | std r30,-16(r1) |
|---|
| 259 | std r29,-24(r1) |
|---|
| 260 | std r28,-32(r1) |
|---|
| 261 | std r27,-40(r1) |
|---|
| 262 | |
|---|
| 263 | srdi r0,r5,5 |
|---|
| 264 | mtctr r0 |
|---|
| 265 | andi. r5,r5,31 |
|---|
| 266 | |
|---|
| 267 | LD rA,0,r3 |
|---|
| 268 | LD rB,0,r4 |
|---|
| 269 | |
|---|
| 270 | LD rC,off8,r3 |
|---|
| 271 | LD rD,off8,r4 |
|---|
| 272 | |
|---|
| 273 | LD rE,off16,r3 |
|---|
| 274 | LD rF,off16,r4 |
|---|
| 275 | |
|---|
| 276 | LD rG,off24,r3 |
|---|
| 277 | LD rH,off24,r4 |
|---|
| 278 | cmpld cr0,rA,rB |
|---|
| 279 | |
|---|
| 280 | addi r3,r3,32 |
|---|
| 281 | addi r4,r4,32 |
|---|
| 282 | |
|---|
| 283 | bdz .Lfirst32 |
|---|
| 284 | |
|---|
| 285 | LD rA,0,r3 |
|---|
| 286 | LD rB,0,r4 |
|---|
| 287 | cmpld cr1,rC,rD |
|---|
| 288 | |
|---|
| 289 | LD rC,off8,r3 |
|---|
| 290 | LD rD,off8,r4 |
|---|
| 291 | cmpld cr6,rE,rF |
|---|
| 292 | |
|---|
| 293 | LD rE,off16,r3 |
|---|
| 294 | LD rF,off16,r4 |
|---|
| 295 | cmpld cr7,rG,rH |
|---|
| 296 | bne cr0,.LcmpAB |
|---|
| 297 | |
|---|
| 298 | LD rG,off24,r3 |
|---|
| 299 | LD rH,off24,r4 |
|---|
| 300 | cmpld cr0,rA,rB |
|---|
| 301 | bne cr1,.LcmpCD |
|---|
| 302 | |
|---|
| 303 | addi r3,r3,32 |
|---|
| 304 | addi r4,r4,32 |
|---|
| 305 | |
|---|
| 306 | bdz .Lsecond32 |
|---|
| 307 | |
|---|
| 308 | .balign 16 |
|---|
| 309 | |
|---|
| 310 | 1: LD rA,0,r3 |
|---|
| 311 | LD rB,0,r4 |
|---|
| 312 | cmpld cr1,rC,rD |
|---|
| 313 | bne cr6,.LcmpEF |
|---|
| 314 | |
|---|
| 315 | LD rC,off8,r3 |
|---|
| 316 | LD rD,off8,r4 |
|---|
| 317 | cmpld cr6,rE,rF |
|---|
| 318 | bne cr7,.LcmpGH |
|---|
| 319 | |
|---|
| 320 | LD rE,off16,r3 |
|---|
| 321 | LD rF,off16,r4 |
|---|
| 322 | cmpld cr7,rG,rH |
|---|
| 323 | bne cr0,.LcmpAB |
|---|
| 324 | |
|---|
| 325 | LD rG,off24,r3 |
|---|
| 326 | LD rH,off24,r4 |
|---|
| 327 | cmpld cr0,rA,rB |
|---|
| 328 | bne cr1,.LcmpCD |
|---|
| 329 | |
|---|
| 330 | addi r3,r3,32 |
|---|
| 331 | addi r4,r4,32 |
|---|
| 332 | |
|---|
| 333 | bdnz 1b |
|---|
| 334 | |
|---|
| 335 | .Lsecond32: |
|---|
| 336 | cmpld cr1,rC,rD |
|---|
| 337 | bne cr6,.LcmpEF |
|---|
| 338 | |
|---|
| 339 | cmpld cr6,rE,rF |
|---|
| 340 | bne cr7,.LcmpGH |
|---|
| 341 | |
|---|
| 342 | cmpld cr7,rG,rH |
|---|
| 343 | bne cr0,.LcmpAB |
|---|
| 344 | |
|---|
| 345 | bne cr1,.LcmpCD |
|---|
| 346 | bne cr6,.LcmpEF |
|---|
| 347 | bne cr7,.LcmpGH |
|---|
| 348 | |
|---|
| 349 | .Ltail: |
|---|
| 350 | ld r31,-8(r1) |
|---|
| 351 | ld r30,-16(r1) |
|---|
| 352 | ld r29,-24(r1) |
|---|
| 353 | ld r28,-32(r1) |
|---|
| 354 | ld r27,-40(r1) |
|---|
| 355 | |
|---|
| 356 | cmpdi r5,0 |
|---|
| 357 | beq .Lzero |
|---|
| 358 | b .Lshort |
|---|
| 359 | |
|---|
| 360 | .Lfirst32: |
|---|
| 361 | cmpld cr1,rC,rD |
|---|
| 362 | cmpld cr6,rE,rF |
|---|
| 363 | cmpld cr7,rG,rH |
|---|
| 364 | |
|---|
| 365 | bne cr0,.LcmpAB |
|---|
| 366 | bne cr1,.LcmpCD |
|---|
| 367 | bne cr6,.LcmpEF |
|---|
| 368 | bne cr7,.LcmpGH |
|---|
| 369 | |
|---|
| 370 | b .Ltail |
|---|
| 371 | |
|---|
| 372 | .LcmpAB: |
|---|
| 373 | li r3,1 |
|---|
| 374 | bgt cr0,.Lout |
|---|
| 375 | li r3,-1 |
|---|
| 376 | b .Lout |
|---|
| 377 | |
|---|
| 378 | .LcmpCD: |
|---|
| 379 | li r3,1 |
|---|
| 380 | bgt cr1,.Lout |
|---|
| 381 | li r3,-1 |
|---|
| 382 | b .Lout |
|---|
| 383 | |
|---|
| 384 | .LcmpEF: |
|---|
| 385 | li r3,1 |
|---|
| 386 | bgt cr6,.Lout |
|---|
| 387 | li r3,-1 |
|---|
| 388 | b .Lout |
|---|
| 389 | |
|---|
| 390 | .LcmpGH: |
|---|
| 391 | li r3,1 |
|---|
| 392 | bgt cr7,.Lout |
|---|
| 393 | li r3,-1 |
|---|
| 394 | |
|---|
| 395 | .Lout: |
|---|
| 396 | ld r31,-8(r1) |
|---|
| 397 | ld r30,-16(r1) |
|---|
| 398 | ld r29,-24(r1) |
|---|
| 399 | ld r28,-32(r1) |
|---|
| 400 | ld r27,-40(r1) |
|---|
| 401 | blr |
|---|
| 402 | |
|---|
| 403 | .LcmpAB_lightweight: /* skip NV GPRS restore */ |
|---|
| 404 | li r3,1 |
|---|
| 405 | bgtlr |
|---|
| 406 | li r3,-1 |
|---|
| 407 | blr |
|---|
| 408 | |
|---|
| 409 | #ifdef CONFIG_ALTIVEC |
|---|
| 410 | .Lsameoffset_vmx_cmp: |
|---|
| 411 | /* Enter with src/dst addrs has the same offset with 8 bytes |
|---|
| 412 | * align boundary. |
|---|
| 413 | * |
|---|
| 414 | * There is an optimization based on following fact: memcmp() |
|---|
| 415 | * prones to fail early at the first 32 bytes. |
|---|
| 416 | * Before applying VMX instructions which will lead to 32x128bits |
|---|
| 417 | * VMX regs load/restore penalty, we compare the first 32 bytes |
|---|
| 418 | * so that we can catch the ~80% fail cases. |
|---|
| 419 | */ |
|---|
| 420 | |
|---|
| 421 | li r0,4 |
|---|
| 422 | mtctr r0 |
|---|
| 423 | .Lsameoffset_prechk_32B_loop: |
|---|
| 424 | LD rA,0,r3 |
|---|
| 425 | LD rB,0,r4 |
|---|
| 426 | cmpld cr0,rA,rB |
|---|
| 427 | addi r3,r3,8 |
|---|
| 428 | addi r4,r4,8 |
|---|
| 429 | bne cr0,.LcmpAB_lightweight |
|---|
| 430 | addi r5,r5,-8 |
|---|
| 431 | bdnz .Lsameoffset_prechk_32B_loop |
|---|
| 432 | |
|---|
| 433 | ENTER_VMX_OPS |
|---|
| 434 | beq cr1,.Llong_novmx_cmp |
|---|
| 435 | |
|---|
| 436 | 3: |
|---|
| 437 | /* need to check whether r4 has the same offset with r3 |
|---|
| 438 | * for 16 bytes boundary. |
|---|
| 439 | */ |
|---|
| 440 | xor r0,r3,r4 |
|---|
| 441 | andi. r0,r0,0xf |
|---|
| 442 | bne .Ldiffoffset_vmx_cmp_start |
|---|
| 443 | |
|---|
| 444 | /* len is no less than 4KB. Need to align with 16 bytes further. |
|---|
| 445 | */ |
|---|
| 446 | andi. rA,r3,8 |
|---|
| 447 | LD rA,0,r3 |
|---|
| 448 | beq 4f |
|---|
| 449 | LD rB,0,r4 |
|---|
| 450 | cmpld cr0,rA,rB |
|---|
| 451 | addi r3,r3,8 |
|---|
| 452 | addi r4,r4,8 |
|---|
| 453 | addi r5,r5,-8 |
|---|
| 454 | |
|---|
| 455 | beq cr0,4f |
|---|
| 456 | /* save and restore cr0 */ |
|---|
| 457 | mfocrf r5,128 |
|---|
| 458 | EXIT_VMX_OPS |
|---|
| 459 | mtocrf 128,r5 |
|---|
| 460 | b .LcmpAB_lightweight |
|---|
| 461 | |
|---|
| 462 | 4: |
|---|
| 463 | /* compare 32 bytes for each loop */ |
|---|
| 464 | srdi r0,r5,5 |
|---|
| 465 | mtctr r0 |
|---|
| 466 | clrldi r5,r5,59 |
|---|
| 467 | li off16,16 |
|---|
| 468 | |
|---|
| 469 | .balign 16 |
|---|
| 470 | 5: |
|---|
| 471 | lvx v0,0,r3 |
|---|
| 472 | lvx v1,0,r4 |
|---|
| 473 | VCMPEQUD_RC(v0,v0,v1) |
|---|
| 474 | bnl cr6,7f |
|---|
| 475 | lvx v0,off16,r3 |
|---|
| 476 | lvx v1,off16,r4 |
|---|
| 477 | VCMPEQUD_RC(v0,v0,v1) |
|---|
| 478 | bnl cr6,6f |
|---|
| 479 | addi r3,r3,32 |
|---|
| 480 | addi r4,r4,32 |
|---|
| 481 | bdnz 5b |
|---|
| 482 | |
|---|
| 483 | EXIT_VMX_OPS |
|---|
| 484 | cmpdi r5,0 |
|---|
| 485 | beq .Lzero |
|---|
| 486 | b .Lcmp_lt32bytes |
|---|
| 487 | |
|---|
| 488 | 6: |
|---|
| 489 | addi r3,r3,16 |
|---|
| 490 | addi r4,r4,16 |
|---|
| 491 | |
|---|
| 492 | 7: |
|---|
| 493 | /* diff the last 16 bytes */ |
|---|
| 494 | EXIT_VMX_OPS |
|---|
| 495 | LD rA,0,r3 |
|---|
| 496 | LD rB,0,r4 |
|---|
| 497 | cmpld cr0,rA,rB |
|---|
| 498 | li off8,8 |
|---|
| 499 | bne cr0,.LcmpAB_lightweight |
|---|
| 500 | |
|---|
| 501 | LD rA,off8,r3 |
|---|
| 502 | LD rB,off8,r4 |
|---|
| 503 | cmpld cr0,rA,rB |
|---|
| 504 | bne cr0,.LcmpAB_lightweight |
|---|
| 505 | b .Lzero |
|---|
| 506 | #endif |
|---|
| 507 | |
|---|
| 508 | .Ldiffoffset_8bytes_make_align_start: |
|---|
| 509 | /* now try to align s1 with 8 bytes */ |
|---|
| 510 | rlwinm r6,r3,3,26,28 |
|---|
| 511 | beq .Ldiffoffset_align_s1_8bytes |
|---|
| 512 | |
|---|
| 513 | clrrdi r3,r3,3 |
|---|
| 514 | LD rA,0,r3 |
|---|
| 515 | LD rB,0,r4 /* unaligned load */ |
|---|
| 516 | sld rA,rA,r6 |
|---|
| 517 | srd rA,rA,r6 |
|---|
| 518 | srd rB,rB,r6 |
|---|
| 519 | cmpld cr0,rA,rB |
|---|
| 520 | srwi r6,r6,3 |
|---|
| 521 | bne cr0,.LcmpAB_lightweight |
|---|
| 522 | |
|---|
| 523 | subfic r6,r6,8 |
|---|
| 524 | subf. r5,r6,r5 |
|---|
| 525 | addi r3,r3,8 |
|---|
| 526 | add r4,r4,r6 |
|---|
| 527 | |
|---|
| 528 | beq .Lzero |
|---|
| 529 | |
|---|
| 530 | .Ldiffoffset_align_s1_8bytes: |
|---|
| 531 | /* now s1 is aligned with 8 bytes. */ |
|---|
| 532 | #ifdef CONFIG_ALTIVEC |
|---|
| 533 | BEGIN_FTR_SECTION |
|---|
| 534 | /* only do vmx ops when the size equal or greater than 4K bytes */ |
|---|
| 535 | cmpdi cr5,r5,VMX_THRESH |
|---|
| 536 | bge cr5,.Ldiffoffset_vmx_cmp |
|---|
| 537 | END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) |
|---|
| 538 | |
|---|
| 539 | .Ldiffoffset_novmx_cmp: |
|---|
| 540 | #endif |
|---|
| 541 | |
|---|
| 542 | |
|---|
| 543 | cmpdi cr5,r5,31 |
|---|
| 544 | ble cr5,.Lcmp_lt32bytes |
|---|
| 545 | |
|---|
| 546 | #ifdef CONFIG_ALTIVEC |
|---|
| 547 | b .Llong_novmx_cmp |
|---|
| 548 | #else |
|---|
| 549 | b .Llong |
|---|
| 550 | #endif |
|---|
| 551 | |
|---|
| 552 | #ifdef CONFIG_ALTIVEC |
|---|
| 553 | .Ldiffoffset_vmx_cmp: |
|---|
| 554 | /* perform a 32 bytes pre-checking before |
|---|
| 555 | * enable VMX operations. |
|---|
| 556 | */ |
|---|
| 557 | li r0,4 |
|---|
| 558 | mtctr r0 |
|---|
| 559 | .Ldiffoffset_prechk_32B_loop: |
|---|
| 560 | LD rA,0,r3 |
|---|
| 561 | LD rB,0,r4 |
|---|
| 562 | cmpld cr0,rA,rB |
|---|
| 563 | addi r3,r3,8 |
|---|
| 564 | addi r4,r4,8 |
|---|
| 565 | bne cr0,.LcmpAB_lightweight |
|---|
| 566 | addi r5,r5,-8 |
|---|
| 567 | bdnz .Ldiffoffset_prechk_32B_loop |
|---|
| 568 | |
|---|
| 569 | ENTER_VMX_OPS |
|---|
| 570 | beq cr1,.Ldiffoffset_novmx_cmp |
|---|
| 571 | |
|---|
| 572 | .Ldiffoffset_vmx_cmp_start: |
|---|
| 573 | /* Firstly try to align r3 with 16 bytes */ |
|---|
| 574 | andi. r6,r3,0xf |
|---|
| 575 | li off16,16 |
|---|
| 576 | beq .Ldiffoffset_vmx_s1_16bytes_align |
|---|
| 577 | |
|---|
| 578 | LVS v3,0,r3 |
|---|
| 579 | LVS v4,0,r4 |
|---|
| 580 | |
|---|
| 581 | lvx v5,0,r3 |
|---|
| 582 | lvx v6,0,r4 |
|---|
| 583 | LD_VSR_CROSS16B(r3,v3,v5,v7,v9) |
|---|
| 584 | LD_VSR_CROSS16B(r4,v4,v6,v8,v10) |
|---|
| 585 | |
|---|
| 586 | VCMPEQUB_RC(v7,v9,v10) |
|---|
| 587 | bnl cr6,.Ldiffoffset_vmx_diff_found |
|---|
| 588 | |
|---|
| 589 | subfic r6,r6,16 |
|---|
| 590 | subf r5,r6,r5 |
|---|
| 591 | add r3,r3,r6 |
|---|
| 592 | add r4,r4,r6 |
|---|
| 593 | |
|---|
| 594 | .Ldiffoffset_vmx_s1_16bytes_align: |
|---|
| 595 | /* now s1 is aligned with 16 bytes */ |
|---|
| 596 | lvx v6,0,r4 |
|---|
| 597 | LVS v4,0,r4 |
|---|
| 598 | srdi r6,r5,5 /* loop for 32 bytes each */ |
|---|
| 599 | clrldi r5,r5,59 |
|---|
| 600 | mtctr r6 |
|---|
| 601 | |
|---|
| 602 | .balign 16 |
|---|
| 603 | .Ldiffoffset_vmx_32bytesloop: |
|---|
| 604 | /* the first qw of r4 was saved in v6 */ |
|---|
| 605 | lvx v9,0,r3 |
|---|
| 606 | LD_VSR_CROSS16B(r4,v4,v6,v8,v10) |
|---|
| 607 | VCMPEQUB_RC(v7,v9,v10) |
|---|
| 608 | vor v6,v8,v8 |
|---|
| 609 | bnl cr6,.Ldiffoffset_vmx_diff_found |
|---|
| 610 | |
|---|
| 611 | addi r3,r3,16 |
|---|
| 612 | addi r4,r4,16 |
|---|
| 613 | |
|---|
| 614 | lvx v9,0,r3 |
|---|
| 615 | LD_VSR_CROSS16B(r4,v4,v6,v8,v10) |
|---|
| 616 | VCMPEQUB_RC(v7,v9,v10) |
|---|
| 617 | vor v6,v8,v8 |
|---|
| 618 | bnl cr6,.Ldiffoffset_vmx_diff_found |
|---|
| 619 | |
|---|
| 620 | addi r3,r3,16 |
|---|
| 621 | addi r4,r4,16 |
|---|
| 622 | |
|---|
| 623 | bdnz .Ldiffoffset_vmx_32bytesloop |
|---|
| 624 | |
|---|
| 625 | EXIT_VMX_OPS |
|---|
| 626 | |
|---|
| 627 | cmpdi r5,0 |
|---|
| 628 | beq .Lzero |
|---|
| 629 | b .Lcmp_lt32bytes |
|---|
| 630 | |
|---|
| 631 | .Ldiffoffset_vmx_diff_found: |
|---|
| 632 | EXIT_VMX_OPS |
|---|
| 633 | /* anyway, the diff will appear in next 16 bytes */ |
|---|
| 634 | li r5,16 |
|---|
| 635 | b .Lcmp_lt32bytes |
|---|
| 636 | |
|---|
| 637 | #endif |
|---|
| 638 | EXPORT_SYMBOL(memcmp) |
|---|
| 639 | |
|---|
