1 | /* |
2 | * Linux/PA-RISC Project (http://www.parisc-linux.org/) |
3 | * |
4 | * System call entry code / Linux gateway page |
5 | * Copyright (c) Matthew Wilcox 1999 <willy@infradead.org> |
6 | * Licensed under the GNU GPL. |
7 | * thanks to Philipp Rumpf, Mike Shaver and various others |
8 | * sorry about the wall, puffin.. |
9 | */ |
10 | |
11 | /* |
12 | How does the Linux gateway page on PA-RISC work? |
13 | ------------------------------------------------ |
14 | The Linux gateway page on PA-RISC is "special". |
15 | It actually has PAGE_GATEWAY bits set (this is linux terminology; in parisc |
16 | terminology it's Execute, promote to PL0) in the page map. So anything |
17 | executing on this page executes with kernel level privilege (there's more to it |
18 | than that: to have this happen, you also have to use a branch with a ,gate |
19 | completer to activate the privilege promotion). The upshot is that everything |
20 | that runs on the gateway page runs at kernel privilege but with the current |
21 | user process address space (although you have access to kernel space via %sr2). |
22 | For the 0x100 syscall entry, we redo the space registers to point to the kernel |
23 | address space (preserving the user address space in %sr3), move to wide mode if |
24 | required, save the user registers and branch into the kernel syscall entry |
25 | point. For all the other functions, we execute at kernel privilege but don't |
26 | flip address spaces. The basic upshot of this is that these code snippets are |
27 | executed atomically (because the kernel can't be pre-empted) and they may |
28 | perform architecturally forbidden (to PL3) operations (like setting control |
29 | registers). |
30 | */ |
31 | |
32 | |
33 | #include <asm/asm-offsets.h> |
34 | #include <asm/unistd.h> |
35 | #include <asm/errno.h> |
36 | #include <asm/page.h> |
37 | #include <asm/psw.h> |
38 | #include <asm/thread_info.h> |
39 | #include <asm/assembly.h> |
40 | #include <asm/processor.h> |
41 | #include <asm/cache.h> |
42 | #include <asm/spinlock_types.h> |
43 | |
44 | #include <linux/linkage.h> |
45 | |
46 | /* We fill the empty parts of the gateway page with |
47 | * something that will kill the kernel or a |
48 | * userspace application. |
49 | */ |
50 | #define KILL_INSN break 0,0 |
51 | |
52 | .level PA_ASM_LEVEL |
53 | |
54 | .macro lws_pagefault_disable reg1,reg2 |
55 | mfctl %cr30, \reg2 |
56 | ldo TASK_PAGEFAULT_DISABLED(\reg2), \reg2 |
57 | ldw 0(%sr2,\reg2), \reg1 |
58 | ldo 1(\reg1), \reg1 |
59 | stw \reg1, 0(%sr2,\reg2) |
60 | .endm |
61 | |
62 | .macro lws_pagefault_enable reg1,reg2 |
63 | mfctl %cr30, \reg2 |
64 | ldo TASK_PAGEFAULT_DISABLED(\reg2), \reg2 |
65 | ldw 0(%sr2,\reg2), \reg1 |
66 | ldo -1(\reg1), \reg1 |
67 | stw \reg1, 0(%sr2,\reg2) |
68 | .endm |
69 | |
70 | /* raise exception if spinlock content is not zero or |
71 | * __ARCH_SPIN_LOCK_UNLOCKED_VAL */ |
72 | .macro spinlock_check spin_val,tmpreg |
73 | #ifdef CONFIG_LIGHTWEIGHT_SPINLOCK_CHECK |
74 | ldi __ARCH_SPIN_LOCK_UNLOCKED_VAL, \tmpreg |
75 | andcm,= \spin_val, \tmpreg, %r0 |
76 | .word SPINLOCK_BREAK_INSN |
77 | #endif |
78 | .endm |
79 | |
80 | .text |
81 | |
82 | .import syscall_exit,code |
83 | .import syscall_exit_rfi,code |
84 | |
85 | /* Linux gateway page is aliased to virtual page 0 in the kernel |
86 | * address space. Since it is a gateway page it cannot be |
87 | * dereferenced, so null pointers will still fault. We start |
88 | * the actual entry point at 0x100. We put break instructions |
89 | * at the beginning of the page to trap null indirect function |
90 | * pointers. |
91 | */ |
92 | |
93 | .align PAGE_SIZE |
94 | ENTRY(linux_gateway_page) |
95 | |
96 | /* ADDRESS 0x00 to 0xb0 = 176 bytes / 4 bytes per insn = 44 insns */ |
97 | .rept 44 |
98 | KILL_INSN |
99 | .endr |
100 | |
101 | /* ADDRESS 0xb0 to 0xb8, lws uses two insns for entry */ |
102 | /* Light-weight-syscall entry must always be located at 0xb0 */ |
103 | /* WARNING: Keep this number updated with table size changes */ |
104 | #define __NR_lws_entries (5) |
105 | |
106 | lws_entry: |
107 | gate lws_start, %r0 /* increase privilege */ |
108 | depi PRIV_USER, 31, 2, %r31 /* Ensure we return into user mode. */ |
109 | |
110 | /* Fill from 0xb8 to 0xe0 */ |
111 | .rept 10 |
112 | KILL_INSN |
113 | .endr |
114 | |
115 | /* This function MUST be located at 0xe0 for glibc's threading |
116 | mechanism to work. DO NOT MOVE THIS CODE EVER! */ |
117 | set_thread_pointer: |
118 | gate .+8, %r0 /* increase privilege */ |
119 | depi PRIV_USER, 31, 2, %r31 /* Ensure we return into user mode. */ |
120 | be 0(%sr7,%r31) /* return to user space */ |
121 | mtctl %r26, %cr27 /* move arg0 to the control register */ |
122 | |
123 | /* Increase the chance of trapping if random jumps occur to this |
124 | address, fill from 0xf0 to 0x100 */ |
125 | .rept 4 |
126 | KILL_INSN |
127 | .endr |
128 | |
129 | /* This address must remain fixed at 0x100 for glibc's syscalls to work */ |
130 | .align LINUX_GATEWAY_ADDR |
131 | linux_gateway_entry: |
132 | gate .+8, %r0 /* become privileged */ |
133 | mtsp %r0,%sr4 /* get kernel space into sr4 */ |
134 | mtsp %r0,%sr5 /* get kernel space into sr5 */ |
135 | mtsp %r0,%sr6 /* get kernel space into sr6 */ |
136 | |
137 | #ifdef CONFIG_64BIT |
138 | /* Store W bit on entry to the syscall in case it's a wide userland |
139 | * process. */ |
140 | ssm PSW_SM_W, %r1 |
141 | extrd,u %r1,PSW_W_BIT,1,%r1 |
142 | /* sp must be aligned on 4, so deposit the W bit setting into |
143 | * the bottom of sp temporarily */ |
144 | or,ev %r1,%r30,%r30 |
145 | b,n 1f |
146 | /* The top halves of argument registers must be cleared on syscall |
147 | * entry from narrow executable. |
148 | */ |
149 | depdi 0, 31, 32, %r26 |
150 | depdi 0, 31, 32, %r25 |
151 | depdi 0, 31, 32, %r24 |
152 | depdi 0, 31, 32, %r23 |
153 | depdi 0, 31, 32, %r22 |
154 | depdi 0, 31, 32, %r21 |
155 | 1: |
156 | #endif |
157 | |
158 | /* We use a rsm/ssm pair to prevent sr3 from being clobbered |
159 | * by external interrupts. |
160 | */ |
161 | mfsp %sr7,%r1 /* save user sr7 */ |
162 | rsm PSW_SM_I, %r0 /* disable interrupts */ |
163 | mtsp %r1,%sr3 /* and store it in sr3 */ |
164 | |
165 | mfctl %cr30,%r1 |
166 | xor %r1,%r30,%r30 /* ye olde xor trick */ |
167 | xor %r1,%r30,%r1 |
168 | xor %r1,%r30,%r30 |
169 | |
170 | LDREG TASK_STACK(%r30),%r30 /* set up kernel stack */ |
171 | ldo FRAME_SIZE(%r30),%r30 |
172 | /* N.B.: It is critical that we don't set sr7 to 0 until r30 |
173 | * contains a valid kernel stack pointer. It is also |
174 | * critical that we don't start using the kernel stack |
175 | * until after sr7 has been set to 0. |
176 | */ |
177 | |
178 | mtsp %r0,%sr7 /* get kernel space into sr7 */ |
179 | ssm PSW_SM_I, %r0 /* enable interrupts */ |
180 | STREGM %r1,FRAME_SIZE(%r30) /* save r1 (usp) here for now */ |
181 | mfctl %cr30,%r1 /* get task ptr in %r1 */ |
182 | |
183 | /* Save some registers for sigcontext and potential task |
184 | switch (see entry.S for the details of which ones are |
185 | saved/restored). TASK_PT_PSW is zeroed so we can see whether |
186 | a process is on a syscall or not. For an interrupt the real |
187 | PSW value is stored. This is needed for gdb and sys_ptrace. */ |
188 | STREG %r0, TASK_PT_PSW(%r1) |
189 | STREG %r2, TASK_PT_GR2(%r1) /* preserve rp */ |
190 | STREG %r19, TASK_PT_GR19(%r1) |
191 | |
192 | LDREGM -FRAME_SIZE(%r30), %r2 /* get users sp back */ |
193 | #ifdef CONFIG_64BIT |
194 | extrd,u %r2,63,1,%r19 /* W hidden in bottom bit */ |
195 | #if 0 |
196 | xor %r19,%r2,%r2 /* clear bottom bit */ |
197 | depd,z %r19,1,1,%r19 |
198 | std %r19,TASK_PT_PSW(%r1) |
199 | #endif |
200 | #endif |
201 | STREG %r2, TASK_PT_GR30(%r1) /* ... and save it */ |
202 | |
203 | STREG %r20, TASK_PT_GR20(%r1) /* Syscall number */ |
204 | STREG %r21, TASK_PT_GR21(%r1) |
205 | STREG %r22, TASK_PT_GR22(%r1) |
206 | STREG %r23, TASK_PT_GR23(%r1) /* 4th argument */ |
207 | STREG %r24, TASK_PT_GR24(%r1) /* 3rd argument */ |
208 | STREG %r25, TASK_PT_GR25(%r1) /* 2nd argument */ |
209 | STREG %r26, TASK_PT_GR26(%r1) /* 1st argument */ |
210 | STREG %r27, TASK_PT_GR27(%r1) /* user dp */ |
211 | STREG %r28, TASK_PT_GR28(%r1) /* return value 0 */ |
212 | STREG %r0, TASK_PT_ORIG_R28(%r1) /* don't prohibit restarts */ |
213 | STREG %r29, TASK_PT_GR29(%r1) /* return value 1 */ |
214 | STREG %r31, TASK_PT_GR31(%r1) /* preserve syscall return ptr */ |
215 | |
216 | ldo TASK_PT_FR0(%r1), %r27 /* save fpregs from the kernel */ |
217 | save_fp %r27 /* or potential task switch */ |
218 | |
219 | mfctl %cr11, %r27 /* i.e. SAR */ |
220 | STREG %r27, TASK_PT_SAR(%r1) |
221 | |
222 | loadgp |
223 | |
224 | #ifdef CONFIG_64BIT |
225 | ldo -16(%r30),%r29 /* Reference param save area */ |
226 | copy %r19,%r2 /* W bit back to r2 */ |
227 | #else |
228 | /* no need to save these on stack in wide mode because the first 8 |
229 | * args are passed in registers */ |
230 | stw %r22, -52(%r30) /* 5th argument */ |
231 | stw %r21, -56(%r30) /* 6th argument */ |
232 | #endif |
233 | |
234 | /* Are we being ptraced? */ |
235 | mfctl %cr30, %r1 |
236 | LDREG TASK_TI_FLAGS(%r1),%r1 |
237 | ldi _TIF_SYSCALL_TRACE_MASK, %r19 |
238 | and,COND(=) %r1, %r19, %r0 |
239 | b,n .Ltracesys |
240 | |
241 | /* Note! We cannot use the syscall table that is mapped |
242 | nearby since the gateway page is mapped execute-only. */ |
243 | |
244 | #ifdef CONFIG_64BIT |
245 | ldil L%sys_call_table, %r1 |
246 | or,= %r2,%r2,%r2 |
247 | addil L%(sys_call_table64-sys_call_table), %r1 |
248 | ldo R%sys_call_table(%r1), %r19 |
249 | or,= %r2,%r2,%r2 |
250 | ldo R%sys_call_table64(%r1), %r19 |
251 | #else |
252 | load32 sys_call_table, %r19 |
253 | #endif |
254 | comiclr,>> __NR_Linux_syscalls, %r20, %r0 |
255 | b,n .Lsyscall_nosys |
256 | |
257 | LDREGX %r20(%r19), %r19 |
258 | |
259 | /* If this is a sys_rt_sigreturn call, and the signal was received |
260 | * when not in_syscall, then we want to return via syscall_exit_rfi, |
261 | * not syscall_exit. Signal no. in r20, in_syscall in r25 (see |
262 | * trampoline code in signal.c). |
263 | */ |
264 | ldi __NR_rt_sigreturn,%r2 |
265 | comb,= %r2,%r20,.Lrt_sigreturn |
266 | .Lin_syscall: |
267 | ldil L%syscall_exit,%r2 |
268 | be 0(%sr7,%r19) |
269 | ldo R%syscall_exit(%r2),%r2 |
270 | .Lrt_sigreturn: |
271 | comib,<> 0,%r25,.Lin_syscall |
272 | ldil L%syscall_exit_rfi,%r2 |
273 | be 0(%sr7,%r19) |
274 | ldo R%syscall_exit_rfi(%r2),%r2 |
275 | |
276 | /* Note! Because we are not running where we were linked, any |
277 | calls to functions external to this file must be indirect. To |
278 | be safe, we apply the opposite rule to functions within this |
279 | file, with local labels given to them to ensure correctness. */ |
280 | |
281 | .Lsyscall_nosys: |
282 | syscall_nosys: |
283 | ldil L%syscall_exit,%r1 |
284 | be R%syscall_exit(%sr7,%r1) |
285 | ldo -ENOSYS(%r0),%r28 /* set errno */ |
286 | |
287 | |
288 | /* Warning! This trace code is a virtual duplicate of the code above so be |
289 | * sure to maintain both! */ |
290 | .Ltracesys: |
291 | tracesys: |
292 | /* Need to save more registers so the debugger can see where we |
293 | * are. This saves only the lower 8 bits of PSW, so that the C |
294 | * bit is still clear on syscalls, and the D bit is set if this |
295 | * full register save path has been executed. We check the D |
296 | * bit on syscall_return_rfi to determine which registers to |
297 | * restore. An interrupt results in a full PSW saved with the |
298 | * C bit set, a non-straced syscall entry results in C and D clear |
299 | * in the saved PSW. |
300 | */ |
301 | mfctl %cr30,%r1 /* get task ptr */ |
302 | ssm 0,%r2 |
303 | STREG %r2,TASK_PT_PSW(%r1) /* Lower 8 bits only!! */ |
304 | mfsp %sr0,%r2 |
305 | STREG %r2,TASK_PT_SR0(%r1) |
306 | mfsp %sr1,%r2 |
307 | STREG %r2,TASK_PT_SR1(%r1) |
308 | mfsp %sr2,%r2 |
309 | STREG %r2,TASK_PT_SR2(%r1) |
310 | mfsp %sr3,%r2 |
311 | STREG %r2,TASK_PT_SR3(%r1) |
312 | STREG %r2,TASK_PT_SR4(%r1) |
313 | STREG %r2,TASK_PT_SR5(%r1) |
314 | STREG %r2,TASK_PT_SR6(%r1) |
315 | STREG %r2,TASK_PT_SR7(%r1) |
316 | STREG %r2,TASK_PT_IASQ0(%r1) |
317 | STREG %r2,TASK_PT_IASQ1(%r1) |
318 | LDREG TASK_PT_GR31(%r1),%r2 |
319 | STREG %r2,TASK_PT_IAOQ0(%r1) |
320 | ldo 4(%r2),%r2 |
321 | STREG %r2,TASK_PT_IAOQ1(%r1) |
322 | ldo TASK_REGS(%r1),%r2 |
323 | /* reg_save %r2 */ |
324 | STREG %r3,PT_GR3(%r2) |
325 | STREG %r4,PT_GR4(%r2) |
326 | STREG %r5,PT_GR5(%r2) |
327 | STREG %r6,PT_GR6(%r2) |
328 | STREG %r7,PT_GR7(%r2) |
329 | STREG %r8,PT_GR8(%r2) |
330 | STREG %r9,PT_GR9(%r2) |
331 | STREG %r10,PT_GR10(%r2) |
332 | STREG %r11,PT_GR11(%r2) |
333 | STREG %r12,PT_GR12(%r2) |
334 | STREG %r13,PT_GR13(%r2) |
335 | STREG %r14,PT_GR14(%r2) |
336 | STREG %r15,PT_GR15(%r2) |
337 | STREG %r16,PT_GR16(%r2) |
338 | STREG %r17,PT_GR17(%r2) |
339 | STREG %r18,PT_GR18(%r2) |
340 | /* Finished saving things for the debugger */ |
341 | |
342 | copy %r2,%r26 |
343 | ldil L%do_syscall_trace_enter,%r1 |
344 | ldil L%tracesys_next,%r2 |
345 | be R%do_syscall_trace_enter(%sr7,%r1) |
346 | ldo R%tracesys_next(%r2),%r2 |
347 | |
348 | tracesys_next: |
349 | /* do_syscall_trace_enter either returned the syscallno, or -1L, |
350 | * so we skip restoring the PT_GR20 below, since we pulled it from |
351 | * task->thread.regs.gr[20] above. |
352 | */ |
353 | copy %ret0,%r20 |
354 | |
355 | mfctl %cr30,%r1 /* get task ptr */ |
356 | LDREG TASK_PT_GR28(%r1), %r28 /* Restore return value */ |
357 | LDREG TASK_PT_GR26(%r1), %r26 /* Restore the users args */ |
358 | LDREG TASK_PT_GR25(%r1), %r25 |
359 | LDREG TASK_PT_GR24(%r1), %r24 |
360 | LDREG TASK_PT_GR23(%r1), %r23 |
361 | LDREG TASK_PT_GR22(%r1), %r22 |
362 | LDREG TASK_PT_GR21(%r1), %r21 |
363 | #ifdef CONFIG_64BIT |
364 | ldo -16(%r30),%r29 /* Reference param save area */ |
365 | #else |
366 | stw %r22, -52(%r30) /* 5th argument */ |
367 | stw %r21, -56(%r30) /* 6th argument */ |
368 | #endif |
369 | |
370 | cmpib,COND(=),n -1,%r20,tracesys_exit /* seccomp may have returned -1 */ |
371 | comiclr,>> __NR_Linux_syscalls, %r20, %r0 |
372 | b,n .Ltracesys_nosys |
373 | |
374 | /* Note! We cannot use the syscall table that is mapped |
375 | nearby since the gateway page is mapped execute-only. */ |
376 | |
377 | #ifdef CONFIG_64BIT |
378 | LDREG TASK_PT_GR30(%r1), %r19 /* get users sp back */ |
379 | extrd,u %r19,63,1,%r2 /* W hidden in bottom bit */ |
380 | |
381 | ldil L%sys_call_table, %r1 |
382 | or,= %r2,%r2,%r2 |
383 | addil L%(sys_call_table64-sys_call_table), %r1 |
384 | ldo R%sys_call_table(%r1), %r19 |
385 | or,= %r2,%r2,%r2 |
386 | ldo R%sys_call_table64(%r1), %r19 |
387 | #else |
388 | load32 sys_call_table, %r19 |
389 | #endif |
390 | |
391 | LDREGX %r20(%r19), %r19 |
392 | |
393 | /* If this is a sys_rt_sigreturn call, and the signal was received |
394 | * when not in_syscall, then we want to return via syscall_exit_rfi, |
395 | * not syscall_exit. Signal no. in r20, in_syscall in r25 (see |
396 | * trampoline code in signal.c). |
397 | */ |
398 | ldi __NR_rt_sigreturn,%r2 |
399 | comb,= %r2,%r20,.Ltrace_rt_sigreturn |
400 | .Ltrace_in_syscall: |
401 | ldil L%tracesys_exit,%r2 |
402 | be 0(%sr7,%r19) |
403 | ldo R%tracesys_exit(%r2),%r2 |
404 | |
405 | .Ltracesys_nosys: |
406 | ldo -ENOSYS(%r0),%r28 /* set errno */ |
407 | |
408 | /* Do *not* call this function on the gateway page, because it |
409 | makes a direct call to syscall_trace. */ |
410 | |
411 | tracesys_exit: |
412 | mfctl %cr30,%r1 /* get task ptr */ |
413 | #ifdef CONFIG_64BIT |
414 | ldo -16(%r30),%r29 /* Reference param save area */ |
415 | #endif |
416 | ldo TASK_REGS(%r1),%r26 |
417 | BL do_syscall_trace_exit,%r2 |
418 | STREG %r28,TASK_PT_GR28(%r1) /* save return value now */ |
419 | mfctl %cr30,%r1 /* get task ptr */ |
420 | LDREG TASK_PT_GR28(%r1), %r28 /* Restore return val. */ |
421 | |
422 | ldil L%syscall_exit,%r1 |
423 | be,n R%syscall_exit(%sr7,%r1) |
424 | |
425 | .Ltrace_rt_sigreturn: |
426 | comib,<> 0,%r25,.Ltrace_in_syscall |
427 | ldil L%tracesys_sigexit,%r2 |
428 | be 0(%sr7,%r19) |
429 | ldo R%tracesys_sigexit(%r2),%r2 |
430 | |
431 | tracesys_sigexit: |
432 | mfctl %cr30,%r1 /* get task ptr */ |
433 | #ifdef CONFIG_64BIT |
434 | ldo -16(%r30),%r29 /* Reference param save area */ |
435 | #endif |
436 | BL do_syscall_trace_exit,%r2 |
437 | ldo TASK_REGS(%r1),%r26 |
438 | |
439 | ldil L%syscall_exit_rfi,%r1 |
440 | be,n R%syscall_exit_rfi(%sr7,%r1) |
441 | |
442 | |
443 | /********************************************************* |
444 | 32/64-bit Light-Weight-Syscall ABI |
445 | |
446 | * - Indicates a hint for userspace inline asm |
447 | implementations. |
448 | |
449 | Syscall number (caller-saves) |
450 | - %r20 |
451 | * In asm clobber. |
452 | |
453 | Argument registers (caller-saves) |
454 | - %r26, %r25, %r24, %r23, %r22 |
455 | * In asm input. |
456 | |
457 | Return registers (caller-saves) |
458 | - %r28 (return), %r21 (errno) |
459 | * In asm output. |
460 | |
461 | Caller-saves registers |
462 | - %r1, %r27, %r29 |
463 | - %r2 (return pointer) |
464 | - %r31 (ble link register) |
465 | * In asm clobber. |
466 | |
467 | Callee-saves registers |
468 | - %r3-%r18 |
469 | - %r30 (stack pointer) |
470 | * Not in asm clobber. |
471 | |
472 | If userspace is 32-bit: |
473 | Callee-saves registers |
474 | - %r19 (32-bit PIC register) |
475 | |
476 | Differences from 32-bit calling convention: |
477 | - Syscall number in %r20 |
478 | - Additional argument register %r22 (arg4) |
479 | - Callee-saves %r19. |
480 | |
481 | If userspace is 64-bit: |
482 | Callee-saves registers |
483 | - %r27 (64-bit PIC register) |
484 | |
485 | Differences from 64-bit calling convention: |
486 | - Syscall number in %r20 |
487 | - Additional argument register %r22 (arg4) |
488 | - Callee-saves %r27. |
489 | |
490 | Error codes returned by entry path: |
491 | |
492 | ENOSYS - r20 was an invalid LWS number. |
493 | |
494 | *********************************************************/ |
495 | lws_start: |
496 | |
497 | #ifdef CONFIG_64BIT |
498 | ssm PSW_SM_W, %r1 |
499 | extrd,u %r1,PSW_W_BIT,1,%r1 |
500 | /* sp must be aligned on 4, so deposit the W bit setting into |
501 | * the bottom of sp temporarily */ |
502 | or,od %r1,%r30,%r30 |
503 | |
504 | /* Clip LWS number to a 32-bit value for 32-bit processes */ |
505 | depdi 0, 31, 32, %r20 |
506 | #endif |
507 | |
508 | /* Is the lws entry number valid? */ |
509 | comiclr,>> __NR_lws_entries, %r20, %r0 |
510 | b,n lws_exit_nosys |
511 | |
512 | /* Load table start */ |
513 | ldil L%lws_table, %r1 |
514 | ldo R%lws_table(%r1), %r28 /* Scratch use of r28 */ |
515 | LDREGX %r20(%sr2,r28), %r21 /* Scratch use of r21 */ |
516 | |
517 | /* Jump to lws, lws table pointers already relocated */ |
518 | be,n 0(%sr2,%r21) |
519 | |
520 | lws_exit_noerror: |
521 | lws_pagefault_enable %r1,%r21 |
522 | ldi __ARCH_SPIN_LOCK_UNLOCKED_VAL, %r21 |
523 | stw,ma %r21, 0(%sr2,%r20) |
524 | ssm PSW_SM_I, %r0 |
525 | b lws_exit |
526 | copy %r0, %r21 |
527 | |
528 | lws_wouldblock: |
529 | ssm PSW_SM_I, %r0 |
530 | ldo 2(%r0), %r28 |
531 | b lws_exit |
532 | ldo -EAGAIN(%r0), %r21 |
533 | |
534 | lws_pagefault: |
535 | lws_pagefault_enable %r1,%r21 |
536 | ldi __ARCH_SPIN_LOCK_UNLOCKED_VAL, %r21 |
537 | stw,ma %r21, 0(%sr2,%r20) |
538 | ssm PSW_SM_I, %r0 |
539 | ldo 3(%r0),%r28 |
540 | b lws_exit |
541 | ldo -EAGAIN(%r0),%r21 |
542 | |
543 | lws_fault: |
544 | ldo 1(%r0),%r28 |
545 | b lws_exit |
546 | ldo -EFAULT(%r0),%r21 |
547 | |
548 | lws_exit_nosys: |
549 | ldo -ENOSYS(%r0),%r21 |
550 | /* Fall through: Return to userspace */ |
551 | |
552 | lws_exit: |
553 | #ifdef CONFIG_64BIT |
554 | /* decide whether to reset the wide mode bit |
555 | * |
556 | * For a syscall, the W bit is stored in the lowest bit |
557 | * of sp. Extract it and reset W if it is zero */ |
558 | extrd,u,*<> %r30,63,1,%r1 |
559 | rsm PSW_SM_W, %r0 |
560 | /* now reset the lowest bit of sp if it was set */ |
561 | xor %r30,%r1,%r30 |
562 | #endif |
563 | be,n 0(%sr7, %r31) |
564 | |
565 | |
566 | |
567 | /*************************************************** |
568 | Implementing 32bit CAS as an atomic operation: |
569 | |
570 | %r26 - Address to examine |
571 | %r25 - Old value to check (old) |
572 | %r24 - New value to set (new) |
573 | %r28 - Return prev through this register. |
574 | %r21 - Kernel error code |
575 | |
576 | %r21 returns the following error codes: |
577 | EAGAIN - CAS is busy, ldcw failed, try again. |
578 | EFAULT - Read or write failed. |
579 | |
580 | If EAGAIN is returned, %r28 indicates the busy reason: |
581 | r28 == 1 - CAS is busy. lock contended. |
582 | r28 == 2 - CAS is busy. ldcw failed. |
583 | r28 == 3 - CAS is busy. page fault. |
584 | |
585 | Scratch: r20, r28, r1 |
586 | |
587 | ****************************************************/ |
588 | |
589 | /* ELF64 Process entry path */ |
590 | lws_compare_and_swap64: |
591 | #ifdef CONFIG_64BIT |
592 | b,n lws_compare_and_swap |
593 | #else |
594 | /* If we are not a 64-bit kernel, then we don't |
595 | * have 64-bit input registers, and calling |
596 | * the 64-bit LWS CAS returns ENOSYS. |
597 | */ |
598 | b,n lws_exit_nosys |
599 | #endif |
600 | |
601 | /* ELF32/ELF64 Process entry path */ |
602 | lws_compare_and_swap32: |
603 | #ifdef CONFIG_64BIT |
604 | /* Wide mode user process? */ |
605 | bb,<,n %sp, 31, lws_compare_and_swap |
606 | |
607 | /* Clip all the input registers for 32-bit processes */ |
608 | depdi 0, 31, 32, %r26 |
609 | depdi 0, 31, 32, %r25 |
610 | depdi 0, 31, 32, %r24 |
611 | #endif |
612 | |
613 | lws_compare_and_swap: |
614 | /* Trigger memory reference interruptions without writing to memory */ |
615 | 1: ldw 0(%r26), %r28 |
616 | 2: stbys,e %r0, 0(%r26) |
617 | |
618 | /* Calculate 8-bit hash index from virtual address */ |
619 | extru_safe %r26, 27, 8, %r20 |
620 | |
621 | /* Load start of lock table */ |
622 | ldil L%lws_lock_start, %r28 |
623 | ldo R%lws_lock_start(%r28), %r28 |
624 | |
625 | /* Find lock to use, the hash index is one of 0 to |
626 | 255, multiplied by 16 (keep it 16-byte aligned) |
627 | and add to the lock table offset. */ |
628 | shlw %r20, 4, %r20 |
629 | add %r20, %r28, %r20 |
630 | |
631 | rsm PSW_SM_I, %r0 /* Disable interrupts */ |
632 | |
633 | /* Try to acquire the lock */ |
634 | LDCW 0(%sr2,%r20), %r28 |
635 | spinlock_check %r28, %r21 |
636 | comclr,<> %r0, %r28, %r0 |
637 | b,n lws_wouldblock |
638 | |
639 | /* Disable page faults to prevent sleeping in critical region */ |
640 | lws_pagefault_disable %r21,%r28 |
641 | |
642 | /* |
643 | prev = *addr; |
644 | if ( prev == old ) |
645 | *addr = new; |
646 | return prev; |
647 | */ |
648 | |
649 | /* NOTES: |
650 | This all works because intr_do_signal |
651 | and schedule both check the return iasq |
652 | and see that we are on the kernel page |
653 | so this process is never scheduled off |
654 | or is ever sent any signal of any sort, |
655 | thus it is wholly atomic from usrspace's |
656 | perspective |
657 | */ |
658 | /* The load and store could fail */ |
659 | 3: ldw 0(%r26), %r28 |
660 | sub,<> %r28, %r25, %r0 |
661 | 4: stw %r24, 0(%r26) |
662 | b,n lws_exit_noerror |
663 | |
664 | /* A fault occurred on load or stbys,e store */ |
665 | 5: b,n lws_fault |
666 | ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 5b-linux_gateway_page) |
667 | ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 5b-linux_gateway_page) |
668 | |
669 | /* A page fault occurred in critical region */ |
670 | 6: b,n lws_pagefault |
671 | ASM_EXCEPTIONTABLE_ENTRY(3b-linux_gateway_page, 6b-linux_gateway_page) |
672 | ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 6b-linux_gateway_page) |
673 | |
674 | |
675 | /*************************************************** |
676 | New CAS implementation which uses pointers and variable size |
677 | information. The value pointed by old and new MUST NOT change |
678 | while performing CAS. The lock only protects the value at %r26. |
679 | |
680 | %r26 - Address to examine |
681 | %r25 - Pointer to the value to check (old) |
682 | %r24 - Pointer to the value to set (new) |
683 | %r23 - Size of the variable (0/1/2/3 for 8/16/32/64 bit) |
684 | %r28 - Return non-zero on failure |
685 | %r21 - Kernel error code |
686 | |
687 | %r21 returns the following error codes: |
688 | EAGAIN - CAS is busy, ldcw failed, try again. |
689 | EFAULT - Read or write failed. |
690 | |
691 | If EAGAIN is returned, %r28 indicates the busy reason: |
692 | r28 == 1 - CAS is busy. lock contended. |
693 | r28 == 2 - CAS is busy. ldcw failed. |
694 | r28 == 3 - CAS is busy. page fault. |
695 | |
696 | Scratch: r20, r22, r28, r29, r1, fr4 (32bit for 64bit CAS only) |
697 | |
698 | ****************************************************/ |
699 | |
700 | lws_compare_and_swap_2: |
701 | #ifdef CONFIG_64BIT |
702 | /* Wide mode user process? */ |
703 | bb,<,n %sp, 31, cas2_begin |
704 | |
705 | /* Clip the input registers for 32-bit processes. We don't |
706 | need to clip %r23 as we only use it for word operations */ |
707 | depdi 0, 31, 32, %r26 |
708 | depdi 0, 31, 32, %r25 |
709 | depdi 0, 31, 32, %r24 |
710 | #endif |
711 | |
712 | cas2_begin: |
713 | /* Check the validity of the size pointer */ |
714 | subi,>>= 3, %r23, %r0 |
715 | b,n lws_exit_nosys |
716 | |
717 | /* Jump to the functions which will load the old and new values into |
718 | registers depending on the their size */ |
719 | shlw %r23, 2, %r29 |
720 | blr %r29, %r0 |
721 | nop |
722 | |
723 | /* 8-bit load */ |
724 | 1: ldb 0(%r25), %r25 |
725 | b cas2_lock_start |
726 | 2: ldb 0(%r24), %r24 |
727 | nop |
728 | nop |
729 | nop |
730 | nop |
731 | nop |
732 | |
733 | /* 16-bit load */ |
734 | 3: ldh 0(%r25), %r25 |
735 | b cas2_lock_start |
736 | 4: ldh 0(%r24), %r24 |
737 | nop |
738 | nop |
739 | nop |
740 | nop |
741 | nop |
742 | |
743 | /* 32-bit load */ |
744 | 5: ldw 0(%r25), %r25 |
745 | b cas2_lock_start |
746 | 6: ldw 0(%r24), %r24 |
747 | nop |
748 | nop |
749 | nop |
750 | nop |
751 | nop |
752 | |
753 | /* 64-bit load */ |
754 | #ifdef CONFIG_64BIT |
755 | 7: ldd 0(%r25), %r25 |
756 | 8: ldd 0(%r24), %r24 |
757 | #else |
758 | /* Load old value into r22/r23 - high/low */ |
759 | 7: ldw 0(%r25), %r22 |
760 | 8: ldw 4(%r25), %r23 |
761 | /* Load new value into fr4 for atomic store later */ |
762 | 9: flddx 0(%r24), %fr4 |
763 | #endif |
764 | |
765 | cas2_lock_start: |
766 | /* Trigger memory reference interruptions without writing to memory */ |
767 | copy %r26, %r28 |
768 | depi_safe 0, 31, 2, %r28 |
769 | 10: ldw 0(%r28), %r1 |
770 | 11: stbys,e %r0, 0(%r28) |
771 | |
772 | /* Calculate 8-bit hash index from virtual address */ |
773 | extru_safe %r26, 27, 8, %r20 |
774 | |
775 | /* Load start of lock table */ |
776 | ldil L%lws_lock_start, %r28 |
777 | ldo R%lws_lock_start(%r28), %r28 |
778 | |
779 | /* Find lock to use, the hash index is one of 0 to |
780 | 255, multiplied by 16 (keep it 16-byte aligned) |
781 | and add to the lock table offset. */ |
782 | shlw %r20, 4, %r20 |
783 | add %r20, %r28, %r20 |
784 | |
785 | rsm PSW_SM_I, %r0 /* Disable interrupts */ |
786 | |
787 | /* Try to acquire the lock */ |
788 | LDCW 0(%sr2,%r20), %r28 |
789 | spinlock_check %r28, %r21 |
790 | comclr,<> %r0, %r28, %r0 |
791 | b,n lws_wouldblock |
792 | |
793 | /* Disable page faults to prevent sleeping in critical region */ |
794 | lws_pagefault_disable %r21,%r28 |
795 | |
796 | /* |
797 | prev = *addr; |
798 | if ( prev == old ) |
799 | *addr = new; |
800 | return prev; |
801 | */ |
802 | |
803 | /* NOTES: |
804 | This all works because intr_do_signal |
805 | and schedule both check the return iasq |
806 | and see that we are on the kernel page |
807 | so this process is never scheduled off |
808 | or is ever sent any signal of any sort, |
809 | thus it is wholly atomic from usrspace's |
810 | perspective |
811 | */ |
812 | |
813 | /* Jump to the correct function */ |
814 | blr %r29, %r0 |
815 | /* Set %r28 as non-zero for now */ |
816 | ldo 1(%r0),%r28 |
817 | |
818 | /* 8-bit CAS */ |
819 | 12: ldb 0(%r26), %r29 |
820 | sub,= %r29, %r25, %r0 |
821 | b,n lws_exit_noerror |
822 | 13: stb %r24, 0(%r26) |
823 | b lws_exit_noerror |
824 | copy %r0, %r28 |
825 | nop |
826 | nop |
827 | |
828 | /* 16-bit CAS */ |
829 | 14: ldh 0(%r26), %r29 |
830 | sub,= %r29, %r25, %r0 |
831 | b,n lws_exit_noerror |
832 | 15: sth %r24, 0(%r26) |
833 | b lws_exit_noerror |
834 | copy %r0, %r28 |
835 | nop |
836 | nop |
837 | |
838 | /* 32-bit CAS */ |
839 | 16: ldw 0(%r26), %r29 |
840 | sub,= %r29, %r25, %r0 |
841 | b,n lws_exit_noerror |
842 | 17: stw %r24, 0(%r26) |
843 | b lws_exit_noerror |
844 | copy %r0, %r28 |
845 | nop |
846 | nop |
847 | |
848 | /* 64-bit CAS */ |
849 | #ifdef CONFIG_64BIT |
850 | 18: ldd 0(%r26), %r29 |
851 | sub,*= %r29, %r25, %r0 |
852 | b,n lws_exit_noerror |
853 | 19: std %r24, 0(%r26) |
854 | copy %r0, %r28 |
855 | #else |
856 | /* Compare first word */ |
857 | 18: ldw 0(%r26), %r29 |
858 | sub,= %r29, %r22, %r0 |
859 | b,n lws_exit_noerror |
860 | /* Compare second word */ |
861 | 19: ldw 4(%r26), %r29 |
862 | sub,= %r29, %r23, %r0 |
863 | b,n lws_exit_noerror |
864 | /* Perform the store */ |
865 | 20: fstdx %fr4, 0(%r26) |
866 | copy %r0, %r28 |
867 | #endif |
868 | b lws_exit_noerror |
869 | copy %r0, %r28 |
870 | |
871 | /* A fault occurred on load or stbys,e store */ |
872 | 30: b,n lws_fault |
873 | ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 30b-linux_gateway_page) |
874 | ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 30b-linux_gateway_page) |
875 | ASM_EXCEPTIONTABLE_ENTRY(3b-linux_gateway_page, 30b-linux_gateway_page) |
876 | ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 30b-linux_gateway_page) |
877 | ASM_EXCEPTIONTABLE_ENTRY(5b-linux_gateway_page, 30b-linux_gateway_page) |
878 | ASM_EXCEPTIONTABLE_ENTRY(6b-linux_gateway_page, 30b-linux_gateway_page) |
879 | ASM_EXCEPTIONTABLE_ENTRY(7b-linux_gateway_page, 30b-linux_gateway_page) |
880 | ASM_EXCEPTIONTABLE_ENTRY(8b-linux_gateway_page, 30b-linux_gateway_page) |
881 | #ifndef CONFIG_64BIT |
882 | ASM_EXCEPTIONTABLE_ENTRY(9b-linux_gateway_page, 30b-linux_gateway_page) |
883 | #endif |
884 | |
885 | ASM_EXCEPTIONTABLE_ENTRY(10b-linux_gateway_page, 30b-linux_gateway_page) |
886 | ASM_EXCEPTIONTABLE_ENTRY(11b-linux_gateway_page, 30b-linux_gateway_page) |
887 | |
888 | /* A page fault occurred in critical region */ |
889 | 31: b,n lws_pagefault |
890 | ASM_EXCEPTIONTABLE_ENTRY(12b-linux_gateway_page, 31b-linux_gateway_page) |
891 | ASM_EXCEPTIONTABLE_ENTRY(13b-linux_gateway_page, 31b-linux_gateway_page) |
892 | ASM_EXCEPTIONTABLE_ENTRY(14b-linux_gateway_page, 31b-linux_gateway_page) |
893 | ASM_EXCEPTIONTABLE_ENTRY(15b-linux_gateway_page, 31b-linux_gateway_page) |
894 | ASM_EXCEPTIONTABLE_ENTRY(16b-linux_gateway_page, 31b-linux_gateway_page) |
895 | ASM_EXCEPTIONTABLE_ENTRY(17b-linux_gateway_page, 31b-linux_gateway_page) |
896 | ASM_EXCEPTIONTABLE_ENTRY(18b-linux_gateway_page, 31b-linux_gateway_page) |
897 | ASM_EXCEPTIONTABLE_ENTRY(19b-linux_gateway_page, 31b-linux_gateway_page) |
898 | #ifndef CONFIG_64BIT |
899 | ASM_EXCEPTIONTABLE_ENTRY(20b-linux_gateway_page, 31b-linux_gateway_page) |
900 | #endif |
901 | |
902 | |
903 | /*************************************************** |
904 | LWS atomic exchange. |
905 | |
906 | %r26 - Exchange address |
907 | %r25 - Size of the variable (0/1/2/3 for 8/16/32/64 bit) |
908 | %r24 - Address of new value |
909 | %r23 - Address of old value |
910 | %r28 - Return non-zero on failure |
911 | %r21 - Kernel error code |
912 | |
913 | %r21 returns the following error codes: |
914 | EAGAIN - CAS is busy, ldcw failed, try again. |
915 | EFAULT - Read or write failed. |
916 | |
917 | If EAGAIN is returned, %r28 indicates the busy reason: |
918 | r28 == 1 - CAS is busy. lock contended. |
919 | r28 == 2 - CAS is busy. ldcw failed. |
920 | r28 == 3 - CAS is busy. page fault. |
921 | |
922 | Scratch: r20, r1 |
923 | |
924 | ****************************************************/ |
925 | |
926 | lws_atomic_xchg: |
927 | #ifdef CONFIG_64BIT |
928 | /* Wide mode user process? */ |
929 | bb,<,n %sp, 31, atomic_xchg_begin |
930 | |
931 | /* Clip the input registers for 32-bit processes. We don't |
932 | need to clip %r23 as we only use it for word operations */ |
933 | depdi 0, 31, 32, %r26 |
934 | depdi 0, 31, 32, %r25 |
935 | depdi 0, 31, 32, %r24 |
936 | depdi 0, 31, 32, %r23 |
937 | #endif |
938 | |
939 | atomic_xchg_begin: |
940 | /* Check the validity of the size pointer */ |
941 | subi,>>= 3, %r25, %r0 |
942 | b,n lws_exit_nosys |
943 | |
944 | /* Jump to the functions which will load the old and new values into |
945 | registers depending on the their size */ |
946 | shlw %r25, 2, %r1 |
947 | blr %r1, %r0 |
948 | nop |
949 | |
950 | /* Perform exception checks */ |
951 | |
952 | /* 8-bit exchange */ |
953 | 1: ldb 0(%r24), %r20 |
954 | copy %r23, %r20 |
955 | depi_safe 0, 31, 2, %r20 |
956 | b atomic_xchg_start |
957 | 2: stbys,e %r0, 0(%r20) |
958 | nop |
959 | nop |
960 | nop |
961 | |
962 | /* 16-bit exchange */ |
963 | 3: ldh 0(%r24), %r20 |
964 | copy %r23, %r20 |
965 | depi_safe 0, 31, 2, %r20 |
966 | b atomic_xchg_start |
967 | 4: stbys,e %r0, 0(%r20) |
968 | nop |
969 | nop |
970 | nop |
971 | |
972 | /* 32-bit exchange */ |
973 | 5: ldw 0(%r24), %r20 |
974 | b atomic_xchg_start |
975 | 6: stbys,e %r0, 0(%r23) |
976 | nop |
977 | nop |
978 | nop |
979 | nop |
980 | nop |
981 | |
982 | /* 64-bit exchange */ |
983 | #ifdef CONFIG_64BIT |
984 | 7: ldd 0(%r24), %r20 |
985 | 8: stdby,e %r0, 0(%r23) |
986 | #else |
987 | 7: ldw 0(%r24), %r20 |
988 | 8: ldw 4(%r24), %r20 |
989 | copy %r23, %r20 |
990 | depi_safe 0, 31, 2, %r20 |
991 | 9: stbys,e %r0, 0(%r20) |
992 | 10: stbys,e %r0, 4(%r20) |
993 | #endif |
994 | |
995 | atomic_xchg_start: |
996 | /* Trigger memory reference interruptions without writing to memory */ |
997 | copy %r26, %r28 |
998 | depi_safe 0, 31, 2, %r28 |
999 | 11: ldw 0(%r28), %r1 |
1000 | 12: stbys,e %r0, 0(%r28) |
1001 | |
1002 | /* Calculate 8-bit hash index from virtual address */ |
1003 | extru_safe %r26, 27, 8, %r20 |
1004 | |
1005 | /* Load start of lock table */ |
1006 | ldil L%lws_lock_start, %r28 |
1007 | ldo R%lws_lock_start(%r28), %r28 |
1008 | |
1009 | /* Find lock to use, the hash index is one of 0 to |
1010 | 255, multiplied by 16 (keep it 16-byte aligned) |
1011 | and add to the lock table offset. */ |
1012 | shlw %r20, 4, %r20 |
1013 | add %r20, %r28, %r20 |
1014 | |
1015 | rsm PSW_SM_I, %r0 /* Disable interrupts */ |
1016 | |
1017 | /* Try to acquire the lock */ |
1018 | LDCW 0(%sr2,%r20), %r28 |
1019 | spinlock_check %r28, %r21 |
1020 | comclr,<> %r0, %r28, %r0 |
1021 | b,n lws_wouldblock |
1022 | |
1023 | /* Disable page faults to prevent sleeping in critical region */ |
1024 | lws_pagefault_disable %r21,%r28 |
1025 | |
1026 | /* NOTES: |
1027 | This all works because intr_do_signal |
1028 | and schedule both check the return iasq |
1029 | and see that we are on the kernel page |
1030 | so this process is never scheduled off |
1031 | or is ever sent any signal of any sort, |
1032 | thus it is wholly atomic from userspace's |
1033 | perspective |
1034 | */ |
1035 | |
1036 | /* Jump to the correct function */ |
1037 | blr %r1, %r0 |
1038 | /* Set %r28 as non-zero for now */ |
1039 | ldo 1(%r0),%r28 |
1040 | |
1041 | /* 8-bit exchange */ |
1042 | 14: ldb 0(%r26), %r1 |
1043 | 15: stb %r1, 0(%r23) |
1044 | 15: ldb 0(%r24), %r1 |
1045 | 17: stb %r1, 0(%r26) |
1046 | b lws_exit_noerror |
1047 | copy %r0, %r28 |
1048 | nop |
1049 | nop |
1050 | |
1051 | /* 16-bit exchange */ |
1052 | 18: ldh 0(%r26), %r1 |
1053 | 19: sth %r1, 0(%r23) |
1054 | 20: ldh 0(%r24), %r1 |
1055 | 21: sth %r1, 0(%r26) |
1056 | b lws_exit_noerror |
1057 | copy %r0, %r28 |
1058 | nop |
1059 | nop |
1060 | |
1061 | /* 32-bit exchange */ |
1062 | 22: ldw 0(%r26), %r1 |
1063 | 23: stw %r1, 0(%r23) |
1064 | 24: ldw 0(%r24), %r1 |
1065 | 25: stw %r1, 0(%r26) |
1066 | b lws_exit_noerror |
1067 | copy %r0, %r28 |
1068 | nop |
1069 | nop |
1070 | |
1071 | /* 64-bit exchange */ |
1072 | #ifdef CONFIG_64BIT |
1073 | 26: ldd 0(%r26), %r1 |
1074 | 27: std %r1, 0(%r23) |
1075 | 28: ldd 0(%r24), %r1 |
1076 | 29: std %r1, 0(%r26) |
1077 | #else |
1078 | 26: flddx 0(%r26), %fr4 |
1079 | 27: fstdx %fr4, 0(%r23) |
1080 | 28: flddx 0(%r24), %fr4 |
1081 | 29: fstdx %fr4, 0(%r26) |
1082 | #endif |
1083 | b lws_exit_noerror |
1084 | copy %r0, %r28 |
1085 | |
1086 | /* A fault occurred on load or stbys,e store */ |
1087 | 30: b,n lws_fault |
1088 | ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 30b-linux_gateway_page) |
1089 | ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 30b-linux_gateway_page) |
1090 | ASM_EXCEPTIONTABLE_ENTRY(3b-linux_gateway_page, 30b-linux_gateway_page) |
1091 | ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 30b-linux_gateway_page) |
1092 | ASM_EXCEPTIONTABLE_ENTRY(5b-linux_gateway_page, 30b-linux_gateway_page) |
1093 | ASM_EXCEPTIONTABLE_ENTRY(6b-linux_gateway_page, 30b-linux_gateway_page) |
1094 | ASM_EXCEPTIONTABLE_ENTRY(7b-linux_gateway_page, 30b-linux_gateway_page) |
1095 | ASM_EXCEPTIONTABLE_ENTRY(8b-linux_gateway_page, 30b-linux_gateway_page) |
1096 | #ifndef CONFIG_64BIT |
1097 | ASM_EXCEPTIONTABLE_ENTRY(9b-linux_gateway_page, 30b-linux_gateway_page) |
1098 | ASM_EXCEPTIONTABLE_ENTRY(10b-linux_gateway_page, 30b-linux_gateway_page) |
1099 | #endif |
1100 | |
1101 | ASM_EXCEPTIONTABLE_ENTRY(11b-linux_gateway_page, 30b-linux_gateway_page) |
1102 | ASM_EXCEPTIONTABLE_ENTRY(12b-linux_gateway_page, 30b-linux_gateway_page) |
1103 | |
1104 | /* A page fault occurred in critical region */ |
1105 | 31: b,n lws_pagefault |
1106 | ASM_EXCEPTIONTABLE_ENTRY(14b-linux_gateway_page, 31b-linux_gateway_page) |
1107 | ASM_EXCEPTIONTABLE_ENTRY(15b-linux_gateway_page, 31b-linux_gateway_page) |
1108 | ASM_EXCEPTIONTABLE_ENTRY(16b-linux_gateway_page, 31b-linux_gateway_page) |
1109 | ASM_EXCEPTIONTABLE_ENTRY(17b-linux_gateway_page, 31b-linux_gateway_page) |
1110 | ASM_EXCEPTIONTABLE_ENTRY(18b-linux_gateway_page, 31b-linux_gateway_page) |
1111 | ASM_EXCEPTIONTABLE_ENTRY(19b-linux_gateway_page, 31b-linux_gateway_page) |
1112 | ASM_EXCEPTIONTABLE_ENTRY(20b-linux_gateway_page, 31b-linux_gateway_page) |
1113 | ASM_EXCEPTIONTABLE_ENTRY(21b-linux_gateway_page, 31b-linux_gateway_page) |
1114 | ASM_EXCEPTIONTABLE_ENTRY(22b-linux_gateway_page, 31b-linux_gateway_page) |
1115 | ASM_EXCEPTIONTABLE_ENTRY(23b-linux_gateway_page, 31b-linux_gateway_page) |
1116 | ASM_EXCEPTIONTABLE_ENTRY(24b-linux_gateway_page, 31b-linux_gateway_page) |
1117 | ASM_EXCEPTIONTABLE_ENTRY(25b-linux_gateway_page, 31b-linux_gateway_page) |
1118 | ASM_EXCEPTIONTABLE_ENTRY(26b-linux_gateway_page, 31b-linux_gateway_page) |
1119 | ASM_EXCEPTIONTABLE_ENTRY(27b-linux_gateway_page, 31b-linux_gateway_page) |
1120 | ASM_EXCEPTIONTABLE_ENTRY(28b-linux_gateway_page, 31b-linux_gateway_page) |
1121 | ASM_EXCEPTIONTABLE_ENTRY(29b-linux_gateway_page, 31b-linux_gateway_page) |
1122 | |
1123 | /*************************************************** |
1124 | LWS atomic store. |
1125 | |
1126 | %r26 - Address to store |
1127 | %r25 - Size of the variable (0/1/2/3 for 8/16/32/64 bit) |
1128 | %r24 - Address of value to store |
1129 | %r28 - Return non-zero on failure |
1130 | %r21 - Kernel error code |
1131 | |
1132 | %r21 returns the following error codes: |
1133 | EAGAIN - CAS is busy, ldcw failed, try again. |
1134 | EFAULT - Read or write failed. |
1135 | |
1136 | If EAGAIN is returned, %r28 indicates the busy reason: |
1137 | r28 == 1 - CAS is busy. lock contended. |
1138 | r28 == 2 - CAS is busy. ldcw failed. |
1139 | r28 == 3 - CAS is busy. page fault. |
1140 | |
1141 | Scratch: r20, r1 |
1142 | |
1143 | ****************************************************/ |
1144 | |
1145 | lws_atomic_store: |
1146 | #ifdef CONFIG_64BIT |
1147 | /* Wide mode user process? */ |
1148 | bb,<,n %sp, 31, atomic_store_begin |
1149 | |
1150 | /* Clip the input registers for 32-bit processes. We don't |
1151 | need to clip %r23 as we only use it for word operations */ |
1152 | depdi 0, 31, 32, %r26 |
1153 | depdi 0, 31, 32, %r25 |
1154 | depdi 0, 31, 32, %r24 |
1155 | #endif |
1156 | |
1157 | atomic_store_begin: |
1158 | /* Check the validity of the size pointer */ |
1159 | subi,>>= 3, %r25, %r0 |
1160 | b,n lws_exit_nosys |
1161 | |
1162 | shlw %r25, 1, %r1 |
1163 | blr %r1, %r0 |
1164 | nop |
1165 | |
1166 | /* Perform exception checks */ |
1167 | |
1168 | /* 8-bit store */ |
1169 | 1: ldb 0(%r24), %r20 |
1170 | b,n atomic_store_start |
1171 | nop |
1172 | nop |
1173 | |
1174 | /* 16-bit store */ |
1175 | 2: ldh 0(%r24), %r20 |
1176 | b,n atomic_store_start |
1177 | nop |
1178 | nop |
1179 | |
1180 | /* 32-bit store */ |
1181 | 3: ldw 0(%r24), %r20 |
1182 | b,n atomic_store_start |
1183 | nop |
1184 | nop |
1185 | |
1186 | /* 64-bit store */ |
1187 | #ifdef CONFIG_64BIT |
1188 | 4: ldd 0(%r24), %r20 |
1189 | #else |
1190 | 4: ldw 0(%r24), %r20 |
1191 | 5: ldw 4(%r24), %r20 |
1192 | #endif |
1193 | |
1194 | atomic_store_start: |
1195 | /* Trigger memory reference interruptions without writing to memory */ |
1196 | copy %r26, %r28 |
1197 | depi_safe 0, 31, 2, %r28 |
1198 | 6: ldw 0(%r28), %r1 |
1199 | 7: stbys,e %r0, 0(%r28) |
1200 | |
1201 | /* Calculate 8-bit hash index from virtual address */ |
1202 | extru_safe %r26, 27, 8, %r20 |
1203 | |
1204 | /* Load start of lock table */ |
1205 | ldil L%lws_lock_start, %r28 |
1206 | ldo R%lws_lock_start(%r28), %r28 |
1207 | |
1208 | /* Find lock to use, the hash index is one of 0 to |
1209 | 255, multiplied by 16 (keep it 16-byte aligned) |
1210 | and add to the lock table offset. */ |
1211 | shlw %r20, 4, %r20 |
1212 | add %r20, %r28, %r20 |
1213 | |
1214 | rsm PSW_SM_I, %r0 /* Disable interrupts */ |
1215 | |
1216 | /* Try to acquire the lock */ |
1217 | LDCW 0(%sr2,%r20), %r28 |
1218 | spinlock_check %r28, %r21 |
1219 | comclr,<> %r0, %r28, %r0 |
1220 | b,n lws_wouldblock |
1221 | |
1222 | /* Disable page faults to prevent sleeping in critical region */ |
1223 | lws_pagefault_disable %r21,%r28 |
1224 | |
1225 | /* NOTES: |
1226 | This all works because intr_do_signal |
1227 | and schedule both check the return iasq |
1228 | and see that we are on the kernel page |
1229 | so this process is never scheduled off |
1230 | or is ever sent any signal of any sort, |
1231 | thus it is wholly atomic from userspace's |
1232 | perspective |
1233 | */ |
1234 | |
1235 | /* Jump to the correct function */ |
1236 | blr %r1, %r0 |
1237 | /* Set %r28 as non-zero for now */ |
1238 | ldo 1(%r0),%r28 |
1239 | |
1240 | /* 8-bit store */ |
1241 | 9: ldb 0(%r24), %r1 |
1242 | 10: stb %r1, 0(%r26) |
1243 | b lws_exit_noerror |
1244 | copy %r0, %r28 |
1245 | |
1246 | /* 16-bit store */ |
1247 | 11: ldh 0(%r24), %r1 |
1248 | 12: sth %r1, 0(%r26) |
1249 | b lws_exit_noerror |
1250 | copy %r0, %r28 |
1251 | |
1252 | /* 32-bit store */ |
1253 | 13: ldw 0(%r24), %r1 |
1254 | 14: stw %r1, 0(%r26) |
1255 | b lws_exit_noerror |
1256 | copy %r0, %r28 |
1257 | |
1258 | /* 64-bit store */ |
1259 | #ifdef CONFIG_64BIT |
1260 | 15: ldd 0(%r24), %r1 |
1261 | 16: std %r1, 0(%r26) |
1262 | #else |
1263 | 15: flddx 0(%r24), %fr4 |
1264 | 16: fstdx %fr4, 0(%r26) |
1265 | #endif |
1266 | b lws_exit_noerror |
1267 | copy %r0, %r28 |
1268 | |
1269 | /* A fault occurred on load or stbys,e store */ |
1270 | 30: b,n lws_fault |
1271 | ASM_EXCEPTIONTABLE_ENTRY(1b-linux_gateway_page, 30b-linux_gateway_page) |
1272 | ASM_EXCEPTIONTABLE_ENTRY(2b-linux_gateway_page, 30b-linux_gateway_page) |
1273 | ASM_EXCEPTIONTABLE_ENTRY(3b-linux_gateway_page, 30b-linux_gateway_page) |
1274 | ASM_EXCEPTIONTABLE_ENTRY(4b-linux_gateway_page, 30b-linux_gateway_page) |
1275 | #ifndef CONFIG_64BIT |
1276 | ASM_EXCEPTIONTABLE_ENTRY(5b-linux_gateway_page, 30b-linux_gateway_page) |
1277 | #endif |
1278 | |
1279 | ASM_EXCEPTIONTABLE_ENTRY(6b-linux_gateway_page, 30b-linux_gateway_page) |
1280 | ASM_EXCEPTIONTABLE_ENTRY(7b-linux_gateway_page, 30b-linux_gateway_page) |
1281 | |
1282 | /* A page fault occurred in critical region */ |
1283 | 31: b,n lws_pagefault |
1284 | ASM_EXCEPTIONTABLE_ENTRY(9b-linux_gateway_page, 31b-linux_gateway_page) |
1285 | ASM_EXCEPTIONTABLE_ENTRY(10b-linux_gateway_page, 31b-linux_gateway_page) |
1286 | ASM_EXCEPTIONTABLE_ENTRY(11b-linux_gateway_page, 31b-linux_gateway_page) |
1287 | ASM_EXCEPTIONTABLE_ENTRY(12b-linux_gateway_page, 31b-linux_gateway_page) |
1288 | ASM_EXCEPTIONTABLE_ENTRY(13b-linux_gateway_page, 31b-linux_gateway_page) |
1289 | ASM_EXCEPTIONTABLE_ENTRY(14b-linux_gateway_page, 31b-linux_gateway_page) |
1290 | ASM_EXCEPTIONTABLE_ENTRY(15b-linux_gateway_page, 31b-linux_gateway_page) |
1291 | ASM_EXCEPTIONTABLE_ENTRY(16b-linux_gateway_page, 31b-linux_gateway_page) |
1292 | |
1293 | /* Make sure nothing else is placed on this page */ |
1294 | .align PAGE_SIZE |
1295 | END(linux_gateway_page) |
1296 | ENTRY(end_linux_gateway_page) |
1297 | |
1298 | /* Relocate symbols assuming linux_gateway_page is mapped |
1299 | to virtual address 0x0 */ |
1300 | |
1301 | #define LWS_ENTRY(_name_) ASM_ULONG_INSN (lws_##_name_ - linux_gateway_page) |
1302 | |
1303 | .section .rodata,"a" |
1304 | |
1305 | .align 8 |
1306 | /* Light-weight-syscall table */ |
1307 | /* Start of lws table. */ |
1308 | ENTRY(lws_table) |
1309 | LWS_ENTRY(compare_and_swap32) /* 0 - ELF32 Atomic 32bit CAS */ |
1310 | LWS_ENTRY(compare_and_swap64) /* 1 - ELF64 Atomic 32bit CAS */ |
1311 | LWS_ENTRY(compare_and_swap_2) /* 2 - Atomic 64bit CAS */ |
1312 | LWS_ENTRY(atomic_xchg) /* 3 - Atomic Exchange */ |
1313 | LWS_ENTRY(atomic_store) /* 4 - Atomic Store */ |
1314 | END(lws_table) |
1315 | /* End of lws table */ |
1316 | |
1317 | #ifdef CONFIG_64BIT |
1318 | #define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, compat) |
1319 | #else |
1320 | #define __SYSCALL_WITH_COMPAT(nr, native, compat) __SYSCALL(nr, native) |
1321 | #endif |
1322 | #define __SYSCALL(nr, entry) ASM_ULONG_INSN entry |
1323 | .align 8 |
1324 | ENTRY(sys_call_table) |
1325 | .export sys_call_table,data |
1326 | #include <asm/syscall_table_32.h> /* 32-bit syscalls */ |
1327 | END(sys_call_table) |
1328 | |
1329 | #ifdef CONFIG_64BIT |
1330 | .align 8 |
1331 | ENTRY(sys_call_table64) |
1332 | #include <asm/syscall_table_64.h> /* 64-bit syscalls */ |
1333 | END(sys_call_table64) |
1334 | #endif |
1335 | |
1336 | /* |
1337 | All light-weight-syscall atomic operations |
1338 | will use this set of locks |
1339 | |
1340 | NOTE: The lws_lock_start symbol must be |
1341 | at least 16-byte aligned for safe use |
1342 | with ldcw. |
1343 | */ |
1344 | .section .data |
1345 | .align L1_CACHE_BYTES |
1346 | ENTRY(lws_lock_start) |
1347 | /* lws locks */ |
1348 | .rept 256 |
1349 | /* Keep locks aligned at 16-bytes */ |
1350 | .word __ARCH_SPIN_LOCK_UNLOCKED_VAL |
1351 | .word 0 |
1352 | .word 0 |
1353 | .word 0 |
1354 | .endr |
1355 | END(lws_lock_start) |
1356 | .previous |
1357 | |
1358 | .end |
1359 | |