1/* Copyright (C) 2002-2019 Free Software Foundation, Inc.
2 This file is part of the GNU C Library.
3 Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
4
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
9
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
14
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <http://www.gnu.org/licenses/>. */
18
19#include <sysdep.h>
20#include <pthread-errnos.h>
21#include <kernel-features.h>
22#include <lowlevellock.h>
23
24#include <stap-probe.h>
25
26 .text
27
28#define LOAD_PRIVATE_FUTEX_WAIT(reg) \
29 movl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
30#define LOAD_PRIVATE_FUTEX_WAKE(reg) \
31 movl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
32#define LOAD_FUTEX_WAIT(reg) \
33 xorl $(FUTEX_WAIT | FUTEX_PRIVATE_FLAG), reg
34#define LOAD_FUTEX_WAIT_ABS(reg) \
35 xorl $(FUTEX_WAIT_BITSET | FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME), reg
36#define LOAD_FUTEX_WAKE(reg) \
37 xorl $(FUTEX_WAKE | FUTEX_PRIVATE_FLAG), reg
38
39
40 .globl __lll_lock_wait_private
41 .type __lll_lock_wait_private,@function
42 .hidden __lll_lock_wait_private
43 .align 16
44__lll_lock_wait_private:
45 cfi_startproc
46 pushq %r10
47 cfi_adjust_cfa_offset(8)
48 pushq %rdx
49 cfi_adjust_cfa_offset(8)
50 cfi_offset(%r10, -16)
51 cfi_offset(%rdx, -24)
52 xorq %r10, %r10 /* No timeout. */
53 movl $2, %edx
54 LOAD_PRIVATE_FUTEX_WAIT (%esi)
55
56 cmpl %edx, %eax /* NB: %edx == 2 */
57 jne 2f
58
591: LIBC_PROBE (lll_lock_wait_private, 1, %rdi)
60 movl $SYS_futex, %eax
61 syscall
62
632: movl %edx, %eax
64 xchgl %eax, (%rdi) /* NB: lock is implied */
65
66 testl %eax, %eax
67 jnz 1b
68
69 popq %rdx
70 cfi_adjust_cfa_offset(-8)
71 cfi_restore(%rdx)
72 popq %r10
73 cfi_adjust_cfa_offset(-8)
74 cfi_restore(%r10)
75 retq
76 cfi_endproc
77 .size __lll_lock_wait_private,.-__lll_lock_wait_private
78
79#if !IS_IN (libc)
80 .globl __lll_lock_wait
81 .type __lll_lock_wait,@function
82 .hidden __lll_lock_wait
83 .align 16
84__lll_lock_wait:
85 cfi_startproc
86 pushq %r10
87 cfi_adjust_cfa_offset(8)
88 pushq %rdx
89 cfi_adjust_cfa_offset(8)
90 cfi_offset(%r10, -16)
91 cfi_offset(%rdx, -24)
92 xorq %r10, %r10 /* No timeout. */
93 movl $2, %edx
94 LOAD_FUTEX_WAIT (%esi)
95
96 cmpl %edx, %eax /* NB: %edx == 2 */
97 jne 2f
98
991: LIBC_PROBE (lll_lock_wait, 2, %rdi, %rsi)
100 movl $SYS_futex, %eax
101 syscall
102
1032: movl %edx, %eax
104 xchgl %eax, (%rdi) /* NB: lock is implied */
105
106 testl %eax, %eax
107 jnz 1b
108
109 popq %rdx
110 cfi_adjust_cfa_offset(-8)
111 cfi_restore(%rdx)
112 popq %r10
113 cfi_adjust_cfa_offset(-8)
114 cfi_restore(%r10)
115 retq
116 cfi_endproc
117 .size __lll_lock_wait,.-__lll_lock_wait
118
119 /* %rdi: futex
120 %rsi: flags
121 %rdx: timeout
122 %eax: futex value
123 */
124 .globl __lll_timedlock_wait
125 .type __lll_timedlock_wait,@function
126 .hidden __lll_timedlock_wait
127 .align 16
128__lll_timedlock_wait:
129 cfi_startproc
130 cmpq $0, (%rdx)
131 js 5f
132
133 pushq %r9
134 cfi_adjust_cfa_offset(8)
135 cfi_rel_offset(%r9, 0)
136
137 movq %rdx, %r10
138 movl $0xffffffff, %r9d
139 LOAD_FUTEX_WAIT_ABS (%esi)
140
141 movl $2, %edx
142 cmpl %edx, %eax
143 jne 2f
144
1451: movl $SYS_futex, %eax
146 movl $2, %edx
147 syscall
148
1492: xchgl %edx, (%rdi) /* NB: lock is implied */
150
151 testl %edx, %edx
152 jz 3f
153
154 cmpl $-ETIMEDOUT, %eax
155 je 4f
156 cmpl $-EINVAL, %eax
157 jne 1b
1584: movl %eax, %edx
159 negl %edx
160
1613: movl %edx, %eax
162 popq %r9
163 cfi_adjust_cfa_offset(-8)
164 cfi_restore(%r9)
165 retq
166
1675: movl $ETIMEDOUT, %eax
168 retq
169
170 cfi_endproc
171 .size __lll_timedlock_wait,.-__lll_timedlock_wait
172#endif
173
174
175 .globl __lll_unlock_wake_private
176 .type __lll_unlock_wake_private,@function
177 .hidden __lll_unlock_wake_private
178 .align 16
179__lll_unlock_wake_private:
180 cfi_startproc
181 pushq %rsi
182 cfi_adjust_cfa_offset(8)
183 pushq %rdx
184 cfi_adjust_cfa_offset(8)
185 cfi_offset(%rsi, -16)
186 cfi_offset(%rdx, -24)
187
188 movl $0, (%rdi)
189 LOAD_PRIVATE_FUTEX_WAKE (%esi)
190 movl $1, %edx /* Wake one thread. */
191 movl $SYS_futex, %eax
192 syscall
193
194 popq %rdx
195 cfi_adjust_cfa_offset(-8)
196 cfi_restore(%rdx)
197 popq %rsi
198 cfi_adjust_cfa_offset(-8)
199 cfi_restore(%rsi)
200 retq
201 cfi_endproc
202 .size __lll_unlock_wake_private,.-__lll_unlock_wake_private
203
204#if !IS_IN (libc)
205 .globl __lll_unlock_wake
206 .type __lll_unlock_wake,@function
207 .hidden __lll_unlock_wake
208 .align 16
209__lll_unlock_wake:
210 cfi_startproc
211 pushq %rsi
212 cfi_adjust_cfa_offset(8)
213 pushq %rdx
214 cfi_adjust_cfa_offset(8)
215 cfi_offset(%rsi, -16)
216 cfi_offset(%rdx, -24)
217
218 movl $0, (%rdi)
219 LOAD_FUTEX_WAKE (%esi)
220 movl $1, %edx /* Wake one thread. */
221 movl $SYS_futex, %eax
222 syscall
223
224 popq %rdx
225 cfi_adjust_cfa_offset(-8)
226 cfi_restore(%rdx)
227 popq %rsi
228 cfi_adjust_cfa_offset(-8)
229 cfi_restore(%rsi)
230 retq
231 cfi_endproc
232 .size __lll_unlock_wake,.-__lll_unlock_wake
233#endif
234