(lll_futex_wait, lll_futex_timedwait, lll_wait_tid): Add "memory" clobber.
[kopensolaris-gnu/glibc.git] / nptl / sysdeps / unix / sysv / linux / i386 / lowlevellock.h
1 /* Copyright (C) 2002, 2003, 2004, 2006 Free Software Foundation, Inc.
2    This file is part of the GNU C Library.
3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
4
5    The GNU C Library is free software; you can redistribute it and/or
6    modify it under the terms of the GNU Lesser General Public
7    License as published by the Free Software Foundation; either
8    version 2.1 of the License, or (at your option) any later version.
9
10    The GNU C Library is distributed in the hope that it will be useful,
11    but WITHOUT ANY WARRANTY; without even the implied warranty of
12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13    Lesser General Public License for more details.
14
15    You should have received a copy of the GNU Lesser General Public
16    License along with the GNU C Library; if not, write to the Free
17    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
18    02111-1307 USA.  */
19
20 #ifndef _LOWLEVELLOCK_H
21 #define _LOWLEVELLOCK_H 1
22
23 #include <time.h>
24 #include <sys/param.h>
25 #include <bits/pthreadtypes.h>
26
27 #ifndef LOCK_INSTR
28 # ifdef UP
29 #  define LOCK_INSTR    /* nothing */
30 # else
31 #  define LOCK_INSTR "lock;"
32 # endif
33 #endif
34
35 #define SYS_futex               240
36 #define FUTEX_WAIT              0
37 #define FUTEX_WAKE              1
38
39
40 /* Initializer for compatibility lock.  */
41 #define LLL_MUTEX_LOCK_INITIALIZER              (0)
42 #define LLL_MUTEX_LOCK_INITIALIZER_LOCKED       (1)
43 #define LLL_MUTEX_LOCK_INITIALIZER_WAITERS      (2)
44
45
46 #ifdef PIC
47 # define LLL_EBX_LOAD   "xchgl %2, %%ebx\n"
48 # define LLL_EBX_REG    "D"
49 #else
50 # define LLL_EBX_LOAD
51 # define LLL_EBX_REG    "b"
52 #endif
53
54 #ifdef I386_USE_SYSENTER
55 # ifdef SHARED
56 #  define LLL_ENTER_KERNEL      "call *%%gs:%P6\n\t"
57 # else
58 #  define LLL_ENTER_KERNEL      "call *_dl_sysinfo\n\t"
59 # endif
60 #else
61 # define LLL_ENTER_KERNEL       "int $0x80\n\t"
62 #endif
63
64 /* Delay in spinlock loop.  */
65 #define BUSY_WAIT_NOP          asm ("rep; nop")
66
67
68 #define lll_futex_wait(futex, val) \
69   ({                                                                          \
70     int __status;                                                             \
71     register __typeof (val) _val asm ("edx") = (val);                         \
72     __asm __volatile (LLL_EBX_LOAD                                            \
73                       LLL_ENTER_KERNEL                                        \
74                       LLL_EBX_LOAD                                            \
75                       : "=a" (__status)                                       \
76                       : "0" (SYS_futex), LLL_EBX_REG (futex), "S" (0),        \
77                         "c" (FUTEX_WAIT), "d" (_val),                         \
78                         "i" (offsetof (tcbhead_t, sysinfo))                   \
79                       : "memory");                                            \
80     __status;                                                                 \
81   })
82
83
84 #define lll_futex_timed_wait(futex, val, timeout)                             \
85   ({                                                                          \
86     int __status;                                                             \
87     register __typeof (val) _val asm ("edx") = (val);                         \
88     __asm __volatile (LLL_EBX_LOAD                                            \
89                       LLL_ENTER_KERNEL                                        \
90                       LLL_EBX_LOAD                                            \
91                       : "=a" (__status)                                       \
92                       : "0" (SYS_futex), LLL_EBX_REG (futex), "S" (timeout),  \
93                         "c" (FUTEX_WAIT), "d" (_val),                         \
94                         "i" (offsetof (tcbhead_t, sysinfo))                   \
95                       : "memory");                                            \
96     __status;                                                                 \
97   })
98
99
100 #define lll_futex_wake(futex, nr) \
101   do {                                                                        \
102     int __ignore;                                                             \
103     register __typeof (nr) _nr asm ("edx") = (nr);                            \
104     __asm __volatile (LLL_EBX_LOAD                                            \
105                       LLL_ENTER_KERNEL                                        \
106                       LLL_EBX_LOAD                                            \
107                       : "=a" (__ignore)                                       \
108                       : "0" (SYS_futex), LLL_EBX_REG (futex),                 \
109                         "c" (FUTEX_WAKE), "d" (_nr),                          \
110                         "i" (0) /* phony, to align next arg's number */,      \
111                         "i" (offsetof (tcbhead_t, sysinfo)));                 \
112   } while (0)
113
114
115 /* Does not preserve %eax and %ecx.  */
116 extern int __lll_mutex_lock_wait (int val, int *__futex)
117      __attribute ((regparm (2))) attribute_hidden;
118 /* Does not preserve %eax, %ecx, and %edx.  */
119 extern int __lll_mutex_timedlock_wait (int val, int *__futex,
120                                        const struct timespec *abstime)
121      __attribute ((regparm (3))) attribute_hidden;
122 /* Preserves all registers but %eax.  */
123 extern int __lll_mutex_unlock_wake (int *__futex)
124      __attribute ((regparm (1))) attribute_hidden;
125
126
127 /* NB: in the lll_mutex_trylock macro we simply return the value in %eax
128    after the cmpxchg instruction.  In case the operation succeded this
129    value is zero.  In case the operation failed, the cmpxchg instruction
130    has loaded the current value of the memory work which is guaranteed
131    to be nonzero.  */
132 #define lll_mutex_trylock(futex) \
133   ({ int ret;                                                                 \
134      __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1"                           \
135                        : "=a" (ret), "=m" (futex)                             \
136                        : "r" (LLL_MUTEX_LOCK_INITIALIZER_LOCKED), "m" (futex),\
137                          "0" (LLL_MUTEX_LOCK_INITIALIZER)                     \
138                        : "memory");                                           \
139      ret; })
140
141
142 #define lll_mutex_cond_trylock(futex) \
143   ({ int ret;                                                                 \
144      __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1"                           \
145                        : "=a" (ret), "=m" (futex)                             \
146                        : "r" (LLL_MUTEX_LOCK_INITIALIZER_WAITERS),            \
147                           "m" (futex), "0" (LLL_MUTEX_LOCK_INITIALIZER)       \
148                        : "memory");                                           \
149      ret; })
150
151
152 #define lll_mutex_lock(futex) \
153   (void) ({ int ignore1, ignore2;                                             \
154             __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t"                \
155                               "jnz _L_mutex_lock_%=\n\t"                      \
156                               ".subsection 1\n\t"                             \
157                               ".type _L_mutex_lock_%=,@function\n"            \
158                               "_L_mutex_lock_%=:\n\t"                         \
159                               "leal %2, %%ecx\n\t"                            \
160                               "call __lll_mutex_lock_wait\n\t"                \
161                               "jmp 1f\n\t"                                    \
162                               ".size _L_mutex_lock_%=,.-_L_mutex_lock_%=\n"   \
163                               ".previous\n"                                   \
164                               "1:"                                            \
165                               : "=a" (ignore1), "=c" (ignore2), "=m" (futex)  \
166                               : "0" (0), "1" (1), "m" (futex)                 \
167                               : "memory"); })
168
169
170 /* Special version of lll_mutex_lock which causes the unlock function to
171    always wakeup waiters.  */
172 #define lll_mutex_cond_lock(futex) \
173   (void) ({ int ignore1, ignore2;                                             \
174             __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t"                \
175                               "jnz _L_mutex_cond_lock_%=\n\t"                 \
176                               ".subsection 1\n\t"                             \
177                               ".type _L_mutex_cond_lock_%=,@function\n"       \
178                               "_L_mutex_cond_lock_%=:\n\t"                    \
179                               "leal %2, %%ecx\n\t"                            \
180                               "call __lll_mutex_lock_wait\n\t"                \
181                               "jmp 1f\n\t"                                    \
182                               ".size _L_mutex_cond_lock_%=,.-_L_mutex_cond_lock_%=\n"   \
183                               ".previous\n"                                   \
184                               "1:"                                            \
185                               : "=a" (ignore1), "=c" (ignore2), "=m" (futex)  \
186                               : "0" (0), "1" (2), "m" (futex)                 \
187                               : "memory"); })
188
189
190 #define lll_mutex_timedlock(futex, timeout) \
191   ({ int result, ignore1, ignore2;                                            \
192      __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t"                       \
193                        "jnz _L_mutex_timedlock_%=\n\t"                        \
194                        ".subsection 1\n\t"                                    \
195                        ".type _L_mutex_timedlock_%=,@function\n"              \
196                        "_L_mutex_timedlock_%=:\n\t"                           \
197                        "leal %3, %%ecx\n\t"                                   \
198                        "movl %7, %%edx\n\t"                                   \
199                        "call __lll_mutex_timedlock_wait\n\t"                  \
200                        "jmp 1f\n\t"                                           \
201                        ".size _L_mutex_timedlock_%=,.-_L_mutex_timedlock_%=\n"\
202                        ".previous\n"                                          \
203                        "1:"                                                   \
204                        : "=a" (result), "=c" (ignore1), "=&d" (ignore2),      \
205                          "=m" (futex)                                         \
206                        : "0" (0), "1" (1), "m" (futex), "m" (timeout)         \
207                        : "memory");                                           \
208      result; })
209
210
211 #define lll_mutex_unlock(futex) \
212   (void) ({ int ignore;                                                       \
213             __asm __volatile (LOCK_INSTR "subl $1,%0\n\t"                     \
214                               "jne _L_mutex_unlock_%=\n\t"                    \
215                               ".subsection 1\n\t"                             \
216                               ".type _L_mutex_unlock_%=,@function\n"          \
217                               "_L_mutex_unlock_%=:\n\t"                       \
218                               "leal %0, %%eax\n\t"                            \
219                               "call __lll_mutex_unlock_wake\n\t"              \
220                               "jmp 1f\n\t"                                    \
221                               ".size _L_mutex_unlock_%=,.-_L_mutex_unlock_%=\n" \
222                               ".previous\n"                                   \
223                               "1:"                                            \
224                               : "=m" (futex), "=&a" (ignore)                  \
225                               : "m" (futex)                                   \
226                               : "memory"); })
227
228
229 #define lll_mutex_islocked(futex) \
230   (futex != 0)
231
232
233 /* We have a separate internal lock implementation which is not tied
234    to binary compatibility.  */
235
236 /* Type for lock object.  */
237 typedef int lll_lock_t;
238
239 /* Initializers for lock.  */
240 #define LLL_LOCK_INITIALIZER            (0)
241 #define LLL_LOCK_INITIALIZER_LOCKED     (1)
242
243
244 extern int __lll_lock_wait (int val, int *__futex)
245      __attribute ((regparm (2))) attribute_hidden;
246 extern int __lll_unlock_wake (int *__futex)
247      __attribute ((regparm (1))) attribute_hidden;
248 extern int lll_unlock_wake_cb (int *__futex) attribute_hidden;
249
250
251 /* The states of a lock are:
252     0  -  untaken
253     1  -  taken by one user
254     2  -  taken by more users */
255
256
257 #if defined NOT_IN_libc || defined UP
258 # define lll_trylock(futex) lll_mutex_trylock (futex)
259 # define lll_lock(futex) lll_mutex_lock (futex)
260 # define lll_unlock(futex) lll_mutex_unlock (futex)
261 #else
262 /* Special versions of the macros for use in libc itself.  They avoid
263    the lock prefix when the thread library is not used.
264
265    XXX In future we might even want to avoid it on UP machines.  */
266 # include <tls.h>
267
268 # define lll_trylock(futex) \
269   ({ unsigned char ret;                                                       \
270      __asm __volatile ("cmpl $0, %%gs:%P5\n\t"                                \
271                        "je,pt 0f\n\t"                                         \
272                        "lock\n"                                               \
273                        "0:\tcmpxchgl %2, %1; setne %0"                        \
274                        : "=a" (ret), "=m" (futex)                             \
275                        : "r" (LLL_MUTEX_LOCK_INITIALIZER_LOCKED), "m" (futex),\
276                          "0" (LLL_MUTEX_LOCK_INITIALIZER),                    \
277                          "i" (offsetof (tcbhead_t, multiple_threads))         \
278                        : "memory");                                           \
279      ret; })
280
281
282 # define lll_lock(futex) \
283   (void) ({ int ignore1, ignore2;                                             \
284             __asm __volatile ("cmpl $0, %%gs:%P6\n\t"                         \
285                               "je,pt 0f\n\t"                                  \
286                               "lock\n"                                        \
287                               "0:\tcmpxchgl %1, %2\n\t"                       \
288                               "jnz _L_mutex_lock_%=\n\t"                      \
289                               ".subsection 1\n\t"                             \
290                               ".type _L_mutex_lock_%=,@function\n"            \
291                               "_L_mutex_lock_%=:\n\t"                         \
292                               "leal %2, %%ecx\n\t"                            \
293                               "call __lll_mutex_lock_wait\n\t"                \
294                               "jmp 1f\n\t"                                    \
295                               ".size _L_mutex_lock_%=,.-_L_mutex_lock_%=\n"   \
296                               ".previous\n"                                   \
297                               "1:"                                            \
298                               : "=a" (ignore1), "=c" (ignore2), "=m" (futex)  \
299                               : "0" (0), "1" (1), "m" (futex),                \
300                                 "i" (offsetof (tcbhead_t, multiple_threads))  \
301                               : "memory"); })
302
303
304 # define lll_unlock(futex) \
305   (void) ({ int ignore;                                                       \
306             __asm __volatile ("cmpl $0, %%gs:%P3\n\t"                         \
307                               "je,pt 0f\n\t"                                  \
308                               "lock\n"                                        \
309                               "0:\tsubl $1,%0\n\t"                    \
310                               "jne _L_mutex_unlock_%=\n\t"                    \
311                               ".subsection 1\n\t"                             \
312                               ".type _L_mutex_unlock_%=,@function\n"          \
313                               "_L_mutex_unlock_%=:\n\t"                       \
314                               "leal %0, %%eax\n\t"                            \
315                               "call __lll_mutex_unlock_wake\n\t"              \
316                               "jmp 1f\n\t"                                    \
317                               ".size _L_mutex_unlock_%=,.-_L_mutex_unlock_%=\n" \
318                               ".previous\n"                                   \
319                               "1:"                                            \
320                               : "=m" (futex), "=&a" (ignore)                  \
321                               : "m" (futex),                                  \
322                                 "i" (offsetof (tcbhead_t, multiple_threads))  \
323                               : "memory"); })
324 #endif
325
326
327 #define lll_islocked(futex) \
328   (futex != LLL_LOCK_INITIALIZER)
329
330
331 /* The kernel notifies a process with uses CLONE_CLEARTID via futex
332    wakeup when the clone terminates.  The memory location contains the
333    thread ID while the clone is running and is reset to zero
334    afterwards.
335
336    The macro parameter must not have any side effect.  */
337 #define lll_wait_tid(tid) \
338   do {                                                                        \
339     int __ignore;                                                             \
340     register __typeof (tid) _tid asm ("edx") = (tid);                         \
341     if (_tid != 0)                                                            \
342       __asm __volatile (LLL_EBX_LOAD                                          \
343                         "1:\tmovl %1, %%eax\n\t"                              \
344                         LLL_ENTER_KERNEL                                      \
345                         "cmpl $0, (%%ebx)\n\t"                                \
346                         "jne,pn 1b\n\t"                                       \
347                         LLL_EBX_LOAD                                          \
348                         : "=&a" (__ignore)                                    \
349                         : "i" (SYS_futex), LLL_EBX_REG (&tid), "S" (0),       \
350                           "c" (FUTEX_WAIT), "d" (_tid),                       \
351                           "i" (offsetof (tcbhead_t, sysinfo))                 \
352                         : "memory");                                          \
353   } while (0)
354
355 extern int __lll_timedwait_tid (int *tid, const struct timespec *abstime)
356      __attribute__ ((regparm (2))) attribute_hidden;
357 #define lll_timedwait_tid(tid, abstime) \
358   ({                                                                          \
359     int __result = 0;                                                         \
360     if (tid != 0)                                                             \
361       {                                                                       \
362         if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)           \
363           __result = EINVAL;                                                  \
364         else                                                                  \
365           __result = __lll_timedwait_tid (&tid, abstime);                     \
366       }                                                                       \
367     __result; })
368
369
370 /* Conditional variable handling.  */
371
372 extern void __lll_cond_wait (pthread_cond_t *cond)
373      __attribute ((regparm (1))) attribute_hidden;
374 extern int __lll_cond_timedwait (pthread_cond_t *cond,
375                                  const struct timespec *abstime)
376      __attribute ((regparm (2))) attribute_hidden;
377 extern void __lll_cond_wake (pthread_cond_t *cond)
378      __attribute ((regparm (1))) attribute_hidden;
379 extern void __lll_cond_broadcast (pthread_cond_t *cond)
380      __attribute ((regparm (1))) attribute_hidden;
381
382
383 #define lll_cond_wait(cond) \
384   __lll_cond_wait (cond)
385 #define lll_cond_timedwait(cond, abstime) \
386   __lll_cond_timedwait (cond, abstime)
387 #define lll_cond_wake(cond) \
388   __lll_cond_wake (cond)
389 #define lll_cond_broadcast(cond) \
390   __lll_cond_broadcast (cond)
391
392
393 #endif  /* lowlevellock.h */