fd1ee4569d5de3ea5d59bd2cedb94347229357b2
[kopensolaris-gnu/glibc.git] / nptl / sysdeps / unix / sysv / linux / i386 / lowlevellock.h
1 /* Copyright (C) 2002, 2003, 2004, 2006 Free Software Foundation, Inc.
2    This file is part of the GNU C Library.
3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
4
5    The GNU C Library is free software; you can redistribute it and/or
6    modify it under the terms of the GNU Lesser General Public
7    License as published by the Free Software Foundation; either
8    version 2.1 of the License, or (at your option) any later version.
9
10    The GNU C Library is distributed in the hope that it will be useful,
11    but WITHOUT ANY WARRANTY; without even the implied warranty of
12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13    Lesser General Public License for more details.
14
15    You should have received a copy of the GNU Lesser General Public
16    License along with the GNU C Library; if not, write to the Free
17    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
18    02111-1307 USA.  */
19
20 #ifndef _LOWLEVELLOCK_H
21 #define _LOWLEVELLOCK_H 1
22
23 #include <time.h>
24 #include <sys/param.h>
25 #include <bits/pthreadtypes.h>
26
27 #ifndef LOCK_INSTR
28 # ifdef UP
29 #  define LOCK_INSTR    /* nothing */
30 # else
31 #  define LOCK_INSTR "lock;"
32 # endif
33 #endif
34
35 #define SYS_futex               240
36 #define FUTEX_WAIT              0
37 #define FUTEX_WAKE              1
38
39
40 /* Initializer for compatibility lock.  */
41 #define LLL_MUTEX_LOCK_INITIALIZER              (0)
42 #define LLL_MUTEX_LOCK_INITIALIZER_LOCKED       (1)
43 #define LLL_MUTEX_LOCK_INITIALIZER_WAITERS      (2)
44
45
46 #ifdef PIC
47 # define LLL_EBX_LOAD   "xchgl %2, %%ebx\n"
48 # define LLL_EBX_REG    "D"
49 #else
50 # define LLL_EBX_LOAD
51 # define LLL_EBX_REG    "b"
52 #endif
53
54 #ifdef I386_USE_SYSENTER
55 # ifdef SHARED
56 #  define LLL_ENTER_KERNEL      "call *%%gs:%P6\n\t"
57 # else
58 #  define LLL_ENTER_KERNEL      "call *_dl_sysinfo\n\t"
59 # endif
60 #else
61 # define LLL_ENTER_KERNEL       "int $0x80\n\t"
62 #endif
63
64 /* Delay in spinlock loop.  */
65 #define BUSY_WAIT_NOP          asm ("rep; nop")
66
67
68 #define lll_futex_wait(futex, val) \
69   ({                                                                          \
70     int __status;                                                             \
71     register __typeof (val) _val asm ("edx") = (val);                         \
72     __asm __volatile (LLL_EBX_LOAD                                            \
73                       LLL_ENTER_KERNEL                                        \
74                       LLL_EBX_LOAD                                            \
75                       : "=a" (__status)                                       \
76                       : "0" (SYS_futex), LLL_EBX_REG (futex), "S" (0),        \
77                         "c" (FUTEX_WAIT), "d" (_val),                         \
78                         "i" (offsetof (tcbhead_t, sysinfo)));                 \
79     __status;                                                                 \
80   })
81
82
83 #define lll_futex_timed_wait(futex, val, timeout)                             \
84   ({                                                                          \
85     int __status;                                                             \
86     register __typeof (val) _val asm ("edx") = (val);                         \
87     __asm __volatile (LLL_EBX_LOAD                                            \
88                       LLL_ENTER_KERNEL                                        \
89                       LLL_EBX_LOAD                                            \
90                       : "=a" (__status)                                       \
91                       : "0" (SYS_futex), LLL_EBX_REG (futex), "S" (timeout),  \
92                         "c" (FUTEX_WAIT), "d" (_val),                         \
93                         "i" (offsetof (tcbhead_t, sysinfo)));                 \
94     __status;                                                                 \
95   })
96
97
98 #define lll_futex_wake(futex, nr) \
99   do {                                                                        \
100     int __ignore;                                                             \
101     register __typeof (nr) _nr asm ("edx") = (nr);                            \
102     __asm __volatile (LLL_EBX_LOAD                                            \
103                       LLL_ENTER_KERNEL                                        \
104                       LLL_EBX_LOAD                                            \
105                       : "=a" (__ignore)                                       \
106                       : "0" (SYS_futex), LLL_EBX_REG (futex),                 \
107                         "c" (FUTEX_WAKE), "d" (_nr),                          \
108                         "i" (0) /* phony, to align next arg's number */,      \
109                         "i" (offsetof (tcbhead_t, sysinfo)));                 \
110   } while (0)
111
112
113 /* Does not preserve %eax and %ecx.  */
114 extern int __lll_mutex_lock_wait (int val, int *__futex)
115      __attribute ((regparm (2))) attribute_hidden;
116 /* Does not preserve %eax, %ecx, and %edx.  */
117 extern int __lll_mutex_timedlock_wait (int val, int *__futex,
118                                        const struct timespec *abstime)
119      __attribute ((regparm (3))) attribute_hidden;
120 /* Preserves all registers but %eax.  */
121 extern int __lll_mutex_unlock_wake (int *__futex)
122      __attribute ((regparm (1))) attribute_hidden;
123
124
125 /* NB: in the lll_mutex_trylock macro we simply return the value in %eax
126    after the cmpxchg instruction.  In case the operation succeded this
127    value is zero.  In case the operation failed, the cmpxchg instruction
128    has loaded the current value of the memory work which is guaranteed
129    to be nonzero.  */
130 #define lll_mutex_trylock(futex) \
131   ({ int ret;                                                                 \
132      __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1"                           \
133                        : "=a" (ret), "=m" (futex)                             \
134                        : "r" (LLL_MUTEX_LOCK_INITIALIZER_LOCKED), "m" (futex),\
135                          "0" (LLL_MUTEX_LOCK_INITIALIZER)                     \
136                        : "memory");                                           \
137      ret; })
138
139
140 #define lll_mutex_cond_trylock(futex) \
141   ({ int ret;                                                                 \
142      __asm __volatile (LOCK_INSTR "cmpxchgl %2, %1"                           \
143                        : "=a" (ret), "=m" (futex)                             \
144                        : "r" (LLL_MUTEX_LOCK_INITIALIZER_WAITERS),            \
145                           "m" (futex), "0" (LLL_MUTEX_LOCK_INITIALIZER)       \
146                        : "memory");                                           \
147      ret; })
148
149
150 #define lll_mutex_lock(futex) \
151   (void) ({ int ignore1, ignore2;                                             \
152             __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t"                \
153                               "jnz _L_mutex_lock_%=\n\t"                      \
154                               ".subsection 1\n\t"                             \
155                               ".type _L_mutex_lock_%=,@function\n"            \
156                               "_L_mutex_lock_%=:\n\t"                         \
157                               "leal %2, %%ecx\n\t"                            \
158                               "call __lll_mutex_lock_wait\n\t"                \
159                               "jmp 1f\n\t"                                    \
160                               ".size _L_mutex_lock_%=,.-_L_mutex_lock_%=\n"   \
161                               ".previous\n"                                   \
162                               "1:"                                            \
163                               : "=a" (ignore1), "=c" (ignore2), "=m" (futex)  \
164                               : "0" (0), "1" (1), "m" (futex)                 \
165                               : "memory"); })
166
167
168 /* Special version of lll_mutex_lock which causes the unlock function to
169    always wakeup waiters.  */
170 #define lll_mutex_cond_lock(futex) \
171   (void) ({ int ignore1, ignore2;                                             \
172             __asm __volatile (LOCK_INSTR "cmpxchgl %1, %2\n\t"                \
173                               "jnz _L_mutex_cond_lock_%=\n\t"                 \
174                               ".subsection 1\n\t"                             \
175                               ".type _L_mutex_cond_lock_%=,@function\n"       \
176                               "_L_mutex_cond_lock_%=:\n\t"                    \
177                               "leal %2, %%ecx\n\t"                            \
178                               "call __lll_mutex_lock_wait\n\t"                \
179                               "jmp 1f\n\t"                                    \
180                               ".size _L_mutex_cond_lock_%=,.-_L_mutex_cond_lock_%=\n"   \
181                               ".previous\n"                                   \
182                               "1:"                                            \
183                               : "=a" (ignore1), "=c" (ignore2), "=m" (futex)  \
184                               : "0" (0), "1" (2), "m" (futex)                 \
185                               : "memory"); })
186
187
188 #define lll_mutex_timedlock(futex, timeout) \
189   ({ int result, ignore1, ignore2;                                            \
190      __asm __volatile (LOCK_INSTR "cmpxchgl %1, %3\n\t"                       \
191                        "jnz _L_mutex_timedlock_%=\n\t"                        \
192                        ".subsection 1\n\t"                                    \
193                        ".type _L_mutex_timedlock_%=,@function\n"              \
194                        "_L_mutex_timedlock_%=:\n\t"                           \
195                        "leal %3, %%ecx\n\t"                                   \
196                        "movl %7, %%edx\n\t"                                   \
197                        "call __lll_mutex_timedlock_wait\n\t"                  \
198                        "jmp 1f\n\t"                                           \
199                        ".size _L_mutex_timedlock_%=,.-_L_mutex_timedlock_%=\n"\
200                        ".previous\n"                                          \
201                        "1:"                                                   \
202                        : "=a" (result), "=c" (ignore1), "=&d" (ignore2),      \
203                          "=m" (futex)                                         \
204                        : "0" (0), "1" (1), "m" (futex), "m" (timeout)         \
205                        : "memory");                                           \
206      result; })
207
208
209 #define lll_mutex_unlock(futex) \
210   (void) ({ int ignore;                                                       \
211             __asm __volatile (LOCK_INSTR "subl $1,%0\n\t"                     \
212                               "jne _L_mutex_unlock_%=\n\t"                    \
213                               ".subsection 1\n\t"                             \
214                               ".type _L_mutex_unlock_%=,@function\n"          \
215                               "_L_mutex_unlock_%=:\n\t"                       \
216                               "leal %0, %%eax\n\t"                            \
217                               "call __lll_mutex_unlock_wake\n\t"              \
218                               "jmp 1f\n\t"                                    \
219                               ".size _L_mutex_unlock_%=,.-_L_mutex_unlock_%=\n" \
220                               ".previous\n"                                   \
221                               "1:"                                            \
222                               : "=m" (futex), "=&a" (ignore)                  \
223                               : "m" (futex)                                   \
224                               : "memory"); })
225
226
227 #define lll_mutex_islocked(futex) \
228   (futex != 0)
229
230
231 /* We have a separate internal lock implementation which is not tied
232    to binary compatibility.  */
233
234 /* Type for lock object.  */
235 typedef int lll_lock_t;
236
237 /* Initializers for lock.  */
238 #define LLL_LOCK_INITIALIZER            (0)
239 #define LLL_LOCK_INITIALIZER_LOCKED     (1)
240
241
242 extern int __lll_lock_wait (int val, int *__futex)
243      __attribute ((regparm (2))) attribute_hidden;
244 extern int __lll_unlock_wake (int *__futex)
245      __attribute ((regparm (1))) attribute_hidden;
246 extern int lll_unlock_wake_cb (int *__futex) attribute_hidden;
247
248
249 /* The states of a lock are:
250     0  -  untaken
251     1  -  taken by one user
252     2  -  taken by more users */
253
254
255 #if defined NOT_IN_libc || defined UP
256 # define lll_trylock(futex) lll_mutex_trylock (futex)
257 # define lll_lock(futex) lll_mutex_lock (futex)
258 # define lll_unlock(futex) lll_mutex_unlock (futex)
259 #else
260 /* Special versions of the macros for use in libc itself.  They avoid
261    the lock prefix when the thread library is not used.
262
263    XXX In future we might even want to avoid it on UP machines.  */
264 # include <tls.h>
265
266 # define lll_trylock(futex) \
267   ({ unsigned char ret;                                                       \
268      __asm __volatile ("cmpl $0, %%gs:%P5\n\t"                                \
269                        "je,pt 0f\n\t"                                         \
270                        "lock\n"                                               \
271                        "0:\tcmpxchgl %2, %1; setne %0"                        \
272                        : "=a" (ret), "=m" (futex)                             \
273                        : "r" (LLL_MUTEX_LOCK_INITIALIZER_LOCKED), "m" (futex),\
274                          "0" (LLL_MUTEX_LOCK_INITIALIZER),                    \
275                          "i" (offsetof (tcbhead_t, multiple_threads))         \
276                        : "memory");                                           \
277      ret; })
278
279
280 # define lll_lock(futex) \
281   (void) ({ int ignore1, ignore2;                                             \
282             __asm __volatile ("cmpl $0, %%gs:%P6\n\t"                         \
283                               "je,pt 0f\n\t"                                  \
284                               "lock\n"                                        \
285                               "0:\tcmpxchgl %1, %2\n\t"                       \
286                               "jnz _L_mutex_lock_%=\n\t"                      \
287                               ".subsection 1\n\t"                             \
288                               ".type _L_mutex_lock_%=,@function\n"            \
289                               "_L_mutex_lock_%=:\n\t"                         \
290                               "leal %2, %%ecx\n\t"                            \
291                               "call __lll_mutex_lock_wait\n\t"                \
292                               "jmp 1f\n\t"                                    \
293                               ".size _L_mutex_lock_%=,.-_L_mutex_lock_%=\n"   \
294                               ".previous\n"                                   \
295                               "1:"                                            \
296                               : "=a" (ignore1), "=c" (ignore2), "=m" (futex)  \
297                               : "0" (0), "1" (1), "m" (futex),                \
298                                 "i" (offsetof (tcbhead_t, multiple_threads))  \
299                               : "memory"); })
300
301
302 # define lll_unlock(futex) \
303   (void) ({ int ignore;                                                       \
304             __asm __volatile ("cmpl $0, %%gs:%P3\n\t"                         \
305                               "je,pt 0f\n\t"                                  \
306                               "lock\n"                                        \
307                               "0:\tsubl $1,%0\n\t"                    \
308                               "jne _L_mutex_unlock_%=\n\t"                    \
309                               ".subsection 1\n\t"                             \
310                               ".type _L_mutex_unlock_%=,@function\n"          \
311                               "_L_mutex_unlock_%=:\n\t"                       \
312                               "leal %0, %%eax\n\t"                            \
313                               "call __lll_mutex_unlock_wake\n\t"              \
314                               "jmp 1f\n\t"                                    \
315                               ".size _L_mutex_unlock_%=,.-_L_mutex_unlock_%=\n" \
316                               ".previous\n"                                   \
317                               "1:"                                            \
318                               : "=m" (futex), "=&a" (ignore)                  \
319                               : "m" (futex),                                  \
320                                 "i" (offsetof (tcbhead_t, multiple_threads))  \
321                               : "memory"); })
322 #endif
323
324
325 #define lll_islocked(futex) \
326   (futex != LLL_LOCK_INITIALIZER)
327
328
329 /* The kernel notifies a process with uses CLONE_CLEARTID via futex
330    wakeup when the clone terminates.  The memory location contains the
331    thread ID while the clone is running and is reset to zero
332    afterwards.
333
334    The macro parameter must not have any side effect.  */
335 #define lll_wait_tid(tid) \
336   do {                                                                        \
337     int __ignore;                                                             \
338     register __typeof (tid) _tid asm ("edx") = (tid);                         \
339     if (_tid != 0)                                                            \
340       __asm __volatile (LLL_EBX_LOAD                                          \
341                         "1:\tmovl %1, %%eax\n\t"                              \
342                         LLL_ENTER_KERNEL                                      \
343                         "cmpl $0, (%%ebx)\n\t"                                \
344                         "jne,pn 1b\n\t"                                       \
345                         LLL_EBX_LOAD                                          \
346                         : "=&a" (__ignore)                                    \
347                         : "i" (SYS_futex), LLL_EBX_REG (&tid), "S" (0),       \
348                           "c" (FUTEX_WAIT), "d" (_tid),                       \
349                           "i" (offsetof (tcbhead_t, sysinfo)));               \
350   } while (0)
351
352 extern int __lll_timedwait_tid (int *tid, const struct timespec *abstime)
353      __attribute__ ((regparm (2))) attribute_hidden;
354 #define lll_timedwait_tid(tid, abstime) \
355   ({                                                                          \
356     int __result = 0;                                                         \
357     if (tid != 0)                                                             \
358       {                                                                       \
359         if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)           \
360           __result = EINVAL;                                                  \
361         else                                                                  \
362           __result = __lll_timedwait_tid (&tid, abstime);                     \
363       }                                                                       \
364     __result; })
365
366
367 /* Conditional variable handling.  */
368
369 extern void __lll_cond_wait (pthread_cond_t *cond)
370      __attribute ((regparm (1))) attribute_hidden;
371 extern int __lll_cond_timedwait (pthread_cond_t *cond,
372                                  const struct timespec *abstime)
373      __attribute ((regparm (2))) attribute_hidden;
374 extern void __lll_cond_wake (pthread_cond_t *cond)
375      __attribute ((regparm (1))) attribute_hidden;
376 extern void __lll_cond_broadcast (pthread_cond_t *cond)
377      __attribute ((regparm (1))) attribute_hidden;
378
379
380 #define lll_cond_wait(cond) \
381   __lll_cond_wait (cond)
382 #define lll_cond_timedwait(cond, abstime) \
383   __lll_cond_timedwait (cond, abstime)
384 #define lll_cond_wake(cond) \
385   __lll_cond_wake (cond)
386 #define lll_cond_broadcast(cond) \
387   __lll_cond_broadcast (cond)
388
389
390 #endif  /* lowlevellock.h */