LCOV - code coverage report
Current view: top level - arch/x86/include/asm - rwsem.h (source / functions) Hit Total Coverage
Test: coverage.info Lines: 2 2 100.0 %
Date: 2017-01-25 Functions: 0 0 -

          Line data    Source code
       1             : /* rwsem.h: R/W semaphores implemented using XADD/CMPXCHG for i486+
       2             :  *
       3             :  * Written by David Howells (dhowells@redhat.com).
       4             :  *
       5             :  * Derived from asm-x86/semaphore.h
       6             :  *
       7             :  *
       8             :  * The MSW of the count is the negated number of active writers and waiting
       9             :  * lockers, and the LSW is the total number of active locks
      10             :  *
      11             :  * The lock count is initialized to 0 (no active and no waiting lockers).
      12             :  *
      13             :  * When a writer subtracts WRITE_BIAS, it'll get 0xffff0001 for the case of an
      14             :  * uncontended lock. This can be determined because XADD returns the old value.
      15             :  * Readers increment by 1 and see a positive value when uncontended, negative
      16             :  * if there are writers (and maybe) readers waiting (in which case it goes to
      17             :  * sleep).
      18             :  *
      19             :  * The value of WAITING_BIAS supports up to 32766 waiting processes. This can
      20             :  * be extended to 65534 by manually checking the whole MSW rather than relying
      21             :  * on the S flag.
      22             :  *
      23             :  * The value of ACTIVE_BIAS supports up to 65535 active processes.
      24             :  *
      25             :  * This should be totally fair - if anything is waiting, a process that wants a
      26             :  * lock will go to the back of the queue. When the currently active lock is
      27             :  * released, if there's a writer at the front of the queue, then that and only
      28             :  * that will be woken up; if there's a bunch of consequtive readers at the
      29             :  * front, then they'll all be woken up, but no other readers will be.
      30             :  */
      31             : 
      32             : #ifndef _ASM_X86_RWSEM_H
      33             : #define _ASM_X86_RWSEM_H
      34             : 
      35             : #ifndef _LINUX_RWSEM_H
      36             : #error "please don't include asm/rwsem.h directly, use linux/rwsem.h instead"
      37             : #endif
      38             : 
      39             : #ifdef __KERNEL__
      40             : 
      41             : #include <linux/list.h>
      42             : #include <linux/spinlock.h>
      43             : #include <linux/lockdep.h>
      44             : #include <asm/asm.h>
      45             : 
      46             : struct rwsem_waiter;
      47             : 
      48             : extern asmregparm struct rw_semaphore *
      49             :  rwsem_down_read_failed(struct rw_semaphore *sem);
      50             : extern asmregparm struct rw_semaphore *
      51             :  rwsem_down_write_failed(struct rw_semaphore *sem);
      52             : extern asmregparm struct rw_semaphore *
      53             :  rwsem_wake(struct rw_semaphore *);
      54             : extern asmregparm struct rw_semaphore *
      55             :  rwsem_downgrade_wake(struct rw_semaphore *sem);
      56             : 
      57             : /*
      58             :  * the semaphore definition
      59             :  *
      60             :  * The bias values and the counter type limits the number of
      61             :  * potential readers/writers to 32767 for 32 bits and 2147483647
      62             :  * for 64 bits.
      63             :  */
      64             : 
      65             : #ifdef CONFIG_X86_64
      66             : # define RWSEM_ACTIVE_MASK              0xffffffffL
      67             : #else
      68             : # define RWSEM_ACTIVE_MASK              0x0000ffffL
      69             : #endif
      70             : 
      71             : #define RWSEM_UNLOCKED_VALUE            0x00000000L
      72             : #define RWSEM_ACTIVE_BIAS               0x00000001L
      73             : #define RWSEM_WAITING_BIAS              (-RWSEM_ACTIVE_MASK-1)
      74             : #define RWSEM_ACTIVE_READ_BIAS          RWSEM_ACTIVE_BIAS
      75             : #define RWSEM_ACTIVE_WRITE_BIAS         (RWSEM_WAITING_BIAS + RWSEM_ACTIVE_BIAS)
      76             : 
      77           1 : typedef signed long rwsem_count_t;
      78           1 : 
      79             : struct rw_semaphore {
      80             :         rwsem_count_t           count;
      81             :         spinlock_t              wait_lock;
      82             :         struct list_head        wait_list;
      83             : #ifdef CONFIG_DEBUG_LOCK_ALLOC
      84             :         struct lockdep_map dep_map;
      85             : #endif
      86             : };
      87             : 
      88             : #ifdef CONFIG_DEBUG_LOCK_ALLOC
      89             : # define __RWSEM_DEP_MAP_INIT(lockname) , .dep_map = { .name = #lockname }
      90             : #else
      91             : # define __RWSEM_DEP_MAP_INIT(lockname)
      92             : #endif
      93             : 
      94             : 
      95             : #define __RWSEM_INITIALIZER(name)                               \
      96             : {                                                               \
      97             :         RWSEM_UNLOCKED_VALUE, __SPIN_LOCK_UNLOCKED((name).wait_lock), \
      98             :         LIST_HEAD_INIT((name).wait_list) __RWSEM_DEP_MAP_INIT(name) \
      99             : }
     100             : 
     101             : #define DECLARE_RWSEM(name)                                     \
     102             :         struct rw_semaphore name = __RWSEM_INITIALIZER(name)
     103             : 
     104             : extern void __init_rwsem(struct rw_semaphore *sem, const char *name,
     105             :                          struct lock_class_key *key);
     106             : 
     107             : #define init_rwsem(sem)                                         \
     108             : do {                                                            \
     109             :         static struct lock_class_key __key;                     \
     110             :                                                                 \
     111             :         __init_rwsem((sem), #sem, &__key);                  \
     112             : } while (0)
     113             : 
     114             : /*
     115             :  * lock for reading
     116             :  */
     117             : static inline void __down_read(struct rw_semaphore *sem)
     118             : {
     119             :         asm volatile("# beginning down_read\n\t"
     120             :                      LOCK_PREFIX _ASM_INC "(%1)\n\t"
     121             :                      /* adds 0x00000001, returns the old value */
     122             :                      "  jns        1f\n"
     123             :                      "  call call_rwsem_down_read_failed\n"
     124             :                      "1:\n\t"
     125             :                      "# ending down_read\n\t"
     126             :                      : "+m" (sem->count)
     127             :                      : "a" (sem)
     128             :                      : "memory", "cc");
     129             : }
     130             : 
     131             : /*
     132             :  * trylock for reading -- returns 1 if successful, 0 if contention
     133             :  */
     134             : static inline int __down_read_trylock(struct rw_semaphore *sem)
     135             : {
     136             :         rwsem_count_t result, tmp;
     137             :         asm volatile("# beginning __down_read_trylock\n\t"
     138             :                      "  mov          %0,%1\n\t"
     139             :                      "1:\n\t"
     140             :                      "  mov          %1,%2\n\t"
     141             :                      "  add          %3,%2\n\t"
     142             :                      "  jle             2f\n\t"
     143             :                      LOCK_PREFIX "  cmpxchg  %2,%0\n\t"
     144             :                      "  jnz             1b\n\t"
     145             :                      "2:\n\t"
     146             :                      "# ending __down_read_trylock\n\t"
     147             :                      : "+m" (sem->count), "=&a" (result), "=&r" (tmp)
     148             :                      : "i" (RWSEM_ACTIVE_READ_BIAS)
     149             :                      : "memory", "cc");
     150             :         return result >= 0 ? 1 : 0;
     151             : }
     152             : 
     153             : /*
     154             :  * lock for writing
     155             :  */
     156             : static inline void __down_write_nested(struct rw_semaphore *sem, int subclass)
     157             : {
     158             :         rwsem_count_t tmp;
     159             : 
     160             :         tmp = RWSEM_ACTIVE_WRITE_BIAS;
     161             :         asm volatile("# beginning down_write\n\t"
     162             :                      LOCK_PREFIX "  xadd      %1,(%2)\n\t"
     163             :                      /* subtract 0x0000ffff, returns the old value */
     164             :                      "  test      %1,%1\n\t"
     165             :                      /* was the count 0 before? */
     166             :                      "  jz        1f\n"
     167             :                      "  call call_rwsem_down_write_failed\n"
     168             :                      "1:\n"
     169             :                      "# ending down_write"
     170             :                      : "+m" (sem->count), "=d" (tmp)
     171             :                      : "a" (sem), "1" (tmp)
     172             :                      : "memory", "cc");
     173             : }
     174             : 
     175             : static inline void __down_write(struct rw_semaphore *sem)
     176             : {
     177             :         __down_write_nested(sem, 0);
     178             : }
     179             : 
     180             : /*
     181             :  * trylock for writing -- returns 1 if successful, 0 if contention
     182             :  */
     183             : static inline int __down_write_trylock(struct rw_semaphore *sem)
     184             : {
     185             :         rwsem_count_t ret = cmpxchg(&sem->count,
     186             :                                     RWSEM_UNLOCKED_VALUE,
     187             :                                     RWSEM_ACTIVE_WRITE_BIAS);
     188             :         if (ret == RWSEM_UNLOCKED_VALUE)
     189             :                 return 1;
     190             :         return 0;
     191             : }
     192             : 
     193             : /*
     194             :  * unlock after reading
     195             :  */
     196             : static inline void __up_read(struct rw_semaphore *sem)
     197             : {
     198             :         rwsem_count_t tmp = -RWSEM_ACTIVE_READ_BIAS;
     199             :         asm volatile("# beginning __up_read\n\t"
     200             :                      LOCK_PREFIX "  xadd      %1,(%2)\n\t"
     201             :                      /* subtracts 1, returns the old value */
     202             :                      "  jns        1f\n\t"
     203             :                      "  call call_rwsem_wake\n"
     204             :                      "1:\n"
     205             :                      "# ending __up_read\n"
     206             :                      : "+m" (sem->count), "=d" (tmp)
     207             :                      : "a" (sem), "1" (tmp)
     208             :                      : "memory", "cc");
     209             : }
     210             : 
     211             : /*
     212             :  * unlock after writing
     213             :  */
     214             : static inline void __up_write(struct rw_semaphore *sem)
     215             : {
     216             :         rwsem_count_t tmp;
     217             :         asm volatile("# beginning __up_write\n\t"
     218             :                      LOCK_PREFIX "  xadd      %1,(%2)\n\t"
     219             :                      /* tries to transition
     220             :                         0xffff0001 -> 0x00000000 */
     221             :                      "  jz       1f\n"
     222             :                      "  call call_rwsem_wake\n"
     223             :                      "1:\n\t"
     224             :                      "# ending __up_write\n"
     225             :                      : "+m" (sem->count), "=d" (tmp)
     226             :                      : "a" (sem), "1" (-RWSEM_ACTIVE_WRITE_BIAS)
     227             :                      : "memory", "cc");
     228             : }
     229             : 
     230             : /*
     231             :  * downgrade write lock to read lock
     232             :  */
     233             : static inline void __downgrade_write(struct rw_semaphore *sem)
     234             : {
     235             :         asm volatile("# beginning __downgrade_write\n\t"
     236             :                      LOCK_PREFIX _ASM_ADD "%2,(%1)\n\t"
     237             :                      /*
     238             :                       * transitions 0xZZZZ0001 -> 0xYYYY0001 (i386)
     239             :                       *     0xZZZZZZZZ00000001 -> 0xYYYYYYYY00000001 (x86_64)
     240             :                       */
     241             :                      "  jns       1f\n\t"
     242             :                      "  call call_rwsem_downgrade_wake\n"
     243             :                      "1:\n\t"
     244             :                      "# ending __downgrade_write\n"
     245             :                      : "+m" (sem->count)
     246             :                      : "a" (sem), "er" (-RWSEM_WAITING_BIAS)
     247             :                      : "memory", "cc");
     248             : }
     249             : 
     250             : /*
     251             :  * implement atomic add functionality
     252             :  */
     253             : static inline void rwsem_atomic_add(rwsem_count_t delta,
     254             :                                     struct rw_semaphore *sem)
     255             : {
     256             :         asm volatile(LOCK_PREFIX _ASM_ADD "%1,%0"
     257             :                      : "+m" (sem->count)
     258             :                      : "er" (delta));
     259             : }
     260             : 
     261             : /*
     262             :  * implement exchange and add functionality
     263             :  */
     264             : static inline rwsem_count_t rwsem_atomic_update(rwsem_count_t delta,
     265             :                                                 struct rw_semaphore *sem)
     266             : {
     267             :         rwsem_count_t tmp = delta;
     268             : 
     269             :         asm volatile(LOCK_PREFIX "xadd %0,%1"
     270             :                      : "+r" (tmp), "+m" (sem->count)
     271             :                      : : "memory");
     272             : 
     273             :         return tmp + delta;
     274             : }
     275             : 
     276             : static inline int rwsem_is_locked(struct rw_semaphore *sem)
     277             : {
     278             :         return (sem->count != 0);
     279             : }
     280             : 
     281             : #endif /* __KERNEL__ */
     282             : #endif /* _ASM_X86_RWSEM_H */

Generated by: LCOV version 1.10