Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
openSUSE:12.1
xen
23099-x86-rwlock-scalability.patch
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File 23099-x86-rwlock-scalability.patch of Package xen
# HG changeset patch # User Keir Fraser <keir@xen.org> # Date 1301126601 0 # Node ID 612171ff82ea51aaf65d98fd1a551eb8d50fb481 # Parent c9f745c153ec8c3775e2ee03adc3cb30370b84f6 rwlock: Allow to scale to 2^31-1 readers on x86. Also rework to match the 'trylock' style of raw function used for spinlocks. Inspired by Jan Beulich's patch to do similar improved scaling. Signed-off-by: Keir Fraser <keir@xen.org> # HG changeset patch # User Keir Fraser <keir@xen.org> # Date 1301214635 -3600 # Node ID 0bc1c4746c8939337f693a513fd837fc03477db1 # Parent 48dac730a93b27ff60a340564e9a7afd7f9385f4 x86_32: Fix _raw_read_trylock() build on some gcc versions. Was broken by 23099:612171ff82ea. A bool_t is a single byte, and needs a 'q' register constraint. Avoid the whole issue by changing the variable to an int, and explicitly specify the operand suffix as 'l' for good measure. Signed-off-by: Keir Fraser <keir@xen.org> Index: xen-4.1.2-testing/xen/common/spinlock.c =================================================================== --- xen-4.1.2-testing.orig/xen/common/spinlock.c +++ xen-4.1.2-testing/xen/common/spinlock.c @@ -234,7 +234,11 @@ void _spin_unlock_recursive(spinlock_t * void _read_lock(rwlock_t *lock) { check_lock(&lock->debug); - _raw_read_lock(&lock->raw); + while ( unlikely(!_raw_read_trylock(&lock->raw)) ) + { + while ( likely(_raw_rw_is_write_locked(&lock->raw)) ) + cpu_relax(); + } preempt_disable(); } @@ -243,7 +247,13 @@ void _read_lock_irq(rwlock_t *lock) ASSERT(local_irq_is_enabled()); local_irq_disable(); check_lock(&lock->debug); - _raw_read_lock(&lock->raw); + while ( unlikely(!_raw_read_trylock(&lock->raw)) ) + { + local_irq_enable(); + while ( likely(_raw_rw_is_write_locked(&lock->raw)) ) + cpu_relax(); + local_irq_disable(); + } preempt_disable(); } @@ -252,11 +262,26 @@ unsigned long _read_lock_irqsave(rwlock_ unsigned long flags; local_irq_save(flags); check_lock(&lock->debug); - _raw_read_lock(&lock->raw); + while ( unlikely(!_raw_read_trylock(&lock->raw)) ) + { + local_irq_restore(flags); + while ( likely(_raw_rw_is_write_locked(&lock->raw)) ) + cpu_relax(); + local_irq_save(flags); + } preempt_disable(); return flags; } +int _read_trylock(rwlock_t *lock) +{ + check_lock(&lock->debug); + if ( !_raw_read_trylock(&lock->raw) ) + return 0; + preempt_disable(); + return 1; +} + void _read_unlock(rwlock_t *lock) { preempt_enable(); @@ -280,7 +305,11 @@ void _read_unlock_irqrestore(rwlock_t *l void _write_lock(rwlock_t *lock) { check_lock(&lock->debug); - _raw_write_lock(&lock->raw); + while ( unlikely(!_raw_write_trylock(&lock->raw)) ) + { + while ( likely(_raw_rw_is_locked(&lock->raw)) ) + cpu_relax(); + } preempt_disable(); } @@ -289,7 +318,13 @@ void _write_lock_irq(rwlock_t *lock) ASSERT(local_irq_is_enabled()); local_irq_disable(); check_lock(&lock->debug); - _raw_write_lock(&lock->raw); + while ( unlikely(!_raw_write_trylock(&lock->raw)) ) + { + local_irq_enable(); + while ( likely(_raw_rw_is_locked(&lock->raw)) ) + cpu_relax(); + local_irq_disable(); + } preempt_disable(); } @@ -298,7 +333,13 @@ unsigned long _write_lock_irqsave(rwlock unsigned long flags; local_irq_save(flags); check_lock(&lock->debug); - _raw_write_lock(&lock->raw); + while ( unlikely(!_raw_write_trylock(&lock->raw)) ) + { + local_irq_restore(flags); + while ( likely(_raw_rw_is_locked(&lock->raw)) ) + cpu_relax(); + local_irq_save(flags); + } preempt_disable(); return flags; } Index: xen-4.1.2-testing/xen/include/asm-ia64/linux-xen/asm/spinlock.h =================================================================== --- xen-4.1.2-testing.orig/xen/include/asm-ia64/linux-xen/asm/spinlock.h +++ xen-4.1.2-testing/xen/include/asm-ia64/linux-xen/asm/spinlock.h @@ -35,17 +35,6 @@ typedef struct { } raw_rwlock_t; #define _RAW_RW_LOCK_UNLOCKED /*(raw_rwlock_t)*/ { 0, 0 } -#define _raw_read_lock(rw) \ -do { \ - raw_rwlock_t *__read_lock_ptr = (rw); \ - \ - while (unlikely(ia64_fetchadd(1, (int *) __read_lock_ptr, acq) < 0)) { \ - ia64_fetchadd(-1, (int *) __read_lock_ptr, rel); \ - while (*(volatile int *)__read_lock_ptr < 0) \ - cpu_relax(); \ - } \ -} while (0) - #define _raw_read_unlock(rw) \ do { \ raw_rwlock_t *__read_lock_ptr = (rw); \ @@ -53,20 +42,6 @@ do { \ } while (0) #ifdef ASM_SUPPORTED -#define _raw_write_lock(rw) \ -do { \ - __asm__ __volatile__ ( \ - "mov ar.ccv = r0\n" \ - "dep r29 = -1, r0, 31, 1;;\n" \ - "1:\n" \ - "ld4 r2 = [%0];;\n" \ - "cmp4.eq p0,p7 = r0,r2\n" \ - "(p7) br.cond.spnt.few 1b \n" \ - "cmpxchg4.acq r2 = [%0], r29, ar.ccv;;\n" \ - "cmp4.eq p0,p7 = r0, r2\n" \ - "(p7) br.cond.spnt.few 1b;;\n" \ - :: "r"(rw) : "ar.ccv", "p7", "r2", "r29", "memory"); \ -} while(0) #define _raw_write_trylock(rw) \ ({ \ @@ -82,16 +57,6 @@ do { \ #else /* !ASM_SUPPORTED */ -#define _raw_write_lock(l) \ -({ \ - __u64 ia64_val, ia64_set_val = ia64_dep_mi(-1, 0, 31, 1); \ - __u32 *ia64_write_lock_ptr = (__u32 *) (l); \ - do { \ - while (*ia64_write_lock_ptr) \ - ia64_barrier(); \ - ia64_val = ia64_cmpxchg4_acq(ia64_write_lock_ptr, ia64_set_val, 0); \ - } while (ia64_val); \ -}) #define _raw_write_trylock(rw) \ ({ \ Index: xen-4.1.2-testing/xen/include/asm-x86/spinlock.h =================================================================== --- xen-4.1.2-testing.orig/xen/include/asm-x86/spinlock.h +++ xen-4.1.2-testing/xen/include/asm-x86/spinlock.h @@ -35,51 +35,29 @@ typedef struct { volatile int lock; } raw_rwlock_t; -#define RW_LOCK_BIAS 0x01000000 -#define _RAW_RW_LOCK_UNLOCKED /*(raw_rwlock_t)*/ { RW_LOCK_BIAS } +#define RW_WRITE_BIAS 0x7fffffff +#define _RAW_RW_LOCK_UNLOCKED /*(raw_rwlock_t)*/ { 0 } -static always_inline void _raw_read_lock(raw_rwlock_t *rw) +static always_inline int _raw_read_trylock(raw_rwlock_t *rw) { - asm volatile ( - "1: lock; decl %0 \n" - " jns 3f \n" - " lock; incl %0 \n" - "2: rep; nop \n" - " cmpl $1,%0 \n" - " js 2b \n" - " jmp 1b \n" - "3:" - : "=m" (rw->lock) : : "memory" ); -} + int acquired; -static always_inline void _raw_write_lock(raw_rwlock_t *rw) -{ asm volatile ( - "1: lock; subl %1,%0 \n" - " jz 3f \n" - " lock; addl %1,%0 \n" - "2: rep; nop \n" - " cmpl %1,%0 \n" - " jne 2b \n" + " lock; decl %0 \n" + " jns 2f \n" + "1: .subsection 1 \n" + "2: lock; incl %0 \n" + " decl %1 \n" " jmp 1b \n" - "3:" - : "=m" (rw->lock) : "i" (RW_LOCK_BIAS) : "memory" ); + " .subsection 0 \n" + : "=m" (rw->lock), "=r" (acquired) : "1" (1) : "memory" ); + + return acquired; } static always_inline int _raw_write_trylock(raw_rwlock_t *rw) { - int rc; - - asm volatile ( - " lock; subl %2,%0 \n" - " jz 1f \n" - " lock; addl %2,%0 \n" - " dec %1 \n" - "1:" - : "=m" (rw->lock), "=r" (rc) : "i" (RW_LOCK_BIAS), "1" (1) - : "memory" ); - - return rc; + return (cmpxchg(&rw->lock, 0, RW_WRITE_BIAS) == 0); } static always_inline void _raw_read_unlock(raw_rwlock_t *rw) @@ -92,11 +70,11 @@ static always_inline void _raw_read_unlo static always_inline void _raw_write_unlock(raw_rwlock_t *rw) { asm volatile ( - "lock ; addl %1,%0" - : "=m" ((rw)->lock) : "i" (RW_LOCK_BIAS) : "memory" ); + "lock ; subl %1,%0" + : "=m" ((rw)->lock) : "i" (RW_WRITE_BIAS) : "memory" ); } -#define _raw_rw_is_locked(x) ((x)->lock < RW_LOCK_BIAS) -#define _raw_rw_is_write_locked(x) ((x)->lock <= 0) +#define _raw_rw_is_locked(x) ((x)->lock != 0) +#define _raw_rw_is_write_locked(x) ((x)->lock > 0) #endif /* __ASM_SPINLOCK_H */ Index: xen-4.1.2-testing/xen/include/xen/spinlock.h =================================================================== --- xen-4.1.2-testing.orig/xen/include/xen/spinlock.h +++ xen-4.1.2-testing/xen/include/xen/spinlock.h @@ -157,6 +157,7 @@ unsigned long _read_lock_irqsave(rwlock_ void _read_unlock(rwlock_t *lock); void _read_unlock_irq(rwlock_t *lock); void _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags); +int _read_trylock(rwlock_t *lock); void _write_lock(rwlock_t *lock); void _write_lock_irq(rwlock_t *lock); @@ -210,6 +211,7 @@ int _rw_is_write_locked(rwlock_t *lock); #define read_unlock(l) _read_unlock(l) #define read_unlock_irq(l) _read_unlock_irq(l) #define read_unlock_irqrestore(l, f) _read_unlock_irqrestore(l, f) +#define read_trylock(l) _read_trylock(l) #define write_lock(l) _write_lock(l) #define write_lock_irq(l) _write_lock_irq(l)
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
