File 23099-x86-rwlock-scalability.patch of Package xen
# HG changeset patch
# User Keir Fraser <keir@xen.org>
# Date 1301126601 0
# Node ID 612171ff82ea51aaf65d98fd1a551eb8d50fb481
# Parent c9f745c153ec8c3775e2ee03adc3cb30370b84f6
rwlock: Allow to scale to 2^31-1 readers on x86.
Also rework to match the 'trylock' style of raw function used for
spinlocks.
Inspired by Jan Beulich's patch to do similar improved scaling.
Signed-off-by: Keir Fraser <keir@xen.org>
# HG changeset patch
# User Keir Fraser <keir@xen.org>
# Date 1301214635 -3600
# Node ID 0bc1c4746c8939337f693a513fd837fc03477db1
# Parent 48dac730a93b27ff60a340564e9a7afd7f9385f4
x86_32: Fix _raw_read_trylock() build on some gcc versions.
Was broken by 23099:612171ff82ea.
A bool_t is a single byte, and needs a 'q' register constraint. Avoid
the whole issue by changing the variable to an int, and explicitly
specify the operand suffix as 'l' for good measure.
Signed-off-by: Keir Fraser <keir@xen.org>
Index: xen-4.1.2-testing/xen/common/spinlock.c
===================================================================
--- xen-4.1.2-testing.orig/xen/common/spinlock.c
+++ xen-4.1.2-testing/xen/common/spinlock.c
@@ -234,7 +234,11 @@ void _spin_unlock_recursive(spinlock_t *
void _read_lock(rwlock_t *lock)
{
check_lock(&lock->debug);
- _raw_read_lock(&lock->raw);
+ while ( unlikely(!_raw_read_trylock(&lock->raw)) )
+ {
+ while ( likely(_raw_rw_is_write_locked(&lock->raw)) )
+ cpu_relax();
+ }
preempt_disable();
}
@@ -243,7 +247,13 @@ void _read_lock_irq(rwlock_t *lock)
ASSERT(local_irq_is_enabled());
local_irq_disable();
check_lock(&lock->debug);
- _raw_read_lock(&lock->raw);
+ while ( unlikely(!_raw_read_trylock(&lock->raw)) )
+ {
+ local_irq_enable();
+ while ( likely(_raw_rw_is_write_locked(&lock->raw)) )
+ cpu_relax();
+ local_irq_disable();
+ }
preempt_disable();
}
@@ -252,11 +262,26 @@ unsigned long _read_lock_irqsave(rwlock_
unsigned long flags;
local_irq_save(flags);
check_lock(&lock->debug);
- _raw_read_lock(&lock->raw);
+ while ( unlikely(!_raw_read_trylock(&lock->raw)) )
+ {
+ local_irq_restore(flags);
+ while ( likely(_raw_rw_is_write_locked(&lock->raw)) )
+ cpu_relax();
+ local_irq_save(flags);
+ }
preempt_disable();
return flags;
}
+int _read_trylock(rwlock_t *lock)
+{
+ check_lock(&lock->debug);
+ if ( !_raw_read_trylock(&lock->raw) )
+ return 0;
+ preempt_disable();
+ return 1;
+}
+
void _read_unlock(rwlock_t *lock)
{
preempt_enable();
@@ -280,7 +305,11 @@ void _read_unlock_irqrestore(rwlock_t *l
void _write_lock(rwlock_t *lock)
{
check_lock(&lock->debug);
- _raw_write_lock(&lock->raw);
+ while ( unlikely(!_raw_write_trylock(&lock->raw)) )
+ {
+ while ( likely(_raw_rw_is_locked(&lock->raw)) )
+ cpu_relax();
+ }
preempt_disable();
}
@@ -289,7 +318,13 @@ void _write_lock_irq(rwlock_t *lock)
ASSERT(local_irq_is_enabled());
local_irq_disable();
check_lock(&lock->debug);
- _raw_write_lock(&lock->raw);
+ while ( unlikely(!_raw_write_trylock(&lock->raw)) )
+ {
+ local_irq_enable();
+ while ( likely(_raw_rw_is_locked(&lock->raw)) )
+ cpu_relax();
+ local_irq_disable();
+ }
preempt_disable();
}
@@ -298,7 +333,13 @@ unsigned long _write_lock_irqsave(rwlock
unsigned long flags;
local_irq_save(flags);
check_lock(&lock->debug);
- _raw_write_lock(&lock->raw);
+ while ( unlikely(!_raw_write_trylock(&lock->raw)) )
+ {
+ local_irq_restore(flags);
+ while ( likely(_raw_rw_is_locked(&lock->raw)) )
+ cpu_relax();
+ local_irq_save(flags);
+ }
preempt_disable();
return flags;
}
Index: xen-4.1.2-testing/xen/include/asm-ia64/linux-xen/asm/spinlock.h
===================================================================
--- xen-4.1.2-testing.orig/xen/include/asm-ia64/linux-xen/asm/spinlock.h
+++ xen-4.1.2-testing/xen/include/asm-ia64/linux-xen/asm/spinlock.h
@@ -35,17 +35,6 @@ typedef struct {
} raw_rwlock_t;
#define _RAW_RW_LOCK_UNLOCKED /*(raw_rwlock_t)*/ { 0, 0 }
-#define _raw_read_lock(rw) \
-do { \
- raw_rwlock_t *__read_lock_ptr = (rw); \
- \
- while (unlikely(ia64_fetchadd(1, (int *) __read_lock_ptr, acq) < 0)) { \
- ia64_fetchadd(-1, (int *) __read_lock_ptr, rel); \
- while (*(volatile int *)__read_lock_ptr < 0) \
- cpu_relax(); \
- } \
-} while (0)
-
#define _raw_read_unlock(rw) \
do { \
raw_rwlock_t *__read_lock_ptr = (rw); \
@@ -53,20 +42,6 @@ do { \
} while (0)
#ifdef ASM_SUPPORTED
-#define _raw_write_lock(rw) \
-do { \
- __asm__ __volatile__ ( \
- "mov ar.ccv = r0\n" \
- "dep r29 = -1, r0, 31, 1;;\n" \
- "1:\n" \
- "ld4 r2 = [%0];;\n" \
- "cmp4.eq p0,p7 = r0,r2\n" \
- "(p7) br.cond.spnt.few 1b \n" \
- "cmpxchg4.acq r2 = [%0], r29, ar.ccv;;\n" \
- "cmp4.eq p0,p7 = r0, r2\n" \
- "(p7) br.cond.spnt.few 1b;;\n" \
- :: "r"(rw) : "ar.ccv", "p7", "r2", "r29", "memory"); \
-} while(0)
#define _raw_write_trylock(rw) \
({ \
@@ -82,16 +57,6 @@ do { \
#else /* !ASM_SUPPORTED */
-#define _raw_write_lock(l) \
-({ \
- __u64 ia64_val, ia64_set_val = ia64_dep_mi(-1, 0, 31, 1); \
- __u32 *ia64_write_lock_ptr = (__u32 *) (l); \
- do { \
- while (*ia64_write_lock_ptr) \
- ia64_barrier(); \
- ia64_val = ia64_cmpxchg4_acq(ia64_write_lock_ptr, ia64_set_val, 0); \
- } while (ia64_val); \
-})
#define _raw_write_trylock(rw) \
({ \
Index: xen-4.1.2-testing/xen/include/asm-x86/spinlock.h
===================================================================
--- xen-4.1.2-testing.orig/xen/include/asm-x86/spinlock.h
+++ xen-4.1.2-testing/xen/include/asm-x86/spinlock.h
@@ -35,51 +35,29 @@ typedef struct {
volatile int lock;
} raw_rwlock_t;
-#define RW_LOCK_BIAS 0x01000000
-#define _RAW_RW_LOCK_UNLOCKED /*(raw_rwlock_t)*/ { RW_LOCK_BIAS }
+#define RW_WRITE_BIAS 0x7fffffff
+#define _RAW_RW_LOCK_UNLOCKED /*(raw_rwlock_t)*/ { 0 }
-static always_inline void _raw_read_lock(raw_rwlock_t *rw)
+static always_inline int _raw_read_trylock(raw_rwlock_t *rw)
{
- asm volatile (
- "1: lock; decl %0 \n"
- " jns 3f \n"
- " lock; incl %0 \n"
- "2: rep; nop \n"
- " cmpl $1,%0 \n"
- " js 2b \n"
- " jmp 1b \n"
- "3:"
- : "=m" (rw->lock) : : "memory" );
-}
+ int acquired;
-static always_inline void _raw_write_lock(raw_rwlock_t *rw)
-{
asm volatile (
- "1: lock; subl %1,%0 \n"
- " jz 3f \n"
- " lock; addl %1,%0 \n"
- "2: rep; nop \n"
- " cmpl %1,%0 \n"
- " jne 2b \n"
+ " lock; decl %0 \n"
+ " jns 2f \n"
+ "1: .subsection 1 \n"
+ "2: lock; incl %0 \n"
+ " decl %1 \n"
" jmp 1b \n"
- "3:"
- : "=m" (rw->lock) : "i" (RW_LOCK_BIAS) : "memory" );
+ " .subsection 0 \n"
+ : "=m" (rw->lock), "=r" (acquired) : "1" (1) : "memory" );
+
+ return acquired;
}
static always_inline int _raw_write_trylock(raw_rwlock_t *rw)
{
- int rc;
-
- asm volatile (
- " lock; subl %2,%0 \n"
- " jz 1f \n"
- " lock; addl %2,%0 \n"
- " dec %1 \n"
- "1:"
- : "=m" (rw->lock), "=r" (rc) : "i" (RW_LOCK_BIAS), "1" (1)
- : "memory" );
-
- return rc;
+ return (cmpxchg(&rw->lock, 0, RW_WRITE_BIAS) == 0);
}
static always_inline void _raw_read_unlock(raw_rwlock_t *rw)
@@ -92,11 +70,11 @@ static always_inline void _raw_read_unlo
static always_inline void _raw_write_unlock(raw_rwlock_t *rw)
{
asm volatile (
- "lock ; addl %1,%0"
- : "=m" ((rw)->lock) : "i" (RW_LOCK_BIAS) : "memory" );
+ "lock ; subl %1,%0"
+ : "=m" ((rw)->lock) : "i" (RW_WRITE_BIAS) : "memory" );
}
-#define _raw_rw_is_locked(x) ((x)->lock < RW_LOCK_BIAS)
-#define _raw_rw_is_write_locked(x) ((x)->lock <= 0)
+#define _raw_rw_is_locked(x) ((x)->lock != 0)
+#define _raw_rw_is_write_locked(x) ((x)->lock > 0)
#endif /* __ASM_SPINLOCK_H */
Index: xen-4.1.2-testing/xen/include/xen/spinlock.h
===================================================================
--- xen-4.1.2-testing.orig/xen/include/xen/spinlock.h
+++ xen-4.1.2-testing/xen/include/xen/spinlock.h
@@ -157,6 +157,7 @@ unsigned long _read_lock_irqsave(rwlock_
void _read_unlock(rwlock_t *lock);
void _read_unlock_irq(rwlock_t *lock);
void _read_unlock_irqrestore(rwlock_t *lock, unsigned long flags);
+int _read_trylock(rwlock_t *lock);
void _write_lock(rwlock_t *lock);
void _write_lock_irq(rwlock_t *lock);
@@ -210,6 +211,7 @@ int _rw_is_write_locked(rwlock_t *lock);
#define read_unlock(l) _read_unlock(l)
#define read_unlock_irq(l) _read_unlock_irq(l)
#define read_unlock_irqrestore(l, f) _read_unlock_irqrestore(l, f)
+#define read_trylock(l) _read_trylock(l)
#define write_lock(l) _write_lock(l)
#define write_lock_irq(l) _write_lock_irq(l)
