File 5faa974f-evtchn-rework-per-channel-lock.patch of Package xen
# Commit 5f2df45ead7c1195142f68b7923047a1e9479d54
# Date 2020-11-10 14:36:15 +0100
# Author Juergen Gross <jgross@suse.com>
# Committer Jan Beulich <jbeulich@suse.com>
xen/evtchn: rework per event channel lock
Currently the lock for a single event channel needs to be taken with
interrupts off, which causes deadlocks in some cases.
Rework the per event channel lock to be non-blocking for the case of
sending an event and removing the need for disabling interrupts for
taking the lock.
The lock is needed for avoiding races between event channel state
changes (creation, closing, binding) against normal operations (set
pending, [un]masking, priority changes).
Use a rwlock, but with some restrictions:
- Changing the state of an event channel (creation, closing, binding)
needs to use write_lock(), with ASSERT()ing that the lock is taken as
writer only when the state of the event channel is either before or
after the locked region appropriate (either free or unbound).
- Sending an event needs to use read_trylock() mostly, in case of not
obtaining the lock the operation is omitted. This is needed as
sending an event can happen with interrupts off (at least in some
cases).
- Dumping the event channel state for debug purposes is using
read_trylock(), too, in order to avoid blocking in case the lock is
taken as writer for a long time.
- All other cases can use read_lock().
Fixes: e045199c7c9c54 ("evtchn: address races with evtchn_reset()")
Signed-off-by: Juergen Gross <jgross@suse.com>
Reviewed-by: Jan Beulich <jbeulich@suse.com>
Acked-by: Julien Grall <jgrall@amazon.com>
# Commit 53bacb86f496fdb11560d9e3b361bca7de60d268
# Date 2020-11-11 08:56:21 +0100
# Author Juergen Gross <jgross@suse.com>
# Committer Jan Beulich <jbeulich@suse.com>
xen/events: fix build
Commit 5f2df45ead7c1195 ("xen/evtchn: rework per event channel lock")
introduced a build failure for NDEBUG builds.
Fixes: 5f2df45ead7c1195 ("xen/evtchn: rework per event channel lock")
Signed-off-by: Juergen Gross <jgross@suse.com>
Signed-off-by: Jan Beulich <jbeulich@suse.com>
--- a/xen/arch/x86/irq.c
+++ b/xen/arch/x86/irq.c
@@ -2316,14 +2316,12 @@ static void dump_irqs(unsigned char key)
pirq = domain_irq_to_pirq(d, irq);
info = pirq_info(d, pirq);
evtchn = evtchn_from_port(d, info->evtchn);
- local_irq_disable();
- if ( spin_trylock(&evtchn->lock) )
+ if ( evtchn_read_trylock(evtchn) )
{
pending = evtchn_is_pending(d, evtchn);
masked = evtchn_is_masked(d, evtchn);
- spin_unlock(&evtchn->lock);
+ evtchn_read_unlock(evtchn);
}
- local_irq_enable();
printk("%u:%3d(%c%c%c)",
d->domain_id, pirq, "-P?"[pending],
"-M?"[masked], info->masked ? 'M' : '-');
--- a/xen/common/event_channel.c
+++ b/xen/common/event_channel.c
@@ -51,6 +51,40 @@
#define consumer_is_xen(e) (!!(e)->xen_consumer)
/*
+ * Lock an event channel exclusively. This is allowed only when the channel is
+ * free or unbound either when taking or when releasing the lock, as any
+ * concurrent operation on the event channel using evtchn_read_trylock() will
+ * just assume the event channel is free or unbound at the moment when the
+ * evtchn_read_trylock() returns false.
+ */
+static inline void evtchn_write_lock(struct evtchn *evtchn)
+{
+ write_lock(&evtchn->lock);
+
+#ifndef NDEBUG
+ evtchn->old_state = evtchn->state;
+#endif
+}
+
+static inline unsigned int old_state(const struct evtchn *evtchn)
+{
+#ifndef NDEBUG
+ return evtchn->old_state;
+#else
+ return ECS_RESERVED; /* Just to allow things to build. */
+#endif
+}
+
+static inline void evtchn_write_unlock(struct evtchn *evtchn)
+{
+ /* Enforce lock discipline. */
+ ASSERT(old_state(evtchn) == ECS_FREE || old_state(evtchn) == ECS_UNBOUND ||
+ evtchn->state == ECS_FREE || evtchn->state == ECS_UNBOUND);
+
+ write_unlock(&evtchn->lock);
+}
+
+/*
* The function alloc_unbound_xen_event_channel() allows an arbitrary
* notifier function to be specified. However, very few unique functions
* are specified in practice, so to prevent bloating the evtchn structure
@@ -138,7 +172,7 @@ static struct evtchn *alloc_evtchn_bucke
return NULL;
}
chn[i].port = port + i;
- spin_lock_init(&chn[i].lock);
+ rwlock_init(&chn[i].lock);
}
return chn;
}
@@ -242,7 +276,6 @@ static long evtchn_alloc_unbound(evtchn_
int port;
domid_t dom = alloc->dom;
long rc;
- unsigned long flags;
d = rcu_lock_domain_by_any_id(dom);
if ( d == NULL )
@@ -258,14 +291,14 @@ static long evtchn_alloc_unbound(evtchn_
if ( rc )
goto out;
- spin_lock_irqsave(&chn->lock, flags);
+ evtchn_write_lock(chn);
chn->state = ECS_UNBOUND;
if ( (chn->u.unbound.remote_domid = alloc->remote_dom) == DOMID_SELF )
chn->u.unbound.remote_domid = current->domain->domain_id;
evtchn_port_init(d, chn);
- spin_unlock_irqrestore(&chn->lock, flags);
+ evtchn_write_unlock(chn);
alloc->port = port;
@@ -278,32 +311,26 @@ static long evtchn_alloc_unbound(evtchn_
}
-static unsigned long double_evtchn_lock(struct evtchn *lchn,
- struct evtchn *rchn)
+static void double_evtchn_lock(struct evtchn *lchn, struct evtchn *rchn)
{
- unsigned long flags;
-
if ( lchn <= rchn )
{
- spin_lock_irqsave(&lchn->lock, flags);
+ evtchn_write_lock(lchn);
if ( lchn != rchn )
- spin_lock(&rchn->lock);
+ evtchn_write_lock(rchn);
}
else
{
- spin_lock_irqsave(&rchn->lock, flags);
- spin_lock(&lchn->lock);
+ evtchn_write_lock(rchn);
+ evtchn_write_lock(lchn);
}
-
- return flags;
}
-static void double_evtchn_unlock(struct evtchn *lchn, struct evtchn *rchn,
- unsigned long flags)
+static void double_evtchn_unlock(struct evtchn *lchn, struct evtchn *rchn)
{
if ( lchn != rchn )
- spin_unlock(&lchn->lock);
- spin_unlock_irqrestore(&rchn->lock, flags);
+ evtchn_write_unlock(lchn);
+ evtchn_write_unlock(rchn);
}
static long evtchn_bind_interdomain(evtchn_bind_interdomain_t *bind)
@@ -313,7 +340,6 @@ static long evtchn_bind_interdomain(evtc
int lport, rport = bind->remote_port;
domid_t rdom = bind->remote_dom;
long rc;
- unsigned long flags;
if ( rdom == DOMID_SELF )
rdom = current->domain->domain_id;
@@ -349,7 +375,7 @@ static long evtchn_bind_interdomain(evtc
if ( rc )
goto out;
- flags = double_evtchn_lock(lchn, rchn);
+ double_evtchn_lock(lchn, rchn);
lchn->u.interdomain.remote_dom = rd;
lchn->u.interdomain.remote_port = rport;
@@ -366,7 +392,7 @@ static long evtchn_bind_interdomain(evtc
*/
evtchn_port_set_pending(ld, lchn->notify_vcpu_id, lchn);
- double_evtchn_unlock(lchn, rchn, flags);
+ double_evtchn_unlock(lchn, rchn);
bind->local_port = lport;
@@ -389,7 +415,6 @@ static long evtchn_bind_virq(evtchn_bind
struct domain *d = current->domain;
int port, virq = bind->virq, vcpu = bind->vcpu;
long rc = 0;
- unsigned long flags;
if ( (virq < 0) || (virq >= ARRAY_SIZE(v->virq_to_evtchn)) )
return -EINVAL;
@@ -411,14 +436,14 @@ static long evtchn_bind_virq(evtchn_bind
chn = evtchn_from_port(d, port);
- spin_lock_irqsave(&chn->lock, flags);
+ evtchn_write_lock(chn);
chn->state = ECS_VIRQ;
chn->notify_vcpu_id = vcpu;
chn->u.virq = virq;
evtchn_port_init(d, chn);
- spin_unlock_irqrestore(&chn->lock, flags);
+ evtchn_write_unlock(chn);
v->virq_to_evtchn[virq] = bind->port = port;
@@ -435,7 +460,6 @@ static long evtchn_bind_ipi(evtchn_bind_
struct domain *d = current->domain;
int port, vcpu = bind->vcpu;
long rc = 0;
- unsigned long flags;
if ( (vcpu < 0) || (vcpu >= d->max_vcpus) ||
(d->vcpu[vcpu] == NULL) )
@@ -448,13 +472,13 @@ static long evtchn_bind_ipi(evtchn_bind_
chn = evtchn_from_port(d, port);
- spin_lock_irqsave(&chn->lock, flags);
+ evtchn_write_lock(chn);
chn->state = ECS_IPI;
chn->notify_vcpu_id = vcpu;
evtchn_port_init(d, chn);
- spin_unlock_irqrestore(&chn->lock, flags);
+ evtchn_write_unlock(chn);
bind->port = port;
@@ -498,7 +522,6 @@ static long evtchn_bind_pirq(evtchn_bind
struct pirq *info;
int port = 0, pirq = bind->pirq;
long rc;
- unsigned long flags;
if ( (pirq < 0) || (pirq >= d->nr_pirqs) )
return -EINVAL;
@@ -531,14 +554,14 @@ static long evtchn_bind_pirq(evtchn_bind
goto out;
}
- spin_lock_irqsave(&chn->lock, flags);
+ evtchn_write_lock(chn);
chn->state = ECS_PIRQ;
chn->u.pirq.irq = pirq;
link_pirq_port(port, chn, v);
evtchn_port_init(d, chn);
- spin_unlock_irqrestore(&chn->lock, flags);
+ evtchn_write_unlock(chn);
bind->port = port;
@@ -559,7 +582,6 @@ static long evtchn_close(struct domain *
struct evtchn *chn1, *chn2;
int port2;
long rc = 0;
- unsigned long flags;
again:
spin_lock(&d1->event_lock);
@@ -659,14 +681,14 @@ static long evtchn_close(struct domain *
BUG_ON(chn2->state != ECS_INTERDOMAIN);
BUG_ON(chn2->u.interdomain.remote_dom != d1);
- flags = double_evtchn_lock(chn1, chn2);
+ double_evtchn_lock(chn1, chn2);
free_evtchn(d1, chn1);
chn2->state = ECS_UNBOUND;
chn2->u.unbound.remote_domid = d1->domain_id;
- double_evtchn_unlock(chn1, chn2, flags);
+ double_evtchn_unlock(chn1, chn2);
goto out;
@@ -674,9 +696,9 @@ static long evtchn_close(struct domain *
BUG();
}
- spin_lock_irqsave(&chn1->lock, flags);
+ evtchn_write_lock(chn1);
free_evtchn(d1, chn1);
- spin_unlock_irqrestore(&chn1->lock, flags);
+ evtchn_write_unlock(chn1);
out:
if ( d2 != NULL )
@@ -696,14 +718,13 @@ int evtchn_send(struct domain *ld, unsig
struct evtchn *lchn, *rchn;
struct domain *rd;
int rport, ret = 0;
- unsigned long flags;
if ( !port_is_valid(ld, lport) )
return -EINVAL;
lchn = evtchn_from_port(ld, lport);
- spin_lock_irqsave(&lchn->lock, flags);
+ evtchn_read_lock(lchn);
/* Guest cannot send via a Xen-attached event channel. */
if ( unlikely(consumer_is_xen(lchn)) )
@@ -738,7 +759,7 @@ int evtchn_send(struct domain *ld, unsig
}
out:
- spin_unlock_irqrestore(&lchn->lock, flags);
+ evtchn_read_unlock(lchn);
return ret;
}
@@ -765,9 +786,11 @@ void send_guest_vcpu_virq(struct vcpu *v
d = v->domain;
chn = evtchn_from_port(d, port);
- spin_lock(&chn->lock);
- evtchn_port_set_pending(d, v->vcpu_id, chn);
- spin_unlock(&chn->lock);
+ if ( evtchn_read_trylock(chn) )
+ {
+ evtchn_port_set_pending(d, v->vcpu_id, chn);
+ evtchn_read_unlock(chn);
+ }
out:
spin_unlock_irqrestore(&v->virq_lock, flags);
@@ -796,9 +819,11 @@ static void send_guest_global_virq(struc
goto out;
chn = evtchn_from_port(d, port);
- spin_lock(&chn->lock);
- evtchn_port_set_pending(d, chn->notify_vcpu_id, chn);
- spin_unlock(&chn->lock);
+ if ( evtchn_read_trylock(chn) )
+ {
+ evtchn_port_set_pending(d, chn->notify_vcpu_id, chn);
+ evtchn_read_unlock(chn);
+ }
out:
spin_unlock_irqrestore(&v->virq_lock, flags);
@@ -808,7 +833,6 @@ void send_guest_pirq(struct domain *d, c
{
int port;
struct evtchn *chn;
- unsigned long flags;
/*
* PV guests: It should not be possible to race with __evtchn_close(). The
@@ -823,9 +847,11 @@ void send_guest_pirq(struct domain *d, c
}
chn = evtchn_from_port(d, port);
- spin_lock_irqsave(&chn->lock, flags);
- evtchn_port_set_pending(d, chn->notify_vcpu_id, chn);
- spin_unlock_irqrestore(&chn->lock, flags);
+ if ( evtchn_read_trylock(chn) )
+ {
+ evtchn_port_set_pending(d, chn->notify_vcpu_id, chn);
+ evtchn_read_unlock(chn);
+ }
}
static struct domain *global_virq_handlers[NR_VIRQS] __read_mostly;
@@ -1021,15 +1047,17 @@ int evtchn_unmask(unsigned int port)
{
struct domain *d = current->domain;
struct evtchn *evtchn;
- unsigned long flags;
if ( unlikely(!port_is_valid(d, port)) )
return -EINVAL;
evtchn = evtchn_from_port(d, port);
- spin_lock_irqsave(&evtchn->lock, flags);
+
+ evtchn_read_lock(evtchn);
+
evtchn_port_unmask(d, evtchn);
- spin_unlock_irqrestore(&evtchn->lock, flags);
+
+ evtchn_read_unlock(evtchn);
return 0;
}
@@ -1275,7 +1303,6 @@ int alloc_unbound_xen_event_channel(
{
struct evtchn *chn;
int port, rc;
- unsigned long flags;
spin_lock(&ld->event_lock);
@@ -1288,14 +1315,14 @@ int alloc_unbound_xen_event_channel(
if ( rc )
goto out;
- spin_lock_irqsave(&chn->lock, flags);
+ evtchn_write_lock(chn);
chn->state = ECS_UNBOUND;
chn->xen_consumer = get_xen_consumer(notification_fn);
chn->notify_vcpu_id = lvcpu;
chn->u.unbound.remote_domid = remote_domid;
- spin_unlock_irqrestore(&chn->lock, flags);
+ evtchn_write_unlock(chn);
write_atomic(&ld->xen_evtchns, ld->xen_evtchns + 1);
@@ -1327,7 +1354,6 @@ void notify_via_xen_event_channel(struct
{
struct evtchn *lchn, *rchn;
struct domain *rd;
- unsigned long flags;
if ( !port_is_valid(ld, lport) )
{
@@ -1342,7 +1368,8 @@ void notify_via_xen_event_channel(struct
lchn = evtchn_from_port(ld, lport);
- spin_lock_irqsave(&lchn->lock, flags);
+ if ( !evtchn_read_trylock(lchn) )
+ return;
if ( likely(lchn->state == ECS_INTERDOMAIN) )
{
@@ -1352,7 +1379,7 @@ void notify_via_xen_event_channel(struct
evtchn_port_set_pending(rd, rchn->notify_vcpu_id, rchn);
}
- spin_unlock_irqrestore(&lchn->lock, flags);
+ evtchn_read_unlock(lchn);
}
void evtchn_check_pollers(struct domain *d, unsigned int port)
--- a/xen/include/xen/event.h
+++ b/xen/include/xen/event.h
@@ -91,6 +91,21 @@ static inline unsigned int max_evtchns(c
: BITS_PER_EVTCHN_WORD(d) * BITS_PER_EVTCHN_WORD(d);
}
+static inline void evtchn_read_lock(struct evtchn *evtchn)
+{
+ read_lock(&evtchn->lock);
+}
+
+static inline bool evtchn_read_trylock(struct evtchn *evtchn)
+{
+ return read_trylock(&evtchn->lock);
+}
+
+static inline void evtchn_read_unlock(struct evtchn *evtchn)
+{
+ read_unlock(&evtchn->lock);
+}
+
static inline bool_t port_is_valid(struct domain *d, unsigned int p)
{
if ( p >= read_atomic(&d->valid_evtchns) )
@@ -225,11 +240,10 @@ static inline bool_t evtchn_port_is_pend
{
struct evtchn *evtchn = evtchn_from_port(d, port);
bool rc;
- unsigned long flags;
- spin_lock_irqsave(&evtchn->lock, flags);
+ evtchn_read_lock(evtchn);
rc = evtchn_is_pending(d, evtchn);
- spin_unlock_irqrestore(&evtchn->lock, flags);
+ evtchn_read_unlock(evtchn);
return rc;
}
@@ -245,11 +259,12 @@ static inline bool_t evtchn_port_is_mask
{
struct evtchn *evtchn = evtchn_from_port(d, port);
bool rc;
- unsigned long flags;
- spin_lock_irqsave(&evtchn->lock, flags);
+ evtchn_read_lock(evtchn);
+
rc = evtchn_is_masked(d, evtchn);
- spin_unlock_irqrestore(&evtchn->lock, flags);
+
+ evtchn_read_unlock(evtchn);
return rc;
}
--- a/xen/include/xen/sched.h
+++ b/xen/include/xen/sched.h
@@ -81,7 +81,7 @@ extern domid_t hardware_domid;
struct evtchn
{
- spinlock_t lock;
+ rwlock_t lock;
#define ECS_FREE 0 /* Channel is available for use. */
#define ECS_RESERVED 1 /* Channel is reserved. */
#define ECS_UNBOUND 2 /* Channel is waiting to bind to a remote domain. */
@@ -110,6 +110,9 @@ struct evtchn
u16 virq; /* state == ECS_VIRQ */
} u;
u8 priority;
+#ifndef NDEBUG
+ u8 old_state; /* State when taking lock in write mode. */
+#endif
u8 last_priority;
u16 last_vcpu_id;
#ifdef CONFIG_XSM
