2013-05-13 11:29:47 +00:00
|
|
|
/*
|
|
|
|
|
* Simple interface for atomic operations.
|
|
|
|
|
*
|
|
|
|
|
* Copyright (C) 2013 Red Hat, Inc.
|
|
|
|
|
*
|
|
|
|
|
* Author: Paolo Bonzini <pbonzini@redhat.com>
|
|
|
|
|
*
|
|
|
|
|
* This work is licensed under the terms of the GNU GPL, version 2 or later.
|
|
|
|
|
* See the COPYING file in the top-level directory.
|
|
|
|
|
*
|
|
|
|
|
*/
|
2010-02-22 16:57:54 +00:00
|
|
|
|
2013-05-13 11:29:47 +00:00
|
|
|
#ifndef __QEMU_ATOMIC_H
|
|
|
|
|
#define __QEMU_ATOMIC_H 1
|
2010-06-25 14:56:49 +00:00
|
|
|
|
2013-05-13 11:29:47 +00:00
|
|
|
#include "qemu/compiler.h"
|
Barriers in qemu-barrier.h should not be x86 specific
qemu-barrier.h contains a few macros implementing memory barrier
primitives used in several places throughout qemu. However, apart
from the compiler-only barrier, the defined wmb() is correct only for
x86, or platforms which are similarly strongly ordered.
This patch addresses the FIXME about this by making the wmb() macro
arch dependent. On x86, it remains a compiler barrier only, but with
a comment explaining in more detail the conditions under which this is
correct. On weakly-ordered powerpc, an "eieio" instruction is used,
again with explanation of the conditions under which it is sufficient.
On other platforms, we use the __sync_synchronize() primitive,
available in sufficiently recent gcc (4.2 and after?). This should
implement a full barrier which will be sufficient on all platforms,
although it may be overkill in some cases. Other platforms can add
optimized versions in future if it's worth it for them.
Without proper memory barriers, it is easy to reproduce ordering
problems with virtio on powerpc; specifically, the QEMU puts new
element into the "used" ring and then updates the ring free-running
counter. Without a barrier between these under the right
circumstances, the guest linux driver can receive an interrupt, read
the counter change but find the ring element to be handled still has
an old value, leading to an "id %u is not a head!\n" error message.
Similar problems are likely to be possible with kvm on other weakly
ordered platforms.
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2011-09-20 02:05:21 +00:00
|
|
|
|
2013-05-13 11:29:47 +00:00
|
|
|
/* For C11 atomic ops */
|
2012-10-06 10:46:15 +00:00
|
|
|
|
2013-05-13 11:29:47 +00:00
|
|
|
/* Compiler barrier */
|
|
|
|
|
#define barrier() ({ asm volatile("" ::: "memory"); (void)0; })
|
|
|
|
|
|
|
|
|
|
#ifndef __ATOMIC_RELAXED
|
2013-02-22 16:36:25 +00:00
|
|
|
|
virtio: add missing mb() on notification
During normal operation, virtio first writes a used index
and then checks whether it should interrupt the guest
by reading guest avail index/flag values.
Guest does the reverse: writes the index/flag,
then checks the used ring.
The ordering is important: if host avail flag read bypasses the used
index write, we could in effect get this timing:
host avail flag read
guest enable interrupts: avail flag write
guest check used ring: ring is empty
host used index write
which results in a lost interrupt: guest will never be notified
about the used ring update.
This actually can happen when using kvm with an io thread,
such that the guest vcpu and qemu run on different host cpus,
and this has actually been observed in the field
(but only seems to trigger on very specific processor types)
with userspace virtio: vhost has the necessary smp_mb()
in place to prevent the regordering, so the same workload stalls
forever waiting for an interrupt with vhost=off but works
fine with vhost=on.
Insert an smp_mb barrier operation in userspace virtio to
ensure the correct ordering.
Applying this patch fixed the race condition we have observed.
Tested on x86_64. I checked the code generated by the new macro
for i386 and ppc but didn't run virtio.
Note: mb could in theory be implemented by __sync_synchronize, but this
would make us hit old GCC bugs. Besides old GCC
not implementing __sync_synchronize at all, there were bugs
http://gcc.gnu.org/bugzilla/show_bug.cgi?id=36793
in this functionality as recently as in 4.3.
As we need asm for rmb,wmb anyway, it's just as well to
use it for mb.
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
2012-04-22 13:45:53 +00:00
|
|
|
/*
|
2013-05-13 11:29:47 +00:00
|
|
|
* We use GCC builtin if it's available, as that can use mfence on
|
|
|
|
|
* 32-bit as well, e.g. if built with -march=pentium-m. However, on
|
|
|
|
|
* i386 the spec is buggy, and the implementation followed it until
|
|
|
|
|
* 4.3 (http://gcc.gnu.org/bugzilla/show_bug.cgi?id=36793).
|
|
|
|
|
*/
|
|
|
|
|
#if defined(__i386__) || defined(__x86_64__)
|
|
|
|
|
#if !QEMU_GNUC_PREREQ(4, 4)
|
|
|
|
|
#if defined __x86_64__
|
|
|
|
|
#define smp_mb() ({ asm volatile("mfence" ::: "memory"); (void)0; })
|
virtio: add missing mb() on notification
During normal operation, virtio first writes a used index
and then checks whether it should interrupt the guest
by reading guest avail index/flag values.
Guest does the reverse: writes the index/flag,
then checks the used ring.
The ordering is important: if host avail flag read bypasses the used
index write, we could in effect get this timing:
host avail flag read
guest enable interrupts: avail flag write
guest check used ring: ring is empty
host used index write
which results in a lost interrupt: guest will never be notified
about the used ring update.
This actually can happen when using kvm with an io thread,
such that the guest vcpu and qemu run on different host cpus,
and this has actually been observed in the field
(but only seems to trigger on very specific processor types)
with userspace virtio: vhost has the necessary smp_mb()
in place to prevent the regordering, so the same workload stalls
forever waiting for an interrupt with vhost=off but works
fine with vhost=on.
Insert an smp_mb barrier operation in userspace virtio to
ensure the correct ordering.
Applying this patch fixed the race condition we have observed.
Tested on x86_64. I checked the code generated by the new macro
for i386 and ppc but didn't run virtio.
Note: mb could in theory be implemented by __sync_synchronize, but this
would make us hit old GCC bugs. Besides old GCC
not implementing __sync_synchronize at all, there were bugs
http://gcc.gnu.org/bugzilla/show_bug.cgi?id=36793
in this functionality as recently as in 4.3.
As we need asm for rmb,wmb anyway, it's just as well to
use it for mb.
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
2012-04-22 13:45:53 +00:00
|
|
|
#else
|
2013-05-13 11:29:47 +00:00
|
|
|
#define smp_mb() ({ asm volatile("lock; addl $0,0(%%esp) " ::: "memory"); (void)0; })
|
|
|
|
|
#endif
|
|
|
|
|
#endif
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#ifdef __alpha__
|
|
|
|
|
#define smp_read_barrier_depends() asm volatile("mb":::"memory")
|
virtio: add missing mb() on notification
During normal operation, virtio first writes a used index
and then checks whether it should interrupt the guest
by reading guest avail index/flag values.
Guest does the reverse: writes the index/flag,
then checks the used ring.
The ordering is important: if host avail flag read bypasses the used
index write, we could in effect get this timing:
host avail flag read
guest enable interrupts: avail flag write
guest check used ring: ring is empty
host used index write
which results in a lost interrupt: guest will never be notified
about the used ring update.
This actually can happen when using kvm with an io thread,
such that the guest vcpu and qemu run on different host cpus,
and this has actually been observed in the field
(but only seems to trigger on very specific processor types)
with userspace virtio: vhost has the necessary smp_mb()
in place to prevent the regordering, so the same workload stalls
forever waiting for an interrupt with vhost=off but works
fine with vhost=on.
Insert an smp_mb barrier operation in userspace virtio to
ensure the correct ordering.
Applying this patch fixed the race condition we have observed.
Tested on x86_64. I checked the code generated by the new macro
for i386 and ppc but didn't run virtio.
Note: mb could in theory be implemented by __sync_synchronize, but this
would make us hit old GCC bugs. Besides old GCC
not implementing __sync_synchronize at all, there were bugs
http://gcc.gnu.org/bugzilla/show_bug.cgi?id=36793
in this functionality as recently as in 4.3.
As we need asm for rmb,wmb anyway, it's just as well to
use it for mb.
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
2012-04-22 13:45:53 +00:00
|
|
|
#endif
|
|
|
|
|
|
2013-05-13 11:29:47 +00:00
|
|
|
#if defined(__i386__) || defined(__x86_64__) || defined(__s390x__)
|
virtio: add missing mb() on notification
During normal operation, virtio first writes a used index
and then checks whether it should interrupt the guest
by reading guest avail index/flag values.
Guest does the reverse: writes the index/flag,
then checks the used ring.
The ordering is important: if host avail flag read bypasses the used
index write, we could in effect get this timing:
host avail flag read
guest enable interrupts: avail flag write
guest check used ring: ring is empty
host used index write
which results in a lost interrupt: guest will never be notified
about the used ring update.
This actually can happen when using kvm with an io thread,
such that the guest vcpu and qemu run on different host cpus,
and this has actually been observed in the field
(but only seems to trigger on very specific processor types)
with userspace virtio: vhost has the necessary smp_mb()
in place to prevent the regordering, so the same workload stalls
forever waiting for an interrupt with vhost=off but works
fine with vhost=on.
Insert an smp_mb barrier operation in userspace virtio to
ensure the correct ordering.
Applying this patch fixed the race condition we have observed.
Tested on x86_64. I checked the code generated by the new macro
for i386 and ppc but didn't run virtio.
Note: mb could in theory be implemented by __sync_synchronize, but this
would make us hit old GCC bugs. Besides old GCC
not implementing __sync_synchronize at all, there were bugs
http://gcc.gnu.org/bugzilla/show_bug.cgi?id=36793
in this functionality as recently as in 4.3.
As we need asm for rmb,wmb anyway, it's just as well to
use it for mb.
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
2012-04-22 13:45:53 +00:00
|
|
|
|
2013-05-13 11:29:47 +00:00
|
|
|
/*
|
|
|
|
|
* Because of the strongly ordered storage model, wmb() and rmb() are nops
|
|
|
|
|
* here (a compiler barrier only). QEMU doesn't do accesses to write-combining
|
|
|
|
|
* qemu memory or non-temporal load/stores from C code.
|
|
|
|
|
*/
|
virtio: add missing mb() on notification
During normal operation, virtio first writes a used index
and then checks whether it should interrupt the guest
by reading guest avail index/flag values.
Guest does the reverse: writes the index/flag,
then checks the used ring.
The ordering is important: if host avail flag read bypasses the used
index write, we could in effect get this timing:
host avail flag read
guest enable interrupts: avail flag write
guest check used ring: ring is empty
host used index write
which results in a lost interrupt: guest will never be notified
about the used ring update.
This actually can happen when using kvm with an io thread,
such that the guest vcpu and qemu run on different host cpus,
and this has actually been observed in the field
(but only seems to trigger on very specific processor types)
with userspace virtio: vhost has the necessary smp_mb()
in place to prevent the regordering, so the same workload stalls
forever waiting for an interrupt with vhost=off but works
fine with vhost=on.
Insert an smp_mb barrier operation in userspace virtio to
ensure the correct ordering.
Applying this patch fixed the race condition we have observed.
Tested on x86_64. I checked the code generated by the new macro
for i386 and ppc but didn't run virtio.
Note: mb could in theory be implemented by __sync_synchronize, but this
would make us hit old GCC bugs. Besides old GCC
not implementing __sync_synchronize at all, there were bugs
http://gcc.gnu.org/bugzilla/show_bug.cgi?id=36793
in this functionality as recently as in 4.3.
As we need asm for rmb,wmb anyway, it's just as well to
use it for mb.
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
2012-04-22 13:45:53 +00:00
|
|
|
#define smp_wmb() barrier()
|
2012-04-23 12:46:22 +00:00
|
|
|
#define smp_rmb() barrier()
|
2013-05-13 11:29:47 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* __sync_lock_test_and_set() is documented to be an acquire barrier only,
|
|
|
|
|
* but it is a full barrier at the hardware level. Add a compiler barrier
|
|
|
|
|
* to make it a full barrier also at the compiler level.
|
|
|
|
|
*/
|
|
|
|
|
#define atomic_xchg(ptr, i) (barrier(), __sync_lock_test_and_set(ptr, i))
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Load/store with Java volatile semantics.
|
|
|
|
|
*/
|
|
|
|
|
#define atomic_mb_set(ptr, i) ((void)atomic_xchg(ptr, i))
|
Barriers in qemu-barrier.h should not be x86 specific
qemu-barrier.h contains a few macros implementing memory barrier
primitives used in several places throughout qemu. However, apart
from the compiler-only barrier, the defined wmb() is correct only for
x86, or platforms which are similarly strongly ordered.
This patch addresses the FIXME about this by making the wmb() macro
arch dependent. On x86, it remains a compiler barrier only, but with
a comment explaining in more detail the conditions under which this is
correct. On weakly-ordered powerpc, an "eieio" instruction is used,
again with explanation of the conditions under which it is sufficient.
On other platforms, we use the __sync_synchronize() primitive,
available in sufficiently recent gcc (4.2 and after?). This should
implement a full barrier which will be sufficient on all platforms,
although it may be overkill in some cases. Other platforms can add
optimized versions in future if it's worth it for them.
Without proper memory barriers, it is easy to reproduce ordering
problems with virtio on powerpc; specifically, the QEMU puts new
element into the "used" ring and then updates the ring free-running
counter. Without a barrier between these under the right
circumstances, the guest linux driver can receive an interrupt, read
the counter change but find the ring element to be handled still has
an old value, leading to an "id %u is not a head!\n" error message.
Similar problems are likely to be possible with kvm on other weakly
ordered platforms.
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2011-09-20 02:05:21 +00:00
|
|
|
|
2011-11-01 09:39:49 +00:00
|
|
|
#elif defined(_ARCH_PPC)
|
Barriers in qemu-barrier.h should not be x86 specific
qemu-barrier.h contains a few macros implementing memory barrier
primitives used in several places throughout qemu. However, apart
from the compiler-only barrier, the defined wmb() is correct only for
x86, or platforms which are similarly strongly ordered.
This patch addresses the FIXME about this by making the wmb() macro
arch dependent. On x86, it remains a compiler barrier only, but with
a comment explaining in more detail the conditions under which this is
correct. On weakly-ordered powerpc, an "eieio" instruction is used,
again with explanation of the conditions under which it is sufficient.
On other platforms, we use the __sync_synchronize() primitive,
available in sufficiently recent gcc (4.2 and after?). This should
implement a full barrier which will be sufficient on all platforms,
although it may be overkill in some cases. Other platforms can add
optimized versions in future if it's worth it for them.
Without proper memory barriers, it is easy to reproduce ordering
problems with virtio on powerpc; specifically, the QEMU puts new
element into the "used" ring and then updates the ring free-running
counter. Without a barrier between these under the right
circumstances, the guest linux driver can receive an interrupt, read
the counter change but find the ring element to be handled still has
an old value, leading to an "id %u is not a head!\n" error message.
Similar problems are likely to be possible with kvm on other weakly
ordered platforms.
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2011-09-20 02:05:21 +00:00
|
|
|
|
|
|
|
|
/*
|
virtio: add missing mb() on notification
During normal operation, virtio first writes a used index
and then checks whether it should interrupt the guest
by reading guest avail index/flag values.
Guest does the reverse: writes the index/flag,
then checks the used ring.
The ordering is important: if host avail flag read bypasses the used
index write, we could in effect get this timing:
host avail flag read
guest enable interrupts: avail flag write
guest check used ring: ring is empty
host used index write
which results in a lost interrupt: guest will never be notified
about the used ring update.
This actually can happen when using kvm with an io thread,
such that the guest vcpu and qemu run on different host cpus,
and this has actually been observed in the field
(but only seems to trigger on very specific processor types)
with userspace virtio: vhost has the necessary smp_mb()
in place to prevent the regordering, so the same workload stalls
forever waiting for an interrupt with vhost=off but works
fine with vhost=on.
Insert an smp_mb barrier operation in userspace virtio to
ensure the correct ordering.
Applying this patch fixed the race condition we have observed.
Tested on x86_64. I checked the code generated by the new macro
for i386 and ppc but didn't run virtio.
Note: mb could in theory be implemented by __sync_synchronize, but this
would make us hit old GCC bugs. Besides old GCC
not implementing __sync_synchronize at all, there were bugs
http://gcc.gnu.org/bugzilla/show_bug.cgi?id=36793
in this functionality as recently as in 4.3.
As we need asm for rmb,wmb anyway, it's just as well to
use it for mb.
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
2012-04-22 13:45:53 +00:00
|
|
|
* We use an eieio() for wmb() on powerpc. This assumes we don't
|
Barriers in qemu-barrier.h should not be x86 specific
qemu-barrier.h contains a few macros implementing memory barrier
primitives used in several places throughout qemu. However, apart
from the compiler-only barrier, the defined wmb() is correct only for
x86, or platforms which are similarly strongly ordered.
This patch addresses the FIXME about this by making the wmb() macro
arch dependent. On x86, it remains a compiler barrier only, but with
a comment explaining in more detail the conditions under which this is
correct. On weakly-ordered powerpc, an "eieio" instruction is used,
again with explanation of the conditions under which it is sufficient.
On other platforms, we use the __sync_synchronize() primitive,
available in sufficiently recent gcc (4.2 and after?). This should
implement a full barrier which will be sufficient on all platforms,
although it may be overkill in some cases. Other platforms can add
optimized versions in future if it's worth it for them.
Without proper memory barriers, it is easy to reproduce ordering
problems with virtio on powerpc; specifically, the QEMU puts new
element into the "used" ring and then updates the ring free-running
counter. Without a barrier between these under the right
circumstances, the guest linux driver can receive an interrupt, read
the counter change but find the ring element to be handled still has
an old value, leading to an "id %u is not a head!\n" error message.
Similar problems are likely to be possible with kvm on other weakly
ordered platforms.
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2011-09-20 02:05:21 +00:00
|
|
|
* need to order cacheable and non-cacheable stores with respect to
|
2013-05-13 11:29:47 +00:00
|
|
|
* each other.
|
|
|
|
|
*
|
|
|
|
|
* smp_mb has the same problem as on x86 for not-very-new GCC
|
|
|
|
|
* (http://patchwork.ozlabs.org/patch/126184/, Nov 2011).
|
Barriers in qemu-barrier.h should not be x86 specific
qemu-barrier.h contains a few macros implementing memory barrier
primitives used in several places throughout qemu. However, apart
from the compiler-only barrier, the defined wmb() is correct only for
x86, or platforms which are similarly strongly ordered.
This patch addresses the FIXME about this by making the wmb() macro
arch dependent. On x86, it remains a compiler barrier only, but with
a comment explaining in more detail the conditions under which this is
correct. On weakly-ordered powerpc, an "eieio" instruction is used,
again with explanation of the conditions under which it is sufficient.
On other platforms, we use the __sync_synchronize() primitive,
available in sufficiently recent gcc (4.2 and after?). This should
implement a full barrier which will be sufficient on all platforms,
although it may be overkill in some cases. Other platforms can add
optimized versions in future if it's worth it for them.
Without proper memory barriers, it is easy to reproduce ordering
problems with virtio on powerpc; specifically, the QEMU puts new
element into the "used" ring and then updates the ring free-running
counter. Without a barrier between these under the right
circumstances, the guest linux driver can receive an interrupt, read
the counter change but find the ring element to be handled still has
an old value, leading to an "id %u is not a head!\n" error message.
Similar problems are likely to be possible with kvm on other weakly
ordered platforms.
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2011-09-20 02:05:21 +00:00
|
|
|
*/
|
2013-05-13 11:29:47 +00:00
|
|
|
#define smp_wmb() ({ asm volatile("eieio" ::: "memory"); (void)0; })
|
2012-04-23 12:46:22 +00:00
|
|
|
#if defined(__powerpc64__)
|
2013-05-13 11:29:47 +00:00
|
|
|
#define smp_rmb() ({ asm volatile("lwsync" ::: "memory"); (void)0; })
|
2012-04-23 12:46:22 +00:00
|
|
|
#else
|
2013-05-13 11:29:47 +00:00
|
|
|
#define smp_rmb() ({ asm volatile("sync" ::: "memory"); (void)0; })
|
2012-04-23 12:46:22 +00:00
|
|
|
#endif
|
2013-05-13 11:29:47 +00:00
|
|
|
#define smp_mb() ({ asm volatile("sync" ::: "memory"); (void)0; })
|
2012-04-23 12:46:22 +00:00
|
|
|
|
2013-05-13 11:29:47 +00:00
|
|
|
#endif /* _ARCH_PPC */
|
Barriers in qemu-barrier.h should not be x86 specific
qemu-barrier.h contains a few macros implementing memory barrier
primitives used in several places throughout qemu. However, apart
from the compiler-only barrier, the defined wmb() is correct only for
x86, or platforms which are similarly strongly ordered.
This patch addresses the FIXME about this by making the wmb() macro
arch dependent. On x86, it remains a compiler barrier only, but with
a comment explaining in more detail the conditions under which this is
correct. On weakly-ordered powerpc, an "eieio" instruction is used,
again with explanation of the conditions under which it is sufficient.
On other platforms, we use the __sync_synchronize() primitive,
available in sufficiently recent gcc (4.2 and after?). This should
implement a full barrier which will be sufficient on all platforms,
although it may be overkill in some cases. Other platforms can add
optimized versions in future if it's worth it for them.
Without proper memory barriers, it is easy to reproduce ordering
problems with virtio on powerpc; specifically, the QEMU puts new
element into the "used" ring and then updates the ring free-running
counter. Without a barrier between these under the right
circumstances, the guest linux driver can receive an interrupt, read
the counter change but find the ring element to be handled still has
an old value, leading to an "id %u is not a head!\n" error message.
Similar problems are likely to be possible with kvm on other weakly
ordered platforms.
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2011-09-20 02:05:21 +00:00
|
|
|
|
2013-05-13 11:29:47 +00:00
|
|
|
#endif /* C11 atomics */
|
Barriers in qemu-barrier.h should not be x86 specific
qemu-barrier.h contains a few macros implementing memory barrier
primitives used in several places throughout qemu. However, apart
from the compiler-only barrier, the defined wmb() is correct only for
x86, or platforms which are similarly strongly ordered.
This patch addresses the FIXME about this by making the wmb() macro
arch dependent. On x86, it remains a compiler barrier only, but with
a comment explaining in more detail the conditions under which this is
correct. On weakly-ordered powerpc, an "eieio" instruction is used,
again with explanation of the conditions under which it is sufficient.
On other platforms, we use the __sync_synchronize() primitive,
available in sufficiently recent gcc (4.2 and after?). This should
implement a full barrier which will be sufficient on all platforms,
although it may be overkill in some cases. Other platforms can add
optimized versions in future if it's worth it for them.
Without proper memory barriers, it is easy to reproduce ordering
problems with virtio on powerpc; specifically, the QEMU puts new
element into the "used" ring and then updates the ring free-running
counter. Without a barrier between these under the right
circumstances, the guest linux driver can receive an interrupt, read
the counter change but find the ring element to be handled still has
an old value, leading to an "id %u is not a head!\n" error message.
Similar problems are likely to be possible with kvm on other weakly
ordered platforms.
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2011-09-20 02:05:21 +00:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* For (host) platforms we don't have explicit barrier definitions
|
|
|
|
|
* for, we use the gcc __sync_synchronize() primitive to generate a
|
|
|
|
|
* full barrier. This should be safe on all platforms, though it may
|
2013-05-13 11:29:47 +00:00
|
|
|
* be overkill for smp_wmb() and smp_rmb().
|
Barriers in qemu-barrier.h should not be x86 specific
qemu-barrier.h contains a few macros implementing memory barrier
primitives used in several places throughout qemu. However, apart
from the compiler-only barrier, the defined wmb() is correct only for
x86, or platforms which are similarly strongly ordered.
This patch addresses the FIXME about this by making the wmb() macro
arch dependent. On x86, it remains a compiler barrier only, but with
a comment explaining in more detail the conditions under which this is
correct. On weakly-ordered powerpc, an "eieio" instruction is used,
again with explanation of the conditions under which it is sufficient.
On other platforms, we use the __sync_synchronize() primitive,
available in sufficiently recent gcc (4.2 and after?). This should
implement a full barrier which will be sufficient on all platforms,
although it may be overkill in some cases. Other platforms can add
optimized versions in future if it's worth it for them.
Without proper memory barriers, it is easy to reproduce ordering
problems with virtio on powerpc; specifically, the QEMU puts new
element into the "used" ring and then updates the ring free-running
counter. Without a barrier between these under the right
circumstances, the guest linux driver can receive an interrupt, read
the counter change but find the ring element to be handled still has
an old value, leading to an "id %u is not a head!\n" error message.
Similar problems are likely to be possible with kvm on other weakly
ordered platforms.
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2011-09-20 02:05:21 +00:00
|
|
|
*/
|
2013-05-13 11:29:47 +00:00
|
|
|
#ifndef smp_mb
|
|
|
|
|
#define smp_mb() __sync_synchronize()
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#ifndef smp_wmb
|
|
|
|
|
#ifdef __ATOMIC_RELEASE
|
|
|
|
|
#define smp_wmb() __atomic_thread_fence(__ATOMIC_RELEASE)
|
|
|
|
|
#else
|
Barriers in qemu-barrier.h should not be x86 specific
qemu-barrier.h contains a few macros implementing memory barrier
primitives used in several places throughout qemu. However, apart
from the compiler-only barrier, the defined wmb() is correct only for
x86, or platforms which are similarly strongly ordered.
This patch addresses the FIXME about this by making the wmb() macro
arch dependent. On x86, it remains a compiler barrier only, but with
a comment explaining in more detail the conditions under which this is
correct. On weakly-ordered powerpc, an "eieio" instruction is used,
again with explanation of the conditions under which it is sufficient.
On other platforms, we use the __sync_synchronize() primitive,
available in sufficiently recent gcc (4.2 and after?). This should
implement a full barrier which will be sufficient on all platforms,
although it may be overkill in some cases. Other platforms can add
optimized versions in future if it's worth it for them.
Without proper memory barriers, it is easy to reproduce ordering
problems with virtio on powerpc; specifically, the QEMU puts new
element into the "used" ring and then updates the ring free-running
counter. Without a barrier between these under the right
circumstances, the guest linux driver can receive an interrupt, read
the counter change but find the ring element to be handled still has
an old value, leading to an "id %u is not a head!\n" error message.
Similar problems are likely to be possible with kvm on other weakly
ordered platforms.
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2011-09-20 02:05:21 +00:00
|
|
|
#define smp_wmb() __sync_synchronize()
|
2013-05-13 11:29:47 +00:00
|
|
|
#endif
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#ifndef smp_rmb
|
|
|
|
|
#ifdef __ATOMIC_ACQUIRE
|
|
|
|
|
#define smp_rmb() __atomic_thread_fence(__ATOMIC_ACQUIRE)
|
|
|
|
|
#else
|
2012-04-23 12:46:22 +00:00
|
|
|
#define smp_rmb() __sync_synchronize()
|
2013-05-13 11:29:47 +00:00
|
|
|
#endif
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#ifndef smp_read_barrier_depends
|
|
|
|
|
#ifdef __ATOMIC_CONSUME
|
|
|
|
|
#define smp_read_barrier_depends() __atomic_thread_fence(__ATOMIC_CONSUME)
|
|
|
|
|
#else
|
|
|
|
|
#define smp_read_barrier_depends() barrier()
|
|
|
|
|
#endif
|
|
|
|
|
#endif
|
Barriers in qemu-barrier.h should not be x86 specific
qemu-barrier.h contains a few macros implementing memory barrier
primitives used in several places throughout qemu. However, apart
from the compiler-only barrier, the defined wmb() is correct only for
x86, or platforms which are similarly strongly ordered.
This patch addresses the FIXME about this by making the wmb() macro
arch dependent. On x86, it remains a compiler barrier only, but with
a comment explaining in more detail the conditions under which this is
correct. On weakly-ordered powerpc, an "eieio" instruction is used,
again with explanation of the conditions under which it is sufficient.
On other platforms, we use the __sync_synchronize() primitive,
available in sufficiently recent gcc (4.2 and after?). This should
implement a full barrier which will be sufficient on all platforms,
although it may be overkill in some cases. Other platforms can add
optimized versions in future if it's worth it for them.
Without proper memory barriers, it is easy to reproduce ordering
problems with virtio on powerpc; specifically, the QEMU puts new
element into the "used" ring and then updates the ring free-running
counter. Without a barrier between these under the right
circumstances, the guest linux driver can receive an interrupt, read
the counter change but find the ring element to be handled still has
an old value, leading to an "id %u is not a head!\n" error message.
Similar problems are likely to be possible with kvm on other weakly
ordered platforms.
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2011-09-20 02:05:21 +00:00
|
|
|
|
2013-05-13 11:29:47 +00:00
|
|
|
#ifndef atomic_read
|
|
|
|
|
#define atomic_read(ptr) (*(__typeof__(*ptr) *volatile) (ptr))
|
Barriers in qemu-barrier.h should not be x86 specific
qemu-barrier.h contains a few macros implementing memory barrier
primitives used in several places throughout qemu. However, apart
from the compiler-only barrier, the defined wmb() is correct only for
x86, or platforms which are similarly strongly ordered.
This patch addresses the FIXME about this by making the wmb() macro
arch dependent. On x86, it remains a compiler barrier only, but with
a comment explaining in more detail the conditions under which this is
correct. On weakly-ordered powerpc, an "eieio" instruction is used,
again with explanation of the conditions under which it is sufficient.
On other platforms, we use the __sync_synchronize() primitive,
available in sufficiently recent gcc (4.2 and after?). This should
implement a full barrier which will be sufficient on all platforms,
although it may be overkill in some cases. Other platforms can add
optimized versions in future if it's worth it for them.
Without proper memory barriers, it is easy to reproduce ordering
problems with virtio on powerpc; specifically, the QEMU puts new
element into the "used" ring and then updates the ring free-running
counter. Without a barrier between these under the right
circumstances, the guest linux driver can receive an interrupt, read
the counter change but find the ring element to be handled still has
an old value, leading to an "id %u is not a head!\n" error message.
Similar problems are likely to be possible with kvm on other weakly
ordered platforms.
Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
Signed-off-by: David Gibson <david@gibson.dropbear.id.au>
Signed-off-by: Anthony Liguori <aliguori@us.ibm.com>
2011-09-20 02:05:21 +00:00
|
|
|
#endif
|
|
|
|
|
|
2013-05-13 11:29:47 +00:00
|
|
|
#ifndef atomic_set
|
|
|
|
|
#define atomic_set(ptr, i) ((*(__typeof__(*ptr) *volatile) (ptr)) = (i))
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
/* These have the same semantics as Java volatile variables.
|
|
|
|
|
* See http://gee.cs.oswego.edu/dl/jmm/cookbook.html:
|
|
|
|
|
* "1. Issue a StoreStore barrier (wmb) before each volatile store."
|
|
|
|
|
* 2. Issue a StoreLoad barrier after each volatile store.
|
|
|
|
|
* Note that you could instead issue one before each volatile load, but
|
|
|
|
|
* this would be slower for typical programs using volatiles in which
|
|
|
|
|
* reads greatly outnumber writes. Alternatively, if available, you
|
|
|
|
|
* can implement volatile store as an atomic instruction (for example
|
|
|
|
|
* XCHG on x86) and omit the barrier. This may be more efficient if
|
|
|
|
|
* atomic instructions are cheaper than StoreLoad barriers.
|
|
|
|
|
* 3. Issue LoadLoad and LoadStore barriers after each volatile load."
|
|
|
|
|
*
|
|
|
|
|
* If you prefer to think in terms of "pairing" of memory barriers,
|
|
|
|
|
* an atomic_mb_read pairs with an atomic_mb_set.
|
|
|
|
|
*
|
|
|
|
|
* And for the few ia64 lovers that exist, an atomic_mb_read is a ld.acq,
|
|
|
|
|
* while an atomic_mb_set is a st.rel followed by a memory barrier.
|
|
|
|
|
*
|
|
|
|
|
* These are a bit weaker than __atomic_load/store with __ATOMIC_SEQ_CST
|
|
|
|
|
* (see docs/atomics.txt), and I'm not sure that __ATOMIC_ACQ_REL is enough.
|
|
|
|
|
* Just always use the barriers manually by the rules above.
|
|
|
|
|
*/
|
|
|
|
|
#ifndef atomic_mb_read
|
|
|
|
|
#define atomic_mb_read(ptr) ({ \
|
|
|
|
|
typeof(*ptr) _val = atomic_read(ptr); \
|
|
|
|
|
smp_rmb(); \
|
|
|
|
|
_val; \
|
|
|
|
|
})
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#ifndef atomic_mb_set
|
|
|
|
|
#define atomic_mb_set(ptr, i) do { \
|
|
|
|
|
smp_wmb(); \
|
|
|
|
|
atomic_set(ptr, i); \
|
|
|
|
|
smp_mb(); \
|
|
|
|
|
} while (0)
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#ifndef atomic_xchg
|
2013-10-22 09:58:41 +00:00
|
|
|
#if defined(__clang__)
|
|
|
|
|
#define atomic_xchg(ptr, i) __sync_swap(ptr, i)
|
|
|
|
|
#elif defined(__ATOMIC_SEQ_CST)
|
2013-05-13 11:29:47 +00:00
|
|
|
#define atomic_xchg(ptr, i) ({ \
|
|
|
|
|
typeof(*ptr) _new = (i), _old; \
|
|
|
|
|
__atomic_exchange(ptr, &_new, &_old, __ATOMIC_SEQ_CST); \
|
|
|
|
|
_old; \
|
|
|
|
|
})
|
|
|
|
|
#else
|
|
|
|
|
/* __sync_lock_test_and_set() is documented to be an acquire barrier only. */
|
|
|
|
|
#define atomic_xchg(ptr, i) (smp_mb(), __sync_lock_test_and_set(ptr, i))
|
|
|
|
|
#endif
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
/* Provide shorter names for GCC atomic builtins. */
|
|
|
|
|
#define atomic_fetch_inc(ptr) __sync_fetch_and_add(ptr, 1)
|
|
|
|
|
#define atomic_fetch_dec(ptr) __sync_fetch_and_add(ptr, -1)
|
|
|
|
|
#define atomic_fetch_add __sync_fetch_and_add
|
|
|
|
|
#define atomic_fetch_sub __sync_fetch_and_sub
|
|
|
|
|
#define atomic_fetch_and __sync_fetch_and_and
|
|
|
|
|
#define atomic_fetch_or __sync_fetch_and_or
|
|
|
|
|
#define atomic_cmpxchg __sync_val_compare_and_swap
|
|
|
|
|
|
|
|
|
|
/* And even shorter names that return void. */
|
|
|
|
|
#define atomic_inc(ptr) ((void) __sync_fetch_and_add(ptr, 1))
|
|
|
|
|
#define atomic_dec(ptr) ((void) __sync_fetch_and_add(ptr, -1))
|
|
|
|
|
#define atomic_add(ptr, n) ((void) __sync_fetch_and_add(ptr, n))
|
|
|
|
|
#define atomic_sub(ptr, n) ((void) __sync_fetch_and_sub(ptr, n))
|
|
|
|
|
#define atomic_and(ptr, n) ((void) __sync_fetch_and_and(ptr, n))
|
|
|
|
|
#define atomic_or(ptr, n) ((void) __sync_fetch_and_or(ptr, n))
|
|
|
|
|
|
2010-02-22 16:57:54 +00:00
|
|
|
#endif
|