[API-NEXT,PATCHv3,15/18] linux-generic: drv: adding atomic

diff --git a/include/odp_drv.h b/include/odp_drv.h
index 6d2f7ff..35e070c 100644
--- a/include/odp_drv.h
+++ b/include/odp_drv.h
@@ -19,6 +19,7 @@  extern C {
 #endif
 
 #include <odp/drv/align.h>
+#include <odp/drv/atomic.h>
 #include <odp/drv/byteorder.h>
 #include <odp/drv/compiler.h>
 #include <odp/drv/std_types.h>
diff --git a/platform/linux-generic/Makefile.am b/platform/linux-generic/Makefile.am
index 20b4b29..109b42e 100644
--- a/platform/linux-generic/Makefile.am
+++ b/platform/linux-generic/Makefile.am
@@ -94,6 +94,7 @@  odpapiplatinclude_HEADERS = \
 odpdrvincludedir = $(includedir)/odp/drv
 odpdrvinclude_HEADERS = \
 		  $(srcdir)/include/odp/drv/align.h \
+		  $(srcdir)/include/odp/drv/atomic.h \
 		  $(srcdir)/include/odp/drv/byteorder.h \
 		  $(srcdir)/include/odp/drv/compiler.h \
 		  $(srcdir)/include/odp/drv/std_types.h \
@@ -101,6 +102,7 @@  odpdrvinclude_HEADERS = \
 
 odpdrvplatincludedir = $(includedir)/odp/drv/plat
 odpdrvplatinclude_HEADERS = \
+		  $(srcdir)/include/odp/drv/plat/atomic_types.h \
 		  $(srcdir)/include/odp/drv/plat/byteorder_types.h
 
 noinst_HEADERS = \
@@ -198,6 +200,7 @@  __LIB__libodp_linux_la_SOURCES = \
 			   odp_traffic_mngr.c \
 			   odp_version.c \
 			   odp_weak.c \
+			   drv_atomic.c \
 			   arch/@ARCH_DIR@/odp_cpu_arch.c \
 			   arch/@ARCH_DIR@/odp_sysinfo_parse.c
 
diff --git a/platform/linux-generic/drv_atomic.c b/platform/linux-generic/drv_atomic.c
new file mode 100644
index 0000000..72c85e8
--- /dev/null
+++ b/platform/linux-generic/drv_atomic.c
@@ -0,0 +1,26 @@ 
+/* Copyright (c) 2016, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+#include <odp/drv/atomic.h>
+
+int odpdrv_atomic_lock_free_u64(odpdrv_atomic_op_t *atomic_op)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+	/* All operations have locks */
+	if (atomic_op)
+		atomic_op->all_bits = 0;
+
+	return 0;
+#else
+	/* All operations are lock-free */
+	if (atomic_op) {
+		atomic_op->all_bits = ~((uint32_t)0);
+		atomic_op->op.init  = 0;
+	}
+
+	return 2;
+#endif
+}
diff --git a/platform/linux-generic/include/odp/drv/atomic.h b/platform/linux-generic/include/odp/drv/atomic.h
new file mode 100644
index 0000000..7d922da
--- /dev/null
+++ b/platform/linux-generic/include/odp/drv/atomic.h
@@ -0,0 +1,430 @@ 
+/* Copyright (c) 2016, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * ODPDRV atomic operations
+ */
+
+#ifndef ODPDRV_PLAT_ATOMIC_H_
+#define ODPDRV_PLAT_ATOMIC_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp/drv/align.h>
+#include <odp/drv/plat/atomic_types.h>
+
+/** @ingroup odpdrv_atomic
+ *  @{
+ */
+
+static inline void odpdrv_atomic_init_u32(odpdrv_atomic_u32_t *atom,
+					  uint32_t val)
+{
+	__atomic_store_n(&atom->v, val, __ATOMIC_RELAXED);
+}
+
+static inline uint32_t odpdrv_atomic_load_u32(odpdrv_atomic_u32_t *atom)
+{
+	return __atomic_load_n(&atom->v, __ATOMIC_RELAXED);
+}
+
+static inline void odpdrv_atomic_store_u32(odpdrv_atomic_u32_t *atom,
+					   uint32_t val)
+{
+	__atomic_store_n(&atom->v, val, __ATOMIC_RELAXED);
+}
+
+static inline uint32_t odpdrv_atomic_fetch_add_u32(odpdrv_atomic_u32_t *atom,
+						   uint32_t val)
+{
+	return __atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED);
+}
+
+static inline void odpdrv_atomic_add_u32(odpdrv_atomic_u32_t *atom,
+					 uint32_t val)
+{
+	(void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED);
+}
+
+static inline uint32_t odpdrv_atomic_fetch_sub_u32(odpdrv_atomic_u32_t *atom,
+						   uint32_t val)
+{
+	return __atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED);
+}
+
+static inline void odpdrv_atomic_sub_u32(odpdrv_atomic_u32_t *atom,
+					 uint32_t val)
+{
+	(void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED);
+}
+
+static inline uint32_t odpdrv_atomic_fetch_inc_u32(odpdrv_atomic_u32_t *atom)
+{
+	return __atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED);
+}
+
+static inline void odpdrv_atomic_inc_u32(odpdrv_atomic_u32_t *atom)
+{
+	(void)__atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED);
+}
+
+static inline uint32_t odpdrv_atomic_fetch_dec_u32(odpdrv_atomic_u32_t *atom)
+{
+	return __atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED);
+}
+
+static inline void odpdrv_atomic_dec_u32(odpdrv_atomic_u32_t *atom)
+{
+	(void)__atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED);
+}
+
+static inline int odpdrv_atomic_cas_u32(odpdrv_atomic_u32_t *atom,
+					uint32_t *old_val, uint32_t new_val)
+{
+	return __atomic_compare_exchange_n(&atom->v, old_val, new_val,
+					   0 /* strong */,
+					   __ATOMIC_RELAXED,
+					   __ATOMIC_RELAXED);
+}
+
+static inline uint32_t odpdrv_atomic_xchg_u32(odpdrv_atomic_u32_t *atom,
+					      uint32_t new_val)
+{
+	return __atomic_exchange_n(&atom->v, new_val, __ATOMIC_RELAXED);
+}
+
+static inline void odpdrv_atomic_max_u32(odpdrv_atomic_u32_t *atom,
+					 uint32_t new_max)
+{
+	uint32_t old_val;
+
+	old_val = odpdrv_atomic_load_u32(atom);
+
+	while (new_max > old_val) {
+		if (odpdrv_atomic_cas_u32(atom, &old_val, new_max))
+			break;
+	}
+}
+
+static inline void odpdrv_atomic_min_u32(odpdrv_atomic_u32_t *atom,
+					 uint32_t new_min)
+{
+	uint32_t old_val;
+
+	old_val = odpdrv_atomic_load_u32(atom);
+
+	while (new_min < old_val) {
+		if (odpdrv_atomic_cas_u32(atom, &old_val, new_min))
+			break;
+	}
+}
+
+static inline void odpdrv_atomic_init_u64(odpdrv_atomic_u64_t *atom,
+					  uint64_t val)
+{
+	atom->v = val;
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+	__atomic_clear(&atom->lock, __ATOMIC_RELAXED);
+#endif
+}
+
+static inline uint64_t odpdrv_atomic_load_u64(odpdrv_atomic_u64_t *atom)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+	return ATOMIC_OP(atom, (void)0);
+#else
+	return __atomic_load_n(&atom->v, __ATOMIC_RELAXED);
+#endif
+}
+
+static inline void odpdrv_atomic_store_u64(odpdrv_atomic_u64_t *atom,
+					   uint64_t val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+	(void)ATOMIC_OP(atom, atom->v = val);
+#else
+	__atomic_store_n(&atom->v, val, __ATOMIC_RELAXED);
+#endif
+}
+
+static inline uint64_t odpdrv_atomic_fetch_add_u64(odpdrv_atomic_u64_t *atom,
+						   uint64_t val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+	return ATOMIC_OP(atom, atom->v += val);
+#else
+	return __atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED);
+#endif
+}
+
+static inline void odpdrv_atomic_add_u64(odpdrv_atomic_u64_t *atom,
+					 uint64_t val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+	(void)ATOMIC_OP(atom, atom->v += val);
+#else
+	(void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELAXED);
+#endif
+}
+
+static inline uint64_t odpdrv_atomic_fetch_sub_u64(odpdrv_atomic_u64_t *atom,
+						   uint64_t val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+	return ATOMIC_OP(atom, atom->v -= val);
+#else
+	return __atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED);
+#endif
+}
+
+static inline void odpdrv_atomic_sub_u64(odpdrv_atomic_u64_t *atom,
+					 uint64_t val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+	(void)ATOMIC_OP(atom, atom->v -= val);
+#else
+	(void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELAXED);
+#endif
+}
+
+static inline uint64_t odpdrv_atomic_fetch_inc_u64(odpdrv_atomic_u64_t *atom)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+	return ATOMIC_OP(atom, atom->v++);
+#else
+	return __atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED);
+#endif
+}
+
+static inline void odpdrv_atomic_inc_u64(odpdrv_atomic_u64_t *atom)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+	(void)ATOMIC_OP(atom, atom->v++);
+#else
+	(void)__atomic_fetch_add(&atom->v, 1, __ATOMIC_RELAXED);
+#endif
+}
+
+static inline uint64_t odpdrv_atomic_fetch_dec_u64(odpdrv_atomic_u64_t *atom)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+	return ATOMIC_OP(atom, atom->v--);
+#else
+	return __atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED);
+#endif
+}
+
+static inline void odpdrv_atomic_dec_u64(odpdrv_atomic_u64_t *atom)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+	(void)ATOMIC_OP(atom, atom->v--);
+#else
+	(void)__atomic_fetch_sub(&atom->v, 1, __ATOMIC_RELAXED);
+#endif
+}
+
+static inline int odpdrv_atomic_cas_u64(odpdrv_atomic_u64_t *atom,
+					uint64_t *old_val, uint64_t new_val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+	int ret;
+	*old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val));
+	return ret;
+#else
+	return __atomic_compare_exchange_n(&atom->v, old_val, new_val,
+					   0 /* strong */,
+					   __ATOMIC_RELAXED,
+					   __ATOMIC_RELAXED);
+#endif
+}
+
+static inline uint64_t odpdrv_atomic_xchg_u64(odpdrv_atomic_u64_t *atom,
+					      uint64_t new_val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+	return ATOMIC_OP(atom, atom->v = new_val);
+#else
+	return __atomic_exchange_n(&atom->v, new_val, __ATOMIC_RELAXED);
+#endif
+}
+
+static inline void odpdrv_atomic_max_u64(odpdrv_atomic_u64_t *atom,
+					 uint64_t new_max)
+{
+	uint64_t old_val;
+
+	old_val = odpdrv_atomic_load_u64(atom);
+
+	while (new_max > old_val) {
+		if (odpdrv_atomic_cas_u64(atom, &old_val, new_max))
+			break;
+	}
+}
+
+static inline void odpdrv_atomic_min_u64(odpdrv_atomic_u64_t *atom,
+					 uint64_t new_min)
+{
+	uint64_t old_val;
+
+	old_val = odpdrv_atomic_load_u64(atom);
+
+	while (new_min < old_val) {
+		if (odpdrv_atomic_cas_u64(atom, &old_val, new_min))
+			break;
+	}
+}
+
+static inline uint32_t odpdrv_atomic_load_acq_u32(odpdrv_atomic_u32_t *atom)
+{
+	return __atomic_load_n(&atom->v, __ATOMIC_ACQUIRE);
+}
+
+static inline void odpdrv_atomic_store_rel_u32(odpdrv_atomic_u32_t *atom,
+					       uint32_t val)
+{
+	__atomic_store_n(&atom->v, val, __ATOMIC_RELEASE);
+}
+
+static inline void odpdrv_atomic_add_rel_u32(odpdrv_atomic_u32_t *atom,
+					     uint32_t val)
+{
+	(void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELEASE);
+}
+
+static inline void odpdrv_atomic_sub_rel_u32(odpdrv_atomic_u32_t *atom,
+					     uint32_t val)
+{
+	(void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELEASE);
+}
+
+static inline int odpdrv_atomic_cas_acq_u32(odpdrv_atomic_u32_t *atom,
+					    uint32_t *old_val, uint32_t new_val)
+{
+	return __atomic_compare_exchange_n(&atom->v, old_val, new_val,
+					   0 /* strong */,
+					   __ATOMIC_ACQUIRE,
+					   __ATOMIC_RELAXED);
+}
+
+static inline int odpdrv_atomic_cas_rel_u32(odpdrv_atomic_u32_t *atom,
+					    uint32_t *old_val, uint32_t new_val)
+{
+	return __atomic_compare_exchange_n(&atom->v, old_val, new_val,
+					   0 /* strong */,
+					   __ATOMIC_RELEASE,
+					   __ATOMIC_RELAXED);
+}
+
+static inline int odpdrv_atomic_cas_acq_rel_u32(odpdrv_atomic_u32_t *atom,
+						uint32_t *old_val,
+						uint32_t new_val)
+{
+	return __atomic_compare_exchange_n(&atom->v, old_val, new_val,
+					   0 /* strong */,
+					   __ATOMIC_ACQ_REL,
+					   __ATOMIC_RELAXED);
+}
+
+static inline uint64_t odpdrv_atomic_load_acq_u64(odpdrv_atomic_u64_t *atom)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+	return ATOMIC_OP(atom, (void)0);
+#else
+	return __atomic_load_n(&atom->v, __ATOMIC_ACQUIRE);
+#endif
+}
+
+static inline void odpdrv_atomic_store_rel_u64(odpdrv_atomic_u64_t *atom,
+					       uint64_t val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+	(void)ATOMIC_OP(atom, atom->v = val);
+#else
+	__atomic_store_n(&atom->v, val, __ATOMIC_RELEASE);
+#endif
+}
+
+static inline void odpdrv_atomic_add_rel_u64(odpdrv_atomic_u64_t *atom,
+					     uint64_t val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+	(void)ATOMIC_OP(atom, atom->v += val);
+#else
+	(void)__atomic_fetch_add(&atom->v, val, __ATOMIC_RELEASE);
+#endif
+}
+
+static inline void odpdrv_atomic_sub_rel_u64(odpdrv_atomic_u64_t *atom,
+					     uint64_t val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+	(void)ATOMIC_OP(atom, atom->v -= val);
+#else
+	(void)__atomic_fetch_sub(&atom->v, val, __ATOMIC_RELEASE);
+#endif
+}
+
+static inline int odpdrv_atomic_cas_acq_u64(odpdrv_atomic_u64_t *atom,
+					    uint64_t *old_val, uint64_t new_val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+	int ret;
+	*old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val));
+	return ret;
+#else
+	return __atomic_compare_exchange_n(&atom->v, old_val, new_val,
+					   0 /* strong */,
+					   __ATOMIC_ACQUIRE,
+					   __ATOMIC_RELAXED);
+#endif
+}
+
+static inline int odpdrv_atomic_cas_rel_u64(odpdrv_atomic_u64_t *atom,
+					    uint64_t *old_val, uint64_t new_val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+	int ret;
+	*old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val));
+	return ret;
+#else
+	return __atomic_compare_exchange_n(&atom->v, old_val, new_val,
+					   0 /* strong */,
+					   __ATOMIC_RELEASE,
+					   __ATOMIC_RELAXED);
+#endif
+}
+
+static inline int odpdrv_atomic_cas_acq_rel_u64(odpdrv_atomic_u64_t *atom,
+						uint64_t *old_val,
+						uint64_t new_val)
+{
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+	int ret;
+	*old_val = ATOMIC_OP(atom, ATOMIC_CAS_OP(&ret, *old_val, new_val));
+	return ret;
+#else
+	return __atomic_compare_exchange_n(&atom->v, old_val, new_val,
+					   0 /* strong */,
+					   __ATOMIC_ACQ_REL,
+					   __ATOMIC_RELAXED);
+#endif
+}
+
+/**
+ * @}
+ */
+
+#include <odp/drv/spec/atomic.h>
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/platform/linux-generic/include/odp/drv/plat/atomic_types.h b/platform/linux-generic/include/odp/drv/plat/atomic_types.h
new file mode 100644
index 0000000..6a7ff0d
--- /dev/null
+++ b/platform/linux-generic/include/odp/drv/plat/atomic_types.h
@@ -0,0 +1,88 @@ 
+/* Copyright (c) 2016, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * ODPDRV atomic operations
+ */
+
+#ifndef ODPDRV_ATOMIC_TYPES_H_
+#define ODPDRV_ATOMIC_TYPES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <odp/drv/std_types.h>
+#include <odp/drv/align.h>
+
+/**
+ * @internal
+ * Atomic 64-bit unsigned integer
+ */
+struct odpdrv_atomic_u64_s {
+	uint64_t v; /**< Actual storage for the atomic variable */
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+	/* Some architectures do not support lock-free operations on 64-bit
+	 * data types. We use a spin lock to ensure atomicity. */
+	char lock; /**< Spin lock (if needed) used to ensure atomic access */
+#endif
+} ODPDRV_ALIGNED(sizeof(uint64_t)); /* Enforce alignement! */;
+
+/**
+ * @internal
+ * Atomic 32-bit unsigned integer
+ */
+struct odpdrv_atomic_u32_s {
+	uint32_t v; /**< Actual storage for the atomic variable */
+} ODPDRV_ALIGNED(sizeof(uint32_t)); /* Enforce alignement! */;
+
+#if __GCC_ATOMIC_LLONG_LOCK_FREE < 2
+
+/**
+ * @internal
+ * CAS operation expression for the ATOMIC_OP macro
+ */
+#define ATOMIC_CAS_OP(ret_ptr, old_val, new_val) \
+({ \
+	if (atom->v == (old_val)) { \
+		atom->v = (new_val); \
+		*(ret_ptr) = 1; \
+	} else { \
+		*(ret_ptr) = 0; \
+	} \
+})
+
+/**
+ * @internal
+ * Helper macro for lock-based atomic operations on 64-bit integers
+ * @param[in,out] atom Pointer to the 64-bit atomic variable
+ * @param expr Expression used update the variable.
+ * @return The old value of the variable.
+ */
+#define ATOMIC_OP(atom, expr) \
+({ \
+	uint64_t _old_val; \
+	/* Loop while lock is already taken, stop when lock becomes clear */ \
+	while (__atomic_test_and_set(&(atom)->lock, __ATOMIC_ACQUIRE)) \
+		(void)0; \
+	_old_val = (atom)->v; \
+	(expr); /* Perform whatever update is desired */ \
+	__atomic_clear(&(atom)->lock, __ATOMIC_RELEASE); \
+	_old_val; /* Return old value */ \
+})
+#endif
+
+typedef struct odpdrv_atomic_u64_s odpdrv_atomic_u64_t;
+
+typedef struct odpdrv_atomic_u32_s odpdrv_atomic_u32_t;
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif