@@ -19,20 +19,13 @@ extern "C" {
#endif
-
#include <odp_std_types.h>
+#include <odp_platform_types.h>
-
-
-
-
-/**
- * ODP buffer
+/** @defgroup odp_buffer ODP BUFFER
+ * Operations on a buffer.
+ * @{
*/
-typedef unsigned long odp_buffer_t;
-
-
-#define ODP_BUFFER_INVALID (unsigned long)(-1L) /**< Invalid buffer */
/**
@@ -74,11 +67,21 @@ int odp_buffer_type(odp_buffer_t buf);
*
* @param buf Buffer handle
*
- * @return 1 if valid, otherwise 0
+ * @retval 1 Buffer handle represents a valid buffer.
+ * @retval 0 Buffer handle does not represent a valid buffer.
*/
int odp_buffer_is_valid(odp_buffer_t buf);
/**
+ * Buffer pool of the buffer
+ *
+ * @param buf Buffer handle
+ *
+ * @return Handle of buffer pool buffer belongs to
+ */
+odp_buffer_pool_t odp_buffer_pool(odp_buffer_t buf);
+
+/**
* Print buffer metadata to STDOUT
*
* @param buf Buffer handle
@@ -86,6 +89,9 @@ int odp_buffer_is_valid(odp_buffer_t buf);
*/
void odp_buffer_print(odp_buffer_t buf);
+/**
+ * @}
+ */
#ifdef __cplusplus
}
@@ -21,66 +21,143 @@ extern "C" {
#include <odp_std_types.h>
+#include <odp_platform_types.h>
#include <odp_buffer.h>
+/** @addtogroup odp_buffer
+ * Operations on a buffer pool.
+ * @{
+ */
+
/** Maximum queue name lenght in chars */
#define ODP_BUFFER_POOL_NAME_LEN 32
-/** Invalid buffer pool */
-#define ODP_BUFFER_POOL_INVALID (unsigned long)(-1L)
-
-/** ODP buffer pool */
-typedef unsigned long odp_buffer_pool_t;
-
+/**
+ * Buffer pool parameters
+ * Used to communicate buffer pool creation options.
+ */
+typedef struct odp_buffer_pool_param_t {
+ uint32_t buf_size; /**< Buffer size in bytes. The maximum
+ number of bytes application will
+ store in each buffer. For packets, this
+ is the maximum packet data length, and
+ configured headroom and tailroom will be
+ added to this number */
+ uint32_t buf_align; /**< Minimum buffer alignment in bytes.
+ Valid values are powers of two. Use 0
+ for default alignment. Default will
+ always be a multiple of 8. */
+ uint32_t num_bufs; /**< Number of buffers in the pool */
+ int buf_type; /**< Buffer type */
+} odp_buffer_pool_param_t;
/**
* Create a buffer pool
+ * This routine is used to create a buffer pool. It take three
+ * arguments: the optional name of the pool to be created, an optional shared
+ * memory handle, and a parameter struct that describes the pool to be
+ * created. If a name is not specified the result is an anonymous pool that
+ * cannot be referenced by odp_buffer_pool_lookup().
+ *
+ * @param name Name of the pool, max ODP_BUFFER_POOL_NAME_LEN-1 chars.
+ * May be specified as NULL for anonymous pools.
+ *
+ * @param shm The shared memory object in which to create the pool.
+ * Use ODP_SHM_NULL to reserve default memory type
+ * for the buffer type.
*
- * @param name Name of the pool (max ODP_BUFFER_POOL_NAME_LEN - 1 chars)
- * @param base_addr Pool base address
- * @param size Pool size in bytes
- * @param buf_size Buffer size in bytes
- * @param buf_align Minimum buffer alignment
- * @param buf_type Buffer type
+ * @param params Buffer pool parameters.
*
- * @return Buffer pool handle
+ * @return Handle of the created buffer pool
+ * @retval ODP_BUFFER_POOL_INVALID Buffer pool could not be created
*/
+
odp_buffer_pool_t odp_buffer_pool_create(const char *name,
- void *base_addr, uint64_t size,
- size_t buf_size, size_t buf_align,
- int buf_type);
+ odp_shm_t shm,
+ odp_buffer_pool_param_t *params);
+/**
+ * Destroy a buffer pool previously created by odp_buffer_pool_create()
+ *
+ * @param pool Handle of the buffer pool to be destroyed
+ *
+ * @retval 0 Success
+ * @retval -1 Failure
+ *
+ * @note This routine destroys a previously created buffer pool. This call
+ * does not destroy any shared memory object passed to
+ * odp_buffer_pool_create() used to store the buffer pool contents. The caller
+ * takes responsibility for that. If no shared memory object was passed as
+ * part of the create call, then this routine will destroy any internal shared
+ * memory objects associated with the buffer pool. Results are undefined if
+ * an attempt is made to destroy a buffer pool that contains allocated or
+ * otherwise active buffers.
+ */
+int odp_buffer_pool_destroy(odp_buffer_pool_t pool);
/**
* Find a buffer pool by name
*
* @param name Name of the pool
*
- * @return Buffer pool handle, or ODP_BUFFER_POOL_INVALID if not found.
+ * @return Handle of found buffer pool
+ * @retval ODP_BUFFER_POOL_INVALID Buffer pool could not be found
+ *
+ * @note This routine cannot be used to look up an anonymous pool (one created
+ * with no name).
*/
odp_buffer_pool_t odp_buffer_pool_lookup(const char *name);
+/**
+ * Buffer pool information struct
+ * Used to get information about a buffer pool.
+ */
+typedef struct odp_buffer_pool_info_t {
+ const char *name; /**< pool name */
+ odp_shm_t shm; /**< handle of shared memory area
+ supplied by application to
+ contain buffer pool, or
+ ODP_SHM_NULL if this pool is
+ managed by ODP */
+ odp_buffer_pool_param_t params; /**< pool parameters */
+} odp_buffer_pool_info_t;
+
+/**
+ * Retrieve information about a buffer pool
+ *
+ * @param pool Buffer pool handle
+ *
+ * @param[out] info Receives an odp_buffer_pool_info_t object
+ * that describes the pool.
+ *
+ * @retval 0 Success
+ * @retval -1 Failure. Info could not be retrieved.
+ */
+
+int odp_buffer_pool_info(odp_buffer_pool_t pool,
+ odp_buffer_pool_info_t *info);
/**
* Print buffer pool info
*
* @param pool Pool handle
*
+ * @note This routine writes implementation-defined information about the
+ * specified buffer pool to the ODP log. The intended use is for debugging.
*/
void odp_buffer_pool_print(odp_buffer_pool_t pool);
-
-
/**
* Buffer alloc
*
+ * The validity of a buffer can be cheked at any time with odp_buffer_is_valid()
* @param pool Pool handle
*
- * @return Buffer handle or ODP_BUFFER_INVALID
+ * @return Handle of allocated buffer
+ * @retval ODP_BUFFER_INVALID Buffer could not be allocated
*/
odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool);
-
/**
* Buffer free
*
@@ -89,8 +166,9 @@ odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool);
*/
void odp_buffer_free(odp_buffer_t buf);
-
-
+/**
+ * @}
+ */
#ifdef __cplusplus
}
@@ -24,50 +24,144 @@ extern "C" {
#include <odp_buffer.h>
#include <odp_debug.h>
#include <odp_align.h>
-#include <rte_mbuf.h>
+#include <odp_align_internal.h>
+#include <odp_config.h>
+#include <odp_byteorder.h>
+#include <odp_thread.h>
-/* TODO: move these to correct files */
+/* DPDK */
+#include <rte_mbuf.h>
-typedef uint64_t odp_phys_addr_t;
+#define ODP_BITSIZE(x) \
+ ((x) <= 2 ? 1 : \
+ ((x) <= 4 ? 2 : \
+ ((x) <= 8 ? 3 : \
+ ((x) <= 16 ? 4 : \
+ ((x) <= 32 ? 5 : \
+ ((x) <= 64 ? 6 : \
+ ((x) <= 128 ? 7 : \
+ ((x) <= 256 ? 8 : \
+ ((x) <= 512 ? 9 : \
+ ((x) <= 1024 ? 10 : \
+ ((x) <= 2048 ? 11 : \
+ ((x) <= 4096 ? 12 : \
+ ((x) <= 8196 ? 13 : \
+ ((x) <= 16384 ? 14 : \
+ ((x) <= 32768 ? 15 : \
+ ((x) <= 65536 ? 16 : \
+ (0/0)))))))))))))))))
+
+_ODP_STATIC_ASSERT(ODP_CONFIG_PACKET_BUF_LEN_MIN >= 256,
+ "ODP Segment size must be a minimum of 256 bytes");
+
+_ODP_STATIC_ASSERT((ODP_CONFIG_PACKET_BUF_LEN_MIN % ODP_CACHE_LINE_SIZE) == 0,
+ "ODP Segment size must be a multiple of cache line size");
+
+_ODP_STATIC_ASSERT((ODP_CONFIG_PACKET_BUF_LEN_MAX %
+ ODP_CONFIG_PACKET_BUF_LEN_MIN) == 0,
+ "Packet max size must be a multiple of segment size");
+
+#define ODP_BUFFER_MAX_SEG \
+ (ODP_CONFIG_PACKET_BUF_LEN_MAX / ODP_CONFIG_PACKET_BUF_LEN_MIN)
+
+/* We can optimize storage of small raw buffers within metadata area */
+#define ODP_MAX_INLINE_BUF ((sizeof(void *)) * (ODP_BUFFER_MAX_SEG - 1))
+
+#define ODP_BUFFER_POOL_BITS ODP_BITSIZE(ODP_CONFIG_BUFFER_POOLS)
+#define ODP_BUFFER_SEG_BITS ODP_BITSIZE(ODP_BUFFER_MAX_SEG)
+#define ODP_BUFFER_INDEX_BITS (32 - ODP_BUFFER_POOL_BITS - ODP_BUFFER_SEG_BITS)
+#define ODP_BUFFER_PREFIX_BITS (ODP_BUFFER_POOL_BITS + ODP_BUFFER_INDEX_BITS)
+#define ODP_BUFFER_MAX_POOLS (1 << ODP_BUFFER_POOL_BITS)
+#define ODP_BUFFER_MAX_BUFFERS (1 << ODP_BUFFER_INDEX_BITS)
#define ODP_BUFFER_MAX_INDEX (ODP_BUFFER_MAX_BUFFERS - 2)
#define ODP_BUFFER_INVALID_INDEX (ODP_BUFFER_MAX_BUFFERS - 1)
-#define ODP_BUFS_PER_CHUNK 16
-#define ODP_BUFS_PER_SCATTER 4
-
-#define ODP_BUFFER_TYPE_CHUNK 0xffff
-
-
-#define ODP_BUFFER_POOL_BITS 4
-#define ODP_BUFFER_INDEX_BITS (32 - ODP_BUFFER_POOL_BITS)
-#define ODP_BUFFER_MAX_POOLS (1 << ODP_BUFFER_POOL_BITS)
-#define ODP_BUFFER_MAX_BUFFERS (1 << ODP_BUFFER_INDEX_BITS)
-
typedef union odp_buffer_bits_t {
uint32_t u32;
odp_buffer_t handle;
struct {
- uint32_t pool:ODP_BUFFER_POOL_BITS;
+#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN
+ uint32_t pool_id:ODP_BUFFER_POOL_BITS;
+ uint32_t index:ODP_BUFFER_INDEX_BITS;
+ uint32_t seg:ODP_BUFFER_SEG_BITS;
+#else
+ uint32_t seg:ODP_BUFFER_SEG_BITS;
uint32_t index:ODP_BUFFER_INDEX_BITS;
+ uint32_t pool_id:ODP_BUFFER_POOL_BITS;
+#endif
};
-} odp_buffer_bits_t;
+ struct {
+#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN
+ uint32_t prefix:ODP_BUFFER_PREFIX_BITS;
+ uint32_t pfxseg:ODP_BUFFER_SEG_BITS;
+#else
+ uint32_t pfxseg:ODP_BUFFER_SEG_BITS;
+ uint32_t prefix:ODP_BUFFER_PREFIX_BITS;
+#endif
+ };
+} odp_buffer_bits_t;
/* forward declaration */
struct odp_buffer_hdr_t;
-
+/* Common buffer header */
typedef struct odp_buffer_hdr_t {
+ /* DPDK specific */
struct rte_mbuf mb; /* Underlying DPDK rte_mbuf */
- struct odp_buffer_hdr_t *next; /* Next buf in a list */
- int type; /* ODP buffer type; not DPDK buf type */
uint32_t index; /* Index in the rte_mempool */
+
+ struct odp_buffer_hdr_t *next; /* next buf in a list */
+ int allocator; /* allocating thread id */
+ odp_buffer_bits_t handle; /* handle */
+ union {
+ uint32_t all;
+ struct {
+ uint32_t zeroized:1; /* Zeroize buf data on free */
+ uint32_t hdrdata:1; /* Data is in buffer hdr */
+ };
+ } flags;
+ int type; /* buffer type */
+ size_t size; /* max data size */
+ odp_atomic_u32_t ref_count; /* reference count */
+ odp_buffer_pool_t pool_hdl; /* buffer pool handle */
+ union {
+ uint64_t buf_u64; /* user u64 */
+ void *buf_ctx; /* user context */
+ const void *buf_cctx; /* const alias for ctx */
+ void *udata_addr; /* user metadata addr */
+ };
+ size_t udata_size; /* size of user metadata */
+ uint32_t segcount; /* segment count */
+ uint32_t segsize; /* segment size */
+ void *addr[ODP_BUFFER_MAX_SEG]; /* block addrs */
+
} odp_buffer_hdr_t;
-int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf);
+typedef struct odp_buffer_hdr_stride {
+ uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_buffer_hdr_t))];
+} odp_buffer_hdr_stride;
+/* Ensure next header starts from 8 byte align */
+_ODP_STATIC_ASSERT((sizeof(odp_buffer_hdr_t) % 8) == 0,
+ "ODP_BUFFER_HDR_T__SIZE_ERROR");
+
+typedef struct odp_buf_blk_t {
+ struct odp_buf_blk_t *next;
+ struct odp_buf_blk_t *prev;
+} odp_buf_blk_t;
+
+/* Raw buffer header */
+typedef struct {
+ odp_buffer_hdr_t buf_hdr; /* common buffer header */
+} odp_raw_buffer_hdr_t;
+
+/* Free buffer marker */
+#define ODP_FREEBUF -1
+/* Forward declarations */
+odp_buffer_t buffer_alloc(odp_buffer_pool_t pool, size_t size);
#ifdef __cplusplus
}
@@ -19,16 +19,48 @@ extern "C" {
#endif
#include <odp_std_types.h>
+#include <odp_align.h>
+#include <odp_align_internal.h>
#include <odp_buffer_pool.h>
#include <odp_buffer_internal.h>
-#include <odp_align.h>
#include <odp_hints.h>
#include <odp_config.h>
#include <odp_debug.h>
+#include <odp_shared_memory.h>
+#include <odp_atomic.h>
+#include <odp_atomic_internal.h>
+#include <string.h>
-/* for DPDK */
+/* DPDK */
#include <rte_mempool.h>
+/**
+ * Buffer initialization routine prototype
+ *
+ * @note Routines of this type MAY be passed as part of the
+ * _odp_buffer_pool_init_t structure to be called whenever a
+ * buffer is allocated to initialize the user metadata
+ * associated with that buffer.
+ */
+typedef void (_odp_buf_init_t)(odp_buffer_t buf, void *buf_init_arg);
+
+/**
+ * Buffer pool initialization parameters
+ * Used to communicate buffer pool initialization options. Internal for now.
+ */
+typedef struct _odp_buffer_pool_init_t {
+ size_t udata_size; /**< Size of user metadata for each buffer */
+ _odp_buf_init_t *buf_init; /**< Buffer initialization routine to use */
+ void *buf_init_arg; /**< Argument to be passed to buf_init() */
+} _odp_buffer_pool_init_t; /**< Type of buffer initialization struct */
+
+/* Local cache for buffer alloc/free acceleration */
+typedef struct local_cache_t {
+ odp_buffer_hdr_t *buf_freelist; /* The local cache */
+ uint64_t bufallocs; /* Local buffer alloc count */
+ uint64_t buffrees; /* Local buffer free count */
+} local_cache_t;
+
/* Use ticketlock instead of spinlock */
#define POOL_USE_TICKETLOCK
@@ -38,11 +70,16 @@ extern "C" {
#ifdef POOL_USE_TICKETLOCK
#include <odp_ticketlock.h>
+#define POOL_LOCK(a) odp_ticketlock_lock(a)
+#define POOL_UNLOCK(a) odp_ticketlock_unlock(a)
+#define POOL_LOCK_INIT(a) odp_ticketlock_init(a)
#else
#include <odp_spinlock.h>
+#define POOL_LOCK(a) odp_spinlock_lock(a)
+#define POOL_UNLOCK(a) odp_spinlock_unlock(a)
+#define POOL_LOCK_INIT(a) odp_spinlock_init(a)
#endif
-
struct pool_entry_s {
#ifdef POOL_USE_TICKETLOCK
odp_ticketlock_t lock ODP_ALIGNED_CACHE;
@@ -50,38 +87,294 @@ struct pool_entry_s {
odp_spinlock_t lock ODP_ALIGNED_CACHE;
#endif
- uint64_t free_bufs;
char name[ODP_BUFFER_POOL_NAME_LEN];
-
-
- odp_buffer_pool_t pool ODP_ALIGNED_CACHE;
- uintptr_t buf_base;
- size_t buf_size;
- size_t buf_offset;
- uint64_t num_bufs;
- void *pool_base_addr;
- uint64_t pool_size;
- size_t payload_size;
- size_t payload_align;
- int buf_type;
- size_t hdr_size;
+ odp_buffer_pool_param_t params;
+ _odp_buffer_pool_init_t init_params;
+ odp_buffer_pool_t pool_hdl;
+ uint32_t pool_id;
+ odp_shm_t pool_shm;
+ union {
+ uint32_t all;
+ struct {
+ uint32_t has_name:1;
+ uint32_t user_supplied_shm:1;
+ uint32_t unsegmented:1;
+ uint32_t zeroized:1;
+ uint32_t predefined:1;
+ };
+ } flags;
+ uint32_t quiesced;
+ uint32_t low_wm_assert;
+ uint8_t *pool_base_addr;
+ uint8_t *pool_mdata_addr;
+ size_t pool_size;
+ uint32_t buf_align;
+ uint32_t buf_stride;
+ _odp_atomic_ptr_t buf_freelist;
+ _odp_atomic_ptr_t blk_freelist;
+ odp_atomic_u32_t bufcount;
+ odp_atomic_u32_t blkcount;
+ odp_atomic_u64_t bufallocs;
+ odp_atomic_u64_t buffrees;
+ odp_atomic_u64_t blkallocs;
+ odp_atomic_u64_t blkfrees;
+ odp_atomic_u64_t bufempty;
+ odp_atomic_u64_t blkempty;
+ odp_atomic_u64_t high_wm_count;
+ odp_atomic_u64_t low_wm_count;
+ uint32_t seg_size;
+ uint32_t high_wm;
+ uint32_t low_wm;
+ uint32_t headroom;
+ uint32_t tailroom;
};
+typedef union pool_entry_u {
+ struct pool_entry_s s;
+
+ uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct pool_entry_s))];
+} pool_entry_t;
extern void *pool_entry_ptr[];
+#if defined(ODP_CONFIG_SECURE_POOLS) && (ODP_CONFIG_SECURE_POOLS == 1)
+#define buffer_is_secure(buf) (buf->flags.zeroized)
+#define pool_is_secure(pool) (pool->flags.zeroized)
+#else
+#define buffer_is_secure(buf) 0
+#define pool_is_secure(pool) 0
+#endif
+
+#define TAG_ALIGN ((size_t)16)
+
+#define odp_cs(ptr, old, new) \
+ _odp_atomic_ptr_cmp_xchg_strong(&ptr, (void **)&old, (void *)new, \
+ _ODP_MEMMODEL_SC, \
+ _ODP_MEMMODEL_SC)
-static inline void *get_pool_entry(odp_buffer_pool_t pool_id)
+/* Helper functions for pointer tagging to avoid ABA race conditions */
+#define odp_tag(ptr) \
+ (((size_t)ptr) & (TAG_ALIGN - 1))
+
+#define odp_detag(ptr) \
+ ((void *)(((size_t)ptr) & -TAG_ALIGN))
+
+#define odp_retag(ptr, tag) \
+ ((void *)(((size_t)ptr) | odp_tag(tag)))
+
+
+static inline void *get_blk(struct pool_entry_s *pool)
+{
+ void *oldhead, *myhead, *newhead;
+
+ oldhead = _odp_atomic_ptr_load(&pool->blk_freelist, _ODP_MEMMODEL_ACQ);
+
+ do {
+ size_t tag = odp_tag(oldhead);
+ myhead = odp_detag(oldhead);
+ if (odp_unlikely(myhead == NULL))
+ break;
+ newhead = odp_retag(((odp_buf_blk_t *)myhead)->next, tag + 1);
+ } while (odp_cs(pool->blk_freelist, oldhead, newhead) == 0);
+
+ if (odp_unlikely(myhead == NULL))
+ odp_atomic_inc_u64(&pool->blkempty);
+ else
+ odp_atomic_dec_u32(&pool->blkcount);
+
+ return (void *)myhead;
+}
+
+static inline void ret_blk(struct pool_entry_s *pool, void *block)
+{
+ void *oldhead, *myhead, *myblock;
+
+ oldhead = _odp_atomic_ptr_load(&pool->blk_freelist, _ODP_MEMMODEL_ACQ);
+
+ do {
+ size_t tag = odp_tag(oldhead);
+ myhead = odp_detag(oldhead);
+ ((odp_buf_blk_t *)block)->next = myhead;
+ myblock = odp_retag(block, tag + 1);
+ } while (odp_cs(pool->blk_freelist, oldhead, myblock) == 0);
+
+ odp_atomic_inc_u32(&pool->blkcount);
+ odp_atomic_inc_u64(&pool->blkfrees);
+}
+
+static inline odp_buffer_hdr_t *get_buf(struct pool_entry_s *pool)
+{
+ odp_buffer_hdr_t *oldhead, *myhead, *newhead;
+
+ oldhead = _odp_atomic_ptr_load(&pool->buf_freelist, _ODP_MEMMODEL_ACQ);
+
+ do {
+ size_t tag = odp_tag(oldhead);
+ myhead = odp_detag(oldhead);
+ if (odp_unlikely(myhead == NULL))
+ break;
+ newhead = odp_retag(myhead->next, tag + 1);
+ } while (odp_cs(pool->buf_freelist, oldhead, newhead) == 0);
+
+ if (odp_unlikely(myhead == NULL)) {
+ odp_atomic_inc_u64(&pool->bufempty);
+ } else {
+ uint64_t bufcount =
+ odp_atomic_fetch_sub_u32(&pool->bufcount, 1) - 1;
+
+ /* Check for low watermark condition */
+ if (bufcount == pool->low_wm && !pool->low_wm_assert) {
+ pool->low_wm_assert = 1;
+ odp_atomic_inc_u64(&pool->low_wm_count);
+ }
+
+ odp_atomic_inc_u64(&pool->bufallocs);
+ myhead->next = myhead; /* Mark buffer allocated */
+ myhead->allocator = odp_thread_id();
+ }
+
+ return (void *)myhead;
+}
+
+static inline void ret_buf(struct pool_entry_s *pool, odp_buffer_hdr_t *buf)
+{
+ odp_buffer_hdr_t *oldhead, *myhead, *mybuf;
+
+ buf->allocator = ODP_FREEBUF; /* Mark buffer free */
+
+ if (!buf->flags.hdrdata && buf->type != ODP_BUFFER_TYPE_RAW) {
+ while (buf->segcount > 0) {
+ if (buffer_is_secure(buf) || pool_is_secure(pool))
+ memset(buf->addr[buf->segcount - 1],
+ 0, buf->segsize);
+ ret_blk(pool, buf->addr[--buf->segcount]);
+ }
+ buf->size = 0;
+ }
+
+ oldhead = _odp_atomic_ptr_load(&pool->buf_freelist, _ODP_MEMMODEL_ACQ);
+
+ do {
+ size_t tag = odp_tag(oldhead);
+ myhead = odp_detag(oldhead);
+ buf->next = myhead;
+ mybuf = odp_retag(buf, tag + 1);
+ } while (odp_cs(pool->buf_freelist, oldhead, mybuf) == 0);
+
+ uint64_t bufcount = odp_atomic_fetch_add_u32(&pool->bufcount, 1) + 1;
+
+ /* Check if low watermark condition should be deasserted */
+ if (bufcount == pool->high_wm && pool->low_wm_assert) {
+ pool->low_wm_assert = 0;
+ odp_atomic_inc_u64(&pool->high_wm_count);
+ }
+
+ odp_atomic_inc_u64(&pool->buffrees);
+}
+
+static inline void *get_local_buf(local_cache_t *buf_cache,
+ struct pool_entry_s *pool,
+ size_t totsize)
+{
+ odp_buffer_hdr_t *buf = buf_cache->buf_freelist;
+
+ if (odp_likely(buf != NULL)) {
+ buf_cache->buf_freelist = buf->next;
+
+ if (odp_unlikely(buf->size < totsize)) {
+ intmax_t needed = totsize - buf->size;
+
+ do {
+ void *blk = get_blk(pool);
+ if (odp_unlikely(blk == NULL)) {
+ ret_buf(pool, buf);
+ buf_cache->buffrees--;
+ return NULL;
+ }
+ buf->addr[buf->segcount++] = blk;
+ needed -= pool->seg_size;
+ } while (needed > 0);
+
+ buf->size = buf->segcount * pool->seg_size;
+ }
+
+ buf_cache->bufallocs++;
+ buf->allocator = odp_thread_id(); /* Mark buffer allocated */
+ }
+
+ return buf;
+}
+
+static inline void ret_local_buf(local_cache_t *buf_cache,
+ odp_buffer_hdr_t *buf)
+{
+ buf->allocator = ODP_FREEBUF;
+ buf->next = buf_cache->buf_freelist;
+ buf_cache->buf_freelist = buf;
+
+ buf_cache->buffrees++;
+}
+
+static inline void flush_cache(local_cache_t *buf_cache,
+ struct pool_entry_s *pool)
+{
+ odp_buffer_hdr_t *buf = buf_cache->buf_freelist;
+ uint32_t flush_count = 0;
+
+ while (buf != NULL) {
+ odp_buffer_hdr_t *next = buf->next;
+ ret_buf(pool, buf);
+ buf = next;
+ flush_count++;
+ }
+
+ odp_atomic_add_u64(&pool->bufallocs, buf_cache->bufallocs);
+ odp_atomic_add_u64(&pool->buffrees, buf_cache->buffrees - flush_count);
+
+ buf_cache->buf_freelist = NULL;
+ buf_cache->bufallocs = 0;
+ buf_cache->buffrees = 0;
+}
+
+static inline odp_buffer_pool_t pool_index_to_handle(uint32_t pool_id)
+{
+ return pool_id;
+}
+
+static inline uint32_t pool_handle_to_index(odp_buffer_pool_t pool_hdl)
+{
+ return pool_hdl;
+}
+
+static inline void *get_pool_entry(uint32_t pool_id)
{
return pool_entry_ptr[pool_id];
}
+static inline pool_entry_t *odp_pool_to_entry(odp_buffer_pool_t pool)
+{
+ return (pool_entry_t *)get_pool_entry(pool_handle_to_index(pool));
+}
+
+static inline pool_entry_t *odp_buf_to_pool(odp_buffer_hdr_t *buf)
+{
+ return odp_pool_to_entry(buf->pool_hdl);
+}
-static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf)
+static inline uint32_t odp_buffer_pool_segment_size(odp_buffer_pool_t pool)
{
- return (odp_buffer_hdr_t *)buf;
+ return odp_pool_to_entry(pool)->s.seg_size;
}
+static inline uint32_t odp_buffer_pool_headroom(odp_buffer_pool_t pool)
+{
+ return odp_pool_to_entry(pool)->s.headroom;
+}
+
+static inline uint32_t odp_buffer_pool_tailroom(odp_buffer_pool_t pool)
+{
+ return odp_pool_to_entry(pool)->s.tailroom;
+}
#ifdef __cplusplus
}
@@ -5,8 +5,10 @@
*/
#include <odp_buffer.h>
-#include <odp_buffer_internal.h>
#include <odp_buffer_pool_internal.h>
+#include <odp_buffer_internal.h>
+#include <odp_buffer_inlines.h>
+#include <odp_debug_internal.h>
#include <string.h>
#include <stdio.h>
@@ -44,13 +46,13 @@ int odp_buffer_is_valid(odp_buffer_t buf)
}
-int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf)
+int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf)
{
odp_buffer_hdr_t *hdr;
int len = 0;
if (!odp_buffer_is_valid(buf)) {
- printf("Buffer is not valid.\n");
+ ODP_PRINT("Buffer is not valid.\n");
return len;
}
@@ -67,7 +69,9 @@ int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf)
len += snprintf(&str[len], n-len,
" size %u\n", hdr->mb.buf_len);
len += snprintf(&str[len], n-len,
- " ref_count %i\n", hdr->mb.refcnt);
+ " ref_count %i\n",
+ odp_atomic_load_u32((odp_atomic_u32_t *)
+ &hdr->mb.refcnt));
len += snprintf(&str[len], n-len,
" dpdk type %i\n", hdr->mb.type);
len += snprintf(&str[len], n-len,
@@ -86,5 +90,5 @@ void odp_buffer_print(odp_buffer_t buf)
len = odp_buffer_snprint(str, max_len-1, buf);
str[len] = 0;
- printf("\n%s\n", str);
+ ODP_PRINT("\n%s\n", str);
}
@@ -6,16 +6,19 @@
#include <odp_std_types.h>
#include <odp_buffer_pool.h>
-#include <odp_buffer_pool_internal.h>
#include <odp_buffer_internal.h>
+#include <odp_buffer_pool_internal.h>
+#include <odp_buffer_inlines.h>
#include <odp_packet_internal.h>
#include <odp_timer_internal.h>
+#include <odp_align_internal.h>
#include <odp_shared_memory.h>
#include <odp_align.h>
#include <odp_internal.h>
#include <odp_config.h>
#include <odp_hints.h>
-#include <odp_debug.h>
+#include <odp_debug_internal.h>
+#include <odp_atomic_internal.h>
#include <string.h>
#include <stdlib.h>
@@ -26,44 +29,27 @@
#define MBUF_SIZE (2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM)
#define NB_MBUF 32768
-#ifdef POOL_USE_TICKETLOCK
-#include <odp_ticketlock.h>
-#define LOCK(a) odp_ticketlock_lock(a)
-#define UNLOCK(a) odp_ticketlock_unlock(a)
-#define LOCK_INIT(a) odp_ticketlock_init(a)
-#else
-#include <odp_spinlock.h>
-#define LOCK(a) odp_spinlock_lock(a)
-#define UNLOCK(a) odp_spinlock_unlock(a)
-#define LOCK_INIT(a) odp_spinlock_init(a)
-#endif
-
-
#if ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS
#error ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS
#endif
-#define NULL_INDEX ((uint32_t)-1)
-
-union buffer_type_any_u {
+typedef union buffer_type_any_u {
odp_buffer_hdr_t buf;
odp_packet_hdr_t pkt;
odp_timeout_hdr_t tmo;
-};
-
-typedef union buffer_type_any_u odp_any_buffer_hdr_t;
+} odp_anybuf_t;
-typedef union pool_entry_u {
- struct pool_entry_s s;
-
- uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct pool_entry_s))];
-
-} pool_entry_t;
+/* Any buffer type header */
+typedef struct {
+ odp_anybuf_t any_hdr; /* any buffer type */
+} odp_any_buffer_hdr_t;
+typedef struct odp_any_hdr_stride {
+ uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_any_buffer_hdr_t))];
+} odp_any_hdr_stride;
typedef struct pool_table_t {
pool_entry_t pool[ODP_CONFIG_BUFFER_POOLS];
-
} pool_table_t;
@@ -73,6 +59,8 @@ static pool_table_t *pool_tbl;
/* Pool entry pointers (for inlining) */
void *pool_entry_ptr[ODP_CONFIG_BUFFER_POOLS];
+/* Local cache for buffer alloc/free acceleration */
+static __thread local_cache_t local_cache[ODP_CONFIG_BUFFER_POOLS];
int odp_buffer_pool_init_global(void)
{
@@ -94,9 +82,9 @@ int odp_buffer_pool_init_global(void)
for (i = 0; i < ODP_CONFIG_BUFFER_POOLS; i++) {
/* init locks */
pool_entry_t *pool = &pool_tbl->pool[i];
- LOCK_INIT(&pool->s.lock);
- pool->s.pool = i;
-
+ POOL_LOCK_INIT(&pool->s.lock);
+ pool->s.pool_hdl = pool_index_to_handle(i);
+ pool->s.pool_id = i;
pool_entry_ptr[i] = pool;
}
@@ -187,10 +175,13 @@ odp_dpdk_mbuf_ctor(struct rte_mempool *mp,
buf_hdr->index = i;
}
+/**
+ * Buffer pool creation
+ */
+
odp_buffer_pool_t odp_buffer_pool_create(const char *name,
- void *base_addr, uint64_t size,
- size_t buf_size, size_t buf_align,
- int buf_type)
+ odp_shm_t shm,
+ odp_buffer_pool_param_t *params)
{
struct rte_mempool *pool = NULL;
struct mbuf_pool_ctor_arg mbp_ctor_arg;
@@ -198,47 +189,39 @@ odp_buffer_pool_t odp_buffer_pool_create(const char *name,
unsigned mb_size;
size_t hdr_size;
- /* Not used for rte_mempool; the new ODP buffer management introduces
- * rte_mempool_create_from_region where base_addr makes sense */
- (void)base_addr;
-
- /* buf_align will be removed soon, no need to wory about it */
- (void)buf_align;
-
- ODP_DBG("odp_buffer_pool_create: %s, %lx, %u, %u, %u, %d\n", name,
- (uint64_t) base_addr, (unsigned) size,
- (unsigned) buf_size, (unsigned) buf_align,
- buf_type);
+ ODP_DBG("odp_buffer_pool_create: %s, %lx, %u, %u, %d\n", name,
+ (uint64_t) shm, (unsigned) params->buf_size,
+ (unsigned) params->buf_align, params->buf_type);
- switch (buf_type) {
+ switch (params->buf_type) {
case ODP_BUFFER_TYPE_RAW:
hdr_size = sizeof(odp_buffer_hdr_t);
- mbp_ctor_arg.seg_buf_size = (uint16_t) buf_size;
+ mbp_ctor_arg.seg_buf_size = (uint16_t) params->buf_size;
break;
case ODP_BUFFER_TYPE_PACKET:
hdr_size = sizeof(odp_packet_hdr_t);
mbp_ctor_arg.seg_buf_size =
- (uint16_t) (RTE_PKTMBUF_HEADROOM + buf_size);
+ (uint16_t) (RTE_PKTMBUF_HEADROOM + params->buf_size);
break;
case ODP_BUFFER_TYPE_TIMEOUT:
hdr_size = sizeof(odp_timeout_hdr_t);
- mbp_ctor_arg.seg_buf_size = (uint16_t) buf_size;
+ mbp_ctor_arg.seg_buf_size = (uint16_t) params->buf_size;
break;
case ODP_BUFFER_TYPE_ANY:
hdr_size = sizeof(odp_any_buffer_hdr_t);
mbp_ctor_arg.seg_buf_size =
- (uint16_t) (RTE_PKTMBUF_HEADROOM + buf_size);
+ (uint16_t) (RTE_PKTMBUF_HEADROOM + params->buf_size);
break;
default:
- ODP_ERR("odp_buffer_pool_create: Bad type %i\n", buf_type);
+ ODP_ERR("odp_buffer_pool_create: Bad type %i\n",
+ params->buf_type);
return ODP_BUFFER_POOL_INVALID;
- break;
}
mb_ctor_arg.seg_buf_offset =
(uint16_t) ODP_CACHE_LINE_SIZE_ROUNDUP(hdr_size);
mb_ctor_arg.seg_buf_size = mbp_ctor_arg.seg_buf_size;
- mb_ctor_arg.buf_type = buf_type;
+ mb_ctor_arg.buf_type = params->buf_type;
mb_size = mb_ctor_arg.seg_buf_offset + mb_ctor_arg.seg_buf_size;
pool = rte_mempool_create(name, NB_MBUF,
@@ -267,6 +250,61 @@ odp_buffer_pool_t odp_buffer_pool_lookup(const char *name)
return (odp_buffer_pool_t)mp;
}
+odp_buffer_t buffer_alloc(odp_buffer_pool_t pool_hdl, size_t size)
+{
+ uint32_t pool_id = pool_handle_to_index(pool_hdl);
+ pool_entry_t *pool = get_pool_entry(pool_id);
+ uintmax_t totsize = pool->s.headroom + size + pool->s.tailroom;
+ odp_anybuf_t *buf;
+
+ /* Reject oversized allocation requests */
+ if ((pool->s.flags.unsegmented && totsize > pool->s.seg_size) ||
+ (!pool->s.flags.unsegmented &&
+ totsize > ODP_CONFIG_PACKET_BUF_LEN_MAX))
+ return ODP_BUFFER_INVALID;
+
+ /* Try to satisfy request from the local cache */
+ buf = (odp_anybuf_t *)(void *)get_local_buf(&local_cache[pool_id],
+ &pool->s, totsize);
+
+ /* If cache is empty, satisfy request from the pool */
+ if (odp_unlikely(buf == NULL)) {
+ buf = (odp_anybuf_t *)(void *)get_buf(&pool->s);
+
+ if (odp_unlikely(buf == NULL))
+ return ODP_BUFFER_INVALID;
+
+ /* Get blocks for this buffer, if pool uses application data */
+ if (buf->buf.size < totsize) {
+ intmax_t needed = totsize - buf->buf.size;
+ do {
+ uint8_t *blk = get_blk(&pool->s);
+ if (blk == NULL) {
+ ret_buf(&pool->s, &buf->buf);
+ return ODP_BUFFER_INVALID;
+ }
+ buf->buf.addr[buf->buf.segcount++] = blk;
+ needed -= pool->s.seg_size;
+ } while (needed > 0);
+ buf->buf.size = buf->buf.segcount * pool->s.seg_size;
+ }
+ }
+
+ /* By default, buffers inherit their pool's zeroization setting */
+ buf->buf.flags.zeroized = pool->s.flags.zeroized;
+
+ if (buf->buf.type == ODP_BUFFER_TYPE_PACKET) {
+ packet_init(pool, &buf->pkt, size);
+
+ if (pool->s.init_params.buf_init != NULL)
+ (*pool->s.init_params.buf_init)
+ (buf->buf.handle.handle,
+ pool->s.init_params.buf_init_arg);
+ }
+
+ return odp_hdr_to_buf(&buf->buf);
+}
+
odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool_id)
{
@@ -279,6 +317,15 @@ void odp_buffer_free(odp_buffer_t buf)
rte_pktmbuf_free((struct rte_mbuf *)buf);
}
+void _odp_flush_caches(void)
+{
+ int i;
+
+ for (i = 0; i < ODP_CONFIG_BUFFER_POOLS; i++) {
+ pool_entry_t *pool = get_pool_entry(i);
+ flush_cache(&local_cache[i], &pool->s);
+ }
+}
void odp_buffer_pool_print(odp_buffer_pool_t pool_id)
{