new file mode 100644
@@ -0,0 +1,219 @@
+/* Copyright (c) 2014, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * Inline functions for ODP buffer mgmt routines - implementation internal
+ */
+
+#ifndef ODP_BUFFER_INLINES_H_
+#define ODP_BUFFER_INLINES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+static inline odp_buffer_t odp_hdr_to_buf(odp_buffer_hdr_t *hdr)
+{
+ return hdr->buf_hdl.handle;
+}
+
+static inline odp_buffer_t odp_buffer_encode_handle(odp_buffer_hdr_t *hdr)
+{
+ odp_buffer_bits_t handle;
+ uint32_t pool_id = pool_handle_to_index(hdr->pool_hdl);
+ struct pool_entry_s *pool = get_pool_entry(pool_id);
+
+ handle.pool_id = pool_id;
+ handle.index = ((uint8_t *)hdr - pool->pool_base_addr) /
+ ODP_CACHE_LINE_SIZE;
+ handle.seg = 0;
+
+ return handle.u32;
+}
+
+static inline odp_buffer_segment_t odp_hdr_to_seg(odp_buffer_hdr_t *hdr,
+ size_t ndx)
+{
+ odp_buffer_bits_t handle;
+ uint32_t pool_id = pool_handle_to_index(hdr->pool_hdl);
+ struct pool_entry_s *pool = get_pool_entry(pool_id);
+
+ handle.pool_id = pool_id;
+ handle.index = ((uint8_t *)hdr - pool->pool_base_addr) /
+ ODP_CACHE_LINE_SIZE;
+ handle.seg = ndx;
+
+ return handle.u32;
+}
+
+static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf)
+{
+ odp_buffer_bits_t handle;
+ uint32_t pool_id;
+ uint32_t index;
+ struct pool_entry_s *pool;
+
+ handle.u32 = buf;
+ pool_id = handle.pool_id;
+ index = handle.index;
+
+#ifdef POOL_ERROR_CHECK
+ if (odp_unlikely(pool_id > ODP_CONFIG_BUFFER_POOLS)) {
+ ODP_ERR("odp_buf_to_hdr: Bad pool id\n");
+ return NULL;
+ }
+#endif
+
+ pool = get_pool_entry(pool_id);
+
+#ifdef POOL_ERROR_CHECK
+ if (odp_unlikely(index > pool->num_bufs - 1)) {
+ ODP_ERR("odp_buf_to_hdr: Bad buffer index\n");
+ return NULL;
+ }
+#endif
+
+ return (odp_buffer_hdr_t *)(void *)
+ (pool->pool_base_addr + (index * ODP_CACHE_LINE_SIZE));
+}
+
+static inline uint32_t odp_buffer_refcount(odp_buffer_hdr_t *buf)
+{
+ return buf->ref_count;
+}
+
+static inline uint32_t odp_buffer_incr_refcount(odp_buffer_hdr_t *buf,
+ uint32_t val)
+{
+ return odp_atomic_fetch_add_u32(&buf->ref_count, val) + val;
+}
+
+static inline uint32_t odp_buffer_decr_refcount(odp_buffer_hdr_t *buf,
+ uint32_t val)
+{
+ uint32_t tmp;
+
+ tmp = odp_atomic_fetch_sub_u32(&buf->ref_count, val);
+
+ if (tmp < val) {
+ odp_atomic_fetch_add_u32(&buf->ref_count, val - tmp);
+ return 0;
+ } else {
+ return tmp - val;
+ }
+}
+
+static inline odp_buffer_hdr_t *validate_buf(odp_buffer_t buf)
+{
+ odp_buffer_bits_t handle;
+ odp_buffer_hdr_t *buf_hdr;
+ handle.u32 = buf;
+
+ /* For buffer handles, segment index must be 0 */
+ if (handle.seg != 0)
+ return NULL;
+
+ pool_entry_t *pool = odp_pool_to_entry(handle.pool_id);
+
+ /* If pool not created, handle is invalid */
+ if (pool->s.shm == ODP_SHM_INVALID)
+ return NULL;
+
+ /* A valid buffer index must be on stride, and must be in range */
+ if ((handle.index % pool->s.mdata_stride != 0) ||
+ ((uint32_t)(handle.index / pool->s.mdata_stride) >=
+ pool->s.num_bufs))
+ return NULL;
+
+ buf_hdr = (odp_buffer_hdr_t *)(void *)
+ (pool->s.pool_base_addr +
+ (handle.index * ODP_CACHE_LINE_SIZE));
+
+ /* Handle is valid, so buffer is valid if it is allocated */
+ if (buf_hdr->segcount == 0)
+ return NULL;
+ else
+ return buf_hdr;
+}
+
+int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf);
+
+static inline void *buffer_map(odp_buffer_hdr_t *buf,
+ size_t offset,
+ size_t *seglen,
+ size_t limit)
+{
+ int seg_index = offset / buf->segsize;
+ int seg_offset = offset % buf->segsize;
+ size_t buf_left = limit - offset;
+
+ *seglen = buf_left < buf->segsize ?
+ buf_left : buf->segsize - seg_offset;
+
+ return (void *)(seg_offset + (uint8_t *)buf->addr[seg_index]);
+}
+
+static inline odp_buffer_segment_t buffer_segment(odp_buffer_hdr_t *buf,
+ size_t ndx)
+{
+ odp_buffer_bits_t seghandle;
+ seghandle.u32 = buf->buf_hdl.u32;
+
+ if (ndx > buf->segcount) {
+ return ODP_SEGMENT_INVALID;
+ } else {
+ seghandle.seg = ndx;
+ return seghandle.handle;
+ }
+}
+
+static inline odp_buffer_segment_t segment_next(odp_buffer_hdr_t *buf,
+ odp_buffer_segment_t seg)
+{
+ odp_buffer_bits_t seghandle;
+ seghandle.u32 = seg;
+
+
+ if (seg == ODP_SEGMENT_INVALID)
+ return (odp_buffer_segment_t)buf->buf_hdl.u32;
+
+ if (seghandle.prefix != buf->buf_hdl.prefix ||
+ seghandle.seg >= buf->segcount) {
+ return ODP_SEGMENT_INVALID;
+ } else {
+ seghandle.seg++;
+ return (odp_buffer_segment_t)seghandle.u32;
+ }
+}
+
+static inline void *segment_map(odp_buffer_hdr_t *buf,
+ odp_buffer_segment_t seg,
+ size_t *seglen,
+ size_t limit)
+{
+ size_t buf_left;
+ odp_buffer_bits_t seghandle;
+ seghandle.u32 = seg;
+
+ if (seghandle.prefix != buf->buf_hdl.prefix ||
+ seghandle.seg > buf->segcount)
+ return NULL;
+
+ buf_left = limit - (seghandle.seg * buf->segsize);
+ *seglen = buf_left < buf->segsize ?
+ buf_left : buf->segsize;
+
+ return buf->addr[seghandle.seg];
+}
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
@@ -20,26 +20,39 @@ extern "C" {
#include <odp_std_types.h>
#include <odp_atomic.h>
-#include <odp_buffer_pool.h>
#include <odp_buffer.h>
#include <odp_debug.h>
#include <odp_align.h>
-
-/* TODO: move these to correct files */
-
-typedef uint64_t odp_phys_addr_t;
-
-#define ODP_BUFFER_MAX_INDEX (ODP_BUFFER_MAX_BUFFERS - 2)
-#define ODP_BUFFER_INVALID_INDEX (ODP_BUFFER_MAX_BUFFERS - 1)
-
-#define ODP_BUFS_PER_CHUNK 16
-#define ODP_BUFS_PER_SCATTER 4
-
-#define ODP_BUFFER_TYPE_CHUNK 0xffff
-
+#include <odp_config.h>
+
+#define ODP_BUFFER_MAX_SEG (ODP_CONFIG_BUF_MAX_SIZE/ODP_CONFIG_BUF_SEG_SIZE)
+
+ODP_STATIC_ASSERT((ODP_CONFIG_BUF_SEG_SIZE % ODP_CACHE_LINE_SIZE) == 0,
+ "ODP Segment size must be a multiple of cache line size");
+
+#define ODP_SEGBITS(x) \
+ ((x) < 2 ? 1 : \
+ ((x) < 4 ? 2 : \
+ ((x) < 8 ? 3 : \
+ ((x) < 16 ? 4 : \
+ ((x) < 32 ? 5 : \
+ ((x) < 64 ? 6 : \
+ ((x) < 128 ? 7 : \
+ ((x) < 256 ? 8 : \
+ ((x) < 512 ? 9 : \
+ ((x) < 1024 ? 10 : \
+ ((x) < 2048 ? 11 : \
+ ((x) < 4096 ? 12 : \
+ (0/0)))))))))))))
+
+ODP_STATIC_ASSERT(ODP_SEGBITS(ODP_BUFFER_MAX_SEG) <
+ ODP_SEGBITS(ODP_CACHE_LINE_SIZE),
+ "Number of segments must not exceed log of cache line size");
#define ODP_BUFFER_POOL_BITS 4
-#define ODP_BUFFER_INDEX_BITS (32 - ODP_BUFFER_POOL_BITS)
+#define ODP_BUFFER_SEG_BITS ODP_SEGBITS(ODP_CACHE_LINE_SIZE)
+#define ODP_BUFFER_INDEX_BITS (32 - ODP_BUFFER_POOL_BITS - ODP_BUFFER_SEG_BITS)
+#define ODP_BUFFER_PREFIX_BITS (ODP_BUFFER_POOL_BITS + ODP_BUFFER_INDEX_BITS)
#define ODP_BUFFER_MAX_POOLS (1 << ODP_BUFFER_POOL_BITS)
#define ODP_BUFFER_MAX_BUFFERS (1 << ODP_BUFFER_INDEX_BITS)
@@ -50,73 +63,48 @@ typedef union odp_buffer_bits_t {
struct {
uint32_t pool_id:ODP_BUFFER_POOL_BITS;
uint32_t index:ODP_BUFFER_INDEX_BITS;
+ uint32_t seg:ODP_BUFFER_SEG_BITS;
};
-} odp_buffer_bits_t;
+ struct {
+ uint32_t prefix:ODP_BUFFER_PREFIX_BITS;
+ uint32_t pfxseg:ODP_BUFFER_SEG_BITS;
+ };
+} odp_buffer_bits_t;
/* forward declaration */
struct odp_buffer_hdr_t;
-
-/*
- * Scatter/gather list of buffers
- */
-typedef struct odp_buffer_scatter_t {
- /* buffer pointers */
- struct odp_buffer_hdr_t *buf[ODP_BUFS_PER_SCATTER];
- int num_bufs; /* num buffers */
- int pos; /* position on the list */
- size_t total_len; /* Total length */
-} odp_buffer_scatter_t;
-
-
-/*
- * Chunk of buffers (in single pool)
- */
-typedef struct odp_buffer_chunk_t {
- uint32_t num_bufs; /* num buffers */
- uint32_t buf_index[ODP_BUFS_PER_CHUNK]; /* buffers */
-} odp_buffer_chunk_t;
-
-
/* Common buffer header */
typedef struct odp_buffer_hdr_t {
struct odp_buffer_hdr_t *next; /* next buf in a list */
- odp_buffer_bits_t handle; /* handle */
- odp_phys_addr_t phys_addr; /* physical data start address */
- void *addr; /* virtual data start address */
- uint32_t index; /* buf index in the pool */
+ odp_buffer_bits_t buf_hdl; /* handle */
size_t size; /* max data size */
- size_t cur_offset; /* current offset */
odp_atomic_u32_t ref_count; /* reference count */
- odp_buffer_scatter_t scatter; /* Scatter/gather list */
- int type; /* type of next header */
+ odp_buffer_type_e type; /* type of next header */
odp_buffer_pool_t pool_hdl; /* buffer pool handle */
-
+ void *udata_addr; /* user meta data addr */
+ size_t udata_size; /* size of user meta data */
+ uint32_t segcount; /* segment count */
+ uint32_t segsize; /* segment size */
+ void *addr[ODP_BUFFER_MAX_SEG]; /* Block addrs */
} odp_buffer_hdr_t;
-/* Ensure next header starts from 8 byte align */
-ODP_STATIC_ASSERT((sizeof(odp_buffer_hdr_t) % 8) == 0, "ODP_BUFFER_HDR_T__SIZE_ERROR");
-
-
-/* Raw buffer header */
-typedef struct {
- odp_buffer_hdr_t buf_hdr; /* common buffer header */
- uint8_t buf_data[]; /* start of buffer data area */
-} odp_raw_buffer_hdr_t;
-
-
-/* Chunk header */
-typedef struct odp_buffer_chunk_hdr_t {
- odp_buffer_hdr_t buf_hdr;
- odp_buffer_chunk_t chunk;
-} odp_buffer_chunk_hdr_t;
-
+typedef struct odp_buffer_hdr_stride {
+ uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_buffer_hdr_t))];
+} odp_buffer_hdr_stride;
-int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf);
+typedef struct odp_buf_blk_t {
+ struct odp_buf_blk_t *next;
+ struct odp_buf_blk_t *prev;
+} odp_buf_blk_t;
-void odp_buffer_copy_scatter(odp_buffer_t buf_dst, odp_buffer_t buf_src);
+/* Forward declarations */
+odp_buffer_t buffer_alloc(odp_buffer_pool_t pool, size_t size);
+int buffer_copy_to_buffer(odp_buffer_t dstbuf, size_t dstoffset,
+ odp_buffer_t srcbuf, size_t srcoffset,
+ size_t len);
#ifdef __cplusplus
}
@@ -24,6 +24,7 @@ extern "C" {
#include <odp_align.h>
#include <odp_hints.h>
#include <odp_config.h>
+#include <odp_shared_memory.h>
#include <odp_debug.h>
/* Use ticketlock instead of spinlock */
@@ -47,66 +48,146 @@ struct pool_entry_s {
odp_spinlock_t lock ODP_ALIGNED_CACHE;
#endif
- odp_buffer_chunk_hdr_t *head;
- uint64_t free_bufs;
char name[ODP_BUFFER_POOL_NAME_LEN];
- odp_buffer_pool_t pool_hdl ODP_ALIGNED_CACHE;
- uintptr_t buf_base;
- size_t buf_size;
- size_t buf_offset;
+ odp_buffer_pool_t pool_hdl;
+ odp_buffer_pool_param_t params;
+ odp_buffer_pool_init_t init_params;
+ odp_shm_t shm;
+ union {
+ uint32_t all;
+ struct {
+ uint32_t unsegmented:1;
+ uint32_t predefined:1;
+ };
+ } flags;
+ uint8_t *pool_base_addr;
+ size_t pool_size;
+ int mdata_stride;
+ uint8_t *udata_base_addr;
+ int buf_udata_size;
+ int udata_stride;
+ odp_buffer_hdr_t *buf_freelist;
+ uint8_t *blk_freelist;
+ odp_atomic_u32_t bufcount;
uint64_t num_bufs;
- void *pool_base_addr;
- uint64_t pool_size;
- size_t user_size;
- size_t user_align;
- int buf_type;
- size_t hdr_size;
+ size_t seg_size;
+ size_t high_wm;
+ size_t low_wm;
+ size_t headroom;
+ size_t tailroom;
};
+typedef union pool_entry_u {
+ struct pool_entry_s s;
+
+ uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct pool_entry_s))];
+
+} pool_entry_t;
extern void *pool_entry_ptr[];
+#if UINTPTR_MAX == 0xffffffffffffffff
+#define odp_at odp_atomic_u64_t
+#define odp_cs(p, o, n) odp_atomic_cmpset_u64((odp_at *)(void *)(p), \
+ (uint64_t)(o), (uint64_t)(n))
+#else
+#define odp_at odp_atomic_u32_t
+#define odp_cs(p, o, n) odp_atomic_cmpset_u32((odp_at *)(void *)(p), \
+ (uint32_t)(o), (uint32_t)(n))
+#endif
-static inline void *get_pool_entry(uint32_t pool_id)
+/* This macro suggested by Shmulik Ladkani */
+#define odp_ref(p) \
+ ((typeof(p))(uintptr_t) *(volatile typeof(p) const *)&(p))
+
+static inline void *get_blk(struct pool_entry_s *pool)
{
- return pool_entry_ptr[pool_id];
+ void *oldhead, *newhead;
+
+ do {
+ oldhead = odp_ref(pool->blk_freelist);
+ if (oldhead == NULL)
+ break;
+ newhead = ((odp_buf_blk_t *)oldhead)->next;
+ } while (odp_cs(&pool->blk_freelist, oldhead, newhead) == 0);
+
+ return (void *)oldhead;
}
+static inline void ret_blk(struct pool_entry_s *pool, void *block)
+{
+ void *oldhead;
-static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf)
+ do {
+ oldhead = odp_ref(pool->blk_freelist);
+ ((odp_buf_blk_t *)block)->next = oldhead;
+ } while (odp_cs(&pool->blk_freelist, oldhead, block) == 0);
+}
+
+static inline odp_buffer_hdr_t *get_buf(struct pool_entry_s *pool)
{
- odp_buffer_bits_t handle;
- uint32_t pool_id;
- uint32_t index;
- struct pool_entry_s *pool;
- odp_buffer_hdr_t *hdr;
-
- handle.u32 = buf;
- pool_id = handle.pool_id;
- index = handle.index;
-
-#ifdef POOL_ERROR_CHECK
- if (odp_unlikely(pool_id > ODP_CONFIG_BUFFER_POOLS)) {
- ODP_ERR("odp_buf_to_hdr: Bad pool id\n");
- return NULL;
+ odp_buffer_hdr_t *oldhead, *newhead;
+
+ do {
+ oldhead = odp_ref(pool->buf_freelist);
+ if (oldhead == NULL)
+ break;
+ newhead = oldhead->next;
+ } while (odp_cs(&pool->buf_freelist, oldhead, newhead) == 0);
+
+ if (oldhead != NULL) {
+ oldhead->next = oldhead;
+ odp_atomic_inc_u32(&pool->bufcount);
}
-#endif
- pool = get_pool_entry(pool_id);
+ return (void *)oldhead;
+}
-#ifdef POOL_ERROR_CHECK
- if (odp_unlikely(index > pool->num_bufs - 1)) {
- ODP_ERR("odp_buf_to_hdr: Bad buffer index\n");
- return NULL;
- }
-#endif
+static inline void ret_buf(struct pool_entry_s *pool, odp_buffer_hdr_t *buf)
+{
+ odp_buffer_hdr_t *oldhead;
- hdr = (odp_buffer_hdr_t *)(pool->buf_base + index * pool->buf_size);
+ while (buf->segcount > 0)
+ ret_blk(pool, buf->addr[--buf->segcount]);
- return hdr;
+ do {
+ oldhead = odp_ref(pool->buf_freelist);
+ buf->next = oldhead;
+ } while (odp_cs(&pool->buf_freelist, oldhead, buf) == 0);
+
+ odp_atomic_dec_u32(&pool->bufcount);
}
+static inline odp_buffer_pool_t pool_index_to_handle(uint32_t pool_id)
+{
+ return pool_id + 1;
+}
+
+static inline uint32_t pool_handle_to_index(odp_buffer_pool_t pool_hdl)
+{
+ return pool_hdl - 1;
+}
+
+static inline void *get_pool_entry(uint32_t pool_id)
+{
+ return pool_entry_ptr[pool_id];
+}
+
+static inline pool_entry_t *odp_pool_to_entry(odp_buffer_pool_t pool)
+{
+ return (pool_entry_t *)get_pool_entry(pool_handle_to_index(pool));
+}
+
+static inline pool_entry_t *odp_buf_to_pool(odp_buffer_hdr_t *buf)
+{
+ return odp_pool_to_entry(buf->pool_hdl);
+}
+
+static inline size_t odp_buffer_pool_segment_size(odp_buffer_pool_t pool)
+{
+ return odp_pool_to_entry(pool)->s.seg_size;
+}
#ifdef __cplusplus
}
@@ -22,6 +22,7 @@ extern "C" {
#include <odp_debug.h>
#include <odp_buffer_internal.h>
#include <odp_buffer_pool_internal.h>
+#include <odp_buffer_inlines.h>
#include <odp_packet.h>
#include <odp_packet_io.h>
@@ -43,6 +44,7 @@ typedef union {
uint32_t vlan:1; /**< VLAN hdr found */
uint32_t vlan_qinq:1; /**< Stacked VLAN found, QinQ */
+ uint32_t snap:1; /**< SNAP */
uint32_t arp:1; /**< ARP */
uint32_t ipv4:1; /**< IPv4 */
@@ -53,7 +55,7 @@ typedef union {
uint32_t udp:1; /**< UDP */
uint32_t tcp:1; /**< TCP */
- uint32_t sctp:1; /**< SCTP */
+ uint32_t tcpopt:1; /**< TCP Options present */
uint32_t icmp:1; /**< ICMP */
};
} input_flags_t;
@@ -69,7 +71,9 @@ typedef union {
struct {
/* Bitfield flags for each detected error */
+ uint32_t app_error:1; /**< Error bit for application use */
uint32_t frame_len:1; /**< Frame length error */
+ uint32_t snap_len:1; /**< Snap length error */
uint32_t l2_chksum:1; /**< L2 checksum error, checks TBD */
uint32_t ip_err:1; /**< IP error, checks TBD */
uint32_t tcp_err:1; /**< TCP error, checks TBD */
@@ -88,7 +92,10 @@ typedef union {
struct {
/* Bitfield flags for each output option */
- uint32_t l4_chksum:1; /**< Request L4 checksum calculation */
+ uint32_t l3_chksum_set:1; /**< L3 chksum bit is valid */
+ uint32_t l3_chksum:1; /**< L3 chksum override */
+ uint32_t l4_chksum_set:1; /**< L3 chksum bit is valid */
+ uint32_t l4_chksum:1; /**< L4 chksum override */
};
} output_flags_t;
@@ -101,29 +108,33 @@ typedef struct {
/* common buffer header */
odp_buffer_hdr_t buf_hdr;
- input_flags_t input_flags;
error_flags_t error_flags;
+ input_flags_t input_flags;
output_flags_t output_flags;
- uint32_t frame_offset; /**< offset to start of frame, even on error */
uint32_t l2_offset; /**< offset to L2 hdr, e.g. Eth */
uint32_t l3_offset; /**< offset to L3 hdr, e.g. IPv4, IPv6 */
uint32_t l4_offset; /**< offset to L4 hdr (TCP, UDP, SCTP, also ICMP) */
+ uint32_t payload_offset; /**< offset to payload */
- uint32_t frame_len;
+ uint32_t vlan_s_tag; /**< Parsed 1st VLAN header (S-TAG) */
+ uint32_t vlan_c_tag; /**< Parsed 2nd VLAN header (C-TAG) */
+ uint32_t l3_protocol; /**< Parsed L3 protocol */
+ uint32_t l3_len; /**< Layer 3 length */
+ uint32_t l4_protocol; /**< Parsed L4 protocol */
+ uint32_t l4_len; /**< Layer 4 length */
- uint64_t user_ctx; /* user context */
+ uint32_t frame_len;
+ uint32_t headroom;
+ uint32_t tailroom;
odp_pktio_t input;
-
- uint32_t pad;
- uint8_t buf_data[]; /* start of buffer data area */
} odp_packet_hdr_t;
-ODP_STATIC_ASSERT(sizeof(odp_packet_hdr_t) == ODP_OFFSETOF(odp_packet_hdr_t, buf_data),
- "ODP_PACKET_HDR_T__SIZE_ERR");
-ODP_STATIC_ASSERT(sizeof(odp_packet_hdr_t) % sizeof(uint64_t) == 0,
- "ODP_PACKET_HDR_T__SIZE_ERR2");
+typedef struct odp_packet_hdr_stride {
+ uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_packet_hdr_t))];
+} odp_packet_hdr_stride;
+
/**
* Return the packet header
@@ -133,10 +144,100 @@ static inline odp_packet_hdr_t *odp_packet_hdr(odp_packet_t pkt)
return (odp_packet_hdr_t *)odp_buf_to_hdr((odp_buffer_t)pkt);
}
+static inline void odp_packet_set_len(odp_packet_t pkt, size_t len)
+{
+ odp_packet_hdr(pkt)->frame_len = len;
+}
+
+static inline odp_packet_hdr_t *odp_packet_hdr_from_buf_hdr(odp_buffer_hdr_t
+ *buf_hdr)
+{
+ return (odp_packet_hdr_t *)buf_hdr;
+}
+
+static inline odp_buffer_hdr_t *odp_packet_hdr_to_buf_hdr(odp_packet_hdr_t *pkt)
+{
+ return &pkt->buf_hdr;
+}
+
+static inline odp_packet_t odp_packet_from_buf_internal(odp_packet_t buf)
+{
+ return (odp_packet_t)buf;
+}
+
+static inline odp_buffer_t odp_packet_to_buf_internal(odp_packet_t pkt)
+{
+ return (odp_buffer_t)pkt;
+}
+
+static inline void packet_init(pool_entry_t *pool,
+ odp_packet_hdr_t *pkt_hdr,
+ size_t size)
+{
+ /* Reset parser metadata */
+ pkt_hdr->error_flags.all = 0;
+ pkt_hdr->input_flags.all = 0;
+ pkt_hdr->output_flags.all = 0;
+ pkt_hdr->l2_offset = 0;
+ pkt_hdr->l3_offset = 0;
+ pkt_hdr->l4_offset = 0;
+ pkt_hdr->payload_offset = 0;
+ pkt_hdr->vlan_s_tag = 0;
+ pkt_hdr->vlan_c_tag = 0;
+ pkt_hdr->l3_protocol = 0;
+ pkt_hdr->l4_protocol = 0;
+
+ /*
+ * Packet headroom is set from the pool's headroom
+ * Packet tailroom is rounded up to fill the last
+ * segment occupied by the allocated length.
+ */
+ pkt_hdr->frame_len = size;
+ pkt_hdr->headroom = pool->s.headroom;
+ pkt_hdr->tailroom =
+ (pool->s.seg_size * pkt_hdr->buf_hdr.segcount) -
+ (pool->s.headroom + size);
+}
+
+#define pull_offset(x, len) (x = x < len ? 0 : x - len)
+
+static inline void push_head(odp_packet_hdr_t *pkt_hdr, size_t len)
+{
+ pkt_hdr->headroom -= len;
+ pkt_hdr->frame_len += len;
+ pkt_hdr->l2_offset += len;
+ pkt_hdr->l3_offset += len;
+ pkt_hdr->l4_offset += len;
+ pkt_hdr->payload_offset += len;
+}
+
+static inline void pull_head(odp_packet_hdr_t *pkt_hdr, size_t len)
+{
+ pkt_hdr->headroom += len;
+ pkt_hdr->frame_len -= len;
+ pull_offset(pkt_hdr->l2_offset, len);
+ pull_offset(pkt_hdr->l3_offset, len);
+ pull_offset(pkt_hdr->l4_offset, len);
+ pull_offset(pkt_hdr->payload_offset, len);
+}
+
+static inline void push_tail(odp_packet_hdr_t *pkt_hdr, size_t len)
+{
+ pkt_hdr->tailroom -= len;
+ pkt_hdr->frame_len += len;
+}
+
+
+static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, size_t len)
+{
+ pkt_hdr->tailroom += len;
+ pkt_hdr->frame_len -= len;
+}
+
/**
* Parse packet and set internal metadata
*/
-void odp_packet_parse(odp_packet_t pkt, size_t len, size_t l2_offset);
+void odph_packet_parse(odp_packet_t pkt, size_t len, size_t l2_offset);
#ifdef __cplusplus
}
@@ -21,8 +21,9 @@ extern "C" {
#include <odp_std_types.h>
#include <odp_queue.h>
#include <odp_buffer.h>
-#include <odp_buffer_internal.h>
#include <odp_buffer_pool_internal.h>
+#include <odp_buffer_internal.h>
+#include <odp_buffer_inlines.h>
#include <odp_timer.h>
struct timeout_t;
@@ -48,17 +49,11 @@ typedef struct odp_timeout_hdr_t {
timeout_t meta;
- uint8_t buf_data[];
} odp_timeout_hdr_t;
-
-
-ODP_STATIC_ASSERT(sizeof(odp_timeout_hdr_t) ==
- ODP_OFFSETOF(odp_timeout_hdr_t, buf_data),
- "ODP_TIMEOUT_HDR_T__SIZE_ERR");
-
-ODP_STATIC_ASSERT(sizeof(odp_timeout_hdr_t) % sizeof(uint64_t) == 0,
- "ODP_TIMEOUT_HDR_T__SIZE_ERR2");
+typedef struct odp_timeout_hdr_stride {
+ uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_timeout_hdr_t))];
+} odp_timeout_hdr_stride;
/**
v3 incorporates bugfix: ret_buf must return to buf_freelist, not blk_freelist Signed-off-by: Bill Fischofer <bill.fischofer@linaro.org> --- .../linux-generic/include/odp_buffer_inlines.h | 219 +++++++++++++++++++++ .../linux-generic/include/odp_buffer_internal.h | 118 +++++------ .../include/odp_buffer_pool_internal.h | 159 +++++++++++---- .../linux-generic/include/odp_packet_internal.h | 129 ++++++++++-- .../linux-generic/include/odp_timer_internal.h | 15 +- 5 files changed, 512 insertions(+), 128 deletions(-) create mode 100644 platform/linux-generic/include/odp_buffer_inlines.h