@@ -29,6 +29,7 @@ include_HEADERS = \
$(top_srcdir)/platform/linux-generic/include/api/odp_barrier.h \
$(srcdir)/include/api/odp_buffer_pool.h \
$(top_srcdir)/platform/linux-generic/include/api/odp_byteorder.h \
+ $(top_srcdir)/platform/linux-generic/include/api/odp_classification.h \
$(top_srcdir)/platform/linux-generic/include/api/odp_compiler.h \
$(top_srcdir)/platform/linux-generic/include/api/odp_config.h \
$(top_srcdir)/platform/linux-generic/include/api/odp_coremask.h \
@@ -52,15 +53,16 @@ include_HEADERS = \
$(top_srcdir)/platform/linux-generic/include/api/odp_time.h \
$(top_srcdir)/platform/linux-generic/include/api/odp_timer.h \
$(top_srcdir)/platform/linux-generic/include/api/odp_version.h \
- $(srcdir)/include/api/odp_pktio_types.h
+ $(srcdir)/include/api/odp_platform_types.h
subdirheadersdir = $(includedir)/helper
subdirheaders_HEADERS = \
$(top_srcdir)/helper/include/odph_chksum.h \
$(top_srcdir)/helper/include/odph_eth.h \
+ $(top_srcdir)/helper/include/odph_icmp.h \
$(top_srcdir)/helper/include/odph_ip.h \
+ $(top_srcdir)/helper/include/odph_ipsec.h \
$(top_srcdir)/helper/include/odph_linux.h \
- $(top_srcdir)/helper/include/odph_packet.h \
$(top_srcdir)/helper/include/odph_ring.h \
$(top_srcdir)/helper/include/odph_udp.h
@@ -68,9 +70,11 @@ __LIB__libodp_la_SOURCES = \
../linux-generic/odp_barrier.c \
odp_buffer.c \
odp_buffer_pool.c \
+ ../linux-generic/odp_classification.c \
../linux-generic/odp_coremask.c \
../linux-generic/odp_crypto.c \
odp_init.c \
+ ../linux-generic/odp_impl.c \
odp_linux.c \
odp_packet.c \
odp_packet_dpdk.c \
@@ -80,11 +84,12 @@ __LIB__libodp_la_SOURCES = \
odp_queue.c \
../linux-generic/odp_ring.c \
../linux-generic/odp_rwlock.c \
- odp_schedule.c \
+ ../linux-generic/odp_schedule.c \
../linux-generic/odp_shared_memory.c \
../linux-generic/odp_spinlock.c \
../linux-generic/odp_system_info.c \
../linux-generic/odp_thread.c \
../linux-generic/odp_ticketlock.c \
../linux-generic/odp_time.c \
- ../linux-generic/odp_timer.c
+ ../linux-generic/odp_timer.c \
+ ../linux-generic/odp_weak.c
@@ -19,20 +19,13 @@ extern "C" {
#endif
-
#include <odp_std_types.h>
+#include <odp_platform_types.h>
-
-
-
-
-/**
- * ODP buffer
+/** @defgroup odp_buffer ODP BUFFER
+ * Operations on a buffer.
+ * @{
*/
-typedef unsigned long odp_buffer_t;
-
-
-#define ODP_BUFFER_INVALID (unsigned long)(-1L) /**< Invalid buffer */
/**
@@ -74,11 +67,21 @@ int odp_buffer_type(odp_buffer_t buf);
*
* @param buf Buffer handle
*
- * @return 1 if valid, otherwise 0
+ * @retval 1 Buffer handle represents a valid buffer.
+ * @retval 0 Buffer handle does not represent a valid buffer.
*/
int odp_buffer_is_valid(odp_buffer_t buf);
/**
+ * Buffer pool of the buffer
+ *
+ * @param buf Buffer handle
+ *
+ * @return Handle of buffer pool buffer belongs to
+ */
+odp_buffer_pool_t odp_buffer_pool(odp_buffer_t buf);
+
+/**
* Print buffer metadata to STDOUT
*
* @param buf Buffer handle
@@ -86,6 +89,9 @@ int odp_buffer_is_valid(odp_buffer_t buf);
*/
void odp_buffer_print(odp_buffer_t buf);
+/**
+ * @}
+ */
#ifdef __cplusplus
}
@@ -21,66 +21,143 @@ extern "C" {
#include <odp_std_types.h>
+#include <odp_platform_types.h>
#include <odp_buffer.h>
+/** @addtogroup odp_buffer
+ * Operations on a buffer pool.
+ * @{
+ */
+
/** Maximum queue name lenght in chars */
#define ODP_BUFFER_POOL_NAME_LEN 32
-/** Invalid buffer pool */
-#define ODP_BUFFER_POOL_INVALID (unsigned long)(-1L)
-
-/** ODP buffer pool */
-typedef unsigned long odp_buffer_pool_t;
-
+/**
+ * Buffer pool parameters
+ * Used to communicate buffer pool creation options.
+ */
+typedef struct odp_buffer_pool_param_t {
+ uint32_t buf_size; /**< Buffer size in bytes. The maximum
+ number of bytes application will
+ store in each buffer. For packets, this
+ is the maximum packet data length, and
+ configured headroom and tailroom will be
+ added to this number */
+ uint32_t buf_align; /**< Minimum buffer alignment in bytes.
+ Valid values are powers of two. Use 0
+ for default alignment. Default will
+ always be a multiple of 8. */
+ uint32_t num_bufs; /**< Number of buffers in the pool */
+ int buf_type; /**< Buffer type */
+} odp_buffer_pool_param_t;
/**
* Create a buffer pool
+ * This routine is used to create a buffer pool. It take three
+ * arguments: the optional name of the pool to be created, an optional shared
+ * memory handle, and a parameter struct that describes the pool to be
+ * created. If a name is not specified the result is an anonymous pool that
+ * cannot be referenced by odp_buffer_pool_lookup().
+ *
+ * @param name Name of the pool, max ODP_BUFFER_POOL_NAME_LEN-1 chars.
+ * May be specified as NULL for anonymous pools.
+ *
+ * @param shm The shared memory object in which to create the pool.
+ * Use ODP_SHM_NULL to reserve default memory type
+ * for the buffer type.
*
- * @param name Name of the pool (max ODP_BUFFER_POOL_NAME_LEN - 1 chars)
- * @param base_addr Pool base address
- * @param size Pool size in bytes
- * @param buf_size Buffer size in bytes
- * @param buf_align Minimum buffer alignment
- * @param buf_type Buffer type
+ * @param params Buffer pool parameters.
*
- * @return Buffer pool handle
+ * @return Handle of the created buffer pool
+ * @retval ODP_BUFFER_POOL_INVALID Buffer pool could not be created
*/
+
odp_buffer_pool_t odp_buffer_pool_create(const char *name,
- void *base_addr, uint64_t size,
- size_t buf_size, size_t buf_align,
- int buf_type);
+ odp_shm_t shm,
+ odp_buffer_pool_param_t *params);
+/**
+ * Destroy a buffer pool previously created by odp_buffer_pool_create()
+ *
+ * @param pool Handle of the buffer pool to be destroyed
+ *
+ * @retval 0 Success
+ * @retval -1 Failure
+ *
+ * @note This routine destroys a previously created buffer pool. This call
+ * does not destroy any shared memory object passed to
+ * odp_buffer_pool_create() used to store the buffer pool contents. The caller
+ * takes responsibility for that. If no shared memory object was passed as
+ * part of the create call, then this routine will destroy any internal shared
+ * memory objects associated with the buffer pool. Results are undefined if
+ * an attempt is made to destroy a buffer pool that contains allocated or
+ * otherwise active buffers.
+ */
+int odp_buffer_pool_destroy(odp_buffer_pool_t pool);
/**
* Find a buffer pool by name
*
* @param name Name of the pool
*
- * @return Buffer pool handle, or ODP_BUFFER_POOL_INVALID if not found.
+ * @return Handle of found buffer pool
+ * @retval ODP_BUFFER_POOL_INVALID Buffer pool could not be found
+ *
+ * @note This routine cannot be used to look up an anonymous pool (one created
+ * with no name).
*/
odp_buffer_pool_t odp_buffer_pool_lookup(const char *name);
+/**
+ * Buffer pool information struct
+ * Used to get information about a buffer pool.
+ */
+typedef struct odp_buffer_pool_info_t {
+ const char *name; /**< pool name */
+ odp_shm_t shm; /**< handle of shared memory area
+ supplied by application to
+ contain buffer pool, or
+ ODP_SHM_NULL if this pool is
+ managed by ODP */
+ odp_buffer_pool_param_t params; /**< pool parameters */
+} odp_buffer_pool_info_t;
+
+/**
+ * Retrieve information about a buffer pool
+ *
+ * @param pool Buffer pool handle
+ *
+ * @param[out] info Receives an odp_buffer_pool_info_t object
+ * that describes the pool.
+ *
+ * @retval 0 Success
+ * @retval -1 Failure. Info could not be retrieved.
+ */
+
+int odp_buffer_pool_info(odp_buffer_pool_t pool,
+ odp_buffer_pool_info_t *info);
/**
* Print buffer pool info
*
* @param pool Pool handle
*
+ * @note This routine writes implementation-defined information about the
+ * specified buffer pool to the ODP log. The intended use is for debugging.
*/
void odp_buffer_pool_print(odp_buffer_pool_t pool);
-
-
/**
* Buffer alloc
*
+ * The validity of a buffer can be cheked at any time with odp_buffer_is_valid()
* @param pool Pool handle
*
- * @return Buffer handle or ODP_BUFFER_INVALID
+ * @return Handle of allocated buffer
+ * @retval ODP_BUFFER_INVALID Buffer could not be allocated
*/
odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool);
-
/**
* Buffer free
*
@@ -89,8 +166,9 @@ odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool);
*/
void odp_buffer_free(odp_buffer_t buf);
-
-
+/**
+ * @}
+ */
#ifdef __cplusplus
}
@@ -12,73 +12,74 @@
#ifndef ODP_DEBUG_H_
#define ODP_DEBUG_H_
-#include <stdio.h>
-#include <stdlib.h>
#ifdef __cplusplus
extern "C" {
#endif
+/** @addtogroup odp_ver_abt_log_dbg
+ * Macros that allows different messages.
+ * @{
+ */
+
#ifdef __GNUC__
-/**
- * Indicate deprecated variables, functions or types
- */
-#define ODP_DEPRECATED __attribute__((__deprecated__))
+
+#if __GNUC__ < 4 || (__GNUC__ == 4 && (__GNUC_MINOR__ < 6))
/**
- * Intentionally unused variables ot functions
+ * _Static_assert was only added in GCC 4.6. Provide a weak replacement
+ * for previous versions.
*/
-#define ODP_UNUSED __attribute__((__unused__))
+#define _Static_assert(e, s) extern int (*static_assert_checker(void)) \
+ [sizeof(struct { unsigned int error_if_negative:(e) ? 1 : -1; })]
-#else
+#endif
-#define ODP_DEPRECATED
-#define ODP_UNUSED
-#endif
-/**
- * Runtime assertion-macro - aborts if 'cond' is false.
- */
-#ifndef ODP_NO_DEBUG
-#define ODP_ASSERT(cond, msg) \
- do { if (!(cond)) {ODP_ERR("%s\n", msg); abort(); } } while (0)
-#else
-#define ODP_ASSERT(cond, msg)
#endif
+
/**
* Compile time assertion-macro - fail compilation if cond is false.
* @note This macro has zero runtime overhead
*/
-#define ODP_STATIC_ASSERT(cond, msg) _Static_assert(1, msg)
+#define _ODP_STATIC_ASSERT(cond, msg) _Static_assert(1, msg)
/**
- * Debug printing macro, which prints output when DEBUG flag is set.
+ * ODP log level.
*/
-#define ODP_DBG(fmt, ...) \
- do { if (ODP_DEBUG_PRINT == 1) \
- printf(fmt, ##__VA_ARGS__); \
- } while (0)
+typedef enum odp_log_level {
+ ODP_LOG_DBG,
+ ODP_LOG_ERR,
+ ODP_LOG_UNIMPLEMENTED,
+ ODP_LOG_ABORT,
+ ODP_LOG_PRINT
+} odp_log_level_e;
/**
- * Print output to stderr (file, line and function).
+ * ODP log function
+ *
+ * Instead of direct prints to stdout/stderr all logging in ODP implementation
+ * should be done via this function or its wrappers.
+ * ODP platform MUST provide a default *weak* implementation of this function.
+ * Application MAY override the function if needed by providing a strong
+ * function.
+ *
+ * @param[in] level Log level
+ * @param[in] fmt printf-style message format
+ *
+ * @return The number of characters logged if succeeded. Otherwise returns
+ * a negative number.
*/
-#define ODP_ERR(fmt, ...) \
-do { fprintf(stderr, "%s:%d:%s(): " fmt, __FILE__, \
- __LINE__, __func__, ##__VA_ARGS__); \
-} while (0)
+extern int odp_override_log(odp_log_level_e level, const char *fmt, ...);
+
+
/**
- * Print output to stderr (file, line and function),
- * then abort.
+ * @}
*/
-#define ODP_ABORT(fmt, ...) \
-do { fprintf(stderr, "%s:%d:%s(): " fmt, __FILE__, \
- __LINE__, __func__, ##__VA_ARGS__); \
- abort(); \
-} while (0)
#ifdef __cplusplus
}
@@ -1,7 +1,7 @@
/* Copyright (c) 2013, Linaro Limited
* All rights reserved.
*
- * SPDX-License-Identifier: BSD-3-Clause
+ * SPDX-License-Identifier: BSD-3-Clause
*/
@@ -19,33 +19,84 @@ extern "C" {
#endif
#include <odp_buffer.h>
+#include <odp_platform_types.h>
+/** @defgroup odp_packet ODP PACKET
+ * Operations on a packet.
+ * @{
+ */
-/**
- * ODP packet descriptor
+
+/*
+ * Packet API v0.5 notes
+ * - Push/pull operations only on packet level
+ * - Push/pull within limits of segment headroom/tailroom/data lengths
+ * - Segment data length must be always at least one byte (i.e. there are no
+ * empty segments)
+ * - Head/tailroom content belong to packet content (in addition to data
+ * and meta-data) and thus is preserved over packet ownership changes.
+ * - _addr refer to a fixed address, which operations do not modify
+ * - _ptr refer to pointer to data, which may be modified by operations
*/
-typedef odp_buffer_t odp_packet_t;
-/** Invalid packet */
-#define ODP_PACKET_INVALID ODP_BUFFER_INVALID
+/*
+ *
+ * Alloc and free
+ * ********************************************************
+ *
+ */
-/** Invalid offset */
-#define ODP_PACKET_OFFSET_INVALID ((size_t)-1)
+/**
+ * Allocate a packet from a buffer pool
+ *
+ * Allocates a packet of the requested length from the specified buffer pool.
+ * Pool must have been created with buffer type ODP_BUFFER_TYPE_PACKET. The
+ * packet is initialized with data pointers and lengths set according to the
+ * specified len, and the default headroom and tailroom length settings. All
+ * other packet metadata are set to their default values.
+ *
+ * @param pool Pool handle
+ * @param len Packet data length
+ *
+ * @return Handle of allocated packet
+ * @retval ODP_PACKET_INVALID Packet could not be allocated
+ *
+ * @note The default headroom and tailroom used for packets is specified by
+ * the ODP_CONFIG_PACKET_HEADROOM and ODP_CONFIG_PACKET_TAILROOM defines in
+ * odp_config.h.
+ */
+odp_packet_t odp_packet_alloc(odp_buffer_pool_t pool, uint32_t len);
+/**
+ * Free packet
+ *
+ * Frees the packet into the buffer pool it was allocated from.
+ *
+ * @param pkt Packet handle
+ */
+void odp_packet_free(odp_packet_t pkt);
/**
- * Initialize the packet
+ * Reset packet
*
- * Needs to be called if the user allocates a packet buffer, i.e. the packet
- * has not been received from I/O through ODP.
+ * Resets all packet meta-data to their default values. Packet length is used
+ * to initialize pointers and lengths. It must be less than the total buffer
+ * length of the packet minus the default headroom length. Packet is not
+ * modified on failure.
*
- * @param pkt Packet handle
+ * @param pkt Packet handle
+ * @param len Packet data length
+ *
+ * @retval 0 Success
+ * @retval Non-zero Failure
+ *
+ * @see odp_packet_buf_len()
*/
-void odp_packet_init(odp_packet_t pkt);
+int odp_packet_reset(odp_packet_t pkt, uint32_t len);
/**
- * Convert from packet handle to buffer handle
+ * Convert a buffer handle to a packet handle
*
* @param buf Buffer handle
*
@@ -54,190 +105,737 @@ void odp_packet_init(odp_packet_t pkt);
odp_packet_t odp_packet_from_buffer(odp_buffer_t buf);
/**
- * Convert from buffer handle to packet handle
+ * Convert a packet handle to a buffer handle
*
* @param pkt Packet handle
*
* @return Buffer handle
*/
-odp_buffer_t odp_buffer_from_packet(odp_packet_t pkt);
+odp_buffer_t odp_packet_to_buffer(odp_packet_t pkt);
+
+
+/*
+ *
+ * Pointers and lengths
+ * ********************************************************
+ *
+ */
+
+/**
+ * Packet head address
+ *
+ * Returns start address of the first segment. Packet level headroom starts
+ * from here. Use odp_packet_data() or odp_packet_l2_ptr() to return the
+ * packet data start address.
+ *
+ * @param pkt Packet handle
+ *
+ * @return Pointer to the start address of the first packet segment
+ *
+ * @see odp_packet_data(), odp_packet_l2_ptr(), odp_packet_headroom()
+ */
+void *odp_packet_head(odp_packet_t pkt);
+
+/**
+ * Total packet buffer length
+ *
+ * Returns sum of buffer lengths over all packet segments.
+ *
+ * @param pkt Packet handle
+ *
+ * @return Total packet buffer length in bytes
+ *
+ * @see odp_packet_reset()
+ */
+uint32_t odp_packet_buf_len(odp_packet_t pkt);
/**
- * Set the packet length
+ * Packet data pointer
+ *
+ * Returns the current packet data pointer. When a packet is received
+ * from packet input, this points to the first byte of the received
+ * packet. Packet level offsets are calculated relative to this position.
+ *
+ * User can adjust the data pointer with head_push/head_pull (does not modify
+ * segmentation) and add_data/rem_data calls (may modify segmentation).
+ *
+ * @param pkt Packet handle
+ *
+ * @return Pointer to the packet data
+ *
+ * @see odp_packet_l2_ptr(), odp_packet_seg_len()
+ */
+void *odp_packet_data(odp_packet_t pkt);
+
+/**
+ * Packet segment data length
+ *
+ * Returns number of data bytes following the current data pointer
+ * (odp_packet_data()) location in the segment.
*
* @param pkt Packet handle
- * @param len Length of packet in bytes
+ *
+ * @return Segment data length in bytes (pointed by odp_packet_data())
+ *
+ * @see odp_packet_data()
*/
-void odp_packet_set_len(odp_packet_t pkt, size_t len);
+uint32_t odp_packet_seg_len(odp_packet_t pkt);
/**
- * Get the packet length
+ * Packet data length
+ *
+ * Returns sum of data lengths over all packet segments.
*
* @param pkt Packet handle
*
- * @return Packet length in bytes
+ * @return Packet data length
*/
-size_t odp_packet_get_len(odp_packet_t pkt);
+uint32_t odp_packet_len(odp_packet_t pkt);
/**
- * Set packet user context
+ * Packet headroom length
*
- * @param buf Packet handle
- * @param ctx User context
+ * Returns the current packet level headroom length.
+ *
+ * @param pkt Packet handle
*
+ * @return Headroom length
*/
-void odp_packet_set_ctx(odp_packet_t buf, const void *ctx);
+uint32_t odp_packet_headroom(odp_packet_t pkt);
/**
- * Get packet user context
+ * Packet tailroom length
*
- * @param buf Packet handle
+ * Returns the current packet level tailroom length.
*
- * @return User context
+ * @param pkt Packet handle
+ *
+ * @return Tailroom length
*/
-void *odp_packet_get_ctx(odp_packet_t buf);
+uint32_t odp_packet_tailroom(odp_packet_t pkt);
/**
- * Get address to the start of the packet buffer
+ * Packet tailroom pointer
*
- * The address of the packet buffer is not necessarily the same as the start
- * address of the received frame, e.g. an eth frame may be offset by 2 or 6
- * bytes to ensure 32 or 64-bit alignment of the IP header.
- * Use odp_packet_l2(pkt) to get the start address of a received valid frame
- * or odp_packet_start(pkt) to get the start address even if no valid L2 header
- * could be found.
+ * Returns pointer to the start of the current packet level tailroom.
+ *
+ * User can adjust the tail pointer with tail_push/tail_pull (does not modify
+ * segmentation) and add_data/rem_data calls (may modify segmentation).
*
* @param pkt Packet handle
*
- * @return Pointer to the start of the packet buffer
+ * @return Tailroom pointer
*
- * @see odp_packet_l2(), odp_packet_start()
+ * @see odp_packet_tailroom()
*/
-uint8_t *odp_packet_buf_addr(odp_packet_t pkt);
+void *odp_packet_tail(odp_packet_t pkt);
/**
- * Get pointer to the start of the received frame
+ * Push out packet head
+ *
+ * Increase packet data length by moving packet head into packet headroom.
+ * Packet headroom is decreased with the same amount. The packet head may be
+ * pushed out up to 'headroom' bytes. Packet is not modified if there's not
+ * enough headroom space.
*
- * The address of the packet buffer is not necessarily the same as the start
- * address of the received frame, e.g. an eth frame may be offset by 2 or 6
- * bytes to ensure 32 or 64-bit alignment of the IP header.
- * Use odp_packet_l2(pkt) to get the start address of a received valid eth frame
+ * odp_packet_xxx:
+ * seg_len += len
+ * len += len
+ * headroom -= len
+ * data -= len
*
- * odp_packet_start() will always return a pointer to the start of the frame,
- * even if the frame is unrecognized and no valid L2 header could be found.
+ * Operation does not modify packet segmentation or move data. Handles and
+ * pointers remain valid. User is responsible to update packet meta-data
+ * offsets when needed.
*
* @param pkt Packet handle
+ * @param len Number of bytes to push the head (0 ... headroom)
*
- * @return Pointer to the start of the received frame
+ * @return The new data pointer
+ * @retval NULL Requested offset exceeds available headroom
*
- * @see odp_packet_l2(), odp_packet_buf_addr()
+ * @see odp_packet_headroom(), odp_packet_pull_head()
*/
-uint8_t *odp_packet_start(odp_packet_t pkt);
+void *odp_packet_push_head(odp_packet_t pkt, uint32_t len);
/**
- * Get pointer to the start of the L2 frame
+ * Pull in packet head
*
- * The L2 frame header address is not necessarily the same as the address of the
- * packet buffer, see odp_packet_buf_addr()
+ * Decrease packet data length by removing data from the head of the packet.
+ * Packet headroom is increased with the same amount. Packet head may be pulled
+ * in up to seg_len - 1 bytes (i.e. packet data pointer must stay in the
+ * first segment). Packet is not modified if there's not enough data.
+ *
+ * odp_packet_xxx:
+ * seg_len -= len
+ * len -= len
+ * headroom += len
+ * data += len
+ *
+ * Operation does not modify packet segmentation or move data. Handles and
+ * pointers remain valid. User is responsible to update packet meta-data
+ * offsets when needed.
*
* @param pkt Packet handle
+ * @param len Number of bytes to pull the head (0 ... seg_len - 1)
*
- * @return Pointer to L2 header or NULL if not found
+ * @return The new data pointer, or NULL in case of an error.
+ * @retval NULL Requested offset exceeds packet segment length
*
- * @see odp_packet_buf_addr(), odp_packet_start()
+ * @see odp_packet_seg_len(), odp_packet_push_head()
*/
-uint8_t *odp_packet_l2(odp_packet_t pkt);
+void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len);
/**
- * Return the byte offset from the packet buffer to the L2 frame
+ * Push out packet tail
+ *
+ * Increase packet data length by moving packet tail into packet tailroom.
+ * Packet tailroom is decreased with the same amount. The packet tail may be
+ * pushed out up to 'tailroom' bytes. Packet is not modified if there's not
+ * enough tailroom.
+ *
+ * last_seg:
+ * data_len += len
+ *
+ * odp_packet_xxx:
+ * len += len
+ * tail += len
+ * tailroom -= len
+ *
+ * Operation does not modify packet segmentation or move data. Handles,
+ * pointers and offsets remain valid.
*
* @param pkt Packet handle
+ * @param len Number of bytes to push the tail (0 ... tailroom)
*
- * @return L2 byte offset or ODP_PACKET_OFFSET_INVALID if not found
+ * @return The old tail pointer
+ * @retval NULL Requested offset exceeds available tailroom
+ *
+ * @see odp_packet_tailroom(), odp_packet_pull_tail()
*/
-size_t odp_packet_l2_offset(odp_packet_t pkt);
+void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len);
/**
- * Set the byte offset to the L2 frame
+ * Pull in packet tail
+ *
+ * Decrease packet data length by removing data from the tail of the packet.
+ * Packet tailroom is increased with the same amount. Packet tail may be pulled
+ * in up to last segment data_len - 1 bytes. (i.e. packet tail must stay in the
+ * last segment). Packet is not modified if there's not enough data.
+ *
+ * last_seg:
+ * data_len -= len
+ *
+ * odp_packet_xxx:
+ * len -= len
+ * tail -= len
+ * tailroom += len
+ *
+ * Operation does not modify packet segmentation or move data. Handles and
+ * pointers remain valid. User is responsible to update packet meta-data
+ * offsets when needed.
+ *
+ * @param pkt Packet handle
+ * @param len Number of bytes to pull the tail (0 ... last_seg:data_len - 1)
+ *
+ * @return The new tail pointer
+ * @retval NULL The specified offset exceeds allowable data length
+ */
+void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len);
+
+/**
+ * Packet offset pointer
+ *
+ * Returns pointer to data in the packet offset. The packet level byte offset is
+ * calculated from the current odp_packet_data() position. Optionally outputs
+ * handle to the segment and number of data bytes in the segment following the
+ * pointer.
+ *
+ * @param pkt Packet handle
+ * @param offset Byte offset into the packet
+ * @param[out] len Number of data bytes remaining in the segment (output).
+ * Ignored when NULL.
+ * @param[out] seg Handle to the segment containing the address (output).
+ * Ignored when NULL.
+ *
+ * @return Pointer to the offset
+ * @retval NULL Requested offset exceeds packet length
+ */
+void *odp_packet_offset(odp_packet_t pkt, uint32_t offset, uint32_t *len,
+ odp_packet_seg_t *seg);
+
+/*
+ *
+ * Meta-data
+ * ********************************************************
+ *
+ */
+
+/**
+ * Packet pool
+ *
+ * Returns handle to the buffer pool where the packet was allocated from.
+ *
+ * @param pkt Packet handle
+ *
+ * @return Buffer pool handle
+ */
+odp_buffer_pool_t odp_packet_pool(odp_packet_t pkt);
+
+/**
+ * Packet input interface
+ *
+ * Returns handle to the packet IO interface which received the packet or
+ * ODP_PKTIO_INVALID when the packet was allocated/reset by the application.
+ *
+ * @param pkt Packet handle
+ *
+ * @return Packet interface handle
+ * @retval ODP_PKTIO_INVALID Packet was not received
+ */
+odp_pktio_t odp_packet_input(odp_packet_t pkt);
+
+/**
+ * User context pointer
+ *
+ * Return previously stored user context pointer.
+ *
+ * @param pkt Packet handle
+ *
+ * @return User context pointer
+ */
+void *odp_packet_user_ptr(odp_packet_t pkt);
+
+/**
+ * Set user context pointer
+ *
+ * Each packet has room for a user defined context. The context can be stored
+ * either as a pointer OR as a uint64_t value, but not both at the same time.
+ * The latest context set operation determines which one has been stored.
+ *
+ * @param pkt Packet handle
+ * @param ctx User context pointer
+ */
+void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ctx);
+
+/**
+ * User context data (uint64_t)
+ *
+ * Return previously stored user context uint64_t value.
+ *
+ * @param pkt Packet handle
+ *
+ * @return User context data
+ */
+uint64_t odp_packet_user_u64(odp_packet_t pkt);
+
+/**
+ * Set user context data (uint64_t)
+ *
+ * Each packet has room for a user defined context. The context can be stored
+ * either as a pointer OR as a uint64_t value, but not both at the same time.
+ * The latest context set operation determines which one has been stored.
+ *
+ * @param pkt Packet handle
+ * @param ctx User context data
+ */
+void odp_packet_user_u64_set(odp_packet_t pkt, uint64_t ctx);
+
+/**
+ * Layer 2 start pointer
+ *
+ * Returns pointer to the start of the layer 2 header. Optionally, outputs
+ * number of data bytes in the segment following the pointer.
+ *
+ * @param pkt Packet handle
+ * @param[out] len Number of data bytes remaining in the segment (output).
+ * Ignored when NULL.
+ *
+ * @return Layer 2 start pointer, or offset 0 by default
+ *
+ * @see odp_packet_l2_offset(), odp_packet_l2_offset_set()
+ */
+void *odp_packet_l2_ptr(odp_packet_t pkt, uint32_t *len);
+
+/**
+ * Layer 2 start offset
+ *
+ * Returns offset to the start of the layer 2 header. The offset is calculated
+ * from the current odp_packet_data() position in bytes.
+ *
+ * User is responsible to update the offset when modifying the packet data
+ * pointer position.
+ *
+ * @param pkt Packet handle
+ *
+ * @return Layer 2 start offset
+ */
+uint32_t odp_packet_l2_offset(odp_packet_t pkt);
+
+/**
+ * Set layer 2 start offset
+ *
+ * Set offset to the start of the layer 2 header. The offset is calculated from
+ * the current odp_packet_data() position in bytes. Offset must not exceed
+ * packet data length. Packet is not modified on an error.
*
* @param pkt Packet handle
- * @param offset L2 byte offset
+ * @param offset Layer 2 start offset (0 ... odp_packet_len()-1)
+ *
+ * @retval 0 Success
+ * @retval Non-zero Failure
*/
-void odp_packet_set_l2_offset(odp_packet_t pkt, size_t offset);
+int odp_packet_l2_offset_set(odp_packet_t pkt, uint32_t offset);
+/**
+ * Layer 3 start pointer
+ *
+ * Returns pointer to the start of the layer 3 header. Optionally, outputs
+ * number of data bytes in the segment following the pointer.
+ *
+ * @param pkt Packet handle
+ * @param[out] len Number of data bytes remaining in the segment (output).
+ * Ignored when NULL.
+ *
+ * @return Layer 3 start pointer, or NULL
+ *
+ * @see odp_packet_l3_offset(), odp_packet_l3_offset_set()
+ */
+void *odp_packet_l3_ptr(odp_packet_t pkt, uint32_t *len);
/**
- * Get pointer to the start of the L3 packet
+ * Layer 3 start offset
+ *
+ * Returns offset to the start of the layer 3 header. The offset is calculated
+ * from the current odp_packet_data() position in bytes.
+ *
+ * User is responsible to update the offset when modifying the packet data
+ * pointer position.
*
* @param pkt Packet handle
*
- * @return Pointer to L3 packet or NULL if not found
+ * @return Layer 3 start offset or ODP_PACKET_OFFSET_INVALID if not found
+ */
+uint32_t odp_packet_l3_offset(odp_packet_t pkt);
+
+/**
+ * Set layer 3 start offset
+ *
+ * Set offset to the start of the layer 3 header. The offset is calculated from
+ * the current odp_packet_data() position in bytes. Offset must not exceed
+ * packet data length. Packet is not modified on an error.
+ *
+ * @param pkt Packet handle
+ * @param offset Layer 3 start offset (0 ... odp_packet_len()-1)
*
+ * @retval 0 Success
+ * @retval Non-zero Failure
*/
-uint8_t *odp_packet_l3(odp_packet_t pkt);
+int odp_packet_l3_offset_set(odp_packet_t pkt, uint32_t offset);
/**
- * Return the byte offset from the packet buffer to the L3 packet
+ * Layer 4 start pointer
+ *
+ * Returns pointer to the start of the layer 4 header. Optionally, outputs
+ * number of data bytes in the segment following the pointer.
+ *
+ * @param pkt Packet handle
+ * @param[out] len Number of data bytes remaining in the segment (output).
+ * Ignored when NULL.
+ *
+ * @return Layer 4 start pointer, or NULL
+ *
+ * @see odp_packet_l4_offset(), odp_packet_l4_offset_set()
+ */
+void *odp_packet_l4_ptr(odp_packet_t pkt, uint32_t *len);
+
+/**
+ * Layer 4 start offset
+ *
+ * Returns offset to the start of the layer 4 header. The offset is calculated
+ * from the current odp_packet_data() position in bytes.
+ *
+ * User is responsible to update the offset when modifying the packet data
+ * pointer position.
*
* @param pkt Packet handle
*
- * @return L3 byte offset or ODP_PACKET_OFFSET_INVALID if not found
+ * @return Layer 4 start offset or ODP_PACKET_OFFSET_INVALID if not found
*/
-size_t odp_packet_l3_offset(odp_packet_t pkt);
+uint32_t odp_packet_l4_offset(odp_packet_t pkt);
/**
- * Set the byte offset to the L3 packet
+ * Set layer 4 start offset
+ *
+ * Set offset to the start of the layer 4 header. The offset is calculated from
+ * the current odp_packet_data() position in bytes. Offset must not exceed
+ * packet data length. Packet is not modified on an error.
*
* @param pkt Packet handle
- * @param offset L3 byte offset
+ * @param offset Layer 4 start offset (0 ... odp_packet_len()-1)
+ *
+ * @retval 0 Success
+ * @retval Non-zero Failure
+ */
+int odp_packet_l4_offset_set(odp_packet_t pkt, uint32_t offset);
+
+/**
+ * Tests if packet is segmented
+ *
+ * @param pkt Packet handle
+ *
+ * @retval 0 Packet is not segmented
+ * @retval 1 Packet is segmented
+ */
+int odp_packet_is_segmented(odp_packet_t pkt);
+
+/**
+ * Number of segments
+ *
+ * Returns number of segments in the packet. A packet has always at least one
+ * segment.
+ *
+ * @param pkt Packet handle
+ *
+ * @return Number of segments (>0)
+ */
+int odp_packet_num_segs(odp_packet_t pkt);
+
+/**
+ * First segment in packet
+ *
+ * A packet has always the first segment (has at least one segment).
+ *
+ * @param pkt Packet handle
+ *
+ * @return Handle to the first segment
+ */
+odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt);
+
+/**
+ * Last segment in packet
+ *
+ * A packet has always the last segment (has at least one segment).
+ *
+ * @param pkt Packet handle
+ *
+ * @return Handle to the last segment
+ */
+odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt);
+
+/**
+ * Next segment in packet
+ *
+ * Returns handle to the next segment after the current segment, or
+ * ODP_PACKET_SEG_INVALID if there are no more segments. Use
+ * odp_packet_first_seg() to get handle to the first segment.
+ *
+ * @param pkt Packet handle
+ * @param seg Current segment handle
+ *
+ * @return Handle to the next segment, or ODP_PACKET_SEG_INVALID
+ */
+odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg);
+
+
+/*
+ *
+ * Segment level
+ * ********************************************************
+ *
*/
-void odp_packet_set_l3_offset(odp_packet_t pkt, size_t offset);
+/**
+ * Segment buffer address
+ *
+ * Returns start address of the segment.
+ *
+ * @param pkt Packet handle
+ * @param seg Segment handle
+ *
+ * @return Start address of the segment, or NULL on an error
+ *
+ * @see odp_packet_seg_buf_len()
+ */
+void *odp_packet_seg_buf_addr(odp_packet_t pkt, odp_packet_seg_t seg);
/**
- * Get pointer to the start of the L4 packet
+ * Segment buffer length
+ *
+ * Returns segment buffer length in bytes.
*
* @param pkt Packet handle
+ * @param seg Segment handle
*
- * @return Pointer to L4 packet or NULL if not found
+ * @return Segment buffer length in bytes
*
+ * @see odp_packet_seg_buf_addr()
*/
-uint8_t *odp_packet_l4(odp_packet_t pkt);
+uint32_t odp_packet_seg_buf_len(odp_packet_t pkt, odp_packet_seg_t seg);
/**
- * Return the byte offset from the packet buffer to the L4 packet
+ * Segment data pointer
+ *
+ * Returns pointer to the first byte of data in the segment.
+ *
+ * @param pkt Packet handle
+ * @param seg Segment handle
+ *
+ * @return Pointer to the segment data, or NULL on an error
+ *
+ * @see odp_packet_seg_data_len()
+ */
+void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg);
+
+/**
+ * Segment data length
+ *
+ * Returns segment data length in bytes.
*
* @param pkt Packet handle
+ * @param seg Segment handle
*
- * @return L4 byte offset or ODP_PACKET_OFFSET_INVALID if not found
+ * @return Segment data length in bytes
+ *
+ * @see odp_packet_seg_data()
*/
-size_t odp_packet_l4_offset(odp_packet_t pkt);
+uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg);
+
+
+/*
+ *
+ * Manipulation
+ * ********************************************************
+ *
+ */
+
/**
- * Set the byte offset to the L4 packet
+ * Add data into an offset
+ *
+ * Increases packet data length by adding new data area into the specified
+ * offset. The operation returns a new packet handle on success. It may modify
+ * packet segmentation and move data. Handles and pointers must be updated
+ * after the operation. User is responsible to update packet meta-data offsets
+ * when needed. The packet is not modified on an error.
*
* @param pkt Packet handle
- * @param offset L4 byte offset
+ * @param offset Byte offset into the packet
+ * @param len Number of bytes to add into the offset
+ *
+ * @return New packet handle, or ODP_PACKET_INVALID in case of an error.
*/
-void odp_packet_set_l4_offset(odp_packet_t pkt, size_t offset);
+odp_packet_t odp_packet_add_data(odp_packet_t pkt, uint32_t offset,
+ uint32_t len);
/**
- * Print (debug) information about the packet
+ * Remove data from an offset
+ *
+ * Decreases packet data length by removing data from the specified offset.
+ * The operation returns a new packet handle on success, and may modify
+ * packet segmentation and move data. Handles and pointers must be updated
+ * after the operation. User is responsible to update packet meta-data offsets
+ * when needed. The packet is not modified on an error.
+ *
+ * @param pkt Packet handle
+ * @param offset Byte offset into the packet
+ * @param len Number of bytes to remove from the offset
+ *
+ * @return New packet handle, or ODP_PACKET_INVALID in case of an error.
+ */
+odp_packet_t odp_packet_rem_data(odp_packet_t pkt, uint32_t offset,
+ uint32_t len);
+
+
+/*
+ *
+ * Copy
+ * ********************************************************
+ *
+ */
+
+/**
+ * Copy packet
+ *
+ * Create a new copy of the packet. The new packet is exact copy of the source
+ * packet (incl. data and meta-data). The pool must have been created with
+ * buffer type ODP_BUFFER_TYPE_PACKET.
+ *
+ * @param pkt Packet handle
+ * @param pool Buffer pool for allocation of the new packet.
+ *
+ * @return Handle to the copy of the packet, or ODP_PACKET_INVALID
+ */
+odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_buffer_pool_t pool);
+
+/**
+ * Copy data from packet
+ *
+ * Copy 'len' bytes of data from the packet level offset to the destination
+ * address.
+ *
+ * @param pkt Packet handle
+ * @param offset Byte offset into the packet
+ * @param len Number of bytes to copy
+ * @param dst Destination address
+ *
+ * @retval 0 Success
+ * @retval Non-zero Failure
+ */
+int odp_packet_copydata_out(odp_packet_t pkt, uint32_t offset,
+ uint32_t len, void *dst);
+
+/**
+ * Copy data into packet
+ *
+ * Copy 'len' bytes of data from the source address into the packet level
+ * offset. Maximum number of bytes to copy is packet data length minus the
+ * offset. Packet is not modified on an error.
+ *
+ * @param pkt Packet handle
+ * @param offset Byte offset into the packet
+ * @param len Number of bytes to copy
+ * @param src Source address
+ *
+ * @retval 0 Success
+ * @retval Non-zero Failure
+ */
+int odp_packet_copydata_in(odp_packet_t pkt, uint32_t offset,
+ uint32_t len, const void *src);
+
+/*
+ *
+ * Debugging
+ * ********************************************************
+ *
+ */
+
+/**
+ * Print packet to the console
+ *
+ * Print all packet debug information to the console.
*
* @param pkt Packet handle
*/
void odp_packet_print(odp_packet_t pkt);
/**
- * Copy contents and metadata from pkt_src to pkt_dst
- * Useful when creating copies of packets
+ * Perform full packet validity check
*
- * @param pkt_dst Destination packet
- * @param pkt_src Source packet
+ * The operation may consume considerable number of cpu cycles depending on
+ * the check level.
+ *
+ * @param pkt Packet handle
*
- * @return 0 if successful
+ * @retval 0 Packet is not valid
+ * @retval 1 Packet is valid
+ */
+int odp_packet_is_valid(odp_packet_t pkt);
+
+
+/**
+ * @}
*/
-int odp_packet_copy(odp_packet_t pkt_dst, odp_packet_t pkt_src);
#ifdef __cplusplus
}
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014, Linaro Limited
+/* Copyright (c) 2013, Linaro Limited
* All rights reserved.
*
* SPDX-License-Identifier: BSD-3-Clause
@@ -19,29 +19,29 @@ extern "C" {
#endif
#include <odp_std_types.h>
+#include <odp_platform_types.h>
#include <odp_buffer_pool.h>
#include <odp_packet.h>
#include <odp_queue.h>
-#include <odp_pktio_types.h>
-
-/** ODP packet IO handle */
-typedef uint32_t odp_pktio_t;
-
-/** Invalid packet IO handle */
-#define ODP_PKTIO_INVALID 0
+/** @defgroup odp_packet_io ODP PACKET IO
+ * Operations on a packet.
+ * @{
+ */
/**
* Open an ODP packet IO instance
*
* @param dev Packet IO device
* @param pool Pool to use for packet IO
- * @param params Set of parameters to pass to the arch dependent implementation
*
* @return ODP packet IO handle or ODP_PKTIO_INVALID on error
+ *
+ * @note dev name loop is specially pktio reserved name for
+ * device used for testing. Usually it's loop back
+ * interface.
*/
-odp_pktio_t odp_pktio_open(const char *dev, odp_buffer_pool_t pool,
- odp_pktio_params_t *params);
+odp_pktio_t odp_pktio_open(const char *dev, odp_buffer_pool_t pool);
/**
* Close an ODP packet IO instance
@@ -130,21 +130,77 @@ void odp_pktio_set_input(odp_packet_t pkt, odp_pktio_t id);
odp_pktio_t odp_pktio_get_input(odp_packet_t pkt);
/**
- * Defines the maximum length of mac address supported by this platform
+ * Configure the MTU for a packet IO interface.
+ *
+ * @param[in] id ODP packet IO handle.
+ * @param[in] mtu The value of MTU that the interface will be configured to
+ * use.
+ *
+ * @retval 0 on success.
+ * @retval -1 if specified mtu can not be handled.
+ * @retval -1 on any other error or illegal input parameters.
+ */
+int odp_pktio_set_mtu(odp_pktio_t id, int mtu);
+
+/**
+ * Return the currently configured MTU value of a packet IO interface.
+ *
+ * @param[in] id ODP packet IO handle.
+ *
+ * @retval MTU value >0 on success.
+ * @retval -1 on any error or not existance pktio id.
+ */
+int odp_pktio_mtu(odp_pktio_t id);
+
+/**
+ * Enable/Disable promiscuous mode on a packet IO interface.
+ *
+ * @param[in] id ODP packet IO handle.
+ * @param[in] enable 1 to enable, 0 to disable.
+ *
+ * @retval 0 on success.
+ * @retval non-zero on any error.
+ */
+int odp_pktio_promisc_mode_set(odp_pktio_t id, odp_bool_t enable);
+
+/**
+ * Determine if promiscuous mode is enabled for a packet IO interface.
+ *
+ * @param[in] id ODP packet IO handle.
+ *
+ * @retval 1 if promiscuous mode is enabled.
+ * @retval 0 if promiscuous mode is disabled.
+ * @retval -1 on any error.
+*/
+int odp_pktio_promisc_mode(odp_pktio_t id);
+
+/**
+ * Get the default MAC address of a packet IO interface.
+ *
+ * @param id ODP packet IO handle.
+ * @param[out] mac_addr Storage for MAC address of the packet IO interface.
+ * @param addr_size Storage size for the address
+ *
+ * @retval Number of bytes written on success, 0 on failure.
*/
-#define ODP_MAC_ADDR_MAX_LENGTH ETH_ALEN
+size_t odp_pktio_mac_addr(odp_pktio_t id, void *mac_addr,
+ size_t addr_size);
/**
* Get mac address of the interface
*
- * @param id ODP packet IO handle
- * @param mac_addr Storage for Mac address of the packet IO interface
- * Storage provided by the caller should be equal
- * to ODP_MAC_ADDR_MAX_LENGTH (filled by function)
+ * @param id ODP packet IO handle
+ * @param mac_addr Storage for Mac address of the packet IO interface
+ * Storage provided by the caller should be equal
+ * to ODP_MAC_ADDR_MAX_LENGTH (filled by function)
* @return 0 on success or -1 on error
-**/
+ */
int odp_pktio_get_mac_addr(odp_pktio_t id, unsigned char *mac_addr);
+/**
+ * @}
+ */
+
#ifdef __cplusplus
}
#endif
deleted file mode 100644
@@ -1,45 +0,0 @@
-
-/* Copyright (c) 2013, Linaro Limited
- * All rights reserved.
- *
- * SPDX-License-Identifier: BSD-3-Clause
- */
-
-#ifndef ODP_PKTIO_TYPES_H
-#define ODP_PKTIO_TYPES_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/* We should ensure that future enum values will never overlap, otherwise
- * applications that want netmap suport might get in trouble if the odp lib
- * was not built with netmap support and there are more types define below
- */
-
-typedef enum {
- ODP_PKTIO_TYPE_SOCKET_BASIC = 0x1,
- ODP_PKTIO_TYPE_SOCKET_MMSG,
- ODP_PKTIO_TYPE_SOCKET_MMAP,
- ODP_PKTIO_TYPE_NETMAP,
- ODP_PKTIO_TYPE_DPDK,
-} odp_pktio_type_t;
-
-#include <odp_pktio_socket.h>
-#ifdef ODP_HAVE_NETMAP
-#include <odp_pktio_netmap.h>
-#endif
-
-typedef union odp_pktio_params_t {
- odp_pktio_type_t type;
- socket_params_t sock_params;
-#ifdef ODP_HAVE_NETMAP
- netmap_params_t nm_params;
-#endif
-} odp_pktio_params_t;
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
new file mode 100644
@@ -0,0 +1,78 @@
+/* Copyright (c) 2014, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+
+/**
+ * @file
+ * ODP implementation types
+ * This file contains all of the implementation-defined types for ODP abstract
+ * definitions. Having this in one file means that other ODP API files are
+ * implementation-independent and avoids circular dependencies for files that
+ * refer to types managed by other components. Included here are typedefs and
+ * related typed constants that are referenced by other ODP API files.
+ */
+
+#ifndef ODP_IMPL_TYPES_H_
+#define ODP_IMPL_TYPES_H_
+
+/** @defgroup odp_platform_types ODP PLATFORM TYPES
+ * Implementation specific definitions for ODP abstract types.
+ * @{
+ */
+
+/** ODP Buffer pool */
+typedef unsigned long odp_buffer_pool_t;
+
+/** Invalid buffer pool */
+#define ODP_BUFFER_POOL_INVALID (0xffffffff)
+
+/** ODP buffer */
+typedef unsigned long odp_buffer_t;
+
+/** Invalid buffer */
+#define ODP_BUFFER_INVALID (0xffffffff)
+
+/** ODP buffer segment */
+typedef odp_buffer_t odp_buffer_seg_t;
+
+/** Invalid segment */
+#define ODP_SEGMENT_INVALID ODP_BUFFER_INVALID
+
+/** ODP packet */
+typedef odp_buffer_t odp_packet_t;
+
+/** Invalid packet */
+#define ODP_PACKET_INVALID ODP_BUFFER_INVALID
+
+/** ODP packet segment */
+typedef odp_buffer_t odp_packet_seg_t;
+
+/** Invalid packet segment */
+#define ODP_PACKET_SEG_INVALID ODP_BUFFER_INVALID
+
+/** ODP packet IO handle */
+typedef uint32_t odp_pktio_t;
+
+/** Invalid packet IO handle */
+#define ODP_PKTIO_INVALID 0
+
+/** odp_pktio_t value to indicate any port */
+#define ODP_PKTIO_ANY ((odp_pktio_t)~0)
+
+/**
+ * ODP shared memory block
+ */
+typedef uint32_t odp_shm_t;
+
+/** Invalid shared memory block */
+#define ODP_SHM_INVALID 0
+#define ODP_SHM_NULL ODP_SHM_INVALID /**< Synonym for buffer pool use */
+
+/**
+ * @}
+ */
+
+#endif
new file mode 100644
@@ -0,0 +1,179 @@
+/* Copyright (c) 2014, Linaro Limited
+ * All rights reserved.
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+/**
+ * @file
+ *
+ * Inline functions for ODP buffer mgmt routines - implementation internal
+ */
+
+#ifndef ODP_BUFFER_INLINES_H_
+#define ODP_BUFFER_INLINES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+static inline odp_buffer_t odp_buffer_encode_handle(odp_buffer_hdr_t *hdr)
+{
+ odp_buffer_bits_t handle;
+ uint32_t pool_id = pool_handle_to_index(hdr->pool_hdl);
+ struct pool_entry_s *pool = get_pool_entry(pool_id);
+
+ handle.pool_id = pool_id;
+ handle.index = ((uint8_t *)hdr - pool->pool_mdata_addr) /
+ ODP_CACHE_LINE_SIZE;
+ handle.seg = 0;
+
+ return handle.u32;
+}
+
+static inline odp_buffer_t odp_hdr_to_buf(odp_buffer_hdr_t *hdr)
+{
+ return hdr->handle.handle;
+}
+
+static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf)
+{
+ return (odp_buffer_hdr_t *)buf;
+}
+
+static inline uint32_t odp_buffer_refcount(odp_buffer_hdr_t *buf)
+{
+ return odp_atomic_load_u32(&buf->ref_count);
+}
+
+static inline uint32_t odp_buffer_incr_refcount(odp_buffer_hdr_t *buf,
+ uint32_t val)
+{
+ return odp_atomic_fetch_add_u32(&buf->ref_count, val) + val;
+}
+
+static inline uint32_t odp_buffer_decr_refcount(odp_buffer_hdr_t *buf,
+ uint32_t val)
+{
+ uint32_t tmp;
+
+ tmp = odp_atomic_fetch_sub_u32(&buf->ref_count, val);
+
+ if (tmp < val) {
+ odp_atomic_fetch_add_u32(&buf->ref_count, val - tmp);
+ return 0;
+ } else {
+ return tmp - val;
+ }
+}
+
+static inline odp_buffer_hdr_t *validate_buf(odp_buffer_t buf)
+{
+ odp_buffer_bits_t handle;
+ odp_buffer_hdr_t *buf_hdr;
+ handle.u32 = buf;
+
+ /* For buffer handles, segment index must be 0 and pool id in range */
+ if (handle.seg != 0 || handle.pool_id >= ODP_CONFIG_BUFFER_POOLS)
+ return NULL;
+
+ pool_entry_t *pool = odp_pool_to_entry(handle.pool_id);
+
+ /* If pool not created, handle is invalid */
+ if (pool->s.pool_shm == ODP_SHM_INVALID)
+ return NULL;
+
+ uint32_t buf_stride = pool->s.buf_stride / ODP_CACHE_LINE_SIZE;
+
+ /* A valid buffer index must be on stride, and must be in range */
+ if ((handle.index % buf_stride != 0) ||
+ ((uint32_t)(handle.index / buf_stride) >= pool->s.params.num_bufs))
+ return NULL;
+
+ buf_hdr = (odp_buffer_hdr_t *)(void *)
+ (pool->s.pool_mdata_addr +
+ (handle.index * ODP_CACHE_LINE_SIZE));
+
+ /* Handle is valid, so buffer is valid if it is allocated */
+ return buf_hdr->allocator == ODP_FREEBUF ? NULL : buf_hdr;
+}
+
+int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf);
+
+static inline void *buffer_map(odp_buffer_hdr_t *buf,
+ uint32_t offset,
+ uint32_t *seglen,
+ uint32_t limit)
+{
+ int seg_index = offset / buf->segsize;
+ int seg_offset = offset % buf->segsize;
+
+ if (seglen != NULL) {
+ uint32_t buf_left = limit - offset;
+ *seglen = buf_left < buf->segsize ?
+ buf_left : buf->segsize - seg_offset;
+ }
+
+ return (void *)(seg_offset + (uint8_t *)buf->addr[seg_index]);
+}
+
+static inline odp_buffer_seg_t segment_next(odp_buffer_hdr_t *buf,
+ odp_buffer_seg_t seg)
+{
+ odp_buffer_bits_t seghandle;
+ seghandle.u32 = seg;
+
+ if (seg == ODP_SEGMENT_INVALID ||
+ seghandle.prefix != buf->handle.prefix ||
+ seghandle.seg >= buf->segcount - 1)
+ return ODP_SEGMENT_INVALID;
+ else {
+ seghandle.seg++;
+ return (odp_buffer_seg_t)seghandle.u32;
+ }
+}
+
+static inline void *segment_map(odp_buffer_hdr_t *buf,
+ odp_buffer_seg_t seg,
+ uint32_t *seglen,
+ uint32_t limit,
+ uint32_t hr)
+{
+ uint32_t seg_offset, buf_left;
+ odp_buffer_bits_t seghandle;
+ uint8_t *seg_addr;
+ seghandle.u32 = seg;
+
+ if (seghandle.prefix != buf->handle.prefix ||
+ seghandle.seg >= buf->segcount)
+ return NULL;
+
+ seg_addr = (uint8_t *)buf->addr[seghandle.seg];
+ seg_offset = seghandle.seg * buf->segsize;
+ limit += hr;
+
+ /* Can't map this segment if it's nothing but headroom or tailroom */
+ if (hr >= seg_offset + buf->segsize || seg_offset > limit)
+ return NULL;
+
+ /* Adjust address & offset if this segment contains any headroom */
+ if (hr > seg_offset) {
+ seg_addr += hr % buf->segsize;
+ seg_offset += hr % buf->segsize;
+ }
+
+ /* Set seglen if caller is asking for it */
+ if (seglen != NULL) {
+ buf_left = limit - seg_offset;
+ *seglen = buf_left < buf->segsize ? buf_left : buf->segsize;
+ }
+
+ return (void *)seg_addr;
+}
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
@@ -24,50 +24,145 @@ extern "C" {
#include <odp_buffer.h>
#include <odp_debug.h>
#include <odp_align.h>
-#include <rte_mbuf.h>
+#include <odp_align_internal.h>
+#include <odp_config.h>
+#include <odp_byteorder.h>
+#include <odp_thread.h>
-/* TODO: move these to correct files */
+/* DPDK */
+#include <rte_mbuf.h>
-typedef uint64_t odp_phys_addr_t;
+#define ODP_BITSIZE(x) \
+ ((x) <= 2 ? 1 : \
+ ((x) <= 4 ? 2 : \
+ ((x) <= 8 ? 3 : \
+ ((x) <= 16 ? 4 : \
+ ((x) <= 32 ? 5 : \
+ ((x) <= 64 ? 6 : \
+ ((x) <= 128 ? 7 : \
+ ((x) <= 256 ? 8 : \
+ ((x) <= 512 ? 9 : \
+ ((x) <= 1024 ? 10 : \
+ ((x) <= 2048 ? 11 : \
+ ((x) <= 4096 ? 12 : \
+ ((x) <= 8196 ? 13 : \
+ ((x) <= 16384 ? 14 : \
+ ((x) <= 32768 ? 15 : \
+ ((x) <= 65536 ? 16 : \
+ (0/0)))))))))))))))))
+
+_ODP_STATIC_ASSERT(ODP_CONFIG_PACKET_BUF_LEN_MIN >= 256,
+ "ODP Segment size must be a minimum of 256 bytes");
+
+_ODP_STATIC_ASSERT((ODP_CONFIG_PACKET_BUF_LEN_MIN % ODP_CACHE_LINE_SIZE) == 0,
+ "ODP Segment size must be a multiple of cache line size");
+
+_ODP_STATIC_ASSERT((ODP_CONFIG_PACKET_BUF_LEN_MAX %
+ ODP_CONFIG_PACKET_BUF_LEN_MIN) == 0,
+ "Packet max size must be a multiple of segment size");
+
+#define ODP_BUFFER_MAX_SEG \
+ (ODP_CONFIG_PACKET_BUF_LEN_MAX / ODP_CONFIG_PACKET_BUF_LEN_MIN)
+
+/* We can optimize storage of small raw buffers within metadata area */
+#define ODP_MAX_INLINE_BUF ((sizeof(void *)) * (ODP_BUFFER_MAX_SEG - 1))
+
+#define ODP_BUFFER_POOL_BITS ODP_BITSIZE(ODP_CONFIG_BUFFER_POOLS)
+#define ODP_BUFFER_SEG_BITS ODP_BITSIZE(ODP_BUFFER_MAX_SEG)
+#define ODP_BUFFER_INDEX_BITS (32 - ODP_BUFFER_POOL_BITS - ODP_BUFFER_SEG_BITS)
+#define ODP_BUFFER_PREFIX_BITS (ODP_BUFFER_POOL_BITS + ODP_BUFFER_INDEX_BITS)
+#define ODP_BUFFER_MAX_POOLS (1 << ODP_BUFFER_POOL_BITS)
+#define ODP_BUFFER_MAX_BUFFERS (1 << ODP_BUFFER_INDEX_BITS)
#define ODP_BUFFER_MAX_INDEX (ODP_BUFFER_MAX_BUFFERS - 2)
#define ODP_BUFFER_INVALID_INDEX (ODP_BUFFER_MAX_BUFFERS - 1)
-#define ODP_BUFS_PER_CHUNK 16
-#define ODP_BUFS_PER_SCATTER 4
-
-#define ODP_BUFFER_TYPE_CHUNK 0xffff
-
-
-#define ODP_BUFFER_POOL_BITS 4
-#define ODP_BUFFER_INDEX_BITS (32 - ODP_BUFFER_POOL_BITS)
-#define ODP_BUFFER_MAX_POOLS (1 << ODP_BUFFER_POOL_BITS)
-#define ODP_BUFFER_MAX_BUFFERS (1 << ODP_BUFFER_INDEX_BITS)
-
typedef union odp_buffer_bits_t {
uint32_t u32;
odp_buffer_t handle;
struct {
- uint32_t pool:ODP_BUFFER_POOL_BITS;
+#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN
+ uint32_t pool_id:ODP_BUFFER_POOL_BITS;
+ uint32_t index:ODP_BUFFER_INDEX_BITS;
+ uint32_t seg:ODP_BUFFER_SEG_BITS;
+#else
+ uint32_t seg:ODP_BUFFER_SEG_BITS;
uint32_t index:ODP_BUFFER_INDEX_BITS;
+ uint32_t pool_id:ODP_BUFFER_POOL_BITS;
+#endif
};
-} odp_buffer_bits_t;
+ struct {
+#if ODP_BYTE_ORDER == ODP_BIG_ENDIAN
+ uint32_t prefix:ODP_BUFFER_PREFIX_BITS;
+ uint32_t pfxseg:ODP_BUFFER_SEG_BITS;
+#else
+ uint32_t pfxseg:ODP_BUFFER_SEG_BITS;
+ uint32_t prefix:ODP_BUFFER_PREFIX_BITS;
+#endif
+ };
+} odp_buffer_bits_t;
/* forward declaration */
struct odp_buffer_hdr_t;
-
+/* Common buffer header */
typedef struct odp_buffer_hdr_t {
+ /* DPDK specific */
struct rte_mbuf mb; /* Underlying DPDK rte_mbuf */
- struct odp_buffer_hdr_t *next; /* Next buf in a list */
- int type; /* ODP buffer type; not DPDK buf type */
uint32_t index; /* Index in the rte_mempool */
+ uint8_t hole[2048];
+
+ struct odp_buffer_hdr_t *next; /* next buf in a list */
+ int allocator; /* allocating thread id */
+ odp_buffer_bits_t handle; /* handle */
+ union {
+ uint32_t all;
+ struct {
+ uint32_t zeroized:1; /* Zeroize buf data on free */
+ uint32_t hdrdata:1; /* Data is in buffer hdr */
+ };
+ } flags;
+ int type; /* buffer type */
+ size_t size; /* max data size */
+ odp_atomic_u32_t ref_count; /* reference count */
+ odp_buffer_pool_t pool_hdl; /* buffer pool handle */
+ union {
+ uint64_t buf_u64; /* user u64 */
+ void *buf_ctx; /* user context */
+ const void *buf_cctx; /* const alias for ctx */
+ void *udata_addr; /* user metadata addr */
+ };
+ size_t udata_size; /* size of user metadata */
+ uint32_t segcount; /* segment count */
+ uint32_t segsize; /* segment size */
+ void *addr[ODP_BUFFER_MAX_SEG]; /* block addrs */
+
} odp_buffer_hdr_t;
-int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf);
+typedef struct odp_buffer_hdr_stride {
+ uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_buffer_hdr_t))];
+} odp_buffer_hdr_stride;
+/* Ensure next header starts from 8 byte align */
+_ODP_STATIC_ASSERT((sizeof(odp_buffer_hdr_t) % 8) == 0,
+ "ODP_BUFFER_HDR_T__SIZE_ERROR");
+
+typedef struct odp_buf_blk_t {
+ struct odp_buf_blk_t *next;
+ struct odp_buf_blk_t *prev;
+} odp_buf_blk_t;
+
+/* Raw buffer header */
+typedef struct {
+ odp_buffer_hdr_t buf_hdr; /* common buffer header */
+} odp_raw_buffer_hdr_t;
+
+/* Free buffer marker */
+#define ODP_FREEBUF -1
+/* Forward declarations */
+odp_buffer_t buffer_alloc(odp_buffer_pool_t pool, size_t size);
#ifdef __cplusplus
}
@@ -19,16 +19,48 @@ extern "C" {
#endif
#include <odp_std_types.h>
+#include <odp_align.h>
+#include <odp_align_internal.h>
#include <odp_buffer_pool.h>
#include <odp_buffer_internal.h>
-#include <odp_align.h>
#include <odp_hints.h>
#include <odp_config.h>
#include <odp_debug.h>
+#include <odp_shared_memory.h>
+#include <odp_atomic.h>
+#include <odp_atomic_internal.h>
+#include <string.h>
-/* for DPDK */
+/* DPDK */
#include <rte_mempool.h>
+/**
+ * Buffer initialization routine prototype
+ *
+ * @note Routines of this type MAY be passed as part of the
+ * _odp_buffer_pool_init_t structure to be called whenever a
+ * buffer is allocated to initialize the user metadata
+ * associated with that buffer.
+ */
+typedef void (_odp_buf_init_t)(odp_buffer_t buf, void *buf_init_arg);
+
+/**
+ * Buffer pool initialization parameters
+ * Used to communicate buffer pool initialization options. Internal for now.
+ */
+typedef struct _odp_buffer_pool_init_t {
+ size_t udata_size; /**< Size of user metadata for each buffer */
+ _odp_buf_init_t *buf_init; /**< Buffer initialization routine to use */
+ void *buf_init_arg; /**< Argument to be passed to buf_init() */
+} _odp_buffer_pool_init_t; /**< Type of buffer initialization struct */
+
+/* Local cache for buffer alloc/free acceleration */
+typedef struct local_cache_t {
+ odp_buffer_hdr_t *buf_freelist; /* The local cache */
+ uint64_t bufallocs; /* Local buffer alloc count */
+ uint64_t buffrees; /* Local buffer free count */
+} local_cache_t;
+
/* Use ticketlock instead of spinlock */
#define POOL_USE_TICKETLOCK
@@ -38,11 +70,16 @@ extern "C" {
#ifdef POOL_USE_TICKETLOCK
#include <odp_ticketlock.h>
+#define POOL_LOCK(a) odp_ticketlock_lock(a)
+#define POOL_UNLOCK(a) odp_ticketlock_unlock(a)
+#define POOL_LOCK_INIT(a) odp_ticketlock_init(a)
#else
#include <odp_spinlock.h>
+#define POOL_LOCK(a) odp_spinlock_lock(a)
+#define POOL_UNLOCK(a) odp_spinlock_unlock(a)
+#define POOL_LOCK_INIT(a) odp_spinlock_init(a)
#endif
-
struct pool_entry_s {
#ifdef POOL_USE_TICKETLOCK
odp_ticketlock_t lock ODP_ALIGNED_CACHE;
@@ -50,38 +87,294 @@ struct pool_entry_s {
odp_spinlock_t lock ODP_ALIGNED_CACHE;
#endif
- uint64_t free_bufs;
char name[ODP_BUFFER_POOL_NAME_LEN];
-
-
- odp_buffer_pool_t pool ODP_ALIGNED_CACHE;
- uintptr_t buf_base;
- size_t buf_size;
- size_t buf_offset;
- uint64_t num_bufs;
- void *pool_base_addr;
- uint64_t pool_size;
- size_t payload_size;
- size_t payload_align;
- int buf_type;
- size_t hdr_size;
+ odp_buffer_pool_param_t params;
+ _odp_buffer_pool_init_t init_params;
+ odp_buffer_pool_t pool_hdl;
+ uint32_t pool_id;
+ odp_shm_t pool_shm;
+ union {
+ uint32_t all;
+ struct {
+ uint32_t has_name:1;
+ uint32_t user_supplied_shm:1;
+ uint32_t unsegmented:1;
+ uint32_t zeroized:1;
+ uint32_t predefined:1;
+ };
+ } flags;
+ uint32_t quiesced;
+ uint32_t low_wm_assert;
+ uint8_t *pool_base_addr;
+ uint8_t *pool_mdata_addr;
+ size_t pool_size;
+ uint32_t buf_align;
+ uint32_t buf_stride;
+ _odp_atomic_ptr_t buf_freelist;
+ _odp_atomic_ptr_t blk_freelist;
+ odp_atomic_u32_t bufcount;
+ odp_atomic_u32_t blkcount;
+ odp_atomic_u64_t bufallocs;
+ odp_atomic_u64_t buffrees;
+ odp_atomic_u64_t blkallocs;
+ odp_atomic_u64_t blkfrees;
+ odp_atomic_u64_t bufempty;
+ odp_atomic_u64_t blkempty;
+ odp_atomic_u64_t high_wm_count;
+ odp_atomic_u64_t low_wm_count;
+ uint32_t seg_size;
+ uint32_t high_wm;
+ uint32_t low_wm;
+ uint32_t headroom;
+ uint32_t tailroom;
};
+typedef union pool_entry_u {
+ struct pool_entry_s s;
+
+ uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct pool_entry_s))];
+} pool_entry_t;
extern void *pool_entry_ptr[];
+#if defined(ODP_CONFIG_SECURE_POOLS) && (ODP_CONFIG_SECURE_POOLS == 1)
+#define buffer_is_secure(buf) (buf->flags.zeroized)
+#define pool_is_secure(pool) (pool->flags.zeroized)
+#else
+#define buffer_is_secure(buf) 0
+#define pool_is_secure(pool) 0
+#endif
+
+#define TAG_ALIGN ((size_t)16)
+
+#define odp_cs(ptr, old, new) \
+ _odp_atomic_ptr_cmp_xchg_strong(&ptr, (void **)&old, (void *)new, \
+ _ODP_MEMMODEL_SC, \
+ _ODP_MEMMODEL_SC)
-static inline void *get_pool_entry(odp_buffer_pool_t pool_id)
+/* Helper functions for pointer tagging to avoid ABA race conditions */
+#define odp_tag(ptr) \
+ (((size_t)ptr) & (TAG_ALIGN - 1))
+
+#define odp_detag(ptr) \
+ ((void *)(((size_t)ptr) & -TAG_ALIGN))
+
+#define odp_retag(ptr, tag) \
+ ((void *)(((size_t)ptr) | odp_tag(tag)))
+
+
+static inline void *get_blk(struct pool_entry_s *pool)
+{
+ void *oldhead, *myhead, *newhead;
+
+ oldhead = _odp_atomic_ptr_load(&pool->blk_freelist, _ODP_MEMMODEL_ACQ);
+
+ do {
+ size_t tag = odp_tag(oldhead);
+ myhead = odp_detag(oldhead);
+ if (odp_unlikely(myhead == NULL))
+ break;
+ newhead = odp_retag(((odp_buf_blk_t *)myhead)->next, tag + 1);
+ } while (odp_cs(pool->blk_freelist, oldhead, newhead) == 0);
+
+ if (odp_unlikely(myhead == NULL))
+ odp_atomic_inc_u64(&pool->blkempty);
+ else
+ odp_atomic_dec_u32(&pool->blkcount);
+
+ return (void *)myhead;
+}
+
+static inline void ret_blk(struct pool_entry_s *pool, void *block)
+{
+ void *oldhead, *myhead, *myblock;
+
+ oldhead = _odp_atomic_ptr_load(&pool->blk_freelist, _ODP_MEMMODEL_ACQ);
+
+ do {
+ size_t tag = odp_tag(oldhead);
+ myhead = odp_detag(oldhead);
+ ((odp_buf_blk_t *)block)->next = myhead;
+ myblock = odp_retag(block, tag + 1);
+ } while (odp_cs(pool->blk_freelist, oldhead, myblock) == 0);
+
+ odp_atomic_inc_u32(&pool->blkcount);
+ odp_atomic_inc_u64(&pool->blkfrees);
+}
+
+static inline odp_buffer_hdr_t *get_buf(struct pool_entry_s *pool)
+{
+ odp_buffer_hdr_t *oldhead, *myhead, *newhead;
+
+ oldhead = _odp_atomic_ptr_load(&pool->buf_freelist, _ODP_MEMMODEL_ACQ);
+
+ do {
+ size_t tag = odp_tag(oldhead);
+ myhead = odp_detag(oldhead);
+ if (odp_unlikely(myhead == NULL))
+ break;
+ newhead = odp_retag(myhead->next, tag + 1);
+ } while (odp_cs(pool->buf_freelist, oldhead, newhead) == 0);
+
+ if (odp_unlikely(myhead == NULL)) {
+ odp_atomic_inc_u64(&pool->bufempty);
+ } else {
+ uint64_t bufcount =
+ odp_atomic_fetch_sub_u32(&pool->bufcount, 1) - 1;
+
+ /* Check for low watermark condition */
+ if (bufcount == pool->low_wm && !pool->low_wm_assert) {
+ pool->low_wm_assert = 1;
+ odp_atomic_inc_u64(&pool->low_wm_count);
+ }
+
+ odp_atomic_inc_u64(&pool->bufallocs);
+ myhead->next = myhead; /* Mark buffer allocated */
+ myhead->allocator = odp_thread_id();
+ }
+
+ return (void *)myhead;
+}
+
+static inline void ret_buf(struct pool_entry_s *pool, odp_buffer_hdr_t *buf)
+{
+ odp_buffer_hdr_t *oldhead, *myhead, *mybuf;
+
+ buf->allocator = ODP_FREEBUF; /* Mark buffer free */
+
+ if (!buf->flags.hdrdata && buf->type != ODP_BUFFER_TYPE_RAW) {
+ while (buf->segcount > 0) {
+ if (buffer_is_secure(buf) || pool_is_secure(pool))
+ memset(buf->addr[buf->segcount - 1],
+ 0, buf->segsize);
+ ret_blk(pool, buf->addr[--buf->segcount]);
+ }
+ buf->size = 0;
+ }
+
+ oldhead = _odp_atomic_ptr_load(&pool->buf_freelist, _ODP_MEMMODEL_ACQ);
+
+ do {
+ size_t tag = odp_tag(oldhead);
+ myhead = odp_detag(oldhead);
+ buf->next = myhead;
+ mybuf = odp_retag(buf, tag + 1);
+ } while (odp_cs(pool->buf_freelist, oldhead, mybuf) == 0);
+
+ uint64_t bufcount = odp_atomic_fetch_add_u32(&pool->bufcount, 1) + 1;
+
+ /* Check if low watermark condition should be deasserted */
+ if (bufcount == pool->high_wm && pool->low_wm_assert) {
+ pool->low_wm_assert = 0;
+ odp_atomic_inc_u64(&pool->high_wm_count);
+ }
+
+ odp_atomic_inc_u64(&pool->buffrees);
+}
+
+static inline void *get_local_buf(local_cache_t *buf_cache,
+ struct pool_entry_s *pool,
+ size_t totsize)
+{
+ odp_buffer_hdr_t *buf = buf_cache->buf_freelist;
+
+ if (odp_likely(buf != NULL)) {
+ buf_cache->buf_freelist = buf->next;
+
+ if (odp_unlikely(buf->size < totsize)) {
+ intmax_t needed = totsize - buf->size;
+
+ do {
+ void *blk = get_blk(pool);
+ if (odp_unlikely(blk == NULL)) {
+ ret_buf(pool, buf);
+ buf_cache->buffrees--;
+ return NULL;
+ }
+ buf->addr[buf->segcount++] = blk;
+ needed -= pool->seg_size;
+ } while (needed > 0);
+
+ buf->size = buf->segcount * pool->seg_size;
+ }
+
+ buf_cache->bufallocs++;
+ buf->allocator = odp_thread_id(); /* Mark buffer allocated */
+ }
+
+ return buf;
+}
+
+static inline void ret_local_buf(local_cache_t *buf_cache,
+ odp_buffer_hdr_t *buf)
+{
+ buf->allocator = ODP_FREEBUF;
+ buf->next = buf_cache->buf_freelist;
+ buf_cache->buf_freelist = buf;
+
+ buf_cache->buffrees++;
+}
+
+static inline void flush_cache(local_cache_t *buf_cache,
+ struct pool_entry_s *pool)
+{
+ odp_buffer_hdr_t *buf = buf_cache->buf_freelist;
+ uint32_t flush_count = 0;
+
+ while (buf != NULL) {
+ odp_buffer_hdr_t *next = buf->next;
+ ret_buf(pool, buf);
+ buf = next;
+ flush_count++;
+ }
+
+ odp_atomic_add_u64(&pool->bufallocs, buf_cache->bufallocs);
+ odp_atomic_add_u64(&pool->buffrees, buf_cache->buffrees - flush_count);
+
+ buf_cache->buf_freelist = NULL;
+ buf_cache->bufallocs = 0;
+ buf_cache->buffrees = 0;
+}
+
+static inline odp_buffer_pool_t pool_index_to_handle(uint32_t pool_id)
+{
+ return pool_id;
+}
+
+static inline uint32_t pool_handle_to_index(odp_buffer_pool_t pool_hdl)
+{
+ return pool_hdl;
+}
+
+static inline void *get_pool_entry(uint32_t pool_id)
{
return pool_entry_ptr[pool_id];
}
+static inline pool_entry_t *odp_pool_to_entry(odp_buffer_pool_t pool)
+{
+ return (pool_entry_t *)get_pool_entry(pool_handle_to_index(pool));
+}
+
+static inline pool_entry_t *odp_buf_to_pool(odp_buffer_hdr_t *buf)
+{
+ return odp_pool_to_entry(buf->pool_hdl);
+}
-static inline odp_buffer_hdr_t *odp_buf_to_hdr(odp_buffer_t buf)
+static inline uint32_t odp_buffer_pool_segment_size(odp_buffer_pool_t pool)
{
- return (odp_buffer_hdr_t *)buf;
+ return odp_pool_to_entry(pool)->s.seg_size;
}
+static inline uint32_t odp_buffer_pool_headroom(odp_buffer_pool_t pool)
+{
+ return odp_pool_to_entry(pool)->s.headroom;
+}
+
+static inline uint32_t odp_buffer_pool_tailroom(odp_buffer_pool_t pool)
+{
+ return odp_pool_to_entry(pool)->s.tailroom;
+}
#ifdef __cplusplus
}
@@ -8,10 +8,8 @@
#define ODP_PACKET_DPDK_H
#include <stdint.h>
-#include <net/if.h>
#include <odph_eth.h>
-#include <odph_packet.h>
#include <odp_align.h>
#include <odp_debug.h>
#include <odp_packet.h>
@@ -22,6 +22,7 @@ extern "C" {
#include <odp_debug.h>
#include <odp_buffer_internal.h>
#include <odp_buffer_pool_internal.h>
+#include <odp_buffer_inlines.h>
#include <odp_packet.h>
#include <odp_packet_io.h>
@@ -43,6 +44,7 @@ typedef union {
uint32_t vlan:1; /**< VLAN hdr found */
uint32_t vlan_qinq:1; /**< Stacked VLAN found, QinQ */
+ uint32_t snap:1; /**< SNAP */
uint32_t arp:1; /**< ARP */
uint32_t ipv4:1; /**< IPv4 */
@@ -53,13 +55,14 @@ typedef union {
uint32_t udp:1; /**< UDP */
uint32_t tcp:1; /**< TCP */
+ uint32_t tcpopt:1; /**< TCP options present */
uint32_t sctp:1; /**< SCTP */
uint32_t icmp:1; /**< ICMP */
};
} input_flags_t;
-ODP_STATIC_ASSERT(sizeof(input_flags_t) == sizeof(uint32_t),
- "INPUT_FLAGS_SIZE_ERROR");
+_ODP_STATIC_ASSERT(sizeof(input_flags_t) == sizeof(uint32_t),
+ "INPUT_FLAGS_SIZE_ERROR");
/**
* Packet error flags
@@ -70,7 +73,9 @@ typedef union {
struct {
/* Bitfield flags for each detected error */
+ uint32_t app_error:1; /**< Error bit for application use */
uint32_t frame_len:1; /**< Frame length error */
+ uint32_t snap_len:1; /**< Snap length error */
uint32_t l2_chksum:1; /**< L2 checksum error, checks TBD */
uint32_t ip_err:1; /**< IP error, checks TBD */
uint32_t tcp_err:1; /**< TCP error, checks TBD */
@@ -78,8 +83,8 @@ typedef union {
};
} error_flags_t;
-ODP_STATIC_ASSERT(sizeof(error_flags_t) == sizeof(uint32_t),
- "ERROR_FLAGS_SIZE_ERROR");
+_ODP_STATIC_ASSERT(sizeof(error_flags_t) == sizeof(uint32_t),
+ "ERROR_FLAGS_SIZE_ERROR");
/**
* Packet output flags
@@ -90,12 +95,15 @@ typedef union {
struct {
/* Bitfield flags for each output option */
- uint32_t l4_chksum:1; /**< Request L4 checksum calculation */
+ uint32_t l3_chksum_set:1; /**< L3 chksum bit is valid */
+ uint32_t l3_chksum:1; /**< L3 chksum override */
+ uint32_t l4_chksum_set:1; /**< L3 chksum bit is valid */
+ uint32_t l4_chksum:1; /**< L4 chksum override */
};
} output_flags_t;
-ODP_STATIC_ASSERT(sizeof(output_flags_t) == sizeof(uint32_t),
- "OUTPUT_FLAGS_SIZE_ERROR");
+_ODP_STATIC_ASSERT(sizeof(output_flags_t) == sizeof(uint32_t),
+ "OUTPUT_FLAGS_SIZE_ERROR");
/**
* Internal Packet header
@@ -108,28 +116,131 @@ typedef struct {
error_flags_t error_flags;
output_flags_t output_flags;
- uint32_t frame_offset; /**< offset to start of frame, even on error */
uint32_t l2_offset; /**< offset to L2 hdr, e.g. Eth */
uint32_t l3_offset; /**< offset to L3 hdr, e.g. IPv4, IPv6 */
uint32_t l4_offset; /**< offset to L4 hdr (TCP, UDP, SCTP, also ICMP) */
+ uint32_t payload_offset; /**< offset to payload */
- uint64_t user_ctx; /**< user context */
+ uint32_t vlan_s_tag; /**< Parsed 1st VLAN header (S-TAG) */
+ uint32_t vlan_c_tag; /**< Parsed 2nd VLAN header (C-TAG) */
+ uint32_t l3_protocol; /**< Parsed L3 protocol */
+ uint32_t l3_len; /**< Layer 3 length */
+ uint32_t l4_protocol; /**< Parsed L4 protocol */
+ uint32_t l4_len; /**< Layer 4 length */
+
+ uint32_t frame_len;
+ uint32_t headroom;
+ uint32_t tailroom;
odp_pktio_t input;
} odp_packet_hdr_t;
+typedef struct odp_packet_hdr_stride {
+ uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_packet_hdr_t))];
+} odp_packet_hdr_stride;
+
+_ODP_STATIC_ASSERT(sizeof(odp_packet_hdr_t) % sizeof(uint64_t) == 0,
+ "ODP_PACKET_HDR_T__SIZE_ERR2");
+
/**
* Return the packet header
*/
static inline odp_packet_hdr_t *odp_packet_hdr(odp_packet_t pkt)
{
- return (odp_packet_hdr_t *)pkt;
+ return (odp_packet_hdr_t *)odp_buf_to_hdr((odp_buffer_t)pkt);
}
/**
- * Parse packet and set internal metadata
+ * Initialize packet buffer
*/
-void odp_packet_parse(odp_packet_t pkt, size_t len, size_t l2_offset);
+static inline void packet_init(pool_entry_t *pool,
+ odp_packet_hdr_t *pkt_hdr,
+ size_t size)
+{
+ /*
+ * Reset parser metadata. Note that we clear via memset to make
+ * this routine indepenent of any additional adds to packet metadata.
+ */
+ struct rte_mbuf *mb;
+ uint8_t *start;
+
+ mb = (struct rte_mbuf *)&pkt_hdr->buf_hdr;
+ mb->buf_len = size;
+ start = (uint8_t *)mb->buf_addr;
+ memset(start, 0, mb->buf_len);
+
+ /*
+ * Packet headroom is set from the pool's headroom
+ * Packet tailroom is rounded up to fill the last
+ * segment occupied by the allocated length.
+ */
+ pkt_hdr->frame_len = size;
+ pkt_hdr->headroom = pool->s.headroom;
+ pkt_hdr->tailroom =
+ (pool->s.seg_size * pkt_hdr->buf_hdr.segcount) -
+ (pool->s.headroom + size);
+}
+
+static inline void *packet_map(odp_packet_hdr_t *pkt_hdr,
+ uint32_t offset, uint32_t *seglen)
+{
+ struct rte_mbuf *mbuf = (struct rte_mbuf *)&pkt_hdr->buf_hdr;
+ if (offset > pkt_hdr->frame_len)
+ return NULL;
+
+ if (seglen != NULL) {
+ *seglen = mbuf->pkt.data_len;
+ pkt_hdr->frame_len = mbuf->pkt.data_len;
+ }
+ return (void *)((uint8_t *)mbuf->pkt.data + offset);
+}
+
+#define pull_offset(x, len) (x = x < len ? 0 : x - len)
+
+static inline void push_head(odp_packet_hdr_t *pkt_hdr, size_t len)
+{
+ pkt_hdr->headroom -= len;
+ pkt_hdr->frame_len += len;
+ pkt_hdr->l2_offset += len;
+ pkt_hdr->l3_offset += len;
+ pkt_hdr->l4_offset += len;
+}
+
+static inline void pull_head(odp_packet_hdr_t *pkt_hdr, size_t len)
+{
+ pkt_hdr->headroom += len;
+ pkt_hdr->frame_len -= len;
+ pull_offset(pkt_hdr->l2_offset, len);
+ pull_offset(pkt_hdr->l3_offset, len);
+ pull_offset(pkt_hdr->l4_offset, len);
+}
+
+static inline void push_tail(odp_packet_hdr_t *pkt_hdr, size_t len)
+{
+ pkt_hdr->tailroom -= len;
+ pkt_hdr->frame_len += len;
+}
+
+
+static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, size_t len)
+{
+ pkt_hdr->tailroom += len;
+ pkt_hdr->frame_len -= len;
+}
+
+static inline void packet_set_len(odp_packet_t pkt, uint32_t len)
+{
+ odp_packet_hdr(pkt)->frame_len = len;
+}
+
+/* Forward declarations */
+int _odp_packet_copy_to_packet(odp_packet_t srcpkt, uint32_t srcoffset,
+ odp_packet_t dstpkt, uint32_t dstoffset,
+ uint32_t len);
+
+odp_packet_t _odp_packet_alloc(odp_buffer_pool_t pool_hdl);
+
+int _odp_packet_parse(odp_packet_t pkt);
#ifdef __cplusplus
}
@@ -20,17 +20,36 @@ extern "C" {
#include <odp_spinlock.h>
#include <odp_packet_socket.h>
-#ifdef ODP_HAVE_NETMAP
-#include <odp_packet_netmap.h>
-#endif
+#include <odp_classification_datamodel.h>
+#include <odp_align_internal.h>
+
+#include <odp_config.h>
+#include <odp_hints.h>
+#include <linux/if.h>
+
+/* DPDK */
#include <odp_packet_dpdk.h>
+/**
+ * Packet IO types
+ */
+typedef enum {
+ ODP_PKTIO_TYPE_SOCKET_BASIC = 0x1,
+ ODP_PKTIO_TYPE_SOCKET_MMSG,
+ ODP_PKTIO_TYPE_SOCKET_MMAP,
+} odp_pktio_type_t;
+
struct pktio_entry {
odp_spinlock_t lock; /**< entry spinlock */
int taken; /**< is entry taken(1) or free(0) */
odp_queue_t inq_default; /**< default input queue, if set */
odp_queue_t outq_default; /**< default out queue */
- odp_pktio_params_t params; /**< pktio parameters */
+ odp_pktio_type_t type; /**< pktio type */
+ pkt_sock_t pkt_sock; /**< using socket API for IO */
+ pkt_sock_mmap_t pkt_sock_mmap; /**< using socket mmap API for IO */
+ classifier_t cls; /**< classifier linked with this pktio*/
+ char name[IFNAMSIZ]; /**< name of pktio provided to
+ pktio_open() */
pkt_dpdk_t pkt_dpdk; /**< using DPDK API for IO */
};
@@ -39,6 +58,21 @@ typedef union {
uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct pktio_entry))];
} pktio_entry_t;
+typedef struct {
+ pktio_entry_t entries[ODP_CONFIG_PKTIO_ENTRIES];
+} pktio_table_t;
+
+extern void *pktio_entry_ptr[];
+
+
+static inline pktio_entry_t *get_pktio_entry(odp_pktio_t id)
+{
+ if (odp_unlikely(id == ODP_PKTIO_INVALID ||
+ id > ODP_CONFIG_PKTIO_ENTRIES))
+ return NULL;
+
+ return pktio_entry_ptr[id - 1];
+}
#ifdef __cplusplus
}
#endif
@@ -5,8 +5,10 @@
*/
#include <odp_buffer.h>
-#include <odp_buffer_internal.h>
#include <odp_buffer_pool_internal.h>
+#include <odp_buffer_internal.h>
+#include <odp_buffer_inlines.h>
+#include <odp_debug_internal.h>
#include <string.h>
#include <stdio.h>
@@ -44,13 +46,13 @@ int odp_buffer_is_valid(odp_buffer_t buf)
}
-int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf)
+int odp_buffer_snprint(char *str, uint32_t n, odp_buffer_t buf)
{
odp_buffer_hdr_t *hdr;
int len = 0;
if (!odp_buffer_is_valid(buf)) {
- printf("Buffer is not valid.\n");
+ ODP_PRINT("Buffer is not valid.\n");
return len;
}
@@ -67,7 +69,9 @@ int odp_buffer_snprint(char *str, size_t n, odp_buffer_t buf)
len += snprintf(&str[len], n-len,
" size %u\n", hdr->mb.buf_len);
len += snprintf(&str[len], n-len,
- " ref_count %i\n", hdr->mb.refcnt);
+ " ref_count %i\n",
+ odp_atomic_load_u32((odp_atomic_u32_t *)
+ &hdr->mb.refcnt));
len += snprintf(&str[len], n-len,
" dpdk type %i\n", hdr->mb.type);
len += snprintf(&str[len], n-len,
@@ -86,5 +90,5 @@ void odp_buffer_print(odp_buffer_t buf)
len = odp_buffer_snprint(str, max_len-1, buf);
str[len] = 0;
- printf("\n%s\n", str);
+ ODP_PRINT("\n%s\n", str);
}
@@ -6,16 +6,19 @@
#include <odp_std_types.h>
#include <odp_buffer_pool.h>
-#include <odp_buffer_pool_internal.h>
#include <odp_buffer_internal.h>
+#include <odp_buffer_pool_internal.h>
+#include <odp_buffer_inlines.h>
#include <odp_packet_internal.h>
#include <odp_timer_internal.h>
+#include <odp_align_internal.h>
#include <odp_shared_memory.h>
#include <odp_align.h>
#include <odp_internal.h>
#include <odp_config.h>
#include <odp_hints.h>
-#include <odp_debug.h>
+#include <odp_debug_internal.h>
+#include <odp_atomic_internal.h>
#include <string.h>
#include <stdlib.h>
@@ -26,44 +29,27 @@
#define MBUF_SIZE (2048 + sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM)
#define NB_MBUF 32768
-#ifdef POOL_USE_TICKETLOCK
-#include <odp_ticketlock.h>
-#define LOCK(a) odp_ticketlock_lock(a)
-#define UNLOCK(a) odp_ticketlock_unlock(a)
-#define LOCK_INIT(a) odp_ticketlock_init(a)
-#else
-#include <odp_spinlock.h>
-#define LOCK(a) odp_spinlock_lock(a)
-#define UNLOCK(a) odp_spinlock_unlock(a)
-#define LOCK_INIT(a) odp_spinlock_init(a)
-#endif
-
-
#if ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS
#error ODP_CONFIG_BUFFER_POOLS > ODP_BUFFER_MAX_POOLS
#endif
-#define NULL_INDEX ((uint32_t)-1)
-
-union buffer_type_any_u {
+typedef union buffer_type_any_u {
odp_buffer_hdr_t buf;
odp_packet_hdr_t pkt;
odp_timeout_hdr_t tmo;
-};
-
-typedef union buffer_type_any_u odp_any_buffer_hdr_t;
+} odp_anybuf_t;
-typedef union pool_entry_u {
- struct pool_entry_s s;
-
- uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(struct pool_entry_s))];
-
-} pool_entry_t;
+/* Any buffer type header */
+typedef struct {
+ odp_anybuf_t any_hdr; /* any buffer type */
+} odp_any_buffer_hdr_t;
+typedef struct odp_any_hdr_stride {
+ uint8_t pad[ODP_CACHE_LINE_SIZE_ROUNDUP(sizeof(odp_any_buffer_hdr_t))];
+} odp_any_hdr_stride;
typedef struct pool_table_t {
pool_entry_t pool[ODP_CONFIG_BUFFER_POOLS];
-
} pool_table_t;
@@ -73,6 +59,8 @@ static pool_table_t *pool_tbl;
/* Pool entry pointers (for inlining) */
void *pool_entry_ptr[ODP_CONFIG_BUFFER_POOLS];
+/* Local cache for buffer alloc/free acceleration */
+static __thread local_cache_t local_cache[ODP_CONFIG_BUFFER_POOLS];
int odp_buffer_pool_init_global(void)
{
@@ -94,9 +82,9 @@ int odp_buffer_pool_init_global(void)
for (i = 0; i < ODP_CONFIG_BUFFER_POOLS; i++) {
/* init locks */
pool_entry_t *pool = &pool_tbl->pool[i];
- LOCK_INIT(&pool->s.lock);
- pool->s.pool = i;
-
+ POOL_LOCK_INIT(&pool->s.lock);
+ pool->s.pool_hdl = pool_index_to_handle(i);
+ pool->s.pool_id = i;
pool_entry_ptr[i] = pool;
}
@@ -187,10 +175,13 @@ odp_dpdk_mbuf_ctor(struct rte_mempool *mp,
buf_hdr->index = i;
}
+/**
+ * Buffer pool creation
+ */
+
odp_buffer_pool_t odp_buffer_pool_create(const char *name,
- void *base_addr, uint64_t size,
- size_t buf_size, size_t buf_align,
- int buf_type)
+ odp_shm_t shm,
+ odp_buffer_pool_param_t *params)
{
struct rte_mempool *pool = NULL;
struct mbuf_pool_ctor_arg mbp_ctor_arg;
@@ -198,47 +189,39 @@ odp_buffer_pool_t odp_buffer_pool_create(const char *name,
unsigned mb_size;
size_t hdr_size;
- /* Not used for rte_mempool; the new ODP buffer management introduces
- * rte_mempool_create_from_region where base_addr makes sense */
- (void)base_addr;
-
- /* buf_align will be removed soon, no need to wory about it */
- (void)buf_align;
-
- ODP_DBG("odp_buffer_pool_create: %s, %lx, %u, %u, %u, %d\n", name,
- (uint64_t) base_addr, (unsigned) size,
- (unsigned) buf_size, (unsigned) buf_align,
- buf_type);
+ ODP_DBG("odp_buffer_pool_create: %s, %lx, %u, %u, %d\n", name,
+ (uint64_t) shm, (unsigned) params->buf_size,
+ (unsigned) params->buf_align, params->buf_type);
- switch (buf_type) {
+ switch (params->buf_type) {
case ODP_BUFFER_TYPE_RAW:
hdr_size = sizeof(odp_buffer_hdr_t);
- mbp_ctor_arg.seg_buf_size = (uint16_t) buf_size;
+ mbp_ctor_arg.seg_buf_size = (uint16_t) params->buf_size;
break;
case ODP_BUFFER_TYPE_PACKET:
hdr_size = sizeof(odp_packet_hdr_t);
mbp_ctor_arg.seg_buf_size =
- (uint16_t) (RTE_PKTMBUF_HEADROOM + buf_size);
+ (uint16_t) (RTE_PKTMBUF_HEADROOM + params->buf_size);
break;
case ODP_BUFFER_TYPE_TIMEOUT:
hdr_size = sizeof(odp_timeout_hdr_t);
- mbp_ctor_arg.seg_buf_size = (uint16_t) buf_size;
+ mbp_ctor_arg.seg_buf_size = (uint16_t) params->buf_size;
break;
case ODP_BUFFER_TYPE_ANY:
hdr_size = sizeof(odp_any_buffer_hdr_t);
mbp_ctor_arg.seg_buf_size =
- (uint16_t) (RTE_PKTMBUF_HEADROOM + buf_size);
+ (uint16_t) (RTE_PKTMBUF_HEADROOM + params->buf_size);
break;
default:
- ODP_ERR("odp_buffer_pool_create: Bad type %i\n", buf_type);
+ ODP_ERR("odp_buffer_pool_create: Bad type %i\n",
+ params->buf_type);
return ODP_BUFFER_POOL_INVALID;
- break;
}
mb_ctor_arg.seg_buf_offset =
(uint16_t) ODP_CACHE_LINE_SIZE_ROUNDUP(hdr_size);
mb_ctor_arg.seg_buf_size = mbp_ctor_arg.seg_buf_size;
- mb_ctor_arg.buf_type = buf_type;
+ mb_ctor_arg.buf_type = params->buf_type;
mb_size = mb_ctor_arg.seg_buf_offset + mb_ctor_arg.seg_buf_size;
pool = rte_mempool_create(name, NB_MBUF,
@@ -267,6 +250,61 @@ odp_buffer_pool_t odp_buffer_pool_lookup(const char *name)
return (odp_buffer_pool_t)mp;
}
+odp_buffer_t buffer_alloc(odp_buffer_pool_t pool_hdl, size_t size)
+{
+ uint32_t pool_id = pool_handle_to_index(pool_hdl);
+ pool_entry_t *pool = get_pool_entry(pool_id);
+ uintmax_t totsize = pool->s.headroom + size + pool->s.tailroom;
+ odp_anybuf_t *buf;
+
+ /* Reject oversized allocation requests */
+ if ((pool->s.flags.unsegmented && totsize > pool->s.seg_size) ||
+ (!pool->s.flags.unsegmented &&
+ totsize > ODP_CONFIG_PACKET_BUF_LEN_MAX))
+ return ODP_BUFFER_INVALID;
+
+ /* Try to satisfy request from the local cache */
+ buf = (odp_anybuf_t *)(void *)get_local_buf(&local_cache[pool_id],
+ &pool->s, totsize);
+
+ /* If cache is empty, satisfy request from the pool */
+ if (odp_unlikely(buf == NULL)) {
+ buf = (odp_anybuf_t *)(void *)get_buf(&pool->s);
+
+ if (odp_unlikely(buf == NULL))
+ return ODP_BUFFER_INVALID;
+
+ /* Get blocks for this buffer, if pool uses application data */
+ if (buf->buf.size < totsize) {
+ intmax_t needed = totsize - buf->buf.size;
+ do {
+ uint8_t *blk = get_blk(&pool->s);
+ if (blk == NULL) {
+ ret_buf(&pool->s, &buf->buf);
+ return ODP_BUFFER_INVALID;
+ }
+ buf->buf.addr[buf->buf.segcount++] = blk;
+ needed -= pool->s.seg_size;
+ } while (needed > 0);
+ buf->buf.size = buf->buf.segcount * pool->s.seg_size;
+ }
+ }
+
+ /* By default, buffers inherit their pool's zeroization setting */
+ buf->buf.flags.zeroized = pool->s.flags.zeroized;
+
+ if (buf->buf.type == ODP_BUFFER_TYPE_PACKET) {
+ packet_init(pool, &buf->pkt, size);
+
+ if (pool->s.init_params.buf_init != NULL)
+ (*pool->s.init_params.buf_init)
+ (buf->buf.handle.handle,
+ pool->s.init_params.buf_init_arg);
+ }
+
+ return odp_hdr_to_buf(&buf->buf);
+}
+
odp_buffer_t odp_buffer_alloc(odp_buffer_pool_t pool_id)
{
@@ -279,6 +317,15 @@ void odp_buffer_free(odp_buffer_t buf)
rte_pktmbuf_free((struct rte_mbuf *)buf);
}
+void _odp_flush_caches(void)
+{
+ int i;
+
+ for (i = 0; i < ODP_CONFIG_BUFFER_POOLS; i++) {
+ pool_entry_t *pool = get_pool_entry(i);
+ flush_cache(&local_cache[i], &pool->s);
+ }
+}
void odp_buffer_pool_print(odp_buffer_pool_t pool_id)
{
@@ -7,6 +7,8 @@
#include <odp_init.h>
#include <odp_internal.h>
#include <odp_debug.h>
+#include <odp_debug_internal.h>
+#include <odp_system_info.h>
#include <odp_packet_dpdk.h>
int odp_init_dpdk(void)
@@ -45,10 +47,9 @@ int odp_init_dpdk(void)
return 0;
}
-int odp_init_global(void)
+int odp_init_global(odp_init_t *params ODP_UNUSED,
+ odp_platform_init_t *platform_params ODP_UNUSED)
{
- odp_thread_init_global();
-
odp_system_info_init();
if (odp_init_dpdk()) {
@@ -61,6 +62,11 @@ int odp_init_global(void)
return -1;
}
+ if (odp_thread_init_global()) {
+ ODP_ERR("ODP thread init failed.\n");
+ return -1;
+ }
+
if (odp_buffer_pool_init_global()) {
ODP_ERR("ODP buffer pool init failed.\n");
return -1;
@@ -91,13 +97,26 @@ int odp_init_global(void)
return -1;
}
+ if (odp_classification_init_global()) {
+ ODP_ERR("ODP classification init failed.\n");
+ return -1;
+ }
+
return 0;
}
+int odp_term_global(void)
+{
+ ODP_UNIMPLEMENTED();
+ return 0;
+}
-int odp_init_local(int thr_id)
+int odp_init_local(void)
{
- odp_thread_init_local(thr_id);
+ if (odp_thread_init_local()) {
+ ODP_ERR("ODP thread local init failed.\n");
+ return -1;
+ }
if (odp_pktio_init_local()) {
ODP_ERR("ODP packet io local init failed.\n");
@@ -111,3 +130,9 @@ int odp_init_local(int thr_id)
return 0;
}
+
+int odp_term_local(void)
+{
+ ODP_UNIMPLEMENTED();
+ return 0;
+}
@@ -8,6 +8,9 @@
#define _GNU_SOURCE
#endif
#include <sched.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/wait.h>
#include <stdlib.h>
#include <string.h>
@@ -19,28 +22,30 @@
#include <odp_thread.h>
#include <odp_init.h>
#include <odp_system_info.h>
+#include <odp_debug_internal.h>
#include <rte_lcore.h>
typedef struct {
- int thr_id;
void *(*start_routine) (void *);
void *arg;
-
} odp_start_args_t;
-
static void *odp_run_start_routine(void *arg)
{
odp_start_args_t *start_args = arg;
/* ODP thread local init */
- odp_init_local(start_args->thr_id);
+ if (odp_init_local()) {
+ ODP_ERR("Local init failed\n");
+ return NULL;
+ }
- return start_args->start_routine(start_args->arg);
+ void *ret = start_args->start_routine(start_args->arg);
+ _odp_flush_caches();
+ return ret;
}
-
void odph_linux_pthread_create(odph_linux_pthread_t *thread_tbl, int num,
int first_core, void *(*start_routine) (void *), void *arg)
{
@@ -62,12 +67,12 @@ void odph_linux_pthread_create(odph_linux_pthread_t *thread_tbl, int num,
cpu = (first_core + i) % core_count;
start_args = malloc(sizeof(odp_start_args_t));
+ if (start_args == NULL)
+ ODP_ABORT("Malloc failed");
memset(start_args, 0, sizeof(odp_start_args_t));
start_args->start_routine = start_routine;
start_args->arg = arg;
- odp_thread_create(cpu);
- start_args->thr_id = cpu;
/* If not master core */
if (cpu != 0) {
rte_eal_remote_launch(
@@ -94,3 +99,99 @@ void odph_linux_pthread_join(odph_linux_pthread_t *thread_tbl, int num)
return;
}
}
+
+int odph_linux_process_fork_n(odph_linux_process_t *proc_tbl,
+ int num, int first_core)
+{
+ cpu_set_t cpu_set;
+ pid_t pid;
+ int core_count;
+ int cpu;
+ int i;
+
+ memset(proc_tbl, 0, num*sizeof(odph_linux_process_t));
+
+ core_count = odp_sys_core_count();
+
+ if (first_core < 0 || first_core >= core_count) {
+ ODP_ERR("Bad first_core\n");
+ return -1;
+ }
+
+ if (num < 0 || num > core_count) {
+ ODP_ERR("Bad num\n");
+ return -1;
+ }
+
+ for (i = 0; i < num; i++) {
+ cpu = (first_core + i) % core_count;
+ pid = fork();
+
+ if (pid < 0) {
+ ODP_ERR("fork() failed\n");
+ return -1;
+ }
+
+ /* Parent continues to fork */
+ if (pid > 0) {
+ proc_tbl[i].pid = pid;
+ proc_tbl[i].core = cpu;
+ continue;
+ }
+
+ /* Child process */
+ CPU_ZERO(&cpu_set);
+ CPU_SET(cpu, &cpu_set);
+
+ if (sched_setaffinity(0, sizeof(cpu_set_t), &cpu_set)) {
+ ODP_ERR("sched_setaffinity() failed\n");
+ return -2;
+ }
+
+ if (odp_init_local()) {
+ ODP_ERR("Local init failed\n");
+ return -2;
+ }
+
+ return 0;
+ }
+
+ return 1;
+}
+
+
+int odph_linux_process_fork(odph_linux_process_t *proc, int core)
+{
+ return odph_linux_process_fork_n(proc, 1, core);
+}
+
+
+int odph_linux_process_wait_n(odph_linux_process_t *proc_tbl, int num)
+{
+ pid_t pid;
+ int i, j;
+ int status;
+
+ for (i = 0; i < num; i++) {
+ pid = wait(&status);
+
+ if (pid < 0) {
+ ODP_ERR("wait() failed\n");
+ return -1;
+ }
+
+ for (j = 0; j < num; j++) {
+ if (proc_tbl[j].pid == pid) {
+ proc_tbl[j].status = status;
+ break;
+ }
+ }
+
+ if (j == num) {
+ ODP_ERR("Bad pid\n");
+ return -1;
+ }
+ }
+
+ return 0;
+}
@@ -6,34 +6,63 @@
#include <odp_packet.h>
#include <odp_packet_internal.h>
+#include <odp_debug_internal.h>
#include <odp_hints.h>
#include <odp_byteorder.h>
#include <odph_eth.h>
#include <odph_ip.h>
+#include <odph_tcp.h>
+#include <odph_udp.h>
#include <string.h>
#include <stdio.h>
-static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr,
- odph_ipv4hdr_t *ipv4, size_t *offset_out);
-static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr,
- odph_ipv6hdr_t *ipv6, size_t *offset_out);
+/*
+ *
+ * Alloc and free
+ * ********************************************************
+ *
+ */
-void odp_packet_init(odp_packet_t pkt)
+odp_packet_t odp_packet_alloc(odp_buffer_pool_t pool_hdl, uint32_t len)
{
- odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt);
- struct rte_mbuf *mb;
- void *start;
+ pool_entry_t *pool = odp_pool_to_entry(pool_hdl);
+
+ if (pool->s.params.buf_type != ODP_BUFFER_TYPE_PACKET)
+ return ODP_PACKET_INVALID;
+
+ /* Handle special case for zero-length packets */
+ if (len == 0) {
+ odp_packet_t pkt =
+ (odp_packet_t)buffer_alloc(pool_hdl,
+ pool->s.params.buf_size);
+ if (pkt != ODP_PACKET_INVALID)
+ pull_tail(odp_packet_hdr(pkt),
+ pool->s.params.buf_size);
+
+ return pkt;
+ }
+
+ return (odp_packet_t)buffer_alloc(pool_hdl, len);
+}
- mb = &pkt_hdr->buf_hdr.mb;
+void odp_packet_free(odp_packet_t pkt)
+{
+ odp_buffer_free((odp_buffer_t)pkt);
+}
+
+int odp_packet_reset(odp_packet_t pkt, uint32_t len)
+{
+ odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt);
+ pool_entry_t *pool = odp_buf_to_pool(&pkt_hdr->buf_hdr);
+ uint32_t totsize = pool->s.headroom + len + pool->s.tailroom;
- start = mb->buf_addr;
- memset(start, 0, mb->buf_len);
+ if (totsize > pkt_hdr->buf_hdr.size)
+ return -1;
- pkt_hdr->l2_offset = (uint32_t) ODP_PACKET_OFFSET_INVALID;
- pkt_hdr->l3_offset = (uint32_t) ODP_PACKET_OFFSET_INVALID;
- pkt_hdr->l4_offset = (uint32_t) ODP_PACKET_OFFSET_INVALID;
+ packet_init(pool, pkt_hdr, len);
+ return 0;
}
odp_packet_t odp_packet_from_buffer(odp_buffer_t buf)
@@ -41,305 +70,478 @@ odp_packet_t odp_packet_from_buffer(odp_buffer_t buf)
return (odp_packet_t)buf;
}
-odp_buffer_t odp_buffer_from_packet(odp_packet_t pkt)
+odp_buffer_t odp_packet_to_buffer(odp_packet_t pkt)
{
return (odp_buffer_t)pkt;
}
-/* Advance the pkt data pointer and set len in one call */
-static int odp_packet_set_offset_len(odp_packet_t pkt, size_t frame_offset,
- size_t len)
+/*
+ *
+ * Pointers and lengths
+ * ********************************************************
+ *
+ */
+
+void *odp_packet_head(odp_packet_t pkt)
{
- struct rte_mbuf *mb = &(odp_packet_hdr(pkt)->buf_hdr.mb);
- uint16_t offset;
- uint16_t data_len;
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ return buffer_map(&pkt_hdr->buf_hdr, 0, NULL, 0);
+}
- /* The pkt buf may have been pulled back into the headroom
- * so we cannot rely on finding the data right after the
- * ODP header and HEADROOM */
- offset = (uint16_t)((unsigned long)mb->pkt.data -
- (unsigned long)mb->buf_addr);
- ODP_ASSERT(mb->buf_len >= offset, "Corrupted mbuf");
- data_len = mb->buf_len - offset;
+uint32_t odp_packet_buf_len(odp_packet_t pkt)
+{
+ return odp_packet_hdr(pkt)->buf_hdr.size;
+}
- if (data_len < frame_offset) {
- ODP_ERR("Frame offset too big");
- return -1;
- }
- mb->pkt.data = (void *)((char *)mb->pkt.data + frame_offset);
- data_len -= frame_offset;
+void *odp_packet_data(odp_packet_t pkt)
+{
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ return packet_map(pkt_hdr, 0, NULL);
+}
- if (data_len < len) {
- ODP_ERR("Packet len too big");
- return -1;
- }
- mb->pkt.pkt_len = len;
- mb->pkt.data_len = len;
+uint32_t odp_packet_seg_len(odp_packet_t pkt)
+{
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ uint32_t seglen;
- return 0;
+ /* Call returns length of 1st data segment */
+ packet_map(pkt_hdr, 0, &seglen);
+ return seglen;
}
-void odp_packet_set_len(odp_packet_t pkt, size_t len)
+uint32_t odp_packet_len(odp_packet_t pkt)
{
- (void)odp_packet_set_offset_len(pkt, 0, len);
+ return odp_packet_hdr(pkt)->frame_len;
}
-size_t odp_packet_get_len(odp_packet_t pkt)
+uint32_t odp_packet_headroom(odp_packet_t pkt)
{
- struct rte_mbuf *mb = &(odp_packet_hdr(pkt)->buf_hdr.mb);
- return mb->pkt.pkt_len;
+ return odp_packet_hdr(pkt)->headroom;
}
-uint8_t *odp_packet_buf_addr(odp_packet_t pkt)
+uint32_t odp_packet_tailroom(odp_packet_t pkt)
{
- return odp_buffer_addr(odp_buffer_from_packet(pkt));
+ return odp_packet_hdr(pkt)->tailroom;
}
-uint8_t *odp_packet_start(odp_packet_t pkt)
+void *odp_packet_tail(odp_packet_t pkt)
{
- struct rte_mbuf *mb = &(odp_packet_hdr(pkt)->buf_hdr.mb);
- return mb->pkt.data;
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ return packet_map(pkt_hdr, pkt_hdr->frame_len, NULL);
}
-
-uint8_t *odp_packet_l2(odp_packet_t pkt)
+void *odp_packet_push_head(odp_packet_t pkt, uint32_t len)
{
- const size_t offset = odp_packet_l2_offset(pkt);
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID))
+ if (len > pkt_hdr->headroom)
return NULL;
- return odp_packet_start(pkt) + offset;
+ push_head(pkt_hdr, len);
+ return packet_map(pkt_hdr, 0, NULL);
}
-size_t odp_packet_l2_offset(odp_packet_t pkt)
+void *odp_packet_pull_head(odp_packet_t pkt, uint32_t len)
{
- return odp_packet_hdr(pkt)->l2_offset;
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+
+ if (len > pkt_hdr->frame_len)
+ return NULL;
+
+ pull_head(pkt_hdr, len);
+ return packet_map(pkt_hdr, 0, NULL);
}
-void odp_packet_set_l2_offset(odp_packet_t pkt, size_t offset)
+void *odp_packet_push_tail(odp_packet_t pkt, uint32_t len)
{
- odp_packet_hdr(pkt)->l2_offset = offset;
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ struct rte_mbuf *mbuf = (struct rte_mbuf *)&pkt_hdr->buf_hdr;
+ uint32_t origin = pkt_hdr->frame_len;
+
+ mbuf->pkt.pkt_len += len;
+ mbuf->pkt.data_len += len;
+
+ if (len > pkt_hdr->tailroom)
+ return NULL;
+
+ push_tail(pkt_hdr, len);
+ return packet_map(pkt_hdr, origin, NULL);
}
-uint8_t *odp_packet_l3(odp_packet_t pkt)
+void *odp_packet_pull_tail(odp_packet_t pkt, uint32_t len)
{
- const size_t offset = odp_packet_l3_offset(pkt);
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID))
+ if (len > pkt_hdr->frame_len)
return NULL;
- return odp_packet_start(pkt) + offset;
+ pull_tail(pkt_hdr, len);
+ return packet_map(pkt_hdr, pkt_hdr->frame_len, NULL);
}
-size_t odp_packet_l3_offset(odp_packet_t pkt)
+void *odp_packet_offset(odp_packet_t pkt, uint32_t offset, uint32_t *len,
+ odp_packet_seg_t *seg)
{
- return odp_packet_hdr(pkt)->l3_offset;
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ void *addr = packet_map(pkt_hdr, offset, len);
+
+ if (addr != NULL && seg != NULL) {
+ odp_buffer_bits_t seghandle;
+ seghandle.u32 = (uint32_t)pkt;
+ seghandle.seg = (pkt_hdr->headroom + offset) /
+ pkt_hdr->buf_hdr.segsize;
+ *seg = seghandle.handle;
+ }
+
+ return addr;
}
-void odp_packet_set_l3_offset(odp_packet_t pkt, size_t offset)
+/*
+ *
+ * Meta-data
+ * ********************************************************
+ *
+ */
+
+odp_buffer_pool_t odp_packet_pool(odp_packet_t pkt)
{
- odp_packet_hdr(pkt)->l3_offset = offset;
+ return odp_packet_hdr(pkt)->buf_hdr.pool_hdl;
}
-uint8_t *odp_packet_l4(odp_packet_t pkt)
+odp_pktio_t odp_packet_input(odp_packet_t pkt)
{
- const size_t offset = odp_packet_l4_offset(pkt);
+ return odp_packet_hdr(pkt)->input;
+}
- if (odp_unlikely(offset == ODP_PACKET_OFFSET_INVALID))
- return NULL;
+void *odp_packet_user_ptr(odp_packet_t pkt)
+{
+ return odp_packet_hdr(pkt)->buf_hdr.buf_ctx;
+}
+
+void odp_packet_user_ptr_set(odp_packet_t pkt, const void *ctx)
+{
+ odp_packet_hdr(pkt)->buf_hdr.buf_cctx = ctx;
+}
+
+uint64_t odp_packet_user_u64(odp_packet_t pkt)
+{
+ return odp_packet_hdr(pkt)->buf_hdr.buf_u64;
+}
- return odp_packet_start(pkt) + offset;
+void odp_packet_user_u64_set(odp_packet_t pkt, uint64_t ctx)
+{
+ odp_packet_hdr(pkt)->buf_hdr.buf_u64 = ctx;
+}
+
+void *odp_packet_l2_ptr(odp_packet_t pkt, uint32_t *len)
+{
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ return packet_map(pkt_hdr, pkt_hdr->l2_offset, len);
+}
+
+uint32_t odp_packet_l2_offset(odp_packet_t pkt)
+{
+ return odp_packet_hdr(pkt)->l2_offset;
}
-size_t odp_packet_l4_offset(odp_packet_t pkt)
+int odp_packet_l2_offset_set(odp_packet_t pkt, uint32_t offset)
+{
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+
+ if (offset >= pkt_hdr->frame_len)
+ return -1;
+
+ pkt_hdr->l2_offset = offset;
+ return 0;
+}
+
+void *odp_packet_l3_ptr(odp_packet_t pkt, uint32_t *len)
+{
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ return packet_map(pkt_hdr, pkt_hdr->l3_offset, len);
+}
+
+uint32_t odp_packet_l3_offset(odp_packet_t pkt)
+{
+ return odp_packet_hdr(pkt)->l3_offset;
+}
+
+int odp_packet_l3_offset_set(odp_packet_t pkt, uint32_t offset)
+{
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+
+ if (offset >= pkt_hdr->frame_len)
+ return -1;
+
+ pkt_hdr->l3_offset = offset;
+ return 0;
+}
+
+void *odp_packet_l4_ptr(odp_packet_t pkt, uint32_t *len)
+{
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ return packet_map(pkt_hdr, pkt_hdr->l4_offset, len);
+}
+
+uint32_t odp_packet_l4_offset(odp_packet_t pkt)
{
return odp_packet_hdr(pkt)->l4_offset;
}
-void odp_packet_set_l4_offset(odp_packet_t pkt, size_t offset)
+int odp_packet_l4_offset_set(odp_packet_t pkt, uint32_t offset)
{
- odp_packet_hdr(pkt)->l4_offset = offset;
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+
+ if (offset >= pkt_hdr->frame_len)
+ return -1;
+
+ pkt_hdr->l4_offset = offset;
+ return 0;
}
-/**
- * Simple packet parser: eth, VLAN, IP, TCP/UDP/ICMP
+int odp_packet_is_segmented(odp_packet_t pkt)
+{
+ return odp_packet_hdr(pkt)->buf_hdr.segcount > 1;
+}
+
+int odp_packet_num_segs(odp_packet_t pkt)
+{
+ return odp_packet_hdr(pkt)->buf_hdr.segcount;
+}
+
+odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt)
+{
+ return (odp_packet_seg_t)pkt;
+}
+
+odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt)
+{
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ odp_buffer_bits_t seghandle;
+
+ seghandle.u32 = (uint32_t)pkt;
+ seghandle.seg = pkt_hdr->buf_hdr.segcount - 1;
+ return seghandle.handle;
+}
+
+odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg)
+{
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+
+ return segment_next(&pkt_hdr->buf_hdr, seg);
+}
+
+/*
*
- * Internal function: caller is resposible for passing only valid packet handles
- * , lengths and offsets (usually done&called in packet input).
+ * Segment level
+ * ********************************************************
*
- * @param pkt Packet handle
- * @param len Packet length in bytes
- * @param frame_offset Byte offset to L2 header
*/
-void odp_packet_parse(odp_packet_t pkt, size_t len, size_t frame_offset)
+
+void *odp_packet_seg_buf_addr(odp_packet_t pkt, odp_packet_seg_t seg)
{
- odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt);
- odph_ethhdr_t *eth;
- odph_vlanhdr_t *vlan;
- odph_ipv4hdr_t *ipv4;
- odph_ipv6hdr_t *ipv6;
- uint16_t ethtype;
- size_t offset = 0;
- uint8_t ip_proto = 0;
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- /* The frame_offset is not relevant for frames from DPDK */
- pkt_hdr->input_flags.eth = 1;
- (void) frame_offset;
- pkt_hdr->frame_offset = 0;
- if (odp_packet_set_offset_len(pkt, 0, len)) {
- return;
- }
+ return segment_map(&pkt_hdr->buf_hdr, seg, NULL,
+ pkt_hdr->headroom + pkt_hdr->frame_len, 0);
+}
- if (odp_unlikely(len < ODPH_ETH_LEN_MIN)) {
- pkt_hdr->error_flags.frame_len = 1;
- return;
- } else if (len > ODPH_ETH_LEN_MAX) {
- pkt_hdr->input_flags.jumbo = 1;
- }
+uint32_t odp_packet_seg_buf_len(odp_packet_t pkt,
+ odp_packet_seg_t seg ODP_UNUSED)
+{
+ return odp_packet_hdr(pkt)->buf_hdr.segsize;
+}
- /* Assume valid L2 header, no CRC/FCS check in SW */
- pkt_hdr->input_flags.l2 = 1;
- pkt_hdr->l2_offset = 0;
+void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg)
+{
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
- eth = (odph_ethhdr_t *)odp_packet_start(pkt);
- ethtype = odp_be_to_cpu_16(eth->type);
- vlan = (odph_vlanhdr_t *)ð->type;
+ return segment_map(&pkt_hdr->buf_hdr, seg, NULL,
+ pkt_hdr->frame_len, pkt_hdr->headroom);
+}
- if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) {
- pkt_hdr->input_flags.vlan_qinq = 1;
- ethtype = odp_be_to_cpu_16(vlan->tpid);
- offset += sizeof(odph_vlanhdr_t);
- vlan = &vlan[1];
- }
+uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg)
+{
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ uint32_t seglen = 0;
- if (ethtype == ODPH_ETHTYPE_VLAN) {
- pkt_hdr->input_flags.vlan = 1;
- ethtype = odp_be_to_cpu_16(vlan->tpid);
- offset += sizeof(odph_vlanhdr_t);
- }
+ segment_map(&pkt_hdr->buf_hdr, seg, &seglen,
+ pkt_hdr->frame_len, pkt_hdr->headroom);
- /* Set l3_offset+flag only for known ethtypes */
- switch (ethtype) {
- case ODPH_ETHTYPE_IPV4:
- pkt_hdr->input_flags.ipv4 = 1;
- pkt_hdr->input_flags.l3 = 1;
- pkt_hdr->l3_offset = ODPH_ETHHDR_LEN + offset;
- ipv4 = (odph_ipv4hdr_t *)odp_packet_l3(pkt);
- ip_proto = parse_ipv4(pkt_hdr, ipv4, &offset);
- break;
- case ODPH_ETHTYPE_IPV6:
- pkt_hdr->input_flags.ipv6 = 1;
- pkt_hdr->input_flags.l3 = 1;
- pkt_hdr->l3_offset = frame_offset + ODPH_ETHHDR_LEN + offset;
- ipv6 = (odph_ipv6hdr_t *)odp_packet_l3(pkt);
- ip_proto = parse_ipv6(pkt_hdr, ipv6, &offset);
- break;
- case ODPH_ETHTYPE_ARP:
- pkt_hdr->input_flags.arp = 1;
- /* fall through */
- default:
- ip_proto = 0;
- break;
- }
+ return seglen;
+}
- switch (ip_proto) {
- case ODPH_IPPROTO_UDP:
- pkt_hdr->input_flags.udp = 1;
- pkt_hdr->input_flags.l4 = 1;
- pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset;
- break;
- case ODPH_IPPROTO_TCP:
- pkt_hdr->input_flags.tcp = 1;
- pkt_hdr->input_flags.l4 = 1;
- pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset;
- break;
- case ODPH_IPPROTO_SCTP:
- pkt_hdr->input_flags.sctp = 1;
- pkt_hdr->input_flags.l4 = 1;
- pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset;
- break;
- case ODPH_IPPROTO_ICMP:
- pkt_hdr->input_flags.icmp = 1;
- pkt_hdr->input_flags.l4 = 1;
- pkt_hdr->l4_offset = pkt_hdr->l3_offset + offset;
- break;
- default:
- /* 0 or unhandled IP protocols, don't set L4 flag+offset */
- if (pkt_hdr->input_flags.ipv6) {
- /* IPv6 next_hdr is not L4, mark as IP-option instead */
- pkt_hdr->input_flags.ipopt = 1;
+/*
+ *
+ * Manipulation
+ * ********************************************************
+ *
+ */
+
+odp_packet_t odp_packet_add_data(odp_packet_t pkt, uint32_t offset,
+ uint32_t len)
+{
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ uint32_t pktlen = pkt_hdr->frame_len;
+ odp_packet_t newpkt;
+
+ if (offset > pktlen)
+ return ODP_PACKET_INVALID;
+
+ newpkt = odp_packet_alloc(pkt_hdr->buf_hdr.pool_hdl, pktlen + len);
+
+ if (newpkt != ODP_PACKET_INVALID) {
+ if (_odp_packet_copy_to_packet(pkt, 0,
+ newpkt, 0, offset) != 0 ||
+ _odp_packet_copy_to_packet(pkt, offset, newpkt,
+ offset + len,
+ pktlen - offset) != 0) {
+ odp_packet_free(newpkt);
+ newpkt = ODP_PACKET_INVALID;
+ } else {
+ odp_packet_hdr_t *new_hdr = odp_packet_hdr(newpkt);
+ new_hdr->buf_hdr.buf_u64 = pkt_hdr->buf_hdr.buf_u64;
+ odp_atomic_store_u32(
+ &new_hdr->buf_hdr.ref_count,
+ odp_atomic_load_u32(
+ &pkt_hdr->buf_hdr.ref_count));
+ _odp_packet_parse(newpkt);
+ odp_packet_free(pkt);
}
- break;
}
+
+ return newpkt;
}
-static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr,
- odph_ipv4hdr_t *ipv4, size_t *offset_out)
+odp_packet_t odp_packet_rem_data(odp_packet_t pkt, uint32_t offset,
+ uint32_t len)
{
- uint8_t ihl;
- uint16_t frag_offset;
-
- ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl);
- if (odp_unlikely(ihl < ODPH_IPV4HDR_IHL_MIN)) {
- pkt_hdr->error_flags.ip_err = 1;
- return 0;
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+ uint32_t pktlen = pkt_hdr->frame_len;
+ odp_packet_t newpkt;
+
+ if (offset > pktlen || offset + len > pktlen)
+ return ODP_PACKET_INVALID;
+
+ newpkt = odp_packet_alloc(pkt_hdr->buf_hdr.pool_hdl, pktlen - len);
+
+ if (newpkt != ODP_PACKET_INVALID) {
+ if (_odp_packet_copy_to_packet(pkt, 0,
+ newpkt, 0, offset) != 0 ||
+ _odp_packet_copy_to_packet(pkt, offset + len,
+ newpkt, offset,
+ pktlen - offset - len) != 0) {
+ odp_packet_free(newpkt);
+ newpkt = ODP_PACKET_INVALID;
+ } else {
+ odp_packet_hdr_t *new_hdr = odp_packet_hdr(newpkt);
+ new_hdr->buf_hdr.buf_u64 = pkt_hdr->buf_hdr.buf_u64;
+ odp_atomic_store_u32(
+ &new_hdr->buf_hdr.ref_count,
+ odp_atomic_load_u32(
+ &pkt_hdr->buf_hdr.ref_count));
+ _odp_packet_parse(newpkt);
+ odp_packet_free(pkt);
+ }
}
- if (odp_unlikely(ihl > ODPH_IPV4HDR_IHL_MIN)) {
- pkt_hdr->input_flags.ipopt = 1;
- return 0;
- }
+ return newpkt;
+}
- /* A packet is a fragment if:
- * "more fragments" flag is set (all fragments except the last)
- * OR
- * "fragment offset" field is nonzero (all fragments except the first)
- */
- frag_offset = odp_be_to_cpu_16(ipv4->frag_offset);
- if (odp_unlikely(ODPH_IPV4HDR_IS_FRAGMENT(frag_offset))) {
- pkt_hdr->input_flags.ipfrag = 1;
- return 0;
- }
+/*
+ *
+ * Copy
+ * ********************************************************
+ *
+ */
- if (ipv4->proto == ODPH_IPPROTO_ESP ||
- ipv4->proto == ODPH_IPPROTO_AH) {
- pkt_hdr->input_flags.ipsec = 1;
- return 0;
+odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_buffer_pool_t pool)
+{
+ odp_packet_hdr_t *srchdr = odp_packet_hdr(pkt);
+ uint32_t pktlen = srchdr->frame_len;
+ uint32_t meta_offset = ODP_FIELD_SIZEOF(odp_packet_hdr_t, buf_hdr);
+ odp_packet_t newpkt = odp_packet_alloc(pool, pktlen);
+
+ if (newpkt != ODP_PACKET_INVALID) {
+ odp_packet_hdr_t *newhdr = odp_packet_hdr(newpkt);
+ uint8_t *newstart, *srcstart;
+
+ /* Must copy meta data first, followed by packet data */
+ newstart = (uint8_t *)newhdr + meta_offset;
+ srcstart = (uint8_t *)srchdr + meta_offset;
+
+ memcpy(newstart, srcstart,
+ sizeof(odp_packet_hdr_t) - meta_offset);
+
+ if (_odp_packet_copy_to_packet(pkt, 0,
+ newpkt, 0, pktlen) != 0) {
+ odp_packet_free(newpkt);
+ newpkt = ODP_PACKET_INVALID;
+ }
}
- /* Set pkt_hdr->input_flags.ipopt when checking L4 hdrs after return */
-
- *offset_out = sizeof(uint32_t) * ihl;
- return ipv4->proto;
+ return newpkt;
}
-static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr,
- odph_ipv6hdr_t *ipv6, size_t *offset_out)
+int odp_packet_copydata_out(odp_packet_t pkt, uint32_t offset,
+ uint32_t len, void *dst)
{
- if (ipv6->next_hdr == ODPH_IPPROTO_ESP ||
- ipv6->next_hdr == ODPH_IPPROTO_AH) {
- pkt_hdr->input_flags.ipopt = 1;
- pkt_hdr->input_flags.ipsec = 1;
- return 0;
+ void *mapaddr;
+ uint32_t seglen = 0; /* GCC */
+ uint32_t cpylen;
+ uint8_t *dstaddr = (uint8_t *)dst;
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+
+ if (offset + len > pkt_hdr->frame_len)
+ return -1;
+
+ while (len > 0) {
+ mapaddr = packet_map(pkt_hdr, offset, &seglen);
+ cpylen = len > seglen ? seglen : len;
+ memcpy(dstaddr, mapaddr, cpylen);
+ offset += cpylen;
+ dstaddr += cpylen;
+ len -= cpylen;
}
- if (odp_unlikely(ipv6->next_hdr == ODPH_IPPROTO_FRAG)) {
- pkt_hdr->input_flags.ipopt = 1;
- pkt_hdr->input_flags.ipfrag = 1;
- return 0;
+ return 0;
+}
+
+int odp_packet_copydata_in(odp_packet_t pkt, uint32_t offset,
+ uint32_t len, const void *src)
+{
+ void *mapaddr;
+ uint32_t seglen = 0; /* GCC */
+ uint32_t cpylen;
+ const uint8_t *srcaddr = (const uint8_t *)src;
+ odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+
+ if (offset + len > pkt_hdr->frame_len)
+ return -1;
+
+ while (len > 0) {
+ mapaddr = packet_map(pkt_hdr, offset, &seglen);
+ cpylen = len > seglen ? seglen : len;
+ memcpy(mapaddr, srcaddr, cpylen);
+ offset += cpylen;
+ srcaddr += cpylen;
+ len -= cpylen;
}
- /* Don't step through more extensions */
- *offset_out = ODPH_IPV6HDR_LEN;
- return ipv6->next_hdr;
+ return 0;
}
+/*
+ *
+ * Debugging
+ * ********************************************************
+ *
+ */
+
void odp_packet_print(odp_packet_t pkt)
{
int max_len = 512;
char str[max_len];
- uint8_t *p;
int len = 0;
int n = max_len-1;
odp_packet_hdr_t *hdr = odp_packet_hdr(pkt);
@@ -353,77 +555,358 @@ void odp_packet_print(odp_packet_t pkt)
len += snprintf(&str[len], n-len,
" output_flags 0x%x\n", hdr->output_flags.all);
len += snprintf(&str[len], n-len,
- " frame_offset %u\n", hdr->frame_offset);
- len += snprintf(&str[len], n-len,
" l2_offset %u\n", hdr->l2_offset);
len += snprintf(&str[len], n-len,
" l3_offset %u\n", hdr->l3_offset);
len += snprintf(&str[len], n-len,
" l4_offset %u\n", hdr->l4_offset);
len += snprintf(&str[len], n-len,
- " frame_len %u\n", hdr->buf_hdr.mb.pkt.pkt_len);
+ " frame_len %u\n", hdr->frame_len);
len += snprintf(&str[len], n-len,
" input %u\n", hdr->input);
str[len] = '\0';
- printf("\n%s\n", str);
- rte_pktmbuf_dump(stdout, &hdr->buf_hdr.mb, 32);
+ ODP_PRINT("\n%s\n", str);
+}
- p = odp_packet_start(pkt);
- printf("00000000: %02X %02X %02X %02X %02X %02X %02X %02X\n",
- p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7]);
- printf("00000008: %02X %02X %02X %02X %02X %02X %02X %02X\n",
- p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
+int odp_packet_is_valid(odp_packet_t pkt)
+{
+ odp_buffer_hdr_t *buf = validate_buf((odp_buffer_t)pkt);
+ return (buf != NULL && buf->type == ODP_BUFFER_TYPE_PACKET);
}
-/* For now we can only copy between packets of the same segment size
- * We should probably refine this API, maybe introduce a clone API */
-int odp_packet_copy(odp_packet_t pkt_dst, odp_packet_t pkt_src)
+/*
+ *
+ * Internal Use Routines
+ * ********************************************************
+ *
+ */
+
+int _odp_packet_copy_to_packet(odp_packet_t srcpkt, uint32_t srcoffset,
+ odp_packet_t dstpkt, uint32_t dstoffset,
+ uint32_t len)
{
- struct rte_mbuf *mb_dst, *mb_src;
- uint8_t nb_segs, i;
+ odp_packet_hdr_t *srchdr = odp_packet_hdr(srcpkt);
+ odp_packet_hdr_t *dsthdr = odp_packet_hdr(dstpkt);
+ void *srcmap;
+ void *dstmap;
+ uint32_t cpylen, minseg;
+ uint32_t srcseglen = 0; /* GCC */
+ uint32_t dstseglen = 0; /* GCC */
+
+ if (srcoffset + len > srchdr->frame_len ||
+ dstoffset + len > dsthdr->frame_len)
+ return -1;
- ODP_ASSERT(odp_buffer_type(pkt_dst) == ODP_BUFFER_TYPE_PACKET &&
- odp_buffer_type(pkt_src) == ODP_BUFFER_TYPE_PACKET,
- "dst_pkt or src_pkt not of type ODP_BUFFER_TYPE_PACKET");
+ while (len > 0) {
+ srcmap = packet_map(srchdr, srcoffset, &srcseglen);
+ dstmap = packet_map(dsthdr, dstoffset, &dstseglen);
- if (pkt_dst == ODP_PACKET_INVALID || pkt_src == ODP_PACKET_INVALID)
- return -1;
+ minseg = dstseglen > srcseglen ? srcseglen : dstseglen;
+ cpylen = len > minseg ? minseg : len;
+ memcpy(dstmap, srcmap, cpylen);
- mb_dst = &(odp_packet_hdr(pkt_dst)->buf_hdr.mb);
- mb_src = &(odp_packet_hdr(pkt_src)->buf_hdr.mb);
+ srcoffset += cpylen;
+ dstoffset += cpylen;
+ len -= cpylen;
+ }
- if (mb_dst->pkt.nb_segs != mb_src->pkt.nb_segs) {
- ODP_ERR("Different nb_segs in pkt_dst and pkt_src");
- return -1;
+ return 0;
+}
+
+odp_packet_t _odp_packet_alloc(odp_buffer_pool_t pool_hdl)
+{
+ pool_entry_t *pool = odp_pool_to_entry(pool_hdl);
+
+ if (pool->s.params.buf_type != ODP_BUFFER_TYPE_PACKET)
+ return ODP_PACKET_INVALID;
+
+ return (odp_packet_t)buffer_alloc(pool_hdl,
+ pool->s.params.buf_size);
+}
+
+/**
+ * Parser helper function for IPv4
+ */
+static inline uint8_t parse_ipv4(odp_packet_hdr_t *pkt_hdr,
+ uint8_t **parseptr, uint32_t *offset)
+{
+ odph_ipv4hdr_t *ipv4 = (odph_ipv4hdr_t *)*parseptr;
+ uint8_t ver = ODPH_IPV4HDR_VER(ipv4->ver_ihl);
+ uint8_t ihl = ODPH_IPV4HDR_IHL(ipv4->ver_ihl);
+ uint16_t frag_offset;
+
+ pkt_hdr->l3_len = odp_be_to_cpu_16(ipv4->tot_len);
+
+ if (odp_unlikely(ihl < ODPH_IPV4HDR_IHL_MIN) ||
+ odp_unlikely(ver != 4) ||
+ (pkt_hdr->l3_len > pkt_hdr->frame_len - *offset)) {
+ pkt_hdr->error_flags.ip_err = 1;
+ return 0;
}
- nb_segs = mb_src->pkt.nb_segs;
+ *offset += ihl * 4;
+ *parseptr += ihl * 4;
- if (mb_dst->buf_len < mb_src->buf_len) {
- ODP_ERR("dst_pkt smaller than src_pkt");
- return -1;
+ if (odp_unlikely(ihl > ODPH_IPV4HDR_IHL_MIN))
+ pkt_hdr->input_flags.ipopt = 1;
+
+ /* A packet is a fragment if:
+ * "more fragments" flag is set (all fragments except the last)
+ * OR
+ * "fragment offset" field is nonzero (all fragments except the first)
+ */
+ frag_offset = odp_be_to_cpu_16(ipv4->frag_offset);
+ if (odp_unlikely(ODPH_IPV4HDR_IS_FRAGMENT(frag_offset)))
+ pkt_hdr->input_flags.ipfrag = 1;
+
+ return ipv4->proto;
+}
+
+/**
+ * Parser helper function for IPv6
+ */
+static inline uint8_t parse_ipv6(odp_packet_hdr_t *pkt_hdr,
+ uint8_t **parseptr, uint32_t *offset)
+{
+ odph_ipv6hdr_t *ipv6 = (odph_ipv6hdr_t *)*parseptr;
+ odph_ipv6hdr_ext_t *ipv6ext;
+
+ pkt_hdr->l3_len = odp_be_to_cpu_16(ipv6->payload_len);
+
+ /* Basic sanity checks on IPv6 header */
+ if ((ipv6->ver_tc_flow >> 28) != 6 ||
+ pkt_hdr->l3_len > pkt_hdr->frame_len - *offset) {
+ pkt_hdr->error_flags.ip_err = 1;
+ return 0;
}
- for (i = 0; i < nb_segs; i++) {
- if (mb_src == NULL || mb_dst == NULL) {
- ODP_ERR("Corrupted packets");
- return -1;
+ /* Skip past IPv6 header */
+ *offset += sizeof(odph_ipv6hdr_t);
+ *parseptr += sizeof(odph_ipv6hdr_t);
+
+
+ /* Skip past any IPv6 extension headers */
+ if (ipv6->next_hdr == ODPH_IPPROTO_HOPOPTS ||
+ ipv6->next_hdr == ODPH_IPPROTO_ROUTE) {
+ pkt_hdr->input_flags.ipopt = 1;
+
+ do {
+ ipv6ext = (odph_ipv6hdr_ext_t *)*parseptr;
+ uint16_t extlen = 8 + ipv6ext->ext_len * 8;
+
+ *offset += extlen;
+ *parseptr += extlen;
+ } while ((ipv6ext->next_hdr == ODPH_IPPROTO_HOPOPTS ||
+ ipv6ext->next_hdr == ODPH_IPPROTO_ROUTE) &&
+ *offset < pkt_hdr->frame_len);
+
+ if (*offset >= pkt_hdr->l3_offset + ipv6->payload_len) {
+ pkt_hdr->error_flags.ip_err = 1;
+ return 0;
}
- memcpy(mb_dst->buf_addr, mb_src->buf_addr, mb_src->buf_len);
- mb_dst = mb_dst->pkt.next;
- mb_src = mb_src->pkt.next;
+
+ if (ipv6ext->next_hdr == ODPH_IPPROTO_FRAG)
+ pkt_hdr->input_flags.ipfrag = 1;
+
+ return ipv6ext->next_hdr;
}
- return 0;
+
+ if (odp_unlikely(ipv6->next_hdr == ODPH_IPPROTO_FRAG)) {
+ pkt_hdr->input_flags.ipopt = 1;
+ pkt_hdr->input_flags.ipfrag = 1;
+ }
+
+ return ipv6->next_hdr;
+}
+
+/**
+ * Parser helper function for TCP
+ */
+static inline void parse_tcp(odp_packet_hdr_t *pkt_hdr,
+ uint8_t **parseptr, uint32_t *offset)
+{
+ odph_tcphdr_t *tcp = (odph_tcphdr_t *)*parseptr;
+
+ if (tcp->hl < sizeof(odph_tcphdr_t)/sizeof(uint32_t))
+ pkt_hdr->error_flags.tcp_err = 1;
+ else if ((uint32_t)tcp->hl * 4 > sizeof(odph_tcphdr_t))
+ pkt_hdr->input_flags.tcpopt = 1;
+
+ pkt_hdr->l4_len = pkt_hdr->l3_len +
+ pkt_hdr->l3_offset - pkt_hdr->l4_offset;
+
+ *offset += (uint32_t)tcp->hl * 4;
+ *parseptr += (uint32_t)tcp->hl * 4;
}
-void odp_packet_set_ctx(odp_packet_t pkt, const void *ctx)
+/**
+ * Parser helper function for UDP
+ */
+static inline void parse_udp(odp_packet_hdr_t *pkt_hdr,
+ uint8_t **parseptr, uint32_t *offset)
{
- odp_packet_hdr(pkt)->user_ctx = (intptr_t)ctx;
+ odph_udphdr_t *udp = (odph_udphdr_t *)*parseptr;
+ uint32_t udplen = odp_be_to_cpu_16(udp->length);
+
+ if (udplen < sizeof(odph_udphdr_t) ||
+ udplen > (pkt_hdr->l3_len +
+ pkt_hdr->l3_offset - pkt_hdr->l4_offset)) {
+ pkt_hdr->error_flags.udp_err = 1;
+ }
+
+ pkt_hdr->l4_len = udplen;
+
+ *offset += sizeof(odph_udphdr_t);
+ *parseptr += sizeof(odph_udphdr_t);
}
-void *odp_packet_get_ctx(odp_packet_t pkt)
+/**
+ * Simple packet parser
+ */
+
+int _odp_packet_parse(odp_packet_t pkt)
{
- return (void *)(intptr_t)odp_packet_hdr(pkt)->user_ctx;
+ odp_packet_hdr_t *const pkt_hdr = odp_packet_hdr(pkt);
+ odph_ethhdr_t *eth;
+ odph_vlanhdr_t *vlan;
+ uint16_t ethtype;
+ uint8_t *parseptr;
+ uint32_t offset, seglen;
+ uint8_t ip_proto = 0;
+
+ /* Reset parser metadata for new parse */
+ pkt_hdr->error_flags.all = 0;
+ pkt_hdr->input_flags.all = 0;
+ pkt_hdr->output_flags.all = 0;
+ pkt_hdr->l2_offset = 0;
+ pkt_hdr->l3_offset = 0;
+ pkt_hdr->l4_offset = 0;
+ pkt_hdr->payload_offset = 0;
+ pkt_hdr->vlan_s_tag = 0;
+ pkt_hdr->vlan_c_tag = 0;
+ pkt_hdr->l3_protocol = 0;
+ pkt_hdr->l4_protocol = 0;
+
+ /* We only support Ethernet for now */
+ pkt_hdr->input_flags.eth = 1;
+
+ /* Detect jumbo frames */
+ if (pkt_hdr->frame_len > ODPH_ETH_LEN_MAX)
+ pkt_hdr->input_flags.jumbo = 1;
+
+ /* Assume valid L2 header, no CRC/FCS check in SW */
+ pkt_hdr->input_flags.l2 = 1;
+
+ eth = (odph_ethhdr_t *)packet_map(pkt_hdr, 0, &seglen);
+ offset = sizeof(odph_ethhdr_t);
+ parseptr = (uint8_t *)ð->type;
+ ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr));
+
+ /* Parse the VLAN header(s), if present */
+ if (ethtype == ODPH_ETHTYPE_VLAN_OUTER) {
+ pkt_hdr->input_flags.vlan_qinq = 1;
+ pkt_hdr->input_flags.vlan = 1;
+ vlan = (odph_vlanhdr_t *)(void *)parseptr;
+ pkt_hdr->vlan_s_tag = ((ethtype << 16) |
+ odp_be_to_cpu_16(vlan->tci));
+ offset += sizeof(odph_vlanhdr_t);
+ parseptr += sizeof(odph_vlanhdr_t);
+ ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr));
+ }
+
+ if (ethtype == ODPH_ETHTYPE_VLAN) {
+ pkt_hdr->input_flags.vlan = 1;
+ vlan = (odph_vlanhdr_t *)(void *)parseptr;
+ pkt_hdr->vlan_c_tag = ((ethtype << 16) |
+ odp_be_to_cpu_16(vlan->tci));
+ offset += sizeof(odph_vlanhdr_t);
+ parseptr += sizeof(odph_vlanhdr_t);
+ ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr));
+ }
+
+ /* Check for SNAP vs. DIX */
+ if (ethtype < ODPH_ETH_LEN_MAX) {
+ pkt_hdr->input_flags.snap = 1;
+ if (ethtype > pkt_hdr->frame_len - offset) {
+ pkt_hdr->error_flags.snap_len = 1;
+ goto parse_exit;
+ }
+ offset += 8;
+ parseptr += 8;
+ ethtype = odp_be_to_cpu_16(*((uint16_t *)(void *)parseptr));
+ }
+
+ /* Consume Ethertype for Layer 3 parse */
+ parseptr += 2;
+
+ /* Set l3_offset+flag only for known ethtypes */
+ pkt_hdr->input_flags.l3 = 1;
+ pkt_hdr->l3_offset = offset;
+ pkt_hdr->l3_protocol = ethtype;
+
+ /* Parse Layer 3 headers */
+ switch (ethtype) {
+ case ODPH_ETHTYPE_IPV4:
+ pkt_hdr->input_flags.ipv4 = 1;
+ ip_proto = parse_ipv4(pkt_hdr, &parseptr, &offset);
+ break;
+
+ case ODPH_ETHTYPE_IPV6:
+ pkt_hdr->input_flags.ipv6 = 1;
+ ip_proto = parse_ipv6(pkt_hdr, &parseptr, &offset);
+ break;
+
+ case ODPH_ETHTYPE_ARP:
+ pkt_hdr->input_flags.arp = 1;
+ ip_proto = 255; /* Reserved invalid by IANA */
+ break;
+
+ default:
+ pkt_hdr->input_flags.l3 = 0;
+ ip_proto = 255; /* Reserved invalid by IANA */
+ }
+
+ /* Set l4_offset+flag only for known ip_proto */
+ pkt_hdr->input_flags.l4 = 1;
+ pkt_hdr->l4_offset = offset;
+ pkt_hdr->l4_protocol = ip_proto;
+
+ /* Parse Layer 4 headers */
+ switch (ip_proto) {
+ case ODPH_IPPROTO_ICMP:
+ pkt_hdr->input_flags.icmp = 1;
+ break;
+
+ case ODPH_IPPROTO_TCP:
+ pkt_hdr->input_flags.tcp = 1;
+ parse_tcp(pkt_hdr, &parseptr, &offset);
+ break;
+
+ case ODPH_IPPROTO_UDP:
+ pkt_hdr->input_flags.udp = 1;
+ parse_udp(pkt_hdr, &parseptr, &offset);
+ break;
+
+ case ODPH_IPPROTO_AH:
+ case ODPH_IPPROTO_ESP:
+ pkt_hdr->input_flags.ipsec = 1;
+ break;
+
+ default:
+ pkt_hdr->input_flags.l4 = 0;
+ break;
+ }
+
+ /*
+ * Anything beyond what we parse here is considered payload.
+ * Note: Payload is really only relevant for TCP and UDP. For
+ * all other protocols, the payload offset will point to the
+ * final header (ARP, ICMP, AH, ESP, or IP Fragment).
+ */
+ pkt_hdr->payload_offset = offset;
+
+parse_exit:
+ return pkt_hdr->error_flags.all != 0;
}
@@ -23,6 +23,8 @@
#include <odp_hints.h>
#include <odp_thread.h>
+#include <odp_system_info.h>
+#include <odp_debug_internal.h>
#include <odp_packet_dpdk.h>
#include <net/if.h>
@@ -164,11 +166,8 @@ int recv_pkt_dpdk(pkt_dpdk_t * const pkt_dpdk, odp_packet_t pkt_table[],
nb_rx = rte_eth_rx_burst((uint8_t)pkt_dpdk->portid,
(uint16_t)pkt_dpdk->queueid,
(struct rte_mbuf **)pkt_table, (uint16_t)len);
- for (i = 0; i < nb_rx; i++) {
- odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt_table[i]);
- struct rte_mbuf *mb = &pkt_hdr->buf_hdr.mb;
- odp_packet_parse(pkt_table[i], mb->pkt.pkt_len, 0);
- }
+ for (i = 0; i < nb_rx; i++)
+ _odp_packet_parse(pkt_table[i]);
return nb_rx;
}
@@ -16,34 +16,21 @@
#ifdef ODP_HAVE_NETMAP
#include <odp_packet_netmap.h>
#endif
-#include <odp_hints.h>
#include <odp_config.h>
#include <odp_queue_internal.h>
#include <odp_schedule_internal.h>
-#include <odp_debug.h>
-
-#include <odp_pktio_socket.h>
-#ifdef ODP_HAVE_NETMAP
-#include <odp_pktio_netmap.h>
-#endif
+#include <odp_classification_internal.h>
+#include <odp_debug_internal.h>
#include <string.h>
-
-typedef struct {
- pktio_entry_t entries[ODP_CONFIG_PKTIO_ENTRIES];
-} pktio_table_t;
+#include <sys/ioctl.h>
+#include <linux/if_arp.h>
+#include <ifaddrs.h>
static pktio_table_t *pktio_tbl;
-
-static pktio_entry_t *get_entry(odp_pktio_t id)
-{
- if (odp_unlikely(id == ODP_PKTIO_INVALID ||
- id > ODP_CONFIG_PKTIO_ENTRIES))
- return NULL;
-
- return &pktio_tbl->entries[id - 1];
-}
+/* pktio pointer entries ( for inlines) */
+void *pktio_entry_ptr[ODP_CONFIG_PKTIO_ENTRIES];
int odp_pktio_init_global(void)
{
@@ -65,10 +52,12 @@ int odp_pktio_init_global(void)
memset(pktio_tbl, 0, sizeof(pktio_table_t));
for (id = 1; id <= ODP_CONFIG_PKTIO_ENTRIES; ++id) {
- pktio_entry = get_entry(id);
+ pktio_entry = &pktio_tbl->entries[id - 1];
odp_spinlock_init(&pktio_entry->s.lock);
+ odp_spinlock_init(&pktio_entry->s.cls.lock);
+ pktio_entry_ptr[id - 1] = pktio_entry;
/* Create a default output queue for each pktio resource */
snprintf(name, sizeof(name), "%i-pktio_outq_default", (int)id);
name[ODP_QUEUE_NAME_LEN-1] = '\0';
@@ -115,16 +104,27 @@ static void unlock_entry(pktio_entry_t *entry)
odp_spinlock_unlock(&entry->s.lock);
}
-static void init_pktio_entry(pktio_entry_t *entry, odp_pktio_params_t *params)
+static void lock_entry_classifier(pktio_entry_t *entry)
+{
+ odp_spinlock_lock(&entry->s.lock);
+ odp_spinlock_lock(&entry->s.cls.lock);
+}
+
+static void unlock_entry_classifier(pktio_entry_t *entry)
+{
+ odp_spinlock_unlock(&entry->s.cls.lock);
+ odp_spinlock_unlock(&entry->s.lock);
+}
+
+static void init_pktio_entry(pktio_entry_t *entry)
{
set_taken(entry);
entry->s.inq_default = ODP_QUEUE_INVALID;
memset(&entry->s.pkt_dpdk, 0, sizeof(entry->s.pkt_dpdk));
- /* Save pktio parameters, type is the most useful */
- memcpy(&entry->s.params, params, sizeof(*params));
+ pktio_classifier_init(entry);
}
-static odp_pktio_t alloc_lock_pktio_entry(odp_pktio_params_t *params)
+static odp_pktio_t alloc_lock_pktio_entry(void)
{
odp_pktio_t id;
pktio_entry_t *entry;
@@ -133,13 +133,13 @@ static odp_pktio_t alloc_lock_pktio_entry(odp_pktio_params_t *params)
for (i = 0; i < ODP_CONFIG_PKTIO_ENTRIES; ++i) {
entry = &pktio_tbl->entries[i];
if (is_free(entry)) {
- lock_entry(entry);
+ lock_entry_classifier(entry);
if (is_free(entry)) {
- init_pktio_entry(entry, params);
+ init_pktio_entry(entry);
id = i + 1;
return id; /* return with entry locked! */
}
- unlock_entry(entry);
+ unlock_entry_classifier(entry);
}
}
@@ -148,7 +148,7 @@ static odp_pktio_t alloc_lock_pktio_entry(odp_pktio_params_t *params)
static int free_pktio_entry(odp_pktio_t id)
{
- pktio_entry_t *entry = get_entry(id);
+ pktio_entry_t *entry = get_pktio_entry(id);
if (entry == NULL)
return -1;
@@ -158,28 +158,22 @@ static int free_pktio_entry(odp_pktio_t id)
return 0;
}
-odp_pktio_t odp_pktio_open(const char *dev, odp_buffer_pool_t pool,
- odp_pktio_params_t *params)
+odp_pktio_t odp_pktio_open(const char *dev, odp_buffer_pool_t pool)
{
odp_pktio_t id;
pktio_entry_t *pktio_entry;
int res;
- if (params == NULL) {
- ODP_ERR("Invalid pktio params\n");
- return ODP_PKTIO_INVALID;
- }
-
ODP_DBG("Allocating dpdk pktio\n");
- id = alloc_lock_pktio_entry(params);
+ id = alloc_lock_pktio_entry();
if (id == ODP_PKTIO_INVALID) {
ODP_ERR("No resources available.\n");
return ODP_PKTIO_INVALID;
}
/* if successful, alloc_pktio_entry() returns with the entry locked */
- pktio_entry = get_entry(id);
+ pktio_entry = get_pktio_entry(id);
res = setup_pkt_dpdk(&pktio_entry->s.pkt_dpdk, dev, pool);
if (res == -1) {
@@ -188,7 +182,7 @@ odp_pktio_t odp_pktio_open(const char *dev, odp_buffer_pool_t pool,
id = ODP_PKTIO_INVALID;
}
- unlock_entry(pktio_entry);
+ unlock_entry_classifier(pktio_entry);
return id;
}
@@ -197,7 +191,7 @@ int odp_pktio_close(odp_pktio_t id)
pktio_entry_t *entry;
int res = -1;
- entry = get_entry(id);
+ entry = get_pktio_entry(id);
if (entry == NULL)
return -1;
@@ -226,7 +220,7 @@ odp_pktio_t odp_pktio_get_input(odp_packet_t pkt)
int odp_pktio_recv(odp_pktio_t id, odp_packet_t pkt_table[], unsigned len)
{
- pktio_entry_t *pktio_entry = get_entry(id);
+ pktio_entry_t *pktio_entry = get_pktio_entry(id);
int pkts;
int i;
@@ -249,7 +243,7 @@ int odp_pktio_recv(odp_pktio_t id, odp_packet_t pkt_table[], unsigned len)
int odp_pktio_send(odp_pktio_t id, odp_packet_t pkt_table[], unsigned len)
{
- pktio_entry_t *pktio_entry = get_entry(id);
+ pktio_entry_t *pktio_entry = get_pktio_entry(id);
int pkts;
if (pktio_entry == NULL)
@@ -265,15 +259,17 @@ int odp_pktio_send(odp_pktio_t id, odp_packet_t pkt_table[], unsigned len)
int odp_pktio_inq_setdef(odp_pktio_t id, odp_queue_t queue)
{
- pktio_entry_t *pktio_entry = get_entry(id);
+ pktio_entry_t *pktio_entry = get_pktio_entry(id);
queue_entry_t *qentry = queue_to_qentry(queue);
- if (pktio_entry == NULL || qentry == NULL)
+ if (pktio_entry == NULL || qentry == ODP_QUEUE_INVALID)
return -1;
if (qentry->s.type != ODP_QUEUE_TYPE_PKTIN)
return -1;
+ qentry = queue_to_qentry(queue);
+
lock_entry(pktio_entry);
pktio_entry->s.inq_default = queue;
unlock_entry(pktio_entry);
@@ -295,7 +291,7 @@ int odp_pktio_inq_remdef(odp_pktio_t id)
odp_queue_t odp_pktio_inq_getdef(odp_pktio_t id)
{
- pktio_entry_t *pktio_entry = get_entry(id);
+ pktio_entry_t *pktio_entry = get_pktio_entry(id);
if (pktio_entry == NULL)
return ODP_QUEUE_INVALID;
@@ -305,7 +301,7 @@ odp_queue_t odp_pktio_inq_getdef(odp_pktio_t id)
odp_queue_t odp_pktio_outq_getdef(odp_pktio_t id)
{
- pktio_entry_t *pktio_entry = get_entry(id);
+ pktio_entry_t *pktio_entry = get_pktio_entry(id);
if (pktio_entry == NULL)
return ODP_QUEUE_INVALID;
@@ -362,32 +358,32 @@ int pktin_enqueue(queue_entry_t *qentry, odp_buffer_hdr_t *buf_hdr)
odp_buffer_hdr_t *pktin_dequeue(queue_entry_t *qentry)
{
odp_buffer_hdr_t *buf_hdr;
+ odp_buffer_t buf;
+ odp_packet_t pkt_tbl[QUEUE_MULTI_MAX];
+ odp_buffer_hdr_t *tmp_hdr_tbl[QUEUE_MULTI_MAX];
+ int pkts, i, j;
buf_hdr = queue_deq(qentry);
+ if (buf_hdr != NULL)
+ return buf_hdr;
- if (buf_hdr == NULL) {
- odp_packet_t pkt;
- odp_buffer_t buf;
- odp_packet_t pkt_tbl[QUEUE_MULTI_MAX];
- odp_buffer_hdr_t *tmp_hdr_tbl[QUEUE_MULTI_MAX];
- int pkts, i, j;
-
- pkts = odp_pktio_recv(qentry->s.pktin, pkt_tbl,
- QUEUE_MULTI_MAX);
-
- if (pkts > 0) {
- pkt = pkt_tbl[0];
- buf = odp_buffer_from_packet(pkt);
- buf_hdr = odp_buf_to_hdr(buf);
+ pkts = odp_pktio_recv(qentry->s.pktin, pkt_tbl, QUEUE_MULTI_MAX);
+ if (pkts <= 0)
+ return NULL;
- for (i = 1, j = 0; i < pkts; ++i) {
- buf = odp_buffer_from_packet(pkt_tbl[i]);
- tmp_hdr_tbl[j++] = odp_buf_to_hdr(buf);
- }
- queue_enq_multi(qentry, tmp_hdr_tbl, j);
- }
+ for (i = 0, j = 0; i < pkts; ++i) {
+ buf = odp_packet_to_buffer(pkt_tbl[i]);
+ buf_hdr = odp_buf_to_hdr(buf);
+ if (0 > packet_classifier(qentry->s.pktin, pkt_tbl[i]))
+ tmp_hdr_tbl[j++] = buf_hdr;
}
+ if (0 == j)
+ return NULL;
+
+ if (j > 1)
+ queue_enq_multi(qentry, &tmp_hdr_tbl[1], j-1);
+ buf_hdr = tmp_hdr_tbl[0];
return buf_hdr;
}
@@ -400,32 +396,179 @@ int pktin_enq_multi(queue_entry_t *qentry, odp_buffer_hdr_t *buf_hdr[], int num)
int pktin_deq_multi(queue_entry_t *qentry, odp_buffer_hdr_t *buf_hdr[], int num)
{
int nbr;
+ odp_packet_t pkt_tbl[QUEUE_MULTI_MAX];
+ odp_buffer_hdr_t *tmp_hdr_tbl[QUEUE_MULTI_MAX];
+ odp_buffer_hdr_t *tmp_hdr;
+ odp_buffer_t buf;
+ int pkts, i, j;
nbr = queue_deq_multi(qentry, buf_hdr, num);
-
- if (nbr < num) {
- odp_packet_t pkt_tbl[QUEUE_MULTI_MAX];
- odp_buffer_hdr_t *tmp_hdr_tbl[QUEUE_MULTI_MAX];
- odp_buffer_t buf;
- int pkts, i;
-
- pkts = odp_pktio_recv(qentry->s.pktin, pkt_tbl,
- QUEUE_MULTI_MAX);
- if (pkts > 0) {
- for (i = 0; i < pkts; ++i) {
- buf = odp_buffer_from_packet(pkt_tbl[i]);
- tmp_hdr_tbl[i] = odp_buf_to_hdr(buf);
- }
- queue_enq_multi(qentry, tmp_hdr_tbl, pkts);
- }
+ if (odp_unlikely(nbr > num))
+ ODP_ABORT("queue_deq_multi req: %d, returned %d\n",
+ num, nbr);
+
+ /** queue already has number of requsted buffers,
+ * do not do receive in that case.
+ */
+ if (nbr == num)
+ return nbr;
+
+ pkts = odp_pktio_recv(qentry->s.pktin, pkt_tbl, QUEUE_MULTI_MAX);
+ if (pkts <= 0)
+ return nbr;
+
+ for (i = 0, j = 0; i < pkts; ++i) {
+ buf = odp_packet_to_buffer(pkt_tbl[i]);
+ tmp_hdr = odp_buf_to_hdr(buf);
+ if (0 > packet_classifier(qentry->s.pktin, pkt_tbl[i]))
+ tmp_hdr_tbl[j++] = tmp_hdr;
}
+ if (j)
+ queue_enq_multi(qentry, tmp_hdr_tbl, j);
return nbr;
}
+int odp_pktio_set_mtu(odp_pktio_t id, int mtu)
+{
+ pktio_entry_t *entry;
+ int ret;
+
+ if (mtu <= 0) {
+ ODP_DBG("illegal MTU value %d\n", mtu);
+ return -1;
+ }
+
+ entry = get_pktio_entry(id);
+ if (entry == NULL) {
+ ODP_DBG("pktio entry %d does not exist\n", id);
+ return -1;
+ }
+
+ lock_entry(entry);
+
+ if (odp_unlikely(is_free(entry))) {
+ unlock_entry(entry);
+ ODP_DBG("already freed pktio\n");
+ return -1;
+ }
+
+ ret = rte_eth_dev_set_mtu(entry->s.pkt_dpdk.portid, (uint16_t)mtu);
+
+ unlock_entry(entry);
+ return ret;
+}
+
+int odp_pktio_mtu(odp_pktio_t id)
+{
+ pktio_entry_t *entry;
+ int mtu;
+
+ entry = get_pktio_entry(id);
+ if (entry == NULL) {
+ ODP_DBG("pktio entry %d does not exist\n", id);
+ return -1;
+ }
+
+ lock_entry(entry);
+
+ if (odp_unlikely(is_free(entry))) {
+ unlock_entry(entry);
+ ODP_DBG("already freed pktio\n");
+ return -1;
+ }
+
+ rte_eth_dev_get_mtu(entry->s.pkt_dpdk.portid , (uint16_t *)&mtu);
+
+ unlock_entry(entry);
+ return mtu;
+}
+
+int odp_pktio_promisc_mode_set(odp_pktio_t id, odp_bool_t enable)
+{
+ pktio_entry_t *entry;
+
+ entry = get_pktio_entry(id);
+ if (entry == NULL) {
+ ODP_DBG("pktio entry %d does not exist\n", id);
+ return -1;
+ }
+
+ lock_entry(entry);
+
+ if (odp_unlikely(is_free(entry))) {
+ unlock_entry(entry);
+ ODP_DBG("already freed pktio\n");
+ return -1;
+ }
+
+ if (enable)
+ rte_eth_promiscuous_enable(entry->s.pkt_dpdk.portid);
+ else
+ rte_eth_promiscuous_disable(entry->s.pkt_dpdk.portid);
+
+ unlock_entry(entry);
+ return 0;
+}
+
+int odp_pktio_promisc_mode(odp_pktio_t id)
+{
+ pktio_entry_t *entry;
+ int promisc;
+
+ entry = get_pktio_entry(id);
+ if (entry == NULL) {
+ ODP_DBG("pktio entry %d does not exist\n", id);
+ return -1;
+ }
+
+ lock_entry(entry);
+
+ if (odp_unlikely(is_free(entry))) {
+ unlock_entry(entry);
+ ODP_DBG("already freed pktio\n");
+ return -1;
+ }
+
+ promisc = rte_eth_promiscuous_get(entry->s.pkt_dpdk.portid);
+
+ unlock_entry(entry);
+
+ return promisc;
+}
+
+size_t odp_pktio_mac_addr(odp_pktio_t id, void *mac_addr,
+ size_t addr_size)
+{
+ pktio_entry_t *entry;
+
+ if (addr_size < ETH_ALEN)
+ return 0;
+
+ entry = get_pktio_entry(id);
+ if (entry == NULL) {
+ ODP_DBG("pktio entry %d does not exist\n", id);
+ return 0;
+ }
+
+ lock_entry(entry);
+
+ if (odp_unlikely(is_free(entry))) {
+ unlock_entry(entry);
+ ODP_DBG("already freed pktio\n");
+ return -1;
+ }
+
+ rte_eth_macaddr_get(entry->s.pkt_dpdk.portid,
+ (struct ether_addr *)mac_addr);
+ unlock_entry(entry);
+
+ return ETH_ALEN;
+}
+
int odp_pktio_get_mac_addr(odp_pktio_t id, unsigned char *mac_addr)
{
- pktio_entry_t *pktio_entry = get_entry(id);
+ pktio_entry_t *pktio_entry = get_pktio_entry(id);
if (!pktio_entry) {
ODP_ERR("Invalid odp_pktio_t value\n");
return -1;
@@ -11,13 +11,14 @@
#include <odp_buffer.h>
#include <odp_buffer_internal.h>
#include <odp_buffer_pool_internal.h>
+#include <odp_buffer_inlines.h>
#include <odp_internal.h>
#include <odp_shared_memory.h>
#include <odp_schedule_internal.h>
#include <odp_config.h>
#include <odp_packet_io_internal.h>
#include <odp_packet_io_queue.h>
-#include <odp_debug.h>
+#include <odp_debug_internal.h>
#include <odp_hints.h>
#ifdef USE_TICKETLOCK
deleted file mode 100644
@@ -1,421 +0,0 @@
-/* Copyright (c) 2013, Linaro Limited
- * All rights reserved.
- *
- * SPDX-License-Identifier: BSD-3-Clause
- */
-
-#include <odp_schedule.h>
-#include <odp_schedule_internal.h>
-#include <odp_align.h>
-#include <odp_queue.h>
-#include <odp_shared_memory.h>
-#include <odp_buffer.h>
-#include <odp_buffer_pool.h>
-#include <odp_internal.h>
-#include <odp_config.h>
-#include <odp_debug.h>
-#include <odp_thread.h>
-#include <odp_time.h>
-#include <odp_spinlock.h>
-#include <odp_hints.h>
-
-#include <odp_queue_internal.h>
-
-
-/* Limits to number of scheduled queues */
-#define SCHED_POOL_SIZE (256*1024)
-
-/* Scheduler sub queues */
-#define QUEUES_PER_PRIO 4
-
-/* TODO: random or queue based selection */
-#define SEL_PRI_QUEUE(x) ((QUEUES_PER_PRIO-1) & (queue_to_id(x)))
-
-/* Maximum number of dequeues */
-#define MAX_DEQ 4
-
-
-/* Mask of queues per priority */
-typedef uint8_t pri_mask_t;
-
-ODP_STATIC_ASSERT((8 * sizeof(pri_mask_t)) >=
- QUEUES_PER_PRIO, "pri_mask_t_is_too_small");
-
-
-typedef struct {
- odp_queue_t pri_queue[ODP_CONFIG_SCHED_PRIOS][QUEUES_PER_PRIO];
- pri_mask_t pri_mask[ODP_CONFIG_SCHED_PRIOS];
- odp_spinlock_t mask_lock;
- odp_buffer_pool_t pool;
-} sched_t;
-
-typedef struct {
- odp_queue_t queue;
-
-} queue_desc_t;
-
-typedef struct {
- odp_queue_t pri_queue;
- odp_buffer_t desc_buf;
-
- odp_buffer_t buf[MAX_DEQ];
- int num;
- int index;
- odp_queue_t queue;
- int pause;
-
-} sched_local_t;
-
-/* Global scheduler context */
-static sched_t *sched;
-
-/* Thread local scheduler context */
-static __thread sched_local_t sched_local;
-
-
-static inline odp_queue_t select_pri_queue(odp_queue_t queue, int prio)
-{
- int id = SEL_PRI_QUEUE(queue);
- return sched->pri_queue[prio][id];
-}
-
-
-int odp_schedule_init_global(void)
-{
- odp_shm_t shm;
- odp_buffer_pool_t pool;
- int i, j;
-
- ODP_DBG("Schedule init ... ");
-
- shm = odp_shm_reserve("odp_scheduler",
- sizeof(sched_t),
- ODP_CACHE_LINE_SIZE, 0);
-
- sched = odp_shm_addr(shm);
-
- if (sched == NULL) {
- ODP_ERR("Schedule init: Shm reserve failed.\n");
- return -1;
- }
-
- pool = odp_buffer_pool_create("odp_sched_pool", NULL,
- SCHED_POOL_SIZE, sizeof(queue_desc_t),
- ODP_CACHE_LINE_SIZE,
- ODP_BUFFER_TYPE_RAW);
-
- if (pool == ODP_BUFFER_POOL_INVALID) {
- ODP_ERR("Schedule init: Pool create failed.\n");
- return -1;
- }
-
- sched->pool = pool;
- odp_spinlock_init(&sched->mask_lock);
-
- for (i = 0; i < ODP_CONFIG_SCHED_PRIOS; i++) {
- odp_queue_t queue;
- char name[] = "odp_priXX_YY";
-
- name[7] = '0' + i / 10;
- name[8] = '0' + i - 10*(i / 10);
-
- for (j = 0; j < QUEUES_PER_PRIO; j++) {
- name[10] = '0' + j / 10;
- name[11] = '0' + j - 10*(j / 10);
-
- queue = odp_queue_create(name,
- ODP_QUEUE_TYPE_POLL, NULL);
-
- if (queue == ODP_QUEUE_INVALID) {
- ODP_ERR("Sched init: Queue create failed.\n");
- return -1;
- }
-
- sched->pri_queue[i][j] = queue;
- sched->pri_mask[i] = 0;
- }
- }
-
- ODP_DBG("done\n");
-
- return 0;
-}
-
-
-int odp_schedule_init_local(void)
-{
- int i;
-
- sched_local.pri_queue = ODP_QUEUE_INVALID;
- sched_local.desc_buf = ODP_BUFFER_INVALID;
-
- for (i = 0; i < MAX_DEQ; i++)
- sched_local.buf[i] = ODP_BUFFER_INVALID;
-
- sched_local.num = 0;
- sched_local.index = 0;
- sched_local.queue = ODP_QUEUE_INVALID;
- sched_local.pause = 0;
-
- return 0;
-}
-
-
-void odp_schedule_mask_set(odp_queue_t queue, int prio)
-{
- int id = SEL_PRI_QUEUE(queue);
-
- odp_spinlock_lock(&sched->mask_lock);
- sched->pri_mask[prio] |= 1 << id;
- odp_spinlock_unlock(&sched->mask_lock);
-}
-
-
-odp_buffer_t odp_schedule_buffer_alloc(odp_queue_t queue)
-{
- odp_buffer_t buf;
-
- buf = odp_buffer_alloc(sched->pool);
-
- if (buf != ODP_BUFFER_INVALID) {
- queue_desc_t *desc;
- desc = odp_buffer_addr(buf);
- desc->queue = queue;
- }
-
- return buf;
-}
-
-
-void odp_schedule_queue(odp_queue_t queue, int prio)
-{
- odp_buffer_t desc_buf;
- odp_queue_t pri_queue;
-
- pri_queue = select_pri_queue(queue, prio);
- desc_buf = queue_sched_buf(queue);
-
- odp_queue_enq(pri_queue, desc_buf);
-}
-
-
-void odp_schedule_release_atomic(void)
-{
- if (sched_local.pri_queue != ODP_QUEUE_INVALID &&
- sched_local.num == 0) {
- /* Release current atomic queue */
- odp_queue_enq(sched_local.pri_queue, sched_local.desc_buf);
- sched_local.pri_queue = ODP_QUEUE_INVALID;
- }
-}
-
-
-static inline int copy_bufs(odp_buffer_t out_buf[], unsigned int max)
-{
- int i = 0;
-
- while (sched_local.num && max) {
- out_buf[i] = sched_local.buf[sched_local.index];
- sched_local.index++;
- sched_local.num--;
- max--;
- i++;
- }
-
- return i;
-}
-
-
-/*
- * Schedule queues
- *
- * TODO: SYNC_ORDERED not implemented yet
- */
-static int schedule(odp_queue_t *out_queue, odp_buffer_t out_buf[],
- unsigned int max_num, unsigned int max_deq)
-{
- int i, j;
- int thr;
- int ret;
-
- if (sched_local.num) {
- ret = copy_bufs(out_buf, max_num);
-
- if (out_queue)
- *out_queue = sched_local.queue;
-
- return ret;
- }
-
- odp_schedule_release_atomic();
-
- if (odp_unlikely(sched_local.pause))
- return 0;
-
- thr = odp_thread_id();
-
- for (i = 0; i < ODP_CONFIG_SCHED_PRIOS; i++) {
- int id;
-
- if (sched->pri_mask[i] == 0)
- continue;
-
- id = thr & (QUEUES_PER_PRIO-1);
-
- for (j = 0; j < QUEUES_PER_PRIO; j++, id++) {
- odp_queue_t pri_q;
- odp_buffer_t desc_buf;
-
- if (id >= QUEUES_PER_PRIO)
- id = 0;
-
- if (odp_unlikely((sched->pri_mask[i] & (1 << id)) == 0))
- continue;
-
- pri_q = sched->pri_queue[i][id];
- desc_buf = odp_queue_deq(pri_q);
-
- if (desc_buf != ODP_BUFFER_INVALID) {
- queue_desc_t *desc;
- odp_queue_t queue;
- int num;
-
- desc = odp_buffer_addr(desc_buf);
- queue = desc->queue;
-
- num = odp_queue_deq_multi(queue,
- sched_local.buf,
- max_deq);
-
- if (num == 0) {
- /* Remove empty queue from scheduling,
- * except packet input queues
- */
- if (odp_queue_type(queue) ==
- ODP_QUEUE_TYPE_PKTIN)
- odp_queue_enq(pri_q, desc_buf);
-
- continue;
- }
-
- sched_local.num = num;
- sched_local.index = 0;
- ret = copy_bufs(out_buf, max_num);
-
- sched_local.queue = queue;
-
- if (queue_sched_atomic(queue)) {
- /* Hold queue during atomic access */
- sched_local.pri_queue = pri_q;
- sched_local.desc_buf = desc_buf;
- } else {
- /* Continue scheduling the queue */
- odp_queue_enq(pri_q, desc_buf);
- }
-
- /* Output the source queue handle */
- if (out_queue)
- *out_queue = queue;
-
- return ret;
- }
- }
- }
-
- return 0;
-}
-
-
-static int schedule_loop(odp_queue_t *out_queue, uint64_t wait,
- odp_buffer_t out_buf[],
- unsigned int max_num, unsigned int max_deq)
-{
- uint64_t start_cycle, cycle, diff;
- int ret;
-
- start_cycle = 0;
-
- while (1) {
- ret = schedule(out_queue, out_buf, max_num, max_deq);
-
- if (ret)
- break;
-
- if (wait == ODP_SCHED_WAIT)
- continue;
-
- if (wait == ODP_SCHED_NO_WAIT)
- break;
-
- if (start_cycle == 0) {
- start_cycle = odp_time_get_cycles();
- continue;
- }
-
- cycle = odp_time_get_cycles();
- diff = odp_time_diff_cycles(start_cycle, cycle);
-
- if (wait < diff)
- break;
- }
-
- return ret;
-}
-
-
-odp_buffer_t odp_schedule(odp_queue_t *out_queue, uint64_t wait)
-{
- odp_buffer_t buf;
-
- buf = ODP_BUFFER_INVALID;
-
- schedule_loop(out_queue, wait, &buf, 1, MAX_DEQ);
-
- return buf;
-}
-
-
-odp_buffer_t odp_schedule_one(odp_queue_t *out_queue, uint64_t wait)
-{
- odp_buffer_t buf;
-
- buf = ODP_BUFFER_INVALID;
-
- schedule_loop(out_queue, wait, &buf, 1, 1);
-
- return buf;
-}
-
-
-int odp_schedule_multi(odp_queue_t *out_queue, uint64_t wait,
- odp_buffer_t out_buf[], unsigned int num)
-{
- return schedule_loop(out_queue, wait, out_buf, num, MAX_DEQ);
-}
-
-
-void odp_schedule_pause(void)
-{
- sched_local.pause = 1;
-}
-
-
-void odp_schedule_resume(void)
-{
- sched_local.pause = 0;
-}
-
-
-uint64_t odp_schedule_wait_time(uint64_t ns)
-{
- if (ns <= ODP_SCHED_NO_WAIT)
- ns = ODP_SCHED_NO_WAIT + 1;
-
- return odp_time_ns_to_cycles(ns);
-}
-
-
-int odp_schedule_num_prio(void)
-{
- return ODP_CONFIG_SCHED_PRIOS;
-}