[v3,6/10] linux-generic: packet: segment manipulation with references

Message ID 1501599610-28388-7-git-send-email-odpbot@yandex.ru
State New
Headers show
Series
  • [v3,1/10] linux-generic: packet: restructure inline routines to use macros
Related show

Commit Message

Github ODP bot Aug. 1, 2017, 3 p.m.
From: Bill Fischofer <bill.fischofer@linaro.org>


Add support for segment operation with references

Signed-off-by: Bill Fischofer <bill.fischofer@linaro.org>

---
/** Email created from pull request 90 (Bill-Fischofer-Linaro:pktrefs)
 ** https://github.com/Linaro/odp/pull/90
 ** Patch: https://github.com/Linaro/odp/pull/90.patch
 ** Base sha: bb0ca193f8c28036fdf8b3d6c8aa5d4f11980e6c
 ** Merge commit sha: dfef320840f96825686124135a6e509db80b7b82
 **/
 .../include/odp/api/plat/packet_inlines.h          |  40 ++---
 .../include/odp/api/plat/packet_inlines_api.h      |  16 --
 .../linux-generic/include/odp_packet_internal.h    |  55 ++++++-
 platform/linux-generic/odp_packet.c                | 170 +++++++++++++++++----
 4 files changed, 212 insertions(+), 69 deletions(-)

Patch hide | download patch | download mbox

diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h
index d0cf1390..06b049fc 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h
@@ -50,7 +50,19 @@  static inline uint32_t _odp_packet_seg_len(odp_packet_t pkt)
 /** @internal Inline function @param pkt @return */
 static inline uint32_t _odp_packet_len(odp_packet_t pkt)
 {
-	return _odp_pkt_get(pkt, uint32_t, frame_len);
+	uint32_t pkt_len = _odp_pkt_get(pkt, uint32_t, frame_len);
+	void *ref_nxt    = _odp_pkt_get(pkt, void *, ref_hdr);
+	void *ref_pkt    = (void *)pkt;
+
+	while (ref_nxt) {
+		pkt_len += _odp_pkt_get(ref_pkt, uint32_t, ref_len) -
+			_odp_pkt_get(ref_pkt, uint32_t, ref_offset);
+
+		ref_pkt = ref_nxt;
+		ref_nxt = _odp_pkt_get(ref_nxt, void *, ref_hdr);
+	}
+
+	return pkt_len;
 }
 
 /** @internal Inline function @param pkt @return */
@@ -78,12 +90,6 @@  static inline odp_pktio_t _odp_packet_input(odp_packet_t pkt)
 }
 
 /** @internal Inline function @param pkt @return */
-static inline int _odp_packet_num_segs(odp_packet_t pkt)
-{
-	return _odp_pkt_get(pkt, uint8_t, segcount);
-}
-
-/** @internal Inline function @param pkt @return */
 static inline void *_odp_packet_user_ptr(odp_packet_t pkt)
 {
 	return _odp_pkt_get(pkt, void *, user_ptr);
@@ -122,7 +128,8 @@  static inline void *_odp_packet_head(odp_packet_t pkt)
 /** @internal Inline function @param pkt @return */
 static inline int _odp_packet_is_segmented(odp_packet_t pkt)
 {
-	return _odp_pkt_get(pkt, uint8_t, segcount) > 1;
+	return _odp_pkt_get(pkt, uint8_t, segcount) > 1 ||
+		_odp_pkt_get(pkt, void *, ref_hdr) != NULL;
 }
 
 /** @internal Inline function @param pkt @return */
@@ -133,23 +140,6 @@  static inline odp_packet_seg_t _odp_packet_first_seg(odp_packet_t pkt)
 	return _odp_packet_seg_from_ndx(0);
 }
 
-/** @internal Inline function @param pkt @return */
-static inline odp_packet_seg_t _odp_packet_last_seg(odp_packet_t pkt)
-{
-	return _odp_packet_seg_from_ndx(_odp_packet_num_segs(pkt) - 1);
-}
-
-/** @internal Inline function @param pkt @param seg @return */
-static inline odp_packet_seg_t _odp_packet_next_seg(odp_packet_t pkt,
-						    odp_packet_seg_t seg)
-{
-	if (odp_unlikely(_odp_packet_seg_to_ndx(seg) >=
-			 _odp_packet_seg_to_ndx(_odp_packet_last_seg(pkt))))
-		return ODP_PACKET_SEG_INVALID;
-
-	return seg + 1;
-}
-
 /** @internal Inline function @param pkt @param offset @param len */
 static inline void _odp_packet_prefetch(odp_packet_t pkt, uint32_t offset,
 					uint32_t len)
diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h b/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h
index 233bc876..f818f820 100644
--- a/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h
+++ b/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h
@@ -48,11 +48,6 @@  _ODP_INLINE odp_pktio_t odp_packet_input(odp_packet_t pkt)
 	return _odp_packet_input(pkt);
 }
 
-_ODP_INLINE int odp_packet_num_segs(odp_packet_t pkt)
-{
-	return _odp_packet_num_segs(pkt);
-}
-
 _ODP_INLINE void *odp_packet_user_ptr(odp_packet_t pkt)
 {
 	return _odp_packet_user_ptr(pkt);
@@ -93,17 +88,6 @@  _ODP_INLINE odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt)
 	return _odp_packet_first_seg(pkt);
 }
 
-_ODP_INLINE odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt)
-{
-	return _odp_packet_last_seg(pkt);
-}
-
-_ODP_INLINE odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt,
-						 odp_packet_seg_t seg)
-{
-	return _odp_packet_next_seg(pkt, seg);
-}
-
 _ODP_INLINE void odp_packet_prefetch(odp_packet_t pkt, uint32_t offset,
 				     uint32_t len)
 {
diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h
index 1edb5b7d..d923ee23 100644
--- a/platform/linux-generic/include/odp_packet_internal.h
+++ b/platform/linux-generic/include/odp_packet_internal.h
@@ -171,6 +171,50 @@  static inline odp_packet_hdr_t *odp_packet_hdr(odp_packet_t pkt)
 	return (odp_packet_hdr_t *)(uintptr_t)pkt;
 }
 
+static inline odp_packet_hdr_t *packet_last_hdr(odp_packet_t pkt,
+						uint32_t *offset)
+{
+	odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt);
+	odp_packet_hdr_t *prev_hdr = pkt_hdr;
+	uint32_t ref_offset = 0;
+
+	while (pkt_hdr->ref_hdr) {
+		ref_offset = pkt_hdr->ref_offset;
+		prev_hdr   = pkt_hdr;
+		pkt_hdr    = pkt_hdr->ref_hdr;
+	}
+
+	if (offset) {
+		if (prev_hdr != pkt_hdr)
+			ref_offset += pkt_hdr->frame_len - prev_hdr->ref_len;
+		*offset = ref_offset;
+	}
+
+	return pkt_hdr;
+}
+
+static inline odp_packet_hdr_t *packet_prev_hdr(odp_packet_hdr_t *pkt_hdr,
+						odp_packet_hdr_t *cur_hdr,
+						uint32_t *offset)
+{
+	uint32_t ref_offset = 0;
+	odp_packet_hdr_t *prev_hdr = pkt_hdr;
+
+	while (pkt_hdr->ref_hdr != cur_hdr) {
+		ref_offset = pkt_hdr->ref_offset;
+		prev_hdr   = pkt_hdr;
+		pkt_hdr    = pkt_hdr->ref_hdr;
+	}
+
+	if (offset) {
+		if (prev_hdr != pkt_hdr)
+			ref_offset += pkt_hdr->frame_len - prev_hdr->ref_len;
+		*offset = ref_offset;
+	}
+
+	return pkt_hdr;
+}
+
 static inline odp_packet_t packet_handle(odp_packet_hdr_t *pkt_hdr)
 {
 	return (odp_packet_t)pkt_hdr;
@@ -257,7 +301,16 @@  static inline void pull_tail(odp_packet_hdr_t *pkt_hdr, uint32_t len)
 
 static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr)
 {
-	return pkt_hdr->frame_len;
+	uint32_t pkt_len = pkt_hdr->frame_len;
+	odp_packet_hdr_t *ref_hdr = pkt_hdr->ref_hdr;
+
+	while (ref_hdr) {
+		pkt_len += (pkt_hdr->ref_len - pkt_hdr->ref_offset);
+		pkt_hdr = ref_hdr;
+		ref_hdr = ref_hdr->ref_hdr;
+	}
+
+	return pkt_len;
 }
 
 static inline uint32_t packet_ref_count(odp_packet_hdr_t *pkt_hdr)
diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c
index 5e1b4f8c..bfe9f795 100644
--- a/platform/linux-generic/odp_packet.c
+++ b/platform/linux-generic/odp_packet.c
@@ -92,12 +92,13 @@  static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr,
 	return pkt_hdr->buf_hdr.seg[seg_idx].len;
 }
 
-static inline void *packet_seg_data(odp_packet_hdr_t *pkt_hdr, uint32_t seg_idx)
+static inline uint8_t *packet_seg_data(odp_packet_hdr_t *pkt_hdr,
+				       uint32_t seg_idx)
 {
 	return pkt_hdr->buf_hdr.seg[seg_idx].data;
 }
 
-static inline int packet_last_seg(odp_packet_hdr_t *pkt_hdr)
+static inline uint32_t packet_last_seg(odp_packet_hdr_t *pkt_hdr)
 {
 	if (CONFIG_PACKET_MAX_SEGS == 1)
 		return 0;
@@ -218,7 +219,15 @@  static inline void *packet_map(odp_packet_hdr_t *pkt_hdr,
 	int seg = 0;
 	int seg_count = pkt_hdr->buf_hdr.segcount;
 
-	if (odp_unlikely(offset >= pkt_hdr->frame_len))
+	/* Special processing for references */
+	while (offset >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) {
+		offset   -= (pkt_hdr->frame_len - pkt_hdr->ref_offset);
+		offset   += (pkt_hdr->ref_hdr->frame_len - pkt_hdr->ref_len);
+		pkt_hdr   = pkt_hdr->ref_hdr;
+		seg_count = pkt_hdr->buf_hdr.segcount;
+	}
+
+	if (odp_unlikely(offset > pkt_hdr->frame_len))
 		return NULL;
 
 	if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || seg_count == 1)) {
@@ -670,13 +679,47 @@  odp_event_t odp_packet_to_event(odp_packet_t pkt)
 uint32_t odp_packet_buf_len(odp_packet_t pkt)
 {
 	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+	uint32_t buf_len = 0;
 
-	return pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount;
+	do {
+		buf_len += pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount;
+		pkt_hdr  = pkt_hdr->ref_hdr;
+	} while (pkt_hdr);
+
+	return buf_len;
 }
 
-void *odp_packet_tail(odp_packet_t pkt)
+uint32_t odp_packet_unshared_len(odp_packet_t pkt)
 {
 	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+	uint32_t pkt_len = 0, offset = 0;
+
+	if (packet_ref_count(pkt_hdr) == 1)
+		pkt_hdr->unshared_len = pkt_hdr->frame_len;
+
+	do {
+		if (packet_ref_count(pkt_hdr) > 1) {
+			if (offset == 0)
+				pkt_len += pkt_hdr->unshared_len;
+			break;
+		}
+
+		pkt_len += pkt_hdr->frame_len - offset;
+		offset   = pkt_hdr->ref_offset;
+
+		if (pkt_hdr->ref_hdr)
+			offset += (pkt_hdr->ref_hdr->frame_len -
+				   pkt_hdr->ref_len);
+
+		pkt_hdr = pkt_hdr->ref_hdr;
+	} while (pkt_hdr);
+
+	return pkt_len;
+}
+
+void *odp_packet_tail(odp_packet_t pkt)
+{
+	odp_packet_hdr_t *pkt_hdr = packet_last_hdr(pkt, NULL);
 
 	return packet_tail(pkt_hdr);
 }
@@ -1286,6 +1329,46 @@  void odp_packet_ts_set(odp_packet_t pkt, odp_time_t timestamp)
 	pkt_hdr->p.input_flags.timestamp = 1;
 }
 
+int odp_packet_num_segs(odp_packet_t pkt)
+{
+	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+	uint32_t segcount = 0, i;
+	uint32_t seg_offset = 0, offset;
+
+	do {
+		segcount += pkt_hdr->buf_hdr.segcount - seg_offset;
+		offset    = pkt_hdr->ref_offset;
+		pkt_hdr   = pkt_hdr->ref_hdr;
+		if (pkt_hdr) {
+			for (i = 0, seg_offset = 0;
+			     i < pkt_hdr->buf_hdr.segcount;
+			     i++, seg_offset++) {
+				if (offset < pkt_hdr->buf_hdr.seg[i].len)
+					break;
+				offset -= pkt_hdr->buf_hdr.seg[i].len;
+			}
+		}
+	} while (pkt_hdr);
+
+	return segcount;
+}
+
+odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt)
+{
+	return _odp_packet_seg_from_ndx(odp_packet_num_segs(pkt) - 1);
+}
+
+odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg)
+{
+	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+
+	if (odp_unlikely(_odp_packet_seg_to_ndx(seg) >=
+			 packet_last_seg(pkt_hdr)))
+		return ODP_PACKET_SEG_INVALID;
+
+	return seg + 1;
+}
+
 /*
  *
  * Segment level
@@ -1296,23 +1379,53 @@  void odp_packet_ts_set(odp_packet_t pkt, odp_time_t timestamp)
 void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg)
 {
 	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+	uint32_t seg_offset = 0, offset = 0, i;
+	uint32_t seg_ndx = _odp_packet_seg_to_ndx(seg);
+
+	while (seg_ndx >= pkt_hdr->buf_hdr.segcount - seg_offset &&
+	       pkt_hdr->ref_hdr) {
+		seg_ndx -= (pkt_hdr->buf_hdr.segcount - seg_offset);
+		offset   = pkt_hdr->ref_offset;
+		pkt_hdr  = pkt_hdr->ref_hdr;
+		for (i = 0, seg_offset = 0;
+		     i < pkt_hdr->buf_hdr.segcount;
+		     i++, seg_offset++) {
+			if (offset < pkt_hdr->buf_hdr.seg[i].len)
+				break;
+			offset -= pkt_hdr->buf_hdr.seg[i].len;
+		}
+	}
 
-	if (odp_unlikely(_odp_packet_seg_to_ndx(seg) >=
-			 pkt_hdr->buf_hdr.segcount))
+	if (odp_unlikely(seg_ndx + seg_offset >= pkt_hdr->buf_hdr.segcount))
 		return NULL;
 
-	return packet_seg_data(pkt_hdr, _odp_packet_seg_to_ndx(seg));
+	return packet_seg_data(pkt_hdr, seg_ndx + seg_offset) + offset;
 }
 
 uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg)
 {
 	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+	uint32_t seg_offset = 0, offset = 0, i;
+	uint32_t seg_ndx = _odp_packet_seg_to_ndx(seg);
+
+	while (seg_ndx >= pkt_hdr->buf_hdr.segcount - seg_offset &&
+	       pkt_hdr->ref_hdr) {
+		seg_ndx -= (pkt_hdr->buf_hdr.segcount - seg_offset);
+		offset   = pkt_hdr->ref_offset;
+		pkt_hdr  = pkt_hdr->ref_hdr;
+		for (i = 0, seg_offset = 0;
+		     i < pkt_hdr->buf_hdr.segcount;
+		     i++, seg_offset++) {
+			if (offset < pkt_hdr->buf_hdr.seg[i].len)
+				break;
+			offset -= pkt_hdr->buf_hdr.seg[i].len;
+		}
+	}
 
-	if (odp_unlikely(_odp_packet_seg_to_ndx(seg) >=
-			 pkt_hdr->buf_hdr.segcount))
+	if (odp_unlikely(seg_ndx + seg_offset >= pkt_hdr->buf_hdr.segcount))
 		return 0;
 
-	return packet_seg_len(pkt_hdr, _odp_packet_seg_to_ndx(seg));
+	return packet_seg_len(pkt_hdr, seg_ndx + seg_offset) - offset;
 }
 
 /*
@@ -1358,13 +1471,15 @@  int odp_packet_rem_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len)
 {
 	odp_packet_t pkt = *pkt_ptr;
 	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
-	uint32_t pktlen = pkt_hdr->frame_len;
+	uint32_t pktlen = packet_len(pkt_hdr);
 	pool_t *pool = pkt_hdr->buf_hdr.pool_ptr;
 	odp_packet_t newpkt;
 
-	if (offset > pktlen || offset + len > pktlen)
+	if (offset + len > pktlen)
 		return -1;
 
+	ODP_ASSERT(odp_packet_unshared_len(*pkt_ptr) >= offset + len);
+
 	newpkt = odp_packet_alloc(pool->pool_hdl, pktlen - len);
 
 	if (newpkt == ODP_PACKET_INVALID)
@@ -1398,6 +1513,8 @@  int odp_packet_align(odp_packet_t *pkt, uint32_t offset, uint32_t len,
 	if (align > ODP_CACHE_LINE_SIZE)
 		return -1;
 
+	ODP_ASSERT(odp_packet_has_ref(*pkt) == 0);
+
 	if (seglen >= len) {
 		misalign = align <= 1 ? 0 :
 			ROUNDUP_ALIGN(uaddr, align) - uaddr;
@@ -1437,10 +1554,13 @@  int odp_packet_concat(odp_packet_t *dst, odp_packet_t src)
 	uint32_t dst_len = dst_hdr->frame_len;
 	uint32_t src_len = src_hdr->frame_len;
 
+	ODP_ASSERT(packet_ref_count(dst_hdr) == 1);
+
 	/* Do a copy if resulting packet would be out of segments or packets
-	 * are from different pools. */
+	 * are from different pools or src is a reference. */
 	if (odp_unlikely((dst_segs + src_segs) > CONFIG_PACKET_MAX_SEGS) ||
-	    odp_unlikely(dst_pool != src_pool)) {
+	    odp_unlikely(dst_pool != src_pool) ||
+	    odp_unlikely(packet_ref_count(src_hdr) > 1)) {
 		if (odp_packet_extend_tail(dst, src_len, NULL, NULL) >= 0) {
 			(void)odp_packet_copy_from_pkt(*dst, dst_len,
 						       src, 0, src_len);
@@ -1489,8 +1609,7 @@  int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail)
 
 odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_pool_t pool)
 {
-	odp_packet_hdr_t *srchdr = packet_hdr(pkt);
-	uint32_t pktlen = srchdr->frame_len;
+	uint32_t pktlen = odp_packet_len(pkt);
 	odp_packet_t newpkt = odp_packet_alloc(pool, pktlen);
 
 	if (newpkt != ODP_PACKET_INVALID) {
@@ -1529,7 +1648,7 @@  int odp_packet_copy_to_mem(odp_packet_t pkt, uint32_t offset,
 	uint8_t *dstaddr = (uint8_t *)dst;
 	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
 
-	if (offset + len > pkt_hdr->frame_len)
+	if (offset + len > packet_len(pkt_hdr))
 		return -1;
 
 	while (len > 0) {
@@ -1553,7 +1672,7 @@  int odp_packet_copy_from_mem(odp_packet_t pkt, uint32_t offset,
 	const uint8_t *srcaddr = (const uint8_t *)src;
 	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
 
-	if (offset + len > pkt_hdr->frame_len)
+	if (offset + len > packet_len(pkt_hdr))
 		return -1;
 
 	ODP_ASSERT(odp_packet_unshared_len(pkt) >= offset + len);
@@ -1583,10 +1702,12 @@  int odp_packet_copy_from_pkt(odp_packet_t dst, uint32_t dst_offset,
 	uint32_t src_seglen = 0; /* GCC */
 	int overlap;
 
-	if (dst_offset + len > dst_hdr->frame_len ||
-	    src_offset + len > src_hdr->frame_len)
+	if (dst_offset + len > packet_len(dst_hdr) ||
+	    src_offset + len > packet_len(src_hdr))
 		return -1;
 
+	ODP_ASSERT(odp_packet_unshared_len(dst) >= dst_offset + len);
+
 	overlap = (dst_hdr == src_hdr &&
 		   ((dst_offset <= src_offset &&
 		     dst_offset + len >= src_offset) ||
@@ -1719,7 +1840,7 @@  void odp_packet_print(odp_packet_t pkt)
 	len += snprintf(&str[len], n - len,
 			"  l4_offset    %" PRIu32 "\n", hdr->p.l4_offset);
 	len += snprintf(&str[len], n - len,
-			"  frame_len    %" PRIu32 "\n", hdr->frame_len);
+			"  frame_len    %" PRIu32 "\n", packet_len(hdr));
 	len += snprintf(&str[len], n - len,
 			"  input        %" PRIu64 "\n",
 			odp_pktio_to_u64(hdr->input));
@@ -2192,11 +2313,6 @@  int odp_packet_has_ref(odp_packet_t pkt)
 	return 0;
 }
 
-uint32_t odp_packet_unshared_len(odp_packet_t pkt)
-{
-	return odp_packet_len(pkt);
-}
-
 /* Include non-inlined versions of API functions */
 #if ODP_ABI_COMPAT == 1
 #include <odp/api/plat/packet_inlines_api.h>