[PATCHv2,08/11] linux-generic: packet: segment manipulation with references

Message ID	20170312123045.3852-9-bill.fischofer@linaro.org
State	New
Headers	show Delivered-To: patch@linaro.org Received-SPF: pass (google.com: domain of lng-odp-bounces@lists.linaro.org designates 54.225.227.206 as permitted sender) client-ip=54.225.227.206; From: Bill Fischofer <bill.fischofer@linaro.org> To: lng-odp@lists.linaro.org Date: Sun, 12 Mar 2017 07:30:42 -0500 Message-Id: <20170312123045.3852-9-bill.fischofer@linaro.org> In-Reply-To: <20170312123045.3852-1-bill.fischofer@linaro.org> References: <20170312123045.3852-1-bill.fischofer@linaro.org> Subject: [lng-odp] [PATCHv2 08/11] linux-generic: packet: segment manipulation with references Precedence: list Errors-To: lng-odp-bounces@lists.linaro.org Sender: "lng-odp" <lng-odp-bounces@lists.linaro.org>
Series	Optimized Packet References \| expand [PATCHv2,00/11] Optimized Packet References [PATCHv2,01/11] linux-generic: pool: restructure pool routines [PATCHv2,02/11] linux-generic: packet: move pull_tail to packet mainline for consistency [PATCHv2,03/11] linux-generic: packet: restructure inline routines to use macros [PATCHv2,04/11] linux-generic: packet: add packet fields needed for references [PATCHv2,05/11] linux-generic: packet: add routines for manipulating reference counts [PATCHv2,06/11] linux-generic: packet: add initializations for new hdr fields [PATCHv2,07/11] linux-generic: packet: unshared_len maintenance [PATCHv2,08/11] linux-generic: packet: segment manipulation with references [PATCHv2,09/11] linux-generic: packet: restructure free routines for references [PATCHv2,10/11] linux-generic: packet: add reference support for head/tail manipulation [PATCHv2,11/11] linux-generic: packet: enable zero-copy reference creation

diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines.h b/platform/linux-generic/include/odp/api/plat/packet_inlines.h index 02295b08..8bfd85fe 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines.h @@ -40,7 +40,19 @@ static inline uint32_t _odp_packet_seg_len(odp_packet_t pkt) /** @internal Inline function @param pkt @return */ static inline uint32_t _odp_packet_len(odp_packet_t pkt) { - return _odp_pkt_get(pkt, uint32_t, frame_len); + uint32_t pkt_len = _odp_pkt_get(pkt, uint32_t, frame_len); + void *ref_nxt = _odp_pkt_get(pkt, void *, ref_hdr); + void *ref_pkt = (void *)pkt; + + while (ref_nxt) { + pkt_len += _odp_pkt_get(ref_pkt, uint32_t, ref_len) - + _odp_pkt_get(ref_pkt, uint32_t, ref_offset); + + ref_pkt = ref_nxt; + ref_nxt = _odp_pkt_get(ref_nxt, void *, ref_hdr); + } + + return pkt_len; } /** @internal Inline function @param pkt @return */ @@ -68,12 +80,6 @@ static inline odp_pktio_t _odp_packet_input(odp_packet_t pkt) } /** @internal Inline function @param pkt @return */ -static inline int _odp_packet_num_segs(odp_packet_t pkt) -{ - return _odp_pkt_get(pkt, uint8_t, segcount); -} - -/** @internal Inline function @param pkt @return */ static inline void *_odp_packet_user_ptr(odp_packet_t pkt) { return _odp_pkt_get(pkt, void *, user_ptr); @@ -112,7 +118,8 @@ static inline void *_odp_packet_head(odp_packet_t pkt) /** @internal Inline function @param pkt @return */ static inline int _odp_packet_is_segmented(odp_packet_t pkt) { - return _odp_pkt_get(pkt, uint8_t, segcount) > 1; + return _odp_pkt_get(pkt, uint8_t, segcount) > 1 || + _odp_pkt_get(pkt, void *, ref_hdr) != NULL; } /** @internal Inline function @param pkt @return */ @@ -123,22 +130,6 @@ static inline odp_packet_seg_t _odp_packet_first_seg(odp_packet_t pkt) return 0; } -/** @internal Inline function @param pkt @return */ -static inline odp_packet_seg_t _odp_packet_last_seg(odp_packet_t pkt) -{ - return _odp_packet_num_segs(pkt) - 1; -} - -/** @internal Inline function @param pkt @param seg @return */ -static inline odp_packet_seg_t _odp_packet_next_seg(odp_packet_t pkt, - odp_packet_seg_t seg) -{ - if (odp_unlikely(seg >= _odp_packet_last_seg(pkt))) - return ODP_PACKET_SEG_INVALID; - - return seg + 1; -} - /** @internal Inline function @param pkt @param offset @param len */ static inline void _odp_packet_prefetch(odp_packet_t pkt, uint32_t offset, uint32_t len) diff --git a/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h b/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h index 233bc876..f818f820 100644 --- a/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h +++ b/platform/linux-generic/include/odp/api/plat/packet_inlines_api.h @@ -48,11 +48,6 @@ _ODP_INLINE odp_pktio_t odp_packet_input(odp_packet_t pkt) return _odp_packet_input(pkt); } -_ODP_INLINE int odp_packet_num_segs(odp_packet_t pkt) -{ - return _odp_packet_num_segs(pkt); -} - _ODP_INLINE void *odp_packet_user_ptr(odp_packet_t pkt) { return _odp_packet_user_ptr(pkt); @@ -93,17 +88,6 @@ _ODP_INLINE odp_packet_seg_t odp_packet_first_seg(odp_packet_t pkt) return _odp_packet_first_seg(pkt); } -_ODP_INLINE odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) -{ - return _odp_packet_last_seg(pkt); -} - -_ODP_INLINE odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, - odp_packet_seg_t seg) -{ - return _odp_packet_next_seg(pkt, seg); -} - _ODP_INLINE void odp_packet_prefetch(odp_packet_t pkt, uint32_t offset, uint32_t len) { diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h index b6afd581..c99e9843 100644 --- a/platform/linux-generic/include/odp_packet_internal.h +++ b/platform/linux-generic/include/odp_packet_internal.h @@ -171,6 +171,50 @@ static inline odp_packet_hdr_t *odp_packet_hdr(odp_packet_t pkt) return (odp_packet_hdr_t *)(uintptr_t)pkt; } +static inline odp_packet_hdr_t *packet_last_hdr(odp_packet_t pkt, + uint32_t *offset) +{ + odp_packet_hdr_t *pkt_hdr = odp_packet_hdr(pkt); + odp_packet_hdr_t *prev_hdr = pkt_hdr; + uint32_t ref_offset = 0; + + while (pkt_hdr->ref_hdr) { + ref_offset = pkt_hdr->ref_offset; + prev_hdr = pkt_hdr; + pkt_hdr = pkt_hdr->ref_hdr; + } + + if (offset) { + if (prev_hdr != pkt_hdr) + ref_offset += pkt_hdr->frame_len - prev_hdr->ref_len; + *offset = ref_offset; + } + + return pkt_hdr; +} + +static inline odp_packet_hdr_t *packet_prev_hdr(odp_packet_hdr_t *pkt_hdr, + odp_packet_hdr_t *cur_hdr, + uint32_t *offset) +{ + uint32_t ref_offset = 0; + odp_packet_hdr_t *prev_hdr = pkt_hdr; + + while (pkt_hdr->ref_hdr != cur_hdr) { + ref_offset = pkt_hdr->ref_offset; + prev_hdr = pkt_hdr; + pkt_hdr = pkt_hdr->ref_hdr; + } + + if (offset) { + if (prev_hdr != pkt_hdr) + ref_offset += pkt_hdr->frame_len - prev_hdr->ref_len; + *offset = ref_offset; + } + + return pkt_hdr; +} + static inline odp_packet_t packet_handle(odp_packet_hdr_t *pkt_hdr) { return (odp_packet_t)pkt_hdr; @@ -194,7 +238,16 @@ static inline void copy_packet_cls_metadata(odp_packet_hdr_t *src_hdr, static inline uint32_t packet_len(odp_packet_hdr_t *pkt_hdr) { - return pkt_hdr->frame_len; + uint32_t pkt_len = pkt_hdr->frame_len; + odp_packet_hdr_t *ref_hdr = pkt_hdr->ref_hdr; + + while (ref_hdr) { + pkt_len += (pkt_hdr->ref_len - pkt_hdr->ref_offset); + pkt_hdr = ref_hdr; + ref_hdr = ref_hdr->ref_hdr; + } + + return pkt_len; } static inline uint32_t packet_ref_count(odp_packet_hdr_t *pkt_hdr) diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c index d4910214..c9cb3ef2 100644 --- a/platform/linux-generic/odp_packet.c +++ b/platform/linux-generic/odp_packet.c @@ -54,7 +54,7 @@ static inline odp_packet_hdr_t *packet_hdr(odp_packet_t pkt) static inline odp_buffer_t buffer_handle(odp_packet_hdr_t *pkt_hdr) { - return pkt_hdr->buf_hdr.handle.handle; + return odp_hdr_to_buf(&pkt_hdr->buf_hdr); } static inline uint32_t packet_ref_inc(odp_packet_hdr_t *pkt_hdr) @@ -78,7 +78,8 @@ static inline uint32_t packet_seg_len(odp_packet_hdr_t *pkt_hdr, return pkt_hdr->buf_hdr.seg[seg_idx].len; } -static inline void *packet_seg_data(odp_packet_hdr_t *pkt_hdr, uint32_t seg_idx) +static inline uint8_t *packet_seg_data(odp_packet_hdr_t *pkt_hdr, + uint32_t seg_idx) { return pkt_hdr->buf_hdr.seg[seg_idx].data; } @@ -216,7 +217,15 @@ static inline void *packet_map(odp_packet_hdr_t *pkt_hdr, int seg = 0; int seg_count = pkt_hdr->buf_hdr.segcount; - if (odp_unlikely(offset >= pkt_hdr->frame_len)) + /* Special processing for references */ + while (offset >= pkt_hdr->frame_len && pkt_hdr->ref_hdr) { + offset -= (pkt_hdr->frame_len - pkt_hdr->ref_offset); + offset += (pkt_hdr->ref_hdr->frame_len - pkt_hdr->ref_len); + pkt_hdr = pkt_hdr->ref_hdr; + seg_count = pkt_hdr->buf_hdr.segcount; + } + + if (odp_unlikely(offset > pkt_hdr->frame_len)) return NULL; if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || seg_count == 1)) { @@ -719,13 +728,47 @@ odp_event_t odp_packet_to_event(odp_packet_t pkt) uint32_t odp_packet_buf_len(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + uint32_t buf_len = 0; + + do { + buf_len += pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount; + pkt_hdr = pkt_hdr->ref_hdr; + } while (pkt_hdr); - return pkt_hdr->buf_hdr.size * pkt_hdr->buf_hdr.segcount; + return buf_len; } -void *odp_packet_tail(odp_packet_t pkt) +uint32_t odp_packet_unshared_len(odp_packet_t pkt) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + uint32_t pkt_len = 0, offset = 0; + + if (packet_ref_count(pkt_hdr) == 1) + pkt_hdr->unshared_len = pkt_hdr->frame_len; + + do { + if (packet_ref_count(pkt_hdr) > 1) { + if (offset == 0) + pkt_len += pkt_hdr->unshared_len; + break; + } + + pkt_len += pkt_hdr->frame_len - offset; + offset = pkt_hdr->ref_offset; + + if (pkt_hdr->ref_hdr) + offset += (pkt_hdr->ref_hdr->frame_len - + pkt_hdr->ref_len); + + pkt_hdr = pkt_hdr->ref_hdr; + } while (pkt_hdr); + + return pkt_len; +} + +void *odp_packet_tail(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = packet_last_hdr(pkt, NULL); return packet_tail(pkt_hdr); } @@ -1360,6 +1403,45 @@ void odp_packet_ts_set(odp_packet_t pkt, odp_time_t timestamp) pkt_hdr->p.input_flags.timestamp = 1; } +int odp_packet_num_segs(odp_packet_t pkt) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + uint32_t segcount = 0, i; + uint32_t seg_offset = 0, offset; + + do { + segcount += pkt_hdr->buf_hdr.segcount - seg_offset; + offset = pkt_hdr->ref_offset; + pkt_hdr = pkt_hdr->ref_hdr; + if (pkt_hdr) { + for (i = 0, seg_offset = 0; + i < pkt_hdr->buf_hdr.segcount; + i++, seg_offset++) { + if (offset < pkt_hdr->buf_hdr.seg[i].len) + break; + offset -= pkt_hdr->buf_hdr.seg[i].len; + } + } + } while (pkt_hdr); + + return segcount; +} + +odp_packet_seg_t odp_packet_last_seg(odp_packet_t pkt) +{ + return (odp_packet_seg_t)(odp_packet_num_segs(pkt) - 1); +} + +odp_packet_seg_t odp_packet_next_seg(odp_packet_t pkt, odp_packet_seg_t seg) +{ + odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + + if (odp_unlikely(seg >= packet_last_seg(pkt_hdr))) + return ODP_PACKET_SEG_INVALID; + + return seg + 1; +} + /* * * Segment level @@ -1370,21 +1452,51 @@ void odp_packet_ts_set(odp_packet_t pkt, odp_time_t timestamp) void *odp_packet_seg_data(odp_packet_t pkt, odp_packet_seg_t seg) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + uint32_t seg_offset = 0, offset = 0, i; + + while (seg >= pkt_hdr->buf_hdr.segcount - seg_offset && + pkt_hdr->ref_hdr) { + seg -= (pkt_hdr->buf_hdr.segcount - seg_offset); + offset = pkt_hdr->ref_offset; + pkt_hdr = pkt_hdr->ref_hdr; + for (i = 0, seg_offset = 0; + i < pkt_hdr->buf_hdr.segcount; + i++, seg_offset++) { + if (offset < pkt_hdr->buf_hdr.seg[i].len) + break; + offset -= pkt_hdr->buf_hdr.seg[i].len; + } + } - if (odp_unlikely(seg >= pkt_hdr->buf_hdr.segcount)) + if (odp_unlikely(seg + seg_offset >= pkt_hdr->buf_hdr.segcount)) return NULL; - return packet_seg_data(pkt_hdr, seg); + return packet_seg_data(pkt_hdr, seg + seg_offset) + offset; } uint32_t odp_packet_seg_data_len(odp_packet_t pkt, odp_packet_seg_t seg) { odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); + uint32_t seg_offset = 0, offset = 0, i; + + while (seg >= pkt_hdr->buf_hdr.segcount - seg_offset && + pkt_hdr->ref_hdr) { + seg -= (pkt_hdr->buf_hdr.segcount - seg_offset); + offset = pkt_hdr->ref_offset; + pkt_hdr = pkt_hdr->ref_hdr; + for (i = 0, seg_offset = 0; + i < pkt_hdr->buf_hdr.segcount; + i++, seg_offset++) { + if (offset < pkt_hdr->buf_hdr.seg[i].len) + break; + offset -= pkt_hdr->buf_hdr.seg[i].len; + } + } - if (odp_unlikely(seg >= pkt_hdr->buf_hdr.segcount)) + if (odp_unlikely(seg + seg_offset >= pkt_hdr->buf_hdr.segcount)) return 0; - return packet_seg_len(pkt_hdr, seg); + return packet_seg_len(pkt_hdr, seg + seg_offset) - offset; } /* @@ -1398,7 +1510,7 @@ int odp_packet_add_data(odp_packet_t *pkt_ptr, uint32_t offset, uint32_t len) { odp_packet_t pkt = *pkt_ptr; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - uint32_t pktlen = pkt_hdr->frame_len; + uint32_t pktlen = packet_len(pkt_hdr); odp_packet_t newpkt; if (offset > pktlen) @@ -1468,6 +1580,8 @@ int odp_packet_align(odp_packet_t *pkt, uint32_t offset, uint32_t len, if (align > ODP_CACHE_LINE_SIZE) return -1; + ODP_ASSERT(odp_packet_has_ref(*pkt) == 0); + if (seglen >= len) { misalign = align <= 1 ? 0 : ROUNDUP_ALIGN(uaddr, align) - uaddr; @@ -1507,10 +1621,13 @@ int odp_packet_concat(odp_packet_t *dst, odp_packet_t src) uint32_t dst_len = dst_hdr->frame_len; uint32_t src_len = src_hdr->frame_len; + ODP_ASSERT(packet_ref_count(dst_hdr) == 1); + /* Do a copy if resulting packet would be out of segments or packets - * are from different pools. */ + * are from different pools or src is a reference. */ if (odp_unlikely((dst_segs + src_segs) > CONFIG_PACKET_MAX_SEGS) || - odp_unlikely(dst_pool != src_pool)) { + odp_unlikely(dst_pool != src_pool) || + odp_unlikely(packet_ref_count(src_hdr)) > 1) { if (odp_packet_extend_tail(dst, src_len, NULL, NULL) >= 0) { (void)odp_packet_copy_from_pkt(*dst, dst_len, src, 0, src_len); @@ -1559,8 +1676,7 @@ int odp_packet_split(odp_packet_t *pkt, uint32_t len, odp_packet_t *tail) odp_packet_t odp_packet_copy(odp_packet_t pkt, odp_pool_t pool) { - odp_packet_hdr_t *srchdr = packet_hdr(pkt); - uint32_t pktlen = srchdr->frame_len; + uint32_t pktlen = odp_packet_len(pkt); odp_packet_t newpkt = odp_packet_alloc(pool, pktlen); if (newpkt != ODP_PACKET_INVALID) { @@ -1599,7 +1715,7 @@ int odp_packet_copy_to_mem(odp_packet_t pkt, uint32_t offset, uint8_t *dstaddr = (uint8_t *)dst; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - if (offset + len > pkt_hdr->frame_len) + if (offset + len > packet_len(pkt_hdr)) return -1; while (len > 0) { @@ -1623,7 +1739,7 @@ int odp_packet_copy_from_mem(odp_packet_t pkt, uint32_t offset, const uint8_t *srcaddr = (const uint8_t *)src; odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt); - if (offset + len > pkt_hdr->frame_len) + if (offset + len > packet_len(pkt_hdr)) return -1; ODP_ASSERT(odp_packet_unshared_len(pkt) >= offset + len); @@ -1653,10 +1769,12 @@ int odp_packet_copy_from_pkt(odp_packet_t dst, uint32_t dst_offset, uint32_t src_seglen = 0; /* GCC */ int overlap; - if (dst_offset + len > dst_hdr->frame_len || - src_offset + len > src_hdr->frame_len) + if (dst_offset + len > packet_len(dst_hdr) || + src_offset + len > packet_len(src_hdr)) return -1; + ODP_ASSERT(odp_packet_unshared_len(dst) >= dst_offset + len); + overlap = (dst_hdr == src_hdr && ((dst_offset <= src_offset && dst_offset + len >= src_offset) || @@ -1741,7 +1859,7 @@ void odp_packet_print(odp_packet_t pkt) len += snprintf(&str[len], n - len, " l4_offset %" PRIu32 "\n", hdr->p.l4_offset); len += snprintf(&str[len], n - len, - " frame_len %" PRIu32 "\n", hdr->frame_len); + " frame_len %" PRIu32 "\n", packet_len(hdr)); len += snprintf(&str[len], n - len, " input %" PRIu64 "\n", odp_pktio_to_u64(hdr->input)); @@ -2268,11 +2386,6 @@ int odp_packet_has_ref(odp_packet_t pkt) return 0; } -uint32_t odp_packet_unshared_len(odp_packet_t pkt) -{ - return odp_packet_len(pkt); -} - /* Include non-inlined versions of API functions */ #if ODP_ABI_COMPAT == 1 #include <odp/api/plat/packet_inlines_api.h>

[PATCHv2,08/11] linux-generic: packet: segment manipulation with references

Commit Message

Patch