[v2,7/11] linux-gen: packet: implement static references

Message ID 1505221212-27163-8-git-send-email-odpbot@yandex.ru
State New
Headers show
Series
  • [v2,1/11] linux-gen: packet: roll back to copy based references
Related show

Commit Message

Github ODP bot Sept. 12, 2017, 1 p.m.
From: Petri Savolainen <petri.savolainen@linaro.org>


Implemented static references with a reference counter. Counter
is zero when not used (reference API not used).

Signed-off-by: Petri Savolainen <petri.savolainen@linaro.org>

---
/** Email created from pull request 170 (psavol:master-packet-ref-rework)
 ** https://github.com/Linaro/odp/pull/170
 ** Patch: https://github.com/Linaro/odp/pull/170.patch
 ** Base sha: fb3f36cec108ce9c55241d9f0e66d4832a552b8a
 ** Merge commit sha: bab9e010b5432ba0f2ff0651313a85a6a1b563c2
 **/
 .../linux-generic/include/odp_buffer_internal.h    | 21 +++--
 .../linux-generic/include/odp_packet_internal.h    |  5 +-
 platform/linux-generic/odp_packet.c                | 92 +++++++++++++++++++---
 platform/linux-generic/odp_pool.c                  |  2 +
 4 files changed, 102 insertions(+), 18 deletions(-)

Patch

diff --git a/platform/linux-generic/include/odp_buffer_internal.h b/platform/linux-generic/include/odp_buffer_internal.h
index 0c873d2d..cd067a08 100644
--- a/platform/linux-generic/include/odp_buffer_internal.h
+++ b/platform/linux-generic/include/odp_buffer_internal.h
@@ -62,11 +62,16 @@  struct odp_buffer_hdr_t {
 	/* Last header of the segment list */
 	void *last_seg;
 
-	/* Initial buffer data pointer and length */
+	/* Initial buffer data pointer */
 	uint8_t  *base_data;
-	uint8_t  *buf_end;
 
-	/* --- 40 bytes --- */
+	/* Reference count */
+	odp_atomic_u32_t ref_cnt;
+
+	/* Event type. Maybe different than pool type (crypto compl event) */
+	int8_t    event_type;
+
+	/* --- 37 bytes --- */
 
 	/* Segments */
 	seg_entry_t seg[CONFIG_PACKET_MAX_SEGS];
@@ -93,14 +98,17 @@  struct odp_buffer_hdr_t {
 	/* Pool pointer */
 	void *pool_ptr;
 
+	/* Initial buffer tail pointer */
+	uint8_t  *buf_end;
+
 	/* User area pointer */
 	void    *uarea_addr;
 
 	/* User area size */
 	uint32_t uarea_size;
 
-	/* Event type. Maybe different than pool type (crypto compl event) */
-	int8_t    event_type;
+	/* Max data size */
+	uint32_t size;
 
 	/* ipc mapped process can not walk over pointers,
 	 * offset has to be used */
@@ -110,9 +118,6 @@  struct odp_buffer_hdr_t {
 	 * inlining */
 	odp_pool_t pool_hdl;
 
-	/* Max data size */
-	uint32_t size;
-
 	/* Data or next header */
 	uint8_t data[0];
 } ODP_ALIGNED_CACHE;
diff --git a/platform/linux-generic/include/odp_packet_internal.h b/platform/linux-generic/include/odp_packet_internal.h
index d8e5766c..d8d4584a 100644
--- a/platform/linux-generic/include/odp_packet_internal.h
+++ b/platform/linux-generic/include/odp_packet_internal.h
@@ -116,11 +116,13 @@  typedef struct {
 
 	packet_parser_t p;
 
+	uint32_t frame_len;
+
 	odp_pktio_t input;
 
-	uint32_t frame_len;
 	uint32_t headroom;
 	uint32_t tailroom;
+	uint32_t shared_len;
 
 	/*
 	 * Members below are not initialized by packet_init()
@@ -217,6 +219,7 @@  static inline void packet_init(odp_packet_hdr_t *pkt_hdr, uint32_t len)
 	* segment occupied by the allocated length.
 	*/
 	pkt_hdr->frame_len = len;
+	pkt_hdr->shared_len = 0;
 	pkt_hdr->headroom  = CONFIG_PACKET_HEADROOM;
 	pkt_hdr->tailroom  = CONFIG_PACKET_MAX_SEG_LEN - seg_len +
 			     CONFIG_PACKET_TAILROOM;
diff --git a/platform/linux-generic/odp_packet.c b/platform/linux-generic/odp_packet.c
index 94fdffab..cde77bb5 100644
--- a/platform/linux-generic/odp_packet.c
+++ b/platform/linux-generic/odp_packet.c
@@ -491,6 +491,36 @@  static inline void copy_buf_hdr(odp_packet_hdr_t *pkt_hdr, int first, int num,
 	}
 }
 
+static inline void packet_free_multi(odp_buffer_hdr_t *hdr[], int num)
+{
+	int i;
+	uint32_t ref_cnt;
+	int num_ref = 0;
+
+	for (i = 0; i < num; i++) {
+		/* Zero when reference API has not been used */
+		ref_cnt = odp_atomic_load_u32(&hdr[i]->ref_cnt);
+
+		if (odp_unlikely(ref_cnt)) {
+			ref_cnt = odp_atomic_fetch_dec_u32(&hdr[i]->ref_cnt);
+
+			if (ref_cnt > 1) {
+				num_ref++;
+				continue;
+			}
+		}
+
+		/* Skip references and pack to be freed headers to array head */
+		if (odp_unlikely(num_ref))
+			hdr[i - num_ref] = hdr[i];
+	}
+
+	num -= num_ref;
+
+	if (odp_likely(num))
+		buffer_free_multi(hdr, num);
+}
+
 static inline void free_all_segments(odp_packet_hdr_t *pkt_hdr, int num)
 {
 	seg_entry_t *seg;
@@ -503,7 +533,7 @@  static inline void free_all_segments(odp_packet_hdr_t *pkt_hdr, int num)
 		buf_hdr[i] = seg->hdr;
 	}
 
-	buffer_free_multi(buf_hdr, num);
+	packet_free_multi(buf_hdr, num);
 }
 
 static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr,
@@ -560,7 +590,7 @@  static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr,
 
 		pkt_hdr = new_hdr;
 
-		buffer_free_multi(buf_hdr, num);
+		packet_free_multi(buf_hdr, num);
 	} else {
 		/* Free last 'num' bufs.
 		 * First, find the last remaining header. */
@@ -575,7 +605,7 @@  static inline odp_packet_hdr_t *free_segments(odp_packet_hdr_t *pkt_hdr,
 			buf_hdr[i] = seg->hdr;
 		}
 
-		buffer_free_multi(buf_hdr, num);
+		packet_free_multi(buf_hdr, num);
 
 		/* Head segment remains, no need to copy or update majority
 		 * of the metadata. */
@@ -699,7 +729,7 @@  void odp_packet_free(odp_packet_t pkt)
 	int num_seg = pkt_hdr->buf_hdr.segcount;
 
 	if (odp_likely(CONFIG_PACKET_MAX_SEGS == 1 || num_seg == 1))
-		buffer_free_multi((odp_buffer_hdr_t **)&hdl, 1);
+		packet_free_multi((odp_buffer_hdr_t **)&hdl, 1);
 	else
 		free_all_segments(pkt_hdr, num_seg);
 }
@@ -707,7 +737,7 @@  void odp_packet_free(odp_packet_t pkt)
 void odp_packet_free_multi(const odp_packet_t pkt[], int num)
 {
 	if (CONFIG_PACKET_MAX_SEGS == 1) {
-		buffer_free_multi((odp_buffer_hdr_t **)(uintptr_t)pkt, num);
+		packet_free_multi((odp_buffer_hdr_t **)(uintptr_t)pkt, num);
 	} else {
 		odp_buffer_hdr_t *buf_hdr[num * CONFIG_PACKET_MAX_SEGS];
 		int i;
@@ -728,7 +758,7 @@  void odp_packet_free_multi(const odp_packet_t pkt[], int num)
 			bufs += num_seg - 1;
 		}
 
-		buffer_free_multi(buf_hdr, bufs);
+		packet_free_multi(buf_hdr, bufs);
 	}
 }
 
@@ -1937,9 +1967,37 @@  uint64_t odp_packet_seg_to_u64(odp_packet_seg_t hdl)
 	return _odp_pri(hdl);
 }
 
+static inline void buffer_ref_inc(odp_buffer_hdr_t *buf_hdr)
+{
+	/* First count increment after alloc */
+	if (odp_atomic_load_u32(&buf_hdr->ref_cnt) == 0)
+		odp_atomic_store_u32(&buf_hdr->ref_cnt, 2);
+	else
+		odp_atomic_inc_u32(&buf_hdr->ref_cnt);
+}
+
+static inline void packet_ref_inc(odp_packet_hdr_t *pkt_hdr)
+{
+	seg_entry_t *seg;
+	int i;
+	int seg_count = pkt_hdr->buf_hdr.segcount;
+	odp_packet_hdr_t *hdr = pkt_hdr;
+	uint8_t idx = 0;
+
+	for (i = 0; i < seg_count; i++) {
+		seg = seg_entry_next(&hdr, &idx);
+		buffer_ref_inc(seg->hdr);
+	}
+}
+
 odp_packet_t odp_packet_ref_static(odp_packet_t pkt)
 {
-	return odp_packet_copy(pkt, odp_packet_pool(pkt));
+	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+
+	packet_ref_inc(pkt_hdr);
+	pkt_hdr->shared_len = packet_len(pkt_hdr);
+
+	return pkt;
 }
 
 odp_packet_t odp_packet_ref(odp_packet_t pkt, uint32_t offset)
@@ -2001,14 +2059,30 @@  odp_packet_t odp_packet_ref_pkt(odp_packet_t pkt, uint32_t offset,
 
 int odp_packet_has_ref(odp_packet_t pkt)
 {
-	(void)pkt;
+	odp_buffer_hdr_t *buf_hdr;
+	seg_entry_t *seg;
+	int i;
+	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+	int seg_count = pkt_hdr->buf_hdr.segcount;
+	odp_packet_hdr_t *hdr = pkt_hdr;
+	uint8_t idx = 0;
+
+	for (i = 0; i < seg_count; i++) {
+		seg = seg_entry_next(&hdr, &idx);
+		buf_hdr = seg->hdr;
+
+		if (odp_atomic_load_u32(&buf_hdr->ref_cnt) > 1)
+			return 1;
+	}
 
 	return 0;
 }
 
 uint32_t odp_packet_unshared_len(odp_packet_t pkt)
 {
-	return odp_packet_len(pkt);
+	odp_packet_hdr_t *pkt_hdr = packet_hdr(pkt);
+
+	return packet_len(pkt_hdr) - pkt_hdr->shared_len;
 }
 
 /* Include non-inlined versions of API functions */
diff --git a/platform/linux-generic/odp_pool.c b/platform/linux-generic/odp_pool.c
index 47a39f5b..2f65cb20 100644
--- a/platform/linux-generic/odp_pool.c
+++ b/platform/linux-generic/odp_pool.c
@@ -277,6 +277,8 @@  static void init_buffers(pool_t *pool)
 		buf_hdr->seg[0].data      = &data[offset];
 		buf_hdr->seg[0].len       = pool->data_size;
 
+		odp_atomic_init_u32(&buf_hdr->ref_cnt, 0);
+
 		/* Store base values for fast init */
 		buf_hdr->base_data = buf_hdr->seg[0].data;
 		buf_hdr->buf_end   = &data[offset + pool->data_size +