================================= test.sh ========================
function test()
{
    flag=$1
    max=$2
    inc=$3
    if [ "x$max" == "x" ]
    then
        max=4096
    fi
    echo $max

    for s in $(seq 64 $inc $max)
    do
        echo >> log
        echo $flag :UDP_STREAM $s >> log
        netperf -H 192.168.122.202 -l 5 -t UDP_STREAM -- -m $s >> log
        echo $flag :UDP_STREAM $s $?
    done

    for s in $(seq 64 $inc $max)
    do
        echo >> log
        echo $flag :TCP_STREAM $s >> log
        netperf -H 192.168.122.202 -l 5 -t TCP_STREAM -- -m $s >> log
        echo $flag :TCP_STREAM $s $?
    done
}

function test_merge()
{
    XDP='no-xdp'

    ssh root@192.168.122.202 ip link set dev eth0 xdp off
    test $XDP

    XDP='xdp-pass'

    ssh root@192.168.122.202 ip link set dev eth0 xdp off
    ssh root@192.168.122.202 ip link set dev eth0 xdp object xdp.o sec xdp
    test $XDP

    XDP='xdp-meta'

    ssh root@192.168.122.202 ip link set dev eth0 xdp off
    ssh root@192.168.122.202 ip link set dev eth0 xdp object xdp_meta.o sec xdp_mark
    test $XDP
}

function test_big()
{
    # set host net dev mtu to 60000
    test "big" 5000
    test "big" 60000 10
}

ssh root@192.168.122.202 ./netserver
echo '' > log

test_merge # or test_big
================================== xdp.c =================================

SEC("xdp")
int _xdp(struct xdp_md *xdp)
{
        return XDP_PASS;
}

char _license[] SEC("license") = "GPL";

================================== xdp_meta.c =================================

static long (*bpf_xdp_adjust_meta)(struct xdp_md *xdp_md, int delta) = (void *) 54;

struct meta_info {
        __u32 mark;
} __attribute__((aligned(4)));

SEC("xdp_mark")
int _xdp_mark(struct xdp_md *ctx)
{
        struct meta_info *meta;
        void *data, *data_end;
        int ret;

        ret = bpf_xdp_adjust_meta(ctx, -(int)sizeof(*meta));
        if (ret < 0)
                return XDP_ABORTED;

        data = (void *)(unsigned long)ctx->data;

        /* Check data_meta have room for meta_info struct */
        meta = (void *)(unsigned long)ctx->data_meta;
        if ((void *)(meta + 1) > data)
                return XDP_ABORTED;

        meta->mark = 42;

        return XDP_PASS;
}

char _license[] SEC("license") = "GPL";
===================================================================

Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com>
---
 drivers/net/virtio_net.c | 309 +++++++++++++++++++++++----------------
 1 file changed, 185 insertions(+), 124 deletions(-)

diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index 7fda2ae4c40f..a117b3496653 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -249,6 +249,35 @@ struct padded_vnet_hdr {
 	char padding[4];
 };
 
+struct virtnet_page_info {
+	struct virtnet_info *vi;
+	struct receive_queue *rq;
+
+	/* this may be the head_page, buf not starts with this page */
+	struct page *page;
+
+	/* the allcated buf. this may point to the headroom */
+	char *buf;
+
+	/* the size of the buf */
+	unsigned int buf_size;
+
+	/* OUT. the offset of the remaining data in the page */
+	unsigned int offset;
+
+	char *virtnet_hdr;
+
+	/* packet data. generally point to eth header */
+	char *packet;
+
+	/* IN. packet len without virtnet hdr
+	 * OUT. the size of the remaining data
+	 */
+	unsigned int len;
+
+	unsigned int metasize;
+};
+
 static bool is_xdp_frame(void *ptr)
 {
 	return (unsigned long)ptr & VIRTIO_XDP_FLAG;
@@ -357,6 +386,89 @@ static void skb_xmit_done(struct virtqueue *vq)
 		netif_wake_subqueue(vi->dev, vq2txq(vq));
 }
 
+static struct sk_buff *virtnet_page_to_skb(struct virtnet_page_info *pinfo)
+{
+	int shinfo_size = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
+	struct virtio_net_hdr_mrg_rxbuf *hdr;
+	struct sk_buff *skb;
+	int tailroom, copy;
+
+	/* In the case of "big", tailroom may be negative, because len can be
+	 * greater than PAGE_SIZE.
+	 */
+	tailroom = pinfo->buf + pinfo->buf_size - (pinfo->packet + pinfo->len);
+
+	if (!NET_IP_ALIGN && tailroom >= shinfo_size) {
+		skb = build_skb(pinfo->buf, pinfo->buf_size);
+		if (unlikely(!skb))
+			return NULL;
+
+		skb_reserve(skb, pinfo->packet - pinfo->buf);
+		skb_put(skb, pinfo->len);
+
+		/* mark. page has been used. */
+		pinfo->page = NULL;
+	} else {
+		/* copy small data so we can reuse these pages for small data
+		 *
+		 * GOOD_COPY_LEN is used to save network headers, such as eth
+		 * header, ip header, tcp header. If you want to save metadata
+		 * information, we should apply for a larger space. Prevent the
+		 * network header cannot fit in the linear space.
+		 */
+		skb = napi_alloc_skb(&pinfo->rq->napi,
+				     pinfo->metasize + GOOD_COPY_LEN);
+		if (unlikely(!skb))
+			return NULL;
+
+		/* Copy all frame if it fits skb->head, otherwise
+		 * we let virtio_net_hdr_to_skb() and GRO pull headers as needed.
+		 */
+		if (pinfo->len <= GOOD_COPY_LEN)
+			copy = pinfo->len;
+		else
+			copy = ETH_HLEN;
+
+		skb_put_data(skb, pinfo->packet - pinfo->metasize,
+			     copy + pinfo->metasize);
+		__skb_pull(skb, pinfo->metasize);
+		pinfo->len -= copy;
+		pinfo->offset = pinfo->packet + copy -
+				(char *)page_address(pinfo->page);
+	}
+
+	if (pinfo->metasize)
+		skb_metadata_set(skb, pinfo->metasize);
+
+	if (pinfo->virtnet_hdr) {
+		hdr = skb_vnet_hdr(skb);
+		memcpy(hdr, pinfo->virtnet_hdr, pinfo->vi->hdr_len);
+	}
+
+	return skb;
+}
+
+static struct sk_buff *virtnet_merge_page_to_skb(struct virtnet_page_info *pinfo)
+{
+	struct sk_buff *skb;
+
+	skb = virtnet_page_to_skb(pinfo);
+	if (unlikely(!skb))
+		return NULL;
+
+	/* page has been used by build_skb() */
+	if (!pinfo->page)
+		return skb;
+
+	if (pinfo->len)
+		skb_add_rx_frag(skb, 0, pinfo->page, pinfo->offset, pinfo->len,
+				pinfo->buf_size);
+	else
+		put_page(pinfo->page);
+
+	return skb;
+}
+
 #define MRG_CTX_HEADER_SHIFT 22
 static void *mergeable_len_to_ctx(unsigned int truesize,
 				  unsigned int headroom)
@@ -375,86 +487,30 @@ static unsigned int mergeable_ctx_to_truesize(void *mrg_ctx)
 }
 
 /* Called from bottom half context */
-static struct sk_buff *page_to_skb(struct virtnet_info *vi,
-				   struct receive_queue *rq,
-				   struct page *page, unsigned int offset,
-				   unsigned int len, unsigned int truesize,
-				   bool hdr_valid, unsigned int metasize,
-				   unsigned int headroom)
+static struct sk_buff *virtnet_big_page_to_skb(struct virtnet_page_info *pinfo)
 {
+	unsigned int len, offset, truesize;
+	struct receive_queue *rq;
 	struct sk_buff *skb;
-	struct virtio_net_hdr_mrg_rxbuf *hdr;
-	unsigned int copy, hdr_len, hdr_padded_len;
-	struct page *page_to_free = NULL;
-	int tailroom, shinfo_size;
-	char *p, *hdr_p, *buf;
+	struct page *page;
 
-	p = page_address(page) + offset;
-	hdr_p = p;
+	/* save next page */
+	page = (struct page *)pinfo->page->private;
 
-	hdr_len = vi->hdr_len;
-	if (vi->mergeable_rx_bufs)
-		hdr_padded_len = sizeof(*hdr);
-	else
-		hdr_padded_len = sizeof(struct padded_vnet_hdr);
-
-	/* If headroom is not 0, there is an offset between the beginning of the
-	 * data and the allocated space, otherwise the data and the allocated
-	 * space are aligned.
-	 */
-	if (headroom) {
-		/* Buffers with headroom use PAGE_SIZE as alloc size,
-		 * see add_recvbuf_mergeable() + get_mergeable_buf_len()
-		 */
-		truesize = PAGE_SIZE;
-		tailroom = truesize - len - offset;
-		buf = page_address(page);
-	} else {
-		tailroom = truesize - len;
-		buf = p;
-	}
-
-	len -= hdr_len;
-	offset += hdr_padded_len;
-	p += hdr_padded_len;
-
-	shinfo_size = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
-
-	/* copy small packet so we can reuse these pages */
-	if (!NET_IP_ALIGN && len > GOOD_COPY_LEN && tailroom >= shinfo_size) {
-		skb = build_skb(buf, truesize);
-		if (unlikely(!skb))
-			return NULL;
-
-		skb_reserve(skb, p - buf);
-		skb_put(skb, len);
-		goto ok;
-	}
-
-	/* copy small packet so we can reuse these pages for small data */
-	skb = napi_alloc_skb(&rq->napi, GOOD_COPY_LEN);
+	skb = virtnet_page_to_skb(pinfo);
 	if (unlikely(!skb))
 		return NULL;
 
-	/* Copy all frame if it fits skb->head, otherwise
-	 * we let virtio_net_hdr_to_skb() and GRO pull headers as needed.
-	 */
-	if (len <= skb_tailroom(skb))
-		copy = len;
-	else
-		copy = ETH_HLEN + metasize;
-	skb_put_data(skb, p, copy);
+	rq = pinfo->rq;
 
-	len -= copy;
-	offset += copy;
+	/* page has been used by build_skb() */
+	if (!pinfo->page)
+		goto end;
 
-	if (vi->mergeable_rx_bufs) {
-		if (len)
-			skb_add_rx_frag(skb, 0, page, offset, len, truesize);
-		else
-			page_to_free = page;
-		goto ok;
-	}
+	page     = pinfo->page;
+	len      = pinfo->len;
+	offset   = pinfo->offset;
+	truesize = pinfo->buf_size;
 
 	/*
 	 * Verify that we can indeed put this data into a skb.
@@ -477,23 +533,10 @@ static struct sk_buff *page_to_skb(struct virtnet_info *vi,
 		offset = 0;
 	}
 
+end:
 	if (page)
 		give_pages(rq, page);
 
-ok:
-	/* hdr_valid means no XDP, so we can copy the vnet header */
-	if (hdr_valid) {
-		hdr = skb_vnet_hdr(skb);
-		memcpy(hdr, hdr_p, hdr_len);
-	}
-	if (page_to_free)
-		put_page(page_to_free);
-
-	if (metasize) {
-		__skb_pull(skb, metasize);
-		skb_metadata_set(skb, metasize);
-	}
-
 	return skb;
 }
 
@@ -654,17 +697,17 @@ static unsigned int virtnet_get_headroom(struct virtnet_info *vi)
  */
 static struct page *xdp_linearize_page(struct receive_queue *rq,
 				       u16 *num_buf,
-				       struct page *p,
-				       int offset,
+				       void *buf,
 				       int page_off,
 				       unsigned int *len)
 {
 	struct page *page = alloc_page(GFP_ATOMIC);
+	struct page *p;
 
 	if (!page)
 		return NULL;
 
-	memcpy(page_address(page) + page_off, page_address(p) + offset, *len);
+	memcpy(page_address(page) + page_off, buf, *len);
 	page_off += *len;
 
 	while (--*num_buf) {
@@ -739,18 +782,18 @@ static struct sk_buff *receive_small(struct net_device *dev,
 			goto err_xdp;
 
 		if (unlikely(xdp_headroom < virtnet_get_headroom(vi))) {
-			int offset = buf - page_address(page) + header_offset;
 			unsigned int tlen = len + vi->hdr_len;
 			u16 num_buf = 1;
 
+			buf += header_offset;
+
 			xdp_headroom = virtnet_get_headroom(vi);
 			header_offset = VIRTNET_RX_PAD + xdp_headroom;
 			headroom = vi->hdr_len + header_offset;
 			buflen = SKB_DATA_ALIGN(GOOD_PACKET_LEN + headroom) +
 				 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
-			xdp_page = xdp_linearize_page(rq, &num_buf, page,
-						      offset, header_offset,
-						      &tlen);
+			xdp_page = xdp_linearize_page(rq, &num_buf, buf,
+						      header_offset, &tlen);
 			if (!xdp_page)
 				goto err_xdp;
 
@@ -842,9 +885,21 @@ static struct sk_buff *receive_big(struct net_device *dev,
 				   unsigned int len,
 				   struct virtnet_rq_stats *stats)
 {
+	struct virtnet_page_info pinfo;
 	struct page *page = buf;
-	struct sk_buff *skb =
-		page_to_skb(vi, rq, page, 0, len, PAGE_SIZE, true, 0, 0);
+	struct sk_buff *skb;
+
+	pinfo.rq          = rq;
+	pinfo.vi          = vi;
+	pinfo.page        = page;
+	pinfo.buf         = page_address(page);
+	pinfo.buf_size    = PAGE_SIZE;
+	pinfo.virtnet_hdr = pinfo.buf;
+	pinfo.packet      = pinfo.virtnet_hdr + sizeof(struct padded_vnet_hdr);
+	pinfo.len         = len - vi->hdr_len;
+	pinfo.metasize    = 0;
+
+	skb = virtnet_big_page_to_skb(&pinfo);
 
 	stats->bytes += len - vi->hdr_len;
 	if (unlikely(!skb))
@@ -870,12 +925,11 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 	struct virtio_net_hdr_mrg_rxbuf *hdr = buf;
 	u16 num_buf = virtio16_to_cpu(vi->vdev, hdr->num_buffers);
 	struct page *page = virt_to_head_page(buf);
-	int offset = buf - page_address(page);
 	struct sk_buff *head_skb, *curr_skb;
+	struct virtnet_page_info pinfo;
 	struct bpf_prog *xdp_prog;
 	unsigned int truesize = mergeable_ctx_to_truesize(ctx);
 	unsigned int headroom = mergeable_ctx_to_headroom(ctx);
-	unsigned int metasize = 0;
 	unsigned int frame_sz;
 	int err;
 
@@ -887,8 +941,8 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 	if (xdp_prog) {
 		struct xdp_frame *xdpf;
 		struct page *xdp_page;
+		void *hard_start;
 		struct xdp_buff xdp;
-		void *data;
 		u32 act;
 
 		/* Transient failure which in theory could occur if
@@ -912,54 +966,47 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 		if (unlikely(num_buf > 1 ||
 			     headroom < virtnet_get_headroom(vi))) {
 			/* linearize data for XDP */
-			xdp_page = xdp_linearize_page(rq, &num_buf,
-						      page, offset,
+			xdp_page = xdp_linearize_page(rq, &num_buf, buf,
 						      VIRTIO_XDP_HEADROOM,
 						      &len);
 			frame_sz = PAGE_SIZE;
 
 			if (!xdp_page)
 				goto err_xdp;
-			offset = VIRTIO_XDP_HEADROOM;
+
+			hard_start = page_address(xdp_page) + vi->hdr_len;
 		} else {
 			xdp_page = page;
+			hard_start = buf + vi->hdr_len - VIRTIO_XDP_HEADROOM;
 		}
 
 		/* Allow consuming headroom but reserve enough space to push
 		 * the descriptor on if we get an XDP_TX return code.
 		 */
-		data = page_address(xdp_page) + offset;
 		xdp_init_buff(&xdp, frame_sz - vi->hdr_len, &rq->xdp_rxq);
-		xdp_prepare_buff(&xdp, data - VIRTIO_XDP_HEADROOM + vi->hdr_len,
-				 VIRTIO_XDP_HEADROOM, len - vi->hdr_len, true);
+		xdp_prepare_buff(&xdp, hard_start, VIRTIO_XDP_HEADROOM,
+				 len - vi->hdr_len, true);
 
 		act = bpf_prog_run_xdp(xdp_prog, &xdp);
 		stats->xdp_packets++;
 
 		switch (act) {
 		case XDP_PASS:
-			metasize = xdp.data - xdp.data_meta;
-
-			/* recalculate offset to account for any header
-			 * adjustments and minus the metasize to copy the
-			 * metadata in page_to_skb(). Note other cases do not
-			 * build an skb and avoid using offset
-			 */
-			offset = xdp.data - page_address(xdp_page) -
-				 vi->hdr_len - metasize;
-
-			/* recalculate len if xdp.data, xdp.data_end or
-			 * xdp.data_meta were adjusted
-			 */
-			len = xdp.data_end - xdp.data + vi->hdr_len + metasize;
+			pinfo.rq          = rq;
+			pinfo.vi          = vi;
+			pinfo.page        = xdp_page;
+			pinfo.buf         = xdp.data_hard_start - vi->hdr_len;
+			pinfo.buf_size    = PAGE_SIZE;
+			pinfo.virtnet_hdr = NULL;
+			pinfo.packet      = xdp.data;
+			pinfo.len         = xdp.data_end - xdp.data;
+			pinfo.metasize    = xdp.data - xdp.data_meta;
 			/* We can only create skb based on xdp_page. */
 			if (unlikely(xdp_page != page)) {
 				rcu_read_unlock();
 				put_page(page);
-				head_skb = page_to_skb(vi, rq, xdp_page, offset,
-						       len, PAGE_SIZE, false,
-						       metasize, headroom);
-				return head_skb;
+
+				return virtnet_merge_page_to_skb(&pinfo);
 			}
 			break;
 		case XDP_TX:
@@ -1005,8 +1052,22 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 				__free_pages(xdp_page, 0);
 			goto err_xdp;
 		}
+		rcu_read_unlock();
+
+		/* pinfo has been filled inside XDP_PASS */
+	} else {
+		rcu_read_unlock();
+
+		pinfo.rq          = rq;
+		pinfo.vi          = vi;
+		pinfo.page        = page;
+		pinfo.buf         = buf - headroom;
+		pinfo.buf_size    = headroom ? PAGE_SIZE : truesize;
+		pinfo.virtnet_hdr = buf;
+		pinfo.packet      = buf + sizeof(*hdr);
+		pinfo.len         = len - sizeof(*hdr);
+		pinfo.metasize    = 0;
 	}
-	rcu_read_unlock();
 
 	if (unlikely(len > truesize)) {
 		pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
@@ -1015,14 +1076,14 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
 		goto err_skb;
 	}
 
-	head_skb = page_to_skb(vi, rq, page, offset, len, truesize, !xdp_prog,
-			       metasize, headroom);
+	head_skb = virtnet_merge_page_to_skb(&pinfo);
 	curr_skb = head_skb;
 
 	if (unlikely(!curr_skb))
 		goto err_skb;
 	while (--num_buf) {
 		int num_skb_frags;
+		int offset;
 
 		buf = virtqueue_get_buf_ctx(rq->vq, &len, &ctx);
 		if (unlikely(!buf)) {
