@@ -1442,6 +1442,42 @@ static int virtnet_receive(struct receive_queue *rq, int budget,
return stats.packets;
}
+static void virt_xsk_complete(struct send_queue *sq, u32 num, bool xsk_wakeup)
+{
+ struct xsk_buff_pool *pool;
+ struct virtnet_xsk_hdr *hdr = NULL;
+ int n;
+
+ rcu_read_lock();
+
+ sq->xsk.hdr_pro += num;
+
+ pool = rcu_dereference(sq->xsk.pool);
+ if (!pool) {
+ if (sq->xsk.hdr_pro - sq->xsk.hdr_con == sq->xsk.hdr_n)
+ hdr = rcu_replace_pointer(sq->xsk.hdr, hdr, true);
+
+ rcu_read_unlock();
+
+ kfree(hdr);
+ return;
+ }
+
+ xsk_tx_completed(pool, num);
+
+ rcu_read_unlock();
+
+ if (!xsk_wakeup || !sq->xsk.wait_slot)
+ return;
+
+ n = sq->xsk.hdr_pro - sq->xsk.hdr_con;
+
+ if (n > sq->xsk.hdr_n / 2) {
+ sq->xsk.wait_slot = false;
+ virtqueue_napi_schedule(&sq->napi, sq->vq);
+ }
+}
+
static void __free_old_xmit_ptr(struct send_queue *sq, bool in_napi,
bool xsk_wakeup,
unsigned int *_packets, unsigned int *_bytes)
@@ -1449,6 +1485,7 @@ static void __free_old_xmit_ptr(struct send_queue *sq, bool in_napi,
unsigned int packets = 0;
unsigned int bytes = 0;
unsigned int len;
+ u64 xsknum = 0;
struct virtnet_xdp_type *xtype;
struct xdp_frame *frame;
struct virtnet_xsk_hdr *xskhdr;
@@ -1469,6 +1506,7 @@ static void __free_old_xmit_ptr(struct send_queue *sq, bool in_napi,
if (xtype->type == XDP_TYPE_XSK) {
xskhdr = (struct virtnet_xsk_hdr *)xtype;
bytes += xskhdr->len;
+ xsknum += 1;
} else {
frame = xtype_get_ptr(xtype);
xdp_return_frame(frame);
@@ -1478,6 +1516,9 @@ static void __free_old_xmit_ptr(struct send_queue *sq, bool in_napi,
packets++;
}
+ if (xsknum)
+ virt_xsk_complete(sq, xsknum, xsk_wakeup);
+
*_packets = packets;
*_bytes = bytes;
}
@@ -3044,10 +3085,13 @@ static void free_receive_page_frags(struct virtnet_info *vi)
static void free_unused_bufs(struct virtnet_info *vi)
{
void *buf;
+ u32 n;
int i;
+ struct send_queue *sq;
for (i = 0; i < vi->max_queue_pairs; i++) {
struct virtqueue *vq = vi->sq[i].vq;
+ sq = vi->sq + i;
while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) {
if (!is_xdp_frame(buf)) {
dev_kfree_skb(buf);
@@ -3060,6 +3104,11 @@ static void free_unused_bufs(struct virtnet_info *vi)
xdp_return_frame(xtype_get_ptr(xtype));
}
}
+
+ n = sq->xsk.hdr_con + sq->xsk.hdr_n;
+ n -= sq->xsk.hdr_pro;
+ if (n)
+ virt_xsk_complete(sq, n, false);
}
for (i = 0; i < vi->max_queue_pairs; i++) {
When recycling packets that have been sent, call xsk_tx_completed to inform xsk which packets have been sent. If necessary, start napi to process the packets in the xsk queue. Signed-off-by: Xuan Zhuo <xuanzhuo@linux.alibaba.com> --- drivers/net/virtio_net.c | 49 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+)