@@ -651,9 +651,11 @@ struct mlx5e_channel {
/* AF_XDP zero-copy */
struct mlx5e_rq xskrq;
struct mlx5e_xdpsq xsksq;
- struct mlx5e_icosq xskicosq;
- /* xskicosq can be accessed from any CPU - the spinlock protects it. */
- spinlock_t xskicosq_lock;
+
+ /* Async ICOSQ */
+ struct mlx5e_icosq async_icosq;
+ /* async_icosq can be accessed from any CPU - the spinlock protects it. */
+ spinlock_t async_icosq_lock;
/* data path - accessed per napi poll */
struct irq_desc *irq_desc;
@@ -34,31 +34,15 @@ bool mlx5e_validate_xsk_param(struct mlx5e_params *params,
}
}
-static void mlx5e_build_xskicosq_param(struct mlx5e_priv *priv,
- u8 log_wq_size,
- struct mlx5e_sq_param *param)
-{
- void *sqc = param->sqc;
- void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
-
- mlx5e_build_sq_param_common(priv, param);
-
- MLX5_SET(wq, wq, log_wq_sz, log_wq_size);
-}
-
static void mlx5e_build_xsk_cparam(struct mlx5e_priv *priv,
struct mlx5e_params *params,
struct mlx5e_xsk_param *xsk,
struct mlx5e_channel_param *cparam)
{
- const u8 xskicosq_size = MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
-
mlx5e_build_rq_param(priv, params, xsk, &cparam->rq);
mlx5e_build_xdpsq_param(priv, params, &cparam->xdp_sq);
- mlx5e_build_xskicosq_param(priv, xskicosq_size, &cparam->icosq);
mlx5e_build_rx_cq_param(priv, params, xsk, &cparam->rx_cq);
mlx5e_build_tx_cq_param(priv, params, &cparam->tx_cq);
- mlx5e_build_ico_cq_param(priv, xskicosq_size, &cparam->icosq_cq);
}
int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params,
@@ -66,7 +50,6 @@ int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params,
struct mlx5e_channel *c)
{
struct mlx5e_channel_param *cparam;
- struct dim_cq_moder icocq_moder = {};
int err;
if (!mlx5e_validate_xsk_param(params, xsk, priv->mdev))
@@ -100,31 +83,12 @@ int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params,
if (unlikely(err))
goto err_close_tx_cq;
- err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->xskicosq.cq);
- if (unlikely(err))
- goto err_close_sq;
-
- /* Create a dedicated SQ for posting NOPs whenever we need an IRQ to be
- * triggered and NAPI to be called on the correct CPU.
- */
- err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->xskicosq);
- if (unlikely(err))
- goto err_close_icocq;
-
kvfree(cparam);
- spin_lock_init(&c->xskicosq_lock);
-
set_bit(MLX5E_CHANNEL_STATE_XSK, c->state);
return 0;
-err_close_icocq:
- mlx5e_close_cq(&c->xskicosq.cq);
-
-err_close_sq:
- mlx5e_close_xdpsq(&c->xsksq);
-
err_close_tx_cq:
mlx5e_close_cq(&c->xsksq.cq);
@@ -148,28 +112,24 @@ void mlx5e_close_xsk(struct mlx5e_channel *c)
mlx5e_close_rq(&c->xskrq);
mlx5e_close_cq(&c->xskrq.cq);
- mlx5e_close_icosq(&c->xskicosq);
- mlx5e_close_cq(&c->xskicosq.cq);
mlx5e_close_xdpsq(&c->xsksq);
mlx5e_close_cq(&c->xsksq.cq);
}
void mlx5e_activate_xsk(struct mlx5e_channel *c)
{
- mlx5e_activate_icosq(&c->xskicosq);
set_bit(MLX5E_RQ_STATE_ENABLED, &c->xskrq.state);
/* TX queue is created active. */
- spin_lock(&c->xskicosq_lock);
- mlx5e_trigger_irq(&c->xskicosq);
- spin_unlock(&c->xskicosq_lock);
+ spin_lock(&c->async_icosq_lock);
+ mlx5e_trigger_irq(&c->async_icosq);
+ spin_unlock(&c->async_icosq_lock);
}
void mlx5e_deactivate_xsk(struct mlx5e_channel *c)
{
mlx5e_deactivate_rq(&c->xskrq);
/* TX queue is disabled on close. */
- mlx5e_deactivate_icosq(&c->xskicosq);
}
static int mlx5e_redirect_xsk_rqt(struct mlx5e_priv *priv, u16 ix, u32 rqn)
@@ -26,19 +26,19 @@ int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags)
return -ENXIO;
if (!napi_if_scheduled_mark_missed(&c->napi)) {
- /* To avoid WQE overrun, don't post a NOP if XSKICOSQ is not
+ /* To avoid WQE overrun, don't post a NOP if async_icosq is not
* active and not polled by NAPI. Return 0, because the upcoming
* activate will trigger the IRQ for us.
*/
- if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &c->xskicosq.state)))
+ if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &c->async_icosq.state)))
return 0;
- if (test_and_set_bit(MLX5E_SQ_STATE_PENDING_XSK_TX, &c->xskicosq.state))
+ if (test_and_set_bit(MLX5E_SQ_STATE_PENDING_XSK_TX, &c->async_icosq.state))
return 0;
- spin_lock(&c->xskicosq_lock);
- mlx5e_trigger_irq(&c->xskicosq);
- spin_unlock(&c->xskicosq_lock);
+ spin_lock(&c->async_icosq_lock);
+ mlx5e_trigger_irq(&c->async_icosq);
+ spin_unlock(&c->async_icosq_lock);
}
return 0;
@@ -1817,10 +1817,14 @@ static int mlx5e_open_queues(struct mlx5e_channel *c,
struct dim_cq_moder icocq_moder = {0, 0};
int err;
- err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->icosq.cq);
+ err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->async_icosq.cq);
if (err)
return err;
+ err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->icosq.cq);
+ if (err)
+ goto err_close_async_icosq_cq;
+
err = mlx5e_open_tx_cqs(c, params, cparam);
if (err)
goto err_close_icosq_cq;
@@ -1841,10 +1845,16 @@ static int mlx5e_open_queues(struct mlx5e_channel *c,
napi_enable(&c->napi);
- err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq);
+ spin_lock_init(&c->async_icosq_lock);
+
+ err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->async_icosq);
if (err)
goto err_disable_napi;
+ err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq);
+ if (err)
+ goto err_close_async_icosq;
+
err = mlx5e_open_sqs(c, params, cparam);
if (err)
goto err_close_icosq;
@@ -1879,6 +1889,9 @@ static int mlx5e_open_queues(struct mlx5e_channel *c,
err_close_icosq:
mlx5e_close_icosq(&c->icosq);
+err_close_async_icosq:
+ mlx5e_close_icosq(&c->async_icosq);
+
err_disable_napi:
napi_disable(&c->napi);
@@ -1897,6 +1910,9 @@ static int mlx5e_open_queues(struct mlx5e_channel *c,
err_close_icosq_cq:
mlx5e_close_cq(&c->icosq.cq);
+err_close_async_icosq_cq:
+ mlx5e_close_cq(&c->async_icosq.cq);
+
return err;
}
@@ -1908,6 +1924,7 @@ static void mlx5e_close_queues(struct mlx5e_channel *c)
mlx5e_close_xdpsq(&c->rq_xdpsq);
mlx5e_close_sqs(c);
mlx5e_close_icosq(&c->icosq);
+ mlx5e_close_icosq(&c->async_icosq);
napi_disable(&c->napi);
if (c->xdp)
mlx5e_close_cq(&c->rq_xdpsq.cq);
@@ -1915,6 +1932,7 @@ static void mlx5e_close_queues(struct mlx5e_channel *c)
mlx5e_close_cq(&c->xdpsq.cq);
mlx5e_close_tx_cqs(c);
mlx5e_close_cq(&c->icosq.cq);
+ mlx5e_close_cq(&c->async_icosq.cq);
}
static u8 mlx5e_enumerate_lag_port(struct mlx5_core_dev *mdev, int ix)
@@ -1995,6 +2013,7 @@ static void mlx5e_activate_channel(struct mlx5e_channel *c)
for (tc = 0; tc < c->num_tc; tc++)
mlx5e_activate_txqsq(&c->sq[tc]);
mlx5e_activate_icosq(&c->icosq);
+ mlx5e_activate_icosq(&c->async_icosq);
mlx5e_activate_rq(&c->rq);
if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
@@ -2009,6 +2028,7 @@ static void mlx5e_deactivate_channel(struct mlx5e_channel *c)
mlx5e_deactivate_xsk(c);
mlx5e_deactivate_rq(&c->rq);
+ mlx5e_deactivate_icosq(&c->async_icosq);
mlx5e_deactivate_icosq(&c->icosq);
for (tc = 0; tc < c->num_tc; tc++)
mlx5e_deactivate_txqsq(&c->sq[tc]);
@@ -149,17 +149,17 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget)
}
mlx5e_poll_ico_cq(&c->icosq.cq);
+ if (mlx5e_poll_ico_cq(&c->async_icosq.cq))
+ /* Don't clear the flag if nothing was polled to prevent
+ * queueing more WQEs and overflowing XSKICOSQ.
+ */
+ clear_bit(MLX5E_SQ_STATE_PENDING_XSK_TX, &c->async_icosq.state);
busy |= INDIRECT_CALL_2(rq->post_wqes,
mlx5e_post_rx_mpwqes,
mlx5e_post_rx_wqes,
rq);
if (xsk_open) {
- if (mlx5e_poll_ico_cq(&c->xskicosq.cq))
- /* Don't clear the flag if nothing was polled to prevent
- * queueing more WQEs and overflowing XSKICOSQ.
- */
- clear_bit(MLX5E_SQ_STATE_PENDING_XSK_TX, &c->xskicosq.state);
busy |= mlx5e_poll_xdpsq_cq(&xsksq->cq);
busy_xsk |= mlx5e_napi_xsk_post(xsksq, xskrq);
}
@@ -189,11 +189,11 @@ int mlx5e_napi_poll(struct napi_struct *napi, int budget)
mlx5e_cq_arm(&rq->cq);
mlx5e_cq_arm(&c->icosq.cq);
+ mlx5e_cq_arm(&c->async_icosq.cq);
mlx5e_cq_arm(&c->xdpsq.cq);
if (xsk_open) {
mlx5e_handle_rx_dim(xskrq);
- mlx5e_cq_arm(&c->xskicosq.cq);
mlx5e_cq_arm(&xsksq->cq);
mlx5e_cq_arm(&xskrq->cq);
}