diff mbox series

[net-next,4/6] ionic: batch rx buffer refilling

Message ID 20201104223354.63856-5-snelson@pensando.io
State Superseded
Headers show
Series [net-next,1/6] ionic: start queues before announcing link up | expand

Commit Message

Shannon Nelson Nov. 4, 2020, 10:33 p.m. UTC
We don't need to refill the rx descriptors on every napi
if only a few were handled.  Waiting until we can batch up
a few together will save us a few Rx cycles.

Signed-off-by: Shannon Nelson <snelson@pensando.io>
---
 .../net/ethernet/pensando/ionic/ionic_dev.h    |  4 +++-
 .../net/ethernet/pensando/ionic/ionic_txrx.c   | 18 ++++++++++--------
 2 files changed, 13 insertions(+), 9 deletions(-)

Comments

Saeed Mahameed Nov. 5, 2020, 1:08 a.m. UTC | #1
On Wed, 2020-11-04 at 14:33 -0800, Shannon Nelson wrote:
> We don't need to refill the rx descriptors on every napi

> if only a few were handled.  Waiting until we can batch up

> a few together will save us a few Rx cycles.

> 

> Signed-off-by: Shannon Nelson <snelson@pensando.io>

> ---

>  .../net/ethernet/pensando/ionic/ionic_dev.h    |  4 +++-

>  .../net/ethernet/pensando/ionic/ionic_txrx.c   | 18 ++++++++++----

> ----

>  2 files changed, 13 insertions(+), 9 deletions(-)

> 

> diff --git a/drivers/net/ethernet/pensando/ionic/ionic_dev.h

> b/drivers/net/ethernet/pensando/ionic/ionic_dev.h

> index 6c243b17312c..9064222a087a 100644

> --- a/drivers/net/ethernet/pensando/ionic/ionic_dev.h

> +++ b/drivers/net/ethernet/pensando/ionic/ionic_dev.h

> @@ -12,8 +12,10 @@

>  

>  #define IONIC_MAX_TX_DESC		8192

>  #define IONIC_MAX_RX_DESC		16384

> -#define IONIC_MIN_TXRX_DESC		16

> +#define IONIC_MIN_TXRX_DESC		64

>  #define IONIC_DEF_TXRX_DESC		4096

> +#define IONIC_RX_FILL_THRESHOLD		64


isn't 64 a bit high ? 64 is the default napi budget 

Many drivers do this with bulks of 8/16 but I couldn't find any with
higher numbers.

also, just for a reference, GRO and XDP they bulk up to 8. but not
more.

IMHO i think you need to relax the re-fill threshold a bit.

> +#define IONIC_RX_FILL_DIV		8

> 

...

> -	if (work_done)

> +	rx_fill_threshold = min_t(u16, IONIC_RX_FILL_THRESHOLD,

> +				  cq->num_descs / IONIC_RX_FILL_DIV);

> +	if (work_done && ionic_q_space_avail(cq->bound_q) >=

> rx_fill_threshold)

>  		ionic_rx_fill(cq->bound_q);

>
Shannon Nelson Nov. 5, 2020, 6:49 p.m. UTC | #2
On 11/4/20 5:08 PM, Saeed Mahameed wrote:
> On Wed, 2020-11-04 at 14:33 -0800, Shannon Nelson wrote:
>> We don't need to refill the rx descriptors on every napi
>> if only a few were handled.  Waiting until we can batch up
>> a few together will save us a few Rx cycles.
>>
>> Signed-off-by: Shannon Nelson <snelson@pensando.io>
>> ---
>>   .../net/ethernet/pensando/ionic/ionic_dev.h    |  4 +++-
>>   .../net/ethernet/pensando/ionic/ionic_txrx.c   | 18 ++++++++++----
>> ----
>>   2 files changed, 13 insertions(+), 9 deletions(-)
>>
>> diff --git a/drivers/net/ethernet/pensando/ionic/ionic_dev.h
>> b/drivers/net/ethernet/pensando/ionic/ionic_dev.h
>> index 6c243b17312c..9064222a087a 100644
>> --- a/drivers/net/ethernet/pensando/ionic/ionic_dev.h
>> +++ b/drivers/net/ethernet/pensando/ionic/ionic_dev.h
>> @@ -12,8 +12,10 @@
>>   
>>   #define IONIC_MAX_TX_DESC		8192
>>   #define IONIC_MAX_RX_DESC		16384
>> -#define IONIC_MIN_TXRX_DESC		16
>> +#define IONIC_MIN_TXRX_DESC		64
>>   #define IONIC_DEF_TXRX_DESC		4096
>> +#define IONIC_RX_FILL_THRESHOLD		64
> isn't 64 a bit high ? 64 is the default napi budget
>
> Many drivers do this with bulks of 8/16 but I couldn't find any with
> higher numbers.
>
> also, just for a reference, GRO and XDP they bulk up to 8. but not
> more.
>
> IMHO i think you need to relax the re-fill threshold a bit.

Yeah, the work being done internally on tuning has been a bit 
aggressive, I can dial this back a bit.
sln

>
>> +#define IONIC_RX_FILL_DIV		8
>>
> ...
>
>> -	if (work_done)
>> +	rx_fill_threshold = min_t(u16, IONIC_RX_FILL_THRESHOLD,
>> +				  cq->num_descs / IONIC_RX_FILL_DIV);
>> +	if (work_done && ionic_q_space_avail(cq->bound_q) >=
>> rx_fill_threshold)
>>   		ionic_rx_fill(cq->bound_q);
>>   
>
diff mbox series

Patch

diff --git a/drivers/net/ethernet/pensando/ionic/ionic_dev.h b/drivers/net/ethernet/pensando/ionic/ionic_dev.h
index 6c243b17312c..9064222a087a 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_dev.h
+++ b/drivers/net/ethernet/pensando/ionic/ionic_dev.h
@@ -12,8 +12,10 @@ 
 
 #define IONIC_MAX_TX_DESC		8192
 #define IONIC_MAX_RX_DESC		16384
-#define IONIC_MIN_TXRX_DESC		16
+#define IONIC_MIN_TXRX_DESC		64
 #define IONIC_DEF_TXRX_DESC		4096
+#define IONIC_RX_FILL_THRESHOLD		64
+#define IONIC_RX_FILL_DIV		8
 #define IONIC_LIFS_MAX			1024
 #define IONIC_WATCHDOG_SECS		5
 #define IONIC_ITR_COAL_USEC_DEFAULT	64
diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
index b3d2250c77d0..9156c9825a16 100644
--- a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
+++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c
@@ -392,11 +392,6 @@  void ionic_rx_fill(struct ionic_queue *q)
 			 q->dbval | q->head_idx);
 }
 
-static void ionic_rx_fill_cb(void *arg)
-{
-	ionic_rx_fill(arg);
-}
-
 void ionic_rx_empty(struct ionic_queue *q)
 {
 	struct ionic_desc_info *desc_info;
@@ -480,6 +475,7 @@  int ionic_rx_napi(struct napi_struct *napi, int budget)
 	struct ionic_cq *cq = napi_to_cq(napi);
 	struct ionic_dev *idev;
 	struct ionic_lif *lif;
+	u16 rx_fill_threshold;
 	u32 work_done = 0;
 	u32 flags = 0;
 
@@ -489,7 +485,9 @@  int ionic_rx_napi(struct napi_struct *napi, int budget)
 	work_done = ionic_cq_service(cq, budget,
 				     ionic_rx_service, NULL, NULL);
 
-	if (work_done)
+	rx_fill_threshold = min_t(u16, IONIC_RX_FILL_THRESHOLD,
+				  cq->num_descs / IONIC_RX_FILL_DIV);
+	if (work_done && ionic_q_space_avail(cq->bound_q) >= rx_fill_threshold)
 		ionic_rx_fill(cq->bound_q);
 
 	if (work_done < budget && napi_complete_done(napi, work_done)) {
@@ -518,6 +516,7 @@  int ionic_txrx_napi(struct napi_struct *napi, int budget)
 	struct ionic_dev *idev;
 	struct ionic_lif *lif;
 	struct ionic_cq *txcq;
+	u16 rx_fill_threshold;
 	u32 rx_work_done = 0;
 	u32 tx_work_done = 0;
 	u32 flags = 0;
@@ -531,8 +530,11 @@  int ionic_txrx_napi(struct napi_struct *napi, int budget)
 
 	rx_work_done = ionic_cq_service(rxcq, budget,
 					ionic_rx_service, NULL, NULL);
-	if (rx_work_done)
-		ionic_rx_fill_cb(rxcq->bound_q);
+
+	rx_fill_threshold = min_t(u16, IONIC_RX_FILL_THRESHOLD,
+				  rxcq->num_descs / IONIC_RX_FILL_DIV);
+	if (rx_work_done && ionic_q_space_avail(rxcq->bound_q) >= rx_fill_threshold)
+		ionic_rx_fill(rxcq->bound_q);
 
 	if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) {
 		ionic_dim_update(qcq);