@@ -65,8 +65,7 @@ efx_tx_desc(struct efx_tx_queue *tx_queue, unsigned int index)
/* Report whether this TX queue would be empty for the given write_count.
* May return false negative.
*/
-static inline bool __efx_nic_tx_is_empty(struct efx_tx_queue *tx_queue,
- unsigned int write_count)
+static inline bool efx_nic_tx_is_empty(struct efx_tx_queue *tx_queue, unsigned int write_count)
{
unsigned int empty_read_count = READ_ONCE(tx_queue->empty_read_count);
@@ -76,17 +75,6 @@ static inline bool __efx_nic_tx_is_empty(struct efx_tx_queue *tx_queue,
return ((empty_read_count ^ write_count) & ~EFX_EMPTY_COUNT_VALID) == 0;
}
-/* Report whether the NIC considers this TX queue empty, using
- * packet_write_count (the write count recorded for the last completable
- * doorbell push). May return false negative. EF10 only, which is OK
- * because only EF10 supports PIO.
- */
-static inline bool efx_nic_tx_is_empty(struct efx_tx_queue *tx_queue)
-{
- EFX_WARN_ON_ONCE_PARANOID(!tx_queue->efx->type->option_descriptors);
- return __efx_nic_tx_is_empty(tx_queue, tx_queue->packet_write_count);
-}
-
/* Get partner of a TX queue, seen as part of the same net core queue */
/* XXX is this a thing on EF100? */
static inline struct efx_tx_queue *efx_tx_queue_partner(struct efx_tx_queue *tx_queue)
@@ -97,20 +85,6 @@ static inline struct efx_tx_queue *efx_tx_queue_partner(struct efx_tx_queue *tx_
return tx_queue + EFX_TXQ_TYPE_OFFLOAD;
}
-/* Decide whether we can use TX PIO, ie. write packet data directly into
- * a buffer on the device. This can reduce latency at the expense of
- * throughput, so we only do this if both hardware and software TX rings
- * are empty. This also ensures that only one packet at a time can be
- * using the PIO buffer.
- */
-static inline bool efx_nic_may_tx_pio(struct efx_tx_queue *tx_queue)
-{
- struct efx_tx_queue *partner = efx_tx_queue_partner(tx_queue);
-
- return tx_queue->piobuf && efx_nic_tx_is_empty(tx_queue) &&
- efx_nic_tx_is_empty(partner);
-}
-
int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue, struct sk_buff *skb,
bool *data_mapped);
@@ -125,7 +99,7 @@ int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue, struct sk_buff *skb,
static inline bool efx_nic_may_push_tx_desc(struct efx_tx_queue *tx_queue,
unsigned int write_count)
{
- bool was_empty = __efx_nic_tx_is_empty(tx_queue, write_count);
+ bool was_empty = efx_nic_tx_is_empty(tx_queue, write_count);
tx_queue->empty_read_count = 0;
return was_empty && tx_queue->write_count - write_count == 1;
@@ -264,6 +264,30 @@ static int efx_enqueue_skb_pio(struct efx_tx_queue *tx_queue,
++tx_queue->insert_count;
return 0;
}
+
+/* Decide whether we can use TX PIO, ie. write packet data directly into
+ * a buffer on the device. This can reduce latency at the expense of
+ * throughput, so we only do this if both hardware and software TX rings
+ * are empty, including all queues for the channel. This also ensures that
+ * only one packet at a time can be using the PIO buffer. If the xmit_more
+ * flag is set then we don't use this - there'll be another packet along
+ * shortly and we want to hold off the doorbell.
+ */
+static bool efx_tx_may_pio(struct efx_tx_queue *tx_queue)
+{
+ struct efx_channel *channel = tx_queue->channel;
+
+ if (!tx_queue->piobuf)
+ return false;
+
+ EFX_WARN_ON_ONCE_PARANOID(!channel->efx->type->option_descriptors);
+
+ efx_for_each_channel_tx_queue(tx_queue, channel)
+ if (!efx_nic_tx_is_empty(tx_queue, tx_queue->packet_write_count))
+ return false;
+
+ return true;
+}
#endif /* EFX_USE_PIO */
/* Send any pending traffic for a channel. xmit_more is shared across all
@@ -326,7 +350,7 @@ netdev_tx_t __efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb
goto err;
#ifdef EFX_USE_PIO
} else if (skb_len <= efx_piobuf_size && !xmit_more &&
- efx_nic_may_tx_pio(tx_queue)) {
+ efx_tx_may_pio(tx_queue)) {
/* Use PIO for short packets with an empty queue. */
if (efx_enqueue_skb_pio(tx_queue, skb))
goto err;
Use efx_for_each_channel_tx_queue() rather than efx_tx_queue_partner(). Make some related simplifications of efx_nic_tx_is_empty() to remove entry points that aren't used. Signed-off-by: Edward Cree <ecree@solarflare.com> --- drivers/net/ethernet/sfc/nic_common.h | 30 ++------------------------- drivers/net/ethernet/sfc/tx.c | 26 ++++++++++++++++++++++- 2 files changed, 27 insertions(+), 29 deletions(-)