freebsd/freebsd-src
1
0 Fork 0
mirror of https://github.com/freebsd/freebsd-src.git synced 2026-11-07 04:36:52 -10:00
Code Issues Packages Projects Releases Wiki Activity

iflib: unify prefetching

Browse Source 
This change makes all prefetching use the iflib prefetch wrappers, rather
than using prefetch() in some places and __builtin_prefetch() in others.

Note that __builtin_prefetch() does not work in the kernel on amd64 as clang
disables it when -mnosse is specified, so we must use the asm directives there,
and builtin_prefetch() elsewhere.

This change enables prefetching on non-x86 (notably arm64), and it enables
prefetching in netmap that has been there since day one, but which has never worked.

This is a no-op on non-netmap amd64 and i386

Differential Revision: https://reviews.freebsd.org/D60602
Reviewed by: glebius, kbowling
Sponsored by: Netflix
This commit is contained in:
Andrew Gallatin committed 2026-10-12 17:27:26 -04:00
1 parent 2b911d947b
commit cdd95034d6
1 file changed
+25 -33
Download Patch File Download Diff File Expand all files Collapse all files
sys/net/iflib.c
+25 -33
View File
@@ -912,6 +912,25 @@ iflib_num_tx_descs(if_ctx_t ctx)
return (scctx->isc_ntxd[first_txq]);
}
static inline void
prefetch(void *x)
{
#if defined(__i386__) || defined(__amd64__)
__asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x));
#else
__builtin_prefetch(x);
#endif
}
static inline void
prefetch2cachelines(void *x)
{
prefetch(x);
#if (CACHE_LINE_SIZE < 128)
prefetch((char *)x + CACHE_LINE_SIZE);
#endif
}
#ifdef DEV_NETMAP
#include <sys/selinfo.h>
#include <net/netmap.h>
@@ -1194,9 +1213,9 @@ iflib_netmap_txsync(struct netmap_kring *kring, int flags)
pi.ipi_qsidx = kring->ring_id;
nic_i = netmap_idx_k2n(kring, nm_i);
__builtin_prefetch(&ring->slot[nm_i]);
__builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i]);
__builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i]);
prefetch(&ring->slot[nm_i]);
prefetch(&txq->ift_sds.ifsd_m[nic_i]);
prefetch(&txq->ift_sds.ifsd_map[nic_i]);
while (nm_i != head) {
struct netmap_slot *slot = &ring->slot[nm_i];
@@ -1244,9 +1263,9 @@ iflib_netmap_txsync(struct netmap_kring *kring, int flags)
}
/* prefetch for next round */
__builtin_prefetch(&ring->slot[nm_i + 1]);
__builtin_prefetch(&txq->ift_sds.ifsd_m[nic_i + 1]);
__builtin_prefetch(&txq->ift_sds.ifsd_map[nic_i + 1]);
prefetch(&ring->slot[nm_i + 1]);
prefetch(&txq->ift_sds.ifsd_m[nic_i + 1]);
prefetch(&txq->ift_sds.ifsd_map[nic_i + 1]);
NM_CHECK_ADDR_LEN_OFF(na, len, offset);
@@ -1548,33 +1567,6 @@ iflib_netmap_timer(void *arg)
#define netmap_rx_irq(ifp, qid, budget) (0)
#endif
#if defined(__i386__) || defined(__amd64__)
static __inline void
prefetch(void *x)
{
__asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x));
}
static __inline void
prefetch2cachelines(void *x)
{
__asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x));
#if (CACHE_LINE_SIZE < 128)
__asm volatile("prefetcht0 %0" :: "m" (*(((unsigned long *)x) + CACHE_LINE_SIZE / (sizeof(unsigned long)))));
#endif
}
#else
static __inline void
prefetch(void *x)
{
}
static __inline void
prefetch2cachelines(void *x)
{
}
#endif
static void
iru_init(if_rxd_update_t iru, iflib_rxq_t rxq, uint8_t flid)
{