rss: Enable portions of RSS globally to enable symmetric hashing

We use the fact that all NICs that support hashing are using the
same hash algorithm and hash key to enable symmetic hashing in
TCP, where a software version of the same hash is used to
establish hashes on outgoing connections.

Sponsored by: Netflix
Reviewed by: adrian, zlei (both early version)
Differential Revision:	https://reviews.freebsd.org/D53089

sys/conf/files

@@ -4238,10 +4238,10 @@ net/route/route_rtentry.c	standard
 net/route/route_subscription.c	standard
 net/route/route_tables.c	standard
 net/route/route_temporal.c	standard
-net/rss_config.c		optional inet rss | inet6 rss
+net/rss_config.c		standard
 net/rtsock.c			standard
 net/slcompress.c		optional netgraph_vjc
-net/toeplitz.c			optional inet rss | inet6 rss | route_mpath
+net/toeplitz.c			optional inet | inet6 | route_mpath
 net/vnet.c			optional vimage
 net80211/ieee80211.c		optional wlan
 net80211/ieee80211_acl.c	optional wlan wlan_acl
@@ -4384,7 +4384,7 @@ netinet/in_pcb.c		optional inet | inet6
 netinet/in_prot.c		optional inet | inet6
 netinet/in_proto.c		optional inet | inet6
 netinet/in_rmx.c		optional inet
-netinet/in_rss.c		optional inet rss
+netinet/in_rss.c		optional inet
 netinet/ip_divert.c		optional ipdivert inet | ipdivert inet6
 netinet/ip_ecn.c		optional inet | inet6
 netinet/ip_encap.c		optional inet | inet6
@@ -4486,7 +4486,7 @@ netinet6/in6_mcast.c		optional inet6
 netinet6/in6_pcb.c		optional inet6
 netinet6/in6_proto.c		optional inet6
 netinet6/in6_rmx.c		optional inet6
-netinet6/in6_rss.c		optional inet6 rss
+netinet6/in6_rss.c		optional inet6
 netinet6/in6_src.c		optional inet6
 netinet6/ip6_fastfwd.c		optional inet6
 netinet6/ip6_forward.c		optional inet6

sys/net/rss_config.c

@@ -29,6 +29,8 @@
 
 
 #include "opt_inet6.h"
+#include "opt_inet.h"
+#include "opt_rss.h"
 
 #include <sys/param.h>
 #include <sys/mbuf.h>
@@ -72,6 +74,10 @@
  *   placement and pcbgroup expectations.
  */
 
+#if !defined(INET) && !defined(INET6)
+#define _net_inet _net
+#define _net_inet_rss _net_rss
+#endif
 SYSCTL_DECL(_net_inet);
 SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
     "Receive-side steering");
@@ -84,6 +90,7 @@ static u_int	rss_hashalgo = RSS_HASH_TOEPLITZ;
 SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RDTUN, &rss_hashalgo, 0,
     "RSS hash algorithm");
 
+#ifdef RSS
 /*
  * Size of the indirection table; at most 128 entries per the RSS spec.  We
  * size it to at least 2 times the number of CPUs by default to allow useful
@@ -132,6 +139,7 @@ static const u_int	rss_basecpu;
 SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD,
     __DECONST(int *, &rss_basecpu), 0, "RSS base CPU");
 
+#endif
 /*
  * Print verbose debugging messages.
  * 0 - disable
@@ -159,6 +167,7 @@ static uint8_t rss_key[RSS_KEYSIZE] = {
 	0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa,
 };
 
+#ifdef RSS
 /*
  * RSS hash->CPU table, which maps hashed packet headers to particular CPUs.
  * Drivers may supplement this table with a separate CPU<->queue table when
@@ -168,13 +177,15 @@ struct rss_table_entry {
 	uint8_t		rte_cpu;	/* CPU affinity of bucket. */
 };
 static struct rss_table_entry	rss_table[RSS_TABLE_MAXLEN];
+#endif
 
 static void
 rss_init(__unused void *arg)
 {
+#ifdef RSS
 	u_int i;
 	u_int cpuid;
-
+#endif
 	/*
 	 * Validate tunables, coerce to sensible values.
 	 */
@@ -189,6 +200,7 @@ rss_init(__unused void *arg)
 		rss_hashalgo = RSS_HASH_TOEPLITZ;
 	}
 
+#ifdef RSS
 	/*
 	 * Count available CPUs.
 	 *
@@ -248,7 +260,7 @@ rss_init(__unused void *arg)
 		rss_table[i].rte_cpu = cpuid;
 		cpuid = CPU_NEXT(cpuid);
 	}
-
+#endif /* RSS */
 	/*
 	 * Randomize rrs_key.
 	 *
@@ -292,6 +304,30 @@ rss_hash(u_int datalen, const uint8_t *data)
 	}
 }
 
+/*
+ * Query the current RSS key; likely to be used by device drivers when
+ * configuring hardware RSS.  Caller must pass an array of size RSS_KEYSIZE.
+ *
+ * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing?
+ */
+void
+rss_getkey(uint8_t *key)
+{
+
+	bcopy(rss_key, key, sizeof(rss_key));
+}
+
+/*
+ * Query the RSS hash algorithm.
+ */
+u_int
+rss_gethashalgo(void)
+{
+
+	return (rss_hashalgo);
+}
+
+#ifdef RSS
 /*
  * Query the number of RSS bits in use.
  */
@@ -406,29 +442,6 @@ rss_m2bucket(struct mbuf *m, uint32_t *bucket_id)
 	    bucket_id));
 }
 
-/*
- * Query the RSS hash algorithm.
- */
-u_int
-rss_gethashalgo(void)
-{
-
-	return (rss_hashalgo);
-}
-
-/*
- * Query the current RSS key; likely to be used by device drivers when
- * configuring hardware RSS.  Caller must pass an array of size RSS_KEYSIZE.
- *
- * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing?
- */
-void
-rss_getkey(uint8_t *key)
-{
-
-	bcopy(rss_key, key, sizeof(rss_key));
-}
-
 /*
  * Query the number of buckets; this may be used by both network device
  * drivers, which will need to populate hardware shadows of the software
@@ -454,6 +467,7 @@ rss_getnumcpus(void)
 	return (rss_ncpus);
 }
 
+#endif
 /*
  * Return the supported RSS hash configuration.
  *
@@ -517,6 +531,7 @@ SYSCTL_PROC(_net_inet_rss, OID_AUTO, key,
     CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key,
     "", "RSS keying material");
 
+#ifdef RSS
 static int
 sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS)
 {
@@ -544,3 +559,4 @@ sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS)
 SYSCTL_PROC(_net_inet_rss, OID_AUTO, bucket_mapping,
     CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0,
     sysctl_rss_bucket_mapping, "", "RSS bucket -> CPU mapping");
+#endif

sys/net/rss_config.h

@@ -104,6 +104,7 @@
 
 extern int	rss_debug;
 
+#ifdef RSS
 /*
  * Device driver interfaces to query RSS properties that must be programmed
  * into hardware.
@@ -112,16 +113,8 @@ u_int	rss_getbits(void);
 u_int	rss_getbucket(u_int hash);
 u_int	rss_get_indirection_to_bucket(u_int index);
 u_int	rss_getcpu(u_int bucket);
-void	rss_getkey(uint8_t *key);
-u_int	rss_gethashalgo(void);
 u_int	rss_getnumbuckets(void);
 u_int	rss_getnumcpus(void);
-u_int	rss_gethashconfig(void);
-
-/*
- * Hash calculation functions.
- */
-uint32_t	rss_hash(u_int datalen, const uint8_t *data);
 
 /*
  * Network stack interface to query desired CPU affinity of a packet.
@@ -132,4 +125,15 @@ int	rss_hash2bucket(uint32_t hash_val, uint32_t hash_type,
 	    uint32_t *bucket_id);
 int	rss_m2bucket(struct mbuf *m, uint32_t *bucket_id);
 
+#endif /* RSS */
+
+void	rss_getkey(uint8_t *key);
+u_int	rss_gethashalgo(void);
+u_int	rss_gethashconfig(void);
+/*
+ * Hash calculation functions.
+ */
+uint32_t	rss_hash(u_int datalen, const uint8_t *data);
+
+
 #endif /* !_NET_RSS_CONFIG_H_ */

sys/netinet/in_rss.c

@@ -29,6 +29,7 @@
 
 
 #include "opt_inet6.h"
+#include "opt_rss.h"
 
 #include <sys/param.h>
 #include <sys/mbuf.h>
@@ -350,6 +351,7 @@ rss_mbuf_software_hash_v4(const struct mbuf *m, int dir, uint32_t *hashval,
 	}
 }
 
+#ifdef RSS
 /*
  * Similar to rss_m2cpuid, but designed to be used by the IP NETISR
  * on incoming frames.
@@ -387,3 +389,4 @@ rss_soft_m2cpuid_v4(struct mbuf *m, uintptr_t source, u_int *cpuid)
 	}
 	return (m);
 }
+#endif

sys/netinet6/in6_rss.c

@@ -29,6 +29,7 @@
 
 
 #include "opt_inet6.h"
+#include "opt_rss.h"
 
 #include <sys/param.h>
 #include <sys/mbuf.h>
@@ -375,6 +376,7 @@ rss_mbuf_software_hash_v6(const struct mbuf *m, int dir, uint32_t *hashval,
 	}
 }
 
+#ifdef RSS
 /*
  * Similar to rss_m2cpuid, but designed to be used by the IPv6 NETISR
  * on incoming frames.
@@ -412,3 +414,4 @@ rss_soft_m2cpuid_v6(struct mbuf *m, uintptr_t source, u_int *cpuid)
 	}
 	return (m);
 }
+#endif