tcp: release nic ktls send tags when entering time wait

When under heavy load or churn, inline ktls offload NICs may run out
of hardware resources described by ktls send tags.  Rather than
waiting for connections to pass through the time_wait state, reclaim
the ktls send tags early, at entry to time_wait. By preventing
potentially tens or hundreds of thousands of sessions from holding
send tags in time_wait, this allows more ktls sessions to be offloaded
to hardware.

Reviewed by: glebius, kib, nickbanks_netflix.com, rrs, tuexen
Sponsored by: Netflix
Differential Revision: https://reviews.freebsd.org/D56610

sys/netinet/tcp_timewait.c

@@ -32,11 +32,15 @@
 #include "opt_inet.h"
 #include "opt_inet6.h"
 #include "opt_ipsec.h"
+#include "opt_kern_tls.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/callout.h>
 #include <sys/kernel.h>
+#ifdef KERN_TLS
+#include <sys/ktls.h>
+#endif
 #include <sys/sysctl.h>
 #include <sys/malloc.h>
 #include <sys/mbuf.h>
@@ -132,6 +136,12 @@ tcp_twstart(struct tcpcb *tp)
 	tcp_free_sackholes(tp);
 	soisdisconnected(inp->inp_socket);
 
+#ifdef KERN_TLS
+	/* release ktls snd tag now that no more data can be sent */
+	if (tptosocket(tp)->so_snd.sb_tls_info != NULL) {
+		ktls_release_snd_tag(tptosocket(tp)->so_snd.sb_tls_info);
+	}
+#endif
 	if (tp->t_flags & TF_ACKNOW)
 		(void) tcp_output(tp);
 

sys/sys/ktls.h

@@ -28,6 +28,7 @@
 #define	_SYS_KTLS_H_
 
 #ifdef _KERNEL
+#include <sys/mbuf.h>
 #include <sys/_null.h>
 #include <sys/refcount.h>
 #include <sys/_task.h>
@@ -285,6 +286,17 @@ ktls_free(struct ktls_session *tls)
 		ktls_destroy(tls);
 }
 
+static inline void
+ktls_release_snd_tag(struct ktls_session *tls)
+{
+	struct m_snd_tag *mst;
+
+	mst = tls->snd_tag;
+	tls->snd_tag = NULL;
+	if (mst != NULL)
+		m_snd_tag_rele(mst);
+}
+
 void ktls_session_to_xktls_onedir(const struct ktls_session *ks,
     bool export_keys, struct xktls_session_onedir *xktls_od);
 void ktls_session_copy_keys(const struct ktls_session *ktls,