libthr: Cache recent pshared_key

For a pshare key, we can actually cache the last one as a fast path
instead of trapping into kernel directly. It makes sense as a lock and
unlock should be a pair on the same key and at the same time. For
example, lock(A) -> unlock(A) -> lock(B) -> unlock(B). Each unlock is
a hit and refresh later (50% hit).

For a single pshare lock, it is evne a 100% hit as the key never
changed.

For nested lock lock(A) -> lock(B) -> lock(C) -> unlock(C) ->
unlock(B) -> unlock(A). It only cache the last one. However, we should
really think about rewritting this logic as it is a pshare lock shared
by many processes and nested should be uncommon.

Reviewed by:    kib
MFC after:      2 weeks
Differential Revision: https://reviews.freebsd.org/D60957

lib/libthr/thread/thr_private.h

@@ -522,6 +522,10 @@ struct pthread {
 	/* Number rwlocks rdlocks held. */
 	int			rdlock_count;
 
+	void			*pshared_key;
+	void			*pshared_val;
+	unsigned long		pshared_gen;
+
 	/*
 	 * Current locks bitmap for rtld. */
 	int			rtld_bits;

lib/libthr/thread/thr_pshared.c

@@ -47,9 +47,14 @@ LIST_HEAD(pshared_hash_head, psh);
 static struct pshared_hash_head pshared_hash[HASH_SIZE];
 #define	PSHARED_KEY_HASH(key)	(((unsigned long)(key) >> 8) % HASH_SIZE)
 /* XXXKIB: lock could be split to per-hash chain, if appears contested */
-static struct urwlock pshared_lock = DEFAULT_URWLOCK;
+static struct urwlock pshared_lock __aligned(CACHE_LINE_SIZE) = DEFAULT_URWLOCK;
 static int page_size;
 
+static struct {
+	unsigned long val;
+	char pad[CACHE_LINE_SIZE - sizeof(unsigned long)];
+} pshared_gen __aligned(CACHE_LINE_SIZE);
+
 #define GC_INTERVAL_USEC 25000
 static struct timeval pshared_gc_time = {0, 0};
 bool __thr_pshared_destroy_imm_gc = false;
@@ -93,6 +98,30 @@ pshared_unlock(struct pthread *curthread)
 	_thr_ast(curthread);
 }
 
+static void
+pshared_gen_bump(void)
+{
+	atomic_add_long(&pshared_gen.val, 1);
+}
+
+static void *
+pshared_cache_lookup(struct pthread *curthread, void *key)
+{
+	if (curthread->pshared_key == key &&
+	    curthread->pshared_gen == atomic_load_long(&pshared_gen.val))
+		return (curthread->pshared_val);
+	return (NULL);
+}
+
+static void
+pshared_cache_insert(struct pthread *curthread, void *key, void *val,
+    unsigned long gen)
+{
+	curthread->pshared_key = key;
+	curthread->pshared_val = val;
+	curthread->pshared_gen = gen;
+}
+
 /*
  * Among all processes sharing a lock only one executes
  * pthread_lock_destroy().  Other processes still have the hash and
@@ -128,6 +157,7 @@ pshared_gc(struct pthread *curthread)
 			if (error == 0)
 				continue;
 			LIST_REMOVE(h, link);
+			pshared_gen_bump();
 			munmap(h->val, page_size);
 			free(h);
 		}
@@ -212,6 +242,7 @@ pshared_remove(void *key)
 	LIST_FOREACH(h, hd, link) {
 		if (h->key == key) {
 			LIST_REMOVE(h, link);
+			pshared_gen_bump();
 			val = h->val;
 			free(h);
 			return (val);
@@ -245,6 +276,7 @@ __thr_pshared_offpage(void *key, int doalloc)
 {
 	struct pthread *curthread;
 	void *res;
+	unsigned long gen;
 	int fd, ins_done;
 
 	curthread = _get_curthread();
@@ -252,11 +284,17 @@ __thr_pshared_offpage(void *key, int doalloc)
 		pshared_destroy(curthread, key);
 		res = NULL;
 	} else {
-		pshared_rlock(curthread);
-		res = pshared_lookup(key);
-		pshared_unlock(curthread);
+		res = pshared_cache_lookup(curthread, key);
 		if (res != NULL)
 			return (res);
+		pshared_rlock(curthread);
+		gen = pshared_gen.val;
+		res = pshared_lookup(key);
+		pshared_unlock(curthread);
+		if (res != NULL) {
+			pshared_cache_insert(curthread, key, res, gen);
+			return (res);
+		}
 	}
 	fd = _umtx_op(NULL, UMTX_OP_SHM, doalloc ? UMTX_SHM_CREAT :
 	    UMTX_SHM_LOOKUP, key, NULL);
@@ -267,11 +305,14 @@ __thr_pshared_offpage(void *key, int doalloc)
 	if (res == MAP_FAILED)
 		return (NULL);
 	pshared_wlock(curthread);
+	gen = pshared_gen.val;
 	ins_done = pshared_insert(key, &res);
 	pshared_unlock(curthread);
 	if (!ins_done) {
 		pshared_clean(key, res);
 		res = NULL;
+	} else {
+		pshared_cache_insert(curthread, key, res, gen);
 	}
 	return (res);
 }