powerpc/radix: acquire the pmap lock in mmu_radix_extract()

mmu_radix_extract() walks the page tables without holding the pmap lock,
unlike its hash MMU counterpart moea64_extract(). A concurrent unmap can
free and recycle the page table page being walked, so the read returns
whatever now occupies that memory and the caller gets a physical address
that never existed.

That is how mmu_radix_sync_icache() came to hand a bogus address to
__syncicache() and panic the machine. Commit 1574ca1955 worked around
it by taking the pmap lock in mmu_radix_sync_icache(), but the machine
independent callers of pmap_extract() - vm_sync_icache(), proc_rwmem()
and the vslock() paths - remain exposed to the same failure.

Rename the existing body to mmu_radix_extract_locked(), which asserts the
lock, and make mmu_radix_extract() a thin wrapper that acquires it.
mmu_radix_sync_icache() already holds the pmap lock, so it calls the
locked variant directly and neither recurses nor reacquires the lock once
per page.

Suggested by:	alc
MFC after:	1 week
Differential Revision:	https://reviews.freebsd.org/D59320
Reviewed by:	markj, jhibbits

sys/powerpc/aim/mmu_radix.c

@@ -3492,19 +3492,19 @@ mmu_radix_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m,
 	PMAP_UNLOCK(pmap);
 }
 
-vm_paddr_t
-mmu_radix_extract(pmap_t pmap, vm_offset_t va)
+static vm_paddr_t
+mmu_radix_extract_locked(pmap_t pmap, vm_offset_t va)
 {
 	pml3_entry_t *l3e;
 	pt_entry_t *pte;
 	vm_paddr_t pa;
 
+	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 	l3e = pmap_pml3e(pmap, va);
 	if (__predict_false(l3e == NULL))
 		return (0);
 	if (be64toh(*l3e) & RPTE_LEAF) {
 		pa = (be64toh(*l3e) & PG_PS_FRAME) | (va & L3_PAGE_MASK);
-		pa |= (va & L3_PAGE_MASK);
 	} else {
 		/*
 		 * Beware of a concurrent promotion that changes the
@@ -3519,11 +3519,21 @@ mmu_radix_extract(pmap_t pmap, vm_offset_t va)
 			return (0);
 		pa = be64toh(*pte);
 		pa = (pa & PG_FRAME) | (va & PAGE_MASK);
-		pa |= (va & PAGE_MASK);
 	}
 	return (pa);
 }
 
+vm_paddr_t
+mmu_radix_extract(pmap_t pmap, vm_offset_t va)
+{
+	vm_paddr_t pa;
+
+	PMAP_LOCK(pmap);
+	pa = mmu_radix_extract_locked(pmap, va);
+	PMAP_UNLOCK(pmap);
+	return (pa);
+}
+
 vm_page_t
 mmu_radix_extract_and_hold(pmap_t pmap, vm_offset_t va, vm_prot_t prot)
 {
@@ -5980,7 +5990,7 @@ mmu_radix_sync_icache(pmap_t pm, vm_offset_t va, vm_size_t sz)
 
 	PMAP_LOCK(pm);
 	while (sz > 0) {
-		pa = pmap_extract(pm, va);
+		pa = mmu_radix_extract_locked(pm, va);
 		sync_sz = PAGE_SIZE - (va & PAGE_MASK);
 		sync_sz = min(sync_sz, sz);
 		if (pa != 0) {