mirror of
https://github.com/FEX-Emu/linux.git
synced 2024-12-27 03:47:43 +00:00
ba8a9229ab
The current tlb flushing code for page table entries violates the s390 architecture in a small detail. The relevant section from the principles of operation (SA22-7832-02 page 3-47): "A valid table entry must not be changed while it is attached to any CPU and may be used for translation by that CPU except to (1) invalidate the entry by using INVALIDATE PAGE TABLE ENTRY or INVALIDATE DAT TABLE ENTRY, (2) alter bits 56-63 of a page-table entry, or (3) make a change by means of a COMPARE AND SWAP AND PURGE instruction that purges the TLB." That means if one thread of a multithreaded applciation uses a vma while another thread does an unmap on it, the page table entries of that vma needs to get removed with IPTE, IDTE or CSP. In some strange and rare situations a cpu could check-stop (die) because a entry has been pushed out of the TLB that is still needed to complete a (milli-coded) instruction. I've never seen it happen with the current code on any of the supported machines, so right now this is a theoretical problem. But I want to fix it nevertheless, to avoid headaches in the futures. To get this implemented correctly without changing common code the primitives ptep_get_and_clear, ptep_get_and_clear_full and ptep_set_wrprotect need to use the IPTE instruction to invalidate the pte before the new pte value gets stored. If IPTE is always used for the three primitives three important operations will have a performace hit: fork, mprotect and exit_mmap. Time for some workarounds: * 1: ptep_get_and_clear_full is used in unmap_vmas to remove page tables entries in a batched tlb gather operation. If the mmu_gather context passed to unmap_vmas has been started with full_mm_flush==1 or if only one cpu is online or if the only user of a mm_struct is the current process then the fullmm indication in the mmu_gather context is set to one. All TLBs for mm_struct are flushed by the tlb_gather_mmu call. No new TLBs can be created while the unmap is in progress. In this case ptep_get_and_clear_full clears the ptes with a simple store. * 2: ptep_get_and_clear is used in change_protection to clear the ptes from the page tables before they are reentered with the new access flags. At the end of the update flush_tlb_range clears the remaining TLBs. In general the ptep_get_and_clear has to issue IPTE for each pte and flush_tlb_range is a nop. But if there is only one user of the mm_struct then ptep_get_and_clear uses simple stores to do the update and flush_tlb_range will flush the TLBs. * 3: Similar to 2, ptep_set_wrprotect is used in copy_page_range for a fork to make all ptes of a cow mapping read-only. At the end of of copy_page_range dup_mmap will flush the TLBs with a call to flush_tlb_mm. Check for mm->mm_users and if there is only one user avoid using IPTE in ptep_set_wrprotect and let flush_tlb_mm clear the TLBs. Overall for single threaded programs the tlb flush code now performs better, for multi threaded programs it is slightly worse. In particular exit_mmap() now does a single IDTE for the mm and then just frees every page cache reference and every page table page directly without a delay over the mmu_gather structure. Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
115 lines
2.9 KiB
C
115 lines
2.9 KiB
C
#ifndef _S390_TLBFLUSH_H
|
|
#define _S390_TLBFLUSH_H
|
|
|
|
#include <linux/mm.h>
|
|
#include <asm/processor.h>
|
|
#include <asm/pgalloc.h>
|
|
|
|
/*
|
|
* Flush all tlb entries on the local cpu.
|
|
*/
|
|
static inline void __tlb_flush_local(void)
|
|
{
|
|
asm volatile("ptlb" : : : "memory");
|
|
}
|
|
|
|
/*
|
|
* Flush all tlb entries on all cpus.
|
|
*/
|
|
static inline void __tlb_flush_global(void)
|
|
{
|
|
extern void smp_ptlb_all(void);
|
|
register unsigned long reg2 asm("2");
|
|
register unsigned long reg3 asm("3");
|
|
register unsigned long reg4 asm("4");
|
|
long dummy;
|
|
|
|
#ifndef __s390x__
|
|
if (!MACHINE_HAS_CSP) {
|
|
smp_ptlb_all();
|
|
return;
|
|
}
|
|
#endif /* __s390x__ */
|
|
|
|
dummy = 0;
|
|
reg2 = reg3 = 0;
|
|
reg4 = ((unsigned long) &dummy) + 1;
|
|
asm volatile(
|
|
" csp %0,%2"
|
|
: : "d" (reg2), "d" (reg3), "d" (reg4), "m" (dummy) : "cc" );
|
|
}
|
|
|
|
/*
|
|
* Flush all tlb entries of a page table on all cpus.
|
|
*/
|
|
static inline void __tlb_flush_idte(pgd_t *pgd)
|
|
{
|
|
asm volatile(
|
|
" .insn rrf,0xb98e0000,0,%0,%1,0"
|
|
: : "a" (2048), "a" (__pa(pgd) & PAGE_MASK) : "cc" );
|
|
}
|
|
|
|
static inline void __tlb_flush_mm(struct mm_struct * mm)
|
|
{
|
|
cpumask_t local_cpumask;
|
|
|
|
if (unlikely(cpus_empty(mm->cpu_vm_mask)))
|
|
return;
|
|
/*
|
|
* If the machine has IDTE we prefer to do a per mm flush
|
|
* on all cpus instead of doing a local flush if the mm
|
|
* only ran on the local cpu.
|
|
*/
|
|
if (MACHINE_HAS_IDTE) {
|
|
pgd_t *shadow_pgd = get_shadow_pgd(mm->pgd);
|
|
|
|
if (shadow_pgd)
|
|
__tlb_flush_idte(shadow_pgd);
|
|
__tlb_flush_idte(mm->pgd);
|
|
return;
|
|
}
|
|
preempt_disable();
|
|
/*
|
|
* If the process only ran on the local cpu, do a local flush.
|
|
*/
|
|
local_cpumask = cpumask_of_cpu(smp_processor_id());
|
|
if (cpus_equal(mm->cpu_vm_mask, local_cpumask))
|
|
__tlb_flush_local();
|
|
else
|
|
__tlb_flush_global();
|
|
preempt_enable();
|
|
}
|
|
|
|
static inline void __tlb_flush_mm_cond(struct mm_struct * mm)
|
|
{
|
|
if (atomic_read(&mm->mm_users) <= 1 && mm == current->active_mm)
|
|
__tlb_flush_mm(mm);
|
|
}
|
|
|
|
/*
|
|
* TLB flushing:
|
|
* flush_tlb() - flushes the current mm struct TLBs
|
|
* flush_tlb_all() - flushes all processes TLBs
|
|
* flush_tlb_mm(mm) - flushes the specified mm context TLB's
|
|
* flush_tlb_page(vma, vmaddr) - flushes one page
|
|
* flush_tlb_range(vma, start, end) - flushes a range of pages
|
|
* flush_tlb_kernel_range(start, end) - flushes a range of kernel pages
|
|
*/
|
|
|
|
/*
|
|
* flush_tlb_mm goes together with ptep_set_wrprotect for the
|
|
* copy_page_range operation and flush_tlb_range is related to
|
|
* ptep_get_and_clear for change_protection. ptep_set_wrprotect and
|
|
* ptep_get_and_clear do not flush the TLBs directly if the mm has
|
|
* only one user. At the end of the update the flush_tlb_mm and
|
|
* flush_tlb_range functions need to do the flush.
|
|
*/
|
|
#define flush_tlb() do { } while (0)
|
|
#define flush_tlb_all() do { } while (0)
|
|
#define flush_tlb_mm(mm) __tlb_flush_mm_cond(mm)
|
|
#define flush_tlb_page(vma, addr) do { } while (0)
|
|
#define flush_tlb_range(vma, start, end) __tlb_flush_mm_cond(mm)
|
|
#define flush_tlb_kernel_range(start, end) __tlb_flush_mm(&init_mm)
|
|
|
|
#endif /* _S390_TLBFLUSH_H */
|