tlb: mmu_gather: add tlb_flush_*_range APIs

commit 2631ed00b0498810f8d5c2163c6b5270d893687b upstream.

tlb_flush_{pte|pmd|pud|p4d}_range() adjust the tlb->start and
tlb->end, then set corresponding cleared_*.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Zhenyu Ye <yezhenyu2@huawei.com>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20200625080314.230-5-yezhenyu2@huawei.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/include/asm-generic/tlb.h b/include/asm-generic/tlb.h
index c716ea81e65317a2dfa5e56b6bc3a17f9f282509..46294ef620ff988256b8f8aeb359ac4ff4ca9ebd 100644 (file)
--- a/include/asm-generic/tlb.h
+++ b/include/asm-generic/tlb.h
@@ -495,6 +495,38 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
 }
 #endif
 
+/*
+ * tlb_flush_{pte|pmd|pud|p4d}_range() adjust the tlb->start and tlb->end,
+ * and set corresponding cleared_*.
+ */
+static inline void tlb_flush_pte_range(struct mmu_gather *tlb,
+                                    unsigned long address, unsigned long size)
+{
+       __tlb_adjust_range(tlb, address, size);
+       tlb->cleared_ptes = 1;
+}
+
+static inline void tlb_flush_pmd_range(struct mmu_gather *tlb,
+                                    unsigned long address, unsigned long size)
+{
+       __tlb_adjust_range(tlb, address, size);
+       tlb->cleared_pmds = 1;
+}
+
+static inline void tlb_flush_pud_range(struct mmu_gather *tlb,
+                                    unsigned long address, unsigned long size)
+{
+       __tlb_adjust_range(tlb, address, size);
+       tlb->cleared_puds = 1;
+}
+
+static inline void tlb_flush_p4d_range(struct mmu_gather *tlb,
+                                    unsigned long address, unsigned long size)
+{
+       __tlb_adjust_range(tlb, address, size);
+       tlb->cleared_p4ds = 1;
+}
+
 #ifndef __tlb_remove_tlb_entry
 #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
 #endif
@@ -508,19 +540,17 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
  */
 #define tlb_remove_tlb_entry(tlb, ptep, address)               \
        do {                                                    \
-               __tlb_adjust_range(tlb, address, PAGE_SIZE);    \
-               tlb->cleared_ptes = 1;                          \
+               tlb_flush_pte_range(tlb, address, PAGE_SIZE);   \
                __tlb_remove_tlb_entry(tlb, ptep, address);     \
        } while (0)
 
 #define tlb_remove_huge_tlb_entry(h, tlb, ptep, address)       \
        do {                                                    \
                unsigned long _sz = huge_page_size(h);          \
-               __tlb_adjust_range(tlb, address, _sz);          \
                if (_sz == PMD_SIZE)                            \
-                       tlb->cleared_pmds = 1;                  \
+                       tlb_flush_pmd_range(tlb, address, _sz); \
                else if (_sz == PUD_SIZE)                       \
-                       tlb->cleared_puds = 1;                  \
+                       tlb_flush_pud_range(tlb, address, _sz); \
                __tlb_remove_tlb_entry(tlb, ptep, address);     \
        } while (0)
 
@@ -534,8 +564,7 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
 
 #define tlb_remove_pmd_tlb_entry(tlb, pmdp, address)                   \
        do {                                                            \
-               __tlb_adjust_range(tlb, address, HPAGE_PMD_SIZE);       \
-               tlb->cleared_pmds = 1;                                  \
+               tlb_flush_pmd_range(tlb, address, HPAGE_PMD_SIZE);      \
                __tlb_remove_pmd_tlb_entry(tlb, pmdp, address);         \
        } while (0)
 
@@ -549,8 +578,7 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
 
 #define tlb_remove_pud_tlb_entry(tlb, pudp, address)                   \
        do {                                                            \
-               __tlb_adjust_range(tlb, address, HPAGE_PUD_SIZE);       \
-               tlb->cleared_puds = 1;                                  \
+               tlb_flush_pud_range(tlb, address, HPAGE_PUD_SIZE);      \
                __tlb_remove_pud_tlb_entry(tlb, pudp, address);         \
        } while (0)
 
@@ -575,9 +603,8 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
 #ifndef pte_free_tlb
 #define pte_free_tlb(tlb, ptep, address)                       \
        do {                                                    \
-               __tlb_adjust_range(tlb, address, PAGE_SIZE);    \
+               tlb_flush_pmd_range(tlb, address, PAGE_SIZE);   \
                tlb->freed_tables = 1;                          \
-               tlb->cleared_pmds = 1;                          \
                __pte_free_tlb(tlb, ptep, address);             \
        } while (0)
 #endif
@@ -585,9 +612,8 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
 #ifndef pmd_free_tlb
 #define pmd_free_tlb(tlb, pmdp, address)                       \
        do {                                                    \
-               __tlb_adjust_range(tlb, address, PAGE_SIZE);    \
+               tlb_flush_pud_range(tlb, address, PAGE_SIZE);   \
                tlb->freed_tables = 1;                          \
-               tlb->cleared_puds = 1;                          \
                __pmd_free_tlb(tlb, pmdp, address);             \
        } while (0)
 #endif
@@ -596,9 +622,8 @@ static inline void tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vm
 #ifndef pud_free_tlb
 #define pud_free_tlb(tlb, pudp, address)                       \
        do {                                                    \
-               __tlb_adjust_range(tlb, address, PAGE_SIZE);    \
+               tlb_flush_p4d_range(tlb, address, PAGE_SIZE);   \
                tlb->freed_tables = 1;                          \
-               tlb->cleared_p4ds = 1;                          \
                __pud_free_tlb(tlb, pudp, address);             \
        } while (0)
 #endif