diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index ac6f18afdf7607d7144ac445b5f79985df7002ce..96b00c399538b262d33e7432727d21927c9ad650 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -6493,6 +6493,13 @@ int huge_pmd_unshare(struct mm_struct *mm, struct vm_area_struct *vma,
 		return 0;
 
 	pud_clear(pud);
+	/*
+	 * Once our caller drops the rmap lock, some other process might be
+	 * using this page table as a normal, non-hugetlb page table.
+	 * Wait for pending gup_fast() in other threads to finish before letting
+	 * that happen.
+	 */
+	tlb_remove_table_sync_one();
 	page_pmd_pts_dec(virt_to_page(ptep));
 	mm_dec_nr_pmds(mm);
 	*addr = ALIGN(*addr, HPAGE_SIZE * PTRS_PER_PTE) - HPAGE_SIZE;