arm64: mm: Force Device mappings for PCIe MMIO

PCIe device drivers may map MMIO space as Normal non-cacheable, for the
purpose of enabling write combining or unaligned accesses.

On many platforms (e.g. Ampere Altra, RK35xx), the PCIe interface cannot
support unaligned outbound transactions. This may lead to data
corruption, for instance, when a regular memcpy is performed by an
application on a GPU's VRAM BAR.

Add an option to force all software that maps PCIe MMIO space as Normal
non-cacheable memory to use Device-nGnRE instead. If the strict
alignment is not met, the CPU will raise alignment faults that can be
further handled by the kernel by enabling CONFIG_ARM64_ALIGNMENT_FIXUPS.

Signed-off-by: Mario Bălănică <mariobalanica02@gmail.com>

Author: mariobalanica

arch/arm64/Kconfig

@@ -1696,6 +1696,23 @@ config ARM64_TAGGED_ADDR_ABI
 	  to system calls as pointer arguments. For details, see
 	  Documentation/arch/arm64/tagged-address-abi.rst.
 
+config ARM64_FORCE_PCIE_MMIO_DEVICE_MAPPINGS
+	bool "Force Device memory mappings for PCIe MMIO space"
+	default y
+	help
+	  PCIe device drivers may map MMIO space as Normal non-cacheable,
+	  for the purpose of enabling write combining or unaligned accesses.
+
+	  On many platforms (e.g. Ampere Altra, RK35xx), the PCIe interface
+	  cannot support unaligned outbound transactions. This may lead to
+	  data corruption, for instance, when a regular memcpy is performed by
+	  an application on a GPU's VRAM BAR.
+
+	  This option forces all software that maps PCIe MMIO space as Normal
+	  non-cacheable memory to use Device-nGnRE instead. If the strict alignment
+	  is not met, the CPU will raise alignment faults that can be further
+	  handled by the kernel by enabling CONFIG_ARM64_ALIGNMENT_FIXUPS.
+
 config ARM64_ALIGNMENT_FIXUPS
 	bool "Fix up misaligned multi-word loads and stores in 64-bit kernel/user space"
 	default y

arch/arm64/include/asm/pgtable.h

@@ -340,11 +340,6 @@ static inline pte_t pte_mkyoung(pte_t pte)
 	return set_pte_bit(pte, __pgprot(PTE_AF));
 }
 
-static inline pte_t pte_mkspecial(pte_t pte)
-{
-	return set_pte_bit(pte, __pgprot(PTE_SPECIAL));
-}
-
 static inline pte_t pte_mkcont(pte_t pte)
 {
 	return set_pte_bit(pte, __pgprot(PTE_CONT));
@@ -804,6 +799,21 @@ static inline void __set_puds(struct mm_struct *mm,
 	__pgprot_modify(prot, PTE_ATTRINDX_MASK, \
 			PTE_ATTRINDX(MT_NORMAL_NC) | PTE_PXN | PTE_UXN)
 
+extern bool range_is_pci(phys_addr_t phys_addr, size_t size);
+
+static inline pte_t pte_mkspecial(pte_t pte)
+{
+#ifdef CONFIG_ARM64_FORCE_PCIE_MMIO_DEVICE_MAPPINGS
+	phys_addr_t phys = __pte_to_phys(pte);
+	pgprot_t prot = __pgprot(pte_val(pte) & ~__phys_to_pte_val(__pte_to_phys(__pte(~0ull))));
+
+	if ((pgprot_val(prot) != pgprot_val(pgprot_device(prot))) &&
+	    range_is_pci(phys, PAGE_SIZE))
+		pte = __pte(__phys_to_pte_val(phys) | pgprot_val(pgprot_device(prot)));
+#endif
+	return set_pte_bit(pte, __pgprot(PTE_SPECIAL));
+}
+
 #define __HAVE_PHYS_MEM_ACCESS_PROT
 struct file;
 extern pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn,

kernel/resource.c

@@ -389,6 +389,7 @@ static int find_next_iomem_res(resource_size_t start, resource_size_t end,
 			.flags = p->flags,
 			.desc = p->desc,
 			.parent = p->parent,
+			.name = p->name,
 		};
 	}
 
@@ -566,6 +567,38 @@ int __weak page_is_ram(unsigned long pfn)
 }
 EXPORT_SYMBOL_GPL(page_is_ram);
 
+static int pci_res_check(struct resource *res, void *arg)
+{
+	if (!res->name)
+		return 1;
+
+	return strncmp(res->name, "PCI", 3);
+}
+
+bool range_is_pci(phys_addr_t phys_addr, size_t size)
+{
+	u64 start, end;
+	int ret;
+
+	start = phys_addr;
+	end = phys_addr + size;
+
+	/* Check 32-bit MMIO */
+	ret = walk_iomem_res_desc(IORES_DESC_NONE, IORESOURCE_MEM,
+				  start, end, NULL, pci_res_check);
+	if (!ret)
+		return true;
+
+	/* Check 64-bit MMIO */
+	ret = walk_iomem_res_desc(IORES_DESC_NONE, IORESOURCE_MEM_64,
+				  start, end, NULL, pci_res_check);
+	if (!ret)
+		return true;
+
+	return false;
+}
+EXPORT_SYMBOL_GPL(range_is_pci);
+
 static int __region_intersects(struct resource *parent, resource_size_t start,
 			       size_t size, unsigned long flags,
 			       unsigned long desc)