iommu: io-pgtable-arm: Implement IOMMU_USE_LLC_NWA

With the GPU moving onto a new SMMU, the override was removed due to
the way MMU-500 was integrated. Hence, the method of hinting the SMMU
of the upstream device's bus attributes would no longer work for GPU.
Instead, encode the MAIR and TCR register through the IOMMU_USE_LLC_NWA
attribute flag to map the memory into System Cache with no Write-Allocate.

MAIR Encoding:
Bits[7:4] => 0b1110 = Outer Write-back read-allocate, no write-allocate
Bits[3:0] => 0b0100 = Inner non-cacheable normal memory

TCR Encoding:
SH   => 0b10 = Outer Shareable
ORGN => 0b11 = Write-Back, no Write-Allocate cacheable

Change-Id: I34db1ebfb5f4e080ca01328176bcabc368e9ddab
Signed-off-by: Sudarshan Rajagopalan <sudaraja@codeaurora.org>

Documentation/DMA-attributes.txt

@@ -156,3 +156,10 @@ accesses to DMA buffers in both privileged "supervisor" and unprivileged
 subsystem that the buffer is fully accessible at the elevated privilege
 level (and ideally inaccessible or at least read-only at the
 lesser-privileged levels).
+
+DMA_ATTR_IOMMU_USE_LLC_NWA
+------------------------------------
+
+DMA_ATTR_IOMMU_USE_LLC_NWA: Overrides the bus attributes to use
+System Cache(LLC) with allocation policy as Inner Non-Cacheable, Outer Cacheable:
+Write-Back, Read-Allocate, No Write-Allocate policy.

arch/arm64/mm/dma-mapping.c

@@ -1092,6 +1092,8 @@ static int __get_iommu_pgprot(unsigned long attrs, int prot,
 		prot |= IOMMU_NOEXEC;
 	if (attrs & DMA_ATTR_IOMMU_USE_UPSTREAM_HINT)
 		prot |= IOMMU_USE_UPSTREAM_HINT;
+	if (attrs & DMA_ATTR_IOMMU_USE_LLC_NWA)
+		prot |= IOMMU_USE_LLC_NWA;
 	if (coherent)
 		prot |= IOMMU_CACHE;
 

drivers/iommu/arm-smmu.c

@@ -1679,7 +1679,10 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 		quirks |= IO_PGTABLE_QUIRK_QCOM_USE_UPSTREAM_HINT;
 	if (is_iommu_pt_coherent(smmu_domain))
 		quirks |= IO_PGTABLE_QUIRK_NO_DMA;
-	if ((quirks & IO_PGTABLE_QUIRK_QCOM_USE_UPSTREAM_HINT) &&
+	if (smmu_domain->attributes & (1 << DOMAIN_ATTR_USE_LLC_NWA))
+		quirks |= IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA;
+	if (((quirks & IO_PGTABLE_QUIRK_QCOM_USE_UPSTREAM_HINT) ||
+	     (quirks & IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA)) &&
 		(smmu->model == QCOM_SMMUV500))
 		quirks |= IO_PGTABLE_QUIRK_QSMMUV500_NON_SHAREABLE;
 
@@ -2819,6 +2822,11 @@ static int arm_smmu_domain_get_attr(struct iommu_domain *domain,
 				   (1 << DOMAIN_ATTR_USE_UPSTREAM_HINT));
 		ret = 0;
 		break;
+	case DOMAIN_ATTR_USE_LLC_NWA:
+		*((int *)data) = !!(smmu_domain->attributes &
+				   (1 << DOMAIN_ATTR_USE_LLC_NWA));
+		ret = 0;
+		break;
 	case DOMAIN_ATTR_EARLY_MAP:
 		*((int *)data) = !!(smmu_domain->attributes
 				    & (1 << DOMAIN_ATTR_EARLY_MAP));
@@ -3001,6 +3009,17 @@ static int arm_smmu_domain_set_attr(struct iommu_domain *domain,
 				1 << DOMAIN_ATTR_USE_UPSTREAM_HINT;
 		ret = 0;
 		break;
+	case DOMAIN_ATTR_USE_LLC_NWA:
+		/* can't be changed while attached */
+		if (smmu_domain->smmu != NULL) {
+			ret = -EBUSY;
+			break;
+		}
+		if (*((int *)data))
+			smmu_domain->attributes |=
+				1 << DOMAIN_ATTR_USE_LLC_NWA;
+		ret = 0;
+		break;
 	case DOMAIN_ATTR_EARLY_MAP: {
 		int early_map = *((int *)data);
 

drivers/iommu/dma-iommu.c

@@ -381,6 +381,9 @@ int dma_info_to_prot(enum dma_data_direction dir, bool coherent,
 	if (attrs & DMA_ATTR_IOMMU_USE_UPSTREAM_HINT)
 		prot |= IOMMU_USE_UPSTREAM_HINT;
 
+	if (attrs & DMA_ATTR_IOMMU_USE_LLC_NWA)
+		prot |= IOMMU_USE_LLC_NWA;
+
 	switch (dir) {
 	case DMA_BIDIRECTIONAL:
 		return prot | IOMMU_READ | IOMMU_WRITE;

drivers/iommu/io-pgtable-arm.c

@@ -169,15 +169,18 @@
 #define ARM_LPAE_TCR_PS_48_BIT		0x5ULL
 
 #define ARM_LPAE_MAIR_ATTR_SHIFT(n)	((n) << 3)
+#define ARM_LPAE_MAIR1_ATTR_SHIFT(n)	((n-4) << 3)
 #define ARM_LPAE_MAIR_ATTR_MASK		0xff
 #define ARM_LPAE_MAIR_ATTR_DEVICE	0x04
 #define ARM_LPAE_MAIR_ATTR_NC		0x44
 #define ARM_LPAE_MAIR_ATTR_WBRWA	0xff
 #define ARM_LPAE_MAIR_ATTR_UPSTREAM	0xf4
+#define ARM_LPAE_MAIR_ATTR_LLC_NWA	0xe4
 #define ARM_LPAE_MAIR_ATTR_IDX_NC	0
 #define ARM_LPAE_MAIR_ATTR_IDX_CACHE	1
 #define ARM_LPAE_MAIR_ATTR_IDX_DEV	2
 #define ARM_LPAE_MAIR_ATTR_IDX_UPSTREAM	3
+#define ARM_LPAE_MAIR_ATTR_IDX_LLC_NWA	0x4ULL
 
 /* IOPTE accessors */
 #define iopte_deref(pte, d)						\
@@ -583,6 +586,9 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data,
 		else if (prot & IOMMU_USE_UPSTREAM_HINT)
 			pte |= (ARM_LPAE_MAIR_ATTR_IDX_UPSTREAM
 				<< ARM_LPAE_PTE_ATTRINDX_SHIFT);
+		else if (prot & IOMMU_USE_LLC_NWA)
+			pte |= (ARM_LPAE_MAIR_ATTR_IDX_LLC_NWA
+				<< ARM_LPAE_PTE_ATTRINDX_SHIFT);
 	} else {
 		pte = ARM_LPAE_PTE_HAP_FAULT;
 		if (prot & IOMMU_READ)
@@ -1117,7 +1123,8 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
 	if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS
 			  | IO_PGTABLE_QUIRK_NO_DMA
 			  | IO_PGTABLE_QUIRK_QCOM_USE_UPSTREAM_HINT
-			  | IO_PGTABLE_QUIRK_QSMMUV500_NON_SHAREABLE))
+			  | IO_PGTABLE_QUIRK_QSMMUV500_NON_SHAREABLE
+			  | IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA))
 		return NULL;
 
 	data = arm_lpae_alloc_pgtable(cfg);
@@ -1138,6 +1145,15 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
 		reg = (ARM_LPAE_TCR_SH_OS << ARM_LPAE_TCR_SH0_SHIFT) |
 			(ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_IRGN0_SHIFT) |
 			(ARM_LPAE_TCR_RGN_WBWA << ARM_LPAE_TCR_ORGN0_SHIFT);
+	else if ((cfg->quirks & IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA) &&
+		(cfg->quirks & IO_PGTABLE_QUIRK_QSMMUV500_NON_SHAREABLE))
+		reg = (ARM_LPAE_TCR_SH_NS << ARM_LPAE_TCR_SH0_SHIFT) |
+			(ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_IRGN0_SHIFT) |
+			(ARM_LPAE_TCR_RGN_WB << ARM_LPAE_TCR_ORGN0_SHIFT);
+	else if (cfg->quirks & IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA)
+		reg = (ARM_LPAE_TCR_SH_OS << ARM_LPAE_TCR_SH0_SHIFT) |
+			(ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_IRGN0_SHIFT) |
+			(ARM_LPAE_TCR_RGN_WB << ARM_LPAE_TCR_ORGN0_SHIFT);
 	else
 		reg = (ARM_LPAE_TCR_SH_OS << ARM_LPAE_TCR_SH0_SHIFT) |
 			(ARM_LPAE_TCR_RGN_NC << ARM_LPAE_TCR_IRGN0_SHIFT) |
@@ -1195,7 +1211,11 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie)
 	       << ARM_LPAE_MAIR_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_UPSTREAM));
 
 	cfg->arm_lpae_s1_cfg.mair[0] = reg;
-	cfg->arm_lpae_s1_cfg.mair[1] = 0;
+
+	reg = ARM_LPAE_MAIR_ATTR_LLC_NWA
+	      << ARM_LPAE_MAIR1_ATTR_SHIFT(ARM_LPAE_MAIR_ATTR_IDX_LLC_NWA);
+
+	cfg->arm_lpae_s1_cfg.mair[1] = reg;
 
 	/* Looking good; allocate a pgd */
 	data->pgd = __arm_lpae_alloc_pages(data->pgd_size, GFP_KERNEL,

drivers/iommu/io-pgtable.h

@@ -93,6 +93,10 @@ struct io_pgtable_cfg {
 	 *	set in TCR for the page table walker. Use attributes specified
 	 *	by the upstream hw instead.
 	 *
+	 * IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA: Override the attributes
+	 *	set in TCR for the page table walker with Write-Back,
+	 *	no Write-Allocate cacheable encoding.
+	 *
 	 */
 	#define IO_PGTABLE_QUIRK_ARM_NS		BIT(0)
 	#define IO_PGTABLE_QUIRK_NO_PERMS	BIT(1)
@@ -101,6 +105,7 @@ struct io_pgtable_cfg {
 	#define IO_PGTABLE_QUIRK_NO_DMA		BIT(4)
 	#define IO_PGTABLE_QUIRK_QSMMUV500_NON_SHAREABLE BIT(5)
 	#define IO_PGTABLE_QUIRK_QCOM_USE_UPSTREAM_HINT	BIT(6)
+	#define IO_PGTABLE_QUIRK_QCOM_USE_LLC_NWA	BIT(7)
 	unsigned long			quirks;
 	unsigned long			pgsize_bitmap;
 	unsigned int			ias;

include/linux/dma-mapping.h

@@ -112,6 +112,13 @@
  */
 #define DMA_ATTR_PRIVILEGED		(1UL << 17)
 
+/*
+ * DMA_ATTR_IOMMU_USE_LLC_NWA: Overrides the bus attributes to use the System
+ * Cache(LLC) with allocation policy as Inner Non-Cacheable, Outer Cacheable:
+ * Write-Back, Read-Allocate, No Write-Allocate policy.
+ */
+#define DMA_ATTR_IOMMU_USE_LLC_NWA	(1UL << 18)
+
 #define DMA_ERROR_CODE       (~(dma_addr_t)0)
 
 /*

include/linux/iommu.h

@@ -46,6 +46,9 @@
 /* Use upstream device's bus attribute */
 #define IOMMU_USE_UPSTREAM_HINT	(1 << 6)
 
+/* Use upstream device's bus attribute with no write-allocate cache policy */
+#define IOMMU_USE_LLC_NWA	(1 << 7)
+
 struct iommu_ops;
 struct iommu_group;
 struct bus_type;
@@ -155,6 +158,7 @@ enum iommu_attr {
 	DOMAIN_ATTR_CB_STALL_DISABLE,
 	DOMAIN_ATTR_BITMAP_IOVA_ALLOCATOR,
 	DOMAIN_ATTR_QCOM_MMU500_ERRATA_MIN_IOVA_ALIGN,
+	DOMAIN_ATTR_USE_LLC_NWA,
 	DOMAIN_ATTR_MAX,
 };