iommu/arm-smmu: Remove io-pgtable spinlock
With the io-pgtable code now robust against (valid) races, we no longer need to serialise all operations with a lock. This might make broken callers who issue concurrent operations on overlapping addresses go even more wrong than before, but hey, they already had little hope of useful or deterministic results. We do however still have to keep a lock around to serialise the ATS1* translation ops, as parallel iova_to_phys() calls could lead to unpredictable hardware behaviour otherwise. Signed-off-by: Robin Murphy <robin.murphy@arm.com> Signed-off-by: Will Deacon <will.deacon@arm.com>
This commit is contained in:
parent
119ff305b0
commit
523d7423e2
|
@ -433,10 +433,10 @@ enum arm_smmu_domain_stage {
|
||||||
struct arm_smmu_domain {
|
struct arm_smmu_domain {
|
||||||
struct arm_smmu_device *smmu;
|
struct arm_smmu_device *smmu;
|
||||||
struct io_pgtable_ops *pgtbl_ops;
|
struct io_pgtable_ops *pgtbl_ops;
|
||||||
spinlock_t pgtbl_lock;
|
|
||||||
struct arm_smmu_cfg cfg;
|
struct arm_smmu_cfg cfg;
|
||||||
enum arm_smmu_domain_stage stage;
|
enum arm_smmu_domain_stage stage;
|
||||||
struct mutex init_mutex; /* Protects smmu pointer */
|
struct mutex init_mutex; /* Protects smmu pointer */
|
||||||
|
spinlock_t cb_lock; /* Serialises ATS1* ops */
|
||||||
struct iommu_domain domain;
|
struct iommu_domain domain;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -1113,7 +1113,7 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type)
|
||||||
}
|
}
|
||||||
|
|
||||||
mutex_init(&smmu_domain->init_mutex);
|
mutex_init(&smmu_domain->init_mutex);
|
||||||
spin_lock_init(&smmu_domain->pgtbl_lock);
|
spin_lock_init(&smmu_domain->cb_lock);
|
||||||
|
|
||||||
return &smmu_domain->domain;
|
return &smmu_domain->domain;
|
||||||
}
|
}
|
||||||
|
@ -1391,35 +1391,23 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
|
||||||
static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
|
static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
|
||||||
phys_addr_t paddr, size_t size, int prot)
|
phys_addr_t paddr, size_t size, int prot)
|
||||||
{
|
{
|
||||||
int ret;
|
struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
|
||||||
unsigned long flags;
|
|
||||||
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
|
|
||||||
struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
|
|
||||||
|
|
||||||
if (!ops)
|
if (!ops)
|
||||||
return -ENODEV;
|
return -ENODEV;
|
||||||
|
|
||||||
spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
|
return ops->map(ops, iova, paddr, size, prot);
|
||||||
ret = ops->map(ops, iova, paddr, size, prot);
|
|
||||||
spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
|
|
||||||
return ret;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
|
static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova,
|
||||||
size_t size)
|
size_t size)
|
||||||
{
|
{
|
||||||
size_t ret;
|
struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
|
||||||
unsigned long flags;
|
|
||||||
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
|
|
||||||
struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
|
|
||||||
|
|
||||||
if (!ops)
|
if (!ops)
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
|
return ops->unmap(ops, iova, size);
|
||||||
ret = ops->unmap(ops, iova, size);
|
|
||||||
spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
|
|
||||||
return ret;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
|
static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
|
||||||
|
@ -1433,10 +1421,11 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
|
||||||
void __iomem *cb_base;
|
void __iomem *cb_base;
|
||||||
u32 tmp;
|
u32 tmp;
|
||||||
u64 phys;
|
u64 phys;
|
||||||
unsigned long va;
|
unsigned long va, flags;
|
||||||
|
|
||||||
cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
|
cb_base = ARM_SMMU_CB(smmu, cfg->cbndx);
|
||||||
|
|
||||||
|
spin_lock_irqsave(&smmu_domain->cb_lock, flags);
|
||||||
/* ATS1 registers can only be written atomically */
|
/* ATS1 registers can only be written atomically */
|
||||||
va = iova & ~0xfffUL;
|
va = iova & ~0xfffUL;
|
||||||
if (smmu->version == ARM_SMMU_V2)
|
if (smmu->version == ARM_SMMU_V2)
|
||||||
|
@ -1446,6 +1435,7 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
|
||||||
|
|
||||||
if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
|
if (readl_poll_timeout_atomic(cb_base + ARM_SMMU_CB_ATSR, tmp,
|
||||||
!(tmp & ATSR_ACTIVE), 5, 50)) {
|
!(tmp & ATSR_ACTIVE), 5, 50)) {
|
||||||
|
spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
|
||||||
dev_err(dev,
|
dev_err(dev,
|
||||||
"iova to phys timed out on %pad. Falling back to software table walk.\n",
|
"iova to phys timed out on %pad. Falling back to software table walk.\n",
|
||||||
&iova);
|
&iova);
|
||||||
|
@ -1453,6 +1443,7 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
|
||||||
}
|
}
|
||||||
|
|
||||||
phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
|
phys = readq_relaxed(cb_base + ARM_SMMU_CB_PAR);
|
||||||
|
spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
|
||||||
if (phys & CB_PAR_F) {
|
if (phys & CB_PAR_F) {
|
||||||
dev_err(dev, "translation fault!\n");
|
dev_err(dev, "translation fault!\n");
|
||||||
dev_err(dev, "PAR = 0x%llx\n", phys);
|
dev_err(dev, "PAR = 0x%llx\n", phys);
|
||||||
|
@ -1465,10 +1456,8 @@ static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
|
||||||
static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
|
static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
|
||||||
dma_addr_t iova)
|
dma_addr_t iova)
|
||||||
{
|
{
|
||||||
phys_addr_t ret;
|
|
||||||
unsigned long flags;
|
|
||||||
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
|
struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
|
||||||
struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
|
struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
|
||||||
|
|
||||||
if (domain->type == IOMMU_DOMAIN_IDENTITY)
|
if (domain->type == IOMMU_DOMAIN_IDENTITY)
|
||||||
return iova;
|
return iova;
|
||||||
|
@ -1476,17 +1465,11 @@ static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
|
||||||
if (!ops)
|
if (!ops)
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
spin_lock_irqsave(&smmu_domain->pgtbl_lock, flags);
|
|
||||||
if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
|
if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
|
||||||
smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
|
smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
|
||||||
ret = arm_smmu_iova_to_phys_hard(domain, iova);
|
return arm_smmu_iova_to_phys_hard(domain, iova);
|
||||||
} else {
|
|
||||||
ret = ops->iova_to_phys(ops, iova);
|
|
||||||
}
|
|
||||||
|
|
||||||
spin_unlock_irqrestore(&smmu_domain->pgtbl_lock, flags);
|
return ops->iova_to_phys(ops, iova);
|
||||||
|
|
||||||
return ret;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool arm_smmu_capable(enum iommu_cap cap)
|
static bool arm_smmu_capable(enum iommu_cap cap)
|
||||||
|
|
Loading…
Reference in New Issue