From 612942d207f957dacffc4bd36dc47b7ad9dfefb0 Mon Sep 17 00:00:00 2001 From: Phil Elwell Date: Wed, 28 Apr 2021 17:46:01 +0100 Subject: [PATCH 363/697] dmaengine: dw-axi-dmac: Fixes for RP1 Don't assume that DMA addresses of devices are the same as their physical addresses - convert correctly. The CFG2 register layout is used when there are more than 8 channels, but also when configured for more than 16 target peripheral devices because the index of the handshake signal has to be made wider. Reset the DMAC on probe The driver goes to the trouble of tracking when transfers have been paused, but then doesn't report that state when queried. Not having APB registers is not an error - for most use cases it's not even of interest, it's expected. Demote the message to debug level, which is disabled by default. Each channel has a descriptor pool, which is shared between transfers. It is unsafe to treat the total number of descriptors allocated from a pool as the number allocated to a specific transfer; doing so leads to releasing buffers that shouldn't be released and walking off the ends of descriptor lists. Instead, give each transfer descriptor its own count. Support partial transfers: Some use cases involve streaming from a device where the transfer only proceeds when the device's FIFO occupancy exceeds a certain threshold. In such cases (e.g. when pulling data from a UART) it is important to know how much data has been transferred so far, in order that remaining bytes can be read from the FIFO directly by software. Add the necessary code to provide this "residue" value with a finer, sub-transfer granularity. In order to prevent the occasional byte getting stuck in the DMA controller's internal buffers, restrict the destination memory width to the source register width. Signed-off-by: Phil Elwell dmaengine: dw-axi-dmac: Fix a non-atomic update dw_axi_dma_interrupt disables interrupts for the duration of the channel handling. It does so by clearing a bit in the DMA_CFG register - an action that involves a read-modify-write. That in itself would be safe because there will be no further interrupts, hence no reentrancy, were it the only bit of code accessing that register. The only neighbour of INT_EN is DMAC_EN - the main enable for the block. That's not the sort of thing you would expect to be modified during the normal course of operation, but bizarrely it is set at the start of the transfer of every block, in axi_chan_block_xfer_star, by a call to axi_dma_enable. This can lead to INT_EN being accidentally cleared, which causes all DMA transfers to time out. One might think that the enabling was being delayed until the first transfer, but the probe function calls axi_dma_resume which in turn calls axi_dma_enable, so that isn't the case. Fix the atomicity problem by removing the spurious call to axi_dma_enable. Signed-off-by: Phil Elwell --- .../dma/dw-axi-dmac/dw-axi-dmac-platform.c | 132 +++++++++++++++--- 1 file changed, 111 insertions(+), 21 deletions(-) --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -95,6 +96,17 @@ axi_chan_iowrite64(struct axi_dma_chan * iowrite32(upper_32_bits(val), chan->chan_regs + reg + 4); } +static inline u64 +axi_chan_ioread64(struct axi_dma_chan *chan, u32 reg) +{ + /* + * We split one 64 bit read into two 32 bit reads as some HW doesn't + * support 64 bit access. + */ + return ((u64)ioread32(chan->chan_regs + reg + 4) << 32) + + ioread32(chan->chan_regs + reg); +} + static inline void axi_chan_config_write(struct axi_dma_chan *chan, struct axi_dma_chan_config *config) { @@ -266,7 +278,18 @@ static void axi_dma_hw_init(struct axi_d { int ret; u32 i; + int retries = 1000; + axi_dma_iowrite32(chip, DMAC_RESET, 1); + while (axi_dma_ioread32(chip, DMAC_RESET)) { + retries--; + if (!retries) { + dev_err(chip->dev, "%s: DMAC failed to reset\n", + __func__); + return; + } + cpu_relax(); + } for (i = 0; i < chip->dw->hdata->nr_channels; i++) { axi_chan_irq_disable(&chip->dw->chan[i], DWAXIDMAC_IRQ_ALL); axi_chan_disable(&chip->dw->chan[i]); @@ -351,6 +374,48 @@ static void vchan_desc_put(struct virt_d axi_desc_put(vd_to_axi_desc(vdesc)); } +static u32 axi_dma_desc_src_pos(struct axi_dma_desc *desc, dma_addr_t addr) +{ + unsigned int idx = 0; + u32 pos = 0; + + while (pos < desc->length) { + struct axi_dma_hw_desc *hw_desc = &desc->hw_desc[idx++]; + u32 len = hw_desc->len; + dma_addr_t start = le64_to_cpu(hw_desc->lli->sar); + + if (addr >= start && addr <= (start + len)) { + pos += addr - start; + break; + } + + pos += len; + } + + return pos; +} + +static u32 axi_dma_desc_dst_pos(struct axi_dma_desc *desc, dma_addr_t addr) +{ + unsigned int idx = 0; + u32 pos = 0; + + while (pos < desc->length) { + struct axi_dma_hw_desc *hw_desc = &desc->hw_desc[idx++]; + u32 len = hw_desc->len; + dma_addr_t start = le64_to_cpu(hw_desc->lli->dar); + + if (addr >= start && addr <= (start + len)) { + pos += addr - start; + break; + } + + pos += len; + } + + return pos; +} + static enum dma_status dma_chan_tx_status(struct dma_chan *dchan, dma_cookie_t cookie, struct dma_tx_state *txstate) @@ -360,10 +425,7 @@ dma_chan_tx_status(struct dma_chan *dcha enum dma_status status; u32 completed_length; unsigned long flags; - u32 completed_blocks; size_t bytes = 0; - u32 length; - u32 len; status = dma_cookie_status(dchan, cookie, txstate); if (status == DMA_COMPLETE || !txstate) @@ -372,16 +434,31 @@ dma_chan_tx_status(struct dma_chan *dcha spin_lock_irqsave(&chan->vc.lock, flags); vdesc = vchan_find_desc(&chan->vc, cookie); - if (vdesc) { - length = vd_to_axi_desc(vdesc)->length; - completed_blocks = vd_to_axi_desc(vdesc)->completed_blocks; - len = vd_to_axi_desc(vdesc)->hw_desc[0].len; - completed_length = completed_blocks * len; - bytes = length - completed_length; + if (vdesc && vdesc == vchan_next_desc(&chan->vc)) { + /* This descriptor is in-progress */ + struct axi_dma_desc *desc = vd_to_axi_desc(vdesc); + dma_addr_t addr; + + if (chan->direction == DMA_MEM_TO_DEV) { + addr = axi_chan_ioread64(chan, CH_SAR); + completed_length = axi_dma_desc_src_pos(desc, addr); + } else if (chan->direction == DMA_DEV_TO_MEM) { + addr = axi_chan_ioread64(chan, CH_DAR); + completed_length = axi_dma_desc_dst_pos(desc, addr); + } else { + completed_length = 0; + } + bytes = desc->length - completed_length; + } else if (vdesc) { + /* Still in the queue so not started */ + bytes = vd_to_axi_desc(vdesc)->length; } - spin_unlock_irqrestore(&chan->vc.lock, flags); + if (chan->is_paused && status == DMA_IN_PROGRESS) + status = DMA_PAUSED; + dma_set_residue(txstate, bytes); + spin_unlock_irqrestore(&chan->vc.lock, flags); return status; } @@ -435,8 +512,6 @@ static void axi_chan_block_xfer_start(st return; } - axi_dma_enable(chan->chip); - config.dst_multblk_type = DWAXIDMAC_MBLK_TYPE_LL; config.src_multblk_type = DWAXIDMAC_MBLK_TYPE_LL; config.tt_fc = DWAXIDMAC_TT_FC_MEM_TO_MEM_DMAC; @@ -569,7 +644,7 @@ static void dw_axi_dma_set_hw_channel(st unsigned long reg_value, val; if (!chip->apb_regs) { - dev_err(chip->dev, "apb_regs not initialized\n"); + dev_dbg(chip->dev, "apb_regs not initialized\n"); return; } @@ -673,18 +748,25 @@ static int dw_axi_dma_set_hw_desc(struct switch (chan->direction) { case DMA_MEM_TO_DEV: reg_width = __ffs(chan->config.dst_addr_width); - device_addr = chan->config.dst_addr; + device_addr = phys_to_dma(chan->chip->dev, chan->config.dst_addr); ctllo = reg_width << CH_CTL_L_DST_WIDTH_POS | mem_width << CH_CTL_L_SRC_WIDTH_POS | + DWAXIDMAC_BURST_TRANS_LEN_1 << CH_CTL_L_DST_MSIZE_POS | + DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_SRC_MSIZE_POS | DWAXIDMAC_CH_CTL_L_NOINC << CH_CTL_L_DST_INC_POS | DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_SRC_INC_POS; block_ts = len >> mem_width; break; case DMA_DEV_TO_MEM: reg_width = __ffs(chan->config.src_addr_width); - device_addr = chan->config.src_addr; + /* Prevent partial access units getting lost */ + if (mem_width > reg_width) + mem_width = reg_width; + device_addr = phys_to_dma(chan->chip->dev, chan->config.src_addr); ctllo = reg_width << CH_CTL_L_SRC_WIDTH_POS | mem_width << CH_CTL_L_DST_WIDTH_POS | + DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_DST_MSIZE_POS | + DWAXIDMAC_BURST_TRANS_LEN_1 << CH_CTL_L_SRC_MSIZE_POS | DWAXIDMAC_CH_CTL_L_INC << CH_CTL_L_DST_INC_POS | DWAXIDMAC_CH_CTL_L_NOINC << CH_CTL_L_SRC_INC_POS; block_ts = len >> reg_width; @@ -720,9 +802,6 @@ static int dw_axi_dma_set_hw_desc(struct } hw_desc->lli->block_ts_lo = cpu_to_le32(block_ts - 1); - - ctllo |= DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_DST_MSIZE_POS | - DWAXIDMAC_BURST_TRANS_LEN_4 << CH_CTL_L_SRC_MSIZE_POS; hw_desc->lli->ctl_lo = cpu_to_le32(ctllo); set_desc_src_master(hw_desc); @@ -817,6 +896,8 @@ dw_axi_dma_chan_prep_cyclic(struct dma_c src_addr += segment_len; } + desc->nr_hw_descs = total_segments; + llp = desc->hw_desc[0].llp; /* Managed transfer list */ @@ -896,6 +977,8 @@ dw_axi_dma_chan_prep_slave_sg(struct dma } while (len >= segment_len); } + desc->nr_hw_descs = loop; + /* Set end-of-link to the last link descriptor of list */ set_desc_last(&desc->hw_desc[num_sgs - 1]); @@ -1003,6 +1086,8 @@ dma_chan_prep_dma_memcpy(struct dma_chan num++; } + desc->nr_hw_descs = num; + /* Set end-of-link to the last link descriptor of list */ set_desc_last(&desc->hw_desc[num - 1]); /* Managed transfer list */ @@ -1051,7 +1136,7 @@ static void axi_chan_dump_lli(struct axi static void axi_chan_list_dump_lli(struct axi_dma_chan *chan, struct axi_dma_desc *desc_head) { - int count = atomic_read(&chan->descs_allocated); + int count = desc_head->nr_hw_descs; int i; for (i = 0; i < count; i++) @@ -1094,11 +1179,11 @@ out: static void axi_chan_block_xfer_complete(struct axi_dma_chan *chan) { - int count = atomic_read(&chan->descs_allocated); struct axi_dma_hw_desc *hw_desc; struct axi_dma_desc *desc; struct virt_dma_desc *vd; unsigned long flags; + int count; u64 llp; int i; @@ -1120,6 +1205,7 @@ static void axi_chan_block_xfer_complete if (chan->cyclic) { desc = vd_to_axi_desc(vd); if (desc) { + count = desc->nr_hw_descs; llp = lo_hi_readq(chan->chan_regs + CH_LLP); for (i = 0; i < count; i++) { hw_desc = &desc->hw_desc[i]; @@ -1397,6 +1483,10 @@ static int parse_device_properties(struc chip->dw->hdata->nr_masters = tmp; + ret = device_property_read_u32(dev, "snps,dma-targets", &tmp); + if (!ret && tmp > 16) + chip->dw->hdata->use_cfg2 = true; + ret = device_property_read_u32(dev, "snps,data-width", &tmp); if (ret) return ret;