dmaengine: dw-axi-dmac: Add channel abort support

drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c

@@ -230,6 +230,32 @@ static inline void axi_chan_disable(struct axi_dma_chan *chan)
 	}
 }
 
+/*
+ * Abort a channel, flushing its FIFO. This is only intended for use when a
+ * channel has failed to respond to a disable request, e.g. because it is
+ * stalled waiting on a peripheral handshake.
+ */
+static inline void axi_chan_abort(struct axi_dma_chan *chan)
+{
+	u64 val;
+
+	if (chan->chip->dw->hdata->reg_map_8_channels) {
+		val = BIT(chan->id) << (DMAC_CHAN_ABORT_SHIFT - 32) |
+		      BIT(chan->id) << (DMAC_CHAN_ABORT_WE_SHIFT - 32);
+		axi_dma_iowrite32(chan->chip, DMAC_CHEN + 4, (u32)val);
+	} else if (chan->id >= DMAC_CHAN_16) {
+		val = (u64)(BIT(chan->id) >> DMAC_CHAN_16)
+			<< (DMAC_CHAN_ABORT2_SHIFT + DMAC_CHAN_BLOCK_SHIFT) |
+		      (u64)(BIT(chan->id) >> DMAC_CHAN_16)
+			<< (DMAC_CHAN_ABORT2_WE_SHIFT + DMAC_CHAN_BLOCK_SHIFT);
+		axi_dma_iowrite64(chan->chip, DMAC_CHABORTREG, val);
+	} else {
+		val = BIT(chan->id) << DMAC_CHAN_ABORT2_SHIFT |
+		      BIT(chan->id) << DMAC_CHAN_ABORT2_WE_SHIFT;
+		axi_dma_iowrite32(chan->chip, DMAC_CHABORTREG, (u32)val);
+	}
+}
+
 static inline void axi_chan_enable(struct axi_dma_chan *chan)
 {
 	u64 val;
@@ -1356,9 +1382,21 @@ static int dma_chan_terminate_all(struct dma_chan *dchan)
 
 	ret = readl_poll_timeout_atomic(chan->chip->regs + DMAC_CHEN, val,
 					!(val & chan_active), 1000, 50000);
-	if (ret == -ETIMEDOUT)
-		dev_warn(dchan2dev(dchan),
-			 "%s failed to stop\n", axi_chan_name(chan));
+	if (ret == -ETIMEDOUT) {
+		/*
+		 * The channel is probably stalled waiting for a handshake
+		 * from the peripheral - abort it. Discarding the contents
+		 * of the FIFO is permitted when terminating, so this is
+		 * not an error.
+		 */
+		dev_dbg(dchan2dev(dchan), "%s aborting\n", axi_chan_name(chan));
+		axi_chan_abort(chan);
+		ret = readl_poll_timeout_atomic(chan->chip->regs + DMAC_CHEN, val,
+						!(val & chan_active), 10, 10000);
+		if (ret == -ETIMEDOUT)
+			dev_err(dchan2dev(dchan),
+				"%s failed to abort\n", axi_chan_name(chan));
+	}
 
 	if (chan->direction != DMA_MEM_TO_MEM)
 		dw_axi_dma_set_hw_channel(chan, false);

drivers/dma/dw-axi-dmac/dw-axi-dmac.h

@@ -221,6 +221,9 @@ static inline struct axi_dma_chan *dchan_to_axi_dma_chan(struct dma_chan *dchan)
 #define DMAC_CHAN_SUSP_SHIFT		16
 #define DMAC_CHAN_SUSP_WE_SHIFT		24
 
+#define DMAC_CHAN_ABORT_SHIFT		32
+#define DMAC_CHAN_ABORT_WE_SHIFT	40
+
 /* DMAC_CHEN2 */
 #define DMAC_CHAN_EN2_WE_SHIFT		16
 
@@ -232,6 +235,10 @@ static inline struct axi_dma_chan *dchan_to_axi_dma_chan(struct dma_chan *dchan)
 #define DMAC_CHAN_SUSP2_SHIFT		0
 #define DMAC_CHAN_SUSP2_WE_SHIFT	16
 
+/* DMAC_CHABORT */
+#define DMAC_CHAN_ABORT2_SHIFT		0
+#define DMAC_CHAN_ABORT2_WE_SHIFT	16
+
 /* CH_CTL_H */
 #define CH_CTL_H_ARLEN_EN		BIT(6)
 #define CH_CTL_H_ARLEN_POS		7

drivers/misc/rp1-pio.c

@@ -69,6 +69,10 @@
 #define HANDLER(_n, _f) \
 	[_IOC_NR(PIO_IOC_ ## _n)] = { #_n, rp1_pio_ ## _f, _IOC_SIZE(PIO_IOC_ ## _n) }
 
+/* A handler that records its progress in its args if it is interrupted */
+#define HANDLER_RESTARTABLE(_n, _f) \
+	[_IOC_NR(PIO_IOC_ ## _n)] = { #_n, rp1_pio_ ## _f, _IOC_SIZE(PIO_IOC_ ## _n), true }
+
 
 #define ROUND_UP(x, y) (((x) + (y) - 1) - (((x) + (y) - 1) % (y)))
 
@@ -95,6 +99,7 @@ struct dma_info {
 	struct dma_chan *chan;
 	size_t buf_size;
 	size_t buf_count;
+	size_t burst_bytes;
 	bool cyclic;
 	unsigned int head_idx;
 	unsigned int tail_idx;
@@ -961,6 +966,42 @@ static void rp1_pio_sm_kernel_dma_callback(void *param)
 	kfree(dxs);
 }
 
+/*
+ * Once the RX FIFO reaches its DMA threshold, the DMA request is held until
+ * the DMAC acknowledges a burst, even if the FIFO is emptied in the meantime.
+ * A request still held when the SM is released makes the next user's first
+ * burst read stale data from the empty FIFO, so retire it (and any data that
+ * is left) with a throwaway transfer. Only one request can be outstanding, and
+ * it is satisfied by whatever is at the head of the FIFO, so a full FIFO is the
+ * most that can be pending - the extra burst is just a margin. The SM must
+ * already be disabled, so that no new data can arrive and any bursts complete
+ * almost immediately; the transfer is then stopped, finished or not.
+ */
+static void rp1_pio_sm_dma_flush_rx(struct dma_info *dma)
+{
+	size_t len = RP1_PIO_FIFO_DEPTH * sizeof(uint32_t) + dma->burst_bytes;
+	struct device *dma_dev = dma->chan->device->dev;
+	struct dma_async_tx_descriptor *desc;
+	dma_addr_t dma_addr;
+	void *buf;
+
+	dmaengine_terminate_sync(dma->chan);
+
+	buf = dma_alloc_coherent(dma_dev, len, &dma_addr, GFP_KERNEL);
+	if (!buf)
+		return;
+
+	desc = dmaengine_prep_slave_single(dma->chan, dma_addr, len,
+					   DMA_DEV_TO_MEM, DMA_CTRL_ACK);
+	if (desc && dmaengine_submit(desc) >= 0) {
+		dma_async_issue_pending(dma->chan);
+		fsleep(1000);
+	}
+
+	dmaengine_terminate_sync(dma->chan);
+	dma_free_coherent(dma_dev, len, buf, dma_addr);
+}
+
 static void rp1_pio_sm_dma_free(struct device *dev, struct dma_info *dma)
 {
 	dmaengine_terminate_all(dma->chan);
@@ -1112,6 +1153,7 @@ static int rp1_pio_sm_config_xfer_internal(struct rp1_pio_client *client, uint s
 		config.dst_maxburst = dma_caps.max_burst;
 	else
 		config.src_maxburst = dma_caps.max_burst;
+	dma->burst_bytes = dma_caps.max_burst * sizeof(uint32_t);
 
 	ret = dmaengine_slave_config(dma->chan, &config);
 	if (ret)
@@ -1195,10 +1237,24 @@ static int rp1_pio_sm_config_xfer_v2_user(struct rp1_pio_client *client, void *p
 					       args->flags);
 }
 
+/*
+ * Record the progress of a transfer in its arguments. If the call is
+ * interrupted, the updated arguments are copied back to the caller, so that
+ * a restarted call resumes where this one stopped rather than repeating it.
+ */
+static void rp1_pio_sm_xfer_progress(struct rp1_pio_sm_xfer_data32_args *args,
+				     size_t bytes)
+{
+	args->data += bytes;
+	args->data_bytes -= bytes;
+}
+
 static int rp1_pio_sm_tx_user(struct rp1_pio_device *pio, struct dma_info *dma,
-				  const void __user *userbuf, size_t bytes)
+			      struct rp1_pio_sm_xfer_data32_args *args)
 {
 	struct platform_device *pdev = pio->pdev;
+	const void __user *userbuf = args->data;
+	size_t bytes = args->data_bytes;
 	struct dma_async_tx_descriptor *desc;
 	struct device *dev = &pdev->dev;
 	int ret = 0;
@@ -1209,20 +1265,17 @@ static int rp1_pio_sm_tx_user(struct rp1_pio_device *pio, struct dma_info *dma,
 
 		/* grab the next free buffer, waiting if they're all full */
 		if (dma->head_idx - dma->tail_idx == dma->buf_count) {
-			if (down_interruptible(&dma->buf_sem)) {
-				dev_err(dev, "DMA bounce interrupted\n");
-				break;
-			}
+			if (down_interruptible(&dma->buf_sem))
+				return -ERESTARTSYS;
 			dma->tail_idx++;
 		}
 
 		dbi = &dma->bufs[dma->head_idx % dma->buf_count];
 
 		sg_dma_len(&dbi->sgl) = copy_bytes;
 
-		ret = copy_from_user(dbi->buf, userbuf, copy_bytes);
-		if (ret < 0)
-			break;
+		if (copy_from_user(dbi->buf, userbuf, copy_bytes))
+			return -EFAULT;
 
 		userbuf += copy_bytes;
 
@@ -1249,6 +1302,7 @@ static int rp1_pio_sm_tx_user(struct rp1_pio_device *pio, struct dma_info *dma,
 
 		dma->head_idx++;
 		bytes -= copy_bytes;
+		rp1_pio_sm_xfer_progress(args, copy_bytes);
 	}
 
 	return ret;
@@ -1297,9 +1351,10 @@ static int rp1_pio_sm_rx_submit(struct rp1_pio_device *pio, struct dma_info *dma
  * chunk (reads can be any size up to buf_size, and need not match it).
  */
 static int rp1_pio_sm_rx_user(struct rp1_pio_device *pio, struct dma_info *dma,
-				  void __user *userbuf, size_t bytes)
+			      struct rp1_pio_sm_xfer_data32_args *args)
 {
-	struct device *dev = &pio->pdev->dev;
+	void __user *userbuf = args->data;
+	size_t bytes = args->data_bytes;
 	int ret;
 
 	if (!bytes)
@@ -1358,16 +1413,16 @@ static int rp1_pio_sm_rx_user(struct rp1_pio_device *pio, struct dma_info *dma,
 		if (len > bytes)
 			return -EINVAL;
 
-		if (down_interruptible(&dma->buf_sem)) {
-			dev_err(dev, "DMA wait interrupted\n");
-			return -ETIMEDOUT;
-		}
+		if (down_interruptible(&dma->buf_sem))
+			return -ERESTARTSYS;
+
 		dma->tail_idx++;
 
 		if (copy_to_user(userbuf, dbi->buf, len))
 			return -EFAULT;
 		userbuf += len;
 		bytes -= len;
+		rp1_pio_sm_xfer_progress(args, len);
 	}
 
 	return 0;
@@ -1394,22 +1449,28 @@ static int rp1_pio_sm_xfer_data32_user(struct rp1_pio_client *client, void *para
 		return -EINVAL;
 
 	if (args->dir == RP1_PIO_DIR_TO_SM)
-		return rp1_pio_sm_tx_user(pio, dma, args->data, args->data_bytes);
+		return rp1_pio_sm_tx_user(pio, dma, args);
 	else
-		return rp1_pio_sm_rx_user(pio, dma, args->data, args->data_bytes);
+		return rp1_pio_sm_rx_user(pio, dma, args);
 }
 
 static int rp1_pio_sm_xfer_data_user(struct rp1_pio_client *client, void *param)
 {
 	struct rp1_pio_sm_xfer_data_args *args = param;
 	struct rp1_pio_sm_xfer_data32_args args32;
+	int ret;
 
 	args32.sm = args->sm;
 	args32.dir = args->dir;
 	args32.data_bytes = args->data_bytes;
 	args32.data = args->data;
 
-	return rp1_pio_sm_xfer_data32_user(client, &args32);
+	ret = rp1_pio_sm_xfer_data32_user(client, &args32);
+
+	args->data_bytes = args32.data_bytes;
+	args->data = args32.data;
+
+	return ret;
 }
 
 int rp1_pio_sm_config_xfer(struct rp1_pio_client *client, uint sm, uint dir,
@@ -1509,10 +1570,11 @@ struct handler_info {
 	const char *name;
 	int (*func)(struct rp1_pio_client *client, void *param);
 	int argsize;
+	bool restartable;
 } ioctl_handlers[] = {
 	HANDLER(SM_CONFIG_XFER, sm_config_xfer_user),
-	HANDLER(SM_XFER_DATA, sm_xfer_data_user),
-	HANDLER(SM_XFER_DATA32, sm_xfer_data32_user),
+	HANDLER_RESTARTABLE(SM_XFER_DATA, sm_xfer_data_user),
+	HANDLER_RESTARTABLE(SM_XFER_DATA32, sm_xfer_data32_user),
 	HANDLER(SM_CONFIG_XFER32, sm_config_xfer32_user),
 	HANDLER(SM_CONFIG_XFER_V2, sm_config_xfer_v2_user),
 
@@ -1647,6 +1709,9 @@ void rp1_pio_close(struct rp1_pio_client *client)
 			struct dma_info *dma = &pio->dma_configs[i >> 1][i & 1];
 
 			claimed &= ~mask;
+			/* The SMs have been disabled, so this is safe */
+			if ((i & 1) == RP1_PIO_DIR_FROM_SM)
+				rp1_pio_sm_dma_flush_rx(dma);
 			rp1_pio_sm_dma_free(&pio->pdev->dev, dma);
 		}
 	}
@@ -1735,6 +1800,10 @@ static long rp1_pio_ioctl(struct file *filp, unsigned int ioctl_num,
 	if (ret > 0) {
 		if (copy_to_user(argp, argbuf, ret))
 			ret = -EFAULT;
+	} else if (ret == -ERESTARTSYS && hdlr->restartable) {
+		/* Pass back the progress, so that a restart resumes from there */
+		if (copy_to_user(argp, argbuf, sz))
+			ret = -EFAULT;
 	}
 
 	return ret;
@@ -1779,27 +1848,45 @@ static long rp1_pio_compat_ioctl(struct file *filp, unsigned int ioctl_num,
 	{
 		struct rp1_pio_sm_xfer_data_args_compat compat_param;
 		struct rp1_pio_sm_xfer_data_args param;
+		int ret;
 
 		if (copy_from_user(&compat_param, compat_ptr(ioctl_param), sizeof(compat_param)))
 			return -EFAULT;
 		param.sm = compat_param.sm;
 		param.dir = compat_param.dir;
 		param.data_bytes = compat_param.data_bytes;
 		param.data = compat_ptr(compat_param.data);
-		return rp1_pio_sm_xfer_data_user(client, &param);
+		ret = rp1_pio_sm_xfer_data_user(client, &param);
+		if (ret == -ERESTARTSYS) {
+			compat_param.data_bytes = param.data_bytes;
+			compat_param.data = ptr_to_compat(param.data);
+			if (copy_to_user(compat_ptr(ioctl_param), &compat_param,
+					 sizeof(compat_param)))
+				ret = -EFAULT;
+		}
+		return ret;
 	}
 	case PIO_IOC_SM_XFER_DATA32_COMPAT:
 	{
 		struct rp1_pio_sm_xfer_data32_args_compat compat_param;
 		struct rp1_pio_sm_xfer_data32_args param;
+		int ret;
 
 		if (copy_from_user(&compat_param, compat_ptr(ioctl_param), sizeof(compat_param)))
 			return -EFAULT;
 		param.sm = compat_param.sm;
 		param.dir = compat_param.dir;
 		param.data_bytes = compat_param.data_bytes;
 		param.data = compat_ptr(compat_param.data);
-		return rp1_pio_sm_xfer_data32_user(client, &param);
+		ret = rp1_pio_sm_xfer_data32_user(client, &param);
+		if (ret == -ERESTARTSYS) {
+			compat_param.data_bytes = param.data_bytes;
+			compat_param.data = ptr_to_compat(param.data);
+			if (copy_to_user(compat_ptr(ioctl_param), &compat_param,
+					 sizeof(compat_param)))
+				ret = -EFAULT;
+		}
+		return ret;
 	}
 
 	case PIO_IOC_READ_HW_COMPAT: