ALSA: memalloc: Support for non-contiguous page allocation

This patch adds the support for allocation of non-contiguous DMA pages
in the common memalloc helper.  It's another SG-buffer type, but
unlike the existing one, this is directional and requires the explicit
sync / invalidation of dirty pages on non-coherent architectures.

For this enhancement, the following points are changed:
- snd_dma_device stores the DMA direction.
- snd_dma_device stores need_sync flag indicating whether the explicit
  sync is required or not.
- A new variant of helper functions, snd_dma_alloc_dir_pages() and
  *_all() are introduced; the old snd_dma_alloc_pages() and *_all()
  kept as just wrappers with DMA_BIDIRECTIONAL.
- A new helper snd_dma_buffer_sync() is introduced; this gets called
  in the appropriate places.
- A new allocation type, SNDRV_DMA_TYPE_NONCONTIG, is introduced.

When the driver allocates pages with this new type, and it may require
the SNDRV_PCM_INFO_EXPLICIT_SYNC flag set to the PCM hardware.info for
taking the full control of PCM applptr and hwptr changes (that implies
disabling the mmap of control/status data).  When the buffer
allocation is managed by snd_pcm_set_managed_buffer(), this flag is
automatically set depending on the result of dma_need_sync()
internally.  Otherwise, if the buffer is managed manually, the driver
has to set the flag explicitly, too.

The explicit sync between CPU and device for non-coherent memory is
performed at the points before and after read/write transfer as well
as the applptr/hwptr syncptr ioctl.  In the case of mmap mode,
user-space is supposed to call the syncptr ioctl with the hwptr flag
to update and fetch the status at first; this corresponds to CPU-sync.
Then user-space advances the applptr via syncptr ioctl again with
applptr flag, and this corresponds to the device sync with flushing.

Other than the DMA direction and the explicit sync, the usage of this
new buffer type is almost equivalent with the existing
SNDRV_DMA_TYPE_DEV_SG; you can get the page and the address via
snd_sgbuf_get_page() and snd_sgbuf_get_addr(), also calculate the
continuous pages via snd_sgbuf_get_chunk_size().

For those SG-page handling, the non-contig type shares the same ops
with the vmalloc handler.  As we do always vmap the SG pages at first,
the actual address can be deduced from the vmapped address easily
without iterating the SG-list.

Link: https://lore.kernel.org/r/20211017074859.24112-2-tiwai@suse.de
Signed-off-by: Takashi Iwai <tiwai@suse.de>

7 files changed, 145 insertions, 11 deletions

diff --git a/sound/core/memalloc.c b/sound/core/memalloc.c
index c7c943c661e6..11f9a68bf94c 100644
--- a/sound/core/memalloc.c
+++ b/sound/core/memalloc.c
@@ -10,6 +10,7 @@
 #include <linux/mm.h>
 #include <linux/dma-mapping.h>
 #include <linux/genalloc.h>
+#include <linux/highmem.h>
 #include <linux/vmalloc.h>
 #ifdef CONFIG_X86
 #include <asm/set_memory.h>
@@ -39,9 +40,11 @@ static void *__snd_dma_alloc_pages(struct snd_dma_buffer *dmab, size_t size)
 }
 
 /**
- * snd_dma_alloc_pages - allocate the buffer area according to the given type
+ * snd_dma_alloc_dir_pages - allocate the buffer area according to the given
+ *	type and direction
  * @type: the DMA buffer type
  * @device: the device pointer
+ * @dir: DMA direction
  * @size: the buffer size to allocate
  * @dmab: buffer allocation record to store the allocated data
  *
@@ -51,8 +54,9 @@ static void *__snd_dma_alloc_pages(struct snd_dma_buffer *dmab, size_t size)
  * Return: Zero if the buffer with the given size is allocated successfully,
  * otherwise a negative value on error.
  */
-int snd_dma_alloc_pages(int type, struct device *device, size_t size,
-			struct snd_dma_buffer *dmab)
+int snd_dma_alloc_dir_pages(int type, struct device *device,
+			    enum dma_data_direction dir, size_t size,
+			    struct snd_dma_buffer *dmab)
 {
 	if (WARN_ON(!size))
 		return -ENXIO;
@@ -62,6 +66,7 @@ int snd_dma_alloc_pages(int type, struct device *device, size_t size,
 	size = PAGE_ALIGN(size);
 	dmab->dev.type = type;
 	dmab->dev.dev = device;
+	dmab->dev.dir = dir;
 	dmab->bytes = 0;
 	dmab->addr = 0;
 	dmab->private_data = NULL;
@@ -71,7 +76,7 @@ int snd_dma_alloc_pages(int type, struct device *device, size_t size,
 	dmab->bytes = size;
 	return 0;
 }
-EXPORT_SYMBOL(snd_dma_alloc_pages);
+EXPORT_SYMBOL(snd_dma_alloc_dir_pages);
 
 /**
  * snd_dma_alloc_pages_fallback - allocate the buffer area according to the given type with fallback
@@ -129,9 +134,10 @@ static void __snd_release_pages(struct device *dev, void *res)
 }
 
 /**
- * snd_devm_alloc_pages - allocate the buffer and manage with devres
+ * snd_devm_alloc_dir_pages - allocate the buffer and manage with devres
  * @dev: the device pointer
  * @type: the DMA buffer type
+ * @dir: DMA direction
  * @size: the buffer size to allocate
  *
  * Allocate buffer pages depending on the given type and manage using devres.
@@ -144,7 +150,8 @@ static void __snd_release_pages(struct device *dev, void *res)
  * The function returns the snd_dma_buffer object at success, or NULL if failed.
  */
 struct snd_dma_buffer *
-snd_devm_alloc_pages(struct device *dev, int type, size_t size)
+snd_devm_alloc_dir_pages(struct device *dev, int type,
+			 enum dma_data_direction dir, size_t size)
 {
 	struct snd_dma_buffer *dmab;
 	int err;
@@ -157,7 +164,7 @@ snd_devm_alloc_pages(struct device *dev, int type, size_t size)
 	if (!dmab)
 		return NULL;
 
-	err = snd_dma_alloc_pages(type, dev, size, dmab);
+	err = snd_dma_alloc_dir_pages(type, dev, dir, size, dmab);
 	if (err < 0) {
 		devres_free(dmab);
 		return NULL;
@@ -166,7 +173,7 @@ snd_devm_alloc_pages(struct device *dev, int type, size_t size)
 	devres_add(dev, dmab);
 	return dmab;
 }
-EXPORT_SYMBOL_GPL(snd_devm_alloc_pages);
+EXPORT_SYMBOL_GPL(snd_devm_alloc_dir_pages);
 
 /**
  * snd_dma_buffer_mmap - perform mmap of the given DMA buffer
@@ -185,6 +192,26 @@ int snd_dma_buffer_mmap(struct snd_dma_buffer *dmab,
 }
 EXPORT_SYMBOL(snd_dma_buffer_mmap);
 
+#ifdef CONFIG_HAS_DMA
+/**
+ * snd_dma_buffer_sync - sync DMA buffer between CPU and device
+ * @dmab: buffer allocation information
+ * @mod: sync mode
+ */
+void snd_dma_buffer_sync(struct snd_dma_buffer *dmab,
+			 enum snd_dma_sync_mode mode)
+{
+	const struct snd_malloc_ops *ops;
+
+	if (!dmab || !dmab->dev.need_sync)
+		return;
+	ops = snd_dma_get_ops(dmab);
+	if (ops && ops->sync)
+		ops->sync(dmab, mode);
+}
+EXPORT_SYMBOL_GPL(snd_dma_buffer_sync);
+#endif /* CONFIG_HAS_DMA */
+
 /**
  * snd_sgbuf_get_addr - return the physical address at the corresponding offset
  * @dmab: buffer allocation information
@@ -468,6 +495,71 @@ static const struct snd_malloc_ops snd_dma_wc_ops = {
 	.mmap = snd_dma_wc_mmap,
 };
 #endif /* CONFIG_X86 */
+
+/*
+ * Non-contiguous pages allocator
+ */
+static void *snd_dma_noncontig_alloc(struct snd_dma_buffer *dmab, size_t size)
+{
+	struct sg_table *sgt;
+	void *p;
+
+	sgt = dma_alloc_noncontiguous(dmab->dev.dev, size, dmab->dev.dir,
+				      DEFAULT_GFP, 0);
+	if (!sgt)
+		return NULL;
+	dmab->dev.need_sync = dma_need_sync(dmab->dev.dev, dmab->dev.dir);
+	p = dma_vmap_noncontiguous(dmab->dev.dev, size, sgt);
+	if (p)
+		dmab->private_data = sgt;
+	else
+		dma_free_noncontiguous(dmab->dev.dev, size, sgt, dmab->dev.dir);
+	return p;
+}
+
+static void snd_dma_noncontig_free(struct snd_dma_buffer *dmab)
+{
+	dma_vunmap_noncontiguous(dmab->dev.dev, dmab->area);
+	dma_free_noncontiguous(dmab->dev.dev, dmab->bytes, dmab->private_data,
+			       dmab->dev.dir);
+}
+
+static int snd_dma_noncontig_mmap(struct snd_dma_buffer *dmab,
+				  struct vm_area_struct *area)
+{
+	return dma_mmap_noncontiguous(dmab->dev.dev, area,
+				      dmab->bytes, dmab->private_data);
+}
+
+static void snd_dma_noncontig_sync(struct snd_dma_buffer *dmab,
+				   enum snd_dma_sync_mode mode)
+{
+	if (mode == SNDRV_DMA_SYNC_CPU) {
+		if (dmab->dev.dir == DMA_TO_DEVICE)
+			return;
+		dma_sync_sgtable_for_cpu(dmab->dev.dev, dmab->private_data,
+					 dmab->dev.dir);
+		invalidate_kernel_vmap_range(dmab->area, dmab->bytes);
+	} else {
+		if (dmab->dev.dir == DMA_FROM_DEVICE)
+			return;
+		flush_kernel_vmap_range(dmab->area, dmab->bytes);
+		dma_sync_sgtable_for_device(dmab->dev.dev, dmab->private_data,
+					    dmab->dev.dir);
+	}
+}
+
+static const struct snd_malloc_ops snd_dma_noncontig_ops = {
+	.alloc = snd_dma_noncontig_alloc,
+	.free = snd_dma_noncontig_free,
+	.mmap = snd_dma_noncontig_mmap,
+	.sync = snd_dma_noncontig_sync,
+	/* re-use vmalloc helpers for get_* ops */
+	.get_addr = snd_dma_vmalloc_get_addr,
+	.get_page = snd_dma_vmalloc_get_page,
+	.get_chunk_size = snd_dma_vmalloc_get_chunk_size,
+};
+
 #endif /* CONFIG_HAS_DMA */
 
 /*
@@ -479,6 +571,7 @@ static const struct snd_malloc_ops *dma_ops[] = {
 #ifdef CONFIG_HAS_DMA
 	[SNDRV_DMA_TYPE_DEV] = &snd_dma_dev_ops,
 	[SNDRV_DMA_TYPE_DEV_WC] = &snd_dma_wc_ops,
+	[SNDRV_DMA_TYPE_NONCONTIG] = &snd_dma_noncontig_ops,
 #ifdef CONFIG_GENERIC_ALLOCATOR
 	[SNDRV_DMA_TYPE_DEV_IRAM] = &snd_dma_iram_ops,
 #endif /* CONFIG_GENERIC_ALLOCATOR */
diff --git a/sound/core/memalloc_local.h b/sound/core/memalloc_local.h
index 9f2e0a608b49..a6f3a87194da 100644
--- a/sound/core/memalloc_local.h
+++ b/sound/core/memalloc_local.h
@@ -10,6 +10,7 @@ struct snd_malloc_ops {
 	unsigned int (*get_chunk_size)(struct snd_dma_buffer *dmab,
 				       unsigned int ofs, unsigned int size);
 	int (*mmap)(struct snd_dma_buffer *dmab, struct vm_area_struct *area);
+	void (*sync)(struct snd_dma_buffer *dmab, enum snd_dma_sync_mode mode);
 };
 
 #ifdef CONFIG_SND_DMA_SGBUF
diff --git a/sound/core/pcm_compat.c b/sound/core/pcm_compat.c
index dfe5a64e19d2..e4e176854ce7 100644
--- a/sound/core/pcm_compat.c
+++ b/sound/core/pcm_compat.c
@@ -453,6 +453,8 @@ static int snd_pcm_ioctl_sync_ptr_x32(struct snd_pcm_substream *substream,
 	sstatus.suspended_state = status->suspended_state;
 	sstatus.audio_tstamp = status->audio_tstamp;
 	snd_pcm_stream_unlock_irq(substream);
+	if (!(sflags & SNDRV_PCM_SYNC_PTR_APPL))
+		snd_pcm_dma_buffer_sync(substream, SNDRV_DMA_SYNC_DEVICE);
 	if (put_user(sstatus.state, &src->s.status.state) ||
 	    put_user(sstatus.hw_ptr, &src->s.status.hw_ptr) ||
 	    put_user(sstatus.tstamp.tv_sec, &src->s.status.tstamp_sec) ||
@@ -533,6 +535,8 @@ static int snd_pcm_ioctl_sync_ptr_buggy(struct snd_pcm_substream *substream,
 	sync_ptr.s.status.suspended_state = status->suspended_state;
 	sync_ptr.s.status.audio_tstamp = status->audio_tstamp;
 	snd_pcm_stream_unlock_irq(substream);
+	if (!(sync_ptr.flags & SNDRV_PCM_SYNC_PTR_APPL))
+		snd_pcm_dma_buffer_sync(substream, SNDRV_DMA_SYNC_DEVICE);
 	if (copy_to_user(_sync_ptr, &sync_ptr, sizeof(sync_ptr)))
 		return -EFAULT;
 	return 0;
diff --git a/sound/core/pcm_lib.c b/sound/core/pcm_lib.c
index a144a3f68e9e..4f4b4739f987 100644
--- a/sound/core/pcm_lib.c
+++ b/sound/core/pcm_lib.c
@@ -106,6 +106,7 @@ void snd_pcm_playback_silence(struct snd_pcm_substream *substream, snd_pcm_ufram
 		frames -= transfer;
 		ofs = 0;
 	}
+	snd_pcm_dma_buffer_sync(substream, SNDRV_DMA_SYNC_DEVICE);
 }
 
 #ifdef CONFIG_SND_DEBUG
@@ -2256,8 +2257,12 @@ snd_pcm_sframes_t __snd_pcm_lib_xfer(struct snd_pcm_substream *substream,
 			goto _end_unlock;
 		}
 		snd_pcm_stream_unlock_irq(substream);
+		if (!is_playback)
+			snd_pcm_dma_buffer_sync(substream, SNDRV_DMA_SYNC_CPU);
 		err = writer(substream, appl_ofs, data, offset, frames,
 			     transfer);
+		if (is_playback)
+			snd_pcm_dma_buffer_sync(substream, SNDRV_DMA_SYNC_DEVICE);
 		snd_pcm_stream_lock_irq(substream);
 		if (err < 0)
 			goto _end_unlock;
diff --git a/sound/core/pcm_local.h b/sound/core/pcm_local.h
index fe9689b8a6a6..ecb21697ae3a 100644
--- a/sound/core/pcm_local.h
+++ b/sound/core/pcm_local.h
@@ -73,4 +73,11 @@ void snd_pcm_sync_stop(struct snd_pcm_substream *substream, bool sync_irq);
 		for ((subs) = (pcm)->streams[str].substream; (subs); \
 		     (subs) = (subs)->next)
 
+static inline void snd_pcm_dma_buffer_sync(struct snd_pcm_substream *substream,
+					   enum snd_dma_sync_mode mode)
+{
+	if (substream->runtime->info & SNDRV_PCM_INFO_EXPLICIT_SYNC)
+		snd_dma_buffer_sync(snd_pcm_get_dma_buf(substream), mode);
+}
+
 #endif	/* __SOUND_CORE_PCM_LOCAL_H */
diff --git a/sound/core/pcm_memory.c b/sound/core/pcm_memory.c
index 7fbd1ccbb5b0..b70ce3b69ab4 100644
--- a/sound/core/pcm_memory.c
+++ b/sound/core/pcm_memory.c
@@ -32,15 +32,20 @@ module_param(max_alloc_per_card, ulong, 0644);
 MODULE_PARM_DESC(max_alloc_per_card, "Max total allocation bytes per card.");
 
 static int do_alloc_pages(struct snd_card *card, int type, struct device *dev,
-			  size_t size, struct snd_dma_buffer *dmab)
+			  int str, size_t size, struct snd_dma_buffer *dmab)
 {
+	enum dma_data_direction dir;
 	int err;
 
 	if (max_alloc_per_card &&
 	    card->total_pcm_alloc_bytes + size > max_alloc_per_card)
 		return -ENOMEM;
 
-	err = snd_dma_alloc_pages(type, dev, size, dmab);
+	if (str == SNDRV_PCM_STREAM_PLAYBACK)
+		dir = DMA_TO_DEVICE;
+	else
+		dir = DMA_FROM_DEVICE;
+	err = snd_dma_alloc_dir_pages(type, dev, dir, size, dmab);
 	if (!err) {
 		mutex_lock(&card->memory_mutex);
 		card->total_pcm_alloc_bytes += dmab->bytes;
@@ -77,7 +82,7 @@ static int preallocate_pcm_pages(struct snd_pcm_substream *substream,
 
 	do {
 		err = do_alloc_pages(card, dmab->dev.type, dmab->dev.dev,
-				     size, dmab);
+				     substream->stream, size, dmab);
 		if (err != -ENOMEM)
 			return err;
 		if (no_fallback)
@@ -177,6 +182,7 @@ static void snd_pcm_lib_preallocate_proc_write(struct snd_info_entry *entry,
 			if (do_alloc_pages(card,
 					   substream->dma_buffer.dev.type,
 					   substream->dma_buffer.dev.dev,
+					   substream->stream,
 					   size, &new_dmab) < 0) {
 				buffer->error = -ENOMEM;
 				pr_debug("ALSA pcmC%dD%d%c,%d:%s: cannot preallocate for size %zu\n",
@@ -418,6 +424,7 @@ int snd_pcm_lib_malloc_pages(struct snd_pcm_substream *substream, size_t size)
 		if (do_alloc_pages(card,
 				   substream->dma_buffer.dev.type,
 				   substream->dma_buffer.dev.dev,
+				   substream->stream,
 				   size, dmab) < 0) {
 			kfree(dmab);
 			pr_debug("ALSA pcmC%dD%d%c,%d:%s: cannot preallocate for size %zu\n",
diff --git a/sound/core/pcm_native.c b/sound/core/pcm_native.c
index 627b201b6084..621883e71194 100644
--- a/sound/core/pcm_native.c
+++ b/sound/core/pcm_native.c
@@ -2685,6 +2685,13 @@ int snd_pcm_open_substream(struct snd_pcm *pcm, int stream,
 		goto error;
 	}
 
+	/* automatically set EXPLICIT_SYNC flag in the managed mode whenever
+	 * the DMA buffer requires it
+	 */
+	if (substream->managed_buffer_alloc &&
+	    substream->dma_buffer.dev.need_sync)
+		substream->runtime->hw.info |= SNDRV_PCM_INFO_EXPLICIT_SYNC;
+
 	*rsubstream = substream;
 	return 0;
 
@@ -2912,6 +2919,8 @@ static snd_pcm_sframes_t snd_pcm_rewind(struct snd_pcm_substream *substream,
 		ret = rewind_appl_ptr(substream, frames,
 				      snd_pcm_hw_avail(substream));
 	snd_pcm_stream_unlock_irq(substream);
+	if (ret >= 0)
+		snd_pcm_dma_buffer_sync(substream, SNDRV_DMA_SYNC_DEVICE);
 	return ret;
 }
 
@@ -2929,6 +2938,8 @@ static snd_pcm_sframes_t snd_pcm_forward(struct snd_pcm_substream *substream,
 		ret = forward_appl_ptr(substream, frames,
 				       snd_pcm_avail(substream));
 	snd_pcm_stream_unlock_irq(substream);
+	if (ret >= 0)
+		snd_pcm_dma_buffer_sync(substream, SNDRV_DMA_SYNC_DEVICE);
 	return ret;
 }
 
@@ -2942,6 +2953,8 @@ static int snd_pcm_delay(struct snd_pcm_substream *substream,
 	if (delay && !err)
 		*delay = snd_pcm_calc_delay(substream);
 	snd_pcm_stream_unlock_irq(substream);
+	snd_pcm_dma_buffer_sync(substream, SNDRV_DMA_SYNC_CPU);
+
 	return err;
 }
 		
@@ -2992,6 +3005,8 @@ static int snd_pcm_sync_ptr(struct snd_pcm_substream *substream,
 	sync_ptr.s.status.suspended_state = status->suspended_state;
 	sync_ptr.s.status.audio_tstamp = status->audio_tstamp;
 	snd_pcm_stream_unlock_irq(substream);
+	if (!(sync_ptr.flags & SNDRV_PCM_SYNC_PTR_APPL))
+		snd_pcm_dma_buffer_sync(substream, SNDRV_DMA_SYNC_DEVICE);
 	if (copy_to_user(_sync_ptr, &sync_ptr, sizeof(sync_ptr)))
 		return -EFAULT;
 	return 0;
@@ -3088,6 +3103,8 @@ static int snd_pcm_ioctl_sync_ptr_compat(struct snd_pcm_substream *substream,
 	sstatus.suspended_state = status->suspended_state;
 	sstatus.audio_tstamp = status->audio_tstamp;
 	snd_pcm_stream_unlock_irq(substream);
+	if (!(sflags & SNDRV_PCM_SYNC_PTR_APPL))
+		snd_pcm_dma_buffer_sync(substream, SNDRV_DMA_SYNC_DEVICE);
 	if (put_user(sstatus.state, &src->s.status.state) ||
 	    put_user(sstatus.hw_ptr, &src->s.status.hw_ptr) ||
 	    put_user(sstatus.tstamp.tv_sec, &src->s.status.tstamp_sec) ||