1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
|
// SPDX-License-Identifier: GPL-2.0-only
/*
* RISC-V specific functions to support DMA for non-coherent devices
*
* Copyright (c) 2021 Western Digital Corporation or its affiliates.
*/
#include <linux/dma-direct.h>
#include <linux/dma-map-ops.h>
#include <linux/mm.h>
#include <asm/cacheflush.h>
#include <asm/dma-noncoherent.h>
static bool noncoherent_supported __ro_after_init;
int dma_cache_alignment __ro_after_init = ARCH_DMA_MINALIGN;
EXPORT_SYMBOL_GPL(dma_cache_alignment);
struct riscv_nonstd_cache_ops noncoherent_cache_ops __ro_after_init = {
.wback = NULL,
.inv = NULL,
.wback_inv = NULL,
};
static inline void arch_dma_cache_wback(phys_addr_t paddr, size_t size)
{
void *vaddr = phys_to_virt(paddr);
#ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS
if (unlikely(noncoherent_cache_ops.wback)) {
noncoherent_cache_ops.wback(paddr, size);
return;
}
#endif
ALT_CMO_OP(CLEAN, vaddr, size, riscv_cbom_block_size);
}
static inline void arch_dma_cache_inv(phys_addr_t paddr, size_t size)
{
void *vaddr = phys_to_virt(paddr);
#ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS
if (unlikely(noncoherent_cache_ops.inv)) {
noncoherent_cache_ops.inv(paddr, size);
return;
}
#endif
ALT_CMO_OP(INVAL, vaddr, size, riscv_cbom_block_size);
}
static inline void arch_dma_cache_wback_inv(phys_addr_t paddr, size_t size)
{
void *vaddr = phys_to_virt(paddr);
#ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS
if (unlikely(noncoherent_cache_ops.wback_inv)) {
noncoherent_cache_ops.wback_inv(paddr, size);
return;
}
#endif
ALT_CMO_OP(FLUSH, vaddr, size, riscv_cbom_block_size);
}
static inline bool arch_sync_dma_clean_before_fromdevice(void)
{
return true;
}
static inline bool arch_sync_dma_cpu_needs_post_dma_flush(void)
{
return true;
}
void arch_sync_dma_for_device(phys_addr_t paddr, size_t size,
enum dma_data_direction dir)
{
switch (dir) {
case DMA_TO_DEVICE:
arch_dma_cache_wback(paddr, size);
break;
case DMA_FROM_DEVICE:
if (!arch_sync_dma_clean_before_fromdevice()) {
arch_dma_cache_inv(paddr, size);
break;
}
fallthrough;
case DMA_BIDIRECTIONAL:
/* Skip the invalidate here if it's done later */
if (IS_ENABLED(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) &&
arch_sync_dma_cpu_needs_post_dma_flush())
arch_dma_cache_wback(paddr, size);
else
arch_dma_cache_wback_inv(paddr, size);
break;
default:
break;
}
}
void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size,
enum dma_data_direction dir)
{
switch (dir) {
case DMA_TO_DEVICE:
break;
case DMA_FROM_DEVICE:
case DMA_BIDIRECTIONAL:
/* FROM_DEVICE invalidate needed if speculative CPU prefetch only */
if (arch_sync_dma_cpu_needs_post_dma_flush())
arch_dma_cache_inv(paddr, size);
break;
default:
break;
}
}
void arch_dma_prep_coherent(struct page *page, size_t size)
{
void *flush_addr = page_address(page);
#ifdef CONFIG_RISCV_NONSTANDARD_CACHE_OPS
if (unlikely(noncoherent_cache_ops.wback_inv)) {
noncoherent_cache_ops.wback_inv(page_to_phys(page), size);
return;
}
#endif
ALT_CMO_OP(FLUSH, flush_addr, size, riscv_cbom_block_size);
}
void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
const struct iommu_ops *iommu, bool coherent)
{
WARN_TAINT(!coherent && riscv_cbom_block_size > ARCH_DMA_MINALIGN,
TAINT_CPU_OUT_OF_SPEC,
"%s %s: ARCH_DMA_MINALIGN smaller than riscv,cbom-block-size (%d < %d)",
dev_driver_string(dev), dev_name(dev),
ARCH_DMA_MINALIGN, riscv_cbom_block_size);
WARN_TAINT(!coherent && !noncoherent_supported, TAINT_CPU_OUT_OF_SPEC,
"%s %s: device non-coherent but no non-coherent operations supported",
dev_driver_string(dev), dev_name(dev));
dev->dma_coherent = coherent;
}
void riscv_noncoherent_supported(void)
{
WARN(!riscv_cbom_block_size,
"Non-coherent DMA support enabled without a block size\n");
noncoherent_supported = true;
}
void __init riscv_set_dma_cache_alignment(void)
{
if (!noncoherent_supported)
dma_cache_alignment = 1;
}
void riscv_noncoherent_register_cache_ops(const struct riscv_nonstd_cache_ops *ops)
{
if (!ops)
return;
noncoherent_cache_ops = *ops;
}
EXPORT_SYMBOL_GPL(riscv_noncoherent_register_cache_ops);
|
