1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
|
// SPDX-License-Identifier: GPL-2.0
/* Copyright (c) 2019 HiSilicon Limited. */
#include <linux/dma-mapping.h>
#include <linux/module.h>
#include "./sgl.h"
#define HISI_ACC_SGL_SGE_NR_MIN 1
#define HISI_ACC_SGL_SGE_NR_MAX 255
#define HISI_ACC_SGL_SGE_NR_DEF 10
#define HISI_ACC_SGL_NR_MAX 256
#define HISI_ACC_SGL_ALIGN_SIZE 64
static int acc_sgl_sge_set(const char *val, const struct kernel_param *kp)
{
int ret;
u32 n;
if (!val)
return -EINVAL;
ret = kstrtou32(val, 10, &n);
if (ret != 0 || n > HISI_ACC_SGL_SGE_NR_MAX || n == 0)
return -EINVAL;
return param_set_int(val, kp);
}
static const struct kernel_param_ops acc_sgl_sge_ops = {
.set = acc_sgl_sge_set,
.get = param_get_int,
};
static u32 acc_sgl_sge_nr = HISI_ACC_SGL_SGE_NR_DEF;
module_param_cb(acc_sgl_sge_nr, &acc_sgl_sge_ops, &acc_sgl_sge_nr, 0444);
MODULE_PARM_DESC(acc_sgl_sge_nr, "Number of sge in sgl(1-255)");
struct acc_hw_sge {
dma_addr_t buf;
void *page_ctrl;
__le32 len;
__le32 pad;
__le32 pad0;
__le32 pad1;
};
/* use default sgl head size 64B */
struct hisi_acc_hw_sgl {
dma_addr_t next_dma;
__le16 entry_sum_in_chain;
__le16 entry_sum_in_sgl;
__le16 entry_length_in_sgl;
__le16 pad0;
__le64 pad1[5];
struct hisi_acc_hw_sgl *next;
struct acc_hw_sge sge_entries[];
} __aligned(1);
/**
* hisi_acc_create_sgl_pool() - Create a hw sgl pool.
* @dev: The device which hw sgl pool belongs to.
* @pool: Pointer of pool.
* @count: Count of hisi_acc_hw_sgl in pool.
*
* This function creates a hw sgl pool, after this user can get hw sgl memory
* from it.
*/
int hisi_acc_create_sgl_pool(struct device *dev,
struct hisi_acc_sgl_pool *pool, u32 count)
{
u32 sgl_size;
u32 size;
if (!dev || !pool || !count)
return -EINVAL;
sgl_size = sizeof(struct acc_hw_sge) * acc_sgl_sge_nr +
sizeof(struct hisi_acc_hw_sgl);
size = sgl_size * count;
pool->sgl = dma_alloc_coherent(dev, size, &pool->sgl_dma, GFP_KERNEL);
if (!pool->sgl)
return -ENOMEM;
pool->size = size;
pool->count = count;
pool->sgl_size = sgl_size;
return 0;
}
EXPORT_SYMBOL_GPL(hisi_acc_create_sgl_pool);
/**
* hisi_acc_free_sgl_pool() - Free a hw sgl pool.
* @dev: The device which hw sgl pool belongs to.
* @pool: Pointer of pool.
*
* This function frees memory of a hw sgl pool.
*/
void hisi_acc_free_sgl_pool(struct device *dev, struct hisi_acc_sgl_pool *pool)
{
dma_free_coherent(dev, pool->size, pool->sgl, pool->sgl_dma);
memset(pool, 0, sizeof(struct hisi_acc_sgl_pool));
}
EXPORT_SYMBOL_GPL(hisi_acc_free_sgl_pool);
struct hisi_acc_hw_sgl *acc_get_sgl(struct hisi_acc_sgl_pool *pool, u32 index,
dma_addr_t *hw_sgl_dma)
{
if (!pool || !hw_sgl_dma || index >= pool->count || !pool->sgl)
return ERR_PTR(-EINVAL);
*hw_sgl_dma = pool->sgl_dma + pool->sgl_size * index;
return (void *)pool->sgl + pool->sgl_size * index;
}
void acc_put_sgl(struct hisi_acc_sgl_pool *pool, u32 index) {}
static void sg_map_to_hw_sg(struct scatterlist *sgl,
struct acc_hw_sge *hw_sge)
{
hw_sge->buf = sgl->dma_address;
hw_sge->len = sgl->dma_length;
}
static void inc_hw_sgl_sge(struct hisi_acc_hw_sgl *hw_sgl)
{
hw_sgl->entry_sum_in_sgl++;
}
static void update_hw_sgl_sum_sge(struct hisi_acc_hw_sgl *hw_sgl, u16 sum)
{
hw_sgl->entry_sum_in_chain = sum;
}
/**
* hisi_acc_sg_buf_map_to_hw_sgl - Map a scatterlist to a hw sgl.
* @dev: The device which hw sgl belongs to.
* @sgl: Scatterlist which will be mapped to hw sgl.
* @pool: Pool which hw sgl memory will be allocated in.
* @index: Index of hisi_acc_hw_sgl in pool.
* @hw_sgl_dma: The dma address of allocated hw sgl.
*
* This function builds hw sgl according input sgl, user can use hw_sgl_dma
* as src/dst in its BD. Only support single hw sgl currently.
*/
struct hisi_acc_hw_sgl *
hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev,
struct scatterlist *sgl,
struct hisi_acc_sgl_pool *pool,
u32 index, dma_addr_t *hw_sgl_dma)
{
struct hisi_acc_hw_sgl *curr_hw_sgl;
dma_addr_t curr_sgl_dma = 0;
struct acc_hw_sge *curr_hw_sge;
struct scatterlist *sg;
int sg_n = sg_nents(sgl);
int i, ret;
if (!dev || !sgl || !pool || !hw_sgl_dma || sg_n > acc_sgl_sge_nr)
return ERR_PTR(-EINVAL);
ret = dma_map_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL);
if (!ret)
return ERR_PTR(-EINVAL);
curr_hw_sgl = acc_get_sgl(pool, index, &curr_sgl_dma);
if (!curr_hw_sgl) {
ret = -ENOMEM;
goto err_unmap_sg;
}
curr_hw_sgl->entry_length_in_sgl = acc_sgl_sge_nr;
curr_hw_sge = curr_hw_sgl->sge_entries;
for_each_sg(sgl, sg, sg_n, i) {
sg_map_to_hw_sg(sg, curr_hw_sge);
inc_hw_sgl_sge(curr_hw_sgl);
curr_hw_sge++;
}
update_hw_sgl_sum_sge(curr_hw_sgl, acc_sgl_sge_nr);
*hw_sgl_dma = curr_sgl_dma;
return curr_hw_sgl;
err_unmap_sg:
dma_unmap_sg(dev, sgl, sg_n, DMA_BIDIRECTIONAL);
return ERR_PTR(ret);
}
EXPORT_SYMBOL_GPL(hisi_acc_sg_buf_map_to_hw_sgl);
/**
* hisi_acc_sg_buf_unmap() - Unmap allocated hw sgl.
* @dev: The device which hw sgl belongs to.
* @sgl: Related scatterlist.
* @hw_sgl: Virtual address of hw sgl.
* @hw_sgl_dma: DMA address of hw sgl.
* @pool: Pool which hw sgl is allocated in.
*
* This function unmaps allocated hw sgl.
*/
void hisi_acc_sg_buf_unmap(struct device *dev, struct scatterlist *sgl,
struct hisi_acc_hw_sgl *hw_sgl)
{
dma_unmap_sg(dev, sgl, sg_nents(sgl), DMA_BIDIRECTIONAL);
hw_sgl->entry_sum_in_chain = 0;
hw_sgl->entry_sum_in_sgl = 0;
hw_sgl->entry_length_in_sgl = 0;
}
EXPORT_SYMBOL_GPL(hisi_acc_sg_buf_unmap);
MODULE_LICENSE("GPL v2");
MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
MODULE_DESCRIPTION("HiSilicon Accelerator SGL support");
|
