1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (c) 2020 Synopsys, Inc. and/or its affiliates. |
4 | * Synopsys DesignWare xData driver |
5 | * |
6 | * Author: Gustavo Pimentel <gustavo.pimentel@synopsys.com> |
7 | */ |
8 | |
9 | #include <linux/miscdevice.h> |
10 | #include <linux/bitfield.h> |
11 | #include <linux/pci-epf.h> |
12 | #include <linux/kernel.h> |
13 | #include <linux/module.h> |
14 | #include <linux/device.h> |
15 | #include <linux/bitops.h> |
16 | #include <linux/mutex.h> |
17 | #include <linux/delay.h> |
18 | #include <linux/pci.h> |
19 | |
20 | #define DW_XDATA_DRIVER_NAME "dw-xdata-pcie" |
21 | |
22 | #define DW_XDATA_EP_MEM_OFFSET 0x8000000 |
23 | |
24 | static DEFINE_IDA(xdata_ida); |
25 | |
26 | #define STATUS_DONE BIT(0) |
27 | |
28 | #define CONTROL_DOORBELL BIT(0) |
29 | #define CONTROL_IS_WRITE BIT(1) |
30 | #define CONTROL_LENGTH(a) FIELD_PREP(GENMASK(13, 2), a) |
31 | #define CONTROL_PATTERN_INC BIT(16) |
32 | #define CONTROL_NO_ADDR_INC BIT(18) |
33 | |
34 | #define XPERF_CONTROL_ENABLE BIT(5) |
35 | |
36 | #define BURST_REPEAT BIT(31) |
37 | #define BURST_VALUE 0x1001 |
38 | |
39 | #define PATTERN_VALUE 0x0 |
40 | |
41 | struct dw_xdata_regs { |
42 | u32 addr_lsb; /* 0x000 */ |
43 | u32 addr_msb; /* 0x004 */ |
44 | u32 burst_cnt; /* 0x008 */ |
45 | u32 control; /* 0x00c */ |
46 | u32 pattern; /* 0x010 */ |
47 | u32 status; /* 0x014 */ |
48 | u32 RAM_addr; /* 0x018 */ |
49 | u32 RAM_port; /* 0x01c */ |
50 | u32 _reserved0[14]; /* 0x020..0x054 */ |
51 | u32 perf_control; /* 0x058 */ |
52 | u32 _reserved1[41]; /* 0x05c..0x0fc */ |
53 | u32 wr_cnt_lsb; /* 0x100 */ |
54 | u32 wr_cnt_msb; /* 0x104 */ |
55 | u32 rd_cnt_lsb; /* 0x108 */ |
56 | u32 rd_cnt_msb; /* 0x10c */ |
57 | } __packed; |
58 | |
59 | struct dw_xdata_region { |
60 | phys_addr_t paddr; /* physical address */ |
61 | void __iomem *vaddr; /* virtual address */ |
62 | }; |
63 | |
64 | struct dw_xdata { |
65 | struct dw_xdata_region rg_region; /* registers */ |
66 | size_t max_wr_len; /* max wr xfer len */ |
67 | size_t max_rd_len; /* max rd xfer len */ |
68 | struct mutex mutex; |
69 | struct pci_dev *pdev; |
70 | struct miscdevice misc_dev; |
71 | }; |
72 | |
73 | static inline struct dw_xdata_regs __iomem *__dw_regs(struct dw_xdata *dw) |
74 | { |
75 | return dw->rg_region.vaddr; |
76 | } |
77 | |
78 | static void dw_xdata_stop(struct dw_xdata *dw) |
79 | { |
80 | u32 burst; |
81 | |
82 | mutex_lock(&dw->mutex); |
83 | |
84 | burst = readl(addr: &(__dw_regs(dw)->burst_cnt)); |
85 | |
86 | if (burst & BURST_REPEAT) { |
87 | burst &= ~(u32)BURST_REPEAT; |
88 | writel(val: burst, addr: &(__dw_regs(dw)->burst_cnt)); |
89 | } |
90 | |
91 | mutex_unlock(lock: &dw->mutex); |
92 | } |
93 | |
94 | static void dw_xdata_start(struct dw_xdata *dw, bool write) |
95 | { |
96 | struct device *dev = &dw->pdev->dev; |
97 | u32 control, status; |
98 | |
99 | /* Stop first if xfer in progress */ |
100 | dw_xdata_stop(dw); |
101 | |
102 | mutex_lock(&dw->mutex); |
103 | |
104 | /* Clear status register */ |
105 | writel(val: 0x0, addr: &(__dw_regs(dw)->status)); |
106 | |
107 | /* Burst count register set for continuous until stopped */ |
108 | writel(BURST_REPEAT | BURST_VALUE, addr: &(__dw_regs(dw)->burst_cnt)); |
109 | |
110 | /* Pattern register */ |
111 | writel(PATTERN_VALUE, addr: &(__dw_regs(dw)->pattern)); |
112 | |
113 | /* Control register */ |
114 | control = CONTROL_DOORBELL | CONTROL_PATTERN_INC | CONTROL_NO_ADDR_INC; |
115 | if (write) { |
116 | control |= CONTROL_IS_WRITE; |
117 | control |= CONTROL_LENGTH(dw->max_wr_len); |
118 | } else { |
119 | control |= CONTROL_LENGTH(dw->max_rd_len); |
120 | } |
121 | writel(val: control, addr: &(__dw_regs(dw)->control)); |
122 | |
123 | /* |
124 | * The xData HW block needs about 100 ms to initiate the traffic |
125 | * generation according this HW block datasheet. |
126 | */ |
127 | usleep_range(min: 100, max: 150); |
128 | |
129 | status = readl(addr: &(__dw_regs(dw)->status)); |
130 | |
131 | mutex_unlock(lock: &dw->mutex); |
132 | |
133 | if (!(status & STATUS_DONE)) |
134 | dev_dbg(dev, "xData: started %s direction\n" , |
135 | write ? "write" : "read" ); |
136 | } |
137 | |
138 | static void dw_xdata_perf_meas(struct dw_xdata *dw, u64 *data, bool write) |
139 | { |
140 | if (write) { |
141 | *data = readl(addr: &(__dw_regs(dw)->wr_cnt_msb)); |
142 | *data <<= 32; |
143 | *data |= readl(addr: &(__dw_regs(dw)->wr_cnt_lsb)); |
144 | } else { |
145 | *data = readl(addr: &(__dw_regs(dw)->rd_cnt_msb)); |
146 | *data <<= 32; |
147 | *data |= readl(addr: &(__dw_regs(dw)->rd_cnt_lsb)); |
148 | } |
149 | } |
150 | |
151 | static u64 dw_xdata_perf_diff(u64 *m1, u64 *m2, u64 time) |
152 | { |
153 | u64 rate = (*m1 - *m2); |
154 | |
155 | rate *= (1000 * 1000 * 1000); |
156 | rate >>= 20; |
157 | rate = DIV_ROUND_CLOSEST_ULL(rate, time); |
158 | |
159 | return rate; |
160 | } |
161 | |
162 | static void dw_xdata_perf(struct dw_xdata *dw, u64 *rate, bool write) |
163 | { |
164 | struct device *dev = &dw->pdev->dev; |
165 | u64 data[2], time[2], diff; |
166 | |
167 | mutex_lock(&dw->mutex); |
168 | |
169 | /* First acquisition of current count frames */ |
170 | writel(val: 0x0, addr: &(__dw_regs(dw)->perf_control)); |
171 | dw_xdata_perf_meas(dw, data: &data[0], write); |
172 | time[0] = jiffies; |
173 | writel(val: (u32)XPERF_CONTROL_ENABLE, addr: &(__dw_regs(dw)->perf_control)); |
174 | |
175 | /* |
176 | * Wait 100ms between the 1st count frame acquisition and the 2nd |
177 | * count frame acquisition, in order to calculate the speed later |
178 | */ |
179 | mdelay(100); |
180 | |
181 | /* Second acquisition of current count frames */ |
182 | writel(val: 0x0, addr: &(__dw_regs(dw)->perf_control)); |
183 | dw_xdata_perf_meas(dw, data: &data[1], write); |
184 | time[1] = jiffies; |
185 | writel(val: (u32)XPERF_CONTROL_ENABLE, addr: &(__dw_regs(dw)->perf_control)); |
186 | |
187 | /* |
188 | * Speed calculation |
189 | * |
190 | * rate = (2nd count frames - 1st count frames) / (time elapsed) |
191 | */ |
192 | diff = jiffies_to_nsecs(j: time[1] - time[0]); |
193 | *rate = dw_xdata_perf_diff(m1: &data[1], m2: &data[0], time: diff); |
194 | |
195 | mutex_unlock(lock: &dw->mutex); |
196 | |
197 | dev_dbg(dev, "xData: time=%llu us, %s=%llu MB/s\n" , |
198 | diff, write ? "write" : "read" , *rate); |
199 | } |
200 | |
201 | static struct dw_xdata *misc_dev_to_dw(struct miscdevice *misc_dev) |
202 | { |
203 | return container_of(misc_dev, struct dw_xdata, misc_dev); |
204 | } |
205 | |
206 | static ssize_t write_show(struct device *dev, struct device_attribute *attr, |
207 | char *buf) |
208 | { |
209 | struct miscdevice *misc_dev = dev_get_drvdata(dev); |
210 | struct dw_xdata *dw = misc_dev_to_dw(misc_dev); |
211 | u64 rate; |
212 | |
213 | dw_xdata_perf(dw, rate: &rate, write: true); |
214 | |
215 | return sysfs_emit(buf, fmt: "%llu\n" , rate); |
216 | } |
217 | |
218 | static ssize_t write_store(struct device *dev, struct device_attribute *attr, |
219 | const char *buf, size_t size) |
220 | { |
221 | struct miscdevice *misc_dev = dev_get_drvdata(dev); |
222 | struct dw_xdata *dw = misc_dev_to_dw(misc_dev); |
223 | bool enabled; |
224 | int ret; |
225 | |
226 | ret = kstrtobool(s: buf, res: &enabled); |
227 | if (ret < 0) |
228 | return ret; |
229 | |
230 | if (enabled) { |
231 | dev_dbg(dev, "xData: requested write transfer\n" ); |
232 | dw_xdata_start(dw, write: true); |
233 | } else { |
234 | dev_dbg(dev, "xData: requested stop transfer\n" ); |
235 | dw_xdata_stop(dw); |
236 | } |
237 | |
238 | return size; |
239 | } |
240 | |
241 | static DEVICE_ATTR_RW(write); |
242 | |
243 | static ssize_t read_show(struct device *dev, struct device_attribute *attr, |
244 | char *buf) |
245 | { |
246 | struct miscdevice *misc_dev = dev_get_drvdata(dev); |
247 | struct dw_xdata *dw = misc_dev_to_dw(misc_dev); |
248 | u64 rate; |
249 | |
250 | dw_xdata_perf(dw, rate: &rate, write: false); |
251 | |
252 | return sysfs_emit(buf, fmt: "%llu\n" , rate); |
253 | } |
254 | |
255 | static ssize_t read_store(struct device *dev, struct device_attribute *attr, |
256 | const char *buf, size_t size) |
257 | { |
258 | struct miscdevice *misc_dev = dev_get_drvdata(dev); |
259 | struct dw_xdata *dw = misc_dev_to_dw(misc_dev); |
260 | bool enabled; |
261 | int ret; |
262 | |
263 | ret = kstrtobool(s: buf, res: &enabled); |
264 | if (ret < 0) |
265 | return ret; |
266 | |
267 | if (enabled) { |
268 | dev_dbg(dev, "xData: requested read transfer\n" ); |
269 | dw_xdata_start(dw, write: false); |
270 | } else { |
271 | dev_dbg(dev, "xData: requested stop transfer\n" ); |
272 | dw_xdata_stop(dw); |
273 | } |
274 | |
275 | return size; |
276 | } |
277 | |
278 | static DEVICE_ATTR_RW(read); |
279 | |
280 | static struct attribute *xdata_attrs[] = { |
281 | &dev_attr_write.attr, |
282 | &dev_attr_read.attr, |
283 | NULL, |
284 | }; |
285 | |
286 | ATTRIBUTE_GROUPS(xdata); |
287 | |
288 | static int dw_xdata_pcie_probe(struct pci_dev *pdev, |
289 | const struct pci_device_id *pid) |
290 | { |
291 | struct device *dev = &pdev->dev; |
292 | struct dw_xdata *dw; |
293 | char name[24]; |
294 | u64 addr; |
295 | int err; |
296 | int id; |
297 | |
298 | /* Enable PCI device */ |
299 | err = pcim_enable_device(pdev); |
300 | if (err) { |
301 | dev_err(dev, "enabling device failed\n" ); |
302 | return err; |
303 | } |
304 | |
305 | /* Mapping PCI BAR regions */ |
306 | err = pcim_iomap_regions(pdev, BIT(BAR_0), name: pci_name(pdev)); |
307 | if (err) { |
308 | dev_err(dev, "xData BAR I/O remapping failed\n" ); |
309 | return err; |
310 | } |
311 | |
312 | pci_set_master(dev: pdev); |
313 | |
314 | /* Allocate memory */ |
315 | dw = devm_kzalloc(dev, size: sizeof(*dw), GFP_KERNEL); |
316 | if (!dw) |
317 | return -ENOMEM; |
318 | |
319 | /* Data structure initialization */ |
320 | mutex_init(&dw->mutex); |
321 | |
322 | dw->rg_region.vaddr = pcim_iomap_table(pdev)[BAR_0]; |
323 | if (!dw->rg_region.vaddr) |
324 | return -ENOMEM; |
325 | |
326 | dw->rg_region.paddr = pdev->resource[BAR_0].start; |
327 | |
328 | dw->max_wr_len = pcie_get_mps(dev: pdev); |
329 | dw->max_wr_len >>= 2; |
330 | |
331 | dw->max_rd_len = pcie_get_readrq(dev: pdev); |
332 | dw->max_rd_len >>= 2; |
333 | |
334 | dw->pdev = pdev; |
335 | |
336 | id = ida_alloc(ida: &xdata_ida, GFP_KERNEL); |
337 | if (id < 0) { |
338 | dev_err(dev, "xData: unable to get id\n" ); |
339 | return id; |
340 | } |
341 | |
342 | snprintf(buf: name, size: sizeof(name), DW_XDATA_DRIVER_NAME ".%d" , id); |
343 | dw->misc_dev.name = kstrdup(s: name, GFP_KERNEL); |
344 | if (!dw->misc_dev.name) { |
345 | err = -ENOMEM; |
346 | goto err_ida_remove; |
347 | } |
348 | |
349 | dw->misc_dev.minor = MISC_DYNAMIC_MINOR; |
350 | dw->misc_dev.parent = dev; |
351 | dw->misc_dev.groups = xdata_groups; |
352 | |
353 | writel(val: 0x0, addr: &(__dw_regs(dw)->RAM_addr)); |
354 | writel(val: 0x0, addr: &(__dw_regs(dw)->RAM_port)); |
355 | |
356 | addr = dw->rg_region.paddr + DW_XDATA_EP_MEM_OFFSET; |
357 | writel(lower_32_bits(addr), addr: &(__dw_regs(dw)->addr_lsb)); |
358 | writel(upper_32_bits(addr), addr: &(__dw_regs(dw)->addr_msb)); |
359 | dev_dbg(dev, "xData: target address = 0x%.16llx\n" , addr); |
360 | |
361 | dev_dbg(dev, "xData: wr_len = %zu, rd_len = %zu\n" , |
362 | dw->max_wr_len * 4, dw->max_rd_len * 4); |
363 | |
364 | /* Saving data structure reference */ |
365 | pci_set_drvdata(pdev, data: dw); |
366 | |
367 | /* Register misc device */ |
368 | err = misc_register(misc: &dw->misc_dev); |
369 | if (err) { |
370 | dev_err(dev, "xData: failed to register device\n" ); |
371 | goto err_kfree_name; |
372 | } |
373 | |
374 | return 0; |
375 | |
376 | err_kfree_name: |
377 | kfree(objp: dw->misc_dev.name); |
378 | |
379 | err_ida_remove: |
380 | ida_free(&xdata_ida, id); |
381 | |
382 | return err; |
383 | } |
384 | |
385 | static void dw_xdata_pcie_remove(struct pci_dev *pdev) |
386 | { |
387 | struct dw_xdata *dw = pci_get_drvdata(pdev); |
388 | int id; |
389 | |
390 | if (sscanf(dw->misc_dev.name, DW_XDATA_DRIVER_NAME ".%d" , &id) != 1) |
391 | return; |
392 | |
393 | if (id < 0) |
394 | return; |
395 | |
396 | dw_xdata_stop(dw); |
397 | misc_deregister(misc: &dw->misc_dev); |
398 | kfree(objp: dw->misc_dev.name); |
399 | ida_free(&xdata_ida, id); |
400 | } |
401 | |
402 | static const struct pci_device_id dw_xdata_pcie_id_table[] = { |
403 | { PCI_DEVICE_DATA(SYNOPSYS, EDDA, NULL) }, |
404 | { } |
405 | }; |
406 | MODULE_DEVICE_TABLE(pci, dw_xdata_pcie_id_table); |
407 | |
408 | static struct pci_driver dw_xdata_pcie_driver = { |
409 | .name = DW_XDATA_DRIVER_NAME, |
410 | .id_table = dw_xdata_pcie_id_table, |
411 | .probe = dw_xdata_pcie_probe, |
412 | .remove = dw_xdata_pcie_remove, |
413 | }; |
414 | |
415 | module_pci_driver(dw_xdata_pcie_driver); |
416 | |
417 | MODULE_LICENSE("GPL v2" ); |
418 | MODULE_DESCRIPTION("Synopsys DesignWare xData PCIe driver" ); |
419 | MODULE_AUTHOR("Gustavo Pimentel <gustavo.pimentel@synopsys.com>" ); |
420 | |
421 | |