1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /** |
3 | * Host side endpoint driver to implement Non-Transparent Bridge functionality |
4 | * |
5 | * Copyright (C) 2020 Texas Instruments |
6 | * Author: Kishon Vijay Abraham I <kishon@ti.com> |
7 | */ |
8 | |
9 | #include <linux/delay.h> |
10 | #include <linux/module.h> |
11 | #include <linux/pci.h> |
12 | #include <linux/slab.h> |
13 | #include <linux/ntb.h> |
14 | |
15 | #define NTB_EPF_COMMAND 0x0 |
16 | #define CMD_CONFIGURE_DOORBELL 1 |
17 | #define CMD_TEARDOWN_DOORBELL 2 |
18 | #define CMD_CONFIGURE_MW 3 |
19 | #define CMD_TEARDOWN_MW 4 |
20 | #define CMD_LINK_UP 5 |
21 | #define CMD_LINK_DOWN 6 |
22 | |
23 | #define NTB_EPF_ARGUMENT 0x4 |
24 | #define MSIX_ENABLE BIT(16) |
25 | |
26 | #define NTB_EPF_CMD_STATUS 0x8 |
27 | #define COMMAND_STATUS_OK 1 |
28 | #define COMMAND_STATUS_ERROR 2 |
29 | |
30 | #define NTB_EPF_LINK_STATUS 0x0A |
31 | #define LINK_STATUS_UP BIT(0) |
32 | |
33 | #define NTB_EPF_TOPOLOGY 0x0C |
34 | #define NTB_EPF_LOWER_ADDR 0x10 |
35 | #define NTB_EPF_UPPER_ADDR 0x14 |
36 | #define NTB_EPF_LOWER_SIZE 0x18 |
37 | #define NTB_EPF_UPPER_SIZE 0x1C |
38 | #define NTB_EPF_MW_COUNT 0x20 |
39 | #define NTB_EPF_MW1_OFFSET 0x24 |
40 | #define NTB_EPF_SPAD_OFFSET 0x28 |
41 | #define NTB_EPF_SPAD_COUNT 0x2C |
42 | #define NTB_EPF_DB_ENTRY_SIZE 0x30 |
43 | #define NTB_EPF_DB_DATA(n) (0x34 + (n) * 4) |
44 | #define NTB_EPF_DB_OFFSET(n) (0xB4 + (n) * 4) |
45 | |
46 | #define NTB_EPF_MIN_DB_COUNT 3 |
47 | #define NTB_EPF_MAX_DB_COUNT 31 |
48 | |
49 | #define NTB_EPF_COMMAND_TIMEOUT 1000 /* 1 Sec */ |
50 | |
51 | enum pci_barno { |
52 | BAR_0, |
53 | BAR_1, |
54 | BAR_2, |
55 | BAR_3, |
56 | BAR_4, |
57 | BAR_5, |
58 | }; |
59 | |
60 | struct ntb_epf_dev { |
61 | struct ntb_dev ntb; |
62 | struct device *dev; |
63 | /* Mutex to protect providing commands to NTB EPF */ |
64 | struct mutex cmd_lock; |
65 | |
66 | enum pci_barno ctrl_reg_bar; |
67 | enum pci_barno peer_spad_reg_bar; |
68 | enum pci_barno db_reg_bar; |
69 | enum pci_barno mw_bar; |
70 | |
71 | unsigned int mw_count; |
72 | unsigned int spad_count; |
73 | unsigned int db_count; |
74 | |
75 | void __iomem *ctrl_reg; |
76 | void __iomem *db_reg; |
77 | void __iomem *peer_spad_reg; |
78 | |
79 | unsigned int self_spad; |
80 | unsigned int peer_spad; |
81 | |
82 | int db_val; |
83 | u64 db_valid_mask; |
84 | }; |
85 | |
86 | #define ntb_ndev(__ntb) container_of(__ntb, struct ntb_epf_dev, ntb) |
87 | |
88 | struct ntb_epf_data { |
89 | /* BAR that contains both control region and self spad region */ |
90 | enum pci_barno ctrl_reg_bar; |
91 | /* BAR that contains peer spad region */ |
92 | enum pci_barno peer_spad_reg_bar; |
93 | /* BAR that contains Doorbell region and Memory window '1' */ |
94 | enum pci_barno db_reg_bar; |
95 | /* BAR that contains memory windows*/ |
96 | enum pci_barno mw_bar; |
97 | }; |
98 | |
99 | static int ntb_epf_send_command(struct ntb_epf_dev *ndev, u32 command, |
100 | u32 argument) |
101 | { |
102 | ktime_t timeout; |
103 | bool timedout; |
104 | int ret = 0; |
105 | u32 status; |
106 | |
107 | mutex_lock(&ndev->cmd_lock); |
108 | writel(val: argument, addr: ndev->ctrl_reg + NTB_EPF_ARGUMENT); |
109 | writel(val: command, addr: ndev->ctrl_reg + NTB_EPF_COMMAND); |
110 | |
111 | timeout = ktime_add_ms(kt: ktime_get(), NTB_EPF_COMMAND_TIMEOUT); |
112 | while (1) { |
113 | timedout = ktime_after(cmp1: ktime_get(), cmp2: timeout); |
114 | status = readw(addr: ndev->ctrl_reg + NTB_EPF_CMD_STATUS); |
115 | |
116 | if (status == COMMAND_STATUS_ERROR) { |
117 | ret = -EINVAL; |
118 | break; |
119 | } |
120 | |
121 | if (status == COMMAND_STATUS_OK) |
122 | break; |
123 | |
124 | if (WARN_ON(timedout)) { |
125 | ret = -ETIMEDOUT; |
126 | break; |
127 | } |
128 | |
129 | usleep_range(min: 5, max: 10); |
130 | } |
131 | |
132 | writew(val: 0, addr: ndev->ctrl_reg + NTB_EPF_CMD_STATUS); |
133 | mutex_unlock(lock: &ndev->cmd_lock); |
134 | |
135 | return ret; |
136 | } |
137 | |
138 | static int ntb_epf_mw_to_bar(struct ntb_epf_dev *ndev, int idx) |
139 | { |
140 | struct device *dev = ndev->dev; |
141 | |
142 | if (idx < 0 || idx > ndev->mw_count) { |
143 | dev_err(dev, "Unsupported Memory Window index %d\n" , idx); |
144 | return -EINVAL; |
145 | } |
146 | |
147 | return idx + 2; |
148 | } |
149 | |
150 | static int ntb_epf_mw_count(struct ntb_dev *ntb, int pidx) |
151 | { |
152 | struct ntb_epf_dev *ndev = ntb_ndev(ntb); |
153 | struct device *dev = ndev->dev; |
154 | |
155 | if (pidx != NTB_DEF_PEER_IDX) { |
156 | dev_err(dev, "Unsupported Peer ID %d\n" , pidx); |
157 | return -EINVAL; |
158 | } |
159 | |
160 | return ndev->mw_count; |
161 | } |
162 | |
163 | static int ntb_epf_mw_get_align(struct ntb_dev *ntb, int pidx, int idx, |
164 | resource_size_t *addr_align, |
165 | resource_size_t *size_align, |
166 | resource_size_t *size_max) |
167 | { |
168 | struct ntb_epf_dev *ndev = ntb_ndev(ntb); |
169 | struct device *dev = ndev->dev; |
170 | int bar; |
171 | |
172 | if (pidx != NTB_DEF_PEER_IDX) { |
173 | dev_err(dev, "Unsupported Peer ID %d\n" , pidx); |
174 | return -EINVAL; |
175 | } |
176 | |
177 | bar = ntb_epf_mw_to_bar(ndev, idx); |
178 | if (bar < 0) |
179 | return bar; |
180 | |
181 | if (addr_align) |
182 | *addr_align = SZ_4K; |
183 | |
184 | if (size_align) |
185 | *size_align = 1; |
186 | |
187 | if (size_max) |
188 | *size_max = pci_resource_len(ndev->ntb.pdev, bar); |
189 | |
190 | return 0; |
191 | } |
192 | |
193 | static u64 ntb_epf_link_is_up(struct ntb_dev *ntb, |
194 | enum ntb_speed *speed, |
195 | enum ntb_width *width) |
196 | { |
197 | struct ntb_epf_dev *ndev = ntb_ndev(ntb); |
198 | u32 status; |
199 | |
200 | status = readw(addr: ndev->ctrl_reg + NTB_EPF_LINK_STATUS); |
201 | |
202 | return status & LINK_STATUS_UP; |
203 | } |
204 | |
205 | static u32 ntb_epf_spad_read(struct ntb_dev *ntb, int idx) |
206 | { |
207 | struct ntb_epf_dev *ndev = ntb_ndev(ntb); |
208 | struct device *dev = ndev->dev; |
209 | u32 offset; |
210 | |
211 | if (idx < 0 || idx >= ndev->spad_count) { |
212 | dev_err(dev, "READ: Invalid ScratchPad Index %d\n" , idx); |
213 | return 0; |
214 | } |
215 | |
216 | offset = readl(addr: ndev->ctrl_reg + NTB_EPF_SPAD_OFFSET); |
217 | offset += (idx << 2); |
218 | |
219 | return readl(addr: ndev->ctrl_reg + offset); |
220 | } |
221 | |
222 | static int ntb_epf_spad_write(struct ntb_dev *ntb, |
223 | int idx, u32 val) |
224 | { |
225 | struct ntb_epf_dev *ndev = ntb_ndev(ntb); |
226 | struct device *dev = ndev->dev; |
227 | u32 offset; |
228 | |
229 | if (idx < 0 || idx >= ndev->spad_count) { |
230 | dev_err(dev, "WRITE: Invalid ScratchPad Index %d\n" , idx); |
231 | return -EINVAL; |
232 | } |
233 | |
234 | offset = readl(addr: ndev->ctrl_reg + NTB_EPF_SPAD_OFFSET); |
235 | offset += (idx << 2); |
236 | writel(val, addr: ndev->ctrl_reg + offset); |
237 | |
238 | return 0; |
239 | } |
240 | |
241 | static u32 ntb_epf_peer_spad_read(struct ntb_dev *ntb, int pidx, int idx) |
242 | { |
243 | struct ntb_epf_dev *ndev = ntb_ndev(ntb); |
244 | struct device *dev = ndev->dev; |
245 | u32 offset; |
246 | |
247 | if (pidx != NTB_DEF_PEER_IDX) { |
248 | dev_err(dev, "Unsupported Peer ID %d\n" , pidx); |
249 | return -EINVAL; |
250 | } |
251 | |
252 | if (idx < 0 || idx >= ndev->spad_count) { |
253 | dev_err(dev, "WRITE: Invalid Peer ScratchPad Index %d\n" , idx); |
254 | return -EINVAL; |
255 | } |
256 | |
257 | offset = (idx << 2); |
258 | return readl(addr: ndev->peer_spad_reg + offset); |
259 | } |
260 | |
261 | static int ntb_epf_peer_spad_write(struct ntb_dev *ntb, int pidx, |
262 | int idx, u32 val) |
263 | { |
264 | struct ntb_epf_dev *ndev = ntb_ndev(ntb); |
265 | struct device *dev = ndev->dev; |
266 | u32 offset; |
267 | |
268 | if (pidx != NTB_DEF_PEER_IDX) { |
269 | dev_err(dev, "Unsupported Peer ID %d\n" , pidx); |
270 | return -EINVAL; |
271 | } |
272 | |
273 | if (idx < 0 || idx >= ndev->spad_count) { |
274 | dev_err(dev, "WRITE: Invalid Peer ScratchPad Index %d\n" , idx); |
275 | return -EINVAL; |
276 | } |
277 | |
278 | offset = (idx << 2); |
279 | writel(val, addr: ndev->peer_spad_reg + offset); |
280 | |
281 | return 0; |
282 | } |
283 | |
284 | static int ntb_epf_link_enable(struct ntb_dev *ntb, |
285 | enum ntb_speed max_speed, |
286 | enum ntb_width max_width) |
287 | { |
288 | struct ntb_epf_dev *ndev = ntb_ndev(ntb); |
289 | struct device *dev = ndev->dev; |
290 | int ret; |
291 | |
292 | ret = ntb_epf_send_command(ndev, CMD_LINK_UP, argument: 0); |
293 | if (ret) { |
294 | dev_err(dev, "Fail to enable link\n" ); |
295 | return ret; |
296 | } |
297 | |
298 | return 0; |
299 | } |
300 | |
301 | static int ntb_epf_link_disable(struct ntb_dev *ntb) |
302 | { |
303 | struct ntb_epf_dev *ndev = ntb_ndev(ntb); |
304 | struct device *dev = ndev->dev; |
305 | int ret; |
306 | |
307 | ret = ntb_epf_send_command(ndev, CMD_LINK_DOWN, argument: 0); |
308 | if (ret) { |
309 | dev_err(dev, "Fail to disable link\n" ); |
310 | return ret; |
311 | } |
312 | |
313 | return 0; |
314 | } |
315 | |
316 | static irqreturn_t ntb_epf_vec_isr(int irq, void *dev) |
317 | { |
318 | struct ntb_epf_dev *ndev = dev; |
319 | int irq_no; |
320 | |
321 | irq_no = irq - pci_irq_vector(dev: ndev->ntb.pdev, nr: 0); |
322 | ndev->db_val = irq_no + 1; |
323 | |
324 | if (irq_no == 0) |
325 | ntb_link_event(ntb: &ndev->ntb); |
326 | else |
327 | ntb_db_event(ntb: &ndev->ntb, vector: irq_no); |
328 | |
329 | return IRQ_HANDLED; |
330 | } |
331 | |
332 | static int ntb_epf_init_isr(struct ntb_epf_dev *ndev, int msi_min, int msi_max) |
333 | { |
334 | struct pci_dev *pdev = ndev->ntb.pdev; |
335 | struct device *dev = ndev->dev; |
336 | u32 argument = MSIX_ENABLE; |
337 | int irq; |
338 | int ret; |
339 | int i; |
340 | |
341 | irq = pci_alloc_irq_vectors(dev: pdev, min_vecs: msi_min, max_vecs: msi_max, PCI_IRQ_MSIX); |
342 | if (irq < 0) { |
343 | dev_dbg(dev, "Failed to get MSIX interrupts\n" ); |
344 | irq = pci_alloc_irq_vectors(dev: pdev, min_vecs: msi_min, max_vecs: msi_max, |
345 | PCI_IRQ_MSI); |
346 | if (irq < 0) { |
347 | dev_err(dev, "Failed to get MSI interrupts\n" ); |
348 | return irq; |
349 | } |
350 | argument &= ~MSIX_ENABLE; |
351 | } |
352 | |
353 | for (i = 0; i < irq; i++) { |
354 | ret = request_irq(irq: pci_irq_vector(dev: pdev, nr: i), handler: ntb_epf_vec_isr, |
355 | flags: 0, name: "ntb_epf" , dev: ndev); |
356 | if (ret) { |
357 | dev_err(dev, "Failed to request irq\n" ); |
358 | goto err_request_irq; |
359 | } |
360 | } |
361 | |
362 | ndev->db_count = irq - 1; |
363 | |
364 | ret = ntb_epf_send_command(ndev, CMD_CONFIGURE_DOORBELL, |
365 | argument: argument | irq); |
366 | if (ret) { |
367 | dev_err(dev, "Failed to configure doorbell\n" ); |
368 | goto err_configure_db; |
369 | } |
370 | |
371 | return 0; |
372 | |
373 | err_configure_db: |
374 | for (i = 0; i < ndev->db_count + 1; i++) |
375 | free_irq(pci_irq_vector(dev: pdev, nr: i), ndev); |
376 | |
377 | err_request_irq: |
378 | pci_free_irq_vectors(dev: pdev); |
379 | |
380 | return ret; |
381 | } |
382 | |
383 | static int ntb_epf_peer_mw_count(struct ntb_dev *ntb) |
384 | { |
385 | return ntb_ndev(ntb)->mw_count; |
386 | } |
387 | |
388 | static int ntb_epf_spad_count(struct ntb_dev *ntb) |
389 | { |
390 | return ntb_ndev(ntb)->spad_count; |
391 | } |
392 | |
393 | static u64 ntb_epf_db_valid_mask(struct ntb_dev *ntb) |
394 | { |
395 | return ntb_ndev(ntb)->db_valid_mask; |
396 | } |
397 | |
398 | static int ntb_epf_db_set_mask(struct ntb_dev *ntb, u64 db_bits) |
399 | { |
400 | return 0; |
401 | } |
402 | |
403 | static int ntb_epf_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx, |
404 | dma_addr_t addr, resource_size_t size) |
405 | { |
406 | struct ntb_epf_dev *ndev = ntb_ndev(ntb); |
407 | struct device *dev = ndev->dev; |
408 | resource_size_t mw_size; |
409 | int bar; |
410 | |
411 | if (pidx != NTB_DEF_PEER_IDX) { |
412 | dev_err(dev, "Unsupported Peer ID %d\n" , pidx); |
413 | return -EINVAL; |
414 | } |
415 | |
416 | bar = idx + ndev->mw_bar; |
417 | |
418 | mw_size = pci_resource_len(ntb->pdev, bar); |
419 | |
420 | if (size > mw_size) { |
421 | dev_err(dev, "Size:%pa is greater than the MW size %pa\n" , |
422 | &size, &mw_size); |
423 | return -EINVAL; |
424 | } |
425 | |
426 | writel(lower_32_bits(addr), addr: ndev->ctrl_reg + NTB_EPF_LOWER_ADDR); |
427 | writel(upper_32_bits(addr), addr: ndev->ctrl_reg + NTB_EPF_UPPER_ADDR); |
428 | writel(lower_32_bits(size), addr: ndev->ctrl_reg + NTB_EPF_LOWER_SIZE); |
429 | writel(upper_32_bits(size), addr: ndev->ctrl_reg + NTB_EPF_UPPER_SIZE); |
430 | ntb_epf_send_command(ndev, CMD_CONFIGURE_MW, argument: idx); |
431 | |
432 | return 0; |
433 | } |
434 | |
435 | static int ntb_epf_mw_clear_trans(struct ntb_dev *ntb, int pidx, int idx) |
436 | { |
437 | struct ntb_epf_dev *ndev = ntb_ndev(ntb); |
438 | struct device *dev = ndev->dev; |
439 | int ret = 0; |
440 | |
441 | ntb_epf_send_command(ndev, CMD_TEARDOWN_MW, argument: idx); |
442 | if (ret) |
443 | dev_err(dev, "Failed to teardown memory window\n" ); |
444 | |
445 | return ret; |
446 | } |
447 | |
448 | static int ntb_epf_peer_mw_get_addr(struct ntb_dev *ntb, int idx, |
449 | phys_addr_t *base, resource_size_t *size) |
450 | { |
451 | struct ntb_epf_dev *ndev = ntb_ndev(ntb); |
452 | u32 offset = 0; |
453 | int bar; |
454 | |
455 | if (idx == 0) |
456 | offset = readl(addr: ndev->ctrl_reg + NTB_EPF_MW1_OFFSET); |
457 | |
458 | bar = idx + ndev->mw_bar; |
459 | |
460 | if (base) |
461 | *base = pci_resource_start(ndev->ntb.pdev, bar) + offset; |
462 | |
463 | if (size) |
464 | *size = pci_resource_len(ndev->ntb.pdev, bar) - offset; |
465 | |
466 | return 0; |
467 | } |
468 | |
469 | static int ntb_epf_peer_db_set(struct ntb_dev *ntb, u64 db_bits) |
470 | { |
471 | struct ntb_epf_dev *ndev = ntb_ndev(ntb); |
472 | u32 interrupt_num = ffs(db_bits) + 1; |
473 | struct device *dev = ndev->dev; |
474 | u32 db_entry_size; |
475 | u32 db_offset; |
476 | u32 db_data; |
477 | |
478 | if (interrupt_num > ndev->db_count) { |
479 | dev_err(dev, "DB interrupt %d greater than Max Supported %d\n" , |
480 | interrupt_num, ndev->db_count); |
481 | return -EINVAL; |
482 | } |
483 | |
484 | db_entry_size = readl(addr: ndev->ctrl_reg + NTB_EPF_DB_ENTRY_SIZE); |
485 | |
486 | db_data = readl(addr: ndev->ctrl_reg + NTB_EPF_DB_DATA(interrupt_num)); |
487 | db_offset = readl(addr: ndev->ctrl_reg + NTB_EPF_DB_OFFSET(interrupt_num)); |
488 | writel(val: db_data, addr: ndev->db_reg + (db_entry_size * interrupt_num) + |
489 | db_offset); |
490 | |
491 | return 0; |
492 | } |
493 | |
494 | static u64 ntb_epf_db_read(struct ntb_dev *ntb) |
495 | { |
496 | struct ntb_epf_dev *ndev = ntb_ndev(ntb); |
497 | |
498 | return ndev->db_val; |
499 | } |
500 | |
501 | static int ntb_epf_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) |
502 | { |
503 | return 0; |
504 | } |
505 | |
506 | static int ntb_epf_db_clear(struct ntb_dev *ntb, u64 db_bits) |
507 | { |
508 | struct ntb_epf_dev *ndev = ntb_ndev(ntb); |
509 | |
510 | ndev->db_val = 0; |
511 | |
512 | return 0; |
513 | } |
514 | |
515 | static const struct ntb_dev_ops ntb_epf_ops = { |
516 | .mw_count = ntb_epf_mw_count, |
517 | .spad_count = ntb_epf_spad_count, |
518 | .peer_mw_count = ntb_epf_peer_mw_count, |
519 | .db_valid_mask = ntb_epf_db_valid_mask, |
520 | .db_set_mask = ntb_epf_db_set_mask, |
521 | .mw_set_trans = ntb_epf_mw_set_trans, |
522 | .mw_clear_trans = ntb_epf_mw_clear_trans, |
523 | .peer_mw_get_addr = ntb_epf_peer_mw_get_addr, |
524 | .link_enable = ntb_epf_link_enable, |
525 | .spad_read = ntb_epf_spad_read, |
526 | .spad_write = ntb_epf_spad_write, |
527 | .peer_spad_read = ntb_epf_peer_spad_read, |
528 | .peer_spad_write = ntb_epf_peer_spad_write, |
529 | .peer_db_set = ntb_epf_peer_db_set, |
530 | .db_read = ntb_epf_db_read, |
531 | .mw_get_align = ntb_epf_mw_get_align, |
532 | .link_is_up = ntb_epf_link_is_up, |
533 | .db_clear_mask = ntb_epf_db_clear_mask, |
534 | .db_clear = ntb_epf_db_clear, |
535 | .link_disable = ntb_epf_link_disable, |
536 | }; |
537 | |
538 | static inline void ntb_epf_init_struct(struct ntb_epf_dev *ndev, |
539 | struct pci_dev *pdev) |
540 | { |
541 | ndev->ntb.pdev = pdev; |
542 | ndev->ntb.topo = NTB_TOPO_NONE; |
543 | ndev->ntb.ops = &ntb_epf_ops; |
544 | } |
545 | |
546 | static int ntb_epf_init_dev(struct ntb_epf_dev *ndev) |
547 | { |
548 | struct device *dev = ndev->dev; |
549 | int ret; |
550 | |
551 | /* One Link interrupt and rest doorbell interrupt */ |
552 | ret = ntb_epf_init_isr(ndev, NTB_EPF_MIN_DB_COUNT + 1, |
553 | NTB_EPF_MAX_DB_COUNT + 1); |
554 | if (ret) { |
555 | dev_err(dev, "Failed to init ISR\n" ); |
556 | return ret; |
557 | } |
558 | |
559 | ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; |
560 | ndev->mw_count = readl(addr: ndev->ctrl_reg + NTB_EPF_MW_COUNT); |
561 | ndev->spad_count = readl(addr: ndev->ctrl_reg + NTB_EPF_SPAD_COUNT); |
562 | |
563 | return 0; |
564 | } |
565 | |
566 | static int ntb_epf_init_pci(struct ntb_epf_dev *ndev, |
567 | struct pci_dev *pdev) |
568 | { |
569 | struct device *dev = ndev->dev; |
570 | size_t spad_sz, spad_off; |
571 | int ret; |
572 | |
573 | pci_set_drvdata(pdev, data: ndev); |
574 | |
575 | ret = pci_enable_device(dev: pdev); |
576 | if (ret) { |
577 | dev_err(dev, "Cannot enable PCI device\n" ); |
578 | goto err_pci_enable; |
579 | } |
580 | |
581 | ret = pci_request_regions(pdev, "ntb" ); |
582 | if (ret) { |
583 | dev_err(dev, "Cannot obtain PCI resources\n" ); |
584 | goto err_pci_regions; |
585 | } |
586 | |
587 | pci_set_master(dev: pdev); |
588 | |
589 | ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); |
590 | if (ret) { |
591 | ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); |
592 | if (ret) { |
593 | dev_err(dev, "Cannot set DMA mask\n" ); |
594 | goto err_pci_regions; |
595 | } |
596 | dev_warn(&pdev->dev, "Cannot DMA highmem\n" ); |
597 | } |
598 | |
599 | ndev->ctrl_reg = pci_iomap(dev: pdev, bar: ndev->ctrl_reg_bar, max: 0); |
600 | if (!ndev->ctrl_reg) { |
601 | ret = -EIO; |
602 | goto err_pci_regions; |
603 | } |
604 | |
605 | if (ndev->peer_spad_reg_bar) { |
606 | ndev->peer_spad_reg = pci_iomap(dev: pdev, bar: ndev->peer_spad_reg_bar, max: 0); |
607 | if (!ndev->peer_spad_reg) { |
608 | ret = -EIO; |
609 | goto err_pci_regions; |
610 | } |
611 | } else { |
612 | spad_sz = 4 * readl(addr: ndev->ctrl_reg + NTB_EPF_SPAD_COUNT); |
613 | spad_off = readl(addr: ndev->ctrl_reg + NTB_EPF_SPAD_OFFSET); |
614 | ndev->peer_spad_reg = ndev->ctrl_reg + spad_off + spad_sz; |
615 | } |
616 | |
617 | ndev->db_reg = pci_iomap(dev: pdev, bar: ndev->db_reg_bar, max: 0); |
618 | if (!ndev->db_reg) { |
619 | ret = -EIO; |
620 | goto err_pci_regions; |
621 | } |
622 | |
623 | return 0; |
624 | |
625 | err_pci_regions: |
626 | pci_disable_device(dev: pdev); |
627 | |
628 | err_pci_enable: |
629 | pci_set_drvdata(pdev, NULL); |
630 | |
631 | return ret; |
632 | } |
633 | |
634 | static void ntb_epf_deinit_pci(struct ntb_epf_dev *ndev) |
635 | { |
636 | struct pci_dev *pdev = ndev->ntb.pdev; |
637 | |
638 | pci_iounmap(dev: pdev, ndev->ctrl_reg); |
639 | pci_iounmap(dev: pdev, ndev->peer_spad_reg); |
640 | pci_iounmap(dev: pdev, ndev->db_reg); |
641 | |
642 | pci_release_regions(pdev); |
643 | pci_disable_device(dev: pdev); |
644 | pci_set_drvdata(pdev, NULL); |
645 | } |
646 | |
647 | static void ntb_epf_cleanup_isr(struct ntb_epf_dev *ndev) |
648 | { |
649 | struct pci_dev *pdev = ndev->ntb.pdev; |
650 | int i; |
651 | |
652 | ntb_epf_send_command(ndev, CMD_TEARDOWN_DOORBELL, argument: ndev->db_count + 1); |
653 | |
654 | for (i = 0; i < ndev->db_count + 1; i++) |
655 | free_irq(pci_irq_vector(dev: pdev, nr: i), ndev); |
656 | pci_free_irq_vectors(dev: pdev); |
657 | } |
658 | |
659 | static int ntb_epf_pci_probe(struct pci_dev *pdev, |
660 | const struct pci_device_id *id) |
661 | { |
662 | enum pci_barno peer_spad_reg_bar = BAR_1; |
663 | enum pci_barno ctrl_reg_bar = BAR_0; |
664 | enum pci_barno db_reg_bar = BAR_2; |
665 | enum pci_barno mw_bar = BAR_2; |
666 | struct device *dev = &pdev->dev; |
667 | struct ntb_epf_data *data; |
668 | struct ntb_epf_dev *ndev; |
669 | int ret; |
670 | |
671 | if (pci_is_bridge(dev: pdev)) |
672 | return -ENODEV; |
673 | |
674 | ndev = devm_kzalloc(dev, size: sizeof(*ndev), GFP_KERNEL); |
675 | if (!ndev) |
676 | return -ENOMEM; |
677 | |
678 | data = (struct ntb_epf_data *)id->driver_data; |
679 | if (data) { |
680 | peer_spad_reg_bar = data->peer_spad_reg_bar; |
681 | ctrl_reg_bar = data->ctrl_reg_bar; |
682 | db_reg_bar = data->db_reg_bar; |
683 | mw_bar = data->mw_bar; |
684 | } |
685 | |
686 | ndev->peer_spad_reg_bar = peer_spad_reg_bar; |
687 | ndev->ctrl_reg_bar = ctrl_reg_bar; |
688 | ndev->db_reg_bar = db_reg_bar; |
689 | ndev->mw_bar = mw_bar; |
690 | ndev->dev = dev; |
691 | |
692 | ntb_epf_init_struct(ndev, pdev); |
693 | mutex_init(&ndev->cmd_lock); |
694 | |
695 | ret = ntb_epf_init_pci(ndev, pdev); |
696 | if (ret) { |
697 | dev_err(dev, "Failed to init PCI\n" ); |
698 | return ret; |
699 | } |
700 | |
701 | ret = ntb_epf_init_dev(ndev); |
702 | if (ret) { |
703 | dev_err(dev, "Failed to init device\n" ); |
704 | goto err_init_dev; |
705 | } |
706 | |
707 | ret = ntb_register_device(ntb: &ndev->ntb); |
708 | if (ret) { |
709 | dev_err(dev, "Failed to register NTB device\n" ); |
710 | goto err_register_dev; |
711 | } |
712 | |
713 | return 0; |
714 | |
715 | err_register_dev: |
716 | ntb_epf_cleanup_isr(ndev); |
717 | |
718 | err_init_dev: |
719 | ntb_epf_deinit_pci(ndev); |
720 | |
721 | return ret; |
722 | } |
723 | |
724 | static void ntb_epf_pci_remove(struct pci_dev *pdev) |
725 | { |
726 | struct ntb_epf_dev *ndev = pci_get_drvdata(pdev); |
727 | |
728 | ntb_unregister_device(ntb: &ndev->ntb); |
729 | ntb_epf_cleanup_isr(ndev); |
730 | ntb_epf_deinit_pci(ndev); |
731 | } |
732 | |
733 | static const struct ntb_epf_data j721e_data = { |
734 | .ctrl_reg_bar = BAR_0, |
735 | .peer_spad_reg_bar = BAR_1, |
736 | .db_reg_bar = BAR_2, |
737 | .mw_bar = BAR_2, |
738 | }; |
739 | |
740 | static const struct ntb_epf_data mx8_data = { |
741 | .ctrl_reg_bar = BAR_0, |
742 | .peer_spad_reg_bar = BAR_0, |
743 | .db_reg_bar = BAR_2, |
744 | .mw_bar = BAR_4, |
745 | }; |
746 | |
747 | static const struct pci_device_id ntb_epf_pci_tbl[] = { |
748 | { |
749 | PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_J721E), |
750 | .class = PCI_CLASS_MEMORY_RAM << 8, .class_mask = 0xffff00, |
751 | .driver_data = (kernel_ulong_t)&j721e_data, |
752 | }, |
753 | { |
754 | PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, 0x0809), |
755 | .class = PCI_CLASS_MEMORY_RAM << 8, .class_mask = 0xffff00, |
756 | .driver_data = (kernel_ulong_t)&mx8_data, |
757 | }, |
758 | { }, |
759 | }; |
760 | |
761 | static struct pci_driver ntb_epf_pci_driver = { |
762 | .name = KBUILD_MODNAME, |
763 | .id_table = ntb_epf_pci_tbl, |
764 | .probe = ntb_epf_pci_probe, |
765 | .remove = ntb_epf_pci_remove, |
766 | }; |
767 | module_pci_driver(ntb_epf_pci_driver); |
768 | |
769 | MODULE_DESCRIPTION("PCI ENDPOINT NTB HOST DRIVER" ); |
770 | MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>" ); |
771 | MODULE_LICENSE("GPL v2" ); |
772 | |