1 | // SPDX-License-Identifier: GPL-2.0-or-later |
2 | /* |
3 | * Copyright 2017 IBM Corp. |
4 | */ |
5 | |
6 | #include <linux/hugetlb.h> |
7 | #include <linux/sched/mm.h> |
8 | #include <asm/opal-api.h> |
9 | #include <asm/pnv-pci.h> |
10 | #include <misc/cxllib.h> |
11 | |
12 | #include "cxl.h" |
13 | |
14 | #define CXL_INVALID_DRA ~0ull |
15 | #define CXL_DUMMY_READ_SIZE 128 |
16 | #define CXL_DUMMY_READ_ALIGN 8 |
17 | #define CXL_CAPI_WINDOW_START 0x2000000000000ull |
18 | #define CXL_CAPI_WINDOW_LOG_SIZE 48 |
19 | #define CXL_XSL_CONFIG_CURRENT_VERSION CXL_XSL_CONFIG_VERSION1 |
20 | |
21 | |
22 | bool cxllib_slot_is_supported(struct pci_dev *dev, unsigned long flags) |
23 | { |
24 | int rc; |
25 | u32 phb_index; |
26 | u64 chip_id, capp_unit_id; |
27 | |
28 | /* No flags currently supported */ |
29 | if (flags) |
30 | return false; |
31 | |
32 | if (!cpu_has_feature(CPU_FTR_HVMODE)) |
33 | return false; |
34 | |
35 | if (!cxl_is_power9()) |
36 | return false; |
37 | |
38 | if (cxl_slot_is_switched(dev)) |
39 | return false; |
40 | |
41 | /* on p9, some pci slots are not connected to a CAPP unit */ |
42 | rc = cxl_calc_capp_routing(dev, chipid: &chip_id, phb_index: &phb_index, capp_unit_id: &capp_unit_id); |
43 | if (rc) |
44 | return false; |
45 | |
46 | return true; |
47 | } |
48 | EXPORT_SYMBOL_GPL(cxllib_slot_is_supported); |
49 | |
50 | static DEFINE_MUTEX(dra_mutex); |
51 | static u64 dummy_read_addr = CXL_INVALID_DRA; |
52 | |
53 | static int allocate_dummy_read_buf(void) |
54 | { |
55 | u64 buf, vaddr; |
56 | size_t buf_size; |
57 | |
58 | /* |
59 | * Dummy read buffer is 128-byte long, aligned on a |
60 | * 256-byte boundary and we need the physical address. |
61 | */ |
62 | buf_size = CXL_DUMMY_READ_SIZE + (1ull << CXL_DUMMY_READ_ALIGN); |
63 | buf = (u64) kzalloc(size: buf_size, GFP_KERNEL); |
64 | if (!buf) |
65 | return -ENOMEM; |
66 | |
67 | vaddr = (buf + (1ull << CXL_DUMMY_READ_ALIGN) - 1) & |
68 | (~0ull << CXL_DUMMY_READ_ALIGN); |
69 | |
70 | WARN((vaddr + CXL_DUMMY_READ_SIZE) > (buf + buf_size), |
71 | "Dummy read buffer alignment issue" ); |
72 | dummy_read_addr = virt_to_phys(address: (void *) vaddr); |
73 | return 0; |
74 | } |
75 | |
76 | int cxllib_get_xsl_config(struct pci_dev *dev, struct cxllib_xsl_config *cfg) |
77 | { |
78 | int rc; |
79 | u32 phb_index; |
80 | u64 chip_id, capp_unit_id; |
81 | |
82 | if (!cpu_has_feature(CPU_FTR_HVMODE)) |
83 | return -EINVAL; |
84 | |
85 | mutex_lock(&dra_mutex); |
86 | if (dummy_read_addr == CXL_INVALID_DRA) { |
87 | rc = allocate_dummy_read_buf(); |
88 | if (rc) { |
89 | mutex_unlock(lock: &dra_mutex); |
90 | return rc; |
91 | } |
92 | } |
93 | mutex_unlock(lock: &dra_mutex); |
94 | |
95 | rc = cxl_calc_capp_routing(dev, chipid: &chip_id, phb_index: &phb_index, capp_unit_id: &capp_unit_id); |
96 | if (rc) |
97 | return rc; |
98 | |
99 | rc = cxl_get_xsl9_dsnctl(dev, capp_unit_id, reg: &cfg->dsnctl); |
100 | if (rc) |
101 | return rc; |
102 | |
103 | cfg->version = CXL_XSL_CONFIG_CURRENT_VERSION; |
104 | cfg->log_bar_size = CXL_CAPI_WINDOW_LOG_SIZE; |
105 | cfg->bar_addr = CXL_CAPI_WINDOW_START; |
106 | cfg->dra = dummy_read_addr; |
107 | return 0; |
108 | } |
109 | EXPORT_SYMBOL_GPL(cxllib_get_xsl_config); |
110 | |
111 | int cxllib_switch_phb_mode(struct pci_dev *dev, enum cxllib_mode mode, |
112 | unsigned long flags) |
113 | { |
114 | int rc = 0; |
115 | |
116 | if (!cpu_has_feature(CPU_FTR_HVMODE)) |
117 | return -EINVAL; |
118 | |
119 | switch (mode) { |
120 | case CXL_MODE_PCI: |
121 | /* |
122 | * We currently don't support going back to PCI mode |
123 | * However, we'll turn the invalidations off, so that |
124 | * the firmware doesn't have to ack them and can do |
125 | * things like reset, etc.. with no worries. |
126 | * So always return EPERM (can't go back to PCI) or |
127 | * EBUSY if we couldn't even turn off snooping |
128 | */ |
129 | rc = pnv_phb_to_cxl_mode(dev, OPAL_PHB_CAPI_MODE_SNOOP_OFF); |
130 | if (rc) |
131 | rc = -EBUSY; |
132 | else |
133 | rc = -EPERM; |
134 | break; |
135 | case CXL_MODE_CXL: |
136 | /* DMA only supported on TVT1 for the time being */ |
137 | if (flags != CXL_MODE_DMA_TVT1) |
138 | return -EINVAL; |
139 | rc = pnv_phb_to_cxl_mode(dev, OPAL_PHB_CAPI_MODE_DMA_TVT1); |
140 | if (rc) |
141 | return rc; |
142 | rc = pnv_phb_to_cxl_mode(dev, OPAL_PHB_CAPI_MODE_SNOOP_ON); |
143 | break; |
144 | default: |
145 | rc = -EINVAL; |
146 | } |
147 | return rc; |
148 | } |
149 | EXPORT_SYMBOL_GPL(cxllib_switch_phb_mode); |
150 | |
151 | /* |
152 | * When switching the PHB to capi mode, the TVT#1 entry for |
153 | * the Partitionable Endpoint is set in bypass mode, like |
154 | * in PCI mode. |
155 | * Configure the device dma to use TVT#1, which is done |
156 | * by calling dma_set_mask() with a mask large enough. |
157 | */ |
158 | int cxllib_set_device_dma(struct pci_dev *dev, unsigned long flags) |
159 | { |
160 | int rc; |
161 | |
162 | if (flags) |
163 | return -EINVAL; |
164 | |
165 | rc = dma_set_mask(dev: &dev->dev, DMA_BIT_MASK(64)); |
166 | return rc; |
167 | } |
168 | EXPORT_SYMBOL_GPL(cxllib_set_device_dma); |
169 | |
170 | int cxllib_get_PE_attributes(struct task_struct *task, |
171 | unsigned long translation_mode, |
172 | struct cxllib_pe_attributes *attr) |
173 | { |
174 | if (translation_mode != CXL_TRANSLATED_MODE && |
175 | translation_mode != CXL_REAL_MODE) |
176 | return -EINVAL; |
177 | |
178 | attr->sr = cxl_calculate_sr(master: false, |
179 | kernel: task == NULL, |
180 | real_mode: translation_mode == CXL_REAL_MODE, |
181 | p9: true); |
182 | attr->lpid = mfspr(SPRN_LPID); |
183 | if (task) { |
184 | struct mm_struct *mm = get_task_mm(task); |
185 | if (mm == NULL) |
186 | return -EINVAL; |
187 | /* |
188 | * Caller is keeping a reference on mm_users for as long |
189 | * as XSL uses the memory context |
190 | */ |
191 | attr->pid = mm->context.id; |
192 | mmput(mm); |
193 | attr->tid = task->thread.tidr; |
194 | } else { |
195 | attr->pid = 0; |
196 | attr->tid = 0; |
197 | } |
198 | return 0; |
199 | } |
200 | EXPORT_SYMBOL_GPL(cxllib_get_PE_attributes); |
201 | |
202 | static int get_vma_info(struct mm_struct *mm, u64 addr, |
203 | u64 *vma_start, u64 *vma_end, |
204 | unsigned long *page_size) |
205 | { |
206 | struct vm_area_struct *vma = NULL; |
207 | int rc = 0; |
208 | |
209 | mmap_read_lock(mm); |
210 | |
211 | vma = find_vma(mm, addr); |
212 | if (!vma) { |
213 | rc = -EFAULT; |
214 | goto out; |
215 | } |
216 | *page_size = vma_kernel_pagesize(vma); |
217 | *vma_start = vma->vm_start; |
218 | *vma_end = vma->vm_end; |
219 | out: |
220 | mmap_read_unlock(mm); |
221 | return rc; |
222 | } |
223 | |
224 | int cxllib_handle_fault(struct mm_struct *mm, u64 addr, u64 size, u64 flags) |
225 | { |
226 | int rc; |
227 | u64 dar, vma_start, vma_end; |
228 | unsigned long page_size; |
229 | |
230 | if (mm == NULL) |
231 | return -EFAULT; |
232 | |
233 | /* |
234 | * The buffer we have to process can extend over several pages |
235 | * and may also cover several VMAs. |
236 | * We iterate over all the pages. The page size could vary |
237 | * between VMAs. |
238 | */ |
239 | rc = get_vma_info(mm, addr, vma_start: &vma_start, vma_end: &vma_end, page_size: &page_size); |
240 | if (rc) |
241 | return rc; |
242 | |
243 | for (dar = (addr & ~(page_size - 1)); dar < (addr + size); |
244 | dar += page_size) { |
245 | if (dar < vma_start || dar >= vma_end) { |
246 | /* |
247 | * We don't hold mm->mmap_lock while iterating, since |
248 | * the lock is required by one of the lower-level page |
249 | * fault processing functions and it could |
250 | * create a deadlock. |
251 | * |
252 | * It means the VMAs can be altered between 2 |
253 | * loop iterations and we could theoretically |
254 | * miss a page (however unlikely). But that's |
255 | * not really a problem, as the driver will |
256 | * retry access, get another page fault on the |
257 | * missing page and call us again. |
258 | */ |
259 | rc = get_vma_info(mm, addr: dar, vma_start: &vma_start, vma_end: &vma_end, |
260 | page_size: &page_size); |
261 | if (rc) |
262 | return rc; |
263 | } |
264 | |
265 | rc = cxl_handle_mm_fault(mm, dsisr: flags, dar); |
266 | if (rc) |
267 | return -EFAULT; |
268 | } |
269 | return 0; |
270 | } |
271 | EXPORT_SYMBOL_GPL(cxllib_handle_fault); |
272 | |