1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* |
3 | * Copyright (c) 2019, Intel Corporation. |
4 | * |
5 | * Heterogeneous Memory Attributes Table (HMAT) representation |
6 | * |
7 | * This program parses and reports the platform's HMAT tables, and registers |
8 | * the applicable attributes with the node's interfaces. |
9 | */ |
10 | |
11 | #define pr_fmt(fmt) "acpi/hmat: " fmt |
12 | |
13 | #include <linux/acpi.h> |
14 | #include <linux/bitops.h> |
15 | #include <linux/device.h> |
16 | #include <linux/init.h> |
17 | #include <linux/list.h> |
18 | #include <linux/mm.h> |
19 | #include <linux/platform_device.h> |
20 | #include <linux/list_sort.h> |
21 | #include <linux/memregion.h> |
22 | #include <linux/memory.h> |
23 | #include <linux/mutex.h> |
24 | #include <linux/node.h> |
25 | #include <linux/sysfs.h> |
26 | #include <linux/dax.h> |
27 | #include <linux/memory-tiers.h> |
28 | |
29 | static u8 hmat_revision; |
30 | static int hmat_disable __initdata; |
31 | |
32 | void __init disable_hmat(void) |
33 | { |
34 | hmat_disable = 1; |
35 | } |
36 | |
37 | static LIST_HEAD(targets); |
38 | static LIST_HEAD(initiators); |
39 | static LIST_HEAD(localities); |
40 | |
41 | static DEFINE_MUTEX(target_lock); |
42 | |
43 | /* |
44 | * The defined enum order is used to prioritize attributes to break ties when |
45 | * selecting the best performing node. |
46 | */ |
47 | enum locality_types { |
48 | WRITE_LATENCY, |
49 | READ_LATENCY, |
50 | WRITE_BANDWIDTH, |
51 | READ_BANDWIDTH, |
52 | }; |
53 | |
54 | static struct memory_locality *localities_types[4]; |
55 | |
56 | struct target_cache { |
57 | struct list_head node; |
58 | struct node_cache_attrs cache_attrs; |
59 | }; |
60 | |
61 | enum { |
62 | NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL = ACCESS_COORDINATE_MAX, |
63 | NODE_ACCESS_CLASS_GENPORT_SINK_CPU, |
64 | NODE_ACCESS_CLASS_MAX, |
65 | }; |
66 | |
67 | struct memory_target { |
68 | struct list_head node; |
69 | unsigned int memory_pxm; |
70 | unsigned int processor_pxm; |
71 | struct resource memregions; |
72 | struct access_coordinate coord[NODE_ACCESS_CLASS_MAX]; |
73 | struct list_head caches; |
74 | struct node_cache_attrs cache_attrs; |
75 | u8 gen_port_device_handle[ACPI_SRAT_DEVICE_HANDLE_SIZE]; |
76 | bool registered; |
77 | bool ext_updated; /* externally updated */ |
78 | }; |
79 | |
80 | struct memory_initiator { |
81 | struct list_head node; |
82 | unsigned int processor_pxm; |
83 | bool has_cpu; |
84 | }; |
85 | |
86 | struct memory_locality { |
87 | struct list_head node; |
88 | struct acpi_hmat_locality *hmat_loc; |
89 | }; |
90 | |
91 | static struct memory_initiator *find_mem_initiator(unsigned int cpu_pxm) |
92 | { |
93 | struct memory_initiator *initiator; |
94 | |
95 | list_for_each_entry(initiator, &initiators, node) |
96 | if (initiator->processor_pxm == cpu_pxm) |
97 | return initiator; |
98 | return NULL; |
99 | } |
100 | |
101 | static struct memory_target *find_mem_target(unsigned int mem_pxm) |
102 | { |
103 | struct memory_target *target; |
104 | |
105 | list_for_each_entry(target, &targets, node) |
106 | if (target->memory_pxm == mem_pxm) |
107 | return target; |
108 | return NULL; |
109 | } |
110 | |
111 | static struct memory_target *acpi_find_genport_target(u32 uid) |
112 | { |
113 | struct memory_target *target; |
114 | u32 target_uid; |
115 | u8 *uid_ptr; |
116 | |
117 | list_for_each_entry(target, &targets, node) { |
118 | uid_ptr = target->gen_port_device_handle + 8; |
119 | target_uid = *(u32 *)uid_ptr; |
120 | if (uid == target_uid) |
121 | return target; |
122 | } |
123 | |
124 | return NULL; |
125 | } |
126 | |
127 | /** |
128 | * acpi_get_genport_coordinates - Retrieve the access coordinates for a generic port |
129 | * @uid: ACPI unique id |
130 | * @coord: The access coordinates written back out for the generic port. |
131 | * Expect 2 levels array. |
132 | * |
133 | * Return: 0 on success. Errno on failure. |
134 | * |
135 | * Only supports device handles that are ACPI. Assume ACPI0016 HID for CXL. |
136 | */ |
137 | int acpi_get_genport_coordinates(u32 uid, |
138 | struct access_coordinate *coord) |
139 | { |
140 | struct memory_target *target; |
141 | |
142 | guard(mutex)(T: &target_lock); |
143 | target = acpi_find_genport_target(uid); |
144 | if (!target) |
145 | return -ENOENT; |
146 | |
147 | coord[ACCESS_COORDINATE_LOCAL] = |
148 | target->coord[NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL]; |
149 | coord[ACCESS_COORDINATE_CPU] = |
150 | target->coord[NODE_ACCESS_CLASS_GENPORT_SINK_CPU]; |
151 | |
152 | return 0; |
153 | } |
154 | EXPORT_SYMBOL_NS_GPL(acpi_get_genport_coordinates, CXL); |
155 | |
156 | static __init void alloc_memory_initiator(unsigned int cpu_pxm) |
157 | { |
158 | struct memory_initiator *initiator; |
159 | |
160 | if (pxm_to_node(cpu_pxm) == NUMA_NO_NODE) |
161 | return; |
162 | |
163 | initiator = find_mem_initiator(cpu_pxm); |
164 | if (initiator) |
165 | return; |
166 | |
167 | initiator = kzalloc(size: sizeof(*initiator), GFP_KERNEL); |
168 | if (!initiator) |
169 | return; |
170 | |
171 | initiator->processor_pxm = cpu_pxm; |
172 | initiator->has_cpu = node_state(node: pxm_to_node(cpu_pxm), state: N_CPU); |
173 | list_add_tail(new: &initiator->node, head: &initiators); |
174 | } |
175 | |
176 | static __init struct memory_target *alloc_target(unsigned int mem_pxm) |
177 | { |
178 | struct memory_target *target; |
179 | |
180 | target = find_mem_target(mem_pxm); |
181 | if (!target) { |
182 | target = kzalloc(size: sizeof(*target), GFP_KERNEL); |
183 | if (!target) |
184 | return NULL; |
185 | target->memory_pxm = mem_pxm; |
186 | target->processor_pxm = PXM_INVAL; |
187 | target->memregions = (struct resource) { |
188 | .name = "ACPI mem" , |
189 | .start = 0, |
190 | .end = -1, |
191 | .flags = IORESOURCE_MEM, |
192 | }; |
193 | list_add_tail(new: &target->node, head: &targets); |
194 | INIT_LIST_HEAD(list: &target->caches); |
195 | } |
196 | |
197 | return target; |
198 | } |
199 | |
200 | static __init void alloc_memory_target(unsigned int mem_pxm, |
201 | resource_size_t start, |
202 | resource_size_t len) |
203 | { |
204 | struct memory_target *target; |
205 | |
206 | target = alloc_target(mem_pxm); |
207 | if (!target) |
208 | return; |
209 | |
210 | /* |
211 | * There are potentially multiple ranges per PXM, so record each |
212 | * in the per-target memregions resource tree. |
213 | */ |
214 | if (!__request_region(&target->memregions, start, n: len, name: "memory target" , |
215 | IORESOURCE_MEM)) |
216 | pr_warn("failed to reserve %#llx - %#llx in pxm: %d\n" , |
217 | start, start + len, mem_pxm); |
218 | } |
219 | |
220 | static __init void alloc_genport_target(unsigned int mem_pxm, u8 *handle) |
221 | { |
222 | struct memory_target *target; |
223 | |
224 | target = alloc_target(mem_pxm); |
225 | if (!target) |
226 | return; |
227 | |
228 | memcpy(target->gen_port_device_handle, handle, |
229 | ACPI_SRAT_DEVICE_HANDLE_SIZE); |
230 | } |
231 | |
232 | static __init const char *hmat_data_type(u8 type) |
233 | { |
234 | switch (type) { |
235 | case ACPI_HMAT_ACCESS_LATENCY: |
236 | return "Access Latency" ; |
237 | case ACPI_HMAT_READ_LATENCY: |
238 | return "Read Latency" ; |
239 | case ACPI_HMAT_WRITE_LATENCY: |
240 | return "Write Latency" ; |
241 | case ACPI_HMAT_ACCESS_BANDWIDTH: |
242 | return "Access Bandwidth" ; |
243 | case ACPI_HMAT_READ_BANDWIDTH: |
244 | return "Read Bandwidth" ; |
245 | case ACPI_HMAT_WRITE_BANDWIDTH: |
246 | return "Write Bandwidth" ; |
247 | default: |
248 | return "Reserved" ; |
249 | } |
250 | } |
251 | |
252 | static __init const char *hmat_data_type_suffix(u8 type) |
253 | { |
254 | switch (type) { |
255 | case ACPI_HMAT_ACCESS_LATENCY: |
256 | case ACPI_HMAT_READ_LATENCY: |
257 | case ACPI_HMAT_WRITE_LATENCY: |
258 | return " nsec" ; |
259 | case ACPI_HMAT_ACCESS_BANDWIDTH: |
260 | case ACPI_HMAT_READ_BANDWIDTH: |
261 | case ACPI_HMAT_WRITE_BANDWIDTH: |
262 | return " MB/s" ; |
263 | default: |
264 | return "" ; |
265 | } |
266 | } |
267 | |
268 | static u32 hmat_normalize(u16 entry, u64 base, u8 type) |
269 | { |
270 | u32 value; |
271 | |
272 | /* |
273 | * Check for invalid and overflow values |
274 | */ |
275 | if (entry == 0xffff || !entry) |
276 | return 0; |
277 | else if (base > (UINT_MAX / (entry))) |
278 | return 0; |
279 | |
280 | /* |
281 | * Divide by the base unit for version 1, convert latency from |
282 | * picosenonds to nanoseconds if revision 2. |
283 | */ |
284 | value = entry * base; |
285 | if (hmat_revision == 1) { |
286 | if (value < 10) |
287 | return 0; |
288 | value = DIV_ROUND_UP(value, 10); |
289 | } else if (hmat_revision == 2) { |
290 | switch (type) { |
291 | case ACPI_HMAT_ACCESS_LATENCY: |
292 | case ACPI_HMAT_READ_LATENCY: |
293 | case ACPI_HMAT_WRITE_LATENCY: |
294 | value = DIV_ROUND_UP(value, 1000); |
295 | break; |
296 | default: |
297 | break; |
298 | } |
299 | } |
300 | return value; |
301 | } |
302 | |
303 | static void hmat_update_target_access(struct memory_target *target, |
304 | u8 type, u32 value, int access) |
305 | { |
306 | switch (type) { |
307 | case ACPI_HMAT_ACCESS_LATENCY: |
308 | target->coord[access].read_latency = value; |
309 | target->coord[access].write_latency = value; |
310 | break; |
311 | case ACPI_HMAT_READ_LATENCY: |
312 | target->coord[access].read_latency = value; |
313 | break; |
314 | case ACPI_HMAT_WRITE_LATENCY: |
315 | target->coord[access].write_latency = value; |
316 | break; |
317 | case ACPI_HMAT_ACCESS_BANDWIDTH: |
318 | target->coord[access].read_bandwidth = value; |
319 | target->coord[access].write_bandwidth = value; |
320 | break; |
321 | case ACPI_HMAT_READ_BANDWIDTH: |
322 | target->coord[access].read_bandwidth = value; |
323 | break; |
324 | case ACPI_HMAT_WRITE_BANDWIDTH: |
325 | target->coord[access].write_bandwidth = value; |
326 | break; |
327 | default: |
328 | break; |
329 | } |
330 | } |
331 | |
332 | int hmat_update_target_coordinates(int nid, struct access_coordinate *coord, |
333 | enum access_coordinate_class access) |
334 | { |
335 | struct memory_target *target; |
336 | int pxm; |
337 | |
338 | if (nid == NUMA_NO_NODE) |
339 | return -EINVAL; |
340 | |
341 | pxm = node_to_pxm(nid); |
342 | guard(mutex)(T: &target_lock); |
343 | target = find_mem_target(mem_pxm: pxm); |
344 | if (!target) |
345 | return -ENODEV; |
346 | |
347 | hmat_update_target_access(target, ACPI_HMAT_READ_LATENCY, |
348 | value: coord->read_latency, access); |
349 | hmat_update_target_access(target, ACPI_HMAT_WRITE_LATENCY, |
350 | value: coord->write_latency, access); |
351 | hmat_update_target_access(target, ACPI_HMAT_READ_BANDWIDTH, |
352 | value: coord->read_bandwidth, access); |
353 | hmat_update_target_access(target, ACPI_HMAT_WRITE_BANDWIDTH, |
354 | value: coord->write_bandwidth, access); |
355 | target->ext_updated = true; |
356 | |
357 | return 0; |
358 | } |
359 | EXPORT_SYMBOL_GPL(hmat_update_target_coordinates); |
360 | |
361 | static __init void hmat_add_locality(struct acpi_hmat_locality *hmat_loc) |
362 | { |
363 | struct memory_locality *loc; |
364 | |
365 | loc = kzalloc(size: sizeof(*loc), GFP_KERNEL); |
366 | if (!loc) { |
367 | pr_notice_once("Failed to allocate HMAT locality\n" ); |
368 | return; |
369 | } |
370 | |
371 | loc->hmat_loc = hmat_loc; |
372 | list_add_tail(new: &loc->node, head: &localities); |
373 | |
374 | switch (hmat_loc->data_type) { |
375 | case ACPI_HMAT_ACCESS_LATENCY: |
376 | localities_types[READ_LATENCY] = loc; |
377 | localities_types[WRITE_LATENCY] = loc; |
378 | break; |
379 | case ACPI_HMAT_READ_LATENCY: |
380 | localities_types[READ_LATENCY] = loc; |
381 | break; |
382 | case ACPI_HMAT_WRITE_LATENCY: |
383 | localities_types[WRITE_LATENCY] = loc; |
384 | break; |
385 | case ACPI_HMAT_ACCESS_BANDWIDTH: |
386 | localities_types[READ_BANDWIDTH] = loc; |
387 | localities_types[WRITE_BANDWIDTH] = loc; |
388 | break; |
389 | case ACPI_HMAT_READ_BANDWIDTH: |
390 | localities_types[READ_BANDWIDTH] = loc; |
391 | break; |
392 | case ACPI_HMAT_WRITE_BANDWIDTH: |
393 | localities_types[WRITE_BANDWIDTH] = loc; |
394 | break; |
395 | default: |
396 | break; |
397 | } |
398 | } |
399 | |
400 | static __init void hmat_update_target(unsigned int tgt_pxm, unsigned int init_pxm, |
401 | u8 mem_hier, u8 type, u32 value) |
402 | { |
403 | struct memory_target *target = find_mem_target(mem_pxm: tgt_pxm); |
404 | |
405 | if (mem_hier != ACPI_HMAT_MEMORY) |
406 | return; |
407 | |
408 | if (target && target->processor_pxm == init_pxm) { |
409 | hmat_update_target_access(target, type, value, |
410 | access: ACCESS_COORDINATE_LOCAL); |
411 | /* If the node has a CPU, update access 1 */ |
412 | if (node_state(node: pxm_to_node(init_pxm), state: N_CPU)) |
413 | hmat_update_target_access(target, type, value, |
414 | access: ACCESS_COORDINATE_CPU); |
415 | } |
416 | } |
417 | |
418 | static __init int hmat_parse_locality(union acpi_subtable_headers *, |
419 | const unsigned long end) |
420 | { |
421 | struct acpi_hmat_locality *hmat_loc = (void *)header; |
422 | unsigned int init, targ, total_size, ipds, tpds; |
423 | u32 *inits, *targs, value; |
424 | u16 *entries; |
425 | u8 type, mem_hier; |
426 | |
427 | if (hmat_loc->header.length < sizeof(*hmat_loc)) { |
428 | pr_notice("Unexpected locality header length: %u\n" , |
429 | hmat_loc->header.length); |
430 | return -EINVAL; |
431 | } |
432 | |
433 | type = hmat_loc->data_type; |
434 | mem_hier = hmat_loc->flags & ACPI_HMAT_MEMORY_HIERARCHY; |
435 | ipds = hmat_loc->number_of_initiator_Pds; |
436 | tpds = hmat_loc->number_of_target_Pds; |
437 | total_size = sizeof(*hmat_loc) + sizeof(*entries) * ipds * tpds + |
438 | sizeof(*inits) * ipds + sizeof(*targs) * tpds; |
439 | if (hmat_loc->header.length < total_size) { |
440 | pr_notice("Unexpected locality header length:%u, minimum required:%u\n" , |
441 | hmat_loc->header.length, total_size); |
442 | return -EINVAL; |
443 | } |
444 | |
445 | pr_info("Locality: Flags:%02x Type:%s Initiator Domains:%u Target Domains:%u Base:%lld\n" , |
446 | hmat_loc->flags, hmat_data_type(type), ipds, tpds, |
447 | hmat_loc->entry_base_unit); |
448 | |
449 | inits = (u32 *)(hmat_loc + 1); |
450 | targs = inits + ipds; |
451 | entries = (u16 *)(targs + tpds); |
452 | for (init = 0; init < ipds; init++) { |
453 | alloc_memory_initiator(cpu_pxm: inits[init]); |
454 | for (targ = 0; targ < tpds; targ++) { |
455 | value = hmat_normalize(entry: entries[init * tpds + targ], |
456 | base: hmat_loc->entry_base_unit, |
457 | type); |
458 | pr_info(" Initiator-Target[%u-%u]:%u%s\n" , |
459 | inits[init], targs[targ], value, |
460 | hmat_data_type_suffix(type)); |
461 | |
462 | hmat_update_target(tgt_pxm: targs[targ], init_pxm: inits[init], |
463 | mem_hier, type, value); |
464 | } |
465 | } |
466 | |
467 | if (mem_hier == ACPI_HMAT_MEMORY) |
468 | hmat_add_locality(hmat_loc); |
469 | |
470 | return 0; |
471 | } |
472 | |
473 | static __init int hmat_parse_cache(union acpi_subtable_headers *, |
474 | const unsigned long end) |
475 | { |
476 | struct acpi_hmat_cache *cache = (void *)header; |
477 | struct memory_target *target; |
478 | struct target_cache *tcache; |
479 | u32 attrs; |
480 | |
481 | if (cache->header.length < sizeof(*cache)) { |
482 | pr_notice("Unexpected cache header length: %u\n" , |
483 | cache->header.length); |
484 | return -EINVAL; |
485 | } |
486 | |
487 | attrs = cache->cache_attributes; |
488 | pr_info("Cache: Domain:%u Size:%llu Attrs:%08x SMBIOS Handles:%d\n" , |
489 | cache->memory_PD, cache->cache_size, attrs, |
490 | cache->number_of_SMBIOShandles); |
491 | |
492 | target = find_mem_target(mem_pxm: cache->memory_PD); |
493 | if (!target) |
494 | return 0; |
495 | |
496 | tcache = kzalloc(size: sizeof(*tcache), GFP_KERNEL); |
497 | if (!tcache) { |
498 | pr_notice_once("Failed to allocate HMAT cache info\n" ); |
499 | return 0; |
500 | } |
501 | |
502 | tcache->cache_attrs.size = cache->cache_size; |
503 | tcache->cache_attrs.level = (attrs & ACPI_HMAT_CACHE_LEVEL) >> 4; |
504 | tcache->cache_attrs.line_size = (attrs & ACPI_HMAT_CACHE_LINE_SIZE) >> 16; |
505 | |
506 | switch ((attrs & ACPI_HMAT_CACHE_ASSOCIATIVITY) >> 8) { |
507 | case ACPI_HMAT_CA_DIRECT_MAPPED: |
508 | tcache->cache_attrs.indexing = NODE_CACHE_DIRECT_MAP; |
509 | break; |
510 | case ACPI_HMAT_CA_COMPLEX_CACHE_INDEXING: |
511 | tcache->cache_attrs.indexing = NODE_CACHE_INDEXED; |
512 | break; |
513 | case ACPI_HMAT_CA_NONE: |
514 | default: |
515 | tcache->cache_attrs.indexing = NODE_CACHE_OTHER; |
516 | break; |
517 | } |
518 | |
519 | switch ((attrs & ACPI_HMAT_WRITE_POLICY) >> 12) { |
520 | case ACPI_HMAT_CP_WB: |
521 | tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_BACK; |
522 | break; |
523 | case ACPI_HMAT_CP_WT: |
524 | tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_THROUGH; |
525 | break; |
526 | case ACPI_HMAT_CP_NONE: |
527 | default: |
528 | tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_OTHER; |
529 | break; |
530 | } |
531 | list_add_tail(new: &tcache->node, head: &target->caches); |
532 | |
533 | return 0; |
534 | } |
535 | |
536 | static int __init hmat_parse_proximity_domain(union acpi_subtable_headers *, |
537 | const unsigned long end) |
538 | { |
539 | struct acpi_hmat_proximity_domain *p = (void *)header; |
540 | struct memory_target *target = NULL; |
541 | |
542 | if (p->header.length != sizeof(*p)) { |
543 | pr_notice("Unexpected address range header length: %u\n" , |
544 | p->header.length); |
545 | return -EINVAL; |
546 | } |
547 | |
548 | if (hmat_revision == 1) |
549 | pr_info("Memory (%#llx length %#llx) Flags:%04x Processor Domain:%u Memory Domain:%u\n" , |
550 | p->reserved3, p->reserved4, p->flags, p->processor_PD, |
551 | p->memory_PD); |
552 | else |
553 | pr_info("Memory Flags:%04x Processor Domain:%u Memory Domain:%u\n" , |
554 | p->flags, p->processor_PD, p->memory_PD); |
555 | |
556 | if ((hmat_revision == 1 && p->flags & ACPI_HMAT_MEMORY_PD_VALID) || |
557 | hmat_revision > 1) { |
558 | target = find_mem_target(mem_pxm: p->memory_PD); |
559 | if (!target) { |
560 | pr_debug("Memory Domain missing from SRAT\n" ); |
561 | return -EINVAL; |
562 | } |
563 | } |
564 | if (target && p->flags & ACPI_HMAT_PROCESSOR_PD_VALID) { |
565 | int p_node = pxm_to_node(p->processor_PD); |
566 | |
567 | if (p_node == NUMA_NO_NODE) { |
568 | pr_debug("Invalid Processor Domain\n" ); |
569 | return -EINVAL; |
570 | } |
571 | target->processor_pxm = p->processor_PD; |
572 | } |
573 | |
574 | return 0; |
575 | } |
576 | |
577 | static int __init hmat_parse_subtable(union acpi_subtable_headers *, |
578 | const unsigned long end) |
579 | { |
580 | struct acpi_hmat_structure *hdr = (void *)header; |
581 | |
582 | if (!hdr) |
583 | return -EINVAL; |
584 | |
585 | switch (hdr->type) { |
586 | case ACPI_HMAT_TYPE_PROXIMITY: |
587 | return hmat_parse_proximity_domain(header, end); |
588 | case ACPI_HMAT_TYPE_LOCALITY: |
589 | return hmat_parse_locality(header, end); |
590 | case ACPI_HMAT_TYPE_CACHE: |
591 | return hmat_parse_cache(header, end); |
592 | default: |
593 | return -EINVAL; |
594 | } |
595 | } |
596 | |
597 | static __init int srat_parse_mem_affinity(union acpi_subtable_headers *, |
598 | const unsigned long end) |
599 | { |
600 | struct acpi_srat_mem_affinity *ma = (void *)header; |
601 | |
602 | if (!ma) |
603 | return -EINVAL; |
604 | if (!(ma->flags & ACPI_SRAT_MEM_ENABLED)) |
605 | return 0; |
606 | alloc_memory_target(mem_pxm: ma->proximity_domain, start: ma->base_address, len: ma->length); |
607 | return 0; |
608 | } |
609 | |
610 | static __init int srat_parse_genport_affinity(union acpi_subtable_headers *, |
611 | const unsigned long end) |
612 | { |
613 | struct acpi_srat_generic_affinity *ga = (void *)header; |
614 | |
615 | if (!ga) |
616 | return -EINVAL; |
617 | |
618 | if (!(ga->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED)) |
619 | return 0; |
620 | |
621 | /* Skip PCI device_handle for now */ |
622 | if (ga->device_handle_type != 0) |
623 | return 0; |
624 | |
625 | alloc_genport_target(mem_pxm: ga->proximity_domain, |
626 | handle: (u8 *)ga->device_handle); |
627 | |
628 | return 0; |
629 | } |
630 | |
631 | static u32 hmat_initiator_perf(struct memory_target *target, |
632 | struct memory_initiator *initiator, |
633 | struct acpi_hmat_locality *hmat_loc) |
634 | { |
635 | unsigned int ipds, tpds, i, idx = 0, tdx = 0; |
636 | u32 *inits, *targs; |
637 | u16 *entries; |
638 | |
639 | ipds = hmat_loc->number_of_initiator_Pds; |
640 | tpds = hmat_loc->number_of_target_Pds; |
641 | inits = (u32 *)(hmat_loc + 1); |
642 | targs = inits + ipds; |
643 | entries = (u16 *)(targs + tpds); |
644 | |
645 | for (i = 0; i < ipds; i++) { |
646 | if (inits[i] == initiator->processor_pxm) { |
647 | idx = i; |
648 | break; |
649 | } |
650 | } |
651 | |
652 | if (i == ipds) |
653 | return 0; |
654 | |
655 | for (i = 0; i < tpds; i++) { |
656 | if (targs[i] == target->memory_pxm) { |
657 | tdx = i; |
658 | break; |
659 | } |
660 | } |
661 | if (i == tpds) |
662 | return 0; |
663 | |
664 | return hmat_normalize(entry: entries[idx * tpds + tdx], |
665 | base: hmat_loc->entry_base_unit, |
666 | type: hmat_loc->data_type); |
667 | } |
668 | |
669 | static bool hmat_update_best(u8 type, u32 value, u32 *best) |
670 | { |
671 | bool updated = false; |
672 | |
673 | if (!value) |
674 | return false; |
675 | |
676 | switch (type) { |
677 | case ACPI_HMAT_ACCESS_LATENCY: |
678 | case ACPI_HMAT_READ_LATENCY: |
679 | case ACPI_HMAT_WRITE_LATENCY: |
680 | if (!*best || *best > value) { |
681 | *best = value; |
682 | updated = true; |
683 | } |
684 | break; |
685 | case ACPI_HMAT_ACCESS_BANDWIDTH: |
686 | case ACPI_HMAT_READ_BANDWIDTH: |
687 | case ACPI_HMAT_WRITE_BANDWIDTH: |
688 | if (!*best || *best < value) { |
689 | *best = value; |
690 | updated = true; |
691 | } |
692 | break; |
693 | } |
694 | |
695 | return updated; |
696 | } |
697 | |
698 | static int initiator_cmp(void *priv, const struct list_head *a, |
699 | const struct list_head *b) |
700 | { |
701 | struct memory_initiator *ia; |
702 | struct memory_initiator *ib; |
703 | |
704 | ia = list_entry(a, struct memory_initiator, node); |
705 | ib = list_entry(b, struct memory_initiator, node); |
706 | |
707 | return ia->processor_pxm - ib->processor_pxm; |
708 | } |
709 | |
710 | static int initiators_to_nodemask(unsigned long *p_nodes) |
711 | { |
712 | struct memory_initiator *initiator; |
713 | |
714 | if (list_empty(head: &initiators)) |
715 | return -ENXIO; |
716 | |
717 | list_for_each_entry(initiator, &initiators, node) |
718 | set_bit(nr: initiator->processor_pxm, addr: p_nodes); |
719 | |
720 | return 0; |
721 | } |
722 | |
723 | static void hmat_update_target_attrs(struct memory_target *target, |
724 | unsigned long *p_nodes, int access) |
725 | { |
726 | struct memory_initiator *initiator; |
727 | unsigned int cpu_nid; |
728 | struct memory_locality *loc = NULL; |
729 | u32 best = 0; |
730 | int i; |
731 | |
732 | /* Don't update if an external agent has changed the data. */ |
733 | if (target->ext_updated) |
734 | return; |
735 | |
736 | /* Don't update for generic port if there's no device handle */ |
737 | if ((access == NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL || |
738 | access == NODE_ACCESS_CLASS_GENPORT_SINK_CPU) && |
739 | !(*(u16 *)target->gen_port_device_handle)) |
740 | return; |
741 | |
742 | bitmap_zero(dst: p_nodes, MAX_NUMNODES); |
743 | /* |
744 | * If the Address Range Structure provides a local processor pxm, set |
745 | * only that one. Otherwise, find the best performance attributes and |
746 | * collect all initiators that match. |
747 | */ |
748 | if (target->processor_pxm != PXM_INVAL) { |
749 | cpu_nid = pxm_to_node(target->processor_pxm); |
750 | if (access == ACCESS_COORDINATE_LOCAL || |
751 | node_state(node: cpu_nid, state: N_CPU)) { |
752 | set_bit(nr: target->processor_pxm, addr: p_nodes); |
753 | return; |
754 | } |
755 | } |
756 | |
757 | if (list_empty(head: &localities)) |
758 | return; |
759 | |
760 | /* |
761 | * We need the initiator list sorted so we can use bitmap_clear for |
762 | * previously set initiators when we find a better memory accessor. |
763 | * We'll also use the sorting to prime the candidate nodes with known |
764 | * initiators. |
765 | */ |
766 | list_sort(NULL, head: &initiators, cmp: initiator_cmp); |
767 | if (initiators_to_nodemask(p_nodes) < 0) |
768 | return; |
769 | |
770 | for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) { |
771 | loc = localities_types[i]; |
772 | if (!loc) |
773 | continue; |
774 | |
775 | best = 0; |
776 | list_for_each_entry(initiator, &initiators, node) { |
777 | u32 value; |
778 | |
779 | if ((access == ACCESS_COORDINATE_CPU || |
780 | access == NODE_ACCESS_CLASS_GENPORT_SINK_CPU) && |
781 | !initiator->has_cpu) { |
782 | clear_bit(nr: initiator->processor_pxm, addr: p_nodes); |
783 | continue; |
784 | } |
785 | if (!test_bit(initiator->processor_pxm, p_nodes)) |
786 | continue; |
787 | |
788 | value = hmat_initiator_perf(target, initiator, hmat_loc: loc->hmat_loc); |
789 | if (hmat_update_best(type: loc->hmat_loc->data_type, value, best: &best)) |
790 | bitmap_clear(map: p_nodes, start: 0, nbits: initiator->processor_pxm); |
791 | if (value != best) |
792 | clear_bit(nr: initiator->processor_pxm, addr: p_nodes); |
793 | } |
794 | if (best) |
795 | hmat_update_target_access(target, type: loc->hmat_loc->data_type, value: best, access); |
796 | } |
797 | } |
798 | |
799 | static void __hmat_register_target_initiators(struct memory_target *target, |
800 | unsigned long *p_nodes, |
801 | int access) |
802 | { |
803 | unsigned int mem_nid, cpu_nid; |
804 | int i; |
805 | |
806 | mem_nid = pxm_to_node(target->memory_pxm); |
807 | hmat_update_target_attrs(target, p_nodes, access); |
808 | for_each_set_bit(i, p_nodes, MAX_NUMNODES) { |
809 | cpu_nid = pxm_to_node(i); |
810 | register_memory_node_under_compute_node(mem_nid, cpu_nid, access); |
811 | } |
812 | } |
813 | |
814 | static void hmat_update_generic_target(struct memory_target *target) |
815 | { |
816 | static DECLARE_BITMAP(p_nodes, MAX_NUMNODES); |
817 | |
818 | hmat_update_target_attrs(target, p_nodes, |
819 | access: NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL); |
820 | hmat_update_target_attrs(target, p_nodes, |
821 | access: NODE_ACCESS_CLASS_GENPORT_SINK_CPU); |
822 | } |
823 | |
824 | static void hmat_register_target_initiators(struct memory_target *target) |
825 | { |
826 | static DECLARE_BITMAP(p_nodes, MAX_NUMNODES); |
827 | |
828 | __hmat_register_target_initiators(target, p_nodes, |
829 | access: ACCESS_COORDINATE_LOCAL); |
830 | __hmat_register_target_initiators(target, p_nodes, |
831 | access: ACCESS_COORDINATE_CPU); |
832 | } |
833 | |
834 | static void hmat_register_target_cache(struct memory_target *target) |
835 | { |
836 | unsigned mem_nid = pxm_to_node(target->memory_pxm); |
837 | struct target_cache *tcache; |
838 | |
839 | list_for_each_entry(tcache, &target->caches, node) |
840 | node_add_cache(nid: mem_nid, cache_attrs: &tcache->cache_attrs); |
841 | } |
842 | |
843 | static void hmat_register_target_perf(struct memory_target *target, int access) |
844 | { |
845 | unsigned mem_nid = pxm_to_node(target->memory_pxm); |
846 | node_set_perf_attrs(nid: mem_nid, coord: &target->coord[access], access); |
847 | } |
848 | |
849 | static void hmat_register_target_devices(struct memory_target *target) |
850 | { |
851 | struct resource *res; |
852 | |
853 | /* |
854 | * Do not bother creating devices if no driver is available to |
855 | * consume them. |
856 | */ |
857 | if (!IS_ENABLED(CONFIG_DEV_DAX_HMEM)) |
858 | return; |
859 | |
860 | for (res = target->memregions.child; res; res = res->sibling) { |
861 | int target_nid = pxm_to_node(target->memory_pxm); |
862 | |
863 | hmem_register_resource(target_nid, r: res); |
864 | } |
865 | } |
866 | |
867 | static void hmat_register_target(struct memory_target *target) |
868 | { |
869 | int nid = pxm_to_node(target->memory_pxm); |
870 | |
871 | /* |
872 | * Devices may belong to either an offline or online |
873 | * node, so unconditionally add them. |
874 | */ |
875 | hmat_register_target_devices(target); |
876 | |
877 | /* |
878 | * Register generic port perf numbers. The nid may not be |
879 | * initialized and is still NUMA_NO_NODE. |
880 | */ |
881 | mutex_lock(&target_lock); |
882 | if (*(u16 *)target->gen_port_device_handle) { |
883 | hmat_update_generic_target(target); |
884 | target->registered = true; |
885 | } |
886 | mutex_unlock(lock: &target_lock); |
887 | |
888 | /* |
889 | * Skip offline nodes. This can happen when memory |
890 | * marked EFI_MEMORY_SP, "specific purpose", is applied |
891 | * to all the memory in a proximity domain leading to |
892 | * the node being marked offline / unplugged, or if |
893 | * memory-only "hotplug" node is offline. |
894 | */ |
895 | if (nid == NUMA_NO_NODE || !node_online(nid)) |
896 | return; |
897 | |
898 | mutex_lock(&target_lock); |
899 | if (!target->registered) { |
900 | hmat_register_target_initiators(target); |
901 | hmat_register_target_cache(target); |
902 | hmat_register_target_perf(target, access: ACCESS_COORDINATE_LOCAL); |
903 | hmat_register_target_perf(target, access: ACCESS_COORDINATE_CPU); |
904 | target->registered = true; |
905 | } |
906 | mutex_unlock(lock: &target_lock); |
907 | } |
908 | |
909 | static void hmat_register_targets(void) |
910 | { |
911 | struct memory_target *target; |
912 | |
913 | list_for_each_entry(target, &targets, node) |
914 | hmat_register_target(target); |
915 | } |
916 | |
917 | static int hmat_callback(struct notifier_block *self, |
918 | unsigned long action, void *arg) |
919 | { |
920 | struct memory_target *target; |
921 | struct memory_notify *mnb = arg; |
922 | int pxm, nid = mnb->status_change_nid; |
923 | |
924 | if (nid == NUMA_NO_NODE || action != MEM_ONLINE) |
925 | return NOTIFY_OK; |
926 | |
927 | pxm = node_to_pxm(nid); |
928 | target = find_mem_target(mem_pxm: pxm); |
929 | if (!target) |
930 | return NOTIFY_OK; |
931 | |
932 | hmat_register_target(target); |
933 | return NOTIFY_OK; |
934 | } |
935 | |
936 | static int hmat_set_default_dram_perf(void) |
937 | { |
938 | int rc; |
939 | int nid, pxm; |
940 | struct memory_target *target; |
941 | struct access_coordinate *attrs; |
942 | |
943 | if (!default_dram_type) |
944 | return -EIO; |
945 | |
946 | for_each_node_mask(nid, default_dram_type->nodes) { |
947 | pxm = node_to_pxm(nid); |
948 | target = find_mem_target(mem_pxm: pxm); |
949 | if (!target) |
950 | continue; |
951 | attrs = &target->coord[1]; |
952 | rc = mt_set_default_dram_perf(nid, perf: attrs, source: "ACPI HMAT" ); |
953 | if (rc) |
954 | return rc; |
955 | } |
956 | |
957 | return 0; |
958 | } |
959 | |
960 | static int hmat_calculate_adistance(struct notifier_block *self, |
961 | unsigned long nid, void *data) |
962 | { |
963 | static DECLARE_BITMAP(p_nodes, MAX_NUMNODES); |
964 | struct memory_target *target; |
965 | struct access_coordinate *perf; |
966 | int *adist = data; |
967 | int pxm; |
968 | |
969 | pxm = node_to_pxm(nid); |
970 | target = find_mem_target(mem_pxm: pxm); |
971 | if (!target) |
972 | return NOTIFY_OK; |
973 | |
974 | mutex_lock(&target_lock); |
975 | hmat_update_target_attrs(target, p_nodes, access: ACCESS_COORDINATE_CPU); |
976 | mutex_unlock(lock: &target_lock); |
977 | |
978 | perf = &target->coord[1]; |
979 | |
980 | if (mt_perf_to_adistance(perf, adist)) |
981 | return NOTIFY_OK; |
982 | |
983 | return NOTIFY_STOP; |
984 | } |
985 | |
986 | static struct notifier_block hmat_adist_nb __meminitdata = { |
987 | .notifier_call = hmat_calculate_adistance, |
988 | .priority = 100, |
989 | }; |
990 | |
991 | static __init void hmat_free_structures(void) |
992 | { |
993 | struct memory_target *target, *tnext; |
994 | struct memory_locality *loc, *lnext; |
995 | struct memory_initiator *initiator, *inext; |
996 | struct target_cache *tcache, *cnext; |
997 | |
998 | list_for_each_entry_safe(target, tnext, &targets, node) { |
999 | struct resource *res, *res_next; |
1000 | |
1001 | list_for_each_entry_safe(tcache, cnext, &target->caches, node) { |
1002 | list_del(entry: &tcache->node); |
1003 | kfree(objp: tcache); |
1004 | } |
1005 | |
1006 | list_del(entry: &target->node); |
1007 | res = target->memregions.child; |
1008 | while (res) { |
1009 | res_next = res->sibling; |
1010 | __release_region(&target->memregions, res->start, |
1011 | resource_size(res)); |
1012 | res = res_next; |
1013 | } |
1014 | kfree(objp: target); |
1015 | } |
1016 | |
1017 | list_for_each_entry_safe(initiator, inext, &initiators, node) { |
1018 | list_del(entry: &initiator->node); |
1019 | kfree(objp: initiator); |
1020 | } |
1021 | |
1022 | list_for_each_entry_safe(loc, lnext, &localities, node) { |
1023 | list_del(entry: &loc->node); |
1024 | kfree(objp: loc); |
1025 | } |
1026 | } |
1027 | |
1028 | static __init int hmat_init(void) |
1029 | { |
1030 | struct acpi_table_header *tbl; |
1031 | enum acpi_hmat_type i; |
1032 | acpi_status status; |
1033 | |
1034 | if (srat_disabled() || hmat_disable) |
1035 | return 0; |
1036 | |
1037 | status = acpi_get_table(ACPI_SIG_SRAT, instance: 0, out_table: &tbl); |
1038 | if (ACPI_FAILURE(status)) |
1039 | return 0; |
1040 | |
1041 | if (acpi_table_parse_entries(ACPI_SIG_SRAT, |
1042 | table_size: sizeof(struct acpi_table_srat), |
1043 | entry_id: ACPI_SRAT_TYPE_MEMORY_AFFINITY, |
1044 | handler: srat_parse_mem_affinity, max_entries: 0) < 0) |
1045 | goto out_put; |
1046 | |
1047 | if (acpi_table_parse_entries(ACPI_SIG_SRAT, |
1048 | table_size: sizeof(struct acpi_table_srat), |
1049 | entry_id: ACPI_SRAT_TYPE_GENERIC_PORT_AFFINITY, |
1050 | handler: srat_parse_genport_affinity, max_entries: 0) < 0) |
1051 | goto out_put; |
1052 | |
1053 | acpi_put_table(table: tbl); |
1054 | |
1055 | status = acpi_get_table(ACPI_SIG_HMAT, instance: 0, out_table: &tbl); |
1056 | if (ACPI_FAILURE(status)) |
1057 | goto out_put; |
1058 | |
1059 | hmat_revision = tbl->revision; |
1060 | switch (hmat_revision) { |
1061 | case 1: |
1062 | case 2: |
1063 | break; |
1064 | default: |
1065 | pr_notice("Ignoring: Unknown revision:%d\n" , hmat_revision); |
1066 | goto out_put; |
1067 | } |
1068 | |
1069 | for (i = ACPI_HMAT_TYPE_PROXIMITY; i < ACPI_HMAT_TYPE_RESERVED; i++) { |
1070 | if (acpi_table_parse_entries(ACPI_SIG_HMAT, |
1071 | table_size: sizeof(struct acpi_table_hmat), entry_id: i, |
1072 | handler: hmat_parse_subtable, max_entries: 0) < 0) { |
1073 | pr_notice("Ignoring: Invalid table" ); |
1074 | goto out_put; |
1075 | } |
1076 | } |
1077 | hmat_register_targets(); |
1078 | |
1079 | /* Keep the table and structures if the notifier may use them */ |
1080 | if (hotplug_memory_notifier(hmat_callback, HMAT_CALLBACK_PRI)) |
1081 | goto out_put; |
1082 | |
1083 | if (!hmat_set_default_dram_perf()) |
1084 | register_mt_adistance_algorithm(nb: &hmat_adist_nb); |
1085 | |
1086 | return 0; |
1087 | out_put: |
1088 | hmat_free_structures(); |
1089 | acpi_put_table(table: tbl); |
1090 | return 0; |
1091 | } |
1092 | subsys_initcall(hmat_init); |
1093 | |