1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (c) 2019, Intel Corporation.
4 *
5 * Heterogeneous Memory Attributes Table (HMAT) representation
6 *
7 * This program parses and reports the platform's HMAT tables, and registers
8 * the applicable attributes with the node's interfaces.
9 */
10
11#define pr_fmt(fmt) "acpi/hmat: " fmt
12
13#include <linux/acpi.h>
14#include <linux/bitops.h>
15#include <linux/device.h>
16#include <linux/init.h>
17#include <linux/list.h>
18#include <linux/mm.h>
19#include <linux/platform_device.h>
20#include <linux/list_sort.h>
21#include <linux/memregion.h>
22#include <linux/memory.h>
23#include <linux/mutex.h>
24#include <linux/node.h>
25#include <linux/sysfs.h>
26#include <linux/dax.h>
27#include <linux/memory-tiers.h>
28
29static u8 hmat_revision;
30static int hmat_disable __initdata;
31
32void __init disable_hmat(void)
33{
34 hmat_disable = 1;
35}
36
37static LIST_HEAD(targets);
38static LIST_HEAD(initiators);
39static LIST_HEAD(localities);
40
41static DEFINE_MUTEX(target_lock);
42
43/*
44 * The defined enum order is used to prioritize attributes to break ties when
45 * selecting the best performing node.
46 */
47enum locality_types {
48 WRITE_LATENCY,
49 READ_LATENCY,
50 WRITE_BANDWIDTH,
51 READ_BANDWIDTH,
52};
53
54static struct memory_locality *localities_types[4];
55
56struct target_cache {
57 struct list_head node;
58 struct node_cache_attrs cache_attrs;
59};
60
61enum {
62 NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL = ACCESS_COORDINATE_MAX,
63 NODE_ACCESS_CLASS_GENPORT_SINK_CPU,
64 NODE_ACCESS_CLASS_MAX,
65};
66
67struct memory_target {
68 struct list_head node;
69 unsigned int memory_pxm;
70 unsigned int processor_pxm;
71 struct resource memregions;
72 struct access_coordinate coord[NODE_ACCESS_CLASS_MAX];
73 struct list_head caches;
74 struct node_cache_attrs cache_attrs;
75 u8 gen_port_device_handle[ACPI_SRAT_DEVICE_HANDLE_SIZE];
76 bool registered;
77 bool ext_updated; /* externally updated */
78};
79
80struct memory_initiator {
81 struct list_head node;
82 unsigned int processor_pxm;
83 bool has_cpu;
84};
85
86struct memory_locality {
87 struct list_head node;
88 struct acpi_hmat_locality *hmat_loc;
89};
90
91static struct memory_initiator *find_mem_initiator(unsigned int cpu_pxm)
92{
93 struct memory_initiator *initiator;
94
95 list_for_each_entry(initiator, &initiators, node)
96 if (initiator->processor_pxm == cpu_pxm)
97 return initiator;
98 return NULL;
99}
100
101static struct memory_target *find_mem_target(unsigned int mem_pxm)
102{
103 struct memory_target *target;
104
105 list_for_each_entry(target, &targets, node)
106 if (target->memory_pxm == mem_pxm)
107 return target;
108 return NULL;
109}
110
111static struct memory_target *acpi_find_genport_target(u32 uid)
112{
113 struct memory_target *target;
114 u32 target_uid;
115 u8 *uid_ptr;
116
117 list_for_each_entry(target, &targets, node) {
118 uid_ptr = target->gen_port_device_handle + 8;
119 target_uid = *(u32 *)uid_ptr;
120 if (uid == target_uid)
121 return target;
122 }
123
124 return NULL;
125}
126
127/**
128 * acpi_get_genport_coordinates - Retrieve the access coordinates for a generic port
129 * @uid: ACPI unique id
130 * @coord: The access coordinates written back out for the generic port.
131 * Expect 2 levels array.
132 *
133 * Return: 0 on success. Errno on failure.
134 *
135 * Only supports device handles that are ACPI. Assume ACPI0016 HID for CXL.
136 */
137int acpi_get_genport_coordinates(u32 uid,
138 struct access_coordinate *coord)
139{
140 struct memory_target *target;
141
142 guard(mutex)(T: &target_lock);
143 target = acpi_find_genport_target(uid);
144 if (!target)
145 return -ENOENT;
146
147 coord[ACCESS_COORDINATE_LOCAL] =
148 target->coord[NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL];
149 coord[ACCESS_COORDINATE_CPU] =
150 target->coord[NODE_ACCESS_CLASS_GENPORT_SINK_CPU];
151
152 return 0;
153}
154EXPORT_SYMBOL_NS_GPL(acpi_get_genport_coordinates, CXL);
155
156static __init void alloc_memory_initiator(unsigned int cpu_pxm)
157{
158 struct memory_initiator *initiator;
159
160 if (pxm_to_node(cpu_pxm) == NUMA_NO_NODE)
161 return;
162
163 initiator = find_mem_initiator(cpu_pxm);
164 if (initiator)
165 return;
166
167 initiator = kzalloc(size: sizeof(*initiator), GFP_KERNEL);
168 if (!initiator)
169 return;
170
171 initiator->processor_pxm = cpu_pxm;
172 initiator->has_cpu = node_state(node: pxm_to_node(cpu_pxm), state: N_CPU);
173 list_add_tail(new: &initiator->node, head: &initiators);
174}
175
176static __init struct memory_target *alloc_target(unsigned int mem_pxm)
177{
178 struct memory_target *target;
179
180 target = find_mem_target(mem_pxm);
181 if (!target) {
182 target = kzalloc(size: sizeof(*target), GFP_KERNEL);
183 if (!target)
184 return NULL;
185 target->memory_pxm = mem_pxm;
186 target->processor_pxm = PXM_INVAL;
187 target->memregions = (struct resource) {
188 .name = "ACPI mem",
189 .start = 0,
190 .end = -1,
191 .flags = IORESOURCE_MEM,
192 };
193 list_add_tail(new: &target->node, head: &targets);
194 INIT_LIST_HEAD(list: &target->caches);
195 }
196
197 return target;
198}
199
200static __init void alloc_memory_target(unsigned int mem_pxm,
201 resource_size_t start,
202 resource_size_t len)
203{
204 struct memory_target *target;
205
206 target = alloc_target(mem_pxm);
207 if (!target)
208 return;
209
210 /*
211 * There are potentially multiple ranges per PXM, so record each
212 * in the per-target memregions resource tree.
213 */
214 if (!__request_region(&target->memregions, start, n: len, name: "memory target",
215 IORESOURCE_MEM))
216 pr_warn("failed to reserve %#llx - %#llx in pxm: %d\n",
217 start, start + len, mem_pxm);
218}
219
220static __init void alloc_genport_target(unsigned int mem_pxm, u8 *handle)
221{
222 struct memory_target *target;
223
224 target = alloc_target(mem_pxm);
225 if (!target)
226 return;
227
228 memcpy(target->gen_port_device_handle, handle,
229 ACPI_SRAT_DEVICE_HANDLE_SIZE);
230}
231
232static __init const char *hmat_data_type(u8 type)
233{
234 switch (type) {
235 case ACPI_HMAT_ACCESS_LATENCY:
236 return "Access Latency";
237 case ACPI_HMAT_READ_LATENCY:
238 return "Read Latency";
239 case ACPI_HMAT_WRITE_LATENCY:
240 return "Write Latency";
241 case ACPI_HMAT_ACCESS_BANDWIDTH:
242 return "Access Bandwidth";
243 case ACPI_HMAT_READ_BANDWIDTH:
244 return "Read Bandwidth";
245 case ACPI_HMAT_WRITE_BANDWIDTH:
246 return "Write Bandwidth";
247 default:
248 return "Reserved";
249 }
250}
251
252static __init const char *hmat_data_type_suffix(u8 type)
253{
254 switch (type) {
255 case ACPI_HMAT_ACCESS_LATENCY:
256 case ACPI_HMAT_READ_LATENCY:
257 case ACPI_HMAT_WRITE_LATENCY:
258 return " nsec";
259 case ACPI_HMAT_ACCESS_BANDWIDTH:
260 case ACPI_HMAT_READ_BANDWIDTH:
261 case ACPI_HMAT_WRITE_BANDWIDTH:
262 return " MB/s";
263 default:
264 return "";
265 }
266}
267
268static u32 hmat_normalize(u16 entry, u64 base, u8 type)
269{
270 u32 value;
271
272 /*
273 * Check for invalid and overflow values
274 */
275 if (entry == 0xffff || !entry)
276 return 0;
277 else if (base > (UINT_MAX / (entry)))
278 return 0;
279
280 /*
281 * Divide by the base unit for version 1, convert latency from
282 * picosenonds to nanoseconds if revision 2.
283 */
284 value = entry * base;
285 if (hmat_revision == 1) {
286 if (value < 10)
287 return 0;
288 value = DIV_ROUND_UP(value, 10);
289 } else if (hmat_revision == 2) {
290 switch (type) {
291 case ACPI_HMAT_ACCESS_LATENCY:
292 case ACPI_HMAT_READ_LATENCY:
293 case ACPI_HMAT_WRITE_LATENCY:
294 value = DIV_ROUND_UP(value, 1000);
295 break;
296 default:
297 break;
298 }
299 }
300 return value;
301}
302
303static void hmat_update_target_access(struct memory_target *target,
304 u8 type, u32 value, int access)
305{
306 switch (type) {
307 case ACPI_HMAT_ACCESS_LATENCY:
308 target->coord[access].read_latency = value;
309 target->coord[access].write_latency = value;
310 break;
311 case ACPI_HMAT_READ_LATENCY:
312 target->coord[access].read_latency = value;
313 break;
314 case ACPI_HMAT_WRITE_LATENCY:
315 target->coord[access].write_latency = value;
316 break;
317 case ACPI_HMAT_ACCESS_BANDWIDTH:
318 target->coord[access].read_bandwidth = value;
319 target->coord[access].write_bandwidth = value;
320 break;
321 case ACPI_HMAT_READ_BANDWIDTH:
322 target->coord[access].read_bandwidth = value;
323 break;
324 case ACPI_HMAT_WRITE_BANDWIDTH:
325 target->coord[access].write_bandwidth = value;
326 break;
327 default:
328 break;
329 }
330}
331
332int hmat_update_target_coordinates(int nid, struct access_coordinate *coord,
333 enum access_coordinate_class access)
334{
335 struct memory_target *target;
336 int pxm;
337
338 if (nid == NUMA_NO_NODE)
339 return -EINVAL;
340
341 pxm = node_to_pxm(nid);
342 guard(mutex)(T: &target_lock);
343 target = find_mem_target(mem_pxm: pxm);
344 if (!target)
345 return -ENODEV;
346
347 hmat_update_target_access(target, ACPI_HMAT_READ_LATENCY,
348 value: coord->read_latency, access);
349 hmat_update_target_access(target, ACPI_HMAT_WRITE_LATENCY,
350 value: coord->write_latency, access);
351 hmat_update_target_access(target, ACPI_HMAT_READ_BANDWIDTH,
352 value: coord->read_bandwidth, access);
353 hmat_update_target_access(target, ACPI_HMAT_WRITE_BANDWIDTH,
354 value: coord->write_bandwidth, access);
355 target->ext_updated = true;
356
357 return 0;
358}
359EXPORT_SYMBOL_GPL(hmat_update_target_coordinates);
360
361static __init void hmat_add_locality(struct acpi_hmat_locality *hmat_loc)
362{
363 struct memory_locality *loc;
364
365 loc = kzalloc(size: sizeof(*loc), GFP_KERNEL);
366 if (!loc) {
367 pr_notice_once("Failed to allocate HMAT locality\n");
368 return;
369 }
370
371 loc->hmat_loc = hmat_loc;
372 list_add_tail(new: &loc->node, head: &localities);
373
374 switch (hmat_loc->data_type) {
375 case ACPI_HMAT_ACCESS_LATENCY:
376 localities_types[READ_LATENCY] = loc;
377 localities_types[WRITE_LATENCY] = loc;
378 break;
379 case ACPI_HMAT_READ_LATENCY:
380 localities_types[READ_LATENCY] = loc;
381 break;
382 case ACPI_HMAT_WRITE_LATENCY:
383 localities_types[WRITE_LATENCY] = loc;
384 break;
385 case ACPI_HMAT_ACCESS_BANDWIDTH:
386 localities_types[READ_BANDWIDTH] = loc;
387 localities_types[WRITE_BANDWIDTH] = loc;
388 break;
389 case ACPI_HMAT_READ_BANDWIDTH:
390 localities_types[READ_BANDWIDTH] = loc;
391 break;
392 case ACPI_HMAT_WRITE_BANDWIDTH:
393 localities_types[WRITE_BANDWIDTH] = loc;
394 break;
395 default:
396 break;
397 }
398}
399
400static __init void hmat_update_target(unsigned int tgt_pxm, unsigned int init_pxm,
401 u8 mem_hier, u8 type, u32 value)
402{
403 struct memory_target *target = find_mem_target(mem_pxm: tgt_pxm);
404
405 if (mem_hier != ACPI_HMAT_MEMORY)
406 return;
407
408 if (target && target->processor_pxm == init_pxm) {
409 hmat_update_target_access(target, type, value,
410 access: ACCESS_COORDINATE_LOCAL);
411 /* If the node has a CPU, update access 1 */
412 if (node_state(node: pxm_to_node(init_pxm), state: N_CPU))
413 hmat_update_target_access(target, type, value,
414 access: ACCESS_COORDINATE_CPU);
415 }
416}
417
418static __init int hmat_parse_locality(union acpi_subtable_headers *header,
419 const unsigned long end)
420{
421 struct acpi_hmat_locality *hmat_loc = (void *)header;
422 unsigned int init, targ, total_size, ipds, tpds;
423 u32 *inits, *targs, value;
424 u16 *entries;
425 u8 type, mem_hier;
426
427 if (hmat_loc->header.length < sizeof(*hmat_loc)) {
428 pr_notice("Unexpected locality header length: %u\n",
429 hmat_loc->header.length);
430 return -EINVAL;
431 }
432
433 type = hmat_loc->data_type;
434 mem_hier = hmat_loc->flags & ACPI_HMAT_MEMORY_HIERARCHY;
435 ipds = hmat_loc->number_of_initiator_Pds;
436 tpds = hmat_loc->number_of_target_Pds;
437 total_size = sizeof(*hmat_loc) + sizeof(*entries) * ipds * tpds +
438 sizeof(*inits) * ipds + sizeof(*targs) * tpds;
439 if (hmat_loc->header.length < total_size) {
440 pr_notice("Unexpected locality header length:%u, minimum required:%u\n",
441 hmat_loc->header.length, total_size);
442 return -EINVAL;
443 }
444
445 pr_info("Locality: Flags:%02x Type:%s Initiator Domains:%u Target Domains:%u Base:%lld\n",
446 hmat_loc->flags, hmat_data_type(type), ipds, tpds,
447 hmat_loc->entry_base_unit);
448
449 inits = (u32 *)(hmat_loc + 1);
450 targs = inits + ipds;
451 entries = (u16 *)(targs + tpds);
452 for (init = 0; init < ipds; init++) {
453 alloc_memory_initiator(cpu_pxm: inits[init]);
454 for (targ = 0; targ < tpds; targ++) {
455 value = hmat_normalize(entry: entries[init * tpds + targ],
456 base: hmat_loc->entry_base_unit,
457 type);
458 pr_info(" Initiator-Target[%u-%u]:%u%s\n",
459 inits[init], targs[targ], value,
460 hmat_data_type_suffix(type));
461
462 hmat_update_target(tgt_pxm: targs[targ], init_pxm: inits[init],
463 mem_hier, type, value);
464 }
465 }
466
467 if (mem_hier == ACPI_HMAT_MEMORY)
468 hmat_add_locality(hmat_loc);
469
470 return 0;
471}
472
473static __init int hmat_parse_cache(union acpi_subtable_headers *header,
474 const unsigned long end)
475{
476 struct acpi_hmat_cache *cache = (void *)header;
477 struct memory_target *target;
478 struct target_cache *tcache;
479 u32 attrs;
480
481 if (cache->header.length < sizeof(*cache)) {
482 pr_notice("Unexpected cache header length: %u\n",
483 cache->header.length);
484 return -EINVAL;
485 }
486
487 attrs = cache->cache_attributes;
488 pr_info("Cache: Domain:%u Size:%llu Attrs:%08x SMBIOS Handles:%d\n",
489 cache->memory_PD, cache->cache_size, attrs,
490 cache->number_of_SMBIOShandles);
491
492 target = find_mem_target(mem_pxm: cache->memory_PD);
493 if (!target)
494 return 0;
495
496 tcache = kzalloc(size: sizeof(*tcache), GFP_KERNEL);
497 if (!tcache) {
498 pr_notice_once("Failed to allocate HMAT cache info\n");
499 return 0;
500 }
501
502 tcache->cache_attrs.size = cache->cache_size;
503 tcache->cache_attrs.level = (attrs & ACPI_HMAT_CACHE_LEVEL) >> 4;
504 tcache->cache_attrs.line_size = (attrs & ACPI_HMAT_CACHE_LINE_SIZE) >> 16;
505
506 switch ((attrs & ACPI_HMAT_CACHE_ASSOCIATIVITY) >> 8) {
507 case ACPI_HMAT_CA_DIRECT_MAPPED:
508 tcache->cache_attrs.indexing = NODE_CACHE_DIRECT_MAP;
509 break;
510 case ACPI_HMAT_CA_COMPLEX_CACHE_INDEXING:
511 tcache->cache_attrs.indexing = NODE_CACHE_INDEXED;
512 break;
513 case ACPI_HMAT_CA_NONE:
514 default:
515 tcache->cache_attrs.indexing = NODE_CACHE_OTHER;
516 break;
517 }
518
519 switch ((attrs & ACPI_HMAT_WRITE_POLICY) >> 12) {
520 case ACPI_HMAT_CP_WB:
521 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_BACK;
522 break;
523 case ACPI_HMAT_CP_WT:
524 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_THROUGH;
525 break;
526 case ACPI_HMAT_CP_NONE:
527 default:
528 tcache->cache_attrs.write_policy = NODE_CACHE_WRITE_OTHER;
529 break;
530 }
531 list_add_tail(new: &tcache->node, head: &target->caches);
532
533 return 0;
534}
535
536static int __init hmat_parse_proximity_domain(union acpi_subtable_headers *header,
537 const unsigned long end)
538{
539 struct acpi_hmat_proximity_domain *p = (void *)header;
540 struct memory_target *target = NULL;
541
542 if (p->header.length != sizeof(*p)) {
543 pr_notice("Unexpected address range header length: %u\n",
544 p->header.length);
545 return -EINVAL;
546 }
547
548 if (hmat_revision == 1)
549 pr_info("Memory (%#llx length %#llx) Flags:%04x Processor Domain:%u Memory Domain:%u\n",
550 p->reserved3, p->reserved4, p->flags, p->processor_PD,
551 p->memory_PD);
552 else
553 pr_info("Memory Flags:%04x Processor Domain:%u Memory Domain:%u\n",
554 p->flags, p->processor_PD, p->memory_PD);
555
556 if ((hmat_revision == 1 && p->flags & ACPI_HMAT_MEMORY_PD_VALID) ||
557 hmat_revision > 1) {
558 target = find_mem_target(mem_pxm: p->memory_PD);
559 if (!target) {
560 pr_debug("Memory Domain missing from SRAT\n");
561 return -EINVAL;
562 }
563 }
564 if (target && p->flags & ACPI_HMAT_PROCESSOR_PD_VALID) {
565 int p_node = pxm_to_node(p->processor_PD);
566
567 if (p_node == NUMA_NO_NODE) {
568 pr_debug("Invalid Processor Domain\n");
569 return -EINVAL;
570 }
571 target->processor_pxm = p->processor_PD;
572 }
573
574 return 0;
575}
576
577static int __init hmat_parse_subtable(union acpi_subtable_headers *header,
578 const unsigned long end)
579{
580 struct acpi_hmat_structure *hdr = (void *)header;
581
582 if (!hdr)
583 return -EINVAL;
584
585 switch (hdr->type) {
586 case ACPI_HMAT_TYPE_PROXIMITY:
587 return hmat_parse_proximity_domain(header, end);
588 case ACPI_HMAT_TYPE_LOCALITY:
589 return hmat_parse_locality(header, end);
590 case ACPI_HMAT_TYPE_CACHE:
591 return hmat_parse_cache(header, end);
592 default:
593 return -EINVAL;
594 }
595}
596
597static __init int srat_parse_mem_affinity(union acpi_subtable_headers *header,
598 const unsigned long end)
599{
600 struct acpi_srat_mem_affinity *ma = (void *)header;
601
602 if (!ma)
603 return -EINVAL;
604 if (!(ma->flags & ACPI_SRAT_MEM_ENABLED))
605 return 0;
606 alloc_memory_target(mem_pxm: ma->proximity_domain, start: ma->base_address, len: ma->length);
607 return 0;
608}
609
610static __init int srat_parse_genport_affinity(union acpi_subtable_headers *header,
611 const unsigned long end)
612{
613 struct acpi_srat_generic_affinity *ga = (void *)header;
614
615 if (!ga)
616 return -EINVAL;
617
618 if (!(ga->flags & ACPI_SRAT_GENERIC_AFFINITY_ENABLED))
619 return 0;
620
621 /* Skip PCI device_handle for now */
622 if (ga->device_handle_type != 0)
623 return 0;
624
625 alloc_genport_target(mem_pxm: ga->proximity_domain,
626 handle: (u8 *)ga->device_handle);
627
628 return 0;
629}
630
631static u32 hmat_initiator_perf(struct memory_target *target,
632 struct memory_initiator *initiator,
633 struct acpi_hmat_locality *hmat_loc)
634{
635 unsigned int ipds, tpds, i, idx = 0, tdx = 0;
636 u32 *inits, *targs;
637 u16 *entries;
638
639 ipds = hmat_loc->number_of_initiator_Pds;
640 tpds = hmat_loc->number_of_target_Pds;
641 inits = (u32 *)(hmat_loc + 1);
642 targs = inits + ipds;
643 entries = (u16 *)(targs + tpds);
644
645 for (i = 0; i < ipds; i++) {
646 if (inits[i] == initiator->processor_pxm) {
647 idx = i;
648 break;
649 }
650 }
651
652 if (i == ipds)
653 return 0;
654
655 for (i = 0; i < tpds; i++) {
656 if (targs[i] == target->memory_pxm) {
657 tdx = i;
658 break;
659 }
660 }
661 if (i == tpds)
662 return 0;
663
664 return hmat_normalize(entry: entries[idx * tpds + tdx],
665 base: hmat_loc->entry_base_unit,
666 type: hmat_loc->data_type);
667}
668
669static bool hmat_update_best(u8 type, u32 value, u32 *best)
670{
671 bool updated = false;
672
673 if (!value)
674 return false;
675
676 switch (type) {
677 case ACPI_HMAT_ACCESS_LATENCY:
678 case ACPI_HMAT_READ_LATENCY:
679 case ACPI_HMAT_WRITE_LATENCY:
680 if (!*best || *best > value) {
681 *best = value;
682 updated = true;
683 }
684 break;
685 case ACPI_HMAT_ACCESS_BANDWIDTH:
686 case ACPI_HMAT_READ_BANDWIDTH:
687 case ACPI_HMAT_WRITE_BANDWIDTH:
688 if (!*best || *best < value) {
689 *best = value;
690 updated = true;
691 }
692 break;
693 }
694
695 return updated;
696}
697
698static int initiator_cmp(void *priv, const struct list_head *a,
699 const struct list_head *b)
700{
701 struct memory_initiator *ia;
702 struct memory_initiator *ib;
703
704 ia = list_entry(a, struct memory_initiator, node);
705 ib = list_entry(b, struct memory_initiator, node);
706
707 return ia->processor_pxm - ib->processor_pxm;
708}
709
710static int initiators_to_nodemask(unsigned long *p_nodes)
711{
712 struct memory_initiator *initiator;
713
714 if (list_empty(head: &initiators))
715 return -ENXIO;
716
717 list_for_each_entry(initiator, &initiators, node)
718 set_bit(nr: initiator->processor_pxm, addr: p_nodes);
719
720 return 0;
721}
722
723static void hmat_update_target_attrs(struct memory_target *target,
724 unsigned long *p_nodes, int access)
725{
726 struct memory_initiator *initiator;
727 unsigned int cpu_nid;
728 struct memory_locality *loc = NULL;
729 u32 best = 0;
730 int i;
731
732 /* Don't update if an external agent has changed the data. */
733 if (target->ext_updated)
734 return;
735
736 /* Don't update for generic port if there's no device handle */
737 if ((access == NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL ||
738 access == NODE_ACCESS_CLASS_GENPORT_SINK_CPU) &&
739 !(*(u16 *)target->gen_port_device_handle))
740 return;
741
742 bitmap_zero(dst: p_nodes, MAX_NUMNODES);
743 /*
744 * If the Address Range Structure provides a local processor pxm, set
745 * only that one. Otherwise, find the best performance attributes and
746 * collect all initiators that match.
747 */
748 if (target->processor_pxm != PXM_INVAL) {
749 cpu_nid = pxm_to_node(target->processor_pxm);
750 if (access == ACCESS_COORDINATE_LOCAL ||
751 node_state(node: cpu_nid, state: N_CPU)) {
752 set_bit(nr: target->processor_pxm, addr: p_nodes);
753 return;
754 }
755 }
756
757 if (list_empty(head: &localities))
758 return;
759
760 /*
761 * We need the initiator list sorted so we can use bitmap_clear for
762 * previously set initiators when we find a better memory accessor.
763 * We'll also use the sorting to prime the candidate nodes with known
764 * initiators.
765 */
766 list_sort(NULL, head: &initiators, cmp: initiator_cmp);
767 if (initiators_to_nodemask(p_nodes) < 0)
768 return;
769
770 for (i = WRITE_LATENCY; i <= READ_BANDWIDTH; i++) {
771 loc = localities_types[i];
772 if (!loc)
773 continue;
774
775 best = 0;
776 list_for_each_entry(initiator, &initiators, node) {
777 u32 value;
778
779 if ((access == ACCESS_COORDINATE_CPU ||
780 access == NODE_ACCESS_CLASS_GENPORT_SINK_CPU) &&
781 !initiator->has_cpu) {
782 clear_bit(nr: initiator->processor_pxm, addr: p_nodes);
783 continue;
784 }
785 if (!test_bit(initiator->processor_pxm, p_nodes))
786 continue;
787
788 value = hmat_initiator_perf(target, initiator, hmat_loc: loc->hmat_loc);
789 if (hmat_update_best(type: loc->hmat_loc->data_type, value, best: &best))
790 bitmap_clear(map: p_nodes, start: 0, nbits: initiator->processor_pxm);
791 if (value != best)
792 clear_bit(nr: initiator->processor_pxm, addr: p_nodes);
793 }
794 if (best)
795 hmat_update_target_access(target, type: loc->hmat_loc->data_type, value: best, access);
796 }
797}
798
799static void __hmat_register_target_initiators(struct memory_target *target,
800 unsigned long *p_nodes,
801 int access)
802{
803 unsigned int mem_nid, cpu_nid;
804 int i;
805
806 mem_nid = pxm_to_node(target->memory_pxm);
807 hmat_update_target_attrs(target, p_nodes, access);
808 for_each_set_bit(i, p_nodes, MAX_NUMNODES) {
809 cpu_nid = pxm_to_node(i);
810 register_memory_node_under_compute_node(mem_nid, cpu_nid, access);
811 }
812}
813
814static void hmat_update_generic_target(struct memory_target *target)
815{
816 static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
817
818 hmat_update_target_attrs(target, p_nodes,
819 access: NODE_ACCESS_CLASS_GENPORT_SINK_LOCAL);
820 hmat_update_target_attrs(target, p_nodes,
821 access: NODE_ACCESS_CLASS_GENPORT_SINK_CPU);
822}
823
824static void hmat_register_target_initiators(struct memory_target *target)
825{
826 static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
827
828 __hmat_register_target_initiators(target, p_nodes,
829 access: ACCESS_COORDINATE_LOCAL);
830 __hmat_register_target_initiators(target, p_nodes,
831 access: ACCESS_COORDINATE_CPU);
832}
833
834static void hmat_register_target_cache(struct memory_target *target)
835{
836 unsigned mem_nid = pxm_to_node(target->memory_pxm);
837 struct target_cache *tcache;
838
839 list_for_each_entry(tcache, &target->caches, node)
840 node_add_cache(nid: mem_nid, cache_attrs: &tcache->cache_attrs);
841}
842
843static void hmat_register_target_perf(struct memory_target *target, int access)
844{
845 unsigned mem_nid = pxm_to_node(target->memory_pxm);
846 node_set_perf_attrs(nid: mem_nid, coord: &target->coord[access], access);
847}
848
849static void hmat_register_target_devices(struct memory_target *target)
850{
851 struct resource *res;
852
853 /*
854 * Do not bother creating devices if no driver is available to
855 * consume them.
856 */
857 if (!IS_ENABLED(CONFIG_DEV_DAX_HMEM))
858 return;
859
860 for (res = target->memregions.child; res; res = res->sibling) {
861 int target_nid = pxm_to_node(target->memory_pxm);
862
863 hmem_register_resource(target_nid, r: res);
864 }
865}
866
867static void hmat_register_target(struct memory_target *target)
868{
869 int nid = pxm_to_node(target->memory_pxm);
870
871 /*
872 * Devices may belong to either an offline or online
873 * node, so unconditionally add them.
874 */
875 hmat_register_target_devices(target);
876
877 /*
878 * Register generic port perf numbers. The nid may not be
879 * initialized and is still NUMA_NO_NODE.
880 */
881 mutex_lock(&target_lock);
882 if (*(u16 *)target->gen_port_device_handle) {
883 hmat_update_generic_target(target);
884 target->registered = true;
885 }
886 mutex_unlock(lock: &target_lock);
887
888 /*
889 * Skip offline nodes. This can happen when memory
890 * marked EFI_MEMORY_SP, "specific purpose", is applied
891 * to all the memory in a proximity domain leading to
892 * the node being marked offline / unplugged, or if
893 * memory-only "hotplug" node is offline.
894 */
895 if (nid == NUMA_NO_NODE || !node_online(nid))
896 return;
897
898 mutex_lock(&target_lock);
899 if (!target->registered) {
900 hmat_register_target_initiators(target);
901 hmat_register_target_cache(target);
902 hmat_register_target_perf(target, access: ACCESS_COORDINATE_LOCAL);
903 hmat_register_target_perf(target, access: ACCESS_COORDINATE_CPU);
904 target->registered = true;
905 }
906 mutex_unlock(lock: &target_lock);
907}
908
909static void hmat_register_targets(void)
910{
911 struct memory_target *target;
912
913 list_for_each_entry(target, &targets, node)
914 hmat_register_target(target);
915}
916
917static int hmat_callback(struct notifier_block *self,
918 unsigned long action, void *arg)
919{
920 struct memory_target *target;
921 struct memory_notify *mnb = arg;
922 int pxm, nid = mnb->status_change_nid;
923
924 if (nid == NUMA_NO_NODE || action != MEM_ONLINE)
925 return NOTIFY_OK;
926
927 pxm = node_to_pxm(nid);
928 target = find_mem_target(mem_pxm: pxm);
929 if (!target)
930 return NOTIFY_OK;
931
932 hmat_register_target(target);
933 return NOTIFY_OK;
934}
935
936static int hmat_set_default_dram_perf(void)
937{
938 int rc;
939 int nid, pxm;
940 struct memory_target *target;
941 struct access_coordinate *attrs;
942
943 if (!default_dram_type)
944 return -EIO;
945
946 for_each_node_mask(nid, default_dram_type->nodes) {
947 pxm = node_to_pxm(nid);
948 target = find_mem_target(mem_pxm: pxm);
949 if (!target)
950 continue;
951 attrs = &target->coord[1];
952 rc = mt_set_default_dram_perf(nid, perf: attrs, source: "ACPI HMAT");
953 if (rc)
954 return rc;
955 }
956
957 return 0;
958}
959
960static int hmat_calculate_adistance(struct notifier_block *self,
961 unsigned long nid, void *data)
962{
963 static DECLARE_BITMAP(p_nodes, MAX_NUMNODES);
964 struct memory_target *target;
965 struct access_coordinate *perf;
966 int *adist = data;
967 int pxm;
968
969 pxm = node_to_pxm(nid);
970 target = find_mem_target(mem_pxm: pxm);
971 if (!target)
972 return NOTIFY_OK;
973
974 mutex_lock(&target_lock);
975 hmat_update_target_attrs(target, p_nodes, access: ACCESS_COORDINATE_CPU);
976 mutex_unlock(lock: &target_lock);
977
978 perf = &target->coord[1];
979
980 if (mt_perf_to_adistance(perf, adist))
981 return NOTIFY_OK;
982
983 return NOTIFY_STOP;
984}
985
986static struct notifier_block hmat_adist_nb __meminitdata = {
987 .notifier_call = hmat_calculate_adistance,
988 .priority = 100,
989};
990
991static __init void hmat_free_structures(void)
992{
993 struct memory_target *target, *tnext;
994 struct memory_locality *loc, *lnext;
995 struct memory_initiator *initiator, *inext;
996 struct target_cache *tcache, *cnext;
997
998 list_for_each_entry_safe(target, tnext, &targets, node) {
999 struct resource *res, *res_next;
1000
1001 list_for_each_entry_safe(tcache, cnext, &target->caches, node) {
1002 list_del(entry: &tcache->node);
1003 kfree(objp: tcache);
1004 }
1005
1006 list_del(entry: &target->node);
1007 res = target->memregions.child;
1008 while (res) {
1009 res_next = res->sibling;
1010 __release_region(&target->memregions, res->start,
1011 resource_size(res));
1012 res = res_next;
1013 }
1014 kfree(objp: target);
1015 }
1016
1017 list_for_each_entry_safe(initiator, inext, &initiators, node) {
1018 list_del(entry: &initiator->node);
1019 kfree(objp: initiator);
1020 }
1021
1022 list_for_each_entry_safe(loc, lnext, &localities, node) {
1023 list_del(entry: &loc->node);
1024 kfree(objp: loc);
1025 }
1026}
1027
1028static __init int hmat_init(void)
1029{
1030 struct acpi_table_header *tbl;
1031 enum acpi_hmat_type i;
1032 acpi_status status;
1033
1034 if (srat_disabled() || hmat_disable)
1035 return 0;
1036
1037 status = acpi_get_table(ACPI_SIG_SRAT, instance: 0, out_table: &tbl);
1038 if (ACPI_FAILURE(status))
1039 return 0;
1040
1041 if (acpi_table_parse_entries(ACPI_SIG_SRAT,
1042 table_size: sizeof(struct acpi_table_srat),
1043 entry_id: ACPI_SRAT_TYPE_MEMORY_AFFINITY,
1044 handler: srat_parse_mem_affinity, max_entries: 0) < 0)
1045 goto out_put;
1046
1047 if (acpi_table_parse_entries(ACPI_SIG_SRAT,
1048 table_size: sizeof(struct acpi_table_srat),
1049 entry_id: ACPI_SRAT_TYPE_GENERIC_PORT_AFFINITY,
1050 handler: srat_parse_genport_affinity, max_entries: 0) < 0)
1051 goto out_put;
1052
1053 acpi_put_table(table: tbl);
1054
1055 status = acpi_get_table(ACPI_SIG_HMAT, instance: 0, out_table: &tbl);
1056 if (ACPI_FAILURE(status))
1057 goto out_put;
1058
1059 hmat_revision = tbl->revision;
1060 switch (hmat_revision) {
1061 case 1:
1062 case 2:
1063 break;
1064 default:
1065 pr_notice("Ignoring: Unknown revision:%d\n", hmat_revision);
1066 goto out_put;
1067 }
1068
1069 for (i = ACPI_HMAT_TYPE_PROXIMITY; i < ACPI_HMAT_TYPE_RESERVED; i++) {
1070 if (acpi_table_parse_entries(ACPI_SIG_HMAT,
1071 table_size: sizeof(struct acpi_table_hmat), entry_id: i,
1072 handler: hmat_parse_subtable, max_entries: 0) < 0) {
1073 pr_notice("Ignoring: Invalid table");
1074 goto out_put;
1075 }
1076 }
1077 hmat_register_targets();
1078
1079 /* Keep the table and structures if the notifier may use them */
1080 if (hotplug_memory_notifier(hmat_callback, HMAT_CALLBACK_PRI))
1081 goto out_put;
1082
1083 if (!hmat_set_default_dram_perf())
1084 register_mt_adistance_algorithm(nb: &hmat_adist_nb);
1085
1086 return 0;
1087out_put:
1088 hmat_free_structures();
1089 acpi_put_table(table: tbl);
1090 return 0;
1091}
1092subsys_initcall(hmat_init);
1093

source code of linux/drivers/acpi/numa/hmat.c