1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * APEI Generic Hardware Error Source support
4 *
5 * Generic Hardware Error Source provides a way to report platform
6 * hardware errors (such as that from chipset). It works in so called
7 * "Firmware First" mode, that is, hardware errors are reported to
8 * firmware firstly, then reported to Linux by firmware. This way,
9 * some non-standard hardware error registers or non-standard hardware
10 * link can be checked by firmware to produce more hardware error
11 * information for Linux.
12 *
13 * For more information about Generic Hardware Error Source, please
14 * refer to ACPI Specification version 4.0, section 17.3.2.6
15 *
16 * Copyright 2010,2011 Intel Corp.
17 * Author: Huang Ying <ying.huang@intel.com>
18 */
19
20#include <linux/arm_sdei.h>
21#include <linux/kernel.h>
22#include <linux/moduleparam.h>
23#include <linux/init.h>
24#include <linux/acpi.h>
25#include <linux/io.h>
26#include <linux/interrupt.h>
27#include <linux/timer.h>
28#include <linux/cper.h>
29#include <linux/platform_device.h>
30#include <linux/mutex.h>
31#include <linux/ratelimit.h>
32#include <linux/vmalloc.h>
33#include <linux/irq_work.h>
34#include <linux/llist.h>
35#include <linux/genalloc.h>
36#include <linux/pci.h>
37#include <linux/pfn.h>
38#include <linux/aer.h>
39#include <linux/nmi.h>
40#include <linux/sched/clock.h>
41#include <linux/uuid.h>
42#include <linux/ras.h>
43#include <linux/task_work.h>
44
45#include <acpi/actbl1.h>
46#include <acpi/ghes.h>
47#include <acpi/apei.h>
48#include <asm/fixmap.h>
49#include <asm/tlbflush.h>
50#include <ras/ras_event.h>
51
52#include "apei-internal.h"
53
54#define GHES_PFX "GHES: "
55
56#define GHES_ESTATUS_MAX_SIZE 65536
57#define GHES_ESOURCE_PREALLOC_MAX_SIZE 65536
58
59#define GHES_ESTATUS_POOL_MIN_ALLOC_ORDER 3
60
61/* This is just an estimation for memory pool allocation */
62#define GHES_ESTATUS_CACHE_AVG_SIZE 512
63
64#define GHES_ESTATUS_CACHES_SIZE 4
65
66#define GHES_ESTATUS_IN_CACHE_MAX_NSEC 10000000000ULL
67/* Prevent too many caches are allocated because of RCU */
68#define GHES_ESTATUS_CACHE_ALLOCED_MAX (GHES_ESTATUS_CACHES_SIZE * 3 / 2)
69
70#define GHES_ESTATUS_CACHE_LEN(estatus_len) \
71 (sizeof(struct ghes_estatus_cache) + (estatus_len))
72#define GHES_ESTATUS_FROM_CACHE(estatus_cache) \
73 ((struct acpi_hest_generic_status *) \
74 ((struct ghes_estatus_cache *)(estatus_cache) + 1))
75
76#define GHES_ESTATUS_NODE_LEN(estatus_len) \
77 (sizeof(struct ghes_estatus_node) + (estatus_len))
78#define GHES_ESTATUS_FROM_NODE(estatus_node) \
79 ((struct acpi_hest_generic_status *) \
80 ((struct ghes_estatus_node *)(estatus_node) + 1))
81
82#define GHES_VENDOR_ENTRY_LEN(gdata_len) \
83 (sizeof(struct ghes_vendor_record_entry) + (gdata_len))
84#define GHES_GDATA_FROM_VENDOR_ENTRY(vendor_entry) \
85 ((struct acpi_hest_generic_data *) \
86 ((struct ghes_vendor_record_entry *)(vendor_entry) + 1))
87
88/*
89 * NMI-like notifications vary by architecture, before the compiler can prune
90 * unused static functions it needs a value for these enums.
91 */
92#ifndef CONFIG_ARM_SDE_INTERFACE
93#define FIX_APEI_GHES_SDEI_NORMAL __end_of_fixed_addresses
94#define FIX_APEI_GHES_SDEI_CRITICAL __end_of_fixed_addresses
95#endif
96
97static ATOMIC_NOTIFIER_HEAD(ghes_report_chain);
98
99static inline bool is_hest_type_generic_v2(struct ghes *ghes)
100{
101 return ghes->generic->header.type == ACPI_HEST_TYPE_GENERIC_ERROR_V2;
102}
103
104/*
105 * This driver isn't really modular, however for the time being,
106 * continuing to use module_param is the easiest way to remain
107 * compatible with existing boot arg use cases.
108 */
109bool ghes_disable;
110module_param_named(disable, ghes_disable, bool, 0);
111
112/*
113 * "ghes.edac_force_enable" forcibly enables ghes_edac and skips the platform
114 * check.
115 */
116static bool ghes_edac_force_enable;
117module_param_named(edac_force_enable, ghes_edac_force_enable, bool, 0);
118
119/*
120 * All error sources notified with HED (Hardware Error Device) share a
121 * single notifier callback, so they need to be linked and checked one
122 * by one. This holds true for NMI too.
123 *
124 * RCU is used for these lists, so ghes_list_mutex is only used for
125 * list changing, not for traversing.
126 */
127static LIST_HEAD(ghes_hed);
128static DEFINE_MUTEX(ghes_list_mutex);
129
130/*
131 * A list of GHES devices which are given to the corresponding EDAC driver
132 * ghes_edac for further use.
133 */
134static LIST_HEAD(ghes_devs);
135static DEFINE_MUTEX(ghes_devs_mutex);
136
137/*
138 * Because the memory area used to transfer hardware error information
139 * from BIOS to Linux can be determined only in NMI, IRQ or timer
140 * handler, but general ioremap can not be used in atomic context, so
141 * the fixmap is used instead.
142 *
143 * This spinlock is used to prevent the fixmap entry from being used
144 * simultaneously.
145 */
146static DEFINE_SPINLOCK(ghes_notify_lock_irq);
147
148struct ghes_vendor_record_entry {
149 struct work_struct work;
150 int error_severity;
151 char vendor_record[];
152};
153
154static struct gen_pool *ghes_estatus_pool;
155
156static struct ghes_estatus_cache __rcu *ghes_estatus_caches[GHES_ESTATUS_CACHES_SIZE];
157static atomic_t ghes_estatus_cache_alloced;
158
159static int ghes_panic_timeout __read_mostly = 30;
160
161static void __iomem *ghes_map(u64 pfn, enum fixed_addresses fixmap_idx)
162{
163 phys_addr_t paddr;
164 pgprot_t prot;
165
166 paddr = PFN_PHYS(pfn);
167 prot = arch_apei_get_mem_attribute(addr: paddr);
168 __set_fixmap(idx: fixmap_idx, phys: paddr, flags: prot);
169
170 return (void __iomem *) __fix_to_virt(fixmap_idx);
171}
172
173static void ghes_unmap(void __iomem *vaddr, enum fixed_addresses fixmap_idx)
174{
175 int _idx = virt_to_fix(vaddr: (unsigned long)vaddr);
176
177 WARN_ON_ONCE(fixmap_idx != _idx);
178 clear_fixmap(fixmap_idx);
179}
180
181int ghes_estatus_pool_init(unsigned int num_ghes)
182{
183 unsigned long addr, len;
184 int rc;
185
186 ghes_estatus_pool = gen_pool_create(GHES_ESTATUS_POOL_MIN_ALLOC_ORDER, -1);
187 if (!ghes_estatus_pool)
188 return -ENOMEM;
189
190 len = GHES_ESTATUS_CACHE_AVG_SIZE * GHES_ESTATUS_CACHE_ALLOCED_MAX;
191 len += (num_ghes * GHES_ESOURCE_PREALLOC_MAX_SIZE);
192
193 addr = (unsigned long)vmalloc(PAGE_ALIGN(len));
194 if (!addr)
195 goto err_pool_alloc;
196
197 rc = gen_pool_add(pool: ghes_estatus_pool, addr, PAGE_ALIGN(len), nid: -1);
198 if (rc)
199 goto err_pool_add;
200
201 return 0;
202
203err_pool_add:
204 vfree(addr: (void *)addr);
205
206err_pool_alloc:
207 gen_pool_destroy(ghes_estatus_pool);
208
209 return -ENOMEM;
210}
211
212/**
213 * ghes_estatus_pool_region_free - free previously allocated memory
214 * from the ghes_estatus_pool.
215 * @addr: address of memory to free.
216 * @size: size of memory to free.
217 *
218 * Returns none.
219 */
220void ghes_estatus_pool_region_free(unsigned long addr, u32 size)
221{
222 gen_pool_free(pool: ghes_estatus_pool, addr, size);
223}
224EXPORT_SYMBOL_GPL(ghes_estatus_pool_region_free);
225
226static int map_gen_v2(struct ghes *ghes)
227{
228 return apei_map_generic_address(reg: &ghes->generic_v2->read_ack_register);
229}
230
231static void unmap_gen_v2(struct ghes *ghes)
232{
233 apei_unmap_generic_address(reg: &ghes->generic_v2->read_ack_register);
234}
235
236static void ghes_ack_error(struct acpi_hest_generic_v2 *gv2)
237{
238 int rc;
239 u64 val = 0;
240
241 rc = apei_read(val: &val, reg: &gv2->read_ack_register);
242 if (rc)
243 return;
244
245 val &= gv2->read_ack_preserve << gv2->read_ack_register.bit_offset;
246 val |= gv2->read_ack_write << gv2->read_ack_register.bit_offset;
247
248 apei_write(val, reg: &gv2->read_ack_register);
249}
250
251static struct ghes *ghes_new(struct acpi_hest_generic *generic)
252{
253 struct ghes *ghes;
254 unsigned int error_block_length;
255 int rc;
256
257 ghes = kzalloc(size: sizeof(*ghes), GFP_KERNEL);
258 if (!ghes)
259 return ERR_PTR(error: -ENOMEM);
260
261 ghes->generic = generic;
262 if (is_hest_type_generic_v2(ghes)) {
263 rc = map_gen_v2(ghes);
264 if (rc)
265 goto err_free;
266 }
267
268 rc = apei_map_generic_address(reg: &generic->error_status_address);
269 if (rc)
270 goto err_unmap_read_ack_addr;
271 error_block_length = generic->error_block_length;
272 if (error_block_length > GHES_ESTATUS_MAX_SIZE) {
273 pr_warn(FW_WARN GHES_PFX
274 "Error status block length is too long: %u for "
275 "generic hardware error source: %d.\n",
276 error_block_length, generic->header.source_id);
277 error_block_length = GHES_ESTATUS_MAX_SIZE;
278 }
279 ghes->estatus = kmalloc(size: error_block_length, GFP_KERNEL);
280 if (!ghes->estatus) {
281 rc = -ENOMEM;
282 goto err_unmap_status_addr;
283 }
284
285 return ghes;
286
287err_unmap_status_addr:
288 apei_unmap_generic_address(reg: &generic->error_status_address);
289err_unmap_read_ack_addr:
290 if (is_hest_type_generic_v2(ghes))
291 unmap_gen_v2(ghes);
292err_free:
293 kfree(objp: ghes);
294 return ERR_PTR(error: rc);
295}
296
297static void ghes_fini(struct ghes *ghes)
298{
299 kfree(objp: ghes->estatus);
300 apei_unmap_generic_address(reg: &ghes->generic->error_status_address);
301 if (is_hest_type_generic_v2(ghes))
302 unmap_gen_v2(ghes);
303}
304
305static inline int ghes_severity(int severity)
306{
307 switch (severity) {
308 case CPER_SEV_INFORMATIONAL:
309 return GHES_SEV_NO;
310 case CPER_SEV_CORRECTED:
311 return GHES_SEV_CORRECTED;
312 case CPER_SEV_RECOVERABLE:
313 return GHES_SEV_RECOVERABLE;
314 case CPER_SEV_FATAL:
315 return GHES_SEV_PANIC;
316 default:
317 /* Unknown, go panic */
318 return GHES_SEV_PANIC;
319 }
320}
321
322static void ghes_copy_tofrom_phys(void *buffer, u64 paddr, u32 len,
323 int from_phys,
324 enum fixed_addresses fixmap_idx)
325{
326 void __iomem *vaddr;
327 u64 offset;
328 u32 trunk;
329
330 while (len > 0) {
331 offset = paddr - (paddr & PAGE_MASK);
332 vaddr = ghes_map(PHYS_PFN(paddr), fixmap_idx);
333 trunk = PAGE_SIZE - offset;
334 trunk = min(trunk, len);
335 if (from_phys)
336 memcpy_fromio(buffer, vaddr + offset, trunk);
337 else
338 memcpy_toio(vaddr + offset, buffer, trunk);
339 len -= trunk;
340 paddr += trunk;
341 buffer += trunk;
342 ghes_unmap(vaddr, fixmap_idx);
343 }
344}
345
346/* Check the top-level record header has an appropriate size. */
347static int __ghes_check_estatus(struct ghes *ghes,
348 struct acpi_hest_generic_status *estatus)
349{
350 u32 len = cper_estatus_len(estatus);
351
352 if (len < sizeof(*estatus)) {
353 pr_warn_ratelimited(FW_WARN GHES_PFX "Truncated error status block!\n");
354 return -EIO;
355 }
356
357 if (len > ghes->generic->error_block_length) {
358 pr_warn_ratelimited(FW_WARN GHES_PFX "Invalid error status block length!\n");
359 return -EIO;
360 }
361
362 if (cper_estatus_check_header(estatus)) {
363 pr_warn_ratelimited(FW_WARN GHES_PFX "Invalid CPER header!\n");
364 return -EIO;
365 }
366
367 return 0;
368}
369
370/* Read the CPER block, returning its address, and header in estatus. */
371static int __ghes_peek_estatus(struct ghes *ghes,
372 struct acpi_hest_generic_status *estatus,
373 u64 *buf_paddr, enum fixed_addresses fixmap_idx)
374{
375 struct acpi_hest_generic *g = ghes->generic;
376 int rc;
377
378 rc = apei_read(val: buf_paddr, reg: &g->error_status_address);
379 if (rc) {
380 *buf_paddr = 0;
381 pr_warn_ratelimited(FW_WARN GHES_PFX
382"Failed to read error status block address for hardware error source: %d.\n",
383 g->header.source_id);
384 return -EIO;
385 }
386 if (!*buf_paddr)
387 return -ENOENT;
388
389 ghes_copy_tofrom_phys(buffer: estatus, paddr: *buf_paddr, len: sizeof(*estatus), from_phys: 1,
390 fixmap_idx);
391 if (!estatus->block_status) {
392 *buf_paddr = 0;
393 return -ENOENT;
394 }
395
396 return 0;
397}
398
399static int __ghes_read_estatus(struct acpi_hest_generic_status *estatus,
400 u64 buf_paddr, enum fixed_addresses fixmap_idx,
401 size_t buf_len)
402{
403 ghes_copy_tofrom_phys(buffer: estatus, paddr: buf_paddr, len: buf_len, from_phys: 1, fixmap_idx);
404 if (cper_estatus_check(estatus)) {
405 pr_warn_ratelimited(FW_WARN GHES_PFX
406 "Failed to read error status block!\n");
407 return -EIO;
408 }
409
410 return 0;
411}
412
413static int ghes_read_estatus(struct ghes *ghes,
414 struct acpi_hest_generic_status *estatus,
415 u64 *buf_paddr, enum fixed_addresses fixmap_idx)
416{
417 int rc;
418
419 rc = __ghes_peek_estatus(ghes, estatus, buf_paddr, fixmap_idx);
420 if (rc)
421 return rc;
422
423 rc = __ghes_check_estatus(ghes, estatus);
424 if (rc)
425 return rc;
426
427 return __ghes_read_estatus(estatus, buf_paddr: *buf_paddr, fixmap_idx,
428 buf_len: cper_estatus_len(estatus));
429}
430
431static void ghes_clear_estatus(struct ghes *ghes,
432 struct acpi_hest_generic_status *estatus,
433 u64 buf_paddr, enum fixed_addresses fixmap_idx)
434{
435 estatus->block_status = 0;
436
437 if (!buf_paddr)
438 return;
439
440 ghes_copy_tofrom_phys(buffer: estatus, paddr: buf_paddr,
441 len: sizeof(estatus->block_status), from_phys: 0,
442 fixmap_idx);
443
444 /*
445 * GHESv2 type HEST entries introduce support for error acknowledgment,
446 * so only acknowledge the error if this support is present.
447 */
448 if (is_hest_type_generic_v2(ghes))
449 ghes_ack_error(gv2: ghes->generic_v2);
450}
451
452/*
453 * Called as task_work before returning to user-space.
454 * Ensure any queued work has been done before we return to the context that
455 * triggered the notification.
456 */
457static void ghes_kick_task_work(struct callback_head *head)
458{
459 struct acpi_hest_generic_status *estatus;
460 struct ghes_estatus_node *estatus_node;
461 u32 node_len;
462
463 estatus_node = container_of(head, struct ghes_estatus_node, task_work);
464 if (IS_ENABLED(CONFIG_ACPI_APEI_MEMORY_FAILURE))
465 memory_failure_queue_kick(cpu: estatus_node->task_work_cpu);
466
467 estatus = GHES_ESTATUS_FROM_NODE(estatus_node);
468 node_len = GHES_ESTATUS_NODE_LEN(cper_estatus_len(estatus));
469 gen_pool_free(pool: ghes_estatus_pool, addr: (unsigned long)estatus_node, size: node_len);
470}
471
472static bool ghes_do_memory_failure(u64 physical_addr, int flags)
473{
474 unsigned long pfn;
475
476 if (!IS_ENABLED(CONFIG_ACPI_APEI_MEMORY_FAILURE))
477 return false;
478
479 pfn = PHYS_PFN(physical_addr);
480 if (!pfn_valid(pfn) && !arch_is_platform_page(paddr: physical_addr)) {
481 pr_warn_ratelimited(FW_WARN GHES_PFX
482 "Invalid address in generic error data: %#llx\n",
483 physical_addr);
484 return false;
485 }
486
487 memory_failure_queue(pfn, flags);
488 return true;
489}
490
491static bool ghes_handle_memory_failure(struct acpi_hest_generic_data *gdata,
492 int sev)
493{
494 int flags = -1;
495 int sec_sev = ghes_severity(severity: gdata->error_severity);
496 struct cper_sec_mem_err *mem_err = acpi_hest_get_payload(gdata);
497
498 if (!(mem_err->validation_bits & CPER_MEM_VALID_PA))
499 return false;
500
501 /* iff following two events can be handled properly by now */
502 if (sec_sev == GHES_SEV_CORRECTED &&
503 (gdata->flags & CPER_SEC_ERROR_THRESHOLD_EXCEEDED))
504 flags = MF_SOFT_OFFLINE;
505 if (sev == GHES_SEV_RECOVERABLE && sec_sev == GHES_SEV_RECOVERABLE)
506 flags = 0;
507
508 if (flags != -1)
509 return ghes_do_memory_failure(physical_addr: mem_err->physical_addr, flags);
510
511 return false;
512}
513
514static bool ghes_handle_arm_hw_error(struct acpi_hest_generic_data *gdata, int sev)
515{
516 struct cper_sec_proc_arm *err = acpi_hest_get_payload(gdata);
517 bool queued = false;
518 int sec_sev, i;
519 char *p;
520
521 log_arm_hw_error(err);
522
523 sec_sev = ghes_severity(severity: gdata->error_severity);
524 if (sev != GHES_SEV_RECOVERABLE || sec_sev != GHES_SEV_RECOVERABLE)
525 return false;
526
527 p = (char *)(err + 1);
528 for (i = 0; i < err->err_info_num; i++) {
529 struct cper_arm_err_info *err_info = (struct cper_arm_err_info *)p;
530 bool is_cache = (err_info->type == CPER_ARM_CACHE_ERROR);
531 bool has_pa = (err_info->validation_bits & CPER_ARM_INFO_VALID_PHYSICAL_ADDR);
532 const char *error_type = "unknown error";
533
534 /*
535 * The field (err_info->error_info & BIT(26)) is fixed to set to
536 * 1 in some old firmware of HiSilicon Kunpeng920. We assume that
537 * firmware won't mix corrected errors in an uncorrected section,
538 * and don't filter out 'corrected' error here.
539 */
540 if (is_cache && has_pa) {
541 queued = ghes_do_memory_failure(physical_addr: err_info->physical_fault_addr, flags: 0);
542 p += err_info->length;
543 continue;
544 }
545
546 if (err_info->type < ARRAY_SIZE(cper_proc_error_type_strs))
547 error_type = cper_proc_error_type_strs[err_info->type];
548
549 pr_warn_ratelimited(FW_WARN GHES_PFX
550 "Unhandled processor error type: %s\n",
551 error_type);
552 p += err_info->length;
553 }
554
555 return queued;
556}
557
558/*
559 * PCIe AER errors need to be sent to the AER driver for reporting and
560 * recovery. The GHES severities map to the following AER severities and
561 * require the following handling:
562 *
563 * GHES_SEV_CORRECTABLE -> AER_CORRECTABLE
564 * These need to be reported by the AER driver but no recovery is
565 * necessary.
566 * GHES_SEV_RECOVERABLE -> AER_NONFATAL
567 * GHES_SEV_RECOVERABLE && CPER_SEC_RESET -> AER_FATAL
568 * These both need to be reported and recovered from by the AER driver.
569 * GHES_SEV_PANIC does not make it to this handling since the kernel must
570 * panic.
571 */
572static void ghes_handle_aer(struct acpi_hest_generic_data *gdata)
573{
574#ifdef CONFIG_ACPI_APEI_PCIEAER
575 struct cper_sec_pcie *pcie_err = acpi_hest_get_payload(gdata);
576
577 if (pcie_err->validation_bits & CPER_PCIE_VALID_DEVICE_ID &&
578 pcie_err->validation_bits & CPER_PCIE_VALID_AER_INFO) {
579 unsigned int devfn;
580 int aer_severity;
581 u8 *aer_info;
582
583 devfn = PCI_DEVFN(pcie_err->device_id.device,
584 pcie_err->device_id.function);
585 aer_severity = cper_severity_to_aer(cper_severity: gdata->error_severity);
586
587 /*
588 * If firmware reset the component to contain
589 * the error, we must reinitialize it before
590 * use, so treat it as a fatal AER error.
591 */
592 if (gdata->flags & CPER_SEC_RESET)
593 aer_severity = AER_FATAL;
594
595 aer_info = (void *)gen_pool_alloc(pool: ghes_estatus_pool,
596 size: sizeof(struct aer_capability_regs));
597 if (!aer_info)
598 return;
599 memcpy(aer_info, pcie_err->aer_info, sizeof(struct aer_capability_regs));
600
601 aer_recover_queue(domain: pcie_err->device_id.segment,
602 bus: pcie_err->device_id.bus,
603 devfn, severity: aer_severity,
604 aer_regs: (struct aer_capability_regs *)
605 aer_info);
606 }
607#endif
608}
609
610static BLOCKING_NOTIFIER_HEAD(vendor_record_notify_list);
611
612int ghes_register_vendor_record_notifier(struct notifier_block *nb)
613{
614 return blocking_notifier_chain_register(nh: &vendor_record_notify_list, nb);
615}
616EXPORT_SYMBOL_GPL(ghes_register_vendor_record_notifier);
617
618void ghes_unregister_vendor_record_notifier(struct notifier_block *nb)
619{
620 blocking_notifier_chain_unregister(nh: &vendor_record_notify_list, nb);
621}
622EXPORT_SYMBOL_GPL(ghes_unregister_vendor_record_notifier);
623
624static void ghes_vendor_record_work_func(struct work_struct *work)
625{
626 struct ghes_vendor_record_entry *entry;
627 struct acpi_hest_generic_data *gdata;
628 u32 len;
629
630 entry = container_of(work, struct ghes_vendor_record_entry, work);
631 gdata = GHES_GDATA_FROM_VENDOR_ENTRY(entry);
632
633 blocking_notifier_call_chain(nh: &vendor_record_notify_list,
634 val: entry->error_severity, v: gdata);
635
636 len = GHES_VENDOR_ENTRY_LEN(acpi_hest_get_record_size(gdata));
637 gen_pool_free(pool: ghes_estatus_pool, addr: (unsigned long)entry, size: len);
638}
639
640static void ghes_defer_non_standard_event(struct acpi_hest_generic_data *gdata,
641 int sev)
642{
643 struct acpi_hest_generic_data *copied_gdata;
644 struct ghes_vendor_record_entry *entry;
645 u32 len;
646
647 len = GHES_VENDOR_ENTRY_LEN(acpi_hest_get_record_size(gdata));
648 entry = (void *)gen_pool_alloc(pool: ghes_estatus_pool, size: len);
649 if (!entry)
650 return;
651
652 copied_gdata = GHES_GDATA_FROM_VENDOR_ENTRY(entry);
653 memcpy(copied_gdata, gdata, acpi_hest_get_record_size(gdata));
654 entry->error_severity = sev;
655
656 INIT_WORK(&entry->work, ghes_vendor_record_work_func);
657 schedule_work(work: &entry->work);
658}
659
660static bool ghes_do_proc(struct ghes *ghes,
661 const struct acpi_hest_generic_status *estatus)
662{
663 int sev, sec_sev;
664 struct acpi_hest_generic_data *gdata;
665 guid_t *sec_type;
666 const guid_t *fru_id = &guid_null;
667 char *fru_text = "";
668 bool queued = false;
669
670 sev = ghes_severity(severity: estatus->error_severity);
671 apei_estatus_for_each_section(estatus, gdata) {
672 sec_type = (guid_t *)gdata->section_type;
673 sec_sev = ghes_severity(severity: gdata->error_severity);
674 if (gdata->validation_bits & CPER_SEC_VALID_FRU_ID)
675 fru_id = (guid_t *)gdata->fru_id;
676
677 if (gdata->validation_bits & CPER_SEC_VALID_FRU_TEXT)
678 fru_text = gdata->fru_text;
679
680 if (guid_equal(u1: sec_type, u2: &CPER_SEC_PLATFORM_MEM)) {
681 struct cper_sec_mem_err *mem_err = acpi_hest_get_payload(gdata);
682
683 atomic_notifier_call_chain(nh: &ghes_report_chain, val: sev, v: mem_err);
684
685 arch_apei_report_mem_error(sev, mem_err);
686 queued = ghes_handle_memory_failure(gdata, sev);
687 }
688 else if (guid_equal(u1: sec_type, u2: &CPER_SEC_PCIE)) {
689 ghes_handle_aer(gdata);
690 }
691 else if (guid_equal(u1: sec_type, u2: &CPER_SEC_PROC_ARM)) {
692 queued = ghes_handle_arm_hw_error(gdata, sev);
693 } else {
694 void *err = acpi_hest_get_payload(gdata);
695
696 ghes_defer_non_standard_event(gdata, sev);
697 log_non_standard_event(sec_type, fru_id, fru_text,
698 sev: sec_sev, err,
699 len: gdata->error_data_length);
700 }
701 }
702
703 return queued;
704}
705
706static void __ghes_print_estatus(const char *pfx,
707 const struct acpi_hest_generic *generic,
708 const struct acpi_hest_generic_status *estatus)
709{
710 static atomic_t seqno;
711 unsigned int curr_seqno;
712 char pfx_seq[64];
713
714 if (pfx == NULL) {
715 if (ghes_severity(severity: estatus->error_severity) <=
716 GHES_SEV_CORRECTED)
717 pfx = KERN_WARNING;
718 else
719 pfx = KERN_ERR;
720 }
721 curr_seqno = atomic_inc_return(v: &seqno);
722 snprintf(buf: pfx_seq, size: sizeof(pfx_seq), fmt: "%s{%u}" HW_ERR, pfx, curr_seqno);
723 printk("%s""Hardware error from APEI Generic Hardware Error Source: %d\n",
724 pfx_seq, generic->header.source_id);
725 cper_estatus_print(pfx: pfx_seq, estatus);
726}
727
728static int ghes_print_estatus(const char *pfx,
729 const struct acpi_hest_generic *generic,
730 const struct acpi_hest_generic_status *estatus)
731{
732 /* Not more than 2 messages every 5 seconds */
733 static DEFINE_RATELIMIT_STATE(ratelimit_corrected, 5*HZ, 2);
734 static DEFINE_RATELIMIT_STATE(ratelimit_uncorrected, 5*HZ, 2);
735 struct ratelimit_state *ratelimit;
736
737 if (ghes_severity(severity: estatus->error_severity) <= GHES_SEV_CORRECTED)
738 ratelimit = &ratelimit_corrected;
739 else
740 ratelimit = &ratelimit_uncorrected;
741 if (__ratelimit(ratelimit)) {
742 __ghes_print_estatus(pfx, generic, estatus);
743 return 1;
744 }
745 return 0;
746}
747
748/*
749 * GHES error status reporting throttle, to report more kinds of
750 * errors, instead of just most frequently occurred errors.
751 */
752static int ghes_estatus_cached(struct acpi_hest_generic_status *estatus)
753{
754 u32 len;
755 int i, cached = 0;
756 unsigned long long now;
757 struct ghes_estatus_cache *cache;
758 struct acpi_hest_generic_status *cache_estatus;
759
760 len = cper_estatus_len(estatus);
761 rcu_read_lock();
762 for (i = 0; i < GHES_ESTATUS_CACHES_SIZE; i++) {
763 cache = rcu_dereference(ghes_estatus_caches[i]);
764 if (cache == NULL)
765 continue;
766 if (len != cache->estatus_len)
767 continue;
768 cache_estatus = GHES_ESTATUS_FROM_CACHE(cache);
769 if (memcmp(p: estatus, q: cache_estatus, size: len))
770 continue;
771 atomic_inc(v: &cache->count);
772 now = sched_clock();
773 if (now - cache->time_in < GHES_ESTATUS_IN_CACHE_MAX_NSEC)
774 cached = 1;
775 break;
776 }
777 rcu_read_unlock();
778 return cached;
779}
780
781static struct ghes_estatus_cache *ghes_estatus_cache_alloc(
782 struct acpi_hest_generic *generic,
783 struct acpi_hest_generic_status *estatus)
784{
785 int alloced;
786 u32 len, cache_len;
787 struct ghes_estatus_cache *cache;
788 struct acpi_hest_generic_status *cache_estatus;
789
790 alloced = atomic_add_return(i: 1, v: &ghes_estatus_cache_alloced);
791 if (alloced > GHES_ESTATUS_CACHE_ALLOCED_MAX) {
792 atomic_dec(v: &ghes_estatus_cache_alloced);
793 return NULL;
794 }
795 len = cper_estatus_len(estatus);
796 cache_len = GHES_ESTATUS_CACHE_LEN(len);
797 cache = (void *)gen_pool_alloc(pool: ghes_estatus_pool, size: cache_len);
798 if (!cache) {
799 atomic_dec(v: &ghes_estatus_cache_alloced);
800 return NULL;
801 }
802 cache_estatus = GHES_ESTATUS_FROM_CACHE(cache);
803 memcpy(cache_estatus, estatus, len);
804 cache->estatus_len = len;
805 atomic_set(v: &cache->count, i: 0);
806 cache->generic = generic;
807 cache->time_in = sched_clock();
808 return cache;
809}
810
811static void ghes_estatus_cache_rcu_free(struct rcu_head *head)
812{
813 struct ghes_estatus_cache *cache;
814 u32 len;
815
816 cache = container_of(head, struct ghes_estatus_cache, rcu);
817 len = cper_estatus_len(GHES_ESTATUS_FROM_CACHE(cache));
818 len = GHES_ESTATUS_CACHE_LEN(len);
819 gen_pool_free(pool: ghes_estatus_pool, addr: (unsigned long)cache, size: len);
820 atomic_dec(v: &ghes_estatus_cache_alloced);
821}
822
823static void
824ghes_estatus_cache_add(struct acpi_hest_generic *generic,
825 struct acpi_hest_generic_status *estatus)
826{
827 unsigned long long now, duration, period, max_period = 0;
828 struct ghes_estatus_cache *cache, *new_cache;
829 struct ghes_estatus_cache __rcu *victim;
830 int i, slot = -1, count;
831
832 new_cache = ghes_estatus_cache_alloc(generic, estatus);
833 if (!new_cache)
834 return;
835
836 rcu_read_lock();
837 now = sched_clock();
838 for (i = 0; i < GHES_ESTATUS_CACHES_SIZE; i++) {
839 cache = rcu_dereference(ghes_estatus_caches[i]);
840 if (cache == NULL) {
841 slot = i;
842 break;
843 }
844 duration = now - cache->time_in;
845 if (duration >= GHES_ESTATUS_IN_CACHE_MAX_NSEC) {
846 slot = i;
847 break;
848 }
849 count = atomic_read(v: &cache->count);
850 period = duration;
851 do_div(period, (count + 1));
852 if (period > max_period) {
853 max_period = period;
854 slot = i;
855 }
856 }
857 rcu_read_unlock();
858
859 if (slot != -1) {
860 /*
861 * Use release semantics to ensure that ghes_estatus_cached()
862 * running on another CPU will see the updated cache fields if
863 * it can see the new value of the pointer.
864 */
865 victim = xchg_release(&ghes_estatus_caches[slot],
866 RCU_INITIALIZER(new_cache));
867
868 /*
869 * At this point, victim may point to a cached item different
870 * from the one based on which we selected the slot. Instead of
871 * going to the loop again to pick another slot, let's just
872 * drop the other item anyway: this may cause a false cache
873 * miss later on, but that won't cause any problems.
874 */
875 if (victim)
876 call_rcu(head: &unrcu_pointer(victim)->rcu,
877 func: ghes_estatus_cache_rcu_free);
878 }
879}
880
881static void __ghes_panic(struct ghes *ghes,
882 struct acpi_hest_generic_status *estatus,
883 u64 buf_paddr, enum fixed_addresses fixmap_idx)
884{
885 __ghes_print_estatus(KERN_EMERG, generic: ghes->generic, estatus);
886
887 ghes_clear_estatus(ghes, estatus, buf_paddr, fixmap_idx);
888
889 /* reboot to log the error! */
890 if (!panic_timeout)
891 panic_timeout = ghes_panic_timeout;
892 panic(fmt: "Fatal hardware error!");
893}
894
895static int ghes_proc(struct ghes *ghes)
896{
897 struct acpi_hest_generic_status *estatus = ghes->estatus;
898 u64 buf_paddr;
899 int rc;
900
901 rc = ghes_read_estatus(ghes, estatus, buf_paddr: &buf_paddr, fixmap_idx: FIX_APEI_GHES_IRQ);
902 if (rc)
903 goto out;
904
905 if (ghes_severity(severity: estatus->error_severity) >= GHES_SEV_PANIC)
906 __ghes_panic(ghes, estatus, buf_paddr, fixmap_idx: FIX_APEI_GHES_IRQ);
907
908 if (!ghes_estatus_cached(estatus)) {
909 if (ghes_print_estatus(NULL, generic: ghes->generic, estatus))
910 ghes_estatus_cache_add(generic: ghes->generic, estatus);
911 }
912 ghes_do_proc(ghes, estatus);
913
914out:
915 ghes_clear_estatus(ghes, estatus, buf_paddr, fixmap_idx: FIX_APEI_GHES_IRQ);
916
917 return rc;
918}
919
920static void ghes_add_timer(struct ghes *ghes)
921{
922 struct acpi_hest_generic *g = ghes->generic;
923 unsigned long expire;
924
925 if (!g->notify.poll_interval) {
926 pr_warn(FW_WARN GHES_PFX "Poll interval is 0 for generic hardware error source: %d, disabled.\n",
927 g->header.source_id);
928 return;
929 }
930 expire = jiffies + msecs_to_jiffies(m: g->notify.poll_interval);
931 ghes->timer.expires = round_jiffies_relative(j: expire);
932 add_timer(timer: &ghes->timer);
933}
934
935static void ghes_poll_func(struct timer_list *t)
936{
937 struct ghes *ghes = from_timer(ghes, t, timer);
938 unsigned long flags;
939
940 spin_lock_irqsave(&ghes_notify_lock_irq, flags);
941 ghes_proc(ghes);
942 spin_unlock_irqrestore(lock: &ghes_notify_lock_irq, flags);
943 if (!(ghes->flags & GHES_EXITING))
944 ghes_add_timer(ghes);
945}
946
947static irqreturn_t ghes_irq_func(int irq, void *data)
948{
949 struct ghes *ghes = data;
950 unsigned long flags;
951 int rc;
952
953 spin_lock_irqsave(&ghes_notify_lock_irq, flags);
954 rc = ghes_proc(ghes);
955 spin_unlock_irqrestore(lock: &ghes_notify_lock_irq, flags);
956 if (rc)
957 return IRQ_NONE;
958
959 return IRQ_HANDLED;
960}
961
962static int ghes_notify_hed(struct notifier_block *this, unsigned long event,
963 void *data)
964{
965 struct ghes *ghes;
966 unsigned long flags;
967 int ret = NOTIFY_DONE;
968
969 spin_lock_irqsave(&ghes_notify_lock_irq, flags);
970 rcu_read_lock();
971 list_for_each_entry_rcu(ghes, &ghes_hed, list) {
972 if (!ghes_proc(ghes))
973 ret = NOTIFY_OK;
974 }
975 rcu_read_unlock();
976 spin_unlock_irqrestore(lock: &ghes_notify_lock_irq, flags);
977
978 return ret;
979}
980
981static struct notifier_block ghes_notifier_hed = {
982 .notifier_call = ghes_notify_hed,
983};
984
985/*
986 * Handlers for CPER records may not be NMI safe. For example,
987 * memory_failure_queue() takes spinlocks and calls schedule_work_on().
988 * In any NMI-like handler, memory from ghes_estatus_pool is used to save
989 * estatus, and added to the ghes_estatus_llist. irq_work_queue() causes
990 * ghes_proc_in_irq() to run in IRQ context where each estatus in
991 * ghes_estatus_llist is processed.
992 *
993 * Memory from the ghes_estatus_pool is also used with the ghes_estatus_cache
994 * to suppress frequent messages.
995 */
996static struct llist_head ghes_estatus_llist;
997static struct irq_work ghes_proc_irq_work;
998
999static void ghes_proc_in_irq(struct irq_work *irq_work)
1000{
1001 struct llist_node *llnode, *next;
1002 struct ghes_estatus_node *estatus_node;
1003 struct acpi_hest_generic *generic;
1004 struct acpi_hest_generic_status *estatus;
1005 bool task_work_pending;
1006 u32 len, node_len;
1007 int ret;
1008
1009 llnode = llist_del_all(head: &ghes_estatus_llist);
1010 /*
1011 * Because the time order of estatus in list is reversed,
1012 * revert it back to proper order.
1013 */
1014 llnode = llist_reverse_order(head: llnode);
1015 while (llnode) {
1016 next = llnode->next;
1017 estatus_node = llist_entry(llnode, struct ghes_estatus_node,
1018 llnode);
1019 estatus = GHES_ESTATUS_FROM_NODE(estatus_node);
1020 len = cper_estatus_len(estatus);
1021 node_len = GHES_ESTATUS_NODE_LEN(len);
1022 task_work_pending = ghes_do_proc(ghes: estatus_node->ghes, estatus);
1023 if (!ghes_estatus_cached(estatus)) {
1024 generic = estatus_node->generic;
1025 if (ghes_print_estatus(NULL, generic, estatus))
1026 ghes_estatus_cache_add(generic, estatus);
1027 }
1028
1029 if (task_work_pending && current->mm) {
1030 estatus_node->task_work.func = ghes_kick_task_work;
1031 estatus_node->task_work_cpu = smp_processor_id();
1032 ret = task_work_add(current, twork: &estatus_node->task_work,
1033 mode: TWA_RESUME);
1034 if (ret)
1035 estatus_node->task_work.func = NULL;
1036 }
1037
1038 if (!estatus_node->task_work.func)
1039 gen_pool_free(pool: ghes_estatus_pool,
1040 addr: (unsigned long)estatus_node, size: node_len);
1041
1042 llnode = next;
1043 }
1044}
1045
1046static void ghes_print_queued_estatus(void)
1047{
1048 struct llist_node *llnode;
1049 struct ghes_estatus_node *estatus_node;
1050 struct acpi_hest_generic *generic;
1051 struct acpi_hest_generic_status *estatus;
1052
1053 llnode = llist_del_all(head: &ghes_estatus_llist);
1054 /*
1055 * Because the time order of estatus in list is reversed,
1056 * revert it back to proper order.
1057 */
1058 llnode = llist_reverse_order(head: llnode);
1059 while (llnode) {
1060 estatus_node = llist_entry(llnode, struct ghes_estatus_node,
1061 llnode);
1062 estatus = GHES_ESTATUS_FROM_NODE(estatus_node);
1063 generic = estatus_node->generic;
1064 ghes_print_estatus(NULL, generic, estatus);
1065 llnode = llnode->next;
1066 }
1067}
1068
1069static int ghes_in_nmi_queue_one_entry(struct ghes *ghes,
1070 enum fixed_addresses fixmap_idx)
1071{
1072 struct acpi_hest_generic_status *estatus, tmp_header;
1073 struct ghes_estatus_node *estatus_node;
1074 u32 len, node_len;
1075 u64 buf_paddr;
1076 int sev, rc;
1077
1078 if (!IS_ENABLED(CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG))
1079 return -EOPNOTSUPP;
1080
1081 rc = __ghes_peek_estatus(ghes, estatus: &tmp_header, buf_paddr: &buf_paddr, fixmap_idx);
1082 if (rc) {
1083 ghes_clear_estatus(ghes, estatus: &tmp_header, buf_paddr, fixmap_idx);
1084 return rc;
1085 }
1086
1087 rc = __ghes_check_estatus(ghes, estatus: &tmp_header);
1088 if (rc) {
1089 ghes_clear_estatus(ghes, estatus: &tmp_header, buf_paddr, fixmap_idx);
1090 return rc;
1091 }
1092
1093 len = cper_estatus_len(estatus: &tmp_header);
1094 node_len = GHES_ESTATUS_NODE_LEN(len);
1095 estatus_node = (void *)gen_pool_alloc(pool: ghes_estatus_pool, size: node_len);
1096 if (!estatus_node)
1097 return -ENOMEM;
1098
1099 estatus_node->ghes = ghes;
1100 estatus_node->generic = ghes->generic;
1101 estatus_node->task_work.func = NULL;
1102 estatus = GHES_ESTATUS_FROM_NODE(estatus_node);
1103
1104 if (__ghes_read_estatus(estatus, buf_paddr, fixmap_idx, buf_len: len)) {
1105 ghes_clear_estatus(ghes, estatus, buf_paddr, fixmap_idx);
1106 rc = -ENOENT;
1107 goto no_work;
1108 }
1109
1110 sev = ghes_severity(severity: estatus->error_severity);
1111 if (sev >= GHES_SEV_PANIC) {
1112 ghes_print_queued_estatus();
1113 __ghes_panic(ghes, estatus, buf_paddr, fixmap_idx);
1114 }
1115
1116 ghes_clear_estatus(ghes, estatus: &tmp_header, buf_paddr, fixmap_idx);
1117
1118 /* This error has been reported before, don't process it again. */
1119 if (ghes_estatus_cached(estatus))
1120 goto no_work;
1121
1122 llist_add(new: &estatus_node->llnode, head: &ghes_estatus_llist);
1123
1124 return rc;
1125
1126no_work:
1127 gen_pool_free(pool: ghes_estatus_pool, addr: (unsigned long)estatus_node,
1128 size: node_len);
1129
1130 return rc;
1131}
1132
1133static int ghes_in_nmi_spool_from_list(struct list_head *rcu_list,
1134 enum fixed_addresses fixmap_idx)
1135{
1136 int ret = -ENOENT;
1137 struct ghes *ghes;
1138
1139 rcu_read_lock();
1140 list_for_each_entry_rcu(ghes, rcu_list, list) {
1141 if (!ghes_in_nmi_queue_one_entry(ghes, fixmap_idx))
1142 ret = 0;
1143 }
1144 rcu_read_unlock();
1145
1146 if (IS_ENABLED(CONFIG_ARCH_HAVE_NMI_SAFE_CMPXCHG) && !ret)
1147 irq_work_queue(work: &ghes_proc_irq_work);
1148
1149 return ret;
1150}
1151
1152#ifdef CONFIG_ACPI_APEI_SEA
1153static LIST_HEAD(ghes_sea);
1154
1155/*
1156 * Return 0 only if one of the SEA error sources successfully reported an error
1157 * record sent from the firmware.
1158 */
1159int ghes_notify_sea(void)
1160{
1161 static DEFINE_RAW_SPINLOCK(ghes_notify_lock_sea);
1162 int rv;
1163
1164 raw_spin_lock(&ghes_notify_lock_sea);
1165 rv = ghes_in_nmi_spool_from_list(&ghes_sea, FIX_APEI_GHES_SEA);
1166 raw_spin_unlock(&ghes_notify_lock_sea);
1167
1168 return rv;
1169}
1170
1171static void ghes_sea_add(struct ghes *ghes)
1172{
1173 mutex_lock(&ghes_list_mutex);
1174 list_add_rcu(&ghes->list, &ghes_sea);
1175 mutex_unlock(&ghes_list_mutex);
1176}
1177
1178static void ghes_sea_remove(struct ghes *ghes)
1179{
1180 mutex_lock(&ghes_list_mutex);
1181 list_del_rcu(&ghes->list);
1182 mutex_unlock(&ghes_list_mutex);
1183 synchronize_rcu();
1184}
1185#else /* CONFIG_ACPI_APEI_SEA */
1186static inline void ghes_sea_add(struct ghes *ghes) { }
1187static inline void ghes_sea_remove(struct ghes *ghes) { }
1188#endif /* CONFIG_ACPI_APEI_SEA */
1189
1190#ifdef CONFIG_HAVE_ACPI_APEI_NMI
1191/*
1192 * NMI may be triggered on any CPU, so ghes_in_nmi is used for
1193 * having only one concurrent reader.
1194 */
1195static atomic_t ghes_in_nmi = ATOMIC_INIT(0);
1196
1197static LIST_HEAD(ghes_nmi);
1198
1199static int ghes_notify_nmi(unsigned int cmd, struct pt_regs *regs)
1200{
1201 static DEFINE_RAW_SPINLOCK(ghes_notify_lock_nmi);
1202 int ret = NMI_DONE;
1203
1204 if (!atomic_add_unless(v: &ghes_in_nmi, a: 1, u: 1))
1205 return ret;
1206
1207 raw_spin_lock(&ghes_notify_lock_nmi);
1208 if (!ghes_in_nmi_spool_from_list(rcu_list: &ghes_nmi, fixmap_idx: FIX_APEI_GHES_NMI))
1209 ret = NMI_HANDLED;
1210 raw_spin_unlock(&ghes_notify_lock_nmi);
1211
1212 atomic_dec(v: &ghes_in_nmi);
1213 return ret;
1214}
1215
1216static void ghes_nmi_add(struct ghes *ghes)
1217{
1218 mutex_lock(&ghes_list_mutex);
1219 if (list_empty(head: &ghes_nmi))
1220 register_nmi_handler(NMI_LOCAL, ghes_notify_nmi, 0, "ghes");
1221 list_add_rcu(new: &ghes->list, head: &ghes_nmi);
1222 mutex_unlock(lock: &ghes_list_mutex);
1223}
1224
1225static void ghes_nmi_remove(struct ghes *ghes)
1226{
1227 mutex_lock(&ghes_list_mutex);
1228 list_del_rcu(entry: &ghes->list);
1229 if (list_empty(head: &ghes_nmi))
1230 unregister_nmi_handler(NMI_LOCAL, "ghes");
1231 mutex_unlock(lock: &ghes_list_mutex);
1232 /*
1233 * To synchronize with NMI handler, ghes can only be
1234 * freed after NMI handler finishes.
1235 */
1236 synchronize_rcu();
1237}
1238#else /* CONFIG_HAVE_ACPI_APEI_NMI */
1239static inline void ghes_nmi_add(struct ghes *ghes) { }
1240static inline void ghes_nmi_remove(struct ghes *ghes) { }
1241#endif /* CONFIG_HAVE_ACPI_APEI_NMI */
1242
1243static void ghes_nmi_init_cxt(void)
1244{
1245 init_irq_work(work: &ghes_proc_irq_work, func: ghes_proc_in_irq);
1246}
1247
1248static int __ghes_sdei_callback(struct ghes *ghes,
1249 enum fixed_addresses fixmap_idx)
1250{
1251 if (!ghes_in_nmi_queue_one_entry(ghes, fixmap_idx)) {
1252 irq_work_queue(work: &ghes_proc_irq_work);
1253
1254 return 0;
1255 }
1256
1257 return -ENOENT;
1258}
1259
1260static int ghes_sdei_normal_callback(u32 event_num, struct pt_regs *regs,
1261 void *arg)
1262{
1263 static DEFINE_RAW_SPINLOCK(ghes_notify_lock_sdei_normal);
1264 struct ghes *ghes = arg;
1265 int err;
1266
1267 raw_spin_lock(&ghes_notify_lock_sdei_normal);
1268 err = __ghes_sdei_callback(ghes, FIX_APEI_GHES_SDEI_NORMAL);
1269 raw_spin_unlock(&ghes_notify_lock_sdei_normal);
1270
1271 return err;
1272}
1273
1274static int ghes_sdei_critical_callback(u32 event_num, struct pt_regs *regs,
1275 void *arg)
1276{
1277 static DEFINE_RAW_SPINLOCK(ghes_notify_lock_sdei_critical);
1278 struct ghes *ghes = arg;
1279 int err;
1280
1281 raw_spin_lock(&ghes_notify_lock_sdei_critical);
1282 err = __ghes_sdei_callback(ghes, FIX_APEI_GHES_SDEI_CRITICAL);
1283 raw_spin_unlock(&ghes_notify_lock_sdei_critical);
1284
1285 return err;
1286}
1287
1288static int apei_sdei_register_ghes(struct ghes *ghes)
1289{
1290 if (!IS_ENABLED(CONFIG_ARM_SDE_INTERFACE))
1291 return -EOPNOTSUPP;
1292
1293 return sdei_register_ghes(ghes, normal_cb: ghes_sdei_normal_callback,
1294 critical_cb: ghes_sdei_critical_callback);
1295}
1296
1297static int apei_sdei_unregister_ghes(struct ghes *ghes)
1298{
1299 if (!IS_ENABLED(CONFIG_ARM_SDE_INTERFACE))
1300 return -EOPNOTSUPP;
1301
1302 return sdei_unregister_ghes(ghes);
1303}
1304
1305static int ghes_probe(struct platform_device *ghes_dev)
1306{
1307 struct acpi_hest_generic *generic;
1308 struct ghes *ghes = NULL;
1309 unsigned long flags;
1310
1311 int rc = -EINVAL;
1312
1313 generic = *(struct acpi_hest_generic **)ghes_dev->dev.platform_data;
1314 if (!generic->enabled)
1315 return -ENODEV;
1316
1317 switch (generic->notify.type) {
1318 case ACPI_HEST_NOTIFY_POLLED:
1319 case ACPI_HEST_NOTIFY_EXTERNAL:
1320 case ACPI_HEST_NOTIFY_SCI:
1321 case ACPI_HEST_NOTIFY_GSIV:
1322 case ACPI_HEST_NOTIFY_GPIO:
1323 break;
1324
1325 case ACPI_HEST_NOTIFY_SEA:
1326 if (!IS_ENABLED(CONFIG_ACPI_APEI_SEA)) {
1327 pr_warn(GHES_PFX "Generic hardware error source: %d notified via SEA is not supported\n",
1328 generic->header.source_id);
1329 rc = -ENOTSUPP;
1330 goto err;
1331 }
1332 break;
1333 case ACPI_HEST_NOTIFY_NMI:
1334 if (!IS_ENABLED(CONFIG_HAVE_ACPI_APEI_NMI)) {
1335 pr_warn(GHES_PFX "Generic hardware error source: %d notified via NMI interrupt is not supported!\n",
1336 generic->header.source_id);
1337 goto err;
1338 }
1339 break;
1340 case ACPI_HEST_NOTIFY_SOFTWARE_DELEGATED:
1341 if (!IS_ENABLED(CONFIG_ARM_SDE_INTERFACE)) {
1342 pr_warn(GHES_PFX "Generic hardware error source: %d notified via SDE Interface is not supported!\n",
1343 generic->header.source_id);
1344 goto err;
1345 }
1346 break;
1347 case ACPI_HEST_NOTIFY_LOCAL:
1348 pr_warn(GHES_PFX "Generic hardware error source: %d notified via local interrupt is not supported!\n",
1349 generic->header.source_id);
1350 goto err;
1351 default:
1352 pr_warn(FW_WARN GHES_PFX "Unknown notification type: %u for generic hardware error source: %d\n",
1353 generic->notify.type, generic->header.source_id);
1354 goto err;
1355 }
1356
1357 rc = -EIO;
1358 if (generic->error_block_length <
1359 sizeof(struct acpi_hest_generic_status)) {
1360 pr_warn(FW_BUG GHES_PFX "Invalid error block length: %u for generic hardware error source: %d\n",
1361 generic->error_block_length, generic->header.source_id);
1362 goto err;
1363 }
1364 ghes = ghes_new(generic);
1365 if (IS_ERR(ptr: ghes)) {
1366 rc = PTR_ERR(ptr: ghes);
1367 ghes = NULL;
1368 goto err;
1369 }
1370
1371 switch (generic->notify.type) {
1372 case ACPI_HEST_NOTIFY_POLLED:
1373 timer_setup(&ghes->timer, ghes_poll_func, 0);
1374 ghes_add_timer(ghes);
1375 break;
1376 case ACPI_HEST_NOTIFY_EXTERNAL:
1377 /* External interrupt vector is GSI */
1378 rc = acpi_gsi_to_irq(gsi: generic->notify.vector, irq: &ghes->irq);
1379 if (rc) {
1380 pr_err(GHES_PFX "Failed to map GSI to IRQ for generic hardware error source: %d\n",
1381 generic->header.source_id);
1382 goto err;
1383 }
1384 rc = request_irq(irq: ghes->irq, handler: ghes_irq_func, IRQF_SHARED,
1385 name: "GHES IRQ", dev: ghes);
1386 if (rc) {
1387 pr_err(GHES_PFX "Failed to register IRQ for generic hardware error source: %d\n",
1388 generic->header.source_id);
1389 goto err;
1390 }
1391 break;
1392
1393 case ACPI_HEST_NOTIFY_SCI:
1394 case ACPI_HEST_NOTIFY_GSIV:
1395 case ACPI_HEST_NOTIFY_GPIO:
1396 mutex_lock(&ghes_list_mutex);
1397 if (list_empty(head: &ghes_hed))
1398 register_acpi_hed_notifier(nb: &ghes_notifier_hed);
1399 list_add_rcu(new: &ghes->list, head: &ghes_hed);
1400 mutex_unlock(lock: &ghes_list_mutex);
1401 break;
1402
1403 case ACPI_HEST_NOTIFY_SEA:
1404 ghes_sea_add(ghes);
1405 break;
1406 case ACPI_HEST_NOTIFY_NMI:
1407 ghes_nmi_add(ghes);
1408 break;
1409 case ACPI_HEST_NOTIFY_SOFTWARE_DELEGATED:
1410 rc = apei_sdei_register_ghes(ghes);
1411 if (rc)
1412 goto err;
1413 break;
1414 default:
1415 BUG();
1416 }
1417
1418 platform_set_drvdata(pdev: ghes_dev, data: ghes);
1419
1420 ghes->dev = &ghes_dev->dev;
1421
1422 mutex_lock(&ghes_devs_mutex);
1423 list_add_tail(new: &ghes->elist, head: &ghes_devs);
1424 mutex_unlock(lock: &ghes_devs_mutex);
1425
1426 /* Handle any pending errors right away */
1427 spin_lock_irqsave(&ghes_notify_lock_irq, flags);
1428 ghes_proc(ghes);
1429 spin_unlock_irqrestore(lock: &ghes_notify_lock_irq, flags);
1430
1431 return 0;
1432
1433err:
1434 if (ghes) {
1435 ghes_fini(ghes);
1436 kfree(objp: ghes);
1437 }
1438 return rc;
1439}
1440
1441static int ghes_remove(struct platform_device *ghes_dev)
1442{
1443 int rc;
1444 struct ghes *ghes;
1445 struct acpi_hest_generic *generic;
1446
1447 ghes = platform_get_drvdata(pdev: ghes_dev);
1448 generic = ghes->generic;
1449
1450 ghes->flags |= GHES_EXITING;
1451 switch (generic->notify.type) {
1452 case ACPI_HEST_NOTIFY_POLLED:
1453 timer_shutdown_sync(timer: &ghes->timer);
1454 break;
1455 case ACPI_HEST_NOTIFY_EXTERNAL:
1456 free_irq(ghes->irq, ghes);
1457 break;
1458
1459 case ACPI_HEST_NOTIFY_SCI:
1460 case ACPI_HEST_NOTIFY_GSIV:
1461 case ACPI_HEST_NOTIFY_GPIO:
1462 mutex_lock(&ghes_list_mutex);
1463 list_del_rcu(entry: &ghes->list);
1464 if (list_empty(head: &ghes_hed))
1465 unregister_acpi_hed_notifier(nb: &ghes_notifier_hed);
1466 mutex_unlock(lock: &ghes_list_mutex);
1467 synchronize_rcu();
1468 break;
1469
1470 case ACPI_HEST_NOTIFY_SEA:
1471 ghes_sea_remove(ghes);
1472 break;
1473 case ACPI_HEST_NOTIFY_NMI:
1474 ghes_nmi_remove(ghes);
1475 break;
1476 case ACPI_HEST_NOTIFY_SOFTWARE_DELEGATED:
1477 rc = apei_sdei_unregister_ghes(ghes);
1478 if (rc)
1479 return rc;
1480 break;
1481 default:
1482 BUG();
1483 break;
1484 }
1485
1486 ghes_fini(ghes);
1487
1488 mutex_lock(&ghes_devs_mutex);
1489 list_del(entry: &ghes->elist);
1490 mutex_unlock(lock: &ghes_devs_mutex);
1491
1492 kfree(objp: ghes);
1493
1494 return 0;
1495}
1496
1497static struct platform_driver ghes_platform_driver = {
1498 .driver = {
1499 .name = "GHES",
1500 },
1501 .probe = ghes_probe,
1502 .remove = ghes_remove,
1503};
1504
1505void __init acpi_ghes_init(void)
1506{
1507 int rc;
1508
1509 sdei_init();
1510
1511 if (acpi_disabled)
1512 return;
1513
1514 switch (hest_disable) {
1515 case HEST_NOT_FOUND:
1516 return;
1517 case HEST_DISABLED:
1518 pr_info(GHES_PFX "HEST is not enabled!\n");
1519 return;
1520 default:
1521 break;
1522 }
1523
1524 if (ghes_disable) {
1525 pr_info(GHES_PFX "GHES is not enabled!\n");
1526 return;
1527 }
1528
1529 ghes_nmi_init_cxt();
1530
1531 rc = platform_driver_register(&ghes_platform_driver);
1532 if (rc)
1533 return;
1534
1535 rc = apei_osc_setup();
1536 if (rc == 0 && osc_sb_apei_support_acked)
1537 pr_info(GHES_PFX "APEI firmware first mode is enabled by APEI bit and WHEA _OSC.\n");
1538 else if (rc == 0 && !osc_sb_apei_support_acked)
1539 pr_info(GHES_PFX "APEI firmware first mode is enabled by WHEA _OSC.\n");
1540 else if (rc && osc_sb_apei_support_acked)
1541 pr_info(GHES_PFX "APEI firmware first mode is enabled by APEI bit.\n");
1542 else
1543 pr_info(GHES_PFX "Failed to enable APEI firmware first mode.\n");
1544}
1545
1546/*
1547 * Known x86 systems that prefer GHES error reporting:
1548 */
1549static struct acpi_platform_list plat_list[] = {
1550 {"HPE ", "Server ", 0, ACPI_SIG_FADT, all_versions},
1551 { } /* End */
1552};
1553
1554struct list_head *ghes_get_devices(void)
1555{
1556 int idx = -1;
1557
1558 if (IS_ENABLED(CONFIG_X86)) {
1559 idx = acpi_match_platform_list(plat: plat_list);
1560 if (idx < 0) {
1561 if (!ghes_edac_force_enable)
1562 return NULL;
1563
1564 pr_warn_once("Force-loading ghes_edac on an unsupported platform. You're on your own!\n");
1565 }
1566 } else if (list_empty(head: &ghes_devs)) {
1567 return NULL;
1568 }
1569
1570 return &ghes_devs;
1571}
1572EXPORT_SYMBOL_GPL(ghes_get_devices);
1573
1574void ghes_register_report_chain(struct notifier_block *nb)
1575{
1576 atomic_notifier_chain_register(nh: &ghes_report_chain, nb);
1577}
1578EXPORT_SYMBOL_GPL(ghes_register_report_chain);
1579
1580void ghes_unregister_report_chain(struct notifier_block *nb)
1581{
1582 atomic_notifier_chain_unregister(nh: &ghes_report_chain, nb);
1583}
1584EXPORT_SYMBOL_GPL(ghes_unregister_report_chain);
1585

source code of linux/drivers/acpi/apei/ghes.c