| 1 | // SPDX-License-Identifier: GPL-2.0-only |
|---|---|
| 2 | #define pr_fmt(fmt) "APIC: " fmt |
| 3 | |
| 4 | #include <asm/apic.h> |
| 5 | |
| 6 | #include "local.h" |
| 7 | |
| 8 | /* |
| 9 | * Use DEFINE_STATIC_CALL_NULL() to avoid having to provide stub functions |
| 10 | * for each callback. The callbacks are setup during boot and all except |
| 11 | * wait_icr_idle() must be initialized before usage. The IPI wrappers |
| 12 | * use static_call() and not static_call_cond() to catch any fails. |
| 13 | */ |
| 14 | #define DEFINE_APIC_CALL(__cb) \ |
| 15 | DEFINE_STATIC_CALL_NULL(apic_call_##__cb, *apic->__cb) |
| 16 | |
| 17 | DEFINE_APIC_CALL(eoi); |
| 18 | DEFINE_APIC_CALL(native_eoi); |
| 19 | DEFINE_APIC_CALL(icr_read); |
| 20 | DEFINE_APIC_CALL(icr_write); |
| 21 | DEFINE_APIC_CALL(read); |
| 22 | DEFINE_APIC_CALL(send_IPI); |
| 23 | DEFINE_APIC_CALL(send_IPI_mask); |
| 24 | DEFINE_APIC_CALL(send_IPI_mask_allbutself); |
| 25 | DEFINE_APIC_CALL(send_IPI_allbutself); |
| 26 | DEFINE_APIC_CALL(send_IPI_all); |
| 27 | DEFINE_APIC_CALL(send_IPI_self); |
| 28 | DEFINE_APIC_CALL(wait_icr_idle); |
| 29 | DEFINE_APIC_CALL(wakeup_secondary_cpu); |
| 30 | DEFINE_APIC_CALL(wakeup_secondary_cpu_64); |
| 31 | DEFINE_APIC_CALL(write); |
| 32 | |
| 33 | EXPORT_STATIC_CALL_TRAMP_GPL(apic_call_send_IPI_mask); |
| 34 | EXPORT_STATIC_CALL_TRAMP_GPL(apic_call_send_IPI_self); |
| 35 | |
| 36 | /* The container for function call overrides */ |
| 37 | struct apic_override __x86_apic_override __initdata; |
| 38 | |
| 39 | #define apply_override(__cb) \ |
| 40 | if (__x86_apic_override.__cb) \ |
| 41 | apic->__cb = __x86_apic_override.__cb |
| 42 | |
| 43 | static __init void restore_override_callbacks(void) |
| 44 | { |
| 45 | apply_override(eoi); |
| 46 | apply_override(native_eoi); |
| 47 | apply_override(write); |
| 48 | apply_override(read); |
| 49 | apply_override(send_IPI); |
| 50 | apply_override(send_IPI_mask); |
| 51 | apply_override(send_IPI_mask_allbutself); |
| 52 | apply_override(send_IPI_allbutself); |
| 53 | apply_override(send_IPI_all); |
| 54 | apply_override(send_IPI_self); |
| 55 | apply_override(icr_read); |
| 56 | apply_override(icr_write); |
| 57 | apply_override(wakeup_secondary_cpu); |
| 58 | apply_override(wakeup_secondary_cpu_64); |
| 59 | } |
| 60 | |
| 61 | #define update_call(__cb) \ |
| 62 | static_call_update(apic_call_##__cb, *apic->__cb) |
| 63 | |
| 64 | static __init void update_static_calls(void) |
| 65 | { |
| 66 | update_call(eoi); |
| 67 | update_call(native_eoi); |
| 68 | update_call(write); |
| 69 | update_call(read); |
| 70 | update_call(send_IPI); |
| 71 | update_call(send_IPI_mask); |
| 72 | update_call(send_IPI_mask_allbutself); |
| 73 | update_call(send_IPI_allbutself); |
| 74 | update_call(send_IPI_all); |
| 75 | update_call(send_IPI_self); |
| 76 | update_call(icr_read); |
| 77 | update_call(icr_write); |
| 78 | update_call(wait_icr_idle); |
| 79 | update_call(wakeup_secondary_cpu); |
| 80 | update_call(wakeup_secondary_cpu_64); |
| 81 | } |
| 82 | |
| 83 | void __init apic_setup_apic_calls(void) |
| 84 | { |
| 85 | /* Ensure that the default APIC has native_eoi populated */ |
| 86 | apic->native_eoi = apic->eoi; |
| 87 | update_static_calls(); |
| 88 | pr_info("Static calls initialized\n"); |
| 89 | } |
| 90 | |
| 91 | void __init apic_install_driver(struct apic *driver) |
| 92 | { |
| 93 | if (apic == driver) |
| 94 | return; |
| 95 | |
| 96 | apic = driver; |
| 97 | |
| 98 | if (IS_ENABLED(CONFIG_X86_X2APIC) && apic->x2apic_set_max_apicid) |
| 99 | apic->max_apic_id = x2apic_max_apicid; |
| 100 | |
| 101 | /* Copy the original eoi() callback as KVM/HyperV might overwrite it */ |
| 102 | if (!apic->native_eoi) |
| 103 | apic->native_eoi = apic->eoi; |
| 104 | |
| 105 | /* Apply any already installed callback overrides */ |
| 106 | restore_override_callbacks(); |
| 107 | update_static_calls(); |
| 108 | |
| 109 | pr_info("Switched APIC routing to: %s\n", driver->name); |
| 110 | } |
| 111 |