1 | /* |
2 | * This file is subject to the terms and conditions of the GNU General Public |
3 | * License. See the file "COPYING" in the main directory of this archive |
4 | * for more details. |
5 | * |
6 | * Copyright (C) 2004-2008, 2009, 2010 Cavium Networks |
7 | */ |
8 | #include <linux/cpu.h> |
9 | #include <linux/delay.h> |
10 | #include <linux/smp.h> |
11 | #include <linux/interrupt.h> |
12 | #include <linux/kernel_stat.h> |
13 | #include <linux/sched.h> |
14 | #include <linux/sched/hotplug.h> |
15 | #include <linux/sched/task_stack.h> |
16 | #include <linux/init.h> |
17 | #include <linux/export.h> |
18 | #include <linux/kexec.h> |
19 | |
20 | #include <asm/mmu_context.h> |
21 | #include <asm/time.h> |
22 | #include <asm/setup.h> |
23 | #include <asm/smp.h> |
24 | |
25 | #include <asm/octeon/octeon.h> |
26 | |
27 | #include "octeon_boot.h" |
28 | |
29 | volatile unsigned long octeon_processor_boot = 0xff; |
30 | volatile unsigned long octeon_processor_sp; |
31 | volatile unsigned long octeon_processor_gp; |
32 | #ifdef CONFIG_RELOCATABLE |
33 | volatile unsigned long octeon_processor_relocated_kernel_entry; |
34 | #endif /* CONFIG_RELOCATABLE */ |
35 | |
36 | #ifdef CONFIG_HOTPLUG_CPU |
37 | uint64_t octeon_bootloader_entry_addr; |
38 | EXPORT_SYMBOL(octeon_bootloader_entry_addr); |
39 | #endif |
40 | |
41 | extern void kernel_entry(unsigned long arg1, ...); |
42 | |
43 | static void octeon_icache_flush(void) |
44 | { |
45 | asm volatile ("synci 0($0)\n" ); |
46 | } |
47 | |
48 | static void (*octeon_message_functions[8])(void) = { |
49 | scheduler_ipi, |
50 | generic_smp_call_function_interrupt, |
51 | octeon_icache_flush, |
52 | }; |
53 | |
54 | static irqreturn_t mailbox_interrupt(int irq, void *dev_id) |
55 | { |
56 | u64 mbox_clrx = CVMX_CIU_MBOX_CLRX(cvmx_get_core_num()); |
57 | u64 action; |
58 | int i; |
59 | |
60 | /* |
61 | * Make sure the function array initialization remains |
62 | * correct. |
63 | */ |
64 | BUILD_BUG_ON(SMP_RESCHEDULE_YOURSELF != (1 << 0)); |
65 | BUILD_BUG_ON(SMP_CALL_FUNCTION != (1 << 1)); |
66 | BUILD_BUG_ON(SMP_ICACHE_FLUSH != (1 << 2)); |
67 | |
68 | /* |
69 | * Load the mailbox register to figure out what we're supposed |
70 | * to do. |
71 | */ |
72 | action = cvmx_read_csr(mbox_clrx); |
73 | |
74 | if (OCTEON_IS_MODEL(OCTEON_CN68XX)) |
75 | action &= 0xff; |
76 | else |
77 | action &= 0xffff; |
78 | |
79 | /* Clear the mailbox to clear the interrupt */ |
80 | cvmx_write_csr(mbox_clrx, action); |
81 | |
82 | for (i = 0; i < ARRAY_SIZE(octeon_message_functions) && action;) { |
83 | if (action & 1) { |
84 | void (*fn)(void) = octeon_message_functions[i]; |
85 | |
86 | if (fn) |
87 | fn(); |
88 | } |
89 | action >>= 1; |
90 | i++; |
91 | } |
92 | return IRQ_HANDLED; |
93 | } |
94 | |
95 | /* |
96 | * Cause the function described by call_data to be executed on the passed |
97 | * cpu. When the function has finished, increment the finished field of |
98 | * call_data. |
99 | */ |
100 | void octeon_send_ipi_single(int cpu, unsigned int action) |
101 | { |
102 | int coreid = cpu_logical_map(cpu); |
103 | /* |
104 | pr_info("SMP: Mailbox send cpu=%d, coreid=%d, action=%u\n", cpu, |
105 | coreid, action); |
106 | */ |
107 | cvmx_write_csr(CVMX_CIU_MBOX_SETX(coreid), action); |
108 | } |
109 | |
110 | static inline void octeon_send_ipi_mask(const struct cpumask *mask, |
111 | unsigned int action) |
112 | { |
113 | unsigned int i; |
114 | |
115 | for_each_cpu(i, mask) |
116 | octeon_send_ipi_single(cpu: i, action); |
117 | } |
118 | |
119 | /* |
120 | * Detect available CPUs, populate cpu_possible_mask |
121 | */ |
122 | static void octeon_smp_hotplug_setup(void) |
123 | { |
124 | #ifdef CONFIG_HOTPLUG_CPU |
125 | struct linux_app_boot_info *labi; |
126 | |
127 | if (!setup_max_cpus) |
128 | return; |
129 | |
130 | labi = (struct linux_app_boot_info *)PHYS_TO_XKSEG_CACHED(LABI_ADDR_IN_BOOTLOADER); |
131 | if (labi->labi_signature != LABI_SIGNATURE) { |
132 | pr_info("The bootloader on this board does not support HOTPLUG_CPU." ); |
133 | return; |
134 | } |
135 | |
136 | octeon_bootloader_entry_addr = labi->InitTLBStart_addr; |
137 | #endif |
138 | } |
139 | |
140 | static void __init octeon_smp_setup(void) |
141 | { |
142 | const int coreid = cvmx_get_core_num(); |
143 | int cpus; |
144 | int id; |
145 | struct cvmx_sysinfo *sysinfo = cvmx_sysinfo_get(); |
146 | |
147 | #ifdef CONFIG_HOTPLUG_CPU |
148 | int core_mask = octeon_get_boot_coremask(); |
149 | unsigned int num_cores = cvmx_octeon_num_cores(); |
150 | #endif |
151 | |
152 | /* The present CPUs are initially just the boot cpu (CPU 0). */ |
153 | for (id = 0; id < NR_CPUS; id++) { |
154 | set_cpu_possible(cpu: id, possible: id == 0); |
155 | set_cpu_present(cpu: id, present: id == 0); |
156 | } |
157 | |
158 | __cpu_number_map[coreid] = 0; |
159 | __cpu_logical_map[0] = coreid; |
160 | |
161 | /* The present CPUs get the lowest CPU numbers. */ |
162 | cpus = 1; |
163 | for (id = 0; id < NR_CPUS; id++) { |
164 | if ((id != coreid) && cvmx_coremask_is_core_set(&sysinfo->core_mask, id)) { |
165 | set_cpu_possible(cpu: cpus, possible: true); |
166 | set_cpu_present(cpu: cpus, present: true); |
167 | __cpu_number_map[id] = cpus; |
168 | __cpu_logical_map[cpus] = id; |
169 | cpus++; |
170 | } |
171 | } |
172 | |
173 | #ifdef CONFIG_HOTPLUG_CPU |
174 | /* |
175 | * The possible CPUs are all those present on the chip. We |
176 | * will assign CPU numbers for possible cores as well. Cores |
177 | * are always consecutively numberd from 0. |
178 | */ |
179 | for (id = 0; setup_max_cpus && octeon_bootloader_entry_addr && |
180 | id < num_cores && id < NR_CPUS; id++) { |
181 | if (!(core_mask & (1 << id))) { |
182 | set_cpu_possible(cpu: cpus, possible: true); |
183 | __cpu_number_map[id] = cpus; |
184 | __cpu_logical_map[cpus] = id; |
185 | cpus++; |
186 | } |
187 | } |
188 | #endif |
189 | |
190 | octeon_smp_hotplug_setup(); |
191 | } |
192 | |
193 | |
194 | #ifdef CONFIG_RELOCATABLE |
195 | int plat_post_relocation(long offset) |
196 | { |
197 | unsigned long entry = (unsigned long)kernel_entry; |
198 | |
199 | /* Send secondaries into relocated kernel */ |
200 | octeon_processor_relocated_kernel_entry = entry + offset; |
201 | |
202 | return 0; |
203 | } |
204 | #endif /* CONFIG_RELOCATABLE */ |
205 | |
206 | /* |
207 | * Firmware CPU startup hook |
208 | */ |
209 | static int octeon_boot_secondary(int cpu, struct task_struct *idle) |
210 | { |
211 | int count; |
212 | |
213 | pr_info("SMP: Booting CPU%02d (CoreId %2d)...\n" , cpu, |
214 | cpu_logical_map(cpu)); |
215 | |
216 | octeon_processor_sp = __KSTK_TOS(idle); |
217 | octeon_processor_gp = (unsigned long)(task_thread_info(idle)); |
218 | octeon_processor_boot = cpu_logical_map(cpu); |
219 | mb(); |
220 | |
221 | count = 10000; |
222 | while (octeon_processor_sp && count) { |
223 | /* Waiting for processor to get the SP and GP */ |
224 | udelay(1); |
225 | count--; |
226 | } |
227 | if (count == 0) { |
228 | pr_err("Secondary boot timeout\n" ); |
229 | return -ETIMEDOUT; |
230 | } |
231 | |
232 | return 0; |
233 | } |
234 | |
235 | /* |
236 | * After we've done initial boot, this function is called to allow the |
237 | * board code to clean up state, if needed |
238 | */ |
239 | static void octeon_init_secondary(void) |
240 | { |
241 | unsigned int sr; |
242 | |
243 | sr = set_c0_status(ST0_BEV); |
244 | write_c0_ebase((u32)ebase); |
245 | write_c0_status(sr); |
246 | |
247 | octeon_check_cpu_bist(); |
248 | octeon_init_cvmcount(); |
249 | |
250 | octeon_irq_setup_secondary(); |
251 | } |
252 | |
253 | /* |
254 | * Callout to firmware before smp_init |
255 | */ |
256 | static void __init octeon_prepare_cpus(unsigned int max_cpus) |
257 | { |
258 | /* |
259 | * Only the low order mailbox bits are used for IPIs, leave |
260 | * the other bits alone. |
261 | */ |
262 | cvmx_write_csr(CVMX_CIU_MBOX_CLRX(cvmx_get_core_num()), 0xffff); |
263 | if (request_irq(irq: OCTEON_IRQ_MBOX0, handler: mailbox_interrupt, |
264 | IRQF_PERCPU | IRQF_NO_THREAD, name: "SMP-IPI" , |
265 | dev: mailbox_interrupt)) { |
266 | panic(fmt: "Cannot request_irq(OCTEON_IRQ_MBOX0)" ); |
267 | } |
268 | } |
269 | |
270 | /* |
271 | * Last chance for the board code to finish SMP initialization before |
272 | * the CPU is "online". |
273 | */ |
274 | static void octeon_smp_finish(void) |
275 | { |
276 | octeon_user_io_init(); |
277 | |
278 | /* to generate the first CPU timer interrupt */ |
279 | write_c0_compare(read_c0_count() + mips_hpt_frequency / HZ); |
280 | local_irq_enable(); |
281 | } |
282 | |
283 | #ifdef CONFIG_HOTPLUG_CPU |
284 | |
285 | /* State of each CPU. */ |
286 | static DEFINE_PER_CPU(int, cpu_state); |
287 | |
288 | static int octeon_cpu_disable(void) |
289 | { |
290 | unsigned int cpu = smp_processor_id(); |
291 | |
292 | if (!octeon_bootloader_entry_addr) |
293 | return -ENOTSUPP; |
294 | |
295 | set_cpu_online(cpu, online: false); |
296 | calculate_cpu_foreign_map(); |
297 | octeon_fixup_irqs(); |
298 | |
299 | __flush_cache_all(); |
300 | local_flush_tlb_all(); |
301 | |
302 | return 0; |
303 | } |
304 | |
305 | static void octeon_cpu_die(unsigned int cpu) |
306 | { |
307 | int coreid = cpu_logical_map(cpu); |
308 | uint32_t mask, new_mask; |
309 | const struct cvmx_bootmem_named_block_desc *block_desc; |
310 | |
311 | while (per_cpu(cpu_state, cpu) != CPU_DEAD) |
312 | cpu_relax(); |
313 | |
314 | /* |
315 | * This is a bit complicated strategics of getting/settig available |
316 | * cores mask, copied from bootloader |
317 | */ |
318 | |
319 | mask = 1 << coreid; |
320 | /* LINUX_APP_BOOT_BLOCK is initialized in bootoct binary */ |
321 | block_desc = cvmx_bootmem_find_named_block(LINUX_APP_BOOT_BLOCK_NAME); |
322 | |
323 | if (!block_desc) { |
324 | struct linux_app_boot_info *labi; |
325 | |
326 | labi = (struct linux_app_boot_info *)PHYS_TO_XKSEG_CACHED(LABI_ADDR_IN_BOOTLOADER); |
327 | |
328 | labi->avail_coremask |= mask; |
329 | new_mask = labi->avail_coremask; |
330 | } else { /* alternative, already initialized */ |
331 | uint32_t *p = (uint32_t *)PHYS_TO_XKSEG_CACHED(block_desc->base_addr + |
332 | AVAIL_COREMASK_OFFSET_IN_LINUX_APP_BOOT_BLOCK); |
333 | *p |= mask; |
334 | new_mask = *p; |
335 | } |
336 | |
337 | pr_info("Reset core %d. Available Coremask = 0x%x \n" , coreid, new_mask); |
338 | mb(); |
339 | cvmx_write_csr(CVMX_CIU_PP_RST, 1 << coreid); |
340 | cvmx_write_csr(CVMX_CIU_PP_RST, 0); |
341 | } |
342 | |
343 | void play_dead(void) |
344 | { |
345 | int cpu = cpu_number_map(cvmx_get_core_num()); |
346 | |
347 | idle_task_exit(); |
348 | cpuhp_ap_report_dead(); |
349 | octeon_processor_boot = 0xff; |
350 | per_cpu(cpu_state, cpu) = CPU_DEAD; |
351 | |
352 | mb(); |
353 | |
354 | while (1) /* core will be reset here */ |
355 | ; |
356 | } |
357 | |
358 | static void start_after_reset(void) |
359 | { |
360 | kernel_entry(arg1: 0, 0, 0); /* set a2 = 0 for secondary core */ |
361 | } |
362 | |
363 | static int octeon_update_boot_vector(unsigned int cpu) |
364 | { |
365 | |
366 | int coreid = cpu_logical_map(cpu); |
367 | uint32_t avail_coremask; |
368 | const struct cvmx_bootmem_named_block_desc *block_desc; |
369 | struct boot_init_vector *boot_vect = |
370 | (struct boot_init_vector *)PHYS_TO_XKSEG_CACHED(BOOTLOADER_BOOT_VECTOR); |
371 | |
372 | block_desc = cvmx_bootmem_find_named_block(LINUX_APP_BOOT_BLOCK_NAME); |
373 | |
374 | if (!block_desc) { |
375 | struct linux_app_boot_info *labi; |
376 | |
377 | labi = (struct linux_app_boot_info *)PHYS_TO_XKSEG_CACHED(LABI_ADDR_IN_BOOTLOADER); |
378 | |
379 | avail_coremask = labi->avail_coremask; |
380 | labi->avail_coremask &= ~(1 << coreid); |
381 | } else { /* alternative, already initialized */ |
382 | avail_coremask = *(uint32_t *)PHYS_TO_XKSEG_CACHED( |
383 | block_desc->base_addr + AVAIL_COREMASK_OFFSET_IN_LINUX_APP_BOOT_BLOCK); |
384 | } |
385 | |
386 | if (!(avail_coremask & (1 << coreid))) { |
387 | /* core not available, assume, that caught by simple-executive */ |
388 | cvmx_write_csr(CVMX_CIU_PP_RST, 1 << coreid); |
389 | cvmx_write_csr(CVMX_CIU_PP_RST, 0); |
390 | } |
391 | |
392 | boot_vect[coreid].app_start_func_addr = |
393 | (uint32_t) (unsigned long) start_after_reset; |
394 | boot_vect[coreid].code_addr = octeon_bootloader_entry_addr; |
395 | |
396 | mb(); |
397 | |
398 | cvmx_write_csr(CVMX_CIU_NMI, (1 << coreid) & avail_coremask); |
399 | |
400 | return 0; |
401 | } |
402 | |
403 | static int register_cavium_notifier(void) |
404 | { |
405 | return cpuhp_setup_state_nocalls(state: CPUHP_MIPS_SOC_PREPARE, |
406 | name: "mips/cavium:prepare" , |
407 | startup: octeon_update_boot_vector, NULL); |
408 | } |
409 | late_initcall(register_cavium_notifier); |
410 | |
411 | #endif /* CONFIG_HOTPLUG_CPU */ |
412 | |
413 | static const struct plat_smp_ops octeon_smp_ops = { |
414 | .send_ipi_single = octeon_send_ipi_single, |
415 | .send_ipi_mask = octeon_send_ipi_mask, |
416 | .init_secondary = octeon_init_secondary, |
417 | .smp_finish = octeon_smp_finish, |
418 | .boot_secondary = octeon_boot_secondary, |
419 | .smp_setup = octeon_smp_setup, |
420 | .prepare_cpus = octeon_prepare_cpus, |
421 | #ifdef CONFIG_HOTPLUG_CPU |
422 | .cpu_disable = octeon_cpu_disable, |
423 | .cpu_die = octeon_cpu_die, |
424 | #endif |
425 | #ifdef CONFIG_KEXEC_CORE |
426 | .kexec_nonboot_cpu = kexec_nonboot_cpu_jump, |
427 | #endif |
428 | }; |
429 | |
430 | static irqreturn_t octeon_78xx_reched_interrupt(int irq, void *dev_id) |
431 | { |
432 | scheduler_ipi(); |
433 | return IRQ_HANDLED; |
434 | } |
435 | |
436 | static irqreturn_t octeon_78xx_call_function_interrupt(int irq, void *dev_id) |
437 | { |
438 | generic_smp_call_function_interrupt(); |
439 | return IRQ_HANDLED; |
440 | } |
441 | |
442 | static irqreturn_t octeon_78xx_icache_flush_interrupt(int irq, void *dev_id) |
443 | { |
444 | octeon_icache_flush(); |
445 | return IRQ_HANDLED; |
446 | } |
447 | |
448 | /* |
449 | * Callout to firmware before smp_init |
450 | */ |
451 | static void octeon_78xx_prepare_cpus(unsigned int max_cpus) |
452 | { |
453 | if (request_irq(OCTEON_IRQ_MBOX0 + 0, |
454 | octeon_78xx_reched_interrupt, |
455 | IRQF_PERCPU | IRQF_NO_THREAD, "Scheduler" , |
456 | octeon_78xx_reched_interrupt)) { |
457 | panic(fmt: "Cannot request_irq for SchedulerIPI" ); |
458 | } |
459 | if (request_irq(OCTEON_IRQ_MBOX0 + 1, |
460 | octeon_78xx_call_function_interrupt, |
461 | IRQF_PERCPU | IRQF_NO_THREAD, "SMP-Call" , |
462 | octeon_78xx_call_function_interrupt)) { |
463 | panic(fmt: "Cannot request_irq for SMP-Call" ); |
464 | } |
465 | if (request_irq(OCTEON_IRQ_MBOX0 + 2, |
466 | octeon_78xx_icache_flush_interrupt, |
467 | IRQF_PERCPU | IRQF_NO_THREAD, "ICache-Flush" , |
468 | octeon_78xx_icache_flush_interrupt)) { |
469 | panic(fmt: "Cannot request_irq for ICache-Flush" ); |
470 | } |
471 | } |
472 | |
473 | static void octeon_78xx_send_ipi_single(int cpu, unsigned int action) |
474 | { |
475 | int i; |
476 | |
477 | for (i = 0; i < 8; i++) { |
478 | if (action & 1) |
479 | octeon_ciu3_mbox_send(cpu, i); |
480 | action >>= 1; |
481 | } |
482 | } |
483 | |
484 | static void octeon_78xx_send_ipi_mask(const struct cpumask *mask, |
485 | unsigned int action) |
486 | { |
487 | unsigned int cpu; |
488 | |
489 | for_each_cpu(cpu, mask) |
490 | octeon_78xx_send_ipi_single(cpu, action); |
491 | } |
492 | |
493 | static const struct plat_smp_ops octeon_78xx_smp_ops = { |
494 | .send_ipi_single = octeon_78xx_send_ipi_single, |
495 | .send_ipi_mask = octeon_78xx_send_ipi_mask, |
496 | .init_secondary = octeon_init_secondary, |
497 | .smp_finish = octeon_smp_finish, |
498 | .boot_secondary = octeon_boot_secondary, |
499 | .smp_setup = octeon_smp_setup, |
500 | .prepare_cpus = octeon_78xx_prepare_cpus, |
501 | #ifdef CONFIG_HOTPLUG_CPU |
502 | .cpu_disable = octeon_cpu_disable, |
503 | .cpu_die = octeon_cpu_die, |
504 | #endif |
505 | #ifdef CONFIG_KEXEC_CORE |
506 | .kexec_nonboot_cpu = kexec_nonboot_cpu_jump, |
507 | #endif |
508 | }; |
509 | |
510 | void __init octeon_setup_smp(void) |
511 | { |
512 | const struct plat_smp_ops *ops; |
513 | |
514 | if (octeon_has_feature(OCTEON_FEATURE_CIU3)) |
515 | ops = &octeon_78xx_smp_ops; |
516 | else |
517 | ops = &octeon_smp_ops; |
518 | |
519 | register_smp_ops(ops); |
520 | } |
521 | |