平面设计招聘唐山,百度优化排名,汕头建站网站模板,微网站制作工具当你看到这样的log#xff0c;会不会很慌张#xff1f;竟然由CPU没有启动成功#xff0c;除了什么故障#xff1f;本文将结合我遇到的一个问题#xff0c;将启动过程中bringup secondary cpu的过程分析一下。smp_init代码如下#xff1a;602 void __init smp_init(void)
…当你看到这样的log会不会很慌张竟然由CPU没有启动成功除了什么故障本文将结合我遇到的一个问题将启动过程中bringup secondary cpu的过程分析一下。smp_init代码如下602 void __init smp_init(void)
603 {
604 int num_nodes, num_cpus;
605 unsigned int cpu;
606
607 idle_threads_init();
608 cpuhp_threads_init();
609
610 pr_info(cpuhp-walk:Bringing up secondary CPUs ...\n);
611
612 /* FIXME: This should be done in userspace --RR */
613 for_each_present_cpu(cpu) {
614 if (num_online_cpus() setup_max_cpus)
615 break;
616 pr_info(cpuhp-walk:before bringing up cpu:%d\n, cpu);
617 if (!cpu_online(cpu))
618 cpu_up(cpu);
619 pr_info(cpuhp-walk:after bringing up cpu:%d\n, cpu);
620 }
621
622 num_nodes num_online_nodes();
623 num_cpus num_online_cpus();
624 pr_info(cpuhp-walk:Brought up %d node%s, %d CPU%s\n,
625 num_nodes, (num_nodes 1 ? s : ),
626 num_cpus, (num_cpus 1 ? s : ));
627
628 /* Any cleanup work */
629 smp_cpus_done(setup_max_cpus);
630 }
可以看得出在smp_init中cpu0依次调用cpu_up启动其他的cpu启动完成一个再启动下一个直到所有cpu都启动完毕才继续内核剩余的启动流程整个过程从smp_init开始从smp_init结束。下面分析具体的过程我们将启动其他cpu成为BP(boot cpu)将被启动的cpu成为AP(不知道A表示哪个单词)BP AP
smp_init | do_cpu_up| -- cpuhp_invoke_callback| -- smpboot_create_threads| perf_event_init_cpu| workqueue_prepare_cpu| hrtimers_prepare_cpu| ...| bringup_cpu| --__cpu_up| --boot_secondary (cpu_ops[cpu]-cpu_boot(cpu))| -- cpu_psci_cpu_boot| --psci_cpu_on (invoke_psci_fn(0xc4000003, cpuid, secondary_entry, 0);)| secondary_entrynext cpu -- secondary_startup| -- __secondary_switched| --secondary_start_kernel | -- notify_cpu_starting| --cpuhp_invoke_callback| --sched_cpu_starting| gic_starting_cpu| arch_timer_starting_cpu| dummy_timer_starting_cpu| | bringup_wait_for_ap可以看出BP通过bringup_cpu函数调用psci的函数请求firmware启动AP然后原地等待AP启动的结果。firmware给AP上电之后AP进入head.S中的secondary_entry然后进入secondary_start_kernel完成一系列回调之后通知BP自己启动成功。BP从bringup_wait_for_ap返回继续启动下一个cpu。在cpuhp_invoke_callback所调用的回调函数的定义在这kernel/cpu.c1426 /* Boot processor state steps */
1427 static struct cpuhp_step cpuhp_hp_states[] {
1428 [CPUHP_OFFLINE] {
1429 .name offline,
1430 .startup.single NULL,
1431 .teardown.single NULL,
1432 },
1433 #ifdef CONFIG_SMP
1434 [CPUHP_CREATE_THREADS] {
1435 .name threads:prepare,
1436 .startup.single smpboot_create_threads,
1437 .teardown.single NULL,
1438 .cant_stop true,
1439 },
1440 [CPUHP_PERF_PREPARE] {
1441 .name perf:prepare,
1442 .startup.single perf_event_init_cpu,
1443 .teardown.single perf_event_exit_cpu,
1444 },
1445 [CPUHP_WORKQUEUE_PREP] {
1446 .name workqueue:prepare,
1447 .startup.single workqueue_prepare_cpu,
1448 .teardown.single NULL,
1449 },
1450 [CPUHP_HRTIMERS_PREPARE] {
1451 .name hrtimers:prepare,
1452 .startup.single hrtimers_prepare_cpu,
1453 .teardown.single hrtimers_dead_cpu,
1454 },
1455 [CPUHP_SMPCFD_PREPARE] {
1456 .name smpcfd:prepare,
1457 .startup.single smpcfd_prepare_cpu,
1458 .teardown.single smpcfd_dead_cpu,
1459 },
1460 [CPUHP_RELAY_PREPARE] {
1461 .name relay:prepare,
1462 .startup.single relay_prepare_cpu,
1463 .teardown.single NULL,
1464 },
1465 [CPUHP_SLAB_PREPARE] {
1466 .name slab:prepare,
1467 .startup.single slab_prepare_cpu,
1468 .teardown.single slab_dead_cpu,
1469 },
1470 [CPUHP_RCUTREE_PREP] {
1471 .name RCU/tree:prepare,
...
1485 /* Kicks the plugged cpu into life */
1486 [CPUHP_BRINGUP_CPU] {
1487 .name cpu:bringup,
1488 .startup.single bringup_cpu,
1489 .teardown.single NULL,
1490 .cant_stop true,
1491 },
1492 /* Final state before CPU kills itself */
1493 [CPUHP_AP_IDLE_DEAD] {
1494 .name idle:dead,
1495 },
1496 /*
1497 * Last state before CPU enters the idle loop to die. Transient state
1498 * for synchronization.
1499 */
1500 [CPUHP_AP_OFFLINE] {
1501 .name ap:offline,
1502 .cant_stop true,
1503 },
1504 /* First state is scheduler control. Interrupts are disabled */
1505 [CPUHP_AP_SCHED_STARTING] {
1506 .name sched:starting,
1507 .startup.single sched_cpu_starting,
1508 .teardown.single sched_cpu_dying,
1509 },
1510 [CPUHP_AP_RCUTREE_DYING] {
1511 .name RCU/tree:dying,
1512 .startup.single NULL,
1513 .teardown.single rcutree_dying_cpu,
1514 },
1515 [CPUHP_AP_SMPCFD_DYING] {
1516 .name smpcfd:dying,
1517 .startup.single NULL,
1518 .teardown.single smpcfd_dying_cpu,
1519 },
1520 /* Entry state on starting. Interrupts enabled from here on. Transient
1521 * state for synchronsization */
1522 [CPUHP_AP_ONLINE] {
1523 .name ap:online,
1524 },
1525 /*
1526 * Handled on controll processor until the plugged processor manages
1527 * this itself.
1528 */
1529 [CPUHP_TEARDOWN_CPU] {
1530 .name cpu:teardown,
1531 .startup.single NULL,
...
1535 /* Handle smpboot threads park/unpark */
1536 [CPUHP_AP_SMPBOOT_THREADS] {
1537 .name smpboot/threads:online,
1538 .startup.single smpboot_unpark_threads,
1539 .teardown.single smpboot_park_threads,
1540 },
1541 [CPUHP_AP_IRQ_AFFINITY_ONLINE] {
1542 .name irq/affinity:online,
1543 .startup.single irq_affinity_online_cpu,
1544 .teardown.single NULL,
1545 },
1546 [CPUHP_AP_PERF_ONLINE] {
1547 .name perf:online,
1548 .startup.single perf_event_init_cpu,
1549 .teardown.single perf_event_exit_cpu,
1550 },
1551 [CPUHP_AP_WATCHDOG_ONLINE] {
1552 .name lockup_detector:online,
1553 .startup.single lockup_detector_online_cpu,
1554 .teardown.single lockup_detector_offline_cpu,
1555 },
1556 [CPUHP_AP_WORKQUEUE_ONLINE] {
1557 .name workqueue:online,
1558 .startup.single workqueue_online_cpu,
1559 .teardown.single workqueue_offline_cpu,
1560 },
1561 [CPUHP_AP_RCUTREE_ONLINE] {
1562 .name RCU/tree:online,
1563 .startup.single rcutree_online_cpu,
1564 .teardown.single rcutree_offline_cpu,
1565 },
1566 #endif
1567 /*
1568 * The dynamically registered state space is here
1569 */
1570
1571 #ifdef CONFIG_SMP
1572 /* Last state is scheduler control setting the cpu active */
1573 [CPUHP_AP_ACTIVE] {
1574 .name sched:active,
1575 .startup.single sched_cpu_activate,
1576 .teardown.single sched_cpu_deactivate,
1577 },
1578 #endif
1579
1580 /* CPU is fully up and running. */
1581 [CPUHP_ONLINE] {
1582 .name online,
1583 .startup.single NULL,
1584 .teardown.single NULL,
1585 },
1586 };1482,3-9 60所有的回调函数一次调用最重要的是bringup_cpu这个是通过firmware直接给cpu上电启动的过程。psci通过设备树节点获取与固件通信的方法methodsmc,smc具体的操作指令可以在include/uapi/linux/psci.h中查找.我在代码中增加一些打印语句完整的smp_init的过程如下[ 0.064050] smp: cpuhp-walk:Bringing up secondary CPUs ...
[ 0.064051] smp: cpuhp-walk:before bringing up cpu:0
[ 0.064052] smp: cpuhp-walk:after bringing up cpu:0
[ 0.064053] smp: cpuhp-walk:before bringing up cpu:1
[ 0.064053] cpuhp_walk:cpu:1,do_cpu_up
[ 0.064060] cphp_walk:cpu:1,cpuhp_up_callbacks
[ 0.064061] cphp_walk:cpu:1,cpuhp_invoke_callback, 1.1, cb:smpboot_create_threads0x0/0xe0
[ 0.104074] cphp_walk:cpu:1,cpuhp_invoke_callback, 1.1, cb:perf_event_init_cpu0x0/0x140
[ 0.104080] cphp_walk:cpu:1,cpuhp_invoke_callback, 1.1, cb:workqueue_prepare_cpu0x0/0x98
[ 0.112080] cphp_walk:cpu:1,cpuhp_invoke_callback, 1.1, cb:hrtimers_prepare_cpu0x0/0xb0
[ 0.112084] cphp_walk:cpu:1,cpuhp_invoke_callback, 1.1, cb:smpcfd_prepare_cpu0x0/0x70
[ 0.112089] cphp_walk:cpu:1,cpuhp_invoke_callback, 1.1, cb:relay_prepare_cpu0x0/0xf8
[ 0.112092] cphp_walk:cpu:1,cpuhp_invoke_callback, 1.1, cb:rcutree_prepare_cpu0x0/0x1c0
[ 0.112099] cphp_walk:cpu:1,cpuhp_invoke_callback, 1.1, cb:timers_prepare_cpu0x0/0x80
[ 0.112101] cphp_walk:cpu:1,cpuhp_invoke_callback, 1.1, cb:bringup_cpu0x0/0x140
[ 0.112104] cpuhp_walk:cpu:1,bringup_cpu
[ 0.112105] cpuhp_walk:cpu:1,__cpu_up
[ 0.112106] cpuhp_walk:cpu:1,boot_secondary, cpu_boot:cpu_psci_cpu_boot0x0/0x94
[ 0.112109] psci: cpuhp_walk:cpu:1,cpu_psci_cpu_boot, cpu_on:psci_cpu_on0x0/0x94
[ 0.112112] psci: cphp_walk:cpuid:1,psci_cpu_on, fn:0xc4000003
[ 0.145924] cphp_walk:cpu:1,secondary_start_kernel
[ 0.145934] Detected PIPT I-cache on CPU1
[ 0.145942] cpuhp_walk:cpu:1,notify_cpu_starting
[ 0.145943] cphp_walk:cpu:1,cpuhp_invoke_callback, 1.1, cb:sched_cpu_starting0x0/0x120
[ 0.145946] cphp_walk:cpu:1,cpuhp_invoke_callback, 1.1, cb:gic_starting_cpu0x0/0x40
[ 0.145950] GICv3: CPU1: found redistributor 1 region 1:0x00000000299a0000
[ 0.145958] GICv3: CPU1: using allocated LPI pending table 0x0000002176620000
[ 0.145964] cphp_walk:cpu:1,cpuhp_invoke_callback, 1.1, cb:arch_timer_starting_cpu0x0/0x318
[ 0.145971] cphp_walk:cpu:1,cpuhp_invoke_callback, 1.1, cb:dummy_timer_starting_cpu0x0/0x88
[ 0.145973] cpuhp_walk, CPU1: Booted secondary processor 0x0000000001 [0x701f6633]
[ 0.145978] cpuhp_walk:cpu:1,bringup_cpu, wait ap
[ 0.145980] cphp_walk:cpu:1,bringup_wait_for_ap, 1
[ 0.145984] cphp_walk:cpu:1,bringup_wait_for_ap, 2
[ 0.145987] cphp_walk:cpu:1,cpuhp_invoke_callback, 1.1, cb:smpboot_unpark_threads0x0/0xb0
[ 0.145994] cphp_walk:cpu:1,cpuhp_invoke_callback, 1.1, cb:irq_affinity_online_cpu0x0/0x110
[ 0.145998] cphp_walk:cpu:1,cpuhp_invoke_callback, 1.1, cb:perf_event_init_cpu0x0/0x140
[ 0.146002] cphp_walk:cpu:1,cpuhp_invoke_callback, 1.1, cb:workqueue_online_cpu0x0/0x228
[ 0.146019] cphp_walk:cpu:1,cpuhp_invoke_callback, 1.1, cb:rcutree_online_cpu0x0/0x98
[ 0.146022] cphp_walk:cpu:1,cpuhp_invoke_callback, 1.1, cb:page_writeback_cpu_online0x0/0x20
[ 0.146025] cphp_walk:cpu:1,cpuhp_invoke_callback, 1.1, cb:vmstat_cpu_online0x0/0x70
[ 0.146029] cphp_walk:cpu:1,cpuhp_invoke_callback, 1.1, cb:sched_cpu_activate0x0/0x168
[ 0.146035] smp: cpuhp-walk:after bringing up cpu:1
[ 0.146036] smp: cpuhp-walk:before bringing up cpu:2
[ 0.146037] cpuhp_walk:cpu:2,do_cpu_up
[ 0.146041] cphp_walk:cpu:2,cpuhp_up_callbacks
[ 0.146042] cphp_walk:cpu:2,cpuhp_invoke_callback, 1.1, cb:smpboot_create_threads0x0/0xe0
[ 0.185939] cphp_walk:cpu:2,cpuhp_invoke_callback, 1.1, cb:perf_event_init_cpu0x0/0x140
[ 0.185943] cphp_walk:cpu:2,cpuhp_invoke_callback, 1.1, cb:workqueue_prepare_cpu0x0/0x98
[ 0.193946] cphp_walk:cpu:2,cpuhp_invoke_callback, 1.1, cb:hrtimers_prepare_cpu0x0/0xb0
[ 0.193949] cphp_walk:cpu:2,cpuhp_invoke_callback, 1.1, cb:smpcfd_prepare_cpu0x0/0x70
[ 0.193952] cphp_walk:cpu:2,cpuhp_invoke_callback, 1.1, cb:relay_prepare_cpu0x0/0xf8
[ 0.193954] cphp_walk:cpu:2,cpuhp_invoke_callback, 1.1, cb:rcutree_prepare_cpu0x0/0x1c0
[ 0.193959] cphp_walk:cpu:2,cpuhp_invoke_callback, 1.1, cb:timers_prepare_cpu0x0/0x80
[ 0.193961] cphp_walk:cpu:2,cpuhp_invoke_callback, 1.1, cb:bringup_cpu0x0/0x140
[ 0.193964] cpuhp_walk:cpu:2,bringup_cpu
[ 0.193964] cpuhp_walk:cpu:2,__cpu_up
[ 0.193965] cpuhp_walk:cpu:2,boot_secondary, cpu_boot:cpu_psci_cpu_boot0x0/0x94
[ 0.193967] psci: cpuhp_walk:cpu:2,cpu_psci_cpu_boot, cpu_on:psci_cpu_on0x0/0x94
[ 0.193969] psci: cphp_walk:cpuid:256,psci_cpu_on, fn:0xc4000003
[ 0.227952] cphp_walk:cpu:2,secondary_start_kernel
[ 0.227961] Detected PIPT I-cache on CPU2
[ 0.227967] cpuhp_walk:cpu:2,notify_cpu_starting
[ 0.227968] cphp_walk:cpu:2,cpuhp_invoke_callback, 1.1, cb:sched_cpu_starting0x0/0x120
[ 0.227972] cphp_walk:cpu:2,cpuhp_invoke_callback, 1.1, cb:gic_starting_cpu0x0/0x40
[ 0.227976] GICv3: CPU2: found redistributor 100 region 2:0x00000000299c0000
[ 0.227994] GICv3: CPU2: using allocated LPI pending table 0x0000002176630000
[ 0.227999] cphp_walk:cpu:2,cpuhp_invoke_callback, 1.1, cb:arch_timer_starting_cpu0x0/0x318
[ 0.228007] cphp_walk:cpu:2,cpuhp_invoke_callback, 1.1, cb:dummy_timer_starting_cpu0x0/0x88
[ 0.228010] cpuhp_walk, CPU2: Booted secondary processor 0x0000000100 [0x701f6633]
[ 0.228017] cpuhp_walk:cpu:2,bringup_cpu, wait ap
[ 0.228019] cphp_walk:cpu:2,bringup_wait_for_ap, 1
[ 0.228022] cphp_walk:cpu:2,bringup_wait_for_ap, 2
[ 0.228029] cphp_walk:cpu:2,cpuhp_invoke_callback, 1.1, cb:smpboot_unpark_threads0x0/0xb0
[ 0.228039] cphp_walk:cpu:2,cpuhp_invoke_callback, 1.1, cb:irq_affinity_online_cpu0x0/0x110
[ 0.228042] cphp_walk:cpu:2,cpuhp_invoke_callback, 1.1, cb:perf_event_init_cpu0x0/0x140
[ 0.228047] cphp_walk:cpu:2,cpuhp_invoke_callback, 1.1, cb:workqueue_online_cpu0x0/0x228
[ 0.228070] cphp_walk:cpu:2,cpuhp_invoke_callback, 1.1, cb:rcutree_online_cpu0x0/0x98
[ 0.228073] cphp_walk:cpu:2,cpuhp_invoke_callback, 1.1, cb:page_writeback_cpu_online0x0/0x20
[ 0.228076] cphp_walk:cpu:2,cpuhp_invoke_callback, 1.1, cb:vmstat_cpu_online0x0/0x70
[ 0.228080] cphp_walk:cpu:2,cpuhp_invoke_callback, 1.1, cb:sched_cpu_activate0x0/0x168
[ 0.228086] smp: cpuhp-walk:after bringing up cpu:2
[ 0.228087] smp: cpuhp-walk:before bringing up cpu:3
[ 0.228088] cpuhp_walk:cpu:3,do_cpu_up
[ 0.228092] cphp_walk:cpu:3,cpuhp_up_callbacks
[ 0.228093] cphp_walk:cpu:3,cpuhp_invoke_callback, 1.1, cb:smpboot_create_threads0x0/0xe0
[ 0.267968] cphp_walk:cpu:3,cpuhp_invoke_callback, 1.1, cb:perf_event_init_cpu0x0/0x140
[ 0.267972] cphp_walk:cpu:3,cpuhp_invoke_callback, 1.1, cb:workqueue_prepare_cpu0x0/0x98
[ 0.275974] cphp_walk:cpu:3,cpuhp_invoke_callback, 1.1, cb:hrtimers_prepare_cpu0x0/0xb0
[ 0.275977] cphp_walk:cpu:3,cpuhp_invoke_callback, 1.1, cb:smpcfd_prepare_cpu0x0/0x70
[ 0.275980] cphp_walk:cpu:3,cpuhp_invoke_callback, 1.1, cb:relay_prepare_cpu0x0/0xf8
[ 0.275983] cphp_walk:cpu:3,cpuhp_invoke_callback, 1.1, cb:rcutree_prepare_cpu0x0/0x1c0
[ 0.275988] cphp_walk:cpu:3,cpuhp_invoke_callback, 1.1, cb:timers_prepare_cpu0x0/0x80
[ 0.275990] cphp_walk:cpu:3,cpuhp_invoke_callback, 1.1, cb:bringup_cpu0x0/0x140
[ 0.275993] cpuhp_walk:cpu:3,bringup_cpu
[ 0.275994] cpuhp_walk:cpu:3,__cpu_up
[ 0.275995] cpuhp_walk:cpu:3,boot_secondary, cpu_boot:cpu_psci_cpu_boot0x0/0x94
[ 0.275996] psci: cpuhp_walk:cpu:3,cpu_psci_cpu_boot, cpu_on:psci_cpu_on0x0/0x94
[ 0.275998] psci: cphp_walk:cpuid:257,psci_cpu_on, fn:0xc4000003
[ 0.309980] cphp_walk:cpu:3,secondary_start_kernel
[ 0.309985] Detected PIPT I-cache on CPU3
[ 0.309989] cpuhp_walk:cpu:3,notify_cpu_starting
[ 0.309990] cphp_walk:cpu:3,cpuhp_invoke_callback, 1.1, cb:sched_cpu_starting0x0/0x120
[ 0.309993] cphp_walk:cpu:3,cpuhp_invoke_callback, 1.1, cb:gic_starting_cpu0x0/0x40
[ 0.309996] GICv3: CPU3: found redistributor 101 region 3:0x00000000299e0000
[ 0.310002] GICv3: CPU3: using allocated LPI pending table 0x0000002176640000
[ 0.310007] cphp_walk:cpu:3,cpuhp_invoke_callback, 1.1, cb:arch_timer_starting_cpu0x0/0x318
[ 0.310012] cphp_walk:cpu:3,cpuhp_invoke_callback, 1.1, cb:dummy_timer_starting_cpu0x0/0x88
[ 0.310014] cpuhp_walk, CPU3: Booted secondary processor 0x0000000101 [0x701f6633]
[ 0.310019] cpuhp_walk:cpu:3,bringup_cpu, wait ap
[ 0.310021] cphp_walk:cpu:3,bringup_wait_for_ap, 1
[ 0.310024] cphp_walk:cpu:3,bringup_wait_for_ap, 2
[ 0.310028] cphp_walk:cpu:3,cpuhp_invoke_callback, 1.1, cb:smpboot_unpark_threads0x0/0xb0
[ 0.310036] cphp_walk:cpu:3,cpuhp_invoke_callback, 1.1, cb:irq_affinity_online_cpu0x0/0x110
[ 0.310039] cphp_walk:cpu:3,cpuhp_invoke_callback, 1.1, cb:perf_event_init_cpu0x0/0x140
[ 0.310043] cphp_walk:cpu:3,cpuhp_invoke_callback, 1.1, cb:workqueue_online_cpu0x0/0x228
[ 0.310050] cphp_walk:cpu:3,cpuhp_invoke_callback, 1.1, cb:rcutree_online_cpu0x0/0x98
[ 0.310052] cphp_walk:cpu:3,cpuhp_invoke_callback, 1.1, cb:page_writeback_cpu_online0x0/0x20
[ 0.310054] cphp_walk:cpu:3,cpuhp_invoke_callback, 1.1, cb:vmstat_cpu_online0x0/0x70
[ 0.310057] cphp_walk:cpu:3,cpuhp_invoke_callback, 1.1, cb:sched_cpu_activate0x0/0x168
[ 0.310063] smp: cpuhp-walk:after bringing up cpu:3
[ 0.310064] smp: cpuhp-walk:Brought up 1 node, 4 CPUs
[ 0.310066] SMP: Total of 4 processors activated.
[ 0.310067] CPU features: detected: 32-bit EL0 Support
[ 0.310069] CPU features: detected: CRC32 instructions
[ 0.310303] CPU: All CPU(s) started at EL2经查是CPU厂商firmware的问题 也就是psci_cpu_on, fn:0xc4000003的问题。所以以后遇到这样的问题熟悉了过程就不慌了。本文的psci部分由我的同事提供同时在阅读和分析代码之前我还参照了如下链接的信息https://www.cnblogs.com/pengdonglin137/p/11925299.htmlhttps://zhuanlan.zhihu.com/p/538782115https://blog.csdn.net/weixin_45264425/article/details/127895018https://zhuanlan.zhihu.com/p/545550388此外由于功耗控制的原因系统运行期间还会通过cpu idle来间断的关闭和打开cpu也就是cpu hotplug打开cpu的操作和上述过程相同关闭cpu的过程与cpu up相反读者自行阅读代码也是通过psci来关闭cpu。
