rk3399pro 内核软死锁

1内核死锁概念

lockup 就是某段内核代码占着CPU不放，lockup 严重时导致整个系统失去响应，lockup 有两个特点：

1.只有内核代码才能引起lockup，因为用户代码是可被抢占的，不可能形成lockup（只有一个例外：SCHED_FIFO 优先级为99的实时进程也是不可抢占的，SCHED_FIFO优先级为99的实时进程即使在用户态也可能使[watchdog/x]内核线程抢不到CPU而形成soft lock。

2.内核代码必须处于禁止抢占状态，因为linux是抢占式内核，只有某些特定代码区才禁止抢占，只有这些区域才能形成lockup。

lockup分为soft lockup和hard lockup。

2检测方法

Linux kernel设计了一个检测lockup的机制，称为NMI Watchdog，是利用NMI中断实现的，用NMI是因为lockup有可能发生在中断被屏蔽的状态下，这时唯一能把CPU抢下来的方法就是通过NMI，因为NMI中断是不可屏蔽的。NMI Watchdog包含两部分:

1.高精度计时器，递增 hrtimer_interrupts，唤醒[watchodg/x]内核线程，更新一个时间戳。

2.NMI中断时，检查 hrtimer_interrupts 是否递增，如果没有说明hard lockup；检查时间戳是否更新，如果没有说明soft lockup。

3内核软死锁

soft lockup是指CPU被内核代码占据，以至于无法执行其它进程。检测soft lockup的原理是给每个CPU分配一个定时执行的内核线程[watchdog/x]，如果该线程在设定的期限内没有得到执行的话就意味着发生了soft lockup，[watchdog/x]是SCHED_FIFO实时进程，优先级为最高的99，拥有优先运行的特权。

3.1测试demo

#include <linux/init.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/kthread.h>
#include <linux/spinlock.h>

struct task_struct *task0;
static spinlock_t spinlock;
int val;

MODULE_LICENSE("GPL");

int task(void *arg)
{
    printk(KERN_INFO "%s:%d\n",__func__,__LINE__);
    /* To generate panic uncomment following */
    /* panic("softlockup: hung tasks"); */

 #if 1
    while(!kthread_should_stop()) {
        printk(KERN_INFO "%s:%d\n",__func__,__LINE__);
        spin_lock(&spinlock);
        /* busy loop in critical section */
        while(1) {
            printk(KERN_INFO "%s:%d\n",__func__,__LINE__);
        }

        spin_unlock(&spinlock);
    }
#endif

    return val;
}

static int softlockup_init(void)
{
    printk(KERN_INFO "%s:%d\n",__func__,__LINE__);
 #if 1
    val = 1;
    spin_lock_init(&spinlock);
    task0 = kthread_run(&task,NULL,"softlockup_thread");
    set_cpus_allowed_ptr(task0, cpumask_of(0));
 #endif
    return 0;
}

static void softlockup_exit(void)
{
    printk(KERN_INFO "%s:%d\n",__func__,__LINE__);
    kthread_stop(task0);
}

module_init(softlockup_init);
module_exit(softlockup_exit);

3.2软死锁log

rk3399pro上运行测试demo的软死锁log信息如下：

[  344.114604] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [softlockup_thre:1821]
[  344.114611] Modules linked in: softlockup_test(PO) bcmdhd
[  344.114612]
[  344.114621] CPU: 5 PID: 1821 Comm: softlockup_thre Tainted: P           O    4.4.000001
[  344.114828]
[  344.114828] X7: 0xffffff8008fde408:
[  344.114834] e408  f1fb7600 ffffffc0 00000000 00000000 00000000 00000000 00110000 00000000
[  344.114840] e428  000f0000 00000000 00000000 00000000 00040000 00000000 00080000 00000000
[  344.114845] e448  00000000 00000000 00000000 00000000 00000001 00000000 00000000 00000000
[  344.114851] e468  00000000 00000001 00000000 00000001 00000002 00000001 00000001 00000000
[  344.114856] e488  00000000 00000000 08d091d9 ffffff80 072b072b dead4ead ffffffff 00000000
[  344.114862] e4a8  ffffffff ffffffff f1840000 ffffffc0 00040000 00000000 00000001 00000000
[  344.114868] e4c8  00000000 dead4ead ffffffff 00000000 ffffffff ffffffff 08fde4e0 ffffff80
[  344.114873] e4e8  08fde4e0 ffffff80 00000000 00000000 00000000 00000000 00000000 00000000
[  344.114875]
[  344.114875] X8: 0xffffff800835b0d4:
[  344.114881] b0d4  9400054a aa1303e0 94000554 52800000 a94153f3 f94013f5 a8c37bfd d65f03c0
[  344.114887] b0f4  f9400413 17fffff6 f9400813 17fffff4 f9400c13 17fffff2 128002a0 17fffff5
[  344.114892] b114  71001c1f 54000161 f9405ce0 f9400c00 b4000140 a9bf7bfd aa0603e2 aa0403e1
[  344.114898] b134  910003fd 940004ce a8c17bfd d65f03c0 128002a0 d65f03c0 12800160 d65f03c0
[  344.114904] b154  a9bb7bfd 7100081f 910003fd a9025bf5 aa0403f6 a90153f3 aa0603f5 a90363f7
[  344.114910] b174  f9405cf3 54000181 f9400660 b4000a20 2a1503e2 aa1603e1 9400047d 52800000
[  344.114915] b194  a94153f3 a9425bf5 a94363f7 a8c57bfd d65f03c0 71000c1f 54000061 f9400a60
[  344.114921] b1b4  17fffff3 71001c1f 54000061 f9400e60 17ffffef 35000820 51000420 7100041f
[  344.114922]
[  344.114922] X10: 0xffffff800910981f:
[  344.114928] 981c  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[  344.114934] 983c  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[  344.114939] 985c  00000000 00000000 00000000 00000000 00000000 00000000 00000000 00000000
[  344.114945] 987c  00000000 00000000 00000000 00000000 00000000 00000001 00000000 3320205b
[  344.114951] 989c  312e3434 31343431 74205d33 3a6b7361 750a3332 6e695f70 333a7469 67670a35
[  344.114957] 98bc  20676e69 20657564 6b206f74 656e7265 6174206c 0a746e69 69617420 2073746e
[  344.114962] 98dc  6e72656b 0a2e6c65 2d206c6c 6f6c6620 6f632077 6f72746e 7872206c 0a78742f
[  344.114968] 98fc  6e6f6973 382e3920 31352e37 2e31312e 28203831 32656334 40646533 69676873
[  344.114974] 991c  28202974 46202972 20444957 632d3130 33393536 0a333561 2020205b 312e3132
[  344.114977]
[  344.114977] X18: 0xffffff8089109817:
[  344.114988] 9814  ******** ******** ******** ******** ******** ******** ******** ********
[  344.114995] 9834  ******** ******** ******** ******** ******** ******** ******** ********
[  344.115002] 9854  ******** ******** ******** ******** ******** ******** ******** ********
[  344.115009] 9874  ******** ******** ******** ******** ******** ******** ******** ********
[  344.115016] 9894  ******** ******** ******** ******** ******** ******** ******** ********
[  344.115023] 98b4  ******** ******** ******** ******** ******** ******** ******** ********
[  344.115030] 98d4  ******** ******** ******** ******** ******** ******** ******** ********
[  344.115037] 98f4  ******** ******** ******** ******** ******** ******** ******** ********
[  344.115043] 9914  ******** ******** ******** ******** ******** ******** ******** ********
[  344.115045]
[  344.115045] X19: 0xffffff8008f8c688:
[  344.115051] c688  08f8c6e8 ffffff80 08f8c6c8 ffffff80 08f8c6a8 ffffff80 00000000 00000000
[  344.115057] c6a8  08ce4005 ffffff80 00000124 00000000 080eb0a0 ffffff80 00000000 00000000
[  344.115062] c6c8  08ce401c ffffff80 00000124 00000000 080eb200 ffffff80 00000000 00000000
[  344.115068] c6e8  08ce402e ffffff80 00000124 00000000 080eb104 ffffff80 00000000 00000000
[  344.115074] c708  0910c570 ffffff80 c4eac4ea dead4ead ffffffff 00000000 ffffffff ffffffff
[  344.4.194 #8
[  344.116373] Hardware name: Rockchip RK3399pro evb v14 board for linux (DT)
[  344.116377] Call trace:
[  344.116394] [<ffffff80080882c0>] dump_backtrace+0x0/0x220
[  344.116408] [<ffffff8008088504>] show_stack+0x24/0x30
[  344.116421] [<ffffff80083b8dec>] dump_stack+0x94/0xbc
[  344.116433] [<ffffff800808cba8>] handle_IPI+0x1e0/0x260
[  344.116444] [<ffffff8008080eec>] gic_handle_irq+0x12c/0x184
[  344.116454] Exception stack(0xffffffc0f6ee3d70 to 0xffffffc0f6ee3ea0)
[  344.116465] 3d60:                                   ffffffc0f6ee3db0 0000008000000000
[  344.116478] 3d80: ffffffc0f6ee3ee0 ffffff8008080ff8 0000000040000145 ffffffc0f1c53d90
[  344.116493] 3da0: 00000000072b0000 ffffffc0f6ee4050 0000000000000040 ffffff8009106d00
[  344.116507] 3dc0: ffffff8008f37000 ffffff8008f75000 ffffff8009106d00 00000040edfa9000
[  344.116520] 3de0: 00000000000a65da 0000000000000005 ffffff8008a6865c ffffff8008081800
[  344.116532] 3e00: 0000000000001000 0000000000000000 0000000034d5d91d 00000040edfa9000
[  344.116545] 3e20: ffffffc0f1c48d80 0000000000000000 0000000000000000 0000000000000000
[  344.116558] 3e40: 0000000030d00800 ffffff8008f37000 ffffff8008f3e000 0000000000000002
[  344.116572] 3e60: ffffff8009106000 0000000000000001 ffffffc0f1c53d90 ffffff8008fe5000
[  344.116585] 3e80: ffffffc0f6ee4050 0000000000c4a244 ffffff80090c6000 ffffffc0f6ee3ee0
[  344.116597] [<ffffff80080827b4>] el1_irq+0xb4/0x140
[  344.116608] [<ffffff800809fd74>] irq_exit+0x88/0xd0
[  344.116621] [<ffffff80080eedd0>] __handle_domain_irq+0x84/0xb8
[  344.116632] [<ffffff8008080e78>] gic_handle_irq+0xb8/0x184
[  344.116641] Exception stack(0xffffffc0f1c53d90 to 0xffffffc0f1c53ec0)
[  344.116651] 3d80:                                   0000000000000000 0000000000000001
[  344.116666] 3da0: 00000040edfa9000 0000000000000002 00ffffffffffffff 00000001ee52bd73
[  344.116679] 3dc0: 00000000000a65da 0000000000000001 00000032b5593519 ffffff8008081800
[  344.116692] 3de0: 0000000000001000 0000000000000000 0000000034d5d91d 00000040edfa9000
[  344.116704] 3e00: ffffffc0f1c48d80 0000000000000000 0000000000000000 0000000000000000
[  344.116717] 3e20: 0000000030d00800 000000501aec8c71 0000000000000001 ffffffc0f0cad200
[  344.116730] 3e40: ffffff80090c6fe0 0000000000000001 000000501adc4714 ffffff8009070dc8
[  344.116744] 3e60: 00000000014d9000 0000000000c4a244 0000000000000000 ffffffc0f1c53ec0
[  344.116758] 3e80: ffffff800877f858 ffffffc0f1c53ec0 ffffff800877f85c 0000000060000145
[  344.116770] 3ea0: ffffffc0f1c53ec0 ffffff800877f858 ffffffffffffffff 0000000000000001
[  344.116781] [<ffffff80080827b4>] el1_irq+0xb4/0x140
[  344.116795] [<ffffff800877f85c>] cpuidle_enter_state+0x1cc/0x25c
[  344.116808] [<ffffff800877f960>] cpuidle_enter+0x34/0x44
[  344.116820] [<ffffff80080e0da4>] call_cpuidle+0x6c/0x74
[  344.116832] [<ffffff80080e1058>] cpu_startup_entry+0x2ac/0x2bc
[  344.116844] [<ffffff800808c570>] secondary_start_kernel+0x190/0x1bc
[  344.116852] [<0000000000c4a188>] 0xc4a188
[  344.116861] CPU2: stopping
[  344.116876] CPU: 2 PID: 0 Comm: swapper/2 Tainted: P           O L  4.4.194 #8
[  344.116882] Hardware name: Rockchip RK3399pro evb v14 board for linux (DT)
[  344.116886] Call trace:
[  344.116904] [<ffffff80080882c0>] dump_backtrace+0x0/0x220
[  344.116918] [<ffffff8008088504>] show_stack+0x24/0x30
[  344.116932] [<ffffff80083b8dec>] dump_stack+0x94/0xbc
[  344.116943] [<ffffff800808cba8>] handle_IPI+0x1e0/0x260
[  344.116954] [<ffffff8008080eec>] gic_handle_irq+0x12c/0x184
[  344.116964] Exception stack(0xffffffc0f1c5bd90 to 0xffffffc0f1c5bec0)
[  344.116975] bd80:                                   ffffffc0f6eff0c0 00000040edfbe000
[  344.116989] bda0: ffffffc0f6eff0c0 000d721088edce80 00ffffffffffffff 00000001ee525fb3
[  344.117002] bdc0: 00000000000a65d8 0000000000000002 00000032b5593519 ffffff8008081800
[  344.117014] bde0: 0000000000001000 0000000000000000 00000

TOP信息：

3.3软死锁信息分析

如何从log中的信息，分析问题出现的原因，首先，从log中看到如下信息：

[  344.114604] NMI watchdog: BUG: soft lockup - CPU#5 stuck for 22s! [softlockup_thre:1821]

log第一行，可以看到出现软死锁了，而出现软死锁的线程是“softlockup_thre”，注意这个地方没打印全，但是跟top看到的是对应的，冒号后面是线程id，跟top看到的一样。说明这个线程里面出现了死锁，或者说出现了独占cpu很长时间的bug，进一步分析这个线程里面的操作，排查造成软死锁的原因并改掉。