你开心就好
ABBA锁介绍
编辑
2025-01-20
工作知识
0
请注意，本文编写于 137 天前，最后修改于 137 天前，其中某些信息可能已经过时。

目录

一、简单介绍几个同步原语
1.1 ABBA锁
代码示例
2.1 Spinlock版本ABBA
2.2 mutex版本ABBA
2.3 semaphore版本ABBA
三、调试锁问题

项目中遇到客户的USB接U盘，如果是3.0系统会直接在内核卡死，卡死在经典函数blk_update_request上，我们知道，这肯定是因为同步导致的，因为io同步需要通过usb controler来执行真正的io，落盘到usb的usbstorage上，问题很简单，usb 控制器驱动那边有问题，来源是硬件。

但是同事有提问，如果是同步原语，那么是自旋锁还是互斥锁还是其他呢？我一时没有给出正确答案，另一个问题，这个问题是解决了，那我到底能了解什么呢，能了解USB吗，其实不能，能了解文件系统吗，其实也不能，我私以为能了解的就只有同步原语，或者直接点，了解锁。这里为了了解锁，介绍一下ABBA锁场景死锁。

本文以原来项目问题为起始，做了锁的ABBA衍生，一方面解答了自旋锁和其他锁的表象问题，一方面简单介绍了经典abba锁问题，供大家知悉，仅此而已。

一、简单介绍几个同步原语

主要的同步原语这里有四类，具体介绍这里就不说明了，很多书都会详细介绍

自旋锁：竞争时自旋等待，临界区无法休眠

互斥锁：竞争时可睡眠，

信号量：同互斥锁

完成量：等待完成

根据这四种，有不同的变体，除了完成量，其他都能在不严谨的编程中造成abba锁。故咱们仅谈谈上述三种的基本锁，不谈其他变体，例如顺序锁，读写锁，读写信号，多值信号等等。

1.1 ABBA锁

什么是ABBA锁呢，如下图示： 两个线程中，thread1拿锁A，此时尝试锁B，然后thread2拿锁B，此时尝试锁A，此时thread1和thread2都拿不到B锁和A锁，在这种状态下，互相锁死，也就是场景的ABBA锁的模型。

代码示例

介绍ABBA锁非常简单，但是代码中ABBA的场景却非常复杂，这里先以最简单的驱动来示例这个ABBA的问题，总代码如下：

#include <linux/init.h>
#include <linux/mutex.h>
#include <linux/module.h>
#include <linux/spinlock.h>
#include <linux/kthread.h>
#include <linux/delay.h>
 
static DEFINE_SPINLOCK(spinlock_a);
static DEFINE_SPINLOCK(spinlock_b);
static DEFINE_MUTEX(mutex_a);
static DEFINE_MUTEX(mutex_b);
static DEFINE_SEMAPHORE(semaphore_a);
static DEFINE_SEMAPHORE(semaphore_b);
 
static int testsuite = 0;
module_param(testsuite, int, S_IWUSR | S_IRUSR | S_IWGRP | S_IRGRP);
MODULE_PARM_DESC(testsuite, "Test Lock Suilte");
 
static struct task_struct *thread1, *thread2;
 
static int thread_1(void *arg)
{
    char *name = (char*)arg;
 
    pr_info("Thread %s starting...\n", name);
    while(!kthread_should_stop()){
        schedule_timeout(msecs_to_jiffies(1000));
        if(testsuite == 0x1){
            spin_lock(&spinlock_a);
            pr_info("Thread %s hold spinlock_a trying spinlock_b \n", name);
            spin_lock(&spinlock_b);
            pr_info("%s:Do something...\n", name);
            spin_unlock(&spinlock_b);
            spin_unlock(&spinlock_a);
            pr_info("Thread %s unlock\n", name);
        }
 
        if(testsuite == 0x2){
            mutex_lock(&mutex_a);
            pr_info("Thread %s hold mutex_a trying mutex_b\n", name);
            mutex_lock(&mutex_b);
            pr_info("%s:Do something...\n", name);
            mutex_unlock(&mutex_b);
            mutex_unlock(&mutex_a);
        }
 
        if(testsuite == 0x3){
            down(&semaphore_a);
            pr_info("Thread %s hold semaphore_a trying semaphore_b\n", name);
            down(&semaphore_b);
            pr_info("%s:Do something...\n", name);
            up(&semaphore_b);
            up(&semaphore_a);
        }
    }
    return 0;
}
 
static int thread_2(void *arg)
{
    char *name = (char*)arg;
 
    pr_info("Thread %s starting...\n", name);
    while(!kthread_should_stop()){
        schedule_timeout(msecs_to_jiffies(1000));
        if(testsuite == 0x1){
            spin_lock(&spinlock_b);
            pr_info("Thread %s hold spinlock_b trying spinlock_a \n", name);
            spin_lock(&spinlock_a);
            pr_info("%s:Do something...\n", name);
            spin_unlock(&spinlock_a);
            spin_unlock(&spinlock_b);
            pr_info("Thread %s unlock\n", name);
        }
        if(testsuite == 0x2){
            mutex_lock(&mutex_b);
            pr_info("Thread %s hold mutex_b trying mutex_a\n", name);
            mutex_lock(&mutex_a);
            pr_info("%s:Do something...\n", name);
            mutex_unlock(&mutex_a);
            mutex_unlock(&mutex_b);
        }
 
        if(testsuite == 0x3){
            down(&semaphore_b);
            pr_info("Thread %s hold semaphore_b trying semaphore_a\n", name);
            down(&semaphore_a);
            pr_info("%s:Do something...\n", name);
            up(&semaphore_a);
            up(&semaphore_b);
        }
    }
    return 0;
}
 
static void start_test(void)
{
        thread1 = kthread_run(thread_1, "Thread-1", "spinlock_thread1");
        thread2 = kthread_run(thread_2, "Thread-2", "spinlock_thread2");
        return ;
}
 
static int __init test_init(void)
{
        start_test();
        return 0;
}
 
 
 
static void __exit test_exit(void)
{
        kthread_stop(thread1);
        kthread_stop(thread2);
        return;
}
 
 
module_init(test_init);
module_exit(test_exit);
 
MODULE_AUTHOR("tangfeng <tangfeng@kylinos.cn>");
MODULE_DESCRIPTION("Test spinlock/mutex/semaphore");
MODULE_LICENSE("GPL");

2.1 Spinlock版本ABBA

为了支持自旋版本的ABBA，两个线程分别如下：

对于thread1：

spin_lock(&spinlock_a);
pr_info("Thread %s hold spinlock_a trying spinlock_b \n", name);
spin_lock(&spinlock_b);
pr_info("%s:Do something...\n", name);
spin_unlock(&spinlock_b);
spin_unlock(&spinlock_a);
pr_info("Thread %s unlock\n", name);

对于thread2：

spin_lock(&spinlock_b);
pr_info("Thread %s hold spinlock_b trying spinlock_a \n", name);
spin_lock(&spinlock_a);
pr_info("%s:Do something...\n", name);
spin_unlock(&spinlock_a);
spin_unlock(&spinlock_b);
pr_info("Thread %s unlock\n", name);

此时我们加载运行，日志如下：

[ 247.149092] test: module is from the staging directory, the quality is unknown, you have been warned.
[ 247.150668] Thread Thread-1 starting...
[ 247.150736] Thread Thread-1 hold spinlock_a trying spinlock_b
[ 247.150740] Thread-1:Do something...
[ 247.150744] Thread Thread-1 unlock
[ 247.150748] Thread Thread-2 starting...
[ 247.150753] Thread Thread-2 hold spinlock_b trying spinlock_a
[ 247.150755] Thread Thread-1 hold spinlock_a trying spinlock_b
[ 307.143419] rcu: INFO: rcu_sched self-detected stall on CPU
[ 307.143446] rcu: 0-....: (17996 ticks this GP) idle=6a2/1/0x4000000000000002 softirq=4790/4790 fqs=5944 last_accelerate: 0000/04c6 dyntick_enabled: 0
[ 307.143453] (t=18000 jiffies g=14105 q=7019)
[ 307.143458] Task dump for CPU 0:
[ 307.143464] task:spinlock_thread state:R running task stack: 0 pid: 7795 ppid: 2 flags:0x0000000a
[ 307.143474] Call trace:
[ 307.143484] dump_backtrace+0x0/0x1e8
[ 307.143491] show_stack+0x1c/0x28
[ 307.143498] sched_show_task+0x154/0x178
[ 307.143505] dump_cpu_task+0x48/0x54
[ 307.143511] rcu_dump_cpu_stacks+0xbc/0xfc
[ 307.143517] rcu_sched_clock_irq+0x8b0/0x9d8
[ 307.143524] update_process_times+0x64/0xa0
[ 307.143530] tick_sched_handle.isra.0+0x38/0x58
[ 307.143535] tick_sched_timer+0x50/0xa0
[ 307.143540] __hrtimer_run_queues+0x148/0x2d8
[ 307.143546] hrtimer_interrupt+0xec/0x240
[ 307.143553] arch_timer_handler_phys+0x38/0x48
[ 307.143559] handle_percpu_devid_irq+0x8c/0x210
[ 307.143565] __handle_domain_irq+0x78/0xd8
[ 307.143571] gic_handle_irq+0x88/0x2d8
[ 307.143576] el1_irq+0xc8/0x180
[ 307.143582] queued_spin_lock_slowpath+0x128/0x3b0
[ 307.143587] do_raw_spin_lock+0xd4/0x130
[ 307.143594] _raw_spin_lock+0x14/0x20
[ 307.143605] thread_1+0x174/0x248 [test]
[ 307.143612] kthread+0x100/0x130
[ 307.143618] ret_from_fork+0x10/0x18
[ 307.143622] Task dump for CPU 3:
[ 307.143626] task:spinlock_thread state:R running task stack: 0 pid: 7796 ppid: 2 flags:0x0000000a
[ 307.143635] Call trace:
[ 307.143640] __switch_to+0xe4/0x138
[ 307.143645] 0xffffffc00d77be00
[ 307.143651] kthread+0x100/0x130
[ 307.143656] ret_from_fork+0x10/0x18
[ 308.190088] rcu: INFO: rcu_sched detected expedited stalls on CPUs/tasks: { 0-... 3-... } 18312 jiffies s: 753 root: 0x9/.
[ 308.190098] rcu: blocking rcu_node structures:
[ 308.190101] Task dump for CPU 0:
[ 308.190103] task:spinlock_thread state:R running task stack: 0 pid: 7795 ppid: 2 flags:0x0000000a
[ 308.190107] Call trace:
[ 308.190112] __switch_to+0xe4/0x138
[ 308.190115] 0xffffffc00d73be00
[ 308.190118] kthread+0x100/0x130
[ 308.190121] ret_from_fork+0x10/0x18
[ 308.190123] Task dump for CPU 3:
[ 308.190124] task:spinlock_thread state:R running task stack: 0 pid: 7796 ppid: 2 flags:0x0000000a
[ 308.190128] Call trace:
[ 308.190130] __switch_to+0xe4/0x138
[ 308.190132] 0xffffffc00d77be00
[ 308.190134] kthread+0x100/0x130
[ 308.190137] ret_from_fork+0x10/0x18

我们可以通过日志发现：

[ 247.150753] Thread Thread-2 hold spinlock_b trying spinlock_a
[ 247.150755] Thread Thread-1 hold spinlock_a trying spinlock_b

thread2先锁住了spinlock_b，然后尝试spinlock_a，而同时thread1锁住了spinlock_a，然后尝试spinlock_b，这就是典型的baab状态。

我们可以留意到如下：

zcat /proc/config.gz | grep CONFIG_RCU_CPU_STALL_TIMEOUT
CONFIG_RCU_CPU_STALL_TIMEOUT=60

可以发现rcu stall检测器正好能够检测到spinlock的abba死锁问题

2.2 mutex版本ABBA

mutex版本的两个thread的代码如下：

thread1：

mutex_lock(&mutex_a);
pr_info("Thread %s hold mutex_a trying mutex_b\n", name);
mutex_lock(&mutex_b);
pr_info("%s:Do something...\n", name);
mutex_unlock(&mutex_b);
mutex_unlock(&mutex_a);

thread2：

mutex_lock(&mutex_b);
pr_info("Thread %s hold mutex_b trying mutex_a\n", name);
mutex_lock(&mutex_a);
pr_info("%s:Do something...\n", name);
mutex_unlock(&mutex_a);
mutex_unlock(&mutex_b);

此时我们加载运行，日志如下：

[ 111.601145] Thread Thread-1 starting...
[ 111.601222] Thread Thread-1 hold mutex_a trying mutex_b
[ 111.601226] Thread-1:Do something...
[ 111.601230] Thread Thread-1 hold mutex_a trying mutex_b
[ 111.601234] Thread-1:Do something...
[ 111.601239] Thread Thread-1 hold mutex_a trying mutex_b
[ 111.601244] Thread Thread-2 starting...
[ 111.601248] Thread Thread-2 hold mutex_b trying mutex_a

thread1先锁住了mutex_a，然后尝试mutex_b，而同时thread2启动，锁住了mutex_b，然后尝试mutex_a，完成了abba死锁状态。

但是可以发现，此时我们rct stall并不会检测死锁原因，因为mutex在竞争不过的时候会休眠，操作系统仍可以正常使用。为了能够检测这种可休眠的锁的竞争问题，我们需要打开hungtask配置，由内核启动khungtaskd来用于检测软锁问题，如下：

CONFIG_DETECT_HUNG_TASK=y
CONFIG_DEFAULT_HUNG_TASK_TIMEOUT=120

当我们的内核支持hungtask后，可以发现内核默认线程khungtaskd启动，如下：

root@kylin:~# ps -ax | grep hungtaskd
 71 ? S 0:00 [khungtaskd]

此时，mutex的abba锁存在堆栈，如下：

[ 248.461569] INFO: task spinlock_thread:4537 blocked for more than 122 seconds.
[ 248.461598] Tainted: G C 5.10.198 #20
[ 248.461603] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 248.461609] task:spinlock_thread state:D stack: 0 pid: 4537 ppid: 2 flags:0x00000008
[ 248.461618] Call trace:
[ 248.461629] __switch_to+0xe4/0x138
[ 248.461637] __schedule+0x2b4/0x818
[ 248.461643] schedule+0x4c/0xd0
[ 248.461649] schedule_preempt_disabled+0x14/0x20
[ 248.461655] __mutex_lock.isra.0+0x184/0x588
[ 248.461660] __mutex_lock_slowpath+0x18/0x20
[ 248.461665] mutex_lock+0x7c/0x88
[ 248.461676] thread_1+0x1f4/0x220 [test]
[ 248.461682] kthread+0x100/0x130
[ 248.461688] ret_from_fork+0x10/0x18
[ 248.461693] INFO: task spinlock_thread:4538 blocked for more than 122 seconds.
[ 248.461698] Tainted: G C 5.10.198 #20
[ 248.461702] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 248.461706] task:spinlock_thread state:D stack: 0 pid: 4538 ppid: 2 flags:0x00000008
[ 248.461712] Call trace:
[ 248.461718] __switch_to+0xe4/0x138
[ 248.461723] __schedule+0x2b4/0x818
[ 248.461728] schedule+0x4c/0xd0
[ 248.461733] schedule_preempt_disabled+0x14/0x20
[ 248.461738] __mutex_lock.isra.0+0x184/0x588
[ 248.461744] __mutex_lock_slowpath+0x18/0x20
[ 248.461754] mutex_lock+0x7c/0x88
[ 248.461761] thread_2+0xec/0x180 [test]
[ 248.461766] kthread+0x100/0x130
[ 248.461771] ret_from_fork+0x10/0x18

2.3 semaphore版本ABBA

semaphore版本的两个thread如下：

thread1：

down(&semaphore_a);
pr_info("Thread %s hold semaphore_a trying semaphore_b\n", name);
down(&semaphore_b);
pr_info("%s:Do something...\n", name);
up(&semaphore_b);
up(&semaphore_a);

thread2：

down(&semaphore_b);
pr_info("Thread %s hold semaphore_b trying semaphore_a\n", name);
down(&semaphore_a);
pr_info("%s:Do something...\n", name);
up(&semaphore_a);
up(&semaphore_b);

此时加载运行，日志如下：

[ 41.624824] Thread Thread-1 starting...
[ 41.625040] Thread Thread-1 hold semaphore_a trying semaphore_b
[ 41.625044] Thread-1:Do something...
[ 41.625050] Thread Thread-2 starting...
[ 41.625051] Thread Thread-1 hold semaphore_a trying semaphore_b
[ 41.625055] Thread-1:Do something...
[ 41.625076] Thread Thread-1 hold semaphore_a trying semaphore_b
[ 41.625130] Thread Thread-2 hold semaphore_b trying semaphore_a
[ 248.458566] INFO: task spinlock_thread:3817 blocked for more than 122 seconds.
[ 248.458597] Tainted: G C 5.10.198 #20
[ 248.458602] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 248.458607] task:spinlock_thread state:D stack: 0 pid: 3817 ppid: 2 flags:0x00000008
[ 248.458616] Call trace:
[ 248.458627] __switch_to+0xe4/0x138
[ 248.458635] __schedule+0x2b4/0x818
[ 248.458640] schedule+0x4c/0xd0
[ 248.458646] schedule_timeout+0x290/0x2f0
[ 248.458652] __down+0x74/0xd0
[ 248.458659] down+0x50/0x68
[ 248.458670] thread_1+0xb4/0x220 [test]
[ 248.458677] kthread+0x100/0x130
[ 248.458683] ret_from_fork+0x10/0x18
[ 248.458688] INFO: task spinlock_thread:3818 blocked for more than 122 seconds.
[ 248.458693] Tainted: G C 5.10.198 #20
[ 248.458696] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
[ 248.458701] task:spinlock_thread state:D stack: 0 pid: 3818 ppid: 2 flags:0x00000008
[ 248.458707] Call trace:
[ 248.458712] __switch_to+0xe4/0x138
[ 248.458718] __schedule+0x2b4/0x818
[ 248.458723] schedule+0x4c/0xd0
[ 248.458728] schedule_timeout+0x290/0x2f0
[ 248.458733] __down+0x74/0xd0
[ 248.458738] down+0x50/0x68
[ 248.458745] thread_2+0x140/0x180 [test]
[ 248.458751] kthread+0x100/0x130
[ 248.458756] ret_from_fork+0x10/0x18

因为semaphore和mutex实现类似，故hungtask仍可以检测semaphore的abba问题

三、调试锁问题

上面可以查看锁的问题，为了调试锁，我们需要打开内核的LOCKDEP。主要配置如下：

CONFIG_LOCK_STAT=y
CONFIG_PROVE_LOCKING=y
CONFIG_DEBUG_LOCKDEP=y

此时我们的日志能够更清晰的查看abba锁的内容信息，举例如下：

[  371.125970] ======================================================
[  371.132162] [ INFO: possible circular locking dependency detected ]
[  371.138445] 4.9.88 #2 Tainted: G           O   
[  371.142987] -------------------------------------------------------
[  371.149265] kworker/0:2/104 is trying to acquire lock:
..............................

为了可调试，可以直接如下：

cat /proc/lockdep

此时出现系统所有lock的地址

all lock classes: 
0000000044beef8b ....: logbuf_lock
00000000c80448bf ....: (console_sem).lock
00000000700ad619 ....: console_lock
00000000ef505732 ....: cgroup_mutex
0000000042291e92 ....: console_owner_lock
000000002e29cf8c ....: console_owner
.....................

也可以看lock的stat，如下

cat /proc/lockdep_stats
 lock-classes: 1851 [max: 8191] 
 direct dependencies: 0 [max: 32768] 
 indirect dependencies: 0 
 all direct dependencies: 0 
 in-hardirq chains: 0
....................