locking/mutex: clear MUTEX_FLAGS if wait_list is empty due to signal

[ Upstream commit 3a010c493271f04578b133de977e0e5dd2848cea ]

When a interruptible mutex locker is interrupted by a signal
without acquiring this lock and removed from the wait queue.
if the mutex isn't contended enough to have a waiter
put into the wait queue again, the setting of the WAITER
bit will force mutex locker to go into the slowpath to
acquire the lock every time, so if the wait queue is empty,
the WAITER bit need to be clear.

Fixes: 040a0a371005 ("mutex: Add support for wound/wait style locks")
Suggested-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Zqiang <qiang.zhang@windriver.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lkml.kernel.org/r/20210517034005.30828-1-qiang.zhang@windriver.com
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/kernel/locking/mutex-debug.c b/kernel/locking/mutex-debug.c
index 9aa713629387c37be9e21a3769213769c7a67313..839df4383799cb0dbdb9e244a1b4b8f178e7be52 100644 (file)
--- a/kernel/locking/mutex-debug.c
+++ b/kernel/locking/mutex-debug.c
@@ -57,7 +57,7 @@ void debug_mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter,
        task->blocked_on = waiter;
 }
 
-void mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter,
+void debug_mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter,
                         struct task_struct *task)
 {
        DEBUG_LOCKS_WARN_ON(list_empty(&waiter->list));
@@ -65,7 +65,7 @@ void mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter,
        DEBUG_LOCKS_WARN_ON(task->blocked_on != waiter);
        task->blocked_on = NULL;
 
-       list_del_init(&waiter->list);
+       INIT_LIST_HEAD(&waiter->list);
        waiter->task = NULL;
 }
 

diff --git a/kernel/locking/mutex-debug.h b/kernel/locking/mutex-debug.h
index 1edd3f45a4ecbad7979f26944a9173b3de028959..53e631e1d76da7858839a9aa92d47bff9603c9c2 100644 (file)
--- a/kernel/locking/mutex-debug.h
+++ b/kernel/locking/mutex-debug.h
@@ -22,7 +22,7 @@ extern void debug_mutex_free_waiter(struct mutex_waiter *waiter);
 extern void debug_mutex_add_waiter(struct mutex *lock,
                                   struct mutex_waiter *waiter,
                                   struct task_struct *task);
-extern void mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter,
+extern void debug_mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter,
                                struct task_struct *task);
 extern void debug_mutex_unlock(struct mutex *lock);
 extern void debug_mutex_init(struct mutex *lock, const char *name,

diff --git a/kernel/locking/mutex.c b/kernel/locking/mutex.c
index b3da782cdfbd73c9f532a59892a466f6bf6cae84..354151fef06aeff54e9ac426b72d1a5cec9ec076 100644 (file)
--- a/kernel/locking/mutex.c
+++ b/kernel/locking/mutex.c
@@ -177,7 +177,7 @@ static inline bool __mutex_waiter_is_first(struct mutex *lock, struct mutex_wait
  * Add @waiter to a given location in the lock wait_list and set the
  * FLAG_WAITERS flag if it's the first waiter.
  */
-static void __sched
+static void
 __mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter,
                   struct list_head *list)
 {
@@ -188,6 +188,16 @@ __mutex_add_waiter(struct mutex *lock, struct mutex_waiter *waiter,
                __mutex_set_flag(lock, MUTEX_FLAG_WAITERS);
 }
 
+static void
+__mutex_remove_waiter(struct mutex *lock, struct mutex_waiter *waiter)
+{
+       list_del(&waiter->list);
+       if (likely(list_empty(&lock->wait_list)))
+               __mutex_clear_flag(lock, MUTEX_FLAGS);
+
+       debug_mutex_remove_waiter(lock, waiter, current);
+}
+
 /*
  * Give up ownership to a specific task, when @task = NULL, this is equivalent
  * to a regular unlock. Sets PICKUP on a handoff, clears HANDOF, preserves
@@ -1040,9 +1050,7 @@ acquired:
                        __ww_mutex_check_waiters(lock, ww_ctx);
        }
 
-       mutex_remove_waiter(lock, &waiter, current);
-       if (likely(list_empty(&lock->wait_list)))
-               __mutex_clear_flag(lock, MUTEX_FLAGS);
+       __mutex_remove_waiter(lock, &waiter);
 
        debug_mutex_free_waiter(&waiter);
 
@@ -1059,7 +1067,7 @@ skip_wait:
 
 err:
        __set_current_state(TASK_RUNNING);
-       mutex_remove_waiter(lock, &waiter, current);
+       __mutex_remove_waiter(lock, &waiter);
 err_early_kill:
        spin_unlock(&lock->wait_lock);
        debug_mutex_free_waiter(&waiter);

diff --git a/kernel/locking/mutex.h b/kernel/locking/mutex.h
index 1c2287d3fa7191589c6d8f0d5f4e67c3282f8dfe..f0c710b1d1927131c2526dcfc7b96010ecaa45af 100644 (file)
--- a/kernel/locking/mutex.h
+++ b/kernel/locking/mutex.h
@@ -10,12 +10,10 @@
  * !CONFIG_DEBUG_MUTEXES case. Most of them are NOPs:
  */
 
-#define mutex_remove_waiter(lock, waiter, task) \
-               __list_del((waiter)->list.prev, (waiter)->list.next)
-
 #define debug_mutex_wake_waiter(lock, waiter)          do { } while (0)
 #define debug_mutex_free_waiter(waiter)                        do { } while (0)
 #define debug_mutex_add_waiter(lock, waiter, ti)       do { } while (0)
+#define debug_mutex_remove_waiter(lock, waiter, ti)     do { } while (0)
 #define debug_mutex_unlock(lock)                       do { } while (0)
 #define debug_mutex_init(lock, name, key)              do { } while (0)