From 34d80c93a5bbf38938e8c215ec6c938807edeaf0 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Fri, 5 Dec 2025 01:35:09 +0000 Subject: [PATCH 01/90] test-ww_mutex: Extend ww_mutex tests to test both classes of ww_mutexes Currently the test-ww_mutex tool only utilizes the wait-die class of ww_mutexes, and thus isn't very helpful in exercising the wait-wound class of ww_mutexes. So extend the test to exercise both classes of ww_mutexes for all of the subtests. Signed-off-by: John Stultz Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251205013515.759030-2-jstultz@google.com --- kernel/locking/test-ww_mutex.c | 152 ++++++++++++++++++++------------- 1 file changed, 92 insertions(+), 60 deletions(-) diff --git a/kernel/locking/test-ww_mutex.c b/kernel/locking/test-ww_mutex.c index bcb1b9fea588..d27aaaa860a3 100644 --- a/kernel/locking/test-ww_mutex.c +++ b/kernel/locking/test-ww_mutex.c @@ -13,7 +13,8 @@ #include #include -static DEFINE_WD_CLASS(ww_class); +static DEFINE_WD_CLASS(wd_class); +static DEFINE_WW_CLASS(ww_class); struct workqueue_struct *wq; #ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH @@ -54,16 +55,16 @@ static void test_mutex_work(struct work_struct *work) ww_mutex_unlock(&mtx->mutex); } -static int __test_mutex(unsigned int flags) +static int __test_mutex(struct ww_class *class, unsigned int flags) { #define TIMEOUT (HZ / 16) struct test_mutex mtx; struct ww_acquire_ctx ctx; int ret; - ww_mutex_init(&mtx.mutex, &ww_class); + ww_mutex_init(&mtx.mutex, class); if (flags & TEST_MTX_CTX) - ww_acquire_init(&ctx, &ww_class); + ww_acquire_init(&ctx, class); INIT_WORK_ONSTACK(&mtx.work, test_mutex_work); init_completion(&mtx.ready); @@ -106,13 +107,13 @@ static int __test_mutex(unsigned int flags) #undef TIMEOUT } -static int test_mutex(void) +static int test_mutex(struct ww_class *class) { int ret; int i; for (i = 0; i < __TEST_MTX_LAST; i++) { - ret = __test_mutex(i); + ret = __test_mutex(class, i); if (ret) return ret; } @@ -120,15 +121,15 @@ static int test_mutex(void) return 0; } -static int test_aa(bool trylock) +static int test_aa(struct ww_class *class, bool trylock) { struct ww_mutex mutex; struct ww_acquire_ctx ctx; int ret; const char *from = trylock ? "trylock" : "lock"; - ww_mutex_init(&mutex, &ww_class); - ww_acquire_init(&ctx, &ww_class); + ww_mutex_init(&mutex, class); + ww_acquire_init(&ctx, class); if (!trylock) { ret = ww_mutex_lock(&mutex, &ctx); @@ -177,6 +178,7 @@ out: struct test_abba { struct work_struct work; + struct ww_class *class; struct ww_mutex a_mutex; struct ww_mutex b_mutex; struct completion a_ready; @@ -191,7 +193,7 @@ static void test_abba_work(struct work_struct *work) struct ww_acquire_ctx ctx; int err; - ww_acquire_init_noinject(&ctx, &ww_class); + ww_acquire_init_noinject(&ctx, abba->class); if (!abba->trylock) ww_mutex_lock(&abba->b_mutex, &ctx); else @@ -217,23 +219,24 @@ static void test_abba_work(struct work_struct *work) abba->result = err; } -static int test_abba(bool trylock, bool resolve) +static int test_abba(struct ww_class *class, bool trylock, bool resolve) { struct test_abba abba; struct ww_acquire_ctx ctx; int err, ret; - ww_mutex_init(&abba.a_mutex, &ww_class); - ww_mutex_init(&abba.b_mutex, &ww_class); + ww_mutex_init(&abba.a_mutex, class); + ww_mutex_init(&abba.b_mutex, class); INIT_WORK_ONSTACK(&abba.work, test_abba_work); init_completion(&abba.a_ready); init_completion(&abba.b_ready); + abba.class = class; abba.trylock = trylock; abba.resolve = resolve; schedule_work(&abba.work); - ww_acquire_init_noinject(&ctx, &ww_class); + ww_acquire_init_noinject(&ctx, class); if (!trylock) ww_mutex_lock(&abba.a_mutex, &ctx); else @@ -278,6 +281,7 @@ static int test_abba(bool trylock, bool resolve) struct test_cycle { struct work_struct work; + struct ww_class *class; struct ww_mutex a_mutex; struct ww_mutex *b_mutex; struct completion *a_signal; @@ -291,7 +295,7 @@ static void test_cycle_work(struct work_struct *work) struct ww_acquire_ctx ctx; int err, erra = 0; - ww_acquire_init_noinject(&ctx, &ww_class); + ww_acquire_init_noinject(&ctx, cycle->class); ww_mutex_lock(&cycle->a_mutex, &ctx); complete(cycle->a_signal); @@ -314,7 +318,7 @@ static void test_cycle_work(struct work_struct *work) cycle->result = err ?: erra; } -static int __test_cycle(unsigned int nthreads) +static int __test_cycle(struct ww_class *class, unsigned int nthreads) { struct test_cycle *cycles; unsigned int n, last = nthreads - 1; @@ -327,7 +331,8 @@ static int __test_cycle(unsigned int nthreads) for (n = 0; n < nthreads; n++) { struct test_cycle *cycle = &cycles[n]; - ww_mutex_init(&cycle->a_mutex, &ww_class); + cycle->class = class; + ww_mutex_init(&cycle->a_mutex, class); if (n == last) cycle->b_mutex = &cycles[0].a_mutex; else @@ -367,13 +372,13 @@ static int __test_cycle(unsigned int nthreads) return ret; } -static int test_cycle(unsigned int ncpus) +static int test_cycle(struct ww_class *class, unsigned int ncpus) { unsigned int n; int ret; for (n = 2; n <= ncpus + 1; n++) { - ret = __test_cycle(n); + ret = __test_cycle(class, n); if (ret) return ret; } @@ -384,6 +389,7 @@ static int test_cycle(unsigned int ncpus) struct stress { struct work_struct work; struct ww_mutex *locks; + struct ww_class *class; unsigned long timeout; int nlocks; }; @@ -443,7 +449,7 @@ static void stress_inorder_work(struct work_struct *work) int contended = -1; int n, err; - ww_acquire_init(&ctx, &ww_class); + ww_acquire_init(&ctx, stress->class); retry: err = 0; for (n = 0; n < nlocks; n++) { @@ -511,7 +517,7 @@ static void stress_reorder_work(struct work_struct *work) order = NULL; do { - ww_acquire_init(&ctx, &ww_class); + ww_acquire_init(&ctx, stress->class); list_for_each_entry(ll, &locks, link) { err = ww_mutex_lock(ll->lock, &ctx); @@ -570,7 +576,7 @@ static void stress_one_work(struct work_struct *work) #define STRESS_ONE BIT(2) #define STRESS_ALL (STRESS_INORDER | STRESS_REORDER | STRESS_ONE) -static int stress(int nlocks, int nthreads, unsigned int flags) +static int stress(struct ww_class *class, int nlocks, int nthreads, unsigned int flags) { struct ww_mutex *locks; struct stress *stress_array; @@ -588,7 +594,7 @@ static int stress(int nlocks, int nthreads, unsigned int flags) } for (n = 0; n < nlocks; n++) - ww_mutex_init(&locks[n], &ww_class); + ww_mutex_init(&locks[n], class); count = 0; for (n = 0; nthreads; n++) { @@ -617,6 +623,7 @@ static int stress(int nlocks, int nthreads, unsigned int flags) stress = &stress_array[count++]; INIT_WORK(&stress->work, fn); + stress->class = class; stress->locks = locks; stress->nlocks = nlocks; stress->timeout = jiffies + 2*HZ; @@ -635,12 +642,70 @@ static int stress(int nlocks, int nthreads, unsigned int flags) return 0; } -static int __init test_ww_mutex_init(void) +static int __init run_tests(struct ww_class *class) { int ncpus = num_online_cpus(); int ret, i; - printk(KERN_INFO "Beginning ww mutex selftests\n"); + ret = test_mutex(class); + if (ret) + return ret; + + ret = test_aa(class, false); + if (ret) + return ret; + + ret = test_aa(class, true); + if (ret) + return ret; + + for (i = 0; i < 4; i++) { + ret = test_abba(class, i & 1, i & 2); + if (ret) + return ret; + } + + ret = test_cycle(class, ncpus); + if (ret) + return ret; + + ret = stress(class, 16, 2 * ncpus, STRESS_INORDER); + if (ret) + return ret; + + ret = stress(class, 16, 2 * ncpus, STRESS_REORDER); + if (ret) + return ret; + + ret = stress(class, 2046, hweight32(STRESS_ALL) * ncpus, STRESS_ALL); + if (ret) + return ret; + + return 0; +} + +static int __init run_test_classes(void) +{ + int ret; + + pr_info("Beginning ww (wound) mutex selftests\n"); + + ret = run_tests(&ww_class); + if (ret) + return ret; + + pr_info("Beginning ww (die) mutex selftests\n"); + ret = run_tests(&wd_class); + if (ret) + return ret; + + pr_info("All ww mutex selftests passed\n"); + return 0; +} + +static int __init test_ww_mutex_init(void) +{ + int ret; prandom_seed_state(&rng, get_random_u64()); @@ -648,42 +713,9 @@ static int __init test_ww_mutex_init(void) if (!wq) return -ENOMEM; - ret = test_mutex(); - if (ret) - return ret; + ret = run_test_classes(); - ret = test_aa(false); - if (ret) - return ret; - - ret = test_aa(true); - if (ret) - return ret; - - for (i = 0; i < 4; i++) { - ret = test_abba(i & 1, i & 2); - if (ret) - return ret; - } - - ret = test_cycle(ncpus); - if (ret) - return ret; - - ret = stress(16, 2*ncpus, STRESS_INORDER); - if (ret) - return ret; - - ret = stress(16, 2*ncpus, STRESS_REORDER); - if (ret) - return ret; - - ret = stress(2046, hweight32(STRESS_ALL)*ncpus, STRESS_ALL); - if (ret) - return ret; - - printk(KERN_INFO "All ww mutex selftests passed\n"); - return 0; + return ret; } static void __exit test_ww_mutex_exit(void) From d327e7166efa24c69719890ea332b55a9dea21a7 Mon Sep 17 00:00:00 2001 From: John Stultz Date: Fri, 5 Dec 2025 01:35:10 +0000 Subject: [PATCH 02/90] test-ww_mutex: Move work to its own UNBOUND workqueue The test-ww_mutex test already allocates its own workqueue so be sure to use it for the mtx.work and abba.work rather then the default system workqueue. This resolves numerous messages of the sort: "workqueue: test_abba_work hogged CPU... consider switching to WQ_UNBOUND" "workqueue: test_mutex_work hogged CPU... consider switching to WQ_UNBOUND" Signed-off-by: John Stultz Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251205013515.759030-3-jstultz@google.com --- kernel/locking/test-ww_mutex.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/locking/test-ww_mutex.c b/kernel/locking/test-ww_mutex.c index d27aaaa860a3..30512b3e95c9 100644 --- a/kernel/locking/test-ww_mutex.c +++ b/kernel/locking/test-ww_mutex.c @@ -72,7 +72,7 @@ static int __test_mutex(struct ww_class *class, unsigned int flags) init_completion(&mtx.done); mtx.flags = flags; - schedule_work(&mtx.work); + queue_work(wq, &mtx.work); wait_for_completion(&mtx.ready); ww_mutex_lock(&mtx.mutex, (flags & TEST_MTX_CTX) ? &ctx : NULL); @@ -234,7 +234,7 @@ static int test_abba(struct ww_class *class, bool trylock, bool resolve) abba.trylock = trylock; abba.resolve = resolve; - schedule_work(&abba.work); + queue_work(wq, &abba.work); ww_acquire_init_noinject(&ctx, class); if (!trylock) From de2c5a1523fde38411b6259064258a0c0a3c896a Mon Sep 17 00:00:00 2001 From: John Stultz Date: Fri, 5 Dec 2025 01:35:11 +0000 Subject: [PATCH 03/90] test-ww_mutex: Allow test to be run (and re-run) from userland In cases where the ww_mutex test was occasionally tripping on hard to find issues, leaving qemu in a reboot loop was my best way to reproduce problems. These reboots however wasted time when I just wanted to run the test-ww_mutex logic. So tweak the test-ww_mutex test so that it can be re-triggered via a sysfs file, so the test can be run repeatedly without doing module loads or restarting. This has been particularly valuable to stressing and finding issues with the proxy-exec series. To use, run as root: echo 1 > /sys/kernel/test_ww_mutex/run_tests Signed-off-by: John Stultz Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251205013515.759030-4-jstultz@google.com --- kernel/locking/test-ww_mutex.c | 51 ++++++++++++++++++++++++++++++++-- 1 file changed, 49 insertions(+), 2 deletions(-) diff --git a/kernel/locking/test-ww_mutex.c b/kernel/locking/test-ww_mutex.c index 30512b3e95c9..79b5e45f8d4c 100644 --- a/kernel/locking/test-ww_mutex.c +++ b/kernel/locking/test-ww_mutex.c @@ -642,7 +642,7 @@ static int stress(struct ww_class *class, int nlocks, int nthreads, unsigned int return 0; } -static int __init run_tests(struct ww_class *class) +static int run_tests(struct ww_class *class) { int ncpus = num_online_cpus(); int ret, i; @@ -684,7 +684,7 @@ static int __init run_tests(struct ww_class *class) return 0; } -static int __init run_test_classes(void) +static int run_test_classes(void) { int ret; @@ -703,6 +703,36 @@ static int __init run_test_classes(void) return 0; } +static DEFINE_MUTEX(run_lock); + +static ssize_t run_tests_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + if (!mutex_trylock(&run_lock)) { + pr_err("Test already running\n"); + return count; + } + + run_test_classes(); + mutex_unlock(&run_lock); + + return count; +} + +static struct kobj_attribute run_tests_attribute = + __ATTR(run_tests, 0664, NULL, run_tests_store); + +static struct attribute *attrs[] = { + &run_tests_attribute.attr, + NULL, /* need to NULL terminate the list of attributes */ +}; + +static struct attribute_group attr_group = { + .attrs = attrs, +}; + +static struct kobject *test_ww_mutex_kobj; + static int __init test_ww_mutex_init(void) { int ret; @@ -713,13 +743,30 @@ static int __init test_ww_mutex_init(void) if (!wq) return -ENOMEM; + test_ww_mutex_kobj = kobject_create_and_add("test_ww_mutex", kernel_kobj); + if (!test_ww_mutex_kobj) { + destroy_workqueue(wq); + return -ENOMEM; + } + + /* Create the files associated with this kobject */ + ret = sysfs_create_group(test_ww_mutex_kobj, &attr_group); + if (ret) { + kobject_put(test_ww_mutex_kobj); + destroy_workqueue(wq); + return ret; + } + + mutex_lock(&run_lock); ret = run_test_classes(); + mutex_unlock(&run_lock); return ret; } static void __exit test_ww_mutex_exit(void) { + kobject_put(test_ww_mutex_kobj); destroy_workqueue(wq); } From de15fecae44df8254fa597bad7eb3680a8b1c10c Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:39:50 +0100 Subject: [PATCH 04/90] compiler_types: Move lock checking attributes to compiler-context-analysis.h The conditional definition of lock checking macros and attributes is about to become more complex. Factor them out into their own header for better readability, and to make it obvious which features are supported by which mode (currently only Sparse). This is the first step towards generalizing towards "context analysis". No functional change intended. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Bart Van Assche Link: https://patch.msgid.link/20251219154418.3592607-2-elver@google.com --- include/linux/compiler-context-analysis.h | 32 +++++++++++++++++++++++ include/linux/compiler_types.h | 18 ++----------- 2 files changed, 34 insertions(+), 16 deletions(-) create mode 100644 include/linux/compiler-context-analysis.h diff --git a/include/linux/compiler-context-analysis.h b/include/linux/compiler-context-analysis.h new file mode 100644 index 000000000000..f8af63045281 --- /dev/null +++ b/include/linux/compiler-context-analysis.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Macros and attributes for compiler-based static context analysis. + */ + +#ifndef _LINUX_COMPILER_CONTEXT_ANALYSIS_H +#define _LINUX_COMPILER_CONTEXT_ANALYSIS_H + +#ifdef __CHECKER__ + +/* Sparse context/lock checking support. */ +# define __must_hold(x) __attribute__((context(x,1,1))) +# define __acquires(x) __attribute__((context(x,0,1))) +# define __cond_acquires(x) __attribute__((context(x,0,-1))) +# define __releases(x) __attribute__((context(x,1,0))) +# define __acquire(x) __context__(x,1) +# define __release(x) __context__(x,-1) +# define __cond_lock(x, c) ((c) ? ({ __acquire(x); 1; }) : 0) + +#else /* !__CHECKER__ */ + +# define __must_hold(x) +# define __acquires(x) +# define __cond_acquires(x) +# define __releases(x) +# define __acquire(x) (void)0 +# define __release(x) (void)0 +# define __cond_lock(x, c) (c) + +#endif /* __CHECKER__ */ + +#endif /* _LINUX_COMPILER_CONTEXT_ANALYSIS_H */ diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index 1280693766b9..ddada1ed88ea 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -41,6 +41,8 @@ # define BTF_TYPE_TAG(value) /* nothing */ #endif +#include + /* sparse defines __CHECKER__; see Documentation/dev-tools/sparse.rst */ #ifdef __CHECKER__ /* address spaces */ @@ -51,14 +53,6 @@ # define __rcu __attribute__((noderef, address_space(__rcu))) static inline void __chk_user_ptr(const volatile void __user *ptr) { } static inline void __chk_io_ptr(const volatile void __iomem *ptr) { } -/* context/locking */ -# define __must_hold(x) __attribute__((context(x,1,1))) -# define __acquires(x) __attribute__((context(x,0,1))) -# define __cond_acquires(x) __attribute__((context(x,0,-1))) -# define __releases(x) __attribute__((context(x,1,0))) -# define __acquire(x) __context__(x,1) -# define __release(x) __context__(x,-1) -# define __cond_lock(x,c) ((c) ? ({ __acquire(x); 1; }) : 0) /* other */ # define __force __attribute__((force)) # define __nocast __attribute__((nocast)) @@ -79,14 +73,6 @@ static inline void __chk_io_ptr(const volatile void __iomem *ptr) { } # define __chk_user_ptr(x) (void)0 # define __chk_io_ptr(x) (void)0 -/* context/locking */ -# define __must_hold(x) -# define __acquires(x) -# define __cond_acquires(x) -# define __releases(x) -# define __acquire(x) (void)0 -# define __release(x) (void)0 -# define __cond_lock(x,c) (c) /* other */ # define __force # define __nocast From 3269701cb25662ae8a9771a864201116626adb50 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:39:51 +0100 Subject: [PATCH 05/90] compiler-context-analysis: Add infrastructure for Context Analysis with Clang Context Analysis is a language extension, which enables statically checking that required contexts are active (or inactive), by acquiring and releasing user-definable "context locks". An obvious application is lock-safety checking for the kernel's various synchronization primitives (each of which represents a "context lock"), and checking that locking rules are not violated. Clang originally called the feature "Thread Safety Analysis" [1]. This was later changed and the feature became more flexible, gaining the ability to define custom "capabilities". Its foundations can be found in "Capability Systems" [2], used to specify the permissibility of operations to depend on some "capability" being held (or not held). Because the feature is not just able to express "capabilities" related to synchronization primitives, and "capability" is already overloaded in the kernel, the naming chosen for the kernel departs from Clang's "Thread Safety" and "capability" nomenclature; we refer to the feature as "Context Analysis" to avoid confusion. The internal implementation still makes references to Clang's terminology in a few places, such as `-Wthread-safety` being the warning option that also still appears in diagnostic messages. [1] https://clang.llvm.org/docs/ThreadSafetyAnalysis.html [2] https://www.cs.cornell.edu/talc/papers/capabilities.pdf See more details in the kernel-doc documentation added in this and subsequent changes. Clang version 22+ is required. [peterz: disable the thing for __CHECKER__ builds] Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-3-elver@google.com --- Makefile | 1 + include/linux/compiler-context-analysis.h | 464 +++++++++++++++++++++- lib/Kconfig.debug | 30 ++ scripts/Makefile.context-analysis | 7 + scripts/Makefile.lib | 10 + 5 files changed, 505 insertions(+), 7 deletions(-) create mode 100644 scripts/Makefile.context-analysis diff --git a/Makefile b/Makefile index e404e4767944..d4c2aa2df79c 100644 --- a/Makefile +++ b/Makefile @@ -1118,6 +1118,7 @@ include-$(CONFIG_RANDSTRUCT) += scripts/Makefile.randstruct include-$(CONFIG_KSTACK_ERASE) += scripts/Makefile.kstack_erase include-$(CONFIG_AUTOFDO_CLANG) += scripts/Makefile.autofdo include-$(CONFIG_PROPELLER_CLANG) += scripts/Makefile.propeller +include-$(CONFIG_WARN_CONTEXT_ANALYSIS) += scripts/Makefile.context-analysis include-$(CONFIG_GCC_PLUGINS) += scripts/Makefile.gcc-plugins include $(addprefix $(srctree)/, $(include-y)) diff --git a/include/linux/compiler-context-analysis.h b/include/linux/compiler-context-analysis.h index f8af63045281..d0b3cf0ebfe9 100644 --- a/include/linux/compiler-context-analysis.h +++ b/include/linux/compiler-context-analysis.h @@ -6,27 +6,477 @@ #ifndef _LINUX_COMPILER_CONTEXT_ANALYSIS_H #define _LINUX_COMPILER_CONTEXT_ANALYSIS_H +#if defined(WARN_CONTEXT_ANALYSIS) && !defined(__CHECKER__) + +/* + * These attributes define new context lock (Clang: capability) types. + * Internal only. + */ +# define __ctx_lock_type(name) __attribute__((capability(#name))) +# define __reentrant_ctx_lock __attribute__((reentrant_capability)) +# define __acquires_ctx_lock(...) __attribute__((acquire_capability(__VA_ARGS__))) +# define __acquires_shared_ctx_lock(...) __attribute__((acquire_shared_capability(__VA_ARGS__))) +# define __try_acquires_ctx_lock(ret, var) __attribute__((try_acquire_capability(ret, var))) +# define __try_acquires_shared_ctx_lock(ret, var) __attribute__((try_acquire_shared_capability(ret, var))) +# define __releases_ctx_lock(...) __attribute__((release_capability(__VA_ARGS__))) +# define __releases_shared_ctx_lock(...) __attribute__((release_shared_capability(__VA_ARGS__))) +# define __returns_ctx_lock(var) __attribute__((lock_returned(var))) + +/* + * The below are used to annotate code being checked. Internal only. + */ +# define __excludes_ctx_lock(...) __attribute__((locks_excluded(__VA_ARGS__))) +# define __requires_ctx_lock(...) __attribute__((requires_capability(__VA_ARGS__))) +# define __requires_shared_ctx_lock(...) __attribute__((requires_shared_capability(__VA_ARGS__))) + +/* + * The "assert_capability" attribute is a bit confusingly named. It does not + * generate a check. Instead, it tells the analysis to *assume* the capability + * is held. This is used for: + * + * 1. Augmenting runtime assertions, that can then help with patterns beyond the + * compiler's static reasoning abilities. + * + * 2. Initialization of context locks, so we can access guarded variables right + * after initialization (nothing else should access the same object yet). + */ +# define __assumes_ctx_lock(...) __attribute__((assert_capability(__VA_ARGS__))) +# define __assumes_shared_ctx_lock(...) __attribute__((assert_shared_capability(__VA_ARGS__))) + +/** + * __guarded_by - struct member and globals attribute, declares variable + * only accessible within active context + * + * Declares that the struct member or global variable is only accessible within + * the context entered by the given context lock. Read operations on the data + * require shared access, while write operations require exclusive access. + * + * .. code-block:: c + * + * struct some_state { + * spinlock_t lock; + * long counter __guarded_by(&lock); + * }; + */ +# define __guarded_by(...) __attribute__((guarded_by(__VA_ARGS__))) + +/** + * __pt_guarded_by - struct member and globals attribute, declares pointed-to + * data only accessible within active context + * + * Declares that the data pointed to by the struct member pointer or global + * pointer is only accessible within the context entered by the given context + * lock. Read operations on the data require shared access, while write + * operations require exclusive access. + * + * .. code-block:: c + * + * struct some_state { + * spinlock_t lock; + * long *counter __pt_guarded_by(&lock); + * }; + */ +# define __pt_guarded_by(...) __attribute__((pt_guarded_by(__VA_ARGS__))) + +/** + * context_lock_struct() - declare or define a context lock struct + * @name: struct name + * + * Helper to declare or define a struct type that is also a context lock. + * + * .. code-block:: c + * + * context_lock_struct(my_handle) { + * int foo; + * long bar; + * }; + * + * struct some_state { + * ... + * }; + * // ... declared elsewhere ... + * context_lock_struct(some_state); + * + * Note: The implementation defines several helper functions that can acquire + * and release the context lock. + */ +# define context_lock_struct(name, ...) \ + struct __ctx_lock_type(name) __VA_ARGS__ name; \ + static __always_inline void __acquire_ctx_lock(const struct name *var) \ + __attribute__((overloadable)) __no_context_analysis __acquires_ctx_lock(var) { } \ + static __always_inline void __acquire_shared_ctx_lock(const struct name *var) \ + __attribute__((overloadable)) __no_context_analysis __acquires_shared_ctx_lock(var) { } \ + static __always_inline bool __try_acquire_ctx_lock(const struct name *var, bool ret) \ + __attribute__((overloadable)) __no_context_analysis __try_acquires_ctx_lock(1, var) \ + { return ret; } \ + static __always_inline bool __try_acquire_shared_ctx_lock(const struct name *var, bool ret) \ + __attribute__((overloadable)) __no_context_analysis __try_acquires_shared_ctx_lock(1, var) \ + { return ret; } \ + static __always_inline void __release_ctx_lock(const struct name *var) \ + __attribute__((overloadable)) __no_context_analysis __releases_ctx_lock(var) { } \ + static __always_inline void __release_shared_ctx_lock(const struct name *var) \ + __attribute__((overloadable)) __no_context_analysis __releases_shared_ctx_lock(var) { } \ + static __always_inline void __assume_ctx_lock(const struct name *var) \ + __attribute__((overloadable)) __assumes_ctx_lock(var) { } \ + static __always_inline void __assume_shared_ctx_lock(const struct name *var) \ + __attribute__((overloadable)) __assumes_shared_ctx_lock(var) { } \ + struct name + +/** + * disable_context_analysis() - disables context analysis + * + * Disables context analysis. Must be paired with a later + * enable_context_analysis(). + */ +# define disable_context_analysis() \ + __diag_push(); \ + __diag_ignore_all("-Wunknown-warning-option", "") \ + __diag_ignore_all("-Wthread-safety", "") \ + __diag_ignore_all("-Wthread-safety-pointer", "") + +/** + * enable_context_analysis() - re-enables context analysis + * + * Re-enables context analysis. Must be paired with a prior + * disable_context_analysis(). + */ +# define enable_context_analysis() __diag_pop() + +/** + * __no_context_analysis - function attribute, disables context analysis + * + * Function attribute denoting that context analysis is disabled for the + * whole function. Prefer use of `context_unsafe()` where possible. + */ +# define __no_context_analysis __attribute__((no_thread_safety_analysis)) + +#else /* !WARN_CONTEXT_ANALYSIS */ + +# define __ctx_lock_type(name) +# define __reentrant_ctx_lock +# define __acquires_ctx_lock(...) +# define __acquires_shared_ctx_lock(...) +# define __try_acquires_ctx_lock(ret, var) +# define __try_acquires_shared_ctx_lock(ret, var) +# define __releases_ctx_lock(...) +# define __releases_shared_ctx_lock(...) +# define __assumes_ctx_lock(...) +# define __assumes_shared_ctx_lock(...) +# define __returns_ctx_lock(var) +# define __guarded_by(...) +# define __pt_guarded_by(...) +# define __excludes_ctx_lock(...) +# define __requires_ctx_lock(...) +# define __requires_shared_ctx_lock(...) +# define __acquire_ctx_lock(var) do { } while (0) +# define __acquire_shared_ctx_lock(var) do { } while (0) +# define __try_acquire_ctx_lock(var, ret) (ret) +# define __try_acquire_shared_ctx_lock(var, ret) (ret) +# define __release_ctx_lock(var) do { } while (0) +# define __release_shared_ctx_lock(var) do { } while (0) +# define __assume_ctx_lock(var) do { (void)(var); } while (0) +# define __assume_shared_ctx_lock(var) do { (void)(var); } while (0) +# define context_lock_struct(name, ...) struct __VA_ARGS__ name +# define disable_context_analysis() +# define enable_context_analysis() +# define __no_context_analysis + +#endif /* WARN_CONTEXT_ANALYSIS */ + +/** + * context_unsafe() - disable context checking for contained code + * + * Disables context checking for contained statements or expression. + * + * .. code-block:: c + * + * struct some_data { + * spinlock_t lock; + * int counter __guarded_by(&lock); + * }; + * + * int foo(struct some_data *d) + * { + * // ... + * // other code that is still checked ... + * // ... + * return context_unsafe(d->counter); + * } + */ +#define context_unsafe(...) \ +({ \ + disable_context_analysis(); \ + __VA_ARGS__; \ + enable_context_analysis() \ +}) + +/** + * __context_unsafe() - function attribute, disable context checking + * @comment: comment explaining why opt-out is safe + * + * Function attribute denoting that context analysis is disabled for the + * whole function. Forces adding an inline comment as argument. + */ +#define __context_unsafe(comment) __no_context_analysis + +/** + * context_unsafe_alias() - helper to insert a context lock "alias barrier" + * @p: pointer aliasing a context lock or object containing context locks + * + * No-op function that acts as a "context lock alias barrier", where the + * analysis rightfully detects that we're switching aliases, but the switch is + * considered safe but beyond the analysis reasoning abilities. + * + * This should be inserted before the first use of such an alias. + * + * Implementation Note: The compiler ignores aliases that may be reassigned but + * their value cannot be determined (e.g. when passing a non-const pointer to an + * alias as a function argument). + */ +#define context_unsafe_alias(p) _context_unsafe_alias((void **)&(p)) +static inline void _context_unsafe_alias(void **p) { } + +/** + * token_context_lock() - declare an abstract global context lock instance + * @name: token context lock name + * + * Helper that declares an abstract global context lock instance @name, but not + * backed by a real data structure (linker error if accidentally referenced). + * The type name is `__ctx_lock_@name`. + */ +#define token_context_lock(name, ...) \ + context_lock_struct(__ctx_lock_##name, ##__VA_ARGS__) {}; \ + extern const struct __ctx_lock_##name *name + +/** + * token_context_lock_instance() - declare another instance of a global context lock + * @ctx: token context lock previously declared with token_context_lock() + * @name: name of additional global context lock instance + * + * Helper that declares an additional instance @name of the same token context + * lock class @ctx. This is helpful where multiple related token contexts are + * declared, to allow using the same underlying type (`__ctx_lock_@ctx`) as + * function arguments. + */ +#define token_context_lock_instance(ctx, name) \ + extern const struct __ctx_lock_##ctx *name + +/* + * Common keywords for static context analysis. Both Clang's "capability + * analysis" and Sparse's "context tracking" are currently supported. + */ #ifdef __CHECKER__ /* Sparse context/lock checking support. */ # define __must_hold(x) __attribute__((context(x,1,1))) +# define __must_not_hold(x) # define __acquires(x) __attribute__((context(x,0,1))) # define __cond_acquires(x) __attribute__((context(x,0,-1))) # define __releases(x) __attribute__((context(x,1,0))) # define __acquire(x) __context__(x,1) # define __release(x) __context__(x,-1) # define __cond_lock(x, c) ((c) ? ({ __acquire(x); 1; }) : 0) +/* For Sparse, there's no distinction between exclusive and shared locks. */ +# define __must_hold_shared __must_hold +# define __acquires_shared __acquires +# define __cond_acquires_shared __cond_acquires +# define __releases_shared __releases +# define __acquire_shared __acquire +# define __release_shared __release +# define __cond_lock_shared __cond_acquire #else /* !__CHECKER__ */ -# define __must_hold(x) -# define __acquires(x) -# define __cond_acquires(x) -# define __releases(x) -# define __acquire(x) (void)0 -# define __release(x) (void)0 -# define __cond_lock(x, c) (c) +/** + * __must_hold() - function attribute, caller must hold exclusive context lock + * @x: context lock instance pointer + * + * Function attribute declaring that the caller must hold the given context + * lock instance @x exclusively. + */ +# define __must_hold(x) __requires_ctx_lock(x) + +/** + * __must_not_hold() - function attribute, caller must not hold context lock + * @x: context lock instance pointer + * + * Function attribute declaring that the caller must not hold the given context + * lock instance @x. + */ +# define __must_not_hold(x) __excludes_ctx_lock(x) + +/** + * __acquires() - function attribute, function acquires context lock exclusively + * @x: context lock instance pointer + * + * Function attribute declaring that the function acquires the given context + * lock instance @x exclusively, but does not release it. + */ +# define __acquires(x) __acquires_ctx_lock(x) + +/** + * __cond_acquires() - function attribute, function conditionally + * acquires a context lock exclusively + * @x: context lock instance pointer + * + * Function attribute declaring that the function conditionally acquires the + * given context lock instance @x exclusively, but does not release it. + */ +# define __cond_acquires(x) __try_acquires_ctx_lock(1, x) + +/** + * __releases() - function attribute, function releases a context lock exclusively + * @x: context lock instance pointer + * + * Function attribute declaring that the function releases the given context + * lock instance @x exclusively. The associated context must be active on + * entry. + */ +# define __releases(x) __releases_ctx_lock(x) + +/** + * __acquire() - function to acquire context lock exclusively + * @x: context lock instance pointer + * + * No-op function that acquires the given context lock instance @x exclusively. + */ +# define __acquire(x) __acquire_ctx_lock(x) + +/** + * __release() - function to release context lock exclusively + * @x: context lock instance pointer + * + * No-op function that releases the given context lock instance @x. + */ +# define __release(x) __release_ctx_lock(x) + +/** + * __cond_lock() - function that conditionally acquires a context lock + * exclusively + * @x: context lock instance pinter + * @c: boolean expression + * + * Return: result of @c + * + * No-op function that conditionally acquires context lock instance @x + * exclusively, if the boolean expression @c is true. The result of @c is the + * return value; for example: + * + * .. code-block:: c + * + * #define spin_trylock(l) __cond_lock(&lock, _spin_trylock(&lock)) + */ +# define __cond_lock(x, c) __try_acquire_ctx_lock(x, c) + +/** + * __must_hold_shared() - function attribute, caller must hold shared context lock + * @x: context lock instance pointer + * + * Function attribute declaring that the caller must hold the given context + * lock instance @x with shared access. + */ +# define __must_hold_shared(x) __requires_shared_ctx_lock(x) + +/** + * __acquires_shared() - function attribute, function acquires context lock shared + * @x: context lock instance pointer + * + * Function attribute declaring that the function acquires the given + * context lock instance @x with shared access, but does not release it. + */ +# define __acquires_shared(x) __acquires_shared_ctx_lock(x) + +/** + * __cond_acquires_shared() - function attribute, function conditionally + * acquires a context lock shared + * @x: context lock instance pointer + * + * Function attribute declaring that the function conditionally acquires the + * given context lock instance @x with shared access, but does not release it. + */ +# define __cond_acquires_shared(x) __try_acquires_shared_ctx_lock(1, x) + +/** + * __releases_shared() - function attribute, function releases a + * context lock shared + * @x: context lock instance pointer + * + * Function attribute declaring that the function releases the given context + * lock instance @x with shared access. The associated context must be active + * on entry. + */ +# define __releases_shared(x) __releases_shared_ctx_lock(x) + +/** + * __acquire_shared() - function to acquire context lock shared + * @x: context lock instance pointer + * + * No-op function that acquires the given context lock instance @x with shared + * access. + */ +# define __acquire_shared(x) __acquire_shared_ctx_lock(x) + +/** + * __release_shared() - function to release context lock shared + * @x: context lock instance pointer + * + * No-op function that releases the given context lock instance @x with shared + * access. + */ +# define __release_shared(x) __release_shared_ctx_lock(x) + +/** + * __cond_lock_shared() - function that conditionally acquires a context lock shared + * @x: context lock instance pinter + * @c: boolean expression + * + * Return: result of @c + * + * No-op function that conditionally acquires context lock instance @x with + * shared access, if the boolean expression @c is true. The result of @c is the + * return value. + */ +# define __cond_lock_shared(x, c) __try_acquire_shared_ctx_lock(x, c) #endif /* __CHECKER__ */ +/** + * __acquire_ret() - helper to acquire context lock of return value + * @call: call expression + * @ret_expr: acquire expression that uses __ret + */ +#define __acquire_ret(call, ret_expr) \ + ({ \ + __auto_type __ret = call; \ + __acquire(ret_expr); \ + __ret; \ + }) + +/** + * __acquire_shared_ret() - helper to acquire context lock shared of return value + * @call: call expression + * @ret_expr: acquire shared expression that uses __ret + */ +#define __acquire_shared_ret(call, ret_expr) \ + ({ \ + __auto_type __ret = call; \ + __acquire_shared(ret_expr); \ + __ret; \ + }) + +/* + * Attributes to mark functions returning acquired context locks. + * + * This is purely cosmetic to help readability, and should be used with the + * above macros as follows: + * + * struct foo { spinlock_t lock; ... }; + * ... + * #define myfunc(...) __acquire_ret(_myfunc(__VA_ARGS__), &__ret->lock) + * struct foo *_myfunc(int bar) __acquires_ret; + * ... + */ +#define __acquires_ret __no_context_analysis +#define __acquires_shared_ret __no_context_analysis + #endif /* _LINUX_COMPILER_CONTEXT_ANALYSIS_H */ diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index ba36939fda79..cd557e7653a4 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -621,6 +621,36 @@ config DEBUG_FORCE_WEAK_PER_CPU To ensure that generic code follows the above rules, this option forces all percpu variables to be defined as weak. +config WARN_CONTEXT_ANALYSIS + bool "Compiler context-analysis warnings" + depends on CC_IS_CLANG && CLANG_VERSION >= 220000 + # Branch profiling re-defines "if", which messes with the compiler's + # ability to analyze __cond_acquires(..), resulting in false positives. + depends on !TRACE_BRANCH_PROFILING + default y + help + Context Analysis is a language extension, which enables statically + checking that required contexts are active (or inactive) by acquiring + and releasing user-definable "context locks". + + Clang's name of the feature is "Thread Safety Analysis". Requires + Clang 22 or later. + + Produces warnings by default. Select CONFIG_WERROR if you wish to + turn these warnings into errors. + + For more details, see Documentation/dev-tools/context-analysis.rst. + +config WARN_CONTEXT_ANALYSIS_ALL + bool "Enable context analysis for all source files" + depends on WARN_CONTEXT_ANALYSIS + depends on EXPERT && !COMPILE_TEST + help + Enable tree-wide context analysis. This is likely to produce a + large number of false positives - enable at your own risk. + + If unsure, say N. + endmenu # "Compiler options" menu "Generic Kernel Debugging Instruments" diff --git a/scripts/Makefile.context-analysis b/scripts/Makefile.context-analysis new file mode 100644 index 000000000000..70549f7fae1a --- /dev/null +++ b/scripts/Makefile.context-analysis @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: GPL-2.0 + +context-analysis-cflags := -DWARN_CONTEXT_ANALYSIS \ + -fexperimental-late-parse-attributes -Wthread-safety \ + -Wthread-safety-pointer -Wthread-safety-beta + +export CFLAGS_CONTEXT_ANALYSIS := $(context-analysis-cflags) diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index 28a1c08e3b22..e429d68b8594 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -105,6 +105,16 @@ _c_flags += $(if $(patsubst n%,, \ -D__KCSAN_INSTRUMENT_BARRIERS__) endif +# +# Enable context analysis flags only where explicitly opted in. +# (depends on variables CONTEXT_ANALYSIS_obj.o, CONTEXT_ANALYSIS) +# +ifeq ($(CONFIG_WARN_CONTEXT_ANALYSIS),y) +_c_flags += $(if $(patsubst n%,, \ + $(CONTEXT_ANALYSIS_$(target-stem).o)$(CONTEXT_ANALYSIS)$(if $(is-kernel-object),$(CONFIG_WARN_CONTEXT_ANALYSIS_ALL))), \ + $(CFLAGS_CONTEXT_ANALYSIS)) +endif + # # Enable AutoFDO build flags except some files or directories we don't want to # enable (depends on variables AUTOFDO_PROFILE_obj.o and AUTOFDO_PROFILE). From 9b00c1609deeb7d6f68a61f3ec6988ab7e6f4535 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:39:52 +0100 Subject: [PATCH 06/90] compiler-context-analysis: Add test stub Add a simple test stub where we will add common supported patterns that should not generate false positives for each new supported context lock. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-4-elver@google.com --- lib/Kconfig.debug | 14 ++++++++++++++ lib/Makefile | 3 +++ lib/test_context-analysis.c | 18 ++++++++++++++++++ 3 files changed, 35 insertions(+) create mode 100644 lib/test_context-analysis.c diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index cd557e7653a4..8ca42526ee43 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -2835,6 +2835,20 @@ config LINEAR_RANGES_TEST If unsure, say N. +config CONTEXT_ANALYSIS_TEST + bool "Compiler context-analysis warnings test" + depends on EXPERT + help + This builds the test for compiler-based context analysis. The test + does not add executable code to the kernel, but is meant to test that + common patterns supported by the analysis do not result in false + positive warnings. + + When adding support for new context locks, it is strongly recommended + to add supported patterns to this test. + + If unsure, say N. + config CMDLINE_KUNIT_TEST tristate "KUnit test for cmdline API" if !KUNIT_ALL_TESTS depends on KUNIT diff --git a/lib/Makefile b/lib/Makefile index aaf677cf4527..89defefbf6c0 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -331,4 +331,7 @@ obj-$(CONFIG_GENERIC_LIB_DEVMEM_IS_ALLOWED) += devmem_is_allowed.o obj-$(CONFIG_FIRMWARE_TABLE) += fw_table.o +CONTEXT_ANALYSIS_test_context-analysis.o := y +obj-$(CONFIG_CONTEXT_ANALYSIS_TEST) += test_context-analysis.o + subdir-$(CONFIG_FORTIFY_SOURCE) += test_fortify diff --git a/lib/test_context-analysis.c b/lib/test_context-analysis.c new file mode 100644 index 000000000000..68f075dec0e0 --- /dev/null +++ b/lib/test_context-analysis.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Compile-only tests for common patterns that should not generate false + * positive errors when compiled with Clang's context analysis. + */ + +#include + +/* + * Test that helper macros work as expected. + */ +static void __used test_common_helpers(void) +{ + BUILD_BUG_ON(context_unsafe(3) != 3); /* plain expression */ + BUILD_BUG_ON(context_unsafe((void)2; 3) != 3); /* does not swallow semi-colon */ + BUILD_BUG_ON(context_unsafe((void)2, 3) != 3); /* does not swallow commas */ + context_unsafe(do { } while (0)); /* works with void statements */ +} From 8f32441d7a532804a8d9e2ae36f9b13c353934d7 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:39:53 +0100 Subject: [PATCH 07/90] Documentation: Add documentation for Compiler-Based Context Analysis Adds documentation in Documentation/dev-tools/context-analysis.rst, and adds it to the index. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-5-elver@google.com --- Documentation/dev-tools/context-analysis.rst | 144 +++++++++++++++++++ Documentation/dev-tools/index.rst | 1 + 2 files changed, 145 insertions(+) create mode 100644 Documentation/dev-tools/context-analysis.rst diff --git a/Documentation/dev-tools/context-analysis.rst b/Documentation/dev-tools/context-analysis.rst new file mode 100644 index 000000000000..47eb547eb716 --- /dev/null +++ b/Documentation/dev-tools/context-analysis.rst @@ -0,0 +1,144 @@ +.. SPDX-License-Identifier: GPL-2.0 +.. Copyright (C) 2025, Google LLC. + +.. _context-analysis: + +Compiler-Based Context Analysis +=============================== + +Context Analysis is a language extension, which enables statically checking +that required contexts are active (or inactive) by acquiring and releasing +user-definable "context locks". An obvious application is lock-safety checking +for the kernel's various synchronization primitives (each of which represents a +"context lock"), and checking that locking rules are not violated. + +The Clang compiler currently supports the full set of context analysis +features. To enable for Clang, configure the kernel with:: + + CONFIG_WARN_CONTEXT_ANALYSIS=y + +The feature requires Clang 22 or later. + +The analysis is *opt-in by default*, and requires declaring which modules and +subsystems should be analyzed in the respective `Makefile`:: + + CONTEXT_ANALYSIS_mymodule.o := y + +Or for all translation units in the directory:: + + CONTEXT_ANALYSIS := y + +It is possible to enable the analysis tree-wide, however, which will result in +numerous false positive warnings currently and is *not* generally recommended:: + + CONFIG_WARN_CONTEXT_ANALYSIS_ALL=y + +Programming Model +----------------- + +The below describes the programming model around using context lock types. + +.. note:: + Enabling context analysis can be seen as enabling a dialect of Linux C with + a Context System. Some valid patterns involving complex control-flow are + constrained (such as conditional acquisition and later conditional release + in the same function). + +Context analysis is a way to specify permissibility of operations to depend on +context locks being held (or not held). Typically we are interested in +protecting data and code in a critical section by requiring a specific context +to be active, for example by holding a specific lock. The analysis ensures that +callers cannot perform an operation without the required context being active. + +Context locks are associated with named structs, along with functions that +operate on struct instances to acquire and release the associated context lock. + +Context locks can be held either exclusively or shared. This mechanism allows +assigning more precise privileges when a context is active, typically to +distinguish where a thread may only read (shared) or also write (exclusive) to +data guarded within a context. + +The set of contexts that are actually active in a given thread at a given point +in program execution is a run-time concept. The static analysis works by +calculating an approximation of that set, called the context environment. The +context environment is calculated for every program point, and describes the +set of contexts that are statically known to be active, or inactive, at that +particular point. This environment is a conservative approximation of the full +set of contexts that will actually be active in a thread at run-time. + +More details are also documented `here +`_. + +.. note:: + Clang's analysis explicitly does not infer context locks acquired or + released by inline functions. It requires explicit annotations to (a) assert + that it's not a bug if a context lock is released or acquired, and (b) to + retain consistency between inline and non-inline function declarations. + +Supported Kernel Primitives +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. Currently the following synchronization primitives are supported: + +For context locks with an initialization function (e.g., `spin_lock_init()`), +calling this function before initializing any guarded members or globals +prevents the compiler from issuing warnings about unguarded initialization. + +Lockdep assertions, such as `lockdep_assert_held()`, inform the compiler's +context analysis that the associated synchronization primitive is held after +the assertion. This avoids false positives in complex control-flow scenarios +and encourages the use of Lockdep where static analysis is limited. For +example, this is useful when a function doesn't *always* require a lock, making +`__must_hold()` inappropriate. + +Keywords +~~~~~~~~ + +.. kernel-doc:: include/linux/compiler-context-analysis.h + :identifiers: context_lock_struct + token_context_lock token_context_lock_instance + __guarded_by __pt_guarded_by + __must_hold + __must_not_hold + __acquires + __cond_acquires + __releases + __must_hold_shared + __acquires_shared + __cond_acquires_shared + __releases_shared + __acquire + __release + __cond_lock + __acquire_shared + __release_shared + __cond_lock_shared + __acquire_ret + __acquire_shared_ret + context_unsafe + __context_unsafe + disable_context_analysis enable_context_analysis + +.. note:: + The function attribute `__no_context_analysis` is reserved for internal + implementation of context lock types, and should be avoided in normal code. + +Background +---------- + +Clang originally called the feature `Thread Safety Analysis +`_, with some keywords +and documentation still using the thread-safety-analysis-only terminology. This +was later changed and the feature became more flexible, gaining the ability to +define custom "capabilities". Its foundations can be found in `Capability +Systems `_, used to +specify the permissibility of operations to depend on some "capability" being +held (or not held). + +Because the feature is not just able to express capabilities related to +synchronization primitives, and "capability" is already overloaded in the +kernel, the naming chosen for the kernel departs from Clang's initial "Thread +Safety" and "capability" nomenclature; we refer to the feature as "Context +Analysis" to avoid confusion. The internal implementation still makes +references to Clang's terminology in a few places, such as `-Wthread-safety` +being the warning option that also still appears in diagnostic messages. diff --git a/Documentation/dev-tools/index.rst b/Documentation/dev-tools/index.rst index 4b8425e348ab..d864b3da4cc7 100644 --- a/Documentation/dev-tools/index.rst +++ b/Documentation/dev-tools/index.rst @@ -21,6 +21,7 @@ Documentation/process/debugging/index.rst checkpatch clang-format coccinelle + context-analysis sparse kcov gcov From 25d3b21e1d41f7b58aeb62b97b05d86d43c91801 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:39:54 +0100 Subject: [PATCH 08/90] checkpatch: Warn about context_unsafe() without comment Warn about applications of context_unsafe() without a comment, to encourage documenting the reasoning behind why it was deemed safe. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-6-elver@google.com --- scripts/checkpatch.pl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index c0250244cf7a..c4fd8bdff528 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -6733,6 +6733,13 @@ sub process { } } +# check for context_unsafe without a comment. + if ($line =~ /\bcontext_unsafe\b/ && + !ctx_has_comment($first_line, $linenr)) { + WARN("CONTEXT_UNSAFE", + "context_unsafe without comment\n" . $herecurr); + } + # check of hardware specific defines if ($line =~ m@^.\s*\#\s*if.*\b(__i386__|__powerpc64__|__sun__|__s390x__)\b@ && $realfile !~ m@include/asm-@) { CHK("ARCH_DEFINES", From 3931d4b980398012b66c8ff203bfa2ab3df71a71 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:39:55 +0100 Subject: [PATCH 09/90] cleanup: Basic compatibility with context analysis Introduce basic compatibility with cleanup.h infrastructure. We need to allow the compiler to see the acquisition and release of the context lock at the start and end of a scope. However, the current "cleanup" helpers wrap the lock in a struct passed through separate helper functions, which hides the lock alias from the compiler (no inter-procedural analysis). While Clang supports scoped guards in C++, it's not possible to apply in C code: https://clang.llvm.org/docs/ThreadSafetyAnalysis.html#scoped-context However, together with recent improvements to Clang's alias analysis abilities, idioms such as this work correctly now: void spin_unlock_cleanup(spinlock_t **l) __releases(*l) { .. } ... { spinlock_t *lock_scope __cleanup(spin_unlock_cleanup) = &lock; spin_lock(&lock); // lock through &lock ... critical section ... } // unlock through lock_scope -[alias]-> &lock (no warnings) To generalize this pattern and make it work with existing lock guards, introduce DECLARE_LOCK_GUARD_1_ATTRS() and WITH_LOCK_GUARD_1_ATTRS(). These allow creating an explicit alias to the context lock instance that is "cleaned" up with a separate cleanup helper. This helper is a dummy function that does nothing at runtime, but has the release attributes to tell the compiler what happens at the end of the scope. Example usage: DECLARE_LOCK_GUARD_1_ATTRS(mutex, __acquires(_T), __releases(*(struct mutex **)_T)) #define class_mutex_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(mutex, _T) Note: To support the for-loop based scoped helpers, the auxiliary variable must be a pointer to the "class" type because it is defined in the same statement as the guard variable. However, we initialize it with the lock pointer (despite the type mismatch, the compiler's alias analysis still works as expected). The "_unlock" attribute receives a pointer to the auxiliary variable (a double pointer to the class type), and must be cast and dereferenced appropriately. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-7-elver@google.com --- include/linux/cleanup.h | 50 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/include/linux/cleanup.h b/include/linux/cleanup.h index 8d41b917c77d..ee6df68c2177 100644 --- a/include/linux/cleanup.h +++ b/include/linux/cleanup.h @@ -278,16 +278,21 @@ const volatile void * __must_check_fn(const volatile void *val) #define DEFINE_CLASS(_name, _type, _exit, _init, _init_args...) \ typedef _type class_##_name##_t; \ +typedef _type lock_##_name##_t; \ static __always_inline void class_##_name##_destructor(_type *p) \ + __no_context_analysis \ { _type _T = *p; _exit; } \ static __always_inline _type class_##_name##_constructor(_init_args) \ + __no_context_analysis \ { _type t = _init; return t; } #define EXTEND_CLASS(_name, ext, _init, _init_args...) \ +typedef lock_##_name##_t lock_##_name##ext##_t; \ typedef class_##_name##_t class_##_name##ext##_t; \ static __always_inline void class_##_name##ext##_destructor(class_##_name##_t *p) \ { class_##_name##_destructor(p); } \ static __always_inline class_##_name##_t class_##_name##ext##_constructor(_init_args) \ + __no_context_analysis \ { class_##_name##_t t = _init; return t; } #define CLASS(_name, var) \ @@ -474,12 +479,14 @@ _label: \ */ #define __DEFINE_UNLOCK_GUARD(_name, _type, _unlock, ...) \ +typedef _type lock_##_name##_t; \ typedef struct { \ _type *lock; \ __VA_ARGS__; \ } class_##_name##_t; \ \ static __always_inline void class_##_name##_destructor(class_##_name##_t *_T) \ + __no_context_analysis \ { \ if (!__GUARD_IS_ERR(_T->lock)) { _unlock; } \ } \ @@ -488,6 +495,7 @@ __DEFINE_GUARD_LOCK_PTR(_name, &_T->lock) #define __DEFINE_LOCK_GUARD_1(_name, _type, _lock) \ static __always_inline class_##_name##_t class_##_name##_constructor(_type *l) \ + __no_context_analysis \ { \ class_##_name##_t _t = { .lock = l }, *_T = &_t; \ _lock; \ @@ -496,6 +504,7 @@ static __always_inline class_##_name##_t class_##_name##_constructor(_type *l) \ #define __DEFINE_LOCK_GUARD_0(_name, _lock) \ static __always_inline class_##_name##_t class_##_name##_constructor(void) \ + __no_context_analysis \ { \ class_##_name##_t _t = { .lock = (void*)1 }, \ *_T __maybe_unused = &_t; \ @@ -503,6 +512,47 @@ static __always_inline class_##_name##_t class_##_name##_constructor(void) \ return _t; \ } +#define DECLARE_LOCK_GUARD_0_ATTRS(_name, _lock, _unlock) \ +static inline class_##_name##_t class_##_name##_constructor(void) _lock;\ +static inline void class_##_name##_destructor(class_##_name##_t *_T) _unlock; + +/* + * To support Context Analysis, we need to allow the compiler to see the + * acquisition and release of the context lock. However, the "cleanup" helpers + * wrap the lock in a struct passed through separate helper functions, which + * hides the lock alias from the compiler (no inter-procedural analysis). + * + * To make it work, we introduce an explicit alias to the context lock instance + * that is "cleaned" up with a separate cleanup helper. This helper is a dummy + * function that does nothing at runtime, but has the "_unlock" attribute to + * tell the compiler what happens at the end of the scope. + * + * To generalize the pattern, the WITH_LOCK_GUARD_1_ATTRS() macro should be used + * to redefine the constructor, which then also creates the alias variable with + * the right "cleanup" attribute, *after* DECLARE_LOCK_GUARD_1_ATTRS() has been + * used. + * + * Example usage: + * + * DECLARE_LOCK_GUARD_1_ATTRS(mutex, __acquires(_T), __releases(*(struct mutex **)_T)) + * #define class_mutex_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(mutex, _T) + * + * Note: To support the for-loop based scoped helpers, the auxiliary variable + * must be a pointer to the "class" type because it is defined in the same + * statement as the guard variable. However, we initialize it with the lock + * pointer (despite the type mismatch, the compiler's alias analysis still works + * as expected). The "_unlock" attribute receives a pointer to the auxiliary + * variable (a double pointer to the class type), and must be cast and + * dereferenced appropriately. + */ +#define DECLARE_LOCK_GUARD_1_ATTRS(_name, _lock, _unlock) \ +static inline class_##_name##_t class_##_name##_constructor(lock_##_name##_t *_T) _lock;\ +static __always_inline void __class_##_name##_cleanup_ctx(class_##_name##_t **_T) \ + __no_context_analysis _unlock { } +#define WITH_LOCK_GUARD_1_ATTRS(_name, _T) \ + class_##_name##_constructor(_T), \ + *__UNIQUE_ID(unlock) __cleanup(__class_##_name##_cleanup_ctx) = (void *)(unsigned long)(_T) + #define DEFINE_LOCK_GUARD_1(_name, _type, _lock, _unlock, ...) \ __DEFINE_CLASS_IS_CONDITIONAL(_name, false); \ __DEFINE_UNLOCK_GUARD(_name, _type, _unlock, __VA_ARGS__) \ From 7c451541743c6c2ef1afc425191f18a23e311019 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:39:56 +0100 Subject: [PATCH 10/90] lockdep: Annotate lockdep assertions for context analysis Clang's context analysis can be made aware of functions that assert that locks are held. Presence of these annotations causes the analysis to assume the context lock is held after calls to the annotated function, and avoid false positives with complex control-flow; for example, where not all control-flow paths in a function require a held lock, and therefore marking the function with __must_hold(..) is inappropriate. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-8-elver@google.com --- include/linux/lockdep.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/include/linux/lockdep.h b/include/linux/lockdep.h index dd634103b014..621566345406 100644 --- a/include/linux/lockdep.h +++ b/include/linux/lockdep.h @@ -282,16 +282,16 @@ extern void lock_unpin_lock(struct lockdep_map *lock, struct pin_cookie); do { WARN_ON_ONCE(debug_locks && !(cond)); } while (0) #define lockdep_assert_held(l) \ - lockdep_assert(lockdep_is_held(l) != LOCK_STATE_NOT_HELD) + do { lockdep_assert(lockdep_is_held(l) != LOCK_STATE_NOT_HELD); __assume_ctx_lock(l); } while (0) #define lockdep_assert_not_held(l) \ lockdep_assert(lockdep_is_held(l) != LOCK_STATE_HELD) #define lockdep_assert_held_write(l) \ - lockdep_assert(lockdep_is_held_type(l, 0)) + do { lockdep_assert(lockdep_is_held_type(l, 0)); __assume_ctx_lock(l); } while (0) #define lockdep_assert_held_read(l) \ - lockdep_assert(lockdep_is_held_type(l, 1)) + do { lockdep_assert(lockdep_is_held_type(l, 1)); __assume_shared_ctx_lock(l); } while (0) #define lockdep_assert_held_once(l) \ lockdep_assert_once(lockdep_is_held(l) != LOCK_STATE_NOT_HELD) @@ -389,10 +389,10 @@ extern int lockdep_is_held(const void *); #define lockdep_assert(c) do { } while (0) #define lockdep_assert_once(c) do { } while (0) -#define lockdep_assert_held(l) do { (void)(l); } while (0) +#define lockdep_assert_held(l) __assume_ctx_lock(l) #define lockdep_assert_not_held(l) do { (void)(l); } while (0) -#define lockdep_assert_held_write(l) do { (void)(l); } while (0) -#define lockdep_assert_held_read(l) do { (void)(l); } while (0) +#define lockdep_assert_held_write(l) __assume_ctx_lock(l) +#define lockdep_assert_held_read(l) __assume_shared_ctx_lock(l) #define lockdep_assert_held_once(l) do { (void)(l); } while (0) #define lockdep_assert_none_held_once() do { } while (0) From f16a802d402d735a55731f8c94952b3bbb5ddfe8 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:39:57 +0100 Subject: [PATCH 11/90] locking/rwlock, spinlock: Support Clang's context analysis Add support for Clang's context analysis for raw_spinlock_t, spinlock_t, and rwlock. This wholesale conversion is required because all three of them are interdependent. To avoid warnings in constructors, the initialization functions mark a lock as acquired when initialized before guarded variables. The test verifies that common patterns do not generate false positives. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-9-elver@google.com --- Documentation/dev-tools/context-analysis.rst | 3 +- include/linux/rwlock.h | 25 ++-- include/linux/rwlock_api_smp.h | 29 ++++- include/linux/rwlock_rt.h | 35 +++-- include/linux/rwlock_types.h | 10 +- include/linux/spinlock.h | 93 +++++++++++--- include/linux/spinlock_api_smp.h | 14 +- include/linux/spinlock_api_up.h | 71 +++++----- include/linux/spinlock_rt.h | 21 +-- include/linux/spinlock_types.h | 10 +- include/linux/spinlock_types_raw.h | 5 +- lib/test_context-analysis.c | 128 +++++++++++++++++++ 12 files changed, 347 insertions(+), 97 deletions(-) diff --git a/Documentation/dev-tools/context-analysis.rst b/Documentation/dev-tools/context-analysis.rst index 47eb547eb716..746a2d275fb2 100644 --- a/Documentation/dev-tools/context-analysis.rst +++ b/Documentation/dev-tools/context-analysis.rst @@ -78,7 +78,8 @@ More details are also documented `here Supported Kernel Primitives ~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. Currently the following synchronization primitives are supported: +Currently the following synchronization primitives are supported: +`raw_spinlock_t`, `spinlock_t`, `rwlock_t`. For context locks with an initialization function (e.g., `spin_lock_init()`), calling this function before initializing any guarded members or globals diff --git a/include/linux/rwlock.h b/include/linux/rwlock.h index 5b87c6f4a243..151f9d5f3288 100644 --- a/include/linux/rwlock.h +++ b/include/linux/rwlock.h @@ -22,23 +22,24 @@ do { \ static struct lock_class_key __key; \ \ __rwlock_init((lock), #lock, &__key); \ + __assume_ctx_lock(lock); \ } while (0) #else # define rwlock_init(lock) \ - do { *(lock) = __RW_LOCK_UNLOCKED(lock); } while (0) + do { *(lock) = __RW_LOCK_UNLOCKED(lock); __assume_ctx_lock(lock); } while (0) #endif #ifdef CONFIG_DEBUG_SPINLOCK - extern void do_raw_read_lock(rwlock_t *lock) __acquires(lock); + extern void do_raw_read_lock(rwlock_t *lock) __acquires_shared(lock); extern int do_raw_read_trylock(rwlock_t *lock); - extern void do_raw_read_unlock(rwlock_t *lock) __releases(lock); + extern void do_raw_read_unlock(rwlock_t *lock) __releases_shared(lock); extern void do_raw_write_lock(rwlock_t *lock) __acquires(lock); extern int do_raw_write_trylock(rwlock_t *lock); extern void do_raw_write_unlock(rwlock_t *lock) __releases(lock); #else -# define do_raw_read_lock(rwlock) do {__acquire(lock); arch_read_lock(&(rwlock)->raw_lock); } while (0) +# define do_raw_read_lock(rwlock) do {__acquire_shared(lock); arch_read_lock(&(rwlock)->raw_lock); } while (0) # define do_raw_read_trylock(rwlock) arch_read_trylock(&(rwlock)->raw_lock) -# define do_raw_read_unlock(rwlock) do {arch_read_unlock(&(rwlock)->raw_lock); __release(lock); } while (0) +# define do_raw_read_unlock(rwlock) do {arch_read_unlock(&(rwlock)->raw_lock); __release_shared(lock); } while (0) # define do_raw_write_lock(rwlock) do {__acquire(lock); arch_write_lock(&(rwlock)->raw_lock); } while (0) # define do_raw_write_trylock(rwlock) arch_write_trylock(&(rwlock)->raw_lock) # define do_raw_write_unlock(rwlock) do {arch_write_unlock(&(rwlock)->raw_lock); __release(lock); } while (0) @@ -49,7 +50,7 @@ do { \ * regardless of whether CONFIG_SMP or CONFIG_PREEMPT are set. The various * methods are defined as nops in the case they are not required. */ -#define read_trylock(lock) __cond_lock(lock, _raw_read_trylock(lock)) +#define read_trylock(lock) __cond_lock_shared(lock, _raw_read_trylock(lock)) #define write_trylock(lock) __cond_lock(lock, _raw_write_trylock(lock)) #define write_lock(lock) _raw_write_lock(lock) @@ -112,12 +113,12 @@ do { \ } while (0) #define write_unlock_bh(lock) _raw_write_unlock_bh(lock) -#define write_trylock_irqsave(lock, flags) \ -({ \ - local_irq_save(flags); \ - write_trylock(lock) ? \ - 1 : ({ local_irq_restore(flags); 0; }); \ -}) +#define write_trylock_irqsave(lock, flags) \ + __cond_lock(lock, ({ \ + local_irq_save(flags); \ + _raw_write_trylock(lock) ? \ + 1 : ({ local_irq_restore(flags); 0; }); \ + })) #ifdef arch_rwlock_is_contended #define rwlock_is_contended(lock) \ diff --git a/include/linux/rwlock_api_smp.h b/include/linux/rwlock_api_smp.h index 31d3d1116323..6d5cc0b7be1f 100644 --- a/include/linux/rwlock_api_smp.h +++ b/include/linux/rwlock_api_smp.h @@ -15,12 +15,12 @@ * Released under the General Public License (GPL). */ -void __lockfunc _raw_read_lock(rwlock_t *lock) __acquires(lock); +void __lockfunc _raw_read_lock(rwlock_t *lock) __acquires_shared(lock); void __lockfunc _raw_write_lock(rwlock_t *lock) __acquires(lock); void __lockfunc _raw_write_lock_nested(rwlock_t *lock, int subclass) __acquires(lock); -void __lockfunc _raw_read_lock_bh(rwlock_t *lock) __acquires(lock); +void __lockfunc _raw_read_lock_bh(rwlock_t *lock) __acquires_shared(lock); void __lockfunc _raw_write_lock_bh(rwlock_t *lock) __acquires(lock); -void __lockfunc _raw_read_lock_irq(rwlock_t *lock) __acquires(lock); +void __lockfunc _raw_read_lock_irq(rwlock_t *lock) __acquires_shared(lock); void __lockfunc _raw_write_lock_irq(rwlock_t *lock) __acquires(lock); unsigned long __lockfunc _raw_read_lock_irqsave(rwlock_t *lock) __acquires(lock); @@ -28,11 +28,11 @@ unsigned long __lockfunc _raw_write_lock_irqsave(rwlock_t *lock) __acquires(lock); int __lockfunc _raw_read_trylock(rwlock_t *lock); int __lockfunc _raw_write_trylock(rwlock_t *lock); -void __lockfunc _raw_read_unlock(rwlock_t *lock) __releases(lock); +void __lockfunc _raw_read_unlock(rwlock_t *lock) __releases_shared(lock); void __lockfunc _raw_write_unlock(rwlock_t *lock) __releases(lock); -void __lockfunc _raw_read_unlock_bh(rwlock_t *lock) __releases(lock); +void __lockfunc _raw_read_unlock_bh(rwlock_t *lock) __releases_shared(lock); void __lockfunc _raw_write_unlock_bh(rwlock_t *lock) __releases(lock); -void __lockfunc _raw_read_unlock_irq(rwlock_t *lock) __releases(lock); +void __lockfunc _raw_read_unlock_irq(rwlock_t *lock) __releases_shared(lock); void __lockfunc _raw_write_unlock_irq(rwlock_t *lock) __releases(lock); void __lockfunc _raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) @@ -145,6 +145,7 @@ static inline int __raw_write_trylock(rwlock_t *lock) #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC) static inline void __raw_read_lock(rwlock_t *lock) + __acquires_shared(lock) __no_context_analysis { preempt_disable(); rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_); @@ -152,6 +153,7 @@ static inline void __raw_read_lock(rwlock_t *lock) } static inline unsigned long __raw_read_lock_irqsave(rwlock_t *lock) + __acquires_shared(lock) __no_context_analysis { unsigned long flags; @@ -163,6 +165,7 @@ static inline unsigned long __raw_read_lock_irqsave(rwlock_t *lock) } static inline void __raw_read_lock_irq(rwlock_t *lock) + __acquires_shared(lock) __no_context_analysis { local_irq_disable(); preempt_disable(); @@ -171,6 +174,7 @@ static inline void __raw_read_lock_irq(rwlock_t *lock) } static inline void __raw_read_lock_bh(rwlock_t *lock) + __acquires_shared(lock) __no_context_analysis { __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET); rwlock_acquire_read(&lock->dep_map, 0, 0, _RET_IP_); @@ -178,6 +182,7 @@ static inline void __raw_read_lock_bh(rwlock_t *lock) } static inline unsigned long __raw_write_lock_irqsave(rwlock_t *lock) + __acquires(lock) __no_context_analysis { unsigned long flags; @@ -189,6 +194,7 @@ static inline unsigned long __raw_write_lock_irqsave(rwlock_t *lock) } static inline void __raw_write_lock_irq(rwlock_t *lock) + __acquires(lock) __no_context_analysis { local_irq_disable(); preempt_disable(); @@ -197,6 +203,7 @@ static inline void __raw_write_lock_irq(rwlock_t *lock) } static inline void __raw_write_lock_bh(rwlock_t *lock) + __acquires(lock) __no_context_analysis { __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET); rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_); @@ -204,6 +211,7 @@ static inline void __raw_write_lock_bh(rwlock_t *lock) } static inline void __raw_write_lock(rwlock_t *lock) + __acquires(lock) __no_context_analysis { preempt_disable(); rwlock_acquire(&lock->dep_map, 0, 0, _RET_IP_); @@ -211,6 +219,7 @@ static inline void __raw_write_lock(rwlock_t *lock) } static inline void __raw_write_lock_nested(rwlock_t *lock, int subclass) + __acquires(lock) __no_context_analysis { preempt_disable(); rwlock_acquire(&lock->dep_map, subclass, 0, _RET_IP_); @@ -220,6 +229,7 @@ static inline void __raw_write_lock_nested(rwlock_t *lock, int subclass) #endif /* !CONFIG_GENERIC_LOCKBREAK || CONFIG_DEBUG_LOCK_ALLOC */ static inline void __raw_write_unlock(rwlock_t *lock) + __releases(lock) { rwlock_release(&lock->dep_map, _RET_IP_); do_raw_write_unlock(lock); @@ -227,6 +237,7 @@ static inline void __raw_write_unlock(rwlock_t *lock) } static inline void __raw_read_unlock(rwlock_t *lock) + __releases_shared(lock) { rwlock_release(&lock->dep_map, _RET_IP_); do_raw_read_unlock(lock); @@ -235,6 +246,7 @@ static inline void __raw_read_unlock(rwlock_t *lock) static inline void __raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) + __releases_shared(lock) { rwlock_release(&lock->dep_map, _RET_IP_); do_raw_read_unlock(lock); @@ -243,6 +255,7 @@ __raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) } static inline void __raw_read_unlock_irq(rwlock_t *lock) + __releases_shared(lock) { rwlock_release(&lock->dep_map, _RET_IP_); do_raw_read_unlock(lock); @@ -251,6 +264,7 @@ static inline void __raw_read_unlock_irq(rwlock_t *lock) } static inline void __raw_read_unlock_bh(rwlock_t *lock) + __releases_shared(lock) { rwlock_release(&lock->dep_map, _RET_IP_); do_raw_read_unlock(lock); @@ -259,6 +273,7 @@ static inline void __raw_read_unlock_bh(rwlock_t *lock) static inline void __raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) + __releases(lock) { rwlock_release(&lock->dep_map, _RET_IP_); do_raw_write_unlock(lock); @@ -267,6 +282,7 @@ static inline void __raw_write_unlock_irqrestore(rwlock_t *lock, } static inline void __raw_write_unlock_irq(rwlock_t *lock) + __releases(lock) { rwlock_release(&lock->dep_map, _RET_IP_); do_raw_write_unlock(lock); @@ -275,6 +291,7 @@ static inline void __raw_write_unlock_irq(rwlock_t *lock) } static inline void __raw_write_unlock_bh(rwlock_t *lock) + __releases(lock) { rwlock_release(&lock->dep_map, _RET_IP_); do_raw_write_unlock(lock); diff --git a/include/linux/rwlock_rt.h b/include/linux/rwlock_rt.h index 7d81fc6918ee..f64d6d319a47 100644 --- a/include/linux/rwlock_rt.h +++ b/include/linux/rwlock_rt.h @@ -22,28 +22,32 @@ do { \ \ init_rwbase_rt(&(rwl)->rwbase); \ __rt_rwlock_init(rwl, #rwl, &__key); \ + __assume_ctx_lock(rwl); \ } while (0) -extern void rt_read_lock(rwlock_t *rwlock) __acquires(rwlock); +extern void rt_read_lock(rwlock_t *rwlock) __acquires_shared(rwlock); extern int rt_read_trylock(rwlock_t *rwlock); -extern void rt_read_unlock(rwlock_t *rwlock) __releases(rwlock); +extern void rt_read_unlock(rwlock_t *rwlock) __releases_shared(rwlock); extern void rt_write_lock(rwlock_t *rwlock) __acquires(rwlock); extern void rt_write_lock_nested(rwlock_t *rwlock, int subclass) __acquires(rwlock); extern int rt_write_trylock(rwlock_t *rwlock); extern void rt_write_unlock(rwlock_t *rwlock) __releases(rwlock); static __always_inline void read_lock(rwlock_t *rwlock) + __acquires_shared(rwlock) { rt_read_lock(rwlock); } static __always_inline void read_lock_bh(rwlock_t *rwlock) + __acquires_shared(rwlock) { local_bh_disable(); rt_read_lock(rwlock); } static __always_inline void read_lock_irq(rwlock_t *rwlock) + __acquires_shared(rwlock) { rt_read_lock(rwlock); } @@ -55,37 +59,43 @@ static __always_inline void read_lock_irq(rwlock_t *rwlock) flags = 0; \ } while (0) -#define read_trylock(lock) __cond_lock(lock, rt_read_trylock(lock)) +#define read_trylock(lock) __cond_lock_shared(lock, rt_read_trylock(lock)) static __always_inline void read_unlock(rwlock_t *rwlock) + __releases_shared(rwlock) { rt_read_unlock(rwlock); } static __always_inline void read_unlock_bh(rwlock_t *rwlock) + __releases_shared(rwlock) { rt_read_unlock(rwlock); local_bh_enable(); } static __always_inline void read_unlock_irq(rwlock_t *rwlock) + __releases_shared(rwlock) { rt_read_unlock(rwlock); } static __always_inline void read_unlock_irqrestore(rwlock_t *rwlock, unsigned long flags) + __releases_shared(rwlock) { rt_read_unlock(rwlock); } static __always_inline void write_lock(rwlock_t *rwlock) + __acquires(rwlock) { rt_write_lock(rwlock); } #ifdef CONFIG_DEBUG_LOCK_ALLOC static __always_inline void write_lock_nested(rwlock_t *rwlock, int subclass) + __acquires(rwlock) { rt_write_lock_nested(rwlock, subclass); } @@ -94,12 +104,14 @@ static __always_inline void write_lock_nested(rwlock_t *rwlock, int subclass) #endif static __always_inline void write_lock_bh(rwlock_t *rwlock) + __acquires(rwlock) { local_bh_disable(); rt_write_lock(rwlock); } static __always_inline void write_lock_irq(rwlock_t *rwlock) + __acquires(rwlock) { rt_write_lock(rwlock); } @@ -114,33 +126,34 @@ static __always_inline void write_lock_irq(rwlock_t *rwlock) #define write_trylock(lock) __cond_lock(lock, rt_write_trylock(lock)) #define write_trylock_irqsave(lock, flags) \ -({ \ - int __locked; \ - \ - typecheck(unsigned long, flags); \ - flags = 0; \ - __locked = write_trylock(lock); \ - __locked; \ -}) + __cond_lock(lock, ({ \ + typecheck(unsigned long, flags); \ + flags = 0; \ + rt_write_trylock(lock); \ + })) static __always_inline void write_unlock(rwlock_t *rwlock) + __releases(rwlock) { rt_write_unlock(rwlock); } static __always_inline void write_unlock_bh(rwlock_t *rwlock) + __releases(rwlock) { rt_write_unlock(rwlock); local_bh_enable(); } static __always_inline void write_unlock_irq(rwlock_t *rwlock) + __releases(rwlock) { rt_write_unlock(rwlock); } static __always_inline void write_unlock_irqrestore(rwlock_t *rwlock, unsigned long flags) + __releases(rwlock) { rt_write_unlock(rwlock); } diff --git a/include/linux/rwlock_types.h b/include/linux/rwlock_types.h index 1948442e7750..d5e7316401e7 100644 --- a/include/linux/rwlock_types.h +++ b/include/linux/rwlock_types.h @@ -22,7 +22,7 @@ * portions Copyright 2005, Red Hat, Inc., Ingo Molnar * Released under the General Public License (GPL). */ -typedef struct { +context_lock_struct(rwlock) { arch_rwlock_t raw_lock; #ifdef CONFIG_DEBUG_SPINLOCK unsigned int magic, owner_cpu; @@ -31,7 +31,8 @@ typedef struct { #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map dep_map; #endif -} rwlock_t; +}; +typedef struct rwlock rwlock_t; #define RWLOCK_MAGIC 0xdeaf1eed @@ -54,13 +55,14 @@ typedef struct { #include -typedef struct { +context_lock_struct(rwlock) { struct rwbase_rt rwbase; atomic_t readers; #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map dep_map; #endif -} rwlock_t; +}; +typedef struct rwlock rwlock_t; #define __RWLOCK_RT_INITIALIZER(name) \ { \ diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index d3561c4a080e..72aabdd4fa3f 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -106,11 +106,12 @@ do { \ static struct lock_class_key __key; \ \ __raw_spin_lock_init((lock), #lock, &__key, LD_WAIT_SPIN); \ + __assume_ctx_lock(lock); \ } while (0) #else # define raw_spin_lock_init(lock) \ - do { *(lock) = __RAW_SPIN_LOCK_UNLOCKED(lock); } while (0) + do { *(lock) = __RAW_SPIN_LOCK_UNLOCKED(lock); __assume_ctx_lock(lock); } while (0) #endif #define raw_spin_is_locked(lock) arch_spin_is_locked(&(lock)->raw_lock) @@ -286,19 +287,19 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock) #define raw_spin_trylock_bh(lock) \ __cond_lock(lock, _raw_spin_trylock_bh(lock)) -#define raw_spin_trylock_irq(lock) \ -({ \ - local_irq_disable(); \ - raw_spin_trylock(lock) ? \ - 1 : ({ local_irq_enable(); 0; }); \ -}) +#define raw_spin_trylock_irq(lock) \ + __cond_lock(lock, ({ \ + local_irq_disable(); \ + _raw_spin_trylock(lock) ? \ + 1 : ({ local_irq_enable(); 0; }); \ + })) -#define raw_spin_trylock_irqsave(lock, flags) \ -({ \ - local_irq_save(flags); \ - raw_spin_trylock(lock) ? \ - 1 : ({ local_irq_restore(flags); 0; }); \ -}) +#define raw_spin_trylock_irqsave(lock, flags) \ + __cond_lock(lock, ({ \ + local_irq_save(flags); \ + _raw_spin_trylock(lock) ? \ + 1 : ({ local_irq_restore(flags); 0; }); \ + })) #ifndef CONFIG_PREEMPT_RT /* Include rwlock functions for !RT */ @@ -334,6 +335,7 @@ do { \ \ __raw_spin_lock_init(spinlock_check(lock), \ #lock, &__key, LD_WAIT_CONFIG); \ + __assume_ctx_lock(lock); \ } while (0) #else @@ -342,21 +344,25 @@ do { \ do { \ spinlock_check(_lock); \ *(_lock) = __SPIN_LOCK_UNLOCKED(_lock); \ + __assume_ctx_lock(_lock); \ } while (0) #endif static __always_inline void spin_lock(spinlock_t *lock) + __acquires(lock) __no_context_analysis { raw_spin_lock(&lock->rlock); } static __always_inline void spin_lock_bh(spinlock_t *lock) + __acquires(lock) __no_context_analysis { raw_spin_lock_bh(&lock->rlock); } static __always_inline int spin_trylock(spinlock_t *lock) + __cond_acquires(lock) __no_context_analysis { return raw_spin_trylock(&lock->rlock); } @@ -364,14 +370,17 @@ static __always_inline int spin_trylock(spinlock_t *lock) #define spin_lock_nested(lock, subclass) \ do { \ raw_spin_lock_nested(spinlock_check(lock), subclass); \ + __release(spinlock_check(lock)); __acquire(lock); \ } while (0) #define spin_lock_nest_lock(lock, nest_lock) \ do { \ raw_spin_lock_nest_lock(spinlock_check(lock), nest_lock); \ + __release(spinlock_check(lock)); __acquire(lock); \ } while (0) static __always_inline void spin_lock_irq(spinlock_t *lock) + __acquires(lock) __no_context_analysis { raw_spin_lock_irq(&lock->rlock); } @@ -379,47 +388,53 @@ static __always_inline void spin_lock_irq(spinlock_t *lock) #define spin_lock_irqsave(lock, flags) \ do { \ raw_spin_lock_irqsave(spinlock_check(lock), flags); \ + __release(spinlock_check(lock)); __acquire(lock); \ } while (0) #define spin_lock_irqsave_nested(lock, flags, subclass) \ do { \ raw_spin_lock_irqsave_nested(spinlock_check(lock), flags, subclass); \ + __release(spinlock_check(lock)); __acquire(lock); \ } while (0) static __always_inline void spin_unlock(spinlock_t *lock) + __releases(lock) __no_context_analysis { raw_spin_unlock(&lock->rlock); } static __always_inline void spin_unlock_bh(spinlock_t *lock) + __releases(lock) __no_context_analysis { raw_spin_unlock_bh(&lock->rlock); } static __always_inline void spin_unlock_irq(spinlock_t *lock) + __releases(lock) __no_context_analysis { raw_spin_unlock_irq(&lock->rlock); } static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) + __releases(lock) __no_context_analysis { raw_spin_unlock_irqrestore(&lock->rlock, flags); } static __always_inline int spin_trylock_bh(spinlock_t *lock) + __cond_acquires(lock) __no_context_analysis { return raw_spin_trylock_bh(&lock->rlock); } static __always_inline int spin_trylock_irq(spinlock_t *lock) + __cond_acquires(lock) __no_context_analysis { return raw_spin_trylock_irq(&lock->rlock); } #define spin_trylock_irqsave(lock, flags) \ -({ \ - raw_spin_trylock_irqsave(spinlock_check(lock), flags); \ -}) + __cond_lock(lock, raw_spin_trylock_irqsave(spinlock_check(lock), flags)) /** * spin_is_locked() - Check whether a spinlock is locked. @@ -535,86 +550,132 @@ void free_bucket_spinlocks(spinlock_t *locks); DEFINE_LOCK_GUARD_1(raw_spinlock, raw_spinlock_t, raw_spin_lock(_T->lock), raw_spin_unlock(_T->lock)) +DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock, __acquires(_T), __releases(*(raw_spinlock_t **)_T)) +#define class_raw_spinlock_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock, _T) DEFINE_LOCK_GUARD_1_COND(raw_spinlock, _try, raw_spin_trylock(_T->lock)) +DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_try, __acquires(_T), __releases(*(raw_spinlock_t **)_T)) +#define class_raw_spinlock_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_try, _T) DEFINE_LOCK_GUARD_1(raw_spinlock_nested, raw_spinlock_t, raw_spin_lock_nested(_T->lock, SINGLE_DEPTH_NESTING), raw_spin_unlock(_T->lock)) +DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_nested, __acquires(_T), __releases(*(raw_spinlock_t **)_T)) +#define class_raw_spinlock_nested_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_nested, _T) DEFINE_LOCK_GUARD_1(raw_spinlock_irq, raw_spinlock_t, raw_spin_lock_irq(_T->lock), raw_spin_unlock_irq(_T->lock)) +DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irq, __acquires(_T), __releases(*(raw_spinlock_t **)_T)) +#define class_raw_spinlock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irq, _T) DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irq, _try, raw_spin_trylock_irq(_T->lock)) +DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irq_try, __acquires(_T), __releases(*(raw_spinlock_t **)_T)) +#define class_raw_spinlock_irq_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irq_try, _T) DEFINE_LOCK_GUARD_1(raw_spinlock_bh, raw_spinlock_t, raw_spin_lock_bh(_T->lock), raw_spin_unlock_bh(_T->lock)) +DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_bh, __acquires(_T), __releases(*(raw_spinlock_t **)_T)) +#define class_raw_spinlock_bh_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_bh, _T) DEFINE_LOCK_GUARD_1_COND(raw_spinlock_bh, _try, raw_spin_trylock_bh(_T->lock)) +DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_bh_try, __acquires(_T), __releases(*(raw_spinlock_t **)_T)) +#define class_raw_spinlock_bh_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_bh_try, _T) DEFINE_LOCK_GUARD_1(raw_spinlock_irqsave, raw_spinlock_t, raw_spin_lock_irqsave(_T->lock, _T->flags), raw_spin_unlock_irqrestore(_T->lock, _T->flags), unsigned long flags) +DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave, __acquires(_T), __releases(*(raw_spinlock_t **)_T)) +#define class_raw_spinlock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave, _T) DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irqsave, _try, raw_spin_trylock_irqsave(_T->lock, _T->flags)) +DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave_try, __acquires(_T), __releases(*(raw_spinlock_t **)_T)) +#define class_raw_spinlock_irqsave_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave_try, _T) DEFINE_LOCK_GUARD_1(spinlock, spinlock_t, spin_lock(_T->lock), spin_unlock(_T->lock)) +DECLARE_LOCK_GUARD_1_ATTRS(spinlock, __acquires(_T), __releases(*(spinlock_t **)_T)) +#define class_spinlock_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock, _T) DEFINE_LOCK_GUARD_1_COND(spinlock, _try, spin_trylock(_T->lock)) +DECLARE_LOCK_GUARD_1_ATTRS(spinlock_try, __acquires(_T), __releases(*(spinlock_t **)_T)) +#define class_spinlock_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_try, _T) DEFINE_LOCK_GUARD_1(spinlock_irq, spinlock_t, spin_lock_irq(_T->lock), spin_unlock_irq(_T->lock)) +DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irq, __acquires(_T), __releases(*(spinlock_t **)_T)) +#define class_spinlock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irq, _T) DEFINE_LOCK_GUARD_1_COND(spinlock_irq, _try, spin_trylock_irq(_T->lock)) +DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irq_try, __acquires(_T), __releases(*(spinlock_t **)_T)) +#define class_spinlock_irq_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irq_try, _T) DEFINE_LOCK_GUARD_1(spinlock_bh, spinlock_t, spin_lock_bh(_T->lock), spin_unlock_bh(_T->lock)) +DECLARE_LOCK_GUARD_1_ATTRS(spinlock_bh, __acquires(_T), __releases(*(spinlock_t **)_T)) +#define class_spinlock_bh_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_bh, _T) DEFINE_LOCK_GUARD_1_COND(spinlock_bh, _try, spin_trylock_bh(_T->lock)) +DECLARE_LOCK_GUARD_1_ATTRS(spinlock_bh_try, __acquires(_T), __releases(*(spinlock_t **)_T)) +#define class_spinlock_bh_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_bh_try, _T) DEFINE_LOCK_GUARD_1(spinlock_irqsave, spinlock_t, spin_lock_irqsave(_T->lock, _T->flags), spin_unlock_irqrestore(_T->lock, _T->flags), unsigned long flags) +DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irqsave, __acquires(_T), __releases(*(spinlock_t **)_T)) +#define class_spinlock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irqsave, _T) DEFINE_LOCK_GUARD_1_COND(spinlock_irqsave, _try, spin_trylock_irqsave(_T->lock, _T->flags)) +DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irqsave_try, __acquires(_T), __releases(*(spinlock_t **)_T)) +#define class_spinlock_irqsave_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irqsave_try, _T) DEFINE_LOCK_GUARD_1(read_lock, rwlock_t, read_lock(_T->lock), read_unlock(_T->lock)) +DECLARE_LOCK_GUARD_1_ATTRS(read_lock, __acquires(_T), __releases(*(rwlock_t **)_T)) +#define class_read_lock_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(read_lock, _T) DEFINE_LOCK_GUARD_1(read_lock_irq, rwlock_t, read_lock_irq(_T->lock), read_unlock_irq(_T->lock)) +DECLARE_LOCK_GUARD_1_ATTRS(read_lock_irq, __acquires(_T), __releases(*(rwlock_t **)_T)) +#define class_read_lock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(read_lock_irq, _T) DEFINE_LOCK_GUARD_1(read_lock_irqsave, rwlock_t, read_lock_irqsave(_T->lock, _T->flags), read_unlock_irqrestore(_T->lock, _T->flags), unsigned long flags) +DECLARE_LOCK_GUARD_1_ATTRS(read_lock_irqsave, __acquires(_T), __releases(*(rwlock_t **)_T)) +#define class_read_lock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(read_lock_irqsave, _T) DEFINE_LOCK_GUARD_1(write_lock, rwlock_t, write_lock(_T->lock), write_unlock(_T->lock)) +DECLARE_LOCK_GUARD_1_ATTRS(write_lock, __acquires(_T), __releases(*(rwlock_t **)_T)) +#define class_write_lock_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(write_lock, _T) DEFINE_LOCK_GUARD_1(write_lock_irq, rwlock_t, write_lock_irq(_T->lock), write_unlock_irq(_T->lock)) +DECLARE_LOCK_GUARD_1_ATTRS(write_lock_irq, __acquires(_T), __releases(*(rwlock_t **)_T)) +#define class_write_lock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(write_lock_irq, _T) DEFINE_LOCK_GUARD_1(write_lock_irqsave, rwlock_t, write_lock_irqsave(_T->lock, _T->flags), write_unlock_irqrestore(_T->lock, _T->flags), unsigned long flags) +DECLARE_LOCK_GUARD_1_ATTRS(write_lock_irqsave, __acquires(_T), __releases(*(rwlock_t **)_T)) +#define class_write_lock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(write_lock_irqsave, _T) #undef __LINUX_INSIDE_SPINLOCK_H #endif /* __LINUX_SPINLOCK_H */ diff --git a/include/linux/spinlock_api_smp.h b/include/linux/spinlock_api_smp.h index 9ecb0ab504e3..d19327e04df9 100644 --- a/include/linux/spinlock_api_smp.h +++ b/include/linux/spinlock_api_smp.h @@ -34,8 +34,8 @@ unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock) unsigned long __lockfunc _raw_spin_lock_irqsave_nested(raw_spinlock_t *lock, int subclass) __acquires(lock); -int __lockfunc _raw_spin_trylock(raw_spinlock_t *lock); -int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock); +int __lockfunc _raw_spin_trylock(raw_spinlock_t *lock) __cond_acquires(lock); +int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock) __cond_acquires(lock); void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock) __releases(lock); void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) __releases(lock); void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) __releases(lock); @@ -84,6 +84,7 @@ _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) #endif static inline int __raw_spin_trylock(raw_spinlock_t *lock) + __cond_acquires(lock) { preempt_disable(); if (do_raw_spin_trylock(lock)) { @@ -102,6 +103,7 @@ static inline int __raw_spin_trylock(raw_spinlock_t *lock) #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC) static inline unsigned long __raw_spin_lock_irqsave(raw_spinlock_t *lock) + __acquires(lock) __no_context_analysis { unsigned long flags; @@ -113,6 +115,7 @@ static inline unsigned long __raw_spin_lock_irqsave(raw_spinlock_t *lock) } static inline void __raw_spin_lock_irq(raw_spinlock_t *lock) + __acquires(lock) __no_context_analysis { local_irq_disable(); preempt_disable(); @@ -121,6 +124,7 @@ static inline void __raw_spin_lock_irq(raw_spinlock_t *lock) } static inline void __raw_spin_lock_bh(raw_spinlock_t *lock) + __acquires(lock) __no_context_analysis { __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET); spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); @@ -128,6 +132,7 @@ static inline void __raw_spin_lock_bh(raw_spinlock_t *lock) } static inline void __raw_spin_lock(raw_spinlock_t *lock) + __acquires(lock) __no_context_analysis { preempt_disable(); spin_acquire(&lock->dep_map, 0, 0, _RET_IP_); @@ -137,6 +142,7 @@ static inline void __raw_spin_lock(raw_spinlock_t *lock) #endif /* !CONFIG_GENERIC_LOCKBREAK || CONFIG_DEBUG_LOCK_ALLOC */ static inline void __raw_spin_unlock(raw_spinlock_t *lock) + __releases(lock) { spin_release(&lock->dep_map, _RET_IP_); do_raw_spin_unlock(lock); @@ -145,6 +151,7 @@ static inline void __raw_spin_unlock(raw_spinlock_t *lock) static inline void __raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) + __releases(lock) { spin_release(&lock->dep_map, _RET_IP_); do_raw_spin_unlock(lock); @@ -153,6 +160,7 @@ static inline void __raw_spin_unlock_irqrestore(raw_spinlock_t *lock, } static inline void __raw_spin_unlock_irq(raw_spinlock_t *lock) + __releases(lock) { spin_release(&lock->dep_map, _RET_IP_); do_raw_spin_unlock(lock); @@ -161,6 +169,7 @@ static inline void __raw_spin_unlock_irq(raw_spinlock_t *lock) } static inline void __raw_spin_unlock_bh(raw_spinlock_t *lock) + __releases(lock) { spin_release(&lock->dep_map, _RET_IP_); do_raw_spin_unlock(lock); @@ -168,6 +177,7 @@ static inline void __raw_spin_unlock_bh(raw_spinlock_t *lock) } static inline int __raw_spin_trylock_bh(raw_spinlock_t *lock) + __cond_acquires(lock) { __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET); if (do_raw_spin_trylock(lock)) { diff --git a/include/linux/spinlock_api_up.h b/include/linux/spinlock_api_up.h index 819aeba1c87e..018f5aabc1be 100644 --- a/include/linux/spinlock_api_up.h +++ b/include/linux/spinlock_api_up.h @@ -24,68 +24,77 @@ * flags straight, to suppress compiler warnings of unused lock * variables, and to add the proper checker annotations: */ -#define ___LOCK(lock) \ - do { __acquire(lock); (void)(lock); } while (0) +#define ___LOCK_void(lock) \ + do { (void)(lock); } while (0) -#define __LOCK(lock) \ - do { preempt_disable(); ___LOCK(lock); } while (0) +#define ___LOCK_(lock) \ + do { __acquire(lock); ___LOCK_void(lock); } while (0) -#define __LOCK_BH(lock) \ - do { __local_bh_disable_ip(_THIS_IP_, SOFTIRQ_LOCK_OFFSET); ___LOCK(lock); } while (0) +#define ___LOCK_shared(lock) \ + do { __acquire_shared(lock); ___LOCK_void(lock); } while (0) -#define __LOCK_IRQ(lock) \ - do { local_irq_disable(); __LOCK(lock); } while (0) +#define __LOCK(lock, ...) \ + do { preempt_disable(); ___LOCK_##__VA_ARGS__(lock); } while (0) -#define __LOCK_IRQSAVE(lock, flags) \ - do { local_irq_save(flags); __LOCK(lock); } while (0) +#define __LOCK_BH(lock, ...) \ + do { __local_bh_disable_ip(_THIS_IP_, SOFTIRQ_LOCK_OFFSET); ___LOCK_##__VA_ARGS__(lock); } while (0) -#define ___UNLOCK(lock) \ +#define __LOCK_IRQ(lock, ...) \ + do { local_irq_disable(); __LOCK(lock, ##__VA_ARGS__); } while (0) + +#define __LOCK_IRQSAVE(lock, flags, ...) \ + do { local_irq_save(flags); __LOCK(lock, ##__VA_ARGS__); } while (0) + +#define ___UNLOCK_(lock) \ do { __release(lock); (void)(lock); } while (0) -#define __UNLOCK(lock) \ - do { preempt_enable(); ___UNLOCK(lock); } while (0) +#define ___UNLOCK_shared(lock) \ + do { __release_shared(lock); (void)(lock); } while (0) -#define __UNLOCK_BH(lock) \ +#define __UNLOCK(lock, ...) \ + do { preempt_enable(); ___UNLOCK_##__VA_ARGS__(lock); } while (0) + +#define __UNLOCK_BH(lock, ...) \ do { __local_bh_enable_ip(_THIS_IP_, SOFTIRQ_LOCK_OFFSET); \ - ___UNLOCK(lock); } while (0) + ___UNLOCK_##__VA_ARGS__(lock); } while (0) -#define __UNLOCK_IRQ(lock) \ - do { local_irq_enable(); __UNLOCK(lock); } while (0) +#define __UNLOCK_IRQ(lock, ...) \ + do { local_irq_enable(); __UNLOCK(lock, ##__VA_ARGS__); } while (0) -#define __UNLOCK_IRQRESTORE(lock, flags) \ - do { local_irq_restore(flags); __UNLOCK(lock); } while (0) +#define __UNLOCK_IRQRESTORE(lock, flags, ...) \ + do { local_irq_restore(flags); __UNLOCK(lock, ##__VA_ARGS__); } while (0) #define _raw_spin_lock(lock) __LOCK(lock) #define _raw_spin_lock_nested(lock, subclass) __LOCK(lock) -#define _raw_read_lock(lock) __LOCK(lock) +#define _raw_read_lock(lock) __LOCK(lock, shared) #define _raw_write_lock(lock) __LOCK(lock) #define _raw_write_lock_nested(lock, subclass) __LOCK(lock) #define _raw_spin_lock_bh(lock) __LOCK_BH(lock) -#define _raw_read_lock_bh(lock) __LOCK_BH(lock) +#define _raw_read_lock_bh(lock) __LOCK_BH(lock, shared) #define _raw_write_lock_bh(lock) __LOCK_BH(lock) #define _raw_spin_lock_irq(lock) __LOCK_IRQ(lock) -#define _raw_read_lock_irq(lock) __LOCK_IRQ(lock) +#define _raw_read_lock_irq(lock) __LOCK_IRQ(lock, shared) #define _raw_write_lock_irq(lock) __LOCK_IRQ(lock) #define _raw_spin_lock_irqsave(lock, flags) __LOCK_IRQSAVE(lock, flags) -#define _raw_read_lock_irqsave(lock, flags) __LOCK_IRQSAVE(lock, flags) +#define _raw_read_lock_irqsave(lock, flags) __LOCK_IRQSAVE(lock, flags, shared) #define _raw_write_lock_irqsave(lock, flags) __LOCK_IRQSAVE(lock, flags) -#define _raw_spin_trylock(lock) ({ __LOCK(lock); 1; }) -#define _raw_read_trylock(lock) ({ __LOCK(lock); 1; }) -#define _raw_write_trylock(lock) ({ __LOCK(lock); 1; }) -#define _raw_spin_trylock_bh(lock) ({ __LOCK_BH(lock); 1; }) +#define _raw_spin_trylock(lock) ({ __LOCK(lock, void); 1; }) +#define _raw_read_trylock(lock) ({ __LOCK(lock, void); 1; }) +#define _raw_write_trylock(lock) ({ __LOCK(lock, void); 1; }) +#define _raw_spin_trylock_bh(lock) ({ __LOCK_BH(lock, void); 1; }) #define _raw_spin_unlock(lock) __UNLOCK(lock) -#define _raw_read_unlock(lock) __UNLOCK(lock) +#define _raw_read_unlock(lock) __UNLOCK(lock, shared) #define _raw_write_unlock(lock) __UNLOCK(lock) #define _raw_spin_unlock_bh(lock) __UNLOCK_BH(lock) #define _raw_write_unlock_bh(lock) __UNLOCK_BH(lock) -#define _raw_read_unlock_bh(lock) __UNLOCK_BH(lock) +#define _raw_read_unlock_bh(lock) __UNLOCK_BH(lock, shared) #define _raw_spin_unlock_irq(lock) __UNLOCK_IRQ(lock) -#define _raw_read_unlock_irq(lock) __UNLOCK_IRQ(lock) +#define _raw_read_unlock_irq(lock) __UNLOCK_IRQ(lock, shared) #define _raw_write_unlock_irq(lock) __UNLOCK_IRQ(lock) #define _raw_spin_unlock_irqrestore(lock, flags) \ __UNLOCK_IRQRESTORE(lock, flags) #define _raw_read_unlock_irqrestore(lock, flags) \ - __UNLOCK_IRQRESTORE(lock, flags) + __UNLOCK_IRQRESTORE(lock, flags, shared) #define _raw_write_unlock_irqrestore(lock, flags) \ __UNLOCK_IRQRESTORE(lock, flags) diff --git a/include/linux/spinlock_rt.h b/include/linux/spinlock_rt.h index f6499c37157d..6bab73ee1384 100644 --- a/include/linux/spinlock_rt.h +++ b/include/linux/spinlock_rt.h @@ -20,6 +20,7 @@ static inline void __rt_spin_lock_init(spinlock_t *lock, const char *name, do { \ rt_mutex_base_init(&(slock)->lock); \ __rt_spin_lock_init(slock, name, key, percpu); \ + __assume_ctx_lock(slock); \ } while (0) #define _spin_lock_init(slock, percpu) \ @@ -40,6 +41,7 @@ extern int rt_spin_trylock_bh(spinlock_t *lock); extern int rt_spin_trylock(spinlock_t *lock); static __always_inline void spin_lock(spinlock_t *lock) + __acquires(lock) { rt_spin_lock(lock); } @@ -82,6 +84,7 @@ static __always_inline void spin_lock(spinlock_t *lock) __spin_lock_irqsave_nested(lock, flags, subclass) static __always_inline void spin_lock_bh(spinlock_t *lock) + __acquires(lock) { /* Investigate: Drop bh when blocking ? */ local_bh_disable(); @@ -89,6 +92,7 @@ static __always_inline void spin_lock_bh(spinlock_t *lock) } static __always_inline void spin_lock_irq(spinlock_t *lock) + __acquires(lock) { rt_spin_lock(lock); } @@ -101,23 +105,27 @@ static __always_inline void spin_lock_irq(spinlock_t *lock) } while (0) static __always_inline void spin_unlock(spinlock_t *lock) + __releases(lock) { rt_spin_unlock(lock); } static __always_inline void spin_unlock_bh(spinlock_t *lock) + __releases(lock) { rt_spin_unlock(lock); local_bh_enable(); } static __always_inline void spin_unlock_irq(spinlock_t *lock) + __releases(lock) { rt_spin_unlock(lock); } static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) + __releases(lock) { rt_spin_unlock(lock); } @@ -132,14 +140,11 @@ static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, __cond_lock(lock, rt_spin_trylock(lock)) #define spin_trylock_irqsave(lock, flags) \ -({ \ - int __locked; \ - \ - typecheck(unsigned long, flags); \ - flags = 0; \ - __locked = spin_trylock(lock); \ - __locked; \ -}) + __cond_lock(lock, ({ \ + typecheck(unsigned long, flags); \ + flags = 0; \ + rt_spin_trylock(lock); \ + })) #define spin_is_contended(lock) (((void)(lock), 0)) diff --git a/include/linux/spinlock_types.h b/include/linux/spinlock_types.h index 2dfa35ffec76..b65bb6e4451c 100644 --- a/include/linux/spinlock_types.h +++ b/include/linux/spinlock_types.h @@ -14,7 +14,7 @@ #ifndef CONFIG_PREEMPT_RT /* Non PREEMPT_RT kernels map spinlock to raw_spinlock */ -typedef struct spinlock { +context_lock_struct(spinlock) { union { struct raw_spinlock rlock; @@ -26,7 +26,8 @@ typedef struct spinlock { }; #endif }; -} spinlock_t; +}; +typedef struct spinlock spinlock_t; #define ___SPIN_LOCK_INITIALIZER(lockname) \ { \ @@ -47,12 +48,13 @@ typedef struct spinlock { /* PREEMPT_RT kernels map spinlock to rt_mutex */ #include -typedef struct spinlock { +context_lock_struct(spinlock) { struct rt_mutex_base lock; #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map dep_map; #endif -} spinlock_t; +}; +typedef struct spinlock spinlock_t; #define __SPIN_LOCK_UNLOCKED(name) \ { \ diff --git a/include/linux/spinlock_types_raw.h b/include/linux/spinlock_types_raw.h index 91cb36b65a17..e5644ab2161f 100644 --- a/include/linux/spinlock_types_raw.h +++ b/include/linux/spinlock_types_raw.h @@ -11,7 +11,7 @@ #include -typedef struct raw_spinlock { +context_lock_struct(raw_spinlock) { arch_spinlock_t raw_lock; #ifdef CONFIG_DEBUG_SPINLOCK unsigned int magic, owner_cpu; @@ -20,7 +20,8 @@ typedef struct raw_spinlock { #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map dep_map; #endif -} raw_spinlock_t; +}; +typedef struct raw_spinlock raw_spinlock_t; #define SPINLOCK_MAGIC 0xdead4ead diff --git a/lib/test_context-analysis.c b/lib/test_context-analysis.c index 68f075dec0e0..273fa9d34657 100644 --- a/lib/test_context-analysis.c +++ b/lib/test_context-analysis.c @@ -5,6 +5,7 @@ */ #include +#include /* * Test that helper macros work as expected. @@ -16,3 +17,130 @@ static void __used test_common_helpers(void) BUILD_BUG_ON(context_unsafe((void)2, 3) != 3); /* does not swallow commas */ context_unsafe(do { } while (0)); /* works with void statements */ } + +#define TEST_SPINLOCK_COMMON(class, type, type_init, type_lock, type_unlock, type_trylock, op) \ + struct test_##class##_data { \ + type lock; \ + int counter __guarded_by(&lock); \ + int *pointer __pt_guarded_by(&lock); \ + }; \ + static void __used test_##class##_init(struct test_##class##_data *d) \ + { \ + type_init(&d->lock); \ + d->counter = 0; \ + } \ + static void __used test_##class(struct test_##class##_data *d) \ + { \ + unsigned long flags; \ + d->pointer++; \ + type_lock(&d->lock); \ + op(d->counter); \ + op(*d->pointer); \ + type_unlock(&d->lock); \ + type_lock##_irq(&d->lock); \ + op(d->counter); \ + op(*d->pointer); \ + type_unlock##_irq(&d->lock); \ + type_lock##_bh(&d->lock); \ + op(d->counter); \ + op(*d->pointer); \ + type_unlock##_bh(&d->lock); \ + type_lock##_irqsave(&d->lock, flags); \ + op(d->counter); \ + op(*d->pointer); \ + type_unlock##_irqrestore(&d->lock, flags); \ + } \ + static void __used test_##class##_trylock(struct test_##class##_data *d) \ + { \ + if (type_trylock(&d->lock)) { \ + op(d->counter); \ + type_unlock(&d->lock); \ + } \ + } \ + static void __used test_##class##_assert(struct test_##class##_data *d) \ + { \ + lockdep_assert_held(&d->lock); \ + op(d->counter); \ + } \ + static void __used test_##class##_guard(struct test_##class##_data *d) \ + { \ + { guard(class)(&d->lock); op(d->counter); } \ + { guard(class##_irq)(&d->lock); op(d->counter); } \ + { guard(class##_irqsave)(&d->lock); op(d->counter); } \ + } + +#define TEST_OP_RW(x) (x)++ +#define TEST_OP_RO(x) ((void)(x)) + +TEST_SPINLOCK_COMMON(raw_spinlock, + raw_spinlock_t, + raw_spin_lock_init, + raw_spin_lock, + raw_spin_unlock, + raw_spin_trylock, + TEST_OP_RW); +static void __used test_raw_spinlock_trylock_extra(struct test_raw_spinlock_data *d) +{ + unsigned long flags; + + if (raw_spin_trylock_irq(&d->lock)) { + d->counter++; + raw_spin_unlock_irq(&d->lock); + } + if (raw_spin_trylock_irqsave(&d->lock, flags)) { + d->counter++; + raw_spin_unlock_irqrestore(&d->lock, flags); + } + scoped_cond_guard(raw_spinlock_try, return, &d->lock) { + d->counter++; + } +} + +TEST_SPINLOCK_COMMON(spinlock, + spinlock_t, + spin_lock_init, + spin_lock, + spin_unlock, + spin_trylock, + TEST_OP_RW); +static void __used test_spinlock_trylock_extra(struct test_spinlock_data *d) +{ + unsigned long flags; + + if (spin_trylock_irq(&d->lock)) { + d->counter++; + spin_unlock_irq(&d->lock); + } + if (spin_trylock_irqsave(&d->lock, flags)) { + d->counter++; + spin_unlock_irqrestore(&d->lock, flags); + } + scoped_cond_guard(spinlock_try, return, &d->lock) { + d->counter++; + } +} + +TEST_SPINLOCK_COMMON(write_lock, + rwlock_t, + rwlock_init, + write_lock, + write_unlock, + write_trylock, + TEST_OP_RW); +static void __used test_write_trylock_extra(struct test_write_lock_data *d) +{ + unsigned long flags; + + if (write_trylock_irqsave(&d->lock, flags)) { + d->counter++; + write_unlock_irqrestore(&d->lock, flags); + } +} + +TEST_SPINLOCK_COMMON(read_lock, + rwlock_t, + rwlock_init, + read_lock, + read_unlock, + read_trylock, + TEST_OP_RO); From 38f1311a2219220a3962fae464ca6300ef60b4c1 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:39:58 +0100 Subject: [PATCH 12/90] compiler-context-analysis: Change __cond_acquires to take return value While Sparse is oblivious to the return value of conditional acquire functions, Clang's context analysis needs to know the return value which indicates successful acquisition. Add the additional argument, and convert existing uses. Notably, Clang's interpretation of the value merely relates to the use in a later conditional branch, i.e. 1 ==> context lock acquired in branch taken if condition non-zero, and 0 ==> context lock acquired in branch taken if condition is zero. Given the precise value does not matter, introduce symbolic variants to use instead of either 0 or 1, which should be more intuitive. No functional change intended. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-10-elver@google.com --- fs/dlm/lock.c | 2 +- include/linux/compiler-context-analysis.h | 31 +++++++++++++++++++---- include/linux/refcount.h | 6 ++--- include/linux/spinlock.h | 6 ++--- include/linux/spinlock_api_smp.h | 8 +++--- net/ipv4/tcp_sigpool.c | 2 +- 6 files changed, 38 insertions(+), 17 deletions(-) diff --git a/fs/dlm/lock.c b/fs/dlm/lock.c index be938fdf17d9..0ce04be0d3de 100644 --- a/fs/dlm/lock.c +++ b/fs/dlm/lock.c @@ -343,7 +343,7 @@ void dlm_hold_rsb(struct dlm_rsb *r) /* TODO move this to lib/refcount.c */ static __must_check bool dlm_refcount_dec_and_write_lock_bh(refcount_t *r, rwlock_t *lock) -__cond_acquires(lock) + __cond_acquires(true, lock) { if (refcount_dec_not_one(r)) return false; diff --git a/include/linux/compiler-context-analysis.h b/include/linux/compiler-context-analysis.h index d0b3cf0ebfe9..a6a34985dbb2 100644 --- a/include/linux/compiler-context-analysis.h +++ b/include/linux/compiler-context-analysis.h @@ -271,7 +271,7 @@ static inline void _context_unsafe_alias(void **p) { } # define __must_hold(x) __attribute__((context(x,1,1))) # define __must_not_hold(x) # define __acquires(x) __attribute__((context(x,0,1))) -# define __cond_acquires(x) __attribute__((context(x,0,-1))) +# define __cond_acquires(ret, x) __attribute__((context(x,0,-1))) # define __releases(x) __attribute__((context(x,1,0))) # define __acquire(x) __context__(x,1) # define __release(x) __context__(x,-1) @@ -314,15 +314,32 @@ static inline void _context_unsafe_alias(void **p) { } */ # define __acquires(x) __acquires_ctx_lock(x) +/* + * Clang's analysis does not care precisely about the value, only that it is + * either zero or non-zero. So the __cond_acquires() interface might be + * misleading if we say that @ret is the value returned if acquired. Instead, + * provide symbolic variants which we translate. + */ +#define __cond_acquires_impl_true(x, ...) __try_acquires##__VA_ARGS__##_ctx_lock(1, x) +#define __cond_acquires_impl_false(x, ...) __try_acquires##__VA_ARGS__##_ctx_lock(0, x) +#define __cond_acquires_impl_nonzero(x, ...) __try_acquires##__VA_ARGS__##_ctx_lock(1, x) +#define __cond_acquires_impl_0(x, ...) __try_acquires##__VA_ARGS__##_ctx_lock(0, x) +#define __cond_acquires_impl_nonnull(x, ...) __try_acquires##__VA_ARGS__##_ctx_lock(1, x) +#define __cond_acquires_impl_NULL(x, ...) __try_acquires##__VA_ARGS__##_ctx_lock(0, x) + /** * __cond_acquires() - function attribute, function conditionally * acquires a context lock exclusively + * @ret: abstract value returned by function if context lock acquired * @x: context lock instance pointer * * Function attribute declaring that the function conditionally acquires the - * given context lock instance @x exclusively, but does not release it. + * given context lock instance @x exclusively, but does not release it. The + * function return value @ret denotes when the context lock is acquired. + * + * @ret may be one of: true, false, nonzero, 0, nonnull, NULL. */ -# define __cond_acquires(x) __try_acquires_ctx_lock(1, x) +# define __cond_acquires(ret, x) __cond_acquires_impl_##ret(x) /** * __releases() - function attribute, function releases a context lock exclusively @@ -389,12 +406,16 @@ static inline void _context_unsafe_alias(void **p) { } /** * __cond_acquires_shared() - function attribute, function conditionally * acquires a context lock shared + * @ret: abstract value returned by function if context lock acquired * @x: context lock instance pointer * * Function attribute declaring that the function conditionally acquires the - * given context lock instance @x with shared access, but does not release it. + * given context lock instance @x with shared access, but does not release it. The + * function return value @ret denotes when the context lock is acquired. + * + * @ret may be one of: true, false, nonzero, 0, nonnull, NULL. */ -# define __cond_acquires_shared(x) __try_acquires_shared_ctx_lock(1, x) +# define __cond_acquires_shared(ret, x) __cond_acquires_impl_##ret(x, _shared) /** * __releases_shared() - function attribute, function releases a diff --git a/include/linux/refcount.h b/include/linux/refcount.h index 80dc023ac2bf..3da377ffb0c2 100644 --- a/include/linux/refcount.h +++ b/include/linux/refcount.h @@ -478,9 +478,9 @@ static inline void refcount_dec(refcount_t *r) extern __must_check bool refcount_dec_if_one(refcount_t *r); extern __must_check bool refcount_dec_not_one(refcount_t *r); -extern __must_check bool refcount_dec_and_mutex_lock(refcount_t *r, struct mutex *lock) __cond_acquires(lock); -extern __must_check bool refcount_dec_and_lock(refcount_t *r, spinlock_t *lock) __cond_acquires(lock); +extern __must_check bool refcount_dec_and_mutex_lock(refcount_t *r, struct mutex *lock) __cond_acquires(true, lock); +extern __must_check bool refcount_dec_and_lock(refcount_t *r, spinlock_t *lock) __cond_acquires(true, lock); extern __must_check bool refcount_dec_and_lock_irqsave(refcount_t *r, spinlock_t *lock, - unsigned long *flags) __cond_acquires(lock); + unsigned long *flags) __cond_acquires(true, lock); #endif /* _LINUX_REFCOUNT_H */ diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index 72aabdd4fa3f..7e560c7a7b23 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -362,7 +362,7 @@ static __always_inline void spin_lock_bh(spinlock_t *lock) } static __always_inline int spin_trylock(spinlock_t *lock) - __cond_acquires(lock) __no_context_analysis + __cond_acquires(true, lock) __no_context_analysis { return raw_spin_trylock(&lock->rlock); } @@ -422,13 +422,13 @@ static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, unsigned lo } static __always_inline int spin_trylock_bh(spinlock_t *lock) - __cond_acquires(lock) __no_context_analysis + __cond_acquires(true, lock) __no_context_analysis { return raw_spin_trylock_bh(&lock->rlock); } static __always_inline int spin_trylock_irq(spinlock_t *lock) - __cond_acquires(lock) __no_context_analysis + __cond_acquires(true, lock) __no_context_analysis { return raw_spin_trylock_irq(&lock->rlock); } diff --git a/include/linux/spinlock_api_smp.h b/include/linux/spinlock_api_smp.h index d19327e04df9..7e7d7d373213 100644 --- a/include/linux/spinlock_api_smp.h +++ b/include/linux/spinlock_api_smp.h @@ -34,8 +34,8 @@ unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock) unsigned long __lockfunc _raw_spin_lock_irqsave_nested(raw_spinlock_t *lock, int subclass) __acquires(lock); -int __lockfunc _raw_spin_trylock(raw_spinlock_t *lock) __cond_acquires(lock); -int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock) __cond_acquires(lock); +int __lockfunc _raw_spin_trylock(raw_spinlock_t *lock) __cond_acquires(true, lock); +int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock) __cond_acquires(true, lock); void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock) __releases(lock); void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) __releases(lock); void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) __releases(lock); @@ -84,7 +84,7 @@ _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) #endif static inline int __raw_spin_trylock(raw_spinlock_t *lock) - __cond_acquires(lock) + __cond_acquires(true, lock) { preempt_disable(); if (do_raw_spin_trylock(lock)) { @@ -177,7 +177,7 @@ static inline void __raw_spin_unlock_bh(raw_spinlock_t *lock) } static inline int __raw_spin_trylock_bh(raw_spinlock_t *lock) - __cond_acquires(lock) + __cond_acquires(true, lock) { __local_bh_disable_ip(_RET_IP_, SOFTIRQ_LOCK_OFFSET); if (do_raw_spin_trylock(lock)) { diff --git a/net/ipv4/tcp_sigpool.c b/net/ipv4/tcp_sigpool.c index d8a4f192873a..10b2e5970c40 100644 --- a/net/ipv4/tcp_sigpool.c +++ b/net/ipv4/tcp_sigpool.c @@ -257,7 +257,7 @@ void tcp_sigpool_get(unsigned int id) } EXPORT_SYMBOL_GPL(tcp_sigpool_get); -int tcp_sigpool_start(unsigned int id, struct tcp_sigpool *c) __cond_acquires(RCU_BH) +int tcp_sigpool_start(unsigned int id, struct tcp_sigpool *c) __cond_acquires(0, RCU_BH) { struct crypto_ahash *hash; From 370f0a345a70fe36d0185abf87c7ee8e70572e06 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:39:59 +0100 Subject: [PATCH 13/90] locking/mutex: Support Clang's context analysis Add support for Clang's context analysis for mutex. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-11-elver@google.com --- Documentation/dev-tools/context-analysis.rst | 2 +- include/linux/mutex.h | 38 +++++++----- include/linux/mutex_types.h | 4 +- lib/test_context-analysis.c | 64 ++++++++++++++++++++ 4 files changed, 90 insertions(+), 18 deletions(-) diff --git a/Documentation/dev-tools/context-analysis.rst b/Documentation/dev-tools/context-analysis.rst index 746a2d275fb2..1864b6cba4d1 100644 --- a/Documentation/dev-tools/context-analysis.rst +++ b/Documentation/dev-tools/context-analysis.rst @@ -79,7 +79,7 @@ Supported Kernel Primitives ~~~~~~~~~~~~~~~~~~~~~~~~~~~ Currently the following synchronization primitives are supported: -`raw_spinlock_t`, `spinlock_t`, `rwlock_t`. +`raw_spinlock_t`, `spinlock_t`, `rwlock_t`, `mutex`. For context locks with an initialization function (e.g., `spin_lock_init()`), calling this function before initializing any guarded members or globals diff --git a/include/linux/mutex.h b/include/linux/mutex.h index bf535f0118bb..89977c215cbd 100644 --- a/include/linux/mutex.h +++ b/include/linux/mutex.h @@ -62,6 +62,7 @@ do { \ static struct lock_class_key __key; \ \ __mutex_init((mutex), #mutex, &__key); \ + __assume_ctx_lock(mutex); \ } while (0) /** @@ -182,13 +183,13 @@ static inline int __must_check __devm_mutex_init(struct device *dev, struct mute * Also see Documentation/locking/mutex-design.rst. */ #ifdef CONFIG_DEBUG_LOCK_ALLOC -extern void mutex_lock_nested(struct mutex *lock, unsigned int subclass); +extern void mutex_lock_nested(struct mutex *lock, unsigned int subclass) __acquires(lock); extern void _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock); extern int __must_check mutex_lock_interruptible_nested(struct mutex *lock, - unsigned int subclass); + unsigned int subclass) __cond_acquires(0, lock); extern int __must_check _mutex_lock_killable(struct mutex *lock, - unsigned int subclass, struct lockdep_map *nest_lock); -extern void mutex_lock_io_nested(struct mutex *lock, unsigned int subclass); + unsigned int subclass, struct lockdep_map *nest_lock) __cond_acquires(0, lock); +extern void mutex_lock_io_nested(struct mutex *lock, unsigned int subclass) __acquires(lock); #define mutex_lock(lock) mutex_lock_nested(lock, 0) #define mutex_lock_interruptible(lock) mutex_lock_interruptible_nested(lock, 0) @@ -211,10 +212,10 @@ do { \ _mutex_lock_killable(lock, subclass, NULL) #else -extern void mutex_lock(struct mutex *lock); -extern int __must_check mutex_lock_interruptible(struct mutex *lock); -extern int __must_check mutex_lock_killable(struct mutex *lock); -extern void mutex_lock_io(struct mutex *lock); +extern void mutex_lock(struct mutex *lock) __acquires(lock); +extern int __must_check mutex_lock_interruptible(struct mutex *lock) __cond_acquires(0, lock); +extern int __must_check mutex_lock_killable(struct mutex *lock) __cond_acquires(0, lock); +extern void mutex_lock_io(struct mutex *lock) __acquires(lock); # define mutex_lock_nested(lock, subclass) mutex_lock(lock) # define mutex_lock_interruptible_nested(lock, subclass) mutex_lock_interruptible(lock) @@ -232,7 +233,7 @@ extern void mutex_lock_io(struct mutex *lock); */ #ifdef CONFIG_DEBUG_LOCK_ALLOC -extern int _mutex_trylock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock); +extern int _mutex_trylock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock) __cond_acquires(true, lock); #define mutex_trylock_nest_lock(lock, nest_lock) \ ( \ @@ -242,17 +243,24 @@ extern int _mutex_trylock_nest_lock(struct mutex *lock, struct lockdep_map *nest #define mutex_trylock(lock) _mutex_trylock_nest_lock(lock, NULL) #else -extern int mutex_trylock(struct mutex *lock); +extern int mutex_trylock(struct mutex *lock) __cond_acquires(true, lock); #define mutex_trylock_nest_lock(lock, nest_lock) mutex_trylock(lock) #endif -extern void mutex_unlock(struct mutex *lock); +extern void mutex_unlock(struct mutex *lock) __releases(lock); -extern int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock); +extern int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock) __cond_acquires(true, lock); -DEFINE_GUARD(mutex, struct mutex *, mutex_lock(_T), mutex_unlock(_T)) -DEFINE_GUARD_COND(mutex, _try, mutex_trylock(_T)) -DEFINE_GUARD_COND(mutex, _intr, mutex_lock_interruptible(_T), _RET == 0) +DEFINE_LOCK_GUARD_1(mutex, struct mutex, mutex_lock(_T->lock), mutex_unlock(_T->lock)) +DEFINE_LOCK_GUARD_1_COND(mutex, _try, mutex_trylock(_T->lock)) +DEFINE_LOCK_GUARD_1_COND(mutex, _intr, mutex_lock_interruptible(_T->lock), _RET == 0) + +DECLARE_LOCK_GUARD_1_ATTRS(mutex, __acquires(_T), __releases(*(struct mutex **)_T)) +#define class_mutex_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(mutex, _T) +DECLARE_LOCK_GUARD_1_ATTRS(mutex_try, __acquires(_T), __releases(*(struct mutex **)_T)) +#define class_mutex_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(mutex_try, _T) +DECLARE_LOCK_GUARD_1_ATTRS(mutex_intr, __acquires(_T), __releases(*(struct mutex **)_T)) +#define class_mutex_intr_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(mutex_intr, _T) extern unsigned long mutex_get_owner(struct mutex *lock); diff --git a/include/linux/mutex_types.h b/include/linux/mutex_types.h index fdf7f515fde8..80975935ec48 100644 --- a/include/linux/mutex_types.h +++ b/include/linux/mutex_types.h @@ -38,7 +38,7 @@ * - detects multi-task circular deadlocks and prints out all affected * locks and tasks (and only those tasks) */ -struct mutex { +context_lock_struct(mutex) { atomic_long_t owner; raw_spinlock_t wait_lock; #ifdef CONFIG_MUTEX_SPIN_ON_OWNER @@ -59,7 +59,7 @@ struct mutex { */ #include -struct mutex { +context_lock_struct(mutex) { struct rt_mutex_base rtmutex; #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map dep_map; diff --git a/lib/test_context-analysis.c b/lib/test_context-analysis.c index 273fa9d34657..2b28d20c5f51 100644 --- a/lib/test_context-analysis.c +++ b/lib/test_context-analysis.c @@ -5,6 +5,7 @@ */ #include +#include #include /* @@ -144,3 +145,66 @@ TEST_SPINLOCK_COMMON(read_lock, read_unlock, read_trylock, TEST_OP_RO); + +struct test_mutex_data { + struct mutex mtx; + int counter __guarded_by(&mtx); +}; + +static void __used test_mutex_init(struct test_mutex_data *d) +{ + mutex_init(&d->mtx); + d->counter = 0; +} + +static void __used test_mutex_lock(struct test_mutex_data *d) +{ + mutex_lock(&d->mtx); + d->counter++; + mutex_unlock(&d->mtx); + mutex_lock_io(&d->mtx); + d->counter++; + mutex_unlock(&d->mtx); +} + +static void __used test_mutex_trylock(struct test_mutex_data *d, atomic_t *a) +{ + if (!mutex_lock_interruptible(&d->mtx)) { + d->counter++; + mutex_unlock(&d->mtx); + } + if (!mutex_lock_killable(&d->mtx)) { + d->counter++; + mutex_unlock(&d->mtx); + } + if (mutex_trylock(&d->mtx)) { + d->counter++; + mutex_unlock(&d->mtx); + } + if (atomic_dec_and_mutex_lock(a, &d->mtx)) { + d->counter++; + mutex_unlock(&d->mtx); + } +} + +static void __used test_mutex_assert(struct test_mutex_data *d) +{ + lockdep_assert_held(&d->mtx); + d->counter++; +} + +static void __used test_mutex_guard(struct test_mutex_data *d) +{ + guard(mutex)(&d->mtx); + d->counter++; +} + +static void __used test_mutex_cond_guard(struct test_mutex_data *d) +{ + scoped_cond_guard(mutex_try, return, &d->mtx) { + d->counter++; + } + scoped_cond_guard(mutex_intr, return, &d->mtx) { + d->counter++; + } +} From 8f8a55f49cda5fee914bbea1ab5af8df3a6ba8af Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:00 +0100 Subject: [PATCH 14/90] locking/seqlock: Support Clang's context analysis Add support for Clang's context analysis for seqlock_t. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-12-elver@google.com --- Documentation/dev-tools/context-analysis.rst | 2 +- include/linux/seqlock.h | 38 ++++++++++++++- include/linux/seqlock_types.h | 5 +- lib/test_context-analysis.c | 50 ++++++++++++++++++++ 4 files changed, 91 insertions(+), 4 deletions(-) diff --git a/Documentation/dev-tools/context-analysis.rst b/Documentation/dev-tools/context-analysis.rst index 1864b6cba4d1..690565910084 100644 --- a/Documentation/dev-tools/context-analysis.rst +++ b/Documentation/dev-tools/context-analysis.rst @@ -79,7 +79,7 @@ Supported Kernel Primitives ~~~~~~~~~~~~~~~~~~~~~~~~~~~ Currently the following synchronization primitives are supported: -`raw_spinlock_t`, `spinlock_t`, `rwlock_t`, `mutex`. +`raw_spinlock_t`, `spinlock_t`, `rwlock_t`, `mutex`, `seqlock_t`. For context locks with an initialization function (e.g., `spin_lock_init()`), calling this function before initializing any guarded members or globals diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h index 221123660e71..113320911a09 100644 --- a/include/linux/seqlock.h +++ b/include/linux/seqlock.h @@ -816,6 +816,7 @@ static __always_inline void write_seqcount_latch_end(seqcount_latch_t *s) do { \ spin_lock_init(&(sl)->lock); \ seqcount_spinlock_init(&(sl)->seqcount, &(sl)->lock); \ + __assume_ctx_lock(sl); \ } while (0) /** @@ -832,6 +833,7 @@ static __always_inline void write_seqcount_latch_end(seqcount_latch_t *s) * Return: count, to be passed to read_seqretry() */ static inline unsigned read_seqbegin(const seqlock_t *sl) + __acquires_shared(sl) __no_context_analysis { return read_seqcount_begin(&sl->seqcount); } @@ -848,6 +850,7 @@ static inline unsigned read_seqbegin(const seqlock_t *sl) * Return: true if a read section retry is required, else false */ static inline unsigned read_seqretry(const seqlock_t *sl, unsigned start) + __releases_shared(sl) __no_context_analysis { return read_seqcount_retry(&sl->seqcount, start); } @@ -872,6 +875,7 @@ static inline unsigned read_seqretry(const seqlock_t *sl, unsigned start) * _irqsave or _bh variants of this function instead. */ static inline void write_seqlock(seqlock_t *sl) + __acquires(sl) __no_context_analysis { spin_lock(&sl->lock); do_write_seqcount_begin(&sl->seqcount.seqcount); @@ -885,6 +889,7 @@ static inline void write_seqlock(seqlock_t *sl) * critical section of given seqlock_t. */ static inline void write_sequnlock(seqlock_t *sl) + __releases(sl) __no_context_analysis { do_write_seqcount_end(&sl->seqcount.seqcount); spin_unlock(&sl->lock); @@ -898,6 +903,7 @@ static inline void write_sequnlock(seqlock_t *sl) * other write side sections, can be invoked from softirq contexts. */ static inline void write_seqlock_bh(seqlock_t *sl) + __acquires(sl) __no_context_analysis { spin_lock_bh(&sl->lock); do_write_seqcount_begin(&sl->seqcount.seqcount); @@ -912,6 +918,7 @@ static inline void write_seqlock_bh(seqlock_t *sl) * write_seqlock_bh(). */ static inline void write_sequnlock_bh(seqlock_t *sl) + __releases(sl) __no_context_analysis { do_write_seqcount_end(&sl->seqcount.seqcount); spin_unlock_bh(&sl->lock); @@ -925,6 +932,7 @@ static inline void write_sequnlock_bh(seqlock_t *sl) * other write sections, can be invoked from hardirq contexts. */ static inline void write_seqlock_irq(seqlock_t *sl) + __acquires(sl) __no_context_analysis { spin_lock_irq(&sl->lock); do_write_seqcount_begin(&sl->seqcount.seqcount); @@ -938,12 +946,14 @@ static inline void write_seqlock_irq(seqlock_t *sl) * seqlock_t write side section opened with write_seqlock_irq(). */ static inline void write_sequnlock_irq(seqlock_t *sl) + __releases(sl) __no_context_analysis { do_write_seqcount_end(&sl->seqcount.seqcount); spin_unlock_irq(&sl->lock); } static inline unsigned long __write_seqlock_irqsave(seqlock_t *sl) + __acquires(sl) __no_context_analysis { unsigned long flags; @@ -976,6 +986,7 @@ static inline unsigned long __write_seqlock_irqsave(seqlock_t *sl) */ static inline void write_sequnlock_irqrestore(seqlock_t *sl, unsigned long flags) + __releases(sl) __no_context_analysis { do_write_seqcount_end(&sl->seqcount.seqcount); spin_unlock_irqrestore(&sl->lock, flags); @@ -998,6 +1009,7 @@ write_sequnlock_irqrestore(seqlock_t *sl, unsigned long flags) * The opened read section must be closed with read_sequnlock_excl(). */ static inline void read_seqlock_excl(seqlock_t *sl) + __acquires_shared(sl) __no_context_analysis { spin_lock(&sl->lock); } @@ -1007,6 +1019,7 @@ static inline void read_seqlock_excl(seqlock_t *sl) * @sl: Pointer to seqlock_t */ static inline void read_sequnlock_excl(seqlock_t *sl) + __releases_shared(sl) __no_context_analysis { spin_unlock(&sl->lock); } @@ -1021,6 +1034,7 @@ static inline void read_sequnlock_excl(seqlock_t *sl) * from softirq contexts. */ static inline void read_seqlock_excl_bh(seqlock_t *sl) + __acquires_shared(sl) __no_context_analysis { spin_lock_bh(&sl->lock); } @@ -1031,6 +1045,7 @@ static inline void read_seqlock_excl_bh(seqlock_t *sl) * @sl: Pointer to seqlock_t */ static inline void read_sequnlock_excl_bh(seqlock_t *sl) + __releases_shared(sl) __no_context_analysis { spin_unlock_bh(&sl->lock); } @@ -1045,6 +1060,7 @@ static inline void read_sequnlock_excl_bh(seqlock_t *sl) * hardirq context. */ static inline void read_seqlock_excl_irq(seqlock_t *sl) + __acquires_shared(sl) __no_context_analysis { spin_lock_irq(&sl->lock); } @@ -1055,11 +1071,13 @@ static inline void read_seqlock_excl_irq(seqlock_t *sl) * @sl: Pointer to seqlock_t */ static inline void read_sequnlock_excl_irq(seqlock_t *sl) + __releases_shared(sl) __no_context_analysis { spin_unlock_irq(&sl->lock); } static inline unsigned long __read_seqlock_excl_irqsave(seqlock_t *sl) + __acquires_shared(sl) __no_context_analysis { unsigned long flags; @@ -1089,6 +1107,7 @@ static inline unsigned long __read_seqlock_excl_irqsave(seqlock_t *sl) */ static inline void read_sequnlock_excl_irqrestore(seqlock_t *sl, unsigned long flags) + __releases_shared(sl) __no_context_analysis { spin_unlock_irqrestore(&sl->lock, flags); } @@ -1125,6 +1144,7 @@ read_sequnlock_excl_irqrestore(seqlock_t *sl, unsigned long flags) * parameter of the next read_seqbegin_or_lock() iteration. */ static inline void read_seqbegin_or_lock(seqlock_t *lock, int *seq) + __acquires_shared(lock) __no_context_analysis { if (!(*seq & 1)) /* Even */ *seq = read_seqbegin(lock); @@ -1140,6 +1160,7 @@ static inline void read_seqbegin_or_lock(seqlock_t *lock, int *seq) * Return: true if a read section retry is required, false otherwise */ static inline int need_seqretry(seqlock_t *lock, int seq) + __releases_shared(lock) __no_context_analysis { return !(seq & 1) && read_seqretry(lock, seq); } @@ -1153,6 +1174,7 @@ static inline int need_seqretry(seqlock_t *lock, int seq) * with read_seqbegin_or_lock() and validated by need_seqretry(). */ static inline void done_seqretry(seqlock_t *lock, int seq) + __no_context_analysis { if (seq & 1) read_sequnlock_excl(lock); @@ -1180,6 +1202,7 @@ static inline void done_seqretry(seqlock_t *lock, int seq) */ static inline unsigned long read_seqbegin_or_lock_irqsave(seqlock_t *lock, int *seq) + __acquires_shared(lock) __no_context_analysis { unsigned long flags = 0; @@ -1205,6 +1228,7 @@ read_seqbegin_or_lock_irqsave(seqlock_t *lock, int *seq) */ static inline void done_seqretry_irqrestore(seqlock_t *lock, int seq, unsigned long flags) + __no_context_analysis { if (seq & 1) read_sequnlock_excl_irqrestore(lock, flags); @@ -1225,6 +1249,7 @@ struct ss_tmp { }; static __always_inline void __scoped_seqlock_cleanup(struct ss_tmp *sst) + __no_context_analysis { if (sst->lock) spin_unlock(sst->lock); @@ -1254,6 +1279,7 @@ extern void __scoped_seqlock_bug(void); static __always_inline void __scoped_seqlock_next(struct ss_tmp *sst, seqlock_t *lock, enum ss_state target) + __no_context_analysis { switch (sst->state) { case ss_done: @@ -1296,9 +1322,19 @@ __scoped_seqlock_next(struct ss_tmp *sst, seqlock_t *lock, enum ss_state target) } } +/* + * Context analysis no-op helper to release seqlock at the end of the for-scope; + * the alias analysis of the compiler will recognize that the pointer @s is an + * alias to @_seqlock passed to read_seqbegin(_seqlock) below. + */ +static __always_inline void __scoped_seqlock_cleanup_ctx(struct ss_tmp **s) + __releases_shared(*((seqlock_t **)s)) __no_context_analysis {} + #define __scoped_seqlock_read(_seqlock, _target, _s) \ for (struct ss_tmp _s __cleanup(__scoped_seqlock_cleanup) = \ - { .state = ss_lockless, .data = read_seqbegin(_seqlock) }; \ + { .state = ss_lockless, .data = read_seqbegin(_seqlock) }, \ + *__UNIQUE_ID(ctx) __cleanup(__scoped_seqlock_cleanup_ctx) =\ + (struct ss_tmp *)_seqlock; \ _s.state != ss_done; \ __scoped_seqlock_next(&_s, _seqlock, _target)) diff --git a/include/linux/seqlock_types.h b/include/linux/seqlock_types.h index dfdf43e3fa3d..2d5d793ef660 100644 --- a/include/linux/seqlock_types.h +++ b/include/linux/seqlock_types.h @@ -81,13 +81,14 @@ SEQCOUNT_LOCKNAME(mutex, struct mutex, true, mutex) * - Comments on top of seqcount_t * - Documentation/locking/seqlock.rst */ -typedef struct { +context_lock_struct(seqlock) { /* * Make sure that readers don't starve writers on PREEMPT_RT: use * seqcount_spinlock_t instead of seqcount_t. Check __SEQ_LOCK(). */ seqcount_spinlock_t seqcount; spinlock_t lock; -} seqlock_t; +}; +typedef struct seqlock seqlock_t; #endif /* __LINUX_SEQLOCK_TYPES_H */ diff --git a/lib/test_context-analysis.c b/lib/test_context-analysis.c index 2b28d20c5f51..53abea0008f2 100644 --- a/lib/test_context-analysis.c +++ b/lib/test_context-analysis.c @@ -6,6 +6,7 @@ #include #include +#include #include /* @@ -208,3 +209,52 @@ static void __used test_mutex_cond_guard(struct test_mutex_data *d) d->counter++; } } + +struct test_seqlock_data { + seqlock_t sl; + int counter __guarded_by(&sl); +}; + +static void __used test_seqlock_init(struct test_seqlock_data *d) +{ + seqlock_init(&d->sl); + d->counter = 0; +} + +static void __used test_seqlock_reader(struct test_seqlock_data *d) +{ + unsigned int seq; + + do { + seq = read_seqbegin(&d->sl); + (void)d->counter; + } while (read_seqretry(&d->sl, seq)); +} + +static void __used test_seqlock_writer(struct test_seqlock_data *d) +{ + unsigned long flags; + + write_seqlock(&d->sl); + d->counter++; + write_sequnlock(&d->sl); + + write_seqlock_irq(&d->sl); + d->counter++; + write_sequnlock_irq(&d->sl); + + write_seqlock_bh(&d->sl); + d->counter++; + write_sequnlock_bh(&d->sl); + + write_seqlock_irqsave(&d->sl, flags); + d->counter++; + write_sequnlock_irqrestore(&d->sl, flags); +} + +static void __used test_seqlock_scoped(struct test_seqlock_data *d) +{ + scoped_seqlock_read (&d->sl, ss_lockless) { + (void)d->counter; + } +} From 5f7ba059710609bb997d50775ba92fbf29be51da Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:01 +0100 Subject: [PATCH 15/90] bit_spinlock: Include missing Including into an empty TU will result in the compiler complaining: ./include/linux/bit_spinlock.h:34:4: error: call to undeclared function 'cpu_relax'; <...> 34 | cpu_relax(); | ^ 1 error generated. Include to allow including bit_spinlock.h where is not otherwise included. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Bart Van Assche Link: https://patch.msgid.link/20251219154418.3592607-13-elver@google.com --- include/linux/bit_spinlock.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/linux/bit_spinlock.h b/include/linux/bit_spinlock.h index c0989b5b0407..59e345f74b0e 100644 --- a/include/linux/bit_spinlock.h +++ b/include/linux/bit_spinlock.h @@ -7,6 +7,8 @@ #include #include +#include /* for cpu_relax() */ + /* * bit-based spin_lock() * From eb7d96a13bf45f86909006a59e7855d8810f020a Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:02 +0100 Subject: [PATCH 16/90] bit_spinlock: Support Clang's context analysis The annotations for bit_spinlock.h have simply been using "bitlock" as the token. For Sparse, that was likely sufficient in most cases. But Clang's context analysis is more precise, and we need to ensure we can distinguish different bitlocks. To do so, add a token context, and a macro __bitlock(bitnum, addr) that is used to construct unique per-bitlock tokens. Add the appropriate test. is implicitly included through other includes, and requires 2 annotations to indicate that acquisition (without release) and release (without prior acquisition) of its bitlock is intended. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-14-elver@google.com --- Documentation/dev-tools/context-analysis.rst | 3 ++- include/linux/bit_spinlock.h | 22 ++++++++++++++--- include/linux/list_bl.h | 2 ++ lib/test_context-analysis.c | 26 ++++++++++++++++++++ 4 files changed, 48 insertions(+), 5 deletions(-) diff --git a/Documentation/dev-tools/context-analysis.rst b/Documentation/dev-tools/context-analysis.rst index 690565910084..b2d69fb4a884 100644 --- a/Documentation/dev-tools/context-analysis.rst +++ b/Documentation/dev-tools/context-analysis.rst @@ -79,7 +79,8 @@ Supported Kernel Primitives ~~~~~~~~~~~~~~~~~~~~~~~~~~~ Currently the following synchronization primitives are supported: -`raw_spinlock_t`, `spinlock_t`, `rwlock_t`, `mutex`, `seqlock_t`. +`raw_spinlock_t`, `spinlock_t`, `rwlock_t`, `mutex`, `seqlock_t`, +`bit_spinlock`. For context locks with an initialization function (e.g., `spin_lock_init()`), calling this function before initializing any guarded members or globals diff --git a/include/linux/bit_spinlock.h b/include/linux/bit_spinlock.h index 59e345f74b0e..7869a6e59b6a 100644 --- a/include/linux/bit_spinlock.h +++ b/include/linux/bit_spinlock.h @@ -9,6 +9,16 @@ #include /* for cpu_relax() */ +/* + * For static context analysis, we need a unique token for each possible bit + * that can be used as a bit_spinlock. The easiest way to do that is to create a + * fake context that we can cast to with the __bitlock(bitnum, addr) macro + * below, which will give us unique instances for each (bit, addr) pair that the + * static analysis can use. + */ +context_lock_struct(__context_bitlock) { }; +#define __bitlock(bitnum, addr) (struct __context_bitlock *)(bitnum + (addr)) + /* * bit-based spin_lock() * @@ -16,6 +26,7 @@ * are significantly faster. */ static __always_inline void bit_spin_lock(int bitnum, unsigned long *addr) + __acquires(__bitlock(bitnum, addr)) { /* * Assuming the lock is uncontended, this never enters @@ -34,13 +45,14 @@ static __always_inline void bit_spin_lock(int bitnum, unsigned long *addr) preempt_disable(); } #endif - __acquire(bitlock); + __acquire(__bitlock(bitnum, addr)); } /* * Return true if it was acquired */ static __always_inline int bit_spin_trylock(int bitnum, unsigned long *addr) + __cond_acquires(true, __bitlock(bitnum, addr)) { preempt_disable(); #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) @@ -49,7 +61,7 @@ static __always_inline int bit_spin_trylock(int bitnum, unsigned long *addr) return 0; } #endif - __acquire(bitlock); + __acquire(__bitlock(bitnum, addr)); return 1; } @@ -57,6 +69,7 @@ static __always_inline int bit_spin_trylock(int bitnum, unsigned long *addr) * bit-based spin_unlock() */ static __always_inline void bit_spin_unlock(int bitnum, unsigned long *addr) + __releases(__bitlock(bitnum, addr)) { #ifdef CONFIG_DEBUG_SPINLOCK BUG_ON(!test_bit(bitnum, addr)); @@ -65,7 +78,7 @@ static __always_inline void bit_spin_unlock(int bitnum, unsigned long *addr) clear_bit_unlock(bitnum, addr); #endif preempt_enable(); - __release(bitlock); + __release(__bitlock(bitnum, addr)); } /* @@ -74,6 +87,7 @@ static __always_inline void bit_spin_unlock(int bitnum, unsigned long *addr) * protecting the rest of the flags in the word. */ static __always_inline void __bit_spin_unlock(int bitnum, unsigned long *addr) + __releases(__bitlock(bitnum, addr)) { #ifdef CONFIG_DEBUG_SPINLOCK BUG_ON(!test_bit(bitnum, addr)); @@ -82,7 +96,7 @@ static __always_inline void __bit_spin_unlock(int bitnum, unsigned long *addr) __clear_bit_unlock(bitnum, addr); #endif preempt_enable(); - __release(bitlock); + __release(__bitlock(bitnum, addr)); } /* diff --git a/include/linux/list_bl.h b/include/linux/list_bl.h index ae1b541446c9..df9eebe6afca 100644 --- a/include/linux/list_bl.h +++ b/include/linux/list_bl.h @@ -144,11 +144,13 @@ static inline void hlist_bl_del_init(struct hlist_bl_node *n) } static inline void hlist_bl_lock(struct hlist_bl_head *b) + __acquires(__bitlock(0, b)) { bit_spin_lock(0, (unsigned long *)b); } static inline void hlist_bl_unlock(struct hlist_bl_head *b) + __releases(__bitlock(0, b)) { __bit_spin_unlock(0, (unsigned long *)b); } diff --git a/lib/test_context-analysis.c b/lib/test_context-analysis.c index 53abea0008f2..be0c5d462a48 100644 --- a/lib/test_context-analysis.c +++ b/lib/test_context-analysis.c @@ -4,6 +4,7 @@ * positive errors when compiled with Clang's context analysis. */ +#include #include #include #include @@ -258,3 +259,28 @@ static void __used test_seqlock_scoped(struct test_seqlock_data *d) (void)d->counter; } } + +struct test_bit_spinlock_data { + unsigned long bits; + int counter __guarded_by(__bitlock(3, &bits)); +}; + +static void __used test_bit_spin_lock(struct test_bit_spinlock_data *d) +{ + /* + * Note, the analysis seems to have false negatives, because it won't + * precisely recognize the bit of the fake __bitlock() token. + */ + bit_spin_lock(3, &d->bits); + d->counter++; + bit_spin_unlock(3, &d->bits); + + bit_spin_lock(3, &d->bits); + d->counter++; + __bit_spin_unlock(3, &d->bits); + + if (bit_spin_trylock(3, &d->bits)) { + d->counter++; + bit_spin_unlock(3, &d->bits); + } +} From fe00f6e84621ad441aa99005f2f0fefd0e5e1a2c Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:03 +0100 Subject: [PATCH 17/90] rcu: Support Clang's context analysis Improve the existing annotations to properly support Clang's context analysis. The old annotations distinguished between RCU, RCU_BH, and RCU_SCHED; however, to more easily be able to express that "hold the RCU read lock" without caring if the normal, _bh(), or _sched() variant was used we'd have to remove the distinction of the latter variants: change the _bh() and _sched() variants to also acquire "RCU". When (and if) we introduce context locks to denote more generally that "IRQ", "BH", "PREEMPT" contexts are disabled, it would make sense to acquire these instead of RCU_BH and RCU_SCHED respectively. The above change also simplified introducing __guarded_by support, where only the "RCU" context lock needs to be held: introduce __rcu_guarded, where Clang's context analysis warns if a pointer is dereferenced without any of the RCU locks held, or updated without the appropriate helpers. The primitives rcu_assign_pointer() and friends are wrapped with context_unsafe(), which enforces using them to update RCU-protected pointers marked with __rcu_guarded. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Acked-by: Paul E. McKenney Link: https://patch.msgid.link/20251219154418.3592607-15-elver@google.com --- Documentation/dev-tools/context-analysis.rst | 2 +- include/linux/rcupdate.h | 77 ++++++++++++------ lib/test_context-analysis.c | 85 ++++++++++++++++++++ 3 files changed, 139 insertions(+), 25 deletions(-) diff --git a/Documentation/dev-tools/context-analysis.rst b/Documentation/dev-tools/context-analysis.rst index b2d69fb4a884..3bc72f71fe25 100644 --- a/Documentation/dev-tools/context-analysis.rst +++ b/Documentation/dev-tools/context-analysis.rst @@ -80,7 +80,7 @@ Supported Kernel Primitives Currently the following synchronization primitives are supported: `raw_spinlock_t`, `spinlock_t`, `rwlock_t`, `mutex`, `seqlock_t`, -`bit_spinlock`. +`bit_spinlock`, RCU. For context locks with an initialization function (e.g., `spin_lock_init()`), calling this function before initializing any guarded members or globals diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index c5b30054cd01..50e63eade019 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -31,6 +31,16 @@ #include #include +token_context_lock(RCU, __reentrant_ctx_lock); +token_context_lock_instance(RCU, RCU_SCHED); +token_context_lock_instance(RCU, RCU_BH); + +/* + * A convenience macro that can be used for RCU-protected globals or struct + * members; adds type qualifier __rcu, and also enforces __guarded_by(RCU). + */ +#define __rcu_guarded __rcu __guarded_by(RCU) + #define ULONG_CMP_GE(a, b) (ULONG_MAX / 2 >= (a) - (b)) #define ULONG_CMP_LT(a, b) (ULONG_MAX / 2 < (a) - (b)) @@ -425,7 +435,8 @@ static inline void rcu_preempt_sleep_check(void) { } // See RCU_LOCKDEP_WARN() for an explanation of the double call to // debug_lockdep_rcu_enabled(). -static inline bool lockdep_assert_rcu_helper(bool c) +static inline bool lockdep_assert_rcu_helper(bool c, const struct __ctx_lock_RCU *ctx) + __assumes_shared_ctx_lock(RCU) __assumes_shared_ctx_lock(ctx) { return debug_lockdep_rcu_enabled() && (c || !rcu_is_watching() || !rcu_lockdep_current_cpu_online()) && @@ -438,7 +449,7 @@ static inline bool lockdep_assert_rcu_helper(bool c) * Splats if lockdep is enabled and there is no rcu_read_lock() in effect. */ #define lockdep_assert_in_rcu_read_lock() \ - WARN_ON_ONCE(lockdep_assert_rcu_helper(!lock_is_held(&rcu_lock_map))) + WARN_ON_ONCE(lockdep_assert_rcu_helper(!lock_is_held(&rcu_lock_map), RCU)) /** * lockdep_assert_in_rcu_read_lock_bh - WARN if not protected by rcu_read_lock_bh() @@ -448,7 +459,7 @@ static inline bool lockdep_assert_rcu_helper(bool c) * actual rcu_read_lock_bh() is required. */ #define lockdep_assert_in_rcu_read_lock_bh() \ - WARN_ON_ONCE(lockdep_assert_rcu_helper(!lock_is_held(&rcu_bh_lock_map))) + WARN_ON_ONCE(lockdep_assert_rcu_helper(!lock_is_held(&rcu_bh_lock_map), RCU_BH)) /** * lockdep_assert_in_rcu_read_lock_sched - WARN if not protected by rcu_read_lock_sched() @@ -458,7 +469,7 @@ static inline bool lockdep_assert_rcu_helper(bool c) * instead an actual rcu_read_lock_sched() is required. */ #define lockdep_assert_in_rcu_read_lock_sched() \ - WARN_ON_ONCE(lockdep_assert_rcu_helper(!lock_is_held(&rcu_sched_lock_map))) + WARN_ON_ONCE(lockdep_assert_rcu_helper(!lock_is_held(&rcu_sched_lock_map), RCU_SCHED)) /** * lockdep_assert_in_rcu_reader - WARN if not within some type of RCU reader @@ -476,17 +487,17 @@ static inline bool lockdep_assert_rcu_helper(bool c) WARN_ON_ONCE(lockdep_assert_rcu_helper(!lock_is_held(&rcu_lock_map) && \ !lock_is_held(&rcu_bh_lock_map) && \ !lock_is_held(&rcu_sched_lock_map) && \ - preemptible())) + preemptible(), RCU)) #else /* #ifdef CONFIG_PROVE_RCU */ #define RCU_LOCKDEP_WARN(c, s) do { } while (0 && (c)) #define rcu_sleep_check() do { } while (0) -#define lockdep_assert_in_rcu_read_lock() do { } while (0) -#define lockdep_assert_in_rcu_read_lock_bh() do { } while (0) -#define lockdep_assert_in_rcu_read_lock_sched() do { } while (0) -#define lockdep_assert_in_rcu_reader() do { } while (0) +#define lockdep_assert_in_rcu_read_lock() __assume_shared_ctx_lock(RCU) +#define lockdep_assert_in_rcu_read_lock_bh() __assume_shared_ctx_lock(RCU_BH) +#define lockdep_assert_in_rcu_read_lock_sched() __assume_shared_ctx_lock(RCU_SCHED) +#define lockdep_assert_in_rcu_reader() __assume_shared_ctx_lock(RCU) #endif /* #else #ifdef CONFIG_PROVE_RCU */ @@ -506,11 +517,11 @@ static inline bool lockdep_assert_rcu_helper(bool c) #endif /* #else #ifdef __CHECKER__ */ #define __unrcu_pointer(p, local) \ -({ \ +context_unsafe( \ typeof(*p) *local = (typeof(*p) *__force)(p); \ rcu_check_sparse(p, __rcu); \ - ((typeof(*p) __force __kernel *)(local)); \ -}) + ((typeof(*p) __force __kernel *)(local)) \ +) /** * unrcu_pointer - mark a pointer as not being RCU protected * @p: pointer needing to lose its __rcu property @@ -586,7 +597,7 @@ static inline bool lockdep_assert_rcu_helper(bool c) * other macros that it invokes. */ #define rcu_assign_pointer(p, v) \ -do { \ +context_unsafe( \ uintptr_t _r_a_p__v = (uintptr_t)(v); \ rcu_check_sparse(p, __rcu); \ \ @@ -594,7 +605,7 @@ do { \ WRITE_ONCE((p), (typeof(p))(_r_a_p__v)); \ else \ smp_store_release(&p, RCU_INITIALIZER((typeof(p))_r_a_p__v)); \ -} while (0) +) /** * rcu_replace_pointer() - replace an RCU pointer, returning its old value @@ -861,9 +872,10 @@ do { \ * only when acquiring spinlocks that are subject to priority inheritance. */ static __always_inline void rcu_read_lock(void) + __acquires_shared(RCU) { __rcu_read_lock(); - __acquire(RCU); + __acquire_shared(RCU); rcu_lock_acquire(&rcu_lock_map); RCU_LOCKDEP_WARN(!rcu_is_watching(), "rcu_read_lock() used illegally while idle"); @@ -891,11 +903,12 @@ static __always_inline void rcu_read_lock(void) * See rcu_read_lock() for more information. */ static inline void rcu_read_unlock(void) + __releases_shared(RCU) { RCU_LOCKDEP_WARN(!rcu_is_watching(), "rcu_read_unlock() used illegally while idle"); rcu_lock_release(&rcu_lock_map); /* Keep acq info for rls diags. */ - __release(RCU); + __release_shared(RCU); __rcu_read_unlock(); } @@ -914,9 +927,11 @@ static inline void rcu_read_unlock(void) * was invoked from some other task. */ static inline void rcu_read_lock_bh(void) + __acquires_shared(RCU) __acquires_shared(RCU_BH) { local_bh_disable(); - __acquire(RCU_BH); + __acquire_shared(RCU); + __acquire_shared(RCU_BH); rcu_lock_acquire(&rcu_bh_lock_map); RCU_LOCKDEP_WARN(!rcu_is_watching(), "rcu_read_lock_bh() used illegally while idle"); @@ -928,11 +943,13 @@ static inline void rcu_read_lock_bh(void) * See rcu_read_lock_bh() for more information. */ static inline void rcu_read_unlock_bh(void) + __releases_shared(RCU) __releases_shared(RCU_BH) { RCU_LOCKDEP_WARN(!rcu_is_watching(), "rcu_read_unlock_bh() used illegally while idle"); rcu_lock_release(&rcu_bh_lock_map); - __release(RCU_BH); + __release_shared(RCU_BH); + __release_shared(RCU); local_bh_enable(); } @@ -952,9 +969,11 @@ static inline void rcu_read_unlock_bh(void) * rcu_read_lock_sched() was invoked from an NMI handler. */ static inline void rcu_read_lock_sched(void) + __acquires_shared(RCU) __acquires_shared(RCU_SCHED) { preempt_disable(); - __acquire(RCU_SCHED); + __acquire_shared(RCU); + __acquire_shared(RCU_SCHED); rcu_lock_acquire(&rcu_sched_lock_map); RCU_LOCKDEP_WARN(!rcu_is_watching(), "rcu_read_lock_sched() used illegally while idle"); @@ -962,9 +981,11 @@ static inline void rcu_read_lock_sched(void) /* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */ static inline notrace void rcu_read_lock_sched_notrace(void) + __acquires_shared(RCU) __acquires_shared(RCU_SCHED) { preempt_disable_notrace(); - __acquire(RCU_SCHED); + __acquire_shared(RCU); + __acquire_shared(RCU_SCHED); } /** @@ -973,22 +994,27 @@ static inline notrace void rcu_read_lock_sched_notrace(void) * See rcu_read_lock_sched() for more information. */ static inline void rcu_read_unlock_sched(void) + __releases_shared(RCU) __releases_shared(RCU_SCHED) { RCU_LOCKDEP_WARN(!rcu_is_watching(), "rcu_read_unlock_sched() used illegally while idle"); rcu_lock_release(&rcu_sched_lock_map); - __release(RCU_SCHED); + __release_shared(RCU_SCHED); + __release_shared(RCU); preempt_enable(); } /* Used by lockdep and tracing: cannot be traced, cannot call lockdep. */ static inline notrace void rcu_read_unlock_sched_notrace(void) + __releases_shared(RCU) __releases_shared(RCU_SCHED) { - __release(RCU_SCHED); + __release_shared(RCU_SCHED); + __release_shared(RCU); preempt_enable_notrace(); } static __always_inline void rcu_read_lock_dont_migrate(void) + __acquires_shared(RCU) { if (IS_ENABLED(CONFIG_PREEMPT_RCU)) migrate_disable(); @@ -996,6 +1022,7 @@ static __always_inline void rcu_read_lock_dont_migrate(void) } static inline void rcu_read_unlock_migrate(void) + __releases_shared(RCU) { rcu_read_unlock(); if (IS_ENABLED(CONFIG_PREEMPT_RCU)) @@ -1041,10 +1068,10 @@ static inline void rcu_read_unlock_migrate(void) * ordering guarantees for either the CPU or the compiler. */ #define RCU_INIT_POINTER(p, v) \ - do { \ + context_unsafe( \ rcu_check_sparse(p, __rcu); \ WRITE_ONCE(p, RCU_INITIALIZER(v)); \ - } while (0) + ) /** * RCU_POINTER_INITIALIZER() - statically initialize an RCU protected pointer @@ -1206,4 +1233,6 @@ DEFINE_LOCK_GUARD_0(rcu, } while (0), rcu_read_unlock()) +DECLARE_LOCK_GUARD_0_ATTRS(rcu, __acquires_shared(RCU), __releases_shared(RCU)) + #endif /* __LINUX_RCUPDATE_H */ diff --git a/lib/test_context-analysis.c b/lib/test_context-analysis.c index be0c5d462a48..559df32fb5f8 100644 --- a/lib/test_context-analysis.c +++ b/lib/test_context-analysis.c @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -284,3 +285,87 @@ static void __used test_bit_spin_lock(struct test_bit_spinlock_data *d) bit_spin_unlock(3, &d->bits); } } + +/* + * Test that we can mark a variable guarded by RCU, and we can dereference and + * write to the pointer with RCU's primitives. + */ +struct test_rcu_data { + long __rcu_guarded *data; +}; + +static void __used test_rcu_guarded_reader(struct test_rcu_data *d) +{ + rcu_read_lock(); + (void)rcu_dereference(d->data); + rcu_read_unlock(); + + rcu_read_lock_bh(); + (void)rcu_dereference(d->data); + rcu_read_unlock_bh(); + + rcu_read_lock_sched(); + (void)rcu_dereference(d->data); + rcu_read_unlock_sched(); +} + +static void __used test_rcu_guard(struct test_rcu_data *d) +{ + guard(rcu)(); + (void)rcu_dereference(d->data); +} + +static void __used test_rcu_guarded_updater(struct test_rcu_data *d) +{ + rcu_assign_pointer(d->data, NULL); + RCU_INIT_POINTER(d->data, NULL); + (void)unrcu_pointer(d->data); +} + +static void wants_rcu_held(void) __must_hold_shared(RCU) { } +static void wants_rcu_held_bh(void) __must_hold_shared(RCU_BH) { } +static void wants_rcu_held_sched(void) __must_hold_shared(RCU_SCHED) { } + +static void __used test_rcu_lock_variants(void) +{ + rcu_read_lock(); + wants_rcu_held(); + rcu_read_unlock(); + + rcu_read_lock_bh(); + wants_rcu_held_bh(); + rcu_read_unlock_bh(); + + rcu_read_lock_sched(); + wants_rcu_held_sched(); + rcu_read_unlock_sched(); +} + +static void __used test_rcu_lock_reentrant(void) +{ + rcu_read_lock(); + rcu_read_lock(); + rcu_read_lock_bh(); + rcu_read_lock_bh(); + rcu_read_lock_sched(); + rcu_read_lock_sched(); + + rcu_read_unlock_sched(); + rcu_read_unlock_sched(); + rcu_read_unlock_bh(); + rcu_read_unlock_bh(); + rcu_read_unlock(); + rcu_read_unlock(); +} + +static void __used test_rcu_assert_variants(void) +{ + lockdep_assert_in_rcu_read_lock(); + wants_rcu_held(); + + lockdep_assert_in_rcu_read_lock_bh(); + wants_rcu_held_bh(); + + lockdep_assert_in_rcu_read_lock_sched(); + wants_rcu_held_sched(); +} From f0b7ce22d71810c8c11abcd912fbd6f57c2e9677 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:04 +0100 Subject: [PATCH 18/90] srcu: Support Clang's context analysis Add support for Clang's context analysis for SRCU. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Acked-by: Paul E. McKenney Link: https://patch.msgid.link/20251219154418.3592607-16-elver@google.com --- Documentation/dev-tools/context-analysis.rst | 2 +- include/linux/srcu.h | 73 ++++++++++++++------ include/linux/srcutiny.h | 6 ++ include/linux/srcutree.h | 10 ++- lib/test_context-analysis.c | 25 +++++++ 5 files changed, 91 insertions(+), 25 deletions(-) diff --git a/Documentation/dev-tools/context-analysis.rst b/Documentation/dev-tools/context-analysis.rst index 3bc72f71fe25..f7736f1c0767 100644 --- a/Documentation/dev-tools/context-analysis.rst +++ b/Documentation/dev-tools/context-analysis.rst @@ -80,7 +80,7 @@ Supported Kernel Primitives Currently the following synchronization primitives are supported: `raw_spinlock_t`, `spinlock_t`, `rwlock_t`, `mutex`, `seqlock_t`, -`bit_spinlock`, RCU. +`bit_spinlock`, RCU, SRCU (`srcu_struct`). For context locks with an initialization function (e.g., `spin_lock_init()`), calling this function before initializing any guarded members or globals diff --git a/include/linux/srcu.h b/include/linux/srcu.h index 344ad51c8f6c..bb44a0bd7696 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h @@ -21,7 +21,7 @@ #include #include -struct srcu_struct; +context_lock_struct(srcu_struct, __reentrant_ctx_lock); #ifdef CONFIG_DEBUG_LOCK_ALLOC @@ -77,7 +77,7 @@ int init_srcu_struct_fast_updown(struct srcu_struct *ssp); #define SRCU_READ_FLAVOR_SLOWGP (SRCU_READ_FLAVOR_FAST | SRCU_READ_FLAVOR_FAST_UPDOWN) // Flavors requiring synchronize_rcu() // instead of smp_mb(). -void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases(ssp); +void __srcu_read_unlock(struct srcu_struct *ssp, int idx) __releases_shared(ssp); #ifdef CONFIG_TINY_SRCU #include @@ -131,14 +131,16 @@ static inline bool same_state_synchronize_srcu(unsigned long oldstate1, unsigned } #ifdef CONFIG_NEED_SRCU_NMI_SAFE -int __srcu_read_lock_nmisafe(struct srcu_struct *ssp) __acquires(ssp); -void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) __releases(ssp); +int __srcu_read_lock_nmisafe(struct srcu_struct *ssp) __acquires_shared(ssp); +void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) __releases_shared(ssp); #else static inline int __srcu_read_lock_nmisafe(struct srcu_struct *ssp) + __acquires_shared(ssp) { return __srcu_read_lock(ssp); } static inline void __srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) + __releases_shared(ssp) { __srcu_read_unlock(ssp, idx); } @@ -210,6 +212,14 @@ static inline int srcu_read_lock_held(const struct srcu_struct *ssp) #endif /* #else #ifdef CONFIG_DEBUG_LOCK_ALLOC */ +/* + * No-op helper to denote that ssp must be held. Because SRCU-protected pointers + * should still be marked with __rcu_guarded, and we do not want to mark them + * with __guarded_by(ssp) as it would complicate annotations for writers, we + * choose the following strategy: srcu_dereference_check() calls this helper + * that checks that the passed ssp is held, and then fake-acquires 'RCU'. + */ +static inline void __srcu_read_lock_must_hold(const struct srcu_struct *ssp) __must_hold_shared(ssp) { } /** * srcu_dereference_check - fetch SRCU-protected pointer for later dereferencing @@ -223,9 +233,15 @@ static inline int srcu_read_lock_held(const struct srcu_struct *ssp) * to 1. The @c argument will normally be a logical expression containing * lockdep_is_held() calls. */ -#define srcu_dereference_check(p, ssp, c) \ - __rcu_dereference_check((p), __UNIQUE_ID(rcu), \ - (c) || srcu_read_lock_held(ssp), __rcu) +#define srcu_dereference_check(p, ssp, c) \ +({ \ + __srcu_read_lock_must_hold(ssp); \ + __acquire_shared_ctx_lock(RCU); \ + __auto_type __v = __rcu_dereference_check((p), __UNIQUE_ID(rcu), \ + (c) || srcu_read_lock_held(ssp), __rcu); \ + __release_shared_ctx_lock(RCU); \ + __v; \ +}) /** * srcu_dereference - fetch SRCU-protected pointer for later dereferencing @@ -268,7 +284,8 @@ static inline int srcu_read_lock_held(const struct srcu_struct *ssp) * invoke srcu_read_unlock() from one task and the matching srcu_read_lock() * from another. */ -static inline int srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp) +static inline int srcu_read_lock(struct srcu_struct *ssp) + __acquires_shared(ssp) { int retval; @@ -304,7 +321,8 @@ static inline int srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp) * contexts where RCU is watching, that is, from contexts where it would * be legal to invoke rcu_read_lock(). Otherwise, lockdep will complain. */ -static inline struct srcu_ctr __percpu *srcu_read_lock_fast(struct srcu_struct *ssp) __acquires(ssp) +static inline struct srcu_ctr __percpu *srcu_read_lock_fast(struct srcu_struct *ssp) __acquires_shared(ssp) + __acquires_shared(ssp) { struct srcu_ctr __percpu *retval; @@ -344,7 +362,7 @@ static inline struct srcu_ctr __percpu *srcu_read_lock_fast(struct srcu_struct * * complain. */ static inline struct srcu_ctr __percpu *srcu_read_lock_fast_updown(struct srcu_struct *ssp) -__acquires(ssp) + __acquires_shared(ssp) { struct srcu_ctr __percpu *retval; @@ -360,7 +378,7 @@ __acquires(ssp) * See srcu_read_lock_fast() for more information. */ static inline struct srcu_ctr __percpu *srcu_read_lock_fast_notrace(struct srcu_struct *ssp) - __acquires(ssp) + __acquires_shared(ssp) { struct srcu_ctr __percpu *retval; @@ -381,7 +399,7 @@ static inline struct srcu_ctr __percpu *srcu_read_lock_fast_notrace(struct srcu_ * and srcu_read_lock_fast(). However, the same definition/initialization * requirements called out for srcu_read_lock_safe() apply. */ -static inline struct srcu_ctr __percpu *srcu_down_read_fast(struct srcu_struct *ssp) __acquires(ssp) +static inline struct srcu_ctr __percpu *srcu_down_read_fast(struct srcu_struct *ssp) __acquires_shared(ssp) { WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi()); RCU_LOCKDEP_WARN(!rcu_is_watching(), "RCU must be watching srcu_down_read_fast()."); @@ -400,7 +418,8 @@ static inline struct srcu_ctr __percpu *srcu_down_read_fast(struct srcu_struct * * then none of the other flavors may be used, whether before, during, * or after. */ -static inline int srcu_read_lock_nmisafe(struct srcu_struct *ssp) __acquires(ssp) +static inline int srcu_read_lock_nmisafe(struct srcu_struct *ssp) + __acquires_shared(ssp) { int retval; @@ -412,7 +431,8 @@ static inline int srcu_read_lock_nmisafe(struct srcu_struct *ssp) __acquires(ssp /* Used by tracing, cannot be traced and cannot invoke lockdep. */ static inline notrace int -srcu_read_lock_notrace(struct srcu_struct *ssp) __acquires(ssp) +srcu_read_lock_notrace(struct srcu_struct *ssp) + __acquires_shared(ssp) { int retval; @@ -443,7 +463,8 @@ srcu_read_lock_notrace(struct srcu_struct *ssp) __acquires(ssp) * which calls to down_read() may be nested. The same srcu_struct may be * used concurrently by srcu_down_read() and srcu_read_lock(). */ -static inline int srcu_down_read(struct srcu_struct *ssp) __acquires(ssp) +static inline int srcu_down_read(struct srcu_struct *ssp) + __acquires_shared(ssp) { WARN_ON_ONCE(in_nmi()); srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL); @@ -458,7 +479,7 @@ static inline int srcu_down_read(struct srcu_struct *ssp) __acquires(ssp) * Exit an SRCU read-side critical section. */ static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx) - __releases(ssp) + __releases_shared(ssp) { WARN_ON_ONCE(idx & ~0x1); srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL); @@ -474,7 +495,7 @@ static inline void srcu_read_unlock(struct srcu_struct *ssp, int idx) * Exit a light-weight SRCU read-side critical section. */ static inline void srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) - __releases(ssp) + __releases_shared(ssp) { srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST); srcu_lock_release(&ssp->dep_map); @@ -490,7 +511,7 @@ static inline void srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ct * Exit an SRCU-fast-updown read-side critical section. */ static inline void -srcu_read_unlock_fast_updown(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) __releases(ssp) +srcu_read_unlock_fast_updown(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) __releases_shared(ssp) { srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN); srcu_lock_release(&ssp->dep_map); @@ -504,7 +525,7 @@ srcu_read_unlock_fast_updown(struct srcu_struct *ssp, struct srcu_ctr __percpu * * See srcu_read_unlock_fast() for more information. */ static inline void srcu_read_unlock_fast_notrace(struct srcu_struct *ssp, - struct srcu_ctr __percpu *scp) __releases(ssp) + struct srcu_ctr __percpu *scp) __releases_shared(ssp) { srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST); __srcu_read_unlock_fast(ssp, scp); @@ -519,7 +540,7 @@ static inline void srcu_read_unlock_fast_notrace(struct srcu_struct *ssp, * the same context as the maching srcu_down_read_fast(). */ static inline void srcu_up_read_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) - __releases(ssp) + __releases_shared(ssp) { WARN_ON_ONCE(IS_ENABLED(CONFIG_PROVE_RCU) && in_nmi()); srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_FAST_UPDOWN); @@ -535,7 +556,7 @@ static inline void srcu_up_read_fast(struct srcu_struct *ssp, struct srcu_ctr __ * Exit an SRCU read-side critical section, but in an NMI-safe manner. */ static inline void srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) - __releases(ssp) + __releases_shared(ssp) { WARN_ON_ONCE(idx & ~0x1); srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NMI); @@ -545,7 +566,7 @@ static inline void srcu_read_unlock_nmisafe(struct srcu_struct *ssp, int idx) /* Used by tracing, cannot be traced and cannot call lockdep. */ static inline notrace void -srcu_read_unlock_notrace(struct srcu_struct *ssp, int idx) __releases(ssp) +srcu_read_unlock_notrace(struct srcu_struct *ssp, int idx) __releases_shared(ssp) { srcu_check_read_flavor(ssp, SRCU_READ_FLAVOR_NORMAL); __srcu_read_unlock(ssp, idx); @@ -560,7 +581,7 @@ srcu_read_unlock_notrace(struct srcu_struct *ssp, int idx) __releases(ssp) * the same context as the maching srcu_down_read(). */ static inline void srcu_up_read(struct srcu_struct *ssp, int idx) - __releases(ssp) + __releases_shared(ssp) { WARN_ON_ONCE(idx & ~0x1); WARN_ON_ONCE(in_nmi()); @@ -600,15 +621,21 @@ DEFINE_LOCK_GUARD_1(srcu, struct srcu_struct, _T->idx = srcu_read_lock(_T->lock), srcu_read_unlock(_T->lock, _T->idx), int idx) +DECLARE_LOCK_GUARD_1_ATTRS(srcu, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T)) +#define class_srcu_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu, _T) DEFINE_LOCK_GUARD_1(srcu_fast, struct srcu_struct, _T->scp = srcu_read_lock_fast(_T->lock), srcu_read_unlock_fast(_T->lock, _T->scp), struct srcu_ctr __percpu *scp) +DECLARE_LOCK_GUARD_1_ATTRS(srcu_fast, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T)) +#define class_srcu_fast_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu_fast, _T) DEFINE_LOCK_GUARD_1(srcu_fast_notrace, struct srcu_struct, _T->scp = srcu_read_lock_fast_notrace(_T->lock), srcu_read_unlock_fast_notrace(_T->lock, _T->scp), struct srcu_ctr __percpu *scp) +DECLARE_LOCK_GUARD_1_ATTRS(srcu_fast_notrace, __acquires_shared(_T), __releases_shared(*(struct srcu_struct **)_T)) +#define class_srcu_fast_notrace_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(srcu_fast_notrace, _T) #endif diff --git a/include/linux/srcutiny.h b/include/linux/srcutiny.h index e0698024667a..dec7cbe015aa 100644 --- a/include/linux/srcutiny.h +++ b/include/linux/srcutiny.h @@ -73,6 +73,7 @@ void synchronize_srcu(struct srcu_struct *ssp); * index that must be passed to the matching srcu_read_unlock(). */ static inline int __srcu_read_lock(struct srcu_struct *ssp) + __acquires_shared(ssp) { int idx; @@ -80,6 +81,7 @@ static inline int __srcu_read_lock(struct srcu_struct *ssp) idx = ((READ_ONCE(ssp->srcu_idx) + 1) & 0x2) >> 1; WRITE_ONCE(ssp->srcu_lock_nesting[idx], READ_ONCE(ssp->srcu_lock_nesting[idx]) + 1); preempt_enable(); + __acquire_shared(ssp); return idx; } @@ -96,22 +98,26 @@ static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr(struct srcu_struct *ss } static inline struct srcu_ctr __percpu *__srcu_read_lock_fast(struct srcu_struct *ssp) + __acquires_shared(ssp) { return __srcu_ctr_to_ptr(ssp, __srcu_read_lock(ssp)); } static inline void __srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) + __releases_shared(ssp) { __srcu_read_unlock(ssp, __srcu_ptr_to_ctr(ssp, scp)); } static inline struct srcu_ctr __percpu *__srcu_read_lock_fast_updown(struct srcu_struct *ssp) + __acquires_shared(ssp) { return __srcu_ctr_to_ptr(ssp, __srcu_read_lock(ssp)); } static inline void __srcu_read_unlock_fast_updown(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) + __releases_shared(ssp) { __srcu_read_unlock(ssp, __srcu_ptr_to_ctr(ssp, scp)); } diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h index d6f978b50472..958cb7ef41cb 100644 --- a/include/linux/srcutree.h +++ b/include/linux/srcutree.h @@ -233,7 +233,7 @@ struct srcu_struct { #define DEFINE_STATIC_SRCU_FAST_UPDOWN(name) \ __DEFINE_SRCU(name, SRCU_READ_FLAVOR_FAST_UPDOWN, static) -int __srcu_read_lock(struct srcu_struct *ssp) __acquires(ssp); +int __srcu_read_lock(struct srcu_struct *ssp) __acquires_shared(ssp); void synchronize_srcu_expedited(struct srcu_struct *ssp); void srcu_barrier(struct srcu_struct *ssp); void srcu_expedite_current(struct srcu_struct *ssp); @@ -286,6 +286,7 @@ static inline struct srcu_ctr __percpu *__srcu_ctr_to_ptr(struct srcu_struct *ss * implementations of this_cpu_inc(). */ static inline struct srcu_ctr __percpu notrace *__srcu_read_lock_fast(struct srcu_struct *ssp) + __acquires_shared(ssp) { struct srcu_ctr __percpu *scp = READ_ONCE(ssp->srcu_ctrp); @@ -294,6 +295,7 @@ static inline struct srcu_ctr __percpu notrace *__srcu_read_lock_fast(struct src else atomic_long_inc(raw_cpu_ptr(&scp->srcu_locks)); // Y, and implicit RCU reader. barrier(); /* Avoid leaking the critical section. */ + __acquire_shared(ssp); return scp; } @@ -308,7 +310,9 @@ static inline struct srcu_ctr __percpu notrace *__srcu_read_lock_fast(struct src */ static inline void notrace __srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) + __releases_shared(ssp) { + __release_shared(ssp); barrier(); /* Avoid leaking the critical section. */ if (!IS_ENABLED(CONFIG_NEED_SRCU_NMI_SAFE)) this_cpu_inc(scp->srcu_unlocks.counter); // Z, and implicit RCU reader. @@ -326,6 +330,7 @@ __srcu_read_unlock_fast(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) */ static inline struct srcu_ctr __percpu notrace *__srcu_read_lock_fast_updown(struct srcu_struct *ssp) + __acquires_shared(ssp) { struct srcu_ctr __percpu *scp = READ_ONCE(ssp->srcu_ctrp); @@ -334,6 +339,7 @@ struct srcu_ctr __percpu notrace *__srcu_read_lock_fast_updown(struct srcu_struc else atomic_long_inc(raw_cpu_ptr(&scp->srcu_locks)); // Y, and implicit RCU reader. barrier(); /* Avoid leaking the critical section. */ + __acquire_shared(ssp); return scp; } @@ -348,7 +354,9 @@ struct srcu_ctr __percpu notrace *__srcu_read_lock_fast_updown(struct srcu_struc */ static inline void notrace __srcu_read_unlock_fast_updown(struct srcu_struct *ssp, struct srcu_ctr __percpu *scp) + __releases_shared(ssp) { + __release_shared(ssp); barrier(); /* Avoid leaking the critical section. */ if (!IS_ENABLED(CONFIG_NEED_SRCU_NMI_SAFE)) this_cpu_inc(scp->srcu_unlocks.counter); // Z, and implicit RCU reader. diff --git a/lib/test_context-analysis.c b/lib/test_context-analysis.c index 559df32fb5f8..39e03790c0f6 100644 --- a/lib/test_context-analysis.c +++ b/lib/test_context-analysis.c @@ -10,6 +10,7 @@ #include #include #include +#include /* * Test that helper macros work as expected. @@ -369,3 +370,27 @@ static void __used test_rcu_assert_variants(void) lockdep_assert_in_rcu_read_lock_sched(); wants_rcu_held_sched(); } + +struct test_srcu_data { + struct srcu_struct srcu; + long __rcu_guarded *data; +}; + +static void __used test_srcu(struct test_srcu_data *d) +{ + init_srcu_struct(&d->srcu); + + int idx = srcu_read_lock(&d->srcu); + long *data = srcu_dereference(d->data, &d->srcu); + (void)data; + srcu_read_unlock(&d->srcu, idx); + + rcu_assign_pointer(d->data, NULL); +} + +static void __used test_srcu_guard(struct test_srcu_data *d) +{ + { guard(srcu)(&d->srcu); (void)srcu_dereference(d->data, &d->srcu); } + { guard(srcu_fast)(&d->srcu); (void)srcu_dereference(d->data, &d->srcu); } + { guard(srcu_fast_notrace)(&d->srcu); (void)srcu_dereference(d->data, &d->srcu); } +} From 5e256db9325e75e9f000ddd64e4f1dbd2a6d8acd Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:05 +0100 Subject: [PATCH 19/90] kref: Add context-analysis annotations Mark functions that conditionally acquire the passed lock. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Bart Van Assche Link: https://patch.msgid.link/20251219154418.3592607-17-elver@google.com --- include/linux/kref.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/linux/kref.h b/include/linux/kref.h index 88e82ab1367c..9bc6abe57572 100644 --- a/include/linux/kref.h +++ b/include/linux/kref.h @@ -81,6 +81,7 @@ static inline int kref_put(struct kref *kref, void (*release)(struct kref *kref) static inline int kref_put_mutex(struct kref *kref, void (*release)(struct kref *kref), struct mutex *mutex) + __cond_acquires(true, mutex) { if (refcount_dec_and_mutex_lock(&kref->refcount, mutex)) { release(kref); @@ -102,6 +103,7 @@ static inline int kref_put_mutex(struct kref *kref, static inline int kref_put_lock(struct kref *kref, void (*release)(struct kref *kref), spinlock_t *lock) + __cond_acquires(true, lock) { if (refcount_dec_and_lock(&kref->refcount, lock)) { release(kref); From e4fd3be884cf33a42c5bcde087b0722a5b8f25ca Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:06 +0100 Subject: [PATCH 20/90] locking/rwsem: Support Clang's context analysis Add support for Clang's context analysis for rw_semaphore. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-18-elver@google.com --- Documentation/dev-tools/context-analysis.rst | 2 +- include/linux/rwsem.h | 72 +++++++++++++------- lib/test_context-analysis.c | 64 +++++++++++++++++ 3 files changed, 112 insertions(+), 26 deletions(-) diff --git a/Documentation/dev-tools/context-analysis.rst b/Documentation/dev-tools/context-analysis.rst index f7736f1c0767..7b660c3003a0 100644 --- a/Documentation/dev-tools/context-analysis.rst +++ b/Documentation/dev-tools/context-analysis.rst @@ -80,7 +80,7 @@ Supported Kernel Primitives Currently the following synchronization primitives are supported: `raw_spinlock_t`, `spinlock_t`, `rwlock_t`, `mutex`, `seqlock_t`, -`bit_spinlock`, RCU, SRCU (`srcu_struct`). +`bit_spinlock`, RCU, SRCU (`srcu_struct`), `rw_semaphore`. For context locks with an initialization function (e.g., `spin_lock_init()`), calling this function before initializing any guarded members or globals diff --git a/include/linux/rwsem.h b/include/linux/rwsem.h index f1aaf676a874..8da14a08a4e1 100644 --- a/include/linux/rwsem.h +++ b/include/linux/rwsem.h @@ -45,7 +45,7 @@ * reduce the chance that they will share the same cacheline causing * cacheline bouncing problem. */ -struct rw_semaphore { +context_lock_struct(rw_semaphore) { atomic_long_t count; /* * Write owner or one of the read owners as well flags regarding @@ -76,11 +76,13 @@ static inline int rwsem_is_locked(struct rw_semaphore *sem) } static inline void rwsem_assert_held_nolockdep(const struct rw_semaphore *sem) + __assumes_ctx_lock(sem) { WARN_ON(atomic_long_read(&sem->count) == RWSEM_UNLOCKED_VALUE); } static inline void rwsem_assert_held_write_nolockdep(const struct rw_semaphore *sem) + __assumes_ctx_lock(sem) { WARN_ON(!(atomic_long_read(&sem->count) & RWSEM_WRITER_LOCKED)); } @@ -119,6 +121,7 @@ do { \ static struct lock_class_key __key; \ \ __init_rwsem((sem), #sem, &__key); \ + __assume_ctx_lock(sem); \ } while (0) /* @@ -148,7 +151,7 @@ extern bool is_rwsem_reader_owned(struct rw_semaphore *sem); #include -struct rw_semaphore { +context_lock_struct(rw_semaphore) { struct rwbase_rt rwbase; #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map dep_map; @@ -172,6 +175,7 @@ do { \ static struct lock_class_key __key; \ \ __init_rwsem((sem), #sem, &__key); \ + __assume_ctx_lock(sem); \ } while (0) static __always_inline int rwsem_is_locked(const struct rw_semaphore *sem) @@ -180,11 +184,13 @@ static __always_inline int rwsem_is_locked(const struct rw_semaphore *sem) } static __always_inline void rwsem_assert_held_nolockdep(const struct rw_semaphore *sem) + __assumes_ctx_lock(sem) { WARN_ON(!rwsem_is_locked(sem)); } static __always_inline void rwsem_assert_held_write_nolockdep(const struct rw_semaphore *sem) + __assumes_ctx_lock(sem) { WARN_ON(!rw_base_is_write_locked(&sem->rwbase)); } @@ -202,6 +208,7 @@ static __always_inline int rwsem_is_contended(struct rw_semaphore *sem) */ static inline void rwsem_assert_held(const struct rw_semaphore *sem) + __assumes_ctx_lock(sem) { if (IS_ENABLED(CONFIG_LOCKDEP)) lockdep_assert_held(sem); @@ -210,6 +217,7 @@ static inline void rwsem_assert_held(const struct rw_semaphore *sem) } static inline void rwsem_assert_held_write(const struct rw_semaphore *sem) + __assumes_ctx_lock(sem) { if (IS_ENABLED(CONFIG_LOCKDEP)) lockdep_assert_held_write(sem); @@ -220,48 +228,62 @@ static inline void rwsem_assert_held_write(const struct rw_semaphore *sem) /* * lock for reading */ -extern void down_read(struct rw_semaphore *sem); -extern int __must_check down_read_interruptible(struct rw_semaphore *sem); -extern int __must_check down_read_killable(struct rw_semaphore *sem); +extern void down_read(struct rw_semaphore *sem) __acquires_shared(sem); +extern int __must_check down_read_interruptible(struct rw_semaphore *sem) __cond_acquires_shared(0, sem); +extern int __must_check down_read_killable(struct rw_semaphore *sem) __cond_acquires_shared(0, sem); /* * trylock for reading -- returns 1 if successful, 0 if contention */ -extern int down_read_trylock(struct rw_semaphore *sem); +extern int down_read_trylock(struct rw_semaphore *sem) __cond_acquires_shared(true, sem); /* * lock for writing */ -extern void down_write(struct rw_semaphore *sem); -extern int __must_check down_write_killable(struct rw_semaphore *sem); +extern void down_write(struct rw_semaphore *sem) __acquires(sem); +extern int __must_check down_write_killable(struct rw_semaphore *sem) __cond_acquires(0, sem); /* * trylock for writing -- returns 1 if successful, 0 if contention */ -extern int down_write_trylock(struct rw_semaphore *sem); +extern int down_write_trylock(struct rw_semaphore *sem) __cond_acquires(true, sem); /* * release a read lock */ -extern void up_read(struct rw_semaphore *sem); +extern void up_read(struct rw_semaphore *sem) __releases_shared(sem); /* * release a write lock */ -extern void up_write(struct rw_semaphore *sem); +extern void up_write(struct rw_semaphore *sem) __releases(sem); -DEFINE_GUARD(rwsem_read, struct rw_semaphore *, down_read(_T), up_read(_T)) -DEFINE_GUARD_COND(rwsem_read, _try, down_read_trylock(_T)) -DEFINE_GUARD_COND(rwsem_read, _intr, down_read_interruptible(_T), _RET == 0) +DEFINE_LOCK_GUARD_1(rwsem_read, struct rw_semaphore, down_read(_T->lock), up_read(_T->lock)) +DEFINE_LOCK_GUARD_1_COND(rwsem_read, _try, down_read_trylock(_T->lock)) +DEFINE_LOCK_GUARD_1_COND(rwsem_read, _intr, down_read_interruptible(_T->lock), _RET == 0) -DEFINE_GUARD(rwsem_write, struct rw_semaphore *, down_write(_T), up_write(_T)) -DEFINE_GUARD_COND(rwsem_write, _try, down_write_trylock(_T)) -DEFINE_GUARD_COND(rwsem_write, _kill, down_write_killable(_T), _RET == 0) +DECLARE_LOCK_GUARD_1_ATTRS(rwsem_read, __acquires_shared(_T), __releases_shared(*(struct rw_semaphore **)_T)) +#define class_rwsem_read_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(rwsem_read, _T) +DECLARE_LOCK_GUARD_1_ATTRS(rwsem_read_try, __acquires_shared(_T), __releases_shared(*(struct rw_semaphore **)_T)) +#define class_rwsem_read_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(rwsem_read_try, _T) +DECLARE_LOCK_GUARD_1_ATTRS(rwsem_read_intr, __acquires_shared(_T), __releases_shared(*(struct rw_semaphore **)_T)) +#define class_rwsem_read_intr_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(rwsem_read_intr, _T) + +DEFINE_LOCK_GUARD_1(rwsem_write, struct rw_semaphore, down_write(_T->lock), up_write(_T->lock)) +DEFINE_LOCK_GUARD_1_COND(rwsem_write, _try, down_write_trylock(_T->lock)) +DEFINE_LOCK_GUARD_1_COND(rwsem_write, _kill, down_write_killable(_T->lock), _RET == 0) + +DECLARE_LOCK_GUARD_1_ATTRS(rwsem_write, __acquires(_T), __releases(*(struct rw_semaphore **)_T)) +#define class_rwsem_write_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(rwsem_write, _T) +DECLARE_LOCK_GUARD_1_ATTRS(rwsem_write_try, __acquires(_T), __releases(*(struct rw_semaphore **)_T)) +#define class_rwsem_write_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(rwsem_write_try, _T) +DECLARE_LOCK_GUARD_1_ATTRS(rwsem_write_kill, __acquires(_T), __releases(*(struct rw_semaphore **)_T)) +#define class_rwsem_write_kill_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(rwsem_write_kill, _T) /* * downgrade write lock to read lock */ -extern void downgrade_write(struct rw_semaphore *sem); +extern void downgrade_write(struct rw_semaphore *sem) __releases(sem) __acquires_shared(sem); #ifdef CONFIG_DEBUG_LOCK_ALLOC /* @@ -277,11 +299,11 @@ extern void downgrade_write(struct rw_semaphore *sem); * lockdep_set_class() at lock initialization time. * See Documentation/locking/lockdep-design.rst for more details.) */ -extern void down_read_nested(struct rw_semaphore *sem, int subclass); -extern int __must_check down_read_killable_nested(struct rw_semaphore *sem, int subclass); -extern void down_write_nested(struct rw_semaphore *sem, int subclass); -extern int down_write_killable_nested(struct rw_semaphore *sem, int subclass); -extern void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest_lock); +extern void down_read_nested(struct rw_semaphore *sem, int subclass) __acquires_shared(sem); +extern int __must_check down_read_killable_nested(struct rw_semaphore *sem, int subclass) __cond_acquires_shared(0, sem); +extern void down_write_nested(struct rw_semaphore *sem, int subclass) __acquires(sem); +extern int down_write_killable_nested(struct rw_semaphore *sem, int subclass) __cond_acquires(0, sem); +extern void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest_lock) __acquires(sem); # define down_write_nest_lock(sem, nest_lock) \ do { \ @@ -295,8 +317,8 @@ do { \ * [ This API should be avoided as much as possible - the * proper abstraction for this case is completions. ] */ -extern void down_read_non_owner(struct rw_semaphore *sem); -extern void up_read_non_owner(struct rw_semaphore *sem); +extern void down_read_non_owner(struct rw_semaphore *sem) __acquires_shared(sem); +extern void up_read_non_owner(struct rw_semaphore *sem) __releases_shared(sem); #else # define down_read_nested(sem, subclass) down_read(sem) # define down_read_killable_nested(sem, subclass) down_read_killable(sem) diff --git a/lib/test_context-analysis.c b/lib/test_context-analysis.c index 39e03790c0f6..1c96c56cf873 100644 --- a/lib/test_context-analysis.c +++ b/lib/test_context-analysis.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -262,6 +263,69 @@ static void __used test_seqlock_scoped(struct test_seqlock_data *d) } } +struct test_rwsem_data { + struct rw_semaphore sem; + int counter __guarded_by(&sem); +}; + +static void __used test_rwsem_init(struct test_rwsem_data *d) +{ + init_rwsem(&d->sem); + d->counter = 0; +} + +static void __used test_rwsem_reader(struct test_rwsem_data *d) +{ + down_read(&d->sem); + (void)d->counter; + up_read(&d->sem); + + if (down_read_trylock(&d->sem)) { + (void)d->counter; + up_read(&d->sem); + } +} + +static void __used test_rwsem_writer(struct test_rwsem_data *d) +{ + down_write(&d->sem); + d->counter++; + up_write(&d->sem); + + down_write(&d->sem); + d->counter++; + downgrade_write(&d->sem); + (void)d->counter; + up_read(&d->sem); + + if (down_write_trylock(&d->sem)) { + d->counter++; + up_write(&d->sem); + } +} + +static void __used test_rwsem_assert(struct test_rwsem_data *d) +{ + rwsem_assert_held_nolockdep(&d->sem); + d->counter++; +} + +static void __used test_rwsem_guard(struct test_rwsem_data *d) +{ + { guard(rwsem_read)(&d->sem); (void)d->counter; } + { guard(rwsem_write)(&d->sem); d->counter++; } +} + +static void __used test_rwsem_cond_guard(struct test_rwsem_data *d) +{ + scoped_cond_guard(rwsem_read_try, return, &d->sem) { + (void)d->counter; + } + scoped_cond_guard(rwsem_write_try, return, &d->sem) { + d->counter++; + } +} + struct test_bit_spinlock_data { unsigned long bits; int counter __guarded_by(__bitlock(3, &bits)); From 8c9c8566e139c0f1398245fbe3aa409fc1a79da8 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:07 +0100 Subject: [PATCH 21/90] locking/local_lock: Include missing headers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Including into an empty TU will result in the compiler complaining: ./include/linux/local_lock.h: In function ‘class_local_lock_irqsave_constructor’: ./include/linux/local_lock_internal.h:95:17: error: implicit declaration of function ‘local_irq_save’; <...> 95 | local_irq_save(flags); \ | ^~~~~~~~~~~~~~ As well as (some architectures only, such as 'sh'): ./include/linux/local_lock_internal.h: In function ‘local_lock_acquire’: ./include/linux/local_lock_internal.h:33:20: error: ‘current’ undeclared (first use in this function) 33 | l->owner = current; Include missing headers to allow including local_lock.h where the required headers are not otherwise included. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Bart Van Assche Link: https://patch.msgid.link/20251219154418.3592607-19-elver@google.com --- include/linux/local_lock_internal.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/linux/local_lock_internal.h b/include/linux/local_lock_internal.h index 8f82b4eb542f..1a1ea1232add 100644 --- a/include/linux/local_lock_internal.h +++ b/include/linux/local_lock_internal.h @@ -4,7 +4,9 @@ #endif #include +#include #include +#include #ifndef CONFIG_PREEMPT_RT From d3febf16dee28a74b01ba43195ee4965edb6208f Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:08 +0100 Subject: [PATCH 22/90] locking/local_lock: Support Clang's context analysis Add support for Clang's context analysis for local_lock_t and local_trylock_t. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-20-elver@google.com --- Documentation/dev-tools/context-analysis.rst | 2 +- include/linux/local_lock.h | 47 ++++++++----- include/linux/local_lock_internal.h | 71 +++++++++++++++---- lib/test_context-analysis.c | 73 ++++++++++++++++++++ 4 files changed, 159 insertions(+), 34 deletions(-) diff --git a/Documentation/dev-tools/context-analysis.rst b/Documentation/dev-tools/context-analysis.rst index 7b660c3003a0..a48b75f45e79 100644 --- a/Documentation/dev-tools/context-analysis.rst +++ b/Documentation/dev-tools/context-analysis.rst @@ -80,7 +80,7 @@ Supported Kernel Primitives Currently the following synchronization primitives are supported: `raw_spinlock_t`, `spinlock_t`, `rwlock_t`, `mutex`, `seqlock_t`, -`bit_spinlock`, RCU, SRCU (`srcu_struct`), `rw_semaphore`. +`bit_spinlock`, RCU, SRCU (`srcu_struct`), `rw_semaphore`, `local_lock_t`. For context locks with an initialization function (e.g., `spin_lock_init()`), calling this function before initializing any guarded members or globals diff --git a/include/linux/local_lock.h b/include/linux/local_lock.h index b0e6ab329b00..99c06e499375 100644 --- a/include/linux/local_lock.h +++ b/include/linux/local_lock.h @@ -14,13 +14,13 @@ * local_lock - Acquire a per CPU local lock * @lock: The lock variable */ -#define local_lock(lock) __local_lock(this_cpu_ptr(lock)) +#define local_lock(lock) __local_lock(__this_cpu_local_lock(lock)) /** * local_lock_irq - Acquire a per CPU local lock and disable interrupts * @lock: The lock variable */ -#define local_lock_irq(lock) __local_lock_irq(this_cpu_ptr(lock)) +#define local_lock_irq(lock) __local_lock_irq(__this_cpu_local_lock(lock)) /** * local_lock_irqsave - Acquire a per CPU local lock, save and disable @@ -29,19 +29,19 @@ * @flags: Storage for interrupt flags */ #define local_lock_irqsave(lock, flags) \ - __local_lock_irqsave(this_cpu_ptr(lock), flags) + __local_lock_irqsave(__this_cpu_local_lock(lock), flags) /** * local_unlock - Release a per CPU local lock * @lock: The lock variable */ -#define local_unlock(lock) __local_unlock(this_cpu_ptr(lock)) +#define local_unlock(lock) __local_unlock(__this_cpu_local_lock(lock)) /** * local_unlock_irq - Release a per CPU local lock and enable interrupts * @lock: The lock variable */ -#define local_unlock_irq(lock) __local_unlock_irq(this_cpu_ptr(lock)) +#define local_unlock_irq(lock) __local_unlock_irq(__this_cpu_local_lock(lock)) /** * local_unlock_irqrestore - Release a per CPU local lock and restore @@ -50,7 +50,7 @@ * @flags: Interrupt flags to restore */ #define local_unlock_irqrestore(lock, flags) \ - __local_unlock_irqrestore(this_cpu_ptr(lock), flags) + __local_unlock_irqrestore(__this_cpu_local_lock(lock), flags) /** * local_trylock_init - Runtime initialize a lock instance @@ -66,7 +66,7 @@ * locking constrains it will _always_ fail to acquire the lock in NMI or * HARDIRQ context on PREEMPT_RT. */ -#define local_trylock(lock) __local_trylock(this_cpu_ptr(lock)) +#define local_trylock(lock) __local_trylock(__this_cpu_local_lock(lock)) #define local_lock_is_locked(lock) __local_lock_is_locked(lock) @@ -81,27 +81,36 @@ * HARDIRQ context on PREEMPT_RT. */ #define local_trylock_irqsave(lock, flags) \ - __local_trylock_irqsave(this_cpu_ptr(lock), flags) + __local_trylock_irqsave(__this_cpu_local_lock(lock), flags) -DEFINE_GUARD(local_lock, local_lock_t __percpu*, - local_lock(_T), - local_unlock(_T)) -DEFINE_GUARD(local_lock_irq, local_lock_t __percpu*, - local_lock_irq(_T), - local_unlock_irq(_T)) +DEFINE_LOCK_GUARD_1(local_lock, local_lock_t __percpu, + local_lock(_T->lock), + local_unlock(_T->lock)) +DEFINE_LOCK_GUARD_1(local_lock_irq, local_lock_t __percpu, + local_lock_irq(_T->lock), + local_unlock_irq(_T->lock)) DEFINE_LOCK_GUARD_1(local_lock_irqsave, local_lock_t __percpu, local_lock_irqsave(_T->lock, _T->flags), local_unlock_irqrestore(_T->lock, _T->flags), unsigned long flags) #define local_lock_nested_bh(_lock) \ - __local_lock_nested_bh(this_cpu_ptr(_lock)) + __local_lock_nested_bh(__this_cpu_local_lock(_lock)) #define local_unlock_nested_bh(_lock) \ - __local_unlock_nested_bh(this_cpu_ptr(_lock)) + __local_unlock_nested_bh(__this_cpu_local_lock(_lock)) -DEFINE_GUARD(local_lock_nested_bh, local_lock_t __percpu*, - local_lock_nested_bh(_T), - local_unlock_nested_bh(_T)) +DEFINE_LOCK_GUARD_1(local_lock_nested_bh, local_lock_t __percpu, + local_lock_nested_bh(_T->lock), + local_unlock_nested_bh(_T->lock)) + +DECLARE_LOCK_GUARD_1_ATTRS(local_lock, __acquires(_T), __releases(*(local_lock_t __percpu **)_T)) +#define class_local_lock_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(local_lock, _T) +DECLARE_LOCK_GUARD_1_ATTRS(local_lock_irq, __acquires(_T), __releases(*(local_lock_t __percpu **)_T)) +#define class_local_lock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(local_lock_irq, _T) +DECLARE_LOCK_GUARD_1_ATTRS(local_lock_irqsave, __acquires(_T), __releases(*(local_lock_t __percpu **)_T)) +#define class_local_lock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(local_lock_irqsave, _T) +DECLARE_LOCK_GUARD_1_ATTRS(local_lock_nested_bh, __acquires(_T), __releases(*(local_lock_t __percpu **)_T)) +#define class_local_lock_nested_bh_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(local_lock_nested_bh, _T) #endif diff --git a/include/linux/local_lock_internal.h b/include/linux/local_lock_internal.h index 1a1ea1232add..e8c4803d8db4 100644 --- a/include/linux/local_lock_internal.h +++ b/include/linux/local_lock_internal.h @@ -10,21 +10,23 @@ #ifndef CONFIG_PREEMPT_RT -typedef struct { +context_lock_struct(local_lock) { #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map dep_map; struct task_struct *owner; #endif -} local_lock_t; +}; +typedef struct local_lock local_lock_t; /* local_trylock() and local_trylock_irqsave() only work with local_trylock_t */ -typedef struct { +context_lock_struct(local_trylock) { #ifdef CONFIG_DEBUG_LOCK_ALLOC struct lockdep_map dep_map; struct task_struct *owner; #endif u8 acquired; -} local_trylock_t; +}; +typedef struct local_trylock local_trylock_t; #ifdef CONFIG_DEBUG_LOCK_ALLOC # define LOCAL_LOCK_DEBUG_INIT(lockname) \ @@ -84,9 +86,14 @@ do { \ 0, LD_WAIT_CONFIG, LD_WAIT_INV, \ LD_LOCK_PERCPU); \ local_lock_debug_init(lock); \ + __assume_ctx_lock(lock); \ } while (0) -#define __local_trylock_init(lock) __local_lock_init((local_lock_t *)lock) +#define __local_trylock_init(lock) \ +do { \ + __local_lock_init((local_lock_t *)lock); \ + __assume_ctx_lock(lock); \ +} while (0) #define __spinlock_nested_bh_init(lock) \ do { \ @@ -97,6 +104,7 @@ do { \ 0, LD_WAIT_CONFIG, LD_WAIT_INV, \ LD_LOCK_NORMAL); \ local_lock_debug_init(lock); \ + __assume_ctx_lock(lock); \ } while (0) #define __local_lock_acquire(lock) \ @@ -119,22 +127,25 @@ do { \ do { \ preempt_disable(); \ __local_lock_acquire(lock); \ + __acquire(lock); \ } while (0) #define __local_lock_irq(lock) \ do { \ local_irq_disable(); \ __local_lock_acquire(lock); \ + __acquire(lock); \ } while (0) #define __local_lock_irqsave(lock, flags) \ do { \ local_irq_save(flags); \ __local_lock_acquire(lock); \ + __acquire(lock); \ } while (0) #define __local_trylock(lock) \ - ({ \ + __try_acquire_ctx_lock(lock, ({ \ local_trylock_t *__tl; \ \ preempt_disable(); \ @@ -148,10 +159,10 @@ do { \ (local_lock_t *)__tl); \ } \ !!__tl; \ - }) + })) #define __local_trylock_irqsave(lock, flags) \ - ({ \ + __try_acquire_ctx_lock(lock, ({ \ local_trylock_t *__tl; \ \ local_irq_save(flags); \ @@ -165,7 +176,7 @@ do { \ (local_lock_t *)__tl); \ } \ !!__tl; \ - }) + })) /* preemption or migration must be disabled before calling __local_lock_is_locked */ #define __local_lock_is_locked(lock) READ_ONCE(this_cpu_ptr(lock)->acquired) @@ -188,18 +199,21 @@ do { \ #define __local_unlock(lock) \ do { \ + __release(lock); \ __local_lock_release(lock); \ preempt_enable(); \ } while (0) #define __local_unlock_irq(lock) \ do { \ + __release(lock); \ __local_lock_release(lock); \ local_irq_enable(); \ } while (0) #define __local_unlock_irqrestore(lock, flags) \ do { \ + __release(lock); \ __local_lock_release(lock); \ local_irq_restore(flags); \ } while (0) @@ -208,13 +222,19 @@ do { \ do { \ lockdep_assert_in_softirq(); \ local_lock_acquire((lock)); \ + __acquire(lock); \ } while (0) #define __local_unlock_nested_bh(lock) \ - local_lock_release((lock)) + do { \ + __release(lock); \ + local_lock_release((lock)); \ + } while (0) #else /* !CONFIG_PREEMPT_RT */ +#include + /* * On PREEMPT_RT local_lock maps to a per CPU spinlock, which protects the * critical section while staying preemptible. @@ -269,7 +289,7 @@ do { \ } while (0) #define __local_trylock(lock) \ - ({ \ + __try_acquire_ctx_lock(lock, context_unsafe(({ \ int __locked; \ \ if (in_nmi() | in_hardirq()) { \ @@ -281,17 +301,40 @@ do { \ migrate_enable(); \ } \ __locked; \ - }) + }))) #define __local_trylock_irqsave(lock, flags) \ - ({ \ + __try_acquire_ctx_lock(lock, ({ \ typecheck(unsigned long, flags); \ flags = 0; \ __local_trylock(lock); \ - }) + })) /* migration must be disabled before calling __local_lock_is_locked */ #define __local_lock_is_locked(__lock) \ (rt_mutex_owner(&this_cpu_ptr(__lock)->lock) == current) #endif /* CONFIG_PREEMPT_RT */ + +#if defined(WARN_CONTEXT_ANALYSIS) +/* + * Because the compiler only knows about the base per-CPU variable, use this + * helper function to make the compiler think we lock/unlock the @base variable, + * and hide the fact we actually pass the per-CPU instance to lock/unlock + * functions. + */ +static __always_inline local_lock_t *__this_cpu_local_lock(local_lock_t __percpu *base) + __returns_ctx_lock(base) __attribute__((overloadable)) +{ + return this_cpu_ptr(base); +} +#ifndef CONFIG_PREEMPT_RT +static __always_inline local_trylock_t *__this_cpu_local_lock(local_trylock_t __percpu *base) + __returns_ctx_lock(base) __attribute__((overloadable)) +{ + return this_cpu_ptr(base); +} +#endif /* CONFIG_PREEMPT_RT */ +#else /* WARN_CONTEXT_ANALYSIS */ +#define __this_cpu_local_lock(base) this_cpu_ptr(base) +#endif /* WARN_CONTEXT_ANALYSIS */ diff --git a/lib/test_context-analysis.c b/lib/test_context-analysis.c index 1c96c56cf873..003e64cac540 100644 --- a/lib/test_context-analysis.c +++ b/lib/test_context-analysis.c @@ -6,7 +6,9 @@ #include #include +#include #include +#include #include #include #include @@ -458,3 +460,74 @@ static void __used test_srcu_guard(struct test_srcu_data *d) { guard(srcu_fast)(&d->srcu); (void)srcu_dereference(d->data, &d->srcu); } { guard(srcu_fast_notrace)(&d->srcu); (void)srcu_dereference(d->data, &d->srcu); } } + +struct test_local_lock_data { + local_lock_t lock; + int counter __guarded_by(&lock); +}; + +static DEFINE_PER_CPU(struct test_local_lock_data, test_local_lock_data) = { + .lock = INIT_LOCAL_LOCK(lock), +}; + +static void __used test_local_lock_init(struct test_local_lock_data *d) +{ + local_lock_init(&d->lock); + d->counter = 0; +} + +static void __used test_local_lock(void) +{ + unsigned long flags; + + local_lock(&test_local_lock_data.lock); + this_cpu_add(test_local_lock_data.counter, 1); + local_unlock(&test_local_lock_data.lock); + + local_lock_irq(&test_local_lock_data.lock); + this_cpu_add(test_local_lock_data.counter, 1); + local_unlock_irq(&test_local_lock_data.lock); + + local_lock_irqsave(&test_local_lock_data.lock, flags); + this_cpu_add(test_local_lock_data.counter, 1); + local_unlock_irqrestore(&test_local_lock_data.lock, flags); + + local_lock_nested_bh(&test_local_lock_data.lock); + this_cpu_add(test_local_lock_data.counter, 1); + local_unlock_nested_bh(&test_local_lock_data.lock); +} + +static void __used test_local_lock_guard(void) +{ + { guard(local_lock)(&test_local_lock_data.lock); this_cpu_add(test_local_lock_data.counter, 1); } + { guard(local_lock_irq)(&test_local_lock_data.lock); this_cpu_add(test_local_lock_data.counter, 1); } + { guard(local_lock_irqsave)(&test_local_lock_data.lock); this_cpu_add(test_local_lock_data.counter, 1); } + { guard(local_lock_nested_bh)(&test_local_lock_data.lock); this_cpu_add(test_local_lock_data.counter, 1); } +} + +struct test_local_trylock_data { + local_trylock_t lock; + int counter __guarded_by(&lock); +}; + +static DEFINE_PER_CPU(struct test_local_trylock_data, test_local_trylock_data) = { + .lock = INIT_LOCAL_TRYLOCK(lock), +}; + +static void __used test_local_trylock_init(struct test_local_trylock_data *d) +{ + local_trylock_init(&d->lock); + d->counter = 0; +} + +static void __used test_local_trylock(void) +{ + local_lock(&test_local_trylock_data.lock); + this_cpu_add(test_local_trylock_data.counter, 1); + local_unlock(&test_local_trylock_data.lock); + + if (local_trylock(&test_local_trylock_data.lock)) { + this_cpu_add(test_local_trylock_data.counter, 1); + local_unlock(&test_local_trylock_data.lock); + } +} From 47907461e4f6fcdce8cf91dd164369192deeb7c4 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:09 +0100 Subject: [PATCH 23/90] locking/ww_mutex: Support Clang's context analysis Add support for Clang's context analysis for ww_mutex. The programming model for ww_mutex is subtly more complex than other locking primitives when using ww_acquire_ctx. Encoding the respective pre-conditions for ww_mutex lock/unlock based on ww_acquire_ctx state using Clang's context analysis makes incorrect use of the API harder. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-21-elver@google.com --- Documentation/dev-tools/context-analysis.rst | 3 +- include/linux/ww_mutex.h | 22 +++++-- lib/test_context-analysis.c | 69 ++++++++++++++++++++ 3 files changed, 87 insertions(+), 7 deletions(-) diff --git a/Documentation/dev-tools/context-analysis.rst b/Documentation/dev-tools/context-analysis.rst index a48b75f45e79..8dd6c0d695aa 100644 --- a/Documentation/dev-tools/context-analysis.rst +++ b/Documentation/dev-tools/context-analysis.rst @@ -80,7 +80,8 @@ Supported Kernel Primitives Currently the following synchronization primitives are supported: `raw_spinlock_t`, `spinlock_t`, `rwlock_t`, `mutex`, `seqlock_t`, -`bit_spinlock`, RCU, SRCU (`srcu_struct`), `rw_semaphore`, `local_lock_t`. +`bit_spinlock`, RCU, SRCU (`srcu_struct`), `rw_semaphore`, `local_lock_t`, +`ww_mutex`. For context locks with an initialization function (e.g., `spin_lock_init()`), calling this function before initializing any guarded members or globals diff --git a/include/linux/ww_mutex.h b/include/linux/ww_mutex.h index 45ff6f7a872b..58e959ee10e9 100644 --- a/include/linux/ww_mutex.h +++ b/include/linux/ww_mutex.h @@ -44,7 +44,7 @@ struct ww_class { unsigned int is_wait_die; }; -struct ww_mutex { +context_lock_struct(ww_mutex) { struct WW_MUTEX_BASE base; struct ww_acquire_ctx *ctx; #ifdef DEBUG_WW_MUTEXES @@ -52,7 +52,7 @@ struct ww_mutex { #endif }; -struct ww_acquire_ctx { +context_lock_struct(ww_acquire_ctx) { struct task_struct *task; unsigned long stamp; unsigned int acquired; @@ -107,6 +107,7 @@ struct ww_acquire_ctx { */ static inline void ww_mutex_init(struct ww_mutex *lock, struct ww_class *ww_class) + __assumes_ctx_lock(lock) { ww_mutex_base_init(&lock->base, ww_class->mutex_name, &ww_class->mutex_key); lock->ctx = NULL; @@ -141,6 +142,7 @@ static inline void ww_mutex_init(struct ww_mutex *lock, */ static inline void ww_acquire_init(struct ww_acquire_ctx *ctx, struct ww_class *ww_class) + __acquires(ctx) __no_context_analysis { ctx->task = current; ctx->stamp = atomic_long_inc_return_relaxed(&ww_class->stamp); @@ -179,6 +181,7 @@ static inline void ww_acquire_init(struct ww_acquire_ctx *ctx, * data structures. */ static inline void ww_acquire_done(struct ww_acquire_ctx *ctx) + __releases(ctx) __acquires_shared(ctx) __no_context_analysis { #ifdef DEBUG_WW_MUTEXES lockdep_assert_held(ctx); @@ -196,6 +199,7 @@ static inline void ww_acquire_done(struct ww_acquire_ctx *ctx) * mutexes have been released with ww_mutex_unlock. */ static inline void ww_acquire_fini(struct ww_acquire_ctx *ctx) + __releases_shared(ctx) __no_context_analysis { #ifdef CONFIG_DEBUG_LOCK_ALLOC mutex_release(&ctx->first_lock_dep_map, _THIS_IP_); @@ -245,7 +249,8 @@ static inline void ww_acquire_fini(struct ww_acquire_ctx *ctx) * * A mutex acquired with this function must be released with ww_mutex_unlock. */ -extern int /* __must_check */ ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx); +extern int /* __must_check */ ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) + __cond_acquires(0, lock) __must_hold(ctx); /** * ww_mutex_lock_interruptible - acquire the w/w mutex, interruptible @@ -278,7 +283,8 @@ extern int /* __must_check */ ww_mutex_lock(struct ww_mutex *lock, struct ww_acq * A mutex acquired with this function must be released with ww_mutex_unlock. */ extern int __must_check ww_mutex_lock_interruptible(struct ww_mutex *lock, - struct ww_acquire_ctx *ctx); + struct ww_acquire_ctx *ctx) + __cond_acquires(0, lock) __must_hold(ctx); /** * ww_mutex_lock_slow - slowpath acquiring of the w/w mutex @@ -305,6 +311,7 @@ extern int __must_check ww_mutex_lock_interruptible(struct ww_mutex *lock, */ static inline void ww_mutex_lock_slow(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) + __acquires(lock) __must_hold(ctx) __no_context_analysis { int ret; #ifdef DEBUG_WW_MUTEXES @@ -342,6 +349,7 @@ ww_mutex_lock_slow(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) static inline int __must_check ww_mutex_lock_slow_interruptible(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) + __cond_acquires(0, lock) __must_hold(ctx) { #ifdef DEBUG_WW_MUTEXES DEBUG_LOCKS_WARN_ON(!ctx->contending_lock); @@ -349,10 +357,11 @@ ww_mutex_lock_slow_interruptible(struct ww_mutex *lock, return ww_mutex_lock_interruptible(lock, ctx); } -extern void ww_mutex_unlock(struct ww_mutex *lock); +extern void ww_mutex_unlock(struct ww_mutex *lock) __releases(lock); extern int __must_check ww_mutex_trylock(struct ww_mutex *lock, - struct ww_acquire_ctx *ctx); + struct ww_acquire_ctx *ctx) + __cond_acquires(true, lock) __must_hold(ctx); /*** * ww_mutex_destroy - mark a w/w mutex unusable @@ -363,6 +372,7 @@ extern int __must_check ww_mutex_trylock(struct ww_mutex *lock, * this function is called. */ static inline void ww_mutex_destroy(struct ww_mutex *lock) + __must_not_hold(lock) { #ifndef CONFIG_PREEMPT_RT mutex_destroy(&lock->base); diff --git a/lib/test_context-analysis.c b/lib/test_context-analysis.c index 003e64cac540..2dc404456497 100644 --- a/lib/test_context-analysis.c +++ b/lib/test_context-analysis.c @@ -14,6 +14,7 @@ #include #include #include +#include /* * Test that helper macros work as expected. @@ -531,3 +532,71 @@ static void __used test_local_trylock(void) local_unlock(&test_local_trylock_data.lock); } } + +static DEFINE_WD_CLASS(ww_class); + +struct test_ww_mutex_data { + struct ww_mutex mtx; + int counter __guarded_by(&mtx); +}; + +static void __used test_ww_mutex_init(struct test_ww_mutex_data *d) +{ + ww_mutex_init(&d->mtx, &ww_class); + d->counter = 0; +} + +static void __used test_ww_mutex_lock_noctx(struct test_ww_mutex_data *d) +{ + if (!ww_mutex_lock(&d->mtx, NULL)) { + d->counter++; + ww_mutex_unlock(&d->mtx); + } + + if (!ww_mutex_lock_interruptible(&d->mtx, NULL)) { + d->counter++; + ww_mutex_unlock(&d->mtx); + } + + if (ww_mutex_trylock(&d->mtx, NULL)) { + d->counter++; + ww_mutex_unlock(&d->mtx); + } + + ww_mutex_lock_slow(&d->mtx, NULL); + d->counter++; + ww_mutex_unlock(&d->mtx); + + ww_mutex_destroy(&d->mtx); +} + +static void __used test_ww_mutex_lock_ctx(struct test_ww_mutex_data *d) +{ + struct ww_acquire_ctx ctx; + + ww_acquire_init(&ctx, &ww_class); + + if (!ww_mutex_lock(&d->mtx, &ctx)) { + d->counter++; + ww_mutex_unlock(&d->mtx); + } + + if (!ww_mutex_lock_interruptible(&d->mtx, &ctx)) { + d->counter++; + ww_mutex_unlock(&d->mtx); + } + + if (ww_mutex_trylock(&d->mtx, &ctx)) { + d->counter++; + ww_mutex_unlock(&d->mtx); + } + + ww_mutex_lock_slow(&d->mtx, &ctx); + d->counter++; + ww_mutex_unlock(&d->mtx); + + ww_acquire_done(&ctx); + ww_acquire_fini(&ctx); + + ww_mutex_destroy(&d->mtx); +} From 6e530e2e31191d88f692e6c8d3bd245e43416e4f Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:10 +0100 Subject: [PATCH 24/90] debugfs: Make debugfs_cancellation a context lock struct When compiling include/linux/debugfs.h with CONTEXT_ANALYSIS enabled, we can see this error: ./include/linux/debugfs.h:239:17: error: use of undeclared identifier 'cancellation' 239 | void __acquires(cancellation) Move the __acquires(..) attribute after the declaration, so that the compiler can see the cancellation function argument, as well as making struct debugfs_cancellation a real context lock to benefit from Clang's context analysis. This change is a preparatory change to allow enabling context analysis in subsystems that include the above header. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Bart Van Assche Link: https://patch.msgid.link/20251219154418.3592607-22-elver@google.com --- include/linux/debugfs.h | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/include/linux/debugfs.h b/include/linux/debugfs.h index 7cecda29447e..4177c4738282 100644 --- a/include/linux/debugfs.h +++ b/include/linux/debugfs.h @@ -239,18 +239,16 @@ ssize_t debugfs_read_file_str(struct file *file, char __user *user_buf, * @cancel: callback to call * @cancel_data: extra data for the callback to call */ -struct debugfs_cancellation { +context_lock_struct(debugfs_cancellation) { struct list_head list; void (*cancel)(struct dentry *, void *); void *cancel_data; }; -void __acquires(cancellation) -debugfs_enter_cancellation(struct file *file, - struct debugfs_cancellation *cancellation); -void __releases(cancellation) -debugfs_leave_cancellation(struct file *file, - struct debugfs_cancellation *cancellation); +void debugfs_enter_cancellation(struct file *file, + struct debugfs_cancellation *cancellation) __acquires(cancellation); +void debugfs_leave_cancellation(struct file *file, + struct debugfs_cancellation *cancellation) __releases(cancellation); #else From 4f109baeea4dc6fa1426ab559159d3bb35e05343 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:11 +0100 Subject: [PATCH 25/90] um: Fix incorrect __acquires/__releases annotations With Clang's context analysis, the compiler is a bit more strict about what goes into the __acquires/__releases annotations and can't refer to non-existent variables. On an UM build, mm_id.h is transitively included into mm_types.h, and we can observe the following error (if context analysis is enabled in e.g. stackdepot.c): In file included from lib/stackdepot.c:17: In file included from include/linux/debugfs.h:15: In file included from include/linux/fs.h:5: In file included from include/linux/fs/super.h:5: In file included from include/linux/fs/super_types.h:7: In file included from include/linux/list_lru.h:14: In file included from include/linux/xarray.h:16: In file included from include/linux/gfp.h:7: In file included from include/linux/mmzone.h:22: In file included from include/linux/mm_types.h:26: In file included from arch/um/include/asm/mmu.h:12: >> arch/um/include/shared/skas/mm_id.h:24:54: error: use of undeclared identifier 'turnstile' 24 | void enter_turnstile(struct mm_id *mm_id) __acquires(turnstile); | ^~~~~~~~~ arch/um/include/shared/skas/mm_id.h:25:53: error: use of undeclared identifier 'turnstile' 25 | void exit_turnstile(struct mm_id *mm_id) __releases(turnstile); | ^~~~~~~~~ One (discarded) option was to use token_context_lock(turnstile) to just define a token with the already used name, but that would not allow the compiler to distinguish between different mm_id-dependent instances. Another constraint is that struct mm_id is only declared and incomplete in the header, so even if we tried to construct an expression to get to the mutex instance, this would fail (including more headers transitively everywhere should also be avoided). Instead, just declare an mm_id-dependent helper to return the mutex, and use the mm_id-dependent call expression in the __acquires/__releases attributes; the compiler will consider the identity of the mutex to be the call expression. Then using __get_turnstile() in the lock/unlock wrappers (with context analysis enabled for mmu.c) the compiler will be able to verify the implementation of the wrappers as-is. We leave context analysis disabled in arch/um/kernel/skas/ for now. This change is a preparatory change to allow enabling context analysis in subsystems that include any of the above headers. No functional change intended. Closes: https://lore.kernel.org/oe-kbuild-all/202512171220.vHlvhpCr-lkp@intel.com/ Reported-by: kernel test robot Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-23-elver@google.com --- arch/um/include/shared/skas/mm_id.h | 5 +++-- arch/um/kernel/skas/mmu.c | 13 ++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/arch/um/include/shared/skas/mm_id.h b/arch/um/include/shared/skas/mm_id.h index fb96c0bd8222..18c0621430d2 100644 --- a/arch/um/include/shared/skas/mm_id.h +++ b/arch/um/include/shared/skas/mm_id.h @@ -21,8 +21,9 @@ struct mm_id { int syscall_fd_map[STUB_MAX_FDS]; }; -void enter_turnstile(struct mm_id *mm_id) __acquires(turnstile); -void exit_turnstile(struct mm_id *mm_id) __releases(turnstile); +struct mutex *__get_turnstile(struct mm_id *mm_id); +void enter_turnstile(struct mm_id *mm_id) __acquires(__get_turnstile(mm_id)); +void exit_turnstile(struct mm_id *mm_id) __releases(__get_turnstile(mm_id)); void notify_mm_kill(int pid); diff --git a/arch/um/kernel/skas/mmu.c b/arch/um/kernel/skas/mmu.c index 00957788591b..b5017096028b 100644 --- a/arch/um/kernel/skas/mmu.c +++ b/arch/um/kernel/skas/mmu.c @@ -23,18 +23,21 @@ static_assert(sizeof(struct stub_data) == STUB_DATA_PAGES * UM_KERN_PAGE_SIZE); static spinlock_t mm_list_lock; static struct list_head mm_list; -void enter_turnstile(struct mm_id *mm_id) __acquires(turnstile) +struct mutex *__get_turnstile(struct mm_id *mm_id) { struct mm_context *ctx = container_of(mm_id, struct mm_context, id); - mutex_lock(&ctx->turnstile); + return &ctx->turnstile; } -void exit_turnstile(struct mm_id *mm_id) __releases(turnstile) +void enter_turnstile(struct mm_id *mm_id) { - struct mm_context *ctx = container_of(mm_id, struct mm_context, id); + mutex_lock(__get_turnstile(mm_id)); +} - mutex_unlock(&ctx->turnstile); +void exit_turnstile(struct mm_id *mm_id) +{ + mutex_unlock(__get_turnstile(mm_id)); } int init_new_context(struct task_struct *task, struct mm_struct *mm) From 5b63d0ae94ccfd64dcbdb693d88eb3650eb3c64c Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:12 +0100 Subject: [PATCH 26/90] compiler-context-analysis: Remove Sparse support Remove Sparse support as discussed at [1]. The kernel codebase is still scattered with numerous places that try to appease Sparse's context tracking ("annotation for sparse", "fake out sparse", "work around sparse", etc.). Eventually, as more subsystems enable Clang's context analysis, these places will show up and need adjustment or removal of the workarounds altogether. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/all/20250207083335.GW7145@noisy.programming.kicks-ass.net/ [1] Link: https://lore.kernel.org/all/Z6XTKTo_LMj9KmbY@elver.google.com/ [2] Link: https://patch.msgid.link/20251219154418.3592607-24-elver@google.com --- Documentation/dev-tools/sparse.rst | 19 ------ include/linux/compiler-context-analysis.h | 83 +++++++---------------- include/linux/rcupdate.h | 15 +--- 3 files changed, 27 insertions(+), 90 deletions(-) diff --git a/Documentation/dev-tools/sparse.rst b/Documentation/dev-tools/sparse.rst index dc791c8d84d1..37b20170835d 100644 --- a/Documentation/dev-tools/sparse.rst +++ b/Documentation/dev-tools/sparse.rst @@ -53,25 +53,6 @@ sure that bitwise types don't get mixed up (little-endian vs big-endian vs cpu-endian vs whatever), and there the constant "0" really _is_ special. -Using sparse for lock checking ------------------------------- - -The following macros are undefined for gcc and defined during a sparse -run to use the "context" tracking feature of sparse, applied to -locking. These annotations tell sparse when a lock is held, with -regard to the annotated function's entry and exit. - -__must_hold - The specified lock is held on function entry and exit. - -__acquires - The specified lock is held on function exit, but not entry. - -__releases - The specified lock is held on function entry, but not exit. - -If the function enters and exits without the lock held, acquiring and -releasing the lock inside the function in a balanced way, no -annotation is needed. The three annotations above are for cases where -sparse would otherwise report a context imbalance. - Getting sparse -------------- diff --git a/include/linux/compiler-context-analysis.h b/include/linux/compiler-context-analysis.h index a6a34985dbb2..cb728822343f 100644 --- a/include/linux/compiler-context-analysis.h +++ b/include/linux/compiler-context-analysis.h @@ -262,57 +262,32 @@ static inline void _context_unsafe_alias(void **p) { } extern const struct __ctx_lock_##ctx *name /* - * Common keywords for static context analysis. Both Clang's "capability - * analysis" and Sparse's "context tracking" are currently supported. + * Common keywords for static context analysis. */ -#ifdef __CHECKER__ - -/* Sparse context/lock checking support. */ -# define __must_hold(x) __attribute__((context(x,1,1))) -# define __must_not_hold(x) -# define __acquires(x) __attribute__((context(x,0,1))) -# define __cond_acquires(ret, x) __attribute__((context(x,0,-1))) -# define __releases(x) __attribute__((context(x,1,0))) -# define __acquire(x) __context__(x,1) -# define __release(x) __context__(x,-1) -# define __cond_lock(x, c) ((c) ? ({ __acquire(x); 1; }) : 0) -/* For Sparse, there's no distinction between exclusive and shared locks. */ -# define __must_hold_shared __must_hold -# define __acquires_shared __acquires -# define __cond_acquires_shared __cond_acquires -# define __releases_shared __releases -# define __acquire_shared __acquire -# define __release_shared __release -# define __cond_lock_shared __cond_acquire - -#else /* !__CHECKER__ */ /** * __must_hold() - function attribute, caller must hold exclusive context lock - * @x: context lock instance pointer * * Function attribute declaring that the caller must hold the given context - * lock instance @x exclusively. + * lock instance(s) exclusively. */ -# define __must_hold(x) __requires_ctx_lock(x) +#define __must_hold(...) __requires_ctx_lock(__VA_ARGS__) /** * __must_not_hold() - function attribute, caller must not hold context lock - * @x: context lock instance pointer * * Function attribute declaring that the caller must not hold the given context - * lock instance @x. + * lock instance(s). */ -# define __must_not_hold(x) __excludes_ctx_lock(x) +#define __must_not_hold(...) __excludes_ctx_lock(__VA_ARGS__) /** * __acquires() - function attribute, function acquires context lock exclusively - * @x: context lock instance pointer * * Function attribute declaring that the function acquires the given context - * lock instance @x exclusively, but does not release it. + * lock instance(s) exclusively, but does not release them. */ -# define __acquires(x) __acquires_ctx_lock(x) +#define __acquires(...) __acquires_ctx_lock(__VA_ARGS__) /* * Clang's analysis does not care precisely about the value, only that it is @@ -339,17 +314,16 @@ static inline void _context_unsafe_alias(void **p) { } * * @ret may be one of: true, false, nonzero, 0, nonnull, NULL. */ -# define __cond_acquires(ret, x) __cond_acquires_impl_##ret(x) +#define __cond_acquires(ret, x) __cond_acquires_impl_##ret(x) /** * __releases() - function attribute, function releases a context lock exclusively - * @x: context lock instance pointer * * Function attribute declaring that the function releases the given context - * lock instance @x exclusively. The associated context must be active on + * lock instance(s) exclusively. The associated context(s) must be active on * entry. */ -# define __releases(x) __releases_ctx_lock(x) +#define __releases(...) __releases_ctx_lock(__VA_ARGS__) /** * __acquire() - function to acquire context lock exclusively @@ -357,7 +331,7 @@ static inline void _context_unsafe_alias(void **p) { } * * No-op function that acquires the given context lock instance @x exclusively. */ -# define __acquire(x) __acquire_ctx_lock(x) +#define __acquire(x) __acquire_ctx_lock(x) /** * __release() - function to release context lock exclusively @@ -365,7 +339,7 @@ static inline void _context_unsafe_alias(void **p) { } * * No-op function that releases the given context lock instance @x. */ -# define __release(x) __release_ctx_lock(x) +#define __release(x) __release_ctx_lock(x) /** * __cond_lock() - function that conditionally acquires a context lock @@ -383,25 +357,23 @@ static inline void _context_unsafe_alias(void **p) { } * * #define spin_trylock(l) __cond_lock(&lock, _spin_trylock(&lock)) */ -# define __cond_lock(x, c) __try_acquire_ctx_lock(x, c) +#define __cond_lock(x, c) __try_acquire_ctx_lock(x, c) /** * __must_hold_shared() - function attribute, caller must hold shared context lock - * @x: context lock instance pointer * * Function attribute declaring that the caller must hold the given context - * lock instance @x with shared access. + * lock instance(s) with shared access. */ -# define __must_hold_shared(x) __requires_shared_ctx_lock(x) +#define __must_hold_shared(...) __requires_shared_ctx_lock(__VA_ARGS__) /** * __acquires_shared() - function attribute, function acquires context lock shared - * @x: context lock instance pointer * * Function attribute declaring that the function acquires the given - * context lock instance @x with shared access, but does not release it. + * context lock instance(s) with shared access, but does not release them. */ -# define __acquires_shared(x) __acquires_shared_ctx_lock(x) +#define __acquires_shared(...) __acquires_shared_ctx_lock(__VA_ARGS__) /** * __cond_acquires_shared() - function attribute, function conditionally @@ -410,23 +382,22 @@ static inline void _context_unsafe_alias(void **p) { } * @x: context lock instance pointer * * Function attribute declaring that the function conditionally acquires the - * given context lock instance @x with shared access, but does not release it. The - * function return value @ret denotes when the context lock is acquired. + * given context lock instance @x with shared access, but does not release it. + * The function return value @ret denotes when the context lock is acquired. * * @ret may be one of: true, false, nonzero, 0, nonnull, NULL. */ -# define __cond_acquires_shared(ret, x) __cond_acquires_impl_##ret(x, _shared) +#define __cond_acquires_shared(ret, x) __cond_acquires_impl_##ret(x, _shared) /** * __releases_shared() - function attribute, function releases a * context lock shared - * @x: context lock instance pointer * * Function attribute declaring that the function releases the given context - * lock instance @x with shared access. The associated context must be active - * on entry. + * lock instance(s) with shared access. The associated context(s) must be + * active on entry. */ -# define __releases_shared(x) __releases_shared_ctx_lock(x) +#define __releases_shared(...) __releases_shared_ctx_lock(__VA_ARGS__) /** * __acquire_shared() - function to acquire context lock shared @@ -435,7 +406,7 @@ static inline void _context_unsafe_alias(void **p) { } * No-op function that acquires the given context lock instance @x with shared * access. */ -# define __acquire_shared(x) __acquire_shared_ctx_lock(x) +#define __acquire_shared(x) __acquire_shared_ctx_lock(x) /** * __release_shared() - function to release context lock shared @@ -444,7 +415,7 @@ static inline void _context_unsafe_alias(void **p) { } * No-op function that releases the given context lock instance @x with shared * access. */ -# define __release_shared(x) __release_shared_ctx_lock(x) +#define __release_shared(x) __release_shared_ctx_lock(x) /** * __cond_lock_shared() - function that conditionally acquires a context lock shared @@ -457,9 +428,7 @@ static inline void _context_unsafe_alias(void **p) { } * shared access, if the boolean expression @c is true. The result of @c is the * return value. */ -# define __cond_lock_shared(x, c) __try_acquire_shared_ctx_lock(x, c) - -#endif /* __CHECKER__ */ +#define __cond_lock_shared(x, c) __try_acquire_shared_ctx_lock(x, c) /** * __acquire_ret() - helper to acquire context lock of return value diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 50e63eade019..d828a4673441 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -1219,20 +1219,7 @@ rcu_head_after_call_rcu(struct rcu_head *rhp, rcu_callback_t f) extern int rcu_expedited; extern int rcu_normal; -DEFINE_LOCK_GUARD_0(rcu, - do { - rcu_read_lock(); - /* - * sparse doesn't call the cleanup function, - * so just release immediately and don't track - * the context. We don't need to anyway, since - * the whole point of the guard is to not need - * the explicit unlock. - */ - __release(RCU); - } while (0), - rcu_read_unlock()) - +DEFINE_LOCK_GUARD_0(rcu, rcu_read_lock(), rcu_read_unlock()) DECLARE_LOCK_GUARD_0_ATTRS(rcu, __acquires_shared(RCU), __releases_shared(RCU)) #endif /* __LINUX_RCUPDATE_H */ From e4588c25c9d122b5847b88e18b184404b6959160 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:13 +0100 Subject: [PATCH 27/90] compiler-context-analysis: Remove __cond_lock() function-like helper As discussed in [1], removing __cond_lock() will improve the readability of trylock code. Now that Sparse context tracking support has been removed, we can also remove __cond_lock(). Change existing APIs to either drop __cond_lock() completely, or make use of the __cond_acquires() function attribute instead. In particular, spinlock and rwlock implementations required switching over to inline helpers rather than statement-expressions for their trylock_* variants. Suggested-by: Peter Zijlstra Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/all/20250207082832.GU7145@noisy.programming.kicks-ass.net/ [1] Link: https://patch.msgid.link/20251219154418.3592607-25-elver@google.com --- Documentation/dev-tools/context-analysis.rst | 2 - Documentation/mm/process_addrs.rst | 6 +- .../net/wireless/intel/iwlwifi/iwl-trans.c | 4 +- .../net/wireless/intel/iwlwifi/iwl-trans.h | 6 +- .../intel/iwlwifi/pcie/gen1_2/internal.h | 5 +- .../intel/iwlwifi/pcie/gen1_2/trans.c | 4 +- include/linux/compiler-context-analysis.h | 31 ---------- include/linux/lockref.h | 4 +- include/linux/mm.h | 33 ++-------- include/linux/rwlock.h | 11 +--- include/linux/rwlock_api_smp.h | 14 ++++- include/linux/rwlock_rt.h | 21 ++++--- include/linux/sched/signal.h | 14 +---- include/linux/spinlock.h | 45 +++++--------- include/linux/spinlock_api_smp.h | 20 ++++++ include/linux/spinlock_api_up.h | 61 ++++++++++++++++--- include/linux/spinlock_rt.h | 26 ++++---- kernel/signal.c | 4 +- kernel/time/posix-timers.c | 13 +--- lib/dec_and_lock.c | 8 +-- lib/lockref.c | 1 - mm/memory.c | 4 +- mm/pgtable-generic.c | 19 +++--- tools/include/linux/compiler_types.h | 2 - 24 files changed, 163 insertions(+), 195 deletions(-) diff --git a/Documentation/dev-tools/context-analysis.rst b/Documentation/dev-tools/context-analysis.rst index 8dd6c0d695aa..e69896e597b6 100644 --- a/Documentation/dev-tools/context-analysis.rst +++ b/Documentation/dev-tools/context-analysis.rst @@ -112,10 +112,8 @@ Keywords __releases_shared __acquire __release - __cond_lock __acquire_shared __release_shared - __cond_lock_shared __acquire_ret __acquire_shared_ret context_unsafe diff --git a/Documentation/mm/process_addrs.rst b/Documentation/mm/process_addrs.rst index 7f2f3e87071d..851680ead45f 100644 --- a/Documentation/mm/process_addrs.rst +++ b/Documentation/mm/process_addrs.rst @@ -583,7 +583,7 @@ To access PTE-level page tables, a helper like :c:func:`!pte_offset_map_lock` or :c:func:`!pte_offset_map` can be used depending on stability requirements. These map the page table into kernel memory if required, take the RCU lock, and depending on variant, may also look up or acquire the PTE lock. -See the comment on :c:func:`!__pte_offset_map_lock`. +See the comment on :c:func:`!pte_offset_map_lock`. Atomicity ^^^^^^^^^ @@ -667,7 +667,7 @@ must be released via :c:func:`!pte_unmap_unlock`. .. note:: There are some variants on this, such as :c:func:`!pte_offset_map_rw_nolock` when we know we hold the PTE stable but for brevity we do not explore this. See the comment for - :c:func:`!__pte_offset_map_lock` for more details. + :c:func:`!pte_offset_map_lock` for more details. When modifying data in ranges we typically only wish to allocate higher page tables as necessary, using these locks to avoid races or overwriting anything, @@ -686,7 +686,7 @@ At the leaf page table, that is the PTE, we can't entirely rely on this pattern as we have separate PMD and PTE locks and a THP collapse for instance might have eliminated the PMD entry as well as the PTE from under us. -This is why :c:func:`!__pte_offset_map_lock` locklessly retrieves the PMD entry +This is why :c:func:`!pte_offset_map_lock` locklessly retrieves the PMD entry for the PTE, carefully checking it is as expected, before acquiring the PTE-specific lock, and then *again* checking that the PMD entry is as expected. diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-trans.c b/drivers/net/wireless/intel/iwlwifi/iwl-trans.c index cc8a84018f70..fa1442246662 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-trans.c +++ b/drivers/net/wireless/intel/iwlwifi/iwl-trans.c @@ -548,11 +548,11 @@ int iwl_trans_read_config32(struct iwl_trans *trans, u32 ofs, return iwl_trans_pcie_read_config32(trans, ofs, val); } -bool _iwl_trans_grab_nic_access(struct iwl_trans *trans) +bool iwl_trans_grab_nic_access(struct iwl_trans *trans) { return iwl_trans_pcie_grab_nic_access(trans); } -IWL_EXPORT_SYMBOL(_iwl_trans_grab_nic_access); +IWL_EXPORT_SYMBOL(iwl_trans_grab_nic_access); void __releases(nic_access) iwl_trans_release_nic_access(struct iwl_trans *trans) diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h index a552669db6e2..688f9fee2821 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h @@ -1063,11 +1063,7 @@ int iwl_trans_sw_reset(struct iwl_trans *trans); void iwl_trans_set_bits_mask(struct iwl_trans *trans, u32 reg, u32 mask, u32 value); -bool _iwl_trans_grab_nic_access(struct iwl_trans *trans); - -#define iwl_trans_grab_nic_access(trans) \ - __cond_lock(nic_access, \ - likely(_iwl_trans_grab_nic_access(trans))) +bool iwl_trans_grab_nic_access(struct iwl_trans *trans); void __releases(nic_access) iwl_trans_release_nic_access(struct iwl_trans *trans); diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/internal.h b/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/internal.h index 207c56e338dd..7b7b35e442f9 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/internal.h +++ b/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/internal.h @@ -553,10 +553,7 @@ void iwl_trans_pcie_free(struct iwl_trans *trans); void iwl_trans_pcie_free_pnvm_dram_regions(struct iwl_dram_regions *dram_regions, struct device *dev); -bool __iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans, bool silent); -#define _iwl_trans_pcie_grab_nic_access(trans, silent) \ - __cond_lock(nic_access_nobh, \ - likely(__iwl_trans_pcie_grab_nic_access(trans, silent))) +bool _iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans, bool silent); void iwl_trans_pcie_check_product_reset_status(struct pci_dev *pdev); void iwl_trans_pcie_check_product_reset_mode(struct pci_dev *pdev); diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans.c b/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans.c index 164d060ec617..415a19ea9f06 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/gen1_2/trans.c @@ -2327,7 +2327,7 @@ EXPORT_SYMBOL(iwl_trans_pcie_reset); * This version doesn't disable BHs but rather assumes they're * already disabled. */ -bool __iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans, bool silent) +bool _iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans, bool silent) { int ret; struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans); @@ -2415,7 +2415,7 @@ bool iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans) bool ret; local_bh_disable(); - ret = __iwl_trans_pcie_grab_nic_access(trans, false); + ret = _iwl_trans_pcie_grab_nic_access(trans, false); if (ret) { /* keep BHs disabled until iwl_trans_pcie_release_nic_access */ return ret; diff --git a/include/linux/compiler-context-analysis.h b/include/linux/compiler-context-analysis.h index cb728822343f..4f7559d7ae91 100644 --- a/include/linux/compiler-context-analysis.h +++ b/include/linux/compiler-context-analysis.h @@ -341,24 +341,6 @@ static inline void _context_unsafe_alias(void **p) { } */ #define __release(x) __release_ctx_lock(x) -/** - * __cond_lock() - function that conditionally acquires a context lock - * exclusively - * @x: context lock instance pinter - * @c: boolean expression - * - * Return: result of @c - * - * No-op function that conditionally acquires context lock instance @x - * exclusively, if the boolean expression @c is true. The result of @c is the - * return value; for example: - * - * .. code-block:: c - * - * #define spin_trylock(l) __cond_lock(&lock, _spin_trylock(&lock)) - */ -#define __cond_lock(x, c) __try_acquire_ctx_lock(x, c) - /** * __must_hold_shared() - function attribute, caller must hold shared context lock * @@ -417,19 +399,6 @@ static inline void _context_unsafe_alias(void **p) { } */ #define __release_shared(x) __release_shared_ctx_lock(x) -/** - * __cond_lock_shared() - function that conditionally acquires a context lock shared - * @x: context lock instance pinter - * @c: boolean expression - * - * Return: result of @c - * - * No-op function that conditionally acquires context lock instance @x with - * shared access, if the boolean expression @c is true. The result of @c is the - * return value. - */ -#define __cond_lock_shared(x, c) __try_acquire_shared_ctx_lock(x, c) - /** * __acquire_ret() - helper to acquire context lock of return value * @call: call expression diff --git a/include/linux/lockref.h b/include/linux/lockref.h index 815d871fadfc..6ded24cdb4a8 100644 --- a/include/linux/lockref.h +++ b/include/linux/lockref.h @@ -49,9 +49,7 @@ static inline void lockref_init(struct lockref *lockref) void lockref_get(struct lockref *lockref); int lockref_put_return(struct lockref *lockref); bool lockref_get_not_zero(struct lockref *lockref); -bool lockref_put_or_lock(struct lockref *lockref); -#define lockref_put_or_lock(_lockref) \ - (!__cond_lock((_lockref)->lock, !lockref_put_or_lock(_lockref))) +bool lockref_put_or_lock(struct lockref *lockref) __cond_acquires(false, &lockref->lock); void lockref_mark_dead(struct lockref *lockref); bool lockref_get_not_dead(struct lockref *lockref); diff --git a/include/linux/mm.h b/include/linux/mm.h index 15076261d0c2..f369cb633516 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2975,15 +2975,8 @@ static inline pud_t pud_mkspecial(pud_t pud) } #endif /* CONFIG_ARCH_SUPPORTS_PUD_PFNMAP */ -extern pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr, - spinlock_t **ptl); -static inline pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr, - spinlock_t **ptl) -{ - pte_t *ptep; - __cond_lock(*ptl, ptep = __get_locked_pte(mm, addr, ptl)); - return ptep; -} +extern pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr, + spinlock_t **ptl); #ifdef __PAGETABLE_P4D_FOLDED static inline int __p4d_alloc(struct mm_struct *mm, pgd_t *pgd, @@ -3337,31 +3330,15 @@ static inline bool pagetable_pte_ctor(struct mm_struct *mm, return true; } -pte_t *___pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp); -static inline pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, - pmd_t *pmdvalp) -{ - pte_t *pte; +pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp); - __cond_lock(RCU, pte = ___pte_offset_map(pmd, addr, pmdvalp)); - return pte; -} static inline pte_t *pte_offset_map(pmd_t *pmd, unsigned long addr) { return __pte_offset_map(pmd, addr, NULL); } -pte_t *__pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, - unsigned long addr, spinlock_t **ptlp); -static inline pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, - unsigned long addr, spinlock_t **ptlp) -{ - pte_t *pte; - - __cond_lock(RCU, __cond_lock(*ptlp, - pte = __pte_offset_map_lock(mm, pmd, addr, ptlp))); - return pte; -} +pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr, spinlock_t **ptlp); pte_t *pte_offset_map_ro_nolock(struct mm_struct *mm, pmd_t *pmd, unsigned long addr, spinlock_t **ptlp); diff --git a/include/linux/rwlock.h b/include/linux/rwlock.h index 151f9d5f3288..65a5b55e1bcd 100644 --- a/include/linux/rwlock.h +++ b/include/linux/rwlock.h @@ -50,8 +50,8 @@ do { \ * regardless of whether CONFIG_SMP or CONFIG_PREEMPT are set. The various * methods are defined as nops in the case they are not required. */ -#define read_trylock(lock) __cond_lock_shared(lock, _raw_read_trylock(lock)) -#define write_trylock(lock) __cond_lock(lock, _raw_write_trylock(lock)) +#define read_trylock(lock) _raw_read_trylock(lock) +#define write_trylock(lock) _raw_write_trylock(lock) #define write_lock(lock) _raw_write_lock(lock) #define read_lock(lock) _raw_read_lock(lock) @@ -113,12 +113,7 @@ do { \ } while (0) #define write_unlock_bh(lock) _raw_write_unlock_bh(lock) -#define write_trylock_irqsave(lock, flags) \ - __cond_lock(lock, ({ \ - local_irq_save(flags); \ - _raw_write_trylock(lock) ? \ - 1 : ({ local_irq_restore(flags); 0; }); \ - })) +#define write_trylock_irqsave(lock, flags) _raw_write_trylock_irqsave(lock, &(flags)) #ifdef arch_rwlock_is_contended #define rwlock_is_contended(lock) \ diff --git a/include/linux/rwlock_api_smp.h b/include/linux/rwlock_api_smp.h index 6d5cc0b7be1f..d903b17c46ca 100644 --- a/include/linux/rwlock_api_smp.h +++ b/include/linux/rwlock_api_smp.h @@ -26,8 +26,8 @@ unsigned long __lockfunc _raw_read_lock_irqsave(rwlock_t *lock) __acquires(lock); unsigned long __lockfunc _raw_write_lock_irqsave(rwlock_t *lock) __acquires(lock); -int __lockfunc _raw_read_trylock(rwlock_t *lock); -int __lockfunc _raw_write_trylock(rwlock_t *lock); +int __lockfunc _raw_read_trylock(rwlock_t *lock) __cond_acquires_shared(true, lock); +int __lockfunc _raw_write_trylock(rwlock_t *lock) __cond_acquires(true, lock); void __lockfunc _raw_read_unlock(rwlock_t *lock) __releases_shared(lock); void __lockfunc _raw_write_unlock(rwlock_t *lock) __releases(lock); void __lockfunc _raw_read_unlock_bh(rwlock_t *lock) __releases_shared(lock); @@ -41,6 +41,16 @@ void __lockfunc _raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) __releases(lock); +static inline bool _raw_write_trylock_irqsave(rwlock_t *lock, unsigned long *flags) + __cond_acquires(true, lock) +{ + local_irq_save(*flags); + if (_raw_write_trylock(lock)) + return true; + local_irq_restore(*flags); + return false; +} + #ifdef CONFIG_INLINE_READ_LOCK #define _raw_read_lock(lock) __raw_read_lock(lock) #endif diff --git a/include/linux/rwlock_rt.h b/include/linux/rwlock_rt.h index f64d6d319a47..37b387dcab21 100644 --- a/include/linux/rwlock_rt.h +++ b/include/linux/rwlock_rt.h @@ -26,11 +26,11 @@ do { \ } while (0) extern void rt_read_lock(rwlock_t *rwlock) __acquires_shared(rwlock); -extern int rt_read_trylock(rwlock_t *rwlock); +extern int rt_read_trylock(rwlock_t *rwlock) __cond_acquires_shared(true, rwlock); extern void rt_read_unlock(rwlock_t *rwlock) __releases_shared(rwlock); extern void rt_write_lock(rwlock_t *rwlock) __acquires(rwlock); extern void rt_write_lock_nested(rwlock_t *rwlock, int subclass) __acquires(rwlock); -extern int rt_write_trylock(rwlock_t *rwlock); +extern int rt_write_trylock(rwlock_t *rwlock) __cond_acquires(true, rwlock); extern void rt_write_unlock(rwlock_t *rwlock) __releases(rwlock); static __always_inline void read_lock(rwlock_t *rwlock) @@ -59,7 +59,7 @@ static __always_inline void read_lock_irq(rwlock_t *rwlock) flags = 0; \ } while (0) -#define read_trylock(lock) __cond_lock_shared(lock, rt_read_trylock(lock)) +#define read_trylock(lock) rt_read_trylock(lock) static __always_inline void read_unlock(rwlock_t *rwlock) __releases_shared(rwlock) @@ -123,14 +123,15 @@ static __always_inline void write_lock_irq(rwlock_t *rwlock) flags = 0; \ } while (0) -#define write_trylock(lock) __cond_lock(lock, rt_write_trylock(lock)) +#define write_trylock(lock) rt_write_trylock(lock) -#define write_trylock_irqsave(lock, flags) \ - __cond_lock(lock, ({ \ - typecheck(unsigned long, flags); \ - flags = 0; \ - rt_write_trylock(lock); \ - })) +static __always_inline bool _write_trylock_irqsave(rwlock_t *rwlock, unsigned long *flags) + __cond_acquires(true, rwlock) +{ + *flags = 0; + return rt_write_trylock(rwlock); +} +#define write_trylock_irqsave(lock, flags) _write_trylock_irqsave(lock, &(flags)) static __always_inline void write_unlock(rwlock_t *rwlock) __releases(rwlock) diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index 7d6449982822..a63f65aa5bdd 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -737,18 +737,8 @@ static inline int thread_group_empty(struct task_struct *p) #define delay_group_leader(p) \ (thread_group_leader(p) && !thread_group_empty(p)) -extern struct sighand_struct *__lock_task_sighand(struct task_struct *task, - unsigned long *flags); - -static inline struct sighand_struct *lock_task_sighand(struct task_struct *task, - unsigned long *flags) -{ - struct sighand_struct *ret; - - ret = __lock_task_sighand(task, flags); - (void)__cond_lock(&task->sighand->siglock, ret); - return ret; -} +extern struct sighand_struct *lock_task_sighand(struct task_struct *task, + unsigned long *flags); static inline void unlock_task_sighand(struct task_struct *task, unsigned long *flags) diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index 7e560c7a7b23..396b8c5d6c1b 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -213,7 +213,7 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock) * various methods are defined as nops in the case they are not * required. */ -#define raw_spin_trylock(lock) __cond_lock(lock, _raw_spin_trylock(lock)) +#define raw_spin_trylock(lock) _raw_spin_trylock(lock) #define raw_spin_lock(lock) _raw_spin_lock(lock) @@ -284,22 +284,11 @@ static inline void do_raw_spin_unlock(raw_spinlock_t *lock) __releases(lock) } while (0) #define raw_spin_unlock_bh(lock) _raw_spin_unlock_bh(lock) -#define raw_spin_trylock_bh(lock) \ - __cond_lock(lock, _raw_spin_trylock_bh(lock)) +#define raw_spin_trylock_bh(lock) _raw_spin_trylock_bh(lock) -#define raw_spin_trylock_irq(lock) \ - __cond_lock(lock, ({ \ - local_irq_disable(); \ - _raw_spin_trylock(lock) ? \ - 1 : ({ local_irq_enable(); 0; }); \ - })) +#define raw_spin_trylock_irq(lock) _raw_spin_trylock_irq(lock) -#define raw_spin_trylock_irqsave(lock, flags) \ - __cond_lock(lock, ({ \ - local_irq_save(flags); \ - _raw_spin_trylock(lock) ? \ - 1 : ({ local_irq_restore(flags); 0; }); \ - })) +#define raw_spin_trylock_irqsave(lock, flags) _raw_spin_trylock_irqsave(lock, &(flags)) #ifndef CONFIG_PREEMPT_RT /* Include rwlock functions for !RT */ @@ -433,8 +422,12 @@ static __always_inline int spin_trylock_irq(spinlock_t *lock) return raw_spin_trylock_irq(&lock->rlock); } -#define spin_trylock_irqsave(lock, flags) \ - __cond_lock(lock, raw_spin_trylock_irqsave(spinlock_check(lock), flags)) +static __always_inline bool _spin_trylock_irqsave(spinlock_t *lock, unsigned long *flags) + __cond_acquires(true, lock) __no_context_analysis +{ + return raw_spin_trylock_irqsave(spinlock_check(lock), *flags); +} +#define spin_trylock_irqsave(lock, flags) _spin_trylock_irqsave(lock, &(flags)) /** * spin_is_locked() - Check whether a spinlock is locked. @@ -512,23 +505,17 @@ static inline int rwlock_needbreak(rwlock_t *lock) * Decrements @atomic by 1. If the result is 0, returns true and locks * @lock. Returns false for all other cases. */ -extern int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock); -#define atomic_dec_and_lock(atomic, lock) \ - __cond_lock(lock, _atomic_dec_and_lock(atomic, lock)) +extern int atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock) __cond_acquires(true, lock); extern int _atomic_dec_and_lock_irqsave(atomic_t *atomic, spinlock_t *lock, - unsigned long *flags); -#define atomic_dec_and_lock_irqsave(atomic, lock, flags) \ - __cond_lock(lock, _atomic_dec_and_lock_irqsave(atomic, lock, &(flags))) + unsigned long *flags) __cond_acquires(true, lock); +#define atomic_dec_and_lock_irqsave(atomic, lock, flags) _atomic_dec_and_lock_irqsave(atomic, lock, &(flags)) -extern int _atomic_dec_and_raw_lock(atomic_t *atomic, raw_spinlock_t *lock); -#define atomic_dec_and_raw_lock(atomic, lock) \ - __cond_lock(lock, _atomic_dec_and_raw_lock(atomic, lock)) +extern int atomic_dec_and_raw_lock(atomic_t *atomic, raw_spinlock_t *lock) __cond_acquires(true, lock); extern int _atomic_dec_and_raw_lock_irqsave(atomic_t *atomic, raw_spinlock_t *lock, - unsigned long *flags); -#define atomic_dec_and_raw_lock_irqsave(atomic, lock, flags) \ - __cond_lock(lock, _atomic_dec_and_raw_lock_irqsave(atomic, lock, &(flags))) + unsigned long *flags) __cond_acquires(true, lock); +#define atomic_dec_and_raw_lock_irqsave(atomic, lock, flags) _atomic_dec_and_raw_lock_irqsave(atomic, lock, &(flags)) int __alloc_bucket_spinlocks(spinlock_t **locks, unsigned int *lock_mask, size_t max_size, unsigned int cpu_mult, diff --git a/include/linux/spinlock_api_smp.h b/include/linux/spinlock_api_smp.h index 7e7d7d373213..bda5e7a390cd 100644 --- a/include/linux/spinlock_api_smp.h +++ b/include/linux/spinlock_api_smp.h @@ -95,6 +95,26 @@ static inline int __raw_spin_trylock(raw_spinlock_t *lock) return 0; } +static __always_inline bool _raw_spin_trylock_irq(raw_spinlock_t *lock) + __cond_acquires(true, lock) +{ + local_irq_disable(); + if (_raw_spin_trylock(lock)) + return true; + local_irq_enable(); + return false; +} + +static __always_inline bool _raw_spin_trylock_irqsave(raw_spinlock_t *lock, unsigned long *flags) + __cond_acquires(true, lock) +{ + local_irq_save(*flags); + if (_raw_spin_trylock(lock)) + return true; + local_irq_restore(*flags); + return false; +} + /* * If lockdep is enabled then we use the non-preemption spin-ops * even on CONFIG_PREEMPTION, because lockdep assumes that interrupts are diff --git a/include/linux/spinlock_api_up.h b/include/linux/spinlock_api_up.h index 018f5aabc1be..a9d5c7c66e03 100644 --- a/include/linux/spinlock_api_up.h +++ b/include/linux/spinlock_api_up.h @@ -24,14 +24,11 @@ * flags straight, to suppress compiler warnings of unused lock * variables, and to add the proper checker annotations: */ -#define ___LOCK_void(lock) \ - do { (void)(lock); } while (0) - #define ___LOCK_(lock) \ - do { __acquire(lock); ___LOCK_void(lock); } while (0) + do { __acquire(lock); (void)(lock); } while (0) #define ___LOCK_shared(lock) \ - do { __acquire_shared(lock); ___LOCK_void(lock); } while (0) + do { __acquire_shared(lock); (void)(lock); } while (0) #define __LOCK(lock, ...) \ do { preempt_disable(); ___LOCK_##__VA_ARGS__(lock); } while (0) @@ -78,10 +75,56 @@ #define _raw_spin_lock_irqsave(lock, flags) __LOCK_IRQSAVE(lock, flags) #define _raw_read_lock_irqsave(lock, flags) __LOCK_IRQSAVE(lock, flags, shared) #define _raw_write_lock_irqsave(lock, flags) __LOCK_IRQSAVE(lock, flags) -#define _raw_spin_trylock(lock) ({ __LOCK(lock, void); 1; }) -#define _raw_read_trylock(lock) ({ __LOCK(lock, void); 1; }) -#define _raw_write_trylock(lock) ({ __LOCK(lock, void); 1; }) -#define _raw_spin_trylock_bh(lock) ({ __LOCK_BH(lock, void); 1; }) + +static __always_inline int _raw_spin_trylock(raw_spinlock_t *lock) + __cond_acquires(true, lock) +{ + __LOCK(lock); + return 1; +} + +static __always_inline int _raw_spin_trylock_bh(raw_spinlock_t *lock) + __cond_acquires(true, lock) +{ + __LOCK_BH(lock); + return 1; +} + +static __always_inline int _raw_spin_trylock_irq(raw_spinlock_t *lock) + __cond_acquires(true, lock) +{ + __LOCK_IRQ(lock); + return 1; +} + +static __always_inline int _raw_spin_trylock_irqsave(raw_spinlock_t *lock, unsigned long *flags) + __cond_acquires(true, lock) +{ + __LOCK_IRQSAVE(lock, *(flags)); + return 1; +} + +static __always_inline int _raw_read_trylock(rwlock_t *lock) + __cond_acquires_shared(true, lock) +{ + __LOCK(lock, shared); + return 1; +} + +static __always_inline int _raw_write_trylock(rwlock_t *lock) + __cond_acquires(true, lock) +{ + __LOCK(lock); + return 1; +} + +static __always_inline int _raw_write_trylock_irqsave(rwlock_t *lock, unsigned long *flags) + __cond_acquires(true, lock) +{ + __LOCK_IRQSAVE(lock, *(flags)); + return 1; +} + #define _raw_spin_unlock(lock) __UNLOCK(lock) #define _raw_read_unlock(lock) __UNLOCK(lock, shared) #define _raw_write_unlock(lock) __UNLOCK(lock) diff --git a/include/linux/spinlock_rt.h b/include/linux/spinlock_rt.h index 6bab73ee1384..0a585768358f 100644 --- a/include/linux/spinlock_rt.h +++ b/include/linux/spinlock_rt.h @@ -37,8 +37,8 @@ extern void rt_spin_lock_nested(spinlock_t *lock, int subclass) __acquires(lock) extern void rt_spin_lock_nest_lock(spinlock_t *lock, struct lockdep_map *nest_lock) __acquires(lock); extern void rt_spin_unlock(spinlock_t *lock) __releases(lock); extern void rt_spin_lock_unlock(spinlock_t *lock); -extern int rt_spin_trylock_bh(spinlock_t *lock); -extern int rt_spin_trylock(spinlock_t *lock); +extern int rt_spin_trylock_bh(spinlock_t *lock) __cond_acquires(true, lock); +extern int rt_spin_trylock(spinlock_t *lock) __cond_acquires(true, lock); static __always_inline void spin_lock(spinlock_t *lock) __acquires(lock) @@ -130,21 +130,19 @@ static __always_inline void spin_unlock_irqrestore(spinlock_t *lock, rt_spin_unlock(lock); } -#define spin_trylock(lock) \ - __cond_lock(lock, rt_spin_trylock(lock)) +#define spin_trylock(lock) rt_spin_trylock(lock) -#define spin_trylock_bh(lock) \ - __cond_lock(lock, rt_spin_trylock_bh(lock)) +#define spin_trylock_bh(lock) rt_spin_trylock_bh(lock) -#define spin_trylock_irq(lock) \ - __cond_lock(lock, rt_spin_trylock(lock)) +#define spin_trylock_irq(lock) rt_spin_trylock(lock) -#define spin_trylock_irqsave(lock, flags) \ - __cond_lock(lock, ({ \ - typecheck(unsigned long, flags); \ - flags = 0; \ - rt_spin_trylock(lock); \ - })) +static __always_inline bool _spin_trylock_irqsave(spinlock_t *lock, unsigned long *flags) + __cond_acquires(true, lock) +{ + *flags = 0; + return rt_spin_trylock(lock); +} +#define spin_trylock_irqsave(lock, flags) _spin_trylock_irqsave(lock, &(flags)) #define spin_is_contended(lock) (((void)(lock), 0)) diff --git a/kernel/signal.c b/kernel/signal.c index e42b8bd6922f..d65d0fe24bfb 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -1355,8 +1355,8 @@ int zap_other_threads(struct task_struct *p) return count; } -struct sighand_struct *__lock_task_sighand(struct task_struct *tsk, - unsigned long *flags) +struct sighand_struct *lock_task_sighand(struct task_struct *tsk, + unsigned long *flags) { struct sighand_struct *sighand; diff --git a/kernel/time/posix-timers.c b/kernel/time/posix-timers.c index 80a8a09a21a0..413e2389f0a5 100644 --- a/kernel/time/posix-timers.c +++ b/kernel/time/posix-timers.c @@ -66,14 +66,7 @@ static const struct k_clock clock_realtime, clock_monotonic; #error "SIGEV_THREAD_ID must not share bit with other SIGEV values!" #endif -static struct k_itimer *__lock_timer(timer_t timer_id); - -#define lock_timer(tid) \ -({ struct k_itimer *__timr; \ - __cond_lock(&__timr->it_lock, __timr = __lock_timer(tid)); \ - __timr; \ -}) - +static struct k_itimer *lock_timer(timer_t timer_id); static inline void unlock_timer(struct k_itimer *timr) { if (likely((timr))) @@ -85,7 +78,7 @@ static inline void unlock_timer(struct k_itimer *timr) #define scoped_timer (scope) -DEFINE_CLASS(lock_timer, struct k_itimer *, unlock_timer(_T), __lock_timer(id), timer_t id); +DEFINE_CLASS(lock_timer, struct k_itimer *, unlock_timer(_T), lock_timer(id), timer_t id); DEFINE_CLASS_IS_COND_GUARD(lock_timer); static struct timer_hash_bucket *hash_bucket(struct signal_struct *sig, unsigned int nr) @@ -600,7 +593,7 @@ COMPAT_SYSCALL_DEFINE3(timer_create, clockid_t, which_clock, } #endif -static struct k_itimer *__lock_timer(timer_t timer_id) +static struct k_itimer *lock_timer(timer_t timer_id) { struct k_itimer *timr; diff --git a/lib/dec_and_lock.c b/lib/dec_and_lock.c index 1dcca8f2e194..8c7c398fd770 100644 --- a/lib/dec_and_lock.c +++ b/lib/dec_and_lock.c @@ -18,7 +18,7 @@ * because the spin-lock and the decrement must be * "atomic". */ -int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock) +int atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock) { /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */ if (atomic_add_unless(atomic, -1, 1)) @@ -32,7 +32,7 @@ int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock) return 0; } -EXPORT_SYMBOL(_atomic_dec_and_lock); +EXPORT_SYMBOL(atomic_dec_and_lock); int _atomic_dec_and_lock_irqsave(atomic_t *atomic, spinlock_t *lock, unsigned long *flags) @@ -50,7 +50,7 @@ int _atomic_dec_and_lock_irqsave(atomic_t *atomic, spinlock_t *lock, } EXPORT_SYMBOL(_atomic_dec_and_lock_irqsave); -int _atomic_dec_and_raw_lock(atomic_t *atomic, raw_spinlock_t *lock) +int atomic_dec_and_raw_lock(atomic_t *atomic, raw_spinlock_t *lock) { /* Subtract 1 from counter unless that drops it to 0 (ie. it was 1) */ if (atomic_add_unless(atomic, -1, 1)) @@ -63,7 +63,7 @@ int _atomic_dec_and_raw_lock(atomic_t *atomic, raw_spinlock_t *lock) raw_spin_unlock(lock); return 0; } -EXPORT_SYMBOL(_atomic_dec_and_raw_lock); +EXPORT_SYMBOL(atomic_dec_and_raw_lock); int _atomic_dec_and_raw_lock_irqsave(atomic_t *atomic, raw_spinlock_t *lock, unsigned long *flags) diff --git a/lib/lockref.c b/lib/lockref.c index 9210fc6ae714..5d8e3ef3860e 100644 --- a/lib/lockref.c +++ b/lib/lockref.c @@ -105,7 +105,6 @@ EXPORT_SYMBOL(lockref_put_return); * @lockref: pointer to lockref structure * Return: 1 if count updated successfully or 0 if count <= 1 and lock taken */ -#undef lockref_put_or_lock bool lockref_put_or_lock(struct lockref *lockref) { CMPXCHG_LOOP( diff --git a/mm/memory.c b/mm/memory.c index 2a55edc48a65..b751e1f85abc 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -2210,8 +2210,8 @@ static pmd_t *walk_to_pmd(struct mm_struct *mm, unsigned long addr) return pmd; } -pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr, - spinlock_t **ptl) +pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr, + spinlock_t **ptl) { pmd_t *pmd = walk_to_pmd(mm, addr); diff --git a/mm/pgtable-generic.c b/mm/pgtable-generic.c index d3aec7a9926a..af7966169d69 100644 --- a/mm/pgtable-generic.c +++ b/mm/pgtable-generic.c @@ -280,7 +280,7 @@ static unsigned long pmdp_get_lockless_start(void) { return 0; } static void pmdp_get_lockless_end(unsigned long irqflags) { } #endif -pte_t *___pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp) +pte_t *__pte_offset_map(pmd_t *pmd, unsigned long addr, pmd_t *pmdvalp) { unsigned long irqflags; pmd_t pmdval; @@ -332,13 +332,12 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, } /* - * pte_offset_map_lock(mm, pmd, addr, ptlp), and its internal implementation - * __pte_offset_map_lock() below, is usually called with the pmd pointer for - * addr, reached by walking down the mm's pgd, p4d, pud for addr: either while - * holding mmap_lock or vma lock for read or for write; or in truncate or rmap - * context, while holding file's i_mmap_lock or anon_vma lock for read (or for - * write). In a few cases, it may be used with pmd pointing to a pmd_t already - * copied to or constructed on the stack. + * pte_offset_map_lock(mm, pmd, addr, ptlp) is usually called with the pmd + * pointer for addr, reached by walking down the mm's pgd, p4d, pud for addr: + * either while holding mmap_lock or vma lock for read or for write; or in + * truncate or rmap context, while holding file's i_mmap_lock or anon_vma lock + * for read (or for write). In a few cases, it may be used with pmd pointing to + * a pmd_t already copied to or constructed on the stack. * * When successful, it returns the pte pointer for addr, with its page table * kmapped if necessary (when CONFIG_HIGHPTE), and locked against concurrent @@ -389,8 +388,8 @@ pte_t *pte_offset_map_rw_nolock(struct mm_struct *mm, pmd_t *pmd, * table, and may not use RCU at all: "outsiders" like khugepaged should avoid * pte_offset_map() and co once the vma is detached from mm or mm_users is zero. */ -pte_t *__pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, - unsigned long addr, spinlock_t **ptlp) +pte_t *pte_offset_map_lock(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr, spinlock_t **ptlp) { spinlock_t *ptl; pmd_t pmdval; diff --git a/tools/include/linux/compiler_types.h b/tools/include/linux/compiler_types.h index d09f9dc172a4..067a5b4e0f7b 100644 --- a/tools/include/linux/compiler_types.h +++ b/tools/include/linux/compiler_types.h @@ -20,7 +20,6 @@ # define __releases(x) __attribute__((context(x,1,0))) # define __acquire(x) __context__(x,1) # define __release(x) __context__(x,-1) -# define __cond_lock(x,c) ((c) ? ({ __acquire(x); 1; }) : 0) #else /* __CHECKER__ */ /* context/locking */ # define __must_hold(x) @@ -28,7 +27,6 @@ # define __releases(x) # define __acquire(x) (void)0 # define __release(x) (void)0 -# define __cond_lock(x,c) (c) #endif /* __CHECKER__ */ /* Compiler specific macros. */ From c237f1ceeef56fa101c2b599a00307b3d690801a Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:14 +0100 Subject: [PATCH 28/90] compiler-context-analysis: Introduce header suppressions While we can opt in individual subsystems which add the required annotations, such subsystems inevitably include headers from other subsystems which may not yet have the right annotations, which then result in false positive warnings. Making compatible by adding annotations across all common headers currently requires an excessive number of __no_context_analysis annotations, or carefully analyzing non-trivial cases to add the correct annotations. While this is desirable long-term, providing an incremental path causes less churn and headaches for maintainers not yet interested in dealing with such warnings. Rather than clutter headers unnecessary and mandate all subsystem maintainers to keep their headers working with context analysis, suppress all -Wthread-safety warnings in headers. Explicitly opt in headers with context-enabled primitives. With this in place, we can start enabling the analysis on more complex subsystems in subsequent changes. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-26-elver@google.com --- scripts/Makefile.context-analysis | 4 +++ scripts/context-analysis-suppression.txt | 32 ++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 scripts/context-analysis-suppression.txt diff --git a/scripts/Makefile.context-analysis b/scripts/Makefile.context-analysis index 70549f7fae1a..cd3bb49d3f09 100644 --- a/scripts/Makefile.context-analysis +++ b/scripts/Makefile.context-analysis @@ -4,4 +4,8 @@ context-analysis-cflags := -DWARN_CONTEXT_ANALYSIS \ -fexperimental-late-parse-attributes -Wthread-safety \ -Wthread-safety-pointer -Wthread-safety-beta +ifndef CONFIG_WARN_CONTEXT_ANALYSIS_ALL +context-analysis-cflags += --warning-suppression-mappings=$(srctree)/scripts/context-analysis-suppression.txt +endif + export CFLAGS_CONTEXT_ANALYSIS := $(context-analysis-cflags) diff --git a/scripts/context-analysis-suppression.txt b/scripts/context-analysis-suppression.txt new file mode 100644 index 000000000000..df25c3d07a5b --- /dev/null +++ b/scripts/context-analysis-suppression.txt @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# The suppressions file should only match common paths such as header files. +# For individual subsytems use Makefile directive CONTEXT_ANALYSIS := [yn]. +# +# The suppressions are ignored when CONFIG_WARN_CONTEXT_ANALYSIS_ALL is +# selected. + +[thread-safety] +src:*arch/*/include/* +src:*include/acpi/* +src:*include/asm-generic/* +src:*include/linux/* +src:*include/net/* + +# Opt-in headers: +src:*include/linux/bit_spinlock.h=emit +src:*include/linux/cleanup.h=emit +src:*include/linux/kref.h=emit +src:*include/linux/list*.h=emit +src:*include/linux/local_lock*.h=emit +src:*include/linux/lockdep.h=emit +src:*include/linux/mutex*.h=emit +src:*include/linux/rcupdate.h=emit +src:*include/linux/refcount.h=emit +src:*include/linux/rhashtable.h=emit +src:*include/linux/rwlock*.h=emit +src:*include/linux/rwsem.h=emit +src:*include/linux/seqlock*.h=emit +src:*include/linux/spinlock*.h=emit +src:*include/linux/srcu*.h=emit +src:*include/linux/ww_mutex.h=emit From 3635ad878242487fc3e8165d0329aedb118e4608 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:15 +0100 Subject: [PATCH 29/90] compiler: Let data_race() imply disabled context analysis Many patterns that involve data-racy accesses often deliberately ignore normal synchronization rules to avoid taking a lock. If we have a lock-guarded variable on which we do a lock-less data-racy access, rather than having to write context_unsafe(data_race(..)), simply make the data_race(..) macro imply context-unsafety. The data_race() macro already denotes the intent that something subtly unsafe is about to happen, so it should be clear enough as-is. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-27-elver@google.com --- include/linux/compiler.h | 2 ++ lib/test_context-analysis.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/include/linux/compiler.h b/include/linux/compiler.h index 04487c9bd751..110b28dfd1d1 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -190,7 +190,9 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, #define data_race(expr) \ ({ \ __kcsan_disable_current(); \ + disable_context_analysis(); \ auto __v = (expr); \ + enable_context_analysis(); \ __kcsan_enable_current(); \ __v; \ }) diff --git a/lib/test_context-analysis.c b/lib/test_context-analysis.c index 2dc404456497..1c5a381461fc 100644 --- a/lib/test_context-analysis.c +++ b/lib/test_context-analysis.c @@ -92,6 +92,8 @@ static void __used test_raw_spinlock_trylock_extra(struct test_raw_spinlock_data { unsigned long flags; + data_race(d->counter++); /* no warning */ + if (raw_spin_trylock_irq(&d->lock)) { d->counter++; raw_spin_unlock_irq(&d->lock); From 48eb4b9a3d5c305f93d3cfd0eddffa305884597f Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:16 +0100 Subject: [PATCH 30/90] MAINTAINERS: Add entry for Context Analysis Add entry for all new files added for Clang's context analysis. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Bart Van Assche Link: https://patch.msgid.link/20251219154418.3592607-28-elver@google.com --- MAINTAINERS | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/MAINTAINERS b/MAINTAINERS index 5b11839cba9d..2953b466107e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -6132,6 +6132,17 @@ M: Nelson Escobar S: Supported F: drivers/infiniband/hw/usnic/ +CLANG CONTEXT ANALYSIS +M: Marco Elver +R: Bart Van Assche +L: llvm@lists.linux.dev +S: Maintained +F: Documentation/dev-tools/context-analysis.rst +F: include/linux/compiler-context-analysis.h +F: lib/test_context-analysis.c +F: scripts/Makefile.context-analysis +F: scripts/context-analysis-suppression.txt + CLANG CONTROL FLOW INTEGRITY SUPPORT M: Sami Tolvanen M: Kees Cook From 0f5d764862aa7f50d77b8ea2b4f75a48a630487a Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:17 +0100 Subject: [PATCH 31/90] kfence: Enable context analysis Enable context analysis for the KFENCE subsystem. Notable, kfence_handle_page_fault() required minor restructure, which also fixed a subtle race; arguably that function is more readable now. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-29-elver@google.com --- mm/kfence/Makefile | 2 ++ mm/kfence/core.c | 20 +++++++++++++------- mm/kfence/kfence.h | 14 ++++++++------ mm/kfence/report.c | 4 ++-- 4 files changed, 25 insertions(+), 15 deletions(-) diff --git a/mm/kfence/Makefile b/mm/kfence/Makefile index 2de2a58d11a1..a503e83e74d9 100644 --- a/mm/kfence/Makefile +++ b/mm/kfence/Makefile @@ -1,5 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 +CONTEXT_ANALYSIS := y + obj-y := core.o report.o CFLAGS_kfence_test.o := -fno-omit-frame-pointer -fno-optimize-sibling-calls diff --git a/mm/kfence/core.c b/mm/kfence/core.c index 577a1699c553..ebf442fb2c2b 100644 --- a/mm/kfence/core.c +++ b/mm/kfence/core.c @@ -133,8 +133,8 @@ struct kfence_metadata *kfence_metadata __read_mostly; static struct kfence_metadata *kfence_metadata_init __read_mostly; /* Freelist with available objects. */ -static struct list_head kfence_freelist = LIST_HEAD_INIT(kfence_freelist); -static DEFINE_RAW_SPINLOCK(kfence_freelist_lock); /* Lock protecting freelist. */ +DEFINE_RAW_SPINLOCK(kfence_freelist_lock); /* Lock protecting freelist. */ +static struct list_head kfence_freelist __guarded_by(&kfence_freelist_lock) = LIST_HEAD_INIT(kfence_freelist); /* * The static key to set up a KFENCE allocation; or if static keys are not used @@ -254,6 +254,7 @@ static bool kfence_unprotect(unsigned long addr) } static inline unsigned long metadata_to_pageaddr(const struct kfence_metadata *meta) + __must_hold(&meta->lock) { unsigned long offset = (meta - kfence_metadata + 1) * PAGE_SIZE * 2; unsigned long pageaddr = (unsigned long)&__kfence_pool[offset]; @@ -289,6 +290,7 @@ static inline bool kfence_obj_allocated(const struct kfence_metadata *meta) static noinline void metadata_update_state(struct kfence_metadata *meta, enum kfence_object_state next, unsigned long *stack_entries, size_t num_stack_entries) + __must_hold(&meta->lock) { struct kfence_track *track = next == KFENCE_OBJECT_ALLOCATED ? &meta->alloc_track : &meta->free_track; @@ -486,7 +488,7 @@ static void *kfence_guarded_alloc(struct kmem_cache *cache, size_t size, gfp_t g alloc_covered_add(alloc_stack_hash, 1); /* Set required slab fields. */ - slab = virt_to_slab((void *)meta->addr); + slab = virt_to_slab(addr); slab->slab_cache = cache; slab->objects = 1; @@ -515,6 +517,7 @@ static void *kfence_guarded_alloc(struct kmem_cache *cache, size_t size, gfp_t g static void kfence_guarded_free(void *addr, struct kfence_metadata *meta, bool zombie) { struct kcsan_scoped_access assert_page_exclusive; + u32 alloc_stack_hash; unsigned long flags; bool init; @@ -547,9 +550,10 @@ static void kfence_guarded_free(void *addr, struct kfence_metadata *meta, bool z /* Mark the object as freed. */ metadata_update_state(meta, KFENCE_OBJECT_FREED, NULL, 0); init = slab_want_init_on_free(meta->cache); + alloc_stack_hash = meta->alloc_stack_hash; raw_spin_unlock_irqrestore(&meta->lock, flags); - alloc_covered_add(meta->alloc_stack_hash, -1); + alloc_covered_add(alloc_stack_hash, -1); /* Check canary bytes for memory corruption. */ check_canary(meta); @@ -594,6 +598,7 @@ static void rcu_guarded_free(struct rcu_head *h) * which partial initialization succeeded. */ static unsigned long kfence_init_pool(void) + __context_unsafe(/* constructor */) { unsigned long addr, start_pfn; int i; @@ -1220,6 +1225,7 @@ bool kfence_handle_page_fault(unsigned long addr, bool is_write, struct pt_regs { const int page_index = (addr - (unsigned long)__kfence_pool) / PAGE_SIZE; struct kfence_metadata *to_report = NULL; + unsigned long unprotected_page = 0; enum kfence_error_type error_type; unsigned long flags; @@ -1253,9 +1259,8 @@ bool kfence_handle_page_fault(unsigned long addr, bool is_write, struct pt_regs if (!to_report) goto out; - raw_spin_lock_irqsave(&to_report->lock, flags); - to_report->unprotected_page = addr; error_type = KFENCE_ERROR_OOB; + unprotected_page = addr; /* * If the object was freed before we took the look we can still @@ -1267,7 +1272,6 @@ bool kfence_handle_page_fault(unsigned long addr, bool is_write, struct pt_regs if (!to_report) goto out; - raw_spin_lock_irqsave(&to_report->lock, flags); error_type = KFENCE_ERROR_UAF; /* * We may race with __kfence_alloc(), and it is possible that a @@ -1279,6 +1283,8 @@ bool kfence_handle_page_fault(unsigned long addr, bool is_write, struct pt_regs out: if (to_report) { + raw_spin_lock_irqsave(&to_report->lock, flags); + to_report->unprotected_page = unprotected_page; kfence_report_error(addr, is_write, regs, to_report, error_type); raw_spin_unlock_irqrestore(&to_report->lock, flags); } else { diff --git a/mm/kfence/kfence.h b/mm/kfence/kfence.h index dfba5ea06b01..f9caea007246 100644 --- a/mm/kfence/kfence.h +++ b/mm/kfence/kfence.h @@ -34,6 +34,8 @@ /* Maximum stack depth for reports. */ #define KFENCE_STACK_DEPTH 64 +extern raw_spinlock_t kfence_freelist_lock; + /* KFENCE object states. */ enum kfence_object_state { KFENCE_OBJECT_UNUSED, /* Object is unused. */ @@ -53,7 +55,7 @@ struct kfence_track { /* KFENCE metadata per guarded allocation. */ struct kfence_metadata { - struct list_head list; /* Freelist node; access under kfence_freelist_lock. */ + struct list_head list __guarded_by(&kfence_freelist_lock); /* Freelist node. */ struct rcu_head rcu_head; /* For delayed freeing. */ /* @@ -91,13 +93,13 @@ struct kfence_metadata { * In case of an invalid access, the page that was unprotected; we * optimistically only store one address. */ - unsigned long unprotected_page; + unsigned long unprotected_page __guarded_by(&lock); /* Allocation and free stack information. */ - struct kfence_track alloc_track; - struct kfence_track free_track; + struct kfence_track alloc_track __guarded_by(&lock); + struct kfence_track free_track __guarded_by(&lock); /* For updating alloc_covered on frees. */ - u32 alloc_stack_hash; + u32 alloc_stack_hash __guarded_by(&lock); #ifdef CONFIG_MEMCG struct slabobj_ext obj_exts; #endif @@ -141,6 +143,6 @@ enum kfence_error_type { void kfence_report_error(unsigned long address, bool is_write, struct pt_regs *regs, const struct kfence_metadata *meta, enum kfence_error_type type); -void kfence_print_object(struct seq_file *seq, const struct kfence_metadata *meta); +void kfence_print_object(struct seq_file *seq, const struct kfence_metadata *meta) __must_hold(&meta->lock); #endif /* MM_KFENCE_KFENCE_H */ diff --git a/mm/kfence/report.c b/mm/kfence/report.c index 10e6802a2edf..787e87c26926 100644 --- a/mm/kfence/report.c +++ b/mm/kfence/report.c @@ -106,6 +106,7 @@ found: static void kfence_print_stack(struct seq_file *seq, const struct kfence_metadata *meta, bool show_alloc) + __must_hold(&meta->lock) { const struct kfence_track *track = show_alloc ? &meta->alloc_track : &meta->free_track; u64 ts_sec = track->ts_nsec; @@ -207,8 +208,6 @@ void kfence_report_error(unsigned long address, bool is_write, struct pt_regs *r if (WARN_ON(type != KFENCE_ERROR_INVALID && !meta)) return; - if (meta) - lockdep_assert_held(&meta->lock); /* * Because we may generate reports in printk-unfriendly parts of the * kernel, such as scheduler code, the use of printk() could deadlock. @@ -263,6 +262,7 @@ void kfence_report_error(unsigned long address, bool is_write, struct pt_regs *r stack_trace_print(stack_entries + skipnr, num_stack_entries - skipnr, 0); if (meta) { + lockdep_assert_held(&meta->lock); pr_err("\n"); kfence_print_object(NULL, meta); } From 6556fde265a7bd408ad8ff15ec08970f99f6201c Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:18 +0100 Subject: [PATCH 32/90] kcov: Enable context analysis Enable context analysis for the KCOV subsystem. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-30-elver@google.com --- kernel/Makefile | 2 ++ kernel/kcov.c | 36 +++++++++++++++++++++++++----------- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/kernel/Makefile b/kernel/Makefile index e83669841b8c..6785982013dc 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -43,6 +43,8 @@ KASAN_SANITIZE_kcov.o := n KCSAN_SANITIZE_kcov.o := n UBSAN_SANITIZE_kcov.o := n KMSAN_SANITIZE_kcov.o := n + +CONTEXT_ANALYSIS_kcov.o := y CFLAGS_kcov.o := $(call cc-option, -fno-conserve-stack) -fno-stack-protector obj-y += sched/ diff --git a/kernel/kcov.c b/kernel/kcov.c index 6563141f5de9..6cbc6e2d8aee 100644 --- a/kernel/kcov.c +++ b/kernel/kcov.c @@ -55,13 +55,13 @@ struct kcov { refcount_t refcount; /* The lock protects mode, size, area and t. */ spinlock_t lock; - enum kcov_mode mode; + enum kcov_mode mode __guarded_by(&lock); /* Size of arena (in long's). */ - unsigned int size; + unsigned int size __guarded_by(&lock); /* Coverage buffer shared with user space. */ - void *area; + void *area __guarded_by(&lock); /* Task for which we collect coverage, or NULL. */ - struct task_struct *t; + struct task_struct *t __guarded_by(&lock); /* Collecting coverage from remote (background) threads. */ bool remote; /* Size of remote area (in long's). */ @@ -391,6 +391,7 @@ void kcov_task_init(struct task_struct *t) } static void kcov_reset(struct kcov *kcov) + __must_hold(&kcov->lock) { kcov->t = NULL; kcov->mode = KCOV_MODE_INIT; @@ -400,6 +401,7 @@ static void kcov_reset(struct kcov *kcov) } static void kcov_remote_reset(struct kcov *kcov) + __must_hold(&kcov->lock) { int bkt; struct kcov_remote *remote; @@ -419,6 +421,7 @@ static void kcov_remote_reset(struct kcov *kcov) } static void kcov_disable(struct task_struct *t, struct kcov *kcov) + __must_hold(&kcov->lock) { kcov_task_reset(t); if (kcov->remote) @@ -435,8 +438,11 @@ static void kcov_get(struct kcov *kcov) static void kcov_put(struct kcov *kcov) { if (refcount_dec_and_test(&kcov->refcount)) { - kcov_remote_reset(kcov); - vfree(kcov->area); + /* Context-safety: no references left, object being destroyed. */ + context_unsafe( + kcov_remote_reset(kcov); + vfree(kcov->area); + ); kfree(kcov); } } @@ -491,6 +497,7 @@ static int kcov_mmap(struct file *filep, struct vm_area_struct *vma) unsigned long size, off; struct page *page; unsigned long flags; + void *area; spin_lock_irqsave(&kcov->lock, flags); size = kcov->size * sizeof(unsigned long); @@ -499,10 +506,11 @@ static int kcov_mmap(struct file *filep, struct vm_area_struct *vma) res = -EINVAL; goto exit; } + area = kcov->area; spin_unlock_irqrestore(&kcov->lock, flags); vm_flags_set(vma, VM_DONTEXPAND); for (off = 0; off < size; off += PAGE_SIZE) { - page = vmalloc_to_page(kcov->area + off); + page = vmalloc_to_page(area + off); res = vm_insert_page(vma, vma->vm_start + off, page); if (res) { pr_warn_once("kcov: vm_insert_page() failed\n"); @@ -522,10 +530,10 @@ static int kcov_open(struct inode *inode, struct file *filep) kcov = kzalloc(sizeof(*kcov), GFP_KERNEL); if (!kcov) return -ENOMEM; + spin_lock_init(&kcov->lock); kcov->mode = KCOV_MODE_DISABLED; kcov->sequence = 1; refcount_set(&kcov->refcount, 1); - spin_lock_init(&kcov->lock); filep->private_data = kcov; return nonseekable_open(inode, filep); } @@ -556,6 +564,7 @@ static int kcov_get_mode(unsigned long arg) * vmalloc fault handling path is instrumented. */ static void kcov_fault_in_area(struct kcov *kcov) + __must_hold(&kcov->lock) { unsigned long stride = PAGE_SIZE / sizeof(unsigned long); unsigned long *area = kcov->area; @@ -584,6 +593,7 @@ static inline bool kcov_check_handle(u64 handle, bool common_valid, static int kcov_ioctl_locked(struct kcov *kcov, unsigned int cmd, unsigned long arg) + __must_hold(&kcov->lock) { struct task_struct *t; unsigned long flags, unused; @@ -814,6 +824,7 @@ static inline bool kcov_mode_enabled(unsigned int mode) } static void kcov_remote_softirq_start(struct task_struct *t) + __must_hold(&kcov_percpu_data.lock) { struct kcov_percpu_data *data = this_cpu_ptr(&kcov_percpu_data); unsigned int mode; @@ -831,6 +842,7 @@ static void kcov_remote_softirq_start(struct task_struct *t) } static void kcov_remote_softirq_stop(struct task_struct *t) + __must_hold(&kcov_percpu_data.lock) { struct kcov_percpu_data *data = this_cpu_ptr(&kcov_percpu_data); @@ -896,10 +908,12 @@ void kcov_remote_start(u64 handle) /* Put in kcov_remote_stop(). */ kcov_get(kcov); /* - * Read kcov fields before unlock to prevent races with - * KCOV_DISABLE / kcov_remote_reset(). + * Read kcov fields before unlocking kcov_remote_lock to prevent races + * with KCOV_DISABLE and kcov_remote_reset(); cannot acquire kcov->lock + * here, because it might lead to deadlock given kcov_remote_lock is + * acquired _after_ kcov->lock elsewhere. */ - mode = kcov->mode; + mode = context_unsafe(kcov->mode); sequence = kcov->sequence; if (in_task()) { size = kcov->remote_size; From 0eaa911f890812a7868a44bbfd656636b2c7caf8 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:19 +0100 Subject: [PATCH 33/90] kcsan: Enable context analysis Enable context analysis for the KCSAN subsystem. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-31-elver@google.com --- kernel/kcsan/Makefile | 2 ++ kernel/kcsan/report.c | 11 ++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/kernel/kcsan/Makefile b/kernel/kcsan/Makefile index a45f3dfc8d14..824f30c93252 100644 --- a/kernel/kcsan/Makefile +++ b/kernel/kcsan/Makefile @@ -1,4 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 +CONTEXT_ANALYSIS := y + KCSAN_SANITIZE := n KCOV_INSTRUMENT := n UBSAN_SANITIZE := n diff --git a/kernel/kcsan/report.c b/kernel/kcsan/report.c index e95ce7d7a76e..11a48b78f8d1 100644 --- a/kernel/kcsan/report.c +++ b/kernel/kcsan/report.c @@ -116,6 +116,7 @@ static DEFINE_RAW_SPINLOCK(report_lock); * been reported since (now - KCSAN_REPORT_ONCE_IN_MS). */ static bool rate_limit_report(unsigned long frame1, unsigned long frame2) + __must_hold(&report_lock) { struct report_time *use_entry = &report_times[0]; unsigned long invalid_before; @@ -366,6 +367,7 @@ static int sym_strcmp(void *addr1, void *addr2) static void print_stack_trace(unsigned long stack_entries[], int num_entries, unsigned long reordered_to) + __must_hold(&report_lock) { stack_trace_print(stack_entries, num_entries, 0); if (reordered_to) @@ -373,6 +375,7 @@ print_stack_trace(unsigned long stack_entries[], int num_entries, unsigned long } static void print_verbose_info(struct task_struct *task) + __must_hold(&report_lock) { if (!task) return; @@ -389,6 +392,7 @@ static void print_report(enum kcsan_value_change value_change, const struct access_info *ai, struct other_info *other_info, u64 old, u64 new, u64 mask) + __must_hold(&report_lock) { unsigned long reordered_to = 0; unsigned long stack_entries[NUM_STACK_ENTRIES] = { 0 }; @@ -496,6 +500,7 @@ static void print_report(enum kcsan_value_change value_change, } static void release_report(unsigned long *flags, struct other_info *other_info) + __releases(&report_lock) { /* * Use size to denote valid/invalid, since KCSAN entirely ignores @@ -507,13 +512,11 @@ static void release_report(unsigned long *flags, struct other_info *other_info) /* * Sets @other_info->task and awaits consumption of @other_info. - * - * Precondition: report_lock is held. - * Postcondition: report_lock is held. */ static void set_other_info_task_blocking(unsigned long *flags, const struct access_info *ai, struct other_info *other_info) + __must_hold(&report_lock) { /* * We may be instrumenting a code-path where current->state is already @@ -572,6 +575,7 @@ static void set_other_info_task_blocking(unsigned long *flags, static void prepare_report_producer(unsigned long *flags, const struct access_info *ai, struct other_info *other_info) + __must_not_hold(&report_lock) { raw_spin_lock_irqsave(&report_lock, *flags); @@ -603,6 +607,7 @@ static void prepare_report_producer(unsigned long *flags, static bool prepare_report_consumer(unsigned long *flags, const struct access_info *ai, struct other_info *other_info) + __cond_acquires(true, &report_lock) { raw_spin_lock_irqsave(&report_lock, *flags); From c3d3023f1cf3de10f2d2f83b0d011fa7cab16cf0 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:20 +0100 Subject: [PATCH 34/90] stackdepot: Enable context analysis Enable context analysis for stackdepot. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-32-elver@google.com --- lib/Makefile | 1 + lib/stackdepot.c | 20 ++++++++++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/lib/Makefile b/lib/Makefile index 89defefbf6c0..e755eee4e76f 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -250,6 +250,7 @@ obj-$(CONFIG_POLYNOMIAL) += polynomial.o # Prevent the compiler from calling builtins like memcmp() or bcmp() from this # file. CFLAGS_stackdepot.o += -fno-builtin +CONTEXT_ANALYSIS_stackdepot.o := y obj-$(CONFIG_STACKDEPOT) += stackdepot.o KASAN_SANITIZE_stackdepot.o := n # In particular, instrumenting stackdepot.c with KMSAN will result in infinite diff --git a/lib/stackdepot.c b/lib/stackdepot.c index de0b0025af2b..166f50ad8391 100644 --- a/lib/stackdepot.c +++ b/lib/stackdepot.c @@ -61,18 +61,18 @@ static unsigned int stack_bucket_number_order; /* Hash mask for indexing the table. */ static unsigned int stack_hash_mask; +/* The lock must be held when performing pool or freelist modifications. */ +static DEFINE_RAW_SPINLOCK(pool_lock); /* Array of memory regions that store stack records. */ -static void **stack_pools; +static void **stack_pools __pt_guarded_by(&pool_lock); /* Newly allocated pool that is not yet added to stack_pools. */ static void *new_pool; /* Number of pools in stack_pools. */ static int pools_num; /* Offset to the unused space in the currently used pool. */ -static size_t pool_offset = DEPOT_POOL_SIZE; +static size_t pool_offset __guarded_by(&pool_lock) = DEPOT_POOL_SIZE; /* Freelist of stack records within stack_pools. */ -static LIST_HEAD(free_stacks); -/* The lock must be held when performing pool or freelist modifications. */ -static DEFINE_RAW_SPINLOCK(pool_lock); +static __guarded_by(&pool_lock) LIST_HEAD(free_stacks); /* Statistics counters for debugfs. */ enum depot_counter_id { @@ -291,6 +291,7 @@ EXPORT_SYMBOL_GPL(stack_depot_init); * Initializes new stack pool, and updates the list of pools. */ static bool depot_init_pool(void **prealloc) + __must_hold(&pool_lock) { lockdep_assert_held(&pool_lock); @@ -338,6 +339,7 @@ static bool depot_init_pool(void **prealloc) /* Keeps the preallocated memory to be used for a new stack depot pool. */ static void depot_keep_new_pool(void **prealloc) + __must_hold(&pool_lock) { lockdep_assert_held(&pool_lock); @@ -357,6 +359,7 @@ static void depot_keep_new_pool(void **prealloc) * the current pre-allocation. */ static struct stack_record *depot_pop_free_pool(void **prealloc, size_t size) + __must_hold(&pool_lock) { struct stack_record *stack; void *current_pool; @@ -391,6 +394,7 @@ static struct stack_record *depot_pop_free_pool(void **prealloc, size_t size) /* Try to find next free usable entry from the freelist. */ static struct stack_record *depot_pop_free(void) + __must_hold(&pool_lock) { struct stack_record *stack; @@ -428,6 +432,7 @@ static inline size_t depot_stack_record_size(struct stack_record *s, unsigned in /* Allocates a new stack in a stack depot pool. */ static struct stack_record * depot_alloc_stack(unsigned long *entries, unsigned int nr_entries, u32 hash, depot_flags_t flags, void **prealloc) + __must_hold(&pool_lock) { struct stack_record *stack = NULL; size_t record_size; @@ -486,6 +491,7 @@ depot_alloc_stack(unsigned long *entries, unsigned int nr_entries, u32 hash, dep } static struct stack_record *depot_fetch_stack(depot_stack_handle_t handle) + __must_not_hold(&pool_lock) { const int pools_num_cached = READ_ONCE(pools_num); union handle_parts parts = { .handle = handle }; @@ -502,7 +508,8 @@ static struct stack_record *depot_fetch_stack(depot_stack_handle_t handle) return NULL; } - pool = stack_pools[pool_index]; + /* @pool_index either valid, or user passed in corrupted value. */ + pool = context_unsafe(stack_pools[pool_index]); if (WARN_ON(!pool)) return NULL; @@ -515,6 +522,7 @@ static struct stack_record *depot_fetch_stack(depot_stack_handle_t handle) /* Links stack into the freelist. */ static void depot_free_stack(struct stack_record *stack) + __must_not_hold(&pool_lock) { unsigned long flags; From 322366b8f13a8cafe169dc1dc6f6ec0d82ff8734 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:21 +0100 Subject: [PATCH 35/90] rhashtable: Enable context analysis Enable context analysis for rhashtable, which was used as an initial test as it contains a combination of RCU, mutex, and bit_spinlock usage. Users of rhashtable now also benefit from annotations on the API, which will now warn if the RCU read lock is not held where required. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-33-elver@google.com --- include/linux/rhashtable.h | 16 +++++++++++++--- lib/Makefile | 2 ++ lib/rhashtable.c | 5 +++-- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/include/linux/rhashtable.h b/include/linux/rhashtable.h index 08e664b21f5a..133ccb39137a 100644 --- a/include/linux/rhashtable.h +++ b/include/linux/rhashtable.h @@ -245,16 +245,17 @@ void *rhashtable_insert_slow(struct rhashtable *ht, const void *key, void rhashtable_walk_enter(struct rhashtable *ht, struct rhashtable_iter *iter); void rhashtable_walk_exit(struct rhashtable_iter *iter); -int rhashtable_walk_start_check(struct rhashtable_iter *iter) __acquires(RCU); +int rhashtable_walk_start_check(struct rhashtable_iter *iter) __acquires_shared(RCU); static inline void rhashtable_walk_start(struct rhashtable_iter *iter) + __acquires_shared(RCU) { (void)rhashtable_walk_start_check(iter); } void *rhashtable_walk_next(struct rhashtable_iter *iter); void *rhashtable_walk_peek(struct rhashtable_iter *iter); -void rhashtable_walk_stop(struct rhashtable_iter *iter) __releases(RCU); +void rhashtable_walk_stop(struct rhashtable_iter *iter) __releases_shared(RCU); void rhashtable_free_and_destroy(struct rhashtable *ht, void (*free_fn)(void *ptr, void *arg), @@ -325,6 +326,7 @@ static inline struct rhash_lock_head __rcu **rht_bucket_insert( static inline unsigned long rht_lock(struct bucket_table *tbl, struct rhash_lock_head __rcu **bkt) + __acquires(__bitlock(0, bkt)) { unsigned long flags; @@ -337,6 +339,7 @@ static inline unsigned long rht_lock(struct bucket_table *tbl, static inline unsigned long rht_lock_nested(struct bucket_table *tbl, struct rhash_lock_head __rcu **bucket, unsigned int subclass) + __acquires(__bitlock(0, bucket)) { unsigned long flags; @@ -349,6 +352,7 @@ static inline unsigned long rht_lock_nested(struct bucket_table *tbl, static inline void rht_unlock(struct bucket_table *tbl, struct rhash_lock_head __rcu **bkt, unsigned long flags) + __releases(__bitlock(0, bkt)) { lock_map_release(&tbl->dep_map); bit_spin_unlock(0, (unsigned long *)bkt); @@ -424,13 +428,14 @@ static inline void rht_assign_unlock(struct bucket_table *tbl, struct rhash_lock_head __rcu **bkt, struct rhash_head *obj, unsigned long flags) + __releases(__bitlock(0, bkt)) { if (rht_is_a_nulls(obj)) obj = NULL; lock_map_release(&tbl->dep_map); rcu_assign_pointer(*bkt, (void *)obj); preempt_enable(); - __release(bitlock); + __release(__bitlock(0, bkt)); local_irq_restore(flags); } @@ -612,6 +617,7 @@ static __always_inline struct rhash_head *__rhashtable_lookup( struct rhashtable *ht, const void *key, const struct rhashtable_params params, const enum rht_lookup_freq freq) + __must_hold_shared(RCU) { struct rhashtable_compare_arg arg = { .ht = ht, @@ -666,6 +672,7 @@ restart: static __always_inline void *rhashtable_lookup( struct rhashtable *ht, const void *key, const struct rhashtable_params params) + __must_hold_shared(RCU) { struct rhash_head *he = __rhashtable_lookup(ht, key, params, RHT_LOOKUP_NORMAL); @@ -676,6 +683,7 @@ static __always_inline void *rhashtable_lookup( static __always_inline void *rhashtable_lookup_likely( struct rhashtable *ht, const void *key, const struct rhashtable_params params) + __must_hold_shared(RCU) { struct rhash_head *he = __rhashtable_lookup(ht, key, params, RHT_LOOKUP_LIKELY); @@ -727,6 +735,7 @@ static __always_inline void *rhashtable_lookup_fast( static __always_inline struct rhlist_head *rhltable_lookup( struct rhltable *hlt, const void *key, const struct rhashtable_params params) + __must_hold_shared(RCU) { struct rhash_head *he = __rhashtable_lookup(&hlt->ht, key, params, RHT_LOOKUP_NORMAL); @@ -737,6 +746,7 @@ static __always_inline struct rhlist_head *rhltable_lookup( static __always_inline struct rhlist_head *rhltable_lookup_likely( struct rhltable *hlt, const void *key, const struct rhashtable_params params) + __must_hold_shared(RCU) { struct rhash_head *he = __rhashtable_lookup(&hlt->ht, key, params, RHT_LOOKUP_LIKELY); diff --git a/lib/Makefile b/lib/Makefile index e755eee4e76f..22d8742bba57 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -50,6 +50,8 @@ lib-$(CONFIG_MIN_HEAP) += min_heap.o lib-y += kobject.o klist.o obj-y += lockref.o +CONTEXT_ANALYSIS_rhashtable.o := y + obj-y += bcd.o sort.o parser.o debug_locks.o random32.o \ bust_spinlocks.o kasprintf.o bitmap.o scatterlist.o \ list_sort.o uuid.o iov_iter.o clz_ctz.o \ diff --git a/lib/rhashtable.c b/lib/rhashtable.c index fde0f0e556f8..6074ed5f66f3 100644 --- a/lib/rhashtable.c +++ b/lib/rhashtable.c @@ -358,6 +358,7 @@ static int rhashtable_rehash_table(struct rhashtable *ht) static int rhashtable_rehash_alloc(struct rhashtable *ht, struct bucket_table *old_tbl, unsigned int size) + __must_hold(&ht->mutex) { struct bucket_table *new_tbl; int err; @@ -392,6 +393,7 @@ static int rhashtable_rehash_alloc(struct rhashtable *ht, * bucket locks or concurrent RCU protected lookups and traversals. */ static int rhashtable_shrink(struct rhashtable *ht) + __must_hold(&ht->mutex) { struct bucket_table *old_tbl = rht_dereference(ht->tbl, ht); unsigned int nelems = atomic_read(&ht->nelems); @@ -724,7 +726,7 @@ EXPORT_SYMBOL_GPL(rhashtable_walk_exit); * resize events and always continue. */ int rhashtable_walk_start_check(struct rhashtable_iter *iter) - __acquires(RCU) + __acquires_shared(RCU) { struct rhashtable *ht = iter->ht; bool rhlist = ht->rhlist; @@ -940,7 +942,6 @@ EXPORT_SYMBOL_GPL(rhashtable_walk_peek); * hash table. */ void rhashtable_walk_stop(struct rhashtable_iter *iter) - __releases(RCU) { struct rhashtable *ht; struct bucket_table *tbl = iter->walker.tbl; From 8ec56d9aaba9667b0c6429de7aeb4ec691944a5e Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:22 +0100 Subject: [PATCH 36/90] printk: Move locking annotation to printk.c With Sparse support gone, Clang is a bit more strict and warns: ./include/linux/console.h:492:50: error: use of undeclared identifier 'console_mutex' 492 | extern void console_list_unlock(void) __releases(console_mutex); Since it does not make sense to make console_mutex itself global, move the annotation to printk.c. Context analysis remains disabled for printk.c. This is needed to enable context analysis for modules that include . Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-34-elver@google.com --- include/linux/console.h | 4 ++-- kernel/printk/printk.c | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/include/linux/console.h b/include/linux/console.h index fc9f5c5c1b04..f882833bedf0 100644 --- a/include/linux/console.h +++ b/include/linux/console.h @@ -492,8 +492,8 @@ static inline bool console_srcu_read_lock_is_held(void) extern int console_srcu_read_lock(void); extern void console_srcu_read_unlock(int cookie); -extern void console_list_lock(void) __acquires(console_mutex); -extern void console_list_unlock(void) __releases(console_mutex); +extern void console_list_lock(void); +extern void console_list_unlock(void); extern struct hlist_head console_list; diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 1d765ad242b8..37d16ef27f13 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -245,6 +245,7 @@ int devkmsg_sysctl_set_loglvl(const struct ctl_table *table, int write, * For console list or console->flags updates */ void console_list_lock(void) + __acquires(&console_mutex) { /* * In unregister_console() and console_force_preferred_locked(), @@ -269,6 +270,7 @@ EXPORT_SYMBOL(console_list_lock); * Counterpart to console_list_lock() */ void console_list_unlock(void) + __releases(&console_mutex) { mutex_unlock(&console_mutex); } From 87335b61a23bd10e4aec132bd3a48a009d406973 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:23 +0100 Subject: [PATCH 37/90] security/tomoyo: Enable context analysis Enable context analysis for security/tomoyo. This demonstrates a larger conversion to use Clang's context analysis. The benefit is additional static checking of locking rules, along with better documentation. Tomoyo makes use of several synchronization primitives, yet its clear design made it relatively straightforward to enable context analysis. One notable finding was: security/tomoyo/gc.c:664:20: error: reading variable 'write_buf' requires holding mutex '&tomoyo_io_buffer::io_sem' 664 | is_write = head->write_buf != NULL; For which Tetsuo writes: "Good catch. This should be data_race(), for tomoyo_write_control() might concurrently update head->write_buf from non-NULL to non-NULL with head->io_sem held." Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-35-elver@google.com --- security/tomoyo/Makefile | 2 + security/tomoyo/common.c | 52 ++++++++++++++++++++++++-- security/tomoyo/common.h | 77 ++++++++++++++++++++------------------- security/tomoyo/domain.c | 1 + security/tomoyo/environ.c | 1 + security/tomoyo/file.c | 5 +++ security/tomoyo/gc.c | 28 ++++++++++---- security/tomoyo/mount.c | 2 + security/tomoyo/network.c | 3 ++ 9 files changed, 122 insertions(+), 49 deletions(-) diff --git a/security/tomoyo/Makefile b/security/tomoyo/Makefile index 55c67b9846a9..e3c0f853aa3b 100644 --- a/security/tomoyo/Makefile +++ b/security/tomoyo/Makefile @@ -1,4 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 +CONTEXT_ANALYSIS := y + obj-y = audit.o common.o condition.o domain.o environ.o file.o gc.o group.o load_policy.o memory.o mount.o network.o realpath.o securityfs_if.o tomoyo.o util.o targets += builtin-policy.h diff --git a/security/tomoyo/common.c b/security/tomoyo/common.c index 0f78898bce09..86ce56c32d37 100644 --- a/security/tomoyo/common.c +++ b/security/tomoyo/common.c @@ -268,6 +268,7 @@ static void tomoyo_io_printf(struct tomoyo_io_buffer *head, const char *fmt, */ static void tomoyo_io_printf(struct tomoyo_io_buffer *head, const char *fmt, ...) + __must_hold(&head->io_sem) { va_list args; size_t len; @@ -416,8 +417,9 @@ static void tomoyo_print_name_union_quoted(struct tomoyo_io_buffer *head, * * Returns nothing. */ -static void tomoyo_print_number_union_nospace -(struct tomoyo_io_buffer *head, const struct tomoyo_number_union *ptr) +static void +tomoyo_print_number_union_nospace(struct tomoyo_io_buffer *head, const struct tomoyo_number_union *ptr) + __must_hold(&head->io_sem) { if (ptr->group) { tomoyo_set_string(head, "@"); @@ -466,6 +468,7 @@ static void tomoyo_print_number_union_nospace */ static void tomoyo_print_number_union(struct tomoyo_io_buffer *head, const struct tomoyo_number_union *ptr) + __must_hold(&head->io_sem) { tomoyo_set_space(head); tomoyo_print_number_union_nospace(head, ptr); @@ -664,6 +667,7 @@ static int tomoyo_set_mode(char *name, const char *value, * Returns 0 on success, negative value otherwise. */ static int tomoyo_write_profile(struct tomoyo_io_buffer *head) + __must_hold(&head->io_sem) { char *data = head->write_buf; unsigned int i; @@ -719,6 +723,7 @@ static int tomoyo_write_profile(struct tomoyo_io_buffer *head) * Caller prints functionality's name. */ static void tomoyo_print_config(struct tomoyo_io_buffer *head, const u8 config) + __must_hold(&head->io_sem) { tomoyo_io_printf(head, "={ mode=%s grant_log=%s reject_log=%s }\n", tomoyo_mode[config & 3], @@ -734,6 +739,7 @@ static void tomoyo_print_config(struct tomoyo_io_buffer *head, const u8 config) * Returns nothing. */ static void tomoyo_read_profile(struct tomoyo_io_buffer *head) + __must_hold(&head->io_sem) { u8 index; struct tomoyo_policy_namespace *ns = @@ -852,6 +858,7 @@ static bool tomoyo_same_manager(const struct tomoyo_acl_head *a, */ static int tomoyo_update_manager_entry(const char *manager, const bool is_delete) + __must_hold_shared(&tomoyo_ss) { struct tomoyo_manager e = { }; struct tomoyo_acl_param param = { @@ -883,6 +890,8 @@ static int tomoyo_update_manager_entry(const char *manager, * Caller holds tomoyo_read_lock(). */ static int tomoyo_write_manager(struct tomoyo_io_buffer *head) + __must_hold_shared(&tomoyo_ss) + __must_hold(&head->io_sem) { char *data = head->write_buf; @@ -901,6 +910,7 @@ static int tomoyo_write_manager(struct tomoyo_io_buffer *head) * Caller holds tomoyo_read_lock(). */ static void tomoyo_read_manager(struct tomoyo_io_buffer *head) + __must_hold_shared(&tomoyo_ss) { if (head->r.eof) return; @@ -927,6 +937,7 @@ static void tomoyo_read_manager(struct tomoyo_io_buffer *head) * Caller holds tomoyo_read_lock(). */ static bool tomoyo_manager(void) + __must_hold_shared(&tomoyo_ss) { struct tomoyo_manager *ptr; const char *exe; @@ -981,6 +992,8 @@ static struct tomoyo_domain_info *tomoyo_find_domain_by_qid */ static bool tomoyo_select_domain(struct tomoyo_io_buffer *head, const char *data) + __must_hold_shared(&tomoyo_ss) + __must_hold(&head->io_sem) { unsigned int pid; struct tomoyo_domain_info *domain = NULL; @@ -1051,6 +1064,7 @@ static bool tomoyo_same_task_acl(const struct tomoyo_acl_info *a, * Caller holds tomoyo_read_lock(). */ static int tomoyo_write_task(struct tomoyo_acl_param *param) + __must_hold_shared(&tomoyo_ss) { int error = -EINVAL; @@ -1079,6 +1093,7 @@ static int tomoyo_write_task(struct tomoyo_acl_param *param) * Caller holds tomoyo_read_lock(). */ static int tomoyo_delete_domain(char *domainname) + __must_hold_shared(&tomoyo_ss) { struct tomoyo_domain_info *domain; struct tomoyo_path_info name; @@ -1118,6 +1133,7 @@ static int tomoyo_delete_domain(char *domainname) static int tomoyo_write_domain2(struct tomoyo_policy_namespace *ns, struct list_head *list, char *data, const bool is_delete) + __must_hold_shared(&tomoyo_ss) { struct tomoyo_acl_param param = { .ns = ns, @@ -1162,6 +1178,8 @@ const char * const tomoyo_dif[TOMOYO_MAX_DOMAIN_INFO_FLAGS] = { * Caller holds tomoyo_read_lock(). */ static int tomoyo_write_domain(struct tomoyo_io_buffer *head) + __must_hold_shared(&tomoyo_ss) + __must_hold(&head->io_sem) { char *data = head->write_buf; struct tomoyo_policy_namespace *ns; @@ -1223,6 +1241,7 @@ static int tomoyo_write_domain(struct tomoyo_io_buffer *head) */ static bool tomoyo_print_condition(struct tomoyo_io_buffer *head, const struct tomoyo_condition *cond) + __must_hold(&head->io_sem) { switch (head->r.cond_step) { case 0: @@ -1364,6 +1383,7 @@ static bool tomoyo_print_condition(struct tomoyo_io_buffer *head, */ static void tomoyo_set_group(struct tomoyo_io_buffer *head, const char *category) + __must_hold(&head->io_sem) { if (head->type == TOMOYO_EXCEPTIONPOLICY) { tomoyo_print_namespace(head); @@ -1383,6 +1403,7 @@ static void tomoyo_set_group(struct tomoyo_io_buffer *head, */ static bool tomoyo_print_entry(struct tomoyo_io_buffer *head, struct tomoyo_acl_info *acl) + __must_hold(&head->io_sem) { const u8 acl_type = acl->type; bool first = true; @@ -1588,6 +1609,8 @@ print_cond_part: */ static bool tomoyo_read_domain2(struct tomoyo_io_buffer *head, struct list_head *list) + __must_hold_shared(&tomoyo_ss) + __must_hold(&head->io_sem) { list_for_each_cookie(head->r.acl, list) { struct tomoyo_acl_info *ptr = @@ -1608,6 +1631,8 @@ static bool tomoyo_read_domain2(struct tomoyo_io_buffer *head, * Caller holds tomoyo_read_lock(). */ static void tomoyo_read_domain(struct tomoyo_io_buffer *head) + __must_hold_shared(&tomoyo_ss) + __must_hold(&head->io_sem) { if (head->r.eof) return; @@ -1686,6 +1711,7 @@ static int tomoyo_write_pid(struct tomoyo_io_buffer *head) * using read()/write() interface rather than sysctl() interface. */ static void tomoyo_read_pid(struct tomoyo_io_buffer *head) + __must_hold(&head->io_sem) { char *buf = head->write_buf; bool global_pid = false; @@ -1746,6 +1772,8 @@ static const char *tomoyo_group_name[TOMOYO_MAX_GROUP] = { * Caller holds tomoyo_read_lock(). */ static int tomoyo_write_exception(struct tomoyo_io_buffer *head) + __must_hold_shared(&tomoyo_ss) + __must_hold(&head->io_sem) { const bool is_delete = head->w.is_delete; struct tomoyo_acl_param param = { @@ -1787,6 +1815,8 @@ static int tomoyo_write_exception(struct tomoyo_io_buffer *head) * Caller holds tomoyo_read_lock(). */ static bool tomoyo_read_group(struct tomoyo_io_buffer *head, const int idx) + __must_hold_shared(&tomoyo_ss) + __must_hold(&head->io_sem) { struct tomoyo_policy_namespace *ns = container_of(head->r.ns, typeof(*ns), namespace_list); @@ -1846,6 +1876,7 @@ static bool tomoyo_read_group(struct tomoyo_io_buffer *head, const int idx) * Caller holds tomoyo_read_lock(). */ static bool tomoyo_read_policy(struct tomoyo_io_buffer *head, const int idx) + __must_hold_shared(&tomoyo_ss) { struct tomoyo_policy_namespace *ns = container_of(head->r.ns, typeof(*ns), namespace_list); @@ -1906,6 +1937,8 @@ static bool tomoyo_read_policy(struct tomoyo_io_buffer *head, const int idx) * Caller holds tomoyo_read_lock(). */ static void tomoyo_read_exception(struct tomoyo_io_buffer *head) + __must_hold_shared(&tomoyo_ss) + __must_hold(&head->io_sem) { struct tomoyo_policy_namespace *ns = container_of(head->r.ns, typeof(*ns), namespace_list); @@ -2097,6 +2130,7 @@ flush: * Returns nothing. */ static void tomoyo_add_entry(struct tomoyo_domain_info *domain, char *header) + __must_hold_shared(&tomoyo_ss) { char *buffer; char *realpath = NULL; @@ -2301,6 +2335,7 @@ static __poll_t tomoyo_poll_query(struct file *file, poll_table *wait) * @head: Pointer to "struct tomoyo_io_buffer". */ static void tomoyo_read_query(struct tomoyo_io_buffer *head) + __must_hold(&head->io_sem) { struct list_head *tmp; unsigned int pos = 0; @@ -2362,6 +2397,7 @@ static void tomoyo_read_query(struct tomoyo_io_buffer *head) * Returns 0 on success, -EINVAL otherwise. */ static int tomoyo_write_answer(struct tomoyo_io_buffer *head) + __must_hold(&head->io_sem) { char *data = head->write_buf; struct list_head *tmp; @@ -2401,6 +2437,7 @@ static int tomoyo_write_answer(struct tomoyo_io_buffer *head) * Returns version information. */ static void tomoyo_read_version(struct tomoyo_io_buffer *head) + __must_hold(&head->io_sem) { if (!head->r.eof) { tomoyo_io_printf(head, "2.6.0"); @@ -2449,6 +2486,7 @@ void tomoyo_update_stat(const u8 index) * Returns nothing. */ static void tomoyo_read_stat(struct tomoyo_io_buffer *head) + __must_hold(&head->io_sem) { u8 i; unsigned int total = 0; @@ -2493,6 +2531,7 @@ static void tomoyo_read_stat(struct tomoyo_io_buffer *head) * Returns 0. */ static int tomoyo_write_stat(struct tomoyo_io_buffer *head) + __must_hold(&head->io_sem) { char *data = head->write_buf; u8 i; @@ -2717,6 +2756,8 @@ ssize_t tomoyo_read_control(struct tomoyo_io_buffer *head, char __user *buffer, * Caller holds tomoyo_read_lock(). */ static int tomoyo_parse_policy(struct tomoyo_io_buffer *head, char *line) + __must_hold_shared(&tomoyo_ss) + __must_hold(&head->io_sem) { /* Delete request? */ head->w.is_delete = !strncmp(line, "delete ", 7); @@ -2969,8 +3010,11 @@ void __init tomoyo_load_builtin_policy(void) break; *end = '\0'; tomoyo_normalize_line(start); - head.write_buf = start; - tomoyo_parse_policy(&head, start); + /* head is stack-local and not shared. */ + context_unsafe( + head.write_buf = start; + tomoyo_parse_policy(&head, start); + ); start = end + 1; } } diff --git a/security/tomoyo/common.h b/security/tomoyo/common.h index 3b2a97d10a5d..4f1704c911ef 100644 --- a/security/tomoyo/common.h +++ b/security/tomoyo/common.h @@ -827,13 +827,13 @@ struct tomoyo_io_buffer { bool is_delete; } w; /* Buffer for reading. */ - char *read_buf; + char *read_buf __guarded_by(&io_sem); /* Size of read buffer. */ - size_t readbuf_size; + size_t readbuf_size __guarded_by(&io_sem); /* Buffer for writing. */ - char *write_buf; + char *write_buf __guarded_by(&io_sem); /* Size of write buffer. */ - size_t writebuf_size; + size_t writebuf_size __guarded_by(&io_sem); /* Type of this interface. */ enum tomoyo_securityfs_interface_index type; /* Users counter protected by tomoyo_io_buffer_list_lock. */ @@ -922,6 +922,35 @@ struct tomoyo_task { struct tomoyo_domain_info *old_domain_info; }; +/********** External variable definitions. **********/ + +extern bool tomoyo_policy_loaded; +extern int tomoyo_enabled; +extern const char * const tomoyo_condition_keyword +[TOMOYO_MAX_CONDITION_KEYWORD]; +extern const char * const tomoyo_dif[TOMOYO_MAX_DOMAIN_INFO_FLAGS]; +extern const char * const tomoyo_mac_keywords[TOMOYO_MAX_MAC_INDEX + + TOMOYO_MAX_MAC_CATEGORY_INDEX]; +extern const char * const tomoyo_mode[TOMOYO_CONFIG_MAX_MODE]; +extern const char * const tomoyo_path_keyword[TOMOYO_MAX_PATH_OPERATION]; +extern const char * const tomoyo_proto_keyword[TOMOYO_SOCK_MAX]; +extern const char * const tomoyo_socket_keyword[TOMOYO_MAX_NETWORK_OPERATION]; +extern const u8 tomoyo_index2category[TOMOYO_MAX_MAC_INDEX]; +extern const u8 tomoyo_pn2mac[TOMOYO_MAX_PATH_NUMBER_OPERATION]; +extern const u8 tomoyo_pnnn2mac[TOMOYO_MAX_MKDEV_OPERATION]; +extern const u8 tomoyo_pp2mac[TOMOYO_MAX_PATH2_OPERATION]; +extern struct list_head tomoyo_condition_list; +extern struct list_head tomoyo_domain_list; +extern struct list_head tomoyo_name_list[TOMOYO_MAX_HASH]; +extern struct list_head tomoyo_namespace_list; +extern struct mutex tomoyo_policy_lock; +extern struct srcu_struct tomoyo_ss; +extern struct tomoyo_domain_info tomoyo_kernel_domain; +extern struct tomoyo_policy_namespace tomoyo_kernel_namespace; +extern unsigned int tomoyo_memory_quota[TOMOYO_MAX_MEMORY_STAT]; +extern unsigned int tomoyo_memory_used[TOMOYO_MAX_MEMORY_STAT]; +extern struct lsm_blob_sizes tomoyo_blob_sizes; + /********** Function prototypes. **********/ int tomoyo_interface_init(void); @@ -971,10 +1000,10 @@ const struct tomoyo_path_info *tomoyo_path_matches_group int tomoyo_check_open_permission(struct tomoyo_domain_info *domain, const struct path *path, const int flag); void tomoyo_close_control(struct tomoyo_io_buffer *head); -int tomoyo_env_perm(struct tomoyo_request_info *r, const char *env); +int tomoyo_env_perm(struct tomoyo_request_info *r, const char *env) __must_hold_shared(&tomoyo_ss); int tomoyo_execute_permission(struct tomoyo_request_info *r, - const struct tomoyo_path_info *filename); -int tomoyo_find_next_domain(struct linux_binprm *bprm); + const struct tomoyo_path_info *filename) __must_hold_shared(&tomoyo_ss); +int tomoyo_find_next_domain(struct linux_binprm *bprm) __must_hold_shared(&tomoyo_ss); int tomoyo_get_mode(const struct tomoyo_policy_namespace *ns, const u8 profile, const u8 index); int tomoyo_init_request_info(struct tomoyo_request_info *r, @@ -1002,6 +1031,7 @@ int tomoyo_socket_listen_permission(struct socket *sock); int tomoyo_socket_sendmsg_permission(struct socket *sock, struct msghdr *msg, int size); int tomoyo_supervisor(struct tomoyo_request_info *r, const char *fmt, ...) + __must_hold_shared(&tomoyo_ss) __printf(2, 3); int tomoyo_update_domain(struct tomoyo_acl_info *new_entry, const int size, struct tomoyo_acl_param *param, @@ -1061,7 +1091,7 @@ void tomoyo_print_ulong(char *buffer, const int buffer_len, const unsigned long value, const u8 type); void tomoyo_put_name_union(struct tomoyo_name_union *ptr); void tomoyo_put_number_union(struct tomoyo_number_union *ptr); -void tomoyo_read_log(struct tomoyo_io_buffer *head); +void tomoyo_read_log(struct tomoyo_io_buffer *head) __must_hold(&head->io_sem); void tomoyo_update_stat(const u8 index); void tomoyo_warn_oom(const char *function); void tomoyo_write_log(struct tomoyo_request_info *r, const char *fmt, ...) @@ -1069,35 +1099,6 @@ void tomoyo_write_log(struct tomoyo_request_info *r, const char *fmt, ...) void tomoyo_write_log2(struct tomoyo_request_info *r, int len, const char *fmt, va_list args) __printf(3, 0); -/********** External variable definitions. **********/ - -extern bool tomoyo_policy_loaded; -extern int tomoyo_enabled; -extern const char * const tomoyo_condition_keyword -[TOMOYO_MAX_CONDITION_KEYWORD]; -extern const char * const tomoyo_dif[TOMOYO_MAX_DOMAIN_INFO_FLAGS]; -extern const char * const tomoyo_mac_keywords[TOMOYO_MAX_MAC_INDEX - + TOMOYO_MAX_MAC_CATEGORY_INDEX]; -extern const char * const tomoyo_mode[TOMOYO_CONFIG_MAX_MODE]; -extern const char * const tomoyo_path_keyword[TOMOYO_MAX_PATH_OPERATION]; -extern const char * const tomoyo_proto_keyword[TOMOYO_SOCK_MAX]; -extern const char * const tomoyo_socket_keyword[TOMOYO_MAX_NETWORK_OPERATION]; -extern const u8 tomoyo_index2category[TOMOYO_MAX_MAC_INDEX]; -extern const u8 tomoyo_pn2mac[TOMOYO_MAX_PATH_NUMBER_OPERATION]; -extern const u8 tomoyo_pnnn2mac[TOMOYO_MAX_MKDEV_OPERATION]; -extern const u8 tomoyo_pp2mac[TOMOYO_MAX_PATH2_OPERATION]; -extern struct list_head tomoyo_condition_list; -extern struct list_head tomoyo_domain_list; -extern struct list_head tomoyo_name_list[TOMOYO_MAX_HASH]; -extern struct list_head tomoyo_namespace_list; -extern struct mutex tomoyo_policy_lock; -extern struct srcu_struct tomoyo_ss; -extern struct tomoyo_domain_info tomoyo_kernel_domain; -extern struct tomoyo_policy_namespace tomoyo_kernel_namespace; -extern unsigned int tomoyo_memory_quota[TOMOYO_MAX_MEMORY_STAT]; -extern unsigned int tomoyo_memory_used[TOMOYO_MAX_MEMORY_STAT]; -extern struct lsm_blob_sizes tomoyo_blob_sizes; - /********** Inlined functions. **********/ /** @@ -1106,6 +1107,7 @@ extern struct lsm_blob_sizes tomoyo_blob_sizes; * Returns index number for tomoyo_read_unlock(). */ static inline int tomoyo_read_lock(void) + __acquires_shared(&tomoyo_ss) { return srcu_read_lock(&tomoyo_ss); } @@ -1118,6 +1120,7 @@ static inline int tomoyo_read_lock(void) * Returns nothing. */ static inline void tomoyo_read_unlock(int idx) + __releases_shared(&tomoyo_ss) { srcu_read_unlock(&tomoyo_ss, idx); } diff --git a/security/tomoyo/domain.c b/security/tomoyo/domain.c index 90cf0e2969df..0612eac7f2f2 100644 --- a/security/tomoyo/domain.c +++ b/security/tomoyo/domain.c @@ -611,6 +611,7 @@ out: * Returns 0 on success, negative value otherwise. */ static int tomoyo_environ(struct tomoyo_execve *ee) + __must_hold_shared(&tomoyo_ss) { struct tomoyo_request_info *r = &ee->r; struct linux_binprm *bprm = ee->bprm; diff --git a/security/tomoyo/environ.c b/security/tomoyo/environ.c index 7f0a471f19b2..bcb05910facc 100644 --- a/security/tomoyo/environ.c +++ b/security/tomoyo/environ.c @@ -32,6 +32,7 @@ static bool tomoyo_check_env_acl(struct tomoyo_request_info *r, * Returns 0 on success, negative value otherwise. */ static int tomoyo_audit_env_log(struct tomoyo_request_info *r) + __must_hold_shared(&tomoyo_ss) { return tomoyo_supervisor(r, "misc env %s\n", r->param.environ.name->name); diff --git a/security/tomoyo/file.c b/security/tomoyo/file.c index 8f3b90b6e03d..e9b67dbb38e7 100644 --- a/security/tomoyo/file.c +++ b/security/tomoyo/file.c @@ -164,6 +164,7 @@ static bool tomoyo_get_realpath(struct tomoyo_path_info *buf, const struct path * Returns 0 on success, negative value otherwise. */ static int tomoyo_audit_path_log(struct tomoyo_request_info *r) + __must_hold_shared(&tomoyo_ss) { return tomoyo_supervisor(r, "file %s %s\n", tomoyo_path_keyword [r->param.path.operation], @@ -178,6 +179,7 @@ static int tomoyo_audit_path_log(struct tomoyo_request_info *r) * Returns 0 on success, negative value otherwise. */ static int tomoyo_audit_path2_log(struct tomoyo_request_info *r) + __must_hold_shared(&tomoyo_ss) { return tomoyo_supervisor(r, "file %s %s %s\n", tomoyo_mac_keywords [tomoyo_pp2mac[r->param.path2.operation]], @@ -193,6 +195,7 @@ static int tomoyo_audit_path2_log(struct tomoyo_request_info *r) * Returns 0 on success, negative value otherwise. */ static int tomoyo_audit_mkdev_log(struct tomoyo_request_info *r) + __must_hold_shared(&tomoyo_ss) { return tomoyo_supervisor(r, "file %s %s 0%o %u %u\n", tomoyo_mac_keywords @@ -210,6 +213,7 @@ static int tomoyo_audit_mkdev_log(struct tomoyo_request_info *r) * Returns 0 on success, negative value otherwise. */ static int tomoyo_audit_path_number_log(struct tomoyo_request_info *r) + __must_hold_shared(&tomoyo_ss) { const u8 type = r->param.path_number.operation; u8 radix; @@ -572,6 +576,7 @@ static int tomoyo_update_path2_acl(const u8 perm, */ static int tomoyo_path_permission(struct tomoyo_request_info *r, u8 operation, const struct tomoyo_path_info *filename) + __must_hold_shared(&tomoyo_ss) { int error; diff --git a/security/tomoyo/gc.c b/security/tomoyo/gc.c index 026e29ea3796..8e2008863af8 100644 --- a/security/tomoyo/gc.c +++ b/security/tomoyo/gc.c @@ -23,11 +23,10 @@ static inline void tomoyo_memory_free(void *ptr) tomoyo_memory_used[TOMOYO_MEMORY_POLICY] -= ksize(ptr); kfree(ptr); } - -/* The list for "struct tomoyo_io_buffer". */ -static LIST_HEAD(tomoyo_io_buffer_list); /* Lock for protecting tomoyo_io_buffer_list. */ static DEFINE_SPINLOCK(tomoyo_io_buffer_list_lock); +/* The list for "struct tomoyo_io_buffer". */ +static __guarded_by(&tomoyo_io_buffer_list_lock) LIST_HEAD(tomoyo_io_buffer_list); /** * tomoyo_struct_used_by_io_buffer - Check whether the list element is used by /sys/kernel/security/tomoyo/ users or not. @@ -385,6 +384,7 @@ static inline void tomoyo_del_number_group(struct list_head *element) */ static void tomoyo_try_to_gc(const enum tomoyo_policy_id type, struct list_head *element) + __must_hold(&tomoyo_policy_lock) { /* * __list_del_entry() guarantees that the list element became no longer @@ -484,6 +484,7 @@ reinject: */ static void tomoyo_collect_member(const enum tomoyo_policy_id id, struct list_head *member_list) + __must_hold(&tomoyo_policy_lock) { struct tomoyo_acl_head *member; struct tomoyo_acl_head *tmp; @@ -504,6 +505,7 @@ static void tomoyo_collect_member(const enum tomoyo_policy_id id, * Returns nothing. */ static void tomoyo_collect_acl(struct list_head *list) + __must_hold(&tomoyo_policy_lock) { struct tomoyo_acl_info *acl; struct tomoyo_acl_info *tmp; @@ -627,8 +629,11 @@ static int tomoyo_gc_thread(void *unused) if (head->users) continue; list_del(&head->list); - kfree(head->read_buf); - kfree(head->write_buf); + /* Safe destruction because no users are left. */ + context_unsafe( + kfree(head->read_buf); + kfree(head->write_buf); + ); kfree(head); } spin_unlock(&tomoyo_io_buffer_list_lock); @@ -656,11 +661,18 @@ void tomoyo_notify_gc(struct tomoyo_io_buffer *head, const bool is_register) head->users = 1; list_add(&head->list, &tomoyo_io_buffer_list); } else { - is_write = head->write_buf != NULL; + /* + * tomoyo_write_control() can concurrently update write_buf from + * a non-NULL to new non-NULL pointer with io_sem held. + */ + is_write = data_race(head->write_buf != NULL); if (!--head->users) { list_del(&head->list); - kfree(head->read_buf); - kfree(head->write_buf); + /* Safe destruction because no users are left. */ + context_unsafe( + kfree(head->read_buf); + kfree(head->write_buf); + ); kfree(head); } } diff --git a/security/tomoyo/mount.c b/security/tomoyo/mount.c index 2755971f50df..322dfd188ada 100644 --- a/security/tomoyo/mount.c +++ b/security/tomoyo/mount.c @@ -28,6 +28,7 @@ static const char * const tomoyo_mounts[TOMOYO_MAX_SPECIAL_MOUNT] = { * Returns 0 on success, negative value otherwise. */ static int tomoyo_audit_mount_log(struct tomoyo_request_info *r) + __must_hold_shared(&tomoyo_ss) { return tomoyo_supervisor(r, "file mount %s %s %s 0x%lX\n", r->param.mount.dev->name, @@ -78,6 +79,7 @@ static int tomoyo_mount_acl(struct tomoyo_request_info *r, const char *dev_name, const struct path *dir, const char *type, unsigned long flags) + __must_hold_shared(&tomoyo_ss) { struct tomoyo_obj_info obj = { }; struct path path; diff --git a/security/tomoyo/network.c b/security/tomoyo/network.c index 8dc61335f65e..cfc2a019de1e 100644 --- a/security/tomoyo/network.c +++ b/security/tomoyo/network.c @@ -363,6 +363,7 @@ int tomoyo_write_unix_network(struct tomoyo_acl_param *param) static int tomoyo_audit_net_log(struct tomoyo_request_info *r, const char *family, const u8 protocol, const u8 operation, const char *address) + __must_hold_shared(&tomoyo_ss) { return tomoyo_supervisor(r, "network %s %s %s %s\n", family, tomoyo_proto_keyword[protocol], @@ -377,6 +378,7 @@ static int tomoyo_audit_net_log(struct tomoyo_request_info *r, * Returns 0 on success, negative value otherwise. */ static int tomoyo_audit_inet_log(struct tomoyo_request_info *r) + __must_hold_shared(&tomoyo_ss) { char buf[128]; int len; @@ -402,6 +404,7 @@ static int tomoyo_audit_inet_log(struct tomoyo_request_info *r) * Returns 0 on success, negative value otherwise. */ static int tomoyo_audit_unix_log(struct tomoyo_request_info *r) + __must_hold_shared(&tomoyo_ss) { return tomoyo_audit_net_log(r, "unix", r->param.unix_network.protocol, r->param.unix_network.operation, From dc36d55d4e7259ff0f91a154744125ccc2228171 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:24 +0100 Subject: [PATCH 38/90] crypto: Enable context analysis Enable context analysis for crypto subsystem. This demonstrates a larger conversion to use Clang's context analysis. The benefit is additional static checking of locking rules, along with better documentation. Note the use of the __acquire_ret macro how to define an API where a function returns a pointer to an object (struct scomp_scratch) with a lock held. Additionally, the analysis only resolves aliases where the analysis unambiguously sees that a variable was not reassigned after initialization, requiring minor code changes. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-36-elver@google.com --- crypto/Makefile | 2 ++ crypto/acompress.c | 6 +++--- crypto/algapi.c | 2 ++ crypto/api.c | 1 + crypto/crypto_engine.c | 2 +- crypto/drbg.c | 5 +++++ crypto/internal.h | 2 +- crypto/proc.c | 3 +++ crypto/scompress.c | 24 ++++++++++++------------ include/crypto/internal/acompress.h | 7 ++++--- include/crypto/internal/engine.h | 2 +- 11 files changed, 35 insertions(+), 21 deletions(-) diff --git a/crypto/Makefile b/crypto/Makefile index 16a35649dd91..db264feab7e7 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -3,6 +3,8 @@ # Cryptographic API # +CONTEXT_ANALYSIS := y + obj-$(CONFIG_CRYPTO) += crypto.o crypto-y := api.o cipher.o diff --git a/crypto/acompress.c b/crypto/acompress.c index be28cbfd22e3..25df368df098 100644 --- a/crypto/acompress.c +++ b/crypto/acompress.c @@ -449,8 +449,8 @@ int crypto_acomp_alloc_streams(struct crypto_acomp_streams *s) } EXPORT_SYMBOL_GPL(crypto_acomp_alloc_streams); -struct crypto_acomp_stream *crypto_acomp_lock_stream_bh( - struct crypto_acomp_streams *s) __acquires(stream) +struct crypto_acomp_stream *_crypto_acomp_lock_stream_bh( + struct crypto_acomp_streams *s) { struct crypto_acomp_stream __percpu *streams = s->streams; int cpu = raw_smp_processor_id(); @@ -469,7 +469,7 @@ struct crypto_acomp_stream *crypto_acomp_lock_stream_bh( spin_lock(&ps->lock); return ps; } -EXPORT_SYMBOL_GPL(crypto_acomp_lock_stream_bh); +EXPORT_SYMBOL_GPL(_crypto_acomp_lock_stream_bh); void acomp_walk_done_src(struct acomp_walk *walk, int used) { diff --git a/crypto/algapi.c b/crypto/algapi.c index e604d0d8b7b4..abc9333327d4 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -244,6 +244,7 @@ EXPORT_SYMBOL_GPL(crypto_remove_spawns); static void crypto_alg_finish_registration(struct crypto_alg *alg, struct list_head *algs_to_put) + __must_hold(&crypto_alg_sem) { struct crypto_alg *q; @@ -299,6 +300,7 @@ static struct crypto_larval *crypto_alloc_test_larval(struct crypto_alg *alg) static struct crypto_larval * __crypto_register_alg(struct crypto_alg *alg, struct list_head *algs_to_put) + __must_hold(&crypto_alg_sem) { struct crypto_alg *q; struct crypto_larval *larval; diff --git a/crypto/api.c b/crypto/api.c index 5724d62e9d07..05629644a688 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -57,6 +57,7 @@ EXPORT_SYMBOL_GPL(crypto_mod_put); static struct crypto_alg *__crypto_alg_lookup(const char *name, u32 type, u32 mask) + __must_hold_shared(&crypto_alg_sem) { struct crypto_alg *q, *alg = NULL; int best = -2; diff --git a/crypto/crypto_engine.c b/crypto/crypto_engine.c index 18e1689efe12..1653a4bf5b31 100644 --- a/crypto/crypto_engine.c +++ b/crypto/crypto_engine.c @@ -453,8 +453,8 @@ struct crypto_engine *crypto_engine_alloc_init_and_set(struct device *dev, snprintf(engine->name, sizeof(engine->name), "%s-engine", dev_name(dev)); - crypto_init_queue(&engine->queue, qlen); spin_lock_init(&engine->queue_lock); + crypto_init_queue(&engine->queue, qlen); engine->kworker = kthread_run_worker(0, "%s", engine->name); if (IS_ERR(engine->kworker)) { diff --git a/crypto/drbg.c b/crypto/drbg.c index 1d433dae9955..0a6f6c05a78f 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -232,6 +232,7 @@ static inline unsigned short drbg_sec_strength(drbg_flag_t flags) */ static int drbg_fips_continuous_test(struct drbg_state *drbg, const unsigned char *entropy) + __must_hold(&drbg->drbg_mutex) { unsigned short entropylen = drbg_sec_strength(drbg->core->flags); int ret = 0; @@ -848,6 +849,7 @@ static inline int __drbg_seed(struct drbg_state *drbg, struct list_head *seed, static inline int drbg_get_random_bytes(struct drbg_state *drbg, unsigned char *entropy, unsigned int entropylen) + __must_hold(&drbg->drbg_mutex) { int ret; @@ -862,6 +864,7 @@ static inline int drbg_get_random_bytes(struct drbg_state *drbg, } static int drbg_seed_from_random(struct drbg_state *drbg) + __must_hold(&drbg->drbg_mutex) { struct drbg_string data; LIST_HEAD(seedlist); @@ -919,6 +922,7 @@ static bool drbg_nopr_reseed_interval_elapsed(struct drbg_state *drbg) */ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers, bool reseed) + __must_hold(&drbg->drbg_mutex) { int ret; unsigned char entropy[((32 + 16) * 2)]; @@ -1153,6 +1157,7 @@ err: static int drbg_generate(struct drbg_state *drbg, unsigned char *buf, unsigned int buflen, struct drbg_string *addtl) + __must_hold(&drbg->drbg_mutex) { int len = 0; LIST_HEAD(addtllist); diff --git a/crypto/internal.h b/crypto/internal.h index b9afd68767c1..8fbe0226d48e 100644 --- a/crypto/internal.h +++ b/crypto/internal.h @@ -61,8 +61,8 @@ enum { /* Maximum number of (rtattr) parameters for each template. */ #define CRYPTO_MAX_ATTRS 32 -extern struct list_head crypto_alg_list; extern struct rw_semaphore crypto_alg_sem; +extern struct list_head crypto_alg_list __guarded_by(&crypto_alg_sem); extern struct blocking_notifier_head crypto_chain; int alg_test(const char *driver, const char *alg, u32 type, u32 mask); diff --git a/crypto/proc.c b/crypto/proc.c index 82f15b967e85..5fb9fe86d023 100644 --- a/crypto/proc.c +++ b/crypto/proc.c @@ -19,17 +19,20 @@ #include "internal.h" static void *c_start(struct seq_file *m, loff_t *pos) + __acquires_shared(&crypto_alg_sem) { down_read(&crypto_alg_sem); return seq_list_start(&crypto_alg_list, *pos); } static void *c_next(struct seq_file *m, void *p, loff_t *pos) + __must_hold_shared(&crypto_alg_sem) { return seq_list_next(p, &crypto_alg_list, pos); } static void c_stop(struct seq_file *m, void *p) + __releases_shared(&crypto_alg_sem) { up_read(&crypto_alg_sem); } diff --git a/crypto/scompress.c b/crypto/scompress.c index 1a7ed8ae65b0..7aee1d50e148 100644 --- a/crypto/scompress.c +++ b/crypto/scompress.c @@ -28,8 +28,8 @@ struct scomp_scratch { spinlock_t lock; union { - void *src; - unsigned long saddr; + void *src __guarded_by(&lock); + unsigned long saddr __guarded_by(&lock); }; }; @@ -38,8 +38,8 @@ static DEFINE_PER_CPU(struct scomp_scratch, scomp_scratch) = { }; static const struct crypto_type crypto_scomp_type; -static int scomp_scratch_users; static DEFINE_MUTEX(scomp_lock); +static int scomp_scratch_users __guarded_by(&scomp_lock); static cpumask_t scomp_scratch_want; static void scomp_scratch_workfn(struct work_struct *work); @@ -67,6 +67,7 @@ static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg) } static void crypto_scomp_free_scratches(void) + __context_unsafe(/* frees @scratch */) { struct scomp_scratch *scratch; int i; @@ -101,7 +102,7 @@ static void scomp_scratch_workfn(struct work_struct *work) struct scomp_scratch *scratch; scratch = per_cpu_ptr(&scomp_scratch, cpu); - if (scratch->src) + if (context_unsafe(scratch->src)) continue; if (scomp_alloc_scratch(scratch, cpu)) break; @@ -111,6 +112,7 @@ static void scomp_scratch_workfn(struct work_struct *work) } static int crypto_scomp_alloc_scratches(void) + __context_unsafe(/* allocates @scratch */) { unsigned int i = cpumask_first(cpu_possible_mask); struct scomp_scratch *scratch; @@ -139,7 +141,8 @@ unlock: return ret; } -static struct scomp_scratch *scomp_lock_scratch(void) __acquires(scratch) +#define scomp_lock_scratch(...) __acquire_ret(_scomp_lock_scratch(__VA_ARGS__), &__ret->lock) +static struct scomp_scratch *_scomp_lock_scratch(void) __acquires_ret { int cpu = raw_smp_processor_id(); struct scomp_scratch *scratch; @@ -159,7 +162,7 @@ static struct scomp_scratch *scomp_lock_scratch(void) __acquires(scratch) } static inline void scomp_unlock_scratch(struct scomp_scratch *scratch) - __releases(scratch) + __releases(&scratch->lock) { spin_unlock(&scratch->lock); } @@ -171,8 +174,6 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) bool src_isvirt = acomp_request_src_isvirt(req); bool dst_isvirt = acomp_request_dst_isvirt(req); struct crypto_scomp *scomp = *tfm_ctx; - struct crypto_acomp_stream *stream; - struct scomp_scratch *scratch; unsigned int slen = req->slen; unsigned int dlen = req->dlen; struct page *spage, *dpage; @@ -232,13 +233,12 @@ static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) } while (0); } - stream = crypto_acomp_lock_stream_bh(&crypto_scomp_alg(scomp)->streams); + struct crypto_acomp_stream *stream = crypto_acomp_lock_stream_bh(&crypto_scomp_alg(scomp)->streams); if (!src_isvirt && !src) { - const u8 *src; + struct scomp_scratch *scratch = scomp_lock_scratch(); + const u8 *src = scratch->src; - scratch = scomp_lock_scratch(); - src = scratch->src; memcpy_from_sglist(scratch->src, req->src, 0, slen); if (dir) diff --git a/include/crypto/internal/acompress.h b/include/crypto/internal/acompress.h index 2d97440028ff..9a3f28baa804 100644 --- a/include/crypto/internal/acompress.h +++ b/include/crypto/internal/acompress.h @@ -191,11 +191,12 @@ static inline bool crypto_acomp_req_virt(struct crypto_acomp *tfm) void crypto_acomp_free_streams(struct crypto_acomp_streams *s); int crypto_acomp_alloc_streams(struct crypto_acomp_streams *s); -struct crypto_acomp_stream *crypto_acomp_lock_stream_bh( - struct crypto_acomp_streams *s) __acquires(stream); +#define crypto_acomp_lock_stream_bh(...) __acquire_ret(_crypto_acomp_lock_stream_bh(__VA_ARGS__), &__ret->lock); +struct crypto_acomp_stream *_crypto_acomp_lock_stream_bh( + struct crypto_acomp_streams *s) __acquires_ret; static inline void crypto_acomp_unlock_stream_bh( - struct crypto_acomp_stream *stream) __releases(stream) + struct crypto_acomp_stream *stream) __releases(&stream->lock) { spin_unlock_bh(&stream->lock); } diff --git a/include/crypto/internal/engine.h b/include/crypto/internal/engine.h index f19ef376833f..6a1d27880615 100644 --- a/include/crypto/internal/engine.h +++ b/include/crypto/internal/engine.h @@ -45,7 +45,7 @@ struct crypto_engine { struct list_head list; spinlock_t queue_lock; - struct crypto_queue queue; + struct crypto_queue queue __guarded_by(&queue_lock); struct device *dev; struct kthread_worker *kworker; From 04e49d926f438134b6453505aa206e70f8cf4cb1 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Fri, 19 Dec 2025 16:40:25 +0100 Subject: [PATCH 39/90] sched: Enable context analysis for core.c and fair.c This demonstrates a larger conversion to use Clang's context analysis. The benefit is additional static checking of locking rules, along with better documentation. Notably, kernel/sched contains sufficiently complex synchronization patterns, and application to core.c & fair.c demonstrates that the latest Clang version has become powerful enough to start applying this to more complex subsystems (with some modest annotations and changes). Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251219154418.3592607-37-elver@google.com --- include/linux/sched.h | 6 +- include/linux/sched/signal.h | 4 +- include/linux/sched/task.h | 6 +- include/linux/sched/wake_q.h | 3 + kernel/sched/Makefile | 3 + kernel/sched/core.c | 89 +++++++++++----- kernel/sched/fair.c | 7 +- kernel/sched/sched.h | 126 ++++++++++++++++------- scripts/context-analysis-suppression.txt | 1 + 9 files changed, 177 insertions(+), 68 deletions(-) diff --git a/include/linux/sched.h b/include/linux/sched.h index d395f2810fac..c4022647282e 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2094,9 +2094,9 @@ static inline int _cond_resched(void) _cond_resched(); \ }) -extern int __cond_resched_lock(spinlock_t *lock); -extern int __cond_resched_rwlock_read(rwlock_t *lock); -extern int __cond_resched_rwlock_write(rwlock_t *lock); +extern int __cond_resched_lock(spinlock_t *lock) __must_hold(lock); +extern int __cond_resched_rwlock_read(rwlock_t *lock) __must_hold_shared(lock); +extern int __cond_resched_rwlock_write(rwlock_t *lock) __must_hold(lock); #define MIGHT_RESCHED_RCU_SHIFT 8 #define MIGHT_RESCHED_PREEMPT_MASK ((1U << MIGHT_RESCHED_RCU_SHIFT) - 1) diff --git a/include/linux/sched/signal.h b/include/linux/sched/signal.h index a63f65aa5bdd..a22248aebcf9 100644 --- a/include/linux/sched/signal.h +++ b/include/linux/sched/signal.h @@ -738,10 +738,12 @@ static inline int thread_group_empty(struct task_struct *p) (thread_group_leader(p) && !thread_group_empty(p)) extern struct sighand_struct *lock_task_sighand(struct task_struct *task, - unsigned long *flags); + unsigned long *flags) + __acquires(&task->sighand->siglock); static inline void unlock_task_sighand(struct task_struct *task, unsigned long *flags) + __releases(&task->sighand->siglock) { spin_unlock_irqrestore(&task->sighand->siglock, *flags); } diff --git a/include/linux/sched/task.h b/include/linux/sched/task.h index 525aa2a632b2..41ed884cffc9 100644 --- a/include/linux/sched/task.h +++ b/include/linux/sched/task.h @@ -214,15 +214,19 @@ static inline struct vm_struct *task_stack_vm_area(const struct task_struct *t) * write_lock_irq(&tasklist_lock), neither inside nor outside. */ static inline void task_lock(struct task_struct *p) + __acquires(&p->alloc_lock) { spin_lock(&p->alloc_lock); } static inline void task_unlock(struct task_struct *p) + __releases(&p->alloc_lock) { spin_unlock(&p->alloc_lock); } -DEFINE_GUARD(task_lock, struct task_struct *, task_lock(_T), task_unlock(_T)) +DEFINE_LOCK_GUARD_1(task_lock, struct task_struct, task_lock(_T->lock), task_unlock(_T->lock)) +DECLARE_LOCK_GUARD_1_ATTRS(task_lock, __acquires(&_T->alloc_lock), __releases(&(*(struct task_struct **)_T)->alloc_lock)) +#define class_task_lock_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(task_lock, _T) #endif /* _LINUX_SCHED_TASK_H */ diff --git a/include/linux/sched/wake_q.h b/include/linux/sched/wake_q.h index 0f28b4623ad4..765bbc3d54be 100644 --- a/include/linux/sched/wake_q.h +++ b/include/linux/sched/wake_q.h @@ -66,6 +66,7 @@ extern void wake_up_q(struct wake_q_head *head); /* Spin unlock helpers to unlock and call wake_up_q with preempt disabled */ static inline void raw_spin_unlock_wake(raw_spinlock_t *lock, struct wake_q_head *wake_q) + __releases(lock) { guard(preempt)(); raw_spin_unlock(lock); @@ -77,6 +78,7 @@ void raw_spin_unlock_wake(raw_spinlock_t *lock, struct wake_q_head *wake_q) static inline void raw_spin_unlock_irq_wake(raw_spinlock_t *lock, struct wake_q_head *wake_q) + __releases(lock) { guard(preempt)(); raw_spin_unlock_irq(lock); @@ -89,6 +91,7 @@ void raw_spin_unlock_irq_wake(raw_spinlock_t *lock, struct wake_q_head *wake_q) static inline void raw_spin_unlock_irqrestore_wake(raw_spinlock_t *lock, unsigned long flags, struct wake_q_head *wake_q) + __releases(lock) { guard(preempt)(); raw_spin_unlock_irqrestore(lock, flags); diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index 8ae86371ddcd..b1f1a367034f 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -1,5 +1,8 @@ # SPDX-License-Identifier: GPL-2.0 +CONTEXT_ANALYSIS_core.o := y +CONTEXT_ANALYSIS_fair.o := y + # The compilers are complaining about unused variables inside an if(0) scope # block. This is daft, shut them up. ccflags-y += $(call cc-disable-warning, unused-but-set-variable) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 41ba0be16911..ae543ee91272 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -396,6 +396,8 @@ static atomic_t sched_core_count; static struct cpumask sched_core_mask; static void sched_core_lock(int cpu, unsigned long *flags) + __context_unsafe(/* acquires multiple */) + __acquires(&runqueues.__lock) /* overapproximation */ { const struct cpumask *smt_mask = cpu_smt_mask(cpu); int t, i = 0; @@ -406,6 +408,8 @@ static void sched_core_lock(int cpu, unsigned long *flags) } static void sched_core_unlock(int cpu, unsigned long *flags) + __context_unsafe(/* releases multiple */) + __releases(&runqueues.__lock) /* overapproximation */ { const struct cpumask *smt_mask = cpu_smt_mask(cpu); int t; @@ -630,6 +634,7 @@ EXPORT_SYMBOL(__trace_set_current_state); */ void raw_spin_rq_lock_nested(struct rq *rq, int subclass) + __context_unsafe() { raw_spinlock_t *lock; @@ -655,6 +660,7 @@ void raw_spin_rq_lock_nested(struct rq *rq, int subclass) } bool raw_spin_rq_trylock(struct rq *rq) + __context_unsafe() { raw_spinlock_t *lock; bool ret; @@ -696,15 +702,16 @@ void double_rq_lock(struct rq *rq1, struct rq *rq2) raw_spin_rq_lock(rq1); if (__rq_lockp(rq1) != __rq_lockp(rq2)) raw_spin_rq_lock_nested(rq2, SINGLE_DEPTH_NESTING); + else + __acquire_ctx_lock(__rq_lockp(rq2)); /* fake acquire */ double_rq_clock_clear_update(rq1, rq2); } /* - * __task_rq_lock - lock the rq @p resides on. + * ___task_rq_lock - lock the rq @p resides on. */ -struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf) - __acquires(rq->lock) +struct rq *___task_rq_lock(struct task_struct *p, struct rq_flags *rf) { struct rq *rq; @@ -727,9 +734,7 @@ struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf) /* * task_rq_lock - lock p->pi_lock and lock the rq @p resides on. */ -struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf) - __acquires(p->pi_lock) - __acquires(rq->lock) +struct rq *_task_rq_lock(struct task_struct *p, struct rq_flags *rf) { struct rq *rq; @@ -2431,6 +2436,7 @@ static inline bool is_cpu_allowed(struct task_struct *p, int cpu) */ static struct rq *move_queued_task(struct rq *rq, struct rq_flags *rf, struct task_struct *p, int new_cpu) + __must_hold(__rq_lockp(rq)) { lockdep_assert_rq_held(rq); @@ -2477,6 +2483,7 @@ struct set_affinity_pending { */ static struct rq *__migrate_task(struct rq *rq, struct rq_flags *rf, struct task_struct *p, int dest_cpu) + __must_hold(__rq_lockp(rq)) { /* Affinity changed (again). */ if (!is_cpu_allowed(p, dest_cpu)) @@ -2513,6 +2520,12 @@ static int migration_cpu_stop(void *data) */ flush_smp_call_function_queue(); + /* + * We may change the underlying rq, but the locks held will + * appropriately be "transferred" when switching. + */ + context_unsafe_alias(rq); + raw_spin_lock(&p->pi_lock); rq_lock(rq, &rf); @@ -2624,6 +2637,8 @@ int push_cpu_stop(void *arg) if (!lowest_rq) goto out_unlock; + lockdep_assert_rq_held(lowest_rq); + // XXX validate p is still the highest prio task if (task_rq(p) == rq) { move_queued_task_locked(rq, lowest_rq, p); @@ -2834,8 +2849,7 @@ void release_user_cpus_ptr(struct task_struct *p) */ static int affine_move_task(struct rq *rq, struct task_struct *p, struct rq_flags *rf, int dest_cpu, unsigned int flags) - __releases(rq->lock) - __releases(p->pi_lock) + __releases(__rq_lockp(rq), &p->pi_lock) { struct set_affinity_pending my_pending = { }, *pending = NULL; bool stop_pending, complete = false; @@ -2990,8 +3004,7 @@ static int __set_cpus_allowed_ptr_locked(struct task_struct *p, struct affinity_context *ctx, struct rq *rq, struct rq_flags *rf) - __releases(rq->lock) - __releases(p->pi_lock) + __releases(__rq_lockp(rq), &p->pi_lock) { const struct cpumask *cpu_allowed_mask = task_cpu_possible_mask(p); const struct cpumask *cpu_valid_mask = cpu_active_mask; @@ -4273,29 +4286,30 @@ static bool __task_needs_rq_lock(struct task_struct *p) */ int task_call_func(struct task_struct *p, task_call_f func, void *arg) { - struct rq *rq = NULL; struct rq_flags rf; int ret; raw_spin_lock_irqsave(&p->pi_lock, rf.flags); - if (__task_needs_rq_lock(p)) - rq = __task_rq_lock(p, &rf); + if (__task_needs_rq_lock(p)) { + struct rq *rq = __task_rq_lock(p, &rf); - /* - * At this point the task is pinned; either: - * - blocked and we're holding off wakeups (pi->lock) - * - woken, and we're holding off enqueue (rq->lock) - * - queued, and we're holding off schedule (rq->lock) - * - running, and we're holding off de-schedule (rq->lock) - * - * The called function (@func) can use: task_curr(), p->on_rq and - * p->__state to differentiate between these states. - */ - ret = func(p, arg); + /* + * At this point the task is pinned; either: + * - blocked and we're holding off wakeups (pi->lock) + * - woken, and we're holding off enqueue (rq->lock) + * - queued, and we're holding off schedule (rq->lock) + * - running, and we're holding off de-schedule (rq->lock) + * + * The called function (@func) can use: task_curr(), p->on_rq and + * p->__state to differentiate between these states. + */ + ret = func(p, arg); - if (rq) __task_rq_unlock(rq, p, &rf); + } else { + ret = func(p, arg); + } raw_spin_unlock_irqrestore(&p->pi_lock, rf.flags); return ret; @@ -4968,6 +4982,8 @@ void balance_callbacks(struct rq *rq, struct balance_callback *head) static inline void prepare_lock_switch(struct rq *rq, struct task_struct *next, struct rq_flags *rf) + __releases(__rq_lockp(rq)) + __acquires(__rq_lockp(this_rq())) { /* * Since the runqueue lock will be released by the next @@ -4981,9 +4997,15 @@ prepare_lock_switch(struct rq *rq, struct task_struct *next, struct rq_flags *rf /* this is a valid case when another task releases the spinlock */ rq_lockp(rq)->owner = next; #endif + /* + * Model the rq reference switcheroo. + */ + __release(__rq_lockp(rq)); + __acquire(__rq_lockp(this_rq())); } static inline void finish_lock_switch(struct rq *rq) + __releases(__rq_lockp(rq)) { /* * If we are tracking spinlock dependencies then we have to @@ -5039,6 +5061,7 @@ static inline void kmap_local_sched_in(void) static inline void prepare_task_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next) + __must_hold(__rq_lockp(rq)) { kcov_prepare_switch(prev); sched_info_switch(rq, prev, next); @@ -5069,7 +5092,7 @@ prepare_task_switch(struct rq *rq, struct task_struct *prev, * because prev may have moved to another CPU. */ static struct rq *finish_task_switch(struct task_struct *prev) - __releases(rq->lock) + __releases(__rq_lockp(this_rq())) { struct rq *rq = this_rq(); struct mm_struct *mm = rq->prev_mm; @@ -5165,7 +5188,7 @@ static struct rq *finish_task_switch(struct task_struct *prev) * @prev: the thread we just switched away from. */ asmlinkage __visible void schedule_tail(struct task_struct *prev) - __releases(rq->lock) + __releases(__rq_lockp(this_rq())) { /* * New tasks start with FORK_PREEMPT_COUNT, see there and @@ -5197,6 +5220,7 @@ asmlinkage __visible void schedule_tail(struct task_struct *prev) static __always_inline struct rq * context_switch(struct rq *rq, struct task_struct *prev, struct task_struct *next, struct rq_flags *rf) + __releases(__rq_lockp(rq)) { prepare_task_switch(rq, prev, next); @@ -5865,6 +5889,7 @@ static void prev_balance(struct rq *rq, struct task_struct *prev, */ static inline struct task_struct * __pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) + __must_hold(__rq_lockp(rq)) { const struct sched_class *class; struct task_struct *p; @@ -5965,6 +5990,7 @@ static void queue_core_balance(struct rq *rq); static struct task_struct * pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) + __must_hold(__rq_lockp(rq)) { struct task_struct *next, *p, *max; const struct cpumask *smt_mask; @@ -6273,6 +6299,7 @@ static bool steal_cookie_task(int cpu, struct sched_domain *sd) } static void sched_core_balance(struct rq *rq) + __must_hold(__rq_lockp(rq)) { struct sched_domain *sd; int cpu = cpu_of(rq); @@ -6418,6 +6445,7 @@ static inline void sched_core_cpu_dying(unsigned int cpu) {} static struct task_struct * pick_next_task(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) + __must_hold(__rq_lockp(rq)) { return __pick_next_task(rq, prev, rf); } @@ -8043,6 +8071,12 @@ static int __balance_push_cpu_stop(void *arg) int cpu; scoped_guard (raw_spinlock_irq, &p->pi_lock) { + /* + * We may change the underlying rq, but the locks held will + * appropriately be "transferred" when switching. + */ + context_unsafe_alias(rq); + cpu = select_fallback_rq(rq->cpu, p); rq_lock(rq, &rf); @@ -8066,6 +8100,7 @@ static DEFINE_PER_CPU(struct cpu_stop_work, push_work); * effective when the hotplug motion is down. */ static void balance_push(struct rq *rq) + __must_hold(__rq_lockp(rq)) { struct task_struct *push_task = rq->curr; diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index da46c3164537..d0c929ecdb6a 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -2860,6 +2860,7 @@ static int preferred_group_nid(struct task_struct *p, int nid) } static void task_numa_placement(struct task_struct *p) + __context_unsafe(/* conditional locking */) { int seq, nid, max_nid = NUMA_NO_NODE; unsigned long max_faults = 0; @@ -4781,7 +4782,8 @@ static inline unsigned long cfs_rq_load_avg(struct cfs_rq *cfs_rq) return cfs_rq->avg.load_avg; } -static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf); +static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf) + __must_hold(__rq_lockp(this_rq)); static inline unsigned long task_util(struct task_struct *p) { @@ -6188,6 +6190,7 @@ next: * used to track this state. */ static int do_sched_cfs_period_timer(struct cfs_bandwidth *cfs_b, int overrun, unsigned long flags) + __must_hold(&cfs_b->lock) { int throttled; @@ -8919,6 +8922,7 @@ static void set_next_task_fair(struct rq *rq, struct task_struct *p, bool first) struct task_struct * pick_next_task_fair(struct rq *rq, struct task_struct *prev, struct rq_flags *rf) + __must_hold(__rq_lockp(rq)) { struct sched_entity *se; struct task_struct *p; @@ -12858,6 +12862,7 @@ static inline void nohz_newidle_balance(struct rq *this_rq) { } * > 0 - success, new (fair) tasks present */ static int sched_balance_newidle(struct rq *this_rq, struct rq_flags *rf) + __must_hold(__rq_lockp(this_rq)) { unsigned long next_balance = jiffies + HZ; int this_cpu = this_rq->cpu; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index d30cca6870f5..25d2ff265227 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1358,8 +1358,13 @@ static inline u32 sched_rng(void) return prandom_u32_state(this_cpu_ptr(&sched_rnd_state)); } +static __always_inline struct rq *__this_rq(void) +{ + return this_cpu_ptr(&runqueues); +} + #define cpu_rq(cpu) (&per_cpu(runqueues, (cpu))) -#define this_rq() this_cpu_ptr(&runqueues) +#define this_rq() __this_rq() #define task_rq(p) cpu_rq(task_cpu(p)) #define cpu_curr(cpu) (cpu_rq(cpu)->curr) #define raw_rq() raw_cpu_ptr(&runqueues) @@ -1404,6 +1409,7 @@ static inline raw_spinlock_t *rq_lockp(struct rq *rq) } static inline raw_spinlock_t *__rq_lockp(struct rq *rq) + __returns_ctx_lock(rq_lockp(rq)) /* alias them */ { if (rq->core_enabled) return &rq->core->__lock; @@ -1503,6 +1509,7 @@ static inline raw_spinlock_t *rq_lockp(struct rq *rq) } static inline raw_spinlock_t *__rq_lockp(struct rq *rq) + __returns_ctx_lock(rq_lockp(rq)) /* alias them */ { return &rq->__lock; } @@ -1545,32 +1552,42 @@ static inline bool rt_group_sched_enabled(void) #endif /* !CONFIG_RT_GROUP_SCHED */ static inline void lockdep_assert_rq_held(struct rq *rq) + __assumes_ctx_lock(__rq_lockp(rq)) { lockdep_assert_held(__rq_lockp(rq)); } -extern void raw_spin_rq_lock_nested(struct rq *rq, int subclass); -extern bool raw_spin_rq_trylock(struct rq *rq); -extern void raw_spin_rq_unlock(struct rq *rq); +extern void raw_spin_rq_lock_nested(struct rq *rq, int subclass) + __acquires(__rq_lockp(rq)); + +extern bool raw_spin_rq_trylock(struct rq *rq) + __cond_acquires(true, __rq_lockp(rq)); + +extern void raw_spin_rq_unlock(struct rq *rq) + __releases(__rq_lockp(rq)); static inline void raw_spin_rq_lock(struct rq *rq) + __acquires(__rq_lockp(rq)) { raw_spin_rq_lock_nested(rq, 0); } static inline void raw_spin_rq_lock_irq(struct rq *rq) + __acquires(__rq_lockp(rq)) { local_irq_disable(); raw_spin_rq_lock(rq); } static inline void raw_spin_rq_unlock_irq(struct rq *rq) + __releases(__rq_lockp(rq)) { raw_spin_rq_unlock(rq); local_irq_enable(); } static inline unsigned long _raw_spin_rq_lock_irqsave(struct rq *rq) + __acquires(__rq_lockp(rq)) { unsigned long flags; @@ -1581,6 +1598,7 @@ static inline unsigned long _raw_spin_rq_lock_irqsave(struct rq *rq) } static inline void raw_spin_rq_unlock_irqrestore(struct rq *rq, unsigned long flags) + __releases(__rq_lockp(rq)) { raw_spin_rq_unlock(rq); local_irq_restore(flags); @@ -1829,18 +1847,16 @@ static inline void rq_repin_lock(struct rq *rq, struct rq_flags *rf) rq->clock_update_flags |= rf->clock_update_flags; } -extern -struct rq *__task_rq_lock(struct task_struct *p, struct rq_flags *rf) - __acquires(rq->lock); +#define __task_rq_lock(...) __acquire_ret(___task_rq_lock(__VA_ARGS__), __rq_lockp(__ret)) +extern struct rq *___task_rq_lock(struct task_struct *p, struct rq_flags *rf) __acquires_ret; -extern -struct rq *task_rq_lock(struct task_struct *p, struct rq_flags *rf) - __acquires(p->pi_lock) - __acquires(rq->lock); +#define task_rq_lock(...) __acquire_ret(_task_rq_lock(__VA_ARGS__), __rq_lockp(__ret)) +extern struct rq *_task_rq_lock(struct task_struct *p, struct rq_flags *rf) + __acquires(&p->pi_lock) __acquires_ret; static inline void __task_rq_unlock(struct rq *rq, struct task_struct *p, struct rq_flags *rf) - __releases(rq->lock) + __releases(__rq_lockp(rq)) { rq_unpin_lock(rq, rf); raw_spin_rq_unlock(rq); @@ -1848,8 +1864,7 @@ __task_rq_unlock(struct rq *rq, struct task_struct *p, struct rq_flags *rf) static inline void task_rq_unlock(struct rq *rq, struct task_struct *p, struct rq_flags *rf) - __releases(rq->lock) - __releases(p->pi_lock) + __releases(__rq_lockp(rq), &p->pi_lock) { __task_rq_unlock(rq, p, rf); raw_spin_unlock_irqrestore(&p->pi_lock, rf->flags); @@ -1859,6 +1874,8 @@ DEFINE_LOCK_GUARD_1(task_rq_lock, struct task_struct, _T->rq = task_rq_lock(_T->lock, &_T->rf), task_rq_unlock(_T->rq, _T->lock, &_T->rf), struct rq *rq; struct rq_flags rf) +DECLARE_LOCK_GUARD_1_ATTRS(task_rq_lock, __acquires(_T->pi_lock), __releases((*(struct task_struct **)_T)->pi_lock)) +#define class_task_rq_lock_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(task_rq_lock, _T) DEFINE_LOCK_GUARD_1(__task_rq_lock, struct task_struct, _T->rq = __task_rq_lock(_T->lock, &_T->rf), @@ -1866,42 +1883,42 @@ DEFINE_LOCK_GUARD_1(__task_rq_lock, struct task_struct, struct rq *rq; struct rq_flags rf) static inline void rq_lock_irqsave(struct rq *rq, struct rq_flags *rf) - __acquires(rq->lock) + __acquires(__rq_lockp(rq)) { raw_spin_rq_lock_irqsave(rq, rf->flags); rq_pin_lock(rq, rf); } static inline void rq_lock_irq(struct rq *rq, struct rq_flags *rf) - __acquires(rq->lock) + __acquires(__rq_lockp(rq)) { raw_spin_rq_lock_irq(rq); rq_pin_lock(rq, rf); } static inline void rq_lock(struct rq *rq, struct rq_flags *rf) - __acquires(rq->lock) + __acquires(__rq_lockp(rq)) { raw_spin_rq_lock(rq); rq_pin_lock(rq, rf); } static inline void rq_unlock_irqrestore(struct rq *rq, struct rq_flags *rf) - __releases(rq->lock) + __releases(__rq_lockp(rq)) { rq_unpin_lock(rq, rf); raw_spin_rq_unlock_irqrestore(rq, rf->flags); } static inline void rq_unlock_irq(struct rq *rq, struct rq_flags *rf) - __releases(rq->lock) + __releases(__rq_lockp(rq)) { rq_unpin_lock(rq, rf); raw_spin_rq_unlock_irq(rq); } static inline void rq_unlock(struct rq *rq, struct rq_flags *rf) - __releases(rq->lock) + __releases(__rq_lockp(rq)) { rq_unpin_lock(rq, rf); raw_spin_rq_unlock(rq); @@ -1912,18 +1929,27 @@ DEFINE_LOCK_GUARD_1(rq_lock, struct rq, rq_unlock(_T->lock, &_T->rf), struct rq_flags rf) +DECLARE_LOCK_GUARD_1_ATTRS(rq_lock, __acquires(__rq_lockp(_T)), __releases(__rq_lockp(*(struct rq **)_T))); +#define class_rq_lock_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(rq_lock, _T) + DEFINE_LOCK_GUARD_1(rq_lock_irq, struct rq, rq_lock_irq(_T->lock, &_T->rf), rq_unlock_irq(_T->lock, &_T->rf), struct rq_flags rf) +DECLARE_LOCK_GUARD_1_ATTRS(rq_lock_irq, __acquires(__rq_lockp(_T)), __releases(__rq_lockp(*(struct rq **)_T))); +#define class_rq_lock_irq_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(rq_lock_irq, _T) + DEFINE_LOCK_GUARD_1(rq_lock_irqsave, struct rq, rq_lock_irqsave(_T->lock, &_T->rf), rq_unlock_irqrestore(_T->lock, &_T->rf), struct rq_flags rf) -static inline struct rq *this_rq_lock_irq(struct rq_flags *rf) - __acquires(rq->lock) +DECLARE_LOCK_GUARD_1_ATTRS(rq_lock_irqsave, __acquires(__rq_lockp(_T)), __releases(__rq_lockp(*(struct rq **)_T))); +#define class_rq_lock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(rq_lock_irqsave, _T) + +#define this_rq_lock_irq(...) __acquire_ret(_this_rq_lock_irq(__VA_ARGS__), __rq_lockp(__ret)) +static inline struct rq *_this_rq_lock_irq(struct rq_flags *rf) __acquires_ret { struct rq *rq; @@ -3050,8 +3076,20 @@ static inline void double_rq_clock_clear_update(struct rq *rq1, struct rq *rq2) #define DEFINE_LOCK_GUARD_2(name, type, _lock, _unlock, ...) \ __DEFINE_UNLOCK_GUARD(name, type, _unlock, type *lock2; __VA_ARGS__) \ static inline class_##name##_t class_##name##_constructor(type *lock, type *lock2) \ + __no_context_analysis \ { class_##name##_t _t = { .lock = lock, .lock2 = lock2 }, *_T = &_t; \ _lock; return _t; } +#define DECLARE_LOCK_GUARD_2_ATTRS(_name, _lock, _unlock1, _unlock2) \ +static inline class_##_name##_t class_##_name##_constructor(lock_##_name##_t *_T1, \ + lock_##_name##_t *_T2) _lock; \ +static __always_inline void __class_##_name##_cleanup_ctx1(class_##_name##_t **_T1) \ + __no_context_analysis _unlock1 { } \ +static __always_inline void __class_##_name##_cleanup_ctx2(class_##_name##_t **_T2) \ + __no_context_analysis _unlock2 { } +#define WITH_LOCK_GUARD_2_ATTRS(_name, _T1, _T2) \ + class_##_name##_constructor(_T1, _T2), \ + *__UNIQUE_ID(unlock1) __cleanup(__class_##_name##_cleanup_ctx1) = (void *)(_T1),\ + *__UNIQUE_ID(unlock2) __cleanup(__class_##_name##_cleanup_ctx2) = (void *)(_T2) static inline bool rq_order_less(struct rq *rq1, struct rq *rq2) { @@ -3079,7 +3117,8 @@ static inline bool rq_order_less(struct rq *rq1, struct rq *rq2) return rq1->cpu < rq2->cpu; } -extern void double_rq_lock(struct rq *rq1, struct rq *rq2); +extern void double_rq_lock(struct rq *rq1, struct rq *rq2) + __acquires(__rq_lockp(rq1), __rq_lockp(rq2)); #ifdef CONFIG_PREEMPTION @@ -3092,9 +3131,8 @@ extern void double_rq_lock(struct rq *rq1, struct rq *rq2); * also adds more overhead and therefore may reduce throughput. */ static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest) - __releases(this_rq->lock) - __acquires(busiest->lock) - __acquires(this_rq->lock) + __must_hold(__rq_lockp(this_rq)) + __acquires(__rq_lockp(busiest)) { raw_spin_rq_unlock(this_rq); double_rq_lock(this_rq, busiest); @@ -3111,12 +3149,16 @@ static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest) * regardless of entry order into the function. */ static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest) - __releases(this_rq->lock) - __acquires(busiest->lock) - __acquires(this_rq->lock) + __must_hold(__rq_lockp(this_rq)) + __acquires(__rq_lockp(busiest)) { - if (__rq_lockp(this_rq) == __rq_lockp(busiest) || - likely(raw_spin_rq_trylock(busiest))) { + if (__rq_lockp(this_rq) == __rq_lockp(busiest)) { + __acquire(__rq_lockp(busiest)); /* already held */ + double_rq_clock_clear_update(this_rq, busiest); + return 0; + } + + if (likely(raw_spin_rq_trylock(busiest))) { double_rq_clock_clear_update(this_rq, busiest); return 0; } @@ -3139,6 +3181,8 @@ static inline int _double_lock_balance(struct rq *this_rq, struct rq *busiest) * double_lock_balance - lock the busiest runqueue, this_rq is locked already. */ static inline int double_lock_balance(struct rq *this_rq, struct rq *busiest) + __must_hold(__rq_lockp(this_rq)) + __acquires(__rq_lockp(busiest)) { lockdep_assert_irqs_disabled(); @@ -3146,14 +3190,17 @@ static inline int double_lock_balance(struct rq *this_rq, struct rq *busiest) } static inline void double_unlock_balance(struct rq *this_rq, struct rq *busiest) - __releases(busiest->lock) + __releases(__rq_lockp(busiest)) { if (__rq_lockp(this_rq) != __rq_lockp(busiest)) raw_spin_rq_unlock(busiest); + else + __release(__rq_lockp(busiest)); /* fake release */ lock_set_subclass(&__rq_lockp(this_rq)->dep_map, 0, _RET_IP_); } static inline void double_lock(spinlock_t *l1, spinlock_t *l2) + __acquires(l1, l2) { if (l1 > l2) swap(l1, l2); @@ -3163,6 +3210,7 @@ static inline void double_lock(spinlock_t *l1, spinlock_t *l2) } static inline void double_lock_irq(spinlock_t *l1, spinlock_t *l2) + __acquires(l1, l2) { if (l1 > l2) swap(l1, l2); @@ -3172,6 +3220,7 @@ static inline void double_lock_irq(spinlock_t *l1, spinlock_t *l2) } static inline void double_raw_lock(raw_spinlock_t *l1, raw_spinlock_t *l2) + __acquires(l1, l2) { if (l1 > l2) swap(l1, l2); @@ -3181,6 +3230,7 @@ static inline void double_raw_lock(raw_spinlock_t *l1, raw_spinlock_t *l2) } static inline void double_raw_unlock(raw_spinlock_t *l1, raw_spinlock_t *l2) + __releases(l1, l2) { raw_spin_unlock(l1); raw_spin_unlock(l2); @@ -3190,6 +3240,13 @@ DEFINE_LOCK_GUARD_2(double_raw_spinlock, raw_spinlock_t, double_raw_lock(_T->lock, _T->lock2), double_raw_unlock(_T->lock, _T->lock2)) +DECLARE_LOCK_GUARD_2_ATTRS(double_raw_spinlock, + __acquires(_T1, _T2), + __releases(*(raw_spinlock_t **)_T1), + __releases(*(raw_spinlock_t **)_T2)); +#define class_double_raw_spinlock_constructor(_T1, _T2) \ + WITH_LOCK_GUARD_2_ATTRS(double_raw_spinlock, _T1, _T2) + /* * double_rq_unlock - safely unlock two runqueues * @@ -3197,13 +3254,12 @@ DEFINE_LOCK_GUARD_2(double_raw_spinlock, raw_spinlock_t, * you need to do so manually after calling. */ static inline void double_rq_unlock(struct rq *rq1, struct rq *rq2) - __releases(rq1->lock) - __releases(rq2->lock) + __releases(__rq_lockp(rq1), __rq_lockp(rq2)) { if (__rq_lockp(rq1) != __rq_lockp(rq2)) raw_spin_rq_unlock(rq2); else - __release(rq2->lock); + __release(__rq_lockp(rq2)); /* fake release */ raw_spin_rq_unlock(rq1); } diff --git a/scripts/context-analysis-suppression.txt b/scripts/context-analysis-suppression.txt index df25c3d07a5b..fd8951d06706 100644 --- a/scripts/context-analysis-suppression.txt +++ b/scripts/context-analysis-suppression.txt @@ -26,6 +26,7 @@ src:*include/linux/refcount.h=emit src:*include/linux/rhashtable.h=emit src:*include/linux/rwlock*.h=emit src:*include/linux/rwsem.h=emit +src:*include/linux/sched*=emit src:*include/linux/seqlock*.h=emit src:*include/linux/spinlock*.h=emit src:*include/linux/srcu*.h=emit From c10d860e0baae0853773dc90a94b26adc5687380 Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Mon, 5 Jan 2026 16:24:05 +0100 Subject: [PATCH 40/90] tags: Add regex for context_lock_struct With the introduction of compiler context analysis (LLVM ThreadSafetyAnalysis) the struct definition of various locks get wrapped in a macro. This hides them from tags based navigation, although clangd/LSP sees right through it and works as expected. Add a regex to the tags script to help it along. Requested-by: Bart Van Assche Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20251220133307.GR3707891@noisy.programming.kicks-ass.net --- scripts/tags.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/tags.sh b/scripts/tags.sh index 99ce427d9a69..243373683f98 100755 --- a/scripts/tags.sh +++ b/scripts/tags.sh @@ -221,6 +221,7 @@ regex_c=( '/^\ Date: Tue, 6 Jan 2026 12:01:58 +0800 Subject: [PATCH 41/90] scripts/atomic: Fix kerneldoc spelling in try_cmpxchg() Fix a typo in the kerneldoc comment template. This changes 'occured' to 'occurred' in generated documentation. Signed-off-by: oldzhu Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260106040158.31461-1-oldrunner999@gmail.com --- include/linux/atomic/atomic-arch-fallback.h | 18 +++++++------- include/linux/atomic/atomic-instrumented.h | 26 ++++++++++----------- include/linux/atomic/atomic-long.h | 10 ++++---- scripts/atomic/kerneldoc/try_cmpxchg | 2 +- 4 files changed, 28 insertions(+), 28 deletions(-) diff --git a/include/linux/atomic/atomic-arch-fallback.h b/include/linux/atomic/atomic-arch-fallback.h index 2f9d36b72bd8..cdc25f8979f7 100644 --- a/include/linux/atomic/atomic-arch-fallback.h +++ b/include/linux/atomic/atomic-arch-fallback.h @@ -2121,7 +2121,7 @@ raw_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) * * Safe to use in noinstr code; prefer atomic_try_cmpxchg() elsewhere. * - * Return: @true if the exchange occured, @false otherwise. + * Return: @true if the exchange occurred, @false otherwise. */ static __always_inline bool raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new) @@ -2155,7 +2155,7 @@ raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new) * * Safe to use in noinstr code; prefer atomic_try_cmpxchg_acquire() elsewhere. * - * Return: @true if the exchange occured, @false otherwise. + * Return: @true if the exchange occurred, @false otherwise. */ static __always_inline bool raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) @@ -2189,7 +2189,7 @@ raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) * * Safe to use in noinstr code; prefer atomic_try_cmpxchg_release() elsewhere. * - * Return: @true if the exchange occured, @false otherwise. + * Return: @true if the exchange occurred, @false otherwise. */ static __always_inline bool raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) @@ -2222,7 +2222,7 @@ raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) * * Safe to use in noinstr code; prefer atomic_try_cmpxchg_relaxed() elsewhere. * - * Return: @true if the exchange occured, @false otherwise. + * Return: @true if the exchange occurred, @false otherwise. */ static __always_inline bool raw_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) @@ -4247,7 +4247,7 @@ raw_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) * * Safe to use in noinstr code; prefer atomic64_try_cmpxchg() elsewhere. * - * Return: @true if the exchange occured, @false otherwise. + * Return: @true if the exchange occurred, @false otherwise. */ static __always_inline bool raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) @@ -4281,7 +4281,7 @@ raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) * * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_acquire() elsewhere. * - * Return: @true if the exchange occured, @false otherwise. + * Return: @true if the exchange occurred, @false otherwise. */ static __always_inline bool raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) @@ -4315,7 +4315,7 @@ raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) * * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_release() elsewhere. * - * Return: @true if the exchange occured, @false otherwise. + * Return: @true if the exchange occurred, @false otherwise. */ static __always_inline bool raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) @@ -4348,7 +4348,7 @@ raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) * * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_relaxed() elsewhere. * - * Return: @true if the exchange occured, @false otherwise. + * Return: @true if the exchange occurred, @false otherwise. */ static __always_inline bool raw_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) @@ -4690,4 +4690,4 @@ raw_atomic64_dec_if_positive(atomic64_t *v) } #endif /* _LINUX_ATOMIC_FALLBACK_H */ -// b565db590afeeff0d7c9485ccbca5bb6e155749f +// 206314f82b8b73a5c3aa69cf7f35ac9e7b5d6b58 diff --git a/include/linux/atomic/atomic-instrumented.h b/include/linux/atomic/atomic-instrumented.h index 37ab6314a9f7..feb3b5dc3e96 100644 --- a/include/linux/atomic/atomic-instrumented.h +++ b/include/linux/atomic/atomic-instrumented.h @@ -1269,7 +1269,7 @@ atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) * * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg() there. * - * Return: @true if the exchange occured, @false otherwise. + * Return: @true if the exchange occurred, @false otherwise. */ static __always_inline bool atomic_try_cmpxchg(atomic_t *v, int *old, int new) @@ -1292,7 +1292,7 @@ atomic_try_cmpxchg(atomic_t *v, int *old, int new) * * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_acquire() there. * - * Return: @true if the exchange occured, @false otherwise. + * Return: @true if the exchange occurred, @false otherwise. */ static __always_inline bool atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) @@ -1314,7 +1314,7 @@ atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) * * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_release() there. * - * Return: @true if the exchange occured, @false otherwise. + * Return: @true if the exchange occurred, @false otherwise. */ static __always_inline bool atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) @@ -1337,7 +1337,7 @@ atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) * * Unsafe to use in noinstr code; use raw_atomic_try_cmpxchg_relaxed() there. * - * Return: @true if the exchange occured, @false otherwise. + * Return: @true if the exchange occurred, @false otherwise. */ static __always_inline bool atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) @@ -2847,7 +2847,7 @@ atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) * * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg() there. * - * Return: @true if the exchange occured, @false otherwise. + * Return: @true if the exchange occurred, @false otherwise. */ static __always_inline bool atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) @@ -2870,7 +2870,7 @@ atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) * * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_acquire() there. * - * Return: @true if the exchange occured, @false otherwise. + * Return: @true if the exchange occurred, @false otherwise. */ static __always_inline bool atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) @@ -2892,7 +2892,7 @@ atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) * * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_release() there. * - * Return: @true if the exchange occured, @false otherwise. + * Return: @true if the exchange occurred, @false otherwise. */ static __always_inline bool atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) @@ -2915,7 +2915,7 @@ atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) * * Unsafe to use in noinstr code; use raw_atomic64_try_cmpxchg_relaxed() there. * - * Return: @true if the exchange occured, @false otherwise. + * Return: @true if the exchange occurred, @false otherwise. */ static __always_inline bool atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) @@ -4425,7 +4425,7 @@ atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) * * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg() there. * - * Return: @true if the exchange occured, @false otherwise. + * Return: @true if the exchange occurred, @false otherwise. */ static __always_inline bool atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) @@ -4448,7 +4448,7 @@ atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) * * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_acquire() there. * - * Return: @true if the exchange occured, @false otherwise. + * Return: @true if the exchange occurred, @false otherwise. */ static __always_inline bool atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) @@ -4470,7 +4470,7 @@ atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) * * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_release() there. * - * Return: @true if the exchange occured, @false otherwise. + * Return: @true if the exchange occurred, @false otherwise. */ static __always_inline bool atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) @@ -4493,7 +4493,7 @@ atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) * * Unsafe to use in noinstr code; use raw_atomic_long_try_cmpxchg_relaxed() there. * - * Return: @true if the exchange occured, @false otherwise. + * Return: @true if the exchange occurred, @false otherwise. */ static __always_inline bool atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) @@ -5050,4 +5050,4 @@ atomic_long_dec_if_positive(atomic_long_t *v) #endif /* _LINUX_ATOMIC_INSTRUMENTED_H */ -// f618ac667f868941a84ce0ab2242f1786e049ed4 +// 9dd948d3012b22c4e75933a5172983f912e46439 diff --git a/include/linux/atomic/atomic-long.h b/include/linux/atomic/atomic-long.h index f86b29d90877..6a4e47d2db35 100644 --- a/include/linux/atomic/atomic-long.h +++ b/include/linux/atomic/atomic-long.h @@ -1449,7 +1449,7 @@ raw_atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) * * Safe to use in noinstr code; prefer atomic_long_try_cmpxchg() elsewhere. * - * Return: @true if the exchange occured, @false otherwise. + * Return: @true if the exchange occurred, @false otherwise. */ static __always_inline bool raw_atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) @@ -1473,7 +1473,7 @@ raw_atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) * * Safe to use in noinstr code; prefer atomic_long_try_cmpxchg_acquire() elsewhere. * - * Return: @true if the exchange occured, @false otherwise. + * Return: @true if the exchange occurred, @false otherwise. */ static __always_inline bool raw_atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) @@ -1497,7 +1497,7 @@ raw_atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) * * Safe to use in noinstr code; prefer atomic_long_try_cmpxchg_release() elsewhere. * - * Return: @true if the exchange occured, @false otherwise. + * Return: @true if the exchange occurred, @false otherwise. */ static __always_inline bool raw_atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) @@ -1521,7 +1521,7 @@ raw_atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) * * Safe to use in noinstr code; prefer atomic_long_try_cmpxchg_relaxed() elsewhere. * - * Return: @true if the exchange occured, @false otherwise. + * Return: @true if the exchange occurred, @false otherwise. */ static __always_inline bool raw_atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) @@ -1809,4 +1809,4 @@ raw_atomic_long_dec_if_positive(atomic_long_t *v) } #endif /* _LINUX_ATOMIC_LONG_H */ -// eadf183c3600b8b92b91839dd3be6bcc560c752d +// 4b882bf19018602c10816c52f8b4ae280adc887b diff --git a/scripts/atomic/kerneldoc/try_cmpxchg b/scripts/atomic/kerneldoc/try_cmpxchg index 3ccff29538f5..4dfc7a167ea1 100644 --- a/scripts/atomic/kerneldoc/try_cmpxchg +++ b/scripts/atomic/kerneldoc/try_cmpxchg @@ -11,6 +11,6 @@ cat < Date: Wed, 7 Jan 2026 23:35:49 +0100 Subject: [PATCH 42/90] compiler-context-analysys: Fix CONFIG_MODVERSION The robot reported CONFIG_MODVERSION fails; which Nathan described as so: > Something about the context analysis makes genksyms fall over, running > it manually on kernel/sched/core.i with '-w' to show warnings reveals > many new "syntax error" instances. I don't see any warnings when using > gendwarfksyms. Maybe it is context_lock_struct, as that is the first > error I see in the list: > > include/linux/spinlock_types_raw.h:14: syntax error Reported-by: kernel test robot Debugged-by: Nathan Chancellor Closes: https://lore.kernel.org/oe-kbuild-all/202512222219.F6EkVNmQ-lkp@intel.com/ Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/aV7fxXjaOBtHhI9X@elver.google.com --- include/linux/compiler-context-analysis.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/compiler-context-analysis.h b/include/linux/compiler-context-analysis.h index 4f7559d7ae91..e86b8a3c2f89 100644 --- a/include/linux/compiler-context-analysis.h +++ b/include/linux/compiler-context-analysis.h @@ -6,7 +6,7 @@ #ifndef _LINUX_COMPILER_CONTEXT_ANALYSIS_H #define _LINUX_COMPILER_CONTEXT_ANALYSIS_H -#if defined(WARN_CONTEXT_ANALYSIS) && !defined(__CHECKER__) +#if defined(WARN_CONTEXT_ANALYSIS) && !defined(__CHECKER__) && !defined(__GENKSYMS__) /* * These attributes define new context lock (Clang: capability) types. From a45026cef17d1080c985adf28234d6c8475ad66f Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Thu, 8 Jan 2026 11:14:24 +0100 Subject: [PATCH 43/90] locking/local_lock: Include more missing headers Ingo reported PREEMPT_RT=y builds fail building lib/test_context-analysis.c. Reported-by: Ingo Molnar Signed-off-by: Peter Zijlstra (Intel) --- include/linux/local_lock_internal.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/linux/local_lock_internal.h b/include/linux/local_lock_internal.h index e8c4803d8db4..7843ab9059c2 100644 --- a/include/linux/local_lock_internal.h +++ b/include/linux/local_lock_internal.h @@ -233,6 +233,7 @@ do { \ #else /* !CONFIG_PREEMPT_RT */ +#include #include /* From 86f4a271dc1962e389ea512d07a77626dbd8c1d8 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 11 Aug 2025 12:14:41 +0000 Subject: [PATCH 44/90] rust: sync: Refactor static_lock_class!() macro By introducing a new_static() constructor, the macro does not need to go through MaybeUninit::uninit().assume_init(), which is a pattern that is best avoided when possible. The safety comment not only requires that the value is leaked, but also that it is stored in the right portion of memory. This is so that the lockdep static_obj() check will succeed when using this constructor. One could argue that lockdep detects this scenario, so that safety requirement isn't needed. However, it simplifies matters to require that static_obj() will succeed and it's not a burdensome requirement on the caller. Suggested-by: Benno Lossin Reviewed-by: Daniel Almeida Reviewed-by: Benno Lossin Signed-off-by: Alice Ryhl Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20250811-lock-class-key-cleanup-v3-1-b12967ee1ca2@google.com --- rust/kernel/sync.rs | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index 5df87e2bd212..1dfbee8e9d00 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -45,6 +45,21 @@ pub struct LockClassKey { unsafe impl Sync for LockClassKey {} impl LockClassKey { + /// Initializes a statically allocated lock class key. + /// + /// This is usually used indirectly through the [`static_lock_class!`] macro. + /// + /// # Safety + /// + /// * Before using the returned value, it must be pinned in a static memory location. + /// * The destructor must never run on the returned `LockClassKey`. + #[doc(hidden)] + pub const unsafe fn new_static() -> Self { + LockClassKey { + inner: Opaque::uninit(), + } + } + /// Initializes a dynamically allocated lock class key. In the common case of using a /// statically allocated lock class key, the static_lock_class! macro should be used instead. /// @@ -101,12 +116,9 @@ impl PinnedDrop for LockClassKey { macro_rules! static_lock_class { () => {{ static CLASS: $crate::sync::LockClassKey = - // Lockdep expects uninitialized memory when it's handed a statically allocated `struct - // lock_class_key`. - // - // SAFETY: `LockClassKey` transparently wraps `Opaque` which permits uninitialized - // memory. - unsafe { ::core::mem::MaybeUninit::uninit().assume_init() }; + // SAFETY: The returned `LockClassKey` is stored in static memory and we pin it. Drop + // never runs on a static global. + unsafe { $crate::sync::LockClassKey::new_static() }; $crate::prelude::Pin::static_ref(&CLASS) }}; } From 106ab474e5a711ea08e0908a42cfa89d691e57ad Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 11 Aug 2025 12:14:42 +0000 Subject: [PATCH 45/90] rust: sync: Clean up LockClassKey and its docs Several aspects of the code and documentation for this type is incomplete. Also several things are hidden from the docs. Thus, clean it up and make it easier to read the rendered html docs. Reviewed-by: Daniel Almeida Signed-off-by: Alice Ryhl Reviewed-by: Benno Lossin Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20250811-lock-class-key-cleanup-v3-2-b12967ee1ca2@google.com --- rust/kernel/sync.rs | 54 +++++++++++++++++++++++++++++++++------------ 1 file changed, 40 insertions(+), 14 deletions(-) diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index 1dfbee8e9d00..b10e576221ff 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -32,7 +32,9 @@ pub use locked_by::LockedBy; pub use refcount::Refcount; pub use set_once::SetOnce; -/// Represents a lockdep class. It's a wrapper around C's `lock_class_key`. +/// Represents a lockdep class. +/// +/// Wraps the kernel's `struct lock_class_key`. #[repr(transparent)] #[pin_data(PinnedDrop)] pub struct LockClassKey { @@ -40,6 +42,10 @@ pub struct LockClassKey { inner: Opaque, } +// SAFETY: Unregistering a lock class key from a different thread than where it was registered is +// allowed. +unsafe impl Send for LockClassKey {} + // SAFETY: `bindings::lock_class_key` is designed to be used concurrently from multiple threads and // provides its own synchronization. unsafe impl Sync for LockClassKey {} @@ -47,28 +53,31 @@ unsafe impl Sync for LockClassKey {} impl LockClassKey { /// Initializes a statically allocated lock class key. /// - /// This is usually used indirectly through the [`static_lock_class!`] macro. + /// This is usually used indirectly through the [`static_lock_class!`] macro. See its + /// documentation for more information. /// /// # Safety /// /// * Before using the returned value, it must be pinned in a static memory location. /// * The destructor must never run on the returned `LockClassKey`. - #[doc(hidden)] pub const unsafe fn new_static() -> Self { LockClassKey { inner: Opaque::uninit(), } } - /// Initializes a dynamically allocated lock class key. In the common case of using a - /// statically allocated lock class key, the static_lock_class! macro should be used instead. + /// Initializes a dynamically allocated lock class key. + /// + /// In the common case of using a statically allocated lock class key, the + /// [`static_lock_class!`] macro should be used instead. /// /// # Examples + /// /// ``` - /// # use kernel::alloc::KBox; - /// # use kernel::types::ForeignOwnable; - /// # use kernel::sync::{LockClassKey, SpinLock}; - /// # use pin_init::stack_pin_init; + /// use kernel::alloc::KBox; + /// use kernel::types::ForeignOwnable; + /// use kernel::sync::{LockClassKey, SpinLock}; + /// use pin_init::stack_pin_init; /// /// let key = KBox::pin_init(LockClassKey::new_dynamic(), GFP_KERNEL)?; /// let key_ptr = key.into_foreign(); @@ -86,7 +95,6 @@ impl LockClassKey { /// // SAFETY: We dropped `num`, the only use of the key, so the result of the previous /// // `borrow` has also been dropped. Thus, it's safe to use from_foreign. /// unsafe { drop(> as ForeignOwnable>::from_foreign(key_ptr)) }; - /// /// # Ok::<(), Error>(()) /// ``` pub fn new_dynamic() -> impl PinInit { @@ -96,7 +104,10 @@ impl LockClassKey { }) } - pub(crate) fn as_ptr(&self) -> *mut bindings::lock_class_key { + /// Returns a raw pointer to the inner C struct. + /// + /// It is up to the caller to use the raw pointer correctly. + pub fn as_ptr(&self) -> *mut bindings::lock_class_key { self.inner.get() } } @@ -104,14 +115,28 @@ impl LockClassKey { #[pinned_drop] impl PinnedDrop for LockClassKey { fn drop(self: Pin<&mut Self>) { - // SAFETY: self.as_ptr was registered with lockdep and self is pinned, so the address - // hasn't changed. Thus, it's safe to pass to unregister. + // SAFETY: `self.as_ptr()` was registered with lockdep and `self` is pinned, so the address + // hasn't changed. Thus, it's safe to pass it to unregister. unsafe { bindings::lockdep_unregister_key(self.as_ptr()) } } } /// Defines a new static lock class and returns a pointer to it. -#[doc(hidden)] +/// +/// # Examples +/// +/// ``` +/// use kernel::c_str; +/// use kernel::sync::{static_lock_class, Arc, SpinLock}; +/// +/// fn new_locked_int() -> Result>> { +/// Arc::pin_init(SpinLock::new( +/// 42, +/// c_str!("new_locked_int"), +/// static_lock_class!(), +/// ), GFP_KERNEL) +/// } +/// ``` #[macro_export] macro_rules! static_lock_class { () => {{ @@ -122,6 +147,7 @@ macro_rules! static_lock_class { $crate::prelude::Pin::static_ref(&CLASS) }}; } +pub use static_lock_class; /// Returns the given string, if one is provided, otherwise generates one based on the source code /// location. From 8a581130b1cbc17c702298b8325e3df98c792760 Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Tue, 16 Dec 2025 09:09:01 +0900 Subject: [PATCH 46/90] rust: sync: set_once: Implement Send and Sync Implement Send and Sync for SetOnce to allow it to be used across thread boundaries. Send: SetOnce can be transferred across threads when T: Send, as the contained value is also transferred and will be dropped on the destination thread. Sync: SetOnce can be shared across threads when T: Sync, as as_ref() provides shared references &T and atomic operations ensure proper synchronization. Since the inner T may be dropped on any thread, we also require T: Send. Signed-off-by: FUJITA Tomonori Reviewed-by: Andreas Hindborg Reviewed-by: Gary Guo Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20251216000901.221375-1-fujita.tomonori@gmail.com --- rust/kernel/sync/set_once.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/rust/kernel/sync/set_once.rs b/rust/kernel/sync/set_once.rs index bdba601807d8..139cef05e935 100644 --- a/rust/kernel/sync/set_once.rs +++ b/rust/kernel/sync/set_once.rs @@ -123,3 +123,11 @@ impl Drop for SetOnce { } } } + +// SAFETY: `SetOnce` can be transferred across thread boundaries iff the data it contains can. +unsafe impl Send for SetOnce {} + +// SAFETY: `SetOnce` synchronises access to the inner value via atomic operations, +// so shared references are safe when `T: Sync`. Since the inner `T` may be dropped +// on any thread, we also require `T: Send`. +unsafe impl Sync for SetOnce {} From 09248ed8cdb6345afc883c02aecd79dfbd9c2a9c Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Thu, 18 Dec 2025 08:25:13 +0000 Subject: [PATCH 47/90] rust: sync: Implement Unpin for ARef The default implementation of Unpin for ARef is conditional on T being Unpin due to its PhantomData field. However, this is overly strict as pointers to T are legal to move even if T itself cannot move. Since commit 66f1ea83d9f8 ("rust: lock: Add a Pin<&mut T> accessor") this causes build failures when combined with a Mutex that contains an field ARef, because almost any type that ARef is used with is !Unpin. Reviewed-by: Daniel Almeida Signed-off-by: Alice Ryhl Reviewed-by: Alexandre Courbot Reviewed-by: Benno Lossin Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20251218-unpin-for-aref-v2-1-30d77129cbc6@google.com --- rust/kernel/sync/aref.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rust/kernel/sync/aref.rs b/rust/kernel/sync/aref.rs index 0d24a0432015..0616c0353c2b 100644 --- a/rust/kernel/sync/aref.rs +++ b/rust/kernel/sync/aref.rs @@ -83,6 +83,9 @@ unsafe impl Send for ARef {} // example, when the reference count reaches zero and `T` is dropped. unsafe impl Sync for ARef {} +// Even if `T` is pinned, pointers to `T` can still move. +impl Unpin for ARef {} + impl ARef { /// Creates a new instance of [`ARef`]. /// From 2cc3d5d6adbee058858f2e66de701a203b032746 Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Thu, 11 Dec 2025 20:38:23 +0900 Subject: [PATCH 48/90] rust: helpers: Add i8/i16 atomic_read_acquire/atomic_set_release helpers Add helper functions to expose smp_load_acquire() and smp_store_release() for i8 and i16 types. The smp_load_acquire() and smp_store_release() macros require type information (sizeof) to generate appropriate architecture-specific memory ordering instructions. Therefore, separate helper functions are needed for each type size. These helpers expose different symbol names than their C counterparts so they are split into atomic_ext.c instead of atomic.c. The symbol names; atomic_[i8|i16]_read_acquire and atomic_[i8|i16]_set_release makes the interface Rust/C clear, consistent with i32/i64. These helpers will be used by the upcoming Atomic and Atomic implementation to provide proper Acquire/Release semantics across all architectures. [boqun: Rename the functions from {load,store} to {read,set} to avoid future adjustment] Signed-off-by: FUJITA Tomonori Reviewed-by: Gary Guo Reviewed-by: Joel Fernandes Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20251211113826.1299077-2-fujita.tomonori@gmail.com --- rust/helpers/atomic_ext.c | 23 +++++++++++++++++++++++ rust/helpers/helpers.c | 1 + 2 files changed, 24 insertions(+) create mode 100644 rust/helpers/atomic_ext.c diff --git a/rust/helpers/atomic_ext.c b/rust/helpers/atomic_ext.c new file mode 100644 index 000000000000..1fb624147aa4 --- /dev/null +++ b/rust/helpers/atomic_ext.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include + +__rust_helper s8 rust_helper_atomic_i8_read_acquire(s8 *ptr) +{ + return smp_load_acquire(ptr); +} + +__rust_helper s16 rust_helper_atomic_i16_read_acquire(s16 *ptr) +{ + return smp_load_acquire(ptr); +} + +__rust_helper void rust_helper_atomic_i8_set_release(s8 *ptr, s8 val) +{ + smp_store_release(ptr, val); +} + +__rust_helper void rust_helper_atomic_i16_set_release(s16 *ptr, s16 val) +{ + smp_store_release(ptr, val); +} diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 79c72762ad9c..15d75578f459 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -8,6 +8,7 @@ */ #include "atomic.c" +#include "atomic_ext.c" #include "auxiliary.c" #include "barrier.c" #include "binder.c" From 300e53b3d3b59e72a972a12ee5c6438aab4860a4 Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Thu, 11 Dec 2025 20:38:24 +0900 Subject: [PATCH 49/90] rust: helpers: Add i8/i16 relaxed atomic helpers Add READ_ONCE/WRITE_ONCE based helpers for i8 and i16 types to support relaxed atomic operations in Rust. While relaxed operations could be implemented purely in Rust using read_volatile() and write_volatile(), using C's READ_ONCE() and WRITE_ONCE() macros ensures complete consistency with the kernel memory model. These helpers expose different symbol names than their C counterparts so they are split into atomic_ext.c instead of atomic.c. The symbol names; the names make the interface Rust/C clear, consistent with i32/i64. [boqun: Rename the functions from {load,store} to {read,set} to avoid future adjustment] Signed-off-by: FUJITA Tomonori Reviewed-by: Gary Guo Reviewed-by: Joel Fernandes Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20251211113826.1299077-3-fujita.tomonori@gmail.com --- rust/helpers/atomic_ext.c | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/rust/helpers/atomic_ext.c b/rust/helpers/atomic_ext.c index 1fb624147aa4..02e05b4246ae 100644 --- a/rust/helpers/atomic_ext.c +++ b/rust/helpers/atomic_ext.c @@ -1,22 +1,43 @@ // SPDX-License-Identifier: GPL-2.0 #include +#include + +__rust_helper s8 rust_helper_atomic_i8_read(s8 *ptr) +{ + return READ_ONCE(*ptr); +} __rust_helper s8 rust_helper_atomic_i8_read_acquire(s8 *ptr) { return smp_load_acquire(ptr); } +__rust_helper s16 rust_helper_atomic_i16_read(s16 *ptr) +{ + return READ_ONCE(*ptr); +} + __rust_helper s16 rust_helper_atomic_i16_read_acquire(s16 *ptr) { return smp_load_acquire(ptr); } +__rust_helper void rust_helper_atomic_i8_set(s8 *ptr, s8 val) +{ + WRITE_ONCE(*ptr, val); +} + __rust_helper void rust_helper_atomic_i8_set_release(s8 *ptr, s8 val) { smp_store_release(ptr, val); } +__rust_helper void rust_helper_atomic_i16_set(s16 *ptr, s16 val) +{ + WRITE_ONCE(*ptr, val); +} + __rust_helper void rust_helper_atomic_i16_set_release(s16 *ptr, s16 val) { smp_store_release(ptr, val); From 5dbc0a692459bc49cdb7add281086291da547750 Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Tue, 23 Dec 2025 15:21:37 +0900 Subject: [PATCH 50/90] rust: helpers: Add i8/i16 atomic xchg helpers Add i8/i16 atomic xchg helpers that call xchg() macro implementing atomic xchg using architecture-specific instructions. [boqun: Use xchg() instead of raw_xchg()] Reviewed-by: Alice Ryhl Signed-off-by: FUJITA Tomonori Reviewed-by: Gary Guo Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20251223062140.938325-2-fujita.tomonori@gmail.com --- rust/helpers/atomic_ext.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/rust/helpers/atomic_ext.c b/rust/helpers/atomic_ext.c index 02e05b4246ae..3136255a84c6 100644 --- a/rust/helpers/atomic_ext.c +++ b/rust/helpers/atomic_ext.c @@ -2,6 +2,7 @@ #include #include +#include __rust_helper s8 rust_helper_atomic_i8_read(s8 *ptr) { @@ -42,3 +43,20 @@ __rust_helper void rust_helper_atomic_i16_set_release(s16 *ptr, s16 val) { smp_store_release(ptr, val); } + +/* + * xchg helpers depend on ARCH_SUPPORTS_ATOMIC_RMW and on the + * architecture provding xchg() support for i8 and i16. + * + * The architectures that currently support Rust (x86_64, armv7, + * arm64, riscv, and loongarch) satisfy these requirements. + */ +__rust_helper s8 rust_helper_atomic_i8_xchg(s8 *ptr, s8 new) +{ + return xchg(ptr, new); +} + +__rust_helper s16 rust_helper_atomic_i16_xchg(s16 *ptr, s16 new) +{ + return xchg(ptr, new); +} From ab717dd98bee964add2161d94193d756fdef614c Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Tue, 23 Dec 2025 15:21:38 +0900 Subject: [PATCH 51/90] rust: helpers: Add i8/i16 atomic xchg_acquire helpers Add i8/i16 atomic xchg_acquire helpers that call xchg_acquire() macro implementing atomic xchg_acquire using architecture-specific instructions. [boqun: Use xchg_acquire() instead of raw_xchg_acquire()] Reviewed-by: Alice Ryhl Signed-off-by: FUJITA Tomonori Reviewed-by: Gary Guo Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20251223062140.938325-3-fujita.tomonori@gmail.com --- rust/helpers/atomic_ext.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/rust/helpers/atomic_ext.c b/rust/helpers/atomic_ext.c index 3136255a84c6..177bb3603e5f 100644 --- a/rust/helpers/atomic_ext.c +++ b/rust/helpers/atomic_ext.c @@ -60,3 +60,13 @@ __rust_helper s16 rust_helper_atomic_i16_xchg(s16 *ptr, s16 new) { return xchg(ptr, new); } + +__rust_helper s8 rust_helper_atomic_i8_xchg_acquire(s8 *ptr, s8 new) +{ + return xchg_acquire(ptr, new); +} + +__rust_helper s16 rust_helper_atomic_i16_xchg_acquire(s16 *ptr, s16 new) +{ + return xchg_acquire(ptr, new); +} From 1bfca1e7e845a55fa82046727666f713d24ebdad Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Tue, 23 Dec 2025 15:21:39 +0900 Subject: [PATCH 52/90] rust: helpers: Add i8/i16 atomic xchg_release helpers Add i8/i16 atomic xchg_release helpers that call xchg_release() macro implementing atomic xchg_release using architecture-specific instructions. [boqun: Use xchg_release() instead of raw_xchg_release()] Reviewed-by: Alice Ryhl Signed-off-by: FUJITA Tomonori Reviewed-by: Gary Guo Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20251223062140.938325-4-fujita.tomonori@gmail.com --- rust/helpers/atomic_ext.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/rust/helpers/atomic_ext.c b/rust/helpers/atomic_ext.c index 177bb3603e5f..2b976a7ad3d7 100644 --- a/rust/helpers/atomic_ext.c +++ b/rust/helpers/atomic_ext.c @@ -70,3 +70,13 @@ __rust_helper s16 rust_helper_atomic_i16_xchg_acquire(s16 *ptr, s16 new) { return xchg_acquire(ptr, new); } + +__rust_helper s8 rust_helper_atomic_i8_xchg_release(s8 *ptr, s8 new) +{ + return xchg_release(ptr, new); +} + +__rust_helper s16 rust_helper_atomic_i16_xchg_release(s16 *ptr, s16 new) +{ + return xchg_release(ptr, new); +} From 910cbddc416cc30d83966baf378f44e59f3dc5d7 Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Tue, 23 Dec 2025 15:21:40 +0900 Subject: [PATCH 53/90] rust: helpers: Add i8/i16 atomic xchg_relaxed helpers Add i8/i16 atomic xchg_relaxed helpers that call xchg_relaxed() macro implementing atomic xchg_relaxed using architecture-specific instructions. [boqun: Use xchg_relaxed() instead of raw_xchg_relaxed()] Reviewed-by: Alice Ryhl Signed-off-by: FUJITA Tomonori Reviewed-by: Gary Guo Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20251223062140.938325-5-fujita.tomonori@gmail.com --- rust/helpers/atomic_ext.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/rust/helpers/atomic_ext.c b/rust/helpers/atomic_ext.c index 2b976a7ad3d7..76e392c39c7b 100644 --- a/rust/helpers/atomic_ext.c +++ b/rust/helpers/atomic_ext.c @@ -80,3 +80,13 @@ __rust_helper s16 rust_helper_atomic_i16_xchg_release(s16 *ptr, s16 new) { return xchg_release(ptr, new); } + +__rust_helper s8 rust_helper_atomic_i8_xchg_relaxed(s8 *ptr, s8 new) +{ + return xchg_relaxed(ptr, new); +} + +__rust_helper s16 rust_helper_atomic_i16_xchg_relaxed(s16 *ptr, s16 new) +{ + return xchg_relaxed(ptr, new); +} From 164e4b5600b32b4ddeac58bb5b37bc1490a1dce4 Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Sat, 27 Dec 2025 20:59:48 +0900 Subject: [PATCH 54/90] rust: helpers: Add i8/i16 atomic try_cmpxchg helpers Add i8/i16 atomic try_cmpxchg helpers that call try_cmpxchg() macro implementing atomic try_cmpxchg using architecture-specific instructions. [boqun: Add comments explaining CONFIG_ARCH_SUPPORTS_ATOMIC_RMW and use try_cmpxchg() instead of raw_try_cmpxchg()] Signed-off-by: FUJITA Tomonori Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20251227115951.1424458-2-fujita.tomonori@gmail.com --- rust/helpers/atomic_ext.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/rust/helpers/atomic_ext.c b/rust/helpers/atomic_ext.c index 76e392c39c7b..5ee127f1cc80 100644 --- a/rust/helpers/atomic_ext.c +++ b/rust/helpers/atomic_ext.c @@ -90,3 +90,20 @@ __rust_helper s16 rust_helper_atomic_i16_xchg_relaxed(s16 *ptr, s16 new) { return xchg_relaxed(ptr, new); } + +/* + * try_cmpxchg helpers depend on ARCH_SUPPORTS_ATOMIC_RMW and on the + * architecture provding try_cmpxchg() support for i8 and i16. + * + * The architectures that currently support Rust (x86_64, armv7, + * arm64, riscv, and loongarch) satisfy these requirements. + */ +__rust_helper bool rust_helper_atomic_i8_try_cmpxchg(s8 *ptr, s8 *old, s8 new) +{ + return try_cmpxchg(ptr, old, new); +} + +__rust_helper bool rust_helper_atomic_i16_try_cmpxchg(s16 *ptr, s16 *old, s16 new) +{ + return try_cmpxchg(ptr, old, new); +} From fed6aaa392c301721144303d64b7c68575d9d5ef Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Sat, 27 Dec 2025 20:59:49 +0900 Subject: [PATCH 55/90] rust: helpers: Add i8/i16 atomic try_cmpxchg_acquire helpers Add i8/i16 atomic try_cmpxchg_acquire helpers that call try_cmpxchg_acquire() macro implementing atomic try_cmpxchg_acquire using architecture-specific instructions. [boqun: Use try_cmpxchg_acquire() instead of raw_try_cmpxchg_acquire()] Signed-off-by: FUJITA Tomonori Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20251227115951.1424458-3-fujita.tomonori@gmail.com --- rust/helpers/atomic_ext.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/rust/helpers/atomic_ext.c b/rust/helpers/atomic_ext.c index 5ee127f1cc80..b6efec14e5b3 100644 --- a/rust/helpers/atomic_ext.c +++ b/rust/helpers/atomic_ext.c @@ -107,3 +107,13 @@ __rust_helper bool rust_helper_atomic_i16_try_cmpxchg(s16 *ptr, s16 *old, s16 ne { return try_cmpxchg(ptr, old, new); } + +__rust_helper bool rust_helper_atomic_i8_try_cmpxchg_acquire(s8 *ptr, s8 *old, s8 new) +{ + return try_cmpxchg_acquire(ptr, old, new); +} + +__rust_helper bool rust_helper_atomic_i16_try_cmpxchg_acquire(s16 *ptr, s16 *old, s16 new) +{ + return try_cmpxchg_acquire(ptr, old, new); +} From b5992f07a9736ab6279181c848f42227af9945bf Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Sat, 27 Dec 2025 20:59:50 +0900 Subject: [PATCH 56/90] rust: helpers: Add i8/i16 atomic try_cmpxchg_release helpers Add i8/i16 atomic try_cmpxchg_release helpers that call try_cmpxchg_release() macro implementing atomic try_cmpxchg_release using architecture-specific instructions. [boqun: Use try_cmpxchg_release() instead of raw_try_cmpxchg_release()] Signed-off-by: FUJITA Tomonori Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20251227115951.1424458-4-fujita.tomonori@gmail.com --- rust/helpers/atomic_ext.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/rust/helpers/atomic_ext.c b/rust/helpers/atomic_ext.c index b6efec14e5b3..962ea05dfb9c 100644 --- a/rust/helpers/atomic_ext.c +++ b/rust/helpers/atomic_ext.c @@ -117,3 +117,13 @@ __rust_helper bool rust_helper_atomic_i16_try_cmpxchg_acquire(s16 *ptr, s16 *old { return try_cmpxchg_acquire(ptr, old, new); } + +__rust_helper bool rust_helper_atomic_i8_try_cmpxchg_release(s8 *ptr, s8 *old, s8 new) +{ + return try_cmpxchg_release(ptr, old, new); +} + +__rust_helper bool rust_helper_atomic_i16_try_cmpxchg_release(s16 *ptr, s16 *old, s16 new) +{ + return try_cmpxchg_release(ptr, old, new); +} From 8de731a6c75547602601a5d219b5cf259ce2b38b Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Sat, 27 Dec 2025 20:59:51 +0900 Subject: [PATCH 57/90] rust: helpers: Add i8/i16 atomic try_cmpxchg_relaxed helpers Add i8/i16 atomic try_cmpxchg_relaxed helpers that call try_cmpxchg_relaxed() macro implementing atomic try_cmpxchg_relaxed using architecture-specific instructions. [boqun: Use try_cmpxchg_relaxed() instead of raw_try_cmpxchg_relaxed()] Signed-off-by: FUJITA Tomonori Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20251227115951.1424458-5-fujita.tomonori@gmail.com --- rust/helpers/atomic_ext.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/rust/helpers/atomic_ext.c b/rust/helpers/atomic_ext.c index 962ea05dfb9c..7d0c2bd340da 100644 --- a/rust/helpers/atomic_ext.c +++ b/rust/helpers/atomic_ext.c @@ -127,3 +127,13 @@ __rust_helper bool rust_helper_atomic_i16_try_cmpxchg_release(s16 *ptr, s16 *old { return try_cmpxchg_release(ptr, old, new); } + +__rust_helper bool rust_helper_atomic_i8_try_cmpxchg_relaxed(s8 *ptr, s8 *old, s8 new) +{ + return try_cmpxchg_relaxed(ptr, old, new); +} + +__rust_helper bool rust_helper_atomic_i16_try_cmpxchg_relaxed(s16 *ptr, s16 *old, s16 new) +{ + return try_cmpxchg_relaxed(ptr, old, new); +} From 2bb8c41e61b29ccdf7b6d716c3a8fe8488aa202a Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Sun, 28 Dec 2025 21:05:44 +0900 Subject: [PATCH 58/90] rust: sync: atomic: Prepare AtomicOps macros for i8/i16 support Rework the internal AtomicOps macro plumbing to generate per-type implementations from a mapping list. Capture the trait definition once and reuse it for both declaration and per-type impl expansion to reduce duplication and keep future extensions simple. This is a preparatory refactor for enabling i8/i16 atomics cleanly. Signed-off-by: FUJITA Tomonori Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20251228120546.1602275-2-fujita.tomonori@gmail.com --- rust/kernel/sync/atomic/internal.rs | 85 ++++++++++++++++++++++------- 1 file changed, 66 insertions(+), 19 deletions(-) diff --git a/rust/kernel/sync/atomic/internal.rs b/rust/kernel/sync/atomic/internal.rs index 6fdd8e59f45b..41b4ce2935e3 100644 --- a/rust/kernel/sync/atomic/internal.rs +++ b/rust/kernel/sync/atomic/internal.rs @@ -156,16 +156,17 @@ macro_rules! impl_atomic_method { } } -// Delcares $ops trait with methods and implements the trait for `i32` and `i64`. -macro_rules! declare_and_impl_atomic_methods { - ($(#[$attr:meta])* $pub:vis trait $ops:ident { - $( - $(#[doc=$doc:expr])* - fn $func:ident [$($variant:ident),*]($($arg_sig:tt)*) $( -> $ret:ty)? { - $unsafe:tt { bindings::#call($($arg:tt)*) } - } - )* - }) => { +macro_rules! declare_atomic_ops_trait { + ( + $(#[$attr:meta])* $pub:vis trait $ops:ident { + $( + $(#[doc=$doc:expr])* + fn $func:ident [$($variant:ident),*]($($arg_sig:tt)*) $( -> $ret:ty)? { + $unsafe:tt { bindings::#call($($arg:tt)*) } + } + )* + } + ) => { $(#[$attr])* $pub trait $ops: AtomicImpl { $( @@ -175,21 +176,25 @@ macro_rules! declare_and_impl_atomic_methods { ); )* } + } +} - impl $ops for i32 { +macro_rules! impl_atomic_ops_for_one { + ( + $ty:ty => $ctype:ident, + $(#[$attr:meta])* $pub:vis trait $ops:ident { $( - impl_atomic_method!( - (atomic) $func[$($variant)*]($($arg_sig)*) $(-> $ret)? { - $unsafe { call($($arg)*) } - } - ); + $(#[doc=$doc:expr])* + fn $func:ident [$($variant:ident),*]($($arg_sig:tt)*) $( -> $ret:ty)? { + $unsafe:tt { bindings::#call($($arg:tt)*) } + } )* } - - impl $ops for i64 { + ) => { + impl $ops for $ty { $( impl_atomic_method!( - (atomic64) $func[$($variant)*]($($arg_sig)*) $(-> $ret)? { + ($ctype) $func[$($variant)*]($($arg_sig)*) $(-> $ret)? { $unsafe { call($($arg)*) } } ); @@ -198,7 +203,47 @@ macro_rules! declare_and_impl_atomic_methods { } } +// Declares $ops trait with methods and implements the trait. +macro_rules! declare_and_impl_atomic_methods { + ( + [ $($map:tt)* ] + $(#[$attr:meta])* $pub:vis trait $ops:ident { $($body:tt)* } + ) => { + declare_and_impl_atomic_methods!( + @with_ops_def + [ $($map)* ] + ( $(#[$attr])* $pub trait $ops { $($body)* } ) + ); + }; + + (@with_ops_def [ $($map:tt)* ] ( $($ops_def:tt)* )) => { + declare_atomic_ops_trait!( $($ops_def)* ); + + declare_and_impl_atomic_methods!( + @munch + [ $($map)* ] + ( $($ops_def)* ) + ); + }; + + (@munch [] ( $($ops_def:tt)* )) => {}; + + (@munch [ $ty:ty => $ctype:ident $(, $($rest:tt)*)? ] ( $($ops_def:tt)* )) => { + impl_atomic_ops_for_one!( + $ty => $ctype, + $($ops_def)* + ); + + declare_and_impl_atomic_methods!( + @munch + [ $($($rest)*)? ] + ( $($ops_def)* ) + ); + }; +} + declare_and_impl_atomic_methods!( + [ i32 => atomic, i64 => atomic64 ] /// Basic atomic operations pub trait AtomicBasicOps { /// Atomic read (load). @@ -216,6 +261,7 @@ declare_and_impl_atomic_methods!( ); declare_and_impl_atomic_methods!( + [ i32 => atomic, i64 => atomic64 ] /// Exchange and compare-and-exchange atomic operations pub trait AtomicExchangeOps { /// Atomic exchange. @@ -243,6 +289,7 @@ declare_and_impl_atomic_methods!( ); declare_and_impl_atomic_methods!( + [ i32 => atomic, i64 => atomic64 ] /// Atomic arithmetic operations pub trait AtomicArithmeticOps { /// Atomic add (wrapping). From cf4c3bc1445152c1949a4b5fef56d07579fadb1e Mon Sep 17 00:00:00 2001 From: Boqun Feng Date: Tue, 6 Jan 2026 11:40:34 +0800 Subject: [PATCH 59/90] arch: um/x86: Select ARCH_SUPPORTS_ATOMIC_RMW for UML_X86 x86 atomic instructions are used for um on UML_X86, therefore atomics on UML_X86 support native atomic RmW as x86 does, hence select ARCH_SUPPORTS_ATOMIC_RMW. Reviewed-by: FUJITA Tomonori Acked-by: Richard Weinberger Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260106034034.60074-1-boqun.feng@gmail.com --- arch/x86/um/Kconfig | 1 + 1 file changed, 1 insertion(+) diff --git a/arch/x86/um/Kconfig b/arch/x86/um/Kconfig index bdd7c8e39b01..44b12e45f9a0 100644 --- a/arch/x86/um/Kconfig +++ b/arch/x86/um/Kconfig @@ -9,6 +9,7 @@ endmenu config UML_X86 def_bool y select ARCH_USE_QUEUED_RWLOCKS + select ARCH_SUPPORTS_ATOMIC_RMW select ARCH_USE_QUEUED_SPINLOCKS select DCACHE_WORD_ACCESS select HAVE_EFFICIENT_UNALIGNED_ACCESS From b33796d554f270e19141c0c1fa0a90705a511d2b Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Thu, 11 Dec 2025 20:38:25 +0900 Subject: [PATCH 60/90] rust: sync: atomic: Add i8/i16 load and store support Add atomic operation support for i8 and i16 types using volatile read/write and smp_load_acquire/smp_store_release helpers. [boqun: Adjust [1] to avoid introduction of impl_atomic_only_load_and_store_ops!() in the middle] Signed-off-by: FUJITA Tomonori Reviewed-by: Gary Guo Reviewed-by: Joel Fernandes Link: https://lore.kernel.org/all/20251228120546.1602275-1-fujita.tomonori@gmail.com/ [1] Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20251211113826.1299077-4-fujita.tomonori@gmail.com --- rust/kernel/sync/atomic/internal.rs | 25 +++++++++++++++++++------ rust/kernel/sync/atomic/predefine.rs | 14 +++++++++++++- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/rust/kernel/sync/atomic/internal.rs b/rust/kernel/sync/atomic/internal.rs index 41b4ce2935e3..1b2a7933bc14 100644 --- a/rust/kernel/sync/atomic/internal.rs +++ b/rust/kernel/sync/atomic/internal.rs @@ -13,17 +13,22 @@ mod private { pub trait Sealed {} } -// `i32` and `i64` are only supported atomic implementations. +// The C side supports atomic primitives only for `i32` and `i64` (`atomic_t` and `atomic64_t`), +// while the Rust side also layers provides atomic support for `i8` and `i16` +// on top of lower-level C primitives. +impl private::Sealed for i8 {} +impl private::Sealed for i16 {} impl private::Sealed for i32 {} impl private::Sealed for i64 {} /// A marker trait for types that implement atomic operations with C side primitives. /// -/// This trait is sealed, and only types that have directly mapping to the C side atomics should -/// impl this: +/// This trait is sealed, and only types that map directly to the C side atomics +/// or can be implemented with lower-level C primitives are allowed to implement this: /// -/// - `i32` maps to `atomic_t`. -/// - `i64` maps to `atomic64_t`. +/// - `i8` and `i16` are implemented with lower-level C primitives. +/// - `i32` map to `atomic_t` +/// - `i64` map to `atomic64_t` pub trait AtomicImpl: Sized + Send + Copy + private::Sealed { /// The type of the delta in arithmetic or logical operations. /// @@ -32,6 +37,14 @@ pub trait AtomicImpl: Sized + Send + Copy + private::Sealed { type Delta; } +impl AtomicImpl for i8 { + type Delta = Self; +} + +impl AtomicImpl for i16 { + type Delta = Self; +} + // `atomic_t` implements atomic operations on `i32`. impl AtomicImpl for i32 { type Delta = Self; @@ -243,7 +256,7 @@ macro_rules! declare_and_impl_atomic_methods { } declare_and_impl_atomic_methods!( - [ i32 => atomic, i64 => atomic64 ] + [ i8 => atomic_i8, i16 => atomic_i16, i32 => atomic, i64 => atomic64 ] /// Basic atomic operations pub trait AtomicBasicOps { /// Atomic read (load). diff --git a/rust/kernel/sync/atomic/predefine.rs b/rust/kernel/sync/atomic/predefine.rs index 45a17985cda4..09b357be59b8 100644 --- a/rust/kernel/sync/atomic/predefine.rs +++ b/rust/kernel/sync/atomic/predefine.rs @@ -5,6 +5,18 @@ use crate::static_assert; use core::mem::{align_of, size_of}; +// SAFETY: `i8` has the same size and alignment with itself, and is round-trip transmutable to +// itself. +unsafe impl super::AtomicType for i8 { + type Repr = i8; +} + +// SAFETY: `i16` has the same size and alignment with itself, and is round-trip transmutable to +// itself. +unsafe impl super::AtomicType for i16 { + type Repr = i16; +} + // SAFETY: `i32` has the same size and alignment with itself, and is round-trip transmutable to // itself. unsafe impl super::AtomicType for i32 { @@ -118,7 +130,7 @@ mod tests { #[test] fn atomic_basic_tests() { - for_each_type!(42 in [i32, i64, u32, u64, isize, usize] |v| { + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { let x = Atomic::new(v); assert_eq!(v, x.load(Relaxed)); From 7b001c97d9bdaea50e1e1834040c58f7ef9f4e89 Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Thu, 11 Dec 2025 20:38:26 +0900 Subject: [PATCH 61/90] rust: sync: atomic: Add store_release/load_acquire tests Add minimum store_release/load_acquire tests. Signed-off-by: FUJITA Tomonori Reviewed-by: Gary Guo Reviewed-by: Joel Fernandes Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20251211113826.1299077-5-fujita.tomonori@gmail.com --- rust/kernel/sync/atomic/predefine.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/rust/kernel/sync/atomic/predefine.rs b/rust/kernel/sync/atomic/predefine.rs index 09b357be59b8..51e9df0cf56e 100644 --- a/rust/kernel/sync/atomic/predefine.rs +++ b/rust/kernel/sync/atomic/predefine.rs @@ -137,6 +137,16 @@ mod tests { }); } + #[test] + fn atomic_acquire_release_tests() { + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { + let x = Atomic::new(0); + + x.store(v, Release); + assert_eq!(v, x.load(Acquire)); + }); + } + #[test] fn atomic_xchg_tests() { for_each_type!(42 in [i32, i64, u32, u64, isize, usize] |v| { From 584f286f822afecc1a6521a27b3caf3e2f515d41 Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Sun, 28 Dec 2025 21:05:46 +0900 Subject: [PATCH 62/90] rust: sync: atomic: Add i8/i16 xchg and cmpxchg support Add atomic xchg and cmpxchg operation support for i8 and i16 types with tests. Note that since the current implementation of Atomic::<{i8,i16}>::{load,store}() is READ_ONCE()/WRITE_ONCE()-based. The atomicity between load/store and xchg/cmpxchg is only guaranteed if the architecture has native RmW support, hence i8/i16 is currently AtomicImpl only when CONFIG_ARCH_SUPPORTS_ATOMIC_RWM=y. [boqun: Make i8/i16 AtomicImpl only when CONFIG_ARCH_SUPPORTS_ATOMIC_RWM=y] Signed-off-by: FUJITA Tomonori Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20251228120546.1602275-4-fujita.tomonori@gmail.com --- rust/kernel/sync/atomic/internal.rs | 8 +++++++- rust/kernel/sync/atomic/predefine.rs | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/rust/kernel/sync/atomic/internal.rs b/rust/kernel/sync/atomic/internal.rs index 1b2a7933bc14..0dac58bca2b3 100644 --- a/rust/kernel/sync/atomic/internal.rs +++ b/rust/kernel/sync/atomic/internal.rs @@ -37,10 +37,16 @@ pub trait AtomicImpl: Sized + Send + Copy + private::Sealed { type Delta; } +// The current helpers of load/store uses `{WRITE,READ}_ONCE()` hence the atomicity is only +// guaranteed against read-modify-write operations if the architecture supports native atomic RmW. +#[cfg(CONFIG_ARCH_SUPPORTS_ATOMIC_RMW)] impl AtomicImpl for i8 { type Delta = Self; } +// The current helpers of load/store uses `{WRITE,READ}_ONCE()` hence the atomicity is only +// guaranteed against read-modify-write operations if the architecture supports native atomic RmW. +#[cfg(CONFIG_ARCH_SUPPORTS_ATOMIC_RMW)] impl AtomicImpl for i16 { type Delta = Self; } @@ -274,7 +280,7 @@ declare_and_impl_atomic_methods!( ); declare_and_impl_atomic_methods!( - [ i32 => atomic, i64 => atomic64 ] + [ i8 => atomic_i8, i16 => atomic_i16, i32 => atomic, i64 => atomic64 ] /// Exchange and compare-and-exchange atomic operations pub trait AtomicExchangeOps { /// Atomic exchange. diff --git a/rust/kernel/sync/atomic/predefine.rs b/rust/kernel/sync/atomic/predefine.rs index 51e9df0cf56e..248d26555ccf 100644 --- a/rust/kernel/sync/atomic/predefine.rs +++ b/rust/kernel/sync/atomic/predefine.rs @@ -149,7 +149,7 @@ mod tests { #[test] fn atomic_xchg_tests() { - for_each_type!(42 in [i32, i64, u32, u64, isize, usize] |v| { + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { let x = Atomic::new(v); let old = v; @@ -162,7 +162,7 @@ mod tests { #[test] fn atomic_cmpxchg_tests() { - for_each_type!(42 in [i32, i64, u32, u64, isize, usize] |v| { + for_each_type!(42 in [i8, i16, i32, i64, u32, u64, isize, usize] |v| { let x = Atomic::new(v); let old = v; From 06bd0e52bfd78eae1c7dd5db163ce64161b495e7 Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Thu, 1 Jan 2026 12:49:21 +0900 Subject: [PATCH 63/90] rust: sync: atomic: Add atomic bool support via i8 representation Add `bool` support, `Atomic` by using `i8` as its underlying representation. Rust specifies that `bool` has size 1 and alignment 1 [1], so it matches `i8` on layout; keep `static_assert!()` checks to enforce this assumption at build time. [boqun: Remove the unnecessary impl AtomicImpl for bool] Link: https://doc.rust-lang.org/reference/types/boolean.html [1] Signed-off-by: FUJITA Tomonori Reviewed-by: Gary Guo Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260101034922.2020334-2-fujita.tomonori@gmail.com --- rust/kernel/sync/atomic/predefine.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/rust/kernel/sync/atomic/predefine.rs b/rust/kernel/sync/atomic/predefine.rs index 248d26555ccf..3fc99174b086 100644 --- a/rust/kernel/sync/atomic/predefine.rs +++ b/rust/kernel/sync/atomic/predefine.rs @@ -5,6 +5,17 @@ use crate::static_assert; use core::mem::{align_of, size_of}; +// Ensure size and alignment requirements are checked. +static_assert!(size_of::() == size_of::()); +static_assert!(align_of::() == align_of::()); + +// SAFETY: `bool` has the same size and alignment as `i8`, and Rust guarantees that `bool` has +// only two valid bit patterns: 0 (false) and 1 (true). Those are valid `i8` values, so `bool` is +// round-trip transmutable to `i8`. +unsafe impl super::AtomicType for bool { + type Repr = i8; +} + // SAFETY: `i8` has the same size and alignment with itself, and is round-trip transmutable to // itself. unsafe impl super::AtomicType for i8 { From 4bac28727a2b3f33e6375aeafdf31df67deff5d0 Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Thu, 1 Jan 2026 12:49:22 +0900 Subject: [PATCH 64/90] rust: sync: atomic: Add atomic bool tests Add tests for Atomic operations. Atomic does not fit into the existing u8/16/32/64 tests so introduce a dedicated test for it. Signed-off-by: FUJITA Tomonori Reviewed-by: Gary Guo Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260101034922.2020334-3-fujita.tomonori@gmail.com --- rust/kernel/sync/atomic/predefine.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/rust/kernel/sync/atomic/predefine.rs b/rust/kernel/sync/atomic/predefine.rs index 3fc99174b086..42067c6a266c 100644 --- a/rust/kernel/sync/atomic/predefine.rs +++ b/rust/kernel/sync/atomic/predefine.rs @@ -199,4 +199,20 @@ mod tests { assert_eq!(v + 25, x.load(Relaxed)); }); } + + #[test] + fn atomic_bool_tests() { + let x = Atomic::new(false); + + assert_eq!(false, x.load(Relaxed)); + x.store(true, Relaxed); + assert_eq!(true, x.load(Relaxed)); + + assert_eq!(true, x.xchg(false, Relaxed)); + assert_eq!(false, x.load(Relaxed)); + + assert_eq!(Err(false), x.cmpxchg(true, true, Relaxed)); + assert_eq!(false, x.load(Relaxed)); + assert_eq!(Ok(false), x.cmpxchg(false, true, Full)); + } } From 323e4bfcbe2dc6c6cac6e007dded0ba4f89a6458 Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Tue, 30 Dec 2025 18:37:17 +0900 Subject: [PATCH 65/90] rust: list: Switch to kernel::sync atomic primitives Convert uses of `AtomicBool` to `Atomic`. Note that the compare_exchange migration simplifies to `try_cmpxchg()`, since `try_cmpxchg()` provides relaxed ordering on failure, making the explicit failure ordering unnecessary. Signed-off-by: FUJITA Tomonori Reviewed-by: Alice Ryhl Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20251230093718.1852322-3-fujita.tomonori@gmail.com --- rust/kernel/list/arc.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/rust/kernel/list/arc.rs b/rust/kernel/list/arc.rs index d92bcf665c89..2282f33913ee 100644 --- a/rust/kernel/list/arc.rs +++ b/rust/kernel/list/arc.rs @@ -6,11 +6,11 @@ use crate::alloc::{AllocError, Flags}; use crate::prelude::*; +use crate::sync::atomic::{ordering, Atomic}; use crate::sync::{Arc, ArcBorrow, UniqueArc}; use core::marker::PhantomPinned; use core::ops::Deref; use core::pin::Pin; -use core::sync::atomic::{AtomicBool, Ordering}; /// Declares that this type has some way to ensure that there is exactly one `ListArc` instance for /// this id. @@ -469,7 +469,7 @@ where /// If the boolean is `false`, then there is no [`ListArc`] for this value. #[repr(transparent)] pub struct AtomicTracker { - inner: AtomicBool, + inner: Atomic, // This value needs to be pinned to justify the INVARIANT: comment in `AtomicTracker::new`. _pin: PhantomPinned, } @@ -480,12 +480,12 @@ impl AtomicTracker { // INVARIANT: Pin-init initializers can't be used on an existing `Arc`, so this value will // not be constructed in an `Arc` that already has a `ListArc`. Self { - inner: AtomicBool::new(false), + inner: Atomic::new(false), _pin: PhantomPinned, } } - fn project_inner(self: Pin<&mut Self>) -> &mut AtomicBool { + fn project_inner(self: Pin<&mut Self>) -> &mut Atomic { // SAFETY: The `inner` field is not structurally pinned, so we may obtain a mutable // reference to it even if we only have a pinned reference to `self`. unsafe { &mut Pin::into_inner_unchecked(self).inner } @@ -500,7 +500,7 @@ impl ListArcSafe for AtomicTracker { unsafe fn on_drop_list_arc(&self) { // INVARIANT: We just dropped a ListArc, so the boolean should be false. - self.inner.store(false, Ordering::Release); + self.inner.store(false, ordering::Release); } } @@ -514,8 +514,6 @@ unsafe impl TryNewListArc for AtomicTracker { fn try_new_list_arc(&self) -> bool { // INVARIANT: If this method returns true, then the boolean used to be false, and is no // longer false, so it is okay for the caller to create a new [`ListArc`]. - self.inner - .compare_exchange(false, true, Ordering::Acquire, Ordering::Relaxed) - .is_ok() + self.inner.cmpxchg(false, true, ordering::Acquire).is_ok() } } From 7f4c8b4dcde7174a3bd5d001790d8453c9aefa3c Mon Sep 17 00:00:00 2001 From: FUJITA Tomonori Date: Tue, 30 Dec 2025 18:37:18 +0900 Subject: [PATCH 66/90] rust_binder: Switch to kernel::sync atomic primitives Convert uses of AtomicBool, AtomicUsize, and AtomicU32. Signed-off-by: FUJITA Tomonori Acked-by: Alice Ryhl Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20251230093718.1852322-4-fujita.tomonori@gmail.com --- drivers/android/binder/rust_binder_main.rs | 20 ++++++++---------- drivers/android/binder/stats.rs | 8 ++++---- drivers/android/binder/thread.rs | 24 ++++++++++------------ drivers/android/binder/transaction.rs | 16 +++++++-------- 4 files changed, 32 insertions(+), 36 deletions(-) diff --git a/drivers/android/binder/rust_binder_main.rs b/drivers/android/binder/rust_binder_main.rs index c79a9e742240..47bfb114cabb 100644 --- a/drivers/android/binder/rust_binder_main.rs +++ b/drivers/android/binder/rust_binder_main.rs @@ -18,6 +18,7 @@ use kernel::{ prelude::*, seq_file::SeqFile, seq_print, + sync::atomic::{ordering::Relaxed, Atomic}, sync::poll::PollTable, sync::Arc, task::Pid, @@ -28,10 +29,7 @@ use kernel::{ use crate::{context::Context, page_range::Shrinker, process::Process, thread::Thread}; -use core::{ - ptr::NonNull, - sync::atomic::{AtomicBool, AtomicUsize, Ordering}, -}; +use core::ptr::NonNull; mod allocation; mod context; @@ -90,9 +88,9 @@ module! { } fn next_debug_id() -> usize { - static NEXT_DEBUG_ID: AtomicUsize = AtomicUsize::new(0); + static NEXT_DEBUG_ID: Atomic = Atomic::new(0); - NEXT_DEBUG_ID.fetch_add(1, Ordering::Relaxed) + NEXT_DEBUG_ID.fetch_add(1, Relaxed) } /// Provides a single place to write Binder return values via the @@ -215,7 +213,7 @@ impl DTRWrap { struct DeliverCode { code: u32, - skip: AtomicBool, + skip: Atomic, } kernel::list::impl_list_arc_safe! { @@ -226,7 +224,7 @@ impl DeliverCode { fn new(code: u32) -> Self { Self { code, - skip: AtomicBool::new(false), + skip: Atomic::new(false), } } @@ -235,7 +233,7 @@ impl DeliverCode { /// This is used instead of removing it from the work list, since `LinkedList::remove` is /// unsafe, whereas this method is not. fn skip(&self) { - self.skip.store(true, Ordering::Relaxed); + self.skip.store(true, Relaxed); } } @@ -245,7 +243,7 @@ impl DeliverToRead for DeliverCode { _thread: &Thread, writer: &mut BinderReturnWriter<'_>, ) -> Result { - if !self.skip.load(Ordering::Relaxed) { + if !self.skip.load(Relaxed) { writer.write_code(self.code)?; } Ok(true) @@ -259,7 +257,7 @@ impl DeliverToRead for DeliverCode { fn debug_print(&self, m: &SeqFile, prefix: &str, _tprefix: &str) -> Result<()> { seq_print!(m, "{}", prefix); - if self.skip.load(Ordering::Relaxed) { + if self.skip.load(Relaxed) { seq_print!(m, "(skipped) "); } if self.code == defs::BR_TRANSACTION_COMPLETE { diff --git a/drivers/android/binder/stats.rs b/drivers/android/binder/stats.rs index 037002651941..ab75e9561cbf 100644 --- a/drivers/android/binder/stats.rs +++ b/drivers/android/binder/stats.rs @@ -5,7 +5,7 @@ //! Keep track of statistics for binder_logs. use crate::defs::*; -use core::sync::atomic::{AtomicU32, Ordering::Relaxed}; +use kernel::sync::atomic::{ordering::Relaxed, Atomic}; use kernel::{ioctl::_IOC_NR, seq_file::SeqFile, seq_print}; const BC_COUNT: usize = _IOC_NR(BC_REPLY_SG) as usize + 1; @@ -14,14 +14,14 @@ const BR_COUNT: usize = _IOC_NR(BR_TRANSACTION_PENDING_FROZEN) as usize + 1; pub(crate) static GLOBAL_STATS: BinderStats = BinderStats::new(); pub(crate) struct BinderStats { - bc: [AtomicU32; BC_COUNT], - br: [AtomicU32; BR_COUNT], + bc: [Atomic; BC_COUNT], + br: [Atomic; BR_COUNT], } impl BinderStats { pub(crate) const fn new() -> Self { #[expect(clippy::declare_interior_mutable_const)] - const ZERO: AtomicU32 = AtomicU32::new(0); + const ZERO: Atomic = Atomic::new(0); Self { bc: [ZERO; BC_COUNT], diff --git a/drivers/android/binder/thread.rs b/drivers/android/binder/thread.rs index 1a8e6fdc0dc4..82264db06507 100644 --- a/drivers/android/binder/thread.rs +++ b/drivers/android/binder/thread.rs @@ -15,6 +15,7 @@ use kernel::{ security, seq_file::SeqFile, seq_print, + sync::atomic::{ordering::Relaxed, Atomic}, sync::poll::{PollCondVar, PollTable}, sync::{Arc, SpinLock}, task::Task, @@ -34,10 +35,7 @@ use crate::{ BinderReturnWriter, DArc, DLArc, DTRWrap, DeliverCode, DeliverToRead, }; -use core::{ - mem::size_of, - sync::atomic::{AtomicU32, Ordering}, -}; +use core::mem::size_of; /// Stores the layout of the scatter-gather entries. This is used during the `translate_objects` /// call and is discarded when it returns. @@ -273,8 +271,8 @@ const LOOPER_POLL: u32 = 0x40; impl InnerThread { fn new() -> Result { fn next_err_id() -> u32 { - static EE_ID: AtomicU32 = AtomicU32::new(0); - EE_ID.fetch_add(1, Ordering::Relaxed) + static EE_ID: Atomic = Atomic::new(0); + EE_ID.fetch_add(1, Relaxed) } Ok(Self { @@ -1537,7 +1535,7 @@ impl Thread { #[pin_data] struct ThreadError { - error_code: AtomicU32, + error_code: Atomic, #[pin] links_track: AtomicTracker, } @@ -1545,18 +1543,18 @@ struct ThreadError { impl ThreadError { fn try_new() -> Result> { DTRWrap::arc_pin_init(pin_init!(Self { - error_code: AtomicU32::new(BR_OK), + error_code: Atomic::new(BR_OK), links_track <- AtomicTracker::new(), })) .map(ListArc::into_arc) } fn set_error_code(&self, code: u32) { - self.error_code.store(code, Ordering::Relaxed); + self.error_code.store(code, Relaxed); } fn is_unused(&self) -> bool { - self.error_code.load(Ordering::Relaxed) == BR_OK + self.error_code.load(Relaxed) == BR_OK } } @@ -1566,8 +1564,8 @@ impl DeliverToRead for ThreadError { _thread: &Thread, writer: &mut BinderReturnWriter<'_>, ) -> Result { - let code = self.error_code.load(Ordering::Relaxed); - self.error_code.store(BR_OK, Ordering::Relaxed); + let code = self.error_code.load(Relaxed); + self.error_code.store(BR_OK, Relaxed); writer.write_code(code)?; Ok(true) } @@ -1583,7 +1581,7 @@ impl DeliverToRead for ThreadError { m, "{}transaction error: {}\n", prefix, - self.error_code.load(Ordering::Relaxed) + self.error_code.load(Relaxed) ); Ok(()) } diff --git a/drivers/android/binder/transaction.rs b/drivers/android/binder/transaction.rs index 4bd3c0e417eb..2273a8e9d01c 100644 --- a/drivers/android/binder/transaction.rs +++ b/drivers/android/binder/transaction.rs @@ -2,11 +2,11 @@ // Copyright (C) 2025 Google LLC. -use core::sync::atomic::{AtomicBool, Ordering}; use kernel::{ prelude::*, seq_file::SeqFile, seq_print, + sync::atomic::{ordering::Relaxed, Atomic}, sync::{Arc, SpinLock}, task::Kuid, time::{Instant, Monotonic}, @@ -33,7 +33,7 @@ pub(crate) struct Transaction { pub(crate) to: Arc, #[pin] allocation: SpinLock>, - is_outstanding: AtomicBool, + is_outstanding: Atomic, code: u32, pub(crate) flags: u32, data_size: usize, @@ -105,7 +105,7 @@ impl Transaction { offsets_size: trd.offsets_size as _, data_address, allocation <- kernel::new_spinlock!(Some(alloc.success()), "Transaction::new"), - is_outstanding: AtomicBool::new(false), + is_outstanding: Atomic::new(false), txn_security_ctx_off, oneway_spam_detected, start_time: Instant::now(), @@ -145,7 +145,7 @@ impl Transaction { offsets_size: trd.offsets_size as _, data_address: alloc.ptr, allocation <- kernel::new_spinlock!(Some(alloc.success()), "Transaction::new"), - is_outstanding: AtomicBool::new(false), + is_outstanding: Atomic::new(false), txn_security_ctx_off: None, oneway_spam_detected, start_time: Instant::now(), @@ -215,8 +215,8 @@ impl Transaction { pub(crate) fn set_outstanding(&self, to_process: &mut ProcessInner) { // No race because this method is only called once. - if !self.is_outstanding.load(Ordering::Relaxed) { - self.is_outstanding.store(true, Ordering::Relaxed); + if !self.is_outstanding.load(Relaxed) { + self.is_outstanding.store(true, Relaxed); to_process.add_outstanding_txn(); } } @@ -227,8 +227,8 @@ impl Transaction { // destructor, which is guaranteed to not race with any other operations on the // transaction. It also cannot race with `set_outstanding`, since submission happens // before delivery. - if self.is_outstanding.load(Ordering::Relaxed) { - self.is_outstanding.store(false, Ordering::Relaxed); + if self.is_outstanding.load(Relaxed) { + self.is_outstanding.store(false, Relaxed); self.to.drop_outstanding_txn(); } } From aa574e0f21a6e7a28e4b8794ad4238d3bfd4f9df Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 5 Jan 2026 12:42:14 +0000 Subject: [PATCH 67/90] rust: barrier: Add __rust_helper to helpers This is needed to inline these helpers into Rust code. Reviewed-by: Boqun Feng Reviewed-by: Gary Guo Signed-off-by: Alice Ryhl Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260105-define-rust-helper-v2-1-51da5f454a67@google.com --- rust/helpers/barrier.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rust/helpers/barrier.c b/rust/helpers/barrier.c index cdf28ce8e511..fed8853745c8 100644 --- a/rust/helpers/barrier.c +++ b/rust/helpers/barrier.c @@ -2,17 +2,17 @@ #include -void rust_helper_smp_mb(void) +__rust_helper void rust_helper_smp_mb(void) { smp_mb(); } -void rust_helper_smp_wmb(void) +__rust_helper void rust_helper_smp_wmb(void) { smp_wmb(); } -void rust_helper_smp_rmb(void) +__rust_helper void rust_helper_smp_rmb(void) { smp_rmb(); } From 71a4d13fa1cf2b7a4f45a6ee41548c27783f7940 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 5 Jan 2026 12:42:15 +0000 Subject: [PATCH 68/90] rust: blk: Add __rust_helper to helpers This is needed to inline these helpers into Rust code. Reviewed-by: Boqun Feng Reviewed-by: Gary Guo Signed-off-by: Alice Ryhl Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260105-define-rust-helper-v2-2-51da5f454a67@google.com --- rust/helpers/blk.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/helpers/blk.c b/rust/helpers/blk.c index cc9f4e6a2d23..20c512e46a7a 100644 --- a/rust/helpers/blk.c +++ b/rust/helpers/blk.c @@ -3,12 +3,12 @@ #include #include -void *rust_helper_blk_mq_rq_to_pdu(struct request *rq) +__rust_helper void *rust_helper_blk_mq_rq_to_pdu(struct request *rq) { return blk_mq_rq_to_pdu(rq); } -struct request *rust_helper_blk_mq_rq_from_pdu(void *pdu) +__rust_helper struct request *rust_helper_blk_mq_rq_from_pdu(void *pdu) { return blk_mq_rq_from_pdu(pdu); } From 1c7a6f48f7eeb3014584d2fc55fc67f0cbaeef69 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 5 Jan 2026 12:42:18 +0000 Subject: [PATCH 69/90] rust: completion: Add __rust_helper to helpers This is needed to inline these helpers into Rust code. Reviewed-by: Boqun Feng Reviewed-by: Gary Guo Signed-off-by: Alice Ryhl Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260105-define-rust-helper-v2-5-51da5f454a67@google.com --- rust/helpers/completion.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/helpers/completion.c b/rust/helpers/completion.c index b2443262a2ae..0126767cc3be 100644 --- a/rust/helpers/completion.c +++ b/rust/helpers/completion.c @@ -2,7 +2,7 @@ #include -void rust_helper_init_completion(struct completion *x) +__rust_helper void rust_helper_init_completion(struct completion *x) { init_completion(x); } From 9f658bd5378d5c357d5eeb1e699f1504a7498dbf Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 5 Jan 2026 12:42:19 +0000 Subject: [PATCH 70/90] rust: cpu: Add __rust_helper to helpers This is needed to inline these helpers into Rust code. Reviewed-by: Boqun Feng Reviewed-by: Gary Guo Signed-off-by: Alice Ryhl Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260105-define-rust-helper-v2-6-51da5f454a67@google.com --- rust/helpers/cpu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/helpers/cpu.c b/rust/helpers/cpu.c index 824e0adb19d4..5759349b2c88 100644 --- a/rust/helpers/cpu.c +++ b/rust/helpers/cpu.c @@ -2,7 +2,7 @@ #include -unsigned int rust_helper_raw_smp_processor_id(void) +__rust_helper unsigned int rust_helper_raw_smp_processor_id(void) { return raw_smp_processor_id(); } From a87e6fe8738fabf9881758b79b0db592c057acbd Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 5 Jan 2026 12:42:26 +0000 Subject: [PATCH 71/90] rust: processor: Add __rust_helper to helpers This is needed to inline these helpers into Rust code. Reviewed-by: Boqun Feng Reviewed-by: Gary Guo Signed-off-by: Alice Ryhl Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260105-define-rust-helper-v2-13-51da5f454a67@google.com --- rust/helpers/processor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/helpers/processor.c b/rust/helpers/processor.c index d41355e14d6e..76fadbb647c5 100644 --- a/rust/helpers/processor.c +++ b/rust/helpers/processor.c @@ -2,7 +2,7 @@ #include -void rust_helper_cpu_relax(void) +__rust_helper void rust_helper_cpu_relax(void) { cpu_relax(); } From 5e03edaed373f41e7a3c8617e01891eb680d62aa Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 5 Jan 2026 12:42:29 +0000 Subject: [PATCH 72/90] rust: rcu: Add __rust_helper to helpers This is needed to inline these helpers into Rust code. Reviewed-by: Boqun Feng Reviewed-by: Gary Guo Signed-off-by: Alice Ryhl Reviewed-by: Joel Fernandes (NVIDIA) Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260105-define-rust-helper-v2-16-51da5f454a67@google.com --- rust/helpers/rcu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/helpers/rcu.c b/rust/helpers/rcu.c index f1cec6583513..481274c05857 100644 --- a/rust/helpers/rcu.c +++ b/rust/helpers/rcu.c @@ -2,12 +2,12 @@ #include -void rust_helper_rcu_read_lock(void) +__rust_helper void rust_helper_rcu_read_lock(void) { rcu_read_lock(); } -void rust_helper_rcu_read_unlock(void) +__rust_helper void rust_helper_rcu_read_unlock(void) { rcu_read_unlock(); } From 9ba1aaf25ab7dadb910348b6857865e87b4c5689 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 5 Jan 2026 12:42:30 +0000 Subject: [PATCH 73/90] rust: refcount: Add __rust_helper to helpers This is needed to inline these helpers into Rust code. Reviewed-by: Boqun Feng Reviewed-by: Gary Guo Signed-off-by: Alice Ryhl Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260105-define-rust-helper-v2-17-51da5f454a67@google.com --- rust/helpers/refcount.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rust/helpers/refcount.c b/rust/helpers/refcount.c index d175898ad7b8..36334a674ee4 100644 --- a/rust/helpers/refcount.c +++ b/rust/helpers/refcount.c @@ -2,27 +2,27 @@ #include -refcount_t rust_helper_REFCOUNT_INIT(int n) +__rust_helper refcount_t rust_helper_REFCOUNT_INIT(int n) { return (refcount_t)REFCOUNT_INIT(n); } -void rust_helper_refcount_set(refcount_t *r, int n) +__rust_helper void rust_helper_refcount_set(refcount_t *r, int n) { refcount_set(r, n); } -void rust_helper_refcount_inc(refcount_t *r) +__rust_helper void rust_helper_refcount_inc(refcount_t *r) { refcount_inc(r); } -void rust_helper_refcount_dec(refcount_t *r) +__rust_helper void rust_helper_refcount_dec(refcount_t *r) { refcount_dec(r); } -bool rust_helper_refcount_dec_and_test(refcount_t *r) +__rust_helper bool rust_helper_refcount_dec_and_test(refcount_t *r) { return refcount_dec_and_test(r); } From d4ad4de929ba27ed241c6ef1098b1687001ced1f Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 5 Jan 2026 12:42:33 +0000 Subject: [PATCH 74/90] rust: sync: Add __rust_helper to helpers This is needed to inline these helpers into Rust code. Reviewed-by: Boqun Feng Reviewed-by: Gary Guo Signed-off-by: Alice Ryhl Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260105-define-rust-helper-v2-20-51da5f454a67@google.com --- rust/helpers/mutex.c | 13 +++++++------ rust/helpers/spinlock.c | 13 +++++++------ rust/helpers/sync.c | 4 ++-- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/rust/helpers/mutex.c b/rust/helpers/mutex.c index e487819125f0..1b07d6e64299 100644 --- a/rust/helpers/mutex.c +++ b/rust/helpers/mutex.c @@ -2,28 +2,29 @@ #include -void rust_helper_mutex_lock(struct mutex *lock) +__rust_helper void rust_helper_mutex_lock(struct mutex *lock) { mutex_lock(lock); } -int rust_helper_mutex_trylock(struct mutex *lock) +__rust_helper int rust_helper_mutex_trylock(struct mutex *lock) { return mutex_trylock(lock); } -void rust_helper___mutex_init(struct mutex *mutex, const char *name, - struct lock_class_key *key) +__rust_helper void rust_helper___mutex_init(struct mutex *mutex, + const char *name, + struct lock_class_key *key) { __mutex_init(mutex, name, key); } -void rust_helper_mutex_assert_is_held(struct mutex *mutex) +__rust_helper void rust_helper_mutex_assert_is_held(struct mutex *mutex) { lockdep_assert_held(mutex); } -void rust_helper_mutex_destroy(struct mutex *lock) +__rust_helper void rust_helper_mutex_destroy(struct mutex *lock) { mutex_destroy(lock); } diff --git a/rust/helpers/spinlock.c b/rust/helpers/spinlock.c index 42c4bf01a23e..4d13062cf253 100644 --- a/rust/helpers/spinlock.c +++ b/rust/helpers/spinlock.c @@ -2,8 +2,9 @@ #include -void rust_helper___spin_lock_init(spinlock_t *lock, const char *name, - struct lock_class_key *key) +__rust_helper void rust_helper___spin_lock_init(spinlock_t *lock, + const char *name, + struct lock_class_key *key) { #ifdef CONFIG_DEBUG_SPINLOCK # if defined(CONFIG_PREEMPT_RT) @@ -16,22 +17,22 @@ void rust_helper___spin_lock_init(spinlock_t *lock, const char *name, #endif /* CONFIG_DEBUG_SPINLOCK */ } -void rust_helper_spin_lock(spinlock_t *lock) +__rust_helper void rust_helper_spin_lock(spinlock_t *lock) { spin_lock(lock); } -void rust_helper_spin_unlock(spinlock_t *lock) +__rust_helper void rust_helper_spin_unlock(spinlock_t *lock) { spin_unlock(lock); } -int rust_helper_spin_trylock(spinlock_t *lock) +__rust_helper int rust_helper_spin_trylock(spinlock_t *lock) { return spin_trylock(lock); } -void rust_helper_spin_assert_is_held(spinlock_t *lock) +__rust_helper void rust_helper_spin_assert_is_held(spinlock_t *lock) { lockdep_assert_held(lock); } diff --git a/rust/helpers/sync.c b/rust/helpers/sync.c index ff7e68b48810..82d6aff73b04 100644 --- a/rust/helpers/sync.c +++ b/rust/helpers/sync.c @@ -2,12 +2,12 @@ #include -void rust_helper_lockdep_register_key(struct lock_class_key *k) +__rust_helper void rust_helper_lockdep_register_key(struct lock_class_key *k) { lockdep_register_key(k); } -void rust_helper_lockdep_unregister_key(struct lock_class_key *k) +__rust_helper void rust_helper_lockdep_unregister_key(struct lock_class_key *k) { lockdep_unregister_key(k); } From 5f1193d55a4311780136044355b1f09e7b5abac7 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 5 Jan 2026 12:42:34 +0000 Subject: [PATCH 75/90] rust: task: Add __rust_helper to helpers This is needed to inline these helpers into Rust code. Reviewed-by: Boqun Feng Reviewed-by: Gary Guo Signed-off-by: Alice Ryhl Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260105-define-rust-helper-v2-21-51da5f454a67@google.com --- rust/helpers/signal.c | 2 +- rust/helpers/task.c | 24 ++++++++++++------------ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/rust/helpers/signal.c b/rust/helpers/signal.c index 1a6bbe9438e2..85111186cf3d 100644 --- a/rust/helpers/signal.c +++ b/rust/helpers/signal.c @@ -2,7 +2,7 @@ #include -int rust_helper_signal_pending(struct task_struct *t) +__rust_helper int rust_helper_signal_pending(struct task_struct *t) { return signal_pending(t); } diff --git a/rust/helpers/task.c b/rust/helpers/task.c index 2c85bbc2727e..c0e1a06ede78 100644 --- a/rust/helpers/task.c +++ b/rust/helpers/task.c @@ -3,60 +3,60 @@ #include #include -void rust_helper_might_resched(void) +__rust_helper void rust_helper_might_resched(void) { might_resched(); } -struct task_struct *rust_helper_get_current(void) +__rust_helper struct task_struct *rust_helper_get_current(void) { return current; } -void rust_helper_get_task_struct(struct task_struct *t) +__rust_helper void rust_helper_get_task_struct(struct task_struct *t) { get_task_struct(t); } -void rust_helper_put_task_struct(struct task_struct *t) +__rust_helper void rust_helper_put_task_struct(struct task_struct *t) { put_task_struct(t); } -kuid_t rust_helper_task_uid(struct task_struct *task) +__rust_helper kuid_t rust_helper_task_uid(struct task_struct *task) { return task_uid(task); } -kuid_t rust_helper_task_euid(struct task_struct *task) +__rust_helper kuid_t rust_helper_task_euid(struct task_struct *task) { return task_euid(task); } #ifndef CONFIG_USER_NS -uid_t rust_helper_from_kuid(struct user_namespace *to, kuid_t uid) +__rust_helper uid_t rust_helper_from_kuid(struct user_namespace *to, kuid_t uid) { return from_kuid(to, uid); } #endif /* CONFIG_USER_NS */ -bool rust_helper_uid_eq(kuid_t left, kuid_t right) +__rust_helper bool rust_helper_uid_eq(kuid_t left, kuid_t right) { return uid_eq(left, right); } -kuid_t rust_helper_current_euid(void) +__rust_helper kuid_t rust_helper_current_euid(void) { return current_euid(); } -struct user_namespace *rust_helper_current_user_ns(void) +__rust_helper struct user_namespace *rust_helper_current_user_ns(void) { return current_user_ns(); } -pid_t rust_helper_task_tgid_nr_ns(struct task_struct *tsk, - struct pid_namespace *ns) +__rust_helper pid_t rust_helper_task_tgid_nr_ns(struct task_struct *tsk, + struct pid_namespace *ns) { return task_tgid_nr_ns(tsk, ns); } From 75b6034780e8dc8c71096313534ccb720fa633f9 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 5 Jan 2026 12:42:35 +0000 Subject: [PATCH 76/90] rust: time: Add __rust_helper to helpers This is needed to inline these helpers into Rust code. Reviewed-by: Boqun Feng Reviewed-by: Gary Guo Signed-off-by: Alice Ryhl Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260105-define-rust-helper-v2-22-51da5f454a67@google.com --- rust/helpers/time.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/rust/helpers/time.c b/rust/helpers/time.c index 67a36ccc3ec4..32f495970493 100644 --- a/rust/helpers/time.c +++ b/rust/helpers/time.c @@ -4,37 +4,37 @@ #include #include -void rust_helper_fsleep(unsigned long usecs) +__rust_helper void rust_helper_fsleep(unsigned long usecs) { fsleep(usecs); } -ktime_t rust_helper_ktime_get_real(void) +__rust_helper ktime_t rust_helper_ktime_get_real(void) { return ktime_get_real(); } -ktime_t rust_helper_ktime_get_boottime(void) +__rust_helper ktime_t rust_helper_ktime_get_boottime(void) { return ktime_get_boottime(); } -ktime_t rust_helper_ktime_get_clocktai(void) +__rust_helper ktime_t rust_helper_ktime_get_clocktai(void) { return ktime_get_clocktai(); } -s64 rust_helper_ktime_to_us(const ktime_t kt) +__rust_helper s64 rust_helper_ktime_to_us(const ktime_t kt) { return ktime_to_us(kt); } -s64 rust_helper_ktime_to_ms(const ktime_t kt) +__rust_helper s64 rust_helper_ktime_to_ms(const ktime_t kt) { return ktime_to_ms(kt); } -void rust_helper_udelay(unsigned long usec) +__rust_helper void rust_helper_udelay(unsigned long usec) { udelay(usec); } From 5628f0510a4c64908c5d2f36a676b092e1e5d174 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Mon, 5 Jan 2026 12:42:38 +0000 Subject: [PATCH 77/90] rust: wait: Add __rust_helper to helpers This is needed to inline these helpers into Rust code. Reviewed-by: Boqun Feng Reviewed-by: Gary Guo Signed-off-by: Alice Ryhl Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260105-define-rust-helper-v2-25-51da5f454a67@google.com --- rust/helpers/wait.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/helpers/wait.c b/rust/helpers/wait.c index ae48e33d9da3..2dde1e451780 100644 --- a/rust/helpers/wait.c +++ b/rust/helpers/wait.c @@ -2,7 +2,7 @@ #include -void rust_helper_init_wait(struct wait_queue_entry *wq_entry) +__rust_helper void rust_helper_init_wait(struct wait_queue_entry *wq_entry) { init_wait(wq_entry); } From abf2111d8d900c834993d443f59b836291b8d0fc Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Wed, 7 Jan 2026 14:14:13 +0000 Subject: [PATCH 78/90] rust: helpers: Move #define __rust_helper out of atomic.c In order to support inline helpers [1], we need to have __rust_helper defined for all helper files. Current we are lucky that atomic.c is the first file in helpers.c, but this is fragile. Thus, move it to helpers.c. [boqun: Reword the commit message and apply file hash changes] Link: https://lore.kernel.org/r/20260105-define-rust-helper-v2-0-51da5f454a67@google.com [1] Signed-off-by: Alice Ryhl Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20260107-move-rust_helper-define-v1-1-4109d58ef275@google.com --- rust/helpers/atomic.c | 7 +------ rust/helpers/helpers.c | 2 ++ scripts/atomic/gen-rust-atomic-helpers.sh | 5 ----- 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/rust/helpers/atomic.c b/rust/helpers/atomic.c index cf06b7ef9a1c..4b24eceef5fc 100644 --- a/rust/helpers/atomic.c +++ b/rust/helpers/atomic.c @@ -11,11 +11,6 @@ #include -// TODO: Remove this after INLINE_HELPERS support is added. -#ifndef __rust_helper -#define __rust_helper -#endif - __rust_helper int rust_helper_atomic_read(const atomic_t *v) { @@ -1037,4 +1032,4 @@ rust_helper_atomic64_dec_if_positive(atomic64_t *v) } #endif /* _RUST_ATOMIC_API_H */ -// 615a0e0c98b5973a47fe4fa65e92935051ca00ed +// e4edb6174dd42a265284958f00a7cea7ddb464b1 diff --git a/rust/helpers/helpers.c b/rust/helpers/helpers.c index 15d75578f459..a3c42e51f00a 100644 --- a/rust/helpers/helpers.c +++ b/rust/helpers/helpers.c @@ -7,6 +7,8 @@ * Sorted alphabetically. */ +#define __rust_helper + #include "atomic.c" #include "atomic_ext.c" #include "auxiliary.c" diff --git a/scripts/atomic/gen-rust-atomic-helpers.sh b/scripts/atomic/gen-rust-atomic-helpers.sh index 45b1e100ed7c..a3732153af29 100755 --- a/scripts/atomic/gen-rust-atomic-helpers.sh +++ b/scripts/atomic/gen-rust-atomic-helpers.sh @@ -47,11 +47,6 @@ cat << EOF #include -// TODO: Remove this after INLINE_HELPERS support is added. -#ifndef __rust_helper -#define __rust_helper -#endif - EOF grep '^[a-z]' "$1" | while read name meta args; do From ccf9e070116a81d29aae30db501d562c8efd1ed8 Mon Sep 17 00:00:00 2001 From: Alice Ryhl Date: Thu, 18 Dec 2025 12:10:23 +0000 Subject: [PATCH 79/90] rust: sync: Inline various lock related methods While debugging a different issue [1], the following relocation was noticed in the rust_binder.ko file: R_AARCH64_CALL26 _RNvXNtNtNtCsdfZWD8DztAw_6kernel4sync4lock8spinlockNtB2_15SpinLockBackendNtB4_7Backend6unlock This relocation (and a similar one for lock) occurred many times throughout the module. That is not really useful because all this function does is call spin_unlock(), so what we actually want here is that a call to spin_unlock() dirctly is generated in favor of this wrapper method. Thus, mark these methods inline. [boqun: Reword the commit message a bit] Link: https://lore.kernel.org/p/20251111-binder-fix-list-remove-v1-0-8ed14a0da63d@google.com Signed-off-by: Alice Ryhl Reviewed-by: Gary Guo Reviewed-by: Daniel Almeida Signed-off-by: Boqun Feng Link: https://patch.msgid.link/20251218-inline-lock-unlock-v2-1-fbadac8bd61b@google.com --- rust/kernel/sync/lock.rs | 7 +++++++ rust/kernel/sync/lock/global.rs | 2 ++ rust/kernel/sync/lock/mutex.rs | 5 +++++ rust/kernel/sync/lock/spinlock.rs | 5 +++++ 4 files changed, 19 insertions(+) diff --git a/rust/kernel/sync/lock.rs b/rust/kernel/sync/lock.rs index 46a57d1fc309..10b6b5e9b024 100644 --- a/rust/kernel/sync/lock.rs +++ b/rust/kernel/sync/lock.rs @@ -156,6 +156,7 @@ impl Lock<(), B> { /// the whole lifetime of `'a`. /// /// [`State`]: Backend::State + #[inline] pub unsafe fn from_raw<'a>(ptr: *mut B::State) -> &'a Self { // SAFETY: // - By the safety contract `ptr` must point to a valid initialised instance of `B::State` @@ -169,6 +170,7 @@ impl Lock<(), B> { impl Lock { /// Acquires the lock and gives the caller access to the data protected by it. + #[inline] pub fn lock(&self) -> Guard<'_, T, B> { // SAFETY: The constructor of the type calls `init`, so the existence of the object proves // that `init` was called. @@ -182,6 +184,7 @@ impl Lock { /// Returns a guard that can be used to access the data protected by the lock if successful. // `Option` is not `#[must_use]` even if `T` is, thus the attribute is needed here. #[must_use = "if unused, the lock will be immediately unlocked"] + #[inline] pub fn try_lock(&self) -> Option> { // SAFETY: The constructor of the type calls `init`, so the existence of the object proves // that `init` was called. @@ -275,6 +278,7 @@ impl<'a, T: ?Sized, B: Backend> Guard<'a, T, B> { impl core::ops::Deref for Guard<'_, T, B> { type Target = T; + #[inline] fn deref(&self) -> &Self::Target { // SAFETY: The caller owns the lock, so it is safe to deref the protected data. unsafe { &*self.lock.data.get() } @@ -285,6 +289,7 @@ impl core::ops::DerefMut for Guard<'_, T, B> where T: Unpin, { + #[inline] fn deref_mut(&mut self) -> &mut Self::Target { // SAFETY: The caller owns the lock, so it is safe to deref the protected data. unsafe { &mut *self.lock.data.get() } @@ -292,6 +297,7 @@ where } impl Drop for Guard<'_, T, B> { + #[inline] fn drop(&mut self) { // SAFETY: The caller owns the lock, so it is safe to unlock it. unsafe { B::unlock(self.lock.state.get(), &self.state) }; @@ -304,6 +310,7 @@ impl<'a, T: ?Sized, B: Backend> Guard<'a, T, B> { /// # Safety /// /// The caller must ensure that it owns the lock. + #[inline] pub unsafe fn new(lock: &'a Lock, state: B::GuardState) -> Self { // SAFETY: The caller can only hold the lock if `Backend::init` has already been called. unsafe { B::assert_is_held(lock.state.get()) }; diff --git a/rust/kernel/sync/lock/global.rs b/rust/kernel/sync/lock/global.rs index eab48108a4ae..aecbdc34738f 100644 --- a/rust/kernel/sync/lock/global.rs +++ b/rust/kernel/sync/lock/global.rs @@ -77,6 +77,7 @@ impl GlobalLock { } /// Lock this global lock. + #[inline] pub fn lock(&'static self) -> GlobalGuard { GlobalGuard { inner: self.inner.lock(), @@ -84,6 +85,7 @@ impl GlobalLock { } /// Try to lock this global lock. + #[inline] pub fn try_lock(&'static self) -> Option> { Some(GlobalGuard { inner: self.inner.try_lock()?, diff --git a/rust/kernel/sync/lock/mutex.rs b/rust/kernel/sync/lock/mutex.rs index 581cee7ab842..cda0203efefb 100644 --- a/rust/kernel/sync/lock/mutex.rs +++ b/rust/kernel/sync/lock/mutex.rs @@ -102,6 +102,7 @@ unsafe impl super::Backend for MutexBackend { type State = bindings::mutex; type GuardState = (); + #[inline] unsafe fn init( ptr: *mut Self::State, name: *const crate::ffi::c_char, @@ -112,18 +113,21 @@ unsafe impl super::Backend for MutexBackend { unsafe { bindings::__mutex_init(ptr, name, key) } } + #[inline] unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState { // SAFETY: The safety requirements of this function ensure that `ptr` points to valid // memory, and that it has been initialised before. unsafe { bindings::mutex_lock(ptr) }; } + #[inline] unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) { // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the // caller is the owner of the mutex. unsafe { bindings::mutex_unlock(ptr) }; } + #[inline] unsafe fn try_lock(ptr: *mut Self::State) -> Option { // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. let result = unsafe { bindings::mutex_trylock(ptr) }; @@ -135,6 +139,7 @@ unsafe impl super::Backend for MutexBackend { } } + #[inline] unsafe fn assert_is_held(ptr: *mut Self::State) { // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. unsafe { bindings::mutex_assert_is_held(ptr) } diff --git a/rust/kernel/sync/lock/spinlock.rs b/rust/kernel/sync/lock/spinlock.rs index d7be38ccbdc7..ef76fa07ca3a 100644 --- a/rust/kernel/sync/lock/spinlock.rs +++ b/rust/kernel/sync/lock/spinlock.rs @@ -101,6 +101,7 @@ unsafe impl super::Backend for SpinLockBackend { type State = bindings::spinlock_t; type GuardState = (); + #[inline] unsafe fn init( ptr: *mut Self::State, name: *const crate::ffi::c_char, @@ -111,18 +112,21 @@ unsafe impl super::Backend for SpinLockBackend { unsafe { bindings::__spin_lock_init(ptr, name, key) } } + #[inline] unsafe fn lock(ptr: *mut Self::State) -> Self::GuardState { // SAFETY: The safety requirements of this function ensure that `ptr` points to valid // memory, and that it has been initialised before. unsafe { bindings::spin_lock(ptr) } } + #[inline] unsafe fn unlock(ptr: *mut Self::State, _guard_state: &Self::GuardState) { // SAFETY: The safety requirements of this function ensure that `ptr` is valid and that the // caller is the owner of the spinlock. unsafe { bindings::spin_unlock(ptr) } } + #[inline] unsafe fn try_lock(ptr: *mut Self::State) -> Option { // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. let result = unsafe { bindings::spin_trylock(ptr) }; @@ -134,6 +138,7 @@ unsafe impl super::Backend for SpinLockBackend { } } + #[inline] unsafe fn assert_is_held(ptr: *mut Self::State) { // SAFETY: The `ptr` pointer is guaranteed to be valid and initialized before use. unsafe { bindings::spin_assert_is_held(ptr) } From e6de07249ef381b674f0d65adf9defcdab76b768 Mon Sep 17 00:00:00 2001 From: Tamir Duberstein Date: Thu, 22 Jan 2026 21:46:24 -0800 Subject: [PATCH 80/90] rust: sync: Replace `kernel::c_str!` with C-Strings C-String literals were added in Rust 1.77. Replace instances of `kernel::c_str!` with C-String literals where possible. Signed-off-by: Tamir Duberstein Signed-off-by: Boqun Feng Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Gary Guo Link: https://patch.msgid.link/20260120-cstr-sync-again-v1-1-2a775a2a36fd@kernel.org Link: https://patch.msgid.link/20260123054624.8226-2-boqun.feng@gmail.com --- rust/kernel/sync.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rust/kernel/sync.rs b/rust/kernel/sync.rs index b10e576221ff..993dbf2caa0e 100644 --- a/rust/kernel/sync.rs +++ b/rust/kernel/sync.rs @@ -126,13 +126,12 @@ impl PinnedDrop for LockClassKey { /// # Examples /// /// ``` -/// use kernel::c_str; /// use kernel::sync::{static_lock_class, Arc, SpinLock}; /// /// fn new_locked_int() -> Result>> { /// Arc::pin_init(SpinLock::new( /// 42, -/// c_str!("new_locked_int"), +/// c"new_locked_int", /// static_lock_class!(), /// ), GFP_KERNEL) /// } From 944e3f7562c55fa37ebcdd58e5f60f296c81a854 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Tue, 27 Jan 2026 12:12:06 +0100 Subject: [PATCH 81/90] tools: Update context analysis macros in compiler_types.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In sync with the main kernel headers, include a stub version of compiler-context-analysis.h in tools/include/linux/compiler_types.h and remove the sparse context tracking definitions. Since tools/ headers are generally self-contained, provide a standalone tools/include/linux/compiler-context-analysis.h with no-op stubs for now. Also clean up redundant stubs in tools/testing/shared/linux/kernel.h that are now redundant. This fixes build errors in tools/testing/radix-tree/ where headers from include/linux/ (like cleanup.h) are used directly and expect these macros to be defined: | cc -I../shared -I. -I../../include -I../../arch/x86/include -I../../../lib -g -Og -Wall -D_LGPL_SOURCE -fsanitize=address -fsanitize=undefined -c -o radix-tree.o radix-tree.c | In file included from ../shared/linux/cleanup.h:2, | from ../shared/linux/../../../../include/linux/idr.h:18, | from ../shared/linux/idr.h:5, | from radix-tree.c:18: | ../shared/linux/../../../../include/linux/idr.h: In function ‘class_idr_alloc_destructor’: | ../shared/linux/../../../../include/linux/cleanup.h:283:9: error: expected declaration specifiers before ‘__no_context_analysis’ | 283 | __no_context_analysis \ | | ^~~~~~~~~~~~~~~~~~~~~ Closes: https://lore.kernel.org/oe-lkp/202601261546.d7ae2447-lkp@intel.com Reported-by: kernel test robot Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Lorenzo Stoakes Tested-by: Lorenzo Stoakes Link: https://patch.msgid.link/20260127111428.3747328-1-elver@google.com --- .../include/linux/compiler-context-analysis.h | 42 +++++++++++++++++++ tools/include/linux/compiler_types.h | 16 +------ tools/testing/shared/linux/kernel.h | 4 -- 3 files changed, 43 insertions(+), 19 deletions(-) create mode 100644 tools/include/linux/compiler-context-analysis.h diff --git a/tools/include/linux/compiler-context-analysis.h b/tools/include/linux/compiler-context-analysis.h new file mode 100644 index 000000000000..13a9115e9e58 --- /dev/null +++ b/tools/include/linux/compiler-context-analysis.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _TOOLS_LINUX_COMPILER_CONTEXT_ANALYSIS_H +#define _TOOLS_LINUX_COMPILER_CONTEXT_ANALYSIS_H + +/* + * Macros and attributes for compiler-based static context analysis. + * No-op stubs for tools. + */ + +#define __guarded_by(...) +#define __pt_guarded_by(...) + +#define context_lock_struct(name, ...) struct __VA_ARGS__ name + +#define __no_context_analysis +#define __context_unsafe(comment) +#define context_unsafe(...) ({ __VA_ARGS__; }) +#define context_unsafe_alias(p) +#define disable_context_analysis() +#define enable_context_analysis() + +#define __must_hold(...) +#define __must_not_hold(...) +#define __acquires(...) +#define __cond_acquires(ret, x) +#define __releases(...) +#define __acquire(x) (void)0 +#define __release(x) (void)0 + +#define __must_hold_shared(...) +#define __acquires_shared(...) +#define __cond_acquires_shared(ret, x) +#define __releases_shared(...) +#define __acquire_shared(x) (void)0 +#define __release_shared(x) (void)0 + +#define __acquire_ret(call, expr) (call) +#define __acquire_shared_ret(call, expr) (call) +#define __acquires_ret +#define __acquires_shared_ret + +#endif /* _TOOLS_LINUX_COMPILER_CONTEXT_ANALYSIS_H */ diff --git a/tools/include/linux/compiler_types.h b/tools/include/linux/compiler_types.h index 067a5b4e0f7b..14e420467eee 100644 --- a/tools/include/linux/compiler_types.h +++ b/tools/include/linux/compiler_types.h @@ -13,21 +13,7 @@ #define __has_builtin(x) (0) #endif -#ifdef __CHECKER__ -/* context/locking */ -# define __must_hold(x) __attribute__((context(x,1,1))) -# define __acquires(x) __attribute__((context(x,0,1))) -# define __releases(x) __attribute__((context(x,1,0))) -# define __acquire(x) __context__(x,1) -# define __release(x) __context__(x,-1) -#else /* __CHECKER__ */ -/* context/locking */ -# define __must_hold(x) -# define __acquires(x) -# define __releases(x) -# define __acquire(x) (void)0 -# define __release(x) (void)0 -#endif /* __CHECKER__ */ +#include /* Compiler specific macros. */ #ifdef __GNUC__ diff --git a/tools/testing/shared/linux/kernel.h b/tools/testing/shared/linux/kernel.h index c0a2bb785b92..dc2b4ccfb185 100644 --- a/tools/testing/shared/linux/kernel.h +++ b/tools/testing/shared/linux/kernel.h @@ -21,9 +21,5 @@ #define schedule() #define PAGE_SHIFT 12 -#define __acquires(x) -#define __releases(x) -#define __must_hold(x) - #define EXPORT_PER_CPU_SYMBOL_GPL(x) #endif /* _KERNEL_H */ From f88a31308db6a856229150039b0f56d59696ed31 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Fri, 23 Jan 2026 10:37:49 -0800 Subject: [PATCH 82/90] seqlock: fix scoped_seqlock_read kernel-doc Eliminate all kernel-doc warnings in seqlock.h: - correct the macro to have "()" immediately following the macro name - don't include the macro parameters in the short description (first line) - make the parameter names in the comments match the actual macro parameter names. - use "::" for the Example WARNING: include/linux/seqlock.h:1341 This comment starts with '/**', but isn't a kernel-doc comment. * scoped_seqlock_read (lock, ss_state) - execute the read side critical Documentation/locking/seqlock:242: include/linux/seqlock.h:1351: WARNING: Definition list ends without a blank line; unexpected unindent. [docutils] Warning: include/linux/seqlock.h:1357 function parameter '_seqlock' not described in 'scoped_seqlock_read' Warning: include/linux/seqlock.h:1357 function parameter '_target' not described in 'scoped_seqlock_read' Fixes: cc39f3872c08 ("seqlock: Introduce scoped_seqlock_read()") Signed-off-by: Randy Dunlap Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260123183749.3997533-1-rdunlap@infradead.org --- include/linux/seqlock.h | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h index 113320911a09..c00063dffba3 100644 --- a/include/linux/seqlock.h +++ b/include/linux/seqlock.h @@ -1339,15 +1339,14 @@ static __always_inline void __scoped_seqlock_cleanup_ctx(struct ss_tmp **s) __scoped_seqlock_next(&_s, _seqlock, _target)) /** - * scoped_seqlock_read (lock, ss_state) - execute the read side critical - * section without manual sequence - * counter handling or calls to other - * helpers - * @lock: pointer to seqlock_t protecting the data - * @ss_state: one of {ss_lock, ss_lock_irqsave, ss_lockless} indicating - * the type of critical read section + * scoped_seqlock_read() - execute the read-side critical section + * without manual sequence counter handling + * or calls to other helpers + * @_seqlock: pointer to seqlock_t protecting the data + * @_target: an enum ss_state: one of {ss_lock, ss_lock_irqsave, ss_lockless} + * indicating the type of critical read section * - * Example: + * Example:: * * scoped_seqlock_read (&lock, ss_lock) { * // read-side critical section From 3b9ed30344a866f6f96896b3ce64303b9074682b Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Mon, 19 Jan 2026 10:05:51 +0100 Subject: [PATCH 83/90] cleanup: Make __DEFINE_LOCK_GUARD handle commas in initializers Initialization macros can expand to structure initializers containing commas, which when used as a "lock" function resulted in errors such as: >> include/linux/spinlock.h:582:56: error: too many arguments provided to function-like macro invocation 582 | DEFINE_LOCK_GUARD_1(raw_spinlock_init, raw_spinlock_t, raw_spin_lock_init(_T->lock), /* */) | ^ include/linux/spinlock.h:113:17: note: expanded from macro 'raw_spin_lock_init' 113 | do { *(lock) = __RAW_SPIN_LOCK_UNLOCKED(lock); } while (0) | ^ include/linux/spinlock_types_raw.h:70:19: note: expanded from macro '__RAW_SPIN_LOCK_UNLOCKED' 70 | (raw_spinlock_t) __RAW_SPIN_LOCK_INITIALIZER(lockname) | ^ include/linux/spinlock_types_raw.h:67:34: note: expanded from macro '__RAW_SPIN_LOCK_INITIALIZER' 67 | RAW_SPIN_DEP_MAP_INIT(lockname) } | ^ include/linux/cleanup.h:496:9: note: macro '__DEFINE_LOCK_GUARD_1' defined here 496 | #define __DEFINE_LOCK_GUARD_1(_name, _type, _lock) \ | ^ include/linux/spinlock.h:582:1: note: parentheses are required around macro argument containing braced initializer list 582 | DEFINE_LOCK_GUARD_1(raw_spinlock_init, raw_spinlock_t, raw_spin_lock_init(_T->lock), /* */) | ^ | ( include/linux/cleanup.h:558:60: note: expanded from macro 'DEFINE_LOCK_GUARD_1' 558 | __DEFINE_UNLOCK_GUARD(_name, _type, _unlock, __VA_ARGS__) \ | ^ Make __DEFINE_LOCK_GUARD_0 and __DEFINE_LOCK_GUARD_1 variadic so that __VA_ARGS__ captures everything. Reported-by: kernel test robot Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260119094029.1344361-2-elver@google.com --- include/linux/cleanup.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/include/linux/cleanup.h b/include/linux/cleanup.h index ee6df68c2177..dbc4162921e9 100644 --- a/include/linux/cleanup.h +++ b/include/linux/cleanup.h @@ -493,22 +493,22 @@ static __always_inline void class_##_name##_destructor(class_##_name##_t *_T) \ \ __DEFINE_GUARD_LOCK_PTR(_name, &_T->lock) -#define __DEFINE_LOCK_GUARD_1(_name, _type, _lock) \ +#define __DEFINE_LOCK_GUARD_1(_name, _type, ...) \ static __always_inline class_##_name##_t class_##_name##_constructor(_type *l) \ __no_context_analysis \ { \ class_##_name##_t _t = { .lock = l }, *_T = &_t; \ - _lock; \ + __VA_ARGS__; \ return _t; \ } -#define __DEFINE_LOCK_GUARD_0(_name, _lock) \ +#define __DEFINE_LOCK_GUARD_0(_name, ...) \ static __always_inline class_##_name##_t class_##_name##_constructor(void) \ __no_context_analysis \ { \ class_##_name##_t _t = { .lock = (void*)1 }, \ *_T __maybe_unused = &_t; \ - _lock; \ + __VA_ARGS__; \ return _t; \ } From d084a73714f818ce509022e1aa9483cabf797c16 Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Mon, 19 Jan 2026 10:05:52 +0100 Subject: [PATCH 84/90] compiler-context-analysis: Introduce scoped init guards Add scoped init guard definitions for common synchronization primitives supported by context analysis. The scoped init guards treat the context as active within initialization scope of the underlying context lock, given initialization implies exclusive access to the underlying object. This allows initialization of guarded members without disabling context analysis, while documenting initialization from subsequent usage. The documentation is updated with the new recommendation. Where scoped init guards are not provided or cannot be implemented (ww_mutex omitted for lack of multi-arg guard initializers), the alternative is to just disable context analysis where guarded members are initialized. Suggested-by: Peter Zijlstra Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/all/20251212095943.GM3911114@noisy.programming.kicks-ass.net/ Link: https://patch.msgid.link/20260119094029.1344361-3-elver@google.com --- Documentation/dev-tools/context-analysis.rst | 30 ++++++++++++++++++-- include/linux/compiler-context-analysis.h | 9 ++---- include/linux/local_lock.h | 8 ++++++ include/linux/local_lock_internal.h | 1 + include/linux/mutex.h | 3 ++ include/linux/rwsem.h | 4 +++ include/linux/seqlock.h | 5 ++++ include/linux/spinlock.h | 12 ++++++++ lib/test_context-analysis.c | 16 +++++------ 9 files changed, 70 insertions(+), 18 deletions(-) diff --git a/Documentation/dev-tools/context-analysis.rst b/Documentation/dev-tools/context-analysis.rst index e69896e597b6..54d9ee28de98 100644 --- a/Documentation/dev-tools/context-analysis.rst +++ b/Documentation/dev-tools/context-analysis.rst @@ -83,9 +83,33 @@ Currently the following synchronization primitives are supported: `bit_spinlock`, RCU, SRCU (`srcu_struct`), `rw_semaphore`, `local_lock_t`, `ww_mutex`. -For context locks with an initialization function (e.g., `spin_lock_init()`), -calling this function before initializing any guarded members or globals -prevents the compiler from issuing warnings about unguarded initialization. +To initialize variables guarded by a context lock with an initialization +function (``type_init(&lock)``), prefer using ``guard(type_init)(&lock)`` or +``scoped_guard(type_init, &lock) { ... }`` to initialize such guarded members +or globals in the enclosing scope. This initializes the context lock and treats +the context as active within the initialization scope (initialization implies +exclusive access to the underlying object). + +For example:: + + struct my_data { + spinlock_t lock; + int counter __guarded_by(&lock); + }; + + void init_my_data(struct my_data *d) + { + ... + guard(spinlock_init)(&d->lock); + d->counter = 0; + ... + } + +Alternatively, initializing guarded variables can be done with context analysis +disabled, preferably in the smallest possible scope (due to lack of any other +checking): either with a ``context_unsafe(var = init)`` expression, or by +marking small initialization functions with the ``__context_unsafe(init)`` +attribute. Lockdep assertions, such as `lockdep_assert_held()`, inform the compiler's context analysis that the associated synchronization primitive is held after diff --git a/include/linux/compiler-context-analysis.h b/include/linux/compiler-context-analysis.h index e86b8a3c2f89..00c074a2ccb0 100644 --- a/include/linux/compiler-context-analysis.h +++ b/include/linux/compiler-context-analysis.h @@ -32,13 +32,8 @@ /* * The "assert_capability" attribute is a bit confusingly named. It does not * generate a check. Instead, it tells the analysis to *assume* the capability - * is held. This is used for: - * - * 1. Augmenting runtime assertions, that can then help with patterns beyond the - * compiler's static reasoning abilities. - * - * 2. Initialization of context locks, so we can access guarded variables right - * after initialization (nothing else should access the same object yet). + * is held. This is used for augmenting runtime assertions, that can then help + * with patterns beyond the compiler's static reasoning abilities. */ # define __assumes_ctx_lock(...) __attribute__((assert_capability(__VA_ARGS__))) # define __assumes_shared_ctx_lock(...) __attribute__((assert_shared_capability(__VA_ARGS__))) diff --git a/include/linux/local_lock.h b/include/linux/local_lock.h index 99c06e499375..b8830148a859 100644 --- a/include/linux/local_lock.h +++ b/include/linux/local_lock.h @@ -104,6 +104,8 @@ DEFINE_LOCK_GUARD_1(local_lock_nested_bh, local_lock_t __percpu, local_lock_nested_bh(_T->lock), local_unlock_nested_bh(_T->lock)) +DEFINE_LOCK_GUARD_1(local_lock_init, local_lock_t, local_lock_init(_T->lock), /* */) + DECLARE_LOCK_GUARD_1_ATTRS(local_lock, __acquires(_T), __releases(*(local_lock_t __percpu **)_T)) #define class_local_lock_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(local_lock, _T) DECLARE_LOCK_GUARD_1_ATTRS(local_lock_irq, __acquires(_T), __releases(*(local_lock_t __percpu **)_T)) @@ -112,5 +114,11 @@ DECLARE_LOCK_GUARD_1_ATTRS(local_lock_irqsave, __acquires(_T), __releases(*(loca #define class_local_lock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(local_lock_irqsave, _T) DECLARE_LOCK_GUARD_1_ATTRS(local_lock_nested_bh, __acquires(_T), __releases(*(local_lock_t __percpu **)_T)) #define class_local_lock_nested_bh_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(local_lock_nested_bh, _T) +DECLARE_LOCK_GUARD_1_ATTRS(local_lock_init, __acquires(_T), __releases(*(local_lock_t **)_T)) +#define class_local_lock_init_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(local_lock_init, _T) + +DEFINE_LOCK_GUARD_1(local_trylock_init, local_trylock_t, local_trylock_init(_T->lock), /* */) +DECLARE_LOCK_GUARD_1_ATTRS(local_trylock_init, __acquires(_T), __releases(*(local_trylock_t **)_T)) +#define class_local_trylock_init_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(local_trylock_init, _T) #endif diff --git a/include/linux/local_lock_internal.h b/include/linux/local_lock_internal.h index 7843ab9059c2..ed2f3fb4c360 100644 --- a/include/linux/local_lock_internal.h +++ b/include/linux/local_lock_internal.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #ifndef CONFIG_PREEMPT_RT diff --git a/include/linux/mutex.h b/include/linux/mutex.h index 89977c215cbd..6b12009351d2 100644 --- a/include/linux/mutex.h +++ b/include/linux/mutex.h @@ -254,6 +254,7 @@ extern int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock) __cond_a DEFINE_LOCK_GUARD_1(mutex, struct mutex, mutex_lock(_T->lock), mutex_unlock(_T->lock)) DEFINE_LOCK_GUARD_1_COND(mutex, _try, mutex_trylock(_T->lock)) DEFINE_LOCK_GUARD_1_COND(mutex, _intr, mutex_lock_interruptible(_T->lock), _RET == 0) +DEFINE_LOCK_GUARD_1(mutex_init, struct mutex, mutex_init(_T->lock), /* */) DECLARE_LOCK_GUARD_1_ATTRS(mutex, __acquires(_T), __releases(*(struct mutex **)_T)) #define class_mutex_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(mutex, _T) @@ -261,6 +262,8 @@ DECLARE_LOCK_GUARD_1_ATTRS(mutex_try, __acquires(_T), __releases(*(struct mutex #define class_mutex_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(mutex_try, _T) DECLARE_LOCK_GUARD_1_ATTRS(mutex_intr, __acquires(_T), __releases(*(struct mutex **)_T)) #define class_mutex_intr_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(mutex_intr, _T) +DECLARE_LOCK_GUARD_1_ATTRS(mutex_init, __acquires(_T), __releases(*(struct mutex **)_T)) +#define class_mutex_init_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(mutex_init, _T) extern unsigned long mutex_get_owner(struct mutex *lock); diff --git a/include/linux/rwsem.h b/include/linux/rwsem.h index 8da14a08a4e1..ea1bbdb57a47 100644 --- a/include/linux/rwsem.h +++ b/include/linux/rwsem.h @@ -280,6 +280,10 @@ DECLARE_LOCK_GUARD_1_ATTRS(rwsem_write_try, __acquires(_T), __releases(*(struct DECLARE_LOCK_GUARD_1_ATTRS(rwsem_write_kill, __acquires(_T), __releases(*(struct rw_semaphore **)_T)) #define class_rwsem_write_kill_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(rwsem_write_kill, _T) +DEFINE_LOCK_GUARD_1(rwsem_init, struct rw_semaphore, init_rwsem(_T->lock), /* */) +DECLARE_LOCK_GUARD_1_ATTRS(rwsem_init, __acquires(_T), __releases(*(struct rw_semaphore **)_T)) +#define class_rwsem_init_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(rwsem_init, _T) + /* * downgrade write lock to read lock */ diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h index c00063dffba3..077c8d5b2afd 100644 --- a/include/linux/seqlock.h +++ b/include/linux/seqlock.h @@ -14,6 +14,7 @@ */ #include +#include #include #include #include @@ -1358,4 +1359,8 @@ static __always_inline void __scoped_seqlock_cleanup_ctx(struct ss_tmp **s) #define scoped_seqlock_read(_seqlock, _target) \ __scoped_seqlock_read(_seqlock, _target, __UNIQUE_ID(seqlock)) +DEFINE_LOCK_GUARD_1(seqlock_init, seqlock_t, seqlock_init(_T->lock), /* */) +DECLARE_LOCK_GUARD_1_ATTRS(seqlock_init, __acquires(_T), __releases(*(seqlock_t **)_T)) +#define class_seqlock_init_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(seqlock_init, _T) + #endif /* __LINUX_SEQLOCK_H */ diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index 396b8c5d6c1b..7b11991c742a 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -582,6 +582,10 @@ DEFINE_LOCK_GUARD_1_COND(raw_spinlock_irqsave, _try, DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave_try, __acquires(_T), __releases(*(raw_spinlock_t **)_T)) #define class_raw_spinlock_irqsave_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_irqsave_try, _T) +DEFINE_LOCK_GUARD_1(raw_spinlock_init, raw_spinlock_t, raw_spin_lock_init(_T->lock), /* */) +DECLARE_LOCK_GUARD_1_ATTRS(raw_spinlock_init, __acquires(_T), __releases(*(raw_spinlock_t **)_T)) +#define class_raw_spinlock_init_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(raw_spinlock_init, _T) + DEFINE_LOCK_GUARD_1(spinlock, spinlock_t, spin_lock(_T->lock), spin_unlock(_T->lock)) @@ -626,6 +630,10 @@ DEFINE_LOCK_GUARD_1_COND(spinlock_irqsave, _try, DECLARE_LOCK_GUARD_1_ATTRS(spinlock_irqsave_try, __acquires(_T), __releases(*(spinlock_t **)_T)) #define class_spinlock_irqsave_try_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_irqsave_try, _T) +DEFINE_LOCK_GUARD_1(spinlock_init, spinlock_t, spin_lock_init(_T->lock), /* */) +DECLARE_LOCK_GUARD_1_ATTRS(spinlock_init, __acquires(_T), __releases(*(spinlock_t **)_T)) +#define class_spinlock_init_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(spinlock_init, _T) + DEFINE_LOCK_GUARD_1(read_lock, rwlock_t, read_lock(_T->lock), read_unlock(_T->lock)) @@ -664,5 +672,9 @@ DEFINE_LOCK_GUARD_1(write_lock_irqsave, rwlock_t, DECLARE_LOCK_GUARD_1_ATTRS(write_lock_irqsave, __acquires(_T), __releases(*(rwlock_t **)_T)) #define class_write_lock_irqsave_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(write_lock_irqsave, _T) +DEFINE_LOCK_GUARD_1(rwlock_init, rwlock_t, rwlock_init(_T->lock), /* */) +DECLARE_LOCK_GUARD_1_ATTRS(rwlock_init, __acquires(_T), __releases(*(rwlock_t **)_T)) +#define class_rwlock_init_constructor(_T) WITH_LOCK_GUARD_1_ATTRS(rwlock_init, _T) + #undef __LINUX_INSIDE_SPINLOCK_H #endif /* __LINUX_SPINLOCK_H */ diff --git a/lib/test_context-analysis.c b/lib/test_context-analysis.c index 1c5a381461fc..0f05943d957f 100644 --- a/lib/test_context-analysis.c +++ b/lib/test_context-analysis.c @@ -35,7 +35,7 @@ static void __used test_common_helpers(void) }; \ static void __used test_##class##_init(struct test_##class##_data *d) \ { \ - type_init(&d->lock); \ + guard(type_init)(&d->lock); \ d->counter = 0; \ } \ static void __used test_##class(struct test_##class##_data *d) \ @@ -83,7 +83,7 @@ static void __used test_common_helpers(void) TEST_SPINLOCK_COMMON(raw_spinlock, raw_spinlock_t, - raw_spin_lock_init, + raw_spinlock_init, raw_spin_lock, raw_spin_unlock, raw_spin_trylock, @@ -109,7 +109,7 @@ static void __used test_raw_spinlock_trylock_extra(struct test_raw_spinlock_data TEST_SPINLOCK_COMMON(spinlock, spinlock_t, - spin_lock_init, + spinlock_init, spin_lock, spin_unlock, spin_trylock, @@ -163,7 +163,7 @@ struct test_mutex_data { static void __used test_mutex_init(struct test_mutex_data *d) { - mutex_init(&d->mtx); + guard(mutex_init)(&d->mtx); d->counter = 0; } @@ -226,7 +226,7 @@ struct test_seqlock_data { static void __used test_seqlock_init(struct test_seqlock_data *d) { - seqlock_init(&d->sl); + guard(seqlock_init)(&d->sl); d->counter = 0; } @@ -275,7 +275,7 @@ struct test_rwsem_data { static void __used test_rwsem_init(struct test_rwsem_data *d) { - init_rwsem(&d->sem); + guard(rwsem_init)(&d->sem); d->counter = 0; } @@ -475,7 +475,7 @@ static DEFINE_PER_CPU(struct test_local_lock_data, test_local_lock_data) = { static void __used test_local_lock_init(struct test_local_lock_data *d) { - local_lock_init(&d->lock); + guard(local_lock_init)(&d->lock); d->counter = 0; } @@ -519,7 +519,7 @@ static DEFINE_PER_CPU(struct test_local_trylock_data, test_local_trylock_data) = static void __used test_local_trylock_init(struct test_local_trylock_data *d) { - local_trylock_init(&d->lock); + guard(local_trylock_init)(&d->lock); d->counter = 0; } From b7be9442a3758a27a4b09b75ad79f3626b16ec3d Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Mon, 19 Jan 2026 10:05:53 +0100 Subject: [PATCH 85/90] kcov: Use scoped init guard Convert lock initialization to scoped guarded initialization where lock-guarded members are initialized in the same scope. This ensures the context analysis treats the context as active during member initialization. This is required to avoid errors once implicit context assertion is removed. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260119094029.1344361-4-elver@google.com --- kernel/kcov.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/kcov.c b/kernel/kcov.c index 6cbc6e2d8aee..5397d0c14127 100644 --- a/kernel/kcov.c +++ b/kernel/kcov.c @@ -530,7 +530,7 @@ static int kcov_open(struct inode *inode, struct file *filep) kcov = kzalloc(sizeof(*kcov), GFP_KERNEL); if (!kcov) return -ENOMEM; - spin_lock_init(&kcov->lock); + guard(spinlock_init)(&kcov->lock); kcov->mode = KCOV_MODE_DISABLED; kcov->sequence = 1; refcount_set(&kcov->refcount, 1); From f39261f55b3ee58d85e96142763c25b945399b2f Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Mon, 19 Jan 2026 10:05:54 +0100 Subject: [PATCH 86/90] crypto: Use scoped init guard Convert lock initialization to scoped guarded initialization where lock-guarded members are initialized in the same scope. This ensures the context analysis treats the context as active during member initialization. This is required to avoid errors once implicit context assertion is removed. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260119094029.1344361-5-elver@google.com --- crypto/crypto_engine.c | 2 +- crypto/drbg.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crypto/crypto_engine.c b/crypto/crypto_engine.c index 1653a4bf5b31..afb6848f7df4 100644 --- a/crypto/crypto_engine.c +++ b/crypto/crypto_engine.c @@ -453,7 +453,7 @@ struct crypto_engine *crypto_engine_alloc_init_and_set(struct device *dev, snprintf(engine->name, sizeof(engine->name), "%s-engine", dev_name(dev)); - spin_lock_init(&engine->queue_lock); + guard(spinlock_init)(&engine->queue_lock); crypto_init_queue(&engine->queue, qlen); engine->kworker = kthread_run_worker(0, "%s", engine->name); diff --git a/crypto/drbg.c b/crypto/drbg.c index 0a6f6c05a78f..21b339c76cca 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -1780,7 +1780,7 @@ static inline int __init drbg_healthcheck_sanity(void) if (!drbg) return -ENOMEM; - mutex_init(&drbg->drbg_mutex); + guard(mutex_init)(&drbg->drbg_mutex); drbg->core = &drbg_cores[coreref]; drbg->reseed_threshold = drbg_max_requests(drbg); From 41539433b32d71aea9f7ada84dc6a8bd014ca50d Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Mon, 19 Jan 2026 10:05:55 +0100 Subject: [PATCH 87/90] tomoyo: Use scoped init guard Convert lock initialization to scoped guarded initialization where lock-guarded members are initialized in the same scope. This ensures the context analysis treats the context as active during member initialization. This is required to avoid errors once implicit context assertion is removed. Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://patch.msgid.link/20260119094029.1344361-6-elver@google.com --- security/tomoyo/common.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/security/tomoyo/common.c b/security/tomoyo/common.c index 86ce56c32d37..7e1f825d903b 100644 --- a/security/tomoyo/common.c +++ b/security/tomoyo/common.c @@ -2557,7 +2557,7 @@ int tomoyo_open_control(const u8 type, struct file *file) if (!head) return -ENOMEM; - mutex_init(&head->io_sem); + guard(mutex_init)(&head->io_sem); head->type = type; switch (type) { case TOMOYO_DOMAINPOLICY: From b682b70d016f6aee20d91dcbaa319a932008a83a Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Mon, 19 Jan 2026 10:05:56 +0100 Subject: [PATCH 88/90] compiler-context-analysis: Remove __assume_ctx_lock from initializers Remove __assume_ctx_lock() from lock initializers. Implicitly asserting an active context during initialization caused false-positive double-lock errors when acquiring a lock immediately after its initialization. Moving forward, guarded member initialization must either: 1. Use guard(type_init)(&lock) or scoped_guard(type_init, ...). 2. Use context_unsafe() for simple initialization. Reported-by: Bart Van Assche Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/all/57062131-e79e-42c2-aa0b-8f931cb8cac2@acm.org/ Link: https://patch.msgid.link/20260119094029.1344361-7-elver@google.com --- include/linux/local_lock_internal.h | 3 --- include/linux/mutex.h | 1 - include/linux/rwlock.h | 3 +-- include/linux/rwlock_rt.h | 1 - include/linux/rwsem.h | 2 -- include/linux/seqlock.h | 1 - include/linux/spinlock.h | 5 +---- include/linux/spinlock_rt.h | 1 - include/linux/ww_mutex.h | 1 - lib/test_context-analysis.c | 6 ------ 10 files changed, 2 insertions(+), 22 deletions(-) diff --git a/include/linux/local_lock_internal.h b/include/linux/local_lock_internal.h index ed2f3fb4c360..eff711bf973f 100644 --- a/include/linux/local_lock_internal.h +++ b/include/linux/local_lock_internal.h @@ -87,13 +87,11 @@ do { \ 0, LD_WAIT_CONFIG, LD_WAIT_INV, \ LD_LOCK_PERCPU); \ local_lock_debug_init(lock); \ - __assume_ctx_lock(lock); \ } while (0) #define __local_trylock_init(lock) \ do { \ __local_lock_init((local_lock_t *)lock); \ - __assume_ctx_lock(lock); \ } while (0) #define __spinlock_nested_bh_init(lock) \ @@ -105,7 +103,6 @@ do { \ 0, LD_WAIT_CONFIG, LD_WAIT_INV, \ LD_LOCK_NORMAL); \ local_lock_debug_init(lock); \ - __assume_ctx_lock(lock); \ } while (0) #define __local_lock_acquire(lock) \ diff --git a/include/linux/mutex.h b/include/linux/mutex.h index 6b12009351d2..ecaa0440f6ec 100644 --- a/include/linux/mutex.h +++ b/include/linux/mutex.h @@ -62,7 +62,6 @@ do { \ static struct lock_class_key __key; \ \ __mutex_init((mutex), #mutex, &__key); \ - __assume_ctx_lock(mutex); \ } while (0) /** diff --git a/include/linux/rwlock.h b/include/linux/rwlock.h index 65a5b55e1bcd..3390d21c95dd 100644 --- a/include/linux/rwlock.h +++ b/include/linux/rwlock.h @@ -22,11 +22,10 @@ do { \ static struct lock_class_key __key; \ \ __rwlock_init((lock), #lock, &__key); \ - __assume_ctx_lock(lock); \ } while (0) #else # define rwlock_init(lock) \ - do { *(lock) = __RW_LOCK_UNLOCKED(lock); __assume_ctx_lock(lock); } while (0) + do { *(lock) = __RW_LOCK_UNLOCKED(lock); } while (0) #endif #ifdef CONFIG_DEBUG_SPINLOCK diff --git a/include/linux/rwlock_rt.h b/include/linux/rwlock_rt.h index 37b387dcab21..5353abbfdc0b 100644 --- a/include/linux/rwlock_rt.h +++ b/include/linux/rwlock_rt.h @@ -22,7 +22,6 @@ do { \ \ init_rwbase_rt(&(rwl)->rwbase); \ __rt_rwlock_init(rwl, #rwl, &__key); \ - __assume_ctx_lock(rwl); \ } while (0) extern void rt_read_lock(rwlock_t *rwlock) __acquires_shared(rwlock); diff --git a/include/linux/rwsem.h b/include/linux/rwsem.h index ea1bbdb57a47..9bf1d93d3d7b 100644 --- a/include/linux/rwsem.h +++ b/include/linux/rwsem.h @@ -121,7 +121,6 @@ do { \ static struct lock_class_key __key; \ \ __init_rwsem((sem), #sem, &__key); \ - __assume_ctx_lock(sem); \ } while (0) /* @@ -175,7 +174,6 @@ do { \ static struct lock_class_key __key; \ \ __init_rwsem((sem), #sem, &__key); \ - __assume_ctx_lock(sem); \ } while (0) static __always_inline int rwsem_is_locked(const struct rw_semaphore *sem) diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h index 077c8d5b2afd..5a40252b8334 100644 --- a/include/linux/seqlock.h +++ b/include/linux/seqlock.h @@ -817,7 +817,6 @@ static __always_inline void write_seqcount_latch_end(seqcount_latch_t *s) do { \ spin_lock_init(&(sl)->lock); \ seqcount_spinlock_init(&(sl)->seqcount, &(sl)->lock); \ - __assume_ctx_lock(sl); \ } while (0) /** diff --git a/include/linux/spinlock.h b/include/linux/spinlock.h index 7b11991c742a..e1e2f144af9b 100644 --- a/include/linux/spinlock.h +++ b/include/linux/spinlock.h @@ -106,12 +106,11 @@ do { \ static struct lock_class_key __key; \ \ __raw_spin_lock_init((lock), #lock, &__key, LD_WAIT_SPIN); \ - __assume_ctx_lock(lock); \ } while (0) #else # define raw_spin_lock_init(lock) \ - do { *(lock) = __RAW_SPIN_LOCK_UNLOCKED(lock); __assume_ctx_lock(lock); } while (0) + do { *(lock) = __RAW_SPIN_LOCK_UNLOCKED(lock); } while (0) #endif #define raw_spin_is_locked(lock) arch_spin_is_locked(&(lock)->raw_lock) @@ -324,7 +323,6 @@ do { \ \ __raw_spin_lock_init(spinlock_check(lock), \ #lock, &__key, LD_WAIT_CONFIG); \ - __assume_ctx_lock(lock); \ } while (0) #else @@ -333,7 +331,6 @@ do { \ do { \ spinlock_check(_lock); \ *(_lock) = __SPIN_LOCK_UNLOCKED(_lock); \ - __assume_ctx_lock(_lock); \ } while (0) #endif diff --git a/include/linux/spinlock_rt.h b/include/linux/spinlock_rt.h index 0a585768358f..373618a4243c 100644 --- a/include/linux/spinlock_rt.h +++ b/include/linux/spinlock_rt.h @@ -20,7 +20,6 @@ static inline void __rt_spin_lock_init(spinlock_t *lock, const char *name, do { \ rt_mutex_base_init(&(slock)->lock); \ __rt_spin_lock_init(slock, name, key, percpu); \ - __assume_ctx_lock(slock); \ } while (0) #define _spin_lock_init(slock, percpu) \ diff --git a/include/linux/ww_mutex.h b/include/linux/ww_mutex.h index 58e959ee10e9..c47d4b9b88b3 100644 --- a/include/linux/ww_mutex.h +++ b/include/linux/ww_mutex.h @@ -107,7 +107,6 @@ context_lock_struct(ww_acquire_ctx) { */ static inline void ww_mutex_init(struct ww_mutex *lock, struct ww_class *ww_class) - __assumes_ctx_lock(lock) { ww_mutex_base_init(&lock->base, ww_class->mutex_name, &ww_class->mutex_key); lock->ctx = NULL; diff --git a/lib/test_context-analysis.c b/lib/test_context-analysis.c index 0f05943d957f..140efa8a9763 100644 --- a/lib/test_context-analysis.c +++ b/lib/test_context-analysis.c @@ -542,12 +542,6 @@ struct test_ww_mutex_data { int counter __guarded_by(&mtx); }; -static void __used test_ww_mutex_init(struct test_ww_mutex_data *d) -{ - ww_mutex_init(&d->mtx, &ww_class); - d->counter = 0; -} - static void __used test_ww_mutex_lock_noctx(struct test_ww_mutex_data *d) { if (!ww_mutex_lock(&d->mtx, NULL)) { From 451c3a0356099483218535aed17fcf94ab5764a3 Mon Sep 17 00:00:00 2001 From: Arnd Bergmann Date: Mon, 2 Feb 2026 10:55:01 +0100 Subject: [PATCH 89/90] rcu: Mark lockdep_assert_rcu_helper() __always_inline There are some configurations in which lockdep_assert_rcu_helper() ends up not being inlined, for some reason. This leads to a link failure because now the caller tries to pass a nonexistant __ctx_lock_RCU structure: ld: lib/test_context-analysis.o: in function `test_rcu_assert_variants': test_context-analysis.c:(.text+0x275c): undefined reference to `RCU' ld: test_context-analysis.c:(.text+0x276c): undefined reference to `RCU_BH' ld: test_context-analysis.c:(.text+0x2774): undefined reference to `RCU_SCHED' I saw this in one out of many 32-bit arm builds using gcc-15.2, but it probably happens in others as well. Mark this function as __always_inline to fix the build. Fixes: fe00f6e84621 ("rcu: Support Clang's context analysis") Signed-off-by: Arnd Bergmann Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Paul E. McKenney Link: https://patch.msgid.link/20260202095507.1237440-1-arnd@kernel.org --- include/linux/rcupdate.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index d828a4673441..f0aa48bb5d12 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -435,7 +435,7 @@ static inline void rcu_preempt_sleep_check(void) { } // See RCU_LOCKDEP_WARN() for an explanation of the double call to // debug_lockdep_rcu_enabled(). -static inline bool lockdep_assert_rcu_helper(bool c, const struct __ctx_lock_RCU *ctx) +static __always_inline bool lockdep_assert_rcu_helper(bool c, const struct __ctx_lock_RCU *ctx) __assumes_shared_ctx_lock(RCU) __assumes_shared_ctx_lock(ctx) { return debug_lockdep_rcu_enabled() && From 7a562d5d2396c9c78fbbced7ae81bcfcfa0fde3f Mon Sep 17 00:00:00 2001 From: Marco Elver Date: Tue, 3 Feb 2026 23:50:10 +0100 Subject: [PATCH 90/90] locking/rwlock: Fix write_trylock_irqsave() with CONFIG_INLINE_WRITE_TRYLOCK Move _raw_write_trylock_irqsave() after the _raw_write_trylock macro to ensure it uses the inlined version, fixing a linker error when inlining is enabled. This is the case on s390: >> ld.lld: error: undefined symbol: _raw_write_trylock >>> referenced by rwlock_api_smp.h:48 (include/linux/rwlock_api_smp.h:48) >>> lib/test_context-analysis.o:(test_write_trylock_extra) in archive vmlinux.a >>> referenced by rwlock_api_smp.h:48 (include/linux/rwlock_api_smp.h:48) >>> lib/test_context-analysis.o:(test_write_trylock_extra) in archive vmlinux.a Closes: https://lore.kernel.org/oe-kbuild-all/202602032101.dbxRfsWO-lkp@intel.com/ Reported-by: kernel test robot Signed-off-by: Marco Elver Signed-off-by: Peter Zijlstra (Intel) Reviewed-by: Bart Van Assche Link: https://patch.msgid.link/20260203225114.3493538-1-elver@google.com --- include/linux/rwlock_api_smp.h | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/include/linux/rwlock_api_smp.h b/include/linux/rwlock_api_smp.h index d903b17c46ca..61a852609eab 100644 --- a/include/linux/rwlock_api_smp.h +++ b/include/linux/rwlock_api_smp.h @@ -41,16 +41,6 @@ void __lockfunc _raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) __releases(lock); -static inline bool _raw_write_trylock_irqsave(rwlock_t *lock, unsigned long *flags) - __cond_acquires(true, lock) -{ - local_irq_save(*flags); - if (_raw_write_trylock(lock)) - return true; - local_irq_restore(*flags); - return false; -} - #ifdef CONFIG_INLINE_READ_LOCK #define _raw_read_lock(lock) __raw_read_lock(lock) #endif @@ -147,6 +137,16 @@ static inline int __raw_write_trylock(rwlock_t *lock) return 0; } +static inline bool _raw_write_trylock_irqsave(rwlock_t *lock, unsigned long *flags) + __cond_acquires(true, lock) __no_context_analysis +{ + local_irq_save(*flags); + if (_raw_write_trylock(lock)) + return true; + local_irq_restore(*flags); + return false; +} + /* * If lockdep is enabled then we use the non-preemption spin-ops * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are