2
0
mirror of git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git synced 2025-09-04 20:19:47 +08:00

17 hotfixes. 13 are cc:stable and the remainder address post-6.16 issues

or aren't considered necessary for -stable kernels.  11 of these fixes are
 for MM.
 
 This includes a three-patch series from Harry Yoo which fixes an
 intermittent boot failure which can occur on x86 systems.  And a two-patch
 series from Alexander Gordeev which fixes a KASAN crash on S390 systems.
 -----BEGIN PGP SIGNATURE-----
 
 iHUEABYKAB0WIQTTMBEPP41GrTpTJgfdBJ7gKXxAjgUCaLY4WAAKCRDdBJ7gKXxA
 jp2qAP92JCCzscR87um+YSc4u6a/X6ucYWkzh9BGhM8bMT8p7wD/UhIuGbYRFLPw
 XbSDkAD6lKpujQkRAudRFQTcZcU7gwg=
 =mPUd
 -----END PGP SIGNATURE-----

Merge tag 'mm-hotfixes-stable-2025-09-01-17-20' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm

Pull misc fixes from Andrew Morton:
 "17 hotfixes. 13 are cc:stable and the remainder address post-6.16
  issues or aren't considered necessary for -stable kernels. 11 of these
  fixes are for MM.

  This includes a three-patch series from Harry Yoo which fixes an
  intermittent boot failure which can occur on x86 systems. And a
  two-patch series from Alexander Gordeev which fixes a KASAN crash on
  S390 systems"

* tag 'mm-hotfixes-stable-2025-09-01-17-20' of git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm:
  mm: fix possible deadlock in kmemleak
  x86/mm/64: define ARCH_PAGE_TABLE_SYNC_MASK and arch_sync_kernel_mappings()
  mm: introduce and use {pgd,p4d}_populate_kernel()
  mm: move page table sync declarations to linux/pgtable.h
  proc: fix missing pde_set_flags() for net proc files
  mm: fix accounting of memmap pages
  mm/damon/core: prevent unnecessary overflow in damos_set_effective_quota()
  kexec: add KEXEC_FILE_NO_CMA as a legal flag
  kasan: fix GCC mem-intrinsic prefix with sw tags
  mm/kasan: avoid lazy MMU mode hazards
  mm/kasan: fix vmalloc shadow memory (de-)population races
  kunit: kasan_test: disable fortify string checker on kasan_strings() test
  selftests/mm: fix FORCE_READ to read input value correctly
  mm/userfaultfd: fix kmap_local LIFO ordering for CONFIG_HIGHPTE
  ocfs2: prevent release journal inode after journal shutdown
  rust: mm: mark VmaNew as transparent
  of_numa: fix uninitialized memory nodes causing kernel panic
This commit is contained in:
Linus Torvalds 2025-09-02 13:18:00 -07:00
commit 8026aed072
27 changed files with 190 additions and 94 deletions

View File

@ -36,6 +36,9 @@ static inline bool pgtable_l5_enabled(void)
#define pgtable_l5_enabled() cpu_feature_enabled(X86_FEATURE_LA57) #define pgtable_l5_enabled() cpu_feature_enabled(X86_FEATURE_LA57)
#endif /* USE_EARLY_PGTABLE_L5 */ #endif /* USE_EARLY_PGTABLE_L5 */
#define ARCH_PAGE_TABLE_SYNC_MASK \
(pgtable_l5_enabled() ? PGTBL_PGD_MODIFIED : PGTBL_P4D_MODIFIED)
extern unsigned int pgdir_shift; extern unsigned int pgdir_shift;
extern unsigned int ptrs_per_p4d; extern unsigned int ptrs_per_p4d;

View File

@ -223,6 +223,24 @@ static void sync_global_pgds(unsigned long start, unsigned long end)
sync_global_pgds_l4(start, end); sync_global_pgds_l4(start, end);
} }
/*
* Make kernel mappings visible in all page tables in the system.
* This is necessary except when the init task populates kernel mappings
* during the boot process. In that case, all processes originating from
* the init task copies the kernel mappings, so there is no issue.
* Otherwise, missing synchronization could lead to kernel crashes due
* to missing page table entries for certain kernel mappings.
*
* Synchronization is performed at the top level, which is the PGD in
* 5-level paging systems. But in 4-level paging systems, however,
* pgd_populate() is a no-op, so synchronization is done at the P4D level.
* sync_global_pgds() handles this difference between paging levels.
*/
void arch_sync_kernel_mappings(unsigned long start, unsigned long end)
{
sync_global_pgds(start, end);
}
/* /*
* NOTE: This function is marked __ref because it calls __init function * NOTE: This function is marked __ref because it calls __init function
* (alloc_bootmem_pages). It's safe to do it ONLY when after_bootmem == 0. * (alloc_bootmem_pages). It's safe to do it ONLY when after_bootmem == 0.

View File

@ -59,8 +59,11 @@ static int __init of_numa_parse_memory_nodes(void)
r = -EINVAL; r = -EINVAL;
} }
for (i = 0; !r && !of_address_to_resource(np, i, &rsrc); i++) for (i = 0; !r && !of_address_to_resource(np, i, &rsrc); i++) {
r = numa_add_memblk(nid, rsrc.start, rsrc.end + 1); r = numa_add_memblk(nid, rsrc.start, rsrc.end + 1);
if (!r)
node_set(nid, numa_nodes_parsed);
}
if (!i || r) { if (!i || r) {
of_node_put(np); of_node_put(np);

View File

@ -1281,6 +1281,9 @@ static void ocfs2_clear_inode(struct inode *inode)
* the journal is flushed before journal shutdown. Thus it is safe to * the journal is flushed before journal shutdown. Thus it is safe to
* have inodes get cleaned up after journal shutdown. * have inodes get cleaned up after journal shutdown.
*/ */
if (!osb->journal)
return;
jbd2_journal_release_jbd_inode(osb->journal->j_journal, jbd2_journal_release_jbd_inode(osb->journal->j_journal,
&oi->ip_jinode); &oi->ip_jinode);
} }

View File

@ -367,6 +367,25 @@ static const struct inode_operations proc_dir_inode_operations = {
.setattr = proc_notify_change, .setattr = proc_notify_change,
}; };
static void pde_set_flags(struct proc_dir_entry *pde)
{
const struct proc_ops *proc_ops = pde->proc_ops;
if (!proc_ops)
return;
if (proc_ops->proc_flags & PROC_ENTRY_PERMANENT)
pde->flags |= PROC_ENTRY_PERMANENT;
if (proc_ops->proc_read_iter)
pde->flags |= PROC_ENTRY_proc_read_iter;
#ifdef CONFIG_COMPAT
if (proc_ops->proc_compat_ioctl)
pde->flags |= PROC_ENTRY_proc_compat_ioctl;
#endif
if (proc_ops->proc_lseek)
pde->flags |= PROC_ENTRY_proc_lseek;
}
/* returns the registered entry, or frees dp and returns NULL on failure */ /* returns the registered entry, or frees dp and returns NULL on failure */
struct proc_dir_entry *proc_register(struct proc_dir_entry *dir, struct proc_dir_entry *proc_register(struct proc_dir_entry *dir,
struct proc_dir_entry *dp) struct proc_dir_entry *dp)
@ -374,6 +393,8 @@ struct proc_dir_entry *proc_register(struct proc_dir_entry *dir,
if (proc_alloc_inum(&dp->low_ino)) if (proc_alloc_inum(&dp->low_ino))
goto out_free_entry; goto out_free_entry;
pde_set_flags(dp);
write_lock(&proc_subdir_lock); write_lock(&proc_subdir_lock);
dp->parent = dir; dp->parent = dir;
if (pde_subdir_insert(dir, dp) == false) { if (pde_subdir_insert(dir, dp) == false) {
@ -561,20 +582,6 @@ struct proc_dir_entry *proc_create_reg(const char *name, umode_t mode,
return p; return p;
} }
static void pde_set_flags(struct proc_dir_entry *pde)
{
if (pde->proc_ops->proc_flags & PROC_ENTRY_PERMANENT)
pde->flags |= PROC_ENTRY_PERMANENT;
if (pde->proc_ops->proc_read_iter)
pde->flags |= PROC_ENTRY_proc_read_iter;
#ifdef CONFIG_COMPAT
if (pde->proc_ops->proc_compat_ioctl)
pde->flags |= PROC_ENTRY_proc_compat_ioctl;
#endif
if (pde->proc_ops->proc_lseek)
pde->flags |= PROC_ENTRY_proc_lseek;
}
struct proc_dir_entry *proc_create_data(const char *name, umode_t mode, struct proc_dir_entry *proc_create_data(const char *name, umode_t mode,
struct proc_dir_entry *parent, struct proc_dir_entry *parent,
const struct proc_ops *proc_ops, void *data) const struct proc_ops *proc_ops, void *data)
@ -585,7 +592,6 @@ struct proc_dir_entry *proc_create_data(const char *name, umode_t mode,
if (!p) if (!p)
return NULL; return NULL;
p->proc_ops = proc_ops; p->proc_ops = proc_ops;
pde_set_flags(p);
return proc_register(parent, p); return proc_register(parent, p);
} }
EXPORT_SYMBOL(proc_create_data); EXPORT_SYMBOL(proc_create_data);
@ -636,7 +642,6 @@ struct proc_dir_entry *proc_create_seq_private(const char *name, umode_t mode,
p->proc_ops = &proc_seq_ops; p->proc_ops = &proc_seq_ops;
p->seq_ops = ops; p->seq_ops = ops;
p->state_size = state_size; p->state_size = state_size;
pde_set_flags(p);
return proc_register(parent, p); return proc_register(parent, p);
} }
EXPORT_SYMBOL(proc_create_seq_private); EXPORT_SYMBOL(proc_create_seq_private);
@ -667,7 +672,6 @@ struct proc_dir_entry *proc_create_single_data(const char *name, umode_t mode,
return NULL; return NULL;
p->proc_ops = &proc_single_ops; p->proc_ops = &proc_single_ops;
p->single_show = show; p->single_show = show;
pde_set_flags(p);
return proc_register(parent, p); return proc_register(parent, p);
} }
EXPORT_SYMBOL(proc_create_single_data); EXPORT_SYMBOL(proc_create_single_data);

View File

@ -460,7 +460,8 @@ bool kexec_load_permitted(int kexec_image_type);
/* List of defined/legal kexec file flags */ /* List of defined/legal kexec file flags */
#define KEXEC_FILE_FLAGS (KEXEC_FILE_UNLOAD | KEXEC_FILE_ON_CRASH | \ #define KEXEC_FILE_FLAGS (KEXEC_FILE_UNLOAD | KEXEC_FILE_ON_CRASH | \
KEXEC_FILE_NO_INITRAMFS | KEXEC_FILE_DEBUG) KEXEC_FILE_NO_INITRAMFS | KEXEC_FILE_DEBUG | \
KEXEC_FILE_NO_CMA)
/* flag to track if kexec reboot is in progress */ /* flag to track if kexec reboot is in progress */
extern bool kexec_in_progress; extern bool kexec_in_progress;

29
include/linux/pgalloc.h Normal file
View File

@ -0,0 +1,29 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef _LINUX_PGALLOC_H
#define _LINUX_PGALLOC_H
#include <linux/pgtable.h>
#include <asm/pgalloc.h>
/*
* {pgd,p4d}_populate_kernel() are defined as macros to allow
* compile-time optimization based on the configured page table levels.
* Without this, linking may fail because callers (e.g., KASAN) may rely
* on calls to these functions being optimized away when passing symbols
* that exist only for certain page table levels.
*/
#define pgd_populate_kernel(addr, pgd, p4d) \
do { \
pgd_populate(&init_mm, pgd, p4d); \
if (ARCH_PAGE_TABLE_SYNC_MASK & PGTBL_PGD_MODIFIED) \
arch_sync_kernel_mappings(addr, addr); \
} while (0)
#define p4d_populate_kernel(addr, p4d, pud) \
do { \
p4d_populate(&init_mm, p4d, pud); \
if (ARCH_PAGE_TABLE_SYNC_MASK & PGTBL_P4D_MODIFIED) \
arch_sync_kernel_mappings(addr, addr); \
} while (0)
#endif /* _LINUX_PGALLOC_H */

View File

@ -1467,6 +1467,22 @@ static inline void modify_prot_commit_ptes(struct vm_area_struct *vma, unsigned
} }
#endif #endif
/*
* Architectures can set this mask to a combination of PGTBL_P?D_MODIFIED values
* and let generic vmalloc, ioremap and page table update code know when
* arch_sync_kernel_mappings() needs to be called.
*/
#ifndef ARCH_PAGE_TABLE_SYNC_MASK
#define ARCH_PAGE_TABLE_SYNC_MASK 0
#endif
/*
* There is no default implementation for arch_sync_kernel_mappings(). It is
* relied upon the compiler to optimize calls out if ARCH_PAGE_TABLE_SYNC_MASK
* is 0.
*/
void arch_sync_kernel_mappings(unsigned long start, unsigned long end);
#endif /* CONFIG_MMU */ #endif /* CONFIG_MMU */
/* /*
@ -1938,10 +1954,11 @@ static inline bool arch_has_pfn_modify_check(void)
/* /*
* Page Table Modification bits for pgtbl_mod_mask. * Page Table Modification bits for pgtbl_mod_mask.
* *
* These are used by the p?d_alloc_track*() set of functions an in the generic * These are used by the p?d_alloc_track*() and p*d_populate_kernel()
* vmalloc/ioremap code to track at which page-table levels entries have been * functions in the generic vmalloc, ioremap and page table update code
* modified. Based on that the code can better decide when vmalloc and ioremap * to track at which page-table levels entries have been modified.
* mapping changes need to be synchronized to other page-tables in the system. * Based on that the code can better decide when page table changes need
* to be synchronized to other page-tables in the system.
*/ */
#define __PGTBL_PGD_MODIFIED 0 #define __PGTBL_PGD_MODIFIED 0
#define __PGTBL_P4D_MODIFIED 1 #define __PGTBL_P4D_MODIFIED 1

View File

@ -219,22 +219,6 @@ extern int remap_vmalloc_range(struct vm_area_struct *vma, void *addr,
int vmap_pages_range(unsigned long addr, unsigned long end, pgprot_t prot, int vmap_pages_range(unsigned long addr, unsigned long end, pgprot_t prot,
struct page **pages, unsigned int page_shift); struct page **pages, unsigned int page_shift);
/*
* Architectures can set this mask to a combination of PGTBL_P?D_MODIFIED values
* and let generic vmalloc and ioremap code know when arch_sync_kernel_mappings()
* needs to be called.
*/
#ifndef ARCH_PAGE_TABLE_SYNC_MASK
#define ARCH_PAGE_TABLE_SYNC_MASK 0
#endif
/*
* There is no default implementation for arch_sync_kernel_mappings(). It is
* relied upon the compiler to optimize calls out if ARCH_PAGE_TABLE_SYNC_MASK
* is 0.
*/
void arch_sync_kernel_mappings(unsigned long start, unsigned long end);
/* /*
* Lowlevel-APIs (not for driver use!) * Lowlevel-APIs (not for driver use!)
*/ */

View File

@ -2073,8 +2073,8 @@ static void damos_set_effective_quota(struct damos_quota *quota)
if (quota->ms) { if (quota->ms) {
if (quota->total_charged_ns) if (quota->total_charged_ns)
throughput = quota->total_charged_sz * 1000000 / throughput = mult_frac(quota->total_charged_sz, 1000000,
quota->total_charged_ns; quota->total_charged_ns);
else else
throughput = PAGE_SIZE * 1024; throughput = PAGE_SIZE * 1024;
esz = min(throughput * quota->ms, esz); esz = min(throughput * quota->ms, esz);

View File

@ -13,9 +13,9 @@
#include <linux/mm.h> #include <linux/mm.h>
#include <linux/pfn.h> #include <linux/pfn.h>
#include <linux/slab.h> #include <linux/slab.h>
#include <linux/pgalloc.h>
#include <asm/page.h> #include <asm/page.h>
#include <asm/pgalloc.h>
#include "kasan.h" #include "kasan.h"
@ -191,7 +191,7 @@ static int __ref zero_p4d_populate(pgd_t *pgd, unsigned long addr,
pud_t *pud; pud_t *pud;
pmd_t *pmd; pmd_t *pmd;
p4d_populate(&init_mm, p4d, p4d_populate_kernel(addr, p4d,
lm_alias(kasan_early_shadow_pud)); lm_alias(kasan_early_shadow_pud));
pud = pud_offset(p4d, addr); pud = pud_offset(p4d, addr);
pud_populate(&init_mm, pud, pud_populate(&init_mm, pud,
@ -212,7 +212,7 @@ static int __ref zero_p4d_populate(pgd_t *pgd, unsigned long addr,
} else { } else {
p = early_alloc(PAGE_SIZE, NUMA_NO_NODE); p = early_alloc(PAGE_SIZE, NUMA_NO_NODE);
pud_init(p); pud_init(p);
p4d_populate(&init_mm, p4d, p); p4d_populate_kernel(addr, p4d, p);
} }
} }
zero_pud_populate(p4d, addr, next); zero_pud_populate(p4d, addr, next);
@ -251,10 +251,10 @@ int __ref kasan_populate_early_shadow(const void *shadow_start,
* puds,pmds, so pgd_populate(), pud_populate() * puds,pmds, so pgd_populate(), pud_populate()
* is noops. * is noops.
*/ */
pgd_populate(&init_mm, pgd, pgd_populate_kernel(addr, pgd,
lm_alias(kasan_early_shadow_p4d)); lm_alias(kasan_early_shadow_p4d));
p4d = p4d_offset(pgd, addr); p4d = p4d_offset(pgd, addr);
p4d_populate(&init_mm, p4d, p4d_populate_kernel(addr, p4d,
lm_alias(kasan_early_shadow_pud)); lm_alias(kasan_early_shadow_pud));
pud = pud_offset(p4d, addr); pud = pud_offset(p4d, addr);
pud_populate(&init_mm, pud, pud_populate(&init_mm, pud,
@ -273,7 +273,7 @@ int __ref kasan_populate_early_shadow(const void *shadow_start,
if (!p) if (!p)
return -ENOMEM; return -ENOMEM;
} else { } else {
pgd_populate(&init_mm, pgd, pgd_populate_kernel(addr, pgd,
early_alloc(PAGE_SIZE, NUMA_NO_NODE)); early_alloc(PAGE_SIZE, NUMA_NO_NODE));
} }
} }

View File

@ -1578,9 +1578,11 @@ static void kasan_strings(struct kunit *test)
ptr = kmalloc(size, GFP_KERNEL | __GFP_ZERO); ptr = kmalloc(size, GFP_KERNEL | __GFP_ZERO);
KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr); KUNIT_ASSERT_NOT_ERR_OR_NULL(test, ptr);
OPTIMIZER_HIDE_VAR(ptr);
src = kmalloc(KASAN_GRANULE_SIZE, GFP_KERNEL | __GFP_ZERO); src = kmalloc(KASAN_GRANULE_SIZE, GFP_KERNEL | __GFP_ZERO);
strscpy(src, "f0cacc1a0000000", KASAN_GRANULE_SIZE); strscpy(src, "f0cacc1a0000000", KASAN_GRANULE_SIZE);
OPTIMIZER_HIDE_VAR(src);
/* /*
* Make sure that strscpy() does not trigger KASAN if it overreads into * Make sure that strscpy() does not trigger KASAN if it overreads into

View File

@ -305,8 +305,7 @@ static int kasan_populate_vmalloc_pte(pte_t *ptep, unsigned long addr,
pte_t pte; pte_t pte;
int index; int index;
if (likely(!pte_none(ptep_get(ptep)))) arch_leave_lazy_mmu_mode();
return 0;
index = PFN_DOWN(addr - data->start); index = PFN_DOWN(addr - data->start);
page = data->pages[index]; page = data->pages[index];
@ -320,6 +319,8 @@ static int kasan_populate_vmalloc_pte(pte_t *ptep, unsigned long addr,
} }
spin_unlock(&init_mm.page_table_lock); spin_unlock(&init_mm.page_table_lock);
arch_enter_lazy_mmu_mode();
return 0; return 0;
} }
@ -461,18 +462,23 @@ int kasan_populate_vmalloc(unsigned long addr, unsigned long size)
static int kasan_depopulate_vmalloc_pte(pte_t *ptep, unsigned long addr, static int kasan_depopulate_vmalloc_pte(pte_t *ptep, unsigned long addr,
void *unused) void *unused)
{ {
unsigned long page; pte_t pte;
int none;
page = (unsigned long)__va(pte_pfn(ptep_get(ptep)) << PAGE_SHIFT); arch_leave_lazy_mmu_mode();
spin_lock(&init_mm.page_table_lock); spin_lock(&init_mm.page_table_lock);
pte = ptep_get(ptep);
if (likely(!pte_none(ptep_get(ptep)))) { none = pte_none(pte);
if (likely(!none))
pte_clear(&init_mm, addr, ptep); pte_clear(&init_mm, addr, ptep);
free_page(page);
}
spin_unlock(&init_mm.page_table_lock); spin_unlock(&init_mm.page_table_lock);
if (likely(!none))
__free_page(pfn_to_page(pte_pfn(pte)));
arch_enter_lazy_mmu_mode();
return 0; return 0;
} }

View File

@ -437,9 +437,15 @@ static struct kmemleak_object *__lookup_object(unsigned long ptr, int alias,
else if (untagged_objp == untagged_ptr || alias) else if (untagged_objp == untagged_ptr || alias)
return object; return object;
else { else {
/*
* Printk deferring due to the kmemleak_lock held.
* This is done to avoid deadlock.
*/
printk_deferred_enter();
kmemleak_warn("Found object by alias at 0x%08lx\n", kmemleak_warn("Found object by alias at 0x%08lx\n",
ptr); ptr);
dump_object_info(object); dump_object_info(object);
printk_deferred_exit();
break; break;
} }
} }
@ -736,6 +742,11 @@ static int __link_object(struct kmemleak_object *object, unsigned long ptr,
else if (untagged_objp + parent->size <= untagged_ptr) else if (untagged_objp + parent->size <= untagged_ptr)
link = &parent->rb_node.rb_right; link = &parent->rb_node.rb_right;
else { else {
/*
* Printk deferring due to the kmemleak_lock held.
* This is done to avoid deadlock.
*/
printk_deferred_enter();
kmemleak_stop("Cannot insert 0x%lx into the object search tree (overlaps existing)\n", kmemleak_stop("Cannot insert 0x%lx into the object search tree (overlaps existing)\n",
ptr); ptr);
/* /*
@ -743,6 +754,7 @@ static int __link_object(struct kmemleak_object *object, unsigned long ptr,
* be freed while the kmemleak_lock is held. * be freed while the kmemleak_lock is held.
*/ */
dump_object_info(parent); dump_object_info(parent);
printk_deferred_exit();
return -EEXIST; return -EEXIST;
} }
} }
@ -856,13 +868,8 @@ static void delete_object_part(unsigned long ptr, size_t size,
raw_spin_lock_irqsave(&kmemleak_lock, flags); raw_spin_lock_irqsave(&kmemleak_lock, flags);
object = __find_and_remove_object(ptr, 1, objflags); object = __find_and_remove_object(ptr, 1, objflags);
if (!object) { if (!object)
#ifdef DEBUG
kmemleak_warn("Partially freeing unknown object at 0x%08lx (size %zu)\n",
ptr, size);
#endif
goto unlock; goto unlock;
}
/* /*
* Create one or two objects that may result from the memory block * Create one or two objects that may result from the memory block
@ -882,8 +889,14 @@ static void delete_object_part(unsigned long ptr, size_t size,
unlock: unlock:
raw_spin_unlock_irqrestore(&kmemleak_lock, flags); raw_spin_unlock_irqrestore(&kmemleak_lock, flags);
if (object) if (object) {
__delete_object(object); __delete_object(object);
} else {
#ifdef DEBUG
kmemleak_warn("Partially freeing unknown object at 0x%08lx (size %zu)\n",
ptr, size);
#endif
}
out: out:
if (object_l) if (object_l)

View File

@ -3108,7 +3108,7 @@ out_free:
#endif /* BUILD_EMBED_FIRST_CHUNK */ #endif /* BUILD_EMBED_FIRST_CHUNK */
#ifdef BUILD_PAGE_FIRST_CHUNK #ifdef BUILD_PAGE_FIRST_CHUNK
#include <asm/pgalloc.h> #include <linux/pgalloc.h>
#ifndef P4D_TABLE_SIZE #ifndef P4D_TABLE_SIZE
#define P4D_TABLE_SIZE PAGE_SIZE #define P4D_TABLE_SIZE PAGE_SIZE
@ -3134,13 +3134,13 @@ void __init __weak pcpu_populate_pte(unsigned long addr)
if (pgd_none(*pgd)) { if (pgd_none(*pgd)) {
p4d = memblock_alloc_or_panic(P4D_TABLE_SIZE, P4D_TABLE_SIZE); p4d = memblock_alloc_or_panic(P4D_TABLE_SIZE, P4D_TABLE_SIZE);
pgd_populate(&init_mm, pgd, p4d); pgd_populate_kernel(addr, pgd, p4d);
} }
p4d = p4d_offset(pgd, addr); p4d = p4d_offset(pgd, addr);
if (p4d_none(*p4d)) { if (p4d_none(*p4d)) {
pud = memblock_alloc_or_panic(PUD_TABLE_SIZE, PUD_TABLE_SIZE); pud = memblock_alloc_or_panic(PUD_TABLE_SIZE, PUD_TABLE_SIZE);
p4d_populate(&init_mm, p4d, pud); p4d_populate_kernel(addr, p4d, pud);
} }
pud = pud_offset(p4d, addr); pud = pud_offset(p4d, addr);

View File

@ -27,9 +27,9 @@
#include <linux/spinlock.h> #include <linux/spinlock.h>
#include <linux/vmalloc.h> #include <linux/vmalloc.h>
#include <linux/sched.h> #include <linux/sched.h>
#include <linux/pgalloc.h>
#include <asm/dma.h> #include <asm/dma.h>
#include <asm/pgalloc.h>
#include <asm/tlbflush.h> #include <asm/tlbflush.h>
#include "hugetlb_vmemmap.h" #include "hugetlb_vmemmap.h"
@ -229,7 +229,7 @@ p4d_t * __meminit vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node)
if (!p) if (!p)
return NULL; return NULL;
pud_init(p); pud_init(p);
p4d_populate(&init_mm, p4d, p); p4d_populate_kernel(addr, p4d, p);
} }
return p4d; return p4d;
} }
@ -241,7 +241,7 @@ pgd_t * __meminit vmemmap_pgd_populate(unsigned long addr, int node)
void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node); void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
if (!p) if (!p)
return NULL; return NULL;
pgd_populate(&init_mm, pgd, p); pgd_populate_kernel(addr, pgd, p);
} }
return pgd; return pgd;
} }
@ -578,11 +578,6 @@ struct page * __meminit __populate_section_memmap(unsigned long pfn,
if (r < 0) if (r < 0)
return NULL; return NULL;
if (system_state == SYSTEM_BOOTING)
memmap_boot_pages_add(DIV_ROUND_UP(end - start, PAGE_SIZE));
else
memmap_pages_add(DIV_ROUND_UP(end - start, PAGE_SIZE));
return pfn_to_page(pfn); return pfn_to_page(pfn);
} }

View File

@ -454,9 +454,6 @@ static void __init sparse_buffer_init(unsigned long size, int nid)
*/ */
sparsemap_buf = memmap_alloc(size, section_map_size(), addr, nid, true); sparsemap_buf = memmap_alloc(size, section_map_size(), addr, nid, true);
sparsemap_buf_end = sparsemap_buf + size; sparsemap_buf_end = sparsemap_buf + size;
#ifndef CONFIG_SPARSEMEM_VMEMMAP
memmap_boot_pages_add(DIV_ROUND_UP(size, PAGE_SIZE));
#endif
} }
static void __init sparse_buffer_fini(void) static void __init sparse_buffer_fini(void)
@ -567,6 +564,8 @@ static void __init sparse_init_nid(int nid, unsigned long pnum_begin,
sparse_buffer_fini(); sparse_buffer_fini();
goto failed; goto failed;
} }
memmap_boot_pages_add(DIV_ROUND_UP(PAGES_PER_SECTION * sizeof(struct page),
PAGE_SIZE));
sparse_init_early_section(nid, map, pnum, 0); sparse_init_early_section(nid, map, pnum, 0);
} }
} }
@ -680,7 +679,6 @@ static void depopulate_section_memmap(unsigned long pfn, unsigned long nr_pages,
unsigned long start = (unsigned long) pfn_to_page(pfn); unsigned long start = (unsigned long) pfn_to_page(pfn);
unsigned long end = start + nr_pages * sizeof(struct page); unsigned long end = start + nr_pages * sizeof(struct page);
memmap_pages_add(-1L * (DIV_ROUND_UP(end - start, PAGE_SIZE)));
vmemmap_free(start, end, altmap); vmemmap_free(start, end, altmap);
} }
static void free_map_bootmem(struct page *memmap) static void free_map_bootmem(struct page *memmap)
@ -856,10 +854,14 @@ static void section_deactivate(unsigned long pfn, unsigned long nr_pages,
* The memmap of early sections is always fully populated. See * The memmap of early sections is always fully populated. See
* section_activate() and pfn_valid() . * section_activate() and pfn_valid() .
*/ */
if (!section_is_early) if (!section_is_early) {
memmap_pages_add(-1L * (DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE)));
depopulate_section_memmap(pfn, nr_pages, altmap); depopulate_section_memmap(pfn, nr_pages, altmap);
else if (memmap) } else if (memmap) {
memmap_boot_pages_add(-1L * (DIV_ROUND_UP(nr_pages * sizeof(struct page),
PAGE_SIZE)));
free_map_bootmem(memmap); free_map_bootmem(memmap);
}
if (empty) if (empty)
ms->section_mem_map = (unsigned long)NULL; ms->section_mem_map = (unsigned long)NULL;
@ -904,6 +906,7 @@ static struct page * __meminit section_activate(int nid, unsigned long pfn,
section_deactivate(pfn, nr_pages, altmap); section_deactivate(pfn, nr_pages, altmap);
return ERR_PTR(-ENOMEM); return ERR_PTR(-ENOMEM);
} }
memmap_pages_add(DIV_ROUND_UP(nr_pages * sizeof(struct page), PAGE_SIZE));
return memmap; return memmap;
} }

View File

@ -1453,10 +1453,15 @@ out:
folio_unlock(src_folio); folio_unlock(src_folio);
folio_put(src_folio); folio_put(src_folio);
} }
if (dst_pte) /*
pte_unmap(dst_pte); * Unmap in reverse order (LIFO) to maintain proper kmap_local
* index ordering when CONFIG_HIGHPTE is enabled. We mapped dst_pte
* first, then src_pte, so we must unmap src_pte first, then dst_pte.
*/
if (src_pte) if (src_pte)
pte_unmap(src_pte); pte_unmap(src_pte);
if (dst_pte)
pte_unmap(dst_pte);
mmu_notifier_invalidate_range_end(&range); mmu_notifier_invalidate_range_end(&range);
if (si) if (si)
put_swap_device(si); put_swap_device(si);

View File

@ -209,6 +209,7 @@ impl VmaMixedMap {
/// ///
/// For the duration of 'a, the referenced vma must be undergoing initialization in an /// For the duration of 'a, the referenced vma must be undergoing initialization in an
/// `f_ops->mmap()` hook. /// `f_ops->mmap()` hook.
#[repr(transparent)]
pub struct VmaNew { pub struct VmaNew {
vma: VmaRef, vma: VmaRef,
} }

View File

@ -86,10 +86,14 @@ kasan_params += hwasan-instrument-stack=$(stack_enable) \
hwasan-use-short-granules=0 \ hwasan-use-short-granules=0 \
hwasan-inline-all-checks=0 hwasan-inline-all-checks=0
# Instrument memcpy/memset/memmove calls by using instrumented __hwasan_mem*(). # Instrument memcpy/memset/memmove calls by using instrumented __(hw)asan_mem*().
ifeq ($(call clang-min-version, 150000)$(call gcc-min-version, 130000),y) ifdef CONFIG_CC_HAS_KASAN_MEMINTRINSIC_PREFIX
kasan_params += hwasan-kernel-mem-intrinsic-prefix=1 ifdef CONFIG_CC_IS_GCC
endif kasan_params += asan-kernel-mem-intrinsic-prefix=1
else
kasan_params += hwasan-kernel-mem-intrinsic-prefix=1
endif
endif # CONFIG_CC_HAS_KASAN_MEMINTRINSIC_PREFIX
endif # CONFIG_KASAN_SW_TAGS endif # CONFIG_KASAN_SW_TAGS

View File

@ -1554,8 +1554,8 @@ static void run_with_zeropage(non_anon_test_fn fn, const char *desc)
} }
/* Read from the page to populate the shared zeropage. */ /* Read from the page to populate the shared zeropage. */
FORCE_READ(mem); FORCE_READ(*mem);
FORCE_READ(smem); FORCE_READ(*smem);
fn(mem, smem, pagesize); fn(mem, smem, pagesize);
munmap: munmap:

View File

@ -145,7 +145,7 @@ static bool try_access_buf(char *ptr, bool write)
if (write) if (write)
*ptr = 'x'; *ptr = 'x';
else else
FORCE_READ(ptr); FORCE_READ(*ptr);
} }
signal_jump_set = false; signal_jump_set = false;

View File

@ -50,8 +50,10 @@ void read_fault_pages(void *addr, unsigned long nr_pages)
unsigned long i; unsigned long i;
for (i = 0; i < nr_pages; i++) { for (i = 0; i < nr_pages; i++) {
unsigned long *addr2 =
((unsigned long *)(addr + (i * huge_page_size)));
/* Prevent the compiler from optimizing out the entire loop: */ /* Prevent the compiler from optimizing out the entire loop: */
FORCE_READ(((unsigned long *)(addr + (i * huge_page_size)))); FORCE_READ(*addr2);
} }
} }

View File

@ -110,7 +110,7 @@ void *access_mem(void *ptr)
* the memory access actually happens and prevents the compiler * the memory access actually happens and prevents the compiler
* from optimizing away this entire loop. * from optimizing away this entire loop.
*/ */
FORCE_READ((uint64_t *)ptr); FORCE_READ(*(uint64_t *)ptr);
} }
return NULL; return NULL;

View File

@ -1525,7 +1525,7 @@ void zeropfn_tests(void)
ret = madvise(mem, hpage_size, MADV_HUGEPAGE); ret = madvise(mem, hpage_size, MADV_HUGEPAGE);
if (!ret) { if (!ret) {
FORCE_READ(mem); FORCE_READ(*mem);
ret = pagemap_ioctl(mem, hpage_size, &vec, 1, 0, ret = pagemap_ioctl(mem, hpage_size, &vec, 1, 0,
0, PAGE_IS_PFNZERO, 0, 0, PAGE_IS_PFNZERO); 0, PAGE_IS_PFNZERO, 0, 0, PAGE_IS_PFNZERO);

View File

@ -439,8 +439,11 @@ int create_pagecache_thp_and_fd(const char *testfile, size_t fd_size, int *fd,
} }
madvise(*addr, fd_size, MADV_HUGEPAGE); madvise(*addr, fd_size, MADV_HUGEPAGE);
for (size_t i = 0; i < fd_size; i++) for (size_t i = 0; i < fd_size; i++) {
FORCE_READ((*addr + i)); char *addr2 = *addr + i;
FORCE_READ(*addr2);
}
if (!check_huge_file(*addr, fd_size / pmd_pagesize, pmd_pagesize)) { if (!check_huge_file(*addr, fd_size / pmd_pagesize, pmd_pagesize)) {
ksft_print_msg("No large pagecache folio generated, please provide a filesystem supporting large folio\n"); ksft_print_msg("No large pagecache folio generated, please provide a filesystem supporting large folio\n");

View File

@ -23,7 +23,7 @@
* anything with it in order to trigger a read page fault. We therefore must use * anything with it in order to trigger a read page fault. We therefore must use
* volatile to stop the compiler from optimising this away. * volatile to stop the compiler from optimising this away.
*/ */
#define FORCE_READ(x) (*(volatile typeof(x) *)x) #define FORCE_READ(x) (*(const volatile typeof(x) *)&(x))
extern unsigned int __page_size; extern unsigned int __page_size;
extern unsigned int __page_shift; extern unsigned int __page_shift;