mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
synced 2026-09-18 22:09:30 +02:00
Merge tag 'x86_urgent_for_7.3-rc4' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
Pull x86 fixes from Dave Hansen: "The most notable fix is THP not silently losing user data and having been around for a couple of years. The main explanation I'd have for its longevity is that it requires a few different things to align at the same time: MADV_FREE, THP and heavy reclaim. - Fix user-space data loss with THP - Fix set_memory oopses - Fix addition of large constants in mul_u64_add_u64_div_u64() - Fix FineIBT hash offset in cfi_get_func_hash() - Fix PCI device reference counting in amd_smn_init()" * tag 'x86_urgent_for_7.3-rc4' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: x86/amd_node: Fix PCI device reference counting in amd_smn_init() x86/div64: Fix addition of large constants in mul_u64_add_u64_div_u64() x86/cfi: Fix FineIBT hash offset in cfi_get_func_hash() x86/mm: Fix user-space data loss with MADV_FREE and THP x86/mm/pat: Allocate split page tables as kernel page tables x86/alternatives: Exclude text poking against change_page_attr() x86/mm/pat: Acquire init_mm read lock on attribute changes to avoid UAF x86/mm/pat: Acquire init_mm write lock on collapse to avoid UAF
This commit is contained in:
Linus Torvalds
@@ -111,7 +111,7 @@ static inline u64 mul_u64_add_u64_div_u64(u64 rax, u64 mul, u64 add, u64 div)
|
||||
|
||||
if (!statically_true(!add))
|
||||
asm ("addq %[add], %[lo]; adcq $0, %[hi]" :
|
||||
[lo] "+r" (rax), [hi] "+r" (rdx) : [add] "irm" (add));
|
||||
[lo] "+r" (rax), [hi] "+r" (rdx) : [add] "erm" (add));
|
||||
|
||||
asm ("divq %[div]" : "+a" (rax), "+d" (rdx) : [div] "rm" (div));
|
||||
|
||||
|
||||
@@ -806,7 +806,7 @@ static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot)
|
||||
pmdval_t val = pmd_val(pmd), oldval = val;
|
||||
pmd_t pmd_result;
|
||||
|
||||
val &= (_HPAGE_CHG_MASK & ~_PAGE_DIRTY);
|
||||
val &= _HPAGE_CHG_MASK;
|
||||
val |= check_pgprot(newprot) & ~_HPAGE_CHG_MASK;
|
||||
val = flip_protnone_guard(oldval, val, PHYSICAL_PMD_PAGE_MASK);
|
||||
|
||||
|
||||
@@ -6,6 +6,9 @@
|
||||
#include <linux/vmalloc.h>
|
||||
#include <linux/memory.h>
|
||||
#include <linux/execmem.h>
|
||||
#include <linux/cleanup.h>
|
||||
#include <linux/kgdb.h>
|
||||
#include <linux/mmap_lock.h>
|
||||
|
||||
#include <asm/text-patching.h>
|
||||
#include <asm/insn.h>
|
||||
@@ -1198,6 +1201,41 @@ static bool cfi_debug __ro_after_init;
|
||||
bool cfi_bhi __ro_after_init = false;
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_FINEIBT
|
||||
/*
|
||||
* <fineibt_preamble_start>:
|
||||
* 0: f3 0f 1e fa endbr64
|
||||
* 4: 2d 78 56 34 12 sub $0x12345678, %eax
|
||||
* 9: 2e 0f 85 03 00 00 00 jne,pn 13 <fineibt_preamble_start+0x13>
|
||||
* 10: 0f 1f 40 d6 nopl -0x2a(%rax)
|
||||
*
|
||||
* Note that the JNE target is the 0xD6 byte inside the NOPL, this decodes as
|
||||
* UDB on x86_64 and raises #UD.
|
||||
*/
|
||||
asm( ".pushsection .rodata \n"
|
||||
"fineibt_preamble_start: \n"
|
||||
" endbr64 \n"
|
||||
" subl $0x12345678, %eax \n"
|
||||
"fineibt_preamble_bhi: \n"
|
||||
" cs jne.d32 fineibt_preamble_start+0x13 \n"
|
||||
"#fineibt_func: \n"
|
||||
" nopl -42(%rax) \n"
|
||||
"fineibt_preamble_end: \n"
|
||||
".popsection\n"
|
||||
);
|
||||
|
||||
extern u8 fineibt_preamble_start[];
|
||||
extern u8 fineibt_preamble_bhi[];
|
||||
extern u8 fineibt_preamble_end[];
|
||||
|
||||
#define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start)
|
||||
#define fineibt_preamble_bhi (fineibt_preamble_bhi - fineibt_preamble_start)
|
||||
#define fineibt_preamble_ud 0x13
|
||||
#define fineibt_preamble_hash 5
|
||||
|
||||
#define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE)
|
||||
#endif /* CONFIG_FINEIBT */
|
||||
|
||||
#ifdef CONFIG_CFI
|
||||
u32 cfi_get_func_hash(void *func)
|
||||
{
|
||||
@@ -1205,9 +1243,11 @@ u32 cfi_get_func_hash(void *func)
|
||||
|
||||
func -= cfi_get_offset();
|
||||
switch (cfi_mode) {
|
||||
#ifdef CONFIG_FINEIBT
|
||||
case CFI_FINEIBT:
|
||||
func += 7;
|
||||
func += fineibt_preamble_hash;
|
||||
break;
|
||||
#endif
|
||||
case CFI_KCFI:
|
||||
func += 1;
|
||||
break;
|
||||
@@ -1363,39 +1403,6 @@ early_param("cfi", cfi_parse_cmdline);
|
||||
* anyway.
|
||||
*/
|
||||
|
||||
/*
|
||||
* <fineibt_preamble_start>:
|
||||
* 0: f3 0f 1e fa endbr64
|
||||
* 4: 2d 78 56 34 12 sub $0x12345678, %eax
|
||||
* 9: 2e 0f 85 03 00 00 00 jne,pn 13 <fineibt_preamble_start+0x13>
|
||||
* 10: 0f 1f 40 d6 nopl -0x2a(%rax)
|
||||
*
|
||||
* Note that the JNE target is the 0xD6 byte inside the NOPL, this decodes as
|
||||
* UDB on x86_64 and raises #UD.
|
||||
*/
|
||||
asm( ".pushsection .rodata \n"
|
||||
"fineibt_preamble_start: \n"
|
||||
" endbr64 \n"
|
||||
" subl $0x12345678, %eax \n"
|
||||
"fineibt_preamble_bhi: \n"
|
||||
" cs jne.d32 fineibt_preamble_start+0x13 \n"
|
||||
"#fineibt_func: \n"
|
||||
" nopl -42(%rax) \n"
|
||||
"fineibt_preamble_end: \n"
|
||||
".popsection\n"
|
||||
);
|
||||
|
||||
extern u8 fineibt_preamble_start[];
|
||||
extern u8 fineibt_preamble_bhi[];
|
||||
extern u8 fineibt_preamble_end[];
|
||||
|
||||
#define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start)
|
||||
#define fineibt_preamble_bhi (fineibt_preamble_bhi - fineibt_preamble_start)
|
||||
#define fineibt_preamble_ud 0x13
|
||||
#define fineibt_preamble_hash 5
|
||||
|
||||
#define fineibt_prefix_size (fineibt_preamble_size - ENDBR_INSN_SIZE)
|
||||
|
||||
/*
|
||||
* <fineibt_caller_start>:
|
||||
* 0: b8 78 56 34 12 mov $0x12345678, %eax
|
||||
@@ -2372,6 +2379,38 @@ static void text_poke_memset(void *dst, const void *src, size_t len)
|
||||
|
||||
typedef void text_poke_f(void *dst, const void *src, size_t len);
|
||||
|
||||
static void __poke_vmalloc_pages(struct page **pages, void *addr,
|
||||
bool cross_page_boundary)
|
||||
{
|
||||
pages[0] = vmalloc_to_page(addr);
|
||||
if (cross_page_boundary)
|
||||
pages[1] = vmalloc_to_page(addr + PAGE_SIZE);
|
||||
}
|
||||
|
||||
static void poke_vmalloc_pages(struct page **pages, void *addr,
|
||||
bool cross_page_boundary)
|
||||
{
|
||||
if (in_dbg_master()) {
|
||||
/*
|
||||
* If called from kgdb cannot sleep, but all other CPUs stopped
|
||||
* anyway so safe to proceed without locks
|
||||
*/
|
||||
__poke_vmalloc_pages(pages, addr, cross_page_boundary);
|
||||
} else {
|
||||
/*
|
||||
* execmem ROX ranges are shared between modules and can be
|
||||
* collapsed to huge PMD entries, and this collapse can happen
|
||||
* concurrently with a racing set_memory_rox().
|
||||
*
|
||||
* Prevent vmalloc_to_page() from racing by acquiring an
|
||||
* init_mm read lock which pairs with the init_mm write lock in
|
||||
* cpa_collapse_large_pages().
|
||||
*/
|
||||
guard(mmap_read_lock)(&init_mm);
|
||||
__poke_vmalloc_pages(pages, addr, cross_page_boundary);
|
||||
}
|
||||
}
|
||||
|
||||
static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t len)
|
||||
{
|
||||
bool cross_page_boundary = offset_in_page(addr) + len > PAGE_SIZE;
|
||||
@@ -2389,9 +2428,7 @@ static void *__text_poke(text_poke_f func, void *addr, const void *src, size_t l
|
||||
BUG_ON(!after_bootmem);
|
||||
|
||||
if (!core_kernel_text((unsigned long)addr)) {
|
||||
pages[0] = vmalloc_to_page(addr);
|
||||
if (cross_page_boundary)
|
||||
pages[1] = vmalloc_to_page(addr + PAGE_SIZE);
|
||||
poke_vmalloc_pages(pages, addr, cross_page_boundary);
|
||||
} else {
|
||||
pages[0] = virt_to_page(addr);
|
||||
WARN_ON(!PageReserved(pages[0]));
|
||||
|
||||
@@ -251,7 +251,7 @@ __setup("amd_smn_debugfs_enable", amd_smn_enable_dfs);
|
||||
static int __init amd_smn_init(void)
|
||||
{
|
||||
u16 count, num_roots, roots_per_node, node, num_nodes;
|
||||
struct pci_dev *root;
|
||||
struct pci_dev *root __free(pci_dev_put) = NULL;
|
||||
|
||||
if (!cpu_feature_enabled(X86_FEATURE_ZEN))
|
||||
return 0;
|
||||
@@ -262,7 +262,6 @@ static int __init amd_smn_init(void)
|
||||
return 0;
|
||||
|
||||
num_roots = 0;
|
||||
root = NULL;
|
||||
while ((root = get_next_root(root))) {
|
||||
pci_dbg(root, "Reserving PCI config space\n");
|
||||
|
||||
@@ -299,14 +298,13 @@ static int __init amd_smn_init(void)
|
||||
|
||||
count = 0;
|
||||
node = 0;
|
||||
root = NULL;
|
||||
while (node < num_nodes && (root = get_next_root(root))) {
|
||||
/* Use one root for each node and skip the rest. */
|
||||
if (count++ % roots_per_node)
|
||||
continue;
|
||||
|
||||
pci_dbg(root, "is root for AMD node %u\n", node);
|
||||
amd_roots[node++] = root;
|
||||
amd_roots[node++] = pci_dev_get(root);
|
||||
}
|
||||
|
||||
if (enable_dfs) {
|
||||
|
||||
@@ -22,6 +22,7 @@
|
||||
#include <linux/cc_platform.h>
|
||||
#include <linux/set_memory.h>
|
||||
#include <linux/memregion.h>
|
||||
#include <linux/cleanup.h>
|
||||
|
||||
#include <asm/e820/api.h>
|
||||
#include <asm/processor.h>
|
||||
@@ -49,7 +50,8 @@ struct cpa_data {
|
||||
unsigned int flags;
|
||||
unsigned int force_split : 1,
|
||||
force_static_prot : 1,
|
||||
force_flush_all : 1;
|
||||
force_flush_all : 1,
|
||||
init_mm_read_locked : 1;
|
||||
struct page **pages;
|
||||
};
|
||||
|
||||
@@ -409,7 +411,7 @@ static void __cpa_flush_tlb(void *data)
|
||||
|
||||
static int collapse_large_pages(unsigned long addr, struct list_head *pgtables);
|
||||
|
||||
static void cpa_collapse_large_pages(struct cpa_data *cpa)
|
||||
static void __cpa_collapse_large_pages(struct cpa_data *cpa)
|
||||
{
|
||||
unsigned long start, addr, end;
|
||||
struct ptdesc *ptdesc, *tmp;
|
||||
@@ -439,10 +441,30 @@ static void cpa_collapse_large_pages(struct cpa_data *cpa)
|
||||
|
||||
list_for_each_entry_safe(ptdesc, tmp, &pgtables, pt_list) {
|
||||
list_del(&ptdesc->pt_list);
|
||||
pagetable_free(ptdesc);
|
||||
/*
|
||||
* Only early alloc'd direct map should not be flagged PG_table
|
||||
* here and those shouldn't be collapsed. However be abundantly
|
||||
* cautious and handle the !PG_table case too.
|
||||
*/
|
||||
if (PageTable((ptdesc_page(ptdesc))))
|
||||
pagetable_dtor_free(ptdesc);
|
||||
else
|
||||
pagetable_free(ptdesc);
|
||||
}
|
||||
}
|
||||
|
||||
static void cpa_collapse_large_pages(struct cpa_data *cpa)
|
||||
{
|
||||
/*
|
||||
* Take the mmap write lock on init_mm to:
|
||||
* - Avoid a use-after-free if raced by ptdump (which takes its own
|
||||
* write lock on init_mm).
|
||||
* - Serialise concurrent CPA walkers.
|
||||
*/
|
||||
scoped_guard(mmap_write_lock, &init_mm)
|
||||
__cpa_collapse_large_pages(cpa);
|
||||
}
|
||||
|
||||
static void cpa_flush(struct cpa_data *cpa, int cache)
|
||||
{
|
||||
unsigned int i;
|
||||
@@ -1120,11 +1142,10 @@ set:
|
||||
|
||||
static int
|
||||
__split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,
|
||||
struct ptdesc *ptdesc)
|
||||
pte_t *pbase)
|
||||
{
|
||||
unsigned long lpaddr, lpinc, ref_pfn, pfn, pfninc = 1;
|
||||
struct page *base = ptdesc_page(ptdesc);
|
||||
pte_t *pbase = (pte_t *)page_address(base);
|
||||
struct page *base = virt_to_page(pbase);
|
||||
unsigned int i, level;
|
||||
pgprot_t ref_prot;
|
||||
bool nx, rw;
|
||||
@@ -1224,16 +1245,20 @@ __split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address,
|
||||
static int split_large_page(struct cpa_data *cpa, pte_t *kpte,
|
||||
unsigned long address)
|
||||
{
|
||||
struct ptdesc *ptdesc;
|
||||
pte_t *pte;
|
||||
|
||||
spin_unlock(&cpa_lock);
|
||||
ptdesc = pagetable_alloc(GFP_KERNEL, 0);
|
||||
if (cpa->init_mm_read_locked)
|
||||
mmap_read_unlock(&init_mm);
|
||||
pte = pte_alloc_one_kernel(&init_mm);
|
||||
if (cpa->init_mm_read_locked)
|
||||
mmap_read_lock(&init_mm);
|
||||
spin_lock(&cpa_lock);
|
||||
if (!ptdesc)
|
||||
if (!pte)
|
||||
return -ENOMEM;
|
||||
|
||||
if (__split_large_page(cpa, kpte, address, ptdesc))
|
||||
pagetable_free(ptdesc);
|
||||
if (__split_large_page(cpa, kpte, address, pte))
|
||||
pte_free_kernel(&init_mm, pte);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -2121,7 +2146,11 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages,
|
||||
cpa.curpage = 0;
|
||||
cpa.force_split = force_split;
|
||||
|
||||
ret = __change_page_attr_set_clr(&cpa, 1);
|
||||
/* Avoid race with concurrent CPA collapse. */
|
||||
cpa.init_mm_read_locked = true;
|
||||
scoped_guard(mmap_read_lock, &init_mm)
|
||||
ret = __change_page_attr_set_clr(&cpa, 1);
|
||||
cpa.init_mm_read_locked = false;
|
||||
|
||||
/*
|
||||
* Check whether we really changed something:
|
||||
|
||||
@@ -630,6 +630,8 @@ static inline void mmap_read_unlock(struct mm_struct *mm)
|
||||
DEFINE_GUARD(mmap_read_lock, struct mm_struct *,
|
||||
mmap_read_lock(_T), mmap_read_unlock(_T))
|
||||
DEFINE_GUARD_COND(mmap_read_lock, _try, mmap_read_trylock(_T))
|
||||
DEFINE_GUARD(mmap_write_lock, struct mm_struct *,
|
||||
mmap_write_lock(_T), mmap_write_unlock(_T))
|
||||
|
||||
static inline void mmap_read_unlock_non_owner(struct mm_struct *mm)
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user