diff --git a/src/sysboost_loader/binfmt_rto_604.c b/src/sysboost_loader/binfmt_rto_604.c index 728938af824b5cab4a8c56edd682c895a367eead..a5a832852b325e6c8ec5f2a0511cd1289af086c0 100644 --- a/src/sysboost_loader/binfmt_rto_604.c +++ b/src/sysboost_loader/binfmt_rto_604.c @@ -94,6 +94,7 @@ static struct global_symbols { proc_symbol(elf_core_write_extra_phdrs); proc_symbol(elf_core_write_extra_data); proc_symbol(get_mm_exe_file); + proc_symbol(find_extend_vma_locked); } g_sym; #define proc_symbol_char(x) #x @@ -128,6 +129,7 @@ static char *global_symbol_names[] = { proc_symbol_char(elf_core_write_extra_phdrs), proc_symbol_char(elf_core_write_extra_data), proc_symbol_char(get_mm_exe_file), + proc_symbol_char(find_extend_vma_locked), }; int init_symbols(void) @@ -539,10 +541,10 @@ create_elf_tables(struct linux_binprm *bprm, const struct elfhdr *exec, * Grow the stack manually; some architectures have a limit on how * far ahead a user-space access may be in order to grow the stack. */ - if (mmap_read_lock_killable(mm)) + if (mmap_write_lock_killable(mm)) return -EINTR; - vma = find_extend_vma(mm, bprm->p); - mmap_read_unlock(mm); + vma = g_sym.find_extend_vma_locked(mm, bprm->p); + mmap_write_unlock(mm); if (!vma) return -EFAULT; diff --git a/src/sysboost_loader/rto_populate_604.c b/src/sysboost_loader/rto_populate_604.c index f66e095abe1f419fbcd45557dcd94e7af2d784aa..19c27fba115e6022ccb8883cf7d576c8dcd7e2d5 100644 --- a/src/sysboost_loader/rto_populate_604.c +++ b/src/sysboost_loader/rto_populate_604.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -39,6 +40,15 @@ struct page *follow_page_mask(struct vm_area_struct *vma, struct follow_page_context *ctx); vm_fault_t do_set_pmd(struct vm_fault *vmf, struct page *page); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) +static struct vm_area_struct *gup_vma_lookup(struct mm_struct *mm, + unsigned long addr); + +static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags); + +struct vm_area_struct *find_extend_vma_locked(struct mm_struct *mm, unsigned long addr); +#endif + #define proc_symbol(SYM) typeof(SYM) *(SYM) static struct global_symbols { proc_symbol(follow_page_mask); @@ -46,6 +56,12 @@ static struct global_symbols { proc_symbol(__anon_vma_prepare); proc_symbol(__pmd_alloc); proc_symbol(do_set_pmd); +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) + proc_symbol(gup_vma_lookup); + proc_symbol(check_vma_flags); + proc_symbol(find_extend_vma_locked); +#endif + #ifdef CONFIG_X86 proc_symbol(__p4d_alloc); proc_symbol(pud_clear_bad); @@ -59,6 +75,11 @@ static char *global_symbol_names[] = { proc_symbol_char(__anon_vma_prepare), proc_symbol_char(__pmd_alloc), proc_symbol_char(do_set_pmd), +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) + proc_symbol_char(gup_vma_lookup), + proc_symbol_char(check_vma_flags), + proc_symbol_char(find_extend_vma_locked), +#endif #ifdef CONFIG_X86 proc_symbol_char(__p4d_alloc), proc_symbol_char(pud_clear_bad), @@ -90,6 +111,10 @@ enum { FOLL_FAST_ONLY = 1 << 20, /* allow unlocking the mmap lock */ FOLL_UNLOCKABLE = 1 << 21, +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) + /* VMA lookup+checks compatible with MADV_POPULATE_(READ|WRITE) */ + FOLL_MADV_POPULATE = 1 << 22, +#endif }; static vm_fault_t __rto_do_huge_pmd_anonymous_page(struct vm_fault *vmf, @@ -530,7 +555,27 @@ static long rto_get_user_pages(struct mm_struct *mm, /* first iteration or cross vma bound */ if (!vma || start >= vma->vm_end) { +#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 6, 0) + /* + * MADV_POPULATE_(READ|WRITE) wants to handle VMA + * lookups+error reporting differently. + */ + if (gup_flags & FOLL_MADV_POPULATE) { + vma = vma_lookup(mm, start); + if (!vma) { + ret = -ENOMEM; + goto out; + } + if (ppl_sym.check_vma_flags(vma, gup_flags)) { + ret = -EINVAL; + goto out; + } + } else { + vma = ppl_sym.gup_vma_lookup(mm, start); + } +#else vma = find_extend_vma(mm, start); +#endif // if (!vma && in_gate_area(mm, start)) { // ret = get_gate_page(mm, start & PAGE_MASK, // gup_flags, &vma, diff --git a/src/sysboostd/bolt.rs b/src/sysboostd/bolt.rs index 56f5ae99aad9a74ed2f2d0a9482eedf01aa6fa49..5684dd1b55661ea385e5a0b944ec0fa17e37158e 100644 --- a/src/sysboostd/bolt.rs +++ b/src/sysboostd/bolt.rs @@ -41,6 +41,7 @@ fn get_profile_path(conf: &RtoConfig) -> String { // support profile fn bolt_optimize_bin(conf: &RtoConfig) -> i32 { let mut args: Vec = Vec::new(); + let mut command = String::new(); args.push("-reorder-blocks=ext-tsp".to_string()); args.push("-reorder-functions=hfsort".to_string()); @@ -68,7 +69,14 @@ fn bolt_optimize_bin(conf: &RtoConfig) -> i32 { if real_profile_path != "" { args.push(format!("-data={}", real_profile_path)); } - let mut ret = run_child("/usr/bin/llvm-bolt", &args); + + let bolt_dir = conf.bolt_dir.clone(); + if bolt_dir.is_empty() { + command = "/usr/bin/llvm-bolt".to_string(); + } else { + command = format!("{}/llvm-bolt", bolt_dir); + } + let mut ret = run_child(&command, &args); if ret != 0 { return ret; } @@ -80,6 +88,7 @@ fn bolt_optimize_bin(conf: &RtoConfig) -> i32 { fn bolt_optimize_so(conf: &RtoConfig) -> i32 { let mut args: Vec = Vec::new(); + let mut command = String::new(); let mut ret = 1; // change layout of basic blocks in a function args.push("-reorder-blocks=ext-tsp".to_string()); @@ -110,7 +119,13 @@ fn bolt_optimize_so(conf: &RtoConfig) -> i32 { args.push(lib_bak_path.to_str().unwrap().to_string()); args.push("-o".to_string()); args.push(lib.split_whitespace().collect()); - ret = run_child("/usr/bin/llvm-bolt", &args); + let bolt_dir = conf.bolt_dir.clone(); + if bolt_dir.is_empty() { + command = "/usr/bin/llvm-bolt".to_string(); + } else { + command = format!("{}/llvm-bolt", bolt_dir); + } + ret = run_child(&command, &args); } return ret; } diff --git a/src/sysboostd/config.rs b/src/sysboostd/config.rs index 173dbc43bff1edafa4ddc18389209cf15c35d555..ba197bf61912fca10e84e32778736f628f5eebe7 100644 --- a/src/sysboostd/config.rs +++ b/src/sysboostd/config.rs @@ -34,6 +34,8 @@ pub struct RtoConfig { pub libs: Vec, // TODO: 修改为字符串, 列表形式影响可读性 // profile文件路径 pub profile_path: Option, + // llvm-bolt命令文件夹路径 + pub bolt_dir: String, // 环境变量 #[serde(rename = "PATH")] pub path: Option, @@ -172,6 +174,17 @@ fn parse_rto_config(sec: String, prop: &Properties) { }, profile_path: prop.get("profile_path").map(|s| s.to_string()), path: prop.get("path").map(|s| s.to_string()), + bolt_dir: match prop.get("bolt_dir") { + Some(p) => { + if p.trim().is_empty() { + String::new() + } + else { + p.to_string() + } + } + None => {String::new()} + }, watch_paths: Vec::new(), }; if rtoconf.elf_path == SYSBOOST_PATH || is_mode_invalid(rtoconf.mode.clone()){ diff --git a/src/sysboostd/interface.rs b/src/sysboostd/interface.rs index c2474815e8d0375893942739cee64c2578c215dd..b8d9e563ad7c7ad0439d5de645a0299e59c91397 100644 --- a/src/sysboostd/interface.rs +++ b/src/sysboostd/interface.rs @@ -93,8 +93,9 @@ pub fn bolt_add_link(file_name: &str) -> i32 { 0 } -pub fn gen_bolt_optimize_bin(name: &str, bolt_option: &str, profile_path: &str) -> i32 { +pub fn gen_bolt_optimize_bin(name: &str, bolt_option: &str, profile_path: &str, bolt_dir: &str) -> i32 { let mut args: Vec = Vec::new(); + let mut command = String::new(); if bolt_option.is_empty() { args.push("-reorder-blocks=ext-tsp".to_string()); args.push("-reorder-functions=hfsort".to_string()); @@ -121,7 +122,12 @@ pub fn gen_bolt_optimize_bin(name: &str, bolt_option: &str, profile_path: &str) args.push("-o".to_string()); args.push(rto_path.to_str().unwrap().to_string()); args.push(format!("-data={}", profile_path)); - let mut ret = run_child("/usr/bin/llvm-bolt", &args); + if bolt_dir.is_empty() { + command = "/usr/bin/llvm-bolt".to_string(); + } else { + command = format!("{}/llvm-bolt", bolt_dir); + } + let mut ret = run_child(&command, &args); if ret != 0 { return ret; } diff --git a/src/sysboostd/main.rs b/src/sysboostd/main.rs index d33c0abedd4bc3e1a6756009c1e5b9bfd57dc578..f1abbb27e90469d24c6ca556bb5d49f0ffef97c6 100644 --- a/src/sysboostd/main.rs +++ b/src/sysboostd/main.rs @@ -59,6 +59,7 @@ fn main() { let mut bolt_option = ""; let mut profile_path = ""; let mut bolt_elf_name = ""; + let mut bolt_dir = ""; let mut is_stop = false; let mut stop_elf_name = ""; @@ -99,6 +100,12 @@ fn main() { } continue; } + if args[i].contains("--bolt-dir=") { + if let Some(index) = args[i].find('=') { + bolt_dir = &args[i][index + 1..]; + } + continue; + } if args[i].contains("--stop=") { if let Some(index) = args[i].find('=') { is_stop = true; @@ -139,7 +146,7 @@ fn main() { if profile_path.is_empty() { profile_path = PROFILE_PATH_DEFAULT; } - let ret = gen_bolt_optimize_bin(bolt_elf_name, bolt_option, profile_path); + let ret = gen_bolt_optimize_bin(bolt_elf_name, bolt_option, profile_path, bolt_dir); if ret < 0 { std::process::exit(-1); }