From e22be507794e6de2cf67a6f1f0fbb0b67e4d71c8 Mon Sep 17 00:00:00 2001 From: jlcoo Date: Mon, 28 Apr 2025 19:04:11 +0800 Subject: [PATCH] feat: add persist repo metadata --- src/depends.rs | 149 ++++++++++++++++++++++++++++------------------- src/install.rs | 8 +-- src/io.rs | 53 ++++------------- src/models.rs | 21 ++++--- src/remove.rs | 4 +- src/repo.rs | 155 +++++++++++++++++++++++++++++++++++++++++++++++-- 6 files changed, 270 insertions(+), 120 deletions(-) diff --git a/src/depends.rs b/src/depends.rs index c0c04f8..bddff8b 100644 --- a/src/depends.rs +++ b/src/depends.rs @@ -1,5 +1,5 @@ use std::process::exit; -use std::collections::HashMap; +use std::collections::{HashMap}; use std::time::{SystemTime, UNIX_EPOCH}; use anyhow::{bail, Ok, Result}; use crate::models::*; @@ -21,17 +21,38 @@ impl InstalledPackageInfo { } impl PackageManager { - - pub fn resolve_appbin_source(&mut self, packages: &mut HashMap) { + pub fn record_appbin_source(&mut self, packages: &mut HashMap) { + let mut tmp_format: Option = None; for pkgline in packages.keys() { - if let Some(spec) = self.pkghash2spec.get(&pkgline[0..32]) { - if let Some(source) = spec.source.clone() { - self.appbin_source.insert(source); - } else { - println!("Not get source, pkgline: {:#?}", pkgline); + let pkg_json = self.load_package_info(pkgline).unwrap(); + if pkg_json.source.is_some() { + self.appbin_source.insert(pkg_json.source.as_ref().unwrap().clone()); + } + if tmp_format.is_none() { + tmp_format = match pkg_json.origin_url { + Some(ref url) => { + get_package_format(url) + }, + None => { + Some("rpm".to_string()) + } + }; + } + } + self.repos_data[0].format = tmp_format; + } + + pub fn change_appbin_flag_same_source(&mut self, packages: &mut HashMap) -> Result<()> { + for (pkgline, package_info) in packages.iter_mut() { + let pkg_json = self.load_package_info(pkgline.as_str()).unwrap(); + if package_info.appbin_flag == false && pkg_json.source.is_some() { + let Some(source) = &pkg_json.source else { continue }; + if self.appbin_source.contains(source) { + package_info.appbin_flag = true; } } } + Ok(()) } fn add_one_package_installing(&self, pkg_name: &str, depth: u8, ebin_flag: bool, @@ -57,7 +78,7 @@ impl PackageManager { for capability in capabilities { if let Some(pkgname) = self.provide2pkgnames.get(capability .as_str()) { - pnames.push(pkgname.clone()); + pnames.push(pkgname[0].clone()); } else { // 输入是真正的pkgname pnames.push(capability .clone()); @@ -75,11 +96,22 @@ impl PackageManager { } } + packages + } + + pub fn collect_essential_packages(&mut self, packages: &mut HashMap) -> Result<()> { + let mut missing_names = Vec::new(); for essential_pkgname in &self.essential_pkgnames { - self.add_one_package_installing(essential_pkgname.as_str(), 0, false, &mut packages, &mut missing_names); + self.add_one_package_installing(essential_pkgname.as_str(), 0, false, packages, &mut missing_names); + } + if !missing_names.is_empty() { + println!("Missing packages: {:#?}", missing_names); + if !self.options.ignore_missing { + exit(1); + } } - packages + Ok(()) } pub fn collect_recursive_depends(&mut self, @@ -87,14 +119,10 @@ impl PackageManager { ) -> Result<()> { let mut depend_packages: HashMap = HashMap::new(); let mut depth = 1; - let mut repo_format: Option = None; - if let Some((pkg_hash, _)) = packages.iter().next() { - let pkg_hash_str = pkg_hash.as_str(); - repo_format = self.pkghash2spec[&pkg_hash_str[0..32]].format.clone(); - } + let repo_format: Option = self.repos_data[0].format.clone(); self.collect_depends(&packages, &mut depend_packages, depth, &repo_format)?; - + while !depend_packages.is_empty() { packages.extend(depend_packages); depend_packages = HashMap::new(); @@ -105,55 +133,56 @@ impl PackageManager { Ok(()) } - fn load_package_info(&self, pkg_hash: &str) -> Result { - let path = format!( - "{}/channel/{}/{}/{}/pkg-info/{}/{}.json", - paths::instance.epkg_cache.display(), - self.env_config.channel.name, - self.pkghash2spec[&pkg_hash[0..32]].repo, - self.options.arch, - &pkg_hash[0..2], - pkg_hash - ); - load_package_json(&path).map_err(|e| e.into()) + fn load_package_info(&mut self, pkgline: &str) -> Result { + if let Some(package) = self.pkghash2pkg.get(&pkgline[0..32]) { + return Ok(package.clone()); + } else { + let path = format!( + "{}/channel/{}/{}/{}/pkg-info/{}/{}.json", + paths::instance.epkg_cache.display(), + self.env_config.channel.name, + self.pkghash2spec[&pkgline[0..32]].repo, + self.options.arch, + &pkgline[0..2], + pkgline + ); + let package = load_package_json(&path).unwrap(); + self.pkghash2pkg.insert(pkgline.to_string(), package.clone()); + return Ok(package); + } } fn process_dependencies( &mut self, - pkg_info: &Package, + dependencies: &Vec, packages: &HashMap, depend_packages: &mut HashMap, depth: u8, missing_deps: &mut Vec, ) -> Result<()> { - if let Some(dependencies) = &pkg_info.depends { - for dep in dependencies { - let Some(spec) = self.pkghash2spec.get(&dep.hash) else { - missing_deps.push(format!("{}-{}", dep.pkgname, dep.hash)); - continue; - }; - - let dep_id = format!( - "{}__{}__{}__{}", - spec.hash, spec.name, spec.version, spec.release + for dep in dependencies { + let Some(spec) = self.pkghash2spec.get(&dep.hash) else { + missing_deps.push(format!("{}-{}", dep.pkgname, dep.hash)); + continue; + }; + + let dep_id = format!( + "{}__{}__{}__{}", + spec.hash, spec.name, spec.version, spec.release + ); + + if !packages.contains_key(&dep_id) && !depend_packages.contains_key(&dep_id) { + depend_packages.insert( + dep_id.clone(), + InstalledPackageInfo::new(depth, false), ); - - if !packages.contains_key(&dep_id) && !depend_packages.contains_key(&dep_id) { - let appbin_flag = spec.source - .as_ref() - .map_or(false, |s| self.appbin_source.contains(s)); - - depend_packages.insert( - dep_id.clone(), - InstalledPackageInfo::new(depth, appbin_flag), - ); - } } } + Ok(()) } - fn process_requirements_impl( + fn process_requirements( &mut self, requirements: &Vec, packages: &HashMap, @@ -180,7 +209,7 @@ impl PackageManager { }; for or_depends in and_deps { for pkg_depend in or_depends { - self.process_requirement( + self.process_requirement_impl( &pkg_depend.capability, pkg_format.as_str(), packages, @@ -194,7 +223,7 @@ impl PackageManager { Ok(()) } - fn process_requirement( + fn process_requirement_impl( &mut self, capability: &str, pkg_format: &str, @@ -213,7 +242,7 @@ impl PackageManager { return Ok(()); } }; - let Some(hashes) = self.pkgname2lines.get(pkg_mapping_name) else { + let Some(hashes) = self.pkgname2lines.get(pkg_mapping_name[0].as_str()) else { missing_deps.push(format!("{}-{}", capability, pkg_format)); return Ok(()); }; @@ -256,11 +285,12 @@ impl PackageManager { repo_format: &Option, ) -> Result<()> { let mut missing_deps = Vec::new(); - for pkg_hash in packages.keys() { - let pkg_info = self.load_package_info(pkg_hash)?; + for pkgline in packages.keys() { + let pkg_info = self.load_package_info(pkgline)?; + if pkg_info.requires_pre.is_some() { let Some(requirements) = &pkg_info.requires_pre else { continue }; - self.process_requirements_impl( + self.process_requirements( requirements, packages, depend_packages, @@ -270,8 +300,9 @@ impl PackageManager { )?; } if pkg_info.depends.is_some() { + let Some(dependencies) = &pkg_info.depends else { continue }; self.process_dependencies( - &pkg_info, + dependencies, packages, depend_packages, depth, @@ -279,7 +310,7 @@ impl PackageManager { )?; } else if pkg_info.requires.is_some() { let Some(requirements) = &pkg_info.requires else { continue }; - self.process_requirements_impl( + self.process_requirements( requirements, packages, depend_packages, diff --git a/src/install.rs b/src/install.rs index 8999db7..23011b5 100644 --- a/src/install.rs +++ b/src/install.rs @@ -271,7 +271,8 @@ impl PackageManager { self.load_installed_packages().unwrap(); let mut packages_to_install = self.resolve_package_info(package_specs.clone()); - self.resolve_appbin_source(&mut packages_to_install); + self.record_appbin_source(&mut packages_to_install); + self.collect_essential_packages(&mut packages_to_install)?; self.collect_recursive_depends(&mut packages_to_install)?; remove_duplicates(&self.installed_packages, &mut packages_to_install, "Warning: Some packages are already installed and will be skipped:"); if packages_to_install.is_empty() { @@ -279,14 +280,13 @@ impl PackageManager { } if self.options.verbose { - println!("appbin_source: {:?}", self.appbin_source); println!("Packages to install:"); print_packages_by_depend_depth(&packages_to_install); } let files = self.download_packages(&packages_to_install)?; self.unpack_packages(files).unwrap(); - + self.change_appbin_flag_same_source(&mut packages_to_install).unwrap(); // create symlinks let symlink_dir = self.get_current_profile()?; let mut appbin_count = 0; @@ -296,7 +296,7 @@ impl PackageManager { let mut pkg_name = String::new(); // appbin_source check if let Some(spec) = self.pkghash2spec.get(&pkgline[0..32]) { - appbin_flag = package_specs.contains(&spec.name) || spec.source.as_ref().map_or(false, |source| self.appbin_source.contains(source)); + appbin_flag = _package_info.appbin_flag; pkg_name = spec.name.clone(); if appbin_flag { appbin_count += 1; diff --git a/src/io.rs b/src/io.rs index 4e0784b..bf66bd3 100644 --- a/src/io.rs +++ b/src/io.rs @@ -8,7 +8,6 @@ use dirs::home_dir; use anyhow::{Context, Result, bail}; use crate::paths; use crate::models::*; -use crate::parse_requires::*; pub fn load_package_json(file_path: &str) -> Result { let contents = fs::read_to_string(&file_path) @@ -43,40 +42,18 @@ pub fn load_repodata_index(file_path: &str) -> Result { } // Function to parse a pkgline into a PackageSpec -fn parse_package_line(pkgline: &str, reponame: &str, channel: &str, arch: &str) -> Result { +fn parse_package_line(pkgline: &str, reponame: &str) -> Result { let parts: Vec<&str> = pkgline.split("__").collect(); if parts.len() != 4 { bail!("Invalid package line format: {}", pkgline); } - let file_path: String = format!("{}/channel/{}/{}/{}/pkg-info/{}/{}.json", - paths::instance.epkg_cache.display(), - channel, - reponame.to_string(), - arch, - &pkgline[0..2], - pkgline, - ); - let pkg_json = load_package_json(&file_path)?; - let format = match pkg_json.origin_url { - Some(ref url) => { - get_package_format(url) - }, - None => { - Some("rpm".to_string()) - } - }; - Ok(PackageSpec { repo: reponame.to_string(), hash: parts[0].to_string(), name: parts[1].to_string(), version: parts[2].to_string(), release: parts[3].to_string(), - source: pkg_json.source, - provides: pkg_json.provides, - priority: pkg_json.priority, - format: format, }) } @@ -115,8 +92,12 @@ impl PackageManager { Ok(path) => { let path_str = path.to_str().with_context(|| format!("Invalid UTF-8 in path: {:?}", path))?; // Call the global function to load repodata - let repodata = load_repodata_index(path_str) + let mut repodata = load_repodata_index(path_str) .with_context(|| format!("Failed to load repodata from {}", path.display()))?; + let provide_path = path.parent().unwrap().join("provide2pkgnames.txt"); + repodata.decode_provide_hashmap(provide_path.to_str().unwrap())?; + let essential_path = path.parent().unwrap().join("essential_pkgnames.txt"); + repodata.decode_essential_hashset(essential_path.to_str().unwrap())?; self.repos_data.push(repodata); }, Err(e) => println!("{:?}", e), @@ -131,6 +112,9 @@ impl PackageManager { self.load_repodata()?; } for repodata in &self.repos_data { + self.provide2pkgnames.extend(repodata.provide2pkgnames.clone()); + self.essential_pkgnames.extend(repodata.essential_pkgnames.clone()); + for entry in &repodata.store_paths { let file_path = format!( "{}/{}", @@ -144,28 +128,11 @@ impl PackageManager { let contents = fs::read_to_string(&file_path) .with_context(|| format!("Failed to load store-paths from {}", file_path))?; for pkgline in contents.lines() { - if let Ok(pkg_spec) = parse_package_line(pkgline, &repodata.name, &self.env_config.channel.name, &self.options.arch) { + if let Ok(pkg_spec) = parse_package_line(pkgline, &repodata.name) { self.pkgname2lines .entry(pkg_spec.name.clone()) .or_insert_with(Vec::new) .push(pkgline.to_string()); - if let Some(provides) = &pkg_spec.provides { - for provide in provides { - if pkg_spec.format.is_some() { - let and_deps = match parse_requires(&pkg_spec.format.clone().unwrap().as_str(), provide) { - Ok(deps) => deps, - Err(e) => { - println!("Failed to parse requirement '{}': {}", provide, e); - continue; - } - }; - self.provide2pkgnames.insert(and_deps[0][0].capability.clone(), pkg_spec.name.clone()); - } - } - } - if pkg_spec.priority.is_some() && pkg_spec.priority.as_ref().unwrap() == "essential" { - self.essential_pkgnames.insert(pkg_spec.name.clone()); - } self.pkghash2spec.insert(pkg_spec.hash.clone(), pkg_spec); } } diff --git a/src/models.rs b/src/models.rs index b42656b..be4ebc4 100644 --- a/src/models.rs +++ b/src/models.rs @@ -4,7 +4,7 @@ use std::os::unix::net::UnixStream; use serde::{Deserialize, Serialize}; #[allow(dead_code)] -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Deserialize)] pub struct Dependency { pub pkgname: String, pub hash: String, @@ -12,7 +12,7 @@ pub struct Dependency { // $HOME/.cache/epkg/channel/${channel}/${repo}/${arch}/pkg-info/{2-char-prefix}/${pkghash}__${pkgname}__${pkgver}__${pkgrel}.json #[allow(dead_code)] -#[derive(Debug, Deserialize)] +#[derive(Debug, Clone, Deserialize)] pub struct Package { pub name: String, pub version: String, @@ -21,6 +21,7 @@ pub struct Package { pub dist: Option, pub hash: String, pub arch: String, + #[serde(rename = "sourcePkg")] pub source: Option, pub summary: Option, @@ -53,12 +54,18 @@ pub struct Repodata { pub name: String, #[serde(skip)] pub dir: String, + #[serde(skip)] + pub format: Option, #[serde(rename = "store-paths")] pub store_paths: Vec, #[serde(rename = "pkg-info")] pub pkg_infos: Vec, #[serde(rename = "pkg-files")] pub pkg_files: Vec, + #[serde(skip)] + pub provide2pkgnames: HashMap>, + #[serde(skip)] + pub essential_pkgnames: HashSet, } // $HOME/.cache/epkg/channel/${channel}/${repo}/${arch}/repodata/store-paths-{filehash}.txt @@ -97,10 +104,6 @@ pub struct PackageSpec { pub name: String, pub version: String, pub release: String, - pub source: Option, - pub provides: Option>, - pub priority: Option, - pub format: Option, } /* @@ -200,14 +203,15 @@ pub struct PackageManager { pub repos_data: Vec, pub env_config: EnvConfig, pub appbin_source: HashSet, - // loaded from repodata.store_paths files // pkghash2spec[hash] = PackageSpec // pkgname2lines[pkgname] = [pkgline] pub pkghash2spec: HashMap, pub pkgname2lines: HashMap>, - pub provide2pkgnames: HashMap, + pub provide2pkgnames: HashMap>, pub essential_pkgnames: HashSet, + // cache need to installing packages info + pub pkghash2pkg: HashMap, // loaded from env installed-packages.json pub installed_packages: HashMap, @@ -217,3 +221,4 @@ pub struct PackageManager { pub ipc_stream: Option, pub child_pid: Option, } + diff --git a/src/remove.rs b/src/remove.rs index cddbf40..338e985 100644 --- a/src/remove.rs +++ b/src/remove.rs @@ -92,7 +92,9 @@ impl PackageManager { let mut packages_to_keep: HashMap = self .installed_packages .iter() - .filter(|(pkgline, info)| info.depend_depth == 0 && !input_package_info.contains_key(*pkgline)) + .filter(|(pkgline, info)| (info.depend_depth == 0 && + !input_package_info.contains_key(*pkgline)) || + self.essential_pkgnames.contains(self.pkghash2spec.get(&pkgline[0..32]).unwrap().name.as_str())) .map(|(key, value)| (key.clone(), (*value).clone())) .collect(); self.collect_recursive_depends(&mut packages_to_keep)?; diff --git a/src/repo.rs b/src/repo.rs index 651d80e..4dc963d 100644 --- a/src/repo.rs +++ b/src/repo.rs @@ -1,6 +1,9 @@ +use std::collections::{HashMap, HashSet}; use std::fs; +use std::fs::File; use std::path::Path; use std::path::PathBuf; +use std::io::{BufRead, BufReader, BufWriter, Write}; use anyhow::Ok; use anyhow::Result; use anyhow::{anyhow, Context}; @@ -8,7 +11,145 @@ use crate::models::*; use crate::store::*; use crate::paths; use crate::download::*; -use crate::io::load_repodata_index; +use crate::parse_requires::*; +use crate::io::{load_repodata_index, load_package_json}; + +impl Repodata { + pub fn encode_provide_hashmap(&mut self, path: &str) -> Result<()> { + let target_path = Path::new(path); + if target_path.exists() { + fs::remove_file(&target_path)?; + } + let file = File::create(target_path)?; + let mut writer = BufWriter::new(file); + + for (key, values) in self.provide2pkgnames.iter() { + writer.write_all(key.as_bytes())?; + writer.write_all(b":")?; + + for (i, value) in values.iter().enumerate() { + writer.write_all(value.as_bytes())?; + if i < values.len() - 1 { + writer.write_all(b",")?; + } + } + + writer.write_all(b"\n")?; + } + + writer.flush()?; + Ok(()) + } + + pub fn decode_provide_hashmap(&mut self, file_path: &str) -> Result<()> { + let file = File::open(file_path)?; + let reader = BufReader::new(file); + let mut map: HashMap> = HashMap::new(); + + for line_result in reader.lines() { + let line = line_result?; + if let Some(colon_index) = line.find(":") { + let key = line[..colon_index].to_string(); + let values_str = &line[colon_index + 1..]; + let values: Vec = values_str + .split(",") + .map(|s| s.to_string()) + .collect(); + + map.insert(key, values); + } + } + self.provide2pkgnames = map; + + Ok(()) + } + + pub fn encode_essential_hashset(&mut self, path: &str) -> Result<()> { + let target_path = Path::new(path); + if target_path.exists() { + fs::remove_file(&target_path)?; + } + let file = File::create(target_path)?; + let mut writer = BufWriter::new(file); + + for item in self.essential_pkgnames.iter() { + writeln!(writer, "{}", item)?; + } + + writer.flush()?; + Ok(()) + } + + pub fn decode_essential_hashset(&mut self, file_path: &str) -> Result<()> { + let file = File::open(file_path)?; + let reader = BufReader::new(file); + let mut hashset: HashSet = HashSet::new(); + + for line in reader.lines() { + let line = line?; + hashset.insert(line); + } + self.essential_pkgnames = hashset; + + Ok(()) + } + + pub fn generate_repo_metadata(&mut self) -> Result<()> { + let pkg_info_dir = Path::new(&self.dir).parent().unwrap().join("pkg-info"); + for entry in &self.store_paths { + let file_path = format!( + "{}/{}", + self.dir, + entry.filename.strip_suffix(".zst").unwrap() + .splitn(3, '-') + .take(2) + .collect::>() + .join("-") + ); + let contents = fs::read_to_string(&file_path) + .with_context(|| format!("Failed to load store-paths from {}", file_path))?; + for pkgline in contents.lines() { + let file_path: String = format!("{}/{}/{}.json", + pkg_info_dir.display(), + &pkgline[0..2], + pkgline, + ); + let pkg_json = load_package_json(&file_path)?; + let format = match pkg_json.origin_url { + Some(ref url) => { + get_package_format(url) + }, + None => { + Some("rpm".to_string()) + } + }; + if let Some(provides) = &pkg_json.provides { + for provide in provides { + if format.is_some() { + let and_deps = match parse_requires(&format.clone().unwrap().as_str(), provide) { + std::result::Result::Ok(deps) => deps, + Err(e) => { + println!("Failed to parse requirement '{}': {}", provide, e); + continue; + } + }; + if let Some(pkgnames) = self.provide2pkgnames.get_mut(and_deps[0][0].capability.as_str()) { + pkgnames.push(pkg_json.name.clone()); + } else { + self.provide2pkgnames.insert(and_deps[0][0].capability.clone(), vec![pkg_json.name.clone()]); + } + } + } + } + if pkg_json.priority.is_some() && pkg_json.priority.as_ref().unwrap() == "essential" { + self.essential_pkgnames.insert(pkg_json.name.clone()); + } + } + } + + Ok(()) + } +} impl PackageManager { pub fn cache_repo(&mut self) -> Result<()> { @@ -50,7 +191,7 @@ fn download_repodata(base_url: &str, repodata_path: &PathBuf) -> Result<()> { Ok(()) } -fn unzst_all_repodatas(repodata_path: &PathBuf) -> Result<()> { +fn unzst_all_repodatas(repodata_path: &PathBuf) -> Result { let index_file_path = repodata_path.join("index.json"); let repo_data = load_repodata_index(index_file_path.to_str().unwrap()) .with_context(|| format!("Failed to load repodata from {}", repodata_path.display()))?; @@ -63,7 +204,7 @@ fn unzst_all_repodatas(repodata_path: &PathBuf) -> Result<()> { untar_zst(pkg_info_zst.to_str().unwrap(), repodata_path.parent().unwrap().to_str().unwrap(), false).unwrap(); } - Ok(()) + Ok(repo_data) } pub fn cache_repo_name(repo_name: &str, repo_url: &str) -> Result<()> { @@ -73,7 +214,8 @@ pub fn cache_repo_name(repo_name: &str, repo_url: &str) -> Result<()> { }; // [TODO] should check index.json pkg-info-xxx.zst store-paths-xxx.zst all valid // Check if index.json already exists - if local_cache_path.join("repodata/index.json").exists() { + if local_cache_path.join("repodata/provide2pkgnames.txt").exists() && + local_cache_path.join("repodata/essential_pkgnames.txt").exists() { return Ok(()); } println!("Caching repodata {} from {}", repo_name, repo_url); @@ -102,7 +244,10 @@ pub fn cache_repo_name(repo_name: &str, repo_url: &str) -> Result<()> { }, _ => return Err(anyhow!("Unsupported repo URL scheme")), } - unzst_all_repodatas(&repodata_path).unwrap(); + let mut repodata = unzst_all_repodatas(&repodata_path).unwrap(); + repodata.generate_repo_metadata().unwrap(); + repodata.encode_provide_hashmap(repodata_path.join("provide2pkgnames.txt").to_str().unwrap()).unwrap(); + repodata.encode_essential_hashset(repodata_path.join("essential_pkgnames.txt").to_str().unwrap()).unwrap(); println!("Cache repodata succeed: {}", repo_name); Ok(()) -- Gitee