From e0f9e4a89259f5cb8d1ade394fa31f95a3232648 Mon Sep 17 00:00:00 2001 From: owent Date: Mon, 28 Aug 2023 14:10:44 +0800 Subject: [PATCH] =?UTF-8?q?=E8=B0=83=E6=95=B4=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 5 +- src/exec/Cargo.toml | 2 +- src/exec/src/main.rs | 442 ++++++++++++++++++++++++---------------- src/protocol/Cargo.toml | 2 +- 4 files changed, 276 insertions(+), 175 deletions(-) diff --git a/README.md b/README.md index 1ef0dfd..c8dff1c 100644 --- a/README.md +++ b/README.md @@ -60,8 +60,9 @@ This can be used to generate string table data source for UnrealEngine(UE). --silence --string-table-pretty # strings will be saved in string-table.json and string-table.txt -# We can also use --string-table-value-regex-rule and --string-table-value-regex-file to filter contents -# and use --string-table-field-path-file to filter contents by which protocol paths +# You can also use --string-table-include-value-regex-rule/--string-table-include-value-regex-file and --string-table-exclude-value-regex-rule/--string-table-exclude-value-regex-file to filter contents. +# Use --string-table-include-field-path-file/--string-table-exclude-field-path-file to filter contents by protocol field paths +# Or use --string-table-include-message-path-file/--string-table-exclude-message-path-file to filter contents by protocol message paths ``` https://doc.rust-lang.org/cargo/reference/config.html diff --git a/src/exec/Cargo.toml b/src/exec/Cargo.toml index d7a4ffe..bf7d196 100644 --- a/src/exec/Cargo.toml +++ b/src/exec/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "xresloader-dump-bin" -version = "2.2.1" +version = "2.3.0" authors = ["owent "] license = "MIT" description = "A tool to dump human readable text from binary output of xresloader" diff --git a/src/exec/src/main.rs b/src/exec/src/main.rs index 19de22d..359d6dc 100644 --- a/src/exec/src/main.rs +++ b/src/exec/src/main.rs @@ -33,58 +33,88 @@ use file_descriptor_index::FileDescriptorIndex; /// Simple program to greet a person #[derive(Parser, Debug)] -#[clap(version, about, long_about = None)] +#[command(version, about, long_about = None)] struct DumpOptions { /// pb files(can be used mulpitle times) - #[clap(short, long, value_parser, action = ArgAction::Append)] + #[arg(short, long, value_parser, action = ArgAction::Append)] pb_file: Vec, /// binary files generated by xresloader(can be used mulpitle times) - #[clap(short, long, value_parser, action = ArgAction::Append)] + #[arg(short, long, value_parser, action = ArgAction::Append)] bin_file: Vec, /// Debug mode - #[clap(long, value_parser, default_value = "false")] + #[arg(long, value_parser, default_value = "false")] debug: bool, /// Pretty mode - #[clap(long, value_parser, default_value = "false")] + #[arg(long, value_parser, default_value = "false")] pretty: bool, /// Plain mode - #[clap(long, value_parser, default_value = "false")] + #[arg(long, value_parser, default_value = "false")] plain: bool, /// head_only mode - #[clap(long, value_parser, default_value = "false")] + #[arg(long, value_parser, default_value = "false")] head_only: bool, /// silence mode - #[clap(long, value_parser, default_value = "false")] + #[arg(long, value_parser, default_value = "false")] silence: bool, /// Output string table as json - #[clap(long, value_parser, default_value = "")] + #[arg( + long, + value_parser, + default_value = "", + value_name = "OUTPUT JSON FILE PATH" + )] output_string_table_json: String, /// Output string table as text lines - #[clap(long, value_parser, default_value = "")] + #[arg( + long, + value_parser, + default_value = "", + value_name = "OUTPUT TEXT FILE PATH" + )] output_string_table_text: String, - /// Field value matching for string table(can be used mulpitle times) - #[clap(long, value_parser, action = ArgAction::Append)] - string_table_value_regex_rule: Vec, + /// Field value include matching rule for string table(can be used mulpitle times) + #[arg(long, value_parser, action = ArgAction::Append, value_name = "REGEX")] + string_table_include_value_regex_rule: Vec, - /// Load field value matching rule(regex) from file for string table(can be used mulpitle times) - #[clap(long, value_parser, action = ArgAction::Append)] - string_table_value_regex_file: Vec, + /// Load field value include matching rule(regex) from file for string table(can be used mulpitle times) + #[arg(long, value_parser, action = ArgAction::Append, value_name = "REGEX")] + string_table_include_value_regex_file: Vec, - /// Load field path from file for string table(can be used mulpitle times) - #[clap(long, value_parser, action = ArgAction::Append)] - string_table_field_path_file: Vec, + /// Field value exclude matching rule for string table(can be used mulpitle times) + #[arg(long, value_parser, action = ArgAction::Append, value_name = "REGEX FILE PATH")] + string_table_exclude_value_regex_rule: Vec, + + /// Load field value exclude matching rule(regex) from file for string table(can be used mulpitle times) + #[arg(long, value_parser, action = ArgAction::Append, value_name = "REGEX FILE PATH")] + string_table_exclude_value_regex_file: Vec, + + /// Load field include path from file for string table(can be used mulpitle times) + #[arg(long, value_parser, action = ArgAction::Append, value_name = "FILE PATH")] + string_table_include_field_path_file: Vec, + + /// Load field exclude path from file for string table(can be used mulpitle times) + #[arg(long, value_parser, action = ArgAction::Append, value_name = "FILE PATH")] + string_table_exclude_field_path_file: Vec, + + /// Load message exclude path from file for string table(can be used mulpitle times) + #[arg(long, value_parser, action = ArgAction::Append, value_name = "FILE PATH")] + string_table_exclude_message_path_file: Vec, + + /// Load message include path from file for string table(can be used mulpitle times) + #[arg(long, value_parser, action = ArgAction::Append, value_name = "FILE PATH")] + string_table_include_message_path_file: Vec, /// String table pretty mode - #[clap(long, value_parser, default_value = "false")] + #[arg(long, value_parser, default_value = "false")] string_table_pretty: bool, } @@ -120,14 +150,75 @@ struct StringTableContent { pub body: HashMap>, } +#[derive(Default)] +struct StringTableFilter { + pub value_include_regex_rules: Vec, + pub value_exclude_regex_rules: Vec, + pub include_message_paths: HashSet, + pub exclude_message_paths: HashSet, + pub include_field_paths: HashSet, + pub exclude_field_paths: HashSet, +} + +impl StringTableFilter { + pub fn filter_value(&self, input: &str) -> bool { + if !self.value_include_regex_rules.is_empty() { + let mut matched = false; + for rule in &self.value_include_regex_rules { + if rule.is_match(input) { + matched = true; + break; + } + } + if !matched { + return false; + } + } + for rule in &self.value_exclude_regex_rules { + if rule.is_match(input) { + return false; + } + } + + true + } + + pub fn filter_field_full_name(&self, input: &str) -> bool { + if !self.include_field_paths.is_empty() && !self.include_field_paths.contains(input) { + return false; + } + + if self.exclude_field_paths.contains(input) { + return false; + } + + true + } + + pub fn filter_message_full_name(&self, input: &str) -> bool { + if !self.include_message_paths.is_empty() && !self.include_message_paths.contains(input) { + return false; + } + + if self.exclude_message_paths.contains(input) { + return false; + } + + true + } +} + impl StringTableContent { pub fn load_message( &mut self, message: &dyn MessageDyn, - value_rules: &Vec, - field_paths: &HashSet, + filter: &StringTableFilter, data_source: &StringTableDataSource, ) { + if !filter.filter_message_full_name(message.descriptor_dyn().full_name()) { + return; + } + message .descriptor_dyn() .fields() @@ -136,27 +227,17 @@ impl StringTableContent { if let Some(v) = field.get_singular(message) { match v { protobuf::reflect::ReflectValueRef::Message(m) => { - self.load_message(m.deref(), value_rules, field_paths, data_source); + self.load_message(m.deref(), filter, data_source); } - protobuf::reflect::ReflectValueRef::String(_s) => { - if !field_paths.is_empty() - && !field_paths.contains(&field.full_name()) - { + protobuf::reflect::ReflectValueRef::String(s) => { + if !filter.filter_field_full_name(&field.full_name()) { return; } - if !value_rules.is_empty() { - let mut matched = false; - for rule in value_rules { - if rule.is_match(v.to_string().as_str()) { - matched = true; - break; - } - } - if !matched { - return; - } + if !filter.filter_value(s) { + return; } + let value = v.to_string(); if let Some(item) = self.body.get_mut(&value) { item.push_back(StringTableItemSource { @@ -176,11 +257,8 @@ impl StringTableContent { } } } - protobuf::reflect::RuntimeFieldType::Repeated(t) => { - if !field_paths.is_empty() - && t == protobuf::reflect::RuntimeType::String - && !field_paths.contains(&field.full_name()) - { + protobuf::reflect::RuntimeFieldType::Repeated(_) => { + if !filter.filter_field_full_name(&field.full_name()) { return; } @@ -189,20 +267,11 @@ impl StringTableContent { .into_iter() .for_each(|v| match v { protobuf::reflect::ReflectValueRef::Message(m) => { - self.load_message(m.deref(), value_rules, field_paths, data_source); + self.load_message(m.deref(), filter, data_source); } - protobuf::reflect::ReflectValueRef::String(_s) => { - if !value_rules.is_empty() { - let mut matched = false; - for rule in value_rules { - if rule.is_match(v.to_string().as_str()) { - matched = true; - break; - } - } - if !matched { - return; - } + protobuf::reflect::ReflectValueRef::String(s) => { + if !filter.filter_value(s) { + return; } let value = v.to_string(); @@ -227,26 +296,15 @@ impl StringTableContent { field.get_map(message).into_iter().for_each(|(k, v)| { match k { protobuf::reflect::ReflectValueRef::Message(m) => { - self.load_message(m.deref(), value_rules, field_paths, data_source); + self.load_message(m.deref(), filter, data_source); } - protobuf::reflect::ReflectValueRef::String(_s) => { - if !field_paths.is_empty() - && !field_paths.contains(&field.full_name()) - { + protobuf::reflect::ReflectValueRef::String(s) => { + if !filter.filter_field_full_name(&field.full_name()) { return; } - if !value_rules.is_empty() { - let mut matched = false; - for rule in value_rules { - if rule.is_match(v.to_string().as_str()) { - matched = true; - break; - } - } - if !matched { - return; - } + if !filter.filter_value(s) { + return; } let value = v.to_string(); @@ -269,26 +327,15 @@ impl StringTableContent { match v { protobuf::reflect::ReflectValueRef::Message(m) => { - self.load_message(m.deref(), value_rules, field_paths, data_source); + self.load_message(m.deref(), filter, data_source); } - protobuf::reflect::ReflectValueRef::String(_s) => { - if !field_paths.is_empty() - && !field_paths.contains(&field.full_name()) - { + protobuf::reflect::ReflectValueRef::String(s) => { + if !filter.filter_field_full_name(&field.full_name()) { return; } - if !value_rules.is_empty() { - let mut matched = false; - for rule in value_rules { - if rule.is_match(v.to_string().as_str()) { - matched = true; - break; - } - } - if !matched { - return; - } + if !filter.filter_value(s) { + return; } let value = v.to_string(); @@ -426,6 +473,138 @@ impl log::Log for Logger { } } +fn load_file_by_lines

(file_path: &str, file_type: &str, has_error: &mut bool, func: P) +where + P: FnMut(&str) -> Result<(), String>, +{ + let mut invoke = func; + match std::fs::OpenOptions::new() + .read(true) + .write(false) + .open(file_path) + { + Ok(f) => { + let reader = BufReader::new(f); + reader + .lines() + .map_while(Result::ok) + .map(|s| s.trim().to_string()) + .filter(|s| !s.is_empty() && !s.starts_with('#')) + .for_each(|line| { + if let Err(e) = invoke(&line) { + error!( + "Invalid {}: \"{}\" in file {}, {}, ignore this line", + file_type, line, file_path, e + ); + *has_error = true; + } + }) + } + Err(e) => { + error!( + "Try to open {} file {} failed, {}, ignore this file", + file_type, file_path, e + ); + *has_error = true; + } + } +} + +fn build_string_table_filter(args: &DumpOptions) -> (bool, StringTableFilter) { + let mut ret: StringTableFilter = StringTableFilter::default(); + let mut has_error = false; + + for regex_rule in &args.string_table_include_value_regex_rule { + match regex::Regex::new(regex_rule) { + Ok(r) => { + ret.value_include_regex_rules.push(r); + } + Err(e) => { + error!( + "Invalid regex rule: {}, {}, ignore this rule", + regex_rule, e + ); + has_error = true; + } + } + } + + for file_path in &args.string_table_include_value_regex_file { + load_file_by_lines( + file_path, + "regex rule", + &mut has_error, + |line| match regex::Regex::new(line) { + Ok(r) => { + ret.value_include_regex_rules.push(r); + Ok(()) + } + Err(e) => Err(format!("{}", e)), + }, + ); + } + + for regex_rule in &args.string_table_exclude_value_regex_rule { + match regex::Regex::new(regex_rule) { + Ok(r) => { + ret.value_exclude_regex_rules.push(r); + } + Err(e) => { + error!( + "Invalid regex rule: {}, {}, ignore this rule", + regex_rule, e + ); + has_error = true; + } + } + } + + for file_path in &args.string_table_exclude_value_regex_file { + load_file_by_lines( + file_path, + "regex rule", + &mut has_error, + |line| match regex::Regex::new(line) { + Ok(r) => { + ret.value_exclude_regex_rules.push(r); + Ok(()) + } + Err(e) => Err(format!("{}", e)), + }, + ); + } + + for field_path_file in &args.string_table_include_field_path_file { + load_file_by_lines(field_path_file, "field path", &mut has_error, |line| { + ret.include_field_paths.insert(line.to_string()); + Ok(()) + }); + } + + for field_path_file in &args.string_table_exclude_field_path_file { + load_file_by_lines(field_path_file, "field path", &mut has_error, |line| { + ret.exclude_field_paths.insert(line.to_string()); + Ok(()) + }); + } + + for field_path_file in &args.string_table_include_message_path_file { + load_file_by_lines(field_path_file, "message path", &mut has_error, |line| { + ret.include_message_paths.insert(line.to_string()); + Ok(()) + }); + } + + for field_path_file in &args.string_table_exclude_message_path_file { + load_file_by_lines(field_path_file, "message path", &mut has_error, |line| { + ret.exclude_message_paths.insert(line.to_string()); + Ok(()) + }); + } + + (has_error, ret) +} + fn main() { let args = DumpOptions::parse(); @@ -437,6 +616,8 @@ fn main() { let mut desc_index = FileDescriptorIndex::new(); + let mut string_tables: Vec = vec![]; + let (has_string_table_error, string_table_filter) = build_string_table_filter(&args); for pb_file in args.pb_file { debug!("Load pb file: {}", pb_file); match std::fs::OpenOptions::new() @@ -474,88 +655,7 @@ fn main() { } } - let mut has_error = false; - - let mut string_tables: Vec = vec![]; - let mut string_table_value_regex_rules: Vec = vec![]; - let mut string_table_field_paths: HashSet = HashSet::new(); - for regex_rule in &args.string_table_value_regex_rule { - match regex::Regex::new(regex_rule) { - Ok(r) => { - string_table_value_regex_rules.push(r); - } - Err(e) => { - error!( - "Invalid regex rule: {}, {}, ignore this rule", - regex_rule, e - ); - has_error = true; - } - } - } - for field_path_file in &args.string_table_value_regex_file { - match std::fs::OpenOptions::new() - .read(true) - .write(false) - .open(field_path_file) - { - Ok(f) => { - let reader = BufReader::new(f); - for line in reader - .lines() - .map_while(Result::ok) - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty() && !s.starts_with('#')) - { - match regex::Regex::new(&line) { - Ok(r) => { - string_table_value_regex_rules.push(r); - } - Err(e) => { - error!( - "Invalid regex rule: {} in file {}, {}, ignore this rule", - line, field_path_file, e - ); - has_error = true; - } - } - } - } - Err(e) => { - error!( - "Try to open regex rule file {} failed, {}, ignore this file", - field_path_file, e - ); - has_error = true; - } - } - } - for field_path_file in &args.string_table_field_path_file { - match std::fs::OpenOptions::new() - .read(true) - .write(false) - .open(field_path_file) - { - Ok(f) => { - let reader = BufReader::new(f); - for line in reader - .lines() - .map_while(Result::ok) - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty() && !s.starts_with('#')) - { - string_table_field_paths.insert(line); - } - } - Err(e) => { - error!( - "Try to open file {} failed, {}, ignore this file", - field_path_file, e - ); - has_error = true; - } - } - } + let mut has_error = has_string_table_error; for bin_file in args.bin_file { debug!("Load xresloader output binary file: {}", bin_file); @@ -671,7 +771,7 @@ fn main() { match message_descriptor.parse_from_bytes(row_data_block) { Ok(message) => { if let Some(ref mut string_table) = current_string_table { - string_table.load_message(message.as_ref(), &string_table_value_regex_rules, &string_table_field_paths, &fallback_data_source); + string_table.load_message(message.as_ref(), &string_table_filter, &fallback_data_source); } if args.head_only || args.silence { diff --git a/src/protocol/Cargo.toml b/src/protocol/Cargo.toml index 079e21e..d25b284 100644 --- a/src/protocol/Cargo.toml +++ b/src/protocol/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "xresloader-protocol" -version = "2.2.0" +version = "2.3.0" authors = ["owent "] license = "MIT" description = "A tool to dump human readable text from binary output of xresloader"