From a8cee79b2f851df3b4911a7f5275ecc58dbda47c Mon Sep 17 00:00:00 2001 From: Wesley Shields Date: Thu, 1 Aug 2024 15:37:22 -0400 Subject: [PATCH 1/7] Add -m to scan. This adds support for the -m flag (print metadata) so that rule metadata is printed when a scan matches. Currently only outputs in text form, json will be next. --- cli/src/commands/scan.rs | 72 ++++++++++++++++++++++++++++++---------- 1 file changed, 55 insertions(+), 17 deletions(-) diff --git a/cli/src/commands/scan.rs b/cli/src/commands/scan.rs index 0be3eef6..f9dd0192 100644 --- a/cli/src/commands/scan.rs +++ b/cli/src/commands/scan.rs @@ -13,7 +13,7 @@ use superconsole::style::Stylize; use superconsole::{Component, Line, Lines, Span}; use yansi::Color::{Cyan, Red, Yellow}; use yansi::Paint; -use yara_x::{Rule, Rules, ScanError, Scanner}; +use yara_x::{MetaValue, Rule, Rules, ScanError, Scanner}; use crate::commands::{ compile_rules, external_var_parser, truncate_with_ellipsis, @@ -46,6 +46,13 @@ pub fn scan() -> Command { .help("Path to the file or directory that will be scanned") .value_parser(value_parser!(PathBuf)) ) + .arg( + arg!(-o --"output-format" ) + .help("Output format for results") + .long_help(help::OUTPUT_FORMAT_LONG_HELP) + .required(false) + .value_parser(value_parser!(OutputFormats)) + ) .arg( arg!(-e --"print-namespace") .help("Print rule namespace") @@ -59,6 +66,10 @@ pub fn scan() -> Command { .help("Print matching patterns, limited to the first N bytes") .value_parser(value_parser!(usize)) ) + .arg( + arg!(-m --"print-meta") + .help("Print rule metadata") + ) .arg( arg!(--"disable-console-logs") .help("Disable printing console log messages") @@ -122,13 +133,6 @@ pub fn scan() -> Command { .value_parser(external_var_parser) .action(ArgAction::Append) ) - .arg( - arg!(-o --"output-format" ) - .help("Output format for results") - .long_help(help::OUTPUT_FORMAT_LONG_HELP) - .required(false) - .value_parser(value_parser!(OutputFormats)) - ) } pub fn exec_scan(args: &ArgMatches) -> anyhow::Result<()> { @@ -373,6 +377,15 @@ fn print_rules_as_json( }) }; + /* + output + .send(Message::Info(format!( + "{}", + matching_rule.metadata().into_json() + ))) + .unwrap(); + */ + if print_strings || print_strings_limit.is_some() { let limit = print_strings_limit.unwrap_or(&STRINGS_LIMIT); for p in matching_rule.patterns() { @@ -432,6 +445,7 @@ fn print_rules_as_text( output: &Sender, ) { let print_namespace = args.get_flag("print-namespace"); + let print_meta = args.get_flag("print-meta"); let print_strings = args.get_flag("print-strings"); let print_strings_limit = args.get_one::("print-strings-limit"); @@ -440,20 +454,44 @@ fn print_rules_as_text( // `the `by_ref` method cannot be invoked on a trait object` #[allow(clippy::while_let_on_iterator)] while let Some(matching_rule) = rules.next() { - let line = if print_namespace { + let name = if print_namespace { format!( - "{}:{} {}", + "{}:{}", matching_rule.namespace().paint(Cyan).bold(), - matching_rule.identifier().paint(Cyan).bold(), - file_path.display(), + matching_rule.identifier().paint(Cyan).bold() ) } else { - format!( - "{} {}", - matching_rule.identifier().paint(Cyan).bold(), - file_path.display() - ) + format!("{}", matching_rule.identifier().paint(Cyan).bold()) + }; + + let meta = if print_meta { + let mut meta_str: String = String::from(""); + for (m, v) in matching_rule.metadata() { + // [a="b",c =1,d=true] + match v { + MetaValue::Bool(v) => { + meta_str.push_str(format!("{}={},", m, v).as_str()) + } + MetaValue::Integer(v) => { + meta_str.push_str(format!("{}={},", m, v).as_str()) + } + MetaValue::Float(v) => { + meta_str.push_str(format!("{}={},", m, v).as_str()) + } + MetaValue::String(v) => { + meta_str.push_str(format!("{}=\"{}\",", m, v).as_str()) + } + MetaValue::Bytes(v) => meta_str.push_str( + format!("{}=\"{}\",", m, v.escape_ascii()).as_str(), + ), + }; + } + format!("[{}]", &meta_str.as_str()[0..meta_str.len() - 1]) + } else { + format!("") }; + + let line = format!("{} {} {}", name, meta, file_path.display()); output.send(Message::Info(line)).unwrap(); if print_strings || print_strings_limit.is_some() { From 9629b5d00c7c6efd92477ce2b7e110f995335d3d Mon Sep 17 00:00:00 2001 From: Wesley Shields Date: Thu, 1 Aug 2024 15:44:45 -0400 Subject: [PATCH 2/7] Implement -m for json output. --- cli/src/commands/scan.rs | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/cli/src/commands/scan.rs b/cli/src/commands/scan.rs index 0b1ee72f..d85c14e2 100644 --- a/cli/src/commands/scan.rs +++ b/cli/src/commands/scan.rs @@ -354,6 +354,7 @@ fn print_rules_as_json( output: &Sender, ) { let print_namespace = args.get_flag("print-namespace"); + let print_meta = args.get_flag("print-meta"); let print_strings = args.get_flag("print-strings"); let print_strings_limit = args.get_one::("print-strings-limit"); @@ -378,14 +379,9 @@ fn print_rules_as_json( }) }; - /* - output - .send(Message::Info(format!( - "{}", - matching_rule.metadata().into_json() - ))) - .unwrap(); - */ + if print_meta { + json_rule["meta"] = matching_rule.metadata().into_json(); + } if print_strings || print_strings_limit.is_some() { let limit = print_strings_limit.unwrap_or(&STRINGS_LIMIT); From 6b608cf4e9b3959aa650df0e5686b9a28602563f Mon Sep 17 00:00:00 2001 From: Wesley Shields Date: Fri, 2 Aug 2024 10:39:20 -0400 Subject: [PATCH 3/7] Append metadata to avoid extra allocations. --- cli/src/commands/scan.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/cli/src/commands/scan.rs b/cli/src/commands/scan.rs index d85c14e2..9b0fc4f9 100644 --- a/cli/src/commands/scan.rs +++ b/cli/src/commands/scan.rs @@ -443,7 +443,7 @@ fn print_rules_as_text( // `the `by_ref` method cannot be invoked on a trait object` #[allow(clippy::while_let_on_iterator)] while let Some(matching_rule) = rules.next() { - let name = if print_namespace { + let mut line = if print_namespace { format!( "{}:{}", matching_rule.namespace().paint(Cyan).bold(), @@ -453,7 +453,7 @@ fn print_rules_as_text( format!("{}", matching_rule.identifier().paint(Cyan).bold()) }; - let meta = if print_meta { + if print_meta { let mut meta_str: String = String::from(""); for (m, v) in matching_rule.metadata() { // [a="b",c =1,d=true] @@ -475,12 +475,14 @@ fn print_rules_as_text( ), }; } - format!("[{}]", &meta_str.as_str()[0..meta_str.len() - 1]) - } else { - format!("") - }; + line = format!( + "{} [{}]", + line, + &meta_str.as_str()[0..meta_str.len() - 1] + ); + } - let line = format!("{} {} {}", name, meta, file_path.display()); + line = format!("{} {}", line, file_path.display()); output.send(Message::Info(line)).unwrap(); if print_strings || print_strings_limit.is_some() { From df11ab2bb5f80126383f84df5bd5ded8487539e7 Mon Sep 17 00:00:00 2001 From: Wesley Shields Date: Fri, 2 Aug 2024 11:48:23 -0400 Subject: [PATCH 4/7] Implement xor output in scan command. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If a string is using the xor modifier we now display the xor information (key and plaintext) in both text and json output modes. ``` wxs@mbp yara-x % ./target/debug/yr scan -o ndjson -s rules/a.yara ~/src/yara/tests/data/xor.out | jq . { "path": "/Users/wxs/src/yara/tests/data/xor.out", "rules": [ { "identifier": "freebsd", "strings": [ { "identifier": "$a", "start": 28, "length": 19, "data": "Uihr!qsnfs`l!b`oonu", "xor_key": 1, "plaintext": "This program cannot" }, { "identifier": "$a", "start": 52, "length": 19, "data": "Vjkq\\\"rpmepco\\\"acllmv", "xor_key": 2, "plaintext": "This program cannot" }, { "identifier": "$b", "start": 4, "length": 19, "data": "This program cannot" } ] } ] } wxs@mbp yara-x % ./target/debug/yr scan -s rules/a.yara ~/src/yara/tests/data/xor.out freebsd /Users/wxs/src/yara/tests/data/xor.out 0x1c:19:$a xor(0x1,This program cannot): Uihr!qsnfs`l!b`oonu 0x34:19:$a xor(0x2,This program cannot): Vjkq\"rpmepco\"acllmv 0x4:19:$b: This program cannot ──────────────────────────────────────────────────────────────────────────────────────────────────────────────── 1 file(s) scanned in 0.0s. 1 file(s) matched. wxs@mbp yara-x % ``` When using --print-strings-limit it looks like this in text mode: ``` wxs@mbp yara-x % ./target/debug/yr scan -s --print-strings-limit 5 rules/a.yara ~/src/yara/tests/data/xor.out freebsd /Users/wxs/src/yara/tests/data/xor.out 0x1c:19:$a xor(0x1,This ): Uihr! ... 14 more bytes 0x34:19:$a xor(0x2,This ): Vjkq\" ... 14 more bytes 0x4:19:$b: This ... 14 more bytes ──────────────────────────────────────────────────────────────────────────────────────────────────────────────── 1 file(s) scanned in 0.0s. 1 file(s) matched. wxs@mbp yara-x % ``` Not sure if we want to print the "... X more bytes" part in the plaintext or just leave it implied. I've also included a bug fix here where we were only printing the last matching pattern. --- cli/src/commands/scan.rs | 46 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/cli/src/commands/scan.rs b/cli/src/commands/scan.rs index 9b0fc4f9..74938756 100644 --- a/cli/src/commands/scan.rs +++ b/cli/src/commands/scan.rs @@ -385,8 +385,8 @@ fn print_rules_as_json( if print_strings || print_strings_limit.is_some() { let limit = print_strings_limit.unwrap_or(&STRINGS_LIMIT); + let mut match_vec: Vec = Vec::new(); for p in matching_rule.patterns() { - let mut match_vec: Vec = Vec::new(); for m in p.matches() { let match_range = m.range(); let match_data = m.data(); @@ -408,12 +408,32 @@ fn print_rules_as_json( .as_str(), ); } - let match_json = serde_json::json!({ + + let mut match_json = serde_json::json!({ "identifier": p.identifier(), "start": match_range.start, "length": match_range.len(), "data": s.as_str() }); + + match m.xor_key() { + Some(k) => { + let mut p = String::with_capacity(s.len()); + for b in + &match_data[..min(match_data.len(), *limit)] + { + for c in (b ^ k).escape_ascii() { + p.push_str( + format!("{}", c as char).as_str(), + ); + } + } + match_json["xor_key"] = serde_json::json!(k); + match_json["plaintext"] = serde_json::json!(p); + } + _ => {} + } + match_vec.push(match_json); } json_rule["strings"] = serde_json::json!(match_vec); @@ -456,7 +476,6 @@ fn print_rules_as_text( if print_meta { let mut meta_str: String = String::from(""); for (m, v) in matching_rule.metadata() { - // [a="b",c =1,d=true] match v { MetaValue::Bool(v) => { meta_str.push_str(format!("{}={},", m, v).as_str()) @@ -493,12 +512,31 @@ fn print_rules_as_text( let match_data = m.data(); let mut msg = format!( - "{:#x}:{}:{}: ", + "{:#x}:{}:{}", match_range.start, match_range.len(), p.identifier(), ); + match m.xor_key() { + Some(k) => { + msg.push_str(format!(" xor({:#x},", k).as_str()); + for b in + &match_data[..min(match_data.len(), *limit)] + { + for c in (b ^ k).escape_ascii() { + msg.push_str( + format!("{}", c as char).as_str(), + ); + } + } + msg.push_str(format!("): ").as_str()); + } + _ => { + msg.push_str(format!(": ").as_str()); + } + } + for b in &match_data[..min(match_data.len(), *limit)] { for c in b.escape_ascii() { msg.push_str(format!("{}", c as char).as_str()); From 160f98d188ab1909390fee94f4652093f8c20e1b Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 2 Aug 2024 18:18:57 +0200 Subject: [PATCH 5/7] chore: some little improvements. --- Cargo.lock | 1 + cli/Cargo.toml | 1 + cli/src/commands/scan.rs | 38 ++++++++++++++++++++++---------------- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2d58a7b1..a89682cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4898,6 +4898,7 @@ dependencies = [ "encoding_rs", "env_logger", "globwalk", + "itertools 0.13.0", "log", "pprof", "protobuf", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index c35e5a09..aaf1607e 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -41,6 +41,7 @@ anyhow = { workspace = true } clap = { workspace = true, features = ["cargo", "derive"] } clap_complete = { workspace = true } globwalk = { workspace = true } +itertools = { workspace = true } enable-ansi-support = { workspace = true } env_logger = { workspace = true, optional = true, features = ["auto-color"] } log = { workspace = true, optional = true } diff --git a/cli/src/commands/scan.rs b/cli/src/commands/scan.rs index 74938756..4552f0c2 100644 --- a/cli/src/commands/scan.rs +++ b/cli/src/commands/scan.rs @@ -9,6 +9,7 @@ use std::time::{Duration, Instant}; use anyhow::{bail, Context, Error}; use clap::{arg, value_parser, ArgAction, ArgMatches, Command, ValueEnum}; use crossbeam::channel::Sender; +use itertools::Itertools; use superconsole::style::Stylize; use superconsole::{Component, Line, Lines, Span}; use yansi::Color::{Cyan, Red, Yellow}; @@ -473,35 +474,40 @@ fn print_rules_as_text( format!("{}", matching_rule.identifier().paint(Cyan).bold()) }; - if print_meta { - let mut meta_str: String = String::from(""); - for (m, v) in matching_rule.metadata() { + let metadata = matching_rule.metadata(); + + if print_meta && !metadata.is_empty() { + line.push_str(" ["); + for (pos, (m, v)) in metadata.with_position() { match v { MetaValue::Bool(v) => { - meta_str.push_str(format!("{}={},", m, v).as_str()) + line.push_str(&format!("{}={}", m, v)) } MetaValue::Integer(v) => { - meta_str.push_str(format!("{}={},", m, v).as_str()) + line.push_str(&format!("{}={}", m, v)) } MetaValue::Float(v) => { - meta_str.push_str(format!("{}={},", m, v).as_str()) + line.push_str(&format!("{}={}", m, v)) } MetaValue::String(v) => { - meta_str.push_str(format!("{}=\"{}\",", m, v).as_str()) + line.push_str(&format!("{}=\"{}\"", m, v)) } - MetaValue::Bytes(v) => meta_str.push_str( - format!("{}=\"{}\",", m, v.escape_ascii()).as_str(), - ), + MetaValue::Bytes(v) => line.push_str(&format!( + "{}=\"{}\"", + m, + v.escape_ascii() + )), }; + if !matches!(pos, itertools::Position::Last) { + line.push(','); + } } - line = format!( - "{} [{}]", - line, - &meta_str.as_str()[0..meta_str.len() - 1] - ); + line.push(']'); } - line = format!("{} {}", line, file_path.display()); + line.push(' '); + line.push_str(&file_path.display().to_string()); + output.send(Message::Info(line)).unwrap(); if print_strings || print_strings_limit.is_some() { From 25a9ac61c33751bfb60d49f86e745fd2b9064a8d Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 2 Aug 2024 18:24:03 +0200 Subject: [PATCH 6/7] style: fix clippy warnings --- cli/src/commands/scan.rs | 29 +++++++++++++---------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/cli/src/commands/scan.rs b/cli/src/commands/scan.rs index 4552f0c2..e21ca510 100644 --- a/cli/src/commands/scan.rs +++ b/cli/src/commands/scan.rs @@ -417,22 +417,19 @@ fn print_rules_as_json( "data": s.as_str() }); - match m.xor_key() { - Some(k) => { - let mut p = String::with_capacity(s.len()); - for b in - &match_data[..min(match_data.len(), *limit)] - { - for c in (b ^ k).escape_ascii() { - p.push_str( - format!("{}", c as char).as_str(), - ); - } + if let Some(k) = m.xor_key() { + let mut p = String::with_capacity(s.len()); + for b in + &match_data[..min(match_data.len(), *limit)] + { + for c in (b ^ k).escape_ascii() { + p.push_str( + format!("{}", c as char).as_str(), + ); } - match_json["xor_key"] = serde_json::json!(k); - match_json["plaintext"] = serde_json::json!(p); } - _ => {} + match_json["xor_key"] = serde_json::json!(k); + match_json["plaintext"] = serde_json::json!(p); } match_vec.push(match_json); @@ -536,10 +533,10 @@ fn print_rules_as_text( ); } } - msg.push_str(format!("): ").as_str()); + msg.push_str("): "); } _ => { - msg.push_str(format!(": ").as_str()); + msg.push_str(": "); } } From 68385153bfb2ee2cc504404b1d1c3f1a63548f3f Mon Sep 17 00:00:00 2001 From: "Victor M. Alvarez" Date: Fri, 2 Aug 2024 18:27:15 +0200 Subject: [PATCH 7/7] style: apply `rustfmt` --- cli/src/commands/scan.rs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/cli/src/commands/scan.rs b/cli/src/commands/scan.rs index e21ca510..c8735ebf 100644 --- a/cli/src/commands/scan.rs +++ b/cli/src/commands/scan.rs @@ -419,19 +419,14 @@ fn print_rules_as_json( if let Some(k) = m.xor_key() { let mut p = String::with_capacity(s.len()); - for b in - &match_data[..min(match_data.len(), *limit)] - { + for b in &match_data[..min(match_data.len(), *limit)] { for c in (b ^ k).escape_ascii() { - p.push_str( - format!("{}", c as char).as_str(), - ); + p.push_str(format!("{}", c as char).as_str()); } } match_json["xor_key"] = serde_json::json!(k); match_json["plaintext"] = serde_json::json!(p); } - match_vec.push(match_json); } json_rule["strings"] = serde_json::json!(match_vec);