Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Exact matches bypass re match for masking #6

Open
wants to merge 3 commits into
base: support-e
Choose a base branch
from
Open

Exact matches bypass re match for masking #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0bcee65
Exact matches bypass re match for masking
bartavelle Mar 14, 2022
a37ae87
WIP
bartavelle Mar 15, 2022
6d2d6b2
Hashed masked values
bartavelle Mar 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions curiefense/curieproxy/rust/curiefense/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ uuid = { version = "0.8", features = ["serde", "v4"] }
multipart = "0.17.1"
xmlparser = "0.13.3"
nom = "6.2.1"
sha2 = "0.10"

# iptools dependencies
rand = "0.8.3"
Expand Down
1 change: 1 addition & 0 deletions curiefense/curieproxy/rust/curiefense/src/config/raw.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,7 @@ pub struct RawWafProfile {
pub cookies: RawWafProperties,
#[serde(default)]
pub path: RawWafProperties,
pub masking_seed: Option<String>,
}

#[derive(Debug, Deserialize, Serialize, Clone)]
Expand Down
3 changes: 3 additions & 0 deletions curiefense/curieproxy/rust/curiefense/src/config/waf.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ pub struct WafProfile {
pub ignore_alphanum: bool,
pub sections: Section<WafSection>,
pub decoding: Vec<Transformation>,
pub masking_seed: String,
}

impl Default for WafProfile {
Expand Down Expand Up @@ -76,6 +77,7 @@ impl Default for WafProfile {
},
},
decoding: Transformation::DEFAULTPOLICY.to_vec(),
masking_seed: "DEFAULT SEED".into(),
}
}
}
Expand Down Expand Up @@ -220,6 +222,7 @@ fn convert_entry(entry: RawWafProfile) -> anyhow::Result<(String, WafProfile)> {
Transformation::HtmlEntitiesDecode,
Transformation::UnicodeDecode,
],
masking_seed: entry.masking_seed.unwrap_or_else(|| "DEFAULTSEED".into()),
},
))
}
Expand Down
33 changes: 25 additions & 8 deletions curiefense/curieproxy/rust/curiefense/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ pub fn inspect_generic_request_map<GH: Grasshopper>(
return (Decision::Pass, tags, reqinfo);
}
};
let masking_seed = securitypolicy.waf_profile.masking_seed.as_bytes();
logs.debug("request tagged");
tags.extend(ntags);
tags.insert_qualified("securitypolicy", &nm);
Expand All @@ -127,14 +128,18 @@ pub fn inspect_generic_request_map<GH: Grasshopper>(
.and_then(|gh| challenge_phase02(gh, &reqinfo.rinfo.qinfo.uri, &reqinfo.headers))
{
// TODO, check for monitor
return (dec, tags, masking(reqinfo, &securitypolicy.waf_profile));
return (dec, tags, masking(masking_seed, reqinfo, &securitypolicy.waf_profile));
}
logs.debug("challenge phase2 ignored");

if let SimpleDecision::Action(action, reason) = globalfilter_dec {
let decision = action.to_decision(is_human, &mgh, &reqinfo.headers, reason);
if decision.is_final() {
return (decision, tags, masking(reqinfo, &securitypolicy.waf_profile));
return (
decision,
tags,
masking(masking_seed, reqinfo, &securitypolicy.waf_profile),
);
}
}

Expand All @@ -145,7 +150,11 @@ pub fn inspect_generic_request_map<GH: Grasshopper>(
Ok(SimpleDecision::Action(a, reason)) => {
let decision = a.to_decision(is_human, &mgh, &reqinfo.headers, reason);
if decision.is_final() {
return (decision, tags, masking(reqinfo, &securitypolicy.waf_profile));
return (
decision,
tags,
masking(masking_seed, reqinfo, &securitypolicy.waf_profile),
);
}
}
}
Expand All @@ -156,7 +165,11 @@ pub fn inspect_generic_request_map<GH: Grasshopper>(
if let SimpleDecision::Action(action, reason) = limit_check {
let decision = action.to_decision(is_human, &mgh, &reqinfo.headers, reason);
if decision.is_final() {
return (decision, tags, masking(reqinfo, &securitypolicy.waf_profile));
return (
decision,
tags,
masking(masking_seed, reqinfo, &securitypolicy.waf_profile),
);
}
}
logs.debug(format!("limit checks done ({} limits)", securitypolicy.limits.len()));
Expand All @@ -168,7 +181,11 @@ pub fn inspect_generic_request_map<GH: Grasshopper>(
AclResult::Passthrough(dec) => {
if dec.allowed {
logs.debug("ACL passthrough detected");
return (Decision::Pass, tags, masking(reqinfo, &securitypolicy.waf_profile));
return (
Decision::Pass,
tags,
masking(masking_seed, reqinfo, &securitypolicy.waf_profile),
);
} else {
logs.debug("ACL force block detected");
Some((0, dec.tags))
Expand Down Expand Up @@ -202,7 +219,7 @@ pub fn inspect_generic_request_map<GH: Grasshopper>(
return (
challenge_phase01(&gh, ua, dtags),
tags,
masking(reqinfo, &securitypolicy.waf_profile),
masking(masking_seed, reqinfo, &securitypolicy.waf_profile),
);
}
(gua, ggh) => {
Expand All @@ -226,7 +243,7 @@ pub fn inspect_generic_request_map<GH: Grasshopper>(
return (
acl_block(true, cde, &tgs),
tags,
masking(reqinfo, &securitypolicy.waf_profile),
masking(masking_seed, reqinfo, &securitypolicy.waf_profile),
);
}
}
Expand Down Expand Up @@ -258,7 +275,7 @@ pub fn inspect_generic_request_map<GH: Grasshopper>(
}
},
tags,
masking(reqinfo, &securitypolicy.waf_profile),
masking(masking_seed, reqinfo, &securitypolicy.waf_profile),
)
}

Expand Down
13 changes: 13 additions & 0 deletions curiefense/curieproxy/rust/curiefense/src/requestfields.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use crate::config::waf::Transformation;
use crate::utils::decoders::DecodingResult;
use crate::utils::masker;
use std::collections::{hash_map, HashMap};

#[derive(Debug, Clone, PartialEq, Eq)]
Expand Down Expand Up @@ -94,6 +95,18 @@ impl RequestField {
out
}

pub fn mask(&mut self, masking_seed: &[u8], key: &str) -> bool {
let mut o = false;
if let Some(v) = self.fields.get_mut(key) {
*v = masker(masking_seed, v);
o = true;
}
if let Some(parent) = key.strip_suffix(":decoded") {
o |= self.mask(masking_seed, parent);
}
o
}

pub fn iter_mut(&mut self) -> hash_map::IterMut<'_, String, String> {
self.fields.iter_mut()
}
Expand Down
10 changes: 10 additions & 0 deletions curiefense/curieproxy/rust/curiefense/src/utils.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ use crate::maxmind::get_country;
use itertools::Itertools;
use maxminddb::geoip2::model;
use serde_json::json;
use sha2::{Digest, Sha256};
use std::collections::HashMap;
use std::net::IpAddr;

Expand Down Expand Up @@ -439,6 +440,15 @@ pub fn check_selector_cond(reqinfo: &RequestInfo, tags: &Tags, sel: &RequestSele
}
}

pub fn masker(seed: &[u8], value: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(seed);
hasher.update(value.as_bytes());
let bytes = hasher.finalize();
let hash_str = format!("{:x}", bytes);
format!("MASKED{{{}}}", &hash_str[0..7])
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down
80 changes: 48 additions & 32 deletions curiefense/curieproxy/rust/curiefense/src/waf.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
use hyperscan::Matching;
use libinjection::{sqli, xss};
use regex::Regex;
use serde_json::{json, Value};
use std::collections::{HashMap, HashSet};

Expand Down Expand Up @@ -323,38 +324,53 @@ fn hyperscan(
})
}

fn mask_section(sec: &mut RequestField, section: &WafSection) -> bool {
fn mask_section(masking_seed: &[u8], sec: &mut RequestField, section: &WafSection) -> bool {
let match_value = |mre: &Option<Regex>, value| mre.as_ref().map(|re| re.is_match(value)).unwrap_or(true);
let to_mask: HashSet<String> = sec
.iter()
.filter_map(|(name, value)| {
if let Some(e) = section.names.get(name) {
if e.mask && match_value(&e.reg, value) {
Some(name.into())
} else {
None
}
} else if section
.regex
.iter()
.any(|(re, wfe)| wfe.mask && re.is_match(name) && match_value(&wfe.reg, value))
{
Some(name.into())
} else {
None
}
})
.collect();
let mut out = false;
for (name, value) in sec.iter_mut() {
if section.names.get(name).map(|e| e.mask).unwrap_or(false)
|| section.regex.iter().any(|(re, v)| v.mask && re.is_match(name))
{
*value = "*MASKED*".to_string();
out = true;
}
for m in to_mask {
out |= sec.mask(masking_seed, &m);
}
out
}

pub fn masking(req: RequestInfo, profile: &WafProfile) -> RequestInfo {
pub fn masking(masking_seed: &[u8], req: RequestInfo, profile: &WafProfile) -> RequestInfo {
let mut ri = req;
mask_section(&mut ri.headers, profile.sections.get(SectionIdx::Headers));
let cookies_masked = mask_section(&mut ri.cookies, profile.sections.get(SectionIdx::Cookies));
mask_section(masking_seed, &mut ri.headers, profile.sections.get(SectionIdx::Headers));
let cookies_masked = mask_section(masking_seed, &mut ri.cookies, profile.sections.get(SectionIdx::Cookies));
if cookies_masked {
ri.headers.fields.insert("cookie".into(), "*REDACTED*".into());
ri.headers.mask(masking_seed, "cookie");
}

let arg_masked = mask_section(&mut ri.rinfo.qinfo.args, profile.sections.get(SectionIdx::Args));
if arg_masked {
ri.rinfo.qinfo.query = "*MASKED*".into();
ri.rinfo.qinfo.uri = "*MASKED*".into();
}
let path_masked = mask_section(&mut ri.rinfo.qinfo.path_as_map, profile.sections.get(SectionIdx::Path));
if path_masked {
ri.rinfo.qinfo.qpath = "*MASKED*".into();
ri.rinfo.qinfo.query = "*MASKED*".into();
ri.rinfo.qinfo.uri = "*MASKED*".into();
}
mask_section(
masking_seed,
&mut ri.rinfo.qinfo.args,
profile.sections.get(SectionIdx::Args),
);
mask_section(
masking_seed,
&mut ri.rinfo.qinfo.path_as_map,
profile.sections.get(SectionIdx::Path),
);
ri
}

Expand Down Expand Up @@ -383,7 +399,7 @@ mod test {
fn no_masking() {
let rinfo = test_request_info();
let profile = WafProfile::default();
let masked = masking(rinfo.clone(), &profile);
let masked = masking(b"DEFAULT SEED", rinfo.clone(), &profile);
assert_eq!(rinfo.headers, masked.headers);
assert_eq!(rinfo.cookies, masked.cookies);
assert_eq!(rinfo.rinfo.qinfo.args, masked.rinfo.qinfo.args);
Expand All @@ -404,11 +420,11 @@ mod test {
let mut profile = WafProfile::default();
let asection = profile.sections.at(SectionIdx::Args);
asection.regex = vec![(regex::Regex::new(".").unwrap(), maskentry())];
let masked = masking(rinfo.clone(), &profile);
let masked = masking(b"DEFAULT SEED", rinfo.clone(), &profile);
assert_eq!(rinfo.headers, masked.headers);
assert_eq!(rinfo.cookies, masked.cookies);
assert_eq!(
RequestField::raw_create(&[], &[("arg1", "*MASKED*"), ("arg2", "*MASKED*")]),
RequestField::raw_create(&[], &[("arg1", "MASKED{172f0f6}"), ("arg2", "MASKED{c8dc0c0}")]),
masked.rinfo.qinfo.args
);
}
Expand All @@ -419,11 +435,11 @@ mod test {
let mut profile = WafProfile::default();
let asection = profile.sections.at(SectionIdx::Args);
asection.regex = vec![(regex::Regex::new("1").unwrap(), maskentry())];
let masked = masking(rinfo.clone(), &profile);
let masked = masking(b"DEFAULT SEED", rinfo.clone(), &profile);
assert_eq!(rinfo.headers, masked.headers);
assert_eq!(rinfo.cookies, masked.cookies);
assert_eq!(
RequestField::raw_create(&[], &[("arg1", "*MASKED*"), ("arg2", "avalue2")]),
RequestField::raw_create(&[], &[("arg1", "MASKED{172f0f6}"), ("arg2", "avalue2")]),
masked.rinfo.qinfo.args
);
}
Expand All @@ -434,11 +450,11 @@ mod test {
let mut profile = WafProfile::default();
let asection = profile.sections.at(SectionIdx::Args);
asection.names = ["arg1"].iter().map(|k| (k.to_string(), maskentry())).collect();
let masked = masking(rinfo.clone(), &profile);
let masked = masking(b"DEFAULT SEED", rinfo.clone(), &profile);
assert_eq!(rinfo.headers, masked.headers);
assert_eq!(rinfo.cookies, masked.cookies);
assert_eq!(
RequestField::raw_create(&[], &[("arg1", "*MASKED*"), ("arg2", "avalue2")]),
RequestField::raw_create(&[], &[("arg1", "MASKED{172f0f6}"), ("arg2", "avalue2")]),
masked.rinfo.qinfo.args
);
}
Expand All @@ -449,11 +465,11 @@ mod test {
let mut profile = WafProfile::default();
let asection = profile.sections.at(SectionIdx::Args);
asection.names = ["arg1", "arg2"].iter().map(|k| (k.to_string(), maskentry())).collect();
let masked = masking(rinfo.clone(), &profile);
let masked = masking(b"DEFAULT SEED", rinfo.clone(), &profile);
assert_eq!(rinfo.headers, masked.headers);
assert_eq!(rinfo.cookies, masked.cookies);
assert_eq!(
RequestField::raw_create(&[], &[("arg1", "*MASKED*"), ("arg2", "*MASKED*")]),
RequestField::raw_create(&[], &[("arg1", "MASKED{172f0f6}"), ("arg2", "MASKED{c8dc0c0}")]),
masked.rinfo.qinfo.args
);
}
Expand Down
11 changes: 7 additions & 4 deletions curiefense/curieproxy/rust/luatests/test.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package.path = package.path .. ";lua/?.lua"
local curiefense = require "curiefense"

local sfmt = string.format
local cjson = require "cjson"
local json_safe = require "cjson.safe"
local json_decode = json_safe.decode
Expand Down Expand Up @@ -211,9 +210,13 @@ local function test_masking(request_path)
local response = run_inspect_request(raw_request_map)
local r = cjson.decode(response)

local p = string.find(response, secret)
if p ~= nil then
error("Could find secret in response: " .. response)
for _, section in pairs({"args", "headers", "cookies", "path"}) do
for k, value in pairs(r.request_map[section]) do
local p = string.find(value, secret)
if p ~= nil then
error("Could find secret in " .. section .. "/" .. k)
end
end
end
end
end
Expand Down