From 2e4525fe3c509a636fc67776cd728819f54a7de6 Mon Sep 17 00:00:00 2001 From: Raymond Hill Date: Mon, 16 Oct 2023 09:47:29 -0400 Subject: [PATCH] Add new static network filter option: `urltransform` The `urltransform` option allows to redirect a non-blocked network request to another URL. There are restrictions on its usage: - require a trusted source -- thus uBO-maintained lists or user filters - the `urltransform` value must start with a `/` If at least one of these conditions is not fulfilled, the filter will be invalid and rejected. The requirement to start with `/` is to enforce that only the path part of a URL can be modified, thus ensuring the network request is redirected to the same scheme and authority (as defined at https://en.wikipedia.org/wiki/Uniform_Resource_Identifier#Syntax). Usage example (redirect requests for CSS resources to a non-existing resource, for demonstration purpose): ||iana.org^$css,urltransform=/notfound.css Name of this option is inspired from DNR API: https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/declarativeNetRequest/URLTransform This commit required to bring the concept of "trusted source" to the static network filtering engine. --- platform/mv3/make-scriptlets.js | 2 +- src/js/1p-filters.js | 1 + src/js/asset-viewer.js | 1 + src/js/benchmarks.js | 3 +- src/js/codemirror/ubo-static-filtering.js | 19 ++++++++- src/js/messaging.js | 1 + src/js/pagestore.js | 24 +++++++---- src/js/redirect-engine.js | 2 - src/js/reverselookup.js | 2 + src/js/scriptlet-filtering.js | 2 +- src/js/static-dnr-filtering.js | 10 +++-- src/js/static-filtering-parser.js | 12 ++++++ src/js/static-net-filtering.js | 50 +++++++++++++++++++---- src/js/storage.js | 12 +++--- 14 files changed, 110 insertions(+), 31 deletions(-) diff --git a/platform/mv3/make-scriptlets.js b/platform/mv3/make-scriptlets.js index 2bc920d07f1a2..9e9559e17e84c 100644 --- a/platform/mv3/make-scriptlets.js +++ b/platform/mv3/make-scriptlets.js @@ -88,7 +88,7 @@ export function compile(details) { const scriptletToken = details.args[0]; const resourceEntry = resourceDetails.get(scriptletToken); if ( resourceEntry === undefined ) { return; } - if ( resourceEntry.requiresTrust && details.isTrusted !== true ) { + if ( resourceEntry.requiresTrust && details.trustedSource !== true ) { console.log(`Rejecting ${scriptletToken}: source is not trusted`); return; } diff --git a/src/js/1p-filters.js b/src/js/1p-filters.js index ccbd7067c3285..31073408e25f3 100644 --- a/src/js/1p-filters.js +++ b/src/js/1p-filters.js @@ -48,6 +48,7 @@ const cmEditor = new CodeMirror(qs$('#userFilters'), { styleActiveLine: { nonEmpty: true, }, + trustedSource: true, }); uBlockDashboard.patchCodeMirrorEditor(cmEditor); diff --git a/src/js/asset-viewer.js b/src/js/asset-viewer.js index 060ea20a389bd..dd69b7da56ef4 100644 --- a/src/js/asset-viewer.js +++ b/src/js/asset-viewer.js @@ -82,6 +82,7 @@ import './codemirror/ubo-static-filtering.js'; what : 'getAssetContent', url: assetKey, }); + cmEditor.setOption('trustedSource', details.trustedSource === true); cmEditor.setValue(details && details.content || ''); if ( subscribeElem !== null ) { diff --git a/src/js/benchmarks.js b/src/js/benchmarks.js index 1d02dc94b70c4..a252d6d530f3d 100644 --- a/src/js/benchmarks.js +++ b/src/js/benchmarks.js @@ -181,7 +181,8 @@ const loadBenchmarkDataset = (( ) => { if ( r === 1 ) { blockCount += 1; } else if ( r === 2 ) { allowCount += 1; } if ( r !== 1 ) { - if ( staticNetFilteringEngine.hasQuery(fctxt) ) { + staticNetFilteringEngine.transformRequest(fctxt); + if ( fctxt.redirectURL !== undefined && staticNetFilteringEngine.hasQuery(fctxt) ) { staticNetFilteringEngine.filterQuery(fctxt, 'removeparam'); } if ( fctxt.type === 'main_frame' || fctxt.type === 'sub_frame' ) { diff --git a/src/js/codemirror/ubo-static-filtering.js b/src/js/codemirror/ubo-static-filtering.js index 963509a836d59..c824e020612c9 100644 --- a/src/js/codemirror/ubo-static-filtering.js +++ b/src/js/codemirror/ubo-static-filtering.js @@ -37,12 +37,21 @@ const preparseDirectiveHints = []; const originHints = []; let hintHelperRegistered = false; +/******************************************************************************/ + +let trustedSource = false; + +CodeMirror.defineOption('trustedSource', false, (cm, state) => { + trustedSource = state; + self.dispatchEvent(new Event('trustedSource')); +}); /******************************************************************************/ CodeMirror.defineMode('ubo-static-filtering', function() { const astParser = new sfp.AstFilterParser({ interactive: true, + trustedSource, nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'), }); const astWalker = astParser.getWalker(); @@ -205,7 +214,11 @@ CodeMirror.defineMode('ubo-static-filtering', function() { return '+'; }; - return { + self.addEventListener('trustedSource', ( ) => { + astParser.options.trustedSource = trustedSource; + }); + + return { lineComment: '!', token: function(stream) { if ( stream.sol() ) { @@ -977,6 +990,10 @@ CodeMirror.registerHelper('fold', 'ubo-static-filtering', (( ) => { } }; + self.addEventListener('trustedSource', ( ) => { + astParser.options.trustedSource = trustedSource; + }); + CodeMirror.defineInitHook(cm => { cm.on('changes', onChanges); cm.on('beforeChange', onBeforeChanges); diff --git a/src/js/messaging.js b/src/js/messaging.js index e5d911d393b97..62fcd93463867 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -108,6 +108,7 @@ const onMessage = function(request, sender, callback) { dontCache: true, needSourceURL: true, }).then(result => { + result.trustedSource = µb.isTrustedList(result.assetKey); callback(result); }); return; diff --git a/src/js/pagestore.js b/src/js/pagestore.js index e51dde815939b..8c1cda295302b 100644 --- a/src/js/pagestore.js +++ b/src/js/pagestore.js @@ -860,7 +860,7 @@ const PageStore = class { if ( (fctxt.itype & fctxt.INLINE_ANY) === 0 ) { if ( result === 1 ) { this.redirectBlockedRequest(fctxt); - } else if ( snfe.hasQuery(fctxt) ) { + } else { this.redirectNonBlockedRequest(fctxt); } } @@ -922,25 +922,31 @@ const PageStore = class { } redirectBlockedRequest(fctxt) { - const directives = staticNetFilteringEngine.redirectRequest( - redirectEngine, - fctxt - ); + const directives = staticNetFilteringEngine.redirectRequest(redirectEngine, fctxt); if ( directives === undefined ) { return; } if ( logger.enabled !== true ) { return; } fctxt.pushFilters(directives.map(a => a.logData())); if ( fctxt.redirectURL === undefined ) { return; } fctxt.pushFilter({ source: 'redirect', - raw: redirectEngine.resourceNameRegister + raw: directives[directives.length-1].value }); } redirectNonBlockedRequest(fctxt) { - const directives = staticNetFilteringEngine.filterQuery(fctxt); - if ( directives === undefined ) { return; } + const transformDirectives = staticNetFilteringEngine.transformRequest(fctxt); + const pruneDirectives = fctxt.redirectURL === undefined && + staticNetFilteringEngine.hasQuery(fctxt) && + staticNetFilteringEngine.filterQuery(fctxt) || + undefined; + if ( transformDirectives === undefined && pruneDirectives === undefined ) { return; } if ( logger.enabled !== true ) { return; } - fctxt.pushFilters(directives.map(a => a.logData())); + if ( transformDirectives !== undefined ) { + fctxt.pushFilters(transformDirectives.map(a => a.logData())); + } + if ( pruneDirectives !== undefined ) { + fctxt.pushFilters(pruneDirectives.map(a => a.logData())); + } if ( fctxt.redirectURL === undefined ) { return; } fctxt.pushFilter({ source: 'redirect', diff --git a/src/js/redirect-engine.js b/src/js/redirect-engine.js index ef4e775414b07..5503c38a7e387 100644 --- a/src/js/redirect-engine.js +++ b/src/js/redirect-engine.js @@ -167,7 +167,6 @@ class RedirectEngine { this.resources = new Map(); this.reset(); this.modifyTime = Date.now(); - this.resourceNameRegister = ''; } reset() { @@ -183,7 +182,6 @@ class RedirectEngine { ) { const entry = this.resources.get(this.aliases.get(token) || token); if ( entry === undefined ) { return; } - this.resourceNameRegister = token; return entry.toURL(fctxt, asDataURI); } diff --git a/src/js/reverselookup.js b/src/js/reverselookup.js index 27f0136f852fd..773eba97bed3b 100644 --- a/src/js/reverselookup.js +++ b/src/js/reverselookup.js @@ -131,6 +131,7 @@ const fromNetFilter = async function(rawFilter) { const writer = new CompiledListWriter(); const parser = new sfp.AstFilterParser({ expertMode: true, + trustedSource: true, maxTokenLength: staticNetFilteringEngine.MAX_TOKEN_LENGTH, nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'), }); @@ -169,6 +170,7 @@ const fromExtendedFilter = async function(details) { const parser = new sfp.AstFilterParser({ expertMode: true, + trustedSource: true, nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'), }); parser.parse(details.rawFilter); diff --git a/src/js/scriptlet-filtering.js b/src/js/scriptlet-filtering.js index 5b50ae67549ca..28677dd419e3c 100644 --- a/src/js/scriptlet-filtering.js +++ b/src/js/scriptlet-filtering.js @@ -306,7 +306,7 @@ scriptletFilteringEngine.compile = function(parser, writer) { // Only exception filters are allowed to be global. const isException = parser.isException(); - const normalized = normalizeRawFilter(parser, writer.properties.get('isTrusted')); + const normalized = normalizeRawFilter(parser, writer.properties.get('trustedSource')); // Can fail if there is a mismatch with trust requirement if ( normalized === undefined ) { return; } diff --git a/src/js/static-dnr-filtering.js b/src/js/static-dnr-filtering.js index ac94475d5002a..de253669cb3fc 100644 --- a/src/js/static-dnr-filtering.js +++ b/src/js/static-dnr-filtering.js @@ -109,8 +109,8 @@ function addExtendedToDNR(context, parser) { let details = context.scriptletFilters.get(argsToken); if ( details === undefined ) { context.scriptletFilters.set(argsToken, details = { args }); - if ( context.isTrusted ) { - details.isTrusted = true; + if ( context.trustedSource ) { + details.trustedSource = true; } } if ( not ) { @@ -299,9 +299,11 @@ function addToDNR(context, list) { if ( parser.isComment() ) { if ( line === `!#trusted on ${context.secret}` ) { - context.isTrusted = true; + parser.trustedSource = true; + context.trustedSource = true; } else if ( line === `!#trusted off ${context.secret}` ) { - context.isTrusted = false; + parser.trustedSource = false; + context.trustedSource = false; } continue; } diff --git a/src/js/static-filtering-parser.js b/src/js/static-filtering-parser.js index 94ee66cd1af20..6141258610264 100644 --- a/src/js/static-filtering-parser.js +++ b/src/js/static-filtering-parser.js @@ -188,6 +188,7 @@ export const NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM = iota++; export const NODE_TYPE_NET_OPTION_NAME_SCRIPT = iota++; export const NODE_TYPE_NET_OPTION_NAME_SHIDE = iota++; export const NODE_TYPE_NET_OPTION_NAME_TO = iota++; +export const NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM = iota++; export const NODE_TYPE_NET_OPTION_NAME_XHR = iota++; export const NODE_TYPE_NET_OPTION_NAME_WEBRTC = iota++; export const NODE_TYPE_NET_OPTION_NAME_WEBSOCKET = iota++; @@ -267,6 +268,7 @@ export const nodeTypeFromOptionName = new Map([ [ 'shide', NODE_TYPE_NET_OPTION_NAME_SHIDE ], /* synonym */ [ 'specifichide', NODE_TYPE_NET_OPTION_NAME_SHIDE ], [ 'to', NODE_TYPE_NET_OPTION_NAME_TO ], + [ 'urltransform', NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM ], [ 'xhr', NODE_TYPE_NET_OPTION_NAME_XHR ], /* synonym */ [ 'xmlhttprequest', NODE_TYPE_NET_OPTION_NAME_XHR ], [ 'webrtc', NODE_TYPE_NET_OPTION_NAME_WEBRTC ], @@ -1315,6 +1317,7 @@ export class AstFilterParser { break; case NODE_TYPE_NET_OPTION_NAME_REDIRECT: case NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE: + case NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM: realBad = isNegated || (isException || hasValue) === false || modifierType !== 0; if ( realBad ) { break; } @@ -1374,6 +1377,14 @@ export class AstFilterParser { realBad = abstractTypeCount || behaviorTypeCount || unredirectableTypeCount; break; } + case NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM: + realBad = abstractTypeCount || behaviorTypeCount || unredirectableTypeCount || + this.options.trustedSource !== true; + if ( realBad !== true ) { + const path = this.getNetOptionValue(NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM); + realBad = path.charCodeAt(0) !== 0x2F /* / */; + } + break; case NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM: realBad = abstractTypeCount || behaviorTypeCount; break; @@ -2973,6 +2984,7 @@ export const netOptionTokenDescriptors = new Map([ [ 'shide', { } ], /* synonym */ [ 'specifichide', { } ], [ 'to', { mustAssign: true } ], + [ 'urltransform', { mustAssign: true } ], [ 'xhr', { canNegate: true } ], /* synonym */ [ 'xmlhttprequest', { canNegate: true } ], [ 'webrtc', { } ], diff --git a/src/js/static-net-filtering.js b/src/js/static-net-filtering.js index 757c5a0a85781..abff5c65dfce0 100644 --- a/src/js/static-net-filtering.js +++ b/src/js/static-net-filtering.js @@ -178,11 +178,14 @@ const typeValueToDNRTypeName = [ 'other', ]; +// Do not change order. Compiled filter lists rely on this order being +// consistent across sessions. const MODIFIER_TYPE_REDIRECT = 1; const MODIFIER_TYPE_REDIRECTRULE = 2; const MODIFIER_TYPE_REMOVEPARAM = 3; const MODIFIER_TYPE_CSP = 4; const MODIFIER_TYPE_PERMISSIONS = 5; +const MODIFIER_TYPE_URLTRANSFORM = 6; const modifierTypeFromName = new Map([ [ 'redirect', MODIFIER_TYPE_REDIRECT ], @@ -190,6 +193,7 @@ const modifierTypeFromName = new Map([ [ 'removeparam', MODIFIER_TYPE_REMOVEPARAM ], [ 'csp', MODIFIER_TYPE_CSP ], [ 'permissions', MODIFIER_TYPE_PERMISSIONS ], + [ 'urltransform', MODIFIER_TYPE_URLTRANSFORM ], ]); const modifierNameFromType = new Map([ @@ -198,6 +202,7 @@ const modifierNameFromType = new Map([ [ MODIFIER_TYPE_REMOVEPARAM, 'removeparam' ], [ MODIFIER_TYPE_CSP, 'csp' ], [ MODIFIER_TYPE_PERMISSIONS, 'permissions' ], + [ MODIFIER_TYPE_URLTRANSFORM, 'urltransform' ], ]); //const typeValueFromCatBits = catBits => (catBits >>> TypeBitsOffset) & 0b11111; @@ -3182,6 +3187,7 @@ class FilterCompiler { [ sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT, MODIFIER_TYPE_REDIRECT ], [ sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE, MODIFIER_TYPE_REDIRECTRULE ], [ sfp.NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM, MODIFIER_TYPE_REMOVEPARAM ], + [ sfp.NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM, MODIFIER_TYPE_URLTRANSFORM ], ]); // These top 100 "bad tokens" are collated using the "miss" histogram // from tokenHistograms(). The "score" is their occurrence among the @@ -3484,6 +3490,12 @@ class FilterCompiler { if ( this.toDomainOpt === '' ) { return false; } this.optionUnitBits |= this.TO_BIT; break; + case sfp.NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM: + if ( this.processModifierOption(id, parser.getNetOptionValue(id)) === false ) { + return false; + } + this.optionUnitBits |= this.REDIRECT_BIT; + break; default: break; } @@ -3575,6 +3587,7 @@ class FilterCompiler { case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE: case sfp.NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM: case sfp.NODE_TYPE_NET_OPTION_NAME_TO: + case sfp.NODE_TYPE_NET_OPTION_NAME_URLTRANSFORM: if ( this.processOptionWithValue(parser, type) === false ) { return this.FILTER_INVALID; } @@ -4521,6 +4534,20 @@ FilterContainer.prototype.dnrFromCompiled = function(op, context, ...args) { dnrAddRuleError(rule, 'Unsupported modifier exception'); } break; + case 'urltransform': { + const path = rule.__modifierValue; + let priority = rule.priority || 1; + if ( rule.__modifierAction !== AllowAction ) { + const transform = { path }; + rule.action.type = 'redirect'; + rule.action.redirect = { transform }; + rule.priority = priority + 1; + } else { + rule.action.type = 'block'; + rule.priority = priority + 2; + } + break; + } default: dnrAddRuleError(rule, `Unsupported modifier ${rule.__modifierType}`); break; @@ -5230,18 +5257,27 @@ FilterContainer.prototype.redirectRequest = function(redirectEngine, fctxt) { } // Redirect to highest-ranked directive const directive = directives[highest]; - if ( (directive.bits & AllowAction) === 0 ) { - const { token } = parseRedirectRequestValue(directive); - fctxt.redirectURL = redirectEngine.tokenToURL(fctxt, token); - if ( fctxt.redirectURL === undefined ) { return; } - } + if ( (directive.bits & AllowAction) !== 0 ) { return directives; } + const { token } = parseRedirectRequestValue(directive); + fctxt.redirectURL = redirectEngine.tokenToURL(fctxt, token); + if ( fctxt.redirectURL === undefined ) { return; } + return directives; +}; + +FilterContainer.prototype.transformRequest = function(fctxt) { + const directives = this.matchAndFetchModifiers(fctxt, 'urltransform'); + if ( directives === undefined ) { return; } + const directive = directives[directives.length-1]; + if ( (directive.bits & AllowAction) !== 0 ) { return directives; } + const redirectURL = new URL(fctxt.url); + redirectURL.pathname = directive.value; + fctxt.redirectURL = redirectURL.href; return directives; }; function parseRedirectRequestValue(directive) { if ( directive.cache === null ) { - directive.cache = - sfp.parseRedirectValue(directive.value); + directive.cache = sfp.parseRedirectValue(directive.value); } return directive.cache; } diff --git a/src/js/storage.js b/src/js/storage.js index 28482e3ce8fdd..575814b576cf4 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -569,7 +569,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { const compiledFilters = this.compileFilters(filters, { assetKey: this.userFiltersPath, - isTrusted: true, + trustedSource: true, }); const snfe = staticNetFilteringEngine; const cfe = cosmeticFilteringEngine; @@ -908,7 +908,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { µb.inMemoryFiltersCompiled = µb.compileFilters(µb.inMemoryFilters.join('\n'), { assetKey: 'in-memory', - isTrusted: true, + trustedSource: true, }); } if ( µb.inMemoryFiltersCompiled !== '' ) { @@ -976,7 +976,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { const compiledContent = this.compileFilters(rawDetails.content, { assetKey, - isTrusted: this.isTrustedList(assetKey), + trustedSource: this.isTrustedList(assetKey), }); io.put(compiledPath, compiledContent); @@ -1041,9 +1041,10 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { // Populate the writer with information potentially useful to the // client compilers. + const trustedSource = details.trustedSource === true; if ( details.assetKey ) { writer.properties.set('name', details.assetKey); - writer.properties.set('isTrusted', details.isTrusted === true); + writer.properties.set('trustedSource', trustedSource); } const assetName = details.assetKey ? details.assetKey : '?'; const expertMode = @@ -1051,6 +1052,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { this.hiddenSettings.filterAuthorMode !== false; const parser = new sfp.AstFilterParser({ expertMode, + trustedSource, maxTokenLength: staticNetFilteringEngine.MAX_TOKEN_LENGTH, nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'), }); @@ -1564,7 +1566,7 @@ self.addEventListener('hiddenSettingsChanged', ( ) => { 'compiled/' + details.assetKey, this.compileFilters(details.content, { assetKey: details.assetKey, - isTrusted: this.isTrustedList(details.assetKey), + trustedSource: this.isTrustedList(details.assetKey), }) ); }