diff --git a/src/3p-filters.html b/src/3p-filters.html index b51e1f1b03eb0..a7791208077a0 100644 --- a/src/3p-filters.html +++ b/src/3p-filters.html @@ -20,7 +20,6 @@
-
diff --git a/src/devtools.html b/src/devtools.html index 5683dc79e355e..af7b972afa5d9 100644 --- a/src/devtools.html +++ b/src/devtools.html @@ -29,6 +29,7 @@ +
diff --git a/src/js/3p-filters.js b/src/js/3p-filters.js index bbdcab505c995..c59365fd73101 100644 --- a/src/js/3p-filters.js +++ b/src/js/3p-filters.js @@ -276,7 +276,7 @@ const renderFilterLists = ( ) => { renderWidgets(); }; - vAPI.messaging.send('dashboard', { + return vAPI.messaging.send('dashboard', { what: 'getLists', }).then(response => { onListsReceived(response); @@ -295,9 +295,6 @@ const renderWidgets = ( ) => { updating === false && qs$('#lists .listEntry.checked.obsolete:not(.toRemove)') === null ); - dom.cl.toggle('#buttonPurgeAll', 'disabled', - updating || qs$('#lists .listEntry.cached:not(.obsolete)') === null - ); }; /******************************************************************************/ @@ -510,8 +507,9 @@ const onPurgeClicked = ev => { } vAPI.messaging.send('dashboard', { - what: 'purgeCaches', + what: 'listsUpdateNow', assetKeys, + preferOrigin: ev.shiftKey, }); // If the cached version is purged, the installed version must be assumed @@ -519,6 +517,7 @@ const onPurgeClicked = ev => { // https://github.com/gorhill/uBlock/issues/1733 // An external filter list must not be marked as obsolete, they will // always be fetched anyways if there is no cached copy. + dom.cl.add(dom.body, 'updating'); dom.cl.add(liEntry, 'obsolete'); if ( qs$(liEntry, 'input[type="checkbox"]').checked ) { @@ -608,25 +607,13 @@ const buttonUpdateHandler = async ( ) => { await selectFilterLists(); dom.cl.add(dom.body, 'updating'); renderWidgets(); - vAPI.messaging.send('dashboard', { what: 'forceUpdateAssets' }); + vAPI.messaging.send('dashboard', { what: 'updateNow' }); }; dom.on('#buttonUpdate', 'click', ( ) => { buttonUpdateHandler(); }); /******************************************************************************/ -const buttonPurgeAllHandler = async hard => { - await vAPI.messaging.send('dashboard', { - what: 'purgeAllCaches', - hard, - }); - renderFilterLists(true); -}; - -dom.on('#buttonPurgeAll', 'click', ev => { buttonPurgeAllHandler(ev.shiftKey); }); - -/******************************************************************************/ - const userSettingCheckboxChanged = ( ) => { const target = event.target; vAPI.messaging.send('dashboard', { @@ -863,6 +850,12 @@ self.hasUnsavedData = function() { /******************************************************************************/ -renderFilterLists(); +renderFilterLists().then(( ) => { + const buttonUpdate = qs$('#buttonUpdate'); + if ( dom.cl.has(buttonUpdate, 'active') ) { return; } + if ( dom.cl.has(buttonUpdate, 'disabled') ) { return; } + if ( listsetDetails.autoUpdate !== true ) { return; } + buttonUpdateHandler(); +}); /******************************************************************************/ diff --git a/src/js/assets.js b/src/js/assets.js index a6b9b73367325..3ba617b7c4ff4 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -39,6 +39,7 @@ const reIsUserAsset = /^user-/; const errorCantConnectTo = i18n$('errorCantConnectTo'); const MS_PER_HOUR = 60 * 60 * 1000; const MS_PER_DAY = 24 * MS_PER_HOUR; +const MINUTES_PER_DAY = 24 * 60; const EXPIRES_DEFAULT = 7; const assets = {}; @@ -175,6 +176,23 @@ const isDiffUpdatableAsset = content => { data.diffPath.startsWith('%') === false; }; +const computedPatchUpdateTime = assetKey => { + const entry = assetCacheRegistry[assetKey]; + if ( entry === undefined ) { return 0; } + if ( typeof entry.diffPath !== 'string' ) { return 0; } + if ( typeof entry.diffExpires !== 'number' ) { return 0; } + const match = /(\d+)\.(\d+)\.(\d+)\.(\d+)/.exec(entry.diffPath); + if ( match === null ) { return getWriteTime(); } + const date = new Date(); + date.setUTCFullYear( + parseInt(match[1], 10), + parseInt(match[2], 10) - 1, + parseInt(match[3], 10) + ); + date.setUTCHours(0, parseInt(match[4], 10) + entry.diffExpires * MINUTES_PER_DAY, 0, 0); + return date.getTime(); +}; + /******************************************************************************/ // favorLocal: avoid making network requests whenever possible @@ -1169,7 +1187,7 @@ assets.getUpdateAges = async function(conditions = {}) { out.push({ assetKey, age, - ageNormalized: age / getUpdateAfterTime(assetKey), + ageNormalized: age / Math.max(1, getUpdateAfterTime(assetKey)), }); } return out; @@ -1221,12 +1239,13 @@ async function diffUpdater() { const assetDetails = getAssetDiffDetails(assetKey); if ( assetDetails === undefined ) { continue; } assetDetails.what = 'update'; - if ( (getWriteTime(assetKey) + assetDetails.diffExpires) > now ) { - assetDetails.fetch = false; - toSoftUpdate.push(assetDetails); - } else { + const computedUpdateTime = computedPatchUpdateTime(assetKey); + if ( computedUpdateTime !== 0 && computedUpdateTime <= now ) { assetDetails.fetch = true; toHardUpdate.push(assetDetails); + } else { + assetDetails.fetch = false; + toSoftUpdate.push(assetDetails); } } if ( toHardUpdate.length === 0 ) { return; } @@ -1432,8 +1451,8 @@ function updateDone() { assets.updateStart = function(details) { const oldUpdateDelay = updaterAssetDelay; - const newUpdateDelay = typeof details.delay === 'number' - ? details.delay + const newUpdateDelay = typeof details.fetchDelay === 'number' + ? details.fetchDelay : updaterAssetDelayDefault; updaterAssetDelay = Math.min(oldUpdateDelay, newUpdateDelay); updaterAuto = details.auto === true; diff --git a/src/js/devtools.js b/src/js/devtools.js index a417708116ec4..93b26978e8975 100644 --- a/src/js/devtools.js +++ b/src/js/devtools.js @@ -134,7 +134,7 @@ dom.on('#console-unfold', 'click', ( ) => { dom.on('#snfe-dump', 'click', ev => { const button = ev.target; dom.attr(button, 'disabled', ''); - vAPI.messaging.send('dashboard', { + vAPI.messaging.send('devTools', { what: 'snfeDump', }).then(result => { log(result); @@ -145,7 +145,7 @@ dom.on('#snfe-dump', 'click', ev => { dom.on('#snfe-todnr', 'click', ev => { const button = ev.target; dom.attr(button, 'disabled', ''); - vAPI.messaging.send('dashboard', { + vAPI.messaging.send('devTools', { what: 'snfeToDNR', }).then(result => { log(result); @@ -153,6 +153,25 @@ dom.on('#snfe-todnr', 'click', ev => { }); }); +dom.on('#cfe-dump', 'click', ev => { + const button = ev.target; + dom.attr(button, 'disabled', ''); + vAPI.messaging.send('devTools', { + what: 'cfeDump', + }).then(result => { + log(result); + dom.attr(button, 'disabled', null); + }); +}); + +dom.on('#purge-all-caches', 'click', ( ) => { + vAPI.messaging.send('devTools', { + what: 'purgeAllCaches' + }).then(result => { + log(result); + }); +}); + vAPI.messaging.send('dashboard', { what: 'getAppData', }).then(appData => { @@ -161,7 +180,7 @@ vAPI.messaging.send('dashboard', { dom.on('#snfe-benchmark', 'click', ev => { const button = ev.target; dom.attr(button, 'disabled', ''); - vAPI.messaging.send('dashboard', { + vAPI.messaging.send('devTools', { what: 'snfeBenchmark', }).then(result => { log(result); @@ -170,15 +189,4 @@ vAPI.messaging.send('dashboard', { }); }); -dom.on('#cfe-dump', 'click', ev => { - const button = ev.target; - dom.attr(button, 'disabled', ''); - vAPI.messaging.send('dashboard', { - what: 'cfeDump', - }).then(result => { - log(result); - dom.attr(button, 'disabled', null); - }); -}); - /******************************************************************************/ diff --git a/src/js/lz4.js b/src/js/lz4.js index d16365e6b5ac2..608cdd853ceaa 100644 --- a/src/js/lz4.js +++ b/src/js/lz4.js @@ -44,7 +44,7 @@ let ttlCount = 0; let ttlDelay = 60000; const init = function() { - ttlDelay = µb.hiddenSettings.autoUpdateAssetFetchPeriod * 1000 + 15000; + ttlDelay = µb.hiddenSettings.autoUpdateAssetFetchPeriod * 2 * 1000; if ( promisedInstance === undefined ) { let flavor; if ( µb.hiddenSettings.disableWebAssembly === true ) { diff --git a/src/js/messaging.js b/src/js/messaging.js index eae8607b5b4d7..cacf32127adda 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -142,124 +142,6 @@ const onMessage = function(request, sender, callback) { }); return; - case 'snfeBenchmark': - µb.benchmarkStaticNetFiltering({ redirectEngine }).then(result => { - callback(result); - }); - return; - - case 'snfeToDNR': { - const listPromises = []; - const listNames = []; - for ( const assetKey of µb.selectedFilterLists ) { - listPromises.push( - io.get(assetKey, { dontCache: true }).then(details => { - listNames.push(assetKey); - return { name: assetKey, text: details.content }; - }) - ); - } - const options = { - extensionPaths: redirectEngine.getResourceDetails().filter(e => - typeof e[1].extensionPath === 'string' && e[1].extensionPath !== '' - ).map(e => - [ e[0], e[1].extensionPath ] - ), - env: vAPI.webextFlavor.env, - }; - const t0 = Date.now(); - dnrRulesetFromRawLists(listPromises, options).then(result => { - const { network } = result; - const replacer = (k, v) => { - if ( k.startsWith('__') ) { return; } - if ( Array.isArray(v) ) { - return v.sort(); - } - if ( v instanceof Object ) { - const sorted = {}; - for ( const kk of Object.keys(v).sort() ) { - sorted[kk] = v[kk]; - } - return sorted; - } - return v; - }; - const isUnsupported = rule => - rule._error !== undefined; - const isRegex = rule => - rule.condition !== undefined && - rule.condition.regexFilter !== undefined; - const isRedirect = rule => - rule.action !== undefined && - rule.action.type === 'redirect' && - rule.action.redirect.extensionPath !== undefined; - const isCsp = rule => - rule.action !== undefined && - rule.action.type === 'modifyHeaders'; - const isRemoveparam = rule => - rule.action !== undefined && - rule.action.type === 'redirect' && - rule.action.redirect.transform !== undefined; - const runtime = Date.now() - t0; - const { ruleset } = network; - const good = ruleset.filter(rule => - isUnsupported(rule) === false && - isRegex(rule) === false && - isRedirect(rule) === false && - isCsp(rule) === false && - isRemoveparam(rule) === false - ); - const unsupported = ruleset.filter(rule => - isUnsupported(rule) - ); - const regexes = ruleset.filter(rule => - isUnsupported(rule) === false && - isRegex(rule) && - isRedirect(rule) === false && - isCsp(rule) === false && - isRemoveparam(rule) === false - ); - const redirects = ruleset.filter(rule => - isUnsupported(rule) === false && - isRedirect(rule) - ); - const headers = ruleset.filter(rule => - isUnsupported(rule) === false && - isCsp(rule) - ); - const removeparams = ruleset.filter(rule => - isUnsupported(rule) === false && - isRemoveparam(rule) - ); - const out = [ - `dnrRulesetFromRawLists(${JSON.stringify(listNames, null, 2)})`, - `Run time: ${runtime} ms`, - `Filters count: ${network.filterCount}`, - `Accepted filter count: ${network.acceptedFilterCount}`, - `Rejected filter count: ${network.rejectedFilterCount}`, - `Un-DNR-able filter count: ${unsupported.length}`, - `Resulting DNR rule count: ${ruleset.length}`, - ]; - out.push(`+ Good filters (${good.length}): ${JSON.stringify(good, replacer, 2)}`); - out.push(`+ Regex-based filters (${regexes.length}): ${JSON.stringify(regexes, replacer, 2)}`); - out.push(`+ 'redirect=' filters (${redirects.length}): ${JSON.stringify(redirects, replacer, 2)}`); - out.push(`+ 'csp=' filters (${headers.length}): ${JSON.stringify(headers, replacer, 2)}`); - out.push(`+ 'removeparam=' filters (${removeparams.length}): ${JSON.stringify(removeparams, replacer, 2)}`); - out.push(`+ Unsupported filters (${unsupported.length}): ${JSON.stringify(unsupported, replacer, 2)}`); - out.push(`+ generichide exclusions (${network.generichideExclusions.length}): ${JSON.stringify(network.generichideExclusions, replacer, 2)}`); - if ( result.specificCosmetic ) { - out.push(`+ Cosmetic filters: ${result.specificCosmetic.size}`); - for ( const details of result.specificCosmetic ) { - out.push(` ${JSON.stringify(details)}`); - } - } else { - out.push(' Cosmetic filters: 0'); - } - callback(out.join('\n')); - }); - return; - } - default: break; } @@ -280,13 +162,6 @@ const onMessage = function(request, sender, callback) { µb.createUserFilters(request); break; - case 'forceUpdateAssets': - µb.scheduleAssetUpdater(0); - io.updateStart({ - delay: µb.hiddenSettings.manualUpdateAssetFetchPeriod - }); - break; - case 'getAppData': response = { name: browser.runtime.getManifest().name, @@ -383,14 +258,6 @@ const onMessage = function(request, sender, callback) { } break; - case 'snfeDump': - response = staticNetFilteringEngine.dump(); - break; - - case 'cfeDump': - response = cosmeticFilteringEngine.dump(); - break; - default: return vAPI.messaging.UNHANDLED; } @@ -1630,19 +1497,25 @@ const onMessage = function(request, sender, callback) { response = getRules(); break; - case 'purgeAllCaches': - if ( request.hard ) { - io.remove(/./); - } else { - io.purge(/./, 'public_suffix_list.dat'); + case 'supportUpdateNow': { + const { assetKeys } = request; + if ( assetKeys.length === 0 ) { return; } + for ( const assetKey of assetKeys ) { + io.purge(assetKey); } + µb.scheduleAssetUpdater({ now: true, fetchDelay: 100 }); break; + } - case 'purgeCaches': - for ( const assetKey of request.assetKeys ) { + case 'listsUpdateNow': { + const { assetKeys, preferOrigin = false } = request; + if ( assetKeys.length === 0 ) { return; } + for ( const assetKey of assetKeys ) { io.purge(assetKey); } + µb.scheduleAssetUpdater({ now: true, fetchDelay: 100, auto: preferOrigin !== true }); break; + } case 'readHiddenSettings': response = { @@ -1660,6 +1533,10 @@ const onMessage = function(request, sender, callback) { resetUserData(); break; + case 'updateNow': + µb.scheduleAssetUpdater({ now: true, fetchDelay: 100, auto: true }); + break; + case 'writeHiddenSettings': µb.changeHiddenSettings(µb.hiddenSettingsFromString(request.content)); break; @@ -1944,6 +1821,181 @@ vAPI.messaging.listen({ /******************************************************************************/ /******************************************************************************/ +// Channel: +// devTools +// privileged + +{ +// >>>>> start of local scope + +const onMessage = function(request, sender, callback) { + // Async + switch ( request.what ) { + case 'purgeAllCaches': + µb.getBytesInUse().then(bytesInUseBefore => + io.remove(/./).then(( ) => + µb.getBytesInUse().then(bytesInUseAfter => { + callback([ + `Storage used before: ${µb.formatCount(bytesInUseBefore)}B`, + `Storage used after: ${µb.formatCount(bytesInUseAfter)}B`, + ].join('\n')); + }) + ) + ); + return; + + case 'snfeBenchmark': + µb.benchmarkStaticNetFiltering({ redirectEngine }).then(result => { + callback(result); + }); + return; + + case 'snfeToDNR': { + const listPromises = []; + const listNames = []; + for ( const assetKey of µb.selectedFilterLists ) { + listPromises.push( + io.get(assetKey, { dontCache: true }).then(details => { + listNames.push(assetKey); + return { name: assetKey, text: details.content }; + }) + ); + } + const options = { + extensionPaths: redirectEngine.getResourceDetails().filter(e => + typeof e[1].extensionPath === 'string' && e[1].extensionPath !== '' + ).map(e => + [ e[0], e[1].extensionPath ] + ), + env: vAPI.webextFlavor.env, + }; + const t0 = Date.now(); + dnrRulesetFromRawLists(listPromises, options).then(result => { + const { network } = result; + const replacer = (k, v) => { + if ( k.startsWith('__') ) { return; } + if ( Array.isArray(v) ) { + return v.sort(); + } + if ( v instanceof Object ) { + const sorted = {}; + for ( const kk of Object.keys(v).sort() ) { + sorted[kk] = v[kk]; + } + return sorted; + } + return v; + }; + const isUnsupported = rule => + rule._error !== undefined; + const isRegex = rule => + rule.condition !== undefined && + rule.condition.regexFilter !== undefined; + const isRedirect = rule => + rule.action !== undefined && + rule.action.type === 'redirect' && + rule.action.redirect.extensionPath !== undefined; + const isCsp = rule => + rule.action !== undefined && + rule.action.type === 'modifyHeaders'; + const isRemoveparam = rule => + rule.action !== undefined && + rule.action.type === 'redirect' && + rule.action.redirect.transform !== undefined; + const runtime = Date.now() - t0; + const { ruleset } = network; + const good = ruleset.filter(rule => + isUnsupported(rule) === false && + isRegex(rule) === false && + isRedirect(rule) === false && + isCsp(rule) === false && + isRemoveparam(rule) === false + ); + const unsupported = ruleset.filter(rule => + isUnsupported(rule) + ); + const regexes = ruleset.filter(rule => + isUnsupported(rule) === false && + isRegex(rule) && + isRedirect(rule) === false && + isCsp(rule) === false && + isRemoveparam(rule) === false + ); + const redirects = ruleset.filter(rule => + isUnsupported(rule) === false && + isRedirect(rule) + ); + const headers = ruleset.filter(rule => + isUnsupported(rule) === false && + isCsp(rule) + ); + const removeparams = ruleset.filter(rule => + isUnsupported(rule) === false && + isRemoveparam(rule) + ); + const out = [ + `dnrRulesetFromRawLists(${JSON.stringify(listNames, null, 2)})`, + `Run time: ${runtime} ms`, + `Filters count: ${network.filterCount}`, + `Accepted filter count: ${network.acceptedFilterCount}`, + `Rejected filter count: ${network.rejectedFilterCount}`, + `Un-DNR-able filter count: ${unsupported.length}`, + `Resulting DNR rule count: ${ruleset.length}`, + ]; + out.push(`+ Good filters (${good.length}): ${JSON.stringify(good, replacer, 2)}`); + out.push(`+ Regex-based filters (${regexes.length}): ${JSON.stringify(regexes, replacer, 2)}`); + out.push(`+ 'redirect=' filters (${redirects.length}): ${JSON.stringify(redirects, replacer, 2)}`); + out.push(`+ 'csp=' filters (${headers.length}): ${JSON.stringify(headers, replacer, 2)}`); + out.push(`+ 'removeparam=' filters (${removeparams.length}): ${JSON.stringify(removeparams, replacer, 2)}`); + out.push(`+ Unsupported filters (${unsupported.length}): ${JSON.stringify(unsupported, replacer, 2)}`); + out.push(`+ generichide exclusions (${network.generichideExclusions.length}): ${JSON.stringify(network.generichideExclusions, replacer, 2)}`); + if ( result.specificCosmetic ) { + out.push(`+ Cosmetic filters: ${result.specificCosmetic.size}`); + for ( const details of result.specificCosmetic ) { + out.push(` ${JSON.stringify(details)}`); + } + } else { + out.push(' Cosmetic filters: 0'); + } + callback(out.join('\n')); + }); + return; + } + default: + break; + } + + // Sync + let response; + + switch ( request.what ) { + case 'snfeDump': + response = staticNetFilteringEngine.dump(); + break; + + case 'cfeDump': + response = cosmeticFilteringEngine.dump(); + break; + + default: + return vAPI.messaging.UNHANDLED; + } + + callback(response); +}; + +vAPI.messaging.listen({ + name: 'devTools', + listener: onMessage, + privileged: true, +}); + +// <<<<< end of local scope +} + +/******************************************************************************/ +/******************************************************************************/ + // Channel: // scriptlets // unprivileged @@ -2122,12 +2174,11 @@ const onMessage = function(request, sender, callback) { io.purge(listkey); } } - µb.scheduleAssetUpdater(0); µb.openNewTab({ url: 'dashboard.html#3p-filters.html', select: true, }); - io.updateStart({ delay: 100, auto: request.manual !== true }); + µb.scheduleAssetUpdater({ now: true, fetchDelay: 100, auto: request.manual !== true }); break; default: diff --git a/src/js/start.js b/src/js/start.js index bc38f2b926cf2..60186bd3000da 100644 --- a/src/js/start.js +++ b/src/js/start.js @@ -474,11 +474,7 @@ lz4Codec.relinquish(); // https://github.com/chrisaljoudi/uBlock/issues/184 // Check for updates not too far in the future. io.addObserver(µb.assetObserver.bind(µb)); -µb.scheduleAssetUpdater( - µb.userSettings.autoUpdate - ? µb.hiddenSettings.autoUpdateDelayAfterLaunch * 1000 - : 0 -); +µb.scheduleAssetUpdater(); // Force an update of the context menu according to the currently // active tab. diff --git a/src/js/storage.js b/src/js/storage.js index ab5cb01fd21e7..b8f85b733089b 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -1497,17 +1497,28 @@ onBroadcast(msg => { const launchTimer = vAPI.defer.create(fetchDelay => { next = 0; - io.updateStart({ delay: fetchDelay, auto: true }); + io.updateStart({ fetchDelay, auto: true }); }); - µb.scheduleAssetUpdater = async function(updateDelay) { + µb.scheduleAssetUpdater = async function(details = {}) { launchTimer.off(); - if ( updateDelay === 0 ) { + if ( details.now ) { next = 0; + io.updateStart(details); return; } + if ( µb.userSettings.autoUpdate === false ) { + if ( details.updateDelay === undefined ) { + next = 0; + return; + } + } + + let updateDelay = details.updateDelay || + this.hiddenSettings.autoUpdatePeriod * 3600000; + const now = Date.now(); let needEmergencyUpdate = false; @@ -1632,13 +1643,7 @@ onBroadcast(msg => { } this.loadFilterLists(); } - if ( this.userSettings.autoUpdate ) { - this.scheduleAssetUpdater( - this.hiddenSettings.autoUpdatePeriod * 3600000 || 25200000 - ); - } else { - this.scheduleAssetUpdater(0); - } + this.scheduleAssetUpdater(); broadcast({ what: 'assetsUpdated', assetKeys: details.assetKeys diff --git a/src/js/support.js b/src/js/support.js index 05ebe60f6f468..9bfd7cbe3b3c2 100644 --- a/src/js/support.js +++ b/src/js/support.js @@ -255,8 +255,7 @@ async function updateFilterLists() { if ( dom.body.dataset.shouldUpdateLists === undefined ) { return false; } dom.cl.add(dom.body, 'updating'); const assetKeys = JSON.parse(dom.body.dataset.shouldUpdateLists); - vAPI.messaging.send('dashboard', { what: 'purgeCaches', assetKeys }); - vAPI.messaging.send('dashboard', { what: 'forceUpdateAssets' }); + vAPI.messaging.send('dashboard', { what: 'supportUpdateNow', assetKeys }); return true; } diff --git a/src/js/ublock.js b/src/js/ublock.js index 18a8b94039774..e963377c3f3c2 100644 --- a/src/js/ublock.js +++ b/src/js/ublock.js @@ -345,7 +345,7 @@ const matchBucket = function(url, hostname, bucket, start) { } break; case 'autoUpdate': - this.scheduleAssetUpdater(value ? 7 * 60 * 1000 : 0); + this.scheduleAssetUpdater({ updateDelay: value ? 2000 : 0 }); break; case 'cnameUncloakEnabled': if ( vAPI.net.canUncloakCnames === true ) {