From 544d9844969d1addd864b296187e56bdb354465c Mon Sep 17 00:00:00 2001 From: yan Date: Wed, 17 Oct 2018 19:19:35 -0700 Subject: [PATCH 1/4] add npm command for running a network audit 'npm run network-audit' produces a JSON file, network-audit-results.json, which contains the URL requests and 307's in the current build. it exits with non-zero status if any of the URLs in the audit are not whitelisted --- .gitignore | 2 ++ lib/start.js | 53 ++++++++++++++++++++++++++++++++++- lib/whitelistedUrlPrefixes.js | 13 +++++++++ package.json | 1 + scripts/commands.js | 1 + 5 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 lib/whitelistedUrlPrefixes.js diff --git a/.gitignore b/.gitignore index d53ebf3c6d8a..74e2b600f7a8 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ npm-debug.log .vscode .cipd .idea +network_log.json +network-audit-results.json # Rendered Sphinx files should be excluded from source control build diff --git a/lib/start.js b/lib/start.js index 378b7a1d5bc7..3429b7d8359f 100644 --- a/lib/start.js +++ b/lib/start.js @@ -1,6 +1,8 @@ const path = require('path') +const fs = require('fs-extra') const config = require('../lib/config') const util = require('../lib/util') +const whitelistedUrlPrefixes = require('./whitelistedUrlPrefixes') const start = (buildConfig = config.defaultBuildConfig, options) => { config.buildConfig = buildConfig @@ -44,8 +46,8 @@ const start = (buildConfig = config.defaultBuildConfig, options) => { if (options.rewards_reconcile_interval) { braveArgs.push(`--rewards-reconcile-interval=${options.rewards_reconcile_interval}`) } + let user_data_dir if (options.user_data_dir_name) { - let user_data_dir if (process.platform === 'darwin') { user_data_dir = path.join(process.env.HOME, 'Library', 'Application\\ Support', 'BraveSoftware', options.user_data_dir_name) } else if (process.platform === 'win32') { @@ -55,9 +57,19 @@ const start = (buildConfig = config.defaultBuildConfig, options) => { } braveArgs.push('--user-data-dir=' + user_data_dir); } + const networkLogFile = path.resolve(path.join(__dirname, '..', 'network_log.json')) + if (options.network_log) { + braveArgs.push(`--log-net-log=${networkLogFile}`) + braveArgs.push(`--net-log-capture-mode=IncludeSocketBytes`) + if (user_data_dir) { + // clear the data directory before doing a network test + fs.removeSync(user_data_dir.replace('\\', '')) + } + } let cmdOptions = { stdio: 'inherit', + timeout: options.network_log ? 120000 : undefined, shell: true } @@ -68,6 +80,45 @@ const start = (buildConfig = config.defaultBuildConfig, options) => { } else { util.run(path.join(config.outputDir, 'brave'), braveArgs, cmdOptions) } + + if (options.network_log) { + let exitCode = 0 + // Read the network log + const jsonOutput = fs.readJsonSync(networkLogFile) + const URL_REQUEST_TYPE = jsonOutput.constants.logSourceType.URL_REQUEST + const URL_REQUEST_FAKE_RESPONSE_HEADERS_CREATED = jsonOutput.constants.logEventTypes.URL_REQUEST_FAKE_RESPONSE_HEADERS_CREATED + const urlRequests = jsonOutput.events.filter((event) => { + if (event.type === URL_REQUEST_FAKE_RESPONSE_HEADERS_CREATED) { + // showing these helps determine which URL requests which don't + // actually hit the network + return true + } + if (event.source.type === URL_REQUEST_TYPE) { + if (!event.params) { + return false + } + const url = event.params.url + if (!url) { + return false + } + if (url.startsWith('http') && url.includes('.')) { + const found = whitelistedUrlPrefixes.find((prefix) => { + return url.startsWith(prefix) + }) + if (!found) { + // This is not a whitelisted URL! log it and exit with non-zero + console.log('Un-whitelisted URL found:', url) + exitCode = 1 + } + return true + } + } + return false + }) + // TODO: parse this and fail travis if it includes unexpected events + fs.writeJsonSync('network-audit-results.json', urlRequests) + process.exit(exitCode) + } } module.exports = start diff --git a/lib/whitelistedUrlPrefixes.js b/lib/whitelistedUrlPrefixes.js new file mode 100644 index 000000000000..b132925a6a42 --- /dev/null +++ b/lib/whitelistedUrlPrefixes.js @@ -0,0 +1,13 @@ +module.exports = [ + 'https://update.googleapis.com/service/update2', // allowed because it 307's to go-updater.brave.com. should never actually connect to googleapis.com. + 'https://no-thanks.invalid/', // fake gaia URL + 'https://go-updater.brave.com/', + 'https://safebrowsing.brave.com/', + 'https://brave-core-ext.s3.brave.com/', + 'https://laptop-updates.brave.com/', + 'https://ledger.mercury.basicattentiontoken.org/', + 'https://balance.mercury.basicattentiontoken.org/', + 'https://publishers.basicattentiontoken.org/', + 'https://updates.bravesoftware.com/', // remove this once updates are moved to the prod environment + 'https://pdfjs.robwu.nl/logpdfjs' // allowed because it gets canceled in tracking protection +] diff --git a/package.json b/package.json index 4399831a7c9b..21a0e4c34e2a 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "update_patches": "node ./scripts/commands.js update_patches", "apply_patches": "node ./scripts/sync.js --run_hooks", "start": "node ./scripts/commands.js start", + "network-audit": "node ./scripts/commands.js start --enable_brave_update --network_log --user_data_dir_name=brave-network-test", "push_l10n": "node ./scripts/commands.js push_l10n", "pull_l10n": "node ./scripts/commands.js pull_l10n", "chromium_rebase_l10n": "node ./scripts/commands.js chromium_rebase_l10n", diff --git a/scripts/commands.js b/scripts/commands.js index 93861b1402eb..6a69afae3788 100644 --- a/scripts/commands.js +++ b/scripts/commands.js @@ -77,6 +77,7 @@ program .option('--rewards_env [server]', 'switch between staging and production', /^(stag|prod)$/i) .option('--rewards_reconcile_interval [reconcile_interval]', 'set reconcile interval for contribution in minutes', parseInt) .option('--single_process', 'use a single process') + .option('--network_log', 'log network activity to network_log.json') .arguments('[build_config]') .action(start) From c5dc59aaa691314bee61eca446cfe6ce7d7f5c95 Mon Sep 17 00:00:00 2001 From: yan Date: Wed, 17 Oct 2018 23:57:38 -0700 Subject: [PATCH 2/4] add npm test-security command --- lib/start.js | 7 +++++-- package.json | 3 ++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/start.js b/lib/start.js index 3429b7d8359f..cf6796a51faa 100644 --- a/lib/start.js +++ b/lib/start.js @@ -73,6 +73,10 @@ const start = (buildConfig = config.defaultBuildConfig, options) => { shell: true } + if (options.network_log) { + console.log('Network audit started. Logging requests for the next 2min...') + } + if (process.platform === 'darwin') { util.run(path.join(config.outputDir, config.macAppName() + '.app', 'Contents', 'MacOS', config.macAppName()), braveArgs, cmdOptions) } else if (process.platform === 'win32') { @@ -107,7 +111,7 @@ const start = (buildConfig = config.defaultBuildConfig, options) => { }) if (!found) { // This is not a whitelisted URL! log it and exit with non-zero - console.log('Un-whitelisted URL found:', url) + console.log('NETWORK AUDIT FAIL:', url) exitCode = 1 } return true @@ -115,7 +119,6 @@ const start = (buildConfig = config.defaultBuildConfig, options) => { } return false }) - // TODO: parse this and fail travis if it includes unexpected events fs.writeJsonSync('network-audit-results.json', urlRequests) process.exit(exitCode) } diff --git a/package.json b/package.json index 21a0e4c34e2a..60d578f8307b 100644 --- a/package.json +++ b/package.json @@ -17,7 +17,8 @@ "push_l10n": "node ./scripts/commands.js push_l10n", "pull_l10n": "node ./scripts/commands.js pull_l10n", "chromium_rebase_l10n": "node ./scripts/commands.js chromium_rebase_l10n", - "test": "node ./scripts/commands.js test" + "test": "node ./scripts/commands.js test", + "test-security": "npm audit && npm run network-audit" }, "config": { "projects": { From c1266de96d23efee6faabc8ef67395e8017031fd Mon Sep 17 00:00:00 2001 From: yan Date: Thu, 18 Oct 2018 00:41:05 -0700 Subject: [PATCH 3/4] add BAT staging urls to whitelist --- lib/whitelistedUrlPrefixes.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/whitelistedUrlPrefixes.js b/lib/whitelistedUrlPrefixes.js index b132925a6a42..8c57d26eeb0a 100644 --- a/lib/whitelistedUrlPrefixes.js +++ b/lib/whitelistedUrlPrefixes.js @@ -6,8 +6,11 @@ module.exports = [ 'https://brave-core-ext.s3.brave.com/', 'https://laptop-updates.brave.com/', 'https://ledger.mercury.basicattentiontoken.org/', + 'https://ledger-staging.mercury.basicattentiontoken.org/', 'https://balance.mercury.basicattentiontoken.org/', + 'https://balance-staging.mercury.basicattentiontoken.org/', 'https://publishers.basicattentiontoken.org/', + 'https://publishers-staging.basicattentiontoken.org/', 'https://updates.bravesoftware.com/', // remove this once updates are moved to the prod environment 'https://pdfjs.robwu.nl/logpdfjs' // allowed because it gets canceled in tracking protection ] From f81bada3d837af73f401d94358d825498894fc4a Mon Sep 17 00:00:00 2001 From: yan Date: Thu, 18 Oct 2018 00:41:43 -0700 Subject: [PATCH 4/4] Allow network audit to be aborted before 2minutes --- lib/start.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/start.js b/lib/start.js index cf6796a51faa..794a7251ada1 100644 --- a/lib/start.js +++ b/lib/start.js @@ -70,11 +70,12 @@ const start = (buildConfig = config.defaultBuildConfig, options) => { let cmdOptions = { stdio: 'inherit', timeout: options.network_log ? 120000 : undefined, + continueOnFail: options.network_log ? true : false, shell: true } if (options.network_log) { - console.log('Network audit started. Logging requests for the next 2min...') + console.log('Network audit started. Logging requests for the next 2min or until you quit Brave...') } if (process.platform === 'darwin') {