Merge pull request #1682 from brave/feature/network-audit

add npm command for running a network audit

.gitignore

@@ -6,6 +6,8 @@ npm-debug.log
 .sccache
 **.sw[po]
 .idea
+network_log.json
+network-audit-results.json
 
 # Clion files
 CMakeLists.txt

lib/start.js

@@ -1,6 +1,8 @@
 const path = require('path')
+const fs = require('fs-extra')
 const config = require('../lib/config')
 const util = require('../lib/util')
+const whitelistedUrlPrefixes = require('./whitelistedUrlPrefixes')
 
 const start = (buildConfig = config.defaultBuildConfig, options) => {
   config.buildConfig = buildConfig
@@ -41,8 +43,8 @@ const start = (buildConfig = config.defaultBuildConfig, options) => {
   if (options.rewards_reconcile_interval) {
     braveArgs.push(`--rewards-reconcile-interval=${options.rewards_reconcile_interval}`)
   }
+  let user_data_dir
   if (options.user_data_dir_name) {
-    let user_data_dir
     if (process.platform === 'darwin') {
       user_data_dir = path.join(process.env.HOME, 'Library', 'Application\\ Support', 'BraveSoftware', options.user_data_dir_name)
     } else if (process.platform === 'win32') {
@@ -52,19 +54,72 @@ const start = (buildConfig = config.defaultBuildConfig, options) => {
     }
     braveArgs.push('--user-data-dir=' + user_data_dir);
   }
+  const networkLogFile = path.resolve(path.join(__dirname, '..', 'network_log.json'))
+  if (options.network_log) {
+    braveArgs.push(`--log-net-log=${networkLogFile}`)
+    braveArgs.push(`--net-log-capture-mode=IncludeSocketBytes`)
+    if (user_data_dir) {
+      // clear the data directory before doing a network test
+      fs.removeSync(user_data_dir.replace('\\', ''))
+    }
+  }
 
   let cmdOptions = {
     stdio: 'inherit',
+    timeout: options.network_log ? 120000 : undefined,
+    continueOnFail: options.network_log ? true : false,
     shell: true
   }
 
+  if (options.network_log) {
+    console.log('Network audit started. Logging requests for the next 2min or until you quit Brave...')
+  }
+
   if (process.platform === 'darwin') {
     util.run(path.join(config.outputDir, config.macAppName() + '.app', 'Contents', 'MacOS', config.macAppName()), braveArgs, cmdOptions)
   } else if (process.platform === 'win32') {
     util.run(path.join(config.outputDir, 'brave.exe'), braveArgs, cmdOptions)
   } else {
     util.run(path.join(config.outputDir, 'brave'), braveArgs, cmdOptions)
   }
+
+  if (options.network_log) {
+    let exitCode = 0
+    // Read the network log
+    const jsonOutput = fs.readJsonSync(networkLogFile)
+    const URL_REQUEST_TYPE = jsonOutput.constants.logSourceType.URL_REQUEST
+    const URL_REQUEST_FAKE_RESPONSE_HEADERS_CREATED = jsonOutput.constants.logEventTypes.URL_REQUEST_FAKE_RESPONSE_HEADERS_CREATED
+    const urlRequests = jsonOutput.events.filter((event) => {
+      if (event.type === URL_REQUEST_FAKE_RESPONSE_HEADERS_CREATED) {
+        // showing these helps determine which URL requests which don't
+        // actually hit the network
+        return true
+      }
+      if (event.source.type === URL_REQUEST_TYPE) {
+        if (!event.params) {
+          return false
+        }
+        const url = event.params.url
+        if (!url) {
+          return false
+        }
+        if (url.startsWith('http') && url.includes('.')) {
+          const found = whitelistedUrlPrefixes.find((prefix) => {
+            return url.startsWith(prefix)
+          })
+          if (!found) {
+            // This is not a whitelisted URL! log it and exit with non-zero
+            console.log('NETWORK AUDIT FAIL:', url)
+            exitCode = 1
+          }
+          return true
+        }
+      }
+      return false
+    })
+    fs.writeJsonSync('network-audit-results.json', urlRequests)
+    process.exit(exitCode)
+  }
 }
 
 module.exports = start

lib/whitelistedUrlPrefixes.js

@@ -0,0 +1,16 @@
+module.exports = [
+  'https://update.googleapis.com/service/update2', // allowed because it 307's to go-updater.brave.com. should never actually connect to googleapis.com.
+  'https://no-thanks.invalid/', // fake gaia URL
+  'https://go-updater.brave.com/',
+  'https://safebrowsing.brave.com/',
+  'https://brave-core-ext.s3.brave.com/',
+  'https://laptop-updates.brave.com/',
+  'https://ledger.mercury.basicattentiontoken.org/',
+  'https://ledger-staging.mercury.basicattentiontoken.org/',
+  'https://balance.mercury.basicattentiontoken.org/',
+  'https://balance-staging.mercury.basicattentiontoken.org/',
+  'https://publishers.basicattentiontoken.org/',
+  'https://publishers-staging.basicattentiontoken.org/',
+  'https://updates.bravesoftware.com/', // remove this once updates are moved to the prod environment
+  'https://pdfjs.robwu.nl/logpdfjs' // allowed because it gets canceled in tracking protection
+]

package.json

@@ -13,10 +13,12 @@
     "update_patches": "node ./scripts/commands.js update_patches",
     "apply_patches": "node ./scripts/sync.js --run_hooks",
     "start": "node ./scripts/commands.js start",
+    "network-audit": "node ./scripts/commands.js start --enable_brave_update --network_log --user_data_dir_name=brave-network-test",
     "push_l10n": "node ./scripts/commands.js push_l10n",
     "pull_l10n": "node ./scripts/commands.js pull_l10n",
     "chromium_rebase_l10n": "node ./scripts/commands.js chromium_rebase_l10n",
-    "test": "node ./scripts/commands.js test"
+    "test": "node ./scripts/commands.js test",
+    "test-security": "npm audit && npm run network-audit"
   },
   "config": {
     "projects": {

scripts/commands.js

@@ -75,6 +75,7 @@ program
   .option('--rewards_env [server]', 'switch between staging and production', /^(stag|prod)$/i)
   .option('--rewards_reconcile_interval [reconcile_interval]', 'set reconcile interval for contribution in minutes', parseInt)
   .option('--single_process', 'use a single process')
+  .option('--network_log', 'log network activity to network_log.json')
   .arguments('[build_config]')
   .action(start)