add npm command for running a network audit

'npm run network-audit' produces a JSON file,
network-audit-results.json, which contains the URL requests and 307's in
the current build.

it exits with non-zero status if any of the URLs in the audit are not
whitelisted

.gitignore

@@ -8,6 +8,8 @@ npm-debug.log
 .vscode
 .cipd
 .idea
+network_log.json
+network-audit-results.json
 
 # Rendered Sphinx files should be excluded from source control
 build

lib/start.js

@@ -1,6 +1,8 @@
 const path = require('path')
+const fs = require('fs-extra')
 const config = require('../lib/config')
 const util = require('../lib/util')
+const whitelistedUrlPrefixes = require('./whitelistedUrlPrefixes')
 
 const start = (buildConfig = config.defaultBuildConfig, options) => {
   config.buildConfig = buildConfig
@@ -44,8 +46,8 @@ const start = (buildConfig = config.defaultBuildConfig, options) => {
   if (options.rewards_reconcile_interval) {
     braveArgs.push(`--rewards-reconcile-interval=${options.rewards_reconcile_interval}`)
   }
+  let user_data_dir
   if (options.user_data_dir_name) {
-    let user_data_dir
     if (process.platform === 'darwin') {
       user_data_dir = path.join(process.env.HOME, 'Library', 'Application\\ Support', 'BraveSoftware', options.user_data_dir_name)
     } else if (process.platform === 'win32') {
@@ -55,9 +57,19 @@ const start = (buildConfig = config.defaultBuildConfig, options) => {
     }
     braveArgs.push('--user-data-dir=' + user_data_dir);
   }
+  const networkLogFile = path.resolve(path.join(__dirname, '..', 'network_log.json'))
+  if (options.network_log) {
+    braveArgs.push(`--log-net-log=${networkLogFile}`)
+    braveArgs.push(`--net-log-capture-mode=IncludeSocketBytes`)
+    if (user_data_dir) {
+      // clear the data directory before doing a network test
+      fs.removeSync(user_data_dir.replace('\\', ''))
+    }
+  }
 
   let cmdOptions = {
     stdio: 'inherit',
+    timeout: options.network_log ? 120000 : undefined,
     shell: true
   }
 
@@ -68,6 +80,45 @@ const start = (buildConfig = config.defaultBuildConfig, options) => {
   } else {
     util.run(path.join(config.outputDir, 'brave'), braveArgs, cmdOptions)
   }
+
+  if (options.network_log) {
+    let exitCode = 0
+    // Read the network log
+    const jsonOutput = fs.readJsonSync(networkLogFile)
+    const URL_REQUEST_TYPE = jsonOutput.constants.logSourceType.URL_REQUEST
+    const URL_REQUEST_FAKE_RESPONSE_HEADERS_CREATED = jsonOutput.constants.logEventTypes.URL_REQUEST_FAKE_RESPONSE_HEADERS_CREATED
+    const urlRequests = jsonOutput.events.filter((event) => {
+      if (event.type === URL_REQUEST_FAKE_RESPONSE_HEADERS_CREATED) {
+        // showing these helps determine which URL requests which don't
+        // actually hit the network
+        return true
+      }
+      if (event.source.type === URL_REQUEST_TYPE) {
+        if (!event.params) {
+          return false
+        }
+        const url = event.params.url
+        if (!url) {
+          return false
+        }
+        if (url.startsWith('http') && url.includes('.')) {
+          const found = whitelistedUrlPrefixes.find((prefix) => {
+            return url.startsWith(prefix)
+          })
+          if (!found) {
+            // This is not a whitelisted URL! log it and exit with non-zero
+            console.log('Un-whitelisted URL found:', url)
+            exitCode = 1
+          }
+          return true
+        }
+      }
+      return false
+    })
+    // TODO: parse this and fail travis if it includes unexpected events
+    fs.writeJsonSync('network-audit-results.json', urlRequests)
+    process.exit(exitCode)
+  }
 }
 
 module.exports = start

lib/whitelistedUrlPrefixes.js

@@ -0,0 +1,13 @@
+module.exports = [
+  'https://update.googleapis.com/service/update2', // allowed because it 307's to go-updater.brave.com. should never actually connect to googleapis.com.
+  'https://no-thanks.invalid/', // fake gaia URL
+  'https://go-updater.brave.com/',
+  'https://safebrowsing.brave.com/',
+  'https://brave-core-ext.s3.brave.com/',
+  'https://laptop-updates.brave.com/',
+  'https://ledger.mercury.basicattentiontoken.org/',
+  'https://balance.mercury.basicattentiontoken.org/',
+  'https://publishers.basicattentiontoken.org/',
+  'https://updates.bravesoftware.com/', // remove this once updates are moved to the prod environment
+  'https://pdfjs.robwu.nl/logpdfjs' // allowed because it gets canceled in tracking protection
+]

package.json

@@ -13,6 +13,7 @@
     "update_patches": "node ./scripts/commands.js update_patches",
     "apply_patches": "node ./scripts/sync.js --run_hooks",
     "start": "node ./scripts/commands.js start",
+    "network-audit": "node ./scripts/commands.js start --enable_brave_update --network_log --user_data_dir_name=brave-network-test",
     "push_l10n": "node ./scripts/commands.js push_l10n",
     "pull_l10n": "node ./scripts/commands.js pull_l10n",
     "chromium_rebase_l10n": "node ./scripts/commands.js chromium_rebase_l10n",

scripts/commands.js

@@ -77,6 +77,7 @@ program
   .option('--rewards_env [server]', 'switch between staging and production', /^(stag|prod)$/i)
   .option('--rewards_reconcile_interval [reconcile_interval]', 'set reconcile interval for contribution in minutes', parseInt)
   .option('--single_process', 'use a single process')
+  .option('--network_log', 'log network activity to network_log.json')
   .arguments('[build_config]')
   .action(start)