diff --git a/README.md b/README.md index 49bb816..7535cbb 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Builds HTTPS Everywhere ruleset files for Brave. ## Configuring -Update `xpiVersion` in `scripts/preloadHTTPSE.js` to the latest release number from https://www.eff.org/https-everywhere. If there are rulesets that are broken and need to be disabled, add them to the `exclusions` list. +If there are rulesets that are broken and need to be disabled, add them to the `exclusions` list. If there is a breaking ruleset format change, bump the version number in `package.json` by one major point release. (Ex: 5.2.21 to 5.3.0. The minor diff --git a/scripts/preloadHTTPSE.js b/scripts/preloadHTTPSE.js index b1800ca..70895ba 100644 --- a/scripts/preloadHTTPSE.js +++ b/scripts/preloadHTTPSE.js @@ -1,34 +1,50 @@ 'use strict' const fs = require('fs') -const http = require('https') +const zlib = require('zlib') +const path = require('path') +const https = require('https') const levelup = require('level') const rmDir = require('./util').rmDir const exec = require('child_process').exec -const xpiVersion = '2018.9.19' // Manually update this to latest version - const downloadRulesets = (dir, cb) => { - const downloadURL = `https://www.eff.org/files/https-everywhere-${xpiVersion}-eff.xpi` - const xpiFile = fs.createWriteStream('httpse.xpi') - http.get(downloadURL, (response) => { - response.pipe(xpiFile) - xpiFile.on('finish', () => { - xpiFile.close(() => { - exec('unzip ../httpse.xpi', { - cwd: 'https-everywhere' - }, (err) => { - if (err) { - throw err - } else { + let timestamp = '' + let baseURL = 'https://www.https-rulesets.org/v1/' + + // Obtain the latest rulesets timestamp from EFF's official endpoint + https.get(baseURL + 'latest-rulesets-timestamp', (response) => { + response.on('data', (data) => { + timestamp += data.toString() + }) + + // Download the rulesets once we obtained the timestamp + response.on('end', () => { + // ${timestamp} comes with trailing newlines, parse it and convert it back + let target = `default.rulesets.${Number(timestamp)}.gz` + + https.get(baseURL + target, (stream) => { + // ${target} is gzipped, gunzip accordingly + // and pipe the output to ${filename} + let filename = path.join(dir, 'default.rulesets') + let output = fs.createWriteStream(filename) + + stream.pipe(zlib.createGunzip()).pipe(output) + + output.on('finish', () => { + output.close(() => { + // everything is fine here cb() - } + }, (err) => { + console.log(`ERROR: Failed to write to ${filename}: ${err}`) + }) }) - }) - }) - }) - .on('error', (err) => { - console.log(`Error downloading ${downloadURL}`, err) + }).on('error', (err) => { + console.log(`ERROR: Failed to retrieve ${target}: ${err}`) + }).end() }) + }).on('error', (err) => { + console.log(`ERROR: Failed to retrieve the latest rulesets timestamp: ${err}`) + }).end() } const buildDataFiles = () => { @@ -39,7 +55,10 @@ const buildDataFiles = () => { 'Digg (partial)': 'breaks digg.com on C70+ with NET::ERR_CERT_SYMANTEC_LEGACY' } - const rulesets = JSON.parse(fs.readFileSync('./https-everywhere/rules/default.rulesets', 'utf8')) + let rulesets = JSON.parse(fs.readFileSync('./https-everywhere/rules/default.rulesets', 'utf8')) + if (rulesets != null) { + rulesets = rulesets.rulesets + } let jsonOutput = { 'rulesetStrings': [], @@ -157,8 +176,9 @@ const buildDataFiles = () => { rmDir('./https-everywhere') fs.mkdirSync('./https-everywhere') +fs.mkdirSync('./https-everywhere/rules') rmDir('./out') fs.mkdirSync('./out') console.log('downloading rulesets') -downloadRulesets('./https-everywhere', buildDataFiles) +downloadRulesets('./https-everywhere/rules', buildDataFiles)