Merge pull request #301 from mozilla-rally/switch-to-scripting-api

fixes issue #284, use new cross-browser scripting api for registering…

README.md

@@ -36,27 +36,27 @@ You will then be able to use those components of the API (e.g., `pageNavigation.
 Documentation for the latest release of WebScience is available at [https://mozilla-rally.github.io/web-science/jsdocs/](https://mozilla-rally.github.io/web-science/jsdocs/). The documentation is automatically generated from JSDoc annotations in the WebScience implementation.
 
 ## API Implementation Progress
-| Module                   | No Breaking Changes Planned | No Known Bugs      | Good Documentation | Good Test Coverage |   
-| ------------------------ | --------------------------- | ------------------ | ------------------ | ------------------ |
-| `debugging`              | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                |
-| `events`                 | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                |
-| `id`                     | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                |
-| `idle`                   | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                |
-| `inline`                 | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                |
-| `linkExposure`           | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                |
-| `linkResolution`         | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                |
-| `matching`               | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                |
-| `messaging`              | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                |
-| `pageManager`            | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                |
-| `pageNavigation`         | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                |
-| `pageText`               | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                |
-| `pageTransition`         | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                |
-| `permissions`            | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                |
-| `randomization`          | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                |
-| `scheduling`             | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                |
-| `socialMediaActivity`    | :x: [#23](https://github.com/mozilla-rally/web-science/issues/23)                        | :white_check_mark: | :white_check_mark: | :x:                |
-| `socialMediaLinkSharing` | :x: [#17](https://github.com/mozilla-rally/web-science/issues/17)                        | :white_check_mark: | :white_check_mark: | :x:                |
-| `storage`                | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                |
-| `timing`                 | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                |
-| `userSurvey`             | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                |
-| `workers`                | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                |
+| Module                   | No Breaking Changes Planned | No Known Bugs      | Good Documentation | Good Test Coverage | Browser Support    |
+| ------------------------ | --------------------------- | ------------------ | ------------------ | ------------------ | ------------------ |
+| `debugging`              | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                | Firefox, Chrome    |
+| `events`                 | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                | Firefox, Chrome    |
+| `id`                     | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                | Firefox, Chrome    |
+| `idle`                   | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                | Firefox, Chrome    |
+| `inline`                 | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                | Firefox, Chrome    |
+| `linkExposure`           | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                | Firefox            |
+| `linkResolution`         | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                | Firefox            |
+| `matching`               | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                | Firefox, Chrome    |
+| `messaging`              | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                | Firefox, Chrome    |
+| `pageManager`            | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                | Firefox, Chrome    |
+| `pageNavigation`         | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                | Firefox, Chrome    |
+| `pageText`               | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                | Firefox            |
+| `pageTransition`         | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                | Firefox            |
+| `permissions`            | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                | Firefox, Chrome    |
+| `randomization`          | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                | Firefox, Chrome    |
+| `scheduling`             | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                | Firefox, Chrome    |
+| `socialMediaActivity`    | :x: [#23](https://github.com/mozilla-rally/web-science/issues/23)                        | :white_check_mark: | :white_check_mark: | :x:                | Firefox            |
+| `socialMediaLinkSharing` | :x: [#17](https://github.com/mozilla-rally/web-science/issues/17)                        | :white_check_mark: | :white_check_mark: | :x:                | Firefox             |
+| `storage`                | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                | Firefox, Chrome    |
+| `timing`                 | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                | Firefox, Chrome    |
+| `userSurvey`             | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                | Firefox            |
+| `workers`                | :white_check_mark:          | :white_check_mark: | :white_check_mark: | :x:                | Firefox, Chrome    |

scripts/integration-test.sh

@@ -13,15 +13,21 @@ XVFB_RUN() {
 
 export PATH=$PATH:./node_modules/.bin
 
-echo "Building test extension..."
+echo "Building test extension for manifest v2..."
 pwd
 pushd tests/integration/extension
 npm install ../../../
-npm install && npm run build && npm run package
+npm install && npm run package:mv2
 popd
-
 echo "Testing Firefox headless with extension"
 npm run test:integration:jest -- --test_browser=firefox --load_extension=true --headless_mode=true  2>&1 | tee integration.log
+
+echo "Building test extension for manifest v3..."
+pwd
+pushd tests/integration/extension
+npm install ../../../
+npm install && npm run package:mv3
+popd
 # NOTE Chrome Headless mode does not support extensions, so we use `xvfb` as the display server.
 echo "Testing Chrome non-headless with extension"
 XVFB_RUN npm run test:integration:jest -- --test_browser=chrome --load_extension=true --headless_mode=false 2>&1 | tee integration.log

src/pageManager.js

@@ -125,7 +125,6 @@ import * as idle from "./idle.js";
 import * as messaging from "./messaging.js";
 import * as permissions from "./permissions.js";
 import * as timing from "./timing.js";
-import * as contentScripts from "./contentScripts.js"
 import pageManagerContentScript from "include:./content-scripts/pageManager.content.js";
 
 /**
@@ -704,7 +703,16 @@ export async function initialize() {
     }
 
     // Register the pageManager content script for all URLs permitted by the extension manifest.
-    contentScripts.registerContentScript(permissions.getManifestOriginMatchPatterns(), pageManagerContentScript);
+    const matchPatterns = permissions.getManifestOriginMatchPatterns();
+
+    // TODO register this content script so it may be unloaded later. 
+    await browser.scripting.registerContentScripts([{
+        id: "pageManager",
+        js: ["dist/browser-polyfill.min.js", pageManagerContentScript],
+        matches: matchPatterns,
+        persistAcrossSessions: false,
+        runAt: "document_start"
+    }]);
 
     initializing = false;
     initialized = true;

src/pageNavigation.js

@@ -9,7 +9,6 @@ import * as events from "./events.js";
 import * as messaging from "./messaging.js";
 import * as pageManager from "./pageManager.js";
 import * as matching from "./matching.js";
-import * as contentScripts from "./contentScripts.js";
 import pageNavigationContentScript from "include:./content-scripts/pageNavigation.content.js";
 
 /**
@@ -36,7 +35,7 @@ import pageNavigationContentScript from "include:./content-scripts/pageNavigatio
  * @typedef {Object} PageDataListenerRecord
  * @property {matching.MatchPatternSet} matchPatternSet - The match patterns for the listener.
  * @property {boolean} privateWindows - Whether to notify the listener about pages in private windows.
- * @property {browser.contentScripts.RegisteredContentScript} contentScript - The content
+ * @property {browser.scripting.RegisteredContentScript} contentScript - The content
  * script associated with the listener.
  * @private
  */
@@ -148,7 +147,13 @@ async function addListener(listener, {
 
     // Compile the match patterns for the listener
     const matchPatternSet = matching.createMatchPatternSet(matchPatterns);
-    const contentScript = await contentScripts.registerContentScript(matchPatterns, pageNavigationContentScript);
+    const contentScript = await browser.scripting.registerContentScripts([{
+        id: "pageNavigation",
+        js: ["dist/browser-polyfill.min.js", pageNavigationContentScript],
+        matches: matchPatterns,
+        persistAcrossSessions: false,
+        runAt: "document_start"
+    }]);
 
     // Store a record for the listener
     pageDataListeners.set(listener, {

src/permissions.js

@@ -202,17 +202,30 @@ export async function check({
  */
 export function getManifestOriginMatchPatterns() {
     const manifest = browser.runtime.getManifest();
+
+    const manifestPermissions = [ ];
+
+    // Manifest v3 requires host match patterns to be in an array under the host_permissions key.
+    // For manifest v2, continue to look in the permissions key.
+    // @see https://developer.chrome.com/docs/extensions/mv3/intro/mv3-migration/#host-permissions
+    ("permissions" in manifest) &&
+    Array.isArray(manifest.permissions) &&
+    manifestPermissions.push(...manifest.permissions);
+
+    ("host_permissions" in manifest) &&
+    Array.isArray(manifest.host_permissions) &&
+    manifestPermissions.push(...manifest.host_permissions);
+
     const matchPatterns = [ ];
-    if(("permissions" in manifest) && Array.isArray(manifest.permissions)) {
-        for(const permission of manifest.permissions) {
-            try {
-                matching.matchPatternsToRegExp([ permission ]);
-                matchPatterns.push(permission);
-            }
-            catch(error) {
-                continue;
-            }
+    for(const permission of manifestPermissions) {
+        try {
+            matching.matchPatternsToRegExp([ permission ]);
+            matchPatterns.push(permission);
+        }
+        catch(error) {
+            continue;
         }
     }
+
     return matchPatterns;
 }

src/webScience.js

@@ -67,7 +67,3 @@ export { workers }
 
 import * as pageTransition from "./pageTransition.js"
 export { pageTransition }
-
-import * as contentScripts from "./contentScripts.js"
-export { contentScripts }
-

tests/integration/extension/manifest.json → tests/integration/extension/manifest.mv2.json

@@ -13,30 +13,16 @@
   },
   "background": {
     "scripts": [
-      "dist/browser-polyfill.min.js",
       "dist/background.js"
     ]
   },
-  "content_scripts": [
-    {
-      "matches": [
-        "<all_urls>"
-      ],
-      "js": [
-        "dist/browser-polyfill.min.js",
-        "dist/webScience/pageManager.content.js",
-        "dist/webScience/pageNavigation.content.js",
-        "dist/test.content.js"
-      ],
-      "run_at": "document_start"
-    }
-  ],
   "permissions": [
-    "<all_urls>",
     "idle",
     "webNavigation",
     "webRequest",
-    "unlimitedStorage"
+    "unlimitedStorage",
+    "scripting",
+    "<all_urls>"
   ],
   "incognito": "not_allowed"
-}
+}

tests/integration/extension/manifest.mv3.json

@@ -0,0 +1,22 @@
+{
+  "description": "A test extension for the WebScience library.",
+  "author": "Mozilla",
+  "manifest_version": 3,
+  "name": "WebScience Test",
+  "version": "1.0.0",
+  "homepage_url": "https://github.com/mozilla-rally/web-science",
+  "background": {
+    "service_worker": "dist/background-loader.js"
+  },
+  "permissions": [
+    "idle",
+    "webNavigation",
+    "webRequest",
+    "unlimitedStorage",
+    "scripting"
+  ],
+  "host_permissions": [
+    "<all_urls>"
+  ],
+  "incognito": "not_allowed"
+}

tests/integration/extension/package.json

@@ -5,8 +5,9 @@
   "main": "dist/background.js",
   "scripts": {
     "build": "rollup -c",
-    "dev": "rollup -c -w --config-enable-developer-mode",
-    "package": "web-ext build --overwrite-dest",
+    "dev": "cp manifest.mv3.json manifest.json && rollup -c -w --config-enable-developer-mode",
+    "package:mv2": "cp manifest.mv2.json manifest.json && rollup -c && web-ext build --overwrite-dest",
+    "package:mv3": "cp manifest.mv3.json manifest.json && rollup -c && web-ext build --overwrite-dest",
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "author": "",

tests/integration/extension/rollup.config.js

@@ -5,6 +5,19 @@ import webScienceRollupPlugin from "@mozilla/web-science/rollup-plugin";
 
 export default (cliArgs) => {
   const rollupConfig = [
+    {
+      input: "src/background-loader.js",
+      output: {
+        file: "dist/background-loader.js",
+      },
+      plugins: [
+        webScienceRollupPlugin(),
+        resolve({
+          browser: true,
+        }),
+        commonjs()
+      ]
+    },
     {
       input: "src/background.js",
       output: {

tests/integration/extension/src/background-loader.js

@@ -0,0 +1,8 @@
+export {};
+
+try {
+    // @ts-ignore
+    importScripts("browser-polyfill.min.js", "background.js");
+} catch (ex) {
+    console.error("Could not load scripts from service worker:", ex);
+}

tests/integration/extension/src/background.js

@@ -2,6 +2,9 @@ import * as webScience from "@mozilla/web-science"
 import browser from "webextension-polyfill";
 
 async function sendMessageToSelenium(message) {
+    // Log to console for Firefox
+    console.debug(message);
+
     // Send message to web content for Chromium
     const tabs = await browser.tabs.query({});
     console.debug(tabs);
@@ -25,4 +28,15 @@ async function pageDataListener(pageData) {
 webScience.pageManager.onPageVisitStart.addListener(pageVisitStartListener);
 webScience.pageManager.onPageVisitStop.addListener(pageVisitStopListener);
 
-webScience.pageNavigation.onPageData.addListener(pageDataListener, { matchPatterns: ["<all_urls>"] });
+webScience.pageNavigation.onPageData.addListener(pageDataListener, { matchPatterns: ["<all_urls>"] });
+
+// Load content script(s) required by this extension.
+browser.scripting.registerContentScripts([{
+    id: "webextension-test",
+    js: ["dist/browser-polyfill.min.js", "dist/test.content.js"],
+    matches: ["<all_urls>"],
+    persistAcrossSessions: false,
+    runAt: "document_start"
+}])
+    .then(result => console.debug(result))
+    .catch(err => console.err(err));

tests/integration/extension/src/test.content.js

@@ -1,3 +1,3 @@
 browser.runtime.onMessage.addListener(request => {
     console.log(JSON.stringify(request));
-});
+});