diff --git a/.github/workflows/custom-actions/set-commit-status/action.yaml b/.github/workflows/custom-actions/set-commit-status/action.yaml new file mode 100644 index 0000000000..6798bd84ef --- /dev/null +++ b/.github/workflows/custom-actions/set-commit-status/action.yaml @@ -0,0 +1,41 @@ +name: Set Commit Status + +description: | + A reusable action that sets the commit status. This is used to set PR status + from workflows with non-PR triggers (such as manually-triggered workflows). + +inputs: + context: + description: An arbitrary string to identify the status check. + required: true + state: + description: Either "pending", "error", "success", or "failure". + required: true + ref: + description: A git ref for which to set the commit status. For PRs, use the head commit, not the merge commit. Defaults to HEAD. + required: false + default: HEAD + token: + description: A GitHub access token. + required: true + +runs: + using: composite + steps: + - name: Report Commit Status + shell: bash + run: | + # This is the URL to view this workflow run on GitHub. It will be + # attached to the commit status, so that when someone clicks "details" + # next to the status on the PR, it will link to this run where they can + # see the logs. + RUN_URL="https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}" + SHA1=$(git rev-parse "${{ inputs.ref }}") + + GITHUB_TOKEN=${{ inputs.token }} \ + gh api \ + -X POST \ + -F "context=${{ inputs.context }}" \ + -F "state=${{ inputs.state }}" \ + -F "target_url=$RUN_URL" \ + "repos/${{ github.repository }}/statuses/$SHA1" diff --git a/.github/workflows/selenium-lab-tests.yaml b/.github/workflows/selenium-lab-tests.yaml new file mode 100644 index 0000000000..c0e25cb47b --- /dev/null +++ b/.github/workflows/selenium-lab-tests.yaml @@ -0,0 +1,93 @@ +name: Selenium Lab Tests + +on: + workflow_dispatch: + # Allows for manual triggering on PRs. They should be reviewed first, to + # avoid malicious code executing in the lab. + inputs: + pr: + description: "A PR number to build and test in the lab. If empty, will build and test from main." + required: false + schedule: + # Runs every night at 2am PST / 10am UTC, testing against the main branch. + - cron: '0 10 * * *' + +# Only one run of this workflow is allowed at a time, since it uses physical +# resources in our lab. +concurrency: selenium-lab + +jobs: + lab-tests: + # This is a self-hosted runner in a Docker container, with access to our + # lab's Selenium grid on port 4444. + runs-on: self-hosted-selenium + + steps: + # This runs on our self-hosted runner, and the Docker image it is based + # on doesn't have GitHub's CLI pre-installed. This installs it. Taken + # verbatim from the official installation instructions at + # https://github.com/cli/cli/blob/trunk/docs/install_linux.md + - name: Install GitHub Actions CLI + run: | + curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null + sudo apt update + sudo apt install gh + + - name: Compute ref + run: | + if [[ "${{ github.event.inputs.pr }}" != "" ]]; then + echo LAB_TEST_REF="refs/pull/${{ github.event.inputs.pr }}/head" >> $GITHUB_ENV + else + echo LAB_TEST_REF="main" >> $GITHUB_ENV + fi + + - uses: actions/checkout@v2 + with: + ref: ${{ env.LAB_TEST_REF }} + + - name: Set Commit Status to Pending + uses: ./.github/workflows/custom-actions/set-commit-status + with: + context: Selenium Lab Tests + state: pending + token: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions/setup-node@v1 + with: + node-version: 16 + registry-url: 'https://registry.npmjs.org' + + # The Docker image for this self-hosted runner doesn't contain java. + - uses: actions/setup-java@v2 + with: + distribution: zulu + java-version: 11 + + # The Docker image for this self-hosted runner has "python3" but not + # plain "python". + - name: Build Player + run: python3 build/all.py + + # Run tests on the Selenium grid in our lab. This uses a private + # hostname and TLS cert to get EME tests working on all platforms + # (since EME only works on https or localhost). + - name: Test Player + run: | + python3 build/test.py \ + --reporters spec --spec-hide-passed \ + --drm \ + --lets-encrypt-folder /etc/shakalab.rocks \ + --hostname karma.shakalab.rocks \ + --port 61731 \ + --grid-config build/shaka-lab.yaml \ + --grid-address selenium-grid.lab:4444 + + - name: Report Final Commit Status + # Will run on success or failure, but not if the workflow is cancelled. + if: ${{ success() || failure() }} + uses: ./.github/workflows/custom-actions/set-commit-status + with: + context: Selenium Lab Tests + state: ${{ job.status }} + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/build/shaka-lab.yaml b/build/shaka-lab.yaml new file mode 100644 index 0000000000..e8c071ffa5 --- /dev/null +++ b/build/shaka-lab.yaml @@ -0,0 +1,163 @@ +# Selenium grid config for the Shaka lab. This is the source of truth for the +# browsers and devices in the Shaka lab at Google. + +# For syntax and general information, see docs/selenium-grid-config.md + + +# A set of variables to contain repeated configurations which can then be +# referenced below. The syntax for the variable is "name: &name", which +# generates an "anchor" with the given name. Later, you can inject the +# contents of the variable with "*name". +vars: + firefox_config: &firefox_config + moz:firefoxOptions: + # Override Firefox default preferences in the temporary profile created + # for each test run. + prefs: + # Overrides Selenium's explicit default setting, to allow Firefox to + # install the Widevine CDM on demand. + media.gmp-manager.updateEnabled: true + # Overrides Firefox's Linux-specific default setting to disable DRM. + media.eme.enabled: true + + minimum_chrome_args: &minimum_chrome_args + # On Chrome m59+ we can test EME on platforms with pop-up prompts, such as + # Android and ChromeOS. Note that this flag does not take a port number + # (domain vs origin). + # This flag requires setting --user-data-dir as well, however, webdriver + # already takes care of that for us (except on Android). + - "--unsafely-allow-protected-media-identifier-for-domain=karma.shakalab.rocks" + # Normally, Chrome disallows autoplaying videos in many cases. Enable it + # for testing. + - "--autoplay-policy=no-user-gesture-required" + + minimum_chrome_android_args: &minimum_chrome_android_args + # There is no way in YAML to natively merge arrays, so we start by + # duplicating the flags from minimum_chrome_args above. + - "--unsafely-allow-protected-media-identifier-for-domain=karma.shakalab.rocks" + - "--autoplay-policy=no-user-gesture-required" + # On Android we must set --user-data-dir. WebDriver does not do it for + # us as it does on other platforms. Without --user-data-dir, + # --unsafely-allow... does not work. + - "--user-data-dir=/data/data/com.android.chrome/cache" + + minimum_chromeos_args: &minimum_chromeos_args + # There is no way in YAML to natively merge arrays, so we start by + # duplicating the flags from minimum_chrome_args above. + - "--unsafely-allow-protected-media-identifier-for-domain=karma.shakalab.rocks" + - "--autoplay-policy=no-user-gesture-required" + # Allow remote attestation even though the device may be in dev mode. This + # is critical for testing involving L1 content licenses. + - "--allow-ra-in-dev-mode" + + chrome_config: &chrome_config + goog:chromeOptions: + args: *minimum_chrome_args + + # Instruct chromedriver not to disable component updater. The + # component updater must run in order for the Widevine CDM to be + # available when using a new user-data-dir. + # TODO(http://crbug.com/613581): Remove once Chrome bug is fixed. + excludeSwitches: + - "disable-component-update" + + chrome_android_config: &chrome_android_config + goog:chromeOptions: + args: *minimum_chrome_android_args + + # Once the new session request reaches chromedriver, it will take + # the androidPackage option as a request to start Chrome through + # adb on the tethered device. + androidPackage: com.android.chrome + + chromeos_config: &chromeos_config + # Pass these client-specified arguments through generic-webdriver-server. + # This array will be appended after "--" instead of being set through one + # specific flag. For example, with ["--foo", "--bar=baz"] in the args, + # this would generate a command like: + # chromeos-webdriver-server.js -- --foo --bar=baz + # Those parameters will be passed on to Chrome instead of used by the + # WebDriver server. + generic:args: *minimum_chromeos_args + + +ChromeMac: + browser: chrome + os: Mac + extra_config: *chrome_config + +FirefoxMac: + browser: firefox + os: Mac + extra_config: *firefox_config + +Safari: + browser: safari + os: Mac + +SafariTP: + # TODO(b/152646297): Safari TP not launching as of Safari 15.4 + disabled: true + browser: safari + os: Mac + extra_config: + safari.options: + technologyPreview: true + +ChromeWindows: + browser: chrome + os: Windows + extra_config: *chrome_config + +FirefoxWindows: + browser: firefox + os: Windows + extra_config: *firefox_config + +IE11: + # IE11 support has been removed from the latest release branch. + disabled: true + browser: internet explorer + os: Windows + extra_config: + se:ieOptions: + # The zoom setting must be checked, or screenshot tests will be way off! + ignoreZoomSetting: false + ignoreProtectedModeSettings: true + +Edge: + browser: msedge + os: Windows + +ChromeLinux: + browser: chrome + os: Linux + extra_config: *chrome_config + +FirefoxLinux: + browser: firefox + os: Linux + extra_config: *firefox_config + +ChromeAndroid: + browser: chrome + os: Android + extra_config: *chrome_android_config + +Chromecast: + browser: chromecast + +Chromebook: + # TODO(b/145916766): Persistent license tests failing + disabled: true + browser: chromeos + version: Pixelbook + extra_config: *chromeos_config + +Tizen: + # TODO(joeyparrish): Get Tizen TV mounted in the lab and connected + disabled: true + browser: tizen + +XboxOne: + browser: xboxone diff --git a/build/test.py b/build/test.py index 62f272fca6..b9bb095a94 100755 --- a/build/test.py +++ b/build/test.py @@ -19,13 +19,22 @@ import argparse import json import logging +import os import platform +import re import build import gendeps import shakaBuildHelpers +# Set a higher default for capture_timeout in grid mode. If the test gets +# queued by the grid, this may prevent Karma from killing the session while +# waiting. +LOCAL_CAPTURE_TIMEOUT = 1 * 60 * 1000 # 1m in ms +SELENIUM_CAPTURE_TIMEOUT = 10 * 60 * 1000 # 10m in ms + + class _HandleMixedListsAction(argparse.Action): '''Action to handle comma-separated and space-separated lists. @@ -100,6 +109,8 @@ def _GetDefaultBrowsers(): raise Error('Unrecognized system: %s' % platform.uname()[0]) +# TODO(joeyparrish): When internal tools using this Launcher system are removed, +# simplify this whole mess. class Launcher: """A stateful object for parsing arguments and running Karma commands. @@ -126,7 +137,7 @@ def __init__(self, description): running_commands = self.parser.add_argument_group( 'Running', - 'These commands affect how tests are ran.') + 'These commands affect how tests are run.') logging_commands = self.parser.add_argument_group( 'Logging', 'These commands affect what gets logged and how the logs will appear.') @@ -230,9 +241,9 @@ def __init__(self, description): running_commands.add_argument( '--capture-timeout', help='Kill the browser if it does not capture in the given time [ms]. ' - '(default %(default)s)', - type=int, - default=60000) + '(default {} for local, {} for Selenium)'.format( + LOCAL_CAPTURE_TIMEOUT, SELENIUM_CAPTURE_TIMEOUT), + type=int) running_commands.add_argument( '--delay-tests', help='Insert an artificial delay between tests, in seconds. ' @@ -273,6 +284,13 @@ def __init__(self, description): help="Don't use Babel to convert ES6 to ES5.", dest='babel', action='store_false') + running_commands.add_argument( + '--grid-address', + help='Address (hostname:port) of a Selenium grid to run tests on.') + running_commands.add_argument( + '--grid-config', + help='Path to a yaml config defining Selenium grid browsers. ' + '(See docs/selenium-grid-config.md)') logging_commands.add_argument( @@ -328,6 +346,17 @@ def __init__(self, description): help='Specify the hostname to be used when capturing browsers. This ' 'defaults to localhost.', default='localhost') + networking_commands.add_argument( + '--tls-key', + help='Specify a TLS key to serve tests over HTTPs.') + networking_commands.add_argument( + '--tls-cert', + help='Specify a TLS cert to serve tests over HTTPs.') + networking_commands.add_argument( + '--lets-encrypt-folder', + help="Specify a Let's Encrypt folder to search for the latest key and " + "cert, to serve tests over HTTPs. This overrides --tls-key and " + "--tls-cert.") pre_launch_commands.add_argument( @@ -360,28 +389,36 @@ def ParseArguments(self, args): pass_through = [ 'auto_watch', 'babel', + 'browsers', 'capture_timeout', 'colors', + 'delay_tests', 'drm', + 'exclude_browsers', 'external', 'filter', + 'grid_address', + 'grid_config', 'hostname', 'html_coverage_report', 'log_level', 'logging', + 'no_browsers', 'port', 'quarantined', 'quick', 'random', + 'reporters', 'report_slower_than', 'seed', 'single_run', - 'uncompiled', - 'delay_tests', 'spec_hide_passed', 'test_custom_asset', 'test_custom_license_server', 'test_timeout', + 'tls_key', + 'tls_cert', + 'uncompiled', ] # Check each value before setting it to avoid passing null values. @@ -390,8 +427,33 @@ def ParseArguments(self, args): if value is not None: self.karma_config[name] = value - if self.parsed_args.reporters: - self.karma_config['reporters'] = self.parsed_args.reporters + if not self.parsed_args.capture_timeout: + # The default for capture_timeout depends on whether or not we are using + # a Selenium grid. + if self.parsed_args.grid_config: + self.karma_config['capture_timeout'] = SELENIUM_CAPTURE_TIMEOUT + else: + self.karma_config['capture_timeout'] = LOCAL_CAPTURE_TIMEOUT + + self._HandleLetsEncryptConfig() + + def _HandleLetsEncryptConfig(self): + folder = self.parsed_args.lets_encrypt_folder + if not folder: + return + + max_serial_number = 0 + # Go through the contents of the folder to find the latest key & cert. + for file_name in os.listdir(folder): + matches = re.match(r'(?:privkey|fullchain)(\d+).pem', file_name) + if matches: + serial_number = int(matches.group(1)) + max_serial_number = max(max_serial_number, serial_number) + + self.karma_config['tls_key'] = os.path.join( + folder, 'privkey{}.pem'.format(max_serial_number)) + self.karma_config['tls_cert'] = os.path.join( + folder, 'fullchain{}.pem'.format(max_serial_number)) def ResolveBrowsers(self, default_browsers): """Decide what browsers we should use. @@ -400,25 +462,7 @@ def ResolveBrowsers(self, default_browsers): additional logic to derive a browser list from the parsed arguments. """ assert(default_browsers and len(default_browsers)) - - if self.parsed_args.no_browsers: - logging.warning('In this mode browsers must manually connect to karma.') - elif self.parsed_args.browsers: - self.karma_config['browsers'] = self.parsed_args.browsers - else: - logging.warning('Using default browsers: %s', default_browsers) - self.karma_config['browsers'] = default_browsers - - # Check if there are any browsers that we should remove - if self.parsed_args.exclude_browsers and 'browsers' in self.karma_config: - all_browsers = set(self.karma_config['browsers']) - bad_browsers = set(self.parsed_args.exclude_browsers) - if bad_browsers - all_browsers: - raise RuntimeError('Attempting to exclude unselected browsers: %s' % - ','.join(bad_browsers - all_browsers)) - - good_browsers = all_browsers - bad_browsers - self.karma_config['browsers'] = list(good_browsers) + self.karma_config['default_browsers'] = default_browsers def RunCommand(self, karma_conf): """Build a command and send it to Karma for execution. diff --git a/docs/tutorials/index.json b/docs/tutorials/index.json index de50bf8976..661b0c118e 100644 --- a/docs/tutorials/index.json +++ b/docs/tutorials/index.json @@ -19,6 +19,7 @@ { "fairplay": { "title": "FairPlay support" } }, { "application-level-redirects": { "title": "Application-Level Redirects" } }, { "blob-url": { "title": "Blob URL" } }, + { "selenium-grid-config": { "title": "Selenium Grid Config" } }, { "faq": { "title": "Frequently Asked Questions" } }, { "upgrade": { "title": "Upgrade Guide", diff --git a/docs/tutorials/selenium-grid-config.md b/docs/tutorials/selenium-grid-config.md new file mode 100644 index 0000000000..6255b3c2d0 --- /dev/null +++ b/docs/tutorials/selenium-grid-config.md @@ -0,0 +1,122 @@ +# Selenium Grid Config + +Shaka Player's test runner (Karma) can be directed to run tests on a Selenium +grid. For this, you need a config file that defines what browsers are +available, and how to request them via WebDriver. + +For a fully-worked, detailed example, see the config for the private grid in +our lab at [build/shaka-lab.yaml](https://github.com/shaka-project/shaka-player/blob/main/build/shaka-lab.yaml) + +## Usage + +```sh +python build/test.py \ + --grid-config grid-config.yaml \ + --grid-address selenium-hub-hostname:4444 +``` + +## Syntax + +The config file is written in YAML. We chose YAML because it has two big +advantages over JSON: + +1. You can add comments +2. You can define variables (with YAML "anchors") to factor out common configs + +### Variables + +You can define any common variables in the `vars` section, then refer to them +elsewhere. For example: + +```yaml +vars: + # Generates an "anchor" with the given name. Later, you can inject the + # contents of the variable with "*name". + firefox_config: &firefox_config + moz:firefoxOptions: + # Override Firefox default preferences in the temporary profile created + # for each test run. + prefs: + # Overrides Selenium's explicit default setting, to allow Firefox to + # install the Widevine CDM on demand. + media.gmp-manager.updateEnabled: true + # Overrides Firefox's Linux-specific default setting to disable DRM. + media.eme.enabled: true + +# These three browser definitions share the same config from above. +FirefoxMac: + browser: firefox + os: Mac + extra_config: *firefox_config + +FirefoxWindows: + browser: firefox + os: Windows + extra_config: *firefox_config + +FirefoxLinux: + browser: firefox + os: Linux + extra_config: *firefox_config +``` + +### Browsers + +Each top-level key in the config file (except for `vars`) is the name of a +browser made available to Karma. Within each of those keys are the following: + + - `browser`: The name of the browser. This is case-sensitive, and must match + the string and case used by Selenium. + - `os` (optional): The name of the OS. This is case-INsensitive, and must + match the name of the platform as used by Selenium. For Generic + WebDriver Server, this can be omitted or set to the name of the host + platform. + - `version` (optional): The version of the browser. This is case-sensitive, + and must match the string and case used in the Selenium node config. + - `disabled` (optional): If true, this browser is disabled and will not be + used unless explicitly requested. + - `extra_config` (optional): A dictionary of extra configs which will be + merged with the WebDriver launcher config in Karma. + +Examples of basic desktop browsers definitions: + +```yaml +ChromeMac: + browser: chrome + os: Mac + +FirefoxMac: + browser: firefox + os: Mac + +Safari: + browser: safari + os: Mac + +SafariTP: + browser: safari + os: Mac + extra_config: + safari.options: + technologyPreview: true + +ChromeWindows: + browser: chrome + os: Windows + +FirefoxWindows: + browser: firefox + os: Windows + +Edge: + browser: msedge + os: Windows + +ChromeLinux: + browser: chrome + os: Linux + +FirefoxLinux: + browser: firefox + os: Linux +``` diff --git a/karma.conf.js b/karma.conf.js index 80982da340..7d7f315a0c 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -15,6 +15,7 @@ const rimraf = require('rimraf'); const {ssim} = require('ssim.js'); const util = require('karma/common/util'); const which = require('which'); +const yaml = require('js-yaml'); /** * @param {Object} config @@ -44,6 +45,58 @@ module.exports = (config) => { const settings = settingsIndex >= 0 ? JSON.parse(args[settingsIndex + 1]) : {}; + if (settings.grid_config) { + const gridBrowserMetadata = + yaml.load(fs.readFileSync(settings.grid_config, 'utf8')); + const customLaunchers = {}; + const [gridHostname, gridPort] = settings.grid_address.split(':'); + console.log(`Using Selenium grid at ${gridHostname}:${gridPort}`); + + // By default, run on all grid browsers instead of the platform-specific + // default. This does not disable local browsers, though. Users can still + // specify a mix of grid and local browsers explicitly. + settings.default_browsers = []; + + for (const name in gridBrowserMetadata) { + if (name == 'vars') { + // Skip variable defs in the YAML file + continue; + } + + const metadata = gridBrowserMetadata[name]; + + const launcher = {}; + customLaunchers[name] = launcher; + + // Disabled-by-default browsers are still defined, but not put in the + // default list. A user can ask for one explicitly. This allows us to + // disable a browser that is down for some reason in the lab, but still + // ask for it manually if we want to test it before re-enabling it for + // everyone. + if (!metadata.disabled) { + settings.default_browsers.push(name); + } + + // Add standard WebDriver configs. + Object.assign(launcher, { + base: 'WebDriver', + config: {hostname: gridHostname, port: gridPort}, + pseudoActivityInterval: 20000, + browserName: metadata.browser, + platform: metadata.os, + version: metadata.version, + }); + + if (metadata.extra_config) { + Object.assign(launcher, metadata.extra_config); + } + } + + config.set({ + customLaunchers: customLaunchers, + }); + } + if (settings.browsers && settings.browsers.length == 1 && settings.browsers[0] == 'help') { console.log('Available browsers:'); @@ -54,6 +107,24 @@ module.exports = (config) => { process.exit(1); } + // Resolve the set of browsers we will use. + const browserSet = new Set(settings.browsers && settings.browsers.length ? + settings.browsers : settings.default_browsers); + if (settings.exclude_browsers) { + for (const excluded of settings.exclude_browsers) { + browserSet.delete(excluded); + } + } + + let browsers = Array.from(browserSet).sort(); + if (settings.no_browsers) { + console.warn( + '--no-browsers: In this mode, you must connect browsers to Karma.'); + browsers = null; + } else { + console.warn('Running tests on: ' + browsers.join(', ')); + } + config.set({ // base path that will be used to resolve all patterns (eg. files, exclude) basePath: '.', @@ -213,7 +284,7 @@ module.exports = (config) => { // Set which browsers to run on. If this is null, then Karma will wait for // an incoming connection. - browsers: settings.browsers, + browsers, // Enable / disable colors in the output (reporters and logs). Defaults // to true. @@ -244,6 +315,7 @@ module.exports = (config) => { specReporter: { suppressSkipped: true, + showBrowser: true, }, }); @@ -357,6 +429,16 @@ module.exports = (config) => { console.log('Using a random test order (--random) with --seed=' + seed); } + + if (settings.tls_key && settings.tls_cert) { + config.set({ + protocol: 'https', + httpsServerOptions: { + key: fs.readFileSync(settings.tls_key), + cert: fs.readFileSync(settings.tls_cert), + }, + }); + } }; /** @@ -446,7 +528,7 @@ function allUsableBrowserLaunchers(config) { browsers.push(...Object.keys(config.customLaunchers)); } - return browsers; + return browsers.sort(); } /**