test: Run tests on a custom Selenium grid (#4117)

Includes a YAML config file for our lab, and documentation on how to
create a custom config for another Selenium grid.

A workflow will run tests nightly in the Shaka lab, using a
self-hosted runner with access to our private grid.

The workflow can also be triggered manually by maintainers to test a
PR in the lab.  This will report status back to the PR.

.github/workflows/custom-actions/set-commit-status/action.yaml

@@ -0,0 +1,41 @@
+name: Set Commit Status
+
+description: |
+  A reusable action that sets the commit status.  This is used to set PR status
+  from workflows with non-PR triggers (such as manually-triggered workflows).
+
+inputs:
+  context:
+    description: An arbitrary string to identify the status check.
+    required: true
+  state:
+    description: Either "pending", "error", "success", or "failure".
+    required: true
+  ref:
+    description: A git ref for which to set the commit status.  For PRs, use the head commit, not the merge commit.  Defaults to HEAD.
+    required: false
+    default: HEAD
+  token:
+    description: A GitHub access token.
+    required: true
+
+runs:
+  using: composite
+  steps:
+    - name: Report Commit Status
+      shell: bash
+      run: |
+        # This is the URL to view this workflow run on GitHub.  It will be
+        # attached to the commit status, so that when someone clicks "details"
+        # next to the status on the PR, it will link to this run where they can
+        # see the logs.
+        RUN_URL="https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}"
+        SHA1=$(git rev-parse "${{ inputs.ref }}")
+
+        GITHUB_TOKEN=${{ inputs.token }} \
+        gh api \
+          -X POST \
+          -F "context=${{ inputs.context }}" \
+          -F "state=${{ inputs.state }}" \
+          -F "target_url=$RUN_URL" \
+          "repos/${{ github.repository }}/statuses/$SHA1"

.github/workflows/selenium-lab-tests.yaml

@@ -0,0 +1,93 @@
+name: Selenium Lab Tests
+
+on:
+  workflow_dispatch:
+    # Allows for manual triggering on PRs.  They should be reviewed first, to
+    # avoid malicious code executing in the lab.
+    inputs:
+      pr:
+        description: "A PR number to build and test in the lab.  If empty, will build and test from main."
+        required: false
+  schedule:
+    # Runs every night at 2am PST / 10am UTC, testing against the main branch.
+    - cron: '0 10 * * *'
+
+# Only one run of this workflow is allowed at a time, since it uses physical
+# resources in our lab.
+concurrency: selenium-lab
+
+jobs:
+  lab-tests:
+    # This is a self-hosted runner in a Docker container, with access to our
+    # lab's Selenium grid on port 4444.
+    runs-on: self-hosted-selenium
+
+    steps:
+      # This runs on our self-hosted runner, and the Docker image it is based
+      # on doesn't have GitHub's CLI pre-installed.  This installs it.  Taken
+      # verbatim from the official installation instructions at
+      # https://github.com/cli/cli/blob/trunk/docs/install_linux.md
+      - name: Install GitHub Actions CLI
+        run: |
+          curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
+          sudo apt update
+          sudo apt install gh
+
+      - name: Compute ref
+        run: |
+          if [[ "${{ github.event.inputs.pr }}" != "" ]]; then
+            echo LAB_TEST_REF="refs/pull/${{ github.event.inputs.pr }}/head" >> $GITHUB_ENV
+          else
+            echo LAB_TEST_REF="main" >> $GITHUB_ENV
+          fi
+
+      - uses: actions/checkout@v2
+        with:
+          ref: ${{ env.LAB_TEST_REF }}
+
+      - name: Set Commit Status to Pending
+        uses: ./.github/workflows/custom-actions/set-commit-status
+        with:
+          context: Selenium Lab Tests
+          state: pending
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 16
+          registry-url: 'https://registry.npmjs.org'
+
+      # The Docker image for this self-hosted runner doesn't contain java.
+      - uses: actions/setup-java@v2
+        with:
+          distribution: zulu
+          java-version: 11
+
+      # The Docker image for this self-hosted runner has "python3" but not
+      # plain "python".
+      - name: Build Player
+        run: python3 build/all.py
+
+      # Run tests on the Selenium grid in our lab.  This uses a private
+      # hostname and TLS cert to get EME tests working on all platforms
+      # (since EME only works on https or localhost).
+      - name: Test Player
+        run: |
+          python3 build/test.py \
+              --reporters spec --spec-hide-passed \
+              --drm \
+              --lets-encrypt-folder /etc/shakalab.rocks \
+              --hostname karma.shakalab.rocks \
+              --port 61731 \
+              --grid-config build/shaka-lab.yaml \
+              --grid-address selenium-grid.lab:4444
+
+      - name: Report Final Commit Status
+        # Will run on success or failure, but not if the workflow is cancelled.
+        if: ${{ success() || failure() }}
+        uses: ./.github/workflows/custom-actions/set-commit-status
+        with:
+          context: Selenium Lab Tests
+          state: ${{ job.status }}
+          token: ${{ secrets.GITHUB_TOKEN }}

build/shaka-lab.yaml

@@ -0,0 +1,163 @@
+# Selenium grid config for the Shaka lab.  This is the source of truth for the
+# browsers and devices in the Shaka lab at Google.
+
+# For syntax and general information, see docs/selenium-grid-config.md
+
+
+# A set of variables to contain repeated configurations which can then be
+# referenced below.  The syntax for the variable is "name: &name", which
+# generates an "anchor" with the given name.  Later, you can inject the
+# contents of the variable with "*name".
+vars:
+  firefox_config: &firefox_config
+    moz:firefoxOptions:
+      # Override Firefox default preferences in the temporary profile created
+      # for each test run.
+      prefs:
+        # Overrides Selenium's explicit default setting, to allow Firefox to
+        # install the Widevine CDM on demand.
+        media.gmp-manager.updateEnabled: true
+        # Overrides Firefox's Linux-specific default setting to disable DRM.
+        media.eme.enabled: true
+
+  minimum_chrome_args: &minimum_chrome_args
+    # On Chrome m59+ we can test EME on platforms with pop-up prompts, such as
+    # Android and ChromeOS.  Note that this flag does not take a port number
+    # (domain vs origin).
+    # This flag requires setting --user-data-dir as well, however, webdriver
+    # already takes care of that for us (except on Android).
+    - "--unsafely-allow-protected-media-identifier-for-domain=karma.shakalab.rocks"
+    # Normally, Chrome disallows autoplaying videos in many cases.  Enable it
+    # for testing.
+    - "--autoplay-policy=no-user-gesture-required"
+
+  minimum_chrome_android_args: &minimum_chrome_android_args
+    # There is no way in YAML to natively merge arrays, so we start by
+    # duplicating the flags from minimum_chrome_args above.
+    - "--unsafely-allow-protected-media-identifier-for-domain=karma.shakalab.rocks"
+    - "--autoplay-policy=no-user-gesture-required"
+    # On Android we must set --user-data-dir.  WebDriver does not do it for
+    # us as it does on other platforms.  Without --user-data-dir,
+    # --unsafely-allow... does not work.
+    - "--user-data-dir=/data/data/com.android.chrome/cache"
+
+  minimum_chromeos_args: &minimum_chromeos_args
+    # There is no way in YAML to natively merge arrays, so we start by
+    # duplicating the flags from minimum_chrome_args above.
+    - "--unsafely-allow-protected-media-identifier-for-domain=karma.shakalab.rocks"
+    - "--autoplay-policy=no-user-gesture-required"
+    # Allow remote attestation even though the device may be in dev mode.  This
+    # is critical for testing involving L1 content licenses.
+    - "--allow-ra-in-dev-mode"
+
+  chrome_config: &chrome_config
+    goog:chromeOptions:
+      args: *minimum_chrome_args
+
+      # Instruct chromedriver not to disable component updater. The
+      # component updater must run in order for the Widevine CDM to be
+      # available when using a new user-data-dir.
+      # TODO(http://crbug.com/613581): Remove once Chrome bug is fixed.
+      excludeSwitches:
+        - "disable-component-update"
+
+  chrome_android_config: &chrome_android_config
+    goog:chromeOptions:
+      args: *minimum_chrome_android_args
+
+      # Once the new session request reaches chromedriver, it will take
+      # the androidPackage option as a request to start Chrome through
+      # adb on the tethered device.
+      androidPackage: com.android.chrome
+
+  chromeos_config: &chromeos_config
+    # Pass these client-specified arguments through generic-webdriver-server.
+    # This array will be appended after "--" instead of being set through one
+    # specific flag.  For example, with ["--foo", "--bar=baz"] in the args,
+    # this would generate a command like:
+    #   chromeos-webdriver-server.js -- --foo --bar=baz
+    # Those parameters will be passed on to Chrome instead of used by the
+    # WebDriver server.
+    generic:args: *minimum_chromeos_args
+
+
+ChromeMac:
+  browser: chrome
+  os: Mac
+  extra_config: *chrome_config
+
+FirefoxMac:
+  browser: firefox
+  os: Mac
+  extra_config: *firefox_config
+
+Safari:
+  browser: safari
+  os: Mac
+
+SafariTP:
+  # TODO(b/152646297): Safari TP not launching as of Safari 15.4
+  disabled: true
+  browser: safari
+  os: Mac
+  extra_config:
+    safari.options:
+      technologyPreview: true
+
+ChromeWindows:
+  browser: chrome
+  os: Windows
+  extra_config: *chrome_config
+
+FirefoxWindows:
+  browser: firefox
+  os: Windows
+  extra_config: *firefox_config
+
+IE11:
+  # IE11 support has been removed from the latest release branch.
+  disabled: true
+  browser: internet explorer
+  os: Windows
+  extra_config:
+    se:ieOptions:
+      # The zoom setting must be checked, or screenshot tests will be way off!
+      ignoreZoomSetting: false
+      ignoreProtectedModeSettings: true
+
+Edge:
+  browser: msedge
+  os: Windows
+
+ChromeLinux:
+  browser: chrome
+  os: Linux
+  extra_config: *chrome_config
+
+FirefoxLinux:
+  browser: firefox
+  os: Linux
+  extra_config: *firefox_config
+
+ChromeAndroid:
+  browser: chrome
+  os: Android
+  extra_config: *chrome_android_config
+
+Chromecast:
+  browser: chromecast
+
+Chromebook:
+  # TODO(b/145916766): Persistent license tests failing
+  disabled: true
+  browser: chromeos
+  version: Pixelbook
+  extra_config: *chromeos_config
+
+Tizen:
+  # TODO(joeyparrish): Get Tizen TV mounted in the lab and connected
+  disabled: true
+  browser: tizen
+
+XboxOne:
+  browser: xboxone