From 3d4fc7852bd7eea40294bdcca5f79a0306089823 Mon Sep 17 00:00:00 2001 From: Powei Feng Date: Thu, 26 Sep 2024 16:06:40 -0700 Subject: [PATCH] github: add software rasterizer job for GL to presubmit (#8158) We use Mesa's gallium swrast to render as the driver with Filament's backend set to GL. We provide a few scripts to parse the tests (as jsons) and run gltf_viewer to produce the rendering. --- .github/actions/ubuntu-apt-add-src/action.yml | 9 ++ .github/workflows/presubmit.yml | 15 ++ test/renderdiff/README.md | 17 +++ test/renderdiff/parse_test_json.py | 142 ++++++++++++++++++ test/renderdiff/run.py | 54 +++++++ test/renderdiff/tests/presubmit.json | 34 +++++ test/renderdiff/tests/sample.json | 34 +++++ test/renderdiff/utils.py | 68 +++++++++ test/renderdiff_tests.sh | 45 ++++++ test/utils/get_mesa.sh | 38 +++++ 10 files changed, 456 insertions(+) create mode 100644 .github/actions/ubuntu-apt-add-src/action.yml create mode 100644 test/renderdiff/README.md create mode 100644 test/renderdiff/parse_test_json.py create mode 100644 test/renderdiff/run.py create mode 100644 test/renderdiff/tests/presubmit.json create mode 100644 test/renderdiff/tests/sample.json create mode 100644 test/renderdiff/utils.py create mode 100755 test/renderdiff_tests.sh create mode 100755 test/utils/get_mesa.sh diff --git a/.github/actions/ubuntu-apt-add-src/action.yml b/.github/actions/ubuntu-apt-add-src/action.yml new file mode 100644 index 00000000000..c104574a632 --- /dev/null +++ b/.github/actions/ubuntu-apt-add-src/action.yml @@ -0,0 +1,9 @@ +name: 'ubuntu apt add deb-src' +runs: + using: "composite" + steps: + - name: "ubuntu apt add deb-src" + run: | + echo "deb-src http://archive.ubuntu.com/ubuntu jammy main restricted universe" | sudo tee /etc/apt/sources.list.d/my.list + sudo apt-get update + shell: bash diff --git a/.github/workflows/presubmit.yml b/.github/workflows/presubmit.yml index 8e0d9ccc08f..151dd375657 100644 --- a/.github/workflows/presubmit.yml +++ b/.github/workflows/presubmit.yml @@ -76,3 +76,18 @@ jobs: - name: Run build script run: | cd build/web && printf "y" | ./build.sh presubmit + + test-renderdiff: + name: test-renderdiff + runs-on: ubuntu-22.04-32core + + steps: + - uses: actions/checkout@v4.1.6 + - uses: ./.github/actions/ubuntu-apt-add-src + - name: Run script + run: | + source ./build/linux/ci-common.sh && bash test/renderdiff_tests.sh + - uses: actions/upload-artifact@v4 + with: + name: presubmit-renderdiff-result + path: ./out/renderdiff_tests diff --git a/test/renderdiff/README.md b/test/renderdiff/README.md new file mode 100644 index 00000000000..f90f2970636 --- /dev/null +++ b/test/renderdiff/README.md @@ -0,0 +1,17 @@ +# Rendering Difference Test + +We created a few scripts to run `gltf_viewer` and produce headless renderings. + +This is mainly useful for continuous integration where GPUs are generally not available on cloud +machines. To perform software rasterization, these scripts are centered around [Mesa]'s software +rasterizers, but nothing bars us from using another rasterizer like [SwiftShader]. Additionally, +we should be able to use GPUs where available (though this is more of a future work). + +The script `run.py` contains the core logic for taking input parameters (such as the test +description file) and then running gltf_viewer to produce the results. + +In the `test` directory is a list of test descriptions that are specified in json. Please see +`sample.json` to parse the structure. + +[Mesa]: https://docs.mesa3d.org +[SwiftShader]: https://github.com/google/swiftshader \ No newline at end of file diff --git a/test/renderdiff/parse_test_json.py b/test/renderdiff/parse_test_json.py new file mode 100644 index 00000000000..2cc260ffec3 --- /dev/null +++ b/test/renderdiff/parse_test_json.py @@ -0,0 +1,142 @@ +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from utils import execute, ArgParseImpl + +import glob +from itertools import chain +import json +import sys +import os +from os import path + +def _is_list_of_strings(field): + return isinstance(field, list) and\ + all(isinstance(item, str) for item in field) + +def _is_string(s): + return isinstance(s, str) + +def _is_dict(s): + return isinstance(s, dict) + +class RenderingConfig(): + def __init__(self, data): + assert 'name' in data + assert _is_string(data['name']) + self.name = data['name'] + + assert 'rendering' in data + assert _is_dict(data['rendering']) + self.rendering = data['rendering'] + +class PresetConfig(RenderingConfig): + def __init__(self, data, existing_models): + RenderingConfig.__init__(self, data) + models = data.get('models') + if models: + assert _is_list_of_strings(models) + assert all(m in existing_models for m in models) + self.models = models + +class TestConfig(RenderingConfig): + def __init__(self, data, existing_models, presets): + RenderingConfig.__init__(self, data) + description = data.get('description') + if description: + assert _is_string(description) + self.description = description + + apply_presets = data.get('apply_presets') + rendering = {} + preset_models = [] + if apply_presets: + given_presets = {p.name: p for p in presets} + assert all((name in given_presets) for name in apply_presets) + for preset in apply_presets: + rendering.update(given_presets[preset].rendering) + preset_models += given_presets[preset].models + + assert 'rendering' in data + rendering.update(data['rendering']) + self.rendering = rendering + + models = data.get('models') + self.models = preset_models + if models: + assert _is_list_of_strings(models) + assert all(m in existing_models for m in models) + self.models = set(models + self.models) + + def to_filament_format(self): + json_out = { + 'name': self.name, + 'base': self.rendering + } + return json.dumps(json_out) + +class RenderTestConfig(): + def __init__(self, data): + assert 'name' in data + name = data['name'] + assert _is_string(name) + self.name = name + + assert 'backends' in data + backends = data['backends'] + assert _is_list_of_strings(backends) + self.backends = backends + + assert 'model_search_paths' in data + model_search_paths = data.get('model_search_paths') + assert _is_list_of_strings(model_search_paths) + assert all(path.isdir(p) for p in model_search_paths) + + model_paths = list( + chain(*(glob.glob(f'{d}/**/*.glb', recursive=True) for d in model_search_paths))) + # This flatten the output for glob.glob + self.models = {path.splitext(path.basename(model))[0]: model for model in model_paths} + + preset_data = data.get('presets') + presets = [] + if preset_data: + presets = [PresetConfig(p, self.models) for p in preset_data] + + assert 'tests' in data + self.tests = [TestConfig(t, self.models, presets) for t in data['tests']] + test_names = list([t.name for t in self.tests]) + + # We cannot have duplicate test names + assert len(test_names) == len(set(test_names)) + +def _remove_comments_from_json_txt(json_txt): + res = [] + for line in json_txt.split('\n'): + if '//' in line: + line = line.split('//')[0] + res.append(line) + return '\n'.join(res) + +def parse_test_config_from_path(config_path): + with open(config_path, 'r') as f: + json_txt = json.loads(_remove_comments_from_json_txt(f.read())) + return RenderTestConfig(json_txt) + + +if __name__ == "__main__": + parser = ArgParseImpl() + parser.add_argument('--test', help='Configuration of the test', required=True) + + args, _ = parser.parse_known_args(sys.argv[1:]) + test = parse_test_config_from_path(args.test) diff --git a/test/renderdiff/run.py b/test/renderdiff/run.py new file mode 100644 index 00000000000..17a8a7acb47 --- /dev/null +++ b/test/renderdiff/run.py @@ -0,0 +1,54 @@ +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from utils import execute, ArgParseImpl + +from parse_test_json import parse_test_config_from_path +import sys +import os + +def run_test(gltf_viewer, pixel_test, output_dir, opengl_lib=None, vk_icd=None): + assert os.path.isdir(output_dir) + assert os.access(gltf_viewer, os.X_OK) + + for test in pixel_test.tests: + test_json_path = f'{output_dir}/{test.name}_simplified.json' + + with open(test_json_path, 'w') as f: + f.write(f'[{test.to_filament_format()}]') + + for backend in pixel_test.backends: + env = None + if backend == 'opengl' and opengl_lib and os.path.isdir(opengl_lib): + env = {'LD_LIBRARY_PATH': opengl_lib} + + for model in test.models: + model_path = pixel_test.models[model] + out_name = f'{test.name}_{model}_{backend}' + execute(f'{gltf_viewer} -a {backend} --batch={test_json_path} -e {model_path} --headless', + env=env, capture_output=False) + execute(f'mv -f {test.name}0.ppm {output_dir}/{out_name}.ppm', capture_output=False) + execute(f'mv -f {test.name}0.json {output_dir}/{test.name}.json', capture_output=False) + +if __name__ == "__main__": + parser = ArgParseImpl() + parser.add_argument('--test', help='Configuration of the test', required=True) + parser.add_argument('--gltf_viewer', help='Path to the gltf_viewer', required=True) + parser.add_argument('--output_dir', help='Output Directory', required=True) + parser.add_argument('--opengl_lib', help='Path to the folder containing OpenGL driver lib (for LD_LIBRARY_PATH)') + parser.add_argument('--vk_icd', help='Path to VK ICD file') + + args, _ = parser.parse_known_args(sys.argv[1:]) + test = parse_test_config_from_path(args.test) + run_test(args.gltf_viewer, test, args.output_dir, opengl_lib=args.opengl_lib, vk_icd=args.vk_icd) diff --git a/test/renderdiff/tests/presubmit.json b/test/renderdiff/tests/presubmit.json new file mode 100644 index 00000000000..78224d419ce --- /dev/null +++ b/test/renderdiff/tests/presubmit.json @@ -0,0 +1,34 @@ +{ + "name": "PresubmitPixelTests", + "backends": ["opengl"], + "model_search_paths": ["third_party/models"], + "presets": [ + { + "name": "Standard", + "models": ["lucy", "DamagedHelmet"], + "rendering": { + "viewer.cameraFocusDistance": 0, + "view.postProcessingEnabled": true + } + } + ], + "tests": [ + { + "name": "BloomFlare", + "description": "Testing bloom and flare", + "apply_presets": ["Standard"], + "rendering": { + "view.bloom.enabled": true, + "view.bloom.lensFlare": true + } + }, + { + "name": "MSAA", + "description": "Testing Multi-sample Anti-aliasing", + "apply_presets": ["Standard"], + "rendering": { + "view.msaa.enabled": true + } + } + ] +} diff --git a/test/renderdiff/tests/sample.json b/test/renderdiff/tests/sample.json new file mode 100644 index 00000000000..eb689d8102d --- /dev/null +++ b/test/renderdiff/tests/sample.json @@ -0,0 +1,34 @@ +{ + "name": "SampleTest" , // [required] + "backends": ["opengl"], // [required] Specifies the backend that will be used to render + // this test + "model_search_paths": ["third_party/models"], // [optional] Base iirectory from which we will glob for + // .glb files and try to match against names in the 'models' + // field. + "presets": [ // [optional] A list of preset configurations that tests can + // inherit from. + { + "name": "Standard", // [required] + "models": ["lucy", "DamagedHelmet"], // [optional] Base name for the gltf file used in the test. For + // example, source files are lucy.glb and DamagedHelmet.gltf + "rendering": { // [required] Rendering settings used in the test. The json format + "viewer.cameraFocusDistance": 0, // is taken from AutomationSpec in libs/viewer + "view.postProcessingEnabled": true + } + } + ], + "tests": [ // [required] List of test configurations + { + "name": "BloomFlare", // [required] + "description": "Testing bloom and flare", // [optional] + "apply_presets": ["Standard"], // [optional] Applies the preset in order. Item must be in + // 'presets' field in the top-level struct. + "model": [], // [optional] List of models used in this test. The list is + // *appended* onto the lists provided by the presets. + "rendering": { // [required] See description in 'presets' + "view.bloom.enabled": true, + "view.bloom.lensFlare": true + } + } + ] +} diff --git a/test/renderdiff/utils.py b/test/renderdiff/utils.py new file mode 100644 index 00000000000..f708d7a6596 --- /dev/null +++ b/test/renderdiff/utils.py @@ -0,0 +1,68 @@ +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess +import os +import argparse +import sys + +def execute(cmd, + cwd=None, + capture_output=True, + stdin=None, + env=None, + raise_errors=False): + in_env = os.environ + in_env.update(env if env else {}) + home = os.environ['HOME'] + if f'{home}/bin' not in in_env['PATH']: + in_env['PATH'] = in_env['PATH'] + f':{home}/bin' + + stdout = subprocess.PIPE if capture_output else sys.stdout + stderr = subprocess.PIPE if capture_output else sys.stdout + output = '' + err_output = '' + return_code = -1 + kwargs = { + 'cwd': cwd, + 'env': in_env, + 'stdout': stdout, + 'stderr': stderr, + 'stdin': stdin, + 'universal_newlines': True + } + if capture_output: + process = subprocess.Popen(cmd.split(' '), **kwargs) + output, err_output = process.communicate() + return_code = process.returncode + else: + return_code = subprocess.call(cmd.split(' '), **kwargs) + + if return_code: + # Error + if raise_errors: + raise subprocess.CalledProcessError(return_code, cmd) + if output: + if type(output) != str: + try: + output = output.decode('utf-8').strip() + except UnicodeDecodeError as e: + print('cannot decode ', output, file=sys.stderr) + return return_code, (output if return_code == 0 else err_output) + +class ArgParseImpl(argparse.ArgumentParser): + def error(self, message): + sys.stderr.write('error: %s\n' % message) + self.print_help() + sys.exit(1) diff --git a/test/renderdiff_tests.sh b/test/renderdiff_tests.sh new file mode 100755 index 00000000000..fc5b14fab04 --- /dev/null +++ b/test/renderdiff_tests.sh @@ -0,0 +1,45 @@ +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/bash + +OUTPUT_DIR="$(pwd)/out/renderdiff_tests" +RENDERDIFF_TEST_DIR="$(pwd)/test/renderdiff" +TEST_UTILS_DIR="$(pwd)/test/utils" +MESA_DIR="$(pwd)/mesa/out/" +MESA_LIB_DIR="${MESA_DIR}/lib/x86_64-linux-gnu" + +function prepare_mesa() { + if [ ! -d ${MESA_LIB_DIR} ]; then + rm -rf mesa + bash ${TEST_UTILS_DIR}/get_mesa.sh + fi +} + +# Following steps are taken: +# - Get and build mesa +# - Build gltf_viewer +# - Run the python script that runs the test +# - Zip up the result + +set -e && set -x && prepare_mesa && \ + mkdir -p ${OUTPUT_DIR} && \ + ./build.sh -X ${MESA_DIR} -p desktop debug gltf_viewer && \ + python3 ${RENDERDIFF_TEST_DIR}/run.py \ + --gltf_viewer="$(pwd)/out/cmake-debug/samples/gltf_viewer" \ + --test=${RENDERDIFF_TEST_DIR}/tests/presubmit.json \ + --output_dir=${OUTPUT_DIR} \ + --opengl_lib=${MESA_LIB_DIR} + +unset MESA_LIB_DIR diff --git a/test/utils/get_mesa.sh b/test/utils/get_mesa.sh new file mode 100755 index 00000000000..575182846d0 --- /dev/null +++ b/test/utils/get_mesa.sh @@ -0,0 +1,38 @@ +# Copyright (C) 2024 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/bash + +set -e +set -x + +sudo apt-get -y build-dep mesa + +git clone https://gitlab.freedesktop.org/mesa/mesa.git + +pushd . + +cd mesa + +git checkout mesa-23.2.1 + +mkdir -p out + +# -Dosmesa=true => builds OSMesa, which is an offscreen GL context +# -Dgallium-drivers=swrast => builds GL software rasterizer +# -Dvulkan-drivers=swrast => builds VK software rasterizer +meson setup builddir/ -Dprefix="$(pwd)/out" -Dosmesa=true -Dglx=xlib -Dgallium-drivers=swrast -Dvulkan-drivers=swrast +meson install -C builddir/ + +popd