Skip to content

Commit

Permalink
ci: add basic test pipeline for shader builders
Browse files Browse the repository at this point in the history
  • Loading branch information
yedpodtrzitko committed Aug 19, 2022
1 parent b04593c commit 388d35b
Show file tree
Hide file tree
Showing 26 changed files with 525 additions and 71 deletions.
6 changes: 5 additions & 1 deletion .github/workflows/static_checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ jobs:
sudo apt-get install -qq dos2unix recode clang-format-13 libxml2-utils
sudo update-alternatives --remove-all clang-format
sudo update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-13 100
sudo pip3 install black==22.3.0 pygments
sudo pip3 install black==22.3.0 pygments pytest
- name: File formatting checks (file_format.sh)
run: |
Expand All @@ -41,6 +41,10 @@ jobs:
run: |
bash ./misc/scripts/black_format.sh
- name: Python builders checks via pytest (pytest_builders.sh)
run: |
bash ./misc/scripts/pytest_builders.sh
- name: JavaScript style and documentation checks via ESLint and JSDoc
run: |
cd platform/javascript
Expand Down
8 changes: 3 additions & 5 deletions gles3_builders.py
Original file line number Diff line number Diff line change
Expand Up @@ -182,17 +182,15 @@ def include_file_in_gles3_header(filename, header_data, depth):
return header_data


def build_gles3_header(filename, include, class_suffix, output_attribs):
header_data = GLES3HeaderStruct()
def build_gles3_header(filename, include, class_suffix, header_data=None):
header_data = header_data or GLES3HeaderStruct()
include_file_in_gles3_header(filename, header_data, 0)

out_file = filename + ".gen.h"
fd = open(out_file, "w")
defspec = 0
defvariant = ""

enum_constants = []

fd.write("/* WARNING, THIS FILE WAS GENERATED, DO NOT EDIT */\n")

out_file_base = out_file
Expand Down Expand Up @@ -552,7 +550,7 @@ def build_gles3_header(filename, include, class_suffix, output_attribs):

def build_gles3_headers(target, source, env):
for x in source:
build_gles3_header(str(x), include="drivers/gles3/shader_gles3.h", class_suffix="GLES3", output_attribs=True)
build_gles3_header(str(x), include="drivers/gles3/shader_gles3.h", class_suffix="GLES3")


if __name__ == "__main__":
Expand Down
126 changes: 62 additions & 64 deletions glsl_builders.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,26 @@
All such functions are invoked in a subprocess on Windows to prevent build flakiness.
"""
import os.path
from platform_methods import subprocess_main


def generate_inline_code(input_lines, insert_newline=True):
"""Take header data and generate inline code
:param: list input_lines: values for shared inline code
:return: str - generated inline value
"""
output = []
for line in input_lines:
if line:
output.append(",".join(str(ord(c)) for c in line))
if insert_newline:
output.append("%s" % ord("\n"))
output.append("0")
return ",".join(output)


class RDHeaderStruct:
def __init__(self):
self.vertex_lines = []
Expand Down Expand Up @@ -57,10 +74,6 @@ def include_file_in_rd_header(filename, header_data, depth):
while line.find("#include ") != -1:
includeline = line.replace("#include ", "").strip()[1:-1]

import os.path

included_file = ""

if includeline.startswith("thirdparty/"):
included_file = os.path.relpath(includeline)

Expand All @@ -82,8 +95,7 @@ def include_file_in_rd_header(filename, header_data, depth):

line = fs.readline()

line = line.replace("\r", "")
line = line.replace("\n", "")
line = line.replace("\r", "").replace("\n", "")

if header_data.reading == "vertex":
header_data.vertex_lines += [line]
Expand All @@ -100,65 +112,53 @@ def include_file_in_rd_header(filename, header_data, depth):
return header_data


def build_rd_header(filename):
header_data = RDHeaderStruct()
def build_rd_header(filename, header_data=None):
header_data = header_data or RDHeaderStruct()
include_file_in_rd_header(filename, header_data, 0)

out_file = filename + ".gen.h"
fd = open(out_file, "w")

fd.write("/* WARNING, THIS FILE WAS GENERATED, DO NOT EDIT */\n")

out_file_base = out_file
out_file_base = out_file_base[out_file_base.rfind("/") + 1 :]
out_file_base = out_file_base[out_file_base.rfind("\\") + 1 :]
out_file_ifdef = out_file_base.replace(".", "_").upper()
fd.write("#ifndef " + out_file_ifdef + "_RD\n")
fd.write("#define " + out_file_ifdef + "_RD\n")

out_file_class = out_file_base.replace(".glsl.gen.h", "").title().replace("_", "").replace(".", "") + "ShaderRD"
fd.write("\n")
fd.write('#include "servers/rendering/renderer_rd/shader_rd.h"\n\n')
fd.write("class " + out_file_class + " : public ShaderRD {\n\n")
fd.write("public:\n\n")

fd.write("\t" + out_file_class + "() {\n\n")
if header_data.compute_lines:
body_parts = [
"static const char _compute_code[] = {\n%s\n\t\t};" % generate_inline_code(header_data.compute_lines),
f'setup(nullptr, nullptr, _compute_code, "{out_file_class}");',
]
else:
body_parts = [
"static const char _vertex_code[] = {\n%s\n\t\t};" % generate_inline_code(header_data.vertex_lines),
"static const char _fragment_code[] = {\n%s\n\t\t};" % generate_inline_code(header_data.fragment_lines),
f'setup(_vertex_code, _fragment_code, nullptr, "{out_file_class}");',
]

if len(header_data.compute_lines):
body_content = "\n\t\t".join(body_parts)

fd.write("\t\tstatic const char _compute_code[] = {\n")
for x in header_data.compute_lines:
for c in x:
fd.write(str(ord(c)) + ",")
fd.write(str(ord("\n")) + ",")
fd.write("\t\t0};\n\n")
# Intended curly brackets are doubled so f-string doesn't eat them up.
shader_template = f"""/* WARNING, THIS FILE WAS GENERATED, DO NOT EDIT */
#ifndef {out_file_ifdef}_RD
#define {out_file_ifdef}_RD
fd.write('\t\tsetup(nullptr, nullptr, _compute_code, "' + out_file_class + '");\n')
fd.write("\t}\n")
#include "servers/rendering/renderer_rd/shader_rd.h"
else:
class {out_file_class} : public ShaderRD {{
fd.write("\t\tstatic const char _vertex_code[] = {\n")
for x in header_data.vertex_lines:
for c in x:
fd.write(str(ord(c)) + ",")
fd.write(str(ord("\n")) + ",")
fd.write("\t\t0};\n\n")
public:
fd.write("\t\tstatic const char _fragment_code[]={\n")
for x in header_data.fragment_lines:
for c in x:
fd.write(str(ord(c)) + ",")
fd.write(str(ord("\n")) + ",")
fd.write("\t\t0};\n\n")
{out_file_class}() {{
fd.write('\t\tsetup(_vertex_code, _fragment_code, nullptr, "' + out_file_class + '");\n')
fd.write("\t}\n")
{body_content}
}}
}};
fd.write("};\n\n")
#endif
"""

fd.write("#endif\n")
fd.close()
with open(out_file, "w") as fd:
fd.write(shader_template)


def build_rd_headers(target, source, env):
Expand All @@ -180,8 +180,6 @@ def include_file_in_raw_header(filename, header_data, depth):
while line.find("#include ") != -1:
includeline = line.replace("#include ", "").strip()[1:-1]

import os.path

included_file = os.path.relpath(os.path.dirname(filename) + "/" + includeline)
include_file_in_raw_header(included_file, header_data, depth + 1)

Expand All @@ -193,28 +191,28 @@ def include_file_in_raw_header(filename, header_data, depth):
fs.close()


def build_raw_header(filename):
header_data = RAWHeaderStruct()
def build_raw_header(filename, header_data=None):
header_data = header_data or RAWHeaderStruct()
include_file_in_raw_header(filename, header_data, 0)

out_file = filename + ".gen.h"
fd = open(out_file, "w")

fd.write("/* WARNING, THIS FILE WAS GENERATED, DO NOT EDIT */\n")

out_file_base = out_file.replace(".glsl.gen.h", "_shader_glsl")
out_file_base = out_file_base[out_file_base.rfind("/") + 1 :]
out_file_base = out_file_base[out_file_base.rfind("\\") + 1 :]
out_file_ifdef = out_file_base.replace(".", "_").upper()
fd.write("#ifndef " + out_file_ifdef + "_RAW_H\n")
fd.write("#define " + out_file_ifdef + "_RAW_H\n")
fd.write("\n")
fd.write("static const char " + out_file_base + "[] = {\n")
for c in header_data.code:
fd.write(str(ord(c)) + ",")
fd.write("\t\t0};\n\n")
fd.write("#endif\n")
fd.close()

shader_template = f"""/* WARNING, THIS FILE WAS GENERATED, DO NOT EDIT */
#ifndef {out_file_ifdef}_RAW_H
#define {out_file_ifdef}_RAW_H
static const char {out_file_base}[] = {{
{generate_inline_code(header_data.code, insert_newline=False)}
}};
#endif
"""

with open(out_file, "w") as f:
f.write(shader_template)


def build_raw_headers(target, source, env):
Expand Down
3 changes: 2 additions & 1 deletion misc/scripts/clang_format.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@ set -uo pipefail

# Loops through all code files tracked by Git.
git ls-files -- '*.c' '*.h' '*.cpp' '*.hpp' '*.cc' '*.hh' '*.cxx' '*.m' '*.mm' '*.inc' '*.java' '*.glsl' \
':!:.git/*' ':!:thirdparty/*' ':!:platform/android/java/lib/src/com/google/*' ':!:*-so_wrap.*' |
':!:.git/*' ':!:thirdparty/*' ':!:platform/android/java/lib/src/com/google/*' ':!:*-so_wrap.*' \
':!:tests/python_build/*' |
while read -r f; do
# Run clang-format.
clang-format --Wno-error=unknown -i "$f"
Expand Down
5 changes: 5 additions & 0 deletions misc/scripts/pytest_builders.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -uo pipefail

echo "Running Python checks for builder system"
pytest ./tests/python_build
26 changes: 26 additions & 0 deletions tests/python_build/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
import os
import sys
from pathlib import Path

import pytest

CWD = Path(__file__).parent
ROOT = CWD.parent.parent
# append directory with build files to sys.path to import them
sys.path.append(str(ROOT))


@pytest.fixture
def shader_files(request):
shader_path = request.param

res = {
"path_input": str(CWD / "fixtures" / f"{shader_path}.glsl"),
"path_output": str(CWD / "fixtures" / f"{shader_path}.glsl.gen.h"),
"path_expected_full": str(CWD / "fixtures" / f"{shader_path}_expected_full.glsl"),
"path_expected_parts": str(CWD / "fixtures" / f"{shader_path}_expected_parts.json"),
}
yield res

if not os.getenv("PYTEST_KEEP_GENERATED_FILES"):
os.remove(res["path_output"])
1 change: 1 addition & 0 deletions tests/python_build/fixtures/gles3/_included.glsl
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
#define M_PI 3.14159265359
34 changes: 34 additions & 0 deletions tests/python_build/fixtures/gles3/vertex_fragment.glsl
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
#include "_included.glsl"

#[modes]

mode_ninepatch = #define USE_NINEPATCH

#[specializations]

DISABLE_LIGHTING = false

#[vertex]

precision highp float;
precision highp int;

layout(location = 0) in highp vec3 vertex;

out highp vec4 position_interp;

void main() {
position_interp = vec4(vertex.x,1,0,1);
}

#[fragment]

precision highp float;
precision highp int;

in highp vec4 position_interp;

void main() {
highp float depth = ((position_interp.z / position_interp.w) + 1.0);
frag_color = vec4(depth);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
/* WARNING, THIS FILE WAS GENERATED, DO NOT EDIT */
#ifndef VERTEX_FRAGMENT_GLSL_GEN_HGLES3_GLES3
#define VERTEX_FRAGMENT_GLSL_GEN_HGLES3_GLES3


#include "drivers/gles3/shader_gles3.h"


class VertexFragmentShaderGLES3 : public ShaderGLES3 {

public:

enum ShaderVariant {
MODE_NINEPATCH,
};

enum Specializations {
DISABLE_LIGHTING=1,
};

_FORCE_INLINE_ void version_bind_shader(RID p_version,ShaderVariant p_variant,uint64_t p_specialization=0) { _version_bind_shader(p_version,p_variant,p_specialization); }

protected:

virtual void _init() override {

static const char **_uniform_strings=nullptr;
static const char* _variant_defines[]={
"#define USE_NINEPATCH",
};

static TexUnitPair *_texunit_pairs=nullptr;
static UBOPair *_ubo_pairs=nullptr;
static Specialization _spec_pairs[]={
{"DISABLE_LIGHTING",false},
};

static const char _vertex_code[]={
10,112,114,101,99,105,115,105,111,110,32,104,105,103,104,112,32,102,108,111,97,116,59,10,112,114,101,99,105,115,105,111,110,32,104,105,103,104,112,32,105,110,116,59,10,10,108,97,121,111,117,116,40,108,111,99,97,116,105,111,110,32,61,32,48,41,32,105,110,32,104,105,103,104,112,32,118,101,99,51,32,118,101,114,116,101,120,59,10,10,111,117,116,32,104,105,103,104,112,32,118,101,99,52,32,112,111,115,105,116,105,111,110,95,105,110,116,101,114,112,59,10,10,118,111,105,100,32,109,97,105,110,40,41,32,123,10,9,112,111,115,105,116,105,111,110,95,105,110,116,101,114,112,32,61,32,118,101,99,52,40,118,101,114,116,101,120,46,120,44,49,44,48,44,49,41,59,10,125,10,10, 0};

static const char _fragment_code[]={
10,112,114,101,99,105,115,105,111,110,32,104,105,103,104,112,32,102,108,111,97,116,59,10,112,114,101,99,105,115,105,111,110,32,104,105,103,104,112,32,105,110,116,59,10,10,105,110,32,104,105,103,104,112,32,118,101,99,52,32,112,111,115,105,116,105,111,110,95,105,110,116,101,114,112,59,10,10,118,111,105,100,32,109,97,105,110,40,41,32,123,10,9,104,105,103,104,112,32,102,108,111,97,116,32,100,101,112,116,104,32,61,32,40,40,112,111,115,105,116,105,111,110,95,105,110,116,101,114,112,46,122,32,47,32,112,111,115,105,116,105,111,110,95,105,110,116,101,114,112,46,119,41,32,43,32,49,46,48,41,59,10,9,102,114,97,103,95,99,111,108,111,114,32,61,32,118,101,99,52,40,100,101,112,116,104,41,59,10,125,10, 0};

_setup(_vertex_code,_fragment_code,"VertexFragmentShaderGLES3",0,_uniform_strings,0,_ubo_pairs,0,_texunit_pairs,1,_spec_pairs,1,_variant_defines);
}

};

#endif

Loading

0 comments on commit 388d35b

Please sign in to comment.