From c5aa08a47000a4ebad56e7bcef0e696118342a5b Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 11 May 2018 17:35:00 -0500 Subject: [PATCH 01/29] Add basic schema --- .../rtd_tests/fixtures/spec/v2/schema.yml | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 readthedocs/rtd_tests/fixtures/spec/v2/schema.yml diff --git a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml new file mode 100644 index 00000000000..c0e2eef3936 --- /dev/null +++ b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml @@ -0,0 +1,49 @@ +# Read the Docs configuration file + +# The version of the spec to be use +# Default: '1' +version: enum('1', '2', required=False) + +# Formats of the documentation to be built +# Default: ['htmlzip', 'pdf', 'epub', 'singlehtmllocalmedia'] +formats: list(enum('htmlzip', 'pdf', 'epub', 'singlehtmllocalmedia'), required=False) + +# The path to the requirements file from the root of the project +requirements_file: str(required=False) + +# Configuration for Conda support +conda: include('conda', required=False) + +# Configuration for the documentation build process +build: include('build', required=False) + +# Configuration of the Python executable to be used +python: include('python', required=False) + +--- + +conda: + # The path to the Conda environment file from the root of the project + file: str() + +build: + # The build docker image to be used + # Default: '2.0' + image: enum('1.0', '2.0', 'latest') + +python: + # The Python version + # Default: '2.7' + version: enum('2', '2.7', '3', '3.3', '3.4', '3.5', '3.6', required=False) + + # Install the project using python setup.py install + # Default: false + setup_py_install: bool(required=False) + + # Install your project with pip + # Default: false + pip_install: bool(required=False) + + # Extra requirements sections to install in addition to the package dependencies + # Default: [] + extra_requirements: list(str(), required=False) From f38259d4acafc413336353d7485cc325207acb8e Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 14 May 2018 11:10:51 -0500 Subject: [PATCH 02/29] Write basic test --- readthedocs/rtd_tests/tests/test_schema.py | 38 ++++++++++++++++++++++ readthedocs/rtd_tests/utils.py | 15 +++++++++ requirements/pip.txt | 1 + 3 files changed, 54 insertions(+) create mode 100644 readthedocs/rtd_tests/tests/test_schema.py diff --git a/readthedocs/rtd_tests/tests/test_schema.py b/readthedocs/rtd_tests/tests/test_schema.py new file mode 100644 index 00000000000..91fa721f4fb --- /dev/null +++ b/readthedocs/rtd_tests/tests/test_schema.py @@ -0,0 +1,38 @@ +from os import path +from os import getcwd + +from django.test import TestCase +import yamale +import pytest + +from readthedocs.rtd_tests.utils import apply_fs + + +class TestSchemaV2(TestCase): + + def setUp(self): + base_path = path.join(getcwd(), 'rtd_tests/fixtures/spec/v2') + self.schema = yamale.make_schema( + path.join(base_path, 'schema.yml') + ) + + @pytest.fixture(autouse=True) + def tmpdir(self, tmpdir): + self.tmpdir = tmpdir + + def create_yaml(self, content): + file = { + 'rtd.yml': content, + } + apply_fs(self.tmpdir, file) + return path.join(self.tmpdir.strpath, 'rtd.yml') + + def test_minimal_config(self): + file = self.create_yaml('') + data = yamale.make_data(file) + yamale.validate(self.schema, data) + + def test_valid_config(self): + file = self.create_yaml('version: "2"') + data = yamale.make_data(file) + yamale.validate(self.schema, data) diff --git a/readthedocs/rtd_tests/utils.py b/readthedocs/rtd_tests/utils.py index 55f7f70b168..d9ed6a7509e 100644 --- a/readthedocs/rtd_tests/utils.py +++ b/readthedocs/rtd_tests/utils.py @@ -121,6 +121,21 @@ def make_test_hg(): return directory +def apply_fs(tmpdir, contents): + """ + Create the directory structure specified in ``contents``. It's a dict of + filenames as keys and the file contents as values. If the value is another + dict, it's a subdirectory. + """ + for filename, content in contents.items(): + if hasattr(content, 'items'): + apply_fs(tmpdir.mkdir(filename), content) + else: + file = tmpdir.join(filename) + file.write(content) + return tmpdir + + def create_user(username, password, **kwargs): user = new(User, username=username, **kwargs) user.set_password(password) diff --git a/requirements/pip.txt b/requirements/pip.txt index 026c570d97c..8a227cdd035 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -64,6 +64,7 @@ Unipath==1.1 django-kombu==0.9.4 mimeparse==0.1.3 mock==2.0.0 +yamale==1.7.0 # stripe 1.20.2 is the latest compatible with our code base (otherwise # gold/tests/test_forms.py fails) From 4c02207b10bb0f83343860147ee5ca2e29f59f4a Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 14 May 2018 11:36:27 -0500 Subject: [PATCH 03/29] Refactor tests --- readthedocs/rtd_tests/tests/test_schema.py | 34 +++++++++++++++------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_schema.py b/readthedocs/rtd_tests/tests/test_schema.py index 91fa721f4fb..971d80cc6a1 100644 --- a/readthedocs/rtd_tests/tests/test_schema.py +++ b/readthedocs/rtd_tests/tests/test_schema.py @@ -1,10 +1,8 @@ -from os import path -from os import getcwd +from os import getcwd, path -from django.test import TestCase -import yamale import pytest - +import yamale +from django.test import TestCase from readthedocs.rtd_tests.utils import apply_fs @@ -27,12 +25,28 @@ def create_yaml(self, content): apply_fs(self.tmpdir, file) return path.join(self.tmpdir.strpath, 'rtd.yml') - def test_minimal_config(self): - file = self.create_yaml('') + def assertValidConfig(self, content): + file = self.create_yaml(content) data = yamale.make_data(file) yamale.validate(self.schema, data) - def test_valid_config(self): - file = self.create_yaml('version: "2"') + def assertInvalidConfig(self, content, exc, msgs=()): + file = self.create_yaml(content) data = yamale.make_data(file) - yamale.validate(self.schema, data) + with pytest.raises(exc) as excinfo: + yamale.validate(self.schema, data) + for msg in msgs: + self.assertIn(msg, str(excinfo.value)) + + def test_minimal_config(self): + self.assertValidConfig('') + + def test_valid_version(self): + self.assertValidConfig('version: "2"') + + def test_invalid_versin(self): + self.assertInvalidConfig( + 'version: "latest"', + ValueError, + ['version: \'latest\' not in'] + ) From a63d39e3edf52a6b04db2fcc113b613e34fb48e3 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 14 May 2018 12:03:53 -0500 Subject: [PATCH 04/29] More tests: formats --- readthedocs/rtd_tests/tests/test_schema.py | 35 +++++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_schema.py b/readthedocs/rtd_tests/tests/test_schema.py index 971d80cc6a1..fc0d23ae875 100644 --- a/readthedocs/rtd_tests/tests/test_schema.py +++ b/readthedocs/rtd_tests/tests/test_schema.py @@ -30,10 +30,10 @@ def assertValidConfig(self, content): data = yamale.make_data(file) yamale.validate(self.schema, data) - def assertInvalidConfig(self, content, exc, msgs=()): + def assertInvalidConfig(self, content, msgs=()): file = self.create_yaml(content) data = yamale.make_data(file) - with pytest.raises(exc) as excinfo: + with pytest.raises(ValueError) as excinfo: yamale.validate(self.schema, data) for msg in msgs: self.assertIn(msg, str(excinfo.value)) @@ -44,9 +44,36 @@ def test_minimal_config(self): def test_valid_version(self): self.assertValidConfig('version: "2"') - def test_invalid_versin(self): + def test_invalid_version(self): self.assertInvalidConfig( 'version: "latest"', - ValueError, ['version: \'latest\' not in'] ) + + def test_valid_formats(self): + content = ''' +version: "2" +formats: + - pdf + - singlehtmllocalmedia + ''' + self.assertValidConfig(content) + + def test_invalid_formats(self): + content = ''' +version: "2" +formats: + - invalidformat + - singlehtmllocalmedia + ''' + self.assertInvalidConfig( + content, + ['formats', '\'invalidformat\' not in'] + ) + + def tets_empty_formats(self): + content = ''' +version: "2" +formats: [] + ''' + self.assertValidConfig(content) From 7fc52b927538aef22de8cffbac1ba07d89571d3e Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 14 May 2018 12:11:34 -0500 Subject: [PATCH 05/29] Version 2 only --- readthedocs/rtd_tests/fixtures/spec/v2/schema.yml | 3 +-- readthedocs/rtd_tests/tests/test_schema.py | 9 ++++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml index c0e2eef3936..fd80b921da2 100644 --- a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml +++ b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml @@ -1,8 +1,7 @@ # Read the Docs configuration file # The version of the spec to be use -# Default: '1' -version: enum('1', '2', required=False) +version: enum('2') # Formats of the documentation to be built # Default: ['htmlzip', 'pdf', 'epub', 'singlehtmllocalmedia'] diff --git a/readthedocs/rtd_tests/tests/test_schema.py b/readthedocs/rtd_tests/tests/test_schema.py index fc0d23ae875..2f28d9e6448 100644 --- a/readthedocs/rtd_tests/tests/test_schema.py +++ b/readthedocs/rtd_tests/tests/test_schema.py @@ -39,9 +39,6 @@ def assertInvalidConfig(self, content, msgs=()): self.assertIn(msg, str(excinfo.value)) def test_minimal_config(self): - self.assertValidConfig('') - - def test_valid_version(self): self.assertValidConfig('version: "2"') def test_invalid_version(self): @@ -50,6 +47,12 @@ def test_invalid_version(self): ['version: \'latest\' not in'] ) + def test_invalid_version_1(self): + self.assertInvalidConfig( + 'version: "1"', + ['version: \'1\' not in'] + ) + def test_valid_formats(self): content = ''' version: "2" From ab7d8fa59b252458ca7c3ba69fd54d84ca99b049 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 14 May 2018 12:17:00 -0500 Subject: [PATCH 06/29] Simplified install options --- readthedocs/rtd_tests/fixtures/spec/v2/schema.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml index fd80b921da2..c0b16d039d8 100644 --- a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml +++ b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml @@ -35,13 +35,8 @@ python: # Default: '2.7' version: enum('2', '2.7', '3', '3.3', '3.4', '3.5', '3.6', required=False) - # Install the project using python setup.py install - # Default: false - setup_py_install: bool(required=False) - - # Install your project with pip - # Default: false - pip_install: bool(required=False) + # Install the project using python setup.py install or pip + install: enum('pip', 'setup.py', required=False) # Extra requirements sections to install in addition to the package dependencies # Default: [] From 602fcd46549f30b63096d8337f8e028d27ab3c48 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 14 May 2018 13:27:49 -0500 Subject: [PATCH 07/29] Tests for all the schema --- readthedocs/rtd_tests/tests/test_schema.py | 151 +++++++++++++++++++++ 1 file changed, 151 insertions(+) diff --git a/readthedocs/rtd_tests/tests/test_schema.py b/readthedocs/rtd_tests/tests/test_schema.py index 2f28d9e6448..a3eff6253ad 100644 --- a/readthedocs/rtd_tests/tests/test_schema.py +++ b/readthedocs/rtd_tests/tests/test_schema.py @@ -58,6 +58,17 @@ def test_valid_formats(self): version: "2" formats: - pdf + - singlehtmllocalmedia + ''' + self.assertValidConfig(content) + + def test_all_valid_formats(self): + content = ''' +version: "2" +formats: + - htmlzip + - pdf + - epub - singlehtmllocalmedia ''' self.assertValidConfig(content) @@ -80,3 +91,143 @@ def tets_empty_formats(self): formats: [] ''' self.assertValidConfig(content) + + def test_valid_requirements_file(self): + content = ''' +version: "2" +requirements_file: docs/requirements.txt + ''' + self.assertValidConfig(content) + + def test_invalid_requirements_file(self): + content = ''' +version: "2" +requirements_file: 23 + ''' + self.assertInvalidConfig( + content, + ['requirements_file: \'23\' is not a str'] + ) + + def test_valid_conda(self): + content = ''' +version: "2" +conda: + file: environment.yml + ''' + self.assertValidConfig(content) + + def test_invalid_conda(self): + content = ''' +version: "2" +conda: + files: environment.yml + ''' + self.assertInvalidConfig( + content, + ['conda.file: Required'] + ) + + def test_valid_build(self): + content = ''' +version: "2" +build: + image: "{image}" + ''' + for image in ['1.0', '2.0', 'latest']: + self.assertValidConfig(content.format(image=image)) + + def test_missing_key_build(self): + content = ''' +version: "2" +build: + imagine: "2.0" + ''' + self.assertInvalidConfig( + content, + ['build.image: Required'] + ) + + def test_invalid_build(self): + content = ''' +version: "2" +build: + image: "4.0" + ''' + self.assertInvalidConfig( + content, + ['build.image: \'4.0\' not in'] + ) + + def test_python_version(self): + content = ''' +version: "2" +python: + version: "{version}" + ''' + versions = ['2', '2.7', '3', '3.3', '3.4', '3.5', '3.6'] + for version in versions: + self.assertValidConfig(content.format(version=version)) + + def test_invalid_python_version(self): + content = ''' +version: "2" +python: + version: "4" + ''' + self.assertInvalidConfig( + content, + ['version: \'4\' not in'] + ) + + def test_no_python_version(self): + content = ''' +version: "2" +python: + guido: true + ''' + self.assertValidConfig(content) + + def test_python_install(self): + content = ''' +version: "2" +python: + version: "3.6" + install: {install} + ''' + for install in ['pip', 'setup.py']: + self.assertValidConfig(content.format(install=install)) + + def test_invalid_python_install(self): + content = ''' +version: "2" +python: + install: guido + ''' + self.assertInvalidConfig( + content, + ['python.install: \'guido\' not in'] + ) + + def test_python_extra_requirements(self): + content = ''' +version: "2" +python: + extra_requirements: + - test + - dev + ''' + self.assertValidConfig(content) + + def test_invalid_python_extra_requirements(self): + content = ''' +version: "2" +python: + extra_requirements: + - 1 + - dev + ''' + self.assertInvalidConfig( + content, + ['\'1\' is not a str'] + ) From b6dd2a20bc55a1b00f633257a4015c0a93aa0220 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 14 May 2018 16:37:01 -0500 Subject: [PATCH 08/29] Add more configurations to the spec --- .../rtd_tests/fixtures/spec/v2/schema.yml | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml index c0b16d039d8..2a427bd959b 100644 --- a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml +++ b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml @@ -19,6 +19,15 @@ build: include('build', required=False) # Configuration of the Python executable to be used python: include('python', required=False) +# Configuration for sphinx +sphinx: include('sphinx', required=False) + +# Submodules configuration +submodules: include('submodules', required=False) + +# Redirects for the current version to be built +redirects: include('redirects', required=False) + --- conda: @@ -41,3 +50,30 @@ python: # Extra requirements sections to install in addition to the package dependencies # Default: [] extra_requirements: list(str(), required=False) + + # Give the virtual environment access to the global site-packages dir + # Default: false + system_packages: bool(required=False) + +sphinx: + # The path to the conf.py file + # Default: rtd will try to find it + file: str(required=False) + +submodules: + # List of submodules to be included + # Default: [] + include: enum(list(str()), 'all', required=False) + + # List of submodules to be ignored + # Default: [] + exclude: enum(list(str()), 'all', required=False) + + # Do a recursive clone? + # Default: false + recursive: bool(required=False) + +redirects: + # Key/value list, represent redirects of type page + # from url -> to url + page: map(str(), str()) From 584ce5e2a951a3dfc08b7361070cb004ec8ed3a7 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 14 May 2018 17:24:28 -0500 Subject: [PATCH 09/29] And more tests --- .../rtd_tests/fixtures/spec/v2/schema.yml | 6 +- readthedocs/rtd_tests/tests/test_schema.py | 104 ++++++++++++++++++ 2 files changed, 107 insertions(+), 3 deletions(-) diff --git a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml index 2a427bd959b..5796ab10da5 100644 --- a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml +++ b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml @@ -5,7 +5,7 @@ version: enum('2') # Formats of the documentation to be built # Default: ['htmlzip', 'pdf', 'epub', 'singlehtmllocalmedia'] -formats: list(enum('htmlzip', 'pdf', 'epub', 'singlehtmllocalmedia'), required=False) +formats: any(list(enum('htmlzip', 'pdf', 'epub', 'singlehtmllocalmedia')), enum('all'), required=False) # The path to the requirements file from the root of the project requirements_file: str(required=False) @@ -63,11 +63,11 @@ sphinx: submodules: # List of submodules to be included # Default: [] - include: enum(list(str()), 'all', required=False) + include: any(list(str()), enum('all'), required=False) # List of submodules to be ignored # Default: [] - exclude: enum(list(str()), 'all', required=False) + exclude: any(list(str()), enum('all'), required=False) # Do a recursive clone? # Default: false diff --git a/readthedocs/rtd_tests/tests/test_schema.py b/readthedocs/rtd_tests/tests/test_schema.py index a3eff6253ad..00681a5732c 100644 --- a/readthedocs/rtd_tests/tests/test_schema.py +++ b/readthedocs/rtd_tests/tests/test_schema.py @@ -73,6 +73,13 @@ def test_all_valid_formats(self): ''' self.assertValidConfig(content) + def test_valid_formats_all(self): + content = ''' +version: "2" +formats: all + ''' + self.assertValidConfig(content) + def test_invalid_formats(self): content = ''' version: "2" @@ -231,3 +238,100 @@ def test_invalid_python_extra_requirements(self): content, ['\'1\' is not a str'] ) + + def test_python_system_packages(self): + content = ''' +version: "2" +python: + system_packages: {option} + ''' + for option in ['true', 'false']: + self.assertValidConfig(content.format(option=option)) + + def test_invalid_python_system_packages(self): + content = ''' +version: "2" +python: + system_packages: not true + ''' + self.assertInvalidConfig(content, ['is not a bool']) + + def test_sphinx(self): + content = ''' +version: "2" +sphinx: + file: docs/conf.py + ''' + self.assertValidConfig(content) + + def test_invalid_sphinx(self): + content = ''' +version: "2" +sphinx: + file: 2 + ''' + self.assertInvalidConfig( + content, + ['is not a str'] + ) + + def test_submodules_include(self): + content = ''' +version: "2" +submodules: + include: + - one + - two + - three + recursive: false + ''' + self.assertValidConfig(content) + + def test_submodules_include_all(self): + content = ''' +version: "2" +submodules: + include: all + ''' + self.assertValidConfig(content) + + def test_submodules_exclude(self): + content = ''' +version: "2" +submodules: + exclude: + - one + - two + - three + ''' + self.assertValidConfig(content) + + def test_submodules_exclude_all(self): + content = ''' +version: "2" +submodules: + exclude: all + recursive: true + ''' + self.assertValidConfig(content) + + def test_redirects(self): + content = ''' +version: "2" +redirects: + page: + 'guides/install.html': 'install.html' + ''' + self.assertValidConfig(content) + + def test_invalid_redirects(self): + content = ''' +version: "2" +redirects: + page: + 'guides/install.html': true + ''' + self.assertInvalidConfig( + content, + ['is not a str'] + ) From e4f5a3a489f6f740f42fe6e7f6f0827351640b5f Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 25 May 2018 11:30:12 -0500 Subject: [PATCH 10/29] Use " instead of scape ' --- readthedocs/rtd_tests/tests/test_schema.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_schema.py b/readthedocs/rtd_tests/tests/test_schema.py index 00681a5732c..1e618d56b8d 100644 --- a/readthedocs/rtd_tests/tests/test_schema.py +++ b/readthedocs/rtd_tests/tests/test_schema.py @@ -6,7 +6,7 @@ from readthedocs.rtd_tests.utils import apply_fs -class TestSchemaV2(TestCase): +class TestYAMLSchemaV2(TestCase): def setUp(self): base_path = path.join(getcwd(), 'rtd_tests/fixtures/spec/v2') @@ -19,10 +19,10 @@ def tmpdir(self, tmpdir): self.tmpdir = tmpdir def create_yaml(self, content): - file = { + fs = { 'rtd.yml': content, } - apply_fs(self.tmpdir, file) + apply_fs(self.tmpdir, fs) return path.join(self.tmpdir.strpath, 'rtd.yml') def assertValidConfig(self, content): @@ -44,13 +44,13 @@ def test_minimal_config(self): def test_invalid_version(self): self.assertInvalidConfig( 'version: "latest"', - ['version: \'latest\' not in'] + ["version: 'latest' not in"] ) def test_invalid_version_1(self): self.assertInvalidConfig( 'version: "1"', - ['version: \'1\' not in'] + ["version: '1' not in"] ) def test_valid_formats(self): @@ -89,7 +89,7 @@ def test_invalid_formats(self): ''' self.assertInvalidConfig( content, - ['formats', '\'invalidformat\' not in'] + ['formats', "'invalidformat' not in"] ) def tets_empty_formats(self): @@ -113,7 +113,7 @@ def test_invalid_requirements_file(self): ''' self.assertInvalidConfig( content, - ['requirements_file: \'23\' is not a str'] + ["requirements_file: '23' is not a str"] ) def test_valid_conda(self): @@ -163,7 +163,7 @@ def test_invalid_build(self): ''' self.assertInvalidConfig( content, - ['build.image: \'4.0\' not in'] + ["build.image: '4.0' not in"] ) def test_python_version(self): @@ -184,7 +184,7 @@ def test_invalid_python_version(self): ''' self.assertInvalidConfig( content, - ['version: \'4\' not in'] + ["version: '4' not in"] ) def test_no_python_version(self): @@ -213,7 +213,7 @@ def test_invalid_python_install(self): ''' self.assertInvalidConfig( content, - ['python.install: \'guido\' not in'] + ["python.install: 'guido' not in"] ) def test_python_extra_requirements(self): @@ -236,7 +236,7 @@ def test_invalid_python_extra_requirements(self): ''' self.assertInvalidConfig( content, - ['\'1\' is not a str'] + ["'1' is not a str"] ) def test_python_system_packages(self): From cd5b5dcf408f549563fac3dad0d1086a513a0e6d Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 25 May 2018 12:29:04 -0500 Subject: [PATCH 11/29] Update spec --- .../rtd_tests/fixtures/spec/v2/schema.yml | 37 ++++++----- readthedocs/rtd_tests/tests/test_schema.py | 64 ++++++++++--------- 2 files changed, 53 insertions(+), 48 deletions(-) diff --git a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml index 5796ab10da5..e39f29ca4da 100644 --- a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml +++ b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml @@ -4,11 +4,8 @@ version: enum('2') # Formats of the documentation to be built -# Default: ['htmlzip', 'pdf', 'epub', 'singlehtmllocalmedia'] -formats: any(list(enum('htmlzip', 'pdf', 'epub', 'singlehtmllocalmedia')), enum('all'), required=False) - -# The path to the requirements file from the root of the project -requirements_file: str(required=False) +# Default: [] +formats: any(list(enum('htmlzip', 'pdf', 'epub')), enum('all'), required=False) # Configuration for Conda support conda: include('conda', required=False) @@ -16,33 +13,41 @@ conda: include('conda', required=False) # Configuration for the documentation build process build: include('build', required=False) -# Configuration of the Python executable to be used +# Configuration of the Python environment to be used python: include('python', required=False) -# Configuration for sphinx +# Configuration for sphinx documentation sphinx: include('sphinx', required=False) +# Configuration for mkdocs documentation +mkdocs: include('mkdocs', required=False) + # Submodules configuration submodules: include('submodules', required=False) # Redirects for the current version to be built -redirects: include('redirects', required=False) +# Key/value list, represent redirects of type `type` +# from url -> to url +redirects: map(enum('page'), map(str(), str()), required=False) --- conda: # The path to the Conda environment file from the root of the project - file: str() + environment: str() build: # The build docker image to be used # Default: '2.0' - image: enum('1.0', '2.0', 'latest') + image: enum('1.0', '2.0', 'latest', required=False) python: # The Python version # Default: '2.7' - version: enum('2', '2.7', '3', '3.3', '3.4', '3.5', '3.6', required=False) + version: enum('2', '2.7', '3', '3.5', '3.6', required=False) + + # The path to the requirements file from the root of the project + requirements: str(required=False) # Install the project using python setup.py install or pip install: enum('pip', 'setup.py', required=False) @@ -58,7 +63,10 @@ python: sphinx: # The path to the conf.py file # Default: rtd will try to find it - file: str(required=False) + configuration: str(required=False) + +mkdocs: + # Something submodules: # List of submodules to be included @@ -72,8 +80,3 @@ submodules: # Do a recursive clone? # Default: false recursive: bool(required=False) - -redirects: - # Key/value list, represent redirects of type page - # from url -> to url - page: map(str(), str()) diff --git a/readthedocs/rtd_tests/tests/test_schema.py b/readthedocs/rtd_tests/tests/test_schema.py index 1e618d56b8d..d35dc30ed6d 100644 --- a/readthedocs/rtd_tests/tests/test_schema.py +++ b/readthedocs/rtd_tests/tests/test_schema.py @@ -58,7 +58,6 @@ def test_valid_formats(self): version: "2" formats: - pdf - - singlehtmllocalmedia ''' self.assertValidConfig(content) @@ -69,7 +68,6 @@ def test_all_valid_formats(self): - htmlzip - pdf - epub - - singlehtmllocalmedia ''' self.assertValidConfig(content) @@ -99,28 +97,11 @@ def tets_empty_formats(self): ''' self.assertValidConfig(content) - def test_valid_requirements_file(self): - content = ''' -version: "2" -requirements_file: docs/requirements.txt - ''' - self.assertValidConfig(content) - - def test_invalid_requirements_file(self): - content = ''' -version: "2" -requirements_file: 23 - ''' - self.assertInvalidConfig( - content, - ["requirements_file: '23' is not a str"] - ) - def test_valid_conda(self): content = ''' version: "2" conda: - file: environment.yml + environment: environment.yml ''' self.assertValidConfig(content) @@ -132,7 +113,7 @@ def test_invalid_conda(self): ''' self.assertInvalidConfig( content, - ['conda.file: Required'] + ['conda.environment: Required'] ) def test_valid_build(self): @@ -150,20 +131,17 @@ def test_missing_key_build(self): build: imagine: "2.0" ''' - self.assertInvalidConfig( - content, - ['build.image: Required'] - ) + self.assertValidConfig(content) def test_invalid_build(self): content = ''' version: "2" build: - image: "4.0" + image: "9.0" ''' self.assertInvalidConfig( content, - ["build.image: '4.0' not in"] + ["build.image: '9.0' not in"] ) def test_python_version(self): @@ -172,7 +150,7 @@ def test_python_version(self): python: version: "{version}" ''' - versions = ['2', '2.7', '3', '3.3', '3.4', '3.5', '3.6'] + versions = ['2', '2.7', '3', '3.5', '3.6'] for version in versions: self.assertValidConfig(content.format(version=version)) @@ -195,6 +173,26 @@ def test_no_python_version(self): ''' self.assertValidConfig(content) + def test_valid_requirements_file(self): + content = ''' +version: "2" +python: + requirements: docs/requirements.txt + ''' + self.assertValidConfig(content) + + def test_invalid_requirements_file(self): + content = ''' +version: "2" +python: + requirements: 23 + ''' + self.assertInvalidConfig( + content, + ["requirements: '23' is not a str"] + ) + + def test_python_install(self): content = ''' version: "2" @@ -252,9 +250,13 @@ def test_invalid_python_system_packages(self): content = ''' version: "2" python: - system_packages: not true + system_packages: {value} ''' - self.assertInvalidConfig(content, ['is not a bool']) + for value in ['not true', "''", '[]']: + self.assertInvalidConfig( + content.format(value=value), + ['is not a bool'] + ) def test_sphinx(self): content = ''' @@ -268,7 +270,7 @@ def test_invalid_sphinx(self): content = ''' version: "2" sphinx: - file: 2 + configuration: 2 ''' self.assertInvalidConfig( content, From 07350b9a10c5bd27098e39a5fb84f40190124e81 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 25 May 2018 12:33:16 -0500 Subject: [PATCH 12/29] Rename --- .../rtd_tests/tests/{test_schema.py => test_yml_schema.py} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename readthedocs/rtd_tests/tests/{test_schema.py => test_yml_schema.py} (99%) diff --git a/readthedocs/rtd_tests/tests/test_schema.py b/readthedocs/rtd_tests/tests/test_yml_schema.py similarity index 99% rename from readthedocs/rtd_tests/tests/test_schema.py rename to readthedocs/rtd_tests/tests/test_yml_schema.py index d35dc30ed6d..4cdb184bab1 100644 --- a/readthedocs/rtd_tests/tests/test_schema.py +++ b/readthedocs/rtd_tests/tests/test_yml_schema.py @@ -6,7 +6,7 @@ from readthedocs.rtd_tests.utils import apply_fs -class TestYAMLSchemaV2(TestCase): +class TestYMLSchemaV2(TestCase): def setUp(self): base_path = path.join(getcwd(), 'rtd_tests/fixtures/spec/v2') From 00e69d9229794429de8c7a5f26bf8d854ebc8400 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 25 May 2018 12:47:22 -0500 Subject: [PATCH 13/29] Refactor tests --- .../rtd_tests/tests/test_yml_schema.py | 41 +++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_yml_schema.py b/readthedocs/rtd_tests/tests/test_yml_schema.py index 4cdb184bab1..4f25b343ad2 100644 --- a/readthedocs/rtd_tests/tests/test_yml_schema.py +++ b/readthedocs/rtd_tests/tests/test_yml_schema.py @@ -44,16 +44,16 @@ def test_minimal_config(self): def test_invalid_version(self): self.assertInvalidConfig( 'version: "latest"', - ["version: 'latest' not in"] + ['version:', "'latest' not in"] ) def test_invalid_version_1(self): self.assertInvalidConfig( 'version: "1"', - ["version: '1' not in"] + ['version', "'1' not in"] ) - def test_valid_formats(self): + def test_formats(self): content = ''' version: "2" formats: @@ -61,7 +61,7 @@ def test_valid_formats(self): ''' self.assertValidConfig(content) - def test_all_valid_formats(self): + def test_formats_all(self): content = ''' version: "2" formats: @@ -71,14 +71,14 @@ def test_all_valid_formats(self): ''' self.assertValidConfig(content) - def test_valid_formats_all(self): + def test_formats_key_all(self): content = ''' version: "2" formats: all ''' self.assertValidConfig(content) - def test_invalid_formats(self): + def test_formats_invalid(self): content = ''' version: "2" formats: @@ -90,14 +90,14 @@ def test_invalid_formats(self): ['formats', "'invalidformat' not in"] ) - def tets_empty_formats(self): + def tets_formats_empty(self): content = ''' version: "2" formats: [] ''' self.assertValidConfig(content) - def test_valid_conda(self): + def test_conda(self): content = ''' version: "2" conda: @@ -105,7 +105,7 @@ def test_valid_conda(self): ''' self.assertValidConfig(content) - def test_invalid_conda(self): + def test_conda_invalid(self): content = ''' version: "2" conda: @@ -116,7 +116,7 @@ def test_invalid_conda(self): ['conda.environment: Required'] ) - def test_valid_build(self): + def test_build(self): content = ''' version: "2" build: @@ -125,15 +125,15 @@ def test_valid_build(self): for image in ['1.0', '2.0', 'latest']: self.assertValidConfig(content.format(image=image)) - def test_missing_key_build(self): + def test_build_missing_image_key(self): content = ''' version: "2" build: - imagine: "2.0" + imagine: "2.0" # note the typo ''' self.assertValidConfig(content) - def test_invalid_build(self): + def test_build_invalid(self): content = ''' version: "2" build: @@ -154,7 +154,7 @@ def test_python_version(self): for version in versions: self.assertValidConfig(content.format(version=version)) - def test_invalid_python_version(self): + def test_python_version_invalid(self): content = ''' version: "2" python: @@ -173,7 +173,7 @@ def test_no_python_version(self): ''' self.assertValidConfig(content) - def test_valid_requirements_file(self): + def test_valid_requirements(self): content = ''' version: "2" python: @@ -189,10 +189,9 @@ def test_invalid_requirements_file(self): ''' self.assertInvalidConfig( content, - ["requirements: '23' is not a str"] + ['requirements:', "'23' is not a str"] ) - def test_python_install(self): content = ''' version: "2" @@ -203,7 +202,7 @@ def test_python_install(self): for install in ['pip', 'setup.py']: self.assertValidConfig(content.format(install=install)) - def test_invalid_python_install(self): + def test_python_install_invalid(self): content = ''' version: "2" python: @@ -224,7 +223,7 @@ def test_python_extra_requirements(self): ''' self.assertValidConfig(content) - def test_invalid_python_extra_requirements(self): + def test_python_extra_requirements_invalid(self): content = ''' version: "2" python: @@ -246,7 +245,7 @@ def test_python_system_packages(self): for option in ['true', 'false']: self.assertValidConfig(content.format(option=option)) - def test_invalid_python_system_packages(self): + def test_python_system_packages_invalid(self): content = ''' version: "2" python: @@ -266,7 +265,7 @@ def test_sphinx(self): ''' self.assertValidConfig(content) - def test_invalid_sphinx(self): + def test_sphinx_invalid(self): content = ''' version: "2" sphinx: From f4ddb429471322d003d82d3892136b9518c74c37 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 25 May 2018 20:28:53 -0500 Subject: [PATCH 14/29] Add path validator --- .../rtd_tests/fixtures/spec/v2/schema.yml | 6 +-- readthedocs/rtdyml/__init__.py | 50 +++++++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 readthedocs/rtdyml/__init__.py diff --git a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml index e39f29ca4da..3a563b63abe 100644 --- a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml +++ b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml @@ -34,7 +34,7 @@ redirects: map(enum('page'), map(str(), str()), required=False) conda: # The path to the Conda environment file from the root of the project - environment: str() + environment: path() build: # The build docker image to be used @@ -47,7 +47,7 @@ python: version: enum('2', '2.7', '3', '3.5', '3.6', required=False) # The path to the requirements file from the root of the project - requirements: str(required=False) + requirements: path(required=False) # Install the project using python setup.py install or pip install: enum('pip', 'setup.py', required=False) @@ -63,7 +63,7 @@ python: sphinx: # The path to the conf.py file # Default: rtd will try to find it - configuration: str(required=False) + configuration: path(required=False) mkdocs: # Something diff --git a/readthedocs/rtdyml/__init__.py b/readthedocs/rtdyml/__init__.py new file mode 100644 index 00000000000..85e0c032e2a --- /dev/null +++ b/readthedocs/rtdyml/__init__.py @@ -0,0 +1,50 @@ +from os import path + +import six + +import yamale +from yamale.validators import DefaultValidators, Validator + +V2_SCHEMA = path.join( + path.dirname(__file__), + '../rtd_tests/fixtures/spec/v2/schema.yml' +) + + +class PathValidator(Validator): + """ + Docstring + """ + + tag = 'path' + constraints = [] + configuration_file = '.' + + def _is_valid(self, value): + if isinstance(value, six.string_types): + file_ = path.join( + path.dirname(self.configuration_file), + value + ) + return path.exists(file_) + return False + + +class BuildConfig(object): + + def __init__(self, configuration_file): + self.configuration_file = configuration_file + self.data = yamale.make_data(self.configuration_file) + self.schema = yamale.make_schema( + V2_SCHEMA, + validators=self.get_validators() + ) + + def get_validators(self): + validators = DefaultValidators.copy() + PathValidator.configuration_file = self.configuration_file + validators[PathValidator.tag] = PathValidator + return validators + + def validate(self): + return yamale.validate(self.schema, self.data) From 89badbf9e6ec4e1619425befcbbe86954c1f36a8 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 25 May 2018 20:30:02 -0500 Subject: [PATCH 15/29] Refactor tests pytest style --- .../rtd_tests/tests/test_yml_schema.py | 480 ++++++++++-------- 1 file changed, 265 insertions(+), 215 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_yml_schema.py b/readthedocs/rtd_tests/tests/test_yml_schema.py index 4f25b343ad2..9204eef6b5c 100644 --- a/readthedocs/rtd_tests/tests/test_yml_schema.py +++ b/readthedocs/rtd_tests/tests/test_yml_schema.py @@ -1,283 +1,327 @@ -from os import getcwd, path +from os import path import pytest -import yamale -from django.test import TestCase +from readthedocs.rtdyml import BuildConfig from readthedocs.rtd_tests.utils import apply_fs -class TestYMLSchemaV2(TestCase): - - def setUp(self): - base_path = path.join(getcwd(), 'rtd_tests/fixtures/spec/v2') - self.schema = yamale.make_schema( - path.join(base_path, 'schema.yml') - ) - - @pytest.fixture(autouse=True) - def tmpdir(self, tmpdir): - self.tmpdir = tmpdir - - def create_yaml(self, content): - fs = { - 'rtd.yml': content, - } - apply_fs(self.tmpdir, fs) - return path.join(self.tmpdir.strpath, 'rtd.yml') - - def assertValidConfig(self, content): - file = self.create_yaml(content) - data = yamale.make_data(file) - yamale.validate(self.schema, data) - - def assertInvalidConfig(self, content, msgs=()): - file = self.create_yaml(content) - data = yamale.make_data(file) - with pytest.raises(ValueError) as excinfo: - yamale.validate(self.schema, data) - for msg in msgs: - self.assertIn(msg, str(excinfo.value)) - - def test_minimal_config(self): - self.assertValidConfig('version: "2"') - - def test_invalid_version(self): - self.assertInvalidConfig( - 'version: "latest"', - ['version:', "'latest' not in"] - ) - - def test_invalid_version_1(self): - self.assertInvalidConfig( - 'version: "1"', - ['version', "'1' not in"] - ) - - def test_formats(self): - content = ''' +def create_yaml(tmpdir, content): + fs = { + 'environment.yml': '', + 'rtd.yml': content, + 'docs': { + 'conf.py': '', + 'requirements.txt': '', + }, + } + apply_fs(tmpdir, fs) + return path.join(tmpdir.strpath, 'rtd.yml') + + +def assertValidConfig(tmpdir, content): + file = create_yaml(tmpdir, content) + build = BuildConfig(file) + build.validate() + + +def assertInvalidConfig(tmpdir, content, msgs=()): + file = create_yaml(tmpdir, content) + with pytest.raises(ValueError) as excinfo: + BuildConfig(file).validate() + for msg in msgs: + msg in str(excinfo.value) + + +def test_minimal_config(tmpdir): + assertValidConfig(tmpdir, 'version: "2"') + + +def test_invalid_version(tmpdir): + assertInvalidConfig( + tmpdir, + 'version: "latest"', + ['version:', "'latest' not in"] + ) + + +def test_invalid_version_1(tmpdir): + assertInvalidConfig( + tmpdir, + 'version: "1"', + ['version', "'1' not in"] + ) + + +def test_formats(tmpdir): + content = ''' version: "2" formats: - pdf - ''' - self.assertValidConfig(content) + ''' + assertValidConfig(tmpdir, content) - def test_formats_all(self): - content = ''' + +def test_formats_all(tmpdir): + content = ''' version: "2" formats: - htmlzip - pdf - epub - ''' - self.assertValidConfig(content) + ''' + assertValidConfig(tmpdir, content) - def test_formats_key_all(self): - content = ''' + +def test_formats_key_all(tmpdir): + content = ''' version: "2" formats: all - ''' - self.assertValidConfig(content) + ''' + assertValidConfig(tmpdir, content) - def test_formats_invalid(self): - content = ''' + +def test_formats_invalid(tmpdir): + content = ''' version: "2" formats: - invalidformat - singlehtmllocalmedia - ''' - self.assertInvalidConfig( - content, - ['formats', "'invalidformat' not in"] - ) + ''' + assertInvalidConfig( + tmpdir, + content, + ['formats', "'invalidformat' not in"] + ) - def tets_formats_empty(self): - content = ''' + +def test_formats_empty(tmpdir): + content = ''' version: "2" formats: [] - ''' - self.assertValidConfig(content) + ''' + assertValidConfig(tmpdir, content) + - def test_conda(self): - content = ''' +def test_conda(tmpdir): + content = ''' version: "2" conda: environment: environment.yml - ''' - self.assertValidConfig(content) + ''' + assertValidConfig(tmpdir, content) + - def test_conda_invalid(self): - content = ''' +def test_conda_invalid(tmpdir): + content = ''' version: "2" conda: - files: environment.yml - ''' - self.assertInvalidConfig( - content, - ['conda.environment: Required'] - ) + environment: environment.yaml + ''' + assertInvalidConfig( + tmpdir, + content, + ['environment.yaml', 'is not a path'] + ) + - def test_build(self): - content = ''' +def test_conda_missing_key(tmpdir): + content = ''' +version: "2" +conda: + files: environment.yml + ''' + assertInvalidConfig( + tmpdir, + content, + ['conda.environment: Required'] + ) + + +@pytest.mark.parametrize('value', ['1.0', '2.0', 'latest']) +def test_build(tmpdir, value): + content = ''' version: "2" build: - image: "{image}" - ''' - for image in ['1.0', '2.0', 'latest']: - self.assertValidConfig(content.format(image=image)) + image: "{value}" + ''' + assertValidConfig(tmpdir, content.format(value=value)) - def test_build_missing_image_key(self): - content = ''' + +def test_build_missing_image_key(tmpdir): + content = ''' version: "2" build: imagine: "2.0" # note the typo - ''' - self.assertValidConfig(content) + ''' + assertValidConfig(tmpdir, content) + - def test_build_invalid(self): - content = ''' +def test_build_invalid(tmpdir): + content = ''' version: "2" build: image: "9.0" - ''' - self.assertInvalidConfig( - content, - ["build.image: '9.0' not in"] - ) + ''' + assertInvalidConfig( + tmpdir, + content, + ["build.image: '9.0' not in"] + ) + - def test_python_version(self): - content = ''' +@pytest.mark.parametrize('value', ['2', '2.7', '3', '3.5', '3.6']) +def test_python_version(tmpdir, value): + content = ''' version: "2" python: - version: "{version}" - ''' - versions = ['2', '2.7', '3', '3.5', '3.6'] - for version in versions: - self.assertValidConfig(content.format(version=version)) + version: "{value}" + ''' + assertValidConfig(tmpdir, content.format(value=value)) + - def test_python_version_invalid(self): - content = ''' +def test_python_version_invalid(tmpdir): + content = ''' version: "2" python: version: "4" - ''' - self.assertInvalidConfig( - content, - ["version: '4' not in"] - ) + ''' + assertInvalidConfig( + tmpdir, + content, + ["version: '4' not in"] + ) + - def test_no_python_version(self): - content = ''' +def test_no_python_version(tmpdir): + content = ''' version: "2" python: guido: true - ''' - self.assertValidConfig(content) + ''' + assertValidConfig(tmpdir, content) + - def test_valid_requirements(self): - content = ''' +def test_valid_requirements(tmpdir): + content = ''' version: "2" python: requirements: docs/requirements.txt - ''' - self.assertValidConfig(content) + ''' + assertValidConfig(tmpdir, content) - def test_invalid_requirements_file(self): - content = ''' + +def test_invalid_requirements_file(tmpdir): + content = ''' version: "2" python: requirements: 23 - ''' - self.assertInvalidConfig( - content, - ['requirements:', "'23' is not a str"] - ) + ''' + assertInvalidConfig( + tmpdir, + content, + ['requirements:', "'23' is not a path"] + ) - def test_python_install(self): - content = ''' + +@pytest.mark.parametrize('value', ['pip', 'setup.py']) +def test_python_install(tmpdir, value): + content = ''' version: "2" python: version: "3.6" - install: {install} - ''' - for install in ['pip', 'setup.py']: - self.assertValidConfig(content.format(install=install)) + install: {value} + ''' + assertValidConfig(tmpdir, content.format(value=value)) + - def test_python_install_invalid(self): - content = ''' +def test_python_install_invalid(tmpdir): + content = ''' version: "2" python: install: guido - ''' - self.assertInvalidConfig( - content, - ["python.install: 'guido' not in"] - ) + ''' + assertInvalidConfig( + tmpdir, + content, + ["python.install: 'guido' not in"] + ) + - def test_python_extra_requirements(self): - content = ''' +def test_python_extra_requirements(tmpdir): + content = ''' version: "2" python: extra_requirements: - test - dev - ''' - self.assertValidConfig(content) + ''' + assertValidConfig(tmpdir, content) + - def test_python_extra_requirements_invalid(self): - content = ''' +def test_python_extra_requirements_invalid(tmpdir): + content = ''' version: "2" python: extra_requirements: - 1 - dev - ''' - self.assertInvalidConfig( - content, - ["'1' is not a str"] - ) + ''' + assertInvalidConfig( + tmpdir, + content, + ["'1' is not a str"] + ) - def test_python_system_packages(self): - content = ''' + +@pytest.mark.parametrize('value', ['true', 'false']) +def test_python_system_packages(tmpdir, value): + content = ''' version: "2" python: - system_packages: {option} - ''' - for option in ['true', 'false']: - self.assertValidConfig(content.format(option=option)) + system_packages: {value} + ''' + assertValidConfig(tmpdir, content.format(value=value)) + - def test_python_system_packages_invalid(self): - content = ''' +@pytest.mark.parametrize('value', ['not true', "''", '[]']) +def test_python_system_packages_invalid(tmpdir, value): + content = ''' version: "2" python: system_packages: {value} - ''' - for value in ['not true', "''", '[]']: - self.assertInvalidConfig( - content.format(value=value), - ['is not a bool'] - ) - - def test_sphinx(self): - content = ''' + ''' + assertInvalidConfig( + tmpdir, + content.format(value=value), + ['is not a bool'] + ) + + +def test_sphinx(tmpdir): + content = ''' version: "2" sphinx: - file: docs/conf.py - ''' - self.assertValidConfig(content) + file: docs/conf.py # Default value for configuration key + ''' + assertValidConfig(tmpdir, content) + - def test_sphinx_invalid(self): - content = ''' +@pytest.mark.parametrize('value', ['2', 'environment.py']) +def test_sphinx_invalid(tmpdir, value): + content = ''' version: "2" sphinx: - configuration: 2 - ''' - self.assertInvalidConfig( - content, - ['is not a str'] - ) + configuration: {value} + ''' + assertInvalidConfig( + tmpdir, + content, + ['is not a path'] + ) + - def test_submodules_include(self): - content = ''' +def test_submodules_include(tmpdir): + content = ''' version: "2" submodules: include: @@ -285,54 +329,60 @@ def test_submodules_include(self): - two - three recursive: false - ''' - self.assertValidConfig(content) + ''' + assertValidConfig(tmpdir, content) + - def test_submodules_include_all(self): - content = ''' +def test_submodules_include_all(tmpdir): + content = ''' version: "2" submodules: - include: all - ''' - self.assertValidConfig(content) +include: all + ''' + assertValidConfig(tmpdir, content) - def test_submodules_exclude(self): - content = ''' + +def test_submodules_exclude(tmpdir): + content = ''' version: "2" submodules: - exclude: - - one - - two - - three - ''' - self.assertValidConfig(content) +exclude: + - one + - two + - three + ''' + assertValidConfig(tmpdir, content) - def test_submodules_exclude_all(self): - content = ''' + +def test_submodules_exclude_all(tmpdir): + content = ''' version: "2" submodules: exclude: all recursive: true - ''' - self.assertValidConfig(content) + ''' + assertValidConfig(tmpdir, content) - def test_redirects(self): - content = ''' + +def test_redirects(tmpdir): + content = ''' version: "2" redirects: page: 'guides/install.html': 'install.html' - ''' - self.assertValidConfig(content) + ''' + assertValidConfig(tmpdir, content) - def test_invalid_redirects(self): - content = ''' + +def test_invalid_redirects(tmpdir): + content = ''' version: "2" redirects: page: 'guides/install.html': true - ''' - self.assertInvalidConfig( - content, - ['is not a str'] - ) + ''' + assertInvalidConfig( + tmpdir, + content, + ['is not a str'] + ) From 7b30c8c519e548444c22699a4030d06b3c291f3b Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 29 May 2018 13:06:23 -0500 Subject: [PATCH 16/29] Docs --- readthedocs/rtdyml/__init__.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/readthedocs/rtdyml/__init__.py b/readthedocs/rtdyml/__init__.py index 85e0c032e2a..f2151be15fd 100644 --- a/readthedocs/rtdyml/__init__.py +++ b/readthedocs/rtdyml/__init__.py @@ -1,3 +1,5 @@ +"""Validator for the RTD configuration file.""" + from os import path import six @@ -12,8 +14,12 @@ class PathValidator(Validator): + """ - Docstring + Path validator + + Checks if the given value is a string and a existing + file. """ tag = 'path' @@ -32,19 +38,22 @@ def _is_valid(self, value): class BuildConfig(object): + """Wrapper object to validate to configuration file.""" + def __init__(self, configuration_file): self.configuration_file = configuration_file self.data = yamale.make_data(self.configuration_file) self.schema = yamale.make_schema( V2_SCHEMA, - validators=self.get_validators() + validators=self._get_validators() ) - def get_validators(self): + def _get_validators(self): validators = DefaultValidators.copy() PathValidator.configuration_file = self.configuration_file validators[PathValidator.tag] = PathValidator return validators def validate(self): + """Validate the current configuration file.""" return yamale.validate(self.schema, self.data) From e03804628f94118a7bfa38848f3dbe948cb458dd Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 29 May 2018 13:07:15 -0500 Subject: [PATCH 17/29] Reuse utils from readthedocs_build --- readthedocs/rtd_tests/tests/test_yml_schema.py | 4 ++-- readthedocs/rtd_tests/utils.py | 15 --------------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_yml_schema.py b/readthedocs/rtd_tests/tests/test_yml_schema.py index 9204eef6b5c..fc1309666df 100644 --- a/readthedocs/rtd_tests/tests/test_yml_schema.py +++ b/readthedocs/rtd_tests/tests/test_yml_schema.py @@ -2,7 +2,7 @@ import pytest from readthedocs.rtdyml import BuildConfig -from readthedocs.rtd_tests.utils import apply_fs +from readthedocs_build.testing import utils def create_yaml(tmpdir, content): @@ -14,7 +14,7 @@ def create_yaml(tmpdir, content): 'requirements.txt': '', }, } - apply_fs(tmpdir, fs) + utils.apply_fs(tmpdir, fs) return path.join(tmpdir.strpath, 'rtd.yml') diff --git a/readthedocs/rtd_tests/utils.py b/readthedocs/rtd_tests/utils.py index d9ed6a7509e..55f7f70b168 100644 --- a/readthedocs/rtd_tests/utils.py +++ b/readthedocs/rtd_tests/utils.py @@ -121,21 +121,6 @@ def make_test_hg(): return directory -def apply_fs(tmpdir, contents): - """ - Create the directory structure specified in ``contents``. It's a dict of - filenames as keys and the file contents as values. If the value is another - dict, it's a subdirectory. - """ - for filename, content in contents.items(): - if hasattr(content, 'items'): - apply_fs(tmpdir.mkdir(filename), content) - else: - file = tmpdir.join(filename) - file.write(content) - return tmpdir - - def create_user(username, password, **kwargs): user = new(User, username=username, **kwargs) user.set_password(password) From b2e5dbc1fa16f2f55d18ef7b182c8648e9e88fc4 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 29 May 2018 19:09:57 -0500 Subject: [PATCH 18/29] Add sphinx and mkdocs to the spec --- .../rtd_tests/fixtures/spec/v2/schema.yml | 15 +++- .../rtd_tests/tests/test_yml_schema.py | 88 +++++++++++++++++++ 2 files changed, 101 insertions(+), 2 deletions(-) diff --git a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml index 3a563b63abe..b44d47cbe4b 100644 --- a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml +++ b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml @@ -60,13 +60,24 @@ python: # Default: false system_packages: bool(required=False) -sphinx: +sphinx: # The path to the conf.py file # Default: rtd will try to find it configuration: path(required=False) + # Add the -W option to sphinx-build + # Default: false + fail_on_warning: bool(required=False) + mkdocs: - # Something + # The path to the mkdocs.yml file + # Default: rtd will try to find it + configuration: path(required=False) + + # Add the --strict optio to mkdocs build + # Default: false + fail_on_warning: bool(required=False) + submodules: # List of submodules to be included diff --git a/readthedocs/rtd_tests/tests/test_yml_schema.py b/readthedocs/rtd_tests/tests/test_yml_schema.py index fc1309666df..4732fa95482 100644 --- a/readthedocs/rtd_tests/tests/test_yml_schema.py +++ b/readthedocs/rtd_tests/tests/test_yml_schema.py @@ -8,6 +8,7 @@ def create_yaml(tmpdir, content): fs = { 'environment.yml': '', + 'mkdocs.yml': '', 'rtd.yml': content, 'docs': { 'conf.py': '', @@ -300,6 +301,15 @@ def test_python_system_packages_invalid(tmpdir, value): def test_sphinx(tmpdir): content = ''' version: "2" +sphinx: + configuration: docs/conf.py + ''' + assertValidConfig(tmpdir, content) + + +def test_sphinx_default_value(tmpdir): + content = ''' +version: "2" sphinx: file: docs/conf.py # Default value for configuration key ''' @@ -320,6 +330,84 @@ def test_sphinx_invalid(tmpdir, value): ) +def test_sphinx_fail_on_warning(tmpdir): + content = ''' +version: "2" +sphinx: + fail_on_warning: true + ''' + assertValidConfig(tmpdir, content) + + +@pytest.mark.parametrize('value', ['not true', "''", '[]']) +def test_sphinx_fail_on_warning_invalid(tmpdir, value): + content = ''' +version: "2" +sphinx: + fail_on_warning: {value} + ''' + assertInvalidConfig( + tmpdir, + content.format(value=value), + ['is not a bool'] + ) + + +def test_mkdocs(tmpdir): + content = ''' +version: "2" +mkdocs: + configuration: mkdocs.yml + ''' + assertValidConfig(tmpdir, content) + + +def test_mkdocs_default_value(tmpdir): + content = ''' +version: "2" +mkdocs: + file: mkdocs.yml # Default value for configuration key + ''' + assertValidConfig(tmpdir, content) + + +@pytest.mark.parametrize('value', ['2', 'environment.py']) +def test_mkdocs_invalid(tmpdir, value): + content = ''' +version: "2" +mkdocs: + configuration: {value} + ''' + assertInvalidConfig( + tmpdir, + content, + ['is not a path'] + ) + + +def test_mkdocs_fail_on_warning(tmpdir): + content = ''' +version: "2" +mkdocs: + fail_on_warning: true + ''' + assertValidConfig(tmpdir, content) + + +@pytest.mark.parametrize('value', ['not true', "''", '[]']) +def test_mkdocs_fail_on_warning_invalid(tmpdir, value): + content = ''' +version: "2" +mkdocs: + fail_on_warning: {value} + ''' + assertInvalidConfig( + tmpdir, + content.format(value=value), + ['is not a bool'] + ) + + def test_submodules_include(tmpdir): content = ''' version: "2" From 40c35a2ce3abf82b32d38536099dcd9ef7b4dc1c Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 29 May 2018 20:10:38 -0500 Subject: [PATCH 19/29] More tests --- .../rtd_tests/fixtures/spec/v2/schema.yml | 3 ++ .../rtd_tests/tests/test_yml_schema.py | 45 +++++++++++++++++-- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml index b44d47cbe4b..d6314bf9795 100644 --- a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml +++ b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml @@ -28,6 +28,7 @@ submodules: include('submodules', required=False) # Redirects for the current version to be built # Key/value list, represent redirects of type `type` # from url -> to url +# Default: null redirects: map(enum('page'), map(str(), str()), required=False) --- @@ -47,9 +48,11 @@ python: version: enum('2', '2.7', '3', '3.5', '3.6', required=False) # The path to the requirements file from the root of the project + # Default: null requirements: path(required=False) # Install the project using python setup.py install or pip + # Default: null install: enum('pip', 'setup.py', required=False) # Extra requirements sections to install in addition to the package dependencies diff --git a/readthedocs/rtd_tests/tests/test_yml_schema.py b/readthedocs/rtd_tests/tests/test_yml_schema.py index 4732fa95482..6f3dace166b 100644 --- a/readthedocs/rtd_tests/tests/test_yml_schema.py +++ b/readthedocs/rtd_tests/tests/test_yml_schema.py @@ -193,7 +193,7 @@ def test_python_version_invalid(tmpdir): ) -def test_no_python_version(tmpdir): +def test_python_version_no_key(tmpdir): content = ''' version: "2" python: @@ -202,7 +202,7 @@ def test_no_python_version(tmpdir): assertValidConfig(tmpdir, content) -def test_valid_requirements(tmpdir): +def test_python_requirements(tmpdir): content = ''' version: "2" python: @@ -211,7 +211,7 @@ def test_valid_requirements(tmpdir): assertValidConfig(tmpdir, content) -def test_invalid_requirements_file(tmpdir): +def test_python_requirements_invalid(tmpdir): content = ''' version: "2" python: @@ -224,6 +224,15 @@ def test_invalid_requirements_file(tmpdir): ) +def test_python_requirements_null(tmpdir): + content = ''' +version: "2" +python: + requirements: null + ''' + assertValidConfig(tmpdir, content) + + @pytest.mark.parametrize('value', ['pip', 'setup.py']) def test_python_install(tmpdir, value): content = ''' @@ -248,6 +257,15 @@ def test_python_install_invalid(tmpdir): ) +def test_python_install_null(tmpdir): + content = ''' +version: "2" +python: + install: null + ''' + assertValidConfig(tmpdir, content) + + def test_python_extra_requirements(tmpdir): content = ''' version: "2" @@ -274,6 +292,16 @@ def test_python_extra_requirements_invalid(tmpdir): ) +@pytest.mark.parametrize('value', ['', 'null', '[]']) +def test_python_extra_requirements_empty(tmpdir, value): + content = ''' +version: "2" +python: + extra_requirements: {value} + ''' + assertValidConfig(tmpdir, content.format(value=value)) + + @pytest.mark.parametrize('value', ['true', 'false']) def test_python_system_packages(tmpdir, value): content = ''' @@ -462,7 +490,7 @@ def test_redirects(tmpdir): assertValidConfig(tmpdir, content) -def test_invalid_redirects(tmpdir): +def test_redirects_invalid(tmpdir): content = ''' version: "2" redirects: @@ -474,3 +502,12 @@ def test_invalid_redirects(tmpdir): content, ['is not a str'] ) + + +@pytest.mark.parametrize('value', ['', 'null', '{}']) +def test_redirects_empty(tmpdir, value): + content = ''' +version: "2" +redirects: {value} + ''' + assertValidConfig(tmpdir, content.format(value=value)) From ace5f8ebf7018788b615e06cd1e0b4552392ac9f Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 30 May 2018 17:12:58 -0500 Subject: [PATCH 20/29] Better names for tests --- readthedocs/rtd_tests/tests/test_yml_schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_yml_schema.py b/readthedocs/rtd_tests/tests/test_yml_schema.py index 6f3dace166b..f42af3aa477 100644 --- a/readthedocs/rtd_tests/tests/test_yml_schema.py +++ b/readthedocs/rtd_tests/tests/test_yml_schema.py @@ -344,7 +344,7 @@ def test_sphinx_default_value(tmpdir): assertValidConfig(tmpdir, content) -@pytest.mark.parametrize('value', ['2', 'environment.py']) +@pytest.mark.parametrize('value', ['2', 'non-existent-file.yml']) def test_sphinx_invalid(tmpdir, value): content = ''' version: "2" @@ -399,7 +399,7 @@ def test_mkdocs_default_value(tmpdir): assertValidConfig(tmpdir, content) -@pytest.mark.parametrize('value', ['2', 'environment.py']) +@pytest.mark.parametrize('value', ['2', 'non-existent-file.yml']) def test_mkdocs_invalid(tmpdir, value): content = ''' version: "2" From 25e962d49262c94f3ef46103fec6fd46f11d1b22 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Wed, 30 May 2018 17:16:00 -0500 Subject: [PATCH 21/29] Update gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 6fc971f5ca5..6442e66e0b1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ *.db +*.rdb *.egg-info *.log *.pyc From 1c3a2fa4c93ebd3e3113516a3834c7512a74b87c Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 31 May 2018 16:47:24 -0500 Subject: [PATCH 22/29] Add validations for unique keys --- .../rtd_tests/fixtures/spec/v2/schema.yml | 2 +- .../rtd_tests/tests/test_yml_schema.py | 35 ++++++ readthedocs/rtdyml/__init__.py | 103 ++++++++++++++++++ 3 files changed, 139 insertions(+), 1 deletion(-) diff --git a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml index d6314bf9795..bac894fbbfe 100644 --- a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml +++ b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml @@ -44,7 +44,7 @@ build: python: # The Python version - # Default: '2.7' + # Default: '3.6' version: enum('2', '2.7', '3', '3.5', '3.6', required=False) # The path to the requirements file from the root of the project diff --git a/readthedocs/rtd_tests/tests/test_yml_schema.py b/readthedocs/rtd_tests/tests/test_yml_schema.py index f42af3aa477..b5ea4259487 100644 --- a/readthedocs/rtd_tests/tests/test_yml_schema.py +++ b/readthedocs/rtd_tests/tests/test_yml_schema.py @@ -344,6 +344,24 @@ def test_sphinx_default_value(tmpdir): assertValidConfig(tmpdir, content) +def test_sphinx_and_mkdocs_invalid(tmpdir): + content = ''' +version: "2" +sphinx: + configuration: docs/conf.py +mkdocs: + configuration: mkdocs.yml + ''' + assertInvalidConfig( + tmpdir, + content, + [ + 'Documentation type can not have', + "['sphinx', 'mkdocs'] at the same time" + ] + ) + + @pytest.mark.parametrize('value', ['2', 'non-existent-file.yml']) def test_sphinx_invalid(tmpdir, value): content = ''' @@ -480,6 +498,23 @@ def test_submodules_exclude_all(tmpdir): assertValidConfig(tmpdir, content) +def test_submodules_exclude_and_include_invalid(tmpdir): + content = ''' +version: "2" +submodules: + exclude: all + include: all + ''' + assertInvalidConfig( + tmpdir, + content, + [ + 'Submodules can not have', + "['submodules.exclude', 'submodules.include'] at the same time" + ] + ) + + def test_redirects(tmpdir): content = ''' version: "2" diff --git a/readthedocs/rtdyml/__init__.py b/readthedocs/rtdyml/__init__.py index f2151be15fd..8aca11c4a57 100644 --- a/readthedocs/rtdyml/__init__.py +++ b/readthedocs/rtdyml/__init__.py @@ -12,6 +12,34 @@ '../rtd_tests/fixtures/spec/v2/schema.yml' ) +ALL = 'all' + + +def flatten(dic, keep_key=False, position=None): + """ + Returns a flattened dictionary from a dictionary of nested dictionaries. + """ + child = {} + + for k, v in dic.items(): + if isinstance(k, six.string_types): + k = k.replace('.', '_') + if position: + item_position = '{}.{}'.format(position, k) + else: + item_position = k + + if isinstance(v, dict): + child.update( + flatten(dic[k], keep_key, item_position) + ) + if keep_key: + child[item_position] = v + else: + child[item_position] = v + + return child + class PathValidator(Validator): @@ -43,6 +71,7 @@ class BuildConfig(object): def __init__(self, configuration_file): self.configuration_file = configuration_file self.data = yamale.make_data(self.configuration_file) + self.defaults = self._get_defaults() self.schema = yamale.make_schema( V2_SCHEMA, validators=self._get_validators() @@ -54,6 +83,80 @@ def _get_validators(self): validators[PathValidator.tag] = PathValidator return validators + def _get_defaults(self): + defaults = { + 'formats': [], + 'conda': None, + 'build': { + 'image': '2.0', + }, + 'python': { + 'version': '3.6', + 'requirements': None, + 'install': None, + 'extra_requirements': [], + 'system_packages': False, + }, + 'sphinx': { + 'configuration': None, + 'fail_on_warning': False, + }, + 'mkdocs': { + 'configuration': None, + 'fail_on_warning': False, + }, + 'submodules': { + 'include': [], + 'exclude': [], + 'recursive': False, + }, + 'redirects': None, + } + return flatten(defaults, keep_key=True) + + def check_constraints(self): + """ + Check constraints between keys, such as relations of uniquiness. + Also set default values. + """ + constraints = { + 'Documentation type': { + 'unique': ['sphinx', 'mkdocs'], + 'default': 'sphinx', + }, + 'Submodules': { + 'unique': ['submodules.include', 'submodules.exclude'], + 'default': 'submodules.include', + } + } + + for subject, constraint in constraints.items(): + present_keys = [ + key + for key in constraint['unique'] + if key in self.data[0] + ] + default_key = constraint.get('default') + if not present_keys and default_key: + self.data[0][default_key] = self.defaults[default_key] + elif len(present_keys) > 1: + raise ValueError( + '{subject} can not have {keys} at the same time'.format( + subject=subject, + keys=constraint['unique'], + ) + ) + + def set_defaults(self): + """ + Set default values to the currently processed data. + """ + for k, v in self.defaults.items(): + if k not in self.data[0]: + self.data[0][k] = v + def validate(self): """Validate the current configuration file.""" + self.check_constraints() + self.set_defaults() return yamale.validate(self.schema, self.data) From b63b0168be3f58e57ddbdc1f140635faa0ffcc15 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Thu, 31 May 2018 17:12:56 -0500 Subject: [PATCH 23/29] Linter --- readthedocs/rtdyml/__init__.py | 99 ++++++++++++++++------------------ 1 file changed, 47 insertions(+), 52 deletions(-) diff --git a/readthedocs/rtdyml/__init__.py b/readthedocs/rtdyml/__init__.py index 8aca11c4a57..fbdd9fe2b0e 100644 --- a/readthedocs/rtdyml/__init__.py +++ b/readthedocs/rtdyml/__init__.py @@ -14,29 +14,56 @@ ALL = 'all' +DEFAULT_VALUES = { + 'formats': [], + 'conda': None, + 'build': { + 'image': '2.0', + }, + 'python': { + 'version': '3.6', + 'requirements': None, + 'install': None, + 'extra_requirements': [], + 'system_packages': False, + }, + 'sphinx': { + 'configuration': None, + 'fail_on_warning': False, + }, + 'mkdocs': { + 'configuration': None, + 'fail_on_warning': False, + }, + 'submodules': { + 'include': [], + 'exclude': [], + 'recursive': False, + }, + 'redirects': None, +} + def flatten(dic, keep_key=False, position=None): - """ - Returns a flattened dictionary from a dictionary of nested dictionaries. - """ + """Returns a flattened dictionary from a dictionary of nested dictionaries.""" child = {} - for k, v in dic.items(): - if isinstance(k, six.string_types): - k = k.replace('.', '_') + for key, value in dic.items(): + if isinstance(key, six.string_types): + key = key.replace('.', '_') if position: - item_position = '{}.{}'.format(position, k) + item_position = '{}.{}'.format(position, key) else: - item_position = k + item_position = key - if isinstance(v, dict): + if isinstance(value, dict): child.update( - flatten(dic[k], keep_key, item_position) + flatten(dic[key], keep_key, item_position) ) if keep_key: - child[item_position] = v + child[item_position] = value else: - child[item_position] = v + child[item_position] = value return child @@ -71,7 +98,7 @@ class BuildConfig(object): def __init__(self, configuration_file): self.configuration_file = configuration_file self.data = yamale.make_data(self.configuration_file) - self.defaults = self._get_defaults() + self.defaults = flatten(DEFAULT_VALUES, keep_key=True) self.schema = yamale.make_schema( V2_SCHEMA, validators=self._get_validators() @@ -83,41 +110,11 @@ def _get_validators(self): validators[PathValidator.tag] = PathValidator return validators - def _get_defaults(self): - defaults = { - 'formats': [], - 'conda': None, - 'build': { - 'image': '2.0', - }, - 'python': { - 'version': '3.6', - 'requirements': None, - 'install': None, - 'extra_requirements': [], - 'system_packages': False, - }, - 'sphinx': { - 'configuration': None, - 'fail_on_warning': False, - }, - 'mkdocs': { - 'configuration': None, - 'fail_on_warning': False, - }, - 'submodules': { - 'include': [], - 'exclude': [], - 'recursive': False, - }, - 'redirects': None, - } - return flatten(defaults, keep_key=True) - def check_constraints(self): """ - Check constraints between keys, such as relations of uniquiness. - Also set default values. + Check constraints between keys. + + Such as relations of uniquiness and set default values for those. """ constraints = { 'Documentation type': { @@ -148,12 +145,10 @@ def check_constraints(self): ) def set_defaults(self): - """ - Set default values to the currently processed data. - """ - for k, v in self.defaults.items(): - if k not in self.data[0]: - self.data[0][k] = v + """Set default values to the currently processed data.""" + for key, value in self.defaults.items(): + if key not in self.data[0]: + self.data[0][key] = value def validate(self): """Validate the current configuration file.""" From 18a6485c8994850fa28ea54de395eee6e07b3049 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 1 Jun 2018 10:16:38 -0500 Subject: [PATCH 24/29] Isort --- readthedocs/rtdyml/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/readthedocs/rtdyml/__init__.py b/readthedocs/rtdyml/__init__.py index fbdd9fe2b0e..9548ffa68f9 100644 --- a/readthedocs/rtdyml/__init__.py +++ b/readthedocs/rtdyml/__init__.py @@ -1,9 +1,10 @@ """Validator for the RTD configuration file.""" +from __future__ import division, print_function, unicode_literals + from os import path import six - import yamale from yamale.validators import DefaultValidators, Validator From dc60f11758cfd670a56d5bb6239e4d66e1e4d965 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 1 Jun 2018 10:50:47 -0500 Subject: [PATCH 25/29] Rename --- .../{rtdyml => buildconfig}/__init__.py | 24 ++++++++++++++++++- ...est_yml_schema.py => test_build_config.py} | 2 +- 2 files changed, 24 insertions(+), 2 deletions(-) rename readthedocs/{rtdyml => buildconfig}/__init__.py (91%) rename readthedocs/rtd_tests/tests/{test_yml_schema.py => test_build_config.py} (99%) diff --git a/readthedocs/rtdyml/__init__.py b/readthedocs/buildconfig/__init__.py similarity index 91% rename from readthedocs/rtdyml/__init__.py rename to readthedocs/buildconfig/__init__.py index 9548ffa68f9..5b596c2427a 100644 --- a/readthedocs/rtdyml/__init__.py +++ b/readthedocs/buildconfig/__init__.py @@ -46,7 +46,29 @@ def flatten(dic, keep_key=False, position=None): - """Returns a flattened dictionary from a dictionary of nested dictionaries.""" + """ + Returns a flattened dictionary + + Given a dictionary of nested dictionaries, it returns + a dictionary with all nested keys at the top level. + + For example + + { + 'key': 'value', + 'nested': { + 'subkey': 'value' + }, + } + + Becomes + + { + 'key': value, + 'nested': ..., + 'nested.subkey': 'value' + } + """ child = {} for key, value in dic.items(): diff --git a/readthedocs/rtd_tests/tests/test_yml_schema.py b/readthedocs/rtd_tests/tests/test_build_config.py similarity index 99% rename from readthedocs/rtd_tests/tests/test_yml_schema.py rename to readthedocs/rtd_tests/tests/test_build_config.py index b5ea4259487..e4ed30cf63d 100644 --- a/readthedocs/rtd_tests/tests/test_yml_schema.py +++ b/readthedocs/rtd_tests/tests/test_build_config.py @@ -1,7 +1,7 @@ from os import path import pytest -from readthedocs.rtdyml import BuildConfig +from readthedocs.buildconfig import BuildConfig from readthedocs_build.testing import utils From 79ebf87494e72319f0815bebda47a1f4800fb0db Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 1 Jun 2018 11:23:27 -0500 Subject: [PATCH 26/29] Extend build class --- readthedocs/buildconfig/__init__.py | 39 +++++++++++++++++------------ 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/readthedocs/buildconfig/__init__.py b/readthedocs/buildconfig/__init__.py index 5b596c2427a..1a0b2a89a23 100644 --- a/readthedocs/buildconfig/__init__.py +++ b/readthedocs/buildconfig/__init__.py @@ -44,6 +44,17 @@ 'redirects': None, } +CONSTRAINTS = { + 'Documentation type': { + 'unique': ['sphinx', 'mkdocs'], + 'default': 'sphinx', + }, + 'Submodules': { + 'unique': ['submodules.include', 'submodules.exclude'], + 'default': 'submodules.include', + } +} + def flatten(dic, keep_key=False, position=None): """ @@ -65,7 +76,7 @@ def flatten(dic, keep_key=False, position=None): { 'key': value, - 'nested': ..., + 'nested': {...}, 'nested.subkey': 'value' } """ @@ -118,16 +129,22 @@ class BuildConfig(object): """Wrapper object to validate to configuration file.""" + _schema = V2_SCHEMA + _default_values = DEFAULT_VALUES + _constraints = CONSTRAINTS + def __init__(self, configuration_file): self.configuration_file = configuration_file self.data = yamale.make_data(self.configuration_file) - self.defaults = flatten(DEFAULT_VALUES, keep_key=True) + self.defaults = flatten(self._default_values, keep_key=True) + self.constraints = self._constraints self.schema = yamale.make_schema( - V2_SCHEMA, + self._schema, validators=self._get_validators() ) def _get_validators(self): + """Custom validators for yamale.""" validators = DefaultValidators.copy() PathValidator.configuration_file = self.configuration_file validators[PathValidator.tag] = PathValidator @@ -139,18 +156,7 @@ def check_constraints(self): Such as relations of uniquiness and set default values for those. """ - constraints = { - 'Documentation type': { - 'unique': ['sphinx', 'mkdocs'], - 'default': 'sphinx', - }, - 'Submodules': { - 'unique': ['submodules.include', 'submodules.exclude'], - 'default': 'submodules.include', - } - } - - for subject, constraint in constraints.items(): + for subject, constraint in self.constraints.items(): present_keys = [ key for key in constraint['unique'] @@ -177,4 +183,5 @@ def validate(self): """Validate the current configuration file.""" self.check_constraints() self.set_defaults() - return yamale.validate(self.schema, self.data) + yamale.validate(self.schema, self.data) + return self.data[0] From 6286a349a4f14804901fa8cb799e1b519d1ad586 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 1 Jun 2018 13:03:15 -0500 Subject: [PATCH 27/29] Just do unit test with yamale --- readthedocs/buildconfig/__init__.py | 187 ------------------ .../rtd_tests/tests/test_build_config.py | 87 ++++---- requirements/pip.txt | 1 - requirements/testing.txt | 1 + 4 files changed, 49 insertions(+), 227 deletions(-) delete mode 100644 readthedocs/buildconfig/__init__.py diff --git a/readthedocs/buildconfig/__init__.py b/readthedocs/buildconfig/__init__.py deleted file mode 100644 index 1a0b2a89a23..00000000000 --- a/readthedocs/buildconfig/__init__.py +++ /dev/null @@ -1,187 +0,0 @@ -"""Validator for the RTD configuration file.""" - -from __future__ import division, print_function, unicode_literals - -from os import path - -import six -import yamale -from yamale.validators import DefaultValidators, Validator - -V2_SCHEMA = path.join( - path.dirname(__file__), - '../rtd_tests/fixtures/spec/v2/schema.yml' -) - -ALL = 'all' - -DEFAULT_VALUES = { - 'formats': [], - 'conda': None, - 'build': { - 'image': '2.0', - }, - 'python': { - 'version': '3.6', - 'requirements': None, - 'install': None, - 'extra_requirements': [], - 'system_packages': False, - }, - 'sphinx': { - 'configuration': None, - 'fail_on_warning': False, - }, - 'mkdocs': { - 'configuration': None, - 'fail_on_warning': False, - }, - 'submodules': { - 'include': [], - 'exclude': [], - 'recursive': False, - }, - 'redirects': None, -} - -CONSTRAINTS = { - 'Documentation type': { - 'unique': ['sphinx', 'mkdocs'], - 'default': 'sphinx', - }, - 'Submodules': { - 'unique': ['submodules.include', 'submodules.exclude'], - 'default': 'submodules.include', - } -} - - -def flatten(dic, keep_key=False, position=None): - """ - Returns a flattened dictionary - - Given a dictionary of nested dictionaries, it returns - a dictionary with all nested keys at the top level. - - For example - - { - 'key': 'value', - 'nested': { - 'subkey': 'value' - }, - } - - Becomes - - { - 'key': value, - 'nested': {...}, - 'nested.subkey': 'value' - } - """ - child = {} - - for key, value in dic.items(): - if isinstance(key, six.string_types): - key = key.replace('.', '_') - if position: - item_position = '{}.{}'.format(position, key) - else: - item_position = key - - if isinstance(value, dict): - child.update( - flatten(dic[key], keep_key, item_position) - ) - if keep_key: - child[item_position] = value - else: - child[item_position] = value - - return child - - -class PathValidator(Validator): - - """ - Path validator - - Checks if the given value is a string and a existing - file. - """ - - tag = 'path' - constraints = [] - configuration_file = '.' - - def _is_valid(self, value): - if isinstance(value, six.string_types): - file_ = path.join( - path.dirname(self.configuration_file), - value - ) - return path.exists(file_) - return False - - -class BuildConfig(object): - - """Wrapper object to validate to configuration file.""" - - _schema = V2_SCHEMA - _default_values = DEFAULT_VALUES - _constraints = CONSTRAINTS - - def __init__(self, configuration_file): - self.configuration_file = configuration_file - self.data = yamale.make_data(self.configuration_file) - self.defaults = flatten(self._default_values, keep_key=True) - self.constraints = self._constraints - self.schema = yamale.make_schema( - self._schema, - validators=self._get_validators() - ) - - def _get_validators(self): - """Custom validators for yamale.""" - validators = DefaultValidators.copy() - PathValidator.configuration_file = self.configuration_file - validators[PathValidator.tag] = PathValidator - return validators - - def check_constraints(self): - """ - Check constraints between keys. - - Such as relations of uniquiness and set default values for those. - """ - for subject, constraint in self.constraints.items(): - present_keys = [ - key - for key in constraint['unique'] - if key in self.data[0] - ] - default_key = constraint.get('default') - if not present_keys and default_key: - self.data[0][default_key] = self.defaults[default_key] - elif len(present_keys) > 1: - raise ValueError( - '{subject} can not have {keys} at the same time'.format( - subject=subject, - keys=constraint['unique'], - ) - ) - - def set_defaults(self): - """Set default values to the currently processed data.""" - for key, value in self.defaults.items(): - if key not in self.data[0]: - self.data[0][key] = value - - def validate(self): - """Validate the current configuration file.""" - self.check_constraints() - self.set_defaults() - yamale.validate(self.schema, self.data) - return self.data[0] diff --git a/readthedocs/rtd_tests/tests/test_build_config.py b/readthedocs/rtd_tests/tests/test_build_config.py index e4ed30cf63d..5d1a95a2de7 100644 --- a/readthedocs/rtd_tests/tests/test_build_config.py +++ b/readthedocs/rtd_tests/tests/test_build_config.py @@ -1,9 +1,41 @@ from os import path import pytest -from readthedocs.buildconfig import BuildConfig +import six from readthedocs_build.testing import utils +import yamale +from yamale.validators import DefaultValidators, Validator + + +V2_SCHEMA = path.join( + path.dirname(__file__), + '../fixtures/spec/v2/schema.yml' +) + + +class PathValidator(Validator): + + """ + Path validator + + Checks if the given value is a string and a existing + file. + """ + + tag = 'path' + constraints = [] + configuration_file = '.' + + def _is_valid(self, value): + if isinstance(value, six.string_types): + file_ = path.join( + path.dirname(self.configuration_file), + value + ) + return path.exists(file_) + return False + def create_yaml(tmpdir, content): fs = { @@ -19,16 +51,28 @@ def create_yaml(tmpdir, content): return path.join(tmpdir.strpath, 'rtd.yml') +def validate_schema(file): + validators = DefaultValidators.copy() + PathValidator.configuration_file = file + validators[PathValidator.tag] = PathValidator + + data = yamale.make_data(file) + schema = yamale.make_schema( + V2_SCHEMA, + validators=validators + ) + yamale.validate(schema, data) + + def assertValidConfig(tmpdir, content): file = create_yaml(tmpdir, content) - build = BuildConfig(file) - build.validate() + validate_schema(file) def assertInvalidConfig(tmpdir, content, msgs=()): file = create_yaml(tmpdir, content) with pytest.raises(ValueError) as excinfo: - BuildConfig(file).validate() + validate_schema(file) for msg in msgs: msg in str(excinfo.value) @@ -344,24 +388,6 @@ def test_sphinx_default_value(tmpdir): assertValidConfig(tmpdir, content) -def test_sphinx_and_mkdocs_invalid(tmpdir): - content = ''' -version: "2" -sphinx: - configuration: docs/conf.py -mkdocs: - configuration: mkdocs.yml - ''' - assertInvalidConfig( - tmpdir, - content, - [ - 'Documentation type can not have', - "['sphinx', 'mkdocs'] at the same time" - ] - ) - - @pytest.mark.parametrize('value', ['2', 'non-existent-file.yml']) def test_sphinx_invalid(tmpdir, value): content = ''' @@ -498,23 +524,6 @@ def test_submodules_exclude_all(tmpdir): assertValidConfig(tmpdir, content) -def test_submodules_exclude_and_include_invalid(tmpdir): - content = ''' -version: "2" -submodules: - exclude: all - include: all - ''' - assertInvalidConfig( - tmpdir, - content, - [ - 'Submodules can not have', - "['submodules.exclude', 'submodules.include'] at the same time" - ] - ) - - def test_redirects(tmpdir): content = ''' version: "2" diff --git a/requirements/pip.txt b/requirements/pip.txt index 8a227cdd035..026c570d97c 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -64,7 +64,6 @@ Unipath==1.1 django-kombu==0.9.4 mimeparse==0.1.3 mock==2.0.0 -yamale==1.7.0 # stripe 1.20.2 is the latest compatible with our code base (otherwise # gold/tests/test_forms.py fails) diff --git a/requirements/testing.txt b/requirements/testing.txt index 110106df658..7d801c59567 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -8,6 +8,7 @@ pytest-xdist==1.22.0 apipkg==1.4 execnet==1.5.0 Mercurial==4.4.2 +yamale==1.7.0 # local debugging tools datadiff From 31fc80064d10dec801060f9e48965f72fe9e6547 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 1 Jun 2018 13:12:39 -0500 Subject: [PATCH 28/29] Change values from spec --- readthedocs/rtd_tests/fixtures/spec/v2/schema.yml | 10 +++++----- readthedocs/rtd_tests/tests/test_build_config.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml index bac894fbbfe..6570a895cd2 100644 --- a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml +++ b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml @@ -39,13 +39,13 @@ conda: build: # The build docker image to be used - # Default: '2.0' - image: enum('1.0', '2.0', 'latest', required=False) + # Default: 'latest' + image: enum('stable', 'latest', required=False) python: - # The Python version - # Default: '3.6' - version: enum('2', '2.7', '3', '3.5', '3.6', required=False) + # The Python version (this depends on the build image) + # Default: '3' + version: enum('2', '2.7', '3', '3.3', '3.4', '3.5', '3.6', required=False) # The path to the requirements file from the root of the project # Default: null diff --git a/readthedocs/rtd_tests/tests/test_build_config.py b/readthedocs/rtd_tests/tests/test_build_config.py index 5d1a95a2de7..c9c40f53ab7 100644 --- a/readthedocs/rtd_tests/tests/test_build_config.py +++ b/readthedocs/rtd_tests/tests/test_build_config.py @@ -182,7 +182,7 @@ def test_conda_missing_key(tmpdir): ) -@pytest.mark.parametrize('value', ['1.0', '2.0', 'latest']) +@pytest.mark.parametrize('value', ['stable', 'latest']) def test_build(tmpdir, value): content = ''' version: "2" From 84d1326661b48fa33bbefd0c04cabda4c6b7872d Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Fri, 1 Jun 2018 13:17:26 -0500 Subject: [PATCH 29/29] Isort --- readthedocs/rtd_tests/tests/test_build_config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_build_config.py b/readthedocs/rtd_tests/tests/test_build_config.py index c9c40f53ab7..73419c5cf48 100644 --- a/readthedocs/rtd_tests/tests/test_build_config.py +++ b/readthedocs/rtd_tests/tests/test_build_config.py @@ -1,13 +1,13 @@ +from __future__ import division, print_function, unicode_literals + from os import path import pytest import six -from readthedocs_build.testing import utils - import yamale +from readthedocs_build.testing import utils from yamale.validators import DefaultValidators, Validator - V2_SCHEMA = path.join( path.dirname(__file__), '../fixtures/spec/v2/schema.yml'