WIP for en#81 template macros and page filtering #152

WARNING:
- 1 failing test in jupman_tools_test.py (test_zip_folder_test_chapter)
- many failing tests in preamble_test.py

related to gitignore #152:

- initializing sphinx ignored files from gitignore using igittigitt
- now supporting sphinx include_patterns
- jupyter-example: added data/trial.json
- created specs and mockups under _test/specs
- zip_ignored holds now patterns relative to project root instead of zip root for uniformity

- now building also stuff in _test/ by default
- added jupman_tools _sphinx_filter and _sphinx_filter_cxt temporary functions
- added jcxt.jpre_subroot param to JupmanContext (needed for proper ignoring when copying trees around)
- created _test/integration_test.py for some gitignore test
- build.py now handles case when no index is present (useful when debugging)

deprecated:
- chapter_patterns, trying to set it different from */ gives a warning

new deps:
- upgraded to Sphinx 5.1.1 from 4.5.0 (to support include_patterns)
- Using igittigitt 2.1.2 to load .gitignore, since it doesn't support negated patterns:
    - challenge-example is no longer published (never liked it anyway)
    - READMEs in private dirs are no longer published  (never liked them anyway)
- for simplicity, requiring test deps also in main build (probably we will
    need them anyway)

.gitignore

@@ -1,26 +1,20 @@
 # important! challenges are *not* versioned
-*_chal_sol.py
-*-chal-sol.ipynb
-*_chal_sol.ipynb  # shouldn't be allowed but just in case
-*-chal-sol.md
-!challenge-example/example-chal-sol.ipynb
-!_test/test-chapter/my_chal_sol.py
-!_test/test-chapter/nb-chal-sol.ipynb
+
+*-chal-sol.*
+*_chal_sol.*
+
 
 _build/
-_private/*
+_private/
 
-/_templates/exam/server/*
-!_templates/exam/server/README.md
+/_templates/exam/server/
 
 #automatically generated by conf.py during sphinx build
-/_static/generated/*
-!/_static/generated/README.md
+/_static/generated/
 
 .cache
 .hypothesis
 
-
 ##### PYTHON
 *.pyc
 __pycache__

_test/chapter_test.py

@@ -33,7 +33,7 @@ def dest_dir():
 def test_copy_chapter_replacements(dest_dir):
 
     replacements_fn = os.path.join(dest_dir, 'replacements.ipynb')
-    jcxt = JupmanContext(make_sphinx_config(), replacements_fn, False)
+    jcxt = JupmanContext(make_sphinx_config(), replacements_fn, False, '')
 
     assert os.path.isfile(replacements_fn)
 
@@ -97,7 +97,7 @@ def test_copy_chapter_replacements(dest_dir):
 def test_copy_chapter_py_files(dest_dir):
 
     py_fn = os.path.join(dest_dir, 'script.py')
-    jcxt = JupmanContext(make_sphinx_config(), py_fn, False)
+    jcxt = JupmanContext(make_sphinx_config(), py_fn, False, '')
     assert os.path.isfile(py_fn)
 
     with open(py_fn, encoding='utf-8') as py_f:
@@ -110,7 +110,7 @@ def test_copy_chapter_py_files(dest_dir):
 
 
     test_fn = os.path.join(dest_dir, 'some_test.py')
-    jcxt = JupmanContext(make_sphinx_config(), test_fn, False)
+    jcxt = JupmanContext(make_sphinx_config(), test_fn, False, '')
     assert os.path.isfile(test_fn)
 
     with open(test_fn, encoding='utf-8') as test_f:
@@ -123,7 +123,7 @@ def test_copy_chapter_py_files(dest_dir):
         assert jcxt.author in test_code        
 
     sol_fn = os.path.join(dest_dir, 'some_sol.py')
-    jcxt = JupmanContext(make_sphinx_config(), sol_fn, False)
+    jcxt = JupmanContext(make_sphinx_config(), sol_fn, False, '')
     assert os.path.isfile(sol_fn)
 
     with open(sol_fn, encoding='utf-8') as py_sol_f:
@@ -156,7 +156,7 @@ def test_copy_chapter_py_files(dest_dir):
 def test_copy_chapter_exercises(dest_dir):
 
     nb_ex_fn = os.path.join(dest_dir, 'nb.ipynb')
-    jcxt = JupmanContext(make_sphinx_config(), nb_ex_fn, False)
+    jcxt = JupmanContext(make_sphinx_config(), nb_ex_fn, False, '')
 
     assert os.path.isfile(nb_ex_fn)
 
@@ -219,7 +219,7 @@ def test_copy_chapter_exercises(dest_dir):
 def test_copy_chapter_solution(dest_dir):
 
     nb_sol_fn = os.path.join(dest_dir, 'nb-sol.ipynb')
-    jcxt = JupmanContext(make_sphinx_config(), nb_sol_fn, False)
+    jcxt = JupmanContext(make_sphinx_config(), nb_sol_fn, False, '')
 
     nb_sol = nbformat.read(nb_sol_fn, nbformat.NO_CONVERT) 
     assert 'stripped!' in nb_sol.cells[8].source   # jupman-strip  strips everything inside exercises
@@ -270,7 +270,7 @@ def test_chapter_solution_web():
     nb_sol_fn = os.path.join('_test/test-chapter', 'nb-sol.ipynb')
     nb_sol_web = nbformat.read(nb_sol_fn, nbformat.NO_CONVERT)
 
-    jcxt = JupmanContext(make_sphinx_config(), os.path.abspath(nb_sol_fn), True)
+    jcxt = JupmanContext(make_sphinx_config(), os.path.abspath(nb_sol_fn), True, '')
 
     jmt._sol_nb_to_ex(jcxt, nb_sol_web)
 
@@ -299,9 +299,8 @@ def test_chapter_solution_web():
         if cell.source == """x = 14\nimport jupman""":
             import_jupman += 1
 
-        debug(f'HELLO*************')
+
         if 'outputs' in cell and len(cell.outputs) > 0 and  'data' in cell.outputs[0]:
-            debug(f'BELLO*************\n{cell.outputs[0]}')
             if '<script src="../../_static/js/pytutor-embed.bundle.min.js' in cell.outputs[0]['data']['text/html']:
                 pytutor_js += 1
 
@@ -341,7 +340,7 @@ def test_copy_chapter_challenge(dest_dir):
 
 
     nb_chal_ex_fn = os.path.join(dest_dir, 'nb2-chal.ipynb')    
-    jcxt = JupmanContext(make_sphinx_config(), os.path.abspath(nb_chal_ex_fn), True)
+    jcxt = JupmanContext(make_sphinx_config(), os.path.abspath(nb_chal_ex_fn), True, '')
     assert os.path.isfile(nb_chal_ex_fn)
     nb_chal_sol_fn = os.path.join(dest_dir, 'nb2-chal-sol.ipynb')
     assert not os.path.isfile(nb_chal_sol_fn)

_test/common_test.py

@@ -41,14 +41,41 @@ def make_sphinx_config() -> dict:
     sc = {}
     sc['author'] = 'People That Write a Lot'
     sc['jm'] = make_jm()
+    #sc['include_patterns'] = ['**']
+    #sc['exclude_patterns'] = []
     return sc
 
 
 def make_jupman_context() -> JupmanContext:
     sc = make_sphinx_config()
-    return JupmanContext(sc, '_private/test', False)
-
-
+    return JupmanContext(sc, '_private/test', False, '')
+
+def make_jcxt_gitignore_non_existing():
+    """
+        @since 3.6
+    """
+    jcxt = make_jupman_context()
+
+    jmt.init_exclude_patterns(jcxt.jm, jcxt.exclude_patterns, gitignore_path='flying-pig.gitignore')
+    return jcxt
+
+def make_jcxt_gitignored():
+    """
+        @since 3.6
+    """
+    jcxt = make_jupman_context()
+    jmt.init_exclude_patterns(jcxt.jm, jcxt.exclude_patterns)
+    return jcxt
+
+def make_jcxt_zip_ignored():
+    """
+        @since 3.6
+    """
+    jcxt = make_jcxt_gitignored()
+    jcxt.jm.zip_ignored.extend(['**/big-*'])
+    return jcxt
+
+
 @pytest.fixture
 def tconf():
     clean()

_test/integration_test.py

@@ -0,0 +1,113 @@
+import sys
+sys.path.append('../')
+sys.path.append('.')  # good lord, without this debugging in VSCode doesn't work
+
+#keep it first so we don't get deprecation warnings
+import jupman_tools as jmt
+
+from hypothesis import given
+from pprint import pprint
+from hypothesis.strategies import text
+from jupman_tools import ignore_spaces, tag_regex, JupmanConfig, SphinxConfig, JupmanContext, JupmanError, JupmanNotFoundError, FileKinds, JupmanPreprocessorError, JupmanEmptyChapterError, JupmanUnsupportedError
+import pytest 
+import re
+from sphinx.application import Sphinx
+import os
+import nbformat
+import time
+import filecmp
+import shutil
+import nbsphinx
+import inspect
+from jupman_tools import debug    
+from pprint import pformat
+
+from common_test import clean, make_jupman_context, make_sphinx_config, make_jcxt_gitignore_non_existing, make_jcxt_gitignored, make_jcxt_zip_ignored, make_jm, make_nb_resources,  tconf
+import datetime
+
+class MockSphinx:
+    def add_config_value(self, a,b,c):
+        pass
+    def add_transform(self, a):
+        pass
+    def add_javascript(self, a):
+        pass
+    def add_stylesheet(self, a):
+        pass
+
+def test_setup(tconf):
+    """ This test runs an entire build of jupman itself
+    """
+
+    mockapp = MockSphinx()
+
+    tconf.setup(mockapp)
+
+    # so tests run smoothly also on non-jupman projects
+    if os.path.exists('jupyter-example'):
+        assert os.path.isfile(os.path.join(tconf.jm.generated, 'jupyter-example.zip'))        
+    if os.path.exists('python-example'):
+        assert os.path.isfile(os.path.join(tconf.jm.generated, 'python-example.zip'))
+    if os.path.exists('jup-and-py-example'):
+        assert os.path.isfile(os.path.join(tconf.jm.generated, 'jup-and-py-example.zip'))
+    if os.path.exists('challenge-example'):
+        assert os.path.isfile(os.path.join(tconf.jm.generated, 'challenge-example.zip'))
+
+    # test reproducible build zips  https://github.com/DavidLeoni/jupman/issues/60
+
+    if os.path.exists('jup-and-py-example'):
+
+        zpath1 = os.path.join(tconf.jm.generated, 'jup-and-py-example.zip')     
+
+
+        zpath2 = os.path.join(tconf.test_tmp, 'jup-and-py-example.zip')
+
+
+        shutil.copyfile(zpath1, zpath2)        
+        time.sleep(2)
+        tconf.setup(mockapp)
+
+        assert filecmp.cmp(zpath1, zpath2, shallow=False)
+
+
+jcxtfs = [ make_jcxt_gitignore_non_existing, 
+           #make_jcxt_gitignored
+           ]
+@pytest.mark.parametrize("jcxtf", jcxtfs)
+def test_is_zip_ignored_after_exclude_init(jcxtf):
+    """ WARNING: since some files we use here are gitignored, you may not have them on your system
+                 (or when a build is performed on github). Test should 'pass' anyway because
+                 when is_zip_ignored doesn't find an existing file, should return True anyway
+                 
+                 Maybe in the future we may recreate them automatically somehow
+        @since 3.6
+    """
+    debug(f"Creating {jcxtf.__name__}")
+    jcxt = jcxtf()
+
+    cchal = "_test/chap1-complete/c-chal-sol.ipynb"
+    dchal = "_test/chap1-complete/d_chal_sol.py"
+
+    assert jmt.is_zip_ignored(jcxt, jcxt.jm.build)
+    assert jmt.is_zip_ignored(jcxt, jcxt.jm.generated) 
+    assert jmt.is_zip_ignored(jcxt, '_private')
+    assert jmt.is_zip_ignored(jcxt, cchal)
+    assert jmt.is_zip_ignored(jcxt, dchal)
+
+    assert jmt.is_zip_ignored(jcxt, '.git/index')
+    assert jmt.is_zip_ignored(jcxt, '.git/')        
+    assert jmt.is_zip_ignored(jcxt, '.git/info/exclude')    
+    assert jmt.is_zip_ignored(jcxt, '.gitignore')
+    assert jmt.is_zip_ignored(jcxt, '.gitattributes')
+
+    # additional check in case you don't have the files on your system:
+
+    assert jcxt.jm.build in jcxt.exclude_patterns
+    assert jcxt.jm.generated in jcxt.exclude_patterns 
+    assert '_private' in jcxt.exclude_patterns
+    assert "**-chal-sol.*" in jcxt.exclude_patterns
+    assert "**-chal-sol.*" in jcxt.exclude_patterns
+
+    assert '.git' in jcxt.exclude_patterns
+    assert '.gitignore' in jcxt.exclude_patterns
+    assert '.gitattributes' in jcxt.exclude_patterns