Canwaf/issue13 (#219)

* New package names * New projects folder structures * Test deploy public-facing packages to pypi * csvqubed-models changes * `pyproject.toml`s updated * Documentation * READMEs updated * new issues on docs locations * imports, unit tests, behave tests fixed * imports in tests: test_dataclassbase.py, test_resource_serialisation.py * Jenkins updates Co-authored-by: @robons and @canwaf
ONSdigital · Oct 29, 2021 · 2150af1 · 2150af1
1 parent c244e18
commit 2150af1
Show file tree

Hide file tree

Showing 187 changed files with 939 additions and 1,116 deletions.
diff --git a/.gitignore b/.gitignore
@@ -137,11 +137,10 @@ dmypy.json
 # Cython debug symbols
 cython_debug/
 
-#Intellij idea
+# IDE ignores
 .idea
-
 .jython_cache
-
 .vscode
 
+# Ignore docs folders
 docs
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -12,17 +12,17 @@ pipeline {
                     // Clean up any unwanted files lying about after the previous build.
                     sh "git clean -fxd --exclude='.venv'"
 
-                    dir('devtools') {
+                    dir('csvcubed-devtools') {
                         sh 'poetry config virtualenvs.in-project true'
                         sh 'poetry install'
                     }
 
-                    dir('sharedmodels') {
+                    dir('csvcubed-models') {
                         sh 'poetry config virtualenvs.in-project true'
                         sh 'poetry install'
                     }
 
-                    dir('pmd') {
+                    dir('csvcubed-pmd') {
                         sh 'poetry config virtualenvs.in-project true'
                         sh 'poetry install'
                         // Patch behave so that it can output the correct format for the Jenkins cucumber tool.
@@ -31,7 +31,7 @@ pipeline {
                         sh "patch -Nf -d \"${venv_location}/lib/python3.9/site-packages/behave/formatter\" -p1 < /cucumber-format.patch || true"
                     }
 
-                    dir('csvqb') {
+                    dir('csvcubed') {
                         sh 'poetry config virtualenvs.in-project true'
                         sh 'poetry install'
                         // Patch behave so that it can output the correct format for the Jenkins cucumber tool.
@@ -50,40 +50,38 @@ pipeline {
                 }
             }
             steps {
-                    dir('devtools') {
+                    dir('csvcubed-devtools') {
                         sh 'poetry run pyright . --lib'
                     }
 
-                    dir('sharedmodels') {
+                    dir('csvcubed-models') {
                         sh 'poetry run pyright . --lib'
                     }
 
-                    dir('pmd') {
+                    dir('csvcubed-pmd') {
                         sh 'poetry run pyright . --lib'
                     }
 
-                    dir('csvqb') {
+                    dir('csvcubed') {
                         sh 'poetry run pyright . --lib'
                     }
             }
         }
         stage('Test') {
             steps {
-                dir('sharedmodels') {
-                    dir('sharedmodels/tests/unit') {
-                        sh "poetry run pytest --junitxml=pytest_results_sharedmodels.xml"
-                    }
+                dir('csvcubed-models/tests/unit') {
+                    sh "poetry run pytest --junitxml=pytest_results_models.xml"
                 }
-                dir('pmd') {
-                    sh 'poetry run behave pmd/tests/behaviour --tags=-skip -f json.cucumber -o pmd/tests/behaviour/test-results.json'
-                    dir('pmd/tests/unit') {
+                dir('csvcubed-pmd') {
+                    sh 'poetry run behave tests/behaviour --tags=-skip -f json.cucumber -o tests/behaviour/test-results.json'
+                    dir('tests/unit') {
                         sh "poetry run pytest --junitxml=pytest_results_pmd.xml"
                     }
                 }
-                dir('csvqb') {
-                    sh 'poetry run behave csvqb/tests/behaviour --tags=-skip -f json.cucumber -o csvqb/tests/behaviour/test-results.json'
-                    dir('csvqb/tests/unit') {
-                        sh "poetry run pytest --junitxml=pytest_results_csvqb.xml"
+                dir('csvcubed') {
+                    sh 'poetry run behave tests/behaviour --tags=-skip -f json.cucumber -o tests/behaviour/test-results.json'
+                    dir('tests/unit') {
+                        sh "poetry run pytest --junitxml=pytest_results_csvcubed.xml"
                     }
                 }
 
@@ -92,19 +90,19 @@ pipeline {
         }
         stage('Package') {
             steps {
-                dir('devtools') {
+                dir('csvcubed-devtools') {
                     sh 'poetry build'
                 }
 
-                dir('sharedmodels') {
+                dir('csvcubed-models') {
                     sh 'poetry build'
                 }
 
-                dir('pmd') {
+                dir('csvcubed-pmd') {
                     sh 'poetry build'
                 }
 
-                dir('csvqb') {
+                dir('csvcubed') {
                     sh 'poetry build'
                 }
 
@@ -114,23 +112,23 @@ pipeline {
         stage('Documentation') {
             steps {
                 script {
-                    dir('devtools') {
-                        sh "poetry run sphinx-apidoc -F -M -a -P --tocfile index.rst -d 10 -E -o docs --implicit-namespaces -o docs devtools \"setup*\""
+                    dir('csvcubed-devtools') {
+                        sh "poetry run sphinx-apidoc -F -M -a -P --tocfile index.rst -d 10 -E -o docs --implicit-namespaces -o docs csvcubeddevtools \"setup*\""
                         sh 'poetry run sphinx-build -W -b html docs docs/_build/html'
                     }
 
-                    dir('sharedmodels') {
-                        sh "poetry run sphinx-apidoc -F -M -a -P --tocfile index.rst -d 10 -E --implicit-namespaces -o docs sharedmodels \"setup*\" \"sharedmodels/scripts\" \"sharedmodels/tests\""
+                    dir('csvcubed-models') {
+                        sh "poetry run sphinx-apidoc -F -M -a -P --tocfile index.rst -d 10 -E --implicit-namespaces -o docs csvcubedmodels \"setup*\" \"csvcubedmodels/scripts\" \"/tests\""
                         sh 'poetry run sphinx-build -W -b html docs docs/_build/html'
                     }
 
-                    dir('pmd') {
-                        sh "poetry run sphinx-apidoc -F -M -a -P --tocfile index.rst -d 10 -E --implicit-namespaces -o docs  pmd \"setup*\" \"pmd/tests\""
+                    dir('csvcubed-pmd') {
+                        sh "poetry run sphinx-apidoc -F -M -a -P --tocfile index.rst -d 10 -E --implicit-namespaces -o docs  csvcubedpmd \"setup*\" \"tests\""
                         sh 'poetry run sphinx-build -W -b html docs docs/_build/html'
                     }
 
-                    dir('csvqb') {
-                        sh "poetry run sphinx-apidoc -F -M -a -P --tocfile index.rst -d 10 -E --implicit-namespaces -o docs csvqb \"setup*\" \"csvqb/tests\""
+                    dir('csvcubed') {
+                        sh "poetry run sphinx-apidoc -F -M -a -P --tocfile index.rst -d 10 -E --implicit-namespaces -o docs csvcubed \"setup*\" \"tests\""
                         sh 'poetry run sphinx-build -W -b html docs docs/_build/html'
                     }
 

diff --git a/README.md b/README.md
@@ -1,15 +1,15 @@
-# csvwlib
+# csvcubed
 
 A refactored version of the functionality required to build *CSV-qb*s along with the functionality necessary to convert said *CSV-qb*s into RDF compatible with the PMD platform.
 
 ## Packages
 
-| Name                                                  | Description                                                                         |
-|:------------------------------------------------------|:------------------------------------------------------------------------------------|
-| [csvqb](./csvqb/csvqb/README.md)                      | The key library helping to transform tidy-data into qb-flavoured CSV-W cubes.       |
-| [sharedmodels](./sharedmodels/sharedmodels/README.md) | Models and RDF serialisation functionality required by the csvqb and pmd packages.  |
-| [pmd](./pmd/pmd/README.md)                            | Transforms a CSV-qb into RDF which is compatible with the Publish My Data platform. |
-| [devtools](./devtools/devtools/README.md)             | Shared test functionality & dev dependencies which are commonly required.           |
+| Name                                                                | Description                                                                         |
+|:--------------------------------------------------------------------|:------------------------------------------------------------------------------------|
+| [csvcubed](./csvcubed/csvcubed/README.md)                           | The key library helping to transform tidy-data into qb-flavoured CSV-W cubes.       |
+| [csvcubed-models](./csvcubed-models/csvcubedmodels/README.md)       | Models and RDF serialisation functionality required by the csvcubed and csvcubed-pmd packages.  |
+| [csvcubed-pmd](./csvcubed-pmd/csvcubedpmd/README.md)                | Transforms a CSV-qb into RDF which is compatible with the Publish My Data platform. |
+| [csvcubed-devtools](./csvcubed-devtools/csvcubeddevtools/README.md) | Shared test functionality & dev dependencies which are commonly required.           |
 
 We're currently keeping all of the packages in one repository to make life easier whilst we break apart and refactor the existing code we have in [GSS-utils](https://github.com/GSS-Cogs/gss-utils). Once we're confident in how our packages look (and they're stable), it's likely that each individual package will be moved to its own independent repository.
 

diff --git a/devtools/devtools/README.md → csvcubed-devtools/csvcubeddevtools/README.md b/devtools/devtools/README.md → csvcubed-devtools/csvcubeddevtools/README.md
@@ -1,19 +1,23 @@
-# csvwlib - devtools
+# csvcubed - devtools
 
 > Shared test functionality & dev dependencies which are commonly required.
 
-The *devtools* package contains common packages necessary for development of *csvwlib* such as:
+The *devtools* package contains common packages necessary for development of *csvcubed* such as:
 
 * behave - for behaviour testing / BDD
-* docker - to execute docker containers, used to transform and test csvwlib outputs.
+* docker - to execute docker containers, used to transform and test csvcubed outputs.
 * black - auto-formatter to ensure consistent code formatting/style.
 
 It also contains shared functionality to support testing, e.g. common *behave* test steps like checking that files exist, using temporary directories when testing as well as tools to copy test files to & from docker containers.
 
 ## Installation
 
-This package should be installed as a [dev dependency](https://python-poetry.org/docs/cli#options-3) to ensure that end-users of *csvwlib* are not required to install development tools such as docker.
+This package should be installed as a [dev dependency](https://python-poetry.org/docs/cli#options-3) to ensure that end-users of *csvcubed* are not required to install development tools such as docker.
+
+
 
 ## API Documentation
 
-The latest API Documentation for the package can be found [here](https://ci.floop.org.uk/job/GSS_data/job/csvwlib/job/main/lastSuccessfulBuild/artifact/devtools/docs/_build/html/index.html).
+The latest API Documentation for the package can be found [here](https://ci.floop.org.uk/job/GSS_data/job/csvcubed/job/main/lastSuccessfulBuild/artifact/devtools/docs/_build/html/index.html).
+
+TODO: #217 Update API documentation location in csvcubed-devtools
diff --git a/devtools/devtools/__init__.py → ...bed-devtools/csvcubeddevtools/__init__.py b/devtools/devtools/__init__.py → ...bed-devtools/csvcubeddevtools/__init__.py
@@ -1,5 +1,5 @@
 """
-Contains functionality helpful to the development of the csvwlib tooling.
+Contains functionality helpful to the development of the csvcubed tooling.
 
 N.B. This library should always be installed as a `dev-package` and never used by an end-user.
 """
diff --git a/csvqb/csvqb/__init__.py → ...ls/csvcubeddevtools/behaviour/__init__.py b/csvqb/csvqb/__init__.py → ...ls/csvcubeddevtools/behaviour/__init__.py
diff --git a/devtools/devtools/behave/csv2rdf.py → ...ols/csvcubeddevtools/behaviour/csv2rdf.py b/devtools/devtools/behave/csv2rdf.py → ...ols/csvcubeddevtools/behaviour/csv2rdf.py
@@ -12,8 +12,8 @@
 from tempfile import TemporaryDirectory
 
 
-from devtools.helpers.tar import dir_to_tar, extract_tar
-from devtools.behave.temporarydirectory import get_context_temp_dir_path
+from csvcubeddevtools.helpers.tar import dir_to_tar, extract_tar
+from csvcubeddevtools.behaviour.temporarydirectory import get_context_temp_dir_path
 
 client = docker.from_env()
 client.images.pull("gsscogs/csv2rdf")

diff --git a/devtools/devtools/behave/csvlint.py → ...ols/csvcubeddevtools/behaviour/csvlint.py b/devtools/devtools/behave/csvlint.py → ...ols/csvcubeddevtools/behaviour/csvlint.py
@@ -10,8 +10,8 @@
 import sys
 from typing import Tuple
 
-from devtools.helpers.tar import dir_to_tar
-from devtools.behave.temporarydirectory import get_context_temp_dir_path
+from csvcubeddevtools.helpers.tar import dir_to_tar
+from csvcubeddevtools.behaviour.temporarydirectory import get_context_temp_dir_path
 
 client = docker.from_env()
 client.images.pull("gsscogs/csvlint")

diff --git a/devtools/devtools/behave/file.py → ...vtools/csvcubeddevtools/behaviour/file.py b/devtools/devtools/behave/file.py → ...vtools/csvcubeddevtools/behaviour/file.py
@@ -70,4 +70,4 @@ def _get_test_cases_dir(start_dir: Path = Path(".")):
             f"Found multiple child test-case directories: {', '.join([str(d) for d in child_test_cases_dirs])}"
         )
 
-    return _get_test_cases_dir(start_dir.parent)
+    return _get_test_cases_dir(start_dir.absolute().parent)
diff --git a/devtools/devtools/behave/rdf.py → ...evtools/csvcubeddevtools/behaviour/rdf.py b/devtools/devtools/behave/rdf.py → ...evtools/csvcubeddevtools/behaviour/rdf.py
diff --git a/devtools/devtools/behave/sparqltests.py → ...csvcubeddevtools/behaviour/sparqltests.py b/devtools/devtools/behave/sparqltests.py → ...csvcubeddevtools/behaviour/sparqltests.py
@@ -10,8 +10,8 @@
 from typing import Tuple, List
 
 
-from devtools.helpers.tar import dir_to_tar
-from devtools.behave.temporarydirectory import get_context_temp_dir_path
+from csvcubeddevtools.helpers.tar import dir_to_tar
+from csvcubeddevtools.behaviour.temporarydirectory import get_context_temp_dir_path
 
 client = docker.from_env()
 client.images.pull("gsscogs/gdp-sparql-tests")

diff --git a/...ols/devtools/behave/temporarydirectory.py → ...ddevtools/behaviour/temporarydirectory.py b/...ols/devtools/behave/temporarydirectory.py → ...ddevtools/behaviour/temporarydirectory.py
diff --git a/devtools/devtools/helpers/__init__.py → ...ools/csvcubeddevtools/helpers/__init__.py b/devtools/devtools/helpers/__init__.py → ...ools/csvcubeddevtools/helpers/__init__.py
diff --git a/devtools/devtools/helpers/csvwhelpers.py → ...s/csvcubeddevtools/helpers/csvwhelpers.py b/devtools/devtools/helpers/csvwhelpers.py → ...s/csvcubeddevtools/helpers/csvwhelpers.py
diff --git a/devtools/devtools/helpers/tar.py → ...-devtools/csvcubeddevtools/helpers/tar.py b/devtools/devtools/helpers/tar.py → ...-devtools/csvcubeddevtools/helpers/tar.py