Merge branch 'new_ui'

.github/workflows/Publish.yml

@@ -16,16 +16,19 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version-file: pyproject.toml
+          python-version-file: scraper/pyproject.toml
           architecture: x64
 
       - name: Build packages
+        working-directory: scraper
         run: |
           pip install -U pip build
           python -m build --sdist --wheel
 
       - name: Upload to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1.8
+        with:
+          packages-dir: scraper/dist/
 
       - name: Build and push Docker image
         uses: openzim/docker-publish-action@v10

.github/workflows/QA.yml

@@ -7,7 +7,7 @@ on:
       - main
 
 jobs:
-  check-qa:
+  check-scraper-qa:
     runs-on: ubuntu-22.04
 
     steps:
@@ -16,19 +16,49 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version-file: pyproject.toml
+          python-version-file: scraper/pyproject.toml
           architecture: x64
 
-      - name: Install dependencies (and project)
+      - name: Install dependencies
+        working-directory: scraper
         run: |
           pip install -U pip
           pip install -e .[lint,check,scripts,test]
 
       - name: Check black formatting
+        working-directory: scraper
         run: inv lint-black
 
       - name: Check ruff
+        working-directory: scraper
         run: inv lint-ruff
 
       - name: Check pyright
+        working-directory: scraper
         run: inv check-pyright
+
+  check-zimui-qa:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: zimui/.node-version
+
+      - name: Install JS dependencies
+        working-directory: zimui
+        run: |
+          yarn install
+
+      - name: Check prettier
+        working-directory: zimui
+        run: |
+          yarn format
+
+      - name: Check eslint
+        working-directory: zimui
+        run: |
+          yarn lint

.github/workflows/Tests.yml

@@ -7,7 +7,7 @@ on:
       - main
 
 jobs:
-  run-tests:
+  test-scraper:
     runs-on: ubuntu-22.04
 
     steps:
@@ -16,39 +16,62 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version-file: pyproject.toml
+          python-version-file: scraper/pyproject.toml
           architecture: x64
 
       - name: Install dependencies (and project)
+        working-directory: scraper
         run: |
           pip install -U pip
           pip install -e .[test,scripts]
 
       - name: Run the tests
+        working-directory: scraper
         run: inv coverage --args "-vvv"
 
       - name: Upload coverage report to codecov
         uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
-  build_python:
+  build-scraper:
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v3
 
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version-file: pyproject.toml
+          python-version-file: scraper/pyproject.toml
           architecture: x64
 
       - name: Ensure we can build Python targets
+        working-directory: scraper
         run: |
           pip install -U pip build
           python3 -m build --sdist --wheel
 
-  build_docker:
+  build-zimui:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: zimui/.node-version
+
+      - name: Install dependencies
+        working-directory: zimui
+        run: |
+          yarn install
+
+      - name: Build
+        working-directory: zimui
+        run: |
+          yarn build
+
+  build-docker:
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v3

.gitignore

@@ -368,16 +368,6 @@ pyrightconfig.json
 
 # assets that we download
 .dockerignore
-src/kolibri2zim/templates/assets/bootstrap/
-src/kolibri2zim/templates/assets/pdfjs/
-src/kolibri2zim/templates/assets/videojs/
-src/kolibri2zim/templates/assets/jquery.min.js
-src/kolibri2zim/templates/assets/ogvjs/
-src/kolibri2zim/templates/assets/videojs-ogvjs.js
-src/kolibri2zim/templates/assets/epub.min.js
-src/kolibri2zim/templates/assets/bootstrap-icons/
-src/kolibri2zim/templates/assets/jszip.min.js
-src/kolibri2zim/templates/assets/perseus/
 
 # output dir
 output

.pre-commit-config.yaml

@@ -4,8 +4,24 @@ repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
   rev: v4.4.0
   hooks:
-  -   id: trailing-whitespace
-  -   id: end-of-file-fixer
+  - id: trailing-whitespace
+  - id: end-of-file-fixer
+- repo: https://github.com/pre-commit/mirrors-prettier
+  rev: v3.0.3
+  hooks:
+  - id: prettier
+    files: zimui\/.*$ # files in zimui folder
+- repo: https://github.com/pre-commit/mirrors-eslint
+  rev: v8.51.0
+  hooks:
+  - id: eslint
+    types: [file]
+    files: zimui\/src\/.*(?:\.[jt]sx?|\.vue)$ # *.js, *.jsx, *.ts, *.tsx, *.vue in zimui/src folder
+    args:
+    - --ignore-path
+    - zimui/.eslintignore
+    - --config
+    - zimui/.eslintrc.cjs
 - repo: https://github.com/psf/black
   rev: "24.2.0"
   hooks:

CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+- Enhance contribution guidelines: Refactored shell scripts, added additional steps. (#91)
+
 ## [1.2.1] - 2024-02-29
 
 ### Changed
@@ -48,13 +50,41 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Default publisher is not correctly spelled (#78)
 - Adapt to hatchling v1.19.0 which mandates packages setting (#79)
 - Small fixes in invoke tasks
+- Force Python version to 3.11 (3.12 is not yet ready in our dependencies)
 
 ### Changed
 
 - Dockerfile: split installation of Python dependencies for more efficiency
 - Github workflow: publish `dev` tag on every push to `main` branch
 - Github workflow: build Docker image + test its startup
 - Github workflow: adopt new standard execution structure (`on` conditions)
+- Scraper (Python code) has been moved to the scraper subfolder
+- Vue.JS is now used as main UI framework
+  - all its code is in the zimui subfolder
+  - it is rendered with Vite to produce a static website
+  - developpers instruction have been adapted
+- QA and Tests workflows have been adapted
+    - to the new folder structure
+    - to also QA and Test the Vue.JS part
+- precommit hooks have been configured for the Vue.JS part
+- Dockerfile has been adapted to first build the Vue.JS part in a dedicated stage and then embed the generated files into the final Python-based image
+- Topics are stored as JSON files in the ZIM
+    - JSON is generated by pydantic
+    - these files are consumed by the Vue.JS UI
+    - content (video, audio, pdf, epub, ...) is still rendered by Jinja2 as before
+- URLs are meaningful slugs
+    - permalink based on Kolibri node title + 4 chars from node ID
+    - generated by Python slugify lib
+- changes in the ZIM "folder" structure:
+    -  files generated by Vite are placed in /
+    -  thumbnails are placed in /thumbnails
+    -  JSON files generated to render topics are placed in /topics
+    -  most Kolibri content (video, audio, ePub, PDF) are placed in /files (some content is still placed at the root to not break some stuff which was found hard to fix for now, will be tackled in specific issues for each content type)
+- legacy MANIFEST.in has been deleted (left-over from migration to hatch)
+- is_front property has been adjusted when adding the item to the ZIM
+- one new CLI argument --zimui-dist to specify the folder where zimui has been built (by Vite)
+
+
 
 ## [1.1.0] - 2023-07-25
 

CONTRIBUTING.md

@@ -4,7 +4,9 @@
 
 * Open issues, bug reports and send PRs [on github](https://github.com/openzim/kolibri2zim).
 * Make sure it's `py3.6+` compatible.
+* Docker (which engine must run in the background).
 * Use [black](https://github.com/psf/black) code formatting.
+* `pre-commit` must be activated before (code must pass all pre-commit checks to succeed in the CI)
 
 ## notes
 
@@ -25,12 +27,44 @@ To add a new locale (`fr` in this example, use only ISO-639-1):
 3. translate the PO file ([poedit](https://poedit.net/) is your friend)
 4. compile updated translation `pybabel compile -d kolibri2zim/locale -l fr`
 
-## releasing
 
-* Update your dependencies: `pip install -U setuptools wheel twine`
-* Make sure CHANGELOG is up-to-date
-* Bump version on `kolibri2zim/VERSION`
-* Build packages `python ./setup.py sdist bdist_wheel`
-* Upload to PyPI `python -m twine upload dist/kolibri2zim-1.0.0*`.
-* Commit your CHANGELOG + version bump changes
-* Tag version on git `git tag -a v1.0.0`
+## Developing the ZIM UI in Vue.JS
+
+Sometimes you need to alter something in the ZIM UI in Vue.JS but for this to work, you need assets which are generated by the scraper (e.g. channel.json, ...).
+
+To simplify this, it is possible to:
+- run the scraper (with original code base or your modified one)
+- extract assets from generated files and place them in a directory where ZIM UI will find them
+- iterate on ZIM UI code
+
+To achieve this, first build the Docker image based on current code base.
+
+```
+docker build -t local-kolibri2zim .
+```
+
+Scrape a channel (here we use the minimal channel, but you could use any other one of interest for your UI developments).
+
+```
+docker run --rm -it -v "$PWD/output":/output local-kolibri2zim kolibri2zim --name "minimal_test" --title "Minimal Kolibri Channel Test" --description "This is a minimal K
+olibri Channel, with new Kolibri UI" --channel-id "7f744ce8d28b471eaf663abd60c92267" --zim-file "Minimal_Test.zim"
+```
+
+Extract interesting ZIM content and move it to `public` folder.
+
+```
+find zimui/public/ -mindepth 1 -maxdepth 1 ! -name ".gitignore" -delete
+docker run -it --rm -v $(pwd)/output:/data ghcr.io/openzim/zim-tools:latest zimdump dump --dir=/data/Minimal_Test /data/Minimal_Test.zim
+sudo chown -R $(id -u -n):$(id -g -n) output/Minimal_Test
+mv output/Minimal_Test/* zimui/public/
+rm -rf output/Minimal_Test
+```
+
+Start ZIM UI locally.
+
+```
+cd zimui
+yarn dev
+```
+
+Do not forget to cleanup `public` folder before building the docker image again, otherwise all assets will be pushed to the ZIM.

Dockerfile

@@ -1,3 +1,10 @@
+FROM node:20-alpine as zimui
+
+WORKDIR /src
+COPY zimui /src
+RUN yarn install --frozen-lockfile
+RUN yarn build
+
 FROM python:3.12-bookworm
 LABEL org.opencontainers.image.source https://github.com/openzim/kolibri
 
@@ -12,22 +19,28 @@ RUN apt-get update \
       pip
 
 # Copy pyproject.toml and its dependencies
-COPY pyproject.toml openzim.toml README.md /src/
-COPY src/kolibri2zim/__about__.py /src/src/kolibri2zim/__about__.py
+COPY README.md /src/
+COPY scraper/pyproject.toml scraper/openzim.toml /src/scraper/
+COPY scraper/src/kolibri2zim/__about__.py /src/scraper/src/kolibri2zim/__about__.py
 
 # Install Python dependencies
-RUN pip install --no-cache-dir /src
+RUN pip install --no-cache-dir /src/scraper
 
 # Copy code + associated artifacts
-COPY src /src/src
-COPY *.md LICENSE *.py /src/
+COPY scraper/src /src/scraper/src
+COPY *.md LICENSE /src/
 
 # Install + cleanup
-RUN pip install --no-cache-dir /src \
- && rm -rf /src
+RUN pip install --no-cache-dir /src/scraper \
+ && rm -rf /src/scraper
+
+# Copy zimui build output
+COPY --from=zimui /src/dist /src/zimui
 
 # default output directory
 RUN mkdir -p /output
 WORKDIR /output
 
+ENV KOLIBRI_ZIMUI_DIST=/src/zimui
+
 CMD ["kolibri2zim", "--help"]