Merge pull request #17 from OpenGenenerativeAI/add-some-good-practice…

…s-from-sicarator

Add some good practices from sicarator

.github/workflows/actions/install_dependencies/action.yml

@@ -0,0 +1,29 @@
+name: "Install Dependencies"
+description: "Action to build the project dependencies from the main versions"
+inputs:
+  python_version:
+    required: true
+    type: string
+    default: "3.11.3"
+  poetry_version:
+    required: true
+    type: string
+    default: "1.5.1"
+
+runs:
+  using: composite
+  steps:
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        version: ${{ inputs.poetry_version }}
+        virtualenvs-create: true
+        virtualenvs-in-project: false
+        installer-parallel: true
+    - uses: actions/setup-python@v4
+      with:
+        python-version: ${{ inputs.python_version }}
+        cache: "poetry"
+    - name: Install Dependencies
+      run: poetry install --no-root
+      shell: bash

.github/workflows/tests.yml

@@ -6,22 +6,62 @@ on:
   pull_request:
     branches: [main]
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.head_ref || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
 jobs:
-  build:
+  setup:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/workflows/actions/install_dependencies
+
+  checks:
+    needs: setup
     runs-on: ubuntu-latest
+    name: ${{ matrix.quality-command }}
+    strategy:
+      matrix:
+        quality-command:
+          - black
+          - ruff
+          - mypy
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/workflows/actions/install_dependencies
+      - name: run ${{ matrix.quality-command }}
+        run: make ${{ matrix.quality-command }}
 
+  test:
+    needs: setup
+    runs-on: ubuntu-latest
+    name: test
     steps:
       - uses: actions/checkout@v3
-      - name: Set up Python 3.11
-        uses: actions/setup-python@v4
+      - uses: ./.github/workflows/actions/install_dependencies
+      - name: run test
+        run: make test
+      # Run even if make test fails for coverage reports
+      # TODO: select a better xml results displayer
+      - name: Archive test results coverage results
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: test_results
+          path: tests-results.xml
+      - name: Archive code coverage results
+        uses: actions/upload-artifact@v3
+        if: always()
         with:
-          python-version: "3.11"
-          cache: "pip"
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install pytest
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-      - name: Test with pytest
-        run: |
-          python -m pytest tests/
+          name: code-coverage-report
+          path: htmlcov/
+
+  all_checks_passed:
+    # Used to easily force requirements checks in GitHub
+    needs:
+      - checks
+      - test
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo "All checks passed"

.gitignore

@@ -1,9 +1,22 @@
-.DS_Store
-**.pyc
+# byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# unit tests / coverage reports
+/tests-results.xml
+/.coverage
+/coverage.xml
+/htmlcov/
 
-local_models/*
+# pyenv
+/.python-version
 
-.env
+# IDE
+.idea/
+.vscode/
+
+# macOS
+.DS_Store
 
-demo/Genoss/*
-genossGPT/*
+# Ignore env from demo
+demo/.env

.pre-commit-config.yaml

@@ -0,0 +1,43 @@
+default_install_hook_types:
+# Mandatory to install both pre-commit and pre-push hooks (see https://pre-commit.com/#top_level-default_install_hook_types)
+# Add new hook types here to ensure automatic installation when running `pre-commit install`
+- pre-commit
+- pre-push
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v4.3.0
+  hooks:
+  - id: trailing-whitespace
+  - id: end-of-file-fixer
+  - id: check-yaml
+  - id: check-json
+  - id: check-toml
+  - id: check-added-large-files
+- repo: local
+  hooks:
+  - id: black
+    name: Formatting (black)
+    entry: black
+    language: system
+    types: [python]
+    stages: [commit]
+  - id: ruff
+    name: Linter (ruff)
+    entry: ruff
+    language: system
+    types: [python]
+    stages: [commit]
+  - id: mypy
+    name: Type checking (mypy)
+    entry: make mypy
+    pass_filenames: false
+    language: system
+    types: [python]
+    stages: [commit]
+  - id: test
+    name: Unit tests (pytest)
+    entry: make test
+    pass_filenames: false
+    language: system
+    types: [python]
+    stages: [push]

CONTRIBUTING.md

@@ -0,0 +1,125 @@
+# GenossGPT
+## Project requirements
+
+### `Python 3.11`
+#### Install with pyenv
+- Install [pyenv](https://github.com/pyenv/pyenv) to manage your Python versions and virtual environments:
+  ```bash
+  curl -sSL https://pyenv.run | bash
+  ```
+  - If you are on MacOS and experiencing errors on python install with pyenv, follow this [comment](https://github.com/pyenv/pyenv/issues/1740#issuecomment-738749988)
+  - Add these lines to your `~/.bashrc` or `~/.zshrc` to be able to activate `pyenv virtualenv`:
+      ```bash
+      eval "$(pyenv init -)"
+      eval "$(pyenv virtualenv-init -)"
+      eval "$(pyenv init --path)"
+      ```
+  - Restart your shell
+
+- Install the right version of `Python` with `pyenv`:
+  ```bash
+  pyenv install 3.11.3
+  ```
+
+### Poetry
+
+- Install [Poetry](https://python-poetry.org) to manage your dependencies and tooling configs:
+  ```bash
+  curl -sSL https://install.python-poetry.org | python - --version 1.5.1
+  ```
+  *If you have not previously installed any Python version, you may need to set your global Python version before installing Poetry:*
+    ```bash
+    pyenv global 3.11.3
+    ```
+
+### Docker Engine
+Install [Docker Engine](https://docs.docker.com/engine/install/) to build and run the API's Docker image locally.
+
+## Installation
+
+### Create a virtual environment
+
+Create your virtual environment and link it to your project folder:
+
+```bash
+pyenv virtualenv 3.11.3 genoss-gpt
+pyenv local genoss-gpt
+```
+Now, every time you are in your project directory your virtualenv will be activated thanks to `pyenv`!
+
+### Install Python dependencies through poetry
+
+```bash
+poetry install --no-root
+```
+
+### Install git hooks (running before commit and push commands)
+
+```bash
+poetry run pre-commit install
+```
+
+## Testing
+
+To run unit tests, run `pytest` with:
+```bash
+pytest tests --cov genoss
+```
+or
+```bash
+make test
+```
+
+## Formatting and static analysis
+
+### Code formatting with `black`
+
+To check code formatting, run `black` with:
+```bash
+black . --check
+```
+or
+```bash
+make black
+```
+
+You can also [integrate it to your IDE](https://black.readthedocs.io/en/stable/integrations/editors.html) to reformat
+your code each time you save a file.
+
+### Static analysis with `ruff`
+
+To run static analysis, run `ruff` with:
+```bash
+ruff check genoss tests
+```
+or
+```bash
+make ruff
+```
+
+To run static analysis and to apply auto-fixes, run `ruff` with:
+```bash
+make fix-ruff
+```
+### Type checking with `mypy`
+
+To type check your code, run `mypy` with:
+```bash
+mypy genoss --explicit-package-bases --namespace-packages
+```
+or
+```bash
+make mypy
+```
+
+## API
+The project includes an API built with [FastAPI](https://fastapi.tiangolo.com/). Its code can be found at `genoss`.
+
+The API is containerized using a [Docker](https://docs.docker.com/get-started/) image, built from the `Dockerfile` and `docker-compose.yml` at the root.
+
+To build and start the API, use the following Makefile command:
+```bash
+make start-api
+```
+
+For more details on the API routes, check the automatically generated [swagger](https://learning.postman.com/docs/getting-started/importing-and-exporting-data/#importing-postman-data) at the `/docs` url.

Dockerfile

@@ -1,20 +1,55 @@
-# Use the same base image as the 'backend-core' container
-FROM python:3.11-bullseye
+# Docker multi-stage building, as recommended by https://fastapi.tiangolo.com/deployment/docker/#docker-image-with-poetry
+FROM python:3.11.3-slim-buster as curl-stage
 
-# Install necessary packages
-RUN apt-get update && apt-get install -y liblzma-dev cmake git
+# Install curl ; remove apt cache to reduce image size
+# TODO: check if liblzma-dev is necessary
+RUN apt-get -y update && apt-get -y install curl liblzma-dev && rm -rf /var/lib/apt/lists/*
+RUN pip install --upgrade pip
 
-# Set the working directory
+FROM curl-stage as poetry-requirements-stage
+
+ARG poetry_groups=main,llms
+
+WORKDIR /tmp
+
+ENV HOME /root
+ENV PATH=${PATH}:$HOME/.local/bin
+
+# Install poetry
+RUN curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.5.1 python3 -
+
+# Export requirements.txt
+COPY ./pyproject.toml ./poetry.lock* /tmp/
+RUN poetry export -f requirements.txt --output requirements.txt --without-hashes --no-interaction --no-cache --only=${poetry_groups}
+
+
+FROM curl-stage
 WORKDIR /app
 
-# Copy the requirements file
-COPY ./requirements.txt /app/requirements.txt
+ENV \
+    # Prevent Python from buffering stdout and stderr and loosing some logs (equivalent to python -u option)
+    PYTHONUNBUFFERED=1 \
+    # Prevent Pip from timing out when installing heavy dependencies
+    PIP_DEFAULT_TIMEOUT=600 \
+    # Prevent Pip from creating a cache directory to reduce image size
+    PIP_NO_CACHE_DIR=1
+
+ENV UVICORN_HOST="0.0.0.0" \
+    UVICORN_PORT=4321
 
-# Install Python dependencies
-RUN pip install --no-cache-dir -r /app/requirements.txt --timeout 100
+# Install dependencies with pip from exported requirements.txt
+COPY --from=poetry-requirements-stage /tmp/requirements.txt /app/requirements.txt
+RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt
 
-# Copy your application's code to the Docker container
+# Copy API files
 COPY . /app
 
-# Start the Uvicorn server on port 5051
-CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "4321"]
+# Add and set a non-root user
+RUN useradd appuser
+USER appuser
+
+# Start FastAPI
+CMD ["uvicorn", "main:app"]
+
+# Healthcheck
+HEALTHCHECK --interval=10s --timeout=1s --retries=3 CMD curl --fail http://localhost:${PORT}/ || exit 1