LAI [wip]

.actions/assistant.py

@@ -83,10 +83,10 @@ def prepare_nightly_version(proj_root: str = _PATH_ROOT) -> None:
         now_date = now.strftime("%Y%m%d")
 
         print(f"prepare init '{path_info}' - replace version by {now_date}")
-        with open(path_info) as fp:
+        with open(path_info, encoding="utf-8") as fp:
             init = fp.read()
         init = re.sub(r'__version__ = [\d\.\w\'"]+', f'__version__ = "{now_date}"', init)
-        with open(path_info, "w") as fp:
+        with open(path_info, "w", encoding="utf-8") as fp:
             fp.write(init)
 
     @staticmethod
@@ -118,8 +118,8 @@ def _prune_packages(req_file: str, packages: Sequence[str]) -> None:
 
     @staticmethod
     def _replace_min(fname: str) -> None:
-        req = open(fname).read().replace(">=", "==")
-        open(fname, "w").write(req)
+        req = open(fname, encoding="utf-8").read().replace(">=", "==")
+        open(fname, "w", encoding="utf-8").write(req)
 
     @staticmethod
     def replace_oldest_ver(requirement_fnames: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:

.github/actions/pkg-check/action.yml

@@ -5,6 +5,7 @@ inputs:
   pkg-name:
     description: package name inside lightning.*
     required: true
+    default: "lightning"
   nb-dirs:
     description: nb of packages in the wrap/distribution
     required: false
@@ -21,10 +22,7 @@ runs:
   - name: Source check
     env:
       PACKAGE_NAME: ${{ inputs.pkg-name }}
-    run: |
-      python setup.py check --metadata --strict
-      # TODO: fix reimporting (F811) which is aftefact after rplacing var assigne with import in meta package
-      flake8 src/lightning/ --ignore E402,F401,E501,W391,E303,F811
+    run: python setup.py check --metadata --strict
     shell: bash
 
   - name: Create package
@@ -40,10 +38,6 @@ runs:
       # python setup.py clean
     shell: bash
 
-  - name: copy/export pkg
-    run: cp dist/* pypi/
-    shell: bash
-
   - name: Unzip packages
     if: ${{ inputs.pkg-name != '' }}
     working-directory: dist

.github/workflows/ci-app-tests.yml

@@ -64,10 +64,11 @@ jobs:
           pip list
 
       - name: DocTests App
-        # todo: run doctest laos on mono-package
-        if: ${{ matrix.pkg-name != 'LAI' }}
         working-directory: src
-        run: python -m pytest lightning_app --ignore-glob="**/cli/*-template/**"
+        run: |
+          pip install pytest -q
+          scope=$(python -c "print('lightning/app' if '${{matrix.pkg-name}}' == 'LAI' else 'lightning_app')")
+          python -m pytest $scope --ignore-glob="**/cli/*-template/**"
 
       - name: Get pip cache
         id: pip-cache

.github/workflows/ci-lite-test-full.yml

@@ -83,10 +83,11 @@ jobs:
         pip list
 
     - name: DocTests Lite
-      # todo: run docstest laos on mono-package
-      if: ${{ matrix.pkg-name != 'LAI' }}
       working-directory: src
-      run: python -m pytest lightning_lite
+      run: |
+        pip install pytest -q
+        scope=$(python -c "print('lightning/lite' if '${{matrix.pkg-name}}' == 'LAI' else 'lightning_lite')")
+        python -m pytest $scope
 
     # Note: This uses an internal pip API and may not always work
     # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow
@@ -102,7 +103,7 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ matrix.pkg-name }}-${{ matrix.release }}-${{ matrix.requires }}-
 
-    - name: Install dependencies
+    - name: Install package & dependencies
       run: |
         pip install -r requirements/lite/devel.txt --upgrade --find-links ${TORCH_URL}
         pip list

.github/workflows/ci-pkg-install.yml

@@ -17,25 +17,14 @@ defaults:
 
 jobs:
 
-  init-temp:
-    runs-on: ubuntu-20.04
-    steps:
-    - run: mkdir pypi && touch pypi/.placeholder
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ci-packages-${{ github.sha }}
-        path: pypi
-
-  install-standalone:
-    needs: init-temp
+  install-pkg:
     runs-on: ${{ matrix.os }}
     strategy:
-      fail-fast: true
-      max-parallel: 1
+      fail-fast: false
       matrix:
         os: [ubuntu-20.04, ubuntu-22.04, macOS-11, macOS-12, windows-2022]
-        pkg: ["app", "lite", "pytorch"]
-        python-version: [3.8]  # , 3.9
+        pkg: ["app", "lite", "pytorch", ""]
+        python-version: ["3.7" , "3.10"]
 
     steps:
       - uses: actions/checkout@v3
@@ -49,109 +38,14 @@ jobs:
           grep 'pytest>' ../requirements/pytorch/test.txt | xargs -0 pip install
           python -m pytest setup_tools.py
 
-      - uses: actions/download-artifact@v3
-        with:
-          name: ci-packages-${{ github.sha }}
-          path: pypi
-      - run: ls -lh pypi/
-
       - run: python -c "print('NB_DIRS=' + str(2 if '${{ matrix.pkg }}' == 'pytorch' else 1))" >> $GITHUB_ENV
       - uses: ./.github/actions/pkg-check
         with:
           pkg-name: ${{ matrix.pkg }}
           nb-dirs: ${{ env.NB_DIRS }}
 
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ci-packages-${{ github.sha }}
-          path: pypi
-
-      - uses: ./.github/actions/pkg-install
-
-  install-meta-src:
-    needs: install-standalone
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      # max-parallel: 1
-      matrix:
-        os: [ubuntu-20.04, ubuntu-22.04, macOS-11, macOS-12, windows-2022]
-        pkg: ["", "lightning"]
-        python-version: [3.8]  # , 3.9
-
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - run: mkdir -p pypi
-      - uses: actions/download-artifact@v3
-        if: matrix.pkg != ''
-        with:
-          name: ci-packages-${{ github.sha }}
-          path: pypi
-      - run: ls -lh pypi/
-
-      - uses: ./.github/actions/pkg-check
-        with:
-          pkg-name: ${{ matrix.pkg }}
-
       - uses: ./.github/actions/pkg-install
-        with:
-          pkg-name: "lightning"
-          pip-flags: "-U --pre --find-links ../pypi/"
-
-      - name: Run CLI
-        run: python -m lightning --version
-
-  install-meta-pypi:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      # max-parallel: 1
-      matrix:
-        os: [ubuntu-20.04, ubuntu-22.04, macOS-11, macOS-12, windows-2022]
-        python-version: [3.8]  # , 3.9
-
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Dowload package
-        # todo: download also lite after it is fist published
-        run: |
-          pip install -q -r .actions/requirements.txt
-          for pkg in 'app' 'pytorch' ; do
-            python .actions/assistant.py download_package --package "$pkg" --folder pypi
-          done
-          ls -lh pypi/
-
-      - name: Unzip packages
-        working-directory: pypi
-        run: for file in `ls *.gz`; do tar -xzf $file; done
-      - name: Show upacked pkgs
-        if: runner.os == 'linux'
-        run: |
-          sudo apt install -y tree
-          tree pypi/ -L 3
-
-      - name: Miror source
-        run: |
-          pip install -q -r .actions/requirements.txt
-          python .actions/assistant.py mirror_pkg2source --pypi_folder pypi --src_folder src
-          ls -R src/
-
-      - uses: ./.github/actions/pkg-check
-        with:
-          pkg-name: "lightning"
-
-      - uses: ./.github/actions/pkg-install
-        with:
-          pkg-name: "lightning"
-          pip-flags: "-U --pre --find-links ../pypi/"
 
       - name: Run CLI
+        if: matrix.pkg == ''
         run: python -m lightning --version

.github/workflows/ci-pytorch-test-full.yml

@@ -111,14 +111,15 @@ jobs:
       env:
         PACKAGE_NAME: ${{ matrix.pkg-name }}
       run: |
-        pip install -e . pytest --upgrade --find-links ${TORCH_URL}
+        pip install -e . --upgrade --find-links ${TORCH_URL}
         pip list
 
     - name: DocTests PL
-      # todo: run docstest laos on mono-package
-      if: ${{ matrix.pkg-name != 'LAI' }}
       working-directory: ./src
-      run: python -m pytest pytorch_lightning
+      run: |
+        pip install pytest -q
+        scope=$(python -c "print('lightning/pytorch' if '${{matrix.pkg-name}}' == 'LAI' else 'pytorch_lightning')")
+        python -m pytest $scope
 
     - name: Get pip cache dir
       id: pip-cache

docs/source-lit/conf.py

@@ -398,13 +398,13 @@ def find_source():
 from typing import Optional
 
 import torch
-import pytorch_lightning as pl
+import lightning.pytorch as pl
 from torch import nn
 from torch.utils.data import IterableDataset, DataLoader, Dataset
-from pytorch_lightning import LightningDataModule, LightningModule, Trainer, seed_everything
-from pytorch_lightning.callbacks import Callback
-from pytorch_lightning.cli import LightningCLI
-from pytorch_lightning.utilities import (
+from lightning.pytorch import LightningDataModule, LightningModule, Trainer, seed_everything
+from lightning.pytorch.callbacks import Callback
+from lightning.pytorch.cli import LightningCLI
+from lightning.pytorch.utilities import (
     _APEX_AVAILABLE,
     _TORCHVISION_AVAILABLE,
     _TORCH_GREATER_EQUAL_1_10,

requirements/app/base.txt

@@ -1,5 +1,5 @@
 lightning-cloud>=0.5.10
-packaging
+typing-extensions>=4.0.0, <4.3.1
 deepdiff>=5.7.0, <=5.8.1
 starsessions>=1.2.1, <2.0 # strict
 fsspec>=2022.5.0, <=2022.7.1

requirements/app/test.txt

@@ -1,8 +1,13 @@
 coverage>=6.4, <=6.4.2
 codecov>=2.1, <=2.1.12
 pytest>=7.0, <=7.1.2
-pytest-timeout <=2.1.0
 pytest-cov <=3.0.0
+pytest-forked <=1.4.0
+pytest-rerunfailures>=10.2
+pre-commit>=1.0
+mypy==0.971
+
+pytest-timeout <=2.1.0
 playwright==1.27.1
 # pytest-flake8
 httpx

requirements/base.txt

@@ -1,2 +0,0 @@
-pytorch_lightning>=1.6.5
-lightning_app>=0.5.2

requirements/lite/devel.txt

@@ -1,4 +1,15 @@
 # install all mandatory dependencies
 -r ./base.txt
 
+
+# install all test utilities
+coverage>=6.4, <=6.4.2
+codecov>=2.1, <=2.1.12
+pytest>=7.0, <=7.1.2
+pytest-cov <=3.0.0
+pytest-forked <=1.4.0
+pytest-rerunfailures>=10.2
+pre-commit>=1.0
+mypy==0.971
+
 -r ./test.txt

setup.py

@@ -25,12 +25,13 @@
     In case you want to install just one package you need to export env. variable before calling `pip`
 
      - for `pytorch-lightning` use `export PACKAGE_NAME=pytorch ; pip install .`
+     - for `lightning-lite` use `export PACKAGE_NAME=lite ; pip install .`
      - for `lightning-app` use `export PACKAGE_NAME=app ; pip install .`
 
 3. Building packages as sdist or binary wheel and installing or publish to PyPI afterwords you use command
     `python setup.py sdist` or `python setup.py bdist_wheel` accordingly.
    In case you want to build just a particular package you would use exporting env. variable as above:
-   `export PACKAGE_NAME=pytorch|app ; python setup.py sdist bdist_wheel`
+   `export PACKAGE_NAME=pytorch|app|lite ; python setup.py sdist bdist_wheel`
 
 4. Automated releasing with GitHub action is natural extension of 3) is composed of three consecutive steps:
     a) determine which packages shall be released based on version increment in `__version__.py` and eventually
@@ -102,9 +103,7 @@ def _load_py_module(name: str, location: str) -> ModuleType:
     if _PACKAGE_NAME not in _PACKAGE_MAPPING:
         _SETUP_TOOLS.set_version_today(os.path.join(_PATH_SRC, "lightning", "__version__.py"))
 
-    for lit_name, pkg_name in _PACKAGE_MAPPING.items():
-        # fixme: if we run creation of meta pkg against stable we shall pull the source
-        _SETUP_TOOLS.create_meta_package(os.path.join(_PATH_ROOT, "src"), pkg_name, lit_name)
+    _SETUP_TOOLS.create_mirror_package(os.path.join(_PATH_ROOT, "src"), _PACKAGE_MAPPING)
 
     _SETUP_MODULE = _load_py_module(name="pkg_setup", location=_PATH_SETUP)
     _SETUP_MODULE._adjust_manifest(pkg_name=_REAL_PKG_NAME)

src/lightning/__init__.py

@@ -1,5 +1,6 @@
 """Root package info."""
 import logging
+import os
 from typing import Any
 
 _DETAIL = 15  # between logging.INFO and logging.DEBUG, used for logging in production use cases
@@ -15,6 +16,7 @@ def _detail(self: Any, message: str, *args: Any, **kwargs: Any) -> None:
 logging.detail = _detail
 logging.Logger.detail = _detail
 
+
 # explicitly don't set root logger's propagation and leave this to subpackages to manage
 _logger = logging.getLogger(__name__)
 _logger.setLevel(logging.INFO)
@@ -39,6 +41,10 @@ def _detail(self: Any, message: str, *args: Any, **kwargs: Any) -> None:
 from lightning.pytorch.trainer import Trainer  # noqa: E402
 from lightning.pytorch.utilities.seed import seed_everything  # noqa: E402
 
+import lightning.app  # isort: skip # noqa: E402
+
+lightning.app._PROJECT_ROOT = os.path.dirname(lightning.app._PROJECT_ROOT)
+
 __all__ = [
     "LightningApp",
     "LightningFlow",