LAI: creating mirror package

.actions/assistant.py

@@ -83,10 +83,10 @@ def prepare_nightly_version(proj_root: str = _PATH_ROOT) -> None:
         now_date = now.strftime("%Y%m%d")
 
         print(f"prepare init '{path_info}' - replace version by {now_date}")
-        with open(path_info) as fp:
+        with open(path_info, encoding="utf-8") as fp:
             init = fp.read()
         init = re.sub(r'__version__ = [\d\.\w\'"]+', f'__version__ = "{now_date}"', init)
-        with open(path_info, "w") as fp:
+        with open(path_info, "w", encoding="utf-8") as fp:
             fp.write(init)
 
     @staticmethod
@@ -118,8 +118,8 @@ def _prune_packages(req_file: str, packages: Sequence[str]) -> None:
 
     @staticmethod
     def _replace_min(fname: str) -> None:
-        req = open(fname).read().replace(">=", "==")
-        open(fname, "w").write(req)
+        req = open(fname, encoding="utf-8").read().replace(">=", "==")
+        open(fname, "w", encoding="utf-8").write(req)
 
     @staticmethod
     def replace_oldest_ver(requirement_fnames: Sequence[str] = REQUIREMENT_FILES_ALL) -> None:
@@ -212,7 +212,7 @@ def copy_replace_imports(
 
                 for source_import, target_import in zip(source_imports, target_imports):
                     for i, ln in enumerate(py):
-                        py[i] = re.sub(rf"(?!_){source_import}(?!_)", target_import, ln)
+                        py[i] = re.sub(rf"([^_]|^){source_import}([^_\w]|$)", rf"\1{target_import}\2", ln)
 
                 if target_dir:
                     fp_new = fp.replace(source_dir, target_dir)

.github/actions/pkg-check/action.yml

@@ -5,6 +5,7 @@ inputs:
   pkg-name:
     description: package name inside lightning.*
     required: true
+    default: "lightning"
   nb-dirs:
     description: nb of packages in the wrap/distribution
     required: false
@@ -21,10 +22,7 @@ runs:
   - name: Source check
     env:
       PACKAGE_NAME: ${{ inputs.pkg-name }}
-    run: |
-      python setup.py check --metadata --strict
-      # TODO: fix reimporting (F811) which is aftefact after rplacing var assigne with import in meta package
-      flake8 src/lightning/ --ignore E402,F401,E501,W391,E303,F811
+    run: python setup.py check --metadata --strict
     shell: bash
 
   - name: Create package
@@ -40,10 +38,6 @@ runs:
       # python setup.py clean
     shell: bash
 
-  - name: copy/export pkg
-    run: cp dist/* pypi/
-    shell: bash
-
   - name: Unzip packages
     if: ${{ inputs.pkg-name != '' }}
     working-directory: dist

.github/workflows/ci-app-examples.yml

@@ -80,6 +80,8 @@ jobs:
         run: npm install -g yarn
 
       - name: Install Lightning as top-level
+        env:
+          PACKAGE_NAME: app
         run: pip install -e .
         shell: bash
 

.github/workflows/ci-app-tests.yml

@@ -64,10 +64,11 @@ jobs:
           pip list
 
       - name: DocTests App
-        # todo: run doctest laos on mono-package
-        if: ${{ matrix.pkg-name != 'LAI' }}
         working-directory: src
-        run: python -m pytest lightning_app --ignore-glob="**/cli/*-template/**"
+        run: |
+          pip install pytest -q
+          scope=$(python -c "print('lightning/app' if '${{matrix.pkg-name}}' == 'LAI' else 'lightning_app')")
+          python -m pytest $scope --ignore-glob="**/cli/*-template/**"
 
       - name: Get pip cache
         id: pip-cache

.github/workflows/ci-lite-test-full.yml

@@ -83,10 +83,11 @@ jobs:
         pip list
 
     - name: DocTests Lite
-      # todo: run docstest laos on mono-package
-      if: ${{ matrix.pkg-name != 'LAI' }}
       working-directory: src
-      run: python -m pytest lightning_lite
+      run: |
+        pip install pytest -q
+        scope=$(python -c "print('lightning/lite' if '${{matrix.pkg-name}}' == 'LAI' else 'lightning_lite')")
+        python -m pytest $scope
 
     # Note: This uses an internal pip API and may not always work
     # https://github.com/actions/cache/blob/master/examples.md#multiple-oss-in-a-workflow

.github/workflows/ci-pkg-install.yml

@@ -17,141 +17,40 @@ defaults:
 
 jobs:
 
-  init-temp:
-    runs-on: ubuntu-20.04
-    steps:
-    - run: mkdir pypi && touch pypi/.placeholder
-    - uses: actions/upload-artifact@v3
-      with:
-        name: ci-packages-${{ github.sha }}
-        path: pypi
-
-  install-standalone:
-    needs: init-temp
+  install-pkg:
     runs-on: ${{ matrix.os }}
     strategy:
-      fail-fast: true
-      max-parallel: 1
+      fail-fast: false
       matrix:
         os: [ubuntu-20.04, ubuntu-22.04, macOS-11, macOS-12, windows-2022]
-        pkg: ["app", "lite", "pytorch"]
-        python-version: [3.8]  # , 3.9
+        pkg: ["app", "lite", "pytorch", ""]
+        python-version: ["3.7" , "3.10"]
 
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Install pytest extensions
+        # TODO: for mono-package create & load requirements/test.txt
+        if: ${{ matrix.pkg != '' }}
+        run: grep 'pytest>' requirements/${{ matrix.pkg }}/test.txt | xargs -0 pip install
+
       - name: DocTests actions
         working-directory: .actions/
         run: |
-          grep 'pytest>' ../requirements/pytorch/test.txt | xargs -0 pip install
+          pip install pytest -q
           python -m pytest setup_tools.py
 
-      - uses: actions/download-artifact@v3
-        with:
-          name: ci-packages-${{ github.sha }}
-          path: pypi
-      - run: ls -lh pypi/
-
       - run: python -c "print('NB_DIRS=' + str(2 if '${{ matrix.pkg }}' == 'pytorch' else 1))" >> $GITHUB_ENV
       - uses: ./.github/actions/pkg-check
         with:
           pkg-name: ${{ matrix.pkg }}
           nb-dirs: ${{ env.NB_DIRS }}
 
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ci-packages-${{ github.sha }}
-          path: pypi
-
-      - uses: ./.github/actions/pkg-install
-
-  install-meta-src:
-    needs: install-standalone
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      # max-parallel: 1
-      matrix:
-        os: [ubuntu-20.04, ubuntu-22.04, macOS-11, macOS-12, windows-2022]
-        pkg: ["", "lightning"]
-        python-version: [3.8]  # , 3.9
-
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - run: mkdir -p pypi
-      - uses: actions/download-artifact@v3
-        if: matrix.pkg != ''
-        with:
-          name: ci-packages-${{ github.sha }}
-          path: pypi
-      - run: ls -lh pypi/
-
-      - uses: ./.github/actions/pkg-check
-        with:
-          pkg-name: ${{ matrix.pkg }}
-
       - uses: ./.github/actions/pkg-install
-        with:
-          pkg-name: "lightning"
-          pip-flags: "-U --pre --find-links ../pypi/"
-
-      - name: Run CLI
-        run: python -m lightning --version
-
-  install-meta-pypi:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      # max-parallel: 1
-      matrix:
-        os: [ubuntu-20.04, ubuntu-22.04, macOS-11, macOS-12, windows-2022]
-        python-version: [3.8]  # , 3.9
-
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Dowload package
-        # todo: download also lite after it is fist published
-        run: |
-          pip install -q -r .actions/requirements.txt
-          for pkg in 'app' 'pytorch' ; do
-            python .actions/assistant.py download_package --package "$pkg" --folder pypi
-          done
-          ls -lh pypi/
-
-      - name: Unzip packages
-        working-directory: pypi
-        run: for file in `ls *.gz`; do tar -xzf $file; done
-      - name: Show upacked pkgs
-        if: runner.os == 'linux'
-        run: |
-          sudo apt install -y tree
-          tree pypi/ -L 3
-
-      - name: Miror source
-        run: |
-          pip install -q -r .actions/requirements.txt
-          python .actions/assistant.py mirror_pkg2source --pypi_folder pypi --src_folder src
-          ls -R src/
-
-      - uses: ./.github/actions/pkg-check
-        with:
-          pkg-name: "lightning"
-
-      - uses: ./.github/actions/pkg-install
-        with:
-          pkg-name: "lightning"
-          pip-flags: "-U --pre --find-links ../pypi/"
 
       - name: Run CLI
+        if: ${{ matrix.pkg == '' }}
         run: python -m lightning --version

.github/workflows/ci-pytorch-test-full.yml

@@ -111,14 +111,15 @@ jobs:
       env:
         PACKAGE_NAME: ${{ matrix.pkg-name }}
       run: |
-        pip install -e . pytest --upgrade --find-links ${TORCH_URL}
+        pip install -e . --upgrade --find-links ${TORCH_URL}
         pip list
 
     - name: DocTests PL
-      # todo: run docstest laos on mono-package
-      if: ${{ matrix.pkg-name != 'LAI' }}
       working-directory: ./src
-      run: python -m pytest pytorch_lightning
+      run: |
+        pip install pytest -q
+        scope=$(python -c "print('lightning/pytorch' if '${{matrix.pkg-name}}' == 'LAI' else 'pytorch_lightning')")
+        python -m pytest $scope
 
     - name: Get pip cache dir
       id: pip-cache

docs/source-lit/conf.py

@@ -398,13 +398,13 @@ def find_source():
 from typing import Optional
 
 import torch
-import pytorch_lightning as pl
+import lightning.pytorch as pl
 from torch import nn
 from torch.utils.data import IterableDataset, DataLoader, Dataset
-from pytorch_lightning import LightningDataModule, LightningModule, Trainer, seed_everything
-from pytorch_lightning.callbacks import Callback
-from pytorch_lightning.cli import LightningCLI
-from pytorch_lightning.utilities import (
+from lightning.pytorch import LightningDataModule, LightningModule, Trainer, seed_everything
+from lightning.pytorch.callbacks import Callback
+from lightning.pytorch.cli import LightningCLI
+from lightning.pytorch.utilities import (
     _APEX_AVAILABLE,
     _TORCHVISION_AVAILABLE,
     _TORCH_GREATER_EQUAL_1_10,

requirements/app/base.txt

@@ -1,5 +1,5 @@
 lightning-cloud>=0.5.10
-packaging
+typing-extensions>=4.0.0, <4.3.1
 deepdiff>=5.7.0, <=5.8.1
 starsessions>=1.2.1, <2.0 # strict
 fsspec>=2022.5.0, <=2022.7.1

requirements/base.txt

@@ -1,2 +0,0 @@
-pytorch_lightning>=1.6.5
-lightning_app>=0.5.2

setup.py

@@ -25,12 +25,13 @@
     In case you want to install just one package you need to export env. variable before calling `pip`
 
      - for `pytorch-lightning` use `export PACKAGE_NAME=pytorch ; pip install .`
+     - for `lightning-lite` use `export PACKAGE_NAME=lite ; pip install .`
      - for `lightning-app` use `export PACKAGE_NAME=app ; pip install .`
 
 3. Building packages as sdist or binary wheel and installing or publish to PyPI afterwords you use command
     `python setup.py sdist` or `python setup.py bdist_wheel` accordingly.
    In case you want to build just a particular package you would use exporting env. variable as above:
-   `export PACKAGE_NAME=pytorch|app ; python setup.py sdist bdist_wheel`
+   `export PACKAGE_NAME=pytorch|app|lite ; python setup.py sdist bdist_wheel`
 
 4. Automated releasing with GitHub action is natural extension of 3) is composed of three consecutive steps:
     a) determine which packages shall be released based on version increment in `__version__.py` and eventually
@@ -102,9 +103,7 @@ def _load_py_module(name: str, location: str) -> ModuleType:
     if _PACKAGE_NAME not in _PACKAGE_MAPPING:
         _SETUP_TOOLS.set_version_today(os.path.join(_PATH_SRC, "lightning", "__version__.py"))
 
-    for lit_name, pkg_name in _PACKAGE_MAPPING.items():
-        # fixme: if we run creation of meta pkg against stable we shall pull the source
-        _SETUP_TOOLS.create_meta_package(os.path.join(_PATH_ROOT, "src"), pkg_name, lit_name)
+    _SETUP_TOOLS.create_mirror_package(os.path.join(_PATH_ROOT, "src"), _PACKAGE_MAPPING)
 
     _SETUP_MODULE = _load_py_module(name="pkg_setup", location=_PATH_SETUP)
     _SETUP_MODULE._adjust_manifest(pkg_name=_REAL_PKG_NAME)

src/lightning/__init__.py

@@ -1,5 +1,6 @@
 """Root package info."""
 import logging
+import os
 from typing import Any
 
 _DETAIL = 15  # between logging.INFO and logging.DEBUG, used for logging in production use cases
@@ -39,6 +40,10 @@ def _detail(self: Any, message: str, *args: Any, **kwargs: Any) -> None:
 from lightning.pytorch.trainer import Trainer  # noqa: E402
 from lightning.pytorch.utilities.seed import seed_everything  # noqa: E402
 
+import lightning.app  # isort: skip # noqa: E402
+
+lightning.app._PROJECT_ROOT = os.path.dirname(lightning.app._PROJECT_ROOT)
+
 __all__ = [
     "LightningApp",
     "LightningFlow",

src/lightning/__setup__.py

@@ -30,21 +30,16 @@ def _adjust_manifest(**kwargs: Any) -> None:
     assert os.path.isfile(manifest_path)
     with open(manifest_path) as fp:
         lines = [ln.rstrip() for ln in fp.readlines()]
-    if kwargs["pkg_name"] == "lightning":
-        lines += [
-            "recursive-include src/lightning *.md",
-            "include requirements/base.txt",
-            # fixme: this is strange, this shall work with setup find package - include
-            "prune src/lightning_app",
-            "prune src/pytorch_lightning",
-        ]
-    else:
-        lines += [
-            "recursive-include src *.md",
-            "recursive-include requirements *.txt",
-            "recursive-include src/lightning_app/ui *",
-            "recursive-include src/lightning_app/cli/*-template *",  # Add templates as build-in
-        ]
+    lines += [
+        "recursive-include src/lightning *.md",
+        "recursive-include requirements *.txt",
+        "recursive-include src/lightning/ui *",
+        "recursive-include src/lightning/cli/*-template *",  # Add templates as build-in
+        # fixme: this is strange, this shall work with setup find package - include
+        "prune src/lightning_app",
+        "prune src/lightning_lite",
+        "prune src/pytorch_lightning",
+    ]
     with open(manifest_path, "w") as fp:
         fp.writelines([ln + os.linesep for ln in lines])
 
@@ -55,12 +50,12 @@ def _setup_args(**kwargs: Any) -> Dict[str, Any]:
     _long_description = _SETUP_TOOLS.load_readme_description(
         _PROJECT_ROOT, homepage=_about.__homepage__, version=_version.version
     )
-    _include_pkgs = ["lightning", "lightning.*"] if kwargs["pkg_name"] == "lightning" else ["*"]
+    _include_pkgs = ["lightning", "lightning.*"]
 
     # TODO: consider invaliding some additional arguments from packages, for example if include data or safe to zip
 
     # TODO: remove this once lightning-ui package is ready as a dependency
-    _SETUP_TOOLS._download_frontend(_PROJECT_ROOT)
+    _SETUP_TOOLS._download_frontend(os.path.join(_SOURCE_ROOT, "lightning", "app"))
 
     return dict(
         name="lightning",

src/lightning/__version__.py

@@ -1 +1 @@
-version = "YYYY.-M.-D"
+version = "1.8.0rc0"