-
Notifications
You must be signed in to change notification settings - Fork 4.5k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[Draft] Python Wheels for PyPi #2010
Changes from 17 commits
983e08b
042c912
a156cfd
50e8ca6
40e8f8f
85f75c0
a6beac7
44e7eb3
2f9b7ff
ab845cd
765f71a
f59468c
7b7f2b7
523d9a2
37dd50d
292705c
79c6f45
65fea9d
ee1abe7
4d45c59
c09212b
53e24d9
1bac7e8
144929e
a19d7ef
b84672a
52b8047
6ecd1ff
4ff2707
e8d6bf0
1d2aca2
d33cbad
1bbb065
22ea5e6
292028a
4eeb943
e4fae99
e20d88c
b7be6fb
a76d33a
3552e94
b996ff1
3d6fade
a971493
eabe76f
461c050
180d081
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,61 @@ | ||
name: "Create Python Wheels" | ||
|
||
on: | ||
push: | ||
branches: [ master ] | ||
pull_request: | ||
branches: [ master ] | ||
# release: | ||
# types: [created] | ||
|
||
env: | ||
STAN_BACKEND: "PYSTAN,CMDSTANPY" | ||
# Env vars, required because CIBW may build inside a Docker container | ||
CIBW_ENVIRONMENT: STAN_BACKEND="PYSTAN,CMDSTANPY" | ||
# Python versions for cibuildwheel | ||
CIBW_BUILD: "cp38-*" | ||
# Architectures for cibuildwheel | ||
CIBW_ARCHS: "native" | ||
# Use pypa/build as the build frontend | ||
CIBW_BUILD_FRONTEND: "build" | ||
|
||
jobs: | ||
make-wheels-macos-linux: | ||
name: ${{ matrix.python-version }}-${{ matrix.architecture }}-${{ matrix.os }} | ||
runs-on: ${{ matrix.os }} | ||
strategy: | ||
matrix: | ||
os: | ||
- "macos-latest" | ||
- "ubuntu-latest" | ||
python-version: | ||
- "3.8" | ||
architecture: | ||
- x64 | ||
|
||
fail-fast: false | ||
|
||
steps: | ||
- name: "Checkout repo" | ||
uses: actions/checkout@v2 | ||
|
||
- name: "Set up Python" | ||
uses: actions/setup-python@v2 | ||
with: | ||
python-version: ${{ matrix.python-version }} | ||
architecture: ${{ matrix.architecture }} | ||
|
||
- name: "Install cibuildwheel" | ||
run: | | ||
python -m pip install --upgrade pip | ||
python -m pip install cibuildwheel build | ||
|
||
- name: "Create wheel" | ||
run: | | ||
cd python && python -m cibuildwheel --output-dir wheelhouse | ||
|
||
- name: "Upload wheel as artifact" | ||
uses: actions/upload-artifact@v2 | ||
with: | ||
name: ${{ matrix.os }}-wheel | ||
path: "./**/*.whl" |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
[build-system] | ||
requires = [ | ||
"setuptools>=42", | ||
"wheel", | ||
"pystan~=2.19.1.1", | ||
"cmdstanpy==0.9.77" | ||
] | ||
build-backend = "setuptools.build_meta" |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,12 +1,14 @@ | ||
Cython>=0.22 | ||
cmdstanpy==0.9.68 | ||
cmdstanpy==0.9.77 | ||
pystan~=2.19.1.1 | ||
numpy>=1.15.4 | ||
pandas>=1.0.4 | ||
matplotlib>=2.0.0 | ||
LunarCalendar>=0.0.9 | ||
convertdate>=2.1.2 | ||
holidays>=0.10.2 | ||
setuptools>=42 | ||
setuptools-git>=1.2 | ||
python-dateutil>=2.8.0 | ||
tqdm>=4.36.1 | ||
wheel>=0.37.0 |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -3,10 +3,11 @@ | |
# This source code is licensed under the MIT license found in the | ||
# LICENSE file in the root directory of this source tree. | ||
|
||
import os.path | ||
import os | ||
import pickle | ||
import platform | ||
import sys | ||
import os | ||
from shutil import copy, rmtree | ||
from pkg_resources import ( | ||
normalize_path, | ||
working_set, | ||
|
@@ -16,26 +17,77 @@ | |
from setuptools import setup, find_packages | ||
from setuptools.command.build_py import build_py | ||
from setuptools.command.develop import develop | ||
from setuptools.command.install import install | ||
from setuptools.command.test import test as test_command | ||
from typing import List | ||
from wheel.bdist_wheel import bdist_wheel as _bdist_wheel | ||
|
||
PLATFORM = 'unix' | ||
if platform.platform().startswith('Win'): | ||
PLATFORM = 'win' | ||
|
||
MODEL_DIR = os.path.join('stan', PLATFORM) | ||
MODEL_TARGET_DIR = os.path.join('prophet', 'stan_model') | ||
|
||
CMDSTAN_VERSION = "2.26.1" | ||
|
||
def get_backends_from_env() -> List[str]: | ||
from prophet.models import StanBackendEnum | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This was causing the build system to fail, since the build happens in an isolated environment with only the dependencies needed for |
||
return os.environ.get("STAN_BACKEND", StanBackendEnum.PYSTAN.name).split(",") | ||
|
||
return os.environ.get("STAN_BACKEND", "PYSTAN").split(",") | ||
|
||
def prune_cmdstan_files(cmdstan_dir): | ||
""" | ||
Remove unnecessary folders from the unbundled cmdstan installation. | ||
We only need to keep the files that will be used to execute the model binary at runtime. | ||
""" | ||
remove_dirs = ["stan"] | ||
for dirname in remove_dirs: | ||
rmtree(os.path.join(cmdstan_dir, dirname)) | ||
raw_binaries = ["linux-stanc", "mac-stanc", "windows-stanc"] # These are converted into "stanc" in the Make process | ||
for fname in raw_binaries: | ||
os.remove(os.path.join(cmdstan_dir, f"bin/{fname}")) | ||
|
||
def build_cmdstan_model(target_dir): | ||
import cmdstanpy | ||
cmdstanpy.install_cmdstan(version=CMDSTAN_VERSION, dir=target_dir, overwrite=True) | ||
cmdstan_dir = os.path.join(target_dir, f"cmdstan-{CMDSTAN_VERSION}") | ||
cmdstanpy.set_cmdstan_path(cmdstan_dir) | ||
model_name = 'prophet.stan' | ||
target_name = 'prophet_model.bin' | ||
sm = cmdstanpy.CmdStanModel(stan_file=os.path.join(MODEL_DIR, model_name)) | ||
sm.compile() | ||
copy(sm.exe_file, os.path.join(target_dir, target_name)) | ||
prune_cmdstan_files(cmdstan_dir) | ||
|
||
def build_pystan_model(target_dir): | ||
import pystan | ||
model_name = 'prophet.stan' | ||
target_name = 'prophet_model.pkl' | ||
with open(os.path.join(MODEL_DIR, model_name)) as f: | ||
model_code = f.read() | ||
sm = pystan.StanModel(model_code=model_code) | ||
with open(os.path.join(target_dir, target_name), 'wb') as f: | ||
pickle.dump(sm, f, protocol=pickle.HIGHEST_PROTOCOL) | ||
|
||
def build_models(target_dir): | ||
from prophet.models import StanBackendEnum | ||
for backend in get_backends_from_env(): | ||
StanBackendEnum.get_backend_class(backend).build_model(target_dir, MODEL_DIR) | ||
print(f"Compiling {backend} model") | ||
if backend == "CMDSTANPY": | ||
build_cmdstan_model(target_dir) | ||
elif backend == "PYSTAN": | ||
build_pystan_model(target_dir) | ||
|
||
|
||
class BDistWheelNonPure(_bdist_wheel): | ||
tcuongd marked this conversation as resolved.
Show resolved
Hide resolved
|
||
def finalize_options(self): | ||
_bdist_wheel.finalize_options(self) | ||
self.root_is_pure = False | ||
|
||
|
||
class InstallPlatlib(install): | ||
"""Ensure wheel has correct platlib specification when being repaired by auditwheel.""" | ||
tcuongd marked this conversation as resolved.
Show resolved
Hide resolved
|
||
def finalize_options(self): | ||
install.finalize_options(self) | ||
if self.distribution.has_ext_modules(): | ||
self.install_lib = self.install_platlib | ||
|
||
|
||
class BuildPyCommand(build_py): | ||
|
@@ -129,15 +181,15 @@ def with_project_on_sys_path(self, func): | |
author_email='[email protected]', | ||
license='MIT', | ||
packages=find_packages(), | ||
setup_requires=[ | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ensures that we read from |
||
], | ||
install_requires=install_requires, | ||
python_requires='>=3', | ||
zip_safe=False, | ||
include_package_data=True, | ||
cmdclass={ | ||
'bdist_wheel': BDistWheelNonPure, | ||
'build_py': BuildPyCommand, | ||
'develop': DevelopCommand, | ||
'install': InstallPlatlib, | ||
'test': TestCommand, | ||
}, | ||
test_suite='prophet.tests', | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Temporary, for testing.