Skip to content

DataFusion 41, arrow-rs 52.2, sqlparser 0.49, pyo3 0.21.1 #930

DataFusion 41, arrow-rs 52.2, sqlparser 0.49, pyo3 0.21.1

DataFusion 41, arrow-rs 52.2, sqlparser 0.49, pyo3 0.21.1 #930

Workflow file for this run

name: build_test
on:
pull_request:
types: [ opened, synchronize ]
jobs:
check-rust:
runs-on: ubuntu-20.04
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- uses: prefix-dev/[email protected]
with:
pixi-version: v0.27.1
cache: true
- name: Cache
uses: actions/cache@v3
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-check-rust
path: |
~/.cargo
target
.pixi
- name: Check fmt
run: pixi run check-rs-fmt
- name: Check warnings
run: pixi run check-rs-warnings
- name: Check clippy
run: pixi run check-rs-clippy
test-rust:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os:
- ubuntu-20.04
- windows-2022
- macos-12
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- uses: prefix-dev/[email protected]
with:
pixi-version: v0.27.1
- name: Cache
uses: actions/cache@v3
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-test-rust
path: |
~/.cargo
target
.pixi
- name: start minio server in the background
run: pixi run start-minio &
- name: Test rust
run: |
pixi run test-rs --release
- name: Upload test artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
if: always()
with:
name: vegafusion-rt-test-images
path: |
vegafusion-runtime/tests/output
build-vegafusion-wasm:
runs-on: ubuntu-20.04
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- uses: prefix-dev/[email protected]
with:
pixi-version: v0.27.1
- name: Cache
uses: actions/cache@v3
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-wasm
path: |
~/.cargo
target
.pixi
- name: Build package
run: |
pixi run pack-wasm
- name: Upload artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
with:
name: vegafusion-wasm-packages
path: vegafusion-wasm/pkg/vegafusion-wasm-*.tgz
# Use maturin action to build linux wheels within proper manylinux compatible containers
# (This is why we don't use the pixi "build-py-embed" action)
build-vegafusion-python-embed-linux-64:
runs-on: ubuntu-latest
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- name: Install latest stable Rust toolchain
uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1
with:
toolchain: stable
override: true
- name: Cache rust dependencies
uses: Swatinem/rust-cache@81d053bdb0871dcd3f10763c8cc60d0adc41762b # pin@v1
with:
cache-on-failure: True
- name: Build wheels (Linux)
uses: messense/maturin-action@60d11847b29f81ca5375519a8eb33cc336ba4bfa # [email protected]
with:
command: build
manylinux: 2014
rust-toolchain: stable
args: --release -m vegafusion-python-embed/Cargo.toml --features=protobuf-src --strip
- name: Upload artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
with:
name: vegafusion-python-embed-wheels
path: |
target/wheels/*.tar.gz
target/wheels/*.whl
build-vegafusion-python-embed-linux-arm64:
runs-on: ubuntu-latest
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- name: Install latest stable Rust toolchain
uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1
with:
toolchain: stable
override: true
- name: Cache rust dependencies
uses: Swatinem/rust-cache@81d053bdb0871dcd3f10763c8cc60d0adc41762b # pin@v1
with:
cache-on-failure: True
- name: Download arm64 toolchain
run: |
rustup target add aarch64-unknown-linux-gnu
- name: Build arm64 wheels
uses: messense/maturin-action@60d11847b29f81ca5375519a8eb33cc336ba4bfa # [email protected]
with:
command: build
manylinux: 2_28
rust-toolchain: stable
args: --release -m vegafusion-python-embed/Cargo.toml --features=protobuf-src --strip --target aarch64-unknown-linux-gnu
- name: Upload artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
with:
name: vegafusion-python-embed-wheels
path: |
target/wheels/*.tar.gz
target/wheels/*.whl
build-vegafusion-python-embed-win-64:
runs-on: windows-2022
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- uses: prefix-dev/[email protected]
with:
pixi-version: v0.27.1
- name: Cache
uses: actions/cache@v3
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-python-embed-win-64
path: |
~/.cargo
target
.pixi
- name: Build vegafusion-python-embed
run: |
pixi run build-py-embed
- name: Upload artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
with:
name: vegafusion-python-embed-wheels
path: |
target/wheels/*.tar.gz
target/wheels/*.whl
build-vegafusion-python-embed-osx-64:
runs-on: macos-12
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- uses: prefix-dev/[email protected]
with:
pixi-version: v0.27.1
- name: Cache
uses: actions/cache@v3
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-python-embed-osx-64
path: |
~/.cargo
target
.pixi
- name: Build vegafusion-python-embed
run: |
pixi run build-py-embed
- name: Upload artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
with:
name: vegafusion-python-embed-wheels
path: |
target/wheels/*.tar.gz
target/wheels/*.whl
build-vegafusion-python-embed-osx-arm64:
runs-on: macos-latest
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- uses: prefix-dev/[email protected]
with:
pixi-version: v0.27.1
- name: Cache
uses: actions/cache@v3
with:
key: osx-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-python-embed-osx-arm64
path: |
~/.cargo
target
.pixi
- name: Build vegafusion-python-embed
run: |
pixi run build-py-embed --target aarch64-apple-darwin
- name: Upload artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
with:
name: vegafusion-python-embed-wheels
path: |
target/wheels/*.tar.gz
target/wheels/*.whl
build-vegafusion-packages:
runs-on: ubuntu-20.04
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- uses: prefix-dev/[email protected]
with:
pixi-version: v0.27.1
- name: Cache
uses: actions/cache@v3
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-packages
path: |
~/.cargo
target
.pixi
- name: Build vegafusion wheel
run: |
pixi run build-py-vegafusion
- name: Upload artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
with:
name: vegafusion-packages
path: |
python/vegafusion/dist/*.whl
python/vegafusion/dist/*.tar.gz
build-vegafusion-jupyter-packages:
runs-on: ubuntu-20.04
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- uses: prefix-dev/[email protected]
with:
pixi-version: v0.27.1
- name: Cache
uses: actions/cache@v3
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-jupyter-packages
path: |
~/.cargo
target
.pixi
- name: Build vegafusion-jupyter wheel
run: |
echo JupyterLab Version: `pixi run jupyter-lab --version`
pixi run build-py-jupyter
- name: Upload artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
with:
name: vegafusion-jupyter-packages
path: |
python/vegafusion-jupyter/dist/*.whl
python/vegafusion-jupyter/dist/*.tar.gz
test-vegafusion-python-linux-64:
runs-on: ubuntu-20.04
needs: [build-vegafusion-python-embed-linux-64, build-vegafusion-packages]
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- name: Setup Python
uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # [email protected]
with:
python-version: '3.11'
- name: Install fonts on Linux
if: runner.os == 'Linux'
run: |
echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | sudo debconf-set-selections
sudo apt-get install ttf-mscorefonts-installer
- name: Download vegafusion-python-embed wheel
uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2
with:
name: vegafusion-python-embed-wheels
path: target/wheels/
- name: Download vegafusion wheel
uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2
with:
name: vegafusion-packages
path: target/wheels/
- name: install wheels
working-directory: target/wheels/
run: |
ls -la
python -m pip install vegafusion-*.whl
python -m pip install vegafusion_python_embed-*manylinux_2_17_x86_64*.whl
python -m pip install pytest vega-datasets polars-lts-cpu "duckdb>=1.0" "vl-convert-python>=1.0.1rc1" scikit-image "pandas>=2.2"
- name: Test lazy imports
working-directory: python/vegafusion/
run: python checks/check_lazy_imports.py
- name: Test vegafusion
working-directory: python/vegafusion/
run: pytest
test-vegafusion-python-osx-64:
runs-on: macos-13
needs: [build-vegafusion-python-embed-osx-64, build-vegafusion-packages]
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- name: Setup Python
uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # [email protected]
with:
python-version: '3.10'
- name: Download vegafusion-python-embed wheel
uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2
with:
name: vegafusion-python-embed-wheels
path: target/wheels/
- name: Download vegafusion wheel
uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2
with:
name: vegafusion-packages
path: target/wheels/
- name: install wheels
working-directory: target/wheels/
run: |
ls -la
python -m pip install vegafusion-*.whl
python -m pip install vegafusion_python_embed-*macosx_10_*_x86_64.whl
python -m pip install pytest vega-datasets polars-lts-cpu "duckdb>=1.0" vl-convert-python scikit-image "pandas>=2.2"
python -m pip install pyarrow==10.0 altair==5.1.2
- name: Test vegafusion
working-directory: python/vegafusion/
run: pytest
test-vegafusion-python-win-64:
runs-on: windows-2022
needs: [build-vegafusion-python-embed-win-64, build-vegafusion-packages]
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- name: Setup Python
uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # [email protected]
with:
python-version: '3.8'
- name: Download vegafusion-python-embed wheel
uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2
with:
name: vegafusion-python-embed-wheels
path: target/wheels/
- name: Download vegafusion wheel
uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2
with:
name: vegafusion-packages
path: target/wheels/
- name: install wheels
working-directory: target/wheels/
shell: powershell
run: |
Get-ChildItem -Force
$vegafusion = Get-ChildItem -Name "vegafusion-*.whl" | Select-Object -First 1
$vegafusion_python_embed = Get-ChildItem -Name "vegafusion_python_embed-*win_amd64.whl" | Select-Object -First 1
python -m pip install $vegafusion
python -m pip install $vegafusion_python_embed
python -m pip install pytest vega-datasets polars[timezone] "duckdb>=1.0" vl-convert-python scikit-image
- name: Test vegafusion
working-directory: python/vegafusion/
run: pytest
test-vegafusion-jupyter-linux-64:
runs-on: ubuntu-20.04
needs:
- build-vegafusion-python-embed-linux-64
- build-vegafusion-packages
- build-vegafusion-jupyter-packages
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- uses: prefix-dev/[email protected]
with:
pixi-version: v0.27.1
- name: Cache
uses: actions/cache@v3
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-test-vegafusion-jupyter-linux-64
path: |
.pixi
- name: Install Chrome
uses: browser-actions/setup-chrome@f0ff752add8c926994566c80b3ceadfd03f24d12 # pin@latest
with:
chrome-version: stable
- name: Download vegafusion-python-embed wheel
uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2
with:
name: vegafusion-python-embed-wheels
path: target/wheels/
- name: Download vegafusion wheel
uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2
with:
name: vegafusion-packages
path: target/wheels/
- name: Download vegafusion-jupyter wheel
uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2
with:
name: vegafusion-jupyter-packages
path: target/wheels/
- name: install wheels
run: |
ls -la target/wheels/
pixi run pip install target/wheels/vegafusion_python_embed-*manylinux_2_17_x86_64*.whl
pixi run pip install target/wheels/vegafusion-*.whl
pixi run pip install target/wheels/vegafusion_jupyter-*.whl
pixi run pip install chromedriver-binary-auto
pixi run pip list
- name: Test vegafusion jupyter
run: |
VEGAFUSION_TEST_HEADLESS=1 pixi run pytest python/vegafusion-jupyter/tests
- name: Upload test artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
if: always()
with:
name: vegafusion-jupyter-test-failures
path: |
python/vegafusion-jupyter/tests/failures/*
build-vegafusion-server-linux-64:
runs-on: ubuntu-20.04
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- uses: prefix-dev/[email protected]
with:
pixi-version: v0.27.1
- name: Cache
uses: actions/cache@v3
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-linux-64
path: |
~/.cargo
target
.pixi
- name: Build vegafusion-server
run: |
pixi run build-rs-vegafusion-server
- name: zip executable
uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # [email protected]
with:
files: target/release/vegafusion-server
dest: vegafusion-server-linux-64.zip
- name: Upload artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
with:
name: vegafusion-server
path: |
vegafusion-server-*
# # Below is an Attempt to cross compile to linux aarch64 within conda environment.
# # It fails in compiling protobuf-src. So instead we fall back to the direct use of
# # cargo (without involving conda).
#
# build-vegafusion-server-linux-arm64:
# runs-on: ubuntu-20.04
# steps:
# - name: Check out repository code
# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
# - name: Install pixi
# run: curl -fsSL https://pixi.sh/install.sh | bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH
# - name: Cache
# uses: actions/cache@v3
# with:
# key: linux-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-server
# path: |
# ~/.cargo
# target
# .pixi
# - name: Build vegafusion-server
# run: |
# pixi run python automation/download_rust_target.py aarch64-unknown-linux-gnu
# pixi add gcc_linux-aarch64 -p linux-64
# export RUSTFLAGS="-C linker=aarch64-linux-gnu-gcc"
# pixi run build-rs-vegafusion-server --features=protobuf-src --target aarch64-unknown-linux-gnu
# - name: zip executable
# uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # [email protected]
# with:
# files: target/aarch64-unknown-linux-gnu/release/vegafusion-server
# dest: vegafusion-server-linux-arm64.zip
# - name: Upload artifacts
# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
# with:
# name: vegafusion-server
# path: |
# vegafusion-server-*
build-vegafusion-server-linux-arm64:
runs-on: ubuntu-latest
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- name: Install latest stable Rust toolchain
uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1
with:
toolchain: stable
override: true
- name: Cache rust dependencies
uses: Swatinem/rust-cache@81d053bdb0871dcd3f10763c8cc60d0adc41762b # pin@v1
with:
cache-on-failure: True
- name: Download arm64 toolchain
run: |
rustup target add aarch64-unknown-linux-gnu
- name: Download gcc for cross compilation
run: |
sudo apt-get update
sudo apt-get install gcc-aarch64-linux-gnu
- name: Build vegafusion-server
uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # pin@v1
env:
RUSTFLAGS: "-C linker=aarch64-linux-gnu-gcc"
with:
command: build
args: -p vegafusion-server --release --features=protobuf-src --target=aarch64-unknown-linux-gnu
- name: zip executable
uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # [email protected]
with:
files: target/aarch64-unknown-linux-gnu/release/vegafusion-server
dest: vegafusion-server-linux-arm64.zip
- name: Upload artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
with:
name: vegafusion-server
path: |
vegafusion-server-*
build-vegafusion-server-win-64:
runs-on: windows-2022
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- uses: prefix-dev/[email protected]
with:
pixi-version: v0.27.1
- name: Cache
uses: actions/cache@v3
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-win-64b
path: |
~/.cargo
target
.pixi
- name: Build vegafusion-server
run: |
pixi install -vvv
pixi run build-rs-vegafusion-server
- name: zip executable
uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # [email protected]
with:
files: target/release/vegafusion-server.exe
dest: vegafusion-server-win-64.zip
- name: Upload artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
with:
name: vegafusion-server
path: |
vegafusion-server-*
build-vegafusion-server-osx-64:
runs-on: macos-13
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- uses: prefix-dev/[email protected]
with:
pixi-version: v0.27.1
- name: Cache
uses: actions/cache@v3
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-osx-64
path: |
~/.cargo
target
.pixi
- name: Build vegafusion-server
run: |
pixi run build-rs-vegafusion-server
- name: zip executable
uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # [email protected]
with:
files: target/release/vegafusion-server
dest: vegafusion-server-osx-64.zip
- name: Upload artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
with:
name: vegafusion-server
path: |
vegafusion-server-*
build-vegafusion-server-osx-arm64:
runs-on: macos-latest
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- uses: prefix-dev/[email protected]
with:
pixi-version: v0.27.1
- name: Cache
uses: actions/cache@v3
with:
key: osx-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-osx-arm64
path: |
~/.cargo
target
.pixi
- name: Build vegafusion-server
run: |
pixi run build-rs-vegafusion-server --target aarch64-apple-darwin
- name: zip executable
uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # [email protected]
with:
files: target/aarch64-apple-darwin/release/vegafusion-server
dest: vegafusion-server-osx-arm64.zip
- name: Upload artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
with:
name: vegafusion-server
path: |
vegafusion-server-*