From cc91dbfc45d57d45e4018fd6a1d0f2decda57b24 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 8 Sep 2022 16:55:21 -0500 Subject: [PATCH 01/28] Initial dependency file. --- .github/workflows/dependency-files.yml | 8 ++ dependencies.yaml | 145 +++++++++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 .github/workflows/dependency-files.yml create mode 100644 dependencies.yaml diff --git a/.github/workflows/dependency-files.yml b/.github/workflows/dependency-files.yml new file mode 100644 index 00000000000..aeda16f41cb --- /dev/null +++ b/.github/workflows/dependency-files.yml @@ -0,0 +1,8 @@ +name: dependency-files + +on: + pull_request: + +jobs: + check-generated-files: + uses: rapidsai/shared-action-workflows/.github/workflows/dependency-files.yaml@main diff --git a/dependencies.yaml b/dependencies.yaml new file mode 100644 index 00000000000..b9a011ff582 --- /dev/null +++ b/dependencies.yaml @@ -0,0 +1,145 @@ +# Dependency list for https://github.com/rapidsai/dependency-file-generator +files: + all: + generate: conda + matrix: + cuda: ["11.5"] + arch: [x86_64] + includes: + - build + - develop + - run + - test + - doc + - notebook + test: + generate: none + includes: + - test +channels: + - rapidsai + - conda-forge +dependencies: + conda_and_requirements: + common: + build: + - arrow-cpp=9 + - cmake>=3.20.1,!=3.23.0 + - cmake_setuptools>=0.1.3 + - cython>=0.29,<0.30 + - dlpack>=0.5,<0.6.0a0 + - librdkafka=1.7.0 + - protobuf>=3.20.1,<3.21.0a0 + - pyarrow=9 + - python>=3.8,<3.10 + - rmm=22.10.* + - scikit-build>=0.13.1 + # Not sure if these are used anywhere? + # - double-conversion + # - rapidjson + run: + - cachetools + - cupy>=9.5.0,<12.0.0a0 # TODO: cupy for conda, cupy-cuda115 for requirements? + - dask>=2022.7.1 + - distributed>=2022.7.1 + - fsspec>=0.6.0 + - numba>=0.54 + - numpy>=1.19 + - nvtx>=0.2.1 + - packaging + - pandas>=1.0,<1.5.0dev0 + - python-confluent-kafka=1.7.0 + - streamz + - typing_extensions + develop: + - black=22.3.0 + - cmakelang=0.6.13 + - doxygen=1.8.20 + - flake8=3.8.3 + - gcovr>=5.0 + - isort=5.10.1 + - pre-commit + - pydocstyle=6.1.1 + test: + - aiobotocore>=2.2.0 + - boto3>=1.21.21 + - botocore>=1.24.21 + - dask-cuda=22.10.* # TODO: This only looks like a testing dependency? + - fastavro>=0.22.9 + - hypothesis + - mimesis<4.1 + - moto>=3.1.6 + - pytest + - pytest-benchmark + - pytest-cases + - pytest-cov + - pytest-xdist + - python-snappy>=0.6.0 + - pytorch<1.12.0 + - s3fs>=2022.3.0 + - scipy + - transformers<=4.10.3 + - werkzeug<2.2.0 # Temporary transient dependency pinning to avoid URL-LIB3 + moto timeouts + doc: + - myst-nb + - nbsphinx + - numpydoc + - pandoc<=2.0.0 + - pydata-sphinx-theme + - sphinx + - sphinx-autobuild + - sphinx-copybutton + - sphinx-markdown-tables + - sphinxcontrib-websupport + notebook: + - ipython + - notebook>=0.5.0 + specific: + - matrix: + cuda: "11.5" + build: + - cuda-python>=11.5,<11.7.1 + conda: + common: + build: + - c-compiler + - cxx-compiler + develop: + - clang=11.1.0 + - clang-tools=11.1.0 + run: + - pip + - pip: + - git+https://github.com/python-streamz/streamz.git@master + - pyorc + - ptxcompiler # Not available on PyPI, may be needed? + specific: + - matrix: + arch: x86_64 + build: + - gcc_linux-64=9.* + - sysroot_linux-64==2.17 + - matrix: + arch: aarch64 + build: + - gcc_linux-aarch64=9.* + - sysroot_linux-aarch64==2.17 + - matrix: + cuda: "11.5" + build: + - cudatoolkit=11.5 + - matrix: + arch: x86_64 + cuda: "11.5" + build: + - nvcc_linux-64=11.5 + - matrix: + arch: aarch64 + cuda: "11.5" + build: + - nvcc_linux-aarch64=11.5 + requirements: + common: + run: + - git+https://github.com/python-streamz/streamz.git@master + - pyorc From b43682c3a525e410eea45862e10bcf160b9b740c Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 8 Sep 2022 16:55:47 -0500 Subject: [PATCH 02/28] Migrate to generated dependency file. --- CONTRIBUTING.md | 7 +- .../all_cuda-115_arch-x86_64.yaml | 84 +++++++++++++++++ conda/environments/cudf_dev_cuda11.5.yml | 91 ------------------- 3 files changed, 86 insertions(+), 96 deletions(-) create mode 100644 conda/environments/all_cuda-115_arch-x86_64.yaml delete mode 100644 conda/environments/cudf_dev_cuda11.5.yml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 38a57caa5f7..e4316103763 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -99,13 +99,13 @@ cd $CUDF_HOME **Note:** Using a conda environment is the easiest way to satisfy the library's dependencies. Instructions for a minimal build environment without conda are included below. -- Create the conda development environment `cudf_dev`: +- Create the conda development environment: ```bash # create the conda environment (assuming in base `cudf` directory) # note: RAPIDS currently doesn't support `channel_priority: strict`; # use `channel_priority: flexible` instead -conda env create --name cudf_dev --file conda/environments/cudf_dev_cuda11.5.yml +conda env create --name cudf_dev --file conda/environments/all_cuda-115_arch-x86_64.yaml # activate the environment conda activate cudf_dev ``` @@ -114,9 +114,6 @@ conda activate cudf_dev development environment may also need to be updated if dependency versions or pinnings are changed. -- For other CUDA versions, check the corresponding `cudf_dev_cuda*.yml` file in - `conda/environments/`. - #### Building without a conda environment - libcudf has the following minimal dependencies (in addition to those listed in the [General diff --git a/conda/environments/all_cuda-115_arch-x86_64.yaml b/conda/environments/all_cuda-115_arch-x86_64.yaml new file mode 100644 index 00000000000..493883c540c --- /dev/null +++ b/conda/environments/all_cuda-115_arch-x86_64.yaml @@ -0,0 +1,84 @@ +# This file is generated by `rapids-dependency-file-generator`. +# To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. +channels: +- rapidsai +- conda-forge +dependencies: +- aiobotocore>=2.2.0 +- arrow-cpp=9 +- black=22.3.0 +- boto3>=1.21.21 +- botocore>=1.24.21 +- c-compiler +- cachetools +- clang-tools=11.1.0 +- clang=11.1.0 +- cmake>=3.20.1,!=3.23.0 +- cmake_setuptools>=0.1.3 +- cmakelang=0.6.13 +- cuda-python>=11.5,<11.7.1 +- cudatoolkit=11.5 +- cupy>=9.5.0,<12.0.0a0 +- cxx-compiler +- cython>=0.29,<0.30 +- dask-cuda=22.10.* +- dask>=2022.7.1 +- distributed>=2022.7.1 +- dlpack>=0.5,<0.6.0a0 +- doxygen=1.8.20 +- fastavro>=0.22.9 +- flake8=3.8.3 +- fsspec>=0.6.0 +- gcc_linux-64=9.* +- gcovr>=5.0 +- hypothesis +- ipython +- isort=5.10.1 +- librdkafka=1.7.0 +- mimesis<4.1 +- moto>=3.1.6 +- myst-nb +- nbsphinx +- notebook>=0.5.0 +- numba>=0.54 +- numpy>=1.19 +- numpydoc +- nvcc_linux-64=11.5 +- nvtx>=0.2.1 +- packaging +- pandas>=1.0,<1.5.0dev0 +- pandoc<=2.0.0 +- pip +- pre-commit +- protobuf>=3.20.1,<3.21.0a0 +- ptxcompiler +- pyarrow=9 +- pydata-sphinx-theme +- pydocstyle=6.1.1 +- pytest +- pytest-benchmark +- pytest-cases +- pytest-cov +- pytest-xdist +- python-confluent-kafka=1.7.0 +- python-snappy>=0.6.0 +- python>=3.8,<3.10 +- pytorch<1.12.0 +- rmm=22.10.* +- s3fs>=2022.3.0 +- scikit-build>=0.13.1 +- scipy +- sphinx +- sphinx-autobuild +- sphinx-copybutton +- sphinx-markdown-tables +- sphinxcontrib-websupport +- streamz +- sysroot_linux-64==2.17 +- transformers<=4.10.3 +- typing_extensions +- werkzeug<2.2.0 +- pip: + - git+https://github.com/python-streamz/streamz.git@master + - pyorc +name: all_cuda-115_arch-x86_64 diff --git a/conda/environments/cudf_dev_cuda11.5.yml b/conda/environments/cudf_dev_cuda11.5.yml deleted file mode 100644 index d8281ca2b75..00000000000 --- a/conda/environments/cudf_dev_cuda11.5.yml +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (c) 2021-2022, NVIDIA CORPORATION. - -name: cudf_dev -channels: - - rapidsai - - nvidia - - rapidsai-nightly - - dask/label/dev - - conda-forge -dependencies: - - c-compiler - - cxx-compiler - - clang=11.1.0 - - clang-tools=11.1.0 - - cupy>=9.5.0,<12.0.0a0 - - rmm=22.10.* - - cmake>=3.20.1,!=3.23.0 - - cmake_setuptools>=0.1.3 - - scikit-build>=0.13.1 - - python>=3.8,<3.10 - - numba>=0.54 - - numpy - - pandas>=1.0,<1.5.0dev0 - - pyarrow=9 - - fastavro>=0.22.9 - - python-snappy>=0.6.0 - - notebook>=0.5.0 - - cython>=0.29,<0.30 - - fsspec>=0.6.0 - - pytest - - pytest-benchmark - - pytest-cases - - pytest-xdist - - sphinx - - sphinxcontrib-websupport - - nbsphinx - - numpydoc - - ipython - - pandoc<=2.0.0 - - cudatoolkit=11.5 - - cuda-python >=11.5,<11.7.1 - - pip - - flake8=3.8.3 - - black=22.3.0 - - isort=5.10.1 - - mypy=0.782 - - doxygen=1.8.20 - - pydocstyle=6.1.1 - - typing_extensions - - pre-commit - - dask>=2022.7.1 - - distributed>=2022.7.1 - - streamz - - arrow-cpp=9 - - dlpack>=0.5,<0.6.0a0 - - double-conversion - - rapidjson - - hypothesis - - sphinx-markdown-tables - - sphinx-copybutton - - sphinx-autobuild - - myst-nb - - scipy - - dask-cuda=22.10.* - - mimesis<4.1 - - packaging - - protobuf>=3.20.1,<3.21.0a0 - - nvtx>=0.2.1 - - cachetools - - transformers<=4.10.3 - - pydata-sphinx-theme - - librdkafka=1.7.0 - - python-confluent-kafka=1.7.0 - - moto>=3.1.6 - - boto3>=1.21.21 - - botocore>=1.24.21 - - aiobotocore>=2.2.0 - - s3fs>=2022.3.0 - - werkzeug<2.2.0 # Temporary transient dependency pinning to avoid URL-LIB3 + moto timeouts - - pytorch<1.12.0 - - pip: - - git+https://github.com/python-streamz/streamz.git@master - - pyorc - - ptxcompiler # [linux64] - - gcc_linux-64=9.* # [linux64] - - sysroot_linux-64==2.17 # [linux64] - - nvcc_linux-64=11.5 - # Un-comment following lines for ARM specific packages. - # - gcc_linux-aarch64=9.* # [aarch64] - # - sysroot_linux-aarch64==2.17 # [aarch64] - # - nvcc_linux-aarch64=11.5 # [aarch64] From e18ebce9acfe0307ff7f1988f88fb825fb920c85 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Thu, 27 Oct 2022 18:13:07 -0500 Subject: [PATCH 03/28] Update versions and TODO notes. --- dependencies.yaml | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index b9a011ff582..cd708a307d9 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -19,12 +19,16 @@ files: channels: - rapidsai - conda-forge +# Not sure whether we need these: +# - nvidia +# - rapidsai-nightly +# - dask/label/dev dependencies: conda_and_requirements: common: build: - arrow-cpp=9 - - cmake>=3.20.1,!=3.23.0 + - cmake>=3.23.1 - cmake_setuptools>=0.1.3 - cython>=0.29,<0.30 - dlpack>=0.5,<0.6.0a0 @@ -32,29 +36,30 @@ dependencies: - protobuf>=3.20.1,<3.21.0a0 - pyarrow=9 - python>=3.8,<3.10 - - rmm=22.10.* + - rmm=22.12.* - scikit-build>=0.13.1 # Not sure if these are used anywhere? # - double-conversion # - rapidjson run: - cachetools + - cubinlinker # TODO: Is this conda or requirements or both? - cupy>=9.5.0,<12.0.0a0 # TODO: cupy for conda, cupy-cuda115 for requirements? - - dask>=2022.7.1 - - distributed>=2022.7.1 + - dask>=2022.9.2 + - distributed>=2022.9.2 - fsspec>=0.6.0 - - numba>=0.54 - - numpy>=1.19 + - numba>=0.56.2 + - numpy # Does this need a lower bound? - nvtx>=0.2.1 - packaging - - pandas>=1.0,<1.5.0dev0 + - pandas>=1.0,<1.6.0dev0 - python-confluent-kafka=1.7.0 - streamz - typing_extensions develop: - black=22.3.0 - cmakelang=0.6.13 - - doxygen=1.8.20 + - doxygen=1.8.20 # We rely on this being installed for pre-commit but pre-commit handles all the rest of this section for us (except pre-commit itself). We can probably remove black, etc. - flake8=3.8.3 - gcovr>=5.0 - isort=5.10.1 @@ -64,10 +69,10 @@ dependencies: - aiobotocore>=2.2.0 - boto3>=1.21.21 - botocore>=1.24.21 - - dask-cuda=22.10.* # TODO: This only looks like a testing dependency? + - dask-cuda=22.12.* # TODO: This only looks like a testing dependency? - fastavro>=0.22.9 - hypothesis - - mimesis<4.1 + - mimesis>=4.1.0 - moto>=3.1.6 - pytest - pytest-benchmark @@ -75,16 +80,16 @@ dependencies: - pytest-cov - pytest-xdist - python-snappy>=0.6.0 - - pytorch<1.12.0 + - pytorch<1.12.0 # We should check and fix all "<=" pinnings - s3fs>=2022.3.0 - - scipy - - transformers<=4.10.3 + - scipy # Does this need a lower bound? + - transformers<=4.10.3 # We should check and fix all "<=" pinnings - werkzeug<2.2.0 # Temporary transient dependency pinning to avoid URL-LIB3 + moto timeouts doc: - myst-nb - nbsphinx - numpydoc - - pandoc<=2.0.0 + - pandoc<=2.0.0 # We should check and fix all "<=" pinnings - pydata-sphinx-theme - sphinx - sphinx-autobuild @@ -98,20 +103,21 @@ dependencies: - matrix: cuda: "11.5" build: - - cuda-python>=11.5,<11.7.1 + - cuda-python>=11.5,<11.7.1 # TODO: Check this, it should be updated very soon for the hotfix conda: common: build: - c-compiler - cxx-compiler develop: + # Are these needed? For what? - clang=11.1.0 - clang-tools=11.1.0 run: - pip - pip: - git+https://github.com/python-streamz/streamz.git@master - - pyorc + - pyorc # I think there is an issue filed about removing this. Maybe need to check that. - ptxcompiler # Not available on PyPI, may be needed? specific: - matrix: From 28e745c36efc0c8acff9ef10a7e8576f0593577a Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 31 Oct 2022 21:48:33 -0500 Subject: [PATCH 04/28] Update to drafted schema changes. --- dependencies.yaml | 258 +++++++++++++++++++++++++--------------------- 1 file changed, 141 insertions(+), 117 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index cd708a307d9..8baf20852b4 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -24,128 +24,152 @@ channels: # - rapidsai-nightly # - dask/label/dev dependencies: - conda_and_requirements: + build: common: - build: - - arrow-cpp=9 - - cmake>=3.23.1 - - cmake_setuptools>=0.1.3 - - cython>=0.29,<0.30 - - dlpack>=0.5,<0.6.0a0 - - librdkafka=1.7.0 - - protobuf>=3.20.1,<3.21.0a0 - - pyarrow=9 - - python>=3.8,<3.10 - - rmm=22.12.* - - scikit-build>=0.13.1 - # Not sure if these are used anywhere? - # - double-conversion - # - rapidjson - run: - - cachetools - - cubinlinker # TODO: Is this conda or requirements or both? - - cupy>=9.5.0,<12.0.0a0 # TODO: cupy for conda, cupy-cuda115 for requirements? - - dask>=2022.9.2 - - distributed>=2022.9.2 - - fsspec>=0.6.0 - - numba>=0.56.2 - - numpy # Does this need a lower bound? - - nvtx>=0.2.1 - - packaging - - pandas>=1.0,<1.6.0dev0 - - python-confluent-kafka=1.7.0 - - streamz - - typing_extensions - develop: - - black=22.3.0 - - cmakelang=0.6.13 - - doxygen=1.8.20 # We rely on this being installed for pre-commit but pre-commit handles all the rest of this section for us (except pre-commit itself). We can probably remove black, etc. - - flake8=3.8.3 - - gcovr>=5.0 - - isort=5.10.1 - - pre-commit - - pydocstyle=6.1.1 - test: - - aiobotocore>=2.2.0 - - boto3>=1.21.21 - - botocore>=1.24.21 - - dask-cuda=22.12.* # TODO: This only looks like a testing dependency? - - fastavro>=0.22.9 - - hypothesis - - mimesis>=4.1.0 - - moto>=3.1.6 - - pytest - - pytest-benchmark - - pytest-cases - - pytest-cov - - pytest-xdist - - python-snappy>=0.6.0 - - pytorch<1.12.0 # We should check and fix all "<=" pinnings - - s3fs>=2022.3.0 - - scipy # Does this need a lower bound? - - transformers<=4.10.3 # We should check and fix all "<=" pinnings - - werkzeug<2.2.0 # Temporary transient dependency pinning to avoid URL-LIB3 + moto timeouts - doc: - - myst-nb - - nbsphinx - - numpydoc - - pandoc<=2.0.0 # We should check and fix all "<=" pinnings - - pydata-sphinx-theme - - sphinx - - sphinx-autobuild - - sphinx-copybutton - - sphinx-markdown-tables - - sphinxcontrib-websupport - notebook: - - ipython - - notebook>=0.5.0 + - output_types: [conda, requirements] + packages: + - arrow-cpp=9 + - cmake>=3.23.1 + - cmake_setuptools>=0.1.3 + - cython>=0.29,<0.30 + - dlpack>=0.5,<0.6.0a0 + - librdkafka=1.7.0 + - protobuf>=3.20.1,<3.21.0a0 + - pyarrow=9 + - python>=3.8,<3.10 + - rmm=22.12.* + - scikit-build>=0.13.1 + # Not sure if these are used anywhere? + # - double-conversion + # - rapidjson + - output_types: conda + packages: + - c-compiler + - cxx-compiler specific: - - matrix: - cuda: "11.5" - build: - - cuda-python>=11.5,<11.7.1 # TODO: Check this, it should be updated very soon for the hotfix - conda: + - output_types: [conda, requirements] + matrices: + - matrix: + cuda: "11.5" + packages: + - cuda-python>=11.5,<11.7.1 # TODO: Check this, it should be updated very soon for the hotfix + - output_types: conda + matrices: + - matrix: + arch: x86_64 + packages: + - gcc_linux-64=9.* + - sysroot_linux-64==2.17 + - matrix: + arch: aarch64 + packages: + - gcc_linux-aarch64=9.* + - sysroot_linux-aarch64==2.17 + - output_types: conda + matrices: + - matrix: + cuda: "11.5" + packages: + - cudatoolkit=11.5 + - output_types: conda + matrices: + - matrix: + packages: # TODO: Test priority of empty matrix! + - matrix: + arch: x86_64 + cuda: "11.5" + packages: + - nvcc_linux-64=11.5 + - matrix: + arch: aarch64 + cuda: "11.5" + packages: + - nvcc_linux-aarch64=11.5 + run: common: - build: - - c-compiler - - cxx-compiler - develop: - # Are these needed? For what? - - clang=11.1.0 - - clang-tools=11.1.0 - run: + - output_types: [conda, requirements] + packages: + - cachetools + - cubinlinker # TODO: Is this conda or requirements or both? + - cupy>=9.5.0,<12.0.0a0 # TODO: cupy for conda, cupy-cuda115 for requirements? + - dask>=2022.9.2 + - distributed>=2022.9.2 + - fsspec>=0.6.0 + - numba>=0.56.2 + - numpy # Does this need a lower bound? + - nvtx>=0.2.1 + - packaging + - pandas>=1.0,<1.6.0dev0 + - python-confluent-kafka=1.7.0 + - streamz + - typing_extensions + - output_types: [conda, requirements] + packages: - pip - pip: - git+https://github.com/python-streamz/streamz.git@master - pyorc # I think there is an issue filed about removing this. Maybe need to check that. - - ptxcompiler # Not available on PyPI, may be needed? - specific: - - matrix: - arch: x86_64 - build: - - gcc_linux-64=9.* - - sysroot_linux-64==2.17 - - matrix: - arch: aarch64 - build: - - gcc_linux-aarch64=9.* - - sysroot_linux-aarch64==2.17 - - matrix: - cuda: "11.5" - build: - - cudatoolkit=11.5 - - matrix: - arch: x86_64 - cuda: "11.5" - build: - - nvcc_linux-64=11.5 - - matrix: - arch: aarch64 - cuda: "11.5" - build: - - nvcc_linux-aarch64=11.5 - requirements: + - ptxcompiler # Not available on PyPI, but may be needed? + - output_types: requirements + packages: + - git+https://github.com/python-streamz/streamz.git@master + - pyorc + develop: + common: + - output_types: [conda, requirements] + packages: + - black=22.3.0 + - cmakelang=0.6.13 + - flake8=3.8.3 + - gcovr>=5.0 + - isort=5.10.1 + - pre-commit + - pydocstyle=6.1.1 + - output_types: conda + packages: + - clang=11.1.0 # Are these clang packages needed? For what? + - clang-tools=11.1.0 # Are these clang packages needed? For what? + - doxygen=1.8.20 # We rely on this being installed for pre-commit but pre-commit handles all the rest of this section for us (except pre-commit itself). We can probably remove black, etc. + test: + common: + - output_types: [conda, requirements] + packages: + - aiobotocore>=2.2.0 + - boto3>=1.21.21 + - botocore>=1.24.21 + - dask-cuda=22.12.* # TODO: This only looks like a testing dependency? + - fastavro>=0.22.9 + - hypothesis + - mimesis>=4.1.0 + - moto>=3.1.6 + - pytest + - pytest-benchmark + - pytest-cases + - pytest-cov + - pytest-xdist + - python-snappy>=0.6.0 + - pytorch<1.12.0 # We should check and fix all "<=" pinnings + - s3fs>=2022.3.0 + - scipy # Does this need a lower bound? + - transformers<=4.10.3 # We should check and fix all "<=" pinnings + - werkzeug<2.2.0 # Temporary transient dependency pinning to avoid URL-LIB3 + moto timeouts + doc: + common: + - output_types: [conda, requirements] + packages: + - myst-nb + - nbsphinx + - numpydoc + - pandoc<=2.0.0 # We should check and fix all "<=" pinnings + - pydata-sphinx-theme + - sphinx + - sphinx-autobuild + - sphinx-copybutton + - sphinx-markdown-tables + - sphinxcontrib-websupport + notebook: common: - run: - - git+https://github.com/python-streamz/streamz.git@master - - pyorc + - output_types: [conda, requirements] + packages: + - ipython + - notebook>=0.5.0 From a08111c59c5a87890d5204a11252587e25d761d4 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 4 Nov 2022 11:04:29 -0500 Subject: [PATCH 05/28] Rerun 0.0.20. --- .../all_cuda-115_arch-x86_64.yaml | 20 +++++++++---------- dependencies.yaml | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/conda/environments/all_cuda-115_arch-x86_64.yaml b/conda/environments/all_cuda-115_arch-x86_64.yaml index 493883c540c..4961797c2da 100644 --- a/conda/environments/all_cuda-115_arch-x86_64.yaml +++ b/conda/environments/all_cuda-115_arch-x86_64.yaml @@ -13,17 +13,18 @@ dependencies: - cachetools - clang-tools=11.1.0 - clang=11.1.0 -- cmake>=3.20.1,!=3.23.0 +- cmake>=3.23.1 - cmake_setuptools>=0.1.3 - cmakelang=0.6.13 +- cubinlinker - cuda-python>=11.5,<11.7.1 - cudatoolkit=11.5 - cupy>=9.5.0,<12.0.0a0 - cxx-compiler - cython>=0.29,<0.30 -- dask-cuda=22.10.* -- dask>=2022.7.1 -- distributed>=2022.7.1 +- dask-cuda=22.12.* +- dask>=2022.9.2 +- distributed>=2022.9.2 - dlpack>=0.5,<0.6.0a0 - doxygen=1.8.20 - fastavro>=0.22.9 @@ -35,18 +36,17 @@ dependencies: - ipython - isort=5.10.1 - librdkafka=1.7.0 -- mimesis<4.1 +- mimesis>=4.1.0 - moto>=3.1.6 - myst-nb - nbsphinx - notebook>=0.5.0 -- numba>=0.54 -- numpy>=1.19 +- numba>=0.56.2 +- numpy - numpydoc -- nvcc_linux-64=11.5 - nvtx>=0.2.1 - packaging -- pandas>=1.0,<1.5.0dev0 +- pandas>=1.0,<1.6.0dev0 - pandoc<=2.0.0 - pip - pre-commit @@ -64,7 +64,7 @@ dependencies: - python-snappy>=0.6.0 - python>=3.8,<3.10 - pytorch<1.12.0 -- rmm=22.10.* +- rmm=22.12.* - s3fs>=2022.3.0 - scikit-build>=0.13.1 - scipy diff --git a/dependencies.yaml b/dependencies.yaml index 8baf20852b4..b7117ed2cd3 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -1,7 +1,7 @@ # Dependency list for https://github.com/rapidsai/dependency-file-generator files: all: - generate: conda + output: conda matrix: cuda: ["11.5"] arch: [x86_64] @@ -13,7 +13,7 @@ files: - doc - notebook test: - generate: none + output: none includes: - test channels: From 78433a5ebe1ea169afa1b046166d6e12ea2ba163 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 4 Nov 2022 11:40:31 -0500 Subject: [PATCH 06/28] Update cuda-python. --- conda/environments/all_cuda-115_arch-x86_64.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/environments/all_cuda-115_arch-x86_64.yaml b/conda/environments/all_cuda-115_arch-x86_64.yaml index 4961797c2da..98f5e3caa33 100644 --- a/conda/environments/all_cuda-115_arch-x86_64.yaml +++ b/conda/environments/all_cuda-115_arch-x86_64.yaml @@ -17,7 +17,7 @@ dependencies: - cmake_setuptools>=0.1.3 - cmakelang=0.6.13 - cubinlinker -- cuda-python>=11.5,<11.7.1 +- cuda-python>=11.7.1,<12.0 - cudatoolkit=11.5 - cupy>=9.5.0,<12.0.0a0 - cxx-compiler From b3a7c601c958cb5b0e9c2524cfcd6b80f04d941e Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 4 Nov 2022 11:45:52 -0500 Subject: [PATCH 07/28] Remove outdated dependencies on double-conversion and rapidjson. --- dependencies.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 8978720fedc..2da31d4c255 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -39,9 +39,6 @@ dependencies: - python>=3.8,<3.10 - rmm=22.12.* - scikit-build>=0.13.1 - # Not sure if these are used anywhere? - # - double-conversion - # - rapidjson - output_types: conda packages: - c-compiler From b14bf807d76b406039679a3e69aa9c795087d943 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 4 Nov 2022 17:00:39 -0500 Subject: [PATCH 08/28] Updating TODO items. --- dependencies.yaml | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 2da31d4c255..0d6ff607fc1 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -28,21 +28,21 @@ dependencies: common: - output_types: [conda, requirements] packages: - - arrow-cpp=9 - cmake>=3.23.1 - cmake_setuptools>=0.1.3 - cython>=0.29,<0.30 - dlpack>=0.5,<0.6.0a0 - - librdkafka=1.7.0 - - protobuf>=3.20.1,<3.21.0a0 - pyarrow=9 - - python>=3.8,<3.10 - rmm=22.12.* - scikit-build>=0.13.1 - output_types: conda packages: + - arrow-cpp=9 - c-compiler - cxx-compiler + - librdkafka=1.7.0 + - protobuf>=3.20.1,<3.21.0a0 + - python>=3.8,<3.10 specific: - output_types: [conda, requirements] matrices: @@ -70,8 +70,6 @@ dependencies: - cudatoolkit=11.5 - output_types: conda matrices: - - matrix: - packages: # TODO: Test priority of empty matrix! - matrix: arch: x86_64 cuda: "11.5" @@ -87,28 +85,32 @@ dependencies: - output_types: [conda, requirements] packages: - cachetools - - cubinlinker # TODO: Is this conda or requirements or both? - - cupy>=9.5.0,<12.0.0a0 # TODO: cupy for conda, cupy-cuda115 for requirements? + - cubinlinker - dask>=2022.9.2 - distributed>=2022.9.2 - fsspec>=0.6.0 - numba>=0.56.2 - - numpy # Does this need a lower bound? + - numpy - nvtx>=0.2.1 - packaging - pandas>=1.0,<1.6.0dev0 + - ptxcompiler - python-confluent-kafka=1.7.0 - streamz - typing_extensions - - output_types: [conda, requirements] + - output_types: conda packages: - - pip - - pip: - - git+https://github.com/python-streamz/streamz.git@master - - pyorc # I think there is an issue filed about removing this. Maybe need to check that. - - ptxcompiler # Not available on PyPI, but may be needed? + - cupy>=9.5.0,<12.0.0a0 + - pip + - pip: + - git+https://github.com/python-streamz/streamz.git@master + - pyorc - output_types: requirements packages: + # pip recognizes the index as a global option for the requirements.txt file + # This index is needed for rmm, cubinlinker, ptxcompiler. + - --extra-index-url=https://pypi.ngc.nvidia.com + - cupy-cuda115>=9.5.0,<12.0.0a0 # Need to check this. ARM requires cupy-cuda11x? - git+https://github.com/python-streamz/streamz.git@master - pyorc develop: From dc50330d095a8cc9cd11bfd78a8b38e23c61ebc4 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 4 Nov 2022 17:02:06 -0500 Subject: [PATCH 09/28] Remove style check packages handled by pre-commit. --- dependencies.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 0d6ff607fc1..eb0446d58f3 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -117,13 +117,8 @@ dependencies: common: - output_types: [conda, requirements] packages: - - black=22.3.0 - - cmakelang=0.6.13 - - flake8=3.8.3 - gcovr>=5.0 - - isort=5.10.1 - pre-commit - - pydocstyle=6.1.1 - output_types: conda packages: - clang=11.1.0 # Are these clang packages needed? For what? From 6fb085882957c84a7a4727de77143986cf0f7fc3 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 4 Nov 2022 17:11:47 -0500 Subject: [PATCH 10/28] Update moto to avoid werkzeug breakage. --- dependencies.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index eb0446d58f3..46afdae6b57 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -135,7 +135,7 @@ dependencies: - fastavro>=0.22.9 - hypothesis - mimesis>=4.1.0 - - moto>=3.1.6 + - moto>=4.0.8 - pytest - pytest-benchmark - pytest-cases @@ -146,7 +146,6 @@ dependencies: - s3fs>=2022.3.0 - scipy # Does this need a lower bound? - transformers<=4.10.3 # We should check and fix all "<=" pinnings - - werkzeug<2.2.0 # Temporary transient dependency pinning to avoid URL-LIB3 + moto timeouts doc: common: - output_types: [conda, requirements] From fe65899cba28290bd7600476efa204c73889e5dc Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 4 Nov 2022 17:23:47 -0500 Subject: [PATCH 11/28] Remove style checkers, gcovr, clang, cmake_setuptools. --- conda/environments/all_cuda-115_arch-x86_64.yaml | 13 ++----------- dependencies.yaml | 10 +++------- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/conda/environments/all_cuda-115_arch-x86_64.yaml b/conda/environments/all_cuda-115_arch-x86_64.yaml index 98f5e3caa33..06c322b1bc9 100644 --- a/conda/environments/all_cuda-115_arch-x86_64.yaml +++ b/conda/environments/all_cuda-115_arch-x86_64.yaml @@ -6,16 +6,11 @@ channels: dependencies: - aiobotocore>=2.2.0 - arrow-cpp=9 -- black=22.3.0 - boto3>=1.21.21 - botocore>=1.24.21 - c-compiler - cachetools -- clang-tools=11.1.0 -- clang=11.1.0 - cmake>=3.23.1 -- cmake_setuptools>=0.1.3 -- cmakelang=0.6.13 - cubinlinker - cuda-python>=11.7.1,<12.0 - cudatoolkit=11.5 @@ -28,22 +23,20 @@ dependencies: - dlpack>=0.5,<0.6.0a0 - doxygen=1.8.20 - fastavro>=0.22.9 -- flake8=3.8.3 - fsspec>=0.6.0 - gcc_linux-64=9.* -- gcovr>=5.0 - hypothesis - ipython -- isort=5.10.1 - librdkafka=1.7.0 - mimesis>=4.1.0 -- moto>=3.1.6 +- moto>=4.0.8 - myst-nb - nbsphinx - notebook>=0.5.0 - numba>=0.56.2 - numpy - numpydoc +- nvcc_linux-64=11.5 - nvtx>=0.2.1 - packaging - pandas>=1.0,<1.6.0dev0 @@ -54,7 +47,6 @@ dependencies: - ptxcompiler - pyarrow=9 - pydata-sphinx-theme -- pydocstyle=6.1.1 - pytest - pytest-benchmark - pytest-cases @@ -77,7 +69,6 @@ dependencies: - sysroot_linux-64==2.17 - transformers<=4.10.3 - typing_extensions -- werkzeug<2.2.0 - pip: - git+https://github.com/python-streamz/streamz.git@master - pyorc diff --git a/dependencies.yaml b/dependencies.yaml index 46afdae6b57..a5a59517b42 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -20,16 +20,15 @@ channels: - rapidsai - conda-forge # Not sure whether we need these: -# - nvidia # - rapidsai-nightly # - dask/label/dev +# - nvidia dependencies: build: common: - output_types: [conda, requirements] packages: - cmake>=3.23.1 - - cmake_setuptools>=0.1.3 - cython>=0.29,<0.30 - dlpack>=0.5,<0.6.0a0 - pyarrow=9 @@ -117,13 +116,10 @@ dependencies: common: - output_types: [conda, requirements] packages: - - gcovr>=5.0 - pre-commit - output_types: conda packages: - - clang=11.1.0 # Are these clang packages needed? For what? - - clang-tools=11.1.0 # Are these clang packages needed? For what? - - doxygen=1.8.20 # We rely on this being installed for pre-commit but pre-commit handles all the rest of this section for us (except pre-commit itself). We can probably remove black, etc. + - doxygen=1.8.20 # pre-commit hook needs a specific version. test: common: - output_types: [conda, requirements] @@ -144,7 +140,7 @@ dependencies: - python-snappy>=0.6.0 - pytorch<1.12.0 # We should check and fix all "<=" pinnings - s3fs>=2022.3.0 - - scipy # Does this need a lower bound? + - scipy - transformers<=4.10.3 # We should check and fix all "<=" pinnings doc: common: From c9367886de6b5007f78e2d69902544b1a5036381 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 4 Nov 2022 18:03:37 -0500 Subject: [PATCH 12/28] Update pinnings (tested locally). --- .../all_cuda-115_arch-x86_64.yaml | 4 ++-- dependencies.yaml | 24 +++++++++++++++---- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/conda/environments/all_cuda-115_arch-x86_64.yaml b/conda/environments/all_cuda-115_arch-x86_64.yaml index 06c322b1bc9..9e1019119a8 100644 --- a/conda/environments/all_cuda-115_arch-x86_64.yaml +++ b/conda/environments/all_cuda-115_arch-x86_64.yaml @@ -45,7 +45,7 @@ dependencies: - pre-commit - protobuf>=3.20.1,<3.21.0a0 - ptxcompiler -- pyarrow=9 +- pyarrow=9.0.0 - pydata-sphinx-theme - pytest - pytest-benchmark @@ -67,7 +67,7 @@ dependencies: - sphinxcontrib-websupport - streamz - sysroot_linux-64==2.17 -- transformers<=4.10.3 +- transformers - typing_extensions - pip: - git+https://github.com/python-streamz/streamz.git@master diff --git a/dependencies.yaml b/dependencies.yaml index a5a59517b42..95d9e5d3ba9 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -31,7 +31,7 @@ dependencies: - cmake>=3.23.1 - cython>=0.29,<0.30 - dlpack>=0.5,<0.6.0a0 - - pyarrow=9 + - pyarrow=9.0.0 - rmm=22.12.* - scikit-build>=0.13.1 - output_types: conda @@ -84,7 +84,6 @@ dependencies: - output_types: [conda, requirements] packages: - cachetools - - cubinlinker - dask>=2022.9.2 - distributed>=2022.9.2 - fsspec>=0.6.0 @@ -93,25 +92,40 @@ dependencies: - nvtx>=0.2.1 - packaging - pandas>=1.0,<1.6.0dev0 - - ptxcompiler - python-confluent-kafka=1.7.0 - streamz - typing_extensions - output_types: conda packages: + - cubinlinker - cupy>=9.5.0,<12.0.0a0 - pip - pip: - git+https://github.com/python-streamz/streamz.git@master - pyorc + - ptxcompiler + - rmm=22.12.* - output_types: requirements packages: # pip recognizes the index as a global option for the requirements.txt file # This index is needed for rmm, cubinlinker, ptxcompiler. - --extra-index-url=https://pypi.ngc.nvidia.com - - cupy-cuda115>=9.5.0,<12.0.0a0 # Need to check this. ARM requires cupy-cuda11x? + - cubinlinker-cu11 - git+https://github.com/python-streamz/streamz.git@master + - ptxcompiler-cu11 - pyorc + - rmm-cu11=22.12.* + specific: + - output_types: requirements + matrices: + - matrix: + arch: x86_64 + packages: + - cupy-cuda115>=9.5.0,<12.0.0a0 # TODO: This might change to cupy-cuda11x? + - matrix: + arch: aarch64 + packages: + - cupy-cuda11x -f https://pip.cupy.dev/aarch64 # TODO: Verify that this works. develop: common: - output_types: [conda, requirements] @@ -141,7 +155,7 @@ dependencies: - pytorch<1.12.0 # We should check and fix all "<=" pinnings - s3fs>=2022.3.0 - scipy - - transformers<=4.10.3 # We should check and fix all "<=" pinnings + - transformers doc: common: - output_types: [conda, requirements] From f5a5675224878e2d2d1bcb33da68c4bd3e09b389 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Fri, 4 Nov 2022 18:06:03 -0500 Subject: [PATCH 13/28] Update comment. --- dependencies.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies.yaml b/dependencies.yaml index 95d9e5d3ba9..60a86da1a4c 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -141,7 +141,7 @@ dependencies: - aiobotocore>=2.2.0 - boto3>=1.21.21 - botocore>=1.24.21 - - dask-cuda=22.12.* # TODO: This only looks like a testing dependency? + - dask-cuda=22.12.* # TODO: Verify this is a testing (not run) dependency. - fastavro>=0.22.9 - hypothesis - mimesis>=4.1.0 From e261a169dd62adf6d892b14122be221b2e8a24ca Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 7 Nov 2022 15:09:50 -0600 Subject: [PATCH 14/28] Update checks. Co-authored-by: AJ Schmidt --- .github/workflows/dependency-files.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/dependency-files.yml b/.github/workflows/dependency-files.yml index aeda16f41cb..3c876079dae 100644 --- a/.github/workflows/dependency-files.yml +++ b/.github/workflows/dependency-files.yml @@ -1,8 +1,12 @@ -name: dependency-files +name: pr on: pull_request: jobs: - check-generated-files: - uses: rapidsai/shared-action-workflows/.github/workflows/dependency-files.yaml@main + checks: + secrets: inherit + uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@main + with: + enable_check_size: false + enable_check_size: false From bd886b62d576566aed9e001837882a83c9b78a2e Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 7 Nov 2022 15:10:05 -0600 Subject: [PATCH 15/28] Update dependency-files.yml --- .github/workflows/dependency-files.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dependency-files.yml b/.github/workflows/dependency-files.yml index 3c876079dae..2ae939292d7 100644 --- a/.github/workflows/dependency-files.yml +++ b/.github/workflows/dependency-files.yml @@ -9,4 +9,4 @@ jobs: uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@main with: enable_check_size: false - enable_check_size: false + enable_check_style: false From ede483337d0a112cabcad9cfda07bd6d6b89a970 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 7 Nov 2022 15:11:00 -0600 Subject: [PATCH 16/28] Put cuda-python in common packages. --- dependencies.yaml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 60a86da1a4c..a7e9998d089 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -29,6 +29,7 @@ dependencies: - output_types: [conda, requirements] packages: - cmake>=3.23.1 + - cuda-python>=11.7.1,<12.0 - cython>=0.29,<0.30 - dlpack>=0.5,<0.6.0a0 - pyarrow=9.0.0 @@ -43,12 +44,6 @@ dependencies: - protobuf>=3.20.1,<3.21.0a0 - python>=3.8,<3.10 specific: - - output_types: [conda, requirements] - matrices: - - matrix: - cuda: "11.5" - packages: - - cuda-python>=11.7.1,<12.0 - output_types: conda matrices: - matrix: From 65bd9ad331b56e35b2575bb069faa11a57099ff8 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 7 Nov 2022 15:15:25 -0600 Subject: [PATCH 17/28] Move cudatoolkit. --- dependencies.yaml | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index a7e9998d089..88777032139 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -7,6 +7,7 @@ files: arch: [x86_64] includes: - build + - cudatoolkit - develop - run - test @@ -15,6 +16,7 @@ files: test: output: none includes: + - cudatoolkit - test channels: - rapidsai @@ -56,12 +58,6 @@ dependencies: packages: - gcc_linux-aarch64=9.* - sysroot_linux-aarch64==2.17 - - output_types: conda - matrices: - - matrix: - cuda: "11.5" - packages: - - cudatoolkit=11.5 - output_types: conda matrices: - matrix: @@ -74,6 +70,22 @@ dependencies: cuda: "11.5" packages: - nvcc_linux-aarch64=11.5 + cudatoolkit: + specific: + - output_types: conda + matrices: + - matrix: + cuda: "11.2" + packages: + - cudatoolkit=11.2 + - matrix: + cuda: "11.4" + packages: + - cudatoolkit=11.4 + - matrix: + cuda: "11.5" + packages: + - cudatoolkit=11.5 run: common: - output_types: [conda, requirements] From 6d68f343e8facb19f7a443546ee4eeb4dd49deb7 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 7 Nov 2022 15:28:46 -0600 Subject: [PATCH 18/28] Update dependencies.yaml versions on release. --- ci/release/update-version.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 52dc22b6c49..9dcfe093643 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -63,9 +63,10 @@ sed_runner 's/version = .*/version = '"'${NEXT_SHORT_TAG}'"'/g' docs/cudf/source sed_runner 's/release = .*/release = '"'${NEXT_FULL_TAG}'"'/g' docs/cudf/source/conf.py # bump rmm & dask-cuda -for FILE in conda/environments/*.yml; do - sed_runner "s/rmm=${CURRENT_SHORT_TAG}/rmm=${NEXT_SHORT_TAG}/g" ${FILE}; +for FILE in conda/environments/*.yaml dependencies.yaml; do sed_runner "s/dask-cuda=${CURRENT_SHORT_TAG}/dask-cuda=${NEXT_SHORT_TAG}/g" ${FILE}; + sed_runner "s/rmm=${CURRENT_SHORT_TAG}/rmm=${NEXT_SHORT_TAG}/g" ${FILE}; + sed_runner "s/rmm-cu11=${CURRENT_SHORT_TAG}/rmm-cu11=${NEXT_SHORT_TAG}/g" ${FILE}; done # Doxyfile update From fd4a7814504f5b63f183f725e55924f911018075 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 7 Nov 2022 15:30:51 -0600 Subject: [PATCH 19/28] Ensure that rapids-dependency-file-generator fails in CI. --- conda/environments/all_cuda-115_arch-x86_64.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/environments/all_cuda-115_arch-x86_64.yaml b/conda/environments/all_cuda-115_arch-x86_64.yaml index 9e1019119a8..d9ae40d6ba0 100644 --- a/conda/environments/all_cuda-115_arch-x86_64.yaml +++ b/conda/environments/all_cuda-115_arch-x86_64.yaml @@ -4,7 +4,7 @@ channels: - rapidsai - conda-forge dependencies: -- aiobotocore>=2.2.0 +- aiobotocore>=99999 - arrow-cpp=9 - boto3>=1.21.21 - botocore>=1.24.21 From aee5118d69ea0f0661d0707b7072583cbc88ed8b Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 7 Nov 2022 15:34:13 -0600 Subject: [PATCH 20/28] Revert "Ensure that rapids-dependency-file-generator fails in CI." This reverts commit fd4a7814504f5b63f183f725e55924f911018075. --- conda/environments/all_cuda-115_arch-x86_64.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/environments/all_cuda-115_arch-x86_64.yaml b/conda/environments/all_cuda-115_arch-x86_64.yaml index d9ae40d6ba0..9e1019119a8 100644 --- a/conda/environments/all_cuda-115_arch-x86_64.yaml +++ b/conda/environments/all_cuda-115_arch-x86_64.yaml @@ -4,7 +4,7 @@ channels: - rapidsai - conda-forge dependencies: -- aiobotocore>=99999 +- aiobotocore>=2.2.0 - arrow-cpp=9 - boto3>=1.21.21 - botocore>=1.24.21 From e814cb5ea16956642c2aeceb5ac5bb47ddf28ec9 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 7 Nov 2022 15:44:44 -0600 Subject: [PATCH 21/28] Split test_cpp, test_python --- dependencies.yaml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 88777032139..71058123cbe 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -13,11 +13,21 @@ files: - test - doc - notebook - test: + test_cpp: output: none includes: - cudatoolkit - - test + test_python: + output: none + includes: + - cudatoolkit + - test_python + checks: + output: none + includes: + - build + - develop + - py_version channels: - rapidsai - conda-forge @@ -141,7 +151,7 @@ dependencies: - output_types: conda packages: - doxygen=1.8.20 # pre-commit hook needs a specific version. - test: + test_python: common: - output_types: [conda, requirements] packages: From a47817aa26dea3a022f922b69368a64eb47d8d7c Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 7 Nov 2022 15:45:49 -0600 Subject: [PATCH 22/28] Sort sections alphabetically. --- dependencies.yaml | 56 +++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 71058123cbe..d1873fdf244 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -96,6 +96,34 @@ dependencies: cuda: "11.5" packages: - cudatoolkit=11.5 + develop: + common: + - output_types: [conda, requirements] + packages: + - pre-commit + - output_types: conda + packages: + - doxygen=1.8.20 # pre-commit hook needs a specific version. + doc: + common: + - output_types: [conda, requirements] + packages: + - myst-nb + - nbsphinx + - numpydoc + - pandoc<=2.0.0 # We should check and fix all "<=" pinnings + - pydata-sphinx-theme + - sphinx + - sphinx-autobuild + - sphinx-copybutton + - sphinx-markdown-tables + - sphinxcontrib-websupport + notebook: + common: + - output_types: [conda, requirements] + packages: + - ipython + - notebook>=0.5.0 run: common: - output_types: [conda, requirements] @@ -143,14 +171,6 @@ dependencies: arch: aarch64 packages: - cupy-cuda11x -f https://pip.cupy.dev/aarch64 # TODO: Verify that this works. - develop: - common: - - output_types: [conda, requirements] - packages: - - pre-commit - - output_types: conda - packages: - - doxygen=1.8.20 # pre-commit hook needs a specific version. test_python: common: - output_types: [conda, requirements] @@ -173,23 +193,3 @@ dependencies: - s3fs>=2022.3.0 - scipy - transformers - doc: - common: - - output_types: [conda, requirements] - packages: - - myst-nb - - nbsphinx - - numpydoc - - pandoc<=2.0.0 # We should check and fix all "<=" pinnings - - pydata-sphinx-theme - - sphinx - - sphinx-autobuild - - sphinx-copybutton - - sphinx-markdown-tables - - sphinxcontrib-websupport - notebook: - common: - - output_types: [conda, requirements] - packages: - - ipython - - notebook>=0.5.0 From 934acb84cc95cfaa0aad5bee93b07e7e0a577247 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 7 Nov 2022 15:48:57 -0600 Subject: [PATCH 23/28] Remove extra channels. --- dependencies.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index d1873fdf244..ab8814255fc 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -31,10 +31,6 @@ files: channels: - rapidsai - conda-forge -# Not sure whether we need these: -# - rapidsai-nightly -# - dask/label/dev -# - nvidia dependencies: build: common: From b3b41a380c81dbd93c078d176e47fd41208da52d Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 7 Nov 2022 15:50:11 -0600 Subject: [PATCH 24/28] Add py_version. --- dependencies.yaml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index ab8814255fc..0de69e61b61 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -9,10 +9,10 @@ files: - build - cudatoolkit - develop - - run - - test - doc - notebook + - run + - test_python test_cpp: output: none includes: @@ -120,6 +120,18 @@ dependencies: packages: - ipython - notebook>=0.5.0 + py_version: + specific: + - output_types: conda + matrices: + - matrix: + py: "3.8" + packages: + - python=3.8 + - matrix: + py: "3.9" + packages: + - python=3.9 run: common: - output_types: [conda, requirements] From 98451f30c499593a4d20090c55bad2d476474ebe Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 7 Nov 2022 15:57:06 -0600 Subject: [PATCH 25/28] Update dependencies.yaml --- dependencies.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/dependencies.yaml b/dependencies.yaml index 0de69e61b61..5c879a0c673 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -21,6 +21,7 @@ files: output: none includes: - cudatoolkit + - py_version - test_python checks: output: none From c6cc1e02c169facb5f7fa0ed33c1d1e566a287d3 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 7 Nov 2022 15:59:54 -0600 Subject: [PATCH 26/28] Add back channels. --- dependencies.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dependencies.yaml b/dependencies.yaml index 5c879a0c673..46441a99a66 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -31,7 +31,10 @@ files: - py_version channels: - rapidsai + - rapidsai-nightly + - dask/label/dev - conda-forge + - nvidia dependencies: build: common: From c29e6bce06da9ad9aabf1b28ead069356da32385 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 7 Nov 2022 17:17:03 -0600 Subject: [PATCH 27/28] Update dependencies.yaml --- dependencies.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies.yaml b/dependencies.yaml index 46441a99a66..b8470f02f86 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -190,7 +190,7 @@ dependencies: - aiobotocore>=2.2.0 - boto3>=1.21.21 - botocore>=1.24.21 - - dask-cuda=22.12.* # TODO: Verify this is a testing (not run) dependency. + - dask-cuda=22.12.* - fastavro>=0.22.9 - hypothesis - mimesis>=4.1.0 From d9e7bb819afac09c5caeb72d4865853661419db8 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Mon, 7 Nov 2022 17:25:24 -0600 Subject: [PATCH 28/28] Update conda/environments/all_cuda-115_arch-x86_64.yaml --- conda/environments/all_cuda-115_arch-x86_64.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conda/environments/all_cuda-115_arch-x86_64.yaml b/conda/environments/all_cuda-115_arch-x86_64.yaml index 9e1019119a8..a7e5f1a04a6 100644 --- a/conda/environments/all_cuda-115_arch-x86_64.yaml +++ b/conda/environments/all_cuda-115_arch-x86_64.yaml @@ -2,7 +2,10 @@ # To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. channels: - rapidsai +- rapidsai-nightly +- dask/label/dev - conda-forge +- nvidia dependencies: - aiobotocore>=2.2.0 - arrow-cpp=9