chore: add support for python 3.12

.github/workflows/compatibility_tests.yml

@@ -7,10 +7,6 @@ on:
     branches:
       - main
 
-  # For debug - uncomment below to run on all PRs
-  # pull_request:
-  #   branches: "*"
-
 env:
   JEST_ENV: prod
 
@@ -32,14 +28,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # note: The `macos-latest` is latest Catalina version, and not Big Sur. So we explicitly ask for Big Sur (`macos-11`)
         os: [ubuntu-latest, macos-latest, macos-13]
-        python-version: ["3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12"]
         cellxgene_build: [main, latest]
         # add anndata pinned version test for subset of matrix configurations,
         # in order to reduce matrix cross-product explosion
         include:
-          - python-version: 3.11
+          - python-version: 3.12
             cellxgene_build: latest
             # TODO: dynamically use the literal version in requirements.txt,
             # to avoid having to update this in manually in the future
@@ -100,7 +95,7 @@ jobs:
           # keep same pip pkg versions as in the cxg release
           sed -i'' -e 's/-r requirements.txt//' server/requirements-dev.txt
           pip install -r server/requirements-dev.txt
-          pip install --force-reinstall numpy==2.0.1 numba>=0.60.0 pandas
+          pip install --force-reinstall numpy==2.0.1 numba>=0.60.0 pandas flatbuffers==2.0.7
       - name: Install anndata version per matrix variable
         run: pip install anndata${{ matrix.anndata_version }}
       - name: Install node

.github/workflows/push_tests.yml

@@ -17,10 +17,10 @@ jobs:
       - uses: actions/checkout@v4
       - run: |
           git fetch --depth=1 origin +${{github.base_ref}}
-      - name: Set up Python 3.11
+      - name: Set up Python 3.12
         uses: actions/setup-python@v5
         with:
-          python-version: 3.11
+          python-version: 3.12
       - name: Node cache
         uses: actions/cache@v4
         with:
@@ -46,10 +46,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python 3.11 (pyenv) # pyenv needed for mlflow in cli annotate tests
+      - name: Set up Python 3.12 (pyenv) # pyenv needed for mlflow in cli annotate tests
         uses: gabrielfalcao/pyenv-action@v9
         with:
-          default: 3.11
+          default: 3.12
           command: pip install -U pip # upgrade pip after installing python
       - run: pip install virtualenv # virtualenv needed for mlflow in cli annotate tests
       - name: Python cache
@@ -79,10 +79,10 @@ jobs:
     timeout-minutes: 20
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python 3.11
+      - name: Set up Python 3.12
         uses: actions/setup-python@v5
         with:
-          python-version: 3.11
+          python-version: 3.12
       - name: Python cache
         uses: actions/cache@v4
         with:
@@ -98,7 +98,9 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-node-
       - name: Install dependencies
-        run: make pydist install-dist
+        run: |
+          pip install setuptools
+          make pydist install-dist
       - name: Smoke tests (without annotations feature)
         run: |
           cd client && make smoke-test

README.md

@@ -27,7 +27,7 @@ Whether you need to visualize one thousand cells or one million, CELLxGENE Annot
 
 ### Quick start
 
-To install CELLxGENE Annotate you need Python 3.6+. We recommend [installing Annotate into a conda or virtual environment.](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md)
+To install CELLxGENE Annotate you need Python 3.10+. We recommend [installing Annotate into a conda or virtual environment.](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md)
 
 Install the package.
 
@@ -66,22 +66,21 @@ For any errors, [report bugs on Github](https://github.com/chanzuckerberg/cellxg
 
 ### Contributing
 
-We warmly welcome contributions from the community! Please see our [contributing guide](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md) and don't hesitate to open an issue or send a pull request to improve CELLxGENE Annotate. Please see the [dev_docs](https://github.com/chanzuckerberg/cellxgene/tree/main/dev_docs) for pull request suggestions, unit test details, local documentation preview, and other development specifics. 
+We warmly welcome contributions from the community! Please see our [contributing guide](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md) and don't hesitate to open an issue or send a pull request to improve CELLxGENE Annotate. Please see the [dev_docs](https://github.com/chanzuckerberg/cellxgene/tree/main/dev_docs) for pull request suggestions, unit test details, local documentation preview, and other development specifics.
 
 This project adheres to the Contributor Covenant [code of conduct](https://github.com/chanzuckerberg/.github/blob/master/CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to [email protected].
 
 ### Reuse
 
-This project was started with the sole goal of empowering the scientific community to explore and understand their data. 
-As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from 
+This project was started with the sole goal of empowering the scientific community to explore and understand their data.
+As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from
 this project. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
 
+Before extending CELLxGENE Annotate, we encourage you to reach out to us with ideas or questions. It might be possible that an
+extension could be directly contributed, which would make it available for a wider audience, or that it's on our
+[roadmap](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/roadmap.md) and under active development.
 
-Before extending CELLxGENE Annotate, we encourage you to reach out to us with ideas or questions. It might be possible that an 
-extension could be directly contributed, which would make it available for a wider audience, or that it's on our 
-[roadmap](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/roadmap.md) and under active development. 
-
-See the [CELLxGENE extensions](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/community-extensions.md) section of our documentation for examples of community use and CELLxGENE extensions. 
+See the [CELLxGENE extensions](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/community-extensions.md) section of our documentation for examples of community use and CELLxGENE extensions.
 
 ### Security
 

client/.nvmrc

@@ -1 +1 @@
-16.20.0
+18.17.0

client/jest-puppeteer.config.js

@@ -14,6 +14,7 @@ const DEFAULT_LAUNCH_CONFIG = {
   headless: !isHeadful,
   args: ["--ignore-certificate-errors", "--ignore-ssl-errors"],
   ignoreHTTPSErrors: true,
+  timeout: 90000,
   defaultViewport: {
     width: 1280,
     height: 960,

client/package.json

@@ -18,8 +18,8 @@
   },
   "engineStrict": true,
   "engines": {
-    "npm": ">=3.0.0",
-    "node": "^16.0.0"
+    "npm": ">=9.6.7",
+    "node": "^18.17.0"
   },
   "eslintConfig": {
     "extends": "./configuration/eslint/eslint.js"

dev_docs/developer_guidelines.md

@@ -3,7 +3,7 @@
 ## Requirements
 
 - npm
-- Python 3.6+
+- Python 3.10+
 - Chrome
 
 [See dev section of README](../README.md)
@@ -148,6 +148,6 @@ If you would like to run the smoke tests against a hot-reloaded version of the c
 
 ### Tips
 
-- You can also install/launch the server side code from npm scrips (requires python3.6 with virtualenv) with the `scripts/backend_dev` script.
+- You can also install/launch the server side code from npm scrips (requires python3.10 with virtualenv) with the `scripts/backend_dev` script.
 
-- Check out [e2e Tests](e2e_tests.md) for more details
+- Check out [e2e Tests](e2e_tests.md) for more details

dev_docs/developer_scripts.md

@@ -11,27 +11,30 @@ $PROJECT_ROOT`.
 ### Build
 
 **Usage:** from the `$PROJECT_ROOT` directory run:
-* `make build` builds whole app client and server
-* `make build-client` runs webpack build
-* `make build-for-server-dev` builds client and copies output directly into
+
+- `make build` builds whole app client and server
+- `make build-client` runs webpack build
+- `make build-for-server-dev` builds client and copies output directly into
   source tree (only for server devlopment)
 
 ### Clean
 
 Deletes generated files.
 
 **Usage:** from the `$PROJECT_ROOT` directory run:
-* `make clean` cleans everything including node modules (means build with take
+
+- `make clean` cleans everything including node modules (means build with take
   a while
-* `make clean-lite` cleans built directories
-* `make clean-server` cleans source tree
+- `make clean-lite` cleans built directories
+- `make clean-server` cleans source tree
 
 ### Distribution
 
 Creates distribution for python module to upload to pypi.
 
 **Usage:** from the `$PROJECT_ROOT` directory run:
-* `make pydist` builds code and then builds sdist
+
+- `make pydist` builds code and then builds sdist
 
 ### Release
 
@@ -42,16 +45,18 @@ See `release_process.md`.
 Installs requirements files.
 
 **Usage:** from the `$PROJECT_ROOT` directory run:
-* `make dev-env` installs requirements and requirments-dev (for building code)
+
+- `make dev-env` installs requirements and requirments-dev (for building code)
 
 ### Installing cellxgene packages
 
 **Usage:** from the `$PROJECT_ROOT` directory:
-* `install-dev` - installs from local source tree
-* `install-release-test` - installs from test pypi
-* `install-release` - installs from pypi
-* `install-dist` - installs from local dist folder
-* `uninstall` - uninstalls cellxgene
+
+- `install-dev` - installs from local source tree
+- `install-release-test` - installs from test pypi
+- `install-release` - installs from pypi
+- `install-dist` - installs from local dist folder
+- `uninstall` - uninstalls cellxgene
 
 ## Client-level scripts
 
@@ -62,8 +67,9 @@ Installs requirements files.
 **About** Serve the current client javascript independently from the `server` code.
 
 **Requires**
-* The server to be running. Best way to do this is with [backend_dev](#backend_dev).
-* `make ci` to install the necessary node modules
+
+- The server to be running. Best way to do this is with [backend_dev](#backend_dev).
+- `make ci` to install the necessary node modules
 
 **Usage:** from the `$PROJECT_ROOT/client` directory run `make start-frontend`
 
@@ -75,23 +81,24 @@ the FE developer gets the current version of the backend with a single command
 and no knowledge of python necessary. It creates and activates a virtual
 environment and installs cellxgene from the current branch.
 
-**Requires** `Python3.6+`, `virtual-env`, `pip`
+**Requires** `Python3.10+`, `virtual-env`, `pip`
 
 **Usage:** from the `$PROJECT_ROOT` directory run `./scripts/backend_dev`
 
 **Options:**
-* In parallel, you can then launch the node development server to serve the
+
+- In parallel, you can then launch the node development server to serve the
   current state of the FE with [`start-frontend`](#start-frontend), usually in
   a different terminal tab.
-* You can also select a specific dataset using `DATASET=<dataset path> ./scripts/backend_dev`.
-* You can also use `CXG_OPTIONS` to pass options to the `cellxgene launch`
+- You can also select a specific dataset using `DATASET=<dataset path> ./scripts/backend_dev`.
+- You can also use `CXG_OPTIONS` to pass options to the `cellxgene launch`
   command, as in `CXG_OPTIONS='--disable-annotations' ./scripts/backend_dev`.
 
 **Breakdown**
 
 | command                                  | purpose                                                    |
 | ---------------------------------------- | ---------------------------------------------------------- |
-| python3.6 -m venv cellxgene              | creates cellxgene virtual environment                      |
+| python3.12 -m venv cellxgene             | creates cellxgene virtual environment                      |
 | source cellxgene/bin/activate            | activates virtual environment                              |
 | yes \| pip uninstall cellxgene \|\| true | uninstalls cellxgene (if installed)                        |
 | pip install -e .                         | installs current local version of cellxgene                |
@@ -102,14 +109,15 @@ environment and installs cellxgene from the current branch.
 Methods used to test the client javascript code
 
 **Usage:** from the `$PROJECT_ROOT/client` directory run:
-* `make unit-test` Runs all unit tests. It excludes any tests in the e2e
+
+- `make unit-test` Runs all unit tests. It excludes any tests in the e2e
   folder. This is used by travis to run unit tests.
-* `make smoke-test` Starts backend development server and runs end to end
+- `make smoke-test` Starts backend development server and runs end to end
   tests. This is what travis runs. It depends on the `e2e` and the
   `backend-dev` targets. One starts the server, the other runs the tests. If
   developing a front-end feature and just checking if tests pass, this is
   probabaly the one you want to run.
-* `npm run e2e` Runs backend tests without starting the server. You will need to
+- `npm run e2e` Runs backend tests without starting the server. You will need to
   start the rest api separately with the pbmc3k.h5ad file. Note you can use
   the `JEST_ENV` environment variable to change how JEST runs in the browser.
   The test runs against `localhost:3000` by default. You can use the

server/requirements.txt

@@ -7,7 +7,7 @@ Flask-Cors>=3.0.9
 Flask-RESTful>=0.3.6
 flask-server-timing>=0.1.2
 flask-talisman>=0.7.0
-flatbuffers>=1.11.0,<2.0.0 # cellxgene is not compatible with 2.0.0. Requires migration
+flatbuffers==2.0.7
 flatten-dict>=0.2.0
 fsspec>0.8.0
 gunicorn>=20.0.4
@@ -19,4 +19,5 @@ pandas>=2.2.2
 PyYAML>=5.4  # CVE-2020-14343
 requests>=2.22.0
 s3fs==0.4.2
-scipy>=1.4
+scipy>=1.4
+setuptools

setup.py

@@ -24,7 +24,7 @@
     long_description=long_description,
     long_description_content_type="text/markdown",
     install_requires=requirements,
-    python_requires=">=3.6",
+    python_requires=">=3.10",
     include_package_data=True,
     zip_safe=False,
     classifiers=[
@@ -37,8 +37,9 @@
         "Operating System :: MacOS :: MacOS X",
         "Programming Language :: JavaScript",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.6",
-        "Programming Language :: Python :: 3.7",
+        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
         "Programming Language :: Python :: 3 :: Only",
         "Topic :: Scientific/Engineering :: Bio-Informatics",
     ],