diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index c9b792aaa13..2d1916b780f 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -9,3 +9,6 @@ d6d0607a845e6f71084ce272a1c1e8c50e244bdd # Apply buildifier to the project f457f19039b82536b35659c1f9cb898a198e6cd1 + +# Apply ruff linter to the project +893774eab71fd7be5000436ff2ff0b5dd85ef073 diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 00000000000..7d379ff4623 --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,25 @@ +target-version = "py312" + +[lint] +select = [ + "B0", # bugbear (all B0* checks enabled by default) + "B904", # bugbear (Within an except clause, raise exceptions with raise ... from err) + "B905", # bugbear (zip() without an explicit strict= parameter set.) + "E", # pycodestyles + "W", # pycodestyles + "F", # pyflakes + "I", # isort + "PGH", # pygrep-hooks + "PLC", # pylint conventions + "PLE", # pylint errors + "UP", # pyupgrade +] +ignore = [ + "E402", # module import not at top of file + "UP038", # Use X | Y in isinstance check instead of (X, Y) +] + +[lint.per-file-ignores] +# Want to preserve compatibility for old Python versions in tools directory so +# disable pyupgrade. Oldest supported version is currently 3.9. +"tools/*" = ["UP"] diff --git a/samples/pyodide-fastapi/worker.py b/samples/pyodide-fastapi/worker.py index ee61748e2d0..53222e959eb 100644 --- a/samples/pyodide-fastapi/worker.py +++ b/samples/pyodide-fastapi/worker.py @@ -7,31 +7,26 @@ async def on_fetch(request): return await asgi.fetch(app, request, env) -def test(): - import fastapi - - # Set up fastapi app from fastapi import FastAPI from pydantic import BaseModel - app = FastAPI() @app.get("/hello") -async def root(env=env): +async def hello(env=env): return {"message": "Hello World", "secret": env.secret} @app.get("/route") -async def root(): +async def route(): return {"message": "this is my custom route"} @app.get("/favicon.ico") -async def root(): +async def favicon(): return {"message": "here's a favicon I guess?"} @@ -53,8 +48,8 @@ async def create_item(item: Item): @app.put("/items/{item_id}") -async def create_item(item_id: int, item: Item, q: str | None = None): - result = {"item_id": item_id, **item.dict()} +async def create_item2(item_id: int, item: Item, q: str | None = None): + result = {"item_id": item_id, **item.model_dump()} if q: result.update({"q": q}) return result diff --git a/samples/pyodide-langchain/worker.py b/samples/pyodide-langchain/worker.py index 835667fa42b..ae61132abd5 100644 --- a/samples/pyodide-langchain/worker.py +++ b/samples/pyodide-langchain/worker.py @@ -1,4 +1,3 @@ -from js import Response from langchain_core.prompts import PromptTemplate from langchain_openai import OpenAI diff --git a/samples/repl-server-python/worker.py b/samples/repl-server-python/worker.py index 17b7d968ae5..5911026f191 100644 --- a/samples/repl-server-python/worker.py +++ b/samples/repl-server-python/worker.py @@ -1,11 +1,8 @@ -from js import Response - -import io - import code - -from io import StringIO import sys +from io import StringIO + +from js import Response sys.stdout = StringIO() diff --git a/src/cloudflare/internal/test/vectorize/vectorize-api-test.py b/src/cloudflare/internal/test/vectorize/vectorize-api-test.py index a8295f40484..2d03fb6ae20 100644 --- a/src/cloudflare/internal/test/vectorize/vectorize-api-test.py +++ b/src/cloudflare/internal/test/vectorize/vectorize-api-test.py @@ -2,11 +2,9 @@ # Licensed under the Apache 2.0 license found in the LICENSE file or at: # https://opensource.org/licenses/Apache-2.0 -from js import Float32Array -from js import JSON +from js import Float32Array, Object from pyodide.ffi import to_js as _to_js -from js import Object def to_js(obj): diff --git a/src/pyodide/internal/asgi.py b/src/pyodide/internal/asgi.py index 57f3e9d8a5b..060ed74729f 100644 --- a/src/pyodide/internal/asgi.py +++ b/src/pyodide/internal/asgi.py @@ -1,7 +1,8 @@ -from asyncio import Future, ensure_future, Queue, sleep -from inspect import isawaitable +from asyncio import Future, Queue, ensure_future, sleep from contextlib import contextmanager -from fastapi import Request, Depends +from inspect import isawaitable + +from fastapi import Depends, Request ASGI = {"spec_version": "2.0", "version": "3.0"} @@ -93,7 +94,8 @@ async def send(got): async def process_request(app, req, env): - from js import Response, Object + from js import Object, Response + from pyodide.ffi import create_proxy status = None diff --git a/src/pyodide/internal/patches/aiohttp.py b/src/pyodide/internal/patches/aiohttp.py index b04bc8b4213..5c7934bf428 100644 --- a/src/pyodide/internal/patches/aiohttp.py +++ b/src/pyodide/internal/patches/aiohttp.py @@ -1,14 +1,20 @@ -# Monkeypatch aiohttp to introduce Fetch API support. -# -# Based on https://github.com/pyodide/pyodide/issues/3711#issuecomment-1773523301 -# with some modifications. +""" +Monkeypatch aiohttp to introduce Fetch API support. -from multidict import CIMultiDict, istr -from aiohttp import payload, InvalidURL, hdrs, ClientSession, ClientTimeout -from aiohttp.client_reqrep import _merge_ssl_params -from aiohttp.helpers import TimeoutHandle, strip_auth_from_url, get_env_proxy_for_url +Based on https://github.com/pyodide/pyodide/issues/3711#issuecomment-1773523301 +with some modifications. +""" + +# ruff: noqa + +from collections.abc import Iterable from contextlib import suppress -from typing import Any, Optional, Iterable +from typing import Any + +from aiohttp import ClientSession, ClientTimeout, InvalidURL, hdrs, payload +from aiohttp.client_reqrep import _merge_ssl_params +from aiohttp.helpers import TimeoutHandle, get_env_proxy_for_url, strip_auth_from_url +from multidict import CIMultiDict, istr from yarl import URL @@ -43,25 +49,25 @@ async def _request( json: Any = None, cookies=None, headers=None, - skip_auto_headers: Optional[Iterable[str]] = None, + skip_auto_headers: Iterable[str] | None = None, auth=None, allow_redirects: bool = True, max_redirects: int = 10, - compress: Optional[str] = None, - chunked: Optional[bool] = None, + compress: str | None = None, + chunked: bool | None = None, expect100: bool = False, raise_for_status=None, read_until_eof: bool = True, proxy=None, proxy_auth=None, timeout=None, - verify_ssl: Optional[bool] = None, - fingerprint: Optional[bytes] = None, + verify_ssl: bool | None = None, + fingerprint: bytes | None = None, ssl_context=None, ssl=None, proxy_headers=None, trace_request_ctx=None, - read_bufsize: Optional[int] = None, + read_bufsize: int | None = None, ): # NOTE: timeout clamps existing connect and read timeouts. We cannot # set the default to None because we need to detect if the user wants @@ -191,7 +197,8 @@ async def _request( loop=req.loop, session=req._session, ) - from js import fetch, Headers + from js import Headers, fetch + from pyodide.ffi import to_js body = None diff --git a/src/pyodide/internal/patches/httpx.py b/src/pyodide/internal/patches/httpx.py index a9c0f3401e0..1ef76c61e5b 100644 --- a/src/pyodide/internal/patches/httpx.py +++ b/src/pyodide/internal/patches/httpx.py @@ -7,9 +7,9 @@ from httpx._transports.default import AsyncResponseStream from httpx._types import AsyncByteStream from httpx._utils import Timer - from js import Headers as js_Headers from js import fetch + from pyodide.ffi import create_proxy @@ -29,8 +29,9 @@ def acquire_buffer(content): async def js_readable_stream_iter(js_readable_stream): - """Readable streams are supposed to be async iterators some day but they aren't yet. - In the meantime, this is an adaptor that produces an async iterator from a readable stream. + """Readable streams are supposed to be async iterators some day but they + aren't yet. In the meantime, this is an adaptor that produces an async + iterator from a readable stream. """ reader = js_readable_stream.getReader() while True: @@ -64,8 +65,8 @@ async def _send_single_request(self, request: Request) -> Response: ) py_headers = Headers(js_resp.headers) - # Unset content-encoding b/c Javascript fetch already handled unpacking. If we leave it we will - # get errors when httpx tries to unpack a second time. + # Unset content-encoding b/c Javascript fetch already handled unpacking. If + # we leave it we will get errors when httpx tries to unpack a second time. py_headers.pop("content-encoding", None) response = Response( status_code=js_resp.status, diff --git a/src/pyodide/internal/process_script_imports.py b/src/pyodide/internal/process_script_imports.py index 049e9ab1b02..b184ea97c89 100644 --- a/src/pyodide/internal/process_script_imports.py +++ b/src/pyodide/internal/process_script_imports.py @@ -1,14 +1,17 @@ -# This script is used to prepare a worker prior to a _package_ memory snapshot being taken. -# All it does is walk through the imports in each of the worker's modules and attempts to import -# them. Local imports are not possible because the worker file path is explicitly removed from the -# module search path. +""" +This script is used to prepare a worker prior to a _package_ memory snapshot +being taken. All it does is walk through the imports in each of the worker's +modules and attempts to import them. Local imports are not possible because +the worker file path is explicitly removed from the module search path. +""" + CF_LOADED_MODULES = [] def _do_it(): import ast - from pathlib import Path import sys + from pathlib import Path def find_imports(source: str) -> list[str]: try: @@ -36,10 +39,11 @@ def process_script(script): pass def process_scripts(): - # Currently this script assumes that it is generating a _package_ snapshot- one that - # only includes non-vendored packages. Because of this we do not wish to import local - # modules, the easiest way to ensure they cannot be imported is to remove - # `/session/metadata` from the sys path. + # Currently this script assumes that it is generating a _package_ + # snapshot- one that only includes non-vendored packages. Because of + # this we do not wish to import local modules, the easiest way to ensure + # they cannot be imported is to remove `/session/metadata` from the sys + # path. worker_files_path = "/session/metadata" sys.path.remove(worker_files_path) for script in Path(worker_files_path).glob("**/*.py"): diff --git a/src/pyodide/internal/topLevelEntropy/entropy_import_context.py b/src/pyodide/internal/topLevelEntropy/entropy_import_context.py index d2d99ec6e7e..e4b66431834 100644 --- a/src/pyodide/internal/topLevelEntropy/entropy_import_context.py +++ b/src/pyodide/internal/topLevelEntropy/entropy_import_context.py @@ -13,11 +13,11 @@ Other rust packages are likely to need similar treatment to pydantic_core. """ -from contextlib import contextmanager +import sys from array import array -from .import_patch_manager import block_calls +from contextlib import contextmanager -import sys +from .import_patch_manager import block_calls RUST_PACKAGES = ["pydantic_core", "tiktoken"] MODULES_TO_PATCH = [ @@ -34,8 +34,9 @@ def get_bad_entropy_flag(): - # simpleRunPython reads out stderr. We put the address there so we can fish it out... - # We could use ctypes instead of array but ctypes weighs an extra 100kb compared to array. + # simpleRunPython reads out stderr. We put the address there so we can fish + # it out... We could use ctypes instead of array but ctypes weighs an extra + # 100kb compared to array. print(ALLOWED_ENTROPY_CALLS.buffer_info()[0], file=sys.stderr) @@ -72,7 +73,8 @@ def get_entropy_import_context(name): if res: return res if name in RUST_PACKAGES: - # Initial import needs one entropy call to initialize std::collections::HashMap hash seed + # Initial import needs one entropy call to initialize + # std::collections::HashMap hash seed return rust_package_context raise Exception(f"Missing context for {name}") @@ -95,7 +97,7 @@ def random_context(module): # instantiating it without a seed will call getentropy() and fail. # Instantiating SystemRandom is fine, calling it's methods will call # getentropy() and fail. - block_calls(module, allowlist=["Random", "SystemRandom"]) + block_calls(module, allowlist=("Random", "SystemRandom")) @contextmanager @@ -109,7 +111,7 @@ def numpy_random_context(module): yield # Calling default_rng() with a given seed is fine, calling it without a seed # will call getentropy() and fail. - block_calls(module, allowlist=["default_rng"]) + block_calls(module, allowlist=("default_rng",)) @contextmanager @@ -125,15 +127,16 @@ def numpy_random_mtrand_context(module): @contextmanager def pydantic_core_context(module): try: - # Initial import needs one entropy call to initialize std::collections::HashMap hash seed + # Initial import needs one entropy call to initialize + # std::collections::HashMap hash seed with allow_bad_entropy_calls(1): yield finally: try: with allow_bad_entropy_calls(1): - # validate_core_schema makes an ahash::AHashMap which makes another entropy call for - # its hash seed. It will throw an error but only after making the needed entropy - # call. + # validate_core_schema makes an ahash::AHashMap which makes + # another entropy call for its hash seed. It will throw an error + # but only after making the needed entropy call. module.validate_core_schema(None) except module.SchemaError: pass @@ -152,7 +155,7 @@ def patched_Random(): try: yield finally: - random.Random = random + random.Random = Random class DeterministicRandomNameSequence: diff --git a/src/pyodide/internal/topLevelEntropy/entropy_patches.py b/src/pyodide/internal/topLevelEntropy/entropy_patches.py index 3c5936d131a..1a73059ae4c 100644 --- a/src/pyodide/internal/topLevelEntropy/entropy_patches.py +++ b/src/pyodide/internal/topLevelEntropy/entropy_patches.py @@ -28,8 +28,8 @@ from functools import wraps from .entropy_import_context import ( - is_bad_entropy_enabled, get_entropy_import_context, + is_bad_entropy_enabled, tempfile_restore_random_name_sequence, ) from .import_patch_manager import ( @@ -46,15 +46,17 @@ def should_allow_entropy_call(): It doesn't really matter that much since we're not likely to recover from these anyways but it feels better. """ - # Allow if we've either entered request context or if we've temporarily enabled entropy. + # Allow if we've either entered request context or if we've temporarily + # enabled entropy. return IN_REQUEST_CONTEXT or is_bad_entropy_enabled() # Step 1. # -# Prevent calls to getentropy(). The intended way for `getentropy()` to fail is to set an EIO error, -# which turns into a Python OSError, so we raise this same error so that if we patch `getentropy` -# from the Emscripten C stdlib we can remove these patches without changing the behavior. +# Prevent calls to getentropy(). The intended way for `getentropy()` to fail is +# to set an EIO error, which turns into a Python OSError, so we raise this same +# error so that if we patch `getentropy` from the Emscripten C stdlib we can +# remove these patches without changing the behavior. EIO = 29 @@ -70,11 +72,12 @@ def patch_urandom(*args): def disable_urandom(): """ - Python os.urandom() calls C getentropy() which calls JS crypto.getRandomValues() which throws at - top level, fatally crashing the interpreter. + Python os.urandom() calls C getentropy() which calls JS + crypto.getRandomValues() which throws at top level, fatally crashing the + interpreter. - TODO: Patch Emscripten's getentropy() to return EIO if `crypto.getRandomValues()` throws. Then - we can remove this. + TODO: Patch Emscripten's getentropy() to return EIO if + `crypto.getRandomValues()` throws. Then we can remove this. """ os.urandom = patch_urandom @@ -89,8 +92,8 @@ def restore_urandom(): @wraps(orig_Random_seed) def patched_seed(self, val): """ - Random.seed calls _PyOs_URandom which will fatally fail in top level. Prevent this by raising a - RuntimeError instead. + Random.seed calls _PyOs_URandom which will fatally fail in top level. + Prevent this by raising a RuntimeError instead. """ if val is None and not should_allow_entropy_call(): raise OSError(EIO, "Cannot get entropy outside of request context") @@ -109,8 +112,8 @@ def restore_random_seed(): def reseed_rng(): """ - Step 5: Have to reseed randomness in the IoContext of the first request since we gave a low - quality seed when it was seeded at top level. + Step 5: Have to reseed randomness in the IoContext of the first request + since we gave a low quality seed when it was seeded at top level. """ from random import seed diff --git a/src/pyodide/internal/topLevelEntropy/import_patch_manager.py b/src/pyodide/internal/topLevelEntropy/import_patch_manager.py index 2599cdda23f..ed28cd54c14 100644 --- a/src/pyodide/internal/topLevelEntropy/import_patch_manager.py +++ b/src/pyodide/internal/topLevelEntropy/import_patch_manager.py @@ -9,8 +9,8 @@ IN_REQUEST_CONTEXT variable. """ -from functools import wraps import sys +from functools import wraps class PatchLoader: @@ -71,7 +71,7 @@ def install(get_import_context): @staticmethod def remove(): - for idx, val in enumerate(sys.meta_path): + for idx, val in enumerate(sys.meta_path): # noqa:B007 if isinstance(val, PatchFinder): break del sys.meta_path[idx] @@ -96,7 +96,7 @@ def remove_import_patch_manager(): ORIG_MODULES = {} -def block_calls(module, *, allowlist=[]): +def block_calls(module, *, allowlist=()): # Called from the import context for modules that need to block calls. sys.modules[module.__name__] = BlockedCallModule(module, allowlist) ORIG_MODULES[module.__name__] = module @@ -141,8 +141,9 @@ def __getattribute__(self, key): if key in super().__getattribute__("_allow_list"): return orig - # If we aren't in a request scope, the value is a callable, and it's not in the allow_list, - # return a wrapper that raises an error if it's called before entering the request scope. + # If we aren't in a request scope, the value is a callable, and it's not + # in the allow_list, return a wrapper that raises an error if it's + # called before entering the request scope. # TODO: this doesn't wrap classes correctly, does it matter? @wraps(orig) def wrapper(*args, **kwargs): diff --git a/src/workerd/server/tests/python/random/worker.py b/src/workerd/server/tests/python/random/worker.py index df63bcfa830..0b9dcbce6b9 100644 --- a/src/workerd/server/tests/python/random/worker.py +++ b/src/workerd/server/tests/python/random/worker.py @@ -4,7 +4,10 @@ Calls to random should only work inside a request context. """ -from random import random, randbytes, choice +# Disable do not `assert False` lint +# ruff: noqa: B011 + +from random import choice, randbytes, random try: random() @@ -38,7 +41,7 @@ def t1(): - from random import random, randbytes + from random import randbytes, random random() randbytes(5) diff --git a/tools/cross/format.py b/tools/cross/format.py index 8d5734eff1f..f43ce957b8b 100644 --- a/tools/cross/format.py +++ b/tools/cross/format.py @@ -6,10 +6,9 @@ import shutil import subprocess from argparse import ArgumentParser, Namespace -from typing import Optional, Callable -from pathlib import Path from dataclasses import dataclass - +from pathlib import Path +from typing import Callable, Optional CLANG_FORMAT = os.environ.get("CLANG_FORMAT", "clang-format") PRETTIER = os.environ.get("PRETTIER", "node_modules/.bin/prettier") @@ -31,7 +30,10 @@ def parse_args() -> Namespace: ) git_parser.add_argument( "--source", - help="consider files modified in the specified commit-ish; if not specified, defaults to all changes in the working directory", + help=( + "consider files modified in the specified commit-ish; " + "if not specified, defaults to all changes in the working directory" + ), type=str, required=False, default=None, @@ -56,7 +58,8 @@ def parse_args() -> Namespace: and (options.source is not None or options.target != "HEAD") ): logging.error( - "--staged cannot be used with --source or --target; use --staged with --source=HEAD" + "--staged cannot be used with --source or --target; " + "use --staged with --source=HEAD" ) exit(1) return options @@ -118,7 +121,9 @@ def buildifier(files: list[Path], check: bool = False) -> bool: def ruff(files: list[Path], check: bool = False) -> bool: - if files and not shutil.which(RUFF): + if not files: + return True + if not shutil.which(RUFF): msg = "Cannot find ruff, will not format Python" if check: # In ci, fail. @@ -129,11 +134,17 @@ def ruff(files: list[Path], check: bool = False) -> bool: # formatting they can install ruff and run again. logging.warning(msg) return True + # lint + cmd = [RUFF, "check"] + if not check: + cmd.append("--fix") + result1 = subprocess.run(cmd + files) + # format cmd = [RUFF, "format"] if check: cmd.append("--diff") - result = subprocess.run(cmd + files) - return result.returncode == 0 + result2 = subprocess.run(cmd + files) + return result1.returncode == 0 and result2.returncode == 0 def git_get_modified_files(