diff --git a/pyo3-macros-backend/src/lib.rs b/pyo3-macros-backend/src/lib.rs index 5d7437a4295..c041b6635da 100644 --- a/pyo3-macros-backend/src/lib.rs +++ b/pyo3-macros-backend/src/lib.rs @@ -15,6 +15,7 @@ mod konst; mod method; mod module; mod params; +mod pycall; mod pyclass; mod pyfunction; mod pyimpl; @@ -24,6 +25,7 @@ mod quotes; pub use frompyobject::build_derive_from_pyobject; pub use module::{pymodule_function_impl, pymodule_module_impl, PyModuleOptions}; +pub use pycall::{build_pycall_output, PycallInput}; pub use pyclass::{build_py_class, build_py_enum, PyClassArgs}; pub use pyfunction::{build_py_function, PyFunctionOptions}; pub use pyimpl::{build_py_methods, PyClassMethodsType}; diff --git a/pyo3-macros-backend/src/pycall.rs b/pyo3-macros-backend/src/pycall.rs new file mode 100644 index 00000000000..a9508faf3fc --- /dev/null +++ b/pyo3-macros-backend/src/pycall.rs @@ -0,0 +1,457 @@ +use std::collections::HashSet; + +use proc_macro2::TokenStream; +use quote::{format_ident, quote, quote_spanned}; +use syn::parse::{Parse, ParseStream}; +use syn::punctuated::Punctuated; +use syn::spanned::Spanned; +use syn::{parenthesized, token, Error, Result, Token}; + +#[derive(Debug)] +pub struct PycallInput { + pyo3_path: syn::Ident, + callable_or_receiver: syn::Expr, + method_name: Option, + args: ArgList, +} + +impl Parse for PycallInput { + fn parse(input: ParseStream<'_>) -> Result { + let pyo3_path = input.parse()?; + let callable = parse_ident_or_parenthesized_expr(&input)?; + let method_name = if input.peek(Token![.]) { + input.parse::()?; + let method_name = if input.peek(syn::Ident) { + PyString::FromIdent(input.parse()?) + } else { + let callable; + parenthesized!(callable in input); + PyString::FromValue(callable.parse()?) + }; + Some(method_name) + } else { + None + }; + let args = input.parse()?; + Ok(Self { + pyo3_path, + callable_or_receiver: callable, + method_name, + args, + }) + } +} + +#[derive(Debug)] +enum PyString { + FromIdent(syn::Ident), + FromValue(syn::Expr), +} + +#[derive(Debug)] +struct ArgList(Punctuated); + +impl Parse for ArgList { + fn parse(input: ParseStream<'_>) -> Result { + let args; + parenthesized!(args in input); + let args = Punctuated::parse_terminated(&args)?; + Ok(Self(args)) + } +} + +#[derive(Debug)] +enum Arg { + Arg(syn::Expr), + Kwarg { + name: PyString, + value: syn::Expr, + }, + UnpackArgs { + unpack_parens: token::Paren, + value: syn::Expr, + }, + UnpackKwargs { + unpack_parens: token::Paren, + value: syn::Expr, + }, +} + +impl Parse for Arg { + fn parse(input: ParseStream<'_>) -> Result { + if input.peek(token::Paren) { + if input.peek2(Token![=]) { + let name; + parenthesized!(name in input); + let name = PyString::FromValue(name.parse()?); + input.parse::()?; + let value = input.parse()?; + return Ok(Arg::Kwarg { name, value }); + } + let in_parens; + let unpack_parens = parenthesized!(in_parens in input.fork()); + if in_parens.parse::().is_ok() { + if in_parens.is_empty() { + let stars; + parenthesized!(stars in input); + // Necessary because syn checks we parsed the full thing. + stars.parse::()?; + let value = input.parse()?; + return Ok(Arg::UnpackArgs { + unpack_parens, + value, + }); + } + if in_parens.parse::().is_ok() && in_parens.is_empty() { + let stars; + parenthesized!(stars in input); + // Necessary because syn checks we parsed the full thing. + stars.parse::()?; + let value = input.parse()?; + return Ok(Arg::UnpackKwargs { + unpack_parens, + value, + }); + } + } + } + if input.peek(syn::Ident) && input.peek2(Token![=]) { + let name = PyString::FromIdent(input.parse()?); + input.parse::()?; + let value = input.parse()?; + return Ok(Arg::Kwarg { name, value }); + } + let value = input.parse()?; + Ok(Arg::Arg(value)) + } +} + +fn parse_ident_or_parenthesized_expr(input: &ParseStream<'_>) -> Result { + if input.peek(syn::Ident) { + Ok(syn::Expr::Path(input.parse()?)) + } else { + let callable; + parenthesized!(callable in input); + callable.parse() + } +} + +pub fn build_pycall_output(input: PycallInput) -> Result { + check_args_order(&input.args)?; + check_duplicate_kwargs(&input.args)?; + let pyo3_path = &input.pyo3_path; + let result = store_values_in_variables(input.args, |args_with_variables| { + let args = build_args(pyo3_path, args_with_variables); + let kwargs = build_kwargs(pyo3_path, args_with_variables); + let callable_or_receiver = &input.callable_or_receiver; + let tokens = match &input.method_name { + Some(method_name) => { + let method_name = match method_name { + PyString::FromIdent(method_name) => { + let method_name = method_name.to_string(); + let method_name = method_name.trim_start_matches("r#"); + quote! { #pyo3_path::intern!(unsafe { #pyo3_path::Python::assume_gil_acquired() }, #method_name) } + } + PyString::FromValue(method_name) => quote! { #method_name }, + }; + quote! { + #pyo3_path::pycall::call_method(&(#callable_or_receiver), #method_name, #args, #kwargs) + } + } + None => quote! { + #pyo3_path::pycall::call(&(#callable_or_receiver), #args, #kwargs) + }, + }; + wrap_call(pyo3_path, tokens) + }); + Ok(result) +} + +fn check_duplicate_kwargs(args: &ArgList) -> Result<()> { + let kwargs = args.0.iter().filter_map(|arg| match arg { + Arg::Kwarg { + name: PyString::FromIdent(name), + value: _, + } => Some(name), + _ => None, + }); + let mut prev_kwargs = HashSet::new(); + let mut errors = Vec::new(); + for kwarg in kwargs { + let kwarg_string = kwarg.to_string().trim_start_matches("r#").to_owned(); + if prev_kwargs.contains(&kwarg_string) { + errors.push(syn::Error::new_spanned(kwarg, "duplicate kwarg")); + } else { + prev_kwargs.insert(kwarg_string); + } + } + let errors = errors.into_iter().reduce(|mut errors, error| { + errors.combine(error); + errors + }); + match errors { + None => Ok(()), + Some(errors) => Err(errors), + } +} + +fn check_args_order(args: &ArgList) -> Result<()> { + let mut started_kwargs = false; + for arg in &args.0 { + match arg { + Arg::Kwarg { .. } | Arg::UnpackKwargs { .. } => started_kwargs = true, + Arg::Arg(arg) => { + if started_kwargs { + return Err(Error::new_spanned(arg, "normal argument after kwargs")); + } + } + Arg::UnpackArgs { unpack_parens, .. } => { + if started_kwargs { + return Err(Error::new( + unpack_parens.span.span(), + "normal arguments unpack after kwargs", + )); + } + } + } + } + Ok(()) +} + +fn wrap_call(pyo3_path: &syn::Ident, call: TokenStream) -> TokenStream { + quote! { + { + #[allow(unused_imports)] + use #pyo3_path::pycall::select_traits::*; + #call + } + } +} + +const MAX_TUPLE_SIZE: usize = 13; + +#[derive(Debug)] +enum ArgWithVariable { + Arg(syn::Expr), + Kwarg { name: PyString, value: syn::Expr }, + UnpackArgs(syn::Ident), + UnpackKwargs(syn::Ident), +} + +fn store_values_in_variables( + args: ArgList, + callback: impl FnOnce(&[ArgWithVariable]) -> TokenStream, +) -> TokenStream { + let idents_exprs = args + .0 + .iter() + .filter_map(|arg| match arg { + Arg::UnpackArgs { + unpack_parens: _, + value, + } + | Arg::UnpackKwargs { + unpack_parens: _, + value, + } => Some(value.clone()), + Arg::Arg(..) | Arg::Kwarg { .. } => None, + }) + .collect::>(); + let args_with_variables = args + .0 + .into_iter() + .enumerate() + .map(|(index, arg)| match arg { + Arg::Arg(expr) => ArgWithVariable::Arg(expr), + Arg::Kwarg { name, value } => ArgWithVariable::Kwarg { name, value }, + Arg::UnpackKwargs { + unpack_parens, + value: _, + } => ArgWithVariable::UnpackKwargs(format_ident!( + "__py_arg_{index}", + span = unpack_parens.span.span() + )), + Arg::UnpackArgs { + unpack_parens, + value: _, + } => ArgWithVariable::UnpackArgs(format_ident!( + "__py_arg_{index}", + span = unpack_parens.span.span() + )), + }) + .collect::>(); + let idents = args_with_variables.iter().filter_map(|arg| match arg { + ArgWithVariable::UnpackArgs(ident) | ArgWithVariable::UnpackKwargs(ident) => Some(ident), + ArgWithVariable::Arg(..) | ArgWithVariable::Kwarg { .. } => None, + }); + let inside_match = callback(&args_with_variables); + quote! { + // `match` so that temporaries will live well. + match ( #( #idents_exprs, )* ) { + ( #( #idents, )* ) => { + #inside_match + } + } + } +} + +fn build_args(pyo3_path: &syn::Ident, args: &[ArgWithVariable]) -> TokenStream { + fn write_normal_args( + pyo3_path: &syn::Ident, + tokens: TokenStream, + consecutive_normal_args: &mut Vec<&syn::Expr>, + ) -> TokenStream { + if consecutive_normal_args.is_empty() { + return tokens; + } + + let new_args = quote! { + #pyo3_path::pycall::non_unpacked_args( ( #( #consecutive_normal_args, )* ) ) + }; + let result = quote! { + #pyo3_path::pycall::concat_args( #tokens, #new_args ) + }; + consecutive_normal_args.clear(); + result + } + + let mut consecutive_normal_args = Vec::new(); + let mut tokens = quote!(#pyo3_path::pycall::EmptyArgsStorage); + for arg in args { + match arg { + ArgWithVariable::Arg(arg) => { + consecutive_normal_args.push(arg); + if consecutive_normal_args.len() == MAX_TUPLE_SIZE { + tokens = write_normal_args(pyo3_path, tokens, &mut consecutive_normal_args); + } + } + ArgWithVariable::UnpackArgs(variable) => { + tokens = write_normal_args(pyo3_path, tokens, &mut consecutive_normal_args); + + let selector = quote! { + (&&&&&&&&&&&#pyo3_path::pycall::ArgsStorageSelector::new(loop { + break None; + // The block is needed because the compiler doesn't respect the `#[allow]` otherwise. + #[allow(unreachable_code)] + { + break Some(#variable); + } + })) + }; + let select = quote_spanned! { variable.span() => + #selector.__py_unpack_args_select(#variable) + }; + tokens = quote_spanned! { variable.span() => + #pyo3_path::pycall::concat_args( #tokens, #select ) + }; + } + ArgWithVariable::Kwarg { .. } | ArgWithVariable::UnpackKwargs(..) => break, + } + } + if !consecutive_normal_args.is_empty() { + tokens = write_normal_args(pyo3_path, tokens, &mut consecutive_normal_args); + } + tokens +} + +fn build_known_kwargs(pyo3_path: &syn::Ident, args: &[ArgWithVariable]) -> Option { + let mut known_kwargs = args.iter().filter_map(|it| match it { + ArgWithVariable::Kwarg { + name: PyString::FromIdent(name), + value, + } => Some((name, value)), + _ => None, + }); + let names = known_kwargs + .clone() + .map(|(name, _)| name.to_string().trim_start_matches("r#").to_owned()); + let mut values = match known_kwargs.next() { + Some((_, first_kwarg)) => quote! { #pyo3_path::pycall::first_known_kwarg( #first_kwarg ) }, + None => return None, + }; + for (_, kwarg) in known_kwargs { + values = quote! { #pyo3_path::pycall::add_known_kwarg( #kwarg, #values ) }; + } + Some(quote! { + #pyo3_path::pycall::known_kwargs_with_names( + #pyo3_path::known_kwargs!( #(#names)* ), + #values, + ) + }) +} + +fn build_unknown_non_unpacked_kwargs( + pyo3_path: &syn::Ident, + args: &[ArgWithVariable], +) -> TokenStream { + fn write_kwargs( + pyo3_path: &syn::Ident, + mut tokens: TokenStream, + consecutive_kwargs: &mut Vec, + ) -> TokenStream { + tokens = quote! { + #pyo3_path::pycall::concat_kwargs( + #tokens, + #pyo3_path::pycall::non_unpacked_kwargs( ( #( #consecutive_kwargs, )* ) ), + ) + }; + consecutive_kwargs.clear(); + tokens + } + + let kwargs = args.iter().filter_map(|it| match it { + ArgWithVariable::Kwarg { + name: PyString::FromValue(name), + value, + } => Some((name, value)), + _ => None, + }); + let mut tokens = quote! { #pyo3_path::pycall::EmptyKwargsStorage }; + let mut consecutive_kwargs = Vec::new(); + for (name, value) in kwargs { + consecutive_kwargs.push(quote! { (#name, #value) }); + + if consecutive_kwargs.len() == MAX_TUPLE_SIZE { + tokens = write_kwargs(pyo3_path, tokens, &mut consecutive_kwargs); + } + } + if !consecutive_kwargs.is_empty() { + tokens = write_kwargs(pyo3_path, tokens, &mut consecutive_kwargs); + } + tokens +} + +fn build_kwargs(pyo3_path: &syn::Ident, args: &[ArgWithVariable]) -> TokenStream { + let known = build_known_kwargs(pyo3_path, args); + let unknown_non_unpacked = build_unknown_non_unpacked_kwargs(pyo3_path, args); + let mut tokens = match known { + Some(known) => { + quote! { #pyo3_path::pycall::concat_kwargs( #known, #unknown_non_unpacked ) } + } + None => unknown_non_unpacked, + }; + + let unpacked_kwargs = args.iter().filter_map(|it| match it { + ArgWithVariable::UnpackKwargs(variable) => Some(variable), + _ => None, + }); + for variable in unpacked_kwargs { + let selector = quote! { + (&&&&&&&&&&&#pyo3_path::pycall::KwargsStorageSelector::new(loop { + break None; + // The block is needed because the compiler doesn't respect the `#[allow]` otherwise. + #[allow(unreachable_code)] + { + break Some(#variable); + } + })) + }; + let select = quote_spanned! { variable.span() => + #selector.__py_unpack_kwargs_select(#variable) + }; + tokens = quote_spanned! { variable.span() => + #pyo3_path::pycall::concat_kwargs( #tokens, #select ) + }; + } + tokens +} diff --git a/pyo3-macros/src/lib.rs b/pyo3-macros/src/lib.rs index c4263a512d3..21e9b3b782e 100644 --- a/pyo3-macros/src/lib.rs +++ b/pyo3-macros/src/lib.rs @@ -6,8 +6,8 @@ use proc_macro::TokenStream; use proc_macro2::TokenStream as TokenStream2; use pyo3_macros_backend::{ build_derive_from_pyobject, build_py_class, build_py_enum, build_py_function, build_py_methods, - pymodule_function_impl, pymodule_module_impl, PyClassArgs, PyClassMethodsType, - PyFunctionOptions, PyModuleOptions, + build_pycall_output, pymodule_function_impl, pymodule_module_impl, PyClassArgs, + PyClassMethodsType, PyFunctionOptions, PyModuleOptions, PycallInput, }; use quote::quote; use syn::{parse_macro_input, Item}; @@ -163,6 +163,21 @@ pub fn derive_from_py_object(item: TokenStream) -> TokenStream { .into() } +#[proc_macro] +pub fn pycall_impl(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as PycallInput); + build_pycall_output(input) + .unwrap_or_else(|err| { + let err = err.into_compile_error(); + // Turn it into an expression, otherwise syn emits a `compile_error!` that looks to the + // compiler like an item. + quote! { + ({ #err }) + } + }) + .into() +} + fn pyclass_impl( attrs: TokenStream, mut ast: syn::ItemStruct, diff --git a/src/instance.rs b/src/instance.rs index b0a4a38d176..ffee2c5cc19 100644 --- a/src/instance.rs +++ b/src/instance.rs @@ -32,6 +32,10 @@ pub trait BoundObject<'py, T>: bound_object_sealed::Sealed { fn into_ptr(self) -> *mut ffi::PyObject; /// Turn this smart pointer into an owned [`Py`] fn unbind(self) -> Py; + /// Turn this smart pointer into a raw pointer, that may be a strong reference or may not be. + fn into_ptr_raw(self) -> *mut ffi::PyObject; + /// Whether this is an owned `Bound` or a `Borrowed`. + const IS_OWNED: bool; } mod bound_object_sealed { @@ -619,6 +623,12 @@ impl<'py, T> BoundObject<'py, T> for Bound<'py, T> { fn unbind(self) -> Py { self.unbind() } + + fn into_ptr_raw(self) -> *mut ffi::PyObject { + self.into_ptr() + } + + const IS_OWNED: bool = true; } /// A borrowed equivalent to `Bound`. @@ -825,6 +835,12 @@ impl<'a, 'py, T> BoundObject<'py, T> for Borrowed<'a, 'py, T> { fn unbind(self) -> Py { (*self).to_owned().unbind() } + + fn into_ptr_raw(self) -> *mut ffi::PyObject { + self.as_ptr() + } + + const IS_OWNED: bool = false; } /// A GIL-independent reference to an object allocated on the Python heap. diff --git a/src/lib.rs b/src/lib.rs index 7f1329c9ea5..8f8b445dd0e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -431,6 +431,8 @@ pub mod pycell; pub mod pyclass; pub mod pyclass_init; +#[doc(hidden)] +pub mod pycall; pub mod type_object; pub mod types; mod version; diff --git a/src/pycall.rs b/src/pycall.rs new file mode 100644 index 00000000000..a4a4d44dd8c --- /dev/null +++ b/src/pycall.rs @@ -0,0 +1,658 @@ +mod args; +mod as_pyobject; +mod kwargs; +mod kwargs_args_adapter; +mod storage; +mod trusted_len; + +pub mod select_traits { + pub use super::args::select_traits::*; + pub use super::kwargs::select_traits::*; +} + +use std::mem::MaybeUninit; + +pub use args::{ArgsStorageSelector, EmptyArgsStorage}; +pub use kwargs::{EmptyKwargsStorage, KwargsStorageSelector}; +pub use pyo3_macros::pycall_impl; + +use crate::exceptions::PyTypeError; +use crate::ffi_ptr_ext::FfiPtrExt; +use crate::prelude::IntoPyObject; +use crate::types::{PyAnyMethods, PyDict, PyDictMethods, PyString, PyTuple}; +use crate::{ffi, Borrowed, Bound, BoundObject, PyAny, PyResult, Python}; +use args::{ + AppendEmptyArgForVectorcall, ArgumentsOffsetFlag, ArrayArgsStorage, ConcatArgsStorages, + ResolveArgs, +}; +use kwargs::{ArrayKwargsStorage, ConcatKwargsStorages, KnownKwargsNames}; +use kwargs_args_adapter::{CombineArgsKwargs, KwargsArgsAdapter}; +use storage::RawStorage; + +type PPPyObject = *mut *mut ffi::PyObject; + +#[inline(always)] +pub fn concat_args<'py, A, B>(a: A, b: B) -> >::Output +where + A: ConcatArgsStorages<'py, B>, + A: args::FundamentalStorage<'py>, + B: args::FundamentalStorage<'py>, +{ + A::concat(a, b) +} + +#[inline(always)] +pub fn concat_kwargs<'py, A, B>(a: A, b: B) -> >::Output +where + A: ConcatKwargsStorages<'py, B>, + A: kwargs::FundamentalStorage<'py>, + B: kwargs::FundamentalStorage<'py>, +{ + A::concat(a, b) +} + +#[inline(always)] +pub fn non_unpacked_args<'py, T: args::Tuple<'py>>(tuple: T) -> ArrayArgsStorage { + ArrayArgsStorage(tuple) +} + +#[inline(always)] +pub fn non_unpacked_kwargs<'py, T: kwargs::Tuple<'py>>(tuple: T) -> ArrayKwargsStorage { + ArrayKwargsStorage(tuple) +} + +#[inline(always)] +pub fn first_known_kwarg<'py, Kwarg: IntoPyObject<'py>>( + kwarg: Kwarg, +) -> kwargs::TypeLevelPyObjectListCons { + kwargs::TypeLevelPyObjectListCons(kwarg, kwargs::TypeLevelPyObjectListNil) +} + +#[inline(always)] +pub fn add_known_kwarg< + 'py, + Kwarg: IntoPyObject<'py>, + T: kwargs::TypeLevelPyObjectListTrait<'py>, +>( + kwarg: Kwarg, + existing_known: T, +) -> kwargs::TypeLevelPyObjectListCons { + kwargs::TypeLevelPyObjectListCons(kwarg, existing_known) +} + +#[inline(always)] +pub fn known_kwargs_with_names<'py, Values: kwargs::TypeLevelPyObjectListTrait<'py>>( + names: &'static KnownKwargsNames, + values: Values, +) -> kwargs::KnownKwargsStorage<'py, Values> { + kwargs::KnownKwargsStorage { + names: names + .0 + .bind_borrowed(unsafe { Python::assume_gil_acquired() }), + values, + } +} + +/// Call any Python callable, with a syntax similar to Python and maximum performance. +/// +/// The syntax for calling a callable is: +/// ``` +/// pycall!( +/// callable( +/// arg1, arg2, (*)unpack_args1, arg3, (*)unpack_args2, +/// (**)unpack_kwargs1, kwarg1=value, kwarg2=value, (**)unpack_kwargs2, +/// (kwarg_name_expression)=value, +/// ) +/// ) +/// ``` +/// An argument can be any expression that implements [`IntoPyObject`]. +/// +/// An unpacked argument can be any `IntoIterator` whose `Item` implements `IntoPyObject`, +/// or any Python iterable. +/// +/// A keyword argument's name can be either an identifier, or an expression surrounded in parentheses +/// that produces any type that implements `IntoPyObject`. Its value can be +/// of any type that implements `IntoPyObject`. +/// +/// An unpacked keyword argument can be either any `IntoIterator` where `K` implements +/// `IntoPyObject` and `V` implements `IntoPyObject`, or any Python mapping. +/// +/// `callable` can either be an identifier or an expression surrounded in parentheses that produces +/// [`Bound`] or [`Borrowed`] or any reference to them. +/// +/// Similarly, a method can be called: +/// ``` +/// pycall!(object.method(...)) +/// pycall!(object.(method_name_expression)(...)) +/// ``` +/// +/// `object` can be any [`Bound`] or [`Borrowed`] or a reference to them. It can either be a single identifier +/// or an expression surrounded in parentheses (similar to `callable`). +/// +/// The method name after the dot +/// can either be an identifier, in which case it is treated as the name of the method to call, +/// or an expression surrounded in parenthesis that produces a type that implements `IntoPyObject`, +/// in which case its value is the method name. +/// +/// Note, `object.method()` and `object.(method)()` are **not** the same thing! The former calls the method named +/// `"method"` on `object`, while the latter calls the method whose name is stored in `method` on `object`. +/// +/// Method arguments rules are identical to non-method. +/// +/// The call returns [PyResult]<[PyAny]>. +/// +/// The macro will try its best to pick the most performant way to call the function. +#[macro_export] +macro_rules! pycall { + ( $($t:tt)* ) => { $crate::pycall::pycall_impl!($crate $($t)*) }; +} + +pub trait BoundPyObject<'py> { + fn py(&self) -> Python<'py>; + fn as_borrowed(&self) -> Borrowed<'_, 'py, PyAny>; +} + +impl<'py, T> BoundPyObject<'py> for Bound<'py, T> { + #[inline(always)] + fn py(&self) -> Python<'py> { + self.py() + } + #[inline(always)] + fn as_borrowed(&self) -> Borrowed<'_, 'py, PyAny> { + self.as_borrowed().into_any() + } +} + +impl<'py, T> BoundPyObject<'py> for Borrowed<'_, 'py, T> { + #[inline(always)] + fn py(&self) -> Python<'py> { + (**self).py() + } + #[inline(always)] + fn as_borrowed(&self) -> Borrowed<'_, 'py, PyAny> { + self.into_any() + } +} + +impl<'py, T: ?Sized + BoundPyObject<'py>> BoundPyObject<'py> for &'_ T { + #[inline(always)] + fn py(&self) -> Python<'py> { + T::py(self) + } + #[inline(always)] + fn as_borrowed(&self) -> Borrowed<'_, 'py, PyAny> { + T::as_borrowed(*self) + } +} + +impl<'py, T: ?Sized + BoundPyObject<'py>> BoundPyObject<'py> for &'_ mut T { + #[inline(always)] + fn py(&self) -> Python<'py> { + T::py(self) + } + #[inline(always)] + fn as_borrowed(&self) -> Borrowed<'_, 'py, PyAny> { + T::as_borrowed(*self) + } +} + +type AppendVectorcallOffset<'py, Args> = + >::Output; + +#[inline(always)] +fn kwargs_to_dict<'py, Kwargs>( + py: Python<'py>, + kwargs: Kwargs, + kwargs_can_be_cheaply_converted_to_pydict: bool, +) -> PyResult> +where + Kwargs: kwargs::ResolveKwargs<'py>, +{ + if kwargs_can_be_cheaply_converted_to_pydict { + return kwargs.into_pydict(py); + } + let kwargs_dict = PyDict::new(py); + let expected_len = kwargs.write_to_dict(kwargs_dict.as_borrowed())?; + // Python doesn't allow us to check efficiently if `PyDict_SetItem()` overwrote + // an existing value, so we check the length instead. + if kwargs_dict.len() != expected_len { + return Err(PyTypeError::new_err( + intern!(py, "got multiple values for keyword argument") + .clone() + .unbind(), + )); + } + Ok(kwargs_dict) +} + +#[inline(always)] +unsafe fn call_tuple_dict<'py, Args>( + py: Python<'py>, + args: Args, + kwargs: *mut ffi::PyObject, + do_call: impl FnOnce(*mut ffi::PyObject, *mut ffi::PyObject) -> *mut ffi::PyObject, +) -> PyResult<*mut ffi::PyObject> +where + Args: args::FundamentalStorage<'py>, +{ + if let Some(args_tuple) = args.as_pytuple(py) { + return Ok(do_call(args_tuple.as_ptr(), kwargs)); + } + + let len = args.len(); + let tuple = if args.has_known_size() { + let tuple = ffi::PyTuple_New( + len.try_into() + .expect("too many arguments requested for a call"), + ) + .assume_owned_or_err(py)? + .downcast_into_unchecked(); + args.write_to_tuple(tuple.as_borrowed(), &mut 0)?; + tuple + } else { + let mut storage = Args::RawStorage::new(len); + let mut base_storage = storage.as_ptr(); + // DO NOT remove the `as *mut PPPyObject`, due to a rustc bug without it you have aliasing violations. + let guard = args.init( + py, + storage.as_init_param(), + &mut base_storage as *mut PPPyObject as *const PPPyObject, + )?; + let tuple = ffi::PyTuple_New(storage.len() as ffi::Py_ssize_t) + .assume_owned_or_err(py)? + .downcast_into_unchecked(); + Args::write_initialized_to_tuple(tuple.as_borrowed(), guard, &mut storage.as_ptr(), &mut 0); + tuple + }; + Ok(do_call(tuple.as_ptr(), kwargs)) +} + +const MAX_STACK_LEN: usize = 11; + +#[inline(always)] +unsafe fn call_vectorcall_with_kwargs_names<'py, Args>( + py: Python<'py>, + all_args: Args, + kwargs_names: *mut ffi::PyObject, + kwargs_len: usize, + do_call: impl FnOnce(PPPyObject, usize, *mut ffi::PyObject) -> *mut ffi::PyObject, +) -> PyResult<*mut ffi::PyObject> +where + Args: args::FundamentalStorage<'py>, +{ + if Args::USE_STACK_FOR_SMALL_LEN && all_args.has_known_size() && all_args.len() <= MAX_STACK_LEN + { + let mut storage = MaybeUninit::<[*mut ffi::PyObject; MAX_STACK_LEN]>::uninit(); + let mut base_storage = storage.as_mut_ptr().cast::<*mut ffi::PyObject>(); + let positional_len = all_args.len() - kwargs_len; + let _guard = all_args.init( + py, + Args::RawStorage::init_param_from_ptr(base_storage), + &mut base_storage as *mut PPPyObject as *const PPPyObject, + )?; + Ok(do_call(base_storage, positional_len, kwargs_names)) + } else { + let mut storage = Args::RawStorage::new(all_args.len()); + let mut base_storage = storage.as_ptr(); + let _guard = all_args.init( + py, + storage.as_init_param(), + &mut base_storage as *mut PPPyObject as *const PPPyObject, + )?; + let positional_len = storage.len() - kwargs_len; + Ok(do_call(storage.as_ptr(), positional_len, kwargs_names)) + } +} + +#[inline(always)] +unsafe fn call_vectorcall<'py, Args, Kwargs>( + py: Python<'py>, + args: Args, + kwargs: Kwargs, + do_call: impl FnOnce(PPPyObject, usize, *mut ffi::PyObject) -> *mut ffi::PyObject, +) -> PyResult<*mut ffi::PyObject> +where + Args: args::FundamentalStorage<'py>, + Kwargs: kwargs::FundamentalStorage<'py>, + Args: for<'a> CombineArgsKwargs<'a, 'py, Kwargs>, +{ + debug_assert!(kwargs.has_known_size()); + if Kwargs::IS_EMPTY { + return call_vectorcall_with_kwargs_names(py, args, std::ptr::null_mut(), 0, do_call); + } + let kwargs_len = kwargs.len(); + if let Some(kwargs_names) = kwargs.as_names_pytuple() { + let all_args = Args::combine_no_names(args, kwargs); + return call_vectorcall_with_kwargs_names( + py, + all_args, + kwargs_names.as_ptr(), + kwargs_len, + do_call, + ); + } + // This will be filled inside `call_vectorcall_with_kwargs_names()`, when we initialize the storage. + let kwargs_names = ffi::PyTuple_New( + kwargs + .len() + .try_into() + .expect("too many arguments requested for a call"), + ) + .assume_owned_or_err(py)? + .downcast_into_unchecked::(); + let all_args = Args::combine( + args, + KwargsArgsAdapter { + kwargs, + kwargs_tuple: kwargs_names.as_borrowed(), + }, + ); + call_vectorcall_with_kwargs_names(py, all_args, kwargs_names.as_ptr(), kwargs_len, do_call) +} + +#[inline] +pub fn call<'py, F, Args, Kwargs>(f: F, args: Args, kwargs: Kwargs) -> PyResult> +where + F: BoundPyObject<'py>, + Args: args::FundamentalStorage<'py>, + Kwargs: kwargs::FundamentalStorage<'py>, + Args: for<'a> CombineArgsKwargs<'a, 'py, Kwargs>, + AppendEmptyArgForVectorcall: ConcatArgsStorages<'py, Args>, + AppendVectorcallOffset<'py, Args>: for<'a> CombineArgsKwargs<'a, 'py, Kwargs>, +{ + // Assertions for extra safety, not required. + if Args::IS_EMPTY { + debug_assert_eq!(args.len(), 0); + } + if Args::IS_ONE { + debug_assert_eq!(args.len(), 1); + } + if Kwargs::IS_EMPTY { + debug_assert_eq!(kwargs.len(), 0); + } + + let py = f.py(); + let f = f.as_borrowed().as_ptr(); + unsafe { + let result = 'result: { + if Args::IS_EMPTY && Kwargs::IS_EMPTY { + break 'result ffi::PyObject_CallNoArgs(f); + } + + if Args::IS_ONE && Kwargs::IS_EMPTY { + let mut storage = Args::RawStorage::new(1); + let mut base_storage = storage.as_ptr(); + let _guard = args.init( + py, + storage.as_init_param(), + &mut base_storage as *mut PPPyObject as *const PPPyObject, + ); + break 'result ffi::PyObject_CallOneArg(f, *storage.as_ptr()); + } + + let kwargs_can_be_cheaply_converted_to_pydict = + kwargs.can_be_cheaply_converted_to_pydict(py); + + // The arguments are readily available as a tuple; do not spend time trying to figure out vectorcall - + // just pass them directly with normal calling convention. + if let Some(tuple_args) = args.as_pytuple(py) { + if Kwargs::IS_EMPTY { + break 'result ffi::PyObject_Call(f, tuple_args.as_ptr(), std::ptr::null_mut()); + } + if kwargs_can_be_cheaply_converted_to_pydict { + let kwargs_dict = kwargs.into_pydict(py)?; + break 'result ffi::PyObject_Call(f, tuple_args.as_ptr(), kwargs_dict.as_ptr()); + } + } + + if Args::IS_EMPTY && kwargs_can_be_cheaply_converted_to_pydict { + let kwargs_dict = kwargs.into_pydict(py)?; + break 'result ffi::PyObject_Call(f, ffi::PyTuple_New(0), kwargs_dict.as_ptr()); + } + + if !kwargs.has_known_size() { + let kwargs_dict = + kwargs_to_dict(py, kwargs, kwargs_can_be_cheaply_converted_to_pydict)?; + break 'result call_tuple_dict(py, args, kwargs_dict.as_ptr(), |args, kwargs| { + ffi::PyObject_Call(f, args, kwargs) + })?; + } + + let vectorcall_fn = ffi::PyVectorcall_Function(f); + match vectorcall_fn { + Some(vectorcall_fn) => { + type CombinedArgsKwargs<'a, 'py, Args, Kwargs> = + >::Output; + + match CombinedArgsKwargs::<'_, 'py, Args, Kwargs>::ARGUMENTS_OFFSET { + ArgumentsOffsetFlag::Normal => { + call_vectorcall( + py, + concat_args(AppendEmptyArgForVectorcall, args), + kwargs, + |args, args_len, kwargs_names| { + // Add 1 to the arguments pointer and subtract 1 from the length because of `PY_VECTORCALL_ARGUMENTS_OFFSET`. + vectorcall_fn( + f, + args.add(1), + args_len - 1 + ffi::PY_VECTORCALL_ARGUMENTS_OFFSET, + kwargs_names, + ) + }, + )? + } + ArgumentsOffsetFlag::DoNotOffset + | ArgumentsOffsetFlag::DoNotOffsetButCanChangeArgs0 => { + call_vectorcall(py, args, kwargs, |args, args_len, kwargs_names| { + vectorcall_fn(f, args, args_len, kwargs_names) + })? + } + } + } + None => { + // vectorcall is not available; instead of spending time converting the arguments, + // when Python will convert them then again to a tuple, just create a tuple directly. + if Kwargs::IS_EMPTY { + break 'result call_tuple_dict( + py, + args, + std::ptr::null_mut(), + |args, kwargs| ffi::PyObject_Call(f, args, kwargs), + )?; + } else { + let kwargs_dict = + kwargs_to_dict(py, kwargs, kwargs_can_be_cheaply_converted_to_pydict)?; + break 'result call_tuple_dict( + py, + args, + kwargs_dict.as_ptr(), + |args, kwargs| ffi::PyObject_Call(f, args, kwargs), + )?; + } + } + } + }; + result.assume_owned_or_err(py) + } +} + +type AppendMethodReceiver<'a, 'py, Args> = + ,)> as ConcatArgsStorages<'py, Args>>::Output; + +#[inline] +pub fn call_method<'a, 'py, Obj, Name, Args, Kwargs>( + obj: &'a Obj, + method_name: Name, + args: Args, + kwargs: Kwargs, +) -> PyResult> +where + Obj: BoundPyObject<'py>, + Name: IntoPyObject<'py, Target = PyString>, + Args: args::FundamentalStorage<'py>, + Kwargs: kwargs::FundamentalStorage<'py>, + ArrayArgsStorage<(Borrowed<'a, 'py, PyAny>,)>: ConcatArgsStorages<'py, Args>, + AppendMethodReceiver<'a, 'py, Args>: for<'b> CombineArgsKwargs<'b, 'py, Kwargs>, +{ + // Assertions for extra safety, not required. + if Args::IS_EMPTY { + debug_assert_eq!(args.len(), 0); + } + if Args::IS_ONE { + debug_assert_eq!(args.len(), 1); + } + if Kwargs::IS_EMPTY { + debug_assert_eq!(kwargs.len(), 0); + } + + let py = obj.py(); + let obj = obj.as_borrowed(); + let method_name = method_name.into_pyobject(py).map_err(Into::into)?; + let method_name = method_name.as_borrowed(); + unsafe { + let result = 'result: { + if Args::IS_EMPTY && Kwargs::IS_EMPTY { + break 'result ffi::PyObject_CallMethodNoArgs(obj.as_ptr(), method_name.as_ptr()); + } + + if Args::IS_ONE && Kwargs::IS_EMPTY { + let mut storage = Args::RawStorage::new(1); + let mut base_storage = storage.as_ptr(); + let _guard = args.init( + py, + storage.as_init_param(), + &mut base_storage as *mut PPPyObject as *const PPPyObject, + ); + break 'result ffi::PyObject_CallMethodOneArg( + obj.as_ptr(), + method_name.as_ptr(), + *storage.as_ptr(), + ); + } + + let kwargs_can_be_cheaply_converted_to_pydict = + kwargs.can_be_cheaply_converted_to_pydict(py); + + // The arguments are readily available as a tuple; do not spend time trying to figure out vectorcall - + // just pass them directly with normal calling convention. + if let Some(tuple_args) = args.as_pytuple(py) { + // FIXME: Benchmark if this is faster than vectorcall. + if Kwargs::IS_EMPTY { + let method = obj.getattr(method_name)?; + break 'result ffi::PyObject_Call( + method.as_ptr(), + tuple_args.as_ptr(), + std::ptr::null_mut(), + ); + } + if kwargs_can_be_cheaply_converted_to_pydict { + let method = obj.getattr(method_name)?; + let kwargs_dict = kwargs.into_pydict(py)?; + break 'result ffi::PyObject_Call( + method.as_ptr(), + tuple_args.as_ptr(), + kwargs_dict.as_ptr(), + ); + } + } + + if Args::IS_EMPTY && kwargs_can_be_cheaply_converted_to_pydict { + let method = obj.getattr(method_name)?; + let kwargs_dict = kwargs.into_pydict(py)?; + break 'result ffi::PyObject_Call( + method.as_ptr(), + ffi::PyTuple_New(0), + kwargs_dict.as_ptr(), + ); + } + + if !kwargs.has_known_size() { + let method = obj.getattr(method_name)?; + let kwargs_dict = + kwargs_to_dict(py, kwargs, kwargs_can_be_cheaply_converted_to_pydict)?; + break 'result call_tuple_dict(py, args, kwargs_dict.as_ptr(), |args, kwargs| { + ffi::PyObject_Call(method.as_ptr(), args, kwargs) + })?; + } + + type CombinedArgsKwargs<'a, 'b, 'py, Args, Kwargs> = + as CombineArgsKwargs<'b, 'py, Kwargs>>::Output; + + match CombinedArgsKwargs::<'_, '_, 'py, Args, Kwargs>::ARGUMENTS_OFFSET { + ArgumentsOffsetFlag::Normal | ArgumentsOffsetFlag::DoNotOffsetButCanChangeArgs0 => { + call_vectorcall( + py, + concat_args(ArrayArgsStorage((obj,)), args), + kwargs, + |args, args_len, kwargs_names| { + ffi::PyObject_VectorcallMethod( + method_name.as_ptr(), + args, + args_len + ffi::PY_VECTORCALL_ARGUMENTS_OFFSET, + kwargs_names, + ) + }, + )? + } + ArgumentsOffsetFlag::DoNotOffset => { + unreachable!("since we concatenate the receiver this is unreachable") + // call_vectorcall( + // py, + // concat_args(ArrayArgsStorage((obj,)), args), + // kwargs, + // |args, args_len, kwargs_names| { + // ffi::PyObject_VectorcallMethod( + // method_name.as_ptr(), + // args, + // args_len, + // kwargs_names, + // ) + // }, + // )? + } + } + }; + result.assume_owned_or_err(py) + } +} + +// TODO: An option to call a method with a list of arguments that includes the receiver +// (as a first argument), that can be more efficient in case there is already a slice +// of pyobjects including the receiver. + +#[cfg(test)] +mod tests { + use crate::types::{PyAnyMethods, PyDict, PyDictMethods, PyModule, PyTuple}; + use crate::{PyResult, Python, ToPyObject}; + + #[test] + pub fn my_special_test() -> PyResult<()> { + Python::with_gil(|py| { + let unpack_args = (1, 2, 3); + let unpack_args2 = ["a", "b", "c"]; + let m = PyModule::from_code( + py, + cr#" +def f(*args, **kwargs): print(args, kwargs) +class mydict(dict): pass + "#, + c"my_module.py", + c"my_module", + )?; + let f = m.getattr("f")?; + let unpack_kwargs = PyDict::new(py); + unpack_kwargs.set_item("1", "hello")?; + pycall!(f(1, 2, 3, (*)unpack_args, 5. + 1.2, (*)unpack_args2, a="b", ("c")="d", (**)unpack_kwargs, d=1, e=2))?; + pycall!(f((*)(1, 2, 3)))?; + pycall!(f(1, "a",))?; + pycall!(f(a = 1))?; + let my_dict = pycall!(m.mydict(a = 1, b = 2, c = "abc"))?.downcast_into::()?; + pycall!(f((*)PyTuple::new(py, [1, 2, 3]), (**)&my_dict))?; + pycall!(f((*)[1.to_object(py), 2.to_object(py), 3.to_object(py)]))?; + pycall!(f((**)std::env::vars().filter(|(name, _)| name.starts_with(char::is_lowercase))))?; + + dbg!(&my_dict, my_dict.get_type()); + Ok(()) + }) + } +} diff --git a/src/pycall/args.rs b/src/pycall/args.rs new file mode 100644 index 00000000000..9963a2b7a44 --- /dev/null +++ b/src/pycall/args.rs @@ -0,0 +1,89 @@ +mod array; +mod concat; +mod empty; +mod existing; +mod helpers; +mod pyobjects; +mod selector; +mod unknown_size; +mod vec; +mod vectorcall_arguments_offset; + +use crate::types::PyTuple; +use crate::{ffi, Borrowed, PyResult, Python}; + +pub use empty::EmptyArgsStorage; +pub use selector::{select_traits, ArgsStorageSelector}; + +pub(super) use array::{ArrayArgsStorage, Tuple}; +pub(super) use concat::{ConcatArgsStorages, FundamentalStorage}; +pub(super) use existing::{ExistingArgListSlice, ExistingArgListSliceTrait}; +pub(super) use unknown_size::{SizedToUnsizedStorage, UnsizedArgsStorage}; +pub(super) use vec::VecArgsStorage; +pub(super) use vectorcall_arguments_offset::AppendEmptyArgForVectorcall; + +use super::storage::RawStorage; +use super::PPPyObject; + +#[derive(Debug)] +pub enum ArgumentsOffsetFlag { + /// Do not add an offset, as this will lead into more expensive conversion. + /// + /// We use this when unpacking an existing slice of args, because then the flag will + /// mean we need to allocate a new space. + DoNotOffset, + /// Like [`ArgumentsOffsetFlag::DoNotOffset`], but the memory is mutable, + /// and as such, when calling a method, that does not require offsetting - only + /// write access to `args[0]`, you can provide that. + DoNotOffsetButCanChangeArgs0, + /// You can add an offset and mutate `args[0]`. + Normal, +} + +pub(super) type InitParam<'a, 'py, T> = + <>::RawStorage as RawStorage>::InitParam<'a>; + +#[diagnostic::on_unimplemented( + message = "`{Self}` cannot be unpacked in a Python call", + note = "the following types can be unpacked in a Python call: \ + any iterable Python object, any `IntoIterator` that yields \ + types that can be converted into Python objects" +)] +pub trait ResolveArgs<'py>: Sized { + type RawStorage: RawStorage; + type Guard; + fn init( + self, + py: Python<'py>, + storage: InitParam<'_, 'py, Self>, + base_storage: *const PPPyObject, + ) -> PyResult; + fn len(&self) -> usize; + fn write_to_tuple( + self, + tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + ) -> PyResult<()>; + fn write_initialized_to_tuple( + tuple: Borrowed<'_, 'py, PyTuple>, + guard: Self::Guard, + raw_storage: &mut PPPyObject, + index: &mut ffi::Py_ssize_t, + ); + #[inline(always)] + fn as_pytuple(&self, _py: Python<'py>) -> Option> { + None + } + fn has_known_size(&self) -> bool; + const ARGUMENTS_OFFSET: ArgumentsOffsetFlag; + const IS_EMPTY: bool; + const IS_ONE: bool; + /// This is false for array storages, are they are already on stack. + const USE_STACK_FOR_SMALL_LEN: bool; +} + +pub struct ConcatStorages(pub(super) A, pub(super) B); + +/// This struct is used to create an array whose size is the sum of two smaller arrays, without generic_const_exprs. +#[repr(C)] +pub struct ConcatArrays(A, B); diff --git a/src/pycall/args/array.rs b/src/pycall/args/array.rs new file mode 100644 index 00000000000..26de29c8fe4 --- /dev/null +++ b/src/pycall/args/array.rs @@ -0,0 +1,370 @@ +use std::mem::MaybeUninit; + +use crate::conversion::IntoPyObject; +use crate::types::PyTuple; +use crate::{ffi, Borrowed, BoundObject, PyResult, Python}; + +use super::helpers::{ + concat_known_sized, write_iter_to_tuple, write_raw_storage_to_tuple, DropManyGuard, + DropOneGuard, WriteToTuple, +}; +use super::{ + ArgumentsOffsetFlag, ConcatArrays, ConcatStorages, PPPyObject, RawStorage, ResolveArgs, +}; + +pub struct ArrayArgsStorage(pub(in super::super) T); + +impl<'py, T: ResolveArgs<'py>> ResolveArgs<'py> for ArrayArgsStorage +where + T::RawStorage: for<'a> RawStorage = PPPyObject> + 'static, +{ + type RawStorage = T::RawStorage; + type Guard = T::Guard; + #[inline(always)] + fn init( + self, + py: Python<'py>, + storage: PPPyObject, + base_storage: *const PPPyObject, + ) -> PyResult { + self.0.init(py, storage, base_storage) + } + #[inline(always)] + fn len(&self) -> usize { + self.0.len() + } + #[inline(always)] + fn write_to_tuple( + self, + tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + self.0.write_to_tuple(tuple, index) + } + #[inline(always)] + fn write_initialized_to_tuple( + tuple: Borrowed<'_, 'py, PyTuple>, + guard: Self::Guard, + raw_storage: &mut PPPyObject, + index: &mut ffi::Py_ssize_t, + ) { + T::write_initialized_to_tuple(tuple, guard, raw_storage, index) + } + #[inline(always)] + fn has_known_size(&self) -> bool { + true + } + const ARGUMENTS_OFFSET: ArgumentsOffsetFlag = ArgumentsOffsetFlag::Normal; + #[inline(always)] + fn as_pytuple(&self, py: Python<'py>) -> Option> { + self.0.as_pytuple(py) + } + const IS_EMPTY: bool = T::IS_EMPTY; + const IS_ONE: bool = T::IS_ONE; + const USE_STACK_FOR_SMALL_LEN: bool = T::USE_STACK_FOR_SMALL_LEN; +} + +impl<'py, A, B> ResolveArgs<'py> for ArrayArgsStorage> +where + A: ResolveArgs<'py>, + B: ResolveArgs<'py>, + A::RawStorage: for<'a> RawStorage = PPPyObject> + 'static, + B::RawStorage: for<'a> RawStorage = PPPyObject> + 'static, +{ + type RawStorage = MaybeUninit>; + type Guard = (A::Guard, B::Guard); + #[inline(always)] + fn init( + self, + py: Python<'py>, + storage: PPPyObject, + base_storage: *const PPPyObject, + ) -> PyResult { + concat_known_sized(self.0 .0, self.0 .1, py, storage, base_storage) + } + #[inline(always)] + fn len(&self) -> usize { + self.0 .0.len() + self.0 .1.len() + } + #[inline(always)] + fn write_to_tuple( + self, + tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + self.0 .0.write_to_tuple(tuple, index)?; + self.0 .1.write_to_tuple(tuple, index) + } + #[inline(always)] + fn write_initialized_to_tuple( + tuple: Borrowed<'_, 'py, PyTuple>, + guard: Self::Guard, + raw_storage: &mut PPPyObject, + index: &mut ffi::Py_ssize_t, + ) { + A::write_initialized_to_tuple(tuple, guard.0, raw_storage, index); + B::write_initialized_to_tuple(tuple, guard.1, raw_storage, index); + } + #[inline(always)] + fn has_known_size(&self) -> bool { + true + } + const ARGUMENTS_OFFSET: ArgumentsOffsetFlag = ArgumentsOffsetFlag::Normal; + const IS_EMPTY: bool = A::IS_EMPTY && B::IS_EMPTY; + const IS_ONE: bool = (A::IS_EMPTY && B::IS_ONE) || (A::IS_ONE && B::IS_EMPTY); + const USE_STACK_FOR_SMALL_LEN: bool = false; +} + +impl<'py, T, const N: usize> ResolveArgs<'py> for [T; N] +where + T: IntoPyObject<'py>, +{ + type RawStorage = MaybeUninit<[*mut ffi::PyObject; N]>; + type Guard = DropManyGuard; + #[inline(always)] + fn init( + self, + py: Python<'py>, + storage: PPPyObject, + base_storage: *const PPPyObject, + ) -> PyResult { + DropManyGuard::from_iter(py, storage, base_storage, self) + } + #[inline(always)] + fn len(&self) -> usize { + N + } + #[inline(always)] + fn write_to_tuple( + self, + tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + write_iter_to_tuple(tuple, self, index) + } + #[inline(always)] + fn write_initialized_to_tuple( + tuple: Borrowed<'_, 'py, PyTuple>, + guard: Self::Guard, + raw_storage: &mut PPPyObject, + index: &mut ffi::Py_ssize_t, + ) { + write_raw_storage_to_tuple::(tuple, raw_storage, index, N); + std::mem::forget(guard); + } + #[inline(always)] + fn has_known_size(&self) -> bool { + true + } + const ARGUMENTS_OFFSET: ArgumentsOffsetFlag = ArgumentsOffsetFlag::Normal; + const IS_EMPTY: bool = N == 0; + const IS_ONE: bool = N == 1; + const USE_STACK_FOR_SMALL_LEN: bool = false; +} + +impl<'a, 'py, T, const N: usize> ResolveArgs<'py> for &'a [T; N] +where + &'a T: IntoPyObject<'py>, +{ + type RawStorage = MaybeUninit<[*mut ffi::PyObject; N]>; + type Guard = DropManyGuard<<&'a T as IntoPyObject<'py>>::Output>; + #[inline(always)] + fn init( + self, + py: Python<'py>, + storage: PPPyObject, + base_storage: *const PPPyObject, + ) -> PyResult { + DropManyGuard::from_iter(py, storage, base_storage, self) + } + #[inline(always)] + fn len(&self) -> usize { + N + } + #[inline(always)] + fn write_to_tuple( + self, + tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + write_iter_to_tuple(tuple, self, index) + } + #[inline(always)] + fn write_initialized_to_tuple( + tuple: Borrowed<'_, 'py, PyTuple>, + guard: Self::Guard, + raw_storage: &mut PPPyObject, + index: &mut ffi::Py_ssize_t, + ) { + write_raw_storage_to_tuple::<<&'a T as IntoPyObject<'py>>::Output, _>( + tuple, + raw_storage, + index, + N, + ); + std::mem::forget(guard); + } + #[inline(always)] + fn has_known_size(&self) -> bool { + true + } + const ARGUMENTS_OFFSET: ArgumentsOffsetFlag = ArgumentsOffsetFlag::Normal; + const IS_EMPTY: bool = N == 0; + const IS_ONE: bool = N == 1; + const USE_STACK_FOR_SMALL_LEN: bool = false; +} + +impl<'a, 'py, T, const N: usize> ResolveArgs<'py> for &'a mut [T; N] +where + &'a T: IntoPyObject<'py>, +{ + type RawStorage = MaybeUninit<[*mut ffi::PyObject; N]>; + type Guard = DropManyGuard<<&'a T as IntoPyObject<'py>>::Output>; + #[inline(always)] + fn init( + self, + py: Python<'py>, + storage: PPPyObject, + base_storage: *const PPPyObject, + ) -> PyResult { + DropManyGuard::from_iter(py, storage, base_storage, self.iter()) + } + #[inline(always)] + fn len(&self) -> usize { + N + } + #[inline(always)] + fn write_to_tuple( + self, + tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + write_iter_to_tuple(tuple, self.iter(), index) + } + #[inline(always)] + fn write_initialized_to_tuple( + tuple: Borrowed<'_, 'py, PyTuple>, + guard: Self::Guard, + raw_storage: &mut PPPyObject, + index: &mut ffi::Py_ssize_t, + ) { + write_raw_storage_to_tuple::<<&'a T as IntoPyObject<'py>>::Output, _>( + tuple, + raw_storage, + index, + N, + ); + std::mem::forget(guard); + } + #[inline(always)] + fn has_known_size(&self) -> bool { + true + } + const ARGUMENTS_OFFSET: ArgumentsOffsetFlag = ArgumentsOffsetFlag::Normal; + const IS_EMPTY: bool = N == 0; + const IS_ONE: bool = N == 1; + const USE_STACK_FOR_SMALL_LEN: bool = false; +} + +/// A helper trait so that we don't have to repeat the macro for tuples both here and in selection. +pub trait Tuple<'py>: ResolveArgs<'py> {} + +macro_rules! impl_resolve_args_for_tuple { + ( @guard_type $ty:ty, $next:ident, $($rest:ident,)* ) => { + impl_resolve_args_for_tuple!( @guard_type ConcatArrays<$ty, $next::Output>, $($rest,)* ) + }; + ( @guard_type $ty:ty, ) => { + $ty + }; + ( @count $t:ident ) => { 1 }; + ( ) => {}; + ( + $first:ident, $( $rest:ident, )* + ) => { + impl<'py, $first, $( $rest, )*> Tuple<'py> for ( $first, $($rest,)* ) + where + $first: IntoPyObject<'py>, + $( + $rest: IntoPyObject<'py>, + )* + {} + + impl<'py, $first, $( $rest, )*> ResolveArgs<'py> for ( $first, $($rest,)* ) + where + $first: IntoPyObject<'py>, + $( + $rest: IntoPyObject<'py>, + )* + { + type RawStorage = MaybeUninit<[*mut ffi::PyObject; 1 $( + impl_resolve_args_for_tuple!(@count $rest) )*]>; + type Guard = DropOneGuard<'py, impl_resolve_args_for_tuple!( @guard_type $first::Output, $($rest,)* )>; + #[inline(always)] + fn init( + self, + py: Python<'py>, + storage: PPPyObject, + base_storage: *const PPPyObject, + ) -> PyResult { + #[allow(non_snake_case)] + let ( $first, $( $rest, )* ) = self; + Ok( + DropOneGuard::from_write(py, storage, base_storage, $first)? + $( .write($rest)? )* + ) + } + #[inline(always)] + fn len(&self) -> usize { + 1 $( + impl_resolve_args_for_tuple!(@count $rest) )* + } + #[inline(always)] + fn write_to_tuple( + self, + tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + #[allow(non_snake_case)] + let ( $first, $( $rest, )* ) = self; + WriteToTuple::new(tuple, index) + .write($first)? + $( .write($rest)? )* + .finish() + } + #[inline(always)] + fn write_initialized_to_tuple( + tuple: Borrowed<'_, 'py, PyTuple>, + guard: Self::Guard, + raw_storage: &mut PPPyObject, + index: &mut ffi::Py_ssize_t, + ) { + let mut p = *raw_storage; + let mut i = *index; + unsafe { + let value = *p; + if !$first::Output::IS_OWNED { + ffi::Py_INCREF(value); + } + ffi::PyTuple_SET_ITEM(tuple.as_ptr(), i, value); + p = p.add(1); + i += 1; + } + *index = i; + *raw_storage = p; + std::mem::forget(guard); + } + #[inline(always)] + fn has_known_size(&self) -> bool { + true + } + const ARGUMENTS_OFFSET: ArgumentsOffsetFlag = ArgumentsOffsetFlag::Normal; + const IS_EMPTY: bool = false; + const IS_ONE: bool = 0 $( + impl_resolve_args_for_tuple!( @count $rest ) )* == 0; + const USE_STACK_FOR_SMALL_LEN: bool = false; + } + + impl_resolve_args_for_tuple!( $($rest,)* ); + }; +} + +// If you are changing the size of the tuple here, make sure to change `build_args()` in +// pyo3-macros-backend/src/pycall.rs too. +impl_resolve_args_for_tuple!(A, B, C, D, E, F, G, H, I, J, K, L, M,); diff --git a/src/pycall/args/concat.rs b/src/pycall/args/concat.rs new file mode 100644 index 00000000000..ec899b5473b --- /dev/null +++ b/src/pycall/args/concat.rs @@ -0,0 +1,243 @@ +//! We have 6 fundamental storages: `ArrayArgsStorage`, `EmptyArgsStorage`, `ExistingArgListSlice`, +//! `UnsizedArgsStorage`, `AppendEmptyArgForVectorcall`, and `VecArgsStorage`. We need to define all +//! combinations between them, that means 5^2+5=30 impls (`AppendEmptyArgForVectorcall` is special: +//! it only concat when it is LHS, and it cannot concat with itself). Fortunately, macros can help with that. + +use super::array::ArrayArgsStorage; +use super::empty::EmptyArgsStorage; +use super::existing::{ + ExistingArgListSlice, ExistingArgListSliceTrait, ExistingArgListVecStorageAdapter, +}; +use super::unknown_size::{SizedToUnsizedStorage, UnsizedArgsStorage}; +use super::vec::VecArgsStorage; +use super::{AppendEmptyArgForVectorcall, ConcatStorages, ResolveArgs}; + +/// A storage that can be concatenated with other storages. +/// +/// A storage can be a remark how to handle some unpacked argument (e.g. tuples implement `ResolveArgs`), +/// or it can also carry instructions how to create the whole list of arguments (e.g. an `ArrayArgsStorage`). +/// This trait signifies the latter. +pub trait FundamentalStorage<'py>: ResolveArgs<'py> {} +macro_rules! impl_fundamental_storage { + ( $($storage:ident)+ ) => { + $( + impl<'py, T> FundamentalStorage<'py> for $storage where + $storage: ResolveArgs<'py> + { + } + )+ + }; +} +impl_fundamental_storage!(ArrayArgsStorage VecArgsStorage UnsizedArgsStorage ExistingArgListSlice); +impl<'py> FundamentalStorage<'py> for EmptyArgsStorage {} +impl<'py> FundamentalStorage<'py> for AppendEmptyArgForVectorcall {} + +pub trait ConcatArgsStorages<'py, Rhs: FundamentalStorage<'py>>: FundamentalStorage<'py> { + type Output: ResolveArgs<'py>; + fn concat(self, other: Rhs) -> Self::Output; +} + +macro_rules! define_concat { + ( + $( + $storage1:ident + $storage2:ident = $result:ident + )+ + ) => { + $( + impl<'py, A, B> ConcatArgsStorages<'py, $storage2> for $storage1 + where + $storage1: ResolveArgs<'py>, + $storage2: ResolveArgs<'py>, + $result, $storage2>>: ResolveArgs<'py>, + { + type Output = $result, $storage2>>; + #[inline(always)] + fn concat(self, other: $storage2) -> Self::Output { + $result(ConcatStorages(self, other)) + } + } + )+ + }; +} +define_concat!( + ArrayArgsStorage + ArrayArgsStorage = ArrayArgsStorage + ArrayArgsStorage + VecArgsStorage = VecArgsStorage + VecArgsStorage + ArrayArgsStorage = VecArgsStorage + VecArgsStorage + VecArgsStorage = VecArgsStorage +); + +macro_rules! define_concat_empty { + ( $( $other:ident )+ ) => { + $( + impl<'py, T> ConcatArgsStorages<'py, $other> for EmptyArgsStorage + where + $other: ResolveArgs<'py>, + { + type Output = $other; + #[inline(always)] + fn concat(self, other: $other) -> Self::Output { + other + } + } + impl<'py, T> ConcatArgsStorages<'py, EmptyArgsStorage> for $other + where + $other: ResolveArgs<'py>, + { + type Output = $other; + #[inline(always)] + fn concat(self, _other: EmptyArgsStorage) -> Self::Output { + self + } + } + )+ + }; +} +define_concat_empty!( + ArrayArgsStorage VecArgsStorage UnsizedArgsStorage ExistingArgListSlice +); +impl<'py> ConcatArgsStorages<'py, EmptyArgsStorage> for EmptyArgsStorage { + #[inline(always)] + fn concat(self, _other: EmptyArgsStorage) -> Self::Output { + EmptyArgsStorage + } + type Output = EmptyArgsStorage; +} + +macro_rules! define_concat_existing { + ( $( $other:ident )+ ) => { + $( + impl<'py, S, T> ConcatArgsStorages<'py, $other> for ExistingArgListSlice + where + ExistingArgListSlice: ResolveArgs<'py>, + $other: ResolveArgs<'py>, + VecArgsStorage>: ConcatArgsStorages<'py, $other>, + { + type Output = > as ConcatArgsStorages<'py, $other>>::Output; + #[inline(always)] + fn concat(self, other: $other) -> Self::Output { + > as ConcatArgsStorages<'py, $other>>::concat( + VecArgsStorage(ExistingArgListVecStorageAdapter(self)), other) + } + } + impl<'py, S, T> ConcatArgsStorages<'py, ExistingArgListSlice> for $other + where + ExistingArgListSlice: ResolveArgs<'py>, + VecArgsStorage>: ResolveArgs<'py>, + $other: ResolveArgs<'py>, + $other: ConcatArgsStorages<'py, VecArgsStorage>>, + { + type Output = <$other as ConcatArgsStorages<'py, VecArgsStorage>>>::Output; + #[inline(always)] + fn concat(self, other: ExistingArgListSlice) -> Self::Output { + <$other as ConcatArgsStorages<'py, VecArgsStorage>>>::concat( + self, VecArgsStorage(ExistingArgListVecStorageAdapter(other))) + } + } + )+ + }; +} +define_concat_existing!(ArrayArgsStorage VecArgsStorage UnsizedArgsStorage); +impl<'py, A, B> ConcatArgsStorages<'py, ExistingArgListSlice> for ExistingArgListSlice +where + ExistingArgListSlice: ResolveArgs<'py>, + ExistingArgListSlice: ResolveArgs<'py>, + VecArgsStorage< + ConcatStorages< + VecArgsStorage>, + VecArgsStorage>, + >, + >: ResolveArgs<'py>, +{ + type Output = VecArgsStorage< + ConcatStorages< + VecArgsStorage>, + VecArgsStorage>, + >, + >; + #[inline(always)] + fn concat(self, other: ExistingArgListSlice) -> Self::Output { + VecArgsStorage(ConcatStorages( + VecArgsStorage(ExistingArgListVecStorageAdapter(self)), + VecArgsStorage(ExistingArgListVecStorageAdapter(other)), + )) + } +} + +macro_rules! define_concat_sized_to_unsized { + ( + $( $other:ident )+ + ) => { + $( + impl<'py, T, U> ConcatArgsStorages<'py, $other> for UnsizedArgsStorage + where + UnsizedArgsStorage: ResolveArgs<'py>, + $other: ResolveArgs<'py>, + UnsizedArgsStorage, UnsizedArgsStorage>>>>: ResolveArgs<'py>, + { + type Output = UnsizedArgsStorage, UnsizedArgsStorage>>>>; + #[inline(always)] + fn concat(self, other: $other) -> Self::Output { + UnsizedArgsStorage(ConcatStorages(self, UnsizedArgsStorage(SizedToUnsizedStorage(other)))) + } + } + impl<'py, T, U> ConcatArgsStorages<'py, UnsizedArgsStorage> for $other + where + UnsizedArgsStorage: ResolveArgs<'py>, + $other: ResolveArgs<'py>, + UnsizedArgsStorage>>, UnsizedArgsStorage>>: ResolveArgs<'py>, + { + type Output = UnsizedArgsStorage>>, UnsizedArgsStorage>>; + #[inline(always)] + fn concat(self, other: UnsizedArgsStorage) -> Self::Output { + UnsizedArgsStorage(ConcatStorages(UnsizedArgsStorage(SizedToUnsizedStorage(self)), other)) + } + } + )+ + }; +} +define_concat_sized_to_unsized!(ArrayArgsStorage VecArgsStorage); +impl<'py, A, B> ConcatArgsStorages<'py, UnsizedArgsStorage> for UnsizedArgsStorage +where + UnsizedArgsStorage: ResolveArgs<'py>, + UnsizedArgsStorage: ResolveArgs<'py>, + UnsizedArgsStorage, UnsizedArgsStorage>>: + ResolveArgs<'py>, +{ + type Output = UnsizedArgsStorage, UnsizedArgsStorage>>; + #[inline(always)] + fn concat(self, other: UnsizedArgsStorage) -> Self::Output { + UnsizedArgsStorage(ConcatStorages(self, other)) + } +} + +macro_rules! define_concat_append_empty_arg_for_vectorcall { + ( $( ( $($generic:ident)? ) $other:ident )+ ) => { + $( + impl<'py, $($generic)?> ConcatArgsStorages<'py, $other<$($generic)?>> for AppendEmptyArgForVectorcall + where + $other<$($generic)?>: ResolveArgs<'py>, + ArrayArgsStorage: ConcatArgsStorages<'py, $other<$($generic)?>>, + { + type Output = as ConcatArgsStorages<'py, $other<$($generic)?>>>::Output; + #[inline(always)] + fn concat(self, other: $other<$($generic)?>) -> Self::Output { + as ConcatArgsStorages<'py, $other<$($generic)?>>>::concat(ArrayArgsStorage(self), other) + } + } + )+ + }; +} +define_concat_append_empty_arg_for_vectorcall!( + (T) ArrayArgsStorage (T) VecArgsStorage (T) UnsizedArgsStorage + () EmptyArgsStorage +); +impl<'py, S> ConcatArgsStorages<'py, ExistingArgListSlice> for AppendEmptyArgForVectorcall +where + S: ExistingArgListSliceTrait, +{ + type Output = ExistingArgListSlice; + #[inline(always)] + fn concat(self, other: ExistingArgListSlice) -> Self::Output { + other + } +} diff --git a/src/pycall/args/empty.rs b/src/pycall/args/empty.rs new file mode 100644 index 00000000000..3ca845f3789 --- /dev/null +++ b/src/pycall/args/empty.rs @@ -0,0 +1,51 @@ +use std::mem::MaybeUninit; + +use crate::pycall::PPPyObject; +use crate::types::PyTuple; +use crate::{ffi, Borrowed, PyResult, Python}; + +use super::{ArgumentsOffsetFlag, ResolveArgs}; + +pub struct EmptyArgsStorage; + +impl<'py> ResolveArgs<'py> for EmptyArgsStorage { + type RawStorage = MaybeUninit<()>; + type Guard = (); + #[inline(always)] + fn init( + self, + _py: Python<'py>, + _storage: PPPyObject, + _base_storage: *const PPPyObject, + ) -> PyResult<()> { + Ok(()) + } + #[inline(always)] + fn len(&self) -> usize { + 0 + } + #[inline(always)] + fn write_to_tuple( + self, + _tuple: Borrowed<'_, 'py, PyTuple>, + _index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + Ok(()) + } + #[inline(always)] + fn write_initialized_to_tuple( + _tuple: Borrowed<'_, 'py, PyTuple>, + _guard: Self::Guard, + _raw_storage: &mut PPPyObject, + _index: &mut ffi::Py_ssize_t, + ) { + } + #[inline(always)] + fn has_known_size(&self) -> bool { + true + } + const ARGUMENTS_OFFSET: ArgumentsOffsetFlag = ArgumentsOffsetFlag::Normal; + const IS_EMPTY: bool = true; + const IS_ONE: bool = false; + const USE_STACK_FOR_SMALL_LEN: bool = false; +} diff --git a/src/pycall/args/existing.rs b/src/pycall/args/existing.rs new file mode 100644 index 00000000000..50136f8ef9e --- /dev/null +++ b/src/pycall/args/existing.rs @@ -0,0 +1,252 @@ +use std::mem::{ManuallyDrop, MaybeUninit}; + +use super::helpers::write_raw_storage_to_tuple; +use super::{ArgumentsOffsetFlag, ResolveArgs}; +use crate::pycall::storage::{DynKnownSizeRawStorage, RawStorage}; +use crate::pycall::PPPyObject; +use crate::types::PyTuple; +use crate::{ffi, Borrowed, Bound, Py, PyAny, PyResult, Python}; + +pub struct ExistingArgListSlice(pub(super) S); + +pub trait ExistingArgListSliceArg { + const IS_OWNED: bool; +} +impl ExistingArgListSliceArg for Py { + const IS_OWNED: bool = true; +} +impl ExistingArgListSliceArg for Bound<'_, T> { + const IS_OWNED: bool = true; +} +impl ExistingArgListSliceArg for Borrowed<'_, '_, T> { + const IS_OWNED: bool = false; +} + +pub trait ExistingArgListSliceTrait: Sized { + fn as_ptr(&mut self) -> PPPyObject; + fn len(&self) -> usize; + /// Deallocate the memory but do not drop the objects inside. + #[inline(always)] + fn dealloc(&mut self) {} + const IS_OWNED: bool; + const CAN_MUTATE: bool; +} +impl ExistingArgListSliceTrait for Vec { + #[inline(always)] + fn as_ptr(&mut self) -> PPPyObject { + self.as_mut_ptr().cast::<*mut ffi::PyObject>() + } + #[inline(always)] + fn len(&self) -> usize { + self.len() + } + #[inline(always)] + fn dealloc(&mut self) { + unsafe { + Vec::>::from_raw_parts( + self.as_mut_ptr().cast::>(), + self.len(), + self.capacity(), + ); + } + } + const IS_OWNED: bool = T::IS_OWNED; + const CAN_MUTATE: bool = true; +} +impl ExistingArgListSliceTrait for &'_ Vec { + #[inline(always)] + fn as_ptr(&mut self) -> PPPyObject { + (**self).as_ptr().cast::<*mut ffi::PyObject>().cast_mut() + } + #[inline(always)] + fn len(&self) -> usize { + (**self).len() + } + const IS_OWNED: bool = false; + const CAN_MUTATE: bool = false; +} +impl ExistingArgListSliceTrait for &'_ mut Vec { + #[inline(always)] + fn as_ptr(&mut self) -> PPPyObject { + (**self).as_mut_ptr().cast::<*mut ffi::PyObject>() + } + #[inline(always)] + fn len(&self) -> usize { + (**self).len() + } + const IS_OWNED: bool = false; + const CAN_MUTATE: bool = true; +} +impl ExistingArgListSliceTrait for &'_ [T] { + #[inline(always)] + fn as_ptr(&mut self) -> PPPyObject { + (**self).as_ptr().cast::<*mut ffi::PyObject>().cast_mut() + } + #[inline(always)] + fn len(&self) -> usize { + (**self).len() + } + const IS_OWNED: bool = false; + const CAN_MUTATE: bool = false; +} +impl ExistingArgListSliceTrait for &'_ mut [T] { + #[inline(always)] + fn as_ptr(&mut self) -> PPPyObject { + (**self).as_mut_ptr().cast::<*mut ffi::PyObject>() + } + #[inline(always)] + fn len(&self) -> usize { + (**self).len() + } + const IS_OWNED: bool = false; + const CAN_MUTATE: bool = true; +} + +pub struct ExistingArgListSliceStorage(MaybeUninit); +impl RawStorage for ExistingArgListSliceStorage { + type InitParam<'a> = &'a mut Self + where + Self: 'a; + #[inline(always)] + fn new(_len: usize) -> Self { + Self(MaybeUninit::uninit()) + } + #[inline(always)] + fn as_init_param(&mut self) -> Self::InitParam<'_> { + self + } + #[inline(always)] + fn as_ptr(&mut self) -> PPPyObject { + unsafe { self.0.assume_init_mut().as_ptr() } + } + #[inline(always)] + fn len(&self) -> usize { + unsafe { self.0.assume_init_ref().len() } + } + #[inline(always)] + fn init_param_from_ptr<'a>(_ptr: PPPyObject) -> Self::InitParam<'a> { + unreachable!("ExistingArgListSliceStorage does not use small stack optimization") + } +} +impl Drop for ExistingArgListSliceStorage { + #[inline(always)] + fn drop(&mut self) { + unsafe { + self.0.assume_init_drop(); + } + } +} +impl<'py, S: ExistingArgListSliceTrait> ResolveArgs<'py> for ExistingArgListSlice { + type RawStorage = ExistingArgListSliceStorage; + type Guard = (); + #[inline(always)] + fn init( + self, + _py: Python<'py>, + storage: &mut ExistingArgListSliceStorage, + _base_storage: *const PPPyObject, + ) -> PyResult { + storage.0 = MaybeUninit::new(self.0); + Ok(()) + } + #[inline(always)] + fn len(&self) -> usize { + self.0.len() + } + #[inline(always)] + fn write_to_tuple( + self, + tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + let mut this = ManuallyDrop::new(self); + unsafe { + let mut p = this.0.as_ptr(); + for i in *index..*index + this.len() as ffi::Py_ssize_t { + let v = *p; + if !S::IS_OWNED { + ffi::Py_INCREF(v); + } + ffi::PyTuple_SET_ITEM(tuple.as_ptr(), i, v); + p = p.add(1); + } + } + *index += this.len() as isize; + this.0.dealloc(); + Ok(()) + } + #[inline(always)] + fn write_initialized_to_tuple( + _tuple: Borrowed<'_, 'py, PyTuple>, + _guard: Self::Guard, + _raw_storage: &mut PPPyObject, + _index: &mut ffi::Py_ssize_t, + ) { + unreachable!( + "ExistingArgListSlice::write_initialized_to_tuple() should never be called: \ + `write_to_tuple()` will be called if the storage is alone, and if it is concatenated, \ + it will be replaced by ExistingArgListVecStorageAdapter" + ) + } + #[inline(always)] + fn has_known_size(&self) -> bool { + true + } + const ARGUMENTS_OFFSET: ArgumentsOffsetFlag = if S::CAN_MUTATE { + ArgumentsOffsetFlag::DoNotOffsetButCanChangeArgs0 + } else { + ArgumentsOffsetFlag::DoNotOffset + }; + const IS_EMPTY: bool = false; + const IS_ONE: bool = false; + const USE_STACK_FOR_SMALL_LEN: bool = false; +} + +pub struct ExistingArgListVecStorageAdapter(pub(super) ExistingArgListSlice); +impl<'py, S: ExistingArgListSliceTrait> ResolveArgs<'py> for ExistingArgListVecStorageAdapter { + type RawStorage = DynKnownSizeRawStorage; + type Guard = S; + #[inline(always)] + fn init( + mut self, + _py: Python<'py>, + storage: PPPyObject, + _base_storage: *const PPPyObject, + ) -> PyResult { + unsafe { + std::ptr::copy_nonoverlapping(self.0 .0.as_ptr(), storage, self.0.len()); + } + Ok(self.0 .0) + } + #[inline(always)] + fn len(&self) -> usize { + self.0.len() + } + #[inline(always)] + fn write_to_tuple( + self, + tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + self.0.write_to_tuple(tuple, index) + } + #[inline(always)] + fn write_initialized_to_tuple( + tuple: Borrowed<'_, 'py, PyTuple>, + guard: Self::Guard, + raw_storage: &mut PPPyObject, + index: &mut ffi::Py_ssize_t, + ) { + let mut guard = ManuallyDrop::new(guard); + write_raw_storage_to_tuple::, _>(tuple, raw_storage, index, guard.len()); + guard.dealloc(); + } + #[inline(always)] + fn has_known_size(&self) -> bool { + true + } + const ARGUMENTS_OFFSET: ArgumentsOffsetFlag = ArgumentsOffsetFlag::Normal; + const IS_EMPTY: bool = false; + const IS_ONE: bool = false; + const USE_STACK_FOR_SMALL_LEN: bool = true; +} diff --git a/src/pycall/args/helpers.rs b/src/pycall/args/helpers.rs new file mode 100644 index 00000000000..08ecd67fe8e --- /dev/null +++ b/src/pycall/args/helpers.rs @@ -0,0 +1,282 @@ +use std::marker::PhantomData; +use std::ops::Range; + +use crate::conversion::IntoPyObject; +use crate::pycall::storage::RawStorage; +use crate::pycall::PPPyObject; +use crate::types::PyTuple; +use crate::{ffi, Borrowed, BoundObject, PyErr, PyResult, Python}; + +use super::{ConcatArrays, ResolveArgs}; + +pub struct DropOneGuard<'py, DropTy> { + base_storage: *const PPPyObject, + base_offset: isize, + py: Python<'py>, + _marker: PhantomData, +} +impl<'py, DropTy> DropOneGuard<'py, DropTy> { + #[inline(always)] + pub(super) fn from_write( + py: Python<'py>, + write_at: PPPyObject, + base_storage: *const PPPyObject, + value: T, + ) -> PyResult + where + T: IntoPyObject<'py, Output = DropTy, Error = E>, + E: Into, + { + const { + assert!( + size_of::<*mut ffi::PyObject>() == size_of::() + && align_of::<*mut ffi::PyObject>() == align_of::(), + ) + } + unsafe { + let base = base_storage.read(); + let base_idx = write_at.offset_from(base); + let value = value.into_pyobject(py).map_err(Into::into)?; + base.offset(base_idx).cast::().write(value); + Ok(Self { + base_storage, + base_offset: base_idx, + py, + _marker: PhantomData, + }) + } + } + #[inline(always)] + pub(super) fn write( + self, + value: T, + ) -> PyResult>> + where + T: IntoPyObject<'py, Output = NextDropTy, Error = E>, + E: Into, + { + const { + assert!( + size_of::<*mut ffi::PyObject>() == size_of::() + && align_of::<*mut ffi::PyObject>() == align_of::(), + ) + } + unsafe { + let value = value.into_pyobject(self.py).map_err(Into::into)?; + self.base_storage + .read() + .offset(self.base_offset) + .cast::() + .add(1) + .cast::() + .write(value); + Ok(DropOneGuard { + base_storage: self.base_storage, + base_offset: self.base_offset, + py: self.py, + _marker: PhantomData, + }) + } + } +} +impl Drop for DropOneGuard<'_, DropTy> { + #[inline(always)] + fn drop(&mut self) { + unsafe { + self.base_storage + .read() + .offset(self.base_offset) + .cast::() + .drop_in_place(); + } + } +} + +pub struct DropManyGuard { + base_storage: *const PPPyObject, + base_range: Range, + _marker: PhantomData, +} +impl DropManyGuard { + #[inline(always)] + pub(super) fn new(base_storage: *const PPPyObject, from: PPPyObject, len: usize) -> Self { + let from = unsafe { from.offset_from(base_storage.read()) }; + Self { + base_storage, + base_range: from..len as isize + from, + _marker: PhantomData, + } + } + #[inline(always)] + pub(super) fn from_iter<'py, T, E>( + py: Python<'py>, + write_at: PPPyObject, + base_storage: *const PPPyObject, + iter: impl IntoIterator, + ) -> PyResult + where + T: IntoPyObject<'py, Output = DropTy, Error = E>, + E: Into, + { + const { + assert!( + size_of::<*mut ffi::PyObject>() == size_of::() + && align_of::<*mut ffi::PyObject>() == align_of::(), + ) + } + + unsafe { + let base_offset = write_at.offset_from(base_storage.read()); + let mut guard = Self { + base_storage, + base_range: base_offset..base_offset, + _marker: PhantomData, + }; + iter.into_iter() + .try_for_each(|item| match item.into_pyobject(py) { + Ok(item) => { + let base = guard.base_storage.read(); + base.offset(guard.base_range.end) + .cast::() + .write(item); + guard.base_range.end += 1; + Ok(()) + } + Err(err) => Err(err.into()), + })?; + Ok(guard) + } + } + #[inline(always)] + pub(super) fn len(&self) -> usize { + self.base_range.end as usize - self.base_range.start as usize + } +} +impl Drop for DropManyGuard { + #[inline(always)] + fn drop(&mut self) { + unsafe { + std::ptr::slice_from_raw_parts_mut( + self.base_storage + .read() + .offset(self.base_range.start) + .cast::(), + (self.base_range.end - self.base_range.start) as usize, + ) + .drop_in_place(); + } + } +} + +pub(super) struct WriteToTuple<'a, 'py> { + tuple: *mut ffi::PyObject, + index: &'a mut ffi::Py_ssize_t, + py: Python<'py>, +} +impl<'a, 'py> WriteToTuple<'a, 'py> { + #[inline(always)] + pub(super) fn new(tuple: Borrowed<'_, 'py, PyTuple>, index: &'a mut ffi::Py_ssize_t) -> Self { + Self { + tuple: tuple.as_ptr(), + index, + py: tuple.py(), + } + } + + #[inline(always)] + pub(super) fn write(self, value: T) -> PyResult + where + T: IntoPyObject<'py, Error = E>, + E: Into, + { + unsafe { + ffi::PyTuple_SET_ITEM( + self.tuple, + *self.index, + value + .into_pyobject(self.py) + .map_err(Into::into)? + .into_bound() + .into_ptr(), + ); + } + *self.index += 1; + Ok(self) + } + + #[inline(always)] + pub(super) fn finish(self) -> PyResult<()> { + Ok(()) + } +} + +#[inline(always)] +pub(super) fn write_iter_to_tuple<'py, T, E>( + tuple: Borrowed<'_, 'py, PyTuple>, + iter: impl IntoIterator, + index: &mut ffi::Py_ssize_t, +) -> PyResult<()> +where + T: IntoPyObject<'py, Error = E>, + E: Into, +{ + iter.into_iter() + .try_for_each(|item| match item.into_pyobject(tuple.py()) { + Ok(item) => { + let item = item.into_bound().into_ptr(); + unsafe { + ffi::PyTuple_SET_ITEM(tuple.as_ptr(), *index, item); + } + *index += 1; + Ok(()) + } + Err(err) => Err(err.into()), + }) +} + +#[inline(always)] +pub(super) fn write_raw_storage_to_tuple<'py, T, U>( + tuple: Borrowed<'_, 'py, PyTuple>, + raw_storage: &mut PPPyObject, + index: &mut ffi::Py_ssize_t, + len: usize, +) where + T: BoundObject<'py, U>, +{ + let end_index = *index + len as ffi::Py_ssize_t; + let mut p = *raw_storage; + for i in *index..end_index { + unsafe { + let value = *p; + if !T::IS_OWNED { + ffi::Py_INCREF(value); + } + ffi::PyTuple_SET_ITEM(tuple.as_ptr(), i, value); + p = p.add(1); + } + } + *raw_storage = p; + *index = end_index; +} + +#[inline(always)] +pub(super) fn concat_known_sized<'py, A, B>( + a: A, + b: B, + py: Python<'py>, + mut storage: PPPyObject, + base_storage: *const PPPyObject, +) -> PyResult<(A::Guard, B::Guard)> +where + A: ResolveArgs<'py>, + B: ResolveArgs<'py>, + A::RawStorage: for<'a> RawStorage = PPPyObject>, + B::RawStorage: for<'a> RawStorage = PPPyObject>, +{ + let len1 = a.len(); + let index = unsafe { storage.offset_from(base_storage.read()) as usize }; + let g1 = a.init(py, storage, base_storage)?; + storage = unsafe { base_storage.read().add(index + len1) }; + let g2 = b.init(py, storage, base_storage)?; + Ok((g1, g2)) +} diff --git a/src/pycall/args/pyobjects.rs b/src/pycall/args/pyobjects.rs new file mode 100644 index 00000000000..e80d485479f --- /dev/null +++ b/src/pycall/args/pyobjects.rs @@ -0,0 +1,331 @@ +use crate::pycall::as_pyobject::AsPyObject; +use crate::pycall::storage::{UnsizedInitParam, UnsizedStorage}; +use crate::pycall::PPPyObject; +use crate::types::{ + PyAnyMethods, PyByteArray, PyBytes, PyDict, PyDictItems, PyDictKeys, PyDictValues, PyFrozenSet, + PyList, PySet, PyTuple, +}; +use crate::{ffi, Borrowed, Bound, BoundObject, PyAny, PyResult, PyTypeInfo, Python}; + +use super::helpers::{write_iter_to_tuple, write_raw_storage_to_tuple, DropManyGuard}; +use super::unknown_size::UnsizedGuard; +use super::{ArgumentsOffsetFlag, ResolveArgs}; + +pub struct PyTupleArgs { + value: T, + is_not_tuple_subclass: bool, + len: usize, +} + +impl<'py, T: AsPyObject<'py, PyObject = PyTuple>> PyTupleArgs { + #[inline(always)] + pub fn new(value: T) -> Self { + let value_borrowed = value.as_borrowed(unsafe { Python::assume_gil_acquired() }); + let is_not_tuple_subclass = value_borrowed.is_exact_instance_of::(); + let len = value_borrowed.len().unwrap_or(0); + Self { + value, + is_not_tuple_subclass, + len, + } + } +} + +impl<'py, T: AsPyObject<'py, PyObject = PyTuple>> ResolveArgs<'py> for PyTupleArgs { + type RawStorage = UnsizedStorage; + type Guard = (Self, UnsizedGuard>); + #[inline(always)] + fn init( + self, + py: Python<'py>, + storage: UnsizedInitParam<'_>, + base_storage: *const PPPyObject, + ) -> PyResult { + if self.is_not_tuple_subclass { + storage.reserve(self.len); + unsafe { + // The buffer might've been invalidated. + base_storage.cast_mut().write(storage.as_mut_ptr()); + } + DropManyGuard::from_iter( + py, + unsafe { storage.as_mut_ptr().add(storage.len()) }, + base_storage, + self.value.as_borrowed(py).iter_borrowed(), + )?; + Ok((self, UnsizedGuard::empty(base_storage))) + } else { + let guard = UnsizedGuard::from_iter( + storage, + base_storage, + self.len, + self.value.as_borrowed(py).as_any().iter()?, + )?; + Ok((self, guard)) + } + } + #[inline(always)] + fn len(&self) -> usize { + self.len + } + #[inline(always)] + fn write_to_tuple( + self, + tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + debug_assert!( + self.is_not_tuple_subclass, + "cannot write an unsized type directly into tuple", + ); + write_iter_to_tuple( + tuple, + self.value.as_borrowed(tuple.py()).iter_borrowed(), + index, + ) + } + #[inline(always)] + fn write_initialized_to_tuple( + tuple: Borrowed<'_, 'py, PyTuple>, + guard: Self::Guard, + raw_storage: &mut PPPyObject, + index: &mut ffi::Py_ssize_t, + ) { + if guard.0.is_not_tuple_subclass { + write_raw_storage_to_tuple::, _>( + tuple, + raw_storage, + index, + guard.0.len, + ); + } else { + write_raw_storage_to_tuple::, _>( + tuple, + raw_storage, + index, + guard.0.len, + ); + } + std::mem::forget(guard.1); + } + #[inline(always)] + fn as_pytuple(&self, py: Python<'py>) -> Option> { + if self.is_not_tuple_subclass { + Some(self.value.as_borrowed(py)) + } else { + None + } + } + #[inline(always)] + fn has_known_size(&self) -> bool { + self.is_not_tuple_subclass + } + const ARGUMENTS_OFFSET: ArgumentsOffsetFlag = ArgumentsOffsetFlag::Normal; + const IS_EMPTY: bool = false; + const IS_ONE: bool = false; + const USE_STACK_FOR_SMALL_LEN: bool = true; +} + +pub struct PyBuiltinIterableArgs { + value: T, + is_not_builtin_subclass: bool, + len: usize, +} + +pub trait IterableBuiltin: PyTypeInfo {} +impl IterableBuiltin for PyByteArray {} +impl IterableBuiltin for PyBytes {} +impl IterableBuiltin for PyDict {} +impl IterableBuiltin for PyDictKeys {} +impl IterableBuiltin for PyDictValues {} +impl IterableBuiltin for PyDictItems {} +impl IterableBuiltin for PySet {} +impl IterableBuiltin for PyFrozenSet {} +impl IterableBuiltin for PyList {} + +impl<'py, T> PyBuiltinIterableArgs +where + T: AsPyObject<'py>, + T::PyObject: IterableBuiltin, +{ + #[inline(always)] + pub fn new(value: T) -> Self { + let value_borrowed = value + .as_borrowed(unsafe { Python::assume_gil_acquired() }) + .into_any(); + let is_not_builtin_subclass = value_borrowed.is_exact_instance_of::(); + let len = value_borrowed.len().unwrap_or(0); + Self { + value, + is_not_builtin_subclass, + len, + } + } +} + +impl<'py, Target, T> ResolveArgs<'py> for PyBuiltinIterableArgs +where + T: AsPyObject<'py, PyObject = Target>, + Target: IterableBuiltin, +{ + type RawStorage = UnsizedStorage; + type Guard = (T, UnsizedGuard>); + #[inline(always)] + fn init( + self, + py: Python<'py>, + storage: UnsizedInitParam<'_>, + base_storage: *const PPPyObject, + ) -> PyResult { + let len = self.len; + if self.is_not_builtin_subclass { + storage.reserve(len); + unsafe { + // The buffer might've been invalidated. + base_storage.cast_mut().write(storage.as_mut_ptr()); + } + let start = unsafe { storage.as_mut_ptr().add(storage.len()) }; + DropManyGuard::from_iter( + py, + start, + base_storage, + self.value.as_borrowed(py).as_any().iter(), + )?; + Ok(( + self.value, + UnsizedGuard::from_range(base_storage, start, len), + )) + } else { + let guard = UnsizedGuard::from_iter( + storage, + base_storage, + len, + self.value.as_borrowed(py).as_any().iter()?, + )?; + Ok((self.value, guard)) + } + } + #[inline(always)] + fn len(&self) -> usize { + self.len + } + #[inline(always)] + fn write_to_tuple( + self, + tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + debug_assert!( + self.is_not_builtin_subclass, + "cannot write an unsized type directly into tuple", + ); + write_iter_to_tuple( + tuple, + self.value.as_borrowed(tuple.py()).as_any().iter(), + index, + ) + } + #[inline(always)] + fn write_initialized_to_tuple( + tuple: Borrowed<'_, 'py, PyTuple>, + guard: Self::Guard, + raw_storage: &mut PPPyObject, + index: &mut ffi::Py_ssize_t, + ) { + write_raw_storage_to_tuple::, _>(tuple, raw_storage, index, guard.1.len()); + std::mem::forget(guard.1); + } + #[inline(always)] + fn as_pytuple(&self, _py: Python<'py>) -> Option> { + None + } + #[inline(always)] + fn has_known_size(&self) -> bool { + self.is_not_builtin_subclass + } + const ARGUMENTS_OFFSET: ArgumentsOffsetFlag = ArgumentsOffsetFlag::Normal; + const IS_EMPTY: bool = false; + const IS_ONE: bool = false; + const USE_STACK_FOR_SMALL_LEN: bool = true; +} + +pub struct AnyPyIterable { + value: T, + len: usize, +} + +impl<'py, T: AsPyObject<'py>> AnyPyIterable { + #[inline(always)] + pub fn new(value: T) -> Self { + let value_borrowed = value + .as_borrowed(unsafe { Python::assume_gil_acquired() }) + .into_any(); + let len = value_borrowed.len().unwrap_or(0); + Self { value, len } + } +} + +impl<'py, T: AsPyObject<'py>> ResolveArgs<'py> for AnyPyIterable { + type RawStorage = UnsizedStorage; + type Guard = (T, UnsizedGuard>); + #[inline(always)] + fn init( + self, + py: Python<'py>, + storage: UnsizedInitParam<'_>, + base_storage: *const PPPyObject, + ) -> PyResult { + let guard = UnsizedGuard::from_iter( + storage, + base_storage, + self.len, + self.value.as_borrowed(py).as_any().iter()?, + )?; + Ok((self.value, guard)) + } + #[inline(always)] + fn len(&self) -> usize { + self.len + } + #[inline(always)] + fn write_to_tuple( + self, + _tuple: Borrowed<'_, 'py, PyTuple>, + _index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + panic!("cannot write an unsized type directly into tuple") + } + #[inline(always)] + fn write_initialized_to_tuple( + tuple: Borrowed<'_, 'py, PyTuple>, + guard: Self::Guard, + raw_storage: &mut PPPyObject, + index: &mut ffi::Py_ssize_t, + ) { + write_raw_storage_to_tuple::, _>(tuple, raw_storage, index, guard.1.len()); + std::mem::forget(guard.1); + } + #[inline(always)] + fn as_pytuple(&self, py: Python<'py>) -> Option> { + let value = self.value.as_borrowed(py).into_any(); + if value.is_exact_instance_of::() { + // FIXME: There is no downcast method for `Borrowed`. + unsafe { + Some(std::mem::transmute::< + Borrowed<'_, 'py, PyAny>, + Borrowed<'_, 'py, PyTuple>, + >(value)) + } + } else { + None + } + } + #[inline(always)] + fn has_known_size(&self) -> bool { + false + } + const ARGUMENTS_OFFSET: ArgumentsOffsetFlag = ArgumentsOffsetFlag::Normal; + const IS_EMPTY: bool = false; + const IS_ONE: bool = false; + const USE_STACK_FOR_SMALL_LEN: bool = false; +} diff --git a/src/pycall/args/selector.rs b/src/pycall/args/selector.rs new file mode 100644 index 00000000000..dc50027c6ba --- /dev/null +++ b/src/pycall/args/selector.rs @@ -0,0 +1,444 @@ +//! This module is responsible, given an unpacked argument, to find the best storage for it. +//! We do that using [autoderef specialization]. We rank each storage implementation, +//! and use the first that can be used. +//! +//! Here are all implementations, ordered by their rank, from the best to the worst: +//! +//! 1. Empty argument set (`()`, `[T; 0]`, `&[T; 0]` and `&mut [T; 0]`). +//! 2. Arrays (`[T; N]`, `&[T; N]` and `&mut [T; N]`). +//! 3. Existing argument slice. +//! 4. `TrustedLen` iterators. +//! 5. `ExactSizeIterator`s. +//! 6. Any `IntoIterator`. +//! 7. `PyTuple`. +//! 8. Python builtin object (sets, lists, etc.). +//! 9. Any Python object. +//! 10. Tuples. +//! +//! They are divided to four groups: +//! +//! - The empty argument is before everything, since it is cancelled by everything, +//! and so its effect on performance is zero. That means it is a better match than anything else. +//! It also enables calling to `PyObject_CallNoArgs()`. +//! - Stack allocated arrays come next. That includes arrays and tuples. They are preferred to +//! existing argument slices because they can be added with `PY_VECTORCALL_ARGUMENTS_OFFSET`. +//! - Existing argument slices come next, since they are better than anything that has to allocate +//! memory. +//! - `TrustedLen`, `ExactSizeIterator` and any iterator have to come in this order specifically +//! since each latter one is a superset of the former, but has a less efficient implementation. +//! - Likewise for `PyTuple`, Python builtin objects and any Python object. `PyTuple` can be used +//! as-is for calls, Python builtins are essentially `TrustedLen` iterators, and other Python iterables +//! are equivalent to any Rust iterator. +//! - Tuples come last not because they are less efficient (in fact they are equivalent to arrays), +//! but because it is more convenient to put them in a "catch-all" bound instead of having to +//! enumerate each tuple type using a macro again. It doesn't matter for performance since +//! nothing else can match tuples. +//! +//! [autoderef specialization]: https://lukaskalbertodt.github.io/2019/12/05/generalized-autoref-based-specialization.html + +use std::marker::PhantomData; + +pub mod select_traits { + pub use super::any_iterator::AnyIteratorSelector as _; + pub use super::any_pyiterable::AnyPyIterableSelector as _; + pub use super::array::ArraySelector as _; + pub use super::empty::EmptySelector as _; + pub use super::exact_size::ExactSizeIteratorSelector as _; + pub use super::existing::ExistingSelector as _; + pub use super::python_builtins::PythonBuiltinsSelector as _; + pub use super::pytuple::PyTupleSelector as _; + pub use super::trusted_len::TrustedLenSelector as _; + pub use super::tuple::TupleSelector as _; +} + +pub struct ArgsStorageSelector(PhantomData); + +impl ArgsStorageSelector { + /// This is called by the macro like the following: + /// + /// ```ignore + /// ArgsStorageSelector::new(loop { + /// break None; + /// break Some(value); + /// }) + /// ``` + /// + /// This way, the compiler infers the correct type, but no code is actually executed. + /// + /// Note that `if false` cannot be used instead, as it can cause borrow checker errors. + /// The borrow checker understands this construct as unreachable, and so won't complain. + #[inline(always)] + pub fn new(_: Option) -> Self { + Self(PhantomData) + } +} + +mod empty { + use crate::prelude::IntoPyObject; + use crate::pycall::args::empty::EmptyArgsStorage; + use crate::pycall::args::FundamentalStorage; + + use super::ArgsStorageSelector; + + #[diagnostic::on_unimplemented( + message = "`{Self}` cannot be unpacked in a Python call", + note = "the following types can be unpacked in a Python call: \ + any iterable Python object, any `IntoIterator` that yields \ + types that can be converted into Python objects" + )] + pub trait EmptySelector<'py, T> { + type Output: FundamentalStorage<'py>; + fn __py_unpack_args_select(self, value: T) -> Self::Output; + } + + impl<'py> EmptySelector<'py, ()> for &&&&&&&&&&&ArgsStorageSelector<()> { + type Output = EmptyArgsStorage; + #[inline(always)] + fn __py_unpack_args_select(self, _value: ()) -> EmptyArgsStorage { + EmptyArgsStorage + } + } + impl<'py, T> EmptySelector<'py, [T; 0]> for &&&&&&&&&&&ArgsStorageSelector<[T; 0]> + where + T: IntoPyObject<'py>, + { + type Output = EmptyArgsStorage; + #[inline(always)] + fn __py_unpack_args_select(self, _value: [T; 0]) -> EmptyArgsStorage { + EmptyArgsStorage + } + } + impl<'py, 'a, T> EmptySelector<'py, &'a [T; 0]> for &&&&&&&&&&&ArgsStorageSelector<&'a [T; 0]> + where + T: IntoPyObject<'py>, + { + type Output = EmptyArgsStorage; + #[inline(always)] + fn __py_unpack_args_select(self, _value: &'a [T; 0]) -> EmptyArgsStorage { + EmptyArgsStorage + } + } + impl<'py, 'a, T> EmptySelector<'py, &'a mut [T; 0]> + for &&&&&&&&&&&ArgsStorageSelector<&'a mut [T; 0]> + where + T: IntoPyObject<'py>, + { + type Output = EmptyArgsStorage; + #[inline(always)] + fn __py_unpack_args_select(self, _value: &'a mut [T; 0]) -> EmptyArgsStorage { + EmptyArgsStorage + } + } +} + +mod array { + use crate::conversion::IntoPyObject; + use crate::pycall::args::array::ArrayArgsStorage; + use crate::pycall::args::FundamentalStorage; + + use super::ArgsStorageSelector; + + #[diagnostic::on_unimplemented( + message = "`{Self}` cannot be unpacked in a Python call", + note = "the following types can be unpacked in a Python call: \ + any iterable Python object, any `IntoIterator` that yields \ + types that can be converted into Python objects" + )] + pub trait ArraySelector<'py, T> { + type Output: FundamentalStorage<'py>; + fn __py_unpack_args_select(self, value: T) -> Self::Output; + } + + impl<'py, T, const N: usize> ArraySelector<'py, [T; N]> for &&&&&&&&&&ArgsStorageSelector<[T; N]> + where + T: IntoPyObject<'py>, + { + type Output = ArrayArgsStorage<[T; N]>; + #[inline(always)] + fn __py_unpack_args_select(self, value: [T; N]) -> ArrayArgsStorage<[T; N]> { + ArrayArgsStorage(value) + } + } + + impl<'a, 'py, T, const N: usize> ArraySelector<'py, &'a [T; N]> + for &&&&&&&&&&ArgsStorageSelector<&'a [T; N]> + where + &'a T: IntoPyObject<'py>, + { + type Output = ArrayArgsStorage<&'a [T; N]>; + #[inline(always)] + fn __py_unpack_args_select(self, value: &'a [T; N]) -> ArrayArgsStorage<&'a [T; N]> { + ArrayArgsStorage(value) + } + } + + impl<'a, 'py, T, const N: usize> ArraySelector<'py, &'a mut [T; N]> + for &&&&&&&&&&ArgsStorageSelector<&'a mut [T; N]> + where + &'a T: IntoPyObject<'py>, + { + type Output = ArrayArgsStorage<&'a mut [T; N]>; + #[inline(always)] + fn __py_unpack_args_select( + self, + value: &'a mut [T; N], + ) -> ArrayArgsStorage<&'a mut [T; N]> { + ArrayArgsStorage(value) + } + } +} + +mod existing { + use crate::pycall::args::existing::{ExistingArgListSlice, ExistingArgListSliceTrait}; + use crate::pycall::args::FundamentalStorage; + + use super::ArgsStorageSelector; + + #[diagnostic::on_unimplemented( + message = "`{Self}` cannot be unpacked in a Python call", + note = "the following types can be unpacked in a Python call: \ + any iterable Python object, any `IntoIterator` that yields \ + types that can be converted into Python objects" + )] + pub trait ExistingSelector<'py, T> { + type Output: FundamentalStorage<'py>; + fn __py_unpack_args_select(self, value: T) -> Self::Output; + } + + impl<'py, T: ExistingArgListSliceTrait> ExistingSelector<'py, T> + for &&&&&&&&&ArgsStorageSelector + { + type Output = ExistingArgListSlice; + #[inline(always)] + fn __py_unpack_args_select(self, value: T) -> ExistingArgListSlice { + ExistingArgListSlice(value) + } + } +} + +mod trusted_len { + use crate::conversion::IntoPyObject; + use crate::pycall::args::vec::{TrustedLenIterator, VecArgsStorage}; + use crate::pycall::args::FundamentalStorage; + use crate::pycall::trusted_len::TrustedLen; + + use super::ArgsStorageSelector; + + #[diagnostic::on_unimplemented( + message = "`{Self}` cannot be unpacked in a Python call", + note = "the following types can be unpacked in a Python call: \ + any iterable Python object, any `IntoIterator` that yields \ + types that can be converted into Python objects" + )] + pub trait TrustedLenSelector<'py, T> { + type Output: FundamentalStorage<'py>; + fn __py_unpack_args_select(self, value: T) -> Self::Output; + } + + impl<'py, T, Item> TrustedLenSelector<'py, T> for &&&&&&&&ArgsStorageSelector + where + T: IntoIterator, + T::IntoIter: TrustedLen, + Item: IntoPyObject<'py>, + { + type Output = VecArgsStorage>; + #[inline(always)] + fn __py_unpack_args_select( + self, + value: T, + ) -> VecArgsStorage> { + VecArgsStorage(TrustedLenIterator(value.into_iter())) + } + } +} + +mod exact_size { + use crate::conversion::IntoPyObject; + use crate::pycall::args::vec::{ExactSizeIterator, VecArgsStorage}; + use crate::pycall::args::FundamentalStorage; + + use super::ArgsStorageSelector; + + #[diagnostic::on_unimplemented( + message = "`{Self}` cannot be unpacked in a Python call", + note = "the following types can be unpacked in a Python call: \ + any iterable Python object, any `IntoIterator` that yields \ + types that can be converted into Python objects" + )] + pub trait ExactSizeIteratorSelector<'py, T> { + type Output: FundamentalStorage<'py>; + fn __py_unpack_args_select(self, value: T) -> Self::Output; + } + + impl<'py, T, Item> ExactSizeIteratorSelector<'py, T> for &&&&&&&ArgsStorageSelector + where + T: IntoIterator, + T::IntoIter: std::iter::ExactSizeIterator, + Item: IntoPyObject<'py>, + { + type Output = VecArgsStorage>; + #[inline(always)] + fn __py_unpack_args_select( + self, + value: T, + ) -> VecArgsStorage> { + VecArgsStorage(ExactSizeIterator::new(value.into_iter())) + } + } +} + +mod any_iterator { + use crate::conversion::IntoPyObject; + use crate::pycall::args::unknown_size::{AnyIteratorArgs, UnsizedArgsStorage}; + use crate::pycall::args::FundamentalStorage; + + use super::ArgsStorageSelector; + + #[diagnostic::on_unimplemented( + message = "`{Self}` cannot be unpacked in a Python call", + note = "the following types can be unpacked in a Python call: \ + any iterable Python object, any `IntoIterator` that yields \ + types that can be converted into Python objects" + )] + pub trait AnyIteratorSelector<'py, T> { + type Output: FundamentalStorage<'py>; + fn __py_unpack_args_select(self, value: T) -> Self::Output; + } + + impl<'py, T, Item> AnyIteratorSelector<'py, T> for &&&&&&ArgsStorageSelector + where + T: IntoIterator, + Item: IntoPyObject<'py>, + { + type Output = UnsizedArgsStorage>; + #[inline(always)] + fn __py_unpack_args_select( + self, + value: T, + ) -> UnsizedArgsStorage> { + UnsizedArgsStorage(AnyIteratorArgs(value.into_iter())) + } + } +} + +mod pytuple { + use crate::pycall::args::pyobjects::PyTupleArgs; + use crate::pycall::args::unknown_size::UnsizedArgsStorage; + use crate::pycall::args::FundamentalStorage; + use crate::pycall::as_pyobject::AsPyObject; + use crate::types::PyTuple; + + use super::ArgsStorageSelector; + + #[diagnostic::on_unimplemented( + message = "`{Self}` cannot be unpacked in a Python call", + note = "the following types can be unpacked in a Python call: \ + any iterable Python object, any `IntoIterator` that yields \ + types that can be converted into Python objects" + )] + pub trait PyTupleSelector<'py, T> { + type Output: FundamentalStorage<'py>; + fn __py_unpack_args_select(self, value: T) -> Self::Output; + } + + impl<'py, T: AsPyObject<'py, PyObject = PyTuple>> PyTupleSelector<'py, T> + for &&&&&ArgsStorageSelector + { + type Output = UnsizedArgsStorage>; + #[inline(always)] + fn __py_unpack_args_select(self, value: T) -> UnsizedArgsStorage> { + UnsizedArgsStorage(PyTupleArgs::new(value)) + } + } +} + +mod python_builtins { + use crate::pycall::args::pyobjects::{IterableBuiltin, PyBuiltinIterableArgs}; + use crate::pycall::args::unknown_size::UnsizedArgsStorage; + use crate::pycall::args::FundamentalStorage; + use crate::pycall::as_pyobject::AsPyObject; + + use super::ArgsStorageSelector; + + #[diagnostic::on_unimplemented( + message = "`{Self}` cannot be unpacked in a Python call", + note = "the following types can be unpacked in a Python call: \ + any iterable Python object, any `IntoIterator` that yields \ + types that can be converted into Python objects" + )] + pub trait PythonBuiltinsSelector<'py, T> { + type Output: FundamentalStorage<'py>; + fn __py_unpack_args_select(self, value: T) -> Self::Output; + } + + impl<'py, T> PythonBuiltinsSelector<'py, T> for &&&&ArgsStorageSelector + where + T: AsPyObject<'py>, + T::PyObject: IterableBuiltin, + { + type Output = UnsizedArgsStorage>; + #[inline(always)] + fn __py_unpack_args_select(self, value: T) -> UnsizedArgsStorage> { + UnsizedArgsStorage(PyBuiltinIterableArgs::new(value)) + } + } +} + +mod any_pyiterable { + use crate::pycall::args::pyobjects::AnyPyIterable; + use crate::pycall::args::unknown_size::UnsizedArgsStorage; + use crate::pycall::args::FundamentalStorage; + use crate::pycall::as_pyobject::AsPyObject; + + use super::ArgsStorageSelector; + + #[diagnostic::on_unimplemented( + message = "`{Self}` cannot be unpacked in a Python call", + note = "the following types can be unpacked in a Python call: \ + any iterable Python object, any `IntoIterator` that yields \ + types that can be converted into Python objects" + )] + pub trait AnyPyIterableSelector<'py, T> { + type Output: FundamentalStorage<'py>; + fn __py_unpack_args_select(self, value: T) -> Self::Output; + } + + impl<'py, T: AsPyObject<'py>> AnyPyIterableSelector<'py, T> for &&&ArgsStorageSelector { + type Output = UnsizedArgsStorage>; + #[inline(always)] + fn __py_unpack_args_select(self, value: T) -> UnsizedArgsStorage> { + UnsizedArgsStorage(AnyPyIterable::new(value)) + } + } +} + +mod tuple { + use crate::pycall::args::array::Tuple; + use crate::pycall::args::{ArrayArgsStorage, FundamentalStorage}; + use crate::pycall::storage::RawStorage; + use crate::pycall::PPPyObject; + + use super::ArgsStorageSelector; + + #[diagnostic::on_unimplemented( + message = "`{Self}` cannot be unpacked in a Python call", + note = "the following types can be unpacked in a Python call: \ + any iterable Python object, any `IntoIterator` that yields \ + types that can be converted into Python objects" + )] + pub trait TupleSelector<'py, T> { + type Output: FundamentalStorage<'py>; + fn __py_unpack_args_select(self, value: T) -> Self::Output; + } + + impl<'py, T: Tuple<'py>> TupleSelector<'py, T> for &&ArgsStorageSelector + where + T::RawStorage: for<'a> RawStorage = PPPyObject> + 'static, + { + type Output = ArrayArgsStorage; + #[inline(always)] + fn __py_unpack_args_select(self, value: T) -> ArrayArgsStorage { + ArrayArgsStorage(value) + } + } +} diff --git a/src/pycall/args/unknown_size.rs b/src/pycall/args/unknown_size.rs new file mode 100644 index 00000000000..2a9bfd7edf6 --- /dev/null +++ b/src/pycall/args/unknown_size.rs @@ -0,0 +1,300 @@ +use std::marker::PhantomData; + +use crate::conversion::IntoPyObject; +use crate::pycall::storage::{RawStorage, UnsizedInitParam, UnsizedStorage}; +use crate::pycall::PPPyObject; +use crate::types::PyTuple; +use crate::{ffi, Borrowed, BoundObject, PyErr, PyResult, Python}; + +use super::helpers::write_raw_storage_to_tuple; +use super::{ArgumentsOffsetFlag, ConcatStorages, ResolveArgs}; + +pub struct UnsizedArgsStorage(pub(in super::super) T); + +impl<'py, T> ResolveArgs<'py> for UnsizedArgsStorage +where + T: ResolveArgs<'py>, + T::RawStorage: for<'a> RawStorage = UnsizedInitParam<'a>> + 'static, +{ + type RawStorage = T::RawStorage; + type Guard = T::Guard; + #[inline(always)] + fn init( + self, + py: Python<'py>, + storage: UnsizedInitParam<'_>, + base_storage: *const PPPyObject, + ) -> PyResult { + self.0.init(py, storage, base_storage) + } + #[inline(always)] + fn len(&self) -> usize { + self.0.len() + } + #[inline(always)] + fn write_to_tuple( + self, + _tuple: Borrowed<'_, 'py, PyTuple>, + _index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + panic!("unsized storages don't support direct writing into tuples") + } + #[inline(always)] + fn write_initialized_to_tuple( + tuple: Borrowed<'_, 'py, PyTuple>, + guard: Self::Guard, + raw_storage: &mut PPPyObject, + index: &mut ffi::Py_ssize_t, + ) { + T::write_initialized_to_tuple(tuple, guard, raw_storage, index) + } + #[inline(always)] + fn as_pytuple(&self, py: Python<'py>) -> Option> { + self.0.as_pytuple(py) + } + #[inline(always)] + fn has_known_size(&self) -> bool { + self.0.has_known_size() + } + const ARGUMENTS_OFFSET: ArgumentsOffsetFlag = ArgumentsOffsetFlag::Normal; + const IS_EMPTY: bool = T::IS_EMPTY; + const IS_ONE: bool = T::IS_ONE; + const USE_STACK_FOR_SMALL_LEN: bool = T::USE_STACK_FOR_SMALL_LEN; +} + +impl<'py, A, B> ResolveArgs<'py> for UnsizedArgsStorage> +where + A: ResolveArgs<'py>, + B: ResolveArgs<'py>, + A::RawStorage: for<'a> RawStorage = UnsizedInitParam<'a>> + 'static, + B::RawStorage: for<'a> RawStorage = UnsizedInitParam<'a>> + 'static, +{ + type RawStorage = UnsizedStorage; + type Guard = (A::Guard, B::Guard); + #[inline(always)] + fn init( + self, + py: Python<'py>, + storage: UnsizedInitParam<'_>, + base_storage: *const PPPyObject, + ) -> PyResult { + let g1 = self.0 .0.init(py, storage, base_storage)?; + let g2 = self.0 .1.init(py, storage, base_storage)?; + Ok((g1, g2)) + } + #[inline(always)] + fn len(&self) -> usize { + self.0 .0.len() + self.0 .1.len() + } + #[inline(always)] + fn write_to_tuple( + self, + _tuple: Borrowed<'_, 'py, PyTuple>, + _index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + panic!("unsized storages don't support direct writing into tuples") + } + #[inline(always)] + fn write_initialized_to_tuple( + tuple: Borrowed<'_, 'py, PyTuple>, + guard: Self::Guard, + raw_storage: &mut PPPyObject, + index: &mut ffi::Py_ssize_t, + ) { + A::write_initialized_to_tuple(tuple, guard.0, raw_storage, index); + B::write_initialized_to_tuple(tuple, guard.1, raw_storage, index); + } + #[inline(always)] + fn has_known_size(&self) -> bool { + self.0 .0.has_known_size() && self.0 .1.has_known_size() + } + const ARGUMENTS_OFFSET: ArgumentsOffsetFlag = ArgumentsOffsetFlag::Normal; + const IS_EMPTY: bool = A::IS_EMPTY && B::IS_EMPTY; + const IS_ONE: bool = (A::IS_EMPTY && B::IS_ONE) || (A::IS_ONE && B::IS_EMPTY); + const USE_STACK_FOR_SMALL_LEN: bool = false; +} + +pub struct SizedToUnsizedStorage(pub(in super::super) T); + +impl<'py, T> ResolveArgs<'py> for UnsizedArgsStorage> +where + T: ResolveArgs<'py>, + T::RawStorage: for<'a> RawStorage = PPPyObject> + 'static, +{ + type RawStorage = UnsizedStorage; + type Guard = T::Guard; + #[inline(always)] + fn init( + self, + py: Python<'py>, + storage: UnsizedInitParam<'_>, + base_storage: *const PPPyObject, + ) -> PyResult { + let len = self.0 .0.len(); + storage.reserve(len); + unsafe { + // The buffer might've been invalidated. + base_storage.cast_mut().write(storage.as_mut_ptr()); + + // FIXME: If the Vec will resize we'll get use-after-free. + let write_to = storage.as_mut_ptr().add(storage.len()); + let guard = self.0 .0.init(py, write_to, base_storage)?; + storage.set_len(storage.len() + len); + Ok(guard) + } + } + #[inline(always)] + fn len(&self) -> usize { + self.0 .0.len() + } + #[inline(always)] + fn write_to_tuple( + self, + _tuple: Borrowed<'_, 'py, PyTuple>, + _index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + panic!("unsized storages don't support direct writing into tuples") + } + #[inline(always)] + fn write_initialized_to_tuple( + tuple: Borrowed<'_, 'py, PyTuple>, + guard: Self::Guard, + raw_storage: &mut PPPyObject, + index: &mut ffi::Py_ssize_t, + ) { + T::write_initialized_to_tuple(tuple, guard, raw_storage, index) + } + #[inline(always)] + fn has_known_size(&self) -> bool { + true + } + #[inline(always)] + fn as_pytuple(&self, py: Python<'py>) -> Option> { + self.0 .0.as_pytuple(py) + } + const ARGUMENTS_OFFSET: ArgumentsOffsetFlag = ArgumentsOffsetFlag::Normal; + const IS_EMPTY: bool = T::IS_EMPTY; + const IS_ONE: bool = T::IS_ONE; + const USE_STACK_FOR_SMALL_LEN: bool = false; +} + +pub struct AnyIteratorArgs(pub(super) I); + +pub struct UnsizedGuard(*const PPPyObject, usize, usize, PhantomData); +impl UnsizedGuard { + #[inline(always)] + pub(super) fn empty(base_storage: *const PPPyObject) -> Self { + Self(base_storage, 0, 0, PhantomData) + } + #[inline(always)] + pub(super) fn from_range( + base_storage: *const PPPyObject, + start: PPPyObject, + len: usize, + ) -> Self { + Self( + base_storage, + unsafe { start.offset_from(base_storage.read()) as usize }, + len, + PhantomData, + ) + } + #[inline(always)] + pub(super) fn from_iter<'py, I, R, S, E>( + storage: UnsizedInitParam<'_>, + base_storage: *const PPPyObject, + size_hint: usize, + mut iter: I, + ) -> PyResult + where + I: Iterator>, + R: BoundObject<'py, S>, + E: Into, + { + storage.reserve(size_hint); + unsafe { + // The buffer might've been invalidated. + base_storage.cast_mut().write(storage.as_mut_ptr()); + } + let mut guard = UnsizedGuard(base_storage, storage.len(), 0, PhantomData); + iter.try_for_each(|item| match item { + Ok(item) => { + storage.push(item.into_ptr_raw()); + unsafe { + // The buffer might've been invalidated. + base_storage.cast_mut().write(storage.as_mut_ptr()) + } + guard.2 += 1; + Ok(()) + } + Err(err) => Err(err.into()), + })?; + Ok(guard) + } + #[inline(always)] + pub(super) fn len(&self) -> usize { + self.2 + } +} +impl Drop for UnsizedGuard { + #[inline(always)] + fn drop(&mut self) { + unsafe { + std::ptr::slice_from_raw_parts_mut(self.0.read().add(self.1).cast::(), self.2) + .drop_in_place(); + } + } +} + +impl<'py, I, Item> ResolveArgs<'py> for AnyIteratorArgs +where + I: Iterator, + Item: IntoPyObject<'py>, +{ + type RawStorage = UnsizedStorage; + type Guard = UnsizedGuard; + #[inline(always)] + fn init( + self, + py: Python<'py>, + storage: UnsizedInitParam<'_>, + base_storage: *const PPPyObject, + ) -> PyResult { + UnsizedGuard::from_iter( + storage, + base_storage, + self.0.size_hint().0, + self.0.map(|item| item.into_pyobject(py)), + ) + } + #[inline(always)] + fn len(&self) -> usize { + self.0.size_hint().0 + } + #[inline(always)] + fn write_to_tuple( + self, + _tuple: Borrowed<'_, 'py, PyTuple>, + _index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + panic!("unsized storages don't support direct writing into tuples") + } + #[inline(always)] + fn write_initialized_to_tuple( + tuple: Borrowed<'_, 'py, PyTuple>, + guard: Self::Guard, + raw_storage: &mut PPPyObject, + index: &mut ffi::Py_ssize_t, + ) { + write_raw_storage_to_tuple::(tuple, raw_storage, index, guard.len()); + std::mem::forget(guard); + } + #[inline(always)] + fn has_known_size(&self) -> bool { + false + } + const ARGUMENTS_OFFSET: ArgumentsOffsetFlag = ArgumentsOffsetFlag::Normal; + const IS_EMPTY: bool = false; + const IS_ONE: bool = false; + const USE_STACK_FOR_SMALL_LEN: bool = false; +} diff --git a/src/pycall/args/vec.rs b/src/pycall/args/vec.rs new file mode 100644 index 00000000000..b4ad0d7a2a9 --- /dev/null +++ b/src/pycall/args/vec.rs @@ -0,0 +1,282 @@ +use std::marker::PhantomData; + +use crate::conversion::IntoPyObject; +use crate::pycall::storage::{DynKnownSizeRawStorage, RawStorage}; +use crate::pycall::trusted_len::TrustedLen; +use crate::pycall::PPPyObject; +use crate::types::PyTuple; +use crate::{ffi, Borrowed, BoundObject, PyResult, Python}; + +use super::helpers::{ + concat_known_sized, write_iter_to_tuple, write_raw_storage_to_tuple, DropManyGuard, +}; +use super::{ArgumentsOffsetFlag, ConcatStorages, ResolveArgs}; + +pub struct VecArgsStorage(pub(in super::super) T); + +impl<'py, T> ResolveArgs<'py> for VecArgsStorage +where + T: ResolveArgs<'py>, + T::RawStorage: for<'a> RawStorage = PPPyObject> + 'static, +{ + type RawStorage = T::RawStorage; + type Guard = T::Guard; + #[inline(always)] + fn init( + self, + py: Python<'py>, + storage: PPPyObject, + base_storage: *const PPPyObject, + ) -> PyResult { + self.0.init(py, storage, base_storage) + } + #[inline(always)] + fn len(&self) -> usize { + self.0.len() + } + #[inline(always)] + fn write_to_tuple( + self, + tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + self.0.write_to_tuple(tuple, index) + } + #[inline(always)] + fn write_initialized_to_tuple( + tuple: Borrowed<'_, 'py, PyTuple>, + guard: Self::Guard, + raw_storage: &mut PPPyObject, + index: &mut ffi::Py_ssize_t, + ) { + T::write_initialized_to_tuple(tuple, guard, raw_storage, index) + } + #[inline(always)] + fn has_known_size(&self) -> bool { + true + } + const ARGUMENTS_OFFSET: ArgumentsOffsetFlag = ArgumentsOffsetFlag::Normal; + #[inline(always)] + fn as_pytuple(&self, py: Python<'py>) -> Option> { + self.0.as_pytuple(py) + } + const IS_EMPTY: bool = T::IS_EMPTY; + const IS_ONE: bool = T::IS_ONE; + const USE_STACK_FOR_SMALL_LEN: bool = T::USE_STACK_FOR_SMALL_LEN; +} + +impl<'py, A, B> ResolveArgs<'py> for VecArgsStorage> +where + A: ResolveArgs<'py>, + B: ResolveArgs<'py>, + A::RawStorage: for<'a> RawStorage = PPPyObject> + 'static, + B::RawStorage: for<'a> RawStorage = PPPyObject> + 'static, +{ + type RawStorage = DynKnownSizeRawStorage; + type Guard = (A::Guard, B::Guard); + #[inline(always)] + fn init( + self, + py: Python<'py>, + storage: PPPyObject, + base_storage: *const PPPyObject, + ) -> PyResult { + concat_known_sized(self.0 .0, self.0 .1, py, storage, base_storage) + } + #[inline(always)] + fn len(&self) -> usize { + self.0 .0.len() + self.0 .1.len() + } + #[inline(always)] + fn write_to_tuple( + self, + tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + self.0 .0.write_to_tuple(tuple, index)?; + self.0 .1.write_to_tuple(tuple, index) + } + #[inline(always)] + fn write_initialized_to_tuple( + tuple: Borrowed<'_, 'py, PyTuple>, + guard: Self::Guard, + raw_storage: &mut PPPyObject, + index: &mut ffi::Py_ssize_t, + ) { + A::write_initialized_to_tuple(tuple, guard.0, raw_storage, index); + B::write_initialized_to_tuple(tuple, guard.1, raw_storage, index); + } + #[inline(always)] + fn has_known_size(&self) -> bool { + true + } + const ARGUMENTS_OFFSET: ArgumentsOffsetFlag = ArgumentsOffsetFlag::Normal; + const IS_EMPTY: bool = A::IS_EMPTY && B::IS_EMPTY; + const IS_ONE: bool = (A::IS_EMPTY && B::IS_ONE) || (A::IS_ONE && B::IS_EMPTY); + const USE_STACK_FOR_SMALL_LEN: bool = true; +} + +pub struct TrustedLenIterator(pub(super) I); + +impl<'py, I, Item> ResolveArgs<'py> for TrustedLenIterator +where + I: TrustedLen, + Item: IntoPyObject<'py>, +{ + type RawStorage = DynKnownSizeRawStorage; + type Guard = DropManyGuard; + #[inline(always)] + fn init( + self, + py: Python<'py>, + storage: PPPyObject, + base_storage: *const PPPyObject, + ) -> PyResult { + DropManyGuard::from_iter(py, storage, base_storage, self.0) + } + #[inline(always)] + fn len(&self) -> usize { + self.0.size_hint().0 + } + #[inline(always)] + fn write_to_tuple( + self, + tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + write_iter_to_tuple(tuple, self.0, index) + } + #[inline(always)] + fn write_initialized_to_tuple( + tuple: Borrowed<'_, 'py, PyTuple>, + guard: Self::Guard, + raw_storage: &mut PPPyObject, + index: &mut ffi::Py_ssize_t, + ) { + write_raw_storage_to_tuple::(tuple, raw_storage, index, guard.len()) + } + #[inline(always)] + fn has_known_size(&self) -> bool { + true + } + const ARGUMENTS_OFFSET: ArgumentsOffsetFlag = ArgumentsOffsetFlag::Normal; + const IS_EMPTY: bool = false; + const IS_ONE: bool = false; + const USE_STACK_FOR_SMALL_LEN: bool = true; +} + +pub struct ExactSizeIterator { + iter: I, + // We cannot rely on the iterator providing the same `len()` every time (this will lead to unsoundness), + // so we save it here. + len: usize, +} + +impl ExactSizeIterator { + #[inline(always)] + pub(super) fn new(iter: I) -> Self { + Self { + len: iter.len(), + iter, + } + } +} + +impl<'py, I, Item> ResolveArgs<'py> for ExactSizeIterator +where + I: std::iter::ExactSizeIterator, + Item: IntoPyObject<'py>, +{ + type RawStorage = DynKnownSizeRawStorage; + type Guard = DropManyGuard; + #[inline(always)] + fn init( + mut self, + py: Python<'py>, + storage: PPPyObject, + base_storage: *const PPPyObject, + ) -> PyResult { + struct Guard(PPPyObject, usize, PhantomData); + impl Drop for Guard { + #[inline(always)] + fn drop(&mut self) { + unsafe { + std::ptr::slice_from_raw_parts_mut(self.0, self.1).drop_in_place(); + } + } + } + unsafe { + let len = self.len; + let mut guard = Guard::(storage, 0, PhantomData); + self.iter.try_for_each(|item| { + if guard.1 >= len { + // FIXME: Maybe this should be an `Err` and not panic? + panic!("an ExactSizeIterator produced more items than it declared"); + } + match item.into_pyobject(py) { + Ok(item) => { + guard.0.add(guard.1).write(item.into_ptr_raw()); + guard.1 += 1; + Ok(()) + } + Err(err) => Err(err.into()), + } + })?; + if guard.1 != len { + panic!("an ExactSizeIterator produced less items than it declared"); + } + Ok(DropManyGuard::new(base_storage, guard.0, guard.1)) + } + } + #[inline(always)] + fn len(&self) -> usize { + self.len + } + #[inline(always)] + fn write_to_tuple( + mut self, + tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + let mut i = *index; + let len = self.len as isize; + unsafe { + self.iter.try_for_each(|item| { + if i >= len { + // FIXME: Maybe this should be an `Err` and not panic? + panic!("an ExactSizeIterator produced more items than it declared"); + } + match item.into_pyobject(tuple.py()) { + Ok(item) => { + ffi::PyTuple_SET_ITEM(tuple.as_ptr(), i, item.into_ptr_raw()); + i += 1; + Ok(()) + } + Err(err) => Err(err.into()), + } + })?; + } + if i != len { + panic!("an ExactSizeIterator produced less items than it declared"); + } + *index = i; + Ok(()) + } + #[inline(always)] + fn write_initialized_to_tuple( + tuple: Borrowed<'_, 'py, PyTuple>, + guard: Self::Guard, + raw_storage: &mut PPPyObject, + index: &mut ffi::Py_ssize_t, + ) { + write_raw_storage_to_tuple::(tuple, raw_storage, index, guard.len()) + } + #[inline(always)] + fn has_known_size(&self) -> bool { + true + } + const ARGUMENTS_OFFSET: ArgumentsOffsetFlag = ArgumentsOffsetFlag::Normal; + const IS_EMPTY: bool = false; + const IS_ONE: bool = false; + const USE_STACK_FOR_SMALL_LEN: bool = true; +} diff --git a/src/pycall/args/vectorcall_arguments_offset.rs b/src/pycall/args/vectorcall_arguments_offset.rs new file mode 100644 index 00000000000..1fb4288e884 --- /dev/null +++ b/src/pycall/args/vectorcall_arguments_offset.rs @@ -0,0 +1,57 @@ +//! `PY_VECTORCALL_ARGUMENTS_OFFSET` needs a first empty arg. + +use std::mem::MaybeUninit; + +use crate::pycall::PPPyObject; +use crate::types::PyTuple; +use crate::{ffi, Borrowed, PyResult, Python}; + +use super::{ArgumentsOffsetFlag, ResolveArgs}; + +pub struct AppendEmptyArgForVectorcall; + +impl<'py> ResolveArgs<'py> for AppendEmptyArgForVectorcall { + type RawStorage = MaybeUninit<*mut ffi::PyObject>; + type Guard = (); + #[inline(always)] + fn init( + self, + _py: Python<'py>, + storage: PPPyObject, + _base_storage: *const PPPyObject, + ) -> PyResult { + unsafe { + storage.write(std::ptr::null_mut()); + } + Ok(()) + } + #[inline(always)] + fn len(&self) -> usize { + 1 + } + #[inline(always)] + fn write_to_tuple( + self, + _tuple: Borrowed<'_, 'py, PyTuple>, + _index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + unreachable!("AppendEmptyArgForVectorcall should never be converted into a tuple, it only exists for vectorcall") + } + #[inline(always)] + fn write_initialized_to_tuple( + _tuple: Borrowed<'_, 'py, PyTuple>, + _guard: Self::Guard, + _raw_storage: &mut PPPyObject, + _index: &mut ffi::Py_ssize_t, + ) { + unreachable!("AppendEmptyArgForVectorcall should never be converted into a tuple, it only exists for vectorcall") + } + #[inline(always)] + fn has_known_size(&self) -> bool { + true + } + const ARGUMENTS_OFFSET: ArgumentsOffsetFlag = ArgumentsOffsetFlag::Normal; + const IS_EMPTY: bool = false; + const IS_ONE: bool = false; + const USE_STACK_FOR_SMALL_LEN: bool = false; +} diff --git a/src/pycall/as_pyobject.rs b/src/pycall/as_pyobject.rs new file mode 100644 index 00000000000..f0d4352bca6 --- /dev/null +++ b/src/pycall/as_pyobject.rs @@ -0,0 +1,73 @@ +use crate::{Borrowed, Bound, Py, Python}; + +pub trait AsPyObject<'py> { + type PyObject; + fn as_borrowed(&self, py: Python<'py>) -> Borrowed<'_, 'py, Self::PyObject>; + const IS_OWNED: bool; + fn into_bound(self, py: Python<'py>) -> Bound<'py, Self::PyObject>; +} + +impl<'py, T> AsPyObject<'py> for Bound<'py, T> { + type PyObject = T; + #[inline(always)] + fn as_borrowed(&self, _py: Python<'py>) -> Borrowed<'_, 'py, Self::PyObject> { + self.as_borrowed() + } + const IS_OWNED: bool = true; + #[inline(always)] + fn into_bound(self, _py: Python<'py>) -> Bound<'py, Self::PyObject> { + self + } +} + +impl<'py, T> AsPyObject<'py> for Borrowed<'_, 'py, T> { + type PyObject = T; + #[inline(always)] + fn as_borrowed(&self, _py: Python<'py>) -> Borrowed<'_, 'py, Self::PyObject> { + *self + } + const IS_OWNED: bool = false; + #[inline(always)] + fn into_bound(self, _py: Python<'py>) -> Bound<'py, Self::PyObject> { + panic!("non-owned AsPyObject cannot be converted into Bound") + } +} + +impl<'py, T> AsPyObject<'py> for Py { + type PyObject = T; + #[inline(always)] + fn as_borrowed(&self, py: Python<'py>) -> Borrowed<'_, 'py, Self::PyObject> { + self.bind_borrowed(py) + } + const IS_OWNED: bool = true; + #[inline(always)] + fn into_bound(self, py: Python<'py>) -> Bound<'py, Self::PyObject> { + self.into_bound(py) + } +} + +impl<'py, T: AsPyObject<'py>> AsPyObject<'py> for &'_ T { + type PyObject = T::PyObject; + #[inline(always)] + fn as_borrowed(&self, py: Python<'py>) -> Borrowed<'_, 'py, Self::PyObject> { + T::as_borrowed(*self, py) + } + const IS_OWNED: bool = false; + #[inline(always)] + fn into_bound(self, _py: Python<'py>) -> Bound<'py, Self::PyObject> { + panic!("non-owned AsPyObject cannot be converted into Bound") + } +} + +impl<'py, T: AsPyObject<'py>> AsPyObject<'py> for &'_ mut T { + type PyObject = T::PyObject; + #[inline(always)] + fn as_borrowed(&self, py: Python<'py>) -> Borrowed<'_, 'py, Self::PyObject> { + T::as_borrowed(*self, py) + } + const IS_OWNED: bool = false; + #[inline(always)] + fn into_bound(self, _py: Python<'py>) -> Bound<'py, Self::PyObject> { + panic!("non-owned AsPyObject cannot be converted into Bound") + } +} diff --git a/src/pycall/kwargs.rs b/src/pycall/kwargs.rs new file mode 100644 index 00000000000..6c36908aa1e --- /dev/null +++ b/src/pycall/kwargs.rs @@ -0,0 +1,190 @@ +mod array; +mod concat; +mod empty; +mod helpers; +mod known; +mod pyobjects; +mod selector; +mod unknown_size; +mod vec; + +pub use empty::EmptyKwargsStorage; +pub use selector::{select_traits, KwargsStorageSelector}; + +pub(super) use array::{ArrayKwargsStorage, Tuple}; +pub(super) use concat::{ConcatKwargsStorages, FundamentalStorage}; +pub(super) use known::{ + KnownKwargsNames, KnownKwargsStorage, TypeLevelPyObjectListCons, TypeLevelPyObjectListNil, + TypeLevelPyObjectListTrait, +}; +pub(super) use unknown_size::UnsizedKwargsStorage; +pub(super) use vec::VecKwargsStorage; + +use std::collections::HashMap; +use std::hash::{BuildHasherDefault, Hash, Hasher}; +use std::mem::ManuallyDrop; + +use crate::exceptions::PyTypeError; +use crate::types::{PyDict, PyString, PyTuple}; +use crate::{ffi, Bound, PyAny}; +use crate::{Borrowed, PyResult, Python}; + +use super::storage::RawStorage; +use super::PPPyObject; + +struct ExistingName { + hash: ffi::Py_hash_t, + ptr: *mut ffi::PyObject, +} + +impl ExistingName { + #[inline] + fn new(name: Borrowed<'_, '_, PyString>) -> Self { + let hash = unsafe { ffi::PyObject_Hash(name.as_ptr()) }; + Self { + hash, + ptr: name.as_ptr(), + } + } +} + +impl Hash for ExistingName { + #[inline] + fn hash(&self, state: &mut H) { + state.write_isize(self.hash); + } +} + +impl PartialEq for ExistingName { + #[inline] + fn eq(&self, other: &Self) -> bool { + // PyUnicode comparison functions don't consider str subclasses. + unsafe { ffi::PyObject_RichCompareBool(self.ptr, other.ptr, ffi::Py_EQ) == 1 } + } +} + +impl Eq for ExistingName {} + +#[derive(Default)] +struct IdentityHasher(isize); + +impl Hasher for IdentityHasher { + #[inline] + fn write_isize(&mut self, i: isize) { + self.0 = i; + } + + fn finish(&self) -> u64 { + self.0 as u64 + } + + fn write(&mut self, _bytes: &[u8]) { + panic!("should only hash isize in IdentityHasher") + } +} + +pub struct ExistingNames(HashMap>); + +impl ExistingNames { + #[inline] + pub(super) fn new(capacity: usize) -> Self { + Self(HashMap::with_capacity_and_hasher( + capacity, + BuildHasherDefault::default(), + )) + } + + #[inline(always)] + fn check_borrowed( + &mut self, + new_name: Borrowed<'_, '_, PyString>, + kwargs_tuple: Borrowed<'_, '_, PyTuple>, + index: ffi::Py_ssize_t, + ) -> PyResult<()> { + match self.0.entry(ExistingName::new(new_name.as_borrowed())) { + std::collections::hash_map::Entry::Occupied(_) => { + return Err(PyTypeError::new_err( + intern!( + kwargs_tuple.py(), + "got multiple values for keyword argument" + ) + .clone() + .unbind(), + )); + } + std::collections::hash_map::Entry::Vacant(entry) => { + entry.insert(()); + unsafe { + ffi::PyTuple_SET_ITEM(kwargs_tuple.as_ptr(), index, new_name.as_ptr()); + } + Ok(()) + } + } + } + + #[inline(always)] + fn insert( + &mut self, + new_name: Bound<'_, PyString>, + new_value: Borrowed<'_, '_, PyAny>, + args: PPPyObject, + kwargs_tuple: Borrowed<'_, '_, PyTuple>, + index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + self.check_borrowed( + ManuallyDrop::new(new_name).as_borrowed(), + kwargs_tuple, + *index, + )?; + unsafe { + args.offset(*index).write(new_value.as_ptr()); + } + *index += 1; + Ok(()) + } +} + +#[diagnostic::on_unimplemented( + message = "`{Self}` cannot be unpacked in a Python call", + note = "the following types can be unpacked in a Python call: \ + any mapping Python object, any `IntoIterator` that yields \ + (IntoPyObject, IntoPyObject)" +)] +pub trait ResolveKwargs<'py>: Sized { + type RawStorage: for<'a> RawStorage = PPPyObject> + 'static; + type Guard; + fn init( + self, + args: PPPyObject, + kwargs_tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + existing_names: &mut ExistingNames, + ) -> PyResult; + #[inline(always)] + fn init_no_names(self, _py: Python<'py>, _args: PPPyObject) -> PyResult { + unreachable!("`ResolveKwargs::init_no_names()` should only be called for known names") + } + fn len(&self) -> usize; + /// This returns the number of kwargs written. + fn write_to_dict(self, dict: Borrowed<'_, 'py, PyDict>) -> PyResult; + #[inline(always)] + fn can_be_cheaply_converted_to_pydict(&self, _py: Python<'py>) -> bool { + false + } + #[inline(always)] + fn into_pydict(self, _py: Python<'py>) -> PyResult> { + panic!("cannot be cheaply converted into PyDict") + } + #[inline(always)] + fn as_names_pytuple(&self) -> Option> { + None + } + fn has_known_size(&self) -> bool; + const IS_EMPTY: bool; +} + +pub struct ConcatStorages(A, B); + +/// This struct is used to create an array whose size is the sum of two smaller arrays, without generic_const_exprs. +#[repr(C)] +pub struct ConcatArrays(A, B); diff --git a/src/pycall/kwargs/array.rs b/src/pycall/kwargs/array.rs new file mode 100644 index 00000000000..85c311e9d90 --- /dev/null +++ b/src/pycall/kwargs/array.rs @@ -0,0 +1,290 @@ +use std::mem::MaybeUninit; + +use crate::conversion::IntoPyObject; +use crate::types::{PyDict, PyString, PyTuple}; +use crate::{ffi, Borrowed, Bound, BoundObject, PyResult, Python}; + +use super::helpers::{set_kwarg, set_kwargs_from_iter, DropManyGuard, DropOneGuard}; +use super::{ConcatArrays, ConcatStorages, ExistingNames, PPPyObject, ResolveKwargs}; + +pub struct ArrayKwargsStorage(pub(in super::super) T); + +impl<'py, T: ResolveKwargs<'py>> ResolveKwargs<'py> for ArrayKwargsStorage { + type RawStorage = T::RawStorage; + type Guard = T::Guard; + #[inline(always)] + fn init( + self, + args: PPPyObject, + kwargs_tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + existing_names: &mut ExistingNames, + ) -> PyResult { + self.0.init(args, kwargs_tuple, index, existing_names) + } + #[inline(always)] + fn len(&self) -> usize { + self.0.len() + } + #[inline(always)] + fn write_to_dict(self, dict: Borrowed<'_, 'py, PyDict>) -> PyResult { + self.0.write_to_dict(dict) + } + #[inline(always)] + fn has_known_size(&self) -> bool { + self.0.has_known_size() + } + const IS_EMPTY: bool = T::IS_EMPTY; + #[inline(always)] + fn as_names_pytuple(&self) -> Option> { + self.0.as_names_pytuple() + } + #[inline(always)] + fn can_be_cheaply_converted_to_pydict(&self, py: Python<'py>) -> bool { + self.0.can_be_cheaply_converted_to_pydict(py) + } + #[inline(always)] + fn init_no_names(self, py: Python<'py>, args: PPPyObject) -> PyResult { + self.0.init_no_names(py, args) + } + #[inline(always)] + fn into_pydict(self, py: Python<'py>) -> PyResult> { + self.0.into_pydict(py) + } +} + +impl<'py, A, B> ResolveKwargs<'py> for ArrayKwargsStorage> +where + A: ResolveKwargs<'py>, + B: ResolveKwargs<'py>, +{ + type RawStorage = MaybeUninit>; + type Guard = (A::Guard, B::Guard); + #[inline(always)] + fn init( + self, + args: PPPyObject, + kwargs_tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + existing_names: &mut ExistingNames, + ) -> PyResult { + let g1 = self.0 .0.init(args, kwargs_tuple, index, existing_names)?; + let g2 = self.0 .1.init(args, kwargs_tuple, index, existing_names)?; + Ok((g1, g2)) + } + #[inline(always)] + fn len(&self) -> usize { + self.0 .0.len() + self.0 .1.len() + } + #[inline(always)] + fn write_to_dict(self, dict: Borrowed<'_, 'py, PyDict>) -> PyResult { + let len1 = self.0 .0.write_to_dict(dict)?; + let len2 = self.0 .1.write_to_dict(dict)?; + Ok(len1 + len2) + } + #[inline(always)] + fn has_known_size(&self) -> bool { + self.0 .0.has_known_size() && self.0 .1.has_known_size() + } + const IS_EMPTY: bool = A::IS_EMPTY && B::IS_EMPTY; +} + +impl<'py, K, V, const N: usize> ResolveKwargs<'py> for [(K, V); N] +where + K: IntoPyObject<'py, Target = PyString>, + V: IntoPyObject<'py>, +{ + type RawStorage = MaybeUninit<[*mut ffi::PyObject; N]>; + type Guard = DropManyGuard; + #[inline(always)] + fn init( + self, + args: PPPyObject, + kwargs_tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + existing_names: &mut ExistingNames, + ) -> PyResult { + DropManyGuard::from_iter(args, kwargs_tuple, self, index, existing_names) + } + #[inline(always)] + fn len(&self) -> usize { + N + } + #[inline(always)] + fn write_to_dict(self, dict: Borrowed<'_, 'py, PyDict>) -> PyResult { + set_kwargs_from_iter(dict, self)?; + Ok(N) + } + #[inline(always)] + fn has_known_size(&self) -> bool { + true + } + const IS_EMPTY: bool = N == 0; +} + +impl<'a, 'py, K, V, const N: usize> ResolveKwargs<'py> for &'a [(K, V); N] +where + &'a K: IntoPyObject<'py, Target = PyString>, + &'a V: IntoPyObject<'py>, +{ + type RawStorage = MaybeUninit<[*mut ffi::PyObject; N]>; + type Guard = DropManyGuard<<&'a V as IntoPyObject<'py>>::Output>; + #[inline(always)] + fn init( + self, + args: PPPyObject, + kwargs_tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + existing_names: &mut ExistingNames, + ) -> PyResult { + DropManyGuard::from_iter( + args, + kwargs_tuple, + self.iter().map(|(k, v)| (k, v)), + index, + existing_names, + ) + } + #[inline(always)] + fn len(&self) -> usize { + N + } + #[inline(always)] + fn write_to_dict(self, dict: Borrowed<'_, 'py, PyDict>) -> PyResult { + set_kwargs_from_iter(dict, self.iter().map(|(k, v)| (k, v)))?; + Ok(N) + } + #[inline(always)] + fn has_known_size(&self) -> bool { + true + } + const IS_EMPTY: bool = N == 0; +} + +impl<'a, 'py, K, V, const N: usize> ResolveKwargs<'py> for &'a mut [(K, V); N] +where + &'a K: IntoPyObject<'py, Target = PyString>, + &'a V: IntoPyObject<'py>, +{ + type RawStorage = MaybeUninit<[*mut ffi::PyObject; N]>; + type Guard = DropManyGuard<<&'a V as IntoPyObject<'py>>::Output>; + #[inline(always)] + fn init( + self, + args: PPPyObject, + kwargs_tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + existing_names: &mut ExistingNames, + ) -> PyResult { + DropManyGuard::from_iter( + args, + kwargs_tuple, + self.iter().map(|(k, v)| (k, v)), + index, + existing_names, + ) + } + #[inline(always)] + fn len(&self) -> usize { + N + } + #[inline(always)] + fn write_to_dict(self, dict: Borrowed<'_, 'py, PyDict>) -> PyResult { + set_kwargs_from_iter(dict, self.iter().map(|(k, v)| (k, v)))?; + Ok(N) + } + #[inline(always)] + fn has_known_size(&self) -> bool { + true + } + const IS_EMPTY: bool = N == 0; +} + +/// A helper trait so that we don't have to repeat the macro for tuples both here and in selection. +pub trait Tuple<'py>: ResolveKwargs<'py> {} + +macro_rules! impl_resolve_args_for_tuple { + ( @guard_type $ty:ty, $next:ident, $($rest:ident,)* ) => { + impl_resolve_args_for_tuple!( @guard_type ConcatArrays<$ty, $next::Output>, $($rest,)* ) + }; + ( @guard_type $ty:ty, ) => { + $ty + }; + ( @count $t:ident ) => { 1 }; + ( ) => {}; + ( + $first_a:ident $first_b:ident, $( $rest_a:ident $rest_b:ident, )* + ) => { + impl<'py, $first_a, $first_b, $( $rest_a, $rest_b, )*> Tuple<'py> for ( ($first_a, $first_b), $( ($rest_a, $rest_b), )* ) + where + $first_a: IntoPyObject<'py, Target = PyString>, + $first_b: IntoPyObject<'py>, + $( + $rest_a: IntoPyObject<'py, Target = PyString>, + $rest_b: IntoPyObject<'py>, + )* + {} + + impl<'py, $first_a, $first_b, $( $rest_a, $rest_b, )*> ResolveKwargs<'py> for ( ($first_a, $first_b), $( ($rest_a, $rest_b), )* ) + where + $first_a: IntoPyObject<'py, Target = PyString>, + $first_b: IntoPyObject<'py>, + $( + $rest_a: IntoPyObject<'py, Target = PyString>, + $rest_b: IntoPyObject<'py>, + )* + { + type RawStorage = MaybeUninit<[*mut ffi::PyObject; 1 $( + impl_resolve_args_for_tuple!( @count $rest_b ) )*]>; + type Guard = DropOneGuard<'py, impl_resolve_args_for_tuple!( @guard_type $first_b::Output, $($rest_b,)* )>; + #[inline(always)] + fn init( + self, + args: PPPyObject, + kwargs_tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + existing_names: &mut ExistingNames, + ) -> PyResult { + #[allow(non_snake_case)] + let ( ($first_a, $first_b), $( ($rest_a, $rest_b), )* ) = self; + Ok( + DropOneGuard::from_write(args, kwargs_tuple, index, existing_names, $first_a, $first_b)? + $( .write(kwargs_tuple, index, existing_names, $rest_a, $rest_b)? )* + ) + } + #[inline(always)] + fn len(&self) -> usize { + 1 $( + impl_resolve_args_for_tuple!( @count $rest_a ) )* + } + #[inline(always)] + fn write_to_dict(self, dict: Borrowed<'_, 'py, PyDict>) -> PyResult { + let py = dict.py(); + #[allow(non_snake_case)] + let ( ($first_a, $first_b), $( ($rest_a, $rest_b), )* ) = self; + set_kwarg( + dict, + $first_a.into_pyobject(py).map_err(Into::into)?.as_borrowed(), + $first_b.into_pyobject(py).map_err(Into::into)?.into_any().as_borrowed(), + )?; + $( + set_kwarg( + dict, + $rest_a.into_pyobject(py).map_err(Into::into)?.as_borrowed(), + $rest_b.into_pyobject(py).map_err(Into::into)?.into_any().as_borrowed(), + )?; + )* + Ok(1 $( + impl_resolve_args_for_tuple!( @count $rest_b ) )*) + } + #[inline(always)] + fn has_known_size(&self) -> bool { + true + } + const IS_EMPTY: bool = false; + } + + impl_resolve_args_for_tuple!( $($rest_a $rest_b,)* ); + }; +} + +// If you are changing the size of the tuple here, make sure to change `build_unknown_non_unpacked_kwargs()` in +// pyo3-macros-backend/src/pycall.rs too. +impl_resolve_args_for_tuple!(A1 A2, B1 B2, C1 C2, D1 D2, E1 E2, F1 F2, G1 G2, H1 H2, I1 I2, J1 J2, K1 K2, L1 L2, M1 M2,); diff --git a/src/pycall/kwargs/concat.rs b/src/pycall/kwargs/concat.rs new file mode 100644 index 00000000000..deeb9a37db5 --- /dev/null +++ b/src/pycall/kwargs/concat.rs @@ -0,0 +1,182 @@ +//! We have 5 fundamental storages: `ArrayKwargsStorage`, `EmptyKwargsStorage`, `KnownKwargsStorage`, +//! `UnsizedKwargsStorage`, and `VecKwargsStorage`. We need to define all +//! combinations between them, that means 4^2+4=20 impls (`KnownKwargsStorage` is special: +//! it only concat when it is LHS, and it cannot concat with itself). Fortunately, macros can help with that. + +use super::array::ArrayKwargsStorage; +use super::empty::EmptyKwargsStorage; +use super::known::KnownKwargsStorage; +use super::unknown_size::UnsizedKwargsStorage; +use super::vec::VecKwargsStorage; +use super::{ConcatStorages, ResolveKwargs, TypeLevelPyObjectListTrait}; + +/// A storage that can be concatenated with other storages. +/// +/// A storage can be a remark how to handle some unpacked argument (e.g. tuples implement `ResolveKwargs`), +/// or it can also carry instructions how to create the whole list of arguments (e.g. an `ArrayKwargsStorage`). +/// This trait signifies the latter. +pub trait FundamentalStorage<'py>: ResolveKwargs<'py> {} +macro_rules! impl_fundamental_storage { + ( $($storage:ident)+ ) => { + $( + impl<'py, T> FundamentalStorage<'py> for $storage where + $storage: ResolveKwargs<'py> + { + } + )+ + }; +} +impl_fundamental_storage!(ArrayKwargsStorage VecKwargsStorage UnsizedKwargsStorage); +impl<'py> FundamentalStorage<'py> for EmptyKwargsStorage {} +impl<'py, Values: TypeLevelPyObjectListTrait<'py>> FundamentalStorage<'py> + for KnownKwargsStorage<'py, Values> +{ +} + +pub trait ConcatKwargsStorages<'py, Rhs: FundamentalStorage<'py>>: FundamentalStorage<'py> { + type Output: ResolveKwargs<'py>; + fn concat(self, other: Rhs) -> Self::Output; +} + +macro_rules! define_concat { + ( + $( + $storage1:ident + $storage2:ident = $result:ident + )+ + ) => { + $( + impl<'py, A, B> ConcatKwargsStorages<'py, $storage2> for $storage1 + where + $storage1: ResolveKwargs<'py>, + $storage2: ResolveKwargs<'py>, + $result, $storage2>>: ResolveKwargs<'py>, + { + type Output = $result, $storage2>>; + #[inline(always)] + fn concat(self, other: $storage2) -> Self::Output { + $result(ConcatStorages(self, other)) + } + } + )+ + }; +} +define_concat!( + ArrayKwargsStorage + ArrayKwargsStorage = ArrayKwargsStorage + ArrayKwargsStorage + VecKwargsStorage = VecKwargsStorage + VecKwargsStorage + ArrayKwargsStorage = VecKwargsStorage + VecKwargsStorage + VecKwargsStorage = VecKwargsStorage +); + +macro_rules! define_concat_empty { + ( $( $other:ident )+ ) => { + $( + impl<'py, T> ConcatKwargsStorages<'py, $other> for EmptyKwargsStorage + where + $other: ResolveKwargs<'py>, + { + type Output = $other; + #[inline(always)] + fn concat(self, other: $other) -> Self::Output { + other + } + } + impl<'py, T> ConcatKwargsStorages<'py, EmptyKwargsStorage> for $other + where + $other: ResolveKwargs<'py>, + { + type Output = $other; + #[inline(always)] + fn concat(self, _other: EmptyKwargsStorage) -> Self::Output { + self + } + } + )+ + }; +} +define_concat_empty!( + ArrayKwargsStorage VecKwargsStorage UnsizedKwargsStorage +); +impl<'py> ConcatKwargsStorages<'py, EmptyKwargsStorage> for EmptyKwargsStorage { + #[inline(always)] + fn concat(self, _other: EmptyKwargsStorage) -> Self::Output { + EmptyKwargsStorage + } + type Output = EmptyKwargsStorage; +} +impl<'py, Values> ConcatKwargsStorages<'py, EmptyKwargsStorage> for KnownKwargsStorage<'py, Values> +where + Values: TypeLevelPyObjectListTrait<'py>, +{ + #[inline(always)] + fn concat(self, _other: EmptyKwargsStorage) -> Self::Output { + self + } + type Output = KnownKwargsStorage<'py, Values>; +} + +macro_rules! define_concat_known { + ( $( $other:ident )+ ) => { + $( + impl<'py, T, Values> ConcatKwargsStorages<'py, $other> for KnownKwargsStorage<'py, Values> + where + Values: TypeLevelPyObjectListTrait<'py>, + $other: ResolveKwargs<'py>, + ArrayKwargsStorage>: ConcatKwargsStorages<'py, $other>, + { + type Output = > as ConcatKwargsStorages<'py, $other>>::Output; + #[inline(always)] + fn concat(self, other: $other) -> Self::Output { + > as ConcatKwargsStorages<'py, $other>>::concat( + ArrayKwargsStorage(self), other) + } + } + )+ + }; +} +define_concat_known!(ArrayKwargsStorage VecKwargsStorage UnsizedKwargsStorage); + +macro_rules! define_concat_sized_to_unsized { + ( + $( $other:ident )+ + ) => { + $( + impl<'py, T, U> ConcatKwargsStorages<'py, $other> for UnsizedKwargsStorage + where + UnsizedKwargsStorage: ResolveKwargs<'py>, + $other: ResolveKwargs<'py>, + { + type Output = UnsizedKwargsStorage, $other>>; + #[inline(always)] + fn concat(self, other: $other) -> Self::Output { + UnsizedKwargsStorage(ConcatStorages(self, other)) + } + } + impl<'py, T, U> ConcatKwargsStorages<'py, UnsizedKwargsStorage> for $other + where + UnsizedKwargsStorage: ResolveKwargs<'py>, + $other: ResolveKwargs<'py>, + { + type Output = UnsizedKwargsStorage, UnsizedKwargsStorage>>; + #[inline(always)] + fn concat(self, other: UnsizedKwargsStorage) -> Self::Output { + UnsizedKwargsStorage(ConcatStorages(self, other)) + } + } + )+ + }; +} +define_concat_sized_to_unsized!(ArrayKwargsStorage VecKwargsStorage); +impl<'py, A, B> ConcatKwargsStorages<'py, UnsizedKwargsStorage> for UnsizedKwargsStorage +where + UnsizedKwargsStorage: ResolveKwargs<'py>, + UnsizedKwargsStorage: ResolveKwargs<'py>, + UnsizedKwargsStorage, UnsizedKwargsStorage>>: + ResolveKwargs<'py>, +{ + type Output = + UnsizedKwargsStorage, UnsizedKwargsStorage>>; + #[inline(always)] + fn concat(self, other: UnsizedKwargsStorage) -> Self::Output { + UnsizedKwargsStorage(ConcatStorages(self, other)) + } +} diff --git a/src/pycall/kwargs/empty.rs b/src/pycall/kwargs/empty.rs new file mode 100644 index 00000000000..5b8f0561aa0 --- /dev/null +++ b/src/pycall/kwargs/empty.rs @@ -0,0 +1,42 @@ +use std::mem::MaybeUninit; + +use crate::types::{PyDict, PyTuple}; +use crate::{ffi, Borrowed, PyResult}; + +use super::{ExistingNames, PPPyObject, ResolveKwargs}; + +pub struct EmptyKwargsStorage; + +impl<'py> ResolveKwargs<'py> for EmptyKwargsStorage { + type RawStorage = MaybeUninit<()>; + type Guard = (); + #[inline(always)] + fn init( + self, + _args: PPPyObject, + _kwargs_tuple: Borrowed<'_, 'py, PyTuple>, + _index: &mut ffi::Py_ssize_t, + _existing_names: &mut ExistingNames, + ) -> PyResult { + unreachable!( + "`EmptyKwargsStorage` should never be converted into a dict or tuple, \ + rather it should pass NULL for kwargs" + ) + } + #[inline(always)] + fn len(&self) -> usize { + 0 + } + #[inline(always)] + fn write_to_dict(self, _dict: Borrowed<'_, 'py, PyDict>) -> PyResult { + unreachable!( + "`EmptyKwargsStorage` should never be converted into a dict or tuple, \ + rather it should pass NULL for kwargs" + ) + } + #[inline(always)] + fn has_known_size(&self) -> bool { + true + } + const IS_EMPTY: bool = true; +} diff --git a/src/pycall/kwargs/helpers.rs b/src/pycall/kwargs/helpers.rs new file mode 100644 index 00000000000..38ef8221eb9 --- /dev/null +++ b/src/pycall/kwargs/helpers.rs @@ -0,0 +1,181 @@ +use crate::err::error_on_minusone; +use crate::types::{PyDict, PyString}; +use crate::{ffi, Borrowed, BoundObject, PyAny, PyErr, PyResult, Python}; +use std::marker::PhantomData; + +use crate::conversion::IntoPyObject; +use crate::pycall::PPPyObject; +use crate::types::PyTuple; + +use super::{ConcatArrays, ExistingNames}; + +pub struct DropOneGuard<'py, DropTy> { + ptr: PPPyObject, + py: Python<'py>, + _marker: PhantomData, +} +impl<'py, DropTy> DropOneGuard<'py, DropTy> { + #[inline(always)] + pub(super) fn from_write( + args: PPPyObject, + kwargs_tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + existing_names: &mut ExistingNames, + name: K, + value: V, + ) -> PyResult + where + K: IntoPyObject<'py, Target = PyString>, + V: IntoPyObject<'py, Output = DropTy>, + DropTy: BoundObject<'py, V::Target>, + { + const { + assert!( + size_of::<*mut ffi::PyObject>() == size_of::() + && align_of::<*mut ffi::PyObject>() == align_of::(), + ) + } + + let py = kwargs_tuple.py(); + let name = name.into_pyobject(py).map_err(Into::into)?.into_bound(); + let value = value.into_pyobject(py).map_err(Into::into)?; + existing_names.insert( + name, + value.as_borrowed().into_any(), + args, + kwargs_tuple, + index, + )?; + Ok(Self { + ptr: args, + py, + _marker: PhantomData, + }) + } + #[inline(always)] + pub(super) fn write( + self, + kwargs_tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + existing_names: &mut ExistingNames, + name: K, + value: V, + ) -> PyResult>> + where + K: IntoPyObject<'py, Target = PyString>, + V: IntoPyObject<'py, Output = NextDropTy>, + NextDropTy: BoundObject<'py, V::Target>, + { + let py = kwargs_tuple.py(); + let name = name.into_pyobject(py).map_err(Into::into)?.into_bound(); + let value = value.into_pyobject(py).map_err(Into::into)?; + existing_names.insert( + name, + value.as_borrowed().into_any(), + self.ptr, + kwargs_tuple, + index, + )?; + Ok(DropOneGuard { + ptr: self.ptr, + py: self.py, + _marker: PhantomData, + }) + } +} +impl Drop for DropOneGuard<'_, DropTy> { + #[inline(always)] + fn drop(&mut self) { + unsafe { + self.ptr.cast::().drop_in_place(); + } + } +} + +pub struct DropManyGuard { + ptr: PPPyObject, + len: usize, + _marker: PhantomData, +} +impl DropManyGuard { + #[inline(always)] + pub(super) fn from_iter<'py, K, V>( + args: PPPyObject, + kwargs_tuple: Borrowed<'_, 'py, PyTuple>, + iter: impl IntoIterator, + index: &mut ffi::Py_ssize_t, + existing_names: &mut ExistingNames, + ) -> PyResult + where + K: IntoPyObject<'py, Target = PyString>, + V: IntoPyObject<'py, Output = DropTy>, + DropTy: BoundObject<'py, V::Target>, + { + let mut guard = Self { + ptr: args, + len: 0, + _marker: PhantomData, + }; + let py = kwargs_tuple.py(); + iter.into_iter().try_for_each(|(name, value)| { + let name = name.into_pyobject(py).map_err(Into::into)?.into_bound(); + let value = value.into_pyobject(py).map_err(Into::into)?; + existing_names.insert( + name, + value.as_borrowed().into_any(), + args, + kwargs_tuple, + index, + )?; + guard.len += 1; + Ok::<_, PyErr>(()) + })?; + Ok(guard) + } +} +impl Drop for DropManyGuard { + #[inline(always)] + fn drop(&mut self) { + unsafe { + std::ptr::slice_from_raw_parts_mut(self.ptr.cast::(), self.len).drop_in_place(); + } + } +} + +#[inline(always)] +pub(super) fn set_kwarg( + dict: Borrowed<'_, '_, PyDict>, + name: Borrowed<'_, '_, PyString>, + value: Borrowed<'_, '_, PyAny>, +) -> PyResult<()> { + unsafe { + error_on_minusone( + dict.py(), + ffi::PyDict_SetItem(dict.as_ptr(), name.as_ptr(), value.as_ptr()), + ) + } +} + +#[inline(always)] +pub(super) fn set_kwargs_from_iter<'py, K, V>( + dict: Borrowed<'_, 'py, PyDict>, + iter: impl IntoIterator, +) -> PyResult<()> +where + K: IntoPyObject<'py, Target = PyString>, + V: IntoPyObject<'py>, +{ + let py = dict.py(); + for (name, value) in iter { + set_kwarg( + dict, + name.into_pyobject(py).map_err(Into::into)?.as_borrowed(), + value + .into_pyobject(py) + .map_err(Into::into)? + .into_any() + .as_borrowed(), + )?; + } + Ok(()) +} diff --git a/src/pycall/kwargs/known.rs b/src/pycall/kwargs/known.rs new file mode 100644 index 00000000000..ebaa55f5018 --- /dev/null +++ b/src/pycall/kwargs/known.rs @@ -0,0 +1,216 @@ +use std::marker::PhantomData; +use std::mem::MaybeUninit; + +use crate::ffi_ptr_ext::FfiPtrExt; +use crate::prelude::IntoPyObject; +use crate::pycall::storage::RawStorage; +use crate::pycall::PPPyObject; +use crate::types::{PyAnyMethods, PyDict, PyString, PyTuple}; +use crate::{ffi, Borrowed, BoundObject, Py, PyResult, Python}; + +use super::{ConcatArrays, ExistingNames, ResolveKwargs}; + +#[macro_export] +#[doc(hidden)] +macro_rules! known_kwargs { + ( $( $names:literal )* ) => {{ + static KNOWN_NAMES: $crate::sync::GILOnceCell<$crate::pycall::KnownKwargsNames> = + $crate::sync::GILOnceCell::new(); + KNOWN_NAMES.get_or_init( + unsafe { $crate::Python::assume_gil_acquired() }, + || $crate::pycall::KnownKwargsNames::new(&[ $($names),* ]), + ) + }}; +} + +pub struct KnownKwargsNames(pub(in super::super) Py); + +impl KnownKwargsNames { + #[inline] + pub fn new(names: &[&'static str]) -> Self { + let tuple = unsafe { + let py = Python::assume_gil_acquired(); + let tuple = ffi::PyTuple_New(names.len() as ffi::Py_ssize_t) + .assume_owned_or_err(py) + .expect("failed to initialize tuple for kwargs") + .downcast_into_unchecked::(); + for (i, name) in names.into_iter().enumerate() { + let name = PyString::new(py, name); + ffi::PyTuple_SET_ITEM(tuple.as_ptr(), i as ffi::Py_ssize_t, name.into_ptr()); + } + tuple + }; + Self(tuple.unbind()) + } +} + +// The list is reversed! +pub trait TypeLevelPyObjectListTrait<'py> { + type RawStorage: for<'a> RawStorage = PPPyObject> + 'static; + fn add_args( + self, + py: Python<'py>, + args: PPPyObject, + index: &mut ffi::Py_ssize_t, + ) -> PyResult<()>; + fn add_to_dict( + self, + dict: Borrowed<'_, 'py, PyDict>, + names: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + ) -> PyResult<()>; + fn drop_arg(arg: &mut PPPyObject); + const LEN: usize; +} + +#[repr(C)] +pub struct TypeLevelPyObjectListCons(pub(in super::super) T, pub(in super::super) Next); + +impl<'py, T, Prev> TypeLevelPyObjectListTrait<'py> for TypeLevelPyObjectListCons +where + T: IntoPyObject<'py>, + Prev: TypeLevelPyObjectListTrait<'py>, +{ + type RawStorage = MaybeUninit>; + #[inline(always)] + fn add_args( + self, + py: Python<'py>, + args: PPPyObject, + index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + unsafe { + self.1.add_args(py, args, index)?; + args.offset(*index) + .write(self.0.into_pyobject(py).map_err(Into::into)?.into_ptr_raw()); + *index += 1; + } + Ok(()) + } + fn add_to_dict( + self, + dict: Borrowed<'_, 'py, PyDict>, + names: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + self.1.add_to_dict(dict, names, index)?; + unsafe { + let name = ffi::PyTuple_GET_ITEM(names.as_ptr(), *index); + let value = self.0.into_pyobject(dict.py()).map_err(Into::into)?; + ffi::PyDict_SetItem(dict.as_ptr(), name, value.as_borrowed().as_ptr()); + } + *index += 1; + Ok(()) + } + #[inline(always)] + fn drop_arg(arg: &mut PPPyObject) { + unsafe { + Prev::drop_arg(arg); + (*arg).cast::().drop_in_place(); + *arg = arg.add(1); + } + } + const LEN: usize = 1 + Prev::LEN; +} + +#[repr(C)] +pub struct TypeLevelPyObjectListNil; + +impl<'py> TypeLevelPyObjectListTrait<'py> for TypeLevelPyObjectListNil { + type RawStorage = MaybeUninit<()>; + #[inline(always)] + fn add_args( + self, + _py: Python<'py>, + _args: PPPyObject, + _index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + Ok(()) + } + #[inline(always)] + fn add_to_dict( + self, + _dict: Borrowed<'_, 'py, PyDict>, + _names: Borrowed<'_, 'py, PyTuple>, + _index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + Ok(()) + } + #[inline(always)] + fn drop_arg(_arg: &mut PPPyObject) {} + const LEN: usize = 0; +} + +pub struct KnownKwargsStorage<'py, Values> { + pub(in super::super) names: Borrowed<'static, 'py, PyTuple>, + pub(in super::super) values: Values, +} + +pub struct KnownKwargsGuard<'py, Values: TypeLevelPyObjectListTrait<'py>> { + args: PPPyObject, + _marker: PhantomData<(Values, &'py ())>, +} + +impl<'py, Values: TypeLevelPyObjectListTrait<'py>> Drop for KnownKwargsGuard<'py, Values> { + #[inline(always)] + fn drop(&mut self) { + Values::drop_arg(&mut { self.args }); + } +} + +impl<'py, Values: TypeLevelPyObjectListTrait<'py>> ResolveKwargs<'py> + for KnownKwargsStorage<'py, Values> +{ + type RawStorage = Values::RawStorage; + type Guard = KnownKwargsGuard<'py, Values>; + #[inline(always)] + fn init( + self, + args: PPPyObject, + kwargs_tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + existing_names: &mut ExistingNames, + ) -> PyResult { + let ptr = unsafe { args.offset(*index) }; + for i in 0..Values::LEN as ffi::Py_ssize_t { + unsafe { + let name = ffi::PyTuple_GET_ITEM(self.names.as_ptr(), i) + .assume_borrowed_unchecked(kwargs_tuple.py()) + .downcast_unchecked(); + existing_names.check_borrowed(name, kwargs_tuple, *index + i)?; + } + } + self.values.add_args(kwargs_tuple.py(), ptr, index)?; + Ok(KnownKwargsGuard { + args: ptr, + _marker: PhantomData, + }) + } + #[inline(always)] + fn len(&self) -> usize { + Values::LEN + } + #[inline(always)] + fn write_to_dict(self, dict: Borrowed<'_, 'py, PyDict>) -> PyResult { + let mut len = 0; + self.values.add_to_dict(dict, self.names, &mut len)?; + Ok(len as usize) + } + #[inline(always)] + fn has_known_size(&self) -> bool { + true + } + const IS_EMPTY: bool = false; + #[inline(always)] + fn as_names_pytuple(&self) -> Option> { + Some(self.names) + } + #[inline(always)] + fn init_no_names(self, py: Python<'py>, args: PPPyObject) -> PyResult { + self.values.add_args(py, args, &mut 0)?; + Ok(KnownKwargsGuard { + args, + _marker: PhantomData, + }) + } +} diff --git a/src/pycall/kwargs/pyobjects.rs b/src/pycall/kwargs/pyobjects.rs new file mode 100644 index 00000000000..083826576d0 --- /dev/null +++ b/src/pycall/kwargs/pyobjects.rs @@ -0,0 +1,188 @@ +use std::ffi::c_int; +use std::mem::MaybeUninit; + +use crate::err::error_on_minusone; +use crate::exceptions::{PyRuntimeError, PyTypeError}; +use crate::pycall::as_pyobject::AsPyObject; +use crate::pycall::storage::DynKnownSizeRawStorage; +use crate::types::{PyAnyMethods, PyDict, PyDictMethods, PyString, PyTuple}; +use crate::{ffi, Borrowed, Bound, BoundObject, PyResult, Python}; + +use super::{ExistingNames, PPPyObject, ResolveKwargs}; + +pub struct PyDictKwargsStorage { + value: T, + is_not_dict_subclass: bool, + len: usize, +} + +impl<'py, T: AsPyObject<'py, PyObject = PyDict>> PyDictKwargsStorage { + #[inline(always)] + pub fn new(value: T) -> Self { + let value_borrowed = value.as_borrowed(unsafe { Python::assume_gil_acquired() }); + let is_not_dict_subclass = value_borrowed.is_exact_instance_of::(); + // Do not call `PyDictMethods::len()`, as it will be incorrect for dict subclasses. + let len = PyAnyMethods::len(&**value_borrowed).unwrap_or(0); + Self { + value, + is_not_dict_subclass, + len, + } + } +} + +const DICT_MERGE_ERR_ON_DUPLICATE: c_int = 2; + +#[inline(always)] +fn copy_dict_if_needed<'py, T: AsPyObject<'py>>( + py: Python<'py>, + value: T, +) -> PyResult> { + if T::IS_OWNED && value.as_borrowed(py).into_any().get_refcnt() == 1 { + Ok(unsafe { value.into_bound(py).into_any().downcast_into_unchecked() }) + } else { + unsafe { + value + .as_borrowed(py) + .into_any() + .downcast_unchecked::() + } + .copy() + } +} + +impl<'py, T: AsPyObject<'py, PyObject = PyDict>> ResolveKwargs<'py> for PyDictKwargsStorage { + type RawStorage = DynKnownSizeRawStorage; + // Need to keep the dict around because we borrow the values from it. + type Guard = T; + #[inline(always)] + fn init( + self, + args: PPPyObject, + kwargs_tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + existing_names: &mut ExistingNames, + ) -> PyResult { + debug_assert!( + self.is_not_dict_subclass, + "dict subclasses have no known size", + ); + let py = kwargs_tuple.py(); + let dict = self.value.as_borrowed(py); + unsafe { + let mut pos = 0; + let mut key = std::ptr::null_mut(); + let mut value = std::ptr::null_mut(); + let mut len = self.len as isize; + let di_used = len; + while ffi::PyDict_Next(dict.as_ptr(), &mut pos, &mut key, &mut value) != 0 { + let ma_used = dict.len() as isize; + + if di_used != ma_used || len == -1 { + return Err(PyRuntimeError::new_err( + intern!(py, "dictionary changed during iteration") + .clone() + .unbind(), + )); + }; + len -= 1; + + let key = Borrowed::from_ptr_unchecked(py, key) + .downcast::() + .map_err(|err| { + let new_err = PyTypeError::new_err( + intern!(py, "keywords must be strings").clone().unbind(), + ); + new_err.set_cause(py, Some(err.into())); + new_err + })? + .to_owned(); + let value = Borrowed::from_ptr_unchecked(py, value); + existing_names.insert(key, value, args, kwargs_tuple, index)?; + } + } + Ok(self.value) + } + #[inline(always)] + fn len(&self) -> usize { + self.len + } + #[inline(always)] + fn write_to_dict(self, dict: Borrowed<'_, 'py, PyDict>) -> PyResult { + unsafe { + error_on_minusone( + dict.py(), + ffi::PyDict_Merge( + dict.as_ptr(), + self.value.as_borrowed(dict.py()).as_ptr(), + DICT_MERGE_ERR_ON_DUPLICATE, + ), + )?; + } + Ok(self.len) + } + #[inline(always)] + fn has_known_size(&self) -> bool { + self.is_not_dict_subclass + } + #[inline(always)] + fn can_be_cheaply_converted_to_pydict(&self, _py: Python<'py>) -> bool { + T::IS_OWNED && self.is_not_dict_subclass + } + #[inline(always)] + fn into_pydict(self, py: Python<'py>) -> PyResult> { + copy_dict_if_needed(py, self.value) + } + const IS_EMPTY: bool = false; +} + +pub struct AnyPyMapping(pub(super) T); + +impl<'py, T: AsPyObject<'py>> ResolveKwargs<'py> for AnyPyMapping { + type RawStorage = MaybeUninit<()>; + type Guard = std::convert::Infallible; + #[inline(always)] + fn init( + self, + _args: PPPyObject, + _kwargs_tuple: Borrowed<'_, 'py, PyTuple>, + _index: &mut ffi::Py_ssize_t, + _existing_names: &mut ExistingNames, + ) -> PyResult { + panic!("Python classes have no known size") + } + #[inline(always)] + fn len(&self) -> usize { + 0 + } + #[inline(always)] + fn write_to_dict(self, dict: Borrowed<'_, 'py, PyDict>) -> PyResult { + unsafe { + let value = self.0.as_borrowed(dict.py()); + let len = value.into_any().len()?; + error_on_minusone( + dict.py(), + ffi::PyDict_Merge(dict.as_ptr(), value.as_ptr(), DICT_MERGE_ERR_ON_DUPLICATE), + )?; + Ok(len) + } + } + #[inline(always)] + fn has_known_size(&self) -> bool { + false + } + #[inline(always)] + fn can_be_cheaply_converted_to_pydict(&self, py: Python<'py>) -> bool { + T::IS_OWNED + && self + .0 + .as_borrowed(py) + .into_any() + .is_exact_instance_of::() + } + #[inline(always)] + fn into_pydict(self, py: Python<'py>) -> PyResult> { + copy_dict_if_needed(py, self.0) + } + const IS_EMPTY: bool = false; +} diff --git a/src/pycall/kwargs/selector.rs b/src/pycall/kwargs/selector.rs new file mode 100644 index 00000000000..013b2c1a9d8 --- /dev/null +++ b/src/pycall/kwargs/selector.rs @@ -0,0 +1,345 @@ +//! This module is responsible, given an unpacked argument, to find the best storage for it. +//! We do that using [autoderef specialization]. We rank each storage implementation, +//! and use the first that can be used. +//! +//! Here are all implementations, ordered by their rank, from the best to the worst: +//! +//! 1. Empty argument set (`()`, `[T; 0]`, `&[T; 0]` and `&mut [T; 0]`). +//! 2. Arrays (`[T; N]`, `&[T; N]` and `&mut [T; N]`). +//! 3. `TrustedLen` iterators. +//! 4. `ExactSizeIterator`s. +//! 5. Any `IntoIterator`. +//! 6. `PyDict`. +//! 7. Any Python object. +//! 8. Tuples. +//! +//! They are divided to four groups: +//! +//! - The empty argument is before everything, since it is cancelled by everything, +//! and so its effect on performance is zero. That means it is a better match than anything else. +//! It also enables calling with NULL kwargs. +//! - Stack allocated arrays come next. That includes arrays and tuples. +//! - `TrustedLen`, `ExactSizeIterator` and any iterator have to come in this order specifically +//! since each latter one is a superset of the former, but has a less efficient implementation. +//! - Likewise for `PyDict` and any Python object. `PyDict` can be used as-is for calls and is `TrustedLen`, +//! while other Python mappings are equivalent to any Rust iterator. +//! - Tuples come last not because they are less efficient (in fact they are equivalent to arrays), +//! but because it is more convenient to put them in a "catch-all" bound instead of having to +//! enumerate each tuple type using a macro again. It doesn't matter for performance since +//! nothing else can match tuples. +//! +//! [autoderef specialization]: https://lukaskalbertodt.github.io/2019/12/05/generalized-autoref-based-specialization.html + +use std::marker::PhantomData; + +pub mod select_traits { + pub use super::any_iterator::AnyIteratorSelector as _; + pub use super::any_pymapping::AnyPyMappingSelector as _; + pub use super::array::ArraySelector as _; + pub use super::empty::EmptySelector as _; + pub use super::exact_size::ExactSizeIteratorSelector as _; + pub use super::pydict::PyDictSelector as _; + pub use super::trusted_len::TrustedLenSelector as _; + pub use super::tuple::TupleSelector as _; +} + +pub struct KwargsStorageSelector(PhantomData); + +impl KwargsStorageSelector { + /// This is called by the macro like the following: + /// + /// ```ignore + /// KwargsStorageSelector::new(loop { + /// break None; + /// break Some(value); + /// }) + /// ``` + /// + /// This way, the compiler infers the correct type, but no code is actually executed. + /// + /// Note that `if false` cannot be used instead, as it can cause borrow checker errors. + /// The borrow checker understands this construct as unreachable, and so won't complain. + #[inline(always)] + pub fn new(_: Option) -> Self { + Self(PhantomData) + } +} + +mod empty { + use crate::conversion::IntoPyObject; + use crate::pycall::kwargs::concat::FundamentalStorage; + use crate::pycall::kwargs::empty::EmptyKwargsStorage; + use crate::types::PyString; + + use super::KwargsStorageSelector; + + pub trait EmptySelector<'py, T> { + type Output: FundamentalStorage<'py>; + fn __py_unpack_kwargs_select(self, value: T) -> Self::Output; + } + + impl<'py> EmptySelector<'py, ()> for &&&&&&&&&&&KwargsStorageSelector<()> { + type Output = EmptyKwargsStorage; + #[inline(always)] + fn __py_unpack_kwargs_select(self, _value: ()) -> EmptyKwargsStorage { + EmptyKwargsStorage + } + } + impl<'py, K, V> EmptySelector<'py, [(K, V); 0]> for &&&&&&&&&&&KwargsStorageSelector<[(K, V); 0]> + where + K: IntoPyObject<'py, Target = PyString>, + V: IntoPyObject<'py>, + { + type Output = EmptyKwargsStorage; + #[inline(always)] + fn __py_unpack_kwargs_select(self, _value: [(K, V); 0]) -> EmptyKwargsStorage { + EmptyKwargsStorage + } + } + impl<'py, 'a, K, V> EmptySelector<'py, &'a [(K, V); 0]> + for &&&&&&&&&&&KwargsStorageSelector<&'a [(K, V); 0]> + where + &'a K: IntoPyObject<'py, Target = PyString>, + &'a V: IntoPyObject<'py>, + { + type Output = EmptyKwargsStorage; + #[inline(always)] + fn __py_unpack_kwargs_select(self, _value: &'a [(K, V); 0]) -> EmptyKwargsStorage { + EmptyKwargsStorage + } + } + impl<'py, 'a, K, V> EmptySelector<'py, &'a mut [(K, V); 0]> + for &&&&&&&&&&&KwargsStorageSelector<&'a mut [(K, V); 0]> + where + &'a K: IntoPyObject<'py, Target = PyString>, + &'a V: IntoPyObject<'py>, + { + type Output = EmptyKwargsStorage; + #[inline(always)] + fn __py_unpack_kwargs_select(self, _value: &'a mut [(K, V); 0]) -> EmptyKwargsStorage { + EmptyKwargsStorage + } + } +} + +mod array { + use crate::conversion::IntoPyObject; + use crate::pycall::kwargs::array::ArrayKwargsStorage; + use crate::pycall::kwargs::concat::FundamentalStorage; + use crate::types::PyString; + + use super::KwargsStorageSelector; + + pub trait ArraySelector<'py, T> { + type Output: FundamentalStorage<'py>; + fn __py_unpack_kwargs_select(self, value: T) -> Self::Output; + } + + impl<'py, K, V, const N: usize> ArraySelector<'py, [(K, V); N]> + for &&&&&&&&&&KwargsStorageSelector<[(K, V); N]> + where + K: IntoPyObject<'py, Target = PyString>, + V: IntoPyObject<'py>, + { + type Output = ArrayKwargsStorage<[(K, V); N]>; + #[inline(always)] + fn __py_unpack_kwargs_select(self, value: [(K, V); N]) -> ArrayKwargsStorage<[(K, V); N]> { + ArrayKwargsStorage(value) + } + } + + impl<'a, 'py, K, V, const N: usize> ArraySelector<'py, &'a [(K, V); N]> + for &&&&&&&&&&KwargsStorageSelector<&'a [(K, V); N]> + where + &'a K: IntoPyObject<'py, Target = PyString>, + &'a V: IntoPyObject<'py>, + { + type Output = ArrayKwargsStorage<&'a [(K, V); N]>; + #[inline(always)] + fn __py_unpack_kwargs_select( + self, + value: &'a [(K, V); N], + ) -> ArrayKwargsStorage<&'a [(K, V); N]> { + ArrayKwargsStorage(value) + } + } + + impl<'a, 'py, K, V, const N: usize> ArraySelector<'py, &'a mut [(K, V); N]> + for &&&&&&&&&&KwargsStorageSelector<&'a mut [(K, V); N]> + where + &'a K: IntoPyObject<'py, Target = PyString>, + &'a V: IntoPyObject<'py>, + { + type Output = ArrayKwargsStorage<&'a mut [(K, V); N]>; + #[inline(always)] + fn __py_unpack_kwargs_select( + self, + value: &'a mut [(K, V); N], + ) -> ArrayKwargsStorage<&'a mut [(K, V); N]> { + ArrayKwargsStorage(value) + } + } +} + +mod trusted_len { + use crate::conversion::IntoPyObject; + use crate::pycall::kwargs::concat::FundamentalStorage; + use crate::pycall::kwargs::vec::{TrustedLenIterator, VecKwargsStorage}; + use crate::pycall::trusted_len::TrustedLen; + use crate::types::PyString; + + use super::KwargsStorageSelector; + + pub trait TrustedLenSelector<'py, T> { + type Output: FundamentalStorage<'py>; + fn __py_unpack_kwargs_select(self, value: T) -> Self::Output; + } + + impl<'py, T, K, V> TrustedLenSelector<'py, T> for &&&&&&&&KwargsStorageSelector + where + T: IntoIterator, + T::IntoIter: TrustedLen, + K: IntoPyObject<'py, Target = PyString>, + V: IntoPyObject<'py>, + { + type Output = VecKwargsStorage>; + #[inline(always)] + fn __py_unpack_kwargs_select( + self, + value: T, + ) -> VecKwargsStorage> { + VecKwargsStorage(TrustedLenIterator(value.into_iter())) + } + } +} + +mod exact_size { + use crate::conversion::IntoPyObject; + use crate::pycall::kwargs::concat::FundamentalStorage; + use crate::pycall::kwargs::vec::{ExactSizeIterator, VecKwargsStorage}; + use crate::types::PyString; + + use super::KwargsStorageSelector; + + pub trait ExactSizeIteratorSelector<'py, T> { + type Output: FundamentalStorage<'py>; + fn __py_unpack_kwargs_select(self, value: T) -> Self::Output; + } + + impl<'py, T, K, V> ExactSizeIteratorSelector<'py, T> for &&&&&&&KwargsStorageSelector + where + T: IntoIterator, + T::IntoIter: std::iter::ExactSizeIterator, + K: IntoPyObject<'py, Target = PyString>, + V: IntoPyObject<'py>, + { + type Output = VecKwargsStorage>; + #[inline(always)] + fn __py_unpack_kwargs_select( + self, + value: T, + ) -> VecKwargsStorage> { + VecKwargsStorage(ExactSizeIterator::new(value.into_iter())) + } + } +} + +mod any_iterator { + use crate::conversion::IntoPyObject; + use crate::pycall::kwargs::concat::FundamentalStorage; + use crate::pycall::kwargs::unknown_size::{AnyIteratorKwargs, UnsizedKwargsStorage}; + use crate::types::PyString; + + use super::KwargsStorageSelector; + + pub trait AnyIteratorSelector<'py, T> { + type Output: FundamentalStorage<'py>; + fn __py_unpack_kwargs_select(self, value: T) -> Self::Output; + } + + impl<'py, T, K, V> AnyIteratorSelector<'py, T> for &&&&&&KwargsStorageSelector + where + T: IntoIterator, + K: IntoPyObject<'py, Target = PyString>, + V: IntoPyObject<'py>, + { + type Output = UnsizedKwargsStorage>; + #[inline(always)] + fn __py_unpack_kwargs_select( + self, + value: T, + ) -> UnsizedKwargsStorage> { + UnsizedKwargsStorage(AnyIteratorKwargs(value.into_iter())) + } + } +} + +mod pydict { + use crate::pycall::as_pyobject::AsPyObject; + use crate::pycall::kwargs::concat::FundamentalStorage; + use crate::pycall::kwargs::pyobjects::PyDictKwargsStorage; + use crate::pycall::kwargs::unknown_size::UnsizedKwargsStorage; + use crate::types::PyDict; + + use super::KwargsStorageSelector; + + pub trait PyDictSelector<'py, T> { + type Output: FundamentalStorage<'py>; + fn __py_unpack_kwargs_select(self, value: T) -> Self::Output; + } + + impl<'py, T: AsPyObject<'py, PyObject = PyDict>> PyDictSelector<'py, T> + for &&&&&KwargsStorageSelector + { + type Output = UnsizedKwargsStorage>; + #[inline(always)] + fn __py_unpack_kwargs_select( + self, + value: T, + ) -> UnsizedKwargsStorage> { + UnsizedKwargsStorage(PyDictKwargsStorage::new(value)) + } + } +} + +mod any_pymapping { + use crate::pycall::as_pyobject::AsPyObject; + use crate::pycall::kwargs::concat::FundamentalStorage; + use crate::pycall::kwargs::pyobjects::AnyPyMapping; + use crate::pycall::kwargs::unknown_size::UnsizedKwargsStorage; + + use super::KwargsStorageSelector; + + pub trait AnyPyMappingSelector<'py, T> { + type Output: FundamentalStorage<'py>; + fn __py_unpack_kwargs_select(self, value: T) -> Self::Output; + } + + impl<'py, T: AsPyObject<'py>> AnyPyMappingSelector<'py, T> for &&&KwargsStorageSelector { + type Output = UnsizedKwargsStorage>; + #[inline(always)] + fn __py_unpack_kwargs_select(self, value: T) -> UnsizedKwargsStorage> { + UnsizedKwargsStorage(AnyPyMapping(value)) + } + } +} + +mod tuple { + use crate::pycall::kwargs::array::{ArrayKwargsStorage, Tuple}; + use crate::pycall::kwargs::concat::FundamentalStorage; + + use super::KwargsStorageSelector; + + pub trait TupleSelector<'py, T> { + type Output: FundamentalStorage<'py>; + fn __py_unpack_kwargs_select(self, value: T) -> Self::Output; + } + + impl<'py, T: Tuple<'py>> TupleSelector<'py, T> for &&KwargsStorageSelector { + type Output = ArrayKwargsStorage; + #[inline(always)] + fn __py_unpack_kwargs_select(self, value: T) -> ArrayKwargsStorage { + ArrayKwargsStorage(value) + } + } +} diff --git a/src/pycall/kwargs/unknown_size.rs b/src/pycall/kwargs/unknown_size.rs new file mode 100644 index 00000000000..065bbb12d0a --- /dev/null +++ b/src/pycall/kwargs/unknown_size.rs @@ -0,0 +1,127 @@ +use std::mem::MaybeUninit; + +use crate::conversion::IntoPyObject; +use crate::pycall::storage::DynKnownSizeRawStorage; +use crate::types::{PyDict, PyString, PyTuple}; +use crate::{ffi, Borrowed, Bound, PyResult, Python}; + +use super::helpers::set_kwargs_from_iter; +use super::{ConcatStorages, ExistingNames, PPPyObject, ResolveKwargs}; + +pub struct UnsizedKwargsStorage(pub(super) T); + +impl<'py, T: ResolveKwargs<'py>> ResolveKwargs<'py> for UnsizedKwargsStorage { + type RawStorage = T::RawStorage; + type Guard = T::Guard; + #[inline(always)] + fn init( + self, + args: PPPyObject, + kwargs_tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + existing_names: &mut ExistingNames, + ) -> PyResult { + self.0.init(args, kwargs_tuple, index, existing_names) + } + #[inline(always)] + fn len(&self) -> usize { + self.0.len() + } + #[inline(always)] + fn write_to_dict(self, dict: Borrowed<'_, 'py, PyDict>) -> PyResult { + self.0.write_to_dict(dict) + } + #[inline(always)] + fn has_known_size(&self) -> bool { + self.0.has_known_size() + } + const IS_EMPTY: bool = T::IS_EMPTY; + #[inline(always)] + fn as_names_pytuple(&self) -> Option> { + self.0.as_names_pytuple() + } + #[inline(always)] + fn can_be_cheaply_converted_to_pydict(&self, py: Python<'py>) -> bool { + self.0.can_be_cheaply_converted_to_pydict(py) + } + #[inline(always)] + fn init_no_names(self, py: Python<'py>, args: PPPyObject) -> PyResult { + self.0.init_no_names(py, args) + } + #[inline(always)] + fn into_pydict(self, py: Python<'py>) -> PyResult> { + self.0.into_pydict(py) + } +} + +impl<'py, A, B> ResolveKwargs<'py> for UnsizedKwargsStorage> +where + A: ResolveKwargs<'py>, + B: ResolveKwargs<'py>, +{ + type RawStorage = DynKnownSizeRawStorage; + type Guard = (A::Guard, B::Guard); + #[inline(always)] + fn init( + self, + args: PPPyObject, + kwargs_tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + existing_names: &mut ExistingNames, + ) -> PyResult { + let g1 = self.0 .0.init(args, kwargs_tuple, index, existing_names)?; + let g2 = self.0 .1.init(args, kwargs_tuple, index, existing_names)?; + Ok((g1, g2)) + } + #[inline(always)] + fn len(&self) -> usize { + self.0 .0.len() + self.0 .1.len() + } + #[inline(always)] + fn write_to_dict(self, dict: Borrowed<'_, 'py, PyDict>) -> PyResult { + self.0 .0.write_to_dict(dict)?; + self.0 .1.write_to_dict(dict) + } + #[inline(always)] + fn has_known_size(&self) -> bool { + self.0 .0.has_known_size() && self.0 .1.has_known_size() + } + const IS_EMPTY: bool = A::IS_EMPTY && B::IS_EMPTY; +} + +pub struct AnyIteratorKwargs(pub(super) I); + +impl<'py, I, K, V> ResolveKwargs<'py> for AnyIteratorKwargs +where + I: Iterator, + K: IntoPyObject<'py, Target = PyString>, + V: IntoPyObject<'py>, +{ + type RawStorage = MaybeUninit<()>; + type Guard = std::convert::Infallible; + #[inline(always)] + fn init( + self, + _args: PPPyObject, + _kwargs_tuple: Borrowed<'_, 'py, PyTuple>, + _index: &mut ffi::Py_ssize_t, + _existing_names: &mut ExistingNames, + ) -> PyResult { + panic!("Any iterator doesn't have a known size") + } + #[inline(always)] + fn len(&self) -> usize { + self.0.size_hint().0 + } + #[inline(always)] + fn write_to_dict(self, dict: Borrowed<'_, 'py, PyDict>) -> PyResult { + let mut len = 0; + set_kwargs_from_iter(dict, self.0.inspect(|_| len += 1))?; + Ok(len) + } + #[inline(always)] + fn has_known_size(&self) -> bool { + false + } + const IS_EMPTY: bool = false; +} diff --git a/src/pycall/kwargs/vec.rs b/src/pycall/kwargs/vec.rs new file mode 100644 index 00000000000..e4855400e51 --- /dev/null +++ b/src/pycall/kwargs/vec.rs @@ -0,0 +1,198 @@ +use crate::conversion::IntoPyObject; +use crate::pycall::storage::DynKnownSizeRawStorage; +use crate::pycall::trusted_len::TrustedLen; +use crate::types::{PyDict, PyString, PyTuple}; +use crate::{ffi, Borrowed, Bound, PyResult, Python}; + +use super::helpers::{set_kwargs_from_iter, DropManyGuard}; +use super::{ConcatStorages, ExistingNames, PPPyObject, ResolveKwargs}; + +pub struct VecKwargsStorage(pub(super) T); + +impl<'py, T: ResolveKwargs<'py>> ResolveKwargs<'py> for VecKwargsStorage { + type RawStorage = T::RawStorage; + type Guard = T::Guard; + #[inline(always)] + fn init( + self, + args: PPPyObject, + kwargs_tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + existing_names: &mut ExistingNames, + ) -> PyResult { + self.0.init(args, kwargs_tuple, index, existing_names) + } + #[inline(always)] + fn len(&self) -> usize { + self.0.len() + } + #[inline(always)] + fn write_to_dict(self, dict: Borrowed<'_, 'py, PyDict>) -> PyResult { + self.0.write_to_dict(dict) + } + #[inline(always)] + fn has_known_size(&self) -> bool { + self.0.has_known_size() + } + const IS_EMPTY: bool = T::IS_EMPTY; + #[inline(always)] + fn as_names_pytuple(&self) -> Option> { + self.0.as_names_pytuple() + } + #[inline(always)] + fn can_be_cheaply_converted_to_pydict(&self, py: Python<'py>) -> bool { + self.0.can_be_cheaply_converted_to_pydict(py) + } + #[inline(always)] + fn init_no_names(self, py: Python<'py>, args: PPPyObject) -> PyResult { + self.0.init_no_names(py, args) + } + #[inline(always)] + fn into_pydict(self, py: Python<'py>) -> PyResult> { + self.0.into_pydict(py) + } +} + +impl<'py, A, B> ResolveKwargs<'py> for VecKwargsStorage> +where + A: ResolveKwargs<'py>, + B: ResolveKwargs<'py>, +{ + type RawStorage = DynKnownSizeRawStorage; + type Guard = (A::Guard, B::Guard); + #[inline(always)] + fn init( + self, + args: PPPyObject, + kwargs_tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + existing_names: &mut ExistingNames, + ) -> PyResult { + let g1 = self.0 .0.init(args, kwargs_tuple, index, existing_names)?; + let g2 = self.0 .1.init(args, kwargs_tuple, index, existing_names)?; + Ok((g1, g2)) + } + #[inline(always)] + fn len(&self) -> usize { + self.0 .0.len() + self.0 .1.len() + } + #[inline(always)] + fn write_to_dict(self, dict: Borrowed<'_, 'py, PyDict>) -> PyResult { + let len1 = self.0 .0.write_to_dict(dict)?; + let len2 = self.0 .1.write_to_dict(dict)?; + Ok(len1 + len2) + } + #[inline(always)] + fn has_known_size(&self) -> bool { + self.0 .0.has_known_size() && self.0 .1.has_known_size() + } + const IS_EMPTY: bool = A::IS_EMPTY && B::IS_EMPTY; +} + +pub struct TrustedLenIterator(pub(super) I); + +impl<'py, I, K, V> ResolveKwargs<'py> for TrustedLenIterator +where + I: TrustedLen, + K: IntoPyObject<'py, Target = PyString>, + V: IntoPyObject<'py>, +{ + type RawStorage = DynKnownSizeRawStorage; + type Guard = DropManyGuard; + #[inline(always)] + fn init( + self, + args: PPPyObject, + kwargs_tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + existing_names: &mut ExistingNames, + ) -> PyResult { + DropManyGuard::from_iter(args, kwargs_tuple, self.0, index, existing_names) + } + #[inline(always)] + fn len(&self) -> usize { + self.0.size_hint().0 + } + #[inline(always)] + fn write_to_dict(self, dict: Borrowed<'_, 'py, PyDict>) -> PyResult { + let len = self.len(); + set_kwargs_from_iter(dict, self.0)?; + Ok(len) + } + #[inline(always)] + fn has_known_size(&self) -> bool { + true + } + const IS_EMPTY: bool = false; +} + +pub struct ExactSizeIterator { + iter: I, + // We cannot rely on the iterator providing the same `len()` every time (this will lead to unsoundness), + // so we save it here. + len: usize, +} + +impl ExactSizeIterator { + #[inline(always)] + pub(super) fn new(iter: I) -> Self { + Self { + len: iter.len(), + iter, + } + } +} + +impl<'py, I, K, V> ResolveKwargs<'py> for ExactSizeIterator +where + I: std::iter::ExactSizeIterator, + K: IntoPyObject<'py, Target = PyString>, + V: IntoPyObject<'py>, +{ + type RawStorage = DynKnownSizeRawStorage; + type Guard = DropManyGuard; + #[inline(always)] + fn init( + self, + args: PPPyObject, + kwargs_tuple: Borrowed<'_, 'py, PyTuple>, + index: &mut ffi::Py_ssize_t, + existing_names: &mut ExistingNames, + ) -> PyResult { + let mut i = 0; + let guard = DropManyGuard::from_iter( + args, + kwargs_tuple, + self.iter.inspect(|_| { + i += 1; + if i > self.len { + panic!("an ExactSizeIterator produced more items than it declared"); + } + }), + index, + existing_names, + ); + if i != self.len { + panic!("an ExactSizeIterator produced less items than it declared"); + } + guard + } + #[inline(always)] + fn len(&self) -> usize { + self.len + } + #[inline(always)] + fn write_to_dict(self, dict: Borrowed<'_, 'py, PyDict>) -> PyResult { + // No need to check for violations of `ExactSizeIterator`, dict can grow as needed. + set_kwargs_from_iter(dict, self.iter)?; + // FIXME: Figure out if it's a problem if an iterator will yield different numbers + // of items than `len` (probably not, the check will be messed up and may fail + // succeed wrongly, but they broke the contract of `ExactSizeIterator` so this is fine). + Ok(self.len) + } + #[inline(always)] + fn has_known_size(&self) -> bool { + true + } + const IS_EMPTY: bool = false; +} diff --git a/src/pycall/kwargs_args_adapter.rs b/src/pycall/kwargs_args_adapter.rs new file mode 100644 index 00000000000..e1b0c3bbbf2 --- /dev/null +++ b/src/pycall/kwargs_args_adapter.rs @@ -0,0 +1,308 @@ +use crate::types::PyTuple; +use crate::{ffi, Borrowed, PyResult, Python}; + +use super::args::{self, ArgumentsOffsetFlag, ConcatStorages, ResolveArgs}; +use super::kwargs::{self, ExistingNames, ResolveKwargs}; +use super::PPPyObject; + +pub struct KwargsArgsAdapter<'a, 'py, Kwargs> { + pub(super) kwargs: Kwargs, + pub(super) kwargs_tuple: Borrowed<'a, 'py, PyTuple>, +} + +impl<'py, Kwargs: ResolveKwargs<'py>> ResolveArgs<'py> for KwargsArgsAdapter<'_, 'py, Kwargs> { + type RawStorage = Kwargs::RawStorage; + type Guard = Kwargs::Guard; + fn init( + self, + _py: Python<'py>, + storage: PPPyObject, + _base_storage: *const PPPyObject, + ) -> PyResult { + let len = self.kwargs.len(); + self.kwargs.init( + storage, + self.kwargs_tuple, + &mut 0, + &mut ExistingNames::new(len), + ) + } + fn len(&self) -> usize { + self.kwargs.len() + } + fn write_to_tuple( + self, + _tuple: Borrowed<'_, 'py, PyTuple>, + _index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + unreachable!("kwargs-args adapters are only used for vectorcall") + } + fn write_initialized_to_tuple( + _tuple: Borrowed<'_, 'py, PyTuple>, + _guard: Self::Guard, + _raw_storage: &mut PPPyObject, + _index: &mut ffi::Py_ssize_t, + ) { + unreachable!("kwargs-args adapters are only used for vectorcall") + } + fn has_known_size(&self) -> bool { + self.kwargs.has_known_size() + } + const ARGUMENTS_OFFSET: ArgumentsOffsetFlag = ArgumentsOffsetFlag::Normal; + const IS_EMPTY: bool = Kwargs::IS_EMPTY; + const IS_ONE: bool = false; + const USE_STACK_FOR_SMALL_LEN: bool = true; +} + +pub struct KwargsArgsNoNamesAdapter(Kwargs); + +impl<'py, Kwargs: ResolveKwargs<'py>> ResolveArgs<'py> for KwargsArgsNoNamesAdapter { + type RawStorage = Kwargs::RawStorage; + type Guard = Kwargs::Guard; + fn init( + self, + py: Python<'py>, + storage: PPPyObject, + _base_storage: *const PPPyObject, + ) -> PyResult { + self.0.init_no_names(py, storage) + } + fn len(&self) -> usize { + self.0.len() + } + fn write_to_tuple( + self, + _tuple: Borrowed<'_, 'py, PyTuple>, + _index: &mut ffi::Py_ssize_t, + ) -> PyResult<()> { + unreachable!("kwargs-args adapters are only used for vectorcall") + } + fn write_initialized_to_tuple( + _tuple: Borrowed<'_, 'py, PyTuple>, + _guard: Self::Guard, + _raw_storage: &mut PPPyObject, + _index: &mut ffi::Py_ssize_t, + ) { + unreachable!("kwargs-args adapters are only used for vectorcall") + } + fn has_known_size(&self) -> bool { + self.0.has_known_size() + } + const ARGUMENTS_OFFSET: ArgumentsOffsetFlag = ArgumentsOffsetFlag::Normal; + const IS_EMPTY: bool = Kwargs::IS_EMPTY; + const IS_ONE: bool = false; + const USE_STACK_FOR_SMALL_LEN: bool = true; +} + +// We have 5 fundamental args storages (`AppendEmptyArgForVectorcall` isn't counted because +// it is always converted to `ArrayArgsStorage` before combined), and 5 fundamental kwargs +// storages. That means 5*5 = 25 combinations. + +pub trait CombineArgsKwargs<'a, 'py, Kwargs> +where + Self: args::FundamentalStorage<'py>, + Kwargs: kwargs::FundamentalStorage<'py>, +{ + type Output: args::FundamentalStorage<'py>; + fn combine(self, kwargs: KwargsArgsAdapter<'a, 'py, Kwargs>) -> Self::Output; + type OutputNoNames: args::FundamentalStorage<'py>; + fn combine_no_names(self, kwargs: Kwargs) -> Self::OutputNoNames; +} + +macro_rules! define_combine { + ( + $( + $args:ident + (( $($py:lifetime)? ) $($b:ident)?) $kwargs:ident = $result:ident + )+ + ) => { + $( + impl<'a, 'py, A, $($b)?> CombineArgsKwargs<'a, 'py, kwargs::$kwargs<$($py,)? $($b)?>> for args::$args + where + args::$args: args::ResolveArgs<'py>, + kwargs::$kwargs<$($py,)? $($b)?>: kwargs::FundamentalStorage<'py>, + args::$result, args::$args>>>>: args::ResolveArgs<'py>, + args::$result, args::$args>>>>: args::ResolveArgs<'py>, + { + type Output = args::$result, args::$args>>>>; + #[inline(always)] + fn combine(self, kwargs: KwargsArgsAdapter<'a, 'py, kwargs::$kwargs<$($py,)? $($b)?>>) -> Self::Output { + args::$result(ConcatStorages(self, args::$args(kwargs))) + } + type OutputNoNames = args::$result, args::$args>>>>; + #[inline(always)] + fn combine_no_names(self, kwargs: kwargs::$kwargs<$($py,)? $($b)?>) -> Self::OutputNoNames { + args::$result(ConcatStorages(self, args::$args(KwargsArgsNoNamesAdapter(kwargs)))) + } + } + )+ + }; +} +define_combine!( + ArrayArgsStorage + (() B) ArrayKwargsStorage = ArrayArgsStorage + VecArgsStorage + (() B) ArrayKwargsStorage = VecArgsStorage + ArrayArgsStorage + (() B) VecKwargsStorage = VecArgsStorage + ArrayArgsStorage + (('py) B) KnownKwargsStorage = ArrayArgsStorage + VecArgsStorage + (('py) B) KnownKwargsStorage = VecArgsStorage + VecArgsStorage + (() B) VecKwargsStorage = VecArgsStorage + // The following will never be used, since we check for empty kwargs and pass NULL. + // But they need to be here to please the compiler. + ArrayArgsStorage + (()) EmptyKwargsStorage = ArrayArgsStorage + VecArgsStorage + (()) EmptyKwargsStorage = ArrayArgsStorage + // The following will never really be unsized used, since we check for unknown size kwargs + // and use normal (tuple and dict) calling convention. But they can be dynamically sized + // if `PyDict`. + ArrayArgsStorage + (() B) UnsizedKwargsStorage = VecArgsStorage + VecArgsStorage + (() B) UnsizedKwargsStorage = VecArgsStorage +); + +macro_rules! define_combine_empty_args { + ( + $( + ( $($py:lifetime)? ) $kwargs:ident = $result:ident + )+ + ) => { + $( + impl<'a, 'py, T> CombineArgsKwargs<'a, 'py, kwargs::$kwargs<$($py,)? T>> for args::EmptyArgsStorage + where + kwargs::$kwargs<$($py,)? T>: kwargs::FundamentalStorage<'py>, + args::$result>>: args::ResolveArgs<'py>, + args::$result>>: args::ResolveArgs<'py>, + { + type Output = args::$result>>; + #[inline(always)] + fn combine(self, kwargs: KwargsArgsAdapter<'a, 'py, kwargs::$kwargs<$($py,)? T>>) -> Self::Output { + args::$result(kwargs) + } + type OutputNoNames = args::$result>>; + #[inline(always)] + fn combine_no_names(self, kwargs: kwargs::$kwargs<$($py,)? T>) -> Self::OutputNoNames { + args::$result(KwargsArgsNoNamesAdapter(kwargs)) + } + } + )+ + }; +} +define_combine_empty_args!( + () ArrayKwargsStorage = ArrayArgsStorage + ('py) KnownKwargsStorage = ArrayArgsStorage + () VecKwargsStorage = VecArgsStorage + // The following will never be used, since we check for unknown size kwargs and use normal + // (tuple and dict) calling convention. But it needs to be here to please the compiler. + () UnsizedKwargsStorage = VecArgsStorage +); +// The following will never be used, since we check for empty kwargs and pass NULL. +// But it needs to be here to please the compiler. +impl<'a, 'py> CombineArgsKwargs<'a, 'py, kwargs::EmptyKwargsStorage> for args::EmptyArgsStorage { + type Output = args::EmptyArgsStorage; + #[inline(always)] + fn combine( + self, + _kwargs: KwargsArgsAdapter<'a, 'py, kwargs::EmptyKwargsStorage>, + ) -> Self::Output { + args::EmptyArgsStorage + } + type OutputNoNames = args::EmptyArgsStorage; + #[inline(always)] + fn combine_no_names(self, _kwargs: kwargs::EmptyKwargsStorage) -> Self::OutputNoNames { + args::EmptyArgsStorage + } +} + +macro_rules! define_combine_sized_to_unsized { + ( + $( + $args:ident + (( $($py:lifetime)? ) $($b:ident)?) $kwargs:ident + )+ + ) => { + $( + impl<'a, 'py, A, $($b)?> CombineArgsKwargs<'a, 'py, kwargs::$kwargs<$($py,)? $($b)?>> for args::$args + where + args::$args: args::ResolveArgs<'py>, + kwargs::$kwargs<$($py,)? $($b)?>: kwargs::FundamentalStorage<'py>, + args::UnsizedArgsStorage, + args::UnsizedArgsStorage>>>, + >>: args::ResolveArgs<'py>, + args::UnsizedArgsStorage, + args::UnsizedArgsStorage>>>, + >>: args::ResolveArgs<'py>, + { + type Output = args::UnsizedArgsStorage, + args::UnsizedArgsStorage>>>, + >>; + #[inline(always)] + fn combine(self, kwargs: KwargsArgsAdapter<'a, 'py, kwargs::$kwargs<$($py,)? $($b)?>>) -> Self::Output { + args::UnsizedArgsStorage(ConcatStorages(self, args::UnsizedArgsStorage(args::SizedToUnsizedStorage(kwargs)))) + } + type OutputNoNames = args::UnsizedArgsStorage, + args::UnsizedArgsStorage>>>, + >>; + #[inline(always)] + fn combine_no_names(self, kwargs: kwargs::$kwargs<$($py,)? $($b)?>) -> Self::OutputNoNames { + args::UnsizedArgsStorage(ConcatStorages(self, args::UnsizedArgsStorage(args::SizedToUnsizedStorage(KwargsArgsNoNamesAdapter(kwargs))))) + } + } + )+ + }; +} +define_combine_sized_to_unsized!( + UnsizedArgsStorage + (() B) ArrayKwargsStorage + UnsizedArgsStorage + (() B) VecKwargsStorage + UnsizedArgsStorage + (('py) B) KnownKwargsStorage + // The following will never be used, since we check for unknown size kwargs and use normal + // (tuple and dict) calling convention. But it needs to be here to please the compiler. + UnsizedArgsStorage + (() B) UnsizedKwargsStorage + // The following will never be used, since we check for empty kwargs and pass NULL. + // But it needs to be here to please the compiler. + UnsizedArgsStorage + (()) EmptyKwargsStorage +); + +macro_rules! define_combine_existing { + ( + $( + (( $($py:lifetime)? ) $($t:ident)?) $kwargs:ident + )+ + ) => { + $( + impl<'a, 'py, S, $($t)?> CombineArgsKwargs<'a, 'py, kwargs::$kwargs<$($py,)? $($t)?>> for args::ExistingArgListSlice + where + S: args::ExistingArgListSliceTrait, + kwargs::$kwargs<$($py,)? $($t)?>: kwargs::FundamentalStorage<'py>, + args::VecArgsStorage>: CombineArgsKwargs<'a, 'py, kwargs::$kwargs<$($py,)? $($t)?>>, + { + type Output = < + args::VecArgsStorage> + as CombineArgsKwargs<'a, 'py, kwargs::$kwargs<$($py,)? $($t)?>> + >::Output; + #[inline(always)] + fn combine(self, kwargs: KwargsArgsAdapter<'a, 'py, kwargs::$kwargs<$($py,)? $($t)?>>) -> Self::Output { + < + args::VecArgsStorage> + as CombineArgsKwargs<'a, 'py, kwargs::$kwargs<$($py,)? $($t)?>> + >::combine(args::VecArgsStorage(self), kwargs) + } + type OutputNoNames = < + args::VecArgsStorage> + as CombineArgsKwargs<'a, 'py, kwargs::$kwargs<$($py,)? $($t)?>> + >::OutputNoNames; + #[inline(always)] + fn combine_no_names(self, kwargs: kwargs::$kwargs<$($py,)? $($t)?>) -> Self::OutputNoNames { + < + args::VecArgsStorage> + as CombineArgsKwargs<'a, 'py, kwargs::$kwargs<$($py,)? $($t)?>> + >::combine_no_names(args::VecArgsStorage(self), kwargs) + } + } + )+ + }; +} +define_combine_existing!( + (() T) ArrayKwargsStorage + (() T) VecKwargsStorage + (() T) UnsizedKwargsStorage + (('py) T) KnownKwargsStorage + (()) EmptyKwargsStorage +); diff --git a/src/pycall/storage.rs b/src/pycall/storage.rs new file mode 100644 index 00000000000..708c82b0257 --- /dev/null +++ b/src/pycall/storage.rs @@ -0,0 +1,117 @@ +use std::alloc::{handle_alloc_error, Layout}; +use std::mem::MaybeUninit; + +use crate::ffi; + +use super::PPPyObject; + +pub trait RawStorage: Sized { + type InitParam<'a> + where + Self: 'a; + fn new(len: usize) -> Self; + fn as_init_param(&mut self) -> Self::InitParam<'_>; + fn as_ptr(&mut self) -> PPPyObject; + fn len(&self) -> usize; + fn init_param_from_ptr<'a>(ptr: PPPyObject) -> Self::InitParam<'a>; +} + +impl RawStorage for MaybeUninit { + type InitParam<'a> = PPPyObject; + #[inline(always)] + fn new(_len: usize) -> Self { + MaybeUninit::uninit() + } + #[inline(always)] + fn as_init_param(&mut self) -> PPPyObject { + self.as_mut_ptr().cast::<*mut ffi::PyObject>() + } + #[inline(always)] + fn as_ptr(&mut self) -> PPPyObject { + (self as *mut Self).cast::<*mut ffi::PyObject>() + } + #[inline(always)] + fn len(&self) -> usize { + size_of::() / size_of::<*mut ffi::PyObject>() + } + #[inline(always)] + fn init_param_from_ptr<'a>(ptr: PPPyObject) -> Self::InitParam<'a> { + ptr + } +} + +pub struct DynKnownSizeRawStorage { + ptr: PPPyObject, + len: usize, +} + +impl Drop for DynKnownSizeRawStorage { + #[inline] + fn drop(&mut self) { + unsafe { + std::alloc::dealloc( + self.ptr.cast::(), + Layout::array::<*mut ffi::PyObject>(self.len).unwrap_unchecked(), + ); + } + } +} + +impl RawStorage for DynKnownSizeRawStorage { + type InitParam<'a> = PPPyObject; + #[inline] + fn new(len: usize) -> Self { + unsafe { + let layout = + Layout::array::<*mut ffi::PyObject>(len).expect("too much memory requested"); + let ptr = std::alloc::alloc(layout).cast::<*mut ffi::PyObject>(); + if ptr.is_null() { + handle_alloc_error(layout); + } + Self { ptr, len } + } + } + #[inline(always)] + fn as_init_param(&mut self) -> PPPyObject { + self.ptr + } + #[inline(always)] + fn as_ptr(&mut self) -> PPPyObject { + self.ptr + } + #[inline(always)] + fn len(&self) -> usize { + self.len + } + #[inline(always)] + fn init_param_from_ptr<'a>(ptr: PPPyObject) -> Self::InitParam<'a> { + ptr + } +} + +pub(super) type UnsizedStorage = Vec<*mut ffi::PyObject>; +pub(super) type UnsizedInitParam<'a> = &'a mut Vec<*mut ffi::PyObject>; + +impl RawStorage for UnsizedStorage { + type InitParam<'a> = UnsizedInitParam<'a>; + #[inline] + fn new(len: usize) -> Self { + Vec::with_capacity(len) + } + #[inline(always)] + fn as_init_param(&mut self) -> UnsizedInitParam<'_> { + self + } + #[inline(always)] + fn as_ptr(&mut self) -> PPPyObject { + self.as_mut_ptr() + } + #[inline(always)] + fn len(&self) -> usize { + self.len() + } + #[inline(always)] + fn init_param_from_ptr<'a>(_ptr: PPPyObject) -> Self::InitParam<'a> { + unreachable!("UnsizedStorage does not use small stack optimization") + } +} diff --git a/src/pycall/trusted_len.rs b/src/pycall/trusted_len.rs new file mode 100644 index 00000000000..db590cc3401 --- /dev/null +++ b/src/pycall/trusted_len.rs @@ -0,0 +1,108 @@ +// Copied from the standard library: https://doc.rust-lang.org/stable/std/iter/trait.TrustedLen.html. + +pub(super) unsafe trait TrustedLen: Iterator {} + +unsafe impl TrustedLen for std::char::ToLowercase {} +unsafe impl TrustedLen for std::char::ToUppercase {} +unsafe impl TrustedLen for std::str::Bytes<'_> {} +unsafe impl TrustedLen for std::slice::Chunks<'_, T> {} +unsafe impl TrustedLen for std::slice::ChunksMut<'_, T> {} +unsafe impl TrustedLen for std::slice::ChunksExact<'_, T> {} +unsafe impl TrustedLen for std::slice::ChunksExactMut<'_, T> {} +unsafe impl TrustedLen for std::slice::RChunks<'_, T> {} +unsafe impl TrustedLen for std::slice::RChunksMut<'_, T> {} +unsafe impl TrustedLen for std::slice::RChunksExact<'_, T> {} +unsafe impl TrustedLen for std::slice::RChunksExactMut<'_, T> {} +unsafe impl TrustedLen for std::slice::Windows<'_, T> {} +unsafe impl TrustedLen for std::iter::Empty {} +unsafe impl TrustedLen for std::iter::Once {} +unsafe impl TrustedLen for std::option::IntoIter {} +unsafe impl TrustedLen for std::option::Iter<'_, T> {} +unsafe impl TrustedLen for std::option::IterMut<'_, T> {} +unsafe impl TrustedLen for std::result::IntoIter {} +unsafe impl TrustedLen for std::result::Iter<'_, T> {} +unsafe impl TrustedLen for std::result::IterMut<'_, T> {} +unsafe impl TrustedLen for std::collections::vec_deque::IntoIter {} +unsafe impl TrustedLen for std::collections::vec_deque::Iter<'_, T> {} +unsafe impl TrustedLen for std::slice::Iter<'_, T> {} +unsafe impl TrustedLen for std::slice::IterMut<'_, T> {} +unsafe impl TrustedLen for std::vec::Drain<'_, T> {} +unsafe impl TrustedLen for std::vec::IntoIter {} +unsafe impl TrustedLen for std::array::IntoIter {} +unsafe impl TrustedLen for std::ops::Range +where + T: TrustedStep, + std::ops::Range: Iterator, +{ +} +unsafe impl TrustedLen for std::ops::RangeFrom +where + T: TrustedStep, + std::ops::RangeFrom: Iterator, +{ +} +unsafe impl TrustedLen for std::ops::RangeInclusive +where + T: TrustedStep, + std::ops::RangeInclusive: Iterator, +{ +} + +unsafe impl<'a, I, T> TrustedLen for std::iter::Cloned +where + T: Clone + 'a, + I: TrustedLen, +{ +} +unsafe impl<'a, I, T> TrustedLen for std::iter::Copied +where + T: Copy + 'a, + I: TrustedLen, +{ +} +unsafe impl TrustedLen for std::iter::Repeat where T: Clone {} +unsafe impl TrustedLen for std::iter::Chain +where + A: TrustedLen, + B: TrustedLen, +{ +} +unsafe impl TrustedLen for std::iter::Zip +where + A: TrustedLen, + B: TrustedLen, +{ +} +unsafe impl TrustedLen for std::iter::OnceWith where F: FnOnce() -> A {} +unsafe impl TrustedLen for std::iter::RepeatWith where F: FnMut() -> A {} +unsafe impl TrustedLen for std::iter::Map +where + I: TrustedLen, + F: FnMut(I::Item) -> B, +{ +} +unsafe impl TrustedLen for std::iter::Enumerate where I: TrustedLen {} +unsafe impl TrustedLen for std::iter::Fuse where I: TrustedLen {} +unsafe impl TrustedLen for std::iter::Peekable where I: TrustedLen {} +unsafe impl TrustedLen for std::iter::Rev where I: TrustedLen + DoubleEndedIterator {} +unsafe impl TrustedLen for std::iter::Take where I: TrustedLen {} + +unsafe impl TrustedLen for &mut I where I: TrustedLen + ?Sized {} + +unsafe trait TrustedStep {} + +unsafe impl TrustedStep for char {} +unsafe impl TrustedStep for i8 {} +unsafe impl TrustedStep for i16 {} +unsafe impl TrustedStep for i32 {} +unsafe impl TrustedStep for i64 {} +unsafe impl TrustedStep for i128 {} +unsafe impl TrustedStep for isize {} +unsafe impl TrustedStep for u8 {} +unsafe impl TrustedStep for u16 {} +unsafe impl TrustedStep for u32 {} +unsafe impl TrustedStep for u64 {} +unsafe impl TrustedStep for u128 {} +unsafe impl TrustedStep for usize {} +unsafe impl TrustedStep for std::net::Ipv4Addr {} +unsafe impl TrustedStep for std::net::Ipv6Addr {}