Skip to content

Commit

Permalink
Minor: remove uncessary #cfg test (#8036)
Browse files Browse the repository at this point in the history 
* Minor: remove uncessary #cfg test

* fmt

* Update datafusion/core/src/datasource/file_format/arrow.rs

Co-authored-by: Ruihang Xia <[email protected]>

---------

Co-authored-by: Daniël Heres <[email protected]>
Co-authored-by: Ruihang Xia <[email protected]>
  • Loading branch information
@alamb @Dandandan @waynexia
3 people authored Nov 3, 2023
1 parent b2a1668 commit 8c42d94
Show file tree
Hide file tree
Showing 2 changed files with 120 additions and 132 deletions.
60 changes: 27 additions & 33 deletions datafusion/core/tests/fuzz_cases/aggregate_fuzz.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,39 +35,33 @@ use datafusion_physical_expr::expressions::{col, Sum};
use datafusion_physical_expr::{AggregateExpr, PhysicalSortExpr};
use test_utils::add_empty_batches;

#[cfg(test)]
#[allow(clippy::items_after_test_module)]
mod tests {
use super::*;

#[tokio::test(flavor = "multi_thread", worker_threads = 8)]
async fn aggregate_test() {
let test_cases = vec![
vec!["a"],
vec!["b", "a"],
vec!["c", "a"],
vec!["c", "b", "a"],
vec!["d", "a"],
vec!["d", "b", "a"],
vec!["d", "c", "a"],
vec!["d", "c", "b", "a"],
];
let n = 300;
let distincts = vec![10, 20];
for distinct in distincts {
let mut handles = Vec::new();
for i in 0..n {
let test_idx = i % test_cases.len();
let group_by_columns = test_cases[test_idx].clone();
let job = tokio::spawn(run_aggregate_test(
make_staggered_batches::<true>(1000, distinct, i as u64),
group_by_columns,
));
handles.push(job);
}
for job in handles {
job.await.unwrap();
}
#[tokio::test(flavor = "multi_thread", worker_threads = 8)]
async fn aggregate_test() {
let test_cases = vec![
vec!["a"],
vec!["b", "a"],
vec!["c", "a"],
vec!["c", "b", "a"],
vec!["d", "a"],
vec!["d", "b", "a"],
vec!["d", "c", "a"],
vec!["d", "c", "b", "a"],
];
let n = 300;
let distincts = vec![10, 20];
for distinct in distincts {
let mut handles = Vec::new();
for i in 0..n {
let test_idx = i % test_cases.len();
let group_by_columns = test_cases[test_idx].clone();
let job = tokio::spawn(run_aggregate_test(
make_staggered_batches::<true>(1000, distinct, i as u64),
group_by_columns,
));
handles.push(job);
}
for job in handles {
job.await.unwrap();
}
}
}
Expand Down
192 changes: 93 additions & 99 deletions datafusion/core/tests/fuzz_cases/window_fuzz.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,108 +44,102 @@ use hashbrown::HashMap;
use rand::rngs::StdRng;
use rand::{Rng, SeedableRng};

#[cfg(test)]
#[allow(clippy::items_after_test_module)]
mod tests {
use super::*;

use datafusion_physical_plan::windows::PartitionSearchMode::{
Linear, PartiallySorted, Sorted,
};
use datafusion_physical_plan::windows::PartitionSearchMode::{
Linear, PartiallySorted, Sorted,
};

#[tokio::test(flavor = "multi_thread", worker_threads = 16)]
async fn window_bounded_window_random_comparison() -> Result<()> {
// make_staggered_batches gives result sorted according to a, b, c
// In the test cases first entry represents partition by columns
// Second entry represents order by columns.
// Third entry represents search mode.
// In sorted mode physical plans are in the form for WindowAggExec
//```
// WindowAggExec
// MemoryExec]
// ```
// and in the form for BoundedWindowAggExec
// ```
// BoundedWindowAggExec
// MemoryExec
// ```
// In Linear and PartiallySorted mode physical plans are in the form for WindowAggExec
//```
// WindowAggExec
// SortExec(required by window function)
// MemoryExec]
// ```
// and in the form for BoundedWindowAggExec
// ```
// BoundedWindowAggExec
// MemoryExec
// ```
let test_cases = vec![
(vec!["a"], vec!["a"], Sorted),
(vec!["a"], vec!["b"], Sorted),
(vec!["a"], vec!["a", "b"], Sorted),
(vec!["a"], vec!["b", "c"], Sorted),
(vec!["a"], vec!["a", "b", "c"], Sorted),
(vec!["b"], vec!["a"], Linear),
(vec!["b"], vec!["a", "b"], Linear),
(vec!["b"], vec!["a", "c"], Linear),
(vec!["b"], vec!["a", "b", "c"], Linear),
(vec!["c"], vec!["a"], Linear),
(vec!["c"], vec!["a", "b"], Linear),
(vec!["c"], vec!["a", "c"], Linear),
(vec!["c"], vec!["a", "b", "c"], Linear),
(vec!["b", "a"], vec!["a"], Sorted),
(vec!["b", "a"], vec!["b"], Sorted),
(vec!["b", "a"], vec!["c"], Sorted),
(vec!["b", "a"], vec!["a", "b"], Sorted),
(vec!["b", "a"], vec!["b", "c"], Sorted),
(vec!["b", "a"], vec!["a", "c"], Sorted),
(vec!["b", "a"], vec!["a", "b", "c"], Sorted),
(vec!["c", "b"], vec!["a"], Linear),
(vec!["c", "b"], vec!["a", "b"], Linear),
(vec!["c", "b"], vec!["a", "c"], Linear),
(vec!["c", "b"], vec!["a", "b", "c"], Linear),
(vec!["c", "a"], vec!["a"], PartiallySorted(vec![1])),
(vec!["c", "a"], vec!["b"], PartiallySorted(vec![1])),
(vec!["c", "a"], vec!["c"], PartiallySorted(vec![1])),
(vec!["c", "a"], vec!["a", "b"], PartiallySorted(vec![1])),
(vec!["c", "a"], vec!["b", "c"], PartiallySorted(vec![1])),
(vec!["c", "a"], vec!["a", "c"], PartiallySorted(vec![1])),
(
vec!["c", "a"],
vec!["a", "b", "c"],
PartiallySorted(vec![1]),
),
(vec!["c", "b", "a"], vec!["a"], Sorted),
(vec!["c", "b", "a"], vec!["b"], Sorted),
(vec!["c", "b", "a"], vec!["c"], Sorted),
(vec!["c", "b", "a"], vec!["a", "b"], Sorted),
(vec!["c", "b", "a"], vec!["b", "c"], Sorted),
(vec!["c", "b", "a"], vec!["a", "c"], Sorted),
(vec!["c", "b", "a"], vec!["a", "b", "c"], Sorted),
];
let n = 300;
let n_distincts = vec![10, 20];
for n_distinct in n_distincts {
let mut handles = Vec::new();
for i in 0..n {
let idx = i % test_cases.len();
let (pb_cols, ob_cols, search_mode) = test_cases[idx].clone();
let job = tokio::spawn(run_window_test(
make_staggered_batches::<true>(1000, n_distinct, i as u64),
i as u64,
pb_cols,
ob_cols,
search_mode,
));
handles.push(job);
}
for job in handles {
job.await.unwrap()?;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 16)]
async fn window_bounded_window_random_comparison() -> Result<()> {
// make_staggered_batches gives result sorted according to a, b, c
// In the test cases first entry represents partition by columns
// Second entry represents order by columns.
// Third entry represents search mode.
// In sorted mode physical plans are in the form for WindowAggExec
//```
// WindowAggExec
// MemoryExec]
// ```
// and in the form for BoundedWindowAggExec
// ```
// BoundedWindowAggExec
// MemoryExec
// ```
// In Linear and PartiallySorted mode physical plans are in the form for WindowAggExec
//```
// WindowAggExec
// SortExec(required by window function)
// MemoryExec]
// ```
// and in the form for BoundedWindowAggExec
// ```
// BoundedWindowAggExec
// MemoryExec
// ```
let test_cases = vec![
(vec!["a"], vec!["a"], Sorted),
(vec!["a"], vec!["b"], Sorted),
(vec!["a"], vec!["a", "b"], Sorted),
(vec!["a"], vec!["b", "c"], Sorted),
(vec!["a"], vec!["a", "b", "c"], Sorted),
(vec!["b"], vec!["a"], Linear),
(vec!["b"], vec!["a", "b"], Linear),
(vec!["b"], vec!["a", "c"], Linear),
(vec!["b"], vec!["a", "b", "c"], Linear),
(vec!["c"], vec!["a"], Linear),
(vec!["c"], vec!["a", "b"], Linear),
(vec!["c"], vec!["a", "c"], Linear),
(vec!["c"], vec!["a", "b", "c"], Linear),
(vec!["b", "a"], vec!["a"], Sorted),
(vec!["b", "a"], vec!["b"], Sorted),
(vec!["b", "a"], vec!["c"], Sorted),
(vec!["b", "a"], vec!["a", "b"], Sorted),
(vec!["b", "a"], vec!["b", "c"], Sorted),
(vec!["b", "a"], vec!["a", "c"], Sorted),
(vec!["b", "a"], vec!["a", "b", "c"], Sorted),
(vec!["c", "b"], vec!["a"], Linear),
(vec!["c", "b"], vec!["a", "b"], Linear),
(vec!["c", "b"], vec!["a", "c"], Linear),
(vec!["c", "b"], vec!["a", "b", "c"], Linear),
(vec!["c", "a"], vec!["a"], PartiallySorted(vec![1])),
(vec!["c", "a"], vec!["b"], PartiallySorted(vec![1])),
(vec!["c", "a"], vec!["c"], PartiallySorted(vec![1])),
(vec!["c", "a"], vec!["a", "b"], PartiallySorted(vec![1])),
(vec!["c", "a"], vec!["b", "c"], PartiallySorted(vec![1])),
(vec!["c", "a"], vec!["a", "c"], PartiallySorted(vec![1])),
(
vec!["c", "a"],
vec!["a", "b", "c"],
PartiallySorted(vec![1]),
),
(vec!["c", "b", "a"], vec!["a"], Sorted),
(vec!["c", "b", "a"], vec!["b"], Sorted),
(vec!["c", "b", "a"], vec!["c"], Sorted),
(vec!["c", "b", "a"], vec!["a", "b"], Sorted),
(vec!["c", "b", "a"], vec!["b", "c"], Sorted),
(vec!["c", "b", "a"], vec!["a", "c"], Sorted),
(vec!["c", "b", "a"], vec!["a", "b", "c"], Sorted),
];
let n = 300;
let n_distincts = vec![10, 20];
for n_distinct in n_distincts {
let mut handles = Vec::new();
for i in 0..n {
let idx = i % test_cases.len();
let (pb_cols, ob_cols, search_mode) = test_cases[idx].clone();
let job = tokio::spawn(run_window_test(
make_staggered_batches::<true>(1000, n_distinct, i as u64),
i as u64,
pb_cols,
ob_cols,
search_mode,
));
handles.push(job);
}
for job in handles {
job.await.unwrap()?;
}
Ok(())
}
Ok(())
}

fn get_random_function(
Expand Down

0 comments on commit 8c42d94

Please sign in to comment.