Add sqllogictests (v0)

.github/workflows/rust.yml

@@ -517,3 +517,31 @@ jobs:
           # If you encounter an error, run './dev/update_config_docs.sh' and commit
           ./dev/update_config_docs.sh
           git diff --exit-code
+
+  # Run sqllogictests
+  sql-logic-tests:
+    name: run sqllogictests
+    needs: [linux-build-lib]
+    runs-on: ubuntu-latest
+    container:
+      image: amd64/rust
+      env:
+        # Disable full debug symbol generation to speed up CI build and keep memory down
+        # "1" means line tables only, which is useful for panic tracebacks.
+        RUSTFLAGS: "-C debuginfo=1"
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+      - name: Cache Cargo
+        uses: actions/cache@v3
+        with:
+          path: /github/home/.cargo
+          # this key equals the ones on `linux-build-lib` for re-use
+          key: cargo-cache-
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup-builder
+        with:
+          rust-version: stable
+      - name: Run sqllogictests
+        run: cargo run -p datafusion-sqllogictests

Cargo.toml

@@ -31,6 +31,7 @@ members = [
     "test-utils",
     "parquet-test-utils",
     "benchmarks",
+    "tests/sqllogictests",
 ]
 
 [profile.release]

tests/sqllogictests/.gitignore

@@ -0,0 +1 @@
+*.py

tests/sqllogictests/Cargo.toml

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "datafusion-sqllogictests"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https:#doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+async-trait = "0.1.58"
+datafusion = { path = "../../datafusion/core" }
+sqllogictest = "0.8.0"
+tokio = { version = "1.0", features = ["macros", "rt-multi-thread"] }

tests/sqllogictests/README.md

@@ -0,0 +1,45 @@
+#### Overview
+
+This is the Datafusion implementation of [sqllogictest](https://www.sqlite.org/sqllogictest/doc/trunk/about.wiki). We use [sqllogictest-rs](https://github.com/risinglightdb/sqllogictest-rs) as a parser/runner of `.slt` files in `test_files`.
+
+#### Running tests
+
+`cargo run -p datafusion-sqllogictests`
+
+#### Setup
+
+
+#### sqllogictests
+
+> :warning: **Warning**:Datafusion's sqllogictest implementation and migration is still in progress. Definitions taken from https://www.sqlite.org/sqllogictest/doc/trunk/about.wiki
+
+sqllogictest is a program originally written for SQLite to verify the correctness of SQL queries against the SQLLite engine. The program is engine-agnostic and can parse sqllogictest files (`.slt`), runs queries against an SQL engine and compare the output to the expected output.
+
+Tests in the `.slt` file are a sequence of query record generally starting with `CREATE` statements to populate tables and then further queries to test the populated data (arrow-datafusion exception).
+
+Query records follow the format:
+```sql
+# <test_name>
+query <type_string> <sort_mode> <label>
+<sql_query>
+----
+<expected_result>
+```
+
+- `test_name`: Uniquely identify the test name (arrow-datafusion only)
+- `type_string`: A short string that specifies the number of result columns and the expected datatype of each result column. There is one character in the <type_string> for each result column. The characters codes are "T" for a text result, "I" for an integer result, and "R" for a floating-point result.
+- (Optional) `label`: sqllogictest stores a hash of the results of this query under the given label. If the label is reused, then sqllogictest verifies that the results are the same. This can be used to verify that two or more queries in the same test script that are logically equivalent always generate the same output. 
+- `expected_result`: In the results section, integer values are rendered as if by printf("%d"). Floating point values are rendered as if by printf("%.3f"). NULL values are rendered as "NULL". Empty strings are rendered as "(empty)". Within non-empty strings, all control characters and unprintable characters are rendered as "@". 
+
+##### Example
+
+```sql
+# group_by_distinct
+query TTI
+SELECT a, b, COUNT(DISTINCT c) FROM my_table GROUP BY a, b ORDER BY a, b
+----
+foo bar 10
+foo baz 5
+foo     4
+        3
+```

tests/sqllogictests/src/main.rs

@@ -0,0 +1,121 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use async_trait::async_trait;
+use datafusion::arrow::csv::WriterBuilder;
+use datafusion::arrow::record_batch::RecordBatch;
+use datafusion::prelude::SessionContext;
+use std::path::PathBuf;
+
+use sqllogictest::TestError;
+pub type Result<T> = std::result::Result<T, TestError>;
+
+mod setup;
+mod utils;
+
+const TEST_DIRECTORY: &str = "tests/sqllogictests/test_files";
+const TEST_CATEGORIES: [TestCategory; 2] =
+    [TestCategory::Aggregate, TestCategory::ArrowTypeOf];
+
+pub enum TestCategory {
+    Aggregate,
+    ArrowTypeOf,
+}
+
+impl TestCategory {
+    fn as_str(&self) -> &'static str {
+        match self {
+            TestCategory::Aggregate => "Aggregate",
+            TestCategory::ArrowTypeOf => "ArrowTypeOf",
+        }
+    }
+
+    fn test_filename(&self) -> &'static str {
+        match self {
+            TestCategory::Aggregate => "aggregate.slt",
+            TestCategory::ArrowTypeOf => "arrow_typeof.slt",
+        }
+    }
+
+    async fn register_test_tables(&self, ctx: &SessionContext) {
+        println!("[{}] Registering tables", self.as_str());
+        match self {
+            TestCategory::Aggregate => setup::register_aggregate_tables(ctx).await,
+            TestCategory::ArrowTypeOf => (),
+        }
+    }
+}
+
+pub struct DataFusion {
+    ctx: SessionContext,
+    test_category: TestCategory,
+}
+
+#[async_trait]
+impl sqllogictest::AsyncDB for DataFusion {
+    type Error = TestError;
+
+    async fn run(&mut self, sql: &str) -> Result<String> {
+        println!(
+            "[{}] Running query: \"{}\"",
+            self.test_category.as_str(),
+            sql
+        );
+        let result = run_query(&self.ctx, sql).await?;
+        Ok(result)
+    }
+}
+
+#[tokio::main]
+pub async fn main() -> Result<()> {
+    for test_category in TEST_CATEGORIES {
+        let filename = PathBuf::from(format!(
+            "{}/{}",
+            TEST_DIRECTORY,
+            test_category.test_filename()
+        ));
+        let ctx = SessionContext::new();
+        test_category.register_test_tables(&ctx).await;
+
+        let mut tester = sqllogictest::Runner::new(DataFusion { ctx, test_category });
+        // TODO: use tester.run_parallel_async()
+        tester.run_file_async(filename).await.unwrap();
+    }
+
+    Ok(())
+}
+
+fn format_batches(batches: &[RecordBatch]) -> Result<String> {
+    let mut bytes = vec![];
+    {
+        let builder = WriterBuilder::new().has_headers(false).with_delimiter(b',');
+        let mut writer = builder.build(&mut bytes);
+        for batch in batches {
+            writer.write(batch).unwrap();
+        }
+    }
+
+    let formatted = String::from_utf8(bytes).unwrap().replace(',', " ");
+    Ok(formatted)
+}
+
+async fn run_query(ctx: &SessionContext, sql: impl Into<String>) -> Result<String> {
+    let df = ctx.sql(&sql.into()).await.unwrap();
+    let results: Vec<RecordBatch> = df.collect().await.unwrap();
+    let formatted_batches = format_batches(&results)?;
+    Ok(formatted_batches)
+}