Merge pull request ClickHouse#56636 from ClickHouse/collect-binary-size

Collect information about binary sizes in the CI database
amosbird · Nov 13, 2023 · b1d8c98 · b1d8c98
2 parents 00b414f + 6e910f0
commit b1d8c98
Show file tree

Hide file tree

Showing 3 changed files with 67 additions and 10 deletions.
diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp
@@ -6,9 +6,7 @@
 #include <Common/HashTable/Hash.h>
 #include <Common/RadixSort.h>
 
-#include <base/unaligned.h>
 #include <base/sort.h>
-#include <base/scope_guard.h>
 
 #include <IO/WriteHelpers.h>
 
@@ -20,8 +18,6 @@
 #include <Processors/Transforms/ColumnGathererTransform.h>
 
 
-template <typename T> bool decimalLess(T x, T y, UInt32 x_scale, UInt32 y_scale);
-
 namespace DB
 {
 

diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py
@@ -433,9 +433,10 @@ def main():
         )
         profile_data_file = temp_path / "profile.json"
         with open(profile_data_file, "wb") as profile_fd:
-            for profile_sourse in profiles_dir.iterdir():
-                with open(profiles_dir / profile_sourse, "rb") as ps_fd:
-                    profile_fd.write(ps_fd.read())
+            for profile_source in profiles_dir.iterdir():
+                if profile_source.name != "binary_sizes.txt":
+                    with open(profiles_dir / profile_source, "rb") as ps_fd:
+                        profile_fd.write(ps_fd.read())
 
         logging.info(
             "::notice ::Log Uploading profile data, path: %s, size: %s, query: %s",
@@ -445,6 +446,32 @@ def main():
         )
         ch_helper.insert_file(url, auth, query, profile_data_file)
 
+        query = f"""INSERT INTO binary_sizes
+(
+    pull_request_number,
+    commit_sha,
+    check_start_time,
+    check_name,
+    instance_type,
+    instance_id,
+    file,
+    size
+)
+SELECT {pr_info.number}, '{pr_info.sha}', '{stopwatch.start_time_str}', '{build_name}', '{instance_type}', '{instance_id}', file, size
+FROM input('size UInt64, file String')
+SETTINGS format_regexp = '^\\s*(\\d+) (.+)$'
+FORMAT Regexp"""
+
+        binary_sizes_file = profiles_dir / "binary_sizes.txt"
+
+        logging.info(
+            "::notice ::Log Uploading binary sizes data, path: %s, size: %s, query: %s",
+            binary_sizes_file,
+            binary_sizes_file.stat().st_size,
+            query,
+        )
+        ch_helper.insert_file(url, auth, query, binary_sizes_file)
+
     # Upload statistics to CI database
     prepared_events = prepare_tests_results_for_clickhouse(
         pr_info,

diff --git a/utils/prepare-time-trace/prepare-time-trace.sh b/utils/prepare-time-trace/prepare-time-trace.sh
@@ -11,12 +11,19 @@
 <<///
 CREATE TABLE build_time_trace
 (
-    -- extra columns here
+    -- Extra columns:
+    pull_request_number UInt32,
+    commit_sha String,
+    check_start_time DateTime,
+    check_name LowCardinality(String),
+    instance_type LowCardinality(String),
+    instance_id String,
+
+    -- Normal columns:
     file String,
     library LowCardinality(String),
     date Date DEFAULT toDate(time),
     time DateTime64(6),
-
     pid UInt32,
     tid UInt32,
     ph Enum8('B', 'E', 'X', 'i', 'I', 'C', 'b', 'n', 'e', 'S', 'T', 'p', 'F', 's', 't', 'f', 'P', 'N', 'O', 'D', 'M'),
@@ -30,7 +37,8 @@ CREATE TABLE build_time_trace
     args_name LowCardinality(String),
     is_total Bool DEFAULT name LIKE 'Total %'
 )
-ENGINE = MergeTree ORDER BY (date, file, name, args_name);
+ENGINE = MergeTree
+ORDER BY (date, file, name, args_name);
 ///
 
 INPUT_DIR=$1
@@ -48,3 +56,29 @@ find "$INPUT_DIR" -name '*.json' | grep -P '\.(c|cpp|cc|cxx)\.json$' | xargs -P
 # Now you can upload it as follows:
 
 #cat "$OUTPUT_DIR"/* | clickhouse-client --progress --query "INSERT INTO build_time_trace (extra_column_names, file, library, time, pid, tid, ph, ts, dur, cat, name, detail, count, avgMs, args_name) FORMAT JSONCompactEachRow"
+
+# Additionally, collect information about the sizes of translation units
+
+<<///
+CREATE TABLE binary_sizes
+(
+    -- Extra columns:
+    pull_request_number UInt32,
+    commit_sha String,
+    check_start_time DateTime,
+    check_name LowCardinality(String),
+    instance_type LowCardinality(String),
+    instance_id String,
+
+    -- Normal columns:
+    file LowCardinality(String),
+    library LowCardinality(String) DEFAULT extract(file, 'CMakeFiles/([^/]+)\.dir/'),
+    size UInt64,
+    date Date DEFAULT toDate(time),
+    time DateTime64(6) DEFAULT now64()
+)
+ENGINE = MergeTree
+ORDER BY (date, file, pull_request_number, commit_sha, check_name);
+///
+
+find "$INPUT_DIR" -type f -executable -or -name '*.o' -or -name '*.a' | grep -v cargo | xargs wc -c | grep -v 'total' > "${OUTPUT_DIR}/binary_sizes.txt"