Skip to content

Commit

Permalink
...
Browse files Browse the repository at this point in the history
  • Loading branch information
mbasmanova committed Oct 19, 2023
1 parent 2edd972 commit f544393
Show file tree
Hide file tree
Showing 8 changed files with 463 additions and 22 deletions.
4 changes: 4 additions & 0 deletions velox/core/PlanNode.h
Original file line number Diff line number Diff line change
Expand Up @@ -2260,6 +2260,10 @@ class TopNRowNumberNode : public PlanNode {
return outputType_;
}

bool canSpill(const QueryConfig& queryConfig) const override {
return !partitionKeys_.empty() && queryConfig.topNRowNumberSpillEnabled();
}

const RowTypePtr& inputType() const {
return sources_[0]->outputType();
}
Expand Down
14 changes: 12 additions & 2 deletions velox/core/QueryConfig.h
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,10 @@ class QueryConfig {
static constexpr const char* kRowNumberSpillEnabled =
"row_number_spill_enabled";

/// TopNRowNumber spilling flag, only applies if "spill_enabled" flag is set.
static constexpr const char* kTopNRowNumberSpillEnabled =
"topn_row_number_spill_enabled";

/// The max memory that a final aggregation can use before spilling. If it 0,
/// then there is no limit.
static constexpr const char* kAggregationSpillMemoryThreshold =
Expand Down Expand Up @@ -469,12 +473,18 @@ class QueryConfig {
return get<bool>(kOrderBySpillEnabled, true);
}

/// Returns 'is row_number spilling enabled' flag. Must also check the
/// spillEnabled()!
/// Returns true if spilling is enabled for RowNumber operator. Must also
/// check the spillEnabled()!
bool rowNumberSpillEnabled() const {
return get<bool>(kRowNumberSpillEnabled, true);
}

/// Returns true if spilling is enabled for TopNRowNumber operator. Must also
/// check the spillEnabled()!
bool topNRowNumberSpillEnabled() const {
return get<bool>(kTopNRowNumberSpillEnabled, true);
}

// Returns a percentage of aggregation or join input batches that
// will be forced to spill for testing. 0 means no extra spilling.
int32_t testingSpillPct() const {
Expand Down
4 changes: 4 additions & 0 deletions velox/docs/configs.rst
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,10 @@ Spilling
- boolean
- true
- When `spill_enabled` is true, determines whether RowNumber operator can spill to disk under memory pressure.
* - topn_row_number_spill_enabled
- boolean
- true
- When `spill_enabled` is true, determines whether TopNRowNumber operator can spill to disk under memory pressure.
* - aggregation_spill_memory_threshold
- integer
- 0
Expand Down
6 changes: 6 additions & 0 deletions velox/exec/PlanNodeStats.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,12 @@ std::string PlanNodeStats::toString(bool includeInputStats) const {
if (numSplits > 0) {
out << ", Splits: " << numSplits;
}

if (spilledRows > 0) {
out << ", Spilled: " << spilledRows << " rows ("
<< succinctBytes(spilledBytes) << ", " << spilledFiles << " files)";
}

return out.str();
}

Expand Down
2 changes: 1 addition & 1 deletion velox/exec/RowNumber.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ void RowNumber::restoreNextSpillPartition() {
}

void RowNumber::ensureInputFits(const RowVectorPtr& input) {
if (!spillConfig_.has_value()) {
if (!spillEnabled()) {
// Spilling is disabled.
return;
}
Expand Down
Loading

0 comments on commit f544393

Please sign in to comment.