From 7e4419531caa5b4d5c198e14929d4d697b856382 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 8 Feb 2023 11:42:45 +0100 Subject: [PATCH 01/81] feat(hogql): select statements --- frontend/src/queries/schema.json | 57 +++--- frontend/src/queries/schema.ts | 14 +- posthog/hogql/ast.py | 28 ++- posthog/hogql/constants.py | 3 + posthog/hogql/context.py | 5 + posthog/hogql/parser.py | 185 +++++++++++++++--- posthog/hogql/printer.py | 73 ++++++- posthog/hogql/test/test_parser.py | 238 ++++++++++++++++++++++- posthog/hogql/test/test_printer.py | 97 +++++++++ posthog/models/event/query_event_list.py | 10 +- posthog/schema.py | 6 +- 11 files changed, 641 insertions(+), 75 deletions(-) diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json index 1ac0c8e20d62e..58353872860da 100644 --- a/frontend/src/queries/schema.json +++ b/frontend/src/queries/schema.json @@ -1600,34 +1600,8 @@ "type": "array" }, "response": { - "additionalProperties": false, - "description": "Cached query response", - "properties": { - "columns": { - "items": { - "type": "string" - }, - "type": "array" - }, - "hasMore": { - "type": "boolean" - }, - "results": { - "items": { - "items": {}, - "type": "array" - }, - "type": "array" - }, - "types": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "required": ["columns", "types", "results"], - "type": "object" + "$ref": "#/definitions/EventsQueryResponse", + "description": "Cached query response" }, "select": { "description": "Return a limited set of data. Required.", @@ -1647,6 +1621,33 @@ "required": ["kind", "select"], "type": "object" }, + "EventsQueryResponse": { + "additionalProperties": false, + "properties": { + "columns": { + "items": {}, + "type": "array" + }, + "hasMore": { + "type": "boolean" + }, + "results": { + "items": { + "items": {}, + "type": "array" + }, + "type": "array" + }, + "types": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": ["columns", "types", "results"], + "type": "object" + }, "FeaturePropertyFilter": { "additionalProperties": false, "properties": { diff --git a/frontend/src/queries/schema.ts b/frontend/src/queries/schema.ts index 5bfebc91f3c6c..161b0b349c4d1 100644 --- a/frontend/src/queries/schema.ts +++ b/frontend/src/queries/schema.ts @@ -134,6 +134,13 @@ export interface NewEntityNode extends EntityNode { event?: string | null } +export interface EventsQueryResponse { + columns: any[] + types: string[] + results: any[][] + hasMore?: boolean +} + export interface EventsQuery extends DataNode { kind: NodeKind.EventsQuery /** Return a limited set of data. Required. */ @@ -170,12 +177,7 @@ export interface EventsQuery extends DataNode { /** Columns to order by */ orderBy?: string[] - response?: { - columns: string[] - types: string[] - results: any[][] - hasMore?: boolean - } + response?: EventsQueryResponse } export interface PersonsNode extends DataNode { diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index afba1f5cb50d5..b43b558fccc23 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -1,6 +1,6 @@ import re from enum import Enum -from typing import Any, List, Literal +from typing import Any, List, Literal, Optional, Union from pydantic import BaseModel, Extra @@ -102,3 +102,29 @@ class Placeholder(Expr): class Call(Expr): name: str args: List[Expr] + + +class JoinExpr(Expr): + table: Optional[Union["SelectQuery", Field]] = None + table_final: Optional[bool] = None + alias: Optional[str] = None + join_type: Optional[str] = None + join_constraint: Optional[Expr] = None + join_expr: Optional["JoinExpr"] = None + + +class SelectQuery(Expr): + select: List[Expr] + select_from: Optional[JoinExpr] = None + where: Optional[Expr] = None + prewhere: Optional[Expr] = None + having: Optional[Expr] = None + group_by: Optional[List[Expr]] = None + order_by: Optional[List[OrderExpr]] = None + limit: Optional[int] = None + offset: Optional[int] = None + distinct: Optional[bool] = None + + +JoinExpr.update_forward_refs(SelectQuery=SelectQuery) +JoinExpr.update_forward_refs(JoinExpr=JoinExpr) diff --git a/posthog/hogql/constants.py b/posthog/hogql/constants.py index 2f2359c104562..a7f7a8357c7ff 100644 --- a/posthog/hogql/constants.py +++ b/posthog/hogql/constants.py @@ -124,3 +124,6 @@ "person.created_at", "person.properties", ] + +# Never return more rows than this in top level HogQL SELECT statements +MAX_SELECT_RETURNED_ROWS = 65535 diff --git a/posthog/hogql/context.py b/posthog/hogql/context.py index 6e2221a3d7348..df3aabbd97e12 100644 --- a/posthog/hogql/context.py +++ b/posthog/hogql/context.py @@ -20,4 +20,9 @@ class HogQLContext: field_access_logs: List[HogQLFieldAccess] = field(default_factory=list) # Did the last calls to translate_hogql since setting these to False contain any of the following found_aggregation: bool = False + # Do we need to join the persons table or not using_person_on_events: bool = True + # If set, allows printing full SELECT queries in ClickHouse + select_team_id: Optional[int] = None + # Do we apply a limit of MAX_SELECT_RETURNED_ROWS=65535 to the topmost select query? + limit_top_select: bool = True diff --git a/posthog/hogql/parser.py b/posthog/hogql/parser.py index bc3eca9e61c15..b1891dddad6fe 100644 --- a/posthog/hogql/parser.py +++ b/posthog/hogql/parser.py @@ -1,4 +1,4 @@ -from typing import Dict, Optional +from typing import Dict, Literal, Optional, cast from antlr4 import CommonTokenStream, InputStream, ParseTreeVisitor from antlr4.error.ErrorListener import ErrorListener @@ -18,6 +18,14 @@ def parse_expr(expr: str, placeholders: Optional[Dict[str, ast.Expr]] = None) -> return node +def parse_select(statement: str, placeholders: Optional[Dict[str, ast.Expr]] = None) -> ast.Expr: + parse_tree = get_parser(statement).select() + node = HogQLParseTreeConverter().visit(parse_tree) + if placeholders: + node = replace_placeholders(node, placeholders) + return node + + def get_parser(query: str) -> HogQLParser: input_stream = InputStream(data=query) lexer = HogQLLexer(input_stream) @@ -35,16 +43,69 @@ def syntaxError(self, recognizer, offendingSymbol, line, column, msg, e): class HogQLParseTreeConverter(ParseTreeVisitor): def visitSelect(self, ctx: HogQLParser.SelectContext): - raise NotImplementedError(f"Unsupported node: SelectQuery") + return self.visit(ctx.selectUnionStmt() or ctx.selectStmt()) def visitSelectUnionStmt(self, ctx: HogQLParser.SelectUnionStmtContext): - raise NotImplementedError(f"Unsupported node: SelectUnionStmt") + selects = ctx.selectStmtWithParens() + if len(selects) != 1: + raise NotImplementedError(f"Unsupported: UNION ALL") + return self.visit(selects[0]) def visitSelectStmtWithParens(self, ctx: HogQLParser.SelectStmtWithParensContext): - raise NotImplementedError(f"Unsupported node: SelectStmtWithParens") + return self.visit(ctx.selectStmt() or ctx.selectUnionStmt()) def visitSelectStmt(self, ctx: HogQLParser.SelectStmtContext): - raise NotImplementedError(f"Unsupported node: SelectStmt") + select = self.visit(ctx.columnExprList()) if ctx.columnExprList() else [] + select_from = self.visit(ctx.fromClause()) if ctx.fromClause() else None + where = self.visit(ctx.whereClause()) if ctx.whereClause() else None + prewhere = self.visit(ctx.prewhereClause()) if ctx.prewhereClause() else None + having = self.visit(ctx.havingClause()) if ctx.havingClause() else None + group_by = self.visit(ctx.groupByClause()) if ctx.groupByClause() else None + order_by = self.visit(ctx.orderByClause()) if ctx.orderByClause() else None + + limit = None + offset = None + if ctx.limitClause() and ctx.limitClause().limitExpr(): + limit_expr = ctx.limitClause().limitExpr() + limit_node = self.visit(limit_expr.columnExpr(0)) + if limit_node is not None: + if isinstance(limit_node, ast.Constant) and isinstance(limit_node.value, int): + limit = limit_node.value + else: + raise Exception(f"LIMIT must be an integer") + if limit_expr.columnExpr(1): + offset_node = self.visit(limit_expr.columnExpr(1)) + if offset_node is not None: + if isinstance(offset_node, ast.Constant) and isinstance(offset_node.value, int): + offset = offset_node.value + else: + raise Exception(f"OFFSET must be an integer") + + if ctx.withClause(): + raise NotImplementedError(f"Unsupported: SelectStmt.withClause()") + if ctx.topClause(): + raise NotImplementedError(f"Unsupported: SelectStmt.topClause()") + if ctx.arrayJoinClause(): + raise NotImplementedError(f"Unsupported: SelectStmt.arrayJoinClause()") + if ctx.windowClause(): + raise NotImplementedError(f"Unsupported: SelectStmt.windowClause()") + if ctx.limitByClause(): + raise NotImplementedError(f"Unsupported: SelectStmt.limitByClause()") + if ctx.settingsClause(): + raise NotImplementedError(f"Unsupported: SelectStmt.settingsClause()") + + return ast.SelectQuery( + select=select, + distinct=True if ctx.DISTINCT() else None, + select_from=select_from, + where=where, + prewhere=prewhere, + having=having, + group_by=group_by, + order_by=order_by, + limit=limit, + offset=offset, + ) def visitWithClause(self, ctx: HogQLParser.WithClauseContext): raise NotImplementedError(f"Unsupported node: WithClause") @@ -53,7 +114,7 @@ def visitTopClause(self, ctx: HogQLParser.TopClauseContext): raise NotImplementedError(f"Unsupported node: TopClause") def visitFromClause(self, ctx: HogQLParser.FromClauseContext): - raise NotImplementedError(f"Unsupported node: FromClause") + return self.visit(ctx.joinExpr()) def visitArrayJoinClause(self, ctx: HogQLParser.ArrayJoinClauseContext): raise NotImplementedError(f"Unsupported node: ArrayJoinClause") @@ -62,19 +123,19 @@ def visitWindowClause(self, ctx: HogQLParser.WindowClauseContext): raise NotImplementedError(f"Unsupported node: WindowClause") def visitPrewhereClause(self, ctx: HogQLParser.PrewhereClauseContext): - raise NotImplementedError(f"Unsupported node: PrewhereClause") + return self.visit(ctx.columnExpr()) def visitWhereClause(self, ctx: HogQLParser.WhereClauseContext): - raise NotImplementedError(f"Unsupported node: WhereClause") + return self.visit(ctx.columnExpr()) def visitGroupByClause(self, ctx: HogQLParser.GroupByClauseContext): - raise NotImplementedError(f"Unsupported node: GroupByClause") + return self.visit(ctx.columnExprList()) def visitHavingClause(self, ctx: HogQLParser.HavingClauseContext): - raise NotImplementedError(f"Unsupported node: HavingClause") + return self.visit(ctx.columnExpr()) def visitOrderByClause(self, ctx: HogQLParser.OrderByClauseContext): - raise NotImplementedError(f"Unsupported node: OrderByClause") + return self.visit(ctx.orderExprList()) def visitProjectionOrderByClause(self, ctx: HogQLParser.ProjectionOrderByClauseContext): raise NotImplementedError(f"Unsupported node: ProjectionOrderByClause") @@ -89,31 +150,102 @@ def visitSettingsClause(self, ctx: HogQLParser.SettingsClauseContext): raise NotImplementedError(f"Unsupported node: SettingsClause") def visitJoinExprOp(self, ctx: HogQLParser.JoinExprOpContext): - raise NotImplementedError(f"Unsupported node: JoinExprOp") + if ctx.GLOBAL(): + raise NotImplementedError(f"Unsupported: GLOBAL JOIN") + if ctx.LOCAL(): + raise NotImplementedError(f"Unsupported: LOCAL JOIN") + + join1: ast.JoinExpr = self.visit(ctx.joinExpr(0)) + join2: ast.JoinExpr = self.visit(ctx.joinExpr(1)) + + if ctx.joinOp(): + join_type = f"{self.visit(ctx.joinOp())} JOIN" + else: + join_type = "JOIN" + join_constraint = self.visit(ctx.joinConstraintClause()) + + join_without_next_expr = join1 + while join_without_next_expr.join_expr: + join_without_next_expr = join_without_next_expr.join_expr + + join_without_next_expr.join_expr = join2 + join_without_next_expr.join_constraint = join_constraint + join_without_next_expr.join_type = join_type + return join1 def visitJoinExprTable(self, ctx: HogQLParser.JoinExprTableContext): - raise NotImplementedError(f"Unsupported node: JoinExprTable") + if ctx.sampleClause(): + raise NotImplementedError(f"Unsupported: SAMPLE (JoinExprTable.sampleClause)") + table = self.visit(ctx.tableExpr()) + table_final = True if ctx.FINAL() else None + if isinstance(table, ast.JoinExpr): + # visitTableExprAlias returns a JoinExpr to pass the alias + table.table_final = table_final + return table + return ast.JoinExpr(table=table, table_final=table_final) def visitJoinExprParens(self, ctx: HogQLParser.JoinExprParensContext): - raise NotImplementedError(f"Unsupported node: JoinExprParens") + return self.visit(ctx.joinExpr()) def visitJoinExprCrossOp(self, ctx: HogQLParser.JoinExprCrossOpContext): raise NotImplementedError(f"Unsupported node: JoinExprCrossOp") def visitJoinOpInner(self, ctx: HogQLParser.JoinOpInnerContext): - raise NotImplementedError(f"Unsupported node: JoinOpInner") + tokens = [] + if ctx.LEFT(): + tokens.append("INNER") + if ctx.ALL(): + tokens.append("ALL") + if ctx.ANTI(): + tokens.append("ANTI") + if ctx.ANY(): + tokens.append("ANY") + if ctx.ASOF(): + tokens.append("ASOF") + return " ".join(tokens) def visitJoinOpLeftRight(self, ctx: HogQLParser.JoinOpLeftRightContext): - raise NotImplementedError(f"Unsupported node: JoinOpLeftRight") + tokens = [] + if ctx.LEFT(): + tokens.append("LEFT") + if ctx.RIGHT(): + tokens.append("RIGHT") + if ctx.OUTER(): + tokens.append("OUTER") + if ctx.SEMI(): + tokens.append("SEMI") + if ctx.ALL(): + tokens.append("ALL") + if ctx.ANTI(): + tokens.append("ANTI") + if ctx.ANY(): + tokens.append("ANY") + if ctx.ASOF(): + tokens.append("ASOF") + return " ".join(tokens) def visitJoinOpFull(self, ctx: HogQLParser.JoinOpFullContext): - raise NotImplementedError(f"Unsupported node: JoinOpFull") + tokens = [] + if ctx.LEFT(): + tokens.append("FULL") + if ctx.OUTER(): + tokens.append("OUTER") + if ctx.ALL(): + tokens.append("ALL") + if ctx.ANY(): + tokens.append("ANY") + return " ".join(tokens) def visitJoinOpCross(self, ctx: HogQLParser.JoinOpCrossContext): raise NotImplementedError(f"Unsupported node: JoinOpCross") def visitJoinConstraintClause(self, ctx: HogQLParser.JoinConstraintClauseContext): - raise NotImplementedError(f"Unsupported node: JoinConstraintClause") + if ctx.USING(): + raise NotImplementedError(f"Unsupported: JOIN ... USING") + column_expr_list = self.visit(ctx.columnExprList()) + if len(column_expr_list) != 1: + raise NotImplementedError(f"Unsupported: JOIN ... ON with multiple expressions") + return column_expr_list[0] def visitSampleClause(self, ctx: HogQLParser.SampleClauseContext): raise NotImplementedError(f"Unsupported node: SampleClause") @@ -122,10 +254,11 @@ def visitLimitExpr(self, ctx: HogQLParser.LimitExprContext): raise NotImplementedError(f"Unsupported node: LimitExpr") def visitOrderExprList(self, ctx: HogQLParser.OrderExprListContext): - raise NotImplementedError(f"Unsupported node: OrderExprList") + return [self.visit(expr) for expr in ctx.orderExpr()] def visitOrderExpr(self, ctx: HogQLParser.OrderExprContext): - raise NotImplementedError(f"Unsupported node: OrderExpr") + order = "DESC" if ctx.DESC() or ctx.DESCENDING() else "ASC" + return ast.OrderExpr(expr=self.visit(ctx.columnExpr()), order=cast(Literal["ASC", "DESC"], order)) def visitRatioExpr(self, ctx: HogQLParser.RatioExprContext): raise NotImplementedError(f"Unsupported node: RatioExpr") @@ -209,7 +342,7 @@ def visitColumnExprNegate(self, ctx: HogQLParser.ColumnExprNegateContext): raise NotImplementedError(f"Unsupported node: ColumnExprNegate") def visitColumnExprSubquery(self, ctx: HogQLParser.ColumnExprSubqueryContext): - raise NotImplementedError(f"Unsupported node: ColumnExprSubquery") + return self.visit(ctx.selectUnionStmt()) def visitColumnExprLiteral(self, ctx: HogQLParser.ColumnExprLiteralContext): return self.visitChildren(ctx) @@ -410,17 +543,17 @@ def visitColumnIdentifier(self, ctx: HogQLParser.ColumnIdentifierContext): return ast.Field(chain=table + nested) def visitNestedIdentifier(self, ctx: HogQLParser.NestedIdentifierContext): - chain = [self.visit(identifier) for identifier in ctx.identifier()] - return chain + return [self.visit(identifier) for identifier in ctx.identifier()] def visitTableExprIdentifier(self, ctx: HogQLParser.TableExprIdentifierContext): - raise NotImplementedError(f"Unsupported node: TableExprIdentifier") + chain = self.visit(ctx.tableIdentifier()) + return ast.Field(chain=chain) def visitTableExprSubquery(self, ctx: HogQLParser.TableExprSubqueryContext): - raise NotImplementedError(f"Unsupported node: TableExprSubquery") + return self.visit(ctx.selectUnionStmt()) def visitTableExprAlias(self, ctx: HogQLParser.TableExprAliasContext): - raise NotImplementedError(f"Unsupported node: TableExprAlias") + return ast.JoinExpr(table=self.visit(ctx.tableExpr()), alias=self.visit(ctx.alias() or ctx.identifier())) def visitTableExprFunction(self, ctx: HogQLParser.TableExprFunctionContext): raise NotImplementedError(f"Unsupported node: TableExprFunction") diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index 61c2ee7020d1b..dfba9de265867 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -7,6 +7,7 @@ EVENT_PERSON_FIELDS, HOGQL_AGGREGATIONS, KEYWORDS, + MAX_SELECT_RETURNED_ROWS, SELECT_STAR_FROM_EVENTS_FIELDS, ) from posthog.hogql.context import HogQLContext, HogQLFieldAccess @@ -14,13 +15,83 @@ from posthog.hogql.print_string import print_hogql_identifier +def guard_where_team_id(where: ast.Expr, context: HogQLContext) -> ast.Expr: + """Add a mandatory "and(team_id, ...)" filter around the expression.""" + if not context.select_team_id: + raise ValueError("context.select_team_id not found") + + from posthog.hogql.parser import parse_expr + + team_clause = parse_expr("team_id = {team_id}", {"team_id": ast.Constant(value=context.select_team_id)}) + if isinstance(where, ast.And): + where = ast.And(exprs=[team_clause] + where.exprs) + elif where: + where = ast.And(exprs=[team_clause, where]) + else: + where = team_clause + return where + + def print_ast( node: ast.AST, stack: List[ast.AST], context: HogQLContext, dialect: Literal["hogql", "clickhouse"] ) -> str: """Translate a parsed HogQL expression in the shape of a Python AST into a Clickhouse expression.""" stack.append(node) - if isinstance(node, ast.BinaryOperation): + if isinstance(node, ast.SelectQuery): + if dialect == "clickhouse" and not context.select_team_id: + raise ValueError("Full SELECT queries are disabled if select_team_id is not set") + + columns = [print_ast(column, stack, context, dialect) for column in node.select] if node.select else ["1"] + + from_table = None + if node.select_from: + if node.select_from.alias is not None: + raise ValueError("Table aliases not yet supported") + if isinstance(node.select_from.table, ast.Field): + if node.select_from.table.chain != ["events"]: + raise ValueError('Only selecting from the "events" table is supported') + from_table = "events" + elif isinstance(node.select_from.table, ast.SelectQuery): + from_table = f"({print_ast(node.select_from.table, stack, context, dialect)})" + else: + raise ValueError("Only selecting from a table or a subquery is supported") + + where = node.where + # Guard with team_id if selecting from the events table and printing ClickHouse SQL + # We do this in the printer, and not in a separate step, to be really sure this gets added. + if dialect == "clickhouse" and from_table == "events": + where = guard_where_team_id(where, context) + where = print_ast(where, stack, context, dialect) if where else None + + having = print_ast(node.having, stack, context, dialect) if node.having else None + prewhere = print_ast(node.prewhere, stack, context, dialect) if node.prewhere else None + group_by = [print_ast(column, stack, context, dialect) for column in node.group_by] if node.group_by else None + order_by = [print_ast(column, stack, context, dialect) for column in node.order_by] if node.order_by else None + + limit = node.limit + if context.limit_top_select: + if limit is not None: + limit = max(0, min(node.limit, MAX_SELECT_RETURNED_ROWS)) + if len(stack) == 1 and limit is None: + limit = MAX_SELECT_RETURNED_ROWS + + clauses = [ + f"SELECT {'DISTINCT ' if node.distinct else ''}{', '.join(columns)}", + f"FROM {from_table}" if from_table else None, + "WHERE " + where if where else None, + f"GROUP BY {', '.join(group_by)}" if group_by and len(group_by) > 0 else None, + "HAVING " + having if having else None, + "PREWHERE " + prewhere if prewhere else None, + f"ORDER BY {', '.join(order_by)}" if order_by and len(order_by) > 0 else None, + f"LIMIT {limit}" if limit is not None else None, + f"OFFSET {node.offset}" if node.offset is not None else None, + ] + response = " ".join([clause for clause in clauses if clause]) + if len(stack) > 1: + response = f"({response})" + + elif isinstance(node, ast.BinaryOperation): if node.op == ast.BinaryOperationType.Add: response = f"plus({print_ast(node.left, stack, context, dialect)}, {print_ast(node.right, stack, context, dialect)})" elif node.op == ast.BinaryOperationType.Sub: diff --git a/posthog/hogql/test/test_parser.py b/posthog/hogql/test/test_parser.py index dfe0d7b169d36..2c6b34a4e5719 100644 --- a/posthog/hogql/test/test_parser.py +++ b/posthog/hogql/test/test_parser.py @@ -1,5 +1,5 @@ from posthog.hogql import ast -from posthog.hogql.parser import parse_expr +from posthog.hogql.parser import parse_expr, parse_select from posthog.test.base import BaseTest @@ -354,3 +354,239 @@ def test_placeholders(self): right=ast.Constant(value=123), ), ) + + def test_select_columns(self): + self.assertEqual(parse_select("select 1"), ast.SelectQuery(select=[ast.Constant(value=1)])) + self.assertEqual( + parse_select("select 1, 4, 'string'"), + ast.SelectQuery(select=[ast.Constant(value=1), ast.Constant(value=4), ast.Constant(value="string")]), + ) + + def test_select_columns_distinct(self): + self.assertEqual( + parse_select("select distinct 1"), ast.SelectQuery(select=[ast.Constant(value=1)], distinct=True) + ) + + def test_select_where(self): + self.assertEqual( + parse_select("select 1 where true"), + ast.SelectQuery(select=[ast.Constant(value=1)], where=ast.Constant(value=True)), + ) + self.assertEqual( + parse_select("select 1 where 1 == 2"), + ast.SelectQuery( + select=[ast.Constant(value=1)], + where=ast.CompareOperation( + op=ast.CompareOperationType.Eq, left=ast.Constant(value=1), right=ast.Constant(value=2) + ), + ), + ) + + def test_select_prewhere(self): + self.assertEqual( + parse_select("select 1 prewhere true"), + ast.SelectQuery(select=[ast.Constant(value=1)], prewhere=ast.Constant(value=True)), + ) + self.assertEqual( + parse_select("select 1 prewhere 1 == 2"), + ast.SelectQuery( + select=[ast.Constant(value=1)], + prewhere=ast.CompareOperation( + op=ast.CompareOperationType.Eq, left=ast.Constant(value=1), right=ast.Constant(value=2) + ), + ), + ) + + def test_select_having(self): + self.assertEqual( + parse_select("select 1 having true"), + ast.SelectQuery(select=[ast.Constant(value=1)], having=ast.Constant(value=True)), + ) + self.assertEqual( + parse_select("select 1 having 1 == 2"), + ast.SelectQuery( + select=[ast.Constant(value=1)], + having=ast.CompareOperation( + op=ast.CompareOperationType.Eq, left=ast.Constant(value=1), right=ast.Constant(value=2) + ), + ), + ) + + def test_select_complex_wheres(self): + self.assertEqual( + parse_select("select 1 prewhere 2 != 3 where 1 == 2 having 'string' like '%a%'"), + ast.SelectQuery( + select=[ast.Constant(value=1)], + where=ast.CompareOperation( + op=ast.CompareOperationType.Eq, left=ast.Constant(value=1), right=ast.Constant(value=2) + ), + prewhere=ast.CompareOperation( + op=ast.CompareOperationType.NotEq, left=ast.Constant(value=2), right=ast.Constant(value=3) + ), + having=ast.CompareOperation( + op=ast.CompareOperationType.Like, left=ast.Constant(value="string"), right=ast.Constant(value="%a%") + ), + ), + ) + + def test_select_from(self): + self.assertEqual( + parse_select("select 1 from events"), + ast.SelectQuery( + select=[ast.Constant(value=1)], select_from=ast.JoinExpr(table=ast.Field(chain=["events"])) + ), + ) + self.assertEqual( + parse_select("select 1 from events as e"), + ast.SelectQuery( + select=[ast.Constant(value=1)], + select_from=ast.JoinExpr(table=ast.Field(chain=["events"]), alias="e"), + ), + ) + self.assertEqual( + parse_select("select 1 from complex.table"), + ast.SelectQuery( + select=[ast.Constant(value=1)], + select_from=ast.JoinExpr(table=ast.Field(chain=["complex", "table"])), + ), + ) + self.assertEqual( + parse_select("select 1 from complex.table as a"), + ast.SelectQuery( + select=[ast.Constant(value=1)], + select_from=ast.JoinExpr(table=ast.Field(chain=["complex", "table"]), alias="a"), + ), + ) + self.assertEqual( + parse_select("select 1 from (select 1 from events)"), + ast.SelectQuery( + select=[ast.Constant(value=1)], + select_from=ast.JoinExpr( + table=ast.SelectQuery( + select=[ast.Constant(value=1)], select_from=ast.JoinExpr(table=ast.Field(chain=["events"])) + ) + ), + ), + ) + self.assertEqual( + parse_select("select 1 from (select 1 from events) as sq"), + ast.SelectQuery( + select=[ast.Constant(value=1)], + select_from=ast.JoinExpr( + table=ast.SelectQuery( + select=[ast.Constant(value=1)], select_from=ast.JoinExpr(table=ast.Field(chain=["events"])) + ), + alias="sq", + ), + ), + ) + + def test_select_from_join(self): + self.assertEqual( + parse_select("select 1 from events JOIN events2 ON 1"), + ast.SelectQuery( + select=[ast.Constant(value=1)], + select_from=ast.JoinExpr( + table=ast.Field(chain=["events"]), + join_type="JOIN", + join_constraint=ast.Constant(value=1), + join_expr=ast.JoinExpr(table=ast.Field(chain=["events2"])), + ), + ), + ) + self.assertEqual( + parse_select("select * from events LEFT OUTER JOIN events2 ON 1"), + ast.SelectQuery( + select=[ast.Field(chain=["*"])], + select_from=ast.JoinExpr( + table=ast.Field(chain=["events"]), + join_type="LEFT OUTER JOIN", + join_constraint=ast.Constant(value=1), + join_expr=ast.JoinExpr(table=ast.Field(chain=["events2"])), + ), + ), + ) + self.assertEqual( + parse_select("select 1 from events LEFT OUTER JOIN events2 ON 1 ANY RIGHT JOIN events3 ON 2"), + ast.SelectQuery( + select=[ast.Constant(value=1)], + select_from=ast.JoinExpr( + table=ast.Field(chain=["events"]), + join_type="LEFT OUTER JOIN", + join_constraint=ast.Constant(value=1), + join_expr=ast.JoinExpr( + table=ast.Field(chain=["events2"]), + join_type="RIGHT ANY JOIN", + join_constraint=ast.Constant(value=2), + join_expr=ast.JoinExpr(table=ast.Field(chain=["events3"])), + ), + ), + ), + ) + + def test_select_group_by(self): + self.assertEqual( + parse_select("select 1 from events GROUP BY 1, event"), + ast.SelectQuery( + select=[ast.Constant(value=1)], + select_from=ast.JoinExpr(table=ast.Field(chain=["events"])), + group_by=[ast.Constant(value=1), ast.Field(chain=["event"])], + ), + ) + + def test_select_order_by(self): + self.assertEqual( + parse_select("select 1 from events ORDER BY 1 ASC, event, timestamp DESC"), + ast.SelectQuery( + select=[ast.Constant(value=1)], + select_from=ast.JoinExpr(table=ast.Field(chain=["events"])), + order_by=[ + ast.OrderExpr(expr=ast.Constant(value=1), order="ASC"), + ast.OrderExpr(expr=ast.Field(chain=["event"]), order="ASC"), + ast.OrderExpr(expr=ast.Field(chain=["timestamp"]), order="DESC"), + ], + ), + ) + + def test_select_limit_offset(self): + self.assertEqual( + parse_select("select 1 from events LIMIT 1"), + ast.SelectQuery( + select=[ast.Constant(value=1)], + select_from=ast.JoinExpr(table=ast.Field(chain=["events"])), + limit=1, + ), + ) + self.assertEqual( + parse_select("select 1 from events LIMIT 1 OFFSET 3"), + ast.SelectQuery( + select=[ast.Constant(value=1)], + select_from=ast.JoinExpr(table=ast.Field(chain=["events"])), + limit=1, + offset=3, + ), + ) + + def test_select_placeholders(self): + self.assertEqual( + parse_select("select 1 where 1 == {hogql_val_1}"), + ast.SelectQuery( + select=[ast.Constant(value=1)], + where=ast.CompareOperation( + op=ast.CompareOperationType.Eq, + left=ast.Constant(value=1), + right=ast.Placeholder(field="hogql_val_1"), + ), + ), + ) + self.assertEqual( + parse_select("select 1 where 1 == {hogql_val_1}", {"hogql_val_1": ast.Constant(value="bar")}), + ast.SelectQuery( + select=[ast.Constant(value=1)], + where=ast.CompareOperation( + op=ast.CompareOperationType.Eq, + left=ast.Constant(value=1), + right=ast.Constant(value="bar"), + ), + ), + ) diff --git a/posthog/hogql/test/test_printer.py b/posthog/hogql/test/test_printer.py index 80e15dcb5f3d2..ded3198704c69 100644 --- a/posthog/hogql/test/test_printer.py +++ b/posthog/hogql/test/test_printer.py @@ -13,6 +13,12 @@ def _expr( ) -> str: return translate_hogql(query, context or HogQLContext(), dialect) + # Helper to always translate HogQL with a blank context, + def _select( + self, query: str, context: Optional[HogQLContext] = None, dialect: Literal["hogql", "clickhouse"] = "clickhouse" + ) -> str: + return translate_hogql(query, context or HogQLContext(select_team_id=42), dialect) + def _assert_expr_error(self, expr, expected_error, dialect: Literal["hogql", "clickhouse"] = "clickhouse"): with self.assertRaises(ValueError) as context: self._expr(expr, None, dialect) @@ -20,6 +26,13 @@ def _assert_expr_error(self, expr, expected_error, dialect: Literal["hogql", "cl raise AssertionError(f"Expected '{expected_error}' in '{str(context.exception)}'") self.assertTrue(expected_error in str(context.exception)) + def _assert_select_error(self, statement, expected_error, dialect: Literal["hogql", "clickhouse"] = "clickhouse"): + with self.assertRaises(ValueError) as context: + self._select(statement, None, dialect) + if expected_error not in str(context.exception): + raise AssertionError(f"Expected '{expected_error}' in '{str(context.exception)}'") + self.assertTrue(expected_error in str(context.exception)) + def test_literals(self): self.assertEqual(self._expr("1 + 2"), "plus(1, 2)") self.assertEqual(self._expr("-1 + 2"), "plus(-1, 2)") @@ -330,3 +343,87 @@ def test_values(self): def test_no_alias_yet(self): self._assert_expr_error("1 as team_id", "Unknown AST node Alias") self._assert_expr_error("1 as `-- select team_id`", "Unknown AST node Alias") + + def test_select(self): + self.assertEqual(self._select("select 1"), "SELECT 1 LIMIT 65535") + self.assertEqual(self._select("select 1 + 2"), "SELECT plus(1, 2) LIMIT 65535") + self.assertEqual(self._select("select 1 + 2, 3"), "SELECT plus(1, 2), 3 LIMIT 65535") + self.assertEqual( + self._select("select 1 + 2, 3 + 4 from events"), + "SELECT plus(1, 2), plus(3, 4) FROM events WHERE equals(team_id, 42) LIMIT 65535", + ) + + def test_select_alias(self): + # currently not supported! + self._assert_select_error("select 1 as b", "Unknown AST node Alias") + self._assert_select_error("select 1 from events as e", "Table aliases not yet supported") + + def test_select_from(self): + self.assertEqual( + self._select("select 1 from events"), "SELECT 1 FROM events WHERE equals(team_id, 42) LIMIT 65535" + ) + self._assert_select_error("select 1 from other", 'Only selecting from the "events" table is supported') + + def test_select_where(self): + self.assertEqual( + self._select("select 1 from events where 1 == 2"), + "SELECT 1 FROM events WHERE and(equals(team_id, 42), equals(1, 2)) LIMIT 65535", + ) + + def test_select_having(self): + self.assertEqual( + self._select("select 1 from events having 1 == 2"), + "SELECT 1 FROM events WHERE equals(team_id, 42) HAVING equals(1, 2) LIMIT 65535", + ) + + def test_select_prewhere(self): + self.assertEqual( + self._select("select 1 from events prewhere 1 == 2"), + "SELECT 1 FROM events WHERE equals(team_id, 42) PREWHERE equals(1, 2) LIMIT 65535", + ) + + def test_select_order_by(self): + self.assertEqual( + self._select("select event from events order by event"), + "SELECT event FROM events WHERE equals(team_id, 42) ORDER BY event ASC LIMIT 65535", + ) + self.assertEqual( + self._select("select event from events order by event desc"), + "SELECT event FROM events WHERE equals(team_id, 42) ORDER BY event DESC LIMIT 65535", + ) + self.assertEqual( + self._select("select event from events order by event desc, timestamp"), + "SELECT event FROM events WHERE equals(team_id, 42) ORDER BY event DESC, timestamp ASC LIMIT 65535", + ) + + def test_select_limit(self): + self.assertEqual( + self._select("select event from events limit 10"), + "SELECT event FROM events WHERE equals(team_id, 42) LIMIT 10", + ) + self.assertEqual( + self._select("select event from events limit 10000000"), + "SELECT event FROM events WHERE equals(team_id, 42) LIMIT 65535", + ) + + def test_select_offset(self): + self.assertEqual( + self._select("select event from events limit 10 offset 10"), + "SELECT event FROM events WHERE equals(team_id, 42) LIMIT 10 OFFSET 10", + ) + self.assertEqual( + self._select("select event from events limit 10 offset 0"), + "SELECT event FROM events WHERE equals(team_id, 42) LIMIT 10 OFFSET 0", + ) + + def test_select_group_by(self): + self.assertEqual( + self._select("select event from events group by event, timestamp"), + "SELECT event FROM events WHERE equals(team_id, 42) GROUP BY event, timestamp LIMIT 65535", + ) + + def test_select_distinct(self): + self.assertEqual( + self._select("select distinct event from events group by event, timestamp"), + "SELECT DISTINCT event FROM events WHERE equals(team_id, 42) GROUP BY event, timestamp LIMIT 65535", + ) diff --git a/posthog/models/event/query_event_list.py b/posthog/models/event/query_event_list.py index 847369a2fc951..878bf00e4ed32 100644 --- a/posthog/models/event/query_event_list.py +++ b/posthog/models/event/query_event_list.py @@ -4,7 +4,6 @@ from dateutil.parser import isoparse from django.utils.timezone import now -from pydantic import BaseModel from posthog.api.utils import get_pk_or_uuid from posthog.clickhouse.client.connection import Workload @@ -22,7 +21,7 @@ from posthog.models.event.util import ElementSerializer from posthog.models.property.util import parse_prop_grouped_clauses from posthog.queries.insight import insight_query_with_columns, insight_sync_execute -from posthog.schema import EventsQuery +from posthog.schema import EventsQuery, EventsQueryResponse from posthog.utils import relative_date_parse # Return at most this number of events in CSV export @@ -31,13 +30,6 @@ QUERY_MAXIMUM_LIMIT = 100_000 -class EventsQueryResponse(BaseModel): - columns: List[str] - types: List[str] - results: List[List] - hasMore: bool - - def determine_event_conditions(conditions: Dict[str, Union[None, str, List[str]]]) -> Tuple[str, Dict]: result = "" params: Dict[str, Union[str, List[str]]] = {} diff --git a/posthog/schema.py b/posthog/schema.py index 28bad7dd7ad5e..a9c22bf6e7321 100644 --- a/posthog/schema.py +++ b/posthog/schema.py @@ -176,11 +176,11 @@ class Config: results: List[EventType] -class Response1(BaseModel): +class EventsQueryResponse(BaseModel): class Config: extra = Extra.forbid - columns: List[str] + columns: List hasMore: Optional[bool] = None results: List[List] types: List[str] @@ -668,7 +668,7 @@ class Config: ] ] ] = Field(None, description="Properties configurable in the interface") - response: Optional[Response1] = Field(None, description="Cached query response") + response: Optional[EventsQueryResponse] = Field(None, description="Cached query response") select: List[str] = Field(..., description="Return a limited set of data. Required.") where: Optional[List[str]] = Field(None, description="HogQL filters to apply on returned data") From a87115f5fa42eb0fa7441fbae6a7b3033c6ed7bf Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 8 Feb 2023 15:16:35 +0100 Subject: [PATCH 02/81] visitor --- posthog/hogql/hogql.py | 7 +++++-- posthog/hogql/test/test_visitor.py | 24 ++++++++++++++++++++++++ posthog/hogql/visitor.py | 26 ++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) diff --git a/posthog/hogql/hogql.py b/posthog/hogql/hogql.py index 81257157e08db..dc16b3cf8e9f6 100644 --- a/posthog/hogql/hogql.py +++ b/posthog/hogql/hogql.py @@ -1,7 +1,7 @@ from typing import Literal from posthog.hogql.context import HogQLContext -from posthog.hogql.parser import parse_expr +from posthog.hogql.parser import parse_expr, parse_select from posthog.hogql.printer import print_ast @@ -11,7 +11,10 @@ def translate_hogql(query: str, context: HogQLContext, dialect: Literal["hogql", raise ValueError("Empty query") try: - node = parse_expr(query) + if context.select_team_id: + node = parse_select(query) + else: + node = parse_expr(query) except SyntaxError as err: raise ValueError(f"SyntaxError: {err.msg}") except NotImplementedError as err: diff --git a/posthog/hogql/test/test_visitor.py b/posthog/hogql/test/test_visitor.py index 42857388f890d..f60812f53aab8 100644 --- a/posthog/hogql/test/test_visitor.py +++ b/posthog/hogql/test/test_visitor.py @@ -65,6 +65,30 @@ def test_everything_visitor(self): ], ) ), + ast.Alias(expr=ast.SelectQuery(select=[ast.Field(chain=["timestamp"])]), alias="f"), + ast.SelectQuery( + select=[ast.Field(chain=["a"])], + select_from=ast.JoinExpr( + table=ast.Field(chain=["b"]), + table_final=True, + alias="c", + join_type="INNER", + join_constraint=ast.CompareOperation( + op=ast.CompareOperationType.Eq, + left=ast.Field(chain=["d"]), + right=ast.Field(chain=["e"]), + ), + join_expr=ast.JoinExpr(table=ast.Field(chain=["f"])), + ), + where=ast.Constant(value=True), + prewhere=ast.Constant(value=True), + having=ast.Constant(value=True), + group_by=[ast.Constant(value=True)], + order_by=[ast.OrderExpr(expr=ast.Constant(value=True), order="DESC")], + limit=1, + offset=0, + distinct=True, + ), ] ) self.assertEqual(node, EverythingVisitor().visit(node)) diff --git a/posthog/hogql/visitor.py b/posthog/hogql/visitor.py index e36ec7d6af6e4..1bca4d05a78f4 100644 --- a/posthog/hogql/visitor.py +++ b/posthog/hogql/visitor.py @@ -3,6 +3,8 @@ class Visitor(object): def visit(self, node: ast.AST): + if node is None: + return node return node.accept(self) @@ -59,3 +61,27 @@ def visit_call(self, call: ast.Call): name=call.name, args=[self.visit(arg) for arg in call.args], ) + + def visit_join_expr(self, node: ast.JoinExpr): + return ast.JoinExpr( + table=self.visit(node.table), + join_expr=self.visit(node.join_expr), + table_final=node.table_final, + alias=node.alias, + join_type=node.join_type, + join_constraint=self.visit(node.join_constraint), + ) + + def visit_select_query(self, node: ast.SelectQuery): + return ast.SelectQuery( + select=[self.visit(expr) for expr in node.select] if node.select else None, + select_from=self.visit(node.select_from), + where=self.visit(node.where), + prewhere=self.visit(node.prewhere), + having=self.visit(node.having), + group_by=[self.visit(expr) for expr in node.group_by] if node.group_by else None, + order_by=[self.visit(expr) for expr in node.order_by] if node.order_by else None, + limit=node.limit, + offset=node.offset, + distinct=node.distinct, + ) From c55f170b9c26f3f6cf5329b0637b4fc30c9ac9ec Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 8 Feb 2023 15:23:08 +0100 Subject: [PATCH 03/81] cleanup --- posthog/hogql/printer.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index dfba9de265867..d07599d080e21 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -20,8 +20,6 @@ def guard_where_team_id(where: ast.Expr, context: HogQLContext) -> ast.Expr: if not context.select_team_id: raise ValueError("context.select_team_id not found") - from posthog.hogql.parser import parse_expr - team_clause = parse_expr("team_id = {team_id}", {"team_id": ast.Constant(value=context.select_team_id)}) if isinstance(where, ast.And): where = ast.And(exprs=[team_clause] + where.exprs) @@ -58,9 +56,10 @@ def print_ast( raise ValueError("Only selecting from a table or a subquery is supported") where = node.where - # Guard with team_id if selecting from the events table and printing ClickHouse SQL + # Guard with team_id if selecting from a table and printing ClickHouse SQL # We do this in the printer, and not in a separate step, to be really sure this gets added. - if dialect == "clickhouse" and from_table == "events": + # This will be improved when we add proper table and column alias support. For now, let's just be safe. + if dialect == "clickhouse" and from_table is not None: where = guard_where_team_id(where, context) where = print_ast(where, stack, context, dialect) if where else None From d47bc616d9905a56ba540df096858cf5ce68f59b Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 8 Feb 2023 16:03:43 +0100 Subject: [PATCH 04/81] parse limit by --- posthog/hogql/ast.py | 8 ++-- posthog/hogql/parser.py | 64 ++++++++++++------------------- posthog/hogql/test/test_parser.py | 26 +++++++++++-- 3 files changed, 52 insertions(+), 46 deletions(-) diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index b43b558fccc23..012a9b9a8b57a 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -115,15 +115,17 @@ class JoinExpr(Expr): class SelectQuery(Expr): select: List[Expr] + distinct: Optional[bool] = None select_from: Optional[JoinExpr] = None where: Optional[Expr] = None prewhere: Optional[Expr] = None having: Optional[Expr] = None group_by: Optional[List[Expr]] = None order_by: Optional[List[OrderExpr]] = None - limit: Optional[int] = None - offset: Optional[int] = None - distinct: Optional[bool] = None + limit: Optional[Expr] = None + limit_by: Optional[List[Expr]] = None + limit_with_ties: Optional[bool] = None + offset: Optional[Expr] = None JoinExpr.update_forward_refs(SelectQuery=SelectQuery) diff --git a/posthog/hogql/parser.py b/posthog/hogql/parser.py index b1891dddad6fe..f393769960196 100644 --- a/posthog/hogql/parser.py +++ b/posthog/hogql/parser.py @@ -55,31 +55,28 @@ def visitSelectStmtWithParens(self, ctx: HogQLParser.SelectStmtWithParensContext return self.visit(ctx.selectStmt() or ctx.selectUnionStmt()) def visitSelectStmt(self, ctx: HogQLParser.SelectStmtContext): - select = self.visit(ctx.columnExprList()) if ctx.columnExprList() else [] - select_from = self.visit(ctx.fromClause()) if ctx.fromClause() else None - where = self.visit(ctx.whereClause()) if ctx.whereClause() else None - prewhere = self.visit(ctx.prewhereClause()) if ctx.prewhereClause() else None - having = self.visit(ctx.havingClause()) if ctx.havingClause() else None - group_by = self.visit(ctx.groupByClause()) if ctx.groupByClause() else None - order_by = self.visit(ctx.orderByClause()) if ctx.orderByClause() else None - - limit = None - offset = None - if ctx.limitClause() and ctx.limitClause().limitExpr(): - limit_expr = ctx.limitClause().limitExpr() - limit_node = self.visit(limit_expr.columnExpr(0)) - if limit_node is not None: - if isinstance(limit_node, ast.Constant) and isinstance(limit_node.value, int): - limit = limit_node.value - else: - raise Exception(f"LIMIT must be an integer") + select_query = ast.SelectQuery( + select=self.visit(ctx.columnExprList()) if ctx.columnExprList() else [], + distinct=True if ctx.DISTINCT() else None, + select_from=self.visit(ctx.fromClause()) if ctx.fromClause() else None, + where=self.visit(ctx.whereClause()) if ctx.whereClause() else None, + prewhere=self.visit(ctx.prewhereClause()) if ctx.prewhereClause() else None, + having=self.visit(ctx.havingClause()) if ctx.havingClause() else None, + group_by=self.visit(ctx.groupByClause()) if ctx.groupByClause() else None, + order_by=self.visit(ctx.orderByClause()) if ctx.orderByClause() else None, + ) + + any_limit_clause = ctx.limitClause() or ctx.limitByClause() + if any_limit_clause and any_limit_clause.limitExpr(): + limit_expr = any_limit_clause.limitExpr() + if limit_expr.columnExpr(0): + select_query.limit = self.visit(limit_expr.columnExpr(0)) if limit_expr.columnExpr(1): - offset_node = self.visit(limit_expr.columnExpr(1)) - if offset_node is not None: - if isinstance(offset_node, ast.Constant) and isinstance(offset_node.value, int): - offset = offset_node.value - else: - raise Exception(f"OFFSET must be an integer") + select_query.offset = self.visit(limit_expr.columnExpr(1)) + if ctx.limitClause() and ctx.limitClause().WITH() and ctx.limitClause().TIES(): + select_query.limit_with_ties = True + if ctx.limitByClause() and ctx.limitByClause().columnExprList(): + select_query.limit_by = self.visit(ctx.limitByClause().columnExprList()) if ctx.withClause(): raise NotImplementedError(f"Unsupported: SelectStmt.withClause()") @@ -89,23 +86,10 @@ def visitSelectStmt(self, ctx: HogQLParser.SelectStmtContext): raise NotImplementedError(f"Unsupported: SelectStmt.arrayJoinClause()") if ctx.windowClause(): raise NotImplementedError(f"Unsupported: SelectStmt.windowClause()") - if ctx.limitByClause(): - raise NotImplementedError(f"Unsupported: SelectStmt.limitByClause()") if ctx.settingsClause(): raise NotImplementedError(f"Unsupported: SelectStmt.settingsClause()") - return ast.SelectQuery( - select=select, - distinct=True if ctx.DISTINCT() else None, - select_from=select_from, - where=where, - prewhere=prewhere, - having=having, - group_by=group_by, - order_by=order_by, - limit=limit, - offset=offset, - ) + return select_query def visitWithClause(self, ctx: HogQLParser.WithClauseContext): raise NotImplementedError(f"Unsupported node: WithClause") @@ -141,10 +125,10 @@ def visitProjectionOrderByClause(self, ctx: HogQLParser.ProjectionOrderByClauseC raise NotImplementedError(f"Unsupported node: ProjectionOrderByClause") def visitLimitByClause(self, ctx: HogQLParser.LimitByClauseContext): - raise NotImplementedError(f"Unsupported node: LimitByClause") + raise Exception(f"Parsed as part of SelectStmt, can't parse directly.") def visitLimitClause(self, ctx: HogQLParser.LimitClauseContext): - raise NotImplementedError(f"Unsupported node: LimitClause") + raise Exception(f"Parsed as part of SelectStmt, can't parse directly.") def visitSettingsClause(self, ctx: HogQLParser.SettingsClauseContext): raise NotImplementedError(f"Unsupported node: SettingsClause") diff --git a/posthog/hogql/test/test_parser.py b/posthog/hogql/test/test_parser.py index 2c6b34a4e5719..55d943a1d197e 100644 --- a/posthog/hogql/test/test_parser.py +++ b/posthog/hogql/test/test_parser.py @@ -554,7 +554,7 @@ def test_select_limit_offset(self): ast.SelectQuery( select=[ast.Constant(value=1)], select_from=ast.JoinExpr(table=ast.Field(chain=["events"])), - limit=1, + limit=ast.Constant(value=1), ), ) self.assertEqual( @@ -562,8 +562,28 @@ def test_select_limit_offset(self): ast.SelectQuery( select=[ast.Constant(value=1)], select_from=ast.JoinExpr(table=ast.Field(chain=["events"])), - limit=1, - offset=3, + limit=ast.Constant(value=1), + offset=ast.Constant(value=3), + ), + ) + self.assertEqual( + parse_select("select 1 from events LIMIT 1 OFFSET 3 WITH TIES"), + ast.SelectQuery( + select=[ast.Constant(value=1)], + select_from=ast.JoinExpr(table=ast.Field(chain=["events"])), + limit=ast.Constant(value=1), + limit_with_ties=True, + offset=ast.Constant(value=3), + ), + ) + self.assertEqual( + parse_select("select 1 from events LIMIT 1 OFFSET 3 BY 1, event"), + ast.SelectQuery( + select=[ast.Constant(value=1)], + select_from=ast.JoinExpr(table=ast.Field(chain=["events"])), + limit=ast.Constant(value=1), + offset=ast.Constant(value=3), + limit_by=[ast.Constant(value=1), ast.Field(chain=["event"])], ), ) From 839d5059b60a3fb5efab513c44b84741d4f2675c Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 8 Feb 2023 16:16:18 +0100 Subject: [PATCH 05/81] parse limit by --- posthog/hogql/printer.py | 20 +++++++++++++++----- posthog/hogql/test/test_printer.py | 19 +++++++++++++++++++ posthog/hogql/test/test_visitor.py | 6 ++++-- posthog/hogql/visitor.py | 6 ++++-- 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index d07599d080e21..e2ebe83dc4d23 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -71,9 +71,12 @@ def print_ast( limit = node.limit if context.limit_top_select: if limit is not None: - limit = max(0, min(node.limit, MAX_SELECT_RETURNED_ROWS)) - if len(stack) == 1 and limit is None: - limit = MAX_SELECT_RETURNED_ROWS + if isinstance(limit, ast.Constant) and isinstance(limit.value, int): + limit.value = min(limit.value, MAX_SELECT_RETURNED_ROWS) + else: + limit = ast.Call(name="min2", args=[ast.Constant(value=MAX_SELECT_RETURNED_ROWS), limit]) + elif len(stack) == 1: + limit = ast.Constant(value=MAX_SELECT_RETURNED_ROWS) clauses = [ f"SELECT {'DISTINCT ' if node.distinct else ''}{', '.join(columns)}", @@ -83,9 +86,16 @@ def print_ast( "HAVING " + having if having else None, "PREWHERE " + prewhere if prewhere else None, f"ORDER BY {', '.join(order_by)}" if order_by and len(order_by) > 0 else None, - f"LIMIT {limit}" if limit is not None else None, - f"OFFSET {node.offset}" if node.offset is not None else None, ] + if limit is not None: + clauses.append(f"LIMIT {print_ast(limit, stack, context, dialect)}"), + if node.offset is not None: + clauses.append(f"OFFSET {print_ast(node.offset, stack, context, dialect)}") + if node.limit_by is not None: + clauses.append(f"BY {', '.join([print_ast(expr, stack, context, dialect) for expr in node.limit_by])}") + if node.limit_with_ties: + clauses.append("WITH TIES") + response = " ".join([clause for clause in clauses if clause]) if len(stack) > 1: response = f"({response})" diff --git a/posthog/hogql/test/test_printer.py b/posthog/hogql/test/test_printer.py index ded3198704c69..e1e67e8106497 100644 --- a/posthog/hogql/test/test_printer.py +++ b/posthog/hogql/test/test_printer.py @@ -405,6 +405,15 @@ def test_select_limit(self): self._select("select event from events limit 10000000"), "SELECT event FROM events WHERE equals(team_id, 42) LIMIT 65535", ) + self.assertEqual( + self._select("select event from events limit (select 1000000000)"), + "SELECT event FROM events WHERE equals(team_id, 42) LIMIT min2(65535, (SELECT 1000000000))", + ) + + self.assertEqual( + self._select("select event from events limit (select 1000000000) with ties"), + "SELECT event FROM events WHERE equals(team_id, 42) LIMIT min2(65535, (SELECT 1000000000)) WITH TIES", + ) def test_select_offset(self): self.assertEqual( @@ -415,6 +424,16 @@ def test_select_offset(self): self._select("select event from events limit 10 offset 0"), "SELECT event FROM events WHERE equals(team_id, 42) LIMIT 10 OFFSET 0", ) + self.assertEqual( + self._select("select event from events limit 10 offset 0 with ties"), + "SELECT event FROM events WHERE equals(team_id, 42) LIMIT 10 OFFSET 0 WITH TIES", + ) + + def test_select_limit_by(self): + self.assertEqual( + self._select("select event from events limit 10 offset 0 by 1,event"), + "SELECT event FROM events WHERE equals(team_id, 42) LIMIT 10 OFFSET 0 BY 1, event", + ) def test_select_group_by(self): self.assertEqual( diff --git a/posthog/hogql/test/test_visitor.py b/posthog/hogql/test/test_visitor.py index f60812f53aab8..738b4ac3aed12 100644 --- a/posthog/hogql/test/test_visitor.py +++ b/posthog/hogql/test/test_visitor.py @@ -85,8 +85,10 @@ def test_everything_visitor(self): having=ast.Constant(value=True), group_by=[ast.Constant(value=True)], order_by=[ast.OrderExpr(expr=ast.Constant(value=True), order="DESC")], - limit=1, - offset=0, + limit=ast.Constant(value=1), + limit_by=[ast.Constant(value=True)], + limit_with_ties=True, + offset=ast.Or(exprs=[ast.Constant(value=1)]), distinct=True, ), ] diff --git a/posthog/hogql/visitor.py b/posthog/hogql/visitor.py index 1bca4d05a78f4..66365f368bbc1 100644 --- a/posthog/hogql/visitor.py +++ b/posthog/hogql/visitor.py @@ -81,7 +81,9 @@ def visit_select_query(self, node: ast.SelectQuery): having=self.visit(node.having), group_by=[self.visit(expr) for expr in node.group_by] if node.group_by else None, order_by=[self.visit(expr) for expr in node.order_by] if node.order_by else None, - limit=node.limit, - offset=node.offset, + limit_by=[self.visit(expr) for expr in node.limit_by] if node.limit_by else None, + limit=self.visit(node.limit), + limit_with_ties=node.limit_with_ties, + offset=self.visit(node.offset), distinct=node.distinct, ) From fef94635ccaa2fc29568f8b11d15e21d4fb3d668 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 8 Feb 2023 16:55:45 +0100 Subject: [PATCH 06/81] merge limit clauses --- posthog/hogql/grammar/HogQLParser.g4 | 4 +- posthog/hogql/grammar/HogQLParser.interp | 3 +- posthog/hogql/grammar/HogQLParser.py | 2164 +++++++++---------- posthog/hogql/grammar/HogQLParserVisitor.py | 5 - posthog/hogql/parser.py | 17 +- posthog/hogql/printer.py | 2 +- 6 files changed, 1063 insertions(+), 1132 deletions(-) diff --git a/posthog/hogql/grammar/HogQLParser.g4 b/posthog/hogql/grammar/HogQLParser.g4 index a2726834f1bca..757cb726f5ce7 100644 --- a/posthog/hogql/grammar/HogQLParser.g4 +++ b/posthog/hogql/grammar/HogQLParser.g4 @@ -21,7 +21,6 @@ selectStmt: groupByClause? (WITH (CUBE | ROLLUP))? (WITH TOTALS)? havingClause? orderByClause? - limitByClause? limitClause? settingsClause? ; @@ -37,8 +36,7 @@ groupByClause: GROUP BY ((CUBE | ROLLUP) LPAREN columnExprList RPAREN | columnEx havingClause: HAVING columnExpr; orderByClause: ORDER BY orderExprList; projectionOrderByClause: ORDER BY columnExprList; -limitByClause: LIMIT limitExpr BY columnExprList; -limitClause: LIMIT limitExpr (WITH TIES)?; +limitClause: LIMIT limitExpr ((WITH TIES) | BY columnExprList)?; settingsClause: SETTINGS settingExprList; joinExpr diff --git a/posthog/hogql/grammar/HogQLParser.interp b/posthog/hogql/grammar/HogQLParser.interp index a48298cae9931..e9aab592cff5b 100644 --- a/posthog/hogql/grammar/HogQLParser.interp +++ b/posthog/hogql/grammar/HogQLParser.interp @@ -488,7 +488,6 @@ groupByClause havingClause orderByClause projectionOrderByClause -limitByClause limitClause settingsClause joinExpr @@ -537,4 +536,4 @@ enumValue atn: -[4, 1, 234, 891, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 2, 60, 7, 60, 1, 0, 1, 0, 3, 0, 125, 8, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 133, 8, 1, 10, 1, 12, 1, 136, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 143, 8, 2, 1, 3, 3, 3, 146, 8, 3, 1, 3, 1, 3, 3, 3, 150, 8, 3, 1, 3, 3, 3, 153, 8, 3, 1, 3, 1, 3, 3, 3, 157, 8, 3, 1, 3, 3, 3, 160, 8, 3, 1, 3, 3, 3, 163, 8, 3, 1, 3, 3, 3, 166, 8, 3, 1, 3, 3, 3, 169, 8, 3, 1, 3, 3, 3, 172, 8, 3, 1, 3, 1, 3, 3, 3, 176, 8, 3, 1, 3, 1, 3, 3, 3, 180, 8, 3, 1, 3, 3, 3, 183, 8, 3, 1, 3, 3, 3, 186, 8, 3, 1, 3, 3, 3, 189, 8, 3, 1, 3, 3, 3, 192, 8, 3, 1, 3, 3, 3, 195, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 204, 8, 5, 1, 6, 1, 6, 1, 6, 1, 7, 3, 7, 210, 8, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 3, 11, 237, 8, 11, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 16, 1, 16, 1, 16, 1, 16, 3, 16, 259, 8, 16, 1, 17, 1, 17, 1, 17, 1, 18, 1, 18, 1, 18, 3, 18, 267, 8, 18, 1, 18, 3, 18, 270, 8, 18, 1, 18, 1, 18, 1, 18, 1, 18, 3, 18, 276, 8, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 1, 18, 3, 18, 284, 8, 18, 1, 18, 3, 18, 287, 8, 18, 1, 18, 1, 18, 1, 18, 1, 18, 5, 18, 293, 8, 18, 10, 18, 12, 18, 296, 9, 18, 1, 19, 3, 19, 299, 8, 19, 1, 19, 1, 19, 1, 19, 3, 19, 304, 8, 19, 1, 19, 3, 19, 307, 8, 19, 1, 19, 3, 19, 310, 8, 19, 1, 19, 1, 19, 3, 19, 314, 8, 19, 1, 19, 1, 19, 3, 19, 318, 8, 19, 1, 19, 3, 19, 321, 8, 19, 3, 19, 323, 8, 19, 1, 19, 3, 19, 326, 8, 19, 1, 19, 1, 19, 3, 19, 330, 8, 19, 1, 19, 1, 19, 3, 19, 334, 8, 19, 1, 19, 3, 19, 337, 8, 19, 3, 19, 339, 8, 19, 3, 19, 341, 8, 19, 1, 20, 3, 20, 344, 8, 20, 1, 20, 1, 20, 1, 20, 3, 20, 349, 8, 20, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 1, 21, 3, 21, 360, 8, 21, 1, 22, 1, 22, 1, 22, 1, 22, 3, 22, 366, 8, 22, 1, 23, 1, 23, 1, 23, 3, 23, 371, 8, 23, 1, 24, 1, 24, 1, 24, 5, 24, 376, 8, 24, 10, 24, 12, 24, 379, 9, 24, 1, 25, 1, 25, 3, 25, 383, 8, 25, 1, 25, 1, 25, 3, 25, 387, 8, 25, 1, 25, 1, 25, 3, 25, 391, 8, 25, 1, 26, 1, 26, 1, 26, 3, 26, 396, 8, 26, 1, 27, 1, 27, 1, 27, 5, 27, 401, 8, 27, 10, 27, 12, 27, 404, 9, 27, 1, 28, 1, 28, 1, 28, 1, 28, 1, 29, 3, 29, 411, 8, 29, 1, 29, 3, 29, 414, 8, 29, 1, 29, 3, 29, 417, 8, 29, 1, 30, 1, 30, 1, 30, 1, 30, 1, 31, 1, 31, 1, 31, 1, 31, 1, 32, 1, 32, 1, 32, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 3, 33, 436, 8, 33, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 1, 34, 3, 34, 450, 8, 34, 1, 35, 1, 35, 1, 35, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 5, 36, 464, 8, 36, 10, 36, 12, 36, 467, 9, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 5, 36, 476, 8, 36, 10, 36, 12, 36, 479, 9, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 5, 36, 488, 8, 36, 10, 36, 12, 36, 491, 9, 36, 1, 36, 1, 36, 1, 36, 1, 36, 1, 36, 3, 36, 498, 8, 36, 1, 36, 1, 36, 3, 36, 502, 8, 36, 1, 37, 1, 37, 1, 37, 5, 37, 507, 8, 37, 10, 37, 12, 37, 510, 9, 37, 1, 38, 1, 38, 1, 38, 3, 38, 515, 8, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 3, 38, 523, 8, 38, 1, 39, 1, 39, 1, 39, 3, 39, 528, 8, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 4, 39, 535, 8, 39, 11, 39, 12, 39, 536, 1, 39, 1, 39, 3, 39, 541, 8, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 3, 39, 572, 8, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 3, 39, 589, 8, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 3, 39, 601, 8, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 3, 39, 611, 8, 39, 1, 39, 3, 39, 614, 8, 39, 1, 39, 1, 39, 3, 39, 618, 8, 39, 1, 39, 3, 39, 621, 8, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 3, 39, 633, 8, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 3, 39, 650, 8, 39, 1, 39, 1, 39, 3, 39, 654, 8, 39, 1, 39, 1, 39, 1, 39, 1, 39, 3, 39, 660, 8, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 3, 39, 667, 8, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 3, 39, 679, 8, 39, 1, 39, 3, 39, 682, 8, 39, 1, 39, 1, 39, 3, 39, 686, 8, 39, 1, 39, 3, 39, 689, 8, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 3, 39, 700, 8, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 3, 39, 724, 8, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 1, 39, 3, 39, 733, 8, 39, 5, 39, 735, 8, 39, 10, 39, 12, 39, 738, 9, 39, 1, 40, 1, 40, 1, 40, 5, 40, 743, 8, 40, 10, 40, 12, 40, 746, 9, 40, 1, 41, 1, 41, 3, 41, 750, 8, 41, 1, 42, 1, 42, 1, 42, 1, 42, 5, 42, 756, 8, 42, 10, 42, 12, 42, 759, 9, 42, 1, 42, 1, 42, 1, 42, 1, 42, 1, 42, 5, 42, 766, 8, 42, 10, 42, 12, 42, 769, 9, 42, 3, 42, 771, 8, 42, 1, 42, 1, 42, 1, 42, 1, 43, 1, 43, 1, 43, 1, 43, 3, 43, 780, 8, 43, 1, 43, 3, 43, 783, 8, 43, 1, 44, 1, 44, 1, 44, 5, 44, 788, 8, 44, 10, 44, 12, 44, 791, 9, 44, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 1, 45, 3, 45, 800, 8, 45, 1, 45, 1, 45, 1, 45, 1, 45, 3, 45, 806, 8, 45, 5, 45, 808, 8, 45, 10, 45, 12, 45, 811, 9, 45, 1, 46, 1, 46, 1, 46, 3, 46, 816, 8, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 3, 47, 823, 8, 47, 1, 47, 1, 47, 1, 48, 1, 48, 1, 48, 5, 48, 830, 8, 48, 10, 48, 12, 48, 833, 9, 48, 1, 49, 1, 49, 1, 49, 3, 49, 838, 8, 49, 1, 50, 1, 50, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 3, 51, 848, 8, 51, 3, 51, 850, 8, 51, 1, 52, 3, 52, 853, 8, 52, 1, 52, 1, 52, 1, 52, 1, 52, 1, 52, 1, 52, 3, 52, 861, 8, 52, 1, 53, 1, 53, 1, 53, 3, 53, 866, 8, 53, 1, 54, 1, 54, 1, 55, 1, 55, 1, 56, 1, 56, 1, 57, 1, 57, 3, 57, 876, 8, 57, 1, 58, 1, 58, 1, 58, 3, 58, 881, 8, 58, 1, 59, 1, 59, 3, 59, 885, 8, 59, 1, 60, 1, 60, 1, 60, 1, 60, 1, 60, 0, 3, 36, 78, 90, 61, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 120, 0, 18, 2, 0, 31, 31, 140, 140, 2, 0, 83, 83, 95, 95, 2, 0, 70, 70, 100, 100, 3, 0, 4, 4, 8, 8, 12, 12, 4, 0, 4, 4, 7, 8, 12, 12, 146, 146, 2, 0, 95, 95, 139, 139, 2, 0, 4, 4, 8, 8, 2, 0, 117, 117, 205, 205, 2, 0, 11, 11, 41, 42, 2, 0, 61, 61, 92, 92, 2, 0, 132, 132, 142, 142, 3, 0, 17, 17, 94, 94, 169, 169, 2, 0, 78, 78, 97, 97, 1, 0, 195, 196, 2, 0, 207, 207, 222, 222, 8, 0, 36, 36, 75, 75, 107, 107, 109, 109, 131, 131, 144, 144, 184, 184, 189, 189, 12, 0, 2, 35, 37, 74, 76, 80, 82, 106, 108, 108, 110, 111, 113, 114, 116, 129, 132, 143, 145, 183, 185, 188, 190, 191, 4, 0, 35, 35, 61, 61, 76, 76, 90, 90, 1000, 0, 124, 1, 0, 0, 0, 2, 128, 1, 0, 0, 0, 4, 142, 1, 0, 0, 0, 6, 145, 1, 0, 0, 0, 8, 196, 1, 0, 0, 0, 10, 199, 1, 0, 0, 0, 12, 205, 1, 0, 0, 0, 14, 209, 1, 0, 0, 0, 16, 215, 1, 0, 0, 0, 18, 222, 1, 0, 0, 0, 20, 225, 1, 0, 0, 0, 22, 228, 1, 0, 0, 0, 24, 238, 1, 0, 0, 0, 26, 241, 1, 0, 0, 0, 28, 245, 1, 0, 0, 0, 30, 249, 1, 0, 0, 0, 32, 254, 1, 0, 0, 0, 34, 260, 1, 0, 0, 0, 36, 275, 1, 0, 0, 0, 38, 340, 1, 0, 0, 0, 40, 348, 1, 0, 0, 0, 42, 359, 1, 0, 0, 0, 44, 361, 1, 0, 0, 0, 46, 367, 1, 0, 0, 0, 48, 372, 1, 0, 0, 0, 50, 380, 1, 0, 0, 0, 52, 392, 1, 0, 0, 0, 54, 397, 1, 0, 0, 0, 56, 405, 1, 0, 0, 0, 58, 410, 1, 0, 0, 0, 60, 418, 1, 0, 0, 0, 62, 422, 1, 0, 0, 0, 64, 426, 1, 0, 0, 0, 66, 435, 1, 0, 0, 0, 68, 449, 1, 0, 0, 0, 70, 451, 1, 0, 0, 0, 72, 501, 1, 0, 0, 0, 74, 503, 1, 0, 0, 0, 76, 522, 1, 0, 0, 0, 78, 653, 1, 0, 0, 0, 80, 739, 1, 0, 0, 0, 82, 749, 1, 0, 0, 0, 84, 770, 1, 0, 0, 0, 86, 782, 1, 0, 0, 0, 88, 784, 1, 0, 0, 0, 90, 799, 1, 0, 0, 0, 92, 812, 1, 0, 0, 0, 94, 822, 1, 0, 0, 0, 96, 826, 1, 0, 0, 0, 98, 837, 1, 0, 0, 0, 100, 839, 1, 0, 0, 0, 102, 849, 1, 0, 0, 0, 104, 852, 1, 0, 0, 0, 106, 865, 1, 0, 0, 0, 108, 867, 1, 0, 0, 0, 110, 869, 1, 0, 0, 0, 112, 871, 1, 0, 0, 0, 114, 875, 1, 0, 0, 0, 116, 880, 1, 0, 0, 0, 118, 884, 1, 0, 0, 0, 120, 886, 1, 0, 0, 0, 122, 125, 3, 2, 1, 0, 123, 125, 3, 6, 3, 0, 124, 122, 1, 0, 0, 0, 124, 123, 1, 0, 0, 0, 125, 126, 1, 0, 0, 0, 126, 127, 5, 0, 0, 1, 127, 1, 1, 0, 0, 0, 128, 134, 3, 4, 2, 0, 129, 130, 5, 175, 0, 0, 130, 131, 5, 4, 0, 0, 131, 133, 3, 4, 2, 0, 132, 129, 1, 0, 0, 0, 133, 136, 1, 0, 0, 0, 134, 132, 1, 0, 0, 0, 134, 135, 1, 0, 0, 0, 135, 3, 1, 0, 0, 0, 136, 134, 1, 0, 0, 0, 137, 143, 3, 6, 3, 0, 138, 139, 5, 218, 0, 0, 139, 140, 3, 2, 1, 0, 140, 141, 5, 228, 0, 0, 141, 143, 1, 0, 0, 0, 142, 137, 1, 0, 0, 0, 142, 138, 1, 0, 0, 0, 143, 5, 1, 0, 0, 0, 144, 146, 3, 8, 4, 0, 145, 144, 1, 0, 0, 0, 145, 146, 1, 0, 0, 0, 146, 147, 1, 0, 0, 0, 147, 149, 5, 145, 0, 0, 148, 150, 5, 48, 0, 0, 149, 148, 1, 0, 0, 0, 149, 150, 1, 0, 0, 0, 150, 152, 1, 0, 0, 0, 151, 153, 3, 10, 5, 0, 152, 151, 1, 0, 0, 0, 152, 153, 1, 0, 0, 0, 153, 154, 1, 0, 0, 0, 154, 156, 3, 74, 37, 0, 155, 157, 3, 12, 6, 0, 156, 155, 1, 0, 0, 0, 156, 157, 1, 0, 0, 0, 157, 159, 1, 0, 0, 0, 158, 160, 3, 14, 7, 0, 159, 158, 1, 0, 0, 0, 159, 160, 1, 0, 0, 0, 160, 162, 1, 0, 0, 0, 161, 163, 3, 16, 8, 0, 162, 161, 1, 0, 0, 0, 162, 163, 1, 0, 0, 0, 163, 165, 1, 0, 0, 0, 164, 166, 3, 18, 9, 0, 165, 164, 1, 0, 0, 0, 165, 166, 1, 0, 0, 0, 166, 168, 1, 0, 0, 0, 167, 169, 3, 20, 10, 0, 168, 167, 1, 0, 0, 0, 168, 169, 1, 0, 0, 0, 169, 171, 1, 0, 0, 0, 170, 172, 3, 22, 11, 0, 171, 170, 1, 0, 0, 0, 171, 172, 1, 0, 0, 0, 172, 175, 1, 0, 0, 0, 173, 174, 5, 188, 0, 0, 174, 176, 7, 0, 0, 0, 175, 173, 1, 0, 0, 0, 175, 176, 1, 0, 0, 0, 176, 179, 1, 0, 0, 0, 177, 178, 5, 188, 0, 0, 178, 180, 5, 168, 0, 0, 179, 177, 1, 0, 0, 0, 179, 180, 1, 0, 0, 0, 180, 182, 1, 0, 0, 0, 181, 183, 3, 24, 12, 0, 182, 181, 1, 0, 0, 0, 182, 183, 1, 0, 0, 0, 183, 185, 1, 0, 0, 0, 184, 186, 3, 26, 13, 0, 185, 184, 1, 0, 0, 0, 185, 186, 1, 0, 0, 0, 186, 188, 1, 0, 0, 0, 187, 189, 3, 30, 15, 0, 188, 187, 1, 0, 0, 0, 188, 189, 1, 0, 0, 0, 189, 191, 1, 0, 0, 0, 190, 192, 3, 32, 16, 0, 191, 190, 1, 0, 0, 0, 191, 192, 1, 0, 0, 0, 192, 194, 1, 0, 0, 0, 193, 195, 3, 34, 17, 0, 194, 193, 1, 0, 0, 0, 194, 195, 1, 0, 0, 0, 195, 7, 1, 0, 0, 0, 196, 197, 5, 188, 0, 0, 197, 198, 3, 74, 37, 0, 198, 9, 1, 0, 0, 0, 199, 200, 5, 167, 0, 0, 200, 203, 5, 196, 0, 0, 201, 202, 5, 188, 0, 0, 202, 204, 5, 163, 0, 0, 203, 201, 1, 0, 0, 0, 203, 204, 1, 0, 0, 0, 204, 11, 1, 0, 0, 0, 205, 206, 5, 67, 0, 0, 206, 207, 3, 36, 18, 0, 207, 13, 1, 0, 0, 0, 208, 210, 7, 1, 0, 0, 209, 208, 1, 0, 0, 0, 209, 210, 1, 0, 0, 0, 210, 211, 1, 0, 0, 0, 211, 212, 5, 9, 0, 0, 212, 213, 5, 89, 0, 0, 213, 214, 3, 74, 37, 0, 214, 15, 1, 0, 0, 0, 215, 216, 5, 187, 0, 0, 216, 217, 3, 116, 58, 0, 217, 218, 5, 10, 0, 0, 218, 219, 5, 218, 0, 0, 219, 220, 3, 58, 29, 0, 220, 221, 5, 228, 0, 0, 221, 17, 1, 0, 0, 0, 222, 223, 5, 128, 0, 0, 223, 224, 3, 78, 39, 0, 224, 19, 1, 0, 0, 0, 225, 226, 5, 186, 0, 0, 226, 227, 3, 78, 39, 0, 227, 21, 1, 0, 0, 0, 228, 229, 5, 72, 0, 0, 229, 236, 5, 18, 0, 0, 230, 231, 7, 0, 0, 0, 231, 232, 5, 218, 0, 0, 232, 233, 3, 74, 37, 0, 233, 234, 5, 228, 0, 0, 234, 237, 1, 0, 0, 0, 235, 237, 3, 74, 37, 0, 236, 230, 1, 0, 0, 0, 236, 235, 1, 0, 0, 0, 237, 23, 1, 0, 0, 0, 238, 239, 5, 73, 0, 0, 239, 240, 3, 78, 39, 0, 240, 25, 1, 0, 0, 0, 241, 242, 5, 121, 0, 0, 242, 243, 5, 18, 0, 0, 243, 244, 3, 48, 24, 0, 244, 27, 1, 0, 0, 0, 245, 246, 5, 121, 0, 0, 246, 247, 5, 18, 0, 0, 247, 248, 3, 74, 37, 0, 248, 29, 1, 0, 0, 0, 249, 250, 5, 98, 0, 0, 250, 251, 3, 46, 23, 0, 251, 252, 5, 18, 0, 0, 252, 253, 3, 74, 37, 0, 253, 31, 1, 0, 0, 0, 254, 255, 5, 98, 0, 0, 255, 258, 3, 46, 23, 0, 256, 257, 5, 188, 0, 0, 257, 259, 5, 163, 0, 0, 258, 256, 1, 0, 0, 0, 258, 259, 1, 0, 0, 0, 259, 33, 1, 0, 0, 0, 260, 261, 5, 149, 0, 0, 261, 262, 3, 54, 27, 0, 262, 35, 1, 0, 0, 0, 263, 264, 6, 18, -1, 0, 264, 266, 3, 90, 45, 0, 265, 267, 5, 60, 0, 0, 266, 265, 1, 0, 0, 0, 266, 267, 1, 0, 0, 0, 267, 269, 1, 0, 0, 0, 268, 270, 3, 44, 22, 0, 269, 268, 1, 0, 0, 0, 269, 270, 1, 0, 0, 0, 270, 276, 1, 0, 0, 0, 271, 272, 5, 218, 0, 0, 272, 273, 3, 36, 18, 0, 273, 274, 5, 228, 0, 0, 274, 276, 1, 0, 0, 0, 275, 263, 1, 0, 0, 0, 275, 271, 1, 0, 0, 0, 276, 294, 1, 0, 0, 0, 277, 278, 10, 3, 0, 0, 278, 279, 3, 40, 20, 0, 279, 280, 3, 36, 18, 4, 280, 293, 1, 0, 0, 0, 281, 283, 10, 4, 0, 0, 282, 284, 7, 2, 0, 0, 283, 282, 1, 0, 0, 0, 283, 284, 1, 0, 0, 0, 284, 286, 1, 0, 0, 0, 285, 287, 3, 38, 19, 0, 286, 285, 1, 0, 0, 0, 286, 287, 1, 0, 0, 0, 287, 288, 1, 0, 0, 0, 288, 289, 5, 89, 0, 0, 289, 290, 3, 36, 18, 0, 290, 291, 3, 42, 21, 0, 291, 293, 1, 0, 0, 0, 292, 277, 1, 0, 0, 0, 292, 281, 1, 0, 0, 0, 293, 296, 1, 0, 0, 0, 294, 292, 1, 0, 0, 0, 294, 295, 1, 0, 0, 0, 295, 37, 1, 0, 0, 0, 296, 294, 1, 0, 0, 0, 297, 299, 7, 3, 0, 0, 298, 297, 1, 0, 0, 0, 298, 299, 1, 0, 0, 0, 299, 300, 1, 0, 0, 0, 300, 307, 5, 83, 0, 0, 301, 303, 5, 83, 0, 0, 302, 304, 7, 3, 0, 0, 303, 302, 1, 0, 0, 0, 303, 304, 1, 0, 0, 0, 304, 307, 1, 0, 0, 0, 305, 307, 7, 3, 0, 0, 306, 298, 1, 0, 0, 0, 306, 301, 1, 0, 0, 0, 306, 305, 1, 0, 0, 0, 307, 341, 1, 0, 0, 0, 308, 310, 7, 4, 0, 0, 309, 308, 1, 0, 0, 0, 309, 310, 1, 0, 0, 0, 310, 311, 1, 0, 0, 0, 311, 313, 7, 5, 0, 0, 312, 314, 5, 122, 0, 0, 313, 312, 1, 0, 0, 0, 313, 314, 1, 0, 0, 0, 314, 323, 1, 0, 0, 0, 315, 317, 7, 5, 0, 0, 316, 318, 5, 122, 0, 0, 317, 316, 1, 0, 0, 0, 317, 318, 1, 0, 0, 0, 318, 320, 1, 0, 0, 0, 319, 321, 7, 4, 0, 0, 320, 319, 1, 0, 0, 0, 320, 321, 1, 0, 0, 0, 321, 323, 1, 0, 0, 0, 322, 309, 1, 0, 0, 0, 322, 315, 1, 0, 0, 0, 323, 341, 1, 0, 0, 0, 324, 326, 7, 6, 0, 0, 325, 324, 1, 0, 0, 0, 325, 326, 1, 0, 0, 0, 326, 327, 1, 0, 0, 0, 327, 329, 5, 68, 0, 0, 328, 330, 5, 122, 0, 0, 329, 328, 1, 0, 0, 0, 329, 330, 1, 0, 0, 0, 330, 339, 1, 0, 0, 0, 331, 333, 5, 68, 0, 0, 332, 334, 5, 122, 0, 0, 333, 332, 1, 0, 0, 0, 333, 334, 1, 0, 0, 0, 334, 336, 1, 0, 0, 0, 335, 337, 7, 6, 0, 0, 336, 335, 1, 0, 0, 0, 336, 337, 1, 0, 0, 0, 337, 339, 1, 0, 0, 0, 338, 325, 1, 0, 0, 0, 338, 331, 1, 0, 0, 0, 339, 341, 1, 0, 0, 0, 340, 306, 1, 0, 0, 0, 340, 322, 1, 0, 0, 0, 340, 338, 1, 0, 0, 0, 341, 39, 1, 0, 0, 0, 342, 344, 7, 2, 0, 0, 343, 342, 1, 0, 0, 0, 343, 344, 1, 0, 0, 0, 344, 345, 1, 0, 0, 0, 345, 346, 5, 30, 0, 0, 346, 349, 5, 89, 0, 0, 347, 349, 5, 205, 0, 0, 348, 343, 1, 0, 0, 0, 348, 347, 1, 0, 0, 0, 349, 41, 1, 0, 0, 0, 350, 351, 5, 118, 0, 0, 351, 360, 3, 74, 37, 0, 352, 353, 5, 178, 0, 0, 353, 354, 5, 218, 0, 0, 354, 355, 3, 74, 37, 0, 355, 356, 5, 228, 0, 0, 356, 360, 1, 0, 0, 0, 357, 358, 5, 178, 0, 0, 358, 360, 3, 74, 37, 0, 359, 350, 1, 0, 0, 0, 359, 352, 1, 0, 0, 0, 359, 357, 1, 0, 0, 0, 360, 43, 1, 0, 0, 0, 361, 362, 5, 143, 0, 0, 362, 365, 3, 52, 26, 0, 363, 364, 5, 117, 0, 0, 364, 366, 3, 52, 26, 0, 365, 363, 1, 0, 0, 0, 365, 366, 1, 0, 0, 0, 366, 45, 1, 0, 0, 0, 367, 370, 3, 78, 39, 0, 368, 369, 7, 7, 0, 0, 369, 371, 3, 78, 39, 0, 370, 368, 1, 0, 0, 0, 370, 371, 1, 0, 0, 0, 371, 47, 1, 0, 0, 0, 372, 377, 3, 50, 25, 0, 373, 374, 5, 205, 0, 0, 374, 376, 3, 50, 25, 0, 375, 373, 1, 0, 0, 0, 376, 379, 1, 0, 0, 0, 377, 375, 1, 0, 0, 0, 377, 378, 1, 0, 0, 0, 378, 49, 1, 0, 0, 0, 379, 377, 1, 0, 0, 0, 380, 382, 3, 78, 39, 0, 381, 383, 7, 8, 0, 0, 382, 381, 1, 0, 0, 0, 382, 383, 1, 0, 0, 0, 383, 386, 1, 0, 0, 0, 384, 385, 5, 116, 0, 0, 385, 387, 7, 9, 0, 0, 386, 384, 1, 0, 0, 0, 386, 387, 1, 0, 0, 0, 387, 390, 1, 0, 0, 0, 388, 389, 5, 25, 0, 0, 389, 391, 5, 198, 0, 0, 390, 388, 1, 0, 0, 0, 390, 391, 1, 0, 0, 0, 391, 51, 1, 0, 0, 0, 392, 395, 3, 104, 52, 0, 393, 394, 5, 230, 0, 0, 394, 396, 3, 104, 52, 0, 395, 393, 1, 0, 0, 0, 395, 396, 1, 0, 0, 0, 396, 53, 1, 0, 0, 0, 397, 402, 3, 56, 28, 0, 398, 399, 5, 205, 0, 0, 399, 401, 3, 56, 28, 0, 400, 398, 1, 0, 0, 0, 401, 404, 1, 0, 0, 0, 402, 400, 1, 0, 0, 0, 402, 403, 1, 0, 0, 0, 403, 55, 1, 0, 0, 0, 404, 402, 1, 0, 0, 0, 405, 406, 3, 116, 58, 0, 406, 407, 5, 211, 0, 0, 407, 408, 3, 106, 53, 0, 408, 57, 1, 0, 0, 0, 409, 411, 3, 60, 30, 0, 410, 409, 1, 0, 0, 0, 410, 411, 1, 0, 0, 0, 411, 413, 1, 0, 0, 0, 412, 414, 3, 62, 31, 0, 413, 412, 1, 0, 0, 0, 413, 414, 1, 0, 0, 0, 414, 416, 1, 0, 0, 0, 415, 417, 3, 64, 32, 0, 416, 415, 1, 0, 0, 0, 416, 417, 1, 0, 0, 0, 417, 59, 1, 0, 0, 0, 418, 419, 5, 125, 0, 0, 419, 420, 5, 18, 0, 0, 420, 421, 3, 74, 37, 0, 421, 61, 1, 0, 0, 0, 422, 423, 5, 121, 0, 0, 423, 424, 5, 18, 0, 0, 424, 425, 3, 48, 24, 0, 425, 63, 1, 0, 0, 0, 426, 427, 7, 10, 0, 0, 427, 428, 3, 66, 33, 0, 428, 65, 1, 0, 0, 0, 429, 436, 3, 68, 34, 0, 430, 431, 5, 16, 0, 0, 431, 432, 3, 68, 34, 0, 432, 433, 5, 6, 0, 0, 433, 434, 3, 68, 34, 0, 434, 436, 1, 0, 0, 0, 435, 429, 1, 0, 0, 0, 435, 430, 1, 0, 0, 0, 436, 67, 1, 0, 0, 0, 437, 438, 5, 32, 0, 0, 438, 450, 5, 141, 0, 0, 439, 440, 5, 174, 0, 0, 440, 450, 5, 127, 0, 0, 441, 442, 5, 174, 0, 0, 442, 450, 5, 63, 0, 0, 443, 444, 3, 104, 52, 0, 444, 445, 5, 127, 0, 0, 445, 450, 1, 0, 0, 0, 446, 447, 3, 104, 52, 0, 447, 448, 5, 63, 0, 0, 448, 450, 1, 0, 0, 0, 449, 437, 1, 0, 0, 0, 449, 439, 1, 0, 0, 0, 449, 441, 1, 0, 0, 0, 449, 443, 1, 0, 0, 0, 449, 446, 1, 0, 0, 0, 450, 69, 1, 0, 0, 0, 451, 452, 3, 78, 39, 0, 452, 453, 5, 0, 0, 1, 453, 71, 1, 0, 0, 0, 454, 502, 3, 116, 58, 0, 455, 456, 3, 116, 58, 0, 456, 457, 5, 218, 0, 0, 457, 458, 3, 116, 58, 0, 458, 465, 3, 72, 36, 0, 459, 460, 5, 205, 0, 0, 460, 461, 3, 116, 58, 0, 461, 462, 3, 72, 36, 0, 462, 464, 1, 0, 0, 0, 463, 459, 1, 0, 0, 0, 464, 467, 1, 0, 0, 0, 465, 463, 1, 0, 0, 0, 465, 466, 1, 0, 0, 0, 466, 468, 1, 0, 0, 0, 467, 465, 1, 0, 0, 0, 468, 469, 5, 228, 0, 0, 469, 502, 1, 0, 0, 0, 470, 471, 3, 116, 58, 0, 471, 472, 5, 218, 0, 0, 472, 477, 3, 120, 60, 0, 473, 474, 5, 205, 0, 0, 474, 476, 3, 120, 60, 0, 475, 473, 1, 0, 0, 0, 476, 479, 1, 0, 0, 0, 477, 475, 1, 0, 0, 0, 477, 478, 1, 0, 0, 0, 478, 480, 1, 0, 0, 0, 479, 477, 1, 0, 0, 0, 480, 481, 5, 228, 0, 0, 481, 502, 1, 0, 0, 0, 482, 483, 3, 116, 58, 0, 483, 484, 5, 218, 0, 0, 484, 489, 3, 72, 36, 0, 485, 486, 5, 205, 0, 0, 486, 488, 3, 72, 36, 0, 487, 485, 1, 0, 0, 0, 488, 491, 1, 0, 0, 0, 489, 487, 1, 0, 0, 0, 489, 490, 1, 0, 0, 0, 490, 492, 1, 0, 0, 0, 491, 489, 1, 0, 0, 0, 492, 493, 5, 228, 0, 0, 493, 502, 1, 0, 0, 0, 494, 495, 3, 116, 58, 0, 495, 497, 5, 218, 0, 0, 496, 498, 3, 74, 37, 0, 497, 496, 1, 0, 0, 0, 497, 498, 1, 0, 0, 0, 498, 499, 1, 0, 0, 0, 499, 500, 5, 228, 0, 0, 500, 502, 1, 0, 0, 0, 501, 454, 1, 0, 0, 0, 501, 455, 1, 0, 0, 0, 501, 470, 1, 0, 0, 0, 501, 482, 1, 0, 0, 0, 501, 494, 1, 0, 0, 0, 502, 73, 1, 0, 0, 0, 503, 508, 3, 76, 38, 0, 504, 505, 5, 205, 0, 0, 505, 507, 3, 76, 38, 0, 506, 504, 1, 0, 0, 0, 507, 510, 1, 0, 0, 0, 508, 506, 1, 0, 0, 0, 508, 509, 1, 0, 0, 0, 509, 75, 1, 0, 0, 0, 510, 508, 1, 0, 0, 0, 511, 512, 3, 94, 47, 0, 512, 513, 5, 209, 0, 0, 513, 515, 1, 0, 0, 0, 514, 511, 1, 0, 0, 0, 514, 515, 1, 0, 0, 0, 515, 516, 1, 0, 0, 0, 516, 523, 5, 201, 0, 0, 517, 518, 5, 218, 0, 0, 518, 519, 3, 2, 1, 0, 519, 520, 5, 228, 0, 0, 520, 523, 1, 0, 0, 0, 521, 523, 3, 78, 39, 0, 522, 514, 1, 0, 0, 0, 522, 517, 1, 0, 0, 0, 522, 521, 1, 0, 0, 0, 523, 77, 1, 0, 0, 0, 524, 525, 6, 39, -1, 0, 525, 527, 5, 19, 0, 0, 526, 528, 3, 78, 39, 0, 527, 526, 1, 0, 0, 0, 527, 528, 1, 0, 0, 0, 528, 534, 1, 0, 0, 0, 529, 530, 5, 185, 0, 0, 530, 531, 3, 78, 39, 0, 531, 532, 5, 162, 0, 0, 532, 533, 3, 78, 39, 0, 533, 535, 1, 0, 0, 0, 534, 529, 1, 0, 0, 0, 535, 536, 1, 0, 0, 0, 536, 534, 1, 0, 0, 0, 536, 537, 1, 0, 0, 0, 537, 540, 1, 0, 0, 0, 538, 539, 5, 51, 0, 0, 539, 541, 3, 78, 39, 0, 540, 538, 1, 0, 0, 0, 540, 541, 1, 0, 0, 0, 541, 542, 1, 0, 0, 0, 542, 543, 5, 52, 0, 0, 543, 654, 1, 0, 0, 0, 544, 545, 5, 20, 0, 0, 545, 546, 5, 218, 0, 0, 546, 547, 3, 78, 39, 0, 547, 548, 5, 10, 0, 0, 548, 549, 3, 72, 36, 0, 549, 550, 5, 228, 0, 0, 550, 654, 1, 0, 0, 0, 551, 552, 5, 35, 0, 0, 552, 654, 5, 198, 0, 0, 553, 554, 5, 58, 0, 0, 554, 555, 5, 218, 0, 0, 555, 556, 3, 108, 54, 0, 556, 557, 5, 67, 0, 0, 557, 558, 3, 78, 39, 0, 558, 559, 5, 228, 0, 0, 559, 654, 1, 0, 0, 0, 560, 561, 5, 85, 0, 0, 561, 562, 3, 78, 39, 0, 562, 563, 3, 108, 54, 0, 563, 654, 1, 0, 0, 0, 564, 565, 5, 154, 0, 0, 565, 566, 5, 218, 0, 0, 566, 567, 3, 78, 39, 0, 567, 568, 5, 67, 0, 0, 568, 571, 3, 78, 39, 0, 569, 570, 5, 64, 0, 0, 570, 572, 3, 78, 39, 0, 571, 569, 1, 0, 0, 0, 571, 572, 1, 0, 0, 0, 572, 573, 1, 0, 0, 0, 573, 574, 5, 228, 0, 0, 574, 654, 1, 0, 0, 0, 575, 576, 5, 165, 0, 0, 576, 654, 5, 198, 0, 0, 577, 578, 5, 170, 0, 0, 578, 579, 5, 218, 0, 0, 579, 580, 7, 11, 0, 0, 580, 581, 5, 198, 0, 0, 581, 582, 5, 67, 0, 0, 582, 583, 3, 78, 39, 0, 583, 584, 5, 228, 0, 0, 584, 654, 1, 0, 0, 0, 585, 586, 3, 116, 58, 0, 586, 588, 5, 218, 0, 0, 587, 589, 3, 74, 37, 0, 588, 587, 1, 0, 0, 0, 588, 589, 1, 0, 0, 0, 589, 590, 1, 0, 0, 0, 590, 591, 5, 228, 0, 0, 591, 592, 1, 0, 0, 0, 592, 593, 5, 124, 0, 0, 593, 594, 5, 218, 0, 0, 594, 595, 3, 58, 29, 0, 595, 596, 5, 228, 0, 0, 596, 654, 1, 0, 0, 0, 597, 598, 3, 116, 58, 0, 598, 600, 5, 218, 0, 0, 599, 601, 3, 74, 37, 0, 600, 599, 1, 0, 0, 0, 600, 601, 1, 0, 0, 0, 601, 602, 1, 0, 0, 0, 602, 603, 5, 228, 0, 0, 603, 604, 1, 0, 0, 0, 604, 605, 5, 124, 0, 0, 605, 606, 3, 116, 58, 0, 606, 654, 1, 0, 0, 0, 607, 613, 3, 116, 58, 0, 608, 610, 5, 218, 0, 0, 609, 611, 3, 74, 37, 0, 610, 609, 1, 0, 0, 0, 610, 611, 1, 0, 0, 0, 611, 612, 1, 0, 0, 0, 612, 614, 5, 228, 0, 0, 613, 608, 1, 0, 0, 0, 613, 614, 1, 0, 0, 0, 614, 615, 1, 0, 0, 0, 615, 617, 5, 218, 0, 0, 616, 618, 5, 48, 0, 0, 617, 616, 1, 0, 0, 0, 617, 618, 1, 0, 0, 0, 618, 620, 1, 0, 0, 0, 619, 621, 3, 80, 40, 0, 620, 619, 1, 0, 0, 0, 620, 621, 1, 0, 0, 0, 621, 622, 1, 0, 0, 0, 622, 623, 5, 228, 0, 0, 623, 654, 1, 0, 0, 0, 624, 654, 3, 106, 53, 0, 625, 626, 5, 207, 0, 0, 626, 654, 3, 78, 39, 17, 627, 628, 5, 114, 0, 0, 628, 654, 3, 78, 39, 12, 629, 630, 3, 94, 47, 0, 630, 631, 5, 209, 0, 0, 631, 633, 1, 0, 0, 0, 632, 629, 1, 0, 0, 0, 632, 633, 1, 0, 0, 0, 633, 634, 1, 0, 0, 0, 634, 654, 5, 201, 0, 0, 635, 636, 5, 218, 0, 0, 636, 637, 3, 2, 1, 0, 637, 638, 5, 228, 0, 0, 638, 654, 1, 0, 0, 0, 639, 640, 5, 218, 0, 0, 640, 641, 3, 78, 39, 0, 641, 642, 5, 228, 0, 0, 642, 654, 1, 0, 0, 0, 643, 644, 5, 218, 0, 0, 644, 645, 3, 74, 37, 0, 645, 646, 5, 228, 0, 0, 646, 654, 1, 0, 0, 0, 647, 649, 5, 216, 0, 0, 648, 650, 3, 74, 37, 0, 649, 648, 1, 0, 0, 0, 649, 650, 1, 0, 0, 0, 650, 651, 1, 0, 0, 0, 651, 654, 5, 227, 0, 0, 652, 654, 3, 86, 43, 0, 653, 524, 1, 0, 0, 0, 653, 544, 1, 0, 0, 0, 653, 551, 1, 0, 0, 0, 653, 553, 1, 0, 0, 0, 653, 560, 1, 0, 0, 0, 653, 564, 1, 0, 0, 0, 653, 575, 1, 0, 0, 0, 653, 577, 1, 0, 0, 0, 653, 585, 1, 0, 0, 0, 653, 597, 1, 0, 0, 0, 653, 607, 1, 0, 0, 0, 653, 624, 1, 0, 0, 0, 653, 625, 1, 0, 0, 0, 653, 627, 1, 0, 0, 0, 653, 632, 1, 0, 0, 0, 653, 635, 1, 0, 0, 0, 653, 639, 1, 0, 0, 0, 653, 643, 1, 0, 0, 0, 653, 647, 1, 0, 0, 0, 653, 652, 1, 0, 0, 0, 654, 736, 1, 0, 0, 0, 655, 659, 10, 16, 0, 0, 656, 660, 5, 201, 0, 0, 657, 660, 5, 230, 0, 0, 658, 660, 5, 221, 0, 0, 659, 656, 1, 0, 0, 0, 659, 657, 1, 0, 0, 0, 659, 658, 1, 0, 0, 0, 660, 661, 1, 0, 0, 0, 661, 735, 3, 78, 39, 17, 662, 666, 10, 15, 0, 0, 663, 667, 5, 222, 0, 0, 664, 667, 5, 207, 0, 0, 665, 667, 5, 206, 0, 0, 666, 663, 1, 0, 0, 0, 666, 664, 1, 0, 0, 0, 666, 665, 1, 0, 0, 0, 667, 668, 1, 0, 0, 0, 668, 735, 3, 78, 39, 16, 669, 688, 10, 14, 0, 0, 670, 689, 5, 210, 0, 0, 671, 689, 5, 211, 0, 0, 672, 689, 5, 220, 0, 0, 673, 689, 5, 217, 0, 0, 674, 689, 5, 212, 0, 0, 675, 689, 5, 219, 0, 0, 676, 689, 5, 213, 0, 0, 677, 679, 5, 70, 0, 0, 678, 677, 1, 0, 0, 0, 678, 679, 1, 0, 0, 0, 679, 681, 1, 0, 0, 0, 680, 682, 5, 114, 0, 0, 681, 680, 1, 0, 0, 0, 681, 682, 1, 0, 0, 0, 682, 683, 1, 0, 0, 0, 683, 689, 5, 79, 0, 0, 684, 686, 5, 114, 0, 0, 685, 684, 1, 0, 0, 0, 685, 686, 1, 0, 0, 0, 686, 687, 1, 0, 0, 0, 687, 689, 7, 12, 0, 0, 688, 670, 1, 0, 0, 0, 688, 671, 1, 0, 0, 0, 688, 672, 1, 0, 0, 0, 688, 673, 1, 0, 0, 0, 688, 674, 1, 0, 0, 0, 688, 675, 1, 0, 0, 0, 688, 676, 1, 0, 0, 0, 688, 678, 1, 0, 0, 0, 688, 685, 1, 0, 0, 0, 689, 690, 1, 0, 0, 0, 690, 735, 3, 78, 39, 15, 691, 692, 10, 11, 0, 0, 692, 693, 5, 6, 0, 0, 693, 735, 3, 78, 39, 12, 694, 695, 10, 10, 0, 0, 695, 696, 5, 120, 0, 0, 696, 735, 3, 78, 39, 11, 697, 699, 10, 9, 0, 0, 698, 700, 5, 114, 0, 0, 699, 698, 1, 0, 0, 0, 699, 700, 1, 0, 0, 0, 700, 701, 1, 0, 0, 0, 701, 702, 5, 16, 0, 0, 702, 703, 3, 78, 39, 0, 703, 704, 5, 6, 0, 0, 704, 705, 3, 78, 39, 10, 705, 735, 1, 0, 0, 0, 706, 707, 10, 8, 0, 0, 707, 708, 5, 223, 0, 0, 708, 709, 3, 78, 39, 0, 709, 710, 5, 204, 0, 0, 710, 711, 3, 78, 39, 8, 711, 735, 1, 0, 0, 0, 712, 713, 10, 19, 0, 0, 713, 714, 5, 216, 0, 0, 714, 715, 3, 78, 39, 0, 715, 716, 5, 227, 0, 0, 716, 735, 1, 0, 0, 0, 717, 718, 10, 18, 0, 0, 718, 719, 5, 209, 0, 0, 719, 735, 5, 196, 0, 0, 720, 721, 10, 13, 0, 0, 721, 723, 5, 87, 0, 0, 722, 724, 5, 114, 0, 0, 723, 722, 1, 0, 0, 0, 723, 724, 1, 0, 0, 0, 724, 725, 1, 0, 0, 0, 725, 735, 5, 115, 0, 0, 726, 732, 10, 7, 0, 0, 727, 733, 3, 114, 57, 0, 728, 729, 5, 10, 0, 0, 729, 733, 3, 116, 58, 0, 730, 731, 5, 10, 0, 0, 731, 733, 5, 198, 0, 0, 732, 727, 1, 0, 0, 0, 732, 728, 1, 0, 0, 0, 732, 730, 1, 0, 0, 0, 733, 735, 1, 0, 0, 0, 734, 655, 1, 0, 0, 0, 734, 662, 1, 0, 0, 0, 734, 669, 1, 0, 0, 0, 734, 691, 1, 0, 0, 0, 734, 694, 1, 0, 0, 0, 734, 697, 1, 0, 0, 0, 734, 706, 1, 0, 0, 0, 734, 712, 1, 0, 0, 0, 734, 717, 1, 0, 0, 0, 734, 720, 1, 0, 0, 0, 734, 726, 1, 0, 0, 0, 735, 738, 1, 0, 0, 0, 736, 734, 1, 0, 0, 0, 736, 737, 1, 0, 0, 0, 737, 79, 1, 0, 0, 0, 738, 736, 1, 0, 0, 0, 739, 744, 3, 82, 41, 0, 740, 741, 5, 205, 0, 0, 741, 743, 3, 82, 41, 0, 742, 740, 1, 0, 0, 0, 743, 746, 1, 0, 0, 0, 744, 742, 1, 0, 0, 0, 744, 745, 1, 0, 0, 0, 745, 81, 1, 0, 0, 0, 746, 744, 1, 0, 0, 0, 747, 750, 3, 84, 42, 0, 748, 750, 3, 78, 39, 0, 749, 747, 1, 0, 0, 0, 749, 748, 1, 0, 0, 0, 750, 83, 1, 0, 0, 0, 751, 752, 5, 218, 0, 0, 752, 757, 3, 116, 58, 0, 753, 754, 5, 205, 0, 0, 754, 756, 3, 116, 58, 0, 755, 753, 1, 0, 0, 0, 756, 759, 1, 0, 0, 0, 757, 755, 1, 0, 0, 0, 757, 758, 1, 0, 0, 0, 758, 760, 1, 0, 0, 0, 759, 757, 1, 0, 0, 0, 760, 761, 5, 228, 0, 0, 761, 771, 1, 0, 0, 0, 762, 767, 3, 116, 58, 0, 763, 764, 5, 205, 0, 0, 764, 766, 3, 116, 58, 0, 765, 763, 1, 0, 0, 0, 766, 769, 1, 0, 0, 0, 767, 765, 1, 0, 0, 0, 767, 768, 1, 0, 0, 0, 768, 771, 1, 0, 0, 0, 769, 767, 1, 0, 0, 0, 770, 751, 1, 0, 0, 0, 770, 762, 1, 0, 0, 0, 771, 772, 1, 0, 0, 0, 772, 773, 5, 200, 0, 0, 773, 774, 3, 78, 39, 0, 774, 85, 1, 0, 0, 0, 775, 783, 5, 199, 0, 0, 776, 777, 3, 94, 47, 0, 777, 778, 5, 209, 0, 0, 778, 780, 1, 0, 0, 0, 779, 776, 1, 0, 0, 0, 779, 780, 1, 0, 0, 0, 780, 781, 1, 0, 0, 0, 781, 783, 3, 88, 44, 0, 782, 775, 1, 0, 0, 0, 782, 779, 1, 0, 0, 0, 783, 87, 1, 0, 0, 0, 784, 789, 3, 116, 58, 0, 785, 786, 5, 209, 0, 0, 786, 788, 3, 116, 58, 0, 787, 785, 1, 0, 0, 0, 788, 791, 1, 0, 0, 0, 789, 787, 1, 0, 0, 0, 789, 790, 1, 0, 0, 0, 790, 89, 1, 0, 0, 0, 791, 789, 1, 0, 0, 0, 792, 793, 6, 45, -1, 0, 793, 800, 3, 94, 47, 0, 794, 800, 3, 92, 46, 0, 795, 796, 5, 218, 0, 0, 796, 797, 3, 2, 1, 0, 797, 798, 5, 228, 0, 0, 798, 800, 1, 0, 0, 0, 799, 792, 1, 0, 0, 0, 799, 794, 1, 0, 0, 0, 799, 795, 1, 0, 0, 0, 800, 809, 1, 0, 0, 0, 801, 805, 10, 1, 0, 0, 802, 806, 3, 114, 57, 0, 803, 804, 5, 10, 0, 0, 804, 806, 3, 116, 58, 0, 805, 802, 1, 0, 0, 0, 805, 803, 1, 0, 0, 0, 806, 808, 1, 0, 0, 0, 807, 801, 1, 0, 0, 0, 808, 811, 1, 0, 0, 0, 809, 807, 1, 0, 0, 0, 809, 810, 1, 0, 0, 0, 810, 91, 1, 0, 0, 0, 811, 809, 1, 0, 0, 0, 812, 813, 3, 116, 58, 0, 813, 815, 5, 218, 0, 0, 814, 816, 3, 96, 48, 0, 815, 814, 1, 0, 0, 0, 815, 816, 1, 0, 0, 0, 816, 817, 1, 0, 0, 0, 817, 818, 5, 228, 0, 0, 818, 93, 1, 0, 0, 0, 819, 820, 3, 100, 50, 0, 820, 821, 5, 209, 0, 0, 821, 823, 1, 0, 0, 0, 822, 819, 1, 0, 0, 0, 822, 823, 1, 0, 0, 0, 823, 824, 1, 0, 0, 0, 824, 825, 3, 116, 58, 0, 825, 95, 1, 0, 0, 0, 826, 831, 3, 98, 49, 0, 827, 828, 5, 205, 0, 0, 828, 830, 3, 98, 49, 0, 829, 827, 1, 0, 0, 0, 830, 833, 1, 0, 0, 0, 831, 829, 1, 0, 0, 0, 831, 832, 1, 0, 0, 0, 832, 97, 1, 0, 0, 0, 833, 831, 1, 0, 0, 0, 834, 838, 3, 88, 44, 0, 835, 838, 3, 92, 46, 0, 836, 838, 3, 106, 53, 0, 837, 834, 1, 0, 0, 0, 837, 835, 1, 0, 0, 0, 837, 836, 1, 0, 0, 0, 838, 99, 1, 0, 0, 0, 839, 840, 3, 116, 58, 0, 840, 101, 1, 0, 0, 0, 841, 850, 5, 194, 0, 0, 842, 843, 5, 209, 0, 0, 843, 850, 7, 13, 0, 0, 844, 845, 5, 196, 0, 0, 845, 847, 5, 209, 0, 0, 846, 848, 7, 13, 0, 0, 847, 846, 1, 0, 0, 0, 847, 848, 1, 0, 0, 0, 848, 850, 1, 0, 0, 0, 849, 841, 1, 0, 0, 0, 849, 842, 1, 0, 0, 0, 849, 844, 1, 0, 0, 0, 850, 103, 1, 0, 0, 0, 851, 853, 7, 14, 0, 0, 852, 851, 1, 0, 0, 0, 852, 853, 1, 0, 0, 0, 853, 860, 1, 0, 0, 0, 854, 861, 3, 102, 51, 0, 855, 861, 5, 195, 0, 0, 856, 861, 5, 196, 0, 0, 857, 861, 5, 197, 0, 0, 858, 861, 5, 81, 0, 0, 859, 861, 5, 112, 0, 0, 860, 854, 1, 0, 0, 0, 860, 855, 1, 0, 0, 0, 860, 856, 1, 0, 0, 0, 860, 857, 1, 0, 0, 0, 860, 858, 1, 0, 0, 0, 860, 859, 1, 0, 0, 0, 861, 105, 1, 0, 0, 0, 862, 866, 3, 104, 52, 0, 863, 866, 5, 198, 0, 0, 864, 866, 5, 115, 0, 0, 865, 862, 1, 0, 0, 0, 865, 863, 1, 0, 0, 0, 865, 864, 1, 0, 0, 0, 866, 107, 1, 0, 0, 0, 867, 868, 7, 15, 0, 0, 868, 109, 1, 0, 0, 0, 869, 870, 7, 16, 0, 0, 870, 111, 1, 0, 0, 0, 871, 872, 7, 17, 0, 0, 872, 113, 1, 0, 0, 0, 873, 876, 5, 193, 0, 0, 874, 876, 3, 112, 56, 0, 875, 873, 1, 0, 0, 0, 875, 874, 1, 0, 0, 0, 876, 115, 1, 0, 0, 0, 877, 881, 5, 193, 0, 0, 878, 881, 3, 108, 54, 0, 879, 881, 3, 110, 55, 0, 880, 877, 1, 0, 0, 0, 880, 878, 1, 0, 0, 0, 880, 879, 1, 0, 0, 0, 881, 117, 1, 0, 0, 0, 882, 885, 3, 116, 58, 0, 883, 885, 5, 115, 0, 0, 884, 882, 1, 0, 0, 0, 884, 883, 1, 0, 0, 0, 885, 119, 1, 0, 0, 0, 886, 887, 5, 198, 0, 0, 887, 888, 5, 211, 0, 0, 888, 889, 3, 104, 52, 0, 889, 121, 1, 0, 0, 0, 115, 124, 134, 142, 145, 149, 152, 156, 159, 162, 165, 168, 171, 175, 179, 182, 185, 188, 191, 194, 203, 209, 236, 258, 266, 269, 275, 283, 286, 292, 294, 298, 303, 306, 309, 313, 317, 320, 322, 325, 329, 333, 336, 338, 340, 343, 348, 359, 365, 370, 377, 382, 386, 390, 395, 402, 410, 413, 416, 435, 449, 465, 477, 489, 497, 501, 508, 514, 522, 527, 536, 540, 571, 588, 600, 610, 613, 617, 620, 632, 649, 653, 659, 666, 678, 681, 685, 688, 699, 723, 732, 734, 736, 744, 749, 757, 767, 770, 779, 782, 789, 799, 805, 809, 815, 822, 831, 837, 847, 849, 852, 860, 865, 875, 880, 884] \ No newline at end of file +[4, 1, 234, 883, 2, 0, 7, 0, 2, 1, 7, 1, 2, 2, 7, 2, 2, 3, 7, 3, 2, 4, 7, 4, 2, 5, 7, 5, 2, 6, 7, 6, 2, 7, 7, 7, 2, 8, 7, 8, 2, 9, 7, 9, 2, 10, 7, 10, 2, 11, 7, 11, 2, 12, 7, 12, 2, 13, 7, 13, 2, 14, 7, 14, 2, 15, 7, 15, 2, 16, 7, 16, 2, 17, 7, 17, 2, 18, 7, 18, 2, 19, 7, 19, 2, 20, 7, 20, 2, 21, 7, 21, 2, 22, 7, 22, 2, 23, 7, 23, 2, 24, 7, 24, 2, 25, 7, 25, 2, 26, 7, 26, 2, 27, 7, 27, 2, 28, 7, 28, 2, 29, 7, 29, 2, 30, 7, 30, 2, 31, 7, 31, 2, 32, 7, 32, 2, 33, 7, 33, 2, 34, 7, 34, 2, 35, 7, 35, 2, 36, 7, 36, 2, 37, 7, 37, 2, 38, 7, 38, 2, 39, 7, 39, 2, 40, 7, 40, 2, 41, 7, 41, 2, 42, 7, 42, 2, 43, 7, 43, 2, 44, 7, 44, 2, 45, 7, 45, 2, 46, 7, 46, 2, 47, 7, 47, 2, 48, 7, 48, 2, 49, 7, 49, 2, 50, 7, 50, 2, 51, 7, 51, 2, 52, 7, 52, 2, 53, 7, 53, 2, 54, 7, 54, 2, 55, 7, 55, 2, 56, 7, 56, 2, 57, 7, 57, 2, 58, 7, 58, 2, 59, 7, 59, 1, 0, 1, 0, 3, 0, 123, 8, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 5, 1, 131, 8, 1, 10, 1, 12, 1, 134, 9, 1, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 3, 2, 141, 8, 2, 1, 3, 3, 3, 144, 8, 3, 1, 3, 1, 3, 3, 3, 148, 8, 3, 1, 3, 3, 3, 151, 8, 3, 1, 3, 1, 3, 3, 3, 155, 8, 3, 1, 3, 3, 3, 158, 8, 3, 1, 3, 3, 3, 161, 8, 3, 1, 3, 3, 3, 164, 8, 3, 1, 3, 3, 3, 167, 8, 3, 1, 3, 3, 3, 170, 8, 3, 1, 3, 1, 3, 3, 3, 174, 8, 3, 1, 3, 1, 3, 3, 3, 178, 8, 3, 1, 3, 3, 3, 181, 8, 3, 1, 3, 3, 3, 184, 8, 3, 1, 3, 3, 3, 187, 8, 3, 1, 3, 3, 3, 190, 8, 3, 1, 4, 1, 4, 1, 4, 1, 5, 1, 5, 1, 5, 1, 5, 3, 5, 199, 8, 5, 1, 6, 1, 6, 1, 6, 1, 7, 3, 7, 205, 8, 7, 1, 7, 1, 7, 1, 7, 1, 7, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 8, 1, 9, 1, 9, 1, 9, 1, 10, 1, 10, 1, 10, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 1, 11, 3, 11, 232, 8, 11, 1, 12, 1, 12, 1, 12, 1, 13, 1, 13, 1, 13, 1, 13, 1, 14, 1, 14, 1, 14, 1, 14, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 1, 15, 3, 15, 251, 8, 15, 1, 16, 1, 16, 1, 16, 1, 17, 1, 17, 1, 17, 3, 17, 259, 8, 17, 1, 17, 3, 17, 262, 8, 17, 1, 17, 1, 17, 1, 17, 1, 17, 3, 17, 268, 8, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 1, 17, 3, 17, 276, 8, 17, 1, 17, 3, 17, 279, 8, 17, 1, 17, 1, 17, 1, 17, 1, 17, 5, 17, 285, 8, 17, 10, 17, 12, 17, 288, 9, 17, 1, 18, 3, 18, 291, 8, 18, 1, 18, 1, 18, 1, 18, 3, 18, 296, 8, 18, 1, 18, 3, 18, 299, 8, 18, 1, 18, 3, 18, 302, 8, 18, 1, 18, 1, 18, 3, 18, 306, 8, 18, 1, 18, 1, 18, 3, 18, 310, 8, 18, 1, 18, 3, 18, 313, 8, 18, 3, 18, 315, 8, 18, 1, 18, 3, 18, 318, 8, 18, 1, 18, 1, 18, 3, 18, 322, 8, 18, 1, 18, 1, 18, 3, 18, 326, 8, 18, 1, 18, 3, 18, 329, 8, 18, 3, 18, 331, 8, 18, 3, 18, 333, 8, 18, 1, 19, 3, 19, 336, 8, 19, 1, 19, 1, 19, 1, 19, 3, 19, 341, 8, 19, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 1, 20, 3, 20, 352, 8, 20, 1, 21, 1, 21, 1, 21, 1, 21, 3, 21, 358, 8, 21, 1, 22, 1, 22, 1, 22, 3, 22, 363, 8, 22, 1, 23, 1, 23, 1, 23, 5, 23, 368, 8, 23, 10, 23, 12, 23, 371, 9, 23, 1, 24, 1, 24, 3, 24, 375, 8, 24, 1, 24, 1, 24, 3, 24, 379, 8, 24, 1, 24, 1, 24, 3, 24, 383, 8, 24, 1, 25, 1, 25, 1, 25, 3, 25, 388, 8, 25, 1, 26, 1, 26, 1, 26, 5, 26, 393, 8, 26, 10, 26, 12, 26, 396, 9, 26, 1, 27, 1, 27, 1, 27, 1, 27, 1, 28, 3, 28, 403, 8, 28, 1, 28, 3, 28, 406, 8, 28, 1, 28, 3, 28, 409, 8, 28, 1, 29, 1, 29, 1, 29, 1, 29, 1, 30, 1, 30, 1, 30, 1, 30, 1, 31, 1, 31, 1, 31, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 1, 32, 3, 32, 428, 8, 32, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 1, 33, 3, 33, 442, 8, 33, 1, 34, 1, 34, 1, 34, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 5, 35, 456, 8, 35, 10, 35, 12, 35, 459, 9, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 5, 35, 468, 8, 35, 10, 35, 12, 35, 471, 9, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 5, 35, 480, 8, 35, 10, 35, 12, 35, 483, 9, 35, 1, 35, 1, 35, 1, 35, 1, 35, 1, 35, 3, 35, 490, 8, 35, 1, 35, 1, 35, 3, 35, 494, 8, 35, 1, 36, 1, 36, 1, 36, 5, 36, 499, 8, 36, 10, 36, 12, 36, 502, 9, 36, 1, 37, 1, 37, 1, 37, 3, 37, 507, 8, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 1, 37, 3, 37, 515, 8, 37, 1, 38, 1, 38, 1, 38, 3, 38, 520, 8, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 4, 38, 527, 8, 38, 11, 38, 12, 38, 528, 1, 38, 1, 38, 3, 38, 533, 8, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 3, 38, 564, 8, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 3, 38, 581, 8, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 3, 38, 593, 8, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 3, 38, 603, 8, 38, 1, 38, 3, 38, 606, 8, 38, 1, 38, 1, 38, 3, 38, 610, 8, 38, 1, 38, 3, 38, 613, 8, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 3, 38, 625, 8, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 3, 38, 642, 8, 38, 1, 38, 1, 38, 3, 38, 646, 8, 38, 1, 38, 1, 38, 1, 38, 1, 38, 3, 38, 652, 8, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 3, 38, 659, 8, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 3, 38, 671, 8, 38, 1, 38, 3, 38, 674, 8, 38, 1, 38, 1, 38, 3, 38, 678, 8, 38, 1, 38, 3, 38, 681, 8, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 3, 38, 692, 8, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 3, 38, 716, 8, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 1, 38, 3, 38, 725, 8, 38, 5, 38, 727, 8, 38, 10, 38, 12, 38, 730, 9, 38, 1, 39, 1, 39, 1, 39, 5, 39, 735, 8, 39, 10, 39, 12, 39, 738, 9, 39, 1, 40, 1, 40, 3, 40, 742, 8, 40, 1, 41, 1, 41, 1, 41, 1, 41, 5, 41, 748, 8, 41, 10, 41, 12, 41, 751, 9, 41, 1, 41, 1, 41, 1, 41, 1, 41, 1, 41, 5, 41, 758, 8, 41, 10, 41, 12, 41, 761, 9, 41, 3, 41, 763, 8, 41, 1, 41, 1, 41, 1, 41, 1, 42, 1, 42, 1, 42, 1, 42, 3, 42, 772, 8, 42, 1, 42, 3, 42, 775, 8, 42, 1, 43, 1, 43, 1, 43, 5, 43, 780, 8, 43, 10, 43, 12, 43, 783, 9, 43, 1, 44, 1, 44, 1, 44, 1, 44, 1, 44, 1, 44, 1, 44, 3, 44, 792, 8, 44, 1, 44, 1, 44, 1, 44, 1, 44, 3, 44, 798, 8, 44, 5, 44, 800, 8, 44, 10, 44, 12, 44, 803, 9, 44, 1, 45, 1, 45, 1, 45, 3, 45, 808, 8, 45, 1, 45, 1, 45, 1, 46, 1, 46, 1, 46, 3, 46, 815, 8, 46, 1, 46, 1, 46, 1, 47, 1, 47, 1, 47, 5, 47, 822, 8, 47, 10, 47, 12, 47, 825, 9, 47, 1, 48, 1, 48, 1, 48, 3, 48, 830, 8, 48, 1, 49, 1, 49, 1, 50, 1, 50, 1, 50, 1, 50, 1, 50, 1, 50, 3, 50, 840, 8, 50, 3, 50, 842, 8, 50, 1, 51, 3, 51, 845, 8, 51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 1, 51, 3, 51, 853, 8, 51, 1, 52, 1, 52, 1, 52, 3, 52, 858, 8, 52, 1, 53, 1, 53, 1, 54, 1, 54, 1, 55, 1, 55, 1, 56, 1, 56, 3, 56, 868, 8, 56, 1, 57, 1, 57, 1, 57, 3, 57, 873, 8, 57, 1, 58, 1, 58, 3, 58, 877, 8, 58, 1, 59, 1, 59, 1, 59, 1, 59, 1, 59, 0, 3, 34, 76, 88, 60, 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44, 46, 48, 50, 52, 54, 56, 58, 60, 62, 64, 66, 68, 70, 72, 74, 76, 78, 80, 82, 84, 86, 88, 90, 92, 94, 96, 98, 100, 102, 104, 106, 108, 110, 112, 114, 116, 118, 0, 18, 2, 0, 31, 31, 140, 140, 2, 0, 83, 83, 95, 95, 2, 0, 70, 70, 100, 100, 3, 0, 4, 4, 8, 8, 12, 12, 4, 0, 4, 4, 7, 8, 12, 12, 146, 146, 2, 0, 95, 95, 139, 139, 2, 0, 4, 4, 8, 8, 2, 0, 117, 117, 205, 205, 2, 0, 11, 11, 41, 42, 2, 0, 61, 61, 92, 92, 2, 0, 132, 132, 142, 142, 3, 0, 17, 17, 94, 94, 169, 169, 2, 0, 78, 78, 97, 97, 1, 0, 195, 196, 2, 0, 207, 207, 222, 222, 8, 0, 36, 36, 75, 75, 107, 107, 109, 109, 131, 131, 144, 144, 184, 184, 189, 189, 12, 0, 2, 35, 37, 74, 76, 80, 82, 106, 108, 108, 110, 111, 113, 114, 116, 129, 132, 143, 145, 183, 185, 188, 190, 191, 4, 0, 35, 35, 61, 61, 76, 76, 90, 90, 993, 0, 122, 1, 0, 0, 0, 2, 126, 1, 0, 0, 0, 4, 140, 1, 0, 0, 0, 6, 143, 1, 0, 0, 0, 8, 191, 1, 0, 0, 0, 10, 194, 1, 0, 0, 0, 12, 200, 1, 0, 0, 0, 14, 204, 1, 0, 0, 0, 16, 210, 1, 0, 0, 0, 18, 217, 1, 0, 0, 0, 20, 220, 1, 0, 0, 0, 22, 223, 1, 0, 0, 0, 24, 233, 1, 0, 0, 0, 26, 236, 1, 0, 0, 0, 28, 240, 1, 0, 0, 0, 30, 244, 1, 0, 0, 0, 32, 252, 1, 0, 0, 0, 34, 267, 1, 0, 0, 0, 36, 332, 1, 0, 0, 0, 38, 340, 1, 0, 0, 0, 40, 351, 1, 0, 0, 0, 42, 353, 1, 0, 0, 0, 44, 359, 1, 0, 0, 0, 46, 364, 1, 0, 0, 0, 48, 372, 1, 0, 0, 0, 50, 384, 1, 0, 0, 0, 52, 389, 1, 0, 0, 0, 54, 397, 1, 0, 0, 0, 56, 402, 1, 0, 0, 0, 58, 410, 1, 0, 0, 0, 60, 414, 1, 0, 0, 0, 62, 418, 1, 0, 0, 0, 64, 427, 1, 0, 0, 0, 66, 441, 1, 0, 0, 0, 68, 443, 1, 0, 0, 0, 70, 493, 1, 0, 0, 0, 72, 495, 1, 0, 0, 0, 74, 514, 1, 0, 0, 0, 76, 645, 1, 0, 0, 0, 78, 731, 1, 0, 0, 0, 80, 741, 1, 0, 0, 0, 82, 762, 1, 0, 0, 0, 84, 774, 1, 0, 0, 0, 86, 776, 1, 0, 0, 0, 88, 791, 1, 0, 0, 0, 90, 804, 1, 0, 0, 0, 92, 814, 1, 0, 0, 0, 94, 818, 1, 0, 0, 0, 96, 829, 1, 0, 0, 0, 98, 831, 1, 0, 0, 0, 100, 841, 1, 0, 0, 0, 102, 844, 1, 0, 0, 0, 104, 857, 1, 0, 0, 0, 106, 859, 1, 0, 0, 0, 108, 861, 1, 0, 0, 0, 110, 863, 1, 0, 0, 0, 112, 867, 1, 0, 0, 0, 114, 872, 1, 0, 0, 0, 116, 876, 1, 0, 0, 0, 118, 878, 1, 0, 0, 0, 120, 123, 3, 2, 1, 0, 121, 123, 3, 6, 3, 0, 122, 120, 1, 0, 0, 0, 122, 121, 1, 0, 0, 0, 123, 124, 1, 0, 0, 0, 124, 125, 5, 0, 0, 1, 125, 1, 1, 0, 0, 0, 126, 132, 3, 4, 2, 0, 127, 128, 5, 175, 0, 0, 128, 129, 5, 4, 0, 0, 129, 131, 3, 4, 2, 0, 130, 127, 1, 0, 0, 0, 131, 134, 1, 0, 0, 0, 132, 130, 1, 0, 0, 0, 132, 133, 1, 0, 0, 0, 133, 3, 1, 0, 0, 0, 134, 132, 1, 0, 0, 0, 135, 141, 3, 6, 3, 0, 136, 137, 5, 218, 0, 0, 137, 138, 3, 2, 1, 0, 138, 139, 5, 228, 0, 0, 139, 141, 1, 0, 0, 0, 140, 135, 1, 0, 0, 0, 140, 136, 1, 0, 0, 0, 141, 5, 1, 0, 0, 0, 142, 144, 3, 8, 4, 0, 143, 142, 1, 0, 0, 0, 143, 144, 1, 0, 0, 0, 144, 145, 1, 0, 0, 0, 145, 147, 5, 145, 0, 0, 146, 148, 5, 48, 0, 0, 147, 146, 1, 0, 0, 0, 147, 148, 1, 0, 0, 0, 148, 150, 1, 0, 0, 0, 149, 151, 3, 10, 5, 0, 150, 149, 1, 0, 0, 0, 150, 151, 1, 0, 0, 0, 151, 152, 1, 0, 0, 0, 152, 154, 3, 72, 36, 0, 153, 155, 3, 12, 6, 0, 154, 153, 1, 0, 0, 0, 154, 155, 1, 0, 0, 0, 155, 157, 1, 0, 0, 0, 156, 158, 3, 14, 7, 0, 157, 156, 1, 0, 0, 0, 157, 158, 1, 0, 0, 0, 158, 160, 1, 0, 0, 0, 159, 161, 3, 16, 8, 0, 160, 159, 1, 0, 0, 0, 160, 161, 1, 0, 0, 0, 161, 163, 1, 0, 0, 0, 162, 164, 3, 18, 9, 0, 163, 162, 1, 0, 0, 0, 163, 164, 1, 0, 0, 0, 164, 166, 1, 0, 0, 0, 165, 167, 3, 20, 10, 0, 166, 165, 1, 0, 0, 0, 166, 167, 1, 0, 0, 0, 167, 169, 1, 0, 0, 0, 168, 170, 3, 22, 11, 0, 169, 168, 1, 0, 0, 0, 169, 170, 1, 0, 0, 0, 170, 173, 1, 0, 0, 0, 171, 172, 5, 188, 0, 0, 172, 174, 7, 0, 0, 0, 173, 171, 1, 0, 0, 0, 173, 174, 1, 0, 0, 0, 174, 177, 1, 0, 0, 0, 175, 176, 5, 188, 0, 0, 176, 178, 5, 168, 0, 0, 177, 175, 1, 0, 0, 0, 177, 178, 1, 0, 0, 0, 178, 180, 1, 0, 0, 0, 179, 181, 3, 24, 12, 0, 180, 179, 1, 0, 0, 0, 180, 181, 1, 0, 0, 0, 181, 183, 1, 0, 0, 0, 182, 184, 3, 26, 13, 0, 183, 182, 1, 0, 0, 0, 183, 184, 1, 0, 0, 0, 184, 186, 1, 0, 0, 0, 185, 187, 3, 30, 15, 0, 186, 185, 1, 0, 0, 0, 186, 187, 1, 0, 0, 0, 187, 189, 1, 0, 0, 0, 188, 190, 3, 32, 16, 0, 189, 188, 1, 0, 0, 0, 189, 190, 1, 0, 0, 0, 190, 7, 1, 0, 0, 0, 191, 192, 5, 188, 0, 0, 192, 193, 3, 72, 36, 0, 193, 9, 1, 0, 0, 0, 194, 195, 5, 167, 0, 0, 195, 198, 5, 196, 0, 0, 196, 197, 5, 188, 0, 0, 197, 199, 5, 163, 0, 0, 198, 196, 1, 0, 0, 0, 198, 199, 1, 0, 0, 0, 199, 11, 1, 0, 0, 0, 200, 201, 5, 67, 0, 0, 201, 202, 3, 34, 17, 0, 202, 13, 1, 0, 0, 0, 203, 205, 7, 1, 0, 0, 204, 203, 1, 0, 0, 0, 204, 205, 1, 0, 0, 0, 205, 206, 1, 0, 0, 0, 206, 207, 5, 9, 0, 0, 207, 208, 5, 89, 0, 0, 208, 209, 3, 72, 36, 0, 209, 15, 1, 0, 0, 0, 210, 211, 5, 187, 0, 0, 211, 212, 3, 114, 57, 0, 212, 213, 5, 10, 0, 0, 213, 214, 5, 218, 0, 0, 214, 215, 3, 56, 28, 0, 215, 216, 5, 228, 0, 0, 216, 17, 1, 0, 0, 0, 217, 218, 5, 128, 0, 0, 218, 219, 3, 76, 38, 0, 219, 19, 1, 0, 0, 0, 220, 221, 5, 186, 0, 0, 221, 222, 3, 76, 38, 0, 222, 21, 1, 0, 0, 0, 223, 224, 5, 72, 0, 0, 224, 231, 5, 18, 0, 0, 225, 226, 7, 0, 0, 0, 226, 227, 5, 218, 0, 0, 227, 228, 3, 72, 36, 0, 228, 229, 5, 228, 0, 0, 229, 232, 1, 0, 0, 0, 230, 232, 3, 72, 36, 0, 231, 225, 1, 0, 0, 0, 231, 230, 1, 0, 0, 0, 232, 23, 1, 0, 0, 0, 233, 234, 5, 73, 0, 0, 234, 235, 3, 76, 38, 0, 235, 25, 1, 0, 0, 0, 236, 237, 5, 121, 0, 0, 237, 238, 5, 18, 0, 0, 238, 239, 3, 46, 23, 0, 239, 27, 1, 0, 0, 0, 240, 241, 5, 121, 0, 0, 241, 242, 5, 18, 0, 0, 242, 243, 3, 72, 36, 0, 243, 29, 1, 0, 0, 0, 244, 245, 5, 98, 0, 0, 245, 250, 3, 44, 22, 0, 246, 247, 5, 188, 0, 0, 247, 251, 5, 163, 0, 0, 248, 249, 5, 18, 0, 0, 249, 251, 3, 72, 36, 0, 250, 246, 1, 0, 0, 0, 250, 248, 1, 0, 0, 0, 250, 251, 1, 0, 0, 0, 251, 31, 1, 0, 0, 0, 252, 253, 5, 149, 0, 0, 253, 254, 3, 52, 26, 0, 254, 33, 1, 0, 0, 0, 255, 256, 6, 17, -1, 0, 256, 258, 3, 88, 44, 0, 257, 259, 5, 60, 0, 0, 258, 257, 1, 0, 0, 0, 258, 259, 1, 0, 0, 0, 259, 261, 1, 0, 0, 0, 260, 262, 3, 42, 21, 0, 261, 260, 1, 0, 0, 0, 261, 262, 1, 0, 0, 0, 262, 268, 1, 0, 0, 0, 263, 264, 5, 218, 0, 0, 264, 265, 3, 34, 17, 0, 265, 266, 5, 228, 0, 0, 266, 268, 1, 0, 0, 0, 267, 255, 1, 0, 0, 0, 267, 263, 1, 0, 0, 0, 268, 286, 1, 0, 0, 0, 269, 270, 10, 3, 0, 0, 270, 271, 3, 38, 19, 0, 271, 272, 3, 34, 17, 4, 272, 285, 1, 0, 0, 0, 273, 275, 10, 4, 0, 0, 274, 276, 7, 2, 0, 0, 275, 274, 1, 0, 0, 0, 275, 276, 1, 0, 0, 0, 276, 278, 1, 0, 0, 0, 277, 279, 3, 36, 18, 0, 278, 277, 1, 0, 0, 0, 278, 279, 1, 0, 0, 0, 279, 280, 1, 0, 0, 0, 280, 281, 5, 89, 0, 0, 281, 282, 3, 34, 17, 0, 282, 283, 3, 40, 20, 0, 283, 285, 1, 0, 0, 0, 284, 269, 1, 0, 0, 0, 284, 273, 1, 0, 0, 0, 285, 288, 1, 0, 0, 0, 286, 284, 1, 0, 0, 0, 286, 287, 1, 0, 0, 0, 287, 35, 1, 0, 0, 0, 288, 286, 1, 0, 0, 0, 289, 291, 7, 3, 0, 0, 290, 289, 1, 0, 0, 0, 290, 291, 1, 0, 0, 0, 291, 292, 1, 0, 0, 0, 292, 299, 5, 83, 0, 0, 293, 295, 5, 83, 0, 0, 294, 296, 7, 3, 0, 0, 295, 294, 1, 0, 0, 0, 295, 296, 1, 0, 0, 0, 296, 299, 1, 0, 0, 0, 297, 299, 7, 3, 0, 0, 298, 290, 1, 0, 0, 0, 298, 293, 1, 0, 0, 0, 298, 297, 1, 0, 0, 0, 299, 333, 1, 0, 0, 0, 300, 302, 7, 4, 0, 0, 301, 300, 1, 0, 0, 0, 301, 302, 1, 0, 0, 0, 302, 303, 1, 0, 0, 0, 303, 305, 7, 5, 0, 0, 304, 306, 5, 122, 0, 0, 305, 304, 1, 0, 0, 0, 305, 306, 1, 0, 0, 0, 306, 315, 1, 0, 0, 0, 307, 309, 7, 5, 0, 0, 308, 310, 5, 122, 0, 0, 309, 308, 1, 0, 0, 0, 309, 310, 1, 0, 0, 0, 310, 312, 1, 0, 0, 0, 311, 313, 7, 4, 0, 0, 312, 311, 1, 0, 0, 0, 312, 313, 1, 0, 0, 0, 313, 315, 1, 0, 0, 0, 314, 301, 1, 0, 0, 0, 314, 307, 1, 0, 0, 0, 315, 333, 1, 0, 0, 0, 316, 318, 7, 6, 0, 0, 317, 316, 1, 0, 0, 0, 317, 318, 1, 0, 0, 0, 318, 319, 1, 0, 0, 0, 319, 321, 5, 68, 0, 0, 320, 322, 5, 122, 0, 0, 321, 320, 1, 0, 0, 0, 321, 322, 1, 0, 0, 0, 322, 331, 1, 0, 0, 0, 323, 325, 5, 68, 0, 0, 324, 326, 5, 122, 0, 0, 325, 324, 1, 0, 0, 0, 325, 326, 1, 0, 0, 0, 326, 328, 1, 0, 0, 0, 327, 329, 7, 6, 0, 0, 328, 327, 1, 0, 0, 0, 328, 329, 1, 0, 0, 0, 329, 331, 1, 0, 0, 0, 330, 317, 1, 0, 0, 0, 330, 323, 1, 0, 0, 0, 331, 333, 1, 0, 0, 0, 332, 298, 1, 0, 0, 0, 332, 314, 1, 0, 0, 0, 332, 330, 1, 0, 0, 0, 333, 37, 1, 0, 0, 0, 334, 336, 7, 2, 0, 0, 335, 334, 1, 0, 0, 0, 335, 336, 1, 0, 0, 0, 336, 337, 1, 0, 0, 0, 337, 338, 5, 30, 0, 0, 338, 341, 5, 89, 0, 0, 339, 341, 5, 205, 0, 0, 340, 335, 1, 0, 0, 0, 340, 339, 1, 0, 0, 0, 341, 39, 1, 0, 0, 0, 342, 343, 5, 118, 0, 0, 343, 352, 3, 72, 36, 0, 344, 345, 5, 178, 0, 0, 345, 346, 5, 218, 0, 0, 346, 347, 3, 72, 36, 0, 347, 348, 5, 228, 0, 0, 348, 352, 1, 0, 0, 0, 349, 350, 5, 178, 0, 0, 350, 352, 3, 72, 36, 0, 351, 342, 1, 0, 0, 0, 351, 344, 1, 0, 0, 0, 351, 349, 1, 0, 0, 0, 352, 41, 1, 0, 0, 0, 353, 354, 5, 143, 0, 0, 354, 357, 3, 50, 25, 0, 355, 356, 5, 117, 0, 0, 356, 358, 3, 50, 25, 0, 357, 355, 1, 0, 0, 0, 357, 358, 1, 0, 0, 0, 358, 43, 1, 0, 0, 0, 359, 362, 3, 76, 38, 0, 360, 361, 7, 7, 0, 0, 361, 363, 3, 76, 38, 0, 362, 360, 1, 0, 0, 0, 362, 363, 1, 0, 0, 0, 363, 45, 1, 0, 0, 0, 364, 369, 3, 48, 24, 0, 365, 366, 5, 205, 0, 0, 366, 368, 3, 48, 24, 0, 367, 365, 1, 0, 0, 0, 368, 371, 1, 0, 0, 0, 369, 367, 1, 0, 0, 0, 369, 370, 1, 0, 0, 0, 370, 47, 1, 0, 0, 0, 371, 369, 1, 0, 0, 0, 372, 374, 3, 76, 38, 0, 373, 375, 7, 8, 0, 0, 374, 373, 1, 0, 0, 0, 374, 375, 1, 0, 0, 0, 375, 378, 1, 0, 0, 0, 376, 377, 5, 116, 0, 0, 377, 379, 7, 9, 0, 0, 378, 376, 1, 0, 0, 0, 378, 379, 1, 0, 0, 0, 379, 382, 1, 0, 0, 0, 380, 381, 5, 25, 0, 0, 381, 383, 5, 198, 0, 0, 382, 380, 1, 0, 0, 0, 382, 383, 1, 0, 0, 0, 383, 49, 1, 0, 0, 0, 384, 387, 3, 102, 51, 0, 385, 386, 5, 230, 0, 0, 386, 388, 3, 102, 51, 0, 387, 385, 1, 0, 0, 0, 387, 388, 1, 0, 0, 0, 388, 51, 1, 0, 0, 0, 389, 394, 3, 54, 27, 0, 390, 391, 5, 205, 0, 0, 391, 393, 3, 54, 27, 0, 392, 390, 1, 0, 0, 0, 393, 396, 1, 0, 0, 0, 394, 392, 1, 0, 0, 0, 394, 395, 1, 0, 0, 0, 395, 53, 1, 0, 0, 0, 396, 394, 1, 0, 0, 0, 397, 398, 3, 114, 57, 0, 398, 399, 5, 211, 0, 0, 399, 400, 3, 104, 52, 0, 400, 55, 1, 0, 0, 0, 401, 403, 3, 58, 29, 0, 402, 401, 1, 0, 0, 0, 402, 403, 1, 0, 0, 0, 403, 405, 1, 0, 0, 0, 404, 406, 3, 60, 30, 0, 405, 404, 1, 0, 0, 0, 405, 406, 1, 0, 0, 0, 406, 408, 1, 0, 0, 0, 407, 409, 3, 62, 31, 0, 408, 407, 1, 0, 0, 0, 408, 409, 1, 0, 0, 0, 409, 57, 1, 0, 0, 0, 410, 411, 5, 125, 0, 0, 411, 412, 5, 18, 0, 0, 412, 413, 3, 72, 36, 0, 413, 59, 1, 0, 0, 0, 414, 415, 5, 121, 0, 0, 415, 416, 5, 18, 0, 0, 416, 417, 3, 46, 23, 0, 417, 61, 1, 0, 0, 0, 418, 419, 7, 10, 0, 0, 419, 420, 3, 64, 32, 0, 420, 63, 1, 0, 0, 0, 421, 428, 3, 66, 33, 0, 422, 423, 5, 16, 0, 0, 423, 424, 3, 66, 33, 0, 424, 425, 5, 6, 0, 0, 425, 426, 3, 66, 33, 0, 426, 428, 1, 0, 0, 0, 427, 421, 1, 0, 0, 0, 427, 422, 1, 0, 0, 0, 428, 65, 1, 0, 0, 0, 429, 430, 5, 32, 0, 0, 430, 442, 5, 141, 0, 0, 431, 432, 5, 174, 0, 0, 432, 442, 5, 127, 0, 0, 433, 434, 5, 174, 0, 0, 434, 442, 5, 63, 0, 0, 435, 436, 3, 102, 51, 0, 436, 437, 5, 127, 0, 0, 437, 442, 1, 0, 0, 0, 438, 439, 3, 102, 51, 0, 439, 440, 5, 63, 0, 0, 440, 442, 1, 0, 0, 0, 441, 429, 1, 0, 0, 0, 441, 431, 1, 0, 0, 0, 441, 433, 1, 0, 0, 0, 441, 435, 1, 0, 0, 0, 441, 438, 1, 0, 0, 0, 442, 67, 1, 0, 0, 0, 443, 444, 3, 76, 38, 0, 444, 445, 5, 0, 0, 1, 445, 69, 1, 0, 0, 0, 446, 494, 3, 114, 57, 0, 447, 448, 3, 114, 57, 0, 448, 449, 5, 218, 0, 0, 449, 450, 3, 114, 57, 0, 450, 457, 3, 70, 35, 0, 451, 452, 5, 205, 0, 0, 452, 453, 3, 114, 57, 0, 453, 454, 3, 70, 35, 0, 454, 456, 1, 0, 0, 0, 455, 451, 1, 0, 0, 0, 456, 459, 1, 0, 0, 0, 457, 455, 1, 0, 0, 0, 457, 458, 1, 0, 0, 0, 458, 460, 1, 0, 0, 0, 459, 457, 1, 0, 0, 0, 460, 461, 5, 228, 0, 0, 461, 494, 1, 0, 0, 0, 462, 463, 3, 114, 57, 0, 463, 464, 5, 218, 0, 0, 464, 469, 3, 118, 59, 0, 465, 466, 5, 205, 0, 0, 466, 468, 3, 118, 59, 0, 467, 465, 1, 0, 0, 0, 468, 471, 1, 0, 0, 0, 469, 467, 1, 0, 0, 0, 469, 470, 1, 0, 0, 0, 470, 472, 1, 0, 0, 0, 471, 469, 1, 0, 0, 0, 472, 473, 5, 228, 0, 0, 473, 494, 1, 0, 0, 0, 474, 475, 3, 114, 57, 0, 475, 476, 5, 218, 0, 0, 476, 481, 3, 70, 35, 0, 477, 478, 5, 205, 0, 0, 478, 480, 3, 70, 35, 0, 479, 477, 1, 0, 0, 0, 480, 483, 1, 0, 0, 0, 481, 479, 1, 0, 0, 0, 481, 482, 1, 0, 0, 0, 482, 484, 1, 0, 0, 0, 483, 481, 1, 0, 0, 0, 484, 485, 5, 228, 0, 0, 485, 494, 1, 0, 0, 0, 486, 487, 3, 114, 57, 0, 487, 489, 5, 218, 0, 0, 488, 490, 3, 72, 36, 0, 489, 488, 1, 0, 0, 0, 489, 490, 1, 0, 0, 0, 490, 491, 1, 0, 0, 0, 491, 492, 5, 228, 0, 0, 492, 494, 1, 0, 0, 0, 493, 446, 1, 0, 0, 0, 493, 447, 1, 0, 0, 0, 493, 462, 1, 0, 0, 0, 493, 474, 1, 0, 0, 0, 493, 486, 1, 0, 0, 0, 494, 71, 1, 0, 0, 0, 495, 500, 3, 74, 37, 0, 496, 497, 5, 205, 0, 0, 497, 499, 3, 74, 37, 0, 498, 496, 1, 0, 0, 0, 499, 502, 1, 0, 0, 0, 500, 498, 1, 0, 0, 0, 500, 501, 1, 0, 0, 0, 501, 73, 1, 0, 0, 0, 502, 500, 1, 0, 0, 0, 503, 504, 3, 92, 46, 0, 504, 505, 5, 209, 0, 0, 505, 507, 1, 0, 0, 0, 506, 503, 1, 0, 0, 0, 506, 507, 1, 0, 0, 0, 507, 508, 1, 0, 0, 0, 508, 515, 5, 201, 0, 0, 509, 510, 5, 218, 0, 0, 510, 511, 3, 2, 1, 0, 511, 512, 5, 228, 0, 0, 512, 515, 1, 0, 0, 0, 513, 515, 3, 76, 38, 0, 514, 506, 1, 0, 0, 0, 514, 509, 1, 0, 0, 0, 514, 513, 1, 0, 0, 0, 515, 75, 1, 0, 0, 0, 516, 517, 6, 38, -1, 0, 517, 519, 5, 19, 0, 0, 518, 520, 3, 76, 38, 0, 519, 518, 1, 0, 0, 0, 519, 520, 1, 0, 0, 0, 520, 526, 1, 0, 0, 0, 521, 522, 5, 185, 0, 0, 522, 523, 3, 76, 38, 0, 523, 524, 5, 162, 0, 0, 524, 525, 3, 76, 38, 0, 525, 527, 1, 0, 0, 0, 526, 521, 1, 0, 0, 0, 527, 528, 1, 0, 0, 0, 528, 526, 1, 0, 0, 0, 528, 529, 1, 0, 0, 0, 529, 532, 1, 0, 0, 0, 530, 531, 5, 51, 0, 0, 531, 533, 3, 76, 38, 0, 532, 530, 1, 0, 0, 0, 532, 533, 1, 0, 0, 0, 533, 534, 1, 0, 0, 0, 534, 535, 5, 52, 0, 0, 535, 646, 1, 0, 0, 0, 536, 537, 5, 20, 0, 0, 537, 538, 5, 218, 0, 0, 538, 539, 3, 76, 38, 0, 539, 540, 5, 10, 0, 0, 540, 541, 3, 70, 35, 0, 541, 542, 5, 228, 0, 0, 542, 646, 1, 0, 0, 0, 543, 544, 5, 35, 0, 0, 544, 646, 5, 198, 0, 0, 545, 546, 5, 58, 0, 0, 546, 547, 5, 218, 0, 0, 547, 548, 3, 106, 53, 0, 548, 549, 5, 67, 0, 0, 549, 550, 3, 76, 38, 0, 550, 551, 5, 228, 0, 0, 551, 646, 1, 0, 0, 0, 552, 553, 5, 85, 0, 0, 553, 554, 3, 76, 38, 0, 554, 555, 3, 106, 53, 0, 555, 646, 1, 0, 0, 0, 556, 557, 5, 154, 0, 0, 557, 558, 5, 218, 0, 0, 558, 559, 3, 76, 38, 0, 559, 560, 5, 67, 0, 0, 560, 563, 3, 76, 38, 0, 561, 562, 5, 64, 0, 0, 562, 564, 3, 76, 38, 0, 563, 561, 1, 0, 0, 0, 563, 564, 1, 0, 0, 0, 564, 565, 1, 0, 0, 0, 565, 566, 5, 228, 0, 0, 566, 646, 1, 0, 0, 0, 567, 568, 5, 165, 0, 0, 568, 646, 5, 198, 0, 0, 569, 570, 5, 170, 0, 0, 570, 571, 5, 218, 0, 0, 571, 572, 7, 11, 0, 0, 572, 573, 5, 198, 0, 0, 573, 574, 5, 67, 0, 0, 574, 575, 3, 76, 38, 0, 575, 576, 5, 228, 0, 0, 576, 646, 1, 0, 0, 0, 577, 578, 3, 114, 57, 0, 578, 580, 5, 218, 0, 0, 579, 581, 3, 72, 36, 0, 580, 579, 1, 0, 0, 0, 580, 581, 1, 0, 0, 0, 581, 582, 1, 0, 0, 0, 582, 583, 5, 228, 0, 0, 583, 584, 1, 0, 0, 0, 584, 585, 5, 124, 0, 0, 585, 586, 5, 218, 0, 0, 586, 587, 3, 56, 28, 0, 587, 588, 5, 228, 0, 0, 588, 646, 1, 0, 0, 0, 589, 590, 3, 114, 57, 0, 590, 592, 5, 218, 0, 0, 591, 593, 3, 72, 36, 0, 592, 591, 1, 0, 0, 0, 592, 593, 1, 0, 0, 0, 593, 594, 1, 0, 0, 0, 594, 595, 5, 228, 0, 0, 595, 596, 1, 0, 0, 0, 596, 597, 5, 124, 0, 0, 597, 598, 3, 114, 57, 0, 598, 646, 1, 0, 0, 0, 599, 605, 3, 114, 57, 0, 600, 602, 5, 218, 0, 0, 601, 603, 3, 72, 36, 0, 602, 601, 1, 0, 0, 0, 602, 603, 1, 0, 0, 0, 603, 604, 1, 0, 0, 0, 604, 606, 5, 228, 0, 0, 605, 600, 1, 0, 0, 0, 605, 606, 1, 0, 0, 0, 606, 607, 1, 0, 0, 0, 607, 609, 5, 218, 0, 0, 608, 610, 5, 48, 0, 0, 609, 608, 1, 0, 0, 0, 609, 610, 1, 0, 0, 0, 610, 612, 1, 0, 0, 0, 611, 613, 3, 78, 39, 0, 612, 611, 1, 0, 0, 0, 612, 613, 1, 0, 0, 0, 613, 614, 1, 0, 0, 0, 614, 615, 5, 228, 0, 0, 615, 646, 1, 0, 0, 0, 616, 646, 3, 104, 52, 0, 617, 618, 5, 207, 0, 0, 618, 646, 3, 76, 38, 17, 619, 620, 5, 114, 0, 0, 620, 646, 3, 76, 38, 12, 621, 622, 3, 92, 46, 0, 622, 623, 5, 209, 0, 0, 623, 625, 1, 0, 0, 0, 624, 621, 1, 0, 0, 0, 624, 625, 1, 0, 0, 0, 625, 626, 1, 0, 0, 0, 626, 646, 5, 201, 0, 0, 627, 628, 5, 218, 0, 0, 628, 629, 3, 2, 1, 0, 629, 630, 5, 228, 0, 0, 630, 646, 1, 0, 0, 0, 631, 632, 5, 218, 0, 0, 632, 633, 3, 76, 38, 0, 633, 634, 5, 228, 0, 0, 634, 646, 1, 0, 0, 0, 635, 636, 5, 218, 0, 0, 636, 637, 3, 72, 36, 0, 637, 638, 5, 228, 0, 0, 638, 646, 1, 0, 0, 0, 639, 641, 5, 216, 0, 0, 640, 642, 3, 72, 36, 0, 641, 640, 1, 0, 0, 0, 641, 642, 1, 0, 0, 0, 642, 643, 1, 0, 0, 0, 643, 646, 5, 227, 0, 0, 644, 646, 3, 84, 42, 0, 645, 516, 1, 0, 0, 0, 645, 536, 1, 0, 0, 0, 645, 543, 1, 0, 0, 0, 645, 545, 1, 0, 0, 0, 645, 552, 1, 0, 0, 0, 645, 556, 1, 0, 0, 0, 645, 567, 1, 0, 0, 0, 645, 569, 1, 0, 0, 0, 645, 577, 1, 0, 0, 0, 645, 589, 1, 0, 0, 0, 645, 599, 1, 0, 0, 0, 645, 616, 1, 0, 0, 0, 645, 617, 1, 0, 0, 0, 645, 619, 1, 0, 0, 0, 645, 624, 1, 0, 0, 0, 645, 627, 1, 0, 0, 0, 645, 631, 1, 0, 0, 0, 645, 635, 1, 0, 0, 0, 645, 639, 1, 0, 0, 0, 645, 644, 1, 0, 0, 0, 646, 728, 1, 0, 0, 0, 647, 651, 10, 16, 0, 0, 648, 652, 5, 201, 0, 0, 649, 652, 5, 230, 0, 0, 650, 652, 5, 221, 0, 0, 651, 648, 1, 0, 0, 0, 651, 649, 1, 0, 0, 0, 651, 650, 1, 0, 0, 0, 652, 653, 1, 0, 0, 0, 653, 727, 3, 76, 38, 17, 654, 658, 10, 15, 0, 0, 655, 659, 5, 222, 0, 0, 656, 659, 5, 207, 0, 0, 657, 659, 5, 206, 0, 0, 658, 655, 1, 0, 0, 0, 658, 656, 1, 0, 0, 0, 658, 657, 1, 0, 0, 0, 659, 660, 1, 0, 0, 0, 660, 727, 3, 76, 38, 16, 661, 680, 10, 14, 0, 0, 662, 681, 5, 210, 0, 0, 663, 681, 5, 211, 0, 0, 664, 681, 5, 220, 0, 0, 665, 681, 5, 217, 0, 0, 666, 681, 5, 212, 0, 0, 667, 681, 5, 219, 0, 0, 668, 681, 5, 213, 0, 0, 669, 671, 5, 70, 0, 0, 670, 669, 1, 0, 0, 0, 670, 671, 1, 0, 0, 0, 671, 673, 1, 0, 0, 0, 672, 674, 5, 114, 0, 0, 673, 672, 1, 0, 0, 0, 673, 674, 1, 0, 0, 0, 674, 675, 1, 0, 0, 0, 675, 681, 5, 79, 0, 0, 676, 678, 5, 114, 0, 0, 677, 676, 1, 0, 0, 0, 677, 678, 1, 0, 0, 0, 678, 679, 1, 0, 0, 0, 679, 681, 7, 12, 0, 0, 680, 662, 1, 0, 0, 0, 680, 663, 1, 0, 0, 0, 680, 664, 1, 0, 0, 0, 680, 665, 1, 0, 0, 0, 680, 666, 1, 0, 0, 0, 680, 667, 1, 0, 0, 0, 680, 668, 1, 0, 0, 0, 680, 670, 1, 0, 0, 0, 680, 677, 1, 0, 0, 0, 681, 682, 1, 0, 0, 0, 682, 727, 3, 76, 38, 15, 683, 684, 10, 11, 0, 0, 684, 685, 5, 6, 0, 0, 685, 727, 3, 76, 38, 12, 686, 687, 10, 10, 0, 0, 687, 688, 5, 120, 0, 0, 688, 727, 3, 76, 38, 11, 689, 691, 10, 9, 0, 0, 690, 692, 5, 114, 0, 0, 691, 690, 1, 0, 0, 0, 691, 692, 1, 0, 0, 0, 692, 693, 1, 0, 0, 0, 693, 694, 5, 16, 0, 0, 694, 695, 3, 76, 38, 0, 695, 696, 5, 6, 0, 0, 696, 697, 3, 76, 38, 10, 697, 727, 1, 0, 0, 0, 698, 699, 10, 8, 0, 0, 699, 700, 5, 223, 0, 0, 700, 701, 3, 76, 38, 0, 701, 702, 5, 204, 0, 0, 702, 703, 3, 76, 38, 8, 703, 727, 1, 0, 0, 0, 704, 705, 10, 19, 0, 0, 705, 706, 5, 216, 0, 0, 706, 707, 3, 76, 38, 0, 707, 708, 5, 227, 0, 0, 708, 727, 1, 0, 0, 0, 709, 710, 10, 18, 0, 0, 710, 711, 5, 209, 0, 0, 711, 727, 5, 196, 0, 0, 712, 713, 10, 13, 0, 0, 713, 715, 5, 87, 0, 0, 714, 716, 5, 114, 0, 0, 715, 714, 1, 0, 0, 0, 715, 716, 1, 0, 0, 0, 716, 717, 1, 0, 0, 0, 717, 727, 5, 115, 0, 0, 718, 724, 10, 7, 0, 0, 719, 725, 3, 112, 56, 0, 720, 721, 5, 10, 0, 0, 721, 725, 3, 114, 57, 0, 722, 723, 5, 10, 0, 0, 723, 725, 5, 198, 0, 0, 724, 719, 1, 0, 0, 0, 724, 720, 1, 0, 0, 0, 724, 722, 1, 0, 0, 0, 725, 727, 1, 0, 0, 0, 726, 647, 1, 0, 0, 0, 726, 654, 1, 0, 0, 0, 726, 661, 1, 0, 0, 0, 726, 683, 1, 0, 0, 0, 726, 686, 1, 0, 0, 0, 726, 689, 1, 0, 0, 0, 726, 698, 1, 0, 0, 0, 726, 704, 1, 0, 0, 0, 726, 709, 1, 0, 0, 0, 726, 712, 1, 0, 0, 0, 726, 718, 1, 0, 0, 0, 727, 730, 1, 0, 0, 0, 728, 726, 1, 0, 0, 0, 728, 729, 1, 0, 0, 0, 729, 77, 1, 0, 0, 0, 730, 728, 1, 0, 0, 0, 731, 736, 3, 80, 40, 0, 732, 733, 5, 205, 0, 0, 733, 735, 3, 80, 40, 0, 734, 732, 1, 0, 0, 0, 735, 738, 1, 0, 0, 0, 736, 734, 1, 0, 0, 0, 736, 737, 1, 0, 0, 0, 737, 79, 1, 0, 0, 0, 738, 736, 1, 0, 0, 0, 739, 742, 3, 82, 41, 0, 740, 742, 3, 76, 38, 0, 741, 739, 1, 0, 0, 0, 741, 740, 1, 0, 0, 0, 742, 81, 1, 0, 0, 0, 743, 744, 5, 218, 0, 0, 744, 749, 3, 114, 57, 0, 745, 746, 5, 205, 0, 0, 746, 748, 3, 114, 57, 0, 747, 745, 1, 0, 0, 0, 748, 751, 1, 0, 0, 0, 749, 747, 1, 0, 0, 0, 749, 750, 1, 0, 0, 0, 750, 752, 1, 0, 0, 0, 751, 749, 1, 0, 0, 0, 752, 753, 5, 228, 0, 0, 753, 763, 1, 0, 0, 0, 754, 759, 3, 114, 57, 0, 755, 756, 5, 205, 0, 0, 756, 758, 3, 114, 57, 0, 757, 755, 1, 0, 0, 0, 758, 761, 1, 0, 0, 0, 759, 757, 1, 0, 0, 0, 759, 760, 1, 0, 0, 0, 760, 763, 1, 0, 0, 0, 761, 759, 1, 0, 0, 0, 762, 743, 1, 0, 0, 0, 762, 754, 1, 0, 0, 0, 763, 764, 1, 0, 0, 0, 764, 765, 5, 200, 0, 0, 765, 766, 3, 76, 38, 0, 766, 83, 1, 0, 0, 0, 767, 775, 5, 199, 0, 0, 768, 769, 3, 92, 46, 0, 769, 770, 5, 209, 0, 0, 770, 772, 1, 0, 0, 0, 771, 768, 1, 0, 0, 0, 771, 772, 1, 0, 0, 0, 772, 773, 1, 0, 0, 0, 773, 775, 3, 86, 43, 0, 774, 767, 1, 0, 0, 0, 774, 771, 1, 0, 0, 0, 775, 85, 1, 0, 0, 0, 776, 781, 3, 114, 57, 0, 777, 778, 5, 209, 0, 0, 778, 780, 3, 114, 57, 0, 779, 777, 1, 0, 0, 0, 780, 783, 1, 0, 0, 0, 781, 779, 1, 0, 0, 0, 781, 782, 1, 0, 0, 0, 782, 87, 1, 0, 0, 0, 783, 781, 1, 0, 0, 0, 784, 785, 6, 44, -1, 0, 785, 792, 3, 92, 46, 0, 786, 792, 3, 90, 45, 0, 787, 788, 5, 218, 0, 0, 788, 789, 3, 2, 1, 0, 789, 790, 5, 228, 0, 0, 790, 792, 1, 0, 0, 0, 791, 784, 1, 0, 0, 0, 791, 786, 1, 0, 0, 0, 791, 787, 1, 0, 0, 0, 792, 801, 1, 0, 0, 0, 793, 797, 10, 1, 0, 0, 794, 798, 3, 112, 56, 0, 795, 796, 5, 10, 0, 0, 796, 798, 3, 114, 57, 0, 797, 794, 1, 0, 0, 0, 797, 795, 1, 0, 0, 0, 798, 800, 1, 0, 0, 0, 799, 793, 1, 0, 0, 0, 800, 803, 1, 0, 0, 0, 801, 799, 1, 0, 0, 0, 801, 802, 1, 0, 0, 0, 802, 89, 1, 0, 0, 0, 803, 801, 1, 0, 0, 0, 804, 805, 3, 114, 57, 0, 805, 807, 5, 218, 0, 0, 806, 808, 3, 94, 47, 0, 807, 806, 1, 0, 0, 0, 807, 808, 1, 0, 0, 0, 808, 809, 1, 0, 0, 0, 809, 810, 5, 228, 0, 0, 810, 91, 1, 0, 0, 0, 811, 812, 3, 98, 49, 0, 812, 813, 5, 209, 0, 0, 813, 815, 1, 0, 0, 0, 814, 811, 1, 0, 0, 0, 814, 815, 1, 0, 0, 0, 815, 816, 1, 0, 0, 0, 816, 817, 3, 114, 57, 0, 817, 93, 1, 0, 0, 0, 818, 823, 3, 96, 48, 0, 819, 820, 5, 205, 0, 0, 820, 822, 3, 96, 48, 0, 821, 819, 1, 0, 0, 0, 822, 825, 1, 0, 0, 0, 823, 821, 1, 0, 0, 0, 823, 824, 1, 0, 0, 0, 824, 95, 1, 0, 0, 0, 825, 823, 1, 0, 0, 0, 826, 830, 3, 86, 43, 0, 827, 830, 3, 90, 45, 0, 828, 830, 3, 104, 52, 0, 829, 826, 1, 0, 0, 0, 829, 827, 1, 0, 0, 0, 829, 828, 1, 0, 0, 0, 830, 97, 1, 0, 0, 0, 831, 832, 3, 114, 57, 0, 832, 99, 1, 0, 0, 0, 833, 842, 5, 194, 0, 0, 834, 835, 5, 209, 0, 0, 835, 842, 7, 13, 0, 0, 836, 837, 5, 196, 0, 0, 837, 839, 5, 209, 0, 0, 838, 840, 7, 13, 0, 0, 839, 838, 1, 0, 0, 0, 839, 840, 1, 0, 0, 0, 840, 842, 1, 0, 0, 0, 841, 833, 1, 0, 0, 0, 841, 834, 1, 0, 0, 0, 841, 836, 1, 0, 0, 0, 842, 101, 1, 0, 0, 0, 843, 845, 7, 14, 0, 0, 844, 843, 1, 0, 0, 0, 844, 845, 1, 0, 0, 0, 845, 852, 1, 0, 0, 0, 846, 853, 3, 100, 50, 0, 847, 853, 5, 195, 0, 0, 848, 853, 5, 196, 0, 0, 849, 853, 5, 197, 0, 0, 850, 853, 5, 81, 0, 0, 851, 853, 5, 112, 0, 0, 852, 846, 1, 0, 0, 0, 852, 847, 1, 0, 0, 0, 852, 848, 1, 0, 0, 0, 852, 849, 1, 0, 0, 0, 852, 850, 1, 0, 0, 0, 852, 851, 1, 0, 0, 0, 853, 103, 1, 0, 0, 0, 854, 858, 3, 102, 51, 0, 855, 858, 5, 198, 0, 0, 856, 858, 5, 115, 0, 0, 857, 854, 1, 0, 0, 0, 857, 855, 1, 0, 0, 0, 857, 856, 1, 0, 0, 0, 858, 105, 1, 0, 0, 0, 859, 860, 7, 15, 0, 0, 860, 107, 1, 0, 0, 0, 861, 862, 7, 16, 0, 0, 862, 109, 1, 0, 0, 0, 863, 864, 7, 17, 0, 0, 864, 111, 1, 0, 0, 0, 865, 868, 5, 193, 0, 0, 866, 868, 3, 110, 55, 0, 867, 865, 1, 0, 0, 0, 867, 866, 1, 0, 0, 0, 868, 113, 1, 0, 0, 0, 869, 873, 5, 193, 0, 0, 870, 873, 3, 106, 53, 0, 871, 873, 3, 108, 54, 0, 872, 869, 1, 0, 0, 0, 872, 870, 1, 0, 0, 0, 872, 871, 1, 0, 0, 0, 873, 115, 1, 0, 0, 0, 874, 877, 3, 114, 57, 0, 875, 877, 5, 115, 0, 0, 876, 874, 1, 0, 0, 0, 876, 875, 1, 0, 0, 0, 877, 117, 1, 0, 0, 0, 878, 879, 5, 198, 0, 0, 879, 880, 5, 211, 0, 0, 880, 881, 3, 102, 51, 0, 881, 119, 1, 0, 0, 0, 114, 122, 132, 140, 143, 147, 150, 154, 157, 160, 163, 166, 169, 173, 177, 180, 183, 186, 189, 198, 204, 231, 250, 258, 261, 267, 275, 278, 284, 286, 290, 295, 298, 301, 305, 309, 312, 314, 317, 321, 325, 328, 330, 332, 335, 340, 351, 357, 362, 369, 374, 378, 382, 387, 394, 402, 405, 408, 427, 441, 457, 469, 481, 489, 493, 500, 506, 514, 519, 528, 532, 563, 580, 592, 602, 605, 609, 612, 624, 641, 645, 651, 658, 670, 673, 677, 680, 691, 715, 724, 726, 728, 736, 741, 749, 759, 762, 771, 774, 781, 791, 797, 801, 807, 814, 823, 829, 839, 841, 844, 852, 857, 867, 872, 876] \ No newline at end of file diff --git a/posthog/hogql/grammar/HogQLParser.py b/posthog/hogql/grammar/HogQLParser.py index 69da91c2a9518..eca17cbe4f6a4 100644 --- a/posthog/hogql/grammar/HogQLParser.py +++ b/posthog/hogql/grammar/HogQLParser.py @@ -10,7 +10,7 @@ def serializedATN(): return [ - 4,1,234,891,2,0,7,0,2,1,7,1,2,2,7,2,2,3,7,3,2,4,7,4,2,5,7,5,2,6, + 4,1,234,883,2,0,7,0,2,1,7,1,2,2,7,2,2,3,7,3,2,4,7,4,2,5,7,5,2,6, 7,6,2,7,7,7,2,8,7,8,2,9,7,9,2,10,7,10,2,11,7,11,2,12,7,12,2,13,7, 13,2,14,7,14,2,15,7,15,2,16,7,16,2,17,7,17,2,18,7,18,2,19,7,19,2, 20,7,20,2,21,7,21,2,22,7,22,2,23,7,23,2,24,7,24,2,25,7,25,2,26,7, @@ -19,348 +19,345 @@ def serializedATN(): 39,2,40,7,40,2,41,7,41,2,42,7,42,2,43,7,43,2,44,7,44,2,45,7,45,2, 46,7,46,2,47,7,47,2,48,7,48,2,49,7,49,2,50,7,50,2,51,7,51,2,52,7, 52,2,53,7,53,2,54,7,54,2,55,7,55,2,56,7,56,2,57,7,57,2,58,7,58,2, - 59,7,59,2,60,7,60,1,0,1,0,3,0,125,8,0,1,0,1,0,1,1,1,1,1,1,1,1,5, - 1,133,8,1,10,1,12,1,136,9,1,1,2,1,2,1,2,1,2,1,2,3,2,143,8,2,1,3, - 3,3,146,8,3,1,3,1,3,3,3,150,8,3,1,3,3,3,153,8,3,1,3,1,3,3,3,157, - 8,3,1,3,3,3,160,8,3,1,3,3,3,163,8,3,1,3,3,3,166,8,3,1,3,3,3,169, - 8,3,1,3,3,3,172,8,3,1,3,1,3,3,3,176,8,3,1,3,1,3,3,3,180,8,3,1,3, - 3,3,183,8,3,1,3,3,3,186,8,3,1,3,3,3,189,8,3,1,3,3,3,192,8,3,1,3, - 3,3,195,8,3,1,4,1,4,1,4,1,5,1,5,1,5,1,5,3,5,204,8,5,1,6,1,6,1,6, - 1,7,3,7,210,8,7,1,7,1,7,1,7,1,7,1,8,1,8,1,8,1,8,1,8,1,8,1,8,1,9, - 1,9,1,9,1,10,1,10,1,10,1,11,1,11,1,11,1,11,1,11,1,11,1,11,1,11,3, - 11,237,8,11,1,12,1,12,1,12,1,13,1,13,1,13,1,13,1,14,1,14,1,14,1, - 14,1,15,1,15,1,15,1,15,1,15,1,16,1,16,1,16,1,16,3,16,259,8,16,1, - 17,1,17,1,17,1,18,1,18,1,18,3,18,267,8,18,1,18,3,18,270,8,18,1,18, - 1,18,1,18,1,18,3,18,276,8,18,1,18,1,18,1,18,1,18,1,18,1,18,3,18, - 284,8,18,1,18,3,18,287,8,18,1,18,1,18,1,18,1,18,5,18,293,8,18,10, - 18,12,18,296,9,18,1,19,3,19,299,8,19,1,19,1,19,1,19,3,19,304,8,19, - 1,19,3,19,307,8,19,1,19,3,19,310,8,19,1,19,1,19,3,19,314,8,19,1, - 19,1,19,3,19,318,8,19,1,19,3,19,321,8,19,3,19,323,8,19,1,19,3,19, - 326,8,19,1,19,1,19,3,19,330,8,19,1,19,1,19,3,19,334,8,19,1,19,3, - 19,337,8,19,3,19,339,8,19,3,19,341,8,19,1,20,3,20,344,8,20,1,20, - 1,20,1,20,3,20,349,8,20,1,21,1,21,1,21,1,21,1,21,1,21,1,21,1,21, - 1,21,3,21,360,8,21,1,22,1,22,1,22,1,22,3,22,366,8,22,1,23,1,23,1, - 23,3,23,371,8,23,1,24,1,24,1,24,5,24,376,8,24,10,24,12,24,379,9, - 24,1,25,1,25,3,25,383,8,25,1,25,1,25,3,25,387,8,25,1,25,1,25,3,25, - 391,8,25,1,26,1,26,1,26,3,26,396,8,26,1,27,1,27,1,27,5,27,401,8, - 27,10,27,12,27,404,9,27,1,28,1,28,1,28,1,28,1,29,3,29,411,8,29,1, - 29,3,29,414,8,29,1,29,3,29,417,8,29,1,30,1,30,1,30,1,30,1,31,1,31, - 1,31,1,31,1,32,1,32,1,32,1,33,1,33,1,33,1,33,1,33,1,33,3,33,436, - 8,33,1,34,1,34,1,34,1,34,1,34,1,34,1,34,1,34,1,34,1,34,1,34,1,34, - 3,34,450,8,34,1,35,1,35,1,35,1,36,1,36,1,36,1,36,1,36,1,36,1,36, - 1,36,1,36,5,36,464,8,36,10,36,12,36,467,9,36,1,36,1,36,1,36,1,36, - 1,36,1,36,1,36,5,36,476,8,36,10,36,12,36,479,9,36,1,36,1,36,1,36, - 1,36,1,36,1,36,1,36,5,36,488,8,36,10,36,12,36,491,9,36,1,36,1,36, - 1,36,1,36,1,36,3,36,498,8,36,1,36,1,36,3,36,502,8,36,1,37,1,37,1, - 37,5,37,507,8,37,10,37,12,37,510,9,37,1,38,1,38,1,38,3,38,515,8, - 38,1,38,1,38,1,38,1,38,1,38,1,38,3,38,523,8,38,1,39,1,39,1,39,3, - 39,528,8,39,1,39,1,39,1,39,1,39,1,39,4,39,535,8,39,11,39,12,39,536, - 1,39,1,39,3,39,541,8,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39, - 1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39, - 1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,3,39,572,8,39,1,39,1,39, - 1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39, - 3,39,589,8,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39, - 3,39,601,8,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,3,39,611,8, - 39,1,39,3,39,614,8,39,1,39,1,39,3,39,618,8,39,1,39,3,39,621,8,39, - 1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,3,39,633,8,39, - 1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39, - 1,39,1,39,3,39,650,8,39,1,39,1,39,3,39,654,8,39,1,39,1,39,1,39,1, - 39,3,39,660,8,39,1,39,1,39,1,39,1,39,1,39,3,39,667,8,39,1,39,1,39, - 1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,3,39,679,8,39,1,39,3,39, - 682,8,39,1,39,1,39,3,39,686,8,39,1,39,3,39,689,8,39,1,39,1,39,1, - 39,1,39,1,39,1,39,1,39,1,39,1,39,3,39,700,8,39,1,39,1,39,1,39,1, - 39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1,39,1, - 39,1,39,1,39,1,39,1,39,1,39,3,39,724,8,39,1,39,1,39,1,39,1,39,1, - 39,1,39,1,39,3,39,733,8,39,5,39,735,8,39,10,39,12,39,738,9,39,1, - 40,1,40,1,40,5,40,743,8,40,10,40,12,40,746,9,40,1,41,1,41,3,41,750, - 8,41,1,42,1,42,1,42,1,42,5,42,756,8,42,10,42,12,42,759,9,42,1,42, - 1,42,1,42,1,42,1,42,5,42,766,8,42,10,42,12,42,769,9,42,3,42,771, - 8,42,1,42,1,42,1,42,1,43,1,43,1,43,1,43,3,43,780,8,43,1,43,3,43, - 783,8,43,1,44,1,44,1,44,5,44,788,8,44,10,44,12,44,791,9,44,1,45, - 1,45,1,45,1,45,1,45,1,45,1,45,3,45,800,8,45,1,45,1,45,1,45,1,45, - 3,45,806,8,45,5,45,808,8,45,10,45,12,45,811,9,45,1,46,1,46,1,46, - 3,46,816,8,46,1,46,1,46,1,47,1,47,1,47,3,47,823,8,47,1,47,1,47,1, - 48,1,48,1,48,5,48,830,8,48,10,48,12,48,833,9,48,1,49,1,49,1,49,3, - 49,838,8,49,1,50,1,50,1,51,1,51,1,51,1,51,1,51,1,51,3,51,848,8,51, - 3,51,850,8,51,1,52,3,52,853,8,52,1,52,1,52,1,52,1,52,1,52,1,52,3, - 52,861,8,52,1,53,1,53,1,53,3,53,866,8,53,1,54,1,54,1,55,1,55,1,56, - 1,56,1,57,1,57,3,57,876,8,57,1,58,1,58,1,58,3,58,881,8,58,1,59,1, - 59,3,59,885,8,59,1,60,1,60,1,60,1,60,1,60,0,3,36,78,90,61,0,2,4, - 6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48, - 50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92, - 94,96,98,100,102,104,106,108,110,112,114,116,118,120,0,18,2,0,31, - 31,140,140,2,0,83,83,95,95,2,0,70,70,100,100,3,0,4,4,8,8,12,12,4, - 0,4,4,7,8,12,12,146,146,2,0,95,95,139,139,2,0,4,4,8,8,2,0,117,117, - 205,205,2,0,11,11,41,42,2,0,61,61,92,92,2,0,132,132,142,142,3,0, - 17,17,94,94,169,169,2,0,78,78,97,97,1,0,195,196,2,0,207,207,222, - 222,8,0,36,36,75,75,107,107,109,109,131,131,144,144,184,184,189, - 189,12,0,2,35,37,74,76,80,82,106,108,108,110,111,113,114,116,129, - 132,143,145,183,185,188,190,191,4,0,35,35,61,61,76,76,90,90,1000, - 0,124,1,0,0,0,2,128,1,0,0,0,4,142,1,0,0,0,6,145,1,0,0,0,8,196,1, - 0,0,0,10,199,1,0,0,0,12,205,1,0,0,0,14,209,1,0,0,0,16,215,1,0,0, - 0,18,222,1,0,0,0,20,225,1,0,0,0,22,228,1,0,0,0,24,238,1,0,0,0,26, - 241,1,0,0,0,28,245,1,0,0,0,30,249,1,0,0,0,32,254,1,0,0,0,34,260, - 1,0,0,0,36,275,1,0,0,0,38,340,1,0,0,0,40,348,1,0,0,0,42,359,1,0, - 0,0,44,361,1,0,0,0,46,367,1,0,0,0,48,372,1,0,0,0,50,380,1,0,0,0, - 52,392,1,0,0,0,54,397,1,0,0,0,56,405,1,0,0,0,58,410,1,0,0,0,60,418, - 1,0,0,0,62,422,1,0,0,0,64,426,1,0,0,0,66,435,1,0,0,0,68,449,1,0, - 0,0,70,451,1,0,0,0,72,501,1,0,0,0,74,503,1,0,0,0,76,522,1,0,0,0, - 78,653,1,0,0,0,80,739,1,0,0,0,82,749,1,0,0,0,84,770,1,0,0,0,86,782, - 1,0,0,0,88,784,1,0,0,0,90,799,1,0,0,0,92,812,1,0,0,0,94,822,1,0, - 0,0,96,826,1,0,0,0,98,837,1,0,0,0,100,839,1,0,0,0,102,849,1,0,0, - 0,104,852,1,0,0,0,106,865,1,0,0,0,108,867,1,0,0,0,110,869,1,0,0, - 0,112,871,1,0,0,0,114,875,1,0,0,0,116,880,1,0,0,0,118,884,1,0,0, - 0,120,886,1,0,0,0,122,125,3,2,1,0,123,125,3,6,3,0,124,122,1,0,0, - 0,124,123,1,0,0,0,125,126,1,0,0,0,126,127,5,0,0,1,127,1,1,0,0,0, - 128,134,3,4,2,0,129,130,5,175,0,0,130,131,5,4,0,0,131,133,3,4,2, - 0,132,129,1,0,0,0,133,136,1,0,0,0,134,132,1,0,0,0,134,135,1,0,0, - 0,135,3,1,0,0,0,136,134,1,0,0,0,137,143,3,6,3,0,138,139,5,218,0, - 0,139,140,3,2,1,0,140,141,5,228,0,0,141,143,1,0,0,0,142,137,1,0, - 0,0,142,138,1,0,0,0,143,5,1,0,0,0,144,146,3,8,4,0,145,144,1,0,0, - 0,145,146,1,0,0,0,146,147,1,0,0,0,147,149,5,145,0,0,148,150,5,48, - 0,0,149,148,1,0,0,0,149,150,1,0,0,0,150,152,1,0,0,0,151,153,3,10, - 5,0,152,151,1,0,0,0,152,153,1,0,0,0,153,154,1,0,0,0,154,156,3,74, - 37,0,155,157,3,12,6,0,156,155,1,0,0,0,156,157,1,0,0,0,157,159,1, - 0,0,0,158,160,3,14,7,0,159,158,1,0,0,0,159,160,1,0,0,0,160,162,1, - 0,0,0,161,163,3,16,8,0,162,161,1,0,0,0,162,163,1,0,0,0,163,165,1, - 0,0,0,164,166,3,18,9,0,165,164,1,0,0,0,165,166,1,0,0,0,166,168,1, - 0,0,0,167,169,3,20,10,0,168,167,1,0,0,0,168,169,1,0,0,0,169,171, - 1,0,0,0,170,172,3,22,11,0,171,170,1,0,0,0,171,172,1,0,0,0,172,175, - 1,0,0,0,173,174,5,188,0,0,174,176,7,0,0,0,175,173,1,0,0,0,175,176, - 1,0,0,0,176,179,1,0,0,0,177,178,5,188,0,0,178,180,5,168,0,0,179, - 177,1,0,0,0,179,180,1,0,0,0,180,182,1,0,0,0,181,183,3,24,12,0,182, - 181,1,0,0,0,182,183,1,0,0,0,183,185,1,0,0,0,184,186,3,26,13,0,185, - 184,1,0,0,0,185,186,1,0,0,0,186,188,1,0,0,0,187,189,3,30,15,0,188, - 187,1,0,0,0,188,189,1,0,0,0,189,191,1,0,0,0,190,192,3,32,16,0,191, - 190,1,0,0,0,191,192,1,0,0,0,192,194,1,0,0,0,193,195,3,34,17,0,194, - 193,1,0,0,0,194,195,1,0,0,0,195,7,1,0,0,0,196,197,5,188,0,0,197, - 198,3,74,37,0,198,9,1,0,0,0,199,200,5,167,0,0,200,203,5,196,0,0, - 201,202,5,188,0,0,202,204,5,163,0,0,203,201,1,0,0,0,203,204,1,0, - 0,0,204,11,1,0,0,0,205,206,5,67,0,0,206,207,3,36,18,0,207,13,1,0, - 0,0,208,210,7,1,0,0,209,208,1,0,0,0,209,210,1,0,0,0,210,211,1,0, - 0,0,211,212,5,9,0,0,212,213,5,89,0,0,213,214,3,74,37,0,214,15,1, - 0,0,0,215,216,5,187,0,0,216,217,3,116,58,0,217,218,5,10,0,0,218, - 219,5,218,0,0,219,220,3,58,29,0,220,221,5,228,0,0,221,17,1,0,0,0, - 222,223,5,128,0,0,223,224,3,78,39,0,224,19,1,0,0,0,225,226,5,186, - 0,0,226,227,3,78,39,0,227,21,1,0,0,0,228,229,5,72,0,0,229,236,5, - 18,0,0,230,231,7,0,0,0,231,232,5,218,0,0,232,233,3,74,37,0,233,234, - 5,228,0,0,234,237,1,0,0,0,235,237,3,74,37,0,236,230,1,0,0,0,236, - 235,1,0,0,0,237,23,1,0,0,0,238,239,5,73,0,0,239,240,3,78,39,0,240, - 25,1,0,0,0,241,242,5,121,0,0,242,243,5,18,0,0,243,244,3,48,24,0, - 244,27,1,0,0,0,245,246,5,121,0,0,246,247,5,18,0,0,247,248,3,74,37, - 0,248,29,1,0,0,0,249,250,5,98,0,0,250,251,3,46,23,0,251,252,5,18, - 0,0,252,253,3,74,37,0,253,31,1,0,0,0,254,255,5,98,0,0,255,258,3, - 46,23,0,256,257,5,188,0,0,257,259,5,163,0,0,258,256,1,0,0,0,258, - 259,1,0,0,0,259,33,1,0,0,0,260,261,5,149,0,0,261,262,3,54,27,0,262, - 35,1,0,0,0,263,264,6,18,-1,0,264,266,3,90,45,0,265,267,5,60,0,0, - 266,265,1,0,0,0,266,267,1,0,0,0,267,269,1,0,0,0,268,270,3,44,22, - 0,269,268,1,0,0,0,269,270,1,0,0,0,270,276,1,0,0,0,271,272,5,218, - 0,0,272,273,3,36,18,0,273,274,5,228,0,0,274,276,1,0,0,0,275,263, - 1,0,0,0,275,271,1,0,0,0,276,294,1,0,0,0,277,278,10,3,0,0,278,279, - 3,40,20,0,279,280,3,36,18,4,280,293,1,0,0,0,281,283,10,4,0,0,282, - 284,7,2,0,0,283,282,1,0,0,0,283,284,1,0,0,0,284,286,1,0,0,0,285, - 287,3,38,19,0,286,285,1,0,0,0,286,287,1,0,0,0,287,288,1,0,0,0,288, - 289,5,89,0,0,289,290,3,36,18,0,290,291,3,42,21,0,291,293,1,0,0,0, - 292,277,1,0,0,0,292,281,1,0,0,0,293,296,1,0,0,0,294,292,1,0,0,0, - 294,295,1,0,0,0,295,37,1,0,0,0,296,294,1,0,0,0,297,299,7,3,0,0,298, - 297,1,0,0,0,298,299,1,0,0,0,299,300,1,0,0,0,300,307,5,83,0,0,301, - 303,5,83,0,0,302,304,7,3,0,0,303,302,1,0,0,0,303,304,1,0,0,0,304, - 307,1,0,0,0,305,307,7,3,0,0,306,298,1,0,0,0,306,301,1,0,0,0,306, - 305,1,0,0,0,307,341,1,0,0,0,308,310,7,4,0,0,309,308,1,0,0,0,309, - 310,1,0,0,0,310,311,1,0,0,0,311,313,7,5,0,0,312,314,5,122,0,0,313, - 312,1,0,0,0,313,314,1,0,0,0,314,323,1,0,0,0,315,317,7,5,0,0,316, - 318,5,122,0,0,317,316,1,0,0,0,317,318,1,0,0,0,318,320,1,0,0,0,319, - 321,7,4,0,0,320,319,1,0,0,0,320,321,1,0,0,0,321,323,1,0,0,0,322, - 309,1,0,0,0,322,315,1,0,0,0,323,341,1,0,0,0,324,326,7,6,0,0,325, - 324,1,0,0,0,325,326,1,0,0,0,326,327,1,0,0,0,327,329,5,68,0,0,328, - 330,5,122,0,0,329,328,1,0,0,0,329,330,1,0,0,0,330,339,1,0,0,0,331, - 333,5,68,0,0,332,334,5,122,0,0,333,332,1,0,0,0,333,334,1,0,0,0,334, - 336,1,0,0,0,335,337,7,6,0,0,336,335,1,0,0,0,336,337,1,0,0,0,337, - 339,1,0,0,0,338,325,1,0,0,0,338,331,1,0,0,0,339,341,1,0,0,0,340, - 306,1,0,0,0,340,322,1,0,0,0,340,338,1,0,0,0,341,39,1,0,0,0,342,344, - 7,2,0,0,343,342,1,0,0,0,343,344,1,0,0,0,344,345,1,0,0,0,345,346, - 5,30,0,0,346,349,5,89,0,0,347,349,5,205,0,0,348,343,1,0,0,0,348, - 347,1,0,0,0,349,41,1,0,0,0,350,351,5,118,0,0,351,360,3,74,37,0,352, - 353,5,178,0,0,353,354,5,218,0,0,354,355,3,74,37,0,355,356,5,228, - 0,0,356,360,1,0,0,0,357,358,5,178,0,0,358,360,3,74,37,0,359,350, - 1,0,0,0,359,352,1,0,0,0,359,357,1,0,0,0,360,43,1,0,0,0,361,362,5, - 143,0,0,362,365,3,52,26,0,363,364,5,117,0,0,364,366,3,52,26,0,365, - 363,1,0,0,0,365,366,1,0,0,0,366,45,1,0,0,0,367,370,3,78,39,0,368, - 369,7,7,0,0,369,371,3,78,39,0,370,368,1,0,0,0,370,371,1,0,0,0,371, - 47,1,0,0,0,372,377,3,50,25,0,373,374,5,205,0,0,374,376,3,50,25,0, - 375,373,1,0,0,0,376,379,1,0,0,0,377,375,1,0,0,0,377,378,1,0,0,0, - 378,49,1,0,0,0,379,377,1,0,0,0,380,382,3,78,39,0,381,383,7,8,0,0, - 382,381,1,0,0,0,382,383,1,0,0,0,383,386,1,0,0,0,384,385,5,116,0, - 0,385,387,7,9,0,0,386,384,1,0,0,0,386,387,1,0,0,0,387,390,1,0,0, - 0,388,389,5,25,0,0,389,391,5,198,0,0,390,388,1,0,0,0,390,391,1,0, - 0,0,391,51,1,0,0,0,392,395,3,104,52,0,393,394,5,230,0,0,394,396, - 3,104,52,0,395,393,1,0,0,0,395,396,1,0,0,0,396,53,1,0,0,0,397,402, - 3,56,28,0,398,399,5,205,0,0,399,401,3,56,28,0,400,398,1,0,0,0,401, - 404,1,0,0,0,402,400,1,0,0,0,402,403,1,0,0,0,403,55,1,0,0,0,404,402, - 1,0,0,0,405,406,3,116,58,0,406,407,5,211,0,0,407,408,3,106,53,0, - 408,57,1,0,0,0,409,411,3,60,30,0,410,409,1,0,0,0,410,411,1,0,0,0, - 411,413,1,0,0,0,412,414,3,62,31,0,413,412,1,0,0,0,413,414,1,0,0, - 0,414,416,1,0,0,0,415,417,3,64,32,0,416,415,1,0,0,0,416,417,1,0, - 0,0,417,59,1,0,0,0,418,419,5,125,0,0,419,420,5,18,0,0,420,421,3, - 74,37,0,421,61,1,0,0,0,422,423,5,121,0,0,423,424,5,18,0,0,424,425, - 3,48,24,0,425,63,1,0,0,0,426,427,7,10,0,0,427,428,3,66,33,0,428, - 65,1,0,0,0,429,436,3,68,34,0,430,431,5,16,0,0,431,432,3,68,34,0, - 432,433,5,6,0,0,433,434,3,68,34,0,434,436,1,0,0,0,435,429,1,0,0, - 0,435,430,1,0,0,0,436,67,1,0,0,0,437,438,5,32,0,0,438,450,5,141, - 0,0,439,440,5,174,0,0,440,450,5,127,0,0,441,442,5,174,0,0,442,450, - 5,63,0,0,443,444,3,104,52,0,444,445,5,127,0,0,445,450,1,0,0,0,446, - 447,3,104,52,0,447,448,5,63,0,0,448,450,1,0,0,0,449,437,1,0,0,0, - 449,439,1,0,0,0,449,441,1,0,0,0,449,443,1,0,0,0,449,446,1,0,0,0, - 450,69,1,0,0,0,451,452,3,78,39,0,452,453,5,0,0,1,453,71,1,0,0,0, - 454,502,3,116,58,0,455,456,3,116,58,0,456,457,5,218,0,0,457,458, - 3,116,58,0,458,465,3,72,36,0,459,460,5,205,0,0,460,461,3,116,58, - 0,461,462,3,72,36,0,462,464,1,0,0,0,463,459,1,0,0,0,464,467,1,0, - 0,0,465,463,1,0,0,0,465,466,1,0,0,0,466,468,1,0,0,0,467,465,1,0, - 0,0,468,469,5,228,0,0,469,502,1,0,0,0,470,471,3,116,58,0,471,472, - 5,218,0,0,472,477,3,120,60,0,473,474,5,205,0,0,474,476,3,120,60, - 0,475,473,1,0,0,0,476,479,1,0,0,0,477,475,1,0,0,0,477,478,1,0,0, - 0,478,480,1,0,0,0,479,477,1,0,0,0,480,481,5,228,0,0,481,502,1,0, - 0,0,482,483,3,116,58,0,483,484,5,218,0,0,484,489,3,72,36,0,485,486, - 5,205,0,0,486,488,3,72,36,0,487,485,1,0,0,0,488,491,1,0,0,0,489, - 487,1,0,0,0,489,490,1,0,0,0,490,492,1,0,0,0,491,489,1,0,0,0,492, - 493,5,228,0,0,493,502,1,0,0,0,494,495,3,116,58,0,495,497,5,218,0, - 0,496,498,3,74,37,0,497,496,1,0,0,0,497,498,1,0,0,0,498,499,1,0, - 0,0,499,500,5,228,0,0,500,502,1,0,0,0,501,454,1,0,0,0,501,455,1, - 0,0,0,501,470,1,0,0,0,501,482,1,0,0,0,501,494,1,0,0,0,502,73,1,0, - 0,0,503,508,3,76,38,0,504,505,5,205,0,0,505,507,3,76,38,0,506,504, - 1,0,0,0,507,510,1,0,0,0,508,506,1,0,0,0,508,509,1,0,0,0,509,75,1, - 0,0,0,510,508,1,0,0,0,511,512,3,94,47,0,512,513,5,209,0,0,513,515, - 1,0,0,0,514,511,1,0,0,0,514,515,1,0,0,0,515,516,1,0,0,0,516,523, - 5,201,0,0,517,518,5,218,0,0,518,519,3,2,1,0,519,520,5,228,0,0,520, - 523,1,0,0,0,521,523,3,78,39,0,522,514,1,0,0,0,522,517,1,0,0,0,522, - 521,1,0,0,0,523,77,1,0,0,0,524,525,6,39,-1,0,525,527,5,19,0,0,526, - 528,3,78,39,0,527,526,1,0,0,0,527,528,1,0,0,0,528,534,1,0,0,0,529, - 530,5,185,0,0,530,531,3,78,39,0,531,532,5,162,0,0,532,533,3,78,39, - 0,533,535,1,0,0,0,534,529,1,0,0,0,535,536,1,0,0,0,536,534,1,0,0, - 0,536,537,1,0,0,0,537,540,1,0,0,0,538,539,5,51,0,0,539,541,3,78, - 39,0,540,538,1,0,0,0,540,541,1,0,0,0,541,542,1,0,0,0,542,543,5,52, - 0,0,543,654,1,0,0,0,544,545,5,20,0,0,545,546,5,218,0,0,546,547,3, - 78,39,0,547,548,5,10,0,0,548,549,3,72,36,0,549,550,5,228,0,0,550, - 654,1,0,0,0,551,552,5,35,0,0,552,654,5,198,0,0,553,554,5,58,0,0, - 554,555,5,218,0,0,555,556,3,108,54,0,556,557,5,67,0,0,557,558,3, - 78,39,0,558,559,5,228,0,0,559,654,1,0,0,0,560,561,5,85,0,0,561,562, - 3,78,39,0,562,563,3,108,54,0,563,654,1,0,0,0,564,565,5,154,0,0,565, - 566,5,218,0,0,566,567,3,78,39,0,567,568,5,67,0,0,568,571,3,78,39, - 0,569,570,5,64,0,0,570,572,3,78,39,0,571,569,1,0,0,0,571,572,1,0, - 0,0,572,573,1,0,0,0,573,574,5,228,0,0,574,654,1,0,0,0,575,576,5, - 165,0,0,576,654,5,198,0,0,577,578,5,170,0,0,578,579,5,218,0,0,579, - 580,7,11,0,0,580,581,5,198,0,0,581,582,5,67,0,0,582,583,3,78,39, - 0,583,584,5,228,0,0,584,654,1,0,0,0,585,586,3,116,58,0,586,588,5, - 218,0,0,587,589,3,74,37,0,588,587,1,0,0,0,588,589,1,0,0,0,589,590, - 1,0,0,0,590,591,5,228,0,0,591,592,1,0,0,0,592,593,5,124,0,0,593, - 594,5,218,0,0,594,595,3,58,29,0,595,596,5,228,0,0,596,654,1,0,0, - 0,597,598,3,116,58,0,598,600,5,218,0,0,599,601,3,74,37,0,600,599, - 1,0,0,0,600,601,1,0,0,0,601,602,1,0,0,0,602,603,5,228,0,0,603,604, - 1,0,0,0,604,605,5,124,0,0,605,606,3,116,58,0,606,654,1,0,0,0,607, - 613,3,116,58,0,608,610,5,218,0,0,609,611,3,74,37,0,610,609,1,0,0, - 0,610,611,1,0,0,0,611,612,1,0,0,0,612,614,5,228,0,0,613,608,1,0, - 0,0,613,614,1,0,0,0,614,615,1,0,0,0,615,617,5,218,0,0,616,618,5, - 48,0,0,617,616,1,0,0,0,617,618,1,0,0,0,618,620,1,0,0,0,619,621,3, - 80,40,0,620,619,1,0,0,0,620,621,1,0,0,0,621,622,1,0,0,0,622,623, - 5,228,0,0,623,654,1,0,0,0,624,654,3,106,53,0,625,626,5,207,0,0,626, - 654,3,78,39,17,627,628,5,114,0,0,628,654,3,78,39,12,629,630,3,94, - 47,0,630,631,5,209,0,0,631,633,1,0,0,0,632,629,1,0,0,0,632,633,1, - 0,0,0,633,634,1,0,0,0,634,654,5,201,0,0,635,636,5,218,0,0,636,637, - 3,2,1,0,637,638,5,228,0,0,638,654,1,0,0,0,639,640,5,218,0,0,640, - 641,3,78,39,0,641,642,5,228,0,0,642,654,1,0,0,0,643,644,5,218,0, - 0,644,645,3,74,37,0,645,646,5,228,0,0,646,654,1,0,0,0,647,649,5, - 216,0,0,648,650,3,74,37,0,649,648,1,0,0,0,649,650,1,0,0,0,650,651, - 1,0,0,0,651,654,5,227,0,0,652,654,3,86,43,0,653,524,1,0,0,0,653, - 544,1,0,0,0,653,551,1,0,0,0,653,553,1,0,0,0,653,560,1,0,0,0,653, - 564,1,0,0,0,653,575,1,0,0,0,653,577,1,0,0,0,653,585,1,0,0,0,653, - 597,1,0,0,0,653,607,1,0,0,0,653,624,1,0,0,0,653,625,1,0,0,0,653, - 627,1,0,0,0,653,632,1,0,0,0,653,635,1,0,0,0,653,639,1,0,0,0,653, - 643,1,0,0,0,653,647,1,0,0,0,653,652,1,0,0,0,654,736,1,0,0,0,655, - 659,10,16,0,0,656,660,5,201,0,0,657,660,5,230,0,0,658,660,5,221, - 0,0,659,656,1,0,0,0,659,657,1,0,0,0,659,658,1,0,0,0,660,661,1,0, - 0,0,661,735,3,78,39,17,662,666,10,15,0,0,663,667,5,222,0,0,664,667, - 5,207,0,0,665,667,5,206,0,0,666,663,1,0,0,0,666,664,1,0,0,0,666, - 665,1,0,0,0,667,668,1,0,0,0,668,735,3,78,39,16,669,688,10,14,0,0, - 670,689,5,210,0,0,671,689,5,211,0,0,672,689,5,220,0,0,673,689,5, - 217,0,0,674,689,5,212,0,0,675,689,5,219,0,0,676,689,5,213,0,0,677, - 679,5,70,0,0,678,677,1,0,0,0,678,679,1,0,0,0,679,681,1,0,0,0,680, - 682,5,114,0,0,681,680,1,0,0,0,681,682,1,0,0,0,682,683,1,0,0,0,683, - 689,5,79,0,0,684,686,5,114,0,0,685,684,1,0,0,0,685,686,1,0,0,0,686, - 687,1,0,0,0,687,689,7,12,0,0,688,670,1,0,0,0,688,671,1,0,0,0,688, - 672,1,0,0,0,688,673,1,0,0,0,688,674,1,0,0,0,688,675,1,0,0,0,688, - 676,1,0,0,0,688,678,1,0,0,0,688,685,1,0,0,0,689,690,1,0,0,0,690, - 735,3,78,39,15,691,692,10,11,0,0,692,693,5,6,0,0,693,735,3,78,39, - 12,694,695,10,10,0,0,695,696,5,120,0,0,696,735,3,78,39,11,697,699, - 10,9,0,0,698,700,5,114,0,0,699,698,1,0,0,0,699,700,1,0,0,0,700,701, - 1,0,0,0,701,702,5,16,0,0,702,703,3,78,39,0,703,704,5,6,0,0,704,705, - 3,78,39,10,705,735,1,0,0,0,706,707,10,8,0,0,707,708,5,223,0,0,708, - 709,3,78,39,0,709,710,5,204,0,0,710,711,3,78,39,8,711,735,1,0,0, - 0,712,713,10,19,0,0,713,714,5,216,0,0,714,715,3,78,39,0,715,716, - 5,227,0,0,716,735,1,0,0,0,717,718,10,18,0,0,718,719,5,209,0,0,719, - 735,5,196,0,0,720,721,10,13,0,0,721,723,5,87,0,0,722,724,5,114,0, - 0,723,722,1,0,0,0,723,724,1,0,0,0,724,725,1,0,0,0,725,735,5,115, - 0,0,726,732,10,7,0,0,727,733,3,114,57,0,728,729,5,10,0,0,729,733, - 3,116,58,0,730,731,5,10,0,0,731,733,5,198,0,0,732,727,1,0,0,0,732, - 728,1,0,0,0,732,730,1,0,0,0,733,735,1,0,0,0,734,655,1,0,0,0,734, - 662,1,0,0,0,734,669,1,0,0,0,734,691,1,0,0,0,734,694,1,0,0,0,734, - 697,1,0,0,0,734,706,1,0,0,0,734,712,1,0,0,0,734,717,1,0,0,0,734, - 720,1,0,0,0,734,726,1,0,0,0,735,738,1,0,0,0,736,734,1,0,0,0,736, - 737,1,0,0,0,737,79,1,0,0,0,738,736,1,0,0,0,739,744,3,82,41,0,740, - 741,5,205,0,0,741,743,3,82,41,0,742,740,1,0,0,0,743,746,1,0,0,0, - 744,742,1,0,0,0,744,745,1,0,0,0,745,81,1,0,0,0,746,744,1,0,0,0,747, - 750,3,84,42,0,748,750,3,78,39,0,749,747,1,0,0,0,749,748,1,0,0,0, - 750,83,1,0,0,0,751,752,5,218,0,0,752,757,3,116,58,0,753,754,5,205, - 0,0,754,756,3,116,58,0,755,753,1,0,0,0,756,759,1,0,0,0,757,755,1, - 0,0,0,757,758,1,0,0,0,758,760,1,0,0,0,759,757,1,0,0,0,760,761,5, - 228,0,0,761,771,1,0,0,0,762,767,3,116,58,0,763,764,5,205,0,0,764, - 766,3,116,58,0,765,763,1,0,0,0,766,769,1,0,0,0,767,765,1,0,0,0,767, - 768,1,0,0,0,768,771,1,0,0,0,769,767,1,0,0,0,770,751,1,0,0,0,770, - 762,1,0,0,0,771,772,1,0,0,0,772,773,5,200,0,0,773,774,3,78,39,0, - 774,85,1,0,0,0,775,783,5,199,0,0,776,777,3,94,47,0,777,778,5,209, - 0,0,778,780,1,0,0,0,779,776,1,0,0,0,779,780,1,0,0,0,780,781,1,0, - 0,0,781,783,3,88,44,0,782,775,1,0,0,0,782,779,1,0,0,0,783,87,1,0, - 0,0,784,789,3,116,58,0,785,786,5,209,0,0,786,788,3,116,58,0,787, - 785,1,0,0,0,788,791,1,0,0,0,789,787,1,0,0,0,789,790,1,0,0,0,790, - 89,1,0,0,0,791,789,1,0,0,0,792,793,6,45,-1,0,793,800,3,94,47,0,794, - 800,3,92,46,0,795,796,5,218,0,0,796,797,3,2,1,0,797,798,5,228,0, - 0,798,800,1,0,0,0,799,792,1,0,0,0,799,794,1,0,0,0,799,795,1,0,0, - 0,800,809,1,0,0,0,801,805,10,1,0,0,802,806,3,114,57,0,803,804,5, - 10,0,0,804,806,3,116,58,0,805,802,1,0,0,0,805,803,1,0,0,0,806,808, - 1,0,0,0,807,801,1,0,0,0,808,811,1,0,0,0,809,807,1,0,0,0,809,810, - 1,0,0,0,810,91,1,0,0,0,811,809,1,0,0,0,812,813,3,116,58,0,813,815, - 5,218,0,0,814,816,3,96,48,0,815,814,1,0,0,0,815,816,1,0,0,0,816, - 817,1,0,0,0,817,818,5,228,0,0,818,93,1,0,0,0,819,820,3,100,50,0, - 820,821,5,209,0,0,821,823,1,0,0,0,822,819,1,0,0,0,822,823,1,0,0, - 0,823,824,1,0,0,0,824,825,3,116,58,0,825,95,1,0,0,0,826,831,3,98, - 49,0,827,828,5,205,0,0,828,830,3,98,49,0,829,827,1,0,0,0,830,833, - 1,0,0,0,831,829,1,0,0,0,831,832,1,0,0,0,832,97,1,0,0,0,833,831,1, - 0,0,0,834,838,3,88,44,0,835,838,3,92,46,0,836,838,3,106,53,0,837, - 834,1,0,0,0,837,835,1,0,0,0,837,836,1,0,0,0,838,99,1,0,0,0,839,840, - 3,116,58,0,840,101,1,0,0,0,841,850,5,194,0,0,842,843,5,209,0,0,843, - 850,7,13,0,0,844,845,5,196,0,0,845,847,5,209,0,0,846,848,7,13,0, - 0,847,846,1,0,0,0,847,848,1,0,0,0,848,850,1,0,0,0,849,841,1,0,0, - 0,849,842,1,0,0,0,849,844,1,0,0,0,850,103,1,0,0,0,851,853,7,14,0, - 0,852,851,1,0,0,0,852,853,1,0,0,0,853,860,1,0,0,0,854,861,3,102, - 51,0,855,861,5,195,0,0,856,861,5,196,0,0,857,861,5,197,0,0,858,861, - 5,81,0,0,859,861,5,112,0,0,860,854,1,0,0,0,860,855,1,0,0,0,860,856, - 1,0,0,0,860,857,1,0,0,0,860,858,1,0,0,0,860,859,1,0,0,0,861,105, - 1,0,0,0,862,866,3,104,52,0,863,866,5,198,0,0,864,866,5,115,0,0,865, - 862,1,0,0,0,865,863,1,0,0,0,865,864,1,0,0,0,866,107,1,0,0,0,867, - 868,7,15,0,0,868,109,1,0,0,0,869,870,7,16,0,0,870,111,1,0,0,0,871, - 872,7,17,0,0,872,113,1,0,0,0,873,876,5,193,0,0,874,876,3,112,56, - 0,875,873,1,0,0,0,875,874,1,0,0,0,876,115,1,0,0,0,877,881,5,193, - 0,0,878,881,3,108,54,0,879,881,3,110,55,0,880,877,1,0,0,0,880,878, - 1,0,0,0,880,879,1,0,0,0,881,117,1,0,0,0,882,885,3,116,58,0,883,885, - 5,115,0,0,884,882,1,0,0,0,884,883,1,0,0,0,885,119,1,0,0,0,886,887, - 5,198,0,0,887,888,5,211,0,0,888,889,3,104,52,0,889,121,1,0,0,0,115, - 124,134,142,145,149,152,156,159,162,165,168,171,175,179,182,185, - 188,191,194,203,209,236,258,266,269,275,283,286,292,294,298,303, - 306,309,313,317,320,322,325,329,333,336,338,340,343,348,359,365, - 370,377,382,386,390,395,402,410,413,416,435,449,465,477,489,497, - 501,508,514,522,527,536,540,571,588,600,610,613,617,620,632,649, - 653,659,666,678,681,685,688,699,723,732,734,736,744,749,757,767, - 770,779,782,789,799,805,809,815,822,831,837,847,849,852,860,865, - 875,880,884 + 59,7,59,1,0,1,0,3,0,123,8,0,1,0,1,0,1,1,1,1,1,1,1,1,5,1,131,8,1, + 10,1,12,1,134,9,1,1,2,1,2,1,2,1,2,1,2,3,2,141,8,2,1,3,3,3,144,8, + 3,1,3,1,3,3,3,148,8,3,1,3,3,3,151,8,3,1,3,1,3,3,3,155,8,3,1,3,3, + 3,158,8,3,1,3,3,3,161,8,3,1,3,3,3,164,8,3,1,3,3,3,167,8,3,1,3,3, + 3,170,8,3,1,3,1,3,3,3,174,8,3,1,3,1,3,3,3,178,8,3,1,3,3,3,181,8, + 3,1,3,3,3,184,8,3,1,3,3,3,187,8,3,1,3,3,3,190,8,3,1,4,1,4,1,4,1, + 5,1,5,1,5,1,5,3,5,199,8,5,1,6,1,6,1,6,1,7,3,7,205,8,7,1,7,1,7,1, + 7,1,7,1,8,1,8,1,8,1,8,1,8,1,8,1,8,1,9,1,9,1,9,1,10,1,10,1,10,1,11, + 1,11,1,11,1,11,1,11,1,11,1,11,1,11,3,11,232,8,11,1,12,1,12,1,12, + 1,13,1,13,1,13,1,13,1,14,1,14,1,14,1,14,1,15,1,15,1,15,1,15,1,15, + 1,15,3,15,251,8,15,1,16,1,16,1,16,1,17,1,17,1,17,3,17,259,8,17,1, + 17,3,17,262,8,17,1,17,1,17,1,17,1,17,3,17,268,8,17,1,17,1,17,1,17, + 1,17,1,17,1,17,3,17,276,8,17,1,17,3,17,279,8,17,1,17,1,17,1,17,1, + 17,5,17,285,8,17,10,17,12,17,288,9,17,1,18,3,18,291,8,18,1,18,1, + 18,1,18,3,18,296,8,18,1,18,3,18,299,8,18,1,18,3,18,302,8,18,1,18, + 1,18,3,18,306,8,18,1,18,1,18,3,18,310,8,18,1,18,3,18,313,8,18,3, + 18,315,8,18,1,18,3,18,318,8,18,1,18,1,18,3,18,322,8,18,1,18,1,18, + 3,18,326,8,18,1,18,3,18,329,8,18,3,18,331,8,18,3,18,333,8,18,1,19, + 3,19,336,8,19,1,19,1,19,1,19,3,19,341,8,19,1,20,1,20,1,20,1,20,1, + 20,1,20,1,20,1,20,1,20,3,20,352,8,20,1,21,1,21,1,21,1,21,3,21,358, + 8,21,1,22,1,22,1,22,3,22,363,8,22,1,23,1,23,1,23,5,23,368,8,23,10, + 23,12,23,371,9,23,1,24,1,24,3,24,375,8,24,1,24,1,24,3,24,379,8,24, + 1,24,1,24,3,24,383,8,24,1,25,1,25,1,25,3,25,388,8,25,1,26,1,26,1, + 26,5,26,393,8,26,10,26,12,26,396,9,26,1,27,1,27,1,27,1,27,1,28,3, + 28,403,8,28,1,28,3,28,406,8,28,1,28,3,28,409,8,28,1,29,1,29,1,29, + 1,29,1,30,1,30,1,30,1,30,1,31,1,31,1,31,1,32,1,32,1,32,1,32,1,32, + 1,32,3,32,428,8,32,1,33,1,33,1,33,1,33,1,33,1,33,1,33,1,33,1,33, + 1,33,1,33,1,33,3,33,442,8,33,1,34,1,34,1,34,1,35,1,35,1,35,1,35, + 1,35,1,35,1,35,1,35,1,35,5,35,456,8,35,10,35,12,35,459,9,35,1,35, + 1,35,1,35,1,35,1,35,1,35,1,35,5,35,468,8,35,10,35,12,35,471,9,35, + 1,35,1,35,1,35,1,35,1,35,1,35,1,35,5,35,480,8,35,10,35,12,35,483, + 9,35,1,35,1,35,1,35,1,35,1,35,3,35,490,8,35,1,35,1,35,3,35,494,8, + 35,1,36,1,36,1,36,5,36,499,8,36,10,36,12,36,502,9,36,1,37,1,37,1, + 37,3,37,507,8,37,1,37,1,37,1,37,1,37,1,37,1,37,3,37,515,8,37,1,38, + 1,38,1,38,3,38,520,8,38,1,38,1,38,1,38,1,38,1,38,4,38,527,8,38,11, + 38,12,38,528,1,38,1,38,3,38,533,8,38,1,38,1,38,1,38,1,38,1,38,1, + 38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1, + 38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,3,38,564,8, + 38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1, + 38,1,38,1,38,3,38,581,8,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1, + 38,1,38,1,38,3,38,593,8,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1, + 38,3,38,603,8,38,1,38,3,38,606,8,38,1,38,1,38,3,38,610,8,38,1,38, + 3,38,613,8,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38, + 3,38,625,8,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38, + 1,38,1,38,1,38,1,38,1,38,3,38,642,8,38,1,38,1,38,3,38,646,8,38,1, + 38,1,38,1,38,1,38,3,38,652,8,38,1,38,1,38,1,38,1,38,1,38,3,38,659, + 8,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,3,38,671, + 8,38,1,38,3,38,674,8,38,1,38,1,38,3,38,678,8,38,1,38,3,38,681,8, + 38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,3,38,692,8,38,1, + 38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1, + 38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,1,38,3,38,716,8,38,1,38,1, + 38,1,38,1,38,1,38,1,38,1,38,3,38,725,8,38,5,38,727,8,38,10,38,12, + 38,730,9,38,1,39,1,39,1,39,5,39,735,8,39,10,39,12,39,738,9,39,1, + 40,1,40,3,40,742,8,40,1,41,1,41,1,41,1,41,5,41,748,8,41,10,41,12, + 41,751,9,41,1,41,1,41,1,41,1,41,1,41,5,41,758,8,41,10,41,12,41,761, + 9,41,3,41,763,8,41,1,41,1,41,1,41,1,42,1,42,1,42,1,42,3,42,772,8, + 42,1,42,3,42,775,8,42,1,43,1,43,1,43,5,43,780,8,43,10,43,12,43,783, + 9,43,1,44,1,44,1,44,1,44,1,44,1,44,1,44,3,44,792,8,44,1,44,1,44, + 1,44,1,44,3,44,798,8,44,5,44,800,8,44,10,44,12,44,803,9,44,1,45, + 1,45,1,45,3,45,808,8,45,1,45,1,45,1,46,1,46,1,46,3,46,815,8,46,1, + 46,1,46,1,47,1,47,1,47,5,47,822,8,47,10,47,12,47,825,9,47,1,48,1, + 48,1,48,3,48,830,8,48,1,49,1,49,1,50,1,50,1,50,1,50,1,50,1,50,3, + 50,840,8,50,3,50,842,8,50,1,51,3,51,845,8,51,1,51,1,51,1,51,1,51, + 1,51,1,51,3,51,853,8,51,1,52,1,52,1,52,3,52,858,8,52,1,53,1,53,1, + 54,1,54,1,55,1,55,1,56,1,56,3,56,868,8,56,1,57,1,57,1,57,3,57,873, + 8,57,1,58,1,58,3,58,877,8,58,1,59,1,59,1,59,1,59,1,59,0,3,34,76, + 88,60,0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40, + 42,44,46,48,50,52,54,56,58,60,62,64,66,68,70,72,74,76,78,80,82,84, + 86,88,90,92,94,96,98,100,102,104,106,108,110,112,114,116,118,0,18, + 2,0,31,31,140,140,2,0,83,83,95,95,2,0,70,70,100,100,3,0,4,4,8,8, + 12,12,4,0,4,4,7,8,12,12,146,146,2,0,95,95,139,139,2,0,4,4,8,8,2, + 0,117,117,205,205,2,0,11,11,41,42,2,0,61,61,92,92,2,0,132,132,142, + 142,3,0,17,17,94,94,169,169,2,0,78,78,97,97,1,0,195,196,2,0,207, + 207,222,222,8,0,36,36,75,75,107,107,109,109,131,131,144,144,184, + 184,189,189,12,0,2,35,37,74,76,80,82,106,108,108,110,111,113,114, + 116,129,132,143,145,183,185,188,190,191,4,0,35,35,61,61,76,76,90, + 90,993,0,122,1,0,0,0,2,126,1,0,0,0,4,140,1,0,0,0,6,143,1,0,0,0,8, + 191,1,0,0,0,10,194,1,0,0,0,12,200,1,0,0,0,14,204,1,0,0,0,16,210, + 1,0,0,0,18,217,1,0,0,0,20,220,1,0,0,0,22,223,1,0,0,0,24,233,1,0, + 0,0,26,236,1,0,0,0,28,240,1,0,0,0,30,244,1,0,0,0,32,252,1,0,0,0, + 34,267,1,0,0,0,36,332,1,0,0,0,38,340,1,0,0,0,40,351,1,0,0,0,42,353, + 1,0,0,0,44,359,1,0,0,0,46,364,1,0,0,0,48,372,1,0,0,0,50,384,1,0, + 0,0,52,389,1,0,0,0,54,397,1,0,0,0,56,402,1,0,0,0,58,410,1,0,0,0, + 60,414,1,0,0,0,62,418,1,0,0,0,64,427,1,0,0,0,66,441,1,0,0,0,68,443, + 1,0,0,0,70,493,1,0,0,0,72,495,1,0,0,0,74,514,1,0,0,0,76,645,1,0, + 0,0,78,731,1,0,0,0,80,741,1,0,0,0,82,762,1,0,0,0,84,774,1,0,0,0, + 86,776,1,0,0,0,88,791,1,0,0,0,90,804,1,0,0,0,92,814,1,0,0,0,94,818, + 1,0,0,0,96,829,1,0,0,0,98,831,1,0,0,0,100,841,1,0,0,0,102,844,1, + 0,0,0,104,857,1,0,0,0,106,859,1,0,0,0,108,861,1,0,0,0,110,863,1, + 0,0,0,112,867,1,0,0,0,114,872,1,0,0,0,116,876,1,0,0,0,118,878,1, + 0,0,0,120,123,3,2,1,0,121,123,3,6,3,0,122,120,1,0,0,0,122,121,1, + 0,0,0,123,124,1,0,0,0,124,125,5,0,0,1,125,1,1,0,0,0,126,132,3,4, + 2,0,127,128,5,175,0,0,128,129,5,4,0,0,129,131,3,4,2,0,130,127,1, + 0,0,0,131,134,1,0,0,0,132,130,1,0,0,0,132,133,1,0,0,0,133,3,1,0, + 0,0,134,132,1,0,0,0,135,141,3,6,3,0,136,137,5,218,0,0,137,138,3, + 2,1,0,138,139,5,228,0,0,139,141,1,0,0,0,140,135,1,0,0,0,140,136, + 1,0,0,0,141,5,1,0,0,0,142,144,3,8,4,0,143,142,1,0,0,0,143,144,1, + 0,0,0,144,145,1,0,0,0,145,147,5,145,0,0,146,148,5,48,0,0,147,146, + 1,0,0,0,147,148,1,0,0,0,148,150,1,0,0,0,149,151,3,10,5,0,150,149, + 1,0,0,0,150,151,1,0,0,0,151,152,1,0,0,0,152,154,3,72,36,0,153,155, + 3,12,6,0,154,153,1,0,0,0,154,155,1,0,0,0,155,157,1,0,0,0,156,158, + 3,14,7,0,157,156,1,0,0,0,157,158,1,0,0,0,158,160,1,0,0,0,159,161, + 3,16,8,0,160,159,1,0,0,0,160,161,1,0,0,0,161,163,1,0,0,0,162,164, + 3,18,9,0,163,162,1,0,0,0,163,164,1,0,0,0,164,166,1,0,0,0,165,167, + 3,20,10,0,166,165,1,0,0,0,166,167,1,0,0,0,167,169,1,0,0,0,168,170, + 3,22,11,0,169,168,1,0,0,0,169,170,1,0,0,0,170,173,1,0,0,0,171,172, + 5,188,0,0,172,174,7,0,0,0,173,171,1,0,0,0,173,174,1,0,0,0,174,177, + 1,0,0,0,175,176,5,188,0,0,176,178,5,168,0,0,177,175,1,0,0,0,177, + 178,1,0,0,0,178,180,1,0,0,0,179,181,3,24,12,0,180,179,1,0,0,0,180, + 181,1,0,0,0,181,183,1,0,0,0,182,184,3,26,13,0,183,182,1,0,0,0,183, + 184,1,0,0,0,184,186,1,0,0,0,185,187,3,30,15,0,186,185,1,0,0,0,186, + 187,1,0,0,0,187,189,1,0,0,0,188,190,3,32,16,0,189,188,1,0,0,0,189, + 190,1,0,0,0,190,7,1,0,0,0,191,192,5,188,0,0,192,193,3,72,36,0,193, + 9,1,0,0,0,194,195,5,167,0,0,195,198,5,196,0,0,196,197,5,188,0,0, + 197,199,5,163,0,0,198,196,1,0,0,0,198,199,1,0,0,0,199,11,1,0,0,0, + 200,201,5,67,0,0,201,202,3,34,17,0,202,13,1,0,0,0,203,205,7,1,0, + 0,204,203,1,0,0,0,204,205,1,0,0,0,205,206,1,0,0,0,206,207,5,9,0, + 0,207,208,5,89,0,0,208,209,3,72,36,0,209,15,1,0,0,0,210,211,5,187, + 0,0,211,212,3,114,57,0,212,213,5,10,0,0,213,214,5,218,0,0,214,215, + 3,56,28,0,215,216,5,228,0,0,216,17,1,0,0,0,217,218,5,128,0,0,218, + 219,3,76,38,0,219,19,1,0,0,0,220,221,5,186,0,0,221,222,3,76,38,0, + 222,21,1,0,0,0,223,224,5,72,0,0,224,231,5,18,0,0,225,226,7,0,0,0, + 226,227,5,218,0,0,227,228,3,72,36,0,228,229,5,228,0,0,229,232,1, + 0,0,0,230,232,3,72,36,0,231,225,1,0,0,0,231,230,1,0,0,0,232,23,1, + 0,0,0,233,234,5,73,0,0,234,235,3,76,38,0,235,25,1,0,0,0,236,237, + 5,121,0,0,237,238,5,18,0,0,238,239,3,46,23,0,239,27,1,0,0,0,240, + 241,5,121,0,0,241,242,5,18,0,0,242,243,3,72,36,0,243,29,1,0,0,0, + 244,245,5,98,0,0,245,250,3,44,22,0,246,247,5,188,0,0,247,251,5,163, + 0,0,248,249,5,18,0,0,249,251,3,72,36,0,250,246,1,0,0,0,250,248,1, + 0,0,0,250,251,1,0,0,0,251,31,1,0,0,0,252,253,5,149,0,0,253,254,3, + 52,26,0,254,33,1,0,0,0,255,256,6,17,-1,0,256,258,3,88,44,0,257,259, + 5,60,0,0,258,257,1,0,0,0,258,259,1,0,0,0,259,261,1,0,0,0,260,262, + 3,42,21,0,261,260,1,0,0,0,261,262,1,0,0,0,262,268,1,0,0,0,263,264, + 5,218,0,0,264,265,3,34,17,0,265,266,5,228,0,0,266,268,1,0,0,0,267, + 255,1,0,0,0,267,263,1,0,0,0,268,286,1,0,0,0,269,270,10,3,0,0,270, + 271,3,38,19,0,271,272,3,34,17,4,272,285,1,0,0,0,273,275,10,4,0,0, + 274,276,7,2,0,0,275,274,1,0,0,0,275,276,1,0,0,0,276,278,1,0,0,0, + 277,279,3,36,18,0,278,277,1,0,0,0,278,279,1,0,0,0,279,280,1,0,0, + 0,280,281,5,89,0,0,281,282,3,34,17,0,282,283,3,40,20,0,283,285,1, + 0,0,0,284,269,1,0,0,0,284,273,1,0,0,0,285,288,1,0,0,0,286,284,1, + 0,0,0,286,287,1,0,0,0,287,35,1,0,0,0,288,286,1,0,0,0,289,291,7,3, + 0,0,290,289,1,0,0,0,290,291,1,0,0,0,291,292,1,0,0,0,292,299,5,83, + 0,0,293,295,5,83,0,0,294,296,7,3,0,0,295,294,1,0,0,0,295,296,1,0, + 0,0,296,299,1,0,0,0,297,299,7,3,0,0,298,290,1,0,0,0,298,293,1,0, + 0,0,298,297,1,0,0,0,299,333,1,0,0,0,300,302,7,4,0,0,301,300,1,0, + 0,0,301,302,1,0,0,0,302,303,1,0,0,0,303,305,7,5,0,0,304,306,5,122, + 0,0,305,304,1,0,0,0,305,306,1,0,0,0,306,315,1,0,0,0,307,309,7,5, + 0,0,308,310,5,122,0,0,309,308,1,0,0,0,309,310,1,0,0,0,310,312,1, + 0,0,0,311,313,7,4,0,0,312,311,1,0,0,0,312,313,1,0,0,0,313,315,1, + 0,0,0,314,301,1,0,0,0,314,307,1,0,0,0,315,333,1,0,0,0,316,318,7, + 6,0,0,317,316,1,0,0,0,317,318,1,0,0,0,318,319,1,0,0,0,319,321,5, + 68,0,0,320,322,5,122,0,0,321,320,1,0,0,0,321,322,1,0,0,0,322,331, + 1,0,0,0,323,325,5,68,0,0,324,326,5,122,0,0,325,324,1,0,0,0,325,326, + 1,0,0,0,326,328,1,0,0,0,327,329,7,6,0,0,328,327,1,0,0,0,328,329, + 1,0,0,0,329,331,1,0,0,0,330,317,1,0,0,0,330,323,1,0,0,0,331,333, + 1,0,0,0,332,298,1,0,0,0,332,314,1,0,0,0,332,330,1,0,0,0,333,37,1, + 0,0,0,334,336,7,2,0,0,335,334,1,0,0,0,335,336,1,0,0,0,336,337,1, + 0,0,0,337,338,5,30,0,0,338,341,5,89,0,0,339,341,5,205,0,0,340,335, + 1,0,0,0,340,339,1,0,0,0,341,39,1,0,0,0,342,343,5,118,0,0,343,352, + 3,72,36,0,344,345,5,178,0,0,345,346,5,218,0,0,346,347,3,72,36,0, + 347,348,5,228,0,0,348,352,1,0,0,0,349,350,5,178,0,0,350,352,3,72, + 36,0,351,342,1,0,0,0,351,344,1,0,0,0,351,349,1,0,0,0,352,41,1,0, + 0,0,353,354,5,143,0,0,354,357,3,50,25,0,355,356,5,117,0,0,356,358, + 3,50,25,0,357,355,1,0,0,0,357,358,1,0,0,0,358,43,1,0,0,0,359,362, + 3,76,38,0,360,361,7,7,0,0,361,363,3,76,38,0,362,360,1,0,0,0,362, + 363,1,0,0,0,363,45,1,0,0,0,364,369,3,48,24,0,365,366,5,205,0,0,366, + 368,3,48,24,0,367,365,1,0,0,0,368,371,1,0,0,0,369,367,1,0,0,0,369, + 370,1,0,0,0,370,47,1,0,0,0,371,369,1,0,0,0,372,374,3,76,38,0,373, + 375,7,8,0,0,374,373,1,0,0,0,374,375,1,0,0,0,375,378,1,0,0,0,376, + 377,5,116,0,0,377,379,7,9,0,0,378,376,1,0,0,0,378,379,1,0,0,0,379, + 382,1,0,0,0,380,381,5,25,0,0,381,383,5,198,0,0,382,380,1,0,0,0,382, + 383,1,0,0,0,383,49,1,0,0,0,384,387,3,102,51,0,385,386,5,230,0,0, + 386,388,3,102,51,0,387,385,1,0,0,0,387,388,1,0,0,0,388,51,1,0,0, + 0,389,394,3,54,27,0,390,391,5,205,0,0,391,393,3,54,27,0,392,390, + 1,0,0,0,393,396,1,0,0,0,394,392,1,0,0,0,394,395,1,0,0,0,395,53,1, + 0,0,0,396,394,1,0,0,0,397,398,3,114,57,0,398,399,5,211,0,0,399,400, + 3,104,52,0,400,55,1,0,0,0,401,403,3,58,29,0,402,401,1,0,0,0,402, + 403,1,0,0,0,403,405,1,0,0,0,404,406,3,60,30,0,405,404,1,0,0,0,405, + 406,1,0,0,0,406,408,1,0,0,0,407,409,3,62,31,0,408,407,1,0,0,0,408, + 409,1,0,0,0,409,57,1,0,0,0,410,411,5,125,0,0,411,412,5,18,0,0,412, + 413,3,72,36,0,413,59,1,0,0,0,414,415,5,121,0,0,415,416,5,18,0,0, + 416,417,3,46,23,0,417,61,1,0,0,0,418,419,7,10,0,0,419,420,3,64,32, + 0,420,63,1,0,0,0,421,428,3,66,33,0,422,423,5,16,0,0,423,424,3,66, + 33,0,424,425,5,6,0,0,425,426,3,66,33,0,426,428,1,0,0,0,427,421,1, + 0,0,0,427,422,1,0,0,0,428,65,1,0,0,0,429,430,5,32,0,0,430,442,5, + 141,0,0,431,432,5,174,0,0,432,442,5,127,0,0,433,434,5,174,0,0,434, + 442,5,63,0,0,435,436,3,102,51,0,436,437,5,127,0,0,437,442,1,0,0, + 0,438,439,3,102,51,0,439,440,5,63,0,0,440,442,1,0,0,0,441,429,1, + 0,0,0,441,431,1,0,0,0,441,433,1,0,0,0,441,435,1,0,0,0,441,438,1, + 0,0,0,442,67,1,0,0,0,443,444,3,76,38,0,444,445,5,0,0,1,445,69,1, + 0,0,0,446,494,3,114,57,0,447,448,3,114,57,0,448,449,5,218,0,0,449, + 450,3,114,57,0,450,457,3,70,35,0,451,452,5,205,0,0,452,453,3,114, + 57,0,453,454,3,70,35,0,454,456,1,0,0,0,455,451,1,0,0,0,456,459,1, + 0,0,0,457,455,1,0,0,0,457,458,1,0,0,0,458,460,1,0,0,0,459,457,1, + 0,0,0,460,461,5,228,0,0,461,494,1,0,0,0,462,463,3,114,57,0,463,464, + 5,218,0,0,464,469,3,118,59,0,465,466,5,205,0,0,466,468,3,118,59, + 0,467,465,1,0,0,0,468,471,1,0,0,0,469,467,1,0,0,0,469,470,1,0,0, + 0,470,472,1,0,0,0,471,469,1,0,0,0,472,473,5,228,0,0,473,494,1,0, + 0,0,474,475,3,114,57,0,475,476,5,218,0,0,476,481,3,70,35,0,477,478, + 5,205,0,0,478,480,3,70,35,0,479,477,1,0,0,0,480,483,1,0,0,0,481, + 479,1,0,0,0,481,482,1,0,0,0,482,484,1,0,0,0,483,481,1,0,0,0,484, + 485,5,228,0,0,485,494,1,0,0,0,486,487,3,114,57,0,487,489,5,218,0, + 0,488,490,3,72,36,0,489,488,1,0,0,0,489,490,1,0,0,0,490,491,1,0, + 0,0,491,492,5,228,0,0,492,494,1,0,0,0,493,446,1,0,0,0,493,447,1, + 0,0,0,493,462,1,0,0,0,493,474,1,0,0,0,493,486,1,0,0,0,494,71,1,0, + 0,0,495,500,3,74,37,0,496,497,5,205,0,0,497,499,3,74,37,0,498,496, + 1,0,0,0,499,502,1,0,0,0,500,498,1,0,0,0,500,501,1,0,0,0,501,73,1, + 0,0,0,502,500,1,0,0,0,503,504,3,92,46,0,504,505,5,209,0,0,505,507, + 1,0,0,0,506,503,1,0,0,0,506,507,1,0,0,0,507,508,1,0,0,0,508,515, + 5,201,0,0,509,510,5,218,0,0,510,511,3,2,1,0,511,512,5,228,0,0,512, + 515,1,0,0,0,513,515,3,76,38,0,514,506,1,0,0,0,514,509,1,0,0,0,514, + 513,1,0,0,0,515,75,1,0,0,0,516,517,6,38,-1,0,517,519,5,19,0,0,518, + 520,3,76,38,0,519,518,1,0,0,0,519,520,1,0,0,0,520,526,1,0,0,0,521, + 522,5,185,0,0,522,523,3,76,38,0,523,524,5,162,0,0,524,525,3,76,38, + 0,525,527,1,0,0,0,526,521,1,0,0,0,527,528,1,0,0,0,528,526,1,0,0, + 0,528,529,1,0,0,0,529,532,1,0,0,0,530,531,5,51,0,0,531,533,3,76, + 38,0,532,530,1,0,0,0,532,533,1,0,0,0,533,534,1,0,0,0,534,535,5,52, + 0,0,535,646,1,0,0,0,536,537,5,20,0,0,537,538,5,218,0,0,538,539,3, + 76,38,0,539,540,5,10,0,0,540,541,3,70,35,0,541,542,5,228,0,0,542, + 646,1,0,0,0,543,544,5,35,0,0,544,646,5,198,0,0,545,546,5,58,0,0, + 546,547,5,218,0,0,547,548,3,106,53,0,548,549,5,67,0,0,549,550,3, + 76,38,0,550,551,5,228,0,0,551,646,1,0,0,0,552,553,5,85,0,0,553,554, + 3,76,38,0,554,555,3,106,53,0,555,646,1,0,0,0,556,557,5,154,0,0,557, + 558,5,218,0,0,558,559,3,76,38,0,559,560,5,67,0,0,560,563,3,76,38, + 0,561,562,5,64,0,0,562,564,3,76,38,0,563,561,1,0,0,0,563,564,1,0, + 0,0,564,565,1,0,0,0,565,566,5,228,0,0,566,646,1,0,0,0,567,568,5, + 165,0,0,568,646,5,198,0,0,569,570,5,170,0,0,570,571,5,218,0,0,571, + 572,7,11,0,0,572,573,5,198,0,0,573,574,5,67,0,0,574,575,3,76,38, + 0,575,576,5,228,0,0,576,646,1,0,0,0,577,578,3,114,57,0,578,580,5, + 218,0,0,579,581,3,72,36,0,580,579,1,0,0,0,580,581,1,0,0,0,581,582, + 1,0,0,0,582,583,5,228,0,0,583,584,1,0,0,0,584,585,5,124,0,0,585, + 586,5,218,0,0,586,587,3,56,28,0,587,588,5,228,0,0,588,646,1,0,0, + 0,589,590,3,114,57,0,590,592,5,218,0,0,591,593,3,72,36,0,592,591, + 1,0,0,0,592,593,1,0,0,0,593,594,1,0,0,0,594,595,5,228,0,0,595,596, + 1,0,0,0,596,597,5,124,0,0,597,598,3,114,57,0,598,646,1,0,0,0,599, + 605,3,114,57,0,600,602,5,218,0,0,601,603,3,72,36,0,602,601,1,0,0, + 0,602,603,1,0,0,0,603,604,1,0,0,0,604,606,5,228,0,0,605,600,1,0, + 0,0,605,606,1,0,0,0,606,607,1,0,0,0,607,609,5,218,0,0,608,610,5, + 48,0,0,609,608,1,0,0,0,609,610,1,0,0,0,610,612,1,0,0,0,611,613,3, + 78,39,0,612,611,1,0,0,0,612,613,1,0,0,0,613,614,1,0,0,0,614,615, + 5,228,0,0,615,646,1,0,0,0,616,646,3,104,52,0,617,618,5,207,0,0,618, + 646,3,76,38,17,619,620,5,114,0,0,620,646,3,76,38,12,621,622,3,92, + 46,0,622,623,5,209,0,0,623,625,1,0,0,0,624,621,1,0,0,0,624,625,1, + 0,0,0,625,626,1,0,0,0,626,646,5,201,0,0,627,628,5,218,0,0,628,629, + 3,2,1,0,629,630,5,228,0,0,630,646,1,0,0,0,631,632,5,218,0,0,632, + 633,3,76,38,0,633,634,5,228,0,0,634,646,1,0,0,0,635,636,5,218,0, + 0,636,637,3,72,36,0,637,638,5,228,0,0,638,646,1,0,0,0,639,641,5, + 216,0,0,640,642,3,72,36,0,641,640,1,0,0,0,641,642,1,0,0,0,642,643, + 1,0,0,0,643,646,5,227,0,0,644,646,3,84,42,0,645,516,1,0,0,0,645, + 536,1,0,0,0,645,543,1,0,0,0,645,545,1,0,0,0,645,552,1,0,0,0,645, + 556,1,0,0,0,645,567,1,0,0,0,645,569,1,0,0,0,645,577,1,0,0,0,645, + 589,1,0,0,0,645,599,1,0,0,0,645,616,1,0,0,0,645,617,1,0,0,0,645, + 619,1,0,0,0,645,624,1,0,0,0,645,627,1,0,0,0,645,631,1,0,0,0,645, + 635,1,0,0,0,645,639,1,0,0,0,645,644,1,0,0,0,646,728,1,0,0,0,647, + 651,10,16,0,0,648,652,5,201,0,0,649,652,5,230,0,0,650,652,5,221, + 0,0,651,648,1,0,0,0,651,649,1,0,0,0,651,650,1,0,0,0,652,653,1,0, + 0,0,653,727,3,76,38,17,654,658,10,15,0,0,655,659,5,222,0,0,656,659, + 5,207,0,0,657,659,5,206,0,0,658,655,1,0,0,0,658,656,1,0,0,0,658, + 657,1,0,0,0,659,660,1,0,0,0,660,727,3,76,38,16,661,680,10,14,0,0, + 662,681,5,210,0,0,663,681,5,211,0,0,664,681,5,220,0,0,665,681,5, + 217,0,0,666,681,5,212,0,0,667,681,5,219,0,0,668,681,5,213,0,0,669, + 671,5,70,0,0,670,669,1,0,0,0,670,671,1,0,0,0,671,673,1,0,0,0,672, + 674,5,114,0,0,673,672,1,0,0,0,673,674,1,0,0,0,674,675,1,0,0,0,675, + 681,5,79,0,0,676,678,5,114,0,0,677,676,1,0,0,0,677,678,1,0,0,0,678, + 679,1,0,0,0,679,681,7,12,0,0,680,662,1,0,0,0,680,663,1,0,0,0,680, + 664,1,0,0,0,680,665,1,0,0,0,680,666,1,0,0,0,680,667,1,0,0,0,680, + 668,1,0,0,0,680,670,1,0,0,0,680,677,1,0,0,0,681,682,1,0,0,0,682, + 727,3,76,38,15,683,684,10,11,0,0,684,685,5,6,0,0,685,727,3,76,38, + 12,686,687,10,10,0,0,687,688,5,120,0,0,688,727,3,76,38,11,689,691, + 10,9,0,0,690,692,5,114,0,0,691,690,1,0,0,0,691,692,1,0,0,0,692,693, + 1,0,0,0,693,694,5,16,0,0,694,695,3,76,38,0,695,696,5,6,0,0,696,697, + 3,76,38,10,697,727,1,0,0,0,698,699,10,8,0,0,699,700,5,223,0,0,700, + 701,3,76,38,0,701,702,5,204,0,0,702,703,3,76,38,8,703,727,1,0,0, + 0,704,705,10,19,0,0,705,706,5,216,0,0,706,707,3,76,38,0,707,708, + 5,227,0,0,708,727,1,0,0,0,709,710,10,18,0,0,710,711,5,209,0,0,711, + 727,5,196,0,0,712,713,10,13,0,0,713,715,5,87,0,0,714,716,5,114,0, + 0,715,714,1,0,0,0,715,716,1,0,0,0,716,717,1,0,0,0,717,727,5,115, + 0,0,718,724,10,7,0,0,719,725,3,112,56,0,720,721,5,10,0,0,721,725, + 3,114,57,0,722,723,5,10,0,0,723,725,5,198,0,0,724,719,1,0,0,0,724, + 720,1,0,0,0,724,722,1,0,0,0,725,727,1,0,0,0,726,647,1,0,0,0,726, + 654,1,0,0,0,726,661,1,0,0,0,726,683,1,0,0,0,726,686,1,0,0,0,726, + 689,1,0,0,0,726,698,1,0,0,0,726,704,1,0,0,0,726,709,1,0,0,0,726, + 712,1,0,0,0,726,718,1,0,0,0,727,730,1,0,0,0,728,726,1,0,0,0,728, + 729,1,0,0,0,729,77,1,0,0,0,730,728,1,0,0,0,731,736,3,80,40,0,732, + 733,5,205,0,0,733,735,3,80,40,0,734,732,1,0,0,0,735,738,1,0,0,0, + 736,734,1,0,0,0,736,737,1,0,0,0,737,79,1,0,0,0,738,736,1,0,0,0,739, + 742,3,82,41,0,740,742,3,76,38,0,741,739,1,0,0,0,741,740,1,0,0,0, + 742,81,1,0,0,0,743,744,5,218,0,0,744,749,3,114,57,0,745,746,5,205, + 0,0,746,748,3,114,57,0,747,745,1,0,0,0,748,751,1,0,0,0,749,747,1, + 0,0,0,749,750,1,0,0,0,750,752,1,0,0,0,751,749,1,0,0,0,752,753,5, + 228,0,0,753,763,1,0,0,0,754,759,3,114,57,0,755,756,5,205,0,0,756, + 758,3,114,57,0,757,755,1,0,0,0,758,761,1,0,0,0,759,757,1,0,0,0,759, + 760,1,0,0,0,760,763,1,0,0,0,761,759,1,0,0,0,762,743,1,0,0,0,762, + 754,1,0,0,0,763,764,1,0,0,0,764,765,5,200,0,0,765,766,3,76,38,0, + 766,83,1,0,0,0,767,775,5,199,0,0,768,769,3,92,46,0,769,770,5,209, + 0,0,770,772,1,0,0,0,771,768,1,0,0,0,771,772,1,0,0,0,772,773,1,0, + 0,0,773,775,3,86,43,0,774,767,1,0,0,0,774,771,1,0,0,0,775,85,1,0, + 0,0,776,781,3,114,57,0,777,778,5,209,0,0,778,780,3,114,57,0,779, + 777,1,0,0,0,780,783,1,0,0,0,781,779,1,0,0,0,781,782,1,0,0,0,782, + 87,1,0,0,0,783,781,1,0,0,0,784,785,6,44,-1,0,785,792,3,92,46,0,786, + 792,3,90,45,0,787,788,5,218,0,0,788,789,3,2,1,0,789,790,5,228,0, + 0,790,792,1,0,0,0,791,784,1,0,0,0,791,786,1,0,0,0,791,787,1,0,0, + 0,792,801,1,0,0,0,793,797,10,1,0,0,794,798,3,112,56,0,795,796,5, + 10,0,0,796,798,3,114,57,0,797,794,1,0,0,0,797,795,1,0,0,0,798,800, + 1,0,0,0,799,793,1,0,0,0,800,803,1,0,0,0,801,799,1,0,0,0,801,802, + 1,0,0,0,802,89,1,0,0,0,803,801,1,0,0,0,804,805,3,114,57,0,805,807, + 5,218,0,0,806,808,3,94,47,0,807,806,1,0,0,0,807,808,1,0,0,0,808, + 809,1,0,0,0,809,810,5,228,0,0,810,91,1,0,0,0,811,812,3,98,49,0,812, + 813,5,209,0,0,813,815,1,0,0,0,814,811,1,0,0,0,814,815,1,0,0,0,815, + 816,1,0,0,0,816,817,3,114,57,0,817,93,1,0,0,0,818,823,3,96,48,0, + 819,820,5,205,0,0,820,822,3,96,48,0,821,819,1,0,0,0,822,825,1,0, + 0,0,823,821,1,0,0,0,823,824,1,0,0,0,824,95,1,0,0,0,825,823,1,0,0, + 0,826,830,3,86,43,0,827,830,3,90,45,0,828,830,3,104,52,0,829,826, + 1,0,0,0,829,827,1,0,0,0,829,828,1,0,0,0,830,97,1,0,0,0,831,832,3, + 114,57,0,832,99,1,0,0,0,833,842,5,194,0,0,834,835,5,209,0,0,835, + 842,7,13,0,0,836,837,5,196,0,0,837,839,5,209,0,0,838,840,7,13,0, + 0,839,838,1,0,0,0,839,840,1,0,0,0,840,842,1,0,0,0,841,833,1,0,0, + 0,841,834,1,0,0,0,841,836,1,0,0,0,842,101,1,0,0,0,843,845,7,14,0, + 0,844,843,1,0,0,0,844,845,1,0,0,0,845,852,1,0,0,0,846,853,3,100, + 50,0,847,853,5,195,0,0,848,853,5,196,0,0,849,853,5,197,0,0,850,853, + 5,81,0,0,851,853,5,112,0,0,852,846,1,0,0,0,852,847,1,0,0,0,852,848, + 1,0,0,0,852,849,1,0,0,0,852,850,1,0,0,0,852,851,1,0,0,0,853,103, + 1,0,0,0,854,858,3,102,51,0,855,858,5,198,0,0,856,858,5,115,0,0,857, + 854,1,0,0,0,857,855,1,0,0,0,857,856,1,0,0,0,858,105,1,0,0,0,859, + 860,7,15,0,0,860,107,1,0,0,0,861,862,7,16,0,0,862,109,1,0,0,0,863, + 864,7,17,0,0,864,111,1,0,0,0,865,868,5,193,0,0,866,868,3,110,55, + 0,867,865,1,0,0,0,867,866,1,0,0,0,868,113,1,0,0,0,869,873,5,193, + 0,0,870,873,3,106,53,0,871,873,3,108,54,0,872,869,1,0,0,0,872,870, + 1,0,0,0,872,871,1,0,0,0,873,115,1,0,0,0,874,877,3,114,57,0,875,877, + 5,115,0,0,876,874,1,0,0,0,876,875,1,0,0,0,877,117,1,0,0,0,878,879, + 5,198,0,0,879,880,5,211,0,0,880,881,3,102,51,0,881,119,1,0,0,0,114, + 122,132,140,143,147,150,154,157,160,163,166,169,173,177,180,183, + 186,189,198,204,231,250,258,261,267,275,278,284,286,290,295,298, + 301,305,309,312,314,317,321,325,328,330,332,335,340,351,357,362, + 369,374,378,382,387,394,402,405,408,427,441,457,469,481,489,493, + 500,506,514,519,528,532,563,580,592,602,605,609,612,624,641,645, + 651,658,670,673,677,680,691,715,724,726,728,736,741,749,759,762, + 771,774,781,791,797,801,807,814,823,829,839,841,844,852,857,867, + 872,876 ] class HogQLParser ( Parser ): @@ -488,68 +485,66 @@ class HogQLParser ( Parser ): RULE_havingClause = 12 RULE_orderByClause = 13 RULE_projectionOrderByClause = 14 - RULE_limitByClause = 15 - RULE_limitClause = 16 - RULE_settingsClause = 17 - RULE_joinExpr = 18 - RULE_joinOp = 19 - RULE_joinOpCross = 20 - RULE_joinConstraintClause = 21 - RULE_sampleClause = 22 - RULE_limitExpr = 23 - RULE_orderExprList = 24 - RULE_orderExpr = 25 - RULE_ratioExpr = 26 - RULE_settingExprList = 27 - RULE_settingExpr = 28 - RULE_windowExpr = 29 - RULE_winPartitionByClause = 30 - RULE_winOrderByClause = 31 - RULE_winFrameClause = 32 - RULE_winFrameExtend = 33 - RULE_winFrameBound = 34 - RULE_expr = 35 - RULE_columnTypeExpr = 36 - RULE_columnExprList = 37 - RULE_columnsExpr = 38 - RULE_columnExpr = 39 - RULE_columnArgList = 40 - RULE_columnArgExpr = 41 - RULE_columnLambdaExpr = 42 - RULE_columnIdentifier = 43 - RULE_nestedIdentifier = 44 - RULE_tableExpr = 45 - RULE_tableFunctionExpr = 46 - RULE_tableIdentifier = 47 - RULE_tableArgList = 48 - RULE_tableArgExpr = 49 - RULE_databaseIdentifier = 50 - RULE_floatingLiteral = 51 - RULE_numberLiteral = 52 - RULE_literal = 53 - RULE_interval = 54 - RULE_keyword = 55 - RULE_keywordForAlias = 56 - RULE_alias = 57 - RULE_identifier = 58 - RULE_identifierOrNull = 59 - RULE_enumValue = 60 + RULE_limitClause = 15 + RULE_settingsClause = 16 + RULE_joinExpr = 17 + RULE_joinOp = 18 + RULE_joinOpCross = 19 + RULE_joinConstraintClause = 20 + RULE_sampleClause = 21 + RULE_limitExpr = 22 + RULE_orderExprList = 23 + RULE_orderExpr = 24 + RULE_ratioExpr = 25 + RULE_settingExprList = 26 + RULE_settingExpr = 27 + RULE_windowExpr = 28 + RULE_winPartitionByClause = 29 + RULE_winOrderByClause = 30 + RULE_winFrameClause = 31 + RULE_winFrameExtend = 32 + RULE_winFrameBound = 33 + RULE_expr = 34 + RULE_columnTypeExpr = 35 + RULE_columnExprList = 36 + RULE_columnsExpr = 37 + RULE_columnExpr = 38 + RULE_columnArgList = 39 + RULE_columnArgExpr = 40 + RULE_columnLambdaExpr = 41 + RULE_columnIdentifier = 42 + RULE_nestedIdentifier = 43 + RULE_tableExpr = 44 + RULE_tableFunctionExpr = 45 + RULE_tableIdentifier = 46 + RULE_tableArgList = 47 + RULE_tableArgExpr = 48 + RULE_databaseIdentifier = 49 + RULE_floatingLiteral = 50 + RULE_numberLiteral = 51 + RULE_literal = 52 + RULE_interval = 53 + RULE_keyword = 54 + RULE_keywordForAlias = 55 + RULE_alias = 56 + RULE_identifier = 57 + RULE_identifierOrNull = 58 + RULE_enumValue = 59 ruleNames = [ "select", "selectUnionStmt", "selectStmtWithParens", "selectStmt", "withClause", "topClause", "fromClause", "arrayJoinClause", "windowClause", "prewhereClause", "whereClause", "groupByClause", "havingClause", "orderByClause", - "projectionOrderByClause", "limitByClause", "limitClause", - "settingsClause", "joinExpr", "joinOp", "joinOpCross", - "joinConstraintClause", "sampleClause", "limitExpr", - "orderExprList", "orderExpr", "ratioExpr", "settingExprList", - "settingExpr", "windowExpr", "winPartitionByClause", - "winOrderByClause", "winFrameClause", "winFrameExtend", - "winFrameBound", "expr", "columnTypeExpr", "columnExprList", - "columnsExpr", "columnExpr", "columnArgList", "columnArgExpr", - "columnLambdaExpr", "columnIdentifier", "nestedIdentifier", - "tableExpr", "tableFunctionExpr", "tableIdentifier", - "tableArgList", "tableArgExpr", "databaseIdentifier", + "projectionOrderByClause", "limitClause", "settingsClause", + "joinExpr", "joinOp", "joinOpCross", "joinConstraintClause", + "sampleClause", "limitExpr", "orderExprList", "orderExpr", + "ratioExpr", "settingExprList", "settingExpr", "windowExpr", + "winPartitionByClause", "winOrderByClause", "winFrameClause", + "winFrameExtend", "winFrameBound", "expr", "columnTypeExpr", + "columnExprList", "columnsExpr", "columnExpr", "columnArgList", + "columnArgExpr", "columnLambdaExpr", "columnIdentifier", + "nestedIdentifier", "tableExpr", "tableFunctionExpr", + "tableIdentifier", "tableArgList", "tableArgExpr", "databaseIdentifier", "floatingLiteral", "numberLiteral", "literal", "interval", "keyword", "keywordForAlias", "alias", "identifier", "identifierOrNull", "enumValue" ] @@ -835,21 +830,21 @@ def select(self): self.enterRule(localctx, 0, self.RULE_select) try: self.enterOuterAlt(localctx, 1) - self.state = 124 + self.state = 122 self._errHandler.sync(self) la_ = self._interp.adaptivePredict(self._input,0,self._ctx) if la_ == 1: - self.state = 122 + self.state = 120 self.selectUnionStmt() pass elif la_ == 2: - self.state = 123 + self.state = 121 self.selectStmt() pass - self.state = 126 + self.state = 124 self.match(HogQLParser.EOF) except RecognitionException as re: localctx.exception = re @@ -905,19 +900,19 @@ def selectUnionStmt(self): self._la = 0 # Token type try: self.enterOuterAlt(localctx, 1) - self.state = 128 + self.state = 126 self.selectStmtWithParens() - self.state = 134 + self.state = 132 self._errHandler.sync(self) _la = self._input.LA(1) while _la==175: - self.state = 129 + self.state = 127 self.match(HogQLParser.UNION) - self.state = 130 + self.state = 128 self.match(HogQLParser.ALL) - self.state = 131 + self.state = 129 self.selectStmtWithParens() - self.state = 136 + self.state = 134 self._errHandler.sync(self) _la = self._input.LA(1) @@ -968,21 +963,21 @@ def selectStmtWithParens(self): localctx = HogQLParser.SelectStmtWithParensContext(self, self._ctx, self.state) self.enterRule(localctx, 4, self.RULE_selectStmtWithParens) try: - self.state = 142 + self.state = 140 self._errHandler.sync(self) token = self._input.LA(1) if token in [145, 188]: self.enterOuterAlt(localctx, 1) - self.state = 137 + self.state = 135 self.selectStmt() pass elif token in [218]: self.enterOuterAlt(localctx, 2) - self.state = 138 + self.state = 136 self.match(HogQLParser.LPAREN) - self.state = 139 + self.state = 137 self.selectUnionStmt() - self.state = 140 + self.state = 138 self.match(HogQLParser.RPAREN) pass else: @@ -1055,10 +1050,6 @@ def orderByClause(self): return self.getTypedRuleContext(HogQLParser.OrderByClauseContext,0) - def limitByClause(self): - return self.getTypedRuleContext(HogQLParser.LimitByClauseContext,0) - - def limitClause(self): return self.getTypedRuleContext(HogQLParser.LimitClauseContext,0) @@ -1104,89 +1095,89 @@ def selectStmt(self): self._la = 0 # Token type try: self.enterOuterAlt(localctx, 1) - self.state = 145 + self.state = 143 self._errHandler.sync(self) _la = self._input.LA(1) if _la==188: - self.state = 144 + self.state = 142 localctx.with_ = self.withClause() - self.state = 147 + self.state = 145 self.match(HogQLParser.SELECT) - self.state = 149 + self.state = 147 self._errHandler.sync(self) la_ = self._interp.adaptivePredict(self._input,4,self._ctx) if la_ == 1: - self.state = 148 + self.state = 146 self.match(HogQLParser.DISTINCT) - self.state = 152 + self.state = 150 self._errHandler.sync(self) la_ = self._interp.adaptivePredict(self._input,5,self._ctx) if la_ == 1: - self.state = 151 + self.state = 149 self.topClause() - self.state = 154 + self.state = 152 localctx.columns = self.columnExprList() - self.state = 156 + self.state = 154 self._errHandler.sync(self) _la = self._input.LA(1) if _la==67: - self.state = 155 + self.state = 153 localctx.from_ = self.fromClause() - self.state = 159 + self.state = 157 self._errHandler.sync(self) _la = self._input.LA(1) if _la==9 or _la==83 or _la==95: - self.state = 158 + self.state = 156 self.arrayJoinClause() - self.state = 162 + self.state = 160 self._errHandler.sync(self) _la = self._input.LA(1) if _la==187: - self.state = 161 + self.state = 159 self.windowClause() - self.state = 165 + self.state = 163 self._errHandler.sync(self) _la = self._input.LA(1) if _la==128: - self.state = 164 + self.state = 162 self.prewhereClause() - self.state = 168 + self.state = 166 self._errHandler.sync(self) _la = self._input.LA(1) if _la==186: - self.state = 167 + self.state = 165 localctx.where = self.whereClause() - self.state = 171 + self.state = 169 self._errHandler.sync(self) _la = self._input.LA(1) if _la==72: - self.state = 170 + self.state = 168 self.groupByClause() - self.state = 175 + self.state = 173 self._errHandler.sync(self) la_ = self._interp.adaptivePredict(self._input,12,self._ctx) if la_ == 1: - self.state = 173 + self.state = 171 self.match(HogQLParser.WITH) - self.state = 174 + self.state = 172 _la = self._input.LA(1) if not(_la==31 or _la==140): self._errHandler.recoverInline(self) @@ -1195,53 +1186,45 @@ def selectStmt(self): self.consume() - self.state = 179 + self.state = 177 self._errHandler.sync(self) _la = self._input.LA(1) if _la==188: - self.state = 177 + self.state = 175 self.match(HogQLParser.WITH) - self.state = 178 + self.state = 176 self.match(HogQLParser.TOTALS) - self.state = 182 + self.state = 180 self._errHandler.sync(self) _la = self._input.LA(1) if _la==73: - self.state = 181 + self.state = 179 self.havingClause() - self.state = 185 + self.state = 183 self._errHandler.sync(self) _la = self._input.LA(1) if _la==121: - self.state = 184 + self.state = 182 self.orderByClause() - self.state = 188 - self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,16,self._ctx) - if la_ == 1: - self.state = 187 - self.limitByClause() - - - self.state = 191 + self.state = 186 self._errHandler.sync(self) _la = self._input.LA(1) if _la==98: - self.state = 190 + self.state = 185 self.limitClause() - self.state = 194 + self.state = 189 self._errHandler.sync(self) _la = self._input.LA(1) if _la==149: - self.state = 193 + self.state = 188 self.settingsClause() @@ -1286,9 +1269,9 @@ def withClause(self): self.enterRule(localctx, 8, self.RULE_withClause) try: self.enterOuterAlt(localctx, 1) - self.state = 196 + self.state = 191 self.match(HogQLParser.WITH) - self.state = 197 + self.state = 192 self.columnExprList() except RecognitionException as re: localctx.exception = re @@ -1336,17 +1319,17 @@ def topClause(self): self.enterRule(localctx, 10, self.RULE_topClause) try: self.enterOuterAlt(localctx, 1) - self.state = 199 + self.state = 194 self.match(HogQLParser.TOP) - self.state = 200 + self.state = 195 self.match(HogQLParser.DECIMAL_LITERAL) - self.state = 203 + self.state = 198 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,19,self._ctx) + la_ = self._interp.adaptivePredict(self._input,18,self._ctx) if la_ == 1: - self.state = 201 + self.state = 196 self.match(HogQLParser.WITH) - self.state = 202 + self.state = 197 self.match(HogQLParser.TIES) @@ -1391,9 +1374,9 @@ def fromClause(self): self.enterRule(localctx, 12, self.RULE_fromClause) try: self.enterOuterAlt(localctx, 1) - self.state = 205 + self.state = 200 self.match(HogQLParser.FROM) - self.state = 206 + self.state = 201 self.joinExpr(0) except RecognitionException as re: localctx.exception = re @@ -1446,11 +1429,11 @@ def arrayJoinClause(self): self._la = 0 # Token type try: self.enterOuterAlt(localctx, 1) - self.state = 209 + self.state = 204 self._errHandler.sync(self) _la = self._input.LA(1) if _la==83 or _la==95: - self.state = 208 + self.state = 203 _la = self._input.LA(1) if not(_la==83 or _la==95): self._errHandler.recoverInline(self) @@ -1459,11 +1442,11 @@ def arrayJoinClause(self): self.consume() - self.state = 211 + self.state = 206 self.match(HogQLParser.ARRAY) - self.state = 212 + self.state = 207 self.match(HogQLParser.JOIN) - self.state = 213 + self.state = 208 self.columnExprList() except RecognitionException as re: localctx.exception = re @@ -1519,17 +1502,17 @@ def windowClause(self): self.enterRule(localctx, 16, self.RULE_windowClause) try: self.enterOuterAlt(localctx, 1) - self.state = 215 + self.state = 210 self.match(HogQLParser.WINDOW) - self.state = 216 + self.state = 211 self.identifier() - self.state = 217 + self.state = 212 self.match(HogQLParser.AS) - self.state = 218 + self.state = 213 self.match(HogQLParser.LPAREN) - self.state = 219 + self.state = 214 self.windowExpr() - self.state = 220 + self.state = 215 self.match(HogQLParser.RPAREN) except RecognitionException as re: localctx.exception = re @@ -1572,9 +1555,9 @@ def prewhereClause(self): self.enterRule(localctx, 18, self.RULE_prewhereClause) try: self.enterOuterAlt(localctx, 1) - self.state = 222 + self.state = 217 self.match(HogQLParser.PREWHERE) - self.state = 223 + self.state = 218 self.columnExpr(0) except RecognitionException as re: localctx.exception = re @@ -1617,9 +1600,9 @@ def whereClause(self): self.enterRule(localctx, 20, self.RULE_whereClause) try: self.enterOuterAlt(localctx, 1) - self.state = 225 + self.state = 220 self.match(HogQLParser.WHERE) - self.state = 226 + self.state = 221 self.columnExpr(0) except RecognitionException as re: localctx.exception = re @@ -1678,31 +1661,31 @@ def groupByClause(self): self._la = 0 # Token type try: self.enterOuterAlt(localctx, 1) - self.state = 228 + self.state = 223 self.match(HogQLParser.GROUP) - self.state = 229 + self.state = 224 self.match(HogQLParser.BY) - self.state = 236 + self.state = 231 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,21,self._ctx) + la_ = self._interp.adaptivePredict(self._input,20,self._ctx) if la_ == 1: - self.state = 230 + self.state = 225 _la = self._input.LA(1) if not(_la==31 or _la==140): self._errHandler.recoverInline(self) else: self._errHandler.reportMatch(self) self.consume() - self.state = 231 + self.state = 226 self.match(HogQLParser.LPAREN) - self.state = 232 + self.state = 227 self.columnExprList() - self.state = 233 + self.state = 228 self.match(HogQLParser.RPAREN) pass elif la_ == 2: - self.state = 235 + self.state = 230 self.columnExprList() pass @@ -1748,9 +1731,9 @@ def havingClause(self): self.enterRule(localctx, 24, self.RULE_havingClause) try: self.enterOuterAlt(localctx, 1) - self.state = 238 + self.state = 233 self.match(HogQLParser.HAVING) - self.state = 239 + self.state = 234 self.columnExpr(0) except RecognitionException as re: localctx.exception = re @@ -1796,11 +1779,11 @@ def orderByClause(self): self.enterRule(localctx, 26, self.RULE_orderByClause) try: self.enterOuterAlt(localctx, 1) - self.state = 241 + self.state = 236 self.match(HogQLParser.ORDER) - self.state = 242 + self.state = 237 self.match(HogQLParser.BY) - self.state = 243 + self.state = 238 self.orderExprList() except RecognitionException as re: localctx.exception = re @@ -1846,11 +1829,11 @@ def projectionOrderByClause(self): self.enterRule(localctx, 28, self.RULE_projectionOrderByClause) try: self.enterOuterAlt(localctx, 1) - self.state = 245 + self.state = 240 self.match(HogQLParser.ORDER) - self.state = 246 + self.state = 241 self.match(HogQLParser.BY) - self.state = 247 + self.state = 242 self.columnExprList() except RecognitionException as re: localctx.exception = re @@ -1861,7 +1844,7 @@ def projectionOrderByClause(self): return localctx - class LimitByClauseContext(ParserRuleContext): + class LimitClauseContext(ParserRuleContext): __slots__ = 'parser' def __init__(self, parser, parent:ParserRuleContext=None, invokingState:int=-1): @@ -1882,55 +1865,6 @@ def columnExprList(self): return self.getTypedRuleContext(HogQLParser.ColumnExprListContext,0) - def getRuleIndex(self): - return HogQLParser.RULE_limitByClause - - def accept(self, visitor:ParseTreeVisitor): - if hasattr( visitor, "visitLimitByClause" ): - return visitor.visitLimitByClause(self) - else: - return visitor.visitChildren(self) - - - - - def limitByClause(self): - - localctx = HogQLParser.LimitByClauseContext(self, self._ctx, self.state) - self.enterRule(localctx, 30, self.RULE_limitByClause) - try: - self.enterOuterAlt(localctx, 1) - self.state = 249 - self.match(HogQLParser.LIMIT) - self.state = 250 - self.limitExpr() - self.state = 251 - self.match(HogQLParser.BY) - self.state = 252 - self.columnExprList() - except RecognitionException as re: - localctx.exception = re - self._errHandler.reportError(self, re) - self._errHandler.recover(self, re) - finally: - self.exitRule() - return localctx - - - class LimitClauseContext(ParserRuleContext): - __slots__ = 'parser' - - def __init__(self, parser, parent:ParserRuleContext=None, invokingState:int=-1): - super().__init__(parent, invokingState) - self.parser = parser - - def LIMIT(self): - return self.getToken(HogQLParser.LIMIT, 0) - - def limitExpr(self): - return self.getTypedRuleContext(HogQLParser.LimitExprContext,0) - - def WITH(self): return self.getToken(HogQLParser.WITH, 0) @@ -1952,24 +1886,32 @@ def accept(self, visitor:ParseTreeVisitor): def limitClause(self): localctx = HogQLParser.LimitClauseContext(self, self._ctx, self.state) - self.enterRule(localctx, 32, self.RULE_limitClause) - self._la = 0 # Token type + self.enterRule(localctx, 30, self.RULE_limitClause) try: self.enterOuterAlt(localctx, 1) - self.state = 254 + self.state = 244 self.match(HogQLParser.LIMIT) - self.state = 255 + self.state = 245 self.limitExpr() - self.state = 258 + self.state = 250 self._errHandler.sync(self) - _la = self._input.LA(1) - if _la==188: - self.state = 256 + token = self._input.LA(1) + if token in [188]: + self.state = 246 self.match(HogQLParser.WITH) - self.state = 257 + self.state = 247 self.match(HogQLParser.TIES) - - + pass + elif token in [18]: + self.state = 248 + self.match(HogQLParser.BY) + self.state = 249 + self.columnExprList() + pass + elif token in [-1, 149, 175, 228]: + pass + else: + pass except RecognitionException as re: localctx.exception = re self._errHandler.reportError(self, re) @@ -2008,12 +1950,12 @@ def accept(self, visitor:ParseTreeVisitor): def settingsClause(self): localctx = HogQLParser.SettingsClauseContext(self, self._ctx, self.state) - self.enterRule(localctx, 34, self.RULE_settingsClause) + self.enterRule(localctx, 32, self.RULE_settingsClause) try: self.enterOuterAlt(localctx, 1) - self.state = 260 + self.state = 252 self.match(HogQLParser.SETTINGS) - self.state = 261 + self.state = 253 self.settingExprList() except RecognitionException as re: localctx.exception = re @@ -2144,34 +2086,34 @@ def joinExpr(self, _p:int=0): _parentState = self.state localctx = HogQLParser.JoinExprContext(self, self._ctx, _parentState) _prevctx = localctx - _startState = 36 - self.enterRecursionRule(localctx, 36, self.RULE_joinExpr, _p) + _startState = 34 + self.enterRecursionRule(localctx, 34, self.RULE_joinExpr, _p) self._la = 0 # Token type try: self.enterOuterAlt(localctx, 1) - self.state = 275 + self.state = 267 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,25,self._ctx) + la_ = self._interp.adaptivePredict(self._input,24,self._ctx) if la_ == 1: localctx = HogQLParser.JoinExprTableContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 264 + self.state = 256 self.tableExpr(0) - self.state = 266 + self.state = 258 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,23,self._ctx) + la_ = self._interp.adaptivePredict(self._input,22,self._ctx) if la_ == 1: - self.state = 265 + self.state = 257 self.match(HogQLParser.FINAL) - self.state = 269 + self.state = 261 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,24,self._ctx) + la_ = self._interp.adaptivePredict(self._input,23,self._ctx) if la_ == 1: - self.state = 268 + self.state = 260 self.sampleClause() @@ -2181,52 +2123,52 @@ def joinExpr(self, _p:int=0): localctx = HogQLParser.JoinExprParensContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 271 + self.state = 263 self.match(HogQLParser.LPAREN) - self.state = 272 + self.state = 264 self.joinExpr(0) - self.state = 273 + self.state = 265 self.match(HogQLParser.RPAREN) pass self._ctx.stop = self._input.LT(-1) - self.state = 294 + self.state = 286 self._errHandler.sync(self) - _alt = self._interp.adaptivePredict(self._input,29,self._ctx) + _alt = self._interp.adaptivePredict(self._input,28,self._ctx) while _alt!=2 and _alt!=ATN.INVALID_ALT_NUMBER: if _alt==1: if self._parseListeners is not None: self.triggerExitRuleEvent() _prevctx = localctx - self.state = 292 + self.state = 284 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,28,self._ctx) + la_ = self._interp.adaptivePredict(self._input,27,self._ctx) if la_ == 1: localctx = HogQLParser.JoinExprCrossOpContext(self, HogQLParser.JoinExprContext(self, _parentctx, _parentState)) self.pushNewRecursionContext(localctx, _startState, self.RULE_joinExpr) - self.state = 277 + self.state = 269 if not self.precpred(self._ctx, 3): from antlr4.error.Errors import FailedPredicateException raise FailedPredicateException(self, "self.precpred(self._ctx, 3)") - self.state = 278 + self.state = 270 self.joinOpCross() - self.state = 279 + self.state = 271 self.joinExpr(4) pass elif la_ == 2: localctx = HogQLParser.JoinExprOpContext(self, HogQLParser.JoinExprContext(self, _parentctx, _parentState)) self.pushNewRecursionContext(localctx, _startState, self.RULE_joinExpr) - self.state = 281 + self.state = 273 if not self.precpred(self._ctx, 4): from antlr4.error.Errors import FailedPredicateException raise FailedPredicateException(self, "self.precpred(self._ctx, 4)") - self.state = 283 + self.state = 275 self._errHandler.sync(self) _la = self._input.LA(1) if _la==70 or _la==100: - self.state = 282 + self.state = 274 _la = self._input.LA(1) if not(_la==70 or _la==100): self._errHandler.recoverInline(self) @@ -2235,26 +2177,26 @@ def joinExpr(self, _p:int=0): self.consume() - self.state = 286 + self.state = 278 self._errHandler.sync(self) _la = self._input.LA(1) if ((_la) & ~0x3f) == 0 and ((1 << _la) & 4496) != 0 or (((_la - 68)) & ~0x3f) == 0 and ((1 << (_la - 68)) & 134250497) != 0 or _la==139 or _la==146: - self.state = 285 + self.state = 277 self.joinOp() - self.state = 288 + self.state = 280 self.match(HogQLParser.JOIN) - self.state = 289 + self.state = 281 self.joinExpr(0) - self.state = 290 + self.state = 282 self.joinConstraintClause() pass - self.state = 296 + self.state = 288 self._errHandler.sync(self) - _alt = self._interp.adaptivePredict(self._input,29,self._ctx) + _alt = self._interp.adaptivePredict(self._input,28,self._ctx) except RecognitionException as re: localctx.exception = re @@ -2360,24 +2302,24 @@ def accept(self, visitor:ParseTreeVisitor): def joinOp(self): localctx = HogQLParser.JoinOpContext(self, self._ctx, self.state) - self.enterRule(localctx, 38, self.RULE_joinOp) + self.enterRule(localctx, 36, self.RULE_joinOp) self._la = 0 # Token type try: - self.state = 340 + self.state = 332 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,43,self._ctx) + la_ = self._interp.adaptivePredict(self._input,42,self._ctx) if la_ == 1: localctx = HogQLParser.JoinOpInnerContext(self, localctx) self.enterOuterAlt(localctx, 1) - self.state = 306 + self.state = 298 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,32,self._ctx) + la_ = self._interp.adaptivePredict(self._input,31,self._ctx) if la_ == 1: - self.state = 298 + self.state = 290 self._errHandler.sync(self) _la = self._input.LA(1) if ((_la) & ~0x3f) == 0 and ((1 << _la) & 4368) != 0: - self.state = 297 + self.state = 289 _la = self._input.LA(1) if not(((_la) & ~0x3f) == 0 and ((1 << _la) & 4368) != 0): self._errHandler.recoverInline(self) @@ -2386,18 +2328,18 @@ def joinOp(self): self.consume() - self.state = 300 + self.state = 292 self.match(HogQLParser.INNER) pass elif la_ == 2: - self.state = 301 + self.state = 293 self.match(HogQLParser.INNER) - self.state = 303 + self.state = 295 self._errHandler.sync(self) _la = self._input.LA(1) if ((_la) & ~0x3f) == 0 and ((1 << _la) & 4368) != 0: - self.state = 302 + self.state = 294 _la = self._input.LA(1) if not(((_la) & ~0x3f) == 0 and ((1 << _la) & 4368) != 0): self._errHandler.recoverInline(self) @@ -2409,7 +2351,7 @@ def joinOp(self): pass elif la_ == 3: - self.state = 305 + self.state = 297 _la = self._input.LA(1) if not(((_la) & ~0x3f) == 0 and ((1 << _la) & 4368) != 0): self._errHandler.recoverInline(self) @@ -2424,15 +2366,15 @@ def joinOp(self): elif la_ == 2: localctx = HogQLParser.JoinOpLeftRightContext(self, localctx) self.enterOuterAlt(localctx, 2) - self.state = 322 + self.state = 314 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,37,self._ctx) + la_ = self._interp.adaptivePredict(self._input,36,self._ctx) if la_ == 1: - self.state = 309 + self.state = 301 self._errHandler.sync(self) _la = self._input.LA(1) if ((_la) & ~0x3f) == 0 and ((1 << _la) & 4496) != 0 or _la==146: - self.state = 308 + self.state = 300 _la = self._input.LA(1) if not(((_la) & ~0x3f) == 0 and ((1 << _la) & 4496) != 0 or _la==146): self._errHandler.recoverInline(self) @@ -2441,44 +2383,44 @@ def joinOp(self): self.consume() - self.state = 311 + self.state = 303 _la = self._input.LA(1) if not(_la==95 or _la==139): self._errHandler.recoverInline(self) else: self._errHandler.reportMatch(self) self.consume() - self.state = 313 + self.state = 305 self._errHandler.sync(self) _la = self._input.LA(1) if _la==122: - self.state = 312 + self.state = 304 self.match(HogQLParser.OUTER) pass elif la_ == 2: - self.state = 315 + self.state = 307 _la = self._input.LA(1) if not(_la==95 or _la==139): self._errHandler.recoverInline(self) else: self._errHandler.reportMatch(self) self.consume() - self.state = 317 + self.state = 309 self._errHandler.sync(self) _la = self._input.LA(1) if _la==122: - self.state = 316 + self.state = 308 self.match(HogQLParser.OUTER) - self.state = 320 + self.state = 312 self._errHandler.sync(self) _la = self._input.LA(1) if ((_la) & ~0x3f) == 0 and ((1 << _la) & 4496) != 0 or _la==146: - self.state = 319 + self.state = 311 _la = self._input.LA(1) if not(((_la) & ~0x3f) == 0 and ((1 << _la) & 4496) != 0 or _la==146): self._errHandler.recoverInline(self) @@ -2495,15 +2437,15 @@ def joinOp(self): elif la_ == 3: localctx = HogQLParser.JoinOpFullContext(self, localctx) self.enterOuterAlt(localctx, 3) - self.state = 338 + self.state = 330 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,42,self._ctx) + la_ = self._interp.adaptivePredict(self._input,41,self._ctx) if la_ == 1: - self.state = 325 + self.state = 317 self._errHandler.sync(self) _la = self._input.LA(1) if _la==4 or _la==8: - self.state = 324 + self.state = 316 _la = self._input.LA(1) if not(_la==4 or _la==8): self._errHandler.recoverInline(self) @@ -2512,34 +2454,34 @@ def joinOp(self): self.consume() - self.state = 327 + self.state = 319 self.match(HogQLParser.FULL) - self.state = 329 + self.state = 321 self._errHandler.sync(self) _la = self._input.LA(1) if _la==122: - self.state = 328 + self.state = 320 self.match(HogQLParser.OUTER) pass elif la_ == 2: - self.state = 331 + self.state = 323 self.match(HogQLParser.FULL) - self.state = 333 + self.state = 325 self._errHandler.sync(self) _la = self._input.LA(1) if _la==122: - self.state = 332 + self.state = 324 self.match(HogQLParser.OUTER) - self.state = 336 + self.state = 328 self._errHandler.sync(self) _la = self._input.LA(1) if _la==4 or _la==8: - self.state = 335 + self.state = 327 _la = self._input.LA(1) if not(_la==4 or _la==8): self._errHandler.recoverInline(self) @@ -2600,19 +2542,19 @@ def accept(self, visitor:ParseTreeVisitor): def joinOpCross(self): localctx = HogQLParser.JoinOpCrossContext(self, self._ctx, self.state) - self.enterRule(localctx, 40, self.RULE_joinOpCross) + self.enterRule(localctx, 38, self.RULE_joinOpCross) self._la = 0 # Token type try: - self.state = 348 + self.state = 340 self._errHandler.sync(self) token = self._input.LA(1) if token in [30, 70, 100]: self.enterOuterAlt(localctx, 1) - self.state = 343 + self.state = 335 self._errHandler.sync(self) _la = self._input.LA(1) if _la==70 or _la==100: - self.state = 342 + self.state = 334 _la = self._input.LA(1) if not(_la==70 or _la==100): self._errHandler.recoverInline(self) @@ -2621,14 +2563,14 @@ def joinOpCross(self): self.consume() - self.state = 345 + self.state = 337 self.match(HogQLParser.CROSS) - self.state = 346 + self.state = 338 self.match(HogQLParser.JOIN) pass elif token in [205]: self.enterOuterAlt(localctx, 2) - self.state = 347 + self.state = 339 self.match(HogQLParser.COMMA) pass else: @@ -2681,36 +2623,36 @@ def accept(self, visitor:ParseTreeVisitor): def joinConstraintClause(self): localctx = HogQLParser.JoinConstraintClauseContext(self, self._ctx, self.state) - self.enterRule(localctx, 42, self.RULE_joinConstraintClause) + self.enterRule(localctx, 40, self.RULE_joinConstraintClause) try: - self.state = 359 + self.state = 351 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,46,self._ctx) + la_ = self._interp.adaptivePredict(self._input,45,self._ctx) if la_ == 1: self.enterOuterAlt(localctx, 1) - self.state = 350 + self.state = 342 self.match(HogQLParser.ON) - self.state = 351 + self.state = 343 self.columnExprList() pass elif la_ == 2: self.enterOuterAlt(localctx, 2) - self.state = 352 + self.state = 344 self.match(HogQLParser.USING) - self.state = 353 + self.state = 345 self.match(HogQLParser.LPAREN) - self.state = 354 + self.state = 346 self.columnExprList() - self.state = 355 + self.state = 347 self.match(HogQLParser.RPAREN) pass elif la_ == 3: self.enterOuterAlt(localctx, 3) - self.state = 357 + self.state = 349 self.match(HogQLParser.USING) - self.state = 358 + self.state = 350 self.columnExprList() pass @@ -2759,20 +2701,20 @@ def accept(self, visitor:ParseTreeVisitor): def sampleClause(self): localctx = HogQLParser.SampleClauseContext(self, self._ctx, self.state) - self.enterRule(localctx, 44, self.RULE_sampleClause) + self.enterRule(localctx, 42, self.RULE_sampleClause) try: self.enterOuterAlt(localctx, 1) - self.state = 361 + self.state = 353 self.match(HogQLParser.SAMPLE) - self.state = 362 + self.state = 354 self.ratioExpr() - self.state = 365 + self.state = 357 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,47,self._ctx) + la_ = self._interp.adaptivePredict(self._input,46,self._ctx) if la_ == 1: - self.state = 363 + self.state = 355 self.match(HogQLParser.OFFSET) - self.state = 364 + self.state = 356 self.ratioExpr() @@ -2820,24 +2762,24 @@ def accept(self, visitor:ParseTreeVisitor): def limitExpr(self): localctx = HogQLParser.LimitExprContext(self, self._ctx, self.state) - self.enterRule(localctx, 46, self.RULE_limitExpr) + self.enterRule(localctx, 44, self.RULE_limitExpr) self._la = 0 # Token type try: self.enterOuterAlt(localctx, 1) - self.state = 367 + self.state = 359 self.columnExpr(0) - self.state = 370 + self.state = 362 self._errHandler.sync(self) _la = self._input.LA(1) if _la==117 or _la==205: - self.state = 368 + self.state = 360 _la = self._input.LA(1) if not(_la==117 or _la==205): self._errHandler.recoverInline(self) else: self._errHandler.reportMatch(self) self.consume() - self.state = 369 + self.state = 361 self.columnExpr(0) @@ -2885,21 +2827,21 @@ def accept(self, visitor:ParseTreeVisitor): def orderExprList(self): localctx = HogQLParser.OrderExprListContext(self, self._ctx, self.state) - self.enterRule(localctx, 48, self.RULE_orderExprList) + self.enterRule(localctx, 46, self.RULE_orderExprList) self._la = 0 # Token type try: self.enterOuterAlt(localctx, 1) - self.state = 372 + self.state = 364 self.orderExpr() - self.state = 377 + self.state = 369 self._errHandler.sync(self) _la = self._input.LA(1) while _la==205: - self.state = 373 + self.state = 365 self.match(HogQLParser.COMMA) - self.state = 374 + self.state = 366 self.orderExpr() - self.state = 379 + self.state = 371 self._errHandler.sync(self) _la = self._input.LA(1) @@ -2962,17 +2904,17 @@ def accept(self, visitor:ParseTreeVisitor): def orderExpr(self): localctx = HogQLParser.OrderExprContext(self, self._ctx, self.state) - self.enterRule(localctx, 50, self.RULE_orderExpr) + self.enterRule(localctx, 48, self.RULE_orderExpr) self._la = 0 # Token type try: self.enterOuterAlt(localctx, 1) - self.state = 380 + self.state = 372 self.columnExpr(0) - self.state = 382 + self.state = 374 self._errHandler.sync(self) _la = self._input.LA(1) if ((_la) & ~0x3f) == 0 and ((1 << _la) & 6597069768704) != 0: - self.state = 381 + self.state = 373 _la = self._input.LA(1) if not(((_la) & ~0x3f) == 0 and ((1 << _la) & 6597069768704) != 0): self._errHandler.recoverInline(self) @@ -2981,13 +2923,13 @@ def orderExpr(self): self.consume() - self.state = 386 + self.state = 378 self._errHandler.sync(self) _la = self._input.LA(1) if _la==116: - self.state = 384 + self.state = 376 self.match(HogQLParser.NULLS) - self.state = 385 + self.state = 377 _la = self._input.LA(1) if not(_la==61 or _la==92): self._errHandler.recoverInline(self) @@ -2996,13 +2938,13 @@ def orderExpr(self): self.consume() - self.state = 390 + self.state = 382 self._errHandler.sync(self) _la = self._input.LA(1) if _la==25: - self.state = 388 + self.state = 380 self.match(HogQLParser.COLLATE) - self.state = 389 + self.state = 381 self.match(HogQLParser.STRING_LITERAL) @@ -3047,18 +2989,18 @@ def accept(self, visitor:ParseTreeVisitor): def ratioExpr(self): localctx = HogQLParser.RatioExprContext(self, self._ctx, self.state) - self.enterRule(localctx, 52, self.RULE_ratioExpr) + self.enterRule(localctx, 50, self.RULE_ratioExpr) try: self.enterOuterAlt(localctx, 1) - self.state = 392 + self.state = 384 self.numberLiteral() - self.state = 395 + self.state = 387 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,53,self._ctx) + la_ = self._interp.adaptivePredict(self._input,52,self._ctx) if la_ == 1: - self.state = 393 + self.state = 385 self.match(HogQLParser.SLASH) - self.state = 394 + self.state = 386 self.numberLiteral() @@ -3106,21 +3048,21 @@ def accept(self, visitor:ParseTreeVisitor): def settingExprList(self): localctx = HogQLParser.SettingExprListContext(self, self._ctx, self.state) - self.enterRule(localctx, 54, self.RULE_settingExprList) + self.enterRule(localctx, 52, self.RULE_settingExprList) self._la = 0 # Token type try: self.enterOuterAlt(localctx, 1) - self.state = 397 + self.state = 389 self.settingExpr() - self.state = 402 + self.state = 394 self._errHandler.sync(self) _la = self._input.LA(1) while _la==205: - self.state = 398 + self.state = 390 self.match(HogQLParser.COMMA) - self.state = 399 + self.state = 391 self.settingExpr() - self.state = 404 + self.state = 396 self._errHandler.sync(self) _la = self._input.LA(1) @@ -3166,14 +3108,14 @@ def accept(self, visitor:ParseTreeVisitor): def settingExpr(self): localctx = HogQLParser.SettingExprContext(self, self._ctx, self.state) - self.enterRule(localctx, 56, self.RULE_settingExpr) + self.enterRule(localctx, 54, self.RULE_settingExpr) try: self.enterOuterAlt(localctx, 1) - self.state = 405 + self.state = 397 self.identifier() - self.state = 406 + self.state = 398 self.match(HogQLParser.EQ_SINGLE) - self.state = 407 + self.state = 399 self.literal() except RecognitionException as re: localctx.exception = re @@ -3218,31 +3160,31 @@ def accept(self, visitor:ParseTreeVisitor): def windowExpr(self): localctx = HogQLParser.WindowExprContext(self, self._ctx, self.state) - self.enterRule(localctx, 58, self.RULE_windowExpr) + self.enterRule(localctx, 56, self.RULE_windowExpr) self._la = 0 # Token type try: self.enterOuterAlt(localctx, 1) - self.state = 410 + self.state = 402 self._errHandler.sync(self) _la = self._input.LA(1) if _la==125: - self.state = 409 + self.state = 401 self.winPartitionByClause() - self.state = 413 + self.state = 405 self._errHandler.sync(self) _la = self._input.LA(1) if _la==121: - self.state = 412 + self.state = 404 self.winOrderByClause() - self.state = 416 + self.state = 408 self._errHandler.sync(self) _la = self._input.LA(1) if _la==132 or _la==142: - self.state = 415 + self.state = 407 self.winFrameClause() @@ -3287,14 +3229,14 @@ def accept(self, visitor:ParseTreeVisitor): def winPartitionByClause(self): localctx = HogQLParser.WinPartitionByClauseContext(self, self._ctx, self.state) - self.enterRule(localctx, 60, self.RULE_winPartitionByClause) + self.enterRule(localctx, 58, self.RULE_winPartitionByClause) try: self.enterOuterAlt(localctx, 1) - self.state = 418 + self.state = 410 self.match(HogQLParser.PARTITION) - self.state = 419 + self.state = 411 self.match(HogQLParser.BY) - self.state = 420 + self.state = 412 self.columnExprList() except RecognitionException as re: localctx.exception = re @@ -3337,14 +3279,14 @@ def accept(self, visitor:ParseTreeVisitor): def winOrderByClause(self): localctx = HogQLParser.WinOrderByClauseContext(self, self._ctx, self.state) - self.enterRule(localctx, 62, self.RULE_winOrderByClause) + self.enterRule(localctx, 60, self.RULE_winOrderByClause) try: self.enterOuterAlt(localctx, 1) - self.state = 422 + self.state = 414 self.match(HogQLParser.ORDER) - self.state = 423 + self.state = 415 self.match(HogQLParser.BY) - self.state = 424 + self.state = 416 self.orderExprList() except RecognitionException as re: localctx.exception = re @@ -3387,18 +3329,18 @@ def accept(self, visitor:ParseTreeVisitor): def winFrameClause(self): localctx = HogQLParser.WinFrameClauseContext(self, self._ctx, self.state) - self.enterRule(localctx, 64, self.RULE_winFrameClause) + self.enterRule(localctx, 62, self.RULE_winFrameClause) self._la = 0 # Token type try: self.enterOuterAlt(localctx, 1) - self.state = 426 + self.state = 418 _la = self._input.LA(1) if not(_la==132 or _la==142): self._errHandler.recoverInline(self) else: self._errHandler.reportMatch(self) self.consume() - self.state = 427 + self.state = 419 self.winFrameExtend() except RecognitionException as re: localctx.exception = re @@ -3471,27 +3413,27 @@ def accept(self, visitor:ParseTreeVisitor): def winFrameExtend(self): localctx = HogQLParser.WinFrameExtendContext(self, self._ctx, self.state) - self.enterRule(localctx, 66, self.RULE_winFrameExtend) + self.enterRule(localctx, 64, self.RULE_winFrameExtend) try: - self.state = 435 + self.state = 427 self._errHandler.sync(self) token = self._input.LA(1) if token in [32, 81, 112, 174, 194, 195, 196, 197, 207, 209, 222]: localctx = HogQLParser.FrameStartContext(self, localctx) self.enterOuterAlt(localctx, 1) - self.state = 429 + self.state = 421 self.winFrameBound() pass elif token in [16]: localctx = HogQLParser.FrameBetweenContext(self, localctx) self.enterOuterAlt(localctx, 2) - self.state = 430 + self.state = 422 self.match(HogQLParser.BETWEEN) - self.state = 431 + self.state = 423 self.winFrameBound() - self.state = 432 + self.state = 424 self.match(HogQLParser.AND) - self.state = 433 + self.state = 425 self.winFrameBound() pass else: @@ -3547,44 +3489,44 @@ def accept(self, visitor:ParseTreeVisitor): def winFrameBound(self): localctx = HogQLParser.WinFrameBoundContext(self, self._ctx, self.state) - self.enterRule(localctx, 68, self.RULE_winFrameBound) + self.enterRule(localctx, 66, self.RULE_winFrameBound) try: self.enterOuterAlt(localctx, 1) - self.state = 449 + self.state = 441 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,59,self._ctx) + la_ = self._interp.adaptivePredict(self._input,58,self._ctx) if la_ == 1: - self.state = 437 + self.state = 429 self.match(HogQLParser.CURRENT) - self.state = 438 + self.state = 430 self.match(HogQLParser.ROW) pass elif la_ == 2: - self.state = 439 + self.state = 431 self.match(HogQLParser.UNBOUNDED) - self.state = 440 + self.state = 432 self.match(HogQLParser.PRECEDING) pass elif la_ == 3: - self.state = 441 + self.state = 433 self.match(HogQLParser.UNBOUNDED) - self.state = 442 + self.state = 434 self.match(HogQLParser.FOLLOWING) pass elif la_ == 4: - self.state = 443 + self.state = 435 self.numberLiteral() - self.state = 444 + self.state = 436 self.match(HogQLParser.PRECEDING) pass elif la_ == 5: - self.state = 446 + self.state = 438 self.numberLiteral() - self.state = 447 + self.state = 439 self.match(HogQLParser.FOLLOWING) pass @@ -3627,12 +3569,12 @@ def accept(self, visitor:ParseTreeVisitor): def expr(self): localctx = HogQLParser.ExprContext(self, self._ctx, self.state) - self.enterRule(localctx, 70, self.RULE_expr) + self.enterRule(localctx, 68, self.RULE_expr) try: self.enterOuterAlt(localctx, 1) - self.state = 451 + self.state = 443 self.columnExpr(0) - self.state = 452 + self.state = 444 self.match(HogQLParser.EOF) except RecognitionException as re: localctx.exception = re @@ -3804,114 +3746,114 @@ def accept(self, visitor:ParseTreeVisitor): def columnTypeExpr(self): localctx = HogQLParser.ColumnTypeExprContext(self, self._ctx, self.state) - self.enterRule(localctx, 72, self.RULE_columnTypeExpr) + self.enterRule(localctx, 70, self.RULE_columnTypeExpr) self._la = 0 # Token type try: - self.state = 501 + self.state = 493 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,64,self._ctx) + la_ = self._interp.adaptivePredict(self._input,63,self._ctx) if la_ == 1: localctx = HogQLParser.ColumnTypeExprSimpleContext(self, localctx) self.enterOuterAlt(localctx, 1) - self.state = 454 + self.state = 446 self.identifier() pass elif la_ == 2: localctx = HogQLParser.ColumnTypeExprNestedContext(self, localctx) self.enterOuterAlt(localctx, 2) - self.state = 455 + self.state = 447 self.identifier() - self.state = 456 + self.state = 448 self.match(HogQLParser.LPAREN) - self.state = 457 + self.state = 449 self.identifier() - self.state = 458 + self.state = 450 self.columnTypeExpr() - self.state = 465 + self.state = 457 self._errHandler.sync(self) _la = self._input.LA(1) while _la==205: - self.state = 459 + self.state = 451 self.match(HogQLParser.COMMA) - self.state = 460 + self.state = 452 self.identifier() - self.state = 461 + self.state = 453 self.columnTypeExpr() - self.state = 467 + self.state = 459 self._errHandler.sync(self) _la = self._input.LA(1) - self.state = 468 + self.state = 460 self.match(HogQLParser.RPAREN) pass elif la_ == 3: localctx = HogQLParser.ColumnTypeExprEnumContext(self, localctx) self.enterOuterAlt(localctx, 3) - self.state = 470 + self.state = 462 self.identifier() - self.state = 471 + self.state = 463 self.match(HogQLParser.LPAREN) - self.state = 472 + self.state = 464 self.enumValue() - self.state = 477 + self.state = 469 self._errHandler.sync(self) _la = self._input.LA(1) while _la==205: - self.state = 473 + self.state = 465 self.match(HogQLParser.COMMA) - self.state = 474 + self.state = 466 self.enumValue() - self.state = 479 + self.state = 471 self._errHandler.sync(self) _la = self._input.LA(1) - self.state = 480 + self.state = 472 self.match(HogQLParser.RPAREN) pass elif la_ == 4: localctx = HogQLParser.ColumnTypeExprComplexContext(self, localctx) self.enterOuterAlt(localctx, 4) - self.state = 482 + self.state = 474 self.identifier() - self.state = 483 + self.state = 475 self.match(HogQLParser.LPAREN) - self.state = 484 + self.state = 476 self.columnTypeExpr() - self.state = 489 + self.state = 481 self._errHandler.sync(self) _la = self._input.LA(1) while _la==205: - self.state = 485 + self.state = 477 self.match(HogQLParser.COMMA) - self.state = 486 + self.state = 478 self.columnTypeExpr() - self.state = 491 + self.state = 483 self._errHandler.sync(self) _la = self._input.LA(1) - self.state = 492 + self.state = 484 self.match(HogQLParser.RPAREN) pass elif la_ == 5: localctx = HogQLParser.ColumnTypeExprParamContext(self, localctx) self.enterOuterAlt(localctx, 5) - self.state = 494 + self.state = 486 self.identifier() - self.state = 495 + self.state = 487 self.match(HogQLParser.LPAREN) - self.state = 497 + self.state = 489 self._errHandler.sync(self) _la = self._input.LA(1) if ((_la) & ~0x3f) == 0 and ((1 << _la) & -4) != 0 or (((_la - 64)) & ~0x3f) == 0 and ((1 << (_la - 64)) & -1) != 0 or (((_la - 128)) & ~0x3f) == 0 and ((1 << (_la - 128)) & -5) != 0 or (((_la - 193)) & ~0x3f) == 0 and ((1 << (_la - 193)) & 578896255) != 0: - self.state = 496 + self.state = 488 self.columnExprList() - self.state = 499 + self.state = 491 self.match(HogQLParser.RPAREN) pass @@ -3960,23 +3902,23 @@ def accept(self, visitor:ParseTreeVisitor): def columnExprList(self): localctx = HogQLParser.ColumnExprListContext(self, self._ctx, self.state) - self.enterRule(localctx, 74, self.RULE_columnExprList) + self.enterRule(localctx, 72, self.RULE_columnExprList) try: self.enterOuterAlt(localctx, 1) - self.state = 503 + self.state = 495 self.columnsExpr() - self.state = 508 + self.state = 500 self._errHandler.sync(self) - _alt = self._interp.adaptivePredict(self._input,65,self._ctx) + _alt = self._interp.adaptivePredict(self._input,64,self._ctx) while _alt!=2 and _alt!=ATN.INVALID_ALT_NUMBER: if _alt==1: - self.state = 504 + self.state = 496 self.match(HogQLParser.COMMA) - self.state = 505 + self.state = 497 self.columnsExpr() - self.state = 510 + self.state = 502 self._errHandler.sync(self) - _alt = self._interp.adaptivePredict(self._input,65,self._ctx) + _alt = self._interp.adaptivePredict(self._input,64,self._ctx) except RecognitionException as re: localctx.exception = re @@ -4067,44 +4009,44 @@ def accept(self, visitor:ParseTreeVisitor): def columnsExpr(self): localctx = HogQLParser.ColumnsExprContext(self, self._ctx, self.state) - self.enterRule(localctx, 76, self.RULE_columnsExpr) + self.enterRule(localctx, 74, self.RULE_columnsExpr) self._la = 0 # Token type try: - self.state = 522 + self.state = 514 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,67,self._ctx) + la_ = self._interp.adaptivePredict(self._input,66,self._ctx) if la_ == 1: localctx = HogQLParser.ColumnsExprAsteriskContext(self, localctx) self.enterOuterAlt(localctx, 1) - self.state = 514 + self.state = 506 self._errHandler.sync(self) _la = self._input.LA(1) if (((_la - 2)) & ~0x3f) == 0 and ((1 << (_la - 2)) & -1) != 0 or (((_la - 66)) & ~0x3f) == 0 and ((1 << (_la - 66)) & -633318697631745) != 0 or (((_la - 131)) & ~0x3f) == 0 and ((1 << (_la - 131)) & 6917529027641081855) != 0: - self.state = 511 + self.state = 503 self.tableIdentifier() - self.state = 512 + self.state = 504 self.match(HogQLParser.DOT) - self.state = 516 + self.state = 508 self.match(HogQLParser.ASTERISK) pass elif la_ == 2: localctx = HogQLParser.ColumnsExprSubqueryContext(self, localctx) self.enterOuterAlt(localctx, 2) - self.state = 517 + self.state = 509 self.match(HogQLParser.LPAREN) - self.state = 518 + self.state = 510 self.selectUnionStmt() - self.state = 519 + self.state = 511 self.match(HogQLParser.RPAREN) pass elif la_ == 3: localctx = HogQLParser.ColumnsExprColumnContext(self, localctx) self.enterOuterAlt(localctx, 3) - self.state = 521 + self.state = 513 self.columnExpr(0) pass @@ -4927,58 +4869,58 @@ def columnExpr(self, _p:int=0): _parentState = self.state localctx = HogQLParser.ColumnExprContext(self, self._ctx, _parentState) _prevctx = localctx - _startState = 78 - self.enterRecursionRule(localctx, 78, self.RULE_columnExpr, _p) + _startState = 76 + self.enterRecursionRule(localctx, 76, self.RULE_columnExpr, _p) self._la = 0 # Token type try: self.enterOuterAlt(localctx, 1) - self.state = 653 + self.state = 645 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,80,self._ctx) + la_ = self._interp.adaptivePredict(self._input,79,self._ctx) if la_ == 1: localctx = HogQLParser.ColumnExprCaseContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 525 + self.state = 517 self.match(HogQLParser.CASE) - self.state = 527 + self.state = 519 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,68,self._ctx) + la_ = self._interp.adaptivePredict(self._input,67,self._ctx) if la_ == 1: - self.state = 526 + self.state = 518 localctx.caseExpr = self.columnExpr(0) - self.state = 534 + self.state = 526 self._errHandler.sync(self) _la = self._input.LA(1) while True: - self.state = 529 + self.state = 521 self.match(HogQLParser.WHEN) - self.state = 530 + self.state = 522 localctx.whenExpr = self.columnExpr(0) - self.state = 531 + self.state = 523 self.match(HogQLParser.THEN) - self.state = 532 + self.state = 524 localctx.thenExpr = self.columnExpr(0) - self.state = 536 + self.state = 528 self._errHandler.sync(self) _la = self._input.LA(1) if not (_la==185): break - self.state = 540 + self.state = 532 self._errHandler.sync(self) _la = self._input.LA(1) if _la==51: - self.state = 538 + self.state = 530 self.match(HogQLParser.ELSE) - self.state = 539 + self.state = 531 localctx.elseExpr = self.columnExpr(0) - self.state = 542 + self.state = 534 self.match(HogQLParser.END) pass @@ -4986,17 +4928,17 @@ def columnExpr(self, _p:int=0): localctx = HogQLParser.ColumnExprCastContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 544 + self.state = 536 self.match(HogQLParser.CAST) - self.state = 545 + self.state = 537 self.match(HogQLParser.LPAREN) - self.state = 546 + self.state = 538 self.columnExpr(0) - self.state = 547 + self.state = 539 self.match(HogQLParser.AS) - self.state = 548 + self.state = 540 self.columnTypeExpr() - self.state = 549 + self.state = 541 self.match(HogQLParser.RPAREN) pass @@ -5004,9 +4946,9 @@ def columnExpr(self, _p:int=0): localctx = HogQLParser.ColumnExprDateContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 551 + self.state = 543 self.match(HogQLParser.DATE) - self.state = 552 + self.state = 544 self.match(HogQLParser.STRING_LITERAL) pass @@ -5014,17 +4956,17 @@ def columnExpr(self, _p:int=0): localctx = HogQLParser.ColumnExprExtractContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 553 + self.state = 545 self.match(HogQLParser.EXTRACT) - self.state = 554 + self.state = 546 self.match(HogQLParser.LPAREN) - self.state = 555 + self.state = 547 self.interval() - self.state = 556 + self.state = 548 self.match(HogQLParser.FROM) - self.state = 557 + self.state = 549 self.columnExpr(0) - self.state = 558 + self.state = 550 self.match(HogQLParser.RPAREN) pass @@ -5032,11 +4974,11 @@ def columnExpr(self, _p:int=0): localctx = HogQLParser.ColumnExprIntervalContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 560 + self.state = 552 self.match(HogQLParser.INTERVAL) - self.state = 561 + self.state = 553 self.columnExpr(0) - self.state = 562 + self.state = 554 self.interval() pass @@ -5044,27 +4986,27 @@ def columnExpr(self, _p:int=0): localctx = HogQLParser.ColumnExprSubstringContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 564 + self.state = 556 self.match(HogQLParser.SUBSTRING) - self.state = 565 + self.state = 557 self.match(HogQLParser.LPAREN) - self.state = 566 + self.state = 558 self.columnExpr(0) - self.state = 567 + self.state = 559 self.match(HogQLParser.FROM) - self.state = 568 + self.state = 560 self.columnExpr(0) - self.state = 571 + self.state = 563 self._errHandler.sync(self) _la = self._input.LA(1) if _la==64: - self.state = 569 + self.state = 561 self.match(HogQLParser.FOR) - self.state = 570 + self.state = 562 self.columnExpr(0) - self.state = 573 + self.state = 565 self.match(HogQLParser.RPAREN) pass @@ -5072,9 +5014,9 @@ def columnExpr(self, _p:int=0): localctx = HogQLParser.ColumnExprTimestampContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 575 + self.state = 567 self.match(HogQLParser.TIMESTAMP) - self.state = 576 + self.state = 568 self.match(HogQLParser.STRING_LITERAL) pass @@ -5082,24 +5024,24 @@ def columnExpr(self, _p:int=0): localctx = HogQLParser.ColumnExprTrimContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 577 + self.state = 569 self.match(HogQLParser.TRIM) - self.state = 578 + self.state = 570 self.match(HogQLParser.LPAREN) - self.state = 579 + self.state = 571 _la = self._input.LA(1) if not(_la==17 or _la==94 or _la==169): self._errHandler.recoverInline(self) else: self._errHandler.reportMatch(self) self.consume() - self.state = 580 + self.state = 572 self.match(HogQLParser.STRING_LITERAL) - self.state = 581 + self.state = 573 self.match(HogQLParser.FROM) - self.state = 582 + self.state = 574 self.columnExpr(0) - self.state = 583 + self.state = 575 self.match(HogQLParser.RPAREN) pass @@ -5107,28 +5049,28 @@ def columnExpr(self, _p:int=0): localctx = HogQLParser.ColumnExprWinFunctionContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 585 + self.state = 577 self.identifier() - self.state = 586 + self.state = 578 self.match(HogQLParser.LPAREN) - self.state = 588 + self.state = 580 self._errHandler.sync(self) _la = self._input.LA(1) if ((_la) & ~0x3f) == 0 and ((1 << _la) & -4) != 0 or (((_la - 64)) & ~0x3f) == 0 and ((1 << (_la - 64)) & -1) != 0 or (((_la - 128)) & ~0x3f) == 0 and ((1 << (_la - 128)) & -5) != 0 or (((_la - 193)) & ~0x3f) == 0 and ((1 << (_la - 193)) & 578896255) != 0: - self.state = 587 + self.state = 579 self.columnExprList() - self.state = 590 + self.state = 582 self.match(HogQLParser.RPAREN) - self.state = 592 + self.state = 584 self.match(HogQLParser.OVER) - self.state = 593 + self.state = 585 self.match(HogQLParser.LPAREN) - self.state = 594 + self.state = 586 self.windowExpr() - self.state = 595 + self.state = 587 self.match(HogQLParser.RPAREN) pass @@ -5136,24 +5078,24 @@ def columnExpr(self, _p:int=0): localctx = HogQLParser.ColumnExprWinFunctionTargetContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 597 + self.state = 589 self.identifier() - self.state = 598 + self.state = 590 self.match(HogQLParser.LPAREN) - self.state = 600 + self.state = 592 self._errHandler.sync(self) _la = self._input.LA(1) if ((_la) & ~0x3f) == 0 and ((1 << _la) & -4) != 0 or (((_la - 64)) & ~0x3f) == 0 and ((1 << (_la - 64)) & -1) != 0 or (((_la - 128)) & ~0x3f) == 0 and ((1 << (_la - 128)) & -5) != 0 or (((_la - 193)) & ~0x3f) == 0 and ((1 << (_la - 193)) & 578896255) != 0: - self.state = 599 + self.state = 591 self.columnExprList() - self.state = 602 + self.state = 594 self.match(HogQLParser.RPAREN) - self.state = 604 + self.state = 596 self.match(HogQLParser.OVER) - self.state = 605 + self.state = 597 self.identifier() pass @@ -5161,45 +5103,45 @@ def columnExpr(self, _p:int=0): localctx = HogQLParser.ColumnExprFunctionContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 607 + self.state = 599 self.identifier() - self.state = 613 + self.state = 605 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,75,self._ctx) + la_ = self._interp.adaptivePredict(self._input,74,self._ctx) if la_ == 1: - self.state = 608 + self.state = 600 self.match(HogQLParser.LPAREN) - self.state = 610 + self.state = 602 self._errHandler.sync(self) _la = self._input.LA(1) if ((_la) & ~0x3f) == 0 and ((1 << _la) & -4) != 0 or (((_la - 64)) & ~0x3f) == 0 and ((1 << (_la - 64)) & -1) != 0 or (((_la - 128)) & ~0x3f) == 0 and ((1 << (_la - 128)) & -5) != 0 or (((_la - 193)) & ~0x3f) == 0 and ((1 << (_la - 193)) & 578896255) != 0: - self.state = 609 + self.state = 601 self.columnExprList() - self.state = 612 + self.state = 604 self.match(HogQLParser.RPAREN) - self.state = 615 + self.state = 607 self.match(HogQLParser.LPAREN) - self.state = 617 + self.state = 609 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,76,self._ctx) + la_ = self._interp.adaptivePredict(self._input,75,self._ctx) if la_ == 1: - self.state = 616 + self.state = 608 self.match(HogQLParser.DISTINCT) - self.state = 620 + self.state = 612 self._errHandler.sync(self) _la = self._input.LA(1) if ((_la) & ~0x3f) == 0 and ((1 << _la) & -4) != 0 or (((_la - 64)) & ~0x3f) == 0 and ((1 << (_la - 64)) & -1) != 0 or (((_la - 128)) & ~0x3f) == 0 and ((1 << (_la - 128)) & -5) != 0 or (((_la - 193)) & ~0x3f) == 0 and ((1 << (_la - 193)) & 578896255) != 0: - self.state = 619 + self.state = 611 self.columnArgList() - self.state = 622 + self.state = 614 self.match(HogQLParser.RPAREN) pass @@ -5207,7 +5149,7 @@ def columnExpr(self, _p:int=0): localctx = HogQLParser.ColumnExprLiteralContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 624 + self.state = 616 self.literal() pass @@ -5215,9 +5157,9 @@ def columnExpr(self, _p:int=0): localctx = HogQLParser.ColumnExprNegateContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 625 + self.state = 617 self.match(HogQLParser.DASH) - self.state = 626 + self.state = 618 self.columnExpr(17) pass @@ -5225,9 +5167,9 @@ def columnExpr(self, _p:int=0): localctx = HogQLParser.ColumnExprNotContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 627 + self.state = 619 self.match(HogQLParser.NOT) - self.state = 628 + self.state = 620 self.columnExpr(12) pass @@ -5235,17 +5177,17 @@ def columnExpr(self, _p:int=0): localctx = HogQLParser.ColumnExprAsteriskContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 632 + self.state = 624 self._errHandler.sync(self) _la = self._input.LA(1) if (((_la - 2)) & ~0x3f) == 0 and ((1 << (_la - 2)) & -1) != 0 or (((_la - 66)) & ~0x3f) == 0 and ((1 << (_la - 66)) & -633318697631745) != 0 or (((_la - 131)) & ~0x3f) == 0 and ((1 << (_la - 131)) & 6917529027641081855) != 0: - self.state = 629 + self.state = 621 self.tableIdentifier() - self.state = 630 + self.state = 622 self.match(HogQLParser.DOT) - self.state = 634 + self.state = 626 self.match(HogQLParser.ASTERISK) pass @@ -5253,11 +5195,11 @@ def columnExpr(self, _p:int=0): localctx = HogQLParser.ColumnExprSubqueryContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 635 + self.state = 627 self.match(HogQLParser.LPAREN) - self.state = 636 + self.state = 628 self.selectUnionStmt() - self.state = 637 + self.state = 629 self.match(HogQLParser.RPAREN) pass @@ -5265,11 +5207,11 @@ def columnExpr(self, _p:int=0): localctx = HogQLParser.ColumnExprParensContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 639 + self.state = 631 self.match(HogQLParser.LPAREN) - self.state = 640 + self.state = 632 self.columnExpr(0) - self.state = 641 + self.state = 633 self.match(HogQLParser.RPAREN) pass @@ -5277,11 +5219,11 @@ def columnExpr(self, _p:int=0): localctx = HogQLParser.ColumnExprTupleContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 643 + self.state = 635 self.match(HogQLParser.LPAREN) - self.state = 644 + self.state = 636 self.columnExprList() - self.state = 645 + self.state = 637 self.match(HogQLParser.RPAREN) pass @@ -5289,17 +5231,17 @@ def columnExpr(self, _p:int=0): localctx = HogQLParser.ColumnExprArrayContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 647 + self.state = 639 self.match(HogQLParser.LBRACKET) - self.state = 649 + self.state = 641 self._errHandler.sync(self) _la = self._input.LA(1) if ((_la) & ~0x3f) == 0 and ((1 << _la) & -4) != 0 or (((_la - 64)) & ~0x3f) == 0 and ((1 << (_la - 64)) & -1) != 0 or (((_la - 128)) & ~0x3f) == 0 and ((1 << (_la - 128)) & -5) != 0 or (((_la - 193)) & ~0x3f) == 0 and ((1 << (_la - 193)) & 578896255) != 0: - self.state = 648 + self.state = 640 self.columnExprList() - self.state = 651 + self.state = 643 self.match(HogQLParser.RBRACKET) pass @@ -5307,50 +5249,50 @@ def columnExpr(self, _p:int=0): localctx = HogQLParser.ColumnExprIdentifierContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 652 + self.state = 644 self.columnIdentifier() pass self._ctx.stop = self._input.LT(-1) - self.state = 736 + self.state = 728 self._errHandler.sync(self) - _alt = self._interp.adaptivePredict(self._input,91,self._ctx) + _alt = self._interp.adaptivePredict(self._input,90,self._ctx) while _alt!=2 and _alt!=ATN.INVALID_ALT_NUMBER: if _alt==1: if self._parseListeners is not None: self.triggerExitRuleEvent() _prevctx = localctx - self.state = 734 + self.state = 726 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,90,self._ctx) + la_ = self._interp.adaptivePredict(self._input,89,self._ctx) if la_ == 1: localctx = HogQLParser.ColumnExprPrecedence1Context(self, HogQLParser.ColumnExprContext(self, _parentctx, _parentState)) localctx.left = _prevctx self.pushNewRecursionContext(localctx, _startState, self.RULE_columnExpr) - self.state = 655 + self.state = 647 if not self.precpred(self._ctx, 16): from antlr4.error.Errors import FailedPredicateException raise FailedPredicateException(self, "self.precpred(self._ctx, 16)") - self.state = 659 + self.state = 651 self._errHandler.sync(self) token = self._input.LA(1) if token in [201]: - self.state = 656 + self.state = 648 localctx.operator = self.match(HogQLParser.ASTERISK) pass elif token in [230]: - self.state = 657 + self.state = 649 localctx.operator = self.match(HogQLParser.SLASH) pass elif token in [221]: - self.state = 658 + self.state = 650 localctx.operator = self.match(HogQLParser.PERCENT) pass else: raise NoViableAltException(self) - self.state = 661 + self.state = 653 localctx.right = self.columnExpr(17) pass @@ -5358,29 +5300,29 @@ def columnExpr(self, _p:int=0): localctx = HogQLParser.ColumnExprPrecedence2Context(self, HogQLParser.ColumnExprContext(self, _parentctx, _parentState)) localctx.left = _prevctx self.pushNewRecursionContext(localctx, _startState, self.RULE_columnExpr) - self.state = 662 + self.state = 654 if not self.precpred(self._ctx, 15): from antlr4.error.Errors import FailedPredicateException raise FailedPredicateException(self, "self.precpred(self._ctx, 15)") - self.state = 666 + self.state = 658 self._errHandler.sync(self) token = self._input.LA(1) if token in [222]: - self.state = 663 + self.state = 655 localctx.operator = self.match(HogQLParser.PLUS) pass elif token in [207]: - self.state = 664 + self.state = 656 localctx.operator = self.match(HogQLParser.DASH) pass elif token in [206]: - self.state = 665 + self.state = 657 localctx.operator = self.match(HogQLParser.CONCAT) pass else: raise NoViableAltException(self) - self.state = 668 + self.state = 660 localctx.right = self.columnExpr(16) pass @@ -5388,79 +5330,79 @@ def columnExpr(self, _p:int=0): localctx = HogQLParser.ColumnExprPrecedence3Context(self, HogQLParser.ColumnExprContext(self, _parentctx, _parentState)) localctx.left = _prevctx self.pushNewRecursionContext(localctx, _startState, self.RULE_columnExpr) - self.state = 669 + self.state = 661 if not self.precpred(self._ctx, 14): from antlr4.error.Errors import FailedPredicateException raise FailedPredicateException(self, "self.precpred(self._ctx, 14)") - self.state = 688 + self.state = 680 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,86,self._ctx) + la_ = self._interp.adaptivePredict(self._input,85,self._ctx) if la_ == 1: - self.state = 670 + self.state = 662 localctx.operator = self.match(HogQLParser.EQ_DOUBLE) pass elif la_ == 2: - self.state = 671 + self.state = 663 localctx.operator = self.match(HogQLParser.EQ_SINGLE) pass elif la_ == 3: - self.state = 672 + self.state = 664 localctx.operator = self.match(HogQLParser.NOT_EQ) pass elif la_ == 4: - self.state = 673 + self.state = 665 localctx.operator = self.match(HogQLParser.LE) pass elif la_ == 5: - self.state = 674 + self.state = 666 localctx.operator = self.match(HogQLParser.GE) pass elif la_ == 6: - self.state = 675 + self.state = 667 localctx.operator = self.match(HogQLParser.LT) pass elif la_ == 7: - self.state = 676 + self.state = 668 localctx.operator = self.match(HogQLParser.GT) pass elif la_ == 8: - self.state = 678 + self.state = 670 self._errHandler.sync(self) _la = self._input.LA(1) if _la==70: - self.state = 677 + self.state = 669 localctx.operator = self.match(HogQLParser.GLOBAL) - self.state = 681 + self.state = 673 self._errHandler.sync(self) _la = self._input.LA(1) if _la==114: - self.state = 680 + self.state = 672 self.match(HogQLParser.NOT) - self.state = 683 + self.state = 675 self.match(HogQLParser.IN) pass elif la_ == 9: - self.state = 685 + self.state = 677 self._errHandler.sync(self) _la = self._input.LA(1) if _la==114: - self.state = 684 + self.state = 676 localctx.operator = self.match(HogQLParser.NOT) - self.state = 687 + self.state = 679 _la = self._input.LA(1) if not(_la==78 or _la==97): self._errHandler.recoverInline(self) @@ -5470,153 +5412,153 @@ def columnExpr(self, _p:int=0): pass - self.state = 690 + self.state = 682 localctx.right = self.columnExpr(15) pass elif la_ == 4: localctx = HogQLParser.ColumnExprAndContext(self, HogQLParser.ColumnExprContext(self, _parentctx, _parentState)) self.pushNewRecursionContext(localctx, _startState, self.RULE_columnExpr) - self.state = 691 + self.state = 683 if not self.precpred(self._ctx, 11): from antlr4.error.Errors import FailedPredicateException raise FailedPredicateException(self, "self.precpred(self._ctx, 11)") - self.state = 692 + self.state = 684 self.match(HogQLParser.AND) - self.state = 693 + self.state = 685 self.columnExpr(12) pass elif la_ == 5: localctx = HogQLParser.ColumnExprOrContext(self, HogQLParser.ColumnExprContext(self, _parentctx, _parentState)) self.pushNewRecursionContext(localctx, _startState, self.RULE_columnExpr) - self.state = 694 + self.state = 686 if not self.precpred(self._ctx, 10): from antlr4.error.Errors import FailedPredicateException raise FailedPredicateException(self, "self.precpred(self._ctx, 10)") - self.state = 695 + self.state = 687 self.match(HogQLParser.OR) - self.state = 696 + self.state = 688 self.columnExpr(11) pass elif la_ == 6: localctx = HogQLParser.ColumnExprBetweenContext(self, HogQLParser.ColumnExprContext(self, _parentctx, _parentState)) self.pushNewRecursionContext(localctx, _startState, self.RULE_columnExpr) - self.state = 697 + self.state = 689 if not self.precpred(self._ctx, 9): from antlr4.error.Errors import FailedPredicateException raise FailedPredicateException(self, "self.precpred(self._ctx, 9)") - self.state = 699 + self.state = 691 self._errHandler.sync(self) _la = self._input.LA(1) if _la==114: - self.state = 698 + self.state = 690 self.match(HogQLParser.NOT) - self.state = 701 + self.state = 693 self.match(HogQLParser.BETWEEN) - self.state = 702 + self.state = 694 self.columnExpr(0) - self.state = 703 + self.state = 695 self.match(HogQLParser.AND) - self.state = 704 + self.state = 696 self.columnExpr(10) pass elif la_ == 7: localctx = HogQLParser.ColumnExprTernaryOpContext(self, HogQLParser.ColumnExprContext(self, _parentctx, _parentState)) self.pushNewRecursionContext(localctx, _startState, self.RULE_columnExpr) - self.state = 706 + self.state = 698 if not self.precpred(self._ctx, 8): from antlr4.error.Errors import FailedPredicateException raise FailedPredicateException(self, "self.precpred(self._ctx, 8)") - self.state = 707 + self.state = 699 self.match(HogQLParser.QUERY) - self.state = 708 + self.state = 700 self.columnExpr(0) - self.state = 709 + self.state = 701 self.match(HogQLParser.COLON) - self.state = 710 + self.state = 702 self.columnExpr(8) pass elif la_ == 8: localctx = HogQLParser.ColumnExprArrayAccessContext(self, HogQLParser.ColumnExprContext(self, _parentctx, _parentState)) self.pushNewRecursionContext(localctx, _startState, self.RULE_columnExpr) - self.state = 712 + self.state = 704 if not self.precpred(self._ctx, 19): from antlr4.error.Errors import FailedPredicateException raise FailedPredicateException(self, "self.precpred(self._ctx, 19)") - self.state = 713 + self.state = 705 self.match(HogQLParser.LBRACKET) - self.state = 714 + self.state = 706 self.columnExpr(0) - self.state = 715 + self.state = 707 self.match(HogQLParser.RBRACKET) pass elif la_ == 9: localctx = HogQLParser.ColumnExprTupleAccessContext(self, HogQLParser.ColumnExprContext(self, _parentctx, _parentState)) self.pushNewRecursionContext(localctx, _startState, self.RULE_columnExpr) - self.state = 717 + self.state = 709 if not self.precpred(self._ctx, 18): from antlr4.error.Errors import FailedPredicateException raise FailedPredicateException(self, "self.precpred(self._ctx, 18)") - self.state = 718 + self.state = 710 self.match(HogQLParser.DOT) - self.state = 719 + self.state = 711 self.match(HogQLParser.DECIMAL_LITERAL) pass elif la_ == 10: localctx = HogQLParser.ColumnExprIsNullContext(self, HogQLParser.ColumnExprContext(self, _parentctx, _parentState)) self.pushNewRecursionContext(localctx, _startState, self.RULE_columnExpr) - self.state = 720 + self.state = 712 if not self.precpred(self._ctx, 13): from antlr4.error.Errors import FailedPredicateException raise FailedPredicateException(self, "self.precpred(self._ctx, 13)") - self.state = 721 + self.state = 713 self.match(HogQLParser.IS) - self.state = 723 + self.state = 715 self._errHandler.sync(self) _la = self._input.LA(1) if _la==114: - self.state = 722 + self.state = 714 self.match(HogQLParser.NOT) - self.state = 725 + self.state = 717 self.match(HogQLParser.NULL_SQL) pass elif la_ == 11: localctx = HogQLParser.ColumnExprAliasContext(self, HogQLParser.ColumnExprContext(self, _parentctx, _parentState)) self.pushNewRecursionContext(localctx, _startState, self.RULE_columnExpr) - self.state = 726 + self.state = 718 if not self.precpred(self._ctx, 7): from antlr4.error.Errors import FailedPredicateException raise FailedPredicateException(self, "self.precpred(self._ctx, 7)") - self.state = 732 + self.state = 724 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,89,self._ctx) + la_ = self._interp.adaptivePredict(self._input,88,self._ctx) if la_ == 1: - self.state = 727 + self.state = 719 self.alias() pass elif la_ == 2: - self.state = 728 + self.state = 720 self.match(HogQLParser.AS) - self.state = 729 + self.state = 721 self.identifier() pass elif la_ == 3: - self.state = 730 + self.state = 722 self.match(HogQLParser.AS) - self.state = 731 + self.state = 723 self.match(HogQLParser.STRING_LITERAL) pass @@ -5624,9 +5566,9 @@ def columnExpr(self, _p:int=0): pass - self.state = 738 + self.state = 730 self._errHandler.sync(self) - _alt = self._interp.adaptivePredict(self._input,91,self._ctx) + _alt = self._interp.adaptivePredict(self._input,90,self._ctx) except RecognitionException as re: localctx.exception = re @@ -5672,21 +5614,21 @@ def accept(self, visitor:ParseTreeVisitor): def columnArgList(self): localctx = HogQLParser.ColumnArgListContext(self, self._ctx, self.state) - self.enterRule(localctx, 80, self.RULE_columnArgList) + self.enterRule(localctx, 78, self.RULE_columnArgList) self._la = 0 # Token type try: self.enterOuterAlt(localctx, 1) - self.state = 739 + self.state = 731 self.columnArgExpr() - self.state = 744 + self.state = 736 self._errHandler.sync(self) _la = self._input.LA(1) while _la==205: - self.state = 740 + self.state = 732 self.match(HogQLParser.COMMA) - self.state = 741 + self.state = 733 self.columnArgExpr() - self.state = 746 + self.state = 738 self._errHandler.sync(self) _la = self._input.LA(1) @@ -5729,20 +5671,20 @@ def accept(self, visitor:ParseTreeVisitor): def columnArgExpr(self): localctx = HogQLParser.ColumnArgExprContext(self, self._ctx, self.state) - self.enterRule(localctx, 82, self.RULE_columnArgExpr) + self.enterRule(localctx, 80, self.RULE_columnArgExpr) try: - self.state = 749 + self.state = 741 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,93,self._ctx) + la_ = self._interp.adaptivePredict(self._input,92,self._ctx) if la_ == 1: self.enterOuterAlt(localctx, 1) - self.state = 747 + self.state = 739 self.columnLambdaExpr() pass elif la_ == 2: self.enterOuterAlt(localctx, 2) - self.state = 748 + self.state = 740 self.columnExpr(0) pass @@ -5804,45 +5746,45 @@ def accept(self, visitor:ParseTreeVisitor): def columnLambdaExpr(self): localctx = HogQLParser.ColumnLambdaExprContext(self, self._ctx, self.state) - self.enterRule(localctx, 84, self.RULE_columnLambdaExpr) + self.enterRule(localctx, 82, self.RULE_columnLambdaExpr) self._la = 0 # Token type try: self.enterOuterAlt(localctx, 1) - self.state = 770 + self.state = 762 self._errHandler.sync(self) token = self._input.LA(1) if token in [218]: - self.state = 751 + self.state = 743 self.match(HogQLParser.LPAREN) - self.state = 752 + self.state = 744 self.identifier() - self.state = 757 + self.state = 749 self._errHandler.sync(self) _la = self._input.LA(1) while _la==205: - self.state = 753 + self.state = 745 self.match(HogQLParser.COMMA) - self.state = 754 + self.state = 746 self.identifier() - self.state = 759 + self.state = 751 self._errHandler.sync(self) _la = self._input.LA(1) - self.state = 760 + self.state = 752 self.match(HogQLParser.RPAREN) pass elif token in [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 113, 114, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 193]: - self.state = 762 + self.state = 754 self.identifier() - self.state = 767 + self.state = 759 self._errHandler.sync(self) _la = self._input.LA(1) while _la==205: - self.state = 763 + self.state = 755 self.match(HogQLParser.COMMA) - self.state = 764 + self.state = 756 self.identifier() - self.state = 769 + self.state = 761 self._errHandler.sync(self) _la = self._input.LA(1) @@ -5850,9 +5792,9 @@ def columnLambdaExpr(self): else: raise NoViableAltException(self) - self.state = 772 + self.state = 764 self.match(HogQLParser.ARROW) - self.state = 773 + self.state = 765 self.columnExpr(0) except RecognitionException as re: localctx.exception = re @@ -5899,29 +5841,29 @@ def accept(self, visitor:ParseTreeVisitor): def columnIdentifier(self): localctx = HogQLParser.ColumnIdentifierContext(self, self._ctx, self.state) - self.enterRule(localctx, 86, self.RULE_columnIdentifier) + self.enterRule(localctx, 84, self.RULE_columnIdentifier) try: - self.state = 782 + self.state = 774 self._errHandler.sync(self) token = self._input.LA(1) if token in [199]: self.enterOuterAlt(localctx, 1) - self.state = 775 + self.state = 767 self.match(HogQLParser.PLACEHOLDER) pass elif token in [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 113, 114, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 193]: self.enterOuterAlt(localctx, 2) - self.state = 779 + self.state = 771 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,97,self._ctx) + la_ = self._interp.adaptivePredict(self._input,96,self._ctx) if la_ == 1: - self.state = 776 + self.state = 768 self.tableIdentifier() - self.state = 777 + self.state = 769 self.match(HogQLParser.DOT) - self.state = 781 + self.state = 773 self.nestedIdentifier() pass else: @@ -5971,23 +5913,23 @@ def accept(self, visitor:ParseTreeVisitor): def nestedIdentifier(self): localctx = HogQLParser.NestedIdentifierContext(self, self._ctx, self.state) - self.enterRule(localctx, 88, self.RULE_nestedIdentifier) + self.enterRule(localctx, 86, self.RULE_nestedIdentifier) try: self.enterOuterAlt(localctx, 1) - self.state = 784 + self.state = 776 self.identifier() - self.state = 789 + self.state = 781 self._errHandler.sync(self) - _alt = self._interp.adaptivePredict(self._input,99,self._ctx) + _alt = self._interp.adaptivePredict(self._input,98,self._ctx) while _alt!=2 and _alt!=ATN.INVALID_ALT_NUMBER: if _alt==1: - self.state = 785 + self.state = 777 self.match(HogQLParser.DOT) - self.state = 786 + self.state = 778 self.identifier() - self.state = 791 + self.state = 783 self._errHandler.sync(self) - _alt = self._interp.adaptivePredict(self._input,99,self._ctx) + _alt = self._interp.adaptivePredict(self._input,98,self._ctx) except RecognitionException as re: localctx.exception = re @@ -6100,19 +6042,19 @@ def tableExpr(self, _p:int=0): _parentState = self.state localctx = HogQLParser.TableExprContext(self, self._ctx, _parentState) _prevctx = localctx - _startState = 90 - self.enterRecursionRule(localctx, 90, self.RULE_tableExpr, _p) + _startState = 88 + self.enterRecursionRule(localctx, 88, self.RULE_tableExpr, _p) try: self.enterOuterAlt(localctx, 1) - self.state = 799 + self.state = 791 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,100,self._ctx) + la_ = self._interp.adaptivePredict(self._input,99,self._ctx) if la_ == 1: localctx = HogQLParser.TableExprIdentifierContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 793 + self.state = 785 self.tableIdentifier() pass @@ -6120,7 +6062,7 @@ def tableExpr(self, _p:int=0): localctx = HogQLParser.TableExprFunctionContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 794 + self.state = 786 self.tableFunctionExpr() pass @@ -6128,19 +6070,19 @@ def tableExpr(self, _p:int=0): localctx = HogQLParser.TableExprSubqueryContext(self, localctx) self._ctx = localctx _prevctx = localctx - self.state = 795 + self.state = 787 self.match(HogQLParser.LPAREN) - self.state = 796 + self.state = 788 self.selectUnionStmt() - self.state = 797 + self.state = 789 self.match(HogQLParser.RPAREN) pass self._ctx.stop = self._input.LT(-1) - self.state = 809 + self.state = 801 self._errHandler.sync(self) - _alt = self._interp.adaptivePredict(self._input,102,self._ctx) + _alt = self._interp.adaptivePredict(self._input,101,self._ctx) while _alt!=2 and _alt!=ATN.INVALID_ALT_NUMBER: if _alt==1: if self._parseListeners is not None: @@ -6148,29 +6090,29 @@ def tableExpr(self, _p:int=0): _prevctx = localctx localctx = HogQLParser.TableExprAliasContext(self, HogQLParser.TableExprContext(self, _parentctx, _parentState)) self.pushNewRecursionContext(localctx, _startState, self.RULE_tableExpr) - self.state = 801 + self.state = 793 if not self.precpred(self._ctx, 1): from antlr4.error.Errors import FailedPredicateException raise FailedPredicateException(self, "self.precpred(self._ctx, 1)") - self.state = 805 + self.state = 797 self._errHandler.sync(self) token = self._input.LA(1) if token in [35, 61, 76, 90, 193]: - self.state = 802 + self.state = 794 self.alias() pass elif token in [10]: - self.state = 803 + self.state = 795 self.match(HogQLParser.AS) - self.state = 804 + self.state = 796 self.identifier() pass else: raise NoViableAltException(self) - self.state = 811 + self.state = 803 self._errHandler.sync(self) - _alt = self._interp.adaptivePredict(self._input,102,self._ctx) + _alt = self._interp.adaptivePredict(self._input,101,self._ctx) except RecognitionException as re: localctx.exception = re @@ -6217,23 +6159,23 @@ def accept(self, visitor:ParseTreeVisitor): def tableFunctionExpr(self): localctx = HogQLParser.TableFunctionExprContext(self, self._ctx, self.state) - self.enterRule(localctx, 92, self.RULE_tableFunctionExpr) + self.enterRule(localctx, 90, self.RULE_tableFunctionExpr) self._la = 0 # Token type try: self.enterOuterAlt(localctx, 1) - self.state = 812 + self.state = 804 self.identifier() - self.state = 813 + self.state = 805 self.match(HogQLParser.LPAREN) - self.state = 815 + self.state = 807 self._errHandler.sync(self) _la = self._input.LA(1) if ((_la) & ~0x3f) == 0 and ((1 << _la) & -4) != 0 or (((_la - 64)) & ~0x3f) == 0 and ((1 << (_la - 64)) & -1) != 0 or (((_la - 128)) & ~0x3f) == 0 and ((1 << (_la - 128)) & -5) != 0 or (((_la - 193)) & ~0x3f) == 0 and ((1 << (_la - 193)) & 536952895) != 0: - self.state = 814 + self.state = 806 self.tableArgList() - self.state = 817 + self.state = 809 self.match(HogQLParser.RPAREN) except RecognitionException as re: localctx.exception = re @@ -6277,20 +6219,20 @@ def accept(self, visitor:ParseTreeVisitor): def tableIdentifier(self): localctx = HogQLParser.TableIdentifierContext(self, self._ctx, self.state) - self.enterRule(localctx, 94, self.RULE_tableIdentifier) + self.enterRule(localctx, 92, self.RULE_tableIdentifier) try: self.enterOuterAlt(localctx, 1) - self.state = 822 + self.state = 814 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,104,self._ctx) + la_ = self._interp.adaptivePredict(self._input,103,self._ctx) if la_ == 1: - self.state = 819 + self.state = 811 self.databaseIdentifier() - self.state = 820 + self.state = 812 self.match(HogQLParser.DOT) - self.state = 824 + self.state = 816 self.identifier() except RecognitionException as re: localctx.exception = re @@ -6336,21 +6278,21 @@ def accept(self, visitor:ParseTreeVisitor): def tableArgList(self): localctx = HogQLParser.TableArgListContext(self, self._ctx, self.state) - self.enterRule(localctx, 96, self.RULE_tableArgList) + self.enterRule(localctx, 94, self.RULE_tableArgList) self._la = 0 # Token type try: self.enterOuterAlt(localctx, 1) - self.state = 826 + self.state = 818 self.tableArgExpr() - self.state = 831 + self.state = 823 self._errHandler.sync(self) _la = self._input.LA(1) while _la==205: - self.state = 827 + self.state = 819 self.match(HogQLParser.COMMA) - self.state = 828 + self.state = 820 self.tableArgExpr() - self.state = 833 + self.state = 825 self._errHandler.sync(self) _la = self._input.LA(1) @@ -6397,26 +6339,26 @@ def accept(self, visitor:ParseTreeVisitor): def tableArgExpr(self): localctx = HogQLParser.TableArgExprContext(self, self._ctx, self.state) - self.enterRule(localctx, 98, self.RULE_tableArgExpr) + self.enterRule(localctx, 96, self.RULE_tableArgExpr) try: - self.state = 837 + self.state = 829 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,106,self._ctx) + la_ = self._interp.adaptivePredict(self._input,105,self._ctx) if la_ == 1: self.enterOuterAlt(localctx, 1) - self.state = 834 + self.state = 826 self.nestedIdentifier() pass elif la_ == 2: self.enterOuterAlt(localctx, 2) - self.state = 835 + self.state = 827 self.tableFunctionExpr() pass elif la_ == 3: self.enterOuterAlt(localctx, 3) - self.state = 836 + self.state = 828 self.literal() pass @@ -6456,10 +6398,10 @@ def accept(self, visitor:ParseTreeVisitor): def databaseIdentifier(self): localctx = HogQLParser.DatabaseIdentifierContext(self, self._ctx, self.state) - self.enterRule(localctx, 100, self.RULE_databaseIdentifier) + self.enterRule(localctx, 98, self.RULE_databaseIdentifier) try: self.enterOuterAlt(localctx, 1) - self.state = 839 + self.state = 831 self.identifier() except RecognitionException as re: localctx.exception = re @@ -6507,22 +6449,22 @@ def accept(self, visitor:ParseTreeVisitor): def floatingLiteral(self): localctx = HogQLParser.FloatingLiteralContext(self, self._ctx, self.state) - self.enterRule(localctx, 102, self.RULE_floatingLiteral) + self.enterRule(localctx, 100, self.RULE_floatingLiteral) self._la = 0 # Token type try: - self.state = 849 + self.state = 841 self._errHandler.sync(self) token = self._input.LA(1) if token in [194]: self.enterOuterAlt(localctx, 1) - self.state = 841 + self.state = 833 self.match(HogQLParser.FLOATING_LITERAL) pass elif token in [209]: self.enterOuterAlt(localctx, 2) - self.state = 842 + self.state = 834 self.match(HogQLParser.DOT) - self.state = 843 + self.state = 835 _la = self._input.LA(1) if not(_la==195 or _la==196): self._errHandler.recoverInline(self) @@ -6532,15 +6474,15 @@ def floatingLiteral(self): pass elif token in [196]: self.enterOuterAlt(localctx, 3) - self.state = 844 + self.state = 836 self.match(HogQLParser.DECIMAL_LITERAL) - self.state = 845 + self.state = 837 self.match(HogQLParser.DOT) - self.state = 847 + self.state = 839 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,107,self._ctx) + la_ = self._interp.adaptivePredict(self._input,106,self._ctx) if la_ == 1: - self.state = 846 + self.state = 838 _la = self._input.LA(1) if not(_la==195 or _la==196): self._errHandler.recoverInline(self) @@ -6609,15 +6551,15 @@ def accept(self, visitor:ParseTreeVisitor): def numberLiteral(self): localctx = HogQLParser.NumberLiteralContext(self, self._ctx, self.state) - self.enterRule(localctx, 104, self.RULE_numberLiteral) + self.enterRule(localctx, 102, self.RULE_numberLiteral) self._la = 0 # Token type try: self.enterOuterAlt(localctx, 1) - self.state = 852 + self.state = 844 self._errHandler.sync(self) _la = self._input.LA(1) if _la==207 or _la==222: - self.state = 851 + self.state = 843 _la = self._input.LA(1) if not(_la==207 or _la==222): self._errHandler.recoverInline(self) @@ -6626,36 +6568,36 @@ def numberLiteral(self): self.consume() - self.state = 860 + self.state = 852 self._errHandler.sync(self) - la_ = self._interp.adaptivePredict(self._input,110,self._ctx) + la_ = self._interp.adaptivePredict(self._input,109,self._ctx) if la_ == 1: - self.state = 854 + self.state = 846 self.floatingLiteral() pass elif la_ == 2: - self.state = 855 + self.state = 847 self.match(HogQLParser.OCTAL_LITERAL) pass elif la_ == 3: - self.state = 856 + self.state = 848 self.match(HogQLParser.DECIMAL_LITERAL) pass elif la_ == 4: - self.state = 857 + self.state = 849 self.match(HogQLParser.HEXADECIMAL_LITERAL) pass elif la_ == 5: - self.state = 858 + self.state = 850 self.match(HogQLParser.INF) pass elif la_ == 6: - self.state = 859 + self.state = 851 self.match(HogQLParser.NAN_SQL) pass @@ -6701,24 +6643,24 @@ def accept(self, visitor:ParseTreeVisitor): def literal(self): localctx = HogQLParser.LiteralContext(self, self._ctx, self.state) - self.enterRule(localctx, 106, self.RULE_literal) + self.enterRule(localctx, 104, self.RULE_literal) try: - self.state = 865 + self.state = 857 self._errHandler.sync(self) token = self._input.LA(1) if token in [81, 112, 194, 195, 196, 197, 207, 209, 222]: self.enterOuterAlt(localctx, 1) - self.state = 862 + self.state = 854 self.numberLiteral() pass elif token in [198]: self.enterOuterAlt(localctx, 2) - self.state = 863 + self.state = 855 self.match(HogQLParser.STRING_LITERAL) pass elif token in [115]: self.enterOuterAlt(localctx, 3) - self.state = 864 + self.state = 856 self.match(HogQLParser.NULL_SQL) pass else: @@ -6779,11 +6721,11 @@ def accept(self, visitor:ParseTreeVisitor): def interval(self): localctx = HogQLParser.IntervalContext(self, self._ctx, self.state) - self.enterRule(localctx, 108, self.RULE_interval) + self.enterRule(localctx, 106, self.RULE_interval) self._la = 0 # Token type try: self.enterOuterAlt(localctx, 1) - self.state = 867 + self.state = 859 _la = self._input.LA(1) if not(_la==36 or (((_la - 75)) & ~0x3f) == 0 and ((1 << (_la - 75)) & 72057615512764417) != 0 or (((_la - 144)) & ~0x3f) == 0 and ((1 << (_la - 144)) & 36283883716609) != 0): self._errHandler.recoverInline(self) @@ -7355,11 +7297,11 @@ def accept(self, visitor:ParseTreeVisitor): def keyword(self): localctx = HogQLParser.KeywordContext(self, self._ctx, self.state) - self.enterRule(localctx, 110, self.RULE_keyword) + self.enterRule(localctx, 108, self.RULE_keyword) self._la = 0 # Token type try: self.enterOuterAlt(localctx, 1) - self.state = 869 + self.state = 861 _la = self._input.LA(1) if not(((_la) & ~0x3f) == 0 and ((1 << _la) & -68719476740) != 0 or (((_la - 64)) & ~0x3f) == 0 and ((1 << (_la - 64)) & -2577255255640065) != 0 or (((_la - 128)) & ~0x3f) == 0 and ((1 << (_la - 128)) & -2377900603251687437) != 0): self._errHandler.recoverInline(self) @@ -7409,11 +7351,11 @@ def accept(self, visitor:ParseTreeVisitor): def keywordForAlias(self): localctx = HogQLParser.KeywordForAliasContext(self, self._ctx, self.state) - self.enterRule(localctx, 112, self.RULE_keywordForAlias) + self.enterRule(localctx, 110, self.RULE_keywordForAlias) self._la = 0 # Token type try: self.enterOuterAlt(localctx, 1) - self.state = 871 + self.state = 863 _la = self._input.LA(1) if not((((_la - 35)) & ~0x3f) == 0 and ((1 << (_la - 35)) & 36030996109328385) != 0): self._errHandler.recoverInline(self) @@ -7458,19 +7400,19 @@ def accept(self, visitor:ParseTreeVisitor): def alias(self): localctx = HogQLParser.AliasContext(self, self._ctx, self.state) - self.enterRule(localctx, 114, self.RULE_alias) + self.enterRule(localctx, 112, self.RULE_alias) try: - self.state = 875 + self.state = 867 self._errHandler.sync(self) token = self._input.LA(1) if token in [193]: self.enterOuterAlt(localctx, 1) - self.state = 873 + self.state = 865 self.match(HogQLParser.IDENTIFIER) pass elif token in [35, 61, 76, 90]: self.enterOuterAlt(localctx, 2) - self.state = 874 + self.state = 866 self.keywordForAlias() pass else: @@ -7518,24 +7460,24 @@ def accept(self, visitor:ParseTreeVisitor): def identifier(self): localctx = HogQLParser.IdentifierContext(self, self._ctx, self.state) - self.enterRule(localctx, 116, self.RULE_identifier) + self.enterRule(localctx, 114, self.RULE_identifier) try: - self.state = 880 + self.state = 872 self._errHandler.sync(self) token = self._input.LA(1) if token in [193]: self.enterOuterAlt(localctx, 1) - self.state = 877 + self.state = 869 self.match(HogQLParser.IDENTIFIER) pass elif token in [36, 75, 107, 109, 131, 144, 184, 189]: self.enterOuterAlt(localctx, 2) - self.state = 878 + self.state = 870 self.interval() pass elif token in [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 76, 77, 78, 79, 80, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 108, 110, 111, 113, 114, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 185, 186, 187, 188, 190, 191]: self.enterOuterAlt(localctx, 3) - self.state = 879 + self.state = 871 self.keyword() pass else: @@ -7579,19 +7521,19 @@ def accept(self, visitor:ParseTreeVisitor): def identifierOrNull(self): localctx = HogQLParser.IdentifierOrNullContext(self, self._ctx, self.state) - self.enterRule(localctx, 118, self.RULE_identifierOrNull) + self.enterRule(localctx, 116, self.RULE_identifierOrNull) try: - self.state = 884 + self.state = 876 self._errHandler.sync(self) token = self._input.LA(1) if token in [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 113, 114, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 193]: self.enterOuterAlt(localctx, 1) - self.state = 882 + self.state = 874 self.identifier() pass elif token in [115]: self.enterOuterAlt(localctx, 2) - self.state = 883 + self.state = 875 self.match(HogQLParser.NULL_SQL) pass else: @@ -7638,14 +7580,14 @@ def accept(self, visitor:ParseTreeVisitor): def enumValue(self): localctx = HogQLParser.EnumValueContext(self, self._ctx, self.state) - self.enterRule(localctx, 120, self.RULE_enumValue) + self.enterRule(localctx, 118, self.RULE_enumValue) try: self.enterOuterAlt(localctx, 1) - self.state = 886 + self.state = 878 self.match(HogQLParser.STRING_LITERAL) - self.state = 887 + self.state = 879 self.match(HogQLParser.EQ_SINGLE) - self.state = 888 + self.state = 880 self.numberLiteral() except RecognitionException as re: localctx.exception = re @@ -7660,9 +7602,9 @@ def enumValue(self): def sempred(self, localctx:RuleContext, ruleIndex:int, predIndex:int): if self._predicates == None: self._predicates = dict() - self._predicates[18] = self.joinExpr_sempred - self._predicates[39] = self.columnExpr_sempred - self._predicates[45] = self.tableExpr_sempred + self._predicates[17] = self.joinExpr_sempred + self._predicates[38] = self.columnExpr_sempred + self._predicates[44] = self.tableExpr_sempred pred = self._predicates.get(ruleIndex, None) if pred is None: raise Exception("No predicate with index:" + str(ruleIndex)) diff --git a/posthog/hogql/grammar/HogQLParserVisitor.py b/posthog/hogql/grammar/HogQLParserVisitor.py index dbaf69ef6fda9..b5bc53c977fc1 100644 --- a/posthog/hogql/grammar/HogQLParserVisitor.py +++ b/posthog/hogql/grammar/HogQLParserVisitor.py @@ -84,11 +84,6 @@ def visitProjectionOrderByClause(self, ctx:HogQLParser.ProjectionOrderByClauseCo return self.visitChildren(ctx) - # Visit a parse tree produced by HogQLParser#limitByClause. - def visitLimitByClause(self, ctx:HogQLParser.LimitByClauseContext): - return self.visitChildren(ctx) - - # Visit a parse tree produced by HogQLParser#limitClause. def visitLimitClause(self, ctx:HogQLParser.LimitClauseContext): return self.visitChildren(ctx) diff --git a/posthog/hogql/parser.py b/posthog/hogql/parser.py index f393769960196..a8347a36a0035 100644 --- a/posthog/hogql/parser.py +++ b/posthog/hogql/parser.py @@ -66,17 +66,17 @@ def visitSelectStmt(self, ctx: HogQLParser.SelectStmtContext): order_by=self.visit(ctx.orderByClause()) if ctx.orderByClause() else None, ) - any_limit_clause = ctx.limitClause() or ctx.limitByClause() - if any_limit_clause and any_limit_clause.limitExpr(): - limit_expr = any_limit_clause.limitExpr() + if ctx.limitClause(): + limit_clause = ctx.limitClause() + limit_expr = limit_clause.limitExpr() if limit_expr.columnExpr(0): select_query.limit = self.visit(limit_expr.columnExpr(0)) if limit_expr.columnExpr(1): select_query.offset = self.visit(limit_expr.columnExpr(1)) - if ctx.limitClause() and ctx.limitClause().WITH() and ctx.limitClause().TIES(): - select_query.limit_with_ties = True - if ctx.limitByClause() and ctx.limitByClause().columnExprList(): - select_query.limit_by = self.visit(ctx.limitByClause().columnExprList()) + if limit_clause.columnExprList(): + select_query.limit_by = self.visit(limit_clause.columnExprList()) + if limit_clause.WITH() and limit_clause.TIES(): + select_query.limit_with_ties = True if ctx.withClause(): raise NotImplementedError(f"Unsupported: SelectStmt.withClause()") @@ -124,9 +124,6 @@ def visitOrderByClause(self, ctx: HogQLParser.OrderByClauseContext): def visitProjectionOrderByClause(self, ctx: HogQLParser.ProjectionOrderByClauseContext): raise NotImplementedError(f"Unsupported node: ProjectionOrderByClause") - def visitLimitByClause(self, ctx: HogQLParser.LimitByClauseContext): - raise Exception(f"Parsed as part of SelectStmt, can't parse directly.") - def visitLimitClause(self, ctx: HogQLParser.LimitClauseContext): raise Exception(f"Parsed as part of SelectStmt, can't parse directly.") diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index e2ebe83dc4d23..65eb95b928eba 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -88,7 +88,7 @@ def print_ast( f"ORDER BY {', '.join(order_by)}" if order_by and len(order_by) > 0 else None, ] if limit is not None: - clauses.append(f"LIMIT {print_ast(limit, stack, context, dialect)}"), + clauses.append(f"LIMIT {print_ast(limit, stack, context, dialect)}") if node.offset is not None: clauses.append(f"OFFSET {print_ast(node.offset, stack, context, dialect)}") if node.limit_by is not None: From 9cd96bccedc8ae501140385b8233ea5fabd6ac51 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 8 Feb 2023 15:27:10 +0000 Subject: [PATCH 07/81] Update snapshots --- .../queries/trends/test/__snapshots__/test_formula.ambr | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/posthog/queries/trends/test/__snapshots__/test_formula.ambr b/posthog/queries/trends/test/__snapshots__/test_formula.ambr index 3ce2fa339fefa..6828f1b8f59cd 100644 --- a/posthog/queries/trends/test/__snapshots__/test_formula.ambr +++ b/posthog/queries/trends/test/__snapshots__/test_formula.ambr @@ -153,7 +153,7 @@ CROSS JOIN (SELECT breakdown_value FROM - (SELECT [36, 0] as breakdown_value) ARRAY + (SELECT [37, 0] as breakdown_value) ARRAY JOIN breakdown_value) as sec ORDER BY breakdown_value, day_start @@ -163,7 +163,7 @@ FROM events e INNER JOIN (SELECT distinct_id, - 36 as value + 37 as value FROM (SELECT distinct_id, argMax(person_id, version) as person_id @@ -222,7 +222,7 @@ CROSS JOIN (SELECT breakdown_value FROM - (SELECT [36, 0] as breakdown_value) ARRAY + (SELECT [37, 0] as breakdown_value) ARRAY JOIN breakdown_value) as sec ORDER BY breakdown_value, day_start @@ -232,7 +232,7 @@ FROM events e INNER JOIN (SELECT distinct_id, - 36 as value + 37 as value FROM (SELECT distinct_id, argMax(person_id, version) as person_id From 2d0d1e84605e5578bb665b86380d7b3bde31a331 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 8 Feb 2023 21:45:12 +0100 Subject: [PATCH 08/81] fix placeholders --- posthog/hogql/hogql.py | 6 +++--- posthog/hogql/parser.py | 11 ++++++++--- posthog/hogql/placeholders.py | 9 +++++++++ posthog/hogql/test/test_placeholders.py | 17 ++++++++++++++++- 4 files changed, 36 insertions(+), 7 deletions(-) diff --git a/posthog/hogql/hogql.py b/posthog/hogql/hogql.py index dc16b3cf8e9f6..8e216b1654b46 100644 --- a/posthog/hogql/hogql.py +++ b/posthog/hogql/hogql.py @@ -6,15 +6,15 @@ def translate_hogql(query: str, context: HogQLContext, dialect: Literal["hogql", "clickhouse"] = "clickhouse") -> str: - """Translate a HogQL expression into a Clickhouse expression.""" + """Translate a HogQL expression into a Clickhouse expression. Raises if any placeholders found.""" if query == "": raise ValueError("Empty query") try: if context.select_team_id: - node = parse_select(query) + node = parse_select(query, no_placeholders=True) else: - node = parse_expr(query) + node = parse_expr(query, no_placeholders=True) except SyntaxError as err: raise ValueError(f"SyntaxError: {err.msg}") except NotImplementedError as err: diff --git a/posthog/hogql/parser.py b/posthog/hogql/parser.py index a8347a36a0035..a2646a16d6973 100644 --- a/posthog/hogql/parser.py +++ b/posthog/hogql/parser.py @@ -7,22 +7,27 @@ from posthog.hogql.grammar.HogQLLexer import HogQLLexer from posthog.hogql.grammar.HogQLParser import HogQLParser from posthog.hogql.parse_string import parse_string, parse_string_literal -from posthog.hogql.placeholders import replace_placeholders +from posthog.hogql.placeholders import assert_no_placeholders, replace_placeholders -def parse_expr(expr: str, placeholders: Optional[Dict[str, ast.Expr]] = None) -> ast.Expr: +def parse_expr(expr: str, placeholders: Optional[Dict[str, ast.Expr]] = None, no_placeholders=False) -> ast.Expr: parse_tree = get_parser(expr).expr() node = HogQLParseTreeConverter().visit(parse_tree) if placeholders: return replace_placeholders(node, placeholders) + elif no_placeholders: + assert_no_placeholders(node) + return node -def parse_select(statement: str, placeholders: Optional[Dict[str, ast.Expr]] = None) -> ast.Expr: +def parse_select(statement: str, placeholders: Optional[Dict[str, ast.Expr]] = None, no_placeholders=False) -> ast.Expr: parse_tree = get_parser(statement).select() node = HogQLParseTreeConverter().visit(parse_tree) if placeholders: node = replace_placeholders(node, placeholders) + elif no_placeholders: + assert_no_placeholders(node) return node diff --git a/posthog/hogql/placeholders.py b/posthog/hogql/placeholders.py index a3b543e2a0804..0399f6dc2d498 100644 --- a/posthog/hogql/placeholders.py +++ b/posthog/hogql/placeholders.py @@ -16,3 +16,12 @@ def visit_placeholder(self, node): if node.field in self.placeholders: return self.placeholders[node.field] raise ValueError(f"Placeholder '{node.field}' not found in provided dict: {', '.join(list(self.placeholders))}") + + +def assert_no_placeholders(node: ast.Expr): + AssertNoPlaceholders().visit(node) + + +class AssertNoPlaceholders(EverythingVisitor): + def visit_placeholder(self, node): + raise ValueError(f"Placeholder '{node.field}' not allowed in this context") diff --git a/posthog/hogql/test/test_placeholders.py b/posthog/hogql/test/test_placeholders.py index 4211c00238dfc..b7a8211c2ada4 100644 --- a/posthog/hogql/test/test_placeholders.py +++ b/posthog/hogql/test/test_placeholders.py @@ -1,6 +1,6 @@ from posthog.hogql import ast from posthog.hogql.parser import parse_expr -from posthog.hogql.placeholders import replace_placeholders +from posthog.hogql.placeholders import assert_no_placeholders, replace_placeholders from posthog.test.base import BaseTest @@ -17,6 +17,15 @@ def test_replace_placeholders_simple(self): ast.Constant(value="bar"), ) + def test_replace_placeholders_error(self): + expr = ast.Placeholder(field="foo") + with self.assertRaises(ValueError) as context: + replace_placeholders(expr, {}) + self.assertTrue("Placeholder 'foo' not found in provided dict:" in str(context.exception)) + with self.assertRaises(ValueError) as context: + replace_placeholders(expr, {"bar": ast.Constant(value=123)}) + self.assertTrue("Placeholder 'foo' not found in provided dict: bar" in str(context.exception)) + def test_replace_placeholders_comparison(self): expr = parse_expr("timestamp < {timestamp}") self.assertEqual( @@ -36,3 +45,9 @@ def test_replace_placeholders_comparison(self): right=ast.Constant(value=123), ), ) + + def test_assert_no_placeholders(self): + expr = ast.Placeholder(field="foo") + with self.assertRaises(ValueError) as context: + assert_no_placeholders(expr) + self.assertTrue("Placeholder 'foo' not allowed in this context" in str(context.exception)) From 5064dad837469816390b2d18286aa57ecd7f63e9 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 9 Feb 2023 11:04:13 +0100 Subject: [PATCH 09/81] resolve symbols for events table --- posthog/hogql/ast.py | 75 ++++++++++++++++- posthog/hogql/parser.py | 4 +- posthog/hogql/placeholders.py | 6 +- posthog/hogql/resolver.py | 122 ++++++++++++++++++++++++++++ posthog/hogql/test/test_parser.py | 14 ++++ posthog/hogql/test/test_resolver.py | 79 ++++++++++++++++++ posthog/hogql/test/test_visitor.py | 6 +- posthog/hogql/visitor.py | 76 ++++++++++++++++- 8 files changed, 368 insertions(+), 14 deletions(-) create mode 100644 posthog/hogql/resolver.py create mode 100644 posthog/hogql/test/test_resolver.py diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index 012a9b9a8b57a..76cea55d685ad 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -1,10 +1,12 @@ import re from enum import Enum -from typing import Any, List, Literal, Optional, Union +from typing import Any, Dict, List, Literal, Optional, Union from pydantic import BaseModel, Extra -# NOTE: when you add new AST fields or nodes, add them to EverythingVisitor as well! +from posthog.hogql.constants import EVENT_FIELDS + +# NOTE: when you add new AST fields or nodes, add them to CloningVisitor as well! camel_case_pattern = re.compile(r"(? "Symbol": + raise NotImplementedError() + + def has_child(self, name: str) -> bool: + return self.get_child(name) is not None + + +class AliasSymbol(Symbol): + expr: "Expr" + + def get_child(self, name: str) -> "Symbol": + if isinstance(self.expr, SelectQuery): + return self.expr.symbol.get_child(name) + elif isinstance(self.expr, Field): + return self.expr.symbol.get_child(name) + + +class TableSymbol(Symbol): + table_name: Literal["events"] + + def has_child(self, name: str) -> bool: + if self.table_name == "events": + return name in EVENT_FIELDS + else: + raise NotImplementedError(f"Can not resolve table: {self.name}") + + def get_child(self, name: str) -> "Symbol": + if self.table_name == "events": + if name in EVENT_FIELDS: + return FieldSymbol(name=name, table=self) + else: + raise NotImplementedError(f"Can not resolve table: {self.name}") + + +class SelectQuerySymbol(Symbol): + # expr: "Expr" + + symbols: Dict[str, Symbol] + tables: Dict[str, Symbol] + + +class FieldSymbol(Symbol): + table: TableSymbol + + def get_child(self, name: str) -> "Symbol": + if self.table.table_name == "events": + if self.name == "properties": + raise NotImplementedError(f"Property symbol resolution not implemented yet") + else: + raise NotImplementedError(f"Can not resolve field {self.name} on table events") + else: + raise NotImplementedError(f"Can not resolve fields on table: {self.name}") + self.table.get_child(name) + + +class PropertySymbol(Symbol): + field: FieldSymbol + + class Expr(AST): - pass + symbol: Optional[Symbol] + + +Symbol.update_forward_refs(Expr=Expr) +AliasSymbol.update_forward_refs(Expr=Expr) +SelectQuerySymbol.update_forward_refs(Expr=Expr) class Alias(Expr): diff --git a/posthog/hogql/parser.py b/posthog/hogql/parser.py index a2646a16d6973..0f2424bb6d9cb 100644 --- a/posthog/hogql/parser.py +++ b/posthog/hogql/parser.py @@ -21,7 +21,9 @@ def parse_expr(expr: str, placeholders: Optional[Dict[str, ast.Expr]] = None, no return node -def parse_select(statement: str, placeholders: Optional[Dict[str, ast.Expr]] = None, no_placeholders=False) -> ast.Expr: +def parse_select( + statement: str, placeholders: Optional[Dict[str, ast.Expr]] = None, no_placeholders=False +) -> ast.SelectQuery: parse_tree = get_parser(statement).select() node = HogQLParseTreeConverter().visit(parse_tree) if placeholders: diff --git a/posthog/hogql/placeholders.py b/posthog/hogql/placeholders.py index 0399f6dc2d498..6d9eb5b017c4a 100644 --- a/posthog/hogql/placeholders.py +++ b/posthog/hogql/placeholders.py @@ -1,14 +1,14 @@ from typing import Dict from posthog.hogql import ast -from posthog.hogql.visitor import EverythingVisitor +from posthog.hogql.visitor import CloningVisitor def replace_placeholders(node: ast.Expr, placeholders: Dict[str, ast.Expr]) -> ast.Expr: return ReplacePlaceholders(placeholders).visit(node) -class ReplacePlaceholders(EverythingVisitor): +class ReplacePlaceholders(CloningVisitor): def __init__(self, placeholders: Dict[str, ast.Expr]): self.placeholders = placeholders @@ -22,6 +22,6 @@ def assert_no_placeholders(node: ast.Expr): AssertNoPlaceholders().visit(node) -class AssertNoPlaceholders(EverythingVisitor): +class AssertNoPlaceholders(CloningVisitor): def visit_placeholder(self, node): raise ValueError(f"Placeholder '{node.field}' not allowed in this context") diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py new file mode 100644 index 0000000000000..649b7bb6df102 --- /dev/null +++ b/posthog/hogql/resolver.py @@ -0,0 +1,122 @@ +from typing import List, Optional + +from posthog.hogql import ast +from posthog.hogql.visitor import TraversingVisitor + + +def resolve_symbols(node: ast.SelectQuery): + Resolver().visit(node) + + +class ResolverException(ValueError): + pass + + +class Resolver(TraversingVisitor): + def __init__(self): + self.scopes: List[ast.SelectQuerySymbol] = [] + + def visit_alias(self, node: ast.Alias): + if node.symbol is not None: + return + + if len(self.scopes) == 0: + raise ResolverException("Aliases are allowed only within SELECT queries") + last_select = self.scopes[-1] + if node.alias in last_select.symbols: + raise ResolverException(f"Found multiple expressions with the same alias: {node.alias}") + if node.alias == "": + raise ResolverException("Alias cannot be empty") + + symbol = ast.AliasSymbol(name=node.alias, expr=node.expr) + last_select.symbols[node.alias] = symbol + self.visit(node.expr) + + def visit_field(self, node): + if node.symbol is not None: + return + if len(node.chain) == 0: + raise Exception("Invalid field access with empty chain") + + # resolve the first part of the chain + name = node.chain[0] + symbol: Optional[ast.Symbol] = None + for scope in reversed(self.scopes): + if name in scope.tables and len(node.chain) > 1: + symbol = scope.tables[name] + break + elif name in scope.symbols: + symbol = scope.symbols[name] + break + else: + fields_on_tables_in_scope = [table for table in scope.tables.values() if table.has_child(name)] + if len(fields_on_tables_in_scope) > 1: + raise ResolverException( + f"Found multiple joined tables with field \"{name}\": {', '.join([symbol.name for symbol in fields_on_tables_in_scope])}. Please specify which table you're selecting from." + ) + elif len(fields_on_tables_in_scope) == 1: + symbol = fields_on_tables_in_scope[0].get_child(name) + break + + if not symbol: + raise ResolverException(f'Cannot resolve symbol: "{name}"') + + # recursively resolve the rest of the chain + for name in node.chain[1:]: + symbol = symbol.get_child(name) + if symbol is None: + raise ResolverException(f"Cannot resolve symbol {', '.join(node.chain)}. Unable to resolve {name}") + + node.symbol = symbol + + def visit_join_expr(self, node): + if node.symbol is not None: + return + if len(self.scopes) == 0: + raise ResolverException("Unexpected JoinExpr outside a SELECT query") + last_select = self.scopes[-1] + if node.alias in last_select.tables: + raise ResolverException(f"Table alias with the same name as another table: {node.alias}") + + if isinstance(node.table, ast.Field): + if node.table.chain == ["events"]: + if node.alias is None: + node.alias = node.table.chain[0] + symbol = ast.TableSymbol(name=node.alias, table_name="events") + else: + raise ResolverException(f"Cannot resolve table {node.table.chain[0]}") + + elif isinstance(node.table, ast.SelectQuery): + symbol = self.visit(node.table) + symbol.name = node.alias + + else: + raise ResolverException(f"JoinExpr with table of type {type(node.table).__name__} not supported") + + node.table.symbol = symbol + last_select.tables[node.alias] = symbol + + self.visit(node.join_expr) + + def visit_select_query(self, node): + if node.symbol is not None: + return + + node.symbol = ast.SelectQuerySymbol(name="", symbols={}, tables={}) + self.scopes.append(node.symbol) + + if node.select_from: + self.visit(node.select_from) + if node.select: + for expr in node.select: + self.visit(expr) + if node.where: + self.visit(node.where) + if node.prewhere: + self.visit(node.prewhere) + if node.having: + self.visit(node.having) + + self.scopes.pop() + + return node.symbol diff --git a/posthog/hogql/test/test_parser.py b/posthog/hogql/test/test_parser.py index 55d943a1d197e..0313923c96c18 100644 --- a/posthog/hogql/test/test_parser.py +++ b/posthog/hogql/test/test_parser.py @@ -443,6 +443,13 @@ def test_select_from(self): select_from=ast.JoinExpr(table=ast.Field(chain=["events"]), alias="e"), ), ) + self.assertEqual( + parse_select("select 1 from events e"), + ast.SelectQuery( + select=[ast.Constant(value=1)], + select_from=ast.JoinExpr(table=ast.Field(chain=["events"]), alias="e"), + ), + ) self.assertEqual( parse_select("select 1 from complex.table"), ast.SelectQuery( @@ -457,6 +464,13 @@ def test_select_from(self): select_from=ast.JoinExpr(table=ast.Field(chain=["complex", "table"]), alias="a"), ), ) + self.assertEqual( + parse_select("select 1 from complex.table a"), + ast.SelectQuery( + select=[ast.Constant(value=1)], + select_from=ast.JoinExpr(table=ast.Field(chain=["complex", "table"]), alias="a"), + ), + ) self.assertEqual( parse_select("select 1 from (select 1 from events)"), ast.SelectQuery( diff --git a/posthog/hogql/test/test_resolver.py b/posthog/hogql/test/test_resolver.py new file mode 100644 index 0000000000000..19f05523a698d --- /dev/null +++ b/posthog/hogql/test/test_resolver.py @@ -0,0 +1,79 @@ +from posthog.hogql import ast +from posthog.hogql.parser import parse_select +from posthog.hogql.resolver import resolve_symbols +from posthog.test.base import BaseTest + + +class TestResolver(BaseTest): + def test_resolve_events_table(self): + expr = parse_select("SELECT event, events.timestamp FROM events WHERE events.event = 'test'") + resolve_symbols(expr) + + events_table_symbol = ast.TableSymbol(name="events", table_name="events") + event_field_symbol = ast.FieldSymbol(name="event", table=events_table_symbol) + timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=events_table_symbol) + select_query_symbol = ast.SelectQuerySymbol( + name="", + symbols={}, + tables={"events": events_table_symbol}, + ) + + self.assertEqual( + expr, + ast.SelectQuery( + select=[ + ast.Field(chain=["event"], symbol=event_field_symbol), + ast.Field(chain=["events", "timestamp"], symbol=timestamp_field_symbol), + ], + select_from=ast.JoinExpr( + table=ast.Field(chain=["events"], symbol=events_table_symbol), + alias="events", + ), + where=ast.CompareOperation( + left=ast.Field(chain=["events", "event"], symbol=event_field_symbol), + op=ast.CompareOperationType.Eq, + right=ast.Constant(value="test"), + ), + symbol=select_query_symbol, + ), + ) + + def test_resolve_events_table_alias(self): + expr = parse_select("SELECT event, e.timestamp FROM events e WHERE e.event = 'test'") + resolve_symbols(expr) + + events_table_symbol = ast.TableSymbol(name="e", table_name="events") + event_field_symbol = ast.FieldSymbol(name="event", table=events_table_symbol) + timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=events_table_symbol) + select_query_symbol = ast.SelectQuerySymbol( + name="", + symbols={}, + tables={"e": events_table_symbol}, + ) + + self.assertEqual( + expr, + ast.SelectQuery( + select=[ + ast.Field(chain=["event"], symbol=event_field_symbol), + ast.Field(chain=["e", "timestamp"], symbol=timestamp_field_symbol), + ], + select_from=ast.JoinExpr( + table=ast.Field(chain=["events"], symbol=events_table_symbol), + alias="e", + ), + where=ast.CompareOperation( + left=ast.Field(chain=["e", "event"], symbol=event_field_symbol), + op=ast.CompareOperationType.Eq, + right=ast.Constant(value="test"), + ), + symbol=select_query_symbol, + ), + ) + + +# "with 2 as a select 1 as a" -> "Different expressions with the same alias a:" +# "with 2 as b, 3 as c select (select 1 as b) as a, b, c" -> "Different expressions with the same alias b:" + + +# "select a, b, e.c from (select 1 as a, 2 as b, 3 as c) as e" -> 1, 2, 3 diff --git a/posthog/hogql/test/test_visitor.py b/posthog/hogql/test/test_visitor.py index 738b4ac3aed12..eedf80e367fc5 100644 --- a/posthog/hogql/test/test_visitor.py +++ b/posthog/hogql/test/test_visitor.py @@ -1,10 +1,10 @@ from posthog.hogql import ast from posthog.hogql.parser import parse_expr -from posthog.hogql.visitor import EverythingVisitor +from posthog.hogql.visitor import CloningVisitor from posthog.test.base import BaseTest -class ConstantVisitor(EverythingVisitor): +class ConstantVisitor(CloningVisitor): def __init__(self): self.constants = [] self.fields = [] @@ -93,4 +93,4 @@ def test_everything_visitor(self): ), ] ) - self.assertEqual(node, EverythingVisitor().visit(node)) + self.assertEqual(node, CloningVisitor().visit(node)) diff --git a/posthog/hogql/visitor.py b/posthog/hogql/visitor.py index 66365f368bbc1..fce103f70e28f 100644 --- a/posthog/hogql/visitor.py +++ b/posthog/hogql/visitor.py @@ -8,7 +8,75 @@ def visit(self, node: ast.AST): return node.accept(self) -class EverythingVisitor(Visitor): +class TraversingVisitor(Visitor): + """Visitor that traverses the AST tree without returning anything""" + + def visit_expr(self, node: ast.Expr): + raise ValueError("Can not visit generic Expr node") + + def visit_alias(self, node: ast.Alias): + self.visit(node.expr) + + def visit_binary_operation(self, node: ast.BinaryOperation): + self.visit(node.left) + self.visit(node.right) + + def visit_and(self, node: ast.And): + for expr in node.exprs: + self.visit(expr) + + def visit_or(self, node: ast.Or): + for expr in node.exprs: + self.visit(expr) + + def visit_compare_operation(self, node: ast.CompareOperation): + self.visit(node.left) + self.visit(node.right) + + def visit_not(self, node: ast.Not): + self.visit(node.expr) + + def visit_order_expr(self, node: ast.OrderExpr): + self.visit(node.expr) + + def visit_constant(self, node: ast.Constant): + pass + + def visit_field(self, node: ast.Field): + pass + + def visit_placeholder(self, node: ast.Placeholder): + pass + + def visit_call(self, node: ast.Call): + for expr in node.args: + self.visit(expr) + + def visit_join_expr(self, node: ast.JoinExpr): + self.visit(node.table) + self.visit(node.join_expr) + self.visit(node.join_constraint) + + def visit_select_query(self, node: ast.SelectQuery): + self.visit(node.select_from) + for expr in node.select or []: + self.visit(expr) + self.visit(node.where) + self.visit(node.prewhere) + self.visit(node.having) + for expr in node.group_by or []: + self.visit(expr) + for expr in node.order_by or []: + self.visit(expr) + for expr in node.limit_by or []: + self.visit(expr) + self.visit(node.limit), + self.visit(node.offset), + + +class CloningVisitor(Visitor): + """Visitor that traverses and clones the AST tree""" + def visit_expr(self, node: ast.Expr): raise ValueError("Can not visit generic Expr node") @@ -56,10 +124,10 @@ def visit_field(self, node: ast.Field): def visit_placeholder(self, node: ast.Placeholder): return node - def visit_call(self, call: ast.Call): + def visit_call(self, node: ast.Call): return ast.Call( - name=call.name, - args=[self.visit(arg) for arg in call.args], + name=node.name, + args=[self.visit(arg) for arg in node.args], ) def visit_join_expr(self, node: ast.JoinExpr): From 62dd464a6a5f6016c9a6786b2ad6357e4173c0e5 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 9 Feb 2023 11:55:31 +0100 Subject: [PATCH 10/81] resolve aliases --- posthog/hogql/ast.py | 20 ++-- posthog/hogql/resolver.py | 6 +- posthog/hogql/test/test_resolver.py | 149 ++++++++++++++++++++++++++++ 3 files changed, 163 insertions(+), 12 deletions(-) diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index 76cea55d685ad..0226bc3538e11 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -33,13 +33,7 @@ def has_child(self, name: str) -> bool: class AliasSymbol(Symbol): - expr: "Expr" - - def get_child(self, name: str) -> "Symbol": - if isinstance(self.expr, SelectQuery): - return self.expr.symbol.get_child(name) - elif isinstance(self.expr, Field): - return self.expr.symbol.get_child(name) + symbol: "Symbol" class TableSymbol(Symbol): @@ -60,11 +54,18 @@ def get_child(self, name: str) -> "Symbol": class SelectQuerySymbol(Symbol): - # expr: "Expr" - symbols: Dict[str, Symbol] tables: Dict[str, Symbol] + def get_child(self, name: str) -> "Symbol": + if name in self.symbols: + return self.symbols[name] + if name in self.tables: + return self.tables[name] + + def has_child(self, name: str) -> bool: + return name in self.symbols or name in self.tables + class FieldSymbol(Symbol): table: TableSymbol @@ -77,7 +78,6 @@ def get_child(self, name: str) -> "Symbol": raise NotImplementedError(f"Can not resolve field {self.name} on table events") else: raise NotImplementedError(f"Can not resolve fields on table: {self.name}") - self.table.get_child(name) class PropertySymbol(Symbol): diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py index 649b7bb6df102..d3ee3d71ef2fa 100644 --- a/posthog/hogql/resolver.py +++ b/posthog/hogql/resolver.py @@ -28,10 +28,11 @@ def visit_alias(self, node: ast.Alias): if node.alias == "": raise ResolverException("Alias cannot be empty") - symbol = ast.AliasSymbol(name=node.alias, expr=node.expr) - last_select.symbols[node.alias] = symbol self.visit(node.expr) + node.symbol = ast.AliasSymbol(name=node.alias, symbol=node.expr.symbol) + last_select.symbols[node.alias] = node.symbol + def visit_field(self, node): if node.symbol is not None: return @@ -43,6 +44,7 @@ def visit_field(self, node): symbol: Optional[ast.Symbol] = None for scope in reversed(self.scopes): if name in scope.tables and len(node.chain) > 1: + # CH assumes you're selecting a field, unless it's with a "." in the field, then check for tables symbol = scope.tables[name] break elif name in scope.symbols: diff --git a/posthog/hogql/test/test_resolver.py b/posthog/hogql/test/test_resolver.py index 19f05523a698d..7153d42f18639 100644 --- a/posthog/hogql/test/test_resolver.py +++ b/posthog/hogql/test/test_resolver.py @@ -71,6 +71,155 @@ def test_resolve_events_table_alias(self): ), ) + def test_resolve_events_table_column_alias(self): + expr = parse_select("SELECT event as ee, ee, ee as e, e.timestamp FROM events e WHERE e.event = 'test'") + resolve_symbols(expr) + + events_table_symbol = ast.TableSymbol(name="e", table_name="events") + event_field_symbol = ast.FieldSymbol(name="event", table=events_table_symbol) + timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=events_table_symbol) + select_query_symbol = ast.SelectQuerySymbol( + name="", + symbols={ + "ee": ast.AliasSymbol(name="ee", symbol=event_field_symbol), + "e": ast.AliasSymbol(name="e", symbol=ast.AliasSymbol(name="ee", symbol=event_field_symbol)), + }, + tables={"e": events_table_symbol}, + ) + + expected_query = ast.SelectQuery( + select=[ + ast.Alias( + alias="ee", + expr=ast.Field(chain=["event"], symbol=event_field_symbol), + symbol=ast.AliasSymbol(name="ee", symbol=event_field_symbol), + ), + ast.Field(chain=["ee"], symbol=ast.AliasSymbol(name="ee", symbol=event_field_symbol)), + ast.Alias( + alias="e", + expr=ast.Field(chain=["ee"], symbol=ast.AliasSymbol(name="ee", symbol=event_field_symbol)), + symbol=ast.AliasSymbol(name="e", symbol=ast.AliasSymbol(name="ee", symbol=event_field_symbol)), + ), + ast.Field(chain=["e", "timestamp"], symbol=timestamp_field_symbol), + ], + select_from=ast.JoinExpr( + table=ast.Field(chain=["events"], symbol=events_table_symbol), + alias="e", + ), + where=ast.CompareOperation( + left=ast.Field(chain=["e", "event"], symbol=event_field_symbol), + op=ast.CompareOperationType.Eq, + right=ast.Constant(value="test"), + ), + symbol=select_query_symbol, + ) + # asserting individually to help debug if something is off + self.assertEqual(expr.select, expected_query.select) + self.assertEqual(expr.select_from, expected_query.select_from) + self.assertEqual(expr.where, expected_query.where) + self.assertEqual(expr.symbol, expected_query.symbol) + self.assertEqual(expr, expected_query) + + def test_resolve_events_table_column_alias_inside_subquery(self): + expr = parse_select("SELECT b FROM (select event as b, timestamp as c from events) e WHERE e.b = 'test'") + resolve_symbols(expr) + events_table_symbol = ast.TableSymbol(name="events", table_name="events") + event_field_symbol = ast.FieldSymbol(name="event", table=events_table_symbol) + timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=events_table_symbol) + expected_query = ast.SelectQuery( + select=[ + ast.Field( + chain=["b"], + symbol=ast.AliasSymbol( + name="b", + symbol=event_field_symbol, + ), + ), + ], + select_from=ast.JoinExpr( + table=ast.SelectQuery( + select=[ + ast.Alias( + alias="b", + expr=ast.Field(chain=["event"], symbol=event_field_symbol), + symbol=ast.AliasSymbol( + name="b", + symbol=event_field_symbol, + ), + ), + ast.Alias( + alias="c", + expr=ast.Field(chain=["timestamp"], symbol=timestamp_field_symbol), + symbol=ast.AliasSymbol( + name="c", + symbol=timestamp_field_symbol, + ), + ), + ], + select_from=ast.JoinExpr( + table=ast.Field(chain=["events"], symbol=events_table_symbol), + alias="events", + ), + symbol=ast.SelectQuerySymbol( + name="e", + symbols={ + "b": ast.AliasSymbol( + name="b", + symbol=event_field_symbol, + ), + "c": ast.AliasSymbol( + name="c", + symbol=timestamp_field_symbol, + ), + }, + tables={ + "events": events_table_symbol, + }, + ), + ), + alias="e", + ), + where=ast.CompareOperation( + left=ast.Field( + chain=["e", "b"], + symbol=ast.AliasSymbol( + name="b", + symbol=event_field_symbol, + ), + ), + op=ast.CompareOperationType.Eq, + right=ast.Constant(value="test"), + ), + symbol=ast.SelectQuerySymbol( + name="", + symbols={}, + tables={ + "e": ast.SelectQuerySymbol( + name="e", + symbols={ + "b": ast.AliasSymbol( + name="b", + symbol=event_field_symbol, + ), + "c": ast.AliasSymbol( + name="c", + symbol=timestamp_field_symbol, + ), + }, + tables={ + "events": events_table_symbol, + }, + ) + }, + ), + ) + # asserting individually to help debug if something is off + self.assertEqual(expr.select, expected_query.select) + self.assertEqual(expr.select_from, expected_query.select_from) + self.assertEqual(expr.where, expected_query.where) + self.assertEqual(expr.symbol, expected_query.symbol) + self.assertEqual(expr, expected_query) + # "with 2 as a select 1 as a" -> "Different expressions with the same alias a:" # "with 2 as b, 3 as c select (select 1 as b) as a, b, c" -> "Different expressions with the same alias b:" From 6c04d20f53cc9e2e65549cb4ab5642a7c1f403e6 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 9 Feb 2023 14:19:20 +0100 Subject: [PATCH 11/81] refactor column and table aliases --- posthog/hogql/ast.py | 28 ++- posthog/hogql/parser.py | 2 +- posthog/hogql/resolver.py | 28 +-- posthog/hogql/test/test_resolver.py | 265 +++++++++++++++++++--------- 4 files changed, 223 insertions(+), 100 deletions(-) diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index 0226bc3538e11..66b8b5b12ed89 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -23,8 +23,6 @@ def accept(self, visitor): class Symbol(AST): - name: str - def get_child(self, name: str) -> "Symbol": raise NotImplementedError() @@ -32,9 +30,27 @@ def has_child(self, name: str) -> bool: return self.get_child(name) is not None -class AliasSymbol(Symbol): +class ColumnAliasSymbol(Symbol): + name: str symbol: "Symbol" + def get_child(self, name: str) -> "Symbol": + return self.symbol.get_child(name) + + def has_child(self, name: str) -> bool: + return self.symbol.has_child(name) + + +class TableAliasSymbol(Symbol): + name: str + symbol: "Symbol" + + def get_child(self, name: str) -> "Symbol": + return self.symbol.get_child(name) + + def has_child(self, name: str) -> bool: + return self.symbol.has_child(name) + class TableSymbol(Symbol): table_name: Literal["events"] @@ -68,6 +84,7 @@ def has_child(self, name: str) -> bool: class FieldSymbol(Symbol): + name: str table: TableSymbol def get_child(self, name: str) -> "Symbol": @@ -81,6 +98,7 @@ def get_child(self, name: str) -> "Symbol": class PropertySymbol(Symbol): + name: str field: FieldSymbol @@ -88,8 +106,8 @@ class Expr(AST): symbol: Optional[Symbol] -Symbol.update_forward_refs(Expr=Expr) -AliasSymbol.update_forward_refs(Expr=Expr) +ColumnAliasSymbol.update_forward_refs(Expr=Expr) +TableAliasSymbol.update_forward_refs(Expr=Expr) SelectQuerySymbol.update_forward_refs(Expr=Expr) diff --git a/posthog/hogql/parser.py b/posthog/hogql/parser.py index 0f2424bb6d9cb..2266073968cf0 100644 --- a/posthog/hogql/parser.py +++ b/posthog/hogql/parser.py @@ -303,7 +303,7 @@ def visitColumnsExprAsterisk(self, ctx: HogQLParser.ColumnsExprAsteriskContext): return ast.Field(chain=["*"]) def visitColumnsExprSubquery(self, ctx: HogQLParser.ColumnsExprSubqueryContext): - raise NotImplementedError(f"Unsupported node: ColumnsExprSubquery") + return self.visit(ctx.selectUnionStmt()) def visitColumnsExprColumn(self, ctx: HogQLParser.ColumnsExprColumnContext): return self.visit(ctx.columnExpr()) diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py index d3ee3d71ef2fa..28d150022f22f 100644 --- a/posthog/hogql/resolver.py +++ b/posthog/hogql/resolver.py @@ -3,6 +3,8 @@ from posthog.hogql import ast from posthog.hogql.visitor import TraversingVisitor +# https://github.com/ClickHouse/ClickHouse/issues/23194 - "Describe how identifiers in SELECT queries are resolved" + def resolve_symbols(node: ast.SelectQuery): Resolver().visit(node) @@ -30,7 +32,7 @@ def visit_alias(self, node: ast.Alias): self.visit(node.expr) - node.symbol = ast.AliasSymbol(name=node.alias, symbol=node.expr.symbol) + node.symbol = ast.ColumnAliasSymbol(name=node.alias, symbol=node.expr.symbol) last_select.symbols[node.alias] = node.symbol def visit_field(self, node): @@ -77,26 +79,30 @@ def visit_join_expr(self, node): if len(self.scopes) == 0: raise ResolverException("Unexpected JoinExpr outside a SELECT query") last_select = self.scopes[-1] - if node.alias in last_select.tables: - raise ResolverException(f"Table alias with the same name as another table: {node.alias}") if isinstance(node.table, ast.Field): + if node.alias is None: + node.alias = node.table.chain[0] + if node.alias in last_select.tables: + raise ResolverException(f"Table alias with the same name as another table: {node.alias}") + if node.table.chain == ["events"]: - if node.alias is None: - node.alias = node.table.chain[0] - symbol = ast.TableSymbol(name=node.alias, table_name="events") + node.table.symbol = ast.TableSymbol(table_name="events") + node.symbol = ast.TableAliasSymbol(name=node.alias, symbol=node.table.symbol) else: raise ResolverException(f"Cannot resolve table {node.table.chain[0]}") elif isinstance(node.table, ast.SelectQuery): - symbol = self.visit(node.table) - symbol.name = node.alias + node.table.symbol = self.visit(node.table) + if node.alias is None: + node.symbol = node.table.symbol + else: + node.symbol = ast.TableAliasSymbol(name=node.alias, symbol=node.table.symbol) else: raise ResolverException(f"JoinExpr with table of type {type(node.table).__name__} not supported") - node.table.symbol = symbol - last_select.tables[node.alias] = symbol + last_select.tables[node.alias] = node.table.symbol self.visit(node.join_expr) @@ -104,7 +110,7 @@ def visit_select_query(self, node): if node.symbol is not None: return - node.symbol = ast.SelectQuerySymbol(name="", symbols={}, tables={}) + node.symbol = ast.SelectQuerySymbol(symbols={}, tables={}) self.scopes.append(node.symbol) if node.select_from: diff --git a/posthog/hogql/test/test_resolver.py b/posthog/hogql/test/test_resolver.py index 7153d42f18639..8ae06e6b8a2c3 100644 --- a/posthog/hogql/test/test_resolver.py +++ b/posthog/hogql/test/test_resolver.py @@ -9,102 +9,114 @@ def test_resolve_events_table(self): expr = parse_select("SELECT event, events.timestamp FROM events WHERE events.event = 'test'") resolve_symbols(expr) - events_table_symbol = ast.TableSymbol(name="events", table_name="events") + events_table_symbol = ast.TableSymbol(table_name="events") event_field_symbol = ast.FieldSymbol(name="event", table=events_table_symbol) timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=events_table_symbol) select_query_symbol = ast.SelectQuerySymbol( - name="", symbols={}, tables={"events": events_table_symbol}, ) - self.assertEqual( - expr, - ast.SelectQuery( - select=[ - ast.Field(chain=["event"], symbol=event_field_symbol), - ast.Field(chain=["events", "timestamp"], symbol=timestamp_field_symbol), - ], - select_from=ast.JoinExpr( - table=ast.Field(chain=["events"], symbol=events_table_symbol), - alias="events", - ), - where=ast.CompareOperation( - left=ast.Field(chain=["events", "event"], symbol=event_field_symbol), - op=ast.CompareOperationType.Eq, - right=ast.Constant(value="test"), - ), - symbol=select_query_symbol, + expected = ast.SelectQuery( + select=[ + ast.Field(chain=["event"], symbol=event_field_symbol), + ast.Field(chain=["events", "timestamp"], symbol=timestamp_field_symbol), + ], + select_from=ast.JoinExpr( + table=ast.Field(chain=["events"], symbol=events_table_symbol), + alias="events", + symbol=ast.TableAliasSymbol(name="events", symbol=events_table_symbol), ), + where=ast.CompareOperation( + left=ast.Field(chain=["events", "event"], symbol=event_field_symbol), + op=ast.CompareOperationType.Eq, + right=ast.Constant(value="test"), + ), + symbol=select_query_symbol, ) + # asserting individually to help debug if something is off + self.assertEqual(expr.select, expected.select) + self.assertEqual(expr.select_from, expected.select_from) + self.assertEqual(expr.where, expected.where) + self.assertEqual(expr.symbol, expected.symbol) + self.assertEqual(expr, expected) + def test_resolve_events_table_alias(self): expr = parse_select("SELECT event, e.timestamp FROM events e WHERE e.event = 'test'") resolve_symbols(expr) - events_table_symbol = ast.TableSymbol(name="e", table_name="events") + events_table_symbol = ast.TableSymbol(table_name="events") event_field_symbol = ast.FieldSymbol(name="event", table=events_table_symbol) timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=events_table_symbol) select_query_symbol = ast.SelectQuerySymbol( - name="", symbols={}, tables={"e": events_table_symbol}, ) - self.assertEqual( - expr, - ast.SelectQuery( - select=[ - ast.Field(chain=["event"], symbol=event_field_symbol), - ast.Field(chain=["e", "timestamp"], symbol=timestamp_field_symbol), - ], - select_from=ast.JoinExpr( - table=ast.Field(chain=["events"], symbol=events_table_symbol), - alias="e", - ), - where=ast.CompareOperation( - left=ast.Field(chain=["e", "event"], symbol=event_field_symbol), - op=ast.CompareOperationType.Eq, - right=ast.Constant(value="test"), - ), - symbol=select_query_symbol, + expected = ast.SelectQuery( + select=[ + ast.Field(chain=["event"], symbol=event_field_symbol), + ast.Field(chain=["e", "timestamp"], symbol=timestamp_field_symbol), + ], + select_from=ast.JoinExpr( + table=ast.Field(chain=["events"], symbol=events_table_symbol), + alias="e", + symbol=ast.TableAliasSymbol(name="e", symbol=events_table_symbol), + ), + where=ast.CompareOperation( + left=ast.Field(chain=["e", "event"], symbol=event_field_symbol), + op=ast.CompareOperationType.Eq, + right=ast.Constant(value="test"), ), + symbol=select_query_symbol, ) + # asserting individually to help debug if something is off + self.assertEqual(expr.select, expected.select) + self.assertEqual(expr.select_from, expected.select_from) + self.assertEqual(expr.where, expected.where) + self.assertEqual(expr.symbol, expected.symbol) + self.assertEqual(expr, expected) + def test_resolve_events_table_column_alias(self): expr = parse_select("SELECT event as ee, ee, ee as e, e.timestamp FROM events e WHERE e.event = 'test'") resolve_symbols(expr) - events_table_symbol = ast.TableSymbol(name="e", table_name="events") + events_table_symbol = ast.TableSymbol(table_name="events") event_field_symbol = ast.FieldSymbol(name="event", table=events_table_symbol) timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=events_table_symbol) select_query_symbol = ast.SelectQuerySymbol( - name="", symbols={ - "ee": ast.AliasSymbol(name="ee", symbol=event_field_symbol), - "e": ast.AliasSymbol(name="e", symbol=ast.AliasSymbol(name="ee", symbol=event_field_symbol)), + "ee": ast.ColumnAliasSymbol(name="ee", symbol=event_field_symbol), + "e": ast.ColumnAliasSymbol( + name="e", symbol=ast.ColumnAliasSymbol(name="ee", symbol=event_field_symbol) + ), }, tables={"e": events_table_symbol}, ) - expected_query = ast.SelectQuery( + expected = ast.SelectQuery( select=[ ast.Alias( alias="ee", expr=ast.Field(chain=["event"], symbol=event_field_symbol), - symbol=ast.AliasSymbol(name="ee", symbol=event_field_symbol), + symbol=ast.ColumnAliasSymbol(name="ee", symbol=event_field_symbol), ), - ast.Field(chain=["ee"], symbol=ast.AliasSymbol(name="ee", symbol=event_field_symbol)), + ast.Field(chain=["ee"], symbol=ast.ColumnAliasSymbol(name="ee", symbol=event_field_symbol)), ast.Alias( alias="e", - expr=ast.Field(chain=["ee"], symbol=ast.AliasSymbol(name="ee", symbol=event_field_symbol)), - symbol=ast.AliasSymbol(name="e", symbol=ast.AliasSymbol(name="ee", symbol=event_field_symbol)), + expr=ast.Field(chain=["ee"], symbol=ast.ColumnAliasSymbol(name="ee", symbol=event_field_symbol)), + symbol=ast.ColumnAliasSymbol( + name="e", symbol=ast.ColumnAliasSymbol(name="ee", symbol=event_field_symbol) + ), ), ast.Field(chain=["e", "timestamp"], symbol=timestamp_field_symbol), ], select_from=ast.JoinExpr( table=ast.Field(chain=["events"], symbol=events_table_symbol), alias="e", + symbol=ast.TableAliasSymbol(name="e", symbol=events_table_symbol), ), where=ast.CompareOperation( left=ast.Field(chain=["e", "event"], symbol=event_field_symbol), @@ -114,23 +126,38 @@ def test_resolve_events_table_column_alias(self): symbol=select_query_symbol, ) # asserting individually to help debug if something is off - self.assertEqual(expr.select, expected_query.select) - self.assertEqual(expr.select_from, expected_query.select_from) - self.assertEqual(expr.where, expected_query.where) - self.assertEqual(expr.symbol, expected_query.symbol) - self.assertEqual(expr, expected_query) + self.assertEqual(expr.select, expected.select) + self.assertEqual(expr.select_from, expected.select_from) + self.assertEqual(expr.where, expected.where) + self.assertEqual(expr.symbol, expected.symbol) + self.assertEqual(expr, expected) def test_resolve_events_table_column_alias_inside_subquery(self): expr = parse_select("SELECT b FROM (select event as b, timestamp as c from events) e WHERE e.b = 'test'") resolve_symbols(expr) - events_table_symbol = ast.TableSymbol(name="events", table_name="events") + events_table_symbol = ast.TableSymbol(table_name="events") event_field_symbol = ast.FieldSymbol(name="event", table=events_table_symbol) timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=events_table_symbol) - expected_query = ast.SelectQuery( + inner_select_symbol = ast.SelectQuerySymbol( + symbols={ + "b": ast.ColumnAliasSymbol( + name="b", + symbol=event_field_symbol, + ), + "c": ast.ColumnAliasSymbol( + name="c", + symbol=timestamp_field_symbol, + ), + }, + tables={ + "events": events_table_symbol, + }, + ) + expected = ast.SelectQuery( select=[ ast.Field( chain=["b"], - symbol=ast.AliasSymbol( + symbol=ast.ColumnAliasSymbol( name="b", symbol=event_field_symbol, ), @@ -142,7 +169,7 @@ def test_resolve_events_table_column_alias_inside_subquery(self): ast.Alias( alias="b", expr=ast.Field(chain=["event"], symbol=event_field_symbol), - symbol=ast.AliasSymbol( + symbol=ast.ColumnAliasSymbol( name="b", symbol=event_field_symbol, ), @@ -150,7 +177,7 @@ def test_resolve_events_table_column_alias_inside_subquery(self): ast.Alias( alias="c", expr=ast.Field(chain=["timestamp"], symbol=timestamp_field_symbol), - symbol=ast.AliasSymbol( + symbol=ast.ColumnAliasSymbol( name="c", symbol=timestamp_field_symbol, ), @@ -159,30 +186,17 @@ def test_resolve_events_table_column_alias_inside_subquery(self): select_from=ast.JoinExpr( table=ast.Field(chain=["events"], symbol=events_table_symbol), alias="events", + symbol=ast.ColumnAliasSymbol(name="events", symbol=events_table_symbol), ), - symbol=ast.SelectQuerySymbol( - name="e", - symbols={ - "b": ast.AliasSymbol( - name="b", - symbol=event_field_symbol, - ), - "c": ast.AliasSymbol( - name="c", - symbol=timestamp_field_symbol, - ), - }, - tables={ - "events": events_table_symbol, - }, - ), + symbol=inner_select_symbol, ), alias="e", + symbol=ast.TableAliasSymbol(name="e", symbol=inner_select_symbol), ), where=ast.CompareOperation( left=ast.Field( chain=["e", "b"], - symbol=ast.AliasSymbol( + symbol=ast.ColumnAliasSymbol( name="b", symbol=event_field_symbol, ), @@ -191,17 +205,15 @@ def test_resolve_events_table_column_alias_inside_subquery(self): right=ast.Constant(value="test"), ), symbol=ast.SelectQuerySymbol( - name="", symbols={}, tables={ "e": ast.SelectQuerySymbol( - name="e", symbols={ - "b": ast.AliasSymbol( + "b": ast.ColumnAliasSymbol( name="b", symbol=event_field_symbol, ), - "c": ast.AliasSymbol( + "c": ast.ColumnAliasSymbol( name="c", symbol=timestamp_field_symbol, ), @@ -214,11 +226,86 @@ def test_resolve_events_table_column_alias_inside_subquery(self): ), ) # asserting individually to help debug if something is off - self.assertEqual(expr.select, expected_query.select) - self.assertEqual(expr.select_from, expected_query.select_from) - self.assertEqual(expr.where, expected_query.where) - self.assertEqual(expr.symbol, expected_query.symbol) - self.assertEqual(expr, expected_query) + self.assertEqual(expr.select, expected.select) + self.assertEqual(expr.select_from, expected.select_from) + self.assertEqual(expr.where, expected.where) + self.assertEqual(expr.symbol, expected.symbol) + self.assertEqual(expr, expected) + + def test_resolve_standard_subquery(self): + expr = parse_select( + "SELECT event, (select count() from events where event = e.event) as c FROM events e where event = '$pageview'" + ) + resolve_symbols(expr) + + outer_events_table_symbol = ast.TableSymbol(table_name="events") + outer_event_field_symbol = ast.FieldSymbol(name="event", table=outer_events_table_symbol) + + inner_events_table_symbol = ast.TableSymbol(table_name="events") + inner_event_field_symbol = ast.FieldSymbol(name="event", table=inner_events_table_symbol) + + expected = ast.SelectQuery( + select=[ + ast.Field( + chain=["event"], + symbol=outer_event_field_symbol, + ), + ast.Alias( + alias="c", + expr=ast.SelectQuery( + select=[ast.Call(name="count", args=[])], + select_from=ast.JoinExpr( + table=ast.Field(chain=["events"], symbol=inner_events_table_symbol), + alias="events", + symbol=ast.ColumnAliasSymbol(name="events", symbol=inner_events_table_symbol), + ), + symbol=ast.SelectQuerySymbol( + symbols={}, + tables={"events": inner_events_table_symbol}, + ), + where=ast.CompareOperation( + left=ast.Field(chain=["event"], symbol=inner_event_field_symbol), + op=ast.CompareOperationType.Eq, + right=ast.Field(chain=["e", "event"], symbol=outer_event_field_symbol), + ), + ), + symbol=ast.ColumnAliasSymbol( + name="c", + symbol=ast.SelectQuerySymbol( + symbols={}, + tables={"events": inner_events_table_symbol}, + ), + ), + ), + ], + select_from=ast.JoinExpr( + table=ast.Field(chain=["events"], symbol=outer_events_table_symbol), + alias="e", + symbol=ast.ColumnAliasSymbol(name="e", symbol=outer_events_table_symbol), + ), + where=ast.CompareOperation( + left=ast.Field( + chain=["event"], + symbol=outer_event_field_symbol, + ), + op=ast.CompareOperationType.Eq, + right=ast.Constant(value="$pageview"), + ), + symbol=ast.SelectQuerySymbol( + symbols={ + "c": ast.ColumnAliasSymbol( + name="c", symbol=ast.SelectQuerySymbol(symbols={}, tables={"events": inner_events_table_symbol}) + ) + }, + tables={"e": outer_events_table_symbol}, + ), + ) + # asserting individually to help debug if something is off + self.assertEqual(expr.select, expected.select) + self.assertEqual(expr.select_from, expected.select_from) + self.assertEqual(expr.where, expected.where) + self.assertEqual(expr.symbol, expected.symbol) + self.assertEqual(expr, expected) # "with 2 as a select 1 as a" -> "Different expressions with the same alias a:" @@ -226,3 +313,15 @@ def test_resolve_events_table_column_alias_inside_subquery(self): # "select a, b, e.c from (select 1 as a, 2 as b, 3 as c) as e" -> 1, 2, 3 + +# # good +# SELECT t.x FROM (SELECT 1 AS x) AS t; +# SELECT t.x FROM (SELECT x FROM tbl) AS t; +# SELECT x FROM (SELECT x FROM tbl) AS t; + +# # bad +# SELECT x, (SELECT 1 AS x); -- does not work, `x` is not visible; +# SELECT x IN (SELECT 1 AS x); -- does not work either; +# SELECT x IN (SELECT 1 AS x) FROM (SELECT 1 AS x); -- this will work, but keep in mind that there are two different `x`. +# SELECT tbl.x FROM (SELECT x FROM tbl) AS t; -- this is wrong, the `tbl` name is not exported +# SELECT t2.x FROM (SELECT x FROM tbl AS t2) AS t; -- this is also wrong, the `t2` alias is not exported From c5501472e94fecf57158453a8d0d8471912c7117 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 9 Feb 2023 21:26:04 +0100 Subject: [PATCH 12/81] column resolver --- posthog/hogql/ast.py | 16 +-- posthog/hogql/resolver.py | 68 ++++++++---- posthog/hogql/test/test_resolver.py | 156 ++++++++-------------------- 3 files changed, 96 insertions(+), 144 deletions(-) diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index 66b8b5b12ed89..83b8974ac0325 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -59,28 +59,32 @@ def has_child(self, name: str) -> bool: if self.table_name == "events": return name in EVENT_FIELDS else: - raise NotImplementedError(f"Can not resolve table: {self.name}") + raise NotImplementedError(f"Can not resolve table: {self.table_name}") def get_child(self, name: str) -> "Symbol": if self.table_name == "events": if name in EVENT_FIELDS: return FieldSymbol(name=name, table=self) else: - raise NotImplementedError(f"Can not resolve table: {self.name}") + raise NotImplementedError(f"Can not resolve table: {self.table_name}") class SelectQuerySymbol(Symbol): - symbols: Dict[str, Symbol] + # all aliases a select query has access to in its scope + aliases: Dict[str, Symbol] + # all symbols a select query exports + columns: Dict[str, Symbol] + # all tables we join in this query on which we look for aliases tables: Dict[str, Symbol] def get_child(self, name: str) -> "Symbol": - if name in self.symbols: - return self.symbols[name] + if name in self.columns: + return self.columns[name] if name in self.tables: return self.tables[name] def has_child(self, name: str) -> bool: - return name in self.symbols or name in self.tables + return name in self.columns or name in self.tables class FieldSymbol(Symbol): diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py index 28d150022f22f..8765448675d33 100644 --- a/posthog/hogql/resolver.py +++ b/posthog/hogql/resolver.py @@ -25,15 +25,15 @@ def visit_alias(self, node: ast.Alias): if len(self.scopes) == 0: raise ResolverException("Aliases are allowed only within SELECT queries") last_select = self.scopes[-1] - if node.alias in last_select.symbols: + if node.alias in last_select.aliases: raise ResolverException(f"Found multiple expressions with the same alias: {node.alias}") if node.alias == "": raise ResolverException("Alias cannot be empty") self.visit(node.expr) - node.symbol = ast.ColumnAliasSymbol(name=node.alias, symbol=node.expr.symbol) - last_select.symbols[node.alias] = node.symbol + node.symbol = ast.ColumnAliasSymbol(name=node.alias, symbol=unwrap_column_alias_symbol(node.expr.symbol)) + last_select.aliases[node.alias] = node.symbol def visit_field(self, node): if node.symbol is not None: @@ -44,23 +44,23 @@ def visit_field(self, node): # resolve the first part of the chain name = node.chain[0] symbol: Optional[ast.Symbol] = None - for scope in reversed(self.scopes): - if name in scope.tables and len(node.chain) > 1: - # CH assumes you're selecting a field, unless it's with a "." in the field, then check for tables - symbol = scope.tables[name] - break - elif name in scope.symbols: - symbol = scope.symbols[name] - break - else: - fields_on_tables_in_scope = [table for table in scope.tables.values() if table.has_child(name)] - if len(fields_on_tables_in_scope) > 1: - raise ResolverException( - f"Found multiple joined tables with field \"{name}\": {', '.join([symbol.name for symbol in fields_on_tables_in_scope])}. Please specify which table you're selecting from." - ) - elif len(fields_on_tables_in_scope) == 1: - symbol = fields_on_tables_in_scope[0].get_child(name) - break + + # to keep things simple, we only allow selecting fields from within this (select x) scope + scope = self.scopes[-1] + + if len(node.chain) > 1 and name in scope.tables: + # CH assumes you're selecting a field, unless it's with a "." in the field, then check for tables + symbol = scope.tables[name] + elif name in scope.aliases: + symbol = scope.aliases[name] + else: + fields_on_tables_in_scope = [table for table in scope.tables.values() if table.has_child(name)] + if len(fields_on_tables_in_scope) > 1: + raise ResolverException( + f'Found multiple joined tables with field "{name}". Please where you\'re selecting from.' + ) + elif len(fields_on_tables_in_scope) == 1: + symbol = fields_on_tables_in_scope[0].get_child(name) if not symbol: raise ResolverException(f'Cannot resolve symbol: "{name}"') @@ -88,7 +88,10 @@ def visit_join_expr(self, node): if node.table.chain == ["events"]: node.table.symbol = ast.TableSymbol(table_name="events") - node.symbol = ast.TableAliasSymbol(name=node.alias, symbol=node.table.symbol) + if node.alias == node.table.symbol.table_name: + node.symbol = node.table.symbol + else: + node.symbol = ast.TableAliasSymbol(name=node.alias, symbol=node.table.symbol) else: raise ResolverException(f"Cannot resolve table {node.table.chain[0]}") @@ -102,7 +105,7 @@ def visit_join_expr(self, node): else: raise ResolverException(f"JoinExpr with table of type {type(node.table).__name__} not supported") - last_select.tables[node.alias] = node.table.symbol + last_select.tables[node.alias] = node.symbol self.visit(node.join_expr) @@ -110,7 +113,7 @@ def visit_select_query(self, node): if node.symbol is not None: return - node.symbol = ast.SelectQuerySymbol(symbols={}, tables={}) + node.symbol = ast.SelectQuerySymbol(aliases={}, columns={}, tables={}) self.scopes.append(node.symbol) if node.select_from: @@ -118,6 +121,15 @@ def visit_select_query(self, node): if node.select: for expr in node.select: self.visit(expr) + if isinstance(expr.symbol, ast.ColumnAliasSymbol): + node.symbol.columns[expr.symbol.name] = expr.symbol + + elif isinstance(expr, ast.Alias): + node.symbol.columns[expr.alias] = expr.symbol + + elif isinstance(expr.symbol, ast.FieldSymbol): + node.symbol.columns[expr.symbol.name] = expr.symbol + if node.where: self.visit(node.where) if node.prewhere: @@ -128,3 +140,13 @@ def visit_select_query(self, node): self.scopes.pop() return node.symbol + + +def unwrap_column_alias_symbol(symbol: ast.Symbol) -> ast.Symbol: + i = 0 + while isinstance(symbol, ast.ColumnAliasSymbol): + symbol = symbol.symbol + i += 1 + if i > 100: + raise ResolverException("ColumnAliasSymbol recursion too deep!") + return symbol diff --git a/posthog/hogql/test/test_resolver.py b/posthog/hogql/test/test_resolver.py index 8ae06e6b8a2c3..30de26cb15346 100644 --- a/posthog/hogql/test/test_resolver.py +++ b/posthog/hogql/test/test_resolver.py @@ -1,6 +1,6 @@ from posthog.hogql import ast from posthog.hogql.parser import parse_select -from posthog.hogql.resolver import resolve_symbols +from posthog.hogql.resolver import ResolverException, resolve_symbols from posthog.test.base import BaseTest @@ -13,7 +13,8 @@ def test_resolve_events_table(self): event_field_symbol = ast.FieldSymbol(name="event", table=events_table_symbol) timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=events_table_symbol) select_query_symbol = ast.SelectQuerySymbol( - symbols={}, + aliases={}, + columns={"event": event_field_symbol, "timestamp": timestamp_field_symbol}, tables={"events": events_table_symbol}, ) @@ -25,7 +26,7 @@ def test_resolve_events_table(self): select_from=ast.JoinExpr( table=ast.Field(chain=["events"], symbol=events_table_symbol), alias="events", - symbol=ast.TableAliasSymbol(name="events", symbol=events_table_symbol), + symbol=events_table_symbol, ), where=ast.CompareOperation( left=ast.Field(chain=["events", "event"], symbol=event_field_symbol), @@ -50,8 +51,9 @@ def test_resolve_events_table_alias(self): event_field_symbol = ast.FieldSymbol(name="event", table=events_table_symbol) timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=events_table_symbol) select_query_symbol = ast.SelectQuerySymbol( - symbols={}, - tables={"e": events_table_symbol}, + aliases={}, + columns={"event": event_field_symbol, "timestamp": timestamp_field_symbol}, + tables={"e": ast.TableAliasSymbol(name="e", symbol=events_table_symbol)}, ) expected = ast.SelectQuery( @@ -86,14 +88,18 @@ def test_resolve_events_table_column_alias(self): events_table_symbol = ast.TableSymbol(table_name="events") event_field_symbol = ast.FieldSymbol(name="event", table=events_table_symbol) timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=events_table_symbol) + select_query_symbol = ast.SelectQuerySymbol( - symbols={ + aliases={ "ee": ast.ColumnAliasSymbol(name="ee", symbol=event_field_symbol), - "e": ast.ColumnAliasSymbol( - name="e", symbol=ast.ColumnAliasSymbol(name="ee", symbol=event_field_symbol) - ), + "e": ast.ColumnAliasSymbol(name="e", symbol=event_field_symbol), + }, + columns={ + "ee": ast.ColumnAliasSymbol(name="ee", symbol=event_field_symbol), + "e": ast.ColumnAliasSymbol(name="e", symbol=event_field_symbol), + "timestamp": timestamp_field_symbol, }, - tables={"e": events_table_symbol}, + tables={"e": ast.TableAliasSymbol(name="e", symbol=events_table_symbol)}, ) expected = ast.SelectQuery( @@ -101,22 +107,20 @@ def test_resolve_events_table_column_alias(self): ast.Alias( alias="ee", expr=ast.Field(chain=["event"], symbol=event_field_symbol), - symbol=ast.ColumnAliasSymbol(name="ee", symbol=event_field_symbol), + symbol=select_query_symbol.aliases["ee"], ), - ast.Field(chain=["ee"], symbol=ast.ColumnAliasSymbol(name="ee", symbol=event_field_symbol)), + ast.Field(chain=["ee"], symbol=select_query_symbol.aliases["ee"]), ast.Alias( alias="e", - expr=ast.Field(chain=["ee"], symbol=ast.ColumnAliasSymbol(name="ee", symbol=event_field_symbol)), - symbol=ast.ColumnAliasSymbol( - name="e", symbol=ast.ColumnAliasSymbol(name="ee", symbol=event_field_symbol) - ), + expr=ast.Field(chain=["ee"], symbol=select_query_symbol.aliases["ee"]), + symbol=select_query_symbol.aliases["e"], ), ast.Field(chain=["e", "timestamp"], symbol=timestamp_field_symbol), ], select_from=ast.JoinExpr( table=ast.Field(chain=["events"], symbol=events_table_symbol), alias="e", - symbol=ast.TableAliasSymbol(name="e", symbol=events_table_symbol), + symbol=select_query_symbol.tables["e"], ), where=ast.CompareOperation( left=ast.Field(chain=["e", "event"], symbol=event_field_symbol), @@ -139,15 +143,13 @@ def test_resolve_events_table_column_alias_inside_subquery(self): event_field_symbol = ast.FieldSymbol(name="event", table=events_table_symbol) timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=events_table_symbol) inner_select_symbol = ast.SelectQuerySymbol( - symbols={ - "b": ast.ColumnAliasSymbol( - name="b", - symbol=event_field_symbol, - ), - "c": ast.ColumnAliasSymbol( - name="c", - symbol=timestamp_field_symbol, - ), + aliases={ + "b": ast.ColumnAliasSymbol(name="b", symbol=event_field_symbol), + "c": ast.ColumnAliasSymbol(name="c", symbol=timestamp_field_symbol), + }, + columns={ + "b": ast.ColumnAliasSymbol(name="b", symbol=event_field_symbol), + "c": ast.ColumnAliasSymbol(name="c", symbol=timestamp_field_symbol), }, tables={ "events": events_table_symbol, @@ -186,7 +188,7 @@ def test_resolve_events_table_column_alias_inside_subquery(self): select_from=ast.JoinExpr( table=ast.Field(chain=["events"], symbol=events_table_symbol), alias="events", - symbol=ast.ColumnAliasSymbol(name="events", symbol=events_table_symbol), + symbol=events_table_symbol, ), symbol=inner_select_symbol, ), @@ -205,24 +207,11 @@ def test_resolve_events_table_column_alias_inside_subquery(self): right=ast.Constant(value="test"), ), symbol=ast.SelectQuerySymbol( - symbols={}, - tables={ - "e": ast.SelectQuerySymbol( - symbols={ - "b": ast.ColumnAliasSymbol( - name="b", - symbol=event_field_symbol, - ), - "c": ast.ColumnAliasSymbol( - name="c", - symbol=timestamp_field_symbol, - ), - }, - tables={ - "events": events_table_symbol, - }, - ) + aliases={}, + columns={ + "b": ast.ColumnAliasSymbol(name="b", symbol=event_field_symbol), }, + tables={"e": ast.TableAliasSymbol(name="e", symbol=inner_select_symbol)}, ), ) # asserting individually to help debug if something is off @@ -232,80 +221,14 @@ def test_resolve_events_table_column_alias_inside_subquery(self): self.assertEqual(expr.symbol, expected.symbol) self.assertEqual(expr, expected) - def test_resolve_standard_subquery(self): + def test_resolve_subquery_no_field_access(self): + # "Aliases defined outside of subquery are not visible in subqueries (but see below)." expr = parse_select( "SELECT event, (select count() from events where event = e.event) as c FROM events e where event = '$pageview'" ) - resolve_symbols(expr) - - outer_events_table_symbol = ast.TableSymbol(table_name="events") - outer_event_field_symbol = ast.FieldSymbol(name="event", table=outer_events_table_symbol) - - inner_events_table_symbol = ast.TableSymbol(table_name="events") - inner_event_field_symbol = ast.FieldSymbol(name="event", table=inner_events_table_symbol) - - expected = ast.SelectQuery( - select=[ - ast.Field( - chain=["event"], - symbol=outer_event_field_symbol, - ), - ast.Alias( - alias="c", - expr=ast.SelectQuery( - select=[ast.Call(name="count", args=[])], - select_from=ast.JoinExpr( - table=ast.Field(chain=["events"], symbol=inner_events_table_symbol), - alias="events", - symbol=ast.ColumnAliasSymbol(name="events", symbol=inner_events_table_symbol), - ), - symbol=ast.SelectQuerySymbol( - symbols={}, - tables={"events": inner_events_table_symbol}, - ), - where=ast.CompareOperation( - left=ast.Field(chain=["event"], symbol=inner_event_field_symbol), - op=ast.CompareOperationType.Eq, - right=ast.Field(chain=["e", "event"], symbol=outer_event_field_symbol), - ), - ), - symbol=ast.ColumnAliasSymbol( - name="c", - symbol=ast.SelectQuerySymbol( - symbols={}, - tables={"events": inner_events_table_symbol}, - ), - ), - ), - ], - select_from=ast.JoinExpr( - table=ast.Field(chain=["events"], symbol=outer_events_table_symbol), - alias="e", - symbol=ast.ColumnAliasSymbol(name="e", symbol=outer_events_table_symbol), - ), - where=ast.CompareOperation( - left=ast.Field( - chain=["event"], - symbol=outer_event_field_symbol, - ), - op=ast.CompareOperationType.Eq, - right=ast.Constant(value="$pageview"), - ), - symbol=ast.SelectQuerySymbol( - symbols={ - "c": ast.ColumnAliasSymbol( - name="c", symbol=ast.SelectQuerySymbol(symbols={}, tables={"events": inner_events_table_symbol}) - ) - }, - tables={"e": outer_events_table_symbol}, - ), - ) - # asserting individually to help debug if something is off - self.assertEqual(expr.select, expected.select) - self.assertEqual(expr.select_from, expected.select_from) - self.assertEqual(expr.where, expected.where) - self.assertEqual(expr.symbol, expected.symbol) - self.assertEqual(expr, expected) + with self.assertRaises(ResolverException) as e: + resolve_symbols(expr) + self.assertEqual(str(e.exception), 'Cannot resolve symbol: "e"') # "with 2 as a select 1 as a" -> "Different expressions with the same alias a:" @@ -318,6 +241,9 @@ def test_resolve_standard_subquery(self): # SELECT t.x FROM (SELECT 1 AS x) AS t; # SELECT t.x FROM (SELECT x FROM tbl) AS t; # SELECT x FROM (SELECT x FROM tbl) AS t; +# SELECT 1 AS x, x, x + 1; +# SELECT x, x + 1, 1 AS x; +# SELECT x, 1 + (2 + (3 AS x)); # # bad # SELECT x, (SELECT 1 AS x); -- does not work, `x` is not visible; From 2253f411c3c42b03667bfe352d24ae43c6a375be Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 9 Feb 2023 21:52:04 +0100 Subject: [PATCH 13/81] make sure some things error --- posthog/hogql/ast.py | 4 +--- posthog/hogql/resolver.py | 16 +++++++++------- posthog/hogql/test/test_resolver.py | 21 ++++++++++++++------- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index 83b8974ac0325..f3193d24ca6d3 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -80,11 +80,9 @@ class SelectQuerySymbol(Symbol): def get_child(self, name: str) -> "Symbol": if name in self.columns: return self.columns[name] - if name in self.tables: - return self.tables[name] def has_child(self, name: str) -> bool: - return name in self.columns or name in self.tables + return name in self.columns class FieldSymbol(Symbol): diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py index 8765448675d33..b2ae30415c1b1 100644 --- a/posthog/hogql/resolver.py +++ b/posthog/hogql/resolver.py @@ -54,22 +54,24 @@ def visit_field(self, node): elif name in scope.aliases: symbol = scope.aliases[name] else: - fields_on_tables_in_scope = [table for table in scope.tables.values() if table.has_child(name)] - if len(fields_on_tables_in_scope) > 1: + fields_in_scope = [table.get_child(name) for table in scope.tables.values() if table.has_child(name)] + if len(fields_in_scope) > 1: raise ResolverException( f'Found multiple joined tables with field "{name}". Please where you\'re selecting from.' ) - elif len(fields_on_tables_in_scope) == 1: - symbol = fields_on_tables_in_scope[0].get_child(name) + elif len(fields_in_scope) == 1: + symbol = fields_in_scope[0] if not symbol: raise ResolverException(f'Cannot resolve symbol: "{name}"') # recursively resolve the rest of the chain - for name in node.chain[1:]: - symbol = symbol.get_child(name) + for child_name in node.chain[1:]: + symbol = symbol.get_child(child_name) if symbol is None: - raise ResolverException(f"Cannot resolve symbol {', '.join(node.chain)}. Unable to resolve {name}") + raise ResolverException( + f"Cannot resolve symbol {'.'.join(node.chain)}. Unable to resolve {child_name} on {name}" + ) node.symbol = symbol diff --git a/posthog/hogql/test/test_resolver.py b/posthog/hogql/test/test_resolver.py index 30de26cb15346..7095b8b43b824 100644 --- a/posthog/hogql/test/test_resolver.py +++ b/posthog/hogql/test/test_resolver.py @@ -230,6 +230,20 @@ def test_resolve_subquery_no_field_access(self): resolve_symbols(expr) self.assertEqual(str(e.exception), 'Cannot resolve symbol: "e"') + def test_resolve_errors(self): + queries = [ + "SELECT event, (select count() from events where event = x.event) as c FROM events x where event = '$pageview'", + "SELECT x, (SELECT 1 AS x)", + "SELECT x IN (SELECT 1 AS x)", + "SELECT events.x FROM (SELECT event as x FROM events) AS t", + "SELECT x.y FROM (SELECT event as y FROM events AS x) AS t", + # "SELECT x IN (SELECT 1 AS x) FROM (SELECT 1 AS x)", + ] + for query in queries: + with self.assertRaises(ResolverException) as e: + resolve_symbols(parse_select(query)) + self.assertEqual(str(e.exception), "Cannot resolve symbol") + # "with 2 as a select 1 as a" -> "Different expressions with the same alias a:" # "with 2 as b, 3 as c select (select 1 as b) as a, b, c" -> "Different expressions with the same alias b:" @@ -244,10 +258,3 @@ def test_resolve_subquery_no_field_access(self): # SELECT 1 AS x, x, x + 1; # SELECT x, x + 1, 1 AS x; # SELECT x, 1 + (2 + (3 AS x)); - -# # bad -# SELECT x, (SELECT 1 AS x); -- does not work, `x` is not visible; -# SELECT x IN (SELECT 1 AS x); -- does not work either; -# SELECT x IN (SELECT 1 AS x) FROM (SELECT 1 AS x); -- this will work, but keep in mind that there are two different `x`. -# SELECT tbl.x FROM (SELECT x FROM tbl) AS t; -- this is wrong, the `tbl` name is not exported -# SELECT t2.x FROM (SELECT x FROM tbl AS t2) AS t; -- this is also wrong, the `t2` alias is not exported From a0068724724d63d59544651aad963b27d15e09b8 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 9 Feb 2023 22:57:53 +0100 Subject: [PATCH 14/81] annotate --- posthog/hogql/resolver.py | 162 +++++++++++++++++++++----------------- 1 file changed, 89 insertions(+), 73 deletions(-) diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py index b2ae30415c1b1..6482d0517c202 100644 --- a/posthog/hogql/resolver.py +++ b/posthog/hogql/resolver.py @@ -18,76 +18,61 @@ class Resolver(TraversingVisitor): def __init__(self): self.scopes: List[ast.SelectQuerySymbol] = [] - def visit_alias(self, node: ast.Alias): - if node.symbol is not None: - return - - if len(self.scopes) == 0: - raise ResolverException("Aliases are allowed only within SELECT queries") - last_select = self.scopes[-1] - if node.alias in last_select.aliases: - raise ResolverException(f"Found multiple expressions with the same alias: {node.alias}") - if node.alias == "": - raise ResolverException("Alias cannot be empty") - - self.visit(node.expr) - - node.symbol = ast.ColumnAliasSymbol(name=node.alias, symbol=unwrap_column_alias_symbol(node.expr.symbol)) - last_select.aliases[node.alias] = node.symbol + def visit_select_query(self, node): + """Visit each SELECT query or subquery.""" - def visit_field(self, node): if node.symbol is not None: return - if len(node.chain) == 0: - raise Exception("Invalid field access with empty chain") - # resolve the first part of the chain - name = node.chain[0] - symbol: Optional[ast.Symbol] = None + # Create a new lexical scope each time we enter a SELECT query. + node.symbol = ast.SelectQuerySymbol(aliases={}, columns={}, tables={}) + # Keep those scopes stacked in a list as we traverse the tree. + self.scopes.append(node.symbol) - # to keep things simple, we only allow selecting fields from within this (select x) scope - scope = self.scopes[-1] + # Visit all the FROM and JOIN tables (JoinExpr nodes) + if node.select_from: + self.visit(node.select_from) - if len(node.chain) > 1 and name in scope.tables: - # CH assumes you're selecting a field, unless it's with a "." in the field, then check for tables - symbol = scope.tables[name] - elif name in scope.aliases: - symbol = scope.aliases[name] - else: - fields_in_scope = [table.get_child(name) for table in scope.tables.values() if table.has_child(name)] - if len(fields_in_scope) > 1: - raise ResolverException( - f'Found multiple joined tables with field "{name}". Please where you\'re selecting from.' - ) - elif len(fields_in_scope) == 1: - symbol = fields_in_scope[0] + # Visit all the SELECT columns. + # Then mark them for export in "columns". This means they will be available outside of this query via: + # SELECT e.event, e.timestamp from (SELECT event, timestamp FROM events) AS e + for expr in node.select or []: + self.visit(expr) + if isinstance(expr.symbol, ast.ColumnAliasSymbol): + node.symbol.columns[expr.symbol.name] = expr.symbol + elif isinstance(expr, ast.Alias): + node.symbol.columns[expr.alias] = expr.symbol + elif isinstance(expr.symbol, ast.FieldSymbol): + node.symbol.columns[expr.symbol.name] = expr.symbol - if not symbol: - raise ResolverException(f'Cannot resolve symbol: "{name}"') + if node.where: + self.visit(node.where) + if node.prewhere: + self.visit(node.prewhere) + if node.having: + self.visit(node.having) - # recursively resolve the rest of the chain - for child_name in node.chain[1:]: - symbol = symbol.get_child(child_name) - if symbol is None: - raise ResolverException( - f"Cannot resolve symbol {'.'.join(node.chain)}. Unable to resolve {child_name} on {name}" - ) + self.scopes.pop() - node.symbol = symbol + return node.symbol def visit_join_expr(self, node): + """Visit each FROM and JOIN table or subquery.""" + if node.symbol is not None: return if len(self.scopes) == 0: raise ResolverException("Unexpected JoinExpr outside a SELECT query") - last_select = self.scopes[-1] + scope = self.scopes[-1] if isinstance(node.table, ast.Field): if node.alias is None: + # Make sure there is a way to call the field in the scope. node.alias = node.table.chain[0] - if node.alias in last_select.tables: - raise ResolverException(f"Table alias with the same name as another table: {node.alias}") + if node.alias in scope.tables: + raise ResolverException(f'Already have a joined table called "{node.alias}", can\'t redefine.') + # Only joining the events table is supported if node.table.chain == ["events"]: node.table.symbol = ast.TableSymbol(table_name="events") if node.alias == node.table.symbol.table_name: @@ -107,41 +92,72 @@ def visit_join_expr(self, node): else: raise ResolverException(f"JoinExpr with table of type {type(node.table).__name__} not supported") - last_select.tables[node.alias] = node.symbol + scope.tables[node.alias] = node.symbol self.visit(node.join_expr) - def visit_select_query(self, node): + def visit_alias(self, node: ast.Alias): + """Visit column aliases. SELECT 1, (select 3 as y) as x.""" if node.symbol is not None: return - node.symbol = ast.SelectQuerySymbol(aliases={}, columns={}, tables={}) - self.scopes.append(node.symbol) + if len(self.scopes) == 0: + raise ResolverException("Aliases are allowed only within SELECT queries") + scope = self.scopes[-1] + if node.alias in scope.aliases: + raise ResolverException(f"Cannot redefine an alias with the name: {node.alias}") + if node.alias == "": + raise ResolverException("Alias cannot be empty") - if node.select_from: - self.visit(node.select_from) - if node.select: - for expr in node.select: - self.visit(expr) - if isinstance(expr.symbol, ast.ColumnAliasSymbol): - node.symbol.columns[expr.symbol.name] = expr.symbol + self.visit(node.expr) + node.symbol = ast.ColumnAliasSymbol(name=node.alias, symbol=unwrap_column_alias_symbol(node.expr.symbol)) + scope.aliases[node.alias] = node.symbol - elif isinstance(expr, ast.Alias): - node.symbol.columns[expr.alias] = expr.symbol + def visit_field(self, node): + """Visit a field such as ast.Field(chain=["e", "properties", "$browser"])""" + if node.symbol is not None: + return + if len(node.chain) == 0: + raise Exception("Invalid field access with empty chain") - elif isinstance(expr.symbol, ast.FieldSymbol): - node.symbol.columns[expr.symbol.name] = expr.symbol + # ClickHouse does not support subqueries accessing "x.event" like this: + # "SELECT event, (select count() from events where event = x.event) as c FROM events x where event = '$pageview'", + # + # But this is supported: + # "SELECT t.big_count FROM (select count() + 100 as big_count from events) as t", + # + # Thus only look into the current scope, for columns and aliases. + scope = self.scopes[-1] + symbol: Optional[ast.Symbol] = None + name = node.chain[0] - if node.where: - self.visit(node.where) - if node.prewhere: - self.visit(node.prewhere) - if node.having: - self.visit(node.having) + if len(node.chain) > 1 and name in scope.tables: + # If the field has a chain of at least one (e.g "e", "event"), the first part could refer to a table. + symbol = scope.tables[name] + elif name in scope.columns: + symbol = scope.columns[name] + elif name in scope.aliases: + symbol = scope.aliases[name] + else: + # Look through all FROM/JOIN tables, if they export a field by this name. + fields_in_scope = [table.get_child(name) for table in scope.tables.values() if table.has_child(name)] + if len(fields_in_scope) > 1: + raise ResolverException(f'Ambiguous query. Found multiple sources for field "{name}".') + elif len(fields_in_scope) == 1: + symbol = fields_in_scope[0] - self.scopes.pop() + if not symbol: + raise ResolverException(f'Cannot resolve symbol: "{name}"') - return node.symbol + # Recursively resolve the rest of the chain until we can point to the deepest node. + for child_name in node.chain[1:]: + symbol = symbol.get_child(child_name) + if symbol is None: + raise ResolverException( + f"Cannot resolve symbol {'.'.join(node.chain)}. Unable to resolve {child_name} on {name}" + ) + + node.symbol = symbol def unwrap_column_alias_symbol(symbol: ast.Symbol) -> ast.Symbol: From b7b5521b82968f58e3a55d4ed663a79b935ec3ff Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 9 Feb 2023 23:12:22 +0100 Subject: [PATCH 15/81] constants --- posthog/hogql/ast.py | 4 ++++ posthog/hogql/resolver.py | 6 ++++++ posthog/hogql/test/test_resolver.py | 14 ++++++-------- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index f3193d24ca6d3..5c001dbd80174 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -99,6 +99,10 @@ def get_child(self, name: str) -> "Symbol": raise NotImplementedError(f"Can not resolve fields on table: {self.name}") +class ConstantSymbol(Symbol): + value: Any + + class PropertySymbol(Symbol): name: str field: FieldSymbol diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py index 6482d0517c202..a3d92e598c72f 100644 --- a/posthog/hogql/resolver.py +++ b/posthog/hogql/resolver.py @@ -159,6 +159,12 @@ def visit_field(self, node): node.symbol = symbol + def visit_constant(self, node): + """Visit a constant""" + if node.symbol is not None: + return + node.symbol = ast.ConstantSymbol(value=node.value) + def unwrap_column_alias_symbol(symbol: ast.Symbol) -> ast.Symbol: i = 0 diff --git a/posthog/hogql/test/test_resolver.py b/posthog/hogql/test/test_resolver.py index 7095b8b43b824..6b82b21672b2c 100644 --- a/posthog/hogql/test/test_resolver.py +++ b/posthog/hogql/test/test_resolver.py @@ -31,7 +31,7 @@ def test_resolve_events_table(self): where=ast.CompareOperation( left=ast.Field(chain=["events", "event"], symbol=event_field_symbol), op=ast.CompareOperationType.Eq, - right=ast.Constant(value="test"), + right=ast.Constant(value="test", symbol=ast.ConstantSymbol(value="test")), ), symbol=select_query_symbol, ) @@ -69,7 +69,7 @@ def test_resolve_events_table_alias(self): where=ast.CompareOperation( left=ast.Field(chain=["e", "event"], symbol=event_field_symbol), op=ast.CompareOperationType.Eq, - right=ast.Constant(value="test"), + right=ast.Constant(value="test", symbol=ast.ConstantSymbol(value="test")), ), symbol=select_query_symbol, ) @@ -125,7 +125,7 @@ def test_resolve_events_table_column_alias(self): where=ast.CompareOperation( left=ast.Field(chain=["e", "event"], symbol=event_field_symbol), op=ast.CompareOperationType.Eq, - right=ast.Constant(value="test"), + right=ast.Constant(value="test", symbol=ast.ConstantSymbol(value="test")), ), symbol=select_query_symbol, ) @@ -204,7 +204,7 @@ def test_resolve_events_table_column_alias_inside_subquery(self): ), ), op=ast.CompareOperationType.Eq, - right=ast.Constant(value="test"), + right=ast.Constant(value="test", symbol=ast.ConstantSymbol(value="test")), ), symbol=ast.SelectQuerySymbol( aliases={}, @@ -237,18 +237,15 @@ def test_resolve_errors(self): "SELECT x IN (SELECT 1 AS x)", "SELECT events.x FROM (SELECT event as x FROM events) AS t", "SELECT x.y FROM (SELECT event as y FROM events AS x) AS t", - # "SELECT x IN (SELECT 1 AS x) FROM (SELECT 1 AS x)", ] for query in queries: with self.assertRaises(ResolverException) as e: resolve_symbols(parse_select(query)) - self.assertEqual(str(e.exception), "Cannot resolve symbol") + self.assertIn("Cannot resolve symbol", str(e.exception)) # "with 2 as a select 1 as a" -> "Different expressions with the same alias a:" # "with 2 as b, 3 as c select (select 1 as b) as a, b, c" -> "Different expressions with the same alias b:" - - # "select a, b, e.c from (select 1 as a, 2 as b, 3 as c) as e" -> 1, 2, 3 # # good @@ -258,3 +255,4 @@ def test_resolve_errors(self): # SELECT 1 AS x, x, x + 1; # SELECT x, x + 1, 1 AS x; # SELECT x, 1 + (2 + (3 AS x)); +# "SELECT x IN (SELECT 1 AS x) FROM (SELECT 1 AS x)", From 60376dd796377b51932b4bc10a386804104594a0 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 10 Feb 2023 00:01:43 +0100 Subject: [PATCH 16/81] simple sql query --- posthog/hogql/ast.py | 2 + posthog/hogql/query.py | 62 +++++++++++++++++++++++++++++ posthog/hogql/resolver.py | 2 + posthog/hogql/test/test_query.py | 24 +++++++++++ posthog/hogql/test/test_resolver.py | 2 +- 5 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 posthog/hogql/query.py create mode 100644 posthog/hogql/test/test_query.py diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index 5c001dbd80174..fd2e252cb036c 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -65,6 +65,7 @@ def get_child(self, name: str) -> "Symbol": if self.table_name == "events": if name in EVENT_FIELDS: return FieldSymbol(name=name, table=self) + raise NotImplementedError(f"Event field not found: {name}") else: raise NotImplementedError(f"Can not resolve table: {self.table_name}") @@ -80,6 +81,7 @@ class SelectQuerySymbol(Symbol): def get_child(self, name: str) -> "Symbol": if name in self.columns: return self.columns[name] + raise NotImplementedError(f"Column not found: {name}") def has_child(self, name: str) -> bool: return name in self.columns diff --git a/posthog/hogql/query.py b/posthog/hogql/query.py new file mode 100644 index 0000000000000..362f22dc6cef3 --- /dev/null +++ b/posthog/hogql/query.py @@ -0,0 +1,62 @@ +from typing import List, Optional, Union + +from pydantic import BaseModel, Extra + +from posthog.clickhouse.client.connection import Workload +from posthog.hogql import ast +from posthog.hogql.hogql import HogQLContext +from posthog.hogql.parser import parse_select +from posthog.hogql.printer import print_ast +from posthog.hogql.resolver import resolve_symbols +from posthog.models import Team +from posthog.queries.insight import insight_sync_execute + + +class HogQLQueryResponse(BaseModel): + class Config: + extra = Extra.forbid + + clickhouse: Optional[str] = None + columns: Optional[List] = None + hogql: Optional[str] = None + query: Optional[str] = None + results: Optional[List] = None + types: Optional[List] = None + + +def execute_hogql_query( + query: Union[str, ast.SelectQuery], + team: Team, + query_type: str = "hogql_query", + workload: Workload = Workload.ONLINE, +) -> HogQLQueryResponse: + if isinstance(query, ast.SelectQuery): + select_query = query + query = None + else: + select_query = parse_select(str(query), no_placeholders=True) + + if select_query.limit is None: + select_query.limit = ast.Constant(value=1000) + + hogql_context = HogQLContext(select_team_id=team.pk) + resolve_symbols(select_query) + clickhouse = print_ast(select_query, [], hogql_context, "clickhouse") + hogql = print_ast(select_query, [], hogql_context, "hogql") + + results, types = insight_sync_execute( + clickhouse, + hogql_context.values, + with_column_types=True, + query_type=query_type, + workload=workload, + ) + print_columns = [print_ast(col, [], HogQLContext(), "hogql") for col in select_query.select] + return HogQLQueryResponse( + query=query, + hogql=hogql, + clickhouse=clickhouse, + results=results, + columns=print_columns, + types=types, + ) diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py index a3d92e598c72f..67da8625e29c8 100644 --- a/posthog/hogql/resolver.py +++ b/posthog/hogql/resolver.py @@ -15,6 +15,8 @@ class ResolverException(ValueError): class Resolver(TraversingVisitor): + """The Resolver visits an AST and assigns Symbols to the nodes.""" + def __init__(self): self.scopes: List[ast.SelectQuerySymbol] = [] diff --git a/posthog/hogql/test/test_query.py b/posthog/hogql/test/test_query.py new file mode 100644 index 0000000000000..1aec779f121ed --- /dev/null +++ b/posthog/hogql/test/test_query.py @@ -0,0 +1,24 @@ +from freezegun import freeze_time + +from posthog.hogql.query import execute_hogql_query +from posthog.test.base import APIBaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events + + +class TestQuery(ClickhouseTestMixin, APIBaseTest): + def test_query(self): + with freeze_time("2020-01-10"): + _create_event( + distinct_id="bla", + event="random event", + team=self.team, + properties={"random_prop": "don't include", "some other prop": "with some text"}, + ) + _create_event( + distinct_id="bla", + event="random event", + team=self.team, + properties={"random_prop": "don't include", "some other prop": "with some text"}, + ) + flush_persons_and_events() + response = execute_hogql_query("select count(), event from events group by event", self.team) + self.assertEqual(response.results, [(2, "random event")]) diff --git a/posthog/hogql/test/test_resolver.py b/posthog/hogql/test/test_resolver.py index 6b82b21672b2c..bffe494cb1946 100644 --- a/posthog/hogql/test/test_resolver.py +++ b/posthog/hogql/test/test_resolver.py @@ -222,7 +222,7 @@ def test_resolve_events_table_column_alias_inside_subquery(self): self.assertEqual(expr, expected) def test_resolve_subquery_no_field_access(self): - # "Aliases defined outside of subquery are not visible in subqueries (but see below)." + # From ClickHouse's GitHub: "Aliases defined outside of subquery are not visible in subqueries (but see below)." expr = parse_select( "SELECT event, (select count() from events where event = e.event) as c FROM events e where event = '$pageview'" ) From 9683e9204e7c18d869e3fbebba93e2581ed3bf9a Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 9 Feb 2023 23:39:35 +0000 Subject: [PATCH 17/81] Update snapshots --- posthog/api/test/__snapshots__/test_element.ambr | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/posthog/api/test/__snapshots__/test_element.ambr b/posthog/api/test/__snapshots__/test_element.ambr index 024b0cd86835d..e01fd97dd1132 100644 --- a/posthog/api/test/__snapshots__/test_element.ambr +++ b/posthog/api/test/__snapshots__/test_element.ambr @@ -95,3 +95,14 @@ WHERE "posthog_organizationmembership"."user_id" = 2 /*controller='element-stats',route='api/element/stats/%3F%24'*/ ' --- +# name: TestElement.test_element_stats_postgres_queries_are_as_expected.3 + ' + SELECT "posthog_instancesetting"."id", + "posthog_instancesetting"."key", + "posthog_instancesetting"."raw_value" + FROM "posthog_instancesetting" + WHERE "posthog_instancesetting"."key" = 'constance:posthog:RATE_LIMIT_ENABLED' + ORDER BY "posthog_instancesetting"."id" ASC + LIMIT 1 /*controller='element-stats',route='api/element/stats/%3F%24'*/ + ' +--- From ac3048e9317965890cdad07455039b539945b8cf Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 10 Feb 2023 22:21:35 +0100 Subject: [PATCH 18/81] introduce "print name" --- posthog/hogql/ast.py | 2 ++ posthog/hogql/printer.py | 37 ++++++++++++++++++++++++++++--------- posthog/hogql/resolver.py | 25 ++++++++++++++++++++++--- 3 files changed, 52 insertions(+), 12 deletions(-) diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index fd2e252cb036c..9064c3f6212b8 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -23,6 +23,8 @@ def accept(self, visitor): class Symbol(AST): + print_name: Optional[str] + def get_child(self, name: str) -> "Symbol": raise NotImplementedError() diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index 65eb95b928eba..c61d1f82a8213 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -44,16 +44,28 @@ def print_ast( from_table = None if node.select_from: - if node.select_from.alias is not None: - raise ValueError("Table aliases not yet supported") - if isinstance(node.select_from.table, ast.Field): - if node.select_from.table.chain != ["events"]: - raise ValueError('Only selecting from the "events" table is supported') - from_table = "events" - elif isinstance(node.select_from.table, ast.SelectQuery): - from_table = f"({print_ast(node.select_from.table, stack, context, dialect)})" + if node.symbol: + if isinstance(node.symbol, ast.TableSymbol): + if node.symbol.table_name != "events": + raise ValueError('Only selecting from the "events" table is supported') + from_table = f"events" + if node.symbol.print_name: + from_table = f"{from_table} AS {node.symbol.print_name}" + elif isinstance(node.symbol, ast.SelectQuerySymbol): + from_table = f"({print_ast(node.select_from.table, stack, context, dialect)})" + if node.symbol.print_name: + from_table = f"{from_table} AS {node.symbol.print_name}" else: - raise ValueError("Only selecting from a table or a subquery is supported") + if node.select_from.alias is not None: + raise ValueError("Table aliases not yet supported") + if isinstance(node.select_from.table, ast.Field): + if node.select_from.table.chain != ["events"]: + raise ValueError('Only selecting from the "events" table is supported') + from_table = "events" + elif isinstance(node.select_from.table, ast.SelectQuery): + from_table = f"({print_ast(node.select_from.table, stack, context, dialect)})" + else: + raise ValueError("Only selecting from a table or a subquery is supported") where = node.where # Guard with team_id if selecting from a table and printing ClickHouse SQL @@ -184,6 +196,13 @@ def print_ast( elif node.chain == ["person"]: query = "tuple(distinct_id, person.id, person.created_at, person.properties.name, person.properties.email)" response = print_ast(parse_expr(query), stack, context, dialect) + elif node.symbol is not None: + if isinstance(node.symbol, ast.FieldSymbol): + response = f"{node.symbol.table.print_name}.{node.symbol.name}" + elif isinstance(node.symbol, ast.TableSymbol): + response = node.symbol.print_name + else: + raise ValueError(f"Unknown Symbol, can not print {type(node.symbol)}") else: field_access = parse_field_access(node.chain, context) context.field_access_logs.append(field_access) diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py index 67da8625e29c8..fb0ce78b41aef 100644 --- a/posthog/hogql/resolver.py +++ b/posthog/hogql/resolver.py @@ -1,4 +1,4 @@ -from typing import List, Optional +from typing import Dict, List, Optional from posthog.hogql import ast from posthog.hogql.visitor import TraversingVisitor @@ -19,6 +19,7 @@ class Resolver(TraversingVisitor): def __init__(self): self.scopes: List[ast.SelectQuerySymbol] = [] + self.global_tables: Dict[str, ast.Symbol] = {} def visit_select_query(self, node): """Visit each SELECT query or subquery.""" @@ -53,6 +54,14 @@ def visit_select_query(self, node): self.visit(node.prewhere) if node.having: self.visit(node.having) + for expr in node.group_by or []: + self.visit(expr) + for expr in node.order_by or []: + self.visit(expr) + for expr in node.limit_by or []: + self.visit(expr) + self.visit(node.limit) + self.visit(node.offset) self.scopes.pop() @@ -76,7 +85,9 @@ def visit_join_expr(self, node): # Only joining the events table is supported if node.table.chain == ["events"]: - node.table.symbol = ast.TableSymbol(table_name="events") + print_name = f"{node.alias[0:1] or 't'}{len(self.global_tables)}" + node.table.symbol = ast.TableSymbol(table_name="events", print_name=print_name) + self.global_tables[print_name] = node.table.symbol if node.alias == node.table.symbol.table_name: node.symbol = node.table.symbol else: @@ -89,13 +100,18 @@ def visit_join_expr(self, node): if node.alias is None: node.symbol = node.table.symbol else: - node.symbol = ast.TableAliasSymbol(name=node.alias, symbol=node.table.symbol) + print_name = self._new_global_table_print_name(node.alias) + node.symbol = ast.TableAliasSymbol(name=node.alias, symbol=node.table.symbol, print_name=print_name) + self.global_tables[print_name] = node.symbol else: raise ResolverException(f"JoinExpr with table of type {type(node.table).__name__} not supported") scope.tables[node.alias] = node.symbol + # node.symbol.print_name = self._new_global_table_print_name(node.alias) + # self.global_tables[node.symbol.print_name] = node.symbol + self.visit(node.join_expr) def visit_alias(self, node: ast.Alias): @@ -167,6 +183,9 @@ def visit_constant(self, node): return node.symbol = ast.ConstantSymbol(value=node.value) + def _new_global_table_print_name(self, table_name): + return f"{table_name[0:1] or 't'}{len(self.global_tables)}" + def unwrap_column_alias_symbol(symbol: ast.Symbol) -> ast.Symbol: i = 0 From 86ddae59cff9350a8eff6881ed6adeac607cd8f8 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 10 Feb 2023 22:42:25 +0100 Subject: [PATCH 19/81] visit_unknown --- posthog/hogql/ast.py | 8 +++-- posthog/hogql/test/test_visitor.py | 54 ++++++++++++++++++++---------- 2 files changed, 42 insertions(+), 20 deletions(-) diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index 9064c3f6212b8..014bf81de40f2 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -18,8 +18,12 @@ class Config: def accept(self, visitor): camel_case_name = camel_case_pattern.sub("_", self.__class__.__name__).lower() method_name = "visit_{}".format(camel_case_name) - visit = getattr(visitor, method_name) - return visit(self) + if hasattr(visitor, method_name): + visit = getattr(visitor, method_name) + return visit(self) + if hasattr(visitor, "visit_unknown"): + return visitor.visit_unknown(self) + raise ValueError("Visitor has no method visit_constant") class Symbol(AST): diff --git a/posthog/hogql/test/test_visitor.py b/posthog/hogql/test/test_visitor.py index eedf80e367fc5..0a67144f659cc 100644 --- a/posthog/hogql/test/test_visitor.py +++ b/posthog/hogql/test/test_visitor.py @@ -1,30 +1,29 @@ from posthog.hogql import ast from posthog.hogql.parser import parse_expr -from posthog.hogql.visitor import CloningVisitor +from posthog.hogql.visitor import CloningVisitor, Visitor from posthog.test.base import BaseTest -class ConstantVisitor(CloningVisitor): - def __init__(self): - self.constants = [] - self.fields = [] - self.operations = [] - - def visit_constant(self, node): - self.constants.append(node.value) - return super().visit_constant(node) +class TestVisitor(BaseTest): + def test_visitor_pattern(self): + class ConstantVisitor(CloningVisitor): + def __init__(self): + self.constants = [] + self.fields = [] + self.operations = [] - def visit_field(self, node): - self.fields.append(node.chain) - return super().visit_field(node) + def visit_constant(self, node): + self.constants.append(node.value) + return super().visit_constant(node) - def visit_binary_operation(self, node: ast.BinaryOperation): - self.operations.append(node.op) - return super().visit_binary_operation(node) + def visit_field(self, node): + self.fields.append(node.chain) + return super().visit_field(node) + def visit_binary_operation(self, node: ast.BinaryOperation): + self.operations.append(node.op) + return super().visit_binary_operation(node) -class TestVisitor(BaseTest): - def test_visitor_pattern(self): visitor = ConstantVisitor() visitor.visit(ast.Constant(value="asd")) self.assertEqual(visitor.constants, ["asd"]) @@ -94,3 +93,22 @@ def test_everything_visitor(self): ] ) self.assertEqual(node, CloningVisitor().visit(node)) + + def test_unknown_visitor(self): + class UnknownVisitor(Visitor): + def visit_unknown(self, node): + return "!!" + + def visit_binary_operation(self, node: ast.BinaryOperation): + return self.visit(node.left) + node.op + self.visit(node.right) + + self.assertEqual(UnknownVisitor().visit(parse_expr("1 + 3 / 'asd2'")), "!!+!!/!!") + + def test_unknown_error_visitor(self): + class UnknownNotDefinedVisitor(Visitor): + def visit_binary_operation(self, node: ast.BinaryOperation): + return self.visit(node.left) + node.op + self.visit(node.right) + + with self.assertRaises(ValueError) as e: + UnknownNotDefinedVisitor().visit(parse_expr("1 + 3 / 'asd2'")) + self.assertEqual(str(e.exception), "Visitor has no method visit_constant") From c48024cca6239e86c5a3cbe6e66d86cd834ca22f Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Fri, 10 Feb 2023 22:56:19 +0100 Subject: [PATCH 20/81] basic printer via a visitor --- posthog/hogql/hogql.py | 2 +- posthog/hogql/printer.py | 195 +++++++++++++++++++++------------------ posthog/hogql/query.py | 6 +- 3 files changed, 111 insertions(+), 92 deletions(-) diff --git a/posthog/hogql/hogql.py b/posthog/hogql/hogql.py index 8e216b1654b46..273cf51ac7864 100644 --- a/posthog/hogql/hogql.py +++ b/posthog/hogql/hogql.py @@ -19,4 +19,4 @@ def translate_hogql(query: str, context: HogQLContext, dialect: Literal["hogql", raise ValueError(f"SyntaxError: {err.msg}") except NotImplementedError as err: raise ValueError(f"NotImplementedError: {err}") - return print_ast(node, [], context, dialect) + return print_ast(node, context, dialect) diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index c61d1f82a8213..553456248b9c9 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -13,6 +13,7 @@ from posthog.hogql.context import HogQLContext, HogQLFieldAccess from posthog.hogql.parser import parse_expr from posthog.hogql.print_string import print_hogql_identifier +from posthog.hogql.visitor import Visitor def guard_where_team_id(where: ast.Expr, context: HogQLContext) -> ast.Expr: @@ -30,17 +31,27 @@ def guard_where_team_id(where: ast.Expr, context: HogQLContext) -> ast.Expr: return where -def print_ast( - node: ast.AST, stack: List[ast.AST], context: HogQLContext, dialect: Literal["hogql", "clickhouse"] -) -> str: - """Translate a parsed HogQL expression in the shape of a Python AST into a Clickhouse expression.""" - stack.append(node) +def print_ast(node: ast.AST, context: HogQLContext, dialect: Literal["hogql", "clickhouse"]) -> str: + return Printer(context=context, dialect=dialect).visit(node) - if isinstance(node, ast.SelectQuery): - if dialect == "clickhouse" and not context.select_team_id: + +class Printer(Visitor): + def __init__(self, context: HogQLContext, dialect: Literal["hogql", "clickhouse"]): + self.context = context + self.dialect = dialect + self.stack: List[ast.AST] = [] + + def visit(self, node: ast.AST): + self.stack.append(node) + response = super().visit(node) + self.stack.pop() + return response + + def visit_select_query(self, node: ast.SelectQuery): + if self.dialect == "clickhouse" and not self.context.select_team_id: raise ValueError("Full SELECT queries are disabled if select_team_id is not set") - columns = [print_ast(column, stack, context, dialect) for column in node.select] if node.select else ["1"] + columns = [self.visit(column) for column in node.select] if node.select else ["1"] from_table = None if node.select_from: @@ -52,7 +63,7 @@ def print_ast( if node.symbol.print_name: from_table = f"{from_table} AS {node.symbol.print_name}" elif isinstance(node.symbol, ast.SelectQuerySymbol): - from_table = f"({print_ast(node.select_from.table, stack, context, dialect)})" + from_table = f"({self.visit(node.select_from.table)})" if node.symbol.print_name: from_table = f"{from_table} AS {node.symbol.print_name}" else: @@ -63,7 +74,7 @@ def print_ast( raise ValueError('Only selecting from the "events" table is supported') from_table = "events" elif isinstance(node.select_from.table, ast.SelectQuery): - from_table = f"({print_ast(node.select_from.table, stack, context, dialect)})" + from_table = f"({self.visit(node.select_from.table)})" else: raise ValueError("Only selecting from a table or a subquery is supported") @@ -71,23 +82,23 @@ def print_ast( # Guard with team_id if selecting from a table and printing ClickHouse SQL # We do this in the printer, and not in a separate step, to be really sure this gets added. # This will be improved when we add proper table and column alias support. For now, let's just be safe. - if dialect == "clickhouse" and from_table is not None: - where = guard_where_team_id(where, context) - where = print_ast(where, stack, context, dialect) if where else None + if self.dialect == "clickhouse" and from_table is not None: + where = guard_where_team_id(where, self.context) + where = self.visit(where) if where else None - having = print_ast(node.having, stack, context, dialect) if node.having else None - prewhere = print_ast(node.prewhere, stack, context, dialect) if node.prewhere else None - group_by = [print_ast(column, stack, context, dialect) for column in node.group_by] if node.group_by else None - order_by = [print_ast(column, stack, context, dialect) for column in node.order_by] if node.order_by else None + having = self.visit(node.having) if node.having else None + prewhere = self.visit(node.prewhere) if node.prewhere else None + group_by = [self.visit(column) for column in node.group_by] if node.group_by else None + order_by = [self.visit(column) for column in node.order_by] if node.order_by else None limit = node.limit - if context.limit_top_select: + if self.context.limit_top_select: if limit is not None: if isinstance(limit, ast.Constant) and isinstance(limit.value, int): limit.value = min(limit.value, MAX_SELECT_RETURNED_ROWS) else: limit = ast.Call(name="min2", args=[ast.Constant(value=MAX_SELECT_RETURNED_ROWS), limit]) - elif len(stack) == 1: + elif len(self.stack) == 1: limit = ast.Constant(value=MAX_SELECT_RETURNED_ROWS) clauses = [ @@ -100,116 +111,125 @@ def print_ast( f"ORDER BY {', '.join(order_by)}" if order_by and len(order_by) > 0 else None, ] if limit is not None: - clauses.append(f"LIMIT {print_ast(limit, stack, context, dialect)}") + clauses.append(f"LIMIT {self.visit(limit)}") if node.offset is not None: - clauses.append(f"OFFSET {print_ast(node.offset, stack, context, dialect)}") + clauses.append(f"OFFSET {self.visit(node.offset)}") if node.limit_by is not None: - clauses.append(f"BY {', '.join([print_ast(expr, stack, context, dialect) for expr in node.limit_by])}") + clauses.append(f"BY {', '.join([self.visit(expr) for expr in node.limit_by])}") if node.limit_with_ties: clauses.append("WITH TIES") response = " ".join([clause for clause in clauses if clause]) - if len(stack) > 1: + if len(self.stack) > 1: response = f"({response})" + return response - elif isinstance(node, ast.BinaryOperation): + def visit_binary_operation(self, node: ast.BinaryOperation): if node.op == ast.BinaryOperationType.Add: - response = f"plus({print_ast(node.left, stack, context, dialect)}, {print_ast(node.right, stack, context, dialect)})" + return f"plus({self.visit(node.left)}, {self.visit(node.right)})" elif node.op == ast.BinaryOperationType.Sub: - response = f"minus({print_ast(node.left, stack, context, dialect)}, {print_ast(node.right, stack, context, dialect)})" + return f"minus({self.visit(node.left)}, {self.visit(node.right)})" elif node.op == ast.BinaryOperationType.Mult: - response = f"multiply({print_ast(node.left, stack, context, dialect)}, {print_ast(node.right, stack, context, dialect)})" + return f"multiply({self.visit(node.left)}, {self.visit(node.right)})" elif node.op == ast.BinaryOperationType.Div: - response = f"divide({print_ast(node.left, stack, context, dialect)}, {print_ast(node.right, stack, context, dialect)})" + return f"divide({self.visit(node.left)}, {self.visit(node.right)})" elif node.op == ast.BinaryOperationType.Mod: - response = f"modulo({print_ast(node.left, stack, context, dialect)}, {print_ast(node.right, stack, context, dialect)})" + return f"modulo({self.visit(node.left)}, {self.visit(node.right)})" else: raise ValueError(f"Unknown BinaryOperationType {node.op}") - elif isinstance(node, ast.And): - response = f"and({', '.join([print_ast(operand, stack, context, dialect) for operand in node.exprs])})" - elif isinstance(node, ast.Or): - response = f"or({', '.join([print_ast(operand, stack, context, dialect) for operand in node.exprs])})" - elif isinstance(node, ast.Not): - response = f"not({print_ast(node.expr, stack, context, dialect)})" - elif isinstance(node, ast.OrderExpr): - response = f"{print_ast(node.expr, stack, context, dialect)} {node.order}" - elif isinstance(node, ast.CompareOperation): - left = print_ast(node.left, stack, context, dialect) - right = print_ast(node.right, stack, context, dialect) + + def visit_and(self, node: ast.And): + return f"and({', '.join([self.visit(operand) for operand in node.exprs])})" + + def visit_or(self, node: ast.Or): + return f"or({', '.join([self.visit(operand) for operand in node.exprs])})" + + def visit_not(self, node: ast.Not): + return f"not({self.visit(node.expr)})" + + def visit_order_expr(self, node: ast.OrderExpr): + return f"{self.visit(node.expr)} {node.order}" + + def visit_compare_operation(self, node: ast.CompareOperation): + left = self.visit(node.left) + right = self.visit(node.right) if node.op == ast.CompareOperationType.Eq: if isinstance(node.right, ast.Constant) and node.right.value is None: - response = f"isNull({left})" + return f"isNull({left})" else: - response = f"equals({left}, {right})" + return f"equals({left}, {right})" elif node.op == ast.CompareOperationType.NotEq: if isinstance(node.right, ast.Constant) and node.right.value is None: - response = f"isNotNull({left})" + return f"isNotNull({left})" else: - response = f"notEquals({left}, {right})" + return f"notEquals({left}, {right})" elif node.op == ast.CompareOperationType.Gt: - response = f"greater({left}, {right})" + return f"greater({left}, {right})" elif node.op == ast.CompareOperationType.GtE: - response = f"greaterOrEquals({left}, {right})" + return f"greaterOrEquals({left}, {right})" elif node.op == ast.CompareOperationType.Lt: - response = f"less({left}, {right})" + return f"less({left}, {right})" elif node.op == ast.CompareOperationType.LtE: - response = f"lessOrEquals({left}, {right})" + return f"lessOrEquals({left}, {right})" elif node.op == ast.CompareOperationType.Like: - response = f"like({left}, {right})" + return f"like({left}, {right})" elif node.op == ast.CompareOperationType.ILike: - response = f"ilike({left}, {right})" + return f"ilike({left}, {right})" elif node.op == ast.CompareOperationType.NotLike: - response = f"not(like({left}, {right}))" + return f"not(like({left}, {right}))" elif node.op == ast.CompareOperationType.NotILike: - response = f"not(ilike({left}, {right}))" + return f"not(ilike({left}, {right}))" elif node.op == ast.CompareOperationType.In: - response = f"in({left}, {right})" + return f"in({left}, {right})" elif node.op == ast.CompareOperationType.NotIn: - response = f"not(in({left}, {right}))" + return f"not(in({left}, {right}))" else: raise ValueError(f"Unknown CompareOperationType: {type(node.op).__name__}") - elif isinstance(node, ast.Constant): - key = f"hogql_val_{len(context.values)}" + + def visit_constant(self, node: ast.Constant): + key = f"hogql_val_{len(self.context.values)}" if isinstance(node.value, bool) and node.value is True: - response = "true" + return "true" elif isinstance(node.value, bool) and node.value is False: - response = "false" + return "false" elif isinstance(node.value, int) or isinstance(node.value, float): # :WATCH_OUT: isinstance(True, int) is True (!), so check for numbers lower down the chain - response = str(node.value) + return str(node.value) elif isinstance(node.value, str) or isinstance(node.value, list): - context.values[key] = node.value - response = f"%({key})s" + self.context.values[key] = node.value + return f"%({key})s" elif node.value is None: - response = "null" + return "null" else: raise ValueError( f"Unknown AST Constant node type '{type(node.value).__name__}' for value '{str(node.value)}'" ) - elif isinstance(node, ast.Field): - if dialect == "hogql": + + def visit_field(self, node: ast.Field): + if self.dialect == "hogql": # When printing HogQL, we print the properties out as a chain instead of converting them to Clickhouse SQL - response = ".".join([print_hogql_identifier(identifier) for identifier in node.chain]) + return ".".join([print_hogql_identifier(identifier) for identifier in node.chain]) elif node.chain == ["*"]: query = f"tuple({','.join(SELECT_STAR_FROM_EVENTS_FIELDS)})" - response = print_ast(parse_expr(query), stack, context, dialect) + return self.visit(parse_expr(query)) elif node.chain == ["person"]: query = "tuple(distinct_id, person.id, person.created_at, person.properties.name, person.properties.email)" - response = print_ast(parse_expr(query), stack, context, dialect) + return self.visit(parse_expr(query)) elif node.symbol is not None: if isinstance(node.symbol, ast.FieldSymbol): - response = f"{node.symbol.table.print_name}.{node.symbol.name}" + return f"{node.symbol.table.print_name}.{node.symbol.name}" elif isinstance(node.symbol, ast.TableSymbol): - response = node.symbol.print_name + return node.symbol.print_name else: raise ValueError(f"Unknown Symbol, can not print {type(node.symbol)}") else: - field_access = parse_field_access(node.chain, context) - context.field_access_logs.append(field_access) - response = field_access.sql - elif isinstance(node, ast.Call): + field_access = parse_field_access(node.chain, self.context) + self.context.field_access_logs.append(field_access) + return field_access.sql + + def visit_call(self, node: ast.Call): if node.name in HOGQL_AGGREGATIONS: - context.found_aggregation = True + self.context.found_aggregation = True required_arg_count = HOGQL_AGGREGATIONS[node.name] if required_arg_count != len(node.args): @@ -218,35 +238,34 @@ def print_ast( ) # check that we're not running inside another aggregate - for stack_node in stack: + for stack_node in self.stack: if stack_node != node and isinstance(stack_node, ast.Call) and stack_node.name in HOGQL_AGGREGATIONS: raise ValueError( f"Aggregation '{node.name}' cannot be nested inside another aggregation '{stack_node.name}'." ) - translated_args = ", ".join([print_ast(arg, stack, context, dialect) for arg in node.args]) - if dialect == "hogql": - response = f"{node.name}({translated_args})" + translated_args = ", ".join([self.visit(arg) for arg in node.args]) + if self.dialect == "hogql": + return f"{node.name}({translated_args})" elif node.name == "count": - response = "count(*)" + return "count(*)" elif node.name == "countDistinct": - response = f"count(distinct {translated_args})" + return f"count(distinct {translated_args})" elif node.name == "countDistinctIf": - response = f"countIf(distinct {translated_args})" + return f"countIf(distinct {translated_args})" else: - response = f"{node.name}({translated_args})" + return f"{node.name}({translated_args})" elif node.name in CLICKHOUSE_FUNCTIONS: - response = f"{CLICKHOUSE_FUNCTIONS[node.name]}({', '.join([print_ast(arg, stack, context, dialect) for arg in node.args])})" + return f"{CLICKHOUSE_FUNCTIONS[node.name]}({', '.join([self.visit(arg) for arg in node.args])})" else: raise ValueError(f"Unsupported function call '{node.name}(...)'") - elif isinstance(node, ast.Placeholder): + + def visit_placeholder(self, node: ast.Placeholder): raise ValueError(f"Found a Placeholder {{{node.field}}} in the tree. Can't generate query!") - else: - raise ValueError(f"Unknown AST node {type(node).__name__}") - stack.pop() - return response + def visit_unknown(self, node: ast.AST): + raise ValueError(f"Unknown AST node {type(node).__name__}") def parse_field_access(chain: List[str], context: HogQLContext) -> HogQLFieldAccess: diff --git a/posthog/hogql/query.py b/posthog/hogql/query.py index 362f22dc6cef3..f47e48707a987 100644 --- a/posthog/hogql/query.py +++ b/posthog/hogql/query.py @@ -41,8 +41,8 @@ def execute_hogql_query( hogql_context = HogQLContext(select_team_id=team.pk) resolve_symbols(select_query) - clickhouse = print_ast(select_query, [], hogql_context, "clickhouse") - hogql = print_ast(select_query, [], hogql_context, "hogql") + clickhouse = print_ast(select_query, hogql_context, "clickhouse") + hogql = print_ast(select_query, hogql_context, "hogql") results, types = insight_sync_execute( clickhouse, @@ -51,7 +51,7 @@ def execute_hogql_query( query_type=query_type, workload=workload, ) - print_columns = [print_ast(col, [], HogQLContext(), "hogql") for col in select_query.select] + print_columns = [print_ast(col, HogQLContext(), "hogql") for col in select_query.select] return HogQLQueryResponse( query=query, hogql=hogql, From afe485d56366569611d4bdb9643197d64fea9c9a Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 13 Feb 2023 12:34:40 +0100 Subject: [PATCH 21/81] completely redo printing --- posthog/hogql/ast.py | 76 ++++---- posthog/hogql/constants.py | 18 +- posthog/hogql/database.py | 97 ++++++++++ posthog/hogql/hogql.py | 12 +- posthog/hogql/parser.py | 10 +- posthog/hogql/printer.py | 286 ++++++++++++++++------------ posthog/hogql/resolver.py | 103 +++++----- posthog/hogql/test/test_printer.py | 57 ++++-- posthog/hogql/test/test_query.py | 12 ++ posthog/hogql/test/test_resolver.py | 43 +++-- 10 files changed, 463 insertions(+), 251 deletions(-) create mode 100644 posthog/hogql/database.py diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index 014bf81de40f2..3471092833ac9 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -4,7 +4,7 @@ from pydantic import BaseModel, Extra -from posthog.hogql.constants import EVENT_FIELDS +from posthog.hogql.database import Table, database # NOTE: when you add new AST fields or nodes, add them to CloningVisitor as well! @@ -27,8 +27,6 @@ def accept(self, visitor): class Symbol(AST): - print_name: Optional[str] - def get_child(self, name: str) -> "Symbol": raise NotImplementedError() @@ -38,20 +36,20 @@ def has_child(self, name: str) -> bool: class ColumnAliasSymbol(Symbol): name: str - symbol: "Symbol" + symbol: Symbol - def get_child(self, name: str) -> "Symbol": + def get_child(self, name: str) -> Symbol: return self.symbol.get_child(name) def has_child(self, name: str) -> bool: return self.symbol.has_child(name) -class TableAliasSymbol(Symbol): +class SelectQueryAliasSymbol(Symbol): name: str - symbol: "Symbol" + symbol: Symbol - def get_child(self, name: str) -> "Symbol": + def get_child(self, name: str) -> Symbol: return self.symbol.get_child(name) def has_child(self, name: str) -> bool: @@ -59,32 +57,39 @@ def has_child(self, name: str) -> bool: class TableSymbol(Symbol): - table_name: Literal["events"] + table: Table def has_child(self, name: str) -> bool: - if self.table_name == "events": - return name in EVENT_FIELDS - else: - raise NotImplementedError(f"Can not resolve table: {self.table_name}") + return name in self.table.__fields__ - def get_child(self, name: str) -> "Symbol": - if self.table_name == "events": - if name in EVENT_FIELDS: - return FieldSymbol(name=name, table=self) - raise NotImplementedError(f"Event field not found: {name}") - else: - raise NotImplementedError(f"Can not resolve table: {self.table_name}") + def get_child(self, name: str) -> Symbol: + if self.has_child(name): + return FieldSymbol(name=name, table=self) + raise NotImplementedError(f"Field not found: {name}") + + +class TableAliasSymbol(Symbol): + name: str + table: TableSymbol + + def get_child(self, name: str) -> Symbol: + return self.table.get_child(name) + + def has_child(self, name: str) -> bool: + return self.table.has_child(name) class SelectQuerySymbol(Symbol): # all aliases a select query has access to in its scope - aliases: Dict[str, Symbol] + aliases: Dict[str, ColumnAliasSymbol] # all symbols a select query exports columns: Dict[str, Symbol] - # all tables we join in this query on which we look for aliases - tables: Dict[str, Symbol] + # all from and join, tables and subqueries with aliases + tables: Dict[str, Union[TableSymbol, TableAliasSymbol, "SelectQuerySymbol", SelectQueryAliasSymbol]] + # all from and join subqueries without aliases + anonymous_tables: List["SelectQuerySymbol"] - def get_child(self, name: str) -> "Symbol": + def get_child(self, name: str) -> Symbol: if name in self.columns: return self.columns[name] raise NotImplementedError(f"Column not found: {name}") @@ -93,14 +98,22 @@ def has_child(self, name: str) -> bool: return name in self.columns +SelectQuerySymbol.update_forward_refs(SelectQuerySymbol=SelectQuerySymbol) + + +class CallSymbol(Symbol): + name: str + args: List[Symbol] + + class FieldSymbol(Symbol): name: str - table: TableSymbol + table: Union[TableSymbol, TableAliasSymbol] - def get_child(self, name: str) -> "Symbol": - if self.table.table_name == "events": + def get_child(self, name: str) -> Symbol: + if self.table.table == database.events: if self.name == "properties": - raise NotImplementedError(f"Property symbol resolution not implemented yet") + return PropertySymbol(name=name, field=self) else: raise NotImplementedError(f"Can not resolve field {self.name} on table events") else: @@ -120,11 +133,6 @@ class Expr(AST): symbol: Optional[Symbol] -ColumnAliasSymbol.update_forward_refs(Expr=Expr) -TableAliasSymbol.update_forward_refs(Expr=Expr) -SelectQuerySymbol.update_forward_refs(Expr=Expr) - - class Alias(Expr): alias: str expr: Expr @@ -215,6 +223,8 @@ class JoinExpr(Expr): class SelectQuery(Expr): + symbol: Optional[SelectQuerySymbol] = None + select: List[Expr] distinct: Optional[bool] = None select_from: Optional[JoinExpr] = None diff --git a/posthog/hogql/constants.py b/posthog/hogql/constants.py index a7f7a8357c7ff..1cb107817c21c 100644 --- a/posthog/hogql/constants.py +++ b/posthog/hogql/constants.py @@ -1,18 +1,3 @@ -# fields you can select from in the events query -EVENT_FIELDS = [ - "id", - "uuid", - "event", - "timestamp", - "properties", - "elements_chain", - "created_at", - "distinct_id", - "team_id", -] -# "person.*" fields you can select from in the events query -EVENT_PERSON_FIELDS = ["id", "created_at", "properties"] - # HogQL -> ClickHouse allowed transformations CLICKHOUSE_FUNCTIONS = { # arithmetic @@ -110,6 +95,9 @@ # Keywords passed to ClickHouse without transformation KEYWORDS = ["true", "false", "null"] +# Keywords you can't alias to +RESERVED_KEYWORDS = ["team_id"] + # Allow-listed fields returned when you select "*" from events. Person and group fields will be nested later. SELECT_STAR_FROM_EVENTS_FIELDS = [ "uuid", diff --git a/posthog/hogql/database.py b/posthog/hogql/database.py new file mode 100644 index 0000000000000..031f63502f0ba --- /dev/null +++ b/posthog/hogql/database.py @@ -0,0 +1,97 @@ +from pydantic import BaseModel, Extra + + +class Field(BaseModel): + class Config: + extra = Extra.forbid + + +class IntegerValue(Field): + pass + + +class StringValue(Field): + pass + + +class StringJSONValue(Field): + pass + + +class DateTimeValue(Field): + pass + + +class BooleanValue(Field): + pass + + +class Table(BaseModel): + class Config: + extra = Extra.forbid + + def clickhouse_table(self): + raise NotImplementedError() + + +class PersonsTable(Table): + id: StringValue = StringValue() + created_at: DateTimeValue = DateTimeValue() + team_id: IntegerValue = IntegerValue() + properties: StringJSONValue = StringJSONValue() + is_identified: BooleanValue = BooleanValue() + is_deleted: BooleanValue = BooleanValue() + version: IntegerValue = IntegerValue() + + def clickhouse_table(self): + return "person" + + +class PersonDistinctIdTable(Table): + team_id: IntegerValue = IntegerValue() + distinct_id: StringValue = StringValue() + person_id: StringValue = StringValue() + is_deleted: BooleanValue = BooleanValue() + version: IntegerValue = IntegerValue() + + def clickhouse_table(self): + return "person_distinct_id2" + + +class PersonFieldsOnEvents(Table): + id: StringValue = StringValue() + created_at: DateTimeValue = DateTimeValue() + properties: StringJSONValue = StringJSONValue() + + +class EventsTable(Table): + uuid: StringValue = StringValue() + event: StringValue = StringValue() + timestamp: DateTimeValue = DateTimeValue() + properties: StringJSONValue = StringJSONValue() + elements_chain: StringValue = StringValue() + created_at: DateTimeValue = DateTimeValue() + distinct_id: StringValue = StringValue() + team_id: IntegerValue = IntegerValue() + person: PersonFieldsOnEvents = PersonFieldsOnEvents() + + def clickhouse_table(self): + return "events" + + +# class NumbersTable(Table): +# args: [IntegerValue, IntegerValue] + + +class Database(BaseModel): + class Config: + extra = Extra.forbid + + # All fields below will be tables users can query from + events: EventsTable = EventsTable() + persons: PersonsTable = PersonsTable() + person_distinct_id: PersonDistinctIdTable = PersonDistinctIdTable() + # numbers: NumbersTable = NumbersTable() + + +database = Database() diff --git a/posthog/hogql/hogql.py b/posthog/hogql/hogql.py index 273cf51ac7864..5a10fae468d51 100644 --- a/posthog/hogql/hogql.py +++ b/posthog/hogql/hogql.py @@ -1,8 +1,11 @@ from typing import Literal +from posthog.hogql import ast from posthog.hogql.context import HogQLContext +from posthog.hogql.database import database from posthog.hogql.parser import parse_expr, parse_select from posthog.hogql.printer import print_ast +from posthog.hogql.resolver import resolve_symbols def translate_hogql(query: str, context: HogQLContext, dialect: Literal["hogql", "clickhouse"] = "clickhouse") -> str: @@ -13,10 +16,17 @@ def translate_hogql(query: str, context: HogQLContext, dialect: Literal["hogql", try: if context.select_team_id: node = parse_select(query, no_placeholders=True) + resolve_symbols(node) + return print_ast(node, context, dialect, stack=[]) else: node = parse_expr(query, no_placeholders=True) + symbol = ast.SelectQuerySymbol( + aliases={}, columns={}, tables={"events": ast.TableSymbol(table=database.events)}, anonymous_tables=[] + ) + resolve_symbols(node, symbol) + return print_ast(node, context, dialect, stack=[ast.SelectQuery(select=[], symbol=symbol)]) + except SyntaxError as err: raise ValueError(f"SyntaxError: {err.msg}") except NotImplementedError as err: raise ValueError(f"NotImplementedError: {err}") - return print_ast(node, context, dialect) diff --git a/posthog/hogql/parser.py b/posthog/hogql/parser.py index 2266073968cf0..c9e99c04a70b9 100644 --- a/posthog/hogql/parser.py +++ b/posthog/hogql/parser.py @@ -4,6 +4,7 @@ from antlr4.error.ErrorListener import ErrorListener from posthog.hogql import ast +from posthog.hogql.constants import KEYWORDS, RESERVED_KEYWORDS from posthog.hogql.grammar.HogQLLexer import HogQLLexer from posthog.hogql.grammar.HogQLParser import HogQLParser from posthog.hogql.parse_string import parse_string, parse_string_literal @@ -321,6 +322,10 @@ def visitColumnExprAlias(self, ctx: HogQLParser.ColumnExprAliasContext): else: raise NotImplementedError(f"Must specify an alias.") expr = self.visit(ctx.columnExpr()) + + if alias in RESERVED_KEYWORDS or alias in KEYWORDS: + raise ValueError(f"Alias '{alias}' is a reserved keyword.") + return ast.Alias(expr=expr, alias=alias) def visitColumnExprExtract(self, ctx: HogQLParser.ColumnExprExtractContext): @@ -541,7 +546,10 @@ def visitTableExprSubquery(self, ctx: HogQLParser.TableExprSubqueryContext): return self.visit(ctx.selectUnionStmt()) def visitTableExprAlias(self, ctx: HogQLParser.TableExprAliasContext): - return ast.JoinExpr(table=self.visit(ctx.tableExpr()), alias=self.visit(ctx.alias() or ctx.identifier())) + alias = self.visit(ctx.alias() or ctx.identifier()) + if alias in RESERVED_KEYWORDS or alias in KEYWORDS: + raise ValueError(f"Alias '{alias}' is a reserved keyword.") + return ast.JoinExpr(table=self.visit(ctx.tableExpr()), alias=alias) def visitTableExprFunction(self, ctx: HogQLParser.TableExprFunctionContext): raise NotImplementedError(f"Unsupported node: TableExprFunction") diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index 553456248b9c9..e10a14faa4fde 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -1,27 +1,26 @@ -from typing import List, Literal +from typing import List, Literal, Optional, Union +from ee.clickhouse.materialized_columns.columns import get_materialized_columns from posthog.hogql import ast -from posthog.hogql.constants import ( - CLICKHOUSE_FUNCTIONS, - EVENT_FIELDS, - EVENT_PERSON_FIELDS, - HOGQL_AGGREGATIONS, - KEYWORDS, - MAX_SELECT_RETURNED_ROWS, - SELECT_STAR_FROM_EVENTS_FIELDS, -) +from posthog.hogql.constants import CLICKHOUSE_FUNCTIONS, HOGQL_AGGREGATIONS, MAX_SELECT_RETURNED_ROWS from posthog.hogql.context import HogQLContext, HogQLFieldAccess -from posthog.hogql.parser import parse_expr from posthog.hogql.print_string import print_hogql_identifier from posthog.hogql.visitor import Visitor -def guard_where_team_id(where: ast.Expr, context: HogQLContext) -> ast.Expr: +def guard_where_team_id( + where: Optional[ast.Expr], table_symbol: Union[ast.TableSymbol, ast.TableAliasSymbol], context: HogQLContext +) -> ast.Expr: """Add a mandatory "and(team_id, ...)" filter around the expression.""" if not context.select_team_id: raise ValueError("context.select_team_id not found") - team_clause = parse_expr("team_id = {team_id}", {"team_id": ast.Constant(value=context.select_team_id)}) + team_clause = ast.CompareOperation( + op=ast.CompareOperationType.Eq, + left=ast.Field(chain=["team_id"], symbol=ast.FieldSymbol(name="team_id", table=table_symbol)), + right=ast.Constant(value=context.select_team_id), + ) + if isinstance(where, ast.And): where = ast.And(exprs=[team_clause] + where.exprs) elif where: @@ -31,15 +30,19 @@ def guard_where_team_id(where: ast.Expr, context: HogQLContext) -> ast.Expr: return where -def print_ast(node: ast.AST, context: HogQLContext, dialect: Literal["hogql", "clickhouse"]) -> str: - return Printer(context=context, dialect=dialect).visit(node) +def print_ast( + node: ast.AST, context: HogQLContext, dialect: Literal["hogql", "clickhouse"], stack: List[ast.AST] = [] +) -> str: + return Printer(context=context, dialect=dialect, stack=stack).visit(node) class Printer(Visitor): - def __init__(self, context: HogQLContext, dialect: Literal["hogql", "clickhouse"]): + def __init__( + self, context: HogQLContext, dialect: Literal["hogql", "clickhouse"], stack: Optional[List[ast.AST]] = None + ): self.context = context self.dialect = dialect - self.stack: List[ast.AST] = [] + self.stack: List[ast.AST] = stack or [] def visit(self, node: ast.AST): self.stack.append(node) @@ -51,39 +54,52 @@ def visit_select_query(self, node: ast.SelectQuery): if self.dialect == "clickhouse" and not self.context.select_team_id: raise ValueError("Full SELECT queries are disabled if select_team_id is not set") - columns = [self.visit(column) for column in node.select] if node.select else ["1"] + where = node.where - from_table = None - if node.select_from: - if node.symbol: - if isinstance(node.symbol, ast.TableSymbol): - if node.symbol.table_name != "events": - raise ValueError('Only selecting from the "events" table is supported') - from_table = f"events" - if node.symbol.print_name: - from_table = f"{from_table} AS {node.symbol.print_name}" - elif isinstance(node.symbol, ast.SelectQuerySymbol): - from_table = f"({self.visit(node.select_from.table)})" - if node.symbol.print_name: - from_table = f"{from_table} AS {node.symbol.print_name}" - else: + select_from = None + if node.select_from is not None: + if not node.select_from.symbol: + raise ValueError("Printing queries with a FROM clause is not permitted before symbol resolution") + + if node.select_from.join_expr: + raise ValueError("Printing queries with a JOIN clause is not yet permitted") + + if isinstance(node.select_from.symbol, ast.TableAliasSymbol): + table_symbol = node.select_from.symbol.table + if table_symbol is None: + raise ValueError(f"Table alias {node.select_from.symbol.name} does not resolve!") + if not isinstance(table_symbol, ast.TableSymbol): + raise ValueError(f"Table alias {node.select_from.symbol.name} does not resolve to a table!") + select_from = print_hogql_identifier(table_symbol.table.clickhouse_table()) if node.select_from.alias is not None: - raise ValueError("Table aliases not yet supported") - if isinstance(node.select_from.table, ast.Field): - if node.select_from.table.chain != ["events"]: - raise ValueError('Only selecting from the "events" table is supported') - from_table = "events" - elif isinstance(node.select_from.table, ast.SelectQuery): - from_table = f"({self.visit(node.select_from.table)})" - else: - raise ValueError("Only selecting from a table or a subquery is supported") + select_from += f" AS {print_hogql_identifier(node.select_from.alias)}" + + # Guard with team_id if selecting from a table and printing ClickHouse SQL + # We do this in the printer, and not in a separate step, to be really sure this gets added. + # This will be improved when we add proper table and column alias support. For now, let's just be safe. + if self.dialect == "clickhouse": + where = guard_where_team_id(where, node.select_from.symbol, self.context) + + elif isinstance(node.select_from.symbol, ast.TableSymbol): + select_from = print_hogql_identifier(node.select_from.symbol.table.clickhouse_table()) + + # Guard with team_id if selecting from a table and printing ClickHouse SQL + # We do this in the printer, and not in a separate step, to be really sure this gets added. + # This will be improved when we add proper table and column alias support. For now, let's just be safe. + if self.dialect == "clickhouse": + where = guard_where_team_id(where, node.select_from.symbol, self.context) + + elif isinstance(node.select_from.symbol, ast.SelectQuerySymbol): + select_from = self.visit(node.select_from.table) + + elif isinstance(node.select_from.symbol, ast.SelectQueryAliasSymbol) and node.select_from.alias is not None: + select_from = self.visit(node.select_from.table) + select_from += f" AS {print_hogql_identifier(node.select_from.alias)}" + else: + raise ValueError("Only selecting from a table or a subquery is supported") + + columns = [self.visit(column) for column in node.select] if node.select else ["1"] - where = node.where - # Guard with team_id if selecting from a table and printing ClickHouse SQL - # We do this in the printer, and not in a separate step, to be really sure this gets added. - # This will be improved when we add proper table and column alias support. For now, let's just be safe. - if self.dialect == "clickhouse" and from_table is not None: - where = guard_where_team_id(where, self.context) where = self.visit(where) if where else None having = self.visit(node.having) if node.having else None @@ -103,7 +119,7 @@ def visit_select_query(self, node: ast.SelectQuery): clauses = [ f"SELECT {'DISTINCT ' if node.distinct else ''}{', '.join(columns)}", - f"FROM {from_table}" if from_table else None, + f"FROM {select_from}" if select_from else None, "WHERE " + where if where else None, f"GROUP BY {', '.join(group_by)}" if group_by and len(group_by) > 0 else None, "HAVING " + having if having else None, @@ -206,26 +222,34 @@ def visit_constant(self, node: ast.Constant): ) def visit_field(self, node: ast.Field): + original_field = ".".join([print_hogql_identifier(identifier) for identifier in node.chain]) + if node.symbol is None: + raise ValueError(f"Field {original_field} has no symbol") + if self.dialect == "hogql": # When printing HogQL, we print the properties out as a chain instead of converting them to Clickhouse SQL return ".".join([print_hogql_identifier(identifier) for identifier in node.chain]) - elif node.chain == ["*"]: - query = f"tuple({','.join(SELECT_STAR_FROM_EVENTS_FIELDS)})" - return self.visit(parse_expr(query)) - elif node.chain == ["person"]: - query = "tuple(distinct_id, person.id, person.created_at, person.properties.name, person.properties.email)" - return self.visit(parse_expr(query)) + # elif node.chain == ["*"]: + # query = f"tuple({','.join(SELECT_STAR_FROM_EVENTS_FIELDS)})" + # return self.visit(parse_expr(query)) + # elif node.chain == ["person"]: + # query = "tuple(distinct_id, person.id, person.created_at, person.properties.name, person.properties.email)" + # return self.visit(parse_expr(query)) elif node.symbol is not None: - if isinstance(node.symbol, ast.FieldSymbol): - return f"{node.symbol.table.print_name}.{node.symbol.name}" - elif isinstance(node.symbol, ast.TableSymbol): - return node.symbol.print_name - else: - raise ValueError(f"Unknown Symbol, can not print {type(node.symbol)}") + # find closest select query's symbol for context + select: Optional[ast.SelectQuerySymbol] = None + for stack_node in reversed(self.stack): + if isinstance(stack_node, ast.SelectQuery): + if isinstance(stack_node.symbol, ast.SelectQuerySymbol): + select = stack_node.symbol + break + raise ValueError(f"Closest SelectQuery to field {original_field} has no symbol!") + if select is None: + raise ValueError(f"Can't find SelectQuerySymbol for field: {original_field}") + + return SymbolPrinter(select=select, context=self.context).visit(node.symbol) else: - field_access = parse_field_access(node.chain, self.context) - self.context.field_access_logs.append(field_access) - return field_access.sql + raise ValueError(f"Unknown Symbol, can not print {type(node.symbol)}") def visit_call(self, node: ast.Call): if node.name in HOGQL_AGGREGATIONS: @@ -249,6 +273,7 @@ def visit_call(self, node: ast.Call): return f"{node.name}({translated_args})" elif node.name == "count": return "count(*)" + # TODO: rework these elif node.name == "countDistinct": return f"count(distinct {translated_args})" elif node.name == "countDistinctIf": @@ -264,68 +289,93 @@ def visit_call(self, node: ast.Call): def visit_placeholder(self, node: ast.Placeholder): raise ValueError(f"Found a Placeholder {{{node.field}}} in the tree. Can't generate query!") + def visit_alias(self, node: ast.Alias): + return f"{self.visit(node.expr)} AS {print_hogql_identifier(node.alias)}" + def visit_unknown(self, node: ast.AST): raise ValueError(f"Unknown AST node {type(node).__name__}") -def parse_field_access(chain: List[str], context: HogQLContext) -> HogQLFieldAccess: - # Circular import otherwise - from posthog.models.property.util import get_property_string_expr - - """Given a list like ['properties', '$browser'] or ['uuid'], translate to the correct ClickHouse expr.""" - if len(chain) == 2: - if chain[0] == "properties": - key = f"hogql_val_{len(context.values)}" - context.values[key] = chain[1] - escaped_key = f"%({key})s" - expression, _ = get_property_string_expr( - "events", - chain[1], - escaped_key, - "properties", - ) - return HogQLFieldAccess(chain, "event.properties", chain[1], expression) - elif chain[0] == "person": - if chain[1] in EVENT_PERSON_FIELDS: - return HogQLFieldAccess(chain, "person", chain[1], f"person_{chain[1]}") - else: - raise ValueError(f"Unknown person field '{chain[1]}'") - elif len(chain) == 3 and chain[0] == "person" and chain[1] == "properties": - key = f"hogql_val_{len(context.values or {})}" - context.values[key] = chain[2] - escaped_key = f"%({key})s" - - if context.using_person_on_events: - expression, _ = get_property_string_expr( - "events", - chain[2], - escaped_key, - "person_properties", - materialised_table_column="person_properties", +class SymbolPrinter(Visitor): + def __init__(self, select: ast.SelectQuerySymbol, context: HogQLContext): + self.select = select + self.context = context + + def visit_table_symbol(self, symbol: ast.TableSymbol): + return print_hogql_identifier(symbol.table.clickhouse_table()) + + def visit_table_alias_symbol(self, symbol: ast.TableAliasSymbol): + return f"{self.visit(symbol.table)} AS {print_hogql_identifier(symbol.name)}" + + def visit_field_symbol(self, symbol: ast.FieldSymbol): + # do we need a table prefix? + table_prefix = self.visit(symbol.table) + printed_field = print_hogql_identifier(symbol.name) + + field_sql = printed_field + + # Field exists as a column name in this scope. Is it the same field? + if symbol.name in self.select.columns: + column_symbol = self.select.columns[symbol.name] + if column_symbol != symbol: + field_sql = f"{table_prefix}.{printed_field}" + + # Field exists as an alias name in this scope. Is it the same field? + if symbol.name in self.select.aliases: + aliased_symbol = self.select.aliases[symbol.name] + if aliased_symbol != symbol: + field_sql = f"{table_prefix}.{printed_field}" + + if printed_field != "properties": + self.context.field_access_logs.append( + HogQLFieldAccess( + [symbol.name], + "event", + symbol.name, + field_sql, + ) ) + return field_sql + + def visit_property_symbol(self, symbol: ast.PropertySymbol): + key = f"hogql_val_{len(self.context.values)}" + self.context.values[key] = symbol.name + + table = symbol.field.table + if isinstance(table, ast.TableAliasSymbol): + table = table.table + + # TODO: cache this + materialized_columns = get_materialized_columns(table.table.clickhouse_table()) + materialized_column = materialized_columns.get((symbol.name, "properties"), None) + + if materialized_column: + property_sql = print_hogql_identifier(materialized_column) else: - expression, _ = get_property_string_expr( - "person", - chain[2], - escaped_key, - "person_props", - materialised_table_column="properties", + field_sql = self.visit(symbol.field) + property_sql = trim_quotes_expr(f"JSONExtractRaw({field_sql}, %({key})s)") + + self.context.field_access_logs.append( + HogQLFieldAccess( + ["properties", symbol.name], + "event.properties", + symbol.name, + property_sql, ) + ) + + return property_sql + + def visit_select_query_alias_symbol(self, symbol: ast.SelectQueryAliasSymbol): + return print_hogql_identifier(symbol.name) + + def visit_column_alias_symbol(self, symbol: ast.SelectQueryAliasSymbol): + return print_hogql_identifier(symbol.name) + + def visit_unknown(self, symbol: ast.AST): + raise ValueError(f"Unknown Symbol {type(symbol).__name__}") - return HogQLFieldAccess(chain, "person.properties", chain[2], expression) - elif len(chain) == 1: - if chain[0] in EVENT_FIELDS: - if chain[0] == "id": - return HogQLFieldAccess(chain, "event", "uuid", "uuid") - elif chain[0] == "properties": - return HogQLFieldAccess(chain, "event", "properties", "properties") - return HogQLFieldAccess(chain, "event", chain[0], chain[0]) - elif chain[0].startswith("person_") and chain[0][7:] in EVENT_PERSON_FIELDS: - return HogQLFieldAccess(chain, "person", chain[0][7:], chain[0]) - elif chain[0].lower() in KEYWORDS: - return HogQLFieldAccess(chain, None, None, chain[0].lower()) - else: - raise ValueError(f"Unknown event field '{chain[0]}'") - raise ValueError(f"Unsupported property access: {chain}") +def trim_quotes_expr(expr: str) -> str: + return f"replaceRegexpAll({expr}, '^\"|\"$', '')" diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py index fb0ce78b41aef..8e3cd16f1351a 100644 --- a/posthog/hogql/resolver.py +++ b/posthog/hogql/resolver.py @@ -1,13 +1,14 @@ -from typing import Dict, List, Optional +from typing import List, Optional from posthog.hogql import ast +from posthog.hogql.database import database from posthog.hogql.visitor import TraversingVisitor # https://github.com/ClickHouse/ClickHouse/issues/23194 - "Describe how identifiers in SELECT queries are resolved" -def resolve_symbols(node: ast.SelectQuery): - Resolver().visit(node) +def resolve_symbols(node: ast.Expr, scope: Optional[ast.SelectQuerySymbol] = None): + Resolver(scope=scope).visit(node) class ResolverException(ValueError): @@ -17,22 +18,20 @@ class ResolverException(ValueError): class Resolver(TraversingVisitor): """The Resolver visits an AST and assigns Symbols to the nodes.""" - def __init__(self): - self.scopes: List[ast.SelectQuerySymbol] = [] - self.global_tables: Dict[str, ast.Symbol] = {} + def __init__(self, scope: Optional[ast.SelectQuerySymbol] = None): + self.scopes: List[ast.SelectQuerySymbol] = [scope] if scope else [] def visit_select_query(self, node): """Visit each SELECT query or subquery.""" - if node.symbol is not None: return - # Create a new lexical scope each time we enter a SELECT query. - node.symbol = ast.SelectQuerySymbol(aliases={}, columns={}, tables={}) - # Keep those scopes stacked in a list as we traverse the tree. + node.symbol = ast.SelectQuerySymbol(aliases={}, columns={}, tables={}, anonymous_tables=[]) + + # Each SELECT query creates a new scope. Store all of them in a list for variable access. self.scopes.append(node.symbol) - # Visit all the FROM and JOIN tables (JoinExpr nodes) + # Visit all the FROM and JOIN clauses (JoinExpr nodes), and register the tables into the scope. if node.select_from: self.visit(node.select_from) @@ -77,41 +76,34 @@ def visit_join_expr(self, node): scope = self.scopes[-1] if isinstance(node.table, ast.Field): - if node.alias is None: - # Make sure there is a way to call the field in the scope. - node.alias = node.table.chain[0] - if node.alias in scope.tables: - raise ResolverException(f'Already have a joined table called "{node.alias}", can\'t redefine.') - - # Only joining the events table is supported - if node.table.chain == ["events"]: - print_name = f"{node.alias[0:1] or 't'}{len(self.global_tables)}" - node.table.symbol = ast.TableSymbol(table_name="events", print_name=print_name) - self.global_tables[print_name] = node.table.symbol - if node.alias == node.table.symbol.table_name: - node.symbol = node.table.symbol - else: - node.symbol = ast.TableAliasSymbol(name=node.alias, symbol=node.table.symbol) + table_name = node.table.chain[0] + table_alias = node.alias or table_name + if table_alias in scope.tables: + raise ResolverException(f'Already have a joined table called "{table_alias}", can\'t redefine.') + + if table_name in database.__fields__: + table = database.__fields__[table_name].default + node.symbol = ast.TableSymbol(table=table) + if table_alias != table_name: + node.symbol = ast.TableAliasSymbol(name=table_alias, table=node.symbol) + scope.tables[table_alias] = node.symbol else: - raise ResolverException(f"Cannot resolve table {node.table.chain[0]}") + raise ResolverException(f'Unknown table "{table_name}". Only "events" is supported.') elif isinstance(node.table, ast.SelectQuery): node.table.symbol = self.visit(node.table) - if node.alias is None: - node.symbol = node.table.symbol + if node.alias is not None: + if node.alias in scope.tables: + raise ResolverException(f'Already have a joined table called "{node.alias}", can\'t redefine.') + node.symbol = ast.SelectQueryAliasSymbol(name=node.alias, symbol=node.table.symbol) + scope.tables[node.alias] = node.symbol else: - print_name = self._new_global_table_print_name(node.alias) - node.symbol = ast.TableAliasSymbol(name=node.alias, symbol=node.table.symbol, print_name=print_name) - self.global_tables[print_name] = node.symbol + node.symbol = node.table.symbol + scope.anonymous_tables.append(node.symbol) else: raise ResolverException(f"JoinExpr with table of type {type(node.table).__name__} not supported") - scope.tables[node.alias] = node.symbol - - # node.symbol.print_name = self._new_global_table_print_name(node.alias) - # self.global_tables[node.symbol.print_name] = node.symbol - self.visit(node.join_expr) def visit_alias(self, node: ast.Alias): @@ -128,9 +120,19 @@ def visit_alias(self, node: ast.Alias): raise ResolverException("Alias cannot be empty") self.visit(node.expr) - node.symbol = ast.ColumnAliasSymbol(name=node.alias, symbol=unwrap_column_alias_symbol(node.expr.symbol)) + if not node.expr.symbol: + raise ResolverException(f"Cannot alias an expression without a symbol: {node.alias}") + node.symbol = ast.ColumnAliasSymbol(name=node.alias, symbol=node.expr.symbol) scope.aliases[node.alias] = node.symbol + def visit_call(self, node: ast.Call): + """Visit function calls.""" + if node.symbol is not None: + return + for arg in node.args: + self.visit(arg) + node.symbol = ast.CallSymbol(name=node.name, args=[arg.symbol for arg in node.args]) + def visit_field(self, node): """Visit a field such as ast.Field(chain=["e", "properties", "$browser"])""" if node.symbol is not None: @@ -142,15 +144,15 @@ def visit_field(self, node): # "SELECT event, (select count() from events where event = x.event) as c FROM events x where event = '$pageview'", # # But this is supported: - # "SELECT t.big_count FROM (select count() + 100 as big_count from events) as t", + # "SELECT t.big_count FROM (select count() + 100 as big_count from events) as t JOIN events e ON (e.event = t.event)", # # Thus only look into the current scope, for columns and aliases. scope = self.scopes[-1] symbol: Optional[ast.Symbol] = None name = node.chain[0] + # More than one field and the first one is a table. Found the first match. if len(node.chain) > 1 and name in scope.tables: - # If the field has a chain of at least one (e.g "e", "event"), the first part could refer to a table. symbol = scope.tables[name] elif name in scope.columns: symbol = scope.columns[name] @@ -158,14 +160,17 @@ def visit_field(self, node): symbol = scope.aliases[name] else: # Look through all FROM/JOIN tables, if they export a field by this name. - fields_in_scope = [table.get_child(name) for table in scope.tables.values() if table.has_child(name)] - if len(fields_in_scope) > 1: - raise ResolverException(f'Ambiguous query. Found multiple sources for field "{name}".') - elif len(fields_in_scope) == 1: - symbol = fields_in_scope[0] + tables_with_field = [table for table in scope.tables.values() if table.has_child(name)] + [ + table for table in scope.anonymous_tables if table.has_child(name) + ] + if len(tables_with_field) > 1: + raise ResolverException(f"Ambiguous query. Found multiple sources for field: {name}") + elif len(tables_with_field) == 1: + # accessed a field on a joined table by name + symbol = tables_with_field[0].get_child(name) if not symbol: - raise ResolverException(f'Cannot resolve symbol: "{name}"') + raise ResolverException(f"Unable to resolve field: {name}") # Recursively resolve the rest of the chain until we can point to the deepest node. for child_name in node.chain[1:]: @@ -183,9 +188,6 @@ def visit_constant(self, node): return node.symbol = ast.ConstantSymbol(value=node.value) - def _new_global_table_print_name(self, table_name): - return f"{table_name[0:1] or 't'}{len(self.global_tables)}" - def unwrap_column_alias_symbol(symbol: ast.Symbol) -> ast.Symbol: i = 0 @@ -195,3 +197,6 @@ def unwrap_column_alias_symbol(symbol: ast.Symbol) -> ast.Symbol: if i > 100: raise ResolverException("ColumnAliasSymbol recursion too deep!") return symbol + + +# select a from (select 1 as a) diff --git a/posthog/hogql/test/test_printer.py b/posthog/hogql/test/test_printer.py index e1e67e8106497..5aea6a45c0747 100644 --- a/posthog/hogql/test/test_printer.py +++ b/posthog/hogql/test/test_printer.py @@ -169,10 +169,17 @@ def test_materialized_fields_and_properties(self): self.assertEqual(1 + 2, 3) return materialize("events", "$browser") - self.assertEqual(self._expr("properties['$browser']"), '"mat_$browser"') + self.assertEqual(self._expr("properties['$browser']"), "mat_$browser") + materialize("events", "$browser and string") + self.assertEqual(self._expr("properties['$browser and string']"), "mat_$browser_and_string") + + materialize("events", "$browser%%%#@!@") + self.assertEqual(self._expr("properties['$browser%%%#@!@']"), "mat_$browser_______") + + # TODO: get person properties working materialize("events", "$initial_waffle", table_column="person_properties") - self.assertEqual(self._expr("person.properties['$initial_waffle']"), '"mat_pp_$initial_waffle"') + self.assertEqual(self._expr("person.properties['$initial_waffle']"), "mat_pp_$initial_waffle") def test_methods(self): self.assertEqual(self._expr("count()"), "count(*)") @@ -189,19 +196,19 @@ def test_functions(self): def test_expr_parse_errors(self): self._assert_expr_error("", "Empty query") - self._assert_expr_error("avg(bla)", "Unknown event field 'bla'") + self._assert_expr_error("avg(bla)", "Unable to resolve field: bla") self._assert_expr_error("count(2)", "Aggregation 'count' requires 0 arguments, found 1") self._assert_expr_error("count(2,4)", "Aggregation 'count' requires 0 arguments, found 2") self._assert_expr_error("countIf()", "Aggregation 'countIf' requires 1 argument, found 0") self._assert_expr_error("countIf(2,4)", "Aggregation 'countIf' requires 1 argument, found 2") - self._assert_expr_error("hamburger(bla)", "Unsupported function call 'hamburger(...)'") - self._assert_expr_error("mad(bla)", "Unsupported function call 'mad(...)'") - self._assert_expr_error("yeet.the.cloud", "Unsupported property access: ['yeet', 'the', 'cloud']") - self._assert_expr_error("chipotle", "Unknown event field 'chipotle'") - self._assert_expr_error("person.chipotle", "Unknown person field 'chipotle'") + self._assert_expr_error("hamburger(event)", "Unsupported function call 'hamburger(...)'") + self._assert_expr_error("mad(event)", "Unsupported function call 'mad(...)'") + self._assert_expr_error("yeet.the.cloud", "Unable to resolve field: yeet") + self._assert_expr_error("chipotle", "Unable to resolve field: chipotle") self._assert_expr_error( "avg(avg(properties.bla))", "Aggregation 'avg' cannot be nested inside another aggregation 'avg'." ) + self._assert_expr_error("person.chipotle", "Unknown person field 'chipotle'") def test_expr_syntax_errors(self): self._assert_expr_error("(", "line 1, column 1: no viable alternative at input '('") @@ -211,8 +218,9 @@ def test_expr_syntax_errors(self): self._assert_expr_error( "select query from events", "line 1, column 13: mismatched input 'from' expecting " ) - self._assert_expr_error("this makes little sense", "Unknown AST node Alias") - self._assert_expr_error("event makes little sense", "Unknown AST node Alias") + self._assert_expr_error("this makes little sense", "Unable to resolve field: this") + # TODO: fix + # self._assert_expr_error("event makes little sense", "event AS makes AS little AS sense") self._assert_expr_error("1;2", "line 1, column 1: mismatched input ';' expecting") self._assert_expr_error("b.a(bla)", "SyntaxError: line 1, column 3: mismatched input '(' expecting '.'") @@ -340,9 +348,14 @@ def test_values(self): ) self.assertEqual(context.values, {"hogql_val_0": "E", "hogql_val_1": "lol", "hogql_val_2": "hoo"}) - def test_no_alias_yet(self): - self._assert_expr_error("1 as team_id", "Unknown AST node Alias") - self._assert_expr_error("1 as `-- select team_id`", "Unknown AST node Alias") + def test_alias_keywords(self): + self._assert_expr_error("1 as team_id", "Alias 'team_id' is a reserved keyword.") + self._assert_expr_error("1 as true", "Alias 'true' is a reserved keyword.") + self._assert_select_error("select 1 as team_id from events", "Alias 'team_id' is a reserved keyword.") + self.assertEqual( + self._select("select 1 as `-- select team_id` from events"), + "SELECT 1 AS `-- select team_id` FROM events WHERE equals(team_id, 42) LIMIT 65535", + ) def test_select(self): self.assertEqual(self._select("select 1"), "SELECT 1 LIMIT 65535") @@ -355,14 +368,16 @@ def test_select(self): def test_select_alias(self): # currently not supported! - self._assert_select_error("select 1 as b", "Unknown AST node Alias") - self._assert_select_error("select 1 from events as e", "Table aliases not yet supported") + self.assertEqual(self._select("select 1 as b"), "SELECT 1 AS b LIMIT 65535") + self.assertEqual( + self._select("select 1 from events as e"), "SELECT 1 FROM events AS e WHERE equals(team_id, 42) LIMIT 65535" + ) def test_select_from(self): self.assertEqual( self._select("select 1 from events"), "SELECT 1 FROM events WHERE equals(team_id, 42) LIMIT 65535" ) - self._assert_select_error("select 1 from other", 'Only selecting from the "events" table is supported') + self._assert_select_error("select 1 from other", 'Unknown table "other". Only "events" is supported.') def test_select_where(self): self.assertEqual( @@ -446,3 +461,13 @@ def test_select_distinct(self): self._select("select distinct event from events group by event, timestamp"), "SELECT DISTINCT event FROM events WHERE equals(team_id, 42) GROUP BY event, timestamp LIMIT 65535", ) + + def test_select_subquery(self): + self.assertEqual( + self._select("SELECT event from (select distinct event from events group by event, timestamp)"), + "SELECT event FROM (SELECT DISTINCT event FROM events WHERE equals(team_id, 42) GROUP BY event, timestamp) LIMIT 65535", + ) + self.assertEqual( + self._select("SELECT event from (select distinct event from events group by event, timestamp) e"), + "SELECT event FROM (SELECT DISTINCT event FROM events WHERE equals(team_id, 42) GROUP BY event, timestamp) AS e LIMIT 65535", + ) diff --git a/posthog/hogql/test/test_query.py b/posthog/hogql/test/test_query.py index 1aec779f121ed..fdf5c263286e1 100644 --- a/posthog/hogql/test/test_query.py +++ b/posthog/hogql/test/test_query.py @@ -21,4 +21,16 @@ def test_query(self): ) flush_persons_and_events() response = execute_hogql_query("select count(), event from events group by event", self.team) + + response = execute_hogql_query( + "select c.count, event from (select count() as count, event from events) as c group by count, event", + self.team, + ) + + self.assertEqual( + response.clickhouse, + "SELECT count(*), e0.event FROM events e0 WHERE equals(e0.team_id, 1) GROUP BY e0.event LIMIT 1000", + ) + self.assertEqual(response.hogql, "SELECT count(), event FROM events e0 GROUP BY event") self.assertEqual(response.results, [(2, "random event")]) + # self.assertEqual(response.types, []) diff --git a/posthog/hogql/test/test_resolver.py b/posthog/hogql/test/test_resolver.py index bffe494cb1946..0c522322151fe 100644 --- a/posthog/hogql/test/test_resolver.py +++ b/posthog/hogql/test/test_resolver.py @@ -1,4 +1,5 @@ from posthog.hogql import ast +from posthog.hogql.database import database from posthog.hogql.parser import parse_select from posthog.hogql.resolver import ResolverException, resolve_symbols from posthog.test.base import BaseTest @@ -9,13 +10,14 @@ def test_resolve_events_table(self): expr = parse_select("SELECT event, events.timestamp FROM events WHERE events.event = 'test'") resolve_symbols(expr) - events_table_symbol = ast.TableSymbol(table_name="events") + events_table_symbol = ast.TableSymbol(table=database.events) event_field_symbol = ast.FieldSymbol(name="event", table=events_table_symbol) timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=events_table_symbol) select_query_symbol = ast.SelectQuerySymbol( aliases={}, columns={"event": event_field_symbol, "timestamp": timestamp_field_symbol}, tables={"events": events_table_symbol}, + anonymous_tables=[], ) expected = ast.SelectQuery( @@ -47,13 +49,14 @@ def test_resolve_events_table_alias(self): expr = parse_select("SELECT event, e.timestamp FROM events e WHERE e.event = 'test'") resolve_symbols(expr) - events_table_symbol = ast.TableSymbol(table_name="events") + events_table_symbol = ast.TableSymbol(table=database.events) event_field_symbol = ast.FieldSymbol(name="event", table=events_table_symbol) timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=events_table_symbol) select_query_symbol = ast.SelectQuerySymbol( aliases={}, columns={"event": event_field_symbol, "timestamp": timestamp_field_symbol}, tables={"e": ast.TableAliasSymbol(name="e", symbol=events_table_symbol)}, + anonymous_tables=[], ) expected = ast.SelectQuery( @@ -85,7 +88,7 @@ def test_resolve_events_table_column_alias(self): expr = parse_select("SELECT event as ee, ee, ee as e, e.timestamp FROM events e WHERE e.event = 'test'") resolve_symbols(expr) - events_table_symbol = ast.TableSymbol(table_name="events") + events_table_symbol = ast.TableSymbol(table=database.events) event_field_symbol = ast.FieldSymbol(name="event", table=events_table_symbol) timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=events_table_symbol) @@ -100,6 +103,7 @@ def test_resolve_events_table_column_alias(self): "timestamp": timestamp_field_symbol, }, tables={"e": ast.TableAliasSymbol(name="e", symbol=events_table_symbol)}, + anonymous_tables=[], ) expected = ast.SelectQuery( @@ -139,21 +143,24 @@ def test_resolve_events_table_column_alias(self): def test_resolve_events_table_column_alias_inside_subquery(self): expr = parse_select("SELECT b FROM (select event as b, timestamp as c from events) e WHERE e.b = 'test'") resolve_symbols(expr) - events_table_symbol = ast.TableSymbol(table_name="events") - event_field_symbol = ast.FieldSymbol(name="event", table=events_table_symbol) - timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=events_table_symbol) + outer_events_table_symbol = ast.TableSymbol(table=database.events) + inner_events_table_symbol = ast.TableAliasSymbol(name="e1", symbol=ast.TableSymbol(table=database.events)) + outer_event_field_symbol = ast.FieldSymbol(name="event", table=outer_events_table_symbol) + inner_event_field_symbol = ast.FieldSymbol(name="event", table=inner_events_table_symbol) + timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=outer_events_table_symbol) inner_select_symbol = ast.SelectQuerySymbol( aliases={ - "b": ast.ColumnAliasSymbol(name="b", symbol=event_field_symbol), + "b": ast.ColumnAliasSymbol(name="b", symbol=outer_event_field_symbol), "c": ast.ColumnAliasSymbol(name="c", symbol=timestamp_field_symbol), }, columns={ - "b": ast.ColumnAliasSymbol(name="b", symbol=event_field_symbol), + "b": ast.ColumnAliasSymbol(name="b", symbol=outer_events_table_symbol), "c": ast.ColumnAliasSymbol(name="c", symbol=timestamp_field_symbol), }, tables={ - "events": events_table_symbol, + "events": outer_events_table_symbol, }, + anonymous_tables=[], ) expected = ast.SelectQuery( select=[ @@ -161,7 +168,7 @@ def test_resolve_events_table_column_alias_inside_subquery(self): chain=["b"], symbol=ast.ColumnAliasSymbol( name="b", - symbol=event_field_symbol, + symbol=outer_event_field_symbol, ), ), ], @@ -170,10 +177,10 @@ def test_resolve_events_table_column_alias_inside_subquery(self): select=[ ast.Alias( alias="b", - expr=ast.Field(chain=["event"], symbol=event_field_symbol), + expr=ast.Field(chain=["event"], symbol=inner_event_field_symbol), symbol=ast.ColumnAliasSymbol( name="b", - symbol=event_field_symbol, + symbol=inner_event_field_symbol, ), ), ast.Alias( @@ -186,9 +193,9 @@ def test_resolve_events_table_column_alias_inside_subquery(self): ), ], select_from=ast.JoinExpr( - table=ast.Field(chain=["events"], symbol=events_table_symbol), + table=ast.Field(chain=["events"], symbol=inner_events_table_symbol), alias="events", - symbol=events_table_symbol, + symbol=inner_events_table_symbol, ), symbol=inner_select_symbol, ), @@ -200,7 +207,7 @@ def test_resolve_events_table_column_alias_inside_subquery(self): chain=["e", "b"], symbol=ast.ColumnAliasSymbol( name="b", - symbol=event_field_symbol, + symbol=outer_event_field_symbol, ), ), op=ast.CompareOperationType.Eq, @@ -209,7 +216,7 @@ def test_resolve_events_table_column_alias_inside_subquery(self): symbol=ast.SelectQuerySymbol( aliases={}, columns={ - "b": ast.ColumnAliasSymbol(name="b", symbol=event_field_symbol), + "b": ast.ColumnAliasSymbol(name="b", symbol=outer_event_field_symbol), }, tables={"e": ast.TableAliasSymbol(name="e", symbol=inner_select_symbol)}, ), @@ -228,7 +235,7 @@ def test_resolve_subquery_no_field_access(self): ) with self.assertRaises(ResolverException) as e: resolve_symbols(expr) - self.assertEqual(str(e.exception), 'Cannot resolve symbol: "e"') + self.assertEqual(str(e.exception), "Unable to resolve field: e") def test_resolve_errors(self): queries = [ @@ -241,7 +248,7 @@ def test_resolve_errors(self): for query in queries: with self.assertRaises(ResolverException) as e: resolve_symbols(parse_select(query)) - self.assertIn("Cannot resolve symbol", str(e.exception)) + self.assertIn("Unable to resolve field: x", str(e.exception)) # "with 2 as a select 1 as a" -> "Different expressions with the same alias a:" From 076916e37db2d9c38988dba6c52837a789a98d23 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 13 Feb 2023 15:39:25 +0100 Subject: [PATCH 22/81] some sample queries --- posthog/hogql/test/test_query.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/posthog/hogql/test/test_query.py b/posthog/hogql/test/test_query.py index fdf5c263286e1..459fb93a52e58 100644 --- a/posthog/hogql/test/test_query.py +++ b/posthog/hogql/test/test_query.py @@ -23,14 +23,17 @@ def test_query(self): response = execute_hogql_query("select count(), event from events group by event", self.team) response = execute_hogql_query( - "select c.count, event from (select count() as count, event from events) as c group by count, event", + "select c.count, event from (select count() as count, event from events group by event) as c group by count, event", self.team, ) self.assertEqual( response.clickhouse, - "SELECT count(*), e0.event FROM events e0 WHERE equals(e0.team_id, 1) GROUP BY e0.event LIMIT 1000", + "SELECT count, event FROM (SELECT count(*) AS count, event FROM events WHERE equals(team_id, 1) GROUP BY event) AS c GROUP BY count, event LIMIT 1000", + ) + self.assertEqual( + response.hogql, + "SELECT c.count, event FROM (SELECT count() AS count, event FROM events GROUP BY event) AS c GROUP BY count, event LIMIT 1000", ) - self.assertEqual(response.hogql, "SELECT count(), event FROM events e0 GROUP BY event") self.assertEqual(response.results, [(2, "random event")]) # self.assertEqual(response.types, []) From fd271ddba733aa661670a2c1c8482df7b1647b5f Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 13 Feb 2023 16:06:00 +0100 Subject: [PATCH 23/81] query tests --- posthog/hogql/test/test_query.py | 42 +++++++++++++++++++------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/posthog/hogql/test/test_query.py b/posthog/hogql/test/test_query.py index 459fb93a52e58..b54595a0de3d3 100644 --- a/posthog/hogql/test/test_query.py +++ b/posthog/hogql/test/test_query.py @@ -1,39 +1,47 @@ from freezegun import freeze_time from posthog.hogql.query import execute_hogql_query +from posthog.models.utils import UUIDT from posthog.test.base import APIBaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events class TestQuery(ClickhouseTestMixin, APIBaseTest): def test_query(self): with freeze_time("2020-01-10"): - _create_event( - distinct_id="bla", - event="random event", - team=self.team, - properties={"random_prop": "don't include", "some other prop": "with some text"}, - ) - _create_event( - distinct_id="bla", - event="random event", - team=self.team, - properties={"random_prop": "don't include", "some other prop": "with some text"}, - ) + random_uuid = str(UUIDT()) + for index in range(2): + _create_event( + distinct_id="bla", + event="random event", + team=self.team, + properties={"random_prop": "don't include", "random_uuid": random_uuid, "index": index}, + ) flush_persons_and_events() - response = execute_hogql_query("select count(), event from events group by event", self.team) response = execute_hogql_query( - "select c.count, event from (select count() as count, event from events group by event) as c group by count, event", + f"select count(), event from events where properties.random_uuid = '{random_uuid}' group by event", self.team, ) + self.assertEqual( + response.clickhouse, + f"SELECT count(*), event FROM events WHERE and(equals(team_id, {self.team.id}), equals(replaceRegexpAll(JSONExtractRaw(properties, %(hogql_val_0)s), '^\"|\"$', ''), %(hogql_val_1)s)) GROUP BY event LIMIT 1000", + ) + self.assertEqual( + response.hogql, + "SELECT count(), event FROM events WHERE equals(properties.random_uuid, %(hogql_val_2)s) GROUP BY event LIMIT 1000", + ) + self.assertEqual(response.results, [(2, "random event")]) + response = execute_hogql_query( + f"select count, event from (select count() as count, event from events where properties.random_uuid = '{random_uuid}' group by event) as c group by count, event", + self.team, + ) self.assertEqual( response.clickhouse, - "SELECT count, event FROM (SELECT count(*) AS count, event FROM events WHERE equals(team_id, 1) GROUP BY event) AS c GROUP BY count, event LIMIT 1000", + f"SELECT count, event FROM (SELECT count(*) AS count, event FROM events WHERE and(equals(team_id, {self.team.id}), equals(replaceRegexpAll(JSONExtractRaw(properties, %(hogql_val_0)s), '^\"|\"$', ''), %(hogql_val_1)s)) GROUP BY event) AS c GROUP BY count, event LIMIT 1000", ) self.assertEqual( response.hogql, - "SELECT c.count, event FROM (SELECT count() AS count, event FROM events GROUP BY event) AS c GROUP BY count, event LIMIT 1000", + "SELECT count, event FROM (SELECT count() AS count, event FROM events WHERE equals(properties.random_uuid, %(hogql_val_2)s) GROUP BY event) AS c GROUP BY count, event LIMIT 1000", ) self.assertEqual(response.results, [(2, "random event")]) - # self.assertEqual(response.types, []) From b41055e69c81c36851a334c099da72485b387b17 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 13 Feb 2023 16:38:28 +0100 Subject: [PATCH 24/81] test selecting from persons and distinct id table --- posthog/hogql/ast.py | 13 ++++--- posthog/hogql/test/test_query.py | 59 ++++++++++++++++++++++++++------ 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index 3471092833ac9..d79eb6ad208ae 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -4,7 +4,7 @@ from pydantic import BaseModel, Extra -from posthog.hogql.database import Table, database +from posthog.hogql.database import StringJSONValue, Table # NOTE: when you add new AST fields or nodes, add them to CloningVisitor as well! @@ -111,13 +111,16 @@ class FieldSymbol(Symbol): table: Union[TableSymbol, TableAliasSymbol] def get_child(self, name: str) -> Symbol: - if self.table.table == database.events: - if self.name == "properties": + db_table = self.table.table + if isinstance(db_table, Table): + if self.name in db_table.__fields__ and isinstance(db_table.__fields__[self.name].default, StringJSONValue): return PropertySymbol(name=name, field=self) else: - raise NotImplementedError(f"Can not resolve field {self.name} on table events") + raise NotImplementedError( + f"Can not access property {name} on field {self.name} because it's not a JSON field" + ) else: - raise NotImplementedError(f"Can not resolve fields on table: {self.name}") + raise NotImplementedError(f"Can not resolve table for field: {self.name}") class ConstantSymbol(Symbol): diff --git a/posthog/hogql/test/test_query.py b/posthog/hogql/test/test_query.py index b54595a0de3d3..06f8a9156a5fe 100644 --- a/posthog/hogql/test/test_query.py +++ b/posthog/hogql/test/test_query.py @@ -2,21 +2,32 @@ from posthog.hogql.query import execute_hogql_query from posthog.models.utils import UUIDT -from posthog.test.base import APIBaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events +from posthog.test.base import APIBaseTest, ClickhouseTestMixin, _create_event, _create_person, flush_persons_and_events class TestQuery(ClickhouseTestMixin, APIBaseTest): + def _create_random_events(self) -> str: + random_uuid = str(UUIDT()) + _create_person( + properties={"email": "tim@posthog.com", "random_uuid": random_uuid}, + team=self.team, + distinct_ids=["bla"], + is_identified=True, + ) + flush_persons_and_events() + for index in range(2): + _create_event( + distinct_id="bla", + event="random event", + team=self.team, + properties={"random_prop": "don't include", "random_uuid": random_uuid, "index": index}, + ) + flush_persons_and_events() + return random_uuid + def test_query(self): with freeze_time("2020-01-10"): - random_uuid = str(UUIDT()) - for index in range(2): - _create_event( - distinct_id="bla", - event="random event", - team=self.team, - properties={"random_prop": "don't include", "random_uuid": random_uuid, "index": index}, - ) - flush_persons_and_events() + random_uuid = self._create_random_events() response = execute_hogql_query( f"select count(), event from events where properties.random_uuid = '{random_uuid}' group by event", @@ -45,3 +56,31 @@ def test_query(self): "SELECT count, event FROM (SELECT count() AS count, event FROM events WHERE equals(properties.random_uuid, %(hogql_val_2)s) GROUP BY event) AS c GROUP BY count, event LIMIT 1000", ) self.assertEqual(response.results, [(2, "random event")]) + + response = execute_hogql_query( + f"select distinct properties.email from persons where properties.random_uuid = '{random_uuid}'", + self.team, + ) + self.assertEqual( + response.clickhouse, + f"SELECT DISTINCT replaceRegexpAll(JSONExtractRaw(properties, %(hogql_val_0)s), '^\"|\"$', '') FROM person WHERE and(equals(team_id, {self.team.id}), equals(replaceRegexpAll(JSONExtractRaw(properties, %(hogql_val_1)s), '^\"|\"$', ''), %(hogql_val_2)s)) LIMIT 1000", + ) + self.assertEqual( + response.hogql, + "SELECT DISTINCT properties.email FROM person WHERE equals(properties.random_uuid, %(hogql_val_3)s) LIMIT 1000", + ) + self.assertEqual(response.results, [("tim@posthog.com",)]) + + response = execute_hogql_query( + f"select distinct person_id, distinct_id from person_distinct_id", + self.team, + ) + self.assertEqual( + response.clickhouse, + f"SELECT DISTINCT person_id, distinct_id FROM person_distinct_id2 WHERE equals(team_id, {self.team.id}) LIMIT 1000", + ) + self.assertEqual( + response.hogql, + "SELECT DISTINCT person_id, distinct_id FROM person_distinct_id2 LIMIT 1000", + ) + self.assertTrue(len(response.results) > 0) From 309a7645572d7f55e3f7d23b80d44f41d9245e54 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 13 Feb 2023 21:27:17 +0100 Subject: [PATCH 25/81] document current join ast --- posthog/hogql/parser.py | 5 +--- posthog/hogql/test/test_parser.py | 46 +++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/posthog/hogql/parser.py b/posthog/hogql/parser.py index c9e99c04a70b9..955b18427dd5e 100644 --- a/posthog/hogql/parser.py +++ b/posthog/hogql/parser.py @@ -181,16 +181,13 @@ def visitJoinExprCrossOp(self, ctx: HogQLParser.JoinExprCrossOpContext): def visitJoinOpInner(self, ctx: HogQLParser.JoinOpInnerContext): tokens = [] - if ctx.LEFT(): - tokens.append("INNER") if ctx.ALL(): tokens.append("ALL") - if ctx.ANTI(): - tokens.append("ANTI") if ctx.ANY(): tokens.append("ANY") if ctx.ASOF(): tokens.append("ASOF") + tokens.append("INNER") return " ".join(tokens) def visitJoinOpLeftRight(self, ctx: HogQLParser.JoinOpLeftRightContext): diff --git a/posthog/hogql/test/test_parser.py b/posthog/hogql/test/test_parser.py index 0313923c96c18..740482c3a7587 100644 --- a/posthog/hogql/test/test_parser.py +++ b/posthog/hogql/test/test_parser.py @@ -537,6 +537,52 @@ def test_select_from_join(self): ), ), ) + self.assertEqual( + parse_select( + """ + SELECT event, timestamp, e.distinct_id, p.id, p.properties.email + FROM events e + LEFT JOIN person_distinct_id pdi + ON pdi.distinct_id = e.distinct_id + LEFT JOIN persons p + ON p.id = pdi.person_id + """, + self.team, + ), + ast.SelectQuery( + select=[ + ast.Field(chain=["event"]), + ast.Field(chain=["timestamp"]), + ast.Field(chain=["e", "distinct_id"]), + ast.Field(chain=["p", "id"]), + ast.Field(chain=["p", "properties", "email"]), + ], + select_from=ast.JoinExpr( + table=ast.Field(chain=["events"]), + alias="e", + join_type="LEFT JOIN", + join_constraint=ast.CompareOperation( + op=ast.CompareOperationType.Eq, + left=ast.Field(chain=["pdi", "distinct_id"]), + right=ast.Field(chain=["e", "distinct_id"]), + ), + join_expr=ast.JoinExpr( + table=ast.Field(chain=["person_distinct_id"]), + alias="pdi", + join_type="LEFT JOIN", + join_constraint=ast.CompareOperation( + op=ast.CompareOperationType.Eq, + left=ast.Field(chain=["p", "id"]), + right=ast.Field(chain=["pdi", "person_id"]), + ), + join_expr=ast.JoinExpr( + table=ast.Field(chain=["persons"]), + alias="p", + ), + ), + ), + ), + ) def test_select_group_by(self): self.assertEqual( From 592a18cdef1a557200c6b9b52ec07435a2a3e0ca Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 13 Feb 2023 21:28:23 +0100 Subject: [PATCH 26/81] children, joins and aliases --- posthog/hogql/ast.py | 13 +++- posthog/hogql/constants.py | 2 + posthog/hogql/printer.py | 127 ++++++++++++++++--------------- posthog/hogql/resolver.py | 49 +++++------- posthog/hogql/test/test_query.py | 46 +++++++++++ 5 files changed, 145 insertions(+), 92 deletions(-) diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index d79eb6ad208ae..784c31222ec36 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -72,12 +72,14 @@ class TableAliasSymbol(Symbol): name: str table: TableSymbol - def get_child(self, name: str) -> Symbol: - return self.table.get_child(name) - def has_child(self, name: str) -> bool: return self.table.has_child(name) + def get_child(self, name: str) -> Symbol: + if self.has_child(name): + return FieldSymbol(name=name, table=self) + return self.table.get_child(name) + class SelectQuerySymbol(Symbol): # all aliases a select query has access to in its scope @@ -111,7 +113,10 @@ class FieldSymbol(Symbol): table: Union[TableSymbol, TableAliasSymbol] def get_child(self, name: str) -> Symbol: - db_table = self.table.table + table_symbol = self.table + while isinstance(table_symbol, TableAliasSymbol): + table_symbol = table_symbol.table + db_table = table_symbol.table if isinstance(db_table, Table): if self.name in db_table.__fields__ and isinstance(db_table.__fields__[self.name].default, StringJSONValue): return PropertySymbol(name=name, field=self) diff --git a/posthog/hogql/constants.py b/posthog/hogql/constants.py index 1cb107817c21c..0a42c31f7d500 100644 --- a/posthog/hogql/constants.py +++ b/posthog/hogql/constants.py @@ -91,6 +91,8 @@ "avgIf": 2, "any": 1, "anyIf": 2, + "argMax": 2, + "argMin": 2, } # Keywords passed to ClickHouse without transformation KEYWORDS = ["true", "false", "null"] diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index e10a14faa4fde..959a1fe7553d3 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -5,6 +5,7 @@ from posthog.hogql.constants import CLICKHOUSE_FUNCTIONS, HOGQL_AGGREGATIONS, MAX_SELECT_RETURNED_ROWS from posthog.hogql.context import HogQLContext, HogQLFieldAccess from posthog.hogql.print_string import print_hogql_identifier +from posthog.hogql.resolver import ResolverException, lookup_field_by_name from posthog.hogql.visitor import Visitor @@ -44,6 +45,12 @@ def __init__( self.dialect = dialect self.stack: List[ast.AST] = stack or [] + def _last_select(self) -> Optional[ast.SelectQuery]: + for node in reversed(self.stack): + if isinstance(node, ast.SelectQuery): + return node + return None + def visit(self, node: ast.AST): self.stack.append(node) response = super().visit(node) @@ -56,47 +63,11 @@ def visit_select_query(self, node: ast.SelectQuery): where = node.where - select_from = None + select_from = "" if node.select_from is not None: if not node.select_from.symbol: raise ValueError("Printing queries with a FROM clause is not permitted before symbol resolution") - - if node.select_from.join_expr: - raise ValueError("Printing queries with a JOIN clause is not yet permitted") - - if isinstance(node.select_from.symbol, ast.TableAliasSymbol): - table_symbol = node.select_from.symbol.table - if table_symbol is None: - raise ValueError(f"Table alias {node.select_from.symbol.name} does not resolve!") - if not isinstance(table_symbol, ast.TableSymbol): - raise ValueError(f"Table alias {node.select_from.symbol.name} does not resolve to a table!") - select_from = print_hogql_identifier(table_symbol.table.clickhouse_table()) - if node.select_from.alias is not None: - select_from += f" AS {print_hogql_identifier(node.select_from.alias)}" - - # Guard with team_id if selecting from a table and printing ClickHouse SQL - # We do this in the printer, and not in a separate step, to be really sure this gets added. - # This will be improved when we add proper table and column alias support. For now, let's just be safe. - if self.dialect == "clickhouse": - where = guard_where_team_id(where, node.select_from.symbol, self.context) - - elif isinstance(node.select_from.symbol, ast.TableSymbol): - select_from = print_hogql_identifier(node.select_from.symbol.table.clickhouse_table()) - - # Guard with team_id if selecting from a table and printing ClickHouse SQL - # We do this in the printer, and not in a separate step, to be really sure this gets added. - # This will be improved when we add proper table and column alias support. For now, let's just be safe. - if self.dialect == "clickhouse": - where = guard_where_team_id(where, node.select_from.symbol, self.context) - - elif isinstance(node.select_from.symbol, ast.SelectQuerySymbol): - select_from = self.visit(node.select_from.table) - - elif isinstance(node.select_from.symbol, ast.SelectQueryAliasSymbol) and node.select_from.alias is not None: - select_from = self.visit(node.select_from.table) - select_from += f" AS {print_hogql_identifier(node.select_from.alias)}" - else: - raise ValueError("Only selecting from a table or a subquery is supported") + select_from = self.visit(node.select_from) columns = [self.visit(column) for column in node.select] if node.select else ["1"] @@ -140,6 +111,52 @@ def visit_select_query(self, node: ast.SelectQuery): response = f"({response})" return response + def visit_join_expr(self, node: ast.JoinExpr) -> str: + select_from = [] + if isinstance(node.symbol, ast.TableAliasSymbol): + table_symbol = node.symbol.table + if table_symbol is None: + raise ValueError(f"Table alias {node.symbol.name} does not resolve!") + if not isinstance(table_symbol, ast.TableSymbol): + raise ValueError(f"Table alias {node.symbol.name} does not resolve to a table!") + select_from.append(print_hogql_identifier(table_symbol.table.clickhouse_table())) + if node.alias is not None: + select_from.append(f"AS {print_hogql_identifier(node.alias)}") + + # Guard with team_id if selecting from a table and printing ClickHouse SQL + # We do this in the printer, and not in a separate step, to be really sure this gets added. + # This will be improved when we add proper table and column alias support. For now, let's just be safe. + if self.dialect == "clickhouse": + select = self._last_select() + select.where = guard_where_team_id(select.where, node.symbol, self.context) + + elif isinstance(node.symbol, ast.TableSymbol): + select_from.append(print_hogql_identifier(node.symbol.table.clickhouse_table())) + + # Guard with team_id if selecting from a table and printing ClickHouse SQL + # We do this in the printer, and not in a separate step, to be really sure this gets added. + # This will be improved when we add proper table and column alias support. For now, let's just be safe. + if self.dialect == "clickhouse": + select = self._last_select() + select.where = guard_where_team_id(select.where, node.symbol, self.context) + + elif isinstance(node.symbol, ast.SelectQuerySymbol): + select_from.append(self.visit(node.table)) + + elif isinstance(node.symbol, ast.SelectQueryAliasSymbol) and node.alias is not None: + select_from.append(self.visit(node.table)) + select_from.append(f"AS {print_hogql_identifier(node.alias)}") + else: + raise ValueError("Only selecting from a table or a subquery is supported") + + if node.join_type is not None and node.join_expr is not None: + select_from.append(f"{node.join_type} {self.visit(node.join_expr)}") + + if node.join_constraint is not None: + select_from.append(f"ON {self.visit(node.join_constraint)}") + + return " ".join(select_from) + def visit_binary_operation(self, node: ast.BinaryOperation): if node.op == ast.BinaryOperationType.Add: return f"plus({self.visit(node.left)}, {self.visit(node.right)})" @@ -236,17 +253,10 @@ def visit_field(self, node: ast.Field): # query = "tuple(distinct_id, person.id, person.created_at, person.properties.name, person.properties.email)" # return self.visit(parse_expr(query)) elif node.symbol is not None: - # find closest select query's symbol for context - select: Optional[ast.SelectQuerySymbol] = None - for stack_node in reversed(self.stack): - if isinstance(stack_node, ast.SelectQuery): - if isinstance(stack_node.symbol, ast.SelectQuerySymbol): - select = stack_node.symbol - break - raise ValueError(f"Closest SelectQuery to field {original_field} has no symbol!") + select_query = self._last_select() + select: Optional[ast.SelectQuerySymbol] = select_query.symbol if select_query else None if select is None: raise ValueError(f"Can't find SelectQuerySymbol for field: {original_field}") - return SymbolPrinter(select=select, context=self.context).visit(node.symbol) else: raise ValueError(f"Unknown Symbol, can not print {type(node.symbol)}") @@ -305,28 +315,25 @@ def visit_table_symbol(self, symbol: ast.TableSymbol): return print_hogql_identifier(symbol.table.clickhouse_table()) def visit_table_alias_symbol(self, symbol: ast.TableAliasSymbol): - return f"{self.visit(symbol.table)} AS {print_hogql_identifier(symbol.name)}" + return print_hogql_identifier(symbol.name) def visit_field_symbol(self, symbol: ast.FieldSymbol): # do we need a table prefix? table_prefix = self.visit(symbol.table) printed_field = print_hogql_identifier(symbol.name) - field_sql = printed_field - - # Field exists as a column name in this scope. Is it the same field? - if symbol.name in self.select.columns: - column_symbol = self.select.columns[symbol.name] - if column_symbol != symbol: - field_sql = f"{table_prefix}.{printed_field}" + try: + symbol_with_name_in_scope = lookup_field_by_name(self.select, symbol.name) + except ResolverException: + symbol_with_name_in_scope = None - # Field exists as an alias name in this scope. Is it the same field? - if symbol.name in self.select.aliases: - aliased_symbol = self.select.aliases[symbol.name] - if aliased_symbol != symbol: - field_sql = f"{table_prefix}.{printed_field}" + if symbol_with_name_in_scope == symbol: + field_sql = printed_field + else: + field_sql = f"{table_prefix}.{printed_field}" if printed_field != "properties": + # TODO: refactor this property access logging self.context.field_access_logs.append( HogQLFieldAccess( [symbol.name], diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py index 8e3cd16f1351a..27a9f5fc72011 100644 --- a/posthog/hogql/resolver.py +++ b/posthog/hogql/resolver.py @@ -88,7 +88,7 @@ def visit_join_expr(self, node): node.symbol = ast.TableAliasSymbol(name=table_alias, table=node.symbol) scope.tables[table_alias] = node.symbol else: - raise ResolverException(f'Unknown table "{table_name}". Only "events" is supported.') + raise ResolverException(f'Unknown table "{table_name}".') elif isinstance(node.table, ast.SelectQuery): node.table.symbol = self.visit(node.table) @@ -105,6 +105,7 @@ def visit_join_expr(self, node): raise ResolverException(f"JoinExpr with table of type {type(node.table).__name__} not supported") self.visit(node.join_expr) + self.visit(node.join_constraint) def visit_alias(self, node: ast.Alias): """Visit column aliases. SELECT 1, (select 3 as y) as x.""" @@ -151,24 +152,11 @@ def visit_field(self, node): symbol: Optional[ast.Symbol] = None name = node.chain[0] - # More than one field and the first one is a table. Found the first match. + # Only look for matching tables if field contains at least two parts. if len(node.chain) > 1 and name in scope.tables: symbol = scope.tables[name] - elif name in scope.columns: - symbol = scope.columns[name] - elif name in scope.aliases: - symbol = scope.aliases[name] - else: - # Look through all FROM/JOIN tables, if they export a field by this name. - tables_with_field = [table for table in scope.tables.values() if table.has_child(name)] + [ - table for table in scope.anonymous_tables if table.has_child(name) - ] - if len(tables_with_field) > 1: - raise ResolverException(f"Ambiguous query. Found multiple sources for field: {name}") - elif len(tables_with_field) == 1: - # accessed a field on a joined table by name - symbol = tables_with_field[0].get_child(name) - + if not symbol: + symbol = lookup_field_by_name(scope, name) if not symbol: raise ResolverException(f"Unable to resolve field: {name}") @@ -189,14 +177,19 @@ def visit_constant(self, node): node.symbol = ast.ConstantSymbol(value=node.value) -def unwrap_column_alias_symbol(symbol: ast.Symbol) -> ast.Symbol: - i = 0 - while isinstance(symbol, ast.ColumnAliasSymbol): - symbol = symbol.symbol - i += 1 - if i > 100: - raise ResolverException("ColumnAliasSymbol recursion too deep!") - return symbol - - -# select a from (select 1 as a) +def lookup_field_by_name(scope: ast.SelectQuerySymbol, name: str) -> Optional[ast.Symbol]: + if name in scope.columns: + return scope.columns[name] + elif name in scope.aliases: + return scope.aliases[name] + else: + named_tables = [table for table in scope.tables.values() if table.has_child(name)] + anonymous_tables = [table for table in scope.anonymous_tables if table.has_child(name)] + tables = named_tables + anonymous_tables + + if len(tables) > 1: + raise ResolverException(f"Ambiguous query. Found multiple sources for field: {name}") + elif len(tables) == 1: + # accessed a field on a joined table by name + return tables[0].get_child(name) + return None diff --git a/posthog/hogql/test/test_query.py b/posthog/hogql/test/test_query.py index 06f8a9156a5fe..27ad2b99f6e03 100644 --- a/posthog/hogql/test/test_query.py +++ b/posthog/hogql/test/test_query.py @@ -84,3 +84,49 @@ def test_query(self): "SELECT DISTINCT person_id, distinct_id FROM person_distinct_id2 LIMIT 1000", ) self.assertTrue(len(response.results) > 0) + + def test_query_joins_simple(self): + with freeze_time("2020-01-10"): + self._create_random_events() + + response = execute_hogql_query( + """ + SELECT event, timestamp, e.distinct_id, p.id, p.properties.email + FROM events e + LEFT JOIN person_distinct_id pdi + ON pdi.distinct_id = e.distinct_id + LEFT JOIN persons p + ON p.id = pdi.person_id + """, + self.team, + ) + self.assertEqual( + response.clickhouse, + f"", + ) + + def test_query_joins(self): + with freeze_time("2020-01-10"): + self._create_random_events() + + response = execute_hogql_query( + """select event, timestamp, pdi.person_id from events e INNER JOIN ( + SELECT distinct_id, + argMax(person_id, version) as person_id + FROM person_distinct_id + WHERE team_id = 1 + GROUP BY distinct_id + HAVING argMax(is_deleted, version) = 0 + ) AS pdi + ON e.distinct_id = pdi.distinct_id""", + self.team, + ) + self.assertEqual( + response.clickhouse, + f"SELECT DISTINCT person_id, distinct_id FROM person_distinct_id2 WHERE equals(team_id, {self.team.id}) LIMIT 1000", + ) + self.assertEqual( + response.hogql, + "SELECT DISTINCT person_id, distinct_id FROM person_distinct_id LIMIT 1000", + ) + self.assertTrue(len(response.results) > 0) From 72da1d92c2cd664d165a47874e92abf76b2cc391 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 13 Feb 2023 22:04:29 +0100 Subject: [PATCH 27/81] turn joins around --- posthog/hogql/ast.py | 8 +-- posthog/hogql/parser.py | 16 +++--- posthog/hogql/printer.py | 27 ++++++---- posthog/hogql/resolver.py | 4 +- posthog/hogql/test/test_parser.py | 81 +++++++++++++++++------------- posthog/hogql/test/test_printer.py | 2 +- posthog/hogql/test/test_visitor.py | 14 +++--- posthog/hogql/visitor.py | 8 +-- 8 files changed, 87 insertions(+), 73 deletions(-) diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index 784c31222ec36..d50181ba1196c 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -222,12 +222,12 @@ class Call(Expr): class JoinExpr(Expr): + join_type: Optional[str] = None table: Optional[Union["SelectQuery", Field]] = None - table_final: Optional[bool] = None alias: Optional[str] = None - join_type: Optional[str] = None - join_constraint: Optional[Expr] = None - join_expr: Optional["JoinExpr"] = None + table_final: Optional[bool] = None + constraint: Optional[Expr] = None + next_join: Optional["JoinExpr"] = None class SelectQuery(Expr): diff --git a/posthog/hogql/parser.py b/posthog/hogql/parser.py index 955b18427dd5e..f0893f9aa6eb5 100644 --- a/posthog/hogql/parser.py +++ b/posthog/hogql/parser.py @@ -148,18 +148,16 @@ def visitJoinExprOp(self, ctx: HogQLParser.JoinExprOpContext): join2: ast.JoinExpr = self.visit(ctx.joinExpr(1)) if ctx.joinOp(): - join_type = f"{self.visit(ctx.joinOp())} JOIN" + join2.join_type = f"{self.visit(ctx.joinOp())} JOIN" else: - join_type = "JOIN" - join_constraint = self.visit(ctx.joinConstraintClause()) + join2.join_type = "JOIN" + join2.constraint = self.visit(ctx.joinConstraintClause()) - join_without_next_expr = join1 - while join_without_next_expr.join_expr: - join_without_next_expr = join_without_next_expr.join_expr + last_join = join1 + while last_join.next_join is not None: + last_join = last_join.next_join + last_join.next_join = join2 - join_without_next_expr.join_expr = join2 - join_without_next_expr.join_constraint = join_constraint - join_without_next_expr.join_type = join_type return join1 def visitJoinExprTable(self, ctx: HogQLParser.JoinExprTableContext): diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index 959a1fe7553d3..f327a082a4e23 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -61,18 +61,15 @@ def visit_select_query(self, node: ast.SelectQuery): if self.dialect == "clickhouse" and not self.context.select_team_id: raise ValueError("Full SELECT queries are disabled if select_team_id is not set") - where = node.where - select_from = "" - if node.select_from is not None: + if isinstance(node.select_from, ast.JoinExpr): if not node.select_from.symbol: raise ValueError("Printing queries with a FROM clause is not permitted before symbol resolution") select_from = self.visit(node.select_from) columns = [self.visit(column) for column in node.select] if node.select else ["1"] - where = self.visit(where) if where else None - + where = self.visit(node.where) if node.where else None having = self.visit(node.having) if node.having else None prewhere = self.visit(node.prewhere) if node.prewhere else None group_by = [self.visit(column) for column in node.group_by] if node.group_by else None @@ -113,6 +110,9 @@ def visit_select_query(self, node: ast.SelectQuery): def visit_join_expr(self, node: ast.JoinExpr) -> str: select_from = [] + if node.join_type is not None: + select_from.append(node.join_type) + if isinstance(node.symbol, ast.TableAliasSymbol): table_symbol = node.symbol.table if table_symbol is None: @@ -128,7 +128,8 @@ def visit_join_expr(self, node: ast.JoinExpr) -> str: # This will be improved when we add proper table and column alias support. For now, let's just be safe. if self.dialect == "clickhouse": select = self._last_select() - select.where = guard_where_team_id(select.where, node.symbol, self.context) + if select is not None: + select.where = guard_where_team_id(select.where, node.symbol, self.context) elif isinstance(node.symbol, ast.TableSymbol): select_from.append(print_hogql_identifier(node.symbol.table.clickhouse_table())) @@ -138,7 +139,8 @@ def visit_join_expr(self, node: ast.JoinExpr) -> str: # This will be improved when we add proper table and column alias support. For now, let's just be safe. if self.dialect == "clickhouse": select = self._last_select() - select.where = guard_where_team_id(select.where, node.symbol, self.context) + if select is not None: + select.where = guard_where_team_id(select.where, node.symbol, self.context) elif isinstance(node.symbol, ast.SelectQuerySymbol): select_from.append(self.visit(node.table)) @@ -149,11 +151,14 @@ def visit_join_expr(self, node: ast.JoinExpr) -> str: else: raise ValueError("Only selecting from a table or a subquery is supported") - if node.join_type is not None and node.join_expr is not None: - select_from.append(f"{node.join_type} {self.visit(node.join_expr)}") + if node.table_final: + select_from.append("FINAL") + + if node.constraint is not None: + select_from.append(f"ON {self.visit(node.constraint)}") - if node.join_constraint is not None: - select_from.append(f"ON {self.visit(node.join_constraint)}") + if node.next_join is not None: + select_from.append(self.visit(node.next_join)) return " ".join(select_from) diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py index 27a9f5fc72011..f2be08fdbadf1 100644 --- a/posthog/hogql/resolver.py +++ b/posthog/hogql/resolver.py @@ -104,8 +104,8 @@ def visit_join_expr(self, node): else: raise ResolverException(f"JoinExpr with table of type {type(node.table).__name__} not supported") - self.visit(node.join_expr) - self.visit(node.join_constraint) + self.visit(node.constraint) + self.visit(node.next_join) def visit_alias(self, node: ast.Alias): """Visit column aliases. SELECT 1, (select 3 as y) as x.""" diff --git a/posthog/hogql/test/test_parser.py b/posthog/hogql/test/test_parser.py index 740482c3a7587..8c51fc282e1f7 100644 --- a/posthog/hogql/test/test_parser.py +++ b/posthog/hogql/test/test_parser.py @@ -502,9 +502,11 @@ def test_select_from_join(self): select=[ast.Constant(value=1)], select_from=ast.JoinExpr( table=ast.Field(chain=["events"]), - join_type="JOIN", - join_constraint=ast.Constant(value=1), - join_expr=ast.JoinExpr(table=ast.Field(chain=["events2"])), + next_join=ast.JoinExpr( + join_type="JOIN", + table=ast.Field(chain=["events2"]), + constraint=ast.Constant(value=1), + ), ), ), ) @@ -514,9 +516,11 @@ def test_select_from_join(self): select=[ast.Field(chain=["*"])], select_from=ast.JoinExpr( table=ast.Field(chain=["events"]), - join_type="LEFT OUTER JOIN", - join_constraint=ast.Constant(value=1), - join_expr=ast.JoinExpr(table=ast.Field(chain=["events2"])), + next_join=ast.JoinExpr( + join_type="LEFT OUTER JOIN", + table=ast.Field(chain=["events2"]), + constraint=ast.Constant(value=1), + ), ), ), ) @@ -526,29 +530,34 @@ def test_select_from_join(self): select=[ast.Constant(value=1)], select_from=ast.JoinExpr( table=ast.Field(chain=["events"]), - join_type="LEFT OUTER JOIN", - join_constraint=ast.Constant(value=1), - join_expr=ast.JoinExpr( + next_join=ast.JoinExpr( + join_type="LEFT OUTER JOIN", table=ast.Field(chain=["events2"]), - join_type="RIGHT ANY JOIN", - join_constraint=ast.Constant(value=2), - join_expr=ast.JoinExpr(table=ast.Field(chain=["events3"])), + constraint=ast.Constant(value=1), + next_join=ast.JoinExpr( + join_type="RIGHT ANY JOIN", + table=ast.Field(chain=["events3"]), + constraint=ast.Constant(value=2), + ), ), ), ), ) - self.assertEqual( - parse_select( - """ - SELECT event, timestamp, e.distinct_id, p.id, p.properties.email - FROM events e - LEFT JOIN person_distinct_id pdi - ON pdi.distinct_id = e.distinct_id - LEFT JOIN persons p - ON p.id = pdi.person_id - """, - self.team, - ), + + def test_select_from_join_multiple(self): + node = parse_select( + """ + SELECT event, timestamp, e.distinct_id, p.id, p.properties.email + FROM events e + LEFT JOIN person_distinct_id pdi + ON pdi.distinct_id = e.distinct_id + LEFT JOIN persons p + ON p.id = pdi.person_id + """, + self.team, + ) + self.assertEqual( + node, ast.SelectQuery( select=[ ast.Field(chain=["event"]), @@ -560,24 +569,24 @@ def test_select_from_join(self): select_from=ast.JoinExpr( table=ast.Field(chain=["events"]), alias="e", - join_type="LEFT JOIN", - join_constraint=ast.CompareOperation( - op=ast.CompareOperationType.Eq, - left=ast.Field(chain=["pdi", "distinct_id"]), - right=ast.Field(chain=["e", "distinct_id"]), - ), - join_expr=ast.JoinExpr( + next_join=ast.JoinExpr( + join_type="LEFT JOIN", table=ast.Field(chain=["person_distinct_id"]), alias="pdi", - join_type="LEFT JOIN", - join_constraint=ast.CompareOperation( + constraint=ast.CompareOperation( op=ast.CompareOperationType.Eq, - left=ast.Field(chain=["p", "id"]), - right=ast.Field(chain=["pdi", "person_id"]), + left=ast.Field(chain=["pdi", "distinct_id"]), + right=ast.Field(chain=["e", "distinct_id"]), ), - join_expr=ast.JoinExpr( + next_join=ast.JoinExpr( + join_type="LEFT JOIN", table=ast.Field(chain=["persons"]), alias="p", + constraint=ast.CompareOperation( + op=ast.CompareOperationType.Eq, + left=ast.Field(chain=["p", "id"]), + right=ast.Field(chain=["pdi", "person_id"]), + ), ), ), ), diff --git a/posthog/hogql/test/test_printer.py b/posthog/hogql/test/test_printer.py index 5aea6a45c0747..fe082d123cc22 100644 --- a/posthog/hogql/test/test_printer.py +++ b/posthog/hogql/test/test_printer.py @@ -377,7 +377,7 @@ def test_select_from(self): self.assertEqual( self._select("select 1 from events"), "SELECT 1 FROM events WHERE equals(team_id, 42) LIMIT 65535" ) - self._assert_select_error("select 1 from other", 'Unknown table "other". Only "events" is supported.') + self._assert_select_error("select 1 from other", 'Unknown table "other".') def test_select_where(self): self.assertEqual( diff --git a/posthog/hogql/test/test_visitor.py b/posthog/hogql/test/test_visitor.py index 0a67144f659cc..dee4fe474bca4 100644 --- a/posthog/hogql/test/test_visitor.py +++ b/posthog/hogql/test/test_visitor.py @@ -71,13 +71,15 @@ def test_everything_visitor(self): table=ast.Field(chain=["b"]), table_final=True, alias="c", - join_type="INNER", - join_constraint=ast.CompareOperation( - op=ast.CompareOperationType.Eq, - left=ast.Field(chain=["d"]), - right=ast.Field(chain=["e"]), + next_join=ast.JoinExpr( + join_type="INNER", + table=ast.Field(chain=["f"]), + constraint=ast.CompareOperation( + op=ast.CompareOperationType.Eq, + left=ast.Field(chain=["d"]), + right=ast.Field(chain=["e"]), + ), ), - join_expr=ast.JoinExpr(table=ast.Field(chain=["f"])), ), where=ast.Constant(value=True), prewhere=ast.Constant(value=True), diff --git a/posthog/hogql/visitor.py b/posthog/hogql/visitor.py index fce103f70e28f..b5628f5d7287c 100644 --- a/posthog/hogql/visitor.py +++ b/posthog/hogql/visitor.py @@ -54,8 +54,8 @@ def visit_call(self, node: ast.Call): def visit_join_expr(self, node: ast.JoinExpr): self.visit(node.table) - self.visit(node.join_expr) - self.visit(node.join_constraint) + self.visit(node.constraint) + self.visit(node.next_join) def visit_select_query(self, node: ast.SelectQuery): self.visit(node.select_from) @@ -133,11 +133,11 @@ def visit_call(self, node: ast.Call): def visit_join_expr(self, node: ast.JoinExpr): return ast.JoinExpr( table=self.visit(node.table), - join_expr=self.visit(node.join_expr), + next_join=self.visit(node.next_join), table_final=node.table_final, alias=node.alias, join_type=node.join_type, - join_constraint=self.visit(node.join_constraint), + constraint=self.visit(node.constraint), ) def visit_select_query(self, node: ast.SelectQuery): From 54e55398d948e892ce631d8b328631df80e1ec7a Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 13 Feb 2023 23:03:11 +0100 Subject: [PATCH 28/81] get some actual joins working --- posthog/hogql/printer.py | 63 ++++++++++++++++++-------------- posthog/hogql/resolver.py | 4 +- posthog/hogql/test/test_query.py | 11 +++++- 3 files changed, 46 insertions(+), 32 deletions(-) diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index f327a082a4e23..3bdffc29d0804 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -61,15 +61,30 @@ def visit_select_query(self, node: ast.SelectQuery): if self.dialect == "clickhouse" and not self.context.select_team_id: raise ValueError("Full SELECT queries are disabled if select_team_id is not set") - select_from = "" - if isinstance(node.select_from, ast.JoinExpr): - if not node.select_from.symbol: + # we will add extra clauses onto this + where = node.where + + select_from = [] + next_join = node.select_from + while isinstance(next_join, ast.JoinExpr): + if next_join.symbol is None: raise ValueError("Printing queries with a FROM clause is not permitted before symbol resolution") - select_from = self.visit(node.select_from) - columns = [self.visit(column) for column in node.select] if node.select else ["1"] + (select_sql, extra_where) = self.visit_join_expr(next_join) + select_from.append(select_sql) - where = self.visit(node.where) if node.where else None + if extra_where is not None: + if where is None: + where = extra_where + elif isinstance(where, ast.And): + where = ast.And(exprs=where.exprs + [extra_where]) + else: + where = ast.And(exprs=[where, extra_where]) + + next_join = next_join.next_join + + columns = [self.visit(column) for column in node.select] if node.select else ["1"] + where = self.visit(where) if where else None having = self.visit(node.having) if node.having else None prewhere = self.visit(node.prewhere) if node.prewhere else None group_by = [self.visit(column) for column in node.group_by] if node.group_by else None @@ -87,7 +102,7 @@ def visit_select_query(self, node: ast.SelectQuery): clauses = [ f"SELECT {'DISTINCT ' if node.distinct else ''}{', '.join(columns)}", - f"FROM {select_from}" if select_from else None, + f"FROM {' '.join(select_from)}" if len(select_from) > 0 else None, "WHERE " + where if where else None, f"GROUP BY {', '.join(group_by)}" if group_by and len(group_by) > 0 else None, "HAVING " + having if having else None, @@ -108,7 +123,10 @@ def visit_select_query(self, node: ast.SelectQuery): response = f"({response})" return response - def visit_join_expr(self, node: ast.JoinExpr) -> str: + def visit_join_expr(self, node: ast.JoinExpr) -> (str, Optional[ast.Expr]): + # return constraints we must place on the select query + extra_where = None + select_from = [] if node.join_type is not None: select_from.append(node.join_type) @@ -123,24 +141,14 @@ def visit_join_expr(self, node: ast.JoinExpr) -> str: if node.alias is not None: select_from.append(f"AS {print_hogql_identifier(node.alias)}") - # Guard with team_id if selecting from a table and printing ClickHouse SQL - # We do this in the printer, and not in a separate step, to be really sure this gets added. - # This will be improved when we add proper table and column alias support. For now, let's just be safe. if self.dialect == "clickhouse": - select = self._last_select() - if select is not None: - select.where = guard_where_team_id(select.where, node.symbol, self.context) + extra_where = guard_where_team_id(None, node.symbol, self.context) elif isinstance(node.symbol, ast.TableSymbol): select_from.append(print_hogql_identifier(node.symbol.table.clickhouse_table())) - # Guard with team_id if selecting from a table and printing ClickHouse SQL - # We do this in the printer, and not in a separate step, to be really sure this gets added. - # This will be improved when we add proper table and column alias support. For now, let's just be safe. if self.dialect == "clickhouse": - select = self._last_select() - if select is not None: - select.where = guard_where_team_id(select.where, node.symbol, self.context) + extra_where = guard_where_team_id(None, node.symbol, self.context) elif isinstance(node.symbol, ast.SelectQuerySymbol): select_from.append(self.visit(node.table)) @@ -157,10 +165,7 @@ def visit_join_expr(self, node: ast.JoinExpr) -> str: if node.constraint is not None: select_from.append(f"ON {self.visit(node.constraint)}") - if node.next_join is not None: - select_from.append(self.visit(node.next_join)) - - return " ".join(select_from) + return (" ".join(select_from), extra_where) def visit_binary_operation(self, node: ast.BinaryOperation): if node.op == ast.BinaryOperationType.Add: @@ -332,10 +337,14 @@ def visit_field_symbol(self, symbol: ast.FieldSymbol): except ResolverException: symbol_with_name_in_scope = None - if symbol_with_name_in_scope == symbol: - field_sql = printed_field - else: + if ( + symbol_with_name_in_scope != symbol + or isinstance(symbol.table, ast.TableAliasSymbol) + or isinstance(symbol.table, ast.SelectQueryAliasSymbol) + ): field_sql = f"{table_prefix}.{printed_field}" + else: + field_sql = printed_field if printed_field != "properties": # TODO: refactor this property access logging diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py index f2be08fdbadf1..94a7d5b510507 100644 --- a/posthog/hogql/resolver.py +++ b/posthog/hogql/resolver.py @@ -178,9 +178,7 @@ def visit_constant(self, node): def lookup_field_by_name(scope: ast.SelectQuerySymbol, name: str) -> Optional[ast.Symbol]: - if name in scope.columns: - return scope.columns[name] - elif name in scope.aliases: + if name in scope.aliases: return scope.aliases[name] else: named_tables = [table for table in scope.tables.values() if table.has_child(name)] diff --git a/posthog/hogql/test/test_query.py b/posthog/hogql/test/test_query.py index 27ad2b99f6e03..dc329218df99b 100644 --- a/posthog/hogql/test/test_query.py +++ b/posthog/hogql/test/test_query.py @@ -91,7 +91,7 @@ def test_query_joins_simple(self): response = execute_hogql_query( """ - SELECT event, timestamp, e.distinct_id, p.id, p.properties.email + SELECT event, timestamp, pdi.distinct_id, p.id, p.properties.email FROM events e LEFT JOIN person_distinct_id pdi ON pdi.distinct_id = e.distinct_id @@ -102,8 +102,15 @@ def test_query_joins_simple(self): ) self.assertEqual( response.clickhouse, - f"", + f"SELECT e.event, e.timestamp, pdi.distinct_id, p.id, replaceRegexpAll(JSONExtractRaw(p.properties, %(hogql_val_0)s), '^\"|\"$', '') FROM events AS e LEFT JOIN person_distinct_id2 AS pdi ON equals(pdi.distinct_id, e.distinct_id) LEFT JOIN person AS p ON equals(p.id, pdi.person_id) WHERE and(equals(e.team_id, {self.team.id}), equals(pdi.team_id, {self.team.id}), equals(p.team_id, {self.team.id})) LIMIT 1000", ) + self.assertEqual( + response.hogql, + "SELECT event, timestamp, pdi.distinct_id, p.id, p.properties.email FROM events AS e LEFT JOIN person_distinct_id2 AS pdi ON equals(pdi.distinct_id, e.distinct_id) LEFT JOIN person AS p ON equals(p.id, pdi.person_id) LIMIT 1000", + ) + self.assertEqual(response.results[0][0], "random event") + self.assertEqual(response.results[0][2], "bla") + self.assertEqual(response.results[0][4], "tim@posthog.com") def test_query_joins(self): with freeze_time("2020-01-10"): From 95b6a5e240126180b1e217c7c2386ef4026cbb17 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 13 Feb 2023 23:31:06 +0100 Subject: [PATCH 29/81] query aliases --- posthog/hogql/ast.py | 27 ++++++++++++++------------- posthog/hogql/printer.py | 7 +++---- posthog/hogql/test/test_query.py | 24 +++++++++++++++++++----- 3 files changed, 36 insertions(+), 22 deletions(-) diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index d50181ba1196c..d69f70d271db1 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -45,17 +45,6 @@ def has_child(self, name: str) -> bool: return self.symbol.has_child(name) -class SelectQueryAliasSymbol(Symbol): - name: str - symbol: Symbol - - def get_child(self, name: str) -> Symbol: - return self.symbol.get_child(name) - - def has_child(self, name: str) -> bool: - return self.symbol.has_child(name) - - class TableSymbol(Symbol): table: Table @@ -81,6 +70,18 @@ def get_child(self, name: str) -> Symbol: return self.table.get_child(name) +class SelectQueryAliasSymbol(Symbol): + name: str + symbol: Symbol + + def get_child(self, name: str) -> Optional[Symbol]: + if self.symbol.has_child(name): + return FieldSymbol(name=name, table=self) + + def has_child(self, name: str) -> bool: + return self.symbol.has_child(name) + + class SelectQuerySymbol(Symbol): # all aliases a select query has access to in its scope aliases: Dict[str, ColumnAliasSymbol] @@ -93,7 +94,7 @@ class SelectQuerySymbol(Symbol): def get_child(self, name: str) -> Symbol: if name in self.columns: - return self.columns[name] + return FieldSymbol(name=name, table=self) raise NotImplementedError(f"Column not found: {name}") def has_child(self, name: str) -> bool: @@ -110,7 +111,7 @@ class CallSymbol(Symbol): class FieldSymbol(Symbol): name: str - table: Union[TableSymbol, TableAliasSymbol] + table: Union[TableSymbol, TableAliasSymbol, SelectQuerySymbol, SelectQueryAliasSymbol] def get_child(self, name: str) -> Symbol: table_symbol = self.table diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index 3bdffc29d0804..02060de6f2153 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -77,9 +77,9 @@ def visit_select_query(self, node: ast.SelectQuery): if where is None: where = extra_where elif isinstance(where, ast.And): - where = ast.And(exprs=where.exprs + [extra_where]) + where = ast.And(exprs=[extra_where] + where.exprs) else: - where = ast.And(exprs=[where, extra_where]) + where = ast.And(exprs=[extra_where, where]) next_join = next_join.next_join @@ -328,8 +328,6 @@ def visit_table_alias_symbol(self, symbol: ast.TableAliasSymbol): return print_hogql_identifier(symbol.name) def visit_field_symbol(self, symbol: ast.FieldSymbol): - # do we need a table prefix? - table_prefix = self.visit(symbol.table) printed_field = print_hogql_identifier(symbol.name) try: @@ -342,6 +340,7 @@ def visit_field_symbol(self, symbol: ast.FieldSymbol): or isinstance(symbol.table, ast.TableAliasSymbol) or isinstance(symbol.table, ast.SelectQueryAliasSymbol) ): + table_prefix = self.visit(symbol.table) field_sql = f"{table_prefix}.{printed_field}" else: field_sql = printed_field diff --git a/posthog/hogql/test/test_query.py b/posthog/hogql/test/test_query.py index dc329218df99b..f39da1e54b93b 100644 --- a/posthog/hogql/test/test_query.py +++ b/posthog/hogql/test/test_query.py @@ -43,13 +43,27 @@ def test_query(self): ) self.assertEqual(response.results, [(2, "random event")]) + response = execute_hogql_query( + f"select count, event from (select count() as count, event from events where properties.random_uuid = '{random_uuid}' group by event) group by count, event", + self.team, + ) + self.assertEqual( + response.clickhouse, + f"SELECT count, event FROM (SELECT count(*) AS count, event FROM events WHERE and(equals(team_id, {self.team.id}), equals(replaceRegexpAll(JSONExtractRaw(properties, %(hogql_val_0)s), '^\"|\"$', ''), %(hogql_val_1)s)) GROUP BY event) GROUP BY count, event LIMIT 1000", + ) + self.assertEqual( + response.hogql, + "SELECT count, event FROM (SELECT count() AS count, event FROM events WHERE equals(properties.random_uuid, %(hogql_val_2)s) GROUP BY event) GROUP BY count, event LIMIT 1000", + ) + self.assertEqual(response.results, [(2, "random event")]) + response = execute_hogql_query( f"select count, event from (select count() as count, event from events where properties.random_uuid = '{random_uuid}' group by event) as c group by count, event", self.team, ) self.assertEqual( response.clickhouse, - f"SELECT count, event FROM (SELECT count(*) AS count, event FROM events WHERE and(equals(team_id, {self.team.id}), equals(replaceRegexpAll(JSONExtractRaw(properties, %(hogql_val_0)s), '^\"|\"$', ''), %(hogql_val_1)s)) GROUP BY event) AS c GROUP BY count, event LIMIT 1000", + f"SELECT c.count, c.event FROM (SELECT count(*) AS count, event FROM events WHERE and(equals(team_id, {self.team.id}), equals(replaceRegexpAll(JSONExtractRaw(properties, %(hogql_val_0)s), '^\"|\"$', ''), %(hogql_val_1)s)) GROUP BY event) AS c GROUP BY c.count, c.event LIMIT 1000", ) self.assertEqual( response.hogql, @@ -102,7 +116,7 @@ def test_query_joins_simple(self): ) self.assertEqual( response.clickhouse, - f"SELECT e.event, e.timestamp, pdi.distinct_id, p.id, replaceRegexpAll(JSONExtractRaw(p.properties, %(hogql_val_0)s), '^\"|\"$', '') FROM events AS e LEFT JOIN person_distinct_id2 AS pdi ON equals(pdi.distinct_id, e.distinct_id) LEFT JOIN person AS p ON equals(p.id, pdi.person_id) WHERE and(equals(e.team_id, {self.team.id}), equals(pdi.team_id, {self.team.id}), equals(p.team_id, {self.team.id})) LIMIT 1000", + f"SELECT e.event, e.timestamp, pdi.distinct_id, p.id, replaceRegexpAll(JSONExtractRaw(p.properties, %(hogql_val_0)s), '^\"|\"$', '') FROM events AS e LEFT JOIN person_distinct_id2 AS pdi ON equals(pdi.distinct_id, e.distinct_id) LEFT JOIN person AS p ON equals(p.id, pdi.person_id) WHERE and(equals(p.team_id, {self.team.id}), equals(pdi.team_id, {self.team.id}), equals(e.team_id, {self.team.id})) LIMIT 1000", ) self.assertEqual( response.hogql, @@ -121,19 +135,19 @@ def test_query_joins(self): SELECT distinct_id, argMax(person_id, version) as person_id FROM person_distinct_id - WHERE team_id = 1 GROUP BY distinct_id HAVING argMax(is_deleted, version) = 0 ) AS pdi ON e.distinct_id = pdi.distinct_id""", self.team, ) + self.assertEqual( response.clickhouse, - f"SELECT DISTINCT person_id, distinct_id FROM person_distinct_id2 WHERE equals(team_id, {self.team.id}) LIMIT 1000", + f"SELECT e.event, e.timestamp, pdi.person_id FROM events AS e INNER JOIN (SELECT distinct_id, argMax(person_distinct_id2.person_id, version) AS person_id FROM person_distinct_id2 WHERE equals(team_id, {self.team.id}) GROUP BY distinct_id HAVING equals(argMax(is_deleted, version), 0)) AS pdi ON equals(e.distinct_id, pdi.distinct_id) WHERE equals(e.team_id, {self.team.id}) LIMIT 1000", ) self.assertEqual( response.hogql, - "SELECT DISTINCT person_id, distinct_id FROM person_distinct_id LIMIT 1000", + "SELECT event, timestamp, pdi.person_id FROM events AS e INNER JOIN (SELECT distinct_id, argMax(person_id, version) AS person_id FROM person_distinct_id2 GROUP BY distinct_id HAVING equals(argMax(is_deleted, version), 0)) AS pdi ON equals(e.distinct_id, pdi.distinct_id) LIMIT 1000", ) self.assertTrue(len(response.results) > 0) From da83eec5da74c1ea0531e021c231fa495dbe060d Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 14 Feb 2023 00:26:15 +0100 Subject: [PATCH 30/81] errors, cleanup, recordings --- posthog/hogql/ast.py | 26 ++++++++++++------------ posthog/hogql/database.py | 31 +++++++++++++++++++++++++++-- posthog/hogql/printer.py | 4 ++-- posthog/hogql/test/test_printer.py | 5 +++-- posthog/hogql/test/test_query.py | 6 +++--- posthog/hogql/test/test_resolver.py | 12 +++++------ 6 files changed, 56 insertions(+), 28 deletions(-) diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index d69f70d271db1..a93e988d79d27 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -54,7 +54,7 @@ def has_child(self, name: str) -> bool: def get_child(self, name: str) -> Symbol: if self.has_child(name): return FieldSymbol(name=name, table=self) - raise NotImplementedError(f"Field not found: {name}") + raise ValueError(f"Field not found: {name}") class TableAliasSymbol(Symbol): @@ -74,9 +74,10 @@ class SelectQueryAliasSymbol(Symbol): name: str symbol: Symbol - def get_child(self, name: str) -> Optional[Symbol]: + def get_child(self, name: str) -> Symbol: if self.symbol.has_child(name): return FieldSymbol(name=name, table=self) + raise ValueError(f"Field not found: {name}") def has_child(self, name: str) -> bool: return self.symbol.has_child(name) @@ -95,7 +96,7 @@ class SelectQuerySymbol(Symbol): def get_child(self, name: str) -> Symbol: if name in self.columns: return FieldSymbol(name=name, table=self) - raise NotImplementedError(f"Column not found: {name}") + raise ValueError(f"Column not found: {name}") def has_child(self, name: str) -> bool: return name in self.columns @@ -117,16 +118,15 @@ def get_child(self, name: str) -> Symbol: table_symbol = self.table while isinstance(table_symbol, TableAliasSymbol): table_symbol = table_symbol.table - db_table = table_symbol.table - if isinstance(db_table, Table): - if self.name in db_table.__fields__ and isinstance(db_table.__fields__[self.name].default, StringJSONValue): - return PropertySymbol(name=name, field=self) - else: - raise NotImplementedError( - f"Can not access property {name} on field {self.name} because it's not a JSON field" - ) - else: - raise NotImplementedError(f"Can not resolve table for field: {self.name}") + + if isinstance(table_symbol, TableSymbol): + db_table = table_symbol.table + if isinstance(db_table, Table): + if self.name in db_table.__fields__ and isinstance( + db_table.__fields__[self.name].default, StringJSONValue + ): + return PropertySymbol(name=name, field=self) + raise ValueError(f"Can not access property {name} on field {self.name}.") class ConstantSymbol(Symbol): diff --git a/posthog/hogql/database.py b/posthog/hogql/database.py index 031f63502f0ba..4cf5eb639ffb3 100644 --- a/posthog/hogql/database.py +++ b/posthog/hogql/database.py @@ -26,6 +26,10 @@ class BooleanValue(Field): pass +class ArrayValue(Field): + field: Field + + class Table(BaseModel): class Config: extra = Extra.forbid @@ -79,6 +83,28 @@ def clickhouse_table(self): return "events" +class SessionRecordingEvents(Table): + uuid: StringValue = StringValue() + timestamp: DateTimeValue = DateTimeValue() + team_id: IntegerValue = IntegerValue() + distinct_id: StringValue = StringValue() + session_id: StringValue = StringValue() + window_id: StringValue = StringValue() + snapshot_data: StringValue = StringValue() + created_at: DateTimeValue = DateTimeValue() + has_full_snapshot: BooleanValue = BooleanValue() + events_summary: ArrayValue = ArrayValue(field=BooleanValue()) + click_count: IntegerValue = IntegerValue() + keypress_count: IntegerValue = IntegerValue() + timestamps_summary: ArrayValue = ArrayValue(field=DateTimeValue()) + first_event_timestamp: DateTimeValue = DateTimeValue() + last_event_timestamp: DateTimeValue = DateTimeValue() + urls: ArrayValue = ArrayValue(field=StringValue()) + + def clickhouse_table(self): + return "session_recording_events" + + # class NumbersTable(Table): # args: [IntegerValue, IntegerValue] @@ -87,10 +113,11 @@ class Database(BaseModel): class Config: extra = Extra.forbid - # All fields below will be tables users can query from + # Users can query from the tables below events: EventsTable = EventsTable() persons: PersonsTable = PersonsTable() - person_distinct_id: PersonDistinctIdTable = PersonDistinctIdTable() + person_distinct_ids: PersonDistinctIdTable = PersonDistinctIdTable() + session_recording_events: SessionRecordingEvents = SessionRecordingEvents() # numbers: NumbersTable = NumbersTable() diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index 02060de6f2153..200edbd18ad51 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -1,4 +1,4 @@ -from typing import List, Literal, Optional, Union +from typing import List, Literal, Optional, Tuple, Union from ee.clickhouse.materialized_columns.columns import get_materialized_columns from posthog.hogql import ast @@ -123,7 +123,7 @@ def visit_select_query(self, node: ast.SelectQuery): response = f"({response})" return response - def visit_join_expr(self, node: ast.JoinExpr) -> (str, Optional[ast.Expr]): + def visit_join_expr(self, node: ast.JoinExpr) -> Tuple[str, Optional[ast.Expr]]: # return constraints we must place on the select query extra_where = None diff --git a/posthog/hogql/test/test_printer.py b/posthog/hogql/test/test_printer.py index fe082d123cc22..bf1a055ee474b 100644 --- a/posthog/hogql/test/test_printer.py +++ b/posthog/hogql/test/test_printer.py @@ -370,7 +370,8 @@ def test_select_alias(self): # currently not supported! self.assertEqual(self._select("select 1 as b"), "SELECT 1 AS b LIMIT 65535") self.assertEqual( - self._select("select 1 from events as e"), "SELECT 1 FROM events AS e WHERE equals(team_id, 42) LIMIT 65535" + self._select("select 1 from events as e"), + "SELECT 1 FROM events AS e WHERE equals(e.team_id, 42) LIMIT 65535", ) def test_select_from(self): @@ -469,5 +470,5 @@ def test_select_subquery(self): ) self.assertEqual( self._select("SELECT event from (select distinct event from events group by event, timestamp) e"), - "SELECT event FROM (SELECT DISTINCT event FROM events WHERE equals(team_id, 42) GROUP BY event, timestamp) AS e LIMIT 65535", + "SELECT e.event FROM (SELECT DISTINCT event FROM events WHERE equals(team_id, 42) GROUP BY event, timestamp) AS e LIMIT 65535", ) diff --git a/posthog/hogql/test/test_query.py b/posthog/hogql/test/test_query.py index f39da1e54b93b..c1cefa49361f0 100644 --- a/posthog/hogql/test/test_query.py +++ b/posthog/hogql/test/test_query.py @@ -86,7 +86,7 @@ def test_query(self): self.assertEqual(response.results, [("tim@posthog.com",)]) response = execute_hogql_query( - f"select distinct person_id, distinct_id from person_distinct_id", + f"select distinct person_id, distinct_id from person_distinct_ids", self.team, ) self.assertEqual( @@ -107,7 +107,7 @@ def test_query_joins_simple(self): """ SELECT event, timestamp, pdi.distinct_id, p.id, p.properties.email FROM events e - LEFT JOIN person_distinct_id pdi + LEFT JOIN person_distinct_ids pdi ON pdi.distinct_id = e.distinct_id LEFT JOIN persons p ON p.id = pdi.person_id @@ -134,7 +134,7 @@ def test_query_joins(self): """select event, timestamp, pdi.person_id from events e INNER JOIN ( SELECT distinct_id, argMax(person_id, version) as person_id - FROM person_distinct_id + FROM person_distinct_ids GROUP BY distinct_id HAVING argMax(is_deleted, version) = 0 ) AS pdi diff --git a/posthog/hogql/test/test_resolver.py b/posthog/hogql/test/test_resolver.py index 0c522322151fe..cb7e8c1927d85 100644 --- a/posthog/hogql/test/test_resolver.py +++ b/posthog/hogql/test/test_resolver.py @@ -55,7 +55,7 @@ def test_resolve_events_table_alias(self): select_query_symbol = ast.SelectQuerySymbol( aliases={}, columns={"event": event_field_symbol, "timestamp": timestamp_field_symbol}, - tables={"e": ast.TableAliasSymbol(name="e", symbol=events_table_symbol)}, + tables={"e": ast.TableAliasSymbol(name="e", table=events_table_symbol)}, anonymous_tables=[], ) @@ -67,7 +67,7 @@ def test_resolve_events_table_alias(self): select_from=ast.JoinExpr( table=ast.Field(chain=["events"], symbol=events_table_symbol), alias="e", - symbol=ast.TableAliasSymbol(name="e", symbol=events_table_symbol), + symbol=ast.TableAliasSymbol(name="e", table=events_table_symbol), ), where=ast.CompareOperation( left=ast.Field(chain=["e", "event"], symbol=event_field_symbol), @@ -102,7 +102,7 @@ def test_resolve_events_table_column_alias(self): "e": ast.ColumnAliasSymbol(name="e", symbol=event_field_symbol), "timestamp": timestamp_field_symbol, }, - tables={"e": ast.TableAliasSymbol(name="e", symbol=events_table_symbol)}, + tables={"e": ast.TableAliasSymbol(name="e", table=events_table_symbol)}, anonymous_tables=[], ) @@ -144,7 +144,7 @@ def test_resolve_events_table_column_alias_inside_subquery(self): expr = parse_select("SELECT b FROM (select event as b, timestamp as c from events) e WHERE e.b = 'test'") resolve_symbols(expr) outer_events_table_symbol = ast.TableSymbol(table=database.events) - inner_events_table_symbol = ast.TableAliasSymbol(name="e1", symbol=ast.TableSymbol(table=database.events)) + inner_events_table_symbol = ast.TableAliasSymbol(name="e1", table=ast.TableSymbol(table=database.events)) outer_event_field_symbol = ast.FieldSymbol(name="event", table=outer_events_table_symbol) inner_event_field_symbol = ast.FieldSymbol(name="event", table=inner_events_table_symbol) timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=outer_events_table_symbol) @@ -200,7 +200,7 @@ def test_resolve_events_table_column_alias_inside_subquery(self): symbol=inner_select_symbol, ), alias="e", - symbol=ast.TableAliasSymbol(name="e", symbol=inner_select_symbol), + symbol=ast.TableAliasSymbol(name="e", table=inner_select_symbol), ), where=ast.CompareOperation( left=ast.Field( @@ -218,7 +218,7 @@ def test_resolve_events_table_column_alias_inside_subquery(self): columns={ "b": ast.ColumnAliasSymbol(name="b", symbol=outer_event_field_symbol), }, - tables={"e": ast.TableAliasSymbol(name="e", symbol=inner_select_symbol)}, + tables={"e": ast.TableAliasSymbol(name="e", table=inner_select_symbol)}, ), ) # asserting individually to help debug if something is off From 2e27c1e82d0c992808e1bb45a997b4fb11a6f71c Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 14 Feb 2023 01:20:23 +0100 Subject: [PATCH 31/81] resolver tests --- posthog/hogql/resolver.py | 6 ++- posthog/hogql/test/test_resolver.py | 78 ++++++++++++++--------------- 2 files changed, 41 insertions(+), 43 deletions(-) diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py index 94a7d5b510507..459acb24c68f2 100644 --- a/posthog/hogql/resolver.py +++ b/posthog/hogql/resolver.py @@ -83,9 +83,11 @@ def visit_join_expr(self, node): if table_name in database.__fields__: table = database.__fields__[table_name].default - node.symbol = ast.TableSymbol(table=table) + node.table.symbol = ast.TableSymbol(table=table) if table_alias != table_name: - node.symbol = ast.TableAliasSymbol(name=table_alias, table=node.symbol) + node.symbol = ast.TableAliasSymbol(name=table_alias, table=node.table.symbol) + else: + node.symbol = node.table.symbol scope.tables[table_alias] = node.symbol else: raise ResolverException(f'Unknown table "{table_name}".') diff --git a/posthog/hogql/test/test_resolver.py b/posthog/hogql/test/test_resolver.py index cb7e8c1927d85..bb8ec23d68e90 100644 --- a/posthog/hogql/test/test_resolver.py +++ b/posthog/hogql/test/test_resolver.py @@ -27,7 +27,6 @@ def test_resolve_events_table(self): ], select_from=ast.JoinExpr( table=ast.Field(chain=["events"], symbol=events_table_symbol), - alias="events", symbol=events_table_symbol, ), where=ast.CompareOperation( @@ -50,12 +49,13 @@ def test_resolve_events_table_alias(self): resolve_symbols(expr) events_table_symbol = ast.TableSymbol(table=database.events) - event_field_symbol = ast.FieldSymbol(name="event", table=events_table_symbol) - timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=events_table_symbol) + events_table_alias_symbol = ast.TableAliasSymbol(name="e", table=events_table_symbol) + event_field_symbol = ast.FieldSymbol(name="event", table=events_table_alias_symbol) + timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=events_table_alias_symbol) select_query_symbol = ast.SelectQuerySymbol( aliases={}, columns={"event": event_field_symbol, "timestamp": timestamp_field_symbol}, - tables={"e": ast.TableAliasSymbol(name="e", table=events_table_symbol)}, + tables={"e": events_table_alias_symbol}, anonymous_tables=[], ) @@ -67,7 +67,7 @@ def test_resolve_events_table_alias(self): select_from=ast.JoinExpr( table=ast.Field(chain=["events"], symbol=events_table_symbol), alias="e", - symbol=ast.TableAliasSymbol(name="e", table=events_table_symbol), + symbol=events_table_alias_symbol, ), where=ast.CompareOperation( left=ast.Field(chain=["e", "event"], symbol=event_field_symbol), @@ -89,20 +89,25 @@ def test_resolve_events_table_column_alias(self): resolve_symbols(expr) events_table_symbol = ast.TableSymbol(table=database.events) - event_field_symbol = ast.FieldSymbol(name="event", table=events_table_symbol) - timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=events_table_symbol) + events_table_alias_symbol = ast.TableAliasSymbol(name="e", table=events_table_symbol) + event_field_symbol = ast.FieldSymbol(name="event", table=events_table_alias_symbol) + timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=events_table_alias_symbol) select_query_symbol = ast.SelectQuerySymbol( aliases={ "ee": ast.ColumnAliasSymbol(name="ee", symbol=event_field_symbol), - "e": ast.ColumnAliasSymbol(name="e", symbol=event_field_symbol), + "e": ast.ColumnAliasSymbol( + name="e", symbol=ast.ColumnAliasSymbol(name="ee", symbol=event_field_symbol) + ), }, columns={ "ee": ast.ColumnAliasSymbol(name="ee", symbol=event_field_symbol), - "e": ast.ColumnAliasSymbol(name="e", symbol=event_field_symbol), + "e": ast.ColumnAliasSymbol( + name="e", symbol=ast.ColumnAliasSymbol(name="ee", symbol=event_field_symbol) + ), "timestamp": timestamp_field_symbol, }, - tables={"e": ast.TableAliasSymbol(name="e", table=events_table_symbol)}, + tables={"e": events_table_alias_symbol}, anonymous_tables=[], ) @@ -117,7 +122,7 @@ def test_resolve_events_table_column_alias(self): ast.Alias( alias="e", expr=ast.Field(chain=["ee"], symbol=select_query_symbol.aliases["ee"]), - symbol=select_query_symbol.aliases["e"], + symbol=select_query_symbol.aliases["e"], # is ee ? ), ast.Field(chain=["e", "timestamp"], symbol=timestamp_field_symbol), ], @@ -143,32 +148,34 @@ def test_resolve_events_table_column_alias(self): def test_resolve_events_table_column_alias_inside_subquery(self): expr = parse_select("SELECT b FROM (select event as b, timestamp as c from events) e WHERE e.b = 'test'") resolve_symbols(expr) - outer_events_table_symbol = ast.TableSymbol(table=database.events) - inner_events_table_symbol = ast.TableAliasSymbol(name="e1", table=ast.TableSymbol(table=database.events)) - outer_event_field_symbol = ast.FieldSymbol(name="event", table=outer_events_table_symbol) - inner_event_field_symbol = ast.FieldSymbol(name="event", table=inner_events_table_symbol) - timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=outer_events_table_symbol) + inner_events_table_symbol = ast.TableSymbol(table=database.events) + inner_event_field_symbol = ast.ColumnAliasSymbol( + name="b", symbol=ast.FieldSymbol(name="event", table=inner_events_table_symbol) + ) + timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=inner_events_table_symbol) + timstamp_alias_symbol = ast.ColumnAliasSymbol(name="c", symbol=timestamp_field_symbol) inner_select_symbol = ast.SelectQuerySymbol( aliases={ - "b": ast.ColumnAliasSymbol(name="b", symbol=outer_event_field_symbol), + "b": inner_event_field_symbol, "c": ast.ColumnAliasSymbol(name="c", symbol=timestamp_field_symbol), }, columns={ - "b": ast.ColumnAliasSymbol(name="b", symbol=outer_events_table_symbol), + "b": inner_event_field_symbol, "c": ast.ColumnAliasSymbol(name="c", symbol=timestamp_field_symbol), }, tables={ - "events": outer_events_table_symbol, + "events": inner_events_table_symbol, }, anonymous_tables=[], ) + select_alias_symbol = ast.SelectQueryAliasSymbol(name="e", symbol=inner_select_symbol) expected = ast.SelectQuery( select=[ ast.Field( chain=["b"], - symbol=ast.ColumnAliasSymbol( + symbol=ast.FieldSymbol( name="b", - symbol=outer_event_field_symbol, + table=ast.SelectQueryAliasSymbol(name="e", symbol=inner_select_symbol), ), ), ], @@ -177,48 +184,37 @@ def test_resolve_events_table_column_alias_inside_subquery(self): select=[ ast.Alias( alias="b", - expr=ast.Field(chain=["event"], symbol=inner_event_field_symbol), - symbol=ast.ColumnAliasSymbol( - name="b", - symbol=inner_event_field_symbol, - ), + expr=ast.Field(chain=["event"], symbol=inner_event_field_symbol.symbol), + symbol=inner_event_field_symbol, ), ast.Alias( alias="c", expr=ast.Field(chain=["timestamp"], symbol=timestamp_field_symbol), - symbol=ast.ColumnAliasSymbol( - name="c", - symbol=timestamp_field_symbol, - ), + symbol=timstamp_alias_symbol, ), ], select_from=ast.JoinExpr( table=ast.Field(chain=["events"], symbol=inner_events_table_symbol), - alias="events", symbol=inner_events_table_symbol, ), symbol=inner_select_symbol, ), alias="e", - symbol=ast.TableAliasSymbol(name="e", table=inner_select_symbol), + symbol=select_alias_symbol, ), where=ast.CompareOperation( left=ast.Field( chain=["e", "b"], - symbol=ast.ColumnAliasSymbol( - name="b", - symbol=outer_event_field_symbol, - ), + symbol=ast.FieldSymbol(name="b", table=select_alias_symbol), ), op=ast.CompareOperationType.Eq, right=ast.Constant(value="test", symbol=ast.ConstantSymbol(value="test")), ), symbol=ast.SelectQuerySymbol( aliases={}, - columns={ - "b": ast.ColumnAliasSymbol(name="b", symbol=outer_event_field_symbol), - }, - tables={"e": ast.TableAliasSymbol(name="e", table=inner_select_symbol)}, + columns={"b": ast.FieldSymbol(name="b", table=select_alias_symbol)}, + tables={"e": select_alias_symbol}, + anonymous_tables=[], ), ) # asserting individually to help debug if something is off @@ -248,7 +244,7 @@ def test_resolve_errors(self): for query in queries: with self.assertRaises(ResolverException) as e: resolve_symbols(parse_select(query)) - self.assertIn("Unable to resolve field: x", str(e.exception)) + self.assertIn("Unable to resolve field:", str(e.exception)) # "with 2 as a select 1 as a" -> "Different expressions with the same alias a:" From 78caa571766742f65dd3fb6da2709f7ec2bf0331 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 14 Feb 2023 11:45:18 +0100 Subject: [PATCH 32/81] cleanup --- posthog/hogql/ast.py | 44 ++++++++++++++++------------- posthog/hogql/hogql.py | 2 +- posthog/hogql/printer.py | 32 ++++++++------------- posthog/hogql/resolver.py | 17 +++++------ posthog/hogql/test/test_resolver.py | 27 ++++++------------ 5 files changed, 54 insertions(+), 68 deletions(-) diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index a93e988d79d27..ee4854ecbe202 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -3,10 +3,11 @@ from typing import Any, Dict, List, Literal, Optional, Union from pydantic import BaseModel, Extra +from pydantic import Field as PydanticField from posthog.hogql.database import StringJSONValue, Table -# NOTE: when you add new AST fields or nodes, add them to CloningVisitor as well! +# NOTE: when you add new AST fields or nodes, add them to the Visitor classes in visitor.py as well! camel_case_pattern = re.compile(r"(? bool: return self.get_child(name) is not None -class ColumnAliasSymbol(Symbol): +class FieldAliasSymbol(Symbol): name: str symbol: Symbol @@ -70,28 +71,17 @@ def get_child(self, name: str) -> Symbol: return self.table.get_child(name) -class SelectQueryAliasSymbol(Symbol): - name: str - symbol: Symbol - - def get_child(self, name: str) -> Symbol: - if self.symbol.has_child(name): - return FieldSymbol(name=name, table=self) - raise ValueError(f"Field not found: {name}") - - def has_child(self, name: str) -> bool: - return self.symbol.has_child(name) - - class SelectQuerySymbol(Symbol): # all aliases a select query has access to in its scope - aliases: Dict[str, ColumnAliasSymbol] + aliases: Dict[str, FieldAliasSymbol] = PydanticField(default_factory=dict) # all symbols a select query exports - columns: Dict[str, Symbol] + columns: Dict[str, Symbol] = PydanticField(default_factory=dict) # all from and join, tables and subqueries with aliases - tables: Dict[str, Union[TableSymbol, TableAliasSymbol, "SelectQuerySymbol", SelectQueryAliasSymbol]] + tables: Dict[ + str, Union[TableSymbol, TableAliasSymbol, "SelectQuerySymbol", "SelectQueryAliasSymbol"] + ] = PydanticField(default_factory=dict) # all from and join subqueries without aliases - anonymous_tables: List["SelectQuerySymbol"] + anonymous_tables: List["SelectQuerySymbol"] = PydanticField(default_factory=list) def get_child(self, name: str) -> Symbol: if name in self.columns: @@ -102,7 +92,21 @@ def has_child(self, name: str) -> bool: return name in self.columns +class SelectQueryAliasSymbol(Symbol): + name: str + symbol: SelectQuerySymbol + + def get_child(self, name: str) -> Symbol: + if self.symbol.has_child(name): + return FieldSymbol(name=name, table=self) + raise ValueError(f"Field not found: {name}") + + def has_child(self, name: str) -> bool: + return self.symbol.has_child(name) + + SelectQuerySymbol.update_forward_refs(SelectQuerySymbol=SelectQuerySymbol) +SelectQuerySymbol.update_forward_refs(SelectQueryAliasSymbol=SelectQueryAliasSymbol) class CallSymbol(Symbol): diff --git a/posthog/hogql/hogql.py b/posthog/hogql/hogql.py index 5a10fae468d51..c6e2c1a962e0d 100644 --- a/posthog/hogql/hogql.py +++ b/posthog/hogql/hogql.py @@ -21,7 +21,7 @@ def translate_hogql(query: str, context: HogQLContext, dialect: Literal["hogql", else: node = parse_expr(query, no_placeholders=True) symbol = ast.SelectQuerySymbol( - aliases={}, columns={}, tables={"events": ast.TableSymbol(table=database.events)}, anonymous_tables=[] + tables={"events": ast.TableSymbol(table=database.events)}, ) resolve_symbols(node, symbol) return print_ast(node, context, dialect, stack=[ast.SelectQuery(select=[], symbol=symbol)]) diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index 200edbd18ad51..47797080192db 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -9,27 +9,17 @@ from posthog.hogql.visitor import Visitor -def guard_where_team_id( - where: Optional[ast.Expr], table_symbol: Union[ast.TableSymbol, ast.TableAliasSymbol], context: HogQLContext -) -> ast.Expr: +def guard_where_team_id(table_symbol: Union[ast.TableSymbol, ast.TableAliasSymbol], context: HogQLContext) -> ast.Expr: """Add a mandatory "and(team_id, ...)" filter around the expression.""" if not context.select_team_id: raise ValueError("context.select_team_id not found") - team_clause = ast.CompareOperation( + return ast.CompareOperation( op=ast.CompareOperationType.Eq, left=ast.Field(chain=["team_id"], symbol=ast.FieldSymbol(name="team_id", table=table_symbol)), right=ast.Constant(value=context.select_team_id), ) - if isinstance(where, ast.And): - where = ast.And(exprs=[team_clause] + where.exprs) - elif where: - where = ast.And(exprs=[team_clause, where]) - else: - where = team_clause - return where - def print_ast( node: ast.AST, context: HogQLContext, dialect: Literal["hogql", "clickhouse"], stack: List[ast.AST] = [] @@ -142,13 +132,13 @@ def visit_join_expr(self, node: ast.JoinExpr) -> Tuple[str, Optional[ast.Expr]]: select_from.append(f"AS {print_hogql_identifier(node.alias)}") if self.dialect == "clickhouse": - extra_where = guard_where_team_id(None, node.symbol, self.context) + extra_where = guard_where_team_id(node.symbol, self.context) elif isinstance(node.symbol, ast.TableSymbol): select_from.append(print_hogql_identifier(node.symbol.table.clickhouse_table())) if self.dialect == "clickhouse": - extra_where = guard_where_team_id(None, node.symbol, self.context) + extra_where = guard_where_team_id(node.symbol, self.context) elif isinstance(node.symbol, ast.SelectQuerySymbol): select_from.append(self.visit(node.table)) @@ -256,12 +246,14 @@ def visit_field(self, node: ast.Field): if self.dialect == "hogql": # When printing HogQL, we print the properties out as a chain instead of converting them to Clickhouse SQL return ".".join([print_hogql_identifier(identifier) for identifier in node.chain]) - # elif node.chain == ["*"]: - # query = f"tuple({','.join(SELECT_STAR_FROM_EVENTS_FIELDS)})" - # return self.visit(parse_expr(query)) - # elif node.chain == ["person"]: - # query = "tuple(distinct_id, person.id, person.created_at, person.properties.name, person.properties.email)" - # return self.visit(parse_expr(query)) + elif node.chain == ["*"]: + raise ValueError("Selecting * not implemented") + # query = f"tuple({','.join(SELECT_STAR_FROM_EVENTS_FIELDS)})" + # return self.visit(parse_expr(query)) + elif node.chain == ["person"]: + raise ValueError("Selecting person not implemented") + # query = "tuple(distinct_id, person.id, person.created_at, person.properties.name, person.properties.email)" + # return self.visit(parse_expr(query)) elif node.symbol is not None: select_query = self._last_select() select: Optional[ast.SelectQuerySymbol] = select_query.symbol if select_query else None diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py index 459acb24c68f2..8c76bf9c58f24 100644 --- a/posthog/hogql/resolver.py +++ b/posthog/hogql/resolver.py @@ -19,6 +19,7 @@ class Resolver(TraversingVisitor): """The Resolver visits an AST and assigns Symbols to the nodes.""" def __init__(self, scope: Optional[ast.SelectQuerySymbol] = None): + # Each SELECT query creates a new scope. Store all of them in a list as we traverse the tree. self.scopes: List[ast.SelectQuerySymbol] = [scope] if scope else [] def visit_select_query(self, node): @@ -26,7 +27,7 @@ def visit_select_query(self, node): if node.symbol is not None: return - node.symbol = ast.SelectQuerySymbol(aliases={}, columns={}, tables={}, anonymous_tables=[]) + node.symbol = ast.SelectQuerySymbol() # Each SELECT query creates a new scope. Store all of them in a list for variable access. self.scopes.append(node.symbol) @@ -40,7 +41,7 @@ def visit_select_query(self, node): # SELECT e.event, e.timestamp from (SELECT event, timestamp FROM events) AS e for expr in node.select or []: self.visit(expr) - if isinstance(expr.symbol, ast.ColumnAliasSymbol): + if isinstance(expr.symbol, ast.FieldAliasSymbol): node.symbol.columns[expr.symbol.name] = expr.symbol elif isinstance(expr, ast.Alias): node.symbol.columns[expr.alias] = expr.symbol @@ -125,7 +126,7 @@ def visit_alias(self, node: ast.Alias): self.visit(node.expr) if not node.expr.symbol: raise ResolverException(f"Cannot alias an expression without a symbol: {node.alias}") - node.symbol = ast.ColumnAliasSymbol(name=node.alias, symbol=node.expr.symbol) + node.symbol = ast.FieldAliasSymbol(name=node.alias, symbol=node.expr.symbol) scope.aliases[node.alias] = node.symbol def visit_call(self, node: ast.Call): @@ -144,12 +145,12 @@ def visit_field(self, node): raise Exception("Invalid field access with empty chain") # ClickHouse does not support subqueries accessing "x.event" like this: - # "SELECT event, (select count() from events where event = x.event) as c FROM events x where event = '$pageview'", - # + # - "SELECT event, (select count() from events where event = x.event) as c FROM events x where event = '$pageview'", # But this is supported: - # "SELECT t.big_count FROM (select count() + 100 as big_count from events) as t JOIN events e ON (e.event = t.event)", + # - "SELECT t.big_count FROM (select count() + 100 as big_count from events) as t JOIN events e ON (e.event = t.event)", # - # Thus only look into the current scope, for columns and aliases. + # Thus we only look into scopes[-1] to see aliases in the current scope, and don't loop over all the scopes. + scope = self.scopes[-1] symbol: Optional[ast.Symbol] = None name = node.chain[0] @@ -180,6 +181,7 @@ def visit_constant(self, node): def lookup_field_by_name(scope: ast.SelectQuerySymbol, name: str) -> Optional[ast.Symbol]: + """Looks for a field in the scope's list of aliases and children for each joined table.""" if name in scope.aliases: return scope.aliases[name] else: @@ -190,6 +192,5 @@ def lookup_field_by_name(scope: ast.SelectQuerySymbol, name: str) -> Optional[as if len(tables) > 1: raise ResolverException(f"Ambiguous query. Found multiple sources for field: {name}") elif len(tables) == 1: - # accessed a field on a joined table by name return tables[0].get_child(name) return None diff --git a/posthog/hogql/test/test_resolver.py b/posthog/hogql/test/test_resolver.py index bb8ec23d68e90..8e4fd16cf095a 100644 --- a/posthog/hogql/test/test_resolver.py +++ b/posthog/hogql/test/test_resolver.py @@ -14,10 +14,8 @@ def test_resolve_events_table(self): event_field_symbol = ast.FieldSymbol(name="event", table=events_table_symbol) timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=events_table_symbol) select_query_symbol = ast.SelectQuerySymbol( - aliases={}, columns={"event": event_field_symbol, "timestamp": timestamp_field_symbol}, tables={"events": events_table_symbol}, - anonymous_tables=[], ) expected = ast.SelectQuery( @@ -53,10 +51,8 @@ def test_resolve_events_table_alias(self): event_field_symbol = ast.FieldSymbol(name="event", table=events_table_alias_symbol) timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=events_table_alias_symbol) select_query_symbol = ast.SelectQuerySymbol( - aliases={}, columns={"event": event_field_symbol, "timestamp": timestamp_field_symbol}, tables={"e": events_table_alias_symbol}, - anonymous_tables=[], ) expected = ast.SelectQuery( @@ -95,20 +91,15 @@ def test_resolve_events_table_column_alias(self): select_query_symbol = ast.SelectQuerySymbol( aliases={ - "ee": ast.ColumnAliasSymbol(name="ee", symbol=event_field_symbol), - "e": ast.ColumnAliasSymbol( - name="e", symbol=ast.ColumnAliasSymbol(name="ee", symbol=event_field_symbol) - ), + "ee": ast.FieldAliasSymbol(name="ee", symbol=event_field_symbol), + "e": ast.FieldAliasSymbol(name="e", symbol=ast.FieldAliasSymbol(name="ee", symbol=event_field_symbol)), }, columns={ - "ee": ast.ColumnAliasSymbol(name="ee", symbol=event_field_symbol), - "e": ast.ColumnAliasSymbol( - name="e", symbol=ast.ColumnAliasSymbol(name="ee", symbol=event_field_symbol) - ), + "ee": ast.FieldAliasSymbol(name="ee", symbol=event_field_symbol), + "e": ast.FieldAliasSymbol(name="e", symbol=ast.FieldAliasSymbol(name="ee", symbol=event_field_symbol)), "timestamp": timestamp_field_symbol, }, tables={"e": events_table_alias_symbol}, - anonymous_tables=[], ) expected = ast.SelectQuery( @@ -149,24 +140,23 @@ def test_resolve_events_table_column_alias_inside_subquery(self): expr = parse_select("SELECT b FROM (select event as b, timestamp as c from events) e WHERE e.b = 'test'") resolve_symbols(expr) inner_events_table_symbol = ast.TableSymbol(table=database.events) - inner_event_field_symbol = ast.ColumnAliasSymbol( + inner_event_field_symbol = ast.FieldAliasSymbol( name="b", symbol=ast.FieldSymbol(name="event", table=inner_events_table_symbol) ) timestamp_field_symbol = ast.FieldSymbol(name="timestamp", table=inner_events_table_symbol) - timstamp_alias_symbol = ast.ColumnAliasSymbol(name="c", symbol=timestamp_field_symbol) + timstamp_alias_symbol = ast.FieldAliasSymbol(name="c", symbol=timestamp_field_symbol) inner_select_symbol = ast.SelectQuerySymbol( aliases={ "b": inner_event_field_symbol, - "c": ast.ColumnAliasSymbol(name="c", symbol=timestamp_field_symbol), + "c": ast.FieldAliasSymbol(name="c", symbol=timestamp_field_symbol), }, columns={ "b": inner_event_field_symbol, - "c": ast.ColumnAliasSymbol(name="c", symbol=timestamp_field_symbol), + "c": ast.FieldAliasSymbol(name="c", symbol=timestamp_field_symbol), }, tables={ "events": inner_events_table_symbol, }, - anonymous_tables=[], ) select_alias_symbol = ast.SelectQueryAliasSymbol(name="e", symbol=inner_select_symbol) expected = ast.SelectQuery( @@ -214,7 +204,6 @@ def test_resolve_events_table_column_alias_inside_subquery(self): aliases={}, columns={"b": ast.FieldSymbol(name="b", table=select_alias_symbol)}, tables={"e": select_alias_symbol}, - anonymous_tables=[], ), ) # asserting individually to help debug if something is off From 48f684fc1b629ee2b4c69714d26d4024c48a26aa Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 14 Feb 2023 12:24:12 +0100 Subject: [PATCH 33/81] more tiny cleanup --- posthog/hogql/constants.py | 2 +- posthog/hogql/database.py | 5 ----- posthog/hogql/hogql.py | 10 ++++++---- posthog/hogql/parser.py | 6 +++--- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/posthog/hogql/constants.py b/posthog/hogql/constants.py index 0a42c31f7d500..92c88a8c65df4 100644 --- a/posthog/hogql/constants.py +++ b/posthog/hogql/constants.py @@ -98,7 +98,7 @@ KEYWORDS = ["true", "false", "null"] # Keywords you can't alias to -RESERVED_KEYWORDS = ["team_id"] +RESERVED_KEYWORDS = KEYWORDS + ["team_id"] # Allow-listed fields returned when you select "*" from events. Person and group fields will be nested later. SELECT_STAR_FROM_EVENTS_FIELDS = [ diff --git a/posthog/hogql/database.py b/posthog/hogql/database.py index 4cf5eb639ffb3..c9fa18fb84a30 100644 --- a/posthog/hogql/database.py +++ b/posthog/hogql/database.py @@ -105,10 +105,6 @@ def clickhouse_table(self): return "session_recording_events" -# class NumbersTable(Table): -# args: [IntegerValue, IntegerValue] - - class Database(BaseModel): class Config: extra = Extra.forbid @@ -118,7 +114,6 @@ class Config: persons: PersonsTable = PersonsTable() person_distinct_ids: PersonDistinctIdTable = PersonDistinctIdTable() session_recording_events: SessionRecordingEvents = SessionRecordingEvents() - # numbers: NumbersTable = NumbersTable() database = Database() diff --git a/posthog/hogql/hogql.py b/posthog/hogql/hogql.py index c6e2c1a962e0d..541d0199b4c1f 100644 --- a/posthog/hogql/hogql.py +++ b/posthog/hogql/hogql.py @@ -15,16 +15,18 @@ def translate_hogql(query: str, context: HogQLContext, dialect: Literal["hogql", try: if context.select_team_id: + # Only parse full SELECT statements if we have a team_id in the context. node = parse_select(query, no_placeholders=True) resolve_symbols(node) return print_ast(node, context, dialect, stack=[]) else: + # Create a fake query that selects from "events". Assume were in its scope when evaluating expressions. + symbol = ast.SelectQuerySymbol(tables={"events": ast.TableSymbol(table=database.events)}) + select_query = ast.SelectQuery(select=[], symbol=symbol) + node = parse_expr(query, no_placeholders=True) - symbol = ast.SelectQuerySymbol( - tables={"events": ast.TableSymbol(table=database.events)}, - ) resolve_symbols(node, symbol) - return print_ast(node, context, dialect, stack=[ast.SelectQuery(select=[], symbol=symbol)]) + return print_ast(node, context, dialect, stack=[select_query]) except SyntaxError as err: raise ValueError(f"SyntaxError: {err.msg}") diff --git a/posthog/hogql/parser.py b/posthog/hogql/parser.py index f0893f9aa6eb5..5b7f4b9eff6fc 100644 --- a/posthog/hogql/parser.py +++ b/posthog/hogql/parser.py @@ -4,7 +4,7 @@ from antlr4.error.ErrorListener import ErrorListener from posthog.hogql import ast -from posthog.hogql.constants import KEYWORDS, RESERVED_KEYWORDS +from posthog.hogql.constants import RESERVED_KEYWORDS from posthog.hogql.grammar.HogQLLexer import HogQLLexer from posthog.hogql.grammar.HogQLParser import HogQLParser from posthog.hogql.parse_string import parse_string, parse_string_literal @@ -318,7 +318,7 @@ def visitColumnExprAlias(self, ctx: HogQLParser.ColumnExprAliasContext): raise NotImplementedError(f"Must specify an alias.") expr = self.visit(ctx.columnExpr()) - if alias in RESERVED_KEYWORDS or alias in KEYWORDS: + if alias in RESERVED_KEYWORDS: raise ValueError(f"Alias '{alias}' is a reserved keyword.") return ast.Alias(expr=expr, alias=alias) @@ -542,7 +542,7 @@ def visitTableExprSubquery(self, ctx: HogQLParser.TableExprSubqueryContext): def visitTableExprAlias(self, ctx: HogQLParser.TableExprAliasContext): alias = self.visit(ctx.alias() or ctx.identifier()) - if alias in RESERVED_KEYWORDS or alias in KEYWORDS: + if alias in RESERVED_KEYWORDS: raise ValueError(f"Alias '{alias}' is a reserved keyword.") return ast.JoinExpr(table=self.visit(ctx.tableExpr()), alias=alias) From 5cea7bdcc95c137311c4ec9bb32f10650f87743a Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 14 Feb 2023 13:02:52 +0100 Subject: [PATCH 34/81] resolver cleanup --- posthog/hogql/database.py | 6 +++++ posthog/hogql/resolver.py | 51 +++++++++++++++++++++------------------ 2 files changed, 33 insertions(+), 24 deletions(-) diff --git a/posthog/hogql/database.py b/posthog/hogql/database.py index c9fa18fb84a30..314b8621a00b9 100644 --- a/posthog/hogql/database.py +++ b/posthog/hogql/database.py @@ -115,5 +115,11 @@ class Config: person_distinct_ids: PersonDistinctIdTable = PersonDistinctIdTable() session_recording_events: SessionRecordingEvents = SessionRecordingEvents() + def get_table(self, table_name: str): + return getattr(self, table_name) + + def has_table(self, table_name: str): + return hasattr(self, table_name) + database = Database() diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py index 8c76bf9c58f24..b86bfc5c286c2 100644 --- a/posthog/hogql/resolver.py +++ b/posthog/hogql/resolver.py @@ -27,26 +27,26 @@ def visit_select_query(self, node): if node.symbol is not None: return + # This symbol keeps track of all joined tables and other field aliases that are in scope. node.symbol = ast.SelectQuerySymbol() - # Each SELECT query creates a new scope. Store all of them in a list for variable access. + # Each SELECT query is a new scope in field name resolution. self.scopes.append(node.symbol) - # Visit all the FROM and JOIN clauses (JoinExpr nodes), and register the tables into the scope. + # Visit all the FROM and JOIN clauses, and register the tables into the scope. See visit_join_expr below. if node.select_from: self.visit(node.select_from) - # Visit all the SELECT columns. - # Then mark them for export in "columns". This means they will be available outside of this query via: + # Visit all the SELECT 1,2,3 columns. Mark each for export in "columns" to make this work: # SELECT e.event, e.timestamp from (SELECT event, timestamp FROM events) AS e for expr in node.select or []: self.visit(expr) if isinstance(expr.symbol, ast.FieldAliasSymbol): node.symbol.columns[expr.symbol.name] = expr.symbol - elif isinstance(expr, ast.Alias): - node.symbol.columns[expr.alias] = expr.symbol elif isinstance(expr.symbol, ast.FieldSymbol): node.symbol.columns[expr.symbol.name] = expr.symbol + elif isinstance(expr, ast.Alias): + node.symbol.columns[expr.alias] = expr.symbol if node.where: self.visit(node.where) @@ -80,15 +80,14 @@ def visit_join_expr(self, node): table_name = node.table.chain[0] table_alias = node.alias or table_name if table_alias in scope.tables: - raise ResolverException(f'Already have a joined table called "{table_alias}", can\'t redefine.') + raise ResolverException(f'Already have joined a table called "{table_alias}". Can\'t redefine.') - if table_name in database.__fields__: - table = database.__fields__[table_name].default - node.table.symbol = ast.TableSymbol(table=table) - if table_alias != table_name: - node.symbol = ast.TableAliasSymbol(name=table_alias, table=node.table.symbol) - else: + if database.has_table(table_name): + node.table.symbol = ast.TableSymbol(table=database.get_table(table_name)) + if table_alias == table_name: node.symbol = node.table.symbol + else: + node.symbol = ast.TableAliasSymbol(name=table_alias, table=node.table.symbol) scope.tables[table_alias] = node.symbol else: raise ResolverException(f'Unknown table "{table_name}".') @@ -97,7 +96,7 @@ def visit_join_expr(self, node): node.table.symbol = self.visit(node.table) if node.alias is not None: if node.alias in scope.tables: - raise ResolverException(f'Already have a joined table called "{node.alias}", can\'t redefine.') + raise ResolverException(f'Already have joined a table called "{node.alias}". Can\'t redefine.') node.symbol = ast.SelectQueryAliasSymbol(name=node.alias, symbol=node.table.symbol) scope.tables[node.alias] = node.symbol else: @@ -133,9 +132,12 @@ def visit_call(self, node: ast.Call): """Visit function calls.""" if node.symbol is not None: return + arg_symbols: List[ast.Symbol] = [] for arg in node.args: self.visit(arg) - node.symbol = ast.CallSymbol(name=node.name, args=[arg.symbol for arg in node.args]) + if arg.symbol is not None: + arg_symbols.append(arg.symbol) + node.symbol = ast.CallSymbol(name=node.name, args=arg_symbols) def visit_field(self, node): """Visit a field such as ast.Field(chain=["e", "properties", "$browser"])""" @@ -144,18 +146,19 @@ def visit_field(self, node): if len(node.chain) == 0: raise Exception("Invalid field access with empty chain") - # ClickHouse does not support subqueries accessing "x.event" like this: + # Only look for fields in the last SELECT scope. + scope = self.scopes[-1] + + # ClickHouse does not support subqueries accessing "x.event". This is forbidden: # - "SELECT event, (select count() from events where event = x.event) as c FROM events x where event = '$pageview'", # But this is supported: # - "SELECT t.big_count FROM (select count() + 100 as big_count from events) as t JOIN events e ON (e.event = t.event)", - # - # Thus we only look into scopes[-1] to see aliases in the current scope, and don't loop over all the scopes. + # Thus we don't have to recursively look into all the past scopes to find a match. - scope = self.scopes[-1] symbol: Optional[ast.Symbol] = None name = node.chain[0] - # Only look for matching tables if field contains at least two parts. + # If the field contains at least two parts, the first might be a table. if len(node.chain) > 1 and name in scope.tables: symbol = scope.tables[name] if not symbol: @@ -187,10 +190,10 @@ def lookup_field_by_name(scope: ast.SelectQuerySymbol, name: str) -> Optional[as else: named_tables = [table for table in scope.tables.values() if table.has_child(name)] anonymous_tables = [table for table in scope.anonymous_tables if table.has_child(name)] - tables = named_tables + anonymous_tables + tables_with_field = named_tables + anonymous_tables - if len(tables) > 1: + if len(tables_with_field) > 1: raise ResolverException(f"Ambiguous query. Found multiple sources for field: {name}") - elif len(tables) == 1: - return tables[0].get_child(name) + elif len(tables_with_field) == 1: + return tables_with_field[0].get_child(name) return None From 9e995fb5dc831a06dd2594fc9bce9417f7d9f9b7 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 14 Feb 2023 13:19:40 +0100 Subject: [PATCH 35/81] bit of printer cleanup --- posthog/hogql/printer.py | 83 ++++++++++++++++++++++++---------------- 1 file changed, 51 insertions(+), 32 deletions(-) diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index 47797080192db..cbce47c79caec 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -1,4 +1,5 @@ -from typing import List, Literal, Optional, Tuple, Union +from dataclasses import dataclass +from typing import List, Literal, Optional, Union from ee.clickhouse.materialized_columns.columns import get_materialized_columns from posthog.hogql import ast @@ -9,7 +10,9 @@ from posthog.hogql.visitor import Visitor -def guard_where_team_id(table_symbol: Union[ast.TableSymbol, ast.TableAliasSymbol], context: HogQLContext) -> ast.Expr: +def team_id_guard_for_table( + table_symbol: Union[ast.TableSymbol, ast.TableAliasSymbol], context: HogQLContext +) -> ast.Expr: """Add a mandatory "and(team_id, ...)" filter around the expression.""" if not context.select_team_id: raise ValueError("context.select_team_id not found") @@ -22,9 +25,15 @@ def guard_where_team_id(table_symbol: Union[ast.TableSymbol, ast.TableAliasSymbo def print_ast( - node: ast.AST, context: HogQLContext, dialect: Literal["hogql", "clickhouse"], stack: List[ast.AST] = [] + node: ast.AST, context: HogQLContext, dialect: Literal["hogql", "clickhouse"], stack: Optional[List[ast.AST]] = None ) -> str: - return Printer(context=context, dialect=dialect, stack=stack).visit(node) + return Printer(context=context, dialect=dialect, stack=stack or []).visit(node) + + +@dataclass +class JoinExprResponse: + printed_sql: str + where: Optional[ast.Expr] = None class Printer(Visitor): @@ -33,9 +42,11 @@ def __init__( ): self.context = context self.dialect = dialect + # Keep track of all traversed nodes. self.stack: List[ast.AST] = stack or [] def _last_select(self) -> Optional[ast.SelectQuery]: + """Find the last SELECT query in the stack.""" for node in reversed(self.stack): if isinstance(node, ast.SelectQuery): return node @@ -49,9 +60,9 @@ def visit(self, node: ast.AST): def visit_select_query(self, node: ast.SelectQuery): if self.dialect == "clickhouse" and not self.context.select_team_id: - raise ValueError("Full SELECT queries are disabled if select_team_id is not set") + raise ValueError("Full SELECT queries are disabled if context.select_team_id is not set") - # we will add extra clauses onto this + # We will add extra clauses onto this from the joined tables where = node.where select_from = [] @@ -60,16 +71,22 @@ def visit_select_query(self, node: ast.SelectQuery): if next_join.symbol is None: raise ValueError("Printing queries with a FROM clause is not permitted before symbol resolution") - (select_sql, extra_where) = self.visit_join_expr(next_join) - select_from.append(select_sql) + visited_join = self.visit_join_expr(next_join) + select_from.append(visited_join.printed_sql) - if extra_where is not None: + # This is an expression we must add to the SELECT's WHERE clause to limit results. + extra_where = visited_join.where + if extra_where is None: + pass + elif isinstance(extra_where, ast.Expr): if where is None: where = extra_where elif isinstance(where, ast.And): where = ast.And(exprs=[extra_where] + where.exprs) else: where = ast.And(exprs=[extra_where, where]) + else: + raise ValueError(f"Invalid where of type {type(extra_where).__name__} returned by join_expr") next_join = next_join.next_join @@ -80,16 +97,6 @@ def visit_select_query(self, node: ast.SelectQuery): group_by = [self.visit(column) for column in node.group_by] if node.group_by else None order_by = [self.visit(column) for column in node.order_by] if node.order_by else None - limit = node.limit - if self.context.limit_top_select: - if limit is not None: - if isinstance(limit, ast.Constant) and isinstance(limit.value, int): - limit.value = min(limit.value, MAX_SELECT_RETURNED_ROWS) - else: - limit = ast.Call(name="min2", args=[ast.Constant(value=MAX_SELECT_RETURNED_ROWS), limit]) - elif len(self.stack) == 1: - limit = ast.Constant(value=MAX_SELECT_RETURNED_ROWS) - clauses = [ f"SELECT {'DISTINCT ' if node.distinct else ''}{', '.join(columns)}", f"FROM {' '.join(select_from)}" if len(select_from) > 0 else None, @@ -99,6 +106,17 @@ def visit_select_query(self, node: ast.SelectQuery): "PREWHERE " + prewhere if prewhere else None, f"ORDER BY {', '.join(order_by)}" if order_by and len(order_by) > 0 else None, ] + + limit = node.limit + if self.context.limit_top_select and len(self.stack) == 1: + if limit is not None: + if isinstance(limit, ast.Constant) and isinstance(limit.value, int): + limit.value = min(limit.value, MAX_SELECT_RETURNED_ROWS) + else: + limit = ast.Call(name="min2", args=[ast.Constant(value=MAX_SELECT_RETURNED_ROWS), limit]) + else: + limit = ast.Constant(value=MAX_SELECT_RETURNED_ROWS) + if limit is not None: clauses.append(f"LIMIT {self.visit(limit)}") if node.offset is not None: @@ -113,9 +131,9 @@ def visit_select_query(self, node: ast.SelectQuery): response = f"({response})" return response - def visit_join_expr(self, node: ast.JoinExpr) -> Tuple[str, Optional[ast.Expr]]: + def visit_join_expr(self, node: ast.JoinExpr) -> JoinExprResponse: # return constraints we must place on the select query - extra_where = None + extra_where: Optional[ast.Expr] = None select_from = [] if node.join_type is not None: @@ -132,13 +150,13 @@ def visit_join_expr(self, node: ast.JoinExpr) -> Tuple[str, Optional[ast.Expr]]: select_from.append(f"AS {print_hogql_identifier(node.alias)}") if self.dialect == "clickhouse": - extra_where = guard_where_team_id(node.symbol, self.context) + extra_where = team_id_guard_for_table(node.symbol, self.context) elif isinstance(node.symbol, ast.TableSymbol): select_from.append(print_hogql_identifier(node.symbol.table.clickhouse_table())) if self.dialect == "clickhouse": - extra_where = guard_where_team_id(node.symbol, self.context) + extra_where = team_id_guard_for_table(node.symbol, self.context) elif isinstance(node.symbol, ast.SelectQuerySymbol): select_from.append(self.visit(node.table)) @@ -155,7 +173,7 @@ def visit_join_expr(self, node: ast.JoinExpr) -> Tuple[str, Optional[ast.Expr]]: if node.constraint is not None: select_from.append(f"ON {self.visit(node.constraint)}") - return (" ".join(select_from), extra_where) + return JoinExprResponse(printed_sql=" ".join(select_from), where=extra_where) def visit_binary_operation(self, node: ast.BinaryOperation): if node.op == ast.BinaryOperationType.Add: @@ -244,16 +262,17 @@ def visit_field(self, node: ast.Field): raise ValueError(f"Field {original_field} has no symbol") if self.dialect == "hogql": - # When printing HogQL, we print the properties out as a chain instead of converting them to Clickhouse SQL + # When printing HogQL, we print the properties out as a chain as they are. return ".".join([print_hogql_identifier(identifier) for identifier in node.chain]) - elif node.chain == ["*"]: - raise ValueError("Selecting * not implemented") + + if node.chain == ["*"]: # query = f"tuple({','.join(SELECT_STAR_FROM_EVENTS_FIELDS)})" # return self.visit(parse_expr(query)) + raise ValueError("Selecting * not yet implemented") elif node.chain == ["person"]: - raise ValueError("Selecting person not implemented") # query = "tuple(distinct_id, person.id, person.created_at, person.properties.name, person.properties.email)" # return self.visit(parse_expr(query)) + raise ValueError("Selecting person not yet implemented") elif node.symbol is not None: select_query = self._last_select() select: Optional[ast.SelectQuerySymbol] = select_query.symbol if select_query else None @@ -261,7 +280,7 @@ def visit_field(self, node: ast.Field): raise ValueError(f"Can't find SelectQuerySymbol for field: {original_field}") return SymbolPrinter(select=select, context=self.context).visit(node.symbol) else: - raise ValueError(f"Unknown Symbol, can not print {type(node.symbol)}") + raise ValueError(f"Unknown Symbol, can not print {type(node.symbol).__name__}") def visit_call(self, node: ast.Call): if node.name in HOGQL_AGGREGATIONS: @@ -337,8 +356,8 @@ def visit_field_symbol(self, symbol: ast.FieldSymbol): else: field_sql = printed_field + # TODO: refactor this property access logging, also add person properties if printed_field != "properties": - # TODO: refactor this property access logging self.context.field_access_logs.append( HogQLFieldAccess( [symbol.name], @@ -355,7 +374,7 @@ def visit_property_symbol(self, symbol: ast.PropertySymbol): self.context.values[key] = symbol.name table = symbol.field.table - if isinstance(table, ast.TableAliasSymbol): + while isinstance(table, ast.TableAliasSymbol): table = table.table # TODO: cache this @@ -382,7 +401,7 @@ def visit_property_symbol(self, symbol: ast.PropertySymbol): def visit_select_query_alias_symbol(self, symbol: ast.SelectQueryAliasSymbol): return print_hogql_identifier(symbol.name) - def visit_column_alias_symbol(self, symbol: ast.SelectQueryAliasSymbol): + def visit_field_alias_symbol(self, symbol: ast.SelectQueryAliasSymbol): return print_hogql_identifier(symbol.name) def visit_unknown(self, symbol: ast.AST): From 166e7fe30356da2d7be9e7e5457d838b4be0464f Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 14 Feb 2023 15:28:40 +0100 Subject: [PATCH 36/81] placeholders in query --- posthog/hogql/query.py | 11 +++++++++-- posthog/hogql/test/test_query.py | 21 +++++++++++++-------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/posthog/hogql/query.py b/posthog/hogql/query.py index f47e48707a987..7938da035b6fd 100644 --- a/posthog/hogql/query.py +++ b/posthog/hogql/query.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Union +from typing import Dict, List, Optional, Union from pydantic import BaseModel, Extra @@ -6,6 +6,7 @@ from posthog.hogql import ast from posthog.hogql.hogql import HogQLContext from posthog.hogql.parser import parse_select +from posthog.hogql.placeholders import assert_no_placeholders, replace_placeholders from posthog.hogql.printer import print_ast from posthog.hogql.resolver import resolve_symbols from posthog.models import Team @@ -28,13 +29,19 @@ def execute_hogql_query( query: Union[str, ast.SelectQuery], team: Team, query_type: str = "hogql_query", + placeholders: Optional[Dict[str, ast.Expr]] = None, workload: Workload = Workload.ONLINE, ) -> HogQLQueryResponse: if isinstance(query, ast.SelectQuery): select_query = query query = None else: - select_query = parse_select(str(query), no_placeholders=True) + select_query = parse_select(str(query)) + + if placeholders: + select_query = replace_placeholders(select_query, placeholders) + else: + assert_no_placeholders(select_query) if select_query.limit is None: select_query.limit = ast.Constant(value=1000) diff --git a/posthog/hogql/test/test_query.py b/posthog/hogql/test/test_query.py index c1cefa49361f0..25c9256bad1fd 100644 --- a/posthog/hogql/test/test_query.py +++ b/posthog/hogql/test/test_query.py @@ -1,5 +1,6 @@ from freezegun import freeze_time +from posthog.hogql import ast from posthog.hogql.query import execute_hogql_query from posthog.models.utils import UUIDT from posthog.test.base import APIBaseTest, ClickhouseTestMixin, _create_event, _create_person, flush_persons_and_events @@ -30,8 +31,9 @@ def test_query(self): random_uuid = self._create_random_events() response = execute_hogql_query( - f"select count(), event from events where properties.random_uuid = '{random_uuid}' group by event", - self.team, + "select count(), event from events where properties.random_uuid = {random_uuid} group by event", + placeholders={"random_uuid": ast.Constant(value=random_uuid)}, + team=self.team, ) self.assertEqual( response.clickhouse, @@ -44,8 +46,9 @@ def test_query(self): self.assertEqual(response.results, [(2, "random event")]) response = execute_hogql_query( - f"select count, event from (select count() as count, event from events where properties.random_uuid = '{random_uuid}' group by event) group by count, event", - self.team, + "select count, event from (select count() as count, event from events where properties.random_uuid = {random_uuid} group by event) group by count, event", + placeholders={"random_uuid": ast.Constant(value=random_uuid)}, + team=self.team, ) self.assertEqual( response.clickhouse, @@ -58,8 +61,9 @@ def test_query(self): self.assertEqual(response.results, [(2, "random event")]) response = execute_hogql_query( - f"select count, event from (select count() as count, event from events where properties.random_uuid = '{random_uuid}' group by event) as c group by count, event", - self.team, + "select count, event from (select count() as count, event from events where properties.random_uuid = {random_uuid} group by event) as c group by count, event", + placeholders={"random_uuid": ast.Constant(value=random_uuid)}, + team=self.team, ) self.assertEqual( response.clickhouse, @@ -72,8 +76,9 @@ def test_query(self): self.assertEqual(response.results, [(2, "random event")]) response = execute_hogql_query( - f"select distinct properties.email from persons where properties.random_uuid = '{random_uuid}'", - self.team, + "select distinct properties.email from persons where properties.random_uuid = {random_uuid}", + placeholders={"random_uuid": ast.Constant(value=random_uuid)}, + team=self.team, ) self.assertEqual( response.clickhouse, From 2287c07033b4846e8b878fdf0425b3c14a0e27ef Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 14 Feb 2023 19:10:36 +0100 Subject: [PATCH 37/81] changes --- posthog/hogql/ast.py | 28 +++--- posthog/hogql/database.py | 136 +++++++++++++++++------------ posthog/hogql/printer.py | 7 +- posthog/hogql/test/test_printer.py | 2 +- 4 files changed, 104 insertions(+), 69 deletions(-) diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index ee4854ecbe202..7c3aedb466185 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -5,7 +5,7 @@ from pydantic import BaseModel, Extra from pydantic import Field as PydanticField -from posthog.hogql.database import StringJSONValue, Table +from posthog.hogql.database import ComplexField, DatabaseField, StringJSONDatabaseField, Table # NOTE: when you add new AST fields or nodes, add them to the Visitor classes in visitor.py as well! @@ -29,7 +29,7 @@ def accept(self, visitor): class Symbol(AST): def get_child(self, name: str) -> "Symbol": - raise NotImplementedError() + raise NotImplementedError("Symbol.get_child not overridden") def has_child(self, name: str) -> bool: return self.get_child(name) is not None @@ -50,7 +50,7 @@ class TableSymbol(Symbol): table: Table def has_child(self, name: str) -> bool: - return name in self.table.__fields__ + return self.table.has_field(name) def get_child(self, name: str) -> Symbol: if self.has_child(name): @@ -114,6 +114,10 @@ class CallSymbol(Symbol): args: List[Symbol] +class ConstantSymbol(Symbol): + value: Any + + class FieldSymbol(Symbol): name: str table: Union[TableSymbol, TableAliasSymbol, SelectQuerySymbol, SelectQueryAliasSymbol] @@ -126,20 +130,22 @@ def get_child(self, name: str) -> Symbol: if isinstance(table_symbol, TableSymbol): db_table = table_symbol.table if isinstance(db_table, Table): - if self.name in db_table.__fields__ and isinstance( - db_table.__fields__[self.name].default, StringJSONValue + if db_table.has_field(self.name) and ( + isinstance(db_table.get_field(self.name), StringJSONDatabaseField) + or isinstance(db_table.get_field(self.name), ComplexField) ): - return PropertySymbol(name=name, field=self) - raise ValueError(f"Can not access property {name} on field {self.name}.") + return PropertySymbol(name=name, property=db_table.get_field(self.name), parent=self) - -class ConstantSymbol(Symbol): - value: Any + raise ValueError(f'Can not access property "{name}" on field "{self.name}".') class PropertySymbol(Symbol): name: str - field: FieldSymbol + property: Union[DatabaseField, ComplexField] + parent: Union[FieldSymbol, "PropertySymbol"] + + +PropertySymbol.update_forward_refs(PropertySymbol=PropertySymbol) class Expr(AST): diff --git a/posthog/hogql/database.py b/posthog/hogql/database.py index 314b8621a00b9..8fe9b25754609 100644 --- a/posthog/hogql/database.py +++ b/posthog/hogql/database.py @@ -1,105 +1,129 @@ +from typing import Union + from pydantic import BaseModel, Extra -class Field(BaseModel): +class DatabaseField(BaseModel): + """Base class for a field in a database table.""" + class Config: extra = Extra.forbid + name: str + array: bool = False -class IntegerValue(Field): - pass +class ComplexField(BaseModel): + """Base class for a complex field with custom properties.""" + + class Config: + extra = Extra.forbid + + def has_child(self, name: str) -> bool: + return hasattr(self, name) -class StringValue(Field): + def get_child(self, name: str) -> DatabaseField: + return getattr(self, name) + + +class IntegerDatabaseField(DatabaseField): pass -class StringJSONValue(Field): +class StringDatabaseField(DatabaseField): pass -class DateTimeValue(Field): +class StringJSONDatabaseField(DatabaseField): pass -class BooleanValue(Field): +class DateTimeDatabaseField(DatabaseField): pass -class ArrayValue(Field): - field: Field +class BooleanDatabaseField(DatabaseField): + pass class Table(BaseModel): class Config: extra = Extra.forbid + def has_field(self, name: str) -> bool: + return hasattr(self, name) + + def get_field(self, name: str) -> Union[DatabaseField, ComplexField]: + if self.has_field(name): + return getattr(self, name) + raise ValueError(f'Field "{name}" not found on table {self.__class__.__name__}') + def clickhouse_table(self): - raise NotImplementedError() + raise NotImplementedError("Table.clickhouse_table not overridden") class PersonsTable(Table): - id: StringValue = StringValue() - created_at: DateTimeValue = DateTimeValue() - team_id: IntegerValue = IntegerValue() - properties: StringJSONValue = StringJSONValue() - is_identified: BooleanValue = BooleanValue() - is_deleted: BooleanValue = BooleanValue() - version: IntegerValue = IntegerValue() + id: StringDatabaseField = StringDatabaseField(name="id") + created_at: DateTimeDatabaseField = DateTimeDatabaseField(name="created_at") + team_id: IntegerDatabaseField = IntegerDatabaseField(name="team_id") + properties: StringJSONDatabaseField = StringJSONDatabaseField(name="properties") + is_identified: BooleanDatabaseField = BooleanDatabaseField(name="is_identified") + is_deleted: BooleanDatabaseField = BooleanDatabaseField(name="is_deleted") + version: IntegerDatabaseField = IntegerDatabaseField(name="version") def clickhouse_table(self): return "person" class PersonDistinctIdTable(Table): - team_id: IntegerValue = IntegerValue() - distinct_id: StringValue = StringValue() - person_id: StringValue = StringValue() - is_deleted: BooleanValue = BooleanValue() - version: IntegerValue = IntegerValue() + team_id: IntegerDatabaseField = IntegerDatabaseField(name="team_id") + distinct_id: StringDatabaseField = StringDatabaseField(name="distinct_id") + person_id: StringDatabaseField = StringDatabaseField(name="person_id") + is_deleted: BooleanDatabaseField = BooleanDatabaseField(name="is_deleted") + version: IntegerDatabaseField = IntegerDatabaseField(name="version") def clickhouse_table(self): return "person_distinct_id2" -class PersonFieldsOnEvents(Table): - id: StringValue = StringValue() - created_at: DateTimeValue = DateTimeValue() - properties: StringJSONValue = StringJSONValue() +class EventsPersonComplexField(ComplexField): + id: StringDatabaseField = StringDatabaseField(name="person_id") + created_at: DateTimeDatabaseField = DateTimeDatabaseField(name="person_created_at") + properties: StringJSONDatabaseField = StringJSONDatabaseField(name="person_properties") class EventsTable(Table): - uuid: StringValue = StringValue() - event: StringValue = StringValue() - timestamp: DateTimeValue = DateTimeValue() - properties: StringJSONValue = StringJSONValue() - elements_chain: StringValue = StringValue() - created_at: DateTimeValue = DateTimeValue() - distinct_id: StringValue = StringValue() - team_id: IntegerValue = IntegerValue() - person: PersonFieldsOnEvents = PersonFieldsOnEvents() + uuid: StringDatabaseField = StringDatabaseField(name="uuid") + event: StringDatabaseField = StringDatabaseField(name="event") + timestamp: DateTimeDatabaseField = DateTimeDatabaseField(name="timestamp") + properties: StringJSONDatabaseField = StringJSONDatabaseField(name="properties") + elements_chain: StringDatabaseField = StringDatabaseField(name="elements_chain") + created_at: DateTimeDatabaseField = DateTimeDatabaseField(name="created_at") + distinct_id: StringDatabaseField = StringDatabaseField(name="distinct_id") + team_id: IntegerDatabaseField = IntegerDatabaseField(name="team_id") + person: EventsPersonComplexField = EventsPersonComplexField() def clickhouse_table(self): return "events" class SessionRecordingEvents(Table): - uuid: StringValue = StringValue() - timestamp: DateTimeValue = DateTimeValue() - team_id: IntegerValue = IntegerValue() - distinct_id: StringValue = StringValue() - session_id: StringValue = StringValue() - window_id: StringValue = StringValue() - snapshot_data: StringValue = StringValue() - created_at: DateTimeValue = DateTimeValue() - has_full_snapshot: BooleanValue = BooleanValue() - events_summary: ArrayValue = ArrayValue(field=BooleanValue()) - click_count: IntegerValue = IntegerValue() - keypress_count: IntegerValue = IntegerValue() - timestamps_summary: ArrayValue = ArrayValue(field=DateTimeValue()) - first_event_timestamp: DateTimeValue = DateTimeValue() - last_event_timestamp: DateTimeValue = DateTimeValue() - urls: ArrayValue = ArrayValue(field=StringValue()) + uuid: StringDatabaseField = StringDatabaseField(name="uuid") + timestamp: DateTimeDatabaseField = DateTimeDatabaseField(name="timestamp") + team_id: IntegerDatabaseField = IntegerDatabaseField(name="team_id") + distinct_id: StringDatabaseField = StringDatabaseField(name="distinct_id") + session_id: StringDatabaseField = StringDatabaseField(name="session_id") + window_id: StringDatabaseField = StringDatabaseField(name="window_id") + snapshot_data: StringDatabaseField = StringDatabaseField(name="snapshot_data") + created_at: DateTimeDatabaseField = DateTimeDatabaseField(name="created_at") + has_full_snapshot: BooleanDatabaseField = BooleanDatabaseField(name="has_full_snapshot") + events_summary: BooleanDatabaseField = BooleanDatabaseField(name="events_summary", array=True) + click_count: IntegerDatabaseField = IntegerDatabaseField(name="click_count") + keypress_count: IntegerDatabaseField = IntegerDatabaseField(name="keypress_count") + timestamps_summary: DateTimeDatabaseField = DateTimeDatabaseField(name="timestamps_summary", array=True) + first_event_timestamp: DateTimeDatabaseField = DateTimeDatabaseField(name="first_event_timestamp") + last_event_timestamp: DateTimeDatabaseField = DateTimeDatabaseField(name="last_event_timestamp") + urls: StringDatabaseField = StringDatabaseField(name="urls", array=True) def clickhouse_table(self): return "session_recording_events" @@ -115,11 +139,13 @@ class Config: person_distinct_ids: PersonDistinctIdTable = PersonDistinctIdTable() session_recording_events: SessionRecordingEvents = SessionRecordingEvents() - def get_table(self, table_name: str): - return getattr(self, table_name) - - def has_table(self, table_name: str): + def has_table(self, table_name: str) -> bool: return hasattr(self, table_name) + def get_table(self, table_name: str) -> Table: + if self.has_table(table_name): + return getattr(self, table_name) + raise ValueError(f'Table "{table_name}" not found in database') + database = Database() diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index cbce47c79caec..4511ce66775ed 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -370,10 +370,13 @@ def visit_field_symbol(self, symbol: ast.FieldSymbol): return field_sql def visit_property_symbol(self, symbol: ast.PropertySymbol): + if isinstance(symbol.parent, ast.PropertySymbol): + return self.visit_property_symbol(symbol.parent) + key = f"hogql_val_{len(self.context.values)}" self.context.values[key] = symbol.name - table = symbol.field.table + table = symbol.parent.table while isinstance(table, ast.TableAliasSymbol): table = table.table @@ -384,7 +387,7 @@ def visit_property_symbol(self, symbol: ast.PropertySymbol): if materialized_column: property_sql = print_hogql_identifier(materialized_column) else: - field_sql = self.visit(symbol.field) + field_sql = self.visit(symbol.parent) property_sql = trim_quotes_expr(f"JSONExtractRaw({field_sql}, %({key})s)") self.context.field_access_logs.append( diff --git a/posthog/hogql/test/test_printer.py b/posthog/hogql/test/test_printer.py index bf1a055ee474b..04136ff5738ec 100644 --- a/posthog/hogql/test/test_printer.py +++ b/posthog/hogql/test/test_printer.py @@ -208,7 +208,7 @@ def test_expr_parse_errors(self): self._assert_expr_error( "avg(avg(properties.bla))", "Aggregation 'avg' cannot be nested inside another aggregation 'avg'." ) - self._assert_expr_error("person.chipotle", "Unknown person field 'chipotle'") + self._assert_expr_error("person.chipotle", 'Can not access property "chipotle" on compelx field "person".') def test_expr_syntax_errors(self): self._assert_expr_error("(", "line 1, column 1: no viable alternative at input '('") From 92983fb36a68637efde78a4df98e7ebd6a2dce8d Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 14 Feb 2023 21:57:25 +0100 Subject: [PATCH 38/81] person properties fake table --- posthog/hogql/ast.py | 32 ++++----- posthog/hogql/database.py | 25 ++----- posthog/hogql/printer.py | 106 +++++++++++++++++++---------- posthog/hogql/resolver.py | 8 +-- posthog/hogql/test/test_printer.py | 1 - 5 files changed, 98 insertions(+), 74 deletions(-) diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index 7c3aedb466185..eb18afdda34a5 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -5,7 +5,7 @@ from pydantic import BaseModel, Extra from pydantic import Field as PydanticField -from posthog.hogql.database import ComplexField, DatabaseField, StringJSONDatabaseField, Table +from posthog.hogql.database import DatabaseField, StringJSONDatabaseField, Table # NOTE: when you add new AST fields or nodes, add them to the Visitor classes in visitor.py as well! @@ -122,30 +122,30 @@ class FieldSymbol(Symbol): name: str table: Union[TableSymbol, TableAliasSymbol, SelectQuerySymbol, SelectQueryAliasSymbol] - def get_child(self, name: str) -> Symbol: + def resolve_database_field(self) -> Optional[Union[DatabaseField, Table]]: table_symbol = self.table while isinstance(table_symbol, TableAliasSymbol): table_symbol = table_symbol.table - if isinstance(table_symbol, TableSymbol): - db_table = table_symbol.table - if isinstance(db_table, Table): - if db_table.has_field(self.name) and ( - isinstance(db_table.get_field(self.name), StringJSONDatabaseField) - or isinstance(db_table.get_field(self.name), ComplexField) - ): - return PropertySymbol(name=name, property=db_table.get_field(self.name), parent=self) + return table_symbol.table.get_field(self.name) + return None - raise ValueError(f'Can not access property "{name}" on field "{self.name}".') + def get_child(self, name: str) -> Symbol: + database_field = self.resolve_database_field() + if database_field is None: + raise ValueError(f'Can not access property "{name}" on field "{self.name}".') + if isinstance(database_field, Table): + return FieldSymbol(name=name, table=TableSymbol(table=database_field)) + if isinstance(database_field, StringJSONDatabaseField): + return PropertySymbol(name=name, parent=self) + raise ValueError( + f'Can not access property "{name}" on field "{self.name}" of type: {type(database_field).__name__}' + ) class PropertySymbol(Symbol): name: str - property: Union[DatabaseField, ComplexField] - parent: Union[FieldSymbol, "PropertySymbol"] - - -PropertySymbol.update_forward_refs(PropertySymbol=PropertySymbol) + parent: FieldSymbol class Expr(AST): diff --git a/posthog/hogql/database.py b/posthog/hogql/database.py index 8fe9b25754609..dc9c8bf87d3ed 100644 --- a/posthog/hogql/database.py +++ b/posthog/hogql/database.py @@ -1,5 +1,3 @@ -from typing import Union - from pydantic import BaseModel, Extra @@ -13,19 +11,6 @@ class Config: array: bool = False -class ComplexField(BaseModel): - """Base class for a complex field with custom properties.""" - - class Config: - extra = Extra.forbid - - def has_child(self, name: str) -> bool: - return hasattr(self, name) - - def get_child(self, name: str) -> DatabaseField: - return getattr(self, name) - - class IntegerDatabaseField(DatabaseField): pass @@ -53,7 +38,7 @@ class Config: def has_field(self, name: str) -> bool: return hasattr(self, name) - def get_field(self, name: str) -> Union[DatabaseField, ComplexField]: + def get_field(self, name: str) -> DatabaseField: if self.has_field(name): return getattr(self, name) raise ValueError(f'Field "{name}" not found on table {self.__class__.__name__}') @@ -86,11 +71,15 @@ def clickhouse_table(self): return "person_distinct_id2" -class EventsPersonComplexField(ComplexField): +class EventsPersonSubTable(Table): id: StringDatabaseField = StringDatabaseField(name="person_id") created_at: DateTimeDatabaseField = DateTimeDatabaseField(name="person_created_at") properties: StringJSONDatabaseField = StringJSONDatabaseField(name="person_properties") + def clickhouse_table(self): + # This is a bit of a hack to make sure person.properties.x works + return "events" + class EventsTable(Table): uuid: StringDatabaseField = StringDatabaseField(name="uuid") @@ -101,7 +90,7 @@ class EventsTable(Table): created_at: DateTimeDatabaseField = DateTimeDatabaseField(name="created_at") distinct_id: StringDatabaseField = StringDatabaseField(name="distinct_id") team_id: IntegerDatabaseField = IntegerDatabaseField(name="team_id") - person: EventsPersonComplexField = EventsPersonComplexField() + person: EventsPersonSubTable = EventsPersonSubTable() def clickhouse_table(self): return "events" diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index 4511ce66775ed..c141298f68efa 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -1,10 +1,11 @@ from dataclasses import dataclass -from typing import List, Literal, Optional, Union +from typing import List, Literal, Optional, Union, cast from ee.clickhouse.materialized_columns.columns import get_materialized_columns from posthog.hogql import ast from posthog.hogql.constants import CLICKHOUSE_FUNCTIONS, HOGQL_AGGREGATIONS, MAX_SELECT_RETURNED_ROWS from posthog.hogql.context import HogQLContext, HogQLFieldAccess +from posthog.hogql.database import database from posthog.hogql.print_string import print_hogql_identifier from posthog.hogql.resolver import ResolverException, lookup_field_by_name from posthog.hogql.visitor import Visitor @@ -339,65 +340,100 @@ def visit_table_alias_symbol(self, symbol: ast.TableAliasSymbol): return print_hogql_identifier(symbol.name) def visit_field_symbol(self, symbol: ast.FieldSymbol): - printed_field = print_hogql_identifier(symbol.name) - try: symbol_with_name_in_scope = lookup_field_by_name(self.select, symbol.name) except ResolverException: symbol_with_name_in_scope = None - if ( - symbol_with_name_in_scope != symbol - or isinstance(symbol.table, ast.TableAliasSymbol) - or isinstance(symbol.table, ast.SelectQueryAliasSymbol) - ): - table_prefix = self.visit(symbol.table) - field_sql = f"{table_prefix}.{printed_field}" - else: - field_sql = printed_field - - # TODO: refactor this property access logging, also add person properties - if printed_field != "properties": - self.context.field_access_logs.append( - HogQLFieldAccess( - [symbol.name], - "event", - symbol.name, - field_sql, + if isinstance(symbol.table, ast.TableSymbol) or isinstance(symbol.table, ast.TableAliasSymbol): + resolved_field = symbol.resolve_database_field() + if resolved_field is None: + raise ValueError(f'Can\'t resolve field "{symbol.name}" on table.') + + field_sql = print_hogql_identifier(resolved_field.name) + if ( + resolved_field.name != symbol.name + or isinstance(symbol.table, ast.TableAliasSymbol) + or symbol_with_name_in_scope != symbol + ): + field_sql = f"{self.visit(symbol.table)}.{field_sql}" + + # TODO: refactor this lefacy logging + if symbol.name != "properties": + real_table = symbol.table + while isinstance(real_table, ast.TableAliasSymbol): + real_table = real_table.table + + self.context.field_access_logs.append( + HogQLFieldAccess( + [symbol.name], + "event" if real_table.table == database.events else "person", + symbol.name, + field_sql, + ) ) - ) + + elif isinstance(symbol.table, ast.SelectQuerySymbol) or isinstance(symbol.table, ast.SelectQueryAliasSymbol): + field_sql = print_hogql_identifier(symbol.name) + if isinstance(symbol.table, ast.SelectQueryAliasSymbol) or symbol_with_name_in_scope != symbol: + field_sql = f"{self.visit(symbol.table)}.{field_sql}" + + else: + raise ValueError(f"Unknown FieldSymbol table type: {type(symbol.table).__name__}") return field_sql def visit_property_symbol(self, symbol: ast.PropertySymbol): - if isinstance(symbol.parent, ast.PropertySymbol): - return self.visit_property_symbol(symbol.parent) + field_symbol = symbol.parent key = f"hogql_val_{len(self.context.values)}" self.context.values[key] = symbol.name - table = symbol.parent.table + table = field_symbol.table while isinstance(table, ast.TableAliasSymbol): table = table.table + if not isinstance(table, ast.TableSymbol): + raise ValueError(f"Unknown PropertySymbol table type: {type(table).__name__}") + + table_name = table.table.clickhouse_table() + + field = field_symbol.resolve_database_field() + if field is None: + raise ValueError(f"Can't resolve field {field_symbol.name} on table {table_name}") + + field_name = cast(Union[Literal["properties"], Literal["person_properties"]], field.name) + # TODO: cache this - materialized_columns = get_materialized_columns(table.table.clickhouse_table()) - materialized_column = materialized_columns.get((symbol.name, "properties"), None) + materialized_columns = get_materialized_columns(table_name) + materialized_column = materialized_columns.get((symbol.name, field_name), None) if materialized_column: property_sql = print_hogql_identifier(materialized_column) else: - field_sql = self.visit(symbol.parent) + field_sql = self.visit(field_symbol) property_sql = trim_quotes_expr(f"JSONExtractRaw({field_sql}, %({key})s)") - self.context.field_access_logs.append( - HogQLFieldAccess( - ["properties", symbol.name], - "event.properties", - symbol.name, - property_sql, + if field_name == "properties": + # TODO: refactor this lefacy logging + self.context.field_access_logs.append( + HogQLFieldAccess( + ["properties", symbol.name], + "event.properties", + symbol.name, + property_sql, + ) + ) + elif field_name == "person_properties": + # TODO: refactor this lefacy logging + self.context.field_access_logs.append( + HogQLFieldAccess( + ["person", "properties", symbol.name], + "person.properties", + symbol.name, + property_sql, + ) ) - ) return property_sql diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py index b86bfc5c286c2..b9ecb1c00bcdb 100644 --- a/posthog/hogql/resolver.py +++ b/posthog/hogql/resolver.py @@ -167,14 +167,14 @@ def visit_field(self, node): raise ResolverException(f"Unable to resolve field: {name}") # Recursively resolve the rest of the chain until we can point to the deepest node. + loop_symbol = symbol for child_name in node.chain[1:]: - symbol = symbol.get_child(child_name) - if symbol is None: + loop_symbol = loop_symbol.get_child(child_name) + if loop_symbol is None: raise ResolverException( f"Cannot resolve symbol {'.'.join(node.chain)}. Unable to resolve {child_name} on {name}" ) - - node.symbol = symbol + node.symbol = loop_symbol def visit_constant(self, node): """Visit a constant""" diff --git a/posthog/hogql/test/test_printer.py b/posthog/hogql/test/test_printer.py index 04136ff5738ec..4d11eaf5dcc94 100644 --- a/posthog/hogql/test/test_printer.py +++ b/posthog/hogql/test/test_printer.py @@ -177,7 +177,6 @@ def test_materialized_fields_and_properties(self): materialize("events", "$browser%%%#@!@") self.assertEqual(self._expr("properties['$browser%%%#@!@']"), "mat_$browser_______") - # TODO: get person properties working materialize("events", "$initial_waffle", table_column="person_properties") self.assertEqual(self._expr("person.properties['$initial_waffle']"), "mat_pp_$initial_waffle") From 66b47ab6bcc97b191dc07f1acacd5af4ec1c645d Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 14 Feb 2023 22:14:33 +0100 Subject: [PATCH 39/81] pass two more tests --- posthog/hogql/test/test_printer.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/posthog/hogql/test/test_printer.py b/posthog/hogql/test/test_printer.py index 4d11eaf5dcc94..ef6bf47e77a7c 100644 --- a/posthog/hogql/test/test_printer.py +++ b/posthog/hogql/test/test_printer.py @@ -74,7 +74,7 @@ def test_fields_and_properties(self): context = HogQLContext() self.assertEqual( self._expr("person.properties.bla", context), - "replaceRegexpAll(JSONExtractRaw(person_properties, %(hogql_val_0)s), '^\"|\"$', '')", + "replaceRegexpAll(JSONExtractRaw(events.person_properties, %(hogql_val_0)s), '^\"|\"$', '')", ) self.assertEqual( context.field_access_logs, @@ -83,7 +83,7 @@ def test_fields_and_properties(self): ["person", "properties", "bla"], "person.properties", "bla", - "replaceRegexpAll(JSONExtractRaw(person_properties, %(hogql_val_0)s), '^\"|\"$', '')", + "replaceRegexpAll(JSONExtractRaw(events.person_properties, %(hogql_val_0)s), '^\"|\"$', '')", ) ], ) @@ -109,14 +109,17 @@ def test_fields_and_properties(self): ) context = HogQLContext() - self.assertEqual(self._expr("person_id", context), "person_id") - self.assertEqual(context.field_access_logs, [HogQLFieldAccess(["person_id"], "person", "id", "person_id")]) + self.assertEqual(self._expr("person.id", context), "events.person_id") + self.assertEqual( + context.field_access_logs, + [HogQLFieldAccess(["id"], "person", "id", "events.person_id")], + ) context = HogQLContext() - self.assertEqual(self._expr("person.created_at", context), "person_created_at") + self.assertEqual(self._expr("person.created_at", context), "events.person_created_at") self.assertEqual( context.field_access_logs, - [HogQLFieldAccess(["person", "created_at"], "person", "created_at", "person_created_at")], + [HogQLFieldAccess(["created_at"], "person", "created_at", "events.person_created_at")], ) def test_hogql_properties(self): @@ -207,7 +210,7 @@ def test_expr_parse_errors(self): self._assert_expr_error( "avg(avg(properties.bla))", "Aggregation 'avg' cannot be nested inside another aggregation 'avg'." ) - self._assert_expr_error("person.chipotle", 'Can not access property "chipotle" on compelx field "person".') + self._assert_expr_error("person.chipotle", 'Field "chipotle" not found on table EventsPersonSubTable') def test_expr_syntax_errors(self): self._assert_expr_error("(", "line 1, column 1: no viable alternative at input '('") From fa22dddee717b030e579d41c39e1ac6153977d92 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 15 Feb 2023 09:09:17 +0100 Subject: [PATCH 40/81] legacy person properties --- posthog/hogql/printer.py | 40 ++++++++++++++++++++---------- posthog/hogql/test/test_printer.py | 4 +-- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index c141298f68efa..4511cd4b9b601 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from typing import List, Literal, Optional, Union, cast -from ee.clickhouse.materialized_columns.columns import get_materialized_columns +from ee.clickhouse.materialized_columns.columns import TablesWithMaterializedColumns, get_materialized_columns from posthog.hogql import ast from posthog.hogql.constants import CLICKHOUSE_FUNCTIONS, HOGQL_AGGREGATIONS, MAX_SELECT_RETURNED_ROWS from posthog.hogql.context import HogQLContext, HogQLFieldAccess @@ -9,6 +9,7 @@ from posthog.hogql.print_string import print_hogql_identifier from posthog.hogql.resolver import ResolverException, lookup_field_by_name from posthog.hogql.visitor import Visitor +from posthog.models.property import PropertyName, TableColumn def team_id_guard_for_table( @@ -333,6 +334,13 @@ def __init__(self, select: ast.SelectQuerySymbol, context: HogQLContext): self.select = select self.context = context + def _get_materialized_column( + self, table_name: TablesWithMaterializedColumns, property_name: PropertyName, field_name: TableColumn + ) -> Optional[str]: + materialized_columns = get_materialized_columns(table_name) + materialized_column = materialized_columns.get((property_name, field_name), None) + return materialized_column + def visit_table_symbol(self, symbol: ast.TableSymbol): return print_hogql_identifier(symbol.table.clickhouse_table()) @@ -351,14 +359,20 @@ def visit_field_symbol(self, symbol: ast.FieldSymbol): raise ValueError(f'Can\'t resolve field "{symbol.name}" on table.') field_sql = print_hogql_identifier(resolved_field.name) - if ( - resolved_field.name != symbol.name - or isinstance(symbol.table, ast.TableAliasSymbol) - or symbol_with_name_in_scope != symbol - ): + + # :KLUDGE: Legacy person properties handling. Assume we're in a context where the tables have been joined, + # and this "person_props" alias is accessible to us. + if resolved_field == database.events.person.properties: + if not self.context.using_person_on_events: + field_sql = "person_props" + + # If the field is called on a table that has an alias, prepend the table alias. + # If there's another field with the same name in the scope that's not this, prepend the full table name. + # Note: we don't prepend a table name for the special "person_properties" field. + elif isinstance(symbol.table, ast.TableAliasSymbol) or symbol_with_name_in_scope != symbol: field_sql = f"{self.visit(symbol.table)}.{field_sql}" - # TODO: refactor this lefacy logging + # TODO: refactor this legacy logging if symbol.name != "properties": real_table = symbol.table while isinstance(real_table, ast.TableAliasSymbol): @@ -367,7 +381,9 @@ def visit_field_symbol(self, symbol: ast.FieldSymbol): self.context.field_access_logs.append( HogQLFieldAccess( [symbol.name], - "event" if real_table.table == database.events else "person", + cast(Literal["event"], "event") + if real_table.table == database.events + else cast(Literal["person"], "person"), symbol.name, field_sql, ) @@ -404,9 +420,7 @@ def visit_property_symbol(self, symbol: ast.PropertySymbol): field_name = cast(Union[Literal["properties"], Literal["person_properties"]], field.name) - # TODO: cache this - materialized_columns = get_materialized_columns(table_name) - materialized_column = materialized_columns.get((symbol.name, field_name), None) + materialized_column = self._get_materialized_column(table_name, symbol.name, field_name) if materialized_column: property_sql = print_hogql_identifier(materialized_column) @@ -415,7 +429,7 @@ def visit_property_symbol(self, symbol: ast.PropertySymbol): property_sql = trim_quotes_expr(f"JSONExtractRaw({field_sql}, %({key})s)") if field_name == "properties": - # TODO: refactor this lefacy logging + # TODO: refactor this legacy logging self.context.field_access_logs.append( HogQLFieldAccess( ["properties", symbol.name], @@ -425,7 +439,7 @@ def visit_property_symbol(self, symbol: ast.PropertySymbol): ) ) elif field_name == "person_properties": - # TODO: refactor this lefacy logging + # TODO: refactor this legacy logging self.context.field_access_logs.append( HogQLFieldAccess( ["person", "properties", symbol.name], diff --git a/posthog/hogql/test/test_printer.py b/posthog/hogql/test/test_printer.py index ef6bf47e77a7c..81b9e5a326208 100644 --- a/posthog/hogql/test/test_printer.py +++ b/posthog/hogql/test/test_printer.py @@ -74,7 +74,7 @@ def test_fields_and_properties(self): context = HogQLContext() self.assertEqual( self._expr("person.properties.bla", context), - "replaceRegexpAll(JSONExtractRaw(events.person_properties, %(hogql_val_0)s), '^\"|\"$', '')", + "replaceRegexpAll(JSONExtractRaw(person_properties, %(hogql_val_0)s), '^\"|\"$', '')", ) self.assertEqual( context.field_access_logs, @@ -83,7 +83,7 @@ def test_fields_and_properties(self): ["person", "properties", "bla"], "person.properties", "bla", - "replaceRegexpAll(JSONExtractRaw(events.person_properties, %(hogql_val_0)s), '^\"|\"$', '')", + "replaceRegexpAll(JSONExtractRaw(person_properties, %(hogql_val_0)s), '^\"|\"$', '')", ) ], ) From 30042486eb93580305eea687834cca8c1f165c22 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 15 Feb 2023 08:14:54 +0000 Subject: [PATCH 41/81] Update snapshots --- ee/clickhouse/queries/test/__snapshots__/test_lifecycle.ambr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ee/clickhouse/queries/test/__snapshots__/test_lifecycle.ambr b/ee/clickhouse/queries/test/__snapshots__/test_lifecycle.ambr index 67ba48d323746..4bc6581b95d6e 100644 --- a/ee/clickhouse/queries/test/__snapshots__/test_lifecycle.ambr +++ b/ee/clickhouse/queries/test/__snapshots__/test_lifecycle.ambr @@ -426,7 +426,7 @@ AND event = '$pageview' AND timestamp >= toDateTime(dateTrunc('day', toDateTime('2021-04-28 00:00:00', 'UTC'))) - INTERVAL 1 day AND timestamp < toDateTime(dateTrunc('day', toDateTime('2021-05-05 23:59:59', 'UTC'))) + INTERVAL 1 day - AND (and(like("mat_$current_url", '%example%'), notEquals('bla', 'a%sd'))) + AND (and(like(mat_$current_url, '%example%'), notEquals('bla', 'a%sd'))) GROUP BY pdi.person_id) GROUP BY start_of_period, status) @@ -576,7 +576,7 @@ AND event = '$pageview' AND timestamp >= toDateTime(dateTrunc('day', toDateTime('2021-04-28 00:00:00', 'UTC'))) - INTERVAL 1 day AND timestamp < toDateTime(dateTrunc('day', toDateTime('2021-05-05 23:59:59', 'UTC'))) + INTERVAL 1 day - AND (like("pmat_email", '%test.com')) + AND (like(mat_pp_email, '%test.com')) GROUP BY pdi.person_id) GROUP BY start_of_period, status) From e7efe0a8756899d3c9668e79ce6eada2b1a60b21 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 15 Feb 2023 08:21:33 +0000 Subject: [PATCH 42/81] Update snapshots --- posthog/api/test/__snapshots__/test_insight.ambr | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/posthog/api/test/__snapshots__/test_insight.ambr b/posthog/api/test/__snapshots__/test_insight.ambr index 6479360ca9019..0aa6381ae13bb 100644 --- a/posthog/api/test/__snapshots__/test_insight.ambr +++ b/posthog/api/test/__snapshots__/test_insight.ambr @@ -207,7 +207,7 @@ /* user_id:0 request:_snapshot_ */ SELECT groupArray(value) FROM - (SELECT if(less(toInt64OrNull("mat_int_value"), 10), 'le%ss', 'more') AS value, + (SELECT if(less(toInt64OrNull(mat_int_value), 10), 'le%ss', 'more') AS value, count(*) as count FROM events e WHERE team_id = 2 @@ -249,13 +249,13 @@ day_start UNION ALL SELECT count(*) as total, toStartOfDay(toTimeZone(toDateTime(timestamp, 'UTC'), 'UTC')) as day_start, - if(less(toInt64OrNull("mat_int_value"), 10), 'le%ss', 'more') as breakdown_value + if(less(toInt64OrNull(mat_int_value), 10), 'le%ss', 'more') as breakdown_value FROM events e WHERE e.team_id = 2 AND event = '$pageview' AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2012-01-08 00:00:00', 'UTC')), 'UTC') AND toTimeZone(timestamp, 'UTC') <= toDateTime('2012-01-15 23:59:59', 'UTC') - AND if(less(toInt64OrNull("mat_int_value"), 10), 'le%ss', 'more') in (['more', 'le%ss']) + AND if(less(toInt64OrNull(mat_int_value), 10), 'le%ss', 'more') in (['more', 'le%ss']) GROUP BY day_start, breakdown_value)) GROUP BY day_start, @@ -395,8 +395,8 @@ AND event = '$pageview' AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2012-01-08 00:00:00', 'UTC')), 'UTC') AND toTimeZone(timestamp, 'UTC') <= toDateTime('2012-01-15 23:59:59', 'UTC') - AND ((and(greater(toInt64OrNull("mat_int_value"), 10), notEquals('bla', 'a%sd'))) - AND (like("pmat_fish", '%fish%'))) + AND ((and(greater(toInt64OrNull(mat_int_value), 10), notEquals('bla', 'a%sd'))) + AND (like(mat_pp_fish, '%fish%'))) GROUP BY date) GROUP BY day_start ORDER BY day_start) @@ -479,8 +479,8 @@ AND event = '$pageview' AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2012-01-08 00:00:00', 'UTC')), 'UTC') AND toTimeZone(timestamp, 'UTC') <= toDateTime('2012-01-15 23:59:59', 'UTC') - AND (and(less(toInt64OrNull("mat_int_value"), 10), notEquals('bla', 'a%sd')) - AND like("pmat_fish", '%fish%')) + AND (and(less(toInt64OrNull(mat_int_value), 10), notEquals('bla', 'a%sd')) + AND like(mat_pp_fish, '%fish%')) GROUP BY date) GROUP BY day_start ORDER BY day_start) From b075bc27df75fed4d1d3647778d11cccaec2446b Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 15 Feb 2023 09:52:03 +0100 Subject: [PATCH 43/81] simple splash and table property printing --- posthog/hogql/ast.py | 4 ++++ posthog/hogql/database.py | 20 ++++++++++++++--- posthog/hogql/printer.py | 36 +++++++++++++++++++++--------- posthog/hogql/resolver.py | 18 ++++++++++----- posthog/hogql/test/test_printer.py | 3 +-- 5 files changed, 61 insertions(+), 20 deletions(-) diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index eb18afdda34a5..2d55753b9d851 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -118,6 +118,10 @@ class ConstantSymbol(Symbol): value: Any +class SplashSymbol(Symbol): + table: Union[TableSymbol, TableAliasSymbol, SelectQuerySymbol, SelectQueryAliasSymbol] + + class FieldSymbol(Symbol): name: str table: Union[TableSymbol, TableAliasSymbol, SelectQuerySymbol, SelectQueryAliasSymbol] diff --git a/posthog/hogql/database.py b/posthog/hogql/database.py index dc9c8bf87d3ed..7341258bb58bc 100644 --- a/posthog/hogql/database.py +++ b/posthog/hogql/database.py @@ -1,3 +1,5 @@ +from typing import List + from pydantic import BaseModel, Extra @@ -46,6 +48,18 @@ def get_field(self, name: str) -> DatabaseField: def clickhouse_table(self): raise NotImplementedError("Table.clickhouse_table not overridden") + def get_splash(self) -> List[str]: + list: List[str] = [] + for field in self.__fields__.values(): + database_field = field.default + if isinstance(database_field, DatabaseField): + list.append(database_field.name) + elif isinstance(database_field, Table): + list.extend(database_field.get_splash()) + else: + raise ValueError(f"Unknown field type {type(database_field).__name__} for splash") + return list + class PersonsTable(Table): id: StringDatabaseField = StringDatabaseField(name="id") @@ -84,12 +98,12 @@ def clickhouse_table(self): class EventsTable(Table): uuid: StringDatabaseField = StringDatabaseField(name="uuid") event: StringDatabaseField = StringDatabaseField(name="event") - timestamp: DateTimeDatabaseField = DateTimeDatabaseField(name="timestamp") properties: StringJSONDatabaseField = StringJSONDatabaseField(name="properties") + timestamp: DateTimeDatabaseField = DateTimeDatabaseField(name="timestamp") + team_id: IntegerDatabaseField = IntegerDatabaseField(name="team_id") + distinct_id: StringDatabaseField = StringDatabaseField(name="distinct_id") elements_chain: StringDatabaseField = StringDatabaseField(name="elements_chain") created_at: DateTimeDatabaseField = DateTimeDatabaseField(name="created_at") - distinct_id: StringDatabaseField = StringDatabaseField(name="distinct_id") - team_id: IntegerDatabaseField = IntegerDatabaseField(name="team_id") person: EventsPersonSubTable = EventsPersonSubTable() def clickhouse_table(self): diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index 4511cd4b9b601..6f836f4ee5672 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -5,7 +5,7 @@ from posthog.hogql import ast from posthog.hogql.constants import CLICKHOUSE_FUNCTIONS, HOGQL_AGGREGATIONS, MAX_SELECT_RETURNED_ROWS from posthog.hogql.context import HogQLContext, HogQLFieldAccess -from posthog.hogql.database import database +from posthog.hogql.database import Table, database from posthog.hogql.print_string import print_hogql_identifier from posthog.hogql.resolver import ResolverException, lookup_field_by_name from posthog.hogql.visitor import Visitor @@ -267,15 +267,7 @@ def visit_field(self, node: ast.Field): # When printing HogQL, we print the properties out as a chain as they are. return ".".join([print_hogql_identifier(identifier) for identifier in node.chain]) - if node.chain == ["*"]: - # query = f"tuple({','.join(SELECT_STAR_FROM_EVENTS_FIELDS)})" - # return self.visit(parse_expr(query)) - raise ValueError("Selecting * not yet implemented") - elif node.chain == ["person"]: - # query = "tuple(distinct_id, person.id, person.created_at, person.properties.name, person.properties.email)" - # return self.visit(parse_expr(query)) - raise ValueError("Selecting person not yet implemented") - elif node.symbol is not None: + if node.symbol is not None: select_query = self._last_select() select: Optional[ast.SelectQuerySymbol] = select_query.symbol if select_query else None if select is None: @@ -357,6 +349,18 @@ def visit_field_symbol(self, symbol: ast.FieldSymbol): resolved_field = symbol.resolve_database_field() if resolved_field is None: raise ValueError(f'Can\'t resolve field "{symbol.name}" on table.') + if isinstance(resolved_field, Table): + # :KLUDGE: only works for events.person.* printing now + if isinstance(symbol.table, ast.TableSymbol): + return self.visit(ast.SplashSymbol(table=ast.TableSymbol(table=resolved_field))) + else: + return self.visit( + ast.SplashSymbol( + table=ast.TableAliasSymbol( + table=ast.TableSymbol(table=resolved_field), name=symbol.table.name + ) + ) + ) field_sql = print_hogql_identifier(resolved_field.name) @@ -457,6 +461,18 @@ def visit_select_query_alias_symbol(self, symbol: ast.SelectQueryAliasSymbol): def visit_field_alias_symbol(self, symbol: ast.SelectQueryAliasSymbol): return print_hogql_identifier(symbol.name) + def visit_splash_symbol(self, symbol: ast.SplashSymbol): + table = symbol.table + while isinstance(table, ast.TableAliasSymbol): + table = table.table + if not isinstance(table, ast.TableSymbol): + raise ValueError(f"Unknown SplashSymbol table type: {type(table).__name__}") + splash_fields = table.table.get_splash() + prefix = ( + f"{print_hogql_identifier(symbol.table.name)}." if isinstance(symbol.table, ast.TableAliasSymbol) else "" + ) + return f"tuple({', '.join(f'{prefix}{print_hogql_identifier(field)}' for field in splash_fields)})" + def visit_unknown(self, symbol: ast.AST): raise ValueError(f"Unknown Symbol {type(symbol).__name__}") diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py index b9ecb1c00bcdb..e08e6215a64ed 100644 --- a/posthog/hogql/resolver.py +++ b/posthog/hogql/resolver.py @@ -146,14 +146,12 @@ def visit_field(self, node): if len(node.chain) == 0: raise Exception("Invalid field access with empty chain") - # Only look for fields in the last SELECT scope. - scope = self.scopes[-1] - - # ClickHouse does not support subqueries accessing "x.event". This is forbidden: + # Only look for fields in the last SELECT scope, instead of all previous scopes. + # That's because ClickHouse does not support subqueries accessing "x.event". This is forbidden: # - "SELECT event, (select count() from events where event = x.event) as c FROM events x where event = '$pageview'", # But this is supported: # - "SELECT t.big_count FROM (select count() + 100 as big_count from events) as t JOIN events e ON (e.event = t.event)", - # Thus we don't have to recursively look into all the past scopes to find a match. + scope = self.scopes[-1] symbol: Optional[ast.Symbol] = None name = node.chain[0] @@ -161,6 +159,16 @@ def visit_field(self, node): # If the field contains at least two parts, the first might be a table. if len(node.chain) > 1 and name in scope.tables: symbol = scope.tables[name] + + if name == "*" and len(node.chain) == 1: + table_count = len(scope.anonymous_tables) + len(scope.tables) + if table_count == 0: + raise ResolverException("Cannot use '*' when there are no tables in the query") + if table_count > 1: + raise ResolverException("Cannot use '*' when there are multiple tables in the query") + table = scope.anonymous_tables[0] if len(scope.anonymous_tables) > 0 else list(scope.tables.values())[0] + symbol = ast.SplashSymbol(table=table) + if not symbol: symbol = lookup_field_by_name(scope, name) if not symbol: diff --git a/posthog/hogql/test/test_printer.py b/posthog/hogql/test/test_printer.py index 81b9e5a326208..52a4f58daca3b 100644 --- a/posthog/hogql/test/test_printer.py +++ b/posthog/hogql/test/test_printer.py @@ -336,9 +336,8 @@ def test_special_root_properties(self): context = HogQLContext() self.assertEqual( self._expr("person", context), - "tuple(distinct_id, person_id, person_created_at, replaceRegexpAll(JSONExtractRaw(person_properties, %(hogql_val_0)s), '^\"|\"$', ''), replaceRegexpAll(JSONExtractRaw(person_properties, %(hogql_val_1)s), '^\"|\"$', ''))", + "tuple(person_id, person_created_at, person_properties)", ) - self.assertEqual(context.values, {"hogql_val_0": "name", "hogql_val_1": "email"}) def test_values(self): context = HogQLContext() From e59ef94f7362b4c87cb618ab324666519ad4e25e Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 15 Feb 2023 09:11:56 +0000 Subject: [PATCH 44/81] Update snapshots --- .../api/test/__snapshots__/test_query.ambr | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/posthog/api/test/__snapshots__/test_query.ambr b/posthog/api/test/__snapshots__/test_query.ambr index de06707004e26..9b049fa990e6c 100644 --- a/posthog/api/test/__snapshots__/test_query.ambr +++ b/posthog/api/test/__snapshots__/test_query.ambr @@ -50,9 +50,9 @@ /* user_id:0 request:_snapshot_ */ SELECT event, distinct_id, - "mat_key", + mat_key, 'a%sd', - concat(event, ' ', "mat_key") + concat(event, ' ', mat_key) FROM events WHERE team_id = 2 AND timestamp < '2020-01-10 12:14:05.000000' @@ -65,9 +65,9 @@ /* user_id:0 request:_snapshot_ */ SELECT event, distinct_id, - "mat_key", + mat_key, 'a%sd', - concat(event, ' ', "mat_key") + concat(event, ' ', mat_key) FROM events WHERE team_id = 2 AND timestamp < '2020-01-10 12:14:05.000000' @@ -81,9 +81,9 @@ /* user_id:0 request:_snapshot_ */ SELECT event, distinct_id, - "mat_key", + mat_key, 'a%sd', - concat(event, ' ', "mat_key") + concat(event, ' ', mat_key) FROM events WHERE team_id = 2 AND timestamp < '2020-01-10 12:14:05.000000' @@ -160,9 +160,9 @@ /* user_id:0 request:_snapshot_ */ SELECT event, distinct_id, - "mat_key", + mat_key, 'a%sd', - concat(event, ' ', "mat_key") + concat(event, ' ', mat_key) FROM events WHERE team_id = 2 AND timestamp < '2020-01-10 12:14:05.000000' @@ -175,9 +175,9 @@ /* user_id:0 request:_snapshot_ */ SELECT event, distinct_id, - "mat_key", + mat_key, 'a%sd', - concat(event, ' ', "mat_key") + concat(event, ' ', mat_key) FROM events WHERE team_id = 2 AND timestamp < '2020-01-10 12:14:05.000000' @@ -191,9 +191,9 @@ /* user_id:0 request:_snapshot_ */ SELECT event, distinct_id, - "mat_key", + mat_key, 'a%sd', - concat(event, ' ', "mat_key") + concat(event, ' ', mat_key) FROM events WHERE team_id = 2 AND timestamp < '2020-01-10 12:14:05.000000' @@ -207,13 +207,13 @@ /* user_id:0 request:_snapshot_ */ SELECT event, distinct_id, - "mat_key", + mat_key, 'a%sd', - concat(event, ' ', "mat_key") + concat(event, ' ', mat_key) FROM events WHERE team_id = 2 AND timestamp < '2020-01-10 12:14:05.000000' - AND equals("mat_key", 'test_val2') + AND equals(mat_key, 'test_val2') ORDER BY event ASC LIMIT 101 ' @@ -283,12 +283,12 @@ # name: TestQuery.test_property_filter_aggregations_materialized ' /* user_id:0 request:_snapshot_ */ - SELECT "mat_key", + SELECT mat_key, count(*) FROM events WHERE team_id = 2 AND timestamp < '2020-01-10 12:14:05.000000' - GROUP BY "mat_key" + GROUP BY mat_key ORDER BY count() DESC LIMIT 101 ' @@ -296,12 +296,12 @@ # name: TestQuery.test_property_filter_aggregations_materialized.1 ' /* user_id:0 request:_snapshot_ */ - SELECT "mat_key", + SELECT mat_key, count(*) FROM events WHERE team_id = 2 AND timestamp < '2020-01-10 12:14:05.000000' - GROUP BY "mat_key" + GROUP BY mat_key HAVING greater(count(*), 1) ORDER BY count() DESC LIMIT 101 From f095fec49a64178e45e12521e998768d23bdf053 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 15 Feb 2023 10:32:42 +0100 Subject: [PATCH 45/81] fix pp --- posthog/hogql/printer.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index 6f836f4ee5672..fd60b538b7af2 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -329,9 +329,13 @@ def __init__(self, select: ast.SelectQuerySymbol, context: HogQLContext): def _get_materialized_column( self, table_name: TablesWithMaterializedColumns, property_name: PropertyName, field_name: TableColumn ) -> Optional[str]: + # :KLUDGE: person property materialised columns support when person on events is off + if not self.context.using_person_on_events and table_name == "events" and field_name == "person_properties": + materialized_columns = get_materialized_columns("person") + return materialized_columns.get(("properties", field_name), None) + materialized_columns = get_materialized_columns(table_name) - materialized_column = materialized_columns.get((property_name, field_name), None) - return materialized_column + return materialized_columns.get((property_name, field_name), None) def visit_table_symbol(self, symbol: ast.TableSymbol): return print_hogql_identifier(symbol.table.clickhouse_table()) From 4715e8a31154e32922e10bb0644315c1de2cc31a Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 15 Feb 2023 11:17:09 +0100 Subject: [PATCH 46/81] revert what was different about hogql access logs --- posthog/hogql/printer.py | 17 ++++++++++++----- posthog/hogql/test/test_printer.py | 8 ++++---- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index fd60b538b7af2..187c66f7a9e74 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -373,10 +373,14 @@ def visit_field_symbol(self, symbol: ast.FieldSymbol): if resolved_field == database.events.person.properties: if not self.context.using_person_on_events: field_sql = "person_props" + elif resolved_field == database.events.person.id: + pass + elif resolved_field == database.events.person.created_at: + pass # If the field is called on a table that has an alias, prepend the table alias. # If there's another field with the same name in the scope that's not this, prepend the full table name. - # Note: we don't prepend a table name for the special "person_properties" field. + # Note: we don't prepend a table name for the special "person" fields. elif isinstance(symbol.table, ast.TableAliasSymbol) or symbol_with_name_in_scope != symbol: field_sql = f"{self.visit(symbol.table)}.{field_sql}" @@ -386,12 +390,15 @@ def visit_field_symbol(self, symbol: ast.FieldSymbol): while isinstance(real_table, ast.TableAliasSymbol): real_table = real_table.table + access_table = ( + cast(Literal["event"], "event") + if real_table.table == database.events + else cast(Literal["person"], "person") + ) self.context.field_access_logs.append( HogQLFieldAccess( - [symbol.name], - cast(Literal["event"], "event") - if real_table.table == database.events - else cast(Literal["person"], "person"), + ["person", symbol.name] if access_table == "person" else [symbol.name], + access_table, symbol.name, field_sql, ) diff --git a/posthog/hogql/test/test_printer.py b/posthog/hogql/test/test_printer.py index 52a4f58daca3b..722f2d62512f7 100644 --- a/posthog/hogql/test/test_printer.py +++ b/posthog/hogql/test/test_printer.py @@ -109,17 +109,17 @@ def test_fields_and_properties(self): ) context = HogQLContext() - self.assertEqual(self._expr("person.id", context), "events.person_id") + self.assertEqual(self._expr("person.id", context), "person_id") self.assertEqual( context.field_access_logs, - [HogQLFieldAccess(["id"], "person", "id", "events.person_id")], + [HogQLFieldAccess(["person", "id"], "person", "id", "person_id")], ) context = HogQLContext() - self.assertEqual(self._expr("person.created_at", context), "events.person_created_at") + self.assertEqual(self._expr("person.created_at", context), "person_created_at") self.assertEqual( context.field_access_logs, - [HogQLFieldAccess(["created_at"], "person", "created_at", "events.person_created_at")], + [HogQLFieldAccess(["person", "created_at"], "person", "created_at", "person_created_at")], ) def test_hogql_properties(self): From 07204aa18fe24d7152f4d0abdf73d7d8c81eec0d Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 15 Feb 2023 13:41:13 +0100 Subject: [PATCH 47/81] explicit names for person non properties --- posthog/hogql/printer.py | 4 ---- posthog/hogql/test/test_printer.py | 8 ++++---- posthog/models/property/util.py | 1 + 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index 187c66f7a9e74..4ba125a20da38 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -373,10 +373,6 @@ def visit_field_symbol(self, symbol: ast.FieldSymbol): if resolved_field == database.events.person.properties: if not self.context.using_person_on_events: field_sql = "person_props" - elif resolved_field == database.events.person.id: - pass - elif resolved_field == database.events.person.created_at: - pass # If the field is called on a table that has an alias, prepend the table alias. # If there's another field with the same name in the scope that's not this, prepend the full table name. diff --git a/posthog/hogql/test/test_printer.py b/posthog/hogql/test/test_printer.py index 722f2d62512f7..3fe1d322c8bc5 100644 --- a/posthog/hogql/test/test_printer.py +++ b/posthog/hogql/test/test_printer.py @@ -109,17 +109,17 @@ def test_fields_and_properties(self): ) context = HogQLContext() - self.assertEqual(self._expr("person.id", context), "person_id") + self.assertEqual(self._expr("person.id", context), "events.person_id") self.assertEqual( context.field_access_logs, - [HogQLFieldAccess(["person", "id"], "person", "id", "person_id")], + [HogQLFieldAccess(["person", "id"], "person", "id", "events.person_id")], ) context = HogQLContext() - self.assertEqual(self._expr("person.created_at", context), "person_created_at") + self.assertEqual(self._expr("person.created_at", context), "events.person_created_at") self.assertEqual( context.field_access_logs, - [HogQLFieldAccess(["person", "created_at"], "person", "created_at", "person_created_at")], + [HogQLFieldAccess(["person", "created_at"], "person", "created_at", "events.person_created_at")], ) def test_hogql_properties(self): diff --git a/posthog/models/property/util.py b/posthog/models/property/util.py index 4dc3952c1becc..1a0b34f5e989a 100644 --- a/posthog/models/property/util.py +++ b/posthog/models/property/util.py @@ -772,6 +772,7 @@ def extract_tables_and_properties(props: List[Property]) -> TCounter[PropertyIde for prop in props: if prop.type == "hogql": context = HogQLContext() + # TODO: Refactor this. Currently it prints and discards a query, just to check the properties. translate_hogql(prop.key, context) for field_access in context.field_access_logs: if field_access.type == "event.properties": From 10023828f4d2c2a1f0a9310dbe97bca1001e0b35 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 15 Feb 2023 13:50:14 +0100 Subject: [PATCH 48/81] consolidate into print_ast --- posthog/hogql/hogql.py | 3 --- posthog/hogql/printer.py | 27 ++++++++++++++++++++++++--- posthog/hogql/query.py | 4 +--- posthog/hogql/visitor.py | 5 +++++ 4 files changed, 30 insertions(+), 9 deletions(-) diff --git a/posthog/hogql/hogql.py b/posthog/hogql/hogql.py index 541d0199b4c1f..e96134efda96f 100644 --- a/posthog/hogql/hogql.py +++ b/posthog/hogql/hogql.py @@ -5,7 +5,6 @@ from posthog.hogql.database import database from posthog.hogql.parser import parse_expr, parse_select from posthog.hogql.printer import print_ast -from posthog.hogql.resolver import resolve_symbols def translate_hogql(query: str, context: HogQLContext, dialect: Literal["hogql", "clickhouse"] = "clickhouse") -> str: @@ -17,7 +16,6 @@ def translate_hogql(query: str, context: HogQLContext, dialect: Literal["hogql", if context.select_team_id: # Only parse full SELECT statements if we have a team_id in the context. node = parse_select(query, no_placeholders=True) - resolve_symbols(node) return print_ast(node, context, dialect, stack=[]) else: # Create a fake query that selects from "events". Assume were in its scope when evaluating expressions. @@ -25,7 +23,6 @@ def translate_hogql(query: str, context: HogQLContext, dialect: Literal["hogql", select_query = ast.SelectQuery(select=[], symbol=symbol) node = parse_expr(query, no_placeholders=True) - resolve_symbols(node, symbol) return print_ast(node, context, dialect, stack=[select_query]) except SyntaxError as err: diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index 4ba125a20da38..fcdf938b4c746 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -7,8 +7,8 @@ from posthog.hogql.context import HogQLContext, HogQLFieldAccess from posthog.hogql.database import Table, database from posthog.hogql.print_string import print_hogql_identifier -from posthog.hogql.resolver import ResolverException, lookup_field_by_name -from posthog.hogql.visitor import Visitor +from posthog.hogql.resolver import ResolverException, lookup_field_by_name, resolve_symbols +from posthog.hogql.visitor import Visitor, clone_expr from posthog.models.property import PropertyName, TableColumn @@ -27,8 +27,25 @@ def team_id_guard_for_table( def print_ast( - node: ast.AST, context: HogQLContext, dialect: Literal["hogql", "clickhouse"], stack: Optional[List[ast.AST]] = None + node: ast.Expr, + context: HogQLContext, + dialect: Literal["hogql", "clickhouse"], + stack: Optional[List[ast.Expr]] = None, ) -> str: + """Print an AST into a string. Does not modify the node.""" + symbol = stack[-1].symbol if stack else None + + # make a clean copy of the object + node = clone_expr(node) + # resolve symbols + resolve_symbols(node, symbol) + + # modify the cloned tree as needed + if dialect == "clickhouse": + # TODO: add team_id checks (currently done in the printer) + # TODO: add joins to person and group tables + pass + return Printer(context=context, dialect=dialect, stack=stack or []).visit(node) @@ -39,6 +56,8 @@ class JoinExprResponse: class Printer(Visitor): + # NOTE: Call "print_ast()", not this class directly. + def __init__( self, context: HogQLContext, dialect: Literal["hogql", "clickhouse"], stack: Optional[List[ast.AST]] = None ): @@ -152,12 +171,14 @@ def visit_join_expr(self, node: ast.JoinExpr) -> JoinExprResponse: select_from.append(f"AS {print_hogql_identifier(node.alias)}") if self.dialect == "clickhouse": + # TODO: do this in a separate pass before printing, along with person joins and other transforms extra_where = team_id_guard_for_table(node.symbol, self.context) elif isinstance(node.symbol, ast.TableSymbol): select_from.append(print_hogql_identifier(node.symbol.table.clickhouse_table())) if self.dialect == "clickhouse": + # TODO: do this in a separate pass before printing, along with person joins and other transforms extra_where = team_id_guard_for_table(node.symbol, self.context) elif isinstance(node.symbol, ast.SelectQuerySymbol): diff --git a/posthog/hogql/query.py b/posthog/hogql/query.py index 7938da035b6fd..e1979d05706d3 100644 --- a/posthog/hogql/query.py +++ b/posthog/hogql/query.py @@ -8,7 +8,6 @@ from posthog.hogql.parser import parse_select from posthog.hogql.placeholders import assert_no_placeholders, replace_placeholders from posthog.hogql.printer import print_ast -from posthog.hogql.resolver import resolve_symbols from posthog.models import Team from posthog.queries.insight import insight_sync_execute @@ -46,8 +45,7 @@ def execute_hogql_query( if select_query.limit is None: select_query.limit = ast.Constant(value=1000) - hogql_context = HogQLContext(select_team_id=team.pk) - resolve_symbols(select_query) + hogql_context = HogQLContext(select_team_id=team.pk, using_person_on_events=team.person_on_events_querying_enabled) clickhouse = print_ast(select_query, hogql_context, "clickhouse") hogql = print_ast(select_query, hogql_context, "hogql") diff --git a/posthog/hogql/visitor.py b/posthog/hogql/visitor.py index b5628f5d7287c..e6d94d2ed9ad4 100644 --- a/posthog/hogql/visitor.py +++ b/posthog/hogql/visitor.py @@ -1,6 +1,11 @@ from posthog.hogql import ast +def clone_expr(self: ast.Expr) -> ast.Expr: + """Clone an expression node. Removes all symbols.""" + return CloningVisitor().visit(self) + + class Visitor(object): def visit(self, node: ast.AST): if node is None: From 1d681f834b078dc01bd7e2bced450c541cceb410 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 15 Feb 2023 14:09:48 +0100 Subject: [PATCH 49/81] merge symbol printer into printer --- posthog/hogql/hogql.py | 6 +- posthog/hogql/print_string.py | 11 +++- posthog/hogql/printer.py | 93 +++++++++++++++--------------- posthog/hogql/test/test_printer.py | 11 ++-- 4 files changed, 65 insertions(+), 56 deletions(-) diff --git a/posthog/hogql/hogql.py b/posthog/hogql/hogql.py index e96134efda96f..f39798ae9347c 100644 --- a/posthog/hogql/hogql.py +++ b/posthog/hogql/hogql.py @@ -19,9 +19,9 @@ def translate_hogql(query: str, context: HogQLContext, dialect: Literal["hogql", return print_ast(node, context, dialect, stack=[]) else: # Create a fake query that selects from "events". Assume were in its scope when evaluating expressions. - symbol = ast.SelectQuerySymbol(tables={"events": ast.TableSymbol(table=database.events)}) - select_query = ast.SelectQuery(select=[], symbol=symbol) - + select_query = ast.SelectQuery( + select=[], symbol=ast.SelectQuerySymbol(tables={"events": ast.TableSymbol(table=database.events)}) + ) node = parse_expr(query, no_placeholders=True) return print_ast(node, context, dialect, stack=[select_query]) diff --git a/posthog/hogql/print_string.py b/posthog/hogql/print_string.py index 7b227acf97d82..351bad47457b4 100644 --- a/posthog/hogql/print_string.py +++ b/posthog/hogql/print_string.py @@ -15,9 +15,18 @@ } -# Copied from clickhouse_driver.util.escape, adapted from single quotes to backquotes. +# Copied from clickhouse_driver.util.escape, adapted from single quotes to backquotes. Added a $. def print_hogql_identifier(identifier: str) -> str: + # HogQL allows dollars in the identifier. if re.match(r"^[A-Za-z_$][A-Za-z0-9_$]*$", identifier): return identifier return "`%s`" % "".join(backquote_escape_chars_map.get(c, c) for c in identifier) + + +# Copied from clickhouse_driver.util.escape, adapted from single quotes to backquotes. +def print_clickhouse_identifier(identifier: str) -> str: + if re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", identifier): + return identifier + + return "`%s`" % "".join(backquote_escape_chars_map.get(c, c) for c in identifier) diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index fcdf938b4c746..9298313474ae1 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -6,7 +6,7 @@ from posthog.hogql.constants import CLICKHOUSE_FUNCTIONS, HOGQL_AGGREGATIONS, MAX_SELECT_RETURNED_ROWS from posthog.hogql.context import HogQLContext, HogQLFieldAccess from posthog.hogql.database import Table, database -from posthog.hogql.print_string import print_hogql_identifier +from posthog.hogql.print_string import print_clickhouse_identifier, print_hogql_identifier from posthog.hogql.resolver import ResolverException, lookup_field_by_name, resolve_symbols from posthog.hogql.visitor import Visitor, clone_expr from posthog.models.property import PropertyName, TableColumn @@ -66,13 +66,6 @@ def __init__( # Keep track of all traversed nodes. self.stack: List[ast.AST] = stack or [] - def _last_select(self) -> Optional[ast.SelectQuery]: - """Find the last SELECT query in the stack.""" - for node in reversed(self.stack): - if isinstance(node, ast.SelectQuery): - return node - return None - def visit(self, node: ast.AST): self.stack.append(node) response = super().visit(node) @@ -166,16 +159,16 @@ def visit_join_expr(self, node: ast.JoinExpr) -> JoinExprResponse: raise ValueError(f"Table alias {node.symbol.name} does not resolve!") if not isinstance(table_symbol, ast.TableSymbol): raise ValueError(f"Table alias {node.symbol.name} does not resolve to a table!") - select_from.append(print_hogql_identifier(table_symbol.table.clickhouse_table())) + select_from.append(self._print_identifier(table_symbol.table.clickhouse_table())) if node.alias is not None: - select_from.append(f"AS {print_hogql_identifier(node.alias)}") + select_from.append(f"AS {self._print_identifier(node.alias)}") if self.dialect == "clickhouse": # TODO: do this in a separate pass before printing, along with person joins and other transforms extra_where = team_id_guard_for_table(node.symbol, self.context) elif isinstance(node.symbol, ast.TableSymbol): - select_from.append(print_hogql_identifier(node.symbol.table.clickhouse_table())) + select_from.append(self._print_identifier(node.symbol.table.clickhouse_table())) if self.dialect == "clickhouse": # TODO: do this in a separate pass before printing, along with person joins and other transforms @@ -186,7 +179,7 @@ def visit_join_expr(self, node: ast.JoinExpr) -> JoinExprResponse: elif isinstance(node.symbol, ast.SelectQueryAliasSymbol) and node.alias is not None: select_from.append(self.visit(node.table)) - select_from.append(f"AS {print_hogql_identifier(node.alias)}") + select_from.append(f"AS {self._print_identifier(node.alias)}") else: raise ValueError("Only selecting from a table or a subquery is supported") @@ -280,20 +273,20 @@ def visit_constant(self, node: ast.Constant): ) def visit_field(self, node: ast.Field): - original_field = ".".join([print_hogql_identifier(identifier) for identifier in node.chain]) + original_field = ".".join([self._print_identifier(identifier) for identifier in node.chain]) if node.symbol is None: raise ValueError(f"Field {original_field} has no symbol") if self.dialect == "hogql": # When printing HogQL, we print the properties out as a chain as they are. - return ".".join([print_hogql_identifier(identifier) for identifier in node.chain]) + return ".".join([self._print_identifier(identifier) for identifier in node.chain]) if node.symbol is not None: select_query = self._last_select() select: Optional[ast.SelectQuerySymbol] = select_query.symbol if select_query else None if select is None: raise ValueError(f"Can't find SelectQuerySymbol for field: {original_field}") - return SymbolPrinter(select=select, context=self.context).visit(node.symbol) + return self.visit(node.symbol) else: raise ValueError(f"Unknown Symbol, can not print {type(node.symbol).__name__}") @@ -336,37 +329,18 @@ def visit_placeholder(self, node: ast.Placeholder): raise ValueError(f"Found a Placeholder {{{node.field}}} in the tree. Can't generate query!") def visit_alias(self, node: ast.Alias): - return f"{self.visit(node.expr)} AS {print_hogql_identifier(node.alias)}" - - def visit_unknown(self, node: ast.AST): - raise ValueError(f"Unknown AST node {type(node).__name__}") - - -class SymbolPrinter(Visitor): - def __init__(self, select: ast.SelectQuerySymbol, context: HogQLContext): - self.select = select - self.context = context - - def _get_materialized_column( - self, table_name: TablesWithMaterializedColumns, property_name: PropertyName, field_name: TableColumn - ) -> Optional[str]: - # :KLUDGE: person property materialised columns support when person on events is off - if not self.context.using_person_on_events and table_name == "events" and field_name == "person_properties": - materialized_columns = get_materialized_columns("person") - return materialized_columns.get(("properties", field_name), None) - - materialized_columns = get_materialized_columns(table_name) - return materialized_columns.get((property_name, field_name), None) + return f"{self.visit(node.expr)} AS {self._print_identifier(node.alias)}" def visit_table_symbol(self, symbol: ast.TableSymbol): - return print_hogql_identifier(symbol.table.clickhouse_table()) + return self._print_identifier(symbol.table.clickhouse_table()) def visit_table_alias_symbol(self, symbol: ast.TableAliasSymbol): - return print_hogql_identifier(symbol.name) + return self._print_identifier(symbol.name) def visit_field_symbol(self, symbol: ast.FieldSymbol): try: - symbol_with_name_in_scope = lookup_field_by_name(self.select, symbol.name) + last_select = self._last_select() + symbol_with_name_in_scope = lookup_field_by_name(last_select.symbol, symbol.name) if last_select else None except ResolverException: symbol_with_name_in_scope = None @@ -387,7 +361,7 @@ def visit_field_symbol(self, symbol: ast.FieldSymbol): ) ) - field_sql = print_hogql_identifier(resolved_field.name) + field_sql = self._print_identifier(resolved_field.name) # :KLUDGE: Legacy person properties handling. Assume we're in a context where the tables have been joined, # and this "person_props" alias is accessible to us. @@ -422,7 +396,7 @@ def visit_field_symbol(self, symbol: ast.FieldSymbol): ) elif isinstance(symbol.table, ast.SelectQuerySymbol) or isinstance(symbol.table, ast.SelectQueryAliasSymbol): - field_sql = print_hogql_identifier(symbol.name) + field_sql = self._print_identifier(symbol.name) if isinstance(symbol.table, ast.SelectQueryAliasSymbol) or symbol_with_name_in_scope != symbol: field_sql = f"{self.visit(symbol.table)}.{field_sql}" @@ -455,7 +429,7 @@ def visit_property_symbol(self, symbol: ast.PropertySymbol): materialized_column = self._get_materialized_column(table_name, symbol.name, field_name) if materialized_column: - property_sql = print_hogql_identifier(materialized_column) + property_sql = self._print_identifier(materialized_column) else: field_sql = self.visit(field_symbol) property_sql = trim_quotes_expr(f"JSONExtractRaw({field_sql}, %({key})s)") @@ -484,10 +458,10 @@ def visit_property_symbol(self, symbol: ast.PropertySymbol): return property_sql def visit_select_query_alias_symbol(self, symbol: ast.SelectQueryAliasSymbol): - return print_hogql_identifier(symbol.name) + return self._print_identifier(symbol.name) def visit_field_alias_symbol(self, symbol: ast.SelectQueryAliasSymbol): - return print_hogql_identifier(symbol.name) + return self._print_identifier(symbol.name) def visit_splash_symbol(self, symbol: ast.SplashSymbol): table = symbol.table @@ -497,12 +471,35 @@ def visit_splash_symbol(self, symbol: ast.SplashSymbol): raise ValueError(f"Unknown SplashSymbol table type: {type(table).__name__}") splash_fields = table.table.get_splash() prefix = ( - f"{print_hogql_identifier(symbol.table.name)}." if isinstance(symbol.table, ast.TableAliasSymbol) else "" + f"{self._print_identifier(symbol.table.name)}." if isinstance(symbol.table, ast.TableAliasSymbol) else "" ) - return f"tuple({', '.join(f'{prefix}{print_hogql_identifier(field)}' for field in splash_fields)})" + return f"tuple({', '.join(f'{prefix}{self._print_identifier(field)}' for field in splash_fields)})" + + def visit_unknown(self, node: ast.AST): + raise ValueError(f"Unknown AST node {type(node).__name__}") + + def _last_select(self) -> Optional[ast.SelectQuery]: + """Find the last SELECT query in the stack.""" + for node in reversed(self.stack): + if isinstance(node, ast.SelectQuery): + return node + return None - def visit_unknown(self, symbol: ast.AST): - raise ValueError(f"Unknown Symbol {type(symbol).__name__}") + def _print_identifier(self, name: str) -> str: + if self.dialect == "clickhouse": + return print_clickhouse_identifier(name) + return print_hogql_identifier(name) + + def _get_materialized_column( + self, table_name: TablesWithMaterializedColumns, property_name: PropertyName, field_name: TableColumn + ) -> Optional[str]: + # :KLUDGE: person property materialised columns support when person on events is off + if not self.context.using_person_on_events and table_name == "events" and field_name == "person_properties": + materialized_columns = get_materialized_columns("person") + return materialized_columns.get(("properties", field_name), None) + + materialized_columns = get_materialized_columns(table_name) + return materialized_columns.get((property_name, field_name), None) def trim_quotes_expr(expr: str) -> str: diff --git a/posthog/hogql/test/test_printer.py b/posthog/hogql/test/test_printer.py index 3fe1d322c8bc5..6a8ec04c86290 100644 --- a/posthog/hogql/test/test_printer.py +++ b/posthog/hogql/test/test_printer.py @@ -172,16 +172,19 @@ def test_materialized_fields_and_properties(self): self.assertEqual(1 + 2, 3) return materialize("events", "$browser") - self.assertEqual(self._expr("properties['$browser']"), "mat_$browser") + self.assertEqual(self._expr("properties['$browser']"), "`mat_$browser`") + + materialize("events", "withoutdollar") + self.assertEqual(self._expr("properties['withoutdollar']"), "mat_withoutdollar") materialize("events", "$browser and string") - self.assertEqual(self._expr("properties['$browser and string']"), "mat_$browser_and_string") + self.assertEqual(self._expr("properties['$browser and string']"), "`mat_$browser_and_string`") materialize("events", "$browser%%%#@!@") - self.assertEqual(self._expr("properties['$browser%%%#@!@']"), "mat_$browser_______") + self.assertEqual(self._expr("properties['$browser%%%#@!@']"), "`mat_$browser_______`") materialize("events", "$initial_waffle", table_column="person_properties") - self.assertEqual(self._expr("person.properties['$initial_waffle']"), "mat_pp_$initial_waffle") + self.assertEqual(self._expr("person.properties['$initial_waffle']"), "`mat_pp_$initial_waffle`") def test_methods(self): self.assertEqual(self._expr("count()"), "count(*)") From b32cb4b682dfcaace53b94860f8a10ee4cdc7c8b Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 15 Feb 2023 14:29:07 +0100 Subject: [PATCH 50/81] move it up --- posthog/hogql/printer.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index 9298313474ae1..bce2d1a9b25b3 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -426,7 +426,11 @@ def visit_property_symbol(self, symbol: ast.PropertySymbol): field_name = cast(Union[Literal["properties"], Literal["person_properties"]], field.name) - materialized_column = self._get_materialized_column(table_name, symbol.name, field_name) + if field_name == "person_properties" and not self.context.using_person_on_events: + # :KLUDGE: person property materialized columns support when person on events is off + materialized_column = self._get_materialized_column("person", symbol.name, "properties") + else: + materialized_column = self._get_materialized_column(table_name, symbol.name, field_name) if materialized_column: property_sql = self._print_identifier(materialized_column) @@ -493,11 +497,6 @@ def _print_identifier(self, name: str) -> str: def _get_materialized_column( self, table_name: TablesWithMaterializedColumns, property_name: PropertyName, field_name: TableColumn ) -> Optional[str]: - # :KLUDGE: person property materialised columns support when person on events is off - if not self.context.using_person_on_events and table_name == "events" and field_name == "person_properties": - materialized_columns = get_materialized_columns("person") - return materialized_columns.get(("properties", field_name), None) - materialized_columns = get_materialized_columns(table_name) return materialized_columns.get((property_name, field_name), None) From 425a02e216ee5094f0ad88d019443afe8a272bcf Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 15 Feb 2023 13:34:51 +0000 Subject: [PATCH 51/81] Update snapshots --- ee/clickhouse/queries/test/__snapshots__/test_lifecycle.ambr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ee/clickhouse/queries/test/__snapshots__/test_lifecycle.ambr b/ee/clickhouse/queries/test/__snapshots__/test_lifecycle.ambr index 4bc6581b95d6e..90a37c4108324 100644 --- a/ee/clickhouse/queries/test/__snapshots__/test_lifecycle.ambr +++ b/ee/clickhouse/queries/test/__snapshots__/test_lifecycle.ambr @@ -426,7 +426,7 @@ AND event = '$pageview' AND timestamp >= toDateTime(dateTrunc('day', toDateTime('2021-04-28 00:00:00', 'UTC'))) - INTERVAL 1 day AND timestamp < toDateTime(dateTrunc('day', toDateTime('2021-05-05 23:59:59', 'UTC'))) + INTERVAL 1 day - AND (and(like(mat_$current_url, '%example%'), notEquals('bla', 'a%sd'))) + AND (and(like(`mat_$current_url`, '%example%'), notEquals('bla', 'a%sd'))) GROUP BY pdi.person_id) GROUP BY start_of_period, status) @@ -576,7 +576,7 @@ AND event = '$pageview' AND timestamp >= toDateTime(dateTrunc('day', toDateTime('2021-04-28 00:00:00', 'UTC'))) - INTERVAL 1 day AND timestamp < toDateTime(dateTrunc('day', toDateTime('2021-05-05 23:59:59', 'UTC'))) + INTERVAL 1 day - AND (like(mat_pp_email, '%test.com')) + AND (like(pmat_email, '%test.com')) GROUP BY pdi.person_id) GROUP BY start_of_period, status) From 0d17eb75c3e1ab31d7db0203fa1425861855906a Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 15 Feb 2023 13:41:32 +0000 Subject: [PATCH 52/81] Update snapshots --- posthog/api/test/__snapshots__/test_insight.ambr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/posthog/api/test/__snapshots__/test_insight.ambr b/posthog/api/test/__snapshots__/test_insight.ambr index 0aa6381ae13bb..c21dfe15be9df 100644 --- a/posthog/api/test/__snapshots__/test_insight.ambr +++ b/posthog/api/test/__snapshots__/test_insight.ambr @@ -396,7 +396,7 @@ AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2012-01-08 00:00:00', 'UTC')), 'UTC') AND toTimeZone(timestamp, 'UTC') <= toDateTime('2012-01-15 23:59:59', 'UTC') AND ((and(greater(toInt64OrNull(mat_int_value), 10), notEquals('bla', 'a%sd'))) - AND (like(mat_pp_fish, '%fish%'))) + AND (like(pmat_fish, '%fish%'))) GROUP BY date) GROUP BY day_start ORDER BY day_start) @@ -480,7 +480,7 @@ AND toTimeZone(timestamp, 'UTC') >= toDateTime(toStartOfDay(toDateTime('2012-01-08 00:00:00', 'UTC')), 'UTC') AND toTimeZone(timestamp, 'UTC') <= toDateTime('2012-01-15 23:59:59', 'UTC') AND (and(less(toInt64OrNull(mat_int_value), 10), notEquals('bla', 'a%sd')) - AND like(mat_pp_fish, '%fish%')) + AND like(pmat_fish, '%fish%')) GROUP BY date) GROUP BY day_start ORDER BY day_start) From 4851b8f2b180c3dd2dc5e9d5fc3eaca3b391c12e Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 15 Feb 2023 13:52:49 +0000 Subject: [PATCH 53/81] Update snapshots --- .../__snapshots__/test_session_recording_list.ambr | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/posthog/queries/session_recordings/test/__snapshots__/test_session_recording_list.ambr b/posthog/queries/session_recordings/test/__snapshots__/test_session_recording_list.ambr index 1a633f0a9edbb..79070c7c9eaf0 100644 --- a/posthog/queries/session_recordings/test/__snapshots__/test_session_recording_list.ambr +++ b/posthog/queries/session_recordings/test/__snapshots__/test_session_recording_list.ambr @@ -483,14 +483,14 @@ any(session_recordings.duration) as duration, any(session_recordings.distinct_id) as distinct_id , countIf(event = '$pageview' - AND (equals("mat_$browser", 'Chrome') - AND equals("pmat_email", 'bla'))) as count_event_match_0 , + AND (equals(`mat_$browser`, 'Chrome') + AND equals(pmat_email, 'bla'))) as count_event_match_0 , groupUniqArrayIf(100)((events.timestamp, events.uuid, events.session_id, events.window_id), event = '$pageview' - AND (equals("mat_$browser", 'Chrome') - AND equals("pmat_email", 'bla'))) as matching_events_0 + AND (equals(`mat_$browser`, 'Chrome') + AND equals(pmat_email, 'bla'))) as matching_events_0 FROM (SELECT uuid, distinct_id, @@ -562,12 +562,12 @@ any(session_recordings.duration) as duration, any(session_recordings.distinct_id) as distinct_id , countIf(event = '$pageview' - AND (equals("mat_$browser", 'Firefox'))) as count_event_match_0 , + AND (equals(`mat_$browser`, 'Firefox'))) as count_event_match_0 , groupUniqArrayIf(100)((events.timestamp, events.uuid, events.session_id, events.window_id), event = '$pageview' - AND (equals("mat_$browser", 'Firefox'))) as matching_events_0 + AND (equals(`mat_$browser`, 'Firefox'))) as matching_events_0 FROM (SELECT uuid, distinct_id, From d17d72578d9e879313fa35977ff3de870e465da1 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 16 Feb 2023 10:31:54 +0100 Subject: [PATCH 54/81] basic hogql node --- .../lib/lemon-ui/LemonTable/LemonTable.tsx | 2 +- frontend/src/queries/examples.ts | 17 +++++++ .../src/queries/nodes/DataNode/DataNode.tsx | 4 +- .../queries/nodes/DataNode/dataNodeLogic.ts | 15 +++++- .../src/queries/nodes/DataTable/DataTable.tsx | 26 ++++++++-- .../queries/nodes/DataTable/dataTableLogic.ts | 7 +-- frontend/src/queries/query.ts | 5 ++ frontend/src/queries/schema.json | 51 +++++++++++++++++++ frontend/src/queries/schema.ts | 19 ++++++- frontend/src/queries/utils.ts | 7 ++- .../scenes/saved-insights/SavedInsights.tsx | 6 +++ posthog/api/query.py | 49 +++++++++--------- posthog/api/test/test_query.py | 47 ++++++++++++++++- posthog/schema.py | 25 ++++++++- 14 files changed, 236 insertions(+), 44 deletions(-) diff --git a/frontend/src/lib/lemon-ui/LemonTable/LemonTable.tsx b/frontend/src/lib/lemon-ui/LemonTable/LemonTable.tsx index aeffd4a92400b..f800eb7f840ad 100644 --- a/frontend/src/lib/lemon-ui/LemonTable/LemonTable.tsx +++ b/frontend/src/lib/lemon-ui/LemonTable/LemonTable.tsx @@ -143,7 +143,7 @@ export function LemonTable>({ ) const columnGroups = ( - 'children' in rawColumns[0] + rawColumns.length > 0 && 'children' in rawColumns[0] ? rawColumns : [ { diff --git a/frontend/src/queries/examples.ts b/frontend/src/queries/examples.ts index 87ca2b5d229c8..dd34e5ad1b947 100644 --- a/frontend/src/queries/examples.ts +++ b/frontend/src/queries/examples.ts @@ -5,6 +5,7 @@ import { EventsNode, EventsQuery, FunnelsQuery, + HogQLQuery, LegacyQuery, LifecycleQuery, Node, @@ -290,6 +291,20 @@ const TimeToSeeDataWaterfall: TimeToSeeDataWaterfallNode = { }, } +const HogQL: HogQLQuery = { + kind: NodeKind.HogQLQuery, + query: 'select 1, 2', +} + +const HogQLTable: DataTableNode = { + kind: NodeKind.DataTableNode, + full: true, + source: { + kind: NodeKind.HogQLQuery, + query: 'select 1, 2', + }, +} + export const examples: Record = { Events, EventsTable, @@ -310,6 +325,8 @@ export const examples: Record = { TimeToSeeDataSessions, TimeToSeeDataWaterfall, TimeToSeeDataJSON, + HogQL, + HogQLTable, } export const stringifiedExamples: Record = Object.fromEntries( diff --git a/frontend/src/queries/nodes/DataNode/DataNode.tsx b/frontend/src/queries/nodes/DataNode/DataNode.tsx index 992bc84126299..465651fa1963d 100644 --- a/frontend/src/queries/nodes/DataNode/DataNode.tsx +++ b/frontend/src/queries/nodes/DataNode/DataNode.tsx @@ -18,7 +18,7 @@ let uniqueNode = 0 export function DataNode(props: DataNodeProps): JSX.Element { const [key] = useState(() => `DataNode.${uniqueNode++}`) const logic = dataNodeLogic({ ...props, key }) - const { response, responseLoading } = useValues(logic) + const { response, responseLoading, responseErrorObject } = useValues(logic) return (
@@ -36,7 +36,7 @@ export function DataNode(props: DataNodeProps): JSX.Element { theme="vs-light" className="border" language={'json'} - value={JSON.stringify(response, null, 2)} + value={JSON.stringify(response ?? responseErrorObject, null, 2)} height={Math.max(height, 300)} /> )} diff --git a/frontend/src/queries/nodes/DataNode/dataNodeLogic.ts b/frontend/src/queries/nodes/DataNode/dataNodeLogic.ts index 44d0a467c599e..9ea069e8800c4 100644 --- a/frontend/src/queries/nodes/DataNode/dataNodeLogic.ts +++ b/frontend/src/queries/nodes/DataNode/dataNodeLogic.ts @@ -166,11 +166,24 @@ export const dataNodeLogic = kea([ // Clear the response if a failure to avoid showing inconsistencies in the UI loadDataFailure: () => null, }, + responseErrorObject: [ + null as Record | null, + { + loadData: () => null, + loadDataFailure: (_, { errorObject }) => errorObject, + loadDataSuccess: () => null, + }, + ], responseError: [ null as string | null, { loadData: () => null, - loadDataFailure: () => 'Error loading data', + loadDataFailure: (_, { error, errorObject }) => { + if (errorObject && 'error' in errorObject) { + return errorObject.error + } + return error ?? 'Error loading data' + }, loadDataSuccess: () => null, }, ], diff --git a/frontend/src/queries/nodes/DataTable/DataTable.tsx b/frontend/src/queries/nodes/DataTable/DataTable.tsx index 3c9acf2be5fbc..93c71c5602b6a 100644 --- a/frontend/src/queries/nodes/DataTable/DataTable.tsx +++ b/frontend/src/queries/nodes/DataTable/DataTable.tsx @@ -21,7 +21,7 @@ import { EventBufferNotice } from 'scenes/events/EventBufferNotice' import clsx from 'clsx' import { SessionPlayerModal } from 'scenes/session-recordings/player/modal/SessionPlayerModal' import { InlineEditorButton } from '~/queries/nodes/Node/InlineEditorButton' -import { isEventsQuery, isHogQlAggregation, isPersonsNode, taxonomicFilterToHogQl } from '~/queries/utils' +import { isEventsQuery, isHogQlAggregation, isHogQLQuery, isPersonsNode, taxonomicFilterToHogQl } from '~/queries/utils' import { PersonPropertyFilters } from '~/queries/nodes/PersonsNode/PersonPropertyFilters' import { PersonsSearch } from '~/queries/nodes/PersonsNode/PersonsSearch' import { PersonDeleteModal } from 'scenes/persons/PersonDeleteModal' @@ -89,8 +89,9 @@ export function DataTable({ query, setQuery, context }: DataTableProps): JSX.Ele } = queryWithDefaults const actionsColumnShown = showActions && isEventsQuery(query.source) && columnsInResponse?.includes('*') + const columnsInLemonTable = isHogQLQuery(query.source) ? columnsInResponse ?? columnsInQuery : columnsInQuery const lemonColumns: LemonTableColumn[] = [ - ...columnsInQuery.map((key, index) => ({ + ...columnsInLemonTable.map((key, index) => ({ dataIndex: key as any, ...renderColumnMeta(key, query, context), render: function RenderDataTableColumn(_: any, { result, label }: DataTableRow) { @@ -98,13 +99,13 @@ export function DataTable({ query, setQuery, context }: DataTableProps): JSX.Ele if (index === (expandable ? 1 : 0)) { return { children: label, - props: { colSpan: columnsInQuery.length + (actionsColumnShown ? 1 : 0) }, + props: { colSpan: columnsInLemonTable.length + (actionsColumnShown ? 1 : 0) }, } } else { return { props: { colSpan: 0 } } } } else if (result) { - if (isEventsQuery(query.source)) { + if (isEventsQuery(query.source) || isHogQLQuery(query.source)) { return renderColumn(key, result[index], result, query, setQuery, context) } return renderColumn(key, result[key], result, query, setQuery, context) @@ -398,7 +399,22 @@ export function DataTable({ query, setQuery, context }: DataTableProps): JSX.Ele }} sorting={null} useURLForSorting={false} - emptyState={responseError ? : } + emptyState={ + responseError ? ( + isHogQLQuery(query.source) ? ( + + ) : ( + + ) + ) : ( + + ) + } expandable={ expandable && isEventsQuery(query.source) && columnsInResponse?.includes('*') ? { diff --git a/frontend/src/queries/nodes/DataTable/dataTableLogic.ts b/frontend/src/queries/nodes/DataTable/dataTableLogic.ts index f9193f38c18c2..359a1c136e434 100644 --- a/frontend/src/queries/nodes/DataTable/dataTableLogic.ts +++ b/frontend/src/queries/nodes/DataTable/dataTableLogic.ts @@ -59,12 +59,7 @@ export const dataTableLogic = kea([ columnsInResponse: [ (s) => [s.response], (response: AnyDataNode['response']): string[] | null => - response && - 'columns' in response && - Array.isArray(response.columns) && - !response.columns.find((c) => typeof c !== 'string') - ? (response?.columns as string[]) - : null, + response && 'columns' in response && Array.isArray(response.columns) ? response?.columns : null, ], dataTableRows: [ (s) => [s.sourceKind, s.orderBy, s.response, s.columnsInQuery, s.columnsInResponse], diff --git a/frontend/src/queries/query.ts b/frontend/src/queries/query.ts index 9e13bad1b38b8..f9ea3b1cd49aa 100644 --- a/frontend/src/queries/query.ts +++ b/frontend/src/queries/query.ts @@ -9,6 +9,7 @@ import { isRecentPerformancePageViewNode, isDataTableNode, isTimeToSeeDataSessionsNode, + isHogQLQuery, } from './utils' import api, { ApiMethodOptions } from 'lib/api' import { getCurrentTeamId } from 'lib/utils/logics' @@ -46,6 +47,8 @@ export function queryExportContext( after: now().subtract(EVENTS_DAYS_FIRST_FETCH, 'day').toISOString(), }, } + } else if (isHogQLQuery(query)) { + return { path: api.queryURL(), method: 'POST', body: query } } else if (isPersonsNode(query)) { return { path: getPersonsEndpoint(query) } } else if (isInsightQueryNode(query)) { @@ -115,6 +118,8 @@ export async function query( } } return await api.query({ after: now().subtract(1, 'year').toISOString(), ...query }, methodOptions) + } else if (isHogQLQuery(query)) { + return api.query(query, methodOptions) } else if (isPersonsNode(query)) { return await api.get(getPersonsEndpoint(query), methodOptions) } else if (isInsightQueryNode(query)) { diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json index 58353872860da..837732a4ce7ec 100644 --- a/frontend/src/queries/schema.json +++ b/frontend/src/queries/schema.json @@ -80,6 +80,9 @@ }, { "$ref": "#/definitions/PersonsNode" + }, + { + "$ref": "#/definitions/HogQLQuery" } ] }, @@ -1273,6 +1276,9 @@ }, { "$ref": "#/definitions/RecentPerformancePageViewNode" + }, + { + "$ref": "#/definitions/HogQLQuery" } ], "description": "Source of the events" @@ -1944,6 +1950,51 @@ "required": ["key", "type"], "type": "object" }, + "HogQLQuery": { + "additionalProperties": false, + "properties": { + "kind": { + "const": "HogQLQuery", + "type": "string" + }, + "query": { + "type": "string" + }, + "response": { + "$ref": "#/definitions/HogQLQueryResponse", + "description": "Cached query response" + } + }, + "required": ["kind", "query"], + "type": "object" + }, + "HogQLQueryResponse": { + "additionalProperties": false, + "properties": { + "clickhouse": { + "type": "string" + }, + "columns": { + "items": {}, + "type": "array" + }, + "hogql": { + "type": "string" + }, + "query": { + "type": "string" + }, + "results": { + "items": {}, + "type": "array" + }, + "types": { + "items": {}, + "type": "array" + } + }, + "type": "object" + }, "InsightQueryNode": { "anyOf": [ { diff --git a/frontend/src/queries/schema.ts b/frontend/src/queries/schema.ts index 03f0134ca698c..c36176a99a696 100644 --- a/frontend/src/queries/schema.ts +++ b/frontend/src/queries/schema.ts @@ -40,6 +40,7 @@ export enum NodeKind { NewEntityNode = 'NewEntityNode', EventsQuery = 'EventsQuery', PersonsNode = 'PersonsNode', + HogQLQuery = 'HogQLQuery', // Interface nodes DataTableNode = 'DataTableNode', @@ -64,7 +65,7 @@ export enum NodeKind { RecentPerformancePageViewNode = 'RecentPerformancePageViewNode', } -export type AnyDataNode = EventsNode | EventsQuery | ActionsNode | PersonsNode +export type AnyDataNode = EventsNode | EventsQuery | ActionsNode | PersonsNode | HogQLQuery export type QuerySchema = // Data nodes (see utils.ts) @@ -101,6 +102,20 @@ export interface DataNode extends Node { response?: Record } +export interface HogQLQueryResponse { + query?: string + hogql?: string + clickhouse?: string + results?: any[] + types?: any[] + columns?: any[] +} + +export interface HogQLQuery extends DataNode { + kind: NodeKind.HogQLQuery + query: string + response?: HogQLQueryResponse +} export interface EntityNode extends DataNode { name?: string custom_name?: string @@ -201,7 +216,7 @@ export type HasPropertiesNode = EventsNode | EventsQuery | PersonsNode export interface DataTableNode extends Node { kind: NodeKind.DataTableNode /** Source of the events */ - source: EventsNode | EventsQuery | PersonsNode | RecentPerformancePageViewNode + source: EventsNode | EventsQuery | PersonsNode | RecentPerformancePageViewNode | HogQLQuery /** Columns shown in the table, unless the `source` provides them. */ columns?: HogQLExpression[] /** Columns that aren't shown in the table, even if in columns or returned data */ diff --git a/frontend/src/queries/utils.ts b/frontend/src/queries/utils.ts index a59c909642a32..2baee0972e2c7 100644 --- a/frontend/src/queries/utils.ts +++ b/frontend/src/queries/utils.ts @@ -4,6 +4,7 @@ import { DateRange, EventsNode, EventsQuery, + HogQLQuery, TrendsQuery, FunnelsQuery, RetentionQuery, @@ -27,7 +28,7 @@ import { import { TaxonomicFilterGroupType, TaxonomicFilterValue } from 'lib/components/TaxonomicFilter/types' export function isDataNode(node?: Node): node is EventsQuery | PersonsNode | TimeToSeeDataSessionsQuery { - return isEventsQuery(node) || isPersonsNode(node) || isTimeToSeeDataSessionsQuery(node) + return isEventsQuery(node) || isPersonsNode(node) || isTimeToSeeDataSessionsQuery(node) || isHogQLQuery(node) } export function isEventsNode(node?: Node): node is EventsNode { @@ -58,6 +59,10 @@ export function isLegacyQuery(node?: Node): node is LegacyQuery { return node?.kind === NodeKind.LegacyQuery } +export function isHogQLQuery(node?: Node): node is HogQLQuery { + return node?.kind === NodeKind.HogQLQuery +} + /* * Insight Queries */ diff --git a/frontend/src/scenes/saved-insights/SavedInsights.tsx b/frontend/src/scenes/saved-insights/SavedInsights.tsx index 0e04c893fc2d5..504743a42098d 100644 --- a/frontend/src/scenes/saved-insights/SavedInsights.tsx +++ b/frontend/src/scenes/saved-insights/SavedInsights.tsx @@ -218,6 +218,12 @@ export const QUERY_TYPES_METADATA: Record = { icon: IconCoffee, inMenu: true, }, + [NodeKind.HogQLQuery]: { + name: 'HogQL', + description: 'Direct HogQL query', + icon: IconCoffee, + inMenu: true, + }, } export const INSIGHT_TYPE_OPTIONS: LemonSelectOptions = [ diff --git a/posthog/api/query.py b/posthog/api/query.py index 95909caafd620..17d2dbc4014b9 100644 --- a/posthog/api/query.py +++ b/posthog/api/query.py @@ -4,6 +4,7 @@ from django.http import HttpResponse, JsonResponse from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import OpenApiParameter +from pydantic import BaseModel from rest_framework import viewsets from rest_framework.exceptions import ValidationError from rest_framework.permissions import IsAuthenticated @@ -11,11 +12,12 @@ from posthog.api.documentation import extend_schema from posthog.api.routing import StructuredViewSetMixin +from posthog.hogql.query import execute_hogql_query from posthog.models import Team from posthog.models.event.query_event_list import run_events_query from posthog.permissions import ProjectMembershipNecessaryPermissions, TeamMemberAccessPermission from posthog.rate_limit import ClickHouseBurstRateThrottle, ClickHouseSustainedRateThrottle -from posthog.schema import EventsQuery +from posthog.schema import EventsQuery, HogQLQuery class QueryViewSet(StructuredViewSetMixin, viewsets.ViewSet): @@ -33,13 +35,27 @@ class QueryViewSet(StructuredViewSetMixin, viewsets.ViewSet): ) def list(self, request: Request, **kw) -> HttpResponse: query_json = self._query_json_from_request(request) - query_result = process_query(self.team, query_json) - return JsonResponse(query_result) + return self._process_query(self.team, query_json) def post(self, request, *args, **kwargs): query_json = request.data - query_result = process_query(self.team, query_json) - return JsonResponse(query_result) + return self._process_query(self.team, query_json) + + def _process_query(self, team: Team, query_json: Dict) -> JsonResponse: + # try: + query_kind = query_json.get("kind") + if query_kind == "EventsQuery": + query = EventsQuery.parse_obj(query_json) + response = run_events_query(query=query, team=team) + return self._response_to_json_response(response) + elif query_kind == "HogQLQuery": + query = HogQLQuery.parse_obj(query_json) + response = execute_hogql_query(query=query.query, team=team) + return self._response_to_json_response(response) + else: + raise ValidationError("Unsupported query kind: %s" % query_kind) + # except Exception as e: + # return JsonResponse({"error": str(e)}, status=400) def _query_json_from_request(self, request): if request.method == "POST": @@ -65,21 +81,8 @@ def parsing_error(ex): raise ValidationError("Invalid JSON: %s" % (str(error_main))) return query - -def process_query(team: Team, query_json: Dict) -> Dict: - query_kind = query_json.get("kind") - if query_kind == "EventsQuery": - query = EventsQuery.parse_obj(query_json) - query_result = run_events_query( - team=team, - query=query, - ) - # :KLUDGE: Calling `query_result.dict()` without the following deconstruction fails with a cryptic error - return { - "columns": query_result.columns, - "types": query_result.types, - "results": query_result.results, - "hasMore": query_result.hasMore, - } - else: - raise ValidationError("Unsupported query kind: %s" % query_kind) + def _response_to_json_response(self, response: BaseModel) -> JsonResponse: + dict = {} + for key in response.__fields__.keys(): + dict[key] = getattr(response, key) + return JsonResponse(dict) diff --git a/posthog/api/test/test_query.py b/posthog/api/test/test_query.py index 4c65e8ee5eec6..8ac341658f368 100644 --- a/posthog/api/test/test_query.py +++ b/posthog/api/test/test_query.py @@ -1,7 +1,15 @@ from freezegun import freeze_time from rest_framework import status -from posthog.schema import EventPropertyFilter, EventsQuery, HogQLPropertyFilter, PersonPropertyFilter, PropertyOperator +from posthog.schema import ( + EventPropertyFilter, + EventsQuery, + HogQLPropertyFilter, + HogQLQuery, + HogQLQueryResponse, + PersonPropertyFilter, + PropertyOperator, +) from posthog.test.base import ( APIBaseTest, ClickhouseTestMixin, @@ -256,3 +264,40 @@ def test_property_filter_aggregations(self): query.where = ["count() > 1"] response = self.client.post(f"/api/projects/{self.team.id}/query/", query.dict()).json() self.assertEqual(len(response["results"]), 1) + + @also_test_with_materialized_columns(event_properties=["key"]) + @snapshot_clickhouse_queries + def test_full_hogql_query(self): + with freeze_time("2020-01-10 12:00:00"): + _create_person( + properties={"email": "tom@posthog.com"}, + distinct_ids=["2", "some-random-uid"], + team=self.team, + immediate=True, + ) + _create_event(team=self.team, event="sign up", distinct_id="2", properties={"key": "test_val1"}) + with freeze_time("2020-01-10 12:11:00"): + _create_event(team=self.team, event="sign out", distinct_id="2", properties={"key": "test_val2"}) + with freeze_time("2020-01-10 12:12:00"): + _create_event(team=self.team, event="sign out", distinct_id="3", properties={"key": "test_val2"}) + with freeze_time("2020-01-10 12:13:00"): + _create_event( + team=self.team, event="sign out", distinct_id="4", properties={"key": "test_val3", "path": "a/b/c"} + ) + flush_persons_and_events() + + with freeze_time("2020-01-10 12:14:00"): + query = HogQLQuery(query="select event, distinct_id, properties.key from events order by timestamp") + api_response = self.client.post(f"/api/projects/{self.team.id}/query/", query.dict()).json() + query.response = HogQLQueryResponse.parse_obj(api_response) + + self.assertEqual(len(query.response.results), 4) + self.assertEqual( + query.response.results, + [ + ["sign up", "2", "test_val1"], + ["sign out", "2", "test_val2"], + ["sign out", "3", "test_val2"], + ["sign out", "4", "test_val3"], + ], + ) diff --git a/posthog/schema.py b/posthog/schema.py index a9c22bf6e7321..f6a89c5905cd7 100644 --- a/posthog/schema.py +++ b/posthog/schema.py @@ -240,6 +240,18 @@ class FunnelCorrelationPersonConverted1(str, Enum): false = "false" +class HogQLQueryResponse(BaseModel): + class Config: + extra = Extra.forbid + + clickhouse: Optional[str] = None + columns: Optional[List] = None + hogql: Optional[str] = None + query: Optional[str] = None + results: Optional[List] = None + types: Optional[List] = None + + class InsightType(str, Enum): TRENDS = "TRENDS" STICKINESS = "STICKINESS" @@ -536,6 +548,15 @@ class Config: value: Optional[Union[str, float, List[Union[str, float]]]] = None +class HogQLQuery(BaseModel): + class Config: + extra = Extra.forbid + + kind: str = Field("HogQLQuery", const=True) + query: str + response: Optional[HogQLQueryResponse] = Field(None, description="Cached query response") + + class LifecycleFilter(BaseModel): class Config: extra = Extra.forbid @@ -828,7 +849,7 @@ class Config: showReload: Optional[bool] = Field(None, description="Show a reload button") showSavedQueries: Optional[bool] = Field(None, description="Shows a list of saved queries") showSearch: Optional[bool] = Field(None, description="Include a free text search field (PersonsNode only)") - source: Union[EventsNode, EventsQuery, PersonsNode, RecentPerformancePageViewNode] = Field( + source: Union[EventsNode, EventsQuery, PersonsNode, RecentPerformancePageViewNode, HogQLQuery] = Field( ..., description="Source of the events" ) @@ -1460,7 +1481,7 @@ class Model(BaseModel): LifecycleQuery, RecentPerformancePageViewNode, TimeToSeeDataSessionsQuery, - Union[EventsNode, EventsQuery, ActionsNode, PersonsNode], + Union[EventsNode, EventsQuery, ActionsNode, PersonsNode, HogQLQuery], ] From cc17eff30d780947644f9549da206616ad9a1faa Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 16 Feb 2023 11:28:00 +0100 Subject: [PATCH 55/81] no need to discard resolved symbols... we use them again and again --- posthog/api/query.py | 28 ++++++++++++++-------------- posthog/hogql/printer.py | 4 +--- posthog/hogql/query.py | 8 +++++++- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/posthog/api/query.py b/posthog/api/query.py index 17d2dbc4014b9..289b1d4d366b2 100644 --- a/posthog/api/query.py +++ b/posthog/api/query.py @@ -42,20 +42,20 @@ def post(self, request, *args, **kwargs): return self._process_query(self.team, query_json) def _process_query(self, team: Team, query_json: Dict) -> JsonResponse: - # try: - query_kind = query_json.get("kind") - if query_kind == "EventsQuery": - query = EventsQuery.parse_obj(query_json) - response = run_events_query(query=query, team=team) - return self._response_to_json_response(response) - elif query_kind == "HogQLQuery": - query = HogQLQuery.parse_obj(query_json) - response = execute_hogql_query(query=query.query, team=team) - return self._response_to_json_response(response) - else: - raise ValidationError("Unsupported query kind: %s" % query_kind) - # except Exception as e: - # return JsonResponse({"error": str(e)}, status=400) + try: + query_kind = query_json.get("kind") + if query_kind == "EventsQuery": + query = EventsQuery.parse_obj(query_json) + response = run_events_query(query=query, team=team) + return self._response_to_json_response(response) + elif query_kind == "HogQLQuery": + query = HogQLQuery.parse_obj(query_json) + response = execute_hogql_query(query=query.query, team=team) + return self._response_to_json_response(response) + else: + raise ValidationError("Unsupported query kind: %s" % query_kind) + except Exception as e: + return JsonResponse({"error": str(e)}, status=400) def _query_json_from_request(self, request): if request.method == "POST": diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index bce2d1a9b25b3..5f27ff8ce3ba5 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -8,7 +8,7 @@ from posthog.hogql.database import Table, database from posthog.hogql.print_string import print_clickhouse_identifier, print_hogql_identifier from posthog.hogql.resolver import ResolverException, lookup_field_by_name, resolve_symbols -from posthog.hogql.visitor import Visitor, clone_expr +from posthog.hogql.visitor import Visitor from posthog.models.property import PropertyName, TableColumn @@ -35,8 +35,6 @@ def print_ast( """Print an AST into a string. Does not modify the node.""" symbol = stack[-1].symbol if stack else None - # make a clean copy of the object - node = clone_expr(node) # resolve symbols resolve_symbols(node, symbol) diff --git a/posthog/hogql/query.py b/posthog/hogql/query.py index e1979d05706d3..45251b69a610b 100644 --- a/posthog/hogql/query.py +++ b/posthog/hogql/query.py @@ -56,7 +56,13 @@ def execute_hogql_query( query_type=query_type, workload=workload, ) - print_columns = [print_ast(col, HogQLContext(), "hogql") for col in select_query.select] + print_columns = [] + for node in select_query.select: + if isinstance(node, ast.Alias): + print_columns.append(node.alias) + else: + print_columns.append(print_ast(node=node, context=hogql_context, dialect="hogql", stack=[select_query])) + return HogQLQueryResponse( query=query, hogql=hogql, From f3e7900d96605d934142c94a6ac00a15e4b75b87 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 16 Feb 2023 11:29:45 +0100 Subject: [PATCH 56/81] actual sql editor --- frontend/src/queries/examples.ts | 7 +-- .../src/queries/nodes/DataTable/DataTable.tsx | 11 +++- .../queries/nodes/DataTable/dataTableLogic.ts | 1 + .../nodes/HogQLQuery/HogQLQueryEditor.tsx | 55 +++++++++++++++++++ .../nodes/HogQLQuery/hogQLQueryEditorLogic.ts | 34 ++++++++++++ frontend/src/queries/schema.ts | 2 + 6 files changed, 102 insertions(+), 8 deletions(-) create mode 100644 frontend/src/queries/nodes/HogQLQuery/HogQLQueryEditor.tsx create mode 100644 frontend/src/queries/nodes/HogQLQuery/hogQLQueryEditorLogic.ts diff --git a/frontend/src/queries/examples.ts b/frontend/src/queries/examples.ts index dd34e5ad1b947..91077af07054d 100644 --- a/frontend/src/queries/examples.ts +++ b/frontend/src/queries/examples.ts @@ -293,16 +293,13 @@ const TimeToSeeDataWaterfall: TimeToSeeDataWaterfallNode = { const HogQL: HogQLQuery = { kind: NodeKind.HogQLQuery, - query: 'select 1, 2', + query: 'select event, count() as event_count from events group by event order by event_count desc', } const HogQLTable: DataTableNode = { kind: NodeKind.DataTableNode, full: true, - source: { - kind: NodeKind.HogQLQuery, - query: 'select 1, 2', - }, + source: HogQL, } export const examples: Record = { diff --git a/frontend/src/queries/nodes/DataTable/DataTable.tsx b/frontend/src/queries/nodes/DataTable/DataTable.tsx index 93c71c5602b6a..298fffde66a7b 100644 --- a/frontend/src/queries/nodes/DataTable/DataTable.tsx +++ b/frontend/src/queries/nodes/DataTable/DataTable.tsx @@ -1,5 +1,5 @@ import './DataTable.scss' -import { DataTableNode, EventsNode, EventsQuery, Node, PersonsNode, QueryContext } from '~/queries/schema' +import { DataTableNode, EventsNode, EventsQuery, HogQLQuery, Node, PersonsNode, QueryContext } from '~/queries/schema' import { useCallback, useState } from 'react' import { BindLogic, useValues } from 'kea' import { dataNodeLogic, DataNodeLogicProps } from '~/queries/nodes/DataNode/dataNodeLogic' @@ -34,6 +34,7 @@ import { extractExpressionComment, removeExpressionComment } from '~/queries/nod import { InsightEmptyState, InsightErrorState } from 'scenes/insights/EmptyStates' import { EventType } from '~/types' import { SavedQueries } from '~/queries/nodes/DataTable/SavedQueries' +import { HogQLQueryEditor } from '~/queries/nodes/HogQLQuery/HogQLQueryEditor' interface DataTableProps { query: DataTableNode @@ -79,6 +80,7 @@ export function DataTable({ query, setQuery, context }: DataTableProps): JSX.Ele showSearch, showEventFilter, showPropertyFilter, + showHogQLEditor, showReload, showExport, showElapsedTime, @@ -293,7 +295,7 @@ export function DataTable({ query, setQuery, context }: DataTableProps): JSX.Ele ].filter((column) => !query.hiddenColumns?.includes(column.dataIndex) && column.dataIndex !== '*') const setQuerySource = useCallback( - (source: EventsNode | EventsQuery | PersonsNode) => setQuery?.({ ...query, source }), + (source: EventsNode | EventsQuery | PersonsNode | HogQLQuery) => setQuery?.({ ...query, source }), [setQuery] ) @@ -338,7 +340,10 @@ export function DataTable({ query, setQuery, context }: DataTableProps): JSX.Ele return ( -
+
+ {showHogQLEditor && isHogQLQuery(query.source) ? ( + + ) : null} {showFirstRow && (
{firstRowLeft} diff --git a/frontend/src/queries/nodes/DataTable/dataTableLogic.ts b/frontend/src/queries/nodes/DataTable/dataTableLogic.ts index 359a1c136e434..f5165b33a6313 100644 --- a/frontend/src/queries/nodes/DataTable/dataTableLogic.ts +++ b/frontend/src/queries/nodes/DataTable/dataTableLogic.ts @@ -142,6 +142,7 @@ export const dataTableLogic = kea([ showColumnConfigurator: query.showColumnConfigurator ?? showIfFull, showSavedQueries: query.showSavedQueries ?? false, showEventsBufferWarning: query.showEventsBufferWarning ?? showIfFull, + showHogQLEditor: query.showHogQLEditor ?? showIfFull, allowSorting: query.allowSorting ?? true, }), } diff --git a/frontend/src/queries/nodes/HogQLQuery/HogQLQueryEditor.tsx b/frontend/src/queries/nodes/HogQLQuery/HogQLQueryEditor.tsx new file mode 100644 index 0000000000000..2ee10bc896419 --- /dev/null +++ b/frontend/src/queries/nodes/HogQLQuery/HogQLQueryEditor.tsx @@ -0,0 +1,55 @@ +import { useActions, useValues } from 'kea' +import { HogQLQuery } from '~/queries/schema' +import { useState } from 'react' +import { hogQLQueryEditorLogic } from './hogQLQueryEditorLogic' +import MonacoEditor from '@monaco-editor/react' +import { AutoSizer } from 'react-virtualized/dist/es/AutoSizer' +import { LemonButton } from 'lib/lemon-ui/LemonButton' + +export interface HogQLQueryEditorProps { + query: HogQLQuery + setQuery?: (query: HogQLQuery) => void +} + +let uniqueNode = 0 +export function HogQLQueryEditor(props: HogQLQueryEditorProps): JSX.Element { + const [key] = useState(() => uniqueNode++) + const hogQLQueryEditorLogicProps = { query: props.query, setQuery: props.setQuery, key } + const { queryInput } = useValues(hogQLQueryEditorLogic(hogQLQueryEditorLogicProps)) + const { setQueryInput, saveQuery } = useActions(hogQLQueryEditorLogic(hogQLQueryEditorLogicProps)) + + return ( +
+
+ + {({ height }) => ( + setQueryInput(v ?? '')} + height={height} + options={{ + minimap: { + enabled: false, + }, + wordWrap: 'on', + }} + /> + )} + +
+ + {!props.setQuery ? 'No permission to update' : 'Update'} + +
+ ) +} diff --git a/frontend/src/queries/nodes/HogQLQuery/hogQLQueryEditorLogic.ts b/frontend/src/queries/nodes/HogQLQuery/hogQLQueryEditorLogic.ts new file mode 100644 index 0000000000000..474926ec9f1d3 --- /dev/null +++ b/frontend/src/queries/nodes/HogQLQuery/hogQLQueryEditorLogic.ts @@ -0,0 +1,34 @@ +import { actions, kea, key, listeners, path, props, propsChanged, reducers } from 'kea' +import { HogQLQuery } from '~/queries/schema' + +import type { hogQLQueryEditorLogicType } from './hogQLQueryEditorLogicType' + +export interface HogQLQueryEditorLogicProps { + key: number + query: HogQLQuery + setQuery?: (query: HogQLQuery) => void +} + +export const hogQLQueryEditorLogic = kea([ + path(['queries', 'nodes', 'HogQLQuery', 'hogQLQueryEditorLogic']), + props({} as HogQLQueryEditorLogicProps), + key((props) => props.key), + propsChanged(({ actions, props }, oldProps) => { + if (props.query.query !== oldProps.query.query) { + actions.setQueryInput(props.query.query) + } + }), + actions({ + saveQuery: true, + setQueryInput: (queryInput: string) => ({ queryInput }), + }), + reducers(({ props }) => ({ + queryInput: [props.query.query, { setQueryInput: (_, { queryInput }) => queryInput }], + })), + listeners(({ actions, props, values }) => ({ + saveQuery: () => { + actions.setQueryInput(values.queryInput) + props.setQuery?.({ ...props.query, query: values.queryInput }) + }, + })), +]) diff --git a/frontend/src/queries/schema.ts b/frontend/src/queries/schema.ts index c36176a99a696..7ca8fafb769ec 100644 --- a/frontend/src/queries/schema.ts +++ b/frontend/src/queries/schema.ts @@ -229,6 +229,8 @@ export interface DataTableNode extends Node { showSearch?: boolean /** Include a property filter above the table */ showPropertyFilter?: boolean + /** Include a HogQL query editor above HogQL tables */ + showHogQLEditor?: boolean /** Show the kebab menu at the end of the row */ showActions?: boolean /** Show date range selector */ From d731524f6471ceadc8837a7dceb134025384562b Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 16 Feb 2023 11:36:47 +0100 Subject: [PATCH 57/81] hide query editor if opening a hogql table --- .../queries/nodes/DataTable/dataTableLogic.ts | 4 ++- frontend/src/scenes/query/QueryScene.tsx | 25 ++++++++++++++++--- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/frontend/src/queries/nodes/DataTable/dataTableLogic.ts b/frontend/src/queries/nodes/DataTable/dataTableLogic.ts index f5165b33a6313..e4bf1edbf9c3d 100644 --- a/frontend/src/queries/nodes/DataTable/dataTableLogic.ts +++ b/frontend/src/queries/nodes/DataTable/dataTableLogic.ts @@ -138,7 +138,9 @@ export const dataTableLogic = kea([ showDateRange: query.showDateRange ?? showIfFull, showExport: query.showExport ?? showIfFull, showReload: query.showReload ?? showIfFull, - showElapsedTime: query.showElapsedTime ?? (flagQueryRunningTimeEnabled ? showIfFull : false), + showElapsedTime: + query.showElapsedTime ?? + (flagQueryRunningTimeEnabled || source.kind === NodeKind.HogQLQuery ? showIfFull : false), showColumnConfigurator: query.showColumnConfigurator ?? showIfFull, showSavedQueries: query.showSavedQueries ?? false, showEventsBufferWarning: query.showEventsBufferWarning ?? showIfFull, diff --git a/frontend/src/scenes/query/QueryScene.tsx b/frontend/src/scenes/query/QueryScene.tsx index 6c0337dd51fb5..fb44c06db3813 100644 --- a/frontend/src/scenes/query/QueryScene.tsx +++ b/frontend/src/scenes/query/QueryScene.tsx @@ -14,6 +14,19 @@ export function QueryScene(): JSX.Element { const { query } = useValues(querySceneLogic) const { setQuery } = useActions(querySceneLogic) + let showEditor = true + try { + const parsed = JSON.parse(query) + if ( + parsed && + parsed.kind == 'DataTableNode' && + parsed.source.kind == 'HogQLQuery' && + (parsed.full || parsed.showHogQLEditor) + ) { + showEditor = false + } + } catch (e) {} + return (
@@ -35,10 +48,14 @@ export function QueryScene(): JSX.Element { ))}
- -
- -
+ {showEditor ? ( + <> + +
+ +
+ + ) : null} setQuery(JSON.stringify(query, null, 2))} />
From 0759e0864fd62d06faeb76f164fa8113c2790bf0 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 16 Feb 2023 11:03:00 +0000 Subject: [PATCH 58/81] Update snapshots --- .../api/test/__snapshots__/test_query.ambr | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/posthog/api/test/__snapshots__/test_query.ambr b/posthog/api/test/__snapshots__/test_query.ambr index 9b049fa990e6c..158a059e51a88 100644 --- a/posthog/api/test/__snapshots__/test_query.ambr +++ b/posthog/api/test/__snapshots__/test_query.ambr @@ -92,6 +92,30 @@ LIMIT 101 ' --- +# name: TestQuery.test_full_hogql_query + ' + /* user_id:0 request:_snapshot_ */ + SELECT event, + distinct_id, + replaceRegexpAll(JSONExtractRaw(properties, 'key'), '^"|"$', '') + FROM events + WHERE equals(team_id, 68) + ORDER BY timestamp ASC + LIMIT 1000 + ' +--- +# name: TestQuery.test_full_hogql_query_materialized + ' + /* user_id:0 request:_snapshot_ */ + SELECT event, + distinct_id, + mat_key + FROM events + WHERE equals(team_id, 69) + ORDER BY timestamp ASC + LIMIT 1000 + ' +--- # name: TestQuery.test_hogql_property_filter ' /* user_id:0 request:_snapshot_ */ From 5605a9440a1d56239670a97200034523ea6c140c Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 16 Feb 2023 11:30:43 +0000 Subject: [PATCH 59/81] Update snapshots --- .../__snapshots__/test_session_recording_list.ambr | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/posthog/queries/session_recordings/test/__snapshots__/test_session_recording_list.ambr b/posthog/queries/session_recordings/test/__snapshots__/test_session_recording_list.ambr index 1a633f0a9edbb..79070c7c9eaf0 100644 --- a/posthog/queries/session_recordings/test/__snapshots__/test_session_recording_list.ambr +++ b/posthog/queries/session_recordings/test/__snapshots__/test_session_recording_list.ambr @@ -483,14 +483,14 @@ any(session_recordings.duration) as duration, any(session_recordings.distinct_id) as distinct_id , countIf(event = '$pageview' - AND (equals("mat_$browser", 'Chrome') - AND equals("pmat_email", 'bla'))) as count_event_match_0 , + AND (equals(`mat_$browser`, 'Chrome') + AND equals(pmat_email, 'bla'))) as count_event_match_0 , groupUniqArrayIf(100)((events.timestamp, events.uuid, events.session_id, events.window_id), event = '$pageview' - AND (equals("mat_$browser", 'Chrome') - AND equals("pmat_email", 'bla'))) as matching_events_0 + AND (equals(`mat_$browser`, 'Chrome') + AND equals(pmat_email, 'bla'))) as matching_events_0 FROM (SELECT uuid, distinct_id, @@ -562,12 +562,12 @@ any(session_recordings.duration) as duration, any(session_recordings.distinct_id) as distinct_id , countIf(event = '$pageview' - AND (equals("mat_$browser", 'Firefox'))) as count_event_match_0 , + AND (equals(`mat_$browser`, 'Firefox'))) as count_event_match_0 , groupUniqArrayIf(100)((events.timestamp, events.uuid, events.session_id, events.window_id), event = '$pageview' - AND (equals("mat_$browser", 'Firefox'))) as matching_events_0 + AND (equals(`mat_$browser`, 'Firefox'))) as matching_events_0 FROM (SELECT uuid, distinct_id, From 1b02f0258fcd12717762e7e05cecfc78c020368b Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 16 Feb 2023 12:38:10 +0100 Subject: [PATCH 60/81] query into snapshot data --- posthog/hogql/database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/posthog/hogql/database.py b/posthog/hogql/database.py index 7341258bb58bc..3bed8aa173c94 100644 --- a/posthog/hogql/database.py +++ b/posthog/hogql/database.py @@ -117,10 +117,10 @@ class SessionRecordingEvents(Table): distinct_id: StringDatabaseField = StringDatabaseField(name="distinct_id") session_id: StringDatabaseField = StringDatabaseField(name="session_id") window_id: StringDatabaseField = StringDatabaseField(name="window_id") - snapshot_data: StringDatabaseField = StringDatabaseField(name="snapshot_data") + snapshot_data: StringJSONDatabaseField = StringJSONDatabaseField(name="snapshot_data") created_at: DateTimeDatabaseField = DateTimeDatabaseField(name="created_at") has_full_snapshot: BooleanDatabaseField = BooleanDatabaseField(name="has_full_snapshot") - events_summary: BooleanDatabaseField = BooleanDatabaseField(name="events_summary", array=True) + events_summary: StringJSONDatabaseField = StringJSONDatabaseField(name="events_summary", array=True) click_count: IntegerDatabaseField = IntegerDatabaseField(name="click_count") keypress_count: IntegerDatabaseField = IntegerDatabaseField(name="keypress_count") timestamps_summary: DateTimeDatabaseField = DateTimeDatabaseField(name="timestamps_summary", array=True) From b59cf91f4c16c5109d50514a8e1b1ff70c8eb047 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 16 Feb 2023 13:28:48 +0100 Subject: [PATCH 61/81] splash expander --- posthog/hogql/ast.py | 5 +++++ posthog/hogql/database.py | 14 +++++++------- posthog/hogql/parser.py | 6 ++++++ posthog/hogql/printer.py | 5 ++++- posthog/hogql/resolver.py | 2 +- posthog/hogql/transforms.py | 28 ++++++++++++++++++++++++++++ 6 files changed, 51 insertions(+), 9 deletions(-) create mode 100644 posthog/hogql/transforms.py diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index 2d55753b9d851..c1b0ad4d3f6e5 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -53,7 +53,12 @@ def has_child(self, name: str) -> bool: return self.table.has_field(name) def get_child(self, name: str) -> Symbol: + if name == "*": + return SplashSymbol(table=self) if self.has_child(name): + field = self.table.get_field(name) + if isinstance(field, Table): + return SplashSymbol(table=TableSymbol(table=field)) return FieldSymbol(name=name, table=self) raise ValueError(f"Field not found: {name}") diff --git a/posthog/hogql/database.py b/posthog/hogql/database.py index 3bed8aa173c94..55e8a54812307 100644 --- a/posthog/hogql/database.py +++ b/posthog/hogql/database.py @@ -1,4 +1,4 @@ -from typing import List +from typing import Dict from pydantic import BaseModel, Extra @@ -48,17 +48,17 @@ def get_field(self, name: str) -> DatabaseField: def clickhouse_table(self): raise NotImplementedError("Table.clickhouse_table not overridden") - def get_splash(self) -> List[str]: - list: List[str] = [] - for field in self.__fields__.values(): + def get_splash(self) -> Dict[str, DatabaseField]: + splash: Dict[str, DatabaseField] = {} + for key, field in self.__fields__.items(): database_field = field.default if isinstance(database_field, DatabaseField): - list.append(database_field.name) + splash[key] = database_field elif isinstance(database_field, Table): - list.extend(database_field.get_splash()) + pass # ignore virtual tables for now else: raise ValueError(f"Unknown field type {type(database_field).__name__} for splash") - return list + return splash class PersonsTable(Table): diff --git a/posthog/hogql/parser.py b/posthog/hogql/parser.py index 5b7f4b9eff6fc..b7be7d161938a 100644 --- a/posthog/hogql/parser.py +++ b/posthog/hogql/parser.py @@ -296,6 +296,9 @@ def visitColumnExprList(self, ctx: HogQLParser.ColumnExprListContext): return [self.visit(c) for c in ctx.columnsExpr()] def visitColumnsExprAsterisk(self, ctx: HogQLParser.ColumnsExprAsteriskContext): + if ctx.tableIdentifier(): + table = self.visit(ctx.tableIdentifier()) + return ast.Field(chain=table + ["*"]) return ast.Field(chain=["*"]) def visitColumnsExprSubquery(self, ctx: HogQLParser.ColumnsExprSubqueryContext): @@ -500,6 +503,9 @@ def visitColumnExprFunction(self, ctx: HogQLParser.ColumnExprFunctionContext): return ast.Call(name=name, args=args) def visitColumnExprAsterisk(self, ctx: HogQLParser.ColumnExprAsteriskContext): + if ctx.tableIdentifier(): + table = self.visit(ctx.tableIdentifier()) + return ast.Field(chain=table + ["*"]) return ast.Field(chain=["*"]) def visitColumnArgList(self, ctx: HogQLParser.ColumnArgListContext): diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index 5f27ff8ce3ba5..c0e20b4619e68 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -8,6 +8,7 @@ from posthog.hogql.database import Table, database from posthog.hogql.print_string import print_clickhouse_identifier, print_hogql_identifier from posthog.hogql.resolver import ResolverException, lookup_field_by_name, resolve_symbols +from posthog.hogql.transforms import expand_splashes from posthog.hogql.visitor import Visitor from posthog.models.property import PropertyName, TableColumn @@ -40,6 +41,8 @@ def print_ast( # modify the cloned tree as needed if dialect == "clickhouse": + expand_splashes(node) + # TODO: add team_id checks (currently done in the printer) # TODO: add joins to person and group tables pass @@ -475,7 +478,7 @@ def visit_splash_symbol(self, symbol: ast.SplashSymbol): prefix = ( f"{self._print_identifier(symbol.table.name)}." if isinstance(symbol.table, ast.TableAliasSymbol) else "" ) - return f"tuple({', '.join(f'{prefix}{self._print_identifier(field)}' for field in splash_fields)})" + return f"tuple({', '.join(f'{prefix}{self._print_identifier(field.name)}' for chain, field in splash_fields.items())})" def visit_unknown(self, node: ast.AST): raise ValueError(f"Unknown AST node {type(node).__name__}") diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py index e08e6215a64ed..965b149864d2c 100644 --- a/posthog/hogql/resolver.py +++ b/posthog/hogql/resolver.py @@ -165,7 +165,7 @@ def visit_field(self, node): if table_count == 0: raise ResolverException("Cannot use '*' when there are no tables in the query") if table_count > 1: - raise ResolverException("Cannot use '*' when there are multiple tables in the query") + raise ResolverException("Cannot use '*' without table name when there are multiple tables in the query") table = scope.anonymous_tables[0] if len(scope.anonymous_tables) > 0 else list(scope.tables.values())[0] symbol = ast.SplashSymbol(table=table) diff --git a/posthog/hogql/transforms.py b/posthog/hogql/transforms.py new file mode 100644 index 0000000000000..0cc05d1f8019e --- /dev/null +++ b/posthog/hogql/transforms.py @@ -0,0 +1,28 @@ +from typing import List + +from posthog.hogql import ast +from posthog.hogql.visitor import TraversingVisitor + + +def expand_splashes(node: ast.Expr): + SplashExpander().visit(node) + + +class SplashExpander(TraversingVisitor): + def visit_select_query(self, node: ast.SelectQuery): + columns: List[ast.Expr] = [] + for column in node.select: + if isinstance(column.symbol, ast.SplashSymbol): + splash = column.symbol + table = splash.table + while isinstance(table, ast.TableAliasSymbol): + table = table.table + if isinstance(table, ast.TableSymbol): + database_fields = table.table.get_splash() + for key in database_fields.keys(): + columns.append(ast.Field(chain=[key], symbol=ast.FieldSymbol(name=key, table=splash.table))) + else: + raise ValueError("Can't expand splash (*) on subquery") + else: + columns.append(column) + node.select = columns From 5cf6e77bd3f188d1baaed934ea44ec7595982751 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 16 Feb 2023 13:31:19 +0100 Subject: [PATCH 62/81] code quality issues --- frontend/src/queries/schema.json | 4 ++++ posthog/api/query.py | 8 ++++---- posthog/api/test/test_query.py | 2 +- posthog/schema.py | 1 + 4 files changed, 10 insertions(+), 5 deletions(-) diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json index 837732a4ce7ec..3c61bf014ab6d 100644 --- a/frontend/src/queries/schema.json +++ b/frontend/src/queries/schema.json @@ -1247,6 +1247,10 @@ "description": "Show the export button", "type": "boolean" }, + "showHogQLEditor": { + "description": "Include a HogQL query editor above HogQL tables", + "type": "boolean" + }, "showPropertyFilter": { "description": "Include a property filter above the table", "type": "boolean" diff --git a/posthog/api/query.py b/posthog/api/query.py index 289b1d4d366b2..80a2aa4c664cb 100644 --- a/posthog/api/query.py +++ b/posthog/api/query.py @@ -45,12 +45,12 @@ def _process_query(self, team: Team, query_json: Dict) -> JsonResponse: try: query_kind = query_json.get("kind") if query_kind == "EventsQuery": - query = EventsQuery.parse_obj(query_json) - response = run_events_query(query=query, team=team) + events_query = EventsQuery.parse_obj(query_json) + response = run_events_query(query=events_query, team=team) return self._response_to_json_response(response) elif query_kind == "HogQLQuery": - query = HogQLQuery.parse_obj(query_json) - response = execute_hogql_query(query=query.query, team=team) + hogql_query = HogQLQuery.parse_obj(query_json) + response = execute_hogql_query(query=hogql_query.query, team=team) return self._response_to_json_response(response) else: raise ValidationError("Unsupported query kind: %s" % query_kind) diff --git a/posthog/api/test/test_query.py b/posthog/api/test/test_query.py index 8ac341658f368..4b164743f5580 100644 --- a/posthog/api/test/test_query.py +++ b/posthog/api/test/test_query.py @@ -291,7 +291,7 @@ def test_full_hogql_query(self): api_response = self.client.post(f"/api/projects/{self.team.id}/query/", query.dict()).json() query.response = HogQLQueryResponse.parse_obj(api_response) - self.assertEqual(len(query.response.results), 4) + self.assertEqual(query.response.results and len(query.response.results), 4) self.assertEqual( query.response.results, [ diff --git a/posthog/schema.py b/posthog/schema.py index f6a89c5905cd7..72e9da3f90757 100644 --- a/posthog/schema.py +++ b/posthog/schema.py @@ -845,6 +845,7 @@ class Config: None, description="Show warning about live events being buffered max 60 sec (default: false)" ) showExport: Optional[bool] = Field(None, description="Show the export button") + showHogQLEditor: Optional[bool] = Field(None, description="Include a HogQL query editor above HogQL tables") showPropertyFilter: Optional[bool] = Field(None, description="Include a property filter above the table") showReload: Optional[bool] = Field(None, description="Show a reload button") showSavedQueries: Optional[bool] = Field(None, description="Shows a list of saved queries") From 44ff1a455ab929443e4c1472276b0241268fbc85 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Thu, 16 Feb 2023 13:37:23 +0100 Subject: [PATCH 63/81] error if unexpected splash --- posthog/hogql/printer.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index c0e20b4619e68..4a49c6a9496e0 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -469,16 +469,7 @@ def visit_field_alias_symbol(self, symbol: ast.SelectQueryAliasSymbol): return self._print_identifier(symbol.name) def visit_splash_symbol(self, symbol: ast.SplashSymbol): - table = symbol.table - while isinstance(table, ast.TableAliasSymbol): - table = table.table - if not isinstance(table, ast.TableSymbol): - raise ValueError(f"Unknown SplashSymbol table type: {type(table).__name__}") - splash_fields = table.table.get_splash() - prefix = ( - f"{self._print_identifier(symbol.table.name)}." if isinstance(symbol.table, ast.TableAliasSymbol) else "" - ) - return f"tuple({', '.join(f'{prefix}{self._print_identifier(field.name)}' for chain, field in splash_fields.items())})" + raise ValueError("Unexpected splash (*). It's only allowed in a SELECT column.") def visit_unknown(self, node: ast.AST): raise ValueError(f"Unknown AST node {type(node).__name__}") From 947e7bf8d235003570d66f238410d98df52f9088 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 20 Feb 2023 23:07:39 +0100 Subject: [PATCH 64/81] asterisk and obelisk --- posthog/hogql/ast.py | 2 +- posthog/hogql/database.py | 6 +++--- posthog/hogql/printer.py | 12 ++++++------ posthog/hogql/resolver.py | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index 2d55753b9d851..d47bd578e0261 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -118,7 +118,7 @@ class ConstantSymbol(Symbol): value: Any -class SplashSymbol(Symbol): +class AsteriskSymbol(Symbol): table: Union[TableSymbol, TableAliasSymbol, SelectQuerySymbol, SelectQueryAliasSymbol] diff --git a/posthog/hogql/database.py b/posthog/hogql/database.py index 7341258bb58bc..76b735e7578a9 100644 --- a/posthog/hogql/database.py +++ b/posthog/hogql/database.py @@ -48,16 +48,16 @@ def get_field(self, name: str) -> DatabaseField: def clickhouse_table(self): raise NotImplementedError("Table.clickhouse_table not overridden") - def get_splash(self) -> List[str]: + def get_asterisk(self) -> List[str]: list: List[str] = [] for field in self.__fields__.values(): database_field = field.default if isinstance(database_field, DatabaseField): list.append(database_field.name) elif isinstance(database_field, Table): - list.extend(database_field.get_splash()) + list.extend(database_field.get_asterisk()) else: - raise ValueError(f"Unknown field type {type(database_field).__name__} for splash") + raise ValueError(f"Unknown field type {type(database_field).__name__} for asterisk") return list diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index bce2d1a9b25b3..f4d9670cc80e9 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -351,10 +351,10 @@ def visit_field_symbol(self, symbol: ast.FieldSymbol): if isinstance(resolved_field, Table): # :KLUDGE: only works for events.person.* printing now if isinstance(symbol.table, ast.TableSymbol): - return self.visit(ast.SplashSymbol(table=ast.TableSymbol(table=resolved_field))) + return self.visit(ast.AsteriskSymbol(table=ast.TableSymbol(table=resolved_field))) else: return self.visit( - ast.SplashSymbol( + ast.AsteriskSymbol( table=ast.TableAliasSymbol( table=ast.TableSymbol(table=resolved_field), name=symbol.table.name ) @@ -467,17 +467,17 @@ def visit_select_query_alias_symbol(self, symbol: ast.SelectQueryAliasSymbol): def visit_field_alias_symbol(self, symbol: ast.SelectQueryAliasSymbol): return self._print_identifier(symbol.name) - def visit_splash_symbol(self, symbol: ast.SplashSymbol): + def visit_asterisk_symbol(self, symbol: ast.AsteriskSymbol): table = symbol.table while isinstance(table, ast.TableAliasSymbol): table = table.table if not isinstance(table, ast.TableSymbol): - raise ValueError(f"Unknown SplashSymbol table type: {type(table).__name__}") - splash_fields = table.table.get_splash() + raise ValueError(f"Unknown AsteriskSymbol table type: {type(table).__name__}") + asterisk_fields = table.table.get_asterisk() prefix = ( f"{self._print_identifier(symbol.table.name)}." if isinstance(symbol.table, ast.TableAliasSymbol) else "" ) - return f"tuple({', '.join(f'{prefix}{self._print_identifier(field)}' for field in splash_fields)})" + return f"tuple({', '.join(f'{prefix}{self._print_identifier(field)}' for field in asterisk_fields)})" def visit_unknown(self, node: ast.AST): raise ValueError(f"Unknown AST node {type(node).__name__}") diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py index e08e6215a64ed..341235000607e 100644 --- a/posthog/hogql/resolver.py +++ b/posthog/hogql/resolver.py @@ -167,7 +167,7 @@ def visit_field(self, node): if table_count > 1: raise ResolverException("Cannot use '*' when there are multiple tables in the query") table = scope.anonymous_tables[0] if len(scope.anonymous_tables) > 0 else list(scope.tables.values())[0] - symbol = ast.SplashSymbol(table=table) + symbol = ast.AsteriskSymbol(table=table) if not symbol: symbol = lookup_field_by_name(scope, name) From 74bf09cbfa631f990020d975cf7c81dfeb5311be Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 20 Feb 2023 23:08:26 +0100 Subject: [PATCH 65/81] yeet --- posthog/hogql/test/test_resolver.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/posthog/hogql/test/test_resolver.py b/posthog/hogql/test/test_resolver.py index 8e4fd16cf095a..9bc9e43e34181 100644 --- a/posthog/hogql/test/test_resolver.py +++ b/posthog/hogql/test/test_resolver.py @@ -234,17 +234,3 @@ def test_resolve_errors(self): with self.assertRaises(ResolverException) as e: resolve_symbols(parse_select(query)) self.assertIn("Unable to resolve field:", str(e.exception)) - - -# "with 2 as a select 1 as a" -> "Different expressions with the same alias a:" -# "with 2 as b, 3 as c select (select 1 as b) as a, b, c" -> "Different expressions with the same alias b:" -# "select a, b, e.c from (select 1 as a, 2 as b, 3 as c) as e" -> 1, 2, 3 - -# # good -# SELECT t.x FROM (SELECT 1 AS x) AS t; -# SELECT t.x FROM (SELECT x FROM tbl) AS t; -# SELECT x FROM (SELECT x FROM tbl) AS t; -# SELECT 1 AS x, x, x + 1; -# SELECT x, x + 1, 1 AS x; -# SELECT x, 1 + (2 + (3 AS x)); -# "SELECT x IN (SELECT 1 AS x) FROM (SELECT 1 AS x)", From d8eb09129b0371545a281edf382a0e2301fca4da Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 20 Feb 2023 23:09:57 +0100 Subject: [PATCH 66/81] class is for internal use only --- posthog/hogql/printer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index f4d9670cc80e9..189038c992540 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -46,7 +46,7 @@ def print_ast( # TODO: add joins to person and group tables pass - return Printer(context=context, dialect=dialect, stack=stack or []).visit(node) + return _Printer(context=context, dialect=dialect, stack=stack or []).visit(node) @dataclass @@ -55,7 +55,7 @@ class JoinExprResponse: where: Optional[ast.Expr] = None -class Printer(Visitor): +class _Printer(Visitor): # NOTE: Call "print_ast()", not this class directly. def __init__( From 9f1ae92fa92e75b1383e6f59cd0febbdcaa35fa5 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 20 Feb 2023 23:15:54 +0100 Subject: [PATCH 67/81] fix aliases --- posthog/hogql/printer.py | 5 ++++- posthog/hogql/test/test_printer.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index 189038c992540..ec7f03fe09aa4 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -329,7 +329,10 @@ def visit_placeholder(self, node: ast.Placeholder): raise ValueError(f"Found a Placeholder {{{node.field}}} in the tree. Can't generate query!") def visit_alias(self, node: ast.Alias): - return f"{self.visit(node.expr)} AS {self._print_identifier(node.alias)}" + inside = self.visit(node.expr) + if isinstance(node.expr, ast.Alias): + inside = f"({inside})" + return f"{inside} AS {self._print_identifier(node.alias)}" def visit_table_symbol(self, symbol: ast.TableSymbol): return self._print_identifier(symbol.table.clickhouse_table()) diff --git a/posthog/hogql/test/test_printer.py b/posthog/hogql/test/test_printer.py index 6a8ec04c86290..46576e453e640 100644 --- a/posthog/hogql/test/test_printer.py +++ b/posthog/hogql/test/test_printer.py @@ -224,8 +224,6 @@ def test_expr_syntax_errors(self): "select query from events", "line 1, column 13: mismatched input 'from' expecting " ) self._assert_expr_error("this makes little sense", "Unable to resolve field: this") - # TODO: fix - # self._assert_expr_error("event makes little sense", "event AS makes AS little AS sense") self._assert_expr_error("1;2", "line 1, column 1: mismatched input ';' expecting") self._assert_expr_error("b.a(bla)", "SyntaxError: line 1, column 3: mismatched input '(' expecting '.'") @@ -360,6 +358,8 @@ def test_alias_keywords(self): self._select("select 1 as `-- select team_id` from events"), "SELECT 1 AS `-- select team_id` FROM events WHERE equals(team_id, 42) LIMIT 65535", ) + # Some aliases are funny, but that's what the antlr syntax permits, and ClickHouse doesn't complain either + self.assertEqual(self._expr("event makes little sense"), "((event AS makes) AS little) AS sense") def test_select(self): self.assertEqual(self._select("select 1"), "SELECT 1 LIMIT 65535") From c767159eeaedff1bf7e9ae82efe1d926b48c5edd Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 20 Feb 2023 23:16:11 +0100 Subject: [PATCH 68/81] not needed --- posthog/hogql/ast.py | 1 - 1 file changed, 1 deletion(-) diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index d47bd578e0261..6c265dfeb4b45 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -105,7 +105,6 @@ def has_child(self, name: str) -> bool: return self.symbol.has_child(name) -SelectQuerySymbol.update_forward_refs(SelectQuerySymbol=SelectQuerySymbol) SelectQuerySymbol.update_forward_refs(SelectQueryAliasSymbol=SelectQueryAliasSymbol) From e496907e05579770354037fb172bc03fb3fa294a Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 20 Feb 2023 23:21:29 +0100 Subject: [PATCH 69/81] fix a few fields --- posthog/hogql/database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/posthog/hogql/database.py b/posthog/hogql/database.py index 76b735e7578a9..186241ef621cf 100644 --- a/posthog/hogql/database.py +++ b/posthog/hogql/database.py @@ -117,10 +117,10 @@ class SessionRecordingEvents(Table): distinct_id: StringDatabaseField = StringDatabaseField(name="distinct_id") session_id: StringDatabaseField = StringDatabaseField(name="session_id") window_id: StringDatabaseField = StringDatabaseField(name="window_id") - snapshot_data: StringDatabaseField = StringDatabaseField(name="snapshot_data") + snapshot_data: StringJSONDatabaseField = StringJSONDatabaseField(name="snapshot_data") created_at: DateTimeDatabaseField = DateTimeDatabaseField(name="created_at") has_full_snapshot: BooleanDatabaseField = BooleanDatabaseField(name="has_full_snapshot") - events_summary: BooleanDatabaseField = BooleanDatabaseField(name="events_summary", array=True) + events_summary: StringJSONDatabaseField = StringJSONDatabaseField(name="events_summary", array=True) click_count: IntegerDatabaseField = IntegerDatabaseField(name="click_count") keypress_count: IntegerDatabaseField = IntegerDatabaseField(name="keypress_count") timestamps_summary: DateTimeDatabaseField = DateTimeDatabaseField(name="timestamps_summary", array=True) From 4e3830c3a201337aec041ce3528fa8820559b850 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 20 Feb 2023 23:56:11 +0100 Subject: [PATCH 70/81] pretty printing --- .../nodes/HogQLQuery/HogQLQueryEditor.tsx | 4 +- .../nodes/HogQLQuery/hogQLQueryEditorLogic.ts | 19 ++++++-- package.json | 1 + pnpm-lock.yaml | 45 ++++++++++++++++++- 4 files changed, 62 insertions(+), 7 deletions(-) diff --git a/frontend/src/queries/nodes/HogQLQuery/HogQLQueryEditor.tsx b/frontend/src/queries/nodes/HogQLQuery/HogQLQueryEditor.tsx index 2ee10bc896419..0d38c3b4beeab 100644 --- a/frontend/src/queries/nodes/HogQLQuery/HogQLQueryEditor.tsx +++ b/frontend/src/queries/nodes/HogQLQuery/HogQLQueryEditor.tsx @@ -19,14 +19,14 @@ export function HogQLQueryEditor(props: HogQLQueryEditorProps): JSX.Element { const { setQueryInput, saveQuery } = useActions(hogQLQueryEditorLogic(hogQLQueryEditorLogicProps)) return ( -
+
{({ height }) => ( setQueryInput(v ?? '')} height={height} diff --git a/frontend/src/queries/nodes/HogQLQuery/hogQLQueryEditorLogic.ts b/frontend/src/queries/nodes/HogQLQuery/hogQLQueryEditorLogic.ts index 474926ec9f1d3..566c2dfaa51fb 100644 --- a/frontend/src/queries/nodes/HogQLQuery/hogQLQueryEditorLogic.ts +++ b/frontend/src/queries/nodes/HogQLQuery/hogQLQueryEditorLogic.ts @@ -1,8 +1,18 @@ import { actions, kea, key, listeners, path, props, propsChanged, reducers } from 'kea' +import { format } from 'sql-formatter' import { HogQLQuery } from '~/queries/schema' import type { hogQLQueryEditorLogicType } from './hogQLQueryEditorLogicType' +function formatSQL(sql: string): string { + return format(sql, { + language: 'mysql', + tabWidth: 2, + keywordCase: 'preserve', + linesBetweenQueries: 2, + indentStyle: 'tabularRight', + }) +} export interface HogQLQueryEditorLogicProps { key: number query: HogQLQuery @@ -15,7 +25,7 @@ export const hogQLQueryEditorLogic = kea([ key((props) => props.key), propsChanged(({ actions, props }, oldProps) => { if (props.query.query !== oldProps.query.query) { - actions.setQueryInput(props.query.query) + actions.setQueryInput(formatSQL(props.query.query)) } }), actions({ @@ -23,12 +33,13 @@ export const hogQLQueryEditorLogic = kea([ setQueryInput: (queryInput: string) => ({ queryInput }), }), reducers(({ props }) => ({ - queryInput: [props.query.query, { setQueryInput: (_, { queryInput }) => queryInput }], + queryInput: [formatSQL(props.query.query), { setQueryInput: (_, { queryInput }) => queryInput }], })), listeners(({ actions, props, values }) => ({ saveQuery: () => { - actions.setQueryInput(values.queryInput) - props.setQuery?.({ ...props.query, query: values.queryInput }) + const formattedQuery = formatSQL(values.queryInput) + actions.setQueryInput(formattedQuery) + props.setQuery?.({ ...props.query, query: formattedQuery }) }, })), ]) diff --git a/package.json b/package.json index 9fde08be2475f..e680edc16a0ab 100644 --- a/package.json +++ b/package.json @@ -139,6 +139,7 @@ "resize-observer-polyfill": "^1.5.1", "rrweb": "^1.1.3", "sass": "^1.26.2", + "sql-formatter": "^12.1.2", "use-debounce": "^6.0.1", "use-resize-observer": "^8.0.0", "wildcard-match": "^5.1.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 331ac49dac7d4..75d70327432c1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -173,6 +173,7 @@ specifiers: rrweb: ^1.1.3 sass: ^1.26.2 sass-loader: ^10.0.1 + sql-formatter: ^12.1.2 storybook-addon-pseudo-states: ^1.15.1 style-loader: ^2.0.0 sucrase: ^3.29.0 @@ -269,6 +270,7 @@ dependencies: resize-observer-polyfill: 1.5.1 rrweb: 1.1.3 sass: 1.56.0 + sql-formatter: 12.1.2 use-debounce: 6.0.1_react@16.14.0 use-resize-observer: 8.0.0_wcqkhtmu7mswc6yz4uyexck3ty wildcard-match: 5.1.2 @@ -5643,6 +5645,10 @@ packages: sprintf-js: 1.0.3 dev: true + /argparse/2.0.1: + resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + dev: false + /aria-hidden/1.2.1_edij4neeagymnxmr7qklvezyj4: resolution: {integrity: sha512-PN344VAf9j1EAi+jyVHOJ8XidQdPVssGco39eNcsGdM4wcsILtxrKLkbuiMfLWYROK1FjRQasMWCBttrhjnr6A==} engines: {node: '>=10'} @@ -8190,6 +8196,10 @@ packages: path-type: 4.0.0 dev: true + /discontinuous-range/1.0.0: + resolution: {integrity: sha512-c68LpLbO+7kP/b1Hr1qs8/BJ09F5khZGTxqxZuhzxpmwJKOgRFHJWIb9/KmqnqHhLdO55aOxFH/EGBvUQbL/RQ==} + dev: false + /doctrine/2.1.0: resolution: {integrity: sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==} engines: {node: '>=0.10.0'} @@ -13283,6 +13293,10 @@ packages: color-name: 1.1.4 dev: true + /moo/0.5.2: + resolution: {integrity: sha512-iSAJLHYKnX41mKcJKjqvnAN9sf0LMDTXDEvFv+ffuRR9a1MIuXLjMNL6EsnDHSkKLTWNqQQ5uo61P4EbU4NU+Q==} + dev: false + /move-concurrently/1.0.1: resolution: {integrity: sha512-hdrFxZOycD/g6A6SoI2bB5NA/5NEqD0569+S47WZhPvm46sD50ZHdYaFmnua5lndde9rCHGjmfK7Z8BuCt/PcQ==} dependencies: @@ -13415,6 +13429,16 @@ packages: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} dev: true + /nearley/2.20.1: + resolution: {integrity: sha512-+Mc8UaAebFzgV+KpI5n7DasuuQCHA89dmwm7JXw3TV43ukfNQ9DnBH3Mdb2g/I4Fdxc26pwimBWvjIw0UAILSQ==} + hasBin: true + dependencies: + commander: 2.20.3 + moo: 0.5.2 + railroad-diagrams: 1.0.0 + randexp: 0.4.6 + dev: false + /needle/3.2.0: resolution: {integrity: sha512-oUvzXnyLiVyVGoianLijF9O/RecZUf7TkBfimjGrLM4eQhXyeJwM6GeAWccwfQ9aa4gMCZKqhAOuLaMIcQxajQ==} engines: {node: '>= 4.4.x'} @@ -14899,10 +14923,22 @@ packages: engines: {node: '>=10'} dev: false + /railroad-diagrams/1.0.0: + resolution: {integrity: sha512-cz93DjNeLY0idrCNOH6PviZGRN9GJhsdm9hpn1YCS879fj4W+x5IFJhhkRZcwVgMmFF7R82UA/7Oh+R8lLZg6A==} + dev: false + /ramda/0.28.0: resolution: {integrity: sha512-9QnLuG/kPVgWvMQ4aODhsBUFKOUmnbUnsSXACv+NCQZcHbeb+v8Lodp8OVxtRULN1/xOyYLLaL6npE6dMq5QTA==} dev: true + /randexp/0.4.6: + resolution: {integrity: sha512-80WNmd9DA0tmZrw9qQa62GPPWfuXJknrmVmLcxvq4uZBdYqb1wYoKTmnlGUchvVWe0XiLupYkBoXVOxz3C8DYQ==} + engines: {node: '>=0.12'} + dependencies: + discontinuous-range: 1.0.0 + ret: 0.1.15 + dev: false + /randombytes/2.1.0: resolution: {integrity: sha512-vYl3iOX+4CKUWuxGi9Ukhie6fsqXqS9FE2Zaic4tNFD2N2QQaXOMFbuKK4QmDHC0JO6B1Zp41J0LpT0oR68amQ==} dependencies: @@ -16240,7 +16276,6 @@ packages: /ret/0.1.15: resolution: {integrity: sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg==} engines: {node: '>=0.12'} - dev: true /reusify/1.0.4: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} @@ -16844,6 +16879,14 @@ packages: resolution: {integrity: sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==} dev: true + /sql-formatter/12.1.2: + resolution: {integrity: sha512-SoFn+9ZflUt8+HYZ/PaifXt1RptcDUn8HXqsWmfXdPV3WeHPgT0qOSJXxHU24d7NOVt9X40MLqf263fNk79XqA==} + hasBin: true + dependencies: + argparse: 2.0.1 + nearley: 2.20.1 + dev: false + /sshpk/1.17.0: resolution: {integrity: sha512-/9HIEs1ZXGhSPE8X6Ccm7Nam1z8KcoCqPdI7ecm1N33EzAetWahvQWVqLZtaZQ+IDKX4IyA2o0gBzqIMkAagHQ==} engines: {node: '>=0.10.0'} From b25fd20ff42ebf32e2f99a11c040d2837c6f05a6 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Mon, 20 Feb 2023 23:59:19 +0100 Subject: [PATCH 71/81] update sample query --- frontend/src/queries/examples.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/frontend/src/queries/examples.ts b/frontend/src/queries/examples.ts index 91077af07054d..5e1847149c1df 100644 --- a/frontend/src/queries/examples.ts +++ b/frontend/src/queries/examples.ts @@ -293,7 +293,14 @@ const TimeToSeeDataWaterfall: TimeToSeeDataWaterfallNode = { const HogQL: HogQLQuery = { kind: NodeKind.HogQLQuery, - query: 'select event, count() as event_count from events group by event order by event_count desc', + query: + ' select event,\n' + + ' properties.$geoip_country_name as `Country Name`,\n' + + ' count() as `Event count`\n' + + ' from events\n' + + ' group by event,\n' + + ' properties.$geoip_country_name\n' + + ' order by count() desc', } const HogQLTable: DataTableNode = { From bf29398dd5e5d58d1878712df65b7e58d8af80a5 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 20 Feb 2023 23:04:31 +0000 Subject: [PATCH 72/81] Update snapshots --- posthog/api/test/__snapshots__/test_feature_flag.ambr | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/posthog/api/test/__snapshots__/test_feature_flag.ambr b/posthog/api/test/__snapshots__/test_feature_flag.ambr index df1466bd75a1c..4669865ffa923 100644 --- a/posthog/api/test/__snapshots__/test_feature_flag.ambr +++ b/posthog/api/test/__snapshots__/test_feature_flag.ambr @@ -405,7 +405,9 @@ ' SELECT pg_sleep(1); - SELECT ("posthog_person"."properties" -> 'email') = '"tim@posthog.com"' AS "flag_X_condition_0", + SELECT (("posthog_person"."properties" -> 'email') = '"tim@posthog.com"' + AND "posthog_person"."properties" ? 'email' + AND NOT (("posthog_person"."properties" -> 'email') = 'null')) AS "flag_X_condition_0", (true) AS "flag_X_condition_0" FROM "posthog_person" INNER JOIN "posthog_persondistinctid" ON ("posthog_person"."id" = "posthog_persondistinctid"."person_id") @@ -443,7 +445,9 @@ --- # name: TestResiliency.test_feature_flags_v3_with_experience_continuity_working_slow_db.3 ' - SELECT ("posthog_person"."properties" -> 'email') = '"tim@posthog.com"' AS "flag_X_condition_0", + SELECT (("posthog_person"."properties" -> 'email') = '"tim@posthog.com"' + AND "posthog_person"."properties" ? 'email' + AND NOT (("posthog_person"."properties" -> 'email') = 'null')) AS "flag_X_condition_0", (true) AS "flag_X_condition_0" FROM "posthog_person" INNER JOIN "posthog_persondistinctid" ON ("posthog_person"."id" = "posthog_persondistinctid"."person_id") From f21c9288ba66bba7dcdbda6eb3004e5a5eff5f0f Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 21 Feb 2023 00:10:03 +0100 Subject: [PATCH 73/81] update sample query --- frontend/src/queries/examples.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/queries/examples.ts b/frontend/src/queries/examples.ts index 5e1847149c1df..c53e9ea62a590 100644 --- a/frontend/src/queries/examples.ts +++ b/frontend/src/queries/examples.ts @@ -300,7 +300,8 @@ const HogQL: HogQLQuery = { ' from events\n' + ' group by event,\n' + ' properties.$geoip_country_name\n' + - ' order by count() desc', + ' order by count() desc\n' + + ' limit 100', } const HogQLTable: DataTableNode = { From b35777f81b7810a7085f90d673ccc76645784bed Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 20 Feb 2023 23:11:45 +0000 Subject: [PATCH 74/81] Update snapshots --- posthog/api/test/__snapshots__/test_decide.ambr | 8 ++++++-- posthog/api/test/__snapshots__/test_insight.ambr | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/posthog/api/test/__snapshots__/test_decide.ambr b/posthog/api/test/__snapshots__/test_decide.ambr index 8d2a98767cd22..a1d88aebb96d2 100644 --- a/posthog/api/test/__snapshots__/test_decide.ambr +++ b/posthog/api/test/__snapshots__/test_decide.ambr @@ -589,7 +589,9 @@ --- # name: TestDecide.test_flag_with_regular_cohorts.5 ' - SELECT ("posthog_person"."properties" -> '$some_prop_1') = '"something_1"' AS "flag_X_condition_0" + SELECT (("posthog_person"."properties" -> '$some_prop_1') = '"something_1"' + AND "posthog_person"."properties" ? '$some_prop_1' + AND NOT (("posthog_person"."properties" -> '$some_prop_1') = 'null')) AS "flag_X_condition_0" FROM "posthog_person" INNER JOIN "posthog_persondistinctid" ON ("posthog_person"."id" = "posthog_persondistinctid"."person_id") WHERE ("posthog_persondistinctid"."distinct_id" = 'example_id_1' @@ -666,7 +668,9 @@ --- # name: TestDecide.test_flag_with_regular_cohorts.8 ' - SELECT ("posthog_person"."properties" -> '$some_prop_1') = '"something_1"' AS "flag_X_condition_0" + SELECT (("posthog_person"."properties" -> '$some_prop_1') = '"something_1"' + AND "posthog_person"."properties" ? '$some_prop_1' + AND NOT (("posthog_person"."properties" -> '$some_prop_1') = 'null')) AS "flag_X_condition_0" FROM "posthog_person" INNER JOIN "posthog_persondistinctid" ON ("posthog_person"."id" = "posthog_persondistinctid"."person_id") WHERE ("posthog_persondistinctid"."distinct_id" = 'another_id' diff --git a/posthog/api/test/__snapshots__/test_insight.ambr b/posthog/api/test/__snapshots__/test_insight.ambr index f6a1a0c197b7d..7d0567823c5a4 100644 --- a/posthog/api/test/__snapshots__/test_insight.ambr +++ b/posthog/api/test/__snapshots__/test_insight.ambr @@ -790,7 +790,7 @@ "posthog_insightcachingstate"."created_at", "posthog_insightcachingstate"."updated_at" FROM "posthog_insightcachingstate" - WHERE ("posthog_insightcachingstate"."cache_key" = 'cache_8927b6bdeed5f8af7eeba3673259bf59' + WHERE ("posthog_insightcachingstate"."cache_key" = 'cache_2dfd15f76fb6ac02f0972f7b0203a789' AND "posthog_insightcachingstate"."insight_id" = 2 AND "posthog_insightcachingstate"."team_id" = 2) /*controller='project_insights-list',route='api/projects/%28%3FP%3Cparent_lookup_team_id%3E%5B%5E/.%5D%2B%29/insights/%3F%24'*/ ' From 7271431fbe16529fc9752af52dda05dc55efd817 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 21 Feb 2023 09:57:49 +0100 Subject: [PATCH 75/81] fix webpack build for storybook --- webpack.config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webpack.config.js b/webpack.config.js index 33228a392c92f..dc958a503b5c2 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -65,7 +65,7 @@ function createEntry(entry) { rules: [ { test: /\.[jt]sx?$/, - exclude: /(node_modules)/, + exclude: /node_modules(?!(\/\.pnpm|)(\/sql-formatter))/, use: { loader: 'babel-loader', }, From 55ba79e8ce9f8b88af6a2477ef52d77709832714 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 21 Feb 2023 10:42:46 +0100 Subject: [PATCH 76/81] asterisks work again --- posthog/hogql/ast.py | 8 +++++--- posthog/hogql/constants.py | 3 --- posthog/hogql/printer.py | 2 +- posthog/hogql/test/test_printer.py | 11 ---------- posthog/hogql/test/test_transforms.py | 6 ++++++ posthog/models/event/query_event_list.py | 26 ++---------------------- 6 files changed, 14 insertions(+), 42 deletions(-) create mode 100644 posthog/hogql/test/test_transforms.py diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index 6ed0906143a09..c0b6a75a1bf0d 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -58,9 +58,9 @@ def get_child(self, name: str) -> Symbol: if self.has_child(name): field = self.table.get_field(name) if isinstance(field, Table): - return AsteriskSymbol(table=TableSymbol(table=field)) + return TableSymbol(table=field) return FieldSymbol(name=name, table=self) - raise ValueError(f"Field not found: {name}") + raise ValueError(f'Field "{name}" not found on table {type(self.table).__name__}') class TableAliasSymbol(Symbol): @@ -71,6 +71,8 @@ def has_child(self, name: str) -> bool: return self.table.has_child(name) def get_child(self, name: str) -> Symbol: + if name == "*": + return AsteriskSymbol(table=self) if self.has_child(name): return FieldSymbol(name=name, table=self) return self.table.get_child(name) @@ -104,7 +106,7 @@ class SelectQueryAliasSymbol(Symbol): def get_child(self, name: str) -> Symbol: if self.symbol.has_child(name): return FieldSymbol(name=name, table=self) - raise ValueError(f"Field not found: {name}") + raise ValueError(f"Field {name} not found on query with alias {self.name}") def has_child(self, name: str) -> bool: return self.symbol.has_child(name) diff --git a/posthog/hogql/constants.py b/posthog/hogql/constants.py index 92c88a8c65df4..27e180bec6574 100644 --- a/posthog/hogql/constants.py +++ b/posthog/hogql/constants.py @@ -110,9 +110,6 @@ "distinct_id", "elements_chain", "created_at", - "person.id", - "person.created_at", - "person.properties", ] # Never return more rows than this in top level HogQL SELECT statements diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index 28a123bd56015..d49e5798cd98e 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -472,7 +472,7 @@ def visit_field_alias_symbol(self, symbol: ast.SelectQueryAliasSymbol): return self._print_identifier(symbol.name) def visit_asterisk_symbol(self, symbol: ast.AsteriskSymbol): - raise ValueError("Unexpected asterisk (*). It's only allowed in a SELECT column.") + raise ValueError("Unexpected ast.AsteriskSymbol. Make sure AsteriskExpander has run on the AST.") def visit_unknown(self, node: ast.AST): raise ValueError(f"Unknown AST node {type(node).__name__}") diff --git a/posthog/hogql/test/test_printer.py b/posthog/hogql/test/test_printer.py index 46576e453e640..e4ce7eba73019 100644 --- a/posthog/hogql/test/test_printer.py +++ b/posthog/hogql/test/test_printer.py @@ -329,17 +329,6 @@ def test_comments(self): context = HogQLContext() self.assertEqual(self._expr("event -- something", context), "event") - def test_special_root_properties(self): - self.assertEqual( - self._expr("*"), - "tuple(uuid, event, properties, timestamp, team_id, distinct_id, elements_chain, created_at, person_id, person_created_at, person_properties)", - ) - context = HogQLContext() - self.assertEqual( - self._expr("person", context), - "tuple(person_id, person_created_at, person_properties)", - ) - def test_values(self): context = HogQLContext() self.assertEqual(self._expr("event == 'E'", context), "equals(event, %(hogql_val_0)s)") diff --git a/posthog/hogql/test/test_transforms.py b/posthog/hogql/test/test_transforms.py new file mode 100644 index 0000000000000..be11795007137 --- /dev/null +++ b/posthog/hogql/test/test_transforms.py @@ -0,0 +1,6 @@ +from posthog.test.base import BaseTest + + +class TestTransforms(BaseTest): + def test_asterisk_expander(self): + pass diff --git a/posthog/models/event/query_event_list.py b/posthog/models/event/query_event_list.py index 878bf00e4ed32..2a4c8f5f5b483 100644 --- a/posthog/models/event/query_event_list.py +++ b/posthog/models/event/query_event_list.py @@ -210,6 +210,8 @@ def run_events_query( for expr in select: hogql_context.found_aggregation = False + if expr == "*": + expr = f'tuple({", ".join(SELECT_STAR_FROM_EVENTS_FIELDS)})' clickhouse_sql = translate_hogql(expr, hogql_context) select_columns.append(clickhouse_sql) if not hogql_context.found_aggregation: @@ -273,13 +275,6 @@ def run_events_query( results[index] = list(result) results[index][star] = convert_star_select_to_dict(result[star]) - # Convert person field from tuple to dict in each result - if "person" in select: - person = select.index("person") - for index, result in enumerate(results): - results[index] = list(result) - results[index][person] = convert_person_select_to_dict(result[person]) - received_extra_row = len(results) == limit # limit was +=1'd above return EventsQueryResponse( @@ -293,23 +288,6 @@ def run_events_query( def convert_star_select_to_dict(select: Tuple[Any]) -> Dict[str, Any]: new_result = dict(zip(SELECT_STAR_FROM_EVENTS_FIELDS, select)) new_result["properties"] = json.loads(new_result["properties"]) - new_result["person"] = { - "id": new_result["person.id"], - "created_at": new_result["person.created_at"], - "properties": json.loads(new_result["person.properties"]), - } - new_result.pop("person.id") - new_result.pop("person.created_at") - new_result.pop("person.properties") if new_result["elements_chain"]: new_result["elements"] = ElementSerializer(chain_to_elements(new_result["elements_chain"]), many=True).data return new_result - - -def convert_person_select_to_dict(select: Tuple[str, str, str, str, str]) -> Dict[str, Any]: - return { - "id": select[1], - "created_at": select[2], - "properties": {"name": select[3], "email": select[4]}, - "distinct_ids": [select[0]], - } From f73998042d36ae5f45a9ee0e3de8f3d8eab60d61 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 21 Feb 2023 09:52:44 +0000 Subject: [PATCH 77/81] Update snapshots --- posthog/api/test/__snapshots__/test_query.ambr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/posthog/api/test/__snapshots__/test_query.ambr b/posthog/api/test/__snapshots__/test_query.ambr index 158a059e51a88..34702611c2ded 100644 --- a/posthog/api/test/__snapshots__/test_query.ambr +++ b/posthog/api/test/__snapshots__/test_query.ambr @@ -348,12 +348,12 @@ # name: TestQuery.test_select_hogql_expressions.1 ' /* user_id:0 request:_snapshot_ */ - SELECT tuple(uuid, event, properties, timestamp, team_id, distinct_id, elements_chain, created_at, person_id, person_created_at, person_properties), + SELECT tuple(uuid, event, properties, timestamp, team_id, distinct_id, elements_chain, created_at), event FROM events WHERE team_id = 2 AND timestamp < '2020-01-10 12:14:05.000000' - ORDER BY tuple(uuid, event, properties, timestamp, team_id, distinct_id, elements_chain, created_at, person_id, person_created_at, person_properties) ASC + ORDER BY tuple(uuid, event, properties, timestamp, team_id, distinct_id, elements_chain, created_at) ASC LIMIT 101 ' --- From 02468b7bd2778166e701c8849c5f8d58eedafff1 Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 21 Feb 2023 11:11:23 +0100 Subject: [PATCH 78/81] test splotch desplotcher --- posthog/hogql/ast.py | 4 + posthog/hogql/test/test_transforms.py | 117 +++++++++++++++++++++++++- posthog/hogql/transforms.py | 35 ++++++-- 3 files changed, 146 insertions(+), 10 deletions(-) diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py index c0b6a75a1bf0d..45c8c71ebe6d4 100644 --- a/posthog/hogql/ast.py +++ b/posthog/hogql/ast.py @@ -91,6 +91,8 @@ class SelectQuerySymbol(Symbol): anonymous_tables: List["SelectQuerySymbol"] = PydanticField(default_factory=list) def get_child(self, name: str) -> Symbol: + if name == "*": + return AsteriskSymbol(table=self) if name in self.columns: return FieldSymbol(name=name, table=self) raise ValueError(f"Column not found: {name}") @@ -104,6 +106,8 @@ class SelectQueryAliasSymbol(Symbol): symbol: SelectQuerySymbol def get_child(self, name: str) -> Symbol: + if name == "*": + return AsteriskSymbol(table=self) if self.symbol.has_child(name): return FieldSymbol(name=name, table=self) raise ValueError(f"Field {name} not found on query with alias {self.name}") diff --git a/posthog/hogql/test/test_transforms.py b/posthog/hogql/test/test_transforms.py index be11795007137..b32353eba1a37 100644 --- a/posthog/hogql/test/test_transforms.py +++ b/posthog/hogql/test/test_transforms.py @@ -1,6 +1,119 @@ +from posthog.hogql import ast +from posthog.hogql.database import database +from posthog.hogql.parser import parse_select +from posthog.hogql.resolver import ResolverException, resolve_symbols +from posthog.hogql.transforms import expand_asterisks from posthog.test.base import BaseTest class TestTransforms(BaseTest): - def test_asterisk_expander(self): - pass + def test_asterisk_expander_table(self): + node = parse_select("select * from events") + resolve_symbols(node) + expand_asterisks(node) + events_table_symbol = ast.TableSymbol(table=database.events) + self.assertEqual( + node.select, + [ + ast.Field(chain=["uuid"], symbol=ast.FieldSymbol(name="uuid", table=events_table_symbol)), + ast.Field(chain=["event"], symbol=ast.FieldSymbol(name="event", table=events_table_symbol)), + ast.Field(chain=["properties"], symbol=ast.FieldSymbol(name="properties", table=events_table_symbol)), + ast.Field(chain=["timestamp"], symbol=ast.FieldSymbol(name="timestamp", table=events_table_symbol)), + ast.Field(chain=["team_id"], symbol=ast.FieldSymbol(name="team_id", table=events_table_symbol)), + ast.Field(chain=["distinct_id"], symbol=ast.FieldSymbol(name="distinct_id", table=events_table_symbol)), + ast.Field( + chain=["elements_chain"], symbol=ast.FieldSymbol(name="elements_chain", table=events_table_symbol) + ), + ast.Field(chain=["created_at"], symbol=ast.FieldSymbol(name="created_at", table=events_table_symbol)), + ], + ) + + def test_asterisk_expander_table_alias(self): + node = parse_select("select * from events e") + resolve_symbols(node) + expand_asterisks(node) + events_table_symbol = ast.TableSymbol(table=database.events) + events_table_alias_symbol = ast.TableAliasSymbol(table=events_table_symbol, name="e") + self.assertEqual( + node.select, + [ + ast.Field(chain=["uuid"], symbol=ast.FieldSymbol(name="uuid", table=events_table_alias_symbol)), + ast.Field(chain=["event"], symbol=ast.FieldSymbol(name="event", table=events_table_alias_symbol)), + ast.Field( + chain=["properties"], symbol=ast.FieldSymbol(name="properties", table=events_table_alias_symbol) + ), + ast.Field( + chain=["timestamp"], symbol=ast.FieldSymbol(name="timestamp", table=events_table_alias_symbol) + ), + ast.Field(chain=["team_id"], symbol=ast.FieldSymbol(name="team_id", table=events_table_alias_symbol)), + ast.Field( + chain=["distinct_id"], symbol=ast.FieldSymbol(name="distinct_id", table=events_table_alias_symbol) + ), + ast.Field( + chain=["elements_chain"], + symbol=ast.FieldSymbol(name="elements_chain", table=events_table_alias_symbol), + ), + ast.Field( + chain=["created_at"], symbol=ast.FieldSymbol(name="created_at", table=events_table_alias_symbol) + ), + ], + ) + + def test_asterisk_expander_subquery(self): + node = parse_select("select * from (select 1 as a, 2 as b)") + resolve_symbols(node) + expand_asterisks(node) + select_subquery_symbol = ast.SelectQuerySymbol( + aliases={ + "a": ast.FieldAliasSymbol(name="a", symbol=ast.ConstantSymbol(value=1)), + "b": ast.FieldAliasSymbol(name="b", symbol=ast.ConstantSymbol(value=2)), + }, + columns={ + "a": ast.FieldAliasSymbol(name="a", symbol=ast.ConstantSymbol(value=1)), + "b": ast.FieldAliasSymbol(name="b", symbol=ast.ConstantSymbol(value=2)), + }, + tables={}, + anonymous_tables=[], + ) + self.assertEqual( + node.select, + [ + ast.Field(chain=["a"], symbol=ast.FieldSymbol(name="a", table=select_subquery_symbol)), + ast.Field(chain=["b"], symbol=ast.FieldSymbol(name="b", table=select_subquery_symbol)), + ], + ) + + def test_asterisk_expander_subquery_alias(self): + node = parse_select("select x.* from (select 1 as a, 2 as b) x") + resolve_symbols(node) + expand_asterisks(node) + select_subquery_symbol = ast.SelectQueryAliasSymbol( + name="x", + symbol=ast.SelectQuerySymbol( + aliases={ + "a": ast.FieldAliasSymbol(name="a", symbol=ast.ConstantSymbol(value=1)), + "b": ast.FieldAliasSymbol(name="b", symbol=ast.ConstantSymbol(value=2)), + }, + columns={ + "a": ast.FieldAliasSymbol(name="a", symbol=ast.ConstantSymbol(value=1)), + "b": ast.FieldAliasSymbol(name="b", symbol=ast.ConstantSymbol(value=2)), + }, + tables={}, + anonymous_tables=[], + ), + ) + self.assertEqual( + node.select, + [ + ast.Field(chain=["a"], symbol=ast.FieldSymbol(name="a", table=select_subquery_symbol)), + ast.Field(chain=["b"], symbol=ast.FieldSymbol(name="b", table=select_subquery_symbol)), + ], + ) + + def test_asterisk_expander_multiple_table_error(self): + node = parse_select("select * from (select 1 as a, 2 as b) x left join (select 1 as a, 2 as b) y on x.a = y.a") + with self.assertRaises(ResolverException) as e: + resolve_symbols(node) + self.assertEqual( + str(e.exception), "Cannot use '*' without table name when there are multiple tables in the query" + ) diff --git a/posthog/hogql/transforms.py b/posthog/hogql/transforms.py index b7a30329407b4..b14555b850ba9 100644 --- a/posthog/hogql/transforms.py +++ b/posthog/hogql/transforms.py @@ -14,15 +14,34 @@ def visit_select_query(self, node: ast.SelectQuery): for column in node.select: if isinstance(column.symbol, ast.AsteriskSymbol): asterisk = column.symbol - table = asterisk.table - while isinstance(table, ast.TableAliasSymbol): - table = table.table - if isinstance(table, ast.TableSymbol): - database_fields = table.table.get_asterisk() - for key in database_fields.keys(): - columns.append(ast.Field(chain=[key], symbol=ast.FieldSymbol(name=key, table=asterisk.table))) + if isinstance(asterisk.table, ast.TableSymbol) or isinstance(asterisk.table, ast.TableAliasSymbol): + table = asterisk.table + while isinstance(table, ast.TableAliasSymbol): + table = table.table + if isinstance(table, ast.TableSymbol): + database_fields = table.table.get_asterisk() + for key in database_fields.keys(): + columns.append( + ast.Field(chain=[key], symbol=ast.FieldSymbol(name=key, table=asterisk.table)) + ) + else: + raise ValueError("Can't expand asterisk (*) on table") + elif isinstance(asterisk.table, ast.SelectQuerySymbol) or isinstance( + asterisk.table, ast.SelectQueryAliasSymbol + ): + select = asterisk.table + while isinstance(select, ast.SelectQueryAliasSymbol): + select = select.symbol + if isinstance(select, ast.SelectQuerySymbol): + for name in select.columns.keys(): + columns.append( + ast.Field(chain=[name], symbol=ast.FieldSymbol(name=name, table=asterisk.table)) + ) + else: + raise ValueError("Can't expand asterisk (*) on subquery") else: - raise ValueError("Can't expand asterisk (*) on subquery") + raise ValueError(f"Can't expand asterisk (*) on a symbol of type {type(asterisk.table).__name__}") + else: columns.append(column) node.select = columns From e84425613147b7edb897d63c42faa9f602dbb6fd Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Tue, 21 Feb 2023 19:07:31 +0100 Subject: [PATCH 79/81] revert tiny changes lost in merge --- posthog/hogql/printer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/posthog/hogql/printer.py b/posthog/hogql/printer.py index e5598d1c6e4be..20930c183a06b 100644 --- a/posthog/hogql/printer.py +++ b/posthog/hogql/printer.py @@ -86,7 +86,7 @@ def visit_select_query(self, node: ast.SelectQuery): visited_join = self.visit_join_expr(next_join) joined_tables.append(visited_join.printed_sql) - # This is an expression we must add to the SELECT's WHERE clause to limit results. + # This is an expression we must add to the SELECT's WHERE clause to limit results, like the team ID guard. extra_where = visited_join.where if extra_where is None: pass @@ -143,6 +143,7 @@ def visit_select_query(self, node: ast.SelectQuery): # If we are printing a SELECT subquery (not the first AST node we are visiting), wrap it in parentheses. if len(self.stack) > 1: response = f"({response})" + return response def visit_join_expr(self, node: ast.JoinExpr) -> JoinExprResponse: From 0456203cabd21f9277ac929c2f9ee76e4ebb7a3e Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 22 Feb 2023 13:51:09 +0100 Subject: [PATCH 80/81] support nested splotches --- posthog/hogql/test/test_transforms.py | 32 +++++++++++++++++++++++++++ posthog/hogql/transforms.py | 14 +++++++----- 2 files changed, 40 insertions(+), 6 deletions(-) diff --git a/posthog/hogql/test/test_transforms.py b/posthog/hogql/test/test_transforms.py index b32353eba1a37..923023b255b35 100644 --- a/posthog/hogql/test/test_transforms.py +++ b/posthog/hogql/test/test_transforms.py @@ -110,6 +110,38 @@ def test_asterisk_expander_subquery_alias(self): ], ) + def test_asterisk_expander_from_subquery_table(self): + node = parse_select("select * from (select * from events) x") + resolve_symbols(node) + expand_asterisks(node) + + events_table_symbol = ast.TableSymbol(table=database.events) + events_table_alias_symbol = ast.TableAliasSymbol(table=events_table_symbol, name="e") + self.assertEqual( + node.select, + [ + ast.Field(chain=["uuid"], symbol=ast.FieldSymbol(name="uuid", table=events_table_alias_symbol)), + ast.Field(chain=["event"], symbol=ast.FieldSymbol(name="event", table=events_table_alias_symbol)), + ast.Field( + chain=["properties"], symbol=ast.FieldSymbol(name="properties", table=events_table_alias_symbol) + ), + ast.Field( + chain=["timestamp"], symbol=ast.FieldSymbol(name="timestamp", table=events_table_alias_symbol) + ), + ast.Field(chain=["team_id"], symbol=ast.FieldSymbol(name="team_id", table=events_table_alias_symbol)), + ast.Field( + chain=["distinct_id"], symbol=ast.FieldSymbol(name="distinct_id", table=events_table_alias_symbol) + ), + ast.Field( + chain=["elements_chain"], + symbol=ast.FieldSymbol(name="elements_chain", table=events_table_alias_symbol), + ), + ast.Field( + chain=["created_at"], symbol=ast.FieldSymbol(name="created_at", table=events_table_alias_symbol) + ), + ], + ) + def test_asterisk_expander_multiple_table_error(self): node = parse_select("select * from (select 1 as a, 2 as b) x left join (select 1 as a, 2 as b) y on x.a = y.a") with self.assertRaises(ResolverException) as e: diff --git a/posthog/hogql/transforms.py b/posthog/hogql/transforms.py index b14555b850ba9..b3d8c91ee5cf0 100644 --- a/posthog/hogql/transforms.py +++ b/posthog/hogql/transforms.py @@ -10,6 +10,8 @@ def expand_asterisks(node: ast.Expr): class AsteriskExpander(TraversingVisitor): def visit_select_query(self, node: ast.SelectQuery): + super().visit_select_query(node) + columns: List[ast.Expr] = [] for column in node.select: if isinstance(column.symbol, ast.AsteriskSymbol): @@ -21,9 +23,9 @@ def visit_select_query(self, node: ast.SelectQuery): if isinstance(table, ast.TableSymbol): database_fields = table.table.get_asterisk() for key in database_fields.keys(): - columns.append( - ast.Field(chain=[key], symbol=ast.FieldSymbol(name=key, table=asterisk.table)) - ) + symbol = ast.FieldSymbol(name=key, table=asterisk.table) + columns.append(ast.Field(chain=[key], symbol=symbol)) + node.symbol.columns[key] = symbol else: raise ValueError("Can't expand asterisk (*) on table") elif isinstance(asterisk.table, ast.SelectQuerySymbol) or isinstance( @@ -34,9 +36,9 @@ def visit_select_query(self, node: ast.SelectQuery): select = select.symbol if isinstance(select, ast.SelectQuerySymbol): for name in select.columns.keys(): - columns.append( - ast.Field(chain=[name], symbol=ast.FieldSymbol(name=name, table=asterisk.table)) - ) + symbol = ast.FieldSymbol(name=name, table=asterisk.table) + columns.append(ast.Field(chain=[name], symbol=symbol)) + node.symbol.columns[name] = symbol else: raise ValueError("Can't expand asterisk (*) on subquery") else: From 873b54c080779217fdc32810048c13778e3cddfe Mon Sep 17 00:00:00 2001 From: Marius Andra Date: Wed, 22 Feb 2023 14:16:36 +0100 Subject: [PATCH 81/81] actually fix the test --- posthog/hogql/test/test_transforms.py | 43 ++++++++++++++++----------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/posthog/hogql/test/test_transforms.py b/posthog/hogql/test/test_transforms.py index 923023b255b35..b1f7864d43f4f 100644 --- a/posthog/hogql/test/test_transforms.py +++ b/posthog/hogql/test/test_transforms.py @@ -111,34 +111,41 @@ def test_asterisk_expander_subquery_alias(self): ) def test_asterisk_expander_from_subquery_table(self): - node = parse_select("select * from (select * from events) x") + node = parse_select("select * from (select * from events)") resolve_symbols(node) expand_asterisks(node) events_table_symbol = ast.TableSymbol(table=database.events) - events_table_alias_symbol = ast.TableAliasSymbol(table=events_table_symbol, name="e") + inner_select_symbol = ast.SelectQuerySymbol( + tables={"events": events_table_symbol}, + anonymous_tables=[], + aliases={}, + columns={ + "uuid": ast.FieldSymbol(name="uuid", table=events_table_symbol), + "event": ast.FieldSymbol(name="event", table=events_table_symbol), + "properties": ast.FieldSymbol(name="properties", table=events_table_symbol), + "timestamp": ast.FieldSymbol(name="timestamp", table=events_table_symbol), + "team_id": ast.FieldSymbol(name="team_id", table=events_table_symbol), + "distinct_id": ast.FieldSymbol(name="distinct_id", table=events_table_symbol), + "elements_chain": ast.FieldSymbol(name="elements_chain", table=events_table_symbol), + "created_at": ast.FieldSymbol(name="created_at", table=events_table_symbol), + }, + ) + self.assertEqual( node.select, [ - ast.Field(chain=["uuid"], symbol=ast.FieldSymbol(name="uuid", table=events_table_alias_symbol)), - ast.Field(chain=["event"], symbol=ast.FieldSymbol(name="event", table=events_table_alias_symbol)), - ast.Field( - chain=["properties"], symbol=ast.FieldSymbol(name="properties", table=events_table_alias_symbol) - ), - ast.Field( - chain=["timestamp"], symbol=ast.FieldSymbol(name="timestamp", table=events_table_alias_symbol) - ), - ast.Field(chain=["team_id"], symbol=ast.FieldSymbol(name="team_id", table=events_table_alias_symbol)), - ast.Field( - chain=["distinct_id"], symbol=ast.FieldSymbol(name="distinct_id", table=events_table_alias_symbol) - ), + ast.Field(chain=["uuid"], symbol=ast.FieldSymbol(name="uuid", table=inner_select_symbol)), + ast.Field(chain=["event"], symbol=ast.FieldSymbol(name="event", table=inner_select_symbol)), + ast.Field(chain=["properties"], symbol=ast.FieldSymbol(name="properties", table=inner_select_symbol)), + ast.Field(chain=["timestamp"], symbol=ast.FieldSymbol(name="timestamp", table=inner_select_symbol)), + ast.Field(chain=["team_id"], symbol=ast.FieldSymbol(name="team_id", table=inner_select_symbol)), + ast.Field(chain=["distinct_id"], symbol=ast.FieldSymbol(name="distinct_id", table=inner_select_symbol)), ast.Field( chain=["elements_chain"], - symbol=ast.FieldSymbol(name="elements_chain", table=events_table_alias_symbol), - ), - ast.Field( - chain=["created_at"], symbol=ast.FieldSymbol(name="created_at", table=events_table_alias_symbol) + symbol=ast.FieldSymbol(name="elements_chain", table=inner_select_symbol), ), + ast.Field(chain=["created_at"], symbol=ast.FieldSymbol(name="created_at", table=inner_select_symbol)), ], )