diff --git a/docs/static/resources/openapi.json b/docs/static/resources/openapi.json index 86060e5470a19..9ecf94ffd6705 100644 --- a/docs/static/resources/openapi.json +++ b/docs/static/resources/openapi.json @@ -1680,7 +1680,7 @@ "type": "string" }, "changed_by": { - "$ref": "#/components/schemas/ChartDataRestApi.get_list.User1" + "$ref": "#/components/schemas/ChartDataRestApi.get_list.User" }, "changed_by_name": { "readOnly": true @@ -1695,7 +1695,7 @@ "readOnly": true }, "created_by": { - "$ref": "#/components/schemas/ChartDataRestApi.get_list.User3" + "$ref": "#/components/schemas/ChartDataRestApi.get_list.User2" }, "created_on_delta_humanized": { "readOnly": true @@ -1742,10 +1742,10 @@ "type": "string" }, "last_saved_by": { - "$ref": "#/components/schemas/ChartDataRestApi.get_list.User" + "$ref": "#/components/schemas/ChartDataRestApi.get_list.User3" }, "owners": { - "$ref": "#/components/schemas/ChartDataRestApi.get_list.User2" + "$ref": "#/components/schemas/ChartDataRestApi.get_list.User1" }, "params": { "nullable": true, @@ -1809,10 +1809,6 @@ "maxLength": 64, "type": "string" }, - "id": { - "format": "int32", - "type": "integer" - }, "last_name": { "maxLength": 64, "type": "string" @@ -1830,14 +1826,23 @@ "maxLength": 64, "type": "string" }, + "id": { + "format": "int32", + "type": "integer" + }, "last_name": { "maxLength": 64, "type": "string" + }, + "username": { + "maxLength": 64, + "type": "string" } }, "required": [ "first_name", - "last_name" + "last_name", + "username" ], "type": "object" }, @@ -1854,16 +1859,11 @@ "last_name": { "maxLength": 64, "type": "string" - }, - "username": { - "maxLength": 64, - "type": "string" } }, "required": [ "first_name", - "last_name", - "username" + "last_name" ], "type": "object" }, @@ -2472,7 +2472,7 @@ "type": "string" }, "changed_by": { - "$ref": "#/components/schemas/ChartRestApi.get_list.User1" + "$ref": "#/components/schemas/ChartRestApi.get_list.User" }, "changed_by_name": { "readOnly": true @@ -2487,7 +2487,7 @@ "readOnly": true }, "created_by": { - "$ref": "#/components/schemas/ChartRestApi.get_list.User3" + "$ref": "#/components/schemas/ChartRestApi.get_list.User2" }, "created_on_delta_humanized": { "readOnly": true @@ -2534,10 +2534,10 @@ "type": "string" }, "last_saved_by": { - "$ref": "#/components/schemas/ChartRestApi.get_list.User" + "$ref": "#/components/schemas/ChartRestApi.get_list.User3" }, "owners": { - "$ref": "#/components/schemas/ChartRestApi.get_list.User2" + "$ref": "#/components/schemas/ChartRestApi.get_list.User1" }, "params": { "nullable": true, @@ -2601,10 +2601,6 @@ "maxLength": 64, "type": "string" }, - "id": { - "format": "int32", - "type": "integer" - }, "last_name": { "maxLength": 64, "type": "string" @@ -2622,14 +2618,23 @@ "maxLength": 64, "type": "string" }, + "id": { + "format": "int32", + "type": "integer" + }, "last_name": { "maxLength": 64, "type": "string" + }, + "username": { + "maxLength": 64, + "type": "string" } }, "required": [ "first_name", - "last_name" + "last_name", + "username" ], "type": "object" }, @@ -2646,16 +2651,11 @@ "last_name": { "maxLength": 64, "type": "string" - }, - "username": { - "maxLength": 64, - "type": "string" } }, "required": [ "first_name", - "last_name", - "username" + "last_name" ], "type": "object" }, @@ -4172,6 +4172,12 @@ }, "DatabaseSSHTunnel": { "properties": { + "id": { + "description": "SSH Tunnel ID (for updates)", + "format": "int32", + "nullable": true, + "type": "integer" + }, "password": { "type": "string" }, @@ -5119,7 +5125,7 @@ "DatasetRestApi.get_list": { "properties": { "changed_by": { - "$ref": "#/components/schemas/DatasetRestApi.get_list.User1" + "$ref": "#/components/schemas/DatasetRestApi.get_list.User" }, "changed_by_name": { "readOnly": true @@ -5162,7 +5168,7 @@ "readOnly": true }, "owners": { - "$ref": "#/components/schemas/DatasetRestApi.get_list.User" + "$ref": "#/components/schemas/DatasetRestApi.get_list.User1" }, "schema": { "maxLength": 255, @@ -5206,14 +5212,6 @@ "maxLength": 64, "type": "string" }, - "id": { - "format": "int32", - "type": "integer" - }, - "last_name": { - "maxLength": 64, - "type": "string" - }, "username": { "maxLength": 64, "type": "string" @@ -5221,7 +5219,6 @@ }, "required": [ "first_name", - "last_name", "username" ], "type": "object" @@ -5232,6 +5229,14 @@ "maxLength": 64, "type": "string" }, + "id": { + "format": "int32", + "type": "integer" + }, + "last_name": { + "maxLength": 64, + "type": "string" + }, "username": { "maxLength": 64, "type": "string" @@ -5239,6 +5244,7 @@ }, "required": [ "first_name", + "last_name", "username" ], "type": "object" @@ -6164,6 +6170,207 @@ }, "type": "object" }, + "RLSRestApi.get": { + "properties": { + "clause": { + "description": "This is the condition that will be added to the WHERE clause. For example, to only return rows for a particular client, you might define a regular filter with the clause `client_id = 9`. To display no rows unless a user belongs to a RLS filter role, a base filter can be created with the clause `1 = 0` (always false).", + "type": "string" + }, + "description": { + "description": "Detailed description", + "type": "string" + }, + "filter_type": { + "description": "Regular filters add where clauses to queries if a user belongs to a role referenced in the filter, base filters apply filters to all queries except the roles defined in the filter, and can be used to define what users can see if no RLS filters within a filter group apply to them.", + "enum": [ + "Regular", + "Base" + ], + "type": "string" + }, + "group_key": { + "description": "Filters with the same group key will be ORed together within the group, while different filter groups will be ANDed together. Undefined group keys are treated as unique groups, i.e. are not grouped together. For example, if a table has three filters, of which two are for departments Finance and Marketing (group key = 'department'), and one refers to the region Europe (group key = 'region'), the filter clause would apply the filter (department = 'Finance' OR department = 'Marketing') AND (region = 'Europe').", + "type": "string" + }, + "id": { + "description": "Unique if of rls filter", + "format": "int32", + "type": "integer" + }, + "name": { + "description": "Name of rls filter", + "type": "string" + }, + "roles": { + "items": { + "$ref": "#/components/schemas/Roles1" + }, + "type": "array" + }, + "tables": { + "items": { + "$ref": "#/components/schemas/Tables" + }, + "type": "array" + } + }, + "type": "object" + }, + "RLSRestApi.get_list": { + "properties": { + "changed_on_delta_humanized": { + "readOnly": true + }, + "clause": { + "description": "This is the condition that will be added to the WHERE clause. For example, to only return rows for a particular client, you might define a regular filter with the clause `client_id = 9`. To display no rows unless a user belongs to a RLS filter role, a base filter can be created with the clause `1 = 0` (always false).", + "type": "string" + }, + "description": { + "description": "Detailed description", + "type": "string" + }, + "filter_type": { + "description": "Regular filters add where clauses to queries if a user belongs to a role referenced in the filter, base filters apply filters to all queries except the roles defined in the filter, and can be used to define what users can see if no RLS filters within a filter group apply to them.", + "enum": [ + "Regular", + "Base" + ], + "type": "string" + }, + "group_key": { + "description": "Filters with the same group key will be ORed together within the group, while different filter groups will be ANDed together. Undefined group keys are treated as unique groups, i.e. are not grouped together. For example, if a table has three filters, of which two are for departments Finance and Marketing (group key = 'department'), and one refers to the region Europe (group key = 'region'), the filter clause would apply the filter (department = 'Finance' OR department = 'Marketing') AND (region = 'Europe').", + "type": "string" + }, + "id": { + "description": "Unique if of rls filter", + "format": "int32", + "type": "integer" + }, + "name": { + "description": "Name of rls filter", + "type": "string" + }, + "roles": { + "items": { + "$ref": "#/components/schemas/Roles1" + }, + "type": "array" + }, + "tables": { + "items": { + "$ref": "#/components/schemas/Tables" + }, + "type": "array" + } + }, + "type": "object" + }, + "RLSRestApi.post": { + "properties": { + "clause": { + "description": "This is the condition that will be added to the WHERE clause. For example, to only return rows for a particular client, you might define a regular filter with the clause `client_id = 9`. To display no rows unless a user belongs to a RLS filter role, a base filter can be created with the clause `1 = 0` (always false).", + "type": "string" + }, + "description": { + "description": "Detailed description", + "nullable": true, + "type": "string" + }, + "filter_type": { + "description": "Regular filters add where clauses to queries if a user belongs to a role referenced in the filter, base filters apply filters to all queries except the roles defined in the filter, and can be used to define what users can see if no RLS filters within a filter group apply to them.", + "enum": [ + "Regular", + "Base" + ], + "type": "string" + }, + "group_key": { + "description": "Filters with the same group key will be ORed together within the group, while different filter groups will be ANDed together. Undefined group keys are treated as unique groups, i.e. are not grouped together. For example, if a table has three filters, of which two are for departments Finance and Marketing (group key = 'department'), and one refers to the region Europe (group key = 'region'), the filter clause would apply the filter (department = 'Finance' OR department = 'Marketing') AND (region = 'Europe').", + "nullable": true, + "type": "string" + }, + "name": { + "description": "Name of rls filter", + "maxLength": 255, + "minLength": 1, + "type": "string" + }, + "roles": { + "description": "For regular filters, these are the roles this filter will be applied to. For base filters, these are the roles that the filter DOES NOT apply to, e.g. Admin if admin should see all data.", + "items": { + "format": "int32", + "type": "integer" + }, + "type": "array" + }, + "tables": { + "description": "These are the tables this filter will be applied to.", + "items": { + "format": "int32", + "type": "integer" + }, + "minItems": 1, + "type": "array" + } + }, + "required": [ + "clause", + "filter_type", + "name", + "roles", + "tables" + ], + "type": "object" + }, + "RLSRestApi.put": { + "properties": { + "clause": { + "description": "This is the condition that will be added to the WHERE clause. For example, to only return rows for a particular client, you might define a regular filter with the clause `client_id = 9`. To display no rows unless a user belongs to a RLS filter role, a base filter can be created with the clause `1 = 0` (always false).", + "type": "string" + }, + "description": { + "description": "Detailed description", + "nullable": true, + "type": "string" + }, + "filter_type": { + "description": "Regular filters add where clauses to queries if a user belongs to a role referenced in the filter, base filters apply filters to all queries except the roles defined in the filter, and can be used to define what users can see if no RLS filters within a filter group apply to them.", + "enum": [ + "Regular", + "Base" + ], + "type": "string" + }, + "group_key": { + "description": "Filters with the same group key will be ORed together within the group, while different filter groups will be ANDed together. Undefined group keys are treated as unique groups, i.e. are not grouped together. For example, if a table has three filters, of which two are for departments Finance and Marketing (group key = 'department'), and one refers to the region Europe (group key = 'region'), the filter clause would apply the filter (department = 'Finance' OR department = 'Marketing') AND (region = 'Europe').", + "nullable": true, + "type": "string" + }, + "name": { + "description": "Name of rls filter", + "maxLength": 255, + "minLength": 1, + "type": "string" + }, + "roles": { + "description": "For regular filters, these are the roles this filter will be applied to. For base filters, these are the roles that the filter DOES NOT apply to, e.g. Admin if admin should see all data.", + "items": { + "format": "int32", + "type": "integer" + }, + "type": "array" + }, + "tables": { + "description": "These are the tables this filter will be applied to.", + "items": { + "format": "int32", + "type": "integer" + }, + "type": "array" + } + }, + "type": "object" + }, "RelatedResponseSchema": { "properties": { "count": { @@ -8220,6 +8427,18 @@ }, "type": "object" }, + "Roles1": { + "properties": { + "id": { + "format": "int32", + "type": "integer" + }, + "name": { + "type": "string" + } + }, + "type": "object" + }, "SavedQueryRestApi.get": { "properties": { "changed_on_delta_humanized": { @@ -8725,6 +8944,21 @@ }, "type": "object" }, + "Tables": { + "properties": { + "id": { + "format": "int32", + "type": "integer" + }, + "schema": { + "type": "string" + }, + "table_name": { + "type": "string" + } + }, + "type": "object" + }, "TemporaryCachePostSchema": { "properties": { "value": { @@ -9083,7 +9317,18 @@ }, "type": "object" }, - "screenshot_query_schema": { + "queries_get_updated_since_schema": { + "properties": { + "last_updated_ms": { + "type": "number" + } + }, + "required": [ + "last_updated_ms" + ], + "type": "object" + }, + "screenshot_query_schema": { "properties": { "force": { "type": "boolean" @@ -14217,6 +14462,10 @@ "engine_information": { "description": "Dict with public properties form the DB Engine", "properties": { + "disable_ssh_tunneling": { + "description": "Whether the engine supports SSH Tunnels", + "type": "boolean" + }, "supports_file_upload": { "description": "Whether the engine supports file uploads", "type": "boolean" @@ -17409,6 +17658,65 @@ ] } }, + "/api/v1/query/updated_since": { + "get": { + "parameters": [ + { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/queries_get_updated_since_schema" + } + } + }, + "in": "query", + "name": "q" + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "result": { + "description": "A List of queries that changed after last_updated_ms", + "items": { + "$ref": "#/components/schemas/QueryRestApi.get" + }, + "type": "array" + } + }, + "type": "object" + } + } + }, + "description": "Queries list" + }, + "400": { + "$ref": "#/components/responses/400" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "404": { + "$ref": "#/components/responses/404" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "summary": "Get a list of queries that changed after last_updated_ms", + "tags": [ + "Queries" + ] + } + }, "/api/v1/query/{pk}": { "get": { "description": "Get query detail information.", @@ -18259,6 +18567,590 @@ ] } }, + "/api/v1/rowlevelsecurity/": { + "delete": { + "description": "Deletes multiple RLS rules in a bulk operation.", + "parameters": [ + { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/get_delete_ids_schema" + } + } + }, + "in": "query", + "name": "q" + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "message": { + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "RLS Rule bulk delete" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "403": { + "$ref": "#/components/responses/403" + }, + "404": { + "$ref": "#/components/responses/404" + }, + "422": { + "$ref": "#/components/responses/422" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Row Level Security" + ] + }, + "get": { + "description": "Get a list of models", + "parameters": [ + { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/get_list_schema" + } + } + }, + "in": "query", + "name": "q" + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "count": { + "description": "The total record count on the backend", + "type": "number" + }, + "description_columns": { + "properties": { + "column_name": { + "description": "The description for the column name. Will be translated by babel", + "example": "A Nice description for the column", + "type": "string" + } + }, + "type": "object" + }, + "ids": { + "description": "A list of item ids, useful when you don't know the column id", + "items": { + "type": "string" + }, + "type": "array" + }, + "label_columns": { + "properties": { + "column_name": { + "description": "The label for the column name. Will be translated by babel", + "example": "A Nice label for the column", + "type": "string" + } + }, + "type": "object" + }, + "list_columns": { + "description": "A list of columns", + "items": { + "type": "string" + }, + "type": "array" + }, + "list_title": { + "description": "A title to render. Will be translated by babel", + "example": "List Items", + "type": "string" + }, + "order_columns": { + "description": "A list of allowed columns to sort", + "items": { + "type": "string" + }, + "type": "array" + }, + "result": { + "description": "The result from the get list query", + "items": { + "$ref": "#/components/schemas/RLSRestApi.get_list" + }, + "type": "array" + } + }, + "type": "object" + } + } + }, + "description": "Items from Model" + }, + "400": { + "$ref": "#/components/responses/400" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "422": { + "$ref": "#/components/responses/422" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Row Level Security" + ] + }, + "post": { + "description": "Create a new RLS Rule", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RLSRestApi.post" + } + } + }, + "description": "RLS schema", + "required": true + }, + "responses": { + "201": { + "content": { + "application/json": { + "schema": { + "properties": { + "id": { + "type": "number" + }, + "result": { + "$ref": "#/components/schemas/RLSRestApi.post" + } + }, + "type": "object" + } + } + }, + "description": "RLS Rule added" + }, + "400": { + "$ref": "#/components/responses/400" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "404": { + "$ref": "#/components/responses/404" + }, + "422": { + "$ref": "#/components/responses/422" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Row Level Security" + ] + } + }, + "/api/v1/rowlevelsecurity/_info": { + "get": { + "description": "Get metadata information about this API resource", + "parameters": [ + { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/get_info_schema" + } + } + }, + "in": "query", + "name": "q" + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "add_columns": { + "type": "object" + }, + "edit_columns": { + "type": "object" + }, + "filters": { + "properties": { + "column_name": { + "items": { + "properties": { + "name": { + "description": "The filter name. Will be translated by babel", + "type": "string" + }, + "operator": { + "description": "The filter operation key to use on list filters", + "type": "string" + } + }, + "type": "object" + }, + "type": "array" + } + }, + "type": "object" + }, + "permissions": { + "description": "The user permissions for this API resource", + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + } + } + }, + "description": "Item from Model" + }, + "400": { + "$ref": "#/components/responses/400" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "422": { + "$ref": "#/components/responses/422" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Row Level Security" + ] + } + }, + "/api/v1/rowlevelsecurity/related/{column_name}": { + "get": { + "parameters": [ + { + "in": "path", + "name": "column_name", + "required": true, + "schema": { + "type": "string" + } + }, + { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/get_related_schema" + } + } + }, + "in": "query", + "name": "q" + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RelatedResponseSchema" + } + } + }, + "description": "Related column data" + }, + "400": { + "$ref": "#/components/responses/400" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "404": { + "$ref": "#/components/responses/404" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Row Level Security" + ] + } + }, + "/api/v1/rowlevelsecurity/{pk}": { + "delete": { + "parameters": [ + { + "in": "path", + "name": "pk", + "required": true, + "schema": { + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "message": { + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Item deleted" + }, + "404": { + "$ref": "#/components/responses/404" + }, + "422": { + "$ref": "#/components/responses/422" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Row Level Security" + ] + }, + "get": { + "description": "Get an item model", + "parameters": [ + { + "in": "path", + "name": "pk", + "required": true, + "schema": { + "type": "integer" + } + }, + { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/get_item_schema" + } + } + }, + "in": "query", + "name": "q" + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "description_columns": { + "properties": { + "column_name": { + "description": "The description for the column name. Will be translated by babel", + "example": "A Nice description for the column", + "type": "string" + } + }, + "type": "object" + }, + "id": { + "description": "The item id", + "type": "string" + }, + "label_columns": { + "properties": { + "column_name": { + "description": "The label for the column name. Will be translated by babel", + "example": "A Nice label for the column", + "type": "string" + } + }, + "type": "object" + }, + "result": { + "$ref": "#/components/schemas/RLSRestApi.get" + }, + "show_columns": { + "description": "A list of columns", + "items": { + "type": "string" + }, + "type": "array" + }, + "show_title": { + "description": "A title to render. Will be translated by babel", + "example": "Show Item Details", + "type": "string" + } + }, + "type": "object" + } + } + }, + "description": "Item from Model" + }, + "400": { + "$ref": "#/components/responses/400" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "404": { + "$ref": "#/components/responses/404" + }, + "422": { + "$ref": "#/components/responses/422" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Row Level Security" + ] + }, + "put": { + "description": "Updates an RLS Rule", + "parameters": [ + { + "description": "The Rule pk", + "in": "path", + "name": "pk", + "required": true, + "schema": { + "type": "integer" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RLSRestApi.put" + } + } + }, + "description": "RLS schema", + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "properties": { + "id": { + "type": "number" + }, + "result": { + "$ref": "#/components/schemas/RLSRestApi.put" + } + }, + "type": "object" + } + } + }, + "description": "Rule changed" + }, + "400": { + "$ref": "#/components/responses/400" + }, + "401": { + "$ref": "#/components/responses/401" + }, + "403": { + "$ref": "#/components/responses/403" + }, + "404": { + "$ref": "#/components/responses/404" + }, + "422": { + "$ref": "#/components/responses/422" + }, + "500": { + "$ref": "#/components/responses/500" + } + }, + "security": [ + { + "jwt": [] + } + ], + "tags": [ + "Row Level Security" + ] + } + }, "/api/v1/saved_query/": { "delete": { "description": "Deletes multiple saved queries in a bulk operation.", diff --git a/superset-frontend/src/SqlLab/components/QueryAutoRefresh/index.tsx b/superset-frontend/src/SqlLab/components/QueryAutoRefresh/index.tsx index eb3e6f4c38f87..2d01e724e2479 100644 --- a/superset-frontend/src/SqlLab/components/QueryAutoRefresh/index.tsx +++ b/superset-frontend/src/SqlLab/components/QueryAutoRefresh/index.tsx @@ -18,10 +18,12 @@ */ import { useState } from 'react'; import { isObject } from 'lodash'; +import rison from 'rison'; import { SupersetClient, Query, runningQueryStateList, + QueryResponse, } from '@superset-ui/core'; import { QueryDictionary } from 'src/SqlLab/types'; import useInterval from 'src/SqlLab/utils/useInterval'; @@ -62,22 +64,30 @@ function QueryAutoRefresh({ refreshQueries, queriesLastUpdate, }: QueryAutoRefreshProps) { - // We do not want to spam requests in the case of slow connections and potentially recieve responses out of order + // We do not want to spam requests in the case of slow connections and potentially receive responses out of order // pendingRequest check ensures we only have one active http call to check for query statuses const [pendingRequest, setPendingRequest] = useState(false); const checkForRefresh = () => { if (!pendingRequest && shouldCheckForQueries(queries)) { + const params = rison.encode({ + last_updated_ms: queriesLastUpdate - QUERY_UPDATE_BUFFER_MS, + }); + setPendingRequest(true); SupersetClient.get({ - endpoint: `/superset/queries/${ - queriesLastUpdate - QUERY_UPDATE_BUFFER_MS - }`, + endpoint: `/api/v1/query/updated_since?q=${params}`, timeout: QUERY_TIMEOUT_LIMIT, }) .then(({ json }) => { if (json) { - refreshQueries?.(json); + const jsonPayload = json as { result?: QueryResponse[] }; + const queries = + jsonPayload?.result?.reduce((acc, current) => { + acc[current.id] = current; + return acc; + }, {}) ?? {}; + refreshQueries?.(queries); } }) .catch(() => {}) diff --git a/superset/constants.py b/superset/constants.py index 10de5c52f04a7..3d2c5c470c2c7 100644 --- a/superset/constants.py +++ b/superset/constants.py @@ -140,6 +140,7 @@ class RouteMethod: # pylint: disable=too-few-public-methods "get_data": "read", "samples": "read", "delete_ssh_tunnel": "write", + "get_updated_since": "read", "stop_query": "read", } diff --git a/superset/queries/api.py b/superset/queries/api.py index 51ba148603285..1784edf167c80 100644 --- a/superset/queries/api.py +++ b/superset/queries/api.py @@ -15,9 +15,10 @@ # specific language governing permissions and limitations # under the License. import logging +from typing import Any import backoff -from flask_appbuilder.api import expose, protect, request, safe +from flask_appbuilder.api import expose, protect, request, rison, safe from flask_appbuilder.models.sqla.interface import SQLAInterface from superset import db, event_logger @@ -29,6 +30,7 @@ from superset.queries.filters import QueryFilter from superset.queries.schemas import ( openapi_spec_methods_override, + queries_get_updated_since_schema, QuerySchema, StopQuerySchema, ) @@ -59,6 +61,11 @@ class QueryRestApi(BaseSupersetModelRestApi): RouteMethod.RELATED, RouteMethod.DISTINCT, "stop_query", + "get_updated_since", + } + + apispec_parameter_schemas = { + "queries_get_updated_since_schema": queries_get_updated_since_schema, } list_columns = [ @@ -142,6 +149,59 @@ class QueryRestApi(BaseSupersetModelRestApi): allowed_rel_fields = {"database", "user"} allowed_distinct_fields = {"status"} + @expose("/updated_since") + @protect() + @safe + @rison(queries_get_updated_since_schema) + @statsd_metrics + @event_logger.log_this_with_context( + action=lambda self, *args, **kwargs: f"{self.__class__.__name__}" + f".get_updated_since", + log_to_statsd=False, + ) + def get_updated_since(self, **kwargs: Any) -> FlaskResponse: + """Get a list of queries that changed after last_updated_ms + --- + get: + summary: Get a list of queries that changed after last_updated_ms + parameters: + - in: query + name: q + content: + application/json: + schema: + $ref: '#/components/schemas/queries_get_updated_since_schema' + responses: + 200: + description: Queries list + content: + application/json: + schema: + type: object + properties: + result: + description: >- + A List of queries that changed after last_updated_ms + type: array + items: + $ref: '#/components/schemas/{{self.__class__.__name__}}.get' + 400: + $ref: '#/components/responses/400' + 401: + $ref: '#/components/responses/401' + 404: + $ref: '#/components/responses/404' + 500: + $ref: '#/components/responses/500' + """ + try: + last_updated_ms = kwargs["rison"].get("last_updated_ms", 0) + queries = QueryDAO.get_queries_changed_after(last_updated_ms) + payload = [q.to_dict() for q in queries] + return self.response(200, result=payload) + except SupersetException as ex: + return self.response(ex.status, message=ex.message) + @expose("/stop", methods=["POST"]) @protect() @safe diff --git a/superset/queries/dao.py b/superset/queries/dao.py index 5867b2917dba0..642a5dd4cbb1c 100644 --- a/superset/queries/dao.py +++ b/superset/queries/dao.py @@ -16,7 +16,7 @@ # under the License. import logging from datetime import datetime -from typing import Any, Dict +from typing import Any, Dict, List, Union from superset import sql_lab from superset.common.db_query_status import QueryStatus @@ -25,6 +25,7 @@ from superset.extensions import db from superset.models.sql_lab import Query, SavedQuery from superset.queries.filters import QueryFilter +from superset.utils.core import get_user_id from superset.utils.dates import now_as_float logger = logging.getLogger(__name__) @@ -61,6 +62,17 @@ def save_metadata(query: Query, payload: Dict[str, Any]) -> None: db.session.add(query) query.set_extra_json_key("columns", columns) + @staticmethod + def get_queries_changed_after(last_updated_ms: Union[float, int]) -> List[Query]: + # UTC date time, same that is stored in the DB. + last_updated_dt = datetime.utcfromtimestamp(last_updated_ms / 1000) + + return ( + db.session.query(Query) + .filter(Query.user_id == get_user_id(), Query.changed_on >= last_updated_dt) + .all() + ) + @staticmethod def stop_query(client_id: str) -> None: query = db.session.query(Query).filter_by(client_id=client_id).one_or_none() diff --git a/superset/queries/schemas.py b/superset/queries/schemas.py index a8c1e2bbcbd5d..c29c1c03b6b69 100644 --- a/superset/queries/schemas.py +++ b/superset/queries/schemas.py @@ -33,6 +33,14 @@ }, } +queries_get_updated_since_schema = { + "type": "object", + "properties": { + "last_updated_ms": {"type": "number"}, + }, + "required": ["last_updated_ms"], +} + class DatabaseSchema(Schema): database_name = fields.String() diff --git a/superset/views/core.py b/superset/views/core.py index 9733ea2910a09..758bce4a13e4b 100755 --- a/superset/views/core.py +++ b/superset/views/core.py @@ -2485,6 +2485,7 @@ def fetch_datasource_metadata(self) -> FlaskResponse: # pylint: disable=no-self @event_logger.log_this @expose("/queries/") @expose("/queries/") + @deprecated() def queries(self, last_updated_ms: Union[float, int]) -> FlaskResponse: """ Get the updated queries. diff --git a/tests/integration_tests/queries/api_tests.py b/tests/integration_tests/queries/api_tests.py index 8c193662a12c7..b491cf8498b0e 100644 --- a/tests/integration_tests/queries/api_tests.py +++ b/tests/integration_tests/queries/api_tests.py @@ -52,6 +52,7 @@ def insert_query( rows: int = 100, tab_name: str = "", status: str = "success", + changed_on: datetime = datetime(2020, 1, 1), ) -> Query: database = db.session.query(Database).get(database_id) user = db.session.query(security_manager.user_model).get(user_id) @@ -67,7 +68,7 @@ def insert_query( rows=rows, tab_name=tab_name, status=status, - changed_on=datetime(2020, 1, 1), + changed_on=changed_on, ) db.session.add(query) db.session.commit() @@ -394,6 +395,60 @@ def test_get_list_query_no_data_access(self): db.session.delete(query) db.session.commit() + def test_get_updated_since(self): + """ + Query API: Test get queries updated since timestamp + """ + now = datetime.utcnow() + client_id = self.get_random_string() + + admin = self.get_user("admin") + example_db = get_example_database() + + old_query = self.insert_query( + example_db.id, + admin.id, + self.get_random_string(), + sql="SELECT col1, col2 from table1", + select_sql="SELECT col1, col2 from table1", + executed_sql="SELECT col1, col2 from table1 LIMIT 100", + changed_on=now - timedelta(days=3), + ) + updated_query = self.insert_query( + example_db.id, + admin.id, + client_id, + sql="SELECT col1, col2 from table1", + select_sql="SELECT col1, col2 from table1", + executed_sql="SELECT col1, col2 from table1 LIMIT 100", + changed_on=now - timedelta(days=1), + ) + + self.login(username="admin") + timestamp = datetime.timestamp(now - timedelta(days=2)) * 1000 + uri = f"api/v1/query/updated_since?q={prison.dumps({'last_updated_ms': timestamp})}" + rv = self.client.get(uri) + self.assertEqual(rv.status_code, 200) + + expected_result = updated_query.to_dict() + data = json.loads(rv.data.decode("utf-8")) + self.assertEqual(len(data["result"]), 1) + for key, value in data["result"][0].items(): + # We can't assert timestamp + if key not in ( + "changedOn", + "changed_on", + "end_time", + "start_running_time", + "start_time", + "id", + ): + self.assertEqual(value, expected_result[key]) + # rollback changes + db.session.delete(old_query) + db.session.delete(updated_query) + db.session.commit() + @mock.patch("superset.sql_lab.cancel_query") @mock.patch("superset.views.core.db.session") def test_stop_query_not_found( diff --git a/tests/unit_tests/dao/queries_test.py b/tests/unit_tests/dao/queries_test.py index 590ba92d48f13..62eeff31065e0 100644 --- a/tests/unit_tests/dao/queries_test.py +++ b/tests/unit_tests/dao/queries_test.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. import json +from datetime import datetime, timedelta from typing import Any, Iterator import pytest @@ -58,6 +59,61 @@ def test_query_dao_save_metadata(session: Session) -> None: assert query.extra.get("columns", None) == [] +def test_query_dao_get_queries_changed_after(session: Session) -> None: + from superset.models.core import Database + from superset.models.sql_lab import Query + + engine = session.get_bind() + Query.metadata.create_all(engine) # pylint: disable=no-member + + db = Database(database_name="my_database", sqlalchemy_uri="sqlite://") + + now = datetime.utcnow() + + old_query_obj = Query( + client_id="foo", + database=db, + tab_name="test_tab", + sql_editor_id="test_editor_id", + sql="select * from bar", + select_sql="select * from bar", + executed_sql="select * from bar", + limit=100, + select_as_cta=False, + rows=100, + error_message="none", + results_key="abc", + changed_on=now - timedelta(days=3), + ) + + updated_query_obj = Query( + client_id="updated_foo", + database=db, + tab_name="test_tab", + sql_editor_id="test_editor_id", + sql="select * from foo", + select_sql="select * from foo", + executed_sql="select * from foo", + limit=100, + select_as_cta=False, + rows=100, + error_message="none", + results_key="abc", + changed_on=now - timedelta(days=1), + ) + + session.add(db) + session.add(old_query_obj) + session.add(updated_query_obj) + + from superset.queries.dao import QueryDAO + + timestamp = datetime.timestamp(now - timedelta(days=2)) * 1000 + result = QueryDAO.get_queries_changed_after(timestamp) + assert len(result) == 1 + assert result[0].client_id == "updated_foo" + + def test_query_dao_stop_query_not_found( mocker: MockFixture, app: Any, session: Session ) -> None: