refactor: Unify all json.(loads|dumps) usage to utils.json (#28702)

Co-authored-by: Eyal Ezer <[email protected]>

.pylintrc

@@ -36,7 +36,7 @@ persistent=yes
 
 # List of plugins (as comma separated values of python modules names) to load,
 # usually to register additional checkers.
-load-plugins=
+load-plugins=superset.extensions.pylint
 
 # Use multiple processes to speed up Pylint.
 jobs=2

superset/annotation_layers/annotations/schemas.py

@@ -19,8 +19,7 @@
 from marshmallow import fields, Schema, ValidationError
 from marshmallow.validate import Length
 
-from superset.exceptions import SupersetException
-from superset.utils import json as json_utils
+from superset.utils import json
 
 openapi_spec_methods_override = {
     "get": {"get": {"summary": "Get an annotation layer"}},
@@ -51,8 +50,8 @@
 
 def validate_json(value: Union[bytes, bytearray, str]) -> None:
     try:
-        json_utils.validate_json(value)
-    except SupersetException as ex:
+        json.validate_json(value)
+    except json.JSONDecodeError as ex:
         raise ValidationError("JSON not valid") from ex
 
 

superset/async_events/async_query_manager.py

@@ -14,7 +14,6 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-import json
 import logging
 import uuid
 from typing import Any, Literal, Optional
@@ -23,6 +22,7 @@
 import redis
 from flask import Flask, Request, request, Response, session
 
+from superset.utils import json
 from superset.utils.core import get_user_id
 
 logger = logging.getLogger(__name__)

superset/charts/api.py

@@ -15,7 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 # pylint: disable=too-many-lines
-import json
 import logging
 from datetime import datetime
 from io import BytesIO
@@ -81,6 +80,7 @@
 from superset.models.slice import Slice
 from superset.tasks.thumbnails import cache_chart_thumbnail
 from superset.tasks.utils import get_current_user
+from superset.utils import json
 from superset.utils.screenshots import ChartScreenshot, DEFAULT_CHART_WINDOW_SIZE
 from superset.utils.urls import get_url_path
 from superset.views.base_api import (

superset/charts/data/api.py

@@ -17,7 +17,6 @@
 from __future__ import annotations
 
 import contextlib
-import json
 import logging
 from typing import Any, TYPE_CHECKING
 
@@ -46,7 +45,7 @@
 from superset.exceptions import QueryObjectValidationError
 from superset.extensions import event_logger
 from superset.models.sql_lab import Query
-from superset.utils import json as json_utils
+from superset.utils import json
 from superset.utils.core import (
     create_zip,
     DatasourceType,
@@ -129,7 +128,7 @@ def get_data(self, pk: int) -> Response:
 
         try:
             json_body = json.loads(chart.query_context)
-        except (TypeError, json.decoder.JSONDecodeError):
+        except (TypeError, json.JSONDecodeError):
             json_body = None
 
         if json_body is None:
@@ -171,7 +170,7 @@ def get_data(self, pk: int) -> Response:
 
         try:
             form_data = json.loads(chart.params)
-        except (TypeError, json.decoder.JSONDecodeError):
+        except (TypeError, json.JSONDecodeError):
             form_data = {}
 
         return self._get_data_response(
@@ -395,9 +394,9 @@ def _process_data(query_data: Any) -> Any:
             )
 
         if result_format == ChartDataResultFormat.JSON:
-            response_data = json_utils.dumps(
+            response_data = json.dumps(
                 {"result": result["queries"]},
-                default=json_utils.json_int_dttm_ser,
+                default=json.json_int_dttm_ser,
                 ignore_nan=True,
             )
             resp = make_response(response_data, 200)

superset/commands/chart/export.py

@@ -16,7 +16,6 @@
 # under the License.
 # isort:skip_file
 
-import json
 import logging
 from collections.abc import Iterator
 from typing import Callable
@@ -30,6 +29,7 @@
 from superset.models.slice import Slice
 from superset.utils.dict_import_export import EXPORT_VERSION
 from superset.utils.file import get_filename
+from superset.utils import json
 
 logger = logging.getLogger(__name__)
 
@@ -64,7 +64,7 @@ def _file_content(model: Slice) -> str:
         if payload.get("params"):
             try:
                 payload["params"] = json.loads(payload["params"])
-            except json.decoder.JSONDecodeError:
+            except json.JSONDecodeError:
                 logger.info("Unable to decode `params` field: %s", payload["params"])
 
         payload["version"] = EXPORT_VERSION

superset/commands/chart/importers/v1/utils.py

@@ -16,7 +16,6 @@
 # under the License.
 
 import copy
-import json
 from inspect import isclass
 from typing import Any
 
@@ -25,6 +24,7 @@
 from superset.migrations.shared.migrate_viz import processors
 from superset.migrations.shared.migrate_viz.base import MigrateViz
 from superset.models.slice import Slice
+from superset.utils import json
 from superset.utils.core import AnnotationType, get_user
 
 
@@ -117,7 +117,7 @@ def migrate_chart(config: dict[str, Any]) -> dict[str, Any]:
     # also update `query_context`
     try:
         query_context = json.loads(output.get("query_context") or "{}")
-    except (json.decoder.JSONDecodeError, TypeError):
+    except (json.JSONDecodeError, TypeError):
         query_context = {}
     if "form_data" in query_context:
         query_context["form_data"] = output["params"]

superset/commands/chart/warm_up_cache.py

@@ -18,7 +18,6 @@
 
 from typing import Any, Optional, Union
 
-import simplejson as json
 from flask import g
 
 from superset.commands.base import BaseCommand
@@ -29,6 +28,7 @@
 )
 from superset.extensions import db
 from superset.models.slice import Slice
+from superset.utils import json
 from superset.utils.core import error_msg_from_exception
 from superset.views.utils import get_dashboard_extra_filters, get_form_data, get_viz
 from superset.viz import viz_types

superset/commands/dashboard/export.py

@@ -16,7 +16,6 @@
 # under the License.
 # isort:skip_file
 
-import json
 import logging
 import random
 import string
@@ -36,6 +35,7 @@
 from superset.models.slice import Slice
 from superset.utils.dict_import_export import EXPORT_VERSION
 from superset.utils.file import get_filename
+from superset.utils import json
 
 logger = logging.getLogger(__name__)
 
@@ -126,7 +126,7 @@ def _file_content(model: Dashboard) -> str:
             if value:
                 try:
                     payload[new_name] = json.loads(value)
-                except (TypeError, json.decoder.JSONDecodeError):
+                except (TypeError, json.JSONDecodeError):
                     logger.info("Unable to decode `%s` field: %s", key, value)
                     payload[new_name] = {}
 
@@ -176,7 +176,7 @@ def _export(
             if value:
                 try:
                     payload[new_name] = json.loads(value)
-                except (TypeError, json.decoder.JSONDecodeError):
+                except (TypeError, json.JSONDecodeError):
                     logger.info("Unable to decode `%s` field: %s", key, value)
                     payload[new_name] = {}
 

superset/commands/dashboard/importers/v0.py

@@ -14,7 +14,6 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-import json
 import logging
 import time
 from copy import copy
@@ -32,6 +31,7 @@
 from superset.migrations.shared.native_filters import migrate_dashboard
 from superset.models.dashboard import Dashboard
 from superset.models.slice import Slice
+from superset.utils import json
 from superset.utils.dashboard_filter_scopes_converter import (
     convert_filter_scopes,
     copy_filter_scopes,

superset/commands/dashboard/importers/v1/utils.py

@@ -15,13 +15,13 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import json
 import logging
 from typing import Any
 
 from superset import db, security_manager
 from superset.commands.exceptions import ImportFailedError
 from superset.models.dashboard import Dashboard
+from superset.utils import json
 from superset.utils.core import get_user
 
 logger = logging.getLogger(__name__)

superset/commands/dashboard/update.py

@@ -14,7 +14,6 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-import json
 import logging
 from typing import Any, Optional
 
@@ -37,6 +36,7 @@
 from superset.extensions import db
 from superset.models.dashboard import Dashboard
 from superset.tags.models import ObjectType
+from superset.utils import json
 
 logger = logging.getLogger(__name__)
 

superset/commands/database/export.py

@@ -16,7 +16,6 @@
 # under the License.
 # isort:skip_file
 import functools
-import json
 import logging
 from typing import Any, Callable
 from collections.abc import Iterator
@@ -30,14 +29,15 @@
 from superset.utils.dict_import_export import EXPORT_VERSION
 from superset.utils.file import get_filename
 from superset.utils.ssh_tunnel import mask_password_info
+from superset.utils import json
 
 logger = logging.getLogger(__name__)
 
 
 def parse_extra(extra_payload: str) -> dict[str, Any]:
     try:
         extra = json.loads(extra_payload)
-    except json.decoder.JSONDecodeError:
+    except json.JSONDecodeError:
         logger.info("Unable to decode `extra` field: %s", extra_payload)
         return {}
 

superset/commands/database/importers/v1/utils.py

@@ -15,7 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import json
 from typing import Any
 
 from superset import app, db, security_manager
@@ -25,6 +24,7 @@
 from superset.exceptions import SupersetSecurityException
 from superset.models.core import Database
 from superset.security.analytics_db_safety import check_sqlalchemy_uri
+from superset.utils import json
 
 
 def import_database(

superset/commands/database/validate.py

@@ -14,7 +14,6 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-import json
 from contextlib import closing
 from typing import Any, Optional
 
@@ -33,6 +32,7 @@
 from superset.errors import ErrorLevel, SupersetError, SupersetErrorType
 from superset.extensions import event_logger
 from superset.models.core import Database
+from superset.utils import json
 
 BYPASS_VALIDATION_ENGINES = {"bigquery"}
 
@@ -82,7 +82,7 @@ def run(self) -> None:
             )
         try:
             encrypted_extra = json.loads(serialized_encrypted_extra)
-        except json.decoder.JSONDecodeError:
+        except json.JSONDecodeError:
             encrypted_extra = {}
 
         # try to connect

superset/commands/dataset/export.py

@@ -16,7 +16,6 @@
 # under the License.
 # isort:skip_file
 
-import json
 import logging
 from collections.abc import Iterator
 from typing import Callable
@@ -31,6 +30,7 @@
 from superset.utils.dict_import_export import EXPORT_VERSION
 from superset.utils.file import get_filename
 from superset.utils.ssh_tunnel import mask_password_info
+from superset.utils import json
 
 logger = logging.getLogger(__name__)
 
@@ -63,14 +63,14 @@ def _file_content(model: SqlaTable) -> str:
             if payload.get(key):
                 try:
                     payload[key] = json.loads(payload[key])
-                except json.decoder.JSONDecodeError:
+                except json.JSONDecodeError:
                     logger.info("Unable to decode `%s` field: %s", key, payload[key])
         for key in ("metrics", "columns"):
             for attributes in payload.get(key, []):
                 if attributes.get("extra"):
                     try:
                         attributes["extra"] = json.loads(attributes["extra"])
-                    except json.decoder.JSONDecodeError:
+                    except json.JSONDecodeError:
                         logger.info(
                             "Unable to decode `extra` field: %s", attributes["extra"]
                         )
@@ -108,7 +108,7 @@ def _export(
             if payload.get("extra"):
                 try:
                     payload["extra"] = json.loads(payload["extra"])
-                except json.decoder.JSONDecodeError:
+                except json.JSONDecodeError:
                     logger.info("Unable to decode `extra` field: %s", payload["extra"])
 
             if ssh_tunnel := DatabaseDAO.get_ssh_tunnel(model.database.id):

superset/commands/dataset/importers/v0.py

@@ -14,7 +14,6 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-import json
 import logging
 from typing import Any, Callable, Optional
 
@@ -34,6 +33,7 @@
     TableColumn,
 )
 from superset.models.core import Database
+from superset.utils import json
 from superset.utils.dict_import_export import DATABASES_KEY
 
 logger = logging.getLogger(__name__)

superset/commands/dataset/importers/v1/utils.py

@@ -15,7 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 import gzip
-import json
 import logging
 import re
 from typing import Any
@@ -33,6 +32,7 @@
 from superset.connectors.sqla.models import SqlaTable
 from superset.models.core import Database
 from superset.sql_parse import Table
+from superset.utils import json
 from superset.utils.core import get_user
 
 logger = logging.getLogger(__name__)