Python: Replace isort, pylint, prettier, pyupgrade & flake8 with ruff

python/.pre-commit-config.yaml

@@ -28,52 +28,28 @@ repos:
       - id: debug-statements
       - id: check-yaml
       - id: check-ast
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version (Used for linting)
+    rev: v0.0.286
+    hooks:
+      - id: ruff
+        args: [ --fix, --exit-non-zero-on-fix ]
   - repo: https://github.com/ambv/black
     rev: 23.3.0
     hooks:
       - id: black
-  - repo: https://github.com/pre-commit/mirrors-isort
-    rev: v5.10.1
-    hooks:
-      - id: isort
-        args: [--settings-path=python/pyproject.toml]
+        args: [--skip-string-normalization]
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.3.0
     hooks:
       - id: mypy
         args:
           [--install-types, --non-interactive, --config=python/pyproject.toml]
-  - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v3.0.0-alpha.9-for-vscode
-    hooks:
-      - id: prettier
-        args: [--plugin=prettier-plugin-toml]
-        additional_dependencies:
-          - [email protected]
-          - [email protected]
   - repo: https://github.com/hadialqattan/pycln
     rev: v2.1.5
     hooks:
       - id: pycln
         args: [--config=python/pyproject.toml]
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.4.0
-    hooks:
-      - id: pyupgrade
-        args: [--py38-plus, --keep-runtime-typing]
-  - repo: https://github.com/pycqa/pylint
-    rev: v3.0.0a6
-    hooks:
-      - id: pylint
-        args: [--rcfile=python/pylintrc]
-  - repo: https://github.com/pycqa/flake8
-    rev: "6.0.0"
-    hooks:
-      - id: flake8
-        args: ["--ignore=E501,W503,E203,B024,B028"]
-        additional_dependencies:
-          - flake8-bugbear==23.3.23
-          - flake8-comprehensions==3.12.0
   - repo: https://github.com/executablebooks/mdformat
     rev: 0.7.16
     hooks:

python/pyiceberg/avro/decoder_fast.pyi

@@ -15,8 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from typing import Tuple, Dict
-
 from pyiceberg.avro.decoder import BinaryDecoder
 
 class CythonBinaryDecoder(BinaryDecoder):
@@ -30,9 +28,9 @@ class CythonBinaryDecoder(BinaryDecoder):
         pass
     def read_int(self) -> int:
         pass
-    def read_ints(self, count: int) -> Tuple[int, ...]:
+    def read_ints(self, count: int) -> tuple[int, ...]:
         pass
-    def read_int_bytes_dict(self, count: int, dest: Dict[int, bytes]) -> None:
+    def read_int_bytes_dict(self, count: int, dest: dict[int, bytes]) -> None:
         pass
     def read_bytes(self) -> bytes:
         pass

python/pyiceberg/avro/decoder_fast.pyx

@@ -23,6 +23,7 @@ from libc.stdint cimport uint64_t, int64_t
 
 import array
 
+
 cdef extern from "decoder_basic.c":
   void decode_zigzag_ints(const unsigned char **buffer, const uint64_t count, uint64_t *result);
   void skip_zigzag_int(const unsigned char **buffer);

python/pyiceberg/catalog/hive.py

@@ -29,16 +29,16 @@
 from urllib.parse import urlparse
 
 from hive_metastore.ThriftHiveMetastore import Client
-from hive_metastore.ttypes import AlreadyExistsException
-from hive_metastore.ttypes import Database as HiveDatabase
 from hive_metastore.ttypes import (
+    AlreadyExistsException,
     FieldSchema,
     InvalidOperationException,
     MetaException,
     NoSuchObjectException,
     SerDeInfo,
     StorageDescriptor,
 )
+from hive_metastore.ttypes import Database as HiveDatabase
 from hive_metastore.ttypes import Table as HiveTable
 from thrift.protocol import TBinaryProtocol
 from thrift.transport import TSocket, TTransport

python/pyiceberg/table/metadata.py

@@ -28,9 +28,8 @@
     Union,
 )
 
-from pydantic import Field
+from pydantic import Field, model_validator
 from pydantic import ValidationError as PydanticValidationError
-from pydantic import model_validator
 from typing_extensions import Annotated
 
 from pyiceberg.exceptions import ValidationError

python/pyiceberg/transforms.py

@@ -20,9 +20,8 @@
 from abc import ABC, abstractmethod
 from enum import IntEnum
 from functools import singledispatch
-from typing import Any, Callable, Generic
+from typing import Any, Callable, Generic, Optional, TypeVar
 from typing import Literal as LiteralType
-from typing import Optional, TypeVar
 from uuid import UUID
 
 import mmh3

python/pyproject.toml

@@ -126,13 +126,6 @@ markers = [
 line-length = 130
 target-version = ['py38']
 
-[tool.isort]
-src_paths = ["pyiceberg/", "tests/"]
-multi_line_output = 3
-profile = 'black'
-line_length = 130
-force_grid_wrap = 4
-
 [tool.pycln]
 all = true
 
@@ -295,3 +288,57 @@ ignore_missing_imports = true
 
 [tool.coverage.run]
 source = ['pyiceberg/']
+
+[tool.ruff]
+src = ['pyiceberg','tests']
+extend-exclude = ["dev/provision.py"]
+# Enable the pycodestyle (`E`) and Pyflakes (`F`) rules by default.
+# Unlike Flake8, Ruff doesn't enable pycodestyle warnings (`W`) or
+# McCabe complexity (`C901`) by default.
+select = ["E", "F", "W", "I", "UP"]
+ignore = ["E501","E203","B024","B028"]
+
+# Allow autofix for all enabled rules (when `--fix`) is provided.
+fixable = ["ALL"]
+unfixable = []
+
+# Exclude a variety of commonly ignored directories.
+exclude = [
+    ".bzr",
+    ".direnv",
+    ".eggs",
+    ".git",
+    ".git-rewrite",
+    ".hg",
+    ".mypy_cache",
+    ".nox",
+    ".pants.d",
+    ".pytype",
+    ".ruff_cache",
+    ".svn",
+    ".tox",
+    ".venv",
+    "__pypackages__",
+    "_build",
+    "buck-out",
+    "build",
+    "dist",
+    "node_modules",
+    "venv",
+]
+per-file-ignores = {}
+# Ignore _all_ violations.
+# Same as Black.
+line-length = 130
+
+# Allow unused variables when underscore-prefixed.
+dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$"
+
+[tool.ruff.pyupgrade]
+# Preserve types, even if a file imports `from __future__ import annotations`.
+keep-runtime-typing = true
+
+[tool.ruff.isort]
+detect-same-package = true
+lines-between-types = 0
+section-order = ["future", "standard-library", "third-party", "first-party", "local-folder"]

python/tests/avro/test_decoder.py

@@ -24,8 +24,7 @@
 from unittest.mock import MagicMock, patch
 
 import pytest
-
-from pyiceberg.avro.decoder import BinaryDecoder, StreamingBinaryDecoder, new_decoder
+from pyiceberg.avro.decoder import BinaryDecoder, StreamingBinaryDecoder, new_decoder  # type: ignore
 from pyiceberg.avro.decoder_fast import CythonBinaryDecoder
 from pyiceberg.avro.resolver import resolve
 from pyiceberg.io import InputStream

python/tests/avro/test_file.py

@@ -21,11 +21,10 @@
 from typing import Any
 from uuid import UUID
 
+import pyiceberg.avro.file as avro
 import pytest
 from _decimal import Decimal
 from fastavro import reader, writer
-
-import pyiceberg.avro.file as avro
 from pyiceberg.avro.codecs import DeflateCodec
 from pyiceberg.avro.file import META_SCHEMA, AvroFileHeader
 from pyiceberg.io.pyarrow import PyArrowFileIO

python/tests/avro/test_reader.py

@@ -19,7 +19,6 @@
 from typing import Callable
 
 import pytest
-
 from pyiceberg.avro.decoder import BinaryDecoder, StreamingBinaryDecoder
 from pyiceberg.avro.decoder_fast import CythonBinaryDecoder
 from pyiceberg.avro.file import AvroFile

python/tests/avro/test_resolver.py

@@ -20,7 +20,6 @@
 
 import pytest
 from pydantic import Field
-
 from pyiceberg.avro.file import AvroFile
 from pyiceberg.avro.reader import (
     DecimalReader,

python/tests/avro/test_writer.py

@@ -22,7 +22,6 @@
 
 import pytest
 from _decimal import Decimal
-
 from pyiceberg.avro.encoder import BinaryEncoder
 from pyiceberg.avro.resolver import construct_writer
 from pyiceberg.avro.writer import (

python/tests/catalog/integration_test_dynamodb.py

@@ -20,7 +20,6 @@
 import boto3
 import pytest
 from botocore.exceptions import ClientError
-
 from pyiceberg.catalog import Catalog
 from pyiceberg.catalog.dynamodb import DynamoDbCatalog
 from pyiceberg.exceptions import (

python/tests/catalog/integration_test_glue.py

@@ -20,7 +20,6 @@
 import boto3
 import pytest
 from botocore.exceptions import ClientError
-
 from pyiceberg.catalog import Catalog
 from pyiceberg.catalog.glue import GlueCatalog
 from pyiceberg.exceptions import (

python/tests/catalog/test_base.py

@@ -25,7 +25,6 @@
 )
 
 import pytest
-
 from pyiceberg.catalog import (
     Catalog,
     Identifier,

python/tests/catalog/test_dynamodb.py

@@ -18,7 +18,6 @@
 
 import pytest
 from moto import mock_dynamodb
-
 from pyiceberg.catalog import METADATA_LOCATION, TABLE_TYPE
 from pyiceberg.catalog.dynamodb import (
     DYNAMODB_COL_CREATED_AT,

python/tests/catalog/test_glue.py

@@ -19,7 +19,6 @@
 
 import pytest
 from moto import mock_glue
-
 from pyiceberg.catalog.glue import GlueCatalog
 from pyiceberg.exceptions import (
     NamespaceAlreadyExistsError,

python/tests/catalog/test_hive.py

@@ -19,9 +19,8 @@
 from unittest.mock import MagicMock, patch
 
 import pytest
-from hive_metastore.ttypes import AlreadyExistsException
-from hive_metastore.ttypes import Database as HiveDatabase
 from hive_metastore.ttypes import (
+    AlreadyExistsException,
     FieldSchema,
     InvalidOperationException,
     MetaException,
@@ -30,8 +29,8 @@
     SkewedInfo,
     StorageDescriptor,
 )
+from hive_metastore.ttypes import Database as HiveDatabase
 from hive_metastore.ttypes import Table as HiveTable
-
 from pyiceberg.catalog import PropertiesUpdateSummary
 from pyiceberg.catalog.hive import HiveCatalog, _construct_hive_storage_descriptor
 from pyiceberg.exceptions import (

python/tests/catalog/test_rest.py

@@ -17,10 +17,8 @@
 # pylint: disable=redefined-outer-name,unused-argument
 from uuid import UUID
 
-import pytest
-from requests_mock import Mocker
-
 import pyiceberg
+import pytest
 from pyiceberg.catalog import PropertiesUpdateSummary, Table
 from pyiceberg.catalog.rest import RestCatalog
 from pyiceberg.exceptions import (
@@ -44,6 +42,7 @@
     NestedField,
     StringType,
 )
+from requests_mock import Mocker
 
 TEST_URI = "https://iceberg-test-catalog/"
 TEST_CREDENTIALS = "client:secret"

python/tests/catalog/test_sql.py

@@ -20,9 +20,6 @@
 from typing import Generator, List
 
 import pytest
-from pytest import TempPathFactory
-from sqlalchemy.exc import ArgumentError, IntegrityError
-
 from pyiceberg.catalog import Identifier
 from pyiceberg.catalog.sql import SqlCatalog
 from pyiceberg.exceptions import (
@@ -41,6 +38,8 @@
     SortOrder,
 )
 from pyiceberg.transforms import IdentityTransform
+from pytest import TempPathFactory
+from sqlalchemy.exc import ArgumentError, IntegrityError
 
 
 @pytest.fixture(name="warehouse", scope="session")

python/tests/cli/test_console.py

@@ -18,15 +18,14 @@
 
 import pytest
 from click.testing import CliRunner
-from pytest_mock import MockFixture
-
 from pyiceberg.cli.console import run
 from pyiceberg.partitioning import PartitionField, PartitionSpec
 from pyiceberg.schema import Schema
 from pyiceberg.transforms import IdentityTransform
 from pyiceberg.typedef import Properties
 from pyiceberg.types import LongType, NestedField
 from pyiceberg.utils.config import Config
+from pytest_mock import MockFixture
 from tests.catalog.test_base import InMemoryCatalog
 
 

python/tests/conftest.py

@@ -53,7 +53,6 @@
 import botocore.model
 import pytest
 from moto import mock_dynamodb, mock_glue, mock_s3
-
 from pyiceberg import schema
 from pyiceberg.catalog import Catalog
 from pyiceberg.catalog.noop import NoopCatalog

python/tests/expressions/test_evaluator.py

@@ -18,7 +18,6 @@
 from typing import Any
 
 import pytest
-
 from pyiceberg.conversions import to_bytes
 from pyiceberg.expressions import (
     And,

python/tests/expressions/test_expressions.py

@@ -22,8 +22,6 @@
 from typing import Any
 
 import pytest
-from typing_extensions import assert_type
-
 from pyiceberg.expressions import (
     AlwaysFalse,
     AlwaysTrue,
@@ -78,6 +76,7 @@
     UUIDType,
 )
 from pyiceberg.utils.singleton import Singleton
+from typing_extensions import assert_type
 
 
 class ExpressionA(BooleanExpression, Singleton):