Regular maintenance 202405

.github/workflows/tests.yml

@@ -58,10 +58,12 @@ jobs:
         poe check
 
     - name: Upload coverage to Codecov
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
       with:
         files: ./coverage.xml
         flags: unittests
         env_vars: OS,PYTHON
         name: codecov-umbrella
-        fail_ci_if_error: false
+        fail_ci_if_error: true

codecov.yml

@@ -0,0 +1,13 @@
+# https://docs.codecov.io/docs/common-recipe-list
+# https://docs.codecov.io/docs/commit-status#patch-status
+
+coverage:
+  status:
+
+    project:
+      default:
+        target: 80%
+
+    patch:
+      default:
+        informational: true

doc/backlog.rst

@@ -101,6 +101,10 @@ Next steps
 - [o] Option to suppress ``NOT NULL`` constraint. /cc @seut
 - [o] Different kinds of sampling methods? /cc @seut
 - [o] Performance considerations / HTTP server
+- [o] Look at JSON Schema for DDL definition.
+
+  - https://pypi.org/project/JSONSchema2DB/
+  - https://pypi.org/project/jsonschema2ddl/
 
 Formats
 =======

examples/infer_ndjson.py

@@ -8,6 +8,7 @@
 - https://en.wikipedia.org/wiki/JSON_streaming
 - https://github.com/ndjson/ndjson.github.io/issues/1#issuecomment-109935996
 """
+
 import io
 import typing as t
 

pyproject.toml

@@ -38,26 +38,6 @@ authors = [
   { name = "Richard Pobering", email = "[email protected]" },
 ]
 requires-python = ">=3.8"
-dependencies = [
-  "click<9",
-  "colorama<1",
-  "crash",
-  "crate[sqlalchemy]",
-  "ddlgenerator<0.2",
-  "frictionless[excel,json,ods,parquet,sql]<5.6",
-  "fsspec[gcs,github,http,s3]==2024.3.1",
-  "json_stream<3",
-  "line-protocol-parser<2",
-  "odfpy<2",
-  "pandas<1.6",
-  "requests<2.32",
-  "sql-formatter<0.7",
-  "sqlmakeuper<0.2",
-  "urllib3<2",
-]
-dynamic = [
-  "version",
-]
 classifiers = [
   "Development Status :: 3 - Alpha",
   "Environment :: Console",
@@ -79,10 +59,12 @@ classifiers = [
   "Operating System :: POSIX :: Linux",
   "Operating System :: Unix",
   "Programming Language :: Python",
+  "Programming Language :: Python :: 3 :: Only",
   "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
   "Topic :: Communications",
   "Topic :: Database",
   "Topic :: Documentation",
@@ -99,14 +81,33 @@ classifiers = [
   "Topic :: Text Processing",
   "Topic :: Utilities",
 ]
+dynamic = [
+  "version",
+]
+dependencies = [
+  "click<9",
+  "colorama<1",
+  "crash",
+  "crate[sqlalchemy]",
+  "ddlgenerator<0.2",
+  "frictionless[excel,json,ods,parquet,sql]<5.6",
+  "fsspec[gcs,github,http,s3]==2024.3.1",
+  "json_stream<3",
+  "line-protocol-parser<2",
+  "odfpy<2",
+  "pandas<2",
+  "requests<2.32",
+  "sql-formatter<0.7",
+  "sqlmakeuper<0.2",
+  "urllib3<2",
+]
 [project.optional-dependencies]
 develop = [
-  "black<24",
   "hunter<4",
-  "mypy==1.8.0",
+  "mypy<1.11",
   "poethepoet<0.27",
-  "pyproject-fmt<1.8",
-  "ruff==0.1.14",
+  "pyproject-fmt<1.9",
+  "ruff<0.5",
   "types-requests<2.32",
   "validate-pyproject<0.17",
 ]
@@ -126,17 +127,14 @@ test = [
   "pytest-cov<5",
   "pytest-timeout<3",
 ]
-
 [project.urls]
 changelog = "https://github.com/daq-tools/skeem/blob/main/CHANGES.rst"
 documentation = "https://github.com/daq-tools/skeem"
 homepage = "https://github.com/daq-tools/skeem"
 repository = "https://github.com/daq-tools/skeem"
-
 [project.scripts]
 skeem = "skeem.cli:cli"
 
-
 [tool.black]
 line-length = 120
 
@@ -177,11 +175,10 @@ markers = [
   "slow",
 ]
 
-
 [tool.ruff]
 line-length = 120
 
-select = [
+lint.select = [
   # Bandit
   "S",
   # Bugbear
@@ -209,7 +206,7 @@ select = [
   "RET",
 ]
 
-extend-ignore = [
+lint.extend-ignore = [
   # zip() without an explicit strict= parameter
   "B905",
   # df is a bad variable name. Be kinder to your future self.
@@ -220,7 +217,7 @@ extend-ignore = [
   "RET505",
 ]
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 "skeem/core.py" = ["E402"]
 "skeem/fastparquet/*" = ["E722"]
 "skeem/pandas/*" = ["S101"]
@@ -246,15 +243,15 @@ check = [
 ]
 
 format = [
-  { cmd = "black ." },
+  { cmd = "ruff format" },
   # Configure Ruff not to auto-fix (remove!) unused variables (F841) and `print` statements (T201).
-  { cmd = "ruff --fix --ignore=ERA --ignore=F401 --ignore=F841 --ignore=T20 ." },
-  { cmd = "pyproject-fmt pyproject.toml" },
+  { cmd = "ruff check --fix --ignore=ERA --ignore=F401 --ignore=F841 --ignore=T20 ." },
+  { cmd = "pyproject-fmt --keep-full-version pyproject.toml" },
 ]
 
 lint = [
-  { cmd = "ruff ." },
-  { cmd = "black --check ." },
+  { cmd = "ruff format --check" },
+  { cmd = "ruff check ." },
   { cmd = "validate-pyproject pyproject.toml" },
   { cmd = "mypy" },
 ]

skeem/io.py

@@ -13,6 +13,10 @@
 from skeem.settings import PEEK_BYTES, PEEK_LINES
 from skeem.type import ContentType, ContentTypeGroup
 
+if t.TYPE_CHECKING:
+    import xarray
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -125,7 +129,8 @@
 
 
 def dataset_to_dataframe(
-    ds: "xarray.Dataset", peek_lines: int  # type: ignore[name-defined]  # noqa: F821
+    ds: "xarray.Dataset",
+    peek_lines: int,
 ) -> pd.DataFrame:
     logger.info(f"Dataset:\n{ds}")
     df = ds.to_dataframe().dropna()

skeem/pandas/io_common.py

@@ -190,8 +190,7 @@ def is_fsspec_url(url: t.Union[FilePath, BaseBuffer]) -> bool:
     something fsspec can handle
     """
     return (
-        isinstance(url, str)
-        and bool(_RFC_3986_PATTERN.match(url))
+        isinstance(url, str) and bool(_RFC_3986_PATTERN.match(url))
         # PATCH for Skeem: Let HTTP requests also be handled by `fsspec`.
         # and not url.startswith(("http://", "https://"))
     )