Merge with master

.gitignore

@@ -59,4 +59,4 @@ playground.py
 scripts/tmp/
 
 # docs
-docs/source/generated/
+docs/source/generated/

CHANGELOG.md

@@ -17,6 +17,24 @@ Changes are grouped as follows
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [6.15.0] - 2023-08-18
+### Added
+- Support for the DocumentsAPI with the implementation `client.documents`.
+- Support for advanced filtering for `Events`, `TimeSeries`, `Assets` and `Sequences`. This is available through the 
+  `.filter()` method, for example, `client.events.filter`.
+- Extended aggregation support for `Events`, `TimeSeries`, `Assets` and `Sequences`. This is available through the five 
+  methods `.aggregate_count(...)`, `aggregate_cardinality_values(...)`, `aggregate_cardinality_properties(...)`, 
+  `.aggregate_unique_values(...)`, and `.aggregate_unique_properties(...)`. For example, 
+  `client.assets.aggregate_count(...)`. 
+- Added helper methods `as_external_ids` and `as_ids` for `EventList`, `TimeSeriesList`, `AssetList`, `SequenceList`,
+  `FileMetaDataList`, `FunctionList`, `ExtractionPipelineList`, and `DataSetList`.
+
+### Deprecated
+- Added `DeprecationWarning` to methods `client.assets.aggregate_metadata_keys` and 
+  `client.assets.aggregate_metadata_values`. The use parameter the `fields` in 
+  `client.events.aggregate_unique_values` will also lead to a deprecation warning. The reason is that the endpoints 
+  these methods are using have been deprecated in the CDF API.
+
 ## [6.14.2] - 2023-08-22
 ### Fixed
 - All data modeling endpoints will now be retried. This was not the case for POST endpoints.

cognite/client/_api/data_modeling/instances.py

@@ -1,12 +1,14 @@
 from __future__ import annotations
 
 import json
+from collections.abc import Iterable
 from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Literal, Optional, Sequence, Type, Union, cast, overload
 
 from cognite.client._api_client import APIClient
 from cognite.client._constants import INSTANCES_LIST_LIMIT_DEFAULT
+from cognite.client.data_classes import filters
 from cognite.client.data_classes._base import CogniteResourceList
-from cognite.client.data_classes.data_modeling.aggregations import (
+from cognite.client.data_classes.aggregations import (
     Aggregation,
     Histogram,
     HistogramValue,
@@ -41,21 +43,40 @@
     QueryResult,
 )
 from cognite.client.data_classes.data_modeling.views import View
-from cognite.client.data_classes.filters import Filter
+from cognite.client.data_classes.filters import Filter, _validate_filter
 from cognite.client.utils._identifier import DataModelingIdentifierSequence
 
 from ._data_modeling_executor import get_data_modeling_executor
 
 if TYPE_CHECKING:
     from cognite.client import CogniteClient
 
+_DATA_MODELING_SUPPORTED_FILTERS: frozenset[type[Filter]] = frozenset(
+    {
+        filters.And,
+        filters.Or,
+        filters.Not,
+        filters.In,
+        filters.Equals,
+        filters.Exists,
+        filters.Range,
+        filters.Prefix,
+        filters.ContainsAny,
+        filters.ContainsAll,
+        filters.Nested,
+        filters.HasData,
+        filters.MatchAll,
+        filters.Overlaps,
+    }
+)
+
 
 class _NodeOrEdgeList(CogniteResourceList):
     _RESOURCE = (Node, Edge)  # type: ignore[assignment]
 
     @classmethod
     def _load(
-        cls, resource_list: list[dict[str, Any]] | str, cognite_client: Optional[CogniteClient] = None
+        cls, resource_list: Iterable[dict[str, Any]] | str, cognite_client: Optional[CogniteClient] = None
     ) -> _NodeOrEdgeList:
         resource_list = json.loads(resource_list) if isinstance(resource_list, str) else resource_list
         resources: list[Node | Edge] = [
@@ -81,7 +102,7 @@ class _NodeOrEdgeApplyResultList(CogniteResourceList):
 
     @classmethod
     def _load(
-        cls, resource_list: list[dict[str, Any]] | str, cognite_client: Optional[CogniteClient] = None
+        cls, resource_list: Iterable[dict[str, Any]] | str, cognite_client: Optional[CogniteClient] = None
     ) -> _NodeOrEdgeApplyResultList:
         resource_list = json.loads(resource_list) if isinstance(resource_list, str) else resource_list
         resources: list[NodeApplyResult | EdgeApplyResult] = [
@@ -178,6 +199,7 @@ def __call__(
         filter: Filter | dict | None = None,
     ) -> Iterator[Edge] | Iterator[EdgeList] | Iterator[Node] | Iterator[NodeList]:
         """Iterate over nodes or edges.
+
         Fetches instances as they are iterated over, so you keep a limited number of instances in memory.
 
         Args:
@@ -193,6 +215,7 @@ def __call__(
         Yields:
             Edge | Node | EdgeList | NodeList: yields Instance one by one if chunk_size is not specified, else NodeList/EdgeList objects.
         """
+        self._validate_filter(filter)
         other_params = self._create_other_params(
             include_typing=include_typing, instance_type=instance_type, sort=sort, sources=sources
         )
@@ -219,7 +242,8 @@ def __call__(
         )
 
     def __iter__(self) -> Iterator[Node]:
-        """Iterate over instances
+        """Iterate over instances.
+
         Fetches instances as they are iterated over, so you keep a limited number of instances in memory.
         Yields:
             Instance: yields Instances one by one.
@@ -580,6 +604,7 @@ def search(
                 ... query="Quentin", properties=["name"], filter=born_after_1970)
 
         """
+        self._validate_filter(filter)
         if instance_type == "node":
             list_cls: Union[Type[NodeList], Type[EdgeList]] = NodeList
         elif instance_type == "edge":
@@ -640,6 +665,8 @@ def aggregate(
         """
         if instance_type not in ("node", "edge"):
             raise ValueError(f"Invalid instance type: {instance_type}")
+        self._validate_filter(filter)
+
         body: Dict[str, Any] = {"view": view.dump(camel_case=True), "instanceType": instance_type, "limit": limit}
         aggregate_seq: Sequence[Aggregation | dict] = aggregates if isinstance(aggregates, Sequence) else [aggregates]
         body["aggregates"] = [
@@ -723,6 +750,8 @@ def histogram(
         """
         if instance_type not in ("node", "edge"):
             raise ValueError(f"Invalid instance type: {instance_type}")
+        self._validate_filter(filter)
+
         body: Dict[str, Any] = {"view": view.dump(camel_case=True), "instanceType": instance_type, "limit": limit}
 
         if isinstance(histograms, Sequence):
@@ -753,7 +782,7 @@ def histogram(
             return [HistogramValue.load(item["aggregates"][0]) for item in res.json()["items"]]
 
     def query(self, query: Query) -> QueryResult:
-        """`Advanced query interface for nodes/edges. <https://developer.cognite.com/api/v1/#tag/Instances/operation/queryContent>`_
+        """`Advanced query interface for nodes/edges <https://developer.cognite.com/api/v1/#tag/Instances/operation/queryContent>`_.
 
         The Data Modelling API exposes an advanced query interface. The query interface supports parameterization,
         recursive edge traversal, chaining of result sets, and granular property selection.
@@ -791,7 +820,7 @@ def query(self, query: Query) -> QueryResult:
         return self._query_or_sync(query, "query")
 
     def sync(self, query: Query) -> QueryResult:
-        """`Subscription to changes for nodes/edges. <https://developer.cognite.com/api/v1/#tag/Instances/operation/syncContent>`_
+        """`Subscription to changes for nodes/edges <https://developer.cognite.com/api/v1/#tag/Instances/operation/syncContent>`_.
 
         Subscribe to changes for nodes and edges in a project, matching a supplied filter.
 
@@ -912,6 +941,7 @@ def list(
                 >>> for instance_list in c.data_modeling.instances(chunk_size=100):
                 ...     instance_list # do something with the instances
         """
+        self._validate_filter(filter)
         other_params = self._create_other_params(
             include_typing=include_typing, instance_type=instance_type, sort=sort, sources=sources
         )
@@ -935,3 +965,6 @@ def list(
                 other_params=other_params,
             ),
         )
+
+    def _validate_filter(self, filter: Filter | dict | None) -> None:
+        _validate_filter(filter, _DATA_MODELING_SUPPORTED_FILTERS, type(self).__name__)

cognite/client/_api/datapoints_subscriptions.py

@@ -37,7 +37,7 @@ def _experimental_warning(self) -> None:
         )
 
     def create(self, subscription: DataPointSubscriptionCreate) -> DatapointSubscription:
-        """`Create a subscription <https://pr-2221.specs.preview.cogniteapp.com/20230101-beta.json.html#tag/Data-point-subscriptions/operation/postSubscriptions>`_
+        """`Create a subscription <https://pr-2221.specs.preview.cogniteapp.com/20230101-beta.json.html#tag/Data-point-subscriptions/operation/postSubscriptions>`_.
 
         Create a subscription that can be used to listen for changes in data points for a set of time series.
 
@@ -81,7 +81,9 @@ def create(self, subscription: DataPointSubscriptionCreate) -> DatapointSubscrip
         )
 
     def delete(self, external_id: str | Sequence[str], ignore_unknown_ids: bool = False) -> None:
-        """`Delete subscription(s). This operation cannot be undone. <https://pr-2221.specs.preview.cogniteapp.com/20230101-beta.json.html#tag/Data-point-subscriptions/operation/deleteSubscriptions>`_
+        """`Delete subscription(s) <https://pr-2221.specs.preview.cogniteapp.com/20230101-beta.json.html#tag/Data-point-subscriptions/operation/deleteSubscriptions>`_.
+
+        This operation cannot be undone.
 
         Args:
             external_id (str | Sequence[str]): External ID or list of external IDs of subscriptions to delete.
@@ -108,7 +110,7 @@ def delete(self, external_id: str | Sequence[str], ignore_unknown_ids: bool = Fa
         )
 
     def retrieve(self, external_id: str, ignore_unknown_ids: bool = False) -> Optional[DatapointSubscription]:
-        """`Retrieve one or more subscriptions by external ID. <https://pr-2221.specs.preview.cogniteapp.com/20230101-beta.json.html#tag/Data-point-subscriptions/operation/getSubscriptionsByIds>`_
+        """`Retrieve one or more subscriptions by external ID <https://pr-2221.specs.preview.cogniteapp.com/20230101-beta.json.html#tag/Data-point-subscriptions/operation/getSubscriptionsByIds>`_.
 
         Args:
             external_id (str | Sequence[str]): External ID or list of external IDs of subscriptions to retrieve.
@@ -140,7 +142,7 @@ def retrieve(self, external_id: str, ignore_unknown_ids: bool = False) -> Option
             return None
 
     def update(self, update: DataPointSubscriptionUpdate) -> DatapointSubscription:
-        """`Update a subscriptions <https://pr-2221.specs.preview.cogniteapp.com/20230101-beta.json.html#tag/Data-point-subscriptions/operation/updateSubscriptions>`_
+        """`Update a subscriptions <https://pr-2221.specs.preview.cogniteapp.com/20230101-beta.json.html#tag/Data-point-subscriptions/operation/updateSubscriptions>`_.
 
         Update a subscription. Note that Fields that are not included in the request are not changed.
         Furthermore, the subscription partition cannot be changed.
@@ -186,7 +188,7 @@ def iterate_data(
         start: str | None = None,
         limit: int = DATAPOINT_SUBSCRIPTION_DATA_LIST_LIMIT_DEFAULT,
     ) -> Iterator[DatapointSubscriptionBatch]:
-        """`Iterate over data from a given subscription. <https://pr-2221.specs.preview.cogniteapp.com/20230101-beta.json.html#tag/Data-point-subscriptions/operation/listSubscriptionData>`_
+        """`Iterate over data from a given subscription <https://pr-2221.specs.preview.cogniteapp.com/20230101-beta.json.html#tag/Data-point-subscriptions/operation/listSubscriptionData>`_.
 
         Data can be ingested datapoints and time ranges where data is deleted. This endpoint will also return changes to
         the subscription itself, that is, if time series are added or removed from the subscription.
@@ -250,7 +252,7 @@ def iterate_data(
             current_partitions = batch.partitions
 
     def list(self, limit: int = DATAPOINT_SUBSCRIPTIONS_LIST_LIMIT_DEFAULT) -> DatapointSubscriptionList:
-        """`List data point subscriptions <https://pr-2221.specs.preview.cogniteapp.com/20230101-beta.json.html#tag/Data-point-subscriptions/operation/listSubscriptions>`_
+        """`List data point subscriptions <https://pr-2221.specs.preview.cogniteapp.com/20230101-beta.json.html#tag/Data-point-subscriptions/operation/listSubscriptions>`_.
 
         Args:
             limit (int, optional): Maximum number of subscriptions to return. Defaults to 100. Set to -1, float("inf") or None