Merge pull request #71 from Dynatrace-James-Kitson/log-exporting

add v2 log exporting

dynatrace/environment_v2/logs.py

@@ -13,20 +13,52 @@
 See the License for the specific language governing permissions and
 limitations under the License.
 """
+from enum import Enum
 from typing import Dict, Any, Union, List
 
 from requests import Response
+from datetime import datetime
+from typing import Optional, Union, Dict, Any, List
 
 from dynatrace.http_client import HttpClient
+from dynatrace.dynatrace_object import DynatraceObject
+from dynatrace.pagination import PaginatedList
+from dynatrace.utils import timestamp_to_string
 
 
 class LogService:
-    # TODO - Add search and get
+    # TODO - Add search and aggregate
     ENDPOINT = "/api/v2/logs"
 
     def __init__(self, http_client: HttpClient):
         self.__http_client = http_client
 
+    def export(
+        self,
+        query: Optional[str] = None,
+        time_from: Optional[Union[datetime, str]] = None,
+        time_to: Optional[Union[datetime, str]] = None,
+        sort: Optional[str] = None,
+        page_size: Optional[int] = None,
+    ) -> PaginatedList["LogRecord"]:
+        """
+        Gets the log records matching the provided criteria. Retrieves all records using pagination.
+        :param query: The log search query
+        :param page_size: Number of results per page
+        :param time_from: Start of the requested timeframe
+        :param time_to: End of the requested timefram
+        :param sort: Defines the ordering of log records
+        :return A list of log records
+        """
+        params = {
+            "query": query,
+            "pageSize": page_size,
+            "from": timestamp_to_string(time_from),
+            "to": timestamp_to_string(time_to),
+            "sort": sort,
+        }
+        return PaginatedList(LogRecord, self.__http_client, "/api/v2/logs/export", params, list_item="results")
+
     def ingest(self, payload: Union[Dict[str, Any], List[Dict[str, Any]]]) -> Response:
         """
         Ingests logs into the Dynatrace log store.
@@ -35,3 +67,23 @@ def ingest(self, payload: Union[Dict[str, Any], List[Dict[str, Any]]]) -> Respon
         """
         headers = {"Content-Type": "application/json; charset=utf-8"}
         return self.__http_client.make_request(f"{self.ENDPOINT}/ingest", params=payload, method="POST", headers=headers)
+
+class LogRecord(DynatraceObject):
+    def _create_from_raw_data(self, raw_element: Dict[str, Any]):
+        self.additional_columns: dict = raw_element.get("additionalColumns")
+        self.event_type: EventType = EventType(raw_element.get("eventType"))
+        self.timestamp: datetime = datetime.utcfromtimestamp(raw_element.get("timestamp") / 1000)
+        self.content: str = raw_element.get("content")
+        self.status: LogRecordStatus = LogRecordStatus(raw_element.get("status"))
+
+class EventType(Enum):
+    K8S = "K8S"
+    LOG = "LOG"
+    SFM = "SFM"
+
+class LogRecordStatus(Enum):
+    ERROR = "ERROR"
+    INFO = "INFO"
+    NONE = "NONE"
+    NOT_APPLICABLE = "NOT_APPLICABLE"
+    WARN = "WARN"

setup.py

@@ -2,7 +2,7 @@
 
 setup(
     name="dt",
-    version="1.1.59",
+    version="1.1.60",
     packages=find_packages(),
     install_requires=["requests>=2.22"],
     tests_require=["pytest", "mock", "tox"],

test/mock_data/GET_api_v2_logs_export_8b3579556e13525.json

@@ -0,0 +1 @@
+{"results": [{"timestamp": 1683574915193, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Error testing endpoint https://192.168.222.10/: HTTPSConnectionPool(host='192.168.222.10', port=443): Max retries exceeded with url: /api/cluster?fields=uuid%2Cname%2Clocation%2Cversion%2Cstatistics (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x000002A3FBFBCDC0>, 'Connection to 192.168.222.10 timed out. (connect timeout=None)'))", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:extension-netapp-ontap"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["c89d11aa-7637-34a3-94cb-bcc07f88c1f0"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574855122, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Error testing endpoint https://192.168.222.10/: HTTPSConnectionPool(host='192.168.222.10', port=443): Max retries exceeded with url: /api/cluster?fields=uuid%2Cname%2Clocation%2Cversion%2Cstatistics (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x000001AFE338CDC0>, 'Connection to 192.168.222.10 timed out. (connect timeout=None)'))", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:extension-netapp-ontap"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["c89d11aa-7637-34a3-94cb-bcc07f88c1f0"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574795002, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Error testing endpoint https://192.168.222.10/: HTTPSConnectionPool(host='192.168.222.10', port=443): Max retries exceeded with url: /api/cluster?fields=uuid%2Cname%2Clocation%2Cversion%2Cstatistics (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x000001D1EC4CCDC0>, 'Connection to 192.168.222.10 timed out. (connect timeout=None)'))", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:extension-netapp-ontap"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["c89d11aa-7637-34a3-94cb-bcc07f88c1f0"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574764953, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Cannot extract extension from C:\\ProgramData\\dynatrace\\remotepluginmodule\\/agent/runtime\\extensions\\download\\custom_sybase-ase-health-ci: checking signature failed", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:sybase-ase-health-ci"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["64132242-27f9-39b7-a456-2b7c3bc359db"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574734903, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Error testing endpoint https://192.168.222.10/: HTTPSConnectionPool(host='192.168.222.10', port=443): Max retries exceeded with url: /api/cluster?fields=uuid%2Cname%2Clocation%2Cversion%2Cstatistics (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x000002646D46CDC0>, 'Connection to 192.168.222.10 timed out. (connect timeout=None)'))", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:extension-netapp-ontap"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["c89d11aa-7637-34a3-94cb-bcc07f88c1f0"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574704874, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Extension custom:sybase-ase-health-ci(0.0.6) not available in cache yet (queued for download)", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:sybase-ase-health-ci"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["64132242-27f9-39b7-a456-2b7c3bc359db"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574674822, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Error testing endpoint https://192.168.222.10/: HTTPSConnectionPool(host='192.168.222.10', port=443): Max retries exceeded with url: /api/cluster?fields=uuid%2Cname%2Clocation%2Cversion%2Cstatistics (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x0000029846F9CDC0>, 'Connection to 192.168.222.10 timed out. (connect timeout=None)'))", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:extension-netapp-ontap"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["c89d11aa-7637-34a3-94cb-bcc07f88c1f0"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574614732, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Error testing endpoint https://192.168.222.10/: HTTPSConnectionPool(host='192.168.222.10', port=443): Max retries exceeded with url: /api/cluster?fields=uuid%2Cname%2Clocation%2Cversion%2Cstatistics (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x00000209E87FCDC0>, 'Connection to 192.168.222.10 timed out. (connect timeout=None)'))", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:extension-netapp-ontap"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["c89d11aa-7637-34a3-94cb-bcc07f88c1f0"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574554719, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Error testing endpoint https://192.168.222.10/: HTTPSConnectionPool(host='192.168.222.10', port=443): Max retries exceeded with url: /api/cluster?fields=uuid%2Cname%2Clocation%2Cversion%2Cstatistics (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x000001EDA798CDC0>, 'Connection to 192.168.222.10 timed out. (connect timeout=None)'))", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:extension-netapp-ontap"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["c89d11aa-7637-34a3-94cb-bcc07f88c1f0"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574494522, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Error testing endpoint https://192.168.222.10/: HTTPSConnectionPool(host='192.168.222.10', port=443): Max retries exceeded with url: /api/cluster?fields=uuid%2Cname%2Clocation%2Cversion%2Cstatistics (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x00000215FC46CDC0>, 'Connection to 192.168.222.10 timed out. (connect timeout=None)'))", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:extension-netapp-ontap"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["c89d11aa-7637-34a3-94cb-bcc07f88c1f0"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574459170, "content": "OK", "status": "INFO", "eventType": "SFM", "additionalColumns": {"loglevel": ["INFO"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.active_gate.id": ["0x16f6e00b"], "dt.extension.name": ["custom:vcenter-extension"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "host.name": ["TAG009444368559"], "dt.extension.config.id": ["834f045c-02d3-31b7-91f8-8bfb3f6c050e"], "dt.extension.status": ["OK"]}}, {"timestamp": 1683574434460, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Error testing endpoint https://192.168.222.10/: HTTPSConnectionPool(host='192.168.222.10', port=443): Max retries exceeded with url: /api/cluster?fields=uuid%2Cname%2Clocation%2Cversion%2Cstatistics (Caused by ConnectTimeoutError(<urllib3.connection.HTTPSConnection object at 0x000001AD6AEDCDC0>, 'Connection to 192.168.222.10 timed out. (connect timeout=None)'))", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:extension-netapp-ontap"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["c89d11aa-7637-34a3-94cb-bcc07f88c1f0"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574409411, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Fast check failed for 1 endpoint(s). Cannot connect to database localhost:0. Exception occured: Socket fail to connect to host:address=(host=localhost)(port=3306)(type=primary). Connection refused: connect. ", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["sqlMySql"], "dt.extension.name": ["com.dynatrace.extension.mysql"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["7fb25156-5681-3f94-8f01-41a46f3d8935"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574405008, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: Cannot extract extension from C:\\ProgramData\\dynatrace\\remotepluginmodule\\/agent/runtime\\extensions\\download\\custom_sybase-ase-health-ci: checking signature failed", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:sybase-ase-health-ci"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["64132242-27f9-39b7-a456-2b7c3bc359db"], "dt.extension.status": ["CUSTOM_ERROR"]}}, {"timestamp": 1683574399171, "content": "Method=query_vsphere: Callback Method=query_vsphere took 69.6077 seconds to execute, which is longer than the interval of 60.0s", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["default"], "dt.extension.ds": ["python"], "dt.active_gate.id": ["0x16f6e00b"], "dt.extension.name": ["custom:vcenter-extension"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "host.name": ["TAG009444368559"], "dt.extension.config.id": ["834f045c-02d3-31b7-91f8-8bfb3f6c050e"], "dt.extension.status": ["GENERIC_ERROR"]}}, {"timestamp": 1683574397440, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: The monitoring configuration requires ActiveGate version 1.256.0 or later that supports data source python (runtime version min: 3.9.0) and belongs to group \"windows\"", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["windows"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:vlocity-query"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["7f3d5e87-0a6c-33db-8e4a-6b137f7d1114"], "dt.extension.status": ["NO_SUITABLE_ACTIVE_MONITOR"]}}, {"timestamp": 1683574397440, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: The monitoring configuration requires ActiveGate version 1.259.0 or later that supports data source python (runtime version min: 3.9.0) and belongs to group \"windows\"", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["windows"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:lloyds-status-page-extension"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["b1f1e72e-1b0e-3d98-aac6-5bae991eb425"], "dt.extension.status": ["NO_SUITABLE_ACTIVE_MONITOR"]}}, {"timestamp": 1683574397439, "content": "Failed to assign monitoring configuration to ActiveGate. Reason: The monitoring configuration requires ActiveGate version 1.256.0 or later that supports data source python (runtime version min: 3.9.0) and belongs to group \"windows\"", "status": "ERROR", "eventType": "SFM", "additionalColumns": {"loglevel": ["ERROR"], "dt.active_gate.group.name": ["windows"], "dt.extension.ds": ["python"], "dt.extension.name": ["custom:remote-unix"], "log.source": ["dsfm"], "dt.event.key": ["extension.status"], "dt.extension.config.id": ["f1d59951-5fa2-35bf-9ead-1117e4e31981"], "dt.extension.status": ["NO_SUITABLE_ACTIVE_MONITOR"]}}], "pageSize": 18, "totalCount": 18}

test/test_logs.py

@@ -0,0 +1,22 @@
+from dynatrace import Dynatrace
+from datetime import datetime
+
+from dynatrace.environment_v2.logs import LogRecord, EventType, LogRecordStatus
+from dynatrace.pagination import PaginatedList
+
+
+def test_export(dt: Dynatrace):
+    logs = dt.logs.export(time_from="now-10m")
+    assert isinstance(logs, PaginatedList)
+
+    logs = list(logs)
+    assert len(logs) == 18
+
+    first = logs[0]
+    assert isinstance(first, LogRecord)
+    assert first.additional_columns['dt.extension.ds'][0] == "python"
+    assert first.content.startswith("Failed to assign")
+    assert first.event_type == EventType.SFM
+    assert first.status == LogRecordStatus.ERROR
+    assert first.timestamp == datetime.utcfromtimestamp(1683574915193 / 1000)
+