From 4d0028b56e2fc607a4a39dc6ba65a924ef59d282 Mon Sep 17 00:00:00 2001 From: Shreyas Damle Date: Tue, 27 Aug 2024 16:15:47 +0530 Subject: [PATCH 1/8] Database minor fixes --- pebblo/app/models/db_models.py | 1 + .../app/service/loader/document/document.py | 1 + .../app/service/loader/loader_doc_service.py | 6 +- pebblo/app/service/local_ui/loader_apps.py | 227 ++++++++++-------- 4 files changed, 132 insertions(+), 103 deletions(-) diff --git a/pebblo/app/models/db_models.py b/pebblo/app/models/db_models.py index 642fd57a..4ceaaae2 100644 --- a/pebblo/app/models/db_models.py +++ b/pebblo/app/models/db_models.py @@ -160,6 +160,7 @@ class AiDocument(BaseModel): loaderSourcePath: str metadata: Metadata sourcePath: str + sourceSize: int lastIngested: str owner: str userIdentities: Optional[List[str]] = [] diff --git a/pebblo/app/service/loader/document/document.py b/pebblo/app/service/loader/document/document.py index aea11ea0..971fb147 100644 --- a/pebblo/app/service/loader/document/document.py +++ b/pebblo/app/service/loader/document/document.py @@ -40,6 +40,7 @@ def _get_or_create_document(self, doc, data_source): "dataSourceId": data_source.get("id"), "metadata": metadata, "sourcePath": doc.get("source_path"), + "sourceSize": doc.get("source_path_size"), "loaderSourcePath": data_source.get("sourcePath"), "owner": doc.get("file_owner"), "userIdentities": doc.get("authorized_identities", []), diff --git a/pebblo/app/service/loader/loader_doc_service.py b/pebblo/app/service/loader/loader_doc_service.py index 2e156a13..0d068833 100644 --- a/pebblo/app/service/loader/loader_doc_service.py +++ b/pebblo/app/service/loader/loader_doc_service.py @@ -330,9 +330,11 @@ def process_request(self, data): logger.error(message) logger.info("Rollback the changes") self.db.session.rollback() - return self._create_return_response(message, 500) + return self._create_return_response(message=message, status_code=500) else: message = "Loader Doc API Request processed successfully" - return self._create_return_response(message, output=loader_response_output) + return self._create_return_response( + message=message, output=loader_response_output + ) finally: self.db.session.close() diff --git a/pebblo/app/service/local_ui/loader_apps.py b/pebblo/app/service/local_ui/loader_apps.py index 8084c94f..001f92da 100644 --- a/pebblo/app/service/local_ui/loader_apps.py +++ b/pebblo/app/service/local_ui/loader_apps.py @@ -81,42 +81,49 @@ def _findings_for_app_entities( """ for entity, entity_data in app_data.get("docEntities").items(): - entity_count += entity_data.get("count") - self.loader_findings += entity_data.get("count") - - findings_exists = False - for findings in self.loader_findings_list: - if findings.get("labelName") == entity: - findings_exists = True - findings["findings"] += entity_data["count"] - findings["snippetCount"] += len(entity_data["snippetIds"]) - findings["fileCount"] = len(app_data["documents"]) - total_snippet_count += findings["snippetCount"] - snippets.extend( - self._get_snippet_details( - entity_data["snippetIds"], app_data["owner"], entity + try: + entity_count += entity_data.get("count") + self.loader_findings += entity_data.get("count") + + findings_exists = False + for findings in self.loader_findings_list: + if findings.get("labelName") == entity: + findings_exists = True + findings["findings"] += entity_data["count"] + findings["snippetCount"] += len( + entity_data["snippetIds"] ) - ) - break - if not findings_exists: - logger.debug("finding not exist") - findings = { - "appName": app_data["name"], - "labelName": entity, - "findings": entity_data["count"], - "findingsType": "entities", - "snippetCount": len(entity_data["snippetIds"]), - "fileCount": len(app_data["documents"]), - "snippets": self._get_snippet_details( - entity_data["snippetIds"], app_data["owner"], entity - ), - } - total_snippet_count += findings["snippetCount"] - shallow_copy = findings.copy() - self.loader_findings_list.append(shallow_copy) - del findings["snippets"] - self.loader_findings_summary_list.append(findings) - return entity_count, snippets, total_snippet_count + findings["fileCount"] = len(app_data["documents"]) + total_snippet_count += findings["snippetCount"] + snippets.extend( + self._get_snippet_details( + entity_data["snippetIds"], app_data["owner"] + ) + ) + break + if not findings_exists: + logger.debug("finding not exist") + findings = { + "appName": app_data["name"], + "labelName": entity, + "findings": entity_data["count"], + "findingsType": "entities", + "snippetCount": len(entity_data["snippetIds"]), + "fileCount": len(app_data["documents"]), + "snippets": self._get_snippet_details( + entity_data["snippetIds"], app_data["owner"] + ), + } + total_snippet_count += findings["snippetCount"] + shallow_copy = findings.copy() + self.loader_findings_list.append(shallow_copy) + del findings["snippets"] + self.loader_findings_summary_list.append(findings) + return entity_count, snippets, total_snippet_count + except Exception as err: + logger.error( + f"Failed in getting docEnties details, Error: {err}" + ) def _findings_for_app_topics( self, app_data, snippets, total_snippet_count, topic_count @@ -124,43 +131,49 @@ def _findings_for_app_topics( """ This function finds findings for apps with topics """ - for topic, topic_data in app_data.get("docTopics").items(): - topic_count += topic_data.get("count") - self.loader_findings += topic_data.get("count") - - findings_exists = False - for findings in self.loader_findings_list: - if findings.get("labelName") == topic: - findings_exists = True - findings["findings"] += topic_data["count"] - findings["snippetCount"] += len(topic_data["snippetIds"]) - findings["fileCount"] = len(app_data["documents"]) - total_snippet_count += findings["snippetCount"] - snippets.extend( - self._get_snippet_details( - topic_data["snippetIds"], app_data["owner"], topic + try: + topic_count += topic_data.get("count") + self.loader_findings += topic_data.get("count") + + findings_exists = False + for findings in self.loader_findings_list: + if findings.get("labelName") == topic: + findings_exists = True + findings["findings"] += topic_data["count"] + findings["snippetCount"] += len( + topic_data["snippetIds"] ) - ) - break - if not findings_exists: - findings = { - "appName": app_data["name"], - "labelName": topic, - "findings": topic_data["count"], - "findingsType": "topics", - "snippetCount": len(topic_data["snippetIds"]), - "fileCount": len(app_data["documents"]), - "snippets": self._get_snippet_details( - topic_data["snippetIds"], app_data["owner"], topic - ), - } - total_snippet_count += findings["snippetCount"] - shallow_copy = findings.copy() - self.loader_findings_list.append(shallow_copy) - del findings["snippets"] - self.loader_findings_summary_list.append(findings) - return topic_count, snippets, total_snippet_count + findings["fileCount"] = len(app_data["documents"]) + total_snippet_count += findings["snippetCount"] + snippets.extend( + self._get_snippet_details( + topic_data["snippetIds"], app_data["owner"] + ) + ) + break + if not findings_exists: + findings = { + "appName": app_data["name"], + "labelName": topic, + "findings": topic_data["count"], + "findingsType": "topics", + "snippetCount": len(topic_data["snippetIds"]), + "fileCount": len(app_data["documents"]), + "snippets": self._get_snippet_details( + topic_data["snippetIds"], app_data["owner"] + ), + } + total_snippet_count += findings["snippetCount"] + shallow_copy = findings.copy() + self.loader_findings_list.append(shallow_copy) + del findings["snippets"] + self.loader_findings_summary_list.append(findings) + return topic_count, snippets, total_snippet_count + except Exception as err: + logger.error( + f"Failed in getting docTopics details, Error: {err}" + ) def _update_loader_datasource( self, app_data, entity_count, topic_count, total_snippet_count @@ -168,51 +181,63 @@ def _update_loader_datasource( """ This function updates loader datasource details and count """ - _, data_sources = self.db.query( AiDataSourceTable, {"loadId": app_data.get("id")} ) for data_source in data_sources: - ds_data = data_source.data - ds_obj = { - "appName": ds_data["appName"], - "name": ds_data["loader"], - "sourcePath": ds_data["sourcePath"], - "sourceType": ds_data["sourceType"], - "findingsEntities": entity_count, - "findingsTopics": topic_count, - "totalSnippetCount": total_snippet_count, - "displayedSnippetCount": min( - ReportConstants.SNIPPET_LIMIT.value, total_snippet_count - ), - } - self.loader_data_source_list.append(ds_obj) + try: + ds_data = data_source.data + ds_obj = { + "appName": ds_data["appName"], + "name": ds_data["loader"], + "sourcePath": ds_data["sourcePath"], + "sourceType": ds_data["sourceType"], + "findingsEntities": entity_count, + "findingsTopics": topic_count, + "totalSnippetCount": total_snippet_count, + "displayedSnippetCount": min( + ReportConstants.SNIPPET_LIMIT.value, total_snippet_count + ), + } + self.loader_data_source_list.append(ds_obj) + except Exception as err: + logger.error( + f"Failed in getting data source details of {data_source.data}, Error: {err}" + ) - # Data Source Count - self.loader_data_source = len(self.loader_data_source_list) + # Documents with findings Count + self.loader_files_findings = len(self.loader_document_with_findings_list) def _get_documents_with_findings(self, app_data): """ Fetch required data for DocumentWithFindings """ - _, documents = self.db.query(AiDocumentTable, {"loadId": app_data.get("id")}) + _, documents = self.db.query( + AiDocumentTable, {"loadId": app_data.get("id")} + ) loader_document_with_findings = app_data.get("documentsWithFindings") documents_with_findings_data = [] for document in documents: - document_detail = document.data - if document_detail["sourcePath"] in loader_document_with_findings: - document_obj = { - "appName": document_detail["appName"], - "owner": document_detail["owner"], - "sourceName": "", - "sourceFilePath": document_detail["sourcePath"], - "lastModified": document_detail["lastIngested"], - "findingsEntities": len(document_detail["topics"].keys()), - "findingsTopics": len(document_detail["entities"].keys()), - "authorizedIdentities": document_detail["userIdentities"], - } - documents_with_findings_data.append(document_obj) + try: + document_detail = document.data + if document_detail["sourcePath"] in loader_document_with_findings: + document_obj = { + "appName": document_detail["appName"], + "owner": document_detail["owner"], + "sourceSize": document_detail["sourceSize"], + "sourceFilePath": document_detail["sourcePath"], + "lastModified": document_detail["lastIngested"], + "findingsEntities": len(document_detail["entities"].keys()), + "findingsTopics": len(document_detail["topics"].keys()), + "authorizedIdentities": document_detail["userIdentities"], + } + documents_with_findings_data.append(document_obj) + except Exception as err: + logger.error( + f"Failed in getting doc details of {document.data}, Error: {err}" + ) + continue self.loader_document_with_findings_list.extend(documents_with_findings_data) From 7bd07eaf69b2625710f0e7844fd48d348ccdd6b2 Mon Sep 17 00:00:00 2001 From: Shreyas Damle Date: Fri, 30 Aug 2024 12:04:56 +0530 Subject: [PATCH 2/8] Snippet Limit Set --- pebblo/app/service/local_ui/loader_apps.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pebblo/app/service/local_ui/loader_apps.py b/pebblo/app/service/local_ui/loader_apps.py index 001f92da..2d5084ff 100644 --- a/pebblo/app/service/local_ui/loader_apps.py +++ b/pebblo/app/service/local_ui/loader_apps.py @@ -44,6 +44,8 @@ def _get_snippet_details(self, snippet_ids, owner, label_name): response = [] for snippet_id in snippet_ids: + if len(response) >= ReportConstants.SNIPPET_LIMIT.value: + break status, output = self.db.query(AiSnippetsTable, {"id": snippet_id}) if not status or len(output) == 0: continue From 12d04870fca383df05b68eb5c8357c8f82e0f9b5 Mon Sep 17 00:00:00 2001 From: Shreyas Damle Date: Mon, 2 Sep 2024 12:46:43 +0530 Subject: [PATCH 3/8] Changes after testing. --- .../app/service/loader/document/document.py | 2 +- pebblo/app/service/local_ui/loader_apps.py | 35 +++++++++---------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/pebblo/app/service/loader/document/document.py b/pebblo/app/service/loader/document/document.py index 971fb147..0fb4cd2d 100644 --- a/pebblo/app/service/loader/document/document.py +++ b/pebblo/app/service/loader/document/document.py @@ -40,7 +40,7 @@ def _get_or_create_document(self, doc, data_source): "dataSourceId": data_source.get("id"), "metadata": metadata, "sourcePath": doc.get("source_path"), - "sourceSize": doc.get("source_path_size"), + "sourceSize": doc.get("source_path_size", 0), "loaderSourcePath": data_source.get("sourcePath"), "owner": doc.get("file_owner"), "userIdentities": doc.get("authorized_identities", []), diff --git a/pebblo/app/service/local_ui/loader_apps.py b/pebblo/app/service/local_ui/loader_apps.py index 2d5084ff..286758a6 100644 --- a/pebblo/app/service/local_ui/loader_apps.py +++ b/pebblo/app/service/local_ui/loader_apps.py @@ -37,7 +37,7 @@ def __init__(self): self.loader_document_with_findings_list = [] self.loader_findings_summary_list = [] - def _get_snippet_details(self, snippet_ids, owner, label_name): + def _get_snippet_details(self, snippet_ids: list, owner: str, label_name: str): """ This function finds snippet details based on labels """ @@ -46,8 +46,8 @@ def _get_snippet_details(self, snippet_ids, owner, label_name): for snippet_id in snippet_ids: if len(response) >= ReportConstants.SNIPPET_LIMIT.value: break - status, output = self.db.query(AiSnippetsTable, {"id": snippet_id}) - if not status or len(output) == 0: + result, output = self.db.query(AiSnippetsTable, {"id": snippet_id}) + if not result or len(output) == 0: continue snippet_details = output[0].data entity_details = {} @@ -81,7 +81,6 @@ def _findings_for_app_entities( """ This function finds findings for apps with entities """ - for entity, entity_data in app_data.get("docEntities").items(): try: entity_count += entity_data.get("count") @@ -95,25 +94,24 @@ def _findings_for_app_entities( findings["snippetCount"] += len( entity_data["snippetIds"] ) - findings["fileCount"] = len(app_data["documents"]) + findings["fileCount"] = len(app_data.get("documents", [])) total_snippet_count += findings["snippetCount"] snippets.extend( self._get_snippet_details( - entity_data["snippetIds"], app_data["owner"] + entity_data["snippetIds"], app_data["owner"], entity ) ) break if not findings_exists: - logger.debug("finding not exist") findings = { "appName": app_data["name"], "labelName": entity, - "findings": entity_data["count"], + "findings": entity_data.get("count", 0), "findingsType": "entities", - "snippetCount": len(entity_data["snippetIds"]), - "fileCount": len(app_data["documents"]), + "snippetCount": len(entity_data.get("snippetIds", [])), + "fileCount": len(app_data.get("documents", [])), "snippets": self._get_snippet_details( - entity_data["snippetIds"], app_data["owner"] + entity_data.get("snippetIds", []), app_data["owner"], entity ), } total_snippet_count += findings["snippetCount"] @@ -124,7 +122,7 @@ def _findings_for_app_entities( return entity_count, snippets, total_snippet_count except Exception as err: logger.error( - f"Failed in getting docEnties details, Error: {err}" + f"Failed in getting docEntities details, Error: {err}" ) def _findings_for_app_topics( @@ -144,13 +142,13 @@ def _findings_for_app_topics( findings_exists = True findings["findings"] += topic_data["count"] findings["snippetCount"] += len( - topic_data["snippetIds"] + topic_data.get("snippetIds", []) ) - findings["fileCount"] = len(app_data["documents"]) + findings["fileCount"] = len(app_data.get("documents", [])) total_snippet_count += findings["snippetCount"] snippets.extend( self._get_snippet_details( - topic_data["snippetIds"], app_data["owner"] + topic_data.get("snippetIds", []), app_data["owner"], topic ) ) break @@ -160,10 +158,10 @@ def _findings_for_app_topics( "labelName": topic, "findings": topic_data["count"], "findingsType": "topics", - "snippetCount": len(topic_data["snippetIds"]), - "fileCount": len(app_data["documents"]), + "snippetCount": len(topic_data.get("snippetIds", [])), + "fileCount": len(app_data.get("documents", [])), "snippets": self._get_snippet_details( - topic_data["snippetIds"], app_data["owner"] + topic_data.get("snippetIds", []), app_data["owner"], topic ), } total_snippet_count += findings["snippetCount"] @@ -361,7 +359,6 @@ def get_loader_app_details(self, db, app_name): documentsWithFindings=self.loader_document_with_findings_list, dataSource=self.loader_data_source_list, ) - report_data = self._generate_final_report( loader_app, loader_response.dict() ) From 6108876dd1c53cc46912ca938a650aaa1c514be1 Mon Sep 17 00:00:00 2001 From: Shreyas Damle Date: Mon, 2 Sep 2024 13:58:42 +0530 Subject: [PATCH 4/8] Minor fix. --- pebblo/app/service/local_ui/loader_apps.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pebblo/app/service/local_ui/loader_apps.py b/pebblo/app/service/local_ui/loader_apps.py index 286758a6..e97cea07 100644 --- a/pebblo/app/service/local_ui/loader_apps.py +++ b/pebblo/app/service/local_ui/loader_apps.py @@ -81,11 +81,10 @@ def _findings_for_app_entities( """ This function finds findings for apps with entities """ - for entity, entity_data in app_data.get("docEntities").items(): + for entity, entity_data in app_data.get("docEntities", {}).items(): try: entity_count += entity_data.get("count") self.loader_findings += entity_data.get("count") - findings_exists = False for findings in self.loader_findings_list: if findings.get("labelName") == entity: @@ -131,7 +130,7 @@ def _findings_for_app_topics( """ This function finds findings for apps with topics """ - for topic, topic_data in app_data.get("docTopics").items(): + for topic, topic_data in app_data.get("docTopics", {}).items(): try: topic_count += topic_data.get("count") self.loader_findings += topic_data.get("count") From 6018cbaf197f9c3d7e50c2aad2c7295c4fa1811c Mon Sep 17 00:00:00 2001 From: Shreyas Damle Date: Mon, 2 Sep 2024 14:37:18 +0530 Subject: [PATCH 5/8] Changes after testing. --- pebblo/app/service/local_ui/loader_apps.py | 35 ++++++++++++---------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/pebblo/app/service/local_ui/loader_apps.py b/pebblo/app/service/local_ui/loader_apps.py index e97cea07..1560b44c 100644 --- a/pebblo/app/service/local_ui/loader_apps.py +++ b/pebblo/app/service/local_ui/loader_apps.py @@ -41,7 +41,6 @@ def _get_snippet_details(self, snippet_ids: list, owner: str, label_name: str): """ This function finds snippet details based on labels """ - response = [] for snippet_id in snippet_ids: if len(response) >= ReportConstants.SNIPPET_LIMIT.value: @@ -83,21 +82,21 @@ def _findings_for_app_entities( """ for entity, entity_data in app_data.get("docEntities", {}).items(): try: - entity_count += entity_data.get("count") - self.loader_findings += entity_data.get("count") + entity_count += entity_data.get("count", 0) + self.loader_findings += entity_data.get("count", 0) findings_exists = False for findings in self.loader_findings_list: if findings.get("labelName") == entity: findings_exists = True findings["findings"] += entity_data["count"] findings["snippetCount"] += len( - entity_data["snippetIds"] + entity_data.get("snippetIds", []) ) findings["fileCount"] = len(app_data.get("documents", [])) total_snippet_count += findings["snippetCount"] snippets.extend( self._get_snippet_details( - entity_data["snippetIds"], app_data["owner"], entity + entity_data.get("snippetIds", []), app_data["owner"], entity ) ) break @@ -118,12 +117,14 @@ def _findings_for_app_entities( self.loader_findings_list.append(shallow_copy) del findings["snippets"] self.loader_findings_summary_list.append(findings) - return entity_count, snippets, total_snippet_count + except Exception as err: logger.error( f"Failed in getting docEntities details, Error: {err}" ) + return entity_count, snippets, total_snippet_count + def _findings_for_app_topics( self, app_data, snippets, total_snippet_count, topic_count ): @@ -132,14 +133,14 @@ def _findings_for_app_topics( """ for topic, topic_data in app_data.get("docTopics", {}).items(): try: - topic_count += topic_data.get("count") - self.loader_findings += topic_data.get("count") + topic_count += topic_data.get("count", 0) + self.loader_findings += topic_data.get("count", 0) findings_exists = False for findings in self.loader_findings_list: if findings.get("labelName") == topic: findings_exists = True - findings["findings"] += topic_data["count"] + findings["findings"] += topic_data.get("count", 0) findings["snippetCount"] += len( topic_data.get("snippetIds", []) ) @@ -155,7 +156,7 @@ def _findings_for_app_topics( findings = { "appName": app_data["name"], "labelName": topic, - "findings": topic_data["count"], + "findings": topic_data.get("count", 0), "findingsType": "topics", "snippetCount": len(topic_data.get("snippetIds", [])), "fileCount": len(app_data.get("documents", [])), @@ -168,12 +169,14 @@ def _findings_for_app_topics( self.loader_findings_list.append(shallow_copy) del findings["snippets"] self.loader_findings_summary_list.append(findings) - return topic_count, snippets, total_snippet_count + except Exception as err: logger.error( f"Failed in getting docTopics details, Error: {err}" ) + return topic_count, snippets, total_snippet_count + def _update_loader_datasource( self, app_data, entity_count, topic_count, total_snippet_count ): @@ -224,7 +227,7 @@ def _get_documents_with_findings(self, app_data): document_obj = { "appName": document_detail["appName"], "owner": document_detail["owner"], - "sourceSize": document_detail["sourceSize"], + "sourceSize": document_detail.get("sourceSize", 0), "sourceFilePath": document_detail["sourcePath"], "lastModified": document_detail["lastIngested"], "findingsEntities": len(document_detail["entities"].keys()), @@ -277,7 +280,7 @@ def get_findings_for_loader_app(self, app_data): owner=app_data.get("owner"), loadId=app_data.get("id"), ) - return app_details.dict() + return app_details.model_dump() def get_all_loader_apps(self): """ @@ -331,7 +334,7 @@ def get_all_loader_apps(self): message = "All loader app response prepared successfully" logger.debug(message) self.db.session.commit() - return loader_response.dict() + return loader_response.model_dump() finally: logger.debug("Closing database session for preparing all loader apps") # Closing the session @@ -359,7 +362,7 @@ def get_loader_app_details(self, db, app_name): dataSource=self.loader_data_source_list, ) report_data = self._generate_final_report( - loader_app, loader_response.dict() + loader_app, loader_response.model_dump() ) except Exception as ex: message = f"[App Detail]: Error in loader app listing. Error:{ex}" @@ -478,7 +481,7 @@ def _generate_final_report(self, app_data, raw_data): pebbloClientVersion=app_data.get("pluginVersion", ""), clientVersion=app_data.get("clientVersion", {}), ) - return report_dict.dict() + return report_dict.model_dump() def _delete(self, db, table_name, filter_query): try: From 71d050d68371125f4c1f5138fdce61f743720a79 Mon Sep 17 00:00:00 2001 From: Shreyas Damle Date: Mon, 2 Sep 2024 15:18:50 +0530 Subject: [PATCH 6/8] Handled corner case. --- pebblo/app/service/local_ui/loader_apps.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pebblo/app/service/local_ui/loader_apps.py b/pebblo/app/service/local_ui/loader_apps.py index 1560b44c..c2127f9b 100644 --- a/pebblo/app/service/local_ui/loader_apps.py +++ b/pebblo/app/service/local_ui/loader_apps.py @@ -88,7 +88,7 @@ def _findings_for_app_entities( for findings in self.loader_findings_list: if findings.get("labelName") == entity: findings_exists = True - findings["findings"] += entity_data["count"] + findings["findings"] += entity_data.get("count",0) findings["snippetCount"] += len( entity_data.get("snippetIds", []) ) @@ -230,8 +230,8 @@ def _get_documents_with_findings(self, app_data): "sourceSize": document_detail.get("sourceSize", 0), "sourceFilePath": document_detail["sourcePath"], "lastModified": document_detail["lastIngested"], - "findingsEntities": len(document_detail["entities"].keys()), - "findingsTopics": len(document_detail["topics"].keys()), + "findingsEntities": len((document_detail.get("entities", {}) or {}).keys()), + "findingsTopics": len((document_detail.get("topics", {}) or {}).keys()), "authorizedIdentities": document_detail["userIdentities"], } documents_with_findings_data.append(document_obj) From 0e126511e2c0abaf96059909007a0f47fa75dfb3 Mon Sep 17 00:00:00 2001 From: Shreyas Damle Date: Mon, 2 Sep 2024 15:20:56 +0530 Subject: [PATCH 7/8] Ruff fixes. --- pebblo/app/service/local_ui/loader_apps.py | 40 ++++++++++++---------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/pebblo/app/service/local_ui/loader_apps.py b/pebblo/app/service/local_ui/loader_apps.py index c2127f9b..3022f291 100644 --- a/pebblo/app/service/local_ui/loader_apps.py +++ b/pebblo/app/service/local_ui/loader_apps.py @@ -88,7 +88,7 @@ def _findings_for_app_entities( for findings in self.loader_findings_list: if findings.get("labelName") == entity: findings_exists = True - findings["findings"] += entity_data.get("count",0) + findings["findings"] += entity_data.get("count", 0) findings["snippetCount"] += len( entity_data.get("snippetIds", []) ) @@ -96,7 +96,9 @@ def _findings_for_app_entities( total_snippet_count += findings["snippetCount"] snippets.extend( self._get_snippet_details( - entity_data.get("snippetIds", []), app_data["owner"], entity + entity_data.get("snippetIds", []), + app_data["owner"], + entity, ) ) break @@ -119,9 +121,7 @@ def _findings_for_app_entities( self.loader_findings_summary_list.append(findings) except Exception as err: - logger.error( - f"Failed in getting docEntities details, Error: {err}" - ) + logger.error(f"Failed in getting docEntities details, Error: {err}") return entity_count, snippets, total_snippet_count @@ -148,7 +148,9 @@ def _findings_for_app_topics( total_snippet_count += findings["snippetCount"] snippets.extend( self._get_snippet_details( - topic_data.get("snippetIds", []), app_data["owner"], topic + topic_data.get("snippetIds", []), + app_data["owner"], + topic, ) ) break @@ -171,9 +173,7 @@ def _findings_for_app_topics( self.loader_findings_summary_list.append(findings) except Exception as err: - logger.error( - f"Failed in getting docTopics details, Error: {err}" - ) + logger.error(f"Failed in getting docTopics details, Error: {err}") return topic_count, snippets, total_snippet_count @@ -215,9 +215,7 @@ def _get_documents_with_findings(self, app_data): Fetch required data for DocumentWithFindings """ - _, documents = self.db.query( - AiDocumentTable, {"loadId": app_data.get("id")} - ) + _, documents = self.db.query(AiDocumentTable, {"loadId": app_data.get("id")}) loader_document_with_findings = app_data.get("documentsWithFindings") documents_with_findings_data = [] for document in documents: @@ -230,8 +228,12 @@ def _get_documents_with_findings(self, app_data): "sourceSize": document_detail.get("sourceSize", 0), "sourceFilePath": document_detail["sourcePath"], "lastModified": document_detail["lastIngested"], - "findingsEntities": len((document_detail.get("entities", {}) or {}).keys()), - "findingsTopics": len((document_detail.get("topics", {}) or {}).keys()), + "findingsEntities": len( + (document_detail.get("entities", {}) or {}).keys() + ), + "findingsTopics": len( + (document_detail.get("topics", {}) or {}).keys() + ), "authorizedIdentities": document_detail["userIdentities"], } documents_with_findings_data.append(document_obj) @@ -256,10 +258,12 @@ def get_findings_for_loader_app(self, app_data): total_snippet_count = 0 snippets = [] if app_data.get("docEntities"): - entity_count, snippets, total_snippet_count = ( - self._findings_for_app_entities( - app_data, snippets, total_snippet_count, entity_count - ) + ( + entity_count, + snippets, + total_snippet_count, + ) = self._findings_for_app_entities( + app_data, snippets, total_snippet_count, entity_count ) if app_data.get("docTopics"): From 891e2d63de06e40fde129f636bee02c376aae1bf Mon Sep 17 00:00:00 2001 From: Shreyas Damle Date: Mon, 2 Sep 2024 17:18:27 +0530 Subject: [PATCH 8/8] Incorporated review comment. --- pebblo/app/service/local_ui/loader_apps.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pebblo/app/service/local_ui/loader_apps.py b/pebblo/app/service/local_ui/loader_apps.py index 3022f291..3970a547 100644 --- a/pebblo/app/service/local_ui/loader_apps.py +++ b/pebblo/app/service/local_ui/loader_apps.py @@ -208,6 +208,7 @@ def _update_loader_datasource( ) # Documents with findings Count + self.loader_data_source = len(self.loader_data_source_list) self.loader_files_findings = len(self.loader_document_with_findings_list) def _get_documents_with_findings(self, app_data):