Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Confidence score changes for DB #523

Merged
merged 3 commits into from
Sep 2, 2024
Merged

Confidence score changes for DB #523

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
9e7ab78
Confidence score changes for DB
Aug 30, 2024
20a3964
Merge branch 'main' into dristy_confidence-score_db
dristysrivastava Sep 2, 2024
ab3c49c
Modularized methods and added comments
Sep 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions pebblo/app/models/db_models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,12 +81,10 @@ class AiDataModel(BaseModel):
data: Optional[Union[list, str]] = None
entityCount: int
entities: dict
entityDetails: Optional[dict] = {}
topicCount: Optional[int] = 0
topics: Optional[dict] = {}

def dict(self, **kwargs):
kwargs["exclude_none"] = True
return super().dict(**kwargs)
topicDetails: Optional[dict] = {}


class RetrievalContext(BaseModel):
Expand Down Expand Up @@ -183,5 +181,7 @@ class AiSnippet(BaseModel):
lastModified: Optional[str] = None
entities: dict
topics: dict
entityDetails: Optional[dict] = {}
topicDetails: Optional[dict] = {}
policyViolations: Optional[List[dict]] = []
# label_feedback: Optional[List[LabelFeedback]] = []
6 changes: 6 additions & 0 deletions pebblo/app/service/loader/loader_doc_service.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,8 +170,10 @@ def _get_doc_classification(self, doc):
data=doc.get("doc", None),
entities={},
entityCount=0,
entityDetails={},
topics={},
topicCount=0,
topicDetails={},
)
try:
if doc_info.data:
Expand All @@ -189,8 +191,10 @@ def _get_doc_classification(self, doc):
)
doc_info.topics = topics
doc_info.entities = entities
doc_info.entityDetails = entity_details
doc_info.topicCount = topic_count
doc_info.entityCount = entity_count
doc_info.topicDetails = topic_details
doc_info.data = anonymized_doc
logger.debug("Doc classification finished.")
return doc_info
Expand All @@ -209,6 +213,8 @@ def _update_doc_details(doc, doc_info):
logger.debug("Update doc details with classification result")
doc["entities"] = doc_info.entities
doc["topics"] = doc_info.topics
doc["entity_details"] = doc_info.entityDetails
doc["topic_details"] = doc_info.topicDetails
logger.debug("Input doc updated with classification result")

@timeit
Expand Down
2 changes: 2 additions & 0 deletions pebblo/app/service/loader/snippet/snippet.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ def create_snippet(self, doc, data_source, document):
"loaderSourcePath": data_source.get("sourcePath"),
"entities": doc.get("entities", {}),
"topics": doc.get("topics", {}),
"entityDetails": doc.get("entity_details", {}),
"topicDetails": doc.get("topic_details", {}),
}
ai_snippet_obj = AiSnippet(**snippet_details)
ai_snippet = ai_snippet_obj.dict()
Expand Down
34 changes: 24 additions & 10 deletions pebblo/app/service/local_ui/loader_apps.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,18 +37,32 @@ def __init__(self):
self.loader_document_with_findings_list = []
self.loader_findings_summary_list = []

def _get_snippet_details(self, snippet_ids, owner):
def _get_snippet_details(self, snippet_ids, owner, label_name):
dristysrivastava marked this conversation as resolved.
Show resolved Hide resolved
response = []
for snippet_id in snippet_ids:
status, output = self.db.query(AiSnippetsTable, {"id": snippet_id})
if not status or len(output) == 0:
continue
snippet_details = output[0].data
entity_details = {}
topic_details = {}
if snippet_details.get("topicDetails") and snippet_details[
"topicDetails"
].get(label_name):
topic_details = {
label_name: snippet_details["topicDetails"].get(label_name)
}
if snippet_details.get("entityDetails") and snippet_details[
"entityDetails"
].get(label_name):
entity_details = {
label_name: snippet_details["entityDetails"].get(label_name)
}
snippet_obj = {
"snippet": snippet_details["doc"],
"sourcePath": snippet_details["sourcePath"],
# "topicDetails": {}, # TODO: To be added post 0.1.18
# "entityDetails": {}, # TODO: to be added post 0.1.18
"topicDetails": topic_details,
"entityDetails": entity_details,
"fileOwner": owner,
"authorizedIdentities": [],
}
Expand All @@ -75,7 +89,7 @@ def get_findings_for_loader_app(self, app_data):
total_snippet_count += findings["snippetCount"]
snippets.extend(
self._get_snippet_details(
entity_data["snippetIds"], app_data["owner"]
entity_data["snippetIds"], app_data["owner"], entity
)
)
break
Expand All @@ -89,7 +103,7 @@ def get_findings_for_loader_app(self, app_data):
"snippetCount": len(entity_data["snippetIds"]),
"fileCount": len(app_data["documents"]),
"snippets": self._get_snippet_details(
entity_data["snippetIds"], app_data["owner"]
entity_data["snippetIds"], app_data["owner"], entity
),
}
total_snippet_count += findings["snippetCount"]
Expand All @@ -113,7 +127,7 @@ def get_findings_for_loader_app(self, app_data):
total_snippet_count += findings["snippetCount"]
snippets.extend(
self._get_snippet_details(
topic_data["snippetIds"], app_data["owner"]
topic_data["snippetIds"], app_data["owner"], topic
)
)
break
Expand All @@ -126,7 +140,7 @@ def get_findings_for_loader_app(self, app_data):
"snippetCount": len(topic_data["snippetIds"]),
"fileCount": len(app_data["documents"]),
"snippets": self._get_snippet_details(
topic_data["snippetIds"], app_data["owner"]
topic_data["snippetIds"], app_data["owner"], topic
),
}
total_snippet_count += findings["snippetCount"]
Expand Down Expand Up @@ -216,9 +230,9 @@ def get_all_loader_apps(self):
continue

self.loader_apps_at_risk += 1
loader_app = self.get_findings_for_loader_app(app_data)
all_loader_apps.append(loader_app)
app_processed.append(app_data["name"])
loader_app = self.get_findings_for_loader_app(app_data)
all_loader_apps.append(loader_app)
app_processed.append(app_data["name"])

# TODO: Sort loader apps
# sorted_loader_apps = self._sort_loader_apps(all_loader_apps)
Expand Down