Skip to content

Commit

Permalink
[pre-commit.ci] auto fixes from pre-commit.com hooks
Browse files Browse the repository at this point in the history
for more information, see https://pre-commit.ci
  • Loading branch information
pre-commit-ci[bot] committed Sep 20, 2024
1 parent 9f32d8a commit 2b540a8
Showing 1 changed file with 15 additions and 9 deletions.
24 changes: 15 additions & 9 deletions evals/metrics/ragas/ragas.py
Original file line number Diff line number Diff line change
@@ -1,20 +1,23 @@

#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0
#
import os
from typing import Dict, Optional, Union
from langchain_huggingface import HuggingFaceEndpoint

from langchain_core.embeddings import Embeddings
from langchain_core.language_models import BaseLanguageModel
from langchain_huggingface import HuggingFaceEndpoint


def format_ragas_metric_name(name: str):
return f"{name} (ragas)"


class RagasMetric:
"""This metric checks if the output is more than 3 letters."""

def __init__(
self,
threshold: float = 0.3,
Expand All @@ -36,21 +39,22 @@ def __init__(
"context_utilization",
# "reference_free_rubrics_score",
]

async def a_measure(self, test_case: Dict):
return self.measure(test_case)

def measure(self, test_case: Dict):
# sends to server
try:
from ragas import evaluate
from ragas.metrics import (
from ragas.metrics import ( # reference_free_rubrics_score,
answer_correctness,
answer_relevancy,
answer_similarity,
context_precision,
context_recall,
context_utilization,
faithfulness,
# reference_free_rubrics_score,
)
except ModuleNotFoundError:
raise ModuleNotFoundError("Please install ragas to use this metric. `pip install ragas`.")
Expand Down Expand Up @@ -117,13 +121,13 @@ def measure(self, test_case: Dict):
for column in list(metric._required_columns.values())[0]:
_required_columns.add(column)
column2field = {
"user_input" : "question",
"response" : "answer",
"reference" : "ground_truth",
"retrieved_contexts" : "contexts"
"user_input": "question",
"response": "answer",
"reference": "ground_truth",
"retrieved_contexts": "contexts",
}
_required_fields = [column2field[column] for column in _required_columns]
data = {field : test_case[field] for field in _required_fields}
data = {field: test_case[field] for field in _required_fields}
dataset = Dataset.from_dict(data)

# evaluate
Expand All @@ -134,8 +138,10 @@ def measure(self, test_case: Dict):
embeddings=self.embeddings,
)
return self.score

def is_successful(self):
return self.success

@property
def __name__(self):
return "RAGAS"

0 comments on commit 2b540a8

Please sign in to comment.