You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from ragas.metrics.base import EvaluationMode, MetricWithLLM, get_segmenter
from ragas_experimental.llms.prompt import PydanticPrompt
if t.TYPE_CHECKING:
from langchain_core.callbacks import Callbacks
from ragas.metrics._faithfulness import HasSegmentMethod
logger = logging.getLogger(__name__)
class FaithfulnessStatements(BaseModel):
question: str = Field(description="The question to answer")
answer: str = Field(description="The answer to the question")
sentences: t.Dict[int, str] = Field(
description="A mapping of sentence index to the sentence"
)
class SentenceComponents(BaseModel):
sentence_index: int = Field(description="The index of the sentence")
simpler_statements: t.List[str] = Field(
description="A list of simpler statements that can be directly inferred from the context"
)
class SentencesSimplified(BaseModel):
sentences: t.List[SentenceComponents] = Field(
description="A list of sentences and their simpler versions"
)
# examples
example_input_1 = FaithfulnessStatements(
question="Who was Albert Einstein and what is he best known for?",
answer="He was a German-born theoretical physicist, widely acknowledged to be one of the greatest and most influential physicists of all time. He was best known for developing the theory of relativity, he also made important contributions to the development of the theory of quantum mechanics.",
sentences={
0: "He was a German-born theoretical physicist, widely acknowledged to be one of the greatest and most influential physicists of all time.",
1: "He was best known for developing the theory of relativity, he also made important contributions to the development of the theory of quantum mechanics.",
},
)
example_output_1 = SentencesSimplified(
sentences=[
SentenceComponents(
sentence_index=0,
simpler_statements=[
"Albert Einstein was a German-born theoretical physicist.",
"Albert Einstein is recognized as one of the greatest and most influential physicists of all time.",
],
),
SentenceComponents(
sentence_index=1,
simpler_statements=[
"Albert Einstein was best known for developing the theory of relativity.",
"Albert Einstein also made important contributions to the development of the theory of quantum mechanics.",
],
),
]
)
class LongFormAnswerPrompt(PydanticPrompt[FaithfulnessStatements, SentencesSimplified]):
instruction = "Given a question, an answer, and sentences from the answer analyze the complexity of each sentence given under 'sentences' and break down each sentence into one or more fully understandable statements while also ensuring no pronouns are used in each statement. Format the outputs in JSON."
input_model = FaithfulnessStatements
output_model = SentencesSimplified
examples = [(example_input_1, example_output_1)]
class StatementFaithfulnessAnswer(BaseModel):
statement: str = Field(..., description="the original statement, word-by-word")
reason: str = Field(..., description="the reason of the verdict")
verdict: int = Field(..., description="the verdict(0/1) of the faithfulness.")
class NLIStatementOutput(BaseModel):
statements: t.List[StatementFaithfulnessAnswer]
class NLIStatementInput(BaseModel):
context: str = Field(..., description="The context of the question")
statements: t.List[str] = Field(..., description="The statements to judge")
class NLIStatementPrompt(PydanticPrompt[NLIStatementInput, NLIStatementOutput]):
instruction = "Your task is to judge the faithfulness of a series of statements based on a given context. For each statement you must return verdict as 1 if the statement can be directly inferred based on the context or 0 if the statement can not be directly inferred based on the context."
input_model = NLIStatementInput
output_model = NLIStatementOutput
examples = [
(
NLIStatementInput(
context="""John is a student at XYZ University. He is pursuing a degree in Computer Science. He is enrolled in several courses this semester, including Data Structures, Algorithms, and Database Management. John is a diligent student and spends a significant amount of time studying and completing assignments. He often stays late in the library to work on his projects.""",
statements=[
"John is majoring in Biology.",
"John is taking a course on Artificial Intelligence.",
"John is a dedicated student.",
"John has a part-time job.",
],
),
NLIStatementOutput(
statements=[
StatementFaithfulnessAnswer(
statement="John is majoring in Biology.",
reason="John's major is explicitly mentioned as Computer Science. There is no information suggesting he is majoring in Biology.",
verdict=0,
),
StatementFaithfulnessAnswer(
statement="John is taking a course on Artificial Intelligence.",
reason="The context mentions the courses John is currently enrolled in, and Artificial Intelligence is not mentioned. Therefore, it cannot be deduced that John is taking a course on AI.",
verdict=0,
),
StatementFaithfulnessAnswer(
statement="John is a dedicated student.",
reason="The context states that he spends a significant amount of time studying and completing assignments. Additionally, it mentions that he often stays late in the library to work on his projects, which implies dedication.",
verdict=1,
),
StatementFaithfulnessAnswer(
statement="John has a part-time job.",
reason="There is no information given in the context about John having a part-time job.",
verdict=0,
),
]
),
),
(
NLIStatementInput(
context="Photosynthesis is a process used by plants, algae, and certain bacteria to convert light energy into chemical energy.",
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: new Prompt object and faithfulness metric #1232
feat: new Prompt object and faithfulness metric #1232
Changes from all commits
d9b9ef8
afc98e2
3bd8f4e
a895985
6798e90
11ed465
576751b
e8c2778
daa6c56
14bf70f
5c26016
6e94723
1834088
42ca4ef
61a12f3
5c89381
File filter
Filter by extension
Conversations
Jump to
There are no files selected for viewing