Skip to content

Commit

Permalink
[chat] update config and prompt (hpcaitech#4139)
Browse files Browse the repository at this point in the history 
* update config and prompt

* update config

---------

Co-authored-by: Qianran Ma <[email protected]>
  • Loading branch information
@MichelleMa8
MichelleMa8 and Qianran Ma authored Aug 21, 2023
1 parent d20dceb commit 285fe7b
Show file tree
Hide file tree
Showing 4 changed files with 202 additions and 26 deletions.
99 changes: 88 additions & 11 deletions applications/Chat/evaluate/config/config_cn.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,16 @@
"chat": {
"GPT": [
"language organization",
"relevance",
"naturalness",
"engagingness",
"reasonableness"
"fidelity"
],
"Metrics": [
"Distinct"
]
},
"classification": {
"GPT": [
"language organization",
"relevance",
"correctness"
],
Expand All @@ -40,7 +38,6 @@
},
"closed_qa": {
"GPT": [
"language organization",
"relevance",
"correctness"
],
Expand All @@ -53,7 +50,6 @@
},
"extraction": {
"GPT": [
"language organization",
"relevance",
"correctness"
],
Expand All @@ -74,7 +70,20 @@
"BLEU",
"ROUGE",
"BERTScore"
]
]
},
"logical_reasoning": {
"GPT": [
"correctness",
"relevance",
"reasonableness"
],
"Metrics": [
"BLEU",
"ROUGE",
"BERTScore",
"CHRF"
]
},
"open_qa": {
"GPT": [
Expand Down Expand Up @@ -117,11 +126,79 @@
"conciseness"
],
"Metrics": [
"BLEU",
"ROUGE",
"BERTScore",
"CHRF"
]
]
},
"Finance": {
"GPT": [
"relevance",
"correctness"
],
"Metrics": [
]
},
"Law": {
"GPT": [
"relevance",
"correctness"
],
"Metrics": [
]
},
"Education": {
"GPT": [
"relevance",
"correctness"
],
"Metrics": [
]
},
"Medical": {
"GPT": [
"relevance",
"correctness"
],
"Metrics": [
]
},
"STEM": {
"GPT": [
"relevance",
"correctness"
],
"Metrics": [
]
},
"SocialScience": {
"GPT": [
"relevance",
"correctness"
],
"Metrics": [
]
},
"Humanity": {
"GPT": [
"relevance",
"correctness"
],
"Metrics": [
]
},
"Other": {
"GPT": [
"relevance",
"correctness"
],
"Metrics": [
]
},
"ethics": {
"GPT": [
"relevance",
"correctness"
],
"Metrics": [
]
}
}
}
117 changes: 106 additions & 11 deletions applications/Chat/evaluate/config/config_en.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,9 @@
"chat": {
"GPT": [
"language organization",
"relevance",
"naturalness",
"engagingness",
"reasonableness"
"fidelity"
],
"Metrics": [
"Distinct"
Expand All @@ -45,7 +44,6 @@
},
"classification": {
"GPT": [
"language organization",
"relevance",
"correctness"
],
Expand All @@ -63,7 +61,6 @@
},
"closed_qa": {
"GPT": [
"language organization",
"relevance",
"correctness"
],
Expand All @@ -81,7 +78,6 @@
},
"extraction": {
"GPT": [
"language organization",
"relevance",
"correctness"
],
Expand Down Expand Up @@ -114,6 +110,21 @@
"data2text-informativeness"
]
},
"logical_reasoning": {
"GPT": [
"correctness",
"relevance",
"reasonableness"
],
"Metrics": [
"BLEU",
"ROUGE",
"BERTScore",
"CHRF"
],
"UniEval": [
]
},
"open_qa": {
"GPT": [
"language organization",
Expand Down Expand Up @@ -176,12 +187,96 @@
"CHRF"
],
"UniEval": [
"summarization-coherence",
"summarization-consistency",
"summarization-fluency",
"summarization-relevance",
"data2text-naturalness",
"data2text-informativeness"
]
},
"Finance": {
"GPT": [
"relevance",
"correctness"
],
"Metrics": [
],
"UniEval": [
]
},
"Law": {
"GPT": [
"relevance",
"correctness"
],
"Metrics": [
],
"UniEval": [
]
},
"Education": {
"GPT": [
"relevance",
"correctness"
],
"Metrics": [
],
"UniEval": [
]
},
"Medical": {
"GPT": [
"relevance",
"correctness"
],
"Metrics": [
],
"UniEval": [
]
},
"STEM": {
"GPT": [
"relevance",
"correctness"
],
"Metrics": [
],
"UniEval": [
]
},
"SocialScience": {
"GPT": [
"relevance",
"correctness"
],
"Metrics": [
],
"UniEval": [
]
},
"Humanity": {
"GPT": [
"relevance",
"correctness"
],
"Metrics": [
],
"UniEval": [
]
},
"Other": {
"GPT": [
"relevance",
"correctness"
],
"Metrics": [
],
"UniEval": [
]
},
"ethics": {
"GPT": [
"relevance",
"correctness"
],
"Metrics": [
],
"UniEval": [
]
}
}
Expand Down
6 changes: 4 additions & 2 deletions applications/Chat/evaluate/prompt/evaluation_prompt/evaluation_prompt_cn.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,16 @@
"relevance": "切题(1-5)：答案内容是否切题，不答非所问，并且严格遵照题目要求。",
"naturalness": "自然(1-5)：答案是否自然，并且符合问题给定的身份。",
"engagingness": "参与感(1-5)：答案是否对前面的对话内容做出了恰当的反应，是否理解对话的语境和背景。",
"reasonableness": "合理性(1-5)：答案是否能够与前面的对话内容形成逻辑上的衔接，是否符合常理，能否在这个上下文中合理存在。"
"reasonableness": "合理性(1-5)：答案是否能够与前面的对话内容形成逻辑上的衔接，是否符合常理，能否在这个上下文中合理存在。",
"fidelity": "保真度(1-5)：答案是否能够严格遵守角色的设定回答给定的请求。"
},
"CoT": {
"language organization": "1. 阅读答案，并检查是否有语法错误、用词不当或其他显著的错误。\n2. 检查答案是否具有逻辑性，能够按照合理的顺序传达信息并且能够自圆其说。\n3. 确定答案是否与问题或主题相关，并且能够传达清晰的信息。\n4. 检查答案是否连贯，是否使用适当的转换和过渡来保持句子和段落之间的连贯性。\n5. 检查答案是否具有明确的结构和组织方式，使得读者可以轻松理解信息的层次和结构。\n6. 根据以上因素综合评估答案的语言组织，并给出一个1到5的分数，其中5表示语言组织非常好，而1表示语言组织非常差。\n\n语言组织：",
"relevance": "1. 阅读题目，确定题目所问的问题是什么，以及需要回答哪些方面的问题。\n2. 阅读答案，确认答案是否直接回答了题目所问的问题。\n3. 检查答案是否严格遵照了题目的要求，包括答题方式、答题长度、答题格式等等。\n4. 根据以上因素综合评估答案的切题程度，并给出一个1到5的分数，其中5表示答案非常切题，而1表示答案完全没有切题。\n\n切题：",
"naturalness": "1. 阅读题目，确定题目提供的身份信息。\n2. 检查答案内容是否符合题目给定的身份。\n3. 根据以上因素，对该回答的自然性进行打分，分数从1到5，其中1表示不自然，5表示非常自然，并符合问题给定的身份。\n\n自然：",
"engagingness": "1. 阅读题目，确定对话的语境和背景。\n2. 检查答案是否充分理解对话的语境和背景，能否自然地融入到对话中而不显得突兀。\n3. 根据以上因素，对该回答的参与感进行打分，分数从1到5，其中1表示没有参与感，5表示非常有参与感，并且恰当地理解了对话的语境和背景。\n\n参与感：",
"reasonableness": "1. 阅读题目，确定对话的主题以及问题期望的回答方向。\n2. 判断答案是否能够与前面的对话内容形成逻辑上的衔接，是否符合常理，能否在这个上下文中合理存在。\n3. 根据以上因素，对该回答的合理性进行打分，分数从1到5，其中1表示不合理，5表示非常合理，并且能够与前面的对话内容形成逻辑上的衔接，并符合常理。\n\n合理性："
"reasonableness": "1. 阅读题目，确定对话的主题以及问题期望的回答方向。\n2. 判断答案是否能够与前面的对话内容形成逻辑上的衔接，是否符合常理，能否在这个上下文中合理存在。\n3. 根据以上因素，对该回答的合理性进行打分，分数从1到5，其中1表示不合理，5表示非常合理，并且能够与前面的对话内容形成逻辑上的衔接，并符合常理。\n\n合理性：",
"fidelity": "1. 仔细阅读问题，了解角色在问题中的设定和表现，包括职业、背景、观点、性格等方面。\n阅读题目的请求，确认回答请求时需要注意的细节。\n3. 对比提供的回答与该角色的设定，评估回答是否能够严格遵守角色的设定。\n4. 结合以上评估结果给出保真度的评分，范围从1到5分，其中1分表示回答与角色设定完全不符，5分表示回答完全符合角色设定且满足给定请求。\n\n保真度："
},
"prompt": "你是一个好助手。请你为下面的“补全对话”问题的答案打分。\n\n问题如下：\n\n{question}\n\n答案如下：\n\n{answer}\n\n评分的指标如下：\n\n{metric}\n\n请你遵照以下的评分步骤：\n\n{steps}"
},
Expand Down
6 changes: 4 additions & 2 deletions applications/Chat/evaluate/prompt/evaluation_prompt/evaluation_prompt_en.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,16 @@
"relevance": "Relevance (1-5): whether the content of the answer is relevant to the topic, does not answer the wrong question, and strictly follows the requirements of the topic.",
"naturalness": "Naturalness (1-5): whether the answer is natural and fits the identity given by the question.",
"engagingness": "Engagingness (1-5): whether the answer responds appropriately to the content of the preceding conversation and whether it understands the context and background of the conversation.",
"reasonableness": "Reasonableness (1-5): Whether the answer can form a logical connection with the content of the previous dialogue, whether it is consistent with common sense, and whether it can reasonably exist in this context."
"reasonableness": "Reasonableness (1-5): Whether the answer can form a logical connection with the content of the previous dialogue, whether it is consistent with common sense, and whether it can reasonably exist in this context.",
"fidelity": "Fidelity (1-5): whether the answer is able to answer the given request in strict compliance with the role setting."
},
"CoT": {
"language organization": "1. Read the answers and check for grammatical errors, poor word choice, or other significant mistakes.\n2. Check that the answer is logical, conveys the information in a logical order, and is self-explanatory.\n3. Determine if the answer is relevant to the question or topic and conveys a clear message.\n4. Check that the answer is coherent and that appropriate transitions and switches are used to maintain coherence between sentences and paragraphs.\n5. Check that the answer is clearly structured and organized in such a way that the reader can easily understand the hierarchy and structure of the information.\n6. Evaluate the language organization of the answer based on a combination of the above factors and give a score of 1 to 5, where 5 indicates very good language organization and 1 indicates very poor language organization.\n\nLanguage organization:",
"relevance": "1. Read the question to determine what the question asks and what aspects of the question need to be answered.\n2. Read the answers to make sure that they directly answer the question asked.\n3. Check that the answer follows the requirements of the question, including the way it is answered, the length of the answer, the format of the answer, etc.\n4. Evaluate how relevant the answer is based on the above factors and give a score of 1 to 5, where 5 means the answer is very relevant and 1 means the answer is not relevant at all.\n\nRelevance:",
"naturalness": "1. Read the question and determine the identity information provided in the question.\n2. Check whether the content of the answer matches the identity given in the question.\n3. Based on the above factors, score the naturalness of the response on a scale from 1 to 5, where 1 means unnatural and 5 means very natural and in accordance with the identity given in the question.\n\nNaturalness:",
"engagingness": "1. Read the questions to determine the context and background of the dialogue.\n2. Check that the answer fully understands the context and background of the conversation and that it fits naturally into the conversation without seeming abrupt.\n3. Based on the above factors, rate the response's engagement on a scale from 1 to 5, where 1 means not engaged and 5 means very engaged and appropriately understands the context and background of the conversation.\n\nEngagingness:",
"reasonableness": "1. Read the question and determine the topic of the conversation and the direction the question expects the answer to go.\n2. Determine whether the answer can be logically connected to the preceding conversation, whether it makes common sense, and whether it can reasonably exist in this context.\n3. Based on the above factors, rate the reasonableness of the answer on a scale from 1 to 5, where 1 means unreasonable and 5 means very reasonable and able to form a logical connection with the preceding dialogue content and consistent with common sense.\n\nReasonableness:"
"reasonableness": "1. Read the question and determine the topic of the conversation and the direction the question expects the answer to go.\n2. Determine whether the answer can be logically connected to the preceding conversation, whether it makes common sense, and whether it can reasonably exist in this context.\n3. Based on the above factors, rate the reasonableness of the answer on a scale from 1 to 5, where 1 means unreasonable and 5 means very reasonable and able to form a logical connection with the preceding dialogue content and consistent with common sense.\n\nReasonableness:",
"fidelity": "1. Read the question carefully to understand how the character is set up and represented in the question, including aspects such as occupation, background, point of view, and personality.\n2. Read the question's request and confirm the details that need to be taken into account when answering the request.\n3. Compare the provided answer with the setting of the role and assess whether the answer can strictly adhere to the setting of the role.\n4. Combine the results of the above assessment to give a fidelity score ranging from 1 to 5, where a score of 1 means that the response does not match the persona at all, and a score of 5 means that the response fully complies with the persona and satisfies the given request.\n\nFidelity:"
},
"prompt": "You are a good assistant. Please rate the given answer to the \"chat\" question below.\n\nThe question is as follows:\n\n{question}\n\nThe answer is as follows:\n\n{answer}\n\nThe metric for evaluation is as follows:\n\n{metric}\n\nYou should follow the following evaluation steps:\n\n{steps}"
},
Expand Down

0 comments on commit 285fe7b

Please sign in to comment.