1 Star 0 Fork 1

YuliQ/document-QA-robot

加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
文件
克隆/下载
SBERT.py 2.42 KB
一键复制 编辑 原始数据 按行查看 历史
ArkShin 提交于 2023-08-15 08:51 . Replace SBERT.py
import scipy
from LLM import LLM
from ErnieBot_turbo import ErnieBot
from GPT_turbo import GPT
from SparkApi import Spark
from rouge import Rouge
import jieba
def match(query, sentence_embedding, model):
query_embeddings = model.encode(query)
distances = scipy.spatial.distance.cdist([query_embeddings], sentence_embedding, "cosine")[0]
results = zip(range(len(distances)), distances)
results = sorted(results, key=lambda x: x[1])
return results
def multi_ans(query, url, file_name, sentence_embedding, model_embed, model_llm, data_path, language, answer_embeddings="", answer=""):
llm=LLM(model_llm)
if language=="en":
query_zh=llm.translation(query)
else:
query_zh=query
results=match(query_zh, sentence_embedding, model_embed)
#rouge=Rouge()
ans_list=""
for idx, distance in results[0:3]:
with open(data_path+file_name[idx], "r", encoding='utf-8') as f:
content=f.read()
ans1=llm.get_QA(query, content.strip(), language)
ans_list = ans_list+"Answer:\n"+ans1
# ans1_tokens = list(jieba.cut(ans1))
# ans_tokens = list(jieba.cut(answer))
# score1 = rouge.get_scores(" ".join(ans1_tokens), " ".join(ans_tokens))
# score1 = rouge.get_scores(ans1, answer)
# print("ROUGE-L分数-GPT:", score1[0]['rouge-l'])
if "追问" not in ans1:
s=f'''References: {url[idx]}'''
print(s)
ans_list = ans_list+"\n"+s+"\n"
###如果要比较和数据集答案的相似度,使用以下代码,否则将这一段注释
#ans_embedding1=model_embed.encode([ans1])
#score = scipy.spatial.distance.cdist(answer_embeddings, ans_embedding1, "cosine")[0]
#print("Similarity Score %.4f "%(1-score))
###比较和数据集答案的相似度使用以上代码,否则将这一段注释
#print("(Similarity Score: %.4f)" % (1-distance)) #这里比较答案和文档间的相似度
print("_____________________________________________________")
elif language=="zh":
print("请根据追问重新输入问题:")
ans_list = ans_list+"\n请根据追问重新输入问题:\n"
elif language=="en":
print("Please re-enter the question based on the follow-up question:")
ans_list = ans_list+"\nPlease re-enter the question based on the follow-up question:\n"
return ans_list
Loading...
马建仓 AI 助手
尝试更多
代码解读
代码找茬
代码优化
1
https://gitee.com/qiuyuli/document-qa-robot.git
git@gitee.com:qiuyuli/document-qa-robot.git
qiuyuli
document-qa-robot
document-QA-robot
main

搜索帮助

0d507c66 1850385 C8b1a773 1850385