From 33f1e1e5a847b196afeceb08718acf8eae9a1341 Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Fri, 21 Apr 2023 16:28:08 +0300 Subject: [PATCH 01/32] feat: create files for llm based response selection --- .../llm_based_response_selector/Dockerfile | 22 +++++++ .../llm_based_response_selector/component.yml | 20 +++++++ .../llm_based_response_selector/pipeline.yml | 10 ++++ .../requirements.txt | 8 +++ .../llm_based_response_selector/server.py | 60 +++++++++++++++++++ 5 files changed, 120 insertions(+) create mode 100644 response_selectors/llm_based_response_selector/Dockerfile create mode 100644 response_selectors/llm_based_response_selector/component.yml create mode 100644 response_selectors/llm_based_response_selector/pipeline.yml create mode 100644 response_selectors/llm_based_response_selector/requirements.txt create mode 100644 response_selectors/llm_based_response_selector/server.py diff --git a/response_selectors/llm_based_response_selector/Dockerfile b/response_selectors/llm_based_response_selector/Dockerfile new file mode 100644 index 0000000000..017cb05de6 --- /dev/null +++ b/response_selectors/llm_based_response_selector/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.7.4 + +RUN mkdir /src + +COPY ./requirements.txt /src/requirements.txt +RUN pip install -r /src/requirements.txt + +ARG LANGUAGE=EN +ENV LANGUAGE ${LANGUAGE} +ARG SERVICE_PORT +ENV SERVICE_PORT ${SERVICE_PORT} +ARG GENERATIVE_SERVICE_URL +ENV GENERATIVE_SERVICE_URL ${GENERATIVE_SERVICE_URL} +ARG GENERATIVE_TIMEOUT +ENV GENERATIVE_TIMEOUT ${GENERATIVE_TIMEOUT} +ARG GENERATIVE_SERVICE_CONFIG +ENV GENERATIVE_SERVICE_CONFIG ${GENERATIVE_SERVICE_CONFIG} + +COPY . /src/ +WORKDIR /src + +CMD gunicorn --workers=1 server:app -b 0.0.0.0:${SERVICE_PORT} --timeout=1200 diff --git a/response_selectors/llm_based_response_selector/component.yml b/response_selectors/llm_based_response_selector/component.yml new file mode 100644 index 0000000000..fb771b8440 --- /dev/null +++ b/response_selectors/llm_based_response_selector/component.yml @@ -0,0 +1,20 @@ +llm-based-response-selector: + name: response_selector + display_name: Response Selector + container_name: llm-based-response-selector + component_type: null + model_type: Dictionary/Pattern-based + is_customizable: false + author: DeepPavlov + description: Algorithm that selects a final responses among the given list of candidate + responses + ram_usage: 100M + gpu_usage: null + port: 8005 + endpoints: + - group: response_selectors + endpoint: respond + build_args: + SERVICE_PORT: '8005' + SERVICE_NAME: response_selector + date_created: '2023-03-16T09:45:32' diff --git a/response_selectors/llm_based_response_selector/pipeline.yml b/response_selectors/llm_based_response_selector/pipeline.yml new file mode 100644 index 0000000000..7d522c7fde --- /dev/null +++ b/response_selectors/llm_based_response_selector/pipeline.yml @@ -0,0 +1,10 @@ +llm-based-response-selector: +- group: response_selectors + connector: + protocol: http + url: http://llm-based-response-selector:8005/respond + dialog_formatter: state_formatters.dp_formatters:full_dialog + response_formatter: state_formatters.dp_formatters:base_response_selector_formatter_service + previous_services: + - response_annotators + state_manager_method: add_bot_utterance diff --git a/response_selectors/llm_based_response_selector/requirements.txt b/response_selectors/llm_based_response_selector/requirements.txt new file mode 100644 index 0000000000..1d2e7fb9f2 --- /dev/null +++ b/response_selectors/llm_based_response_selector/requirements.txt @@ -0,0 +1,8 @@ +flask==1.1.1 +itsdangerous==2.0.1 +gunicorn==19.9.0 +requests==2.22.0 +numpy==1.17.2 +sentry-sdk==0.12.3 +jinja2<=3.0.3 +Werkzeug<=2.0.3 diff --git a/response_selectors/llm_based_response_selector/server.py b/response_selectors/llm_based_response_selector/server.py new file mode 100644 index 0000000000..873aca5952 --- /dev/null +++ b/response_selectors/llm_based_response_selector/server.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python + +import logging +import numpy as np +import time +from os import getenv + +import sentry_sdk +from flask import Flask, request, jsonify + + +sentry_sdk.init(getenv("SENTRY_DSN")) + +logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=logging.INFO) +logger = logging.getLogger(__name__) + +app = Flask(__name__) + +GENERATIVE_SERVICE_URL = getenv("GENERATIVE_SERVICE_URL") +GENERATIVE_TIMEOUT = getenv("GENERATIVE_TIMEOUT") +GENERATIVE_SERVICE_CONFIG = getenv("GENERATIVE_SERVICE_CONFIG") + + +@app.route("/respond", methods=["POST"]) +def respond(): + st_time = time.time() + + dialogs = request.json["dialogs"] + response_candidates = [dialog["utterances"][-1]["hypotheses"] for dialog in dialogs] + + selected_skill_names = [] + selected_responses = [] + selected_confidences = [] + + for i, dialog in enumerate(dialogs): + confidences = [] + responses = [] + skill_names = [] + + for skill_data in response_candidates[i]: + if skill_data["text"] and skill_data["confidence"]: + logger.info(f"Skill {skill_data['skill_name']} returned non-empty hypothesis with non-zero confidence.") + + confidences += [skill_data["confidence"]] + responses += [skill_data["text"]] + skill_names += [skill_data["skill_name"]] + + best_id = np.argmax(confidences) + + selected_skill_names.append(skill_names[best_id]) + selected_responses.append(responses[best_id]) + selected_confidences.append(confidences[best_id]) + + total_time = time.time() - st_time + logger.info(f"llm_based_response_selector exec time = {total_time:.3f}s") + return jsonify(list(zip(selected_skill_names, selected_responses, selected_confidences))) + + +if __name__ == "__main__": + app.run(debug=False, host="0.0.0.0", port=3000) From 93006d9900e85e7e8e25d318fa55c23e35fa0001 Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Fri, 21 Apr 2023 16:29:11 +0300 Subject: [PATCH 02/32] feat: tests --- .../llm_based_response_selector/test.py | 19 +++++++++++++++++++ .../llm_based_response_selector/test.sh | 3 +++ 2 files changed, 22 insertions(+) create mode 100644 response_selectors/llm_based_response_selector/test.py create mode 100755 response_selectors/llm_based_response_selector/test.sh diff --git a/response_selectors/llm_based_response_selector/test.py b/response_selectors/llm_based_response_selector/test.py new file mode 100644 index 0000000000..7ed8204c19 --- /dev/null +++ b/response_selectors/llm_based_response_selector/test.py @@ -0,0 +1,19 @@ +import requests +import json +from os import getenv + + +SERVICE_PORT = getenv("SERVICE_PORT") + + +def main(): + with open("test_data.json", "r") as f: + data = json.load(f) + # To skip "Oh, and remember this dialog's id" that raises error due to absence of 'dialog_id' field in test_data. + data["dialogs"][0]["human_utterances"].append(data["dialogs"][0]["human_utterances"][0]) + result = requests.post(f"http://0.0.0.0:{SERVICE_PORT}/respond", json=data).json() + assert result[0][0] == "program_y", print(result) + + +if __name__ == "__main__": + main() diff --git a/response_selectors/llm_based_response_selector/test.sh b/response_selectors/llm_based_response_selector/test.sh new file mode 100755 index 0000000000..61672db785 --- /dev/null +++ b/response_selectors/llm_based_response_selector/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +python test.py From da416c76f89372154a3d3903c7102a0c407f852d Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Fri, 21 Apr 2023 17:46:56 +0300 Subject: [PATCH 03/32] feat: algorithm --- .../llm_based_response_selector/Dockerfile | 4 + .../llm_based_response_selector/server.py | 81 ++++++++++++++----- .../scenario/response.py | 27 ++++--- .../scenario/response.py | 27 ++++--- 4 files changed, 99 insertions(+), 40 deletions(-) diff --git a/response_selectors/llm_based_response_selector/Dockerfile b/response_selectors/llm_based_response_selector/Dockerfile index 017cb05de6..838cea95aa 100644 --- a/response_selectors/llm_based_response_selector/Dockerfile +++ b/response_selectors/llm_based_response_selector/Dockerfile @@ -15,6 +15,10 @@ ARG GENERATIVE_TIMEOUT ENV GENERATIVE_TIMEOUT ${GENERATIVE_TIMEOUT} ARG GENERATIVE_SERVICE_CONFIG ENV GENERATIVE_SERVICE_CONFIG ${GENERATIVE_SERVICE_CONFIG} +ARG ENVVARS_TO_SEND +ENV ENVVARS_TO_SEND ${ENVVARS_TO_SEND} +ARG N_UTTERANCES_CONTEXT=5 +ENV N_UTTERANCES_CONTEXT ${N_UTTERANCES_CONTEXT} COPY . /src/ WORKDIR /src diff --git a/response_selectors/llm_based_response_selector/server.py b/response_selectors/llm_based_response_selector/server.py index 873aca5952..6322a92182 100644 --- a/response_selectors/llm_based_response_selector/server.py +++ b/response_selectors/llm_based_response_selector/server.py @@ -2,6 +2,7 @@ import logging import numpy as np +import requests import time from os import getenv @@ -19,6 +20,49 @@ GENERATIVE_SERVICE_URL = getenv("GENERATIVE_SERVICE_URL") GENERATIVE_TIMEOUT = getenv("GENERATIVE_TIMEOUT") GENERATIVE_SERVICE_CONFIG = getenv("GENERATIVE_SERVICE_CONFIG") +N_UTTERANCES_CONTEXT = int(getenv("N_UTTERANCES_CONTEXT")) +DEFAULT_CRITERION = "the most appropriate, relevant and non-toxic" +CRITERION = getenv("CRITERION", DEFAULT_CRITERION) +PROMPT = f"""Select {CRITERION} response among the hypotheses to the given dialog context. """ \ + """Return only the selected response without extra explanations.""" +ENVVARS_TO_SEND = getenv("ENVVARS_TO_SEND", None) +ENVVARS_TO_SEND = [] if ENVVARS_TO_SEND is None else ENVVARS_TO_SEND.split(",") +sending_variables = {f"{var}_list": [getenv(var, None)] for var in ENVVARS_TO_SEND} +# check if at least one of the env variables is not None +if len(sending_variables.keys()) > 0 and all([var_value is None for var_value in sending_variables.values()]): + raise NotImplementedError( + "ERROR: All environmental variables have None values. At least one of the variables must have not None value" + ) + + +def select_response_by_confidence(hypotheses, confidences): + best_id = np.argmax(confidences) + result = hypotheses[best_id] + return result, best_id + + +def select_response(dialog_context, hypotheses, confidences): + try: + response = requests.post( + GENERATIVE_SERVICE_URL, + json={ + "dialog_contexts": [dialog_context], + "prompts": [PROMPT], + "configs": [GENERATIVE_SERVICE_CONFIG], + **sending_variables, + }, + timeout=GENERATIVE_TIMEOUT, + ) + # batch of a list of one string [["this is the response"]] + result = response.json()[0][0] + except Exception as e: + sentry_sdk.capture_exception(e) + logger.exception(e) + result = select_response_by_confidence(hypotheses, confidences)[0] + logger.info(f"Exception in LLM's invocation. Selected a response with the highest confidence.") + logger.info(f"llm-based-response-selector selected:\n`{result}`") + + return result @app.route("/respond", methods=["POST"]) @@ -26,30 +70,31 @@ def respond(): st_time = time.time() dialogs = request.json["dialogs"] - response_candidates = [dialog["utterances"][-1]["hypotheses"] for dialog in dialogs] selected_skill_names = [] selected_responses = [] selected_confidences = [] for i, dialog in enumerate(dialogs): - confidences = [] - responses = [] - skill_names = [] - - for skill_data in response_candidates[i]: - if skill_data["text"] and skill_data["confidence"]: - logger.info(f"Skill {skill_data['skill_name']} returned non-empty hypothesis with non-zero confidence.") - - confidences += [skill_data["confidence"]] - responses += [skill_data["text"]] - skill_names += [skill_data["skill_name"]] - - best_id = np.argmax(confidences) - - selected_skill_names.append(skill_names[best_id]) - selected_responses.append(responses[best_id]) - selected_confidences.append(confidences[best_id]) + confidences = [hyp["confidence"] for hyp in dialog["human_utterances"][-1]["hypotheses"]] + skill_names = [hyp["skill_name"] for hyp in dialog["human_utterances"][-1]["hypotheses"]] + hypotheses = [hyp["text"] for hyp in dialog["human_utterances"][-1]["hypotheses"]] + dialog_context = [uttr["text"] for uttr in dialog["utterances"][-N_UTTERANCES_CONTEXT:]] + selected_resp = select_response(dialog_context, hypotheses, confidences) + try: + best_id = hypotheses.index(selected_resp) + selected_skill_names.append(skill_names[best_id]) + selected_responses.append(selected_resp) + selected_confidences.append(confidences[best_id]) + except Exception as e: + sentry_sdk.capture_exception(e) + logger.exception(e) + logger.info("Exception in finding selected by LLM response in hypotheses. " + "Selected a response with the highest confidence.") + selected_resp, best_id = select_response_by_confidence(hypotheses, confidences) + selected_skill_names.append(skill_names[best_id]) + selected_responses.append(selected_resp) + selected_confidences.append(confidences[best_id]) total_time = time.time() - st_time logger.info(f"llm_based_response_selector exec time = {total_time:.3f}s") diff --git a/skills/dff_template_prompted_skill/scenario/response.py b/skills/dff_template_prompted_skill/scenario/response.py index 1dd4e53434..5ad0c1bea0 100644 --- a/skills/dff_template_prompted_skill/scenario/response.py +++ b/skills/dff_template_prompted_skill/scenario/response.py @@ -93,17 +93,22 @@ def gathering_responses(reply, confidence, human_attr, bot_attr, attr): logger.info(f"dialog_context: {dialog_context}") if len(dialog_context) > 0: - response = requests.post( - GENERATIVE_SERVICE_URL, - json={ - "dialog_contexts": [dialog_context], - "prompts": [prompt if len(prompt) > 0 and ALLOW_PROMPT_RESET else PROMPT], - "configs": [GENERATIVE_SERVICE_CONFIG], - **sending_variables, - }, - timeout=GENERATIVE_TIMEOUT, - ) - hypotheses = response.json()[0] + try: + response = requests.post( + GENERATIVE_SERVICE_URL, + json={ + "dialog_contexts": [dialog_context], + "prompts": [prompt if len(prompt) > 0 and ALLOW_PROMPT_RESET else PROMPT], + "configs": [GENERATIVE_SERVICE_CONFIG], + **sending_variables, + }, + timeout=GENERATIVE_TIMEOUT, + ) + hypotheses = response.json()[0] + except Exception as e: + sentry_sdk.capture_exception(e) + logger.exception(e) + hypotheses = [] else: hypotheses = [] logger.info(f"generated hypotheses: {hypotheses}") diff --git a/skills/dff_universal_prompted_skill/scenario/response.py b/skills/dff_universal_prompted_skill/scenario/response.py index c4b9a7324a..60b9fccdfb 100644 --- a/skills/dff_universal_prompted_skill/scenario/response.py +++ b/skills/dff_universal_prompted_skill/scenario/response.py @@ -113,17 +113,22 @@ def gathering_responses(reply, confidence, human_attr, bot_attr, attr): sending_variables = {} if len(dialog_context) > 0: - response = requests.post( - CONSIDERED_LM_SERVICES[lm_service]["url"], - json={ - "dialog_contexts": [dialog_context], - "prompts": [prompt], - "configs": [CONSIDERED_LM_SERVICES[lm_service]["config"]], - **sending_variables, - }, - timeout=GENERATIVE_TIMEOUT, - ) - hypotheses = response.json()[0] + try: + response = requests.post( + CONSIDERED_LM_SERVICES[lm_service]["url"], + json={ + "dialog_contexts": [dialog_context], + "prompts": [prompt], + "configs": [CONSIDERED_LM_SERVICES[lm_service]["config"]], + **sending_variables, + }, + timeout=GENERATIVE_TIMEOUT, + ) + hypotheses = response.json()[0] + except Exception as e: + sentry_sdk.capture_exception(e) + logger.exception(e) + hypotheses = [] else: hypotheses = [] logger.info(f"generated hypotheses: {hypotheses}") From 3035f08368839867ce05f93d442ded7c6d71cde0 Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Sun, 23 Apr 2023 21:54:32 +0300 Subject: [PATCH 04/32] fix: filter bad or toxic uttrs --- common/utils.py | 13 ++++++++++--- .../llm_based_response_selector/Dockerfile | 2 ++ .../llm_based_response_selector/server.py | 19 +++++++++++++++++-- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/common/utils.py b/common/utils.py index 5ea01022c0..3fb58decc3 100644 --- a/common/utils.py +++ b/common/utils.py @@ -1278,15 +1278,22 @@ def find_first_complete_sentence(sentences): return None -def is_toxic_or_badlisted_utterance(annotated_utterance): +def is_toxic_utterance(annotated_utterance): toxic_result = get_toxic(annotated_utterance, probs=False) toxic_result = [] if "not_toxic" in toxic_result else toxic_result # now toxic_result is empty if not toxic utterance - toxic_result = True if len(toxic_result) > 0 else False + return True if len(toxic_result) > 0 else False + + +def is_badlisted_utterance(annotated_utterance): default_badlist = {"bad_words": False} badlist_result = annotated_utterance.get("annotations", {}).get("badlisted_words", default_badlist) - return toxic_result or any([badlist_result.get(bad, False) for bad in ["bad_words", "inappropriate", "profanity"]]) + return any([badlist_result.get(bad, False) for bad in ["bad_words", "inappropriate", "profanity"]]) + + +def is_toxic_or_badlisted_utterance(annotated_utterance): + return is_toxic_utterance(annotated_utterance) or is_badlisted_utterance(annotated_utterance) FACTOID_PATTERNS = re.compile( diff --git a/response_selectors/llm_based_response_selector/Dockerfile b/response_selectors/llm_based_response_selector/Dockerfile index 838cea95aa..e27fc9d68e 100644 --- a/response_selectors/llm_based_response_selector/Dockerfile +++ b/response_selectors/llm_based_response_selector/Dockerfile @@ -19,6 +19,8 @@ ARG ENVVARS_TO_SEND ENV ENVVARS_TO_SEND ${ENVVARS_TO_SEND} ARG N_UTTERANCES_CONTEXT=5 ENV N_UTTERANCES_CONTEXT ${N_UTTERANCES_CONTEXT} +ARG FILTER_TOXIC_OR_BADLISTED=1 +ENV FILTER_TOXIC_OR_BADLISTED ${FILTER_TOXIC_OR_BADLISTED} COPY . /src/ WORKDIR /src diff --git a/response_selectors/llm_based_response_selector/server.py b/response_selectors/llm_based_response_selector/server.py index 6322a92182..145c0ce92a 100644 --- a/response_selectors/llm_based_response_selector/server.py +++ b/response_selectors/llm_based_response_selector/server.py @@ -4,10 +4,12 @@ import numpy as np import requests import time +from copy import deepcopy from os import getenv import sentry_sdk from flask import Flask, request, jsonify +from common.utils import is_toxic_or_badlisted_utterance sentry_sdk.init(getenv("SENTRY_DSN")) @@ -20,6 +22,7 @@ GENERATIVE_SERVICE_URL = getenv("GENERATIVE_SERVICE_URL") GENERATIVE_TIMEOUT = getenv("GENERATIVE_TIMEOUT") GENERATIVE_SERVICE_CONFIG = getenv("GENERATIVE_SERVICE_CONFIG") +FILTER_TOXIC_OR_BADLISTED = int(getenv("FILTER_TOXIC_OR_BADLISTED")) N_UTTERANCES_CONTEXT = int(getenv("N_UTTERANCES_CONTEXT")) DEFAULT_CRITERION = "the most appropriate, relevant and non-toxic" CRITERION = getenv("CRITERION", DEFAULT_CRITERION) @@ -35,6 +38,15 @@ ) +def filter_out_badlisted_or_toxic(hypotheses): + clean_hypotheses = [] + for hyp in hypotheses: + is_toxic = is_toxic_or_badlisted_utterance(hyp) + if not is_toxic: + clean_hypotheses += [deepcopy(hyp)] + return clean_hypotheses + + def select_response_by_confidence(hypotheses, confidences): best_id = np.argmax(confidences) result = hypotheses[best_id] @@ -76,9 +88,12 @@ def respond(): selected_confidences = [] for i, dialog in enumerate(dialogs): - confidences = [hyp["confidence"] for hyp in dialog["human_utterances"][-1]["hypotheses"]] - skill_names = [hyp["skill_name"] for hyp in dialog["human_utterances"][-1]["hypotheses"]] hypotheses = [hyp["text"] for hyp in dialog["human_utterances"][-1]["hypotheses"]] + if FILTER_TOXIC_OR_BADLISTED: + hypotheses = filter_out_badlisted_or_toxic(hypotheses) + + confidences = [hyp["confidence"] for hyp in hypotheses] + skill_names = [hyp["skill_name"] for hyp in hypotheses] dialog_context = [uttr["text"] for uttr in dialog["utterances"][-N_UTTERANCES_CONTEXT:]] selected_resp = select_response(dialog_context, hypotheses, confidences) try: From 062d187f39eb43aadcf55078e2ae1d58fd471dd3 Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Sun, 23 Apr 2023 22:01:58 +0300 Subject: [PATCH 05/32] feat: ranking based response selector --- .../llm_based_response_selector/server.py | 6 +- .../Dockerfile | 24 ++++ .../component.yml | 20 ++++ .../pipeline.yml | 10 ++ .../requirements.txt | 8 ++ .../ranking_based_response_selector/server.py | 104 ++++++++++++++++++ .../ranking_based_response_selector/test.py | 19 ++++ .../ranking_based_response_selector/test.sh | 3 + 8 files changed, 191 insertions(+), 3 deletions(-) create mode 100644 response_selectors/ranking_based_response_selector/Dockerfile create mode 100644 response_selectors/ranking_based_response_selector/component.yml create mode 100644 response_selectors/ranking_based_response_selector/pipeline.yml create mode 100644 response_selectors/ranking_based_response_selector/requirements.txt create mode 100644 response_selectors/ranking_based_response_selector/server.py create mode 100644 response_selectors/ranking_based_response_selector/test.py create mode 100644 response_selectors/ranking_based_response_selector/test.sh diff --git a/response_selectors/llm_based_response_selector/server.py b/response_selectors/llm_based_response_selector/server.py index 145c0ce92a..6745dfedab 100644 --- a/response_selectors/llm_based_response_selector/server.py +++ b/response_selectors/llm_based_response_selector/server.py @@ -18,13 +18,13 @@ logger = logging.getLogger(__name__) app = Flask(__name__) +DEFAULT_CRITERION = "the most appropriate, relevant and non-toxic" GENERATIVE_SERVICE_URL = getenv("GENERATIVE_SERVICE_URL") -GENERATIVE_TIMEOUT = getenv("GENERATIVE_TIMEOUT") +GENERATIVE_TIMEOUT = int(getenv("GENERATIVE_TIMEOUT")) GENERATIVE_SERVICE_CONFIG = getenv("GENERATIVE_SERVICE_CONFIG") FILTER_TOXIC_OR_BADLISTED = int(getenv("FILTER_TOXIC_OR_BADLISTED")) N_UTTERANCES_CONTEXT = int(getenv("N_UTTERANCES_CONTEXT")) -DEFAULT_CRITERION = "the most appropriate, relevant and non-toxic" CRITERION = getenv("CRITERION", DEFAULT_CRITERION) PROMPT = f"""Select {CRITERION} response among the hypotheses to the given dialog context. """ \ """Return only the selected response without extra explanations.""" @@ -72,7 +72,7 @@ def select_response(dialog_context, hypotheses, confidences): logger.exception(e) result = select_response_by_confidence(hypotheses, confidences)[0] logger.info(f"Exception in LLM's invocation. Selected a response with the highest confidence.") - logger.info(f"llm-based-response-selector selected:\n`{result}`") + logger.info(f"llm_based_response_selector selected:\n`{result}`") return result diff --git a/response_selectors/ranking_based_response_selector/Dockerfile b/response_selectors/ranking_based_response_selector/Dockerfile new file mode 100644 index 0000000000..d4411c8b7e --- /dev/null +++ b/response_selectors/ranking_based_response_selector/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.7.4 + +RUN mkdir /src + +COPY ./requirements.txt /src/requirements.txt +RUN pip install -r /src/requirements.txt + +ARG LANGUAGE=EN +ENV LANGUAGE ${LANGUAGE} +ARG SERVICE_PORT +ENV SERVICE_PORT ${SERVICE_PORT} +ARG SENTENCE_RANKER_SERVICE_URL +ENV SENTENCE_RANKER_SERVICE_URL ${SENTENCE_RANKER_SERVICE_URL} +ARG SENTENCE_RANKER_TIMEOUT +ENV SENTENCE_RANKER_TIMEOUT ${SENTENCE_RANKER_TIMEOUT} +ARG N_UTTERANCES_CONTEXT=5 +ENV N_UTTERANCES_CONTEXT ${N_UTTERANCES_CONTEXT} +ARG FILTER_TOXIC_OR_BADLISTED=1 +ENV FILTER_TOXIC_OR_BADLISTED ${FILTER_TOXIC_OR_BADLISTED} + +COPY . /src/ +WORKDIR /src + +CMD gunicorn --workers=1 server:app -b 0.0.0.0:${SERVICE_PORT} --timeout=1200 diff --git a/response_selectors/ranking_based_response_selector/component.yml b/response_selectors/ranking_based_response_selector/component.yml new file mode 100644 index 0000000000..fb771b8440 --- /dev/null +++ b/response_selectors/ranking_based_response_selector/component.yml @@ -0,0 +1,20 @@ +llm-based-response-selector: + name: response_selector + display_name: Response Selector + container_name: llm-based-response-selector + component_type: null + model_type: Dictionary/Pattern-based + is_customizable: false + author: DeepPavlov + description: Algorithm that selects a final responses among the given list of candidate + responses + ram_usage: 100M + gpu_usage: null + port: 8005 + endpoints: + - group: response_selectors + endpoint: respond + build_args: + SERVICE_PORT: '8005' + SERVICE_NAME: response_selector + date_created: '2023-03-16T09:45:32' diff --git a/response_selectors/ranking_based_response_selector/pipeline.yml b/response_selectors/ranking_based_response_selector/pipeline.yml new file mode 100644 index 0000000000..7d522c7fde --- /dev/null +++ b/response_selectors/ranking_based_response_selector/pipeline.yml @@ -0,0 +1,10 @@ +llm-based-response-selector: +- group: response_selectors + connector: + protocol: http + url: http://llm-based-response-selector:8005/respond + dialog_formatter: state_formatters.dp_formatters:full_dialog + response_formatter: state_formatters.dp_formatters:base_response_selector_formatter_service + previous_services: + - response_annotators + state_manager_method: add_bot_utterance diff --git a/response_selectors/ranking_based_response_selector/requirements.txt b/response_selectors/ranking_based_response_selector/requirements.txt new file mode 100644 index 0000000000..1d2e7fb9f2 --- /dev/null +++ b/response_selectors/ranking_based_response_selector/requirements.txt @@ -0,0 +1,8 @@ +flask==1.1.1 +itsdangerous==2.0.1 +gunicorn==19.9.0 +requests==2.22.0 +numpy==1.17.2 +sentry-sdk==0.12.3 +jinja2<=3.0.3 +Werkzeug<=2.0.3 diff --git a/response_selectors/ranking_based_response_selector/server.py b/response_selectors/ranking_based_response_selector/server.py new file mode 100644 index 0000000000..de1b74a887 --- /dev/null +++ b/response_selectors/ranking_based_response_selector/server.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python + +import logging +import numpy as np +import requests +import time +from copy import deepcopy +from os import getenv + +import sentry_sdk +from flask import Flask, request, jsonify +from common.utils import is_toxic_or_badlisted_utterance + + +sentry_sdk.init(getenv("SENTRY_DSN")) + +logging.basicConfig(format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=logging.INFO) +logger = logging.getLogger(__name__) + +app = Flask(__name__) + +SENTENCE_RANKER_SERVICE_URL = getenv("SENTENCE_RANKER_SERVICE_URL") +SENTENCE_RANKER_TIMEOUT = int(getenv("SENTENCE_RANKER_TIMEOUT")) +FILTER_TOXIC_OR_BADLISTED = int(getenv("FILTER_TOXIC_OR_BADLISTED")) +N_UTTERANCES_CONTEXT = int(getenv("N_UTTERANCES_CONTEXT")) + + +def filter_out_badlisted_or_toxic(hypotheses): + clean_hypotheses = [] + for hyp in hypotheses: + is_toxic = is_toxic_or_badlisted_utterance(hyp) + if not is_toxic: + clean_hypotheses += [deepcopy(hyp)] + return clean_hypotheses + + +def select_response_by_confidence(hypotheses, confidences): + best_id = np.argmax(confidences) + result = hypotheses[best_id] + return result, best_id + + +def select_response(dialog_context, hypotheses, confidences): + try: + dialog_context = "\n".join(dialog_context) + pairs = [[dialog_context, hyp["text"]] for hyp in hypotheses] + scores = requests.post( + SENTENCE_RANKER_SERVICE_URL, + json={"sentence_pairs": pairs}, + timeout=SENTENCE_RANKER_TIMEOUT, + ).json() + scores = np.array(scores[0]["batch"]) + result = select_response_by_confidence(hypotheses, scores)[0] + except Exception as e: + sentry_sdk.capture_exception(e) + logger.exception(e) + result = select_response_by_confidence(hypotheses, confidences)[0] + logger.info(f"Exception in LLM's invocation. Selected a response with the highest confidence.") + logger.info(f"ranking_based_response_selector selected:\n`{result}`") + + return result + + +@app.route("/respond", methods=["POST"]) +def respond(): + st_time = time.time() + + dialogs = request.json["dialogs"] + + selected_skill_names = [] + selected_responses = [] + selected_confidences = [] + + for i, dialog in enumerate(dialogs): + hypotheses = [hyp["text"] for hyp in dialog["human_utterances"][-1]["hypotheses"]] + if FILTER_TOXIC_OR_BADLISTED: + hypotheses = filter_out_badlisted_or_toxic(hypotheses) + + confidences = [hyp["confidence"] for hyp in hypotheses] + skill_names = [hyp["skill_name"] for hyp in hypotheses] + dialog_context = [uttr["text"] for uttr in dialog["utterances"][-N_UTTERANCES_CONTEXT:]] + selected_resp = select_response(dialog_context, hypotheses, confidences) + try: + best_id = hypotheses.index(selected_resp) + selected_skill_names.append(skill_names[best_id]) + selected_responses.append(selected_resp) + selected_confidences.append(confidences[best_id]) + except Exception as e: + sentry_sdk.capture_exception(e) + logger.exception(e) + logger.info("Exception in finding selected by LLM response in hypotheses. " + "Selected a response with the highest confidence.") + selected_resp, best_id = select_response_by_confidence(hypotheses, confidences) + selected_skill_names.append(skill_names[best_id]) + selected_responses.append(selected_resp) + selected_confidences.append(confidences[best_id]) + + total_time = time.time() - st_time + logger.info(f"ranking_based_response_selector exec time = {total_time:.3f}s") + return jsonify(list(zip(selected_skill_names, selected_responses, selected_confidences))) + + +if __name__ == "__main__": + app.run(debug=False, host="0.0.0.0", port=3000) diff --git a/response_selectors/ranking_based_response_selector/test.py b/response_selectors/ranking_based_response_selector/test.py new file mode 100644 index 0000000000..7ed8204c19 --- /dev/null +++ b/response_selectors/ranking_based_response_selector/test.py @@ -0,0 +1,19 @@ +import requests +import json +from os import getenv + + +SERVICE_PORT = getenv("SERVICE_PORT") + + +def main(): + with open("test_data.json", "r") as f: + data = json.load(f) + # To skip "Oh, and remember this dialog's id" that raises error due to absence of 'dialog_id' field in test_data. + data["dialogs"][0]["human_utterances"].append(data["dialogs"][0]["human_utterances"][0]) + result = requests.post(f"http://0.0.0.0:{SERVICE_PORT}/respond", json=data).json() + assert result[0][0] == "program_y", print(result) + + +if __name__ == "__main__": + main() diff --git a/response_selectors/ranking_based_response_selector/test.sh b/response_selectors/ranking_based_response_selector/test.sh new file mode 100644 index 0000000000..61672db785 --- /dev/null +++ b/response_selectors/ranking_based_response_selector/test.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +python test.py From b4df900adaea96a5e158c1c5ab8ecaba7e93b0ac Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Mon, 24 Apr 2023 11:53:33 +0300 Subject: [PATCH 06/32] feat: model cards --- .../llm_based_response_selector/component.yml | 13 ++++++++----- .../llm_based_response_selector/pipeline.yml | 4 ++-- .../ranking_based_response_selector/component.yml | 8 ++++---- .../ranking_based_response_selector/pipeline.yml | 4 ++-- state_formatters/dp_formatters.py | 7 +++++++ 5 files changed, 23 insertions(+), 13 deletions(-) diff --git a/response_selectors/llm_based_response_selector/component.yml b/response_selectors/llm_based_response_selector/component.yml index fb771b8440..d04f5849fe 100644 --- a/response_selectors/llm_based_response_selector/component.yml +++ b/response_selectors/llm_based_response_selector/component.yml @@ -4,17 +4,20 @@ llm-based-response-selector: container_name: llm-based-response-selector component_type: null model_type: Dictionary/Pattern-based - is_customizable: false + is_customizable: true author: DeepPavlov description: Algorithm that selects a final responses among the given list of candidate - responses + responses via asking the given LLM service the question `Select {CRITERION} response + among the hypotheses to the given dialog context. Return only the selected response + without extra explanations.` where default CRITERION is `the most appropriate, + relevant and non-toxic` and could be customized ram_usage: 100M gpu_usage: null - port: 8005 + port: 8003 endpoints: - group: response_selectors endpoint: respond build_args: - SERVICE_PORT: '8005' + SERVICE_PORT: '8003' SERVICE_NAME: response_selector - date_created: '2023-03-16T09:45:32' + date_created: '2023-04-24T09:45:32' diff --git a/response_selectors/llm_based_response_selector/pipeline.yml b/response_selectors/llm_based_response_selector/pipeline.yml index 7d522c7fde..e85c048e0c 100644 --- a/response_selectors/llm_based_response_selector/pipeline.yml +++ b/response_selectors/llm_based_response_selector/pipeline.yml @@ -2,8 +2,8 @@ llm-based-response-selector: - group: response_selectors connector: protocol: http - url: http://llm-based-response-selector:8005/respond - dialog_formatter: state_formatters.dp_formatters:full_dialog + url: http://llm-based-response-selector:8003/respond + dialog_formatter: state_formatters.dp_formatters:cropped_dialog response_formatter: state_formatters.dp_formatters:base_response_selector_formatter_service previous_services: - response_annotators diff --git a/response_selectors/ranking_based_response_selector/component.yml b/response_selectors/ranking_based_response_selector/component.yml index fb771b8440..ddc9795e56 100644 --- a/response_selectors/ranking_based_response_selector/component.yml +++ b/response_selectors/ranking_based_response_selector/component.yml @@ -7,14 +7,14 @@ llm-based-response-selector: is_customizable: false author: DeepPavlov description: Algorithm that selects a final responses among the given list of candidate - responses + responses via the given ranking service ram_usage: 100M gpu_usage: null - port: 8005 + port: 8002 endpoints: - group: response_selectors endpoint: respond build_args: - SERVICE_PORT: '8005' + SERVICE_PORT: '8002' SERVICE_NAME: response_selector - date_created: '2023-03-16T09:45:32' + date_created: '2023-04-24T09:45:32' diff --git a/response_selectors/ranking_based_response_selector/pipeline.yml b/response_selectors/ranking_based_response_selector/pipeline.yml index 7d522c7fde..b0ac27e1e8 100644 --- a/response_selectors/ranking_based_response_selector/pipeline.yml +++ b/response_selectors/ranking_based_response_selector/pipeline.yml @@ -2,8 +2,8 @@ llm-based-response-selector: - group: response_selectors connector: protocol: http - url: http://llm-based-response-selector:8005/respond - dialog_formatter: state_formatters.dp_formatters:full_dialog + url: http://llm-based-response-selector:8002/respond + dialog_formatter: state_formatters.dp_formatters:cropped_dialog response_formatter: state_formatters.dp_formatters:base_response_selector_formatter_service previous_services: - response_annotators diff --git a/state_formatters/dp_formatters.py b/state_formatters/dp_formatters.py index fd3a237c2d..8af9c5d625 100755 --- a/state_formatters/dp_formatters.py +++ b/state_formatters/dp_formatters.py @@ -521,6 +521,13 @@ def persona_bot_formatter(dialog: Dict): ] +def cropped_dialog(dialog: Dict): + dialog = utils.get_last_n_turns(dialog) + dialog = utils.remove_clarification_turns_from_dialog(dialog) + dialog = utils.replace_with_annotated_utterances(dialog, mode="punct_sent") + return [{"dialogs": [dialog]}] + + def full_dialog(dialog: Dict): return [{"dialogs": [dialog]}] From a845b9bdfb6cb090cb4016520d0675dc0e319672 Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Mon, 24 Apr 2023 13:02:26 +0300 Subject: [PATCH 07/32] feat: ranking and refactoring --- .../llm_based_response_selector/server.py | 8 +-- .../Dockerfile | 2 + .../ranking_based_response_selector/server.py | 65 +++++++++++-------- 3 files changed, 45 insertions(+), 30 deletions(-) diff --git a/response_selectors/llm_based_response_selector/server.py b/response_selectors/llm_based_response_selector/server.py index 6745dfedab..99162b7117 100644 --- a/response_selectors/llm_based_response_selector/server.py +++ b/response_selectors/llm_based_response_selector/server.py @@ -47,8 +47,8 @@ def filter_out_badlisted_or_toxic(hypotheses): return clean_hypotheses -def select_response_by_confidence(hypotheses, confidences): - best_id = np.argmax(confidences) +def select_response_by_scores(hypotheses, scores): + best_id = np.argmax(scores) result = hypotheses[best_id] return result, best_id @@ -70,7 +70,7 @@ def select_response(dialog_context, hypotheses, confidences): except Exception as e: sentry_sdk.capture_exception(e) logger.exception(e) - result = select_response_by_confidence(hypotheses, confidences)[0] + result = select_response_by_scores(hypotheses, confidences)[0] logger.info(f"Exception in LLM's invocation. Selected a response with the highest confidence.") logger.info(f"llm_based_response_selector selected:\n`{result}`") @@ -106,7 +106,7 @@ def respond(): logger.exception(e) logger.info("Exception in finding selected by LLM response in hypotheses. " "Selected a response with the highest confidence.") - selected_resp, best_id = select_response_by_confidence(hypotheses, confidences) + selected_resp, best_id = select_response_by_scores(hypotheses, confidences) selected_skill_names.append(skill_names[best_id]) selected_responses.append(selected_resp) selected_confidences.append(confidences[best_id]) diff --git a/response_selectors/ranking_based_response_selector/Dockerfile b/response_selectors/ranking_based_response_selector/Dockerfile index d4411c8b7e..3788831fd0 100644 --- a/response_selectors/ranking_based_response_selector/Dockerfile +++ b/response_selectors/ranking_based_response_selector/Dockerfile @@ -9,6 +9,8 @@ ARG LANGUAGE=EN ENV LANGUAGE ${LANGUAGE} ARG SERVICE_PORT ENV SERVICE_PORT ${SERVICE_PORT} +ARG SENTENCE_RANKER_ANNOTATION_NAME +ENV SENTENCE_RANKER_ANNOTATION_NAME ${SENTENCE_RANKER_ANNOTATION_NAME} ARG SENTENCE_RANKER_SERVICE_URL ENV SENTENCE_RANKER_SERVICE_URL ${SENTENCE_RANKER_SERVICE_URL} ARG SENTENCE_RANKER_TIMEOUT diff --git a/response_selectors/ranking_based_response_selector/server.py b/response_selectors/ranking_based_response_selector/server.py index de1b74a887..06b4be77e4 100644 --- a/response_selectors/ranking_based_response_selector/server.py +++ b/response_selectors/ranking_based_response_selector/server.py @@ -19,10 +19,14 @@ app = Flask(__name__) +SENTENCE_RANKER_ANNOTATION_NAME = getenv("SENTENCE_RANKER_ANNOTATION_NAME") SENTENCE_RANKER_SERVICE_URL = getenv("SENTENCE_RANKER_SERVICE_URL") SENTENCE_RANKER_TIMEOUT = int(getenv("SENTENCE_RANKER_TIMEOUT")) FILTER_TOXIC_OR_BADLISTED = int(getenv("FILTER_TOXIC_OR_BADLISTED")) N_UTTERANCES_CONTEXT = int(getenv("N_UTTERANCES_CONTEXT")) +assert SENTENCE_RANKER_ANNOTATION_NAME or SENTENCE_RANKER_SERVICE_URL, logger.error( + "Ranker service URL or annotator name should be given" +) def filter_out_badlisted_or_toxic(hypotheses): @@ -34,28 +38,39 @@ def filter_out_badlisted_or_toxic(hypotheses): return clean_hypotheses -def select_response_by_confidence(hypotheses, confidences): - best_id = np.argmax(confidences) +def select_response_by_scores(hypotheses, scores): + best_id = np.argmax(scores) result = hypotheses[best_id] return result, best_id -def select_response(dialog_context, hypotheses, confidences): - try: - dialog_context = "\n".join(dialog_context) - pairs = [[dialog_context, hyp["text"]] for hyp in hypotheses] - scores = requests.post( - SENTENCE_RANKER_SERVICE_URL, - json={"sentence_pairs": pairs}, - timeout=SENTENCE_RANKER_TIMEOUT, - ).json() - scores = np.array(scores[0]["batch"]) - result = select_response_by_confidence(hypotheses, scores)[0] - except Exception as e: - sentry_sdk.capture_exception(e) - logger.exception(e) - result = select_response_by_confidence(hypotheses, confidences)[0] - logger.info(f"Exception in LLM's invocation. Selected a response with the highest confidence.") +def get_scores(dialog_context, hypotheses): + if all([SENTENCE_RANKER_ANNOTATION_NAME in hyp.get("annotations", {}) for hyp in hypotheses]): + scores = [hyp.get("annotations", {}).get("SENTENCE_RANKER_ANNOTATION_NAME", 0.0) for hyp in hypotheses] + logger.info(f"Selected a response via Sentence Ranker Annotator.") + else: + try: + dialog_context = "\n".join(dialog_context) + pairs = [[dialog_context, hyp["text"]] for hyp in hypotheses] + scores = requests.post( + SENTENCE_RANKER_SERVICE_URL, + json={"sentence_pairs": pairs}, + timeout=SENTENCE_RANKER_TIMEOUT, + ).json() + scores = np.array(scores[0]["batch"]) + logger.info(f"Selected a response via Sentence Ranker Service.") + except Exception as e: + sentry_sdk.capture_exception(e) + scores = [hyp["confidence"] for hyp in hypotheses] + logger.exception(e) + logger.info(f"Selected a response via Confidence.") + return scores + + +def select_response(dialog_context, hypotheses): + + scores = get_scores(dialog_context, hypotheses) + result = select_response_by_scores(hypotheses, scores)[0] logger.info(f"ranking_based_response_selector selected:\n`{result}`") return result @@ -76,24 +91,22 @@ def respond(): if FILTER_TOXIC_OR_BADLISTED: hypotheses = filter_out_badlisted_or_toxic(hypotheses) - confidences = [hyp["confidence"] for hyp in hypotheses] - skill_names = [hyp["skill_name"] for hyp in hypotheses] dialog_context = [uttr["text"] for uttr in dialog["utterances"][-N_UTTERANCES_CONTEXT:]] - selected_resp = select_response(dialog_context, hypotheses, confidences) + selected_resp = select_response(dialog_context, hypotheses) try: best_id = hypotheses.index(selected_resp) - selected_skill_names.append(skill_names[best_id]) selected_responses.append(selected_resp) - selected_confidences.append(confidences[best_id]) + selected_skill_names.append(hypotheses[best_id]["skill_name"]) + selected_confidences.append(hypotheses[best_id]["confidence"]) except Exception as e: sentry_sdk.capture_exception(e) logger.exception(e) logger.info("Exception in finding selected by LLM response in hypotheses. " "Selected a response with the highest confidence.") - selected_resp, best_id = select_response_by_confidence(hypotheses, confidences) - selected_skill_names.append(skill_names[best_id]) + selected_resp, best_id = select_response_by_scores(hypotheses, [hyp["confidence"] for hyp in hypotheses]) selected_responses.append(selected_resp) - selected_confidences.append(confidences[best_id]) + selected_skill_names.append(hypotheses[best_id]["skill_name"]) + selected_confidences.append(hypotheses[best_id]["confidence"]) total_time = time.time() - st_time logger.info(f"ranking_based_response_selector exec time = {total_time:.3f}s") From 0e77064bca95baa7d92a83605b56c3b0683bd97d Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Mon, 24 Apr 2023 15:53:11 +0300 Subject: [PATCH 08/32] feat: component files --- .../llm_based_response_selector/component.yml | 7 +++++++ .../llm_based_response_selector/server.py | 2 +- .../ranking_based_response_selector/component.yml | 10 ++++++++-- .../ranking_based_response_selector/pipeline.yml | 4 ++-- .../ranking_based_response_selector/server.py | 2 +- 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/response_selectors/llm_based_response_selector/component.yml b/response_selectors/llm_based_response_selector/component.yml index d04f5849fe..6796c15c4b 100644 --- a/response_selectors/llm_based_response_selector/component.yml +++ b/response_selectors/llm_based_response_selector/component.yml @@ -20,4 +20,11 @@ llm-based-response-selector: build_args: SERVICE_PORT: '8003' SERVICE_NAME: response_selector + LANGUAGE: EN + GENERATIVE_SERVICE_URL: http://openai-api-davinci3:8131/respond + GENERATIVE_SERVICE_CONFIG: openai-text-davinci-003-long.json + GENERATIVE_TIMEOUT: 7 + ENVVARS_TO_SEND: OPENAI_API_KEY,OPENAI_ORGANIZATION + N_UTTERANCES_CONTEXT: 5 + FILTER_TOXIC_OR_BADLISTED: 1 date_created: '2023-04-24T09:45:32' diff --git a/response_selectors/llm_based_response_selector/server.py b/response_selectors/llm_based_response_selector/server.py index 99162b7117..67f8714efd 100644 --- a/response_selectors/llm_based_response_selector/server.py +++ b/response_selectors/llm_based_response_selector/server.py @@ -104,7 +104,7 @@ def respond(): except Exception as e: sentry_sdk.capture_exception(e) logger.exception(e) - logger.info("Exception in finding selected by LLM response in hypotheses. " + logger.info("Exception in finding selected response in hypotheses. " "Selected a response with the highest confidence.") selected_resp, best_id = select_response_by_scores(hypotheses, confidences) selected_skill_names.append(skill_names[best_id]) diff --git a/response_selectors/ranking_based_response_selector/component.yml b/response_selectors/ranking_based_response_selector/component.yml index ddc9795e56..2df1ba7549 100644 --- a/response_selectors/ranking_based_response_selector/component.yml +++ b/response_selectors/ranking_based_response_selector/component.yml @@ -1,7 +1,7 @@ -llm-based-response-selector: +ranking-based-response-selector: name: response_selector display_name: Response Selector - container_name: llm-based-response-selector + container_name: ranking-based-response-selector component_type: null model_type: Dictionary/Pattern-based is_customizable: false @@ -17,4 +17,10 @@ llm-based-response-selector: build_args: SERVICE_PORT: '8002' SERVICE_NAME: response_selector + LANGUAGE: EN + SENTENCE_RANKER_ANNOTATION_NAME: + SENTENCE_RANKER_SERVICE_URL: + SENTENCE_RANKER_TIMEOUT: 3 + N_UTTERANCES_CONTEXT: 5 + FILTER_TOXIC_OR_BADLISTED: 1 date_created: '2023-04-24T09:45:32' diff --git a/response_selectors/ranking_based_response_selector/pipeline.yml b/response_selectors/ranking_based_response_selector/pipeline.yml index b0ac27e1e8..8028c2575a 100644 --- a/response_selectors/ranking_based_response_selector/pipeline.yml +++ b/response_selectors/ranking_based_response_selector/pipeline.yml @@ -1,8 +1,8 @@ -llm-based-response-selector: +ranking-based-response-selector: - group: response_selectors connector: protocol: http - url: http://llm-based-response-selector:8002/respond + url: http://ranking-based-response-selector:8002/respond dialog_formatter: state_formatters.dp_formatters:cropped_dialog response_formatter: state_formatters.dp_formatters:base_response_selector_formatter_service previous_services: diff --git a/response_selectors/ranking_based_response_selector/server.py b/response_selectors/ranking_based_response_selector/server.py index 06b4be77e4..5dddcf0b23 100644 --- a/response_selectors/ranking_based_response_selector/server.py +++ b/response_selectors/ranking_based_response_selector/server.py @@ -101,7 +101,7 @@ def respond(): except Exception as e: sentry_sdk.capture_exception(e) logger.exception(e) - logger.info("Exception in finding selected by LLM response in hypotheses. " + logger.info("Exception in finding selected response in hypotheses. " "Selected a response with the highest confidence.") selected_resp, best_id = select_response_by_scores(hypotheses, [hyp["confidence"] for hyp in hypotheses]) selected_responses.append(selected_resp) From 5cd64f778aac20904396bf86c6f294ce382ac03e Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Mon, 24 Apr 2023 16:10:16 +0300 Subject: [PATCH 09/32] feat: component description --- .../llm_based_response_selector/component.yml | 92 ++- .../llm_based_response_selector/pipeline.yml | 34 +- .../test_data.json | 681 ++++++++++++++++++ .../test_data.json | 681 ++++++++++++++++++ 4 files changed, 1484 insertions(+), 4 deletions(-) create mode 100644 response_selectors/llm_based_response_selector/test_data.json create mode 100644 response_selectors/ranking_based_response_selector/test_data.json diff --git a/response_selectors/llm_based_response_selector/component.yml b/response_selectors/llm_based_response_selector/component.yml index 6796c15c4b..0bef0c738b 100644 --- a/response_selectors/llm_based_response_selector/component.yml +++ b/response_selectors/llm_based_response_selector/component.yml @@ -1,7 +1,7 @@ -llm-based-response-selector: +davinci3-based-response-selector: name: response_selector display_name: Response Selector - container_name: llm-based-response-selector + container_name: davinci3-based-response-selector component_type: null model_type: Dictionary/Pattern-based is_customizable: true @@ -28,3 +28,91 @@ llm-based-response-selector: N_UTTERANCES_CONTEXT: 5 FILTER_TOXIC_OR_BADLISTED: 1 date_created: '2023-04-24T09:45:32' +chatgpt-based-response-selector: + name: response_selector + display_name: Response Selector + container_name: chatgpt-based-response-selector + component_type: null + model_type: Dictionary/Pattern-based + is_customizable: true + author: DeepPavlov + description: Algorithm that selects a final responses among the given list of candidate + responses via asking the given LLM service the question `Select {CRITERION} response + among the hypotheses to the given dialog context. Return only the selected response + without extra explanations.` where default CRITERION is `the most appropriate, + relevant and non-toxic` and could be customized + ram_usage: 100M + gpu_usage: null + port: 8003 + endpoints: + - group: response_selectors + endpoint: respond + build_args: + SERVICE_PORT: '8003' + SERVICE_NAME: response_selector + LANGUAGE: EN + GENERATIVE_SERVICE_URL: http://openai-api-chatgpt:8145/respond + GENERATIVE_SERVICE_CONFIG: openai-chatgpt.json + GENERATIVE_TIMEOUT: 7 + ENVVARS_TO_SEND: OPENAI_API_KEY,OPENAI_ORGANIZATION + N_UTTERANCES_CONTEXT: 5 + FILTER_TOXIC_OR_BADLISTED: 1 + date_created: '2023-04-24T09:45:32' +oasst12b-based-response-selector: + name: response_selector + display_name: Response Selector + container_name: oasst12b-based-response-selector + component_type: null + model_type: Dictionary/Pattern-based + is_customizable: true + author: DeepPavlov + description: Algorithm that selects a final responses among the given list of candidate + responses via asking the given LLM service the question `Select {CRITERION} response + among the hypotheses to the given dialog context. Return only the selected response + without extra explanations.` where default CRITERION is `the most appropriate, + relevant and non-toxic` and could be customized + ram_usage: 100M + gpu_usage: null + port: 8003 + endpoints: + - group: response_selectors + endpoint: respond + build_args: + SERVICE_PORT: '8003' + SERVICE_NAME: response_selector + LANGUAGE: EN + GENERATIVE_SERVICE_URL: http://transformers-lm-oasst12b:8158/respond + GENERATIVE_SERVICE_CONFIG: default_generative_config.json + GENERATIVE_TIMEOUT: 5 + N_UTTERANCES_CONTEXT: 5 + FILTER_TOXIC_OR_BADLISTED: 1 + date_created: '2023-04-24T09:45:32' +gptj-based-response-selector: + name: response_selector + display_name: Response Selector + container_name: gptj-based-response-selector + component_type: null + model_type: Dictionary/Pattern-based + is_customizable: true + author: DeepPavlov + description: Algorithm that selects a final responses among the given list of candidate + responses via asking the given LLM service the question `Select {CRITERION} response + among the hypotheses to the given dialog context. Return only the selected response + without extra explanations.` where default CRITERION is `the most appropriate, + relevant and non-toxic` and could be customized + ram_usage: 100M + gpu_usage: null + port: 8003 + endpoints: + - group: response_selectors + endpoint: respond + build_args: + SERVICE_PORT: '8003' + SERVICE_NAME: response_selector + LANGUAGE: EN + GENERATIVE_SERVICE_URL: http://transformers-lm-gptj:8130/respond + GENERATIVE_SERVICE_CONFIG: default_generative_config.json + GENERATIVE_TIMEOUT: 5 + N_UTTERANCES_CONTEXT: 5 + FILTER_TOXIC_OR_BADLISTED: 1 + date_created: '2023-04-24T09:45:32' diff --git a/response_selectors/llm_based_response_selector/pipeline.yml b/response_selectors/llm_based_response_selector/pipeline.yml index e85c048e0c..3a02488613 100644 --- a/response_selectors/llm_based_response_selector/pipeline.yml +++ b/response_selectors/llm_based_response_selector/pipeline.yml @@ -1,8 +1,38 @@ -llm-based-response-selector: +davinci3-based-response-selector: - group: response_selectors connector: protocol: http - url: http://llm-based-response-selector:8003/respond + url: http://davinci3-based-response-selector:8003/respond + dialog_formatter: state_formatters.dp_formatters:cropped_dialog + response_formatter: state_formatters.dp_formatters:base_response_selector_formatter_service + previous_services: + - response_annotators + state_manager_method: add_bot_utterance +chatgpt-based-response-selector: +- group: response_selectors + connector: + protocol: http + url: http://chatgpt-based-response-selector:8003/respond + dialog_formatter: state_formatters.dp_formatters:cropped_dialog + response_formatter: state_formatters.dp_formatters:base_response_selector_formatter_service + previous_services: + - response_annotators + state_manager_method: add_bot_utterance +oasst12b-based-response-selector: +- group: response_selectors + connector: + protocol: http + url: http://oasst12b-based-response-selector:8003/respond + dialog_formatter: state_formatters.dp_formatters:cropped_dialog + response_formatter: state_formatters.dp_formatters:base_response_selector_formatter_service + previous_services: + - response_annotators + state_manager_method: add_bot_utterance +gptj-based-response-selector: +- group: response_selectors + connector: + protocol: http + url: http://gptj-based-response-selector:8003/respond dialog_formatter: state_formatters.dp_formatters:cropped_dialog response_formatter: state_formatters.dp_formatters:base_response_selector_formatter_service previous_services: diff --git a/response_selectors/llm_based_response_selector/test_data.json b/response_selectors/llm_based_response_selector/test_data.json new file mode 100644 index 0000000000..c08ed424ff --- /dev/null +++ b/response_selectors/llm_based_response_selector/test_data.json @@ -0,0 +1,681 @@ +{ + "dialogs": [ + { + "id": "374475c4139a489fa74981941c194cb0", + "utterances": [ + { + "text": "hey.", + "user": { + "id": "5e61287da7d26764f5d4a0ed", + "user_telegram_id": "dsadas", + "persona": {}, + "profile": { + "name": null, + "gender": null, + "birthdate": null, + "location": null, + "home_coordinates": null, + "work_coordinates": null, + "occupation": null, + "income_per_year": null + }, + "attributes": {}, + "user_type": "human" + }, + "annotations": { + "badlisted_words": { + "inappropriate": false, + "profanity": false, + "restricted_topics": false + }, + "asr": { + "asr_confidence": "undefined" + }, + "sentiment_classification": { + "text": [ + "positive", + 0.7487750053405762 + ] + }, + "emotion_classification": { + "text": { + "anger": 0.2401413470506668, + "fear": 0.2673250734806061, + "joy": 0.25821077823638916, + "disgust": 0.3577530086040497, + "sadness": 0.21969465911388397, + "surprise": 0.2795693576335907, + "neutral": 0.9991581439971924 + } + }, + "toxic_classification": { + "identity_hate": 0.0024989843368530273, + "insult": 0.0030942559242248535, + "obscene": 0.00198972225189209, + "severe_toxic": 0.0007079243659973145, + "sexual_explicit": 0.0022467062808573246, + "threat": 0.0014101153938099742, + "toxic": 0.007221863605082035 + }, + "sentseg": { + "punct_sent": "hey.", + "segments": [ + "hey." + ] + }, + "intent_catcher": { + "cant_do": { + "confidence": 0.0, + "detected": 0 + }, + "doing_well": { + "confidence": 0.0, + "detected": 0 + }, + "dont_understand": { + "confidence": 0.0, + "detected": 0 + }, + "exit": { + "confidence": 0.0, + "detected": 0 + }, + "lets_chat_about": { + "confidence": 0.0, + "detected": 0 + }, + "no": { + "confidence": 0.0, + "detected": 0 + }, + "opinion_request": { + "confidence": 0.0, + "detected": 0 + }, + "repeat": { + "confidence": 0.0, + "detected": 0 + }, + "stupid": { + "confidence": 0.0, + "detected": 0 + }, + "tell_me_a_story": { + "confidence": 0.0, + "detected": 0 + }, + "tell_me_more": { + "confidence": 0.0, + "detected": 0 + }, + "topic_switching": { + "confidence": 0.0, + "detected": 0 + }, + "weather_forecast_intent": { + "confidence": 0.0, + "detected": 0 + }, + "what_can_you_do": { + "confidence": 0.0, + "detected": 0 + }, + "what_is_your_job": { + "confidence": 0.0, + "detected": 0 + }, + "what_is_your_name": { + "confidence": 0.0, + "detected": 0 + }, + "what_time": { + "confidence": 0.0, + "detected": 0 + }, + "where_are_you_from": { + "confidence": 0.0, + "detected": 0 + }, + "who_made_you": { + "confidence": 0.0, + "detected": 0 + }, + "yes": { + "confidence": 0.0, + "detected": 0 + } + }, + "spacy_nounphrases": [], + "ner": [ + [ + { + "confidence": 1, + "end_pos": 1, + "start_pos": 0, + "text": "hey", + "type": "LOC" + } + ] + ], + "cobot_dialogact_intents": { + "text": [ + "General_ChatIntent" + ] + }, + "cobot_dialogact_topics": { + "text": [ + "Phatic" + ] + }, + "cobot_offensiveness": { + "text": [ + "non-toxic" + ], + "confidence": [ + 1.0309148e-05 + ], + "is_badlisted": [ + "not badlist" + ] + }, + "cobot_topics": { + "text": [ + "Phatic" + ] + }, + "sentrewrite": { + "clusters": [], + "modified_sents": [ + "hey." + ] + } + }, + "hypotheses": [ + { + "skill_name": "dummy_skill", + "annotations": { + "toxic_classification": { + "identity_hate": 0.00024211406707763672, + "insult": 0.0004429817199707031, + "obscene": 0.0001055598258972168, + "severe_toxic": 5.415081977844238e-05, + "sexual_explicit": 9.942054748535156e-05, + "threat": 0.00014013051986694336, + "toxic": 0.0009675323963165283 + }, + "stop_detect": { + "stop": 0.6065886616706848, + "continue": 0.4208565950393677 + }, + "convers_evaluator_annotator": { + "isResponseComprehensible": 0.909, + "isResponseErroneous": 0.808, + "isResponseInteresting": 0.073, + "isResponseOnTopic": 0.158, + "responseEngagesUser": 0.469 + }, + "badlisted_words": { + "inappropriate": false, + "profanity": false, + "restricted_topics": false + } + }, + "text": "I didn't get it. Sorry.", + "confidence": 0.5, + "type": "dummy" + }, + { + "skill_name": "dummy_skill", + "annotations": { + "toxic_classification": { + "identity_hate": 0.00024211406707763672, + "insult": 0.0004429817199707031, + "obscene": 0.0001055598258972168, + "severe_toxic": 5.415081977844238e-05, + "sexual_explicit": 9.942054748535156e-05, + "threat": 0.00014013051986694336, + "toxic": 0.0009675323963165283 + }, + "stop_detect": { + "stop": 0.6065886616706848, + "continue": 0.4208565950393677 + }, + "convers_evaluator_annotator": { + "isResponseComprehensible": 0.909, + "isResponseErroneous": 0.808, + "isResponseInteresting": 0.073, + "isResponseOnTopic": 0.158, + "responseEngagesUser": 0.469 + }, + "badlisted_words": { + "inappropriate": false, + "profanity": false, + "restricted_topics": false + } + }, + "text": "are you a comedy fan?", + "confidence": 0.6, + "type": "topic_question" + }, + { + "skill_name": "movie_tfidf_retrieval", + "annotations": { + "toxic_classification": { + "identity_hate": 0.0001259446144104004, + "insult": 0.00027686357498168945, + "obscene": 5.97834587097168e-05, + "severe_toxic": 3.403425216674805e-05, + "sexual_explicit": 8.13603401184082e-05, + "threat": 0.00012931227684020996, + "toxic": 0.0005629658699035645 + }, + "stop_detect": { + "stop": 0.5833511352539062, + "continue": 0.46003755927085876 + }, + "convers_evaluator_annotator": { + "isResponseComprehensible": 0.281, + "isResponseErroneous": 0.531, + "isResponseInteresting": 0.228, + "isResponseOnTopic": 0.254, + "responseEngagesUser": 0.536 + }, + "badlisted_words": { + "inappropriate": false, + "profanity": false, + "restricted_topics": false + } + }, + "text": "i got you haha. what do you think abuot celebrities? judge judy makes 123, 000 per episode apparently!", + "confidence": 0.38232852805460565 + }, + { + "skill_name": "program_y", + "annotations": { + "toxic_classification": { + "identity_hate": 8.749961853027344e-05, + "insult": 0.00024232268333435059, + "obscene": 2.828240394592285e-05, + "severe_toxic": 1.8358230590820312e-05, + "sexual_explicit": 2.9712915420532227e-05, + "threat": 6.490945816040039e-05, + "toxic": 0.00043845176696777344 + }, + "stop_detect": { + "stop": 0.5808720588684082, + "continue": 0.45234695076942444 + }, + "convers_evaluator_annotator": { + "isResponseComprehensible": 0.984, + "isResponseErroneous": 0.614, + "isResponseInteresting": 0.253, + "isResponseOnTopic": 0.226, + "responseEngagesUser": 0.56 + }, + "badlisted_words": { + "inappropriate": false, + "profanity": false, + "restricted_topics": false + } + }, + "text": "Good Morning, this is an Alexa Prize Socialbot! How are you?", + "confidence": 0.98 + } + ], + "date_time": "2020-03-05 17:07:20.926781", + "attributes": {} + } + ], + "human_utterances": [ + { + "text": "hey.", + "user": { + "id": "5e61287da7d26764f5d4a0ed", + "user_telegram_id": "dsadas", + "persona": {}, + "profile": { + "name": null, + "gender": null, + "birthdate": null, + "location": null, + "home_coordinates": null, + "work_coordinates": null, + "occupation": null, + "income_per_year": null + }, + "attributes": {}, + "user_type": "human" + }, + "annotations": { + "badlisted_words": { + "inappropriate": false, + "profanity": false, + "restricted_topics": false + }, + "asr": { + "asr_confidence": "undefined" + }, + "sentiment_classification": { + "text": [ + "positive", + 0.7487750053405762 + ] + }, + "emotion_classification": { + "text": { + "anger": 0.2401413470506668, + "fear": 0.2673250734806061, + "joy": 0.25821077823638916, + "disgust": 0.3577530086040497, + "sadness": 0.21969465911388397, + "surprise": 0.2795693576335907, + "neutral": 0.9991581439971924 + } + }, + "toxic_classification": { + "identity_hate": 0.0024989843368530273, + "insult": 0.0030942559242248535, + "obscene": 0.00198972225189209, + "severe_toxic": 0.0007079243659973145, + "sexual_explicit": 0.0022467062808573246, + "threat": 0.0014101153938099742, + "toxic": 0.007221863605082035 + }, + "sentseg": { + "punct_sent": "hey.", + "segments": [ + "hey." + ] + }, + "intent_catcher": { + "cant_do": { + "confidence": 0.0, + "detected": 0 + }, + "doing_well": { + "confidence": 0.0, + "detected": 0 + }, + "dont_understand": { + "confidence": 0.0, + "detected": 0 + }, + "exit": { + "confidence": 0.0, + "detected": 0 + }, + "lets_chat_about": { + "confidence": 0.0, + "detected": 0 + }, + "no": { + "confidence": 0.0, + "detected": 0 + }, + "opinion_request": { + "confidence": 0.0, + "detected": 0 + }, + "repeat": { + "confidence": 0.0, + "detected": 0 + }, + "stupid": { + "confidence": 0.0, + "detected": 0 + }, + "tell_me_a_story": { + "confidence": 0.0, + "detected": 0 + }, + "tell_me_more": { + "confidence": 0.0, + "detected": 0 + }, + "topic_switching": { + "confidence": 0.0, + "detected": 0 + }, + "weather_forecast_intent": { + "confidence": 0.0, + "detected": 0 + }, + "what_can_you_do": { + "confidence": 0.0, + "detected": 0 + }, + "what_is_your_job": { + "confidence": 0.0, + "detected": 0 + }, + "what_is_your_name": { + "confidence": 0.0, + "detected": 0 + }, + "what_time": { + "confidence": 0.0, + "detected": 0 + }, + "where_are_you_from": { + "confidence": 0.0, + "detected": 0 + }, + "who_made_you": { + "confidence": 0.0, + "detected": 0 + }, + "yes": { + "confidence": 0.0, + "detected": 0 + } + }, + "spacy_nounphrases": [], + "ner": [ + [ + { + "confidence": 1, + "end_pos": 1, + "start_pos": 0, + "text": "hey", + "type": "LOC" + } + ] + ], + "cobot_dialogact_intents": { + "text": [ + "General_ChatIntent" + ]}, + "cobot_dialogact_topics": { + "text": [ + "Phatic" + ] + }, + "cobot_offensiveness": { + "text": [ + "non-toxic" + ], + "confidence": [ + 1.0309148e-05 + ], + "is_badlisted": [ + "not badlist" + ] + }, + "cobot_topics": { + "text": [ + "Phatic" + ] + }, + "sentrewrite": { + "clusters": [], + "modified_sents": [ + "hey." + ] + } + }, + "hypotheses": [ + { + "skill_name": "dummy_skill", + "annotations": { + "toxic_classification": { + "identity_hate": 0.00024211406707763672, + "insult": 0.0004429817199707031, + "obscene": 0.0001055598258972168, + "severe_toxic": 5.415081977844238e-05, + "sexual_explicit": 9.942054748535156e-05, + "threat": 0.00014013051986694336, + "toxic": 0.0009675323963165283 + }, + "stop_detect": { + "stop": 0.6065886616706848, + "continue": 0.4208565950393677 + }, + "convers_evaluator_annotator": { + "isResponseComprehensible": 0.909, + "isResponseErroneous": 0.808, + "isResponseInteresting": 0.073, + "isResponseOnTopic": 0.158, + "responseEngagesUser": 0.469 + }, + "badlisted_words": { + "inappropriate": false, + "profanity": false, + "restricted_topics": false + } + }, + "text": "I didn't get it. Sorry.", + "confidence": 0.5, + "type": "dummy" + }, + { + "skill_name": "dummy_skill", + "annotations": { + "toxic_classification": { + "identity_hate": 0.00024211406707763672, + "insult": 0.0004429817199707031, + "obscene": 0.0001055598258972168, + "severe_toxic": 5.415081977844238e-05, + "sexual_explicit": 9.942054748535156e-05, + "threat": 0.00014013051986694336, + "toxic": 0.0009675323963165283 + }, + "stop_detect": { + "stop": 0.6065886616706848, + "continue": 0.4208565950393677 + }, + "convers_evaluator_annotator": { + "isResponseComprehensible": 0.909, + "isResponseErroneous": 0.808, + "isResponseInteresting": 0.073, + "isResponseOnTopic": 0.158, + "responseEngagesUser": 0.469 + }, + "badlisted_words": { + "inappropriate": false, + "profanity": false, + "restricted_topics": false + } + }, + "text": "are you a comedy fan?", + "confidence": 0.6, + "type": "topic_question" + }, + { + "skill_name": "movie_tfidf_retrieval", + "annotations": { + "toxic_classification": { + "identity_hate": 0.0001259446144104004, + "insult": 0.00027686357498168945, + "obscene": 5.97834587097168e-05, + "severe_toxic": 3.403425216674805e-05, + "sexual_explicit": 8.13603401184082e-05, + "threat": 0.00012931227684020996, + "toxic": 0.0005629658699035645 + }, + "stop_detect": { + "stop": 0.5833511352539062, + "continue": 0.46003755927085876 + }, + "convers_evaluator_annotator": { + "isResponseComprehensible": 0.281, + "isResponseErroneous": 0.531, + "isResponseInteresting": 0.228, + "isResponseOnTopic": 0.254, + "responseEngagesUser": 0.536 + }, + "badlisted_words": { + "inappropriate": false, + "profanity": false, + "restricted_topics": false + } + }, + "text": "i got you haha. what do you think abuot celebrities? judge judy makes 123, 000 per episode apparently!", + "confidence": 0.38232852805460565 + }, + { + "skill_name": "program_y", + "annotations": { + "toxic_classification": { + "identity_hate": 8.749961853027344e-05, + "insult": 0.00024232268333435059, + "obscene": 2.828240394592285e-05, + "severe_toxic": 1.8358230590820312e-05, + "sexual_explicit": 2.9712915420532227e-05, + "threat": 6.490945816040039e-05, + "toxic": 0.00043845176696777344 + }, + "stop_detect": { + "stop": 0.5808720588684082, + "continue": 0.45234695076942444 + }, + "convers_evaluator_annotator": { + "isResponseComprehensible": 0.984, + "isResponseErroneous": 0.614, + "isResponseInteresting": 0.253, + "isResponseOnTopic": 0.226, + "responseEngagesUser": 0.56 + }, + "badlisted_words": { + "inappropriate": false, + "profanity": false, + "restricted_topics": false + } + }, + "text": "Good Morning, this is an Alexa Prize Socialbot! How are you?", + "confidence": 0.98 + } + ], + "date_time": "2020-03-05 17:07:20.926781", + "attributes": {} + } + ], + "bot_utterances": [], + "human": { + "id": "5e61287da7d26764f5d4a0ed", + "user_telegram_id": "dsadas", + "persona": {}, + "profile": { + "name": null, + "gender": null, + "birthdate": null, + "location": null, + "home_coordinates": null, + "work_coordinates": null, + "occupation": null, + "income_per_year": null + }, + "attributes": {}, + "user_type": "human" + }, + "bot": { + "id": "d3479979f64a4f0eab7804fade8a9fc2", + "persona": {}, + "attributes": {}, + "user_type": "bot" + }, + "channel_type": "http_client", + "date_start": "None", + "date_finish": "None" + } + ] +} diff --git a/response_selectors/ranking_based_response_selector/test_data.json b/response_selectors/ranking_based_response_selector/test_data.json new file mode 100644 index 0000000000..c08ed424ff --- /dev/null +++ b/response_selectors/ranking_based_response_selector/test_data.json @@ -0,0 +1,681 @@ +{ + "dialogs": [ + { + "id": "374475c4139a489fa74981941c194cb0", + "utterances": [ + { + "text": "hey.", + "user": { + "id": "5e61287da7d26764f5d4a0ed", + "user_telegram_id": "dsadas", + "persona": {}, + "profile": { + "name": null, + "gender": null, + "birthdate": null, + "location": null, + "home_coordinates": null, + "work_coordinates": null, + "occupation": null, + "income_per_year": null + }, + "attributes": {}, + "user_type": "human" + }, + "annotations": { + "badlisted_words": { + "inappropriate": false, + "profanity": false, + "restricted_topics": false + }, + "asr": { + "asr_confidence": "undefined" + }, + "sentiment_classification": { + "text": [ + "positive", + 0.7487750053405762 + ] + }, + "emotion_classification": { + "text": { + "anger": 0.2401413470506668, + "fear": 0.2673250734806061, + "joy": 0.25821077823638916, + "disgust": 0.3577530086040497, + "sadness": 0.21969465911388397, + "surprise": 0.2795693576335907, + "neutral": 0.9991581439971924 + } + }, + "toxic_classification": { + "identity_hate": 0.0024989843368530273, + "insult": 0.0030942559242248535, + "obscene": 0.00198972225189209, + "severe_toxic": 0.0007079243659973145, + "sexual_explicit": 0.0022467062808573246, + "threat": 0.0014101153938099742, + "toxic": 0.007221863605082035 + }, + "sentseg": { + "punct_sent": "hey.", + "segments": [ + "hey." + ] + }, + "intent_catcher": { + "cant_do": { + "confidence": 0.0, + "detected": 0 + }, + "doing_well": { + "confidence": 0.0, + "detected": 0 + }, + "dont_understand": { + "confidence": 0.0, + "detected": 0 + }, + "exit": { + "confidence": 0.0, + "detected": 0 + }, + "lets_chat_about": { + "confidence": 0.0, + "detected": 0 + }, + "no": { + "confidence": 0.0, + "detected": 0 + }, + "opinion_request": { + "confidence": 0.0, + "detected": 0 + }, + "repeat": { + "confidence": 0.0, + "detected": 0 + }, + "stupid": { + "confidence": 0.0, + "detected": 0 + }, + "tell_me_a_story": { + "confidence": 0.0, + "detected": 0 + }, + "tell_me_more": { + "confidence": 0.0, + "detected": 0 + }, + "topic_switching": { + "confidence": 0.0, + "detected": 0 + }, + "weather_forecast_intent": { + "confidence": 0.0, + "detected": 0 + }, + "what_can_you_do": { + "confidence": 0.0, + "detected": 0 + }, + "what_is_your_job": { + "confidence": 0.0, + "detected": 0 + }, + "what_is_your_name": { + "confidence": 0.0, + "detected": 0 + }, + "what_time": { + "confidence": 0.0, + "detected": 0 + }, + "where_are_you_from": { + "confidence": 0.0, + "detected": 0 + }, + "who_made_you": { + "confidence": 0.0, + "detected": 0 + }, + "yes": { + "confidence": 0.0, + "detected": 0 + } + }, + "spacy_nounphrases": [], + "ner": [ + [ + { + "confidence": 1, + "end_pos": 1, + "start_pos": 0, + "text": "hey", + "type": "LOC" + } + ] + ], + "cobot_dialogact_intents": { + "text": [ + "General_ChatIntent" + ] + }, + "cobot_dialogact_topics": { + "text": [ + "Phatic" + ] + }, + "cobot_offensiveness": { + "text": [ + "non-toxic" + ], + "confidence": [ + 1.0309148e-05 + ], + "is_badlisted": [ + "not badlist" + ] + }, + "cobot_topics": { + "text": [ + "Phatic" + ] + }, + "sentrewrite": { + "clusters": [], + "modified_sents": [ + "hey." + ] + } + }, + "hypotheses": [ + { + "skill_name": "dummy_skill", + "annotations": { + "toxic_classification": { + "identity_hate": 0.00024211406707763672, + "insult": 0.0004429817199707031, + "obscene": 0.0001055598258972168, + "severe_toxic": 5.415081977844238e-05, + "sexual_explicit": 9.942054748535156e-05, + "threat": 0.00014013051986694336, + "toxic": 0.0009675323963165283 + }, + "stop_detect": { + "stop": 0.6065886616706848, + "continue": 0.4208565950393677 + }, + "convers_evaluator_annotator": { + "isResponseComprehensible": 0.909, + "isResponseErroneous": 0.808, + "isResponseInteresting": 0.073, + "isResponseOnTopic": 0.158, + "responseEngagesUser": 0.469 + }, + "badlisted_words": { + "inappropriate": false, + "profanity": false, + "restricted_topics": false + } + }, + "text": "I didn't get it. Sorry.", + "confidence": 0.5, + "type": "dummy" + }, + { + "skill_name": "dummy_skill", + "annotations": { + "toxic_classification": { + "identity_hate": 0.00024211406707763672, + "insult": 0.0004429817199707031, + "obscene": 0.0001055598258972168, + "severe_toxic": 5.415081977844238e-05, + "sexual_explicit": 9.942054748535156e-05, + "threat": 0.00014013051986694336, + "toxic": 0.0009675323963165283 + }, + "stop_detect": { + "stop": 0.6065886616706848, + "continue": 0.4208565950393677 + }, + "convers_evaluator_annotator": { + "isResponseComprehensible": 0.909, + "isResponseErroneous": 0.808, + "isResponseInteresting": 0.073, + "isResponseOnTopic": 0.158, + "responseEngagesUser": 0.469 + }, + "badlisted_words": { + "inappropriate": false, + "profanity": false, + "restricted_topics": false + } + }, + "text": "are you a comedy fan?", + "confidence": 0.6, + "type": "topic_question" + }, + { + "skill_name": "movie_tfidf_retrieval", + "annotations": { + "toxic_classification": { + "identity_hate": 0.0001259446144104004, + "insult": 0.00027686357498168945, + "obscene": 5.97834587097168e-05, + "severe_toxic": 3.403425216674805e-05, + "sexual_explicit": 8.13603401184082e-05, + "threat": 0.00012931227684020996, + "toxic": 0.0005629658699035645 + }, + "stop_detect": { + "stop": 0.5833511352539062, + "continue": 0.46003755927085876 + }, + "convers_evaluator_annotator": { + "isResponseComprehensible": 0.281, + "isResponseErroneous": 0.531, + "isResponseInteresting": 0.228, + "isResponseOnTopic": 0.254, + "responseEngagesUser": 0.536 + }, + "badlisted_words": { + "inappropriate": false, + "profanity": false, + "restricted_topics": false + } + }, + "text": "i got you haha. what do you think abuot celebrities? judge judy makes 123, 000 per episode apparently!", + "confidence": 0.38232852805460565 + }, + { + "skill_name": "program_y", + "annotations": { + "toxic_classification": { + "identity_hate": 8.749961853027344e-05, + "insult": 0.00024232268333435059, + "obscene": 2.828240394592285e-05, + "severe_toxic": 1.8358230590820312e-05, + "sexual_explicit": 2.9712915420532227e-05, + "threat": 6.490945816040039e-05, + "toxic": 0.00043845176696777344 + }, + "stop_detect": { + "stop": 0.5808720588684082, + "continue": 0.45234695076942444 + }, + "convers_evaluator_annotator": { + "isResponseComprehensible": 0.984, + "isResponseErroneous": 0.614, + "isResponseInteresting": 0.253, + "isResponseOnTopic": 0.226, + "responseEngagesUser": 0.56 + }, + "badlisted_words": { + "inappropriate": false, + "profanity": false, + "restricted_topics": false + } + }, + "text": "Good Morning, this is an Alexa Prize Socialbot! How are you?", + "confidence": 0.98 + } + ], + "date_time": "2020-03-05 17:07:20.926781", + "attributes": {} + } + ], + "human_utterances": [ + { + "text": "hey.", + "user": { + "id": "5e61287da7d26764f5d4a0ed", + "user_telegram_id": "dsadas", + "persona": {}, + "profile": { + "name": null, + "gender": null, + "birthdate": null, + "location": null, + "home_coordinates": null, + "work_coordinates": null, + "occupation": null, + "income_per_year": null + }, + "attributes": {}, + "user_type": "human" + }, + "annotations": { + "badlisted_words": { + "inappropriate": false, + "profanity": false, + "restricted_topics": false + }, + "asr": { + "asr_confidence": "undefined" + }, + "sentiment_classification": { + "text": [ + "positive", + 0.7487750053405762 + ] + }, + "emotion_classification": { + "text": { + "anger": 0.2401413470506668, + "fear": 0.2673250734806061, + "joy": 0.25821077823638916, + "disgust": 0.3577530086040497, + "sadness": 0.21969465911388397, + "surprise": 0.2795693576335907, + "neutral": 0.9991581439971924 + } + }, + "toxic_classification": { + "identity_hate": 0.0024989843368530273, + "insult": 0.0030942559242248535, + "obscene": 0.00198972225189209, + "severe_toxic": 0.0007079243659973145, + "sexual_explicit": 0.0022467062808573246, + "threat": 0.0014101153938099742, + "toxic": 0.007221863605082035 + }, + "sentseg": { + "punct_sent": "hey.", + "segments": [ + "hey." + ] + }, + "intent_catcher": { + "cant_do": { + "confidence": 0.0, + "detected": 0 + }, + "doing_well": { + "confidence": 0.0, + "detected": 0 + }, + "dont_understand": { + "confidence": 0.0, + "detected": 0 + }, + "exit": { + "confidence": 0.0, + "detected": 0 + }, + "lets_chat_about": { + "confidence": 0.0, + "detected": 0 + }, + "no": { + "confidence": 0.0, + "detected": 0 + }, + "opinion_request": { + "confidence": 0.0, + "detected": 0 + }, + "repeat": { + "confidence": 0.0, + "detected": 0 + }, + "stupid": { + "confidence": 0.0, + "detected": 0 + }, + "tell_me_a_story": { + "confidence": 0.0, + "detected": 0 + }, + "tell_me_more": { + "confidence": 0.0, + "detected": 0 + }, + "topic_switching": { + "confidence": 0.0, + "detected": 0 + }, + "weather_forecast_intent": { + "confidence": 0.0, + "detected": 0 + }, + "what_can_you_do": { + "confidence": 0.0, + "detected": 0 + }, + "what_is_your_job": { + "confidence": 0.0, + "detected": 0 + }, + "what_is_your_name": { + "confidence": 0.0, + "detected": 0 + }, + "what_time": { + "confidence": 0.0, + "detected": 0 + }, + "where_are_you_from": { + "confidence": 0.0, + "detected": 0 + }, + "who_made_you": { + "confidence": 0.0, + "detected": 0 + }, + "yes": { + "confidence": 0.0, + "detected": 0 + } + }, + "spacy_nounphrases": [], + "ner": [ + [ + { + "confidence": 1, + "end_pos": 1, + "start_pos": 0, + "text": "hey", + "type": "LOC" + } + ] + ], + "cobot_dialogact_intents": { + "text": [ + "General_ChatIntent" + ]}, + "cobot_dialogact_topics": { + "text": [ + "Phatic" + ] + }, + "cobot_offensiveness": { + "text": [ + "non-toxic" + ], + "confidence": [ + 1.0309148e-05 + ], + "is_badlisted": [ + "not badlist" + ] + }, + "cobot_topics": { + "text": [ + "Phatic" + ] + }, + "sentrewrite": { + "clusters": [], + "modified_sents": [ + "hey." + ] + } + }, + "hypotheses": [ + { + "skill_name": "dummy_skill", + "annotations": { + "toxic_classification": { + "identity_hate": 0.00024211406707763672, + "insult": 0.0004429817199707031, + "obscene": 0.0001055598258972168, + "severe_toxic": 5.415081977844238e-05, + "sexual_explicit": 9.942054748535156e-05, + "threat": 0.00014013051986694336, + "toxic": 0.0009675323963165283 + }, + "stop_detect": { + "stop": 0.6065886616706848, + "continue": 0.4208565950393677 + }, + "convers_evaluator_annotator": { + "isResponseComprehensible": 0.909, + "isResponseErroneous": 0.808, + "isResponseInteresting": 0.073, + "isResponseOnTopic": 0.158, + "responseEngagesUser": 0.469 + }, + "badlisted_words": { + "inappropriate": false, + "profanity": false, + "restricted_topics": false + } + }, + "text": "I didn't get it. Sorry.", + "confidence": 0.5, + "type": "dummy" + }, + { + "skill_name": "dummy_skill", + "annotations": { + "toxic_classification": { + "identity_hate": 0.00024211406707763672, + "insult": 0.0004429817199707031, + "obscene": 0.0001055598258972168, + "severe_toxic": 5.415081977844238e-05, + "sexual_explicit": 9.942054748535156e-05, + "threat": 0.00014013051986694336, + "toxic": 0.0009675323963165283 + }, + "stop_detect": { + "stop": 0.6065886616706848, + "continue": 0.4208565950393677 + }, + "convers_evaluator_annotator": { + "isResponseComprehensible": 0.909, + "isResponseErroneous": 0.808, + "isResponseInteresting": 0.073, + "isResponseOnTopic": 0.158, + "responseEngagesUser": 0.469 + }, + "badlisted_words": { + "inappropriate": false, + "profanity": false, + "restricted_topics": false + } + }, + "text": "are you a comedy fan?", + "confidence": 0.6, + "type": "topic_question" + }, + { + "skill_name": "movie_tfidf_retrieval", + "annotations": { + "toxic_classification": { + "identity_hate": 0.0001259446144104004, + "insult": 0.00027686357498168945, + "obscene": 5.97834587097168e-05, + "severe_toxic": 3.403425216674805e-05, + "sexual_explicit": 8.13603401184082e-05, + "threat": 0.00012931227684020996, + "toxic": 0.0005629658699035645 + }, + "stop_detect": { + "stop": 0.5833511352539062, + "continue": 0.46003755927085876 + }, + "convers_evaluator_annotator": { + "isResponseComprehensible": 0.281, + "isResponseErroneous": 0.531, + "isResponseInteresting": 0.228, + "isResponseOnTopic": 0.254, + "responseEngagesUser": 0.536 + }, + "badlisted_words": { + "inappropriate": false, + "profanity": false, + "restricted_topics": false + } + }, + "text": "i got you haha. what do you think abuot celebrities? judge judy makes 123, 000 per episode apparently!", + "confidence": 0.38232852805460565 + }, + { + "skill_name": "program_y", + "annotations": { + "toxic_classification": { + "identity_hate": 8.749961853027344e-05, + "insult": 0.00024232268333435059, + "obscene": 2.828240394592285e-05, + "severe_toxic": 1.8358230590820312e-05, + "sexual_explicit": 2.9712915420532227e-05, + "threat": 6.490945816040039e-05, + "toxic": 0.00043845176696777344 + }, + "stop_detect": { + "stop": 0.5808720588684082, + "continue": 0.45234695076942444 + }, + "convers_evaluator_annotator": { + "isResponseComprehensible": 0.984, + "isResponseErroneous": 0.614, + "isResponseInteresting": 0.253, + "isResponseOnTopic": 0.226, + "responseEngagesUser": 0.56 + }, + "badlisted_words": { + "inappropriate": false, + "profanity": false, + "restricted_topics": false + } + }, + "text": "Good Morning, this is an Alexa Prize Socialbot! How are you?", + "confidence": 0.98 + } + ], + "date_time": "2020-03-05 17:07:20.926781", + "attributes": {} + } + ], + "bot_utterances": [], + "human": { + "id": "5e61287da7d26764f5d4a0ed", + "user_telegram_id": "dsadas", + "persona": {}, + "profile": { + "name": null, + "gender": null, + "birthdate": null, + "location": null, + "home_coordinates": null, + "work_coordinates": null, + "occupation": null, + "income_per_year": null + }, + "attributes": {}, + "user_type": "human" + }, + "bot": { + "id": "d3479979f64a4f0eab7804fade8a9fc2", + "persona": {}, + "attributes": {}, + "user_type": "bot" + }, + "channel_type": "http_client", + "date_start": "None", + "date_finish": "None" + } + ] +} From 6fd0fdeb8e3c63dc23e7c4299a804fb7cb5f6c86 Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Mon, 24 Apr 2023 16:18:36 +0300 Subject: [PATCH 10/32] feat: component description --- .../llm_based_response_selector/component.yml | 8 +-- .../component.yml | 59 +++++++++++++++++-- .../pipeline.yml | 24 +++++++- 3 files changed, 80 insertions(+), 11 deletions(-) diff --git a/response_selectors/llm_based_response_selector/component.yml b/response_selectors/llm_based_response_selector/component.yml index 0bef0c738b..9e6cd3902c 100644 --- a/response_selectors/llm_based_response_selector/component.yml +++ b/response_selectors/llm_based_response_selector/component.yml @@ -18,7 +18,7 @@ davinci3-based-response-selector: - group: response_selectors endpoint: respond build_args: - SERVICE_PORT: '8003' + SERVICE_PORT: 8003 SERVICE_NAME: response_selector LANGUAGE: EN GENERATIVE_SERVICE_URL: http://openai-api-davinci3:8131/respond @@ -48,7 +48,7 @@ chatgpt-based-response-selector: - group: response_selectors endpoint: respond build_args: - SERVICE_PORT: '8003' + SERVICE_PORT: 8003 SERVICE_NAME: response_selector LANGUAGE: EN GENERATIVE_SERVICE_URL: http://openai-api-chatgpt:8145/respond @@ -78,7 +78,7 @@ oasst12b-based-response-selector: - group: response_selectors endpoint: respond build_args: - SERVICE_PORT: '8003' + SERVICE_PORT: 8003 SERVICE_NAME: response_selector LANGUAGE: EN GENERATIVE_SERVICE_URL: http://transformers-lm-oasst12b:8158/respond @@ -107,7 +107,7 @@ gptj-based-response-selector: - group: response_selectors endpoint: respond build_args: - SERVICE_PORT: '8003' + SERVICE_PORT: 8003 SERVICE_NAME: response_selector LANGUAGE: EN GENERATIVE_SERVICE_URL: http://transformers-lm-gptj:8130/respond diff --git a/response_selectors/ranking_based_response_selector/component.yml b/response_selectors/ranking_based_response_selector/component.yml index 2df1ba7549..d56c73d79a 100644 --- a/response_selectors/ranking_based_response_selector/component.yml +++ b/response_selectors/ranking_based_response_selector/component.yml @@ -1,7 +1,7 @@ -ranking-based-response-selector: +ranking-based-response-selector-en-service: name: response_selector display_name: Response Selector - container_name: ranking-based-response-selector + container_name: ranking-based-response-selector-en-service component_type: null model_type: Dictionary/Pattern-based is_customizable: false @@ -15,12 +15,61 @@ ranking-based-response-selector: - group: response_selectors endpoint: respond build_args: - SERVICE_PORT: '8002' + SERVICE_PORT: 8002 SERVICE_NAME: response_selector LANGUAGE: EN - SENTENCE_RANKER_ANNOTATION_NAME: - SENTENCE_RANKER_SERVICE_URL: + SENTENCE_RANKER_ANNOTATION_NAME: null + SENTENCE_RANKER_SERVICE_URL: http://sentence-ranker:8128/respond SENTENCE_RANKER_TIMEOUT: 3 N_UTTERANCES_CONTEXT: 5 FILTER_TOXIC_OR_BADLISTED: 1 date_created: '2023-04-24T09:45:32' +ranking-based-response-selector-ru-service: + name: response_selector + display_name: Response Selector + container_name: ranking-based-response-selector-ru-service + component_type: null + model_type: Dictionary/Pattern-based + is_customizable: false + author: DeepPavlov + description: Algorithm that selects a final responses among the given list of candidate + responses via the given ranking service + ram_usage: 100M + gpu_usage: null + port: 8002 + endpoints: + - group: response_selectors + endpoint: respond + build_args: + SERVICE_PORT: 8002 + SERVICE_NAME: response_selector + LANGUAGE: RU + SENTENCE_RANKER_ANNOTATION_NAME: null + SENTENCE_RANKER_SERVICE_URL: http://dialogrpt-ru:8122/rank_sentences + SENTENCE_RANKER_TIMEOUT: 3 + N_UTTERANCES_CONTEXT: 5 + FILTER_TOXIC_OR_BADLISTED: 1 + date_created: '2023-04-24T09:45:32' +ranking-based-response-selector-ru: + name: response_selector + display_name: Response Selector + container_name: ranking-based-response-selector-ru + component_type: null + model_type: Dictionary/Pattern-based + is_customizable: false + author: DeepPavlov + description: Algorithm that selects a final responses among the given list of candidate + responses via the given ranking service + ram_usage: 100M + gpu_usage: null + port: 8002 + endpoints: + - group: response_selectors + endpoint: respond + build_args: + SERVICE_PORT: 8002 + SERVICE_NAME: response_selector + LANGUAGE: RU + SENTENCE_RANKER_ANNOTATION_NAME: dialogrpt + FILTER_TOXIC_OR_BADLISTED: 1 + date_created: '2023-04-24T09:45:32' \ No newline at end of file diff --git a/response_selectors/ranking_based_response_selector/pipeline.yml b/response_selectors/ranking_based_response_selector/pipeline.yml index 8028c2575a..c08e53f692 100644 --- a/response_selectors/ranking_based_response_selector/pipeline.yml +++ b/response_selectors/ranking_based_response_selector/pipeline.yml @@ -1,8 +1,28 @@ -ranking-based-response-selector: +ranking-based-response-selector-en-service: - group: response_selectors connector: protocol: http - url: http://ranking-based-response-selector:8002/respond + url: http://ranking-based-response-selector-en-service:8002/respond + dialog_formatter: state_formatters.dp_formatters:cropped_dialog + response_formatter: state_formatters.dp_formatters:base_response_selector_formatter_service + previous_services: + - response_annotators + state_manager_method: add_bot_utterance +ranking-based-response-selector-ru-service: +- group: response_selectors + connector: + protocol: http + url: http://ranking-based-response-selector-ru-service:8002/respond + dialog_formatter: state_formatters.dp_formatters:cropped_dialog + response_formatter: state_formatters.dp_formatters:base_response_selector_formatter_service + previous_services: + - response_annotators + state_manager_method: add_bot_utterance +ranking-based-response-selector-ru: +- group: response_selectors + connector: + protocol: http + url: http://ranking-based-response-selector-ru:8002/respond dialog_formatter: state_formatters.dp_formatters:cropped_dialog response_formatter: state_formatters.dp_formatters:base_response_selector_formatter_service previous_services: From f2fdec58d845921b4be468b4eb4b55b74a70950c Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Mon, 24 Apr 2023 16:30:16 +0300 Subject: [PATCH 11/32] feat: use gpt-j resp selector --- .../dream_persona_prompted/dev.yml | 6 ++-- .../docker-compose.override.yml | 33 +++++++------------ .../dream_persona_prompted/pipeline_conf.json | 8 ++--- .../empathetic_marketing_assistant/dev.yml | 6 ++-- .../docker-compose.override.yml | 33 +++++++------------ .../pipeline_conf.json | 8 ++--- 6 files changed, 36 insertions(+), 58 deletions(-) diff --git a/assistant_dists/dream_persona_prompted/dev.yml b/assistant_dists/dream_persona_prompted/dev.yml index 21332c6442..6d33070357 100644 --- a/assistant_dists/dream_persona_prompted/dev.yml +++ b/assistant_dists/dream_persona_prompted/dev.yml @@ -10,12 +10,12 @@ services: - "./annotators/SentSeg:/src" ports: - 8011:8011 - convers-evaluation-no-scripts-selector: + gptj-based-response-selector: volumes: - - "./response_selectors/convers_evaluation_based_selector:/src" + - "./response_selectors/llm_based_response_selector:/src" - "./common:/src/common" ports: - - 8009:8009 + - 8003:8003 badlisted-words: volumes: - "./annotators/BadlistedWordsDetector:/src" diff --git a/assistant_dists/dream_persona_prompted/docker-compose.override.yml b/assistant_dists/dream_persona_prompted/docker-compose.override.yml index cf99ce4c85..b67fe10904 100644 --- a/assistant_dists/dream_persona_prompted/docker-compose.override.yml +++ b/assistant_dists/dream_persona_prompted/docker-compose.override.yml @@ -2,7 +2,7 @@ services: agent: command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/dream_persona_prompted/pipeline_conf.json' environment: - WAIT_HOSTS: "sentseg:8011, convers-evaluation-no-scripts-selector:8009, badlisted-words:8018, combined-classification:8087, + WAIT_HOSTS: "sentseg:8011, gptj-based-response-selector:8003, badlisted-words:8018, combined-classification:8087, sentence-ranker:8128, prompt-selector:8135, transformers-lm-gptj:8130, dff-dream-persona-gpt-j-prompted-skill:8134" WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1000} @@ -38,32 +38,21 @@ services: reservations: memory: 2G - convers-evaluation-no-scripts-selector: + gptj-based-response-selector: env_file: [ .env ] build: args: - TAG_BASED_SELECTION: 1 - CALL_BY_NAME_PROBABILITY: 0.5 - PROMPT_PROBA: 0.1 - ACKNOWLEDGEMENT_PROBA: 0.3 - PRIORITIZE_WITH_REQUIRED_ACT: 0 - PRIORITIZE_NO_DIALOG_BREAKDOWN: 0 - PRIORITIZE_WITH_SAME_TOPIC_ENTITY: 0 - IGNORE_DISLIKED_SKILLS: 0 - GREETING_FIRST: 0 - RESTRICTION_FOR_SENSITIVE_CASE: 1 - PRIORITIZE_PROMTS_WHEN_NO_SCRIPTS: 0 - MAX_TURNS_WITHOUT_SCRIPTS: 7 - ADD_ACKNOWLEDGMENTS_IF_POSSIBLE: 1 - PRIORITIZE_SCRIPTED_SKILLS: 0 - CONFIDENCE_STRENGTH: 0.8 - CONV_EVAL_STRENGTH: 0.4 - PRIORITIZE_HUMAN_INITIATIVE: 1 - QUESTION_TO_QUESTION_DOWNSCORE_COEF: 0.8 + SERVICE_PORT: 8003 + SERVICE_NAME: response_selector LANGUAGE: EN + GENERATIVE_SERVICE_URL: http://transformers-lm-gptj:8130/respond + GENERATIVE_SERVICE_CONFIG: default_generative_config.json + GENERATIVE_TIMEOUT: 5 + N_UTTERANCES_CONTEXT: 5 + FILTER_TOXIC_OR_BADLISTED: 1 context: . - dockerfile: ./response_selectors/convers_evaluation_based_selector/Dockerfile - command: flask run -h 0.0.0.0 -p 8009 + dockerfile: ./response_selectors/llm_based_response_selector/Dockerfile + command: flask run -h 0.0.0.0 -p 8003 environment: - FLASK_APP=server deploy: diff --git a/assistant_dists/dream_persona_prompted/pipeline_conf.json b/assistant_dists/dream_persona_prompted/pipeline_conf.json index e351a38b49..324861ab00 100644 --- a/assistant_dists/dream_persona_prompted/pipeline_conf.json +++ b/assistant_dists/dream_persona_prompted/pipeline_conf.json @@ -275,9 +275,9 @@ "connector": { "protocol": "http", "timeout": 1.0, - "url": "http://convers-evaluation-no-scripts-selector:8009/respond" + "url": "http://gptj-based-response-selector:8003/respond" }, - "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", + "dialog_formatter": "state_formatters.dp_formatters:cropped_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", "previous_services": [ "candidate_annotators" @@ -285,8 +285,8 @@ "state_manager_method": "add_bot_utterance", "is_enabled": true, "source": { - "directory": "response_selectors/convers_evaluation_based_selector", - "container": "convers-evaluation-no-scripts-selector", + "directory": "response_selectors/llm_based_response_selector", + "container": "gptj-based-response-selector", "endpoint": "respond" } } diff --git a/assistant_dists/empathetic_marketing_assistant/dev.yml b/assistant_dists/empathetic_marketing_assistant/dev.yml index f28059d91e..06835391aa 100644 --- a/assistant_dists/empathetic_marketing_assistant/dev.yml +++ b/assistant_dists/empathetic_marketing_assistant/dev.yml @@ -10,12 +10,12 @@ services: - "./annotators/SentSeg:/src" ports: - 8011:8011 - no-restrictions-selector: + gptj-based-response-selector: volumes: - - "./response_selectors/convers_evaluation_based_selector:/src" + - "./response_selectors/llm_based_response_selector:/src" - "./common:/src/common" ports: - - 8009:8009 + - 8003:8003 badlisted-words: volumes: - "./annotators/BadlistedWordsDetector:/src" diff --git a/assistant_dists/empathetic_marketing_assistant/docker-compose.override.yml b/assistant_dists/empathetic_marketing_assistant/docker-compose.override.yml index 244db666aa..75de208ef4 100644 --- a/assistant_dists/empathetic_marketing_assistant/docker-compose.override.yml +++ b/assistant_dists/empathetic_marketing_assistant/docker-compose.override.yml @@ -2,7 +2,7 @@ services: agent: command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/empathetic_marketing_assistant/pipeline_conf.json' environment: - WAIT_HOSTS: "sentseg:8011, no-restrictions-selector:8009, badlisted-words:8018, combined-classification:8087, + WAIT_HOSTS: "sentseg:8011, gptj-based-response-selector:8003, badlisted-words:8018, combined-classification:8087, sentence-ranker:8128, prompt-selector:8135, transformers-lm-gptj:8130, dff-empathetic-marketing-prompted-skill:8138" WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1000} @@ -38,32 +38,21 @@ services: reservations: memory: 2G - no-restrictions-selector: + gptj-based-response-selector: env_file: [ .env ] build: args: - TAG_BASED_SELECTION: 1 - CALL_BY_NAME_PROBABILITY: 0.5 - PROMPT_PROBA: 0 - ACKNOWLEDGEMENT_PROBA: 0.3 - PRIORITIZE_WITH_REQUIRED_ACT: 0 - PRIORITIZE_NO_DIALOG_BREAKDOWN: 0 - PRIORITIZE_WITH_SAME_TOPIC_ENTITY: 0 - IGNORE_DISLIKED_SKILLS: 0 - GREETING_FIRST: 0 - RESTRICTION_FOR_SENSITIVE_CASE: 0 - PRIORITIZE_PROMTS_WHEN_NO_SCRIPTS: 0 - MAX_TURNS_WITHOUT_SCRIPTS: 100 - ADD_ACKNOWLEDGMENTS_IF_POSSIBLE: 0 - PRIORITIZE_SCRIPTED_SKILLS: 0 - CONFIDENCE_STRENGTH: 0.8 - CONV_EVAL_STRENGTH: 0.4 - PRIORITIZE_HUMAN_INITIATIVE: 1 - QUESTION_TO_QUESTION_DOWNSCORE_COEF: 0.8 + SERVICE_PORT: 8003 + SERVICE_NAME: response_selector LANGUAGE: EN + GENERATIVE_SERVICE_URL: http://transformers-lm-gptj:8130/respond + GENERATIVE_SERVICE_CONFIG: default_generative_config.json + GENERATIVE_TIMEOUT: 5 + N_UTTERANCES_CONTEXT: 5 + FILTER_TOXIC_OR_BADLISTED: 1 context: . - dockerfile: ./response_selectors/convers_evaluation_based_selector/Dockerfile - command: flask run -h 0.0.0.0 -p 8009 + dockerfile: ./response_selectors/llm_based_response_selector/Dockerfile + command: flask run -h 0.0.0.0 -p 8003 environment: - FLASK_APP=server deploy: diff --git a/assistant_dists/empathetic_marketing_assistant/pipeline_conf.json b/assistant_dists/empathetic_marketing_assistant/pipeline_conf.json index 7801c0acce..7663e8dab4 100644 --- a/assistant_dists/empathetic_marketing_assistant/pipeline_conf.json +++ b/assistant_dists/empathetic_marketing_assistant/pipeline_conf.json @@ -273,9 +273,9 @@ "connector": { "protocol": "http", "timeout": 1.0, - "url": "http://no-restrictions-selector:8009/respond" + "url": "http://gptj-based-response-selector:8003/respond" }, - "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", + "dialog_formatter": "state_formatters.dp_formatters:cropped_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", "previous_services": [ "candidate_annotators" @@ -283,8 +283,8 @@ "state_manager_method": "add_bot_utterance", "is_enabled": true, "source": { - "directory": "response_selectors/convers_evaluation_based_selector", - "container": "no-restrictions-selector", + "directory": "response_selectors/llm_based_response_selector", + "container": "gptj-based-response-selector", "endpoint": "respond" } } From a18cee9a0451b5261bbe294c6c31b47de8003a10 Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Mon, 24 Apr 2023 16:41:20 +0300 Subject: [PATCH 12/32] feat: context --- response_selectors/llm_based_response_selector/Dockerfile | 4 ++-- response_selectors/ranking_based_response_selector/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/response_selectors/llm_based_response_selector/Dockerfile b/response_selectors/llm_based_response_selector/Dockerfile index e27fc9d68e..8b0a599117 100644 --- a/response_selectors/llm_based_response_selector/Dockerfile +++ b/response_selectors/llm_based_response_selector/Dockerfile @@ -2,7 +2,7 @@ FROM python:3.7.4 RUN mkdir /src -COPY ./requirements.txt /src/requirements.txt +COPY ./response_selectors/llm_based_response_selector/requirements.txt /src/requirements.txt RUN pip install -r /src/requirements.txt ARG LANGUAGE=EN @@ -22,7 +22,7 @@ ENV N_UTTERANCES_CONTEXT ${N_UTTERANCES_CONTEXT} ARG FILTER_TOXIC_OR_BADLISTED=1 ENV FILTER_TOXIC_OR_BADLISTED ${FILTER_TOXIC_OR_BADLISTED} -COPY . /src/ +COPY ./response_selectors/llm_based_response_selector/ /src/ WORKDIR /src CMD gunicorn --workers=1 server:app -b 0.0.0.0:${SERVICE_PORT} --timeout=1200 diff --git a/response_selectors/ranking_based_response_selector/Dockerfile b/response_selectors/ranking_based_response_selector/Dockerfile index 3788831fd0..4e611f1f26 100644 --- a/response_selectors/ranking_based_response_selector/Dockerfile +++ b/response_selectors/ranking_based_response_selector/Dockerfile @@ -2,7 +2,7 @@ FROM python:3.7.4 RUN mkdir /src -COPY ./requirements.txt /src/requirements.txt +COPY ./response_selectors/ranking_based_response_selector/requirements.txt /src/requirements.txt RUN pip install -r /src/requirements.txt ARG LANGUAGE=EN @@ -20,7 +20,7 @@ ENV N_UTTERANCES_CONTEXT ${N_UTTERANCES_CONTEXT} ARG FILTER_TOXIC_OR_BADLISTED=1 ENV FILTER_TOXIC_OR_BADLISTED ${FILTER_TOXIC_OR_BADLISTED} -COPY . /src/ +COPY ./response_selectors/ranking_based_response_selector/ /src/ WORKDIR /src CMD gunicorn --workers=1 server:app -b 0.0.0.0:${SERVICE_PORT} --timeout=1200 From 6e283b81ad2d2206ec802636479fa8c02178ca38 Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Mon, 24 Apr 2023 16:59:07 +0300 Subject: [PATCH 13/32] feat: utilize hypotheses correctly --- .../llm_based_response_selector/server.py | 20 +++++++++---------- .../ranking_based_response_selector/server.py | 2 +- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/response_selectors/llm_based_response_selector/server.py b/response_selectors/llm_based_response_selector/server.py index 67f8714efd..61303af21d 100644 --- a/response_selectors/llm_based_response_selector/server.py +++ b/response_selectors/llm_based_response_selector/server.py @@ -53,7 +53,7 @@ def select_response_by_scores(hypotheses, scores): return result, best_id -def select_response(dialog_context, hypotheses, confidences): +def select_response(dialog_context, hypotheses): try: response = requests.post( GENERATIVE_SERVICE_URL, @@ -70,7 +70,7 @@ def select_response(dialog_context, hypotheses, confidences): except Exception as e: sentry_sdk.capture_exception(e) logger.exception(e) - result = select_response_by_scores(hypotheses, confidences)[0] + result = select_response_by_scores(hypotheses, [hyp["confidence"] for hyp in hypotheses])[0] logger.info(f"Exception in LLM's invocation. Selected a response with the highest confidence.") logger.info(f"llm_based_response_selector selected:\n`{result}`") @@ -88,28 +88,26 @@ def respond(): selected_confidences = [] for i, dialog in enumerate(dialogs): - hypotheses = [hyp["text"] for hyp in dialog["human_utterances"][-1]["hypotheses"]] + hypotheses = [hyp for hyp in dialog["human_utterances"][-1]["hypotheses"]] if FILTER_TOXIC_OR_BADLISTED: hypotheses = filter_out_badlisted_or_toxic(hypotheses) - confidences = [hyp["confidence"] for hyp in hypotheses] - skill_names = [hyp["skill_name"] for hyp in hypotheses] dialog_context = [uttr["text"] for uttr in dialog["utterances"][-N_UTTERANCES_CONTEXT:]] - selected_resp = select_response(dialog_context, hypotheses, confidences) + selected_resp = select_response(dialog_context, hypotheses) try: best_id = hypotheses.index(selected_resp) - selected_skill_names.append(skill_names[best_id]) selected_responses.append(selected_resp) - selected_confidences.append(confidences[best_id]) + selected_skill_names.append(hypotheses[best_id]["skill_name"]) + selected_confidences.append(hypotheses[best_id]["confidence"]) except Exception as e: sentry_sdk.capture_exception(e) logger.exception(e) logger.info("Exception in finding selected response in hypotheses. " "Selected a response with the highest confidence.") - selected_resp, best_id = select_response_by_scores(hypotheses, confidences) - selected_skill_names.append(skill_names[best_id]) + selected_resp, best_id = select_response_by_scores(hypotheses, [hyp["confidence"] for hyp in hypotheses]) selected_responses.append(selected_resp) - selected_confidences.append(confidences[best_id]) + selected_skill_names.append(hypotheses[best_id]["skill_name"]) + selected_confidences.append(hypotheses[best_id]["confidence"]) total_time = time.time() - st_time logger.info(f"llm_based_response_selector exec time = {total_time:.3f}s") diff --git a/response_selectors/ranking_based_response_selector/server.py b/response_selectors/ranking_based_response_selector/server.py index 5dddcf0b23..91bb459d14 100644 --- a/response_selectors/ranking_based_response_selector/server.py +++ b/response_selectors/ranking_based_response_selector/server.py @@ -87,7 +87,7 @@ def respond(): selected_confidences = [] for i, dialog in enumerate(dialogs): - hypotheses = [hyp["text"] for hyp in dialog["human_utterances"][-1]["hypotheses"]] + hypotheses = [hyp for hyp in dialog["human_utterances"][-1]["hypotheses"]] if FILTER_TOXIC_OR_BADLISTED: hypotheses = filter_out_badlisted_or_toxic(hypotheses) From 9433c242f05c03ecece9ff54cc073864903f85a8 Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Mon, 24 Apr 2023 17:21:17 +0300 Subject: [PATCH 14/32] fix: configs --- .../generative_configs/default_generative_config.json | 8 ++++++++ .../generative_configs/openai-chatgpt.json | 7 +++++++ .../generative_configs/openai-text-davinci-003-long.json | 7 +++++++ .../generative_configs/openai-text-davinci-003.json | 7 +++++++ response_selectors/llm_based_response_selector/server.py | 4 ++++ 5 files changed, 33 insertions(+) create mode 100644 response_selectors/llm_based_response_selector/generative_configs/default_generative_config.json create mode 100644 response_selectors/llm_based_response_selector/generative_configs/openai-chatgpt.json create mode 100644 response_selectors/llm_based_response_selector/generative_configs/openai-text-davinci-003-long.json create mode 100644 response_selectors/llm_based_response_selector/generative_configs/openai-text-davinci-003.json diff --git a/response_selectors/llm_based_response_selector/generative_configs/default_generative_config.json b/response_selectors/llm_based_response_selector/generative_configs/default_generative_config.json new file mode 100644 index 0000000000..1edf84f186 --- /dev/null +++ b/response_selectors/llm_based_response_selector/generative_configs/default_generative_config.json @@ -0,0 +1,8 @@ +{ + "max_length": 60, + "min_length": 8, + "top_p": 0.9, + "temperature": 0.9, + "do_sample": true, + "num_return_sequences": 2 +} \ No newline at end of file diff --git a/response_selectors/llm_based_response_selector/generative_configs/openai-chatgpt.json b/response_selectors/llm_based_response_selector/generative_configs/openai-chatgpt.json new file mode 100644 index 0000000000..107e944b80 --- /dev/null +++ b/response_selectors/llm_based_response_selector/generative_configs/openai-chatgpt.json @@ -0,0 +1,7 @@ +{ + "max_tokens": 64, + "temperature": 0.4, + "top_p": 1.0, + "frequency_penalty": 0, + "presence_penalty": 0 +} \ No newline at end of file diff --git a/response_selectors/llm_based_response_selector/generative_configs/openai-text-davinci-003-long.json b/response_selectors/llm_based_response_selector/generative_configs/openai-text-davinci-003-long.json new file mode 100644 index 0000000000..a109e4db88 --- /dev/null +++ b/response_selectors/llm_based_response_selector/generative_configs/openai-text-davinci-003-long.json @@ -0,0 +1,7 @@ +{ + "max_tokens": 128, + "temperature": 0.4, + "top_p": 1.0, + "frequency_penalty": 0, + "presence_penalty": 0 +} \ No newline at end of file diff --git a/response_selectors/llm_based_response_selector/generative_configs/openai-text-davinci-003.json b/response_selectors/llm_based_response_selector/generative_configs/openai-text-davinci-003.json new file mode 100644 index 0000000000..107e944b80 --- /dev/null +++ b/response_selectors/llm_based_response_selector/generative_configs/openai-text-davinci-003.json @@ -0,0 +1,7 @@ +{ + "max_tokens": 64, + "temperature": 0.4, + "top_p": 1.0, + "frequency_penalty": 0, + "presence_penalty": 0 +} \ No newline at end of file diff --git a/response_selectors/llm_based_response_selector/server.py b/response_selectors/llm_based_response_selector/server.py index 61303af21d..4ba7b69a04 100644 --- a/response_selectors/llm_based_response_selector/server.py +++ b/response_selectors/llm_based_response_selector/server.py @@ -1,5 +1,6 @@ #!/usr/bin/env python +import json import logging import numpy as np import requests @@ -23,6 +24,9 @@ GENERATIVE_SERVICE_URL = getenv("GENERATIVE_SERVICE_URL") GENERATIVE_TIMEOUT = int(getenv("GENERATIVE_TIMEOUT")) GENERATIVE_SERVICE_CONFIG = getenv("GENERATIVE_SERVICE_CONFIG") +if GENERATIVE_SERVICE_CONFIG: + with open(f"generative_configs/{GENERATIVE_SERVICE_CONFIG}", "r") as f: + GENERATIVE_SERVICE_CONFIG = json.load(f) FILTER_TOXIC_OR_BADLISTED = int(getenv("FILTER_TOXIC_OR_BADLISTED")) N_UTTERANCES_CONTEXT = int(getenv("N_UTTERANCES_CONTEXT")) CRITERION = getenv("CRITERION", DEFAULT_CRITERION) From a115036923c249d4077ffda4ec8c9ff3b0f63912 Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Mon, 24 Apr 2023 17:26:30 +0300 Subject: [PATCH 15/32] fix: configs --- assistant_dists/ai_faq_assistant/dev.yml | 4 +-- .../docker-compose.override.yml | 33 +++++++------------ .../ai_faq_assistant/pipeline_conf.json | 4 +-- .../da_costa_clothes_assistant/dev.yml | 4 +-- .../docker-compose.override.yml | 33 +++++++------------ .../pipeline_conf.json | 4 +-- assistant_dists/nutrition_assistant/dev.yml | 4 +-- .../docker-compose.override.yml | 33 +++++++------------ .../nutrition_assistant/pipeline_conf.json | 4 +-- .../universal_prompted_assistant/dev.yml | 4 +-- .../docker-compose.override.yml | 33 +++++++------------ .../pipeline_conf.json | 4 +-- 12 files changed, 60 insertions(+), 104 deletions(-) diff --git a/assistant_dists/ai_faq_assistant/dev.yml b/assistant_dists/ai_faq_assistant/dev.yml index 72cf200379..ae1256384f 100644 --- a/assistant_dists/ai_faq_assistant/dev.yml +++ b/assistant_dists/ai_faq_assistant/dev.yml @@ -10,12 +10,12 @@ services: - "./annotators/SentSeg:/src" ports: - 8011:8011 - no-restrictions-selector: + oasst12b-based-response-selector: volumes: - "./response_selectors/convers_evaluation_based_selector:/src" - "./common:/src/common" ports: - - 8009:8009 + - 8003:8003 badlisted-words: volumes: - "./annotators/BadlistedWordsDetector:/src" diff --git a/assistant_dists/ai_faq_assistant/docker-compose.override.yml b/assistant_dists/ai_faq_assistant/docker-compose.override.yml index 8cd7f7c978..ac16efad51 100644 --- a/assistant_dists/ai_faq_assistant/docker-compose.override.yml +++ b/assistant_dists/ai_faq_assistant/docker-compose.override.yml @@ -2,7 +2,7 @@ services: agent: command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/ai_faq_assistant/pipeline_conf.json' environment: - WAIT_HOSTS: "sentseg:8011, no-restrictions-selector:8009, badlisted-words:8018, combined-classification:8087, + WAIT_HOSTS: "sentseg:8011, oasst12b-based-response-selector:8003, badlisted-words:8018, combined-classification:8087, sentence-ranker:8128, prompt-selector:8135, transformers-lm-oasst12b:8158, dff-ai-faq-prompted-skill:8143" WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1000} @@ -38,32 +38,21 @@ services: reservations: memory: 2G - no-restrictions-selector: + oasst12b-based-response-selector: env_file: [ .env ] build: args: - TAG_BASED_SELECTION: 1 - CALL_BY_NAME_PROBABILITY: 0.5 - PROMPT_PROBA: 0 - ACKNOWLEDGEMENT_PROBA: 0.3 - PRIORITIZE_WITH_REQUIRED_ACT: 0 - PRIORITIZE_NO_DIALOG_BREAKDOWN: 0 - PRIORITIZE_WITH_SAME_TOPIC_ENTITY: 0 - IGNORE_DISLIKED_SKILLS: 0 - GREETING_FIRST: 0 - RESTRICTION_FOR_SENSITIVE_CASE: 0 - PRIORITIZE_PROMTS_WHEN_NO_SCRIPTS: 0 - MAX_TURNS_WITHOUT_SCRIPTS: 100 - ADD_ACKNOWLEDGMENTS_IF_POSSIBLE: 0 - PRIORITIZE_SCRIPTED_SKILLS: 0 - CONFIDENCE_STRENGTH: 0.8 - CONV_EVAL_STRENGTH: 0.4 - PRIORITIZE_HUMAN_INITIATIVE: 1 - QUESTION_TO_QUESTION_DOWNSCORE_COEF: 0.8 + SERVICE_PORT: 8003 + SERVICE_NAME: response_selector LANGUAGE: EN + GENERATIVE_SERVICE_URL: http://transformers-lm-oasst12b:8158/respond + GENERATIVE_SERVICE_CONFIG: default_generative_config.json + GENERATIVE_TIMEOUT: 5 + N_UTTERANCES_CONTEXT: 5 + FILTER_TOXIC_OR_BADLISTED: 1 context: . - dockerfile: ./response_selectors/convers_evaluation_based_selector/Dockerfile - command: flask run -h 0.0.0.0 -p 8009 + dockerfile: ./response_selectors/llm_based_response_selector/Dockerfile + command: flask run -h 0.0.0.0 -p 8003 environment: - FLASK_APP=server deploy: diff --git a/assistant_dists/ai_faq_assistant/pipeline_conf.json b/assistant_dists/ai_faq_assistant/pipeline_conf.json index ce4b5a91e9..7e60619764 100644 --- a/assistant_dists/ai_faq_assistant/pipeline_conf.json +++ b/assistant_dists/ai_faq_assistant/pipeline_conf.json @@ -273,7 +273,7 @@ "connector": { "protocol": "http", "timeout": 1.0, - "url": "http://no-restrictions-selector:8009/respond" + "url": "http://oasst12b-based-response-selector:8003/respond" }, "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", @@ -284,7 +284,7 @@ "is_enabled": true, "source": { "directory": "response_selectors/convers_evaluation_based_selector", - "container": "no-restrictions-selector", + "container": "oasst12b-based-response-selector", "endpoint": "respond" } } diff --git a/assistant_dists/da_costa_clothes_assistant/dev.yml b/assistant_dists/da_costa_clothes_assistant/dev.yml index 5515d1836c..a5209b0eac 100644 --- a/assistant_dists/da_costa_clothes_assistant/dev.yml +++ b/assistant_dists/da_costa_clothes_assistant/dev.yml @@ -10,12 +10,12 @@ services: - "./annotators/SentSeg:/src" ports: - 8011:8011 - no-restrictions-selector: + oasst12b-based-response-selector: volumes: - "./response_selectors/convers_evaluation_based_selector:/src" - "./common:/src/common" ports: - - 8009:8009 + - 8003:8003 badlisted-words: volumes: - "./annotators/BadlistedWordsDetector:/src" diff --git a/assistant_dists/da_costa_clothes_assistant/docker-compose.override.yml b/assistant_dists/da_costa_clothes_assistant/docker-compose.override.yml index 15932e3073..20b28ddf90 100644 --- a/assistant_dists/da_costa_clothes_assistant/docker-compose.override.yml +++ b/assistant_dists/da_costa_clothes_assistant/docker-compose.override.yml @@ -2,7 +2,7 @@ services: agent: command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/da_costa_clothes_assistant/pipeline_conf.json' environment: - WAIT_HOSTS: "sentseg:8011, no-restrictions-selector:8009, badlisted-words:8018, combined-classification:8087, + WAIT_HOSTS: "sentseg:8011, oasst12b-based-response-selector:8003, badlisted-words:8018, combined-classification:8087, sentence-ranker:8128, prompt-selector:8135, transformers-lm-oasst12b:8158, dff-da-costa-clothes-prompted-skill:8144" WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1000} @@ -38,32 +38,21 @@ services: reservations: memory: 2G - no-restrictions-selector: + oasst12b-based-response-selector: env_file: [ .env ] build: args: - TAG_BASED_SELECTION: 1 - CALL_BY_NAME_PROBABILITY: 0.5 - PROMPT_PROBA: 0 - ACKNOWLEDGEMENT_PROBA: 0.3 - PRIORITIZE_WITH_REQUIRED_ACT: 0 - PRIORITIZE_NO_DIALOG_BREAKDOWN: 0 - PRIORITIZE_WITH_SAME_TOPIC_ENTITY: 0 - IGNORE_DISLIKED_SKILLS: 0 - GREETING_FIRST: 0 - RESTRICTION_FOR_SENSITIVE_CASE: 0 - PRIORITIZE_PROMTS_WHEN_NO_SCRIPTS: 0 - MAX_TURNS_WITHOUT_SCRIPTS: 100 - ADD_ACKNOWLEDGMENTS_IF_POSSIBLE: 0 - PRIORITIZE_SCRIPTED_SKILLS: 0 - CONFIDENCE_STRENGTH: 0.8 - CONV_EVAL_STRENGTH: 0.4 - PRIORITIZE_HUMAN_INITIATIVE: 1 - QUESTION_TO_QUESTION_DOWNSCORE_COEF: 0.8 + SERVICE_PORT: 8003 + SERVICE_NAME: response_selector LANGUAGE: EN + GENERATIVE_SERVICE_URL: http://transformers-lm-oasst12b:8158/respond + GENERATIVE_SERVICE_CONFIG: default_generative_config.json + GENERATIVE_TIMEOUT: 5 + N_UTTERANCES_CONTEXT: 5 + FILTER_TOXIC_OR_BADLISTED: 1 context: . - dockerfile: ./response_selectors/convers_evaluation_based_selector/Dockerfile - command: flask run -h 0.0.0.0 -p 8009 + dockerfile: ./response_selectors/llm_based_response_selector/Dockerfile + command: flask run -h 0.0.0.0 -p 8003 environment: - FLASK_APP=server deploy: diff --git a/assistant_dists/da_costa_clothes_assistant/pipeline_conf.json b/assistant_dists/da_costa_clothes_assistant/pipeline_conf.json index 5a91f53ad7..3a4991be22 100644 --- a/assistant_dists/da_costa_clothes_assistant/pipeline_conf.json +++ b/assistant_dists/da_costa_clothes_assistant/pipeline_conf.json @@ -273,7 +273,7 @@ "connector": { "protocol": "http", "timeout": 1.0, - "url": "http://no-restrictions-selector:8009/respond" + "url": "http://oasst12b-based-response-selector:8003/respond" }, "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", @@ -284,7 +284,7 @@ "is_enabled": true, "source": { "directory": "response_selectors/convers_evaluation_based_selector", - "container": "no-restrictions-selector", + "container": "oasst12b-based-response-selector", "endpoint": "respond" } } diff --git a/assistant_dists/nutrition_assistant/dev.yml b/assistant_dists/nutrition_assistant/dev.yml index 3ec5688216..6b0c4b7098 100644 --- a/assistant_dists/nutrition_assistant/dev.yml +++ b/assistant_dists/nutrition_assistant/dev.yml @@ -10,12 +10,12 @@ services: - "./annotators/SentSeg:/src" ports: - 8011:8011 - no-restrictions-selector: + oasst12b-based-response-selector: volumes: - "./response_selectors/convers_evaluation_based_selector:/src" - "./common:/src/common" ports: - - 8009:8009 + - 8003:8003 badlisted-words: volumes: - "./annotators/BadlistedWordsDetector:/src" diff --git a/assistant_dists/nutrition_assistant/docker-compose.override.yml b/assistant_dists/nutrition_assistant/docker-compose.override.yml index 9d055e5139..5545d43733 100644 --- a/assistant_dists/nutrition_assistant/docker-compose.override.yml +++ b/assistant_dists/nutrition_assistant/docker-compose.override.yml @@ -2,7 +2,7 @@ services: agent: command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/nutrition_assistant/pipeline_conf.json' environment: - WAIT_HOSTS: "sentseg:8011, no-restrictions-selector:8009, badlisted-words:8018, combined-classification:8087, + WAIT_HOSTS: "sentseg:8011, oasst12b-based-response-selector:8003, badlisted-words:8018, combined-classification:8087, sentence-ranker:8128, prompt-selector:8135, transformers-lm-oasst12b:8158, dff-nutrition-prompted-skill:8141" WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1000} @@ -38,32 +38,21 @@ services: reservations: memory: 2G - no-restrictions-selector: + oasst12b-based-response-selector: env_file: [ .env ] build: args: - TAG_BASED_SELECTION: 1 - CALL_BY_NAME_PROBABILITY: 0.5 - PROMPT_PROBA: 0 - ACKNOWLEDGEMENT_PROBA: 0.3 - PRIORITIZE_WITH_REQUIRED_ACT: 0 - PRIORITIZE_NO_DIALOG_BREAKDOWN: 0 - PRIORITIZE_WITH_SAME_TOPIC_ENTITY: 0 - IGNORE_DISLIKED_SKILLS: 0 - GREETING_FIRST: 0 - RESTRICTION_FOR_SENSITIVE_CASE: 0 - PRIORITIZE_PROMTS_WHEN_NO_SCRIPTS: 0 - MAX_TURNS_WITHOUT_SCRIPTS: 100 - ADD_ACKNOWLEDGMENTS_IF_POSSIBLE: 0 - PRIORITIZE_SCRIPTED_SKILLS: 0 - CONFIDENCE_STRENGTH: 0.8 - CONV_EVAL_STRENGTH: 0.4 - PRIORITIZE_HUMAN_INITIATIVE: 1 - QUESTION_TO_QUESTION_DOWNSCORE_COEF: 0.8 + SERVICE_PORT: 8003 + SERVICE_NAME: response_selector LANGUAGE: EN + GENERATIVE_SERVICE_URL: http://transformers-lm-oasst12b:8158/respond + GENERATIVE_SERVICE_CONFIG: default_generative_config.json + GENERATIVE_TIMEOUT: 5 + N_UTTERANCES_CONTEXT: 5 + FILTER_TOXIC_OR_BADLISTED: 1 context: . - dockerfile: ./response_selectors/convers_evaluation_based_selector/Dockerfile - command: flask run -h 0.0.0.0 -p 8009 + dockerfile: ./response_selectors/llm_based_response_selector/Dockerfile + command: flask run -h 0.0.0.0 -p 8003 environment: - FLASK_APP=server deploy: diff --git a/assistant_dists/nutrition_assistant/pipeline_conf.json b/assistant_dists/nutrition_assistant/pipeline_conf.json index 134cfd8d51..f5002d21cb 100644 --- a/assistant_dists/nutrition_assistant/pipeline_conf.json +++ b/assistant_dists/nutrition_assistant/pipeline_conf.json @@ -273,7 +273,7 @@ "connector": { "protocol": "http", "timeout": 1.0, - "url": "http://no-restrictions-selector:8009/respond" + "url": "http://oasst12b-based-response-selector:8003/respond" }, "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", @@ -284,7 +284,7 @@ "is_enabled": true, "source": { "directory": "response_selectors/convers_evaluation_based_selector", - "container": "no-restrictions-selector", + "container": "oasst12b-based-response-selector", "endpoint": "respond" } } diff --git a/assistant_dists/universal_prompted_assistant/dev.yml b/assistant_dists/universal_prompted_assistant/dev.yml index 24031872f2..3094e3de58 100644 --- a/assistant_dists/universal_prompted_assistant/dev.yml +++ b/assistant_dists/universal_prompted_assistant/dev.yml @@ -10,12 +10,12 @@ services: - "./annotators/SentSeg:/src" ports: - 8011:8011 - no-restrictions-selector: + oasst12b-based-response-selector: volumes: - "./response_selectors/convers_evaluation_based_selector:/src" - "./common:/src/common" ports: - - 8009:8009 + - 8003:8003 badlisted-words: volumes: - "./annotators/BadlistedWordsDetector:/src" diff --git a/assistant_dists/universal_prompted_assistant/docker-compose.override.yml b/assistant_dists/universal_prompted_assistant/docker-compose.override.yml index c5576ab847..61058e7a9e 100644 --- a/assistant_dists/universal_prompted_assistant/docker-compose.override.yml +++ b/assistant_dists/universal_prompted_assistant/docker-compose.override.yml @@ -2,7 +2,7 @@ services: agent: command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/universal_prompted_assistant/pipeline_conf.json' environment: - WAIT_HOSTS: "sentseg:8011, no-restrictions-selector:8009, badlisted-words:8018, combined-classification:8087, + WAIT_HOSTS: "sentseg:8011, oasst12b-based-response-selector:8003, badlisted-words:8018, combined-classification:8087, sentence-ranker:8128, transformers-lm-gptj:8130, transformers-lm-oasst12b:8158, openai-api-chatgpt:8145, openai-api-davinci3:8131, dff-universal-prompted-skill:8147" @@ -39,32 +39,21 @@ services: reservations: memory: 2G - no-restrictions-selector: + oasst12b-based-response-selector: env_file: [ .env ] build: args: - TAG_BASED_SELECTION: 1 - CALL_BY_NAME_PROBABILITY: 0.5 - PROMPT_PROBA: 0 - ACKNOWLEDGEMENT_PROBA: 0.3 - PRIORITIZE_WITH_REQUIRED_ACT: 0 - PRIORITIZE_NO_DIALOG_BREAKDOWN: 0 - PRIORITIZE_WITH_SAME_TOPIC_ENTITY: 0 - IGNORE_DISLIKED_SKILLS: 0 - GREETING_FIRST: 0 - RESTRICTION_FOR_SENSITIVE_CASE: 0 - PRIORITIZE_PROMTS_WHEN_NO_SCRIPTS: 0 - MAX_TURNS_WITHOUT_SCRIPTS: 100 - ADD_ACKNOWLEDGMENTS_IF_POSSIBLE: 0 - PRIORITIZE_SCRIPTED_SKILLS: 0 - CONFIDENCE_STRENGTH: 0.8 - CONV_EVAL_STRENGTH: 0.4 - PRIORITIZE_HUMAN_INITIATIVE: 1 - QUESTION_TO_QUESTION_DOWNSCORE_COEF: 0.8 + SERVICE_PORT: 8003 + SERVICE_NAME: response_selector LANGUAGE: EN + GENERATIVE_SERVICE_URL: http://transformers-lm-oasst12b:8158/respond + GENERATIVE_SERVICE_CONFIG: default_generative_config.json + GENERATIVE_TIMEOUT: 5 + N_UTTERANCES_CONTEXT: 5 + FILTER_TOXIC_OR_BADLISTED: 1 context: . - dockerfile: ./response_selectors/convers_evaluation_based_selector/Dockerfile - command: flask run -h 0.0.0.0 -p 8009 + dockerfile: ./response_selectors/llm_based_response_selector/Dockerfile + command: flask run -h 0.0.0.0 -p 8003 environment: - FLASK_APP=server deploy: diff --git a/assistant_dists/universal_prompted_assistant/pipeline_conf.json b/assistant_dists/universal_prompted_assistant/pipeline_conf.json index ee1eb9d900..f493c2ca70 100644 --- a/assistant_dists/universal_prompted_assistant/pipeline_conf.json +++ b/assistant_dists/universal_prompted_assistant/pipeline_conf.json @@ -255,7 +255,7 @@ "connector": { "protocol": "http", "timeout": 1.0, - "url": "http://no-restrictions-selector:8009/respond" + "url": "http://oasst12b-based-response-selector:8003/respond" }, "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", @@ -266,7 +266,7 @@ "is_enabled": true, "source": { "directory": "response_selectors/convers_evaluation_based_selector", - "container": "no-restrictions-selector", + "container": "oasst12b-based-response-selector", "endpoint": "respond" } } From b3cbc1bd9c733a11b4caa39e3d45f9c08cc66c4a Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Mon, 24 Apr 2023 17:38:53 +0300 Subject: [PATCH 16/32] fix: mapping --- assistant_dists/ai_faq_assistant/dev.yml | 2 +- assistant_dists/ai_faq_assistant/pipeline_conf.json | 2 +- assistant_dists/da_costa_clothes_assistant/dev.yml | 2 +- assistant_dists/da_costa_clothes_assistant/pipeline_conf.json | 2 +- assistant_dists/nutrition_assistant/dev.yml | 2 +- assistant_dists/nutrition_assistant/pipeline_conf.json | 2 +- assistant_dists/universal_prompted_assistant/dev.yml | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/assistant_dists/ai_faq_assistant/dev.yml b/assistant_dists/ai_faq_assistant/dev.yml index ae1256384f..8d553c0de7 100644 --- a/assistant_dists/ai_faq_assistant/dev.yml +++ b/assistant_dists/ai_faq_assistant/dev.yml @@ -12,7 +12,7 @@ services: - 8011:8011 oasst12b-based-response-selector: volumes: - - "./response_selectors/convers_evaluation_based_selector:/src" + - "./response_selectors/llm_based_response_selector:/src" - "./common:/src/common" ports: - 8003:8003 diff --git a/assistant_dists/ai_faq_assistant/pipeline_conf.json b/assistant_dists/ai_faq_assistant/pipeline_conf.json index 7e60619764..a2b4026093 100644 --- a/assistant_dists/ai_faq_assistant/pipeline_conf.json +++ b/assistant_dists/ai_faq_assistant/pipeline_conf.json @@ -283,7 +283,7 @@ "state_manager_method": "add_bot_utterance", "is_enabled": true, "source": { - "directory": "response_selectors/convers_evaluation_based_selector", + "directory": "response_selectors/llm_based_response_selector", "container": "oasst12b-based-response-selector", "endpoint": "respond" } diff --git a/assistant_dists/da_costa_clothes_assistant/dev.yml b/assistant_dists/da_costa_clothes_assistant/dev.yml index a5209b0eac..3e0c57a165 100644 --- a/assistant_dists/da_costa_clothes_assistant/dev.yml +++ b/assistant_dists/da_costa_clothes_assistant/dev.yml @@ -12,7 +12,7 @@ services: - 8011:8011 oasst12b-based-response-selector: volumes: - - "./response_selectors/convers_evaluation_based_selector:/src" + - "./response_selectors/llm_based_response_selector:/src" - "./common:/src/common" ports: - 8003:8003 diff --git a/assistant_dists/da_costa_clothes_assistant/pipeline_conf.json b/assistant_dists/da_costa_clothes_assistant/pipeline_conf.json index 3a4991be22..7e56d89b81 100644 --- a/assistant_dists/da_costa_clothes_assistant/pipeline_conf.json +++ b/assistant_dists/da_costa_clothes_assistant/pipeline_conf.json @@ -283,7 +283,7 @@ "state_manager_method": "add_bot_utterance", "is_enabled": true, "source": { - "directory": "response_selectors/convers_evaluation_based_selector", + "directory": "response_selectors/llm_based_response_selector", "container": "oasst12b-based-response-selector", "endpoint": "respond" } diff --git a/assistant_dists/nutrition_assistant/dev.yml b/assistant_dists/nutrition_assistant/dev.yml index 6b0c4b7098..dbb0bc7066 100644 --- a/assistant_dists/nutrition_assistant/dev.yml +++ b/assistant_dists/nutrition_assistant/dev.yml @@ -12,7 +12,7 @@ services: - 8011:8011 oasst12b-based-response-selector: volumes: - - "./response_selectors/convers_evaluation_based_selector:/src" + - "./response_selectors/llm_based_response_selector:/src" - "./common:/src/common" ports: - 8003:8003 diff --git a/assistant_dists/nutrition_assistant/pipeline_conf.json b/assistant_dists/nutrition_assistant/pipeline_conf.json index f5002d21cb..cb58ff0d5d 100644 --- a/assistant_dists/nutrition_assistant/pipeline_conf.json +++ b/assistant_dists/nutrition_assistant/pipeline_conf.json @@ -283,7 +283,7 @@ "state_manager_method": "add_bot_utterance", "is_enabled": true, "source": { - "directory": "response_selectors/convers_evaluation_based_selector", + "directory": "response_selectors/llm_based_response_selector", "container": "oasst12b-based-response-selector", "endpoint": "respond" } diff --git a/assistant_dists/universal_prompted_assistant/dev.yml b/assistant_dists/universal_prompted_assistant/dev.yml index 3094e3de58..e7bd16de45 100644 --- a/assistant_dists/universal_prompted_assistant/dev.yml +++ b/assistant_dists/universal_prompted_assistant/dev.yml @@ -12,7 +12,7 @@ services: - 8011:8011 oasst12b-based-response-selector: volumes: - - "./response_selectors/convers_evaluation_based_selector:/src" + - "./response_selectors/llm_based_response_selector:/src" - "./common:/src/common" ports: - 8003:8003 From 3c244c385880a0f5793c8769ccd0d965947b2005 Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Mon, 24 Apr 2023 18:21:56 +0300 Subject: [PATCH 17/32] fix: utilize ranker based selector --- assistant_dists/ai_faq_assistant/dev.yml | 6 ++-- .../docker-compose.override.yml | 16 ++++----- .../ai_faq_assistant/pipeline_conf.json | 6 ++-- .../da_costa_clothes_assistant/dev.yml | 6 ++-- .../docker-compose.override.yml | 16 ++++----- .../pipeline_conf.json | 6 ++-- assistant_dists/deeppavlov_assistant/dev.yml | 4 +-- .../docker-compose.override.yml | 33 +++++++------------ .../deeppavlov_assistant/pipeline_conf.json | 4 +-- .../dream_persona_openai_prompted/dev.yml | 4 +-- .../docker-compose.override.yml | 33 +++++++------------ .../pipeline_conf.json | 4 +-- .../dream_persona_prompted/dev.yml | 6 ++-- .../docker-compose.override.yml | 16 ++++----- .../dream_persona_prompted/pipeline_conf.json | 6 ++-- .../empathetic_marketing_assistant/dev.yml | 6 ++-- .../docker-compose.override.yml | 16 ++++----- .../pipeline_conf.json | 6 ++-- assistant_dists/fairytale_assistant/dev.yml | 4 +-- .../docker-compose.override.yml | 33 +++++++------------ .../fairytale_assistant/pipeline_conf.json | 4 +-- assistant_dists/nutrition_assistant/dev.yml | 6 ++-- .../docker-compose.override.yml | 16 ++++----- .../nutrition_assistant/pipeline_conf.json | 6 ++-- .../rhodes_coaching_assistant/dev.yml | 4 +-- .../docker-compose.override.yml | 33 +++++++------------ .../pipeline_conf.json | 4 +-- .../universal_prompted_assistant/dev.yml | 6 ++-- .../docker-compose.override.yml | 16 ++++----- .../pipeline_conf.json | 6 ++-- .../component.yml | 30 +++++++++++++++-- .../pipeline.yml | 14 ++++++-- 32 files changed, 184 insertions(+), 192 deletions(-) diff --git a/assistant_dists/ai_faq_assistant/dev.yml b/assistant_dists/ai_faq_assistant/dev.yml index 8d553c0de7..075cec3252 100644 --- a/assistant_dists/ai_faq_assistant/dev.yml +++ b/assistant_dists/ai_faq_assistant/dev.yml @@ -10,12 +10,12 @@ services: - "./annotators/SentSeg:/src" ports: - 8011:8011 - oasst12b-based-response-selector: + ranking-based-response-selector: volumes: - - "./response_selectors/llm_based_response_selector:/src" + - "./response_selectors/ranking_based_response_selector:/src" - "./common:/src/common" ports: - - 8003:8003 + - 8002:8002 badlisted-words: volumes: - "./annotators/BadlistedWordsDetector:/src" diff --git a/assistant_dists/ai_faq_assistant/docker-compose.override.yml b/assistant_dists/ai_faq_assistant/docker-compose.override.yml index ac16efad51..06e7a380bc 100644 --- a/assistant_dists/ai_faq_assistant/docker-compose.override.yml +++ b/assistant_dists/ai_faq_assistant/docker-compose.override.yml @@ -2,7 +2,7 @@ services: agent: command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/ai_faq_assistant/pipeline_conf.json' environment: - WAIT_HOSTS: "sentseg:8011, oasst12b-based-response-selector:8003, badlisted-words:8018, combined-classification:8087, + WAIT_HOSTS: "sentseg:8011, ranking-based-response-selector:8002, badlisted-words:8018, combined-classification:8087, sentence-ranker:8128, prompt-selector:8135, transformers-lm-oasst12b:8158, dff-ai-faq-prompted-skill:8143" WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1000} @@ -38,21 +38,21 @@ services: reservations: memory: 2G - oasst12b-based-response-selector: + ranking-based-response-selector: env_file: [ .env ] build: args: - SERVICE_PORT: 8003 + SERVICE_PORT: 8002 SERVICE_NAME: response_selector LANGUAGE: EN - GENERATIVE_SERVICE_URL: http://transformers-lm-oasst12b:8158/respond - GENERATIVE_SERVICE_CONFIG: default_generative_config.json - GENERATIVE_TIMEOUT: 5 + SENTENCE_RANKER_ANNOTATION_NAME: sentence_ranker + SENTENCE_RANKER_SERVICE_URL: http://sentence-ranker:8128/respond + SENTENCE_RANKER_TIMEOUT: 3 N_UTTERANCES_CONTEXT: 5 FILTER_TOXIC_OR_BADLISTED: 1 context: . - dockerfile: ./response_selectors/llm_based_response_selector/Dockerfile - command: flask run -h 0.0.0.0 -p 8003 + dockerfile: ./response_selectors/ranking_based_response_selector/Dockerfile + command: flask run -h 0.0.0.0 -p 8002 environment: - FLASK_APP=server deploy: diff --git a/assistant_dists/ai_faq_assistant/pipeline_conf.json b/assistant_dists/ai_faq_assistant/pipeline_conf.json index a2b4026093..4d80ce47ca 100644 --- a/assistant_dists/ai_faq_assistant/pipeline_conf.json +++ b/assistant_dists/ai_faq_assistant/pipeline_conf.json @@ -273,7 +273,7 @@ "connector": { "protocol": "http", "timeout": 1.0, - "url": "http://oasst12b-based-response-selector:8003/respond" + "url": "http://ranking-based-response-selector:8002/respond" }, "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", @@ -283,8 +283,8 @@ "state_manager_method": "add_bot_utterance", "is_enabled": true, "source": { - "directory": "response_selectors/llm_based_response_selector", - "container": "oasst12b-based-response-selector", + "directory": "response_selectors/ranking_based_response_selector", + "container": "ranking-based-response-selector", "endpoint": "respond" } } diff --git a/assistant_dists/da_costa_clothes_assistant/dev.yml b/assistant_dists/da_costa_clothes_assistant/dev.yml index 3e0c57a165..fbc5aadac6 100644 --- a/assistant_dists/da_costa_clothes_assistant/dev.yml +++ b/assistant_dists/da_costa_clothes_assistant/dev.yml @@ -10,12 +10,12 @@ services: - "./annotators/SentSeg:/src" ports: - 8011:8011 - oasst12b-based-response-selector: + ranking-based-response-selector: volumes: - - "./response_selectors/llm_based_response_selector:/src" + - "./response_selectors/ranking_based_response_selector:/src" - "./common:/src/common" ports: - - 8003:8003 + - 8002:8002 badlisted-words: volumes: - "./annotators/BadlistedWordsDetector:/src" diff --git a/assistant_dists/da_costa_clothes_assistant/docker-compose.override.yml b/assistant_dists/da_costa_clothes_assistant/docker-compose.override.yml index 20b28ddf90..ceeecbb3d7 100644 --- a/assistant_dists/da_costa_clothes_assistant/docker-compose.override.yml +++ b/assistant_dists/da_costa_clothes_assistant/docker-compose.override.yml @@ -2,7 +2,7 @@ services: agent: command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/da_costa_clothes_assistant/pipeline_conf.json' environment: - WAIT_HOSTS: "sentseg:8011, oasst12b-based-response-selector:8003, badlisted-words:8018, combined-classification:8087, + WAIT_HOSTS: "sentseg:8011, ranking-based-response-selector:8002, badlisted-words:8018, combined-classification:8087, sentence-ranker:8128, prompt-selector:8135, transformers-lm-oasst12b:8158, dff-da-costa-clothes-prompted-skill:8144" WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1000} @@ -38,21 +38,21 @@ services: reservations: memory: 2G - oasst12b-based-response-selector: + ranking-based-response-selector: env_file: [ .env ] build: args: - SERVICE_PORT: 8003 + SERVICE_PORT: 8002 SERVICE_NAME: response_selector LANGUAGE: EN - GENERATIVE_SERVICE_URL: http://transformers-lm-oasst12b:8158/respond - GENERATIVE_SERVICE_CONFIG: default_generative_config.json - GENERATIVE_TIMEOUT: 5 + SENTENCE_RANKER_ANNOTATION_NAME: sentence_ranker + SENTENCE_RANKER_SERVICE_URL: http://sentence-ranker:8128/respond + SENTENCE_RANKER_TIMEOUT: 3 N_UTTERANCES_CONTEXT: 5 FILTER_TOXIC_OR_BADLISTED: 1 context: . - dockerfile: ./response_selectors/llm_based_response_selector/Dockerfile - command: flask run -h 0.0.0.0 -p 8003 + dockerfile: ./response_selectors/ranking_based_response_selector/Dockerfile + command: flask run -h 0.0.0.0 -p 8002 environment: - FLASK_APP=server deploy: diff --git a/assistant_dists/da_costa_clothes_assistant/pipeline_conf.json b/assistant_dists/da_costa_clothes_assistant/pipeline_conf.json index 7e56d89b81..2fce861f0d 100644 --- a/assistant_dists/da_costa_clothes_assistant/pipeline_conf.json +++ b/assistant_dists/da_costa_clothes_assistant/pipeline_conf.json @@ -273,7 +273,7 @@ "connector": { "protocol": "http", "timeout": 1.0, - "url": "http://oasst12b-based-response-selector:8003/respond" + "url": "http://ranking-based-response-selector:8002/respond" }, "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", @@ -283,8 +283,8 @@ "state_manager_method": "add_bot_utterance", "is_enabled": true, "source": { - "directory": "response_selectors/llm_based_response_selector", - "container": "oasst12b-based-response-selector", + "directory": "response_selectors/ranking_based_response_selector", + "container": "ranking-based-response-selector", "endpoint": "respond" } } diff --git a/assistant_dists/deeppavlov_assistant/dev.yml b/assistant_dists/deeppavlov_assistant/dev.yml index e56fbc76ce..d9b16338c8 100644 --- a/assistant_dists/deeppavlov_assistant/dev.yml +++ b/assistant_dists/deeppavlov_assistant/dev.yml @@ -7,10 +7,10 @@ services: - 4242:4242 no-restrictions-selector: volumes: - - "./response_selectors/convers_evaluation_based_selector:/src" + - "./response_selectors/ranking_based_response_selector:/src" - "./common:/src/common" ports: - - 8009:8009 + - 8002:8002 sentence-ranker: volumes: - "./services/sentence_ranker:/src" diff --git a/assistant_dists/deeppavlov_assistant/docker-compose.override.yml b/assistant_dists/deeppavlov_assistant/docker-compose.override.yml index a4a3ef2d2f..aa483f327e 100644 --- a/assistant_dists/deeppavlov_assistant/docker-compose.override.yml +++ b/assistant_dists/deeppavlov_assistant/docker-compose.override.yml @@ -2,7 +2,7 @@ services: agent: command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/deeppavlov_assistant/pipeline_conf.json' environment: - WAIT_HOSTS: "no-restrictions-selector:8009, sentence-ranker:8128, prompt-selector:8135, openai-api-chatgpt:8145, + WAIT_HOSTS: "no-restrictions-selector:8002, sentence-ranker:8128, prompt-selector:8135, openai-api-chatgpt:8145, dff-deeppavlov-prompted-skill:8146" WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1000} HIGH_PRIORITY_INTENTS: 1 @@ -10,32 +10,21 @@ services: ALWAYS_TURN_ON_ALL_SKILLS: 0 LANGUAGE: EN - no-restrictions-selector: + ranking-based-response-selector: env_file: [ .env ] build: args: - TAG_BASED_SELECTION: 1 - CALL_BY_NAME_PROBABILITY: 0.5 - PROMPT_PROBA: 0 - ACKNOWLEDGEMENT_PROBA: 0.3 - PRIORITIZE_WITH_REQUIRED_ACT: 0 - PRIORITIZE_NO_DIALOG_BREAKDOWN: 0 - PRIORITIZE_WITH_SAME_TOPIC_ENTITY: 0 - IGNORE_DISLIKED_SKILLS: 0 - GREETING_FIRST: 0 - RESTRICTION_FOR_SENSITIVE_CASE: 0 - PRIORITIZE_PROMTS_WHEN_NO_SCRIPTS: 0 - MAX_TURNS_WITHOUT_SCRIPTS: 100 - ADD_ACKNOWLEDGMENTS_IF_POSSIBLE: 0 - PRIORITIZE_SCRIPTED_SKILLS: 0 - CONFIDENCE_STRENGTH: 0.8 - CONV_EVAL_STRENGTH: 0.4 - PRIORITIZE_HUMAN_INITIATIVE: 1 - QUESTION_TO_QUESTION_DOWNSCORE_COEF: 0.8 + SERVICE_PORT: 8002 + SERVICE_NAME: response_selector LANGUAGE: EN + SENTENCE_RANKER_ANNOTATION_NAME: sentence_ranker + SENTENCE_RANKER_SERVICE_URL: http://sentence-ranker:8128/respond + SENTENCE_RANKER_TIMEOUT: 3 + N_UTTERANCES_CONTEXT: 5 + FILTER_TOXIC_OR_BADLISTED: 1 context: . - dockerfile: ./response_selectors/convers_evaluation_based_selector/Dockerfile - command: flask run -h 0.0.0.0 -p 8009 + dockerfile: ./response_selectors/ranking_based_response_selector/Dockerfile + command: flask run -h 0.0.0.0 -p 8002 environment: - FLASK_APP=server deploy: diff --git a/assistant_dists/deeppavlov_assistant/pipeline_conf.json b/assistant_dists/deeppavlov_assistant/pipeline_conf.json index 55cd139471..4dc919c2fe 100644 --- a/assistant_dists/deeppavlov_assistant/pipeline_conf.json +++ b/assistant_dists/deeppavlov_assistant/pipeline_conf.json @@ -159,7 +159,7 @@ "connector": { "protocol": "http", "timeout": 1.0, - "url": "http://no-restrictions-selector:8009/respond" + "url": "http://no-restrictions-selector:8002/respond" }, "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", @@ -169,7 +169,7 @@ "state_manager_method": "add_bot_utterance", "is_enabled": true, "source": { - "directory": "response_selectors/convers_evaluation_based_selector", + "directory": "response_selectors/ranking_based_response_selector", "container": "no-restrictions-selector", "endpoint": "respond" } diff --git a/assistant_dists/dream_persona_openai_prompted/dev.yml b/assistant_dists/dream_persona_openai_prompted/dev.yml index 5e19a6ebe8..3e2bafff20 100644 --- a/assistant_dists/dream_persona_openai_prompted/dev.yml +++ b/assistant_dists/dream_persona_openai_prompted/dev.yml @@ -12,10 +12,10 @@ services: - 8011:8011 convers-evaluation-no-scripts-selector: volumes: - - "./response_selectors/convers_evaluation_based_selector:/src" + - "./response_selectors/ranking_based_response_selector:/src" - "./common:/src/common" ports: - - 8009:8009 + - 8002:8002 badlisted-words: volumes: - "./annotators/BadlistedWordsDetector:/src" diff --git a/assistant_dists/dream_persona_openai_prompted/docker-compose.override.yml b/assistant_dists/dream_persona_openai_prompted/docker-compose.override.yml index b337cb34fe..46081b747d 100644 --- a/assistant_dists/dream_persona_openai_prompted/docker-compose.override.yml +++ b/assistant_dists/dream_persona_openai_prompted/docker-compose.override.yml @@ -2,7 +2,7 @@ services: agent: command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/dream_persona_openai_prompted/pipeline_conf.json' environment: - WAIT_HOSTS: "sentseg:8011, convers-evaluation-no-scripts-selector:8009, badlisted-words:8018, combined-classification:8087, + WAIT_HOSTS: "sentseg:8011, convers-evaluation-no-scripts-selector:8002, badlisted-words:8018, combined-classification:8087, sentence-ranker:8128, prompt-selector:8135, openai-api-davinci3:8131, dff-dream-persona-davinci3-prompted-skill:8137" WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1000} @@ -38,32 +38,21 @@ services: reservations: memory: 2G - convers-evaluation-no-scripts-selector: + ranking-based-response-selector: env_file: [ .env ] build: args: - TAG_BASED_SELECTION: 1 - CALL_BY_NAME_PROBABILITY: 0.5 - PROMPT_PROBA: 0.1 - ACKNOWLEDGEMENT_PROBA: 0.3 - PRIORITIZE_WITH_REQUIRED_ACT: 0 - PRIORITIZE_NO_DIALOG_BREAKDOWN: 0 - PRIORITIZE_WITH_SAME_TOPIC_ENTITY: 0 - IGNORE_DISLIKED_SKILLS: 0 - GREETING_FIRST: 0 - RESTRICTION_FOR_SENSITIVE_CASE: 1 - PRIORITIZE_PROMTS_WHEN_NO_SCRIPTS: 0 - MAX_TURNS_WITHOUT_SCRIPTS: 7 - ADD_ACKNOWLEDGMENTS_IF_POSSIBLE: 1 - PRIORITIZE_SCRIPTED_SKILLS: 0 - CONFIDENCE_STRENGTH: 0.8 - CONV_EVAL_STRENGTH: 0.4 - PRIORITIZE_HUMAN_INITIATIVE: 1 - QUESTION_TO_QUESTION_DOWNSCORE_COEF: 0.8 + SERVICE_PORT: 8002 + SERVICE_NAME: response_selector LANGUAGE: EN + SENTENCE_RANKER_ANNOTATION_NAME: sentence_ranker + SENTENCE_RANKER_SERVICE_URL: http://sentence-ranker:8128/respond + SENTENCE_RANKER_TIMEOUT: 3 + N_UTTERANCES_CONTEXT: 5 + FILTER_TOXIC_OR_BADLISTED: 1 context: . - dockerfile: ./response_selectors/convers_evaluation_based_selector/Dockerfile - command: flask run -h 0.0.0.0 -p 8009 + dockerfile: ./response_selectors/ranking_based_response_selector/Dockerfile + command: flask run -h 0.0.0.0 -p 8002 environment: - FLASK_APP=server deploy: diff --git a/assistant_dists/dream_persona_openai_prompted/pipeline_conf.json b/assistant_dists/dream_persona_openai_prompted/pipeline_conf.json index 4c50270199..d17e62e737 100644 --- a/assistant_dists/dream_persona_openai_prompted/pipeline_conf.json +++ b/assistant_dists/dream_persona_openai_prompted/pipeline_conf.json @@ -273,7 +273,7 @@ "connector": { "protocol": "http", "timeout": 1.0, - "url": "http://convers-evaluation-no-scripts-selector:8009/respond" + "url": "http://convers-evaluation-no-scripts-selector:8002/respond" }, "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", @@ -283,7 +283,7 @@ "state_manager_method": "add_bot_utterance", "is_enabled": true, "source": { - "directory": "response_selectors/convers_evaluation_based_selector", + "directory": "response_selectors/ranking_based_response_selector", "container": "convers-evaluation-no-scripts-selector", "endpoint": "respond" } diff --git a/assistant_dists/dream_persona_prompted/dev.yml b/assistant_dists/dream_persona_prompted/dev.yml index 6d33070357..36aaaba3f2 100644 --- a/assistant_dists/dream_persona_prompted/dev.yml +++ b/assistant_dists/dream_persona_prompted/dev.yml @@ -10,12 +10,12 @@ services: - "./annotators/SentSeg:/src" ports: - 8011:8011 - gptj-based-response-selector: + ranking-based-response-selector: volumes: - - "./response_selectors/llm_based_response_selector:/src" + - "./response_selectors/ranking_based_response_selector:/src" - "./common:/src/common" ports: - - 8003:8003 + - 8002:8002 badlisted-words: volumes: - "./annotators/BadlistedWordsDetector:/src" diff --git a/assistant_dists/dream_persona_prompted/docker-compose.override.yml b/assistant_dists/dream_persona_prompted/docker-compose.override.yml index b67fe10904..d34ccaf555 100644 --- a/assistant_dists/dream_persona_prompted/docker-compose.override.yml +++ b/assistant_dists/dream_persona_prompted/docker-compose.override.yml @@ -2,7 +2,7 @@ services: agent: command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/dream_persona_prompted/pipeline_conf.json' environment: - WAIT_HOSTS: "sentseg:8011, gptj-based-response-selector:8003, badlisted-words:8018, combined-classification:8087, + WAIT_HOSTS: "sentseg:8011, ranking-based-response-selector:8002, badlisted-words:8018, combined-classification:8087, sentence-ranker:8128, prompt-selector:8135, transformers-lm-gptj:8130, dff-dream-persona-gpt-j-prompted-skill:8134" WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1000} @@ -38,21 +38,21 @@ services: reservations: memory: 2G - gptj-based-response-selector: + ranking-based-response-selector: env_file: [ .env ] build: args: - SERVICE_PORT: 8003 + SERVICE_PORT: 8002 SERVICE_NAME: response_selector LANGUAGE: EN - GENERATIVE_SERVICE_URL: http://transformers-lm-gptj:8130/respond - GENERATIVE_SERVICE_CONFIG: default_generative_config.json - GENERATIVE_TIMEOUT: 5 + SENTENCE_RANKER_ANNOTATION_NAME: sentence_ranker + SENTENCE_RANKER_SERVICE_URL: http://sentence-ranker:8128/respond + SENTENCE_RANKER_TIMEOUT: 3 N_UTTERANCES_CONTEXT: 5 FILTER_TOXIC_OR_BADLISTED: 1 context: . - dockerfile: ./response_selectors/llm_based_response_selector/Dockerfile - command: flask run -h 0.0.0.0 -p 8003 + dockerfile: ./response_selectors/ranking_based_response_selector/Dockerfile + command: flask run -h 0.0.0.0 -p 8002 environment: - FLASK_APP=server deploy: diff --git a/assistant_dists/dream_persona_prompted/pipeline_conf.json b/assistant_dists/dream_persona_prompted/pipeline_conf.json index 324861ab00..97da4b4bdc 100644 --- a/assistant_dists/dream_persona_prompted/pipeline_conf.json +++ b/assistant_dists/dream_persona_prompted/pipeline_conf.json @@ -275,7 +275,7 @@ "connector": { "protocol": "http", "timeout": 1.0, - "url": "http://gptj-based-response-selector:8003/respond" + "url": "http://ranking-based-response-selector:8002/respond" }, "dialog_formatter": "state_formatters.dp_formatters:cropped_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", @@ -285,8 +285,8 @@ "state_manager_method": "add_bot_utterance", "is_enabled": true, "source": { - "directory": "response_selectors/llm_based_response_selector", - "container": "gptj-based-response-selector", + "directory": "response_selectors/ranking_based_response_selector", + "container": "ranking-based-response-selector", "endpoint": "respond" } } diff --git a/assistant_dists/empathetic_marketing_assistant/dev.yml b/assistant_dists/empathetic_marketing_assistant/dev.yml index 06835391aa..3285930ff7 100644 --- a/assistant_dists/empathetic_marketing_assistant/dev.yml +++ b/assistant_dists/empathetic_marketing_assistant/dev.yml @@ -10,12 +10,12 @@ services: - "./annotators/SentSeg:/src" ports: - 8011:8011 - gptj-based-response-selector: + ranking-based-response-selector: volumes: - - "./response_selectors/llm_based_response_selector:/src" + - "./response_selectors/ranking_based_response_selector:/src" - "./common:/src/common" ports: - - 8003:8003 + - 8002:8002 badlisted-words: volumes: - "./annotators/BadlistedWordsDetector:/src" diff --git a/assistant_dists/empathetic_marketing_assistant/docker-compose.override.yml b/assistant_dists/empathetic_marketing_assistant/docker-compose.override.yml index 75de208ef4..3cf7a62ee0 100644 --- a/assistant_dists/empathetic_marketing_assistant/docker-compose.override.yml +++ b/assistant_dists/empathetic_marketing_assistant/docker-compose.override.yml @@ -2,7 +2,7 @@ services: agent: command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/empathetic_marketing_assistant/pipeline_conf.json' environment: - WAIT_HOSTS: "sentseg:8011, gptj-based-response-selector:8003, badlisted-words:8018, combined-classification:8087, + WAIT_HOSTS: "sentseg:8011, ranking-based-response-selector:8002, badlisted-words:8018, combined-classification:8087, sentence-ranker:8128, prompt-selector:8135, transformers-lm-gptj:8130, dff-empathetic-marketing-prompted-skill:8138" WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1000} @@ -38,21 +38,21 @@ services: reservations: memory: 2G - gptj-based-response-selector: + ranking-based-response-selector: env_file: [ .env ] build: args: - SERVICE_PORT: 8003 + SERVICE_PORT: 8002 SERVICE_NAME: response_selector LANGUAGE: EN - GENERATIVE_SERVICE_URL: http://transformers-lm-gptj:8130/respond - GENERATIVE_SERVICE_CONFIG: default_generative_config.json - GENERATIVE_TIMEOUT: 5 + SENTENCE_RANKER_ANNOTATION_NAME: sentence_ranker + SENTENCE_RANKER_SERVICE_URL: http://sentence-ranker:8128/respond + SENTENCE_RANKER_TIMEOUT: 3 N_UTTERANCES_CONTEXT: 5 FILTER_TOXIC_OR_BADLISTED: 1 context: . - dockerfile: ./response_selectors/llm_based_response_selector/Dockerfile - command: flask run -h 0.0.0.0 -p 8003 + dockerfile: ./response_selectors/ranking_based_response_selector/Dockerfile + command: flask run -h 0.0.0.0 -p 8002 environment: - FLASK_APP=server deploy: diff --git a/assistant_dists/empathetic_marketing_assistant/pipeline_conf.json b/assistant_dists/empathetic_marketing_assistant/pipeline_conf.json index 7663e8dab4..2e33ba970a 100644 --- a/assistant_dists/empathetic_marketing_assistant/pipeline_conf.json +++ b/assistant_dists/empathetic_marketing_assistant/pipeline_conf.json @@ -273,7 +273,7 @@ "connector": { "protocol": "http", "timeout": 1.0, - "url": "http://gptj-based-response-selector:8003/respond" + "url": "http://ranking-based-response-selector:8002/respond" }, "dialog_formatter": "state_formatters.dp_formatters:cropped_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", @@ -283,8 +283,8 @@ "state_manager_method": "add_bot_utterance", "is_enabled": true, "source": { - "directory": "response_selectors/llm_based_response_selector", - "container": "gptj-based-response-selector", + "directory": "response_selectors/ranking_based_response_selector", + "container": "ranking-based-response-selector", "endpoint": "respond" } } diff --git a/assistant_dists/fairytale_assistant/dev.yml b/assistant_dists/fairytale_assistant/dev.yml index 061529a4b0..f7fa23a775 100644 --- a/assistant_dists/fairytale_assistant/dev.yml +++ b/assistant_dists/fairytale_assistant/dev.yml @@ -12,10 +12,10 @@ services: - 8011:8011 no-restrictions-selector: volumes: - - "./response_selectors/convers_evaluation_based_selector:/src" + - "./response_selectors/ranking_based_response_selector:/src" - "./common:/src/common" ports: - - 8009:8009 + - 8002:8002 badlisted-words: volumes: - "./annotators/BadlistedWordsDetector:/src" diff --git a/assistant_dists/fairytale_assistant/docker-compose.override.yml b/assistant_dists/fairytale_assistant/docker-compose.override.yml index 3c3dcbffd2..44547b61ab 100644 --- a/assistant_dists/fairytale_assistant/docker-compose.override.yml +++ b/assistant_dists/fairytale_assistant/docker-compose.override.yml @@ -2,7 +2,7 @@ services: agent: command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/fairytale_assistant/pipeline_conf.json' environment: - WAIT_HOSTS: "sentseg:8011, no-restrictions-selector:8009, badlisted-words:8018, combined-classification:8087, + WAIT_HOSTS: "sentseg:8011, no-restrictions-selector:8002, badlisted-words:8018, combined-classification:8087, sentence-ranker:8128, prompt-selector:8135, openai-api-chatgpt:8145, dff-fairytale-prompted-skill:8139" WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1000} @@ -38,32 +38,21 @@ services: reservations: memory: 2G - no-restrictions-selector: + ranking-based-response-selector: env_file: [ .env ] build: args: - TAG_BASED_SELECTION: 1 - CALL_BY_NAME_PROBABILITY: 0.5 - PROMPT_PROBA: 0 - ACKNOWLEDGEMENT_PROBA: 0.3 - PRIORITIZE_WITH_REQUIRED_ACT: 0 - PRIORITIZE_NO_DIALOG_BREAKDOWN: 0 - PRIORITIZE_WITH_SAME_TOPIC_ENTITY: 0 - IGNORE_DISLIKED_SKILLS: 0 - GREETING_FIRST: 0 - RESTRICTION_FOR_SENSITIVE_CASE: 0 - PRIORITIZE_PROMTS_WHEN_NO_SCRIPTS: 0 - MAX_TURNS_WITHOUT_SCRIPTS: 100 - ADD_ACKNOWLEDGMENTS_IF_POSSIBLE: 0 - PRIORITIZE_SCRIPTED_SKILLS: 0 - CONFIDENCE_STRENGTH: 0.8 - CONV_EVAL_STRENGTH: 0.4 - PRIORITIZE_HUMAN_INITIATIVE: 1 - QUESTION_TO_QUESTION_DOWNSCORE_COEF: 0.8 + SERVICE_PORT: 8002 + SERVICE_NAME: response_selector LANGUAGE: EN + SENTENCE_RANKER_ANNOTATION_NAME: sentence_ranker + SENTENCE_RANKER_SERVICE_URL: http://sentence-ranker:8128/respond + SENTENCE_RANKER_TIMEOUT: 3 + N_UTTERANCES_CONTEXT: 5 + FILTER_TOXIC_OR_BADLISTED: 1 context: . - dockerfile: ./response_selectors/convers_evaluation_based_selector/Dockerfile - command: flask run -h 0.0.0.0 -p 8009 + dockerfile: ./response_selectors/ranking_based_response_selector/Dockerfile + command: flask run -h 0.0.0.0 -p 8002 environment: - FLASK_APP=server deploy: diff --git a/assistant_dists/fairytale_assistant/pipeline_conf.json b/assistant_dists/fairytale_assistant/pipeline_conf.json index 737a4b61da..6a1d12bf9e 100644 --- a/assistant_dists/fairytale_assistant/pipeline_conf.json +++ b/assistant_dists/fairytale_assistant/pipeline_conf.json @@ -273,7 +273,7 @@ "connector": { "protocol": "http", "timeout": 1.0, - "url": "http://no-restrictions-selector:8009/respond" + "url": "http://no-restrictions-selector:8002/respond" }, "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", @@ -283,7 +283,7 @@ "state_manager_method": "add_bot_utterance", "is_enabled": true, "source": { - "directory": "response_selectors/convers_evaluation_based_selector", + "directory": "response_selectors/ranking_based_response_selector", "container": "no-restrictions-selector", "endpoint": "respond" } diff --git a/assistant_dists/nutrition_assistant/dev.yml b/assistant_dists/nutrition_assistant/dev.yml index dbb0bc7066..93fe4c1d74 100644 --- a/assistant_dists/nutrition_assistant/dev.yml +++ b/assistant_dists/nutrition_assistant/dev.yml @@ -10,12 +10,12 @@ services: - "./annotators/SentSeg:/src" ports: - 8011:8011 - oasst12b-based-response-selector: + ranking-based-response-selector: volumes: - - "./response_selectors/llm_based_response_selector:/src" + - "./response_selectors/ranking_based_response_selector:/src" - "./common:/src/common" ports: - - 8003:8003 + - 8002:8002 badlisted-words: volumes: - "./annotators/BadlistedWordsDetector:/src" diff --git a/assistant_dists/nutrition_assistant/docker-compose.override.yml b/assistant_dists/nutrition_assistant/docker-compose.override.yml index 5545d43733..47284caf33 100644 --- a/assistant_dists/nutrition_assistant/docker-compose.override.yml +++ b/assistant_dists/nutrition_assistant/docker-compose.override.yml @@ -2,7 +2,7 @@ services: agent: command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/nutrition_assistant/pipeline_conf.json' environment: - WAIT_HOSTS: "sentseg:8011, oasst12b-based-response-selector:8003, badlisted-words:8018, combined-classification:8087, + WAIT_HOSTS: "sentseg:8011, ranking-based-response-selector:8002, badlisted-words:8018, combined-classification:8087, sentence-ranker:8128, prompt-selector:8135, transformers-lm-oasst12b:8158, dff-nutrition-prompted-skill:8141" WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1000} @@ -38,21 +38,21 @@ services: reservations: memory: 2G - oasst12b-based-response-selector: + ranking-based-response-selector: env_file: [ .env ] build: args: - SERVICE_PORT: 8003 + SERVICE_PORT: 8002 SERVICE_NAME: response_selector LANGUAGE: EN - GENERATIVE_SERVICE_URL: http://transformers-lm-oasst12b:8158/respond - GENERATIVE_SERVICE_CONFIG: default_generative_config.json - GENERATIVE_TIMEOUT: 5 + SENTENCE_RANKER_ANNOTATION_NAME: sentence_ranker + SENTENCE_RANKER_SERVICE_URL: http://sentence-ranker:8128/respond + SENTENCE_RANKER_TIMEOUT: 3 N_UTTERANCES_CONTEXT: 5 FILTER_TOXIC_OR_BADLISTED: 1 context: . - dockerfile: ./response_selectors/llm_based_response_selector/Dockerfile - command: flask run -h 0.0.0.0 -p 8003 + dockerfile: ./response_selectors/ranking_based_response_selector/Dockerfile + command: flask run -h 0.0.0.0 -p 8002 environment: - FLASK_APP=server deploy: diff --git a/assistant_dists/nutrition_assistant/pipeline_conf.json b/assistant_dists/nutrition_assistant/pipeline_conf.json index cb58ff0d5d..f94f231ae5 100644 --- a/assistant_dists/nutrition_assistant/pipeline_conf.json +++ b/assistant_dists/nutrition_assistant/pipeline_conf.json @@ -273,7 +273,7 @@ "connector": { "protocol": "http", "timeout": 1.0, - "url": "http://oasst12b-based-response-selector:8003/respond" + "url": "http://ranking-based-response-selector:8002/respond" }, "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", @@ -283,8 +283,8 @@ "state_manager_method": "add_bot_utterance", "is_enabled": true, "source": { - "directory": "response_selectors/llm_based_response_selector", - "container": "oasst12b-based-response-selector", + "directory": "response_selectors/ranking_based_response_selector", + "container": "ranking-based-response-selector", "endpoint": "respond" } } diff --git a/assistant_dists/rhodes_coaching_assistant/dev.yml b/assistant_dists/rhodes_coaching_assistant/dev.yml index 0e8c4bb0bb..5f939782f9 100644 --- a/assistant_dists/rhodes_coaching_assistant/dev.yml +++ b/assistant_dists/rhodes_coaching_assistant/dev.yml @@ -12,10 +12,10 @@ services: - 8011:8011 no-restrictions-selector: volumes: - - "./response_selectors/convers_evaluation_based_selector:/src" + - "./response_selectors/ranking_based_response_selector:/src" - "./common:/src/common" ports: - - 8009:8009 + - 8002:8002 badlisted-words: volumes: - "./annotators/BadlistedWordsDetector:/src" diff --git a/assistant_dists/rhodes_coaching_assistant/docker-compose.override.yml b/assistant_dists/rhodes_coaching_assistant/docker-compose.override.yml index d0ba0bde39..879b0d7419 100644 --- a/assistant_dists/rhodes_coaching_assistant/docker-compose.override.yml +++ b/assistant_dists/rhodes_coaching_assistant/docker-compose.override.yml @@ -2,7 +2,7 @@ services: agent: command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/rhodes_coaching_assistant/pipeline_conf.json' environment: - WAIT_HOSTS: "sentseg:8011, no-restrictions-selector:8009, badlisted-words:8018, combined-classification:8087, + WAIT_HOSTS: "sentseg:8011, no-restrictions-selector:8002, badlisted-words:8018, combined-classification:8087, sentence-ranker:8128, prompt-selector:8135, openai-api-davinci3:8131, dff-rhodes-coaching-prompted-skill:8142" WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1000} @@ -38,32 +38,21 @@ services: reservations: memory: 2G - no-restrictions-selector: + ranking-based-response-selector: env_file: [ .env ] build: args: - TAG_BASED_SELECTION: 1 - CALL_BY_NAME_PROBABILITY: 0.5 - PROMPT_PROBA: 0 - ACKNOWLEDGEMENT_PROBA: 0.3 - PRIORITIZE_WITH_REQUIRED_ACT: 0 - PRIORITIZE_NO_DIALOG_BREAKDOWN: 0 - PRIORITIZE_WITH_SAME_TOPIC_ENTITY: 0 - IGNORE_DISLIKED_SKILLS: 0 - GREETING_FIRST: 0 - RESTRICTION_FOR_SENSITIVE_CASE: 0 - PRIORITIZE_PROMTS_WHEN_NO_SCRIPTS: 0 - MAX_TURNS_WITHOUT_SCRIPTS: 100 - ADD_ACKNOWLEDGMENTS_IF_POSSIBLE: 0 - PRIORITIZE_SCRIPTED_SKILLS: 0 - CONFIDENCE_STRENGTH: 0.8 - CONV_EVAL_STRENGTH: 0.4 - PRIORITIZE_HUMAN_INITIATIVE: 1 - QUESTION_TO_QUESTION_DOWNSCORE_COEF: 0.8 + SERVICE_PORT: 8002 + SERVICE_NAME: response_selector LANGUAGE: EN + SENTENCE_RANKER_ANNOTATION_NAME: sentence_ranker + SENTENCE_RANKER_SERVICE_URL: http://sentence-ranker:8128/respond + SENTENCE_RANKER_TIMEOUT: 3 + N_UTTERANCES_CONTEXT: 5 + FILTER_TOXIC_OR_BADLISTED: 1 context: . - dockerfile: ./response_selectors/convers_evaluation_based_selector/Dockerfile - command: flask run -h 0.0.0.0 -p 8009 + dockerfile: ./response_selectors/ranking_based_response_selector/Dockerfile + command: flask run -h 0.0.0.0 -p 8002 environment: - FLASK_APP=server deploy: diff --git a/assistant_dists/rhodes_coaching_assistant/pipeline_conf.json b/assistant_dists/rhodes_coaching_assistant/pipeline_conf.json index 1b53d7f204..33536a030b 100644 --- a/assistant_dists/rhodes_coaching_assistant/pipeline_conf.json +++ b/assistant_dists/rhodes_coaching_assistant/pipeline_conf.json @@ -273,7 +273,7 @@ "connector": { "protocol": "http", "timeout": 1.0, - "url": "http://no-restrictions-selector:8009/respond" + "url": "http://no-restrictions-selector:8002/respond" }, "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", @@ -283,7 +283,7 @@ "state_manager_method": "add_bot_utterance", "is_enabled": true, "source": { - "directory": "response_selectors/convers_evaluation_based_selector", + "directory": "response_selectors/ranking_based_response_selector", "container": "no-restrictions-selector", "endpoint": "respond" } diff --git a/assistant_dists/universal_prompted_assistant/dev.yml b/assistant_dists/universal_prompted_assistant/dev.yml index e7bd16de45..fa089a620c 100644 --- a/assistant_dists/universal_prompted_assistant/dev.yml +++ b/assistant_dists/universal_prompted_assistant/dev.yml @@ -10,12 +10,12 @@ services: - "./annotators/SentSeg:/src" ports: - 8011:8011 - oasst12b-based-response-selector: + ranking-based-response-selector: volumes: - - "./response_selectors/llm_based_response_selector:/src" + - "./response_selectors/ranking_based_response_selector:/src" - "./common:/src/common" ports: - - 8003:8003 + - 8002:8002 badlisted-words: volumes: - "./annotators/BadlistedWordsDetector:/src" diff --git a/assistant_dists/universal_prompted_assistant/docker-compose.override.yml b/assistant_dists/universal_prompted_assistant/docker-compose.override.yml index 61058e7a9e..ef9480ad7f 100644 --- a/assistant_dists/universal_prompted_assistant/docker-compose.override.yml +++ b/assistant_dists/universal_prompted_assistant/docker-compose.override.yml @@ -2,7 +2,7 @@ services: agent: command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/universal_prompted_assistant/pipeline_conf.json' environment: - WAIT_HOSTS: "sentseg:8011, oasst12b-based-response-selector:8003, badlisted-words:8018, combined-classification:8087, + WAIT_HOSTS: "sentseg:8011, ranking-based-response-selector:8002, badlisted-words:8018, combined-classification:8087, sentence-ranker:8128, transformers-lm-gptj:8130, transformers-lm-oasst12b:8158, openai-api-chatgpt:8145, openai-api-davinci3:8131, dff-universal-prompted-skill:8147" @@ -39,21 +39,21 @@ services: reservations: memory: 2G - oasst12b-based-response-selector: + ranking-based-response-selector: env_file: [ .env ] build: args: - SERVICE_PORT: 8003 + SERVICE_PORT: 8002 SERVICE_NAME: response_selector LANGUAGE: EN - GENERATIVE_SERVICE_URL: http://transformers-lm-oasst12b:8158/respond - GENERATIVE_SERVICE_CONFIG: default_generative_config.json - GENERATIVE_TIMEOUT: 5 + SENTENCE_RANKER_ANNOTATION_NAME: sentence_ranker + SENTENCE_RANKER_SERVICE_URL: http://sentence-ranker:8128/respond + SENTENCE_RANKER_TIMEOUT: 3 N_UTTERANCES_CONTEXT: 5 FILTER_TOXIC_OR_BADLISTED: 1 context: . - dockerfile: ./response_selectors/llm_based_response_selector/Dockerfile - command: flask run -h 0.0.0.0 -p 8003 + dockerfile: ./response_selectors/ranking_based_response_selector/Dockerfile + command: flask run -h 0.0.0.0 -p 8002 environment: - FLASK_APP=server deploy: diff --git a/assistant_dists/universal_prompted_assistant/pipeline_conf.json b/assistant_dists/universal_prompted_assistant/pipeline_conf.json index f493c2ca70..a9508657e5 100644 --- a/assistant_dists/universal_prompted_assistant/pipeline_conf.json +++ b/assistant_dists/universal_prompted_assistant/pipeline_conf.json @@ -255,7 +255,7 @@ "connector": { "protocol": "http", "timeout": 1.0, - "url": "http://oasst12b-based-response-selector:8003/respond" + "url": "http://ranking-based-response-selector:8002/respond" }, "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", @@ -265,8 +265,8 @@ "state_manager_method": "add_bot_utterance", "is_enabled": true, "source": { - "directory": "response_selectors/convers_evaluation_based_selector", - "container": "oasst12b-based-response-selector", + "directory": "response_selectors/ranking_based_response_selector", + "container": "ranking-based-response-selector", "endpoint": "respond" } } diff --git a/response_selectors/ranking_based_response_selector/component.yml b/response_selectors/ranking_based_response_selector/component.yml index d56c73d79a..62e8ad349d 100644 --- a/response_selectors/ranking_based_response_selector/component.yml +++ b/response_selectors/ranking_based_response_selector/component.yml @@ -1,7 +1,7 @@ -ranking-based-response-selector-en-service: +ranking-based-response-selector-service: name: response_selector display_name: Response Selector - container_name: ranking-based-response-selector-en-service + container_name: ranking-based-response-selector-service component_type: null model_type: Dictionary/Pattern-based is_customizable: false @@ -72,4 +72,30 @@ ranking-based-response-selector-ru: LANGUAGE: RU SENTENCE_RANKER_ANNOTATION_NAME: dialogrpt FILTER_TOXIC_OR_BADLISTED: 1 + date_created: '2023-04-24T09:45:32' +ranking-based-response-selector: + name: response_selector + display_name: Response Selector + container_name: ranking-based-response-selector + component_type: null + model_type: Dictionary/Pattern-based + is_customizable: false + author: DeepPavlov + description: Algorithm that selects a final responses among the given list of candidate + responses via the given ranking service + ram_usage: 100M + gpu_usage: null + port: 8002 + endpoints: + - group: response_selectors + endpoint: respond + build_args: + SERVICE_PORT: 8002 + SERVICE_NAME: response_selector + LANGUAGE: EN + SENTENCE_RANKER_ANNOTATION_NAME: sentence_ranker + SENTENCE_RANKER_SERVICE_URL: http://sentence-ranker:8128/respond + SENTENCE_RANKER_TIMEOUT: 3 + N_UTTERANCES_CONTEXT: 5 + FILTER_TOXIC_OR_BADLISTED: 1 date_created: '2023-04-24T09:45:32' \ No newline at end of file diff --git a/response_selectors/ranking_based_response_selector/pipeline.yml b/response_selectors/ranking_based_response_selector/pipeline.yml index c08e53f692..8d922583ea 100644 --- a/response_selectors/ranking_based_response_selector/pipeline.yml +++ b/response_selectors/ranking_based_response_selector/pipeline.yml @@ -1,8 +1,8 @@ -ranking-based-response-selector-en-service: +ranking-based-response-selector-service: - group: response_selectors connector: protocol: http - url: http://ranking-based-response-selector-en-service:8002/respond + url: http://ranking-based-response-selector-service:8002/respond dialog_formatter: state_formatters.dp_formatters:cropped_dialog response_formatter: state_formatters.dp_formatters:base_response_selector_formatter_service previous_services: @@ -28,3 +28,13 @@ ranking-based-response-selector-ru: previous_services: - response_annotators state_manager_method: add_bot_utterance +ranking-based-response-selector: +- group: response_selectors + connector: + protocol: http + url: http://ranking-based-response-selector:8002/respond + dialog_formatter: state_formatters.dp_formatters:cropped_dialog + response_formatter: state_formatters.dp_formatters:base_response_selector_formatter_service + previous_services: + - response_annotators + state_manager_method: add_bot_utterance From 115e83a046dc506e101aeee675fe6f4015c920c3 Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Mon, 24 Apr 2023 18:43:40 +0300 Subject: [PATCH 18/32] fix: decay dummy --- response_selectors/ranking_based_response_selector/server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/response_selectors/ranking_based_response_selector/server.py b/response_selectors/ranking_based_response_selector/server.py index 91bb459d14..9c504890a1 100644 --- a/response_selectors/ranking_based_response_selector/server.py +++ b/response_selectors/ranking_based_response_selector/server.py @@ -70,6 +70,7 @@ def get_scores(dialog_context, hypotheses): def select_response(dialog_context, hypotheses): scores = get_scores(dialog_context, hypotheses) + scores = [score if hyp["skill_name"] != "dummy_skill" else scores / 10 for score, hyp in zip(scores, hypotheses)] result = select_response_by_scores(hypotheses, scores)[0] logger.info(f"ranking_based_response_selector selected:\n`{result}`") From effe87ed916be700bd54ef47a2eeefcdc7b0705e Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Mon, 24 Apr 2023 18:46:15 +0300 Subject: [PATCH 19/32] fix: decay dummy --- response_selectors/ranking_based_response_selector/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/response_selectors/ranking_based_response_selector/server.py b/response_selectors/ranking_based_response_selector/server.py index 9c504890a1..6eae50937c 100644 --- a/response_selectors/ranking_based_response_selector/server.py +++ b/response_selectors/ranking_based_response_selector/server.py @@ -70,7 +70,7 @@ def get_scores(dialog_context, hypotheses): def select_response(dialog_context, hypotheses): scores = get_scores(dialog_context, hypotheses) - scores = [score if hyp["skill_name"] != "dummy_skill" else scores / 10 for score, hyp in zip(scores, hypotheses)] + scores = [score if hyp["skill_name"] != "dummy_skill" else score / 10 for score, hyp in zip(scores, hypotheses)] result = select_response_by_scores(hypotheses, scores)[0] logger.info(f"ranking_based_response_selector selected:\n`{result}`") From 0ad3e75e8b27df517ae7d7ac82a95fd298bc13ce Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Mon, 24 Apr 2023 18:49:37 +0300 Subject: [PATCH 20/32] fix: response --- response_selectors/llm_based_response_selector/server.py | 4 ++-- .../ranking_based_response_selector/server.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/response_selectors/llm_based_response_selector/server.py b/response_selectors/llm_based_response_selector/server.py index 4ba7b69a04..a2132634f0 100644 --- a/response_selectors/llm_based_response_selector/server.py +++ b/response_selectors/llm_based_response_selector/server.py @@ -100,7 +100,7 @@ def respond(): selected_resp = select_response(dialog_context, hypotheses) try: best_id = hypotheses.index(selected_resp) - selected_responses.append(selected_resp) + selected_responses.append(selected_resp["text"]) selected_skill_names.append(hypotheses[best_id]["skill_name"]) selected_confidences.append(hypotheses[best_id]["confidence"]) except Exception as e: @@ -109,7 +109,7 @@ def respond(): logger.info("Exception in finding selected response in hypotheses. " "Selected a response with the highest confidence.") selected_resp, best_id = select_response_by_scores(hypotheses, [hyp["confidence"] for hyp in hypotheses]) - selected_responses.append(selected_resp) + selected_responses.append(selected_resp["text"]) selected_skill_names.append(hypotheses[best_id]["skill_name"]) selected_confidences.append(hypotheses[best_id]["confidence"]) diff --git a/response_selectors/ranking_based_response_selector/server.py b/response_selectors/ranking_based_response_selector/server.py index 6eae50937c..a64f365ad9 100644 --- a/response_selectors/ranking_based_response_selector/server.py +++ b/response_selectors/ranking_based_response_selector/server.py @@ -70,7 +70,7 @@ def get_scores(dialog_context, hypotheses): def select_response(dialog_context, hypotheses): scores = get_scores(dialog_context, hypotheses) - scores = [score if hyp["skill_name"] != "dummy_skill" else score / 10 for score, hyp in zip(scores, hypotheses)] + scores = [score if hyp["skill_name"] != "dummy_skill" else score / 100 for score, hyp in zip(scores, hypotheses)] result = select_response_by_scores(hypotheses, scores)[0] logger.info(f"ranking_based_response_selector selected:\n`{result}`") @@ -96,7 +96,7 @@ def respond(): selected_resp = select_response(dialog_context, hypotheses) try: best_id = hypotheses.index(selected_resp) - selected_responses.append(selected_resp) + selected_responses.append(selected_resp["text"]) selected_skill_names.append(hypotheses[best_id]["skill_name"]) selected_confidences.append(hypotheses[best_id]["confidence"]) except Exception as e: @@ -105,7 +105,7 @@ def respond(): logger.info("Exception in finding selected response in hypotheses. " "Selected a response with the highest confidence.") selected_resp, best_id = select_response_by_scores(hypotheses, [hyp["confidence"] for hyp in hypotheses]) - selected_responses.append(selected_resp) + selected_responses.append(selected_resp["text"]) selected_skill_names.append(hypotheses[best_id]["skill_name"]) selected_confidences.append(hypotheses[best_id]["confidence"]) From 072e4a837e736bdef35595a15b13db9d1cedcd5d Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Mon, 24 Apr 2023 18:55:35 +0300 Subject: [PATCH 21/32] fix: remove old selector --- assistant_dists/deeppavlov_assistant/dev.yml | 2 +- .../deeppavlov_assistant/docker-compose.override.yml | 2 +- assistant_dists/deeppavlov_assistant/pipeline_conf.json | 4 ++-- assistant_dists/fairytale_assistant/dev.yml | 2 +- .../fairytale_assistant/docker-compose.override.yml | 2 +- assistant_dists/fairytale_assistant/pipeline_conf.json | 4 ++-- assistant_dists/rhodes_coaching_assistant/dev.yml | 2 +- .../rhodes_coaching_assistant/docker-compose.override.yml | 2 +- assistant_dists/rhodes_coaching_assistant/pipeline_conf.json | 4 ++-- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/assistant_dists/deeppavlov_assistant/dev.yml b/assistant_dists/deeppavlov_assistant/dev.yml index d9b16338c8..b25d085d99 100644 --- a/assistant_dists/deeppavlov_assistant/dev.yml +++ b/assistant_dists/deeppavlov_assistant/dev.yml @@ -5,7 +5,7 @@ services: - ".:/dp-agent" ports: - 4242:4242 - no-restrictions-selector: + ranking-based-response-selector: volumes: - "./response_selectors/ranking_based_response_selector:/src" - "./common:/src/common" diff --git a/assistant_dists/deeppavlov_assistant/docker-compose.override.yml b/assistant_dists/deeppavlov_assistant/docker-compose.override.yml index aa483f327e..551e09a94a 100644 --- a/assistant_dists/deeppavlov_assistant/docker-compose.override.yml +++ b/assistant_dists/deeppavlov_assistant/docker-compose.override.yml @@ -2,7 +2,7 @@ services: agent: command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/deeppavlov_assistant/pipeline_conf.json' environment: - WAIT_HOSTS: "no-restrictions-selector:8002, sentence-ranker:8128, prompt-selector:8135, openai-api-chatgpt:8145, + WAIT_HOSTS: "ranking-based-response-selector:8002, sentence-ranker:8128, prompt-selector:8135, openai-api-chatgpt:8145, dff-deeppavlov-prompted-skill:8146" WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1000} HIGH_PRIORITY_INTENTS: 1 diff --git a/assistant_dists/deeppavlov_assistant/pipeline_conf.json b/assistant_dists/deeppavlov_assistant/pipeline_conf.json index 4dc919c2fe..a9c962866a 100644 --- a/assistant_dists/deeppavlov_assistant/pipeline_conf.json +++ b/assistant_dists/deeppavlov_assistant/pipeline_conf.json @@ -159,7 +159,7 @@ "connector": { "protocol": "http", "timeout": 1.0, - "url": "http://no-restrictions-selector:8002/respond" + "url": "http://ranking-based-response-selector:8002/respond" }, "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", @@ -170,7 +170,7 @@ "is_enabled": true, "source": { "directory": "response_selectors/ranking_based_response_selector", - "container": "no-restrictions-selector", + "container": "ranking-based-response-selector", "endpoint": "respond" } } diff --git a/assistant_dists/fairytale_assistant/dev.yml b/assistant_dists/fairytale_assistant/dev.yml index f7fa23a775..79d58e8ef1 100644 --- a/assistant_dists/fairytale_assistant/dev.yml +++ b/assistant_dists/fairytale_assistant/dev.yml @@ -10,7 +10,7 @@ services: - "./annotators/SentSeg:/src" ports: - 8011:8011 - no-restrictions-selector: + ranking-based-response-selector: volumes: - "./response_selectors/ranking_based_response_selector:/src" - "./common:/src/common" diff --git a/assistant_dists/fairytale_assistant/docker-compose.override.yml b/assistant_dists/fairytale_assistant/docker-compose.override.yml index 44547b61ab..173b23034d 100644 --- a/assistant_dists/fairytale_assistant/docker-compose.override.yml +++ b/assistant_dists/fairytale_assistant/docker-compose.override.yml @@ -2,7 +2,7 @@ services: agent: command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/fairytale_assistant/pipeline_conf.json' environment: - WAIT_HOSTS: "sentseg:8011, no-restrictions-selector:8002, badlisted-words:8018, combined-classification:8087, + WAIT_HOSTS: "sentseg:8011, ranking-based-response-selector:8002, badlisted-words:8018, combined-classification:8087, sentence-ranker:8128, prompt-selector:8135, openai-api-chatgpt:8145, dff-fairytale-prompted-skill:8139" WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1000} diff --git a/assistant_dists/fairytale_assistant/pipeline_conf.json b/assistant_dists/fairytale_assistant/pipeline_conf.json index 6a1d12bf9e..b2e41629cd 100644 --- a/assistant_dists/fairytale_assistant/pipeline_conf.json +++ b/assistant_dists/fairytale_assistant/pipeline_conf.json @@ -273,7 +273,7 @@ "connector": { "protocol": "http", "timeout": 1.0, - "url": "http://no-restrictions-selector:8002/respond" + "url": "http://ranking-based-response-selector:8002/respond" }, "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", @@ -284,7 +284,7 @@ "is_enabled": true, "source": { "directory": "response_selectors/ranking_based_response_selector", - "container": "no-restrictions-selector", + "container": "ranking-based-response-selector", "endpoint": "respond" } } diff --git a/assistant_dists/rhodes_coaching_assistant/dev.yml b/assistant_dists/rhodes_coaching_assistant/dev.yml index 5f939782f9..f2d8f9e9ac 100644 --- a/assistant_dists/rhodes_coaching_assistant/dev.yml +++ b/assistant_dists/rhodes_coaching_assistant/dev.yml @@ -10,7 +10,7 @@ services: - "./annotators/SentSeg:/src" ports: - 8011:8011 - no-restrictions-selector: + ranking-based-response-selector: volumes: - "./response_selectors/ranking_based_response_selector:/src" - "./common:/src/common" diff --git a/assistant_dists/rhodes_coaching_assistant/docker-compose.override.yml b/assistant_dists/rhodes_coaching_assistant/docker-compose.override.yml index 879b0d7419..49bd51600d 100644 --- a/assistant_dists/rhodes_coaching_assistant/docker-compose.override.yml +++ b/assistant_dists/rhodes_coaching_assistant/docker-compose.override.yml @@ -2,7 +2,7 @@ services: agent: command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/rhodes_coaching_assistant/pipeline_conf.json' environment: - WAIT_HOSTS: "sentseg:8011, no-restrictions-selector:8002, badlisted-words:8018, combined-classification:8087, + WAIT_HOSTS: "sentseg:8011, ranking-based-response-selector:8002, badlisted-words:8018, combined-classification:8087, sentence-ranker:8128, prompt-selector:8135, openai-api-davinci3:8131, dff-rhodes-coaching-prompted-skill:8142" WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1000} diff --git a/assistant_dists/rhodes_coaching_assistant/pipeline_conf.json b/assistant_dists/rhodes_coaching_assistant/pipeline_conf.json index 33536a030b..094f997529 100644 --- a/assistant_dists/rhodes_coaching_assistant/pipeline_conf.json +++ b/assistant_dists/rhodes_coaching_assistant/pipeline_conf.json @@ -273,7 +273,7 @@ "connector": { "protocol": "http", "timeout": 1.0, - "url": "http://no-restrictions-selector:8002/respond" + "url": "http://ranking-based-response-selector:8002/respond" }, "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", @@ -284,7 +284,7 @@ "is_enabled": true, "source": { "directory": "response_selectors/ranking_based_response_selector", - "container": "no-restrictions-selector", + "container": "ranking-based-response-selector", "endpoint": "respond" } } From 90814098b1b7497675c608d2dbcb8abb083d7f30 Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Mon, 24 Apr 2023 20:14:42 +0300 Subject: [PATCH 22/32] fix: logs --- response_selectors/ranking_based_response_selector/server.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/response_selectors/ranking_based_response_selector/server.py b/response_selectors/ranking_based_response_selector/server.py index a64f365ad9..8321281766 100644 --- a/response_selectors/ranking_based_response_selector/server.py +++ b/response_selectors/ranking_based_response_selector/server.py @@ -35,6 +35,8 @@ def filter_out_badlisted_or_toxic(hypotheses): is_toxic = is_toxic_or_badlisted_utterance(hyp) if not is_toxic: clean_hypotheses += [deepcopy(hyp)] + else: + logger.info(f"Filter out toxic candidate: {hyp['text']}") return clean_hypotheses @@ -71,6 +73,7 @@ def select_response(dialog_context, hypotheses): scores = get_scores(dialog_context, hypotheses) scores = [score if hyp["skill_name"] != "dummy_skill" else score / 100 for score, hyp in zip(scores, hypotheses)] + logger.info(f"Scores for selection:\n`{scores}`") result = select_response_by_scores(hypotheses, scores)[0] logger.info(f"ranking_based_response_selector selected:\n`{result}`") From 5b2f3b4de6fc96c1ab0daf64b3493c1cb56043a3 Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Mon, 24 Apr 2023 20:16:27 +0300 Subject: [PATCH 23/32] fix: getting annotations --- response_selectors/ranking_based_response_selector/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/response_selectors/ranking_based_response_selector/server.py b/response_selectors/ranking_based_response_selector/server.py index 8321281766..1ab0aa44ac 100644 --- a/response_selectors/ranking_based_response_selector/server.py +++ b/response_selectors/ranking_based_response_selector/server.py @@ -48,7 +48,7 @@ def select_response_by_scores(hypotheses, scores): def get_scores(dialog_context, hypotheses): if all([SENTENCE_RANKER_ANNOTATION_NAME in hyp.get("annotations", {}) for hyp in hypotheses]): - scores = [hyp.get("annotations", {}).get("SENTENCE_RANKER_ANNOTATION_NAME", 0.0) for hyp in hypotheses] + scores = [hyp.get("annotations", {}).get(SENTENCE_RANKER_ANNOTATION_NAME, 0.0) for hyp in hypotheses] logger.info(f"Selected a response via Sentence Ranker Annotator.") else: try: From 9914ac03aa73e0eb8a5da0ae1929a62f7117d2aa Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Mon, 24 Apr 2023 20:18:58 +0300 Subject: [PATCH 24/32] fix: code style --- .../llm_based_response_selector/server.py | 12 ++++++++---- .../ranking_based_response_selector/server.py | 6 ++++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/response_selectors/llm_based_response_selector/server.py b/response_selectors/llm_based_response_selector/server.py index a2132634f0..20222cf270 100644 --- a/response_selectors/llm_based_response_selector/server.py +++ b/response_selectors/llm_based_response_selector/server.py @@ -30,8 +30,10 @@ FILTER_TOXIC_OR_BADLISTED = int(getenv("FILTER_TOXIC_OR_BADLISTED")) N_UTTERANCES_CONTEXT = int(getenv("N_UTTERANCES_CONTEXT")) CRITERION = getenv("CRITERION", DEFAULT_CRITERION) -PROMPT = f"""Select {CRITERION} response among the hypotheses to the given dialog context. """ \ - """Return only the selected response without extra explanations.""" +PROMPT = ( + f"""Select {CRITERION} response among the hypotheses to the given dialog context. """ + """Return only the selected response without extra explanations.""" +) ENVVARS_TO_SEND = getenv("ENVVARS_TO_SEND", None) ENVVARS_TO_SEND = [] if ENVVARS_TO_SEND is None else ENVVARS_TO_SEND.split(",") sending_variables = {f"{var}_list": [getenv(var, None)] for var in ENVVARS_TO_SEND} @@ -106,8 +108,10 @@ def respond(): except Exception as e: sentry_sdk.capture_exception(e) logger.exception(e) - logger.info("Exception in finding selected response in hypotheses. " - "Selected a response with the highest confidence.") + logger.info( + "Exception in finding selected response in hypotheses. " + "Selected a response with the highest confidence." + ) selected_resp, best_id = select_response_by_scores(hypotheses, [hyp["confidence"] for hyp in hypotheses]) selected_responses.append(selected_resp["text"]) selected_skill_names.append(hypotheses[best_id]["skill_name"]) diff --git a/response_selectors/ranking_based_response_selector/server.py b/response_selectors/ranking_based_response_selector/server.py index 1ab0aa44ac..6624603b3d 100644 --- a/response_selectors/ranking_based_response_selector/server.py +++ b/response_selectors/ranking_based_response_selector/server.py @@ -105,8 +105,10 @@ def respond(): except Exception as e: sentry_sdk.capture_exception(e) logger.exception(e) - logger.info("Exception in finding selected response in hypotheses. " - "Selected a response with the highest confidence.") + logger.info( + "Exception in finding selected response in hypotheses. " + "Selected a response with the highest confidence." + ) selected_resp, best_id = select_response_by_scores(hypotheses, [hyp["confidence"] for hyp in hypotheses]) selected_responses.append(selected_resp["text"]) selected_skill_names.append(hypotheses[best_id]["skill_name"]) From 4f1823e5d4d22e3ee7ba42829027c429f5b2f976 Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Mon, 24 Apr 2023 20:24:34 +0300 Subject: [PATCH 25/32] fix: deepy uses resp selector --- assistant_dists/deepy_assistant/cpu.yml | 4 - assistant_dists/deepy_assistant/dev.yml | 28 +--- .../docker-compose.override.yml | 105 ++----------- .../deepy_assistant/pipeline_conf.json | 139 +----------------- assistant_dists/deepy_assistant/proxy.yml | 36 ----- 5 files changed, 22 insertions(+), 290 deletions(-) diff --git a/assistant_dists/deepy_assistant/cpu.yml b/assistant_dists/deepy_assistant/cpu.yml index 3362e09ae0..0a91edfc65 100644 --- a/assistant_dists/deepy_assistant/cpu.yml +++ b/assistant_dists/deepy_assistant/cpu.yml @@ -1,9 +1,5 @@ version: '3.7' services: - combined-classification: - environment: - DEVICE: cpu - CUDA_VISIBLE_DEVICES: "" sentence-ranker: environment: DEVICE: cpu diff --git a/assistant_dists/deepy_assistant/dev.yml b/assistant_dists/deepy_assistant/dev.yml index 43c42be2b9..5d1ff5fb6a 100644 --- a/assistant_dists/deepy_assistant/dev.yml +++ b/assistant_dists/deepy_assistant/dev.yml @@ -5,34 +5,12 @@ services: - ".:/dp-agent" ports: - 4242:4242 - sentseg: - volumes: - - "./annotators/SentSeg:/src" - ports: - - 8011:8011 - no-restrictions-selector: - volumes: - - "./response_selectors/convers_evaluation_based_selector:/src" - - "./common:/src/common" - ports: - - 8009:8009 - badlisted-words: - volumes: - - "./annotators/BadlistedWordsDetector:/src" - - "./common:/src/common" - ports: - - 8018:8018 - spelling-preprocessing: - volumes: - - "./annotators/spelling_preprocessing:/src" - ports: - - 8074:8074 - combined-classification: + ranking-based-response-selector: volumes: + - "./response_selectors/ranking_based_response_selector:/src" - "./common:/src/common" - - "./annotators/combined_classification:/src" ports: - - 8087:8087 + - 8002:8002 sentence-ranker: volumes: - "./services/sentence_ranker:/src" diff --git a/assistant_dists/deepy_assistant/docker-compose.override.yml b/assistant_dists/deepy_assistant/docker-compose.override.yml index 82e981d5e4..d6e94b2add 100644 --- a/assistant_dists/deepy_assistant/docker-compose.override.yml +++ b/assistant_dists/deepy_assistant/docker-compose.override.yml @@ -2,102 +2,29 @@ services: agent: command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/deepy_assistant/pipeline_conf.json' environment: - WAIT_HOSTS: "sentseg:8011, no-restrictions-selector:8009, badlisted-words:8018, combined-classification:8087, - spelling-preprocessing:8074, sentence-ranker:8128, prompt-selector:8135, openai-api-davinci3:8131, - dff-deepy-prompted-skill:8148" + WAIT_HOSTS: "ranking-based-response-selector:8002, sentence-ranker:8128, prompt-selector:8135, openai-api-davinci3:8131, + dff-deepy-prompted-skill:8148" WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1000} + HIGH_PRIORITY_INTENTS: 1 + RESTRICTION_FOR_SENSITIVE_CASE: 1 + ALWAYS_TURN_ON_ALL_SKILLS: 0 + LANGUAGE: EN - sentseg: - env_file: [ .env ] - build: - context: ./annotators/SentSeg/ - command: flask run -h 0.0.0.0 -p 8011 - environment: - - FLASK_APP=server - deploy: - resources: - limits: - memory: 1.5G - reservations: - memory: 1.5G - - combined-classification: + ranking-based-response-selector: env_file: [ .env ] build: args: - CONFIG: combined_classifier.json - SERVICE_PORT: 8087 - context: . - dockerfile: ./annotators/combined_classification/Dockerfile - environment: - - CUDA_VISIBLE_DEVICES=0 - deploy: - resources: - limits: - memory: 2G - reservations: - memory: 2G - - no-restrictions-selector: - env_file: [ .env ] - build: - args: - TAG_BASED_SELECTION: 1 - CALL_BY_NAME_PROBABILITY: 0.5 - PROMPT_PROBA: 0 - ACKNOWLEDGEMENT_PROBA: 0.3 - PRIORITIZE_WITH_REQUIRED_ACT: 0 - PRIORITIZE_NO_DIALOG_BREAKDOWN: 0 - PRIORITIZE_WITH_SAME_TOPIC_ENTITY: 0 - IGNORE_DISLIKED_SKILLS: 0 - GREETING_FIRST: 0 - RESTRICTION_FOR_SENSITIVE_CASE: 0 - PRIORITIZE_PROMTS_WHEN_NO_SCRIPTS: 0 - MAX_TURNS_WITHOUT_SCRIPTS: 100 - ADD_ACKNOWLEDGMENTS_IF_POSSIBLE: 0 - PRIORITIZE_SCRIPTED_SKILLS: 0 - CONFIDENCE_STRENGTH: 0.8 - CONV_EVAL_STRENGTH: 0.4 - PRIORITIZE_HUMAN_INITIATIVE: 1 - QUESTION_TO_QUESTION_DOWNSCORE_COEF: 0.8 + SERVICE_PORT: 8002 + SERVICE_NAME: response_selector LANGUAGE: EN + SENTENCE_RANKER_ANNOTATION_NAME: sentence_ranker + SENTENCE_RANKER_SERVICE_URL: http://sentence-ranker:8128/respond + SENTENCE_RANKER_TIMEOUT: 3 + N_UTTERANCES_CONTEXT: 5 + FILTER_TOXIC_OR_BADLISTED: 1 context: . - dockerfile: ./response_selectors/convers_evaluation_based_selector/Dockerfile - command: flask run -h 0.0.0.0 -p 8009 - environment: - - FLASK_APP=server - deploy: - resources: - limits: - memory: 100M - reservations: - memory: 100M - - badlisted-words: - env_file: [ .env ] - build: - args: - SERVICE_PORT: 8018 - SERVICE_NAME: badlisted_words - context: annotators/BadlistedWordsDetector/ - command: flask run -h 0.0.0.0 -p 8018 - environment: - - FLASK_APP=server - deploy: - resources: - limits: - memory: 256M - reservations: - memory: 256M - - spelling-preprocessing: - env_file: [ .env ] - build: - args: - SERVICE_PORT: 8074 - SERVICE_NAME: spelling_preprocessing - context: ./annotators/spelling_preprocessing/ - command: flask run -h 0.0.0.0 -p 8074 + dockerfile: ./response_selectors/ranking_based_response_selector/Dockerfile + command: flask run -h 0.0.0.0 -p 8002 environment: - FLASK_APP=server deploy: diff --git a/assistant_dists/deepy_assistant/pipeline_conf.json b/assistant_dists/deepy_assistant/pipeline_conf.json index f09684cab1..a1ee3f2eca 100644 --- a/assistant_dists/deepy_assistant/pipeline_conf.json +++ b/assistant_dists/deepy_assistant/pipeline_conf.json @@ -1,11 +1,4 @@ { - "connectors": { - "sentseg": { - "protocol": "http", - "timeout": 1.5, - "url": "http://sentseg:8011/sentseg" - } - }, "services": { "last_chance_service": { "connector": { @@ -13,13 +6,6 @@ "class_name": "PredefinedTextConnector", "response_text": "Sorry, something went wrong inside. Please tell me, what did you say.", "annotations": { - "sentseg": { - "punct_sent": "Sorry, something went wrong inside. Please tell me, what did you say.", - "segments": [ - "Sorry, something went wrong inside.", - "Please tell me, what did you say." - ] - } } }, "state_manager_method": "add_bot_utterance_last_chance", @@ -37,13 +23,6 @@ "class_name": "PredefinedTextConnector", "response_text": "Sorry, I need to think more on that. Let's talk about something else.", "annotations": { - "sentseg": { - "punct_sent": "Sorry, I need to think more on that. Let's talk about something else.", - "segments": [ - "Sorry, I need to think more on that.", - "Let's talk about something else." - ] - } } }, "state_manager_method": "add_bot_utterance_last_chance", @@ -74,81 +53,8 @@ } }, "response_annotators": { - "sentseg": { - "connector": { - "protocol": "http", - "timeout": 1.5, - "url": "http://sentseg:8011/sentseg" - }, - "dialog_formatter": "state_formatters.dp_formatters:last_bot_utt_dialog", - "response_formatter": "state_formatters.dp_formatters:simple_formatter_service", - "previous_services": [ - "response_annotator_selectors" - ], - "state_manager_method": "add_annotation_prev_bot_utt", - "is_enabled": true, - "source": { - "directory": "annotators/SentSeg", - "container": "sentseg", - "endpoint": "sentseg" - } - } }, "annotators": { - "spelling_preprocessing": { - "connector": { - "protocol": "http", - "timeout": 1.0, - "url": "http://spelling-preprocessing:8074/respond" - }, - "dialog_formatter": "state_formatters.dp_formatters:last_utt_dialog", - "response_formatter": "state_formatters.dp_formatters:simple_formatter_service", - "state_manager_method": "add_annotation_and_reset_human_attributes_for_first_turn", - "is_enabled": true, - "source": { - "directory": "annotators/spelling_preprocessing", - "container": "spelling-preprocessing", - "endpoint": "respond" - } - }, - "sentseg": { - "connector": { - "protocol": "http", - "timeout": 1.5, - "url": "http://sentseg:8011/sentseg" - }, - "dialog_formatter": "state_formatters.dp_formatters:preproc_last_human_utt_dialog", - "response_formatter": "state_formatters.dp_formatters:simple_formatter_service", - "previous_services": [ - "annotators.spelling_preprocessing" - ], - "state_manager_method": "add_annotation", - "is_enabled": true, - "source": { - "directory": "annotators/SentSeg", - "container": "sentseg", - "endpoint": "sentseg" - } - }, - "badlisted_words": { - "connector": { - "protocol": "http", - "timeout": 1.0, - "url": "http://badlisted-words:8018/badlisted_words" - }, - "dialog_formatter": "state_formatters.dp_formatters:preproc_last_human_utt_dialog", - "response_formatter": "state_formatters.dp_formatters:simple_formatter_service", - "previous_services": [ - "annotators.spelling_preprocessing" - ], - "state_manager_method": "add_annotation", - "is_enabled": true, - "source": { - "directory": "annotators/BadlistedWordsDetector", - "container": "badlisted-words", - "endpoint": "badlisted_words" - } - }, "prompt_selector": { "connector": { "protocol": "http", @@ -158,7 +64,6 @@ "dialog_formatter": "state_formatters.dp_formatters:context_formatter_dialog", "response_formatter": "state_formatters.dp_formatters:simple_formatter_service", "previous_services": [ - "annotators.spelling_preprocessing" ], "state_manager_method": "add_annotation", "is_enabled": true, @@ -229,44 +134,6 @@ } }, "candidate_annotators": { - "badlisted_words": { - "connector": { - "protocol": "http", - "timeout": 1.0, - "url": "http://badlisted-words:8018/badlisted_words_batch" - }, - "dialog_formatter": "state_formatters.dp_formatters:hypotheses_list", - "response_formatter": "state_formatters.dp_formatters:simple_formatter_service", - "previous_services": [ - "skills" - ], - "state_manager_method": "add_hypothesis_annotation_batch", - "is_enabled": true, - "source": { - "directory": "annotators/BadlistedWordsDetector", - "container": "badlisted-words", - "endpoint": "badlisted_words_batch" - } - }, - "combined_classification": { - "connector": { - "protocol": "http", - "timeout": 2.0, - "url": "http://combined-classification:8087/batch_model" - }, - "dialog_formatter": "state_formatters.dp_formatters:hypothesis_histories_list", - "response_formatter": "state_formatters.dp_formatters:simple_formatter_service", - "previous_services": [ - "skills" - ], - "state_manager_method": "add_hypothesis_annotation_batch", - "is_enabled": true, - "source": { - "directory": "annotators/combined_classification", - "container": "combined-classification", - "endpoint": "batch_model" - } - }, "sentence_ranker": { "connector": { "protocol": "http", @@ -292,7 +159,7 @@ "connector": { "protocol": "http", "timeout": 1.0, - "url": "http://no-restrictions-selector:8009/respond" + "url": "http://ranking-based-response-selector:8002/respond" }, "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", @@ -302,8 +169,8 @@ "state_manager_method": "add_bot_utterance", "is_enabled": true, "source": { - "directory": "response_selectors/convers_evaluation_based_selector", - "container": "no-restrictions-selector", + "directory": "response_selectors/ranking_based_response_selector", + "container": "ranking-based-response-selector", "endpoint": "respond" } } diff --git a/assistant_dists/deepy_assistant/proxy.yml b/assistant_dists/deepy_assistant/proxy.yml index 70915cacf2..402411ea97 100644 --- a/assistant_dists/deepy_assistant/proxy.yml +++ b/assistant_dists/deepy_assistant/proxy.yml @@ -1,41 +1,5 @@ services: - sentseg: - command: ["nginx", "-g", "daemon off;"] - build: - context: dp/proxy/ - dockerfile: Dockerfile - environment: - - PROXY_PASS=dream.deeppavlov.ai:8011 - - PORT=8011 - - badlisted-words: - command: ["nginx", "-g", "daemon off;"] - build: - context: dp/proxy/ - dockerfile: Dockerfile - environment: - - PROXY_PASS=dream.deeppavlov.ai:8018 - - PORT=8018 - - spelling-preprocessing: - command: ["nginx", "-g", "daemon off;"] - build: - context: dp/proxy/ - dockerfile: Dockerfile - environment: - - PROXY_PASS=dream.deeppavlov.ai:8074 - - PORT=8074 - - combined-classification: - command: ["nginx", "-g", "daemon off;"] - build: - context: dp/proxy/ - dockerfile: Dockerfile - environment: - - PROXY_PASS=dream.deeppavlov.ai:8087 - - PORT=8087 - sentence-ranker: command: [ "nginx", "-g", "daemon off;" ] build: From 7d5a4b7481171221ee06886a6aba058f89c75f9b Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Mon, 24 Apr 2023 20:28:54 +0300 Subject: [PATCH 26/32] fix: no language --- .../dream_persona_rullama_prompted/dev.yml | 6 ++-- .../docker-compose.override.yml | 36 +++++++------------ .../pipeline_conf.json | 8 ++--- .../llm_based_response_selector/Dockerfile | 2 -- .../llm_based_response_selector/component.yml | 4 --- .../Dockerfile | 2 -- .../component.yml | 7 ++-- 7 files changed, 22 insertions(+), 43 deletions(-) diff --git a/assistant_dists/dream_persona_rullama_prompted/dev.yml b/assistant_dists/dream_persona_rullama_prompted/dev.yml index b67fdf2598..ae356c1dc6 100644 --- a/assistant_dists/dream_persona_rullama_prompted/dev.yml +++ b/assistant_dists/dream_persona_rullama_prompted/dev.yml @@ -5,12 +5,12 @@ services: - ".:/dp-agent" ports: - 4242:4242 - no-restrictions-selector-ru: + ranking-based-response-selector: volumes: - - "./response_selectors/convers_evaluation_based_selector:/src" + - "./response_selectors/ranking_based_response_selector:/src" - "./common:/src/common" ports: - - 8009:8009 + - 8002:8002 badlisted-words-ru: volumes: - "./annotators/BadlistedWordsDetector_ru:/src" diff --git a/assistant_dists/dream_persona_rullama_prompted/docker-compose.override.yml b/assistant_dists/dream_persona_rullama_prompted/docker-compose.override.yml index 06d3fb90fd..76705d4e68 100644 --- a/assistant_dists/dream_persona_rullama_prompted/docker-compose.override.yml +++ b/assistant_dists/dream_persona_rullama_prompted/docker-compose.override.yml @@ -2,7 +2,7 @@ services: agent: command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/dream_persona_rullama_prompted/pipeline_conf.json' environment: - WAIT_HOSTS: "no-restrictions-selector-ru:8009, badlisted-words-ru:8018, + WAIT_HOSTS: "ranking-based-response-selector:8002, badlisted-words-ru:8018, dialogrpt-ru:8122, prompt-selector-ru:8135, transformers-lm-llama7bru:8149, dff-dream-persona-llama7bru-prompted-skill:8151" WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1800} @@ -11,32 +11,20 @@ services: ALWAYS_TURN_ON_ALL_SKILLS: 0 LANGUAGE: RU - no-restrictions-selector-ru: - env_file: [ .env ] + ranking-based-response-selector-ru: + env_file: [ .env_ru ] build: args: - TAG_BASED_SELECTION: 1 - CALL_BY_NAME_PROBABILITY: 0.5 - PROMPT_PROBA: 0 - ACKNOWLEDGEMENT_PROBA: 0.3 - PRIORITIZE_WITH_REQUIRED_ACT: 0 - PRIORITIZE_NO_DIALOG_BREAKDOWN: 0 - PRIORITIZE_WITH_SAME_TOPIC_ENTITY: 0 - IGNORE_DISLIKED_SKILLS: 0 - GREETING_FIRST: 0 - RESTRICTION_FOR_SENSITIVE_CASE: 0 - PRIORITIZE_PROMTS_WHEN_NO_SCRIPTS: 0 - MAX_TURNS_WITHOUT_SCRIPTS: 100 - ADD_ACKNOWLEDGMENTS_IF_POSSIBLE: 0 - PRIORITIZE_SCRIPTED_SKILLS: 0 - CONFIDENCE_STRENGTH: 0.8 - CONV_EVAL_STRENGTH: 0.4 - PRIORITIZE_HUMAN_INITIATIVE: 1 - QUESTION_TO_QUESTION_DOWNSCORE_COEF: 0.8 - LANGUAGE: RU + SERVICE_PORT: 8002 + SERVICE_NAME: response_selector + SENTENCE_RANKER_ANNOTATION_NAME: dialogrpt + SENTENCE_RANKER_SERVICE_URL: http://dialogrpt-ru:8122/rank_sentences + SENTENCE_RANKER_TIMEOUT: 3 + N_UTTERANCES_CONTEXT: 5 + FILTER_TOXIC_OR_BADLISTED: 1 context: . - dockerfile: ./response_selectors/convers_evaluation_based_selector/Dockerfile - command: flask run -h 0.0.0.0 -p 8009 + dockerfile: ./response_selectors/ranking_based_response_selector/Dockerfile + command: flask run -h 0.0.0.0 -p 8002 environment: - FLASK_APP=server deploy: diff --git a/assistant_dists/dream_persona_rullama_prompted/pipeline_conf.json b/assistant_dists/dream_persona_rullama_prompted/pipeline_conf.json index 424ab2ff57..fa1a7bb479 100644 --- a/assistant_dists/dream_persona_rullama_prompted/pipeline_conf.json +++ b/assistant_dists/dream_persona_rullama_prompted/pipeline_conf.json @@ -193,9 +193,9 @@ "connector": { "protocol": "http", "timeout": 1.0, - "url": "http://no-restrictions-selector-ru:8009/respond" + "url": "http://ranking-based-response-selector-ru:8002/respond" }, - "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", + "dialog_formatter": "state_formatters.dp_formatters:cropped_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", "previous_services": [ "candidate_annotators" @@ -203,8 +203,8 @@ "state_manager_method": "add_bot_utterance", "is_enabled": true, "source": { - "directory": "response_selectors/convers_evaluation_based_selector", - "container": "no-restrictions-selector-ru", + "directory": "response_selectors/ranking_based_response_selector", + "container": "ranking-based-response-selector-ru", "endpoint": "respond" } } diff --git a/response_selectors/llm_based_response_selector/Dockerfile b/response_selectors/llm_based_response_selector/Dockerfile index 8b0a599117..2c58aa37e3 100644 --- a/response_selectors/llm_based_response_selector/Dockerfile +++ b/response_selectors/llm_based_response_selector/Dockerfile @@ -5,8 +5,6 @@ RUN mkdir /src COPY ./response_selectors/llm_based_response_selector/requirements.txt /src/requirements.txt RUN pip install -r /src/requirements.txt -ARG LANGUAGE=EN -ENV LANGUAGE ${LANGUAGE} ARG SERVICE_PORT ENV SERVICE_PORT ${SERVICE_PORT} ARG GENERATIVE_SERVICE_URL diff --git a/response_selectors/llm_based_response_selector/component.yml b/response_selectors/llm_based_response_selector/component.yml index 9e6cd3902c..cefaac088a 100644 --- a/response_selectors/llm_based_response_selector/component.yml +++ b/response_selectors/llm_based_response_selector/component.yml @@ -20,7 +20,6 @@ davinci3-based-response-selector: build_args: SERVICE_PORT: 8003 SERVICE_NAME: response_selector - LANGUAGE: EN GENERATIVE_SERVICE_URL: http://openai-api-davinci3:8131/respond GENERATIVE_SERVICE_CONFIG: openai-text-davinci-003-long.json GENERATIVE_TIMEOUT: 7 @@ -50,7 +49,6 @@ chatgpt-based-response-selector: build_args: SERVICE_PORT: 8003 SERVICE_NAME: response_selector - LANGUAGE: EN GENERATIVE_SERVICE_URL: http://openai-api-chatgpt:8145/respond GENERATIVE_SERVICE_CONFIG: openai-chatgpt.json GENERATIVE_TIMEOUT: 7 @@ -80,7 +78,6 @@ oasst12b-based-response-selector: build_args: SERVICE_PORT: 8003 SERVICE_NAME: response_selector - LANGUAGE: EN GENERATIVE_SERVICE_URL: http://transformers-lm-oasst12b:8158/respond GENERATIVE_SERVICE_CONFIG: default_generative_config.json GENERATIVE_TIMEOUT: 5 @@ -109,7 +106,6 @@ gptj-based-response-selector: build_args: SERVICE_PORT: 8003 SERVICE_NAME: response_selector - LANGUAGE: EN GENERATIVE_SERVICE_URL: http://transformers-lm-gptj:8130/respond GENERATIVE_SERVICE_CONFIG: default_generative_config.json GENERATIVE_TIMEOUT: 5 diff --git a/response_selectors/ranking_based_response_selector/Dockerfile b/response_selectors/ranking_based_response_selector/Dockerfile index 4e611f1f26..66b8ca036b 100644 --- a/response_selectors/ranking_based_response_selector/Dockerfile +++ b/response_selectors/ranking_based_response_selector/Dockerfile @@ -5,8 +5,6 @@ RUN mkdir /src COPY ./response_selectors/ranking_based_response_selector/requirements.txt /src/requirements.txt RUN pip install -r /src/requirements.txt -ARG LANGUAGE=EN -ENV LANGUAGE ${LANGUAGE} ARG SERVICE_PORT ENV SERVICE_PORT ${SERVICE_PORT} ARG SENTENCE_RANKER_ANNOTATION_NAME diff --git a/response_selectors/ranking_based_response_selector/component.yml b/response_selectors/ranking_based_response_selector/component.yml index 62e8ad349d..41ca23519c 100644 --- a/response_selectors/ranking_based_response_selector/component.yml +++ b/response_selectors/ranking_based_response_selector/component.yml @@ -17,7 +17,6 @@ ranking-based-response-selector-service: build_args: SERVICE_PORT: 8002 SERVICE_NAME: response_selector - LANGUAGE: EN SENTENCE_RANKER_ANNOTATION_NAME: null SENTENCE_RANKER_SERVICE_URL: http://sentence-ranker:8128/respond SENTENCE_RANKER_TIMEOUT: 3 @@ -43,7 +42,6 @@ ranking-based-response-selector-ru-service: build_args: SERVICE_PORT: 8002 SERVICE_NAME: response_selector - LANGUAGE: RU SENTENCE_RANKER_ANNOTATION_NAME: null SENTENCE_RANKER_SERVICE_URL: http://dialogrpt-ru:8122/rank_sentences SENTENCE_RANKER_TIMEOUT: 3 @@ -69,8 +67,10 @@ ranking-based-response-selector-ru: build_args: SERVICE_PORT: 8002 SERVICE_NAME: response_selector - LANGUAGE: RU SENTENCE_RANKER_ANNOTATION_NAME: dialogrpt + SENTENCE_RANKER_SERVICE_URL: http://dialogrpt-ru:8122/rank_sentences + SENTENCE_RANKER_TIMEOUT: 3 + N_UTTERANCES_CONTEXT: 5 FILTER_TOXIC_OR_BADLISTED: 1 date_created: '2023-04-24T09:45:32' ranking-based-response-selector: @@ -92,7 +92,6 @@ ranking-based-response-selector: build_args: SERVICE_PORT: 8002 SERVICE_NAME: response_selector - LANGUAGE: EN SENTENCE_RANKER_ANNOTATION_NAME: sentence_ranker SENTENCE_RANKER_SERVICE_URL: http://sentence-ranker:8128/respond SENTENCE_RANKER_TIMEOUT: 3 From db2eba379bcd27e515b3b046b10170b1cc25c5a1 Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Mon, 24 Apr 2023 20:34:35 +0300 Subject: [PATCH 27/32] fix: ru usage --- assistant_dists/dream_persona_rullama_prompted/dev.yml | 2 +- .../dream_persona_rullama_prompted/docker-compose.override.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/assistant_dists/dream_persona_rullama_prompted/dev.yml b/assistant_dists/dream_persona_rullama_prompted/dev.yml index ae356c1dc6..4c3d7ad5a0 100644 --- a/assistant_dists/dream_persona_rullama_prompted/dev.yml +++ b/assistant_dists/dream_persona_rullama_prompted/dev.yml @@ -5,7 +5,7 @@ services: - ".:/dp-agent" ports: - 4242:4242 - ranking-based-response-selector: + ranking-based-response-selector-ru: volumes: - "./response_selectors/ranking_based_response_selector:/src" - "./common:/src/common" diff --git a/assistant_dists/dream_persona_rullama_prompted/docker-compose.override.yml b/assistant_dists/dream_persona_rullama_prompted/docker-compose.override.yml index 76705d4e68..d505c3a491 100644 --- a/assistant_dists/dream_persona_rullama_prompted/docker-compose.override.yml +++ b/assistant_dists/dream_persona_rullama_prompted/docker-compose.override.yml @@ -2,7 +2,7 @@ services: agent: command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/dream_persona_rullama_prompted/pipeline_conf.json' environment: - WAIT_HOSTS: "ranking-based-response-selector:8002, badlisted-words-ru:8018, + WAIT_HOSTS: "ranking-based-response-selector-ru:8002, badlisted-words-ru:8018, dialogrpt-ru:8122, prompt-selector-ru:8135, transformers-lm-llama7bru:8149, dff-dream-persona-llama7bru-prompted-skill:8151" WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1800} From 7dd32df6821571b1045769c17f135e8d9874c99e Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Mon, 24 Apr 2023 21:26:31 +0300 Subject: [PATCH 28/32] fix: usage of formatter --- assistant_dists/ai_faq_assistant/pipeline_conf.json | 2 +- .../da_costa_clothes_assistant/pipeline_conf.json | 2 +- assistant_dists/deeppavlov_assistant/pipeline_conf.json | 2 +- assistant_dists/deepy_assistant/pipeline_conf.json | 2 +- assistant_dists/dream_persona_openai_prompted/dev.yml | 2 +- .../docker-compose.override.yml | 2 +- .../dream_persona_openai_prompted/pipeline_conf.json | 6 +++--- assistant_dists/fairytale_assistant/pipeline_conf.json | 2 +- assistant_dists/nutrition_assistant/pipeline_conf.json | 2 +- .../rhodes_coaching_assistant/pipeline_conf.json | 2 +- .../universal_prompted_assistant/pipeline_conf.json | 2 +- 11 files changed, 13 insertions(+), 13 deletions(-) diff --git a/assistant_dists/ai_faq_assistant/pipeline_conf.json b/assistant_dists/ai_faq_assistant/pipeline_conf.json index 4d80ce47ca..8e99eb2e5c 100644 --- a/assistant_dists/ai_faq_assistant/pipeline_conf.json +++ b/assistant_dists/ai_faq_assistant/pipeline_conf.json @@ -275,7 +275,7 @@ "timeout": 1.0, "url": "http://ranking-based-response-selector:8002/respond" }, - "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", + "dialog_formatter": "state_formatters.dp_formatters:cropped_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", "previous_services": [ "candidate_annotators" diff --git a/assistant_dists/da_costa_clothes_assistant/pipeline_conf.json b/assistant_dists/da_costa_clothes_assistant/pipeline_conf.json index 2fce861f0d..1d9f589440 100644 --- a/assistant_dists/da_costa_clothes_assistant/pipeline_conf.json +++ b/assistant_dists/da_costa_clothes_assistant/pipeline_conf.json @@ -275,7 +275,7 @@ "timeout": 1.0, "url": "http://ranking-based-response-selector:8002/respond" }, - "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", + "dialog_formatter": "state_formatters.dp_formatters:cropped_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", "previous_services": [ "candidate_annotators" diff --git a/assistant_dists/deeppavlov_assistant/pipeline_conf.json b/assistant_dists/deeppavlov_assistant/pipeline_conf.json index a9c962866a..e69c087c6c 100644 --- a/assistant_dists/deeppavlov_assistant/pipeline_conf.json +++ b/assistant_dists/deeppavlov_assistant/pipeline_conf.json @@ -161,7 +161,7 @@ "timeout": 1.0, "url": "http://ranking-based-response-selector:8002/respond" }, - "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", + "dialog_formatter": "state_formatters.dp_formatters:cropped_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", "previous_services": [ "candidate_annotators" diff --git a/assistant_dists/deepy_assistant/pipeline_conf.json b/assistant_dists/deepy_assistant/pipeline_conf.json index a1ee3f2eca..495ae52560 100644 --- a/assistant_dists/deepy_assistant/pipeline_conf.json +++ b/assistant_dists/deepy_assistant/pipeline_conf.json @@ -161,7 +161,7 @@ "timeout": 1.0, "url": "http://ranking-based-response-selector:8002/respond" }, - "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", + "dialog_formatter": "state_formatters.dp_formatters:cropped_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", "previous_services": [ "candidate_annotators" diff --git a/assistant_dists/dream_persona_openai_prompted/dev.yml b/assistant_dists/dream_persona_openai_prompted/dev.yml index 3e2bafff20..aa40605bcd 100644 --- a/assistant_dists/dream_persona_openai_prompted/dev.yml +++ b/assistant_dists/dream_persona_openai_prompted/dev.yml @@ -10,7 +10,7 @@ services: - "./annotators/SentSeg:/src" ports: - 8011:8011 - convers-evaluation-no-scripts-selector: + ranking-based-response-selector: volumes: - "./response_selectors/ranking_based_response_selector:/src" - "./common:/src/common" diff --git a/assistant_dists/dream_persona_openai_prompted/docker-compose.override.yml b/assistant_dists/dream_persona_openai_prompted/docker-compose.override.yml index 46081b747d..b747b3704c 100644 --- a/assistant_dists/dream_persona_openai_prompted/docker-compose.override.yml +++ b/assistant_dists/dream_persona_openai_prompted/docker-compose.override.yml @@ -2,7 +2,7 @@ services: agent: command: sh -c 'bin/wait && python -m deeppavlov_agent.run agent.pipeline_config=assistant_dists/dream_persona_openai_prompted/pipeline_conf.json' environment: - WAIT_HOSTS: "sentseg:8011, convers-evaluation-no-scripts-selector:8002, badlisted-words:8018, combined-classification:8087, + WAIT_HOSTS: "sentseg:8011, ranking-based-response-selector:8002, badlisted-words:8018, combined-classification:8087, sentence-ranker:8128, prompt-selector:8135, openai-api-davinci3:8131, dff-dream-persona-davinci3-prompted-skill:8137" WAIT_HOSTS_TIMEOUT: ${WAIT_TIMEOUT:-1000} diff --git a/assistant_dists/dream_persona_openai_prompted/pipeline_conf.json b/assistant_dists/dream_persona_openai_prompted/pipeline_conf.json index d17e62e737..72007e2ac7 100644 --- a/assistant_dists/dream_persona_openai_prompted/pipeline_conf.json +++ b/assistant_dists/dream_persona_openai_prompted/pipeline_conf.json @@ -273,9 +273,9 @@ "connector": { "protocol": "http", "timeout": 1.0, - "url": "http://convers-evaluation-no-scripts-selector:8002/respond" + "url": "http://ranking-based-response-selector:8002/respond" }, - "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", + "dialog_formatter": "state_formatters.dp_formatters:cropped_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", "previous_services": [ "candidate_annotators" @@ -284,7 +284,7 @@ "is_enabled": true, "source": { "directory": "response_selectors/ranking_based_response_selector", - "container": "convers-evaluation-no-scripts-selector", + "container": "ranking-based-response-selector", "endpoint": "respond" } } diff --git a/assistant_dists/fairytale_assistant/pipeline_conf.json b/assistant_dists/fairytale_assistant/pipeline_conf.json index b2e41629cd..b4cf796159 100644 --- a/assistant_dists/fairytale_assistant/pipeline_conf.json +++ b/assistant_dists/fairytale_assistant/pipeline_conf.json @@ -275,7 +275,7 @@ "timeout": 1.0, "url": "http://ranking-based-response-selector:8002/respond" }, - "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", + "dialog_formatter": "state_formatters.dp_formatters:cropped_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", "previous_services": [ "candidate_annotators" diff --git a/assistant_dists/nutrition_assistant/pipeline_conf.json b/assistant_dists/nutrition_assistant/pipeline_conf.json index f94f231ae5..830ae0652e 100644 --- a/assistant_dists/nutrition_assistant/pipeline_conf.json +++ b/assistant_dists/nutrition_assistant/pipeline_conf.json @@ -275,7 +275,7 @@ "timeout": 1.0, "url": "http://ranking-based-response-selector:8002/respond" }, - "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", + "dialog_formatter": "state_formatters.dp_formatters:cropped_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", "previous_services": [ "candidate_annotators" diff --git a/assistant_dists/rhodes_coaching_assistant/pipeline_conf.json b/assistant_dists/rhodes_coaching_assistant/pipeline_conf.json index 094f997529..1248ae8b9d 100644 --- a/assistant_dists/rhodes_coaching_assistant/pipeline_conf.json +++ b/assistant_dists/rhodes_coaching_assistant/pipeline_conf.json @@ -275,7 +275,7 @@ "timeout": 1.0, "url": "http://ranking-based-response-selector:8002/respond" }, - "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", + "dialog_formatter": "state_formatters.dp_formatters:cropped_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", "previous_services": [ "candidate_annotators" diff --git a/assistant_dists/universal_prompted_assistant/pipeline_conf.json b/assistant_dists/universal_prompted_assistant/pipeline_conf.json index a9508657e5..4cffe74eec 100644 --- a/assistant_dists/universal_prompted_assistant/pipeline_conf.json +++ b/assistant_dists/universal_prompted_assistant/pipeline_conf.json @@ -257,7 +257,7 @@ "timeout": 1.0, "url": "http://ranking-based-response-selector:8002/respond" }, - "dialog_formatter": "state_formatters.dp_formatters:full_history_dialog", + "dialog_formatter": "state_formatters.dp_formatters:cropped_dialog", "response_formatter": "state_formatters.dp_formatters:base_response_selector_formatter_service", "previous_services": [ "candidate_annotators" From be136406af97cbfbda0a38d2e5db344200be4aa8 Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Tue, 25 Apr 2023 17:53:08 +0300 Subject: [PATCH 29/32] fix: codestyle --- response_selectors/llm_based_response_selector/server.py | 2 +- .../ranking_based_response_selector/server.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/response_selectors/llm_based_response_selector/server.py b/response_selectors/llm_based_response_selector/server.py index 20222cf270..f860463f0d 100644 --- a/response_selectors/llm_based_response_selector/server.py +++ b/response_selectors/llm_based_response_selector/server.py @@ -77,7 +77,7 @@ def select_response(dialog_context, hypotheses): sentry_sdk.capture_exception(e) logger.exception(e) result = select_response_by_scores(hypotheses, [hyp["confidence"] for hyp in hypotheses])[0] - logger.info(f"Exception in LLM's invocation. Selected a response with the highest confidence.") + logger.info("Exception in LLM's invocation. Selected a response with the highest confidence.") logger.info(f"llm_based_response_selector selected:\n`{result}`") return result diff --git a/response_selectors/ranking_based_response_selector/server.py b/response_selectors/ranking_based_response_selector/server.py index 6624603b3d..9e02e71baf 100644 --- a/response_selectors/ranking_based_response_selector/server.py +++ b/response_selectors/ranking_based_response_selector/server.py @@ -49,7 +49,7 @@ def select_response_by_scores(hypotheses, scores): def get_scores(dialog_context, hypotheses): if all([SENTENCE_RANKER_ANNOTATION_NAME in hyp.get("annotations", {}) for hyp in hypotheses]): scores = [hyp.get("annotations", {}).get(SENTENCE_RANKER_ANNOTATION_NAME, 0.0) for hyp in hypotheses] - logger.info(f"Selected a response via Sentence Ranker Annotator.") + logger.info("Selected a response via Sentence Ranker Annotator.") else: try: dialog_context = "\n".join(dialog_context) @@ -60,17 +60,16 @@ def get_scores(dialog_context, hypotheses): timeout=SENTENCE_RANKER_TIMEOUT, ).json() scores = np.array(scores[0]["batch"]) - logger.info(f"Selected a response via Sentence Ranker Service.") + logger.info("Selected a response via Sentence Ranker Service.") except Exception as e: sentry_sdk.capture_exception(e) scores = [hyp["confidence"] for hyp in hypotheses] logger.exception(e) - logger.info(f"Selected a response via Confidence.") + logger.info("Selected a response via Confidence.") return scores def select_response(dialog_context, hypotheses): - scores = get_scores(dialog_context, hypotheses) scores = [score if hyp["skill_name"] != "dummy_skill" else score / 100 for score, hyp in zip(scores, hypotheses)] logger.info(f"Scores for selection:\n`{scores}`") From 89cb92b44c152f841546af6fde33a5bb0dcba5a9 Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Tue, 25 Apr 2023 19:02:56 +0300 Subject: [PATCH 30/32] fix: logs --- response_selectors/llm_based_response_selector/server.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/response_selectors/llm_based_response_selector/server.py b/response_selectors/llm_based_response_selector/server.py index f860463f0d..3941f69454 100644 --- a/response_selectors/llm_based_response_selector/server.py +++ b/response_selectors/llm_based_response_selector/server.py @@ -19,8 +19,8 @@ logger = logging.getLogger(__name__) app = Flask(__name__) -DEFAULT_CRITERION = "the most appropriate, relevant and non-toxic" +DEFAULT_CRITERION = "the most appropriate, relevant and non-toxic" GENERATIVE_SERVICE_URL = getenv("GENERATIVE_SERVICE_URL") GENERATIVE_TIMEOUT = int(getenv("GENERATIVE_TIMEOUT")) GENERATIVE_SERVICE_CONFIG = getenv("GENERATIVE_SERVICE_CONFIG") @@ -50,6 +50,8 @@ def filter_out_badlisted_or_toxic(hypotheses): is_toxic = is_toxic_or_badlisted_utterance(hyp) if not is_toxic: clean_hypotheses += [deepcopy(hyp)] + else: + logger.info(f"Filter out toxic candidate: {hyp['text']}") return clean_hypotheses From e82020ff9d0232c8cf488669cdaf9f72932b2d67 Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Tue, 25 Apr 2023 21:59:49 +0300 Subject: [PATCH 31/32] fix: after review --- .../llm_based_response_selector/component.yml | 44 ++++++++++--------- .../llm_based_response_selector/server.py | 3 +- .../component.yml | 8 ++-- 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/response_selectors/llm_based_response_selector/component.yml b/response_selectors/llm_based_response_selector/component.yml index cefaac088a..f6bb290445 100644 --- a/response_selectors/llm_based_response_selector/component.yml +++ b/response_selectors/llm_based_response_selector/component.yml @@ -6,11 +6,11 @@ davinci3-based-response-selector: model_type: Dictionary/Pattern-based is_customizable: true author: DeepPavlov - description: Algorithm that selects a final responses among the given list of candidate - responses via asking the given LLM service the question `Select {CRITERION} response - among the hypotheses to the given dialog context. Return only the selected response - without extra explanations.` where default CRITERION is `the most appropriate, - relevant and non-toxic` and could be customized + description: Algorithm that selects the final response among the given list of candidate + responses via asking the given LLM service to `Select {CRITERION} response + among the hypotheses to the given dialog context. Return only the selected response + without extra explanations.` where the default CRITERION is `the most appropriate, + relevant and non-toxic`. CRITERION can be customized. ram_usage: 100M gpu_usage: null port: 8003 @@ -26,6 +26,7 @@ davinci3-based-response-selector: ENVVARS_TO_SEND: OPENAI_API_KEY,OPENAI_ORGANIZATION N_UTTERANCES_CONTEXT: 5 FILTER_TOXIC_OR_BADLISTED: 1 + CRITERION: the most appropriate, relevant and non-toxic date_created: '2023-04-24T09:45:32' chatgpt-based-response-selector: name: response_selector @@ -35,11 +36,11 @@ chatgpt-based-response-selector: model_type: Dictionary/Pattern-based is_customizable: true author: DeepPavlov - description: Algorithm that selects a final responses among the given list of candidate - responses via asking the given LLM service the question `Select {CRITERION} response - among the hypotheses to the given dialog context. Return only the selected response - without extra explanations.` where default CRITERION is `the most appropriate, - relevant and non-toxic` and could be customized + description: Algorithm that selects the final response among the given list of candidate + responses via asking the given LLM service to `Select {CRITERION} response + among the hypotheses to the given dialog context. Return only the selected response + without extra explanations.` where the default CRITERION is `the most appropriate, + relevant and non-toxic`. CRITERION can be customized. ram_usage: 100M gpu_usage: null port: 8003 @@ -55,6 +56,7 @@ chatgpt-based-response-selector: ENVVARS_TO_SEND: OPENAI_API_KEY,OPENAI_ORGANIZATION N_UTTERANCES_CONTEXT: 5 FILTER_TOXIC_OR_BADLISTED: 1 + CRITERION: the most appropriate, relevant and non-toxic date_created: '2023-04-24T09:45:32' oasst12b-based-response-selector: name: response_selector @@ -64,11 +66,11 @@ oasst12b-based-response-selector: model_type: Dictionary/Pattern-based is_customizable: true author: DeepPavlov - description: Algorithm that selects a final responses among the given list of candidate - responses via asking the given LLM service the question `Select {CRITERION} response - among the hypotheses to the given dialog context. Return only the selected response - without extra explanations.` where default CRITERION is `the most appropriate, - relevant and non-toxic` and could be customized + description: Algorithm that selects the final response among the given list of candidate + responses via asking the given LLM service to `Select {CRITERION} response + among the hypotheses to the given dialog context. Return only the selected response + without extra explanations.` where the default CRITERION is `the most appropriate, + relevant and non-toxic`. CRITERION can be customized. ram_usage: 100M gpu_usage: null port: 8003 @@ -83,6 +85,7 @@ oasst12b-based-response-selector: GENERATIVE_TIMEOUT: 5 N_UTTERANCES_CONTEXT: 5 FILTER_TOXIC_OR_BADLISTED: 1 + CRITERION: the most appropriate, relevant and non-toxic date_created: '2023-04-24T09:45:32' gptj-based-response-selector: name: response_selector @@ -92,11 +95,11 @@ gptj-based-response-selector: model_type: Dictionary/Pattern-based is_customizable: true author: DeepPavlov - description: Algorithm that selects a final responses among the given list of candidate - responses via asking the given LLM service the question `Select {CRITERION} response - among the hypotheses to the given dialog context. Return only the selected response - without extra explanations.` where default CRITERION is `the most appropriate, - relevant and non-toxic` and could be customized + description: Algorithm that selects the final response among the given list of candidate + responses via asking the given LLM service to `Select {CRITERION} response + among the hypotheses to the given dialog context. Return only the selected response + without extra explanations.` where the default CRITERION is `the most appropriate, + relevant and non-toxic`. CRITERION can be customized. ram_usage: 100M gpu_usage: null port: 8003 @@ -111,4 +114,5 @@ gptj-based-response-selector: GENERATIVE_TIMEOUT: 5 N_UTTERANCES_CONTEXT: 5 FILTER_TOXIC_OR_BADLISTED: 1 + CRITERION: the most appropriate, relevant and non-toxic date_created: '2023-04-24T09:45:32' diff --git a/response_selectors/llm_based_response_selector/server.py b/response_selectors/llm_based_response_selector/server.py index 3941f69454..3c84a1dfca 100644 --- a/response_selectors/llm_based_response_selector/server.py +++ b/response_selectors/llm_based_response_selector/server.py @@ -20,7 +20,6 @@ app = Flask(__name__) -DEFAULT_CRITERION = "the most appropriate, relevant and non-toxic" GENERATIVE_SERVICE_URL = getenv("GENERATIVE_SERVICE_URL") GENERATIVE_TIMEOUT = int(getenv("GENERATIVE_TIMEOUT")) GENERATIVE_SERVICE_CONFIG = getenv("GENERATIVE_SERVICE_CONFIG") @@ -29,7 +28,7 @@ GENERATIVE_SERVICE_CONFIG = json.load(f) FILTER_TOXIC_OR_BADLISTED = int(getenv("FILTER_TOXIC_OR_BADLISTED")) N_UTTERANCES_CONTEXT = int(getenv("N_UTTERANCES_CONTEXT")) -CRITERION = getenv("CRITERION", DEFAULT_CRITERION) +CRITERION = getenv("CRITERION", "the most appropriate, relevant and non-toxic") PROMPT = ( f"""Select {CRITERION} response among the hypotheses to the given dialog context. """ """Return only the selected response without extra explanations.""" diff --git a/response_selectors/ranking_based_response_selector/component.yml b/response_selectors/ranking_based_response_selector/component.yml index 41ca23519c..d429b1fcf0 100644 --- a/response_selectors/ranking_based_response_selector/component.yml +++ b/response_selectors/ranking_based_response_selector/component.yml @@ -6,7 +6,7 @@ ranking-based-response-selector-service: model_type: Dictionary/Pattern-based is_customizable: false author: DeepPavlov - description: Algorithm that selects a final responses among the given list of candidate + description: Algorithm that selects the final response among the given list of candidate responses via the given ranking service ram_usage: 100M gpu_usage: null @@ -31,7 +31,7 @@ ranking-based-response-selector-ru-service: model_type: Dictionary/Pattern-based is_customizable: false author: DeepPavlov - description: Algorithm that selects a final responses among the given list of candidate + description: Algorithm that selects the final response among the given list of candidate responses via the given ranking service ram_usage: 100M gpu_usage: null @@ -56,7 +56,7 @@ ranking-based-response-selector-ru: model_type: Dictionary/Pattern-based is_customizable: false author: DeepPavlov - description: Algorithm that selects a final responses among the given list of candidate + description: Algorithm that selects the final response among the given list of candidate responses via the given ranking service ram_usage: 100M gpu_usage: null @@ -81,7 +81,7 @@ ranking-based-response-selector: model_type: Dictionary/Pattern-based is_customizable: false author: DeepPavlov - description: Algorithm that selects a final responses among the given list of candidate + description: Algorithm that selects the final response among the given list of candidate responses via the given ranking service ram_usage: 100M gpu_usage: null From 06da7c30285aa0a39c31e6f43ea7e00645d1aee5 Mon Sep 17 00:00:00 2001 From: dilyararimovna Date: Tue, 25 Apr 2023 22:35:52 +0300 Subject: [PATCH 32/32] fix: sync --- response_selectors/llm_based_response_selector/Dockerfile | 2 ++ response_selectors/ranking_based_response_selector/Dockerfile | 2 ++ .../ranking_based_response_selector/component.yml | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/response_selectors/llm_based_response_selector/Dockerfile b/response_selectors/llm_based_response_selector/Dockerfile index 2c58aa37e3..926c22f48d 100644 --- a/response_selectors/llm_based_response_selector/Dockerfile +++ b/response_selectors/llm_based_response_selector/Dockerfile @@ -21,6 +21,8 @@ ARG FILTER_TOXIC_OR_BADLISTED=1 ENV FILTER_TOXIC_OR_BADLISTED ${FILTER_TOXIC_OR_BADLISTED} COPY ./response_selectors/llm_based_response_selector/ /src/ +COPY ./common/ ./common/ + WORKDIR /src CMD gunicorn --workers=1 server:app -b 0.0.0.0:${SERVICE_PORT} --timeout=1200 diff --git a/response_selectors/ranking_based_response_selector/Dockerfile b/response_selectors/ranking_based_response_selector/Dockerfile index 66b8ca036b..d8306092f7 100644 --- a/response_selectors/ranking_based_response_selector/Dockerfile +++ b/response_selectors/ranking_based_response_selector/Dockerfile @@ -19,6 +19,8 @@ ARG FILTER_TOXIC_OR_BADLISTED=1 ENV FILTER_TOXIC_OR_BADLISTED ${FILTER_TOXIC_OR_BADLISTED} COPY ./response_selectors/ranking_based_response_selector/ /src/ +COPY ./common/ ./common/ + WORKDIR /src CMD gunicorn --workers=1 server:app -b 0.0.0.0:${SERVICE_PORT} --timeout=1200 diff --git a/response_selectors/ranking_based_response_selector/component.yml b/response_selectors/ranking_based_response_selector/component.yml index d429b1fcf0..cc91c8b6ad 100644 --- a/response_selectors/ranking_based_response_selector/component.yml +++ b/response_selectors/ranking_based_response_selector/component.yml @@ -97,4 +97,4 @@ ranking-based-response-selector: SENTENCE_RANKER_TIMEOUT: 3 N_UTTERANCES_CONTEXT: 5 FILTER_TOXIC_OR_BADLISTED: 1 - date_created: '2023-04-24T09:45:32' \ No newline at end of file + date_created: '2023-04-24T09:45:32'