From 0339654a406a3a79efda6d9cc58b273a0a575deb Mon Sep 17 00:00:00 2001 From: Sylvain Gugger <35901082+sgugger@users.noreply.github.com> Date: Wed, 24 May 2023 16:09:13 -0400 Subject: [PATCH] Enable prompts on the Hub (#23662) * Enable prompts on the Hub * Update src/transformers/tools/prompts.py Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> * Address review comments --------- Co-authored-by: amyeroberts <22614925+amyeroberts@users.noreply.github.com> --- docs/source/en/custom_tools.mdx | 7 ++ src/transformers/tools/agents.py | 39 +++++-- src/transformers/tools/prompts.py | 180 ++++-------------------------- 3 files changed, 56 insertions(+), 170 deletions(-) diff --git a/docs/source/en/custom_tools.mdx b/docs/source/en/custom_tools.mdx index 23ca4bccbadca4..feb4c7ac46d779 100644 --- a/docs/source/en/custom_tools.mdx +++ b/docs/source/en/custom_tools.mdx @@ -414,6 +414,13 @@ of the tools, it has available to it. +In both cases, you can pass a repo ID instead of the prompt template if you would like to use a template hosted by someone in the community. The default prompts live in [this repo](https://huggingface.co/datasets/huggingface-tools/default-prompts) as an example. + +To upload your custom prompt on a repo on the Hub and share it with the community just make sure: +- to use a dataset repository +- to put the prompt template for the `run` command in a file named `run_prompt_template.txt` +- to put the prompt template for the `chat` command in a file named `chat_prompt_template.txt` + ## Using custom tools In this section, we'll be leveraging two existing custom tools that are specific to image generation: diff --git a/src/transformers/tools/agents.py b/src/transformers/tools/agents.py index b04f622e15cf99..0ecb600d9212a9 100644 --- a/src/transformers/tools/agents.py +++ b/src/transformers/tools/agents.py @@ -27,7 +27,7 @@ from ..models.auto import AutoTokenizer from ..utils import is_openai_available, is_torch_available, logging from .base import TASK_MAPPING, TOOL_CONFIG_FILE, Tool, load_tool, supports_remote -from .prompts import CHAT_MESSAGE_PROMPT, CHAT_PROMPT_TEMPLATE, RUN_PROMPT_TEMPLATE +from .prompts import CHAT_MESSAGE_PROMPT, download_prompt from .python_interpreter import evaluate @@ -193,9 +193,13 @@ class Agent: Args: chat_prompt_template (`str`, *optional*): - Pass along your own prompt if you want to override the default template for the `chat` method. + Pass along your own prompt if you want to override the default template for the `chat` method. Can be the + actual prompt template or a repo ID (on the Hugging Face Hub). The prompt should be in a file named + `chat_prompt_template.txt` in this repo in this case. run_prompt_template (`str`, *optional*): - Pass along your own prompt if you want to override the default template for the `run` method. + Pass along your own prompt if you want to override the default template for the `run` method. Can be the + actual prompt template or a repo ID (on the Hugging Face Hub). The prompt should be in a file named + `run_prompt_template.txt` in this repo in this case. additional_tools ([`Tool`], list of tools or dictionary with tool values, *optional*): Any additional tools to include on top of the default ones. If you pass along a tool with the same name as one of the default tools, that default tool will be overridden. @@ -204,8 +208,9 @@ class Agent: def __init__(self, chat_prompt_template=None, run_prompt_template=None, additional_tools=None): _setup_default_tools() - self.chat_prompt_template = CHAT_PROMPT_TEMPLATE if chat_prompt_template is None else chat_prompt_template - self.run_prompt_template = RUN_PROMPT_TEMPLATE if run_prompt_template is None else run_prompt_template + agent_name = self.__class__.__name__ + self.chat_prompt_template = download_prompt(chat_prompt_template, agent_name, mode="chat") + self.run_prompt_template = download_prompt(run_prompt_template, agent_name, mode="run") self._toolbox = HUGGINGFACE_DEFAULT_TOOLS.copy() self.log = print if additional_tools is not None: @@ -367,9 +372,13 @@ class OpenAiAgent(Agent): api_key (`str`, *optional*): The API key to use. If unset, will look for the environment variable `"OPENAI_API_KEY"`. chat_prompt_template (`str`, *optional*): - Pass along your own prompt if you want to override the default template for the `chat` method. + Pass along your own prompt if you want to override the default template for the `chat` method. Can be the + actual prompt template or a repo ID (on the Hugging Face Hub). The prompt should be in a file named + `chat_prompt_template.txt` in this repo in this case. run_prompt_template (`str`, *optional*): - Pass along your own prompt if you want to override the default template for the `run` method. + Pass along your own prompt if you want to override the default template for the `run` method. Can be the + actual prompt template or a repo ID (on the Hugging Face Hub). The prompt should be in a file named + `run_prompt_template.txt` in this repo in this case. additional_tools ([`Tool`], list of tools or dictionary with tool values, *optional*): Any additional tools to include on top of the default ones. If you pass along a tool with the same name as one of the default tools, that default tool will be overridden. @@ -455,9 +464,13 @@ class HfAgent(Agent): The token to use as HTTP bearer authorization for remote files. If unset, will use the token generated when running `huggingface-cli login` (stored in `~/.huggingface`). chat_prompt_template (`str`, *optional*): - Pass along your own prompt if you want to override the default template for the `chat` method. + Pass along your own prompt if you want to override the default template for the `chat` method. Can be the + actual prompt template or a repo ID (on the Hugging Face Hub). The prompt should be in a file named + `chat_prompt_template.txt` in this repo in this case. run_prompt_template (`str`, *optional*): - Pass along your own prompt if you want to override the default template for the `run` method. + Pass along your own prompt if you want to override the default template for the `run` method. Can be the + actual prompt template or a repo ID (on the Hugging Face Hub). The prompt should be in a file named + `run_prompt_template.txt` in this repo in this case. additional_tools ([`Tool`], list of tools or dictionary with tool values, *optional*): Any additional tools to include on top of the default ones. If you pass along a tool with the same name as one of the default tools, that default tool will be overridden. @@ -521,9 +534,13 @@ class LocalAgent(Agent): tokenizer ([`PreTrainedTokenizer`]): The tokenizer to use for the agent. chat_prompt_template (`str`, *optional*): - Pass along your own prompt if you want to override the default template for the `chat` method. + Pass along your own prompt if you want to override the default template for the `chat` method. Can be the + actual prompt template or a repo ID (on the Hugging Face Hub). The prompt should be in a file named + `chat_prompt_template.txt` in this repo in this case. run_prompt_template (`str`, *optional*): - Pass along your own prompt if you want to override the default template for the `run` method. + Pass along your own prompt if you want to override the default template for the `run` method. Can be the + actual prompt template or a repo ID (on the Hugging Face Hub). The prompt should be in a file named + `run_prompt_template.txt` in this repo in this case. additional_tools ([`Tool`], list of tools or dictionary with tool values, *optional*): Any additional tools to include on top of the default ones. If you pass along a tool with the same name as one of the default tools, that default tool will be overridden. diff --git a/src/transformers/tools/prompts.py b/src/transformers/tools/prompts.py index 796b3c242ccfe1..2dbb799f859ffe 100644 --- a/src/transformers/tools/prompts.py +++ b/src/transformers/tools/prompts.py @@ -14,173 +14,35 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import re -# docstyle-ignore -RUN_PROMPT_TEMPLATE = """I will ask you to perform a task, your job is to come up with a series of simple commands in Python that will perform the task. -To help you, I will give you access to a set of tools that you can use. Each tool is a Python function and has a description explaining the task it performs, the inputs it expects and the outputs it returns. -You should first explain which tool you will use to perform the task and for what reason, then write the code in Python. -Each instruction in Python should be a simple assignment. You can print intermediate results if it makes sense to do so. - -Tools: -<> - - -Task: "Answer the question in the variable `question` about the image stored in the variable `image`. The question is in French." - -I will use the following tools: `translator` to translate the question into English and then `image_qa` to answer the question on the input image. - -Answer: -```py -translated_question = translator(question=question, src_lang="French", tgt_lang="English") -print(f"The translated question is {translated_question}.") -answer = image_qa(image=image, question=translated_question) -print(f"The answer is {answer}") -``` - -Task: "Identify the oldest person in the `document` and create an image showcasing the result." - -I will use the following tools: `document_qa` to find the oldest person in the document, then `image_generator` to generate an image according to the answer. - -Answer: -```py -answer = document_qa(document, question="What is the oldest person?") -print(f"The answer is {answer}.") -image = image_generator(answer) -``` - -Task: "Generate an image using the text given in the variable `caption`." - -I will use the following tool: `image_generator` to generate an image. - -Answer: -```py -image = image_generator(prompt=caption) -``` - -Task: "Summarize the text given in the variable `text` and read it out loud." - -I will use the following tools: `summarizer` to create a summary of the input text, then `text_reader` to read it out loud. - -Answer: -```py -summarized_text = summarizer(text) -print(f"Summary: {summarized_text}") -audio_summary = text_reader(summarized_text) -``` - -Task: "Answer the question in the variable `question` about the text in the variable `text`. Use the answer to generate an image." - -I will use the following tools: `text_qa` to create the answer, then `image_generator` to generate an image according to the answer. - -Answer: -```py -answer = text_qa(text=text, question=question) -print(f"The answer is {answer}.") -image = image_generator(answer) -``` - -Task: "Caption the following `image`." - -I will use the following tool: `image_captioner` to generate a caption for the image. - -Answer: -```py -caption = image_captioner(image) -``` - -Task: "<>" - -I will use the following""" +from ..utils import cached_file # docstyle-ignore -CHAT_PROMPT_TEMPLATE = """Below are a series of dialogues between various people and an AI assistant specialized in coding. The AI assistant tries to be helpful, polite, honest, and humble-but-knowledgeable. - -The job of the AI assistant is to come up with a series of simple commands in Python that will perform the task the human wants to perform. -To help with that, the AI assistant has access to a set of tools. Each tool is a Python function and has a description explaining the task it performs, the inputs it expects and the outputs it returns. -The AI assistant should first explain the tools it will use to perform the task and for what reason, then write the code in Python. -Each instruction in Python should be a simple assignment. The AI assistant can print intermediate results if it makes sense to do so. - -Tools: -<> - -===== - -Human: Answer the question in the variable `question` about the image stored in the variable `image`. - -Assistant: I will use the tool `image_qa` to answer the question on the input image. - -```py -answer = image_qa(text=question, image=image) -print(f"The answer is {answer}") -``` - -Human: I tried this code, it worked but didn't give me a good result. The question is in French - -Assistant: In this case, the question needs to be translated first. I will use the tool `translator` to do this. - -```py -translated_question = translator(question=question, src_lang="French", tgt_lang="English") -print(f"The translated question is {translated_question}.") -answer = image_qa(text=translated_question, image=image) -print(f"The answer is {answer}") -``` - -===== - -Human: Identify the oldest person in the `document`. - -Assistant: I will use the tool `document_qa` to find the oldest person in the document. - -```py -answer = document_qa(document, question="What is the oldest person?") -print(f"The answer is {answer}.") -``` - -Human: Can you generate an image with the result? - -Assistant: I will use the tool `image_generator` to do that. - -```py -image = image_generator(answer) -``` - -===== - -Human: Summarize the text given in the variable `text` and read it out loud. - -Assistant: I will use the tool `summarizer` to create a summary of the input text, then the tool `text_reader` to read it out loud. - -```py -summarized_text = summarizer(text) -print(f"Summary: {summarized_text}") -audio_summary = text_reader(text=summary) -``` - -Human: I got the following error: "The variable `summary` is not defined." - -Assistant: My bad! Let's try this code instead. - -```py -summarized_text = summarizer(text) -print(f"Summary: {summarized_text}") -audio_summary = text_reader(text=summarized_text) -``` +CHAT_MESSAGE_PROMPT = """ +Human: <> -Human: It worked! Can you translate the summary in German? +Assistant: """ -Assistant: I will use the tool `translator` to translate the text in German. -```py -translated_summary = translator(summarized_text, src_lang="English", tgt_lang="German") -``` +DEFAULT_PROMPTS_REPO = "huggingface-tools/default-prompts" +PROMPT_FILES = {"chat": "chat_prompt_template.txt", "run": "run_prompt_template.txt"} -==== -""" +def download_prompt(prompt_or_repo_id, agent_name, mode="run"): + """ + Downloads and caches the prompt from a repo and returns it contents (if necessary) + """ + if prompt_or_repo_id is None: + prompt_or_repo_id = DEFAULT_PROMPTS_REPO -# docstyle-ignore -CHAT_MESSAGE_PROMPT = """ -Human: <> + # prompt is considered a repo ID when it does not contain any kind of space + if re.search("\\s", prompt_or_repo_id) is not None: + return prompt_or_repo_id -Assistant: """ + prompt_file = cached_file( + prompt_or_repo_id, PROMPT_FILES[mode], repo_type="dataset", user_agent={"agent": agent_name} + ) + with open(prompt_file, "r", encoding="utf-8") as f: + return f.read()