From eaefebb2c90729c9652c0a056104ca03169847be Mon Sep 17 00:00:00 2001 From: slaren Date: Sat, 18 Nov 2023 14:53:34 +0100 Subject: [PATCH 1/5] gguf-py : export chat templates --- gguf-py/gguf/constants.py | 29 +++++++++++++++-------------- gguf-py/gguf/gguf_writer.py | 3 +++ gguf-py/gguf/vocab.py | 6 ++++++ 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 7f63361bd32bc..8bd82daca38a7 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -56,20 +56,21 @@ class Rope: SCALING_FINETUNED = "{arch}.rope.scaling.finetuned" class Tokenizer: - MODEL = "tokenizer.ggml.model" - LIST = "tokenizer.ggml.tokens" - TOKEN_TYPE = "tokenizer.ggml.token_type" - SCORES = "tokenizer.ggml.scores" - MERGES = "tokenizer.ggml.merges" - BOS_ID = "tokenizer.ggml.bos_token_id" - EOS_ID = "tokenizer.ggml.eos_token_id" - UNK_ID = "tokenizer.ggml.unknown_token_id" - SEP_ID = "tokenizer.ggml.seperator_token_id" - PAD_ID = "tokenizer.ggml.padding_token_id" - ADD_BOS = "tokenizer.ggml.add_bos_token" - ADD_EOS = "tokenizer.ggml.add_eos_token" - HF_JSON = "tokenizer.huggingface.json" - RWKV = "tokenizer.rwkv.world" + MODEL = "tokenizer.ggml.model" + LIST = "tokenizer.ggml.tokens" + TOKEN_TYPE = "tokenizer.ggml.token_type" + SCORES = "tokenizer.ggml.scores" + MERGES = "tokenizer.ggml.merges" + BOS_ID = "tokenizer.ggml.bos_token_id" + EOS_ID = "tokenizer.ggml.eos_token_id" + UNK_ID = "tokenizer.ggml.unknown_token_id" + SEP_ID = "tokenizer.ggml.seperator_token_id" + PAD_ID = "tokenizer.ggml.padding_token_id" + ADD_BOS = "tokenizer.ggml.add_bos_token" + ADD_EOS = "tokenizer.ggml.add_eos_token" + HF_JSON = "tokenizer.huggingface.json" + RWKV = "tokenizer.rwkv.world" + CHAT_TEMPLATE = "tokenizer.chat_template" # diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index c3b8c588f17cd..ab7382c44f44e 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -399,6 +399,9 @@ def add_add_bos_token(self, value: bool) -> None: def add_add_eos_token(self, value: bool) -> None: self.add_bool(Keys.Tokenizer.ADD_EOS, value) + def add_chat_template(self, value: str) -> None: + self.add_string(Keys.Tokenizer.CHAT_TEMPLATE, value) + def _pack(self, fmt: str, value: Any, skip_pack_prefix: bool = False) -> bytes: pack_prefix = '' if not skip_pack_prefix: diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.py index b9f50a0afed7a..cc3f6bf4c53c0 100644 --- a/gguf-py/gguf/vocab.py +++ b/gguf-py/gguf/vocab.py @@ -13,6 +13,7 @@ class SpecialVocab: merges: list[str] add_special_token: dict[str, bool] special_token_ids: dict[str, int] + chat_template: str | None def __init__( self, path: str | os.PathLike[str], load_merges: bool = False, @@ -67,6 +68,10 @@ def add_to_gguf(self, gw: GGUFWriter, quiet: bool = False) -> None: if not quiet: print(f'gguf: Setting add_{typ}_token to {value}') add_handler(value) + if self.chat_template is not None: + if not quiet: + print(f'gguf: Setting chat_template to {self.chat_template}') + gw.add_chat_template(self.chat_template) def _load(self, path: Path) -> None: self._try_load_from_tokenizer_json(path) @@ -156,6 +161,7 @@ def _try_load_from_tokenizer_json(self, path: Path) -> bool: None, ) self._set_special_token(typ, maybe_token_id) + self.chat_template = tokenizer_config.get('chat_template') return True def _try_load_from_config_json(self, path: Path) -> bool: From 9829b5b38becc050e22c759fde73c3ef8a860898 Mon Sep 17 00:00:00 2001 From: slaren Date: Sat, 18 Nov 2023 15:59:32 +0100 Subject: [PATCH 2/5] llama.cpp : escape new lines in gguf kv info prints --- llama.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llama.cpp b/llama.cpp index 56d8e765cc112..062c9757b5398 100644 --- a/llama.cpp +++ b/llama.cpp @@ -1871,6 +1871,7 @@ struct llama_model_loader { if (value.size() > MAX_VALUE_LEN) { value = format("%s...", value.substr(0, MAX_VALUE_LEN - 3).c_str()); } + replace_all(value, "\n", "\\n"); LLAMA_LOG_INFO("%s: - kv %3d: %42s %-16s = %s\n", __func__, i, name, type_name.c_str(), value.c_str()); } From 72c3a5d28a8f60050f3f5e19dad8f2e3783789cd Mon Sep 17 00:00:00 2001 From: slaren Date: Sat, 18 Nov 2023 16:11:06 +0100 Subject: [PATCH 3/5] gguf-py : bump version --- gguf-py/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gguf-py/pyproject.toml b/gguf-py/pyproject.toml index 6e3f9e85549d0..e6374bfe898a4 100644 --- a/gguf-py/pyproject.toml +++ b/gguf-py/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gguf" -version = "0.5.3" +version = "0.6.0" description = "Read and write ML models in GGUF for GGML" authors = ["GGML "] packages = [ From edd98313ca7362c7cee7fad9c5f4ce5da32f0a89 Mon Sep 17 00:00:00 2001 From: slaren Date: Sat, 18 Nov 2023 16:59:22 +0100 Subject: [PATCH 4/5] gguf-py : check chat_template type --- gguf-py/gguf/vocab.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.py index cc3f6bf4c53c0..f75286ecae060 100644 --- a/gguf-py/gguf/vocab.py +++ b/gguf-py/gguf/vocab.py @@ -137,6 +137,14 @@ def _try_load_from_tokenizer_json(self, path: Path) -> bool: return True with open(tokenizer_config_file, encoding = 'utf-8') as f: tokenizer_config = json.load(f) + chat_template = tokenizer_config.get('chat_template') + if chat_template is None or isinstance(chat_template, str): + self.chat_template = chat_template + else: + print( + f'gguf: WARNING: Bad type for chat_template field in {tokenizer_config_file!r} - ignoring', + file = sys.stderr + ) for typ in self.special_token_types: add_entry = tokenizer_config.get(f'add_{typ}_token') if isinstance(add_entry, bool): @@ -161,7 +169,6 @@ def _try_load_from_tokenizer_json(self, path: Path) -> bool: None, ) self._set_special_token(typ, maybe_token_id) - self.chat_template = tokenizer_config.get('chat_template') return True def _try_load_from_config_json(self, path: Path) -> bool: From b044ba7aaf690447559c185c246a04c8cc0af946 Mon Sep 17 00:00:00 2001 From: slaren Date: Sat, 18 Nov 2023 17:16:33 +0100 Subject: [PATCH 5/5] gguf-py : initialize chat_template --- gguf-py/gguf/vocab.py | 1 + 1 file changed, 1 insertion(+) diff --git a/gguf-py/gguf/vocab.py b/gguf-py/gguf/vocab.py index f75286ecae060..de3e5edb557d7 100644 --- a/gguf-py/gguf/vocab.py +++ b/gguf-py/gguf/vocab.py @@ -25,6 +25,7 @@ def __init__( self.n_vocab = n_vocab self.load_merges = load_merges self.merges = [] + self.chat_template = None if special_token_types is not None: self.special_token_types = special_token_types else: