diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 0ec8d6d8d03b5..8a6dbcd8214fb 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -65,6 +65,7 @@ add_library(${TARGET} STATIC train.cpp ngram-cache.h ngram-cache.cpp + chaton_meta.cpp ) if (BUILD_SHARED_LIBS) @@ -83,5 +84,5 @@ if (LLAMA_CURL) endif () target_include_directories(${TARGET} PUBLIC .) -target_compile_features(${TARGET} PUBLIC cxx_std_11) +target_compile_features(${TARGET} PUBLIC cxx_std_17) target_link_libraries(${TARGET} PRIVATE ${LLAMA_COMMON_EXTRA_LIBS} PUBLIC llama) diff --git a/common/chaton.hpp b/common/chaton.hpp new file mode 100644 index 0000000000000..2ea9fed7a6efd --- /dev/null +++ b/common/chaton.hpp @@ -0,0 +1,823 @@ +#pragma once + +/** + * + * Generic tagging logic + configurable template data based chat templates handling + * by Humans for All + * + * ## Overview + * + * Helps chat with models, by tagging chat messages based on the specified + * chat-handshake-template-standard. This uses a generic tagging code driven + * by configurable template data which is either builtin or loaded from text/ + * json file, which specifies the handshake template details. + * + * This can be used by + * + * * examples/main, to build on existing interactive flow and its in-prefix, in-suffix + * and antiprompt/reverse-prompt + * + * * examples/server|..., by replacing its existing llama_chat_apply_template with the + * equivalent helper here. + * + * + * ## The common pattern + * + * As a convention, the tagging used by LLMs to differentiate between the + * different parts when chatting with them normally follows a general pattern of + * + * * + * + * * The Roles could include System, User and Assistant (ie the Model) + * + * * A chat normally consists of + * + * * a System message/prompt followed by + * + * * multiple user message/query - model message/response pairs + * + * The different models will normally have all or some subset of the tagging mentioned above. + * + * You may also notice some common patterns like + * + * * Because a user message is normally followed by model/assistant response, in most models + * + * * user messages wont have EndOfSentenceTag and + * + * * the following model response wont have BeginOfSentenceTag + * + * * Because a system message will normally be immidiately followed by a user query, + * + * * in many models, there wont be a EndOfSentenceTag following the system message and + * BeginOfSentenceTag wrt the 1st user message following the system message. + * + * * in some models there wont even be a RoleSuffixTag following system message + * and RolePrefixTag wrt the 1st user message following the system message. + * + * * however in many of these models, the subsequent user messages will have the + * BeginOfSentenceTag and or RolePrefixTag. + * + * * Some models may require a BoS for a group of messages, independent of BoS (if any) + * wrt individual roles. + * + * + * ## The Strategy + * + * The configurable template data allows the user to specify the above mentioned tags wrt + * each of the Role as well as any global tag for a group of messages. Depending on whether + * a given model uses/needs a given tag or not you either specify the required tag or else + * you specify a empty string. + * + * A tag could be a single word or multiple words, and may include newline char specified + * using \n and so on. The tag is always demarcated using double quotes and thus also allows + * spaces at the begining or end of the tag, if needed. + * + * In order to account for the conditionality of tags between the system message and the + * following 1st user message, flags are provided to explicitly control whether each of + * these possible tags is used by a specific model or not, as part of its template info. + * + * The Roles are identified in the configurable template data using "system", "user" and + * "assistant". However the model may use different words to identify these roles, in which + * case setup RolePrefix and or RoleSuffix appropriately. + * + * To identify that model is finished with generating response to user query, depending on + * the model's handshake template standard, one will need to set the reverse-prompt to either + * the assistant's suffix or end tag or to the user's begin or prefix tag, depending on what + * is generated by the model at the end of its response. + * + * Currently flags for trimming wrt user text (be it wrt system or user role) is not added. + * + * + * ## Configurable template data and related optional text/JSON file + * + * Can contain the template info wrt multiple models/handshake-standards. And inturn each + * unique template is identified by a unique template id string. + * + * The fields that make up a given chat-handshake-template-standard include + * + * * global -> begin & end + * + * * system -> begin, prefix, suffix & end + * + * * user -> begin, prefix, suffix & end + * + * * assistant -> begin, prefix, suffix & end + * + * * reverse-prompt + * + * * systemuser-system-has-suffix, systemuser-system-has-end, + * systemuser-1st-user-has-begin and systemuser-1st-user-has-prefix + * + * By default one can preload at compile time. Additionally one could update/load + * more at runtime. A compile time optionally enabled load from json helper is + * provided. For any reason, if one doesnt want to use the json based mechanism, + * and instead wants a simple mechanism for runtime updating/loading, one could + * update ChatTemplates to extend from SimpCfg and inturn use its load from a + * simple text file based flow. + * + * + * ## Usage + * + * One could use the logic along with compile time builtin configurable template data as is + * or one could optionally load configurable template data from a text/json file containing + * the template meta data and inturn call the other helper functions as needed. + * + * NOTE: One could either make do with a pre-compiled chat templates info, or allow users + * to update/modify/override the pre-compiled info and or extend with info for new models + * or chat-handshake-template-standards at runtime. + * + * Inturn one can use the helper functions to either extract a given tag or to apply all + * tags specified wrt a given role to the passed message or to apply tags as needed for + * a bunch of messages in one go. + * + * The single message tagging helper setup to apply all tags specified wrt that role. + * + * The multiple messages tagging helper chaton-tmpl-apply[-ex][-capi], will look at the + * boolean flags when tagging the passed messages. In this the system suffix, system end, + * user begin and user prefix get included only if corresponding flag is set, the 1st time + * system + user message is encountered. + * + * The multi messages tagging is provided in two versions. + * * one which returns a single string which contains the tagged message(s) + * * one which returns [ex version] + * * [tagged msg] the string containing the tagged message(s) + * * [parts lengths] an array of integers, which specifies the part lengths, + * which divides the returned string into parts. + * * [parts types] a string where each character indicates whether the corresponding + * part is a normal part which needs to be tokenized without parse_special + * or is a special part which needs to be tokenized with parse-special. + * + * A single message wrapper is provided for the simple (no extended) version. + * + * chaton_llama_tokenize_ex is provided to show how the extended helpers additional + * subparts info wrt tagged message could be used to tokenize with and without + * parse_special to the appropriate subparts that make up the tagged message. + * + * + * ## example/main + * + * The interactive commandline program under example/main, uses + * + * * the system role related tags to tag the system prompt + * * the system prompt includes contents of -p if any + * * followed by contents of file specified using -f if any + * * the user begin+prefix to map to in-prefix + * * the user suffix+end to map to in-suffix + * * the reverse-prompt to map to antiprompt + * * wrt tokenization + * * the user specified system prompt is tokenized with parse_special flag. + * * however the user messages are tokenized with/without parse_special flag, + * based on interactive-specials. + * + * Currently Main doesnt use chaton-tmpl-apply, but only + * * chaton-tmpl-apply-single (for system prompt) and + * * chaton_tmpl_role_getkeys, used to map the user prefix and suffix + * to in-prefix, in-suffix of main. + * * chaton_tmpl_getkey_str, used to map reverse-prompt to main's antiprompt. + * These always adds any role specific begin+prefix and suffix+end around + * the passed message. + * + * ## other uses be it wrt llama.cpp-as-library or examples/server or ... + * + * This module exposes a c-api which is equivalent to the current hardcoded + * templating logic's llama_chat_apply_template. So any program using llama.cpp's + * chat templating logic can be easily migrated to make use of this generic code + * with text based config file based flow. + * + * If a program doesnt want to bring in json dependency into their project, + * one can make do with the pre initialized configurable template data which + * is compiled in. + * + * Additionally, if runtime configurability required without json dependency, + * the ChatTemplates can be updated to extend SimpCfg from common/simpcfg.hpp, + * which provides a simple text based config file format, along with the + * corresponding parser for the same. This should be relatively easy, if needed. + * + * ## Adding support for new model / chat-handshake-template-standard + * + * 1. Add suitable entries wrt configurable template data, either as part of the + * compile time builtin initialisation or the text/json file loaded at runtime, + * for that model/standard. This in itself should work for most of the models. + * + * 2. If some new model introduces a totally different kind of chat-templating + * tag inter/intra mixing, Try to reuse and update the generic flow in + * chaton-tmpl-apply-ex, as much as possible, before trying to add any custom logic. + * + * If you update the generic flow, cross check if existing text/json files will + * need to be updated or not. + * + * + * ## Notes + * + * Look at the sample chaton_meta.json in examples folder for how the above may apply to + * the different llm's out there like + * + * * llama2, llama3, gemma, zephyr, deepseek(normal and coder), monarch, mistral, phi3 + * * chatml, command-r, orion, openchat, vicuna + * + */ + +#include +#include +#include + +#include "log.h" +#include "llama.h" + +#define LOGXLN LOG_TEELN + +const auto K_SYSTEM = "system"; +const auto K_USER = "user"; +const auto K_ASSISTANT = "assistant"; +const auto K_PREFIX = "prefix"; +const auto K_SUFFIX = "suffix"; +const auto K_BEGIN = "begin"; +const auto K_END = "end"; +const auto K_GLOBAL = "global"; +const auto K_SYSTEMUSER_SYSTEM_HAS_SUFFIX = "systemuser-system-has-suffix"; +const auto K_SYSTEMUSER_SYSTEM_HAS_END = "systemuser-system-has-end"; +const auto K_SYSTEMUSER_1ST_USER_HAS_BEGIN = "systemuser-1st-user-has-begin"; +const auto K_SYSTEMUSER_1ST_USER_HAS_PREFIX = "systemuser-1st-user-has-prefix"; +const auto K_REVERSE_PROMPT = "reverse-prompt"; + + + +/** + * Helps keep user prompt and chat-hs-template tag parts seperate, but in sequence. + * Inturn gives the flexibility to tokenize with or without parse_special flag, wrt the different parts of the chat msg(s). + * One could use the triplet of str, get_types and get_partslens to achieve the above mentioned flexibility. + */ +class ChatParts { + + std::vector parts = {}; + std::string types = {""}; + +public: + // Identify string with special tokens that need to be processed. + static const auto S = 's'; + // Identify string which shouldnt have special token processing done. + static const auto N = 'n'; + // Identify no string condition and or ignore string. + static const auto X = '?'; + + ChatParts() : parts{}, types{""} {} + + char last_type() { + if (types.length() == 0) { + return ChatParts::X; + } + return types[types.length()-1]; + } + + void add_part(char type, const std::string &part) { + if (last_type() == type) { + parts[parts.size()-1] += part; + } else { + parts.emplace_back(part); + types += type; + } + } + + std::string str() { + std::string allin = ""; + for(auto part: parts) { + allin += part; + } + return allin; + } + + std::string get_partstypes() { + return types; + } + + std::vector get_partslens() { + std::vector lens = {}; + for(auto part: parts) { + lens.push_back(part.length()); + } + return lens; + } + + std::string name() { + return typeid(*this).name(); + } + + std::string dump(const std::string &msgTag) { + std::stringstream ss; + std::string me = name() + ":" + __func__; + ss << msgTag << ":NumTypes:" << types.length() << std::endl; + ss << msgTag << ":NumParts:" << parts.size() << std::endl; + ss << msgTag << ":StrLength:" << str().length() << std::endl; + if (parts.size() != types.length()) { + LOG_TEELN("DBUG:%s:Mismatch between parts[%zu] and types[%zu]", me.c_str(), parts.size(), types.length()); + } + int i = 0; + for(auto part: parts) { + ss << msgTag << ":Part:" << i << ":" << types[i] << ":" << part << std::endl; + i += 1; + } + return ss.str(); + } + +}; + + +class ChatTemplates : public GroupKV { + +public: + + std::string tmplFallback; + + ChatTemplates(GroupKVMapMapVariant defaultMap) : GroupKV(defaultMap) {} + + /** + * Check if the specified chat-template exists or not. + * NOTE: This doesnt cross check, if the template inturn contains all the required fields or not. + */ + bool tmpl_exists(const std::string &tmpl, const std::string &msgTag="") { + if (!group_exists(tmpl)) { + LOG_TEELN("WARN:CT:%s:%s:Specified template-id [%s] not found...", __func__, msgTag.c_str(), tmpl.c_str()); + return false; + } + return true; + } + + /** + * Check if all expected keys/fields are present wrt the specified chat-template. + * If any key/field is missing, expect a exception. + * + * Additionally also return a string containing info about all the fields. + */ + bool tmpl_basiccheck(const std::string &tmpl, std::stringstream &ss, const std::string &msgTag) { + + if (!tmpl_exists(tmpl, msgTag)) { + return false; + } + + std::string globalBegin = get_value(tmpl, { K_GLOBAL, K_BEGIN }); + std::string globalEnd = get_value(tmpl, { K_GLOBAL, K_END }); + + std::string systemBegin = get_value(tmpl, { K_SYSTEM, K_BEGIN }); + std::string systemPrefix = get_value(tmpl, { K_SYSTEM, K_PREFIX }); + std::string systemSuffix = get_value(tmpl, { K_SYSTEM, K_SUFFIX }); + std::string systemEnd = get_value(tmpl, { K_SYSTEM, K_END }); + + std::string userBegin = get_value(tmpl, { K_USER, K_BEGIN }); + std::string userPrefix = get_value(tmpl, { K_USER, K_PREFIX }); + std::string userSuffix = get_value(tmpl, { K_USER, K_SUFFIX }); + std::string userEnd = get_value(tmpl, { K_USER, K_END }); + + std::string assistantBegin = get_value(tmpl, { K_ASSISTANT, K_BEGIN }); + std::string assistantPrefix = get_value(tmpl, { K_ASSISTANT, K_PREFIX }); + std::string assistantSuffix = get_value(tmpl, { K_ASSISTANT, K_SUFFIX }); + std::string assistantEnd = get_value(tmpl, { K_ASSISTANT, K_END }); + + std::string reversePrompt = get_value(tmpl, { K_REVERSE_PROMPT }); + + bool systemHasSuffix = get_value(tmpl, { K_SYSTEMUSER_SYSTEM_HAS_SUFFIX }); + bool systemHasEnd = get_value(tmpl, { K_SYSTEMUSER_SYSTEM_HAS_END }); + bool userHasBegin = get_value(tmpl, { K_SYSTEMUSER_1ST_USER_HAS_BEGIN }); + bool userHasPrefix = get_value(tmpl, { K_SYSTEMUSER_1ST_USER_HAS_PREFIX }); + + ss << msgTag << ":" + tmpl + ":" << "global-begin" << ":" << globalBegin << std::endl; + ss << msgTag << ":" + tmpl + ":" << "global-end" << ":" << globalEnd << std::endl; + ss << msgTag << ":" + tmpl + ":" << "system-begin" << ":" << systemBegin << std::endl; + ss << msgTag << ":" + tmpl + ":" << "system-prefix" << ":" << systemPrefix << std::endl; + ss << msgTag << ":" + tmpl + ":" << "system-suffix" << ":" << systemSuffix << std::endl; + ss << msgTag << ":" + tmpl + ":" << "system-end" << ":" << systemEnd << std::endl; + ss << msgTag << ":" + tmpl + ":" << "user-begin" << ":" << userBegin << std::endl; + ss << msgTag << ":" + tmpl + ":" << "user-prefix" << ":" << userPrefix << std::endl; + ss << msgTag << ":" + tmpl + ":" << "user-suffix" << ":" << userSuffix << std::endl; + ss << msgTag << ":" + tmpl + ":" << "user-end" << ":" << userEnd << std::endl; + ss << msgTag << ":" + tmpl + ":" << "assistant-begin" << ":" << assistantBegin << std::endl; + ss << msgTag << ":" + tmpl + ":" << "assistant-prefix" << ":" << assistantPrefix << std::endl; + ss << msgTag << ":" + tmpl + ":" << "assistant-suffix" << ":" << assistantSuffix << std::endl; + ss << msgTag << ":" + tmpl + ":" << "assistant-end" << ":" << assistantEnd << std::endl; + ss << msgTag << ":" + tmpl + ":" << K_REVERSE_PROMPT << ":" << reversePrompt << std::endl; + ss << msgTag << ":" + tmpl + ":" << K_SYSTEMUSER_SYSTEM_HAS_SUFFIX << ":" << systemHasSuffix << std::endl; + ss << msgTag << ":" + tmpl + ":" << K_SYSTEMUSER_SYSTEM_HAS_END << ":" << systemHasEnd << std::endl; + ss << msgTag << ":" + tmpl + ":" << K_SYSTEMUSER_1ST_USER_HAS_BEGIN << ":" << userHasBegin << std::endl; + ss << msgTag << ":" + tmpl + ":" << K_SYSTEMUSER_1ST_USER_HAS_PREFIX << ":" << userHasPrefix << std::endl; + + if (!userEnd.empty()) { + LOG_TEELN("WARN:CT:%s:User-End seems to be set to [%s], do cross check if this is proper and needed", msgTag.c_str(), userEnd.c_str()); + } + if (!assistantBegin.empty()) { + LOG_TEELN("WARN:CT:%s:Assistant-Begin seems to be set to [%s], do cross check if this is proper and needed", msgTag.c_str(), assistantBegin.c_str()); + } + + return true; + } + + /** + * For the specified chat-template, get the value associated with the specified key/field. + */ + template + SupportedDataType tmpl_getkey(const std::string &tmpl, const std::string &key, const SupportedDataType &defaultValue) { + return get_value(tmpl, {key}, defaultValue, "CTTmplGetKey"); + } + + /** + * For the specified chat-template and the role within, cumulate the values of the specified keys/fields + * and return the same. + */ + std::string tmpl_role_getkeys(const std::string &tmpl, const std::string &role, const std::vector &keys) { + std::string got = ""; + std::string sKeys = ""; + for(auto key: keys) { + got += get_value(tmpl, {role, key}, "", "CTTmplRoleGetKeys"); + sKeys += "+"; + sKeys += key; + } + LDBUG_LN("DBUG:CT:%s:%s:%s:%s:%s", __func__, tmpl.c_str(), role.c_str(), sKeys.c_str(), got.c_str()); + return got; + } + + /** + * Given the template model/standard id and a bunch of messages including their roles, + * this returns tagged messages, subPartsTypes string and subPartsLens vector. + * The returned subParts types string and lens vector help identify the parts of the + * tagged msgs string, which relate to passed msgs and added tags. + * + * * a string containing the tagged messages + * [global-begin] + 1 or more [[role-begin] + [role-prefix] + msg + [role-suffix] +[role-end]] + [global-end] + * * a string where the chars contain info about + * type of sub-strings/parts that make up the tagged messages string. + * * a vector of ints, + * which give the length of each part in the tagged messages string. + * + * If a combination of system-user messages is passed, then tags between the 1st system and + * the 1st user message, is based on the flags set wrt the corresponding template standard. + * If you dont want this behaviour, pass non 0 values wrt the optional cntSystemMsgCnt and + * cntUserMsgCnt arguments. + */ + bool chaton_tmpl_apply_ex( + const std::string &tmplId, + const std::vector &msgs, + bool alertAssistantAtEnd, + bool applyGlobalIfAny, + std::string &tagged, + std::string &types, + std::vector &lens, + int curSystemMsgCnt = 0, + int curUserMsgCnt = 0 + ) { + std::string tmpl = tmplId; + if (!tmpl_exists(tmpl)) { + tmpl = tmplFallback; + LDBUG_LN("DBUG:%s:Tmpl [%s] missing, fallback to [%s]", __func__, tmplId.c_str(), tmpl.c_str()); + if (!tmpl_exists(tmpl)) { + return false; + } + } + ChatParts cp = {}; + if (applyGlobalIfAny) { + std::string globalBegin = tmpl_role_getkeys(tmpl, K_GLOBAL, {K_BEGIN}); + cp.add_part(ChatParts::S, globalBegin); + } + int cntSystem = curSystemMsgCnt; + int cntUser = curUserMsgCnt; + int cntOthers = 0; + for(const auto msg: msgs) { + auto role = msg->role; + auto content = msg->content; + std::string begin = tmpl_role_getkeys(tmpl, role, {K_BEGIN}); + auto prefix = tmpl_role_getkeys(tmpl, role, {K_PREFIX}); + auto suffix = tmpl_role_getkeys(tmpl, role, {K_SUFFIX}); + auto end = tmpl_role_getkeys(tmpl, role, {K_END}); + if (role == K_SYSTEM) { + cntSystem += 1; + cp.add_part(ChatParts::S, begin); + cp.add_part(ChatParts::S, prefix); + } else if (role == K_USER) { + cntUser += 1; + if ((cntSystem == 1) && (cntUser == 1)) { + if (tmpl_getkey(tmpl, K_SYSTEMUSER_1ST_USER_HAS_BEGIN, true)) { + cp.add_part(ChatParts::S, begin); + } + if (tmpl_getkey(tmpl, K_SYSTEMUSER_1ST_USER_HAS_PREFIX, true)) { + cp.add_part(ChatParts::S, prefix); + } + } else { + cp.add_part(ChatParts::S, begin); + cp.add_part(ChatParts::S, prefix); + } + } else { + cntOthers += 1; + cp.add_part(ChatParts::S, begin); + cp.add_part(ChatParts::S, prefix); + } + cp.add_part(ChatParts::N, content); + if (role == K_SYSTEM) { + if (cntSystem == 1) { + if (tmpl_getkey(tmpl, K_SYSTEMUSER_SYSTEM_HAS_SUFFIX, true)) { + cp.add_part(ChatParts::S, suffix); + } + if (tmpl_getkey(tmpl, K_SYSTEMUSER_SYSTEM_HAS_END, true)) { + cp.add_part(ChatParts::S, end); + } + } else { + cp.add_part(ChatParts::S, suffix); + cp.add_part(ChatParts::S, end); + } + } else { + cp.add_part(ChatParts::S, suffix); + cp.add_part(ChatParts::S, end); + } + } + if (alertAssistantAtEnd) { + auto assistantBeginPrefix = tmpl_role_getkeys(tmpl, K_ASSISTANT, {K_BEGIN, K_PREFIX}); + cp.add_part(ChatParts::S, assistantBeginPrefix); + } + if (applyGlobalIfAny) { + auto globalEnd = tmpl_role_getkeys(tmpl, K_GLOBAL, {K_END}); + cp.add_part(ChatParts::S, globalEnd); + } + LDBUG_LN("DBUG:CT:%s", cp.dump("INFO:ChatOnTmplApplyEx").c_str()); + tagged = cp.str(); + LDBUG_LN("DBUG:CT:%s:%s:%s", __func__, tmpl.c_str(), tagged.c_str()); + LDBUG_LN("DBUG:CT:%s:CntSys[%d]:CntUsr[%d]:CntOthers[%d]", __func__, cntSystem, cntUser, cntOthers); + types = cp.get_partstypes(); + lens = cp.get_partslens(); + return true; + } + +}; + +// The compiled-in configurable template data (the meta) from chaton_meta.cpp +extern ChatTemplates gCT; + + +inline bool chaton_tmpl_exists(const std::string &tmpl) { + return gCT.tmpl_exists(tmpl); +} + +inline std::string chaton_tmpl_role_getkeys(const std::string &tmpl, const std::string &role, const std::vector &keys) { + return gCT.tmpl_role_getkeys(tmpl, role, keys); +} + +inline std::string chaton_tmpl_getkey_str(const std::string &tmpl, const std::string &key) { + return gCT.tmpl_getkey(tmpl, {key}, ""); +} + +inline bool chaton_tmpl_getkey_bool(const std::string &tmpl, const std::string &key) { + return gCT.tmpl_getkey(tmpl, {key}, false); +} + + +// Given the template standard and a bunch of messages including their roles, this returns +// the tagged messages as a string. +// global-begin + 1 or more [[role-begin] + [role-prefix] + msg + [role-suffix] +[role-end]] + global-end +// +// Additionally also return info about the parts that make up the tagged message. +inline bool chaton_tmpl_apply_ex( + const std::string &tmpl, + const std::vector &msgs, + bool alertAssistantAtEnd, + bool applyGlobalIfAny, + std::string &tagged, + std::string &types, + std::vector &lens, + int curSystemMsgCnt = 0, + int curUserMsgCnt = 0 + ) { + return gCT.chaton_tmpl_apply_ex(tmpl, msgs, alertAssistantAtEnd, applyGlobalIfAny, tagged, types, lens, curSystemMsgCnt, curUserMsgCnt); +} + +// Given the template standard and a bunch of messages including their roles, this returns +// the tagged messages as a string. +// global-begin + 1 or more [[role-begin] + [role-prefix] + msg + [role-suffix] +[role-end]] + global-end +inline int32_t chaton_tmpl_apply( + const std::string &tmpl, + const std::vector &msgs, + bool alertAssistantAtEnd, + bool applyGlobalIfAny, + std::string &tagged + ) { + std::string types; + std::vector lens; + if (!chaton_tmpl_apply_ex(tmpl, msgs, alertAssistantAtEnd, applyGlobalIfAny, tagged, types, lens)) { + return -1; + } + return tagged.size(); +} + +const int BYPASS_MSGCNT = 101; +// +// Given the template standard, role and a message, this creates the tagged message. +// +// string containing the tagged message +// * role-(begin+prefix) + msg + role-(suffix+end) +// +// ALERT: This currently assumes/behaves as if the system or user message it is working on +// is a non-1st message belonging to that role. +// +inline size_t chaton_tmpl_apply_single( + const std::string &tmpl, + const std::string &role, + const std::string &content, + bool alertAssistantAtEnd, + bool applyGlobalIfAny, + std::string &tagged + ) { + std::string types; + std::vector lens; + llama_chat_message cm {role.c_str(), content.c_str()}; + if (!chaton_tmpl_apply_ex(tmpl, {&cm}, alertAssistantAtEnd, applyGlobalIfAny, tagged, types, lens, BYPASS_MSGCNT, BYPASS_MSGCNT)) { + return -1; + } + return tagged.size(); +} + +// Given the template standard and a bunch of messages including their roles, this returns +// the tagged messages as a string. +// global-begin + 1 or more [[role-begin] + [role-prefix] + msg + [role-suffix] +[role-end]] + global-end +// +// If the passed char array is smaller than that required for the tagged messages string, +// * part of the tagged messages string which fits within dest buffer is copied +// * the returned value, indicates the size of the actual tagged message +// +// NOTE: +// * ideally the passed char array should be able to fit the tagged messages string + 0|null char. +// * if the return value from this function is larger than or equal to destLength, +// then you will have to increase the size of the dest buffer, and call this +// function a second time, to ensure that one gets the full tagged messages string. +inline int32_t chaton_tmpl_apply_capi( + const char *tmpl, + const struct llama_chat_message *msgs, + const size_t numMsgs, + bool alertAssistantAtEnd, + char *dest, + int32_t destLength + ) { + if ((tmpl == nullptr) || (dest == nullptr)) { + return -1; + } + std::vector vMsgs; + for(size_t i=0; i 0) { + strlcpy(dest, taggedMsgs.c_str(), destLength); + } + return taggedLength; +} + +// +// In addition to the semantic provided by chaton_tmpl_apply_capi +// this additionally also returns info about the parts that make up +// the returned tagged message. +// +// partsTypes and partsLengths should be arrays that can accomodate the +// same number of elements belonging to its respective type. +// Inturn the pNumParts should point to a int which specifies the +// number of elements. +// If the generated tagged message has more parts than the specified +// *pNumParts, then the logic copies partsTypes and partsLengths to the +// specified length/NumOfParts only. Parallely it updates *pNumParts +// to the actual needed length (not including any terminating null char or so). +// +inline int32_t chaton_tmpl_apply_ex_capi( + const char *tmpl, + const struct llama_chat_message *msgs, + const size_t numMsgs, + bool alertAssistantAtEnd, + char *dest, + int32_t destLength, + char *partsTypes, + int32_t *partsLengths, + int32_t *pNumParts + ) { + if ((tmpl == nullptr) || (dest == nullptr) || (pNumParts == nullptr)) { + return -1; + } + std::vector vMsgs; + for(size_t i=0; i lens; + if (!chaton_tmpl_apply_ex(tmpl, vMsgs, alertAssistantAtEnd, true, taggedMsgs, types, lens)) { + return -1; + } + int32_t taggedLength = taggedMsgs.size(); + if (taggedLength < 0) { + return taggedLength; + } + if (destLength > 0) { + strlcpy(dest, taggedMsgs.c_str(), destLength); + } + if (*pNumParts > 0) { + if (partsTypes != nullptr) { + strlcpy(partsTypes, types.c_str(), *pNumParts); + } + if (partsLengths != nullptr) { + memcpy(partsLengths, lens.data(), (*pNumParts)*sizeof(int32_t)); + } + } + *pNumParts = types.length(); + return taggedLength; +} + +// Copied from common.cpp, updated wrt model and logging flow. +inline std::vector chaton_llama_tokenize( + const struct llama_model * model, + const std::string & text, + bool add_special, + bool parse_special) { + LDBUG_LN("DBUG:%s:%s:special[add:%d, parse:%d]", __func__, text.c_str(), add_special, parse_special); + if (model == nullptr) { + LOG_TEELN("ERRR:%s:Model NOT Provided:%s:special[add:%d, parse:%d]", __func__, text.c_str(), add_special, parse_special); + return std::vector{}; + } + // upper limit for the number of tokens + int n_tokens = text.length() + 2 * add_special; + std::vector result(n_tokens); + n_tokens = llama_tokenize(model, text.data(), text.length(), result.data(), result.size(), add_special, parse_special); + if (n_tokens < 0) { + result.resize(-n_tokens); + int check = llama_tokenize(model, text.data(), text.length(), result.data(), result.size(), add_special, parse_special); + GGML_ASSERT(check == -n_tokens); + } else { + result.resize(n_tokens); + } + return result; +} + +// Tokenize the passed taggedText, keeping in mind the subparts within and +// inturn whether to parse special tokens in them or not (partsTypes). +// If you want to parse special tokens in the taggedText, independent of what +// partsTypes specifies, then set forceParseSpecial to true. +inline std::vector chaton_llama_tokenize_ex( + const struct llama_model *model, + const std::string &taggedText, + const std::string &partsTypes, + const std::vector &partsLengths, + bool addSpecial, + bool forceParseSpecial + ) { + std::vector tokens; + int iPart = 0; + int iStart = 0; + for(auto partLen: partsLengths) { + auto partType = partsTypes[iPart]; + iPart += 1; + auto msgPart = taggedText.substr(iStart, partLen); + iStart += partLen; + auto parseSpecial = partType == ChatParts::S ? true : false; + parseSpecial |= forceParseSpecial; + auto curTokens = chaton_llama_tokenize(model, msgPart, addSpecial, parseSpecial); + tokens.insert(tokens.end(), curTokens.begin(), curTokens.end()); + } + return tokens; +} + + +inline void chaton_meta_fallback(std::string &tmpl, ChatTemplates *ct=nullptr) { + if (ct == nullptr) { + ct = &gCT; + } + ct->tmplFallback = tmpl; +} + +/** + * Validate specified chaton-template-id and inturn dump the contents related to that + * specific chat-handshake-template-standard, wrt the specified ChatTemplates. + * If ct is nullptr, then map to the compiled-in ChatTemplates global instance. + * + * ALERT: If no template-id is specified, it is ignored with a warning. + * NOTE: It optionally dumps the full loaded chaton templates data + * NOTE: It uses tmpl_basiccheck, which raises exception, if all the required + * keys/fields are not present wrt the specified template-standard/model-id. + */ +inline bool _chaton_meta_validate_dump(std::string &tmpl, ChatTemplates *ct=nullptr) { + if (ct == nullptr) { + ct = &gCT; + } + LDBUG_LN("\n\nINFO:%s:%s:\n%s", __func__, tmpl.c_str(), ct->dump("", "INFO:ChatOnMetaValidateDump").c_str()); + if (tmpl.empty()) { + return true; + } + std::stringstream ss; + if (ct->tmpl_basiccheck(tmpl, ss, "INFO:ChatOnMetaValidateDump")) { + LOGXLN("%s", ss.str().c_str()); + } else { + return false; + } + return true; +} + +/** + * In the passed ChatTemplates instance, verify that specified chaton-template-id + * contains required fields using meta-validate-dump. + * If ct is nullptr, then map to the compiled-in ChatTemplates global instance. + */ +inline bool chaton_meta_ok(std::string &tmpl, ChatTemplates *ct=nullptr) { + return _chaton_meta_validate_dump(tmpl, ct); +} diff --git a/common/chaton_json.hpp b/common/chaton_json.hpp new file mode 100644 index 0000000000000..7915adcc43533 --- /dev/null +++ b/common/chaton_json.hpp @@ -0,0 +1,100 @@ +#pragma once + +/** + * Helper to load chaton's configurable template data from json file + * By Humans for All + * + * Any program which wants to load configurable template data from json file, + * can include this file to get the needed helpers for same. +*/ + +#include "chaton.hpp" + +#include +using json = nlohmann::ordered_json; + + +// Get value corresponding to the specified hierarchy/chain of keys. +// Also throw a more informative exception, if it is not found. +template +inline SupportedType json_get(json &j, const std::vector &keys, const std::string &msgTag) { + json curJ = j; + std::stringstream skey; + int i = 0; + for(auto key: keys) { + if (i != 0) skey << "-"; + i += 1; + skey << key; + if (curJ.contains(key)) { + curJ = curJ[key]; + } else { + std::stringstream ss; + ss << "ERRR:ChatON:" << __func__ << ":" << msgTag << ":KeyChain [" << skey.str() << "] is missing"; + throw std::runtime_error(ss.str()); + } + } + return curJ; +} + +// Update/Extend the configurable template data in specified ChatTemplates instance from the specified json file. +// If nullptr is passed wrt ct, then update/extend the global compiled-in configurable template data. +inline bool chaton_meta_load_json(const std::string &fname, ChatTemplates *ct=nullptr) { + if (ct == nullptr) { + ct = &gCT; + } + std::ifstream f(fname); + json conMeta = json::parse(f); + for(auto it=conMeta.begin(); it != conMeta.end(); ++it) { + + auto group = it.key(); + auto curTmpl = conMeta[group]; + + std::string globalBegin = json_get(curTmpl, { K_GLOBAL, K_BEGIN }, group); + ct->set_value(group, { K_GLOBAL, K_BEGIN }, globalBegin); + std::string globalEnd = json_get(curTmpl, { K_GLOBAL, K_END }, group); + ct->set_value(group, { K_GLOBAL, K_END }, globalEnd); + + std::string systemBegin = json_get(curTmpl, { K_SYSTEM, K_BEGIN }, group); + ct->set_value(group, { K_SYSTEM, K_BEGIN }, systemBegin); + std::string systemPrefix = json_get(curTmpl, { K_SYSTEM, K_PREFIX }, group); + ct->set_value(group, { K_SYSTEM, K_PREFIX }, systemPrefix); + std::string systemSuffix = json_get(curTmpl, { K_SYSTEM, K_SUFFIX }, group); + ct->set_value(group, { K_SYSTEM, K_SUFFIX }, systemSuffix); + std::string systemEnd = json_get(curTmpl, { K_SYSTEM, K_END }, group); + ct->set_value(group, { K_SYSTEM, K_END }, systemEnd); + + std::string userBegin = json_get(curTmpl, { K_USER, K_BEGIN }, group); + ct->set_value(group, { K_USER, K_BEGIN }, userBegin); + std::string userPrefix = json_get(curTmpl, { K_USER, K_PREFIX }, group); + ct->set_value(group, { K_USER, K_PREFIX }, userPrefix); + std::string userSuffix = json_get(curTmpl, { K_USER, K_SUFFIX }, group); + ct->set_value(group, { K_USER, K_SUFFIX }, userSuffix); + std::string userEnd = json_get(curTmpl, { K_USER, K_END }, group); + ct->set_value(group, { K_USER, K_END }, userEnd); + + std::string assistantBegin = json_get(curTmpl, { K_ASSISTANT, K_BEGIN }, group); + ct->set_value(group, { K_ASSISTANT, K_BEGIN }, assistantBegin); + std::string assistantPrefix = json_get(curTmpl, { K_ASSISTANT, K_PREFIX }, group); + ct->set_value(group, { K_ASSISTANT, K_PREFIX }, assistantPrefix); + std::string assistantSuffix = json_get(curTmpl, { K_ASSISTANT, K_SUFFIX }, group); + ct->set_value(group, { K_ASSISTANT, K_SUFFIX }, assistantSuffix); + std::string assistantEnd = json_get(curTmpl, { K_ASSISTANT, K_END }, group); + ct->set_value(group, { K_ASSISTANT, K_END }, assistantEnd); + + std::string reversePrompt = json_get(curTmpl, { K_REVERSE_PROMPT }, group); + ct->set_value(group, { K_REVERSE_PROMPT }, reversePrompt); + + bool systemHasSuffix = json_get(curTmpl, { K_SYSTEMUSER_SYSTEM_HAS_SUFFIX }, group); + ct->set_value(group, { K_SYSTEMUSER_SYSTEM_HAS_SUFFIX }, systemHasSuffix); + bool systemHasEnd = json_get(curTmpl, { K_SYSTEMUSER_SYSTEM_HAS_END }, group); + ct->set_value(group, { K_SYSTEMUSER_SYSTEM_HAS_END }, systemHasEnd); + + bool userHasBegin = json_get(curTmpl, { K_SYSTEMUSER_1ST_USER_HAS_BEGIN }, group); + ct->set_value(group, { K_SYSTEMUSER_1ST_USER_HAS_BEGIN }, userHasBegin); + bool userHasPrefix = json_get(curTmpl, { K_SYSTEMUSER_1ST_USER_HAS_PREFIX }, group); + ct->set_value(group, { K_SYSTEMUSER_1ST_USER_HAS_PREFIX }, userHasPrefix); + + } + LDBUG_LN("%s", ct->dump("", "DBUG:ChatONMetaLoad:ChatTemplates").c_str()); + return true; +} diff --git a/common/chaton_meta.cpp b/common/chaton_meta.cpp new file mode 100644 index 0000000000000..d0013b519d052 --- /dev/null +++ b/common/chaton_meta.cpp @@ -0,0 +1,300 @@ +//This is auto created/converted from chaton-meta-json file + +#include "chaton.hpp" + +ChatTemplates gCT = {{ + { "llama2", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "" }, + { "system-prefix", "[INST] <>\n" }, + { "system-suffix", "\n<> " }, + { "system-end", "[/INST]\n\n" }, + { "user-begin", "" }, + { "user-prefix", "[INST] " }, + { "user-suffix", " [/INST]\n\n" }, + { "user-end", "" }, + { "assistant-begin", "" }, + { "assistant-prefix", "" }, + { "assistant-suffix", "" }, + { "assistant-end", "" }, + { "reverse-prompt", "" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", false }, + { "systemuser-1st-user-has-begin", false }, + { "systemuser-1st-user-has-prefix", false }, + }}, + { "llama3", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "" }, + { "system-prefix", "<|start_header_id|>system<|end_header_id|>\n" }, + { "system-suffix", "<|eot_id|>\n\n" }, + { "system-end", "" }, + { "user-begin", "" }, + { "user-prefix", "<|start_header_id|>user<|end_header_id|>\n" }, + { "user-suffix", "<|eot_id|>\n\n" }, + { "user-end", "" }, + { "assistant-begin", "" }, + { "assistant-prefix", "<|start_header_id|>assistant<|end_header_id|>\n" }, + { "assistant-suffix", "<|eot_id|>\n\n" }, + { "assistant-end", "" }, + { "reverse-prompt", "<|eot_id|>" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", true }, + { "systemuser-1st-user-has-begin", true }, + { "systemuser-1st-user-has-prefix", true }, + }}, + { "chatml", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "" }, + { "system-prefix", "<|im_start|>system\n" }, + { "system-suffix", "<|im_end|>\n" }, + { "system-end", "" }, + { "user-begin", "" }, + { "user-prefix", "<|im_start|>user\n" }, + { "user-suffix", "<|im_end|>\n" }, + { "user-end", "" }, + { "assistant-begin", "" }, + { "assistant-prefix", "<|im_start|>assistant\n" }, + { "assistant-suffix", "<|im_end|>\n" }, + { "assistant-end", "" }, + { "reverse-prompt", "<|im_start|>user\n" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", true }, + { "systemuser-1st-user-has-begin", true }, + { "systemuser-1st-user-has-prefix", true }, + }}, + { "zephyr", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "" }, + { "system-prefix", "<|system|>\n" }, + { "system-suffix", "" }, + { "system-end", "\n" }, + { "user-begin", "" }, + { "user-prefix", "<|user|>\n" }, + { "user-suffix", "" }, + { "user-end", "\n" }, + { "assistant-begin", "" }, + { "assistant-prefix", "<|assistant|>\n" }, + { "assistant-suffix", "" }, + { "assistant-end", "\n" }, + { "reverse-prompt", "" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", true }, + { "systemuser-1st-user-has-begin", true }, + { "systemuser-1st-user-has-prefix", true }, + }}, + { "gemma", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "" }, + { "system-prefix", "user\n" }, + { "system-suffix", "\n" }, + { "system-end", "" }, + { "user-begin", "" }, + { "user-prefix", "user\n" }, + { "user-suffix", "\n" }, + { "user-end", "" }, + { "assistant-begin", "" }, + { "assistant-prefix", "model\n" }, + { "assistant-suffix", "\n" }, + { "assistant-end", "" }, + { "reverse-prompt", "" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", false }, + { "systemuser-1st-user-has-begin", true }, + { "systemuser-1st-user-has-prefix", true }, + }}, + { "deepseek-coder", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "\n<｜begin▁of▁sentence｜>" }, + { "system-prefix", "" }, + { "system-suffix", "\n" }, + { "system-end", "" }, + { "user-begin", "" }, + { "user-prefix", "### Instruction:\n" }, + { "user-suffix", "\n" }, + { "user-end", "" }, + { "assistant-begin", "" }, + { "assistant-prefix", "### Response:\n" }, + { "assistant-suffix", "\n<|EOT|>\n" }, + { "assistant-end", "" }, + { "reverse-prompt", "<|EOT|>" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", false }, + { "systemuser-1st-user-has-begin", false }, + { "systemuser-1st-user-has-prefix", true }, + }}, + { "deepseek", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "<｜begin▁of▁sentence｜>" }, + { "system-prefix", "" }, + { "system-suffix", "\n\n" }, + { "system-end", "" }, + { "user-begin", "<｜begin▁of▁sentence｜>" }, + { "user-prefix", "User: " }, + { "user-suffix", "\n\n" }, + { "user-end", "" }, + { "assistant-begin", "" }, + { "assistant-prefix", "Assistant: " }, + { "assistant-suffix", " <｜end▁of▁sentence｜>\n" }, + { "assistant-end", "" }, + { "reverse-prompt", "<｜end▁of▁sentence｜>" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", false }, + { "systemuser-1st-user-has-begin", false }, + { "systemuser-1st-user-has-prefix", true }, + }}, + { "monarch", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "" }, + { "system-prefix", "system\n" }, + { "system-suffix", "" }, + { "system-end", "\n" }, + { "user-begin", "" }, + { "user-prefix", "user\n" }, + { "user-suffix", "" }, + { "user-end", "\n" }, + { "assistant-begin", "" }, + { "assistant-prefix", "assistant\n" }, + { "assistant-suffix", "" }, + { "assistant-end", "\n" }, + { "reverse-prompt", "" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", true }, + { "systemuser-1st-user-has-begin", true }, + { "systemuser-1st-user-has-prefix", true }, + }}, + { "mistral", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "" }, + { "system-prefix", "[INST] " }, + { "system-suffix", " [/INST]" }, + { "system-end", "\n" }, + { "user-begin", "" }, + { "user-prefix", "[INST] " }, + { "user-suffix", " [/INST]" }, + { "user-end", "\n" }, + { "assistant-begin", "" }, + { "assistant-prefix", "" }, + { "assistant-suffix", "" }, + { "assistant-end", " \n" }, + { "reverse-prompt", "" }, + { "systemuser-system-has-suffix", false }, + { "systemuser-system-has-end", true }, + { "systemuser-1st-user-has-begin", false }, + { "systemuser-1st-user-has-prefix", false }, + }}, + { "phi3", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "" }, + { "system-prefix", "<|system|>\n" }, + { "system-suffix", "<|end|>\n" }, + { "system-end", "" }, + { "user-begin", "" }, + { "user-prefix", "<|user|>\n" }, + { "user-suffix", "<|end|>\n" }, + { "user-end", "" }, + { "assistant-begin", "" }, + { "assistant-prefix", "<|assistant|>\n" }, + { "assistant-suffix", "<|end|>\n" }, + { "assistant-end", "" }, + { "reverse-prompt", "<|end|>" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", true }, + { "systemuser-1st-user-has-begin", true }, + { "systemuser-1st-user-has-prefix", true }, + }}, + { "command-r", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "" }, + { "system-prefix", "<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>" }, + { "system-suffix", "<|END_OF_TURN_TOKEN|>" }, + { "system-end", "" }, + { "user-begin", "" }, + { "user-prefix", "<|START_OF_TURN_TOKEN|><|USER_TOKEN|>" }, + { "user-suffix", "<|END_OF_TURN_TOKEN|>" }, + { "user-end", "" }, + { "assistant-begin", "" }, + { "assistant-prefix", "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>" }, + { "assistant-suffix", "<|END_OF_TURN_TOKEN|>" }, + { "assistant-end", "" }, + { "reverse-prompt", "<|END_OF_TURN_TOKEN|>" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", true }, + { "systemuser-1st-user-has-begin", true }, + { "systemuser-1st-user-has-prefix", true }, + }}, + { "orion", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "" }, + { "system-prefix", "Human: " }, + { "system-suffix", "\n\n" }, + { "system-end", "" }, + { "user-begin", "" }, + { "user-prefix", "Human: " }, + { "user-suffix", "\n\nAssistant: " }, + { "user-end", "" }, + { "assistant-begin", "" }, + { "assistant-prefix", "" }, + { "assistant-suffix", "" }, + { "assistant-end", "" }, + { "reverse-prompt", "" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", true }, + { "systemuser-1st-user-has-begin", false }, + { "systemuser-1st-user-has-prefix", false }, + }}, + { "openchat", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "" }, + { "system-prefix", "GPT4 Correct System: " }, + { "system-suffix", "<|end_of_turn|>" }, + { "system-end", "" }, + { "user-begin", "" }, + { "user-prefix", "GPT4 Correct User: " }, + { "user-suffix", "<|end_of_turn|>" }, + { "user-end", "" }, + { "assistant-begin", "" }, + { "assistant-prefix", "GPT4 Correct Assistant: " }, + { "assistant-suffix", "<|end_of_turn|>" }, + { "assistant-end", "" }, + { "reverse-prompt", "<|end_of_turn|>" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", true }, + { "systemuser-1st-user-has-begin", true }, + { "systemuser-1st-user-has-prefix", true }, + }}, + { "vicuna", { + { "global-begin", "" }, + { "global-end", "" }, + { "system-begin", "" }, + { "system-prefix", "SYSTEM: " }, + { "system-suffix", "\n\n" }, + { "system-end", "" }, + { "user-begin", "" }, + { "user-prefix", "USER: " }, + { "user-suffix", "\n" }, + { "user-end", "" }, + { "assistant-begin", "" }, + { "assistant-prefix", "ASSISTANT: " }, + { "assistant-suffix", "\n" }, + { "assistant-end", "" }, + { "reverse-prompt", "" }, + { "systemuser-system-has-suffix", true }, + { "systemuser-system-has-end", true }, + { "systemuser-1st-user-has-begin", true }, + { "systemuser-1st-user-has-prefix", true }, + }}, +}}; diff --git a/common/common.cpp b/common/common.cpp index 96130ad543553..037627a9a5a19 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -925,6 +925,28 @@ bool gpt_params_find_arg(int argc, char ** argv, const std::string & arg, gpt_pa params.chatml = true; return true; } + if (arg == "--chaton-meta-json") { + if (++i >= argc) { + invalid_param = true; + return true; + } + params.chaton_meta_json = argv[i]; + return true; + } + if (arg == "--chaton-template-id") { + if (++i >= argc) { + invalid_param = true; + return true; + } + std::string got = argv[i]; + std::regex whitespaces(R"(\s+)"); + std::string trimmed = std::regex_replace(got, whitespaces, ""); + if (!trimmed.empty()) { + params.chaton_template_id = trimmed; + params.chaton = true; + } + return true; + } if (arg == "--infill") { params.infill = true; return true; @@ -1437,6 +1459,10 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) { printf(" -cnv, --conversation run in conversation mode (does not print special tokens and suffix/prefix)\n"); printf(" -ins, --instruct run in instruction mode (use with Alpaca models)\n"); printf(" -cml, --chatml run in chatml mode (use with ChatML-compatible models)\n"); + printf(" --chaton-meta-json JsonFile\n"); + printf(" specify the json file containing chat-handshake-template-standard(s)\n"); + printf(" --chaton-template-id ChatHandshakeTemplateId\n"); + printf(" specify the specific template standard to use from loaded json file\n"); printf(" --multiline-input allows you to write or paste multiple lines without ending each in '\\'\n"); printf(" -r PROMPT, --reverse-prompt PROMPT\n"); printf(" halt generation at PROMPT, return control in interactive mode\n"); diff --git a/common/common.h b/common/common.h index 566490e2f881a..9659b246f94b2 100644 --- a/common/common.h +++ b/common/common.h @@ -144,6 +144,9 @@ struct gpt_params { bool interactive_specials = false; // whether to allow special tokens from user, during interactive mode bool conversation = false; // conversation mode (does not print special tokens and suffix/prefix) bool chatml = false; // chatml mode (used for models trained on chatml syntax) + bool chaton = false; // whether chaton is enabled or disabled + std::string chaton_meta_json = ""; // name of the json file containing the chaton templates + std::string chaton_template_id = ""; // the specific chat-handshake-template-standard to use bool prompt_cache_all = false; // save user input and generations to prompt cache bool prompt_cache_ro = false; // open the prompt cache read-only and do not update it diff --git a/common/datautils_string.hpp b/common/datautils_string.hpp new file mode 100644 index 0000000000000..f790d48b738b9 --- /dev/null +++ b/common/datautils_string.hpp @@ -0,0 +1,231 @@ +#pragma once + +/** + * A bunch of helper routines to work with strings. + * by Humans for All + * + * ## Some notes for later + * + * NativeCharSize encoded char refers to chars which fit within the size of char type in a given + * type of c++ string or base bitsize of a encoding standard, like 1 byte in case of std::string, + * utf-8, ... + * * example english alphabets in utf-8 encoding space are 1byte chars, in its variable length + * encoding space. + * + * MultiNativeCharSize encoded char refers to chars which occupy multiple base-char-bit-size of + * a c++ string type or char encoding standard. + * * example indian scripts alphabets in utf-8 encoding space occupy multiple bytes in its variable + * length encoding space. + * + * Sane variable length encoding - refers to encoding where the values of NativeCharSized chars of + * a char encoding space cant overlap with values in NativeCharSize subparts of MultiNativeCharSized + * chars of the same char encoding standard. + * * utf-8 shows this behaviour + * * chances are utf-16 and utf-32 also show this behaviour (need to cross check once) + * +*/ + +#include +#include + +#include "log.h" + + +#undef DUS_DEBUG_VERBOSE + +#undef DUS_STR_OVERSMART +#ifdef DUS_STR_OVERSMART +#define str_trim str_trim_oversmart +#else +#define str_trim str_trim_dumb +#endif + + +inline size_t wcs_to_mbs(std::string &sDest, const std::wstring &wSrc) { + std::mbstate_t mbState = std::mbstate_t(); + const wchar_t *wSrcP = wSrc.c_str(); + auto reqLen = std::wcsrtombs(nullptr, &wSrcP, 0, &mbState); + if (reqLen == static_cast(-1)) { + throw std::runtime_error("ERRR:WCS2MBS:Failed probing of size..."); + } + sDest.resize(reqLen); + return std::wcsrtombs(sDest.data(), &wSrcP, sDest.length(), &mbState); +} + +inline size_t mbs_to_wcs(std::wstring &wDest, const std::string &sSrc) { + std::mbstate_t mbState = std::mbstate_t(); + const char *sSrcP = sSrc.c_str(); + auto reqLen = std::mbsrtowcs(nullptr, &sSrcP, 0, &mbState); + if (reqLen == static_cast(-1)) { + throw std::runtime_error("ERRR:MBS2WCS:Failed probing of size..."); + } + wDest.resize(reqLen); + return std::mbsrtowcs(wDest.data(), &sSrcP, wDest.length(), &mbState); +} + +inline std::string uint8_as_hex(uint8_t c) { + char hex[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e', 'f' }; + std::string out = "00"; + out[0] = hex[((c & 0xf0) >> 4)]; + out[1] = hex[(c & 0x0f)]; + return out; +} + +template +inline std::string string_as_hex(const TString &sIn){ + std::stringstream ssout; + ssout << "[ "; + for(auto c: sIn) { + auto cSize = sizeof(c); + if (cSize == 1) { + ssout << uint8_as_hex(c) << ", "; + } else if (cSize == 2) { + ssout << std::setfill('0') << std::setw(cSize*2) << std::hex << static_cast(c) << ", "; + } else if (cSize == 4) { + ssout << std::setfill('0') << std::setw(cSize*2) << std::hex << static_cast(c) << ", "; + } else { + std::stringstream ss; + ss << "ERRR:" << __func__ << ":Unsupported char type with size [" << cSize << "]"; + throw std::runtime_error( ss.str().c_str() ); + } + } + ssout << " ]"; + return ssout.str(); +} + +// Remove chars from begin and end of the passed string, provided the char +// belongs to one of the chars in trimChars. +// +// NOTE: This will work perfectly provided the string being trimmed as well as +// chars being trimmed are made up of NativeCharSize chars from same encoded space. +// For utf-8, this means the ascii equivalent 1byteSized chars of utf8 and not +// variable length MultiNativeCharSize (ie multibye in case of utf-8) ones. +// NOTE: It will also work, if atleast either end of string as well as trimChars +// have NativeCharSize chars from their encoding space, rather than variable +// length MultiNativeCharSize based chars if any. There needs to be NativeCharSized +// chars beyond any chars that get trimmed, on either side. +// +// NOTE: Given the way UTF-8 char encoding is designed, where NativeCharSize 1byte +// encoded chars are fully unique and dont overlap with any bytes from any of the +// variable length MultiNativeCharSize encoded chars in the utf-8 space, so as long as +// the trimChars belong to NativeCharSize chars subset, the logic should work, even +// if string has a mixture of NativeCharSize and MultiNativeCharSize encoded chars. +// Chances are utf-16 and utf-32 also have similar characteristics wrt thier +// NativeCharSize encoded chars (ie those fully encoded within single 16bit and 32bit +// value respectively), and so equivalent semantic applies to them also. +// +// ALERT: Given that this simple minded logic, works at individual NativeCharSize level +// only, If trimChars involve variable length MultiNativeCharSize encoded chars, then +// * because different NativeCharSize subparts (bytes in case of utf-8) from different +// MultiNativeCharSize trim chars when clubbed together can map to some other new char +// in a variable length encoded char space, if there is that new char at either end +// of the string, it may get trimmed, because of the possibility of mix up mentioned. +// * given that different variable length MultiNativeCharSize encoded chars may have +// some common NativeCharSize subparts (bytes in case of utf-8) between them, if one +// of these chars is at either end of the string and another char is in trimChars, +// then string may get partially trimmed wrt such a char at either end. +// +template +inline TString str_trim_dumb(TString sin, const TString &trimChars=" \t\n") { +#ifdef DUS_DEBUG_VERBOSE + LOG_TEELN("DBUG:StrTrimDumb:Str:%s", string_as_hex(sin).c_str()); + LOG_TEELN("DBUG:StrTrimDumb:TrimChars:%s", string_as_hex(trimChars).c_str()); +#endif + sin.erase(sin.find_last_not_of(trimChars)+1); + sin.erase(0, sin.find_first_not_of(trimChars)); + return sin; +} + +// Remove chars from begin and end of the passed string, provided the char belongs +// to one of the chars in trimChars. +// NOTE: Internally converts to wchar/wstring to try and support proper trimming, +// wrt possibly more languages, to some extent. IE even if the passed string +// contains multibyte encoded characters in it in utf-8 space (ie MultiNativeCharSize), +// it may get converted to NativeCharSize chars in the expanded wchar_t encoding space, +// thus leading to fixed NativeCharSize driven logic itself handling things sufficiently. +// Look at str_trim_dumb comments for additional aspects. +inline std::string str_trim_oversmart(std::string sIn, const std::string &trimChars=" \t\n") { + std::wstring wIn; + mbs_to_wcs(wIn, sIn); + std::wstring wTrimChars; + mbs_to_wcs(wTrimChars, trimChars); + auto wOut = str_trim_dumb(wIn, wTrimChars); + std::string sOut; + wcs_to_mbs(sOut, wOut); + return sOut; +} + +// Remove atmost 1 char at the begin and 1 char at the end of the passed string, +// provided the char belongs to one of the chars in trimChars. +// +// NOTE: Chars being trimmed (ie in trimChars) needs to be part of NativeCharSize +// subset of the string's encoded char space, to avoid mix up when working with +// strings which can be utf-8/utf-16/utf-32/sane-variable-length encoded strings. +// +// NOTE:UTF8: This will work provided the string being trimmed as well the chars +// being trimmed are made up of 1byte encoded chars in case of utf8 encoding space. +// If the string being trimmed includes multibyte (ie MultiNativeCharSize) encoded +// characters at either end, then trimming can mess things up, if you have multibyte +// encoded utf-8 chars in the trimChars set. +// +// Currently given that SimpCfg only uses this with NativeCharSize chars in the +// trimChars and most of the platforms are likely to be using utf-8 based char +// space (which is a realtively sane variable length char encoding from this +// logics perspective), so not providing oversmart variant. +// +template +inline TString str_trim_single(TString sin, const TString& trimChars=" \t\n") { + if (sin.empty()) return sin; + for(auto c: trimChars) { + if (c == sin.front()) { + sin = sin.substr(1, TString::npos); + break; + } + } + if (sin.empty()) return sin; + for(auto c: trimChars) { + if (c == sin.back()) { + sin = sin.substr(0, sin.length()-1); + break; + } + } + return sin; +} + +// Convert to lower case, if language has upper and lower case semantic +// +// This works for fixed size encoded char spaces. +// +// For variable length encoded char spaces, it can work +// * if one is doing the conversion for languages which fit into NativeCharSized chars in it +// * AND if one is working with a sane variable length encoding standard +// * ex: this will work if trying to do the conversion for english language within utf-8 +// +template +inline TString str_tolower(const TString &sin) { + TString sout; + sout.resize(sin.size()); + std::transform(sin.begin(), sin.end(), sout.begin(), [](auto c)->auto {return std::tolower(c);}); +#ifdef DUS_DEBUG_VERBOSE + LOG_TEELN("DBUG:StrToLower:in:%s", string_as_hex(sin).c_str()); + LOG_TEELN("DBUG:StrToLower:out:%s", string_as_hex(sout).c_str()); +#endif + return sout; +} + +inline void str_compare_dump(const std::string &s1, const std::string &s2) { + LOG_TEELN("DBUG:%s:%s:Len:%zu", __func__, s1.c_str(), s1.length()); + LOG_TEELN("DBUG:%s:%s:Len:%zu", __func__, s2.c_str(), s2.length()); + int minLen = s1.length() < s2.length() ? s1.length() : s2.length(); + for(int i=0; i +std::string as_str(TypeWithStrSupp value) { + std::stringstream ss; + ss << value; + return ss.str(); +} diff --git a/common/groupkv.hpp b/common/groupkv.hpp new file mode 100644 index 0000000000000..220ee9b33b49f --- /dev/null +++ b/common/groupkv.hpp @@ -0,0 +1,188 @@ +#pragma once + +/** + * Allows grouping key-value pairs into groups. + * by Humans for All + * + * Allows one to maintain a bunch of groups, where each group contains key-value pairs in them. + * The values could belong to any one of these types ie strings, int, float and bool. + * + * It also tries to provide a crude expanded form of array wrt any of the above supported types. + * For this one needs to define keys using the pattern TheKeyName-0, TheKeyName-1, .... + */ + +#include +#include +#include +#include +#include + + +#define GKV_DEBUGLOG_ON + +#include "log.h" +#define LINFO_LN LOG_TEELN +#ifdef GKV_DEBUGLOG_ON +#define LDBUG LOG +#define LDBUG_LN LOGLN +#else +#define LDBUG_LN(...) +#define LDBUG(...) +#endif +#define LERRR_LN LOG_TEELN +#define LWARN_LN LOG_TEELN + + +typedef std::variant GroupKVData; +typedef std::vector MultiPart; +typedef std::map> GroupKVMapMapVariant; + +class GroupKV { + +private: + + GroupKVMapMapVariant gkv = {}; + +public: + + GroupKV(GroupKVMapMapVariant defaultMap) : gkv(defaultMap) {} + + static std::string joiner(const MultiPart& parts) { + std::stringstream joined; + int iCnt = 0; + for(auto part: parts) { + if (iCnt != 0) { + joined << "-"; + } + iCnt += 1; + joined << part; + } + return joined.str(); + } + + std::string the_type(const GroupKVData &value) { + if (std::holds_alternative(value)) { + return "string"; + } else if (std::holds_alternative(value)) { + return "bool"; + } else if (std::holds_alternative(value)) { + return "int32_t"; + } else if (std::holds_alternative(value)) { + return "int64_t"; + } else if (std::holds_alternative(value)) { + return "double"; + } + return "unknown"; + } + + std::string to_str(const GroupKVData &value) { + auto visitor = [](auto value) -> auto { + std::stringstream ss; + ss << value; + return ss.str(); + }; + return std::visit(visitor, value); + } + + template + std::string to_str(std::vector values) { + std::stringstream ss; + ss << "[ "; + int cnt = 0; + for(auto value: values) { + cnt += 1; + if (cnt != 1) ss << ", "; + ss << value; + } + ss << " ]"; + return ss.str(); + } + + bool group_exists(const std::string &group) { + if (gkv.find(group) == gkv.end()) { + return false; + } + return true; + } + + template + void set_value(const std::string &group, const MultiPart &keyParts, const SupportedDataType &value, const std::string &callerName="") { + auto key = joiner(keyParts); + auto &gm = gkv[group]; + gm[key] = value; + LDBUG_LN("DBUG:GKV:%s_%s:%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), the_type(value).c_str(), to_str(value).c_str()); + } + + // Dump info about the specified group. + // If group is empty, then dump info about all groups maintained in this instance. + std::string dump(const std::string &group, const std::string &msgTag = "") { + std::stringstream ss; + for (auto gm: gkv) { + if (!group.empty() && (gm.first != group)) { + LDBUG_LN("DBUG:GKV:%s:%s:%s:Skipping...", __func__, msgTag.c_str(), gm.first.c_str()); + continue; + } + ss << "\n" << msgTag + ":" << gm.first << ":\n"; + for(auto k: gm.second) { +#ifdef GKV_DEBUGLOG_ON + ss << msgTag + ":" << "\t" << k.first << ":" + the_type(k.second) + ":" << to_str(k.second) << "\n"; +#else + ss << msgTag + ":" << "\t" << k.first << ":" << to_str(k.second) << "\n"; +#endif + } + } + return ss.str(); + } + + // If the specified key is missing, an exception will be thrown. + template + SupportedDataType get_value(const std::string &group, const MultiPart &keyParts) { + auto key = joiner(keyParts); + auto gm = gkv[group]; + if (gm.find(key) == gm.end()) { + std::stringstream ss; + ss << "WARN:GKV:" << __func__ << ":" << group << ":Key [" << key << "] not found"; + throw std::range_error(ss.str()); + } + auto value = gm[key]; + return std::get(value); + } + + // If the specified key is missing, then the provided default value will be returned. + template + SupportedDataType get_value(const std::string &group, const MultiPart &keyParts, const SupportedDataType &defaultValue, const std::string &callerName="") { + try { + auto value = get_value(group, keyParts); + LDBUG_LN("DBUG:GKV:%s_%s:%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), to_str(keyParts).c_str(), the_type(value).c_str(), to_str(value).c_str()); + return value; + } catch (std::exception &e) { + } + LDBUG_LN("WARN:GKV:%s_%s:%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), to_str(keyParts).c_str(), the_type(defaultValue).c_str(), to_str(defaultValue).c_str()); + return defaultValue; + } + + template + std::vector get_vector(const std::string &group, const MultiPart &keyParts, const std::vector &defaultValue, const std::string &callerName="") { + auto key = joiner(keyParts); + auto gm = gkv[group]; + std::vector array; + int i = 0; + while(true) { + std::stringstream ssArrayKey; + ssArrayKey << key << "-" << i; + auto arrayKey = ssArrayKey.str(); + if (gm.find(arrayKey) == gm.end()) { + break; + } + array.push_back(std::get(gm[arrayKey])); + i += 1; + } + if (array.empty()) { + LDBUG_LN("WARN:GKV:%s_%s:%s:%s:%s[default]", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(defaultValue).c_str()); + return defaultValue; + } + LDBUG_LN("DBUG:GKV:%s_%s:%s:%s:%s", __func__, callerName.c_str(), group.c_str(), key.c_str(), to_str(array).c_str()); + return array; + } + +}; diff --git a/common/simpcfg.hpp b/common/simpcfg.hpp new file mode 100644 index 0000000000000..ffcadc3f473e9 --- /dev/null +++ b/common/simpcfg.hpp @@ -0,0 +1,197 @@ +#pragma once + +/** + * Provides a simple direct 1-level only config file logic + * by Humans for All + * + * This builds on the GroupKV class. + * + * ## File format + * + * It can consist of multiple config groups. + * * the group name needs to start at the begining of the line. + * Each group can inturn contain multiple config fields (key:value pairs) wrt that group. + * * the group fields need to have 1 or more space at the begining of line. + * + * ## Supported data types + * + * The fields can have values belonging to ane one of the below types + * * strings - enclosed in double quotes + * this is also the fallback catch all type, but dont rely on this behaviour. + * * int - using decimal number system + * * float - needs to have a decimal point and or e/E + * if decimal point is used, there should be atleast one decimal number on its either side + * * bool - either true or false + * + * It tries to provide a crude expanded form of array wrt any of the above supported types. + * For this one needs to define keys using the pattern TheKeyName-0, TheKeyName-1, .... + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "groupkv.hpp" +#include "datautils_string.hpp" + + + +class SimpCfg : public GroupKV { + +private: + std::regex rInt {R"(^[-+]?\d+$)"}; + std::regex rFloat {R"(^[-+]?\d+(?:\.\d+)?(?:[eE][-+]?\d+)?$)"}; + +public: + + SimpCfg(GroupKVMapMapVariant defaultMap) : GroupKV(defaultMap) {} + + + void set_string(const std::string &group, const MultiPart &keyParts, const std::string &value) { + set_value(group, keyParts, value, __func__); + } + + void set_bool(const std::string &group, const MultiPart &keyParts, bool value) { + set_value(group, keyParts, value, __func__); + } + + void set_bool(const std::string &group, const MultiPart &keyParts, const std::string &value) { + std::string sValue = str_tolower(value); + bool bValue = sValue == "true" ? true : false; + //LDBUG_LN("DBUG:%s:%s:%s:%d", __func__, value.c_str(), sValue.c_str(), bValue); + set_bool(group, keyParts, bValue); + } + + void set_int32(const std::string &group, const MultiPart &keyParts, int32_t value) { + set_value(group, keyParts, value, __func__); + } + + void set_int32(const std::string &group, const MultiPart &keyParts, std::string &value) { + auto ivalue = strtol(value.c_str(), nullptr, 0); + set_int32(group, keyParts, ivalue); + } + + void set_int64(const std::string &group, const MultiPart &keyParts, int64_t value) { + set_value(group, keyParts, value, __func__); + } + + void set_int64(const std::string &group, const MultiPart &keyParts, std::string &value) { + auto ivalue = strtoll(value.c_str(), nullptr, 0); + set_int64(group, keyParts, ivalue); + } + + void set_double(const std::string &group, const MultiPart &keyParts, double value) { + set_value(group, keyParts, value, __func__); + } + + void set_double(const std::string &group, const MultiPart &keyParts, std::string &value) { + auto dvalue = strtod(value.c_str(), nullptr); + set_double(group, keyParts, dvalue); + } + + + std::string get_string(const std::string &group, const MultiPart &keyParts, const std::string &defaultValue) { + return get_value(group, keyParts, defaultValue, __func__); + } + + bool get_bool(const std::string &group, const MultiPart &keyParts, bool defaultValue) { + return get_value(group, keyParts, defaultValue, __func__); + } + + int32_t get_int32(const std::string &group, const MultiPart &keyParts, int32_t defaultValue) { + return get_value(group, keyParts, defaultValue, __func__); + } + + int64_t get_int64(const std::string &group, const MultiPart &keyParts, int64_t defaultValue) { + return get_value(group, keyParts, defaultValue, __func__); + } + + double get_double(const std::string &group, const MultiPart &keyParts, double defaultValue) { + return get_value(group, keyParts, defaultValue, __func__); + } + + + static void locale_prepare(std::string &sSavedLocale) { + sSavedLocale = std::setlocale(LC_ALL, nullptr); + auto sUpdatedLocale = std::setlocale(LC_ALL, "en_US.UTF-8"); + LDBUG_LN("DBUG:%s:Locale:Prev:%s:Cur:%s", __func__, sSavedLocale.c_str(), sUpdatedLocale); + } + + static void locale_restore(const std::string &sSavedLocale) { + auto sCurLocale = std::setlocale(LC_ALL, sSavedLocale.c_str()); + LDBUG_LN("DBUG:%s:Locale:Requested:%s:Got:%s", __func__, sSavedLocale.c_str(), sCurLocale); + } + + void load(const std::string &fname) { + std::ifstream f {fname}; + if (!f) { + LERRR_LN("ERRR:SC:%s:%s:failed to load...", __func__, fname.c_str()); + throw std::runtime_error { "ERRR:SimpCfg:File not found" }; + } else { + LDBUG_LN("DBUG:SC:%s:%s", __func__, fname.c_str()); + } + std::string group; + int iLine = 0; + while(!f.eof()) { + iLine += 1; + std::string curL; + getline(f, curL); + if (curL.empty()) { + continue; + } + if (curL[0] == '#') { + continue; + } + bool bGroup = !isspace(curL[0]); + curL = str_trim(curL); + if (bGroup) { + curL = str_trim_single(curL, {"\""}); + group = curL; + LDBUG_LN("DBUG:SC:%s:group:%s", __func__, group.c_str()); + continue; + } + auto dPos = curL.find(':'); + if (dPos == std::string::npos) { + LERRR_LN("ERRR:SC:%s:%d:invalid key value line:%s", __func__, iLine, curL.c_str()); + throw std::runtime_error { "ERRR:SimpCfg:Invalid key value line" }; + } + auto dEnd = curL.length() - dPos; + if ((dPos == 0) || (dEnd < 2)) { + LERRR_LN("ERRR:SC:%s:%d:invalid key value line:%s", __func__, iLine, curL.c_str()); + throw std::runtime_error { "ERRR:SimpCfg:Invalid key value line" }; + } + std::string key = curL.substr(0, dPos); + key = str_trim(key); + key = str_trim_single(key, {"\""}); + std::string value = curL.substr(dPos+1); + value = str_trim(value); + value = str_trim(value, {","}); + std::string vtype = "bool"; + auto valueLower = str_tolower(value); + if ((valueLower.compare("true") == 0) || (valueLower == "false")) { + set_bool(group, {key}, value); + } else if (std::regex_match(value, rInt)) { + vtype = "int"; + set_int64(group, {key}, value); + } else if (std::regex_match(value, rFloat)) { + vtype = "float"; + set_double(group, {key}, value); + } else { + vtype = "string"; + if (!value.empty() && (value.front() != '"')) { + LWARN_LN("WARN:SC:%s:%d:%s:k:%s:v:%s:is this string?", __func__, iLine, group.c_str(), key.c_str(), value.c_str()); + } + value = str_trim_single(value, {"\""}); + set_string(group, {key}, value); + } + //LDBUG_LN("DBUG:SC:%s:%d:kv:%s:%s:%s:%s", __func__, iLine, group.c_str(), key.c_str(), vtype.c_str(), value.c_str()); + } + } + +}; diff --git a/examples/chaton_meta.json b/examples/chaton_meta.json new file mode 100644 index 0000000000000..70c05359d17ae --- /dev/null +++ b/examples/chaton_meta.json @@ -0,0 +1,413 @@ + +{ + "llama2": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "begin": "", + "prefix": "[INST] <>\n", + "suffix": "\n<> ", + "end": "[/INST]\n\n" + }, + "user": { + "begin": "", + "prefix": "[INST] ", + "suffix": " [/INST]\n\n", + "end": "" + }, + "assistant": { + "begin": "", + "prefix": "", + "suffix": "", + "end": "" + }, + "reverse-prompt": "", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": false, + "systemuser-1st-user-has-begin": false, + "systemuser-1st-user-has-prefix": false + }, + "llama3": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "begin": "", + "prefix": "<|start_header_id|>system<|end_header_id|>\n", + "suffix": "<|eot_id|>\n\n", + "end": "" + }, + "user": { + "begin": "", + "prefix": "<|start_header_id|>user<|end_header_id|>\n", + "suffix": "<|eot_id|>\n\n", + "end": "" + }, + "assistant": { + "begin": "", + "prefix": "<|start_header_id|>assistant<|end_header_id|>\n", + "suffix": "<|eot_id|>\n\n", + "end": "" + }, + "reverse-prompt": "<|eot_id|>", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + }, + "chatml": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "begin": "", + "prefix": "<|im_start|>system\n", + "suffix": "<|im_end|>\n", + "end": "" + }, + "user": { + "begin": "", + "prefix": "<|im_start|>user\n", + "suffix": "<|im_end|>\n", + "end": "" + }, + "assistant": { + "begin": "", + "prefix": "<|im_start|>assistant\n", + "suffix": "<|im_end|>\n", + "end": "" + }, + "reverse-prompt": "<|im_start|>user\n", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + }, + "zephyr": { + "global": { + "alt-end": "<|endoftext|>\n", + "begin": "", + "end": "" + }, + "system": { + "begin": "", + "prefix": "<|system|>\n", + "suffix": "", + "end": "\n" + }, + "user": { + "begin": "", + "prefix": "<|user|>\n", + "suffix": "", + "end": "\n" + }, + "assistant": { + "end-alt": "<|endoftext|>\n", + "begin": "", + "prefix": "<|assistant|>\n", + "suffix": "", + "end": "\n" + }, + "reverse-prompt": "", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + }, + "gemma": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "begin": "", + "prefix": "user\n", + "suffix": "\n", + "end": "" + }, + "user": { + "begin": "", + "prefix": "user\n", + "suffix": "\n", + "end": "" + }, + "assistant": { + "begin": "", + "prefix": "model\n", + "suffix": "\n", + "end": "" + }, + "reverse-prompt": "", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": false, + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + }, + "deepseek-coder": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "begin": "\n<｜begin▁of▁sentence｜>", + "prefix": "", + "suffix": "\n", + "end": "" + }, + "user": { + "begin": "", + "prefix": "### Instruction:\n", + "suffix": "\n", + "end": "" + }, + "assistant": { + "begin": "", + "prefix": "### Response:\n", + "suffix": "\n<|EOT|>\n", + "end": "" + }, + "reverse-prompt": "<|EOT|>", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": false, + "systemuser-1st-user-has-begin": false, + "systemuser-1st-user-has-prefix": true + }, + "deepseek": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "begin": "<｜begin▁of▁sentence｜>", + "prefix": "", + "suffix": "\n\n", + "end": "" + }, + "user": { + "begin": "<｜begin▁of▁sentence｜>", + "prefix": "User: ", + "suffix": "\n\n", + "end": "" + }, + "assistant": { + "begin": "", + "prefix": "Assistant: ", + "suffix": " <｜end▁of▁sentence｜>\n", + "end": "" + }, + "reverse-prompt": "<｜end▁of▁sentence｜>", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": false, + "systemuser-1st-user-has-begin": false, + "systemuser-1st-user-has-prefix": true + }, + "monarch": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "begin": "", + "prefix": "system\n", + "suffix": "", + "end": "\n" + }, + "user": { + "begin": "", + "prefix": "user\n", + "suffix": "", + "end": "\n" + }, + "assistant": { + "begin": "", + "prefix": "assistant\n", + "suffix": "", + "end": "\n" + }, + "reverse-prompt": "", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + }, + "mistral": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "begin": "", + "prefix": "[INST] ", + "suffix": " [/INST]", + "end": "\n" + }, + "user": { + "begin": "", + "prefix": "[INST] ", + "suffix": " [/INST]", + "end": "\n" + }, + "assistant": { + "begin": "", + "prefix": "", + "suffix": "", + "end": " \n" + }, + "reverse-prompt": "", + "systemuser-system-has-suffix": false, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": false, + "systemuser-1st-user-has-prefix": false + }, + "phi3": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "begin": "", + "prefix": "<|system|>\n", + "suffix": "<|end|>\n", + "end": "" + }, + "user": { + "begin": "", + "prefix": "<|user|>\n", + "suffix": "<|end|>\n", + "end": "" + }, + "assistant": { + "begin": "", + "prefix": "<|assistant|>\n", + "suffix": "<|end|>\n", + "end": "" + }, + "reverse-prompt": "<|end|>", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + }, + "command-r": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "begin": "", + "prefix": "<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>", + "suffix": "<|END_OF_TURN_TOKEN|>", + "end": "" + }, + "user": { + "begin": "", + "prefix": "<|START_OF_TURN_TOKEN|><|USER_TOKEN|>", + "suffix": "<|END_OF_TURN_TOKEN|>", + "end": "" + }, + "assistant": { + "begin": "", + "prefix": "<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", + "suffix": "<|END_OF_TURN_TOKEN|>", + "end": "" + }, + "reverse-prompt": "<|END_OF_TURN_TOKEN|>", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + }, + "orion": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "begin": "", + "prefix": "Human: ", + "suffix": "\n\n", + "end": "" + }, + "user": { + "begin": "", + "prefix": "Human: ", + "suffix": "\n\nAssistant: ", + "end": "" + }, + "assistant": { + "begin": "", + "prefix": "", + "suffix": "", + "end": "" + }, + "reverse-prompt": "", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": false, + "systemuser-1st-user-has-prefix": false + }, + "openchat": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "begin": "", + "prefix": "GPT4 Correct System: ", + "suffix": "<|end_of_turn|>", + "end": "" + }, + "user": { + "begin": "", + "prefix": "GPT4 Correct User: ", + "suffix": "<|end_of_turn|>", + "end": "" + }, + "assistant": { + "begin": "", + "prefix": "GPT4 Correct Assistant: ", + "suffix": "<|end_of_turn|>", + "end": "" + }, + "reverse-prompt": "<|end_of_turn|>", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + }, + "vicuna": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "begin": "", + "prefix": "SYSTEM: ", + "suffix": "\n\n", + "end": "" + }, + "user": { + "begin": "", + "prefix": "USER: ", + "suffix": "\n", + "end": "" + }, + "assistant": { + "begin": "", + "prefix": "ASSISTANT: ", + "suffix": "\n", + "end": "" + }, + "reverse-prompt": "", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + } + +} + diff --git a/examples/chaton_meta.old_simple.json b/examples/chaton_meta.old_simple.json new file mode 100644 index 0000000000000..25f05a17076e3 --- /dev/null +++ b/examples/chaton_meta.old_simple.json @@ -0,0 +1,180 @@ + +{ + "llama2": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "prefix": "[INST] <>\n", + "suffix": "\n<> [/INST]\n\n" + }, + "user": { + "begin": "", + "prefix": "[INST] ", + "suffix": " [/INST]\n\n" + }, + "assistant": { + "prefix": "", + "suffix": "" + }, + "reverse-prompt": "", + "systemuser-1st-user-has-begin": false, + "systemuser-1st-user-has-prefix": false + }, + "llama3": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "prefix": "<|start_header_id|>system<|end_header_id|>\n", + "suffix": "<|eot_id|>\n\n" + }, + "user": { + "begin": "", + "prefix": "<|start_header_id|>user<|end_header_id|>\n", + "suffix": "<|eot_id|>\n\n" + }, + "assistant": { + "prefix": "<|start_header_id|>assistant<|end_header_id|>\n", + "suffix": "<|eot_id|>\n\n" + }, + "reverse-prompt": "<|eot_id|>", + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + }, + "chatml": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "prefix": "<|im_start|>system\n", + "suffix": "<|im_end|>\n" + }, + "user": { + "begin": "", + "prefix": "\n<|im_start|>user\n", + "suffix": "<|im_end|>\n" + }, + "assistant": { + "prefix": "<|im_start|>assistant\n", + "suffix": "<|im_end|>\n" + }, + "reverse-prompt": "<|im_start|>user\n", + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + }, + "zephyr": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "prefix": "<|system|>\n", + "suffix": "<|endoftext|>\n" + }, + "user": { + "begin": "", + "prefix": "<|user|>\n", + "suffix": "<|endoftext|>\n" + }, + "assistant": { + "prefix": "<|assistant|>\n", + "suffix": "<|endoftext|>\n" + }, + "reverse-prompt": "", + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + }, + "gemma": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "prefix": "", + "suffix": "" + }, + "user": { + "begin": "", + "prefix": "user\n", + "suffix": "\n" + }, + "assistant": { + "prefix": "model\n", + "suffix": "\n" + }, + "reverse-prompt": "", + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + }, + "deepseek-alt": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "prefix": "", + "suffix": "\n" + }, + "user": { + "begin": "", + "prefix": "### Instruction:\n", + "suffix": "\n" + }, + "assistant": { + "prefix": "### Response:\n", + "suffix": "\n<|EOT|>\n" + }, + "reverse-prompt": "<|EOT|>", + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + }, + "deepseek": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "prefix": "", + "suffix": "\n\n" + }, + "user": { + "begin": "", + "prefix": "User: ", + "suffix": "\n\n" + }, + "assistant": { + "prefix": "Assistant: ", + "suffix": " <｜end▁of▁sentence｜>\n" + }, + "reverse-prompt": "<｜end▁of▁sentence｜>", + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + }, + "monarch": { + "global": { + "begin": "", + "end": "" + }, + "system": { + "prefix": "system\n", + "suffix": "\n" + }, + "user": { + "begin": "", + "prefix": "user\n", + "suffix": "\n" + }, + "assistant": { + "prefix": "assistant\n", + "suffix": " \n" + }, + "reverse-prompt": "", + "systemuser-1st-user-has-begin": false, + "systemuser-1st-user-has-prefix": true + } +} + diff --git a/examples/chaton_meta.simpcfg b/examples/chaton_meta.simpcfg new file mode 100644 index 0000000000000..8bf12d8990384 --- /dev/null +++ b/examples/chaton_meta.simpcfg @@ -0,0 +1,239 @@ +"llama2" + "global-begin": "" + "global-end": "" + + "system-begin": "" + "system-prefix": "[INST] <>\n" + "system-suffix": "\n<> " + "system-end": "[/INST]\n\n" + + "user-begin": "" + "user-prefix": "[INST] " + "user-suffix": " [/INST]\n\n" + "user-end": "" + + "assistant-begin": "" + "assistant-prefix": "" + "assistant-suffix": "" + "assistant-end": "" + + "reverse-prompt": "", + "systemuser-system-has-suffix": tRUe, + "systemuser-system-has-end": faLSe, + "systemuser-1st-user-has-begin": false, + "systemuser-1st-user-has-prefix": false + +"llama3" + "global-begin": "" + "global-end": "" + + "system-begin": "" + "system-prefix": "<|start_header_id|>system<|end_header_id|>\n" + "system-suffix": "<|eot_id|>\n\n" + "system-end": "" + + "user-begin": "" + "user-prefix": "<|start_header_id|>user<|end_header_id|>\n" + "user-suffix": "<|eot_id|>\n\n" + "user-end": "" + + "assistant-begin": "" + "assistant-prefix": "<|start_header_id|>assistant<|end_header_id|>\n" + "assistant-suffix": "<|eot_id|>\n\n" + "assistant-end": "" + + "reverse-prompt": "<|eot_id|>", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + +"chatml" + "global-begin": "" + "global-end": "" + + "system-begin": "" + "system-prefix": "<|im_start|>system\n" + "system-suffix": "<|im_end|>\n" + "system-end": "" + + "user-begin": "" + "user-prefix": "<|im_start|>user\n" + "user-suffix": "<|im_end|>\n" + "user-end": "" + + "assistant-begin": "" + "assistant-prefix": "<|im_start|>assistant\n" + "assistant-suffix": "<|im_end|>\n" + "assistant-end": "" + + "reverse-prompt": "<|im_start|>user\n", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + +"zephyr" + "global-alt-end": "<|endoftext|>\n" + "global-begin": "" + "global-end": "" + + "system-begin": "" + "system-prefix": "<|system|>\n" + "system-suffix": "" + "system-end": "\n" + + "user-begin": "" + "user-prefix": "<|user|>\n" + "user-suffix": "" + "user-end": "\n" + + "assistant-end-alt": "<|endoftext|>\n" + "assistant-begin": "" + "assistant-prefix": "<|assistant|>\n" + "assistant-suffix": "" + "assistant-end": "\n" + + "reverse-prompt": "", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + +"gemma" + "global-begin": "" + "global-end": "" + + "system-begin": "" + "system-prefix": "user\n" + "system-suffix": "\n" + "system-end": "" + + "user-begin": "" + "user-prefix": "user\n" + "user-suffix": "\n" + "user-end": "" + + "assistant-begin": "" + "assistant-prefix": "model\n" + "assistant-suffix": "\n" + "assistant-end": "" + + "reverse-prompt": "", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": false, + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + +"deepseek-coder" + "global-begin": "" + "global-end": "" + + "system-begin": "\n<｜begin▁of▁sentence｜>" + "system-prefix": "" + "system-suffix": "\n" + "system-end": "" + + "user-begin": "" + "user-prefix": "### Instruction:\n" + "user-suffix": "\n" + "user-end": "" + + "assistant-prefix": "### Response:\n" + "assistant-suffix": "\n<|EOT|>\n + + "reverse-prompt": "<|EOT|>" + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": false, + "systemuser-1st-user-has-begin": false, + "systemuser-1st-user-has-prefix": true + +"deepseek" + "global-begin": "" + "global-end": "" + + "system-begin": "<｜begin▁of▁sentence｜>" + "system-prefix": "" + "system-suffix": "\n\n" + "system-end": "" + + "user-begin": "<｜begin▁of▁sentence｜>" + "user-prefix": "User: " + "user-suffix": "\n\n" + "user-end": "" + + "assistant-begin": "" + "assistant-prefix": "Assistant: " + "assistant-suffix": " <｜end▁of▁sentence｜>\n" + "assistant-end": "" + + "reverse-prompt": "<｜end▁of▁sentence｜>", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": false, + "systemuser-1st-user-has-begin": false, + "systemuser-1st-user-has-prefix": true + +"monarch" + "global-begin": "" + "global-end": "" + + "system-begin": "" + "system-prefix": "system\n" + "system-suffix": "" + "system-end": "\n" + + "user-begin": "" + "user-prefix": "user\n" + "user-suffix": "" + "user-end": "\n" + + "assistant-begin": "" + "assistant-prefix": "assistant\n" + "assistant-suffix": "" + "assistant-end": "\n" + + "reverse-prompt": "", + "systemuser-system-has-suffix": true, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": true, + "systemuser-1st-user-has-prefix": true + +"mistral" + "global-begin": "" + "global-end": "" + + "system-begin": "" + "system-prefix": "[INST] " + "system-suffix": " [/INST]" + "system-end": "\n" + + "user-begin": "" + "user-prefix": "[INST] " + "user-suffix": " [/INST]" + "user-end": "\n" + + "assistant-begin": "" + "assistant-prefix": "" + "assistant-suffix": "" + "assistant-end": " \n" + + "reverse-prompt": "", + "systemuser-system-has-suffix": false, + "systemuser-system-has-end": true, + "systemuser-1st-user-has-begin": false, + "systemuser-1st-user-has-prefix": false + +"testme" + "int": 1234 + "sint": -9876543210 + "another int": 0xff + "float": 12.34 + "double1": +12.0e-123 + "double2": -12.0e-123678 + "double3": -12.0e300 + "isit double1": -12.e300 + "isit double2": -12. + "yes double2": -12.0 + "isit double3": .0123 + "yes double3": 0.0123 + diff --git a/examples/main/main.cpp b/examples/main/main.cpp index 9dee41001f12c..70e2e562a2859 100644 --- a/examples/main/main.cpp +++ b/examples/main/main.cpp @@ -1,4 +1,5 @@ #include "common.h" +#include "chaton_json.hpp" #include "console.h" #include "llama.h" @@ -141,6 +142,15 @@ int main(int argc, char ** argv) { console::init(params.simple_io, params.use_color); atexit([]() { console::cleanup(); }); + if (params.chaton) { + if (!params.chaton_meta_json.empty()) { + chaton_meta_load_json(params.chaton_meta_json); + } + if (!chaton_meta_ok(params.chaton_template_id)) { + exit(1); + } + } + if (params.logits_all) { printf("\n************\n"); printf("%s: please use the 'perplexity' tool for perplexity calculations\n", __func__); @@ -250,11 +260,14 @@ int main(int argc, char ** argv) { std::vector embd_inp; - if (params.interactive_first || params.instruct || params.chatml || !params.prompt.empty() || session_tokens.empty()) { + if (params.interactive_first || params.instruct || params.chatml || params.chaton || !params.prompt.empty() || session_tokens.empty()) { LOG("tokenize the prompt\n"); if (params.chatml) { params.prompt = "<|im_start|>system\n" + params.prompt + "<|im_end|>"; } + if (params.chaton) { + chaton_tmpl_apply_single(params.chaton_template_id, K_SYSTEM, params.prompt, false, false, params.prompt); + } embd_inp = ::llama_tokenize(ctx, params.prompt, true, true); } else { LOG("use session tokens\n"); @@ -332,7 +345,7 @@ int main(int argc, char ** argv) { } // number of tokens to keep when resetting context - if (params.n_keep < 0 || params.n_keep > (int) embd_inp.size() || params.instruct || params.chatml) { + if (params.n_keep < 0 || params.n_keep > (int) embd_inp.size() || params.instruct || params.chatml || params.chaton) { params.n_keep = (int)embd_inp.size(); } else { params.n_keep += add_bos; // always keep the BOS token @@ -366,6 +379,16 @@ int main(int argc, char ** argv) { params.interactive_first = true; } + // chaton mode + const auto chaton_assitant_prefix = ::llama_tokenize(ctx, chaton_tmpl_role_getkeys(params.chaton_template_id, K_ASSISTANT, {K_BEGIN, K_PREFIX}), false, true); + if (params.chaton) { + params.interactive = true; // may remove later, by requiring user to explicitly request interactive mode + params.interactive_first = true; + params.input_prefix = chaton_tmpl_role_getkeys(params.chaton_template_id, K_USER, {K_BEGIN, K_PREFIX}); + params.input_suffix = chaton_tmpl_role_getkeys(params.chaton_template_id, K_USER, {K_SUFFIX, K_END}); + params.antiprompt.emplace_back(chaton_tmpl_getkey_str(params.chaton_template_id, K_REVERSE_PROMPT)); + } + // enable interactive mode if interactive start is specified if (params.interactive_first) { params.interactive = true; @@ -823,7 +846,7 @@ int main(int argc, char ** argv) { if (n_past > 0 && is_interacting) { LOG("waiting for user input\n"); - if (params.conversation || params.instruct || params.chatml) { + if (params.conversation || params.instruct || params.chatml || params.chaton) { printf("\n> "); } @@ -902,6 +925,11 @@ int main(int argc, char ** argv) { LOG("inserting chatml suffix\n"); embd_inp.insert(embd_inp.end(), cml_sfx.begin(), cml_sfx.end()); } + // chaton mode: insert assistant prefix + if (params.chaton) { + LOG("inserting chaton assistant prefix\n"); + embd_inp.insert(embd_inp.end(), chaton_assitant_prefix.begin(), chaton_assitant_prefix.end()); + } for (size_t i = original_size; i < embd_inp.size(); ++i) { const llama_token token = embd_inp[i]; diff --git a/scripts/chaton-meta-json-to-hpp.py b/scripts/chaton-meta-json-to-hpp.py new file mode 100755 index 0000000000000..3329b9b7f43ed --- /dev/null +++ b/scripts/chaton-meta-json-to-hpp.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# Convert chaton meta json file to equivalent c++ cpp format +# by Humans for All + +import sys +import json + + +def kkv_str(j, tmpl, k1, k2, comma): + print("\t\t{{ \"{}\", \"{}\" }}{}".format("{}-{}".format(k1,k2), repr(j[tmpl][k1][k2])[1:-1], comma)) + +def kv_str(j, tmpl, k1, comma): + print("\t\t{{ \"{}\", \"{}\" }}{}".format(k1, repr(j[tmpl][k1])[1:-1], comma)) + +def kv_bool(j, tmpl, k1, comma): + print("\t\t{{ \"{}\", {} }}{}".format(k1, repr(j[tmpl][k1]).lower(), comma)) + + +fp=open(sys.argv[1]) +j=json.load(fp) +print("//This is auto created/converted from chaton-meta-json file") +print("\n\n#include \"chaton.hpp\"\n\nChatTemplates gCT = {{") + +for tmpl in j: + print("\t{{ \"{}\", {{".format(tmpl)) + + kkv_str(j, tmpl, "global", "begin", ",") + kkv_str(j, tmpl, "global", "end", ",") + + kkv_str(j, tmpl, "system", "begin", ",") + kkv_str(j, tmpl, "system", "prefix", ",") + kkv_str(j, tmpl, "system", "suffix", ",") + kkv_str(j, tmpl, "system", "end", ",") + + kkv_str(j, tmpl, "user", "begin", ",") + kkv_str(j, tmpl, "user", "prefix", ",") + kkv_str(j, tmpl, "user", "suffix", ",") + kkv_str(j, tmpl, "user", "end", ",") + + kkv_str(j, tmpl, "assistant", "begin", ",") + kkv_str(j, tmpl, "assistant", "prefix", ",") + kkv_str(j, tmpl, "assistant", "suffix", ",") + kkv_str(j, tmpl, "assistant", "end", ",") + + kv_str(j, tmpl, "reverse-prompt", ",") + + kv_bool(j, tmpl, "systemuser-system-has-suffix", ",") + kv_bool(j, tmpl, "systemuser-system-has-end", ",") + kv_bool(j, tmpl, "systemuser-1st-user-has-begin", ",") + kv_bool(j, tmpl, "systemuser-1st-user-has-prefix", ",") + + print("\t}},") + +print("}};") + diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 766a017524237..d6f970edb2b5b 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -116,6 +116,12 @@ llama_target_and_test(test-quantize-fns.cpp) llama_target_and_test(test-quantize-perf.cpp) llama_target_and_test(test-sampling.cpp) llama_target_and_test(test-chat-template.cpp) +llama_target_and_test(test-chat-template-chaton.cpp) +llama_target_and_test(test-chaton-groupkv.cpp) +llama_target_and_test(test-chaton-simpcfg.cpp) +#target_compile_features(test-chaton-simpcfg PUBLIC cxx_std_20) +target_compile_options(test-chaton-simpcfg PUBLIC -std=c++20) + llama_target_and_test(test-grammar-parser.cpp) llama_target_and_test(test-llama-grammar.cpp) diff --git a/tests/test-chat-template-chaton.cpp b/tests/test-chat-template-chaton.cpp new file mode 100644 index 0000000000000..7e352738c94f5 --- /dev/null +++ b/tests/test-chat-template-chaton.cpp @@ -0,0 +1,179 @@ +#include +#include +#include +#include + +#undef NDEBUG +#include + +#include "llama.h" +#include "chaton_json.hpp" + + +std::vector templateIds = { "llama2", "llama3", "chatml", + "zephyr", "gemma", "deepseek-coder", "deepseek", "monarch", "mistral" }; + +llama_chat_message conversation[] = { + {"system", "You are a helpful assistant"}, + {"user", "Hello"}, + {"assistant", "Hi there"}, + {"user", "Who are you"}, + {"assistant", " I am an assistant "}, + {"user", "Another question"}, +}; +size_t message_count = 6; + +std::vector templates = { + // teknium/OpenHermes-2.5-Mistral-7B + "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}", + // mistralai/Mistral-7B-Instruct-v0.2 + "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}", + // TheBloke/FusionNet_34Bx2_MoE-AWQ + "{%- for idx in range(0, messages|length) -%}\\n{%- if messages[idx]['role'] == 'user' -%}\\n{%- if idx > 1 -%}\\n{{- bos_token + '[INST] ' + messages[idx]['content'] + ' [/INST]' -}}\\n{%- else -%}\\n{{- messages[idx]['content'] + ' [/INST]' -}}\\n{%- endif -%}\\n{% elif messages[idx]['role'] == 'system' %}\\n{{- '[INST] <>\\\\n' + messages[idx]['content'] + '\\\\n<>\\\\n\\\\n' -}}\\n{%- elif messages[idx]['role'] == 'assistant' -%}\\n{{- ' ' + messages[idx]['content'] + ' ' + eos_token -}}\\n{% endif %}\\n{% endfor %}", + // bofenghuang/vigogne-2-70b-chat + "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% elif true == true and not '<>' in messages[0]['content'] %}{% set loop_messages = messages %}{% set system_message = 'Vous êtes Vigogne, un assistant IA créé par Zaion Lab. Vous suivez extrêmement bien les instructions. Aidez autant que vous le pouvez.' %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if loop.index0 == 0 and system_message != false %}{% set content = '<>\\\\n' + system_message + '\\\\n<>\\\\n\\\\n' + message['content'] %}{% else %}{% set content = message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + content.strip() + ' [/INST]' }}{% elif message['role'] == 'system' %}{{ '<>\\\\n' + content.strip() + '\\\\n<>\\\\n\\\\n' }}{% elif message['role'] == 'assistant' %}{{ ' ' + content.strip() + ' ' + eos_token }}{% endif %}{% endfor %}", + // mlabonne/AlphaMonarch-7B + "{% for message in messages %}{{bos_token + message['role'] + '\\n' + message['content'] + eos_token + '\\n'}}{% endfor %}{% if add_generation_prompt %}{{ bos_token + 'assistant\\n' }}{% endif %}", + // google/gemma-7b-it + "{% if messages[0]['role'] == 'system' %}{{ raise_exception('System role not supported') }}{% endif %}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if (message['role'] == 'assistant') %}{% set role = 'model' %}{% else %}{% set role = message['role'] %}{% endif %}{{ '' + role + '\\n' + message['content'] | trim + '\\n' }}{% endfor %}{% if add_generation_prompt %}{{'model\\n'}}{% endif %}", + // OrionStarAI/Orion-14B-Chat + "{% for message in messages %}{% if loop.first %}{{ bos_token }}{% endif %}{% if message['role'] == 'user' %}{{ 'Human: ' + message['content'] + '\\n\\nAssistant: ' + eos_token }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token }}{% endif %}{% endfor %}", + // openchat/openchat-3.5-0106 + // The included chat_template differs from the author's suggestions here: https://huggingface.co/openchat/openchat_3.5/discussions/5#65448109b4a3f3a2f486fd9d + // So we match against the included template but implement the suggested version. + "{{ bos_token }}{% for message in messages %}{{ 'GPT4 Correct ' + message['role'].title() + ': ' + message['content'] + '<|end_of_turn|>'}}{% endfor %}{% if add_generation_prompt %}{{ 'GPT4 Correct Assistant:' }}{% endif %}", + // deepseek-ai/deepseek-coder-33b-instruct + "{% if not add_generation_prompt is defined %}\n{% set add_generation_prompt = false %}\n{% endif %}\n{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n {%- if message['role'] == 'system' -%}\n {%- set ns.found = true -%}\n {%- endif -%}\n{%- endfor -%}\n{{bos_token}}{%- if not ns.found -%}\n{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\\n'}}\n{%- endif %}\n{%- for message in messages %}\n {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n {%- else %}\n {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|EOT|>\\n'}}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{% if add_generation_prompt %}\n{{'### Response:'}}\n{% endif %}", + // eachadea/vicuna-13b-1.1 + // No template included in tokenizer_config.json, so this template likely needs to be manually set. + "{%- for message in messages %}{%- if message['role'] == 'system' -%}{{- '' + message['content'] + '\n\n' -}}{%- else -%}{%- if message['role'] == 'user' -%}{{-'USER: ' + message['content'] + '\n'-}}{%- else -%}{{-'ASSISTANT: ' + message['content'] + '\n' -}}{%- endif -%}{%- endif -%}{%- endfor -%}{%- if add_generation_prompt -%}{{-'ASSISTANT:'-}}{%- endif -%}", + // Orca-Vicuna + // No template included in tokenizer_config.json, so this template likely needs to be manually set. + "{%- for message in messages %}{%- if message['role'] == 'system' -%}{{-'SYSTEM: ' + message['content'] + '\n' -}}{%- else -%}{%- if message['role'] == 'user' -%}{{-'USER: ' + message['content'] + '\n'-}}{%- else -%}{{-'ASSISTANT: ' + message['content'] + '\n' -}}{%- endif -%}{%- endif -%}{%- endfor -%}{%- if add_generation_prompt -%}{{-'ASSISTANT:'-}}{%- endif -%}", + // CohereForAI/c4ai-command-r-plus + "{{ bos_token }}{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% elif false == true %}{% set loop_messages = messages %}{% set system_message = 'You are Command-R, a brilliant, sophisticated, AI-assistant trained to assist human users by providing thorough responses. You are trained by Cohere.' %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% if system_message != false %}{{ '<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>' + system_message + '<|END_OF_TURN_TOKEN|>' }}{% endif %}{% for message in loop_messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|START_OF_TURN_TOKEN|><|USER_TOKEN|>' + content.strip() + '<|END_OF_TURN_TOKEN|>' }}{% elif message['role'] == 'assistant' %}{{ '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>' + content.strip() + '<|END_OF_TURN_TOKEN|>' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>' }}{% endif %}", + // Llama-3 + "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}", +}; +std::vector expected_output = { + // teknium/OpenHermes-2.5-Mistral-7B + "<|im_start|>system\nYou are a helpful assistant<|im_end|>\n<|im_start|>user\nHello<|im_end|>\n<|im_start|>assistant\nHi there<|im_end|>\n<|im_start|>user\nWho are you<|im_end|>\n<|im_start|>assistant\n I am an assistant <|im_end|>\n<|im_start|>user\nAnother question<|im_end|>\n<|im_start|>assistant\n", + // mistralai/Mistral-7B-Instruct-v0.2 + "[INST] You are a helpful assistant\nHello [/INST]Hi there[INST] Who are you [/INST] I am an assistant [INST] Another question [/INST]", + // TheBloke/FusionNet_34Bx2_MoE-AWQ + "[INST] <>\nYou are a helpful assistant\n<>\n\nHello [/INST] Hi there [INST] Who are you [/INST] I am an assistant [INST] Another question [/INST]", + // bofenghuang/vigogne-2-70b-chat + "[INST] <>\nYou are a helpful assistant\n<>\n\nHello [/INST] Hi there [INST] Who are you [/INST] I am an assistant [INST] Another question [/INST]", + // mlabonne/AlphaMonarch-7B + "system\nYou are a helpful assistant\nuser\nHello\nassistant\nHi there\nuser\nWho are you\nassistant\n I am an assistant \nuser\nAnother question\nassistant\n", + // google/gemma-7b-it + "user\nYou are a helpful assistant\n\nHello\nmodel\nHi there\nuser\nWho are you\nmodel\nI am an assistant\nuser\nAnother question\nmodel\n", + // OrionStarAI/Orion-14B-Chat + "Human: You are a helpful assistant\n\nHello\n\nAssistant: Hi thereHuman: Who are you\n\nAssistant: I am an assistant Human: Another question\n\nAssistant: ", + // openchat/openchat-3.5-0106 + "You are a helpful assistant<|end_of_turn|>GPT4 Correct User: Hello<|end_of_turn|>GPT4 Correct Assistant: Hi there<|end_of_turn|>GPT4 Correct User: Who are you<|end_of_turn|>GPT4 Correct Assistant: I am an assistant <|end_of_turn|>GPT4 Correct User: Another question<|end_of_turn|>GPT4 Correct Assistant:", + // deepseek-ai/deepseek-coder-33b-instruct + "You are a helpful assistant### Instruction:\nHello\n### Response:\nHi there\n<|EOT|>\n### Instruction:\nWho are you\n### Response:\n I am an assistant \n<|EOT|>\n### Instruction:\nAnother question\n### Response:\n", + // eachadea/vicuna-13b-1.1 + "You are a helpful assistant\n\nUSER: Hello\nASSISTANT: Hi there\nUSER: Who are you\nASSISTANT: I am an assistant \nUSER: Another question\nASSISTANT:", + // Orca-Vicuna + "SYSTEM: You are a helpful assistant\nUSER: Hello\nASSISTANT: Hi there\nUSER: Who are you\nASSISTANT: I am an assistant \nUSER: Another question\nASSISTANT:", + // CohereForAI/c4ai-command-r-plus + "<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>You are a helpful assistant<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|USER_TOKEN|>Hello<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>Hi there<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|USER_TOKEN|>Who are you<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>I am an assistant<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|USER_TOKEN|>Another question<|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>", + // Llama 3 + "<|start_header_id|>system<|end_header_id|>\n\nYou are a helpful assistant<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nHello<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nHi there<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nWho are you<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nI am an assistant<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nAnother question<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", +}; + +static void check_default() { + std::vector formatted_chat(1024); + int32_t res; + + // test invalid chat template + res = llama_chat_apply_template(nullptr, "INVALID TEMPLATE", conversation, message_count, true, formatted_chat.data(), formatted_chat.size()); + assert(res < 0); + + for (size_t i = 0; i < templates.size(); i++) { + std::string custom_template = templates[i]; + std::string expected = expected_output[i]; + formatted_chat.resize(1024); + res = llama_chat_apply_template( + nullptr, + custom_template.c_str(), + conversation, + message_count, + true, + formatted_chat.data(), + formatted_chat.size() + ); + formatted_chat.resize(res); + std::string output(formatted_chat.data(), formatted_chat.size()); + std::cout << output << "\n-------------------------\n"; + assert(output == expected); + } +} + +static void check_chaton(std::string &metaJson) { + std::vector formatted_chat(1024); + int32_t res; + + chaton_meta_load_json(metaJson); + for(auto tmplId: templateIds) { + formatted_chat.resize(1024); + std::cout << "\n----------" << tmplId << "---------------\n"; + if (!chaton_meta_ok(tmplId)) { + exit(1); + } + res = chaton_tmpl_apply_capi(tmplId.c_str(), conversation, message_count, true, formatted_chat.data(), formatted_chat.size()); + assert(res > 0); + formatted_chat.resize(res); + std::string output(formatted_chat.data(), formatted_chat.size()); + std::cout << "*** TAGGEDMSG ***\n" << output; + std::cout << "\n----------" << tmplId << "---------------\n"; + } + +} + +static void check_chaton_ex(std::string &metaJson) { + std::vector formatted_chat(1024); + int32_t res; + + chaton_meta_load_json(metaJson); + for(auto tmplId: templateIds) { + formatted_chat.resize(1024); + std::cout << "\n----------" << tmplId << "---------------\n"; + if (!chaton_meta_ok(tmplId)) { + exit(1); + } + std::vector partsTypes; + std::vector partsLens; + int numParts = message_count * 3; + int numPartsCrossCheck = numParts; + partsLens.resize(numParts); + partsTypes.resize(numParts); + res = chaton_tmpl_apply_ex_capi(tmplId.c_str(), conversation, message_count, true, formatted_chat.data(), formatted_chat.size(), partsTypes.data(), partsLens.data(), &numParts); + assert(res > 0); + assert(numParts <= numPartsCrossCheck); + formatted_chat.resize(res); + std::string output(formatted_chat.data(), formatted_chat.size()); + std::cout << "*** TAGGEDMSG ***\n" << output; + std::cout << "\n-----------------------------------------\n"; + partsLens.resize(numParts); + partsTypes.resize(numParts); + std::string sPartsTypes(partsTypes.data(), partsTypes.size()); + chaton_llama_tokenize_ex(nullptr, output, sPartsTypes, partsLens, false, false); + std::cout << "\n----------" << tmplId << "---------------\n"; + } + +} + + +int main(int argc, char **argv) { + if (argc != 2) { + std::cout << argv[0] << " meta.json" << std::endl; + exit(1); + } + std::string metaJson(argv[1]); + //check_chaton(metaJson); + check_chaton_ex(metaJson); + return 0; +} diff --git a/tests/test-chaton-groupkv.cpp b/tests/test-chaton-groupkv.cpp new file mode 100644 index 0000000000000..150adf4f67888 --- /dev/null +++ b/tests/test-chaton-groupkv.cpp @@ -0,0 +1,66 @@ +// +// Test GroupKV +// + +#include "groupkv.hpp" + + +static void gkv_inited() { + GroupKV gkv = {{ + {"Group1",{ + {"testkey11", 11}, + {"testkey12", true} + }}, + {"Group2", { + {"key21", "val21"}, + {"key22", 22}, + {"key23", 2.3} + }} + }}; + + std::cout << "**** gkv inited **** " << std::endl; + std::cout << gkv.dump("", "INFO:GKV:Inited") << std::endl; + +} + +static void gkv_set() { + + std::cout << "**** gkv set **** " << std::endl; + GroupKV gkv = {{}}; + std::cout << gkv.dump("", "INFO:GKV:Set:Initial") << std::endl; + + gkv.get_value("testme", {"key101b"}, false); + gkv.get_value("testme", {"key101s"}, "Not found"); + gkv.get_value("testme", {"key101i"}, 123456); + gkv.get_value("testme", {"key101d"}, 123456.789); + + gkv.set_value("testme", {"key201b"}, true); + gkv.set_value("testme", {"key201s"}, "hello world"); + gkv.set_value("testme", {"key201i"}, 987654); + gkv.set_value("testme", {"key201d"}, 9988.7766); + + std::cout << gkv.dump("testme", "INFO:GKV:Set:After testme set") << std::endl; + gkv.get_value("testme", {"key201b"}, false); + gkv.get_value("testme", {"key201s"}, "Not found"); + gkv.get_value("testme", {"key201i"}, 123456); + gkv.get_value("testme", {"key201d"}, 123456.789); + + gkv.get_vector("testme", {"keyA100"}, {1, 2, 3}); + gkv.get_vector("testme", {"keyA100"}, { "A", "അ", "अ", "ಅ" }); + gkv.set_value("testme", {"keyA300-0"}, 330); + gkv.set_value("testme", {"keyA300-1"}, 331); + gkv.set_value("testme", {"keyA300-2"}, 332); + gkv.set_value("testme", {"keyA301-0"}, "India"); + gkv.set_value("testme", {"keyA301", "1"}, "World"); + gkv.set_value("testme", {"keyA301", "2"}, "AkashaGanga"); + gkv.get_vector("testme", {"keyA300"}, {1, 2, 3}); + gkv.get_vector("testme", {"keyA301"}, { "yes 1", "No 2", "very well 3" }); +} + +int main(int argc, char **argv) { + log_set_target(log_filename_generator("chaton-groupkv", "log")); + log_dump_cmdline(argc, argv); + gkv_inited(); + gkv_set(); + return 0; +} diff --git a/tests/test-chaton-simpcfg.cpp b/tests/test-chaton-simpcfg.cpp new file mode 100644 index 0000000000000..3703a50a18015 --- /dev/null +++ b/tests/test-chaton-simpcfg.cpp @@ -0,0 +1,176 @@ +// +// Test SimpCfg +// + +#include "simpcfg.hpp" + +#include +#include + + +static void check_string() { + std::vector vStandard = { "123", "1अ3" }; + std::cout << "**** string **** " << vStandard.size() << std::endl; + for(auto sCur: vStandard) { + std::cout << std::format("string: [{}] len[{}] size[{}]", sCur, sCur.length(), sCur.size()) << std::endl; + int i = 0; + for(auto c: sCur) { + std::cout << std::format("string:{}:pos:{}:char:{}[0x{:x}]\n", sCur, i, c, (uint8_t)c); + i += 1; + } + } +} + +static void check_u8string() { + std::vector vU8s = { u8"123", u8"1अ3" }; + std::cout << "**** u8string **** " << vU8s.size() << std::endl; + for(auto sCur: vU8s) { + std::string sCurx (sCur.begin(), sCur.end()); + std::cout << std::format("u8string: [{}] len[{}] size[{}]", sCurx, sCur.length(), sCur.size()) << std::endl; + int i = 0; + for(auto c: sCur) { + //std::cout << c << std::endl; + std::cout << std::format("u8string:{}:pos:{}:char:{}[0x{:x}]\n", sCurx, i, (unsigned char)c, (unsigned char)c); + i += 1; + } + } +} + +static void check_wstring_wcout() { + std::wcout.imbue(std::locale("en_US.UTF-8")); + std::vector vWide = { L"123", L"1अ3" }; + std::cout << "**** wstring wcout **** " << vWide.size() << std::endl; + for(auto sCur: vWide) { + std::wcout << sCur << std::endl; + std::wcout << std::format(L"wstring: [{}] len[{}] size[{}]", sCur, sCur.length(), sCur.size()) << std::endl; + int i = 0; + for(auto c: sCur) { + std::wcout << std::format(L"wstring:{}:pos:{}:char:{}[0x{:x}]\n", sCur, i, c, c); + i += 1; + } + } +} + +static void check_wstring_cout() { + std::vector vWide = { L"123", L"1अ3" }; + std::cout << "**** wstring cout **** " << vWide.size() << std::endl; + for(auto sCur: vWide) { + std::string sCury; + wcs_to_mbs(sCury, sCur); + std::cout << std::format("wstring: [{}] len[{}] size[{}]", sCury, sCur.length(), sCur.size()) << std::endl; + int i = 0; + for(auto c: sCur) { + std::wstringstream wsc; + wsc << c; + std::string ssc; + wcs_to_mbs(ssc, wsc.str()); + std::cout << std::format("wstring:{}:pos:{}:char:{}[0x{:x}]\n", sCury, i, ssc, (uint32_t)c); + i += 1; + } + } +} + +static void check_nonenglish() { + std::cout << "**** non english **** " << std::endl; + std::vector vTest1 = { "\n\tAഅअಅ\n\t", "\n\tAഅअಅ " }; + for (auto sTest: vTest1) { + std::string sGotDumb = str_trim_dumb(sTest, {" \n\t"}); + std::string sGotOSmart = str_trim_oversmart(sTest, {" \n\t"}); + std::string sLower = str_tolower(sTest); + std::cout << std::format("{}: Test1 [{}]\n\tTrimDumb[{}]\n\tTrimOverSmart[{}]\n\tLowerDumb[{}]", __func__, sTest, sGotDumb, sGotOSmart, sLower) << std::endl; + } + // The string "\n\tthis र remove 0s and अs at end 000रअ0\xa4अ ", + // * will mess up str_trim_dumb, + // * but will rightly trigger a exception with oversmart. + std::vector vTest2 = { "\n\t this र remove 0s at end 000 ", "\n\tthis र remove 0s, अs, ഇs at end 000रअ0अ ", "\n\tthis र remove 0s, अs, ഇs at end 000रअ0इअ "}; + std::string trimChars = {" \n\tഇ0अ"}; + for (auto sTest: vTest2) { + std::string sGotDumb = str_trim_dumb(sTest, trimChars); + std::string sGotOSmart = str_trim_oversmart(sTest, trimChars); + std::cout << std::format("{}: Test2 [{}]\n\tDumb[{}]\n\tOverSmart[{}]", __func__, sTest, sGotDumb, sGotOSmart) << std::endl; + } +} + +static void check_strings() { + std::string sSavedLocale; + SimpCfg::locale_prepare(sSavedLocale); + check_string(); + check_u8string(); + check_wstring_wcout(); + check_wstring_cout(); + check_nonenglish(); + SimpCfg::locale_restore(sSavedLocale); +} + +static void sc_inited() { + SimpCfg sc = {{ + {"Group1",{ + {"testkey11", 11}, + {"testkey12", true} + }}, + {"Group2", { + {"key21", "val21"}, + {"key22", 22}, + {"key23", 2.3} + }} + }}; + + std::cout << "**** sc inited **** " << std::endl; + std::cout << sc.dump("", "INFO:SC:Inited") << std::endl; + +} + +static void sc_set(const std::string &fname) { + + std::cout << "**** sc set **** " << std::endl; + SimpCfg sc = {{}}; + sc.load(fname); + std::cout << sc.dump("", "INFO:SC:Set:AfterLoad") << std::endl; + + sc.get_bool("testme", {"key101b"}, false); + sc.get_string("testme", {"key101s"}, "Not found"); + sc.get_int64("testme", {"key101i"}, 123456); + sc.get_double("testme", {"key101d"}, 123456.789); + + sc.set_bool("testme", {"key201b"}, true); + sc.set_string("testme", {"key201s"}, "hello world"); + sc.set_int64("testme", {"key201i"}, 987654); + sc.set_double("testme", {"key201d"}, 9988.7766); + + std::cout << sc.dump("testme", "INFO:SC:Set:AfterSet") << std::endl; + sc.get_bool("testme", {"key201b"}, false); + sc.get_string("testme", {"key201s"}, "Not found"); + sc.get_int64("testme", {"key201i"}, 123456); + sc.get_double("testme", {"key201d"}, 123456.789); + + sc.get_string("mistral", {"system-prefix"}, "Not found"); + sc.get_string("\"mistral\"", {"\"system-prefix\""}, "Not found"); + + sc.get_vector("testme", {"keyA100"}, {1, 2, 3}); + sc.get_vector("testme", {"keyA100"}, { "A", "അ", "अ", "ಅ" }); + sc.set_int64("testme", {"keyA300-0"}, 330); + sc.set_int64("testme", {"keyA300-1"}, 331); + sc.set_int64("testme", {"keyA300-2"}, 332); + sc.set_string("testme", {"keyA301-0"}, "India"); + sc.set_value("testme", {"keyA301", "1"}, "World"); + sc.set_string("testme", {"keyA301", "2"}, "AkashaGanga"); + sc.get_vector("testme", {"keyA300"}, {1, 2, 3}); + sc.get_vector("testme", {"keyA301"}, { "yes 1", "No 2", "very well 3" }); +} + +int main(int argc, char **argv) { + if (argc != 2) { + LERRR_LN("USAGE:%s simp.cfg", argv[0]); + exit(1); + } + + log_set_target(log_filename_generator("chaton-simpcfg", "log")); + log_dump_cmdline(argc, argv); + + check_strings(); + sc_inited(); + std::string fname {argv[1]}; + sc_set(fname); + + return 0; +}