diff --git a/code/handlers/card_common_action.go b/code/handlers/card_common_action.go index 29ef6a8e..1f1a7ac8 100644 --- a/code/handlers/card_common_action.go +++ b/code/handlers/card_common_action.go @@ -4,8 +4,6 @@ import ( "context" "encoding/json" "fmt" - "start-feishubot/logger" - larkcard "github.com/larksuite/oapi-sdk-go/v3/card" ) @@ -20,11 +18,13 @@ func NewCardHandler(m MessageHandler) CardHandlerFunc { handlers := []CardHandlerMeta{ NewClearCardHandler, NewPicResolutionHandler, + NewVisionResolutionHandler, NewPicTextMoreHandler, NewPicModeChangeHandler, NewRoleTagCardHandler, NewRoleCardHandler, NewAIModeCardHandler, + NewVisionModeChangeHandler, } return func(ctx context.Context, cardAction *larkcard.CardAction) (interface{}, error) { @@ -35,7 +35,7 @@ func NewCardHandler(m MessageHandler) CardHandlerFunc { return nil, err } //pp.Println(cardMsg) - logger.Debug("cardMsg ", cardMsg) + //logger.Debug("cardMsg ", cardMsg) for _, handler := range handlers { h := handler(cardMsg, m) i, err := h(ctx, cardAction) diff --git a/code/handlers/card_vision_action.go b/code/handlers/card_vision_action.go new file mode 100644 index 00000000..9e056492 --- /dev/null +++ b/code/handlers/card_vision_action.go @@ -0,0 +1,74 @@ +package handlers + +import ( + "context" + "fmt" + larkcard "github.com/larksuite/oapi-sdk-go/v3/card" + larkcore "github.com/larksuite/oapi-sdk-go/v3/core" + "start-feishubot/services" +) + +func NewVisionResolutionHandler(cardMsg CardMsg, + m MessageHandler) CardHandlerFunc { + return func(ctx context.Context, cardAction *larkcard.CardAction) (interface{}, error) { + if cardMsg.Kind == VisionStyleKind { + CommonProcessVisionStyle(cardMsg, cardAction, m.sessionCache) + return nil, nil + } + return nil, ErrNextHandler + } +} +func NewVisionModeChangeHandler(cardMsg CardMsg, + m MessageHandler) CardHandlerFunc { + return func(ctx context.Context, cardAction *larkcard.CardAction) (interface{}, error) { + if cardMsg.Kind == VisionModeChangeKind { + newCard, err, done := CommonProcessVisionModeChange(cardMsg, m.sessionCache) + if done { + return newCard, err + } + return nil, nil + } + return nil, ErrNextHandler + } +} + +func CommonProcessVisionStyle(msg CardMsg, + cardAction *larkcard.CardAction, + cache services.SessionServiceCacheInterface) { + option := cardAction.Action.Option + fmt.Println(larkcore.Prettify(msg)) + cache.SetVisionDetail(msg.SessionId, services.VisionDetail(option)) + //send text + replyMsg(context.Background(), "图片解析度调整为："+option, + &msg.MsgId) +} + +func CommonProcessVisionModeChange(cardMsg CardMsg, + session services.SessionServiceCacheInterface) ( + interface{}, error, bool) { + if cardMsg.Value == "1" { + + sessionId := cardMsg.SessionId + session.Clear(sessionId) + session.SetMode(sessionId, + services.ModeVision) + session.SetVisionDetail(sessionId, + services.VisionDetailLow) + + newCard, _ := + newSendCard( + withHeader("🕵️️ 已进入图片推理模式", larkcard.TemplateBlue), + withVisionDetailLevelBtn(&sessionId), + withNote("提醒：回复图片，让LLM和你一起推理图片的内容。")) + return newCard, nil, true + } + if cardMsg.Value == "0" { + newCard, _ := newSendCard( + withHeader("️🎒 机器人提醒", larkcard.TemplateGreen), + withMainMd("依旧保留此话题的上下文信息"), + withNote("我们可以继续探讨这个话题,期待和您聊天。如果您有其他问题或者想要讨论的话题，请告诉我哦"), + ) + return newCard, nil, true + } + return nil, nil, false +} diff --git a/code/handlers/common.go b/code/handlers/common.go index 1d0f7b64..ffe4912a 100644 --- a/code/handlers/common.go +++ b/code/handlers/common.go @@ -13,7 +13,6 @@ func msgFilter(msg string) string { //replace @到下一个非空的字段 为 '' regex := regexp.MustCompile(`@[^ ]*`) return regex.ReplaceAllString(msg, "") - } // Parse rich text json to text @@ -47,6 +46,33 @@ func parsePostContent(content string) string { return msgFilter(text) } +func parsePostImageKeys(content string) []string { + var contentMap map[string]interface{} + err := json.Unmarshal([]byte(content), &contentMap) + + if err != nil { + fmt.Println(err) + return nil + } + + var imageKeys []string + + if contentMap["content"] == nil { + return imageKeys + } + + contentList := contentMap["content"].([]interface{}) + for _, v := range contentList { + for _, v1 := range v.([]interface{}) { + if v1.(map[string]interface{})["tag"] == "img" { + imageKeys = append(imageKeys, v1.(map[string]interface{})["image_key"].(string)) + } + } + } + + return imageKeys +} + func parseContent(content, msgType string) string { //"{\"text\":\"@_user_1 hahaha\"}", //only get text content hahaha diff --git a/code/handlers/event_common_action.go b/code/handlers/event_common_action.go index 5139060d..190d66bd 100644 --- a/code/handlers/event_common_action.go +++ b/code/handlers/event_common_action.go @@ -19,6 +19,7 @@ type MsgInfo struct { qParsed string fileKey string imageKey string + imageKeys []string // post 消息卡片中的图片组 sessionId *string mention []*larkim.MentionEvent } diff --git a/code/handlers/event_msg_action.go b/code/handlers/event_msg_action.go index da7e5b4b..f9e10f71 100644 --- a/code/handlers/event_msg_action.go +++ b/code/handlers/event_msg_action.go @@ -23,6 +23,23 @@ func setDefaultPrompt(msg []openai.Messages) []openai.Messages { return msg } +//func setDefaultVisionPrompt(msg []openai.VisionMessages) []openai.VisionMessages { +// if !hasSystemRole(msg) { +// msg = append(msg, openai.VisionMessages{ +// Role: "system", Content: []openai.ContentType{ +// {Type: "text", Text: "You are ChatGPT4V, " + +// "You are ChatGPT4V, " + +// "a large language and picture model trained by" + +// " OpenAI. " + +// "Answer in user's language as concisely as" + +// " possible. Knowledge cutoff: 20230601 " + +// "Current date" + time.Now().Format("20060102"), +// }}, +// }) +// } +// return msg +//} + type MessageAction struct { /*消息*/ } diff --git a/code/handlers/event_vision_action.go b/code/handlers/event_vision_action.go new file mode 100644 index 00000000..ae67873b --- /dev/null +++ b/code/handlers/event_vision_action.go @@ -0,0 +1,160 @@ +package handlers + +import ( + "context" + "fmt" + "os" + "start-feishubot/initialization" + "start-feishubot/services" + "start-feishubot/services/openai" + "start-feishubot/utils" + + larkim "github.com/larksuite/oapi-sdk-go/v3/service/im/v1" +) + +type VisionAction struct { /*图片推理*/ +} + +func (va *VisionAction) Execute(a *ActionInfo) bool { + if !AzureModeCheck(a) { + return true + } + + if isVisionCommand(a) { + initializeVisionMode(a) + sendVisionInstructionCard(*a.ctx, a.info.sessionId, a.info.msgId) + return false + } + + mode := a.handler.sessionCache.GetMode(*a.info.sessionId) + + if a.info.msgType == "image" { + if mode != services.ModeVision { + sendVisionModeCheckCard(*a.ctx, a.info.sessionId, a.info.msgId) + return false + } + + return va.handleVisionImage(a) + } + + if a.info.msgType == "post" && mode == services.ModeVision { + return va.handleVisionPost(a) + } + + return true +} + +func isVisionCommand(a *ActionInfo) bool { + _, foundPic := utils.EitherTrimEqual(a.info.qParsed, "/vision", "图片推理") + return foundPic +} + +func initializeVisionMode(a *ActionInfo) { + a.handler.sessionCache.Clear(*a.info.sessionId) + a.handler.sessionCache.SetMode(*a.info.sessionId, services.ModeVision) + a.handler.sessionCache.SetVisionDetail(*a.info.sessionId, services.VisionDetailHigh) +} + +func (va *VisionAction) handleVisionImage(a *ActionInfo) bool { + detail := a.handler.sessionCache.GetVisionDetail(*a.info.sessionId) + base64, err := downloadAndEncodeImage(a.info.imageKey, a.info.msgId) + if err != nil { + replyWithErrorMsg(*a.ctx, err, a.info.msgId) + return false + } + + return va.processImageAndReply(a, base64, detail) +} + +func (va *VisionAction) handleVisionPost(a *ActionInfo) bool { + detail := a.handler.sessionCache.GetVisionDetail(*a.info.sessionId) + var base64s []string + + for _, imageKey := range a.info.imageKeys { + if imageKey == "" { + continue + } + base64, err := downloadAndEncodeImage(imageKey, a.info.msgId) + if err != nil { + replyWithErrorMsg(*a.ctx, err, a.info.msgId) + return false + } + base64s = append(base64s, base64) + } + + if len(base64s) == 0 { + replyMsg(*a.ctx, "🤖️：请发送一张图片", a.info.msgId) + return false + } + + return va.processMultipleImagesAndReply(a, base64s, detail) +} + +func downloadAndEncodeImage(imageKey string, msgId *string) (string, error) { + f := fmt.Sprintf("%s.png", imageKey) + defer os.Remove(f) + + req := larkim.NewGetMessageResourceReqBuilder().MessageId(*msgId).FileKey(imageKey).Type("image").Build() + resp, err := initialization.GetLarkClient().Im.MessageResource.Get(context.Background(), req) + if err != nil { + return "", err + } + + resp.WriteFile(f) + return openai.GetBase64FromImage(f) +} + +func replyWithErrorMsg(ctx context.Context, err error, msgId *string) { + replyMsg(ctx, fmt.Sprintf("🤖️：图片下载失败，请稍后再试～\n 错误信息: %v", err), msgId) +} + +func (va *VisionAction) processImageAndReply(a *ActionInfo, base64 string, detail string) bool { + msg := createVisionMessages("解释这个图片", base64, detail) + completions, err := a.handler.gpt.GetVisionInfo(msg) + if err != nil { + replyWithErrorMsg(*a.ctx, err, a.info.msgId) + return false + } + sendVisionTopicCard(*a.ctx, a.info.sessionId, a.info.msgId, completions.Content) + return false +} + +func (va *VisionAction) processMultipleImagesAndReply(a *ActionInfo, base64s []string, detail string) bool { + msg := createMultipleVisionMessages(a.info.qParsed, base64s, detail) + completions, err := a.handler.gpt.GetVisionInfo(msg) + if err != nil { + replyWithErrorMsg(*a.ctx, err, a.info.msgId) + return false + } + sendVisionTopicCard(*a.ctx, a.info.sessionId, a.info.msgId, completions.Content) + return false +} + +func createVisionMessages(query, base64Image, detail string) []openai.VisionMessages { + return []openai.VisionMessages{ + { + Role: "user", + Content: []openai.ContentType{ + {Type: "text", Text: query}, + {Type: "image_url", ImageURL: &openai.ImageURL{ + URL: "data:image/jpeg;base64," + base64Image, + Detail: detail, + }}, + }, + }, + } +} + +func createMultipleVisionMessages(query string, base64Images []string, detail string) []openai.VisionMessages { + content := []openai.ContentType{{Type: "text", Text: query}} + for _, base64Image := range base64Images { + content = append(content, openai.ContentType{ + Type: "image_url", + ImageURL: &openai.ImageURL{ + URL: "data:image/jpeg;base64," + base64Image, + Detail: detail, + }, + }) + } + return []openai.VisionMessages{{Role: "user", Content: content}} +} diff --git a/code/handlers/handler.go b/code/handlers/handler.go index 3458c99d..d7622f1b 100644 --- a/code/handlers/handler.go +++ b/code/handlers/handler.go @@ -82,6 +82,7 @@ func (m MessageHandler) msgReceivedHandler(ctx context.Context, event *larkim.P2 qParsed: strings.Trim(parseContent(*content, msgType), " "), fileKey: parseFileKey(*content), imageKey: parseImageKey(*content), + imageKeys: parsePostImageKeys(*content), sessionId: sessionId, mention: mention, } @@ -94,8 +95,8 @@ func (m MessageHandler) msgReceivedHandler(ctx context.Context, event *larkim.P2 &ProcessedUniqueAction{}, //避免重复处理 &ProcessMentionAction{}, //判断机器人是否应该被调用 &AudioAction{}, //语音处理 - &EmptyAction{}, //空消息处理 &ClearAction{}, //清除消息处理 + &VisionAction{}, //图片推理处理 &PicAction{}, //图片处理 &AIModeAction{}, //模式切换处理 &RoleListAction{}, //角色列表处理 @@ -103,8 +104,8 @@ func (m MessageHandler) msgReceivedHandler(ctx context.Context, event *larkim.P2 &BalanceAction{}, //余额处理 &RolePlayAction{}, //角色扮演处理 &MessageAction{}, //消息处理 + &EmptyAction{}, //空消息处理 &StreamMessageAction{}, //流式消息处理 - } chain(data, actions...) return nil diff --git a/code/handlers/msg.go b/code/handlers/msg.go index 1b359864..3b752149 100644 --- a/code/handlers/msg.go +++ b/code/handlers/msg.go @@ -21,15 +21,17 @@ type CardKind string type CardChatType string var ( - ClearCardKind = CardKind("clear") // 清空上下文 - PicModeChangeKind = CardKind("pic_mode_change") // 切换图片创作模式 - PicResolutionKind = CardKind("pic_resolution") // 图片分辨率调整 - PicStyleKind = CardKind("pic_style") // 图片风格调整 - PicTextMoreKind = CardKind("pic_text_more") // 重新根据文本生成图片 - PicVarMoreKind = CardKind("pic_var_more") // 变量图片 - RoleTagsChooseKind = CardKind("role_tags_choose") // 内置角色所属标签选择 - RoleChooseKind = CardKind("role_choose") // 内置角色选择 - AIModeChooseKind = CardKind("ai_mode_choose") // 发散模式选择 + ClearCardKind = CardKind("clear") // 清空上下文 + PicModeChangeKind = CardKind("pic_mode_change") // 切换图片创作模式 + VisionModeChangeKind = CardKind("vision_mode") // 切换图片解析模式 + PicResolutionKind = CardKind("pic_resolution") // 图片分辨率调整 + PicStyleKind = CardKind("pic_style") // 图片风格调整 + VisionStyleKind = CardKind("vision_style") // 图片推理级别调整 + PicTextMoreKind = CardKind("pic_text_more") // 重新根据文本生成图片 + PicVarMoreKind = CardKind("pic_var_more") // 变量图片 + RoleTagsChooseKind = CardKind("role_tags_choose") // 内置角色所属标签选择 + RoleChooseKind = CardKind("role_choose") // 内置角色选择 + AIModeChooseKind = CardKind("ai_mode_choose") // AI模式选择 ) var ( @@ -311,6 +313,30 @@ func withPicModeDoubleCheckBtn(sessionID *string) larkcard. return actions } +func withVisionModeDoubleCheckBtn(sessionID *string) larkcard. + MessageCardElement { + confirmBtn := newBtn("切换模式", map[string]interface{}{ + "value": "1", + "kind": VisionModeChangeKind, + "chatType": UserChatType, + "sessionId": *sessionID, + }, larkcard.MessageCardButtonTypeDanger, + ) + cancelBtn := newBtn("我再想想", map[string]interface{}{ + "value": "0", + "kind": VisionModeChangeKind, + "sessionId": *sessionID, + "chatType": UserChatType, + }, + larkcard.MessageCardButtonTypeDefault) + + actions := larkcard.NewMessageCardAction(). + Actions([]larkcard.MessageCardActionElement{confirmBtn, cancelBtn}). + Layout(larkcard.MessageCardActionLayoutBisected.Ptr()). + Build() + + return actions +} func withOneBtn(btn *larkcard.MessageCardEmbedButton) larkcard. MessageCardElement { @@ -380,6 +406,32 @@ func withPicResolutionBtn(sessionID *string) larkcard. return actions } +func withVisionDetailLevelBtn(sessionID *string) larkcard. + MessageCardElement { + detailMenu := newMenu("选择图片解析度，默认为高", + map[string]interface{}{ + "value": "0", + "kind": VisionStyleKind, + "sessionId": *sessionID, + "msgId": *sessionID, + }, + MenuOption{ + label: "高", + value: string(services.VisionDetailHigh), + }, + MenuOption{ + label: "低", + value: string(services.VisionDetailLow), + }, + ) + + actions := larkcard.NewMessageCardAction(). + Actions([]larkcard.MessageCardActionElement{detailMenu}). + Layout(larkcard.MessageCardActionLayoutBisected.Ptr()). + Build() + + return actions +} func withRoleTagsBtn(sessionID *string, tags ...string) larkcard. MessageCardElement { var menuOptions []MenuOption @@ -669,6 +721,15 @@ func sendPicCreateInstructionCard(ctx context.Context, replyCard(ctx, msgId, newCard) } +func sendVisionInstructionCard(ctx context.Context, + sessionId *string, msgId *string) { + newCard, _ := newSendCard( + withHeader("🕵️️ 已进入图片推理模式", larkcard.TemplateBlue), + withVisionDetailLevelBtn(sessionId), + withNote("提醒：回复图片，让LLM和你一起推理图片的内容。")) + replyCard(ctx, msgId, newCard) +} + func sendPicModeCheckCard(ctx context.Context, sessionId *string, msgId *string) { newCard, _ := newSendCard( @@ -678,6 +739,15 @@ func sendPicModeCheckCard(ctx context.Context, withPicModeDoubleCheckBtn(sessionId)) replyCard(ctx, msgId, newCard) } +func sendVisionModeCheckCard(ctx context.Context, + sessionId *string, msgId *string) { + newCard, _ := newSendCard( + withHeader("🕵️ 机器人提醒", larkcard.TemplateBlue), + withMainMd("检测到图片，是否进入图片推理模式？"), + withNote("请注意，这将开始一个全新的对话，您将无法利用之前话题的历史信息"), + withVisionModeDoubleCheckBtn(sessionId)) + replyCard(ctx, msgId, newCard) +} func sendNewTopicCard(ctx context.Context, sessionId *string, msgId *string, content string) { @@ -697,11 +767,20 @@ func sendOldTopicCard(ctx context.Context, replyCard(ctx, msgId, newCard) } +func sendVisionTopicCard(ctx context.Context, + sessionId *string, msgId *string, content string) { + newCard, _ := newSendCard( + withHeader("🕵️图片推理结果", larkcard.TemplateBlue), + withMainText(content), + withNote("让LLM和你一起推理图片的内容~")) + replyCard(ctx, msgId, newCard) +} + func sendHelpCard(ctx context.Context, sessionId *string, msgId *string) { newCard, _ := newSendCard( withHeader("🎒需要帮助吗？", larkcard.TemplateBlue), - withMainMd("**我是小飞机，一款基于chatGpt技术的智能聊天机器人！**"), + withMainMd("**🤠你好呀~ 我来自企联AI，一款基于OpenAI的智能助手！**"), withSplitLine(), withMdAndExtraBtn( "** 🆑 清除话题上下文**\n文本回复 *清除* 或 */clear*", @@ -722,6 +801,8 @@ func sendHelpCard(ctx context.Context, withSplitLine(), withMainMd("🎨 **图片创作模式**\n回复*图片创作* 或 */picture*"), withSplitLine(), + withMainMd("🕵️ **图片推理模式** \n"+" 文本回复 *图片推理* 或 */vision*"), + withSplitLine(), withMainMd("🎰 **Token余额查询**\n回复*余额* 或 */balance*"), withSplitLine(), withMainMd("🔃️ **历史话题回档** 🚧\n"+" 进入话题的回复详情页,文本回复 *恢复* 或 */reload*"), diff --git a/code/services/openai/common.go b/code/services/openai/common.go index e7db23b6..85f6341e 100644 --- a/code/services/openai/common.go +++ b/code/services/openai/common.go @@ -18,6 +18,9 @@ import ( type PlatForm string +const ( + MaxRetries = 3 +) const ( AzureApiUrlV1 = "openai.azure.com/openai/deployments/" ) @@ -104,6 +107,7 @@ func (gpt *ChatGPT) doAPIRequestWithRetry(url, method string, return errors.New("no available API") } + //fmt.Println("requestBodyData", string(requestBodyData)) req, err := http.NewRequest(method, url, bytes.NewReader(requestBodyData)) if err != nil { return err @@ -182,7 +186,7 @@ func (gpt *ChatGPT) sendRequestWithBodyType(link, method string, } err = gpt.doAPIRequestWithRetry(link, method, bodyType, - requestBody, responseBody, client, 3) + requestBody, responseBody, client, MaxRetries) return err } @@ -252,3 +256,8 @@ func GetProxyClient(proxyString string) (*http.Client, error) { } return client, nil } + +func (gpt *ChatGPT) ChangeMode(model string) *ChatGPT { + gpt.Model = model + return gpt +} diff --git a/code/services/openai/gpt3_test.go b/code/services/openai/gpt3_test.go index fb442972..e13c4347 100644 --- a/code/services/openai/gpt3_test.go +++ b/code/services/openai/gpt3_test.go @@ -23,6 +23,28 @@ func TestCompletions(t *testing.T) { fmt.Println(resp.Content, resp.Role) } +func TestVisionOnePic(t *testing.T) { + config := initialization.LoadConfig("../../config.yaml") + content := []ContentType{ + {Type: "text", Text: "What’s in this image?", ImageURL: nil}, + {Type: "image_url", ImageURL: &ImageURL{ + URL: "https://resource.liaobots." + + "com/1849d492904448a0ac17f975f0b7ca8b.jpg", + Detail: "high", + }}, + } + + msgs := []VisionMessages{ + {Role: "assistant", Content: content}, + } + gpt := NewChatGPT(*config) + resp, err := gpt.GetVisionInfo(msgs) + if err != nil { + t.Errorf("TestCompletions failed with error: %v", err) + } + fmt.Println(resp.Content, resp.Role) +} + func TestGenerateOneImage(t *testing.T) { config := initialization.LoadConfig("../../config.yaml") gpt := NewChatGPT(*config) diff --git a/code/services/openai/picture.go b/code/services/openai/picture.go index d93bedda..1c0ace43 100644 --- a/code/services/openai/picture.go +++ b/code/services/openai/picture.go @@ -2,11 +2,13 @@ package openai import ( "bufio" + "encoding/base64" "fmt" "image" "image/jpeg" "image/png" "io" + "io/ioutil" "mime/multipart" "os" ) @@ -300,3 +302,22 @@ func GetImageCompressionType(path string) (string, error) { // 返回压缩类型 return format, nil } + +func GetBase64FromImage(imagePath string) (string, error) { + // 打开文件 + // 读取图片文件 + imageFile, err := os.Open(imagePath) + if err != nil { + return "", err + } + defer imageFile.Close() + // 读取图片内容 + imageData, err := ioutil.ReadAll(imageFile) + if err != nil { + return "", err + } + // 将图片内容转换为base64编码 + base64String := base64.StdEncoding.EncodeToString(imageData) + + return base64String, nil +} diff --git a/code/services/openai/vision.go b/code/services/openai/vision.go new file mode 100644 index 00000000..8523ab3f --- /dev/null +++ b/code/services/openai/vision.go @@ -0,0 +1,53 @@ +package openai + +import ( + "errors" + "start-feishubot/logger" +) + +type ImageURL struct { + URL string `json:"url,omitempty"` + Detail string `json:"detail,omitempty"` +} + +type ContentType struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + ImageURL *ImageURL `json:"image_url,omitempty"` +} +type VisionMessages struct { + Role string `json:"role"` + Content interface{} `json:"content"` +} + +type VisionRequestBody struct { + Model string `json:"model"` + Messages []VisionMessages `json:"messages"` + MaxTokens int `json:"max_tokens"` +} + +func (gpt *ChatGPT) GetVisionInfo(msg []VisionMessages) ( + resp Messages, err error) { + requestBody := VisionRequestBody{ + Model: "gpt-4-vision-preview", + Messages: msg, + MaxTokens: gpt.MaxTokens, + } + gptResponseBody := &ChatGPTResponseBody{} + url := gpt.FullUrl("chat/completions") + logger.Debug("request body ", requestBody) + if url == "" { + return resp, errors.New("无法获取openai请求地址") + } + //gpt.ChangeMode("gpt-4-vision-preview") + //fmt.Println("model", gpt.Model) + err = gpt.sendRequestWithBodyType(url, "POST", jsonBody, requestBody, gptResponseBody) + if err == nil && len(gptResponseBody.Choices) > 0 { + resp = gptResponseBody.Choices[0].Message + } else { + logger.Errorf("ERROR %v", err) + resp = Messages{} + err = errors.New("openai 请求失败") + } + return resp, err +} diff --git a/code/services/sessionCache.go b/code/services/sessionCache.go index 3fd9f195..21c12ce8 100644 --- a/code/services/sessionCache.go +++ b/code/services/sessionCache.go @@ -8,6 +8,7 @@ import ( ) type SessionMode string +type VisionDetail string type SessionService struct { cache *cache.Cache } @@ -19,10 +20,11 @@ type Resolution string type PicStyle string type SessionMeta struct { - Mode SessionMode `json:"mode"` - Msg []openai.Messages `json:"msg,omitempty"` - PicSetting PicSetting `json:"pic_setting,omitempty"` - AIMode openai.AIMode `json:"ai_mode,omitempty"` + Mode SessionMode `json:"mode"` + Msg []openai.Messages `json:"msg,omitempty"` + PicSetting PicSetting `json:"pic_setting,omitempty"` + AIMode openai.AIMode `json:"ai_mode,omitempty"` + VisionDetail VisionDetail `json:"vision_detail,omitempty"` } const ( @@ -36,10 +38,15 @@ const ( PicStyleVivid PicStyle = "vivid" PicStyleNatural PicStyle = "natural" ) +const ( + VisionDetailHigh VisionDetail = "high" + VisionDetailLow VisionDetail = "low" +) const ( ModePicCreate SessionMode = "pic_create" ModePicVary SessionMode = "pic_vary" ModeGPT SessionMode = "gpt" + ModeVision SessionMode = "vision" ) type SessionServiceCacheInterface interface { @@ -52,9 +59,11 @@ type SessionServiceCacheInterface interface { GetAIMode(sessionId string) openai.AIMode SetAIMode(sessionId string, aiMode openai.AIMode) SetPicResolution(sessionId string, resolution Resolution) - SetPicStyle(sessionId string, resolution PicStyle) GetPicResolution(sessionId string) string + SetPicStyle(sessionId string, resolution PicStyle) GetPicStyle(sessionId string) string + SetVisionDetail(sessionId string, visionDetail VisionDetail) + GetVisionDetail(sessionId string) string Clear(sessionId string) } @@ -218,6 +227,29 @@ func (s *SessionService) Clear(sessionId string) { s.cache.Delete(sessionId) } +func (s *SessionService) GetVisionDetail(sessionId string) string { + sessionContext, ok := s.cache.Get(sessionId) + if !ok { + return "" + } + sessionMeta := sessionContext.(*SessionMeta) + return string(sessionMeta.VisionDetail) +} + +func (s *SessionService) SetVisionDetail(sessionId string, + visionDetail VisionDetail) { + maxCacheTime := time.Hour * 12 + sessionContext, ok := s.cache.Get(sessionId) + if !ok { + sessionMeta := &SessionMeta{VisionDetail: visionDetail} + s.cache.Set(sessionId, sessionMeta, maxCacheTime) + return + } + sessionMeta := sessionContext.(*SessionMeta) + sessionMeta.VisionDetail = visionDetail + s.cache.Set(sessionId, sessionMeta, maxCacheTime) +} + func GetSessionCache() SessionServiceCacheInterface { if sessionServices == nil { sessionServices = &SessionService{cache: cache.New(time.Hour*12, time.Hour*1)} diff --git a/readme.md b/readme.md index eae9410b..0c0dfb38 100644 --- a/readme.md +++ b/readme.md @@ -94,8 +94,8 @@ ## 🌟 项目特点 -- 🍏 对话基于 OpenAI (https://platform.openai.com/account/api-keys) 接口 -- 🍎 通过 lark，将 ChatGPT 接入[飞书](https://open.feishu.cn/app)和[飞书国际版](https://www.larksuite.com/) +- 🍏 支持 OpenAI (https://platform.openai.com/account/api-keys) 主要Chat接口:GPT4、DALL·E-3、Whisper、GPT-4V +- 🍎 将 ChatGPT 接入[飞书](https://open.feishu.cn/app)和[飞书国际版](https://www.larksuite.com/) - 🥒 支持[Serverless 云函数](https://github.com/serverless-devs/serverless-devs)、[本地环境](https://dashboard.cpolar.com/login)、[Docker](https://www.docker.com/)、[二进制安装包](https://github.com/Leizhenpeng/feishu-chatgpt/releases/) 等多种渠道部署