From 263041aa6a1f5cfd858269d1c28700b86284895b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E8=82=A5=E7=BE=8A?= <1048382248@qq.com> Date: Fri, 1 Nov 2024 18:16:21 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=20gpt=E8=AF=B4=E8=AF=9D?= =?UTF-8?q?=E4=BA=BA=E6=B5=81=E7=A8=8B=20=E8=AE=A1=E7=AE=97token=E6=95=B0?= =?UTF-8?q?=E9=87=8F=E6=B5=81=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VideoAnalysis/Program.cs | 3 ++ .../AICore/ChatGPT/Dto/CallGPTRes.cs | 40 +++++++++++++++++ .../AICore/ChatGPT/KIMI/KIMI_GPT.cs | 45 +++++++++++++------ .../AICore/ChatGPT/KIMI/MoonshotClient.cs | 6 ++- .../AICore/ChatGPT/KIMI/MoonshotModel.cs | 14 ++++++ VideoAnalysisCore/Common/AppCommon.cs | 10 ++--- VideoAnalysisCore/Enum/QuestionTypeEnum.cs | 20 +++++++++ .../Model/Dto/SpeakerCaptionsDto.cs | 6 ++- 8 files changed, 123 insertions(+), 21 deletions(-) create mode 100644 VideoAnalysisCore/AICore/ChatGPT/Dto/CallGPTRes.cs create mode 100644 VideoAnalysisCore/Enum/QuestionTypeEnum.cs diff --git a/VideoAnalysis/Program.cs b/VideoAnalysis/Program.cs index 19d389c..f6efdb5 100644 --- a/VideoAnalysis/Program.cs +++ b/VideoAnalysis/Program.cs @@ -8,12 +8,15 @@ using VideoAnalysisCore.AICore.ChatGPT.KIMI; using VideoAnalysisCore.AICore.SherpaOnnx; using SqlSugar; + + namespace Learn.VideoAnalysis { public class Program { public static void Main(string[] args) { + var builder = WebApplication.CreateBuilder(args); // Add services to the container. diff --git a/VideoAnalysisCore/AICore/ChatGPT/Dto/CallGPTRes.cs b/VideoAnalysisCore/AICore/ChatGPT/Dto/CallGPTRes.cs new file mode 100644 index 0000000..6988f5f --- /dev/null +++ b/VideoAnalysisCore/AICore/ChatGPT/Dto/CallGPTRes.cs @@ -0,0 +1,40 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using VideoAnalysisCore.Enum; +using VideoAnalysisCore.Model.Dto; + +namespace VideoAnalysisCore.AICore.ChatGPT.Dto +{ + public class Assessment + { + + } + public class CallGPTRes + { + /// + /// 教师发言时间 + /// 秒 + /// + public decimal TeacherSpeaking { get; set; } = 0; + /// + /// 学生发言时间 + /// 秒 + /// + public decimal StudentSpeaking { get; set; } = 0; + /// + /// 视频时间轴 + /// + public IEnumerable? TimeBase { get; set; } + /// + /// GPT模型id + /// + public string GPTModel { get; set; } = string.Empty; + /// + /// AI综合评估 + /// + public Assessment Assessment { get; set; } + } +} diff --git a/VideoAnalysisCore/AICore/ChatGPT/KIMI/KIMI_GPT.cs b/VideoAnalysisCore/AICore/ChatGPT/KIMI/KIMI_GPT.cs index 22e46a2..6239b65 100644 --- a/VideoAnalysisCore/AICore/ChatGPT/KIMI/KIMI_GPT.cs +++ b/VideoAnalysisCore/AICore/ChatGPT/KIMI/KIMI_GPT.cs @@ -9,6 +9,9 @@ using System.Text; using FFmpeg.NET.Services; using Microsoft.Extensions.Primitives; using VideoAnalysisCore.AICore.ChatGPT.Dto; +using System.ComponentModel.DataAnnotations; +using VideoAnalysisCore.Enum; +using System.Reflection; namespace VideoAnalysisCore.AICore.ChatGPT.KIMI { @@ -50,15 +53,21 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI criteriaBuilder.Append(item.NamePrompt); criteriaBuilder.Append("返回bool值|"); } - criteriaBuilder.Append("100:分析授课中使用的高频词10个|"); - criteriaBuilder.Append("101:分析授课中引发学生思考问题的时间段数组|"); - criteriaBuilder.Append("102:分析授课中让学生产生讨论欲望的时间段|"); - criteriaBuilder.Append("104:分析授课中巧妙地引入学习主题的时间段|"); - criteriaBuilder.Append("105:分析授课中教学语言简明清晰准确生动的时间段|"); - criteriaBuilder.Append("106:分析授课中教师激发学生思考的时间段|"); - criteriaBuilder.Append("107:统计授课中教师提问类型的次数 类型重复回答,老师追问,简单性表演,老师补充答案,表扬并补充答案|"); + //拼接枚举提问 + foreach (var value in System.Enum.GetValues(typeof(QuestionTypeEnum))) + { + var enumValue = (QuestionTypeEnum)value; + var displayAttribute = enumValue.GetType() + .GetField(enumValue.ToString())? + .GetCustomAttribute(); + if (displayAttribute == null) continue; + criteriaBuilder.Append(enumValue.GetHashCode()); + criteriaBuilder.Append(":"); + criteriaBuilder.Append(displayAttribute.Prompt); + criteriaBuilder.Append("|"); + } - var resFormat = "问题编号:int,结果:array,问题解释:string"; + var resFormat = "问题编号:int,结果:array|bool,问题解释:string"; var postMessages = $"以下是一段音频的字幕,分析这段字幕(格式 说话人:开始秒:结束秒:内容|下一段字幕)." + $"来简明的回答提出的问题 问题列表 {criteriaBuilder} " + @@ -67,24 +76,32 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI var modelsResp = await moonshotClient.ListModels(); if (modelsResp is null || modelsResp.data.Count == 0) throw new Exception("未获取KIMI模型类型"); + var reqTokenCount =await moonshotClient.GetAsTiMateTokenCount(postMessages); + var modelId = reqTokenCount > 32 * 1000 ? "moonshot-v1-128k" : "moonshot-v1-32k"; var chatRep = new ChatReq { - max_tokens = postMessages.Length * 2, + max_tokens =1000 * 31, temperature = 0.3, frequency_penalty = 0, presence_penalty = 0, - model = modelsResp.data.First(s=>s.id.Contains("v1-32k")).id, + model = modelId, messages = new List(){ new MessagesItem(postMessages,"system"), + //todo 规定返回json格式 + //new MessagesItem(postMessages,"assistant"), } }; var chatResp = await moonshotClient.Chat(chatRep); - var chatResContent = chatResp?.choices.FirstOrDefault()?.message.content; - if(chatResp is null || string.IsNullOrEmpty(chatResContent)) + if(chatResp is null || chatResp.error != null) throw new Exception($"KIMI模型返回异常 Chat 请求参数: {JsonSerializer.Serialize(chatRep)} " + - $" chatResp {chatResp} chatResContent {chatResContent}"); + $" chatResp {JsonSerializer.Serialize(chatResp)}"); + var chatResContent = chatResp?.choices.FirstOrDefault()?.message.content; var questionRes = JsonSerializer.Deserialize(chatResContent); - //分析gpt返回结果 + //todo 分析gpt返回结果 + //todo 分析上课时间段情况 分析 独立学习 小组合作 随堂练习等情况 + + + } } } diff --git a/VideoAnalysisCore/AICore/ChatGPT/KIMI/MoonshotClient.cs b/VideoAnalysisCore/AICore/ChatGPT/KIMI/MoonshotClient.cs index fad7e51..9419182 100644 --- a/VideoAnalysisCore/AICore/ChatGPT/KIMI/MoonshotClient.cs +++ b/VideoAnalysisCore/AICore/ChatGPT/KIMI/MoonshotClient.cs @@ -56,12 +56,16 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI { var requestBody = Newtonsoft.Json.JsonConvert.SerializeObject(chatReq); var chatResp = await PostJsonStreamAsync("/v1/chat/completions", requestBody); + var resStr = chatResp.Content.ReadAsStringAsync(); return await chatResp.Content.ReadFromJsonAsync(); } /// - /// Get as timate token count + /// 计算token长度 /// + /// 文本 + /// + /// public async Task GetAsTiMateTokenCount(string chatReqText) { var response = await PostJsonAsync("/v1/tokenizers/estimate-token-count", chatReqText); diff --git a/VideoAnalysisCore/AICore/ChatGPT/KIMI/MoonshotModel.cs b/VideoAnalysisCore/AICore/ChatGPT/KIMI/MoonshotModel.cs index 1e46c86..024d224 100644 --- a/VideoAnalysisCore/AICore/ChatGPT/KIMI/MoonshotModel.cs +++ b/VideoAnalysisCore/AICore/ChatGPT/KIMI/MoonshotModel.cs @@ -16,6 +16,7 @@ /// /// public string role { get; set; } + public bool partial { get; set; } = false; /// /// @@ -134,8 +135,21 @@ } + public class ChatResError + { + /// + /// 错误信息 + /// + public string message { get; set; } = string.Empty; + /// + /// 错误类型 + /// + public string type { get; set; } = string.Empty; + + } public class ChatRes { + public ChatResError? error { get; set; } public string id { get; set; } public int created { get; set; } /// diff --git a/VideoAnalysisCore/Common/AppCommon.cs b/VideoAnalysisCore/Common/AppCommon.cs index 4b468bb..bd7f747 100644 --- a/VideoAnalysisCore/Common/AppCommon.cs +++ b/VideoAnalysisCore/Common/AppCommon.cs @@ -164,10 +164,11 @@ namespace VideoAnalysisCore.Common else studentSpeaking += speakerRes.Total; spList.Add(speakerRes.SpeakerIndex); - break; } } - results.Add(segment, spList); + var sp = spList.Distinct().ToList(); + if(sp.Count>0) + results.Add(segment, sp); } //拼接 提示词字幕源 var stringBuilder = new StringBuilder(); @@ -182,8 +183,6 @@ namespace VideoAnalysisCore.Common stringBuilder.Append(item.Key.Text); stringBuilder.Append("|"); } - //todo 分析上课时间段情况 分析 独立学习 小组合作 随堂练习等情况 - return new TotalCaptionsDto { StudentSpeaking = studentSpeaking, @@ -193,7 +192,8 @@ namespace VideoAnalysisCore.Common { Start = s.Key.Start.TotalSeconds, End = s.Key.End.TotalSeconds, - Type = s.Value.Count == 1 && s.Value.First() == techerId + Content = s.Key.Text, + TimeBaseType = s.Value.Count == 1 && s.Value.First() == techerId ? TimeBaseTypeEnum.教师讲授 : TimeBaseTypeEnum.互动交流 }) diff --git a/VideoAnalysisCore/Enum/QuestionTypeEnum.cs b/VideoAnalysisCore/Enum/QuestionTypeEnum.cs new file mode 100644 index 0000000..26b2f9e --- /dev/null +++ b/VideoAnalysisCore/Enum/QuestionTypeEnum.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel.DataAnnotations; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace VideoAnalysisCore.Enum +{ + enum QuestionTypeEnum + { + + [Display(Prompt = "分析授课中使用的高频词10个")] + 高频词 =100, + [Display(Prompt = "分析字幕中每5分钟的一个概览 返回结构 array[0]=概览 时间段")] + 时间段概览 = 101, + [Display(Prompt = "统计授课中教师提问类型的次数 类型重复回答,老师追问,简单性表演,老师补充答案,表扬并补充答案")] + 提问类型 = 102, + } +} diff --git a/VideoAnalysisCore/Model/Dto/SpeakerCaptionsDto.cs b/VideoAnalysisCore/Model/Dto/SpeakerCaptionsDto.cs index 5c3e340..62bc431 100644 --- a/VideoAnalysisCore/Model/Dto/SpeakerCaptionsDto.cs +++ b/VideoAnalysisCore/Model/Dto/SpeakerCaptionsDto.cs @@ -20,9 +20,13 @@ namespace VideoAnalysisCore.Model.Dto /// public double End { get; set; } /// + /// 字幕内容 + /// + public string Content { get; set; } + /// /// 时间段 类型 /// - public TimeBaseTypeEnum Type { get; set; } + public TimeBaseTypeEnum TimeBaseType { get; set; } } public class TotalCaptionsDto