From 3dfe2fc087c14cd4c686c52a134da3d7b037c6d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E8=82=A5=E7=BE=8A?= <1048382248@qq.com> Date: Thu, 31 Oct 2024 17:59:07 +0800 Subject: [PATCH] =?UTF-8?q?=E7=BC=96=E5=86=99=20kimi=20gpt=E6=8F=90?= =?UTF-8?q?=E7=A4=BA=E8=AF=8D=20=E8=BF=94=E5=9B=9E=E5=9B=BA=E5=AE=9Ajson?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VideoAnalysis/Controllers/ApiController.cs | 26 ++++------ .../AICore/ChatGPT/Dto/QuestionRes.cs | 25 ++++++++++ .../AICore/ChatGPT/KIMI/KIMI_GPT.cs | 48 +++++++++++++++--- .../AICore/ChatGPT/KIMI/MoonshotClient.cs | 34 +++++++------ .../AICore/ChatGPT/KIMI/MoonshotModel.cs | 50 +++++++++++++++++++ .../AICore/SherpaOnnx/Speaker.cs | 8 +++ VideoAnalysisCore/Common/AppCommon.cs | 12 ++++- VideoAnalysisCore/Common/RedisExpand.cs | 6 ++- VideoAnalysisCore/Model/VideoTask.cs | 22 +++++++- 9 files changed, 186 insertions(+), 45 deletions(-) create mode 100644 VideoAnalysisCore/AICore/ChatGPT/Dto/QuestionRes.cs diff --git a/VideoAnalysis/Controllers/ApiController.cs b/VideoAnalysis/Controllers/ApiController.cs index 1e30681..b2d464e 100644 --- a/VideoAnalysis/Controllers/ApiController.cs +++ b/VideoAnalysis/Controllers/ApiController.cs @@ -14,20 +14,11 @@ namespace Learn.VideoAnalysis.Controllers public class ApiController : ControllerBase { private readonly ILogger _logger; - public ApiController(ILogger logger) + private readonly Repository videoTaskDB; + public ApiController(ILogger logger, Repository videoTaskDB) { _logger = logger; - } - /// - /// 音频转换wav_16k - /// - /// 文件路径 - /// - [HttpGet(Name = "Audio2WAV16K")] - public async Task Audio2WAV16K(string filePath) - { - await FFMPGEHandle.Audio2WAV16KAsync(filePath); - return Ok(); + this.videoTaskDB = videoTaskDB; } private string GetClientIpAddress() @@ -43,7 +34,7 @@ namespace Learn.VideoAnalysis.Controllers /// - /// 测试 插入队列 + /// 插入队列 /// /// /// @@ -61,7 +52,7 @@ namespace Learn.VideoAnalysis.Controllers /// 请求体 /// [HttpPost(Name = "VideoAnalysis")] - public IActionResult VideoAnalysis(VideoAnalysisReq req) + public async Task VideoAnalysis(VideoAnalysisReq req) { if (!ModelState.IsValid) return BadRequest(ModelState); @@ -74,14 +65,15 @@ namespace Learn.VideoAnalysis.Controllers Tag = req.Tag, CallBackUrl = req.CallBackUrl, }; + //入库 + task.Id = await videoTaskDB.InsertReturnBigIdentityAsync(task); var hashEntries = task.GetType() .GetProperties(BindingFlags.Public | BindingFlags.Instance) .ToDictionary(s => s.Name, s => s.GetValue(task)); RedisExpand.Redis.HMSet(RedisExpandKey.Task(task.Id), hashEntries); - RedisExpand.InsertChannel(RedisChannelEnum.DownloadFile - , task.Id); - return Ok(); + RedisExpand.InsertChannel(RedisChannelEnum.DownloadFile,task.Id); + return Ok(task.Id); } } } diff --git a/VideoAnalysisCore/AICore/ChatGPT/Dto/QuestionRes.cs b/VideoAnalysisCore/AICore/ChatGPT/Dto/QuestionRes.cs new file mode 100644 index 0000000..df90eee --- /dev/null +++ b/VideoAnalysisCore/AICore/ChatGPT/Dto/QuestionRes.cs @@ -0,0 +1,25 @@ +锘縰sing System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace VideoAnalysisCore.AICore.ChatGPT.Dto +{ + public class QuestionRes + { + /// + /// 瀵瑰簲闂鐨刬d + /// + public int 闂缂栧彿 { get; set; } + /// + /// 缁撴灉 + /// + public object[]? 缁撴灉 { get; set; } + /// + /// 闂瑙ｉ噴 + /// + public string 闂瑙ｉ噴 { get; set; } + + } +} diff --git a/VideoAnalysisCore/AICore/ChatGPT/KIMI/KIMI_GPT.cs b/VideoAnalysisCore/AICore/ChatGPT/KIMI/KIMI_GPT.cs index 6ec29a2..22e46a2 100644 --- a/VideoAnalysisCore/AICore/ChatGPT/KIMI/KIMI_GPT.cs +++ b/VideoAnalysisCore/AICore/ChatGPT/KIMI/KIMI_GPT.cs @@ -4,6 +4,11 @@ using System.Threading.Tasks; using Whisper.net; using VideoAnalysisCore.AICore.ChatGPT; using System.Text.Json; +using VideoAnalysisCore.Model; +using System.Text; +using FFmpeg.NET.Services; +using Microsoft.Extensions.Primitives; +using VideoAnalysisCore.AICore.ChatGPT.Dto; namespace VideoAnalysisCore.AICore.ChatGPT.KIMI { @@ -13,17 +18,19 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI public class KIMI_GPT : IBserGPT { private readonly MoonshotClient moonshotClient; + private readonly Repository criteriaDB; /// /// 鍒濆鍖 /// /// /// - public KIMI_GPT(MoonshotClient moonshotClient) + public KIMI_GPT(MoonshotClient moonshotClient, Repository criteria) { MoonshotClient.Host = AppCommon.Config.ChatGpt.KIMI.Host; MoonshotClient.ApiKey = AppCommon.Config.ChatGpt.KIMI.ApiKey; this.moonshotClient = moonshotClient; + this.criteriaDB = criteria; } /// /// 璁块棶GPT @@ -33,26 +40,51 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI public async Task CallGPT(string task) { var captions = ExpandFunction.GetSpeakerCaptions(task); + var criteriaArr = await criteriaDB.GetListAsync(); + var criteriaBuilder = new StringBuilder(); + foreach (var item in criteriaArr) + { + criteriaBuilder.Append(item.Id); + criteriaBuilder.Append(":"); + criteriaBuilder.Append(item.NamePrompt); + criteriaBuilder.Append("杩斿洖bool鍊紎"); + } + criteriaBuilder.Append("100:鍒嗘瀽鎺堣涓娇鐢ㄧ殑楂橀璇10涓獆"); + criteriaBuilder.Append("101:鍒嗘瀽鎺堣涓紩鍙戝鐢熸濊冮棶棰樼殑鏃堕棿娈垫暟缁剕"); + criteriaBuilder.Append("102:鍒嗘瀽鎺堣涓瀛︾敓浜х敓璁ㄨ娆叉湜鐨勬椂闂存|"); + criteriaBuilder.Append("104:鍒嗘瀽鎺堣涓阀濡欏湴寮曞叆瀛︿範涓婚鐨勬椂闂存|"); + criteriaBuilder.Append("105:鍒嗘瀽鎺堣涓暀瀛﹁瑷绠鏄庢竻鏅板噯纭敓鍔ㄧ殑鏃堕棿娈祙"); + criteriaBuilder.Append("106:鍒嗘瀽鎺堣涓暀甯堟縺鍙戝鐢熸濊冪殑鏃堕棿娈祙"); + criteriaBuilder.Append("107:缁熻鎺堣涓暀甯堟彁闂被鍨嬬殑娆℃暟 绫诲瀷閲嶅鍥炵瓟,鑰佸笀杩介棶,绠鍗曟ц〃婕,鑰佸笀琛ュ厖绛旀,琛ㄦ壃骞惰ˉ鍏呯瓟妗坾"); + + var resFormat = "闂缂栧彿:int,缁撴灉:array,闂瑙ｉ噴:string"; + var postMessages = + $"浠ヤ笅鏄竴娈甸煶棰戠殑瀛楀箷,鍒嗘瀽杩欐瀛楀箷(鏍煎紡 璇磋瘽浜:寮濮嬬:缁撴潫绉:鍐呭|涓嬩竴娈靛瓧骞)." + + $"鏉ョ畝鏄庣殑鍥炵瓟鎻愬嚭鐨勯棶棰 闂鍒楄〃 {criteriaBuilder} " + + $"瀛楀箷鍒楄〃 {captions.Captions} " + + $"杩斿洖鍥哄畾鐨凧SON鏁扮粍鏍煎紡({resFormat})."; var modelsResp = await moonshotClient.ListModels(); if (modelsResp is null || modelsResp.data.Count == 0) throw new Exception("鏈幏鍙朘IMI妯″瀷绫诲瀷"); var chatRep = new ChatReq { - max_tokens = 1048 * 16, + max_tokens = postMessages.Length * 2, temperature = 0.3, frequency_penalty = 0, presence_penalty = 0, - model = modelsResp.data.First().id, + model = modelsResp.data.First(s=>s.id.Contains("v1-32k")).id, messages = new List(){ - new MessagesItem("浠ヤ笅鏄拰涓涓狝I鍔╂墜鐨勫璇濄傝繖浣嶅姪鎵嬩箰浜庡姪浜猴紝瀵屾湁鍒涢犲姏锛岃仾鏄庯紝鑰屼笖闈炲父鍙嬪ソ銆","system"), - new MessagesItem("缇庡浗鐨勪汉绫婚鏈熷鍛芥槸澶氬皯锛") + new MessagesItem(postMessages,"system"), } }; - Console.WriteLine($"Chat 璇锋眰鍙傛暟: {JsonSerializer.Serialize(chatRep)}"); var chatResp = await moonshotClient.Chat(chatRep); - var chatRespBody = await chatResp.Content.ReadAsStringAsync(); - Console.WriteLine($"Chat 鎴愬姛杩斿洖鍊: {chatRespBody}"); + var chatResContent = chatResp?.choices.FirstOrDefault()?.message.content; + if(chatResp is null || string.IsNullOrEmpty(chatResContent)) + throw new Exception($"KIMI妯″瀷杩斿洖寮傚父 Chat 璇锋眰鍙傛暟: {JsonSerializer.Serialize(chatRep)} " + + $" chatResp {chatResp} chatResContent {chatResContent}"); + var questionRes = JsonSerializer.Deserialize(chatResContent); + //鍒嗘瀽gpt杩斿洖缁撴灉 } } } diff --git a/VideoAnalysisCore/AICore/ChatGPT/KIMI/MoonshotClient.cs b/VideoAnalysisCore/AICore/ChatGPT/KIMI/MoonshotClient.cs index d08c622..fad7e51 100644 --- a/VideoAnalysisCore/AICore/ChatGPT/KIMI/MoonshotClient.cs +++ b/VideoAnalysisCore/AICore/ChatGPT/KIMI/MoonshotClient.cs @@ -4,12 +4,15 @@ using System.Text; using Microsoft.Extensions.Logging; using Newtonsoft.Json.Linq; using System.Net.Http; +using Newtonsoft.Json; +using System.Net.Http.Json; +/// +/// https://platform.moonshot.cn/docs/api-reference +/// namespace VideoAnalysisCore.AICore.ChatGPT.KIMI { - /// - /// https://platform.moonshot.cn/docs/api-reference - /// + public class MoonshotClient { private readonly ILogger _logger; @@ -37,9 +40,11 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI /// /// /// Return HttpResponseMessage for SSE - public async Task Chat(string requestBody) + public async Task Chat(string requestBody) { - return await PostJsonStreamAsync("/v1/chat/completions", requestBody); + var chatResp = await PostJsonStreamAsync("/v1/chat/completions", requestBody); + return await chatResp.Content.ReadFromJsonAsync(); + } /// @@ -47,10 +52,11 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI /// /// /// Return HttpResponseMessage for SSE - public async Task Chat(ChatReq chatReq) + public async Task Chat(ChatReq chatReq) { - var requestBody = System.Text.Json.JsonSerializer.Serialize(chatReq); - return await PostJsonStreamAsync("/v1/chat/completions", requestBody); + var requestBody = Newtonsoft.Json.JsonConvert.SerializeObject(chatReq); + var chatResp = await PostJsonStreamAsync("/v1/chat/completions", requestBody); + return await chatResp.Content.ReadFromJsonAsync(); } /// @@ -65,7 +71,7 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI var responseObj = JToken.Parse(responseText); return responseObj?["data"]?["total_tokens"]?.ToObject(); } - var error = System.Text.Json.JsonSerializer.Deserialize(responseText); + var error = Newtonsoft.Json.JsonConvert.DeserializeObject(responseText); _logger.LogError($"{error?.error?.type}: {error?.error?.message}"); throw new Exception($"{error?.error.type}: {error?.error.message}"); } @@ -78,7 +84,7 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI /// public async Task GetAsTiMateTokenCount(ChatReq chatReq) { - var chatReqText =System.Text.Json.JsonSerializer.Serialize(chatReq); + var chatReqText = Newtonsoft.Json.JsonConvert.SerializeObject(chatReq); return await GetAsTiMateTokenCount(chatReqText); } @@ -177,7 +183,7 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI var uriBuilder = new UriBuilder(Host + path); request.RequestUri = uriBuilder.Uri; request.Method = new HttpMethod("POST"); - request.Headers.Host = new Uri(Host).Host; + request.Headers.Host = (new Uri(Host)).Host; return request; } @@ -191,9 +197,9 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI var responseText = await response.Content.ReadAsStringAsync(); if (response.IsSuccessStatusCode) { - return System.Text.Json.JsonSerializer.Deserialize(responseText) ?? default; + return Newtonsoft.Json.JsonConvert.DeserializeObject(responseText) ?? default; } - var error = System.Text.Json.JsonSerializer.Deserialize(responseText); + var error = Newtonsoft.Json.JsonConvert.DeserializeObject(responseText); _logger.LogError($"{error?.error.type}: {error?.error.message}"); throw new Exception($"{error?.error.type}: {error?.error.message}"); } @@ -241,4 +247,4 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI } } -} +} \ No newline at end of file diff --git a/VideoAnalysisCore/AICore/ChatGPT/KIMI/MoonshotModel.cs b/VideoAnalysisCore/AICore/ChatGPT/KIMI/MoonshotModel.cs index d045c7d..1e46c86 100644 --- a/VideoAnalysisCore/AICore/ChatGPT/KIMI/MoonshotModel.cs +++ b/VideoAnalysisCore/AICore/ChatGPT/KIMI/MoonshotModel.cs @@ -133,6 +133,56 @@ public string is_blocking { get; set; } } + + public class ChatRes + { + public string id { get; set; } + public int created { get; set; } + /// + /// 妯″瀷id + /// + public string model { get; set; } + /// + /// 瀵硅瘽 + /// + public Choice[] choices { get; set; } + /// + /// token浣跨敤鎯呭喌 + /// + public Usage usage { get; set; } + } + /// + /// token浣跨敤鎯呭喌 + /// + public class Usage + { + /// + /// 杈撳叆token鏁伴噺 + /// + public int prompt_tokens { get; set; } + /// + /// 杩斿洖token鏁伴噺 + /// + public int completion_tokens { get; set; } + /// + /// 鎬昏token鏁伴噺 + /// + public int total_tokens { get; set; } + } + + public class Choice + { + public int index { get; set; } + public Message message { get; set; } + public string finish_reason { get; set; } + } + + public class Message + { + public string role { get; set; } + public string content { get; set; } + } + public class ModelInfo { /// diff --git a/VideoAnalysisCore/AICore/SherpaOnnx/Speaker.cs b/VideoAnalysisCore/AICore/SherpaOnnx/Speaker.cs index 4be39f8..02b68ba 100644 --- a/VideoAnalysisCore/AICore/SherpaOnnx/Speaker.cs +++ b/VideoAnalysisCore/AICore/SherpaOnnx/Speaker.cs @@ -85,6 +85,14 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx /// 璁茶瘽浜虹储寮 /// public int SpeakerIndex { get; set; } + + /// + /// 璁茶瘽浜烘棩蹇楃粨鏋 + /// + /// + public OfflineSpeakerRes() + { + } /// /// 璁茶瘽浜烘棩蹇楃粨鏋 /// diff --git a/VideoAnalysisCore/Common/AppCommon.cs b/VideoAnalysisCore/Common/AppCommon.cs index d2978ee..4b468bb 100644 --- a/VideoAnalysisCore/Common/AppCommon.cs +++ b/VideoAnalysisCore/Common/AppCommon.cs @@ -172,8 +172,16 @@ namespace VideoAnalysisCore.Common //鎷兼帴 鎻愮ず璇嶅瓧骞曟簮 var stringBuilder = new StringBuilder(); foreach (var item in results) - stringBuilder.Append($"{item.Value.First()}:{item.Key.Start.TotalSeconds}:{item.Key.End.TotalSeconds}:{item.Key.Text},"); - + { + stringBuilder.Append(item.Value.First()); + stringBuilder.Append(":"); + stringBuilder.Append(item.Key.Start.TotalSeconds); + stringBuilder.Append(":"); + stringBuilder.Append(item.Key.End.TotalSeconds); + stringBuilder.Append(":"); + stringBuilder.Append(item.Key.Text); + stringBuilder.Append("|"); + } //todo 鍒嗘瀽涓婅鏃堕棿娈垫儏鍐 鍒嗘瀽 鐙珛瀛︿範 灏忕粍鍚堜綔 闅忓爞缁冧範绛夋儏鍐 return new TotalCaptionsDto diff --git a/VideoAnalysisCore/Common/RedisExpand.cs b/VideoAnalysisCore/Common/RedisExpand.cs index 0323e5f..d7e6a36 100644 --- a/VideoAnalysisCore/Common/RedisExpand.cs +++ b/VideoAnalysisCore/Common/RedisExpand.cs @@ -95,6 +95,8 @@ namespace VideoAnalysisCore.Common /// 浠诲姟id public static void InsertChannel(RedisChannelEnum @enum, object taskId) { + if (Redis is null) throw new Exception("redis鏈垵濮嬪寲"); + var startTime = Redis.HMGet>(RedisExpandKey.Task(taskId), "StartTime").FirstOrDefault(); if (startTime is null) startTime = new Dictionary(); @@ -102,9 +104,9 @@ namespace VideoAnalysisCore.Common startTime.Add(@enum, DateTime.Now); else startTime[@enum] = DateTime.Now; - Redis.HMSet(RedisExpandKey.Task(taskId), "StartTime", startTime); - if (Redis is null) throw new Exception("redis鏈垵濮嬪寲"); + + Redis.HMSet(RedisExpandKey.Task(taskId), "StartTime", startTime); Redis.LPush(RedisExpandKey.EnumKey(@enum), taskId); } diff --git a/VideoAnalysisCore/Model/VideoTask.cs b/VideoAnalysisCore/Model/VideoTask.cs index f208163..97bf0a4 100644 --- a/VideoAnalysisCore/Model/VideoTask.cs +++ b/VideoAnalysisCore/Model/VideoTask.cs @@ -1,6 +1,7 @@ 锘縰sing SqlSugar; using System.ComponentModel.DataAnnotations; using System.Net; +using System.Text.Json; using VideoAnalysisCore.AICore.SherpaOnnx; using VideoAnalysisCore.Enum; using Whisper.net; @@ -75,8 +76,25 @@ namespace VideoAnalysisCore.Model /// /// 寮濮嬫椂闂磋酱 /// - [SugarColumn(ColumnDataType = "varchar", Length =255)] - public Dictionary StartTime { get; set; } + [SugarColumn(ColumnDataType = "varchar", Length = 255)] + public string StartTime { get; set; } ="{}"; + /// + /// 寮濮嬫椂闂磋酱 + /// 閫昏緫瀛楁 + /// + [SugarColumn(IsIgnore = true)] + public Dictionary StartTimeDic + { + get + { + return JsonSerializer.Deserialize>(StartTime??"{}") + ??new Dictionary(); + } + set + { + StartTime = JsonSerializer.Serialize(value); + } + } }