diff --git a/VideoAnalysis/Controllers/ApiController.cs b/VideoAnalysis/Controllers/ApiController.cs index b2d464e..54a370c 100644 --- a/VideoAnalysis/Controllers/ApiController.cs +++ b/VideoAnalysis/Controllers/ApiController.cs @@ -6,6 +6,8 @@ using System.Reflection; using VideoAnalysisCore.Enum; using VideoAnalysisCore.Model; using VideoAnalysisCore.AICore.FFMPGE; +using VideoAnalysisCore.Model.Dto; +using VideoAnalysisCore.AICore.ChatGPT.Dto; namespace Learn.VideoAnalysis.Controllers { @@ -32,6 +34,20 @@ namespace Learn.VideoAnalysis.Controllers throw new Exception("未能获取到客户端ip地址"); } + /// + /// 插入队列 + /// + /// + /// + /// + [HttpGet(Name = "TaskInfo")] + public async Task TaskInfo(long taskId) + { + var task = await videoTaskDB.GetFirstAsync(s => s.Id == taskId); + if (task.LastEnum != RedisChannelEnum.EndTask) + return BadRequest(new { Enum = task.LastEnum ,Task = task.ChatAnalysis}); + return Ok(new { Enum = task.LastEnum, Task = task.ChatAnalysis }); + } /// /// 插入队列 diff --git a/VideoAnalysis/Program.cs b/VideoAnalysis/Program.cs index a4669c5..6b4d9c2 100644 --- a/VideoAnalysis/Program.cs +++ b/VideoAnalysis/Program.cs @@ -8,6 +8,9 @@ using VideoAnalysisCore.AICore.ChatGPT.KIMI; using VideoAnalysisCore.AICore.SherpaOnnx; using SqlSugar; using Mapster; +using VideoAnalysisCore.AICore.ChatGPT.Dto; +using System.Text.Json; +using Newtonsoft.Json; diff --git a/VideoAnalysisCore/AICore/ChatGPT/BserGPT.cs b/VideoAnalysisCore/AICore/ChatGPT/BserGPT.cs index 7fe6c27..9dcea2a 100644 --- a/VideoAnalysisCore/AICore/ChatGPT/BserGPT.cs +++ b/VideoAnalysisCore/AICore/ChatGPT/BserGPT.cs @@ -1,4 +1,5 @@ -锘縰sing VideoAnalysisCore.AICore.SherpaOnnx; +锘縰sing VideoAnalysisCore.AICore.ChatGPT.Dto; +using VideoAnalysisCore.AICore.SherpaOnnx; using VideoAnalysisCore.Common; using Whisper.net; @@ -14,6 +15,6 @@ namespace VideoAnalysisCore.AICore.ChatGPT /// /// 浠诲姟id /// - public Task CallGPT(string task); + public Task CallGPT(string task); } } diff --git a/VideoAnalysisCore/AICore/ChatGPT/Dto/CallGPTRes.cs b/VideoAnalysisCore/AICore/ChatGPT/Dto/CallGPTRes.cs index 6988f5f..0d09fa3 100644 --- a/VideoAnalysisCore/AICore/ChatGPT/Dto/CallGPTRes.cs +++ b/VideoAnalysisCore/AICore/ChatGPT/Dto/CallGPTRes.cs @@ -1,4 +1,5 @@ -锘縰sing System; +锘縰sing FreeRedis; +using System; using System.Collections.Generic; using System.Linq; using System.Text; @@ -8,12 +9,17 @@ using VideoAnalysisCore.Model.Dto; namespace VideoAnalysisCore.AICore.ChatGPT.Dto { - public class Assessment - { - - } public class CallGPTRes { + public CallGPTRes() + { + } + public CallGPTRes(TotalCaptionsDto captions) + { + this.TeacherSpeaking = captions.TeacherSpeaking; + this.TimeBase = captions.TimeBase; + this.StudentSpeaking = captions.StudentSpeaking; + } /// /// 鏁欏笀鍙戣█鏃堕棿 /// 绉 @@ -25,6 +31,19 @@ namespace VideoAnalysisCore.AICore.ChatGPT.Dto /// public decimal StudentSpeaking { get; set; } = 0; /// + /// 楂橀璇嶆眹 浠庨珮鍒颁綆 + /// + public string[] Hotwords { get; set; } + /// + /// 鏁欏笀鎻愰棶绫诲瀷 + /// + public KeyValue[]? QuestionType { get; set; } + + /// + /// 鏃堕棿娈垫瑙 + /// + public IEnumerable? TimeOverview { get; set; } + /// /// 瑙嗛鏃堕棿杞 /// public IEnumerable? TimeBase { get; set; } @@ -35,6 +54,6 @@ namespace VideoAnalysisCore.AICore.ChatGPT.Dto /// /// AI缁煎悎璇勪及 /// - public Assessment Assessment { get; set; } + public AssessmentDto Assessment { get; set; } = default!; } } diff --git a/VideoAnalysisCore/AICore/ChatGPT/Dto/QuestionRes.cs b/VideoAnalysisCore/AICore/ChatGPT/Dto/QuestionRes.cs index 372f6a5..3cc2613 100644 --- a/VideoAnalysisCore/AICore/ChatGPT/Dto/QuestionRes.cs +++ b/VideoAnalysisCore/AICore/ChatGPT/Dto/QuestionRes.cs @@ -1,7 +1,10 @@ -锘縰sing System; +锘縰sing Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System; using System.Collections.Generic; using System.Linq; using System.Text; +using System.Text.Json.Nodes; using System.Threading.Tasks; namespace VideoAnalysisCore.AICore.ChatGPT.Dto @@ -11,15 +14,54 @@ namespace VideoAnalysisCore.AICore.ChatGPT.Dto /// /// 瀵瑰簲闂鐨刬d /// - public int 闂缂栧彿 { get; set; } + public long 闂缂栧彿 { get; set; } /// /// 缁撴灉 /// - public object? 缁撴灉 { get; set; } + public JToken 缁撴灉 { get; set; } /// /// 闂瑙ｉ噴 /// public string? 闂瑙ｉ噴 { get; set; } } + /// + /// 璇惧爞AI璇勪环 + /// + public class CourseCriteria + { + /// + /// AI瑙ｆ瀽 + /// + public string Analyze { get; set; } = string.Empty; + /// + /// 闂缂栧彿 + /// + public long Id { get; set; } + /// + /// 鎻愯瘝 + /// + public string Prompt { get; set; } = string.Empty; + /// + /// 鏀硅繘鎰忚 + /// 閫傜敤浜:宸殑璇勪环 + /// + public string? ImprovedMethods { get; set; } + + } + /// + /// 璇惧爞AI璇勪环缁撴灉 + /// + public class AssessmentDto + { + /// + /// 浼樼鐨勮瘎浠 + /// + public CourseCriteria[]? Merit { get; set; } + /// + /// 宸殑璇勪环 + /// + public CourseCriteria[]? Bad { get; set; } + } + } diff --git a/VideoAnalysisCore/AICore/ChatGPT/KIMI/KIMI_GPT.cs b/VideoAnalysisCore/AICore/ChatGPT/KIMI/KIMI_GPT.cs index 88ec897..9d1d5f5 100644 --- a/VideoAnalysisCore/AICore/ChatGPT/KIMI/KIMI_GPT.cs +++ b/VideoAnalysisCore/AICore/ChatGPT/KIMI/KIMI_GPT.cs @@ -12,6 +12,8 @@ using VideoAnalysisCore.AICore.ChatGPT.Dto; using System.ComponentModel.DataAnnotations; using VideoAnalysisCore.Enum; using System.Reflection; +using FreeRedis; +using VideoAnalysisCore.Model.Dto; namespace VideoAnalysisCore.AICore.ChatGPT.KIMI { @@ -22,36 +24,37 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI { private readonly MoonshotClient moonshotClient; private readonly Repository criteriaDB; + private readonly Repository videoTaskDB; /// /// 鍒濆鍖 /// /// /// - public KIMI_GPT(MoonshotClient moonshotClient, Repository criteria) + public KIMI_GPT(MoonshotClient moonshotClient, Repository criteria, Repository videoTaskDB) { MoonshotClient.Host = AppCommon.Config.ChatGpt.KIMI.Host; MoonshotClient.ApiKey = AppCommon.Config.ChatGpt.KIMI.ApiKey; this.moonshotClient = moonshotClient; this.criteriaDB = criteria; + this.videoTaskDB = videoTaskDB; } /// /// 璁块棶GPT /// /// 浠诲姟id /// - public async Task CallGPT(string task) + public async Task CallGPT(string task) { var captions = ExpandFunction.GetSpeakerCaptions(task); var criteriaArr = await criteriaDB.GetListAsync(); - var criteriaBuilder = new StringBuilder(); foreach (var item in criteriaArr) { criteriaBuilder.Append(item.Id); criteriaBuilder.Append(":"); criteriaBuilder.Append(item.NamePrompt); - criteriaBuilder.Append("杩斿洖bool鍊紎"); + criteriaBuilder.Append("缁撴灉绫诲瀷 bool|"); } //鎷兼帴鏋氫妇鎻愰棶 foreach (var value in System.Enum.GetValues(typeof(QuestionTypeEnum))) @@ -68,7 +71,7 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI } var resFormat = "[{闂缂栧彿:int,缁撴灉:array|bool,闂瑙ｉ噴:string}]"; - var postMessages = + var postMessages = $"浠ヤ笅鏄竴娈甸煶棰戠殑瀛楀箷,鍒嗘瀽杩欐瀛楀箷(鏍煎紡 璇磋瘽浜:寮濮嬬:缁撴潫绉:鍐呭|涓嬩竴娈靛瓧骞)." + $"鏉ョ畝鏄庣殑鍥炵瓟鎻愬嚭鐨勯棶棰 闂鍒楄〃 {criteriaBuilder} " + $"瀛楀箷鍒楄〃 {captions.Captions} " + @@ -76,7 +79,7 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI var modelsResp = await moonshotClient.ListModels(); if (modelsResp is null || modelsResp.data.Count == 0) throw new Exception("鏈幏鍙朘IMI妯″瀷绫诲瀷"); - var reqTokenCount =await moonshotClient.GetAsTiMateTokenCount(postMessages); + var reqTokenCount = await moonshotClient.GetAsTiMateTokenCount(postMessages); var modelId = reqTokenCount > 32 * 1000 ? "moonshot-v1-128k" : "moonshot-v1-32k"; var chatRep = new ChatReq { @@ -91,16 +94,62 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI } }; var chatResp = await moonshotClient.Chat(chatRep); - if(chatResp is null || chatResp.error != null) + if (chatResp is null || chatResp.error != null) throw new Exception($"KIMI妯″瀷杩斿洖寮傚父 Chat 璇锋眰鍙傛暟: {JsonSerializer.Serialize(chatRep)} " + $" chatResp {JsonSerializer.Serialize(chatResp)}"); var chatResContent = chatResp?.choices.FirstOrDefault()?.message.content; + var questionRes = JsonSerializer.Deserialize(chatResContent); - //todo 鍒嗘瀽gpt杩斿洖缁撴灉 + var gptRes = new CallGPTRes(captions); + if (questionRes is null) + throw new Exception("KIMIGPT杩斿洖鏃犳晥缁撴灉"); + //澶勭悊 ai闂瓟鎻愰棶 + var arr1 = questionRes.Where(s => s.闂缂栧彿 < 100); + var arr2 = questionRes.Where(s => s.闂缂栧彿 >= 100).ToDictionary(s => s.闂缂栧彿); + //AI缁煎悎璇勪及 + var criteriaDic = criteriaArr.ToDictionary(s => s.Id); + gptRes.Assessment = new AssessmentDto() + { + Bad = arr1.Where(s => !s.缁撴灉.ToObject()) + .Select(s => new CourseCriteria() + { + Id = criteriaDic[s.闂缂栧彿].Id, + ImprovedMethods = criteriaDic[s.闂缂栧彿].ImprovedMethods, + Analyze = s.闂瑙ｉ噴??string.Empty, + Prompt = criteriaDic[s.闂缂栧彿].Flaw, + }).ToArray(), + Merit = arr1.Where(s => s.缁撴灉.ToObject()) + .Select(s => new CourseCriteria() + { + Id = criteriaDic[s.闂缂栧彿].Id, + ImprovedMethods = criteriaDic[s.闂缂栧彿].ImprovedMethods, + Prompt = criteriaDic[s.闂缂栧彿].ImprovedMethods, + }).ToArray(), + }; + + //楂橀璇嶆眹 + gptRes.Hotwords = arr2[(int)QuestionTypeEnum.楂橀璇峕.缁撴灉.ToObject()??["鏆傛棤鏁版嵁"]; + //鎻愰棶绫诲瀷 + gptRes.QuestionType = arr2[(int)QuestionTypeEnum.鎻愰棶绫诲瀷].缁撴灉.ToObject[]>(); + //鏃堕棿娈垫瑙 + gptRes.TimeOverview = arr2[(int)QuestionTypeEnum.鏃堕棿娈垫瑙圿 + .缁撴灉.ToObject[]>()?.Select(s => + new TimeBase + { + Start = double.Parse(s.key.Split(":")[1]??0.ToString()), + End = double.Parse(s.key.Split(":")[2]??0.ToString()), + Content = s.value, + }); //todo 鍒嗘瀽涓婅鏃堕棿娈垫儏鍐 鍒嗘瀽 鐙珛瀛︿範 灏忕粍鍚堜綔 闅忓爞缁冧範绛夋儏鍐 - + var tId = long.Parse(task); + var taskData = await videoTaskDB.GetFirstAsync(s => s.Id == tId); + taskData.ChatAnalysis = gptRes; + taskData.LastEnum = RedisChannelEnum.EndTask; + await videoTaskDB.AsUpdateable(taskData) + .UpdateColumns(it => new { it.ChatAnalysis }).ExecuteCommandAsync(); + return gptRes; } } } diff --git a/VideoAnalysisCore/AICore/SherpaOnnx/SenseVoice.cs b/VideoAnalysisCore/AICore/SherpaOnnx/SenseVoice.cs index 22e9a23..02c20cd 100644 --- a/VideoAnalysisCore/AICore/SherpaOnnx/SenseVoice.cs +++ b/VideoAnalysisCore/AICore/SherpaOnnx/SenseVoice.cs @@ -18,9 +18,9 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx /// /// 鍒濆鍖 SenseVoice /// - /// - /// - public static void Init(int speakerNumber = 0, double threshold = 0.6) + /// + /// 鏄惁浣跨敤gpu 鎶ラ敊璇风湅瀹夎CUDA鐜 + public static void Init(int numThreads =4,bool useGPU=false) { Console.WriteLine("鍒濆鍖 SenseVoice"); OfflineRecognizerConfig config = new OfflineRecognizerConfig(); @@ -34,8 +34,13 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx config.ModelConfig.SenseVoice.Model = Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-sense-voice-24-07-17", "model.onnx"); //1 浣跨敤閫嗘枃鏈鑼冨寲澶勭悊鎰熷畼璇煶銆 config.ModelConfig.SenseVoice.UseInverseTextNormalization =1; + config.ModelConfig.SenseVoice.Language = "zh"; //妯″瀷绫诲瀷 config.ModelConfig.ModelType = string.Empty; + config.ModelConfig.NumThreads = numThreads; + //闇瑕佷娇鐢℅PU + if (!useGPU) + config.ModelConfig.Provider = "cuda"; #region 鏈夋晥鐨勮В鐮佹柟娉 //璐┆鎼滅储[greedy_search] 鏀硅繘鐨勬尝鏉熸悳绱 [modified_beam_search] @@ -56,7 +61,7 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx //鍙嶈浆鏂囨湰瑙勮寖鍖栬鍒 fst 鐨勮矾寰 config.RuleFsts = string.Empty; - config.ModelConfig.Debug = 0; + config.ModelConfig.Debug = 1; OR = new OfflineRecognizer(config); diff --git a/VideoAnalysisCore/AICore/SherpaOnnx/Speaker.cs b/VideoAnalysisCore/AICore/SherpaOnnx/Speaker.cs index f39132e..d60e450 100644 --- a/VideoAnalysisCore/AICore/SherpaOnnx/Speaker.cs +++ b/VideoAnalysisCore/AICore/SherpaOnnx/Speaker.cs @@ -20,7 +20,8 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx /// /// /// - public static void Init(int speakerNumber = 0, double threshold = 0.6) + /// + public static void Init(int speakerNumber = 2, double threshold = 0.6,bool useGPU = false) { Console.WriteLine("鍒濆鍖 Speaker"); var config = new OfflineSpeakerDiarizationConfig(); @@ -30,6 +31,9 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx config.Embedding.Model = Path.Combine(AppCommon.AIModelFile, "wespeaker", "wespeaker_zh_cnceleb_resnet34_LM.onnx"); //璇磋瘽浜烘暟閲 config.Clustering.NumClusters = speakerNumber; + //闇瑕佷娇鐢℅PU + if (!useGPU) + config.Embedding.Provider = "cuda"; //璇磋瘽浜哄垽瀹氶槇鍊 config.Clustering.Threshold = (float)threshold; SD = new OfflineSpeakerDiarization(config); diff --git a/VideoAnalysisCore/Enum/QuestionTypeEnum.cs b/VideoAnalysisCore/Enum/QuestionTypeEnum.cs index 26b2f9e..7519d0b 100644 --- a/VideoAnalysisCore/Enum/QuestionTypeEnum.cs +++ b/VideoAnalysisCore/Enum/QuestionTypeEnum.cs @@ -10,11 +10,11 @@ namespace VideoAnalysisCore.Enum enum QuestionTypeEnum { - [Display(Prompt = "鍒嗘瀽鎺堣涓娇鐢ㄧ殑楂橀璇10涓")] + [Display(Prompt = "鍒嗘瀽鎺堣涓娇鐢ㄧ殑楂橀璇10涓鐜囦粠楂樺埌浣 缁撴灉绫诲瀷[]")] 楂橀璇 =100, - [Display(Prompt = "鍒嗘瀽瀛楀箷涓瘡5鍒嗛挓鐨勪竴涓瑙 杩斿洖缁撴瀯 array[0]=姒傝 鏃堕棿娈")] + [Display(Prompt = "鍒嗘瀽瀛楀箷涓瘡5鍒嗛挓鐨勪竴涓瑙 缁撴灉绫诲瀷[{key:鏃堕棿娈,value:姒傝}]")] 鏃堕棿娈垫瑙 = 101, - [Display(Prompt = "缁熻鎺堣涓暀甯堟彁闂被鍨嬬殑娆℃暟 绫诲瀷閲嶅鍥炵瓟,鑰佸笀杩介棶,绠鍗曟ц〃婕,鑰佸笀琛ュ厖绛旀,琛ㄦ壃骞惰ˉ鍏呯瓟妗")] + [Display(Prompt = "缁熻鎺堣涓暀甯堟彁闂被鍨嬬殑娆℃暟 绫诲瀷[閲嶅鍥炵瓟,鑰佸笀杩介棶,绠鍗曟ц〃婕,鑰佸笀琛ュ厖绛旀,琛ㄦ壃骞惰ˉ鍏呯瓟妗圿 缁撴灉绫诲瀷[{key:绫诲瀷,value:娆℃暟}]")] 鎻愰棶绫诲瀷 = 102, } } diff --git a/VideoAnalysisCore/Model/Dto/SpeakerCaptionsDto.cs b/VideoAnalysisCore/Model/Dto/SpeakerCaptionsDto.cs index 62bc431..58e52bd 100644 --- a/VideoAnalysisCore/Model/Dto/SpeakerCaptionsDto.cs +++ b/VideoAnalysisCore/Model/Dto/SpeakerCaptionsDto.cs @@ -20,13 +20,14 @@ namespace VideoAnalysisCore.Model.Dto /// public double End { get; set; } /// - /// 瀛楀箷鍐呭 + /// 鍐呭 /// public string Content { get; set; } /// /// 鏃堕棿娈 绫诲瀷 + /// 鏃朵负 null /// - public TimeBaseTypeEnum TimeBaseType { get; set; } + public TimeBaseTypeEnum? TimeBaseType { get; set; } } public class TotalCaptionsDto diff --git a/VideoAnalysisCore/Model/VideoTask.cs b/VideoAnalysisCore/Model/VideoTask.cs index f280a4d..1cc119f 100644 --- a/VideoAnalysisCore/Model/VideoTask.cs +++ b/VideoAnalysisCore/Model/VideoTask.cs @@ -2,6 +2,7 @@ using System.ComponentModel.DataAnnotations; using System.Net; using System.Text.Json; +using VideoAnalysisCore.AICore.ChatGPT.Dto; using VideoAnalysisCore.AICore.SherpaOnnx; using VideoAnalysisCore.AICore.Whisper; using VideoAnalysisCore.Enum; @@ -88,9 +89,9 @@ namespace VideoAnalysisCore.Model /// Chat妯″瀷鍒嗘瀽缂撳瓨 /// [SugarColumn(IsIgnore = true)] - public object[]? ChatAnalysis + public CallGPTRes? ChatAnalysis { - get => JsonSerializer.Deserialize(_ChatAnalysis ?? "[]"); + get => JsonSerializer.Deserialize(_ChatAnalysis ?? "[]"); set => _ChatAnalysis = JsonSerializer.Serialize(value); }