新增 查询任务id接口

新增 ai任务使用gpu方法

VideoAnalysis/Controllers/ApiController.cs

@@ -6,6 +6,8 @@ using System.Reflection;
 using VideoAnalysisCore.Enum;
 using VideoAnalysisCore.Model;
 using VideoAnalysisCore.AICore.FFMPGE;
+using VideoAnalysisCore.Model.Dto;
+using VideoAnalysisCore.AICore.ChatGPT.Dto;
 
 namespace Learn.VideoAnalysis.Controllers
 {
@@ -32,6 +34,20 @@ namespace Learn.VideoAnalysis.Controllers
             throw new Exception("未能获取到客户端ip地址");
         }
 
+        /// <summary>
+        /// ²åÈë¶ÓÁÐ
+        /// </summary>
+        /// <param name="taskId"></param>
+        /// <param name="msg"></param>
+        /// <returns></returns>
+        [HttpGet(Name = "TaskInfo")]
+        public async Task<IActionResult> TaskInfo(long taskId)
+        {
+            var  task = await videoTaskDB.GetFirstAsync(s => s.Id == taskId);
+            if (task.LastEnum != RedisChannelEnum.EndTask)
+                return BadRequest(new { Enum = task.LastEnum ,Task = task.ChatAnalysis});
+            return Ok(new { Enum = task.LastEnum, Task = task.ChatAnalysis });
+        }
 
         /// <summary>
         /// 插入队列

VideoAnalysis/Program.cs

@@ -8,6 +8,9 @@ using VideoAnalysisCore.AICore.ChatGPT.KIMI;
 using VideoAnalysisCore.AICore.SherpaOnnx;
 using SqlSugar;
 using Mapster;
+using VideoAnalysisCore.AICore.ChatGPT.Dto;
+using System.Text.Json;
+using Newtonsoft.Json;
 
 
 

VideoAnalysisCore/AICore/ChatGPT/BserGPT.cs

@@ -1,4 +1,5 @@
-using VideoAnalysisCore.AICore.SherpaOnnx;
+using VideoAnalysisCore.AICore.ChatGPT.Dto;
+using VideoAnalysisCore.AICore.SherpaOnnx;
 using VideoAnalysisCore.Common;
 using Whisper.net;
 
@@ -14,6 +15,6 @@ namespace VideoAnalysisCore.AICore.ChatGPT
         /// </summary>
         /// <param name="task">任务id</param>
         /// <returns></returns>
-        public Task CallGPT(string task);
+        public  Task<CallGPTRes> CallGPT(string task);
     }
 }

VideoAnalysisCore/AICore/ChatGPT/Dto/CallGPTRes.cs

@@ -1,4 +1,5 @@
-using System;
+using FreeRedis;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
@@ -8,12 +9,17 @@ using VideoAnalysisCore.Model.Dto;
 
 namespace VideoAnalysisCore.AICore.ChatGPT.Dto
 {
-    public class Assessment 
-    {
-    
-    }
     public class CallGPTRes
     {
+        public CallGPTRes()
+        {
+        }
+        public CallGPTRes(TotalCaptionsDto captions)
+        {
+            this.TeacherSpeaking = captions.TeacherSpeaking;
+            this.TimeBase = captions.TimeBase;
+            this.StudentSpeaking = captions.StudentSpeaking;
+        }
         /// <summary>
         /// 教师发言时间
         /// <para>秒</para>
@@ -25,6 +31,19 @@ namespace VideoAnalysisCore.AICore.ChatGPT.Dto
         /// </summary>
         public decimal StudentSpeaking { get; set; } = 0;
         /// <summary>
+        /// 高频词汇 从高到低
+        /// </summary>
+        public string[] Hotwords { get; set; } 
+        /// <summary>
+        /// 教师提问类型
+        /// </summary>
+        public KeyValue<int>[]? QuestionType { get; set; }
+
+        /// <summary>
+        /// 时间段概览
+        /// </summary>
+        public IEnumerable<TimeBase>? TimeOverview { get; set; }
+        /// <summary>
         /// 视频时间轴
         /// </summary>
         public IEnumerable<TimeBase>? TimeBase { get; set; }
@@ -35,6 +54,6 @@ namespace VideoAnalysisCore.AICore.ChatGPT.Dto
         /// <summary>
         /// AI综合评估
         /// </summary>
-        public Assessment Assessment { get; set; }
+        public AssessmentDto Assessment { get; set; } = default!;
     }
 }

VideoAnalysisCore/AICore/ChatGPT/Dto/QuestionRes.cs

@@ -1,7 +1,10 @@
-using System;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
+using System.Text.Json.Nodes;
 using System.Threading.Tasks;
 
 namespace VideoAnalysisCore.AICore.ChatGPT.Dto
@@ -11,15 +14,54 @@ namespace VideoAnalysisCore.AICore.ChatGPT.Dto
         /// <summary>
         /// 对应问题的id
         /// </summary>
-        public int 问题编号 { get; set; }
+        public long 问题编号 { get; set; }
         /// <summary>
         /// 结果
         /// </summary>
-        public object? 结果 { get; set; }
+        public JToken 结果 { get; set; }
         /// <summary>
         /// 问题解释
         /// </summary>
         public string? 问题解释 { get; set; }
 
     }
+    /// <summary>
+    /// 课堂AI评价
+    /// </summary>
+    public class CourseCriteria
+    {
+        /// <summary>
+        /// AI解析
+        /// </summary>
+        public string Analyze { get; set; } = string.Empty;
+        /// <summary>
+        /// 问题编号
+        /// </summary>
+        public long Id { get; set; }
+        /// <summary>
+        /// 提词
+        /// </summary>         
+        public string Prompt { get; set; } = string.Empty;
+        /// <summary>
+        /// 改进意见
+        /// <para>适用于:差的评价<see cref="AssessmentDto.Bad"/></para>
+        /// </summary>           
+        public string? ImprovedMethods { get; set; }
+
+    }
+    /// <summary>
+    /// 课堂AI评价结果
+    /// </summary>
+    public class AssessmentDto
+    {
+        /// <summary>
+        /// 优秀的评价
+        /// </summary>
+        public CourseCriteria[]? Merit { get; set; }
+        /// <summary>
+        /// 差的评价
+        /// </summary>
+        public CourseCriteria[]? Bad { get; set; }
+    }
+
 }

VideoAnalysisCore/AICore/ChatGPT/KIMI/KIMI_GPT.cs

@@ -12,6 +12,8 @@ using VideoAnalysisCore.AICore.ChatGPT.Dto;
 using System.ComponentModel.DataAnnotations;
 using VideoAnalysisCore.Enum;
 using System.Reflection;
+using FreeRedis;
+using VideoAnalysisCore.Model.Dto;
 
 namespace VideoAnalysisCore.AICore.ChatGPT.KIMI
 {
@@ -22,36 +24,37 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI
     {
         private readonly MoonshotClient moonshotClient;
         private readonly Repository<CourseGradingCriteria> criteriaDB;
+        private readonly Repository<VideoTask> videoTaskDB;
         /// <summary>
         /// 初始化
         /// </summary>
         /// <param name="moonshotClient"></param>
         /// <param name="logger"></param>
-        public KIMI_GPT(MoonshotClient moonshotClient, Repository<CourseGradingCriteria> criteria)
+        public KIMI_GPT(MoonshotClient moonshotClient, Repository<CourseGradingCriteria> criteria, Repository<VideoTask> videoTaskDB)
         {
             MoonshotClient.Host = AppCommon.Config.ChatGpt.KIMI.Host;
             MoonshotClient.ApiKey = AppCommon.Config.ChatGpt.KIMI.ApiKey;
 
             this.moonshotClient = moonshotClient;
             this.criteriaDB = criteria;
+            this.videoTaskDB = videoTaskDB;
         }
         /// <summary>
         /// 访问GPT
         /// </summary>
         /// <param name="task">任务id</param>
         /// <returns></returns>
-        public async Task CallGPT(string task)
+        public async Task<CallGPTRes> CallGPT(string task)
         {
             var captions = ExpandFunction.GetSpeakerCaptions(task);
             var criteriaArr = await criteriaDB.GetListAsync();
-
             var criteriaBuilder = new StringBuilder();
             foreach (var item in criteriaArr)
             {
                 criteriaBuilder.Append(item.Id);
                 criteriaBuilder.Append(":");
                 criteriaBuilder.Append(item.NamePrompt);
-                criteriaBuilder.Append("返回bool值|");
+                criteriaBuilder.Append("结果类型 bool|");
             }
             //拼接枚举提问
             foreach (var value in System.Enum.GetValues(typeof(QuestionTypeEnum)))
@@ -68,7 +71,7 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI
             }
 
             var resFormat = "[{问题编号:int,结果:array|bool,问题解释:string}]";
-            var postMessages = 
+            var postMessages =
                 $"以下是一段音频的字幕,分析这段字幕(格式 说话人:开始秒:结束秒:内容|下一段字幕)." +
                 $"来简明的回答提出的问题 问题列表 {criteriaBuilder} " +
                 $"字幕列表 {captions.Captions} " +
@@ -76,7 +79,7 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI
             var modelsResp = await moonshotClient.ListModels();
             if (modelsResp is null || modelsResp.data.Count == 0)
                 throw new Exception("未获取KIMI模型类型");
-            var reqTokenCount =await moonshotClient.GetAsTiMateTokenCount(postMessages);
+            var reqTokenCount = await moonshotClient.GetAsTiMateTokenCount(postMessages);
             var modelId = reqTokenCount > 32 * 1000 ? "moonshot-v1-128k" : "moonshot-v1-32k";
             var chatRep = new ChatReq
             {
@@ -91,16 +94,62 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI
                 }
             };
             var chatResp = await moonshotClient.Chat(chatRep);
-           if(chatResp is null || chatResp.error != null)
+            if (chatResp is null || chatResp.error != null)
                 throw new Exception($"KIMI模型返回异常 Chat 请求参数: {JsonSerializer.Serialize(chatRep)} " +
                     $" chatResp {JsonSerializer.Serialize(chatResp)}");
             var chatResContent = chatResp?.choices.FirstOrDefault()?.message.content;
+
             var questionRes = JsonSerializer.Deserialize<QuestionRes[]>(chatResContent);
-            //todo 分析gpt返回结果
+            var gptRes = new CallGPTRes(captions);
+            if (questionRes is null)
+                throw new Exception("KIMIGPT返回无效结果");
+            //处理 ai问答提问
+            var arr1 = questionRes.Where(s => s.问题编号 < 100);
+            var arr2 = questionRes.Where(s => s.问题编号 >= 100).ToDictionary(s => s.问题编号);
+            //AI综合评估
+            var criteriaDic = criteriaArr.ToDictionary(s => s.Id);
+            gptRes.Assessment = new AssessmentDto()
+            {
+                Bad = arr1.Where(s => !s.结果.ToObject<bool>())
+                    .Select(s => new CourseCriteria()
+                    {
+                        Id = criteriaDic[s.问题编号].Id,
+                        ImprovedMethods = criteriaDic[s.问题编号].ImprovedMethods,
+                        Analyze = s.问题解释??string.Empty,
+                        Prompt = criteriaDic[s.问题编号].Flaw,
+                    }).ToArray(),
+                Merit = arr1.Where(s => s.结果.ToObject<bool>())
+                    .Select(s => new CourseCriteria()
+                    {
+                        Id = criteriaDic[s.问题编号].Id,
+                        ImprovedMethods = criteriaDic[s.问题编号].ImprovedMethods,
+                        Prompt = criteriaDic[s.问题编号].ImprovedMethods,
+                    }).ToArray(),
+            };
+
+            //高频词汇
+            gptRes.Hotwords = arr2[(int)QuestionTypeEnum.高频词].结果.ToObject<string[]>()??["暂无数据"];
+            //提问类型
+            gptRes.QuestionType = arr2[(int)QuestionTypeEnum.提问类型].结果.ToObject<KeyValue<int>[]>();
+            //时间段概览
+            gptRes.TimeOverview = arr2[(int)QuestionTypeEnum.时间段概览]
+                .结果.ToObject<KeyValue<string>[]>()?.Select(s => 
+                    new TimeBase
+                    {
+                        Start = double.Parse(s.key.Split(":")[1]??0.ToString()),
+                        End = double.Parse(s.key.Split(":")[2]??0.ToString()),
+                        Content = s.value,
+                    });
             //todo 分析上课时间段情况 分析 独立学习 小组合作 随堂练习等情况
 
 
-
+            var tId = long.Parse(task);
+            var  taskData = await videoTaskDB.GetFirstAsync(s => s.Id == tId);
+            taskData.ChatAnalysis = gptRes;
+            taskData.LastEnum = RedisChannelEnum.EndTask;
+            await videoTaskDB.AsUpdateable(taskData)
+                .UpdateColumns(it => new { it.ChatAnalysis }).ExecuteCommandAsync();
+            return gptRes;
         }
     }
 }

VideoAnalysisCore/AICore/SherpaOnnx/SenseVoice.cs

@@ -18,9 +18,9 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx
         /// <summary>
         /// 初始化 SenseVoice
         /// </summary>
-        /// <param name="speakerNumber"></param>
+        /// <param name="numThreads"></param>
-        /// <param name="threshold"></param>
+        /// <param name="useGPU">是否使用gpu 报错请看安装CUDA环境<see cref="https://k2-fsa.github.io/sherpa/onnx/pretrained_models/whisper/large-v3.html#run-with-gpu-float32"/></param>
-        public static void Init(int speakerNumber = 0, double threshold = 0.6)
+        public static void Init(int numThreads =4,bool useGPU=false)
         {
             Console.WriteLine("初始化 SenseVoice");
             OfflineRecognizerConfig config = new OfflineRecognizerConfig();
@@ -34,8 +34,13 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx
             config.ModelConfig.SenseVoice.Model = Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-sense-voice-24-07-17", "model.onnx");
             //1 使用逆文本规范化处理感官语音。
             config.ModelConfig.SenseVoice.UseInverseTextNormalization =1;
+            config.ModelConfig.SenseVoice.Language = "zh";
             //模型类型
             config.ModelConfig.ModelType = string.Empty;
+            config.ModelConfig.NumThreads = numThreads;
+            //需要使用GPU
+            if (!useGPU) 
+                config.ModelConfig.Provider = "cuda";
 
             #region 有效的解码方法
             //贪婪搜索[greedy_search] 改进的波束搜索 [modified_beam_search]
@@ -56,7 +61,7 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx
             //反转文本规范化规则 fst 的路径
             config.RuleFsts =  string.Empty;
 
-            config.ModelConfig.Debug = 0;
+            config.ModelConfig.Debug = 1;
 
             OR = new OfflineRecognizer(config);
 

VideoAnalysisCore/AICore/SherpaOnnx/Speaker.cs

@@ -20,7 +20,8 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx
         /// </summary>
         /// <param name="speakerNumber"></param>
         /// <param name="threshold"></param>
-        public static void Init(int speakerNumber = 0, double threshold = 0.6)
+        /// <param name="useGPU"></param>
+        public static void Init(int speakerNumber = 2, double threshold = 0.6,bool useGPU = false)
         {
             Console.WriteLine("初始化 Speaker");
             var config = new OfflineSpeakerDiarizationConfig();
@@ -30,6 +31,9 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx
             config.Embedding.Model = Path.Combine(AppCommon.AIModelFile, "wespeaker", "wespeaker_zh_cnceleb_resnet34_LM.onnx");
             //说话人数量
             config.Clustering.NumClusters = speakerNumber;
+            //需要使用GPU
+            if (!useGPU)
+                config.Embedding.Provider = "cuda";
             //说话人判定阈值
             config.Clustering.Threshold = (float)threshold;
             SD = new OfflineSpeakerDiarization(config);

VideoAnalysisCore/Enum/QuestionTypeEnum.cs

@@ -10,11 +10,11 @@ namespace VideoAnalysisCore.Enum
     enum QuestionTypeEnum
     {
 
-        [Display(Prompt = "分析授课中使用的高频词10个")]
+        [Display(Prompt = "分析授课中使用的高频词10个频率从高到低 结果类型[]")]
         高频词 =100,
-        [Display(Prompt = "分析字幕中每5分钟的一个概览 返回结构 array[0]=概览 时间段")]
+        [Display(Prompt = "分析字幕中每5分钟的一个概览 结果类型[{key:时间段,value:概览}]")]
         时间段概览 = 101,
-        [Display(Prompt = "统计授课中教师提问类型的次数 类型重复回答,老师追问,简单性表演,老师补充答案,表扬并补充答案")]
+        [Display(Prompt = "统计授课中教师提问类型的次数 类型[重复回答,老师追问,简单性表演,老师补充答案,表扬并补充答案] 结果类型[{key:类型,value:次数}]")]
         提问类型 = 102,
     }
 }

VideoAnalysisCore/Model/Dto/SpeakerCaptionsDto.cs

@@ -20,13 +20,14 @@ namespace VideoAnalysisCore.Model.Dto
         /// </summary>
         public double End { get; set; }
         /// <summary>
-        /// 字幕内容
+        /// 内容
         /// </summary>
         public string Content { get; set; }
         /// <summary>
         /// 时间段 类型
+        /// <para><see cref="VideoAnalysisCore.AICore.ChatGPT.Dto.CallGPTRes.TimeOverview"/> 时为 null</para>
         /// </summary>
-        public TimeBaseTypeEnum TimeBaseType { get; set; }
+        public TimeBaseTypeEnum? TimeBaseType { get; set; }
 
     }
     public class TotalCaptionsDto

VideoAnalysisCore/Model/VideoTask.cs

@@ -2,6 +2,7 @@
 using System.ComponentModel.DataAnnotations;
 using System.Net;
 using System.Text.Json;
+using VideoAnalysisCore.AICore.ChatGPT.Dto;
 using VideoAnalysisCore.AICore.SherpaOnnx;
 using VideoAnalysisCore.AICore.Whisper;
 using VideoAnalysisCore.Enum;
@@ -88,9 +89,9 @@ namespace VideoAnalysisCore.Model
         /// Chat模型分析缓存
         /// </summary>
         [SugarColumn(IsIgnore = true)]
-        public object[]? ChatAnalysis
+        public CallGPTRes? ChatAnalysis
         {
-            get => JsonSerializer.Deserialize<object[]>(_ChatAnalysis ?? "[]");
+            get => JsonSerializer.Deserialize<CallGPTRes>(_ChatAnalysis ?? "[]");
             set => _ChatAnalysis = JsonSerializer.Serialize(value);
         }