新增 AI优化字幕流程

优化 AI分析流程

VideoAnalysis/Components/Pages/VideoTaskShow.razor

@@ -21,6 +21,11 @@
         <source  type="video/mp4" />
     </video>
     <div id="subtitleArea" class="subtitles"></div>
+    
+    <div id="subtitleArea1" class="subtitles" style="
+    bottom: 101px;
+    background-color: rgb(99 129 103 / 50%);
+    "></div>
 
 </div>
 
@@ -28,16 +33,19 @@
 <script>
     window.b = []
     window.subtitles = []
+    window.subtitles1 = []
     var displayButton = []
     var lastSegments = null;
     function init() {
         const videoPlayer = document.getElementById('videoPlayer');
         const subtitleArea = document.getElementById('subtitleArea');
+        const subtitleArea1 = document.getElementById('subtitleArea1');
         //视频时间变化时
         videoPlayer.addEventListener('timeupdate', function () {
             if (displayButton.length == 0) initKD()
             const currentTime = videoPlayer.currentTime;
             subtitleArea.textContent = '';
+            subtitleArea1.textContent = '';
             //字幕
             window.subtitles.forEach(subtitle => {
                 let textContent = subtitle.text 
@@ -45,6 +53,7 @@
                     && currentTime <= subtitle.end
                     && subtitleArea.textContent != textContent) {
                     subtitleArea.textContent = textContent;
+                    subtitleArea1.textContent = window.subtitles1[window.subtitles.indexOf(subtitle)].text;
                 }
             });
             //时间片
@@ -62,9 +71,10 @@
         displayButton = window.b.map((s, i) => { return { ...s, button: btns[i] } })
     }
     //后端传递初始化数据
-    function setDB(a, b,c) {
+    function setDB(a,a1, b,c) {
-        console.log("setDB", a, b,c)
+        console.log("setDB", a1,a, b, c)
         window.subtitles = a
+        window.subtitles1 = a1
         window.b = b
         const videoPlayer = document.getElementById('videoPlayer');
         videoPlayer.src = c

VideoAnalysis/Components/Pages/VideoTaskShow.razor.cs

@@ -35,10 +35,6 @@ namespace Learn.VideoAnalysis.Components.Pages
         private VideoTask nowTask { get; set; } = default!;
         private string videoPath { get; set; } = default!;
 
-        /// <summary>
-        /// 字幕
-        /// </summary>
-        private SenseVoiceRes[] captionsArr { get; set; } = default!;
         /// <summary>
         /// 分段
         /// </summary>
@@ -76,7 +72,8 @@ namespace Learn.VideoAnalysis.Components.Pages
             nowTask = await taskDB.GetFirstAsync(s => s.Id == taskId);
             if (nowTask is null)
                 return;
-            captionsArr = JsonSerializer.Deserialize<SenseVoiceRes[]>(nowTask.Captions);
+            var captionsArr = JsonSerializer.Deserialize<SenseVoiceRes[]>(nowTask.Captions);
+            var captionsArr1 = JsonSerializer.Deserialize<SenseVoiceRes[]>(nowTask.CaptionsAI??"[]") ;
             RedisExpand.Redis.HMGet<SenseVoiceRes[]>(RedisExpandKey.Task(taskId), "Captions").FirstOrDefault();
 
             var konwArr = await videoKonwPointDB.AsQueryable()
@@ -94,7 +91,7 @@ namespace Learn.VideoAnalysis.Components.Pages
                     KnowPoint = string.Join(',', s.Select(x => x.KnowPoint))
                 }).ToArray();
             videoPath = AppCommon.GetVideoPath(nowTask.Id.ToString());
-            await JSRuntime.InvokeVoidAsync("setDB", captionsArr, videoKnows, videoPath);
+            await JSRuntime.InvokeVoidAsync("setDB", captionsArr, captionsArr1, videoKnows, videoPath);
             StateHasChanged();
         }
 

VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs

@@ -455,7 +455,8 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
                 .OrderBy(s => s.Id)
                 .GroupBy(s => s.Name)
                 .ToDictionary(s => s.First().Name, s => s.First().Id);
-            questionRes = questionRes.Where(s => s != null).OrderBy(s => s.StartTime).ToList();
+            questionRes = questionRes.Where(s => s != null)
+                .OrderBy(s => s.StartTime).ToList();
             var thems = JsonSerializer.Serialize(questionRes.Adapt<VideoKnowQueryDto[]>());// string.Join(',', questionRes.Select(s => s.StartTime + "->" + s.Theme));
             var checkResFormat1 = """[{"StartTime":开始秒(number),"KnowPoint":知识点名称(string),"KnowPointId":知识点Id(string)}]""";
             var knowMessages =
@@ -493,6 +494,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
                         KnowPointId = knowDic[x].ToString(),
                         TagId = taskInfo.TagId,
                         VideoTaskId = taskInfo.Id,
+                        Stage =s?.Stage?.ToEnum<StageEnum>()
                     });
                 }).ToList();
 
@@ -568,16 +570,18 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
         /// 优化字幕
         /// </summary>
         /// <returns></returns>
-        private async Task<List<SenseVoiceRes>> 优化字幕(VideoTask taskInfo,
+        private async Task<SenseVoiceRes[]> 优化字幕(VideoTask taskInfo,
             SenseVoiceRes[] captionsArr, string sections)
         {
+            if (!string.IsNullOrEmpty(taskInfo.CaptionsAI))
+                return JsonSerializer.Deserialize<SenseVoiceRes[]>(taskInfo.CaptionsAI);
             var subject = taskInfo.Subject.ToString();
             var newCaptionsList = new List<SenseVoiceRes>(captionsArr.Length);
             var spanCount = 50;
             var totalCount = captionsArr.Length / spanCount + 1;
 
             await Parallel.ForAsync(0, totalCount,
-                new ParallelOptions() { MaxDegreeOfParallelism = 4 },
+                new ParallelOptions() { MaxDegreeOfParallelism =10 },
                 async (s, c) =>
             {
                 while (true)
@@ -616,13 +620,14 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
                     return;
                 }
             });
+            var res = newCaptionsList.OrderBy(s => s.Start).ToArray();
             Console.WriteLine(DateTime.Now + $"=>字幕优化执行完成");
-            var jsonData = JsonSerializer.Serialize(newCaptionsList.OrderBy(s=>s.Start));
+            var jsonData = JsonSerializer.Serialize(res);
             await videoTaskDB.AsUpdateable()
                     .SetColumns(it => it.CaptionsAI == jsonData)
                     .Where(it => it.Id == taskInfo.Id)
                     .ExecuteCommandAsync();
-            return newCaptionsList;
+            return res;
         }
         /// <summary>
         /// 视频AI分析字幕
@@ -705,7 +710,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
             //处理视频授课章节
             var sections = await GetSections(taskInfo, Course_Id);
             //AI优化字幕 
-            await 优化字幕(taskInfo, captionsArr, sections);
+            captionsArr = await 优化字幕(taskInfo, captionsArr, sections);
             //合并字幕
             var captions = ExpandFunction.GetSpeakerCaptions(captionsArr);
             var maxVideoTime = captions?.TimeBase?.LastOrDefault()?.End ?? 0;
@@ -747,12 +752,8 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
             await RedisExpand.Redis
                .HMSetAsync(RedisExpandKey.Task(task), "VideoKnows", questionRes);
 
-            var gptRes = new TaskRes(captions);
-            await RedisExpand.Redis
-                .HMSetAsync(RedisExpandKey.Task(task), "ChatAnalysis", gptRes);
-
             RedisExpand.InsertChannel(RedisChannelEnum.EndTask, task);
-            return gptRes;
+            return null;
         }
 
 

VideoAnalysisCore/Common/AppCommon.cs

@@ -253,15 +253,15 @@ namespace VideoAnalysisCore.Common
         /// <summary>
         /// 转化枚举
         /// </summary>
-        /// <param name="data"></param>
+        /// <param name="value"></param>
         /// <returns></returns>
-        public static T? ToEnum<T>(this object data) where T : struct, System.Enum
+        public static T? ToEnum<T>(this object value) where T : struct, Enum
         {
             try
             {
-                if (data is null || string.IsNullOrEmpty(data?.ToString()))
+                if (Enum.TryParse<T>(value.ToString(), true, out var result) && Enum.IsDefined(typeof(T), result))
-                    return null;
+                    return result;
-                return System.Enum.Parse<T>(data.ToString());
+                return null;
             }
             catch (Exception)
             {

VideoAnalysisCore/Common/RedisExpand.cs

@@ -177,29 +177,29 @@ namespace VideoAnalysisCore.Common
         public static async Task TaskEnd(string task)
         {
             var tId = long.Parse(task);
-            var gptRes = (await Redis
+            //var gptRes = (await Redis
-               .HMGetAsync<TaskRes>(RedisExpandKey.Task(task), "ChatAnalysis")).FirstOrDefault();
+            //   .HMGetAsync<TaskRes>(RedisExpandKey.Task(task), "ChatAnalysis")).FirstOrDefault();
-            if (gptRes is null)
+            //if (gptRes is null)
-                throw new Exception("未能读取到GPT处理结果");
+            //    throw new Exception("未能读取到GPT处理结果");
 
             var taskData = await DbScoped.Sugar.Queryable<VideoTask>()
                 .FirstAsync(s => s.Id == tId);
             if (taskData.Captions == "[]")
                 taskData.Captions = (await Redis.HMGetAsync(RedisExpandKey.Task(task), "Captions")).First();
-            if (taskData.Speaker == "[]")
+            //if (taskData.Speaker == "[]")
-                taskData.Speaker = (await Redis.HMGetAsync(RedisExpandKey.Task(task), "Speaker"))?.FirstOrDefault()??"[]";
+            //    taskData.Speaker = (await Redis.HMGetAsync(RedisExpandKey.Task(task), "Speaker"))?.FirstOrDefault()??"[]";
 
 
-
+            //未使用结果暂时屏蔽
-            taskData.ChatAnalysis = JsonSerializer.Serialize(gptRes);
+            //taskData.ChatAnalysis = JsonSerializer.Serialize(gptRes);
-            taskData.ChatAnalysisScore = gptRes?.Assessment?.Merit?.Sum(s => s.Score) ?? 0;
+            taskData.ChatAnalysisScore =0;
             taskData.ErrorMessage = string.Empty;
             taskData.LastEnum = RedisChannelEnum.EndTask;
             taskData.EndTime = DateTime.Now;
             await DbScoped.Sugar.Updateable(taskData)
                 .UpdateColumns(it => new
                 {
-                    it.ChatAnalysis,
+                    //it.ChatAnalysis,
                     it.Captions,
                     it.Speaker,
                     it.ChatAnalysisScore,

VideoAnalysisCore/Model/VideoKonwPoint.cs

@@ -5,6 +5,7 @@ using System.Text.Json;
 using UserCenter.Model.Enum;
 using VideoAnalysisCore.AICore.GPT.Dto;
 using VideoAnalysisCore.AICore.SherpaOnnx;
+using VideoAnalysisCore.Model.Enum;
 using VideoAnalysisCore.Model.Interface;
 using Whisper.net;
 
@@ -63,5 +64,9 @@ namespace VideoAnalysisCore.Model
         /// 内容总结
         /// </summary>
         public string? Content { get; set; }
+        /// <summary>
+        /// 课程阶段
+        /// </summary>
+        public virtual StageEnum? Stage { get; set; }
     }
 }

VideoAnalysisCore/Model/VideoTask.cs

@@ -87,7 +87,7 @@ namespace VideoAnalysisCore.Model
         /// 字幕缓存[AI优化]
         /// </summary>
         [SugarColumn(ColumnName = "CaptionsAI", ColumnDataType = "longtext", IsNullable = true)]
-        public string CaptionsAI { get; set; } = "[]";
+        public string? CaptionsAI { get; set; } 
         /// <summary>
         /// 说话人日志解析缓存
         /// </summary>