优化 计算帧差异算法

修复 复习课试题匹配错误情况

VideoAnalysisCore/AICore/FFMPGE/FFMPGEHandle.cs

@@ -3,7 +3,6 @@ using FFmpeg.NET;
 using VideoAnalysisCore.AICore.SherpaOnnx;
 using VideoAnalysisCore.Common;
 using System.Threading.Tasks;
-using static System.Runtime.InteropServices.JavaScript.JSType;
 using System.Xml.Linq;
 using System.Runtime.InteropServices;
 using SqlSugar.IOC;
@@ -41,6 +40,7 @@ namespace VideoAnalysisCore.AICore.FFMPGE
             //间隔秒
             var intervalSec = 5;
             var threshold = 8.15;
+            var ssimThreshold = 0.9;
             var PPTVideoCode = await DbScoped.Sugar
                 .Queryable<VideoTask>()
                 .Where(s => s.Id == long.Parse(task))
@@ -65,38 +65,37 @@ namespace VideoAnalysisCore.AICore.FFMPGE
             var frameFiles = Directory.GetFiles(localPath, "*.jpg")
               .OrderBy(f => f)
               .ToList();
-            
             RedisExpand.SetTaskProgress(task, "Frame=>50%");
             Image<Rgb24> prevFrame = null;
-            var keyFrames = new List<int>(5);
+            var keyFrames = new List<int>(10) { 5};
             foreach (var frameFile in frameFiles)
             {
                 using (var currFrame = Image.Load<Rgb24>(frameFile))
                 {
                     if (prevFrame != null)
                     {
-                        double diff = CalculateFrameDifference(prevFrame, currFrame);
+                        double ssim = SSIMCalculator.CalculateFrameSSIM(prevFrame, currFrame);
+                        //double diff = CalculateFrameDifference(prevFrame, currFrame);
                         double timestamp = GetTimestampFromFileName(frameFile) * intervalSec;
 
-                        if (diff > threshold)
+                        //if (diff > threshold)
+                        if (ssim < ssimThreshold)
                         {
                             keyFrames.Add((int)timestamp);
                             //string outputPath = Path.Combine(outputDir, $"change_{timestamp:0000}.jpg");
                             //currFrame.Save(outputPath);
-                            Console.WriteLine($"变化帧: {timestamp}秒，差异值: {diff:F2}");
+                            Console.WriteLine($"变化帧: {timestamp}秒，差异值: {ssim:F2}");
                         }
-                        //else
+                        //Console.WriteLine($"帧: {timestamp}秒，SSIM{ssim:F2} 差异值: {ssim:F2} ");
-                            //Console.WriteLine($"帧: {timestamp}秒，差异值: {diff:F2}");
 
                     }
                     prevFrame?.Dispose();
                     prevFrame = currFrame.Clone();
                 }
             } 
-            // 遍历数组
+            // 去掉相邻的重复图片
             for (int i = 1; i < keyFrames.Count(); i++)
             {
-                keyFrames[i] += 5;//ppt与课堂视频时间修正
                 if (keyFrames[i] - keyFrames[i - 1] < 10)
                     keyFrames[i] = -1;
             }
@@ -109,6 +108,7 @@ namespace VideoAnalysisCore.AICore.FFMPGE
                 .ExecuteCommandAsync();
 
         }
+
         /// <summary>
         /// 计算帧差异
         /// </summary>

VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeekClient.cs

@@ -49,6 +49,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
         {
             //chatReq.model = "deepseek-r1";
             if (chatReq.stream)  return await ChatSSE(chatReq);
+
             postStar:
             var requestBody = chatReq.ToJson();
             HttpResponseMessage chatResp = PostJsonStream(Path, requestBody);
@@ -57,7 +58,6 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
             {
                 Console.WriteLine(DateTime.Now + $"=>GPT请求失败重试 Code = {chatResp.StatusCode} Res={res1}");
                 goto postStar;
-            
             }
             //throw new Exception($" GPT模型返回异常 返回参数: " +
             //    $" {System.Text.Json.JsonSerializer.Serialize(res1)}");

VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs

@@ -402,6 +402,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
 
         
 
+
         /// <summary>
         /// 提取试题
         /// </summary>
@@ -423,9 +424,9 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
             foreach (var item in farmeArr)
             {
                 var knowInfoArr = videoKnowArr
-                    .Where(s => item + 20 >= s.StartTime && item <= s.EndTime)
+                    .Where(s => item + 20 >= s.StartTime && item < s.EndTime)
-                    .ToArray();
+                    .FirstOrDefault();
-                if (knowInfoArr is null || knowInfoArr.Count() == 0)
+                if (knowInfoArr is null)
                     continue;
                 var tryCount = 50;
                 while (tryCount > 1)
@@ -440,7 +441,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
                             continue;
                         if (sRes.Result.res.value.Trim().Length < 10)//总试题内容长度小于10 视为无效题目
                             break;
-                        Console.WriteLine(DateTime.Now + $"=>{taskInfo.Id} 提取{knowInfoArr.First().StartTime}秒试题的试题内容");
+                        Console.WriteLine(DateTime.Now + $"=>{taskInfo.Id} 提取{knowInfoArr.StartTime}秒试题的试题内容");
                         Console.WriteLine(sRes.Result.res.value);
                         //var knowArr=JsonSerializer.Serialize(knowInfoArr.Select(s => new { s.KnowPointId, s.KnowPoint }));
                         var resFormat = """[{"Type":string(试题类型),"TopicStem":string(试题题干),"QuestionArr":[{"Question":string(子问题),"KnowPointId":(string)知识点ID}]}]""";
@@ -470,7 +471,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
                                 vq.StartTime = item;
                                 vq.FilePath = filePath;
                                 vq.VideoTaskId = taskInfo.Id;
-                                vq.StageId = knowInfoArr.First().StageId;
+                                vq.StageId = knowInfoArr.StageId;
                                 vq.Question = qt.Question;
                                 vq.TopicId = TopicId;
                                 vq.Type = q.Type;
@@ -497,7 +498,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
                     }
                     catch (Exception ex)
                     {
-                        Console.WriteLine(DateTime.Now + $"=>{taskInfo.Id} 提取{knowInfoArr.First().StartTime}秒试题出现错误 {ex.Message}");
+                        Console.WriteLine(DateTime.Now + $"=>{taskInfo.Id} 提取{knowInfoArr.StartTime}秒试题出现错误 {ex.Message}");
                     }
                 }
             }
@@ -661,7 +662,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
             var postMessages =
                $"我将提供一段视频的字幕内容,请你帮我分析这堂课的课程类型是什么。" +
                $"授课类型限定在我提供的范围内 [{videoTypeStr}]" +
-               $"其中如果是[复习/习题课/试题讲解课程]那么课程类型视为'复习'" +
+               $"其中如果是[习题课/长篇幅的试题讲解课程]那么课程类型视为'复习'" +
                $"请简介的说明分析出课程类型的原因,如果分析出的课程类型与限定条件不匹配则返回NULL" +
                $"输出内容只返回json格式为({resFormat})" +
                $"以下是字幕内容" +

VideoAnalysisCore/Common/DownloadFile.cs

@@ -132,10 +132,6 @@ namespace VideoAnalysisCore.Common
             var fileUrl = taskInfo.MediaUrl;
             if (string.IsNullOrEmpty(fileUrl))
             {
-                switch (taskInfo.VideoType)
-                {
-                    case AttachmentsInfoType.新课:
-                    case AttachmentsInfoType.复习:
                 var videoInfo = await vodClient.GetPlayInfoAsync(new AlibabaCloud.SDK.Vod20170321.Models.GetPlayInfoRequest()
                 {
                     VideoId = taskInfo.TagId,
@@ -146,10 +142,6 @@ namespace VideoAnalysisCore.Common
                 if (videoInfo is null || videoInfo.StatusCode != 200 && !videoInfo.Body.PlayInfoList.PlayInfo.Any())
                     throw new Exception($"{DateTime.Now} 视频订阅=>获取阿里云视频信息失败 VideoCode {taskInfo.TagId} StatusCode {videoInfo?.StatusCode}");
                 fileUrl = videoInfo.Body.PlayInfoList.PlayInfo.First().PlayURL;
-                        break;
-                    default:
-                        break;
-                }
             }
             if (string.IsNullOrEmpty(fileUrl))
                 throw new Exception($"任务id[{task}] 资源地址无效 {fileUrl}");

VideoAnalysisCore/Common/RedisExpand.cs

@@ -174,7 +174,8 @@ namespace VideoAnalysisCore.Common
 
                     Redis.HMSet(RedisExpandKey.Task(taskId), "StartTime", startTime);
 
-                    await SubscribeList[@enum](tId);
+                   await TouchChannel(@enum, tId, SubscribeList[@enum]);
+                   //await SubscribeList[@enum](tId);
                     var e = @enum.NextEnum();
                     if (e is null)
                         break;
@@ -233,24 +234,18 @@ namespace VideoAnalysisCore.Common
         {
             if (Redis is null) throw new Exception("redis未初始化");
 
-            SubscribeList.Add(RedisChannelEnum.下载文件,
+            SubscribeList.Add(RedisChannelEnum.下载文件, (task) =>
-                async (msg) => await TouchChannel(RedisChannelEnum.下载文件, msg,
-                    (task) =>
             {
                 using var scope = AppCommon.Services?.CreateScope();
                 if (scope is null || scope.ServiceProvider.GetService<DownloadFile>() is null)
                     throw new Exception("DownloadFile 未注入");
                 else
                     return scope.ServiceProvider.GetService<DownloadFile>()?.RunTask(task) ?? Task.CompletedTask;
-                    }));
+            });
-            SubscribeList.Add(RedisChannelEnum.分离音频,
+            SubscribeList.Add(RedisChannelEnum.分离音频, FFMPGEHandle.RunAsync);
-                async (msg) => await TouchChannel(RedisChannelEnum.分离音频, msg, FFMPGEHandle.RunAsync));
+            SubscribeList.Add(RedisChannelEnum.解析字幕, SenseVoice.RunTask);
-            SubscribeList.Add(RedisChannelEnum.解析字幕,
+            //SubscribeList.Add(RedisChannelEnum.解析说话人,Speaker.Run);
-                async (msg) => await TouchChannel(RedisChannelEnum.解析字幕, msg, SenseVoice.RunTask));
-            //SubscribeList.Add(RedisChannelEnum.解析说话人,
-            //    async (msg) => await TouchChannel(RedisChannelEnum.解析说话人, msg, Speaker.Run));
             SubscribeList.Add(RedisChannelEnum.AI课程类型, 
-                async (msg) => await TouchChannel(RedisChannelEnum.AI课程类型, msg,
             (task) =>
             {
                 using var scope = AppCommon.Services?.CreateScope();
@@ -258,29 +253,24 @@ namespace VideoAnalysisCore.Common
                     throw new Exception("IBserGPT 未注入");
                 else
                     return scope.ServiceProvider.GetService<IBserGPT>()?.GetVideoType(task) ?? Task.CompletedTask;
-                    }));
+            });
-            SubscribeList.Add(RedisChannelEnum.AI模型分析,
+            SubscribeList.Add(RedisChannelEnum.AI模型分析, (task) =>
-                async (msg) => await TouchChannel(RedisChannelEnum.AI模型分析, msg,
-                    (task) =>
             {
                 using var scope = AppCommon.Services?.CreateScope();
                 if (scope is null || scope.ServiceProvider.GetService<IBserGPT>() is null)
                     throw new Exception("IBserGPT 未注入");
                 else
                     return scope.ServiceProvider.GetService<IBserGPT>()?.GetKnow(task) ?? Task.CompletedTask;
-                    }));
+            });
-            SubscribeList.Add(RedisChannelEnum.AI分析试题,
+            SubscribeList.Add(RedisChannelEnum.AI分析试题, (task) =>
-                async (msg) => await TouchChannel(RedisChannelEnum.AI分析试题, msg,
-                    (task) =>
             {
                 using var scope = AppCommon.Services?.CreateScope();
                 if (scope is null || scope.ServiceProvider.GetService<IBserGPT>() is null)
                     throw new Exception("IBserGPT 未注入");
                 else
                     return scope.ServiceProvider.GetService<IBserGPT>()?.GetVideoQuestion(task) ?? Task.CompletedTask;
-                    }));
+            });
-            SubscribeList.Add(RedisChannelEnum.结束任务,
+            SubscribeList.Add(RedisChannelEnum.结束任务, TaskEnd);
-                async (msg) => await TouchChannel(RedisChannelEnum.结束任务, msg, TaskEnd));
 
             ReceivingTaskAsync();
 

VideoAnalysisCore/Common/SSIMCalculator.cs

@@ -0,0 +1,97 @@
+using SixLabors.ImageSharp;
+using SixLabors.ImageSharp.PixelFormats;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using SixLabors.ImageSharp.Processing;
+
+namespace VideoAnalysisCore.Common
+{
+    /// <summary>
+    /// Ssim计算器
+    /// </summary>
+    public class SSIMCalculator
+    {
+        // SSIM计算常量 (基于8-bit图像范围0-255)
+        private const double C1 = (0.01 * 255) * (0.01 * 255);
+        private const double C2 = (0.03 * 255) * (0.03 * 255);
+
+        /// <summary>
+        /// 计算连续帧的SSIM 值
+        /// </summary>
+        /// <param name="img1"></param>
+        /// <param name="img2"></param>
+        /// <returns>返回阈值 0-1 越小变化越大<para>清晰视频：阈值 0.90-0.95 </para> <para>低质量视频：阈值 0.85-0.90</para></returns>
+        public static double CalculateFrameSSIM(Image<Rgb24> img1, Image<Rgb24> img2)
+        {
+            // 转换为灰度图
+            var gray1 = CreateResizedGrayImage(img1);
+            var gray2 = CreateResizedGrayImage(img2);
+
+            // 计算全局统计量
+            CalculateStats(gray1, gray2, out double mean1, out double mean2,
+                          out double var1, out double var2, out double covar);
+
+            // 计算SSIM分量
+            double luminance = (2 * mean1 * mean2 + C1) / (mean1 * mean1 + mean2 * mean2 + C1);
+            double contrast = (2 * Math.Sqrt(var1) * Math.Sqrt(var2) + C2) / (var1 + var2 + C2);
+            double structure = (covar + C2 / 2) / (Math.Sqrt(var1) * Math.Sqrt(var2) + C2 / 2);
+
+            // 返回SSIM值 (值越接近1表示越相似)
+            return luminance * contrast * structure;
+        }
+
+        private static Image<L8> CreateResizedGrayImage(Image<Rgb24> image)
+        {
+            return image
+                .Clone(x => x.Grayscale())
+                .CloneAs<L8>(); // 转换为8位灰度格式
+        }
+
+        private static void CalculateStats(
+            Image<L8> img1,
+            Image<L8> img2,
+            out double mean1,
+            out double mean2,
+            out double var1,
+            out double var2,
+            out double covar)
+        {
+            int width = img1.Width;
+            int height = img1.Height;
+            int totalPixels = width * height;
+
+            double sum1 = 0, sum2 = 0;
+            double sum1Sq = 0, sum2Sq = 0, sumProduct = 0;
+
+            // 单次遍历计算所有统计量
+            for (int y = 0; y < height; y++)
+            {
+                for (int x = 0; x < width; x++)
+                {
+                    double val1 = img1[x, y].PackedValue;
+                    double val2 = img2[x, y].PackedValue;
+
+                    sum1 += val1;
+                    sum2 += val2;
+                    sum1Sq += val1 * val1;
+                    sum2Sq += val2 * val2;
+                    sumProduct += val1 * val2;
+                }
+            }
+
+            // 计算均值
+            mean1 = sum1 / totalPixels;
+            mean2 = sum2 / totalPixels;
+
+            // 计算方差: Var(X) = E[X²] - E[X]²
+            var1 = (sum1Sq / totalPixels) - (mean1 * mean1);
+            var2 = (sum2Sq / totalPixels) - (mean2 * mean2);
+
+            // 计算协方差: Cov(X,Y) = E[XY] - E[X]E[Y]
+            covar = (sumProduct / totalPixels) - (mean1 * mean2);
+        }
+    }
+}

VideoAnalysisCore/Controllers/LJZK_Controller.cs

@@ -141,7 +141,6 @@ namespace VideoAnalysisCore.Controllers
         /// <summary>
         /// 获取视频知识点片段<para>taskId/tagId二选一</para>
         /// </summary>
-        /// <param name="taskId"></param>
         /// <param name="tagId">自定义id</param>
         /// <returns></returns>
         [HttpGet(Name = "TaskKnowInfo")]