优化计算帧差异算法

修复复习课试题匹配错误情况
2025-07-02 16:30:56 +08:00 · 2025-07-02 16:30:56 +08:00 · 0bd93c14bb
parent 3d112773c9
commit 0bd93c14bb
7 changed files with 170 additions and 91 deletions
--- a/VideoAnalysisCore/AICore/FFMPGE/FFMPGEHandle.cs
+++ b/VideoAnalysisCore/AICore/FFMPGE/FFMPGEHandle.cs
@ -3,7 +3,6 @@ using FFmpeg.NET;
 using VideoAnalysisCore.AICore.SherpaOnnx;
 using VideoAnalysisCore.Common;
 using System.Threading.Tasks;
-using static System.Runtime.InteropServices.JavaScript.JSType;
 using System.Xml.Linq;
 using System.Runtime.InteropServices;
 using SqlSugar.IOC;
@ -41,6 +40,7 @@ namespace VideoAnalysisCore.AICore.FFMPGE
            //间隔秒
            var intervalSec = 5;
            var threshold = 8.15;
+            var ssimThreshold = 0.9;
            var PPTVideoCode = await DbScoped.Sugar
                .Queryable<VideoTask>()
                .Where(s => s.Id == long.Parse(task))
@ -65,38 +65,37 @@ namespace VideoAnalysisCore.AICore.FFMPGE
            var frameFiles = Directory.GetFiles(localPath, "*.jpg")
              .OrderBy(f => f)
              .ToList();
-            
            RedisExpand.SetTaskProgress(task, "Frame=>50%");
            Image<Rgb24> prevFrame = null;
-            var keyFrames = new List<int>(5);
+            var keyFrames = new List<int>(10) { 5};
            foreach (var frameFile in frameFiles)
            {
                using (var currFrame = Image.Load<Rgb24>(frameFile))
                {
                    if (prevFrame != null)
                    {
-                        double diff = CalculateFrameDifference(prevFrame, currFrame);
+                        double ssim = SSIMCalculator.CalculateFrameSSIM(prevFrame, currFrame);
+                        //double diff = CalculateFrameDifference(prevFrame, currFrame);
                        double timestamp = GetTimestampFromFileName(frameFile) * intervalSec;

-                        if (diff > threshold)
+                        //if (diff > threshold)
+                        if (ssim < ssimThreshold)
                        {
                            keyFrames.Add((int)timestamp);
                            //string outputPath = Path.Combine(outputDir, $"change_{timestamp:0000}.jpg");
                            //currFrame.Save(outputPath);
-                            Console.WriteLine($"变化帧: {timestamp}秒，差异值: {diff:F2}");
+                            Console.WriteLine($"变化帧: {timestamp}秒，差异值: {ssim:F2}");
                        }
-                        //else
-                            //Console.WriteLine($"帧: {timestamp}秒，差异值: {diff:F2}");
+                        //Console.WriteLine($"帧: {timestamp}秒，SSIM{ssim:F2} 差异值: {ssim:F2} ");

                    }
                    prevFrame?.Dispose();
                    prevFrame = currFrame.Clone();
                }
            } 
-            // 遍历数组
+            // 去掉相邻的重复图片
            for (int i = 1; i < keyFrames.Count(); i++)
            {
-                keyFrames[i] += 5;//ppt与课堂视频时间修正
                if (keyFrames[i] - keyFrames[i - 1] < 10)
                    keyFrames[i] = -1;
            }
@ -109,6 +108,7 @@ namespace VideoAnalysisCore.AICore.FFMPGE
                .ExecuteCommandAsync();

        }
+
        /// <summary>
        /// 计算帧差异
        /// </summary>
--- a/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeekClient.cs
+++ b/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeekClient.cs
@ -49,6 +49,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
        {
            //chatReq.model = "deepseek-r1";
            if (chatReq.stream)  return await ChatSSE(chatReq);
+
            postStar:
            var requestBody = chatReq.ToJson();
            HttpResponseMessage chatResp = PostJsonStream(Path, requestBody);
@ -57,7 +58,6 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
            {
                Console.WriteLine(DateTime.Now + $"=>GPT请求失败重试 Code = {chatResp.StatusCode} Res={res1}");
                goto postStar;
-            
            }
            //throw new Exception($" GPT模型返回异常 返回参数: " +
            //    $" {System.Text.Json.JsonSerializer.Serialize(res1)}");
--- a/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs
+++ b/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs
@ -402,6 +402,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek

        

+
        /// <summary>
        /// 提取试题
        /// </summary>
@ -423,9 +424,9 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
            foreach (var item in farmeArr)
            {
                var knowInfoArr = videoKnowArr
-                    .Where(s => item + 20 >= s.StartTime && item <= s.EndTime)
-                    .ToArray();
-                if (knowInfoArr is null || knowInfoArr.Count() == 0)
+                    .Where(s => item + 20 >= s.StartTime && item < s.EndTime)
+                    .FirstOrDefault();
+                if (knowInfoArr is null)
                    continue;
                var tryCount = 50;
                while (tryCount > 1)
@ -440,7 +441,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
                            continue;
                        if (sRes.Result.res.value.Trim().Length < 10)//总试题内容长度小于10 视为无效题目
                            break;
-                        Console.WriteLine(DateTime.Now + $"=>{taskInfo.Id} 提取{knowInfoArr.First().StartTime}秒试题的试题内容");
+                        Console.WriteLine(DateTime.Now + $"=>{taskInfo.Id} 提取{knowInfoArr.StartTime}秒试题的试题内容");
                        Console.WriteLine(sRes.Result.res.value);
                        //var knowArr=JsonSerializer.Serialize(knowInfoArr.Select(s => new { s.KnowPointId, s.KnowPoint }));
                        var resFormat = """[{"Type":string(试题类型),"TopicStem":string(试题题干),"QuestionArr":[{"Question":string(子问题),"KnowPointId":(string)知识点ID}]}]""";
@ -470,7 +471,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
                                vq.StartTime = item;
                                vq.FilePath = filePath;
                                vq.VideoTaskId = taskInfo.Id;
-                                vq.StageId = knowInfoArr.First().StageId;
+                                vq.StageId = knowInfoArr.StageId;
                                vq.Question = qt.Question;
                                vq.TopicId = TopicId;
                                vq.Type = q.Type;
@ -497,7 +498,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
                    }
                    catch (Exception ex)
                    {
-                        Console.WriteLine(DateTime.Now + $"=>{taskInfo.Id} 提取{knowInfoArr.First().StartTime}秒试题出现错误 {ex.Message}");
+                        Console.WriteLine(DateTime.Now + $"=>{taskInfo.Id} 提取{knowInfoArr.StartTime}秒试题出现错误 {ex.Message}");
                    }
                }
            }
@ -661,7 +662,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
            var postMessages =
               $"我将提供一段视频的字幕内容,请你帮我分析这堂课的课程类型是什么。" +
               $"授课类型限定在我提供的范围内 [{videoTypeStr}]" +
-               $"其中如果是[复习/习题课/试题讲解课程]那么课程类型视为'复习'" +
+               $"其中如果是[习题课/长篇幅的试题讲解课程]那么课程类型视为'复习'" +
               $"请简介的说明分析出课程类型的原因,如果分析出的课程类型与限定条件不匹配则返回NULL" +
               $"输出内容只返回json格式为({resFormat})" +
               $"以下是字幕内容" +
--- a/VideoAnalysisCore/Common/DownloadFile.cs
+++ b/VideoAnalysisCore/Common/DownloadFile.cs
@ -132,10 +132,6 @@ namespace VideoAnalysisCore.Common
            var fileUrl = taskInfo.MediaUrl;
            if (string.IsNullOrEmpty(fileUrl))
            {
-                switch (taskInfo.VideoType)
-                {
-                    case AttachmentsInfoType.新课:
-                    case AttachmentsInfoType.复习:
                var videoInfo = await vodClient.GetPlayInfoAsync(new AlibabaCloud.SDK.Vod20170321.Models.GetPlayInfoRequest()
                {
                    VideoId = taskInfo.TagId,
@ -146,10 +142,6 @@ namespace VideoAnalysisCore.Common
                if (videoInfo is null || videoInfo.StatusCode != 200 && !videoInfo.Body.PlayInfoList.PlayInfo.Any())
                    throw new Exception($"{DateTime.Now} 视频订阅=>获取阿里云视频信息失败 VideoCode {taskInfo.TagId} StatusCode {videoInfo?.StatusCode}");
                fileUrl = videoInfo.Body.PlayInfoList.PlayInfo.First().PlayURL;
-                        break;
-                    default:
-                        break;
-                }
            }
            if (string.IsNullOrEmpty(fileUrl))
                throw new Exception($"任务id[{task}] 资源地址无效 {fileUrl}");
--- a/VideoAnalysisCore/Common/RedisExpand.cs
+++ b/VideoAnalysisCore/Common/RedisExpand.cs
@ -174,7 +174,8 @@ namespace VideoAnalysisCore.Common

                    Redis.HMSet(RedisExpandKey.Task(taskId), "StartTime", startTime);

-                    await SubscribeList[@enum](tId);
+                   await TouchChannel(@enum, tId, SubscribeList[@enum]);
+                   //await SubscribeList[@enum](tId);
                    var e = @enum.NextEnum();
                    if (e is null)
                        break;
@ -233,24 +234,18 @@ namespace VideoAnalysisCore.Common
        {
            if (Redis is null) throw new Exception("redis未初始化");

-            SubscribeList.Add(RedisChannelEnum.下载文件,
-                async (msg) => await TouchChannel(RedisChannelEnum.下载文件, msg,
-                    (task) =>
+            SubscribeList.Add(RedisChannelEnum.下载文件, (task) =>
            {
                using var scope = AppCommon.Services?.CreateScope();
                if (scope is null || scope.ServiceProvider.GetService<DownloadFile>() is null)
                    throw new Exception("DownloadFile 未注入");
                else
                    return scope.ServiceProvider.GetService<DownloadFile>()?.RunTask(task) ?? Task.CompletedTask;
-                    }));
-            SubscribeList.Add(RedisChannelEnum.分离音频,
-                async (msg) => await TouchChannel(RedisChannelEnum.分离音频, msg, FFMPGEHandle.RunAsync));
-            SubscribeList.Add(RedisChannelEnum.解析字幕,
-                async (msg) => await TouchChannel(RedisChannelEnum.解析字幕, msg, SenseVoice.RunTask));
-            //SubscribeList.Add(RedisChannelEnum.解析说话人,
-            //    async (msg) => await TouchChannel(RedisChannelEnum.解析说话人, msg, Speaker.Run));
+            });
+            SubscribeList.Add(RedisChannelEnum.分离音频, FFMPGEHandle.RunAsync);
+            SubscribeList.Add(RedisChannelEnum.解析字幕, SenseVoice.RunTask);
+            //SubscribeList.Add(RedisChannelEnum.解析说话人,Speaker.Run);
            SubscribeList.Add(RedisChannelEnum.AI课程类型, 
-                async (msg) => await TouchChannel(RedisChannelEnum.AI课程类型, msg,
            (task) =>
            {
                using var scope = AppCommon.Services?.CreateScope();
@ -258,29 +253,24 @@ namespace VideoAnalysisCore.Common
                    throw new Exception("IBserGPT 未注入");
                else
                    return scope.ServiceProvider.GetService<IBserGPT>()?.GetVideoType(task) ?? Task.CompletedTask;
-                    }));
-            SubscribeList.Add(RedisChannelEnum.AI模型分析,
-                async (msg) => await TouchChannel(RedisChannelEnum.AI模型分析, msg,
-                    (task) =>
+            });
+            SubscribeList.Add(RedisChannelEnum.AI模型分析, (task) =>
            {
                using var scope = AppCommon.Services?.CreateScope();
                if (scope is null || scope.ServiceProvider.GetService<IBserGPT>() is null)
                    throw new Exception("IBserGPT 未注入");
                else
                    return scope.ServiceProvider.GetService<IBserGPT>()?.GetKnow(task) ?? Task.CompletedTask;
-                    }));
-            SubscribeList.Add(RedisChannelEnum.AI分析试题,
-                async (msg) => await TouchChannel(RedisChannelEnum.AI分析试题, msg,
-                    (task) =>
+            });
+            SubscribeList.Add(RedisChannelEnum.AI分析试题, (task) =>
            {
                using var scope = AppCommon.Services?.CreateScope();
                if (scope is null || scope.ServiceProvider.GetService<IBserGPT>() is null)
                    throw new Exception("IBserGPT 未注入");
                else
                    return scope.ServiceProvider.GetService<IBserGPT>()?.GetVideoQuestion(task) ?? Task.CompletedTask;
-                    }));
-            SubscribeList.Add(RedisChannelEnum.结束任务,
-                async (msg) => await TouchChannel(RedisChannelEnum.结束任务, msg, TaskEnd));
+            });
+            SubscribeList.Add(RedisChannelEnum.结束任务, TaskEnd);

            ReceivingTaskAsync();

--- a/VideoAnalysisCore/Common/SSIMCalculator.cs
+++ b/VideoAnalysisCore/Common/SSIMCalculator.cs
@ -0,0 +1,97 @@
+using SixLabors.ImageSharp;
+using SixLabors.ImageSharp.PixelFormats;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using SixLabors.ImageSharp.Processing;
+
+namespace VideoAnalysisCore.Common
+{
+    /// <summary>
+    /// Ssim计算器
+    /// </summary>
+    public class SSIMCalculator
+    {
+        // SSIM计算常量 (基于8-bit图像范围0-255)
+        private const double C1 = (0.01 * 255) * (0.01 * 255);
+        private const double C2 = (0.03 * 255) * (0.03 * 255);
+
+        /// <summary>
+        /// 计算连续帧的SSIM 值
+        /// </summary>
+        /// <param name="img1"></param>
+        /// <param name="img2"></param>
+        /// <returns>返回阈值 0-1 越小变化越大<para>清晰视频：阈值 0.90-0.95 </para> <para>低质量视频：阈值 0.85-0.90</para></returns>
+        public static double CalculateFrameSSIM(Image<Rgb24> img1, Image<Rgb24> img2)
+        {
+            // 转换为灰度图
+            var gray1 = CreateResizedGrayImage(img1);
+            var gray2 = CreateResizedGrayImage(img2);
+
+            // 计算全局统计量
+            CalculateStats(gray1, gray2, out double mean1, out double mean2,
+                          out double var1, out double var2, out double covar);
+
+            // 计算SSIM分量
+            double luminance = (2 * mean1 * mean2 + C1) / (mean1 * mean1 + mean2 * mean2 + C1);
+            double contrast = (2 * Math.Sqrt(var1) * Math.Sqrt(var2) + C2) / (var1 + var2 + C2);
+            double structure = (covar + C2 / 2) / (Math.Sqrt(var1) * Math.Sqrt(var2) + C2 / 2);
+
+            // 返回SSIM值 (值越接近1表示越相似)
+            return luminance * contrast * structure;
+        }
+
+        private static Image<L8> CreateResizedGrayImage(Image<Rgb24> image)
+        {
+            return image
+                .Clone(x => x.Grayscale())
+                .CloneAs<L8>(); // 转换为8位灰度格式
+        }
+
+        private static void CalculateStats(
+            Image<L8> img1,
+            Image<L8> img2,
+            out double mean1,
+            out double mean2,
+            out double var1,
+            out double var2,
+            out double covar)
+        {
+            int width = img1.Width;
+            int height = img1.Height;
+            int totalPixels = width * height;
+
+            double sum1 = 0, sum2 = 0;
+            double sum1Sq = 0, sum2Sq = 0, sumProduct = 0;
+
+            // 单次遍历计算所有统计量
+            for (int y = 0; y < height; y++)
+            {
+                for (int x = 0; x < width; x++)
+                {
+                    double val1 = img1[x, y].PackedValue;
+                    double val2 = img2[x, y].PackedValue;
+
+                    sum1 += val1;
+                    sum2 += val2;
+                    sum1Sq += val1 * val1;
+                    sum2Sq += val2 * val2;
+                    sumProduct += val1 * val2;
+                }
+            }
+
+            // 计算均值
+            mean1 = sum1 / totalPixels;
+            mean2 = sum2 / totalPixels;
+
+            // 计算方差: Var(X) = E[X²] - E[X]²
+            var1 = (sum1Sq / totalPixels) - (mean1 * mean1);
+            var2 = (sum2Sq / totalPixels) - (mean2 * mean2);
+
+            // 计算协方差: Cov(X,Y) = E[XY] - E[X]E[Y]
+            covar = (sumProduct / totalPixels) - (mean1 * mean2);
+        }
+    }
+}
--- a/VideoAnalysisCore/Controllers/LJZK_Controller.cs
+++ b/VideoAnalysisCore/Controllers/LJZK_Controller.cs
@ -141,7 +141,6 @@ namespace VideoAnalysisCore.Controllers
        /// <summary>
        /// 获取视频知识点片段<para>taskId/tagId二选一</para>
        /// </summary>
-        /// <param name="taskId"></param>
        /// <param name="tagId">自定义id</param>
        /// <returns></returns>
        [HttpGet(Name = "TaskKnowInfo")]