From 0bd93c14bb4b4d418e02a18a3a79e58a288a7fda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E8=82=A5=E7=BE=8A?= <1048382248@qq.com> Date: Wed, 2 Jul 2025 16:30:56 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=20=E8=AE=A1=E7=AE=97?= =?UTF-8?q?=E5=B8=A7=E5=B7=AE=E5=BC=82=E7=AE=97=E6=B3=95=20=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=20=E5=A4=8D=E4=B9=A0=E8=AF=BE=E8=AF=95=E9=A2=98?= =?UTF-8?q?=E5=8C=B9=E9=85=8D=E9=94=99=E8=AF=AF=E6=83=85=E5=86=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../AICore/FFMPGE/FFMPGEHandle.cs | 32 +++--- .../AICore/GPT/DeepSeek/DeepSeekClient.cs | 2 +- .../AICore/GPT/DeepSeek/DeepSeek_GPT.cs | 15 +-- VideoAnalysisCore/Common/DownloadFile.cs | 26 ++--- VideoAnalysisCore/Common/RedisExpand.cs | 88 ++++++++--------- VideoAnalysisCore/Common/SSIMCalculator.cs | 97 +++++++++++++++++++ .../Controllers/LJZK_Controller.cs | 1 - 7 files changed, 170 insertions(+), 91 deletions(-) create mode 100644 VideoAnalysisCore/Common/SSIMCalculator.cs diff --git a/VideoAnalysisCore/AICore/FFMPGE/FFMPGEHandle.cs b/VideoAnalysisCore/AICore/FFMPGE/FFMPGEHandle.cs index c907e64..617076d 100644 --- a/VideoAnalysisCore/AICore/FFMPGE/FFMPGEHandle.cs +++ b/VideoAnalysisCore/AICore/FFMPGE/FFMPGEHandle.cs @@ -3,7 +3,6 @@ using FFmpeg.NET; using VideoAnalysisCore.AICore.SherpaOnnx; using VideoAnalysisCore.Common; using System.Threading.Tasks; -using static System.Runtime.InteropServices.JavaScript.JSType; using System.Xml.Linq; using System.Runtime.InteropServices; using SqlSugar.IOC; @@ -41,6 +40,7 @@ namespace VideoAnalysisCore.AICore.FFMPGE //间隔秒 var intervalSec = 5; var threshold = 8.15; + var ssimThreshold = 0.9; var PPTVideoCode = await DbScoped.Sugar .Queryable() .Where(s => s.Id == long.Parse(task)) @@ -65,38 +65,37 @@ namespace VideoAnalysisCore.AICore.FFMPGE var frameFiles = Directory.GetFiles(localPath, "*.jpg") .OrderBy(f => f) .ToList(); - RedisExpand.SetTaskProgress(task, "Frame=>50%"); Image prevFrame = null; - var keyFrames = new List(5); + var keyFrames = new List(10) { 5}; foreach (var frameFile in frameFiles) { using (var currFrame = Image.Load(frameFile)) { if (prevFrame != null) { - double diff = CalculateFrameDifference(prevFrame, currFrame); + double ssim = SSIMCalculator.CalculateFrameSSIM(prevFrame, currFrame); + //double diff = CalculateFrameDifference(prevFrame, currFrame); double timestamp = GetTimestampFromFileName(frameFile) * intervalSec; - if (diff > threshold) + //if (diff > threshold) + if (ssim < ssimThreshold) { keyFrames.Add((int)timestamp); //string outputPath = Path.Combine(outputDir, $"change_{timestamp:0000}.jpg"); //currFrame.Save(outputPath); - Console.WriteLine($"变化帧: {timestamp}秒，差异值: {diff:F2}"); + Console.WriteLine($"变化帧: {timestamp}秒，差异值: {ssim:F2}"); } - //else - //Console.WriteLine($"帧: {timestamp}秒，差异值: {diff:F2}"); + //Console.WriteLine($"帧: {timestamp}秒，SSIM{ssim:F2} 差异值: {ssim:F2} "); } prevFrame?.Dispose(); prevFrame = currFrame.Clone(); } } - // 遍历数组 + // 去掉相邻的重复图片 for (int i = 1; i < keyFrames.Count(); i++) { - keyFrames[i] += 5;//ppt与课堂视频时间修正 if (keyFrames[i] - keyFrames[i - 1] < 10) keyFrames[i] = -1; } @@ -108,13 +107,14 @@ namespace VideoAnalysisCore.AICore.FFMPGE .Where(it => it.Id == taskID) .ExecuteCommandAsync(); - } + } + /// - /// 计算帧差异 - /// - /// - /// - /// + /// 计算帧差异 + /// + /// + /// + /// static double CalculateFrameDifference(Image img1, Image img2) { // 统一调整为64x64 diff --git a/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeekClient.cs b/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeekClient.cs index cb345f6..dd7b7a8 100644 --- a/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeekClient.cs +++ b/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeekClient.cs @@ -49,6 +49,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek { //chatReq.model = "deepseek-r1"; if (chatReq.stream) return await ChatSSE(chatReq); + postStar: var requestBody = chatReq.ToJson(); HttpResponseMessage chatResp = PostJsonStream(Path, requestBody); @@ -57,7 +58,6 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek { Console.WriteLine(DateTime.Now + $"=>GPT请求失败重试 Code = {chatResp.StatusCode} Res={res1}"); goto postStar; - } //throw new Exception($" GPT模型返回异常 返回参数: " + // $" {System.Text.Json.JsonSerializer.Serialize(res1)}"); diff --git a/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs b/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs index 59391e2..2146eb0 100644 --- a/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs +++ b/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs @@ -400,6 +400,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek throw new Exception(DateTime.Now + "=>ChatGPT请求失败次数过多!!!"); } + /// @@ -423,9 +424,9 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek foreach (var item in farmeArr) { var knowInfoArr = videoKnowArr - .Where(s => item + 20 >= s.StartTime && item <= s.EndTime) - .ToArray(); - if (knowInfoArr is null || knowInfoArr.Count() == 0) + .Where(s => item + 20 >= s.StartTime && item < s.EndTime) + .FirstOrDefault(); + if (knowInfoArr is null) continue; var tryCount = 50; while (tryCount > 1) @@ -440,7 +441,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek continue; if (sRes.Result.res.value.Trim().Length < 10)//总试题内容长度小于10 视为无效题目 break; - Console.WriteLine(DateTime.Now + $"=>{taskInfo.Id} 提取{knowInfoArr.First().StartTime}秒试题的试题内容"); + Console.WriteLine(DateTime.Now + $"=>{taskInfo.Id} 提取{knowInfoArr.StartTime}秒试题的试题内容"); Console.WriteLine(sRes.Result.res.value); //var knowArr=JsonSerializer.Serialize(knowInfoArr.Select(s => new { s.KnowPointId, s.KnowPoint })); var resFormat = """[{"Type":string(试题类型),"TopicStem":string(试题题干),"QuestionArr":[{"Question":string(子问题),"KnowPointId":(string)知识点ID}]}]"""; @@ -470,7 +471,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek vq.StartTime = item; vq.FilePath = filePath; vq.VideoTaskId = taskInfo.Id; - vq.StageId = knowInfoArr.First().StageId; + vq.StageId = knowInfoArr.StageId; vq.Question = qt.Question; vq.TopicId = TopicId; vq.Type = q.Type; @@ -497,7 +498,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek } catch (Exception ex) { - Console.WriteLine(DateTime.Now + $"=>{taskInfo.Id} 提取{knowInfoArr.First().StartTime}秒试题出现错误 {ex.Message}"); + Console.WriteLine(DateTime.Now + $"=>{taskInfo.Id} 提取{knowInfoArr.StartTime}秒试题出现错误 {ex.Message}"); } } } @@ -661,7 +662,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek var postMessages = $"我将提供一段视频的字幕内容,请你帮我分析这堂课的课程类型是什么。" + $"授课类型限定在我提供的范围内 [{videoTypeStr}]" + - $"其中如果是[复习/习题课/试题讲解课程]那么课程类型视为'复习'" + + $"其中如果是[习题课/长篇幅的试题讲解课程]那么课程类型视为'复习'" + $"请简介的说明分析出课程类型的原因,如果分析出的课程类型与限定条件不匹配则返回NULL" + $"输出内容只返回json格式为({resFormat})" + $"以下是字幕内容" + diff --git a/VideoAnalysisCore/Common/DownloadFile.cs b/VideoAnalysisCore/Common/DownloadFile.cs index b9a629b..f79a042 100644 --- a/VideoAnalysisCore/Common/DownloadFile.cs +++ b/VideoAnalysisCore/Common/DownloadFile.cs @@ -132,24 +132,16 @@ namespace VideoAnalysisCore.Common var fileUrl = taskInfo.MediaUrl; if (string.IsNullOrEmpty(fileUrl)) { - switch (taskInfo.VideoType) + var videoInfo = await vodClient.GetPlayInfoAsync(new AlibabaCloud.SDK.Vod20170321.Models.GetPlayInfoRequest() { - case AttachmentsInfoType.新课: - case AttachmentsInfoType.复习: - var videoInfo = await vodClient.GetPlayInfoAsync(new AlibabaCloud.SDK.Vod20170321.Models.GetPlayInfoRequest() - { - VideoId = taskInfo.TagId, - Formats = "mp4", - OutputType = "cdn", - AuthTimeout = 3600 * 24 * 12, - }); - if (videoInfo is null || videoInfo.StatusCode != 200 && !videoInfo.Body.PlayInfoList.PlayInfo.Any()) - throw new Exception($"{DateTime.Now} 视频订阅=>获取阿里云视频信息失败 VideoCode {taskInfo.TagId} StatusCode {videoInfo?.StatusCode}"); - fileUrl = videoInfo.Body.PlayInfoList.PlayInfo.First().PlayURL; - break; - default: - break; - } + VideoId = taskInfo.TagId, + Formats = "mp4", + OutputType = "cdn", + AuthTimeout = 3600 * 24 * 12, + }); + if (videoInfo is null || videoInfo.StatusCode != 200 && !videoInfo.Body.PlayInfoList.PlayInfo.Any()) + throw new Exception($"{DateTime.Now} 视频订阅=>获取阿里云视频信息失败 VideoCode {taskInfo.TagId} StatusCode {videoInfo?.StatusCode}"); + fileUrl = videoInfo.Body.PlayInfoList.PlayInfo.First().PlayURL; } if (string.IsNullOrEmpty(fileUrl)) throw new Exception($"任务id[{task}] 资源地址无效 {fileUrl}"); diff --git a/VideoAnalysisCore/Common/RedisExpand.cs b/VideoAnalysisCore/Common/RedisExpand.cs index e1b18c9..630aaaa 100644 --- a/VideoAnalysisCore/Common/RedisExpand.cs +++ b/VideoAnalysisCore/Common/RedisExpand.cs @@ -174,7 +174,8 @@ namespace VideoAnalysisCore.Common Redis.HMSet(RedisExpandKey.Task(taskId), "StartTime", startTime); - await SubscribeList[@enum](tId); + await TouchChannel(@enum, tId, SubscribeList[@enum]); + //await SubscribeList[@enum](tId); var e = @enum.NextEnum(); if (e is null) break; @@ -233,54 +234,43 @@ namespace VideoAnalysisCore.Common { if (Redis is null) throw new Exception("redis未初始化"); - SubscribeList.Add(RedisChannelEnum.下载文件, - async (msg) => await TouchChannel(RedisChannelEnum.下载文件, msg, - (task) => - { - using var scope = AppCommon.Services?.CreateScope(); - if (scope is null || scope.ServiceProvider.GetService() is null) - throw new Exception("DownloadFile 未注入"); - else - return scope.ServiceProvider.GetService()?.RunTask(task) ?? Task.CompletedTask; - })); - SubscribeList.Add(RedisChannelEnum.分离音频, - async (msg) => await TouchChannel(RedisChannelEnum.分离音频, msg, FFMPGEHandle.RunAsync)); - SubscribeList.Add(RedisChannelEnum.解析字幕, - async (msg) => await TouchChannel(RedisChannelEnum.解析字幕, msg, SenseVoice.RunTask)); - //SubscribeList.Add(RedisChannelEnum.解析说话人, - // async (msg) => await TouchChannel(RedisChannelEnum.解析说话人, msg, Speaker.Run)); - SubscribeList.Add(RedisChannelEnum.AI课程类型, - async (msg) => await TouchChannel(RedisChannelEnum.AI课程类型, msg, - (task) => - { - using var scope = AppCommon.Services?.CreateScope(); - if (scope is null || scope.ServiceProvider.GetService() is null) - throw new Exception("IBserGPT 未注入"); - else - return scope.ServiceProvider.GetService()?.GetVideoType(task) ?? Task.CompletedTask; - })); - SubscribeList.Add(RedisChannelEnum.AI模型分析, - async (msg) => await TouchChannel(RedisChannelEnum.AI模型分析, msg, - (task) => - { - using var scope = AppCommon.Services?.CreateScope(); - if (scope is null || scope.ServiceProvider.GetService() is null) - throw new Exception("IBserGPT 未注入"); - else - return scope.ServiceProvider.GetService()?.GetKnow(task) ?? Task.CompletedTask; - })); - SubscribeList.Add(RedisChannelEnum.AI分析试题, - async (msg) => await TouchChannel(RedisChannelEnum.AI分析试题, msg, - (task) => - { - using var scope = AppCommon.Services?.CreateScope(); - if (scope is null || scope.ServiceProvider.GetService() is null) - throw new Exception("IBserGPT 未注入"); - else - return scope.ServiceProvider.GetService()?.GetVideoQuestion(task) ?? Task.CompletedTask; - })); - SubscribeList.Add(RedisChannelEnum.结束任务, - async (msg) => await TouchChannel(RedisChannelEnum.结束任务, msg, TaskEnd)); + SubscribeList.Add(RedisChannelEnum.下载文件, (task) => + { + using var scope = AppCommon.Services?.CreateScope(); + if (scope is null || scope.ServiceProvider.GetService() is null) + throw new Exception("DownloadFile 未注入"); + else + return scope.ServiceProvider.GetService()?.RunTask(task) ?? Task.CompletedTask; + }); + SubscribeList.Add(RedisChannelEnum.分离音频, FFMPGEHandle.RunAsync); + SubscribeList.Add(RedisChannelEnum.解析字幕, SenseVoice.RunTask); + //SubscribeList.Add(RedisChannelEnum.解析说话人,Speaker.Run); + SubscribeList.Add(RedisChannelEnum.AI课程类型, + (task) => + { + using var scope = AppCommon.Services?.CreateScope(); + if (scope is null || scope.ServiceProvider.GetService() is null) + throw new Exception("IBserGPT 未注入"); + else + return scope.ServiceProvider.GetService()?.GetVideoType(task) ?? Task.CompletedTask; + }); + SubscribeList.Add(RedisChannelEnum.AI模型分析, (task) => + { + using var scope = AppCommon.Services?.CreateScope(); + if (scope is null || scope.ServiceProvider.GetService() is null) + throw new Exception("IBserGPT 未注入"); + else + return scope.ServiceProvider.GetService()?.GetKnow(task) ?? Task.CompletedTask; + }); + SubscribeList.Add(RedisChannelEnum.AI分析试题, (task) => + { + using var scope = AppCommon.Services?.CreateScope(); + if (scope is null || scope.ServiceProvider.GetService() is null) + throw new Exception("IBserGPT 未注入"); + else + return scope.ServiceProvider.GetService()?.GetVideoQuestion(task) ?? Task.CompletedTask; + }); + SubscribeList.Add(RedisChannelEnum.结束任务, TaskEnd); ReceivingTaskAsync(); diff --git a/VideoAnalysisCore/Common/SSIMCalculator.cs b/VideoAnalysisCore/Common/SSIMCalculator.cs new file mode 100644 index 0000000..db099c9 --- /dev/null +++ b/VideoAnalysisCore/Common/SSIMCalculator.cs @@ -0,0 +1,97 @@ +using SixLabors.ImageSharp; +using SixLabors.ImageSharp.PixelFormats; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using SixLabors.ImageSharp.Processing; + +namespace VideoAnalysisCore.Common +{ + /// + /// Ssim计算器 + /// + public class SSIMCalculator + { + // SSIM计算常量 (基于8-bit图像范围0-255) + private const double C1 = (0.01 * 255) * (0.01 * 255); + private const double C2 = (0.03 * 255) * (0.03 * 255); + + /// + /// 计算连续帧的SSIM 值 + /// + /// + /// + /// 返回阈值 0-1 越小变化越大清晰视频：阈值 0.90-0.95 低质量视频：阈值 0.85-0.90 + public static double CalculateFrameSSIM(Image img1, Image img2) + { + // 转换为灰度图 + var gray1 = CreateResizedGrayImage(img1); + var gray2 = CreateResizedGrayImage(img2); + + // 计算全局统计量 + CalculateStats(gray1, gray2, out double mean1, out double mean2, + out double var1, out double var2, out double covar); + + // 计算SSIM分量 + double luminance = (2 * mean1 * mean2 + C1) / (mean1 * mean1 + mean2 * mean2 + C1); + double contrast = (2 * Math.Sqrt(var1) * Math.Sqrt(var2) + C2) / (var1 + var2 + C2); + double structure = (covar + C2 / 2) / (Math.Sqrt(var1) * Math.Sqrt(var2) + C2 / 2); + + // 返回SSIM值 (值越接近1表示越相似) + return luminance * contrast * structure; + } + + private static Image CreateResizedGrayImage(Image image) + { + return image + .Clone(x => x.Grayscale()) + .CloneAs(); // 转换为8位灰度格式 + } + + private static void CalculateStats( + Image img1, + Image img2, + out double mean1, + out double mean2, + out double var1, + out double var2, + out double covar) + { + int width = img1.Width; + int height = img1.Height; + int totalPixels = width * height; + + double sum1 = 0, sum2 = 0; + double sum1Sq = 0, sum2Sq = 0, sumProduct = 0; + + // 单次遍历计算所有统计量 + for (int y = 0; y < height; y++) + { + for (int x = 0; x < width; x++) + { + double val1 = img1[x, y].PackedValue; + double val2 = img2[x, y].PackedValue; + + sum1 += val1; + sum2 += val2; + sum1Sq += val1 * val1; + sum2Sq += val2 * val2; + sumProduct += val1 * val2; + } + } + + // 计算均值 + mean1 = sum1 / totalPixels; + mean2 = sum2 / totalPixels; + + // 计算方差: Var(X) = E[X²] - E[X]² + var1 = (sum1Sq / totalPixels) - (mean1 * mean1); + var2 = (sum2Sq / totalPixels) - (mean2 * mean2); + + // 计算协方差: Cov(X,Y) = E[XY] - E[X]E[Y] + covar = (sumProduct / totalPixels) - (mean1 * mean2); + } + } +} diff --git a/VideoAnalysisCore/Controllers/LJZK_Controller.cs b/VideoAnalysisCore/Controllers/LJZK_Controller.cs index 50a87ca..a23dbe8 100644 --- a/VideoAnalysisCore/Controllers/LJZK_Controller.cs +++ b/VideoAnalysisCore/Controllers/LJZK_Controller.cs @@ -141,7 +141,6 @@ namespace VideoAnalysisCore.Controllers /// /// ȡƵ֪ʶƬtaskId/tagIdѡһ /// - /// /// Զid /// [HttpGet(Name = "TaskKnowInfo")]