marking
/
Learn.VideoAnalysis
19
0
Fork 0
Code Pull Requests Actions Releases Activity

优化 计算帧差异算法 

修复 复习课试题匹配错误情况
This commit is contained in:
小肥羊 2025-07-02 16:30:56 +08:00
parent 3d112773c9
commit 0bd93c14bb
7 changed files with 170 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
VideoAnalysisCore/AICore/FFMPGE/FFMPGEHandle.cs
View File

@ -3,7 +3,6 @@ using FFmpeg.NET;
using VideoAnalysisCore.AICore.SherpaOnnx; using VideoAnalysisCore.AICore.SherpaOnnx;
using VideoAnalysisCore.Common; using VideoAnalysisCore.Common;
using System.Threading.Tasks; using System.Threading.Tasks;
using static System.Runtime.InteropServices.JavaScript.JSType;
using System.Xml.Linq; using System.Xml.Linq;
using System.Runtime.InteropServices; using System.Runtime.InteropServices;
using SqlSugar.IOC; using SqlSugar.IOC;
@ -41,6 +40,7 @@ namespace VideoAnalysisCore.AICore.FFMPGE
//间隔秒 //间隔秒
var intervalSec = 5; var intervalSec = 5;
var threshold = 8.15; var threshold = 8.15;
var ssimThreshold = 0.9;
var PPTVideoCode = await DbScoped.Sugar var PPTVideoCode = await DbScoped.Sugar
.Queryable<VideoTask>() .Queryable<VideoTask>()
.Where(s => s.Id == long.Parse(task)) .Where(s => s.Id == long.Parse(task))
@ -65,38 +65,37 @@ namespace VideoAnalysisCore.AICore.FFMPGE
var frameFiles = Directory.GetFiles(localPath, "*.jpg") var frameFiles = Directory.GetFiles(localPath, "*.jpg")
.OrderBy(f => f) .OrderBy(f => f)
.ToList(); .ToList();
RedisExpand.SetTaskProgress(task, "Frame=>50%"); RedisExpand.SetTaskProgress(task, "Frame=>50%");
Image<Rgb24> prevFrame = null; Image<Rgb24> prevFrame = null;
var keyFrames = new List<int>(5); var keyFrames = new List<int>(10) { 5};
foreach (var frameFile in frameFiles) foreach (var frameFile in frameFiles)
{ {
using (var currFrame = Image.Load<Rgb24>(frameFile)) using (var currFrame = Image.Load<Rgb24>(frameFile))
{ {
if (prevFrame != null) if (prevFrame != null)
{ {
double diff = CalculateFrameDifference(prevFrame, currFrame); double ssim = SSIMCalculator.CalculateFrameSSIM(prevFrame, currFrame);
//double diff = CalculateFrameDifference(prevFrame, currFrame);
double timestamp = GetTimestampFromFileName(frameFile) * intervalSec; double timestamp = GetTimestampFromFileName(frameFile) * intervalSec;
if (diff > threshold) //if (diff > threshold)
if (ssim < ssimThreshold)
{ {
keyFrames.Add((int)timestamp); keyFrames.Add((int)timestamp);
//string outputPath = Path.Combine(outputDir, $"change_{timestamp:0000}.jpg"); //string outputPath = Path.Combine(outputDir, $"change_{timestamp:0000}.jpg");
//currFrame.Save(outputPath); //currFrame.Save(outputPath);
Console.WriteLine($"变化帧: {timestamp}秒，差异值: {diff:F2}"); Console.WriteLine($"变化帧: {timestamp}秒，差异值: {ssim:F2}");
} }
//else //Console.WriteLine($"帧: {timestamp}秒，SSIM{ssim:F2} 差异值: {ssim:F2} ");
//Console.WriteLine($"帧: {timestamp}秒，差异值: {diff:F2}");
} }
prevFrame?.Dispose(); prevFrame?.Dispose();
prevFrame = currFrame.Clone(); prevFrame = currFrame.Clone();
} }
} }
// 遍历数组 // 去掉相邻的重复图片
for (int i = 1; i < keyFrames.Count(); i++) for (int i = 1; i < keyFrames.Count(); i++)
{ {
keyFrames[i] += 5;//ppt与课堂视频时间修正
if (keyFrames[i] - keyFrames[i - 1] < 10) if (keyFrames[i] - keyFrames[i - 1] < 10)
keyFrames[i] = -1; keyFrames[i] = -1;
} }
@ -108,13 +107,14 @@ namespace VideoAnalysisCore.AICore.FFMPGE
.Where(it => it.Id == taskID) .Where(it => it.Id == taskID)
.ExecuteCommandAsync(); .ExecuteCommandAsync();
} }
/// <summary> /// <summary>
/// 计算帧差异 /// 计算帧差异
/// </summary> /// </summary>
/// <param name="img1"></param> /// <param name="img1"></param>
/// <param name="img2"></param> /// <param name="img2"></param>
/// <returns></returns> /// <returns></returns>
static double CalculateFrameDifference(Image<Rgb24> img1, Image<Rgb24> img2) static double CalculateFrameDifference(Image<Rgb24> img1, Image<Rgb24> img2)
{ {
// 统一调整为64x64 // 统一调整为64x64

2
VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeekClient.cs
View File

@ -49,6 +49,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
{ {
//chatReq.model = "deepseek-r1"; //chatReq.model = "deepseek-r1";
if (chatReq.stream) return await ChatSSE(chatReq); if (chatReq.stream) return await ChatSSE(chatReq);
postStar: postStar:
var requestBody = chatReq.ToJson(); var requestBody = chatReq.ToJson();
HttpResponseMessage chatResp = PostJsonStream(Path, requestBody); HttpResponseMessage chatResp = PostJsonStream(Path, requestBody);
@ -57,7 +58,6 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
{ {
Console.WriteLine(DateTime.Now + $"=>GPT请求失败重试 Code = {chatResp.StatusCode} Res={res1}"); Console.WriteLine(DateTime.Now + $"=>GPT请求失败重试 Code = {chatResp.StatusCode} Res={res1}");
goto postStar; goto postStar;
} }
//throw new Exception($" GPT模型返回异常 返回参数: " + //throw new Exception($" GPT模型返回异常 返回参数: " +
// $" {System.Text.Json.JsonSerializer.Serialize(res1)}"); // $" {System.Text.Json.JsonSerializer.Serialize(res1)}");

15
VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs
View File

@ -400,6 +400,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
throw new Exception(DateTime.Now + "=>ChatGPT请求失败次数过多!!!"); throw new Exception(DateTime.Now + "=>ChatGPT请求失败次数过多!!!");
} }
/// <summary> /// <summary>
@ -423,9 +424,9 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
foreach (var item in farmeArr) foreach (var item in farmeArr)
{ {
var knowInfoArr = videoKnowArr var knowInfoArr = videoKnowArr
.Where(s => item + 20 >= s.StartTime && item <= s.EndTime) .Where(s => item + 20 >= s.StartTime && item < s.EndTime)
.ToArray(); .FirstOrDefault();
if (knowInfoArr is null || knowInfoArr.Count() == 0) if (knowInfoArr is null)
continue; continue;
var tryCount = 50; var tryCount = 50;
while (tryCount > 1) while (tryCount > 1)
@ -440,7 +441,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
continue; continue;
if (sRes.Result.res.value.Trim().Length < 10)//总试题内容长度小于10 视为无效题目 if (sRes.Result.res.value.Trim().Length < 10)//总试题内容长度小于10 视为无效题目
break; break;
Console.WriteLine(DateTime.Now + $"=>{taskInfo.Id} 提取{knowInfoArr.First().StartTime}秒试题的试题内容"); Console.WriteLine(DateTime.Now + $"=>{taskInfo.Id} 提取{knowInfoArr.StartTime}秒试题的试题内容");
Console.WriteLine(sRes.Result.res.value); Console.WriteLine(sRes.Result.res.value);
//var knowArr=JsonSerializer.Serialize(knowInfoArr.Select(s => new { s.KnowPointId, s.KnowPoint })); //var knowArr=JsonSerializer.Serialize(knowInfoArr.Select(s => new { s.KnowPointId, s.KnowPoint }));
var resFormat = """[{"Type":string(试题类型),"TopicStem":string(试题题干),"QuestionArr":[{"Question":string(子问题),"KnowPointId":(string)知识点ID}]}]"""; var resFormat = """[{"Type":string(试题类型),"TopicStem":string(试题题干),"QuestionArr":[{"Question":string(子问题),"KnowPointId":(string)知识点ID}]}]""";
@ -470,7 +471,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
vq.StartTime = item; vq.StartTime = item;
vq.FilePath = filePath; vq.FilePath = filePath;
vq.VideoTaskId = taskInfo.Id; vq.VideoTaskId = taskInfo.Id;
vq.StageId = knowInfoArr.First().StageId; vq.StageId = knowInfoArr.StageId;
vq.Question = qt.Question; vq.Question = qt.Question;
vq.TopicId = TopicId; vq.TopicId = TopicId;
vq.Type = q.Type; vq.Type = q.Type;
@ -497,7 +498,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
} }
catch (Exception ex) catch (Exception ex)
{ {
Console.WriteLine(DateTime.Now + $"=>{taskInfo.Id} 提取{knowInfoArr.First().StartTime}秒试题出现错误 {ex.Message}"); Console.WriteLine(DateTime.Now + $"=>{taskInfo.Id} 提取{knowInfoArr.StartTime}秒试题出现错误 {ex.Message}");
} }
} }
} }
@ -661,7 +662,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
var postMessages = var postMessages =
$"我将提供一段视频的字幕内容,请你帮我分析这堂课的课程类型是什么。" + $"我将提供一段视频的字幕内容,请你帮我分析这堂课的课程类型是什么。" +
$"授课类型限定在我提供的范围内 [{videoTypeStr}]" + $"授课类型限定在我提供的范围内 [{videoTypeStr}]" +
$"其中如果是[复习/习题课/试题讲解课程]那么课程类型视为'复习'" + $"其中如果是[习题课/长篇幅的试题讲解课程]那么课程类型视为'复习'" +
$"请简介的说明分析出课程类型的原因,如果分析出的课程类型与限定条件不匹配则返回NULL" + $"请简介的说明分析出课程类型的原因,如果分析出的课程类型与限定条件不匹配则返回NULL" +
$"输出内容只返回json格式为({resFormat})" + $"输出内容只返回json格式为({resFormat})" +
$"以下是字幕内容" + $"以下是字幕内容" +

26
VideoAnalysisCore/Common/DownloadFile.cs
View File

@ -132,24 +132,16 @@ namespace VideoAnalysisCore.Common
var fileUrl = taskInfo.MediaUrl; var fileUrl = taskInfo.MediaUrl;
if (string.IsNullOrEmpty(fileUrl)) if (string.IsNullOrEmpty(fileUrl))
{ {
switch (taskInfo.VideoType) var videoInfo = await vodClient.GetPlayInfoAsync(new AlibabaCloud.SDK.Vod20170321.Models.GetPlayInfoRequest()
{ {
case AttachmentsInfoType.新课: VideoId = taskInfo.TagId,
case AttachmentsInfoType.复习: Formats = "mp4",
var videoInfo = await vodClient.GetPlayInfoAsync(new AlibabaCloud.SDK.Vod20170321.Models.GetPlayInfoRequest() OutputType = "cdn",
{ AuthTimeout = 3600 * 24 * 12,
VideoId = taskInfo.TagId, });
Formats = "mp4", if (videoInfo is null || videoInfo.StatusCode != 200 && !videoInfo.Body.PlayInfoList.PlayInfo.Any())
OutputType = "cdn", throw new Exception($"{DateTime.Now} 视频订阅=>获取阿里云视频信息失败 VideoCode {taskInfo.TagId} StatusCode {videoInfo?.StatusCode}");
AuthTimeout = 3600 * 24 * 12, fileUrl = videoInfo.Body.PlayInfoList.PlayInfo.First().PlayURL;
});
if (videoInfo is null || videoInfo.StatusCode != 200 && !videoInfo.Body.PlayInfoList.PlayInfo.Any())
throw new Exception($"{DateTime.Now} 视频订阅=>获取阿里云视频信息失败 VideoCode {taskInfo.TagId} StatusCode {videoInfo?.StatusCode}");
fileUrl = videoInfo.Body.PlayInfoList.PlayInfo.First().PlayURL;
break;
default:
break;
}
} }
if (string.IsNullOrEmpty(fileUrl)) if (string.IsNullOrEmpty(fileUrl))
throw new Exception($"任务id[{task}] 资源地址无效 {fileUrl}"); throw new Exception($"任务id[{task}] 资源地址无效 {fileUrl}");

88
VideoAnalysisCore/Common/RedisExpand.cs
View File

@ -174,7 +174,8 @@ namespace VideoAnalysisCore.Common
Redis.HMSet(RedisExpandKey.Task(taskId), "StartTime", startTime); Redis.HMSet(RedisExpandKey.Task(taskId), "StartTime", startTime);
await SubscribeList[@enum](tId); await TouchChannel(@enum, tId, SubscribeList[@enum]);
//await SubscribeList[@enum](tId);
var e = @enum.NextEnum(); var e = @enum.NextEnum();
if (e is null) if (e is null)
break; break;
@ -233,54 +234,43 @@ namespace VideoAnalysisCore.Common
{ {
if (Redis is null) throw new Exception("redis未初始化"); if (Redis is null) throw new Exception("redis未初始化");
SubscribeList.Add(RedisChannelEnum.下载文件, SubscribeList.Add(RedisChannelEnum.下载文件, (task) =>
async (msg) => await TouchChannel(RedisChannelEnum.下载文件, msg, {
(task) => using var scope = AppCommon.Services?.CreateScope();
{ if (scope is null || scope.ServiceProvider.GetService<DownloadFile>() is null)
using var scope = AppCommon.Services?.CreateScope(); throw new Exception("DownloadFile 未注入");
if (scope is null || scope.ServiceProvider.GetService<DownloadFile>() is null) else
throw new Exception("DownloadFile 未注入"); return scope.ServiceProvider.GetService<DownloadFile>()?.RunTask(task) ?? Task.CompletedTask;
else });
return scope.ServiceProvider.GetService<DownloadFile>()?.RunTask(task) ?? Task.CompletedTask; SubscribeList.Add(RedisChannelEnum.分离音频, FFMPGEHandle.RunAsync);
})); SubscribeList.Add(RedisChannelEnum.解析字幕, SenseVoice.RunTask);
SubscribeList.Add(RedisChannelEnum.分离音频, //SubscribeList.Add(RedisChannelEnum.解析说话人,Speaker.Run);
async (msg) => await TouchChannel(RedisChannelEnum.分离音频, msg, FFMPGEHandle.RunAsync)); SubscribeList.Add(RedisChannelEnum.AI课程类型,
SubscribeList.Add(RedisChannelEnum.解析字幕, (task) =>
async (msg) => await TouchChannel(RedisChannelEnum.解析字幕, msg, SenseVoice.RunTask)); {
//SubscribeList.Add(RedisChannelEnum.解析说话人, using var scope = AppCommon.Services?.CreateScope();
// async (msg) => await TouchChannel(RedisChannelEnum.解析说话人, msg, Speaker.Run)); if (scope is null || scope.ServiceProvider.GetService<IBserGPT>() is null)
SubscribeList.Add(RedisChannelEnum.AI课程类型, throw new Exception("IBserGPT 未注入");
async (msg) => await TouchChannel(RedisChannelEnum.AI课程类型, msg, else
(task) => return scope.ServiceProvider.GetService<IBserGPT>()?.GetVideoType(task) ?? Task.CompletedTask;
{ });
using var scope = AppCommon.Services?.CreateScope(); SubscribeList.Add(RedisChannelEnum.AI模型分析, (task) =>
if (scope is null || scope.ServiceProvider.GetService<IBserGPT>() is null) {
throw new Exception("IBserGPT 未注入"); using var scope = AppCommon.Services?.CreateScope();
else if (scope is null || scope.ServiceProvider.GetService<IBserGPT>() is null)
return scope.ServiceProvider.GetService<IBserGPT>()?.GetVideoType(task) ?? Task.CompletedTask; throw new Exception("IBserGPT 未注入");
})); else
SubscribeList.Add(RedisChannelEnum.AI模型分析, return scope.ServiceProvider.GetService<IBserGPT>()?.GetKnow(task) ?? Task.CompletedTask;
async (msg) => await TouchChannel(RedisChannelEnum.AI模型分析, msg, });
(task) => SubscribeList.Add(RedisChannelEnum.AI分析试题, (task) =>
{ {
using var scope = AppCommon.Services?.CreateScope(); using var scope = AppCommon.Services?.CreateScope();
if (scope is null || scope.ServiceProvider.GetService<IBserGPT>() is null) if (scope is null || scope.ServiceProvider.GetService<IBserGPT>() is null)
throw new Exception("IBserGPT 未注入"); throw new Exception("IBserGPT 未注入");
else else
return scope.ServiceProvider.GetService<IBserGPT>()?.GetKnow(task) ?? Task.CompletedTask; return scope.ServiceProvider.GetService<IBserGPT>()?.GetVideoQuestion(task) ?? Task.CompletedTask;
})); });
SubscribeList.Add(RedisChannelEnum.AI分析试题, SubscribeList.Add(RedisChannelEnum.结束任务, TaskEnd);
async (msg) => await TouchChannel(RedisChannelEnum.AI分析试题, msg,
(task) =>
{
using var scope = AppCommon.Services?.CreateScope();
if (scope is null || scope.ServiceProvider.GetService<IBserGPT>() is null)
throw new Exception("IBserGPT 未注入");
else
return scope.ServiceProvider.GetService<IBserGPT>()?.GetVideoQuestion(task) ?? Task.CompletedTask;
}));
SubscribeList.Add(RedisChannelEnum.结束任务,
async (msg) => await TouchChannel(RedisChannelEnum.结束任务, msg, TaskEnd));
ReceivingTaskAsync(); ReceivingTaskAsync();

97
VideoAnalysisCore/Common/SSIMCalculator.cs Normal file
View File

@ -0,0 +1,97 @@
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using SixLabors.ImageSharp.Processing;
namespace VideoAnalysisCore.Common
{
/// <summary>
/// Ssim计算器
/// </summary>
public class SSIMCalculator
{
// SSIM计算常量 (基于8-bit图像范围0-255)
private const double C1 = (0.01 * 255) * (0.01 * 255);
private const double C2 = (0.03 * 255) * (0.03 * 255);
/// <summary>
/// 计算连续帧的SSIM 值
/// </summary>
/// <param name="img1"></param>
/// <param name="img2"></param>
/// <returns>返回阈值 0-1 越小变化越大<para>清晰视频：阈值 0.90-0.95 </para> <para>低质量视频：阈值 0.85-0.90</para></returns>
public static double CalculateFrameSSIM(Image<Rgb24> img1, Image<Rgb24> img2)
{
// 转换为灰度图
var gray1 = CreateResizedGrayImage(img1);
var gray2 = CreateResizedGrayImage(img2);
// 计算全局统计量
CalculateStats(gray1, gray2, out double mean1, out double mean2,
out double var1, out double var2, out double covar);
// 计算SSIM分量
double luminance = (2 * mean1 * mean2 + C1) / (mean1 * mean1 + mean2 * mean2 + C1);
double contrast = (2 * Math.Sqrt(var1) * Math.Sqrt(var2) + C2) / (var1 + var2 + C2);
double structure = (covar + C2 / 2) / (Math.Sqrt(var1) * Math.Sqrt(var2) + C2 / 2);
// 返回SSIM值 (值越接近1表示越相似)
return luminance * contrast * structure;
}
private static Image<L8> CreateResizedGrayImage(Image<Rgb24> image)
{
return image
.Clone(x => x.Grayscale())
.CloneAs<L8>(); // 转换为8位灰度格式
}
private static void CalculateStats(
Image<L8> img1,
Image<L8> img2,
out double mean1,
out double mean2,
out double var1,
out double var2,
out double covar)
{
int width = img1.Width;
int height = img1.Height;
int totalPixels = width * height;
double sum1 = 0, sum2 = 0;
double sum1Sq = 0, sum2Sq = 0, sumProduct = 0;
// 单次遍历计算所有统计量
for (int y = 0; y < height; y++)
{
for (int x = 0; x < width; x++)
{
double val1 = img1[x, y].PackedValue;
double val2 = img2[x, y].PackedValue;
sum1 += val1;
sum2 += val2;
sum1Sq += val1 * val1;
sum2Sq += val2 * val2;
sumProduct += val1 * val2;
}
}
// 计算均值
mean1 = sum1 / totalPixels;
mean2 = sum2 / totalPixels;
// 计算方差: Var(X) = E[X²] - E[X]²
var1 = (sum1Sq / totalPixels) - (mean1 * mean1);
var2 = (sum2Sq / totalPixels) - (mean2 * mean2);
// 计算协方差: Cov(X,Y) = E[XY] - E[X]E[Y]
covar = (sumProduct / totalPixels) - (mean1 * mean2);
}
}
}

1
VideoAnalysisCore/Controllers/LJZK_Controller.cs
View File

@ -141,7 +141,6 @@ namespace VideoAnalysisCore.Controllers
/// <summary> /// <summary>
/// 获取视频知识点片段<para>taskId/tagId二选一</para> /// 获取视频知识点片段<para>taskId/tagId二选一</para>
/// </summary> /// </summary>
/// <param name="taskId"></param>
/// <param name="tagId">自定义id</param> /// <param name="tagId">自定义id</param>
/// <returns></returns> /// <returns></returns>
[HttpGet(Name = "TaskKnowInfo")] [HttpGet(Name = "TaskKnowInfo")]