Compare commits

..

No commits in common. "0bd93c14bb4b4d418e02a18a3a79e58a288a7fda" and "d36fb2fc9f73eac36868cdeb60e70cb7e3123714" have entirely different histories.

14 changed files with 87 additions and 290 deletions

View File

@ -3,6 +3,7 @@ using FFmpeg.NET;
using VideoAnalysisCore.AICore.SherpaOnnx;
using VideoAnalysisCore.Common;
using System.Threading.Tasks;
using static System.Runtime.InteropServices.JavaScript.JSType;
using System.Xml.Linq;
using System.Runtime.InteropServices;
using SqlSugar.IOC;
@ -40,7 +41,6 @@ namespace VideoAnalysisCore.AICore.FFMPGE
//间隔秒
var intervalSec = 5;
var threshold = 8.15;
var ssimThreshold = 0.9;
var PPTVideoCode = await DbScoped.Sugar
.Queryable<VideoTask>()
.Where(s => s.Id == long.Parse(task))
@ -65,37 +65,38 @@ namespace VideoAnalysisCore.AICore.FFMPGE
var frameFiles = Directory.GetFiles(localPath, "*.jpg")
.OrderBy(f => f)
.ToList();
RedisExpand.SetTaskProgress(task, "Frame=>50%");
Image<Rgb24> prevFrame = null;
var keyFrames = new List<int>(10) { 5};
var keyFrames = new List<int>(5);
foreach (var frameFile in frameFiles)
{
using (var currFrame = Image.Load<Rgb24>(frameFile))
{
if (prevFrame != null)
{
double ssim = SSIMCalculator.CalculateFrameSSIM(prevFrame, currFrame);
//double diff = CalculateFrameDifference(prevFrame, currFrame);
double diff = CalculateFrameDifference(prevFrame, currFrame);
double timestamp = GetTimestampFromFileName(frameFile) * intervalSec;
//if (diff > threshold)
if (ssim < ssimThreshold)
if (diff > threshold)
{
keyFrames.Add((int)timestamp);
//string outputPath = Path.Combine(outputDir, $"change_{timestamp:0000}.jpg");
//currFrame.Save(outputPath);
Console.WriteLine($"变化帧: {timestamp}秒,差异值: {ssim:F2}");
Console.WriteLine($"变化帧: {timestamp}秒,差异值: {diff:F2}");
}
//Console.WriteLine($"帧: {timestamp}秒SSIM{ssim:F2} 差异值: {ssim:F2} ");
//else
//Console.WriteLine($"帧: {timestamp}秒,差异值: {diff:F2}");
}
prevFrame?.Dispose();
prevFrame = currFrame.Clone();
}
}
// 去掉相邻的重复图片
// 遍历数组
for (int i = 1; i < keyFrames.Count(); i++)
{
keyFrames[i] += 5;//ppt与课堂视频时间修正
if (keyFrames[i] - keyFrames[i - 1] < 10)
keyFrames[i] = -1;
}
@ -108,7 +109,6 @@ namespace VideoAnalysisCore.AICore.FFMPGE
.ExecuteCommandAsync();
}
/// <summary>
/// 计算帧差异
/// </summary>

View File

@ -22,11 +22,5 @@ namespace VideoAnalysisCore.AICore.GPT
/// <param name="task">任务id</param>
/// <returns></returns>
public Task GetVideoQuestion(string task);
/// <summary>
/// 获取视频类型
/// </summary>
/// <param name="task"></param>
/// <returns></returns>
public Task GetVideoType(string task);
}
}

View File

@ -423,10 +423,5 @@ namespace VideoAnalysisCore.AICore.GPT.ChatGPT
{
throw new NotImplementedException();
}
Task IBserGPT.GetVideoType(string task)
{
throw new NotImplementedException();
}
}
}

View File

@ -49,7 +49,6 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
{
//chatReq.model = "deepseek-r1";
if (chatReq.stream) return await ChatSSE(chatReq);
postStar:
var requestBody = chatReq.ToJson();
HttpResponseMessage chatResp = PostJsonStream(Path, requestBody);
@ -58,6 +57,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
{
Console.WriteLine(DateTime.Now + $"=>GPT请求失败重试 Code = {chatResp.StatusCode} Res={res1}");
goto postStar;
}
//throw new Exception($" GPT模型返回异常 返回参数: " +
// $" {System.Text.Json.JsonSerializer.Serialize(res1)}");

View File

@ -20,7 +20,6 @@ using Aliyun.OSS;
using Yitter.IdGenerator;
using VideoAnalysisCore.Common.Expand;
using System.Collections.Generic;
using UserCenter.Model.Enum;
namespace VideoAnalysisCore.AICore.GPT.DeepSeek
{
@ -347,7 +346,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
if (max_tokens != -1)
chatRep.max_tokens = max_tokens;
var tryCount = 10;
while (tryCount-- > 0)
while (--tryCount > 0)
{
try
{
@ -393,8 +392,8 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
Console.WriteLine(DateTime.Now + $"=>ChatGPT结果解析错误 重试剩余{tryCount}");
Console.WriteLine(ex.Message);
}
}
throw new Exception(DateTime.Now + "=>ChatGPT请求失败次数过多!!!");
@ -402,7 +401,6 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
/// <summary>
/// 提取试题
/// </summary>
@ -424,9 +422,9 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
foreach (var item in farmeArr)
{
var knowInfoArr = videoKnowArr
.Where(s => item + 20 >= s.StartTime && item < s.EndTime)
.FirstOrDefault();
if (knowInfoArr is null)
.Where(s => item + 20 >= s.StartTime && item <= s.EndTime)
.ToArray();
if (knowInfoArr is null || knowInfoArr.Count() == 0)
continue;
var tryCount = 50;
while (tryCount > 1)
@ -441,7 +439,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
continue;
if (sRes.Result.res.value.Trim().Length < 10)//总试题内容长度小于10 视为无效题目
break;
Console.WriteLine(DateTime.Now + $"=>{taskInfo.Id} 提取{knowInfoArr.StartTime}秒试题的试题内容");
Console.WriteLine(DateTime.Now + $"=>{taskInfo.Id} 提取{knowInfoArr.First().StartTime}秒试题的试题内容");
Console.WriteLine(sRes.Result.res.value);
//var knowArr=JsonSerializer.Serialize(knowInfoArr.Select(s => new { s.KnowPointId, s.KnowPoint }));
var resFormat = """[{"Type":string(试题类型),"TopicStem":string(试题题干),"QuestionArr":[{"Question":string(子问题),"KnowPointId":(string)知识点ID}]}]""";
@ -471,7 +469,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
vq.StartTime = item;
vq.FilePath = filePath;
vq.VideoTaskId = taskInfo.Id;
vq.StageId = knowInfoArr.StageId;
vq.StageId = knowInfoArr.First().StageId;
vq.Question = qt.Question;
vq.TopicId = TopicId;
vq.Type = q.Type;
@ -498,7 +496,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
}
catch (Exception ex)
{
Console.WriteLine(DateTime.Now + $"=>{taskInfo.Id} 提取{knowInfoArr.StartTime}秒试题出现错误 {ex.Message}");
Console.WriteLine(DateTime.Now + $"=>{taskInfo.Id} 提取{knowInfoArr.First().StartTime}秒试题出现错误 {ex.Message}");
}
}
}
@ -618,6 +616,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
var subject = taskInfo.Subject.ToString();
var Course_Id = taskInfo.CourseId;
var captionsArr = JsonSerializer.Deserialize<SenseVoiceRes[]>(taskInfo.Captions);
//处理视频授课章节
var sections = await GetSections(taskInfo, Course_Id);
var know = await knowledgeInfoDB.GetFirstAsync(s => s.Course_Id == Course_Id && s.Name == sections);
@ -631,51 +630,5 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
return;
}
/// <summary>
/// 分析视频是否为复习课
/// </summary>
/// <param name="task"></param>
/// <returns></returns>
public async Task GetVideoType(string task)
{
var taskId = long.Parse(task);
var taskInfo = await videoTaskDB.AsQueryable()
.Where(s => s.Id == taskId)
.FirstAsync();
var subject = taskInfo.Subject.ToString();
var Course_Id = taskInfo.CourseId;
var videoTypeStr = string.Join(',', Enum.GetNames(typeof(AttachmentsInfoType)));
//处理视频授课章节
var sections = await GetSections(taskInfo, Course_Id);
var captionsArr = JsonSerializer.Deserialize<SenseVoiceRes[]>(taskInfo.Captions);
captionsArr = await OptimizeSubtitles(taskInfo, captionsArr, sections);
//合并字幕
var captions = ExpandFunction.GetSpeakerCaptions(captionsArr);
var resFormat = """{ "VideoType":string(课程类型),"Reason":string(分析原因)}""";
var postMessages =
$"我将提供一段视频的字幕内容,请你帮我分析这堂课的课程类型是什么。" +
$"授课类型限定在我提供的范围内 [{videoTypeStr}]" +
$"其中如果是[习题课/长篇幅的试题讲解课程]那么课程类型视为'复习'" +
$"请简介的说明分析出课程类型的原因,如果分析出的课程类型与限定条件不匹配则返回NULL" +
$"输出内容只返回json格式为({resFormat})" +
$"以下是字幕内容" +
$"`{captions.Captions}`";
var resData = await ChatAsync<AIVideoTypeDto?>(taskInfo.Id.ToString(), postMessages, "视频类型", "deepseek-chat");
var res = resData.VideoType.ToEnum<AttachmentsInfoType>();
if (res != null)
{
await videoTaskDB.AsUpdateable()
.SetColumns(it => it.VideoType == res)
.Where(it => it.Id == taskInfo.Id)
.ExecuteCommandAsync();
}
}
}
}

View File

@ -1,18 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using VideoAnalysisCore.Model.Enum;
namespace VideoAnalysisCore.AICore.GPT.Dto
{
public class AIVideoTypeDto
{
public string? VideoType { get; set; }
/// <summary>
/// 分析结果原因
/// </summary>
public string Reason { get; set; }
}
}

View File

@ -132,6 +132,10 @@ namespace VideoAnalysisCore.Common
var fileUrl = taskInfo.MediaUrl;
if (string.IsNullOrEmpty(fileUrl))
{
switch (taskInfo.VideoType)
{
case AttachmentsInfoType.:
case AttachmentsInfoType.:
var videoInfo = await vodClient.GetPlayInfoAsync(new AlibabaCloud.SDK.Vod20170321.Models.GetPlayInfoRequest()
{
VideoId = taskInfo.TagId,
@ -142,6 +146,10 @@ namespace VideoAnalysisCore.Common
if (videoInfo is null || videoInfo.StatusCode != 200 && !videoInfo.Body.PlayInfoList.PlayInfo.Any())
throw new Exception($"{DateTime.Now} 视频订阅=>获取阿里云视频信息失败 VideoCode {taskInfo.TagId} StatusCode {videoInfo?.StatusCode}");
fileUrl = videoInfo.Body.PlayInfoList.PlayInfo.First().PlayURL;
break;
default:
break;
}
}
if (string.IsNullOrEmpty(fileUrl))
throw new Exception($"任务id[{task}] 资源地址无效 {fileUrl}");

View File

@ -174,8 +174,7 @@ namespace VideoAnalysisCore.Common
Redis.HMSet(RedisExpandKey.Task(taskId), "StartTime", startTime);
await TouchChannel(@enum, tId, SubscribeList[@enum]);
//await SubscribeList[@enum](tId);
await SubscribeList[@enum](tId);
var e = @enum.NextEnum();
if (e is null)
break;
@ -234,43 +233,44 @@ namespace VideoAnalysisCore.Common
{
if (Redis is null) throw new Exception("redis未初始化");
SubscribeList.Add(RedisChannelEnum., (task) =>
SubscribeList.Add(RedisChannelEnum.,
async (msg) => await TouchChannel(RedisChannelEnum., msg,
(task) =>
{
using var scope = AppCommon.Services?.CreateScope();
if (scope is null || scope.ServiceProvider.GetService<DownloadFile>() is null)
throw new Exception("DownloadFile 未注入");
else
return scope.ServiceProvider.GetService<DownloadFile>()?.RunTask(task) ?? Task.CompletedTask;
});
SubscribeList.Add(RedisChannelEnum., FFMPGEHandle.RunAsync);
SubscribeList.Add(RedisChannelEnum., SenseVoice.RunTask);
//SubscribeList.Add(RedisChannelEnum.解析说话人,Speaker.Run);
SubscribeList.Add(RedisChannelEnum.AI课程类型,
}));
SubscribeList.Add(RedisChannelEnum.,
async (msg) => await TouchChannel(RedisChannelEnum., msg, FFMPGEHandle.RunAsync));
SubscribeList.Add(RedisChannelEnum.,
async (msg) => await TouchChannel(RedisChannelEnum., msg, SenseVoice.RunTask));
//SubscribeList.Add(RedisChannelEnum.解析说话人,
// async (msg) => await TouchChannel(RedisChannelEnum.解析说话人, msg, Speaker.Run));
SubscribeList.Add(RedisChannelEnum.AI模型分析,
async (msg) => await TouchChannel(RedisChannelEnum.AI模型分析, msg,
(task) =>
{
using var scope = AppCommon.Services?.CreateScope();
if (scope is null || scope.ServiceProvider.GetService<IBserGPT>() is null)
throw new Exception("IBserGPT 未注入");
else
return scope.ServiceProvider.GetService<IBserGPT>()?.GetVideoType(task) ?? Task.CompletedTask;
});
SubscribeList.Add(RedisChannelEnum.AI模型分析, (task) =>
{
using var scope = AppCommon.Services?.CreateScope();
if (scope is null || scope.ServiceProvider.GetService<IBserGPT>() is null)
throw new Exception("IBserGPT 未注入");
else
return scope.ServiceProvider.GetService<IBserGPT>()?.GetKnow(task) ?? Task.CompletedTask;
});
SubscribeList.Add(RedisChannelEnum.AI分析试题, (task) =>
}));
SubscribeList.Add(RedisChannelEnum.AI分析试题,
async (msg) => await TouchChannel(RedisChannelEnum.AI分析试题, msg,
(task) =>
{
using var scope = AppCommon.Services?.CreateScope();
if (scope is null || scope.ServiceProvider.GetService<IBserGPT>() is null)
throw new Exception("IBserGPT 未注入");
else
return scope.ServiceProvider.GetService<IBserGPT>()?.GetVideoQuestion(task) ?? Task.CompletedTask;
});
SubscribeList.Add(RedisChannelEnum., TaskEnd);
}));
SubscribeList.Add(RedisChannelEnum.,
async (msg) => await TouchChannel(RedisChannelEnum., msg, TaskEnd));
ReceivingTaskAsync();

View File

@ -1,97 +0,0 @@
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using SixLabors.ImageSharp.Processing;
namespace VideoAnalysisCore.Common
{
/// <summary>
/// Ssim计算器
/// </summary>
public class SSIMCalculator
{
// SSIM计算常量 (基于8-bit图像范围0-255)
private const double C1 = (0.01 * 255) * (0.01 * 255);
private const double C2 = (0.03 * 255) * (0.03 * 255);
/// <summary>
/// 计算连续帧的SSIM 值
/// </summary>
/// <param name="img1"></param>
/// <param name="img2"></param>
/// <returns>返回阈值 0-1 越小变化越大<para>清晰视频:阈值 0.90-0.95 </para> <para>低质量视频:阈值 0.85-0.90</para></returns>
public static double CalculateFrameSSIM(Image<Rgb24> img1, Image<Rgb24> img2)
{
// 转换为灰度图
var gray1 = CreateResizedGrayImage(img1);
var gray2 = CreateResizedGrayImage(img2);
// 计算全局统计量
CalculateStats(gray1, gray2, out double mean1, out double mean2,
out double var1, out double var2, out double covar);
// 计算SSIM分量
double luminance = (2 * mean1 * mean2 + C1) / (mean1 * mean1 + mean2 * mean2 + C1);
double contrast = (2 * Math.Sqrt(var1) * Math.Sqrt(var2) + C2) / (var1 + var2 + C2);
double structure = (covar + C2 / 2) / (Math.Sqrt(var1) * Math.Sqrt(var2) + C2 / 2);
// 返回SSIM值 (值越接近1表示越相似)
return luminance * contrast * structure;
}
private static Image<L8> CreateResizedGrayImage(Image<Rgb24> image)
{
return image
.Clone(x => x.Grayscale())
.CloneAs<L8>(); // 转换为8位灰度格式
}
private static void CalculateStats(
Image<L8> img1,
Image<L8> img2,
out double mean1,
out double mean2,
out double var1,
out double var2,
out double covar)
{
int width = img1.Width;
int height = img1.Height;
int totalPixels = width * height;
double sum1 = 0, sum2 = 0;
double sum1Sq = 0, sum2Sq = 0, sumProduct = 0;
// 单次遍历计算所有统计量
for (int y = 0; y < height; y++)
{
for (int x = 0; x < width; x++)
{
double val1 = img1[x, y].PackedValue;
double val2 = img2[x, y].PackedValue;
sum1 += val1;
sum2 += val2;
sum1Sq += val1 * val1;
sum2Sq += val2 * val2;
sumProduct += val1 * val2;
}
}
// 计算均值
mean1 = sum1 / totalPixels;
mean2 = sum2 / totalPixels;
// 计算方差: Var(X) = E[X²] - E[X]²
var1 = (sum1Sq / totalPixels) - (mean1 * mean1);
var2 = (sum2Sq / totalPixels) - (mean2 * mean2);
// 计算协方差: Cov(X,Y) = E[XY] - E[X]E[Y]
covar = (sumProduct / totalPixels) - (mean1 * mean2);
}
}
}

View File

@ -16,8 +16,6 @@ using Microsoft.AspNetCore.Http;
using VideoAnalysisCore.Model.Dto;
using VideoAnalysisCore.Controllers.Dto;
using VideoAnalysisCore.Common.Expand;
using Microsoft.Extensions.DependencyInjection;
using System.Threading.Tasks;
namespace VideoAnalysisCore.Controllers
{
@ -70,22 +68,6 @@ namespace VideoAnalysisCore.Controllers
#endif
/// <summary>
/// 初始化主库表
/// </summary>
/// <param name="url">文件流</param>
/// <returns></returns>
[HttpGet(Name = "tets1")]
public IActionResult tets1(string task)
{
using var scope = AppCommon.Services?.CreateScope();
if (scope is null || scope.ServiceProvider.GetService<IBserGPT>() is null)
throw new Exception("IBserGPT 未注入");
else
scope.ServiceProvider.GetService<IBserGPT>()?.GetVideoType(task);
return Ok();
}
/// <summary>
/// ÓïÒôʶ±ð
/// </summary>

View File

@ -141,6 +141,7 @@ namespace VideoAnalysisCore.Controllers
/// <summary>
/// 获取视频知识点片段<para>taskId/tagId二选一</para>
/// </summary>
/// <param name="taskId"></param>
/// <param name="tagId">自定义id</param>
/// <returns></returns>
[HttpGet(Name = "TaskKnowInfo")]

View File

@ -47,19 +47,10 @@ namespace VideoAnalysisCore.Job
var taskArr = await nodePackageInfoDB.AsQueryable()
.Where(s => s.SuccessTime == null)
.ToArrayAsync();
var videoArr = await videoTaskDB.AsQueryable()
var videoIdArr = await videoTaskDB.AsQueryable()
.Where(s => s.EndTime != null)
.Select(s =>new VideoTask {
Id = s.Id,
TagId = s.TagId,
VideoType=s.VideoType
})
.Select(s => s.TagId)
.ToArrayAsync();
var videoDic = videoArr
.Where(s=>s.TagId != null)
.GroupBy(s => s.TagId!)
.ToDictionary(s => s.Key,s=>s.First());
var videoIdArr = videoArr.Select(s => s.TagId).ToArray();
var videoPPTArr = await videoTaskDB.AsQueryable()
.Where(s => s.EndTime != null)
@ -70,13 +61,6 @@ namespace VideoAnalysisCore.Job
{
if (videoIdArr.Contains(item.VideoCode)|| videoPPTArr.Contains(item.VideoCode))
{
if (videoDic.ContainsKey(item.VideoCode))
{
var v = videoDic[item.VideoCode];
if (v.VideoType!=null && v.VideoType != item.CourseType)
item.CourseType = v.VideoType.Value;//额外同步课程类型[新课 复习课]
}
postData.Add(item);
item.SuccessTime = DateTime.Now;
}

View File

@ -25,11 +25,6 @@
///// 解析说话人
///// </summary>
//解析说话人 = 30,
/// <summary>
/// AI课程类型
/// </summary>
AI课程类型 = 31,
/// <summary>
/// Chat模型分析
/// </summary>

View File

@ -75,7 +75,7 @@ namespace VideoAnalysisCore.Model
/// <summary>
/// 通知回调地址
/// </summary>
[SugarColumn(IsNullable = true,Length = 500)]
[SugarColumn(Length = 500)]
public string CallBackUrl { get; set; }
/// <summary>
/// 请求区域