marking
/
Learn.VideoAnalysis
19
0
Fork 0
Code Pull Requests Actions Releases Activity

新增 AI优化字幕 

优化 获取片段流程
This commit is contained in:
小肥羊 2025-05-12 18:14:39 +08:00
parent 7cbd4588f5
commit 61af4e9827
2 changed files with 163 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

220
VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs
View File

@ -81,8 +81,8 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
// $"章节范围限定在[{string.Join(',', xkwKnows)}]范围内." +
// $"输出格式 json字符串 对象格式{fileNameResFormat}";
var rCaptionArr = string.Join(',', captionsArr
.Where((s,i)=>i%3==0)
.Take((int)(captionsArr?.Length??0 / 2.2))
.Where((s, i) => i % 3 == 0)
.Take((int)(captionsArr?.Length ?? 0 / 2.2))
.Select(s => s.Text));
var fileNamePostMessages =
@ -307,7 +307,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
var lastTime = 0;
var endTime = 0;
var timeSpan = 700;
while (endTime + 60< maxVideoTime)
while (endTime + 60 < maxVideoTime)
{
try
{
@ -331,7 +331,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
Console.WriteLine(DateTime.Now + $"=>1.开始分析视频内容 {lastTime}~{endTime}");
questionRes.AddRange(await ChatAsync<VideoKnowRes[]>(task, postMessages, null));
lastTime = (int)questionRes.Last().EndTime.Value - (lastTime==0?0: 30);
lastTime = (int)questionRes.Last().EndTime.Value - (lastTime == 0 ? 0 : 30);
}
catch (Exception ex)
{
@ -438,7 +438,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
/// 获取内容对应的章节
/// </summary>
/// <returns></returns>
public async Task<List<VideoKonwPoint>> GetVideoKnow(List<VideoKnowRes> questionRes,VideoTask taskInfo, string sections, int course_Id)
private async Task<List<VideoKonwPoint>> GetVideoKnow(List<VideoKnowRes> questionRes, VideoTask taskInfo, string sections, int course_Id)
{
var know = await knowledgeInfoDB.GetFirstAsync(s => s.Course_Id == course_Id && s.Name == sections);
@ -502,8 +502,11 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
/// 获取内容对应的章节
/// </summary>
/// <returns></returns>
public async Task<string?> GetSections(VideoTask taskInfo,int course_Id)
private async Task<string?> GetSections(VideoTask taskInfo, int course_Id)
{
if (!string.IsNullOrEmpty(taskInfo.Sections))
return taskInfo.Sections;
var xkwKnows = await knowledgeInfoDB.AsQueryable()
.Where(s => s.Course_Id == course_Id
&& (s.Depth == 3
@ -527,6 +530,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
var task = taskInfo.Id.ToString();
var fileNameInfoRes = await ChatAsync<FileNameInfo>
(task, fileNamePostMessages, null);
taskInfo.Sections = fileNameInfoRes.授课章节;
await videoTaskDB.AsUpdateable()
.SetColumns(it => it.Sections == fileNameInfoRes.授课章节)
.Where(it => it.Id == taskInfo.Id)
@ -536,6 +540,143 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
return fileNameInfoRes?.授课章节;
}
/// <summary>
/// 检查AI切片结果质量
/// </summary>
/// <returns></returns>
private async Task<CheckMessageDto> VerifySpanQuality(List<VideoKnowRes> questionRes, VideoTask taskInfo, TotalCaptionsDto captions, string sections, int course_Id)
{
//校验结果质量
var thems = JsonSerializer.Serialize(questionRes.Adapt<VideoKnowQueryDto[]>());
var checkResFormat = """{"Score":打分(number),"Evaluation":评价(string)""";//,"Data":优化后的分段(array)}""";
var checkMessage = "我为视频的讲解内容做了一些分段,希望你能通读字幕内容后检查下的分段是否符合我的要求?" +
$"检查这些分段的时间是否合理 与相邻的时间段间隔是否处于合理区间30~900秒之间?" +
$"分段的主题内容,知识点分配是否合理符合实际吗?" +
$"请给出你的打分(0-100,70分及格)以及打分原因。" +
$"这是我的分段 {thems}。" +
$"后续的内容是包含时间戳的视频字幕的固定格式文本。" +
$"字幕格式(说话人:开始秒:结束秒:内容|下一段字幕).以下是包含时间的视频字幕文本。字幕列表 {captions.Captions}。" +
$"最后输出格式为json({checkResFormat})";
Console.WriteLine(DateTime.Now + "=>3.开始检查视频分段结果");
return await ChatAsync<CheckMessageDto>(taskInfo.Id.ToString(), checkMessage, null);
}
/// <summary>
/// 优化字幕
/// </summary>
/// <returns></returns>
private async Task<List<SenseVoiceRes>> 优化字幕(VideoTask taskInfo,
SenseVoiceRes[] captionsArr, string sections)
{
var subject = taskInfo.Subject.ToString();
var newCaptionsList = new List<SenseVoiceRes>(captionsArr.Length);
var spanCount = 50;
var totalCount = captionsArr.Length / spanCount + 1;
await Parallel.ForAsync(0, totalCount,
new ParallelOptions() { MaxDegreeOfParallelism = 4 },
async (s, c) =>
{
while (true)
{
var cArr = captionsArr
.Skip(spanCount * s)
.Take(spanCount);
if (cArr.Count() == 0)
return;
var nowCaptionStr = string.Join('|', cArr.Select(s => /*s.Start + ":" +*/ s.Text));
var resFormat = """[string(修改结果)]""";
var postMessages =
$"这是一堂中国课的字幕由结果是语音识别提供。" +
$"字幕内容与{subject}学科{sections}章节相关。" +
$"需要你帮我修复其中的错别字,修复公式。" +
$"请注意 只允许对字幕进行修改不允许删除字幕的字或者添加字。" +
$"请确保输出字幕条数与输入字幕条数一致!!!" +
$"输出内容只返回json格式({resFormat})" +
$"字幕内容(使用|符号分割)" +
$"`{nowCaptionStr}`" +
$"字幕结束。 ";
Console.WriteLine(DateTime.Now + $"=>字幕优化 分段{s}/{totalCount}开始...");
var resData = await ChatAsync<string[]>(taskInfo.Id.ToString(), postMessages, null, "deepseek-chat");
if (resData.Count() != cArr.Count())
{
Console.WriteLine(DateTime.Now + $"=>字幕优化 分段{s}/{totalCount} AI结果数量不匹配,重试");
continue;
}
newCaptionsList.AddRange(resData.Select((text, i) => new SenseVoiceRes()
{
Start = captionsArr[spanCount * s + i].Start,
End = captionsArr[spanCount * s + i].End,
Text = text,
}));
Console.WriteLine(DateTime.Now + $"=>字幕优化 分段{s}/{totalCount}完成√");
return;
}
});
Console.WriteLine(DateTime.Now + $"=>字幕优化执行完成");
var jsonData = JsonSerializer.Serialize(newCaptionsList.OrderBy(s=>s.Start));
await videoTaskDB.AsUpdateable()
.SetColumns(it => it.CaptionsAI == jsonData)
.Where(it => it.Id == taskInfo.Id)
.ExecuteCommandAsync();
return newCaptionsList;
}
/// <summary>
/// 视频AI分析字幕
/// </summary>
/// <returns></returns>
private async Task Analytics(List<VideoKnowRes> questionRes, VideoTask taskInfo,
SenseVoiceRes[] captionsArr, string sections)
{
var lastTime = 0;
var endTime = 0;
var maxVideoTime = captionsArr?.LastOrDefault()?.End ?? 0;
var subject = taskInfo.Subject.ToString();
var timeSpan = (int)(maxVideoTime * 0.5);
while (endTime + 60 < maxVideoTime)
{
try
{
endTime = lastTime + timeSpan;
var nowCaptionStr = string.Join('|', captionsArr
.Where(s => s.Text != "。")
.Where((s, i) => s.Start > lastTime && s.End < endTime)
.Select(s => s.Start + ":" + s.Text));
var keyFrameArr = string.IsNullOrEmpty(taskInfo?.PPTVideoCode)
? string.Empty
: $"视频授课内容发生了变化的时间节点{taskInfo.PPTKeyFrame},授课阶段应当在附近时间发生变化。";
var resFormat = """[{"StartTime":开始秒(number),"EndTime":结束秒(number),"Stage":阶段(string),"Theme":主题(string),"Content":内容总结(string)}]""";
var postMessages =
$"请通过视频字幕内容分析出视频中{subject}课堂的授课阶段。" +
$"课堂内容与{sections}章节相关。" +
$"{keyFrameArr}" +
$"完整的课堂标准流程包含以下5个阶段：课程引入/新知讲解/例题精讲/课堂练习/知识总结。" +
$"通过授课阶段的主要讲解内容分析出对应的授课阶段内容总结。" +
$"通过生成的内容总结分析出对应的授课阶段主题。 " +
$"请注意 本次分析的视频字幕只是其中一部分 不需要分析出所有类型的授课阶段。" +
$"最后请检查每个授课阶段的时长,不允许出现超出800秒或者低于50秒的授课阶段。" +
$"输出内容只返回json格式({resFormat})" +
$"字幕格式(开始秒:内容|下一段字幕).以下是包含时间的视频字幕文本。" +
$"字幕列表 {nowCaptionStr} 字幕结束!";
Console.WriteLine(DateTime.Now + $"=>{taskInfo.Id.ToString()}.开始分析视频内容 {lastTime}~{endTime}");
questionRes.AddRange(await ChatAsync<VideoKnowRes[]>(taskInfo.Id.ToString(), postMessages, null));
lastTime = (int)questionRes.Last().EndTime.Value - (lastTime == 0 ? 0 : 30);
}
catch (Exception ex)
{
Console.WriteLine(DateTime.Now + $"=>分析视频内容失败 {lastTime}~{endTime}");
Console.WriteLine(DateTime.Now + ex.Message);
Console.WriteLine(DateTime.Now + ex.StackTrace);
endTime = lastTime - timeSpan;
}
}
}
/// <summary>
/// 获取知识点
/// </summary>
@ -563,6 +704,8 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
var captionsArr = JsonSerializer.Deserialize<SenseVoiceRes[]>(taskInfo.Captions);
//处理视频授课章节
var sections = await GetSections(taskInfo, Course_Id);
//AI优化字幕
await 优化字幕(taskInfo, captionsArr, sections);
//合并字幕
var captions = ExpandFunction.GetSpeakerCaptions(captionsArr);
var maxVideoTime = captions?.TimeBase?.LastOrDefault()?.End ?? 0;
@ -571,64 +714,14 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
while (true)
{
questionRes = new List<VideoKnowRes>();
var lastTime = 0;
var endTime = 0;
var timeSpan =(int)(maxVideoTime * 0.5);
while (endTime + 60 < maxVideoTime)
{
try
{
endTime = lastTime + timeSpan;
var nowCaptionStr = string.Join('|', captionsArr
.Where(s => s.Text != "。")
.Where((s, i) => s.Start > lastTime && s.End < endTime)
.Select(s => s.Start + ":" + s.Text));
var keyFrameArr = string.IsNullOrEmpty(taskInfo?.PPTVideoCode)
?string.Empty
: $"视频授课内容发生了变化的时间节点{taskInfo.PPTKeyFrame},授课阶段应当在附近时间发生变化。" ;
var resFormat = """[{"StartTime":开始秒(number),"EndTime":结束秒(number),"Stage":阶段(string),"Theme":主题(string),"Content":内容总结(string)}]""";
var postMessages =
$"请通过视频字幕内容分析出视频中{subject}课堂的授课阶段。" +
$"课堂内容与{sections}章节相关。" +
$"{keyFrameArr}" +
$"完整的课堂标准流程包含以下5个阶段：课程引入/新知讲解/例题精讲/课堂练习/知识总结。" +
$"通过授课阶段的主要讲解内容分析出对应的授课阶段内容总结。" +
$"通过生成的内容总结分析出对应的授课阶段主题。 " +
$"请注意 本次分析的视频字幕只是其中一部分 不需要分析出所有类型的授课阶段。" +
$"最后请检查每个授课阶段的时长,不允许出现超出800秒或者低于50秒的授课阶段。" +
$"输出内容只返回json格式({resFormat})" +
$"字幕格式(开始秒:内容|下一段字幕).以下是包含时间的视频字幕文本。" +
$"字幕列表 {nowCaptionStr} 字幕结束!";
//视频字幕分析
await Analytics(questionRes, taskInfo, captionsArr, sections);
Console.WriteLine(DateTime.Now + $"=>1.开始分析视频内容 {lastTime}~{endTime}");
questionRes.AddRange(await ChatAsync<VideoKnowRes[]>(task, postMessages, null));
lastTime = (int)questionRes.Last().EndTime.Value - (lastTime == 0 ? 0 : 30);
}
catch (Exception ex)
{
Console.WriteLine(DateTime.Now + $"=>分析视频内容失败 {lastTime}~{endTime}");
Console.WriteLine(DateTime.Now + ex.Message);
Console.WriteLine(DateTime.Now + ex.StackTrace);
endTime = lastTime - timeSpan;
}
}
if (questionRes.Count == 0) continue;
//处理分段 知识点
var insertData = await GetVideoKnow(questionRes, taskInfo, sections, Course_Id);
//校验结果质量
var thems = JsonSerializer.Serialize(questionRes.Adapt<VideoKnowQueryDto[]>());
var checkResFormat = """{"Score":打分(number),"Evaluation":评价(string)""";//,"Data":优化后的分段(array)}""";
var checkMessage = "我为视频的讲解内容做了一些分段,希望你能通读字幕内容后检查下的分段是否符合我的要求?" +
$"检查这些分段的时间是否合理 与相邻的时间段间隔是否处于合理区间30~900秒之间?" +
$"分段的主题内容,知识点分配是否合理符合实际吗?" +
$"请给出你的打分(0-100,70分及格)以及打分原因。" +
$"这是我的分段 {thems}。" +
$"后续的内容是包含时间戳的视频字幕的固定格式文本。" +
$"字幕格式(说话人:开始秒:结束秒:内容|下一段字幕).以下是包含时间的视频字幕文本。字幕列表 {captions.Captions}。" +
$"最后输出格式为json({checkResFormat})";
Console.WriteLine(DateTime.Now + "=>3.开始检查视频分段结果");
var checkRes = await ChatAsync<CheckMessageDto>(task, checkMessage, null);
var checkRes = await VerifySpanQuality(questionRes, taskInfo, captions, sections, Course_Id);
if (checkRes != null && checkRes.Score >= 80)
{
@ -651,9 +744,6 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
}
await RedisExpand.Redis
.HMSetAsync(RedisExpandKey.Task(task), "VideoKnows", questionRes);
@ -677,12 +767,12 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
{
taskId = task,
model = model,
stream = model == "deepseek-reasoner",
stream = true,
temperature = 0.2f,
messages = messageArr
};
var tryCount = 10;
while (--tryCount >0)
while (--tryCount > 0)
{
try
{
@ -726,7 +816,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
Console.WriteLine(ex.Message);
}
}
throw new Exception(DateTime.Now+ "=>ChatGPT请求失败次数过多!!!");
throw new Exception(DateTime.Now + "=>ChatGPT请求失败次数过多!!!");
}
}
}

5
VideoAnalysisCore/Model/VideoTask.cs
View File

@ -84,6 +84,11 @@ namespace VideoAnalysisCore.Model
[SugarColumn(ColumnName = "Captions", ColumnDataType = "longtext", IsNullable = true)]
public string Captions { get; set; } = "[]";
/// <summary>
/// 字幕缓存[AI优化]
/// </summary>
[SugarColumn(ColumnName = "CaptionsAI", ColumnDataType = "longtext", IsNullable = true)]
public string CaptionsAI { get; set; } = "[]";
/// <summary>
/// 说话人日志解析缓存
/// </summary>
[SugarColumn(ColumnName = "Speaker", ColumnDataType = "longtext", IsNullable = true)]