marking
/
Learn.VideoAnalysis
19
0
Fork 0
Code Pull Requests Actions Releases Activity

新增 AI优化字幕 

优化 获取片段流程
This commit is contained in:
小肥羊 2025-05-12 18:14:39 +08:00
parent 7cbd4588f5
commit 61af4e9827
2 changed files with 163 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

182
VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs
View File

@ -438,7 +438,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
/// 获取内容对应的章节
/// </summary>
/// <returns></returns>
public async Task<List<VideoKonwPoint>> GetVideoKnow(List<VideoKnowRes> questionRes,VideoTask taskInfo, string sections, int course_Id)
private async Task<List<VideoKonwPoint>> GetVideoKnow(List<VideoKnowRes> questionRes, VideoTask taskInfo, string sections, int course_Id)
{
var know = await knowledgeInfoDB.GetFirstAsync(s => s.Course_Id == course_Id && s.Name == sections);
@ -502,8 +502,11 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
/// 获取内容对应的章节
/// </summary>
/// <returns></returns>
public async Task<string?> GetSections(VideoTask taskInfo,int course_Id)
private async Task<string?> GetSections(VideoTask taskInfo, int course_Id)
{
if (!string.IsNullOrEmpty(taskInfo.Sections))
return taskInfo.Sections;
var xkwKnows = await knowledgeInfoDB.AsQueryable()
.Where(s => s.Course_Id == course_Id
&& (s.Depth == 3
@ -527,6 +530,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
var task = taskInfo.Id.ToString();
var fileNameInfoRes = await ChatAsync<FileNameInfo>
(task, fileNamePostMessages, null);
taskInfo.Sections = fileNameInfoRes.授课章节;
await videoTaskDB.AsUpdateable()
.SetColumns(it => it.Sections == fileNameInfoRes.授课章节)
.Where(it => it.Id == taskInfo.Id)
@ -536,43 +540,101 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
return fileNameInfoRes?.授课章节;
}
/// <summary>
/// 获取知识点
/// 检查AI切片结果质量
/// </summary>
/// <param name="task">任务id</param>
/// <returns></returns>
public async Task<TaskRes> GetKnow(string task)
private async Task<CheckMessageDto> VerifySpanQuality(List<VideoKnowRes> questionRes, VideoTask taskInfo, TotalCaptionsDto captions, string sections, int course_Id)
{
var taskId = long.Parse(task);
var taskInfo = await videoTaskDB.AsQueryable()
.Where(s => s.Id == taskId)
.FirstAsync();
var subject = taskInfo.Subject.ToString();
var Course_Id = 27;
switch (taskInfo.Type)//处理不同任务类型的知识点树
{
case TaskTypeEnum.蓝鲸智库_中职视频分段:
Course_Id = 51;
break;
case TaskTypeEnum.蓝鲸智库_视频分段:
default:
Course_Id = 27;
break;
//校验结果质量
var thems = JsonSerializer.Serialize(questionRes.Adapt<VideoKnowQueryDto[]>());
var checkResFormat = """{"Score":打分(number),"Evaluation":评价(string)""";//,"Data":优化后的分段(array)}""";
var checkMessage = "我为视频的讲解内容做了一些分段,希望你能通读字幕内容后检查下的分段是否符合我的要求?" +
$"检查这些分段的时间是否合理 与相邻的时间段间隔是否处于合理区间30~900秒之间?" +
$"分段的主题内容,知识点分配是否合理符合实际吗?" +
$"请给出你的打分(0-100,70分及格)以及打分原因。" +
$"这是我的分段 {thems}。" +
$"后续的内容是包含时间戳的视频字幕的固定格式文本。" +
$"字幕格式(说话人:开始秒:结束秒:内容|下一段字幕).以下是包含时间的视频字幕文本。字幕列表 {captions.Captions}。" +
$"最后输出格式为json({checkResFormat})";
Console.WriteLine(DateTime.Now + "=>3.开始检查视频分段结果");
return await ChatAsync<CheckMessageDto>(taskInfo.Id.ToString(), checkMessage, null);
}
var captionsArr = JsonSerializer.Deserialize<SenseVoiceRes[]>(taskInfo.Captions);
//处理视频授课章节
var sections = await GetSections(taskInfo, Course_Id);
//合并字幕
var captions = ExpandFunction.GetSpeakerCaptions(captionsArr);
var maxVideoTime = captions?.TimeBase?.LastOrDefault()?.End ?? 0;
/// <summary>
/// 优化字幕
/// </summary>
/// <returns></returns>
private async Task<List<SenseVoiceRes>> 优化字幕(VideoTask taskInfo,
SenseVoiceRes[] captionsArr, string sections)
{
var subject = taskInfo.Subject.ToString();
var newCaptionsList = new List<SenseVoiceRes>(captionsArr.Length);
var spanCount = 50;
var totalCount = captionsArr.Length / spanCount + 1;
var questionRes = new List<VideoKnowRes>();
await Parallel.ForAsync(0, totalCount,
new ParallelOptions() { MaxDegreeOfParallelism = 4 },
async (s, c) =>
{
while (true)
{
questionRes = new List<VideoKnowRes>();
var cArr = captionsArr
.Skip(spanCount * s)
.Take(spanCount);
if (cArr.Count() == 0)
return;
var nowCaptionStr = string.Join('|', cArr.Select(s => /*s.Start + ":" +*/ s.Text));
var resFormat = """[string(修改结果)]""";
var postMessages =
$"这是一堂中国课的字幕由结果是语音识别提供。" +
$"字幕内容与{subject}学科{sections}章节相关。" +
$"需要你帮我修复其中的错别字,修复公式。" +
$"请注意 只允许对字幕进行修改不允许删除字幕的字或者添加字。" +
$"请确保输出字幕条数与输入字幕条数一致!!!" +
$"输出内容只返回json格式({resFormat})" +
$"字幕内容(使用|符号分割)" +
$"`{nowCaptionStr}`" +
$"字幕结束。 ";
Console.WriteLine(DateTime.Now + $"=>字幕优化 分段{s}/{totalCount}开始...");
var resData = await ChatAsync<string[]>(taskInfo.Id.ToString(), postMessages, null, "deepseek-chat");
if (resData.Count() != cArr.Count())
{
Console.WriteLine(DateTime.Now + $"=>字幕优化 分段{s}/{totalCount} AI结果数量不匹配,重试");
continue;
}
newCaptionsList.AddRange(resData.Select((text, i) => new SenseVoiceRes()
{
Start = captionsArr[spanCount * s + i].Start,
End = captionsArr[spanCount * s + i].End,
Text = text,
}));
Console.WriteLine(DateTime.Now + $"=>字幕优化 分段{s}/{totalCount}完成√");
return;
}
});
Console.WriteLine(DateTime.Now + $"=>字幕优化执行完成");
var jsonData = JsonSerializer.Serialize(newCaptionsList.OrderBy(s=>s.Start));
await videoTaskDB.AsUpdateable()
.SetColumns(it => it.CaptionsAI == jsonData)
.Where(it => it.Id == taskInfo.Id)
.ExecuteCommandAsync();
return newCaptionsList;
}
/// <summary>
/// 视频AI分析字幕
/// </summary>
/// <returns></returns>
private async Task Analytics(List<VideoKnowRes> questionRes, VideoTask taskInfo,
SenseVoiceRes[] captionsArr, string sections)
{
var lastTime = 0;
var endTime = 0;
var maxVideoTime = captionsArr?.LastOrDefault()?.End ?? 0;
var subject = taskInfo.Subject.ToString();
var timeSpan = (int)(maxVideoTime * 0.5);
while (endTime + 60 < maxVideoTime)
{
@ -600,8 +662,8 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
$"字幕格式(开始秒:内容|下一段字幕).以下是包含时间的视频字幕文本。" +
$"字幕列表 {nowCaptionStr} 字幕结束!";
Console.WriteLine(DateTime.Now + $"=>1.开始分析视频内容 {lastTime}~{endTime}");
questionRes.AddRange(await ChatAsync<VideoKnowRes[]>(task, postMessages, null));
Console.WriteLine(DateTime.Now + $"=>{taskInfo.Id.ToString()}.开始分析视频内容 {lastTime}~{endTime}");
questionRes.AddRange(await ChatAsync<VideoKnowRes[]>(taskInfo.Id.ToString(), postMessages, null));
lastTime = (int)questionRes.Last().EndTime.Value - (lastTime == 0 ? 0 : 30);
}
catch (Exception ex)
@ -613,22 +675,53 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
}
}
}
/// <summary>
/// 获取知识点
/// </summary>
/// <param name="task">任务id</param>
/// <returns></returns>
public async Task<TaskRes> GetKnow(string task)
{
var taskId = long.Parse(task);
var taskInfo = await videoTaskDB.AsQueryable()
.Where(s => s.Id == taskId)
.FirstAsync();
var subject = taskInfo.Subject.ToString();
var Course_Id = 27;
switch (taskInfo.Type)//处理不同任务类型的知识点树
{
case TaskTypeEnum.蓝鲸智库_中职视频分段:
Course_Id = 51;
break;
case TaskTypeEnum.蓝鲸智库_视频分段:
default:
Course_Id = 27;
break;
}
var captionsArr = JsonSerializer.Deserialize<SenseVoiceRes[]>(taskInfo.Captions);
//处理视频授课章节
var sections = await GetSections(taskInfo, Course_Id);
//AI优化字幕
await 优化字幕(taskInfo, captionsArr, sections);
//合并字幕
var captions = ExpandFunction.GetSpeakerCaptions(captionsArr);
var maxVideoTime = captions?.TimeBase?.LastOrDefault()?.End ?? 0;
var questionRes = new List<VideoKnowRes>();
while (true)
{
questionRes = new List<VideoKnowRes>();
//视频字幕分析
await Analytics(questionRes, taskInfo, captionsArr, sections);
if (questionRes.Count == 0) continue;
//处理分段 知识点
var insertData = await GetVideoKnow(questionRes, taskInfo, sections, Course_Id);
//校验结果质量
var thems = JsonSerializer.Serialize(questionRes.Adapt<VideoKnowQueryDto[]>());
var checkResFormat = """{"Score":打分(number),"Evaluation":评价(string)""";//,"Data":优化后的分段(array)}""";
var checkMessage = "我为视频的讲解内容做了一些分段,希望你能通读字幕内容后检查下的分段是否符合我的要求?" +
$"检查这些分段的时间是否合理 与相邻的时间段间隔是否处于合理区间30~900秒之间?" +
$"分段的主题内容,知识点分配是否合理符合实际吗?" +
$"请给出你的打分(0-100,70分及格)以及打分原因。" +
$"这是我的分段 {thems}。" +
$"后续的内容是包含时间戳的视频字幕的固定格式文本。" +
$"字幕格式(说话人:开始秒:结束秒:内容|下一段字幕).以下是包含时间的视频字幕文本。字幕列表 {captions.Captions}。" +
$"最后输出格式为json({checkResFormat})";
Console.WriteLine(DateTime.Now + "=>3.开始检查视频分段结果");
var checkRes = await ChatAsync<CheckMessageDto>(task, checkMessage, null);
var checkRes = await VerifySpanQuality(questionRes, taskInfo, captions, sections, Course_Id);
if (checkRes != null && checkRes.Score >= 80)
{
@ -651,9 +744,6 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
}
await RedisExpand.Redis
.HMSetAsync(RedisExpandKey.Task(task), "VideoKnows", questionRes);
@ -677,7 +767,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
{
taskId = task,
model = model,
stream = model == "deepseek-reasoner",
stream = true,
temperature = 0.2f,
messages = messageArr
};

5
VideoAnalysisCore/Model/VideoTask.cs
View File

@ -84,6 +84,11 @@ namespace VideoAnalysisCore.Model
[SugarColumn(ColumnName = "Captions", ColumnDataType = "longtext", IsNullable = true)]
public string Captions { get; set; } = "[]";
/// <summary>
/// 字幕缓存[AI优化]
/// </summary>
[SugarColumn(ColumnName = "CaptionsAI", ColumnDataType = "longtext", IsNullable = true)]
public string CaptionsAI { get; set; } = "[]";
/// <summary>
/// 说话人日志解析缓存
/// </summary>
[SugarColumn(ColumnName = "Speaker", ColumnDataType = "longtext", IsNullable = true)]