marking
/
Learn.VideoAnalysis
19
0
Fork 0
Code Pull Requests Actions Releases Activity

优化 ai分析流程

This commit is contained in:
小肥羊 2025-05-09 18:14:33 +08:00
parent c616581918
commit 7cbd4588f5
5 changed files with 158 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeekClient.cs
View File

@ -24,14 +24,14 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
//public static string Host = AppCommon.Config.ChatGpt.aliyun.Host;
//public static string ApiKey = AppCommon.Config.ChatGpt.aliyun.ApiKey;
//private readonly string Path = "";
//public static string Host = AppCommon.Config.ChatGpt.DeepSeek.Host;
//public static string ApiKey = AppCommon.Config.ChatGpt.DeepSeek.ApiKey;
private readonly string Path = "";
public static string Host = AppCommon.Config.ChatGpt.DeepSeek.Host;
public static string ApiKey = AppCommon.Config.ChatGpt.DeepSeek.ApiKey;
public static string Host = AppCommon.Config.ChatGpt.ChatGpt.Host;
public static string ApiKey = AppCommon.Config.ChatGpt.ChatGpt.ApiKey;
private readonly string Path = "v1/chat/completions";
//public static string Host = AppCommon.Config.ChatGpt.ChatGpt.Host;
//public static string ApiKey = AppCommon.Config.ChatGpt.ChatGpt.ApiKey;
//private readonly string Path = "v1/chat/completions";
private readonly IHttpClientFactory _httpClientFactory;

221
VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs
View File

@ -13,6 +13,8 @@ using VideoAnalysisCore.Model.蓝鲸智库;
using VideoAnalysisCore.Model.Enum;
using Mapster;
using System.Linq;
using System.Security.Cryptography;
using static System.Collections.Specialized.BitVector32;
namespace VideoAnalysisCore.AICore.GPT.DeepSeek
{
@ -432,38 +434,82 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
RedisExpand.InsertChannel(RedisChannelEnum.EndTask, task);
return gptRes;
}
/// <summary>
/// 获取知识点
/// 获取内容对应的章节
/// </summary>
/// <param name="task">任务id</param>
/// <returns></returns>
public async Task<TaskRes> GetKnow(string task)
public async Task<List<VideoKonwPoint>> GetVideoKnow(List<VideoKnowRes> questionRes,VideoTask taskInfo, string sections, int course_Id)
{
var know = await knowledgeInfoDB.GetFirstAsync(s => s.Course_Id == course_Id && s.Name == sections);
if (know is null)
throw new Exception("未能找到对应知识点=>" + sections);
var subject = taskInfo.Subject.ToString();
var kInfo = await knowledgeInfoDB.GetByIdAsync(know.Parent_Id);
var knowledgeInfos = await knowledgeInfoDB.AsQueryable()
.ToChildListAsync(s => s.Parent_Id, kInfo.Parent_Id == 0 ? kInfo.Id : kInfo.Parent_Id);
var knows = string.Join(',', knowledgeInfos.Select(s => s.Id + "|" + s.Name));
var knowDic = knowledgeInfos
.OrderBy(s => s.Id)
.GroupBy(s => s.Name)
.ToDictionary(s => s.First().Name, s => s.First().Id);
questionRes = questionRes.Where(s => s != null).OrderBy(s => s.StartTime).ToList();
var thems = JsonSerializer.Serialize(questionRes.Adapt<VideoKnowQueryDto[]>());// string.Join(',', questionRes.Select(s => s.StartTime + "->" + s.Theme));
var checkResFormat1 = """[{"StartTime":开始秒(number),"KnowPoint":知识点名称(string),"KnowPointId":知识点Id(string)}]""";
var knowMessages =
$"我针对{subject}课堂授课视频分析出了视频的授课阶段片段。" +
$"现在需要你通过每个片段的内容总结来分配正确的知识点(单个片段允许多个知识点用逗号','分割)。" +
$"这是我的分段 {thems}。" +
$"课堂内容与{sections}章节相关" +
$"最后请确保分配的知识点是用户提供的,并且一定正确合理!" +
$"输出内容只返回json格式({checkResFormat1})" +
$" 格式 (方法点Id|方法点名称) " +
$"提供的知识点名称({knows})。";
Console.WriteLine(DateTime.Now + "=>2.开始分析视频内容知识点");
var konwRes = await ChatAsync<VideoKnowRes[]>(taskInfo.Id.ToString(), knowMessages, null);
for (int i = 0; i < konwRes.Count(); i++)
questionRes[i].KnowPoint = konwRes[i].KnowPoint;
//todo 未包含的知识点片段 如何处理
return questionRes
.Where(s => !string.IsNullOrEmpty(s.KnowPoint))
.SelectMany(
s =>
{
var ks = s.KnowPoint.Split(",").Distinct();
return ks.Where(x => knowDic.ContainsKey(x))
.Select(x => new VideoKonwPoint()
{
Content = s.Content,
Theme = s.Theme,
StartTime = s.StartTime,
EndTime = s.EndTime,
KnowPoint = x,
KnowPointId = knowDic[x].ToString(),
TagId = taskInfo.TagId,
VideoTaskId = taskInfo.Id,
});
}).ToList();
}
/// <summary>
/// 获取内容对应的章节
/// </summary>
/// <returns></returns>
public async Task<string?> GetSections(VideoTask taskInfo,int course_Id)
{
var taskId = long.Parse(task);
var taskInfo = await videoTaskDB.AsQueryable()
.Where(s => s.Id == taskId)
.FirstAsync();
var subject = "数学";
var Course_Id = 27;
switch (taskInfo.Type)//处理不同任务类型的知识点树
{
case TaskTypeEnum.蓝鲸智库_中职视频分段:
Course_Id = 51;
break;
case TaskTypeEnum.蓝鲸智库_视频分段:
default:
Course_Id = 27;
break;
}
var xkwKnows = await knowledgeInfoDB.AsQueryable()
.Where(s => s.Course_Id == Course_Id
&& (s.Depth == 3
|| s.Depth == 2))
.Select(s => s.Name).ToArrayAsync();
.Where(s => s.Course_Id == course_Id
&& (s.Depth == 3
|| s.Depth == 2))
.Select(s => s.Name).ToArrayAsync();
string title = taskInfo.MediaName;
var speakerArr = JsonSerializer.Deserialize<OfflineSpeakerRes[]>(taskInfo.Speaker);
var captionsArr = JsonSerializer.Deserialize<SenseVoiceRes[]>(taskInfo.Captions);
var fileNameResFormat = "{授课章节: string|null}";
@ -478,29 +524,49 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
$"以下是包含时间的视频字幕文本。" +
$"字幕列表 {rCaptionArr}。" +
$"输出格式 json字符串 对象格式{fileNameResFormat}";
var task = taskInfo.Id.ToString();
var fileNameInfoRes = await ChatAsync<FileNameInfo>
(task, fileNamePostMessages, null);
#if DEBUG
//fileNameInfoRes = new FileNameInfo() { 授课章节 = "一元二次不等式" };
#endif
var captions = ExpandFunction.GetSpeakerCaptions(captionsArr, speakerArr);
var maxVideoTime = captions?.TimeBase?.LastOrDefault()?.End ?? 0;
var criteriaBuilder = new StringBuilder();
var know = await knowledgeInfoDB.GetFirstAsync(s => s.Course_Id == Course_Id && s.Name == fileNameInfoRes.授课章节);
if (know is null)
throw new Exception("未能找到对应知识点=>" + fileNameInfoRes.授课章节);
await videoTaskDB.AsUpdateable()
.SetColumns(it => it.Sections == fileNameInfoRes.授课章节)
.Where(it => it.Id == taskInfo.Id)
.ExecuteCommandAsync();
await RedisExpand.Redis
.HMSetAsync(RedisExpandKey.Task(task), "学科章节", fileNameInfoRes.授课章节);
//提升到父级
var kInfo = await knowledgeInfoDB.GetByIdAsync(know.Parent_Id);
var knowledgeInfos = await knowledgeInfoDB.AsQueryable().ToChildListAsync(s => s.Parent_Id, kInfo.Parent_Id == 0 ? kInfo.Id : kInfo.Parent_Id);
var knows = string.Join(',', knowledgeInfos.Select(s => s.Id + "|" + s.Name));
var knowDic = knowledgeInfos
.OrderBy(s => s.Id)
.GroupBy(s => s.Name)
.ToDictionary(s => s.First().Name, s => s.First().Id);
return fileNameInfoRes?.授课章节;
}
/// <summary>
/// 获取知识点
/// </summary>
/// <param name="task">任务id</param>
/// <returns></returns>
public async Task<TaskRes> GetKnow(string task)
{
var taskId = long.Parse(task);
var taskInfo = await videoTaskDB.AsQueryable()
.Where(s => s.Id == taskId)
.FirstAsync();
var subject = taskInfo.Subject.ToString();
var Course_Id = 27;
switch (taskInfo.Type)//处理不同任务类型的知识点树
{
case TaskTypeEnum.蓝鲸智库_中职视频分段:
Course_Id = 51;
break;
case TaskTypeEnum.蓝鲸智库_视频分段:
default:
Course_Id = 27;
break;
}
var captionsArr = JsonSerializer.Deserialize<SenseVoiceRes[]>(taskInfo.Captions);
//处理视频授课章节
var sections = await GetSections(taskInfo, Course_Id);
//合并字幕
var captions = ExpandFunction.GetSpeakerCaptions(captionsArr);
var maxVideoTime = captions?.TimeBase?.LastOrDefault()?.End ?? 0;
var questionRes = new List<VideoKnowRes>();
while (true)
{
@ -523,7 +589,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
var resFormat = """[{"StartTime":开始秒(number),"EndTime":结束秒(number),"Stage":阶段(string),"Theme":主题(string),"Content":内容总结(string)}]""";
var postMessages =
$"请通过视频字幕内容分析出视频中{subject}课堂的授课阶段。" +
$"课堂内容与{fileNameInfoRes.授课章节}章节相关。" +
$"课堂内容与{sections}章节相关。" +
$"{keyFrameArr}" +
$"完整的课堂标准流程包含以下5个阶段：课程引入/新知讲解/例题精讲/课堂练习/知识总结。" +
$"通过授课阶段的主要讲解内容分析出对应的授课阶段内容总结。" +
@ -548,36 +614,10 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
}
}
if (questionRes.Count == 0) continue;
questionRes = questionRes.OrderBy(s => s.StartTime).ToList();
var thems = JsonSerializer.Serialize(questionRes.Adapt<VideoKnowQueryDto[]>());// string.Join(',', questionRes.Select(s => s.StartTime + "->" + s.Theme));
var checkResFormat1 = """[{"StartTime":开始秒(number),"KnowPoint":知识点名称(string),"KnowPointId":知识点Id(string)}]""";
var knowMessages =
$"我针对{subject}课堂授课视频分析出了视频的授课阶段片段。" +
$"现在需要你通过每个片段的内容总结来分配正确的知识点(单个片段允许多个知识点用逗号','分割)。" +
$"这是我的分段 {thems}。" +
$"课堂内容与{fileNameInfoRes.授课章节}章节相关" +
$"最后请确保分配的知识点是用户提供的,并且一定正确合理!" +
$"输出内容只返回json格式({checkResFormat1})" +
$" 格式 (方法点Id|方法点名称) " +
$"提供的知识点名称({knows})。";
Console.WriteLine(DateTime.Now + "=>2.开始分析视频内容知识点");
var konwRes = await ChatAsync<VideoKnowRes[]>(task, knowMessages, null);
for (int i = 0; i < konwRes.Count(); i++)
questionRes[i].KnowPoint = konwRes[i].KnowPoint;
VideoKnowRes lastVideoKnow = null;
for (int i = 0; i < questionRes.Count(); i++)
{
var item = questionRes[i];
// 阶段类型相等,且范围包含上一阶段
if (lastVideoKnow != null && lastVideoKnow.Stage == item.Stage && item.StartTime < lastVideoKnow.EndTime)
{
questionRes[i] = null;
lastVideoKnow.EndTime= item.EndTime;
}
lastVideoKnow = questionRes[i];
}
thems = JsonSerializer.Serialize(questionRes.Adapt<VideoKnowQueryDto[]>());
//处理分段 知识点
var insertData = await GetVideoKnow(questionRes, taskInfo, sections, Course_Id);
//校验结果质量
var thems = JsonSerializer.Serialize(questionRes.Adapt<VideoKnowQueryDto[]>());
var checkResFormat = """{"Score":打分(number),"Evaluation":评价(string)""";//,"Data":优化后的分段(array)}""";
var checkMessage = "我为视频的讲解内容做了一些分段,希望你能通读字幕内容后检查下的分段是否符合我的要求?" +
$"检查这些分段的时间是否合理 与相邻的时间段间隔是否处于合理区间30~900秒之间?" +
@ -587,11 +627,14 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
$"后续的内容是包含时间戳的视频字幕的固定格式文本。" +
$"字幕格式(说话人:开始秒:结束秒:内容|下一段字幕).以下是包含时间的视频字幕文本。字幕列表 {captions.Captions}。" +
$"最后输出格式为json({checkResFormat})";
Console.WriteLine(DateTime.Now + "=>3.开始检查视频分段结果");
var checkRes = await ChatAsync<CheckMessageDto>(task, checkMessage, null);
if (checkRes != null && checkRes.Score >= 80)
{
//写入知识点
await videoKonwPointDB.DeleteAsync(s => s.VideoTaskId == taskInfo.Id);
await videoKonwPointDB.InsertRangeAsync(insertData);
break;
}
else
@ -608,29 +651,6 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
}
//todo 未包含的知识点片段 如何处理
var insertData = questionRes
.Where(s => !string.IsNullOrEmpty(s.KnowPoint))
.SelectMany(
s =>
{
var ks = s.KnowPoint.Split(",").Distinct();
return ks.Where(x => knowDic.ContainsKey(x))
.Select(x => new VideoKonwPoint()
{
Content = s.Content,
Theme = s.Theme,
StartTime = s.StartTime,
EndTime = s.EndTime,
KnowPoint = x,
KnowPointId = knowDic[x].ToString(),
TagId = taskInfo.TagId,
VideoTaskId = taskInfo.Id,
});
}).ToList();
await videoKonwPointDB.DeleteAsync(s => s.VideoTaskId == taskId);
await videoKonwPointDB.InsertRangeAsync(insertData);
@ -640,6 +660,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
var gptRes = new TaskRes(captions);
await RedisExpand.Redis
.HMSetAsync(RedisExpandKey.Task(task), "ChatAnalysis", gptRes);
RedisExpand.InsertChannel(RedisChannelEnum.EndTask, task);
return gptRes;
}

2
VideoAnalysisCore/Common/AppCommon.cs
View File

@ -178,7 +178,7 @@ namespace VideoAnalysisCore.Common
/// <summary>
/// 获取Task处理后的 说话人字幕
/// </summary>
public static TotalCaptionsDto GetSpeakerCaptions(SenseVoiceRes[] captionsArr, OfflineSpeakerRes[] speakerArr)
public static TotalCaptionsDto GetSpeakerCaptions(SenseVoiceRes[] captionsArr, OfflineSpeakerRes[]? speakerArr=null)
{
if (captionsArr is null || captionsArr.Length == 0)
//|| speakerArr is null || speakerArr.Length == 0)

41
VideoAnalysisCore/Job/NodePackageJob.cs
View File

@ -62,31 +62,36 @@ namespace VideoAnalysisCore.Job
return;
//var responseMessage = await new HttpClient()
// .PostAsJsonAsync(AppCommon.Config.Subsystem.蓝鲸智库.APIUrl + "/api/callback/platform/videosAnalyze", postData);
foreach (var item in taskArr) {
foreach (var item in taskArr)
{
HttpResponseMessage responseMessage = null;
if (!string.IsNullOrWhiteSpace(item.CallBackUrl))
{
responseMessage = await new HttpClient()
.PostAsJsonAsync(item.CallBackUrl, postData);
}
else
try
{
var postUrl = !string.IsNullOrWhiteSpace(item.CallBackUrl)
? item.CallBackUrl
: AppCommon.Config.Subsystem.蓝鲸智库.APIUrl + "/api/callback/platform/videosAnalyze";
responseMessage = await new HttpClient()
.PostAsJsonAsync(AppCommon.Config.Subsystem.蓝鲸智库.APIUrl + "/api/callback/platform/videosAnalyze", postData);
}
if (responseMessage.IsSuccessStatusCode)
{
var res = await responseMessage.Content.ReadAsStringAsync();
Console.WriteLine($"{DateTime.Now} 执行=>文件包任务 回调结果 {res}");
.PostAsJsonAsync(postUrl, postData);
if (responseMessage.IsSuccessStatusCode)
{
var res = await responseMessage.Content.ReadAsStringAsync();
Console.WriteLine($"{DateTime.Now} 执行=>文件包任务 回调结果 {res}");
await nodePackageInfoDB.AsUpdateable(postData)
.UpdateColumns(it => new { it.SuccessTime })
.ExecuteCommandAsync();
await nodePackageInfoDB.AsUpdateable(postData)
.UpdateColumns(it => new { it.SuccessTime })
.ExecuteCommandAsync();
}
else
{
var res = await responseMessage.Content.ReadAsStringAsync();
Console.WriteLine($"{DateTime.Now} 执行=>文件包任务 回调失败!!! {responseMessage.StatusCode} {res}");
}
}
else
catch (Exception ex)
{
var res = await responseMessage.Content.ReadAsStringAsync();
var res = await responseMessage?.Content?.ReadAsStringAsync()??string.Empty;
Console.WriteLine($"{DateTime.Now} 执行=>文件包任务 回调失败!!! {responseMessage.StatusCode} {res}");
Console.WriteLine(ex);
}
await Task.Delay(1000);
}

8
VideoAnalysisCore/Model/VideoTask.cs
View File

@ -71,7 +71,7 @@ namespace VideoAnalysisCore.Model
/// <summary>
/// 授课视频对应PPT视频ID
/// </summary>
[SugarColumn(Length = 50, ColumnDataType = "varchar", IsNullable = true)]
[SugarColumn(Length = 255, ColumnDataType = "varchar", IsNullable = true)]
public string? PPTVideoCode { get; set; }
/// <summary>
/// 授课视频对应PPT视频关键帧
@ -117,5 +117,11 @@ namespace VideoAnalysisCore.Model
[SugarColumn( IsNullable = true)]
public DateTime? EndTime { get; set; }
/// <summary>
/// 授课章节
/// </summary>
[SugarColumn(IsNullable = true)]
public string? Sections { get; set; }
}
}