marking
/
Learn.VideoAnalysis
19
0
Fork 0
Code Pull Requests Actions Releases Activity

新增 AI优化字幕流程 

优化 AI分析流程
This commit is contained in:
小肥羊 2025-05-14 09:53:07 +08:00
parent 61af4e9827
commit 7da037806f
7 changed files with 48 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
VideoAnalysis/Components/Pages/VideoTaskShow.razor
View File

@ -21,6 +21,11 @@
<source type="video/mp4" />
</video>
<div id="subtitleArea" class="subtitles"></div>
<div id="subtitleArea1" class="subtitles" style="
bottom: 101px;
background-color: rgb(99 129 103 / 50%);
"></div>
</div>
@ -28,16 +33,19 @@
<script>
window.b = []
window.subtitles = []
window.subtitles1 = []
var displayButton = []
var lastSegments = null;
function init() {
const videoPlayer = document.getElementById('videoPlayer');
const subtitleArea = document.getElementById('subtitleArea');
const subtitleArea1 = document.getElementById('subtitleArea1');
//视频时间变化时
videoPlayer.addEventListener('timeupdate', function () {
if (displayButton.length == 0) initKD()
const currentTime = videoPlayer.currentTime;
subtitleArea.textContent = '';
subtitleArea1.textContent = '';
//字幕
window.subtitles.forEach(subtitle => {
let textContent = subtitle.text
@ -45,6 +53,7 @@
&& currentTime <= subtitle.end
&& subtitleArea.textContent != textContent) {
subtitleArea.textContent = textContent;
subtitleArea1.textContent = window.subtitles1[window.subtitles.indexOf(subtitle)].text;
}
});
//时间片
@ -62,9 +71,10 @@
displayButton = window.b.map((s, i) => { return { ...s, button: btns[i] } })
}
//后端传递初始化数据
function setDB(a, b,c) {
console.log("setDB", a, b,c)
function setDB(a,a1, b,c) {
console.log("setDB", a1,a, b, c)
window.subtitles = a
window.subtitles1 = a1
window.b = b
const videoPlayer = document.getElementById('videoPlayer');
videoPlayer.src = c

9
VideoAnalysis/Components/Pages/VideoTaskShow.razor.cs
View File

@ -35,10 +35,6 @@ namespace Learn.VideoAnalysis.Components.Pages
private VideoTask nowTask { get; set; } = default!;
private string videoPath { get; set; } = default!;
/// <summary>
/// 字幕
/// </summary>
private SenseVoiceRes[] captionsArr { get; set; } = default!;
/// <summary>
/// 分段
/// </summary>
@ -76,7 +72,8 @@ namespace Learn.VideoAnalysis.Components.Pages
nowTask = await taskDB.GetFirstAsync(s => s.Id == taskId);
if (nowTask is null)
return;
captionsArr = JsonSerializer.Deserialize<SenseVoiceRes[]>(nowTask.Captions);
var captionsArr = JsonSerializer.Deserialize<SenseVoiceRes[]>(nowTask.Captions);
var captionsArr1 = JsonSerializer.Deserialize<SenseVoiceRes[]>(nowTask.CaptionsAI??"[]") ;
RedisExpand.Redis.HMGet<SenseVoiceRes[]>(RedisExpandKey.Task(taskId), "Captions").FirstOrDefault();
var konwArr = await videoKonwPointDB.AsQueryable()
@ -94,7 +91,7 @@ namespace Learn.VideoAnalysis.Components.Pages
KnowPoint = string.Join(',', s.Select(x => x.KnowPoint))
}).ToArray();
videoPath = AppCommon.GetVideoPath(nowTask.Id.ToString());
await JSRuntime.InvokeVoidAsync("setDB", captionsArr, videoKnows, videoPath);
await JSRuntime.InvokeVoidAsync("setDB", captionsArr, captionsArr1, videoKnows, videoPath);
StateHasChanged();
}

23
VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs
View File

@ -455,7 +455,8 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
.OrderBy(s => s.Id)
.GroupBy(s => s.Name)
.ToDictionary(s => s.First().Name, s => s.First().Id);
questionRes = questionRes.Where(s => s != null).OrderBy(s => s.StartTime).ToList();
questionRes = questionRes.Where(s => s != null)
.OrderBy(s => s.StartTime).ToList();
var thems = JsonSerializer.Serialize(questionRes.Adapt<VideoKnowQueryDto[]>());// string.Join(',', questionRes.Select(s => s.StartTime + "->" + s.Theme));
var checkResFormat1 = """[{"StartTime":开始秒(number),"KnowPoint":知识点名称(string),"KnowPointId":知识点Id(string)}]""";
var knowMessages =
@ -493,6 +494,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
KnowPointId = knowDic[x].ToString(),
TagId = taskInfo.TagId,
VideoTaskId = taskInfo.Id,
Stage =s?.Stage?.ToEnum<StageEnum>()
});
}).ToList();
@ -568,16 +570,18 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
/// 优化字幕
/// </summary>
/// <returns></returns>
private async Task<List<SenseVoiceRes>> 优化字幕(VideoTask taskInfo,
private async Task<SenseVoiceRes[]> 优化字幕(VideoTask taskInfo,
SenseVoiceRes[] captionsArr, string sections)
{
if (!string.IsNullOrEmpty(taskInfo.CaptionsAI))
return JsonSerializer.Deserialize<SenseVoiceRes[]>(taskInfo.CaptionsAI);
var subject = taskInfo.Subject.ToString();
var newCaptionsList = new List<SenseVoiceRes>(captionsArr.Length);
var spanCount = 50;
var totalCount = captionsArr.Length / spanCount + 1;
await Parallel.ForAsync(0, totalCount,
new ParallelOptions() { MaxDegreeOfParallelism = 4 },
new ParallelOptions() { MaxDegreeOfParallelism =10 },
async (s, c) =>
{
while (true)
@ -616,13 +620,14 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
return;
}
});
var res = newCaptionsList.OrderBy(s => s.Start).ToArray();
Console.WriteLine(DateTime.Now + $"=>字幕优化执行完成");
var jsonData = JsonSerializer.Serialize(newCaptionsList.OrderBy(s=>s.Start));
var jsonData = JsonSerializer.Serialize(res);
await videoTaskDB.AsUpdateable()
.SetColumns(it => it.CaptionsAI == jsonData)
.Where(it => it.Id == taskInfo.Id)
.ExecuteCommandAsync();
return newCaptionsList;
return res;
}
/// <summary>
/// 视频AI分析字幕
@ -705,7 +710,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
//处理视频授课章节
var sections = await GetSections(taskInfo, Course_Id);
//AI优化字幕
await 优化字幕(taskInfo, captionsArr, sections);
captionsArr = await 优化字幕(taskInfo, captionsArr, sections);
//合并字幕
var captions = ExpandFunction.GetSpeakerCaptions(captionsArr);
var maxVideoTime = captions?.TimeBase?.LastOrDefault()?.End ?? 0;
@ -747,12 +752,8 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
await RedisExpand.Redis
.HMSetAsync(RedisExpandKey.Task(task), "VideoKnows", questionRes);
var gptRes = new TaskRes(captions);
await RedisExpand.Redis
.HMSetAsync(RedisExpandKey.Task(task), "ChatAnalysis", gptRes);
RedisExpand.InsertChannel(RedisChannelEnum.EndTask, task);
return gptRes;
return null;
}

10
VideoAnalysisCore/Common/AppCommon.cs
View File

@ -253,15 +253,15 @@ namespace VideoAnalysisCore.Common
/// <summary>
/// 转化枚举
/// </summary>
/// <param name="data"></param>
/// <param name="value"></param>
/// <returns></returns>
public static T? ToEnum<T>(this object data) where T : struct, System.Enum
public static T? ToEnum<T>(this object value) where T : struct, Enum
{
try
{
if (data is null || string.IsNullOrEmpty(data?.ToString()))
return null;
return System.Enum.Parse<T>(data.ToString());
if (Enum.TryParse<T>(value.ToString(), true, out var result) && Enum.IsDefined(typeof(T), result))
return result;
return null;
}
catch (Exception)
{

20
VideoAnalysisCore/Common/RedisExpand.cs
View File

@ -177,29 +177,29 @@ namespace VideoAnalysisCore.Common
public static async Task TaskEnd(string task)
{
var tId = long.Parse(task);
var gptRes = (await Redis
.HMGetAsync<TaskRes>(RedisExpandKey.Task(task), "ChatAnalysis")).FirstOrDefault();
if (gptRes is null)
throw new Exception("未能读取到GPT处理结果");
//var gptRes = (await Redis
// .HMGetAsync<TaskRes>(RedisExpandKey.Task(task), "ChatAnalysis")).FirstOrDefault();
//if (gptRes is null)
// throw new Exception("未能读取到GPT处理结果");
var taskData = await DbScoped.Sugar.Queryable<VideoTask>()
.FirstAsync(s => s.Id == tId);
if (taskData.Captions == "[]")
taskData.Captions = (await Redis.HMGetAsync(RedisExpandKey.Task(task), "Captions")).First();
if (taskData.Speaker == "[]")
taskData.Speaker = (await Redis.HMGetAsync(RedisExpandKey.Task(task), "Speaker"))?.FirstOrDefault()??"[]";
//if (taskData.Speaker == "[]")
// taskData.Speaker = (await Redis.HMGetAsync(RedisExpandKey.Task(task), "Speaker"))?.FirstOrDefault()??"[]";
taskData.ChatAnalysis = JsonSerializer.Serialize(gptRes);
taskData.ChatAnalysisScore = gptRes?.Assessment?.Merit?.Sum(s => s.Score) ?? 0;
//未使用结果暂时屏蔽
//taskData.ChatAnalysis = JsonSerializer.Serialize(gptRes);
taskData.ChatAnalysisScore =0;
taskData.ErrorMessage = string.Empty;
taskData.LastEnum = RedisChannelEnum.EndTask;
taskData.EndTime = DateTime.Now;
await DbScoped.Sugar.Updateable(taskData)
.UpdateColumns(it => new
{
it.ChatAnalysis,
//it.ChatAnalysis,
it.Captions,
it.Speaker,
it.ChatAnalysisScore,

5
VideoAnalysisCore/Model/VideoKonwPoint.cs
View File

@ -5,6 +5,7 @@ using System.Text.Json;
using UserCenter.Model.Enum;
using VideoAnalysisCore.AICore.GPT.Dto;
using VideoAnalysisCore.AICore.SherpaOnnx;
using VideoAnalysisCore.Model.Enum;
using VideoAnalysisCore.Model.Interface;
using Whisper.net;
@ -63,5 +64,9 @@ namespace VideoAnalysisCore.Model
/// 内容总结
/// </summary>
public string? Content { get; set; }
/// <summary>
/// 课程阶段
/// </summary>
public virtual StageEnum? Stage { get; set; }
}
}

2
VideoAnalysisCore/Model/VideoTask.cs
View File

@ -87,7 +87,7 @@ namespace VideoAnalysisCore.Model
/// 字幕缓存[AI优化]
/// </summary>
[SugarColumn(ColumnName = "CaptionsAI", ColumnDataType = "longtext", IsNullable = true)]
public string CaptionsAI { get; set; } = "[]";
public string? CaptionsAI { get; set; }
/// <summary>
/// 说话人日志解析缓存
/// </summary>