From 1b239201a2d2f9396c04159c758e45205cadbd79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E8=82=A5=E7=BE=8A?= <1048382248@qq.com> Date: Thu, 27 Feb 2025 18:28:12 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=20=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=AD=98=E5=82=A8=E5=90=8C=E6=AD=A5=E5=88=B0=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VideoAnalysis/Controllers/ApiController.cs | 51 +++++++++- VideoAnalysis/Controllers/Dto/ApiDto.cs | 97 ++++++++++++++++--- .../AICore/FFMPGE/FFMPGEHandle.cs | 7 +- .../AICore/GPT/ChatGPT/Chat_GPT.cs | 11 ++- .../AICore/GPT/DeepSeek/DeepSeek_GPT.cs | 7 +- VideoAnalysisCore/AICore/GPT/KIMI/KIMI_GPT.cs | 4 +- .../AICore/SherpaOnnx/SenseVoice.cs | 9 ++ .../AICore/SherpaOnnx/Speaker.cs | 8 ++ VideoAnalysisCore/Common/AppCommon.cs | 6 +- VideoAnalysisCore/Common/DownloadFile.cs | 8 +- VideoAnalysisCore/Model/VideoKonwPoint.cs | 2 +- 11 files changed, 184 insertions(+), 26 deletions(-) diff --git a/VideoAnalysis/Controllers/ApiController.cs b/VideoAnalysis/Controllers/ApiController.cs index 8c984ae..d8846c2 100644 --- a/VideoAnalysis/Controllers/ApiController.cs +++ b/VideoAnalysis/Controllers/ApiController.cs @@ -19,14 +19,16 @@ namespace Learn.VideoAnalysis.Controllers private readonly ILogger _logger; private readonly IMapper mp; private readonly Repository videoTaskDB; + private readonly Repository videoKonwDB; private readonly IBserGPT chatGPT; public ApiController(ILogger logger, Repository videoTaskDB, - IMapper mp, IBserGPT chatGPT) + IMapper mp, IBserGPT chatGPT, Repository videoKonwDB) { _logger = logger; this.videoTaskDB = videoTaskDB; this.mp = mp; this.chatGPT = chatGPT; + this.videoKonwDB = videoKonwDB; } private string GetClientIpAddress() @@ -151,6 +153,53 @@ namespace Learn.VideoAnalysis.Controllers , task.Id); return Ok(); } + + /// + /// 获取视频知识点片段taskId/tagId二选一 + /// + /// + /// 自定义id + /// + [HttpGet(Name = "TaskKnowInfo")] + public async Task TaskKnowInfo(long taskId, string? tagId) + { + if (taskId == 0 && string.IsNullOrEmpty(tagId)) + return BadRequest(); + var task = await videoTaskDB.AsQueryable() + .WhereIF(taskId != 0, s => s.Id == taskId) + .WhereIF(!string.IsNullOrEmpty(tagId), s => s.TagId == tagId) + .FirstAsync(); + if(task is null) + return BadRequest("无效任务"); + + var konwArr = await videoKonwDB.AsQueryable() + .Where(s=>s.VideoTaskId == task.Id) + .ToArrayAsync(); + if (konwArr is null || konwArr.Length ==0) + return BadRequest("无效任务"); + return Ok(new TaskKnowRes() + { + TagId = task.TagId, + Status = task.LastEnum, + VideoTaskId = task.Id, + KnowBlockArr = konwArr + .GroupBy(s=>s.StartTime) + .Select(s=>new TaskKnowBlock() + { + Id = s.First().Id, + Content = s.First().Content, + StartTime = s.First().StartTime, + EndTime = s.First().EndTime, + Theme =s.First().Theme, + Know=s.Select(x=>new TaskKnowInfo() + { + Id=x.Id, + KnowPoint=x.KnowPoint, + KnowPointId = x.KnowPointId + }).ToArray() + }).ToArray() + }); + } /// /// 获取视频信息taskId/tagId二选一 /// diff --git a/VideoAnalysis/Controllers/Dto/ApiDto.cs b/VideoAnalysis/Controllers/Dto/ApiDto.cs index 5a1f5b1..5ad1098 100644 --- a/VideoAnalysis/Controllers/Dto/ApiDto.cs +++ b/VideoAnalysis/Controllers/Dto/ApiDto.cs @@ -1,4 +1,5 @@ 锘縰sing AntDesign; +using SqlSugar; using System.ComponentModel.DataAnnotations; using UserCenter.Model.Enum; using VideoAnalysisCore.AICore.GPT.Dto; @@ -71,6 +72,76 @@ namespace Learn.VideoAnalysis.Controllers.Dto public string Text { get; set; } public object Value { get; set; } } + public class TaskKnowInfo + { + /// + ///瑙嗛鐗囨鐭ヨ瘑鐐 id + /// + public long Id { get; set; } + /// + /// 鐭ヨ瘑鐐 + /// + public string KnowPoint { get; set; } + /// + /// 鐭ヨ瘑鐐笽D + /// + public string KnowPointId { get; set; } + + } + public class TaskKnowBlock + { + public long Id { get; set; } + + /// + /// 寮濮嬫椂闂 + /// + public float? StartTime { get; set; } + /// + /// 缁撴潫鏃堕棿 + /// + public float? EndTime { get; set; } + /// + /// 鎸佺画鏃堕棿 + /// + [SugarColumn(IsIgnore = true)] + public float? KeepTime => (EndTime ?? 0) - StartTime ?? 0; + /// + /// 涓婚 + /// + public string? Theme { get; set; } + /// + /// 鍐呭鎬荤粨 + /// + public string? Content { get; set; } + /// + /// 鐭ヨ瘑鐐瑰垪琛 + /// + public TaskKnowInfo[] Know { get; set; } + } + /// + /// 瑙嗛鐗囨鐭ヨ瘑鐐圭粨鏋 + /// + public class TaskKnowRes + { + /// + /// 鑷畾涔塈d [浠诲姟瑙嗛鑷畾涔塱d] + /// + /// + public string? TagId { get; set; } + /// + /// 浠诲姟褰撳墠鎵ц鐘舵 + /// + public RedisChannelEnum Status { get; set; } + /// + /// 瑙嗛浠诲姟id + /// + public long VideoTaskId { get; set; } + /// + /// 瑙嗛鐭ヨ瘑蹇 + /// + public TaskKnowBlock[] KnowBlockArr { get; set; } + + } public class TaskInfoRes: TaskRes { public TaskInfoRes() @@ -81,19 +152,19 @@ namespace Learn.VideoAnalysis.Controllers.Dto /// 浠诲姟褰撳墠鎵ц鐘舵 /// public RedisChannelEnum Status { get; set; } - /// - /// 鏃堕棿杞寸姸鎬佹灇涓 - /// - public Dictionary TimeTypeEnum => - Enum.GetValues(typeof(TimeBaseTypeEnum)) - .Cast() - .ToDictionary(x => (int)x, x => x.ToString()); + ///// + ///// 鏃堕棿杞寸姸鎬佹灇涓 + ///// + //public Dictionary TimeTypeEnum => + // Enum.GetValues(typeof(TimeBaseTypeEnum)) + // .Cast() + // .ToDictionary(x => (int)x, x => x.ToString()); - /// - /// 鏃堕棿杞村悎璁 - /// - public Dictionary? TimeBaseTotal => - TimeBase?.GroupBy(s => s.TimeBaseType??TimeBaseTypeEnum.鏁欏笀璁叉巿)? - .ToDictionary(s => s.Key, s => new TextValue(s.Sum(x => x.End - x.Start))); + ///// + ///// 鏃堕棿杞村悎璁 + ///// + //public Dictionary? TimeBaseTotal => + // TimeBase?.GroupBy(s => s.TimeBaseType??TimeBaseTypeEnum.鏁欏笀璁叉巿)? + // .ToDictionary(s => s.Key, s => new TextValue(s.Sum(x => x.End - x.Start))); } } diff --git a/VideoAnalysisCore/AICore/FFMPGE/FFMPGEHandle.cs b/VideoAnalysisCore/AICore/FFMPGE/FFMPGEHandle.cs index 89f3b7e..b92a2ec 100644 --- a/VideoAnalysisCore/AICore/FFMPGE/FFMPGEHandle.cs +++ b/VideoAnalysisCore/AICore/FFMPGE/FFMPGEHandle.cs @@ -6,6 +6,8 @@ using System.Threading.Tasks; using static System.Runtime.InteropServices.JavaScript.JSType; using System.Xml.Linq; using System.Runtime.InteropServices; +using SqlSugar.IOC; +using VideoAnalysisCore.Model; namespace VideoAnalysisCore.AICore.FFMPGE { @@ -30,7 +32,10 @@ namespace VideoAnalysisCore.AICore.FFMPGE public static async Task Audio2WAV16KAsync(string task) { Task = task; - var filePath = RedisExpand.Redis.HGet(RedisExpandKey.Task(task), "LocalMediaPath"); + var filePath = await DbScoped.SugarScope + .Queryable() + .Where(s => s.Id == long.Parse(task)) + .Select(s=>s.LocalMediaPath).FirstAsync(); if (string.IsNullOrEmpty(filePath)) throw new Exception($"浠诲姟id[{task}] 鏃犳晥"); diff --git a/VideoAnalysisCore/AICore/GPT/ChatGPT/Chat_GPT.cs b/VideoAnalysisCore/AICore/GPT/ChatGPT/Chat_GPT.cs index 034ce00..86cd8e6 100644 --- a/VideoAnalysisCore/AICore/GPT/ChatGPT/Chat_GPT.cs +++ b/VideoAnalysisCore/AICore/GPT/ChatGPT/Chat_GPT.cs @@ -9,6 +9,7 @@ using VideoAnalysisCore.Model.Dto; using VideoAnalysisCore.AICore.GPT.Dto; using VideoAnalysisCore.AICore.GPT; using System.Threading.Tasks; +using VideoAnalysisCore.AICore.SherpaOnnx; namespace VideoAnalysisCore.AICore.GPT.ChatGPT { @@ -89,7 +90,10 @@ namespace VideoAnalysisCore.AICore.GPT.ChatGPT $"杈撳嚭鏍煎紡 json瀛楃涓 瀵硅薄鏍煎紡{fileNameResFormat}"; var fileNameInfoRes = await ChatAsync(task, fileNamePostMessages, null, fileNameResFormat); - var captions = ExpandFunction.GetSpeakerCaptions(task); + var speakerArr = JsonSerializer.Deserialize(taskInfo.Speaker); + var captionsArr = JsonSerializer.Deserialize(taskInfo.Captions); + var captions = ExpandFunction.GetSpeakerCaptions(captionsArr, speakerArr); + var maxVideoTime = captions?.TimeBase?.LastOrDefault()?.End ?? 0; var criteriaBuilder = new StringBuilder(); @@ -245,7 +249,10 @@ namespace VideoAnalysisCore.AICore.GPT.ChatGPT var taskInfo = await videoTaskDB.AsQueryable() .Where(s => s.Id == taskId) .FirstAsync(); - var captions = ExpandFunction.GetSpeakerCaptions(task); + + var speakerArr = JsonSerializer.Deserialize(taskInfo.Speaker); + var captionsArr = JsonSerializer.Deserialize(taskInfo.Captions); + var captions = ExpandFunction.GetSpeakerCaptions(captionsArr, speakerArr); var criteriaArr = await criteriaDB.GetListAsync(s => s.Subject == taskInfo.Subject); var criteriaBuilder = new StringBuilder(); foreach (var item in criteriaArr) diff --git a/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs b/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs index 6aacd65..5f9f3fa 100644 --- a/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs +++ b/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs @@ -9,6 +9,7 @@ using VideoAnalysisCore.Model.Dto; using VideoAnalysisCore.AICore.GPT.Dto; using System.Threading.Tasks; using VideoAnalysisCore.AICore.GPT.ChatGPT; +using VideoAnalysisCore.AICore.SherpaOnnx; namespace VideoAnalysisCore.AICore.GPT.DeepSeek { @@ -62,7 +63,9 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek $"杈撳嚭鏍煎紡 json瀛楃涓 瀵硅薄鏍煎紡{fileNameResFormat}"; var fileNameInfoRes = await ChatAsync(task, fileNamePostMessages, null);//, "deepseek-chat"); - var captions = ExpandFunction.GetSpeakerCaptions(task); + var speakerArr = JsonSerializer.Deserialize(taskInfo.Speaker); + var captionsArr = JsonSerializer.Deserialize(taskInfo.Captions); + var captions = ExpandFunction.GetSpeakerCaptions(captionsArr, speakerArr); var maxVideoTime = captions?.TimeBase?.LastOrDefault()?.End ?? 0; var criteriaBuilder = new StringBuilder(); @@ -123,7 +126,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek { Console.WriteLine(DateTime.Now + $"=>{task} 寰楀垎杩囦綆 " + checkRes?.Score ); Console.WriteLine( checkRes.Evaluation); - Console.WriteLine( ); + Console.WriteLine(); } } diff --git a/VideoAnalysisCore/AICore/GPT/KIMI/KIMI_GPT.cs b/VideoAnalysisCore/AICore/GPT/KIMI/KIMI_GPT.cs index 0e34f5d..9678c03 100644 --- a/VideoAnalysisCore/AICore/GPT/KIMI/KIMI_GPT.cs +++ b/VideoAnalysisCore/AICore/GPT/KIMI/KIMI_GPT.cs @@ -53,7 +53,9 @@ namespace VideoAnalysisCore.AICore.GPT.KIMI var taskInfo = await videoTaskDB.AsQueryable() .Where(s => s.Id == taskId) .FirstAsync(); - var captions = ExpandFunction.GetSpeakerCaptions(task); + var speakerArr = JsonSerializer.Deserialize(taskInfo.Speaker); + var captionsArr = JsonSerializer.Deserialize(taskInfo.Captions); + var captions = ExpandFunction.GetSpeakerCaptions(captionsArr, speakerArr); var criteriaArr = await criteriaDB.GetListAsync(s => s.Subject == taskInfo.Subject); var criteriaBuilder = new StringBuilder(); foreach (var item in criteriaArr) diff --git a/VideoAnalysisCore/AICore/SherpaOnnx/SenseVoice.cs b/VideoAnalysisCore/AICore/SherpaOnnx/SenseVoice.cs index ae2b5cd..fbdfb15 100644 --- a/VideoAnalysisCore/AICore/SherpaOnnx/SenseVoice.cs +++ b/VideoAnalysisCore/AICore/SherpaOnnx/SenseVoice.cs @@ -1,14 +1,17 @@ 锘縰sing Microsoft.Extensions.Options; using SherpaOnnx; +using SqlSugar.IOC; using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; using System.Text; +using System.Text.Json; using System.Text.RegularExpressions; using System.Threading.Tasks; using VideoAnalysisCore.Common; +using VideoAnalysisCore.Model; using static System.Runtime.InteropServices.JavaScript.JSType; namespace VideoAnalysisCore.AICore.SherpaOnnx @@ -256,6 +259,12 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx } Console.WriteLine(DateTime.Now + "=> SenseVoice 瀛楀箷鏁伴噺"+ res.Count); + + var captionsStr = JsonSerializer.Serialize(res); + DbScoped.SugarScope + .Updateable() + .SetColumns(it => it.Captions == captionsStr) + .Where(it => it.Id == long.Parse(task)); await RedisExpand.Redis.HMSetAsync(RedisExpandKey.Task(task), "Captions", res); //RedisExpand.InsertChannel(Enum.RedisChannelEnum.ParsingSpeaker, task); RedisExpand.InsertChannel(Enum.RedisChannelEnum.ChatModelAnalysis, task); diff --git a/VideoAnalysisCore/AICore/SherpaOnnx/Speaker.cs b/VideoAnalysisCore/AICore/SherpaOnnx/Speaker.cs index fd55497..c434ee4 100644 --- a/VideoAnalysisCore/AICore/SherpaOnnx/Speaker.cs +++ b/VideoAnalysisCore/AICore/SherpaOnnx/Speaker.cs @@ -4,6 +4,9 @@ using System.Linq; using System.Text.Json.Serialization; using VideoAnalysisCore.Common; using SherpaOnnx; +using SqlSugar.IOC; +using VideoAnalysisCore.Model; +using System.Text.Json; namespace VideoAnalysisCore.AICore.SherpaOnnx { @@ -72,6 +75,11 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx }, nint.Zero); var res = segments.Select(s => new OfflineSpeakerRes(s)); await RedisExpand.Redis.HSetAsync(RedisExpandKey.Task(task), "Speaker", res); + var speakerStr = JsonSerializer.Serialize(res); + DbScoped.SugarScope + .Updateable() + .SetColumns(it => it.Speaker == speakerStr) + .Where(it => it.Id == long.Parse(task)); //鍔犲叆涓嬩竴闃熷垪 RedisExpand.InsertChannel(Enum.RedisChannelEnum.ChatModelAnalysis, task); diff --git a/VideoAnalysisCore/Common/AppCommon.cs b/VideoAnalysisCore/Common/AppCommon.cs index 306e33e..4650deb 100644 --- a/VideoAnalysisCore/Common/AppCommon.cs +++ b/VideoAnalysisCore/Common/AppCommon.cs @@ -178,10 +178,8 @@ namespace VideoAnalysisCore.Common /// /// 鑾峰彇Task澶勭悊鍚庣殑 璇磋瘽浜哄瓧骞 /// - public static TotalCaptionsDto GetSpeakerCaptions(string task) + public static TotalCaptionsDto GetSpeakerCaptions(SenseVoiceRes[] captionsArr, OfflineSpeakerRes[] speakerArr) { - var captionsArr = RedisExpand.Redis.HMGet(RedisExpandKey.Task(task), "Captions").FirstOrDefault(); - var speakerArr = RedisExpand.Redis.HMGet(RedisExpandKey.Task(task), "Speaker").FirstOrDefault(); if (captionsArr is null || captionsArr.Length == 0) //|| speakerArr is null || speakerArr.Length == 0) throw new Exception("闊抽瑙ｆ瀽鏁版嵁寮傚父"); @@ -193,9 +191,9 @@ namespace VideoAnalysisCore.Common var teacherSpeaking = 0f; var studentSpeaking = 0f; var results = new Dictionary>(); + var ss = new List { 1 }; if (speakerArr is null || speakerArr.Count() == 0) { - var ss = new List { 1 }; results = captionsArr.ToDictionary(s => s, s=> ss); } else diff --git a/VideoAnalysisCore/Common/DownloadFile.cs b/VideoAnalysisCore/Common/DownloadFile.cs index 0f86a0b..cc828a6 100644 --- a/VideoAnalysisCore/Common/DownloadFile.cs +++ b/VideoAnalysisCore/Common/DownloadFile.cs @@ -1,11 +1,13 @@ 锘縰sing AntDesign; using Downloader; +using SqlSugar.IOC; using System; using System.ComponentModel; using System.Diagnostics; using System.IO; using System.Net; using System.Threading.Tasks; +using VideoAnalysisCore.Model; namespace VideoAnalysisCore.Common { @@ -122,7 +124,11 @@ namespace VideoAnalysisCore.Common var outputPath = Path.Combine(localPath, task + fileExtension); if (!Directory.Exists(localPath)) Directory.CreateDirectory(localPath); - RedisExpand.Redis.HSet(RedisExpandKey.Task(task), "LocalMediaPath", outputPath); + DbScoped.SugarScope + .Updateable() + .SetColumns(it => it.LocalMediaPath == outputPath) + .Where(it => it.Id == long.Parse(task)); + IDownload download = DownloadBuilder.New() .WithUrl(fileUrl) .WithDirectory(localPath) diff --git a/VideoAnalysisCore/Model/VideoKonwPoint.cs b/VideoAnalysisCore/Model/VideoKonwPoint.cs index 1247011..3d5d1e7 100644 --- a/VideoAnalysisCore/Model/VideoKonwPoint.cs +++ b/VideoAnalysisCore/Model/VideoKonwPoint.cs @@ -12,7 +12,7 @@ using Whisper.net; namespace VideoAnalysisCore.Model { /// - /// 瑙嗛浠诲姟鐭ヨ瘑鐗囨 + /// 瑙嗛鐗囨鐭ヨ瘑鐐 /// [SugarTable("videokonwpoint")] public class VideoKonwPoint : IDB