From 59c8171ce82b25aded5f7ca98656937da06d6181 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E8=82=A5=E7=BE=8A?= <1048382248@qq.com> Date: Thu, 9 Jan 2025 11:53:48 +0800 Subject: [PATCH] =?UTF-8?q?=E5=90=8C=E6=AD=A5=E4=BB=A3=E7=A0=81=20?= =?UTF-8?q?=E5=BC=95=E5=85=A5=20=E7=9F=A5=E8=AF=86=E7=82=B9=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E5=BA=93=20=E9=87=8D=E5=86=99=20AI=E5=88=86=E6=9E=90?= =?UTF-8?q?=E8=A7=86=E9=A2=91=E7=89=87=E6=AE=B5=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Components/Layouts/BasicLayout.razor.cs | 7 + .../Components/Pages/VideoTaskPage.razor | 4 + .../Components/Pages/VideoTaskPage.razor.cs | 5 + .../Components/Pages/VideoTaskShow.razor | 80 ++++++++ .../Components/Pages/VideoTaskShow.razor.cs | 72 ++++++++ .../Components/Pages/VideoTaskShow.razor.css | 59 ++++++ VideoAnalysis/Controllers/ApiController.cs | 4 +- VideoAnalysis/Controllers/Dto/ApiDto.cs | 6 + VideoAnalysis/Program.cs | 11 +- VideoAnalysis/appsettings.json | 9 +- VideoAnalysisCore/AICore/GPT/BserGPT.cs | 1 + .../AICore/GPT/ChatGPT/Chat_GPT.cs | 171 +++++++++++++++++- .../AICore/GPT/Dto/QuestionRes.cs | 22 +++ VideoAnalysisCore/AICore/GPT/KIMI/KIMI_GPT.cs | 2 +- .../AICore/SherpaOnnx/SenseVoice.cs | 3 +- VideoAnalysisCore/Common/AppCommon.cs | 76 +++++--- VideoAnalysisCore/Common/RedisExpand.cs | 3 +- VideoAnalysisCore/Common/Repository.cs | 6 +- VideoAnalysisCore/Common/SqlSugarExpand.cs | 7 + VideoAnalysisCore/Interface/IDB.cs | 15 ++ VideoAnalysisCore/Interface/IKnowsDB.cs | 15 ++ .../Model/CourseGradingCriteria.cs | 3 +- VideoAnalysisCore/Model/Dto/VideoTaskDto.cs | 5 + VideoAnalysisCore/Model/KnowledgeInfo.cs | 45 +++++ VideoAnalysisCore/Model/VideoTask.cs | 7 +- 25 files changed, 603 insertions(+), 35 deletions(-) create mode 100644 VideoAnalysis/Components/Pages/VideoTaskShow.razor create mode 100644 VideoAnalysis/Components/Pages/VideoTaskShow.razor.cs create mode 100644 VideoAnalysis/Components/Pages/VideoTaskShow.razor.css create mode 100644 VideoAnalysisCore/Interface/IDB.cs create mode 100644 VideoAnalysisCore/Interface/IKnowsDB.cs create mode 100644 VideoAnalysisCore/Model/KnowledgeInfo.cs diff --git a/VideoAnalysis/Components/Layouts/BasicLayout.razor.cs b/VideoAnalysis/Components/Layouts/BasicLayout.razor.cs index 47de42b..a53b2c2 100644 --- a/VideoAnalysis/Components/Layouts/BasicLayout.razor.cs +++ b/VideoAnalysis/Components/Layouts/BasicLayout.razor.cs @@ -56,6 +56,13 @@ namespace VideoAnalysisRazor.Layouts Name = "登录页", Key = "Login", HideInMenu = true, + }, + new MenuDataItem + { + Path = "/VideoTaskShow", + Name = "视频任务预览", + Key = "VideoTaskShow", + HideInMenu = true, } ]; } diff --git a/VideoAnalysis/Components/Pages/VideoTaskPage.razor b/VideoAnalysis/Components/Pages/VideoTaskPage.razor index 0376d28..02f68a9 100644 --- a/VideoAnalysis/Components/Pages/VideoTaskPage.razor +++ b/VideoAnalysis/Components/Pages/VideoTaskPage.razor @@ -45,6 +45,10 @@ + + diff --git a/VideoAnalysis/Components/Pages/VideoTaskPage.razor.cs b/VideoAnalysis/Components/Pages/VideoTaskPage.razor.cs index 1b0f224..30732a8 100644 --- a/VideoAnalysis/Components/Pages/VideoTaskPage.razor.cs +++ b/VideoAnalysis/Components/Pages/VideoTaskPage.razor.cs @@ -20,6 +20,7 @@ namespace Learn.VideoAnalysis.Components.Pages [Inject] private ConfirmService ComfirmService { get; set; } = default!; [Inject] private Repository taskDB { get; set; } = default!; + [Inject] private NavigationManager NavigationManager { get; set; } = default!; [Inject] private INotificationService _notice { get; set; } = default!; @@ -56,6 +57,10 @@ namespace Learn.VideoAnalysis.Components.Pages reStartTask = query; modalShow = true; } + void PreviewTask(VideoTaskDto task) + { + NavigationManager.NavigateTo("/VideoTaskShow/"+task.Id); + } /// /// 重试 /// diff --git a/VideoAnalysis/Components/Pages/VideoTaskShow.razor b/VideoAnalysis/Components/Pages/VideoTaskShow.razor new file mode 100644 index 0000000..fd8cd11 --- /dev/null +++ b/VideoAnalysis/Components/Pages/VideoTaskShow.razor @@ -0,0 +1,80 @@ +@page "/VideoTaskShow/{tid}" +
+ + + +
+ +
+ +
+
+ +
+ + + + +@code { + +} diff --git a/VideoAnalysis/Components/Pages/VideoTaskShow.razor.cs b/VideoAnalysis/Components/Pages/VideoTaskShow.razor.cs new file mode 100644 index 0000000..87a9449 --- /dev/null +++ b/VideoAnalysis/Components/Pages/VideoTaskShow.razor.cs @@ -0,0 +1,72 @@ +using AntDesign; +using AntDesign.TableModels; +using FFmpeg.NET.Services; +using Learn.VideoAnalysis.Controllers.Dto; +using Microsoft.AspNetCore.Components; +using Microsoft.AspNetCore.Components.Web; +using Microsoft.AspNetCore.DataProtection.KeyManagement; +using SqlSugar; +using System.Linq.Expressions; +using System.Threading.Tasks; +using VideoAnalysisCore.AICore.GPT.Dto; +using VideoAnalysisCore.AICore.SherpaOnnx; +using VideoAnalysisCore.Common; +using VideoAnalysisCore.Enum; +using VideoAnalysisCore.Model; +using VideoAnalysisCore.Model.Dto; + +namespace Learn.VideoAnalysis.Components.Pages +{ + public partial class VideoTaskShow : ComponentBase + { + + [Inject] private ConfirmService ComfirmService { get; set; } = default!; + [Inject] private IHttpContextAccessor HttpContext { get; set; } = default!; + [Inject] private Repository taskDB { get; set; } = default!; + + private VideoTask nowTask { get; set; } = default!; + private string videoPath { get; set; } = default!; + + /// + /// 字幕 + /// + private SenseVoiceRes[] captionsArr { get; set; } = default!; + /// + /// 分段 + /// + private VideoKnowRes[] videoKnows { get; set; } = default!; + /// + /// 在渲染页面之后 + /// + /// + /// + protected override async Task OnAfterRenderAsync(bool firstRender) + { + + + } + /// + /// 初始化 + /// + protected override async void OnInitialized() + { + var routeData = HttpContext.HttpContext.GetRouteData(); + if (routeData is null) + return; + long taskId = (long)routeData.Values["id"]; + nowTask = await taskDB.GetFirstAsync(s => s.Id == taskId); + if(nowTask is null) + return; + captionsArr = RedisExpand.Redis.HMGet(RedisExpandKey.Task(taskId), "Captions").FirstOrDefault(); + videoKnows = RedisExpand.Redis.HMGet(RedisExpandKey.Task(taskId), "VideoKnows").FirstOrDefault(); + videoPath = AppCommon.GetVideoPath(nowTask.Id.ToString()); + } + + private async Task Comfirm(string message) + { + return await ComfirmService.Show(message, "提示", ConfirmButtons.YesNo, ConfirmIcon.Warning) == ConfirmResult.Yes; + } + + } + +} diff --git a/VideoAnalysis/Components/Pages/VideoTaskShow.razor.css b/VideoAnalysis/Components/Pages/VideoTaskShow.razor.css new file mode 100644 index 0000000..9ca6b8f --- /dev/null +++ b/VideoAnalysis/Components/Pages/VideoTaskShow.razor.css @@ -0,0 +1,59 @@ + +* { + padding: 0; + margin: 0; +} + +#video-container { + position: relative; + width: 1600px; + height: 850px; + float: left; +} + +video { + width: 100%; + height: 100%; +} + +.subtitles { + position: absolute; + bottom: -40px; + width: 100%; + text-align: center; + color: white; + background-color: rgba(0, 0, 0, 0.7); + font-size: 18px; +} + +#segmentsContainer { + display: flex; + flex-direction: column; + width: 265px; + height: 850px; + gap: 10px; + overflow: hidden; + overflow-y: scroll; + float: left; + flex-wrap: nowrap; + padding: 10px; + align-content: flex-start; + justify-content: flex-start; + align-items: center; +} + + #segmentsContainer button { + width: 100%; + height: 60px; + font-size: 1.3rem; + text-align: left; + cursor: pointer; + color: rgb(103, 194, 58); + background-color: rgb(240, 249, 235); + border: 1px solid rgb(179, 225, 157); + } + + #segmentsContainer button:hover { + background-color: rgb(248, 230, 191) !important; + border: 1px solid rgb(206, 187, 81); + } diff --git a/VideoAnalysis/Controllers/ApiController.cs b/VideoAnalysis/Controllers/ApiController.cs index ca4dcae..24ac22e 100644 --- a/VideoAnalysis/Controllers/ApiController.cs +++ b/VideoAnalysis/Controllers/ApiController.cs @@ -83,8 +83,7 @@ namespace Learn.VideoAnalysis.Controllers public async Task Test(long taskId) { //¿ʼִGPT - RedisExpand.InsertChannel(RedisChannelEnum.ChatModelAnalysis - , taskId); + chatGPT.GetKnow(taskId.ToString()); return Ok(); } @@ -206,6 +205,7 @@ namespace Learn.VideoAnalysis.Controllers Subject = req.Subject, Tag = req.Tag, TagId = req.TagId, + MediaName = req.MediaName }; // task.Id = await videoTaskDB.InsertReturnBigIdentityAsync(task); diff --git a/VideoAnalysis/Controllers/Dto/ApiDto.cs b/VideoAnalysis/Controllers/Dto/ApiDto.cs index 11d5104..361207b 100644 --- a/VideoAnalysis/Controllers/Dto/ApiDto.cs +++ b/VideoAnalysis/Controllers/Dto/ApiDto.cs @@ -18,6 +18,12 @@ namespace Learn.VideoAnalysis.Controllers.Dto [Url(ErrorMessage = "请输入有效的 URL")] public string MediaUrl { get; set; } = string.Empty; /// + /// 资源名称 + /// + [Required(ErrorMessage = "资源名称是必要的")] + [Url(ErrorMessage = "请输入有效的 资源名称")] + public string MediaName { get; set; } = string.Empty; + /// /// ApiKey /// [Required(ErrorMessage = "接口Token是必填项")] diff --git a/VideoAnalysis/Program.cs b/VideoAnalysis/Program.cs index 5efc6a4..f8e077b 100644 --- a/VideoAnalysis/Program.cs +++ b/VideoAnalysis/Program.cs @@ -7,6 +7,7 @@ using Mapster; using VideoAnalysisCore.AICore.GPT; using VideoAnalysisCore.AICore.GPT.KIMI; using VideoAnalysisCore.AICore.GPT.ChatGPT; +using Microsoft.Extensions.FileProviders; @@ -101,7 +102,15 @@ namespace Learn.VideoAnalysis //} - app.UseStaticFiles(); + app.UseStaticFiles(new StaticFileOptions + { + FileProvider = new PhysicalFileProvider(AppCommon.TaskCachedFile), + RequestPath = "/video", + //OnPrepareResponse = ctx => // + //{ + // ctx.Context.Response.Headers[Microsoft.Net.Http.Headers.HeaderNames.CacheControl] = "public,max - age = 31536000"; + //} + }); app.UseAntiforgery(); app.MapRazorComponents() diff --git a/VideoAnalysis/appsettings.json b/VideoAnalysis/appsettings.json index 453bffd..aaabb52 100644 --- a/VideoAnalysis/appsettings.json +++ b/VideoAnalysis/appsettings.json @@ -27,14 +27,19 @@ "ApiKey": "sk-8BvvhESZIkgUbiaaJhglPxFa4o2X9H3xEv9lXELrWWwGxHWY" }, "ChatGpt": { - "Host": "https://api.oaibest.com/", + "Host": "https://api.g4f.icu/", + //"Host": "https://api.oaibest.com/", "ApiKey": "sk-D15tBln31N7dI9Fi7lds7OySFv5tOEK7DMNsG5rY2E6DCr4s" } }, "DB": { "ConnectionString": "AllowLoadLocalInfile=true;Server=192.168.2.9;User ID=root;Password=qwe123!@#;Port=3306;Database=learn.videoanalysis;CharSet=utf8mb4;pooling=true;SslMode=None", "SqlType": "MySql", - "UpdateTable": true, + "UpdateTable": true + }, + "KnowsDB": { + "ConnectionString": "Server=47.109.35.116;Database=ResourceBank;UID=live;Password=Woshiren^&*();MultipleActiveResultSets=true;Encrypt=True;TrustServerCertificate=True;", + "SqlType": "SqlServer" } } } diff --git a/VideoAnalysisCore/AICore/GPT/BserGPT.cs b/VideoAnalysisCore/AICore/GPT/BserGPT.cs index eaa9368..4cc4911 100644 --- a/VideoAnalysisCore/AICore/GPT/BserGPT.cs +++ b/VideoAnalysisCore/AICore/GPT/BserGPT.cs @@ -16,5 +16,6 @@ namespace VideoAnalysisCore.AICore.GPT /// 任务id /// public Task CallGPT(string task); + public Task GetKnow(string task); } } diff --git a/VideoAnalysisCore/AICore/GPT/ChatGPT/Chat_GPT.cs b/VideoAnalysisCore/AICore/GPT/ChatGPT/Chat_GPT.cs index 7a0bf79..5edd72d 100644 --- a/VideoAnalysisCore/AICore/GPT/ChatGPT/Chat_GPT.cs +++ b/VideoAnalysisCore/AICore/GPT/ChatGPT/Chat_GPT.cs @@ -8,6 +8,7 @@ using System.Reflection; using VideoAnalysisCore.Model.Dto; using VideoAnalysisCore.AICore.GPT.Dto; using VideoAnalysisCore.AICore.GPT; +using System.Threading.Tasks; namespace VideoAnalysisCore.AICore.GPT.ChatGPT { @@ -19,16 +20,184 @@ namespace VideoAnalysisCore.AICore.GPT.ChatGPT private readonly ChatGPTClient chatClient; private readonly Repository criteriaDB; private readonly Repository videoTaskDB; + private readonly Repository knowledgeInfoDB; /// /// 初始化 /// /// /// - public Chat_GPT(ChatGPTClient moonshotClient, Repository criteria, Repository videoTaskDB) + public Chat_GPT(ChatGPTClient moonshotClient, Repository criteria, Repository videoTaskDB, Repository knowledgeInfoDB) { this.chatClient = moonshotClient; criteriaDB = criteria; this.videoTaskDB = videoTaskDB; + this.knowledgeInfoDB = knowledgeInfoDB; + } + + private static List MergeRes(VideoKnowRes[] timeBases) + { + if (timeBases == null || timeBases.Count() == 0) + { + return new List(); + } + var mergedList = new List(); + // 初始化合并段 + var current = timeBases.First(); + foreach (var next in timeBases) + { + // 如果类型相同，则扩展时间段 + if (current.主题 == next.主题) + { + current.结束秒 = Math.Max(current.结束秒.Value, next.结束秒.Value); + current.内容总结 += next.内容总结; + } + else + { + // 类型不同，将当前时间段加入结果列表，并开始新时间段 + mergedList.Add(current); + current = next; + } + } + // 添加最后的时间段 + mergedList.Add(current); + return mergedList; + } + + + /// + /// 获取知识点 + /// + /// 任务id + /// + public async Task GetKnow(string task) + { + var taskId = long.Parse(task); + var taskInfo = await videoTaskDB.AsQueryable() + .Where(s => s.Id == taskId) + .FirstAsync(); + var subject = "数学"; + var xkwKnows = await knowledgeInfoDB.AsQueryable() + .Where(s => s.Course_Id == 27 + && s.Depth == 2) + .Select(s => s.Name).ToArrayAsync(); + string title = "周三（1.3）《第八章——统计与概率：超几何分布的极限为二项分布》"; + var fileNameResFormat = "{授课章节: string|null, 授课内容:string}"; + var fileNamePostMessages = title + + " 这是一堂课的标题,请你帮我分析一些关于课堂方面的内容." + + $"1.分析出高中{subject}课堂授课的主要章节(例如 章节: 数列),章节范围限定在[{xkwKnows}]范围." + + $"2.分析出这堂课的主要授课内容." + + $"输出格式 json字符串 对象格式{fileNameResFormat}"; + var fileNameInfoRes = await ChatAsync(task, fileNamePostMessages, fileNameResFormat); + + var captions = ExpandFunction.GetSpeakerCaptions(task); + var criteriaBuilder = new StringBuilder(); + + var resFormat = """[{"StartTime":开始秒(number),"EndTime":结束秒(number),"Section":章节(string),"Theme":主题(string),"Content":内容总结(string)}]"""; + var know = await knowledgeInfoDB.GetFirstAsync(s => s.Name == fileNameInfoRes.授课章节); + var knowledgeInfos = await knowledgeInfoDB.AsQueryable().ToChildListAsync(s => s.Parent_Id, know.Id); + var knows = "数列的概念,数列的定义,项的表示,数列的表示方法,通项公式,递推公式,图像表示,数列的类型,等差数列,等比数列,其他特殊数列,数列的性质,单调性,有限性,数列的求和,等差数列求和公式,等比数列求和公式,数列极限,递推关系"; + knows = string.Join(',', knowledgeInfos.Select(s => s.Name)); + var postMessages = + $"你的任务是分析视频字幕内容并提取出中国高考考试试题方法点,然后根据步骤分析出内容片段" + + $"按以下步骤完成:" + + $"1.识别方法点:提取字幕内容中与{subject}考试相关的方法点。" + + $"2.分类方法点:按学科方法点,细化到具体章节与主题（例如“章节:数列 主题:数列的基本概念”）。" + + $"3.分析总结:基于提取出的方法点名称来匹配我提供的方法点名称" + + $"提供的方法点名称(基本概念,课堂练习,{knows},例题讲解)。" + + $"4.关联合并相似的知识点来合并为内容片段。" + + $"内容片段使用关联知识点中的最小(开始秒)和(最大)结束秒,主题为关联知识点的主题分析,内容总结为关联知识点的内容总结分析。" + + $"延长内容片段时间区间来获取更加详细的上下文。" + + $"输入：包含时间戳的视频字幕文本。" + + $"以下是包含时间的视频字幕文本。" + + $"字幕格式(说话人:开始秒:结束秒:内容|下一段字幕).字幕列表 {captions.Captions}" + + $"输出格式({resFormat})"; + //var postMessages = + // $"你是一名专业的教育视频内容分析助手，主要任务是分析视频字幕内容，精准提取出与中国高考数学考试相关的试题方法点。" + + // $"按以下步骤完成:" + + // $"1.准确识别方法点：从字幕中提取与{subject}考试紧密相关的方法点，尤其关注与给定方法点类别相关的内容" + + // $"2.深入分析总结：依据给定的方法点类别进行约束，确定提取出的方法点所属类别。" + + // $"给定方法点包括(基本概念,课堂练习,例题讲解,解题技巧,{gjz})。" + + // $"3.细致分类方法点：按照学科方法点进行分类，具体细化到特定章节与主题，格式为 “章节：具体章节名称 主题：具体主题名称" + + // $"4.合理合并相似方法点为内容片段，确保内容的连贯性和逻辑性" + + // $"5.关联每个内容片段的方法点所有的时间并结构化输出。" + + // $"尽可能延内容片段时间区间,以获取详细的方法点上下文。" + + // $"输入：包含时间戳的视频字幕文本。输出格式：开始秒,结束秒,主题,内容总结" + + // $"以下是包含时间的视频字幕文本。" + + // $"字幕格式(说话人:开始秒:结束秒:内容|下一段字幕).字幕列表 {captions.Captions}" + + // $"返回固定的JSON格式({resFormat})"; + + var questionRes =await ChatAsync(task, postMessages, resFormat); + await RedisExpand.Redis + .HMSetAsync(RedisExpandKey.Task(task), "VideoKnows", questionRes); + + var postMessages1 = + $"你的任务是分析json内容并合并含义相似的主题为新的主题" + + $"按以下步骤完成:" + + $"1.合理合并主题字段重复相似的对象为新的json对象，确保内容的连贯性和逻辑性。" + + $"2.合并对象属性持续时间低于60秒的对象" + + $"3.结构化输出。" + + $"输入：json对象 包含总结开始秒,结束秒,持续时间,主题,章节,内容总结" + + $"以下是包含json内容的文本。" + + $" {JsonSerializer.Serialize(questionRes)}" + + $"返回固定的JSON格式({resFormat})"; + + + var questionRes1 = await ChatAsync(task, postMessages1, resFormat); + //questionRes1 = MergeRes(questionRes1).ToArray(); + + var gptRes = new TaskRes(captions); + await RedisExpand.Redis + .HMSetAsync(RedisExpandKey.Task(task), "ChatAnalysis", gptRes); + RedisExpand.InsertChannel(RedisChannelEnum.EndTask, task); + return gptRes; + } + public async Task ChatAsync(string task,string postMessages,string resFormat) + { + var maxTokens = 4000; + var chatRep = new ChatRequest + { + max_tokens = maxTokens, + temperature = 0.3f, + messages = [ + new Message(postMessages,"system"), + new Message(resFormat,"assistant"), + ] + }; + + RedisExpand.SetTaskGPTReqCached(task, chatRep); + var chatResp = await chatClient.Chat(chatRep); + var chatResContent = chatResp?.res; + if (string.IsNullOrEmpty(chatResContent)) + throw new Exception("GPT返回message无效结果"); + if (chatResp != null) + RedisExpand.SetTaskGPTCached(task, new object[] { chatResp.Value.res, chatResp.Value.u }); + + chatResContent = chatResContent?.Replace("字幕内容", "课堂情况"); + chatResContent = chatResContent?.Replace("\n", ""); + chatResContent = chatResContent?.Replace("```json", ""); + chatResContent = chatResContent?.Replace("```", ""); + chatResContent = chatResContent?.Replace("}{", "},{"); + chatResContent = chatResContent?.Replace("}|{", "},{"); + chatResContent = chatResContent?.Trim(); + var startsStr = typeof(T).IsArray? "[" :"{"; + var endStr = typeof(T).IsArray? "]" : "}"; + if (!chatResContent.StartsWith(startsStr)) + chatResContent = startsStr + chatResContent; + if (!chatResContent.EndsWith(endStr)) + chatResContent = chatResContent + endStr; + var questionRes = JsonSerializer.Deserialize(chatResContent); + if (questionRes is null) + throw new Exception("ChatGPT返回无效结果"); + //var totalTokens = chatResp?.u.total_tokens ?? 0; + //if (totalTokens > 1) + //{ + // var tid = long.Parse(task); + // await videoTaskDB.AsUpdateable() + // .SetColumns(it => it.TotalTokens == totalTokens)//SetColumns是可以叠加的 写2个就2个字段赋值 + // .Where(it => it.Id == tid) + // .ExecuteCommandAsync(); + //} + return questionRes; } /// /// 访问GPT diff --git a/VideoAnalysisCore/AICore/GPT/Dto/QuestionRes.cs b/VideoAnalysisCore/AICore/GPT/Dto/QuestionRes.cs index 30d7007..d760934 100644 --- a/VideoAnalysisCore/AICore/GPT/Dto/QuestionRes.cs +++ b/VideoAnalysisCore/AICore/GPT/Dto/QuestionRes.cs @@ -10,6 +10,28 @@ using System.Threading.Tasks; namespace VideoAnalysisCore.AICore.GPT.Dto { + public class VideoKnowRes + { + /// + /// 问题解释 + /// + public float? 开始秒 { get; set; } + public float? 结束秒 { get; set; } + public float? 持续时间 => (结束秒 ?? 0) - 开始秒??0; + public string? 主题 { get; set; } + public string? 章节 { get; set; } + public string? 内容总结 { get; set; } + + } + public class FileNameInfo + { + /// + /// 问题解释 + /// + public string? 授课章节 { get; set; } + public string? 授课内容 { get; set; } + + } public class QuestionRes { /// diff --git a/VideoAnalysisCore/AICore/GPT/KIMI/KIMI_GPT.cs b/VideoAnalysisCore/AICore/GPT/KIMI/KIMI_GPT.cs index 36a2dd5..0e34f5d 100644 --- a/VideoAnalysisCore/AICore/GPT/KIMI/KIMI_GPT.cs +++ b/VideoAnalysisCore/AICore/GPT/KIMI/KIMI_GPT.cs @@ -23,7 +23,7 @@ namespace VideoAnalysisCore.AICore.GPT.KIMI /// /// kimi 文本模型 /// - public class KIMI_GPT : IBserGPT + public class KIMI_GPT // : IBserGPT { private readonly MoonshotClient moonshotClient; private readonly Repository criteriaDB; diff --git a/VideoAnalysisCore/AICore/SherpaOnnx/SenseVoice.cs b/VideoAnalysisCore/AICore/SherpaOnnx/SenseVoice.cs index f296b83..0c4dfef 100644 --- a/VideoAnalysisCore/AICore/SherpaOnnx/SenseVoice.cs +++ b/VideoAnalysisCore/AICore/SherpaOnnx/SenseVoice.cs @@ -233,7 +233,8 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx } await RedisExpand.Redis.HMSetAsync(RedisExpandKey.Task(task), "Captions", res); - RedisExpand.InsertChannel(Enum.RedisChannelEnum.ParsingSpeaker, task); + //RedisExpand.InsertChannel(Enum.RedisChannelEnum.ParsingSpeaker, task); + RedisExpand.InsertChannel(Enum.RedisChannelEnum.ChatModelAnalysis, task); } } diff --git a/VideoAnalysisCore/Common/AppCommon.cs b/VideoAnalysisCore/Common/AppCommon.cs index 1c5330b..9f9c210 100644 --- a/VideoAnalysisCore/Common/AppCommon.cs +++ b/VideoAnalysisCore/Common/AppCommon.cs @@ -4,12 +4,16 @@ using FreeRedis; using Microsoft.Extensions.DependencyModel; using SqlSugar; using SqlSugar.IOC; +using System.Collections.Generic; +using System.Linq; using System.Reflection; using System.Runtime.Loader; using System.Text; using System.Threading.Tasks; +using UserCenter.Model.Interface; using VideoAnalysisCore.AICore.SherpaOnnx; using VideoAnalysisCore.Enum; +using VideoAnalysisCore.Interface; using VideoAnalysisCore.Model.Dto; using Whisper.net; @@ -29,22 +33,26 @@ namespace VideoAnalysisCore.Common /// 主库数据库表类型 /// public static readonly IEnumerable DbMatserType; + public static readonly IEnumerable KnowsType; static AppCommon() { try { - Assemblies = ExpandFunction.GetAssemblies(); var assembliesType = Assemblies.Where(s => s.FullName.Contains("VideoAnalysis")).SelectMany(s => s.ExportedTypes .Where(u => !u.IsInterface && !u.IsAbstract && u.IsClass && u.IsDefined(typeof(SugarTable), false))); - DbMatserType = assembliesType; + DbMatserType = assembliesType + .Where(u => u.GetInterfaces().Contains(typeof(IDB))); + KnowsType = assembliesType + .Where(u =>u.GetInterfaces().Contains(typeof(IKnowsDB))); } catch { + throw; } - //.Where(u => !u.IsDefined(typeof(SplitTableAttribute), false)) - //.Where(u => !typeof(Model.DataCenterYH.IDataCenterYHModel).IsAssignableFrom(u)) - //.Where(u => !u.IsSubclassOf(typeof(YQ_BaseEntity))); + //.Where(u => !u.IsDefined(typeof(SplitTableAttribute), false)) + //.Where(u => !typeof(Model.DataCenterYH.IDataCenterYHModel).IsAssignableFrom(u)) + //.Where(u => !u.IsSubclassOf(typeof(YQ_BaseEntity))); } /// @@ -66,6 +74,13 @@ namespace VideoAnalysisCore.Common public static string AIModelFile = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "AICore", "_Static"); + /// + /// 获取视频路径 + /// + /// + /// + public static string GetVideoPath(string tid) => + Path.Combine(TaskCachedFile, tid, tid + ".mp4"); } /// @@ -140,35 +155,46 @@ namespace VideoAnalysisCore.Common { var captionsArr = RedisExpand.Redis.HMGet(RedisExpandKey.Task(task), "Captions").FirstOrDefault(); var speakerArr = RedisExpand.Redis.HMGet(RedisExpandKey.Task(task), "Speaker").FirstOrDefault(); - if (captionsArr is null || captionsArr.Length == 0 - || speakerArr is null || speakerArr.Length == 0) + if (captionsArr is null || captionsArr.Length == 0) + //|| speakerArr is null || speakerArr.Length == 0) throw new Exception("音频解析数据异常"); // 教师说话人Id - var techerId = speakerArr.GroupBy(s=>s.SpeakerIndex).Select(s => (s.Key,s.Sum(x=>x.Total))) + var techerId = speakerArr is null || !speakerArr.Any() + ? 0 + :speakerArr.GroupBy(s=>s.SpeakerIndex).Select(s => (s.Key,s.Sum(x=>x.Total))) .OrderByDescending(s=>s.Item2).First().Key; var teacherSpeaking = 0f; var studentSpeaking = 0f; var results = new Dictionary>(); - foreach (var segment in captionsArr) + if (speakerArr is null || speakerArr.Count() == 0) { - var spList = new List(); - foreach (var speakerRes in speakerArr) + var ss = new List { 1 }; + results = captionsArr.ToDictionary(s => s, s=> ss); + } + else + { + foreach (var segment in captionsArr) { - if (speakerRes.Start > segment.End) - break; - if (segment.Start <= speakerRes.End - && segment.End >= speakerRes.Start) + var spList = new List(); + foreach (var speakerRes in speakerArr) { - if (speakerRes.SpeakerIndex == techerId) - teacherSpeaking += speakerRes.Total; - else - studentSpeaking += speakerRes.Total; - spList.Add(speakerRes.SpeakerIndex); + if (speakerRes.Start > segment.End) + break; + if (segment.Start <= speakerRes.End + && segment.End >= speakerRes.Start) + { + if (speakerRes.SpeakerIndex == techerId) + teacherSpeaking += speakerRes.Total; + else + studentSpeaking += speakerRes.Total; + spList.Add(speakerRes.SpeakerIndex); + } } + var sp = spList.Distinct().ToList(); + if (sp.Count > 0) + results.Add(segment, sp); } - var sp = spList.Distinct().ToList(); - if(sp.Count>0) - results.Add(segment, sp); + } //拼接 提示词字幕源 var stringBuilder = new StringBuilder(); @@ -352,6 +378,10 @@ namespace VideoAnalysisCore.Common /// 数据库配置 /// public DBConfig DB { get; set; } = new DBConfig(); + /// + /// 知识点数据库 + /// + public DBConfig KnowsDB { get; set; } = new DBConfig(); } } diff --git a/VideoAnalysisCore/Common/RedisExpand.cs b/VideoAnalysisCore/Common/RedisExpand.cs index 4dcc4d9..bd6f154 100644 --- a/VideoAnalysisCore/Common/RedisExpand.cs +++ b/VideoAnalysisCore/Common/RedisExpand.cs @@ -213,7 +213,7 @@ namespace VideoAnalysisCore.Common if (scope is null || scope.ServiceProvider.GetService() is null) throw new Exception("IBserGPT 未注入"); else - return scope.ServiceProvider.GetService()?.CallGPT(task) ?? Task.CompletedTask; + return scope.ServiceProvider.GetService()?.GetKnow(task) ?? Task.CompletedTask; }); }); SubscribeList.Add(RedisChannelEnum.EndTask, @@ -277,6 +277,7 @@ namespace VideoAnalysisCore.Common .Where(it => it.Id == taskID) .ExecuteCommandAsync() == 1; } + /// /// 触发 /// diff --git a/VideoAnalysisCore/Common/Repository.cs b/VideoAnalysisCore/Common/Repository.cs index d1afcce..235516c 100644 --- a/VideoAnalysisCore/Common/Repository.cs +++ b/VideoAnalysisCore/Common/Repository.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; +using VideoAnalysisCore.Interface; using VideoAnalysisCore.Model; namespace VideoAnalysisCore.Common @@ -13,7 +14,10 @@ namespace VideoAnalysisCore.Common { public Repository() { - base.Context = DbScoped.SugarScope; + if(typeof(T).GetInterfaces().Contains(typeof(IKnowsDB))) + base.Context = DbScoped.SugarScope.GetConnection(1001); + else + base.Context = DbScoped.SugarScope; } } diff --git a/VideoAnalysisCore/Common/SqlSugarExpand.cs b/VideoAnalysisCore/Common/SqlSugarExpand.cs index 52d959a..a1493f3 100644 --- a/VideoAnalysisCore/Common/SqlSugarExpand.cs +++ b/VideoAnalysisCore/Common/SqlSugarExpand.cs @@ -27,6 +27,13 @@ namespace VideoAnalysisCore.Common ConnectionString = AppCommon.Config.DB.ConnectionString, DbType =AppCommon.Config.DB.SqlType, IsAutoCloseConnection = true//自动释放 + }, + new IocConfig() + { + ConfigId =1001, + ConnectionString = AppCommon.Config.KnowsDB.ConnectionString, + DbType =AppCommon.Config.KnowsDB.SqlType, + IsAutoCloseConnection = true//自动释放 }, }; services.AddSingleton(typeof(Repository<>)); diff --git a/VideoAnalysisCore/Interface/IDB.cs b/VideoAnalysisCore/Interface/IDB.cs new file mode 100644 index 0000000..9bb9511 --- /dev/null +++ b/VideoAnalysisCore/Interface/IDB.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace VideoAnalysisCore.Interface +{ + /// + /// 表属于IDB + /// + interface IDB + { + } +} diff --git a/VideoAnalysisCore/Interface/IKnowsDB.cs b/VideoAnalysisCore/Interface/IKnowsDB.cs new file mode 100644 index 0000000..4aa5abe --- /dev/null +++ b/VideoAnalysisCore/Interface/IKnowsDB.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace VideoAnalysisCore.Interface +{ + /// + /// 表属于KnowsDB + /// + interface IKnowsDB + { + } +} diff --git a/VideoAnalysisCore/Model/CourseGradingCriteria.cs b/VideoAnalysisCore/Model/CourseGradingCriteria.cs index d2e5323..1cfd29d 100644 --- a/VideoAnalysisCore/Model/CourseGradingCriteria.cs +++ b/VideoAnalysisCore/Model/CourseGradingCriteria.cs @@ -6,6 +6,7 @@ using System.Net; using UserCenter.Model.Enum; using VideoAnalysisCore.AICore.SherpaOnnx; using VideoAnalysisCore.Enum; +using VideoAnalysisCore.Interface; using Whisper.net; namespace VideoAnalysisCore.Model @@ -14,7 +15,7 @@ namespace VideoAnalysisCore.Model /// 课堂评分标准 /// [SugarTable("coursegradingcriteria")] - public class CourseGradingCriteria + public class CourseGradingCriteria: IDB { /// /// Id diff --git a/VideoAnalysisCore/Model/Dto/VideoTaskDto.cs b/VideoAnalysisCore/Model/Dto/VideoTaskDto.cs index 4b9fa01..574a58e 100644 --- a/VideoAnalysisCore/Model/Dto/VideoTaskDto.cs +++ b/VideoAnalysisCore/Model/Dto/VideoTaskDto.cs @@ -57,6 +57,11 @@ namespace VideoAnalysisCore.Model.Dto [DisplayName("媒体路径")] public string MediaUrl { get; set; } = string.Empty; /// + /// 媒体名称 + /// + [DisplayName("媒体名称")] + public string MediaName { get; set; } = string.Empty; + /// /// 自定义ID /// [DisplayName("自定义ID")] diff --git a/VideoAnalysisCore/Model/KnowledgeInfo.cs b/VideoAnalysisCore/Model/KnowledgeInfo.cs new file mode 100644 index 0000000..8eea235 --- /dev/null +++ b/VideoAnalysisCore/Model/KnowledgeInfo.cs @@ -0,0 +1,45 @@ +using SqlSugar; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using VideoAnalysisCore.Interface; + +namespace VideoAnalysisCore.Model.Dto +{ + [SugarTable("knowledgeinfo")] + public class KnowledgeInfo: IKnowsDB + { + [SugarColumn(IsPrimaryKey = true, ColumnDescription = "Id 主键", ColumnName = "id")] + public long Id { get; set; } + [SugarColumn(ColumnDescription = "自己数据库Id", ColumnName = "thisid")] + public long ThisId { get; set; } + [SugarColumn(InsertServerTime = true, ColumnDescription = "创建时间", ColumnName = "createtime")] + public DateTime CreateTime { get; set; } + [SugarColumn(UpdateServerTime = true, InsertServerTime = true, IsNullable = true, ColumnDescription = "更新时间", ColumnName = "updatetime")] + public DateTime UpdateTime { get; set; } + [SugarColumn(ColumnDescription = "是否删除 true=删除", DefaultValue = "false", ColumnName = "deletestate")] + public bool DeleteState { get; set; } + [SugarColumn(ColumnDescription = "排序", ColumnName = "ordinal", DefaultValue = "0")] + public int Ordinal { get; set; } + + [SugarColumn(ColumnDataType = "nvarchar(200)", ColumnDescription = "知识点名称", ColumnName = "name", IsNullable = true)] + public string Name { get; set; } + [SugarColumn(ColumnDescription = "节点深度，一级节点的深度为1，二级节点的深度为2，以此类推。", ColumnName = "depth", IsNullable = true)] + public long Depth { get; set; } + [SugarColumn(ColumnDescription = "适用于精简版", ColumnName = "for_lite", IsNullable = true)] + public bool For_Lite { get; set; } + [SugarColumn(ColumnDescription = "父节点ID", ColumnName = "parent_id", IsNullable = true)] + public long Parent_Id { get; set; } + [SugarColumn(ColumnDescription = "root节点的ID", ColumnName = "root_id", IsNullable = true)] + public long Root_Id { get; set; } + [SugarColumn(ColumnDataType = "nvarchar(100)", ColumnDescription = "节点类型，可用值：NODE、KNOWLEDGE_POINT、TESTING_POINT，分别代表普通节点、知识点、考点。", ColumnName = "type", IsNullable = true)] + public string Type { get; set; } + [SugarColumn(ColumnDescription = "课程ID", ColumnName = "course_id", IsNullable = true)] + public long Course_Id { get; set; } + [SugarColumn(InsertServerTime = true, ColumnDescription = "学科网创建时间", ColumnName = "create_time", IsNullable = true)] + public DateTime Create_Time { get; set; } + } + +} diff --git a/VideoAnalysisCore/Model/VideoTask.cs b/VideoAnalysisCore/Model/VideoTask.cs index ae9f489..f7afd95 100644 --- a/VideoAnalysisCore/Model/VideoTask.cs +++ b/VideoAnalysisCore/Model/VideoTask.cs @@ -6,6 +6,7 @@ using UserCenter.Model.Enum; using VideoAnalysisCore.AICore.GPT.Dto; using VideoAnalysisCore.AICore.SherpaOnnx; using VideoAnalysisCore.Enum; +using VideoAnalysisCore.Interface; using Whisper.net; namespace VideoAnalysisCore.Model @@ -14,7 +15,7 @@ namespace VideoAnalysisCore.Model /// 视频任务模型 /// [SugarTable("videotask")] - public class VideoTask + public class VideoTask: IDB { /// /// 任务id @@ -27,6 +28,10 @@ namespace VideoAnalysisCore.Model /// public string MediaUrl { get; set; } = string.Empty; /// + /// 媒体文件名称 + /// + public string MediaName { get; set; } = string.Empty; + /// /// 下载后本地媒体目录 /// public string LocalMediaPath { get; set; } = string.Empty;