From c39fbb750175198f90947cb3a6dfabab1dda006f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E8=82=A5=E7=BE=8A?= <1048382248@qq.com> Date: Tue, 11 Mar 2025 14:22:56 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=20AI=E5=88=86=E6=9E=90?= =?UTF-8?q?=E6=B5=81=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VideoAnalysis/Controllers/LJZK_Controller.cs | 57 ++++++- VideoAnalysis/Expand/CoravelExpand.cs | 7 +- VideoAnalysis/Expand/SqlSugarExpand.cs | 8 +- VideoAnalysis/Learn.VideoAnalysis.csproj | 1 + VideoAnalysis/Program.cs | 11 +- VideoAnalysis/appsettings.json | 9 +- .../AICore/GPT/DeepSeek/DeepSeek_GPT.cs | 144 +++++++++++++----- .../AICore/GPT/Dto/QuestionRes.cs | 40 +++-- .../AICore/SherpaOnnx/SenseVoice.cs | 5 +- .../AICore/SherpaOnnx/Speaker.cs | 7 +- .../Common/AlibabaCloudVodExpand.cs | 56 +++++++ VideoAnalysisCore/Common/AppCommon.cs | 48 ++++++ VideoAnalysisCore/Common/DownloadFile.cs | 14 +- VideoAnalysisCore/Common/RedisExpand.cs | 24 +-- VideoAnalysisCore/Job/NodeSubscriptionJob.cs | 65 +++++--- VideoAnalysisCore/Job/TaskFileClearJob.cs | 79 ++++++++++ VideoAnalysisCore/VideoAnalysisCore.csproj | 1 + 17 files changed, 476 insertions(+), 100 deletions(-) create mode 100644 VideoAnalysisCore/Common/AlibabaCloudVodExpand.cs create mode 100644 VideoAnalysisCore/Job/TaskFileClearJob.cs diff --git a/VideoAnalysis/Controllers/LJZK_Controller.cs b/VideoAnalysis/Controllers/LJZK_Controller.cs index 8ced3cc..879e8eb 100644 --- a/VideoAnalysis/Controllers/LJZK_Controller.cs +++ b/VideoAnalysis/Controllers/LJZK_Controller.cs @@ -24,14 +24,18 @@ namespace Learn.VideoAnalysis.Controllers private readonly ILogger _logger; private readonly IMapper mp; private readonly Repository nodesubscriptionDB; + private readonly Repository videoTaskDB; + private readonly Repository videoKonwPointDB; private readonly IBserGPT chatGPT; public LJZK_Controller(ILogger logger, - IMapper mp, IBserGPT chatGPT, Repository nodesubscriptionDB) + IMapper mp, IBserGPT chatGPT, Repository nodesubscriptionDB, Repository videoTaskDB = null, Repository videoKonwPointDB = null) { _logger = logger; this.mp = mp; this.chatGPT = chatGPT; this.nodesubscriptionDB = nodesubscriptionDB; + this.videoTaskDB = videoTaskDB; + this.videoKonwPointDB = videoKonwPointDB; } @@ -43,12 +47,16 @@ namespace Learn.VideoAnalysis.Controllers [HttpPost(Name = "NodeSubscription")] public async Task NodeSubscription(NodeMonitoringReq req) { + if(req is null || req.NodeId ==0) + return BadRequest("无效的提交数据"); if (nodesubscriptionDB.IsAny(s => s.NodeId == req.NodeId)) return BadRequest("重复添加了节点监控任务" + req.NodeId); var res = await nodesubscriptionDB.InsertReturnEntityAsync(new NodeSubscription() { NodeId = req.NodeId, - TaskType = req.Type ?? default + TaskType = req.Type ?? default, + Subject = req.Subject?? default, + }); return Ok(res); } @@ -76,7 +84,52 @@ namespace Learn.VideoAnalysis.Controllers .Select(s => new { Text = s.ToString(), Value = (int)s })); } + /// + /// 获取视频知识点片段taskId/tagId二选一 + /// + /// + /// 自定义id + /// + [HttpGet(Name = "TaskKnowInfo")] + public async Task TaskKnowInfo(long taskId, string? tagId) + { + if (taskId == 0 && string.IsNullOrEmpty(tagId)) + return BadRequest(); + var task = await videoTaskDB.AsQueryable() + .WhereIF(taskId != 0, s => s.Id == taskId) + .WhereIF(!string.IsNullOrEmpty(tagId), s => s.TagId == tagId) + .FirstAsync(); + if (task is null) + return BadRequest("无效任务"); + var konwArr = await videoKonwPointDB.AsQueryable() + .Where(s => s.VideoTaskId == task.Id) + .ToArrayAsync(); + if (konwArr is null || konwArr.Length == 0) + return BadRequest("无效任务"); + return Ok(new TaskKnowRes() + { + TagId = task.TagId, + Status = task.LastEnum, + VideoTaskId = task.Id, + KnowBlockArr = konwArr + .GroupBy(s => s.StartTime) + .Select(s => new TaskKnowBlock() + { + Id = s.First().Id, + Content = s.First().Content, + StartTime = s.First().StartTime, + EndTime = s.First().EndTime, + Theme = s.First().Theme, + Know = s.Select(x => new TaskKnowInfo() + { + Id = x.Id, + KnowPoint = x.KnowPoint, + KnowPointId = x.KnowPointId + }).ToArray() + }).ToArray() + }); + } diff --git a/VideoAnalysis/Expand/CoravelExpand.cs b/VideoAnalysis/Expand/CoravelExpand.cs index 77d3aa2..06a2727 100644 --- a/VideoAnalysis/Expand/CoravelExpand.cs +++ b/VideoAnalysis/Expand/CoravelExpand.cs @@ -13,9 +13,10 @@ namespace Learn.VideoAnalysis.Expand public static class CoravelExpand { public static int MyProperty { get; set; } - public static void Init(IServiceCollection service) + public static void AddCoravel(this IServiceCollection service) { + Console.WriteLine($"{DateTime.Now}=>鍒濆鍖 Coravel"); service.AddScheduler(); service.AddTransient(); } @@ -24,7 +25,9 @@ namespace Learn.VideoAnalysis.Expand provider.UseScheduler(scheduler => { //姣5鍒嗛挓鎵ц涓娆 鏈鐞嗚棰戞壂鎻 - scheduler.Schedule().EveryFiveMinutes(); + //scheduler.Schedule().EveryFiveMinutes(); + //姣忓ぉ涓ょ偣 + scheduler.Schedule().DailyAtHour(2); }); } } diff --git a/VideoAnalysis/Expand/SqlSugarExpand.cs b/VideoAnalysis/Expand/SqlSugarExpand.cs index 8847e02..fd80c3b 100644 --- a/VideoAnalysis/Expand/SqlSugarExpand.cs +++ b/VideoAnalysis/Expand/SqlSugarExpand.cs @@ -5,6 +5,7 @@ using SqlSugar; using SqlSugar.IOC; using System; using System.Collections.Generic; +using System.Configuration; using System.Diagnostics; using System.Linq; using System.Reflection; @@ -17,8 +18,13 @@ namespace Learn.VideoAnalysis.Expand public static class SqlSugarExpand { public static bool ShowSQL = false; - public static void InitSqlSugar(this IServiceCollection services) + public static void AddSqlSugarExpand(this IServiceCollection services) { + + Console.WriteLine($"{DateTime.Now}=>鍒濆鍖 YitId闆姳ID"); + var options = new IdGeneratorOptions(ushort.Parse(AppCommon.Config.ID)); + YitIdHelper.SetIdGenerator(options); + #region SqlSugar娉ㄥ叆 var dbList = new List() { new IocConfig() diff --git a/VideoAnalysis/Learn.VideoAnalysis.csproj b/VideoAnalysis/Learn.VideoAnalysis.csproj index 8844de3..bb84fdb 100644 --- a/VideoAnalysis/Learn.VideoAnalysis.csproj +++ b/VideoAnalysis/Learn.VideoAnalysis.csproj @@ -31,6 +31,7 @@ + diff --git a/VideoAnalysis/Program.cs b/VideoAnalysis/Program.cs index 0394844..3587e85 100644 --- a/VideoAnalysis/Program.cs +++ b/VideoAnalysis/Program.cs @@ -53,14 +53,15 @@ namespace Learn.VideoAnalysis c.OrderActionsBy(o => o.RelativePath); // 对action的名称进行排序，如果有多个，就可以看见效果了。 }); - //绑定 appsetting 配置 + //绑定 appsetting 配置 builder.Configuration.GetSection("AppConfig").Bind(AppCommon.Config); //初始化 插件 - builder.Services.InitSqlSugar(); - RedisExpand.Init(); - Speaker.Init(); - CoravelExpand.Init(builder.Services); + builder.Services.AddSqlSugarExpand(); + builder.Services.AddAlibabaCloudVod(); + builder.Services.AddRedisExpand(); + builder.Services.AddSpeakerAI(); + builder.Services.AddCoravel(); //SenseVoice.Init(); //异常过滤器 builder.Services.AddControllersWithViews(options => diff --git a/VideoAnalysis/appsettings.json b/VideoAnalysis/appsettings.json index d184168..ddfd483 100644 --- a/VideoAnalysis/appsettings.json +++ b/VideoAnalysis/appsettings.json @@ -7,7 +7,7 @@ }, "AllowedHosts": "*", "AppConfig": { - "ID": "APP0001", //绋嬪簭鍞竴鍊 + "ID": 1, //绋嬪簭鍞竴鍊 "Admin": { "Account": "admin", "Password": "q1w2e3!@#" @@ -43,7 +43,12 @@ "DB": { "ConnectionString": "AllowLoadLocalInfile=true;Server=192.168.2.9;User ID=root;Password=qwe123!@#;Port=3306;Database=learn.videoanalysis;CharSet=utf8mb4;pooling=true;SslMode=None", "SqlType": "MySql", - "UpdateTable": true + "UpdateTable": false + }, + "AlibabaCloudVod": { + "AccessKeyId": "LTAI5tDC6p9h747B7FHbgwkH", + "AccessKeySecret": "vRKgmbp1LB05LaGOjh3ZrZxbHSLYLF", + "EndPoint": "vod.cn-shanghai.aliyuncs.com" //涓婁紶鑺傜偣 }, "OtherDBArr": [ { diff --git a/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs b/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs index 26c7806..e747dce 100644 --- a/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs +++ b/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs @@ -11,6 +11,7 @@ using VideoAnalysisCore.AICore.GPT.ChatGPT; using VideoAnalysisCore.AICore.SherpaOnnx; using VideoAnalysisCore.Model.钃濋哺鏅哄簱; using VideoAnalysisCore.Model.Enum; +using Mapster; namespace VideoAnalysisCore.AICore.GPT.DeepSeek { @@ -51,18 +52,30 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek .Where(s => s.Id == taskId) .FirstAsync(); var subject = "鏁板"; + var Course_Id = 27; + switch (taskInfo.Type)//澶勭悊涓嶅悓浠诲姟绫诲瀷鐨勭煡璇嗙偣鏍 + { + case TaskTypeEnum.钃濋哺鏅哄簱_涓亴瑙嗛鍒嗘: + Course_Id = 51; + break; + case TaskTypeEnum.钃濋哺鏅哄簱_瑙嗛鍒嗘: + default: + Course_Id = 27; + break; + } var xkwKnows = await knowledgeInfoDB.AsQueryable() - .Where(s => s.Course_Id == 27 + .Where(s => s.Course_Id == Course_Id && (s.Depth == 3 - || s.Depth == 2 )) + || s.Depth == 2)) .Select(s => s.Name).ToArrayAsync(); string title = taskInfo.MediaName; var fileNameResFormat = "{鎺堣绔犺妭: string|null}"; var fileNamePostMessages = title + " 杩欐槸涓鍫傝鐨勬爣棰,璇蜂綘鍩轰簬鏍囬甯垜鍒嗘瀽鍑鸿繖鍫傝鎵璁叉巿鐨勫唴瀹逛笌鏈鎭板綋鐨勬巿璇剧珷鑺(鍏宠仈鏈璐村垏鐨勭珷鑺,淇濈暀涓涓珷鑺!)." + - $"绔犺妭鑼冨洿闄愬畾鍦╗{string.Join(',', xkwKnows)}]鑼冨洿." + + $"绔犺妭鑼冨洿闄愬畾鍦╗{string.Join(',', xkwKnows)}]鑼冨洿鍐." + $"杈撳嚭鏍煎紡 json瀛楃涓 瀵硅薄鏍煎紡{fileNameResFormat}"; - var fileNameInfoRes = await ChatAsync(task, fileNamePostMessages, null);//, "deepseek-chat"); + var fileNameInfoRes = await ChatAsync + (task, fileNamePostMessages, null);//, "deepseek-chat"); var speakerArr = JsonSerializer.Deserialize(taskInfo.Speaker); var captionsArr = JsonSerializer.Deserialize(taskInfo.Captions); @@ -70,31 +83,42 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek var maxVideoTime = captions?.TimeBase?.LastOrDefault()?.End ?? 0; var criteriaBuilder = new StringBuilder(); - var resFormat = """[{"StartTime":寮濮嬬(number),"Theme":涓婚(string),"KnowPoint":涓婚鏂规硶鐐(string),"KnowPointId":涓婚鏂规硶鐐笽d(string)"Content":鍐呭鎬荤粨(string)}]"""; - var know = await knowledgeInfoDB.GetFirstAsync(s =>s.Course_Id==27 && s.Name == fileNameInfoRes.鎺堣绔犺妭); + var know = await knowledgeInfoDB.GetFirstAsync(s => s.Course_Id == Course_Id && s.Name == fileNameInfoRes.鎺堣绔犺妭); if (know is null) - throw new Exception("鏈兘鎵惧埌瀵瑰簲鐭ヨ瘑鐐=>"+ fileNameInfoRes.鎺堣绔犺妭); - var knowledgeInfos = await knowledgeInfoDB.AsQueryable().ToChildListAsync(s => s.Parent_Id, know.Id); - var knows = string.Join(',', knowledgeInfos.Select(s => s.Id + "|"+ s.Name)); + throw new Exception("鏈兘鎵惧埌瀵瑰簲鐭ヨ瘑鐐=>" + fileNameInfoRes.鎺堣绔犺妭); + //鎻愬崌鍒扮埗绾 + var kInfo = await knowledgeInfoDB.GetByIdAsync(know.Parent_Id); + var knowledgeInfos = await knowledgeInfoDB.AsQueryable().ToChildListAsync(s => s.Parent_Id, kInfo.Parent_Id == 0? kInfo.Id: kInfo.Parent_Id); + var knows = string.Join(',', knowledgeInfos.Select(s => s.Id + "|" + s.Name)); var knowDic = knowledgeInfos - .OrderBy(s=>s.Id) - .GroupBy(s=>s.Name) + .OrderBy(s => s.Id) + .GroupBy(s => s.Name) .ToDictionary(s => s.First().Name, s => s.First().Id); VideoKnowRes[] questionRes; while (true) { + // var postMessages = + //$"浣犵殑浠诲姟鏄垎鏋愯棰戝瓧骞曞唴瀹瑰苟鎻愬彇鍑轰腑鍥介珮鑰冭冭瘯璇曢鏂规硶鐐,鐒跺悗鍒嗘瀽鍑<鐭ヨ瘑鍧>,鏉ュ府鍔╁鐢熷揩閫熶簡瑙ｈ棰戝瓧骞曠殑鍐呭" + + //$"閫氳繃闃呰骞剁悊瑙ｅ瓧骞曞唴瀹.鐒跺悗璇嗗埆鍑簕subject}瀛︾涓睘浜巤fileNameInfoRes.鎺堣绔犺妭}绔犺妭鐩稿叧鐨勬柟娉曠偣浠ュ強瀵瑰簲鐨勬椂闂存銆" + + //$"鍏宠仈鍚堝苟鐭ヨ瘑鍐呭鐩镐技鐨勭煡璇嗙偣鏉ュ悎骞朵负<鐭ヨ瘑鍧>銆" + + //$"鍒嗛厤绌轰綑鏈娇鐢ㄧ殑鏃堕棿娈靛埌鍐呭鐩歌繎鐨<鐭ヨ瘑鍧>鏃堕棿鍖洪棿鏉ヨ幏鍙栨洿鍔犺缁嗙殑涓婁笅鏂,浣嗘槸璇烽伩鍏<鐭ヨ瘑鍧>涔嬮棿鏃堕棿閲嶅悎銆" + + //$"浠庢彁鍙栧嚭鐨<鐭ヨ瘑鍧>鍐呭绉版潵鍖归厤鎴戞彁渚涚殑鐭ヨ瘑鐐,鏉ヤ綔涓虹墖娈电殑鐭ヨ瘑鐐(璇风‘淇濆尮閰嶇殑鐭ヨ瘑鐐规槸鐢ㄦ埛鎻愪緵鐨,鍚﹀垯鐗囨鐭ヨ瘑鐐瑰间负绌哄瓧绗︿覆)銆" + + //$"鎻愪緵鐨勬柟娉曠偣鍚嶇О({knows})銆 鏍煎紡 (鏂规硶鐐笽d|鏂规硶鐐瑰悕绉) 濡傛灉涓涓<鐭ヨ瘑鍧>鍑虹幇澶氫釜鐭ヨ瘑鐐归偅涔堢煡璇嗙偣Id涓庣煡璇嗙偣鍚嶇О閮界敤閫楀彿','鍒嗗壊銆" + + //$"杩欐槸杈撳叆鐨勮棰戝瓧骞曞苟涓旀槸鍖呭惈鏃堕棿鎴崇殑瑙嗛瀛楀箷鐨勫浐瀹氭牸寮忔枃鏈" + + ////$"瀛楀箷鏍煎紡(璇磋瘽浜:寮濮嬬:缁撴潫绉:鍐呭|涓嬩竴娈靛瓧骞).浠ヤ笅鏄寘鍚椂闂寸殑瑙嗛瀛楀箷鏂囨湰銆傚瓧骞曞垪琛 {captions.Captions}銆" + + //$"瀛楀箷鏍煎紡(寮濮嬬:缁撴潫绉:鍐呭|涓嬩竴娈靛瓧骞).浠ヤ笅鏄寘鍚椂闂寸殑瑙嗛瀛楀箷鏂囨湰銆傚瓧骞曞垪琛 {captions.Captions}銆" + + //$"鏈鍚庤妫鏌ユ煇浜<鐭ヨ瘑鍧>涔嬮棿鐨勮繃娓℃槸鍚﹁嚜鐒,濡傛灉<鐭ヨ瘑鍧>鏃堕暱瓒呰繃500绉掑垯鑰冭檻鎷嗗皝涓轰袱涓洿鍔犺创鍒囩殑<鐭ヨ瘑鍧>.鎴栬<鐭ヨ瘑鍧>鏃堕暱灏忎簬30绉掑垯鑰冭檻鍚堝苟<鐭ヨ瘑鍧>鍒扮浉閭荤殑<鐭ヨ瘑鍧>)銆" + + //$"杈撳嚭鍐呭鍙繑鍥瀓son鏍煎紡({resFormat})"; + + var resFormat = """[{"StartTime":寮濮嬬(number),"Theme":涓婚(string),"Content":鍐呭鎬荤粨(string)}]"""; var postMessages = $"浣犵殑浠诲姟鏄垎鏋愯棰戝瓧骞曞唴瀹瑰苟鎻愬彇鍑轰腑鍥介珮鑰冭冭瘯璇曢鏂规硶鐐,鐒跺悗鍒嗘瀽鍑<鐭ヨ瘑鍧>,鏉ュ府鍔╁鐢熷揩閫熶簡瑙ｈ棰戝瓧骞曠殑鍐呭" + - $"閫氳繃闃呰骞剁悊瑙ｅ瓧骞曞唴瀹.鐒跺悗璇嗗埆鍑簕subject}瀛︾涓睘浜巤fileNameInfoRes.鎺堣绔犺妭}绔犺妭鐩稿叧鐨勬柟娉曠偣浠ュ強瀵瑰簲鐨勬椂闂存銆" + + $"閫氳繃闃呰骞剁悊瑙ｅ瓧骞曞唴瀹.鐒跺悗璇嗗埆鍑簕subject}瀛︾涓睘浜巤fileNameInfoRes.鎺堣绔犺妭}绔犺妭鐩稿叧鐨勬椂闂存銆" + $"鍏宠仈鍚堝苟鐭ヨ瘑鍐呭鐩镐技鐨勭煡璇嗙偣鏉ュ悎骞朵负<鐭ヨ瘑鍧>銆" + $"鍒嗛厤绌轰綑鏈娇鐢ㄧ殑鏃堕棿娈靛埌鍐呭鐩歌繎鐨<鐭ヨ瘑鍧>鏃堕棿鍖洪棿鏉ヨ幏鍙栨洿鍔犺缁嗙殑涓婁笅鏂,浣嗘槸璇烽伩鍏<鐭ヨ瘑鍧>涔嬮棿鏃堕棿閲嶅悎銆" + - $"浠庢彁鍙栧嚭鐨<鐭ヨ瘑鍧>鍐呭绉版潵鍖归厤鎴戞彁渚涚殑鐭ヨ瘑鐐,鏉ヤ綔涓虹墖娈电殑鐭ヨ瘑鐐(璇风‘淇濆尮閰嶇殑鐭ヨ瘑鐐规槸鐢ㄦ埛鎻愪緵鐨,鍚﹀垯鐗囨鐭ヨ瘑鐐瑰间负绌哄瓧绗︿覆)銆" + - $"鎻愪緵鐨勬柟娉曠偣鍚嶇О({knows})銆 鏍煎紡 (鏂规硶鐐笽d|鏂规硶鐐瑰悕绉) 濡傛灉涓涓<鐭ヨ瘑鍧>鍑虹幇澶氫釜鐭ヨ瘑鐐归偅涔堢煡璇嗙偣Id涓庣煡璇嗙偣鍚嶇О閮界敤閫楀彿','鍒嗗壊銆" + - $"杩欐槸杈撳叆鐨勮棰戝瓧骞曞苟涓旀槸鍖呭惈鏃堕棿鎴崇殑瑙嗛瀛楀箷鐨勫浐瀹氭牸寮忔枃鏈" + - //$"瀛楀箷鏍煎紡(璇磋瘽浜:寮濮嬬:缁撴潫绉:鍐呭|涓嬩竴娈靛瓧骞).浠ヤ笅鏄寘鍚椂闂寸殑瑙嗛瀛楀箷鏂囨湰銆傚瓧骞曞垪琛 {captions.Captions}銆" + $"瀛楀箷鏍煎紡(寮濮嬬:缁撴潫绉:鍐呭|涓嬩竴娈靛瓧骞).浠ヤ笅鏄寘鍚椂闂寸殑瑙嗛瀛楀箷鏂囨湰銆傚瓧骞曞垪琛 {captions.Captions}銆" + $"鏈鍚庤妫鏌ユ煇浜<鐭ヨ瘑鍧>涔嬮棿鐨勮繃娓℃槸鍚﹁嚜鐒,濡傛灉<鐭ヨ瘑鍧>鏃堕暱瓒呰繃500绉掑垯鑰冭檻鎷嗗皝涓轰袱涓洿鍔犺创鍒囩殑<鐭ヨ瘑鍧>.鎴栬<鐭ヨ瘑鍧>鏃堕暱灏忎簬30绉掑垯鑰冭檻鍚堝苟<鐭ヨ瘑鍧>鍒扮浉閭荤殑<鐭ヨ瘑鍧>)銆" + - $"杈撳嚭鏍煎紡({resFormat})"; + $"杈撳嚭鍐呭鍙繑鍥瀓son鏍煎紡({resFormat})"; Console.WriteLine(DateTime.Now + "=>寮濮嬪垎鏋愯棰戝唴瀹"); @@ -103,37 +127,58 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek if (questionRes.Length <= 3) throw new Exception("瑙嗛鍒嗘鏁伴噺杩囦綆 =>" + questionRes.Length); questionRes = questionRes.OrderBy(s => s.StartTime).ToArray(); - for (int i = 0; i < questionRes.Length; i++) - { - var item = questionRes[i]; - if (i == questionRes.Length - 1) - item.EndTime = maxVideoTime; - else - item.EndTime = (int)(questionRes[i + 1]?.StartTime ?? 0) - 1; - } - var thems = string.Join(',', questionRes.Select(s => s.StartTime + "->" + s.Theme)); - var checkResFormat = """{"Score":鎵撳垎(number),"Evaluation":璇勪环(string),"Data":浼樺寲鍚庣殑鍒嗘(array)}"""; - var checkResFormat1 = """[{"Start":寮濮嬬(number),"Theme":浼樺寲鍚庣殑鍒嗘涓婚(string)}]"""; - var checkMessage = "鎴戜负瑙嗛鐨勮瑙ｅ唴瀹瑰仛浜嗕竴浜涘垎娈,浣犺兘甯垜妫鏌ヤ笅杩欎簺鍒嗘鐨勬椂闂村垎閰嶆槸鍚﹀悎鐞,鏍囬鍐呭绗﹀悎瀹為檯鍚?" + - $"璇风粰鍑轰綘鐨勬墦鍒(0-100,70鍒嗗強鏍)浠ュ強鎵撳垎鍘熷洜,骞朵笖缁欏嚭浼樺寲鍚庣殑鍒嗘 鍒嗘鏍煎紡(${checkResFormat1})" + + var thems = JsonSerializer.Serialize(questionRes.Adapt());// string.Join(',', questionRes.Select(s => s.StartTime + "->" + s.Theme)); + var checkResFormat1 = """[{"StartTime":寮濮嬬(number),"KnowPoint":鐭ヨ瘑鐐瑰悕绉(string),"KnowPointId":鐭ヨ瘑鐐笽d(string)}]"""; + var knowStr = string.Join(',', knowledgeInfos.Select(s => s.Name)); + var knowMessages = + $"鎴戦拡瀵硅棰<{title}>鍒嗘瀽鍑轰簡涓浜涜棰戠殑鐭ヨ瘑鐗囨,鐜板湪闇瑕佷綘甯垜灏嗘瘡涓墖娈靛垎閰嶆伆褰撶殑鐭ヨ瘑鐐(鍗曚釜鐗囨鍏佽澶氫釜鐭ヨ瘑鐐圭敤閫楀彿','鍒嗗壊)銆" + + $"杩欐槸鎴戠殑鍒嗘 {thems}銆" + + $"鎻愪緵鐨勭煡璇嗙偣鍚嶇О({knows})銆 鏍煎紡 (鏂规硶鐐笽d|鏂规硶鐐瑰悕绉) " + + $"鏈鍚庤纭繚鍒嗛厤鐨勭煡璇嗙偣鏄敤鎴锋彁渚涚殑,鍚﹀垯鐗囨鐭ヨ瘑鐐瑰肩暀绌!銆" + + $"杈撳嚭鍐呭鍙繑鍥瀓son鏍煎紡({checkResFormat1})"; + Console.WriteLine(DateTime.Now + "=>寮濮嬪垎鏋愯棰戝唴瀹圭煡璇嗙偣"); + var konwRes = await ChatAsync(task, knowMessages, null); + + + var checkResFormat = """{"Score":鎵撳垎(number),"Evaluation":璇勪环(string)""";//,"Data":浼樺寲鍚庣殑鍒嗘(array)}"""; + var checkMessage = "鎴戜负瑙嗛鐨勮瑙ｅ唴瀹瑰仛浜嗕竴浜涘垎娈,浣犺兘甯垜妫鏌ヤ笅杩欎簺鍒嗘鐨勬椂闂,涓婚,鐭ヨ瘑鐐瑰垎閰嶆槸鍚﹀悎鐞嗙鍚堝疄闄呭悧?" + + $"璇风粰鍑轰綘鐨勬墦鍒(0-100,70鍒嗗強鏍)浠ュ強鎵撳垎鍘熷洜" +//,骞朵笖缁欏嚭浼樺寲鍚庣殑鍒嗘 鍒嗘鏍煎紡(${checkResFormat1})" + + //$"濡傛灉涓嶅悎鐞嗙殑璇濆己甯垜淇敼浼樺寲浠栦滑(娉ㄦ剰浼樺寲鐭ヨ瘑鐐规椂鍙厑璁镐娇鐢ㄥ凡瀛樺湪鐨勭煡璇嗙偣涓嶅厑璁告潨鎾扮寽娴嬫崗閫)" + $"杩欐槸鎴戠殑鍒嗘 {thems}." + $"鍚庣画鐨勫唴瀹规槸鍖呭惈鏃堕棿鎴崇殑瑙嗛瀛楀箷鐨勫浐瀹氭牸寮忔枃鏈" + $"瀛楀箷鏍煎紡(璇磋瘽浜:寮濮嬬:缁撴潫绉:鍐呭|涓嬩竴娈靛瓧骞).浠ヤ笅鏄寘鍚椂闂寸殑瑙嗛瀛楀箷鏂囨湰銆傚瓧骞曞垪琛 {captions.Captions}銆" + - $"杈撳嚭鏍煎紡涓簀son({checkResFormat})"; + $"鏈鍚庤緭鍑烘牸寮忎负json({checkResFormat})"; Console.WriteLine(DateTime.Now + "=>寮濮嬫鏌ヨ棰戝垎娈电粨鏋"); - var checkRes = await ChatAsync (task, checkMessage, null); - if (checkRes != null && checkRes.Score >= 80) + var checkRes = await ChatAsync(task, checkMessage, null); + if (checkRes != null && checkRes.Score >= 80 && questionRes.Count() == checkRes.Data.Count()) + { + for (int i = 0; i < checkRes.Data.Count(); i++) + { + questionRes[i].Theme = checkRes.Data[i].Theme; + questionRes[i].StartTime = checkRes.Data[i].StartTime; + } break; + } else { - Console.WriteLine(DateTime.Now + $"=>{task} 寰楀垎杩囦綆 " + checkRes?.Score ); - Console.WriteLine( checkRes.Evaluation); + + Console.WriteLine(DateTime.Now + $"=>{task} 寰楀垎杩囦綆/鍒嗘闀垮害涓嶅尮閰 寰楀垎{checkRes?.Score} 闀垮害 {questionRes.Count()}/{checkRes.Data.Count()}"); + Console.WriteLine(checkRes.Evaluation); Console.WriteLine(); } } + for (int i = 0; i < questionRes.Length; i++) + { + var item = questionRes[i]; + if (i == questionRes.Length - 1) + item.EndTime = maxVideoTime; + else + item.EndTime = (int)(questionRes[i + 1]?.StartTime ?? 0) - 1; + } + //todo 鏈寘鍚殑鐭ヨ瘑鐐圭墖娈 濡備綍澶勭悊 var insertData = questionRes .Where(s => !string.IsNullOrEmpty(s.KnowPoint)) @@ -169,7 +214,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek RedisExpand.InsertChannel(RedisChannelEnum.EndTask, task); return gptRes; } - public async Task ChatAsync(string task, string postMessages, string postMessages1, string model= "deepseek-reasoner") + public async Task ChatAsync(string task, string postMessages, string postMessages1, string model = "deepseek-reasoner") { var maxTokens = 4000; Message[] messageArr = [ @@ -180,7 +225,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek var chatRep = new ChatRequest { model = model, - stream = model== "deepseek-reasoner", + stream = model == "deepseek-reasoner", max_tokens = maxTokens, temperature = 0.2f, messages = messageArr @@ -200,17 +245,32 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek chatResContent = chatResContent?.Replace("```", ""); chatResContent = chatResContent?.Replace("}{", "},{"); chatResContent = chatResContent?.Replace("}|{", "},{"); - chatResContent = chatResContent?.Trim(); + chatResContent = chatResContent?.Trim().ExtractJson().FirstOrDefault(); + var startsStr = typeof(T).IsArray ? "[" : "{"; var endStr = typeof(T).IsArray ? "]" : "}"; if (!chatResContent.StartsWith(startsStr)) chatResContent = startsStr + chatResContent; if (!chatResContent.EndsWith(endStr)) chatResContent = chatResContent + endStr; - var questionRes = JsonSerializer.Deserialize(chatResContent); - if (questionRes is null) - throw new Exception("ChatGPT杩斿洖鏃犳晥缁撴灉"); - return questionRes; + var options = new JsonSerializerOptions + { + // 鍏佽瑙ｆ瀽涓嶄弗鏍肩鍚 JSON 瑙勮寖鐨勫瓧绗︿覆 + AllowTrailingCommas = true, + // 澶勭悊涓嶅尮閰嶇殑 JSON 瀛楃 + ReadCommentHandling = JsonCommentHandling.Skip + }; + try + { + var questionRes = JsonSerializer.Deserialize(chatResContent, options); + if (questionRes is null) + throw new Exception("ChatGPT杩斿洖鏃犳晥缁撴灉"); + return questionRes; + } + catch (Exception ex) + { + throw new Exception("ChatGPT缁撴灉瑙ｆ瀽閿欒 " + ex.Message + ex.Message); + } } } } diff --git a/VideoAnalysisCore/AICore/GPT/Dto/QuestionRes.cs b/VideoAnalysisCore/AICore/GPT/Dto/QuestionRes.cs index 88ab059..bc8a6c1 100644 --- a/VideoAnalysisCore/AICore/GPT/Dto/QuestionRes.cs +++ b/VideoAnalysisCore/AICore/GPT/Dto/QuestionRes.cs @@ -10,33 +10,53 @@ using System.Threading.Tasks; namespace VideoAnalysisCore.AICore.GPT.Dto { + public class VideoKnowQueryDto + { + /// + /// 闂瑙ｉ噴 + /// + public virtual float? StartTime { get; set; } + /// + /// 涓婚 + /// + public virtual string? Theme { get; set; } + /// + /// 鐭ヨ瘑鐐 + /// + public virtual string? KnowPoint { get; set; } + /// + /// 鍐呭鎬荤粨 + /// + public virtual string? Content { get; set; } + + } public class VideoKnowRes { /// /// 闂瑙ｉ噴 /// - public float? StartTime { get; set; } - public float? EndTime { get; set; } - ///// - ///// 鎸佺画鏃堕棿 - ///// - public float? KeepTime => (EndTime ?? 0) - StartTime ?? 0; + public virtual float? StartTime { get; set; } + public virtual float? EndTime { get; set; } + /// + /// 鎸佺画鏃堕棿 + /// + public virtual float? KeepTime => (EndTime ?? 0) - StartTime ?? 0; /// /// 涓婚 /// - public string? Theme { get; set; } + public virtual string? Theme { get; set; } /// /// 鐭ヨ瘑鐐 /// - public string? KnowPoint { get; set; } + public virtual string? KnowPoint { get; set; } /// /// 鐭ヨ瘑鐐笽D /// - public string? KnowPointId { get; set; } + public virtual string? KnowPointId { get; set; } /// /// 鍐呭鎬荤粨 /// - public string? Content { get; set; } + public virtual string? Content { get; set; } } public class FileNameInfo diff --git a/VideoAnalysisCore/AICore/SherpaOnnx/SenseVoice.cs b/VideoAnalysisCore/AICore/SherpaOnnx/SenseVoice.cs index 878c816..3ceac26 100644 --- a/VideoAnalysisCore/AICore/SherpaOnnx/SenseVoice.cs +++ b/VideoAnalysisCore/AICore/SherpaOnnx/SenseVoice.cs @@ -262,10 +262,11 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx Console.WriteLine(DateTime.Now + "=> SenseVoice 瀛楀箷鏁伴噺"+ res.Count); var captionsStr = JsonSerializer.Serialize(res); - DbScoped.SugarScope + await DbScoped.SugarScope .Updateable() .SetColumns(it => it.Captions == captionsStr) - .Where(it => it.Id == long.Parse(task)); + .Where(it => it.Id == long.Parse(task)) + .ExecuteCommandAsync(); await RedisExpand.Redis.HMSetAsync(RedisExpandKey.Task(task), "Captions", res); //RedisExpand.InsertChannel(Enum.RedisChannelEnum.ParsingSpeaker, task); RedisExpand.InsertChannel(RedisChannelEnum.ChatModelAnalysis, task); diff --git a/VideoAnalysisCore/AICore/SherpaOnnx/Speaker.cs b/VideoAnalysisCore/AICore/SherpaOnnx/Speaker.cs index 219eb7f..034e045 100644 --- a/VideoAnalysisCore/AICore/SherpaOnnx/Speaker.cs +++ b/VideoAnalysisCore/AICore/SherpaOnnx/Speaker.cs @@ -8,6 +8,7 @@ using SqlSugar.IOC; using VideoAnalysisCore.Model; using System.Text.Json; using VideoAnalysisCore.Model.Enum; +using Microsoft.Extensions.DependencyInjection; namespace VideoAnalysisCore.AICore.SherpaOnnx { @@ -16,7 +17,7 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx /// 璇磋瘽浜烘ā鍨 /// pyannote /// - public class Speaker + public static class Speaker { private static OfflineSpeakerDiarization? SD; /// @@ -25,9 +26,9 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx /// /// /// - public static void Init(int speakerNumber = 2, float threshold = 0.6f,bool useGPU = false) + public static void AddSpeakerAI(this IServiceCollection service,int speakerNumber = 2, float threshold = 0.6f,bool useGPU = false) { - Console.WriteLine("鍒濆鍖 Speaker"); + Console.WriteLine($"{DateTime.Now}=>鍒濆鍖 Speaker"); var config = new OfflineSpeakerDiarizationConfig(); //Pyannote妯″瀷鍦板潃 config.Segmentation.Pyannote.Model = Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-pyannote-segmentation-3-0", "model.onnx"); diff --git a/VideoAnalysisCore/Common/AlibabaCloudVodExpand.cs b/VideoAnalysisCore/Common/AlibabaCloudVodExpand.cs new file mode 100644 index 0000000..47bff00 --- /dev/null +++ b/VideoAnalysisCore/Common/AlibabaCloudVodExpand.cs @@ -0,0 +1,56 @@ +锘縰sing System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using AlibabaCloud.OpenApiClient.Models; +using AlibabaCloud.SDK.Vod20170321; +using AlibabaCloud.SDK.Vod20170321.Models; +using AlibabaCloud.TeaUtil.Models; +using Microsoft.Extensions.DependencyInjection; +using VideoAnalysisCore.Job; + +namespace VideoAnalysisCore.Common +{ + public class AlibabaCloudVodConfig + { + /// + /// id + /// + public string AccessKeyId { get; set; } + /// + ///瀵嗛挜 + /// + public string AccessKeySecret { get; set; } + public string Endpoint { get; set; } = "vod.cn-shanghai.aliyuncs.com"; + } + + /// + /// 闃块噷浜 瑙嗛鐐规挱鎷撳睍 + /// + public static class AlibabaCloudVodExpand + { + /// + /// 浣跨敤闃块噷浜 vod鎷撳睍 + /// + /// + /// + public static void AddAlibabaCloudVod(this IServiceCollection service) + { + Console.WriteLine($"{DateTime.Now}=>鍒濆鍖 闃块噷浜慥OD"); + // 宸ョ▼浠ｇ爜娉勯湶鍙兘浼氬鑷 AccessKey 娉勯湶锛屽苟濞佽儊璐﹀彿涓嬫墍鏈夎祫婧愮殑瀹夊叏鎬с備互涓嬩唬鐮佺ず渚嬩粎渚涘弬鑰冦 + // 寤鸿浣跨敤鏇村畨鍏ㄧ殑 STS 鏂瑰紡锛屾洿澶氶壌鏉冭闂柟寮忚鍙傝锛歨ttps://help.aliyun.com/document_detail/378671.html銆 + Config config = new() + { + // 蹇呭～锛岃纭繚浠ｇ爜杩愯鐜璁剧疆浜嗙幆澧冨彉閲 ALIBABA_CLOUD_ACCESS_KEY_ID銆 + AccessKeyId = AppCommon.Config.AlibabaCloudVod.AccessKeyId, + // 蹇呭～锛岃纭繚浠ｇ爜杩愯鐜璁剧疆浜嗙幆澧冨彉閲 ALIBABA_CLOUD_ACCESS_KEY_SECRET銆 + AccessKeySecret = AppCommon.Config.AlibabaCloudVod.AccessKeySecret, + Endpoint = AppCommon.Config.AlibabaCloudVod.Endpoint + }; + // Endpoint 璇峰弬鑰 https://api.aliyun.com/product/vod + var c = new Client(config); + service.AddSingleton(c); + } + } +} diff --git a/VideoAnalysisCore/Common/AppCommon.cs b/VideoAnalysisCore/Common/AppCommon.cs index 838aa06..9317202 100644 --- a/VideoAnalysisCore/Common/AppCommon.cs +++ b/VideoAnalysisCore/Common/AppCommon.cs @@ -87,6 +87,50 @@ namespace VideoAnalysisCore.Common static Dictionary FormulaData; static string FormulaDataKey; /// + /// 璇嗗埆瀛楃涓蹭腑鐨刯son瀛楃涓 + /// + /// + /// + public static List ExtractJson(this string input) + { + List jsonList = new List(); + int index = 0; + + while (index < input.Length) + { + if (input[index] == '{' || input[index] == '[') + { + int startIndex = index; + int openCount = 1; + index++; + + while (index < input.Length && openCount > 0) + { + if (input[index] == '{' || input[index] == '[') + { + openCount++; + } + else if (input[index] == '}' || input[index] == ']') + { + openCount--; + } + index++; + } + + if (openCount == 0) + { + string json = input.Substring(startIndex, index - startIndex); + jsonList.Add(json); + } + } + else + { + index++; + } + } + return jsonList; + } + /// /// 澶勭悊鏁板鍏紡 /// /// @@ -399,6 +443,10 @@ namespace VideoAnalysisCore.Common /// ChatGpt /// public ChatGptConfig ChatGpt { get; set; } = new ChatGptConfig(); + /// + /// 闃块噷浜戣棰戠偣鎾厤缃 + /// + public AlibabaCloudVodConfig AlibabaCloudVod { get; set; } = new AlibabaCloudVodConfig(); /// diff --git a/VideoAnalysisCore/Common/DownloadFile.cs b/VideoAnalysisCore/Common/DownloadFile.cs index 2cd469a..16b319b 100644 --- a/VideoAnalysisCore/Common/DownloadFile.cs +++ b/VideoAnalysisCore/Common/DownloadFile.cs @@ -1,5 +1,6 @@ 锘縰sing AntDesign; using Downloader; +using SqlSugar; using SqlSugar.IOC; using System; using System.ComponentModel; @@ -106,9 +107,11 @@ namespace VideoAnalysisCore.Common { if (Opt is null) Init(); + var taskId = long.Parse(task); //鑾峰彇璧勬簮鏂囦欢 鍦板潃 - var fileUrl = RedisExpand.Redis.HMGet(RedisExpandKey.Task(task), "MediaUrl") - .FirstOrDefault(); + var fileUrl =await DbScoped.SugarScope.Queryable() + .Where(s => s.Id == taskId) + .Select(s=>s.MediaUrl).FirstAsync(); if (string.IsNullOrEmpty(fileUrl)) throw new Exception($"浠诲姟id[{task}] 璧勬簮鍦板潃鏃犳晥 {fileUrl}"); @@ -125,10 +128,11 @@ namespace VideoAnalysisCore.Common var outputPath = Path.Combine(localPath, task + fileExtension); if (!Directory.Exists(localPath)) Directory.CreateDirectory(localPath); - DbScoped.SugarScope + await DbScoped.SugarScope .Updateable() .SetColumns(it => it.LocalMediaPath == outputPath) - .Where(it => it.Id == long.Parse(task)); + .Where(it => it.Id == long.Parse(task)) + .ExecuteCommandAsync(); IDownload download = DownloadBuilder.New() .WithUrl(fileUrl) @@ -147,7 +151,7 @@ namespace VideoAnalysisCore.Common { if (download.Status == DownloadStatus.Failed && e.Error != null) { - await RedisExpand.SetTaskErrorMessage(long.Parse(task), e.Error) + await RedisExpand.SetTaskErrorMessage(taskId, e.Error) .ConfigureAwait(false);//涓嶅垏鍥炰笂涓嬫枃 return; } diff --git a/VideoAnalysisCore/Common/RedisExpand.cs b/VideoAnalysisCore/Common/RedisExpand.cs index fedaf31..5bae5bb 100644 --- a/VideoAnalysisCore/Common/RedisExpand.cs +++ b/VideoAnalysisCore/Common/RedisExpand.cs @@ -67,14 +67,14 @@ namespace VideoAnalysisCore.Common /// 浠诲姟瀵硅薄鍦板潃 /// public static string Task(object taskId) => BaseKey + "Task:" + taskId; - public static string IDTask => BaseKey + AppCommon.Config.ID; + public static string IDTask => BaseKey + "Services:" + AppCommon.Config.ID; public static string TaskGPT(object taskId) => Task(taskId) + ":GPTCached"; } /// /// redis鎷撳睍 /// - public class RedisExpand + public static class RedisExpand { /// /// redis 杩炴帴 @@ -90,9 +90,9 @@ namespace VideoAnalysisCore.Common /// 鍒濆鍖 redis /// 闇瑕佸湪鍒濆鍖栭厤缃枃浠舵椂鍊欒皟鐢 /// - public static void Init() + public static void AddRedisExpand(this IServiceCollection service) { - Console.WriteLine("鍒濆鍖 redis"); + Console.WriteLine($"{DateTime.Now}=>鍒濆鍖 Redis"); Redis.Serialize = obj => JsonSerializer.Serialize(obj); Redis.Deserialize = (json, type) => JsonSerializer.Deserialize(json, type); Task.Run(() => @@ -117,10 +117,16 @@ namespace VideoAnalysisCore.Common /// /// 鍔犲叆鍒版秷璐归槦鍒 /// - /// - public static void JoinQueue(params long[] taskId) - { - Redis.LPush(RedisExpandKey.ChannelKey, taskId); + /// + public static void JoinQueue(params long[] taskIds) + { //浜嬪姟 + using (var tran = Redis.Multi()) + { + foreach (var item in taskIds) + tran.LPush(RedisExpandKey.ChannelKey, item); + object[] ret = tran.Exec(); + Console.WriteLine(ret[0] + ", " + ret[2]); + } } /// /// 鑾峰彇浠诲姟杩涘害 @@ -179,7 +185,7 @@ namespace VideoAnalysisCore.Common if (taskData.Captions == "[]") taskData.Captions = (await Redis.HMGetAsync(RedisExpandKey.Task(task), "Captions")).First(); if (taskData.Speaker == "[]") - taskData.Speaker = (await Redis.HMGetAsync(RedisExpandKey.Task(task), "Speaker")).First(); + taskData.Speaker = (await Redis.HMGetAsync(RedisExpandKey.Task(task), "Speaker"))?.FirstOrDefault()??"[]"; diff --git a/VideoAnalysisCore/Job/NodeSubscriptionJob.cs b/VideoAnalysisCore/Job/NodeSubscriptionJob.cs index 834eb0e..2a7fc83 100644 --- a/VideoAnalysisCore/Job/NodeSubscriptionJob.cs +++ b/VideoAnalysisCore/Job/NodeSubscriptionJob.cs @@ -1,14 +1,18 @@ -锘縰sing Coravel.Invocable; +锘縰sing AlibabaCloud.SDK.Vod20170321; +using Coravel.Invocable; +using Microsoft.AspNetCore.Http; using System; using System.Collections.Generic; using System.Linq; using System.Reflection; using System.Text; using System.Threading.Tasks; +using UserCenter.Model.Enum; using VideoAnalysisCore.Common; using VideoAnalysisCore.Model; using VideoAnalysisCore.Model.Dto; using VideoAnalysisCore.Model.钃濋哺鏅哄簱; +using Yitter.IdGenerator; namespace VideoAnalysisCore.Job { @@ -17,26 +21,32 @@ namespace VideoAnalysisCore.Job /// public class NodeSubscriptionJob : IInvocable { + /// + /// 姣忎釜鎵弿鏂囦欢鍖呮瘡娆″彇鍑簕20}涓 + /// + private readonly int TopLength = 20; private readonly Repository nodesubscriptionDB; private readonly Repository attachmentsDB; private readonly Repository videotaskDB; - public NodeSubscriptionJob(Repository videoTaskDB, Repository nodesubscriptionDB, Repository videotaskDB) + private readonly Client vodClient; + public NodeSubscriptionJob(Repository videoTaskDB, Repository nodesubscriptionDB, Repository videotaskDB, Client vodClient) { this.attachmentsDB = videoTaskDB; this.nodesubscriptionDB = nodesubscriptionDB; this.videotaskDB = videotaskDB; + this.vodClient = vodClient; } public async Task Invoke() { Console.WriteLine($"{DateTime.Now} Invoke=>{this.GetType().FullName}"); - var tasks = await nodesubscriptionDB.GetListAsync(s => s.Enable); + var tasks = await nodesubscriptionDB.GetListAsync(s => s.Enable && s.Subject ==SubjectEnum.鏁板); foreach (var item in tasks) { var fileNodeId = item.NodeId; var data = attachmentsDB.Context.Ado .SqlQuery($""" - SELECT + SELECT top {TopLength} * FROM Attachments @@ -62,22 +72,43 @@ namespace VideoAnalysisCore.Job AND ( NAME NOT LIKE '%PPT%' OR NAME NOT LIKE '%ppt%' - ) + ) and id>{item.LastId} """); + if (data is null || data.Count == 0) + continue; + Console.WriteLine($"{DateTime.Now} 瑙嗛璁㈤槄=>Node:{item.NodeId} 鏁伴噺{data.Count}"); - var videos = data.Select(s => new VideoTask() + var videos = new List(data.Count); + foreach (var s in data) { - ComeFrom = "127.0.0.1", - ApiToken = "", - Type = item.TaskType, - Subject = item.Subject, - Tag = string.Empty, - TagId = s.VideoCode, - MediaUrl = s.Url, - MediaName = s.Name - }).ToArray(); - - //鍏ュ簱 + var videoInfo = await vodClient.GetPlayInfoAsync(new AlibabaCloud.SDK.Vod20170321.Models.GetPlayInfoRequest() + { + VideoId = s.VideoCode, Formats="mp4",OutputType = "cdn", AuthTimeout = 3600 * 24 * 3, + }); + if (videoInfo is null || videoInfo.StatusCode != 200 && !videoInfo.Body.PlayInfoList.PlayInfo.Any()) + { + Console.WriteLine($"{DateTime.Now} 瑙嗛璁㈤槄=>鑾峰彇闃块噷浜戣棰戜俊鎭け璐 VideoCode {s.VideoCode} StatusCode {videoInfo?.StatusCode}"); + continue; + } + videos.Add(new VideoTask() + { + Id = YitIdHelper.NextId(), + ComeFrom = "127.0.0.1", + ApiToken = "", + Type = item.TaskType, + Subject = item.Subject, + Tag = item.NodeId.ToString(), + TagId = s.VideoCode, + MediaUrl = videoInfo.Body.PlayInfoList.PlayInfo.First().PlayURL, + MediaName = s.Name + }); + } + var maxId = data.Max(s => s.Id); + //鍏ュ簱 鏇存柊鏈鍚庢壂鎻忚褰 + await nodesubscriptionDB.AsUpdateable() + .SetColumns(it => it.LastId == maxId) + .Where(it => it.Id == item.Id) + .ExecuteCommandAsync(); await videotaskDB.InsertRangeAsync(videos); var ids = videos.Select(s => s.Id).ToArray(); RedisExpand.JoinQueue(ids); diff --git a/VideoAnalysisCore/Job/TaskFileClearJob.cs b/VideoAnalysisCore/Job/TaskFileClearJob.cs new file mode 100644 index 0000000..248b8bf --- /dev/null +++ b/VideoAnalysisCore/Job/TaskFileClearJob.cs @@ -0,0 +1,79 @@ +锘縰sing AlibabaCloud.SDK.Vod20170321; +using Coravel.Invocable; +using Microsoft.AspNetCore.Http; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading.Tasks; +using UserCenter.Model.Enum; +using VideoAnalysisCore.Common; +using VideoAnalysisCore.Model; +using VideoAnalysisCore.Model.Dto; +using VideoAnalysisCore.Model.Interface; +using VideoAnalysisCore.Model.钃濋哺鏅哄簱; +using Yitter.IdGenerator; + +namespace VideoAnalysisCore.Job +{ + /// + /// 涓嬭浇瑙嗛鐨勪换鍔℃枃浠舵竻鐞嗕换鍔 + /// + public class TaskFileClearJob : IInvocable + { + private readonly Repository videotaskDB; + + public TaskFileClearJob(Repository videotaskDB) + { + this.videotaskDB = videotaskDB; + } + public void DeleteOldCompletedTaskCaches() + { + try + { + var startTime = -2; + var timeSpan = startTime - 3; + // 璁＄畻 2 澶╁墠鐨勬椂闂 + DateTime twoDaysAgo = DateTime.Now.AddDays(startTime); + DateTime endDaysAgo = DateTime.Now.AddDays(timeSpan); + + // 鏌ヨ 2 澶╁墠浠诲姟鎵ц瀹屾垚鐨勮褰 + var completedTasks = videotaskDB.AsQueryable() + .Where(t => + t.LastEnum == Model.Enum.RedisChannelEnum.EndTask + && t.CreateTime < twoDaysAgo + && t.CreateTime > endDaysAgo) + .ToList(); + + // 閬嶅巻鏌ヨ缁撴灉锛屽垹闄ょ紦瀛樻枃浠 + foreach (var task in completedTasks) + { + var path = task.Id.ToString().LocalPath(); + if (!string.IsNullOrEmpty(path) && Directory.Exists(path)) + { + try + { + Directory.Delete(path, true); + Console.WriteLine($"宸插垹闄ょ紦瀛樻枃浠: {task.LocalMediaPath}"); + } + catch (Exception ex) + { + Console.WriteLine($"鍒犻櫎缂撳瓨鏂囦欢 {task.Id} 鏃跺嚭閿: {ex.Message}"); + } + } + } + } + catch (Exception ex) + { + Console.WriteLine($"娓呯悊缂撳瓨鏃跺嚭閿: {ex.Message}"); + } + } + public async Task Invoke() + { + Console.WriteLine($"{DateTime.Now} Invoke=>{this.GetType().FullName}"); + + DeleteOldCompletedTaskCaches(); + } + } +} diff --git a/VideoAnalysisCore/VideoAnalysisCore.csproj b/VideoAnalysisCore/VideoAnalysisCore.csproj index ee585c8..b1145ad 100644 --- a/VideoAnalysisCore/VideoAnalysisCore.csproj +++ b/VideoAnalysisCore/VideoAnalysisCore.csproj @@ -57,6 +57,7 @@ +