diff --git a/VideoAnalysis/Controllers/ApiController.cs b/VideoAnalysis/Controllers/ApiController.cs index e6e8afc..8c984ae 100644 --- a/VideoAnalysis/Controllers/ApiController.cs +++ b/VideoAnalysis/Controllers/ApiController.cs @@ -8,6 +8,7 @@ using VideoAnalysisCore.AICore.SherpaOnnx; using UserCenter.Model.Enum; using VideoAnalysisCore.AICore.GPT.ChatGPT; using VideoAnalysisCore.AICore.GPT; +using System.Text.Json; namespace Learn.VideoAnalysis.Controllers { @@ -75,16 +76,24 @@ namespace Learn.VideoAnalysis.Controllers /// - /// Test + /// 获取FTS_Data str /// - /// + /// 路径 /// - [HttpGet(Name = "Test")] - public async Task Test(long taskId) + [HttpGet(Name = "fts_data")] + public async Task FTS_Data(string path= "itn_subject_sx.fst") { - //重新开始执行GPT分析 - chatGPT.GetKnow(taskId.ToString()); - return Ok(); + var hotwords = JsonSerializer + .Deserialize(System.IO.File.ReadAllText(Path.Combine(AppCommon.AIModelFile, "Hotwords.json"))); + var res = new List(100); + foreach (var element in hotwords.OrderByDescending(s => s.key.Count())) + foreach (var e in element.v) + res.Add($"""("{e}", "{element.key}")"""); + var pyFile = System.IO.File.ReadAllText(Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-fst.py")); + var resStr = pyFile + .Replace("(fts_data)", "[" + string.Join(',', res) + "]") + .Replace("(path)", path); + return Ok(resStr); } [NonAction] diff --git a/VideoAnalysis/Program.cs b/VideoAnalysis/Program.cs index ae26073..2c6ae30 100644 --- a/VideoAnalysis/Program.cs +++ b/VideoAnalysis/Program.cs @@ -8,6 +8,7 @@ using VideoAnalysisCore.AICore.GPT; using VideoAnalysisCore.AICore.GPT.KIMI; using VideoAnalysisCore.AICore.GPT.ChatGPT; using Microsoft.Extensions.FileProviders; +using VideoAnalysisCore.AICore.GPT.DeepSeek; @@ -81,8 +82,10 @@ namespace Learn.VideoAnalysis builder.Services.AddHttpClient(); builder.Services.AddSingleton(); + builder.Services.AddSingleton(); //builder.Services.AddSingleton(); - builder.Services.AddSingleton(); + //builder.Services.AddSingleton(); + builder.Services.AddSingleton(); var app = builder.Build(); diff --git a/VideoAnalysis/appsettings.json b/VideoAnalysis/appsettings.json index aaabb52..6276787 100644 --- a/VideoAnalysis/appsettings.json +++ b/VideoAnalysis/appsettings.json @@ -30,6 +30,10 @@ "Host": "https://api.g4f.icu/", //"Host": "https://api.oaibest.com/", "ApiKey": "sk-D15tBln31N7dI9Fi7lds7OySFv5tOEK7DMNsG5rY2E6DCr4s" + }, + "DeepSeek": { + "Host": "https://api.deepseek.com/chat/completions", + "ApiKey": "sk-88d3d2bc3dae4d50854b2569b281cf76" } }, "DB": { diff --git a/VideoAnalysisCore/AICore/GPT/BserGPT.cs b/VideoAnalysisCore/AICore/GPT/BserGPT.cs index 4cc4911..0c46d53 100644 --- a/VideoAnalysisCore/AICore/GPT/BserGPT.cs +++ b/VideoAnalysisCore/AICore/GPT/BserGPT.cs @@ -15,7 +15,6 @@ namespace VideoAnalysisCore.AICore.GPT /// /// 浠诲姟id /// - public Task CallGPT(string task); public Task GetKnow(string task); } } diff --git a/VideoAnalysisCore/AICore/GPT/ChatGPT/Chat_GPT.cs b/VideoAnalysisCore/AICore/GPT/ChatGPT/Chat_GPT.cs index 04b60d6..1c60f19 100644 --- a/VideoAnalysisCore/AICore/GPT/ChatGPT/Chat_GPT.cs +++ b/VideoAnalysisCore/AICore/GPT/ChatGPT/Chat_GPT.cs @@ -87,25 +87,24 @@ namespace VideoAnalysisCore.AICore.GPT.ChatGPT $"1.鍒嗘瀽鍑洪珮涓瓄subject}璇惧爞鎺堣鐨勪富瑕佺珷鑺(渚嬪 绔犺妭: 鏁板垪),绔犺妭鑼冨洿闄愬畾鍦╗{string.Join(',', xkwKnows)}]鑼冨洿." + $"2.鍒嗘瀽鍑鸿繖鍫傝鐨勪富瑕佹巿璇惧唴瀹." + $"杈撳嚭鏍煎紡 json瀛楃涓 瀵硅薄鏍煎紡{fileNameResFormat}"; - var fileNameInfoRes = await ChatAsync(task, fileNamePostMessages,null, fileNameResFormat); + var fileNameInfoRes = await ChatAsync(task, fileNamePostMessages, null, fileNameResFormat); var captions = ExpandFunction.GetSpeakerCaptions(task); - var maxVideoTime = captions?.TimeBase?.LastOrDefault()?.End??0; + var maxVideoTime = captions?.TimeBase?.LastOrDefault()?.End ?? 0; var criteriaBuilder = new StringBuilder(); //var resFormat = """[{"StartTime":寮濮嬬(number),"EndTime":缁撴潫绉(number),"Section":绔犺妭(string),"Theme":涓婚(string),"ThemeDetalis":涓婚璇︽儏(string),"Content":鍐呭鎬荤粨(string)}]"""; var resFormat = """[{"StartTime":寮濮嬬(number),"Section":绔犺妭(string),"Theme":涓婚(string),"ThemeDetalis":涓婚璇︽儏(string),"Content":鍐呭鎬荤粨(string)}]"""; var know = await knowledgeInfoDB.GetFirstAsync(s => s.Name == fileNameInfoRes.鎺堣绔犺妭); var knowledgeInfos = await knowledgeInfoDB.AsQueryable().ToChildListAsync(s => s.Parent_Id, know.Id); - var knows = "鏁板垪鐨勬蹇,鏁板垪鐨勫畾涔,椤圭殑琛ㄧず,鏁板垪鐨勮〃绀烘柟娉,閫氶」鍏紡,閫掓帹鍏紡,鍥惧儚琛ㄧず,鏁板垪鐨勭被鍨,绛夊樊鏁板垪,绛夋瘮鏁板垪,鍏朵粬鐗规畩鏁板垪,鏁板垪鐨勬ц川,鍗曡皟鎬,鏈夐檺鎬,鏁板垪鐨勬眰鍜,绛夊樊鏁板垪姹傚拰鍏紡,绛夋瘮鏁板垪姹傚拰鍏紡,鏁板垪鏋侀檺,閫掓帹鍏崇郴"; - knows = string.Join(',', knowledgeInfos.Select(s => s.Name)); + var knows = string.Join(',', knowledgeInfos.Select(s => s.Name)); var postMessages = $"浣犵殑浠诲姟鏄垎鏋愯棰戝瓧骞曞唴瀹瑰苟鎻愬彇鍑轰腑鍥介珮鑰冭冭瘯璇曢鏂规硶鐐,鐒跺悗鏍规嵁姝ラ鍒嗘瀽鍑虹煡璇嗙墖娈" + $"鎸変互涓嬫楠ゅ畬鎴:" + $"1.璇嗗埆鏂规硶鐐:鎻愬彇瀛楀箷鍐呭涓笌{subject}鑰冭瘯灞炰簬{fileNameInfoRes.鎺堣绔犺妭}绔犺妭鐩稿叧鐨勬柟娉曠偣銆" + $"2.鍒嗘瀽鎬荤粨:鍩轰簬鎻愬彇鍑虹殑鏂规硶鐐瑰悕绉版潵鍖归厤鎴戞彁渚涚殑鏂规硶鐐瑰悕绉" + - $"鎻愪緵鐨勬柟娉曠偣鍚嶇О(鍩烘湰姒傚康,璇惧爞缁冧範,渚嬮璁茶В,{knows})銆" + + $"鎻愪緵鐨勬柟娉曠偣鍚嶇О({fileNameInfoRes.鎺堣绔犺妭}鐨勫熀鏈蹇,{fileNameInfoRes.鎺堣绔犺妭}鐨勭粌涔犱笌搴旂敤,{fileNameInfoRes.鎺堣绔犺妭}鐨勪緥棰樿瑙,{knows})銆" + $"3.鍏宠仈鍚堝苟鐭ヨ瘑鍐呭鐩镐技鐨勭煡璇嗙偣鏉ュ悎骞朵负鐭ヨ瘑鐗囨銆" + $"鐭ヨ瘑鐗囨浣跨敤鍏宠仈鐭ヨ瘑鐐逛腑鐨勬渶灏忓紑濮嬫椂闂翠富棰樹负鍏宠仈鐭ヨ瘑鐐圭殑涓婚鍒嗘瀽,鍐呭鎬荤粨涓哄叧鑱旂煡璇嗙偣鐨勫唴瀹规荤粨鍒嗘瀽銆" + $"4.鍩轰簬'鐭ヨ瘑鐗囨'鐨'鍐呭鎬荤粨'鍔犱笂'涓婚'鏉ュ垎鏋愯繖涓墖娈靛涓婚鐨勮瑙ｅ唴瀹逛负鏂扮殑涓婚 渚嬪(鏁板垪鐨勫熀鏈蹇)銆" + @@ -114,6 +113,20 @@ namespace VideoAnalysisCore.AICore.GPT.ChatGPT $"浠ヤ笅鏄寘鍚椂闂寸殑瑙嗛瀛楀箷鏂囨湰銆" + $"瀛楀箷鏍煎紡(璇磋瘽浜:寮濮嬬:缁撴潫绉:鍐呭|涓嬩竴娈靛瓧骞).瀛楀箷鍒楄〃 {captions.Captions}" + $"杈撳嚭鏍煎紡({resFormat})"; + postMessages = + $"浣犵幇鍦ㄩ渶瑕佸鐞嗙敤鎴锋彁渚涚殑瑙嗛瀛楀箷鍐呭锛屼粠涓彁鍙栦笌涓浗楂樿冩暟瀛﹁冭瘯{fileNameInfoRes.鎺堣绔犺妭}鐩稿叧鐨勬柟娉曠偣锛屽苟鎸夌収鎸囧畾鐨勬楠よ繘琛屽垎鏋愩傞鍏堬紝浣犲緱浠旂粏闃呰鐢ㄦ埛鐨勯渶姹傦紝纭繚姣忎竴姝ラ兘姝ｇ‘鎵ц" + + $"鐢ㄦ埛鐨勯渶姹:" + + $"1.瑕佹眰璇嗗埆瀛楀箷涓殑鏂规硶鐐癸紝杩欎簺鏂规硶鐐归渶瑕佸睘浜巤fileNameInfoRes.鎺堣绔犺妭}绔犺妭銆傞渶瑕佷綘閫氳瀛楀箷锛屾壘鍒拌繖浜涘叧閿瘝鍑虹幇鐨勫湴鏂癸紝骞惰褰曞搴旂殑鏃堕棿鎴" + + $"2.鏄牴鎹彁鍙栫殑鏂规硶鐐瑰悕绉板尮閰嶇敤鎴锋彁渚涚殑鍒楄〃銆備緥濡傦紝濡傛灉瀛楀箷涓彁鍒扳滈掑鏁板垪鈥濇垨鈥滈氶」鍏紡鈥濓紝浣犻渶瑕佺‘璁よ繖浜涙槸鍚﹀湪鐢ㄦ埛缁欏嚭鐨勬柟娉曠偣鍒楄〃涓紝骞跺皢瀹冧滑褰掔被鍒版纭殑鍚嶇О涓" + + $"鐢ㄦ埛鎻愪緵鐨勬柟娉曠偣鍚嶇О({fileNameInfoRes.鎺堣绔犺妭}鐨勫熀鏈蹇,{fileNameInfoRes.鎺堣绔犺妭}鐨勭粌涔犱笌搴旂敤,{fileNameInfoRes.鎺堣绔犺妭}鐨勪緥棰樿瑙,{knows})銆" + + $"3.鍏宠仈鍚堝苟鐩镐技鐨勭煡璇嗙偣锛屽舰鎴愮煡璇嗙墖娈点傛瘮濡傦紝濡傛灉澶氫釜鏃堕棿娈甸兘璁ㄨ鏁板垪鐨勫熀鏈蹇碉紝浣犻渶瑕佸皢瀹冧滑鍚堝苟锛屽苟閫夋嫨鏈鏃╃殑鏃堕棿浣滀负寮濮嬫椂闂淬傚悓鏃讹紝闇瑕佹荤粨杩欎簺鐗囨鐨勫唴瀹癸紝骞剁敓鎴愭柊鐨勪富棰樺悕绉帮紝濡傗滄暟鍒楃殑鍩烘湰姒傚康鈥濄" + + $"濡傛灉澶氫釜鐗囨閮芥秹鍙婃暟鍒楃殑瀹氫箟鍜屼緥瀛愶紝鍙兘鍚堝苟涓衡滄暟鍒楃殑鍩烘湰姒傚康鈥" + + $"4.鍒嗛厤鏈娇鐢ㄧ殑鏃堕棿娈靛埌鐩歌繎鐨勭煡璇嗙墖娈碉紝閬垮厤鏃堕棿閲嶅彔銆傝繖鍙兘闇瑕佹鏌ユ槸鍚︽湁鍓╀綑鐨勬椂闂存鏈褰掔被锛屽苟灏嗗叾鍚堝苟鍒板悎閫傜殑鐭ヨ瘑鐗囨涓紝浠ヤ赴瀵屼笂涓嬫枃" + + $"杈撳叆锛氬寘鍚椂闂存埑鐨勮棰戝瓧骞曟枃鏈" + + $"浠ヤ笅鏄寘鍚椂闂寸殑瑙嗛瀛楀箷鏂囨湰銆" + + $"瀛楀箷鏍煎紡(璇磋瘽浜:寮濮嬬:缁撴潫绉:鍐呭|涓嬩竴娈靛瓧骞).瀛楀箷鍒楄〃 {captions.Captions}" + + $"杈撳嚭鏍煎紡({resFormat})"; + var questionRes = await ChatAsync(task, postMessages, null, resFormat); //var postMessages = // $"浣犵殑浠诲姟鏄垎鏋愯棰戝瓧骞曞唴瀹癸紝绮惧噯鎻愬彇鍑轰笌涓浗楂樿冩暟瀛﹁冭瘯鐩稿叧鐨勮瘯棰樻柟娉曠偣銆" + @@ -133,9 +146,9 @@ namespace VideoAnalysisCore.AICore.GPT.ChatGPT //var questionRes = await ChatAsync(task, postMessages, postM2, resFormat); if (questionRes.Length <= 3) - throw new Exception("瑙嗛鍒嗘鏁伴噺杩囦綆 =>"+ questionRes.Length); + throw new Exception("瑙嗛鍒嗘鏁伴噺杩囦綆 =>" + questionRes.Length); - if (questionRes.Count(s=>s.ThemeDetalis == questionRes.First().ThemeDetalis) >= 3) + if (questionRes.Count(s => s.ThemeDetalis == questionRes.First().ThemeDetalis) >= 3) throw new Exception("瑙嗛鍒嗘涓婚閲嶅 =>" + questionRes.First().ThemeDetalis); for (int i = 0; i < questionRes.Length; i++) @@ -144,7 +157,7 @@ namespace VideoAnalysisCore.AICore.GPT.ChatGPT if (i == questionRes.Length - 1) item.EndTime = maxVideoTime; else - item.EndTime = (int)(questionRes[i + 1]?.StartTime??0) - 1; + item.EndTime = (int)(questionRes[i + 1]?.StartTime ?? 0) - 1; } await RedisExpand.Redis @@ -171,7 +184,7 @@ namespace VideoAnalysisCore.AICore.GPT.ChatGPT RedisExpand.InsertChannel(RedisChannelEnum.EndTask, task); return gptRes; } - public async Task ChatAsync(string task, string postMessages,string postMessages1, string resFormat) + public async Task ChatAsync(string task, string postMessages, string postMessages1, string resFormat) { var maxTokens = 4000; Message[] messageArr = [ diff --git a/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeekClient.cs b/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeekClient.cs new file mode 100644 index 0000000..0b7f300 --- /dev/null +++ b/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeekClient.cs @@ -0,0 +1,142 @@ +锘縰sing VideoAnalysisCore.Common; +using System.Net.Http.Headers; +using System.Text; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json.Linq; +using System.Net.Http; +using Newtonsoft.Json; +using System.Net.Http.Json; +using AntDesign; +using OneOf.Types; +using System.Net; +using VideoAnalysisCore.AICore.GPT.KIMI; + +namespace VideoAnalysisCore.AICore.GPT.DeepSeek +{ + + public class DeepSeekGPTClient + { + public static string Host = AppCommon.Config.ChatGpt.DeepSeek.Host; + public static string ApiKey = AppCommon.Config.ChatGpt.DeepSeek.ApiKey; + + private readonly IHttpClientFactory _httpClientFactory; + + public DeepSeekGPTClient(IHttpClientFactory httpClientFactory) + { + _httpClientFactory = httpClientFactory; + } + + + /// + /// Chat + /// + /// + /// Return HttpResponseMessage for SSE + public async Task<(Usage u, string res)?> Chat(ChatRequest chatReq) + { + if (chatReq.stream) return await ChatSSE(chatReq); + var requestBody = System.Text.Json.JsonSerializer.Serialize(chatReq); + var chatResp = await PostJsonStreamAsync("v1/chat/completions", requestBody); + var res1 = await chatResp.Content.ReadAsStringAsync(); + var res = await chatResp.Content.ReadFromJsonAsync(); + if (res is null || res.error != null) + throw new Exception($" ChatGPT妯″瀷杩斿洖寮傚父 杩斿洖鍙傛暟: " + + $" {System.Text.Json.JsonSerializer.Serialize(res)}"); + var chatResContent = res?.choices.FirstOrDefault()?.message.content.Trim(); + + if (string.IsNullOrEmpty(chatResContent)) + return null; + return (res.usage, chatResContent); + } + + + private async Task PostJsonStreamAsync(string path, string json) + { + var uriBuilder = new UriBuilder(Host + path); + var maxRestart = 4; + var errorMSG = new Exception[maxRestart]; + for (int i = 0; i < maxRestart; i++) + { + try + { + var client = _httpClientFactory.CreateClient(); + client.DefaultRequestHeaders.Authorization = + new AuthenticationHeaderValue("Bearer", ApiKey); + client.Timeout = TimeSpan.FromSeconds(60 * 20);//瓒呮椂鏃堕棿20鍒嗛挓 + client.DefaultRequestVersion = HttpVersion.Version20; + client.DefaultVersionPolicy = HttpVersionPolicy.RequestVersionOrLower; + client.DefaultRequestHeaders.ConnectionClose = true; + + var content = new StringContent(json, Encoding.UTF8, "application/json"); + return await client.PostAsync(uriBuilder.Uri, content); + } + catch (Exception e) + { + errorMSG[i] = e; + Console.WriteLine("====================[璇锋眰寮傚父,閲嶈瘯]===================="); + Console.WriteLine(uriBuilder.Uri); + Console.WriteLine(e.Message); + Console.WriteLine(e.StackTrace); + Console.WriteLine("=============================================="); + + } + Thread.Sleep(1000); + } + throw errorMSG.Last(s => s != null); + } + + + + + /// + /// ChatSSE[娴佸紡浼犺緭 鏇寸ǔ瀹歖 + /// + /// + /// Return HttpResponseMessage for SSE + public async Task<(Usage u, string res)?> ChatSSE(ChatRequest chatReq) + { + chatReq.stream = true; + var requestBody = System.Text.Json.JsonSerializer.Serialize(chatReq); + var chatResp = await PostJsonStreamAsync("/v1/chat/completions", requestBody); + using var stream = await chatResp.Content.ReadAsStreamAsync(); + using var reader = new StreamReader(stream, Encoding.UTF8); + string line; + var messageBuilder = new StringBuilder(); + var lastChat = new ChatResSSE(); + var splitCount = "data:".Length; + while ((line = await reader.ReadLineAsync()) != null) + { + if (line.EndsWith("[DONE]")) + { + // 琛ㄧず涓鏉℃秷鎭粨鏉 + string message = messageBuilder.ToString(); + messageBuilder.Clear(); + var u = lastChat?.usage; + if (u == null || string.IsNullOrEmpty(message)) + return null; + return (u, message); + } + else if (line.StartsWith("data:")) + { + try + { + var data = System.Text.Json.JsonSerializer.Deserialize(line.Substring(splitCount).Trim()); + lastChat = data; + var str = data?.choices.FirstOrDefault()?.delta.content; + if (!string.IsNullOrEmpty(str)) + messageBuilder.Append(str); + } + catch (Exception e) + { + Console.WriteLine("寮傚父 ChatSSE=>"); + Console.WriteLine(line); + Console.WriteLine(e.Message); + Console.WriteLine(e.StackTrace); + } + } + } + return null; + } + + } +} \ No newline at end of file diff --git a/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeekModel.cs b/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeekModel.cs new file mode 100644 index 0000000..b55e12a --- /dev/null +++ b/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeekModel.cs @@ -0,0 +1,137 @@ +锘縰sing System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using VideoAnalysisCore.AICore.GPT.KIMI; + +namespace VideoAnalysisCore.AICore.GPT.DeepSeek +{ + /// + /// 璇锋眰鏁版嵁 + /// + public class ChatRequest + { + /// + /// 瀵硅瘽 + /// + public Message[] messages { get; set; } + /// + /// 鎻愰棶绉嶅瓙鍊糩鐢ㄦ潵纭繚 鐩稿悓鍙傛暟璇锋眰灏藉彲鑳借繑鍥炵浉鍚屽弬鏁癩 + /// 榛樿:null + /// 姝ゅ姛鑳藉浜 Beta 闃舵銆 濡傛灉鎸囧畾锛屾垜浠殑绯荤粺灏嗗敖鏈澶у姫鍔涚‘瀹氭у湴閲囨牱锛岃繖鏍峰叿鏈夌浉鍚 and 鍙傛暟鐨勯噸澶嶈姹傚簲璇ヨ繑鍥炵浉鍚岀殑缁撴灉銆 鏃犳硶淇濊瘉纭畾鎬э紝鎮ㄥ簲璇ュ弬鑰 response 鍙傛暟鏉ョ洃鎺у悗绔殑鍙樺寲 + /// + public int? seed { get; set; } =null; + /// + /// 鎺ㄧ悊妯″瀷 (deepseek-reasoner) + /// + public string model { get; set; } = "deepseek-reasoner"; + + public float max_tokens { get; set; } = 4000; + /// + /// 瑕佷娇鐢ㄧ殑閲囨牱娓╁害锛屼粙浜 0 鍜 2 涔嬮棿銆傝緝楂樼殑鍊硷紙濡 0.8锛夊皢浣胯緭鍑烘洿鍔犻殢鏈猴紝鑰岃緝浣庣殑鍊硷紙濡 0.2锛夊皢浣垮叾鏇村姞闆嗕腑鍜岀‘瀹氥 鎴戜滑閫氬父寤鸿鏇存敼姝ら」鎴栧悓鏃舵洿鏀逛袱鑰呫倀op_p + /// 榛樿涓 1 + /// 鑱斿姩 + /// + public float temperature { get; set; } = 0.2f; + /// + /// 涓绉嶆浛浠ｆ俯搴﹂噰鏍风殑鏂规硶锛岀О涓哄師瀛愭牳閲囨牱锛 鍏朵腑锛屾ā鍨嬭冭檻鍏锋湁top_p姒傜巼鐨勬爣璁扮殑缁撴灉 璐ㄩ噺銆傛墍浠 0.1 琛ㄧず浠呭寘鍚墠 10% 姒傜巼璐ㄩ噺鐨勪唬甯 琚 + /// 寤鸿涓鑱斿姩 + /// + public float top_p { get; set; } = 0.5f; + /// + /// 涓涓璞★紝鐢ㄤ簬鎸囧畾妯″瀷蹇呴』杈撳嚭鐨勬牸寮忋傝缃负 enable 缁撴瀯鍖栬緭鍑猴紝纭繚妯″瀷涓庢偍鎻愪緵鐨 JSON 鍖归厤 鍥惧紡銆 + /// + public object response_format = new { type = "json_object" }; + /// + /// 娴佸紡杩斿洖 + /// + public bool stream =false; + /// + /// 鎮ㄥ笇鏈涙ā鍨嬩负姝よ姹傜敓鎴愮殑 Output types銆 澶у鏁版ā鍨嬮兘鑳藉鐢熸垚鏂囨湰锛岃繖鏄 + /// 榛樿璁剧疆锛 ["text"] + /// 璇ユā鍨嬭繕鍙敤浜庣敓鎴愰煶棰戙傝嚜 璇锋眰姝ゆā鍨嬪悓鏃剁敓鎴愭枃鏈拰闊抽鍝嶅簲锛屾偍鍙互 鐢細gpt-4o-audio-preview["text", "audio"] + /// + public string modalities = "[\"json\"]"; + +} + public class Message + { + public Message(string content, string role) + { + this.role = role; + this.content = content; + } + public string role { get; set; } + public string content { get; set; } + /// + /// 鎺ㄧ悊鍐呭 + /// + public string reasoning_content { get; set; } + public string refusal { get; set; } + } + + + /// + /// gpt杩斿洖鍊 + /// + public class ChatRes + { + public string id { get; set; } + public string _object { get; set; } + public int created { get; set; } + public string model { get; set; } + public ChatResError error { get; set; } + public Choice[] choices { get; set; } + public Usage usage { get; set; } + /// + /// 绯荤粺鎸囩汗 + /// + public string system_fingerprint { get; set; } + } + + public class Usage + { + public int prompt_tokens { get; set; } + public int completion_tokens { get; set; } + public int total_tokens { get; set; } + } + + + public class Choice + { + public int index { get; set; } + public Message message { get; set; } + public object logprobs { get; set; } + public string finish_reason { get; set; } + } + public class ChatResError + { + public string message { get; set; } + public string type { get; set; } + } + + public class ChatResSSE + { + public string id { get; set; } + public int created { get; set; } + /// + /// 妯″瀷id + /// + public string model { get; set; } + /// + /// 瀵硅瘽 + /// + public ChoiceSSE[] choices { get; set; } + /// + /// token浣跨敤鎯呭喌 + /// + public Usage usage { get; set; } + } + public class ChoiceSSE + { + public int index { get; set; } + public Message delta { get; set; } + public string finish_reason { get; set; } + } +} diff --git a/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs b/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs new file mode 100644 index 0000000..4a5c50f --- /dev/null +++ b/VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs @@ -0,0 +1,202 @@ +锘縰sing VideoAnalysisCore.Common; +using System.Text.Json; +using VideoAnalysisCore.Model; +using System.Text; +using System.ComponentModel.DataAnnotations; +using VideoAnalysisCore.Enum; +using System.Reflection; +using VideoAnalysisCore.Model.Dto; +using VideoAnalysisCore.AICore.GPT.Dto; +using System.Threading.Tasks; +using VideoAnalysisCore.AICore.GPT.ChatGPT; + +namespace VideoAnalysisCore.AICore.GPT.DeepSeek +{ + /// + /// chatgpt 鏂囨湰妯″瀷 + /// + public class DeepSeek_GPT : IBserGPT + { + private readonly DeepSeekGPTClient chatClient; + private readonly Repository criteriaDB; + private readonly Repository videoTaskDB; + private readonly Repository knowledgeInfoDB; + /// + /// 鍒濆鍖 + /// + /// + /// + public DeepSeek_GPT(DeepSeekGPTClient moonshotClient, Repository criteria, Repository videoTaskDB, Repository knowledgeInfoDB) + { + chatClient = moonshotClient; + criteriaDB = criteria; + this.videoTaskDB = videoTaskDB; + this.knowledgeInfoDB = knowledgeInfoDB; + } + + + /// + /// 鑾峰彇鐭ヨ瘑鐐 + /// + /// 浠诲姟id + /// + public async Task GetKnow(string task) + { + var taskId = long.Parse(task); + var taskInfo = await videoTaskDB.AsQueryable() + .Where(s => s.Id == taskId) + .FirstAsync(); + var subject = "鏁板"; + var xkwKnows = await knowledgeInfoDB.AsQueryable() + .Where(s => s.Course_Id == 27 + && s.Depth == 2) + .Select(s => s.Name).ToArrayAsync(); + string title = taskInfo.MediaName; + var fileNameResFormat = "{鎺堣绔犺妭: string|null, 鎺堣鍐呭:string}"; + var fileNamePostMessages = title + + " 杩欐槸涓鍫傝鐨勬爣棰,璇蜂綘甯垜鍒嗘瀽涓浜涘叧浜庤鍫傛柟闈㈢殑鍐呭." + + $"1.鍒嗘瀽鍑洪珮涓瓄subject}璇惧爞鎺堣鐨勪富瑕佺珷鑺(渚嬪 绔犺妭: 鏁板垪),绔犺妭鑼冨洿闄愬畾鍦╗{string.Join(',', xkwKnows)}]鑼冨洿." + + $"2.鍒嗘瀽鍑鸿繖鍫傝鐨勪富瑕佹巿璇惧唴瀹." + + $"杈撳嚭鏍煎紡 json瀛楃涓 瀵硅薄鏍煎紡{fileNameResFormat}"; + var fileNameInfoRes = await ChatAsync(task, fileNamePostMessages, null, "deepseek-chat"); + + var captions = ExpandFunction.GetSpeakerCaptions(task); + var maxVideoTime = captions?.TimeBase?.LastOrDefault()?.End ?? 0; + var criteriaBuilder = new StringBuilder(); + + //var resFormat = """[{"StartTime":寮濮嬬(number),"EndTime":缁撴潫绉(number),"Section":绔犺妭(string),"Theme":涓婚(string),"ThemeDetalis":涓婚璇︽儏(string),"Content":鍐呭鎬荤粨(string)}]"""; + var resFormat = """[{"StartTime":寮濮嬬(number),"Section":绔犺妭(string),"Theme":涓婚(string),"ThemeDetalis":涓婚璇︽儏(string),"Content":鍐呭鎬荤粨(string)}]"""; + var know = await knowledgeInfoDB.GetFirstAsync(s => s.Name == fileNameInfoRes.鎺堣绔犺妭); + var knowledgeInfos = await knowledgeInfoDB.AsQueryable().ToChildListAsync(s => s.Parent_Id, know.Id); + var knows = string.Join(',', knowledgeInfos.Select(s => s.Name)); + + var postMessages = + $"浣犵殑浠诲姟鏄垎鏋愯棰戝瓧骞曞唴瀹瑰苟鎻愬彇鍑轰腑鍥介珮鑰冭冭瘯璇曢鏂规硶鐐,鐒跺悗鏍规嵁姝ラ鍒嗘瀽鍑虹煡璇嗙墖娈" + + $"鎸変互涓嬫楠ゅ畬鎴:" + + $"1.璇嗗埆鏂规硶鐐:鎻愬彇瀛楀箷鍐呭涓笌{subject}鑰冭瘯灞炰簬{fileNameInfoRes.鎺堣绔犺妭}绔犺妭鐩稿叧鐨勬柟娉曠偣銆" + + $"2.鍒嗘瀽鎬荤粨:鍩轰簬鎻愬彇鍑虹殑鏂规硶鐐瑰悕绉版潵鍖归厤鎴戞彁渚涚殑鏂规硶鐐瑰悕绉" + + $"鎻愪緵鐨勬柟娉曠偣鍚嶇О({fileNameInfoRes.鎺堣绔犺妭}鐨勫熀鏈蹇,{fileNameInfoRes.鎺堣绔犺妭}鐨勭粌涔犱笌搴旂敤,{fileNameInfoRes.鎺堣绔犺妭}鐨勪緥棰樿瑙,{knows})銆" + + $"3.鍏宠仈鍚堝苟鐭ヨ瘑鍐呭鐩镐技鐨勭煡璇嗙偣鏉ュ悎骞朵负鐭ヨ瘑鐗囨銆" + + $"鐭ヨ瘑鐗囨浣跨敤鍏宠仈鐭ヨ瘑鐐逛腑鐨勬渶灏忓紑濮嬫椂闂翠富棰樹负鍏宠仈鐭ヨ瘑鐐圭殑涓婚鍒嗘瀽,鍐呭鎬荤粨涓哄叧鑱旂煡璇嗙偣鐨勫唴瀹规荤粨鍒嗘瀽銆" + + $"4.鍩轰簬'鐭ヨ瘑鐗囨'鐨'鍐呭鎬荤粨'鍔犱笂'涓婚'鏉ュ垎鏋愯繖涓墖娈靛涓婚鐨勮瑙ｅ唴瀹逛负鏂扮殑涓婚 渚嬪(鏁板垪鐨勫熀鏈蹇)銆" + + $"5.鍒嗛厤绌轰綑鏈娇鐢ㄧ殑鏃堕棿娈靛埌鍐呭鐩歌繎鐨勭煡璇嗙墖娈垫椂闂村尯闂存潵鑾峰彇鏇村姞璇︾粏鐨勪笂涓嬫枃,浣嗘槸璇烽伩鍏嶇煡璇嗙墖娈典箣闂存椂闂撮噸鍚堛" + + $"杈撳叆锛氬寘鍚椂闂存埑鐨勮棰戝瓧骞曟枃鏈" + + $"浠ヤ笅鏄寘鍚椂闂寸殑瑙嗛瀛楀箷鏂囨湰銆" + + $"瀛楀箷鏍煎紡(璇磋瘽浜:寮濮嬬:缁撴潫绉:鍐呭|涓嬩竴娈靛瓧骞).瀛楀箷鍒楄〃 {captions.Captions}" + + $"杈撳嚭鏍煎紡({resFormat})"; + postMessages = + $"浣犵幇鍦ㄩ渶瑕佸鐞嗙敤鎴锋彁渚涚殑瑙嗛瀛楀箷鍐呭锛屼粠涓彁鍙栦笌涓浗楂樿冩暟瀛﹁冭瘯{fileNameInfoRes.鎺堣绔犺妭}鐩稿叧鐨勬柟娉曠偣锛屽苟鎸夌収鎸囧畾鐨勬楠よ繘琛屽垎鏋愩傞鍏堬紝浣犲緱浠旂粏闃呰鐢ㄦ埛鐨勯渶姹傦紝纭繚姣忎竴姝ラ兘姝ｇ‘鎵ц" + + $"鐢ㄦ埛鐨勯渶姹:" + + $"1.瑕佹眰璇嗗埆瀛楀箷涓殑鏂规硶鐐癸紝杩欎簺鏂规硶鐐归渶瑕佸睘浜巤fileNameInfoRes.鎺堣绔犺妭}绔犺妭銆傞渶瑕佷綘閫氳瀛楀箷锛屾壘鍒拌繖浜涘叧閿瘝鍑虹幇鐨勫湴鏂癸紝骞惰褰曞搴旂殑鏃堕棿鎴" + + $"2.鏄牴鎹彁鍙栫殑鏂规硶鐐瑰悕绉板尮閰嶇敤鎴锋彁渚涚殑鍒楄〃銆備緥濡傦紝濡傛灉瀛楀箷涓彁鍒扳滈掑鏁板垪鈥濇垨鈥滈氶」鍏紡鈥濓紝浣犻渶瑕佺‘璁よ繖浜涙槸鍚﹀湪鐢ㄦ埛缁欏嚭鐨勬柟娉曠偣鍒楄〃涓紝骞跺皢瀹冧滑褰掔被鍒版纭殑鍚嶇О涓" + + $"鐢ㄦ埛鎻愪緵鐨勬柟娉曠偣鍚嶇О({fileNameInfoRes.鎺堣绔犺妭}鐨勫熀鏈蹇,{fileNameInfoRes.鎺堣绔犺妭}鐨勭粌涔犱笌搴旂敤,{fileNameInfoRes.鎺堣绔犺妭}鐨勪緥棰樿瑙,{knows})銆" + + $"3.鍏宠仈鍚堝苟鐩镐技鐨勭煡璇嗙偣锛屽舰鎴愮煡璇嗙墖娈点傛瘮濡傦紝濡傛灉澶氫釜鏃堕棿娈甸兘璁ㄨ鏁板垪鐨勫熀鏈蹇碉紝浣犻渶瑕佸皢瀹冧滑鍚堝苟锛屽苟閫夋嫨鏈鏃╃殑鏃堕棿浣滀负寮濮嬫椂闂淬傚悓鏃讹紝闇瑕佹荤粨杩欎簺鐗囨鐨勫唴瀹癸紝骞剁敓鎴愭柊鐨勪富棰樺悕绉帮紝濡傗滄暟鍒楃殑鍩烘湰姒傚康鈥濄" + + $"濡傛灉澶氫釜鐗囨閮芥秹鍙婃暟鍒楃殑瀹氫箟鍜屼緥瀛愶紝鍙兘鍚堝苟涓衡滄暟鍒楃殑鍩烘湰姒傚康鈥" + + $"4.鍒嗛厤鏈娇鐢ㄧ殑鏃堕棿娈靛埌鐩歌繎鐨勭煡璇嗙墖娈碉紝閬垮厤鏃堕棿閲嶅彔銆傝繖鍙兘闇瑕佹鏌ユ槸鍚︽湁鍓╀綑鐨勬椂闂存鏈褰掔被锛屽苟灏嗗叾鍚堝苟鍒板悎閫傜殑鐭ヨ瘑鐗囨涓紝浠ヤ赴瀵屼笂涓嬫枃" + + $"杈撳叆锛氬寘鍚椂闂存埑鐨勮棰戝瓧骞曟枃鏈" + + $"浠ヤ笅鏄寘鍚椂闂寸殑瑙嗛瀛楀箷鏂囨湰銆" + + $"瀛楀箷鏍煎紡(璇磋瘽浜:寮濮嬬:缁撴潫绉:鍐呭|涓嬩竴娈靛瓧骞).瀛楀箷鍒楄〃 {captions.Captions}" + + $"杈撳嚭鏍煎紡({resFormat})"; + + var questionRes = await ChatAsync(task, postMessages, null); + //var postMessages = + // $"浣犵殑浠诲姟鏄垎鏋愯棰戝瓧骞曞唴瀹癸紝绮惧噯鎻愬彇鍑轰笌涓浗楂樿冩暟瀛﹁冭瘯鐩稿叧鐨勮瘯棰樻柟娉曠偣銆" + + // $"鎸変互涓嬫楠ゅ畬鎴:" + + // $"1.鍑嗙‘璇嗗埆鏂规硶鐐癸細浠庡瓧骞曚腑鎻愬彇涓庤冭瘯绱у瘑鐩稿叧鐨勬柟娉曠偣锛屽挨鍏跺叧娉ㄤ笌缁欏畾鏂规硶鐐圭被鍒浉鍏崇殑鍐呭" + + // $"2.娣卞叆鍒嗘瀽鎬荤粨锛氫緷鎹粰瀹氱殑鏂规硶鐐圭被鍒繘琛岀害鏉燂紝纭畾鎻愬彇鍑虹殑鏂规硶鐐规墍灞炵被鍒" + + // $"缁欏畾鏂规硶鐐瑰寘鎷(鍩烘湰姒傚康,璇惧爞缁冧範,渚嬮璁茶В,瑙ｉ鎶宸...)銆" + + // $"3.缁嗚嚧鍒嗙被鏂规硶鐐癸細鎸夌収瀛︾鏂规硶鐐硅繘琛屽垎绫伙紝鍏蜂綋缁嗗寲鍒扮壒瀹氱珷鑺備笌涓婚锛屾牸寮忎负 鈥滅珷鑺傦細鍏蜂綋绔犺妭鍚嶇О 涓婚锛氬叿浣撲富棰樺悕绉" + + // $"4.鍚堢悊鍚堝苟鐩镐技鏂规硶鐐逛负鍐呭鐗囨锛岀‘淇濆唴瀹圭殑杩炶疮鎬у拰閫昏緫鎬" + + // $"5.鍏宠仈姣忎釜鍐呭鐗囨鐨勬柟娉曠偣鎵鏈夌殑鏃堕棿骞剁粨鏋勫寲杈撳嚭銆" + + // $"6.鍩轰簬鍐呭鐗囨鐨勫唴瀹规荤粨鍔犱笂涓婚鏉ュ垎鏋愯繖涓墖娈靛涓婚鐨勮瑙ｅ唴瀹逛负鏂扮殑涓婚 渚(鏁板垪鐨勫熀鏈蹇)銆" + + // $"灏藉彲鑳藉欢鍐呭鐗囨鏃堕棿鍖洪棿,浠ヨ幏鍙栬缁嗙殑鏂规硶鐐逛笂涓嬫枃銆" + + // $"杈撳叆锛氬寘鍚椂闂存埑鐨勮棰戝瓧骞曟枃鏈傝緭鍑烘牸寮忥細寮濮嬬,缁撴潫绉,涓婚,鍐呭鎬荤粨" ; + //var postM2 = $"浠ヤ笅鏄寘鍚椂闂寸殑瑙嗛瀛楀箷鏂囨湰銆" + + // $"瀛楀箷鏍煎紡(璇磋瘽浜:寮濮嬬:缁撴潫绉:鍐呭|涓嬩竴娈靛瓧骞).瀛楀箷鍒楄〃 {captions.Captions}" + + // $"杩斿洖鍥哄畾鐨凧SON鏍煎紡({resFormat})"; + + //var questionRes = await ChatAsync(task, postMessages, postM2, resFormat); + if (questionRes.Length <= 3) + throw new Exception("瑙嗛鍒嗘鏁伴噺杩囦綆 =>" + questionRes.Length); + + if (questionRes.Count(s => s.ThemeDetalis == questionRes.First().ThemeDetalis) >= 3) + throw new Exception("瑙嗛鍒嗘涓婚閲嶅 =>" + questionRes.First().ThemeDetalis); + + for (int i = 0; i < questionRes.Length; i++) + { + var item = questionRes[i]; + if (i == questionRes.Length - 1) + item.EndTime = maxVideoTime; + else + item.EndTime = (int)(questionRes[i + 1]?.StartTime ?? 0) - 1; + } + + await RedisExpand.Redis + .HMSetAsync(RedisExpandKey.Task(task), "VideoKnows", questionRes); + + //var postMessages1 = + // $"浣犵殑浠诲姟鏄垎鏋恓son鍐呭骞跺悎骞跺惈涔夌浉浼肩殑涓婚涓烘柊鐨勪富棰" + + // $"鎸変互涓嬫楠ゅ畬鎴:" + + // $"1.鍚堢悊鍚堝苟涓婚瀛楁閲嶅鐩镐技鐨勫璞′负鏂扮殑json瀵硅薄锛岀‘淇濆唴瀹圭殑杩炶疮鎬у拰閫昏緫鎬с" + + // $"2.鍚堝苟瀵硅薄灞炴ф寔缁椂闂翠綆浜60绉掔殑瀵硅薄" + + // $"3.缁撴瀯鍖栬緭鍑恒" + + // $"杈撳叆锛歫son瀵硅薄 鍖呭惈鎬荤粨寮濮嬬,缁撴潫绉,鎸佺画鏃堕棿,涓婚,绔犺妭,鍐呭鎬荤粨" + + // $"浠ヤ笅鏄寘鍚玧son鍐呭鐨勬枃鏈" + + // $" {JsonSerializer.Serialize(questionRes)}" + + // $"杩斿洖鍥哄畾鐨凧SON鏍煎紡({resFormat})"; + + + //var questionRes1 = await ChatAsync(task, postMessages1, resFormat); + ////questionRes1 = MergeRes(questionRes1).ToArray(); + + var gptRes = new TaskRes(captions); + await RedisExpand.Redis + .HMSetAsync(RedisExpandKey.Task(task), "ChatAnalysis", gptRes); + RedisExpand.InsertChannel(RedisChannelEnum.EndTask, task); + return gptRes; + } + public async Task ChatAsync(string task, string postMessages, string postMessages1, string model= "deepseek-reasoner") + { + var maxTokens = 4000; + Message[] messageArr = [ + new Message(postMessages,"user"), + string.IsNullOrEmpty(postMessages1)?null:new Message(postMessages1,"user"), + ]; + messageArr = messageArr.Where(s => s != null).ToArray(); + var chatRep = new ChatRequest + { + model = model, + stream = model== "deepseek-reasoner", + max_tokens = maxTokens, + temperature = 0.2f, + messages = messageArr + }; + + RedisExpand.SetTaskGPTReqCached(task, chatRep); + var chatResp = await chatClient.Chat(chatRep); + var chatResContent = chatResp?.res; + if (string.IsNullOrEmpty(chatResContent)) + throw new Exception("GPT杩斿洖message鏃犳晥缁撴灉"); + if (chatResp != null) + RedisExpand.SetTaskGPTCached(task, new object[] { chatResp.Value.res, chatResp.Value.u }); + + chatResContent = chatResContent?.Replace("瀛楀箷鍐呭", "璇惧爞鎯呭喌"); + chatResContent = chatResContent?.Replace("\n", ""); + chatResContent = chatResContent?.Replace("```json", ""); + chatResContent = chatResContent?.Replace("```", ""); + chatResContent = chatResContent?.Replace("}{", "},{"); + chatResContent = chatResContent?.Replace("}|{", "},{"); + chatResContent = chatResContent?.Trim(); + var startsStr = typeof(T).IsArray ? "[" : "{"; + var endStr = typeof(T).IsArray ? "]" : "}"; + if (!chatResContent.StartsWith(startsStr)) + chatResContent = startsStr + chatResContent; + if (!chatResContent.EndsWith(endStr)) + chatResContent = chatResContent + endStr; + var questionRes = JsonSerializer.Deserialize(chatResContent); + if (questionRes is null) + throw new Exception("ChatGPT杩斿洖鏃犳晥缁撴灉"); + return questionRes; + } + } +} diff --git a/VideoAnalysisCore/AICore/SherpaOnnx/SenseVoice.cs b/VideoAnalysisCore/AICore/SherpaOnnx/SenseVoice.cs index 7ca92a5..130a328 100644 --- a/VideoAnalysisCore/AICore/SherpaOnnx/SenseVoice.cs +++ b/VideoAnalysisCore/AICore/SherpaOnnx/SenseVoice.cs @@ -22,9 +22,9 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx /// /// 鍒濆鍖 SenseVoice /// - /// + /// 榛樿6绾跨▼ /// 鏄惁浣跨敤gpu 鎶ラ敊璇风湅瀹夎CUDA鐜 - public static void Init(int numThreads =4,bool useGPU=false,bool useHotwords = false) + public static void Init(int numThreads =6,bool useGPU=false,bool useHotwords = false) { Console.WriteLine("鍒濆鍖 SenseVoice"); OfflineRecognizerConfig config = new OfflineRecognizerConfig(); @@ -57,27 +57,29 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx ////瀹冩寚瀹氭悳绱㈣繃绋嬩腑瑕佷繚鐣欑殑娲诲姩璺緞鏁 //config.MaxActivePaths =4; #endregion - //鍚敤鐑瘝鍔熻兘 - if (false) - { - //鐑瘝鐩綍 - config.HotwordsFile = Path.Combine(AppCommon.AIModelFile, "Hotwords.txt"); - config.DecodingMethod = "modified_beam_search"; - //鐑瘝寰楀垎 - config.HotwordsScore = 1.5f; - config.ModelConfig.ModelingUnit = "cjkchar+bpe"; - config.ModelConfig.BpeVocab = Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", "bpe.model"); - config.ModelConfig.Transducer = new OfflineTransducerModelConfig() - { - Decoder = Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", "decoder-epoch-99-avg-1.onnx"), - Encoder = Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", "encoder-epoch-99-avg-1.onnx"), - Joiner = Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", "joiner-epoch-99-avg-1.onnx"), - }; + #region 鐑瘝鍔熻兘[鏃犳晥] + //if (false) + //{ + // //鐑瘝鐩綍 + // config.HotwordsFile = Path.Combine(AppCommon.AIModelFile, "Hotwords.txt"); + // config.DecodingMethod = "modified_beam_search"; + // //鐑瘝寰楀垎 + // config.HotwordsScore = 1.5f; + + // config.ModelConfig.ModelingUnit = "cjkchar+bpe"; + // config.ModelConfig.BpeVocab = Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", "bpe.model"); + // config.ModelConfig.Transducer = new OfflineTransducerModelConfig() + // { + // Decoder = Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", "decoder-epoch-99-avg-1.onnx"), + // Encoder = Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", "encoder-epoch-99-avg-1.onnx"), + // Joiner = Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", "joiner-epoch-99-avg-1.onnx"), + // }; + //} + #endregion - } //鍙嶈浆鏂囨湰瑙勮寖鍖栬鍒 fst 鐨勮矾寰 - config.RuleFsts = string.Empty; + config.RuleFsts = Path.Combine(AppCommon.AIModelFile, "itn_subject_sx.fst"); #if DEBUG config.ModelConfig.Debug = 1; @@ -214,7 +216,8 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx { res.Add(new() { - Text = ExpandFunction.HandleFormula(stream.Result.Text), + Text = stream.Result.Text, + //Text = ExpandFunction.HandleFormula(stream.Result.Text), Start = (float)Math.Round(startTime, 2, MidpointRounding.AwayFromZero), End = (float)Math.Round(startTime + duration, 2, MidpointRounding.AwayFromZero), }); @@ -241,7 +244,9 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx { res.Add(new() { - Text = ExpandFunction.HandleFormula(stream.Result.Text), + + Text = stream.Result.Text, + //Text = ExpandFunction.HandleFormula(stream.Result.Text), Start = (float)Math.Round(startTime, 2, MidpointRounding.AwayFromZero), End = (float)Math.Round(startTime + duration, 2, MidpointRounding.AwayFromZero), }); diff --git a/VideoAnalysisCore/Common/AppCommon.cs b/VideoAnalysisCore/Common/AppCommon.cs index b53ffc6..769d453 100644 --- a/VideoAnalysisCore/Common/AppCommon.cs +++ b/VideoAnalysisCore/Common/AppCommon.cs @@ -10,12 +10,14 @@ using System.Linq; using System.Reflection; using System.Runtime.Loader; using System.Text; +using System.Text.Json; using System.Text.RegularExpressions; using System.Threading.Tasks; using UserCenter.Model.Interface; using VideoAnalysisCore.AICore.SherpaOnnx; using VideoAnalysisCore.Enum; using VideoAnalysisCore.Interface; +using VideoAnalysisCore.Model; using VideoAnalysisCore.Model.Dto; using Whisper.net; using static System.Runtime.InteropServices.JavaScript.JSType; @@ -90,18 +92,8 @@ namespace VideoAnalysisCore.Common /// public static class ExpandFunction { - static Dictionary FormulaData = new Dictionary() - { - { "闃垮皵娉","伪"}, - { "璐濆","尾"}, - { "浼介┈","纬"}, - { "寰峰皵濉","螖"}, - { "娲","蟺"}, - { "瑗挎牸椹","鈭"}, - { "娆х背浼","惟"}, - { "鏅タ","唯"}, - }; - static string FormulaDataKey = string.Join("|", FormulaData.Keys); + static Dictionary FormulaData; + static string FormulaDataKey; /// /// 澶勭悊鏁板鍏紡 /// @@ -109,6 +101,15 @@ namespace VideoAnalysisCore.Common /// public static string HandleFormula(string f) { + if (FormulaData is null) + { + var hotwords = JsonSerializer.Deserialize(File.ReadAllText(Path.Combine(AppCommon.AIModelFile, "Hotwords.json"))); + foreach (var item in hotwords.OrderByDescending(s=>s.key.Count())) + foreach (var key in item.v) + FormulaData.Add(key, item.key); + } + if (string.IsNullOrEmpty(FormulaDataKey)) + FormulaDataKey = string.Join("|", FormulaData.Keys.Count); if (string.IsNullOrEmpty(f)) return f; return Regex.Replace(f, FormulaDataKey, @@ -307,6 +308,7 @@ namespace VideoAnalysisCore.Common /// /// public GptConfig ChatGpt { get; set; } = new GptConfig(); + public GptConfig DeepSeek { get; set; } = new GptConfig(); public GptConfig KIMI { get; set; } = new GptConfig(); } diff --git a/VideoAnalysisCore/Model/HotwordMode.cs b/VideoAnalysisCore/Model/HotwordMode.cs new file mode 100644 index 0000000..b842e37 --- /dev/null +++ b/VideoAnalysisCore/Model/HotwordMode.cs @@ -0,0 +1,14 @@ +锘縰sing System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace VideoAnalysisCore.Model +{ + public class HotwordMode + { + public string key { get; set; } + public string[] v { get; set; } + } +} diff --git a/VideoAnalysisCore/Model/KnowledgeInfo.cs b/VideoAnalysisCore/Model/KnowledgeInfo.cs index 8eea235..39905bb 100644 --- a/VideoAnalysisCore/Model/KnowledgeInfo.cs +++ b/VideoAnalysisCore/Model/KnowledgeInfo.cs @@ -6,10 +6,10 @@ using System.Text; using System.Threading.Tasks; using VideoAnalysisCore.Interface; -namespace VideoAnalysisCore.Model.Dto +namespace VideoAnalysisCore.Model { [SugarTable("knowledgeinfo")] - public class KnowledgeInfo: IKnowsDB + public class KnowledgeInfo : IKnowsDB { [SugarColumn(IsPrimaryKey = true, ColumnDescription = "Id 涓婚敭", ColumnName = "id")] public long Id { get; set; } diff --git a/VideoAnalysisCore/VideoAnalysisCore.csproj b/VideoAnalysisCore/VideoAnalysisCore.csproj index 70c82a8..7722e4f 100644 --- a/VideoAnalysisCore/VideoAnalysisCore.csproj +++ b/VideoAnalysisCore/VideoAnalysisCore.csproj @@ -8,7 +8,9 @@ - + + + @@ -22,7 +24,13 @@ Never - + + Always + + + Always + + Always