marking
/
Learn.VideoAnalysis
19
0
Fork 0
Code Pull Requests Actions Releases Activity

新增 deepseek_api 

新增 fst语音转文本后处理流程
This commit is contained in:
小肥羊 2025-02-07 17:35:18 +08:00
parent c14842a99d
commit 6c5c7db4f5
13 changed files with 594 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
VideoAnalysis/Controllers/ApiController.cs
View File

@ -8,6 +8,7 @@ using VideoAnalysisCore.AICore.SherpaOnnx;
using UserCenter.Model.Enum;
using VideoAnalysisCore.AICore.GPT.ChatGPT;
using VideoAnalysisCore.AICore.GPT;
using System.Text.Json;
namespace Learn.VideoAnalysis.Controllers
{
@ -75,16 +76,24 @@ namespace Learn.VideoAnalysis.Controllers
/// <summary>
/// Test
/// »ñÈ¡FTS_Data str
/// </summary>
/// <param name="taskId"></param>
/// <param name="path">·¾¶</param>
/// <returns></returns>
[HttpGet(Name = "Test")]
public async Task<IActionResult> Test(long taskId)
[HttpGet(Name = "fts_data")]
public async Task<IActionResult> FTS_Data(string path= "itn_subject_sx.fst")
{
//重新开始执行GPT分析
chatGPT.GetKnow(taskId.ToString());
return Ok();
var hotwords = JsonSerializer
.Deserialize<HotwordMode[]>(System.IO.File.ReadAllText(Path.Combine(AppCommon.AIModelFile, "Hotwords.json")));
var res = new List<string>(100);
foreach (var element in hotwords.OrderByDescending(s => s.key.Count()))
foreach (var e in element.v)
res.Add($"""("{e}", "{element.key}")""");
var pyFile = System.IO.File.ReadAllText(Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-fst.py"));
var resStr = pyFile
.Replace("(fts_data)", "[" + string.Join(',', res) + "]")
.Replace("(path)", path);
return Ok(resStr);
}
[NonAction]

5
VideoAnalysis/Program.cs
View File

@ -8,6 +8,7 @@ using VideoAnalysisCore.AICore.GPT;
using VideoAnalysisCore.AICore.GPT.KIMI;
using VideoAnalysisCore.AICore.GPT.ChatGPT;
using Microsoft.Extensions.FileProviders;
using VideoAnalysisCore.AICore.GPT.DeepSeek;
@ -81,8 +82,10 @@ namespace Learn.VideoAnalysis
builder.Services.AddHttpClient();
builder.Services.AddSingleton<ChatGPTClient>();
builder.Services.AddSingleton<DeepSeekGPTClient>();
//builder.Services.AddSingleton<IBserGPT, KIMI_GPT>();
builder.Services.AddSingleton<IBserGPT, Chat_GPT>();
//builder.Services.AddSingleton<IBserGPT, Chat_GPT>();
builder.Services.AddSingleton<IBserGPT, DeepSeek_GPT>();
var app = builder.Build();

4
VideoAnalysis/appsettings.json
View File

@ -30,6 +30,10 @@
"Host": "https://api.g4f.icu/",
//"Host": "https://api.oaibest.com/",
"ApiKey": "sk-D15tBln31N7dI9Fi7lds7OySFv5tOEK7DMNsG5rY2E6DCr4s"
},
"DeepSeek": {
"Host": "https://api.deepseek.com/chat/completions",
"ApiKey": "sk-88d3d2bc3dae4d50854b2569b281cf76"
}
},
"DB": {

1
VideoAnalysisCore/AICore/GPT/BserGPT.cs
View File

@ -15,7 +15,6 @@ namespace VideoAnalysisCore.AICore.GPT
/// </summary>
/// <param name="task">任务id</param>
/// <returns></returns>
public Task<TaskRes> CallGPT(string task);
public Task<TaskRes> GetKnow(string task);
}
}

31
VideoAnalysisCore/AICore/GPT/ChatGPT/Chat_GPT.cs
View File

@ -87,25 +87,24 @@ namespace VideoAnalysisCore.AICore.GPT.ChatGPT
$"1.分析出高中{subject}课堂授课的主要章节(例如 章节: 数列),章节范围限定在[{string.Join(',', xkwKnows)}]范围." +
$"2.分析出这堂课的主要授课内容." +
$"输出格式 json字符串 对象格式{fileNameResFormat}";
var fileNameInfoRes = await ChatAsync<FileNameInfo>(task, fileNamePostMessages,null, fileNameResFormat);
var fileNameInfoRes = await ChatAsync<FileNameInfo>(task, fileNamePostMessages, null, fileNameResFormat);
var captions = ExpandFunction.GetSpeakerCaptions(task);
var maxVideoTime = captions?.TimeBase?.LastOrDefault()?.End??0;
var maxVideoTime = captions?.TimeBase?.LastOrDefault()?.End ?? 0;
var criteriaBuilder = new StringBuilder();
//var resFormat = """[{"StartTime":开始秒(number),"EndTime":结束秒(number),"Section":章节(string),"Theme":主题(string),"ThemeDetalis":主题详情(string),"Content":内容总结(string)}]""";
var resFormat = """[{"StartTime":开始秒(number),"Section":章节(string),"Theme":主题(string),"ThemeDetalis":主题详情(string),"Content":内容总结(string)}]""";
var know = await knowledgeInfoDB.GetFirstAsync(s => s.Name == fileNameInfoRes.授课章节);
var knowledgeInfos = await knowledgeInfoDB.AsQueryable().ToChildListAsync(s => s.Parent_Id, know.Id);
var knows = "数列的概念,数列的定义,项的表示,数列的表示方法,通项公式,递推公式,图像表示,数列的类型,等差数列,等比数列,其他特殊数列,数列的性质,单调性,有限性,数列的求和,等差数列求和公式,等比数列求和公式,数列极限,递推关系";
knows = string.Join(',', knowledgeInfos.Select(s => s.Name));
var knows = string.Join(',', knowledgeInfos.Select(s => s.Name));
var postMessages =
$"你的任务是分析视频字幕内容并提取出中国高考考试试题方法点,然后根据步骤分析出知识片段" +
$"按以下步骤完成:" +
$"1.识别方法点:提取字幕内容中与{subject}考试属于{fileNameInfoRes.授课章节}章节相关的方法点。" +
$"2.分析总结:基于提取出的方法点名称来匹配我提供的方法点名称" +
$"提供的方法点名称(基本概念,课堂练习,例题讲解,{knows})。" +
$"提供的方法点名称({fileNameInfoRes.授课章节}的基本概念,{fileNameInfoRes.授课章节}的练习与应用,{fileNameInfoRes.授课章节}的例题讲解,{knows})。" +
$"3.关联合并知识内容相似的知识点来合并为知识片段。" +
$"知识片段使用关联知识点中的最小开始时间主题为关联知识点的主题分析,内容总结为关联知识点的内容总结分析。" +
$"4.基于'知识片段'的'内容总结'加上'主题'来分析这个片段对主题的讲解内容为新的主题 例如(数列的基本概念)。" +
@ -114,6 +113,20 @@ namespace VideoAnalysisCore.AICore.GPT.ChatGPT
$"以下是包含时间的视频字幕文本。" +
$"字幕格式(说话人:开始秒:结束秒:内容|下一段字幕).字幕列表 {captions.Captions}" +
$"输出格式({resFormat})";
postMessages =
$"你现在需要处理用户提供的视频字幕内容，从中提取与中国高考数学考试{fileNameInfoRes.授课章节}相关的方法点，并按照指定的步骤进行分析。首先，你得仔细阅读用户的需求，确保每一步都正确执行" +
$"用户的需求:" +
$"1.要求识别字幕中的方法点，这些方法点需要属于{fileNameInfoRes.授课章节}章节。需要你通读字幕，找到这些关键词出现的地方，并记录对应的时间戳" +
$"2.是根据提取的方法点名称匹配用户提供的列表。例如，如果字幕中提到“递增数列”或“通项公式”，你需要确认这些是否在用户给出的方法点列表中，并将它们归类到正确的名称下" +
$"用户提供的方法点名称({fileNameInfoRes.授课章节}的基本概念,{fileNameInfoRes.授课章节}的练习与应用,{fileNameInfoRes.授课章节}的例题讲解,{knows})。" +
$"3.关联合并相似的知识点，形成知识片段。比如，如果多个时间段都讨论数列的基本概念，你需要将它们合并，并选择最早的时间作为开始时间。同时，需要总结这些片段的内容，并生成新的主题名称，如“数列的基本概念”。" +
$"如果多个片段都涉及数列的定义和例子，可能合并为“数列的基本概念”" +
$"4.分配未使用的时间段到相近的知识片段，避免时间重叠。这可能需要检查是否有剩余的时间段未被归类，并将其合并到合适的知识片段中，以丰富上下文" +
$"输入：包含时间戳的视频字幕文本。" +
$"以下是包含时间的视频字幕文本。" +
$"字幕格式(说话人:开始秒:结束秒:内容|下一段字幕).字幕列表 {captions.Captions}" +
$"输出格式({resFormat})";
var questionRes = await ChatAsync<VideoKnowRes[]>(task, postMessages, null, resFormat);
//var postMessages =
// $"你的任务是分析视频字幕内容，精准提取出与中国高考数学考试相关的试题方法点。" +
@ -133,9 +146,9 @@ namespace VideoAnalysisCore.AICore.GPT.ChatGPT
//var questionRes = await ChatAsync<VideoKnowRes[]>(task, postMessages, postM2, resFormat);
if (questionRes.Length <= 3)
throw new Exception("视频分段数量过低 =>"+ questionRes.Length);
throw new Exception("视频分段数量过低 =>" + questionRes.Length);
if (questionRes.Count(s=>s.ThemeDetalis == questionRes.First().ThemeDetalis) >= 3)
if (questionRes.Count(s => s.ThemeDetalis == questionRes.First().ThemeDetalis) >= 3)
throw new Exception("视频分段主题重复 =>" + questionRes.First().ThemeDetalis);
for (int i = 0; i < questionRes.Length; i++)
@ -144,7 +157,7 @@ namespace VideoAnalysisCore.AICore.GPT.ChatGPT
if (i == questionRes.Length - 1)
item.EndTime = maxVideoTime;
else
item.EndTime = (int)(questionRes[i + 1]?.StartTime??0) - 1;
item.EndTime = (int)(questionRes[i + 1]?.StartTime ?? 0) - 1;
}
await RedisExpand.Redis
@ -171,7 +184,7 @@ namespace VideoAnalysisCore.AICore.GPT.ChatGPT
RedisExpand.InsertChannel(RedisChannelEnum.EndTask, task);
return gptRes;
}
public async Task<T> ChatAsync<T>(string task, string postMessages,string postMessages1, string resFormat)
public async Task<T> ChatAsync<T>(string task, string postMessages, string postMessages1, string resFormat)
{
var maxTokens = 4000;
Message[] messageArr = [

142
VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeekClient.cs Normal file
View File

@ -0,0 +1,142 @@
using VideoAnalysisCore.Common;
using System.Net.Http.Headers;
using System.Text;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json.Linq;
using System.Net.Http;
using Newtonsoft.Json;
using System.Net.Http.Json;
using AntDesign;
using OneOf.Types;
using System.Net;
using VideoAnalysisCore.AICore.GPT.KIMI;
namespace VideoAnalysisCore.AICore.GPT.DeepSeek
{
public class DeepSeekGPTClient
{
public static string Host = AppCommon.Config.ChatGpt.DeepSeek.Host;
public static string ApiKey = AppCommon.Config.ChatGpt.DeepSeek.ApiKey;
private readonly IHttpClientFactory _httpClientFactory;
public DeepSeekGPTClient(IHttpClientFactory httpClientFactory)
{
_httpClientFactory = httpClientFactory;
}
/// <summary>
/// Chat
/// </summary>
/// <param name="chatReq"></param>
/// <returns>Return HttpResponseMessage for SSE</returns>
public async Task<(Usage u, string res)?> Chat(ChatRequest chatReq)
{
if (chatReq.stream) return await ChatSSE(chatReq);
var requestBody = System.Text.Json.JsonSerializer.Serialize(chatReq);
var chatResp = await PostJsonStreamAsync("v1/chat/completions", requestBody);
var res1 = await chatResp.Content.ReadAsStringAsync();
var res = await chatResp.Content.ReadFromJsonAsync<ChatRes>();
if (res is null || res.error != null)
throw new Exception($" ChatGPT模型返回异常 返回参数: " +
$" {System.Text.Json.JsonSerializer.Serialize(res)}");
var chatResContent = res?.choices.FirstOrDefault()?.message.content.Trim();
if (string.IsNullOrEmpty(chatResContent))
return null;
return (res.usage, chatResContent);
}
private async Task<HttpResponseMessage> PostJsonStreamAsync(string path, string json)
{
var uriBuilder = new UriBuilder(Host + path);
var maxRestart = 4;
var errorMSG = new Exception[maxRestart];
for (int i = 0; i < maxRestart; i++)
{
try
{
var client = _httpClientFactory.CreateClient();
client.DefaultRequestHeaders.Authorization =
new AuthenticationHeaderValue("Bearer", ApiKey);
client.Timeout = TimeSpan.FromSeconds(60 * 20);//超时时间20分钟
client.DefaultRequestVersion = HttpVersion.Version20;
client.DefaultVersionPolicy = HttpVersionPolicy.RequestVersionOrLower;
client.DefaultRequestHeaders.ConnectionClose = true;
var content = new StringContent(json, Encoding.UTF8, "application/json");
return await client.PostAsync(uriBuilder.Uri, content);
}
catch (Exception e)
{
errorMSG[i] = e;
Console.WriteLine("====================[请求异常,重试]====================");
Console.WriteLine(uriBuilder.Uri);
Console.WriteLine(e.Message);
Console.WriteLine(e.StackTrace);
Console.WriteLine("==============================================");
}
Thread.Sleep(1000);
}
throw errorMSG.Last(s => s != null);
}
/// <summary>
/// ChatSSE[流式传输 更稳定]
/// </summary>
/// <param name="chatReq"></param>
/// <returns>Return HttpResponseMessage for SSE</returns>
public async Task<(Usage u, string res)?> ChatSSE(ChatRequest chatReq)
{
chatReq.stream = true;
var requestBody = System.Text.Json.JsonSerializer.Serialize(chatReq);
var chatResp = await PostJsonStreamAsync("/v1/chat/completions", requestBody);
using var stream = await chatResp.Content.ReadAsStreamAsync();
using var reader = new StreamReader(stream, Encoding.UTF8);
string line;
var messageBuilder = new StringBuilder();
var lastChat = new ChatResSSE();
var splitCount = "data:".Length;
while ((line = await reader.ReadLineAsync()) != null)
{
if (line.EndsWith("[DONE]"))
{
// 表示一条消息结束
string message = messageBuilder.ToString();
messageBuilder.Clear();
var u = lastChat?.usage;
if (u == null || string.IsNullOrEmpty(message))
return null;
return (u, message);
}
else if (line.StartsWith("data:"))
{
try
{
var data = System.Text.Json.JsonSerializer.Deserialize<ChatResSSE>(line.Substring(splitCount).Trim());
lastChat = data;
var str = data?.choices.FirstOrDefault()?.delta.content;
if (!string.IsNullOrEmpty(str))
messageBuilder.Append(str);
}
catch (Exception e)
{
Console.WriteLine("异常 ChatSSE=>");
Console.WriteLine(line);
Console.WriteLine(e.Message);
Console.WriteLine(e.StackTrace);
}
}
}
return null;
}
}
}

137
VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeekModel.cs Normal file
View File

@ -0,0 +1,137 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using VideoAnalysisCore.AICore.GPT.KIMI;
namespace VideoAnalysisCore.AICore.GPT.DeepSeek
{
/// <summary>
/// 请求数据
/// </summary>
public class ChatRequest
{
/// <summary>
/// 对话
/// </summary>
public Message[] messages { get; set; }
/// <summary>
/// 提问种子值[用来确保 相同参数请求尽可能返回相同参数]
/// <para>默认:null</para>
/// <para>此功能处于 Beta 阶段。 如果指定，我们的系统将尽最大努力确定性地采样，这样具有相同 and 参数的重复请求应该返回相同的结果。 无法保证确定性，您应该参考 response 参数来监控后端的变化</para>
/// </summary>
public int? seed { get; set; } =null;
/// <summary>
/// 推理模型 (deepseek-reasoner)
/// </summary>
public string model { get; set; } = "deepseek-reasoner";
public float max_tokens { get; set; } = 4000;
/// <summary>
/// 要使用的采样温度，介于 0 和 2 之间。较高的值（如 0.8）将使输出更加随机，而较低的值（如 0.2）将使其更加集中和确定。 我们通常建议更改此项或同时更改两者。top_p
/// <para> 默认为 1</para>
/// <para> <see cref="ChatRequest.top_p"/>联动</para>
/// </summary>
public float temperature { get; set; } = 0.2f;
/// <summary>
/// 一种替代温度采样的方法，称为原子核采样， 其中，模型考虑具有top_p概率的标记的结果 质量。所以 0.1 表示仅包含前 10% 概率质量的代币 被考
/// <para>建议与<see cref="ChatRequest.temperature"/>联动</para>
/// </summary>
public float top_p { get; set; } = 0.5f;
/// <summary>
/// 一个对象，用于指定模型必须输出的格式。设置为 enable 结构化输出，确保模型与您提供的 JSON 匹配 图式。
/// </summary>
public object response_format = new { type = "json_object" };
/// <summary>
/// 流式返回
/// </summary>
public bool stream =false;
/// <summary>
/// 您希望模型为此请求生成的 Output types。 大多数模型都能够生成文本，这是
/// <para>默认设置： ["text"]</para>
/// <para>该模型还可用于生成音频。自 请求此模型同时生成文本和音频响应，您可以 用：gpt-4o-audio-preview["text", "audio"]</para>
/// </summary>
public string modalities = "[\"json\"]";
}
public class Message
{
public Message(string content, string role)
{
this.role = role;
this.content = content;
}
public string role { get; set; }
public string content { get; set; }
/// <summary>
/// 推理内容
/// </summary>
public string reasoning_content { get; set; }
public string refusal { get; set; }
}
/// <summary>
/// gpt返回值
/// </summary>
public class ChatRes
{
public string id { get; set; }
public string _object { get; set; }
public int created { get; set; }
public string model { get; set; }
public ChatResError error { get; set; }
public Choice[] choices { get; set; }
public Usage usage { get; set; }
/// <summary>
/// 系统指纹
/// </summary>
public string system_fingerprint { get; set; }
}
public class Usage
{
public int prompt_tokens { get; set; }
public int completion_tokens { get; set; }
public int total_tokens { get; set; }
}
public class Choice
{
public int index { get; set; }
public Message message { get; set; }
public object logprobs { get; set; }
public string finish_reason { get; set; }
}
public class ChatResError
{
public string message { get; set; }
public string type { get; set; }
}
public class ChatResSSE
{
public string id { get; set; }
public int created { get; set; }
/// <summary>
/// 模型id
/// </summary>
public string model { get; set; }
/// <summary>
/// 对话
/// </summary>
public ChoiceSSE[] choices { get; set; }
/// <summary>
/// token使用情况
/// </summary>
public Usage usage { get; set; }
}
public class ChoiceSSE
{
public int index { get; set; }
public Message delta { get; set; }
public string finish_reason { get; set; }
}
}

202
VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs Normal file
View File

@ -0,0 +1,202 @@
using VideoAnalysisCore.Common;
using System.Text.Json;
using VideoAnalysisCore.Model;
using System.Text;
using System.ComponentModel.DataAnnotations;
using VideoAnalysisCore.Enum;
using System.Reflection;
using VideoAnalysisCore.Model.Dto;
using VideoAnalysisCore.AICore.GPT.Dto;
using System.Threading.Tasks;
using VideoAnalysisCore.AICore.GPT.ChatGPT;
namespace VideoAnalysisCore.AICore.GPT.DeepSeek
{
/// <summary>
/// chatgpt 文本模型
/// </summary>
public class DeepSeek_GPT : IBserGPT
{
private readonly DeepSeekGPTClient chatClient;
private readonly Repository<CourseGradingCriteria> criteriaDB;
private readonly Repository<VideoTask> videoTaskDB;
private readonly Repository<KnowledgeInfo> knowledgeInfoDB;
/// <summary>
/// 初始化
/// </summary>
/// <param name="moonshotClient"></param>
/// <param name="logger"></param>
public DeepSeek_GPT(DeepSeekGPTClient moonshotClient, Repository<CourseGradingCriteria> criteria, Repository<VideoTask> videoTaskDB, Repository<KnowledgeInfo> knowledgeInfoDB)
{
chatClient = moonshotClient;
criteriaDB = criteria;
this.videoTaskDB = videoTaskDB;
this.knowledgeInfoDB = knowledgeInfoDB;
}
/// <summary>
/// 获取知识点
/// </summary>
/// <param name="task">任务id</param>
/// <returns></returns>
public async Task<TaskRes> GetKnow(string task)
{
var taskId = long.Parse(task);
var taskInfo = await videoTaskDB.AsQueryable()
.Where(s => s.Id == taskId)
.FirstAsync();
var subject = "数学";
var xkwKnows = await knowledgeInfoDB.AsQueryable()
.Where(s => s.Course_Id == 27
&& s.Depth == 2)
.Select(s => s.Name).ToArrayAsync();
string title = taskInfo.MediaName;
var fileNameResFormat = "{授课章节: string|null, 授课内容:string}";
var fileNamePostMessages = title +
" 这是一堂课的标题,请你帮我分析一些关于课堂方面的内容." +
$"1.分析出高中{subject}课堂授课的主要章节(例如 章节: 数列),章节范围限定在[{string.Join(',', xkwKnows)}]范围." +
$"2.分析出这堂课的主要授课内容." +
$"输出格式 json字符串 对象格式{fileNameResFormat}";
var fileNameInfoRes = await ChatAsync<FileNameInfo>(task, fileNamePostMessages, null, "deepseek-chat");
var captions = ExpandFunction.GetSpeakerCaptions(task);
var maxVideoTime = captions?.TimeBase?.LastOrDefault()?.End ?? 0;
var criteriaBuilder = new StringBuilder();
//var resFormat = """[{"StartTime":开始秒(number),"EndTime":结束秒(number),"Section":章节(string),"Theme":主题(string),"ThemeDetalis":主题详情(string),"Content":内容总结(string)}]""";
var resFormat = """[{"StartTime":开始秒(number),"Section":章节(string),"Theme":主题(string),"ThemeDetalis":主题详情(string),"Content":内容总结(string)}]""";
var know = await knowledgeInfoDB.GetFirstAsync(s => s.Name == fileNameInfoRes.授课章节);
var knowledgeInfos = await knowledgeInfoDB.AsQueryable().ToChildListAsync(s => s.Parent_Id, know.Id);
var knows = string.Join(',', knowledgeInfos.Select(s => s.Name));
var postMessages =
$"你的任务是分析视频字幕内容并提取出中国高考考试试题方法点,然后根据步骤分析出知识片段" +
$"按以下步骤完成:" +
$"1.识别方法点:提取字幕内容中与{subject}考试属于{fileNameInfoRes.授课章节}章节相关的方法点。" +
$"2.分析总结:基于提取出的方法点名称来匹配我提供的方法点名称" +
$"提供的方法点名称({fileNameInfoRes.授课章节}的基本概念,{fileNameInfoRes.授课章节}的练习与应用,{fileNameInfoRes.授课章节}的例题讲解,{knows})。" +
$"3.关联合并知识内容相似的知识点来合并为知识片段。" +
$"知识片段使用关联知识点中的最小开始时间主题为关联知识点的主题分析,内容总结为关联知识点的内容总结分析。" +
$"4.基于'知识片段'的'内容总结'加上'主题'来分析这个片段对主题的讲解内容为新的主题 例如(数列的基本概念)。" +
$"5.分配空余未使用的时间段到内容相近的知识片段时间区间来获取更加详细的上下文,但是请避免知识片段之间时间重合。" +
$"输入：包含时间戳的视频字幕文本。" +
$"以下是包含时间的视频字幕文本。" +
$"字幕格式(说话人:开始秒:结束秒:内容|下一段字幕).字幕列表 {captions.Captions}" +
$"输出格式({resFormat})";
postMessages =
$"你现在需要处理用户提供的视频字幕内容，从中提取与中国高考数学考试{fileNameInfoRes.授课章节}相关的方法点，并按照指定的步骤进行分析。首先，你得仔细阅读用户的需求，确保每一步都正确执行" +
$"用户的需求:" +
$"1.要求识别字幕中的方法点，这些方法点需要属于{fileNameInfoRes.授课章节}章节。需要你通读字幕，找到这些关键词出现的地方，并记录对应的时间戳" +
$"2.是根据提取的方法点名称匹配用户提供的列表。例如，如果字幕中提到“递增数列”或“通项公式”，你需要确认这些是否在用户给出的方法点列表中，并将它们归类到正确的名称下" +
$"用户提供的方法点名称({fileNameInfoRes.授课章节}的基本概念,{fileNameInfoRes.授课章节}的练习与应用,{fileNameInfoRes.授课章节}的例题讲解,{knows})。" +
$"3.关联合并相似的知识点，形成知识片段。比如，如果多个时间段都讨论数列的基本概念，你需要将它们合并，并选择最早的时间作为开始时间。同时，需要总结这些片段的内容，并生成新的主题名称，如“数列的基本概念”。" +
$"如果多个片段都涉及数列的定义和例子，可能合并为“数列的基本概念”" +
$"4.分配未使用的时间段到相近的知识片段，避免时间重叠。这可能需要检查是否有剩余的时间段未被归类，并将其合并到合适的知识片段中，以丰富上下文" +
$"输入：包含时间戳的视频字幕文本。" +
$"以下是包含时间的视频字幕文本。" +
$"字幕格式(说话人:开始秒:结束秒:内容|下一段字幕).字幕列表 {captions.Captions}" +
$"输出格式({resFormat})";
var questionRes = await ChatAsync<VideoKnowRes[]>(task, postMessages, null);
//var postMessages =
// $"你的任务是分析视频字幕内容，精准提取出与中国高考数学考试相关的试题方法点。" +
// $"按以下步骤完成:" +
// $"1.准确识别方法点：从字幕中提取与考试紧密相关的方法点，尤其关注与给定方法点类别相关的内容" +
// $"2.深入分析总结：依据给定的方法点类别进行约束，确定提取出的方法点所属类别。" +
// $"给定方法点包括(基本概念,课堂练习,例题讲解,解题技巧...)。" +
// $"3.细致分类方法点：按照学科方法点进行分类，具体细化到特定章节与主题，格式为 “章节：具体章节名称 主题：具体主题名称" +
// $"4.合理合并相似方法点为内容片段，确保内容的连贯性和逻辑性" +
// $"5.关联每个内容片段的方法点所有的时间并结构化输出。" +
// $"6.基于内容片段的内容总结加上主题来分析这个片段对主题的讲解内容为新的主题 例(数列的基本概念)。" +
// $"尽可能延内容片段时间区间,以获取详细的方法点上下文。" +
// $"输入：包含时间戳的视频字幕文本。输出格式：开始秒,结束秒,主题,内容总结" ;
//var postM2 = $"以下是包含时间的视频字幕文本。" +
// $"字幕格式(说话人:开始秒:结束秒:内容|下一段字幕).字幕列表 {captions.Captions}" +
// $"返回固定的JSON格式({resFormat})";
//var questionRes = await ChatAsync<VideoKnowRes[]>(task, postMessages, postM2, resFormat);
if (questionRes.Length <= 3)
throw new Exception("视频分段数量过低 =>" + questionRes.Length);
if (questionRes.Count(s => s.ThemeDetalis == questionRes.First().ThemeDetalis) >= 3)
throw new Exception("视频分段主题重复 =>" + questionRes.First().ThemeDetalis);
for (int i = 0; i < questionRes.Length; i++)
{
var item = questionRes[i];
if (i == questionRes.Length - 1)
item.EndTime = maxVideoTime;
else
item.EndTime = (int)(questionRes[i + 1]?.StartTime ?? 0) - 1;
}
await RedisExpand.Redis
.HMSetAsync(RedisExpandKey.Task(task), "VideoKnows", questionRes);
//var postMessages1 =
// $"你的任务是分析json内容并合并含义相似的主题为新的主题" +
// $"按以下步骤完成:" +
// $"1.合理合并主题字段重复相似的对象为新的json对象，确保内容的连贯性和逻辑性。" +
// $"2.合并对象属性持续时间低于60秒的对象" +
// $"3.结构化输出。" +
// $"输入：json对象 包含总结开始秒,结束秒,持续时间,主题,章节,内容总结" +
// $"以下是包含json内容的文本。" +
// $" {JsonSerializer.Serialize(questionRes)}" +
// $"返回固定的JSON格式({resFormat})";
//var questionRes1 = await ChatAsync<VideoKnowRes[]>(task, postMessages1, resFormat);
////questionRes1 = MergeRes(questionRes1).ToArray();
var gptRes = new TaskRes(captions);
await RedisExpand.Redis
.HMSetAsync(RedisExpandKey.Task(task), "ChatAnalysis", gptRes);
RedisExpand.InsertChannel(RedisChannelEnum.EndTask, task);
return gptRes;
}
public async Task<T> ChatAsync<T>(string task, string postMessages, string postMessages1, string model= "deepseek-reasoner")
{
var maxTokens = 4000;
Message[] messageArr = [
new Message(postMessages,"user"),
string.IsNullOrEmpty(postMessages1)?null:new Message(postMessages1,"user"),
];
messageArr = messageArr.Where(s => s != null).ToArray();
var chatRep = new ChatRequest
{
model = model,
stream = model== "deepseek-reasoner",
max_tokens = maxTokens,
temperature = 0.2f,
messages = messageArr
};
RedisExpand.SetTaskGPTReqCached(task, chatRep);
var chatResp = await chatClient.Chat(chatRep);
var chatResContent = chatResp?.res;
if (string.IsNullOrEmpty(chatResContent))
throw new Exception("GPT返回message无效结果");
if (chatResp != null)
RedisExpand.SetTaskGPTCached(task, new object[] { chatResp.Value.res, chatResp.Value.u });
chatResContent = chatResContent?.Replace("字幕内容", "课堂情况");
chatResContent = chatResContent?.Replace("\n", "");
chatResContent = chatResContent?.Replace("```json", "");
chatResContent = chatResContent?.Replace("```", "");
chatResContent = chatResContent?.Replace("}{", "},{");
chatResContent = chatResContent?.Replace("}|{", "},{");
chatResContent = chatResContent?.Trim();
var startsStr = typeof(T).IsArray ? "[" : "{";
var endStr = typeof(T).IsArray ? "]" : "}";
if (!chatResContent.StartsWith(startsStr))
chatResContent = startsStr + chatResContent;
if (!chatResContent.EndsWith(endStr))
chatResContent = chatResContent + endStr;
var questionRes = JsonSerializer.Deserialize<T>(chatResContent);
if (questionRes is null)
throw new Exception("ChatGPT返回无效结果");
return questionRes;
}
}
}

49
VideoAnalysisCore/AICore/SherpaOnnx/SenseVoice.cs
View File

@ -22,9 +22,9 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx
/// <summary>
/// 初始化 SenseVoice
/// </summary>
/// <param name="numThreads"></param>
/// <param name="numThreads">默认6线程</param>
/// <param name="useGPU">是否使用gpu 报错请看安装CUDA环境<see cref="https://k2-fsa.github.io/sherpa/onnx/pretrained_models/whisper/large-v3.html#run-with-gpu-float32"/></param>
public static void Init(int numThreads =4,bool useGPU=false,bool useHotwords = false)
public static void Init(int numThreads =6,bool useGPU=false,bool useHotwords = false)
{
Console.WriteLine("初始化 SenseVoice");
OfflineRecognizerConfig config = new OfflineRecognizerConfig();
@ -57,27 +57,29 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx
////它指定搜索过程中要保留的活动路径数
//config.MaxActivePaths =4;
#endregion
//启用热词功能
if (false)
{
//热词目录
config.HotwordsFile = Path.Combine(AppCommon.AIModelFile, "Hotwords.txt");
config.DecodingMethod = "modified_beam_search";
//热词得分
config.HotwordsScore = 1.5f;
config.ModelConfig.ModelingUnit = "cjkchar+bpe";
config.ModelConfig.BpeVocab = Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", "bpe.model");
config.ModelConfig.Transducer = new OfflineTransducerModelConfig()
{
Decoder = Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", "decoder-epoch-99-avg-1.onnx"),
Encoder = Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", "encoder-epoch-99-avg-1.onnx"),
Joiner = Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", "joiner-epoch-99-avg-1.onnx"),
};
#region 热词功能[无效]
//if (false)
//{
// //热词目录
// config.HotwordsFile = Path.Combine(AppCommon.AIModelFile, "Hotwords.txt");
// config.DecodingMethod = "modified_beam_search";
// //热词得分
// config.HotwordsScore = 1.5f;
// config.ModelConfig.ModelingUnit = "cjkchar+bpe";
// config.ModelConfig.BpeVocab = Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", "bpe.model");
// config.ModelConfig.Transducer = new OfflineTransducerModelConfig()
// {
// Decoder = Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", "decoder-epoch-99-avg-1.onnx"),
// Encoder = Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", "encoder-epoch-99-avg-1.onnx"),
// Joiner = Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20", "joiner-epoch-99-avg-1.onnx"),
// };
//}
#endregion
}
//反转文本规范化规则 fst 的路径
config.RuleFsts = string.Empty;
config.RuleFsts = Path.Combine(AppCommon.AIModelFile, "itn_subject_sx.fst");
#if DEBUG
config.ModelConfig.Debug = 1;
@ -214,7 +216,8 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx
{
res.Add(new()
{
Text = ExpandFunction.HandleFormula(stream.Result.Text),
Text = stream.Result.Text,
//Text = ExpandFunction.HandleFormula(stream.Result.Text),
Start = (float)Math.Round(startTime, 2, MidpointRounding.AwayFromZero),
End = (float)Math.Round(startTime + duration, 2, MidpointRounding.AwayFromZero),
});
@ -241,7 +244,9 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx
{
res.Add(new()
{
Text = ExpandFunction.HandleFormula(stream.Result.Text),
Text = stream.Result.Text,
//Text = ExpandFunction.HandleFormula(stream.Result.Text),
Start = (float)Math.Round(startTime, 2, MidpointRounding.AwayFromZero),
End = (float)Math.Round(startTime + duration, 2, MidpointRounding.AwayFromZero),
});

26
VideoAnalysisCore/Common/AppCommon.cs
View File

@ -10,12 +10,14 @@ using System.Linq;
using System.Reflection;
using System.Runtime.Loader;
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using UserCenter.Model.Interface;
using VideoAnalysisCore.AICore.SherpaOnnx;
using VideoAnalysisCore.Enum;
using VideoAnalysisCore.Interface;
using VideoAnalysisCore.Model;
using VideoAnalysisCore.Model.Dto;
using Whisper.net;
using static System.Runtime.InteropServices.JavaScript.JSType;
@ -90,18 +92,8 @@ namespace VideoAnalysisCore.Common
/// </summary>
public static class ExpandFunction
{
static Dictionary<string, string> FormulaData = new Dictionary<string, string>()
{
{ "阿尔法","α"},
{ "贝塔","β"},
{ "伽马","γ"},
{ "德尔塔","Δ"},
{ "派","π"},
{ "西格马","∑"},
{ "欧米伽","Ω"},
{ "普西","Ψ"},
};
static string FormulaDataKey = string.Join("|", FormulaData.Keys);
static Dictionary<string, string> FormulaData;
static string FormulaDataKey;
/// <summary>
/// 处理数学公式
/// </summary>
@ -109,6 +101,15 @@ namespace VideoAnalysisCore.Common
/// <returns></returns>
public static string HandleFormula(string f)
{
if (FormulaData is null)
{
var hotwords = JsonSerializer.Deserialize<HotwordMode[]>(File.ReadAllText(Path.Combine(AppCommon.AIModelFile, "Hotwords.json")));
foreach (var item in hotwords.OrderByDescending(s=>s.key.Count()))
foreach (var key in item.v)
FormulaData.Add(key, item.key);
}
if (string.IsNullOrEmpty(FormulaDataKey))
FormulaDataKey = string.Join("|", FormulaData.Keys.Count);
if (string.IsNullOrEmpty(f))
return f;
return Regex.Replace(f, FormulaDataKey,
@ -307,6 +308,7 @@ namespace VideoAnalysisCore.Common
/// <para></para>
/// </summary>
public GptConfig ChatGpt { get; set; } = new GptConfig();
public GptConfig DeepSeek { get; set; } = new GptConfig();
public GptConfig KIMI { get; set; } = new GptConfig();
}

14
VideoAnalysisCore/Model/HotwordMode.cs Normal file
View File

@ -0,0 +1,14 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace VideoAnalysisCore.Model
{
public class HotwordMode
{
public string key { get; set; }
public string[] v { get; set; }
}
}

4
VideoAnalysisCore/Model/KnowledgeInfo.cs
View File

@ -6,10 +6,10 @@ using System.Text;
using System.Threading.Tasks;
using VideoAnalysisCore.Interface;
namespace VideoAnalysisCore.Model.Dto
namespace VideoAnalysisCore.Model
{
[SugarTable("knowledgeinfo")]
public class KnowledgeInfo: IKnowsDB
public class KnowledgeInfo : IKnowsDB
{
[SugarColumn(IsPrimaryKey = true, ColumnDescription = "Id 主键", ColumnName = "id")]
public long Id { get; set; }

12
VideoAnalysisCore/VideoAnalysisCore.csproj
View File

@ -8,7 +8,9 @@
<ItemGroup>
<None Remove="AICore\_Static\ffmpeg.exe" />
<None Remove="AICore\_Static\Hotwords.txt" />
<None Remove="AICore\_Static\Hotwords.json" />
<None Remove="AICore\_Static\itn_subject_sx.fst" />
<None Remove="AICore\_Static\sherpa-onnx-fst.py" />
<None Remove="AICore\_Static\sherpa-onnx-pyannote-segmentation-3-0\model.int8.onnx" />
<None Remove="AICore\_Static\sherpa-onnx-pyannote-segmentation-3-0\model.onnx" />
<None Remove="AICore\_Static\sherpa-onnx-streaming-zipformer-bilingual-zh-en-2023-02-20\bpe.model" />
@ -22,7 +24,13 @@
<Content Include="AICore\_Static\ffmpeg.exe">
<CopyToOutputDirectory>Never</CopyToOutputDirectory>
</Content>
<Content Include="AICore\_Static\Hotwords.txt">
<Content Include="AICore\_Static\Hotwords.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</Content>
<Content Include="AICore\_Static\itn_subject_sx.fst">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</Content>
<Content Include="AICore\_Static\sherpa-onnx-fst.py">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</Content>
<Content Include="AICore\_Static\sherpa-onnx-pyannote-segmentation-3-0\model.int8.onnx">