marking
/
Learn.VideoAnalysis
19
0
Fork 0
Code Pull Requests Actions Releases Activity

新增 查询任务id接口 

新增 ai任务使用gpu方法
This commit is contained in:
小肥羊 2024-11-07 14:46:30 +08:00
parent a2ef1d5d6d
commit 8bf9caedf1
11 changed files with 173 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
VideoAnalysis/Controllers/ApiController.cs
View File

@ -6,6 +6,8 @@ using System.Reflection;
using VideoAnalysisCore.Enum;
using VideoAnalysisCore.Model;
using VideoAnalysisCore.AICore.FFMPGE;
using VideoAnalysisCore.Model.Dto;
using VideoAnalysisCore.AICore.ChatGPT.Dto;
namespace Learn.VideoAnalysis.Controllers
{
@ -32,6 +34,20 @@ namespace Learn.VideoAnalysis.Controllers
throw new Exception("未能获取到客户端ip地址");
}
/// <summary>
/// ²åÈë¶ÓÁÐ
/// </summary>
/// <param name="taskId"></param>
/// <param name="msg"></param>
/// <returns></returns>
[HttpGet(Name = "TaskInfo")]
public async Task<IActionResult> TaskInfo(long taskId)
{
var task = await videoTaskDB.GetFirstAsync(s => s.Id == taskId);
if (task.LastEnum != RedisChannelEnum.EndTask)
return BadRequest(new { Enum = task.LastEnum ,Task = task.ChatAnalysis});
return Ok(new { Enum = task.LastEnum, Task = task.ChatAnalysis });
}
/// <summary>
/// 插入队列

3
VideoAnalysis/Program.cs
View File

@ -8,6 +8,9 @@ using VideoAnalysisCore.AICore.ChatGPT.KIMI;
using VideoAnalysisCore.AICore.SherpaOnnx;
using SqlSugar;
using Mapster;
using VideoAnalysisCore.AICore.ChatGPT.Dto;
using System.Text.Json;
using Newtonsoft.Json;

5
VideoAnalysisCore/AICore/ChatGPT/BserGPT.cs
View File

@ -1,4 +1,5 @@
using VideoAnalysisCore.AICore.SherpaOnnx;
using VideoAnalysisCore.AICore.ChatGPT.Dto;
using VideoAnalysisCore.AICore.SherpaOnnx;
using VideoAnalysisCore.Common;
using Whisper.net;
@ -14,6 +15,6 @@ namespace VideoAnalysisCore.AICore.ChatGPT
/// </summary>
/// <param name="task">任务id</param>
/// <returns></returns>
public Task CallGPT(string task);
public Task<CallGPTRes> CallGPT(string task);
}
}

31
VideoAnalysisCore/AICore/ChatGPT/Dto/CallGPTRes.cs
View File

@ -1,4 +1,5 @@
using System;
using FreeRedis;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
@ -8,12 +9,17 @@ using VideoAnalysisCore.Model.Dto;
namespace VideoAnalysisCore.AICore.ChatGPT.Dto
{
public class Assessment
{
}
public class CallGPTRes
{
public CallGPTRes()
{
}
public CallGPTRes(TotalCaptionsDto captions)
{
this.TeacherSpeaking = captions.TeacherSpeaking;
this.TimeBase = captions.TimeBase;
this.StudentSpeaking = captions.StudentSpeaking;
}
/// <summary>
/// 教师发言时间
/// <para>秒</para>
@ -25,6 +31,19 @@ namespace VideoAnalysisCore.AICore.ChatGPT.Dto
/// </summary>
public decimal StudentSpeaking { get; set; } = 0;
/// <summary>
/// 高频词汇 从高到低
/// </summary>
public string[] Hotwords { get; set; }
/// <summary>
/// 教师提问类型
/// </summary>
public KeyValue<int>[]? QuestionType { get; set; }
/// <summary>
/// 时间段概览
/// </summary>
public IEnumerable<TimeBase>? TimeOverview { get; set; }
/// <summary>
/// 视频时间轴
/// </summary>
public IEnumerable<TimeBase>? TimeBase { get; set; }
@ -35,6 +54,6 @@ namespace VideoAnalysisCore.AICore.ChatGPT.Dto
/// <summary>
/// AI综合评估
/// </summary>
public Assessment Assessment { get; set; }
public AssessmentDto Assessment { get; set; } = default!;
}
}

48
VideoAnalysisCore/AICore/ChatGPT/Dto/QuestionRes.cs
View File

@ -1,7 +1,10 @@
using System;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.Json.Nodes;
using System.Threading.Tasks;
namespace VideoAnalysisCore.AICore.ChatGPT.Dto
@ -11,15 +14,54 @@ namespace VideoAnalysisCore.AICore.ChatGPT.Dto
/// <summary>
/// 对应问题的id
/// </summary>
public int 问题编号 { get; set; }
public long 问题编号 { get; set; }
/// <summary>
/// 结果
/// </summary>
public object? 结果 { get; set; }
public JToken 结果 { get; set; }
/// <summary>
/// 问题解释
/// </summary>
public string? 问题解释 { get; set; }
}
/// <summary>
/// 课堂AI评价
/// </summary>
public class CourseCriteria
{
/// <summary>
/// AI解析
/// </summary>
public string Analyze { get; set; } = string.Empty;
/// <summary>
/// 问题编号
/// </summary>
public long Id { get; set; }
/// <summary>
/// 提词
/// </summary>
public string Prompt { get; set; } = string.Empty;
/// <summary>
/// 改进意见
/// <para>适用于:差的评价<see cref="AssessmentDto.Bad"/></para>
/// </summary>
public string? ImprovedMethods { get; set; }
}
/// <summary>
/// 课堂AI评价结果
/// </summary>
public class AssessmentDto
{
/// <summary>
/// 优秀的评价
/// </summary>
public CourseCriteria[]? Merit { get; set; }
/// <summary>
/// 差的评价
/// </summary>
public CourseCriteria[]? Bad { get; set; }
}
}

67
VideoAnalysisCore/AICore/ChatGPT/KIMI/KIMI_GPT.cs
View File

@ -12,6 +12,8 @@ using VideoAnalysisCore.AICore.ChatGPT.Dto;
using System.ComponentModel.DataAnnotations;
using VideoAnalysisCore.Enum;
using System.Reflection;
using FreeRedis;
using VideoAnalysisCore.Model.Dto;
namespace VideoAnalysisCore.AICore.ChatGPT.KIMI
{
@ -22,36 +24,37 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI
{
private readonly MoonshotClient moonshotClient;
private readonly Repository<CourseGradingCriteria> criteriaDB;
private readonly Repository<VideoTask> videoTaskDB;
/// <summary>
/// 初始化
/// </summary>
/// <param name="moonshotClient"></param>
/// <param name="logger"></param>
public KIMI_GPT(MoonshotClient moonshotClient, Repository<CourseGradingCriteria> criteria)
public KIMI_GPT(MoonshotClient moonshotClient, Repository<CourseGradingCriteria> criteria, Repository<VideoTask> videoTaskDB)
{
MoonshotClient.Host = AppCommon.Config.ChatGpt.KIMI.Host;
MoonshotClient.ApiKey = AppCommon.Config.ChatGpt.KIMI.ApiKey;
this.moonshotClient = moonshotClient;
this.criteriaDB = criteria;
this.videoTaskDB = videoTaskDB;
}
/// <summary>
/// 访问GPT
/// </summary>
/// <param name="task">任务id</param>
/// <returns></returns>
public async Task CallGPT(string task)
public async Task<CallGPTRes> CallGPT(string task)
{
var captions = ExpandFunction.GetSpeakerCaptions(task);
var criteriaArr = await criteriaDB.GetListAsync();
var criteriaBuilder = new StringBuilder();
foreach (var item in criteriaArr)
{
criteriaBuilder.Append(item.Id);
criteriaBuilder.Append(":");
criteriaBuilder.Append(item.NamePrompt);
criteriaBuilder.Append("返回bool值|");
criteriaBuilder.Append("结果类型 bool|");
}
//拼接枚举提问
foreach (var value in System.Enum.GetValues(typeof(QuestionTypeEnum)))
@ -68,7 +71,7 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI
}
var resFormat = "[{问题编号:int,结果:array|bool,问题解释:string}]";
var postMessages =
var postMessages =
$"以下是一段音频的字幕,分析这段字幕(格式 说话人:开始秒:结束秒:内容|下一段字幕)." +
$"来简明的回答提出的问题 问题列表 {criteriaBuilder} " +
$"字幕列表 {captions.Captions} " +
@ -76,7 +79,7 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI
var modelsResp = await moonshotClient.ListModels();
if (modelsResp is null || modelsResp.data.Count == 0)
throw new Exception("未获取KIMI模型类型");
var reqTokenCount =await moonshotClient.GetAsTiMateTokenCount(postMessages);
var reqTokenCount = await moonshotClient.GetAsTiMateTokenCount(postMessages);
var modelId = reqTokenCount > 32 * 1000 ? "moonshot-v1-128k" : "moonshot-v1-32k";
var chatRep = new ChatReq
{
@ -91,16 +94,62 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI
}
};
var chatResp = await moonshotClient.Chat(chatRep);
if(chatResp is null || chatResp.error != null)
if (chatResp is null || chatResp.error != null)
throw new Exception($"KIMI模型返回异常 Chat 请求参数: {JsonSerializer.Serialize(chatRep)} " +
$" chatResp {JsonSerializer.Serialize(chatResp)}");
var chatResContent = chatResp?.choices.FirstOrDefault()?.message.content;
var questionRes = JsonSerializer.Deserialize<QuestionRes[]>(chatResContent);
//todo 分析gpt返回结果
var gptRes = new CallGPTRes(captions);
if (questionRes is null)
throw new Exception("KIMIGPT返回无效结果");
//处理 ai问答提问
var arr1 = questionRes.Where(s => s.问题编号 < 100);
var arr2 = questionRes.Where(s => s.问题编号 >= 100).ToDictionary(s => s.问题编号);
//AI综合评估
var criteriaDic = criteriaArr.ToDictionary(s => s.Id);
gptRes.Assessment = new AssessmentDto()
{
Bad = arr1.Where(s => !s.结果.ToObject<bool>())
.Select(s => new CourseCriteria()
{
Id = criteriaDic[s.问题编号].Id,
ImprovedMethods = criteriaDic[s.问题编号].ImprovedMethods,
Analyze = s.问题解释??string.Empty,
Prompt = criteriaDic[s.问题编号].Flaw,
}).ToArray(),
Merit = arr1.Where(s => s.结果.ToObject<bool>())
.Select(s => new CourseCriteria()
{
Id = criteriaDic[s.问题编号].Id,
ImprovedMethods = criteriaDic[s.问题编号].ImprovedMethods,
Prompt = criteriaDic[s.问题编号].ImprovedMethods,
}).ToArray(),
};
//高频词汇
gptRes.Hotwords = arr2[(int)QuestionTypeEnum.高频词].结果.ToObject<string[]>()??["暂无数据"];
//提问类型
gptRes.QuestionType = arr2[(int)QuestionTypeEnum.提问类型].结果.ToObject<KeyValue<int>[]>();
//时间段概览
gptRes.TimeOverview = arr2[(int)QuestionTypeEnum.时间段概览]
.结果.ToObject<KeyValue<string>[]>()?.Select(s =>
new TimeBase
{
Start = double.Parse(s.key.Split(":")[1]??0.ToString()),
End = double.Parse(s.key.Split(":")[2]??0.ToString()),
Content = s.value,
});
//todo 分析上课时间段情况 分析 独立学习 小组合作 随堂练习等情况
var tId = long.Parse(task);
var taskData = await videoTaskDB.GetFirstAsync(s => s.Id == tId);
taskData.ChatAnalysis = gptRes;
taskData.LastEnum = RedisChannelEnum.EndTask;
await videoTaskDB.AsUpdateable(taskData)
.UpdateColumns(it => new { it.ChatAnalysis }).ExecuteCommandAsync();
return gptRes;
}
}
}

13
VideoAnalysisCore/AICore/SherpaOnnx/SenseVoice.cs
View File

@ -18,9 +18,9 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx
/// <summary>
/// 初始化 SenseVoice
/// </summary>
/// <param name="speakerNumber"></param>
/// <param name="threshold"></param>
public static void Init(int speakerNumber = 0, double threshold = 0.6)
/// <param name="numThreads"></param>
/// <param name="useGPU">是否使用gpu 报错请看安装CUDA环境<see cref="https://k2-fsa.github.io/sherpa/onnx/pretrained_models/whisper/large-v3.html#run-with-gpu-float32"/></param>
public static void Init(int numThreads =4,bool useGPU=false)
{
Console.WriteLine("初始化 SenseVoice");
OfflineRecognizerConfig config = new OfflineRecognizerConfig();
@ -34,8 +34,13 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx
config.ModelConfig.SenseVoice.Model = Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-sense-voice-24-07-17", "model.onnx");
//1 使用逆文本规范化处理感官语音。
config.ModelConfig.SenseVoice.UseInverseTextNormalization =1;
config.ModelConfig.SenseVoice.Language = "zh";
//模型类型
config.ModelConfig.ModelType = string.Empty;
config.ModelConfig.NumThreads = numThreads;
//需要使用GPU
if (!useGPU)
config.ModelConfig.Provider = "cuda";
#region 有效的解码方法
//贪婪搜索[greedy_search] 改进的波束搜索 [modified_beam_search]
@ -56,7 +61,7 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx
//反转文本规范化规则 fst 的路径
config.RuleFsts = string.Empty;
config.ModelConfig.Debug = 0;
config.ModelConfig.Debug = 1;
OR = new OfflineRecognizer(config);

6
VideoAnalysisCore/AICore/SherpaOnnx/Speaker.cs
View File

@ -20,7 +20,8 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx
/// </summary>
/// <param name="speakerNumber"></param>
/// <param name="threshold"></param>
public static void Init(int speakerNumber = 0, double threshold = 0.6)
/// <param name="useGPU"></param>
public static void Init(int speakerNumber = 2, double threshold = 0.6,bool useGPU = false)
{
Console.WriteLine("初始化 Speaker");
var config = new OfflineSpeakerDiarizationConfig();
@ -30,6 +31,9 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx
config.Embedding.Model = Path.Combine(AppCommon.AIModelFile, "wespeaker", "wespeaker_zh_cnceleb_resnet34_LM.onnx");
//说话人数量
config.Clustering.NumClusters = speakerNumber;
//需要使用GPU
if (!useGPU)
config.Embedding.Provider = "cuda";
//说话人判定阈值
config.Clustering.Threshold = (float)threshold;
SD = new OfflineSpeakerDiarization(config);

6
VideoAnalysisCore/Enum/QuestionTypeEnum.cs
View File

@ -10,11 +10,11 @@ namespace VideoAnalysisCore.Enum
enum QuestionTypeEnum
{
[Display(Prompt = "分析授课中使用的高频词10个")]
[Display(Prompt = "分析授课中使用的高频词10个频率从高到低 结果类型[]")]
高频词 =100,
[Display(Prompt = "分析字幕中每5分钟的一个概览 返回结构 array[0]=概览 时间段")]
[Display(Prompt = "分析字幕中每5分钟的一个概览 结果类型[{key:时间段,value:概览}]")]
时间段概览 = 101,
[Display(Prompt = "统计授课中教师提问类型的次数 类型重复回答,老师追问,简单性表演,老师补充答案,表扬并补充答案")]
[Display(Prompt = "统计授课中教师提问类型的次数 类型[重复回答,老师追问,简单性表演,老师补充答案,表扬并补充答案] 结果类型[{key:类型,value:次数}]")]
提问类型 = 102,
}
}

5
VideoAnalysisCore/Model/Dto/SpeakerCaptionsDto.cs
View File

@ -20,13 +20,14 @@ namespace VideoAnalysisCore.Model.Dto
/// </summary>
public double End { get; set; }
/// <summary>
/// 字幕内容
/// 内容
/// </summary>
public string Content { get; set; }
/// <summary>
/// 时间段 类型
/// <para><see cref="VideoAnalysisCore.AICore.ChatGPT.Dto.CallGPTRes.TimeOverview"/> 时为 null</para>
/// </summary>
public TimeBaseTypeEnum TimeBaseType { get; set; }
public TimeBaseTypeEnum? TimeBaseType { get; set; }
}
public class TotalCaptionsDto

5
VideoAnalysisCore/Model/VideoTask.cs
View File

@ -2,6 +2,7 @@
using System.ComponentModel.DataAnnotations;
using System.Net;
using System.Text.Json;
using VideoAnalysisCore.AICore.ChatGPT.Dto;
using VideoAnalysisCore.AICore.SherpaOnnx;
using VideoAnalysisCore.AICore.Whisper;
using VideoAnalysisCore.Enum;
@ -88,9 +89,9 @@ namespace VideoAnalysisCore.Model
/// Chat模型分析缓存
/// </summary>
[SugarColumn(IsIgnore = true)]
public object[]? ChatAnalysis
public CallGPTRes? ChatAnalysis
{
get => JsonSerializer.Deserialize<object[]>(_ChatAnalysis ?? "[]");
get => JsonSerializer.Deserialize<CallGPTRes>(_ChatAnalysis ?? "[]");
set => _ChatAnalysis = JsonSerializer.Serialize(value);
}