marking
/
Learn.VideoAnalysis
19
0
Fork 0
Code Pull Requests Actions Releases Activity

编写 kimi gpt提示词 返回固定json

This commit is contained in:
小肥羊 2024-10-31 17:59:07 +08:00
parent 6d67743869
commit 3dfe2fc087
9 changed files with 186 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
VideoAnalysis/Controllers/ApiController.cs
View File

@ -14,20 +14,11 @@ namespace Learn.VideoAnalysis.Controllers
public class ApiController : ControllerBase
{
private readonly ILogger<ApiController> _logger;
public ApiController(ILogger<ApiController> logger)
private readonly Repository<VideoTask> videoTaskDB;
public ApiController(ILogger<ApiController> logger, Repository<VideoTask> videoTaskDB)
{
_logger = logger;
}
/// <summary>
/// 音频转换wav_16k
/// </summary>
/// <param name="filePath">文件路径</param>
/// <returns></returns>
[HttpGet(Name = "Audio2WAV16K")]
public async Task<IActionResult> Audio2WAV16K(string filePath)
{
await FFMPGEHandle.Audio2WAV16KAsync(filePath);
return Ok();
this.videoTaskDB = videoTaskDB;
}
private string GetClientIpAddress()
@ -43,7 +34,7 @@ namespace Learn.VideoAnalysis.Controllers
/// <summary>
/// 测试 插入队列
/// ²åÈë¶ÓÁÐ
/// </summary>
/// <param name="enum"></param>
/// <param name="msg"></param>
@ -61,7 +52,7 @@ namespace Learn.VideoAnalysis.Controllers
/// <param name="req">ÇëÇóÌå</param>
/// <returns></returns>
[HttpPost(Name = "VideoAnalysis")]
public IActionResult VideoAnalysis(VideoAnalysisReq req)
public async Task<IActionResult> VideoAnalysis(VideoAnalysisReq req)
{
if (!ModelState.IsValid) return BadRequest(ModelState);
@ -74,14 +65,15 @@ namespace Learn.VideoAnalysis.Controllers
Tag = req.Tag,
CallBackUrl = req.CallBackUrl,
};
//Èë¿â
task.Id = await videoTaskDB.InsertReturnBigIdentityAsync(task);
var hashEntries = task.GetType()
.GetProperties(BindingFlags.Public | BindingFlags.Instance)
.ToDictionary(s => s.Name, s => s.GetValue(task));
RedisExpand.Redis.HMSet(RedisExpandKey.Task(task.Id), hashEntries);
RedisExpand.InsertChannel(RedisChannelEnum.DownloadFile
, task.Id);
return Ok();
RedisExpand.InsertChannel(RedisChannelEnum.DownloadFile,task.Id);
return Ok(task.Id);
}
}
}

25
VideoAnalysisCore/AICore/ChatGPT/Dto/QuestionRes.cs Normal file
View File

@ -0,0 +1,25 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace VideoAnalysisCore.AICore.ChatGPT.Dto
{
public class QuestionRes
{
/// <summary>
/// 对应问题的id
/// </summary>
public int 问题编号 { get; set; }
/// <summary>
/// 结果
/// </summary>
public object[]? 结果 { get; set; }
/// <summary>
/// 问题解释
/// </summary>
public string 问题解释 { get; set; }
}
}

48
VideoAnalysisCore/AICore/ChatGPT/KIMI/KIMI_GPT.cs
View File

@ -4,6 +4,11 @@ using System.Threading.Tasks;
using Whisper.net;
using VideoAnalysisCore.AICore.ChatGPT;
using System.Text.Json;
using VideoAnalysisCore.Model;
using System.Text;
using FFmpeg.NET.Services;
using Microsoft.Extensions.Primitives;
using VideoAnalysisCore.AICore.ChatGPT.Dto;
namespace VideoAnalysisCore.AICore.ChatGPT.KIMI
{
@ -13,17 +18,19 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI
public class KIMI_GPT : IBserGPT
{
private readonly MoonshotClient moonshotClient;
private readonly Repository<CourseGradingCriteria> criteriaDB;
/// <summary>
/// 初始化
/// </summary>
/// <param name="moonshotClient"></param>
/// <param name="logger"></param>
public KIMI_GPT(MoonshotClient moonshotClient)
public KIMI_GPT(MoonshotClient moonshotClient, Repository<CourseGradingCriteria> criteria)
{
MoonshotClient.Host = AppCommon.Config.ChatGpt.KIMI.Host;
MoonshotClient.ApiKey = AppCommon.Config.ChatGpt.KIMI.ApiKey;
this.moonshotClient = moonshotClient;
this.criteriaDB = criteria;
}
/// <summary>
/// 访问GPT
@ -33,26 +40,51 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI
public async Task CallGPT(string task)
{
var captions = ExpandFunction.GetSpeakerCaptions(task);
var criteriaArr = await criteriaDB.GetListAsync();
var criteriaBuilder = new StringBuilder();
foreach (var item in criteriaArr)
{
criteriaBuilder.Append(item.Id);
criteriaBuilder.Append(":");
criteriaBuilder.Append(item.NamePrompt);
criteriaBuilder.Append("返回bool值|");
}
criteriaBuilder.Append("100:分析授课中使用的高频词10个|");
criteriaBuilder.Append("101:分析授课中引发学生思考问题的时间段数组|");
criteriaBuilder.Append("102:分析授课中让学生产生讨论欲望的时间段|");
criteriaBuilder.Append("104:分析授课中巧妙地引入学习主题的时间段|");
criteriaBuilder.Append("105:分析授课中教学语言简明清晰准确生动的时间段|");
criteriaBuilder.Append("106:分析授课中教师激发学生思考的时间段|");
criteriaBuilder.Append("107:统计授课中教师提问类型的次数 类型重复回答,老师追问,简单性表演,老师补充答案,表扬并补充答案|");
var resFormat = "问题编号:int,结果:array,问题解释:string";
var postMessages =
$"以下是一段音频的字幕,分析这段字幕(格式 说话人:开始秒:结束秒:内容|下一段字幕)." +
$"来简明的回答提出的问题 问题列表 {criteriaBuilder} " +
$"字幕列表 {captions.Captions} " +
$"返回固定的JSON数组格式({resFormat}).";
var modelsResp = await moonshotClient.ListModels();
if (modelsResp is null || modelsResp.data.Count == 0)
throw new Exception("未获取KIMI模型类型");
var chatRep = new ChatReq
{
max_tokens = 1048 * 16,
max_tokens = postMessages.Length * 2,
temperature = 0.3,
frequency_penalty = 0,
presence_penalty = 0,
model = modelsResp.data.First().id,
model = modelsResp.data.First(s=>s.id.Contains("v1-32k")).id,
messages = new List<MessagesItem>(){
new MessagesItem("以下是和一个AI助手的对话。这位助手乐于助人，富有创造力，聪明，而且非常友好。","system"),
new MessagesItem("美国的人类预期寿命是多少？")
new MessagesItem(postMessages,"system"),
}
};
Console.WriteLine($"Chat 请求参数: {JsonSerializer.Serialize(chatRep)}");
var chatResp = await moonshotClient.Chat(chatRep);
var chatRespBody = await chatResp.Content.ReadAsStringAsync();
Console.WriteLine($"Chat 成功返回值: {chatRespBody}");
var chatResContent = chatResp?.choices.FirstOrDefault()?.message.content;
if(chatResp is null || string.IsNullOrEmpty(chatResContent))
throw new Exception($"KIMI模型返回异常 Chat 请求参数: {JsonSerializer.Serialize(chatRep)} " +
$" chatResp {chatResp} chatResContent {chatResContent}");
var questionRes = JsonSerializer.Deserialize<QuestionRes[]>(chatResContent);
//分析gpt返回结果
}
}
}

34
VideoAnalysisCore/AICore/ChatGPT/KIMI/MoonshotClient.cs
View File

@ -4,12 +4,15 @@ using System.Text;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json.Linq;
using System.Net.Http;
using Newtonsoft.Json;
using System.Net.Http.Json;
/// <summary>
/// https://platform.moonshot.cn/docs/api-reference
/// </summary>
namespace VideoAnalysisCore.AICore.ChatGPT.KIMI
{
/// <summary>
/// https://platform.moonshot.cn/docs/api-reference
/// </summary>
public class MoonshotClient
{
private readonly ILogger<MoonshotClient> _logger;
@ -37,9 +40,11 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI
/// </summary>
/// <param name="requestBody"></param>
/// <returns>Return HttpResponseMessage for SSE</returns>
public async Task<HttpResponseMessage> Chat(string requestBody)
public async Task<ChatRes?> Chat(string requestBody)
{
return await PostJsonStreamAsync("/v1/chat/completions", requestBody);
var chatResp = await PostJsonStreamAsync("/v1/chat/completions", requestBody);
return await chatResp.Content.ReadFromJsonAsync<ChatRes>();
}
/// <summary>
@ -47,10 +52,11 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI
/// </summary>
/// <param name="chatReq"></param>
/// <returns>Return HttpResponseMessage for SSE</returns>
public async Task<HttpResponseMessage> Chat(ChatReq chatReq)
public async Task<ChatRes?> Chat(ChatReq chatReq)
{
var requestBody = System.Text.Json.JsonSerializer.Serialize(chatReq);
return await PostJsonStreamAsync("/v1/chat/completions", requestBody);
var requestBody = Newtonsoft.Json.JsonConvert.SerializeObject(chatReq);
var chatResp = await PostJsonStreamAsync("/v1/chat/completions", requestBody);
return await chatResp.Content.ReadFromJsonAsync<ChatRes>();
}
/// <summary>
@ -65,7 +71,7 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI
var responseObj = JToken.Parse(responseText);
return responseObj?["data"]?["total_tokens"]?.ToObject<int>();
}
var error = System.Text.Json.JsonSerializer.Deserialize<ErrorResponse>(responseText);
var error = Newtonsoft.Json.JsonConvert.DeserializeObject<ErrorResponse>(responseText);
_logger.LogError($"{error?.error?.type}: {error?.error?.message}");
throw new Exception($"{error?.error.type}: {error?.error.message}");
}
@ -78,7 +84,7 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI
/// <returns></returns>
public async Task<int?> GetAsTiMateTokenCount(ChatReq chatReq)
{
var chatReqText =System.Text.Json.JsonSerializer.Serialize(chatReq);
var chatReqText = Newtonsoft.Json.JsonConvert.SerializeObject(chatReq);
return await GetAsTiMateTokenCount(chatReqText);
}
@ -177,7 +183,7 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI
var uriBuilder = new UriBuilder(Host + path);
request.RequestUri = uriBuilder.Uri;
request.Method = new HttpMethod("POST");
request.Headers.Host = new Uri(Host).Host;
request.Headers.Host = (new Uri(Host)).Host;
return request;
}
@ -191,9 +197,9 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI
var responseText = await response.Content.ReadAsStringAsync();
if (response.IsSuccessStatusCode)
{
return System.Text.Json.JsonSerializer.Deserialize<T>(responseText) ?? default;
return Newtonsoft.Json.JsonConvert.DeserializeObject<T>(responseText) ?? default;
}
var error = System.Text.Json.JsonSerializer.Deserialize<ErrorResponse>(responseText);
var error = Newtonsoft.Json.JsonConvert.DeserializeObject<ErrorResponse>(responseText);
_logger.LogError($"{error?.error.type}: {error?.error.message}");
throw new Exception($"{error?.error.type}: {error?.error.message}");
}
@ -241,4 +247,4 @@ namespace VideoAnalysisCore.AICore.ChatGPT.KIMI
}
}
}
}

50
VideoAnalysisCore/AICore/ChatGPT/KIMI/MoonshotModel.cs
View File

@ -133,6 +133,56 @@
public string is_blocking { get; set; }
}
public class ChatRes
{
public string id { get; set; }
public int created { get; set; }
/// <summary>
/// 模型id
/// </summary>
public string model { get; set; }
/// <summary>
/// 对话
/// </summary>
public Choice[] choices { get; set; }
/// <summary>
/// token使用情况
/// </summary>
public Usage usage { get; set; }
}
/// <summary>
/// token使用情况
/// </summary>
public class Usage
{
/// <summary>
/// 输入token数量
/// </summary>
public int prompt_tokens { get; set; }
/// <summary>
/// 返回token数量
/// </summary>
public int completion_tokens { get; set; }
/// <summary>
/// 总计token数量
/// </summary>
public int total_tokens { get; set; }
}
public class Choice
{
public int index { get; set; }
public Message message { get; set; }
public string finish_reason { get; set; }
}
public class Message
{
public string role { get; set; }
public string content { get; set; }
}
public class ModelInfo
{
/// <summary>

8
VideoAnalysisCore/AICore/SherpaOnnx/Speaker.cs
View File

@ -85,6 +85,14 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx
/// 讲话人索引
/// </summary>
public int SpeakerIndex { get; set; }
/// <summary>
/// 讲话人日志结果
/// </summary>
/// <param name="sds"></param>
public OfflineSpeakerRes()
{
}
/// <summary>
/// 讲话人日志结果
/// </summary>

12
VideoAnalysisCore/Common/AppCommon.cs
View File

@ -172,8 +172,16 @@ namespace VideoAnalysisCore.Common
//拼接 提示词字幕源
var stringBuilder = new StringBuilder();
foreach (var item in results)
stringBuilder.Append($"{item.Value.First()}:{item.Key.Start.TotalSeconds}:{item.Key.End.TotalSeconds}:{item.Key.Text},");
{
stringBuilder.Append(item.Value.First());
stringBuilder.Append(":");
stringBuilder.Append(item.Key.Start.TotalSeconds);
stringBuilder.Append(":");
stringBuilder.Append(item.Key.End.TotalSeconds);
stringBuilder.Append(":");
stringBuilder.Append(item.Key.Text);
stringBuilder.Append("|");
}
//todo 分析上课时间段情况 分析 独立学习 小组合作 随堂练习等情况
return new TotalCaptionsDto

6
VideoAnalysisCore/Common/RedisExpand.cs
View File

@ -95,6 +95,8 @@ namespace VideoAnalysisCore.Common
/// <param name="taskId">任务id</param>
public static void InsertChannel(RedisChannelEnum @enum, object taskId)
{
if (Redis is null) throw new Exception("redis未初始化");
var startTime = Redis.HMGet<Dictionary<RedisChannelEnum, DateTime>>(RedisExpandKey.Task(taskId), "StartTime").FirstOrDefault();
if (startTime is null)
startTime = new Dictionary<RedisChannelEnum, DateTime>();
@ -102,9 +104,9 @@ namespace VideoAnalysisCore.Common
startTime.Add(@enum, DateTime.Now);
else
startTime[@enum] = DateTime.Now;
Redis.HMSet(RedisExpandKey.Task(taskId), "StartTime", startTime);
if (Redis is null) throw new Exception("redis未初始化");
Redis.HMSet(RedisExpandKey.Task(taskId), "StartTime", startTime);
Redis.LPush(RedisExpandKey.EnumKey(@enum), taskId);
}

22
VideoAnalysisCore/Model/VideoTask.cs
View File

@ -1,6 +1,7 @@
using SqlSugar;
using System.ComponentModel.DataAnnotations;
using System.Net;
using System.Text.Json;
using VideoAnalysisCore.AICore.SherpaOnnx;
using VideoAnalysisCore.Enum;
using Whisper.net;
@ -75,8 +76,25 @@ namespace VideoAnalysisCore.Model
/// <summary>
/// 开始时间轴
/// </summary>
[SugarColumn(ColumnDataType = "varchar", Length =255)]
public Dictionary<RedisChannelEnum, DateTime> StartTime { get; set; }
[SugarColumn(ColumnDataType = "varchar", Length = 255)]
public string StartTime { get; set; } ="{}";
/// <summary>
/// 开始时间轴
/// <para>逻辑字段</para>
/// </summary>
[SugarColumn(IsIgnore = true)]
public Dictionary<RedisChannelEnum, DateTime> StartTimeDic
{
get
{
return JsonSerializer.Deserialize<Dictionary<RedisChannelEnum, DateTime>>(StartTime??"{}")
??new Dictionary<RedisChannelEnum, DateTime>();
}
set
{
StartTime = JsonSerializer.Serialize(value);
}
}
}