using FFmpeg.NET.Services; using MapsterMapper; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using SqlSugar; using System; using System.Diagnostics; using System.Security.Claims; using System.Text.Json; using System.Threading.Tasks; using UserCenter.Model; using UserCenter.Model.Enum; using VideoAnalysisCore.AICore.GPT.Dto; using VideoAnalysisCore.AICore.SherpaOnnx; using VideoAnalysisCore.Common; using VideoAnalysisCore.Common.Expand; using VideoAnalysisCore.Controllers.Dto; using VideoAnalysisCore.Model; using VideoAnalysisCore.Model.Dto; using VideoAnalysisCore.Model.Enum; using Yitter.IdGenerator; namespace VideoAnalysisCore.Controllers { /// /// 路由菜单 /// public class VideoTaskController : BackController { readonly Repository baseService; readonly Repository videoQuestionDB; readonly Repository videoKonwPointDB; readonly Repository videoQuestionKonwDB; readonly Repository taskLogDB; readonly RedisManager redisManager; public readonly SenseVoice senseVoice; private readonly IMapper mp; public VideoTaskController(Repository baseService, RedisManager redisManager, Repository videoQuestionDB, Repository videoQuestionKonwDB, Repository videoKonwPointDB, SenseVoice senseVoice, IMapper mp, Repository taskLogDB) : base(baseService) { this.baseService = baseService; this.redisManager = redisManager; this.videoQuestionDB = videoQuestionDB; this.videoQuestionKonwDB = videoQuestionKonwDB; this.videoKonwPointDB = videoKonwPointDB; this.senseVoice = senseVoice; this.mp = mp; this.taskLogDB = taskLogDB; } private string GetClientIpAddress() { // 检查 X-Forwarded-For 请求头 if (HttpContext.Request.Headers.ContainsKey("X-Forwarded-For") && !string.IsNullOrEmpty(HttpContext.Request.Headers["X-Forwarded-For"])) return HttpContext.Request.Headers["X-Forwarded-For"].ToString(); if (HttpContext.Connection.RemoteIpAddress != null) return HttpContext.Connection.RemoteIpAddress.ToString(); throw new Exception("未能获取到客户端ip地址"); } #if DEBUG /// /// 初始化主库表 /// /// 文件流 /// [HttpGet(Name = "InitDbTable")] public IActionResult InitDbTable() { var b = AppCommon.Config.DB.UpdateTable; AppCommon.Config.DB.UpdateTable = true; SqlSugarExpand.InitDbTable(); AppCommon.Config.DB.UpdateTable = b; return Ok(); } #endif /// /// 插入批量任务id /// /// 是否执行任务 /// [HttpPost(Name = "JoinQueue")] public IActionResult JoinQueue(long[] ids) { if (ids == null || ids.Count() == 0) return BadRequest("录入数据无效"); redisManager.JoinQueue(ids); return Ok(); } /// /// 当前消费者 继续执行任务 /// /// 是否执行任务 /// [HttpGet(Name = "StartTask")] public IActionResult StartTask(bool task) { if (task) redisManager.RestartTask(); else redisManager.StopTaskAsync(); return Ok(); } /// /// 语音识别 /// /// 文件流 /// [HttpGet(Name = "AudioRecognitionUrl")] public async Task AudioRecognitionUrl(string url) { try { using HttpClient client = new HttpClient(); // 发送GET请求获取网络文件流 using var networkStream = await client.GetStreamAsync(url); var res = senseVoice.RunTask(networkStream); return Ok(res); } catch (Exception ex) { return BadRequest(ex.Message); } } /// /// 语音识别 /// /// 文件流 /// [HttpPost(Name = "AudioRecognition")] public IActionResult AudioRecognition(IFormFile file) { using var s = file.OpenReadStream(); var res = senseVoice.RunTask(s); return Ok(res); } /// /// 获取FTS_Data str /// /// 路径 /// [HttpGet(Name = "fts_data")] public async Task FTS_Data(string path = "itn_subject_sx.fst") { var hotwords = JsonSerializer .Deserialize(System.IO.File.ReadAllText(Path.Combine(AppCommon.AIModelFile, "Hotwords.json"))); var res = new List(100); foreach (var element in hotwords.OrderByDescending(s => s.key.Count())) foreach (var e in element.v) res.Add($"""("{e}", "{element.key}")"""); var pyFile = System.IO.File.ReadAllText(Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-fst.py")); var resStr = pyFile .Replace("(fts_data)", "[" + string.Join(',', res) + "]") .Replace("(path)", path); return Ok(resStr); } /// /// 重新开始执行GPT分析taskId/tagId二选一 /// /// /// 自定义id /// 切换任务所属学科 null忽略 /// [HttpGet] public async Task ReStartTask(long taskId, string? tagId, SubjectEnum? subject) { var task = await baseService.AsQueryable() .WhereIF(taskId != 0, s => s.Id == taskId) .WhereIF(!string.IsNullOrEmpty(tagId), s => s.TagId == tagId) .FirstAsync(); if (task is null) return BadRequest("未能找到对应任务"); if (subject is not null) { task.Subject = subject; await baseService.UpdateAsync(task); } //todo重新开始执行GPT分析 return BadRequest("任务未实现"); return Ok(); } /// /// 视频处理[批量] /// /// 请求体 /// [HttpPost(Name = "VideoAnalysis_Batch")] public async Task VideoAnalysis_Batch(VideoAnalysisReq[] req) { foreach (var item in req) await VideoAnalysis(item); return Ok(); } /// /// 视频处理 /// /// 请求体 /// [HttpPost(Name = "VideoAnalysis")] public async Task VideoAnalysis(VideoAnalysisReq req) { if (!ModelState.IsValid) return BadRequest(ModelState); if (await baseService.IsAnyAsync(s => s.TagId == req.TagId)) return BadRequest("重复添加"); // 自动映射属性到哈希 var task = new VideoTask() { Id = YitIdHelper.NextId(), ComeFrom = GetClientIpAddress(), MediaUrl = req.MediaUrl, ApiToken = req.ApiToken, CourseId = req.CourseId, Subject = req.Subject, Tag = req.Tag, TagId = req.TagId, PPTVideoCode = req.PPTVideoCode, PPTVideoUrl = req.PPTVideoUrl, VideoType = req.VideoType }; //入库 await baseService.InsertAsync(task); redisManager.Redis.LPush(RedisExpandKey.ChannelKey, task.Id); return Ok(task.Id); } public override async Task PageList([FromBody] QueryRequestBase model) { var sqlquery = base.BaseQuery(model) .Select(s => new VideoTask { Id = s.Id, TagId = s.TagId, VideoType = s.VideoType, LastEnum = s.LastEnum, Subject = s.Subject, ComeFrom = s.ComeFrom, MediaUrl = s.MediaUrl, CreateTime = s.CreateTime, ErrorMessage = s.ErrorMessage, }); RefAsync total = 0; var data = await sqlquery.ToPageListAsync(model.PageIndex + 1, model.PageSize, total); return new PageResult() { Data = data, Total = total }; } public override Task Edit([FromBody] VideoTask model) => throw new NotImplementedException(); public override Task Del([FromBody] params long[] ids) => throw new NotImplementedException(); /// /// 重试任务 /// /// 任务id /// 任务类型 /// [HttpGet] public async Task ReStart(long id, RedisChannelEnum selectEnum) { await redisManager.AddTaskLog(id,"手动重试任务"); await redisManager.ClearTaskError(id); _ = Task.Run(async () => await redisManager.InsertChannel(selectEnum, id) ); } /// /// 刷新数据 /// /// 任务id /// [HttpGet] public async Task RowRload(long id) { if (id == 0) return BadRequest("无效id"); var d = await redisManager.Redis.HMGetAsync(RedisExpandKey.Task(id), "Progress", "LastEnum", "StartTime", "ErrorMessage"); var logArr = await taskLogDB.AsQueryable() .Where(s => s.VideoTaskId == id) .ToArrayAsync(); var insertData = (await redisManager.Redis .LRangeAsync(RedisExpandKey.TaskLog, 0, 99)) .Where(s => s.VideoTaskId == id); logArr = logArr.Concat(insertData).ToArray(); return Ok(new { Progress = d[0], LastEnum = d[1]?.ToEnum().ToString(), StartTime = d[2] != null ? JsonSerializer.Deserialize>(d[2]) : null, ErrorMessage = d[3], Logs = logArr, }); } /// /// 预览任务结果 /// /// 任务id /// [HttpGet] public async Task ShowTaskInfo(long id) { var nowTask = await baseService.GetFirstAsync(s => s.Id == id); if (nowTask is null) return BadRequest("无效任务"); var captionsArr = JsonSerializer.Deserialize(nowTask.Captions); var captionsArr1 = JsonSerializer.Deserialize(nowTask.CaptionsAI ?? "[]"); var konwArr = await videoKonwPointDB.AsQueryable() .Where(s => s.VideoTaskId == nowTask.Id) .ToArrayAsync(); var videoKnows = konwArr .GroupBy(s => s.StartTime) .Select(s => new VideoKnowRes() { Content = s.First().Content, StartTime = s.First().StartTime, EndTime = s.First().EndTime, Theme = s.First().Theme, StageId = s.First().StageId, KnowPoint = string.Join(',', s.Select(x => x.KnowPoint)) }).ToArray(); if (nowTask.VideoType == AttachmentsInfoType.复习) { var questionArr = await videoQuestionDB .AsQueryable().Where(s => s.VideoTaskId == nowTask.Id) .Select() .ToArrayAsync(); var konwDic = (await videoQuestionKonwDB .AsQueryable().Where(s => s.VideoTaskId == nowTask.Id) .ToArrayAsync()).GroupBy(s => s.VideoQuestionId) .ToDictionary(s => s.Key); foreach (var item in questionArr.Where(s => konwDic.ContainsKey(s.Id))) item.KonwArr = konwDic[item.Id].ToArray(); foreach (var item in videoKnows) item.QuestionArr = questionArr .Where(s => s.StageId == item.StageId).ToArray(); } return Ok(new { Captions = captionsArr, Captions1 = captionsArr1, VideoKnows = videoKnows, MediaUrl = nowTask.MediaUrl }); } /// /// 执行中的任务 /// /// 查询模型 /// [HttpPost] public async Task RunningTaskList([FromBody] QueryRequestBase model) { var oldTaskArr = redisManager.Redis.LRange(RedisExpandKey.IDTask, 0, 999); var sqlquery = base.BaseQuery(model) .Where(s => oldTaskArr.Contains(s.Id)) .Select(s => new VideoTask { Id = s.Id, TagId = s.TagId, VideoType = s.VideoType, LastEnum = s.LastEnum, Subject = s.Subject, ComeFrom = s.ComeFrom, MediaUrl = s.MediaUrl, CreateTime = s.CreateTime, }); RefAsync total = 0; var data = await sqlquery.ToPageListAsync(model.PageIndex + 1, model.PageSize, total); return new PageResult() { Data = data, Total = total }; } /// /// 错误的任务 /// /// 查询模型 /// [HttpPost] public async Task ErrorTaskList([FromBody] QueryRequestBase model) { var sqlquery = base.BaseQuery(model) .Where(s => s.ErrorMessage!=null && s.ErrorMessage !="") .Select(s => new VideoTask { Id = s.Id, TagId = s.TagId, VideoType = s.VideoType, LastEnum = s.LastEnum, Subject = s.Subject, ComeFrom = s.ComeFrom, MediaUrl = s.MediaUrl, ErrorMessage = s.ErrorMessage, CreateTime = s.CreateTime, }); RefAsync total = 0; var data = await sqlquery.ToPageListAsync(model.PageIndex + 1, model.PageSize, total); return new PageResult() { Data = data, Total = total }; } /// /// 任务日志 /// /// 查询模型 /// [HttpGet] public async Task> TaskLog(long id ) { var logArr = await taskLogDB.AsQueryable() .Where(s => s.VideoTaskId == id) .ToArrayAsync(); var insertData = (await redisManager.Redis .LRangeAsync(RedisExpandKey.TaskLog, 0, 99)) .Where(s=>s.VideoTaskId == id); return logArr.Concat(insertData); } } }