Learn.VideoAnalysis/VideoAnalysisCore/Controllers/VideoTaskController.cs

627 lines
23 KiB
C#

using FFmpeg.NET.Services;
using MapsterMapper;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.DependencyInjection;
using SqlSugar;
using System;
using System.Diagnostics;
using System.Security.Claims;
using System.Text.Json;
using System.Threading.Tasks;
using UserCenter.Model;
using UserCenter.Model.Enum;
using VideoAnalysisCore.AICore.GPT.Dto;
using VideoAnalysisCore.AICore.SherpaOnnx;
using VideoAnalysisCore.Common;
using VideoAnalysisCore.Common.Expand;
using VideoAnalysisCore.Controllers.Dto;
using VideoAnalysisCore.Model;
using VideoAnalysisCore.Model.Dto;
using VideoAnalysisCore.Model.Enum;
using Yitter.IdGenerator;
namespace VideoAnalysisCore.Controllers
{
public class RunningTaskListReq : QueryRequestBase
{
/// <summary>
/// 设备ID (可选,若为空则默认获取当前节点)
/// </summary>
public string? DeviceId { get; set; }
}
/// <summary>
/// 路由菜单
/// </summary>
public class VideoTaskController : BackController<VideoTask>
{
readonly Repository<VideoTask> baseService;
readonly Repository<VideoQuestion> videoQuestionDB;
readonly Repository<VideoKonwPoint> videoKonwPointDB;
readonly Repository<VideoTaskStage> videoTaskStageDB;
readonly Repository<VideoQuestionKonw> videoQuestionKonwDB;
readonly Repository<TaskLog> taskLogDB;
readonly RedisManager redisManager;
readonly TidySlideWorkflowManager tidySlideWorkflowManager;
readonly VideoSliceWorkflowManager videoSliceWorkflowManager;
public readonly SenseVoice senseVoice;
public readonly FunASRNano funASRNano;
private readonly IMapper mp;
public VideoTaskController(Repository<VideoTask> baseService, RedisManager redisManager,
Repository<VideoQuestion> videoQuestionDB,
Repository<VideoQuestionKonw> videoQuestionKonwDB, Repository<VideoKonwPoint> videoKonwPointDB, SenseVoice senseVoice, IMapper mp, Repository<TaskLog> taskLogDB, FunASRNano funASRNano, Repository<VideoTaskStage> videoTaskStageDB, TidySlideWorkflowManager tidySlideWorkflowManager, VideoSliceWorkflowManager videoSliceWorkflowManager) : base(baseService)
{
this.baseService = baseService;
this.redisManager = redisManager;
this.videoQuestionDB = videoQuestionDB;
this.videoQuestionKonwDB = videoQuestionKonwDB;
this.videoKonwPointDB = videoKonwPointDB;
this.senseVoice = senseVoice;
this.mp = mp;
this.taskLogDB = taskLogDB;
this.funASRNano = funASRNano;
this.videoTaskStageDB = videoTaskStageDB;
this.tidySlideWorkflowManager = tidySlideWorkflowManager;
this.videoSliceWorkflowManager = videoSliceWorkflowManager;
}
private string GetClientIpAddress()
{
// 检查 X-Forwarded-For 请求头
if (HttpContext.Request.Headers.ContainsKey("X-Forwarded-For")
&& !string.IsNullOrEmpty(HttpContext.Request.Headers["X-Forwarded-For"]))
return HttpContext.Request.Headers["X-Forwarded-For"].ToString();
if (HttpContext.Connection.RemoteIpAddress != null)
return HttpContext.Connection.RemoteIpAddress.ToString();
throw new Exception("未能获取到客户端ip地址");
}
#if DEBUG
/// <summary>
/// 初始化主库表
/// </summary>
/// <param name="url">文件流</param>
/// <returns></returns>
[HttpGet(Name = "InitDbTable")]
[AllowAnonymous]
public IActionResult InitDbTable()
{
var b = AppCommon.Config.DB.UpdateTable;
AppCommon.Config.DB.UpdateTable = true;
SqlSugarExpand.InitDbTable();
AppCommon.Config.DB.UpdateTable = b;
return Ok();
}
#endif
/// <summary>
/// 添加任务到 TidySlide队列
/// </summary>
/// <param name="ids">是否执行任务</param>
/// <returns></returns>
[HttpPost(Name = "JoinQueueVideoSlice")]
public IActionResult JoinQueue( long[] ids)
{
if (ids == null || ids.Length == 0)
return BadRequest("录入数据无效");
videoSliceWorkflowManager.JoinQueue(ids);
return Ok();
}
/// <summary>
/// 添加任务到 TidySlide队列
/// </summary>
/// <param name="ids"></param>
/// <returns></returns>
[HttpPost(Name = "JoinQueueTidySlide")]
public IActionResult JoinTidySlideQueue( long[] ids)
{
if (ids == null || ids.Length == 0)
return BadRequest("录入数据无效");
tidySlideWorkflowManager.JoinQueue(ids);
return Ok();
}
/// <summary>
/// 当前消费者 继续执行任务
/// </summary>
/// <param name="task">是否执行任务</param>
/// <returns></returns>
[HttpGet(Name = "StartTask")]
public IActionResult StartTask(bool task)
{
if (task)
videoSliceWorkflowManager.StopTask = false;
else
videoSliceWorkflowManager.StopTask = true;
return Ok();
}
/// <summary>
/// 语音识别
/// </summary>
/// <param name="url">文件流</param>
/// <returns></returns>
[HttpGet(Name = "AudioRecognitionUrl")]
public async Task<IActionResult> AudioRecognitionUrl(string url)
{
try
{
using HttpClient client = new HttpClient();
// 发送GET请求获取网络文件流
using var networkStream = await client.GetStreamAsync(url);
var res = senseVoice.RunTask(networkStream);
return Ok(res);
}
catch (Exception ex)
{
return BadRequest(ex.Message);
}
}
/// <summary>
/// 语音识别
/// </summary>
/// <param name="file">文件流</param>
/// <returns></returns>
[HttpPost(Name = "AudioRecognition")]
public IActionResult AudioRecognition(IFormFile file)
{
using var s = file.OpenReadStream();
var res = senseVoice.RunTask(s);
return Ok(res);
}
/// <summary>
/// 语音识别
/// </summary>
/// <param name="file">文件流</param>
/// <returns></returns>
[HttpPost(Name = "AudioRecognition_test")]
public IActionResult AudioRecognition_test(IFormFile file)
{
using var s = file.OpenReadStream();
var x = AppCommon.Services.GetService<FunASRNano>();
x.Init();
senseVoice.RunTask(s);
for (int i = 0; i < SenseVoice.cachedValue.Count(); i++)
{
Console.WriteLine($"字幕索引=>{i}");
Console.WriteLine($"ssv=>{SenseVoice.cachedValue[i].z1}");
Console.WriteLine($"fun=>{SenseVoice.cachedValue[i].z2}");
Console.WriteLine();
}
return Ok();
}
/// <summary>
/// 获取FTS_Data str
/// </summary>
/// <param name="path">路径</param>
/// <returns></returns>
[HttpGet(Name = "fts_data")]
public async Task<IActionResult> FTS_Data(string path = "itn_subject_sx.fst")
{
var hotwords = JsonSerializer
.Deserialize<HotwordMode[]>(System.IO.File.ReadAllText(Path.Combine(AppCommon.AIModelFile, "Hotwords.json")));
var res = new List<string>(100);
foreach (var element in hotwords.OrderByDescending(s => s.key.Count()))
foreach (var e in element.v)
res.Add($"""("{e}", "{element.key}")""");
var pyFile = System.IO.File.ReadAllText(Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-fst.py"));
var resStr = pyFile
.Replace("(fts_data)", "[" + string.Join(',', res) + "]")
.Replace("(path)", path);
return Ok(resStr);
}
/// <summary>
/// 重新开始执行GPT分析<para>taskId/tagId二选一</para>
/// </summary>
/// <param name="taskId"></param>
/// <param name="tagId">自定义id</param>
/// <param name="subject">切换任务所属学科 null忽略</param>
/// <returns></returns>
[HttpGet]
public async Task<IActionResult> ReStartTask(long taskId, string? tagId, SubjectEnum? subject)
{
var task = await baseService.AsQueryable()
.WhereIF(taskId != 0, s => s.Id == taskId)
.WhereIF(!string.IsNullOrEmpty(tagId), s => s.TagId == tagId)
.FirstAsync();
if (task is null)
return BadRequest("未能找到对应任务");
if (subject is not null)
{
task.Subject = subject;
await baseService.UpdateAsync(task);
}
//todo重新开始执行GPT分析
return BadRequest("任务未实现");
return Ok();
}
/// <summary>
/// 视频处理[批量]
/// </summary>
/// <param name="req">请求体</param>
/// <returns></returns>
[HttpPost(Name = "VideoAnalysis_Batch")]
public async Task<IActionResult> VideoAnalysis_Batch(VideoAnalysisReq[] req)
{
foreach (var item in req)
await VideoAnalysis(item);
return Ok();
}
/// <summary>
/// 视频处理
/// </summary>
/// <param name="req">请求体</param>
/// <returns></returns>
[HttpPost(Name = "VideoAnalysis")]
public async Task<IActionResult> VideoAnalysis(VideoAnalysisReq req)
{
if (!ModelState.IsValid) return BadRequest(ModelState);
if (await baseService.IsAnyAsync(s => s.TagId == req.TagId))
return BadRequest("重复添加");
// 自动映射属性到哈希
var task = new VideoTask()
{
Id = YitIdHelper.NextId(),
ComeFrom = GetClientIpAddress(),
MediaUrl = req.MediaUrl,
ApiToken = req.ApiToken,
CourseId = req.CourseId,
Subject = req.Subject,
Tag = req.Tag,
TagId = req.TagId,
PPTVideoCode = req.PPTVideoCode,
PPTVideoUrl = req.PPTVideoUrl,
VideoType = req.VideoType,
};
//入库
await baseService.InsertAsync(task);
redisManager.Redis.LPush(RedisExpandKey.ChannelKey, task.Id);
return Ok(task.Id);
}
public override async Task<dynamic> PageList([FromBody] QueryRequestBase model)
{
var sqlquery = base.BaseQuery(model)
.Select(s => new VideoTask
{
Id = s.Id,
TagId = s.TagId,
VideoType = s.VideoType,
LastEnum = s.LastEnum,
Subject = s.Subject,
ComeFrom = s.ComeFrom,
MediaUrl = s.MediaUrl,
CreateTime = s.CreateTime,
ErrorMessage = s.ErrorMessage,
});
RefAsync<int> total = 0;
var data = await sqlquery.ToPageListAsync(model.PageIndex + 1, model.PageSize, total);
return new PageResult<VideoTask>() { Data = data, Total = total };
}
public override Task<bool> Edit([FromBody] VideoTask model) => throw new NotImplementedException();
public override Task<bool> Del([FromBody] params long[] ids) => throw new NotImplementedException();
/// <summary>
/// 重试任务
/// </summary>
/// <param name="id">任务id</param>
/// <param name="selectEnum">任务类型</param>
/// <returns></returns>
[HttpGet]
public async Task ReStart(long id, RedisChannelEnum selectEnum)
{
await videoSliceWorkflowManager.AddTaskLog(id, "手动重试任务");
await videoSliceWorkflowManager.ClearTaskError(id);
_ = Task.Run(async () =>
await videoSliceWorkflowManager.InsertChannel(selectEnum, id.ToString())
);
}
/// <summary>
/// 重试任务 (TidySlide 工作流)
/// </summary>
/// <param name="id">任务id</param>
/// <param name="selectEnum">任务类型</param>
/// <returns></returns>
[HttpGet]
public async Task ReStartTidySlide(long id, RedisTidySlideChannelEnum selectEnum)
{
await tidySlideWorkflowManager.AddTaskLog(id, "手动重试 TidySlide 任务");
await tidySlideWorkflowManager.ClearTaskError(id);
_ = Task.Run(async () =>
await tidySlideWorkflowManager.InsertChannel(selectEnum, id.ToString())
);
}
/// <summary>
/// 刷新数据
/// </summary>
/// <param name="id">任务id</param>
/// <returns></returns>
[HttpGet]
public async Task<IActionResult> RowRload(long id)
{
if (id == 0)
return BadRequest("无效id");
var d = await redisManager.Redis.HMGetAsync<string>(RedisExpandKey.Task(id),
"Progress", "LastEnum", "StartTime", "ErrorMessage", "Progress:TidySlideWorkflow"); // 获取所有可能的进度字段
var logArr = await taskLogDB.AsQueryable()
.Where(s => s.VideoTaskId == id)
.ToArrayAsync();
var insertData = (await redisManager.Redis
.LRangeAsync<TaskLog>(RedisExpandKey.TaskLog, 0, 99))
.Where(s => s.VideoTaskId == id);
logArr = logArr.Concat(insertData).ToArray();
// 获取所有相关工作流的状态
var workflows = await baseService.Context.Queryable<VideoTaskWorkflow>()
.Where(w => w.VideoTaskId == id)
.ToListAsync();
return Ok(new
{
Progress = d[0],
TidySlideProgress = d[4], // 返回 TidySlideWorkflow 的进度
LastEnum = d[1]?.ToEnum<RedisChannelEnum>().ToString(),
StartTime = d[2] != null
? JsonSerializer.Deserialize<Dictionary<RedisChannelEnum, DateTime>>(d[2])
: null,
ErrorMessage = d[3],
Logs = logArr,
Workflows = workflows // 返回工作流列表
});
}
/// <summary>
/// 预览任务结果
/// </summary>
/// <param name="id">任务id</param>
/// <returns></returns>
[HttpGet]
public async Task<object> ShowTaskInfo(long id)
{
var nowTask = await baseService.GetFirstAsync(s => s.Id == id);
if (nowTask is null)
return BadRequest("无效任务");
var captionsArr = JsonSerializer.Deserialize<SenseVoiceRes[]>(nowTask.Captions);
var captionsArr1 = JsonSerializer.Deserialize<SenseVoiceRes[]>(nowTask.CaptionsAI ?? "[]");
var konwArr = await videoKonwPointDB.AsQueryable()
.Where(s => s.VideoTaskId == nowTask.Id)
.ToArrayAsync();
var stageArr = await videoTaskStageDB.AsQueryable()
.Where(s => s.VideoTaskId == nowTask.Id)
.ToArrayAsync();
var videoKnowDic = konwArr
.GroupBy(s => s.StageId)
.ToDictionary(s => s.Key);
var videoKnows = stageArr
.Select(s => new VideoKnowRes()
{
Content = s.Content,
StartTime = s.StartTime,
EndTime = s.EndTime,
Theme = s.Theme,
StageId = s.Id,
KnowPoint = videoKnowDic.ContainsKey(s.Id)
? string.Join(',', videoKnowDic[s.Id].Select(x => x.KnowPoint))
: string.Empty
}).ToArray();
if (nowTask.VideoType == AttachmentsInfoType.)
{
var questionArr = await videoQuestionDB
.AsQueryable().Where(s => s.VideoTaskId == nowTask.Id)
.Select<VideoQuestionShowDto>()
.ToArrayAsync();
var konwDic = (await videoQuestionKonwDB
.AsQueryable().Where(s => s.VideoTaskId == nowTask.Id)
.ToArrayAsync()).GroupBy(s => s.VideoQuestionId)
.ToDictionary(s => s.Key);
foreach (var item in questionArr.Where(s => konwDic.ContainsKey(s.Id)))
item.KonwArr = konwDic[item.Id].ToArray();
foreach (var item in videoKnows)
item.QuestionArr = questionArr
.Where(s => s.StageId == item.StageId).ToArray();
}
return Ok(new
{
Captions = captionsArr,
Captions1 = captionsArr1,
VideoKnows = videoKnows,
MediaUrl = nowTask.MediaUrl
});
}
/// <summary>
/// 获取在线设备列表
/// </summary>
/// <returns></returns>
[HttpGet]
public IActionResult OnlineDevices()
{
// 扫描 Heartbeat Key
var pattern = RedisExpandKey.DeviceHeartbeat("*");
var keys = new List<string>();
long cursor = 0;
do
{
var scanResult = redisManager.Redis.Scan(cursor, pattern, 1000, null);
keys.AddRange(scanResult.items);
cursor = scanResult.cursor;
} while (cursor != 0);
var prefix = RedisExpandKey.DeviceHeartbeat("");
var devices = keys.Select(k => k.Replace(prefix, "")).ToList();
return Ok(devices);
}
/// <summary>
/// 执行中的任务
/// </summary>
/// <param name="model">查询模型</param>
/// <returns></returns>
[HttpPost]
public async Task<object> RunningTaskList([FromBody] RunningTaskListReq model)
{
List<long> oldTaskArr;
if (string.IsNullOrEmpty(model.DeviceId))
{
// 默认获取当前节点
oldTaskArr = redisManager.Redis.LRange<long>(RedisExpandKey.IDTask, 0, 999).ToList();
}
else if (model.DeviceId == "all")
{
// 获取所有在线节点
oldTaskArr = new List<long>();
// 直接扫描 Heartbeat Key 获取在线设备
var pattern = RedisExpandKey.DeviceHeartbeat("*");
var keys = new List<string>();
long cursor = 0;
do
{
var scanResult = redisManager.Redis.Scan(cursor, pattern, 1000, null);
keys.AddRange(scanResult.items);
cursor = scanResult.cursor;
} while (cursor != 0);
var prefix = RedisExpandKey.DeviceHeartbeat("");
var onlineDevices = keys.Select(k => k.Replace(prefix, "")).ToList();
foreach (var deviceId in onlineDevices)
{
var key = RedisExpandKey.BaseKey + "Services:" + deviceId;
var tasks = redisManager.Redis.LRange<long>(key, 0, 999);
oldTaskArr.AddRange(tasks);
}
}
else
{
// 获取指定节点
var key = RedisExpandKey.BaseKey + "Services:" + model.DeviceId;
oldTaskArr = redisManager.Redis.LRange<long>(key, 0, 999).ToList();
}
var sqlquery = base.BaseQuery(model)
.Where(s => oldTaskArr.Contains(s.Id))
.Select(s => new VideoTask
{
Id = s.Id,
TagId = s.TagId,
VideoType = s.VideoType,
LastEnum = s.LastEnum,
Subject = s.Subject,
ComeFrom = s.ComeFrom,
MediaUrl = s.MediaUrl,
CreateTime = s.CreateTime,
});
RefAsync<int> total = 0;
var data = await sqlquery.ToPageListAsync(model.PageIndex + 1, model.PageSize, total);
return new PageResult<VideoTask>() { Data = data, Total = total };
}
/// <summary>
/// 错误的任务
/// </summary>
/// <param name="model">查询模型</param>
/// <returns></returns>
[HttpPost]
public async Task<object> ErrorTaskList([FromBody] QueryRequestBase model)
{
var sqlquery = base.BaseQuery(model)
.Where(s => s.ErrorMessage != null && s.ErrorMessage != "")
.Select(s => new VideoTask
{
Id = s.Id,
TagId = s.TagId,
VideoType = s.VideoType,
LastEnum = s.LastEnum,
Subject = s.Subject,
ComeFrom = s.ComeFrom,
MediaUrl = s.MediaUrl,
ErrorMessage = s.ErrorMessage,
CreateTime = s.CreateTime,
});
RefAsync<int> total = 0;
var data = await sqlquery.ToPageListAsync(model.PageIndex + 1, model.PageSize, total);
return new PageResult<VideoTask>() { Data = data, Total = total };
}
/// <summary>
/// 任务日志
/// </summary>
/// <param name="id">查询模型</param>
/// <returns></returns>
[HttpGet]
public async Task<IEnumerable<TaskLog>> TaskLog(long id)
{
var logArr = await taskLogDB.AsQueryable()
.Where(s => s.VideoTaskId == id)
.ToArrayAsync();
var insertData = (await redisManager.Redis
.LRangeAsync<TaskLog>(RedisExpandKey.TaskLog, 0, 99))
.Where(s => s.VideoTaskId == id);
return logArr.Concat(insertData);
}
/// <summary>
/// 预览 TidySlide 任务结果
/// </summary>
/// <param name="id">任务id</param>
/// <returns></returns>
[HttpGet]
public async Task<object> ShowTidySlideTaskInfo(long id)
{
var db = baseService.Context;
var result = await db.Queryable<TidySlideTaskResult>()
.Where(s => s.VideoTaskId == id)
.FirstAsync();
if (result == null)
return BadRequest("未找到 TidySlide 任务结果");
return Ok(result);
}
}
}