327 lines
12 KiB
C#
327 lines
12 KiB
C#
using FreeRedis;
|
||
using FreeRedis.Internal;
|
||
using Microsoft.Extensions.DependencyInjection;
|
||
using Microsoft.IdentityModel.Tokens;
|
||
using NetTaste;
|
||
using Newtonsoft.Json.Schema;
|
||
using SqlSugar.IOC;
|
||
using System;
|
||
using System.Security.Cryptography;
|
||
using System.Text.Json;
|
||
using System.Threading.Channels;
|
||
using System.Threading.Tasks;
|
||
using System.Collections.Concurrent;
|
||
using System.Xml.Linq;
|
||
using UserCenter.Model.Enum;
|
||
using VideoAnalysisCore.AICore.FFMPGE;
|
||
using VideoAnalysisCore.AICore.GPT;
|
||
using VideoAnalysisCore.AICore.GPT.Dto;
|
||
|
||
|
||
//using VideoAnalysisCore.AICore.FFMPGE;
|
||
using VideoAnalysisCore.AICore.SherpaOnnx;
|
||
using VideoAnalysisCore.AICore.Whisper;
|
||
using VideoAnalysisCore.Model;
|
||
using VideoAnalysisCore.Model.Dto;
|
||
using VideoAnalysisCore.Model.Enum;
|
||
|
||
namespace VideoAnalysisCore.Common
|
||
{
|
||
/// <summary>
|
||
/// redis key
|
||
/// </summary>
|
||
public static class RedisExpandKey
|
||
{
|
||
|
||
/// <summary>
|
||
/// 基础key
|
||
/// </summary>
|
||
public const string BaseKey = "VideoAnalysis:";
|
||
/// <summary>
|
||
/// 基础Channel key
|
||
/// </summary>
|
||
public const string ChannelKey = BaseKey + "TaskChannel";
|
||
/// <summary>
|
||
/// 上传工作流 Channel key
|
||
/// </summary>
|
||
public const string UploadChannelKey = BaseKey + "UploadTaskChannel";
|
||
/// <summary>
|
||
/// 下载文件
|
||
/// </summary>
|
||
public const string DownloadFile = ChannelKey + "DownloadFile";
|
||
/// <summary>
|
||
/// 分离音频
|
||
/// </summary>
|
||
public const string SeparateAudio = ChannelKey + "SeparateAudio";
|
||
/// <summary>
|
||
/// 解析字幕
|
||
/// </summary>
|
||
public const string ParsingCaptions = ChannelKey + "ParsingCaptions";
|
||
/// <summary>
|
||
/// 解析说话人
|
||
/// </summary>
|
||
public const string ParsingSpeaker = ChannelKey + "ParsingSpeaker";
|
||
/// <summary>
|
||
/// Chat模型分析
|
||
/// </summary>
|
||
public const string ChatModelAnalysis = ChannelKey + "ChatModelAnalysis";
|
||
|
||
/// <summary>
|
||
/// 任务数组
|
||
/// </summary>
|
||
public const string TaskArr = BaseKey + "TaskArr";
|
||
/// <summary>
|
||
/// 任务日志缓存
|
||
/// </summary>
|
||
public static string TaskLog => BaseKey + "TaskLog:" + AppCommon.Config.ID;
|
||
|
||
|
||
/// <summary>
|
||
/// 任务对象地址
|
||
/// </summary>
|
||
public static string Task(object taskId) => BaseKey + "Info:" + taskId;
|
||
public static string IDTask => BaseKey + "Services:" + AppCommon.Config.ID;
|
||
public static string TaskGPT(object taskId) => BaseKey + "GPTCached:" + taskId;
|
||
/// <summary>
|
||
/// 初始化 redis
|
||
/// <para>需要在初始化配置文件时候调用</para>
|
||
/// </summary>
|
||
public static void AddTaskSubscribe(this IServiceCollection service)
|
||
{
|
||
|
||
Console.WriteLine($"{DateTime.Now}=>初始化 Redis任务队列");
|
||
service.AddSingleton<RedisInit>();
|
||
}
|
||
/// <summary>
|
||
/// redis连接拓展(包含消息队列任务)
|
||
/// </summary>
|
||
/// <param name="service"></param>
|
||
public static void AddRedisExpand(this IServiceCollection service)
|
||
{
|
||
Console.WriteLine($"{DateTime.Now}=>初始化 Redis");
|
||
var redis = new RedisClient(AppCommon.Config.Redis.ConnectionString);
|
||
redis.Serialize = obj =>
|
||
JsonSerializer.Serialize(obj);
|
||
redis.Deserialize = (json, type) =>
|
||
JsonSerializer.Deserialize(json, type);
|
||
service.AddSingleton(redis);
|
||
service.AddSingleton<RedisManager>();
|
||
service.AddVideoSliceWorkflow();
|
||
service.AddUploadWorkflow();
|
||
}
|
||
}
|
||
|
||
public class RedisInit
|
||
{
|
||
|
||
public RedisInit(IServiceProvider serviceProvider)
|
||
{
|
||
serviceProvider.GetService<VideoSliceWorkflowInit>();
|
||
serviceProvider.GetService<UploadWorkflowInit>();
|
||
// serviceProvider.GetService<RedisManager>().InitChannel(); // 已废弃，由各工作流自行初始化
|
||
}
|
||
}
|
||
/// <summary>
|
||
/// redis拓展
|
||
/// </summary>
|
||
public class RedisManager
|
||
{
|
||
public static bool StopTask { get; set; } = false;
|
||
public static Dictionary<RedisChannelEnum, Func<string, Task>> SubscribeList = new Dictionary<RedisChannelEnum, Func<string, Task>>();
|
||
|
||
/// <summary>
|
||
/// 正在后台运行的任务集合
|
||
/// </summary>
|
||
public static ConcurrentDictionary<string, Task> RunningTasks = new ConcurrentDictionary<string, Task>();
|
||
|
||
private static CancellationTokenSource? _cts;
|
||
private static Task? _workerTask;
|
||
|
||
public RedisClient Redis { get; set; }
|
||
public Repository<VideoTask> videoTaskDB { get; set; }
|
||
public Repository<TaskLog> taskLogDB { get; set; }
|
||
|
||
public RedisManager(RedisClient redis, Repository<VideoTask> videoTaskDB, Repository<TaskLog> taskLogDB)
|
||
{
|
||
Redis = redis;
|
||
this.videoTaskDB = videoTaskDB;
|
||
this.taskLogDB = taskLogDB;
|
||
}
|
||
|
||
|
||
/// <summary>
|
||
/// 缓存GPT任务缓存
|
||
/// </summary>
|
||
/// <param name="taskId"></param>
|
||
public void SetTaskGPTCached(object taskId, string time, object? data)
|
||
{
|
||
Redis.Set(RedisExpandKey.TaskGPT(taskId) + ":" + time, data, timeoutSeconds: 3600 * 24);
|
||
}
|
||
/// <summary>
|
||
/// 加入到消费队列
|
||
/// </summary>
|
||
/// <param name="taskIds"></param>
|
||
public void JoinQueue(params long[] taskIds)
|
||
{ //事务
|
||
if (taskIds is null || taskIds.Length == 0)
|
||
return;
|
||
using (var tran = Redis.Multi())
|
||
{
|
||
foreach (var item in taskIds)
|
||
tran.LPush(RedisExpandKey.ChannelKey, item);
|
||
tran.Exec();
|
||
}
|
||
}
|
||
/// <summary>
|
||
/// 添加日志
|
||
/// </summary>
|
||
/// <param name="taskId">任务id</param>
|
||
/// <param name="msg">内容</param>
|
||
public async Task AddTaskLog(object taskId, string msg)
|
||
{
|
||
#if DEBUG
|
||
Console.WriteLine($"{DateTime.Now.ToString("MM-dd HH:mm:ss")} => {taskId} \r\n{msg}\r\n");
|
||
#endif
|
||
await Redis.RPushAsync(RedisExpandKey.TaskLog,
|
||
new TaskLog()
|
||
{
|
||
VideoTaskId = long.Parse(taskId.ToString()),
|
||
CreateTime = DateTime.Now,
|
||
Message = msg,
|
||
DeviceId = AppCommon.Config.ID
|
||
});
|
||
var count = 50;
|
||
lock (RedisExpandKey.TaskLog)
|
||
{
|
||
var oldTaskCount = Redis.LLen(RedisExpandKey.TaskLog);
|
||
if (oldTaskCount > count)
|
||
{
|
||
try
|
||
{
|
||
var insertData = Redis.LRange<TaskLog>(RedisExpandKey.TaskLog, 0, count -1);
|
||
taskLogDB.CopyNew().AsInsertable(insertData).ExecuteCommand();
|
||
//同步删除redis
|
||
Redis.LTrim(RedisExpandKey.TaskLog, count, 1000);
|
||
}
|
||
catch (Exception ex)
|
||
{
|
||
Console.WriteLine("写入任务日志出错" + "\r\n" + ex.Message + "\r\n" + ex.StackTrace);
|
||
}
|
||
}
|
||
}
|
||
}
|
||
/// <summary>
|
||
/// 获取任务进度
|
||
/// </summary>
|
||
/// <param name="taskId"></param>
|
||
public float GetTaskProgress(object taskId)
|
||
{
|
||
return Redis.HMGet<float>(RedisExpandKey.Task(taskId), "Progress")[0];
|
||
}
|
||
/// <summary>
|
||
/// 设置任务进度
|
||
/// </summary>
|
||
/// <param name="p">进度百分比</param>
|
||
/// <param name="taskId"></param>
|
||
public void SetTaskProgress(object taskId, object p)
|
||
{
|
||
Redis.HMSet(RedisExpandKey.Task(taskId), "Progress", p.ToString());
|
||
|
||
}
|
||
|
||
public async Task TaskEnd(string task)
|
||
{
|
||
var tId = long.Parse(task);
|
||
//var gptRes = (await Redis
|
||
// .HMGetAsync<TaskRes>(RedisExpandKey.Task(task), "ChatAnalysis")).FirstOrDefault();
|
||
//if (gptRes is null)
|
||
// throw new Exception("未能读取到GPT处理结果");
|
||
//删除任务执行状态
|
||
await Redis.LRemAsync(RedisExpandKey.IDTask, 1, task);
|
||
var taskData = await videoTaskDB
|
||
.CopyNew()
|
||
.GetFirstAsync(s => s.Id == tId);
|
||
if (taskData.Captions == "[]")
|
||
taskData.Captions = (await Redis.HMGetAsync(RedisExpandKey.Task(task), "Captions")).First();
|
||
//if (taskData.Speaker == "[]")
|
||
// taskData.Speaker = (await Redis.HMGetAsync(RedisExpandKey.Task(task), "Speaker"))?.FirstOrDefault()??"[]";
|
||
|
||
|
||
//未使用结果暂时屏蔽
|
||
//taskData.ChatAnalysis = JsonSerializer.Serialize(gptRes);
|
||
taskData.ChatAnalysisScore = 0;
|
||
taskData.ErrorMessage = string.Empty;
|
||
taskData.LastEnum = RedisChannelEnum.结束任务;
|
||
taskData.EndTime = DateTime.Now;
|
||
await videoTaskDB.CopyNew().AsUpdateable(taskData)
|
||
.UpdateColumns(it => new
|
||
{
|
||
//it.ChatAnalysis,
|
||
it.Captions,
|
||
it.Speaker,
|
||
it.ChatAnalysisScore,
|
||
it.ErrorMessage,
|
||
it.TotalTokens,
|
||
it.LastEnum,
|
||
it.EndTime
|
||
}).ExecuteCommandAsync();
|
||
try
|
||
{
|
||
//await ExpandFunction.DeleteTaskFileAsync(tId, this);
|
||
|
||
await ExpandFunction.DeleteTaskAllFileAsync(tId, this);
|
||
}
|
||
catch (Exception)
|
||
{
|
||
|
||
throw;
|
||
}
|
||
//NewTask();
|
||
}
|
||
|
||
/// <summary>
|
||
/// 写入任务异常
|
||
/// </summary>
|
||
/// <param name="taskID"></param>
|
||
/// <param name="ex"></param>
|
||
/// <returns></returns>
|
||
public async Task<bool> SetTaskErrorMessage(long taskID, Exception? ex)
|
||
{
|
||
var error = string.Empty;
|
||
if (ex != null)
|
||
{
|
||
|
||
await Redis.LRemAsync(RedisExpandKey.IDTask, 1, taskID.ToString());
|
||
//执行任务时出现异常
|
||
error = ex.Message + ex.StackTrace;
|
||
await AddTaskLog(taskID, $""" 出现异常 {ex.Message} {ex.StackTrace} """);
|
||
//清除失败任务 重新接收任务
|
||
// NewTask(); // 已废弃，工作流会自动处理
|
||
}
|
||
return await SetTaskError(taskID, error);
|
||
}
|
||
/// <summary>
|
||
/// 清除 任务的错误信息
|
||
/// </summary>
|
||
/// <param name="taskID"></param>
|
||
/// <returns></returns>
|
||
public async Task<bool> ClearTaskError(long taskID) => await SetTaskError(taskID, string.Empty);
|
||
/// <summary>
|
||
/// 修改任务的错误信息
|
||
/// </summary>
|
||
/// <param name="taskID"></param>
|
||
/// <param name="error"></param>
|
||
/// <returns></returns>
|
||
public async Task<bool> SetTaskError(long taskID, string? error)
|
||
{
|
||
var vDB = AppCommon.Services.GetService<Repository<VideoTask>>();
|
||
Redis.HMSet(RedisExpandKey.Task(taskID), "ErrorMessage", error);
|
||
return await vDB.CopyNew().AsUpdateable()
|
||
.SetColumns(it => it.ErrorMessage == error)//SetColumns是可以叠加的 写2个就2个字段赋值
|
||
.Where(it => it.Id == taskID)
|
||
.ExecuteCommandAsync() == 1;
|
||
}
|
||
|
||
}
|
||
}
|