diff --git a/Learn.VideoAnalysis.API/Expand/CoravelExpand.cs b/Learn.VideoAnalysis.API/Expand/CoravelExpand.cs index 27b251c..7912973 100644 --- a/Learn.VideoAnalysis.API/Expand/CoravelExpand.cs +++ b/Learn.VideoAnalysis.API/Expand/CoravelExpand.cs @@ -18,8 +18,8 @@ namespace Learn.VideoAnalysis.API.Expand Console.WriteLine($"{DateTime.Now}=>初始化 Coravel"); service.AddScheduler(); - service.AddTransient(); - service.AddTransient(); + //service.AddTransient(); + //service.AddTransient(); service.AddTransient(); } diff --git a/Learn.VideoAnalysis.API/Program.cs b/Learn.VideoAnalysis.API/Program.cs index 5272232..c0d1ead 100644 --- a/Learn.VideoAnalysis.API/Program.cs +++ b/Learn.VideoAnalysis.API/Program.cs @@ -51,7 +51,7 @@ namespace Learn.VideoAnalysis.API builder.Services.AddHttpClient(); builder.Services.AddSqlSugarExpand(); - builder.Services.AddRedisExpand(); + builder.Services.AddRedisExpand(false); builder.Services.AddCoravel(); builder.Services.AddCorsExpand(); diff --git a/VideoAnalysis/Program.cs b/VideoAnalysis/Program.cs index f6d6179..c7e713d 100644 --- a/VideoAnalysis/Program.cs +++ b/VideoAnalysis/Program.cs @@ -18,6 +18,9 @@ using System.Text.Unicode; using System.Text.Json; using Microsoft.AspNetCore.Hosting.Server.Features; using Microsoft.AspNetCore.Hosting.Server; +using System.IO.Compression; +using System.Text; +using System.Text.Json.Nodes; @@ -25,8 +28,91 @@ namespace Learn.VideoAnalysis { public class Program { + public static void CleanHarFile(string inputPath, string outputPath) + { + try + { + // 1. 读取原始文件 + var jsonString = File.ReadAllText(inputPath); + var root = JsonNode.Parse(jsonString)!; + var entries = root["log"]?["entries"]?.AsArray(); + + if (entries == null) return; + + // 过滤关键词 + var blackList = new[] { "google", "gstatic", "doubleclick", "analytics", "facebook", "recaptcha" }; + + // 2. 逻辑清理 (从后往前) + for (int i = entries.Count - 1; i >= 0; i--) + { + var entry = entries[i]; + string url = entry?["request"]?["url"]?.GetValue().ToLower() ?? ""; + string resourceType = entry?["_resourceType"]?.GetValue().ToLower() ?? ""; + + // 判定是否保留 (必须是 API 且 不在黑名单内) + bool isApi = (resourceType == "xhr" || resourceType == "fetch"); + bool isBlacklisted = blackList.Any(k => url.Contains(k)); + + if (!isApi || isBlacklisted) + { + entries.RemoveAt(i); + continue; + } + + // 3. 结构瘦身 + if (entry?["_initiator"] is JsonObject initiator) + { + initiator.Remove("stack"); + initiator.Remove("callFrames"); + } + + // 移除所有以下划线开头的非标准扩展字段 (Chrome 专用字段) + var entryObj = entry.AsObject(); + var keysToRemove = entryObj.Where(kv => kv.Key.StartsWith("_")).Select(kv => kv.Key).ToList(); + foreach (var key in keysToRemove) entryObj.Remove(key); + } + + // 4. 极限压缩配置 + var options = new JsonSerializerOptions + { + // 核心：禁用缩进，输出单行压缩格式 + WriteIndented = false, + // 核心：防止斜杠等字符被转义成 \u002F，保持原始字符减少长度 + Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping, + // 忽略值为 null 的字段，进一步减小体积 + DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull + }; + + // 5. 写入文件 + string compressedJson = root.ToJsonString(options); + File.WriteAllText(outputPath, compressedJson); + + long originalSize = new FileInfo(inputPath).Length; + long newSize = new FileInfo(outputPath).Length; + + Console.WriteLine("--- 压缩统计 ---"); + Console.WriteLine($"原始大小: {originalSize / 1024.0:F2} KB"); + Console.WriteLine($"压缩后大小: {newSize / 1024.0:F2} KB"); + Console.WriteLine($"压缩率: {(1.0 - (double)newSize / originalSize) * 100:F2}%"); + Console.WriteLine($"保留请求数: {entries.Count}"); + } + catch (Exception ex) + { + Console.WriteLine($"处理失败: {ex.Message}"); + } + } public static void Main(string[] args) { + // 调用方法 + CleanHarFile("C:\\Users\\Administrator\\Downloads\\kream.co.kr.har", "C:\\Users\\Administrator\\Downloads\\kream.co.kr.API压缩.har"); + + + + + + + + //交互式环境选择函数 AppConfigExpand.SelectEnvironment(ref args); diff --git a/VideoAnalysis/WebUI/src/views/welcome/index.vue b/VideoAnalysis/WebUI/src/views/welcome/index.vue index 7752794..529adee 100644 --- a/VideoAnalysis/WebUI/src/views/welcome/index.vue +++ b/VideoAnalysis/WebUI/src/views/welcome/index.vue @@ -196,24 +196,20 @@ async function RloadTaskInfo(row: any) { // 兼容旧逻辑：如果 workflows 为空，根据 LastEnum 构造一个默认的 VideoSliceWorkflow 状态 if (!row.TaskInfo.workflows || row.TaskInfo.workflows.length === 0) { - row.TaskInfo.workflows = [ - { + row.TaskInfo.workflows = []; + } + for (const key in workflowRegistry) { + let wf = workflowRegistry[key]; + if(!row.TaskInfo.workflows.find((w) => w.workflowName === wf.name)){ + row.TaskInfo.workflows.push({ id: 0, videoTaskId: row.id, - workflowName: "VideoSliceWorkflow", - currentStep: row.TaskInfo.lastEnum, + workflowName: wf.name, + currentStep: 0, currentStepValue: 0, // 暂不重要 updateTime: new Date().toISOString(), - }, - { - id: 0, - videoTaskId: row.id, - workflowName: "TidySlideWorkflow", - currentStep: row.TaskInfo.lastEnum, - currentStepValue: 0, // 暂不重要 - updateTime: new Date().toISOString(), - }, - ]; + }) + } } } function formatDateToChinese(dateString) { diff --git a/VideoAnalysisCore/Common/Expand/TidySlideExpand.cs b/VideoAnalysisCore/Common/Expand/TidySlideExpand.cs index 8c49179..0ab4563 100644 --- a/VideoAnalysisCore/Common/Expand/TidySlideExpand.cs +++ b/VideoAnalysisCore/Common/Expand/TidySlideExpand.cs @@ -21,7 +21,7 @@ namespace VideoAnalysisCore.Common.Expand public static void AddTidySlideExpand(this IServiceCollection services) { services.AddSingleton(); - services.AddTransient>(); + //services.AddTransient>(); } } @@ -234,5 +234,18 @@ namespace VideoAnalysisCore.Common.Expand throw; } } + + public async Task CheckTask(string task) + { + var id = long.Parse(task); + var res = await _tidySlideTaskResultDB.IsAnyAsync(s => s.VideoTaskId == id); + if (res) + { + Oh.Error("此视频已被上传过,所以取消任务"); + return; + } + + } + } } diff --git a/VideoAnalysisCore/Common/RedisExpand.cs b/VideoAnalysisCore/Common/RedisExpand.cs index 7b54446..d8e5c8f 100644 --- a/VideoAnalysisCore/Common/RedisExpand.cs +++ b/VideoAnalysisCore/Common/RedisExpand.cs @@ -1,4 +1,4 @@ -using FreeRedis; +using FreeRedis; using FreeRedis.Internal; using Microsoft.Extensions.DependencyInjection; using Microsoft.IdentityModel.Tokens; @@ -104,7 +104,7 @@ namespace VideoAnalysisCore.Common /// redis连接拓展(包含消息队列任务) /// /// - public static void AddRedisExpand(this IServiceCollection service) + public static void AddRedisExpand(this IServiceCollection service,bool needWorkflow=true) { Console.WriteLine($"{DateTime.Now}=>初始化 Redis"); var redis = new RedisClient(AppCommon.Config.Redis.ConnectionString); @@ -114,8 +114,11 @@ namespace VideoAnalysisCore.Common JsonSerializer.Deserialize(json, type); service.AddSingleton(redis); service.AddSingleton(); - service.AddVideoSliceWorkflow(); - service.AddTidySlideWorkflow(); + if (needWorkflow) + { + service.AddVideoSliceWorkflow(); + service.AddTidySlideWorkflow(); + } // 注册心跳 Job // service.AddTransient(); // 迁移到 CoravelExpand 中统一管理 diff --git a/VideoAnalysisCore/Common/TidySlideWorkflowManager.cs b/VideoAnalysisCore/Common/TidySlideWorkflowManager.cs index 65ea86a..4387cfa 100644 --- a/VideoAnalysisCore/Common/TidySlideWorkflowManager.cs +++ b/VideoAnalysisCore/Common/TidySlideWorkflowManager.cs @@ -46,6 +46,12 @@ namespace VideoAnalysisCore.Common { var SubscribeList = _manager.SubscribeList; SubscribeList.Add(RedisTidySlideChannelEnum.排队中, async (task) => await Task.CompletedTask); + SubscribeList.Add(RedisTidySlideChannelEnum.任务校验, async (task) => + { + using var scope = _serviceProvider.CreateScope(); + var s = scope.ServiceProvider.GetRequiredService(); + await s.CheckTask(task); + }); SubscribeList.Add(RedisTidySlideChannelEnum.下载文件, async (task) => { using var scope = _serviceProvider.CreateScope(); diff --git a/VideoAnalysisCore/Common/VideoSliceWorkflowManager.cs b/VideoAnalysisCore/Common/VideoSliceWorkflowManager.cs index d297727..1392796 100644 --- a/VideoAnalysisCore/Common/VideoSliceWorkflowManager.cs +++ b/VideoAnalysisCore/Common/VideoSliceWorkflowManager.cs @@ -100,7 +100,7 @@ namespace VideoAnalysisCore.Common SubscribeList.Add(RedisChannelEnum.结束任务, _manager.TaskEnd); } } - + public class VideoSliceWorkflowManager : WorkflowBase { public VideoSliceWorkflowManager(RedisClient redis, RedisManager redisManager) : base(redis, redisManager) @@ -137,7 +137,7 @@ namespace VideoAnalysisCore.Common // 4. 特殊分流：解析字幕完成后，后续步骤转后台并行处理 if (currentStep == RedisChannelEnum.解析字幕) { - await DispatchBackgroundFlow(nextStep, taskId, taskId); + await DispatchBackgroundFlow( nextStep, taskId, taskId); throw new WorkflowFlowSwitchException(); // 抛出异常以中断当前流程（基类捕获） } await Task.CompletedTask; diff --git a/VideoAnalysisCore/Common/WorkflowBase.cs b/VideoAnalysisCore/Common/WorkflowBase.cs index 50e72b4..9c3ba8d 100644 --- a/VideoAnalysisCore/Common/WorkflowBase.cs +++ b/VideoAnalysisCore/Common/WorkflowBase.cs @@ -38,7 +38,7 @@ namespace VideoAnalysisCore.Common /// 工作流名称 (e.g. "VideoSliceWorkflow") /// 默认通过类名去除 "Manager" 后缀生成，子类可重写 /// - protected virtual string WorkflowName => this.GetType().Name.Replace("Manager", ""); + protected virtual string WorkflowName => "未配置的工作流名称"; public WorkflowBase(RedisClient redis, RedisManager redisManager) { @@ -136,7 +136,7 @@ namespace VideoAnalysisCore.Common var index = i; var task = Task.Run(async () => { - Console.WriteLine($"{DateTime.Now} ==> 开始监听 [{typeof(TEnum).Name}] 队列 [{index}]..."); + Console.WriteLine($"{DateTime.Now} ==> 开始监听 [{typeof(TEnum).Name}] [{ChannelKey}] 队列 [{index}]..."); while (!token.IsCancellationRequested && !StopTask) { try @@ -206,9 +206,6 @@ namespace VideoAnalysisCore.Common await AddTaskLog(tId, "==> 手动停止任务 ", WorkflowName); return; } - // 1. 记录步骤开始时间 (需要转换 RedisChannelEnum 才能调用 UpdateStepTimeAsync，如果类型不匹配则需要适配) - // 这里简化，暂不记录非主流程的时间，或者需要在 RedisManager 增加泛型支持 - // await RedisManager.UpdateStepTimeAsync(taskId, currentStep); // 2. 执行当前步骤 await TouchChannel(currentStep, tId, SubscribeList[currentStep]); @@ -255,21 +252,18 @@ namespace VideoAnalysisCore.Common var stepValue = Convert.ToInt32(step); using var scope = AppCommon.Services.CreateScope(); - var db = scope.ServiceProvider.GetService(); + var db = scope.ServiceProvider.GetService>(); if (db == null) return; - - // 尝试更新或插入 WorkflowState - // 注意：这里假设 VideoTaskWorkflow 表存在且已正确配置 - // 如果不想引入新表依赖，也可以在这里留空，由子类实现 + try { // 使用 upsert 逻辑 - var existing = await db.Queryable() + var existing = await db.AsQueryable() .FirstAsync(it => it.VideoTaskId == tID && it.WorkflowName == WorkflowName); if (existing == null) { - await db.Insertable(new VideoTaskWorkflow + await db.AsInsertable(new VideoTaskWorkflow { Id = Yitter.IdGenerator.YitIdHelper.NextId(), VideoTaskId = tID, @@ -284,7 +278,7 @@ namespace VideoAnalysisCore.Common existing.CurrentStep = stepName; existing.CurrentStepValue = stepValue; existing.UpdateTime = DateTime.Now; - await db.Updateable(existing).ExecuteCommandAsync(); + await db.AsUpdateable(existing).ExecuteCommandAsync(); } } catch (Exception ex) @@ -309,9 +303,9 @@ namespace VideoAnalysisCore.Common await action(taskId); } - catch (Exception ex) + catch { - await AddTaskLog(taskId, $""" 出现异常 {ex.Message} {ex.StackTrace} """); + //await AddTaskLog(taskId, $""" 出现异常 {ex.Message} {ex.StackTrace} """); throw; } } diff --git a/VideoAnalysisCore/Controllers/VideoTaskController.cs b/VideoAnalysisCore/Controllers/VideoTaskController.cs index cee4ec6..c7e4834 100644 --- a/VideoAnalysisCore/Controllers/VideoTaskController.cs +++ b/VideoAnalysisCore/Controllers/VideoTaskController.cs @@ -110,12 +110,12 @@ namespace VideoAnalysisCore.Controllers #endif /// - /// 添加任务到 TidySlide队列 + /// 插入批量任务id [VideoSlice工作流] /// /// 是否执行任务 /// - [HttpPost(Name = "JoinQueueVideoSlice")] - public IActionResult JoinQueue( long[] ids) + [HttpPost] + public IActionResult JoinQueueVideoSlice( long[] ids) { if (ids == null || ids.Length == 0) return BadRequest("录入数据无效"); @@ -124,12 +124,12 @@ namespace VideoAnalysisCore.Controllers } /// - /// 添加任务到 TidySlide队列 + /// 插入批量任务id [TidySlide工作流] /// - /// + /// 是否执行任务 /// - [HttpPost(Name = "JoinQueueTidySlide")] - public IActionResult JoinTidySlideQueue( long[] ids) + [HttpPost] + public IActionResult JoinQueueTidySlide(long[] ids) { if (ids == null || ids.Length == 0) return BadRequest("录入数据无效"); @@ -381,6 +381,7 @@ namespace VideoAnalysisCore.Controllers var logArr = await taskLogDB.AsQueryable() .Where(s => s.VideoTaskId == id) .ToArrayAsync(); + //任务日志只能读取数据库里的 未能写入的缓存需要写入库后其他设备才能读取到 var insertData = (await redisManager.Redis .LRangeAsync(RedisExpandKey.TaskLog, 0, 99)) .Where(s => s.VideoTaskId == id); diff --git a/VideoAnalysisCore/Model/Enum/CourselevelType.cs b/VideoAnalysisCore/Model/Enum/CourselevelType.cs index ab57330..471f3f4 100644 --- a/VideoAnalysisCore/Model/Enum/CourselevelType.cs +++ b/VideoAnalysisCore/Model/Enum/CourselevelType.cs @@ -13,6 +13,7 @@ namespace VideoAnalysisCore.Model.Enum { 一层次 = 1, 二层次 = 2, - 三层次 = 3 + 三层次 = 3, + 无层次 = 10 } } diff --git a/VideoAnalysisCore/Model/Enum/RedisTidySlideChannelEnum.cs b/VideoAnalysisCore/Model/Enum/RedisTidySlideChannelEnum.cs index cfdf420..e3402d5 100644 --- a/VideoAnalysisCore/Model/Enum/RedisTidySlideChannelEnum.cs +++ b/VideoAnalysisCore/Model/Enum/RedisTidySlideChannelEnum.cs @@ -16,6 +16,10 @@ namespace VideoAnalysisCore.Model.Enum /// 排队中 = 0, /// + /// 任务校验 + /// + 任务校验 = 5, + /// /// 下载文件 /// 下载文件 = 10, diff --git a/VideoAnalysisCore/VideoAnalysisCore.csproj b/VideoAnalysisCore/VideoAnalysisCore.csproj index 0ad094d..1d3a695 100644 --- a/VideoAnalysisCore/VideoAnalysisCore.csproj +++ b/VideoAnalysisCore/VideoAnalysisCore.csproj @@ -27,7 +27,7 @@ - +