修复 PPT清洗工作流的异常退出

2026-03-25 16:17:05 +08:00 · 2026-03-25 16:17:05 +08:00 · a2b3c53898
parent 3df0615dc4
commit a2b3c53898
13 changed files with 152 additions and 48 deletions
--- a/Learn.VideoAnalysis.API/Expand/CoravelExpand.cs
+++ b/Learn.VideoAnalysis.API/Expand/CoravelExpand.cs
@ -18,8 +18,8 @@ namespace Learn.VideoAnalysis.API.Expand

            Console.WriteLine($"{DateTime.Now}=>初始化 Coravel");
            service.AddScheduler();
-            service.AddTransient<TaskFileClearJob>();
-            service.AddTransient<ClearAllCacheJob>();
+            //service.AddTransient<TaskFileClearJob>();
+            //service.AddTransient<ClearAllCacheJob>();
            service.AddTransient<NodePackageJob>();

        }
--- a/Learn.VideoAnalysis.API/Program.cs
+++ b/Learn.VideoAnalysis.API/Program.cs
@ -51,7 +51,7 @@ namespace Learn.VideoAnalysis.API

            builder.Services.AddHttpClient();
            builder.Services.AddSqlSugarExpand();
-            builder.Services.AddRedisExpand();
+            builder.Services.AddRedisExpand(false);

            builder.Services.AddCoravel();
            builder.Services.AddCorsExpand();
--- a/VideoAnalysis/Program.cs
+++ b/VideoAnalysis/Program.cs
@ -18,6 +18,9 @@ using System.Text.Unicode;
 using System.Text.Json;
 using Microsoft.AspNetCore.Hosting.Server.Features;
 using Microsoft.AspNetCore.Hosting.Server;
+using System.IO.Compression;
+using System.Text;
+using System.Text.Json.Nodes;



@ -25,8 +28,91 @@ namespace Learn.VideoAnalysis
 {
    public class Program
    {
+        public static void CleanHarFile(string inputPath, string outputPath)
+        {
+            try
+            {
+                // 1. 读取原始文件
+                var jsonString = File.ReadAllText(inputPath);
+                var root = JsonNode.Parse(jsonString)!;
+                var entries = root["log"]?["entries"]?.AsArray();
+
+                if (entries == null) return;
+
+                // 过滤关键词
+                var blackList = new[] { "google", "gstatic", "doubleclick", "analytics", "facebook", "recaptcha" };
+
+                // 2. 逻辑清理 (从后往前)
+                for (int i = entries.Count - 1; i >= 0; i--)
+                {
+                    var entry = entries[i];
+                    string url = entry?["request"]?["url"]?.GetValue<string>().ToLower() ?? "";
+                    string resourceType = entry?["_resourceType"]?.GetValue<string>().ToLower() ?? "";
+
+                    // 判定是否保留 (必须是 API 且 不在黑名单内)
+                    bool isApi = (resourceType == "xhr" || resourceType == "fetch");
+                    bool isBlacklisted = blackList.Any(k => url.Contains(k));
+
+                    if (!isApi || isBlacklisted)
+                    {
+                        entries.RemoveAt(i);
+                        continue;
+                    }
+
+                    // 3. 结构瘦身
+                    if (entry?["_initiator"] is JsonObject initiator)
+                    {
+                        initiator.Remove("stack");
+                        initiator.Remove("callFrames");
+                    }
+
+                    // 移除所有以下划线开头的非标准扩展字段 (Chrome 专用字段)
+                    var entryObj = entry.AsObject();
+                    var keysToRemove = entryObj.Where(kv => kv.Key.StartsWith("_")).Select(kv => kv.Key).ToList();
+                    foreach (var key in keysToRemove) entryObj.Remove(key);
+                }
+
+                // 4. 极限压缩配置
+                var options = new JsonSerializerOptions
+                {
+                    // 核心：禁用缩进，输出单行压缩格式
+                    WriteIndented = false,
+                    // 核心：防止斜杠等字符被转义成 \u002F，保持原始字符减少长度
+                    Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+                    // 忽略值为 null 的字段，进一步减小体积
+                    DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull
+                };
+
+                // 5. 写入文件
+                string compressedJson = root.ToJsonString(options);
+                File.WriteAllText(outputPath, compressedJson);
+
+                long originalSize = new FileInfo(inputPath).Length;
+                long newSize = new FileInfo(outputPath).Length;
+
+                Console.WriteLine("--- 压缩统计 ---");
+                Console.WriteLine($"原始大小: {originalSize / 1024.0:F2} KB");
+                Console.WriteLine($"压缩后大小: {newSize / 1024.0:F2} KB");
+                Console.WriteLine($"压缩率: {(1.0 - (double)newSize / originalSize) * 100:F2}%");
+                Console.WriteLine($"保留请求数: {entries.Count}");
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"处理失败: {ex.Message}");
+            }
+        }
        public static void Main(string[] args)
        {
+            // 调用方法
+            CleanHarFile("C:\\Users\\Administrator\\Downloads\\kream.co.kr.har", "C:\\Users\\Administrator\\Downloads\\kream.co.kr.API压缩.har");
+
+
+
+
+
+
+
+
            //交互式环境选择函数
            AppConfigExpand.SelectEnvironment(ref args);

--- a/VideoAnalysis/WebUI/src/views/welcome/index.vue
+++ b/VideoAnalysis/WebUI/src/views/welcome/index.vue
@ -196,24 +196,20 @@ async function RloadTaskInfo(row: any) {

  // 兼容旧逻辑：如果 workflows 为空，根据 LastEnum 构造一个默认的 VideoSliceWorkflow 状态
  if (!row.TaskInfo.workflows || row.TaskInfo.workflows.length === 0) {
-    row.TaskInfo.workflows = [
-      {
+    row.TaskInfo.workflows = [];
+  }
+   for (const key in workflowRegistry) {
+    let wf = workflowRegistry[key];
+    if(!row.TaskInfo.workflows.find((w) => w.workflowName === wf.name)){
+      row.TaskInfo.workflows.push({
        id: 0,
        videoTaskId: row.id,
-        workflowName: "VideoSliceWorkflow",
-        currentStep: row.TaskInfo.lastEnum,
+        workflowName: wf.name,
+        currentStep: 0,
        currentStepValue: 0, // 暂不重要
        updateTime: new Date().toISOString(),
-      },
-      {
-        id: 0,
-        videoTaskId: row.id,
-        workflowName: "TidySlideWorkflow",
-        currentStep: row.TaskInfo.lastEnum,
-        currentStepValue: 0, // 暂不重要
-        updateTime: new Date().toISOString(),
-      },
-    ];
+      })
+    }
  }
 }
 function formatDateToChinese(dateString) {
--- a/VideoAnalysisCore/Common/Expand/TidySlideExpand.cs
+++ b/VideoAnalysisCore/Common/Expand/TidySlideExpand.cs
@ -21,7 +21,7 @@ namespace VideoAnalysisCore.Common.Expand
        public static void AddTidySlideExpand(this IServiceCollection services)
        {
            services.AddSingleton<TidySlideHandle>();
-            services.AddTransient<Repository<TidySlideTaskResult>>();
+            //services.AddTransient<Repository<TidySlideTaskResult>>();
        }
    }

@ -234,5 +234,18 @@ namespace VideoAnalysisCore.Common.Expand
                throw;
            }
        }
+
+        public async Task CheckTask(string task)
+        {
+            var id = long.Parse(task);
+            var res = await _tidySlideTaskResultDB.IsAnyAsync(s => s.VideoTaskId == id);
+            if (res) 
+            {
+                Oh.Error("此视频已被上传过,所以取消任务");   
+                return;
+            }
+
+        }
+
    }
 }
--- a/VideoAnalysisCore/Common/RedisExpand.cs
+++ b/VideoAnalysisCore/Common/RedisExpand.cs
@ -1,4 +1,4 @@
-using FreeRedis;
+using FreeRedis;
 using FreeRedis.Internal;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.IdentityModel.Tokens;
@ -104,7 +104,7 @@ namespace VideoAnalysisCore.Common
        /// redis连接拓展(包含消息队列任务)
        /// </summary>
        /// <param name="service"></param>
-        public static void AddRedisExpand(this IServiceCollection service)
+        public static void AddRedisExpand(this IServiceCollection service,bool needWorkflow=true)
        {
            Console.WriteLine($"{DateTime.Now}=>初始化 Redis");
            var redis = new RedisClient(AppCommon.Config.Redis.ConnectionString);
@ -114,8 +114,11 @@ namespace VideoAnalysisCore.Common
                JsonSerializer.Deserialize(json, type);
            service.AddSingleton(redis);
            service.AddSingleton<RedisManager>();
-            service.AddVideoSliceWorkflow();
-            service.AddTidySlideWorkflow();
+            if (needWorkflow)
+            {
+                service.AddVideoSliceWorkflow();
+                service.AddTidySlideWorkflow();
+            }
            
            // 注册心跳 Job
            // service.AddTransient<DeviceHeartbeatJob>(); // 迁移到 CoravelExpand 中统一管理
--- a/VideoAnalysisCore/Common/TidySlideWorkflowManager.cs
+++ b/VideoAnalysisCore/Common/TidySlideWorkflowManager.cs
@ -46,6 +46,12 @@ namespace VideoAnalysisCore.Common
        {
            var SubscribeList = _manager.SubscribeList;
            SubscribeList.Add(RedisTidySlideChannelEnum.排队中, async (task) => await Task.CompletedTask);
+            SubscribeList.Add(RedisTidySlideChannelEnum.任务校验, async (task) =>
+            {
+                using var scope = _serviceProvider.CreateScope();
+                var s = scope.ServiceProvider.GetRequiredService<TidySlideHandle>();
+                await s.CheckTask(task);
+            });
            SubscribeList.Add(RedisTidySlideChannelEnum.下载文件, async (task) =>
            {
                using var scope = _serviceProvider.CreateScope();
--- a/VideoAnalysisCore/Common/VideoSliceWorkflowManager.cs
+++ b/VideoAnalysisCore/Common/VideoSliceWorkflowManager.cs
@ -100,7 +100,7 @@ namespace VideoAnalysisCore.Common
            SubscribeList.Add(RedisChannelEnum.结束任务, _manager.TaskEnd);
        }
    }
-
+    
    public class VideoSliceWorkflowManager : WorkflowBase<RedisChannelEnum>
    {
        public VideoSliceWorkflowManager(RedisClient redis, RedisManager redisManager) : base(redis, redisManager)
@ -137,7 +137,7 @@ namespace VideoAnalysisCore.Common
            // 4. 特殊分流：解析字幕完成后，后续步骤转后台并行处理
            if (currentStep == RedisChannelEnum.解析字幕)
            {
-                await DispatchBackgroundFlow(nextStep, taskId, taskId);
+                await DispatchBackgroundFlow( nextStep, taskId, taskId);
                throw new WorkflowFlowSwitchException(); // 抛出异常以中断当前流程（基类捕获）
            }
            await Task.CompletedTask;
--- a/VideoAnalysisCore/Common/WorkflowBase.cs
+++ b/VideoAnalysisCore/Common/WorkflowBase.cs
@ -38,7 +38,7 @@ namespace VideoAnalysisCore.Common
        /// 工作流名称 (e.g. "VideoSliceWorkflow")
        /// <para>默认通过类名去除 "Manager" 后缀生成，子类可重写</para>
        /// </summary>
-        protected virtual string WorkflowName => this.GetType().Name.Replace("Manager", "");
+        protected virtual string WorkflowName => "未配置的工作流名称";

        public WorkflowBase(RedisClient redis, RedisManager redisManager)
        {
@ -136,7 +136,7 @@ namespace VideoAnalysisCore.Common
                var index = i;
                var task = Task.Run(async () =>
                {
-                    Console.WriteLine($"{DateTime.Now} ==> 开始监听 [{typeof(TEnum).Name}] 队列 [{index}]...");
+                    Console.WriteLine($"{DateTime.Now} ==> 开始监听 [{typeof(TEnum).Name}] [{ChannelKey}] 队列 [{index}]...");
                    while (!token.IsCancellationRequested && !StopTask)
                    {
                        try
@ -206,9 +206,6 @@ namespace VideoAnalysisCore.Common
                        await AddTaskLog(tId, "==> 手动停止任务 ", WorkflowName);
                        return;
                    }
-                    // 1. 记录步骤开始时间 (需要转换 RedisChannelEnum 才能调用 UpdateStepTimeAsync，如果类型不匹配则需要适配)
-                    // 这里简化，暂不记录非主流程的时间，或者需要在 RedisManager 增加泛型支持
-                    // await RedisManager.UpdateStepTimeAsync(taskId, currentStep);

                    // 2. 执行当前步骤
                    await TouchChannel(currentStep, tId, SubscribeList[currentStep]);
@ -255,21 +252,18 @@ namespace VideoAnalysisCore.Common
            var stepValue = Convert.ToInt32(step);

            using var scope = AppCommon.Services.CreateScope();
-            var db = scope.ServiceProvider.GetService<ISqlSugarClient>();
+            var db = scope.ServiceProvider.GetService<Repository<VideoTaskWorkflow>>();
            if (db == null) return;
-            
-            // 尝试更新或插入 WorkflowState
-            // 注意：这里假设 VideoTaskWorkflow 表存在且已正确配置
-            // 如果不想引入新表依赖，也可以在这里留空，由子类实现
+           
            try 
            {
                // 使用 upsert 逻辑
-                var existing = await db.Queryable<VideoTaskWorkflow>()
+                var existing = await db.AsQueryable()
                    .FirstAsync(it => it.VideoTaskId == tID && it.WorkflowName == WorkflowName);
                
                if (existing == null)
                {
-                    await db.Insertable(new VideoTaskWorkflow
+                    await db.AsInsertable(new VideoTaskWorkflow
                    {
                        Id = Yitter.IdGenerator.YitIdHelper.NextId(),
                        VideoTaskId = tID,
@ -284,7 +278,7 @@ namespace VideoAnalysisCore.Common
                    existing.CurrentStep = stepName;
                    existing.CurrentStepValue = stepValue;
                    existing.UpdateTime = DateTime.Now;
-                    await db.Updateable(existing).ExecuteCommandAsync();
+                    await db.AsUpdateable(existing).ExecuteCommandAsync();
                }
            }
            catch (Exception ex)
@ -309,9 +303,9 @@ namespace VideoAnalysisCore.Common
                
                await action(taskId);
            }
-            catch (Exception ex)
+            catch
            {
-                 await AddTaskLog(taskId, $""" 出现异常 {ex.Message} {ex.StackTrace}  """);
+                 //await AddTaskLog(taskId, $""" 出现异常 {ex.Message} {ex.StackTrace}  """);
                 throw;
            }
        }
--- a/VideoAnalysisCore/Controllers/VideoTaskController.cs
+++ b/VideoAnalysisCore/Controllers/VideoTaskController.cs
@ -110,12 +110,12 @@ namespace VideoAnalysisCore.Controllers
 #endif

        /// <summary>
-        /// 添加任务到 TidySlide队列
+        /// 插入批量任务id  [VideoSlice工作流]
        /// </summary>
        /// <param name="ids">是否执行任务</param>
        /// <returns></returns>
-        [HttpPost(Name = "JoinQueueVideoSlice")]
-        public IActionResult JoinQueue( long[] ids)
+        [HttpPost]
+        public IActionResult JoinQueueVideoSlice( long[] ids)
        {
            if (ids == null || ids.Length == 0)
                return BadRequest("录入数据无效");
@ -124,12 +124,12 @@ namespace VideoAnalysisCore.Controllers
        }

        /// <summary>
-        /// 添加任务到 TidySlide队列
+        /// 插入批量任务id [TidySlide工作流]
        /// </summary>
-        /// <param name="ids"></param>
+        /// <param name="ids">是否执行任务</param>
        /// <returns></returns>
-        [HttpPost(Name = "JoinQueueTidySlide")]
-        public IActionResult JoinTidySlideQueue( long[] ids)
+        [HttpPost]
+        public IActionResult JoinQueueTidySlide(long[] ids)
        {
            if (ids == null || ids.Length == 0)
                return BadRequest("录入数据无效");
@ -381,6 +381,7 @@ namespace VideoAnalysisCore.Controllers
            var logArr = await taskLogDB.AsQueryable()
               .Where(s => s.VideoTaskId == id)
               .ToArrayAsync();
+            //任务日志只能读取数据库里的 未能写入的缓存需要写入库后其他设备才能读取到
            var insertData = (await redisManager.Redis
                .LRangeAsync<TaskLog>(RedisExpandKey.TaskLog, 0, 99))
                .Where(s => s.VideoTaskId == id);
--- a/VideoAnalysisCore/Model/Enum/CourselevelType.cs
+++ b/VideoAnalysisCore/Model/Enum/CourselevelType.cs
@ -13,6 +13,7 @@ namespace VideoAnalysisCore.Model.Enum
    {
        一层次 = 1,
        二层次 = 2,
-        三层次 = 3
+        三层次 = 3,
+        无层次 = 10
    }
 }
--- a/VideoAnalysisCore/Model/Enum/RedisTidySlideChannelEnum.cs
+++ b/VideoAnalysisCore/Model/Enum/RedisTidySlideChannelEnum.cs
@ -16,6 +16,10 @@ namespace VideoAnalysisCore.Model.Enum
        /// </summary>
        排队中 = 0,
        /// <summary>
+        /// 任务校验
+        /// </summary>
+        任务校验 = 5,
+        /// <summary>
        /// 下载文件
        /// </summary>
        下载文件 = 10,
--- a/VideoAnalysisCore/VideoAnalysisCore.csproj
+++ b/VideoAnalysisCore/VideoAnalysisCore.csproj
@ -27,7 +27,7 @@
 		<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
 		<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
 		<PackageReference Include="org.k2fsa.sherpa.onnx" Version="1.12.22" />
-		<PackageReference Include="SixLabors.ImageSharp" Version="3.1.7" />
+		<PackageReference Include="SixLabors.ImageSharp" Version="3.1.12" />
 		<PackageReference Include="SqlSugar.IOC" Version="2.0.0" />
 		<PackageReference Include="SqlSugarCore" Version="5.1.4.205" />
 		<PackageReference Include="UserCenter.Model" Version="1.4.9" />