using FFmpeg.NET.Events; using FFmpeg.NET; using VideoAnalysisCore.AICore.SherpaOnnx; using VideoAnalysisCore.Common; using System.Threading.Tasks; using System.Xml.Linq; using System.Runtime.InteropServices; using SqlSugar.IOC; using VideoAnalysisCore.Model; using VideoAnalysisCore.Model.Enum; using SixLabors.ImageSharp; using SixLabors.ImageSharp.PixelFormats; using SixLabors.ImageSharp.Processing; using System.Text.Json; using System; using Microsoft.VisualBasic.FileIO; using Microsoft.Extensions.DependencyInjection; namespace VideoAnalysisCore.AICore.FFMPGE { public static class FFMPGEExpand { /// /// 添加FFPMPEG拓展 /// /// public static void AddFFMPGEExpand(this IServiceCollection services) { services.AddSingleton(); } } /// /// Ffmpeg处理程序 /// public class FFMPGEHandle { /// /// /// public static string FFmpegPath = RuntimeInformation.IsOSPlatform(OSPlatform.Linux) ? $"/usr/bin/ffmpeg" : Path.Combine(AppCommon.AIModelFile, "ffmpeg.exe"); private Repository videoTaskDB { get; set; } private VideoSliceWorkflowManager _workflowManager { get; set; } public FFMPGEHandle(VideoSliceWorkflowManager workflowManager, Repository videoTaskDB) { _workflowManager = workflowManager; this.videoTaskDB = videoTaskDB; } /// /// 识别视频关键帧 /// /// 任务id /// public async Task VideoKeyFrames(string task) { var taskID = long.Parse(task); //间隔秒 var intervalSec = 5; var threshold = 8.15; var ssimThreshold = 0.89; var taskInfo = await videoTaskDB.CopyNew().AsQueryable() .Where(s => s.Id == long.Parse(task)).FirstAsync(); if (string.IsNullOrEmpty(taskInfo.PPTVideoCode) || string.IsNullOrEmpty(taskInfo.PPTVideoUrl)) return; //视频切帧 var localPath = task.LocalPath(); var filePath = Path.Combine(localPath, "ppt.mp4"); if (!File.Exists(filePath)) { _workflowManager.AddTaskLog(task,"存在PPT Code但未能找到对应资源文件"); return; } var ffmpeg = new Engine(FFmpegPath); var cToken = new CancellationToken(); _workflowManager.SetTaskProgress(task, "Frame=>10%"); foreach (string jpgFile in Directory.GetFiles(localPath, "*.jpg")) File.Delete(jpgFile); _workflowManager.SetTaskProgress(task, "Frame=>20%"); await ffmpeg.ExecuteAsync($"-i {filePath} -vf \"fps=1/{intervalSec},scale=960:540\" {localPath}/{ExpandFunction.FrameName}%03d.jpg", cToken); //视频关键帧分析 var frameFiles = Directory.GetFiles(localPath, "*.jpg") .OrderBy(f => f) .ToList(); _workflowManager.SetTaskProgress(task, "Frame=>80%"); Image prevFrame = null; var keyFrames = new List(10) { 5}; foreach (var frameFile in frameFiles) { using (var currFrame = Image.Load(frameFile)) { if (prevFrame != null) { double ssim = SSIMCalculator.CalculateFrameSSIM(prevFrame, currFrame); //double diff = CalculateFrameDifference(prevFrame, currFrame); double timestamp = GetTimestampFromFileName(frameFile) * intervalSec; //if (diff > threshold) if (ssim < ssimThreshold) { keyFrames.Add((int)timestamp); //string outputPath = Path.Combine(outputDir, $"change_{timestamp:0000}.jpg"); //currFrame.Save(outputPath); //await redisManager.AddTaskLog(chatReq.taskId, $"变化帧: {timestamp}秒,差异值: {ssim:F2}"); } //await redisManager.AddTaskLog(chatReq.taskId, $"帧: {timestamp}秒,SSIM{ssim:F2} 差异值: {ssim:F2} "); } prevFrame?.Dispose(); prevFrame = currFrame.Clone(); } } // 去掉相邻的重复图片 for (int i = 1; i < keyFrames.Count(); i++) { if (keyFrames[i] - keyFrames[i - 1] < 10) keyFrames[i] = -1; } //写入数据库 var keyFramStr = keyFrames.Where(s => s != -1).ToJson(); await videoTaskDB.CopyNew().AsUpdateable() .SetColumns(it => it.PPTKeyFrame == keyFramStr) .Where(it => it.Id == taskID) .ExecuteCommandAsync(); } /// /// 计算帧差异 /// /// /// /// double CalculateFrameDifference(Image img1, Image img2) { // 统一调整为64x64 var resized1 = img1.Clone(x => x.Grayscale()); var resized2 = img2.Clone(x => x.Grayscale()); long diff = 0; for (int y = 0; y < resized1.Height; y++) { for (int x = 0; x < resized1.Width; x++) { var pixel1 = resized1[x, y]; var pixel2 = resized2[x, y]; diff += Math.Abs(pixel1.R - pixel2.R); } } return diff / (double)(resized1.Width * resized1.Height); } double GetTimestampFromFileName(string filePath) { string fileName = Path.GetFileNameWithoutExtension(filePath); return double.Parse(fileName.Split('_')[1]); } /// /// 执行视频FFMPEG处理任务 /// /// /// public async Task RunAsync(string task) { await VideoKeyFrames(task); await Audio2WAV16KAsync(task); } /// /// 音频转码为 wav_16k /// /// 任务id /// public async Task Audio2WAV16KAsync(string task) { var filePath = await videoTaskDB.CopyNew().AsQueryable() .Where(s => s.Id == long.Parse(task)) .Select(s=>s.LocalMediaPath).FirstAsync(); if (string.IsNullOrEmpty(filePath)) throw new Exception($"任务id[{task}] 无效"); // 打开输入文件 var inputFile = new InputFile(filePath); var outputFile = new OutputFile(Path.Combine(task.LocalPath(), Path.GetFileNameWithoutExtension(filePath) + ".wav")); var ffmpeg = new Engine(FFmpegPath); ffmpeg.Error += (sender, e) => { var ee = new Exception($"音频转码出现异常 \r\n[{e.Input.Name} => {e.Output.Name}]: 错误: {e.Exception.Message}"); throw ee; }; var conversionOptions = new ConversionOptions { ExtraArguments = "-ar 16000 -ac 1" }; try { await ffmpeg.ConvertAsync(inputFile, outputFile, conversionOptions); } catch { throw; } } /// /// 合并音频和视频并切片 /// /// /// public async Task MergeAndSliceAsync(string task) { var taskID = long.Parse(task); var localPath = task.LocalPath(); var pptPath = Path.Combine(localPath, "ppt.mp4"); var taskPath = Path.Combine(localPath, "task.mp4"); var mergedPath = Path.Combine(localPath, "merged.mp4"); var m3u8Path = Path.Combine(localPath, "out.m3u8"); if (!File.Exists(pptPath)) throw new FileNotFoundException("PPT视频文件未找到", pptPath); if (!File.Exists(taskPath)) throw new FileNotFoundException("任务视频文件未找到", taskPath); var ffmpeg = new Engine(FFmpegPath); var cToken = new CancellationToken(); // 1. 合并 PPT视频(画面) + 任务视频(音频) -> merged.mp4 // -map 0:v 取第一个输入(ppt)的视频流 // -map 1:a 取第二个输入(task)的音频流 // -c:v copy 复制视频流不转码 // -c:a aac 音频转码为aac (兼容性好) // -strict experimental 允许使用aac // -shortest 以最短的流为准 var mergeArgs = $"-i \"{pptPath}\" -i \"{taskPath}\" -map 0:v -map 1:a -c:v copy -c:a aac -strict experimental -shortest \"{mergedPath}\" -y"; await _workflowManager.AddTaskLog(task, "开始合并视频与音频..."); await ffmpeg.ExecuteAsync(mergeArgs, cToken); if (!File.Exists(mergedPath)) throw new Exception("视频合并失败"); // 2. 切片 merged.mp4 -> out.m3u8 // -c copy 直接复制流 (因为上一步已经是 mp4/aac) // -f hls HLS格式 // -hls_time 10 切片时长10秒 // -hls_list_size 0 包含所有切片 // -hls_segment_filename out%03d.ts 切片文件名 var sliceArgs = $"-i \"{mergedPath}\" -c copy -f hls -hls_time 10 -hls_list_size 0 -hls_segment_filename \"{Path.Combine(localPath, "out%03d.ts")}\" \"{m3u8Path}\" -y"; await _workflowManager.AddTaskLog(task, "开始视频切片..."); await ffmpeg.ExecuteAsync(sliceArgs, cToken); if (!File.Exists(m3u8Path)) throw new Exception("视频切片失败"); // 更新任务状态或路径? 目前只需要生成文件 await _workflowManager.AddTaskLog(task, "视频处理完成"); } } }