using FFmpeg.NET.Events;
using FFmpeg.NET;
using VideoAnalysisCore.AICore.SherpaOnnx;
using VideoAnalysisCore.Common;
using System.Threading.Tasks;
using System.Xml.Linq;
using System.Runtime.InteropServices;
using SqlSugar.IOC;
using VideoAnalysisCore.Model;
using VideoAnalysisCore.Model.Enum;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Processing;
using System.Text.Json;
using System;
using Microsoft.VisualBasic.FileIO;
using Microsoft.Extensions.DependencyInjection;
namespace VideoAnalysisCore.AICore.FFMPGE
{
public static class FFMPGEExpand
{
///
/// 添加FFPMPEG拓展
///
///
public static void AddFFMPGEExpand(this IServiceCollection services)
{
services.AddSingleton();
}
}
///
/// Ffmpeg处理程序
///
public class FFMPGEHandle
{
///
///
///
public static string FFmpegPath = RuntimeInformation.IsOSPlatform(OSPlatform.Linux)
? $"/usr/bin/ffmpeg"
: Path.Combine(AppCommon.AIModelFile, "ffmpeg.exe");
private Repository videoTaskDB { get; set; }
private VideoSliceWorkflowManager _workflowManager { get; set; }
public FFMPGEHandle(VideoSliceWorkflowManager workflowManager, Repository videoTaskDB)
{
_workflowManager = workflowManager;
this.videoTaskDB = videoTaskDB;
}
///
/// 识别视频关键帧
///
/// 任务id
///
public async Task VideoKeyFrames(string task)
{
var taskID = long.Parse(task);
//间隔秒
var intervalSec = 5;
var threshold = 8.15;
var ssimThreshold = 0.89;
var taskInfo = await videoTaskDB.CopyNew().AsQueryable()
.Where(s => s.Id == long.Parse(task)).FirstAsync();
if (string.IsNullOrEmpty(taskInfo.PPTVideoCode) || string.IsNullOrEmpty(taskInfo.PPTVideoUrl)) return;
//视频切帧
var localPath = task.LocalPath();
var filePath = Path.Combine(localPath, "ppt.mp4");
if (!File.Exists(filePath))
{
_workflowManager.AddTaskLog(task,"存在PPT Code但未能找到对应资源文件");
return;
}
var ffmpeg = new Engine(FFmpegPath);
var cToken = new CancellationToken();
_workflowManager.SetTaskProgress(task, "Frame=>10%");
foreach (string jpgFile in Directory.GetFiles(localPath, "*.jpg"))
File.Delete(jpgFile);
_workflowManager.SetTaskProgress(task, "Frame=>20%");
await ffmpeg.ExecuteAsync($"-i {filePath} -vf \"fps=1/{intervalSec},scale=960:540\" {localPath}/{ExpandFunction.FrameName}%03d.jpg", cToken);
//视频关键帧分析
var frameFiles = Directory.GetFiles(localPath, "*.jpg")
.OrderBy(f => f)
.ToList();
_workflowManager.SetTaskProgress(task, "Frame=>80%");
Image prevFrame = null;
var keyFrames = new List(10) { 5};
foreach (var frameFile in frameFiles)
{
using (var currFrame = Image.Load(frameFile))
{
if (prevFrame != null)
{
double ssim = SSIMCalculator.CalculateFrameSSIM(prevFrame, currFrame);
//double diff = CalculateFrameDifference(prevFrame, currFrame);
double timestamp = GetTimestampFromFileName(frameFile) * intervalSec;
//if (diff > threshold)
if (ssim < ssimThreshold)
{
keyFrames.Add((int)timestamp);
//string outputPath = Path.Combine(outputDir, $"change_{timestamp:0000}.jpg");
//currFrame.Save(outputPath);
//await redisManager.AddTaskLog(chatReq.taskId, $"变化帧: {timestamp}秒,差异值: {ssim:F2}");
}
//await redisManager.AddTaskLog(chatReq.taskId, $"帧: {timestamp}秒,SSIM{ssim:F2} 差异值: {ssim:F2} ");
}
prevFrame?.Dispose();
prevFrame = currFrame.Clone();
}
}
// 去掉相邻的重复图片
for (int i = 1; i < keyFrames.Count(); i++)
{
if (keyFrames[i] - keyFrames[i - 1] < 10)
keyFrames[i] = -1;
}
//写入数据库
var keyFramStr = keyFrames.Where(s => s != -1).ToJson();
await videoTaskDB.CopyNew().AsUpdateable()
.SetColumns(it => it.PPTKeyFrame == keyFramStr)
.Where(it => it.Id == taskID)
.ExecuteCommandAsync();
}
///
/// 计算帧差异
///
///
///
///
double CalculateFrameDifference(Image img1, Image img2)
{
// 统一调整为64x64
var resized1 = img1.Clone(x => x.Grayscale());
var resized2 = img2.Clone(x => x.Grayscale());
long diff = 0;
for (int y = 0; y < resized1.Height; y++)
{
for (int x = 0; x < resized1.Width; x++)
{
var pixel1 = resized1[x, y];
var pixel2 = resized2[x, y];
diff += Math.Abs(pixel1.R - pixel2.R);
}
}
return diff / (double)(resized1.Width * resized1.Height);
}
double GetTimestampFromFileName(string filePath)
{
string fileName = Path.GetFileNameWithoutExtension(filePath);
return double.Parse(fileName.Split('_')[1]);
}
///
/// 执行视频FFMPEG处理任务
///
///
///
public async Task RunAsync(string task)
{
await VideoKeyFrames(task);
await Audio2WAV16KAsync(task);
}
///
/// 音频转码为 wav_16k
///
/// 任务id
///
public async Task Audio2WAV16KAsync(string task)
{
var filePath = await videoTaskDB.CopyNew().AsQueryable()
.Where(s => s.Id == long.Parse(task))
.Select(s=>s.LocalMediaPath).FirstAsync();
if (string.IsNullOrEmpty(filePath))
throw new Exception($"任务id[{task}] 无效");
// 打开输入文件
var inputFile = new InputFile(filePath);
var outputFile = new OutputFile(Path.Combine(task.LocalPath(), Path.GetFileNameWithoutExtension(filePath) + ".wav"));
var ffmpeg = new Engine(FFmpegPath);
ffmpeg.Error += (sender, e) =>
{
var ee = new Exception($"音频转码出现异常 \r\n[{e.Input.Name} => {e.Output.Name}]: 错误: {e.Exception.Message}");
throw ee;
};
var conversionOptions = new ConversionOptions
{
ExtraArguments = "-ar 16000 -ac 1"
};
try
{
await ffmpeg.ConvertAsync(inputFile, outputFile, conversionOptions);
}
catch
{
throw;
}
}
///
/// 合并音频和视频并切片
///
///
///
public async Task MergeAndSliceAsync(string task)
{
var taskID = long.Parse(task);
var localPath = task.LocalPath();
var pptPath = Path.Combine(localPath, "ppt.mp4");
var taskPath = Path.Combine(localPath, "task.mp4");
var mergedPath = Path.Combine(localPath, "merged.mp4");
var m3u8Path = Path.Combine(localPath, "out.m3u8");
if (!File.Exists(pptPath)) throw new FileNotFoundException("PPT视频文件未找到", pptPath);
if (!File.Exists(taskPath)) throw new FileNotFoundException("任务视频文件未找到", taskPath);
var ffmpeg = new Engine(FFmpegPath);
var cToken = new CancellationToken();
// 1. 合并 PPT视频(画面) + 任务视频(音频) -> merged.mp4
// -map 0:v 取第一个输入(ppt)的视频流
// -map 1:a 取第二个输入(task)的音频流
// -c:v copy 复制视频流不转码
// -c:a aac 音频转码为aac (兼容性好)
// -strict experimental 允许使用aac
// -shortest 以最短的流为准
var mergeArgs = $"-i \"{pptPath}\" -i \"{taskPath}\" -map 0:v -map 1:a -c:v copy -c:a aac -strict experimental -shortest \"{mergedPath}\" -y";
await _workflowManager.AddTaskLog(task, "开始合并视频与音频...");
await ffmpeg.ExecuteAsync(mergeArgs, cToken);
if (!File.Exists(mergedPath)) throw new Exception("视频合并失败");
// 2. 切片 merged.mp4 -> out.m3u8
// -c copy 直接复制流 (因为上一步已经是 mp4/aac)
// -f hls HLS格式
// -hls_time 10 切片时长10秒
// -hls_list_size 0 包含所有切片
// -hls_segment_filename out%03d.ts 切片文件名
var sliceArgs = $"-i \"{mergedPath}\" -c copy -f hls -hls_time 10 -hls_list_size 0 -hls_segment_filename \"{Path.Combine(localPath, "out%03d.ts")}\" \"{m3u8Path}\" -y";
await _workflowManager.AddTaskLog(task, "开始视频切片...");
await ffmpeg.ExecuteAsync(sliceArgs, cToken);
if (!File.Exists(m3u8Path)) throw new Exception("视频切片失败");
// 更新任务状态或路径? 目前只需要生成文件
await _workflowManager.AddTaskLog(task, "视频处理完成");
}
}
}