263 lines
10 KiB
C#
263 lines
10 KiB
C#
using FFmpeg.NET.Events;
|
||
using FFmpeg.NET;
|
||
using VideoAnalysisCore.AICore.SherpaOnnx;
|
||
using VideoAnalysisCore.Common;
|
||
using System.Threading.Tasks;
|
||
using System.Xml.Linq;
|
||
using System.Runtime.InteropServices;
|
||
using SqlSugar.IOC;
|
||
using VideoAnalysisCore.Model;
|
||
using VideoAnalysisCore.Model.Enum;
|
||
using SixLabors.ImageSharp;
|
||
using SixLabors.ImageSharp.PixelFormats;
|
||
using SixLabors.ImageSharp.Processing;
|
||
using System.Text.Json;
|
||
using System;
|
||
using Microsoft.VisualBasic.FileIO;
|
||
using Microsoft.Extensions.DependencyInjection;
|
||
|
||
namespace VideoAnalysisCore.AICore.FFMPGE
|
||
{
|
||
|
||
public static class FFMPGEExpand
|
||
{
|
||
/// <summary>
|
||
/// 添加FFPMPEG拓展
|
||
/// </summary>
|
||
/// <param name="services"></param>
|
||
public static void AddFFMPGEExpand(this IServiceCollection services)
|
||
{
|
||
services.AddSingleton<FFMPGEHandle>();
|
||
}
|
||
}
|
||
/// <summary>
|
||
/// Ffmpeg处理程序
|
||
/// </summary>
|
||
public class FFMPGEHandle
|
||
{
|
||
/// <summary>
|
||
///
|
||
/// </summary>
|
||
public static string FFmpegPath = RuntimeInformation.IsOSPlatform(OSPlatform.Linux)
|
||
? $"/usr/bin/ffmpeg"
|
||
: Path.Combine(AppCommon.AIModelFile, "ffmpeg.exe");
|
||
private Repository<VideoTask> videoTaskDB { get; set; }
|
||
private VideoSliceWorkflowManager _workflowManager { get; set; }
|
||
public FFMPGEHandle(VideoSliceWorkflowManager workflowManager, Repository<VideoTask> videoTaskDB)
|
||
{
|
||
_workflowManager = workflowManager;
|
||
this.videoTaskDB = videoTaskDB;
|
||
}
|
||
|
||
/// <summary>
|
||
/// 识别视频关键帧
|
||
/// </summary>
|
||
/// <param name="task">任务id</param>
|
||
/// <returns></returns>
|
||
public async Task VideoKeyFrames(string task)
|
||
{
|
||
var taskID = long.Parse(task);
|
||
//间隔秒
|
||
var intervalSec = 5;
|
||
var threshold = 8.15;
|
||
var ssimThreshold = 0.89;
|
||
var taskInfo = await videoTaskDB.CopyNew().AsQueryable()
|
||
.Where(s => s.Id == long.Parse(task)).FirstAsync();
|
||
if (string.IsNullOrEmpty(taskInfo.PPTVideoCode) || string.IsNullOrEmpty(taskInfo.PPTVideoUrl)) return;
|
||
//视频切帧
|
||
var localPath = task.LocalPath();
|
||
var filePath = Path.Combine(localPath, "ppt.mp4");
|
||
if (!File.Exists(filePath))
|
||
{
|
||
_workflowManager.AddTaskLog(task,"存在PPT Code但未能找到对应资源文件");
|
||
return;
|
||
}
|
||
var ffmpeg = new Engine(FFmpegPath);
|
||
var cToken = new CancellationToken();
|
||
_workflowManager.SetTaskProgress(task, "Frame=>10%");
|
||
|
||
foreach (string jpgFile in Directory.GetFiles(localPath, "*.jpg"))
|
||
File.Delete(jpgFile);
|
||
_workflowManager.SetTaskProgress(task, "Frame=>20%");
|
||
|
||
await ffmpeg.ExecuteAsync($"-i {filePath} -vf \"fps=1/{intervalSec},scale=960:540\" {localPath}/{ExpandFunction.FrameName}%03d.jpg", cToken);
|
||
|
||
//视频关键帧分析
|
||
var frameFiles = Directory.GetFiles(localPath, "*.jpg")
|
||
.OrderBy(f => f)
|
||
.ToList();
|
||
_workflowManager.SetTaskProgress(task, "Frame=>80%");
|
||
Image<Rgb24> prevFrame = null;
|
||
var keyFrames = new List<int>(10) { 5};
|
||
foreach (var frameFile in frameFiles)
|
||
{
|
||
using (var currFrame = Image.Load<Rgb24>(frameFile))
|
||
{
|
||
if (prevFrame != null)
|
||
{
|
||
double ssim = SSIMCalculator.CalculateFrameSSIM(prevFrame, currFrame);
|
||
//double diff = CalculateFrameDifference(prevFrame, currFrame);
|
||
double timestamp = GetTimestampFromFileName(frameFile) * intervalSec;
|
||
|
||
//if (diff > threshold)
|
||
if (ssim < ssimThreshold)
|
||
{
|
||
keyFrames.Add((int)timestamp);
|
||
//string outputPath = Path.Combine(outputDir, $"change_{timestamp:0000}.jpg");
|
||
//currFrame.Save(outputPath);
|
||
//await redisManager.AddTaskLog(chatReq.taskId, $"变化帧: {timestamp}秒，差异值: {ssim:F2}");
|
||
}
|
||
//await redisManager.AddTaskLog(chatReq.taskId, $"帧: {timestamp}秒，SSIM{ssim:F2} 差异值: {ssim:F2} ");
|
||
|
||
}
|
||
prevFrame?.Dispose();
|
||
prevFrame = currFrame.Clone();
|
||
}
|
||
}
|
||
// 去掉相邻的重复图片
|
||
for (int i = 1; i < keyFrames.Count(); i++)
|
||
{
|
||
if (keyFrames[i] - keyFrames[i - 1] < 10)
|
||
keyFrames[i] = -1;
|
||
}
|
||
//写入数据库
|
||
var keyFramStr = keyFrames.Where(s => s != -1).ToJson();
|
||
await videoTaskDB.CopyNew().AsUpdateable()
|
||
.SetColumns(it => it.PPTKeyFrame == keyFramStr)
|
||
.Where(it => it.Id == taskID)
|
||
.ExecuteCommandAsync();
|
||
|
||
}
|
||
|
||
/// <summary>
|
||
/// 计算帧差异
|
||
/// </summary>
|
||
/// <param name="img1"></param>
|
||
/// <param name="img2"></param>
|
||
/// <returns></returns>
|
||
double CalculateFrameDifference(Image<Rgb24> img1, Image<Rgb24> img2)
|
||
{
|
||
// 统一调整为64x64
|
||
var resized1 = img1.Clone(x => x.Grayscale());
|
||
var resized2 = img2.Clone(x => x.Grayscale());
|
||
|
||
long diff = 0;
|
||
for (int y = 0; y < resized1.Height; y++)
|
||
{
|
||
for (int x = 0; x < resized1.Width; x++)
|
||
{
|
||
var pixel1 = resized1[x, y];
|
||
var pixel2 = resized2[x, y];
|
||
diff += Math.Abs(pixel1.R - pixel2.R);
|
||
}
|
||
}
|
||
return diff / (double)(resized1.Width * resized1.Height);
|
||
}
|
||
|
||
double GetTimestampFromFileName(string filePath)
|
||
{
|
||
string fileName = Path.GetFileNameWithoutExtension(filePath);
|
||
return double.Parse(fileName.Split('_')[1]);
|
||
}
|
||
/// <summary>
|
||
/// 执行视频FFMPEG处理任务
|
||
/// </summary>
|
||
/// <param name="task"></param>
|
||
/// <returns></returns>
|
||
public async Task RunAsync(string task)
|
||
{
|
||
await VideoKeyFrames(task);
|
||
await Audio2WAV16KAsync(task);
|
||
}
|
||
/// <summary>
|
||
/// 音频转码为 wav_16k
|
||
/// </summary>
|
||
/// <param name="task">任务id</param>
|
||
/// <returns></returns>
|
||
public async Task Audio2WAV16KAsync(string task)
|
||
{
|
||
var filePath = await videoTaskDB.CopyNew().AsQueryable()
|
||
.Where(s => s.Id == long.Parse(task))
|
||
.Select(s=>s.LocalMediaPath).FirstAsync();
|
||
if (string.IsNullOrEmpty(filePath))
|
||
throw new Exception($"任务id[{task}] 无效");
|
||
|
||
// 打开输入文件
|
||
var inputFile = new InputFile(filePath);
|
||
var outputFile = new OutputFile(Path.Combine(task.LocalPath(), Path.GetFileNameWithoutExtension(filePath) + ".wav"));
|
||
var ffmpeg = new Engine(FFmpegPath);
|
||
|
||
ffmpeg.Error += (sender, e) =>
|
||
{
|
||
var ee = new Exception($"音频转码出现异常 \r\n[{e.Input.Name} => {e.Output.Name}]: 错误: {e.Exception.Message}");
|
||
throw ee;
|
||
};
|
||
|
||
var conversionOptions = new ConversionOptions
|
||
{
|
||
ExtraArguments = "-ar 16000 -ac 1"
|
||
};
|
||
try
|
||
{
|
||
await ffmpeg.ConvertAsync(inputFile, outputFile, conversionOptions);
|
||
}
|
||
catch
|
||
{
|
||
throw;
|
||
}
|
||
|
||
}
|
||
|
||
/// <summary>
|
||
/// 合并音频和视频并切片
|
||
/// </summary>
|
||
/// <param name="task"></param>
|
||
/// <returns></returns>
|
||
public async Task MergeAndSliceAsync(string task)
|
||
{
|
||
var taskID = long.Parse(task);
|
||
var localPath = task.LocalPath();
|
||
var pptPath = Path.Combine(localPath, "ppt.mp4");
|
||
var taskPath = Path.Combine(localPath, "task.mp4");
|
||
var mergedPath = Path.Combine(localPath, "merged.mp4");
|
||
var m3u8Path = Path.Combine(localPath, "out.m3u8");
|
||
|
||
if (!File.Exists(pptPath)) throw new FileNotFoundException("PPT视频文件未找到", pptPath);
|
||
if (!File.Exists(taskPath)) throw new FileNotFoundException("任务视频文件未找到", taskPath);
|
||
|
||
var ffmpeg = new Engine(FFmpegPath);
|
||
var cToken = new CancellationToken();
|
||
|
||
// 1. 合并 PPT视频(画面) + 任务视频(音频) -> merged.mp4
|
||
// -map 0:v 取第一个输入(ppt)的视频流
|
||
// -map 1:a 取第二个输入(task)的音频流
|
||
// -c:v copy 复制视频流不转码
|
||
// -c:a aac 音频转码为aac (兼容性好)
|
||
// -strict experimental 允许使用aac
|
||
// -shortest 以最短的流为准
|
||
var mergeArgs = $"-i \"{pptPath}\" -i \"{taskPath}\" -map 0:v -map 1:a -c:v copy -c:a aac -strict experimental -shortest \"{mergedPath}\" -y";
|
||
|
||
await _workflowManager.AddTaskLog(task, "开始合并视频与音频...");
|
||
await ffmpeg.ExecuteAsync(mergeArgs, cToken);
|
||
|
||
if (!File.Exists(mergedPath)) throw new Exception("视频合并失败");
|
||
|
||
// 2. 切片 merged.mp4 -> out.m3u8
|
||
// -c copy 直接复制流 (因为上一步已经是 mp4/aac)
|
||
// -f hls HLS格式
|
||
// -hls_time 10 切片时长10秒
|
||
// -hls_list_size 0 包含所有切片
|
||
// -hls_segment_filename out%03d.ts 切片文件名
|
||
var sliceArgs = $"-i \"{mergedPath}\" -c copy -f hls -hls_time 10 -hls_list_size 0 -hls_segment_filename \"{Path.Combine(localPath, "out%03d.ts")}\" \"{m3u8Path}\" -y";
|
||
|
||
await _workflowManager.AddTaskLog(task, "开始视频切片...");
|
||
await ffmpeg.ExecuteAsync(sliceArgs, cToken);
|
||
|
||
if (!File.Exists(m3u8Path)) throw new Exception("视频切片失败");
|
||
|
||
// 更新任务状态或路径? 目前只需要生成文件
|
||
await _workflowManager.AddTaskLog(task, "视频处理完成");
|
||
}
|
||
}
|
||
}
|