Learn.VideoAnalysis/VideoAnalysisCore/AICore/FFMPGE/FFMPGEHandle.cs

using FFmpeg.NET.Events;
using FFmpeg.NET;
using VideoAnalysisCore.AICore.SherpaOnnx;
using VideoAnalysisCore.Common;
using System.Threading.Tasks;
using System.Xml.Linq;
using System.Runtime.InteropServices;
using SqlSugar.IOC;
using VideoAnalysisCore.Model;
using VideoAnalysisCore.Model.Enum;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Processing;
using System.Text.Json;
using System;
using Microsoft.VisualBasic.FileIO;
using Microsoft.Extensions.DependencyInjection;

namespace VideoAnalysisCore.AICore.FFMPGE
{

    public static class FFMPGEExpand
    {
        /// <summary>
        /// 添加跨域拓展
        /// </summary>
        /// <param name="services"></param>
        public static void AddFFMPGEExpand(this IServiceCollection services)
        {
            services.AddSingleton<FFMPGEHandle>();
        }
    }
    /// <summary>
    /// Ffmpeg处理程序
    /// </summary>
    public class FFMPGEHandle
    {
        /// <summary>
        /// 
        /// </summary>
        public static string FFmpegPath = RuntimeInformation.IsOSPlatform(OSPlatform.Linux)
                    ? $"/usr/bin/ffmpeg"
                    : Path.Combine(AppCommon.AIModelFile, "ffmpeg.exe");
        private Repository<VideoTask> videoTaskDB { get; set; }
        private RedisManager redisManager { get; set; }
        public FFMPGEHandle(RedisManager redisManager, Repository<VideoTask> videoTaskDB)
        {
            this.redisManager = redisManager;
            this.videoTaskDB = videoTaskDB;
        }

        /// <summary>
        /// 识别视频关键帧
        /// </summary>
        /// <param name="task">任务id</param>
        /// <returns></returns>
        public async Task VideoKeyFrames(string task)
        {
            var taskID = long.Parse(task);
            //间隔秒
            var intervalSec = 5;
            var threshold = 8.15;
            var ssimThreshold = 0.9;
            var taskInfo = await videoTaskDB.CopyNew().AsQueryable()
                .Where(s => s.Id == long.Parse(task)).FirstAsync();
            if (string.IsNullOrEmpty(taskInfo.PPTVideoCode) || string.IsNullOrEmpty(taskInfo.PPTVideoUrl)) return;
            //视频切帧
            var localPath = task.LocalPath();
            var filePath = Path.Combine(localPath, "ppt.mp4");
            if (!File.Exists(filePath))
            {
                redisManager.AddTaskLog(task,"存在PPT Code但未能找到对应资源文件"); 
                return; 
            }
            var ffmpeg = new Engine(FFmpegPath);
            var cToken = new CancellationToken();
            redisManager.SetTaskProgress(task, "Frame=>10%");

            foreach (string jpgFile in Directory.GetFiles(localPath, "*.jpg"))
                File.Delete(jpgFile);
            redisManager.SetTaskProgress(task, "Frame=>20%");

            await ffmpeg.ExecuteAsync($"-i {filePath} -vf \"fps=1/{intervalSec},scale=960:540\" {localPath}/{ExpandFunction.FrameName}%03d.jpg", cToken);

            //视频关键帧分析
            var frameFiles = Directory.GetFiles(localPath, "*.jpg")
              .OrderBy(f => f)
              .ToList();
            redisManager.SetTaskProgress(task, "Frame=>80%");
            Image<Rgb24> prevFrame = null;
            var keyFrames = new List<int>(10) { 5};
            foreach (var frameFile in frameFiles)
            {
                using (var currFrame = Image.Load<Rgb24>(frameFile))
                {
                    if (prevFrame != null)
                    {
                        double ssim = SSIMCalculator.CalculateFrameSSIM(prevFrame, currFrame);
                        //double diff = CalculateFrameDifference(prevFrame, currFrame);
                        double timestamp = GetTimestampFromFileName(frameFile) * intervalSec;

                        //if (diff > threshold)
                        if (ssim < ssimThreshold)
                        {
                            keyFrames.Add((int)timestamp);
                            //string outputPath = Path.Combine(outputDir, $"change_{timestamp:0000}.jpg");
                            //currFrame.Save(outputPath);
                            //await redisManager.AddTaskLog(chatReq.taskId, $"变化帧: {timestamp}秒，差异值: {ssim:F2}");
                        }
                        //await redisManager.AddTaskLog(chatReq.taskId, $"帧: {timestamp}秒，SSIM{ssim:F2} 差异值: {ssim:F2} ");

                    }
                    prevFrame?.Dispose();
                    prevFrame = currFrame.Clone();
                }
            } 
            // 去掉相邻的重复图片
            for (int i = 1; i < keyFrames.Count(); i++)
            {
                if (keyFrames[i] - keyFrames[i - 1] < 10)
                    keyFrames[i] = -1;
            }
            //写入数据库
            var keyFramStr = keyFrames.Where(s => s != -1).ToJson();
            await videoTaskDB.CopyNew().AsUpdateable()
                .SetColumns(it => it.PPTKeyFrame == keyFramStr)
                .Where(it => it.Id == taskID)
                .ExecuteCommandAsync();

        }

        /// <summary>
        /// 计算帧差异
        /// </summary>
        /// <param name="img1"></param>
        /// <param name="img2"></param>
        /// <returns></returns>
        double CalculateFrameDifference(Image<Rgb24> img1, Image<Rgb24> img2)
        {
            // 统一调整为64x64
            var resized1 = img1.Clone(x => x.Grayscale());
            var resized2 = img2.Clone(x => x.Grayscale());

            long diff = 0;
            for (int y = 0; y < resized1.Height; y++)
            {
                for (int x = 0; x < resized1.Width; x++)
                {
                    var pixel1 = resized1[x, y];
                    var pixel2 = resized2[x, y];
                    diff += Math.Abs(pixel1.R - pixel2.R);
                }
            }
            return diff / (double)(resized1.Width * resized1.Height);
        }

        double GetTimestampFromFileName(string filePath)
        {
            string fileName = Path.GetFileNameWithoutExtension(filePath);
            return double.Parse(fileName.Split('_')[1]);
        }
        /// <summary>
        /// 执行视频FFMPEG处理任务
        /// </summary>
        /// <param name="task"></param>
        /// <returns></returns>
        public async Task RunAsync(string task)
        {
            await VideoKeyFrames(task);
            await Audio2WAV16KAsync(task);
        }
        /// <summary>
        /// 音频转码为 wav_16k
        /// </summary>
        /// <param name="task">任务id</param>
        /// <returns></returns>
        public async Task Audio2WAV16KAsync(string task)
        {
            var filePath = await videoTaskDB.CopyNew().AsQueryable()
                .Where(s => s.Id == long.Parse(task))
                .Select(s=>s.LocalMediaPath).FirstAsync();
            if (string.IsNullOrEmpty(filePath))
                throw new Exception($"任务id[{task}] 无效");

            // 打开输入文件
            var inputFile = new InputFile(filePath);
            var outputFile = new OutputFile(Path.Combine(task.LocalPath(), Path.GetFileNameWithoutExtension(filePath) + ".wav"));
            var ffmpeg = new Engine(FFmpegPath);

            ffmpeg.Error += (sender, e) =>
            {
                var ee = new Exception($"音频转码出现异常 \r\n[{e.Input.Name} => {e.Output.Name}]: 错误: {e.Exception.Message}");
                throw ee;
            };

            var conversionOptions = new ConversionOptions
            {
                ExtraArguments = "-ar 16000 -ac 1"
            };
            try
            {
                await ffmpeg.ConvertAsync(inputFile, outputFile, conversionOptions);
            }
            catch
            {
                throw;
            }

        }

    }
}