Learn.VideoAnalysis/VideoAnalysisCore/AICore/FFMPGE/FFMPGEHandle.cs

using FFmpeg.NET.Events;
using FFmpeg.NET;
using VideoAnalysisCore.AICore.SherpaOnnx;
using VideoAnalysisCore.Common;
using System.Threading.Tasks;
using static System.Runtime.InteropServices.JavaScript.JSType;
using System.Xml.Linq;
using System.Runtime.InteropServices;
using SqlSugar.IOC;
using VideoAnalysisCore.Model;
using VideoAnalysisCore.Model.Enum;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Processing;
using System.Text.Json;
using System;
using Microsoft.VisualBasic.FileIO;

namespace VideoAnalysisCore.AICore.FFMPGE
{
    /// <summary>
    /// Ffmpeg处理程序
    /// </summary>
    public class FFMPGEHandle
    {
        /// <summary>
        /// 
        /// </summary>
        public static string FFmpegPath = RuntimeInformation.IsOSPlatform(OSPlatform.Linux)
                    ? $"/usr/bin/ffmpeg"
                    : Path.Combine(AppCommon.AIModelFile, "ffmpeg.exe");

        /// <summary>
        /// 识别视频关键帧
        /// </summary>
        /// <param name="task">任务id</param>
        /// <returns></returns>
        public static async Task VideoKeyFrames(string task)
        {
            var taskID = long.Parse(task);
            //间隔秒
            var intervalSec = 5;
            var threshold = 8.15;
            var PPTVideoCode = await DbScoped.Sugar
                .Queryable<VideoTask>()
                .Where(s => s.Id == long.Parse(task))
                .Select(s => s.PPTVideoCode).FirstAsync();
            if (string.IsNullOrEmpty(PPTVideoCode)) return;
            //视频切帧
            var localPath = task.LocalPath();
            var filePath = Path.Combine(localPath, "ppt.mp4");
            if (!File.Exists(filePath))
                throw new Exception("存在PPTCOde但未能找到对应资源文件");
            var ffmpeg = new Engine(FFmpegPath);
            var cToken = new CancellationToken();
            RedisExpand.SetTaskProgress(task, "Frame=>10%");

            foreach (string jpgFile in Directory.GetFiles(localPath, "*.jpg"))
                FileSystem.DeleteFile(jpgFile, UIOption.OnlyErrorDialogs, RecycleOption.DeletePermanently);
            RedisExpand.SetTaskProgress(task, "Frame=>20%");

            await ffmpeg.ExecuteAsync($"-i {filePath} -vf \"fps=1/{intervalSec},scale=320:180\" {localPath}/frame_%03d.jpg", cToken);

            //视频关键帧分析
            var frameFiles = Directory.GetFiles(localPath, "*.jpg")
              .OrderBy(f => f)
              .ToList();
            
            RedisExpand.SetTaskProgress(task, "Frame=>50%");
            Image<Rgb24> prevFrame = null;
            var keyFrames = new List<int>();
            foreach (var frameFile in frameFiles)
            {
                using (var currFrame = Image.Load<Rgb24>(frameFile))
                {
                    if (prevFrame != null)
                    {
                        double diff = CalculateFrameDifference(prevFrame, currFrame);
                        double timestamp = GetTimestampFromFileName(frameFile) * intervalSec;

                        if (diff > threshold)
                        {
                            keyFrames.Add((int)timestamp);
                            //string outputPath = Path.Combine(outputDir, $"change_{timestamp:0000}.jpg");
                            //currFrame.Save(outputPath);
                            Console.WriteLine($"变化帧: {timestamp}秒，差异值: {diff:F2}");
                        }
                        //else
                            //Console.WriteLine($"帧: {timestamp}秒，差异值: {diff:F2}");

                    }
                    prevFrame?.Dispose();
                    prevFrame = currFrame.Clone();
                }
            } 
            // 遍历数组
            for (int i = 1; i < keyFrames.Count(); i++)
            {
                keyFrames[i] += 5;//ppt与课堂视频时间修正
                if (keyFrames[i] - keyFrames[i - 1] < 10)
                    keyFrames[i] = -1;
            }
            //写入数据库
            var keyFramStr = JsonSerializer.Serialize(keyFrames.Where(s=>s!=-1));
            await DbScoped.Sugar
                .Updateable<VideoTask>()
                .SetColumns(it => it.PPTKeyFrame == keyFramStr)
                .Where(it => it.Id == taskID)
                .ExecuteCommandAsync();

        } 
        /// <summary>
           /// 计算帧差异
           /// </summary>
           /// <param name="img1"></param>
           /// <param name="img2"></param>
           /// <returns></returns>
        static double CalculateFrameDifference(Image<Rgb24> img1, Image<Rgb24> img2)
        {
            // 统一调整为64x64
            var resized1 = img1.Clone(x => x.Grayscale());
            var resized2 = img2.Clone(x => x.Grayscale());

            long diff = 0;
            for (int y = 0; y < resized1.Height; y++)
            {
                for (int x = 0; x < resized1.Width; x++)
                {
                    var pixel1 = resized1[x, y];
                    var pixel2 = resized2[x, y];
                    diff += Math.Abs(pixel1.R - pixel2.R);
                }
            }
            return diff / (double)(resized1.Width * resized1.Height);
        }

        static double GetTimestampFromFileName(string filePath)
        {
            string fileName = Path.GetFileNameWithoutExtension(filePath);
            return double.Parse(fileName.Split('_')[1]);
        }
        /// <summary>
        /// 执行视频FFMPEG处理任务
        /// </summary>
        /// <param name="task"></param>
        /// <returns></returns>
        public static async Task RunAsync(string task)
        {
            await VideoKeyFrames(task);
            await Audio2WAV16KAsync(task);
        }
        /// <summary>
        /// 音频转码为 wav_16k
        /// </summary>
        /// <param name="task">任务id</param>
        /// <returns></returns>
        public static async Task Audio2WAV16KAsync(string task)
        {
            var filePath = await DbScoped.Sugar
                .Queryable<VideoTask>()
                .Where(s => s.Id == long.Parse(task))
                .Select(s=>s.LocalMediaPath).FirstAsync();
            if (string.IsNullOrEmpty(filePath))
                throw new Exception($"任务id[{task}] 无效");

            // 打开输入文件
            var inputFile = new InputFile(filePath);
            var outputFile = new OutputFile(Path.Combine(task.LocalPath(), Path.GetFileNameWithoutExtension(filePath) + ".wav"));
            var ffmpeg = new Engine(FFmpegPath);

            ffmpeg.Error += (sender, e) =>
            {
                var ee = new Exception($"音频转码出现异常 \r\n[{e.Input.Name} => {e.Output.Name}]: 错误: {e.Exception.Message}");
                RedisExpand.SetTaskErrorMessage(long.Parse(task), ee);
                throw ee;
            };

            var conversionOptions = new ConversionOptions
            {
                ExtraArguments = "-ar 16000 -ac 1"
                //+ (AppCommon.AppSetting.FFmpeg.TimeSlice == 0
                //?string.Empty
                //: $"-f segment -reset_timestamps 1 -segment_time {AppCommon.AppSetting.FFmpeg.TimeSlice}")
            };
            var res = await ffmpeg.ConvertAsync(inputFile, outputFile, conversionOptions);

            Console.WriteLine($"{DateTime.Now}=>音频转码完成");
            //加入下一队列
            RedisExpand.InsertChannel(RedisChannelEnum.ParsingCaptions, task);
        }

    }
}