marking
/
Learn.VideoAnalysis
19
0
Fork 0
Code Pull Requests Actions Releases Activity

优化 迁移控制器到Core层复用 

新增 视频关键帧切片流程
This commit is contained in:
小肥羊 2025-04-17 18:13:12 +08:00
parent 7ef238e4e2
commit 56853c2243
30 changed files with 914 additions and 875 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

196
Learn.VideoAnalysis.API/Controllers/LJZK_Controller.cs
View File

@ -1,196 +0,0 @@
using VideoAnalysisCore.Common;
using Microsoft.AspNetCore.Mvc;
using System.Reflection;
using MapsterMapper;
using Mapster;
using VideoAnalysisCore.AICore.SherpaOnnx;
using UserCenter.Model.Enum;
using VideoAnalysisCore.AICore.GPT.ChatGPT;
using VideoAnalysisCore.AICore.GPT;
using System.Text.Json;
using Microsoft.AspNetCore.Authorization;
using VideoAnalysisCore.Model.Enum;
using FFmpeg.NET.Services;
using Yitter.IdGenerator;
using VideoAnalysisCore.AICore.GPT.Dto;
using VideoAnalysisCore.Model;
using Learn.VideoAnalysis.API.Controllers.Dto;
namespace Learn.VideoAnalysis.API.Controllers
{
/// <summary>
/// 蓝鲸字库接口
/// </summary>
[ApiController]
[Route("LJZK/[action]")]
public class LJZK_Controller : ControllerBase
{
private readonly ILogger<LJZK_Controller> _logger;
private readonly IMapper mp;
private readonly Repository<NodeSubscription> nodesubscriptionDB;
private readonly Repository<VideoTask> videoTaskDB;
private readonly Repository<VideoKonwPoint> videoKonwPointDB;
private readonly Repository<NodePackageInfo> nodePackageInfoDB;
public LJZK_Controller(ILogger<LJZK_Controller> logger,
IMapper mp, Repository<NodeSubscription> nodesubscriptionDB,
Repository<VideoTask> videoTaskDB = null, Repository<VideoKonwPoint> videoKonwPointDB = null
, Repository<NodePackageInfo> nodePackageInfoDB = null)
{
_logger = logger;
this.mp = mp;
this.nodesubscriptionDB = nodesubscriptionDB;
this.videoTaskDB = videoTaskDB;
this.videoKonwPointDB = videoKonwPointDB;
this.nodePackageInfoDB = nodePackageInfoDB;
}
/// <summary>
/// 蓝鲸智库_添加文件节点监控
/// </summary>
/// <param name="req">请求体</param>
/// <returns></returns>
[HttpPost(Name = "NodeSubscription")]
public async Task<IActionResult> NodeSubscription(NodeMonitoringReq req)
{
if (req is null || req.NodeId == 0)
return BadRequest("无效的提交数据");
if (nodesubscriptionDB.IsAny(s => s.NodeId == req.NodeId))
return BadRequest("重复添加了节点监控任务" + req.NodeId);
var res = await nodesubscriptionDB.InsertReturnEntityAsync(new NodeSubscription()
{
NodeId = req.NodeId,
TaskType = req.Type ?? default,
Subject = req.Subject ?? default,
});
return Ok(res);
}
/// <summary>
/// 蓝鲸智库_文件包订阅
/// </summary>
/// <param name="req">请求体</param>
/// <returns></returns>
[HttpPost(Name = "NodePackage")]
public async Task<IActionResult> NodePackage(NodePackageReq req)
{
Console.WriteLine($"{DateTime.Now} 文件包订阅请求 req=" + JsonSerializer.Serialize(req));
if (req.AnalyzeItems is null || req.AnalyzeItems.Count() == 0)
return BadRequest("无效视频列表数据");
var videos = new List<VideoTask>(req.AnalyzeItems.Count);
var nodePackages = new List<NodePackageInfo>(req.AnalyzeItems.Count);
var videoIdArr = videoTaskDB.AsQueryable().Select(v => v.TagId).Distinct().ToArray();
foreach (var s in req.AnalyzeItems)
{
var np = new NodePackageInfo()
{
VideoCode = s.VideoCode,
AttachmentsInfoType = s.AttachmentsInfoType,
MaterialId = s.MaterialId,
StructurePageContentId = s.StructurePageContentId,
VideoName = s.VideoName,
NodeId = req.NodeId,
TaskType = req.TaskType,
SubjectType = req.SubjectType,
};
nodePackages.Add(np);
if (videoIdArr.Contains(s.VideoCode))
continue;
videos.Add(new VideoTask()
{
Id = YitIdHelper.NextId(),
ComeFrom = "127.0.0.1",
ApiToken = "",
Type = req.TaskType,
Subject = req.SubjectType,
TagId = s.VideoCode,
MediaUrl = string.Empty,
MediaName = s.VideoName
});
}
await nodePackageInfoDB.InsertRangeAsync(nodePackages);
await videoTaskDB.InsertRangeAsync(videos);
if (videos is null || videos.Count == 0)
return Ok();
var ids = videos.Select(s => s.Id).ToArray();
RedisExpand.JoinQueue(ids);
return Ok();
}
/// <summary>
/// 获取任务类型
/// </summary>
/// <returns></returns>
[HttpGet(Name = "TaskTypList")]
public IActionResult TaskType()
{
Type type = typeof(TaskTypeEnum);
return Ok(Enum.GetValues(type).Cast<object>()
.Select(s => new { Text = s.ToString(), Value = (int)s }));
}
/// <summary>
/// 获取学科类型
/// </summary>
/// <returns></returns>
[HttpGet(Name = "SubjectList")]
public IActionResult Subject()
{
Type type = typeof(SubjectEnum);
return Ok(Enum.GetValues(type).Cast<object>()
.Select(s => new { Text = s.ToString(), Value = (int)s }));
}
/// <summary>
/// 获取视频知识点片段<para>taskId/tagId二选一</para>
/// </summary>
/// <param name="taskId"></param>
/// <param name="tagId">自定义id</param>
/// <returns></returns>
[HttpGet(Name = "TaskKnowInfo")]
public async Task<IActionResult> TaskKnowInfo(long taskId, string? tagId)
{
if (taskId == 0 && string.IsNullOrEmpty(tagId))
return BadRequest();
var task = await videoTaskDB.AsQueryable()
.WhereIF(taskId != 0, s => s.Id == taskId)
.WhereIF(!string.IsNullOrEmpty(tagId), s => s.TagId == tagId)
.FirstAsync();
if (task is null)
return BadRequest("无效任务");
var konwArr = await videoKonwPointDB.AsQueryable()
.Where(s => s.VideoTaskId == task.Id)
.ToArrayAsync();
if (konwArr is null || konwArr.Length == 0)
return BadRequest("无效任务");
return Ok(new TaskKnowRes()
{
TagId = task.TagId,
Status = task.LastEnum,
VideoTaskId = task.Id,
KnowBlockArr = konwArr
.GroupBy(s => s.StartTime)
.Select(s => new TaskKnowBlock()
{
Id = s.First().Id,
Content = s.First().Content,
StartTime = s.First().StartTime,
EndTime = s.First().EndTime,
Theme = s.First().Theme,
Know = s.Select(x => new TaskKnowInfo()
{
Id = x.Id,
KnowPoint = x.KnowPoint,
KnowPointId = x.KnowPointId
}).ToArray()
}).ToArray()
});
}
}
}

2
Learn.VideoAnalysis.API/Program.cs
View File

@ -22,7 +22,9 @@ namespace Learn.VideoAnalysis.API
builder.Services.AddSwaggerGen(c =>
{
var file = Path.Combine(AppContext.BaseDirectory, "Learn.VideoAnalysis.API.xml"); // xml文档绝对路径
var file1 = Path.Combine(AppContext.BaseDirectory, "VideoAnalysisCore.xml"); // xml文档绝对路径
c.IncludeXmlComments(file, true); // true : 显示控制器层注释
c.IncludeXmlComments(file1, true); // true : 显示控制器层注释
c.OrderActionsBy(o => o.RelativePath); // 对action的名称进行排序，如果有多个，就可以看见效果了。
});

2
Learn.VideoAnalysis.sln
View File

@ -7,7 +7,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Learn.VideoAnalysis", "Vide
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "VideoAnalysisCore", "VideoAnalysisCore\VideoAnalysisCore.csproj", "{69F4243A-B22E-431B-8F0B-ECD8729B8665}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Learn.VideoAnalysis.API", "Learn.VideoAnalysis.API\Learn.VideoAnalysis.API.csproj", "{D31BA4AB-73FC-47B1-A10A-34FD5E921F4A}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Learn.VideoAnalysis.API", "Learn.VideoAnalysis.API\Learn.VideoAnalysis.API.csproj", "{D31BA4AB-73FC-47B1-A10A-34FD5E921F4A}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution

7
VideoAnalysis/Components/Layouts/BasicLayout.razor.cs
View File

@ -2,6 +2,7 @@
using AntDesign.ProLayout;
using Microsoft.AspNetCore.Components;
using Microsoft.AspNetCore.Components.Server.ProtectedBrowserStorage;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Identity.Client.Extensions.Msal;
using Microsoft.JSInterop;
using System.Globalization;
@ -27,14 +28,14 @@ namespace VideoAnalysisRazor.Layouts
protected override async Task OnAfterRenderAsync(bool firstRender)
{
}
protected override async Task OnInitializedAsync()
{
if (!await CheckLogin())
{
NavigationManager.NavigateTo("/Login");
}
}
protected override async Task OnInitializedAsync()
{
_menuData = [
new MenuDataItem
{

1
VideoAnalysis/Components/Pages/EvaluationProject.razor.cs
View File

@ -1,7 +1,6 @@
using AntDesign;
using AntDesign.TableModels;
using FreeRedis;
using Learn.VideoAnalysis.Controllers.Dto;
using Microsoft.AspNetCore.Components;
using Microsoft.AspNetCore.Components.Web;
using SqlSugar;

1
VideoAnalysis/Components/Pages/NodeSubscriptionPage.razor.cs
View File

@ -1,7 +1,6 @@
using AntDesign;
using AntDesign.TableModels;
using FreeRedis;
using Learn.VideoAnalysis.Controllers.Dto;
using Microsoft.AspNetCore.Components;
using Microsoft.AspNetCore.Components.Web;
using SqlSugar;

2
VideoAnalysis/Components/Pages/VideoTaskPage.razor
View File

@ -2,7 +2,7 @@
@using AntDesign
@using AntDesign.TableModels
@using System.ComponentModel.DataAnnotations
@using Learn.VideoAnalysis.Controllers.Dto
@using VideoAnalysisCore.Controllers.Dto
@using SqlSugar
@using VideoAnalysisCore.Model
@using VideoAnalysisCore.Model.Dto

4
VideoAnalysis/Components/Pages/VideoTaskPage.razor.cs
View File

@ -2,7 +2,6 @@
using AntDesign.TableModels;
using FFmpeg.NET.Services;
using Learn.VideoAnalysis.API.Expand;
using Learn.VideoAnalysis.Controllers.Dto;
using Microsoft.AspNetCore.Components;
using Microsoft.AspNetCore.Components.Web;
using Microsoft.AspNetCore.DataProtection.KeyManagement;
@ -11,6 +10,7 @@ using System.Data.Common;
using System.Linq.Expressions;
using System.Threading.Tasks;
using VideoAnalysisCore.Common;
using VideoAnalysisCore.Controllers.Dto;
using VideoAnalysisCore.Model;
using VideoAnalysisCore.Model.Dto;
using VideoAnalysisCore.Model.Enum;
@ -127,7 +127,7 @@ namespace Learn.VideoAnalysis.Components.Pages
return;
var data = RedisExpand.Redis.HMGet<string>(RedisExpandKey.Task(item.Id),
"Progress", "LastEnum", "StartTime", "ErrorMessage");
item.Progress = float.Parse(data[0]);
item.Progress = data[0];
item.LastEnum = data[1].ToEnum<RedisChannelEnum>() ?? default;
item.StartTimeDic = System.Text.Json.JsonSerializer.Deserialize<Dictionary<RedisChannelEnum, DateTime>>(data[2]) ?? null;
item.ErrorMessage = data[3];

1
VideoAnalysis/Components/Pages/VideoTaskShow.razor.cs
View File

@ -1,7 +1,6 @@
using AntDesign;
using AntDesign.TableModels;
using FFmpeg.NET.Services;
using Learn.VideoAnalysis.Controllers.Dto;
using Microsoft.AspNetCore.Components;
using Microsoft.AspNetCore.Components.Web;
using Microsoft.AspNetCore.DataProtection.KeyManagement;

296
VideoAnalysis/Controllers/ApiController.cs
View File

@ -1,296 +0,0 @@
using VideoAnalysisCore.Common;
using Microsoft.AspNetCore.Mvc;
using System.Reflection;
using MapsterMapper;
using Mapster;
using VideoAnalysisCore.AICore.SherpaOnnx;
using UserCenter.Model.Enum;
using VideoAnalysisCore.AICore.GPT.ChatGPT;
using VideoAnalysisCore.AICore.GPT;
using System.Text.Json;
using VideoAnalysisCore.Model.Enum;
using Yitter.IdGenerator;
namespace Learn.VideoAnalysis.Controllers
{
[ApiController]
[Route("[controller]/[action]")]
public class ApiController : ControllerBase
{
private readonly ILogger<ApiController> _logger;
private readonly IMapper mp;
private readonly Repository<VideoTask> videoTaskDB;
private readonly Repository<VideoKonwPoint> videoKonwDB;
private readonly IBserGPT chatGPT;
public ApiController(ILogger<ApiController> logger, Repository<VideoTask> videoTaskDB,
IMapper mp, IBserGPT chatGPT, Repository<VideoKonwPoint> videoKonwDB)
{
_logger = logger;
this.videoTaskDB = videoTaskDB;
this.mp = mp;
this.chatGPT = chatGPT;
this.videoKonwDB = videoKonwDB;
}
private string GetClientIpAddress()
{
// 检查 X-Forwarded-For 请求头
if (HttpContext.Request.Headers.ContainsKey("X-Forwarded-For")
&& !string.IsNullOrEmpty(HttpContext.Request.Headers["X-Forwarded-For"]))
return HttpContext.Request.Headers["X-Forwarded-For"].ToString();
if (HttpContext.Connection.RemoteIpAddress != null)
return HttpContext.Connection.RemoteIpAddress.ToString();
throw new Exception("未能获取到客户端ip地址");
}
/// <summary>
/// 语音识别
/// </summary>
/// <param name="url">文件流</param>
/// <returns></returns>
[HttpGet(Name = "AudioRecognitionUrl")]
public async Task<IActionResult> AudioRecognitionUrl(string url)
{
try
{
using HttpClient client = new HttpClient();
// 发送GET请求获取网络文件流
using var networkStream = await client.GetStreamAsync(url);
var res = await SenseVoice.RunTask(networkStream);
return Ok(res);
}
catch (Exception ex)
{
return BadRequest(ex.Message);
}
}
/// <summary>
/// 语音识别
/// </summary>
/// <param name="file">文件流</param>
/// <returns></returns>
[HttpPost(Name = "AudioRecognition")]
public async Task<IActionResult> AudioRecognition(IFormFile file)
{
using var s = file.OpenReadStream();
var res = await SenseVoice.RunTask(s);
return Ok(res);
}
/// <summary>
/// 获取FTS_Data str
/// </summary>
/// <param name="path">路径</param>
/// <returns></returns>
[HttpGet(Name = "fts_data")]
public async Task<IActionResult> FTS_Data(string path = "itn_subject_sx.fst")
{
var hotwords = JsonSerializer
.Deserialize<HotwordMode[]>(System.IO.File.ReadAllText(Path.Combine(AppCommon.AIModelFile, "Hotwords.json")));
var res = new List<string>(100);
foreach (var element in hotwords.OrderByDescending(s => s.key.Count()))
foreach (var e in element.v)
res.Add($"""("{e}", "{element.key}")""");
var pyFile = System.IO.File.ReadAllText(Path.Combine(AppCommon.AIModelFile, "sherpa-onnx-fst.py"));
var resStr = pyFile
.Replace("(fts_data)", "[" + string.Join(',', res) + "]")
.Replace("(path)", path);
return Ok(resStr);
}
/// <summary>
/// 重新开始执行GPT分析<para>taskId/tagId二选一</para>
/// </summary>
/// <param name="taskId"></param>
/// <param name="tagId">自定义id</param>
/// <param name="subject">切换任务所属学科 null忽略</param>
/// <returns></returns>
[HttpGet(Name = "ReStart")]
public async Task<IActionResult> ReStart(long taskId, string? tagId, SubjectEnum? subject)
{
var task = await videoTaskDB.AsQueryable()
.WhereIF(taskId != 0, s => s.Id == taskId)
.WhereIF(!string.IsNullOrEmpty(tagId), s => s.TagId == tagId)
.FirstAsync();
if (task is null)
return BadRequest("未能找到对应任务");
if (subject is not null)
{
task.Subject = subject;
await videoTaskDB.UpdateAsync(task);
}
//重新开始执行GPT分析
RedisExpand.InsertChannel(RedisChannelEnum.ChatModelAnalysis
, task.Id);
return Ok();
}
/// <summary>
/// 插入队列
/// </summary>
/// <param name="enum"></param>
/// <param name="msg"></param>
/// <returns></returns>
[HttpPost(Name = "TestInsertChannel")]
public IActionResult TestInsertChannel(int @enum = 1, string msg = "1")
{
RedisExpand.InsertChannel(@enum.ToEnum<RedisChannelEnum>().Value
, msg);
return Ok();
}
/// <summary>
/// 视频处理
/// </summary>
/// <param name="req">请求体</param>
/// <returns></returns>
[HttpPost(Name = "VideoAnalysis")]
public async Task<IActionResult> VideoAnalysis(VideoAnalysisReq req)
{
if (!ModelState.IsValid) return BadRequest(ModelState);
if (await videoTaskDB.IsAnyAsync(s => s.TagId == req.TagId))
return BadRequest("重复添加");
// 自动映射属性到哈希
var task = new VideoTask()
{
Id=YitIdHelper.NextId(),
ComeFrom = GetClientIpAddress(),
MediaUrl = req.MediaUrl,
ApiToken = req.ApiToken,
Type = req.Type,
Subject = req.Subject,
Tag = req.Tag,
TagId = req.TagId,
MediaName = req.Name
};
//入库
var hashEntries = task.GetType()
.GetProperties(BindingFlags.Public | BindingFlags.Instance)
.ToDictionary(s => s.Name, s => s.GetValue(task));
await videoTaskDB.InsertAsync(task);
RedisExpand.Redis.HMSet(RedisExpandKey.Task(task.Id), hashEntries);
RedisExpand.Redis.LPush(RedisExpandKey.ChannelKey, task.Id);
return Ok(task.Id);
}
///// <summary>
///// 获取视频知识点片段<para>taskId/tagId二选一</para>
///// </summary>
///// <param name="taskId"></param>
///// <param name="tagId">自定义id</param>
///// <returns></returns>
//[HttpGet(Name = "TaskKnowInfo")]
//public async Task<IActionResult> TaskKnowInfo(long taskId, string? tagId)
//{
// if (taskId == 0 && string.IsNullOrEmpty(tagId))
// return BadRequest();
// var task = await videoTaskDB.AsQueryable()
// .WhereIF(taskId != 0, s => s.Id == taskId)
// .WhereIF(!string.IsNullOrEmpty(tagId), s => s.TagId == tagId)
// .FirstAsync();
// if (task is null)
// return BadRequest("无效任务");
// var konwArr = await videoKonwDB.AsQueryable()
// .Where(s => s.VideoTaskId == task.Id)
// .ToArrayAsync();
// if (konwArr is null || konwArr.Length == 0)
// return BadRequest("无效任务");
// return Ok(new TaskKnowRes()
// {
// TagId = task.TagId,
// Status = task.LastEnum,
// VideoTaskId = task.Id,
// KnowBlockArr = konwArr
// .GroupBy(s => s.StartTime)
// .Select(s => new TaskKnowBlock()
// {
// Id = s.First().Id,
// Content = s.First().Content,
// StartTime = s.First().StartTime,
// EndTime = s.First().EndTime,
// Theme = s.First().Theme,
// Know = s.Select(x => new TaskKnowInfo()
// {
// Id = x.Id,
// KnowPoint = x.KnowPoint,
// KnowPointId = x.KnowPointId
// }).ToArray()
// }).ToArray()
// });
//}
///// <summary>
///// 获取视频信息<para>taskId/tagId二选一</para>
///// </summary>
///// <param name="taskId"></param>
///// <param name="tagId">自定义id</param>
///// <returns></returns>
//[HttpGet(Name = "TaskInfo")]
//public async Task<IActionResult> TaskInfo(long taskId, string? tagId)
//{
// if (taskId == 0 && string.IsNullOrEmpty(tagId))
// return BadRequest();
// var task = await videoTaskDB.AsQueryable()
// .WhereIF(taskId != 0, s => s.Id == taskId)
// .WhereIF(!string.IsNullOrEmpty(tagId), s => s.TagId == tagId)
// .FirstAsync();
// if (task is null)
// return BadRequest();
// var taskData = task.ChatAnalysis.Adapt<TaskInfoRes>();
// if (taskData is null)
// return BadRequest();
// taskData.Status = task.LastEnum;
// if (task.LastEnum != RedisChannelEnum.EndTask)
// return BadRequest(taskData);
// if (taskData != null && taskData.TimeBase != null)
// taskData.TimeBase = MergeTimeBases(taskData.TimeBase);
// return Ok(taskData);
//}
//[NonAction]
//private static List<TimeBase> MergeTimeBases(IEnumerable<TimeBase> timeBases)
//{
// if (timeBases == null || timeBases.Count() == 0)
// {
// return new List<TimeBase>();
// }
// var mergedList = new List<TimeBase>();
// // 初始化合并段
// var current = timeBases.First();
// current.Content = string.Empty;
// foreach (var next in timeBases)
// {
// // 如果类型相同，则扩展时间段
// if (current.TimeBaseType == next.TimeBaseType)
// current.End = Math.Max(current.End, next.End);
// else
// {
// // 类型不同，将当前时间段加入结果列表，并开始新时间段
// mergedList.Add(current);
// current = next;
// current.Content = string.Empty;
// }
// }
// // 添加最后的时间段
// mergedList.Add(current);
// return mergedList;
//}
}
}

247
VideoAnalysis/Controllers/Dto/ApiDto.cs
View File

@ -1,247 +0,0 @@
using AntDesign;
using SqlSugar;
using System.ComponentModel.DataAnnotations;
using UserCenter.Model.Enum;
using VideoAnalysisCore.AICore.GPT.Dto;
using VideoAnalysisCore.Model.Enum;
namespace Learn.VideoAnalysis.Controllers.Dto
{
/// <summary>
/// 视频列表项
/// </summary>
public class StructurePageContentAnalyzeItem
{
/// <summary>
/// 录播内容编号
/// </summary>
public long StructurePageContentId { get; set; }
/// <summary>
/// 素材ID
/// </summary>
public long MaterialId { get; set; }
/// <summary>
/// 视频编码
/// </summary>
public string VideoCode { get; set; }
/// <summary>
/// 视频文件名称
/// </summary>
public string VideoName { get; set; }
/// <summary>
/// 内容类型
/// </summary>
public AttachmentsInfoType AttachmentsInfoType { get; set; }
}
public class NodePackageReq
{
/// <summary>
/// 录播结构目录节点编号
/// </summary>
[Required(ErrorMessage = "目录节点编号是必填项")]
public long NodeId { get; set; }
/// <summary>
/// 科目类型
/// </summary>
[Required(ErrorMessage = "科目类型是必填项")]
public SubjectEnum SubjectType { get; set; }
/// <summary>
/// 任务类型
/// </summary>
[Required(ErrorMessage = "任务类型是必填项")]
public TaskTypeEnum TaskType { get; set; }
/// <summary>
/// 视频列表
/// </summary>
[Required(ErrorMessage = "文件数量是必填项")]
public List<StructurePageContentAnalyzeItem> AnalyzeItems { get; set; }
}
/// <summary>
/// 视频处理 请求
/// </summary>
public class NodeMonitoringReq
{
/// <summary>
/// 媒体路径
/// </summary>
[Required(ErrorMessage = "文件节点ID是必填项")]
public long NodeId { get; set; }
/// <summary>
/// 任务类型
/// </summary>
public TaskTypeEnum? Type { get; set; }
/// <summary>
/// 学科类型
/// </summary>
public SubjectEnum? Subject { get; set; }
}
/// <summary>
/// 视频处理 请求
/// </summary>
public class VideoAnalysisReq
{
/// <summary>
/// 媒体路径
/// </summary>
[Required(ErrorMessage = "资源URL是必填项")]
public string MediaUrl { get; set; } = string.Empty;
/// <summary>
/// 资源名称
/// </summary>
[Required(ErrorMessage = "资源名称是必要的")]
public string Name { get; set; } = string.Empty;
/// <summary>
/// ApiKey
/// </summary>
[Required(ErrorMessage = "接口Token是必填项")]
public string ApiToken { get; set; } = string.Empty;
/// <summary>
/// 内容所属学科
/// </summary>
public SubjectEnum? Subject { get; set; }
/// <summary>
/// 任务类型
/// </summary>
public TaskTypeEnum? Type { get; set; }
/// <summary>
/// 自定义值 任务完成后附带通知
/// </summary>
public string Tag { get; set; } = string.Empty;
/// <summary>
/// 自定义Id可用于任务完成之后的查询
/// </summary>
public string? TagId { get; set; }
/// <summary>
///回调Api地址
/// </summary>
//[Required(ErrorMessage = "回调Api地址是必填项")]
//[Url(ErrorMessage = "请输入有效的 URL")]
//public string CallBackUrl { get; set; } = string.Empty;
}
public class TextValue
{
public TextValue(float v)
{
var s = TimeSpan.FromSeconds((double)v);
var td = new[] { s.Hours, s.Minutes, s.Seconds };
Text = string.Join(':', td.Where(s => s > 0));
Value = v;
}
public TextValue(string t,object v)
{
Text = t;
Value = v;
}
public TextValue()
{
}
public string Text { get; set; }
public object Value { get; set; }
}
public class TaskKnowInfo
{
/// <summary>
///视频片段知识点 id
/// </summary>
public long Id { get; set; }
/// <summary>
/// 知识点
/// </summary>
public string KnowPoint { get; set; }
/// <summary>
/// 知识点ID
/// </summary>
public string KnowPointId { get; set; }
}
public class TaskKnowBlock
{
public long Id { get; set; }
/// <summary>
/// 开始时间
/// </summary>
public float? StartTime { get; set; }
/// <summary>
/// 结束时间
/// </summary>
public float? EndTime { get; set; }
/// <summary>
/// 持续时间
/// </summary>
[SugarColumn(IsIgnore = true)]
public float? KeepTime => (EndTime ?? 0) - StartTime ?? 0;
/// <summary>
/// 主题
/// </summary>
public string? Theme { get; set; }
/// <summary>
/// 内容总结
/// </summary>
public string? Content { get; set; }
/// <summary>
/// 知识点列表
/// </summary>
public TaskKnowInfo[] Know { get; set; }
}
/// <summary>
/// 视频片段知识点结果
/// </summary>
public class TaskKnowRes
{
/// <summary>
/// 自定义Id [任务视频自定义id]
/// <see cref="VideoTask.TagId"/>
/// </summary>
public string? TagId { get; set; }
/// <summary>
/// 任务当前执行状态
/// </summary>
public RedisChannelEnum Status { get; set; }
/// <summary>
/// 视频任务id
/// </summary>
public long VideoTaskId { get; set; }
/// <summary>
/// 视频知识快
/// </summary>
public TaskKnowBlock[] KnowBlockArr { get; set; }
}
public class TaskInfoRes: TaskRes
{
public TaskInfoRes()
{
}
/// <summary>
/// 任务当前执行状态
/// </summary>
public RedisChannelEnum Status { get; set; }
///// <summary>
///// 时间轴状态枚举
///// </summary>
//public Dictionary<int, string> TimeTypeEnum =>
// Enum.GetValues(typeof(TimeBaseTypeEnum))
// .Cast<TimeBaseTypeEnum>()
// .ToDictionary(x => (int)x, x => x.ToString());
///// <summary>
///// 时间轴合计
///// </summary>
//public Dictionary<TimeBaseTypeEnum, TextValue>? TimeBaseTotal =>
// TimeBase?.GroupBy(s => s.TimeBaseType??TimeBaseTypeEnum.教师讲授)?
// .ToDictionary(s => s.Key, s => new TextValue(s.Sum(x => x.End - x.Start)));
}
}

1
VideoAnalysis/GlobalUsings.cs
View File

@ -3,4 +3,3 @@
global using VideoAnalysisCore.Model;
global using VideoAnalysisCore.Model.Dto;
global using VideoAnalysisCore.Model.Enum;
global using Learn.VideoAnalysis.Controllers.Dto;

13
VideoAnalysis/Program.cs
View File

@ -11,6 +11,10 @@ using VideoAnalysisCore.AICore.GPT.DeepSeek;
using Microsoft.Extensions.DependencyInjection;
using VideoAnalysisCore.Common.Expand;
using Learn.VideoAnalysis.Expand;
using Microsoft.AspNetCore.Mvc.Formatters;
using System.Security.Cryptography;
using System.Diagnostics;
using VideoAnalysisCore.AICore.FFMPGE;
@ -20,6 +24,7 @@ namespace Learn.VideoAnalysis
{
public static void Main(string[] args)
{
var builder = WebApplication.CreateBuilder(args);
// Add services to the container.
builder.Services.AddRazorComponents()
@ -46,7 +51,9 @@ namespace Learn.VideoAnalysis
Description = "教学视频分析平台v1"
});
var file = Path.Combine(AppContext.BaseDirectory, "Learn.VideoAnalysis.xml"); // xml文档绝对路径
var file1 = Path.Combine(AppContext.BaseDirectory, "VideoAnalysisCore.xml"); // xml文档绝对路径
c.IncludeXmlComments(file, true); // true : 显示控制器层注释
c.IncludeXmlComments(file1, true); // true : 显示控制器层注释
c.OrderActionsBy(o => o.RelativePath); // 对action的名称进行排序，如果有多个，就可以看见效果了。
});
@ -95,7 +102,7 @@ namespace Learn.VideoAnalysis
//builder.Services.AddSingleton<IBserGPT, KIMI_GPT>();
//builder.Services.AddSingleton<IBserGPT, Chat_GPT>();
builder.Services.AddSingleton<IBserGPT, DeepSeek_GPT>();
var app = builder.Build();
@ -120,8 +127,8 @@ namespace Learn.VideoAnalysis
app.MapRazorComponents<Learn.VideoAnalysis.Components.App>()
.AddInteractiveServerRenderMode();
//.AddInteractiveWebAssemblyRenderMode()
//.AddAdditionalAssemblies(typeof(VideoAnalysisRazor._Imports).Assembly);
//.AddInteractiveWebAssemblyRenderMode()
//.AddAdditionalAssemblies(typeof(VideoAnalysisRazor._Imports).Assembly);
app.MapControllers();

138
VideoAnalysisCore/AICore/FFMPGE/FFMPGEHandle.cs
View File

@ -9,6 +9,10 @@ using System.Runtime.InteropServices;
using SqlSugar.IOC;
using VideoAnalysisCore.Model;
using VideoAnalysisCore.Model.Enum;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Processing;
using System.Text.Json;
namespace VideoAnalysisCore.AICore.FFMPGE
{
@ -24,7 +28,110 @@ namespace VideoAnalysisCore.AICore.FFMPGE
? $"/usr/bin/ffmpeg"
: Path.Combine(AppCommon.AIModelFile, "ffmpeg.exe");
public static string Task = string.Empty;
/// <summary>
/// 识别视频关键帧
/// </summary>
/// <param name="task">任务id</param>
/// <returns></returns>
public static async Task VideoKeyFrames(string task)
{
var taskID = long.Parse(task);
//间隔秒
var intervalSec = 5;
var threshold = 15.0;
var PPTVideoCode = await DbScoped.Sugar
.Queryable<VideoTask>()
.Where(s => s.Id == long.Parse(task))
.Select(s => s.PPTVideoCode).FirstAsync();
if (string.IsNullOrEmpty(PPTVideoCode)) return;
//视频切帧
var localPath = task.LocalPath();
var filePath = Path.Combine(localPath, "ppt.mp4");
if (!File.Exists(filePath))
throw new Exception("存在PPTCOde但未能找到对应资源文件");
var ffmpeg = new Engine(FFmpegPath);
var cToken = new CancellationToken();
await ffmpeg.ExecuteAsync($"-i {filePath} -vf \"fps=1/{intervalSec},scale=320:180\" {localPath}/frame_%03d.jpg", cToken);
//视频关键帧分析
var frameFiles = Directory.GetFiles(localPath, "*.jpg")
.OrderBy(f => f)
.ToList();
Image<Rgb24> prevFrame = null;
string outputDir = "output";
Directory.CreateDirectory(outputDir);
var keyFrames = new List<int>();
foreach (var frameFile in frameFiles)
{
using (var currFrame = Image.Load<Rgb24>(frameFile))
{
if (prevFrame != null)
{
double diff = CalculateFrameDifference(prevFrame, currFrame);
double timestamp = GetTimestampFromFileName(frameFile) * intervalSec;
if (diff > threshold)
{
keyFrames.Add((int)timestamp);
//string outputPath = Path.Combine(outputDir, $"change_{timestamp:0000}.jpg");
//currFrame.Save(outputPath);
//Console.WriteLine($"变化帧: {timestamp}秒，差异值: {diff:F2}");
}
}
prevFrame?.Dispose();
prevFrame = currFrame.Clone();
}
}
//写入数据库
var keyFramStr = JsonSerializer.Serialize(keyFrames);
await DbScoped.Sugar
.Updateable<VideoTask>()
.SetColumns(it => it.PPTKeyFrame == keyFramStr)
.Where(it => it.Id == taskID)
.ExecuteCommandAsync();
}
/// <summary>
/// 计算帧差异
/// </summary>
/// <param name="img1"></param>
/// <param name="img2"></param>
/// <returns></returns>
static double CalculateFrameDifference(Image<Rgb24> img1, Image<Rgb24> img2)
{
// 统一调整为64x64
var resized1 = img1.Clone(x => x.Grayscale());
var resized2 = img2.Clone(x => x.Grayscale());
long diff = 0;
for (int y = 0; y < resized1.Height; y++)
{
for (int x = 0; x < resized1.Width; x++)
{
var pixel1 = resized1[x, y];
var pixel2 = resized2[x, y];
diff += Math.Abs(pixel1.R - pixel2.R);
}
}
return diff / (double)(resized1.Width * resized1.Height);
}
static double GetTimestampFromFileName(string filePath)
{
string fileName = Path.GetFileNameWithoutExtension(filePath);
return double.Parse(fileName.Split('_')[1]);
}
/// <summary>
/// 执行视频FFMPEG处理任务
/// </summary>
/// <param name="task"></param>
/// <returns></returns>
public static async Task RunAsync(string task)
{
await VideoKeyFrames(task);
await Audio2WAV16KAsync(task);
}
/// <summary>
/// 音频转码为 wav_16k
/// </summary>
@ -32,7 +139,6 @@ namespace VideoAnalysisCore.AICore.FFMPGE
/// <returns></returns>
public static async Task Audio2WAV16KAsync(string task)
{
Task = task;
var filePath = await DbScoped.Sugar
.Queryable<VideoTask>()
.Where(s => s.Id == long.Parse(task))
@ -45,9 +151,6 @@ namespace VideoAnalysisCore.AICore.FFMPGE
var outputFile = new OutputFile(Path.Combine(task.LocalPath(), Path.GetFileNameWithoutExtension(filePath) + ".wav"));
var ffmpeg = new Engine(FFmpegPath);
//ffmpeg.Progress += OnProgress;
//ffmpeg.Data += OnData;
ffmpeg.Complete += OnComplete;
ffmpeg.Error += (sender, e) =>
{
var ee = new Exception($"音频转码出现异常 \r\n[{e.Input.Name} => {e.Output.Name}]: 错误: {e.Exception.Message}");
@ -62,31 +165,12 @@ namespace VideoAnalysisCore.AICore.FFMPGE
//?string.Empty
//: $"-f segment -reset_timestamps 1 -segment_time {AppCommon.AppSetting.FFmpeg.TimeSlice}")
};
var res = await ffmpeg.ConvertAsync(inputFile, outputFile, conversionOptions);
}
private static void OnProgress(object sender, ConversionProgressEventArgs e)
{
Console.WriteLine("[{0} => {1}]", e.Input.MetaData.FileInfo.Name, e.Output.Name);
Console.WriteLine("比特率: {0}", e.Bitrate);
Console.WriteLine("Fps: {0}", e.Fps);
Console.WriteLine("基本框架: {0}", e.Frame);
Console.WriteLine("处理持续时间: {0}", e.ProcessedDuration);
Console.WriteLine("Size: {0} kb", e.SizeKb);
Console.WriteLine("总持续时间: {0}\n", e.TotalDuration);
}
private static void OnData(object sender, ConversionDataEventArgs e)
{
Console.WriteLine(e.Data);
}
private static void OnComplete(object sender, ConversionCompleteEventArgs e)
{
Console.WriteLine("转换完成=>" + e.Output.Name);
Console.WriteLine($"{DateTime.Now}=>音频转码完成");
//加入下一队列
RedisExpand.InsertChannel(RedisChannelEnum.ParsingCaptions, Task);
RedisExpand.InsertChannel(RedisChannelEnum.ParsingCaptions, task);
}
}
}

518
VideoAnalysisCore/AICore/GPT/DeepSeek/DeepSeek_GPT.cs
View File

@ -46,7 +46,7 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
/// </summary>
/// <param name="task">任务id</param>
/// <returns></returns>
public async Task<TaskRes> GetKnow(string task)
public async Task<TaskRes> GetKnow1(string task)
{
var taskId = long.Parse(task);
var taskInfo = await videoTaskDB.AsQueryable()
@ -136,11 +136,14 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
var thems = JsonSerializer.Serialize(questionRes.Adapt<VideoKnowQueryDto[]>());// string.Join(',', questionRes.Select(s => s.StartTime + "->" + s.Theme));
var checkResFormat1 = """[{"StartTime":开始秒(number),"KnowPoint":知识点名称(string),"KnowPointId":知识点Id(string)}]""";
var knowMessages =
$"我针对视频<{title}>分析出了一些视频的知识片段,现在需要你帮我将每个片段分配恰当的知识点(单个片段允许多个知识点用逗号','分割)。" +
$"我针对{subject}课堂授课视频分析出了视频的授课阶段片段。" +
$"现在需要你通过每个片段的内容总结来分配正确的知识点(单个片段允许多个知识点用逗号','分割)。" +
$"这是我的分段 {thems}。" +
$"提供的知识点名称({knows})。 格式 (方法点Id|方法点名称) " +
$"最后请确保分配的知识点是用户提供的,否则片段知识点值留空!。" +
$"输出内容只返回json格式({checkResFormat1})";
$"课堂内容与{fileNameInfoRes.授课章节}章节相关" +
$"最后请确保分配的知识点是用户提供的,并且一定正确合理!" +
$"输出内容只返回json格式({checkResFormat1})" +
$" 格式 (方法点Id|方法点名称) " +
$"提供的知识点名称({knows})。";
Console.WriteLine(DateTime.Now + "=>2.开始分析视频内容知识点");
var konwRes = await ChatAsync<VideoKnowRes[]>(task, knowMessages, null);
@ -220,6 +223,428 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
RedisExpand.InsertChannel(RedisChannelEnum.EndTask, task);
return gptRes;
}
/// <summary>
/// 获取知识点
/// </summary>
/// <param name="task">任务id</param>
/// <returns></returns>
public async Task<TaskRes> GetKnow_v1(string task)
{
var taskId = long.Parse(task);
var taskInfo = await videoTaskDB.AsQueryable()
.Where(s => s.Id == taskId)
.FirstAsync();
var subject = "数学";
var Course_Id = 27;
switch (taskInfo.Type)//处理不同任务类型的知识点树
{
case TaskTypeEnum.蓝鲸智库_中职视频分段:
Course_Id = 51;
break;
case TaskTypeEnum.蓝鲸智库_视频分段:
default:
Course_Id = 27;
break;
}
var xkwKnows = await knowledgeInfoDB.AsQueryable()
.Where(s => s.Course_Id == Course_Id
&& (s.Depth == 3
|| s.Depth == 2))
.Select(s => s.Name).ToArrayAsync();
string title = taskInfo.MediaName;
var speakerArr = JsonSerializer.Deserialize<OfflineSpeakerRes[]>(taskInfo.Speaker);
var captionsArr = JsonSerializer.Deserialize<SenseVoiceRes[]>(taskInfo.Captions);
var fileNameResFormat = "{授课章节: string|null}";
//var fileNamePostMessages = title +
// " 这是一堂课的标题,请你基于标题帮我分析出这堂课所讲授的内容与最恰当的授课章节(关联最贴切的章节,保留一个章节!)." +
// $"章节范围限定在[{string.Join(',', xkwKnows)}]范围内." +
// $"输出格式 json字符串 对象格式{fileNameResFormat}";
var rCaptionArr = string.Join(',', captionsArr
.Where((s, i) => i % 3 == 0)
.Take((int)(captionsArr?.Length ?? 0 / 2.2))
.Select(s => s.Text));
var fileNamePostMessages =
"这是一堂课的部分授课字幕,请你基于字幕内容帮我分析出这堂课所讲授的内容与最恰当的授课章节(关联最贴切的章节,保留一个章节!)." +
$"章节范围限定在[{string.Join(',', xkwKnows)}]范围内." +
$"以下是包含时间的视频字幕文本。" +
$"字幕列表 {rCaptionArr}。" +
$"输出格式 json字符串 对象格式{fileNameResFormat}";
var fileNameInfoRes = await ChatAsync<FileNameInfo>
(task, fileNamePostMessages, null);
#if DEBUG
fileNameInfoRes = new FileNameInfo() { 授课章节 = "一元二次不等式" };
#endif
var captions = ExpandFunction.GetSpeakerCaptions(captionsArr, speakerArr);
var maxVideoTime = captions?.TimeBase?.LastOrDefault()?.End ?? 0;
var criteriaBuilder = new StringBuilder();
var know = await knowledgeInfoDB.GetFirstAsync(s => s.Course_Id == Course_Id && s.Name == fileNameInfoRes.授课章节);
if (know is null)
throw new Exception("未能找到对应知识点=>" + fileNameInfoRes.授课章节);
await RedisExpand.Redis
.HMSetAsync(RedisExpandKey.Task(task), "学科章节", fileNameInfoRes.授课章节);
//提升到父级
var kInfo = await knowledgeInfoDB.GetByIdAsync(know.Parent_Id);
var knowledgeInfos = await knowledgeInfoDB.AsQueryable().ToChildListAsync(s => s.Parent_Id, kInfo.Parent_Id == 0 ? kInfo.Id : kInfo.Parent_Id);
var knows = string.Join(',', knowledgeInfos.Select(s => s.Id + "|" + s.Name));
var knowDic = knowledgeInfos
.OrderBy(s => s.Id)
.GroupBy(s => s.Name)
.ToDictionary(s => s.First().Name, s => s.First().Id);
var questionRes = new List<VideoKnowRes>();
while (true)
{
questionRes = new List<VideoKnowRes>();
var lastTime = 0;
var endTime = 0;
var timeSpan = 700;
while (endTime + 60< maxVideoTime)
{
try
{
endTime = lastTime + timeSpan;
var nowCaptionStr = string.Join('|', captionsArr
.Where(s => s.Text != "。")
.Where((s, i) => s.Start > lastTime && s.End < endTime)
.Select(s => s.Start + ":" + s.Text));
var resFormat = """[{"StartTime":开始秒(number),"EndTime":结束秒(number),"Stage":阶段(string),"Theme":主题(string),"Content":内容总结(string)}]""";
var postMessages =
$"你的任务是分析出中国{subject}课堂标准流程的授课阶段智能分段" +
$"课堂内容与{fileNameInfoRes.授课章节}章节相关" +
$"课堂标准流程包含以下7个阶段：课程引入/新知讲解/例题精讲/课堂练习/互动讨论/课程总结/作业布置。" +
$"通过时间段的主要讲解内容分析出对应的时间段内容总结。" +
$"通过生成的内容总结分析出对应的时间段主题。 " +
$"本次只分析一个阶段(优先按照阶段顺序 例如 课程引入优先)。 " +
$"最后请检查某些时间段的时常 如果超出800秒或者低于30秒则不符合条件!" +
$"输出内容只返回json格式({resFormat})" +
$"字幕格式(开始秒:内容|下一段字幕).以下是包含时间的部分视频字幕文本。" +
$"字幕列表 {nowCaptionStr} 字幕结束!";
Console.WriteLine(DateTime.Now + $"=>1.开始分析视频内容 {lastTime}~{endTime}");
questionRes.AddRange(await ChatAsync<VideoKnowRes[]>(task, postMessages, null));
lastTime = (int)questionRes.Last().EndTime.Value - (lastTime==0?0: 30);
}
catch (Exception ex)
{
Console.WriteLine(DateTime.Now + $"=>分析视频内容失败 {lastTime}~{endTime}");
Console.WriteLine(DateTime.Now + ex.Message);
Console.WriteLine(DateTime.Now + ex.StackTrace);
endTime = lastTime - timeSpan;
}
}
questionRes = questionRes.OrderBy(s => s.StartTime).ToList();
var thems = JsonSerializer.Serialize(questionRes.Adapt<VideoKnowQueryDto[]>());// string.Join(',', questionRes.Select(s => s.StartTime + "->" + s.Theme));
var checkResFormat1 = """[{"StartTime":开始秒(number),"KnowPoint":知识点名称(string),"KnowPointId":知识点Id(string)}]""";
var knowMessages =
$"我针对{subject}课堂授课视频分析出了视频的授课阶段片段。" +
$"现在需要你通过每个片段的内容总结来分配正确的知识点(单个片段允许多个知识点用逗号','分割)。" +
$"这是我的分段 {thems}。" +
$"课堂内容与{fileNameInfoRes.授课章节}章节相关" +
$"最后请确保分配的知识点是用户提供的,并且一定正确合理!" +
$"输出内容只返回json格式({checkResFormat1})" +
$" 格式 (方法点Id|方法点名称) " +
$"提供的知识点名称({knows})。";
Console.WriteLine(DateTime.Now + "=>2.开始分析视频内容知识点");
var konwRes = await ChatAsync<VideoKnowRes[]>(task, knowMessages, null);
for (int i = 0; i < konwRes.Count(); i++)
questionRes[i].KnowPoint = konwRes[i].KnowPoint;
//for (int i = 0; i < questionRes.Length; i++)
//{
// var item = questionRes[i];
// if (i == questionRes.Length - 1)
// item.EndTime = maxVideoTime;
// else
// item.EndTime = (int)(questionRes[i + 1]?.StartTime ?? 0) - 1;
//}
thems = JsonSerializer.Serialize(questionRes.Adapt<VideoKnowQueryDto[]>());
var checkResFormat = """{"Score":打分(number),"Evaluation":评价(string)""";//,"Data":优化后的分段(array)}""";
var checkMessage = "我为视频的讲解内容做了一些分段,希望你能通读字幕内容后检查下的分段是否符合我的要求?" +
$"检查这些分段的时间是否合理 与相邻的时间段间隔是否处于合理区间30~900秒之间?" +
$"分段的主题内容,知识点分配是否合理符合实际吗?" +
$"请给出你的打分(0-100,70分及格)以及打分原因。" +
$"这是我的分段 {thems}。" +
$"后续的内容是包含时间戳的视频字幕的固定格式文本。" +
$"字幕格式(说话人:开始秒:结束秒:内容|下一段字幕).以下是包含时间的视频字幕文本。字幕列表 {captions.Captions}。" +
$"最后输出格式为json({checkResFormat})";
Console.WriteLine(DateTime.Now + "=>3.开始检查视频分段结果");
var checkRes = await ChatAsync<CheckMessageDto>(task, checkMessage, null);
if (checkRes != null && checkRes.Score >= 80)
{
break;
}
else
{
Console.WriteLine(DateTime.Now + $"=>{task} 得分过低/分段长度不匹配 得分{checkRes?.Score} ");
Console.WriteLine(checkRes.Evaluation);
Console.WriteLine();
}
if (questionRes.Any(s => s.KeepTime < 30))
{
Console.WriteLine(DateTime.Now + "=>视频分段过短!! 重新进行AI分析");
continue;
}
}
//todo 未包含的知识点片段 如何处理
var insertData = questionRes
.Where(s => !string.IsNullOrEmpty(s.KnowPoint))
.SelectMany(
s =>
{
var ks = s.KnowPoint.Split(",").Distinct();
return ks.Where(x => knowDic.ContainsKey(x))
.Select(x => new VideoKonwPoint()
{
Content = s.Content,
Theme = s.Theme,
StartTime = s.StartTime,
EndTime = s.EndTime,
KnowPoint = x,
KnowPointId = knowDic[x].ToString(),
TagId = taskInfo.TagId,
VideoTaskId = taskInfo.Id,
});
}).ToList();
await videoKonwPointDB.DeleteAsync(s => s.VideoTaskId == taskId);
await videoKonwPointDB.InsertRangeAsync(insertData);
await RedisExpand.Redis
.HMSetAsync(RedisExpandKey.Task(task), "VideoKnows", questionRes);
var gptRes = new TaskRes(captions);
await RedisExpand.Redis
.HMSetAsync(RedisExpandKey.Task(task), "ChatAnalysis", gptRes);
RedisExpand.InsertChannel(RedisChannelEnum.EndTask, task);
return gptRes;
}
/// <summary>
/// 获取知识点
/// </summary>
/// <param name="task">任务id</param>
/// <returns></returns>
public async Task<TaskRes> GetKnow(string task)
{
var taskId = long.Parse(task);
var taskInfo = await videoTaskDB.AsQueryable()
.Where(s => s.Id == taskId)
.FirstAsync();
var subject = "数学";
var Course_Id = 27;
switch (taskInfo.Type)//处理不同任务类型的知识点树
{
case TaskTypeEnum.蓝鲸智库_中职视频分段:
Course_Id = 51;
break;
case TaskTypeEnum.蓝鲸智库_视频分段:
default:
Course_Id = 27;
break;
}
var xkwKnows = await knowledgeInfoDB.AsQueryable()
.Where(s => s.Course_Id == Course_Id
&& (s.Depth == 3
|| s.Depth == 2))
.Select(s => s.Name).ToArrayAsync();
string title = taskInfo.MediaName;
var speakerArr = JsonSerializer.Deserialize<OfflineSpeakerRes[]>(taskInfo.Speaker);
var captionsArr = JsonSerializer.Deserialize<SenseVoiceRes[]>(taskInfo.Captions);
var fileNameResFormat = "{授课章节: string|null}";
var rCaptionArr = string.Join(',', captionsArr
.Where((s, i) => i % 3 == 0)
.Take((int)(captionsArr?.Length ?? 0 / 2.2))
.Select(s => s.Text));
var fileNamePostMessages =
"这是一堂课的部分授课字幕,请你基于字幕内容帮我分析出这堂课所讲授的内容与最恰当的授课章节(关联最贴切的章节,保留一个章节!)." +
$"章节范围限定在[{string.Join(',', xkwKnows)}]范围内." +
$"以下是包含时间的视频字幕文本。" +
$"字幕列表 {rCaptionArr}。" +
$"输出格式 json字符串 对象格式{fileNameResFormat}";
var fileNameInfoRes = await ChatAsync<FileNameInfo>
(task, fileNamePostMessages, null);
#if DEBUG
fileNameInfoRes = new FileNameInfo() { 授课章节 = "一元二次不等式" };
#endif
var captions = ExpandFunction.GetSpeakerCaptions(captionsArr, speakerArr);
var maxVideoTime = captions?.TimeBase?.LastOrDefault()?.End ?? 0;
var criteriaBuilder = new StringBuilder();
var know = await knowledgeInfoDB.GetFirstAsync(s => s.Course_Id == Course_Id && s.Name == fileNameInfoRes.授课章节);
if (know is null)
throw new Exception("未能找到对应知识点=>" + fileNameInfoRes.授课章节);
await RedisExpand.Redis
.HMSetAsync(RedisExpandKey.Task(task), "学科章节", fileNameInfoRes.授课章节);
//提升到父级
var kInfo = await knowledgeInfoDB.GetByIdAsync(know.Parent_Id);
var knowledgeInfos = await knowledgeInfoDB.AsQueryable().ToChildListAsync(s => s.Parent_Id, kInfo.Parent_Id == 0 ? kInfo.Id : kInfo.Parent_Id);
var knows = string.Join(',', knowledgeInfos.Select(s => s.Id + "|" + s.Name));
var knowDic = knowledgeInfos
.OrderBy(s => s.Id)
.GroupBy(s => s.Name)
.ToDictionary(s => s.First().Name, s => s.First().Id);
var questionRes = new List<VideoKnowRes>();
while (true)
{
questionRes = new List<VideoKnowRes>();
var lastTime = 0;
var endTime = 0;
var timeSpan =(int)(maxVideoTime * 0.5);
while (endTime + 60 < maxVideoTime)
{
try
{
endTime = lastTime + timeSpan;
var nowCaptionStr = string.Join('|', captionsArr
.Where(s => s.Text != "。")
.Where((s, i) => s.Start > lastTime && s.End < endTime)
.Select(s => s.Start + ":" + s.Text));
var keyFrameArr = string.IsNullOrEmpty(taskInfo?.PPTVideoCode)
?string.Empty
: $"通过分析视频图像得到了视频授课内容发生了变化的时间节点{taskInfo.PPTKeyFrame},授课阶段应当在附近时间发生变化。" ;
var resFormat = """[{"StartTime":开始秒(number),"EndTime":结束秒(number),"Stage":阶段(string),"Theme":主题(string),"Content":内容总结(string)}]""";
var postMessages =
$"请通过视频字幕内容分析出视频中{subject}课堂的授课阶段。" +
$"课堂内容与{fileNameInfoRes.授课章节}章节相关" +
$"{keyFrameArr}" +
$"完整的课堂标准流程包含以下5个阶段：课程引入/新知讲解/例题精讲/课堂练习/课程总结。" +
$"通过授课阶段的主要讲解内容分析出对应的授课阶段内容总结。" +
$"通过生成的内容总结分析出对应的授课阶段主题。 " +
$"请注意 本次分析的视频字幕只是其中一部分 不需要分析出所有类型的授课阶段。" +
$"最后请检查每个授课阶段的时长,不允许出现超出800秒或者低于50秒的授课阶段。" +
$"输出内容只返回json格式({resFormat})" +
$"字幕格式(开始秒:内容|下一段字幕).以下是包含时间的视频字幕文本。" +
$"字幕列表 {nowCaptionStr} 字幕结束!";
Console.WriteLine(DateTime.Now + $"=>1.开始分析视频内容 {lastTime}~{endTime}");
questionRes.AddRange(await ChatAsync<VideoKnowRes[]>(task, postMessages, null));
lastTime = (int)questionRes.Last().EndTime.Value - (lastTime == 0 ? 0 : 30);
}
catch (Exception ex)
{
Console.WriteLine(DateTime.Now + $"=>分析视频内容失败 {lastTime}~{endTime}");
Console.WriteLine(DateTime.Now + ex.Message);
Console.WriteLine(DateTime.Now + ex.StackTrace);
endTime = lastTime - timeSpan;
}
}
questionRes = questionRes.OrderBy(s => s.StartTime).ToList();
var thems = JsonSerializer.Serialize(questionRes.Adapt<VideoKnowQueryDto[]>());// string.Join(',', questionRes.Select(s => s.StartTime + "->" + s.Theme));
var checkResFormat1 = """[{"StartTime":开始秒(number),"KnowPoint":知识点名称(string),"KnowPointId":知识点Id(string)}]""";
var knowMessages =
$"我针对{subject}课堂授课视频分析出了视频的授课阶段片段。" +
$"现在需要你通过每个片段的内容总结来分配正确的知识点(单个片段允许多个知识点用逗号','分割)。" +
$"这是我的分段 {thems}。" +
$"课堂内容与{fileNameInfoRes.授课章节}章节相关" +
$"最后请确保分配的知识点是用户提供的,并且一定正确合理!" +
$"输出内容只返回json格式({checkResFormat1})" +
$" 格式 (方法点Id|方法点名称) " +
$"提供的知识点名称({knows})。";
Console.WriteLine(DateTime.Now + "=>2.开始分析视频内容知识点");
var konwRes = await ChatAsync<VideoKnowRes[]>(task, knowMessages, null);
for (int i = 0; i < konwRes.Count(); i++)
questionRes[i].KnowPoint = konwRes[i].KnowPoint;
VideoKnowRes lastVideoKnow = null;
for (int i = 0; i < questionRes.Count(); i++)
{
var item = questionRes[i];
// 阶段类型相等,且范围包含上一阶段
if (lastVideoKnow != null && lastVideoKnow.Stage == item.Stage && item.StartTime < lastVideoKnow.EndTime)
{
questionRes[i] = null;
lastVideoKnow.EndTime= item.EndTime;
}
lastVideoKnow = questionRes[i];
}
thems = JsonSerializer.Serialize(questionRes.Adapt<VideoKnowQueryDto[]>());
var checkResFormat = """{"Score":打分(number),"Evaluation":评价(string)""";//,"Data":优化后的分段(array)}""";
var checkMessage = "我为视频的讲解内容做了一些分段,希望你能通读字幕内容后检查下的分段是否符合我的要求?" +
$"检查这些分段的时间是否合理 与相邻的时间段间隔是否处于合理区间30~900秒之间?" +
$"分段的主题内容,知识点分配是否合理符合实际吗?" +
$"请给出你的打分(0-100,70分及格)以及打分原因。" +
$"这是我的分段 {thems}。" +
$"后续的内容是包含时间戳的视频字幕的固定格式文本。" +
$"字幕格式(说话人:开始秒:结束秒:内容|下一段字幕).以下是包含时间的视频字幕文本。字幕列表 {captions.Captions}。" +
$"最后输出格式为json({checkResFormat})";
Console.WriteLine(DateTime.Now + "=>3.开始检查视频分段结果");
var checkRes = await ChatAsync<CheckMessageDto>(task, checkMessage, null);
if (checkRes != null && checkRes.Score >= 80)
{
break;
}
else
{
Console.WriteLine(DateTime.Now + $"=>{task} 得分过低/分段长度不匹配 得分{checkRes?.Score} ");
Console.WriteLine(checkRes.Evaluation);
Console.WriteLine();
}
if (questionRes.Any(s => s.KeepTime < 30))
{
Console.WriteLine(DateTime.Now + "=>视频分段过短!! 重新进行AI分析");
continue;
}
}
//todo 未包含的知识点片段 如何处理
var insertData = questionRes
.Where(s => !string.IsNullOrEmpty(s.KnowPoint))
.SelectMany(
s =>
{
var ks = s.KnowPoint.Split(",").Distinct();
return ks.Where(x => knowDic.ContainsKey(x))
.Select(x => new VideoKonwPoint()
{
Content = s.Content,
Theme = s.Theme,
StartTime = s.StartTime,
EndTime = s.EndTime,
KnowPoint = x,
KnowPointId = knowDic[x].ToString(),
TagId = taskInfo.TagId,
VideoTaskId = taskInfo.Id,
});
}).ToList();
await videoKonwPointDB.DeleteAsync(s => s.VideoTaskId == taskId);
await videoKonwPointDB.InsertRangeAsync(insertData);
await RedisExpand.Redis
.HMSetAsync(RedisExpandKey.Task(task), "VideoKnows", questionRes);
var gptRes = new TaskRes(captions);
await RedisExpand.Redis
.HMSetAsync(RedisExpandKey.Task(task), "ChatAnalysis", gptRes);
RedisExpand.InsertChannel(RedisChannelEnum.EndTask, task);
return gptRes;
}
public async Task<T> ChatAsync<T>(string task, string postMessages, string postMessages1, string model = "deepseek-reasoner")
{
var maxTokens = 4000;
@ -236,47 +661,52 @@ namespace VideoAnalysisCore.AICore.GPT.DeepSeek
temperature = 0.2f,
messages = messageArr
};
RedisExpand.SetTaskGPTReqCached(task, chatRep);
var chatResp = await chatClient.Chat(chatRep);
var chatResContent = chatResp?.res;
if (string.IsNullOrEmpty(chatResContent))
throw new Exception("GPT返回message无效结果");
if (chatResp != null)
RedisExpand.SetTaskGPTCached(task, new object[] { chatResp.Value.res, chatResp.Value.u, chatResp.Value.reasoning });
chatResContent = chatResContent?.Replace("字幕内容", "课堂情况");
chatResContent = chatResContent?.Replace("\n", "");
chatResContent = chatResContent?.Replace("```json", "");
chatResContent = chatResContent?.Replace("```", "");
chatResContent = chatResContent?.Replace("}{", "},{");
chatResContent = chatResContent?.Replace("}|{", "},{");
chatResContent = chatResContent?.Trim().ExtractJson().FirstOrDefault();
var startsStr = typeof(T).IsArray ? "[" : "{";
var endStr = typeof(T).IsArray ? "]" : "}";
if (!chatResContent.StartsWith(startsStr))
chatResContent = startsStr + chatResContent;
if (!chatResContent.EndsWith(endStr))
chatResContent = chatResContent + endStr;
var options = new JsonSerializerOptions
var tryCount = 10;
while (--tryCount >0)
{
// 允许解析不严格符合 JSON 规范的字符串
AllowTrailingCommas = true,
// 处理不匹配的 JSON 字符
ReadCommentHandling = JsonCommentHandling.Skip
};
try
{
var questionRes = JsonSerializer.Deserialize<T>(chatResContent, options);
if (questionRes is null)
throw new Exception("ChatGPT返回无效结果");
return questionRes;
}
catch (Exception ex)
{
throw new Exception("ChatGPT结果解析错误 " + ex.Message + ex.Message);
try
{
RedisExpand.SetTaskGPTReqCached(task, chatRep);
var chatResp = await chatClient.Chat(chatRep);
var chatResContent = chatResp?.res;
if (string.IsNullOrEmpty(chatResContent))
throw new Exception("GPT返回message无效结果");
if (chatResp != null)
RedisExpand.SetTaskGPTCached(task, new object[] { chatResp.Value.res, chatResp.Value.u, chatResp.Value.reasoning });
chatResContent = chatResContent?.Replace("字幕内容", "课堂情况");
chatResContent = chatResContent?.Replace("\n", "");
chatResContent = chatResContent?.Replace("```json", "");
chatResContent = chatResContent?.Replace("```", "");
chatResContent = chatResContent?.Replace("}{", "},{");
chatResContent = chatResContent?.Replace("}|{", "},{");
chatResContent = chatResContent?.Trim().ExtractJson().FirstOrDefault();
var startsStr = typeof(T).IsArray ? "[" : "{";
var endStr = typeof(T).IsArray ? "]" : "}";
if (!chatResContent.StartsWith(startsStr))
chatResContent = startsStr + chatResContent;
if (!chatResContent.EndsWith(endStr))
chatResContent = chatResContent + endStr;
var options = new JsonSerializerOptions
{
// 允许解析不严格符合 JSON 规范的字符串
AllowTrailingCommas = true,
// 处理不匹配的 JSON 字符
ReadCommentHandling = JsonCommentHandling.Skip
};
var questionRes = JsonSerializer.Deserialize<T>(chatResContent, options);
if (questionRes is null)
throw new Exception("ChatGPT返回无效结果");
return questionRes;
}
catch (Exception ex)
{
Console.WriteLine(DateTime.Now + $"=>ChatGPT结果解析错误 重试剩余{tryCount}");
Console.WriteLine(ex.Message);
}
}
throw new Exception(DateTime.Now+ "=>ChatGPT请求失败次数过多!!!");
}
}
}

5
VideoAnalysisCore/AICore/GPT/Dto/QuestionRes.cs
View File

@ -7,6 +7,7 @@ using System.Text;
using System.Text.Json;
using System.Text.Json.Nodes;
using System.Threading.Tasks;
using VideoAnalysisCore.Model.Enum;
namespace VideoAnalysisCore.AICore.GPT.Dto
{
@ -54,6 +55,10 @@ namespace VideoAnalysisCore.AICore.GPT.Dto
/// </summary>
public virtual string? KnowPointId { get; set; }
/// <summary>
/// 课程阶段
/// </summary>
public virtual string ? Stage { get; set; }
/// <summary>
/// 内容总结
/// </summary>
public virtual string? Content { get; set; }

5
VideoAnalysisCore/AICore/SherpaOnnx/Speaker.cs
View File

@ -79,10 +79,11 @@ namespace VideoAnalysisCore.AICore.SherpaOnnx
var res = segments.Select(s => new OfflineSpeakerRes(s));
await RedisExpand.Redis.HSetAsync(RedisExpandKey.Task(task), "Speaker", res);
var speakerStr = JsonSerializer.Serialize(res);
DbScoped.Sugar
await DbScoped.Sugar
.Updateable<VideoTask>()
.SetColumns(it => it.Speaker == speakerStr)
.Where(it => it.Id == long.Parse(task));
.Where(it => it.Id == long.Parse(task))
.ExecuteCommandAsync();
//加入下一队列
RedisExpand.InsertChannel(RedisChannelEnum.ChatModelAnalysis, task);

70
VideoAnalysisCore/Common/DownloadFile.cs
View File

@ -175,38 +175,86 @@ namespace VideoAnalysisCore.Common
.SetColumns(it => it.LocalMediaPath == outputPath)
.Where(it => it.Id == long.Parse(task))
.ExecuteCommandAsync();
//下载PPT视频
if (!string.IsNullOrEmpty(taskInfo.PPTVideoCode))
{
try
{
var videoInfo = await vodClient.GetPlayInfoAsync(new AlibabaCloud.SDK.Vod20170321.Models.GetPlayInfoRequest()
{
VideoId = taskInfo.PPTVideoCode,
Formats = "mp4",
OutputType = "cdn",
AuthTimeout = 3600 * 24 * 12,
});
if (videoInfo is null || videoInfo.StatusCode != 200 && !videoInfo.Body.PlayInfoList.PlayInfo.Any())
throw new Exception($"{DateTime.Now} 视频订阅=>获取阿里云视频信息失败 VideoCode {taskInfo.TagId} StatusCode {videoInfo?.StatusCode}");
var url = videoInfo.Body.PlayInfoList.PlayInfo.First().PlayURL;
await Download(url, localPath, "ppt.mp4",
(s, e) => RedisExpand.SetTaskProgress(task, "PPT->" + Math.Round(e.ProgressPercentage, 1)
));
}
catch (Exception e)
{
await RedisExpand.SetTaskErrorMessage(taskId, e);
}
}
try
{//下载原视频
await Download(fileUrl, localPath, task + fileExtension,
(s, e) => RedisExpand.SetTaskProgress(task, Math.Round(e.ProgressPercentage,1)
));
IDownload download = DownloadBuilder.New()
//加入下一队列
RedisExpand.InsertChannel(RedisChannelEnum.SeparateAudio, task);
}
catch (Exception e)
{
await RedisExpand.SetTaskErrorMessage(taskId, e);
}
}
/// <summary>
/// 下载文件
/// </summary>
/// <param name="fileUrl"></param>
/// <param name="localPath"></param>
/// <param name="fileName"></param>
/// <returns></returns>
public async Task Download(string fileUrl, string localPath, string fileName,Action<object? , Downloader.DownloadProgressChangedEventArgs> change)
{
var res = new TaskCompletionSource();
using IDownload download = DownloadBuilder.New()
.WithUrl(fileUrl)
.WithDirectory(localPath)
.WithFileName(task + fileExtension)
.WithFileName(fileName)
.WithConfiguration(Opt)
.Build();
var pI = 0;
download.DownloadProgressChanged += (object? sender, Downloader.DownloadProgressChangedEventArgs e) =>
download.DownloadProgressChanged += (object? sender, Downloader.DownloadProgressChangedEventArgs e) =>
{
pI++;
if (pI % 20 == 0)
RedisExpand.SetTaskProgress(task, e.ProgressPercentage);
change(sender, e);
};
download.DownloadFileCompleted += async (object? sender, AsyncCompletedEventArgs e) =>
{
if (download.Status == DownloadStatus.Failed && e.Error != null)
{
await RedisExpand.SetTaskErrorMessage(taskId, e.Error)
.ConfigureAwait(false);//不切回上下文
return;
res.SetException(e.Error);
}
else if (download.Status == DownloadStatus.Completed)
{
//加入下一队列
RedisExpand.InsertChannel(RedisChannelEnum.SeparateAudio, task);
return;
res.SetResult();
}
};
await download.StartAsync();
// 等待回调函数完成
await res.Task;
}
}
}

3
VideoAnalysisCore/Common/Expand/SqlSugarExpand.cs
View File

@ -76,7 +76,8 @@ namespace VideoAnalysisCore.Common.Expand
if (ex.Parametres == null) return;
//var originColor = Console.ForegroundColor;
//Console.ForegroundColor = ConsoleColor.DarkRed;
Console.WriteLine($"【{DateTime.Now}——错误SQL - [{config.ConfigId}]】\r\n" + UtilMethods.GetSqlString(config.DbType, ex.Sql, (SugarParameter[])ex.Parametres) + "\r\n");
Console.WriteLine($"【{DateTime.Now}——错误SQL - [{config.ConfigId}]】\r\n"+ ex.Message + "\r\n" + UtilMethods.GetSqlString(config.DbType, ex.Sql, (SugarParameter[])ex.Parametres) + "\r\n");
Console.WriteLine();
//Console.ForegroundColor = originColor;
};
db.Aop.DataExecuting = (oldValue, entityInfo) =>

23
VideoAnalysisCore/Common/RedisExpand.cs
View File

@ -108,11 +108,11 @@ namespace VideoAnalysisCore.Common
/// <param name="taskId"></param>
public static void SetTaskGPTCached(object taskId, object? data)
{
Redis.Set(RedisExpandKey.TaskGPT(taskId) + ":Res_" + DateTime.Now.ToString("yyyy/MM/dd_HH/mm/ss"), data, 3600 * 24);
Redis.Set(RedisExpandKey.TaskGPT(taskId) + ":" + DateTime.Now.ToString("MMddHHmmss") + "01", data, 3600 * 24);
}
public static void SetTaskGPTReqCached(object taskId, object? data)
{
Redis.Set(RedisExpandKey.TaskGPT(taskId) + ":Req_" + DateTime.Now.ToString("yyyy/MM/dd_HH/mm/ss"), data, 3600 * 24);
Redis.Set(RedisExpandKey.TaskGPT(taskId) + ":" + DateTime.Now.ToString("MMddHHmmss")+ "00", data, 3600 * 24);
}
/// <summary>
/// 加入到消费队列
@ -143,9 +143,9 @@ namespace VideoAnalysisCore.Common
/// </summary>
/// <param name="p">进度百分比</param>
/// <param name="taskId"></param>
public static void SetTaskProgress(object taskId, double p)
public static void SetTaskProgress(object taskId, object p)
{
Redis.HMSet(RedisExpandKey.Task(taskId), "Progress", Math.Round(p, 2));
Redis.HMSet(RedisExpandKey.Task(taskId), "Progress", p.ToString());
}
/// <summary>
@ -220,19 +220,20 @@ namespace VideoAnalysisCore.Common
if (Redis is null) throw new Exception("redis未初始化");
SubscribeList.Add(RedisChannelEnum.DownloadFile,
(msg) => { TouchChannel(RedisChannelEnum.DownloadFile, msg,
(task) =>
(Action<string>)((msg) => {
TouchChannel(RedisChannelEnum.DownloadFile, msg,
(Func<string, Task>)((task) =>
{
using var scope = AppCommon.Services?.CreateScope();
if (scope is null || scope.ServiceProvider.GetService<DownloadFile>() is null)
if (scope is null || ServiceProviderServiceExtensions.GetService<DownloadFile>(scope.ServiceProvider) is null)
throw new Exception("DownloadFile 未注入");
else
return scope.ServiceProvider.GetService<DownloadFile>()?.RunTask(task) ?? Task.CompletedTask;
});
});
return (Task)(scope.ServiceProvider.GetService<DownloadFile>()?.RunTask(task) ?? Task.CompletedTask);
}));
}));
SubscribeList.Add(RedisChannelEnum.SeparateAudio,
(msg) => { TouchChannel(RedisChannelEnum.SeparateAudio, msg, FFMPGEHandle.Audio2WAV16KAsync); });
(msg) => { TouchChannel(RedisChannelEnum.SeparateAudio, msg, FFMPGEHandle.RunAsync); });
SubscribeList.Add(RedisChannelEnum.ParsingCaptions,
(msg) => { TouchChannel(RedisChannelEnum.ParsingCaptions, msg, SenseVoice.RunTask); });

12
Learn.VideoAnalysis.API/Controllers/ApiController.cs → VideoAnalysisCore/Controllers/ApiController.cs
View File

@ -12,24 +12,23 @@ using System.Text.Json;
using VideoAnalysisCore.Model.Enum;
using Yitter.IdGenerator;
using VideoAnalysisCore.Model;
using Microsoft.AspNetCore.Http;
using VideoAnalysisCore.Model.Dto;
using Learn.VideoAnalysis.API.Controllers.Dto;
using VideoAnalysisCore.Controllers.Dto;
namespace Learn.VideoAnalysis.API.Controllers
namespace VideoAnalysisCore.Controllers
{
[ApiController]
[Route("[controller]/[action]")]
public class ApiController : ControllerBase
{
private readonly ILogger<ApiController> _logger;
private readonly IMapper mp;
private readonly Repository<VideoTask> videoTaskDB;
private readonly Repository<VideoKonwPoint> videoKonwDB;
private readonly IBserGPT chatGPT;
public ApiController(ILogger<ApiController> logger, Repository<VideoTask> videoTaskDB,
public ApiController(Repository<VideoTask> videoTaskDB,
IMapper mp, IBserGPT chatGPT, Repository<VideoKonwPoint> videoKonwDB)
{
_logger = logger;
this.videoTaskDB = videoTaskDB;
this.mp = mp;
this.chatGPT = chatGPT;
@ -169,7 +168,8 @@ namespace Learn.VideoAnalysis.API.Controllers
Subject = req.Subject,
Tag = req.Tag,
TagId = req.TagId,
MediaName = req.Name
MediaName = req.Name,
PPTVideoCode = req.PPTVideoCode,
};
//Èë¿â
var hashEntries = task.GetType()

6
Learn.VideoAnalysis.API/Controllers/Dto/ApiDto.cs → VideoAnalysisCore/Controllers/Dto/ApiDto.cs
View File

@ -4,7 +4,7 @@ using UserCenter.Model.Enum;
using VideoAnalysisCore.AICore.GPT.Dto;
using VideoAnalysisCore.Model.Enum;
namespace Learn.VideoAnalysis.API.Controllers.Dto
namespace VideoAnalysisCore.Controllers.Dto
{
/// <summary>
/// 视频列表项
@ -120,6 +120,10 @@ namespace Learn.VideoAnalysis.API.Controllers.Dto
/// </summary>
public string? TagId { get; set; }
/// <summary>
/// 课程对应ppt视频
/// </summary>
public string? PPTVideoCode { get; set; }
/// <summary>
///回调Api地址
/// </summary>
//[Required(ErrorMessage = "回调Api地址是必填项")]

21
VideoAnalysis/Controllers/LJZK_Controller.cs → VideoAnalysisCore/Controllers/LJZK_Controller.cs
View File

@ -16,8 +16,10 @@ using static FFmpeg.NET.MetaData;
using static System.Runtime.InteropServices.JavaScript.JSType;
using Yitter.IdGenerator;
using VideoAnalysisCore.AICore.GPT.Dto;
using VideoAnalysisCore.Model;
using VideoAnalysisCore.Controllers.Dto;
namespace Learn.VideoAnalysis.Controllers
namespace VideoAnalysisCore.Controllers
{
/// <summary>
/// À¶¾¨×Ö¿â½Ó¿Ú
@ -26,19 +28,16 @@ namespace Learn.VideoAnalysis.Controllers
[Route("LJZK/[action]")]
public class LJZK_Controller : ControllerBase
{
private readonly ILogger<LJZK_Controller> _logger;
private readonly IMapper mp;
private readonly Repository<NodeSubscription> nodesubscriptionDB;
private readonly Repository<VideoTask> videoTaskDB;
private readonly Repository<VideoKonwPoint> videoKonwPointDB;
private readonly Repository<NodePackageInfo> nodePackageInfoDB;
private readonly IBserGPT chatGPT;
public LJZK_Controller(ILogger<LJZK_Controller> logger,
IMapper mp, IBserGPT chatGPT, Repository<NodeSubscription> nodesubscriptionDB,
public LJZK_Controller( IMapper mp, IBserGPT chatGPT, Repository<NodeSubscription> nodesubscriptionDB,
Repository<VideoTask> videoTaskDB = null, Repository<VideoKonwPoint> videoKonwPointDB = null
, Repository<NodePackageInfo> nodePackageInfoDB = null)
{
_logger = logger;
this.mp = mp;
this.chatGPT = chatGPT;
this.nodesubscriptionDB = nodesubscriptionDB;
@ -96,9 +95,12 @@ namespace Learn.VideoAnalysis.Controllers
NodeId = req.NodeId,
TaskType = req.TaskType,
SubjectType = req.SubjectType,
};
nodePackages.Add(np);
if (videoIdArr.Contains(s.VideoCode))
if (s.AttachmentsInfoType == AttachmentsInfoType.PPT)
continue;
if (videoIdArr.Contains(s.VideoCode))
continue;
videos.Add(new VideoTask()
{
@ -108,8 +110,11 @@ namespace Learn.VideoAnalysis.Controllers
Type = req.TaskType,
Subject = req.SubjectType,
TagId = s.VideoCode,
MediaUrl =string.Empty,
MediaName = s.VideoName
MediaUrl = string.Empty,
MediaName = s.VideoName,
PPTVideoCode = req.AnalyzeItems
.FirstOrDefault(x => x.AttachmentsInfoType == AttachmentsInfoType.PPT && s.StructurePageContentId == x.StructurePageContentId)
?.VideoCode,
});
}
await nodePackageInfoDB.InsertRangeAsync(nodePackages);

2
VideoAnalysisCore/Model/Dto/VideoTaskDto.cs
View File

@ -44,7 +44,7 @@ namespace VideoAnalysisCore.Model.Dto
/// 执行进度
/// </summary>
[DisplayName("进度")]
public float Progress { get; set; }
public string Progress { get; set; }
/// <summary>
/// 错误信息
/// </summary>

19
VideoAnalysisCore/Model/Enum/StageEnum.cs Normal file
View File

@ -0,0 +1,19 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace VideoAnalysisCore.Model.Enum
{
public enum StageEnum
{
课程引入,
新知讲解,
例题精讲,
课堂练习,
互动讨论,
课程总结,
作业布置
}
}

16
VideoAnalysisCore/Model/Enum/StepLevelEnum.cs Normal file
View File

@ -0,0 +1,16 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace VideoAnalysisCore.Model.Enum
{
public enum StepLevelEnum
{
初段 = 1,
中段 = 2,
尾段 = 3,
}
}

23
VideoAnalysisCore/Model/VideoTask.cs
View File

@ -61,14 +61,24 @@ namespace VideoAnalysisCore.Model
/// <summary>
/// 自定义值 任务完成后附带通知
/// </summary>
[SugarColumn(Length = 500)]
public string Tag { get; set; }
[SugarColumn(Length = 500, IsNullable = true)]
public string? Tag { get; set; }
/// <summary>
/// 自定义值Id 任务完成后的凭证
/// 自定义值Id
/// </summary>
[SugarColumn(Length = 500,IsNullable =true)]
[SugarColumn(Length = 50, ColumnDataType = "varchar", IsNullable =true)]
public string? TagId { get; set; }
/// <summary>
/// 授课视频对应PPT视频ID
/// </summary>
[SugarColumn(Length = 50, ColumnDataType = "varchar", IsNullable = true)]
public string? PPTVideoCode { get; set; }
/// <summary>
/// 授课视频对应PPT视频关键帧
/// </summary>
[SugarColumn(ColumnDataType = "longtext",IsNullable = true)]
public string? PPTKeyFrame { get; set; }
/// <summary>
/// 字幕缓存
/// </summary>
[SugarColumn(ColumnName = "Captions", ColumnDataType = "longtext", IsNullable = true)]
@ -106,11 +116,6 @@ namespace VideoAnalysisCore.Model
/// </summary>
[SugarColumn( IsNullable = true)]
public DateTime? EndTime { get; set; }
/// <summary>
/// 开始时间轴
/// </summary>
[SugarColumn(ColumnDataType = "varchar", Length = 255)]
public string StartTime { get; set; } ="{}";
}
}

5
VideoAnalysisCore/VideoAnalysisCore.csproj
View File

@ -4,6 +4,7 @@
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<GenerateDocumentationFile>True</GenerateDocumentationFile>
</PropertyGroup>
<ItemGroup>
@ -64,11 +65,13 @@
<PackageReference Include="Mapster" Version="7.4.1-pre01" />
<PackageReference Include="Microsoft.AspNetCore.Cors" Version="2.3.0" />
<PackageReference Include="Microsoft.AspNetCore.Http" Version="2.3.0" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Abstractions" Version="2.2.0" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Abstractions" Version="2.3.0" />
<PackageReference Include="Microsoft.AspNetCore.Mvc.Core" Version="2.3.0" />
<PackageReference Include="Microsoft.Extensions.DependencyModel" Version="7.0.0" />
<PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
<PackageReference Include="org.k2fsa.sherpa.onnx" Version="1.10.32" />
<PackageReference Include="SixLabors.ImageSharp" Version="3.1.7" />
<PackageReference Include="SqlSugar.IOC" Version="2.0.0" />
<PackageReference Include="SqlSugarCore" Version="5.1.4.170" />
<PackageReference Include="UserCenter.Model" Version="1.3.5" />

131
testC/Program.cs Normal file
View File

@ -0,0 +1,131 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Processing;
using MediaToolkit;
using MediaToolkit.Model;
using MediaToolkit.Options;
using VideoAnalysisCore.Common;
using VideoAnalysisCore.AICore.FFMPGE;
using Microsoft.Extensions.Options;
using System.Diagnostics;
namespace VideoChangeDetector
{
class Program
{
static void Main(string[] args)
{
var taskID = "665310769946693";
var inputFile = $"F:\\Learn.VideoAnalysis\\testC\\bin\\Debug\\net8.0\\video\\{taskID}.mp4";
//inputFile = "F:\\Learn.VideoAnalysis\\testC\\bin\\Debug\\net8.0\\video\\ppt.mp4";
string tempFrameDir = "temp_frames";
var intervalSec = 5;
double threshold = 15.0;
Stopwatch sw = new Stopwatch();
sw.Start();
ExtractVideoFrames(inputFile, tempFrameDir, intervalSec);
sw.Stop();
Console.WriteLine("视频切帧总共花费{0}ms.", sw.Elapsed.TotalMilliseconds);
Stopwatch sw1 = new Stopwatch();
sw1.Start();
DetectChanges(tempFrameDir, threshold);
sw1.Stop();
Console.WriteLine("视频分析帧总共花费{0}ms.", sw1.Elapsed.TotalMilliseconds);
Console.WriteLine("处理完成!");
}
/// <summary>
/// 视频切片
/// </summary>
/// <param name="inputPath">输入</param>
/// <param name="outputDir">输出</param>
/// <param name="intervalSec">间隔多少秒</param>
/// <param name="ProcessorCount">线程数</param>
static void ExtractVideoFrames(string inputPath, string outputDir, int intervalSec, int ProcessorCount = 6)
{
Directory.CreateDirectory(outputDir);
var inputFile = new MediaFile { Filename = inputPath };
using var engine = new Engine(FFMPGEHandle.FFmpegPath);
engine.CustomCommand($"-i {inputPath} -vf \"fps=1/{intervalSec},scale=320:180\" {outputDir}/frame_%03d.jpg");
}
/// <summary>
/// 差异比对
/// </summary>
/// <param name="frameDir"></param>
/// <param name="threshold"></param>
/// <param name="ProcessorCount"></param>
static void DetectChanges(string frameDir, double threshold,int ProcessorCount = 6)
{
var frameFiles = Directory.GetFiles(frameDir, "*.jpg")
.OrderBy(f => f)
.ToList();
Image<Rgb24> prevFrame = null;
string outputDir = "output";
Directory.CreateDirectory(outputDir);
var options = new ParallelOptions { MaxDegreeOfParallelism = ProcessorCount };
foreach (var frameFile in frameFiles)
{
using (var currFrame = Image.Load<Rgb24>(frameFile))
{
if (prevFrame != null)
{
double diff = CalculateFrameDifference(prevFrame, currFrame);
double timestamp = GetTimestampFromFileName(frameFile) * 5 ;
if (diff > threshold)
{
string outputPath = Path.Combine(outputDir, $"change_{timestamp:0000}.jpg");
currFrame.Save(outputPath);
Console.WriteLine($"变化帧: {timestamp}秒，差异值: {diff:F2}");
}
else
{
Console.WriteLine($"-------: {timestamp}秒，差异值: {diff:F2}");
}
}
prevFrame?.Dispose();
prevFrame = currFrame.Clone();
}
}
//Parallel.ForEach(frameFiles, options, frameFile =>
//{
//});
}
/// <summary>
/// 计算帧差异
/// </summary>
/// <param name="img1"></param>
/// <param name="img2"></param>
/// <returns></returns>
static double CalculateFrameDifference(Image<Rgb24> img1, Image<Rgb24> img2)
{
// 统一调整为64x64
var resized1 = img1.Clone(x => x.Resize(96, 96).Grayscale());
var resized2 = img2.Clone(x => x.Resize(96, 96).Grayscale());
long diff = 0;
for (int y = 0; y < resized1.Height; y++)
{
for (int x = 0; x < resized1.Width; x++)
{
var pixel1 = resized1[x, y];
var pixel2 = resized2[x, y];
diff += Math.Abs(pixel1.R - pixel2.R);
}
}
return diff / (double)(resized1.Width * resized1.Height);
}
static double GetTimestampFromFileName(string filePath)
{
string fileName = Path.GetFileNameWithoutExtension(filePath);
return double.Parse(fileName.Split('_')[1]);
}
}
}

19
testC/testC.csproj Normal file
View File

@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="MediaToolkit" Version="1.1.0.1" />
<PackageReference Include="SixLabors.ImageSharp" Version="3.1.7" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\VideoAnalysisCore\VideoAnalysisCore.csproj" />
</ItemGroup>
</Project>